xref: /reactos/ntoskrnl/ke/amd64/trap.S (revision c8d07514) 
1/*
2 * FILE:            ntoskrnl/ke/amd64/trap.S
3 * COPYRIGHT:       See COPYING in the top level directory
4 * PURPOSE:         System Traps, Entrypoints and Exitpoints
5 * PROGRAMMER:      Timo Kreuzer (timo.kreuzer@reactos.org)
6 */
7
8/* INCLUDES ******************************************************************/
9
10#include <asm.inc>
11#include <ksamd64.inc>
12#include <trapamd64.inc>
13
14EXTERN KiDispatchException:PROC
15EXTERN KeBugCheckWithTf:PROC
16EXTERN MmAccessFault:PROC
17EXTERN KiSystemFatalException:PROC
18EXTERN KiNpxNotAvailableFaultHandler:PROC
19EXTERN KiGeneralProtectionFaultHandler:PROC
20EXTERN KiXmmExceptionHandler:PROC
21EXTERN KiDeliverApc:PROC
22EXTERN KiDpcInterruptHandler:PROC
23EXTERN PsConvertToGuiThread:PROC
24EXTERN MmCreateKernelStack:PROC
25EXTERN MmDeleteKernelStack:PROC
26
27EXTERN KdSetOwedBreakpoints:PROC
28
29
30/* Helper Macros *************************************************************/
31
32MACRO(DispatchException, Status, Number, P1, P2, P3)
33    mov eax, Status
34    mov edx, Number
35    mov r9, P1
36    mov r10, P2
37    mov r11, P3
38    call InternalDispatchException
39ENDM
40
41MACRO(Fatal, BugcheckCode)
42    /* Bugcheck */
43    mov ecx, BugcheckCode
44    mov rdx, rbp
45    call KiSystemFatalException
46ENDM
47
48
49/* FUNCTIONS *****************************************************************/
50
51.code64
52
53ALIGN 8
54
55MACRO(UnexpectedVectorStub, Vector)
56    /* This nop is to make the relative jmp address 4 bytes aligned and to
57       make the whole code 8 bytes long */
58    nop
59    /* This is a push instruction with 8bit operand. Since the instruction
60       sign extends the value to 32 bits, we need to offset it */
61PUBLIC KxUnexpectedInterrupt&Vector
62KxUnexpectedInterrupt&Vector:
63    push (Vector - 128)
64    jmp KiUnexpectedInterrupt
65ENDM
66
67PUBLIC KiUnexpectedRange
68KiUnexpectedRange:
69Vector = 0
70REPEAT 256
71    UnexpectedVectorStub %Vector
72    Vector = Vector+1
73ENDR
74PUBLIC KiUnexpectedRangeEnd
75KiUnexpectedRangeEnd:
76
77PUBLIC KiInterruptDispatchTemplate
78KiInterruptDispatchTemplate:
79    /* This instruction pushes the return address on the stack, which is the
80       address of the interrupt object's DispatchCode member, then jumps
81       to the address stored in the interrupt object's DispatchAddress member */
82    call qword ptr KiInterruptDispatchTemplate[rip - KINTERRUPT_DispatchCode + KINTERRUPT_DispatchAddress]
83
84
85// rbp = TrapFrame, eax = ExceptionCode, edx = NumParams, r9,r10,r11 = params
86FUNC InternalDispatchException
87
88    /* Allocate stack space for EXCEPTION_RECORD and KEXCEPTION_FRAME */
89    sub rsp, EXCEPTION_RECORD_LENGTH + KEXCEPTION_FRAME_LENGTH
90    .allocstack (EXCEPTION_RECORD_LENGTH + KEXCEPTION_FRAME_LENGTH)
91    .endprolog
92
93    /* Set up EXCEPTION_RECORD */
94    lea rcx, [rsp + KEXCEPTION_FRAME_LENGTH]
95    mov [rcx + EXCEPTION_RECORD_ExceptionCode], eax
96    xor rax, rax
97    mov [rcx + EXCEPTION_RECORD_ExceptionFlags], eax
98    mov [rcx + EXCEPTION_RECORD_ExceptionRecord], rax
99    mov rax, [rbp + KTRAP_FRAME_Rip]
100    mov [rcx + EXCEPTION_RECORD_ExceptionAddress], rax
101    mov [rcx + EXCEPTION_RECORD_NumberParameters], edx
102    mov [rcx + EXCEPTION_RECORD_ExceptionInformation + HEX(00)], r9
103    mov [rcx + EXCEPTION_RECORD_ExceptionInformation + HEX(08)], r10
104    mov [rcx + EXCEPTION_RECORD_ExceptionInformation + HEX(10)], r11
105
106    /* Set up KEXCEPTION_FRAME */
107    mov rax, [rbp + KTRAP_FRAME_Rbp]
108    mov [rsp + KEXCEPTION_FRAME_Rbp], rax
109    mov [rsp + KEXCEPTION_FRAME_Rbx], rbx
110    mov [rsp + KEXCEPTION_FRAME_Rdi], rdi
111    mov [rsp + KEXCEPTION_FRAME_Rsi], rsi
112    mov [rsp + KEXCEPTION_FRAME_R12], r12
113    mov [rsp + KEXCEPTION_FRAME_R13], r13
114    mov [rsp + KEXCEPTION_FRAME_R14], r14
115    mov [rsp + KEXCEPTION_FRAME_R15], r15
116    movdqa [rsp + KEXCEPTION_FRAME_Xmm6], xmm6
117    movdqa [rsp + KEXCEPTION_FRAME_Xmm7], xmm7
118    movdqa [rsp + KEXCEPTION_FRAME_Xmm8], xmm8
119    movdqa [rsp + KEXCEPTION_FRAME_Xmm9], xmm9
120    movdqa [rsp + KEXCEPTION_FRAME_Xmm10], xmm10
121    movdqa [rsp + KEXCEPTION_FRAME_Xmm11], xmm11
122    movdqa [rsp + KEXCEPTION_FRAME_Xmm12], xmm12
123    movdqa [rsp + KEXCEPTION_FRAME_Xmm13], xmm13
124    movdqa [rsp + KEXCEPTION_FRAME_Xmm14], xmm14
125    movdqa [rsp + KEXCEPTION_FRAME_Xmm15], xmm15
126    mov qword ptr [rsp + KEXCEPTION_FRAME_Return], 0
127
128    /* Call KiDispatchException */
129    // rcx already points to ExceptionRecord
130    mov rdx, rsp                                // ExceptionFrame
131    mov r8, rbp                                 // TrapFrame
132    mov r9b, [r8 + KTRAP_FRAME_PreviousMode]    // PreviousMode
133    mov byte ptr [rsp + KEXCEPTION_FRAME_P5], 1 // FirstChance
134    call KiDispatchException
135
136    /* Restore registers */
137    mov r12, [rsp + KEXCEPTION_FRAME_R12]
138    mov r13, [rsp + KEXCEPTION_FRAME_R13]
139    mov r14, [rsp + KEXCEPTION_FRAME_R14]
140    mov r15, [rsp + KEXCEPTION_FRAME_R15]
141    movdqa xmm6, [rsp + KEXCEPTION_FRAME_Xmm6]
142    movdqa xmm7, [rsp + KEXCEPTION_FRAME_Xmm7]
143    movdqa xmm8, [rsp + KEXCEPTION_FRAME_Xmm8]
144    movdqa xmm9, [rsp + KEXCEPTION_FRAME_Xmm9]
145    movdqa xmm10, [rsp + KEXCEPTION_FRAME_Xmm10]
146    movdqa xmm11, [rsp + KEXCEPTION_FRAME_Xmm11]
147    movdqa xmm12, [rsp + KEXCEPTION_FRAME_Xmm12]
148    movdqa xmm13, [rsp + KEXCEPTION_FRAME_Xmm13]
149    movdqa xmm14, [rsp + KEXCEPTION_FRAME_Xmm14]
150    movdqa xmm15, [rsp + KEXCEPTION_FRAME_Xmm15]
151
152    add rsp, EXCEPTION_RECORD_LENGTH + KEXCEPTION_FRAME_LENGTH
153    ret
154ENDFUNC
155
156
157/* CPU EXCEPTION HANDLERS ****************************************************/
158
159PUBLIC KiDivideErrorFault
160FUNC KiDivideErrorFault
161    /* Push pseudo error code */
162    EnterTrap TF_SAVE_ALL
163
164    /* Enable interrupts */
165    sti
166
167    /* Dispatch the exception */
168    DispatchException STATUS_INTEGER_DIVIDE_BY_ZERO, 0, 0, 0, 0
169
170    /* Return */
171    ExitTrap TF_SAVE_ALL
172ENDFUNC
173
174
175PUBLIC KiDebugTrapOrFault
176FUNC KiDebugTrapOrFault
177    /* Push pseudo error code */
178    EnterTrap TF_SAVE_ALL
179
180    /* Check if the frame was from kernelmode */
181    test word ptr [rbp + KTRAP_FRAME_SegCs], 3
182    jz KiDebugTrapOrFaultKMode
183
184    /* Enable interrupts for user-mode */
185    sti
186
187KiDebugTrapOrFaultKMode:
188    /* Dispatch the exception */
189    DispatchException STATUS_SINGLE_STEP, 0, 0, 0, 0
190
191    /* Return */
192    ExitTrap TF_SAVE_ALL
193ENDFUNC
194
195
196PUBLIC KiNmiInterrupt
197FUNC KiNmiInterrupt
198    /* Push pseudo error code */
199    EnterTrap TF_SAVE_ALL
200
201    UNIMPLEMENTED KiNmiInterrupt
202    int 3
203
204    /* Return */
205    ExitTrap TF_SAVE_ALL
206ENDFUNC
207
208
209PUBLIC KiBreakpointTrap
210FUNC KiBreakpointTrap
211    /* Push pseudo error code */
212    EnterTrap TF_SAVE_ALL
213
214    /* Check if the frame was from kernelmode */
215    test word ptr [rbp + KTRAP_FRAME_SegCs], 3
216    jz KiBreakpointTrapKMode
217
218    /* Enable interrupts for user-mode */
219    sti
220
221KiBreakpointTrapKMode:
222    /* Dispatch the exception */
223    DispatchException STATUS_BREAKPOINT, 3, BREAKPOINT_BREAK, 0, 0
224
225    /* Return */
226    ExitTrap TF_SAVE_ALL
227ENDFUNC
228
229
230PUBLIC KiOverflowTrap
231FUNC KiOverflowTrap
232    /* Push pseudo error code */
233    EnterTrap TF_SAVE_ALL
234
235    /* Enable interrupts */
236    sti
237
238    /* Dispatch the exception */
239    DispatchException STATUS_INTEGER_OVERFLOW, 3, 0, 0, 0
240
241    /* Return */
242    ExitTrap TF_SAVE_ALL
243ENDFUNC
244
245
246PUBLIC KiBoundFault
247FUNC KiBoundFault
248    /* Push pseudo error code */
249    EnterTrap TF_SAVE_ALL
250
251    /* Check if the frame was from kernelmode */
252    test word ptr [rbp + KTRAP_FRAME_SegCs], 3
253    jnz KiBoundFaultUserMode
254
255    /* Bugcheck */
256    Fatal EXCEPTION_BOUND_CHECK
257
258KiBoundFaultUserMode:
259    /* Enable interrupts for user-mode */
260    sti
261
262    /* Dispatch the exception */
263    DispatchException STATUS_ARRAY_BOUNDS_EXCEEDED, 0, 0, 0, 0
264
265    /* Return */
266    ExitTrap TF_SAVE_ALL
267ENDFUNC
268
269
270PUBLIC KiInvalidOpcodeFault
271FUNC KiInvalidOpcodeFault
272    /* Push pseudo error code */
273    EnterTrap TF_SAVE_ALL
274
275    /* Enable interrupts */
276    sti
277
278    /* Check if the frame was from kernelmode */
279    test word ptr [rbp + KTRAP_FRAME_SegCs], 3
280    jz KiInvalidOpcodeKernel
281
282    // FIXME: handle STATUS_INVALID_LOCK_SEQUENCE
283
284KiInvalidOpcodeKernel:
285    /* Kernel mode fault */
286
287    /* Dispatch the exception */
288    DispatchException STATUS_ILLEGAL_INSTRUCTION, 3, 0, 0, 0
289
290    /* Return */
291    ExitTrap TF_SAVE_ALL
292ENDFUNC
293
294
295PUBLIC KiNpxNotAvailableFault
296FUNC KiNpxNotAvailableFault
297    /* Push pseudo error code */
298    EnterTrap TF_SAVE_ALL
299
300    /* Call the C handler */
301    mov rcx, rbp
302    call KiNpxNotAvailableFaultHandler
303
304    /* Check the return status code */
305    test eax, eax
306    jz KiNpxNotAvailableFaultExit
307
308    /* Dispatch the exception */
309    DispatchException eax, 3, 0, 0, 0
310
311KiNpxNotAvailableFaultExit:
312    /* Return */
313    ExitTrap TF_SAVE_ALL
314ENDFUNC
315
316
317PUBLIC KiDoubleFaultAbort
318FUNC KiDoubleFaultAbort
319    /* A zero error code is pushed */
320    EnterTrap (TF_HAS_ERROR_CODE OR TF_SAVE_ALL)
321
322    int 3
323
324    /* Bugcheck */
325    Fatal 8 // EXCEPTION_DOUBLE_FAULT
326    jmp $
327ENDFUNC
328
329
330PUBLIC KiNpxSegmentOverrunAbort
331FUNC KiNpxSegmentOverrunAbort
332    /* Push pseudo error code */
333    EnterTrap TF_SAVE_ALL
334
335    /* Bugcheck */
336    Fatal EXCEPTION_NPX_OVERRUN
337
338    jmp $
339ENDFUNC
340
341
342PUBLIC KiInvalidTssFault
343FUNC KiInvalidTssFault
344    /* We have an error code */
345    EnterTrap (TF_HAS_ERROR_CODE OR TF_SAVE_ALL)
346
347    /* Bugcheck */
348    Fatal EXCEPTION_INVALID_TSS
349    jmp $
350ENDFUNC
351
352
353PUBLIC KiSegmentNotPresentFault
354FUNC KiSegmentNotPresentFault
355    /* We have an error code */
356    EnterTrap (TF_HAS_ERROR_CODE OR TF_SAVE_ALL)
357
358    /* Bugcheck */
359    Fatal EXCEPTION_SEGMENT_NOT_PRESENT
360    jmp $
361ENDFUNC
362
363
364PUBLIC KiStackFault
365FUNC KiStackFault
366    /* We have an error code */
367    EnterTrap (TF_HAS_ERROR_CODE OR TF_SAVE_ALL)
368
369    /* Bugcheck */
370    Fatal EXCEPTION_STACK_FAULT
371    jmp $
372ENDFUNC
373
374
375PUBLIC KiGeneralProtectionFault
376FUNC KiGeneralProtectionFault
377    /* We have an error code */
378    EnterTrap (TF_HAS_ERROR_CODE OR TF_SAVE_ALL)
379
380    /* Call the C handler */
381    mov rcx, rbp
382    call KiGeneralProtectionFaultHandler
383
384    /* Check for success */
385    test eax, eax
386    jge KiGpfExit
387
388    /* Dispatch the exception */
389    DispatchException eax, 3, 0, 0, 0
390
391KiGpfFatal:
392
393    /* Bugcheck */
394    mov ecx, UNEXPECTED_KERNEL_MODE_TRAP
395    mov rdx, HEX(000D) // EXCEPTION_GP_FAULT
396    xor r8, r8
397    mov r9, [rbp + KTRAP_FRAME_ErrorCode] // error code
398    sub rsp, 8
399    mov [rsp + KTRAP_FRAME_P5+8], rbp // trap frame
400    call KeBugCheckWithTf
401
402KiGpfExit:
403    /* Return */
404    /* Return */
405    ExitTrap TF_SAVE_ALL
406ENDFUNC
407
408
409PUBLIC KiPageFault
410FUNC KiPageFault
411    /* We have an error code */
412    EnterTrap (TF_HAS_ERROR_CODE OR TF_SAVE_ALL)
413
414    /* Save page fault address */
415    mov rdx, cr2
416    mov [rbp  + KTRAP_FRAME_FaultAddress], rdx
417
418    /* Enable interrupts for the page fault handler */
419    sti
420
421    /* Call page fault handler */
422    mov ecx, [rbp + KTRAP_FRAME_ErrorCode] // FaultCode
423    // rdx == Address
424    mov r8b, [rbp + KTRAP_FRAME_SegCs] // Mode
425    and r8b, 1
426    mov r9, rbp // TrapInformation
427    call MmAccessFault
428
429    /* Check for success */
430    test eax, eax
431    jl PageFaultError
432
433    /* Check whether the kernel debugger has owed breakpoints to be inserted */
434    call KdSetOwedBreakpoints
435    /* We succeeded, return */
436    jmp PageFaultReturn
437
438PageFaultError:
439
440    /* Disable interrupts again for the debugger */
441    cli
442
443    /* Set parameter 1 to error code */
444    mov r9d, [rbp + KTRAP_FRAME_ErrorCode]
445
446    /* Set parameter2 to faulting address */
447    mov r10, cr2  // Param2 = faulting address
448
449    cmp eax, STATUS_ACCESS_VIOLATION
450    je AccessViolation
451    cmp eax, STATUS_GUARD_PAGE_VIOLATION
452    je SpecialCode
453    cmp eax, STATUS_STACK_OVERFLOW
454    je SpecialCode
455
456InPageException:
457    /* Dispatch in-page exception */
458    mov r11d, eax // Param3 = Status
459    mov eax, STATUS_IN_PAGE_ERROR // ExceptionCode
460    mov edx, 3    // ParamCount
461    call InternalDispatchException
462    jmp PageFaultReturn
463
464AccessViolation:
465    /* Use more proper status code */
466    mov eax, KI_EXCEPTION_ACCESS_VIOLATION
467
468SpecialCode:
469    /* Setup a normal page fault exception */
470    mov edx, 2   // ParamCount
471    call InternalDispatchException
472
473PageFaultReturn:
474    /* Return */
475    ExitTrap (TF_SAVE_ALL or TF_CHECKUSERAPC)
476ENDFUNC
477
478
479PUBLIC KiFloatingErrorFault
480FUNC KiFloatingErrorFault
481    /* Push pseudo error code */
482    EnterTrap TF_SAVE_ALL
483
484    UNIMPLEMENTED KiFloatingErrorFault
485    int 3
486
487    /* Return */
488    ExitTrap TF_SAVE_ALL
489ENDFUNC
490
491
492PUBLIC KiAlignmentFault
493FUNC KiAlignmentFault
494    /* We have an error code */
495    EnterTrap (TF_HAS_ERROR_CODE OR TF_SAVE_ALL)
496
497    /* Bugcheck */
498    Fatal EXCEPTION_ALIGNMENT_CHECK
499    jmp $
500ENDFUNC
501
502
503PUBLIC KiMcheckAbort
504FUNC KiMcheckAbort
505    /* Push pseudo error code */
506    EnterTrap TF_SAVE_ALL
507
508    /* Bugcheck */
509    Fatal HEX(12)
510    jmp $
511ENDFUNC
512
513
514PUBLIC KiXmmException
515FUNC KiXmmException
516    /* Push pseudo error code */
517    EnterTrap TF_SAVE_ALL
518
519    /* Call the C handler */
520    mov rcx, rbp
521    call KiXmmExceptionHandler
522
523    /* Check for success */
524    test eax, eax
525    jge KiXmmExit
526
527    /* Dispatch the exception */
528    DispatchException eax, 3, 0, 0, 0
529
530KiXmmExit:
531    /* Return */
532    ExitTrap TF_SAVE_ALL
533ENDFUNC
534
535
536/* SOFTWARE INTERRUPT SERVICES ***********************************************/
537
538PUBLIC KiRaiseAssertion
539FUNC KiRaiseAssertion
540    /* We have an error code */
541    EnterTrap (TF_SAVE_ALL)
542
543    /* Decrement RIP to point to the INT2C instruction (2 bytes, not 1 like INT3) */
544    sub qword ptr [rbp + KTRAP_FRAME_Rip], 2
545
546    /* Dispatch the exception */
547    DispatchException STATUS_ASSERTION_FAILURE, 0, 0, 0, 0
548
549    /* Return */
550    ExitTrap TF_SAVE_ALL
551ENDFUNC
552
553
554PUBLIC KiDebugServiceTrap
555FUNC KiDebugServiceTrap
556   /* No error code */
557    EnterTrap TF_SAVE_ALL
558
559    /* Increase Rip to skip the int3 */
560    inc qword ptr [rbp + KTRAP_FRAME_Rip]
561
562    /* Dispatch the exception (Params = service, buffer, legth) */
563    DispatchException STATUS_BREAKPOINT, 3, [rbp+KTRAP_FRAME_Rax], [rbp+KTRAP_FRAME_Rcx], [rbp+KTRAP_FRAME_Rdx]
564
565    /* Return */
566    ExitTrap TF_SAVE_ALL
567ENDFUNC
568
569
570PUBLIC KiApcInterrupt
571.PROC KiApcInterrupt
572   /* No error code */
573    EnterTrap (TF_VOLATILES or TF_IRQL)
574
575    /* Raise to APC_LEVEL */
576    mov rax, APC_LEVEL
577    mov cr8, rax
578
579    /* End the interrupt */
580    mov dword ptr [APIC_EOI], 0
581
582    /* Enable interrupts  */
583    sti
584
585    /* Call the worker routine */
586    mov cl, [rbp + KTRAP_FRAME_SegCs] // ProcessorMode
587    and cl, 1
588    mov rdx, 0                        // ExceptionFrame
589    mov r8, rbp                       // TrapFrame
590    call KiDeliverApc
591
592    /* Disable interrupts */
593    cli
594
595    /* Lower IRQL back to PASSIVE */
596    mov rax, PASSIVE_LEVEL
597    mov cr8, rax
598
599    /* Return */
600    ExitTrap (TF_VOLATILES or TF_IRQL)
601.ENDP
602
603EXTERN KiRetireDpcList:PROC
604PUBLIC KiRetireDpcListInDpcStack
605.PROC KiRetireDpcListInDpcStack
606    push rbp
607    .pushreg rbp
608    mov rbp, rsp
609    .setframe rbp, 0
610    .endprolog
611
612    /* Switch stack and call the function */
613    mov rsp, rdx
614    sub rsp, 40
615    call KiRetireDpcList
616
617    /* Restore stack, cleanup and return */
618    mov rsp, rbp
619    pop rbp
620    ret
621.ENDP
622
623PUBLIC KiDpcInterrupt
624.PROC KiDpcInterrupt
625   /* No error code */
626    EnterTrap (TF_VOLATILES or TF_IRQL)
627
628    /* Call the worker routine */
629    call KiDpcInterruptHandler
630
631    /* Return, but don't send an EOI! */
632    ExitTrap (TF_VOLATILES or TF_IRQL)
633.ENDP
634
635
636PUBLIC KiIpiInterrupt
637.PROC KiIpiInterrupt
638   /* No error code */
639    EnterTrap (TF_VOLATILES or TF_IRQL)
640
641    /* Raise to IPI_LEVEL */
642    mov rax, IPI_LEVEL
643    mov cr8, rax
644
645    /* End the interrupt */
646    mov dword ptr [APIC_EOI], 0
647
648    int 3
649
650    /* Return */
651    ExitTrap (TF_VOLATILES or TF_IRQL)
652.ENDP
653
654
655PUBLIC KiUnexpectedInterrupt
656FUNC KiUnexpectedInterrupt
657    /* The error code is the vector */
658    EnterTrap (TF_HAS_ERROR_CODE OR TF_SAVE_ALL)
659
660#if 0
661    /* Set bugcheck parameters */
662    mov ecx, TRAP_CAUSE_UNKNOWN
663    mov rdx, [rbp + KTRAP_FRAME_ErrorCode] // the vector
664    mov r8, 0 // The unknown floating-point exception
665    mov r9, 0 // The enabled and asserted status bits
666    sub rsp, 8
667    mov [rbp + KTRAP_FRAME_P5 + 8], rbp // trap frame
668    call KeBugCheckWithTf
669    jmp $
670#endif
671    /* Return */
672    ExitTrap TF_SAVE_ALL
673ENDFUNC
674
675PUBLIC KiInterruptDispatch
676FUNC KiInterruptDispatch
677    /* The error code is a pointer to the interrupt object's code */
678    EnterTrap (TF_HAS_ERROR_CODE or TF_SAVE_ALL or TF_IRQL)
679
680    /* Increase interrupt count */
681    inc dword ptr gs:[PcInterruptCount];
682
683    /* Load the address of the interrupt object into rcx */
684    mov rcx, [rbp + KTRAP_FRAME_ErrorCode]
685
686    /* Substract offset of the DispatchCode member plus 6 for the call instruction */
687    sub rcx, KINTERRUPT_DispatchCode + 6
688
689    /* Raise IRQL to SynchronizeIrql */
690    movzx rax, byte ptr [rcx + KINTERRUPT_SynchronizeIrql]
691    mov cr8, rax
692
693#ifdef CONFIG_SMP
694    /* Acquire interrupt lock */
695    mov r8, [rcx + KINTERRUPT_ActualLock]
696
697    //KxAcquireSpinLock(Interrupt->ActualLock);
698#endif
699
700    /* Call the ISR */
701    mov rdx, [rcx + KINTERRUPT_ServiceContext]
702    call qword ptr [rcx + KINTERRUPT_ServiceRoutine]
703
704#ifdef CONFIG_SMP
705    /* Release interrupt lock */
706    //KxReleaseSpinLock(Interrupt->ActualLock);
707#endif
708
709    /* Go back to old irql */
710    movzx rax, byte ptr [rbp + KTRAP_FRAME_PreviousIrql]
711    mov cr8, rax
712
713    /* Return */
714    ExitTrap (TF_SAVE_ALL or TF_SEND_EOI)
715ENDFUNC
716
717
718#define MAX_SYSCALL_PARAM_SIZE (16 * 8)
719
720EXTERN KiSystemCallHandler:PROC
721
722/*! \name KiSystemCallEntry64
723 *
724 *  \brief This is the entrypoint for syscalls from 64bit user mode
725 *
726 *  \param rax - The system call number
727 *  \param rcx - User mode return address, set by the syscall instruction
728 *  \param rdx,r8,r9 - Parameters 2-4 to the service function
729 *  \param r10 - Parameter 1 to the service function
730 *  \param r11 - RFLAGS saved by the syscall instruction
731 *--*/
732PUBLIC KiSystemCallEntry64
733.PROC KiSystemCallEntry64
734
735    /* Old stack pointer is in rcx, lie and say we saved it in rbp */
736    .setframe rbp, 0
737    .endprolog
738
739    /* Swap gs to kernel, so we can access the PCR */
740    swapgs
741
742    /* Save the user mode rsp in the PCR */
743    mov gs:[PcUserRsp], rsp
744
745    /* Get the kernel stack from the PCR */
746    mov rsp, gs:[PcRspBase]
747
748    /* Allocate a TRAP_FRAME and space for parameters */
749    sub rsp, (KTRAP_FRAME_LENGTH + MAX_SYSCALL_PARAM_SIZE)
750#if DBG
751    /* Save rbp and load it with the old stack pointer */
752    mov [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rbp], rbp
753    mov rbp, gs:[PcUserRsp]
754#endif
755
756    /* Save important registers in the trap frame */
757    mov [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rax], rax
758    mov [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rcx], r10
759    mov [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_EFlags], r11
760
761    /* Set sane segments */
762    mov ax, (KGDT64_R3_DATA or RPL_MASK)
763    mov ds, ax
764    mov es, ax
765
766.ENDP
767
768.PROC KiSystemCall64Again
769
770    /* Old stack pointer is in rcx, lie and say we saved it in rbp */
771    .setframe rbp, 0
772    .endprolog
773
774    /* Call the C-handler (will enable interrupts) */
775    call KiSystemCallHandler
776
777    /* The return value from KiSystemCallHandler is the address of the Nt-function */
778    mov rcx, [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rcx]
779    mov rdx, [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rdx]
780    mov r8, [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_R8]
781    mov r9, [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_R9]
782    call rax
783
784.ENDP
785
786PUBLIC KiSystemServiceExit
787.PROC KiSystemServiceExit
788
789    /* Old stack pointer is in rcx, lie and say we saved it in rbp */
790    .setframe rbp, 0
791    .endprolog
792
793#if DBG
794    /* Restore rbp */
795    mov rbp, [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rbp]
796
797    test dword ptr [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_EFlags], HEX(200)
798    jnz IntsEnabled
799    int 3
800IntsEnabled:
801#endif
802
803    /* Check for pending user APC */
804    mov rcx, gs:qword ptr [PcCurrentThread]
805    cmp byte ptr [rcx + ThApcState + AsUserApcPending], 0
806    jz no_user_apc_pending
807    call KiInitiateUserApc
808no_user_apc_pending:
809
810    /* Disable interrupts for return */
811    cli
812
813    /* Restore old trap frame */
814    mov rcx, gs:[PcCurrentThread]
815    mov rdx, [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_TrapFrame]
816    mov [rcx + KTHREAD_TrapFrame], rdx
817
818    /* Prepare user mode return address (rcx) and eflags (r11) for sysret */
819    mov rcx, [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rip]
820    mov r11, [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_EFlags]
821
822    /* Load user mode stack (It was copied to the trap frame) */
823    mov rsp, [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rsp]
824
825    /* Swap gs back to user */
826    swapgs
827
828    ; Zero out volatiles
829    pxor xmm0, xmm0
830    pxor xmm1, xmm1
831    pxor xmm2, xmm2
832    pxor xmm3, xmm3
833    pxor xmm4, xmm4
834    pxor xmm5, xmm5
835    xor rdx, rdx
836    xor r10, r10
837
838    /* return to user mode */
839    .byte HEX(48) // REX prefix to return to long mode
840    sysret
841
842.ENDP
843
844
845/*!
846 * VOID
847 * DECLSPEC_NORETURN
848 * KiServiceExit(IN PKTRAP_FRAME TrapFrame, IN NTSTATUS Status));
849 */
850PUBLIC KiServiceExit
851.PROC KiServiceExit
852    .endprolog
853
854    lea rsp, [rcx - MAX_SYSCALL_PARAM_SIZE]
855    jmp KiSystemServiceExit
856
857.ENDP
858
859
860/*!
861 * VOID
862 * DECLSPEC_NORETURN
863 * KiServiceExit2(IN PKTRAP_FRAME TrapFrame);
864 */
865PUBLIC KiServiceExit2
866.PROC KiServiceExit2
867    .ENDPROLOG
868
869    mov rbp, rcx
870    mov rsp, rcx
871
872    /* Return */
873    ExitTrap TF_SAVE_ALL
874.ENDP
875
876
877PUBLIC KiSystemCallEntry32
878KiSystemCallEntry32:
879    swapgs
880    int 3
881
882
883PUBLIC KiZwSystemService
884FUNC KiZwSystemService
885    push rbp
886    .pushreg rbp
887    sub rsp, KTRAP_FRAME_LENGTH
888    .allocstack KTRAP_FRAME_LENGTH
889    mov [rsp + KTRAP_FRAME_Rsi], rsi
890    .savereg rsi, KTRAP_FRAME_Rsi
891    mov [rsp + KTRAP_FRAME_Rdi], rdi
892    .savereg rdi, KTRAP_FRAME_Rdi
893    mov rbp, rsp
894    .setframe rbp, 0
895    .endprolog
896
897    /* Get current thread */
898    mov r11, gs:[PcCurrentThread]
899
900    /* Save PreviousMode in the trap frame */
901    mov dil, byte ptr [r11 + KTHREAD_PreviousMode]
902    mov byte ptr [rbp + KTRAP_FRAME_PreviousMode], dil
903
904    /* Save the old trap frame in TrapFrame.Rdx */
905    mov rdi, [r11 + KTHREAD_TrapFrame]
906    mov [rbp + KTRAP_FRAME_Rdx], rdi
907
908    /* Set the new trap frame and previous mode */
909    mov [r11 + ThTrapFrame], rbp
910    mov byte ptr [r11 + KTHREAD_PreviousMode], 0
911
912    /* allocate space for parameters */
913    sub rsp, r10
914    and rsp, HEX(0fffffffffffffff0)
915
916    /* Save rcx */
917    mov [rbp + KTRAP_FRAME_Rcx], rcx
918
919    /* copy parameters to the new location */
920    lea rsi, [rbp + KTRAP_FRAME_LENGTH + 16]
921    lea rdi, [rsp]
922    mov rcx, r10
923    shr rcx, 3
924    rep movsq
925
926    /* Restore rcx */
927    mov rcx, [rbp + KTRAP_FRAME_Rcx]
928
929    /* Call the service function */
930    call rax
931
932    /* Restore the old trap frame */
933    mov r11, gs:[PcCurrentThread]
934    mov rsi, [rbp + KTRAP_FRAME_Rdx]
935    mov [r11 + KTHREAD_TrapFrame], rsi
936
937    /* Restore PreviousMode from the trap frame */
938    mov dil, byte ptr [rbp + KTRAP_FRAME_PreviousMode]
939    mov byte ptr [r11 + KTHREAD_PreviousMode], dil
940
941    /* Restore rdi and rsi */
942    mov rsi, [rbp + KTRAP_FRAME_Rsi]
943    mov rdi, [rbp + KTRAP_FRAME_Rdi]
944
945    /* Cleanup the stack and return */
946    lea rsp, [rbp + KTRAP_FRAME_LENGTH]
947    pop rbp
948    ret
949
950ENDFUNC
951
952PUBLIC KiConvertToGuiThread
953FUNC KiConvertToGuiThread
954
955    sub rsp, 40
956    .allocstack 40
957    .endprolog
958
959    // NewStack = (ULONG_PTR)MmCreateKernelStack(TRUE, 0);
960    mov cl, 1
961    xor rdx, rdx
962    call MmCreateKernelStack
963
964    /* Check for failure */
965    test rax, rax
966    jz KiConvertToGuiThreadFailed
967
968    ; OldStack = KeSwitchKernelStack((PVOID)NewStack, (PVOID)(NewStack - KERNEL_STACK_SIZE));
969    mov rcx, rax
970    mov rdx, rax
971    sub rdx, KERNEL_STACK_SIZE
972    call KeSwitchKernelStack
973
974    // MmDeleteKernelStack(OldStack, FALSE);
975    mov rcx, rax
976    xor rdx, rdx
977    call MmDeleteKernelStack
978
979    /* Call the worker function */
980    call PsConvertToGuiThread
981
982    /* Check for failure */
983    test rax, rax
984    js KiConvertToGuiThreadFailed
985
986    /* Disable interrupts for return */
987    cli
988
989    // FIXME: should just do the trap frame switch in KiSystemCallHandler64
990    /* Restore old trap frame */
991    mov rcx, gs:[PcCurrentThread]
992    mov rdx, [rsp + 48 + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_TrapFrame]
993    mov [rcx + KTHREAD_TrapFrame], rdx
994
995    // Restore register parameters
996    mov rcx, [rsp + 48 + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rip]
997    mov rdx, [rsp + 48 + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rdx]
998    mov r8, [rsp + 48 + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_R8]
999    mov r9, [rsp + 48 + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_R9]
1000
1001    /* Run KiSystemCallHandler again */
1002    add rsp, 48
1003    jmp KiSystemCall64Again
1004
1005KiConvertToGuiThreadFailed:
1006
1007    /* Clean up the stack and return failure */
1008    add rsp, 40
1009    mov eax, HEX(C0000017) // STATUS_NO_MEMORY
1010    ret
1011
1012ENDFUNC
1013
1014;
1015; VOID
1016; KiDeliverApc(
1017;     _In_ KPROCESSOR_MODE DeliveryMode,
1018;     _In_ PKEXCEPTION_FRAME ExceptionFrame,
1019;     _In_ PKTRAP_FRAME TrapFrame);
1020;
1021EXTERN KiDeliverApc:PROC
1022
1023PUBLIC KiInitiateUserApc
1024.PROC KiInitiateUserApc
1025
1026    ; Generate a KEXCEPTION_FRAME on the stack
1027    GENERATE_EXCEPTION_FRAME
1028
1029    ; Raise IRQL to APC_LEVEL
1030    mov rax, APC_LEVEL
1031    mov cr8, rax
1032
1033    ; Enable interrupts
1034    sti
1035
1036    ; Get the current trap frame
1037    mov rax, gs:[PcCurrentThread]
1038    mov r8, [rax + KTHREAD_TrapFrame]
1039
1040    ; Call the C function
1041    mov ecx, 1
1042    mov rdx, rsp
1043    call KiDeliverApc
1044
1045    ; Disable interrupts again
1046    cli
1047
1048    ; Restore the registers from the KEXCEPTION_FRAME
1049    RESTORE_EXCEPTION_STATE
1050
1051    ; Return
1052    ret
1053
1054.ENDP
1055
1056
1057PUBLIC KiInitializeSegments
1058KiInitializeSegments:
1059    mov ax, KGDT64_R3_DATA or RPL_MASK
1060    mov gs, ax
1061    swapgs
1062    mov gs, ax
1063    ret
1064
1065/*!
1066 * VOID
1067 * KiSwitchKernelStackHelper(
1068 *     LONG_PTR StackOffset,
1069 *     PVOID OldStackBase);
1070 */
1071PUBLIC KiSwitchKernelStackHelper
1072KiSwitchKernelStackHelper:
1073
1074    /* Pop return address from the current stack */
1075    pop rax
1076
1077    /* Switch to new stack */
1078    lea rsp, [rsp + rcx]
1079
1080    /* Push return address on the new stack */
1081    push rax
1082
1083    /* Return on new stack */
1084    mov rax, rdx
1085    ret
1086
1087EXTERN KiSwitchKernelStack:PROC
1088
1089PUBLIC KeSwitchKernelStack
1090FUNC KeSwitchKernelStack
1091
1092    sub rsp, 40
1093    .allocstack 40
1094
1095    ; Save rcx
1096    mov [rsp], rcx
1097    .savereg rcx, 0
1098    .endprolog
1099
1100    ; Call the C handler, which returns the old stack in rax
1101   call KiSwitchKernelStack
1102
1103    ; Restore rcx (StackBase)
1104    mov rcx, [rsp]
1105
1106    ; Switch to new stack: RSP += (StackBase - OldStackBase)
1107    sub rcx, rax
1108    add rsp, rcx
1109
1110    ; Deallocate the home frame
1111    add rsp, 40
1112    ret
1113
1114ENDFUNC
1115
1116
1117
1118
1119#ifdef _MSC_VER
1120#undef lgdt
1121#undef lidt
1122
1123//void __lgdt(void *Source);
1124PUBLIC __lgdt
1125__lgdt:
1126    lgdt fword ptr [rcx]
1127    ret
1128
1129//void __sgdt(void *Destination);
1130PUBLIC __sgdt
1131__sgdt:
1132    sgdt fword ptr [rcx]
1133    ret
1134
1135// void __lldt(unsigned short Value)
1136PUBLIC __lldt
1137__lldt:
1138    lldt cx
1139    ret
1140
1141//void __sldt(void *Destination);
1142PUBLIC __sldt
1143__sldt:
1144    sldt word ptr [rcx]
1145    ret
1146
1147//void __ltr(unsigned short Source);
1148PUBLIC __ltr
1149__ltr:
1150    ltr cx
1151    ret
1152
1153//void __str(unsigned short *Destination);
1154PUBLIC __str
1155__str:
1156    str word ptr [rcx]
1157    ret
1158
1159PUBLIC __swapgs
1160__swapgs:
1161    swapgs
1162    ret
1163
1164#endif
1165
1166END
1167