xref: /reactos/ntoskrnl/ke/amd64/trap.S (revision b3194e32) 
1/*
2 * FILE:            ntoskrnl/ke/amd64/trap.S
3 * COPYRIGHT:       See COPYING in the top level directory
4 * PURPOSE:         System Traps, Entrypoints and Exitpoints
5 * PROGRAMMER:      Timo Kreuzer (timo.kreuzer@reactos.org)
6 */
7
8/* INCLUDES ******************************************************************/
9
10#include <asm.inc>
11#include <ksamd64.inc>
12#include <trapamd64.inc>
13
14EXTERN KiDispatchException:PROC
15EXTERN KeBugCheckWithTf:PROC
16EXTERN MmAccessFault:PROC
17EXTERN KiSystemFatalException:PROC
18EXTERN KiNpxNotAvailableFaultHandler:PROC
19EXTERN KiGeneralProtectionFaultHandler:PROC
20EXTERN KiXmmExceptionHandler:PROC
21EXTERN KiDeliverApc:PROC
22EXTERN KiDpcInterruptHandler:PROC
23EXTERN PsConvertToGuiThread:PROC
24EXTERN MmCreateKernelStack:PROC
25EXTERN MmDeleteKernelStack:PROC
26
27EXTERN KdSetOwedBreakpoints:PROC
28
29
30/* Helper Macros *************************************************************/
31
32MACRO(DispatchException, Status, Number, P1, P2, P3)
33    mov eax, Status
34    mov edx, Number
35    mov r9, P1
36    mov r10, P2
37    mov r11, P3
38    call InternalDispatchException
39ENDM
40
41MACRO(Fatal, BugcheckCode)
42    /* Bugcheck */
43    mov ecx, BugcheckCode
44    mov rdx, rbp
45    call KiSystemFatalException
46ENDM
47
48
49/* FUNCTIONS *****************************************************************/
50
51.code64
52
53ALIGN 8
54
55MACRO(UnexpectedVectorStub, Vector)
56    /* This nop is to make the relative jmp address 4 bytes aligned and to
57       make the whole code 8 bytes long */
58    nop
59    /* This is a push instruction with 8bit operand. Since the instruction
60       sign extends the value to 32 bits, we need to offset it */
61PUBLIC KxUnexpectedInterrupt&Vector
62KxUnexpectedInterrupt&Vector:
63    push (Vector - 128)
64    jmp KiUnexpectedInterrupt
65ENDM
66
67PUBLIC KiUnexpectedRange
68KiUnexpectedRange:
69Vector = 0
70REPEAT 256
71    UnexpectedVectorStub %Vector
72    Vector = Vector+1
73ENDR
74PUBLIC KiUnexpectedRangeEnd
75KiUnexpectedRangeEnd:
76
77PUBLIC KiInterruptDispatchTemplate
78KiInterruptDispatchTemplate:
79    /* This instruction pushes the return address on the stack, which is the
80       address of the interrupt object's DispatchCode member, then jumps
81       to the address stored in the interrupt object's DispatchAddress member */
82    call qword ptr KiInterruptDispatchTemplate[rip - KINTERRUPT_DispatchCode + KINTERRUPT_DispatchAddress]
83
84
85// rbp = TrapFrame, eax = ExceptionCode, edx = NumParams, r9,r10,r11 = params
86FUNC InternalDispatchException
87
88    /* Allocate stack space for EXCEPTION_RECORD and KEXCEPTION_FRAME */
89    sub rsp, EXCEPTION_RECORD_LENGTH + KEXCEPTION_FRAME_LENGTH
90    .allocstack (EXCEPTION_RECORD_LENGTH + KEXCEPTION_FRAME_LENGTH)
91    .endprolog
92
93    /* Set up EXCEPTION_RECORD */
94    lea rcx, [rsp + KEXCEPTION_FRAME_LENGTH]
95    mov [rcx + EXCEPTION_RECORD_ExceptionCode], eax
96    xor rax, rax
97    mov [rcx + EXCEPTION_RECORD_ExceptionFlags], eax
98    mov [rcx + EXCEPTION_RECORD_ExceptionRecord], rax
99    mov rax, [rbp + KTRAP_FRAME_Rip]
100    mov [rcx + EXCEPTION_RECORD_ExceptionAddress], rax
101    mov [rcx + EXCEPTION_RECORD_NumberParameters], edx
102    mov [rcx + EXCEPTION_RECORD_ExceptionInformation + HEX(00)], r9
103    mov [rcx + EXCEPTION_RECORD_ExceptionInformation + HEX(08)], r10
104    mov [rcx + EXCEPTION_RECORD_ExceptionInformation + HEX(10)], r11
105
106    /* Set up KEXCEPTION_FRAME */
107    mov rax, [rbp + KTRAP_FRAME_Rbp]
108    mov [rsp + KEXCEPTION_FRAME_Rbp], rax
109    mov [rsp + KEXCEPTION_FRAME_Rbx], rbx
110    mov [rsp + KEXCEPTION_FRAME_Rdi], rdi
111    mov [rsp + KEXCEPTION_FRAME_Rsi], rsi
112    mov [rsp + KEXCEPTION_FRAME_R12], r12
113    mov [rsp + KEXCEPTION_FRAME_R13], r13
114    mov [rsp + KEXCEPTION_FRAME_R14], r14
115    mov [rsp + KEXCEPTION_FRAME_R15], r15
116    movdqa [rsp + KEXCEPTION_FRAME_Xmm6], xmm6
117    movdqa [rsp + KEXCEPTION_FRAME_Xmm7], xmm7
118    movdqa [rsp + KEXCEPTION_FRAME_Xmm8], xmm8
119    movdqa [rsp + KEXCEPTION_FRAME_Xmm9], xmm9
120    movdqa [rsp + KEXCEPTION_FRAME_Xmm10], xmm10
121    movdqa [rsp + KEXCEPTION_FRAME_Xmm11], xmm11
122    movdqa [rsp + KEXCEPTION_FRAME_Xmm12], xmm12
123    movdqa [rsp + KEXCEPTION_FRAME_Xmm13], xmm13
124    movdqa [rsp + KEXCEPTION_FRAME_Xmm14], xmm14
125    movdqa [rsp + KEXCEPTION_FRAME_Xmm15], xmm15
126    mov qword ptr [rsp + KEXCEPTION_FRAME_Return], 0
127
128    /* Call KiDispatchException */
129    // rcx already points to ExceptionRecord
130    mov rdx, rsp                                // ExceptionFrame
131    mov r8, rbp                                 // TrapFrame
132    mov r9b, [r8 + KTRAP_FRAME_PreviousMode]    // PreviousMode
133    mov byte ptr [rsp + KEXCEPTION_FRAME_P5], 1 // FirstChance
134    call KiDispatchException
135
136    /* Restore registers */
137    mov r12, [rsp + KEXCEPTION_FRAME_R12]
138    mov r13, [rsp + KEXCEPTION_FRAME_R13]
139    mov r14, [rsp + KEXCEPTION_FRAME_R14]
140    mov r15, [rsp + KEXCEPTION_FRAME_R15]
141    movdqa xmm6, [rsp + KEXCEPTION_FRAME_Xmm6]
142    movdqa xmm7, [rsp + KEXCEPTION_FRAME_Xmm7]
143    movdqa xmm8, [rsp + KEXCEPTION_FRAME_Xmm8]
144    movdqa xmm9, [rsp + KEXCEPTION_FRAME_Xmm9]
145    movdqa xmm10, [rsp + KEXCEPTION_FRAME_Xmm10]
146    movdqa xmm11, [rsp + KEXCEPTION_FRAME_Xmm11]
147    movdqa xmm12, [rsp + KEXCEPTION_FRAME_Xmm12]
148    movdqa xmm13, [rsp + KEXCEPTION_FRAME_Xmm13]
149    movdqa xmm14, [rsp + KEXCEPTION_FRAME_Xmm14]
150    movdqa xmm15, [rsp + KEXCEPTION_FRAME_Xmm15]
151
152    add rsp, EXCEPTION_RECORD_LENGTH + KEXCEPTION_FRAME_LENGTH
153    ret
154ENDFUNC
155
156
157/* CPU EXCEPTION HANDLERS ****************************************************/
158
159PUBLIC KiDivideErrorFault
160FUNC KiDivideErrorFault
161    /* Push pseudo error code */
162    EnterTrap TF_SAVE_ALL
163
164    /* Enable interrupts */
165    sti
166
167    /* Dispatch the exception */
168    DispatchException STATUS_INTEGER_DIVIDE_BY_ZERO, 0, 0, 0, 0
169
170    /* Return */
171    ExitTrap TF_SAVE_ALL
172ENDFUNC
173
174
175PUBLIC KiDebugTrapOrFault
176FUNC KiDebugTrapOrFault
177    /* Push pseudo error code */
178    EnterTrap TF_SAVE_ALL
179
180    /* Check if the frame was from kernelmode */
181    test word ptr [rbp + KTRAP_FRAME_SegCs], 3
182    jz KiDebugTrapOrFaultKMode
183
184    /* Enable interrupts for user-mode */
185    sti
186
187KiDebugTrapOrFaultKMode:
188    /* Dispatch the exception */
189    DispatchException STATUS_SINGLE_STEP, 0, 0, 0, 0
190
191    /* Return */
192    ExitTrap TF_SAVE_ALL
193ENDFUNC
194
195
196PUBLIC KiNmiInterrupt
197FUNC KiNmiInterrupt
198    /* Push pseudo error code */
199    EnterTrap TF_SAVE_ALL
200
201    UNIMPLEMENTED KiNmiInterrupt
202    int 3
203
204    /* Return */
205    ExitTrap TF_SAVE_ALL
206ENDFUNC
207
208
209PUBLIC KiBreakpointTrap
210FUNC KiBreakpointTrap
211    /* Push pseudo error code */
212    EnterTrap TF_SAVE_ALL
213
214    /* Check if the frame was from kernelmode */
215    test word ptr [rbp + KTRAP_FRAME_SegCs], 3
216    jz KiBreakpointTrapKMode
217
218    /* Enable interrupts for user-mode */
219    sti
220
221KiBreakpointTrapKMode:
222    /* Dispatch the exception */
223    DispatchException STATUS_BREAKPOINT, 3, BREAKPOINT_BREAK, 0, 0
224
225    /* Return */
226    ExitTrap TF_SAVE_ALL
227ENDFUNC
228
229
230PUBLIC KiOverflowTrap
231FUNC KiOverflowTrap
232    /* Push pseudo error code */
233    EnterTrap TF_SAVE_ALL
234
235    /* Enable interrupts */
236    sti
237
238    /* Dispatch the exception */
239    DispatchException STATUS_INTEGER_OVERFLOW, 3, 0, 0, 0
240
241    /* Return */
242    ExitTrap TF_SAVE_ALL
243ENDFUNC
244
245
246PUBLIC KiBoundFault
247FUNC KiBoundFault
248    /* Push pseudo error code */
249    EnterTrap TF_SAVE_ALL
250
251    /* Check if the frame was from kernelmode */
252    test word ptr [rbp + KTRAP_FRAME_SegCs], 3
253    jnz KiBoundFaultUserMode
254
255    /* Bugcheck */
256    Fatal EXCEPTION_BOUND_CHECK
257
258KiBoundFaultUserMode:
259    /* Enable interrupts for user-mode */
260    sti
261
262    /* Dispatch the exception */
263    DispatchException STATUS_ARRAY_BOUNDS_EXCEEDED, 0, 0, 0, 0
264
265    /* Return */
266    ExitTrap TF_SAVE_ALL
267ENDFUNC
268
269
270PUBLIC KiInvalidOpcodeFault
271FUNC KiInvalidOpcodeFault
272    /* Push pseudo error code */
273    EnterTrap TF_SAVE_ALL
274
275    /* Enable interrupts */
276    sti
277
278    /* Check if the frame was from kernelmode */
279    test word ptr [rbp + KTRAP_FRAME_SegCs], 3
280    jz KiInvalidOpcodeKernel
281
282    // FIXME: handle STATUS_INVALID_LOCK_SEQUENCE
283
284KiInvalidOpcodeKernel:
285    /* Kernel mode fault */
286
287    /* Dispatch the exception */
288    DispatchException STATUS_ILLEGAL_INSTRUCTION, 0, 0, 0, 0
289
290    /* Return */
291    ExitTrap TF_SAVE_ALL
292ENDFUNC
293
294
295PUBLIC KiNpxNotAvailableFault
296FUNC KiNpxNotAvailableFault
297    /* Push pseudo error code */
298    EnterTrap TF_SAVE_ALL
299
300    /* Call the C handler */
301    mov rcx, rbp
302    call KiNpxNotAvailableFaultHandler
303
304    /* Check the return status code */
305    test eax, eax
306    jz KiNpxNotAvailableFaultExit
307
308    /* Dispatch the exception */
309    DispatchException eax, 3, 0, 0, 0
310
311KiNpxNotAvailableFaultExit:
312    /* Return */
313    ExitTrap TF_SAVE_ALL
314ENDFUNC
315
316
317PUBLIC KiDoubleFaultAbort
318FUNC KiDoubleFaultAbort
319
320    /* Hack for VBox, which "forgets" to push an error code on the stack! */
321    and rsp, HEX(FFFFFFFFFFFFFFF0)
322
323    /* A zero error code is pushed */
324    EnterTrap (TF_HAS_ERROR_CODE OR TF_SAVE_ALL)
325
326    int 3
327
328    /* Bugcheck */
329    Fatal 8 // EXCEPTION_DOUBLE_FAULT
330    jmp $
331ENDFUNC
332
333
334PUBLIC KiNpxSegmentOverrunAbort
335FUNC KiNpxSegmentOverrunAbort
336    /* Push pseudo error code */
337    EnterTrap TF_SAVE_ALL
338
339    /* Bugcheck */
340    Fatal EXCEPTION_NPX_OVERRUN
341
342    jmp $
343ENDFUNC
344
345
346PUBLIC KiInvalidTssFault
347FUNC KiInvalidTssFault
348    /* We have an error code */
349    EnterTrap (TF_HAS_ERROR_CODE OR TF_SAVE_ALL)
350
351    /* Bugcheck */
352    Fatal EXCEPTION_INVALID_TSS
353    jmp $
354ENDFUNC
355
356
357PUBLIC KiSegmentNotPresentFault
358FUNC KiSegmentNotPresentFault
359    /* We have an error code */
360    EnterTrap (TF_HAS_ERROR_CODE OR TF_SAVE_ALL)
361
362    /* Bugcheck */
363    Fatal EXCEPTION_SEGMENT_NOT_PRESENT
364    jmp $
365ENDFUNC
366
367
368PUBLIC KiStackFault
369FUNC KiStackFault
370    /* We have an error code */
371    EnterTrap (TF_HAS_ERROR_CODE OR TF_SAVE_ALL)
372
373    /* Bugcheck */
374    Fatal EXCEPTION_STACK_FAULT
375    jmp $
376ENDFUNC
377
378
379PUBLIC KiGeneralProtectionFault
380FUNC KiGeneralProtectionFault
381    /* We have an error code */
382    EnterTrap (TF_HAS_ERROR_CODE OR TF_SAVE_ALL)
383
384    /* Call the C handler */
385    mov rcx, rbp
386    call KiGeneralProtectionFaultHandler
387
388    /* Check for success */
389    test eax, eax
390    jge KiGpfExit
391
392    /* Check for access violation */
393    cmp eax, STATUS_ACCESS_VIOLATION
394    je DispatchAccessViolation
395
396    /* Dispatch privileged instruction fault */
397    DispatchException eax, 0, 0, 0, 0
398    jmp KiGpfExit
399
400DispatchAccessViolation:
401
402    /* Dispatch access violation */
403    DispatchException eax, 2, 0, -1, 0
404
405KiGpfExit:
406    /* Return */
407    ExitTrap TF_SAVE_ALL
408ENDFUNC
409
410
411PUBLIC KiPageFault
412FUNC KiPageFault
413    /* We have an error code */
414    EnterTrap (TF_HAS_ERROR_CODE OR TF_SAVE_ALL)
415
416    /* Save page fault address */
417    mov rdx, cr2
418    mov [rbp  + KTRAP_FRAME_FaultAddress], rdx
419
420    /* If interrupts are off, do not enable them */
421    test dword ptr [rbp + KTRAP_FRAME_EFlags], EFLAGS_IF_MASK
422    jz IntsDisabled
423
424    /* Enable interrupts for the page fault handler */
425    sti
426
427IntsDisabled:
428
429    /* Call page fault handler */
430    mov ecx, [rbp + KTRAP_FRAME_ErrorCode] // FaultCode
431    // rdx == Address
432    mov r8b, [rbp + KTRAP_FRAME_SegCs] // Mode
433    and r8b, 1
434    mov r9, rbp // TrapInformation
435    call MmAccessFault
436
437    /* Check for success */
438    test eax, eax
439    jl PageFaultError
440
441    /* Check whether the kernel debugger has owed breakpoints to be inserted */
442    call KdSetOwedBreakpoints
443    /* We succeeded, return */
444    jmp PageFaultReturn
445
446PageFaultError:
447
448    /* Set parameter 1 to error code */
449    mov r9d, [rbp + KTRAP_FRAME_ErrorCode]
450
451    /* Set parameter 2 to faulting address */
452    mov r10, cr2  // Param2 = faulting address
453
454    cmp eax, STATUS_ACCESS_VIOLATION
455    je AccessViolation
456    cmp eax, STATUS_GUARD_PAGE_VIOLATION
457    je SpecialCode
458    cmp eax, STATUS_STACK_OVERFLOW
459    je SpecialCode
460
461InPageException:
462    /* Dispatch in-page exception */
463    mov r11d, eax // Param3 = Status
464    mov eax, STATUS_IN_PAGE_ERROR // ExceptionCode
465    mov edx, 3    // ParamCount
466    call InternalDispatchException
467    jmp PageFaultReturn
468
469AccessViolation:
470    /* Use more proper status code */
471    mov eax, KI_EXCEPTION_ACCESS_VIOLATION
472
473SpecialCode:
474    /* Setup a normal page fault exception */
475    mov edx, 2   // ParamCount
476    call InternalDispatchException
477
478PageFaultReturn:
479
480    /* Disable interrupts for the return */
481    cli
482
483    /* Return */
484    ExitTrap (TF_SAVE_ALL or TF_CHECKUSERAPC)
485ENDFUNC
486
487
488PUBLIC KiFloatingErrorFault
489FUNC KiFloatingErrorFault
490    /* Push pseudo error code */
491    EnterTrap TF_SAVE_ALL
492
493    UNIMPLEMENTED KiFloatingErrorFault
494    int 3
495
496    /* Return */
497    ExitTrap TF_SAVE_ALL
498ENDFUNC
499
500
501PUBLIC KiAlignmentFault
502FUNC KiAlignmentFault
503    /* We have an error code */
504    EnterTrap (TF_HAS_ERROR_CODE OR TF_SAVE_ALL)
505
506    /* Bugcheck */
507    Fatal EXCEPTION_ALIGNMENT_CHECK
508    jmp $
509ENDFUNC
510
511
512PUBLIC KiMcheckAbort
513FUNC KiMcheckAbort
514    /* Push pseudo error code */
515    EnterTrap TF_SAVE_ALL
516
517    /* Bugcheck */
518    Fatal HEX(12)
519    jmp $
520ENDFUNC
521
522
523PUBLIC KiXmmException
524FUNC KiXmmException
525    /* Push pseudo error code */
526    EnterTrap TF_SAVE_ALL
527
528    /* Call the C handler */
529    mov rcx, rbp
530    call KiXmmExceptionHandler
531
532    /* Check for success */
533    test eax, eax
534    jge KiXmmExit
535
536    /* Dispatch the exception */
537    DispatchException eax, 2, 0, [rbp+KTRAP_FRAME_MxCsr], 0
538
539    // FIXME: STATUS_FLOAT_MULTIPLE_TRAPS / STATUS_FLOAT_MULTIPLE_FAULTS
540
541KiXmmExit:
542    /* Return */
543    ExitTrap TF_SAVE_ALL
544ENDFUNC
545
546
547/* SOFTWARE INTERRUPT SERVICES ***********************************************/
548
549PUBLIC KiRaiseAssertion
550FUNC KiRaiseAssertion
551    /* We have an error code */
552    EnterTrap (TF_SAVE_ALL)
553
554    /* Decrement RIP to point to the INT2C instruction (2 bytes, not 1 like INT3) */
555    sub qword ptr [rbp + KTRAP_FRAME_Rip], 2
556
557    /* Dispatch the exception */
558    DispatchException STATUS_ASSERTION_FAILURE, 0, 0, 0, 0
559
560    /* Return */
561    ExitTrap TF_SAVE_ALL
562ENDFUNC
563
564
565PUBLIC KiDebugServiceTrap
566FUNC KiDebugServiceTrap
567   /* No error code */
568    EnterTrap TF_SAVE_ALL
569
570    /* Increase Rip to skip the int3 */
571    inc qword ptr [rbp + KTRAP_FRAME_Rip]
572
573    /* Dispatch the exception (Params = service, buffer, legth) */
574    DispatchException STATUS_BREAKPOINT, 3, [rbp+KTRAP_FRAME_Rax], [rbp+KTRAP_FRAME_Rcx], [rbp+KTRAP_FRAME_Rdx]
575
576    /* Return */
577    ExitTrap TF_SAVE_ALL
578ENDFUNC
579
580
581PUBLIC KiApcInterrupt
582.PROC KiApcInterrupt
583   /* No error code */
584    EnterTrap (TF_SAVE_ALL or TF_IRQL)
585
586    /* Raise to APC_LEVEL */
587    mov rax, APC_LEVEL
588    mov cr8, rax
589
590    /* End the interrupt */
591    mov dword ptr [APIC_EOI], 0
592
593    /* Enable interrupts  */
594    sti
595
596    /* Call the worker routine */
597    mov cl, [rbp + KTRAP_FRAME_SegCs] // ProcessorMode
598    and cl, 1
599    mov rdx, 0                        // ExceptionFrame
600    mov r8, rbp                       // TrapFrame
601    call KiDeliverApc
602
603    /* Disable interrupts */
604    cli
605
606    /* Lower IRQL back to PASSIVE */
607    mov rax, PASSIVE_LEVEL
608    mov cr8, rax
609
610    /* Return */
611    ExitTrap (TF_SAVE_ALL or TF_IRQL)
612.ENDP
613
614/*
615 * VOID
616 * KiRetireDpcList(
617 *     PKPRCB Prcb);
618 */
619EXTERN KiRetireDpcList:PROC
620
621/*
622 * VOID
623 * KiRetireDpcListInDpcStack(
624 *     PKPRCB Prcb,
625 *     PVOID DpcStack);
626 */
627PUBLIC KiRetireDpcListInDpcStack
628.PROC KiRetireDpcListInDpcStack
629    push rbp
630    .pushreg rbp
631    mov rbp, rsp
632    .setframe rbp, 0
633    .endprolog
634
635    /* Switch to the DpcStack */
636    mov rsp, rdx
637
638    /* The stack is 16 byte aligned, allocate 32 bytes home space */
639    sub rsp, 32
640
641    /* Call KiRetireDpcList on the given stack */
642    call KiRetireDpcList
643
644    /* Restore stack, cleanup and return */
645    mov rsp, rbp
646    pop rbp
647    ret
648.ENDP
649
650PUBLIC KiDpcInterrupt
651.PROC KiDpcInterrupt
652   /* No error code */
653    EnterTrap (TF_SAVE_ALL or TF_IRQL)
654
655    /* Call the worker routine */
656    call KiDpcInterruptHandler
657
658    /* Return, but don't send an EOI! */
659    ExitTrap (TF_SAVE_ALL or TF_IRQL)
660.ENDP
661
662
663PUBLIC KiIpiInterrupt
664.PROC KiIpiInterrupt
665   /* No error code */
666    EnterTrap (TF_SAVE_ALL or TF_IRQL)
667
668    /* Raise to IPI_LEVEL */
669    mov rax, IPI_LEVEL
670    mov cr8, rax
671
672    /* End the interrupt */
673    mov dword ptr [APIC_EOI], 0
674
675    int 3
676
677    /* Return */
678    ExitTrap (TF_SAVE_ALL or TF_IRQL)
679.ENDP
680
681
682PUBLIC KiUnexpectedInterrupt
683FUNC KiUnexpectedInterrupt
684    /* The error code is the vector */
685    EnterTrap (TF_HAS_ERROR_CODE OR TF_SAVE_ALL)
686
687#if 0
688    /* Set bugcheck parameters */
689    mov ecx, TRAP_CAUSE_UNKNOWN
690    mov rdx, [rbp + KTRAP_FRAME_ErrorCode] // the vector
691    mov r8, 0 // The unknown floating-point exception
692    mov r9, 0 // The enabled and asserted status bits
693    sub rsp, 8
694    mov [rbp + KTRAP_FRAME_P5 + 8], rbp // trap frame
695    call KeBugCheckWithTf
696    jmp $
697#endif
698    /* Return */
699    ExitTrap TF_SAVE_ALL
700ENDFUNC
701
702PUBLIC KiInterruptDispatch
703FUNC KiInterruptDispatch
704    /* The error code is a pointer to the interrupt object's code */
705    EnterTrap (TF_HAS_ERROR_CODE or TF_SAVE_ALL or TF_IRQL)
706
707    /* Increase interrupt count */
708    inc dword ptr gs:[PcInterruptCount];
709
710    /* Save rbx and rsi in the trap frame */
711    mov [rbp + KTRAP_FRAME_Rbx], rbx
712    mov [rbp + KTRAP_FRAME_Rsi], rsi
713
714    /* Load the address of the dispatch code into rbx */
715    mov rbx, [rbp + KTRAP_FRAME_ErrorCode]
716
717    /* Substract offset of the DispatchCode member plus 6 for the call instruction */
718    sub rbx, KINTERRUPT_DispatchCode + 6
719
720    /* Save the address of the InterruptListEntry in rsi */
721    lea rsi, [rbx + KINTERRUPT_InterruptListEntry]
722
723.DoDispatchInterrupt:
724    /* Raise IRQL to SynchronizeIrql */
725    movzx rax, byte ptr [rbx + KINTERRUPT_SynchronizeIrql]
726    mov cr8, rax
727
728#ifdef CONFIG_SMP
729    /* Acquire interrupt lock */
730    mov r8, [rbx + KINTERRUPT_ActualLock]
731
732    //KxAcquireSpinLock(Interrupt->ActualLock);
733#endif
734
735    /* Call the ISR */
736    mov rcx, rbx
737    mov rdx, [rbx + KINTERRUPT_ServiceContext]
738    call qword ptr [rbx + KINTERRUPT_ServiceRoutine]
739
740#ifdef CONFIG_SMP
741    /* Release interrupt lock */
742    //KxReleaseSpinLock(Interrupt->ActualLock);
743#endif
744
745    /* Go back to old irql */
746    movzx rax, byte ptr [rbp + KTRAP_FRAME_PreviousIrql]
747    mov cr8, rax
748
749    /* Check for chained interrupts */
750    mov rax, [rbx + KINTERRUPT_InterruptListEntry]
751    cmp rax, rsi
752    je .Done
753
754    /* Load the next interrupt object into rbx and repeat */
755    lea rbx, [rax - KINTERRUPT_InterruptListEntry]
756    jmp .DoDispatchInterrupt
757
758.Done:
759    /* Restore rbx and rsi */
760    mov rbx, [rbp + KTRAP_FRAME_Rbx]
761    mov rsi, [rbp + KTRAP_FRAME_Rsi]
762
763    /* Return */
764    ExitTrap (TF_SAVE_ALL or TF_SEND_EOI)
765ENDFUNC
766
767EXTERN KiSystemCallHandler:PROC
768
769/*! \name KiSystemCallEntry64
770 *
771 *  \brief This is the entrypoint for syscalls from 64bit user mode
772 *
773 *  \param rax - The system call number
774 *  \param rcx - User mode return address, set by the syscall instruction
775 *  \param rdx,r8,r9 - Parameters 2-4 to the service function
776 *  \param r10 - Parameter 1 to the service function
777 *  \param r11 - RFLAGS saved by the syscall instruction
778 *--*/
779PUBLIC KiSystemCallEntry64
780.PROC KiSystemCallEntry64
781    /* The unwind info pretends we have a machine frame */
782    .PUSHFRAME
783    .ALLOCSTACK (KTRAP_FRAME_LENGTH + MAX_SYSCALL_PARAM_SIZE - MachineFrameLength)
784    .ENDPROLOG
785
786    /* Swap gs to kernel, so we can access the PCR */
787    swapgs
788
789    /* Save the user mode rsp in the PCR */
790    mov gs:[PcUserRsp], rsp
791
792    /* Get the kernel stack from the PCR */
793    mov rsp, gs:[PcRspBase]
794
795    /* Allocate a TRAP_FRAME and space for parameters */
796    sub rsp, (KTRAP_FRAME_LENGTH + MAX_SYSCALL_PARAM_SIZE)
797
798    /* Save volatile registers in the trap frame */
799    mov [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rax], rax
800    mov [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rip], rcx
801    mov [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rdx], rdx
802    mov [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_R8], r8
803    mov [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_R9], r9
804    mov [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rcx], r10
805    mov [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_EFlags], r11
806
807    /* Store user stack pointer in the trap frame */
808    mov rax, gs:[PcUserRsp]
809    mov [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rsp], rax
810
811    /* Set sane segments */
812    mov ax, (KGDT64_R3_DATA or RPL_MASK)
813    mov ds, ax
814    mov es, ax
815
816    /* Save MCXSR and set kernel value */
817    stmxcsr [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_MxCsr]
818    ldmxcsr gs:[PcMxCsr]
819
820#if DBG
821    /* Check IRQL */
822    mov rax, cr8
823    test eax, eax
824    jz KiSystemCall64Again
825    int HEX(2C)
826#endif
827
828GLOBAL_LABEL KiSystemCall64Again
829
830    /* Call the C-handler (will enable interrupts) */
831    call KiSystemCallHandler
832
833    /* The return value from KiSystemCallHandler is the address of the Nt-function */
834    mov rcx, [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rcx]
835    mov rdx, [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rdx]
836    mov r8, [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_R8]
837    mov r9, [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_R9]
838    call rax
839
840GLOBAL_LABEL KiSystemServiceExit
841
842#if DBG
843    test dword ptr [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_EFlags], HEX(200)
844    jnz IntsEnabled
845    int 3
846IntsEnabled:
847#endif
848
849    /* Check for pending user APC */
850    mov rcx, gs:qword ptr [PcCurrentThread]
851    cmp byte ptr [rcx + ThApcState + AsUserApcPending], 0
852    jz no_user_apc_pending
853    call KiInitiateUserApc
854no_user_apc_pending:
855
856    /* Disable interrupts for return */
857    cli
858
859    /* Restore MCXSR */
860    ldmxcsr [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_MxCsr]
861
862    /* Restore old trap frame */
863    mov rcx, gs:[PcCurrentThread]
864    mov rdx, [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_TrapFrame]
865    mov [rcx + KTHREAD_TrapFrame], rdx
866
867    /* Prepare user mode return address (rcx) and eflags (r11) for sysret */
868    mov rcx, [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rip]
869    mov r11, [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_EFlags]
870
871    /* Load user mode stack (It was copied to the trap frame) */
872    mov rsp, [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rsp]
873
874    /* r8 points to the user stack */
875    mov r8, rsp
876
877    /* r9 matches rbp */
878    mov r9, rbp
879
880    /* Swap gs back to user */
881    swapgs
882
883    /* Zero out volatiles */
884    pxor xmm0, xmm0
885    pxor xmm1, xmm1
886    pxor xmm2, xmm2
887    pxor xmm3, xmm3
888    pxor xmm4, xmm4
889    pxor xmm5, xmm5
890    xor rdx, rdx
891    xor r10, r10
892
893    /* return to user mode */
894    .byte HEX(48) // REX prefix to return to long mode
895    sysretq
896
897.ENDP
898
899
900/*!
901 * VOID
902 * DECLSPEC_NORETURN
903 * KiServiceExit(IN PKTRAP_FRAME TrapFrame, IN NTSTATUS Status));
904 */
905PUBLIC KiServiceExit
906.PROC KiServiceExit
907    .endprolog
908
909    lea rsp, [rcx - MAX_SYSCALL_PARAM_SIZE]
910    jmp KiSystemServiceExit
911
912.ENDP
913
914
915/*!
916 * VOID
917 * DECLSPEC_NORETURN
918 * KiServiceExit2(IN PKTRAP_FRAME TrapFrame);
919 */
920PUBLIC KiServiceExit2
921.PROC KiServiceExit2
922    .ENDPROLOG
923
924    // FIXME: this should probably also restore an exception frame
925
926    mov rsp, rcx
927.ENDP
928
929PUBLIC KiServiceExit3
930.PROC KiServiceExit3
931    .PUSHFRAME
932    .ALLOCSTACK (KTRAP_FRAME_LENGTH - MachineFrameLength)
933    .ENDPROLOG
934
935#if DBG
936    /* Get the current IRQL and compare it to the trap frame */
937    mov rax, cr8
938    cmp byte ptr [rsp + KTRAP_FRAME_PreviousIrql], al
939    je KiServiceExit2_ok1
940    int HEX(2C)
941
942KiServiceExit2_ok1:
943    /* Check if this is a user mode exit */
944    mov ah, byte ptr [rsp + KTRAP_FRAME_SegCs]
945    test ah, 1
946    jz KiServiceExit2_kernel
947
948    /* Validate that we are at PASSIVE_LEVEL */
949    test al, al
950    jz KiServiceExit2_kernel
951    int HEX(2C)
952
953KiServiceExit2_kernel:
954#endif
955
956    /* Return */
957    mov rbp, rsp
958    ExitTrap TF_SAVE_ALL
959.ENDP
960
961
962PUBLIC KiSystemCallEntry32
963KiSystemCallEntry32:
964    swapgs
965    int 3
966
967
968PUBLIC KiZwSystemService
969FUNC KiZwSystemService
970    push rbp
971    .pushreg rbp
972    sub rsp, KTRAP_FRAME_LENGTH
973    .allocstack KTRAP_FRAME_LENGTH
974    mov [rsp + KTRAP_FRAME_Rsi], rsi
975    .savereg rsi, KTRAP_FRAME_Rsi
976    mov [rsp + KTRAP_FRAME_Rdi], rdi
977    .savereg rdi, KTRAP_FRAME_Rdi
978    mov rbp, rsp
979    .setframe rbp, 0
980    .endprolog
981
982    /* Get current thread */
983    mov r11, gs:[PcCurrentThread]
984
985    /* Save PreviousMode in the trap frame */
986    mov dil, byte ptr [r11 + KTHREAD_PreviousMode]
987    mov byte ptr [rbp + KTRAP_FRAME_PreviousMode], dil
988
989    /* Save the old trap frame in TrapFrame.Rdx */
990    mov rdi, [r11 + KTHREAD_TrapFrame]
991    mov [rbp + KTRAP_FRAME_Rdx], rdi
992
993    /* Set the new trap frame and previous mode */
994    mov [r11 + ThTrapFrame], rbp
995    mov byte ptr [r11 + KTHREAD_PreviousMode], 0
996
997    /* allocate space for parameters */
998    sub rsp, r10
999    and rsp, HEX(0fffffffffffffff0)
1000
1001    /* Save rcx */
1002    mov [rbp + KTRAP_FRAME_Rcx], rcx
1003
1004    /* copy parameters to the new location */
1005    lea rsi, [rbp + KTRAP_FRAME_LENGTH + 16]
1006    lea rdi, [rsp]
1007    mov rcx, r10
1008    shr rcx, 3
1009    rep movsq
1010
1011    /* Restore rcx */
1012    mov rcx, [rbp + KTRAP_FRAME_Rcx]
1013
1014    /* Call the service function */
1015    call rax
1016
1017    /* Restore the old trap frame */
1018    mov r11, gs:[PcCurrentThread]
1019    mov rsi, [rbp + KTRAP_FRAME_Rdx]
1020    mov [r11 + KTHREAD_TrapFrame], rsi
1021
1022    /* Restore PreviousMode from the trap frame */
1023    mov dil, byte ptr [rbp + KTRAP_FRAME_PreviousMode]
1024    mov byte ptr [r11 + KTHREAD_PreviousMode], dil
1025
1026    /* Restore rdi and rsi */
1027    mov rsi, [rbp + KTRAP_FRAME_Rsi]
1028    mov rdi, [rbp + KTRAP_FRAME_Rdi]
1029
1030    /* Cleanup the stack and return */
1031    lea rsp, [rbp + KTRAP_FRAME_LENGTH]
1032    pop rbp
1033    ret
1034
1035ENDFUNC
1036
1037PUBLIC KiConvertToGuiThread
1038FUNC KiConvertToGuiThread
1039
1040    sub rsp, 40
1041    .allocstack 40
1042    .endprolog
1043
1044    /* Check if we already have a large stack */
1045    mov rax, gs:[PcCurrentThread]
1046    cmp byte ptr [rax + KTHREAD_LargeStack], 0
1047    jnz AlreadyLargeStack
1048
1049    // NewStack = (ULONG_PTR)MmCreateKernelStack(TRUE, 0);
1050    mov cl, 1
1051    xor rdx, rdx
1052    call MmCreateKernelStack
1053
1054    /* Check for failure */
1055    test rax, rax
1056    jz KiConvertToGuiThreadFailed
1057
1058    /* OldStack = KeSwitchKernelStack((PVOID)NewStack, (PVOID)(NewStack - KERNEL_LARGE_STACK_COMMIT )); */
1059    mov rcx, rax
1060    mov rdx, rax
1061    sub rdx, KERNEL_LARGE_STACK_COMMIT
1062    call KeSwitchKernelStack
1063
1064    // MmDeleteKernelStack(OldStack, FALSE);
1065    mov rcx, rax
1066    xor rdx, rdx
1067    call MmDeleteKernelStack
1068
1069AlreadyLargeStack:
1070
1071    /* Call the worker function */
1072    call PsConvertToGuiThread
1073
1074    /* Check for failure */
1075    test eax, eax
1076    js KiConvertToGuiThreadFailed
1077
1078    /* Disable interrupts for return */
1079    cli
1080
1081    // FIXME: should just do the trap frame switch in KiSystemCallHandler64
1082    /* Restore old trap frame */
1083    mov rcx, gs:[PcCurrentThread]
1084    mov rdx, [rsp + 48 + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_TrapFrame]
1085    mov [rcx + KTHREAD_TrapFrame], rdx
1086
1087    // Restore register parameters
1088    mov rcx, [rsp + 48 + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rip]
1089    mov rdx, [rsp + 48 + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rdx]
1090    mov r8, [rsp + 48 + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_R8]
1091    mov r9, [rsp + 48 + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_R9]
1092
1093    /* Run KiSystemCallHandler again */
1094    add rsp, 48
1095    jmp KiSystemCall64Again
1096
1097KiConvertToGuiThreadFailed:
1098
1099    /* Clean up the stack and return failure */
1100    add rsp, 40
1101    mov eax, HEX(C0000017) // STATUS_NO_MEMORY
1102    ret
1103
1104ENDFUNC
1105
1106
1107EXTERN KiSetTrapContextInternal:PROC
1108
1109/*
1110 * VOID
1111 * KiSetTrapContext(
1112 *     _Out_ PKTRAP_FRAME TrapFrame,
1113 *     _In_ PCONTEXT Context,
1114 *     _In_ KPROCESSOR_MODE RequestorMode);
1115 */
1116PUBLIC KiSetTrapContext
1117.PROC KiSetTrapContext
1118
1119    /* Generate a KEXCEPTION_FRAME on the stack */
1120    GENERATE_EXCEPTION_FRAME
1121
1122    call KiSetTrapContextInternal
1123
1124    /* Restore the registers from the KEXCEPTION_FRAME */
1125    RESTORE_EXCEPTION_STATE
1126
1127    /* Return */
1128    ret
1129
1130.ENDP
1131
1132
1133/*
1134 * VOID
1135 * KiDeliverApc(
1136 *     _In_ KPROCESSOR_MODE DeliveryMode,
1137 *     _In_ PKEXCEPTION_FRAME ExceptionFrame,
1138 *     _In_ PKTRAP_FRAME TrapFrame);
1139 *
1140 */
1141EXTERN KiDeliverApc:PROC
1142
1143PUBLIC KiInitiateUserApc
1144.PROC KiInitiateUserApc
1145
1146    /* Generate a KEXCEPTION_FRAME on the stack */
1147    GENERATE_EXCEPTION_FRAME
1148
1149    /* Raise IRQL to APC_LEVEL */
1150    mov rax, APC_LEVEL
1151    mov cr8, rax
1152
1153    /* Enable interrupts */
1154    sti
1155
1156    /* Get the current trap frame */
1157    mov rax, gs:[PcCurrentThread]
1158    mov r8, [rax + KTHREAD_TrapFrame]
1159
1160    /* Call the C function */
1161    mov ecx, 1
1162    mov rdx, rsp
1163    call KiDeliverApc
1164
1165    /* Disable interrupts again */
1166    cli
1167
1168    /* Go back to PASSIVE_LEVEL */
1169    mov rax, PASSIVE_LEVEL
1170    mov cr8, rax
1171
1172    /* Restore the registers from the KEXCEPTION_FRAME */
1173    RESTORE_EXCEPTION_STATE
1174
1175    /* Return */
1176    ret
1177
1178.ENDP
1179
1180
1181PUBLIC KiInitializeSegments
1182KiInitializeSegments:
1183    mov ax, KGDT64_R3_DATA or RPL_MASK
1184    mov gs, ax
1185    swapgs
1186    mov gs, ax
1187    ret
1188
1189/*!
1190 * VOID
1191 * KiSwitchKernelStackHelper(
1192 *     LONG_PTR StackOffset,
1193 *     PVOID OldStackBase);
1194 */
1195PUBLIC KiSwitchKernelStackHelper
1196KiSwitchKernelStackHelper:
1197
1198    /* Pop return address from the current stack */
1199    pop rax
1200
1201    /* Switch to new stack */
1202    lea rsp, [rsp + rcx]
1203
1204    /* Push return address on the new stack */
1205    push rax
1206
1207    /* Return on new stack */
1208    mov rax, rdx
1209    ret
1210
1211EXTERN KiSwitchKernelStack:PROC
1212
1213PUBLIC KeSwitchKernelStack
1214FUNC KeSwitchKernelStack
1215
1216    /* Save rcx and allocate callee home space */
1217    mov [rsp + P1Home], rcx
1218    .savereg rcx, P1Home
1219    sub rsp, 40
1220    .allocstack 40
1221    .endprolog
1222
1223    /* Call the C handler, which returns the old stack in rax */
1224    call KiSwitchKernelStack
1225
1226    /* Restore rcx (StackBase) */
1227    mov rcx, [rsp + 40 + P1Home]
1228
1229    /* Switch to new stack: RSP += (StackBase - OldStackBase) */
1230    sub rcx, rax
1231    add rsp, rcx
1232
1233    /* Deallocate the home frame */
1234    add rsp, 40
1235    ret
1236
1237ENDFUNC
1238
1239
1240
1241
1242#ifdef _MSC_VER
1243#undef lgdt
1244#undef lidt
1245
1246//void __lgdt(void *Source);
1247PUBLIC __lgdt
1248__lgdt:
1249    lgdt fword ptr [rcx]
1250    ret
1251
1252//void __sgdt(void *Destination);
1253PUBLIC __sgdt
1254__sgdt:
1255    sgdt fword ptr [rcx]
1256    ret
1257
1258// void __lldt(unsigned short Value)
1259PUBLIC __lldt
1260__lldt:
1261    lldt cx
1262    ret
1263
1264//void __sldt(void *Destination);
1265PUBLIC __sldt
1266__sldt:
1267    sldt word ptr [rcx]
1268    ret
1269
1270//void __ltr(unsigned short Source);
1271PUBLIC __ltr
1272__ltr:
1273    ltr cx
1274    ret
1275
1276//void __str(unsigned short *Destination);
1277PUBLIC __str
1278__str:
1279    str word ptr [rcx]
1280    ret
1281
1282PUBLIC __swapgs
1283__swapgs:
1284    swapgs
1285    ret
1286
1287#endif
1288
1289END
1290