xref: /reactos/ntoskrnl/ke/amd64/trap.S (revision ff531eaa) 
1/*
2 * FILE:            ntoskrnl/ke/amd64/trap.S
3 * COPYRIGHT:       See COPYING in the top level directory
4 * PURPOSE:         System Traps, Entrypoints and Exitpoints
5 * PROGRAMMER:      Timo Kreuzer (timo.kreuzer@reactos.org)
6 */
7
8/* INCLUDES ******************************************************************/
9
10#include <asm.inc>
11#include <ksamd64.inc>
12#include <trapamd64.inc>
13
14EXTERN KiDispatchException:PROC
15EXTERN KeBugCheckWithTf:PROC
16EXTERN MmAccessFault:PROC
17EXTERN KiSystemFatalException:PROC
18EXTERN KiNpxNotAvailableFaultHandler:PROC
19EXTERN KiGeneralProtectionFaultHandler:PROC
20EXTERN KiXmmExceptionHandler:PROC
21EXTERN KiDeliverApc:PROC
22EXTERN KiDpcInterruptHandler:PROC
23EXTERN PsConvertToGuiThread:PROC
24EXTERN MmCreateKernelStack:PROC
25EXTERN MmDeleteKernelStack:PROC
26
27EXTERN KdSetOwedBreakpoints:PROC
28
29
30/* Helper Macros *************************************************************/
31
32MACRO(DispatchException, Status, Number, P1, P2, P3)
33    mov eax, Status
34    mov edx, Number
35    mov r9, P1
36    mov r10, P2
37    mov r11, P3
38    call InternalDispatchException
39ENDM
40
41MACRO(Fatal, BugcheckCode)
42    /* Bugcheck */
43    mov ecx, BugcheckCode
44    mov rdx, rbp
45    call KiSystemFatalException
46ENDM
47
48
49/* FUNCTIONS *****************************************************************/
50
51.code64
52
53ALIGN 8
54
55MACRO(UnexpectedVectorStub, Vector)
56    /* This nop is to make the relative jmp address 4 bytes aligned and to
57       make the whole code 8 bytes long */
58    nop
59    /* This is a push instruction with 8bit operand. Since the instruction
60       sign extends the value to 32 bits, we need to offset it */
61PUBLIC KxUnexpectedInterrupt&Vector
62KxUnexpectedInterrupt&Vector:
63    push (Vector - 128)
64    jmp KiUnexpectedInterrupt
65ENDM
66
67PUBLIC KiUnexpectedRange
68KiUnexpectedRange:
69Vector = 0
70REPEAT 256
71    UnexpectedVectorStub %Vector
72    Vector = Vector+1
73ENDR
74PUBLIC KiUnexpectedRangeEnd
75KiUnexpectedRangeEnd:
76
77PUBLIC KiInterruptDispatchTemplate
78KiInterruptDispatchTemplate:
79    /* This instruction pushes the return address on the stack, which is the
80       address of the interrupt object's DispatchCode member, then jumps
81       to the address stored in the interrupt object's DispatchAddress member */
82    call qword ptr KiInterruptDispatchTemplate[rip - KINTERRUPT_DispatchCode + KINTERRUPT_DispatchAddress]
83
84
85// rbp = TrapFrame, eax = ExceptionCode, edx = NumParams, r9,r10,r11 = params
86FUNC InternalDispatchException
87
88    /* Allocate stack space for EXCEPTION_RECORD and KEXCEPTION_FRAME */
89    sub rsp, EXCEPTION_RECORD_LENGTH + KEXCEPTION_FRAME_LENGTH
90    .allocstack (EXCEPTION_RECORD_LENGTH + KEXCEPTION_FRAME_LENGTH)
91    .endprolog
92
93    /* Set up EXCEPTION_RECORD */
94    lea rcx, [rsp + KEXCEPTION_FRAME_LENGTH]
95    mov [rcx + EXCEPTION_RECORD_ExceptionCode], eax
96    xor rax, rax
97    mov [rcx + EXCEPTION_RECORD_ExceptionFlags], eax
98    mov [rcx + EXCEPTION_RECORD_ExceptionRecord], rax
99    mov rax, [rbp + KTRAP_FRAME_Rip]
100    mov [rcx + EXCEPTION_RECORD_ExceptionAddress], rax
101    mov [rcx + EXCEPTION_RECORD_NumberParameters], edx
102    mov [rcx + EXCEPTION_RECORD_ExceptionInformation + HEX(00)], r9
103    mov [rcx + EXCEPTION_RECORD_ExceptionInformation + HEX(08)], r10
104    mov [rcx + EXCEPTION_RECORD_ExceptionInformation + HEX(10)], r11
105
106    /* Set up KEXCEPTION_FRAME */
107    mov rax, [rbp + KTRAP_FRAME_Rbp]
108    mov [rsp + KEXCEPTION_FRAME_Rbp], rax
109    mov [rsp + KEXCEPTION_FRAME_Rbx], rbx
110    mov [rsp + KEXCEPTION_FRAME_Rdi], rdi
111    mov [rsp + KEXCEPTION_FRAME_Rsi], rsi
112    mov [rsp + KEXCEPTION_FRAME_R12], r12
113    mov [rsp + KEXCEPTION_FRAME_R13], r13
114    mov [rsp + KEXCEPTION_FRAME_R14], r14
115    mov [rsp + KEXCEPTION_FRAME_R15], r15
116    movdqa [rsp + KEXCEPTION_FRAME_Xmm6], xmm6
117    movdqa [rsp + KEXCEPTION_FRAME_Xmm7], xmm7
118    movdqa [rsp + KEXCEPTION_FRAME_Xmm8], xmm8
119    movdqa [rsp + KEXCEPTION_FRAME_Xmm9], xmm9
120    movdqa [rsp + KEXCEPTION_FRAME_Xmm10], xmm10
121    movdqa [rsp + KEXCEPTION_FRAME_Xmm11], xmm11
122    movdqa [rsp + KEXCEPTION_FRAME_Xmm12], xmm12
123    movdqa [rsp + KEXCEPTION_FRAME_Xmm13], xmm13
124    movdqa [rsp + KEXCEPTION_FRAME_Xmm14], xmm14
125    movdqa [rsp + KEXCEPTION_FRAME_Xmm15], xmm15
126    mov qword ptr [rsp + KEXCEPTION_FRAME_Return], 0
127
128    /* Call KiDispatchException */
129    // rcx already points to ExceptionRecord
130    mov rdx, rsp                                // ExceptionFrame
131    mov r8, rbp                                 // TrapFrame
132    mov r9b, [r8 + KTRAP_FRAME_PreviousMode]    // PreviousMode
133    mov byte ptr [rsp + KEXCEPTION_FRAME_P5], 1 // FirstChance
134    call KiDispatchException
135
136    /* Restore registers */
137    mov r12, [rsp + KEXCEPTION_FRAME_R12]
138    mov r13, [rsp + KEXCEPTION_FRAME_R13]
139    mov r14, [rsp + KEXCEPTION_FRAME_R14]
140    mov r15, [rsp + KEXCEPTION_FRAME_R15]
141    movdqa xmm6, [rsp + KEXCEPTION_FRAME_Xmm6]
142    movdqa xmm7, [rsp + KEXCEPTION_FRAME_Xmm7]
143    movdqa xmm8, [rsp + KEXCEPTION_FRAME_Xmm8]
144    movdqa xmm9, [rsp + KEXCEPTION_FRAME_Xmm9]
145    movdqa xmm10, [rsp + KEXCEPTION_FRAME_Xmm10]
146    movdqa xmm11, [rsp + KEXCEPTION_FRAME_Xmm11]
147    movdqa xmm12, [rsp + KEXCEPTION_FRAME_Xmm12]
148    movdqa xmm13, [rsp + KEXCEPTION_FRAME_Xmm13]
149    movdqa xmm14, [rsp + KEXCEPTION_FRAME_Xmm14]
150    movdqa xmm15, [rsp + KEXCEPTION_FRAME_Xmm15]
151
152    add rsp, EXCEPTION_RECORD_LENGTH + KEXCEPTION_FRAME_LENGTH
153    ret
154ENDFUNC
155
156
157/* CPU EXCEPTION HANDLERS ****************************************************/
158
159PUBLIC KiDivideErrorFault
160FUNC KiDivideErrorFault
161    /* Push pseudo error code */
162    EnterTrap TF_SAVE_ALL
163
164    /* Enable interrupts */
165    sti
166
167    /* Dispatch the exception */
168    DispatchException STATUS_INTEGER_DIVIDE_BY_ZERO, 0, 0, 0, 0
169
170    /* Return */
171    ExitTrap TF_SAVE_ALL
172ENDFUNC
173
174
175PUBLIC KiDebugTrapOrFault
176FUNC KiDebugTrapOrFault
177    /* Push pseudo error code */
178    EnterTrap TF_SAVE_ALL
179
180    /* Check if the frame was from kernelmode */
181    test word ptr [rbp + KTRAP_FRAME_SegCs], 3
182    jz KiDebugTrapOrFaultKMode
183
184    /* Enable interrupts for user-mode */
185    sti
186
187KiDebugTrapOrFaultKMode:
188    /* Dispatch the exception */
189    DispatchException STATUS_SINGLE_STEP, 0, 0, 0, 0
190
191    /* Return */
192    ExitTrap TF_SAVE_ALL
193ENDFUNC
194
195
196PUBLIC KiNmiInterrupt
197FUNC KiNmiInterrupt
198    /* Push pseudo error code */
199    EnterTrap TF_SAVE_ALL
200
201    UNIMPLEMENTED KiNmiInterrupt
202    int 3
203
204    /* Return */
205    ExitTrap TF_SAVE_ALL
206ENDFUNC
207
208
209PUBLIC KiBreakpointTrap
210FUNC KiBreakpointTrap
211    /* Push pseudo error code */
212    EnterTrap TF_SAVE_ALL
213
214    /* Check if the frame was from kernelmode */
215    test word ptr [rbp + KTRAP_FRAME_SegCs], 3
216    jz KiBreakpointTrapKMode
217
218    /* Enable interrupts for user-mode */
219    sti
220
221KiBreakpointTrapKMode:
222    /* Dispatch the exception */
223    DispatchException STATUS_BREAKPOINT, 3, BREAKPOINT_BREAK, 0, 0
224
225    /* Return */
226    ExitTrap TF_SAVE_ALL
227ENDFUNC
228
229
230PUBLIC KiOverflowTrap
231FUNC KiOverflowTrap
232    /* Push pseudo error code */
233    EnterTrap TF_SAVE_ALL
234
235    /* Enable interrupts */
236    sti
237
238    /* Dispatch the exception */
239    DispatchException STATUS_INTEGER_OVERFLOW, 3, 0, 0, 0
240
241    /* Return */
242    ExitTrap TF_SAVE_ALL
243ENDFUNC
244
245
246PUBLIC KiBoundFault
247FUNC KiBoundFault
248    /* Push pseudo error code */
249    EnterTrap TF_SAVE_ALL
250
251    /* Check if the frame was from kernelmode */
252    test word ptr [rbp + KTRAP_FRAME_SegCs], 3
253    jnz KiBoundFaultUserMode
254
255    /* Bugcheck */
256    Fatal EXCEPTION_BOUND_CHECK
257
258KiBoundFaultUserMode:
259    /* Enable interrupts for user-mode */
260    sti
261
262    /* Dispatch the exception */
263    DispatchException STATUS_ARRAY_BOUNDS_EXCEEDED, 0, 0, 0, 0
264
265    /* Return */
266    ExitTrap TF_SAVE_ALL
267ENDFUNC
268
269
270PUBLIC KiInvalidOpcodeFault
271FUNC KiInvalidOpcodeFault
272    /* Push pseudo error code */
273    EnterTrap TF_SAVE_ALL
274
275    /* Enable interrupts */
276    sti
277
278    /* Check if the frame was from kernelmode */
279    test word ptr [rbp + KTRAP_FRAME_SegCs], 3
280    jz KiInvalidOpcodeKernel
281
282    // FIXME: handle STATUS_INVALID_LOCK_SEQUENCE
283
284KiInvalidOpcodeKernel:
285    /* Kernel mode fault */
286
287    /* Dispatch the exception */
288    DispatchException STATUS_ILLEGAL_INSTRUCTION, 0, 0, 0, 0
289
290    /* Return */
291    ExitTrap TF_SAVE_ALL
292ENDFUNC
293
294
295PUBLIC KiNpxNotAvailableFault
296FUNC KiNpxNotAvailableFault
297    /* Push pseudo error code */
298    EnterTrap TF_SAVE_ALL
299
300    /* Call the C handler */
301    mov rcx, rbp
302    call KiNpxNotAvailableFaultHandler
303
304    /* Check the return status code */
305    test eax, eax
306    jz KiNpxNotAvailableFaultExit
307
308    /* Dispatch the exception */
309    DispatchException eax, 3, 0, 0, 0
310
311KiNpxNotAvailableFaultExit:
312    /* Return */
313    ExitTrap TF_SAVE_ALL
314ENDFUNC
315
316
317PUBLIC KiDoubleFaultAbort
318FUNC KiDoubleFaultAbort
319
320    /* Hack for VBox, which "forgets" to push an error code on the stack! */
321    and rsp, HEX(FFFFFFFFFFFFFFF0)
322
323    /* A zero error code is pushed */
324    EnterTrap (TF_HAS_ERROR_CODE OR TF_SAVE_ALL)
325
326    int 3
327
328    /* Bugcheck */
329    Fatal 8 // EXCEPTION_DOUBLE_FAULT
330    jmp $
331ENDFUNC
332
333
334PUBLIC KiNpxSegmentOverrunAbort
335FUNC KiNpxSegmentOverrunAbort
336    /* Push pseudo error code */
337    EnterTrap TF_SAVE_ALL
338
339    /* Bugcheck */
340    Fatal EXCEPTION_NPX_OVERRUN
341
342    jmp $
343ENDFUNC
344
345
346PUBLIC KiInvalidTssFault
347FUNC KiInvalidTssFault
348    /* We have an error code */
349    EnterTrap (TF_HAS_ERROR_CODE OR TF_SAVE_ALL)
350
351    /* Bugcheck */
352    Fatal EXCEPTION_INVALID_TSS
353    jmp $
354ENDFUNC
355
356
357PUBLIC KiSegmentNotPresentFault
358FUNC KiSegmentNotPresentFault
359    /* We have an error code */
360    EnterTrap (TF_HAS_ERROR_CODE OR TF_SAVE_ALL)
361
362    /* Bugcheck */
363    Fatal EXCEPTION_SEGMENT_NOT_PRESENT
364    jmp $
365ENDFUNC
366
367
368PUBLIC KiStackFault
369FUNC KiStackFault
370    /* We have an error code */
371    EnterTrap (TF_HAS_ERROR_CODE OR TF_SAVE_ALL)
372
373    /* Bugcheck */
374    Fatal EXCEPTION_STACK_FAULT
375    jmp $
376ENDFUNC
377
378
379PUBLIC KiGeneralProtectionFault
380FUNC KiGeneralProtectionFault
381    /* We have an error code */
382    EnterTrap (TF_HAS_ERROR_CODE OR TF_SAVE_ALL)
383
384    /* Call the C handler */
385    mov rcx, rbp
386    call KiGeneralProtectionFaultHandler
387
388    /* Check for success */
389    test eax, eax
390    jge KiGpfExit
391
392    /* Check for access violation */
393    cmp eax, STATUS_ACCESS_VIOLATION
394    je DispatchAccessViolation
395
396    /* Dispatch privileged instruction fault */
397    DispatchException eax, 0, 0, 0, 0
398    jmp KiGpfExit
399
400DispatchAccessViolation:
401
402    /* Dispatch access violation */
403    DispatchException eax, 2, 0, -1, 0
404
405KiGpfExit:
406    /* Return */
407    ExitTrap TF_SAVE_ALL
408ENDFUNC
409
410
411PUBLIC KiPageFault
412FUNC KiPageFault
413    /* We have an error code */
414    EnterTrap (TF_HAS_ERROR_CODE OR TF_SAVE_ALL)
415
416    /* Save page fault address */
417    mov rdx, cr2
418    mov [rbp  + KTRAP_FRAME_FaultAddress], rdx
419
420    /* If interrupts are off, do not enable them */
421    test dword ptr [rbp + KTRAP_FRAME_EFlags], EFLAGS_IF_MASK
422    jz IntsDisabled
423
424    /* Enable interrupts for the page fault handler */
425    sti
426
427IntsDisabled:
428
429    /* Call page fault handler */
430    mov ecx, [rbp + KTRAP_FRAME_ErrorCode] // FaultCode
431    // rdx == Address
432    mov r8b, [rbp + KTRAP_FRAME_SegCs] // Mode
433    and r8b, 1
434    mov r9, rbp // TrapInformation
435    call MmAccessFault
436
437    /* Check for success */
438    test eax, eax
439    jl PageFaultError
440
441    /* Check whether the kernel debugger has owed breakpoints to be inserted */
442    call KdSetOwedBreakpoints
443    /* We succeeded, return */
444    jmp PageFaultReturn
445
446PageFaultError:
447
448    /* Set parameter 1 to error code */
449    mov r9d, [rbp + KTRAP_FRAME_ErrorCode]
450
451    /* Set parameter 2 to faulting address */
452    mov r10, cr2  // Param2 = faulting address
453
454    cmp eax, STATUS_ACCESS_VIOLATION
455    je AccessViolation
456    cmp eax, STATUS_GUARD_PAGE_VIOLATION
457    je SpecialCode
458    cmp eax, STATUS_STACK_OVERFLOW
459    je SpecialCode
460
461InPageException:
462    /* Dispatch in-page exception */
463    mov r11d, eax // Param3 = Status
464    mov eax, STATUS_IN_PAGE_ERROR // ExceptionCode
465    mov edx, 3    // ParamCount
466    call InternalDispatchException
467    jmp PageFaultReturn
468
469AccessViolation:
470    /* Use more proper status code */
471    mov eax, KI_EXCEPTION_ACCESS_VIOLATION
472
473SpecialCode:
474    /* Setup a normal page fault exception */
475    mov edx, 2   // ParamCount
476    call InternalDispatchException
477
478PageFaultReturn:
479
480    /* Disable interrupts for the return */
481    cli
482
483    /* Return */
484    ExitTrap (TF_SAVE_ALL or TF_CHECKUSERAPC)
485ENDFUNC
486
487
488PUBLIC KiFloatingErrorFault
489FUNC KiFloatingErrorFault
490    /* Push pseudo error code */
491    EnterTrap TF_SAVE_ALL
492
493    UNIMPLEMENTED KiFloatingErrorFault
494    int 3
495
496    /* Return */
497    ExitTrap TF_SAVE_ALL
498ENDFUNC
499
500
501PUBLIC KiAlignmentFault
502FUNC KiAlignmentFault
503    /* We have an error code */
504    EnterTrap (TF_HAS_ERROR_CODE OR TF_SAVE_ALL)
505
506    /* Bugcheck */
507    Fatal EXCEPTION_ALIGNMENT_CHECK
508    jmp $
509ENDFUNC
510
511
512PUBLIC KiMcheckAbort
513FUNC KiMcheckAbort
514    /* Push pseudo error code */
515    EnterTrap TF_SAVE_ALL
516
517    /* Bugcheck */
518    Fatal HEX(12)
519    jmp $
520ENDFUNC
521
522
523PUBLIC KiXmmException
524FUNC KiXmmException
525    /* Push pseudo error code */
526    EnterTrap TF_SAVE_ALL
527
528    /* Call the C handler */
529    mov rcx, rbp
530    call KiXmmExceptionHandler
531
532    /* Check for success */
533    test eax, eax
534    jge KiXmmExit
535
536    /* Dispatch the exception */
537    DispatchException eax, 2, 0, [rbp+KTRAP_FRAME_MxCsr], 0
538
539    // FIXME: STATUS_FLOAT_MULTIPLE_TRAPS / STATUS_FLOAT_MULTIPLE_FAULTS
540
541KiXmmExit:
542    /* Return */
543    ExitTrap TF_SAVE_ALL
544ENDFUNC
545
546
547/* SOFTWARE INTERRUPT SERVICES ***********************************************/
548
549PUBLIC KiRaiseAssertion
550FUNC KiRaiseAssertion
551    /* We have an error code */
552    EnterTrap (TF_SAVE_ALL)
553
554    /* Decrement RIP to point to the INT2C instruction (2 bytes, not 1 like INT3) */
555    sub qword ptr [rbp + KTRAP_FRAME_Rip], 2
556
557    /* Dispatch the exception */
558    DispatchException STATUS_ASSERTION_FAILURE, 0, 0, 0, 0
559
560    /* Return */
561    ExitTrap TF_SAVE_ALL
562ENDFUNC
563
564
565PUBLIC KiDebugServiceTrap
566FUNC KiDebugServiceTrap
567   /* No error code */
568    EnterTrap TF_SAVE_ALL
569
570    /* Increase Rip to skip the int3 */
571    inc qword ptr [rbp + KTRAP_FRAME_Rip]
572
573    /* Dispatch the exception (Params = service, buffer, legth) */
574    DispatchException STATUS_BREAKPOINT, 3, [rbp+KTRAP_FRAME_Rax], [rbp+KTRAP_FRAME_Rcx], [rbp+KTRAP_FRAME_Rdx]
575
576    /* Return */
577    ExitTrap TF_SAVE_ALL
578ENDFUNC
579
580
581PUBLIC KiApcInterrupt
582.PROC KiApcInterrupt
583   /* No error code */
584    EnterTrap (TF_SAVE_ALL or TF_IRQL)
585
586    /* Raise to APC_LEVEL */
587    mov rax, APC_LEVEL
588    mov cr8, rax
589
590    /* End the interrupt */
591    mov dword ptr [APIC_EOI], 0
592
593    /* Enable interrupts  */
594    sti
595
596    /* Call the worker routine */
597    mov cl, [rbp + KTRAP_FRAME_SegCs] // ProcessorMode
598    and cl, 1
599    mov rdx, 0                        // ExceptionFrame
600    mov r8, rbp                       // TrapFrame
601    call KiDeliverApc
602
603    /* Disable interrupts */
604    cli
605
606    /* Lower IRQL back to PASSIVE */
607    mov rax, PASSIVE_LEVEL
608    mov cr8, rax
609
610    /* Return */
611    ExitTrap (TF_SAVE_ALL or TF_IRQL)
612.ENDP
613
614/*
615 * VOID
616 * KiRetireDpcList(
617 *     PKPRCB Prcb);
618 */
619EXTERN KiRetireDpcList:PROC
620
621/*
622 * VOID
623 * KiRetireDpcListInDpcStack(
624 *     PKPRCB Prcb,
625 *     PVOID DpcStack);
626 */
627PUBLIC KiRetireDpcListInDpcStack
628.PROC KiRetireDpcListInDpcStack
629    push rbp
630    .pushreg rbp
631    mov rbp, rsp
632    .setframe rbp, 0
633    .endprolog
634
635    /* Switch to the DpcStack */
636    mov rsp, rdx
637
638    /* The stack is 16 byte aligned, allocate 32 bytes home space */
639    sub rsp, 32
640
641    /* Call KiRetireDpcList on the given stack */
642    call KiRetireDpcList
643
644    /* Restore stack, cleanup and return */
645    mov rsp, rbp
646    pop rbp
647    ret
648.ENDP
649
650PUBLIC KiDpcInterrupt
651.PROC KiDpcInterrupt
652   /* No error code */
653    EnterTrap (TF_SAVE_ALL or TF_IRQL)
654
655    /* Call the worker routine */
656    call KiDpcInterruptHandler
657
658    /* Return, but don't send an EOI! */
659    ExitTrap (TF_SAVE_ALL or TF_IRQL)
660.ENDP
661
662
663PUBLIC KiIpiInterrupt
664.PROC KiIpiInterrupt
665   /* No error code */
666    EnterTrap (TF_SAVE_ALL or TF_IRQL)
667
668    /* Raise to IPI_LEVEL */
669    mov rax, IPI_LEVEL
670    mov cr8, rax
671
672    /* End the interrupt */
673    mov dword ptr [APIC_EOI], 0
674
675    int 3
676
677    /* Return */
678    ExitTrap (TF_SAVE_ALL or TF_IRQL)
679.ENDP
680
681
682PUBLIC KiUnexpectedInterrupt
683FUNC KiUnexpectedInterrupt
684    /* The error code is the vector */
685    EnterTrap (TF_HAS_ERROR_CODE OR TF_SAVE_ALL)
686
687#if 0
688    /* Set bugcheck parameters */
689    mov ecx, TRAP_CAUSE_UNKNOWN
690    mov rdx, [rbp + KTRAP_FRAME_ErrorCode] // the vector
691    mov r8, 0 // The unknown floating-point exception
692    mov r9, 0 // The enabled and asserted status bits
693    sub rsp, 8
694    mov [rbp + KTRAP_FRAME_P5 + 8], rbp // trap frame
695    call KeBugCheckWithTf
696    jmp $
697#endif
698    /* Return */
699    ExitTrap TF_SAVE_ALL
700ENDFUNC
701
702PUBLIC KiInterruptDispatch
703FUNC KiInterruptDispatch
704    /* The error code is a pointer to the interrupt object's code */
705    EnterTrap (TF_HAS_ERROR_CODE or TF_SAVE_ALL or TF_IRQL)
706
707    /* Increase interrupt count */
708    inc dword ptr gs:[PcInterruptCount];
709
710    /* Save rbx and rsi in the trap frame */
711    mov [rbp + KTRAP_FRAME_Rbx], rbx
712    mov [rbp + KTRAP_FRAME_Rsi], rsi
713
714    /* Load the address of the dispatch code into rbx */
715    mov rbx, [rbp + KTRAP_FRAME_ErrorCode]
716
717    /* Substract offset of the DispatchCode member plus 6 for the call instruction */
718    sub rbx, KINTERRUPT_DispatchCode + 6
719
720    /* Save the address of the InterruptListEntry in rsi */
721    lea rsi, [rbx + KINTERRUPT_InterruptListEntry]
722
723.DoDispatchInterrupt:
724    /* Raise IRQL to SynchronizeIrql */
725    movzx rax, byte ptr [rbx + KINTERRUPT_SynchronizeIrql]
726    mov cr8, rax
727
728#ifdef CONFIG_SMP
729    /* Acquire interrupt lock */
730    mov r8, [rbx + KINTERRUPT_ActualLock]
731
732    //KxAcquireSpinLock(Interrupt->ActualLock);
733#endif
734
735    /* Call the ISR */
736    mov rcx, rbx
737    mov rdx, [rbx + KINTERRUPT_ServiceContext]
738    call qword ptr [rbx + KINTERRUPT_ServiceRoutine]
739
740#ifdef CONFIG_SMP
741    /* Release interrupt lock */
742    //KxReleaseSpinLock(Interrupt->ActualLock);
743#endif
744
745    /* Go back to old irql */
746    movzx rax, byte ptr [rbp + KTRAP_FRAME_PreviousIrql]
747    mov cr8, rax
748
749    /* Check for chained interrupts */
750    mov rax, [rbx + KINTERRUPT_InterruptListEntry]
751    cmp rax, rsi
752    je .Done
753
754    /* Load the next interrupt object into rbx and repeat */
755    lea rbx, [rax - KINTERRUPT_InterruptListEntry]
756    jmp .DoDispatchInterrupt
757
758.Done:
759    /* Restore rbx and rsi */
760    mov rbx, [rbp + KTRAP_FRAME_Rbx]
761    mov rsi, [rbp + KTRAP_FRAME_Rsi]
762
763    /* Return */
764    ExitTrap (TF_SAVE_ALL or TF_SEND_EOI)
765ENDFUNC
766
767EXTERN KiSystemCallHandler:PROC
768
769/*! \name KiSystemCallEntry64
770 *
771 *  \brief This is the entrypoint for syscalls from 64bit user mode
772 *
773 *  \param rax - The system call number
774 *  \param rcx - User mode return address, set by the syscall instruction
775 *  \param rdx,r8,r9 - Parameters 2-4 to the service function
776 *  \param r10 - Parameter 1 to the service function
777 *  \param r11 - RFLAGS saved by the syscall instruction
778 *--*/
779PUBLIC KiSystemCallEntry64
780.PROC KiSystemCallEntry64
781    /* The unwind info pretends we have a machine frame */
782    .PUSHFRAME
783    .ALLOCSTACK (KTRAP_FRAME_LENGTH + MAX_SYSCALL_PARAM_SIZE - MachineFrameLength)
784    .ENDPROLOG
785
786    /* Swap gs to kernel, so we can access the PCR */
787    swapgs
788
789    /* Save the user mode rsp in the PCR */
790    mov gs:[PcUserRsp], rsp
791
792    /* Get the kernel stack from the PCR */
793    mov rsp, gs:[PcRspBase]
794
795    /* Allocate a TRAP_FRAME and space for parameters */
796    sub rsp, (KTRAP_FRAME_LENGTH + MAX_SYSCALL_PARAM_SIZE)
797
798    /* Save volatile registers in the trap frame */
799    mov [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rax], rax
800    mov [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rip], rcx
801    mov [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rdx], rdx
802    mov [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_R8], r8
803    mov [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_R9], r9
804    mov [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rcx], r10
805    mov [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_EFlags], r11
806
807    /* Store user stack pointer in the trap frame */
808    mov rax, gs:[PcUserRsp]
809    mov [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rsp], rax
810
811    /* Set sane segments */
812    mov ax, (KGDT64_R3_DATA or RPL_MASK)
813    mov ds, ax
814    mov es, ax
815
816    /* Save MCXSR and set kernel value */
817    stmxcsr [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_MxCsr]
818    ldmxcsr gs:[PcMxCsr]
819
820    /* Get the current thread and the trap frame */
821    mov rax, gs:[PcCurrentThread]
822    mov rcx, [rax + ThTrapFrame]
823
824    /* Save the old trap frame */
825    lea rdx, [rsp + MAX_SYSCALL_PARAM_SIZE]
826    mov [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_TrapFrame], rcx
827    mov [rax + ThTrapFrame], rdx
828
829#if DBG
830    /* Check IRQL */
831    mov rax, cr8
832    test eax, eax
833    jz KiSystemCall64Again
834    int HEX(2C)
835#endif
836
837GLOBAL_LABEL KiSystemCall64Again
838
839    /* Call the C-handler (will enable interrupts) */
840    call KiSystemCallHandler
841
842    /* The return value from KiSystemCallHandler is the address of the Nt-function */
843    mov rcx, [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rcx]
844    mov rdx, [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rdx]
845    mov r8, [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_R8]
846    mov r9, [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_R9]
847    call rax
848
849GLOBAL_LABEL KiSystemServiceExit
850
851    ASSERT_TRAP_FRAME_INTS_ENABLED rsp + MAX_SYSCALL_PARAM_SIZE
852
853    /* Check for pending user APCs */
854    mov rcx, gs:[PcCurrentThread]
855    cmp byte ptr [rcx + ThApcState + AsUserApcPending], 0
856    jz NoUserApcPending
857
858    /* Save missing regs in the trap frame */
859    mov [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rax], rax
860    mov [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rbp], rbp
861    mov [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_R9], rbp
862    mov rax, [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rsp]
863    mov [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_R8], rax
864    mov rax, [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rip]
865    mov [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rcx], rax
866    mov rax, [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_EFlags]
867    mov [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_R11], rax
868    xor rax, rax
869    mov [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rdx], rax
870    mov [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_R10], rax
871    pxor xmm0, xmm0
872    movdqa [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Xmm0], xmm0
873    movdqa [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Xmm1], xmm0
874    movdqa [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Xmm2], xmm0
875    movdqa [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Xmm3], xmm0
876    movdqa [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Xmm4], xmm0
877    movdqa [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Xmm5], xmm0
878
879    lea rcx, [rsp + MAX_SYSCALL_PARAM_SIZE]
880    call KiInitiateUserApc
881
882NoUserApcPending:
883    /* Disable interrupts for return */
884    cli
885
886    /* Restore MCXSR */
887    ldmxcsr [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_MxCsr]
888
889    /* Restore old trap frame */
890    mov rcx, gs:[PcCurrentThread]
891    mov rdx, [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_TrapFrame]
892    mov [rcx + KTHREAD_TrapFrame], rdx
893
894    /* Prepare user mode return address (rcx) and eflags (r11) for sysret */
895    mov rcx, [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rip]
896    mov r11, [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_EFlags]
897
898    /* Load user mode stack (It was copied to the trap frame) */
899    mov rsp, [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rsp]
900
901    /* r8 points to the user stack */
902    mov r8, rsp
903
904    /* r9 matches rbp */
905    mov r9, rbp
906
907    /* Swap gs back to user */
908    swapgs
909
910    /* Zero out volatiles */
911    pxor xmm0, xmm0
912    pxor xmm1, xmm1
913    pxor xmm2, xmm2
914    pxor xmm3, xmm3
915    pxor xmm4, xmm4
916    pxor xmm5, xmm5
917    xor rdx, rdx
918    xor r10, r10
919
920    /* return to user mode */
921    sysretq
922
923.ENDP
924
925
926/*!
927 * VOID
928 * DECLSPEC_NORETURN
929 * KiServiceExit(IN PKTRAP_FRAME TrapFrame, IN NTSTATUS Status));
930 */
931PUBLIC KiServiceExit
932.PROC KiServiceExit
933    .endprolog
934
935    lea rsp, [rcx - MAX_SYSCALL_PARAM_SIZE]
936    jmp KiSystemServiceExit
937
938.ENDP
939
940
941/*!
942 * VOID
943 * DECLSPEC_NORETURN
944 * KiServiceExit2(IN PKTRAP_FRAME TrapFrame);
945 */
946PUBLIC KiServiceExit2
947.PROC KiServiceExit2
948    .ENDPROLOG
949
950    // FIXME: this should probably also restore an exception frame
951
952    mov rsp, rcx
953.ENDP
954
955PUBLIC KiServiceExit3
956.PROC KiServiceExit3
957    .PUSHFRAME
958    .ALLOCSTACK (KTRAP_FRAME_LENGTH - MachineFrameLength)
959    .ENDPROLOG
960
961#if DBG
962    /* Get the current IRQL and compare it to the trap frame */
963    mov rax, cr8
964    cmp byte ptr [rsp + KTRAP_FRAME_PreviousIrql], al
965    je KiServiceExit2_ok1
966    int HEX(2C)
967
968KiServiceExit2_ok1:
969    /* Check if this is a user mode exit */
970    mov ah, byte ptr [rsp + KTRAP_FRAME_SegCs]
971    test ah, 1
972    jz KiServiceExit2_kernel
973
974    /* Validate that we are at PASSIVE_LEVEL */
975    test al, al
976    jz KiServiceExit2_kernel
977    int HEX(2C)
978
979KiServiceExit2_kernel:
980#endif
981
982    /* Return */
983    mov rbp, rsp
984    ExitTrap TF_SAVE_ALL
985.ENDP
986
987
988PUBLIC KiSystemCallEntry32
989KiSystemCallEntry32:
990    swapgs
991    int 3
992
993
994PUBLIC KiZwSystemService
995FUNC KiZwSystemService
996    push rbp
997    .pushreg rbp
998    sub rsp, KTRAP_FRAME_LENGTH
999    .allocstack KTRAP_FRAME_LENGTH
1000    mov [rsp + KTRAP_FRAME_Rsi], rsi
1001    .savereg rsi, KTRAP_FRAME_Rsi
1002    mov [rsp + KTRAP_FRAME_Rdi], rdi
1003    .savereg rdi, KTRAP_FRAME_Rdi
1004    mov rbp, rsp
1005    .setframe rbp, 0
1006    .endprolog
1007
1008    /* Get current thread */
1009    mov r11, gs:[PcCurrentThread]
1010
1011    /* Save PreviousMode in the trap frame */
1012    mov dil, byte ptr [r11 + KTHREAD_PreviousMode]
1013    mov byte ptr [rbp + KTRAP_FRAME_PreviousMode], dil
1014
1015    /* Save the old trap frame in TrapFrame.Rdx */
1016    mov rdi, [r11 + KTHREAD_TrapFrame]
1017    mov [rbp + KTRAP_FRAME_Rdx], rdi
1018
1019    /* Set the new trap frame and previous mode */
1020    mov [r11 + ThTrapFrame], rbp
1021    mov byte ptr [r11 + KTHREAD_PreviousMode], 0
1022
1023    /* allocate space for parameters */
1024    sub rsp, r10
1025    and rsp, HEX(0fffffffffffffff0)
1026
1027    /* Save rcx */
1028    mov [rbp + KTRAP_FRAME_Rcx], rcx
1029
1030    /* copy parameters to the new location */
1031    lea rsi, [rbp + KTRAP_FRAME_LENGTH + 16]
1032    lea rdi, [rsp]
1033    mov rcx, r10
1034    shr rcx, 3
1035    rep movsq
1036
1037    /* Restore rcx */
1038    mov rcx, [rbp + KTRAP_FRAME_Rcx]
1039
1040    /* Call the service function */
1041    call rax
1042
1043    /* Restore the old trap frame */
1044    mov r11, gs:[PcCurrentThread]
1045    mov rsi, [rbp + KTRAP_FRAME_Rdx]
1046    mov [r11 + KTHREAD_TrapFrame], rsi
1047
1048    /* Restore PreviousMode from the trap frame */
1049    mov dil, byte ptr [rbp + KTRAP_FRAME_PreviousMode]
1050    mov byte ptr [r11 + KTHREAD_PreviousMode], dil
1051
1052    /* Restore rdi and rsi */
1053    mov rsi, [rbp + KTRAP_FRAME_Rsi]
1054    mov rdi, [rbp + KTRAP_FRAME_Rdi]
1055
1056    /* Cleanup the stack and return */
1057    lea rsp, [rbp + KTRAP_FRAME_LENGTH]
1058    pop rbp
1059    ret
1060
1061ENDFUNC
1062
1063PUBLIC KiConvertToGuiThread
1064FUNC KiConvertToGuiThread
1065
1066    sub rsp, 40
1067    .allocstack 40
1068    .endprolog
1069
1070    /* Check if we already have a large stack */
1071    mov rax, gs:[PcCurrentThread]
1072    cmp byte ptr [rax + KTHREAD_LargeStack], 0
1073    jnz AlreadyLargeStack
1074
1075    // NewStack = (ULONG_PTR)MmCreateKernelStack(TRUE, 0);
1076    mov cl, 1
1077    xor rdx, rdx
1078    call MmCreateKernelStack
1079
1080    /* Check for failure */
1081    test rax, rax
1082    jz KiConvertToGuiThreadFailed
1083
1084    /* OldStack = KeSwitchKernelStack((PVOID)NewStack, (PVOID)(NewStack - KERNEL_LARGE_STACK_COMMIT )); */
1085    mov rcx, rax
1086    mov rdx, rax
1087    sub rdx, KERNEL_LARGE_STACK_COMMIT
1088    call KeSwitchKernelStack
1089
1090    // MmDeleteKernelStack(OldStack, FALSE);
1091    mov rcx, rax
1092    xor rdx, rdx
1093    call MmDeleteKernelStack
1094
1095AlreadyLargeStack:
1096
1097    /* Call the worker function */
1098    call PsConvertToGuiThread
1099
1100    /* Check for failure */
1101    test eax, eax
1102    js KiConvertToGuiThreadFailed
1103
1104    /* Disable interrupts for return */
1105    cli
1106
1107    // Restore register parameters
1108    mov rcx, [rsp + 48 + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rip]
1109    mov rdx, [rsp + 48 + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rdx]
1110    mov r8, [rsp + 48 + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_R8]
1111    mov r9, [rsp + 48 + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_R9]
1112
1113    /* Run KiSystemCallHandler again */
1114    add rsp, 48
1115    jmp KiSystemCall64Again
1116
1117KiConvertToGuiThreadFailed:
1118
1119    /* Clean up the stack and return failure */
1120    add rsp, 40
1121    mov eax, HEX(C0000017) // STATUS_NO_MEMORY
1122    ret
1123
1124ENDFUNC
1125
1126
1127EXTERN KiSetTrapContextInternal:PROC
1128
1129/*
1130 * VOID
1131 * KiSetTrapContext(
1132 *     _Out_ PKTRAP_FRAME TrapFrame,
1133 *     _In_ PCONTEXT Context,
1134 *     _In_ KPROCESSOR_MODE RequestorMode);
1135 */
1136PUBLIC KiSetTrapContext
1137.PROC KiSetTrapContext
1138
1139    /* Generate a KEXCEPTION_FRAME on the stack */
1140    GENERATE_EXCEPTION_FRAME
1141
1142    call KiSetTrapContextInternal
1143
1144    /* Restore the registers from the KEXCEPTION_FRAME */
1145    RESTORE_EXCEPTION_STATE
1146
1147    /* Return */
1148    ret
1149
1150.ENDP
1151
1152
1153/*
1154 * VOID
1155 * KiDeliverApc(
1156 *     _In_ KPROCESSOR_MODE DeliveryMode,
1157 *     _In_ PKEXCEPTION_FRAME ExceptionFrame,
1158 *     _In_ PKTRAP_FRAME TrapFrame);
1159 *
1160 */
1161EXTERN KiDeliverApc:PROC
1162
1163/*
1164 * VOID
1165 * KiInitiateUserApc(
1166 *     _In_ PKTRAP_FRAME TrapFrame@<rcx>);
1167 *
1168 * This function is called to deliver user mode APCs.
1169 * It clobbers all non-volatile registers, except rax.
1170 */
1171PUBLIC KiInitiateUserApc
1172.PROC KiInitiateUserApc
1173
1174    /* Generate a KEXCEPTION_FRAME on the stack */
1175    GENERATE_EXCEPTION_FRAME
1176
1177    /* Raise IRQL to APC_LEVEL */
1178    mov rax, APC_LEVEL
1179    mov cr8, rax
1180
1181    /* Get the current thread */
1182    mov rbp, gs:[PcCurrentThread]
1183
1184    /* Save the trap frame in rsi */
1185    mov rsi, rcx
1186
1187    /* Enable interrupts */
1188    sti
1189
1190    /* Call the C function */
1191    mov ecx, 1
1192    mov rdx, rsp
1193    mov r8, rsi
1194    call KiDeliverApc
1195
1196    /* Disable interrupts again */
1197    cli
1198
1199    /* Go back to PASSIVE_LEVEL */
1200    mov rax, PASSIVE_LEVEL
1201    mov cr8, rax
1202
1203    /* Restore the registers from the KEXCEPTION_FRAME */
1204    RESTORE_EXCEPTION_STATE
1205
1206    /* Return */
1207    ret
1208
1209.ENDP
1210
1211
1212PUBLIC KiInitializeSegments
1213KiInitializeSegments:
1214    mov ax, KGDT64_R3_DATA or RPL_MASK
1215    mov gs, ax
1216    swapgs
1217    mov gs, ax
1218    ret
1219
1220/*!
1221 * VOID
1222 * KiSwitchKernelStackHelper(
1223 *     LONG_PTR StackOffset,
1224 *     PVOID OldStackBase);
1225 */
1226PUBLIC KiSwitchKernelStackHelper
1227KiSwitchKernelStackHelper:
1228
1229    /* Pop return address from the current stack */
1230    pop rax
1231
1232    /* Switch to new stack */
1233    lea rsp, [rsp + rcx]
1234
1235    /* Push return address on the new stack */
1236    push rax
1237
1238    /* Return on new stack */
1239    mov rax, rdx
1240    ret
1241
1242EXTERN KiSwitchKernelStack:PROC
1243
1244PUBLIC KeSwitchKernelStack
1245FUNC KeSwitchKernelStack
1246
1247    /* Save rcx and allocate callee home space */
1248    mov [rsp + P1Home], rcx
1249    .savereg rcx, P1Home
1250    sub rsp, 40
1251    .allocstack 40
1252    .endprolog
1253
1254    /* Call the C handler, which returns the old stack in rax */
1255    call KiSwitchKernelStack
1256
1257    /* Restore rcx (StackBase) */
1258    mov rcx, [rsp + 40 + P1Home]
1259
1260    /* Switch to new stack: RSP += (StackBase - OldStackBase) */
1261    sub rcx, rax
1262    add rsp, rcx
1263
1264    /* Deallocate the home frame */
1265    add rsp, 40
1266    ret
1267
1268ENDFUNC
1269
1270
1271
1272
1273#ifdef _MSC_VER
1274#undef lgdt
1275#undef lidt
1276
1277//void __lgdt(void *Source);
1278PUBLIC __lgdt
1279__lgdt:
1280    lgdt fword ptr [rcx]
1281    ret
1282
1283//void __sgdt(void *Destination);
1284PUBLIC __sgdt
1285__sgdt:
1286    sgdt fword ptr [rcx]
1287    ret
1288
1289// void __lldt(unsigned short Value)
1290PUBLIC __lldt
1291__lldt:
1292    lldt cx
1293    ret
1294
1295//void __sldt(void *Destination);
1296PUBLIC __sldt
1297__sldt:
1298    sldt word ptr [rcx]
1299    ret
1300
1301//void __ltr(unsigned short Source);
1302PUBLIC __ltr
1303__ltr:
1304    ltr cx
1305    ret
1306
1307//void __str(unsigned short *Destination);
1308PUBLIC __str
1309__str:
1310    str word ptr [rcx]
1311    ret
1312
1313PUBLIC __swapgs
1314__swapgs:
1315    swapgs
1316    ret
1317
1318#endif
1319
1320END
1321