xref: /reactos/ntoskrnl/ke/amd64/trap.S (revision 43067402) 
1/*
2 * FILE:            ntoskrnl/ke/amd64/trap.S
3 * COPYRIGHT:       See COPYING in the top level directory
4 * PURPOSE:         System Traps, Entrypoints and Exitpoints
5 * PROGRAMMER:      Timo Kreuzer (timo.kreuzer@reactos.org)
6 */
7
8/* INCLUDES ******************************************************************/
9
10#include <asm.inc>
11#include <ksamd64.inc>
12#include <trapamd64.inc>
13
14EXTERN KiDispatchException:PROC
15EXTERN KeBugCheckWithTf:PROC
16EXTERN MmAccessFault:PROC
17EXTERN KiSystemFatalException:PROC
18EXTERN KiNpxNotAvailableFaultHandler:PROC
19EXTERN KiGeneralProtectionFaultHandler:PROC
20EXTERN KiXmmExceptionHandler:PROC
21EXTERN KiDeliverApc:PROC
22EXTERN KiDpcInterruptHandler:PROC
23EXTERN PsConvertToGuiThread:PROC
24EXTERN MmCreateKernelStack:PROC
25EXTERN MmDeleteKernelStack:PROC
26
27EXTERN KdSetOwedBreakpoints:PROC
28
29
30/* Helper Macros *************************************************************/
31
32MACRO(DispatchException, Status, Number, P1, P2, P3)
33    mov eax, Status
34    mov edx, Number
35    mov r9, P1
36    mov r10, P2
37    mov r11, P3
38    call InternalDispatchException
39ENDM
40
41MACRO(Fatal, BugcheckCode)
42    /* Bugcheck */
43    mov ecx, BugcheckCode
44    mov rdx, rbp
45    call KiSystemFatalException
46ENDM
47
48
49/* FUNCTIONS *****************************************************************/
50
51.code64
52
53ALIGN 8
54
55MACRO(UnexpectedVectorStub, Vector)
56    /* This nop is to make the relative jmp address 4 bytes aligned and to
57       make the whole code 8 bytes long */
58    nop
59    /* This is a push instruction with 8bit operand. Since the instruction
60       sign extends the value to 32 bits, we need to offset it */
61PUBLIC KxUnexpectedInterrupt&Vector
62KxUnexpectedInterrupt&Vector:
63    push (Vector - 128)
64    jmp KiUnexpectedInterrupt
65ENDM
66
67PUBLIC KiUnexpectedRange
68KiUnexpectedRange:
69Vector = 0
70REPEAT 256
71    UnexpectedVectorStub %Vector
72    Vector = Vector+1
73ENDR
74PUBLIC KiUnexpectedRangeEnd
75KiUnexpectedRangeEnd:
76
77PUBLIC KiInterruptDispatchTemplate
78KiInterruptDispatchTemplate:
79    /* This instruction pushes the return address on the stack, which is the
80       address of the interrupt object's DispatchCode member, then jumps
81       to the address stored in the interrupt object's DispatchAddress member */
82    call qword ptr KiInterruptDispatchTemplate[rip - KINTERRUPT_DispatchCode + KINTERRUPT_DispatchAddress]
83
84
85// rbp = TrapFrame, eax = ExceptionCode, edx = NumParams, r9,r10,r11 = params
86FUNC InternalDispatchException
87
88    /* Allocate stack space for EXCEPTION_RECORD and KEXCEPTION_FRAME */
89    sub rsp, EXCEPTION_RECORD_LENGTH + KEXCEPTION_FRAME_LENGTH
90    .allocstack (EXCEPTION_RECORD_LENGTH + KEXCEPTION_FRAME_LENGTH)
91    .endprolog
92
93    /* Set up EXCEPTION_RECORD */
94    lea rcx, [rsp + KEXCEPTION_FRAME_LENGTH]
95    mov [rcx + EXCEPTION_RECORD_ExceptionCode], eax
96    xor rax, rax
97    mov [rcx + EXCEPTION_RECORD_ExceptionFlags], eax
98    mov [rcx + EXCEPTION_RECORD_ExceptionRecord], rax
99    mov rax, [rbp + KTRAP_FRAME_Rip]
100    mov [rcx + EXCEPTION_RECORD_ExceptionAddress], rax
101    mov [rcx + EXCEPTION_RECORD_NumberParameters], edx
102    mov [rcx + EXCEPTION_RECORD_ExceptionInformation + HEX(00)], r9
103    mov [rcx + EXCEPTION_RECORD_ExceptionInformation + HEX(08)], r10
104    mov [rcx + EXCEPTION_RECORD_ExceptionInformation + HEX(10)], r11
105
106    /* Set up KEXCEPTION_FRAME */
107    mov rax, [rbp + KTRAP_FRAME_Rbp]
108    mov [rsp + KEXCEPTION_FRAME_Rbp], rax
109    mov [rsp + KEXCEPTION_FRAME_Rbx], rbx
110    mov [rsp + KEXCEPTION_FRAME_Rdi], rdi
111    mov [rsp + KEXCEPTION_FRAME_Rsi], rsi
112    mov [rsp + KEXCEPTION_FRAME_R12], r12
113    mov [rsp + KEXCEPTION_FRAME_R13], r13
114    mov [rsp + KEXCEPTION_FRAME_R14], r14
115    mov [rsp + KEXCEPTION_FRAME_R15], r15
116    movdqa [rsp + KEXCEPTION_FRAME_Xmm6], xmm6
117    movdqa [rsp + KEXCEPTION_FRAME_Xmm7], xmm7
118    movdqa [rsp + KEXCEPTION_FRAME_Xmm8], xmm8
119    movdqa [rsp + KEXCEPTION_FRAME_Xmm9], xmm9
120    movdqa [rsp + KEXCEPTION_FRAME_Xmm10], xmm10
121    movdqa [rsp + KEXCEPTION_FRAME_Xmm11], xmm11
122    movdqa [rsp + KEXCEPTION_FRAME_Xmm12], xmm12
123    movdqa [rsp + KEXCEPTION_FRAME_Xmm13], xmm13
124    movdqa [rsp + KEXCEPTION_FRAME_Xmm14], xmm14
125    movdqa [rsp + KEXCEPTION_FRAME_Xmm15], xmm15
126    mov qword ptr [rsp + KEXCEPTION_FRAME_Return], 0
127
128    /* Call KiDispatchException */
129    // rcx already points to ExceptionRecord
130    mov rdx, rsp                                // ExceptionFrame
131    mov r8, rbp                                 // TrapFrame
132    mov r9b, [r8 + KTRAP_FRAME_PreviousMode]    // PreviousMode
133    mov byte ptr [rsp + KEXCEPTION_FRAME_P5], 1 // FirstChance
134    call KiDispatchException
135
136    /* Restore registers */
137    mov r12, [rsp + KEXCEPTION_FRAME_R12]
138    mov r13, [rsp + KEXCEPTION_FRAME_R13]
139    mov r14, [rsp + KEXCEPTION_FRAME_R14]
140    mov r15, [rsp + KEXCEPTION_FRAME_R15]
141    movdqa xmm6, [rsp + KEXCEPTION_FRAME_Xmm6]
142    movdqa xmm7, [rsp + KEXCEPTION_FRAME_Xmm7]
143    movdqa xmm8, [rsp + KEXCEPTION_FRAME_Xmm8]
144    movdqa xmm9, [rsp + KEXCEPTION_FRAME_Xmm9]
145    movdqa xmm10, [rsp + KEXCEPTION_FRAME_Xmm10]
146    movdqa xmm11, [rsp + KEXCEPTION_FRAME_Xmm11]
147    movdqa xmm12, [rsp + KEXCEPTION_FRAME_Xmm12]
148    movdqa xmm13, [rsp + KEXCEPTION_FRAME_Xmm13]
149    movdqa xmm14, [rsp + KEXCEPTION_FRAME_Xmm14]
150    movdqa xmm15, [rsp + KEXCEPTION_FRAME_Xmm15]
151
152    add rsp, EXCEPTION_RECORD_LENGTH + KEXCEPTION_FRAME_LENGTH
153    ret
154ENDFUNC
155
156
157/* CPU EXCEPTION HANDLERS ****************************************************/
158
159PUBLIC KiDivideErrorFault
160FUNC KiDivideErrorFault
161    /* Push pseudo error code */
162    EnterTrap TF_SAVE_ALL
163
164    /* Enable interrupts */
165    sti
166
167    /* Dispatch the exception */
168    DispatchException STATUS_INTEGER_DIVIDE_BY_ZERO, 0, 0, 0, 0
169
170    /* Return */
171    ExitTrap TF_SAVE_ALL
172ENDFUNC
173
174
175PUBLIC KiDebugTrapOrFault
176FUNC KiDebugTrapOrFault
177    /* Push pseudo error code */
178    EnterTrap TF_SAVE_ALL
179
180    /* Check if the frame was from kernelmode */
181    test word ptr [rbp + KTRAP_FRAME_SegCs], 3
182    jz KiDebugTrapOrFaultKMode
183
184    /* Enable interrupts for user-mode */
185    sti
186
187KiDebugTrapOrFaultKMode:
188    /* Dispatch the exception */
189    DispatchException STATUS_SINGLE_STEP, 0, 0, 0, 0
190
191    /* Return */
192    ExitTrap TF_SAVE_ALL
193ENDFUNC
194
195
196PUBLIC KiNmiInterrupt
197FUNC KiNmiInterrupt
198    /* Push pseudo error code */
199    EnterTrap TF_SAVE_ALL
200
201    UNIMPLEMENTED KiNmiInterrupt
202    int 3
203
204    /* Return */
205    ExitTrap TF_SAVE_ALL
206ENDFUNC
207
208
209PUBLIC KiBreakpointTrap
210FUNC KiBreakpointTrap
211    /* Push pseudo error code */
212    EnterTrap TF_SAVE_ALL
213
214    /* Check if the frame was from kernelmode */
215    test word ptr [rbp + KTRAP_FRAME_SegCs], 3
216    jz KiBreakpointTrapKMode
217
218    /* Enable interrupts for user-mode */
219    sti
220
221KiBreakpointTrapKMode:
222    /* Dispatch the exception */
223    DispatchException STATUS_BREAKPOINT, 3, BREAKPOINT_BREAK, 0, 0
224
225    /* Return */
226    ExitTrap TF_SAVE_ALL
227ENDFUNC
228
229
230PUBLIC KiOverflowTrap
231FUNC KiOverflowTrap
232    /* Push pseudo error code */
233    EnterTrap TF_SAVE_ALL
234
235    /* Enable interrupts */
236    sti
237
238    /* Dispatch the exception */
239    DispatchException STATUS_INTEGER_OVERFLOW, 3, 0, 0, 0
240
241    /* Return */
242    ExitTrap TF_SAVE_ALL
243ENDFUNC
244
245
246PUBLIC KiBoundFault
247FUNC KiBoundFault
248    /* Push pseudo error code */
249    EnterTrap TF_SAVE_ALL
250
251    /* Check if the frame was from kernelmode */
252    test word ptr [rbp + KTRAP_FRAME_SegCs], 3
253    jnz KiBoundFaultUserMode
254
255    /* Bugcheck */
256    Fatal EXCEPTION_BOUND_CHECK
257
258KiBoundFaultUserMode:
259    /* Enable interrupts for user-mode */
260    sti
261
262    /* Dispatch the exception */
263    DispatchException STATUS_ARRAY_BOUNDS_EXCEEDED, 0, 0, 0, 0
264
265    /* Return */
266    ExitTrap TF_SAVE_ALL
267ENDFUNC
268
269
270PUBLIC KiInvalidOpcodeFault
271FUNC KiInvalidOpcodeFault
272    /* Push pseudo error code */
273    EnterTrap TF_SAVE_ALL
274
275    /* Enable interrupts */
276    sti
277
278    /* Check if the frame was from kernelmode */
279    test word ptr [rbp + KTRAP_FRAME_SegCs], 3
280    jz KiInvalidOpcodeKernel
281
282    // FIXME: handle STATUS_INVALID_LOCK_SEQUENCE
283
284KiInvalidOpcodeKernel:
285    /* Kernel mode fault */
286
287    /* Dispatch the exception */
288    DispatchException STATUS_ILLEGAL_INSTRUCTION, 0, 0, 0, 0
289
290    /* Return */
291    ExitTrap TF_SAVE_ALL
292ENDFUNC
293
294
295PUBLIC KiNpxNotAvailableFault
296FUNC KiNpxNotAvailableFault
297    /* Push pseudo error code */
298    EnterTrap TF_SAVE_ALL
299
300    /* Call the C handler */
301    mov rcx, rbp
302    call KiNpxNotAvailableFaultHandler
303
304    /* Check the return status code */
305    test eax, eax
306    jz KiNpxNotAvailableFaultExit
307
308    /* Dispatch the exception */
309    DispatchException eax, 3, 0, 0, 0
310
311KiNpxNotAvailableFaultExit:
312    /* Return */
313    ExitTrap TF_SAVE_ALL
314ENDFUNC
315
316
317PUBLIC KiDoubleFaultAbort
318FUNC KiDoubleFaultAbort
319
320    /* Hack for VBox, which "forgets" to push an error code on the stack! */
321    and rsp, HEX(FFFFFFFFFFFFFFF0)
322
323    /* A zero error code is pushed */
324    EnterTrap (TF_HAS_ERROR_CODE OR TF_SAVE_ALL)
325
326    int 3
327
328    /* Bugcheck */
329    Fatal 8 // EXCEPTION_DOUBLE_FAULT
330    jmp $
331ENDFUNC
332
333
334PUBLIC KiNpxSegmentOverrunAbort
335FUNC KiNpxSegmentOverrunAbort
336    /* Push pseudo error code */
337    EnterTrap TF_SAVE_ALL
338
339    /* Bugcheck */
340    Fatal EXCEPTION_NPX_OVERRUN
341
342    jmp $
343ENDFUNC
344
345
346PUBLIC KiInvalidTssFault
347FUNC KiInvalidTssFault
348    /* We have an error code */
349    EnterTrap (TF_HAS_ERROR_CODE OR TF_SAVE_ALL)
350
351    /* Bugcheck */
352    Fatal EXCEPTION_INVALID_TSS
353    jmp $
354ENDFUNC
355
356
357PUBLIC KiSegmentNotPresentFault
358FUNC KiSegmentNotPresentFault
359    /* We have an error code */
360    EnterTrap (TF_HAS_ERROR_CODE OR TF_SAVE_ALL)
361
362    /* Bugcheck */
363    Fatal EXCEPTION_SEGMENT_NOT_PRESENT
364    jmp $
365ENDFUNC
366
367
368PUBLIC KiStackFault
369FUNC KiStackFault
370    /* We have an error code */
371    EnterTrap (TF_HAS_ERROR_CODE OR TF_SAVE_ALL)
372
373    /* Bugcheck */
374    Fatal EXCEPTION_STACK_FAULT
375    jmp $
376ENDFUNC
377
378
379PUBLIC KiGeneralProtectionFault
380FUNC KiGeneralProtectionFault
381    /* We have an error code */
382    EnterTrap (TF_HAS_ERROR_CODE OR TF_SAVE_ALL)
383
384    /* Call the C handler */
385    mov rcx, rbp
386    call KiGeneralProtectionFaultHandler
387
388    /* Check for success */
389    test eax, eax
390    jge KiGpfExit
391
392    /* Check for access violation */
393    cmp eax, STATUS_ACCESS_VIOLATION
394    je DispatchAccessViolation
395
396    /* Dispatch privileged instruction fault */
397    DispatchException eax, 0, 0, 0, 0
398    jmp KiGpfFatal
399
400DispatchAccessViolation:
401
402    /* Dispatch access violation */
403    DispatchException eax, 2, 0, -1, 0
404
405KiGpfFatal:
406
407    /* Bugcheck */
408    mov ecx, UNEXPECTED_KERNEL_MODE_TRAP
409    mov rdx, HEX(000D) // EXCEPTION_GP_FAULT
410    xor r8, r8
411    mov r9, [rbp + KTRAP_FRAME_ErrorCode] // error code
412    sub rsp, 8
413    mov [rsp + KTRAP_FRAME_P5+8], rbp // trap frame
414    call KeBugCheckWithTf
415
416KiGpfExit:
417    /* Return */
418    /* Return */
419    ExitTrap TF_SAVE_ALL
420ENDFUNC
421
422
423PUBLIC KiPageFault
424FUNC KiPageFault
425    /* We have an error code */
426    EnterTrap (TF_HAS_ERROR_CODE OR TF_SAVE_ALL)
427
428    /* Save page fault address */
429    mov rdx, cr2
430    mov [rbp  + KTRAP_FRAME_FaultAddress], rdx
431
432    /* If interrupts are off, treat this as an access violation */
433    test dword ptr [rbp + KTRAP_FRAME_EFlags], EFLAGS_IF_MASK
434    jz AccessViolation
435
436    /* Enable interrupts for the page fault handler */
437    sti
438
439    /* Call page fault handler */
440    mov ecx, [rbp + KTRAP_FRAME_ErrorCode] // FaultCode
441    // rdx == Address
442    mov r8b, [rbp + KTRAP_FRAME_SegCs] // Mode
443    and r8b, 1
444    mov r9, rbp // TrapInformation
445    call MmAccessFault
446
447    /* Check for success */
448    test eax, eax
449    jl PageFaultError
450
451    /* Check whether the kernel debugger has owed breakpoints to be inserted */
452    call KdSetOwedBreakpoints
453    /* We succeeded, return */
454    jmp PageFaultReturn
455
456PageFaultError:
457
458    /* Disable interrupts again for the debugger */
459    cli
460
461    /* Set parameter 1 to error code */
462    mov r9d, [rbp + KTRAP_FRAME_ErrorCode]
463
464    /* Set parameter2 to faulting address */
465    mov r10, cr2  // Param2 = faulting address
466
467    cmp eax, STATUS_ACCESS_VIOLATION
468    je AccessViolation
469    cmp eax, STATUS_GUARD_PAGE_VIOLATION
470    je SpecialCode
471    cmp eax, STATUS_STACK_OVERFLOW
472    je SpecialCode
473
474InPageException:
475    /* Dispatch in-page exception */
476    mov r11d, eax // Param3 = Status
477    mov eax, STATUS_IN_PAGE_ERROR // ExceptionCode
478    mov edx, 3    // ParamCount
479    call InternalDispatchException
480    jmp PageFaultReturn
481
482AccessViolation:
483    /* Use more proper status code */
484    mov eax, KI_EXCEPTION_ACCESS_VIOLATION
485
486SpecialCode:
487    /* Setup a normal page fault exception */
488    mov edx, 2   // ParamCount
489    call InternalDispatchException
490
491PageFaultReturn:
492    /* Return */
493    ExitTrap (TF_SAVE_ALL or TF_CHECKUSERAPC)
494ENDFUNC
495
496
497PUBLIC KiFloatingErrorFault
498FUNC KiFloatingErrorFault
499    /* Push pseudo error code */
500    EnterTrap TF_SAVE_ALL
501
502    UNIMPLEMENTED KiFloatingErrorFault
503    int 3
504
505    /* Return */
506    ExitTrap TF_SAVE_ALL
507ENDFUNC
508
509
510PUBLIC KiAlignmentFault
511FUNC KiAlignmentFault
512    /* We have an error code */
513    EnterTrap (TF_HAS_ERROR_CODE OR TF_SAVE_ALL)
514
515    /* Bugcheck */
516    Fatal EXCEPTION_ALIGNMENT_CHECK
517    jmp $
518ENDFUNC
519
520
521PUBLIC KiMcheckAbort
522FUNC KiMcheckAbort
523    /* Push pseudo error code */
524    EnterTrap TF_SAVE_ALL
525
526    /* Bugcheck */
527    Fatal HEX(12)
528    jmp $
529ENDFUNC
530
531
532PUBLIC KiXmmException
533FUNC KiXmmException
534    /* Push pseudo error code */
535    EnterTrap TF_SAVE_ALL
536
537    /* Call the C handler */
538    mov rcx, rbp
539    call KiXmmExceptionHandler
540
541    /* Check for success */
542    test eax, eax
543    jge KiXmmExit
544
545    /* Dispatch the exception */
546    DispatchException eax, 2, 0, [rbp+KTRAP_FRAME_MxCsr], 0
547
548    // FIXME: STATUS_FLOAT_MULTIPLE_TRAPS / STATUS_FLOAT_MULTIPLE_FAULTS
549
550KiXmmExit:
551    /* Return */
552    ExitTrap TF_SAVE_ALL
553ENDFUNC
554
555
556/* SOFTWARE INTERRUPT SERVICES ***********************************************/
557
558PUBLIC KiRaiseAssertion
559FUNC KiRaiseAssertion
560    /* We have an error code */
561    EnterTrap (TF_SAVE_ALL)
562
563    /* Decrement RIP to point to the INT2C instruction (2 bytes, not 1 like INT3) */
564    sub qword ptr [rbp + KTRAP_FRAME_Rip], 2
565
566    /* Dispatch the exception */
567    DispatchException STATUS_ASSERTION_FAILURE, 0, 0, 0, 0
568
569    /* Return */
570    ExitTrap TF_SAVE_ALL
571ENDFUNC
572
573
574PUBLIC KiDebugServiceTrap
575FUNC KiDebugServiceTrap
576   /* No error code */
577    EnterTrap TF_SAVE_ALL
578
579    /* Increase Rip to skip the int3 */
580    inc qword ptr [rbp + KTRAP_FRAME_Rip]
581
582    /* Dispatch the exception (Params = service, buffer, legth) */
583    DispatchException STATUS_BREAKPOINT, 3, [rbp+KTRAP_FRAME_Rax], [rbp+KTRAP_FRAME_Rcx], [rbp+KTRAP_FRAME_Rdx]
584
585    /* Return */
586    ExitTrap TF_SAVE_ALL
587ENDFUNC
588
589
590PUBLIC KiApcInterrupt
591.PROC KiApcInterrupt
592   /* No error code */
593    EnterTrap (TF_VOLATILES or TF_IRQL)
594
595    /* Raise to APC_LEVEL */
596    mov rax, APC_LEVEL
597    mov cr8, rax
598
599    /* End the interrupt */
600    mov dword ptr [APIC_EOI], 0
601
602    /* Enable interrupts  */
603    sti
604
605    /* Call the worker routine */
606    mov cl, [rbp + KTRAP_FRAME_SegCs] // ProcessorMode
607    and cl, 1
608    mov rdx, 0                        // ExceptionFrame
609    mov r8, rbp                       // TrapFrame
610    call KiDeliverApc
611
612    /* Disable interrupts */
613    cli
614
615    /* Lower IRQL back to PASSIVE */
616    mov rax, PASSIVE_LEVEL
617    mov cr8, rax
618
619    /* Return */
620    ExitTrap (TF_VOLATILES or TF_IRQL)
621.ENDP
622
623/*
624 * VOID
625 * KiRetireDpcList(
626 *     PKPRCB Prcb);
627 */
628EXTERN KiRetireDpcList:PROC
629
630/*
631 * VOID
632 * KiRetireDpcListInDpcStack(
633 *     PKPRCB Prcb,
634 *     PVOID DpcStack);
635 */
636PUBLIC KiRetireDpcListInDpcStack
637.PROC KiRetireDpcListInDpcStack
638    push rbp
639    .pushreg rbp
640    mov rbp, rsp
641    .setframe rbp, 0
642    .endprolog
643
644    /* Switch to the DpcStack */
645    mov rsp, rdx
646
647    /* The stack is 16 byte aligned, allocate 32 bytes home space */
648    sub rsp, 32
649
650    /* Call KiRetireDpcList on the given stack */
651    call KiRetireDpcList
652
653    /* Restore stack, cleanup and return */
654    mov rsp, rbp
655    pop rbp
656    ret
657.ENDP
658
659PUBLIC KiDpcInterrupt
660.PROC KiDpcInterrupt
661   /* No error code */
662    EnterTrap (TF_VOLATILES or TF_IRQL)
663
664    /* Call the worker routine */
665    call KiDpcInterruptHandler
666
667    /* Return, but don't send an EOI! */
668    ExitTrap (TF_VOLATILES or TF_IRQL)
669.ENDP
670
671
672PUBLIC KiIpiInterrupt
673.PROC KiIpiInterrupt
674   /* No error code */
675    EnterTrap (TF_VOLATILES or TF_IRQL)
676
677    /* Raise to IPI_LEVEL */
678    mov rax, IPI_LEVEL
679    mov cr8, rax
680
681    /* End the interrupt */
682    mov dword ptr [APIC_EOI], 0
683
684    int 3
685
686    /* Return */
687    ExitTrap (TF_VOLATILES or TF_IRQL)
688.ENDP
689
690
691PUBLIC KiUnexpectedInterrupt
692FUNC KiUnexpectedInterrupt
693    /* The error code is the vector */
694    EnterTrap (TF_HAS_ERROR_CODE OR TF_SAVE_ALL)
695
696#if 0
697    /* Set bugcheck parameters */
698    mov ecx, TRAP_CAUSE_UNKNOWN
699    mov rdx, [rbp + KTRAP_FRAME_ErrorCode] // the vector
700    mov r8, 0 // The unknown floating-point exception
701    mov r9, 0 // The enabled and asserted status bits
702    sub rsp, 8
703    mov [rbp + KTRAP_FRAME_P5 + 8], rbp // trap frame
704    call KeBugCheckWithTf
705    jmp $
706#endif
707    /* Return */
708    ExitTrap TF_SAVE_ALL
709ENDFUNC
710
711PUBLIC KiInterruptDispatch
712FUNC KiInterruptDispatch
713    /* The error code is a pointer to the interrupt object's code */
714    EnterTrap (TF_HAS_ERROR_CODE or TF_SAVE_ALL or TF_IRQL)
715
716    /* Increase interrupt count */
717    inc dword ptr gs:[PcInterruptCount];
718
719    /* Save rbx and rsi in the trap frame */
720    mov [rbp + KTRAP_FRAME_Rbx], rbx
721    mov [rbp + KTRAP_FRAME_Rsi], rsi
722
723    /* Load the address of the dispatch code into rbx */
724    mov rbx, [rbp + KTRAP_FRAME_ErrorCode]
725
726    /* Substract offset of the DispatchCode member plus 6 for the call instruction */
727    sub rbx, KINTERRUPT_DispatchCode + 6
728
729    /* Save the address of the InterruptListEntry in rsi */
730    lea rsi, [rbx + KINTERRUPT_InterruptListEntry]
731
732.DoDispatchInterrupt:
733    /* Raise IRQL to SynchronizeIrql */
734    movzx rax, byte ptr [rbx + KINTERRUPT_SynchronizeIrql]
735    mov cr8, rax
736
737#ifdef CONFIG_SMP
738    /* Acquire interrupt lock */
739    mov r8, [rbx + KINTERRUPT_ActualLock]
740
741    //KxAcquireSpinLock(Interrupt->ActualLock);
742#endif
743
744    /* Call the ISR */
745    mov rcx, rbx
746    mov rdx, [rbx + KINTERRUPT_ServiceContext]
747    call qword ptr [rbx + KINTERRUPT_ServiceRoutine]
748
749#ifdef CONFIG_SMP
750    /* Release interrupt lock */
751    //KxReleaseSpinLock(Interrupt->ActualLock);
752#endif
753
754    /* Go back to old irql */
755    movzx rax, byte ptr [rbp + KTRAP_FRAME_PreviousIrql]
756    mov cr8, rax
757
758    /* Check for chained interrupts */
759    mov rax, [rbx + KINTERRUPT_InterruptListEntry]
760    cmp rax, rsi
761    je .Done
762
763    /* Load the next interrupt object into rbx and repeat */
764    lea rbx, [rax - KINTERRUPT_InterruptListEntry]
765    jmp .DoDispatchInterrupt
766
767.Done:
768    /* Restore rbx and rsi */
769    mov rbx, [rbp + KTRAP_FRAME_Rbx]
770    mov rsi, [rbp + KTRAP_FRAME_Rsi]
771
772    /* Return */
773    ExitTrap (TF_SAVE_ALL or TF_SEND_EOI)
774ENDFUNC
775
776EXTERN KiSystemCallHandler:PROC
777
778/*! \name KiSystemCallEntry64
779 *
780 *  \brief This is the entrypoint for syscalls from 64bit user mode
781 *
782 *  \param rax - The system call number
783 *  \param rcx - User mode return address, set by the syscall instruction
784 *  \param rdx,r8,r9 - Parameters 2-4 to the service function
785 *  \param r10 - Parameter 1 to the service function
786 *  \param r11 - RFLAGS saved by the syscall instruction
787 *--*/
788PUBLIC KiSystemCallEntry64
789.PROC KiSystemCallEntry64
790    /* The unwind info pretends we have a machine frame */
791    .PUSHFRAME
792    .ALLOCSTACK (KTRAP_FRAME_LENGTH + MAX_SYSCALL_PARAM_SIZE - MachineFrameLength)
793    .ENDPROLOG
794
795    /* Swap gs to kernel, so we can access the PCR */
796    swapgs
797
798    /* Save the user mode rsp in the PCR */
799    mov gs:[PcUserRsp], rsp
800
801    /* Get the kernel stack from the PCR */
802    mov rsp, gs:[PcRspBase]
803
804    /* Allocate a TRAP_FRAME and space for parameters */
805    sub rsp, (KTRAP_FRAME_LENGTH + MAX_SYSCALL_PARAM_SIZE)
806
807    /* Save volatile registers in the trap frame */
808    mov [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rax], rax
809    mov [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rip], rcx
810    mov [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rdx], rdx
811    mov [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_R8], r8
812    mov [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_R9], r9
813    mov [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rcx], r10
814    mov [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_EFlags], r11
815
816    /* Store user stack pointer in the trap frame */
817    mov rax, gs:[PcUserRsp]
818    mov [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rsp], rax
819
820    /* Set sane segments */
821    mov ax, (KGDT64_R3_DATA or RPL_MASK)
822    mov ds, ax
823    mov es, ax
824
825    /* Save MCXSR and set kernel value */
826    stmxcsr [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_MxCsr]
827    ldmxcsr gs:[PcMxCsr]
828
829#if DBG
830    /* Check IRQL */
831    mov rax, cr8
832    test eax, eax
833    jz KiSystemCall64Again
834    int HEX(2C)
835#endif
836
837GLOBAL_LABEL KiSystemCall64Again
838
839    /* Call the C-handler (will enable interrupts) */
840    call KiSystemCallHandler
841
842    /* The return value from KiSystemCallHandler is the address of the Nt-function */
843    mov rcx, [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rcx]
844    mov rdx, [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rdx]
845    mov r8, [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_R8]
846    mov r9, [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_R9]
847    call rax
848
849GLOBAL_LABEL KiSystemServiceExit
850
851#if DBG
852    test dword ptr [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_EFlags], HEX(200)
853    jnz IntsEnabled
854    int 3
855IntsEnabled:
856#endif
857
858    /* Check for pending user APC */
859    mov rcx, gs:qword ptr [PcCurrentThread]
860    cmp byte ptr [rcx + ThApcState + AsUserApcPending], 0
861    jz no_user_apc_pending
862    call KiInitiateUserApc
863no_user_apc_pending:
864
865    /* Disable interrupts for return */
866    cli
867
868    /* Restore MCXSR */
869    ldmxcsr [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_MxCsr]
870
871    /* Restore old trap frame */
872    mov rcx, gs:[PcCurrentThread]
873    mov rdx, [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_TrapFrame]
874    mov [rcx + KTHREAD_TrapFrame], rdx
875
876    /* Prepare user mode return address (rcx) and eflags (r11) for sysret */
877    mov rcx, [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rip]
878    mov r11, [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_EFlags]
879
880    /* Load user mode stack (It was copied to the trap frame) */
881    mov rsp, [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rsp]
882
883    /* r8 points to the user stack */
884    mov r8, rsp
885
886    /* r9 matches rbp */
887    mov r9, rbp
888
889    /* Swap gs back to user */
890    swapgs
891
892    /* Zero out volatiles */
893    pxor xmm0, xmm0
894    pxor xmm1, xmm1
895    pxor xmm2, xmm2
896    pxor xmm3, xmm3
897    pxor xmm4, xmm4
898    pxor xmm5, xmm5
899    xor rdx, rdx
900    xor r10, r10
901
902    /* return to user mode */
903    .byte HEX(48) // REX prefix to return to long mode
904    sysretq
905
906.ENDP
907
908
909/*!
910 * VOID
911 * DECLSPEC_NORETURN
912 * KiServiceExit(IN PKTRAP_FRAME TrapFrame, IN NTSTATUS Status));
913 */
914PUBLIC KiServiceExit
915.PROC KiServiceExit
916    .endprolog
917
918    lea rsp, [rcx - MAX_SYSCALL_PARAM_SIZE]
919    jmp KiSystemServiceExit
920
921.ENDP
922
923
924/*!
925 * VOID
926 * DECLSPEC_NORETURN
927 * KiServiceExit2(IN PKTRAP_FRAME TrapFrame);
928 */
929PUBLIC KiServiceExit2
930.PROC KiServiceExit2
931    .ENDPROLOG
932
933    // FIXME: this should probably also restore an exception frame
934
935    mov rsp, rcx
936.ENDP
937
938PUBLIC KiServiceExit3
939.PROC KiServiceExit3
940    .PUSHFRAME
941    .ALLOCSTACK (KTRAP_FRAME_LENGTH - MachineFrameLength)
942    .ENDPROLOG
943
944#if DBG
945    /* Get the current IRQL and compare it to the trap frame */
946    mov rax, cr8
947    cmp byte ptr [rsp + KTRAP_FRAME_PreviousIrql], al
948    je KiServiceExit2_ok1
949    int HEX(2C)
950
951KiServiceExit2_ok1:
952    /* Check if this is a user mode exit */
953    mov ah, byte ptr [rsp + KTRAP_FRAME_SegCs]
954    test ah, 1
955    jz KiServiceExit2_kernel
956
957    /* Validate that we are at PASSIVE_LEVEL */
958    test al, al
959    jz KiServiceExit2_kernel
960    int HEX(2C)
961
962KiServiceExit2_kernel:
963#endif
964
965    /* Return */
966    mov rbp, rsp
967    ExitTrap TF_SAVE_ALL
968.ENDP
969
970
971PUBLIC KiSystemCallEntry32
972KiSystemCallEntry32:
973    swapgs
974    int 3
975
976
977PUBLIC KiZwSystemService
978FUNC KiZwSystemService
979    push rbp
980    .pushreg rbp
981    sub rsp, KTRAP_FRAME_LENGTH
982    .allocstack KTRAP_FRAME_LENGTH
983    mov [rsp + KTRAP_FRAME_Rsi], rsi
984    .savereg rsi, KTRAP_FRAME_Rsi
985    mov [rsp + KTRAP_FRAME_Rdi], rdi
986    .savereg rdi, KTRAP_FRAME_Rdi
987    mov rbp, rsp
988    .setframe rbp, 0
989    .endprolog
990
991    /* Get current thread */
992    mov r11, gs:[PcCurrentThread]
993
994    /* Save PreviousMode in the trap frame */
995    mov dil, byte ptr [r11 + KTHREAD_PreviousMode]
996    mov byte ptr [rbp + KTRAP_FRAME_PreviousMode], dil
997
998    /* Save the old trap frame in TrapFrame.Rdx */
999    mov rdi, [r11 + KTHREAD_TrapFrame]
1000    mov [rbp + KTRAP_FRAME_Rdx], rdi
1001
1002    /* Set the new trap frame and previous mode */
1003    mov [r11 + ThTrapFrame], rbp
1004    mov byte ptr [r11 + KTHREAD_PreviousMode], 0
1005
1006    /* allocate space for parameters */
1007    sub rsp, r10
1008    and rsp, HEX(0fffffffffffffff0)
1009
1010    /* Save rcx */
1011    mov [rbp + KTRAP_FRAME_Rcx], rcx
1012
1013    /* copy parameters to the new location */
1014    lea rsi, [rbp + KTRAP_FRAME_LENGTH + 16]
1015    lea rdi, [rsp]
1016    mov rcx, r10
1017    shr rcx, 3
1018    rep movsq
1019
1020    /* Restore rcx */
1021    mov rcx, [rbp + KTRAP_FRAME_Rcx]
1022
1023    /* Call the service function */
1024    call rax
1025
1026    /* Restore the old trap frame */
1027    mov r11, gs:[PcCurrentThread]
1028    mov rsi, [rbp + KTRAP_FRAME_Rdx]
1029    mov [r11 + KTHREAD_TrapFrame], rsi
1030
1031    /* Restore PreviousMode from the trap frame */
1032    mov dil, byte ptr [rbp + KTRAP_FRAME_PreviousMode]
1033    mov byte ptr [r11 + KTHREAD_PreviousMode], dil
1034
1035    /* Restore rdi and rsi */
1036    mov rsi, [rbp + KTRAP_FRAME_Rsi]
1037    mov rdi, [rbp + KTRAP_FRAME_Rdi]
1038
1039    /* Cleanup the stack and return */
1040    lea rsp, [rbp + KTRAP_FRAME_LENGTH]
1041    pop rbp
1042    ret
1043
1044ENDFUNC
1045
1046PUBLIC KiConvertToGuiThread
1047FUNC KiConvertToGuiThread
1048
1049    sub rsp, 40
1050    .allocstack 40
1051    .endprolog
1052
1053    // NewStack = (ULONG_PTR)MmCreateKernelStack(TRUE, 0);
1054    mov cl, 1
1055    xor rdx, rdx
1056    call MmCreateKernelStack
1057
1058    /* Check for failure */
1059    test rax, rax
1060    jz KiConvertToGuiThreadFailed
1061
1062    /* OldStack = KeSwitchKernelStack((PVOID)NewStack, (PVOID)(NewStack - KERNEL_STACK_SIZE)); */
1063    mov rcx, rax
1064    mov rdx, rax
1065    sub rdx, KERNEL_STACK_SIZE
1066    call KeSwitchKernelStack
1067
1068    // MmDeleteKernelStack(OldStack, FALSE);
1069    mov rcx, rax
1070    xor rdx, rdx
1071    call MmDeleteKernelStack
1072
1073    /* Call the worker function */
1074    call PsConvertToGuiThread
1075
1076    /* Check for failure */
1077    test rax, rax
1078    js KiConvertToGuiThreadFailed
1079
1080    /* Disable interrupts for return */
1081    cli
1082
1083    // FIXME: should just do the trap frame switch in KiSystemCallHandler64
1084    /* Restore old trap frame */
1085    mov rcx, gs:[PcCurrentThread]
1086    mov rdx, [rsp + 48 + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_TrapFrame]
1087    mov [rcx + KTHREAD_TrapFrame], rdx
1088
1089    // Restore register parameters
1090    mov rcx, [rsp + 48 + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rip]
1091    mov rdx, [rsp + 48 + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rdx]
1092    mov r8, [rsp + 48 + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_R8]
1093    mov r9, [rsp + 48 + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_R9]
1094
1095    /* Run KiSystemCallHandler again */
1096    add rsp, 48
1097    jmp KiSystemCall64Again
1098
1099KiConvertToGuiThreadFailed:
1100
1101    /* Clean up the stack and return failure */
1102    add rsp, 40
1103    mov eax, HEX(C0000017) // STATUS_NO_MEMORY
1104    ret
1105
1106ENDFUNC
1107
1108
1109EXTERN KiSetTrapContextInternal:PROC
1110
1111/*
1112 * VOID
1113 * KiSetTrapContext(
1114 *     _Out_ PKTRAP_FRAME TrapFrame,
1115 *     _In_ PCONTEXT Context,
1116 *     _In_ KPROCESSOR_MODE RequestorMode);
1117 */
1118PUBLIC KiSetTrapContext
1119.PROC KiSetTrapContext
1120
1121    /* Generate a KEXCEPTION_FRAME on the stack */
1122    GENERATE_EXCEPTION_FRAME
1123
1124    call KiSetTrapContextInternal
1125
1126    /* Restore the registers from the KEXCEPTION_FRAME */
1127    RESTORE_EXCEPTION_STATE
1128
1129    /* Return */
1130    ret
1131
1132.ENDP
1133
1134
1135/*
1136 * VOID
1137 * KiDeliverApc(
1138 *     _In_ KPROCESSOR_MODE DeliveryMode,
1139 *     _In_ PKEXCEPTION_FRAME ExceptionFrame,
1140 *     _In_ PKTRAP_FRAME TrapFrame);
1141 *
1142 */
1143EXTERN KiDeliverApc:PROC
1144
1145PUBLIC KiInitiateUserApc
1146.PROC KiInitiateUserApc
1147
1148    /* Generate a KEXCEPTION_FRAME on the stack */
1149    GENERATE_EXCEPTION_FRAME
1150
1151    /* Raise IRQL to APC_LEVEL */
1152    mov rax, APC_LEVEL
1153    mov cr8, rax
1154
1155    /* Enable interrupts */
1156    sti
1157
1158    /* Get the current trap frame */
1159    mov rax, gs:[PcCurrentThread]
1160    mov r8, [rax + KTHREAD_TrapFrame]
1161
1162    /* Call the C function */
1163    mov ecx, 1
1164    mov rdx, rsp
1165    call KiDeliverApc
1166
1167    /* Disable interrupts again */
1168    cli
1169
1170    /* Go back to PASSIVE_LEVEL */
1171    mov rax, PASSIVE_LEVEL
1172    mov cr8, rax
1173
1174    /* Restore the registers from the KEXCEPTION_FRAME */
1175    RESTORE_EXCEPTION_STATE
1176
1177    /* Return */
1178    ret
1179
1180.ENDP
1181
1182
1183PUBLIC KiInitializeSegments
1184KiInitializeSegments:
1185    mov ax, KGDT64_R3_DATA or RPL_MASK
1186    mov gs, ax
1187    swapgs
1188    mov gs, ax
1189    ret
1190
1191/*!
1192 * VOID
1193 * KiSwitchKernelStackHelper(
1194 *     LONG_PTR StackOffset,
1195 *     PVOID OldStackBase);
1196 */
1197PUBLIC KiSwitchKernelStackHelper
1198KiSwitchKernelStackHelper:
1199
1200    /* Pop return address from the current stack */
1201    pop rax
1202
1203    /* Switch to new stack */
1204    lea rsp, [rsp + rcx]
1205
1206    /* Push return address on the new stack */
1207    push rax
1208
1209    /* Return on new stack */
1210    mov rax, rdx
1211    ret
1212
1213EXTERN KiSwitchKernelStack:PROC
1214
1215PUBLIC KeSwitchKernelStack
1216FUNC KeSwitchKernelStack
1217
1218    sub rsp, 40
1219    .allocstack 40
1220
1221    /* Save rcx */
1222    mov [rsp], rcx
1223    .savereg rcx, 0
1224    .endprolog
1225
1226    /* Call the C handler, which returns the old stack in rax */
1227    call KiSwitchKernelStack
1228
1229    /* Restore rcx (StackBase) */
1230    mov rcx, [rsp]
1231
1232    /* Switch to new stack: RSP += (StackBase - OldStackBase) */
1233    sub rcx, rax
1234    add rsp, rcx
1235
1236    /* Deallocate the home frame */
1237    add rsp, 40
1238    ret
1239
1240ENDFUNC
1241
1242
1243
1244
1245#ifdef _MSC_VER
1246#undef lgdt
1247#undef lidt
1248
1249//void __lgdt(void *Source);
1250PUBLIC __lgdt
1251__lgdt:
1252    lgdt fword ptr [rcx]
1253    ret
1254
1255//void __sgdt(void *Destination);
1256PUBLIC __sgdt
1257__sgdt:
1258    sgdt fword ptr [rcx]
1259    ret
1260
1261// void __lldt(unsigned short Value)
1262PUBLIC __lldt
1263__lldt:
1264    lldt cx
1265    ret
1266
1267//void __sldt(void *Destination);
1268PUBLIC __sldt
1269__sldt:
1270    sldt word ptr [rcx]
1271    ret
1272
1273//void __ltr(unsigned short Source);
1274PUBLIC __ltr
1275__ltr:
1276    ltr cx
1277    ret
1278
1279//void __str(unsigned short *Destination);
1280PUBLIC __str
1281__str:
1282    str word ptr [rcx]
1283    ret
1284
1285PUBLIC __swapgs
1286__swapgs:
1287    swapgs
1288    ret
1289
1290#endif
1291
1292END
1293