xref: /reactos/ntoskrnl/ke/amd64/trap.S (revision 3c774903) 
1/*
2 * FILE:            ntoskrnl/ke/amd64/trap.S
3 * COPYRIGHT:       See COPYING in the top level directory
4 * PURPOSE:         System Traps, Entrypoints and Exitpoints
5 * PROGRAMMER:      Timo Kreuzer (timo.kreuzer@reactos.org)
6 */
7
8/* INCLUDES ******************************************************************/
9
10#include <asm.inc>
11#include <ksamd64.inc>
12#include <trapamd64.inc>
13
14EXTERN KiDispatchException:PROC
15EXTERN FrLdrDbgPrint:DWORD
16EXTERN KeBugCheckWithTf:PROC
17EXTERN MmAccessFault:PROC
18EXTERN KiSystemFatalException:PROC
19EXTERN KiNpxNotAvailableFaultHandler:PROC
20EXTERN KiGeneralProtectionFaultHandler:PROC
21EXTERN KiXmmExceptionHandler:PROC
22EXTERN KiDeliverApc:PROC
23EXTERN KiDpcInterruptHandler:PROC
24
25#ifdef _WINKD_
26EXTERN KdSetOwedBreakpoints:PROC
27#endif
28
29
30/* GLOBALS *******************************************************************/
31
32.data
33
34PUBLIC MsgUnimplemented
35MsgUnimplemented:
36.asciz "WARNING:  %s at %s:%d is UNIMPLEMENTED!\n"
37
38MsgPageFault:
39.asciz "Page fault! Code = 0x%x, RIP = %p, FaultingAddress = %p\n"
40
41MsgGeneralProtFault:
42.asciz "General protection fault at %p!\n"
43
44MsgBreakpointTrap:
45.asciz "BreakpointTrap at %p\n"
46
47MsgUnexpectedInterrupt:
48.asciz "UnexpectedInterrupt Vector=0x%02lx\n"
49
50MsgInvalidOpcodeFault:
51.asciz "Invalid opcode fault at %p!\n"
52
53MsgDoubleFault:
54.asciz "Double fault at %p, rbp=%p!\n"
55
56MsgTrapInfo:
57.asciz "Trap: %s at %p\n"
58
59MACRO(TRAPINFO, func)
60LOCAL label1, label2
61#if 0
62    jmp label2
63label1:  .asciz "\func"
64label2:
65    push rax
66    push rcx
67    push rdx
68    push r8
69    push r9
70    push r10
71    push r11
72    sub rsp, 32
73    lea rcx, MsgTrapInfo[rip]
74    lea rdx, 1b[rip]
75    mov r8, [rbp + KTRAP_FRAME_Rip]
76    call qword ptr FrLdrDbgPrint[rip]
77    pop r11
78    pop r10
79    pop r9
80    pop r8
81    pop rdx
82    pop rcx
83    pop rax
84    add rsp, 32
85#endif
86ENDM
87
88/* Helper Macros *************************************************************/
89
90MACRO(DispatchException, Status, Number, P1, P2, P3)
91    mov eax, Status
92    mov edx, Number
93    mov r9, P1
94    mov r10, P2
95    mov r11, P3
96    call InternalDispatchException
97ENDM
98
99MACRO(Fatal, BugcheckCode)
100    /* Bugcheck */
101    mov ecx, BugcheckCode
102    mov rdx, rbp
103    call KiSystemFatalException
104ENDM
105
106
107/* FUNCTIONS *****************************************************************/
108
109.code64
110
111ALIGN 8
112
113MACRO(UnexpectedVectorStub, Vector)
114    /* This nop is to make the relative jmp address 4 bytes aligned and to
115       make the whole code 8 bytes long */
116    nop
117    /* This is a push instruction with 8bit operand. Since the instruction
118       sign extends the value to 32 bits, we need to offset it */
119PUBLIC KxUnexpectedInterrupt&Vector
120KxUnexpectedInterrupt&Vector:
121    push (Vector - 128)
122    jmp KiUnexpectedInterrupt
123ENDM
124
125PUBLIC KiUnexpectedRange
126KiUnexpectedRange:
127Vector = 0
128REPEAT 256
129    UnexpectedVectorStub %Vector
130    Vector = Vector+1
131ENDR
132PUBLIC KiUnexpectedRangeEnd
133KiUnexpectedRangeEnd:
134
135PUBLIC KiInterruptDispatchTemplate
136KiInterruptDispatchTemplate:
137    /* This instruction pushes the return address on the stack, which is the
138       address of the interrupt object's DispatchCode member, then jumps
139       to the address stored in the interrupt object's DispatchAddress member */
140    call qword ptr KiInterruptDispatchTemplate[rip - KINTERRUPT_DispatchCode + KINTERRUPT_DispatchAddress]
141
142
143// rbp = TrapFrame, eax = ExceptionCode, edx = NumParams, r9,r10,r11 = params
144FUNC InternalDispatchException
145
146    /* Allocate stack space for EXCEPTION_RECORD and KEXCEPTION_FRAME */
147    sub rsp, EXCEPTION_RECORD_LENGTH + KEXCEPTION_FRAME_LENGTH
148    .allocstack (EXCEPTION_RECORD_LENGTH + KEXCEPTION_FRAME_LENGTH)
149    .endprolog
150
151    /* Set up EXCEPTION_RECORD */
152    lea rcx, [rsp + KEXCEPTION_FRAME_LENGTH]
153    mov [rcx + EXCEPTION_RECORD_ExceptionCode], eax
154    xor rax, rax
155    mov [rcx + EXCEPTION_RECORD_ExceptionFlags], eax
156    mov [rcx + EXCEPTION_RECORD_ExceptionRecord], rax
157    mov rax, [rbp + KTRAP_FRAME_Rip]
158    mov [rcx + EXCEPTION_RECORD_ExceptionAddress], rax
159    mov [rcx + EXCEPTION_RECORD_NumberParameters], edx
160    mov [rcx + EXCEPTION_RECORD_ExceptionInformation + HEX(00)], r9
161    mov [rcx + EXCEPTION_RECORD_ExceptionInformation + HEX(08)], r10
162    mov [rcx + EXCEPTION_RECORD_ExceptionInformation + HEX(10)], r11
163
164    /* Set up KEXCEPTION_FRAME */
165    mov rax, [rbp + KTRAP_FRAME_Rbp]
166    mov [rsp + KEXCEPTION_FRAME_Rbp], rax
167    mov [rsp + KEXCEPTION_FRAME_Rbx], rbx
168    mov [rsp + KEXCEPTION_FRAME_Rdi], rdi
169    mov [rsp + KEXCEPTION_FRAME_Rsi], rsi
170    mov [rsp + KEXCEPTION_FRAME_R12], r12
171    mov [rsp + KEXCEPTION_FRAME_R13], r13
172    mov [rsp + KEXCEPTION_FRAME_R14], r14
173    mov [rsp + KEXCEPTION_FRAME_R15], r15
174    movdqa [rsp + KEXCEPTION_FRAME_Xmm6], xmm6
175    movdqa [rsp + KEXCEPTION_FRAME_Xmm7], xmm7
176    movdqa [rsp + KEXCEPTION_FRAME_Xmm8], xmm8
177    movdqa [rsp + KEXCEPTION_FRAME_Xmm9], xmm9
178    movdqa [rsp + KEXCEPTION_FRAME_Xmm10], xmm10
179    movdqa [rsp + KEXCEPTION_FRAME_Xmm11], xmm11
180    movdqa [rsp + KEXCEPTION_FRAME_Xmm12], xmm12
181    movdqa [rsp + KEXCEPTION_FRAME_Xmm13], xmm13
182    movdqa [rsp + KEXCEPTION_FRAME_Xmm14], xmm14
183    movdqa [rsp + KEXCEPTION_FRAME_Xmm15], xmm15
184    mov qword ptr [rsp + KEXCEPTION_FRAME_Return], 0
185
186    /* Call KiDispatchException */
187    // rcx already points to ExceptionRecord
188    mov rdx, rsp                                // ExceptionFrame
189    mov r8, rbp                                 // TrapFrame
190    mov r9b, [r8 + KTRAP_FRAME_PreviousMode]    // PreviousMode
191    mov byte ptr [rsp + KEXCEPTION_FRAME_P5], 1 // FirstChance
192    call KiDispatchException
193
194    /* Restore registers */
195    mov r12, [rsp + KEXCEPTION_FRAME_R12]
196    mov r13, [rsp + KEXCEPTION_FRAME_R13]
197    mov r14, [rsp + KEXCEPTION_FRAME_R14]
198    mov r15, [rsp + KEXCEPTION_FRAME_R15]
199    movdqa xmm6, [rsp + KEXCEPTION_FRAME_Xmm6]
200    movdqa xmm7, [rsp + KEXCEPTION_FRAME_Xmm7]
201    movdqa xmm8, [rsp + KEXCEPTION_FRAME_Xmm8]
202    movdqa xmm9, [rsp + KEXCEPTION_FRAME_Xmm9]
203    movdqa xmm10, [rsp + KEXCEPTION_FRAME_Xmm10]
204    movdqa xmm11, [rsp + KEXCEPTION_FRAME_Xmm11]
205    movdqa xmm12, [rsp + KEXCEPTION_FRAME_Xmm12]
206    movdqa xmm13, [rsp + KEXCEPTION_FRAME_Xmm13]
207    movdqa xmm14, [rsp + KEXCEPTION_FRAME_Xmm14]
208    movdqa xmm15, [rsp + KEXCEPTION_FRAME_Xmm15]
209
210    add rsp, EXCEPTION_RECORD_LENGTH + KEXCEPTION_FRAME_LENGTH
211    ret
212ENDFUNC
213
214
215/* CPU EXCEPTION HANDLERS ****************************************************/
216
217PUBLIC KiDivideErrorFault
218FUNC KiDivideErrorFault
219    /* Push pseudo error code */
220    EnterTrap TF_SAVE_ALL
221
222    /* Enable interrupts */
223    sti
224
225    /* Dispatch the exception */
226    DispatchException STATUS_INTEGER_DIVIDE_BY_ZERO, 0, 0, 0, 0
227
228    /* Return */
229    ExitTrap TF_SAVE_ALL
230ENDFUNC
231
232
233PUBLIC KiDebugTrapOrFault
234FUNC KiDebugTrapOrFault
235    /* Push pseudo error code */
236    EnterTrap TF_SAVE_ALL
237
238    TRAPINFO KiDebugTrapOrFault
239
240    /* Check if the frame was from kernelmode */
241    test word ptr [rbp + KTRAP_FRAME_SegCs], 3
242    jz KiDebugTrapOrFaultKMode
243
244    /* Enable interrupts for user-mode */
245    sti
246
247KiDebugTrapOrFaultKMode:
248    /* Dispatch the exception */
249    DispatchException STATUS_SINGLE_STEP, 0, 0, 0, 0
250
251    /* Return */
252    ExitTrap TF_SAVE_ALL
253ENDFUNC
254
255
256PUBLIC KiNmiInterrupt
257FUNC KiNmiInterrupt
258    /* Push pseudo error code */
259    EnterTrap TF_SAVE_ALL
260
261    UNIMPLEMENTED KiNmiInterrupt
262    int 3
263
264    /* Return */
265    ExitTrap TF_SAVE_ALL
266ENDFUNC
267
268
269PUBLIC KiBreakpointTrap
270FUNC KiBreakpointTrap
271    /* Push pseudo error code */
272    EnterTrap TF_SAVE_ALL
273
274    /* Check if the frame was from kernelmode */
275    test word ptr [rbp + KTRAP_FRAME_SegCs], 3
276    jz KiBreakpointTrapKMode
277
278    /* Enable interrupts for user-mode */
279    sti
280
281KiBreakpointTrapKMode:
282    /* Dispatch the exception */
283    DispatchException STATUS_BREAKPOINT, 3, BREAKPOINT_BREAK, 0, 0
284
285    /* Return */
286    ExitTrap TF_SAVE_ALL
287ENDFUNC
288
289
290PUBLIC KiOverflowTrap
291FUNC KiOverflowTrap
292    /* Push pseudo error code */
293    EnterTrap TF_SAVE_ALL
294
295    /* Enable interrupts */
296    sti
297
298    /* Dispatch the exception */
299    DispatchException STATUS_INTEGER_OVERFLOW, 3, 0, 0, 0
300
301    /* Return */
302    ExitTrap TF_SAVE_ALL
303ENDFUNC
304
305
306PUBLIC KiBoundFault
307FUNC KiBoundFault
308    /* Push pseudo error code */
309    EnterTrap TF_SAVE_ALL
310
311    /* Check if the frame was from kernelmode */
312    test word ptr [rbp + KTRAP_FRAME_SegCs], 3
313    jnz KiBoundFaultUserMode
314
315    /* Bugcheck */
316    Fatal EXCEPTION_BOUND_CHECK
317
318KiBoundFaultUserMode:
319    /* Enable interrupts for user-mode */
320    sti
321
322    /* Dispatch the exception */
323    DispatchException STATUS_ARRAY_BOUNDS_EXCEEDED, 0, 0, 0, 0
324
325    /* Return */
326    ExitTrap TF_SAVE_ALL
327ENDFUNC
328
329
330PUBLIC KiInvalidOpcodeFault
331FUNC KiInvalidOpcodeFault
332    /* Push pseudo error code */
333    EnterTrap TF_SAVE_ALL
334
335    TRAPINFO KiInvalidOpcodeFault
336
337    mov rdx, [rbp + KTRAP_FRAME_Rip]
338    lea rcx, MsgInvalidOpcodeFault[rip]
339    call qword ptr FrLdrDbgPrint[rip]
340
341    /* Enable interrupts */
342    sti
343
344    /* Check if the frame was from kernelmode */
345    test word ptr [rbp + KTRAP_FRAME_SegCs], 3
346    jz KiInvalidOpcodeKernel
347
348    // FIXME: handle STATUS_INVALID_LOCK_SEQUENCE
349
350KiInvalidOpcodeKernel:
351    /* Kernel mode fault */
352
353    /* Dispatch the exception */
354    DispatchException STATUS_ILLEGAL_INSTRUCTION, 3, 0, 0, 0
355
356    /* Return */
357    ExitTrap TF_SAVE_ALL
358ENDFUNC
359
360
361PUBLIC KiNpxNotAvailableFault
362FUNC KiNpxNotAvailableFault
363    /* Push pseudo error code */
364    EnterTrap TF_SAVE_ALL
365
366    /* Call the C handler */
367    mov rcx, rbp
368    call KiNpxNotAvailableFaultHandler
369
370    /* Check the return status code */
371    test eax, eax
372    jz KiNpxNotAvailableFaultExit
373
374    /* Dispatch the exception */
375    DispatchException eax, 3, 0, 0, 0
376
377KiNpxNotAvailableFaultExit:
378    /* Return */
379    ExitTrap TF_SAVE_ALL
380ENDFUNC
381
382
383PUBLIC KiDoubleFaultAbort
384FUNC KiDoubleFaultAbort
385    /* A zero error code is pushed */
386    EnterTrap (TF_HAS_ERROR_CODE OR TF_SAVE_ALL)
387
388    lea rcx, MsgDoubleFault[rip]
389    mov rdx, [rbp + KTRAP_FRAME_FaultAddress]
390    mov r8, rbp
391    call qword ptr FrLdrDbgPrint[rip]
392
393    /* Bugcheck */
394    Fatal 8 // EXCEPTION_DOUBLE_FAULT
395    jmp $
396ENDFUNC
397
398
399PUBLIC KiNpxSegmentOverrunAbort
400FUNC KiNpxSegmentOverrunAbort
401    /* Push pseudo error code */
402    EnterTrap TF_SAVE_ALL
403
404    /* Bugcheck */
405    Fatal EXCEPTION_NPX_OVERRUN
406
407    jmp $
408ENDFUNC
409
410
411PUBLIC KiInvalidTssFault
412FUNC KiInvalidTssFault
413    /* We have an error code */
414    EnterTrap (TF_HAS_ERROR_CODE OR TF_SAVE_ALL)
415
416    /* Bugcheck */
417    Fatal EXCEPTION_INVALID_TSS
418    jmp $
419ENDFUNC
420
421
422PUBLIC KiSegmentNotPresentFault
423FUNC KiSegmentNotPresentFault
424    /* We have an error code */
425    EnterTrap (TF_HAS_ERROR_CODE OR TF_SAVE_ALL)
426
427    /* Bugcheck */
428    Fatal EXCEPTION_SEGMENT_NOT_PRESENT
429    jmp $
430ENDFUNC
431
432
433PUBLIC KiStackFault
434FUNC KiStackFault
435    /* We have an error code */
436    EnterTrap (TF_HAS_ERROR_CODE OR TF_SAVE_ALL)
437
438    /* Bugcheck */
439    Fatal EXCEPTION_STACK_FAULT
440    jmp $
441ENDFUNC
442
443
444PUBLIC KiGeneralProtectionFault
445FUNC KiGeneralProtectionFault
446    /* We have an error code */
447    EnterTrap (TF_HAS_ERROR_CODE OR TF_SAVE_ALL)
448
449    //TRAPINFO KiGeneralProtectionFault
450    //mov rdx, [rbp + KTRAP_FRAME_Rip]
451    //lea rcx, MsgGeneralProtFault[rip]
452    //call qword ptr FrLdrDbgPrint[rip]
453
454    /* Call the C handler */
455    mov rcx, rbp
456    call KiGeneralProtectionFaultHandler
457
458    /* Check for success */
459    test eax, eax
460    jge KiGpfExit
461
462    /* Dispatch the exception */
463    DispatchException eax, 3, 0, 0, 0
464
465KiGpfFatal:
466
467    /* Bugcheck */
468    mov ecx, UNEXPECTED_KERNEL_MODE_TRAP
469    mov rdx, HEX(000D) // EXCEPTION_GP_FAULT
470    xor r8, r8
471    mov r9, [rbp + KTRAP_FRAME_ErrorCode] // error code
472    sub rsp, 8
473    mov [rsp + KTRAP_FRAME_P5+8], rbp // trap frame
474    call KeBugCheckWithTf
475
476KiGpfExit:
477    /* Return */
478    /* Return */
479    ExitTrap TF_SAVE_ALL
480ENDFUNC
481
482
483PUBLIC KiPageFault
484FUNC KiPageFault
485    /* We have an error code */
486    EnterTrap (TF_HAS_ERROR_CODE OR TF_SAVE_ALL)
487
488    TRAPINFO KiPageFault
489
490#if 0
491    lea rcx, MsgPageFault[rip]
492    mov rdx, [rbp + KTRAP_FRAME_ErrorCode]
493    mov r8, [rbp + KTRAP_FRAME_Rip]
494    mov r9, [rbp + KTRAP_FRAME_FaultAddress]
495    call qword ptr FrLdrDbgPrint[rip]
496#endif
497
498    /* Save page fault address */
499    mov rdx, cr2
500    mov [rbp  + KTRAP_FRAME_FaultAddress], rdx
501
502    /* Enable interrupts for the page fault handler */
503    sti
504
505    /* Call page fault handler */
506    mov ecx, [rbp + KTRAP_FRAME_ErrorCode] // FaultCode
507    // rdx == Address
508    mov r8b, [rbp + KTRAP_FRAME_SegCs] // Mode
509    and r8b, 1
510    mov r9, rbp // TrapInformation
511    call MmAccessFault
512
513    /* Check for success */
514    test eax, eax
515#ifndef _WINKD_
516    jge PageFaultReturn
517#else
518    jl PageFaultError
519
520    /* Check whether the kernel debugger has owed breakpoints to be inserted */
521    call KdSetOwedBreakpoints
522    /* We succeeded, return */
523    jmp PageFaultReturn
524
525PageFaultError:
526#endif
527
528    /* Disable interrupts again for the debugger */
529    cli
530
531    /* Set parameter 1 to error code */
532    mov r9d, [rbp + KTRAP_FRAME_ErrorCode]
533
534    /* Set parameter2 to faulting address */
535    mov r10, cr2  // Param2 = faulting address
536
537    cmp eax, STATUS_ACCESS_VIOLATION
538    je AccessViolation
539    cmp eax, STATUS_GUARD_PAGE_VIOLATION
540    je SpecialCode
541    cmp eax, STATUS_STACK_OVERFLOW
542    je SpecialCode
543
544InPageException:
545    /* Dispatch in-page exception */
546    mov r11d, eax // Param3 = Status
547    mov eax, STATUS_IN_PAGE_ERROR // ExceptionCode
548    mov edx, 3    // ParamCount
549    call InternalDispatchException
550    jmp PageFaultReturn
551
552AccessViolation:
553    /* Use more proper status code */
554    mov eax, KI_EXCEPTION_ACCESS_VIOLATION
555
556SpecialCode:
557    /* Setup a normal page fault exception */
558    mov edx, 2   // ParamCount
559    call InternalDispatchException
560
561PageFaultReturn:
562    /* Return */
563    ExitTrap (TF_SAVE_ALL or TF_CHECKUSERAPC)
564ENDFUNC
565
566
567PUBLIC KiFloatingErrorFault
568FUNC KiFloatingErrorFault
569    /* Push pseudo error code */
570    EnterTrap TF_SAVE_ALL
571
572    UNIMPLEMENTED KiFloatingErrorFault
573    int 3
574
575    /* Return */
576    ExitTrap TF_SAVE_ALL
577ENDFUNC
578
579
580PUBLIC KiAlignmentFault
581FUNC KiAlignmentFault
582    /* We have an error code */
583    EnterTrap (TF_HAS_ERROR_CODE OR TF_SAVE_ALL)
584
585    /* Bugcheck */
586    Fatal EXCEPTION_ALIGNMENT_CHECK
587    jmp $
588ENDFUNC
589
590
591PUBLIC KiMcheckAbort
592FUNC KiMcheckAbort
593    /* Push pseudo error code */
594    EnterTrap TF_SAVE_ALL
595
596    /* Bugcheck */
597    Fatal HEX(12)
598    jmp $
599ENDFUNC
600
601
602PUBLIC KiXmmException
603FUNC KiXmmException
604    /* Push pseudo error code */
605    EnterTrap TF_SAVE_ALL
606
607    /* Call the C handler */
608    mov rcx, rbp
609    call KiXmmExceptionHandler
610
611    /* Check for success */
612    test eax, eax
613    jge KiXmmExit
614
615    /* Dispatch the exception */
616    DispatchException eax, 3, 0, 0, 0
617
618KiXmmExit:
619    /* Return */
620    ExitTrap TF_SAVE_ALL
621ENDFUNC
622
623
624/* SOFTWARE INTERRUPT SERVICES ***********************************************/
625
626PUBLIC KiRaiseAssertion
627FUNC KiRaiseAssertion
628    /* We have an error code */
629    EnterTrap (TF_SAVE_ALL)
630
631    /* Decrement RIP to point to the INT2C instruction (2 bytes, not 1 like INT3) */
632    sub qword ptr [rbp + KTRAP_FRAME_Rip], 2
633
634    /* Dispatch the exception */
635    DispatchException STATUS_ASSERTION_FAILURE, 0, 0, 0, 0
636
637    /* Return */
638    ExitTrap TF_SAVE_ALL
639ENDFUNC
640
641
642PUBLIC KiDebugServiceTrap
643FUNC KiDebugServiceTrap
644   /* No error code */
645    EnterTrap TF_SAVE_ALL
646
647    TRAPINFO KiDebugServiceTrap
648
649    /* Increase Rip to skip the int3 */
650    inc qword ptr [rbp + KTRAP_FRAME_Rip]
651
652    /* Dispatch the exception (Params = service, buffer, legth) */
653    DispatchException STATUS_BREAKPOINT, 3, [rbp+KTRAP_FRAME_Rax], [rbp+KTRAP_FRAME_Rcx], [rbp+KTRAP_FRAME_Rdx]
654
655    /* Return */
656    ExitTrap TF_SAVE_ALL
657ENDFUNC
658
659
660PUBLIC KiApcInterrupt
661.PROC KiApcInterrupt
662   /* No error code */
663    EnterTrap (TF_VOLATILES or TF_IRQL)
664
665    /* Raise to APC_LEVEL */
666    mov rax, APC_LEVEL
667    mov cr8, rax
668
669    /* End the interrupt */
670    mov dword ptr [APIC_EOI], 0
671
672    /* Enable interrupts  */
673    sti
674
675    /* Call the worker routine */
676    mov cl, [rbp + KTRAP_FRAME_SegCs] // ProcessorMode
677    and cl, 1
678    mov rdx, 0                        // ExceptionFrame
679    mov r8, rdx                       // TrapFrame
680    call KiDeliverApc
681
682    /* Disable interrupts */
683    cli
684
685    /* Lower IRQL back to PASSIVE */
686    mov rax, PASSIVE_LEVEL
687    mov cr8, rax
688
689    /* Return */
690    ExitTrap (TF_VOLATILES or TF_IRQL)
691.ENDP
692
693EXTERN KiRetireDpcList:PROC
694PUBLIC KiRetireDpcListInDpcStack
695.PROC KiRetireDpcListInDpcStack
696    push rbp
697    .pushreg rbp
698    mov rbp, rsp
699    .setframe rbp, 0
700    .endprolog
701
702    /* Switch stack and call the function */
703    mov rsp, rdx
704    sub rsp, 40
705    call KiRetireDpcList
706
707    /* Restore stack, cleanup and return */
708    mov rsp, rbp
709    pop rbp
710    ret
711.ENDP
712
713PUBLIC KiDpcInterrupt
714.PROC KiDpcInterrupt
715   /* No error code */
716    EnterTrap (TF_VOLATILES or TF_IRQL)
717
718    /* Call the worker routine */
719    call KiDpcInterruptHandler
720
721    /* Return, but don't send an EOI! */
722    ExitTrap (TF_VOLATILES or TF_IRQL)
723.ENDP
724
725
726PUBLIC KiIpiInterrupt
727.PROC KiIpiInterrupt
728   /* No error code */
729    EnterTrap (TF_VOLATILES or TF_IRQL)
730
731    /* Raise to IPI_LEVEL */
732    mov rax, IPI_LEVEL
733    mov cr8, rax
734
735    /* End the interrupt */
736    mov dword ptr [APIC_EOI], 0
737
738    int 3
739
740    /* Return */
741    ExitTrap (TF_VOLATILES or TF_IRQL)
742.ENDP
743
744
745PUBLIC KiUnexpectedInterrupt
746FUNC KiUnexpectedInterrupt
747    /* The error code is the vector */
748    EnterTrap (TF_HAS_ERROR_CODE OR TF_SAVE_ALL)
749
750#if 0
751    /* Set bugcheck parameters */
752    mov ecx, TRAP_CAUSE_UNKNOWN
753    mov rdx, [rbp + KTRAP_FRAME_ErrorCode] // the vector
754    mov r8, 0 // The unknown floating-point exception
755    mov r9, 0 // The enabled and asserted status bits
756    sub rsp, 8
757    mov [rbp + KTRAP_FRAME_P5 + 8], rbp // trap frame
758    call KeBugCheckWithTf
759    jmp $
760#endif
761    /* Return */
762    ExitTrap TF_SAVE_ALL
763ENDFUNC
764
765PUBLIC KiInterruptDispatch
766FUNC KiInterruptDispatch
767    /* The error code is a pointer to the interrupt object's code */
768    EnterTrap (TF_HAS_ERROR_CODE or TF_SAVE_ALL or TF_IRQL)
769
770    /* Increase interrupt count */
771    inc dword ptr gs:[PcInterruptCount];
772
773    /* Load the address of the interrupt object into rcx */
774    mov rcx, [rbp + KTRAP_FRAME_ErrorCode]
775
776    /* Substract offset of the DispatchCode member plus 6 for the call instruction */
777    sub rcx, KINTERRUPT_DispatchCode + 6
778
779    /* Raise IRQL to SynchronizeIrql */
780    movzx rax, byte ptr [rcx + KINTERRUPT_SynchronizeIrql]
781    mov cr8, rax
782
783#ifdef CONFIG_SMP
784    /* Acquire interrupt lock */
785    mov r8, [rcx + KINTERRUPT_ActualLock]
786
787    //KxAcquireSpinLock(Interrupt->ActualLock);
788#endif
789
790    /* Call the ISR */
791    mov rdx, [rcx + KINTERRUPT_ServiceContext]
792    call qword ptr [rcx + KINTERRUPT_ServiceRoutine]
793
794#ifdef CONFIG_SMP
795    /* Release interrupt lock */
796    //KxReleaseSpinLock(Interrupt->ActualLock);
797#endif
798
799    /* Go back to old irql */
800    movzx rax, byte ptr [rbp + KTRAP_FRAME_PreviousIrql]
801    mov cr8, rax
802
803    /* Return */
804    ExitTrap (TF_SAVE_ALL or TF_SEND_EOI)
805ENDFUNC
806
807
808#define MAX_SYSCALL_PARAM_SIZE (16 * 8)
809#define HOME_SIZE 6*8
810#define SYSCALL_ALLOCATION (MAX_SYSCALL_PARAM_SIZE + HOME_SIZE)
811
812EXTERN KiSystemCallHandler:PROC
813
814/*! \name KiSystemCallEntry64
815 *
816 *  \brief This is the entrypoint for syscalls from 64bit user mode
817 *
818 *  \param rax - The system call number
819 *  \param rcx - User mode return address, set by the syscall instruction
820 *  \param rdx,r8,r9 - Parameters 2-4 to the service function
821 *  \param r10 - Parameter 1 to the service function
822 *  \param r11 - RFLAGS saved by the syscall instruction
823 *--*/
824PUBLIC KiSystemCallEntry64
825.PROC KiSystemCallEntry64
826
827    /* Old stack pointer is in rcx, lie and say we saved it in rbp */
828    .setframe rbp, 0
829    .endprolog
830
831    /* Swap gs to kernel, so we can access the PCR */
832    swapgs
833
834    /* Save the user mode rsp in the PCR */
835    mov gs:[PcUserRsp], rsp
836
837    /* Get the kernel stack from the PCR */
838    mov rsp, gs:[PcRspBase]
839
840    /* Allocate a TRAP_FRAME and space for parameters */
841    sub rsp, (KTRAP_FRAME_LENGTH + MAX_SYSCALL_PARAM_SIZE + HOME_SIZE)
842#if DBG
843    /* Save rbp and load it with the old stack pointer */
844    mov [rsp + HOME_SIZE + MAX_SYSCALL_PARAM_SIZE + HOME_SIZE + KTRAP_FRAME_Rbp], rbp
845    mov rbp, gs:[PcUserRsp]
846#endif
847
848    /* Save important volatiles in the trap frame */
849    mov [rsp + HOME_SIZE + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rax], rax
850    mov [rsp + HOME_SIZE + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rcx], rcx
851    mov [rsp + HOME_SIZE + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_R10], r10
852    mov [rsp + HOME_SIZE + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_R11], r11
853
854    /* Set sane segments */
855    mov ax, (KGDT64_R3_DATA or RPL_MASK)
856    mov ds, ax
857    mov es, ax
858
859    /* Call the C-handler (will enable interrupts) */
860    lea rcx, [rsp + SYSCALL_ALLOCATION]
861    call KiSystemCallHandler
862
863    /* Deallocate the handlers home stack frame */
864    add rsp, HOME_SIZE
865
866    /* The return value is the address of the Nt-function */
867    mov rcx, [rsp + 0]
868    mov rdx, [rsp + 8]
869    mov r8, [rsp + 16]
870    mov r9, [rsp + 24]
871    call rax
872
873#if DBG
874    /* Restore rbp */
875    mov rbp, [rsp + SYSCALL_ALLOCATION + KTRAP_FRAME_Rbp]
876#endif
877
878    /* Disable interrupts for return */
879    cli
880
881    /* Restore old trap frame */
882    mov rcx, gs:[PcCurrentThread]
883    mov rdx, [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_TrapFrame]
884    mov [rcx + KTHREAD_TrapFrame], rdx
885
886    /* Prepare user mode return address (rcx) and eflags (r11) for sysret */
887    mov rcx, [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rcx]
888    mov r11, [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_R11]
889
890    /* Load user mode stack (It was copied to the trap frame) */
891    mov rsp, [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rsp]
892
893    /* Swap gs back to user */
894    swapgs
895
896    /* return to user mode */
897    .byte HEX(48) // REX prefix to return to long mode
898    sysret
899.ENDP
900
901
902PUBLIC KiSystemCallEntry32
903KiSystemCallEntry32:
904    swapgs
905    int 3
906
907
908PUBLIC KiZwSystemService
909FUNC KiZwSystemService
910    push rbp
911    .pushreg rbp
912    sub rsp, KTRAP_FRAME_LENGTH
913    .allocstack KTRAP_FRAME_LENGTH
914    mov [rsp + KTRAP_FRAME_Rsi], rsi
915    .savereg rsi, KTRAP_FRAME_Rsi
916    mov [rsp + KTRAP_FRAME_Rdi], rdi
917    .savereg rdi, KTRAP_FRAME_Rdi
918    mov rbp, rsp
919    .setframe rbp, 0
920    .endprolog
921
922    /* Get current thread */
923    mov r11, gs:[PcCurrentThread]
924
925    /* Save the old trap frame in TrapFrame.Rdx */
926    mov rdi, [r11 + KTHREAD_TrapFrame]
927    mov [rbp + KTRAP_FRAME_Rdx], rdi
928
929    /* Set the new trap frame and previous mode */
930    mov [r11 + ThTrapFrame], rbp
931    mov byte ptr [r11 + KTHREAD_PreviousMode], 0
932
933    /* allocate space for parameters */
934    sub rsp, r10
935    and rsp, HEX(0fffffffffffffff0)
936
937    /* Save rcx */
938    mov [rbp + KTRAP_FRAME_Rcx], rcx
939
940    /* copy parameters to the new location */
941    lea rsi, [rbp + KTRAP_FRAME_LENGTH + 16]
942    lea rdi, [rsp]
943    mov rcx, r10
944    shr rcx, 3
945    rep movsq
946
947    /* Restore rcx */
948    mov rcx, [rbp + KTRAP_FRAME_Rcx]
949
950    /* Call the service function */
951    call rax
952
953    /* Restore the old trap frame */
954    mov r11, gs:[PcCurrentThread]
955    mov rsi, [rsp + KTRAP_FRAME_Rdx]
956    mov [r11 + KTHREAD_TrapFrame], rsi
957
958    /* Restore rdi and rsi */
959    mov rsi, [rbp + KTRAP_FRAME_Rsi]
960    mov rdi, [rbp + KTRAP_FRAME_Rdi]
961
962    /* Cleanup the stack and return */
963    lea rsp, [rbp + KTRAP_FRAME_LENGTH]
964    pop rbp
965    ret
966
967ENDFUNC
968
969
970KiExitToUserApc:
971    int 3
972
973/*!
974 * VOID
975 * DECLSPEC_NORETURN
976 * KiServiceExit(IN PKTRAP_FRAME TrapFrame, IN NTSTATUS Status));
977 */
978PUBLIC KiServiceExit
979KiServiceExit:
980    mov [rcx + KTRAP_FRAME_Rax], rdx
981    mov rbp, rcx
982    mov rsp, rcx
983
984    /* Return */
985    //ExitTrap TF_SAVE_ALL
986
987/*!
988 * VOID
989 * DECLSPEC_NORETURN
990 * KiServiceExit2(IN PKTRAP_FRAME TrapFrame);
991 */
992PUBLIC KiServiceExit2
993.PROC KiServiceExit2
994    .ENDPROLOG
995
996    mov rbp, rcx
997    mov rsp, rcx
998
999    /* Return */
1000    ExitTrap TF_SAVE_ALL
1001.ENDP
1002
1003PUBLIC KiInitializeSegments
1004KiInitializeSegments:
1005    mov ax, KGDT64_R3_DATA or RPL_MASK
1006    mov gs, ax
1007    swapgs
1008    mov gs, ax
1009    ret
1010
1011
1012#ifdef _MSC_VER
1013#undef lgdt
1014#undef lidt
1015
1016//void __lgdt(void *Source);
1017PUBLIC __lgdt
1018__lgdt:
1019    lgdt fword ptr [rcx]
1020    ret
1021
1022//void __sgdt(void *Destination);
1023PUBLIC __sgdt
1024__sgdt:
1025    sgdt fword ptr [rcx]
1026    ret
1027
1028// void __lldt(unsigned short Value)
1029PUBLIC __lldt
1030__lldt:
1031    lldt cx
1032    ret
1033
1034//void __sldt(void *Destination);
1035PUBLIC __sldt
1036__sldt:
1037    sldt word ptr [rcx]
1038    ret
1039
1040//void __ltr(unsigned short Source);
1041PUBLIC __ltr
1042__ltr:
1043    ltr cx
1044    ret
1045
1046//void __str(unsigned short *Destination);
1047PUBLIC __str
1048__str:
1049    str word ptr [rcx]
1050    ret
1051
1052PUBLIC __swapgs
1053__swapgs:
1054    swapgs
1055    ret
1056
1057#endif
1058
1059END
1060