xref: /reactos/ntoskrnl/ke/amd64/trap.S (revision db93cb17) 
1/*
2 * FILE:            ntoskrnl/ke/amd64/trap.S
3 * COPYRIGHT:       See COPYING in the top level directory
4 * PURPOSE:         System Traps, Entrypoints and Exitpoints
5 * PROGRAMMER:      Timo Kreuzer (timo.kreuzer@reactos.org)
6 */
7
8/* INCLUDES ******************************************************************/
9
10#include <asm.inc>
11#include <ksamd64.inc>
12#include <trapamd64.inc>
13
14EXTERN KiDispatchException:PROC
15EXTERN KeBugCheckWithTf:PROC
16EXTERN MmAccessFault:PROC
17EXTERN KiSystemFatalException:PROC
18EXTERN KiNpxNotAvailableFaultHandler:PROC
19EXTERN KiGeneralProtectionFaultHandler:PROC
20EXTERN KiXmmExceptionHandler:PROC
21EXTERN KiDeliverApc:PROC
22EXTERN KiDpcInterruptHandler:PROC
23EXTERN PsConvertToGuiThread:PROC
24EXTERN MmCreateKernelStack:PROC
25EXTERN MmDeleteKernelStack:PROC
26
27EXTERN KdSetOwedBreakpoints:PROC
28
29
30/* Helper Macros *************************************************************/
31
32MACRO(DispatchException, Status, Number, P1, P2, P3)
33    mov eax, Status
34    mov edx, Number
35    mov r9, P1
36    mov r10, P2
37    mov r11, P3
38    call InternalDispatchException
39ENDM
40
41MACRO(Fatal, BugcheckCode)
42    /* Bugcheck */
43    mov ecx, BugcheckCode
44    mov rdx, rbp
45    call KiSystemFatalException
46ENDM
47
48
49/* FUNCTIONS *****************************************************************/
50
51.code64
52
53ALIGN 8
54
55MACRO(UnexpectedVectorStub, Vector)
56    /* This nop is to make the relative jmp address 4 bytes aligned and to
57       make the whole code 8 bytes long */
58    nop
59    /* This is a push instruction with 8bit operand. Since the instruction
60       sign extends the value to 32 bits, we need to offset it */
61PUBLIC KxUnexpectedInterrupt&Vector
62KxUnexpectedInterrupt&Vector:
63    push (Vector - 128)
64    jmp KiUnexpectedInterrupt
65ENDM
66
67PUBLIC KiUnexpectedRange
68KiUnexpectedRange:
69Vector = 0
70REPEAT 256
71    UnexpectedVectorStub %Vector
72    Vector = Vector+1
73ENDR
74PUBLIC KiUnexpectedRangeEnd
75KiUnexpectedRangeEnd:
76
77PUBLIC KiInterruptDispatchTemplate
78KiInterruptDispatchTemplate:
79    /* This instruction pushes the return address on the stack, which is the
80       address of the interrupt object's DispatchCode member, then jumps
81       to the address stored in the interrupt object's DispatchAddress member */
82    call qword ptr KiInterruptDispatchTemplate[rip - KINTERRUPT_DispatchCode + KINTERRUPT_DispatchAddress]
83
84
85// rbp = TrapFrame, eax = ExceptionCode, edx = NumParams, r9,r10,r11 = params
86FUNC InternalDispatchException
87
88    /* Allocate stack space for EXCEPTION_RECORD and KEXCEPTION_FRAME */
89    sub rsp, EXCEPTION_RECORD_LENGTH + KEXCEPTION_FRAME_LENGTH
90    .allocstack (EXCEPTION_RECORD_LENGTH + KEXCEPTION_FRAME_LENGTH)
91    .endprolog
92
93    /* Set up EXCEPTION_RECORD */
94    lea rcx, [rsp + KEXCEPTION_FRAME_LENGTH]
95    mov [rcx + EXCEPTION_RECORD_ExceptionCode], eax
96    xor rax, rax
97    mov [rcx + EXCEPTION_RECORD_ExceptionFlags], eax
98    mov [rcx + EXCEPTION_RECORD_ExceptionRecord], rax
99    mov rax, [rbp + KTRAP_FRAME_Rip]
100    mov [rcx + EXCEPTION_RECORD_ExceptionAddress], rax
101    mov [rcx + EXCEPTION_RECORD_NumberParameters], edx
102    mov [rcx + EXCEPTION_RECORD_ExceptionInformation + HEX(00)], r9
103    mov [rcx + EXCEPTION_RECORD_ExceptionInformation + HEX(08)], r10
104    mov [rcx + EXCEPTION_RECORD_ExceptionInformation + HEX(10)], r11
105
106    /* Set up KEXCEPTION_FRAME */
107    mov rax, [rbp + KTRAP_FRAME_Rbp]
108    mov [rsp + KEXCEPTION_FRAME_Rbp], rax
109    mov [rsp + KEXCEPTION_FRAME_Rbx], rbx
110    mov [rsp + KEXCEPTION_FRAME_Rdi], rdi
111    mov [rsp + KEXCEPTION_FRAME_Rsi], rsi
112    mov [rsp + KEXCEPTION_FRAME_R12], r12
113    mov [rsp + KEXCEPTION_FRAME_R13], r13
114    mov [rsp + KEXCEPTION_FRAME_R14], r14
115    mov [rsp + KEXCEPTION_FRAME_R15], r15
116    movdqa [rsp + KEXCEPTION_FRAME_Xmm6], xmm6
117    movdqa [rsp + KEXCEPTION_FRAME_Xmm7], xmm7
118    movdqa [rsp + KEXCEPTION_FRAME_Xmm8], xmm8
119    movdqa [rsp + KEXCEPTION_FRAME_Xmm9], xmm9
120    movdqa [rsp + KEXCEPTION_FRAME_Xmm10], xmm10
121    movdqa [rsp + KEXCEPTION_FRAME_Xmm11], xmm11
122    movdqa [rsp + KEXCEPTION_FRAME_Xmm12], xmm12
123    movdqa [rsp + KEXCEPTION_FRAME_Xmm13], xmm13
124    movdqa [rsp + KEXCEPTION_FRAME_Xmm14], xmm14
125    movdqa [rsp + KEXCEPTION_FRAME_Xmm15], xmm15
126    mov qword ptr [rsp + KEXCEPTION_FRAME_Return], 0
127
128    /* Call KiDispatchException */
129    // rcx already points to ExceptionRecord
130    mov rdx, rsp                                // ExceptionFrame
131    mov r8, rbp                                 // TrapFrame
132    mov r9b, [r8 + KTRAP_FRAME_PreviousMode]    // PreviousMode
133    mov byte ptr [rsp + KEXCEPTION_FRAME_P5], 1 // FirstChance
134    call KiDispatchException
135
136    /* Restore registers */
137    mov r12, [rsp + KEXCEPTION_FRAME_R12]
138    mov r13, [rsp + KEXCEPTION_FRAME_R13]
139    mov r14, [rsp + KEXCEPTION_FRAME_R14]
140    mov r15, [rsp + KEXCEPTION_FRAME_R15]
141    movdqa xmm6, [rsp + KEXCEPTION_FRAME_Xmm6]
142    movdqa xmm7, [rsp + KEXCEPTION_FRAME_Xmm7]
143    movdqa xmm8, [rsp + KEXCEPTION_FRAME_Xmm8]
144    movdqa xmm9, [rsp + KEXCEPTION_FRAME_Xmm9]
145    movdqa xmm10, [rsp + KEXCEPTION_FRAME_Xmm10]
146    movdqa xmm11, [rsp + KEXCEPTION_FRAME_Xmm11]
147    movdqa xmm12, [rsp + KEXCEPTION_FRAME_Xmm12]
148    movdqa xmm13, [rsp + KEXCEPTION_FRAME_Xmm13]
149    movdqa xmm14, [rsp + KEXCEPTION_FRAME_Xmm14]
150    movdqa xmm15, [rsp + KEXCEPTION_FRAME_Xmm15]
151
152    add rsp, EXCEPTION_RECORD_LENGTH + KEXCEPTION_FRAME_LENGTH
153    ret
154ENDFUNC
155
156
157/* CPU EXCEPTION HANDLERS ****************************************************/
158
159PUBLIC KiDivideErrorFault
160FUNC KiDivideErrorFault
161    /* Push pseudo error code */
162    EnterTrap TF_SAVE_ALL
163
164    /* Enable interrupts */
165    sti
166
167    /* Dispatch the exception */
168    DispatchException STATUS_INTEGER_DIVIDE_BY_ZERO, 0, 0, 0, 0
169
170    /* Return */
171    ExitTrap TF_SAVE_ALL
172ENDFUNC
173
174
175PUBLIC KiDebugTrapOrFault
176FUNC KiDebugTrapOrFault
177    /* Push pseudo error code */
178    EnterTrap TF_SAVE_ALL
179
180    /* Check if the frame was from kernelmode */
181    test word ptr [rbp + KTRAP_FRAME_SegCs], 3
182    jz KiDebugTrapOrFaultKMode
183
184    /* Enable interrupts for user-mode */
185    sti
186
187KiDebugTrapOrFaultKMode:
188    /* Dispatch the exception */
189    DispatchException STATUS_SINGLE_STEP, 0, 0, 0, 0
190
191    /* Return */
192    ExitTrap TF_SAVE_ALL
193ENDFUNC
194
195EXTERN KiNmiInterruptHandler:PROC
196
197FUNC KiNmiInterruptWithEf
198    /* Generate a KEXCEPTION_FRAME on the stack */
199    GENERATE_EXCEPTION_FRAME
200
201    /* Call the C handler */
202    lea rcx, [rsp + KEXCEPTION_FRAME_LENGTH]
203    lea rdx, [rsp]
204    call KiNmiInterruptHandler
205
206    /* Restore the registers from the KEXCEPTION_FRAME */
207    RESTORE_EXCEPTION_STATE
208
209    /* Return */
210    ret
211ENDFUNC
212
213PUBLIC KiNmiInterrupt
214FUNC KiNmiInterrupt
215    /* Push pseudo error code */
216    EnterTrap TF_SAVE_ALL
217
218    call KiNmiInterruptWithEf
219
220    /* Return */
221    ExitTrap TF_SAVE_ALL
222ENDFUNC
223
224
225PUBLIC KiBreakpointTrap
226FUNC KiBreakpointTrap
227    /* Push pseudo error code */
228    EnterTrap TF_SAVE_ALL
229
230    /* Check if the frame was from kernelmode */
231    test word ptr [rbp + KTRAP_FRAME_SegCs], 3
232    jz KiBreakpointTrapKMode
233
234    /* Enable interrupts for user-mode */
235    sti
236
237KiBreakpointTrapKMode:
238    /* Dispatch the exception */
239    DispatchException STATUS_BREAKPOINT, 3, BREAKPOINT_BREAK, 0, 0
240
241    /* Return */
242    ExitTrap TF_SAVE_ALL
243ENDFUNC
244
245
246PUBLIC KiOverflowTrap
247FUNC KiOverflowTrap
248    /* Push pseudo error code */
249    EnterTrap TF_SAVE_ALL
250
251    /* Enable interrupts */
252    sti
253
254    /* Dispatch the exception */
255    DispatchException STATUS_INTEGER_OVERFLOW, 3, 0, 0, 0
256
257    /* Return */
258    ExitTrap TF_SAVE_ALL
259ENDFUNC
260
261
262PUBLIC KiBoundFault
263FUNC KiBoundFault
264    /* Push pseudo error code */
265    EnterTrap TF_SAVE_ALL
266
267    /* Check if the frame was from kernelmode */
268    test word ptr [rbp + KTRAP_FRAME_SegCs], 3
269    jnz KiBoundFaultUserMode
270
271    /* Bugcheck */
272    Fatal EXCEPTION_BOUND_CHECK
273
274KiBoundFaultUserMode:
275    /* Enable interrupts for user-mode */
276    sti
277
278    /* Dispatch the exception */
279    DispatchException STATUS_ARRAY_BOUNDS_EXCEEDED, 0, 0, 0, 0
280
281    /* Return */
282    ExitTrap TF_SAVE_ALL
283ENDFUNC
284
285
286PUBLIC KiInvalidOpcodeFault
287FUNC KiInvalidOpcodeFault
288    /* Push pseudo error code */
289    EnterTrap TF_SAVE_ALL
290
291    /* Enable interrupts */
292    sti
293
294    /* Check if the frame was from kernelmode */
295    test word ptr [rbp + KTRAP_FRAME_SegCs], 3
296    jz KiInvalidOpcodeKernel
297
298    // FIXME: handle STATUS_INVALID_LOCK_SEQUENCE
299
300KiInvalidOpcodeKernel:
301    /* Kernel mode fault */
302
303    /* Dispatch the exception */
304    DispatchException STATUS_ILLEGAL_INSTRUCTION, 0, 0, 0, 0
305
306    /* Return */
307    ExitTrap TF_SAVE_ALL
308ENDFUNC
309
310
311PUBLIC KiNpxNotAvailableFault
312FUNC KiNpxNotAvailableFault
313    /* Push pseudo error code */
314    EnterTrap TF_SAVE_ALL
315
316    /* Call the C handler */
317    mov rcx, rbp
318    call KiNpxNotAvailableFaultHandler
319
320    /* Check the return status code */
321    test eax, eax
322    jz KiNpxNotAvailableFaultExit
323
324    /* Dispatch the exception */
325    DispatchException eax, 3, 0, 0, 0
326
327KiNpxNotAvailableFaultExit:
328    /* Return */
329    ExitTrap TF_SAVE_ALL
330ENDFUNC
331
332
333PUBLIC KiDoubleFaultAbort
334FUNC KiDoubleFaultAbort
335
336    /* Hack for VBox, which "forgets" to push an error code on the stack! */
337    and rsp, HEX(FFFFFFFFFFFFFFF0)
338
339    /* A zero error code is pushed */
340    EnterTrap (TF_HAS_ERROR_CODE OR TF_SAVE_ALL)
341
342    int 3
343
344    /* Bugcheck */
345    Fatal 8 // EXCEPTION_DOUBLE_FAULT
346    jmp $
347ENDFUNC
348
349
350PUBLIC KiNpxSegmentOverrunAbort
351FUNC KiNpxSegmentOverrunAbort
352    /* Push pseudo error code */
353    EnterTrap TF_SAVE_ALL
354
355    /* Bugcheck */
356    Fatal EXCEPTION_NPX_OVERRUN
357
358    jmp $
359ENDFUNC
360
361
362PUBLIC KiInvalidTssFault
363FUNC KiInvalidTssFault
364    /* We have an error code */
365    EnterTrap (TF_HAS_ERROR_CODE OR TF_SAVE_ALL)
366
367    /* Bugcheck */
368    Fatal EXCEPTION_INVALID_TSS
369    jmp $
370ENDFUNC
371
372
373PUBLIC KiSegmentNotPresentFault
374FUNC KiSegmentNotPresentFault
375    /* We have an error code */
376    EnterTrap (TF_HAS_ERROR_CODE OR TF_SAVE_ALL)
377
378    /* Bugcheck */
379    Fatal EXCEPTION_SEGMENT_NOT_PRESENT
380    jmp $
381ENDFUNC
382
383
384PUBLIC KiStackFault
385FUNC KiStackFault
386    /* We have an error code */
387    EnterTrap (TF_HAS_ERROR_CODE OR TF_SAVE_ALL)
388
389    /* Bugcheck */
390    Fatal EXCEPTION_STACK_FAULT
391    jmp $
392ENDFUNC
393
394
395PUBLIC KiGeneralProtectionFault
396FUNC KiGeneralProtectionFault
397    /* We have an error code */
398    EnterTrap (TF_HAS_ERROR_CODE OR TF_SAVE_ALL)
399
400    /* Call the C handler */
401    mov rcx, rbp
402    call KiGeneralProtectionFaultHandler
403
404    /* Check for success */
405    test eax, eax
406    jge KiGpfExit
407
408    /* Check for access violation */
409    cmp eax, STATUS_ACCESS_VIOLATION
410    je DispatchAccessViolation
411
412    /* Dispatch privileged instruction fault */
413    DispatchException eax, 0, 0, 0, 0
414    jmp KiGpfExit
415
416DispatchAccessViolation:
417
418    /* Dispatch access violation */
419    DispatchException eax, 2, 0, -1, 0
420
421KiGpfExit:
422    /* Return */
423    ExitTrap TF_SAVE_ALL
424ENDFUNC
425
426
427PUBLIC KiPageFault
428FUNC KiPageFault
429    /* We have an error code */
430    EnterTrap (TF_HAS_ERROR_CODE OR TF_SAVE_ALL)
431
432    /* Save page fault address */
433    mov rdx, cr2
434    mov [rbp  + KTRAP_FRAME_FaultAddress], rdx
435
436    /* If interrupts are off, do not enable them */
437    test dword ptr [rbp + KTRAP_FRAME_EFlags], EFLAGS_IF_MASK
438    jz IntsDisabled
439
440    /* Enable interrupts for the page fault handler */
441    sti
442
443IntsDisabled:
444
445    /* Call page fault handler */
446    mov ecx, [rbp + KTRAP_FRAME_ErrorCode] // FaultCode
447    // rdx == Address
448    mov r8b, [rbp + KTRAP_FRAME_SegCs] // Mode
449    and r8b, 1
450    mov r9, rbp // TrapInformation
451    call MmAccessFault
452
453    /* Check for success */
454    test eax, eax
455    jl PageFaultError
456
457    /* Check whether the kernel debugger has owed breakpoints to be inserted */
458    call KdSetOwedBreakpoints
459    /* We succeeded, return */
460    jmp PageFaultReturn
461
462PageFaultError:
463
464    /* Set parameter 1 to error code */
465    mov r9d, [rbp + KTRAP_FRAME_ErrorCode]
466
467    /* Set parameter 2 to faulting address */
468    mov r10, cr2  // Param2 = faulting address
469
470    cmp eax, STATUS_ACCESS_VIOLATION
471    je AccessViolation
472    cmp eax, STATUS_GUARD_PAGE_VIOLATION
473    je SpecialCode
474    cmp eax, STATUS_STACK_OVERFLOW
475    je SpecialCode
476
477InPageException:
478    /* Dispatch in-page exception */
479    mov r11d, eax // Param3 = Status
480    mov eax, STATUS_IN_PAGE_ERROR // ExceptionCode
481    mov edx, 3    // ParamCount
482    call InternalDispatchException
483    jmp PageFaultReturn
484
485AccessViolation:
486    /* Use more proper status code */
487    mov eax, KI_EXCEPTION_ACCESS_VIOLATION
488
489SpecialCode:
490    /* Setup a normal page fault exception */
491    mov edx, 2   // ParamCount
492    call InternalDispatchException
493
494PageFaultReturn:
495
496    /* Disable interrupts for the return */
497    cli
498
499    /* Return */
500    ExitTrap (TF_SAVE_ALL or TF_CHECKUSERAPC)
501ENDFUNC
502
503
504PUBLIC KiFloatingErrorFault
505FUNC KiFloatingErrorFault
506    /* Push pseudo error code */
507    EnterTrap TF_SAVE_ALL
508
509    UNIMPLEMENTED KiFloatingErrorFault
510    int 3
511
512    /* Return */
513    ExitTrap TF_SAVE_ALL
514ENDFUNC
515
516
517PUBLIC KiAlignmentFault
518FUNC KiAlignmentFault
519    /* We have an error code */
520    EnterTrap (TF_HAS_ERROR_CODE OR TF_SAVE_ALL)
521
522    /* Bugcheck */
523    Fatal EXCEPTION_ALIGNMENT_CHECK
524    jmp $
525ENDFUNC
526
527
528PUBLIC KiMcheckAbort
529FUNC KiMcheckAbort
530    /* Push pseudo error code */
531    EnterTrap TF_SAVE_ALL
532
533    /* Bugcheck */
534    Fatal HEX(12)
535    jmp $
536ENDFUNC
537
538
539PUBLIC KiXmmException
540FUNC KiXmmException
541    /* Push pseudo error code */
542    EnterTrap TF_SAVE_ALL
543
544    /* Call the C handler */
545    mov rcx, rbp
546    call KiXmmExceptionHandler
547
548    /* Check for success */
549    test eax, eax
550    jge KiXmmExit
551
552    /* Dispatch the exception */
553    DispatchException eax, 2, 0, [rbp+KTRAP_FRAME_MxCsr], 0
554
555    // FIXME: STATUS_FLOAT_MULTIPLE_TRAPS / STATUS_FLOAT_MULTIPLE_FAULTS
556
557KiXmmExit:
558    /* Return */
559    ExitTrap TF_SAVE_ALL
560ENDFUNC
561
562
563/* SOFTWARE INTERRUPT SERVICES ***********************************************/
564
565PUBLIC KiRaiseAssertion
566FUNC KiRaiseAssertion
567    /* We have an error code */
568    EnterTrap (TF_SAVE_ALL)
569
570    /* Decrement RIP to point to the INT2C instruction (2 bytes, not 1 like INT3) */
571    sub qword ptr [rbp + KTRAP_FRAME_Rip], 2
572
573    /* Dispatch the exception */
574    DispatchException STATUS_ASSERTION_FAILURE, 0, 0, 0, 0
575
576    /* Return */
577    ExitTrap TF_SAVE_ALL
578ENDFUNC
579
580
581PUBLIC KiDebugServiceTrap
582FUNC KiDebugServiceTrap
583   /* No error code */
584    EnterTrap TF_SAVE_ALL
585
586    /* Increase Rip to skip the int3 */
587    inc qword ptr [rbp + KTRAP_FRAME_Rip]
588
589    /* Dispatch the exception (Params = service, buffer, legth) */
590    DispatchException STATUS_BREAKPOINT, 3, [rbp+KTRAP_FRAME_Rax], [rbp+KTRAP_FRAME_Rcx], [rbp+KTRAP_FRAME_Rdx]
591
592    /* Return */
593    ExitTrap TF_SAVE_ALL
594ENDFUNC
595
596
597PUBLIC KiApcInterrupt
598.PROC KiApcInterrupt
599   /* No error code */
600    EnterTrap (TF_SAVE_ALL or TF_IRQL)
601
602    /* Raise to APC_LEVEL */
603    mov rax, APC_LEVEL
604    mov cr8, rax
605
606    /* End the interrupt */
607    mov dword ptr [APIC_EOI], 0
608
609    /* Enable interrupts  */
610    sti
611
612    /* Call the worker routine */
613    mov cl, [rbp + KTRAP_FRAME_SegCs] // ProcessorMode
614    and cl, 1
615    mov rdx, 0                        // ExceptionFrame
616    mov r8, rbp                       // TrapFrame
617    call KiDeliverApc
618
619    /* Disable interrupts */
620    cli
621
622    /* Lower IRQL back to PASSIVE */
623    mov rax, PASSIVE_LEVEL
624    mov cr8, rax
625
626    /* Return */
627    ExitTrap (TF_SAVE_ALL or TF_IRQL)
628.ENDP
629
630/*
631 * VOID
632 * KiRetireDpcList(
633 *     PKPRCB Prcb);
634 */
635EXTERN KiRetireDpcList:PROC
636
637/*
638 * VOID
639 * KiRetireDpcListInDpcStack(
640 *     PKPRCB Prcb,
641 *     PVOID DpcStack);
642 */
643PUBLIC KiRetireDpcListInDpcStack
644.PROC KiRetireDpcListInDpcStack
645    push rbp
646    .pushreg rbp
647    mov rbp, rsp
648    .setframe rbp, 0
649    .endprolog
650
651    /* Switch to the DpcStack */
652    mov rsp, rdx
653
654    /* The stack is 16 byte aligned, allocate 32 bytes home space */
655    sub rsp, 32
656
657    /* Call KiRetireDpcList on the given stack */
658    call KiRetireDpcList
659
660    /* Restore stack, cleanup and return */
661    mov rsp, rbp
662    pop rbp
663    ret
664.ENDP
665
666PUBLIC KiDpcInterrupt
667.PROC KiDpcInterrupt
668   /* No error code */
669    EnterTrap (TF_SAVE_ALL or TF_IRQL)
670
671    /* Call the worker routine */
672    call KiDpcInterruptHandler
673
674    /* Return, but don't send an EOI! */
675    ExitTrap (TF_SAVE_ALL or TF_IRQL)
676.ENDP
677
678
679PUBLIC KiIpiInterrupt
680.PROC KiIpiInterrupt
681   /* No error code */
682    EnterTrap (TF_SAVE_ALL or TF_IRQL)
683
684    /* Raise to IPI_LEVEL */
685    mov rax, IPI_LEVEL
686    mov cr8, rax
687
688    /* End the interrupt */
689    mov dword ptr [APIC_EOI], 0
690
691    int 3
692
693    /* Return */
694    ExitTrap (TF_SAVE_ALL or TF_IRQL)
695.ENDP
696
697
698PUBLIC KiUnexpectedInterrupt
699FUNC KiUnexpectedInterrupt
700    /* The error code is the vector */
701    EnterTrap (TF_HAS_ERROR_CODE OR TF_SAVE_ALL)
702
703#if 0
704    /* Set bugcheck parameters */
705    mov ecx, TRAP_CAUSE_UNKNOWN
706    mov rdx, [rbp + KTRAP_FRAME_ErrorCode] // the vector
707    mov r8, 0 // The unknown floating-point exception
708    mov r9, 0 // The enabled and asserted status bits
709    sub rsp, 8
710    mov [rbp + KTRAP_FRAME_P5 + 8], rbp // trap frame
711    call KeBugCheckWithTf
712    jmp $
713#endif
714    /* Return */
715    ExitTrap TF_SAVE_ALL
716ENDFUNC
717
718PUBLIC KiInterruptDispatch
719FUNC KiInterruptDispatch
720    /* The error code is a pointer to the interrupt object's code */
721    EnterTrap (TF_HAS_ERROR_CODE or TF_SAVE_ALL or TF_IRQL)
722
723    /* Increase interrupt count */
724    inc dword ptr gs:[PcInterruptCount];
725
726    /* Save rbx and rsi in the trap frame */
727    mov [rbp + KTRAP_FRAME_Rbx], rbx
728    mov [rbp + KTRAP_FRAME_Rsi], rsi
729
730    /* Load the address of the dispatch code into rbx */
731    mov rbx, [rbp + KTRAP_FRAME_ErrorCode]
732
733    /* Substract offset of the DispatchCode member plus 6 for the call instruction */
734    sub rbx, KINTERRUPT_DispatchCode + 6
735
736    /* Save the address of the InterruptListEntry in rsi */
737    lea rsi, [rbx + KINTERRUPT_InterruptListEntry]
738
739.DoDispatchInterrupt:
740    /* Raise IRQL to SynchronizeIrql */
741    movzx rax, byte ptr [rbx + KINTERRUPT_SynchronizeIrql]
742    mov cr8, rax
743
744#ifdef CONFIG_SMP
745    /* Acquire interrupt lock */
746    mov r8, [rbx + KINTERRUPT_ActualLock]
747
748    //KxAcquireSpinLock(Interrupt->ActualLock);
749#endif
750
751    /* Call the ISR */
752    mov rcx, rbx
753    mov rdx, [rbx + KINTERRUPT_ServiceContext]
754    call qword ptr [rbx + KINTERRUPT_ServiceRoutine]
755
756#ifdef CONFIG_SMP
757    /* Release interrupt lock */
758    //KxReleaseSpinLock(Interrupt->ActualLock);
759#endif
760
761    /* Go back to old irql */
762    movzx rax, byte ptr [rbp + KTRAP_FRAME_PreviousIrql]
763    mov cr8, rax
764
765    /* Check for chained interrupts */
766    mov rax, [rbx + KINTERRUPT_InterruptListEntry]
767    cmp rax, rsi
768    je .Done
769
770    /* Load the next interrupt object into rbx and repeat */
771    lea rbx, [rax - KINTERRUPT_InterruptListEntry]
772    jmp .DoDispatchInterrupt
773
774.Done:
775    /* Restore rbx and rsi */
776    mov rbx, [rbp + KTRAP_FRAME_Rbx]
777    mov rsi, [rbp + KTRAP_FRAME_Rsi]
778
779    /* Return */
780    ExitTrap (TF_SAVE_ALL or TF_SEND_EOI)
781ENDFUNC
782
783EXTERN KiSystemCallHandler:PROC
784
785/*! \name KiSystemCallEntry64
786 *
787 *  \brief This is the entrypoint for syscalls from 64bit user mode
788 *
789 *  \param rax - The system call number
790 *  \param rcx - User mode return address, set by the syscall instruction
791 *  \param rdx,r8,r9 - Parameters 2-4 to the service function
792 *  \param r10 - Parameter 1 to the service function
793 *  \param r11 - RFLAGS saved by the syscall instruction
794 *--*/
795PUBLIC KiSystemCallEntry64
796.PROC KiSystemCallEntry64
797    /* The unwind info pretends we have a machine frame */
798    .PUSHFRAME
799    .ALLOCSTACK (KTRAP_FRAME_LENGTH + MAX_SYSCALL_PARAM_SIZE - MachineFrameLength)
800    .SAVEREG rbp, MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rbp
801    .ENDPROLOG
802
803    /* Swap gs to kernel, so we can access the PCR */
804    swapgs
805
806    /* Save the user mode rsp in the PCR */
807    mov gs:[PcUserRsp], rsp
808
809    /* Get the kernel stack from the PCR */
810    mov rsp, gs:[PcRspBase]
811
812    /* Allocate a TRAP_FRAME and space for parameters */
813    sub rsp, (KTRAP_FRAME_LENGTH + MAX_SYSCALL_PARAM_SIZE)
814
815    /* Save volatile registers and rbp in the trap frame */
816    mov [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rax], rax
817    mov [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rip], rcx
818    mov [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rdx], rdx
819    mov [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_R8], r8
820    mov [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_R9], r9
821    mov [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rcx], r10
822    mov [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_EFlags], r11
823    mov [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rbp], rbp
824
825    /* Store user stack pointer in the trap frame */
826    mov rax, gs:[PcUserRsp]
827    mov [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rsp], rax
828
829    /* Set sane segments */
830    mov ax, (KGDT64_R3_DATA or RPL_MASK)
831    mov ds, ax
832    mov es, ax
833
834    /* Save MCXSR and set kernel value */
835    stmxcsr [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_MxCsr]
836    ldmxcsr gs:[PcMxCsr]
837
838    /* Get the current thread and the trap frame */
839    mov rax, gs:[PcCurrentThread]
840    mov rcx, [rax + ThTrapFrame]
841
842    /* Save the old trap frame */
843    lea rdx, [rsp + MAX_SYSCALL_PARAM_SIZE]
844    mov [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_TrapFrame], rcx
845    mov [rax + ThTrapFrame], rdx
846
847#if DBG
848    /* Check IRQL */
849    mov rax, cr8
850    test eax, eax
851    jz KiSystemCall64Again
852    int HEX(2C)
853#endif
854
855GLOBAL_LABEL KiSystemCall64Again
856
857    /* Call the C-handler (will enable interrupts) */
858    call KiSystemCallHandler
859
860    /* The return value from KiSystemCallHandler is the address of the Nt-function */
861    mov rcx, [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rcx]
862    mov rdx, [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rdx]
863    mov r8, [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_R8]
864    mov r9, [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_R9]
865    call rax
866
867GLOBAL_LABEL KiSystemServiceExit
868
869    ASSERT_TRAP_FRAME_INTS_ENABLED rsp + MAX_SYSCALL_PARAM_SIZE
870
871    /* Check for pending user APCs */
872    mov rcx, gs:[PcCurrentThread]
873    cmp byte ptr [rcx + ThApcState + AsUserApcPending], 0
874    jz NoUserApcPending
875
876    /* Save missing regs in the trap frame */
877    mov [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rax], rax
878    mov [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rbp], rbp
879    mov [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_R9], rbp
880    mov rax, [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rsp]
881    mov [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_R8], rax
882    mov rax, [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rip]
883    mov [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rcx], rax
884    mov rax, [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_EFlags]
885    mov [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_R11], rax
886    xor rax, rax
887    mov [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rdx], rax
888    mov [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_R10], rax
889    pxor xmm0, xmm0
890    movdqa [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Xmm0], xmm0
891    movdqa [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Xmm1], xmm0
892    movdqa [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Xmm2], xmm0
893    movdqa [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Xmm3], xmm0
894    movdqa [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Xmm4], xmm0
895    movdqa [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Xmm5], xmm0
896
897    lea rcx, [rsp + MAX_SYSCALL_PARAM_SIZE]
898    call KiInitiateUserApc
899
900NoUserApcPending:
901    /* Disable interrupts for return */
902    cli
903
904    /* Restore MCXSR */
905    ldmxcsr [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_MxCsr]
906
907    /* Restore old trap frame */
908    mov rcx, gs:[PcCurrentThread]
909    mov rdx, [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_TrapFrame]
910    mov [rcx + KTHREAD_TrapFrame], rdx
911
912    /* Prepare user mode return address (rcx) and eflags (r11) for sysret */
913    mov rcx, [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rip]
914    mov r11, [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_EFlags]
915
916    /* Load user mode stack (It was copied to the trap frame) */
917    mov rsp, [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rsp]
918
919    /* r8 points to the user stack */
920    mov r8, rsp
921
922    /* r9 matches rbp */
923    mov r9, rbp
924
925    /* Swap gs back to user */
926    swapgs
927
928    /* Zero out volatiles */
929    pxor xmm0, xmm0
930    pxor xmm1, xmm1
931    pxor xmm2, xmm2
932    pxor xmm3, xmm3
933    pxor xmm4, xmm4
934    pxor xmm5, xmm5
935    xor rdx, rdx
936    xor r10, r10
937
938    /* return to user mode */
939    sysretq
940
941.ENDP
942
943
944/*!
945 * VOID
946 * DECLSPEC_NORETURN
947 * KiExceptionExit(
948 *     _In_ PKTRAP_FRAME TrapFrame@<rcx>,
949 *     _In_ PKEXCEPTION_FRAME ExceptionFrame@<rdx>);
950 */
951PUBLIC KiExceptionExit
952KiExceptionExit:
953
954    /* Restore registers from exception frame */
955    movaps xmm6, [rdx + ExXmm6]
956    movaps xmm7, [rdx + ExXmm7]
957    movaps xmm8, [rdx + ExXmm8]
958    movaps xmm9, [rdx + ExXmm9]
959    movaps xmm10, [rdx + ExXmm10]
960    movaps xmm11, [rdx + ExXmm11]
961    movaps xmm12, [rdx + ExXmm12]
962    movaps xmm13, [rdx + ExXmm13]
963    movaps xmm14, [rdx + ExXmm14]
964    movaps xmm15, [rdx + ExXmm15]
965    mov rbx, [rdx + ExRbx]
966    mov rdi, [rdx + ExRdi]
967    mov rsi, [rdx + ExRsi]
968    mov r12, [rdx + ExR12]
969    mov r13, [rdx + ExR13]
970    mov r14, [rdx + ExR14]
971    mov r15, [rdx + ExR15]
972
973    /* Point rsp at the trap frame */
974    mov rsp, rcx
975    /* Fall through */
976
977/*
978 * Internal function. Exits to user-mode with rsp pointing to the trap frame.
979 * All non-volatile register context must be set up already.
980 * Used by KiInitializeContextThread to set up the init path for a new thread.
981 */
982PUBLIC KiServiceExit3
983.PROC KiServiceExit3
984    .PUSHFRAME
985    .ALLOCSTACK (KTRAP_FRAME_LENGTH - MachineFrameLength)
986    .ENDPROLOG
987
988    /* Return */
989    mov rbp, rsp
990    ExitTrap (TF_SEGMENTS or TF_CHECKUSERAPC)
991.ENDP
992
993
994PUBLIC KiSystemCallEntry32
995KiSystemCallEntry32:
996    swapgs
997    int 3
998
999
1000PUBLIC KiZwSystemService
1001FUNC KiZwSystemService
1002    push rbp
1003    .pushreg rbp
1004    sub rsp, KTRAP_FRAME_LENGTH
1005    .allocstack KTRAP_FRAME_LENGTH
1006    mov [rsp + KTRAP_FRAME_Rsi], rsi
1007    .savereg rsi, KTRAP_FRAME_Rsi
1008    mov [rsp + KTRAP_FRAME_Rdi], rdi
1009    .savereg rdi, KTRAP_FRAME_Rdi
1010    mov rbp, rsp
1011    .setframe rbp, 0
1012    .endprolog
1013
1014    /* Get current thread */
1015    mov r11, gs:[PcCurrentThread]
1016
1017    /* Save PreviousMode in the trap frame */
1018    mov dil, byte ptr [r11 + KTHREAD_PreviousMode]
1019    mov byte ptr [rbp + KTRAP_FRAME_PreviousMode], dil
1020
1021    /* Save the old trap frame in TrapFrame.Rdx */
1022    mov rdi, [r11 + KTHREAD_TrapFrame]
1023    mov [rbp + KTRAP_FRAME_Rdx], rdi
1024
1025    /* Set the new trap frame and previous mode */
1026    mov [r11 + ThTrapFrame], rbp
1027    mov byte ptr [r11 + KTHREAD_PreviousMode], 0
1028
1029    /* allocate space for parameters */
1030    sub rsp, r10
1031    and rsp, HEX(0fffffffffffffff0)
1032
1033    /* Save rcx */
1034    mov [rbp + KTRAP_FRAME_Rcx], rcx
1035
1036    /* copy parameters to the new location */
1037    lea rsi, [rbp + KTRAP_FRAME_LENGTH + 16]
1038    lea rdi, [rsp]
1039    mov rcx, r10
1040    shr rcx, 3
1041    rep movsq
1042
1043    /* Restore rcx */
1044    mov rcx, [rbp + KTRAP_FRAME_Rcx]
1045
1046    /* Call the service function */
1047    call rax
1048
1049    /* Restore the old trap frame */
1050    mov r11, gs:[PcCurrentThread]
1051    mov rsi, [rbp + KTRAP_FRAME_Rdx]
1052    mov [r11 + KTHREAD_TrapFrame], rsi
1053
1054    /* Restore PreviousMode from the trap frame */
1055    mov dil, byte ptr [rbp + KTRAP_FRAME_PreviousMode]
1056    mov byte ptr [r11 + KTHREAD_PreviousMode], dil
1057
1058    /* Restore rdi and rsi */
1059    mov rsi, [rbp + KTRAP_FRAME_Rsi]
1060    mov rdi, [rbp + KTRAP_FRAME_Rdi]
1061
1062    /* Cleanup the stack and return */
1063    lea rsp, [rbp + KTRAP_FRAME_LENGTH]
1064    pop rbp
1065    ret
1066
1067ENDFUNC
1068
1069PUBLIC KiConvertToGuiThread
1070FUNC KiConvertToGuiThread
1071
1072    sub rsp, 40
1073    .allocstack 40
1074    .endprolog
1075
1076    /* Check if we already have a large stack */
1077    mov rax, gs:[PcCurrentThread]
1078    cmp byte ptr [rax + KTHREAD_LargeStack], 0
1079    jnz AlreadyLargeStack
1080
1081    // NewStack = (ULONG_PTR)MmCreateKernelStack(TRUE, 0);
1082    mov cl, 1
1083    xor rdx, rdx
1084    call MmCreateKernelStack
1085
1086    /* Check for failure */
1087    test rax, rax
1088    jz KiConvertToGuiThreadFailed
1089
1090    /* OldStack = KeSwitchKernelStack((PVOID)NewStack, (PVOID)(NewStack - KERNEL_LARGE_STACK_COMMIT )); */
1091    mov rcx, rax
1092    mov rdx, rax
1093    sub rdx, KERNEL_LARGE_STACK_COMMIT
1094    call KeSwitchKernelStack
1095
1096    // MmDeleteKernelStack(OldStack, FALSE);
1097    mov rcx, rax
1098    xor rdx, rdx
1099    call MmDeleteKernelStack
1100
1101AlreadyLargeStack:
1102
1103    /* Call the worker function */
1104    call PsConvertToGuiThread
1105
1106    /* Check for failure */
1107    test eax, eax
1108    js KiConvertToGuiThreadFailed
1109
1110    /* Disable interrupts for return */
1111    cli
1112
1113    // Restore register parameters
1114    mov rcx, [rsp + 48 + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rip]
1115    mov rdx, [rsp + 48 + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rdx]
1116    mov r8, [rsp + 48 + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_R8]
1117    mov r9, [rsp + 48 + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_R9]
1118
1119    /* Run KiSystemCallHandler again */
1120    add rsp, 48
1121    jmp KiSystemCall64Again
1122
1123KiConvertToGuiThreadFailed:
1124
1125    /* Clean up the stack and return failure */
1126    add rsp, 40
1127    mov eax, HEX(C0000017) // STATUS_NO_MEMORY
1128    ret
1129
1130ENDFUNC
1131
1132
1133EXTERN KiSetTrapContextInternal:PROC
1134
1135/*
1136 * VOID
1137 * KiSetTrapContext(
1138 *     _Out_ PKTRAP_FRAME TrapFrame,
1139 *     _In_ PCONTEXT Context,
1140 *     _In_ KPROCESSOR_MODE RequestorMode);
1141 */
1142PUBLIC KiSetTrapContext
1143.PROC KiSetTrapContext
1144
1145    /* Generate a KEXCEPTION_FRAME on the stack */
1146    GENERATE_EXCEPTION_FRAME
1147
1148    call KiSetTrapContextInternal
1149
1150    /* Restore the registers from the KEXCEPTION_FRAME */
1151    RESTORE_EXCEPTION_STATE
1152
1153    /* Return */
1154    ret
1155
1156.ENDP
1157
1158
1159/*
1160 * VOID
1161 * KiDeliverApc(
1162 *     _In_ KPROCESSOR_MODE DeliveryMode,
1163 *     _In_ PKEXCEPTION_FRAME ExceptionFrame,
1164 *     _In_ PKTRAP_FRAME TrapFrame);
1165 *
1166 */
1167EXTERN KiDeliverApc:PROC
1168
1169/*
1170 * VOID
1171 * KiInitiateUserApc(
1172 *     _In_ PKTRAP_FRAME TrapFrame@<rcx>);
1173 *
1174 * This function is called to deliver user mode APCs.
1175 * It clobbers all non-volatile registers, except rax.
1176 */
1177PUBLIC KiInitiateUserApc
1178.PROC KiInitiateUserApc
1179
1180    /* Generate a KEXCEPTION_FRAME on the stack */
1181    GENERATE_EXCEPTION_FRAME
1182
1183    /* Raise IRQL to APC_LEVEL */
1184    mov rax, APC_LEVEL
1185    mov cr8, rax
1186
1187    /* Get the current thread */
1188    mov rbp, gs:[PcCurrentThread]
1189
1190    /* Save the trap frame in rsi */
1191    mov rsi, rcx
1192
1193    /* Enable interrupts */
1194    sti
1195
1196    /* Call the C function */
1197    mov ecx, 1
1198    mov rdx, rsp
1199    mov r8, rsi
1200    call KiDeliverApc
1201
1202    /* Disable interrupts again */
1203    cli
1204
1205    /* Go back to PASSIVE_LEVEL */
1206    mov rax, PASSIVE_LEVEL
1207    mov cr8, rax
1208
1209    /* Restore the registers from the KEXCEPTION_FRAME */
1210    RESTORE_EXCEPTION_STATE
1211
1212    /* Return */
1213    ret
1214
1215.ENDP
1216
1217
1218PUBLIC KiInitializeSegments
1219KiInitializeSegments:
1220    mov ax, KGDT64_R3_DATA or RPL_MASK
1221    mov gs, ax
1222    swapgs
1223    mov gs, ax
1224    ret
1225
1226/*!
1227 * VOID
1228 * KiSwitchKernelStackHelper(
1229 *     LONG_PTR StackOffset,
1230 *     PVOID OldStackBase);
1231 */
1232PUBLIC KiSwitchKernelStackHelper
1233KiSwitchKernelStackHelper:
1234
1235    /* Pop return address from the current stack */
1236    pop rax
1237
1238    /* Switch to new stack */
1239    lea rsp, [rsp + rcx]
1240
1241    /* Push return address on the new stack */
1242    push rax
1243
1244    /* Return on new stack */
1245    mov rax, rdx
1246    ret
1247
1248EXTERN KiSwitchKernelStack:PROC
1249
1250PUBLIC KeSwitchKernelStack
1251FUNC KeSwitchKernelStack
1252
1253    /* Save rcx and allocate callee home space */
1254    mov [rsp + P1Home], rcx
1255    .savereg rcx, P1Home
1256    sub rsp, 40
1257    .allocstack 40
1258    .endprolog
1259
1260    /* Call the C handler, which returns the old stack in rax */
1261    call KiSwitchKernelStack
1262
1263    /* Restore rcx (StackBase) */
1264    mov rcx, [rsp + 40 + P1Home]
1265
1266    /* Switch to new stack: RSP += (StackBase - OldStackBase) */
1267    sub rcx, rax
1268    add rsp, rcx
1269
1270    /* Deallocate the home frame */
1271    add rsp, 40
1272    ret
1273
1274ENDFUNC
1275
1276
1277
1278
1279#ifdef _MSC_VER
1280#undef lgdt
1281#undef lidt
1282
1283//void __lgdt(void *Source);
1284PUBLIC __lgdt
1285__lgdt:
1286    lgdt fword ptr [rcx]
1287    ret
1288
1289//void __sgdt(void *Destination);
1290PUBLIC __sgdt
1291__sgdt:
1292    sgdt fword ptr [rcx]
1293    ret
1294
1295// void __lldt(unsigned short Value)
1296PUBLIC __lldt
1297__lldt:
1298    lldt cx
1299    ret
1300
1301//void __sldt(void *Destination);
1302PUBLIC __sldt
1303__sldt:
1304    sldt word ptr [rcx]
1305    ret
1306
1307//void __ltr(unsigned short Source);
1308PUBLIC __ltr
1309__ltr:
1310    ltr cx
1311    ret
1312
1313//void __str(unsigned short *Destination);
1314PUBLIC __str
1315__str:
1316    str word ptr [rcx]
1317    ret
1318
1319PUBLIC __swapgs
1320__swapgs:
1321    swapgs
1322    ret
1323
1324#endif
1325
1326END
1327