xref: /reactos/ntoskrnl/ke/amd64/trap.S (revision 2913ef5c) 
1/*
2 * FILE:            ntoskrnl/ke/amd64/trap.S
3 * COPYRIGHT:       See COPYING in the top level directory
4 * PURPOSE:         System Traps, Entrypoints and Exitpoints
5 * PROGRAMMER:      Timo Kreuzer (timo.kreuzer@reactos.org)
6 */
7
8/* INCLUDES ******************************************************************/
9
10#include <asm.inc>
11#include <ksamd64.inc>
12#include <trapamd64.inc>
13
14EXTERN KiDispatchException:PROC
15EXTERN KeBugCheckWithTf:PROC
16EXTERN MmAccessFault:PROC
17EXTERN KiSystemFatalException:PROC
18EXTERN KiNpxNotAvailableFaultHandler:PROC
19EXTERN KiGeneralProtectionFaultHandler:PROC
20EXTERN KiXmmExceptionHandler:PROC
21EXTERN KiDeliverApc:PROC
22EXTERN KiDpcInterruptHandler:PROC
23EXTERN PsConvertToGuiThread:PROC
24EXTERN MmCreateKernelStack:PROC
25EXTERN MmDeleteKernelStack:PROC
26
27EXTERN KdSetOwedBreakpoints:PROC
28
29
30/* Helper Macros *************************************************************/
31
32MACRO(DispatchException, Status, Number, P1, P2, P3)
33    mov eax, Status
34    mov edx, Number
35    mov r9, P1
36    mov r10, P2
37    mov r11, P3
38    call InternalDispatchException
39ENDM
40
41MACRO(Fatal, BugcheckCode)
42    /* Bugcheck */
43    mov ecx, BugcheckCode
44    mov rdx, rbp
45    call KiSystemFatalException
46ENDM
47
48
49/* FUNCTIONS *****************************************************************/
50
51.code64
52
53ALIGN 8
54
55MACRO(UnexpectedVectorStub, Vector)
56    /* This nop is to make the relative jmp address 4 bytes aligned and to
57       make the whole code 8 bytes long */
58    nop
59    /* This is a push instruction with 8bit operand. Since the instruction
60       sign extends the value to 32 bits, we need to offset it */
61PUBLIC KxUnexpectedInterrupt&Vector
62KxUnexpectedInterrupt&Vector:
63    push (Vector - 128)
64    jmp KiUnexpectedInterrupt
65ENDM
66
67PUBLIC KiUnexpectedRange
68KiUnexpectedRange:
69Vector = 0
70REPEAT 256
71    UnexpectedVectorStub %Vector
72    Vector = Vector+1
73ENDR
74PUBLIC KiUnexpectedRangeEnd
75KiUnexpectedRangeEnd:
76
77PUBLIC KiInterruptDispatchTemplate
78KiInterruptDispatchTemplate:
79    /* This instruction pushes the return address on the stack, which is the
80       address of the interrupt object's DispatchCode member, then jumps
81       to the address stored in the interrupt object's DispatchAddress member */
82    call qword ptr KiInterruptDispatchTemplate[rip - KINTERRUPT_DispatchCode + KINTERRUPT_DispatchAddress]
83
84
85// rbp = TrapFrame, eax = ExceptionCode, edx = NumParams, r9,r10,r11 = params
86FUNC InternalDispatchException
87
88    /* Allocate stack space for EXCEPTION_RECORD and KEXCEPTION_FRAME */
89    sub rsp, EXCEPTION_RECORD_LENGTH + KEXCEPTION_FRAME_LENGTH
90    .allocstack (EXCEPTION_RECORD_LENGTH + KEXCEPTION_FRAME_LENGTH)
91    .endprolog
92
93    /* Set up EXCEPTION_RECORD */
94    lea rcx, [rsp + KEXCEPTION_FRAME_LENGTH]
95    mov [rcx + EXCEPTION_RECORD_ExceptionCode], eax
96    xor rax, rax
97    mov [rcx + EXCEPTION_RECORD_ExceptionFlags], eax
98    mov [rcx + EXCEPTION_RECORD_ExceptionRecord], rax
99    mov rax, [rbp + KTRAP_FRAME_Rip]
100    mov [rcx + EXCEPTION_RECORD_ExceptionAddress], rax
101    mov [rcx + EXCEPTION_RECORD_NumberParameters], edx
102    mov [rcx + EXCEPTION_RECORD_ExceptionInformation + HEX(00)], r9
103    mov [rcx + EXCEPTION_RECORD_ExceptionInformation + HEX(08)], r10
104    mov [rcx + EXCEPTION_RECORD_ExceptionInformation + HEX(10)], r11
105
106    /* Set up KEXCEPTION_FRAME */
107    mov rax, [rbp + KTRAP_FRAME_Rbp]
108    mov [rsp + KEXCEPTION_FRAME_Rbp], rax
109    mov [rsp + KEXCEPTION_FRAME_Rbx], rbx
110    mov [rsp + KEXCEPTION_FRAME_Rdi], rdi
111    mov [rsp + KEXCEPTION_FRAME_Rsi], rsi
112    mov [rsp + KEXCEPTION_FRAME_R12], r12
113    mov [rsp + KEXCEPTION_FRAME_R13], r13
114    mov [rsp + KEXCEPTION_FRAME_R14], r14
115    mov [rsp + KEXCEPTION_FRAME_R15], r15
116    movdqa [rsp + KEXCEPTION_FRAME_Xmm6], xmm6
117    movdqa [rsp + KEXCEPTION_FRAME_Xmm7], xmm7
118    movdqa [rsp + KEXCEPTION_FRAME_Xmm8], xmm8
119    movdqa [rsp + KEXCEPTION_FRAME_Xmm9], xmm9
120    movdqa [rsp + KEXCEPTION_FRAME_Xmm10], xmm10
121    movdqa [rsp + KEXCEPTION_FRAME_Xmm11], xmm11
122    movdqa [rsp + KEXCEPTION_FRAME_Xmm12], xmm12
123    movdqa [rsp + KEXCEPTION_FRAME_Xmm13], xmm13
124    movdqa [rsp + KEXCEPTION_FRAME_Xmm14], xmm14
125    movdqa [rsp + KEXCEPTION_FRAME_Xmm15], xmm15
126    mov qword ptr [rsp + KEXCEPTION_FRAME_Return], 0
127
128    /* Call KiDispatchException */
129    // rcx already points to ExceptionRecord
130    mov rdx, rsp                                // ExceptionFrame
131    mov r8, rbp                                 // TrapFrame
132    mov r9b, [r8 + KTRAP_FRAME_PreviousMode]    // PreviousMode
133    mov byte ptr [rsp + KEXCEPTION_FRAME_P5], 1 // FirstChance
134    call KiDispatchException
135
136    /* Restore registers */
137    mov r12, [rsp + KEXCEPTION_FRAME_R12]
138    mov r13, [rsp + KEXCEPTION_FRAME_R13]
139    mov r14, [rsp + KEXCEPTION_FRAME_R14]
140    mov r15, [rsp + KEXCEPTION_FRAME_R15]
141    movdqa xmm6, [rsp + KEXCEPTION_FRAME_Xmm6]
142    movdqa xmm7, [rsp + KEXCEPTION_FRAME_Xmm7]
143    movdqa xmm8, [rsp + KEXCEPTION_FRAME_Xmm8]
144    movdqa xmm9, [rsp + KEXCEPTION_FRAME_Xmm9]
145    movdqa xmm10, [rsp + KEXCEPTION_FRAME_Xmm10]
146    movdqa xmm11, [rsp + KEXCEPTION_FRAME_Xmm11]
147    movdqa xmm12, [rsp + KEXCEPTION_FRAME_Xmm12]
148    movdqa xmm13, [rsp + KEXCEPTION_FRAME_Xmm13]
149    movdqa xmm14, [rsp + KEXCEPTION_FRAME_Xmm14]
150    movdqa xmm15, [rsp + KEXCEPTION_FRAME_Xmm15]
151
152    add rsp, EXCEPTION_RECORD_LENGTH + KEXCEPTION_FRAME_LENGTH
153    ret
154ENDFUNC
155
156
157/* CPU EXCEPTION HANDLERS ****************************************************/
158
159PUBLIC KiDivideErrorFault
160FUNC KiDivideErrorFault
161    /* Push pseudo error code */
162    EnterTrap TF_SAVE_ALL
163
164    /* Enable interrupts */
165    sti
166
167    /* Dispatch the exception */
168    DispatchException STATUS_INTEGER_DIVIDE_BY_ZERO, 0, 0, 0, 0
169
170    /* Return */
171    ExitTrap TF_SAVE_ALL
172ENDFUNC
173
174
175PUBLIC KiDebugTrapOrFault
176FUNC KiDebugTrapOrFault
177    /* Push pseudo error code */
178    EnterTrap TF_SAVE_ALL
179
180    /* Check if the frame was from kernelmode */
181    test word ptr [rbp + KTRAP_FRAME_SegCs], 3
182    jz KiDebugTrapOrFaultKMode
183
184    /* Enable interrupts for user-mode */
185    sti
186
187KiDebugTrapOrFaultKMode:
188    /* Dispatch the exception */
189    DispatchException STATUS_SINGLE_STEP, 0, 0, 0, 0
190
191    /* Return */
192    ExitTrap TF_SAVE_ALL
193ENDFUNC
194
195EXTERN KiNmiInterruptHandler:PROC
196
197FUNC KiNmiInterruptWithEf
198    /* Generate a KEXCEPTION_FRAME on the stack */
199    GENERATE_EXCEPTION_FRAME
200
201    /* Call the C handler */
202    lea rcx, [rsp + KEXCEPTION_FRAME_LENGTH]
203    lea rdx, [rsp]
204    call KiNmiInterruptHandler
205
206    /* Restore the registers from the KEXCEPTION_FRAME */
207    RESTORE_EXCEPTION_STATE
208
209    /* Return */
210    ret
211ENDFUNC
212
213PUBLIC KiNmiInterrupt
214FUNC KiNmiInterrupt
215    /* Push pseudo error code */
216    EnterTrap TF_SAVE_ALL
217
218    call KiNmiInterruptWithEf
219
220    /* Return */
221    ExitTrap TF_SAVE_ALL
222ENDFUNC
223
224
225PUBLIC KiBreakpointTrap
226FUNC KiBreakpointTrap
227    /* Push pseudo error code */
228    EnterTrap TF_SAVE_ALL
229
230    /* Check if the frame was from kernelmode */
231    test word ptr [rbp + KTRAP_FRAME_SegCs], 3
232    jz KiBreakpointTrapKMode
233
234    /* Enable interrupts for user-mode */
235    sti
236
237KiBreakpointTrapKMode:
238    /* Dispatch the exception */
239    DispatchException STATUS_BREAKPOINT, 3, BREAKPOINT_BREAK, 0, 0
240
241    /* Return */
242    ExitTrap TF_SAVE_ALL
243ENDFUNC
244
245
246PUBLIC KiOverflowTrap
247FUNC KiOverflowTrap
248    /* Push pseudo error code */
249    EnterTrap TF_SAVE_ALL
250
251    /* Enable interrupts */
252    sti
253
254    /* Dispatch the exception */
255    DispatchException STATUS_INTEGER_OVERFLOW, 3, 0, 0, 0
256
257    /* Return */
258    ExitTrap TF_SAVE_ALL
259ENDFUNC
260
261
262PUBLIC KiBoundFault
263FUNC KiBoundFault
264    /* Push pseudo error code */
265    EnterTrap TF_SAVE_ALL
266
267    /* Check if the frame was from kernelmode */
268    test word ptr [rbp + KTRAP_FRAME_SegCs], 3
269    jnz KiBoundFaultUserMode
270
271    /* Bugcheck */
272    Fatal EXCEPTION_BOUND_CHECK
273
274KiBoundFaultUserMode:
275    /* Enable interrupts for user-mode */
276    sti
277
278    /* Dispatch the exception */
279    DispatchException STATUS_ARRAY_BOUNDS_EXCEEDED, 0, 0, 0, 0
280
281    /* Return */
282    ExitTrap TF_SAVE_ALL
283ENDFUNC
284
285
286PUBLIC KiInvalidOpcodeFault
287FUNC KiInvalidOpcodeFault
288    /* Push pseudo error code */
289    EnterTrap TF_SAVE_ALL
290
291    /* Enable interrupts */
292    sti
293
294    /* Check if the frame was from kernelmode */
295    test word ptr [rbp + KTRAP_FRAME_SegCs], 3
296    jz KiInvalidOpcodeKernel
297
298    // FIXME: handle STATUS_INVALID_LOCK_SEQUENCE
299
300KiInvalidOpcodeKernel:
301    /* Kernel mode fault */
302
303    /* Dispatch the exception */
304    DispatchException STATUS_ILLEGAL_INSTRUCTION, 0, 0, 0, 0
305
306    /* Return */
307    ExitTrap TF_SAVE_ALL
308ENDFUNC
309
310
311PUBLIC KiNpxNotAvailableFault
312FUNC KiNpxNotAvailableFault
313    /* Push pseudo error code */
314    EnterTrap TF_SAVE_ALL
315
316    /* Call the C handler */
317    mov rcx, rbp
318    call KiNpxNotAvailableFaultHandler
319
320    /* Check the return status code */
321    test eax, eax
322    jz KiNpxNotAvailableFaultExit
323
324    /* Dispatch the exception */
325    DispatchException eax, 3, 0, 0, 0
326
327KiNpxNotAvailableFaultExit:
328    /* Return */
329    ExitTrap TF_SAVE_ALL
330ENDFUNC
331
332
333PUBLIC KiDoubleFaultAbort
334FUNC KiDoubleFaultAbort
335
336    /* Hack for VBox, which "forgets" to push an error code on the stack! */
337    and rsp, HEX(FFFFFFFFFFFFFFF0)
338
339    /* A zero error code is pushed */
340    EnterTrap (TF_HAS_ERROR_CODE OR TF_SAVE_ALL)
341
342    int 3
343
344    /* Bugcheck */
345    Fatal 8 // EXCEPTION_DOUBLE_FAULT
346    jmp $
347ENDFUNC
348
349
350PUBLIC KiNpxSegmentOverrunAbort
351FUNC KiNpxSegmentOverrunAbort
352    /* Push pseudo error code */
353    EnterTrap TF_SAVE_ALL
354
355    /* Bugcheck */
356    Fatal EXCEPTION_NPX_OVERRUN
357
358    jmp $
359ENDFUNC
360
361
362PUBLIC KiInvalidTssFault
363FUNC KiInvalidTssFault
364    /* We have an error code */
365    EnterTrap (TF_HAS_ERROR_CODE OR TF_SAVE_ALL)
366
367    /* Bugcheck */
368    Fatal EXCEPTION_INVALID_TSS
369    jmp $
370ENDFUNC
371
372
373PUBLIC KiSegmentNotPresentFault
374FUNC KiSegmentNotPresentFault
375    /* We have an error code */
376    EnterTrap (TF_HAS_ERROR_CODE OR TF_SAVE_ALL)
377
378    /* Bugcheck */
379    Fatal EXCEPTION_SEGMENT_NOT_PRESENT
380    jmp $
381ENDFUNC
382
383
384PUBLIC KiStackFault
385FUNC KiStackFault
386    /* We have an error code */
387    EnterTrap (TF_HAS_ERROR_CODE OR TF_SAVE_ALL)
388
389    /* Bugcheck */
390    Fatal EXCEPTION_STACK_FAULT
391    jmp $
392ENDFUNC
393
394
395PUBLIC KiGeneralProtectionFault
396FUNC KiGeneralProtectionFault
397    /* We have an error code */
398    EnterTrap (TF_HAS_ERROR_CODE OR TF_SAVE_ALL)
399
400    /* Call the C handler */
401    mov rcx, rbp
402    call KiGeneralProtectionFaultHandler
403
404    /* Check for success */
405    test eax, eax
406    jge KiGpfExit
407
408    /* Check for access violation */
409    cmp eax, STATUS_ACCESS_VIOLATION
410    je DispatchAccessViolation
411
412    /* Dispatch privileged instruction fault */
413    DispatchException eax, 0, 0, 0, 0
414    jmp KiGpfExit
415
416DispatchAccessViolation:
417
418    /* Dispatch access violation */
419    DispatchException eax, 2, 0, -1, 0
420
421KiGpfExit:
422    /* Return */
423    ExitTrap TF_SAVE_ALL
424ENDFUNC
425
426
427PUBLIC KiPageFault
428FUNC KiPageFault
429    /* We have an error code */
430    EnterTrap (TF_HAS_ERROR_CODE OR TF_SAVE_ALL)
431
432    /* Save page fault address */
433    mov rdx, cr2
434    mov [rbp  + KTRAP_FRAME_FaultAddress], rdx
435
436    /* If interrupts are off, do not enable them */
437    test dword ptr [rbp + KTRAP_FRAME_EFlags], EFLAGS_IF_MASK
438    jz IntsDisabled
439
440    /* Enable interrupts for the page fault handler */
441    sti
442
443IntsDisabled:
444
445    /* Call page fault handler */
446    mov ecx, [rbp + KTRAP_FRAME_ErrorCode] // FaultCode
447    // rdx == Address
448    mov r8b, [rbp + KTRAP_FRAME_SegCs] // Mode
449    and r8b, 1
450    mov r9, rbp // TrapInformation
451    call MmAccessFault
452
453    /* Check for success */
454    test eax, eax
455    jl PageFaultError
456
457    /* Check whether the kernel debugger has owed breakpoints to be inserted */
458    call KdSetOwedBreakpoints
459    /* We succeeded, return */
460    jmp PageFaultReturn
461
462PageFaultError:
463
464    /* Set parameter 1 to write/execute flag.
465       See https://learn.microsoft.com/en-us/windows/win32/api/winnt/ns-winnt-exception_record */
466    mov r9d, [rbp + KTRAP_FRAME_ErrorCode]
467    shr r9d, 1
468    and r9d, 9
469
470    /* Set parameter 2 to faulting address */
471    mov r10, cr2  // Param2 = faulting address
472
473    cmp eax, STATUS_ACCESS_VIOLATION
474    je AccessViolation
475    cmp eax, STATUS_GUARD_PAGE_VIOLATION
476    je SpecialCode
477    cmp eax, STATUS_STACK_OVERFLOW
478    je SpecialCode
479
480InPageException:
481    /* Dispatch in-page exception */
482    mov r11d, eax // Param3 = Status
483    mov eax, STATUS_IN_PAGE_ERROR // ExceptionCode
484    mov edx, 3    // ParamCount
485    call InternalDispatchException
486    jmp PageFaultReturn
487
488AccessViolation:
489    /* Use more proper status code */
490    mov eax, KI_EXCEPTION_ACCESS_VIOLATION
491
492SpecialCode:
493    /* Setup a normal page fault exception */
494    mov edx, 2   // ParamCount
495    call InternalDispatchException
496
497PageFaultReturn:
498
499    /* Disable interrupts for the return */
500    cli
501
502    /* Return */
503    ExitTrap (TF_SAVE_ALL or TF_CHECKUSERAPC)
504ENDFUNC
505
506
507PUBLIC KiFloatingErrorFault
508FUNC KiFloatingErrorFault
509    /* Push pseudo error code */
510    EnterTrap TF_SAVE_ALL
511
512    UNIMPLEMENTED KiFloatingErrorFault
513    int 3
514
515    /* Return */
516    ExitTrap TF_SAVE_ALL
517ENDFUNC
518
519
520PUBLIC KiAlignmentFault
521FUNC KiAlignmentFault
522    /* We have an error code */
523    EnterTrap (TF_HAS_ERROR_CODE OR TF_SAVE_ALL)
524
525    /* Bugcheck */
526    Fatal EXCEPTION_ALIGNMENT_CHECK
527    jmp $
528ENDFUNC
529
530
531PUBLIC KiMcheckAbort
532FUNC KiMcheckAbort
533    /* Push pseudo error code */
534    EnterTrap TF_SAVE_ALL
535
536    /* Bugcheck */
537    Fatal HEX(12)
538    jmp $
539ENDFUNC
540
541
542PUBLIC KiXmmException
543FUNC KiXmmException
544    /* Push pseudo error code */
545    EnterTrap TF_SAVE_ALL
546
547    /* Call the C handler */
548    mov rcx, rbp
549    call KiXmmExceptionHandler
550
551    /* Check for success */
552    test eax, eax
553    jge KiXmmExit
554
555    /* Dispatch the exception */
556    DispatchException eax, 2, 0, [rbp+KTRAP_FRAME_MxCsr], 0
557
558    // FIXME: STATUS_FLOAT_MULTIPLE_TRAPS / STATUS_FLOAT_MULTIPLE_FAULTS
559
560KiXmmExit:
561    /* Return */
562    ExitTrap TF_SAVE_ALL
563ENDFUNC
564
565
566/* SOFTWARE INTERRUPT SERVICES ***********************************************/
567
568PUBLIC KiRaiseAssertion
569FUNC KiRaiseAssertion
570    /* We have an error code */
571    EnterTrap (TF_SAVE_ALL)
572
573    /* Decrement RIP to point to the INT2C instruction (2 bytes, not 1 like INT3) */
574    sub qword ptr [rbp + KTRAP_FRAME_Rip], 2
575
576    /* Dispatch the exception */
577    DispatchException STATUS_ASSERTION_FAILURE, 0, 0, 0, 0
578
579    /* Return */
580    ExitTrap TF_SAVE_ALL
581ENDFUNC
582
583
584PUBLIC KiDebugServiceTrap
585FUNC KiDebugServiceTrap
586   /* No error code */
587    EnterTrap TF_SAVE_ALL
588
589    /* Increase Rip to skip the int3 */
590    inc qword ptr [rbp + KTRAP_FRAME_Rip]
591
592    /* Dispatch the exception (Params = service, buffer, legth) */
593    DispatchException STATUS_BREAKPOINT, 3, [rbp+KTRAP_FRAME_Rax], [rbp+KTRAP_FRAME_Rcx], [rbp+KTRAP_FRAME_Rdx]
594
595    /* Return */
596    ExitTrap TF_SAVE_ALL
597ENDFUNC
598
599
600PUBLIC KiApcInterrupt
601.PROC KiApcInterrupt
602   /* No error code */
603    EnterTrap (TF_SAVE_ALL or TF_IRQL)
604
605    /* Raise to APC_LEVEL */
606    mov rax, APC_LEVEL
607    mov cr8, rax
608
609    /* End the interrupt */
610    mov dword ptr [APIC_EOI], 0
611
612    /* Enable interrupts  */
613    sti
614
615    /* Call the worker routine */
616    mov cl, [rbp + KTRAP_FRAME_SegCs] // ProcessorMode
617    and cl, 1
618    mov rdx, 0                        // ExceptionFrame
619    mov r8, rbp                       // TrapFrame
620    call KiDeliverApc
621
622    /* Disable interrupts */
623    cli
624
625    /* Lower IRQL back to PASSIVE */
626    mov rax, PASSIVE_LEVEL
627    mov cr8, rax
628
629    /* Return */
630    ExitTrap (TF_SAVE_ALL or TF_IRQL)
631.ENDP
632
633/*
634 * VOID
635 * KiRetireDpcList(
636 *     PKPRCB Prcb);
637 */
638EXTERN KiRetireDpcList:PROC
639
640/*
641 * VOID
642 * KiRetireDpcListInDpcStack(
643 *     PKPRCB Prcb,
644 *     PVOID DpcStack);
645 */
646PUBLIC KiRetireDpcListInDpcStack
647.PROC KiRetireDpcListInDpcStack
648    push rbp
649    .pushreg rbp
650    mov rbp, rsp
651    .setframe rbp, 0
652    .endprolog
653
654    /* Switch to the DpcStack */
655    mov rsp, rdx
656
657    /* The stack is 16 byte aligned, allocate 32 bytes home space */
658    sub rsp, 32
659
660    /* Call KiRetireDpcList on the given stack */
661    call KiRetireDpcList
662
663    /* Restore stack, cleanup and return */
664    mov rsp, rbp
665    pop rbp
666    ret
667.ENDP
668
669PUBLIC KiDpcInterrupt
670.PROC KiDpcInterrupt
671   /* No error code */
672    EnterTrap (TF_SAVE_ALL or TF_IRQL)
673
674    /* Call the worker routine */
675    call KiDpcInterruptHandler
676
677    /* Return, but don't send an EOI! */
678    ExitTrap (TF_SAVE_ALL or TF_IRQL)
679.ENDP
680
681
682PUBLIC KiIpiInterrupt
683.PROC KiIpiInterrupt
684   /* No error code */
685    EnterTrap (TF_SAVE_ALL or TF_IRQL)
686
687    /* Raise to IPI_LEVEL */
688    mov rax, IPI_LEVEL
689    mov cr8, rax
690
691    /* End the interrupt */
692    mov dword ptr [APIC_EOI], 0
693
694    int 3
695
696    /* Return */
697    ExitTrap (TF_SAVE_ALL or TF_IRQL)
698.ENDP
699
700
701PUBLIC KiUnexpectedInterrupt
702FUNC KiUnexpectedInterrupt
703    /* The error code is the vector */
704    EnterTrap (TF_HAS_ERROR_CODE OR TF_SAVE_ALL)
705
706#if 0
707    /* Set bugcheck parameters */
708    mov ecx, TRAP_CAUSE_UNKNOWN
709    mov rdx, [rbp + KTRAP_FRAME_ErrorCode] // the vector
710    mov r8, 0 // The unknown floating-point exception
711    mov r9, 0 // The enabled and asserted status bits
712    sub rsp, 8
713    mov [rbp + KTRAP_FRAME_P5 + 8], rbp // trap frame
714    call KeBugCheckWithTf
715    jmp $
716#endif
717    /* Return */
718    ExitTrap TF_SAVE_ALL
719ENDFUNC
720
721PUBLIC KiInterruptDispatch
722FUNC KiInterruptDispatch
723    /* The error code is a pointer to the interrupt object's code */
724    EnterTrap (TF_HAS_ERROR_CODE or TF_SAVE_ALL or TF_IRQL)
725
726    /* Increase interrupt count */
727    inc dword ptr gs:[PcInterruptCount];
728
729    /* Save rbx and rsi in the trap frame */
730    mov [rbp + KTRAP_FRAME_Rbx], rbx
731    mov [rbp + KTRAP_FRAME_Rsi], rsi
732
733    /* Load the address of the dispatch code into rbx */
734    mov rbx, [rbp + KTRAP_FRAME_ErrorCode]
735
736    /* Substract offset of the DispatchCode member plus 6 for the call instruction */
737    sub rbx, KINTERRUPT_DispatchCode + 6
738
739    /* Save the address of the InterruptListEntry in rsi */
740    lea rsi, [rbx + KINTERRUPT_InterruptListEntry]
741
742.DoDispatchInterrupt:
743    /* Raise IRQL to SynchronizeIrql */
744    movzx rax, byte ptr [rbx + KINTERRUPT_SynchronizeIrql]
745    mov cr8, rax
746
747#ifdef CONFIG_SMP
748    /* Acquire interrupt lock */
749    mov r8, [rbx + KINTERRUPT_ActualLock]
750
751    //KxAcquireSpinLock(Interrupt->ActualLock);
752#endif
753
754    /* Call the ISR */
755    mov rcx, rbx
756    mov rdx, [rbx + KINTERRUPT_ServiceContext]
757    call qword ptr [rbx + KINTERRUPT_ServiceRoutine]
758
759#ifdef CONFIG_SMP
760    /* Release interrupt lock */
761    //KxReleaseSpinLock(Interrupt->ActualLock);
762#endif
763
764    /* Go back to old irql */
765    movzx rax, byte ptr [rbp + KTRAP_FRAME_PreviousIrql]
766    mov cr8, rax
767
768    /* Check for chained interrupts */
769    mov rax, [rbx + KINTERRUPT_InterruptListEntry]
770    cmp rax, rsi
771    je .Done
772
773    /* Load the next interrupt object into rbx and repeat */
774    lea rbx, [rax - KINTERRUPT_InterruptListEntry]
775    jmp .DoDispatchInterrupt
776
777.Done:
778    /* Restore rbx and rsi */
779    mov rbx, [rbp + KTRAP_FRAME_Rbx]
780    mov rsi, [rbp + KTRAP_FRAME_Rsi]
781
782    /* Return */
783    ExitTrap (TF_SAVE_ALL or TF_SEND_EOI)
784ENDFUNC
785
786EXTERN KiSystemCallHandler:PROC
787
788/*! \name KiSystemCallEntry64
789 *
790 *  \brief This is the entrypoint for syscalls from 64bit user mode
791 *
792 *  \param rax - The system call number
793 *  \param rcx - User mode return address, set by the syscall instruction
794 *  \param rdx,r8,r9 - Parameters 2-4 to the service function
795 *  \param r10 - Parameter 1 to the service function
796 *  \param r11 - RFLAGS saved by the syscall instruction
797 *--*/
798PUBLIC KiSystemCallEntry64
799.PROC KiSystemCallEntry64
800    /* The unwind info pretends we have a machine frame */
801    .PUSHFRAME
802    .ALLOCSTACK (KTRAP_FRAME_LENGTH + MAX_SYSCALL_PARAM_SIZE - MachineFrameLength)
803    .SAVEREG rbp, MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rbp
804    .ENDPROLOG
805
806    /* Swap gs to kernel, so we can access the PCR */
807    swapgs
808
809    /* Save the user mode rsp in the PCR */
810    mov gs:[PcUserRsp], rsp
811
812    /* Get the kernel stack from the PCR */
813    mov rsp, gs:[PcRspBase]
814
815    /* Allocate a TRAP_FRAME and space for parameters */
816    sub rsp, (KTRAP_FRAME_LENGTH + MAX_SYSCALL_PARAM_SIZE)
817
818    /* Save volatile registers and rbp in the trap frame */
819    mov [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rax], rax
820    mov [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rip], rcx
821    mov [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rdx], rdx
822    mov [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_R8], r8
823    mov [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_R9], r9
824    mov [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rcx], r10
825    mov [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_EFlags], r11
826    mov [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rbp], rbp
827
828    /* Store user stack pointer in the trap frame */
829    mov rax, gs:[PcUserRsp]
830    mov [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rsp], rax
831
832    /* Set sane segments */
833    mov ax, (KGDT64_R3_DATA or RPL_MASK)
834    mov ds, ax
835    mov es, ax
836
837    /* Save MCXSR and set kernel value */
838    stmxcsr [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_MxCsr]
839    ldmxcsr gs:[PcMxCsr]
840
841    /* Get the current thread and the trap frame */
842    mov rax, gs:[PcCurrentThread]
843    mov rcx, [rax + ThTrapFrame]
844
845    /* Save the old trap frame */
846    lea rdx, [rsp + MAX_SYSCALL_PARAM_SIZE]
847    mov [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_TrapFrame], rcx
848    mov [rax + ThTrapFrame], rdx
849
850#if DBG
851    /* Check IRQL */
852    mov rax, cr8
853    test eax, eax
854    jz KiSystemCall64Again
855    int HEX(2C)
856#endif
857
858GLOBAL_LABEL KiSystemCall64Again
859
860    /* Call the C-handler (will enable interrupts) */
861    call KiSystemCallHandler
862
863    /* The return value from KiSystemCallHandler is the address of the Nt-function */
864    mov rcx, [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rcx]
865    mov rdx, [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rdx]
866    mov r8, [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_R8]
867    mov r9, [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_R9]
868    call rax
869
870GLOBAL_LABEL KiSystemServiceExit
871
872    ASSERT_TRAP_FRAME_INTS_ENABLED rsp + MAX_SYSCALL_PARAM_SIZE
873
874    /* Check for pending user APCs */
875    mov rcx, gs:[PcCurrentThread]
876    cmp byte ptr [rcx + ThApcState + AsUserApcPending], 0
877    jz NoUserApcPending
878
879    /* Save missing regs in the trap frame */
880    mov [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rax], rax
881    mov [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rbp], rbp
882    mov [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_R9], rbp
883    mov rax, [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rsp]
884    mov [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_R8], rax
885    mov rax, [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rip]
886    mov [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rcx], rax
887    mov rax, [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_EFlags]
888    mov [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_R11], rax
889    xor rax, rax
890    mov [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rdx], rax
891    mov [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_R10], rax
892    pxor xmm0, xmm0
893    movdqa [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Xmm0], xmm0
894    movdqa [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Xmm1], xmm0
895    movdqa [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Xmm2], xmm0
896    movdqa [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Xmm3], xmm0
897    movdqa [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Xmm4], xmm0
898    movdqa [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Xmm5], xmm0
899
900    lea rcx, [rsp + MAX_SYSCALL_PARAM_SIZE]
901    call KiInitiateUserApc
902
903NoUserApcPending:
904    /* Disable interrupts for return */
905    cli
906
907    /* Restore MCXSR */
908    ldmxcsr [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_MxCsr]
909
910    /* Restore old trap frame */
911    mov rcx, gs:[PcCurrentThread]
912    mov rdx, [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_TrapFrame]
913    mov [rcx + KTHREAD_TrapFrame], rdx
914
915    /* Prepare user mode return address (rcx) and eflags (r11) for sysret */
916    mov rcx, [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rip]
917    mov r11, [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_EFlags]
918
919    /* Load user mode stack (It was copied to the trap frame) */
920    mov rsp, [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rsp]
921
922    /* r8 points to the user stack */
923    mov r8, rsp
924
925    /* r9 matches rbp */
926    mov r9, rbp
927
928    /* Swap gs back to user */
929    swapgs
930
931    /* Zero out volatiles */
932    pxor xmm0, xmm0
933    pxor xmm1, xmm1
934    pxor xmm2, xmm2
935    pxor xmm3, xmm3
936    pxor xmm4, xmm4
937    pxor xmm5, xmm5
938    xor rdx, rdx
939    xor r10, r10
940
941    /* return to user mode */
942    sysretq
943
944.ENDP
945
946
947/*!
948 * VOID
949 * DECLSPEC_NORETURN
950 * KiExceptionExit(
951 *     _In_ PKTRAP_FRAME TrapFrame@<rcx>,
952 *     _In_ PKEXCEPTION_FRAME ExceptionFrame@<rdx>);
953 */
954PUBLIC KiExceptionExit
955KiExceptionExit:
956
957    /* Restore registers from exception frame */
958    movaps xmm6, [rdx + ExXmm6]
959    movaps xmm7, [rdx + ExXmm7]
960    movaps xmm8, [rdx + ExXmm8]
961    movaps xmm9, [rdx + ExXmm9]
962    movaps xmm10, [rdx + ExXmm10]
963    movaps xmm11, [rdx + ExXmm11]
964    movaps xmm12, [rdx + ExXmm12]
965    movaps xmm13, [rdx + ExXmm13]
966    movaps xmm14, [rdx + ExXmm14]
967    movaps xmm15, [rdx + ExXmm15]
968    mov rbx, [rdx + ExRbx]
969    mov rdi, [rdx + ExRdi]
970    mov rsi, [rdx + ExRsi]
971    mov r12, [rdx + ExR12]
972    mov r13, [rdx + ExR13]
973    mov r14, [rdx + ExR14]
974    mov r15, [rdx + ExR15]
975
976    /* Point rsp at the trap frame */
977    mov rsp, rcx
978    /* Fall through */
979
980/*
981 * Internal function. Exits to user-mode with rsp pointing to the trap frame.
982 * All non-volatile register context must be set up already.
983 * Used by KiInitializeContextThread to set up the init path for a new thread.
984 */
985PUBLIC KiServiceExit3
986.PROC KiServiceExit3
987    .PUSHFRAME
988    .ALLOCSTACK (KTRAP_FRAME_LENGTH - MachineFrameLength)
989    .ENDPROLOG
990
991    /* Return */
992    mov rbp, rsp
993    ExitTrap (TF_SEGMENTS or TF_CHECKUSERAPC)
994.ENDP
995
996
997PUBLIC KiSystemCallEntry32
998KiSystemCallEntry32:
999    swapgs
1000    int 3
1001
1002
1003PUBLIC KiZwSystemService
1004FUNC KiZwSystemService
1005    push rbp
1006    .pushreg rbp
1007    sub rsp, KTRAP_FRAME_LENGTH
1008    .allocstack KTRAP_FRAME_LENGTH
1009    mov [rsp + KTRAP_FRAME_Rsi], rsi
1010    .savereg rsi, KTRAP_FRAME_Rsi
1011    mov [rsp + KTRAP_FRAME_Rdi], rdi
1012    .savereg rdi, KTRAP_FRAME_Rdi
1013    mov rbp, rsp
1014    .setframe rbp, 0
1015    .endprolog
1016
1017    /* Get current thread */
1018    mov r11, gs:[PcCurrentThread]
1019
1020    /* Save PreviousMode in the trap frame */
1021    mov dil, byte ptr [r11 + KTHREAD_PreviousMode]
1022    mov byte ptr [rbp + KTRAP_FRAME_PreviousMode], dil
1023
1024    /* Save the old trap frame in TrapFrame.Rdx */
1025    mov rdi, [r11 + KTHREAD_TrapFrame]
1026    mov [rbp + KTRAP_FRAME_Rdx], rdi
1027
1028    /* Set the new trap frame and previous mode */
1029    mov [r11 + ThTrapFrame], rbp
1030    mov byte ptr [r11 + KTHREAD_PreviousMode], 0
1031
1032    /* allocate space for parameters */
1033    sub rsp, r10
1034    and rsp, HEX(0fffffffffffffff0)
1035
1036    /* Save rcx */
1037    mov [rbp + KTRAP_FRAME_Rcx], rcx
1038
1039    /* copy parameters to the new location */
1040    lea rsi, [rbp + KTRAP_FRAME_LENGTH + 16]
1041    lea rdi, [rsp]
1042    mov rcx, r10
1043    shr rcx, 3
1044    rep movsq
1045
1046    /* Restore rcx */
1047    mov rcx, [rbp + KTRAP_FRAME_Rcx]
1048
1049    /* Call the service function */
1050    call rax
1051
1052    /* Restore the old trap frame */
1053    mov r11, gs:[PcCurrentThread]
1054    mov rsi, [rbp + KTRAP_FRAME_Rdx]
1055    mov [r11 + KTHREAD_TrapFrame], rsi
1056
1057    /* Restore PreviousMode from the trap frame */
1058    mov dil, byte ptr [rbp + KTRAP_FRAME_PreviousMode]
1059    mov byte ptr [r11 + KTHREAD_PreviousMode], dil
1060
1061    /* Restore rdi and rsi */
1062    mov rsi, [rbp + KTRAP_FRAME_Rsi]
1063    mov rdi, [rbp + KTRAP_FRAME_Rdi]
1064
1065    /* Cleanup the stack and return */
1066    lea rsp, [rbp + KTRAP_FRAME_LENGTH]
1067    pop rbp
1068    ret
1069
1070ENDFUNC
1071
1072PUBLIC KiConvertToGuiThread
1073FUNC KiConvertToGuiThread
1074
1075    sub rsp, 40
1076    .allocstack 40
1077    .endprolog
1078
1079    /* Check if we already have a large stack */
1080    mov rax, gs:[PcCurrentThread]
1081    cmp byte ptr [rax + KTHREAD_LargeStack], 0
1082    jnz AlreadyLargeStack
1083
1084    // NewStack = (ULONG_PTR)MmCreateKernelStack(TRUE, 0);
1085    mov cl, 1
1086    xor rdx, rdx
1087    call MmCreateKernelStack
1088
1089    /* Check for failure */
1090    test rax, rax
1091    jz KiConvertToGuiThreadFailed
1092
1093    /* OldStack = KeSwitchKernelStack((PVOID)NewStack, (PVOID)(NewStack - KERNEL_LARGE_STACK_COMMIT )); */
1094    mov rcx, rax
1095    mov rdx, rax
1096    sub rdx, KERNEL_LARGE_STACK_COMMIT
1097    call KeSwitchKernelStack
1098
1099    // MmDeleteKernelStack(OldStack, FALSE);
1100    mov rcx, rax
1101    xor rdx, rdx
1102    call MmDeleteKernelStack
1103
1104AlreadyLargeStack:
1105
1106    /* Call the worker function */
1107    call PsConvertToGuiThread
1108
1109    /* Check for failure */
1110    test eax, eax
1111    js KiConvertToGuiThreadFailed
1112
1113    /* Disable interrupts for return */
1114    cli
1115
1116    // Restore register parameters
1117    mov rcx, [rsp + 48 + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rip]
1118    mov rdx, [rsp + 48 + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rdx]
1119    mov r8, [rsp + 48 + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_R8]
1120    mov r9, [rsp + 48 + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_R9]
1121
1122    /* Run KiSystemCallHandler again */
1123    add rsp, 48
1124    jmp KiSystemCall64Again
1125
1126KiConvertToGuiThreadFailed:
1127
1128    /* Clean up the stack and return failure */
1129    add rsp, 40
1130    mov eax, HEX(C0000017) // STATUS_NO_MEMORY
1131    ret
1132
1133ENDFUNC
1134
1135
1136EXTERN KiSetTrapContextInternal:PROC
1137
1138/*
1139 * VOID
1140 * KiSetTrapContext(
1141 *     _Out_ PKTRAP_FRAME TrapFrame,
1142 *     _In_ PCONTEXT Context,
1143 *     _In_ KPROCESSOR_MODE RequestorMode);
1144 */
1145PUBLIC KiSetTrapContext
1146.PROC KiSetTrapContext
1147
1148    /* Generate a KEXCEPTION_FRAME on the stack */
1149    GENERATE_EXCEPTION_FRAME
1150
1151    call KiSetTrapContextInternal
1152
1153    /* Restore the registers from the KEXCEPTION_FRAME */
1154    RESTORE_EXCEPTION_STATE
1155
1156    /* Return */
1157    ret
1158
1159.ENDP
1160
1161
1162/*
1163 * VOID
1164 * KiDeliverApc(
1165 *     _In_ KPROCESSOR_MODE DeliveryMode,
1166 *     _In_ PKEXCEPTION_FRAME ExceptionFrame,
1167 *     _In_ PKTRAP_FRAME TrapFrame);
1168 *
1169 */
1170EXTERN KiDeliverApc:PROC
1171
1172/*
1173 * VOID
1174 * KiInitiateUserApc(
1175 *     _In_ PKTRAP_FRAME TrapFrame@<rcx>);
1176 *
1177 * This function is called to deliver user mode APCs.
1178 * It clobbers all non-volatile registers, except rax.
1179 */
1180PUBLIC KiInitiateUserApc
1181.PROC KiInitiateUserApc
1182
1183    /* Generate a KEXCEPTION_FRAME on the stack */
1184    GENERATE_EXCEPTION_FRAME
1185
1186    /* Raise IRQL to APC_LEVEL */
1187    mov rax, APC_LEVEL
1188    mov cr8, rax
1189
1190    /* Get the current thread */
1191    mov rbp, gs:[PcCurrentThread]
1192
1193    /* Save the trap frame in rsi */
1194    mov rsi, rcx
1195
1196    /* Enable interrupts */
1197    sti
1198
1199    /* Call the C function */
1200    mov ecx, 1
1201    mov rdx, rsp
1202    mov r8, rsi
1203    call KiDeliverApc
1204
1205    /* Disable interrupts again */
1206    cli
1207
1208    /* Go back to PASSIVE_LEVEL */
1209    mov rax, PASSIVE_LEVEL
1210    mov cr8, rax
1211
1212    /* Restore the registers from the KEXCEPTION_FRAME */
1213    RESTORE_EXCEPTION_STATE
1214
1215    /* Return */
1216    ret
1217
1218.ENDP
1219
1220
1221PUBLIC KiInitializeSegments
1222KiInitializeSegments:
1223    mov ax, KGDT64_R3_DATA or RPL_MASK
1224    mov gs, ax
1225    swapgs
1226    mov gs, ax
1227    ret
1228
1229/*!
1230 * VOID
1231 * KiSwitchKernelStackHelper(
1232 *     LONG_PTR StackOffset,
1233 *     PVOID OldStackBase);
1234 */
1235PUBLIC KiSwitchKernelStackHelper
1236KiSwitchKernelStackHelper:
1237
1238    /* Pop return address from the current stack */
1239    pop rax
1240
1241    /* Switch to new stack */
1242    lea rsp, [rsp + rcx]
1243
1244    /* Push return address on the new stack */
1245    push rax
1246
1247    /* Return on new stack */
1248    mov rax, rdx
1249    ret
1250
1251EXTERN KiSwitchKernelStack:PROC
1252
1253PUBLIC KeSwitchKernelStack
1254FUNC KeSwitchKernelStack
1255
1256    /* Save rcx and allocate callee home space */
1257    mov [rsp + P1Home], rcx
1258    .savereg rcx, P1Home
1259    sub rsp, 40
1260    .allocstack 40
1261    .endprolog
1262
1263    /* Call the C handler, which returns the old stack in rax */
1264    call KiSwitchKernelStack
1265
1266    /* Restore rcx (StackBase) */
1267    mov rcx, [rsp + 40 + P1Home]
1268
1269    /* Switch to new stack: RSP += (StackBase - OldStackBase) */
1270    sub rcx, rax
1271    add rsp, rcx
1272
1273    /* Deallocate the home frame */
1274    add rsp, 40
1275    ret
1276
1277ENDFUNC
1278
1279
1280
1281
1282#ifdef _MSC_VER
1283#undef lgdt
1284#undef lidt
1285
1286//void __lgdt(void *Source);
1287PUBLIC __lgdt
1288__lgdt:
1289    lgdt fword ptr [rcx]
1290    ret
1291
1292//void __sgdt(void *Destination);
1293PUBLIC __sgdt
1294__sgdt:
1295    sgdt fword ptr [rcx]
1296    ret
1297
1298// void __lldt(unsigned short Value)
1299PUBLIC __lldt
1300__lldt:
1301    lldt cx
1302    ret
1303
1304//void __sldt(void *Destination);
1305PUBLIC __sldt
1306__sldt:
1307    sldt word ptr [rcx]
1308    ret
1309
1310//void __ltr(unsigned short Source);
1311PUBLIC __ltr
1312__ltr:
1313    ltr cx
1314    ret
1315
1316//void __str(unsigned short *Destination);
1317PUBLIC __str
1318__str:
1319    str word ptr [rcx]
1320    ret
1321
1322PUBLIC __swapgs
1323__swapgs:
1324    swapgs
1325    ret
1326
1327#endif
1328
1329END
1330