xref: /reactos/ntoskrnl/ke/amd64/trap.S (revision 67231bd6) 
1/*
2 * FILE:            ntoskrnl/ke/amd64/trap.S
3 * COPYRIGHT:       See COPYING in the top level directory
4 * PURPOSE:         System Traps, Entrypoints and Exitpoints
5 * PROGRAMMER:      Timo Kreuzer (timo.kreuzer@reactos.org)
6 */
7
8/* INCLUDES ******************************************************************/
9
10#include <asm.inc>
11#include <ksamd64.inc>
12#include <trapamd64.inc>
13
14EXTERN KiDispatchException:PROC
15EXTERN KeBugCheckWithTf:PROC
16EXTERN MmAccessFault:PROC
17EXTERN KiSystemFatalException:PROC
18EXTERN KiNpxNotAvailableFaultHandler:PROC
19EXTERN KiGeneralProtectionFaultHandler:PROC
20EXTERN KiXmmExceptionHandler:PROC
21EXTERN KiDeliverApc:PROC
22EXTERN KiDpcInterruptHandler:PROC
23EXTERN PsConvertToGuiThread:PROC
24EXTERN MmCreateKernelStack:PROC
25EXTERN MmDeleteKernelStack:PROC
26EXTERN KdSetOwedBreakpoints:PROC
27EXTERN KeAcquireSpinLockAtDpcLevel:PROC
28EXTERN KeReleaseSpinLockFromDpcLevel:PROC
29
30
31/* Helper Macros *************************************************************/
32
33MACRO(DispatchException, Status, Number, P1, P2, P3)
34    mov eax, Status
35    mov edx, Number
36    mov r9, P1
37    mov r10, P2
38    mov r11, P3
39    call InternalDispatchException
40ENDM
41
42MACRO(Fatal, BugcheckCode)
43    /* Bugcheck */
44    mov ecx, BugcheckCode
45    mov rdx, rbp
46    call KiSystemFatalException
47ENDM
48
49
50/* FUNCTIONS *****************************************************************/
51
52.code64
53
54ALIGN 8
55
56MACRO(UnexpectedVectorStub, Vector)
57    /* This nop is to make the relative jmp address 4 bytes aligned and to
58       make the whole code 8 bytes long */
59    nop
60    /* This is a push instruction with 8bit operand. Since the instruction
61       sign extends the value to 32 bits, we need to offset it */
62PUBLIC KxUnexpectedInterrupt&Vector
63KxUnexpectedInterrupt&Vector:
64    push (Vector - 128)
65    jmp KiUnexpectedInterrupt
66ENDM
67
68PUBLIC KiUnexpectedRange
69KiUnexpectedRange:
70Vector = 0
71REPEAT 256
72    UnexpectedVectorStub %Vector
73    Vector = Vector+1
74ENDR
75PUBLIC KiUnexpectedRangeEnd
76KiUnexpectedRangeEnd:
77
78PUBLIC KiInterruptDispatchTemplate
79KiInterruptDispatchTemplate:
80    /* This instruction pushes the return address on the stack, which is the
81       address of the interrupt object's DispatchCode member, then jumps
82       to the address stored in the interrupt object's DispatchAddress member */
83    call qword ptr KiInterruptDispatchTemplate[rip - KINTERRUPT_DispatchCode + KINTERRUPT_DispatchAddress]
84
85
86// rbp = TrapFrame, eax = ExceptionCode, edx = NumParams, r9,r10,r11 = params
87FUNC InternalDispatchException
88
89    /* Allocate stack space for EXCEPTION_RECORD and KEXCEPTION_FRAME */
90    sub rsp, EXCEPTION_RECORD_LENGTH + KEXCEPTION_FRAME_LENGTH
91    .allocstack (EXCEPTION_RECORD_LENGTH + KEXCEPTION_FRAME_LENGTH)
92    .endprolog
93
94    /* Set up EXCEPTION_RECORD */
95    lea rcx, [rsp + KEXCEPTION_FRAME_LENGTH]
96    mov [rcx + EXCEPTION_RECORD_ExceptionCode], eax
97    xor rax, rax
98    mov [rcx + EXCEPTION_RECORD_ExceptionFlags], eax
99    mov [rcx + EXCEPTION_RECORD_ExceptionRecord], rax
100    mov rax, [rbp + KTRAP_FRAME_Rip]
101    mov [rcx + EXCEPTION_RECORD_ExceptionAddress], rax
102    mov [rcx + EXCEPTION_RECORD_NumberParameters], edx
103    mov [rcx + EXCEPTION_RECORD_ExceptionInformation + HEX(00)], r9
104    mov [rcx + EXCEPTION_RECORD_ExceptionInformation + HEX(08)], r10
105    mov [rcx + EXCEPTION_RECORD_ExceptionInformation + HEX(10)], r11
106
107    /* Set up KEXCEPTION_FRAME */
108    mov rax, [rbp + KTRAP_FRAME_Rbp]
109    mov [rsp + KEXCEPTION_FRAME_Rbp], rax
110    mov [rsp + KEXCEPTION_FRAME_Rbx], rbx
111    mov [rsp + KEXCEPTION_FRAME_Rdi], rdi
112    mov [rsp + KEXCEPTION_FRAME_Rsi], rsi
113    mov [rsp + KEXCEPTION_FRAME_R12], r12
114    mov [rsp + KEXCEPTION_FRAME_R13], r13
115    mov [rsp + KEXCEPTION_FRAME_R14], r14
116    mov [rsp + KEXCEPTION_FRAME_R15], r15
117    movdqa [rsp + KEXCEPTION_FRAME_Xmm6], xmm6
118    movdqa [rsp + KEXCEPTION_FRAME_Xmm7], xmm7
119    movdqa [rsp + KEXCEPTION_FRAME_Xmm8], xmm8
120    movdqa [rsp + KEXCEPTION_FRAME_Xmm9], xmm9
121    movdqa [rsp + KEXCEPTION_FRAME_Xmm10], xmm10
122    movdqa [rsp + KEXCEPTION_FRAME_Xmm11], xmm11
123    movdqa [rsp + KEXCEPTION_FRAME_Xmm12], xmm12
124    movdqa [rsp + KEXCEPTION_FRAME_Xmm13], xmm13
125    movdqa [rsp + KEXCEPTION_FRAME_Xmm14], xmm14
126    movdqa [rsp + KEXCEPTION_FRAME_Xmm15], xmm15
127    mov qword ptr [rsp + KEXCEPTION_FRAME_Return], 0
128
129    /* Call KiDispatchException */
130    // rcx already points to ExceptionRecord
131    mov rdx, rsp                                // ExceptionFrame
132    mov r8, rbp                                 // TrapFrame
133    mov r9b, [r8 + KTRAP_FRAME_PreviousMode]    // PreviousMode
134    mov byte ptr [rsp + KEXCEPTION_FRAME_P5], 1 // FirstChance
135    call KiDispatchException
136
137    /* Restore registers */
138    mov r12, [rsp + KEXCEPTION_FRAME_R12]
139    mov r13, [rsp + KEXCEPTION_FRAME_R13]
140    mov r14, [rsp + KEXCEPTION_FRAME_R14]
141    mov r15, [rsp + KEXCEPTION_FRAME_R15]
142    movdqa xmm6, [rsp + KEXCEPTION_FRAME_Xmm6]
143    movdqa xmm7, [rsp + KEXCEPTION_FRAME_Xmm7]
144    movdqa xmm8, [rsp + KEXCEPTION_FRAME_Xmm8]
145    movdqa xmm9, [rsp + KEXCEPTION_FRAME_Xmm9]
146    movdqa xmm10, [rsp + KEXCEPTION_FRAME_Xmm10]
147    movdqa xmm11, [rsp + KEXCEPTION_FRAME_Xmm11]
148    movdqa xmm12, [rsp + KEXCEPTION_FRAME_Xmm12]
149    movdqa xmm13, [rsp + KEXCEPTION_FRAME_Xmm13]
150    movdqa xmm14, [rsp + KEXCEPTION_FRAME_Xmm14]
151    movdqa xmm15, [rsp + KEXCEPTION_FRAME_Xmm15]
152
153    add rsp, EXCEPTION_RECORD_LENGTH + KEXCEPTION_FRAME_LENGTH
154    ret
155ENDFUNC
156
157
158/* CPU EXCEPTION HANDLERS ****************************************************/
159
160PUBLIC KiDivideErrorFault
161FUNC KiDivideErrorFault
162    /* Push pseudo error code */
163    EnterTrap TF_SAVE_ALL
164
165    /* Enable interrupts */
166    sti
167
168    /* Dispatch the exception */
169    DispatchException STATUS_INTEGER_DIVIDE_BY_ZERO, 0, 0, 0, 0
170
171    /* Return */
172    ExitTrap TF_SAVE_ALL
173ENDFUNC
174
175
176PUBLIC KiDebugTrapOrFault
177FUNC KiDebugTrapOrFault
178    /* Push pseudo error code */
179    EnterTrap TF_SAVE_ALL
180
181    /* Check if the frame was from kernelmode */
182    test word ptr [rbp + KTRAP_FRAME_SegCs], 3
183    jz KiDebugTrapOrFaultKMode
184
185    /* Enable interrupts for user-mode */
186    sti
187
188KiDebugTrapOrFaultKMode:
189    /* Dispatch the exception */
190    DispatchException STATUS_SINGLE_STEP, 0, 0, 0, 0
191
192    /* Return */
193    ExitTrap TF_SAVE_ALL
194ENDFUNC
195
196EXTERN KiNmiInterruptHandler:PROC
197
198FUNC KiNmiInterruptWithEf
199    /* Generate a KEXCEPTION_FRAME on the stack */
200    GENERATE_EXCEPTION_FRAME
201
202    /* Call the C handler */
203    lea rcx, [rsp + KEXCEPTION_FRAME_LENGTH]
204    lea rdx, [rsp]
205    call KiNmiInterruptHandler
206
207    /* Restore the registers from the KEXCEPTION_FRAME */
208    RESTORE_EXCEPTION_STATE
209
210    /* Return */
211    ret
212ENDFUNC
213
214PUBLIC KiNmiInterrupt
215FUNC KiNmiInterrupt
216    /* Push pseudo error code */
217    EnterTrap TF_SAVE_ALL
218
219    call KiNmiInterruptWithEf
220
221    /* Return */
222    ExitTrap TF_SAVE_ALL
223ENDFUNC
224
225
226PUBLIC KiBreakpointTrap
227FUNC KiBreakpointTrap
228    /* Push pseudo error code */
229    EnterTrap TF_SAVE_ALL
230
231    /* Check if the frame was from kernelmode */
232    test word ptr [rbp + KTRAP_FRAME_SegCs], 3
233    jz KiBreakpointTrapKMode
234
235    /* Enable interrupts for user-mode */
236    sti
237
238KiBreakpointTrapKMode:
239    /* Dispatch the exception */
240    DispatchException STATUS_BREAKPOINT, 3, BREAKPOINT_BREAK, 0, 0
241
242    /* Return */
243    ExitTrap TF_SAVE_ALL
244ENDFUNC
245
246
247PUBLIC KiOverflowTrap
248FUNC KiOverflowTrap
249    /* Push pseudo error code */
250    EnterTrap TF_SAVE_ALL
251
252    /* Enable interrupts */
253    sti
254
255    /* Dispatch the exception */
256    DispatchException STATUS_INTEGER_OVERFLOW, 3, 0, 0, 0
257
258    /* Return */
259    ExitTrap TF_SAVE_ALL
260ENDFUNC
261
262
263PUBLIC KiBoundFault
264FUNC KiBoundFault
265    /* Push pseudo error code */
266    EnterTrap TF_SAVE_ALL
267
268    /* Check if the frame was from kernelmode */
269    test word ptr [rbp + KTRAP_FRAME_SegCs], 3
270    jnz KiBoundFaultUserMode
271
272    /* Bugcheck */
273    Fatal EXCEPTION_BOUND_CHECK
274
275KiBoundFaultUserMode:
276    /* Enable interrupts for user-mode */
277    sti
278
279    /* Dispatch the exception */
280    DispatchException STATUS_ARRAY_BOUNDS_EXCEEDED, 0, 0, 0, 0
281
282    /* Return */
283    ExitTrap TF_SAVE_ALL
284ENDFUNC
285
286
287PUBLIC KiInvalidOpcodeFault
288FUNC KiInvalidOpcodeFault
289    /* Push pseudo error code */
290    EnterTrap TF_SAVE_ALL
291
292    /* Enable interrupts */
293    sti
294
295    /* Check if the frame was from kernelmode */
296    test word ptr [rbp + KTRAP_FRAME_SegCs], 3
297    jz KiInvalidOpcodeKernel
298
299    // FIXME: handle STATUS_INVALID_LOCK_SEQUENCE
300
301KiInvalidOpcodeKernel:
302    /* Kernel mode fault */
303
304    /* Dispatch the exception */
305    DispatchException STATUS_ILLEGAL_INSTRUCTION, 0, 0, 0, 0
306
307    /* Return */
308    ExitTrap TF_SAVE_ALL
309ENDFUNC
310
311
312PUBLIC KiNpxNotAvailableFault
313FUNC KiNpxNotAvailableFault
314    /* Push pseudo error code */
315    EnterTrap TF_SAVE_ALL
316
317    /* Call the C handler */
318    mov rcx, rbp
319    call KiNpxNotAvailableFaultHandler
320
321    /* Check the return status code */
322    test eax, eax
323    jz KiNpxNotAvailableFaultExit
324
325    /* Dispatch the exception */
326    DispatchException eax, 3, 0, 0, 0
327
328KiNpxNotAvailableFaultExit:
329    /* Return */
330    ExitTrap TF_SAVE_ALL
331ENDFUNC
332
333
334PUBLIC KiDoubleFaultAbort
335FUNC KiDoubleFaultAbort
336
337    /* Hack for VBox, which "forgets" to push an error code on the stack! */
338    and rsp, HEX(FFFFFFFFFFFFFFF0)
339
340    /* A zero error code is pushed */
341    EnterTrap (TF_HAS_ERROR_CODE OR TF_SAVE_ALL)
342
343    int 3
344
345    /* Bugcheck */
346    Fatal 8 // EXCEPTION_DOUBLE_FAULT
347    jmp $
348ENDFUNC
349
350
351PUBLIC KiNpxSegmentOverrunAbort
352FUNC KiNpxSegmentOverrunAbort
353    /* Push pseudo error code */
354    EnterTrap TF_SAVE_ALL
355
356    /* Bugcheck */
357    Fatal EXCEPTION_NPX_OVERRUN
358
359    jmp $
360ENDFUNC
361
362
363PUBLIC KiInvalidTssFault
364FUNC KiInvalidTssFault
365    /* We have an error code */
366    EnterTrap (TF_HAS_ERROR_CODE OR TF_SAVE_ALL)
367
368    /* Bugcheck */
369    Fatal EXCEPTION_INVALID_TSS
370    jmp $
371ENDFUNC
372
373
374PUBLIC KiSegmentNotPresentFault
375FUNC KiSegmentNotPresentFault
376    /* We have an error code */
377    EnterTrap (TF_HAS_ERROR_CODE OR TF_SAVE_ALL)
378
379    /* Bugcheck */
380    Fatal EXCEPTION_SEGMENT_NOT_PRESENT
381    jmp $
382ENDFUNC
383
384
385PUBLIC KiStackFault
386FUNC KiStackFault
387    /* We have an error code */
388    EnterTrap (TF_HAS_ERROR_CODE OR TF_SAVE_ALL)
389
390    /* Bugcheck */
391    Fatal EXCEPTION_STACK_FAULT
392    jmp $
393ENDFUNC
394
395
396PUBLIC KiGeneralProtectionFault
397FUNC KiGeneralProtectionFault
398    /* We have an error code */
399    EnterTrap (TF_HAS_ERROR_CODE OR TF_SAVE_ALL)
400
401    /* Call the C handler */
402    mov rcx, rbp
403    call KiGeneralProtectionFaultHandler
404
405    /* Check for success */
406    test eax, eax
407    jge KiGpfExit
408
409    /* Check for access violation */
410    cmp eax, STATUS_ACCESS_VIOLATION
411    je DispatchAccessViolation
412
413    /* Dispatch privileged instruction fault */
414    DispatchException eax, 0, 0, 0, 0
415    jmp KiGpfExit
416
417DispatchAccessViolation:
418
419    /* Dispatch access violation */
420    DispatchException eax, 2, 0, -1, 0
421
422KiGpfExit:
423    /* Return */
424    ExitTrap TF_SAVE_ALL
425ENDFUNC
426
427
428PUBLIC KiPageFault
429FUNC KiPageFault
430    /* We have an error code */
431    EnterTrap (TF_HAS_ERROR_CODE OR TF_SAVE_ALL)
432
433    /* Save page fault address */
434    mov rdx, cr2
435    mov [rbp  + KTRAP_FRAME_FaultAddress], rdx
436
437    /* If interrupts are off, do not enable them */
438    test dword ptr [rbp + KTRAP_FRAME_EFlags], EFLAGS_IF_MASK
439    jz IntsDisabled
440
441    /* Enable interrupts for the page fault handler */
442    sti
443
444IntsDisabled:
445
446    /* Call page fault handler */
447    mov ecx, [rbp + KTRAP_FRAME_ErrorCode] // FaultCode
448    // rdx == Address
449    mov r8b, [rbp + KTRAP_FRAME_SegCs] // Mode
450    and r8b, 1
451    mov r9, rbp // TrapInformation
452    call MmAccessFault
453
454    /* Check for success */
455    test eax, eax
456    jl PageFaultError
457
458    /* Check whether the kernel debugger has owed breakpoints to be inserted */
459    call KdSetOwedBreakpoints
460    /* We succeeded, return */
461    jmp PageFaultReturn
462
463PageFaultError:
464
465    /* Set parameter 1 to write/execute flag.
466       See https://learn.microsoft.com/en-us/windows/win32/api/winnt/ns-winnt-exception_record */
467    mov r9d, [rbp + KTRAP_FRAME_ErrorCode]
468    shr r9d, 1
469    and r9d, 9
470
471    /* Set parameter 2 to faulting address */
472    mov r10, cr2  // Param2 = faulting address
473
474    cmp eax, STATUS_ACCESS_VIOLATION
475    je AccessViolation
476    cmp eax, STATUS_GUARD_PAGE_VIOLATION
477    je SpecialCode
478    cmp eax, STATUS_STACK_OVERFLOW
479    je SpecialCode
480
481InPageException:
482    /* Dispatch in-page exception */
483    mov r11d, eax // Param3 = Status
484    mov eax, STATUS_IN_PAGE_ERROR // ExceptionCode
485    mov edx, 3    // ParamCount
486    call InternalDispatchException
487    jmp PageFaultReturn
488
489AccessViolation:
490    /* Use more proper status code */
491    mov eax, KI_EXCEPTION_ACCESS_VIOLATION
492
493SpecialCode:
494    /* Setup a normal page fault exception */
495    mov edx, 2   // ParamCount
496    call InternalDispatchException
497
498PageFaultReturn:
499
500    /* Disable interrupts for the return */
501    cli
502
503    /* Return */
504    ExitTrap (TF_SAVE_ALL or TF_CHECKUSERAPC)
505ENDFUNC
506
507
508PUBLIC KiFloatingErrorFault
509FUNC KiFloatingErrorFault
510    /* Push pseudo error code */
511    EnterTrap TF_SAVE_ALL
512
513    UNIMPLEMENTED KiFloatingErrorFault
514    int 3
515
516    /* Return */
517    ExitTrap TF_SAVE_ALL
518ENDFUNC
519
520
521PUBLIC KiAlignmentFault
522FUNC KiAlignmentFault
523    /* We have an error code */
524    EnterTrap (TF_HAS_ERROR_CODE OR TF_SAVE_ALL)
525
526    /* Bugcheck */
527    Fatal EXCEPTION_ALIGNMENT_CHECK
528    jmp $
529ENDFUNC
530
531
532PUBLIC KiMcheckAbort
533FUNC KiMcheckAbort
534    /* Push pseudo error code */
535    EnterTrap TF_SAVE_ALL
536
537    /* Bugcheck */
538    Fatal HEX(12)
539    jmp $
540ENDFUNC
541
542
543PUBLIC KiXmmException
544FUNC KiXmmException
545    /* Push pseudo error code */
546    EnterTrap TF_SAVE_ALL
547
548    /* Call the C handler */
549    mov rcx, rbp
550    call KiXmmExceptionHandler
551
552    /* Check for success */
553    test eax, eax
554    jge KiXmmExit
555
556    /* Dispatch the exception */
557    DispatchException eax, 2, 0, [rbp+KTRAP_FRAME_MxCsr], 0
558
559    // FIXME: STATUS_FLOAT_MULTIPLE_TRAPS / STATUS_FLOAT_MULTIPLE_FAULTS
560
561KiXmmExit:
562    /* Return */
563    ExitTrap TF_SAVE_ALL
564ENDFUNC
565
566
567/* SOFTWARE INTERRUPT SERVICES ***********************************************/
568
569PUBLIC KiRaiseAssertion
570FUNC KiRaiseAssertion
571    /* We have an error code */
572    EnterTrap (TF_SAVE_ALL)
573
574    /* Decrement RIP to point to the INT2C instruction (2 bytes, not 1 like INT3) */
575    sub qword ptr [rbp + KTRAP_FRAME_Rip], 2
576
577    /* Dispatch the exception */
578    DispatchException STATUS_ASSERTION_FAILURE, 0, 0, 0, 0
579
580    /* Return */
581    ExitTrap TF_SAVE_ALL
582ENDFUNC
583
584
585PUBLIC KiDebugServiceTrap
586FUNC KiDebugServiceTrap
587   /* No error code */
588    EnterTrap TF_SAVE_ALL
589
590    /* Increase Rip to skip the int3 */
591    inc qword ptr [rbp + KTRAP_FRAME_Rip]
592
593    /* Dispatch the exception (Params = service, buffer, legth) */
594    DispatchException STATUS_BREAKPOINT, 3, [rbp+KTRAP_FRAME_Rax], [rbp+KTRAP_FRAME_Rcx], [rbp+KTRAP_FRAME_Rdx]
595
596    /* Return */
597    ExitTrap TF_SAVE_ALL
598ENDFUNC
599
600
601PUBLIC KiApcInterrupt
602.PROC KiApcInterrupt
603   /* No error code */
604    EnterTrap (TF_SAVE_ALL or TF_IRQL)
605
606    /* Raise to APC_LEVEL */
607    mov rax, APC_LEVEL
608    mov cr8, rax
609
610    /* End the interrupt */
611    mov dword ptr [APIC_EOI], 0
612
613    /* Enable interrupts  */
614    sti
615
616    /* Call the worker routine */
617    mov cl, [rbp + KTRAP_FRAME_SegCs] // ProcessorMode
618    and cl, 1
619    mov rdx, 0                        // ExceptionFrame
620    mov r8, rbp                       // TrapFrame
621    call KiDeliverApc
622
623    /* Disable interrupts */
624    cli
625
626    /* Lower IRQL back to PASSIVE */
627    mov rax, PASSIVE_LEVEL
628    mov cr8, rax
629
630    /* Return */
631    ExitTrap (TF_SAVE_ALL or TF_IRQL)
632.ENDP
633
634/*
635 * VOID
636 * KiRetireDpcList(
637 *     PKPRCB Prcb);
638 */
639EXTERN KiRetireDpcList:PROC
640
641/*
642 * VOID
643 * KiRetireDpcListInDpcStack(
644 *     PKPRCB Prcb,
645 *     PVOID DpcStack);
646 */
647PUBLIC KiRetireDpcListInDpcStack
648.PROC KiRetireDpcListInDpcStack
649    push rbp
650    .pushreg rbp
651    mov rbp, rsp
652    .setframe rbp, 0
653    .endprolog
654
655    /* Switch to the DpcStack */
656    mov rsp, rdx
657
658    /* The stack is 16 byte aligned, allocate 32 bytes home space */
659    sub rsp, 32
660
661    /* Call KiRetireDpcList on the given stack */
662    call KiRetireDpcList
663
664    /* Restore stack, cleanup and return */
665    mov rsp, rbp
666    pop rbp
667    ret
668.ENDP
669
670PUBLIC KiDpcInterrupt
671.PROC KiDpcInterrupt
672   /* No error code */
673    EnterTrap (TF_SAVE_ALL or TF_IRQL)
674
675    /* Call the worker routine */
676    call KiDpcInterruptHandler
677
678    /* Return, but don't send an EOI! */
679    ExitTrap (TF_SAVE_ALL or TF_IRQL)
680.ENDP
681
682
683PUBLIC KiIpiInterrupt
684.PROC KiIpiInterrupt
685   /* No error code */
686    EnterTrap (TF_SAVE_ALL or TF_IRQL)
687
688    /* Raise to IPI_LEVEL */
689    mov rax, IPI_LEVEL
690    mov cr8, rax
691
692    /* End the interrupt */
693    mov dword ptr [APIC_EOI], 0
694
695    int 3
696
697    /* Return */
698    ExitTrap (TF_SAVE_ALL or TF_IRQL)
699.ENDP
700
701
702PUBLIC KiUnexpectedInterrupt
703FUNC KiUnexpectedInterrupt
704    /* The error code is the vector */
705    EnterTrap (TF_HAS_ERROR_CODE OR TF_SAVE_ALL)
706
707#if 0
708    /* Set bugcheck parameters */
709    mov ecx, TRAP_CAUSE_UNKNOWN
710    mov rdx, [rbp + KTRAP_FRAME_ErrorCode] // the vector
711    mov r8, 0 // The unknown floating-point exception
712    mov r9, 0 // The enabled and asserted status bits
713    sub rsp, 8
714    mov [rbp + KTRAP_FRAME_P5 + 8], rbp // trap frame
715    call KeBugCheckWithTf
716    jmp $
717#endif
718    /* Return */
719    ExitTrap TF_SAVE_ALL
720ENDFUNC
721
722PUBLIC KiInterruptDispatch
723FUNC KiInterruptDispatch
724    /* The error code is a pointer to the interrupt object's code */
725    EnterTrap (TF_HAS_ERROR_CODE or TF_SAVE_ALL or TF_IRQL)
726
727    /* Increase interrupt count */
728    inc dword ptr gs:[PcInterruptCount];
729
730    /* Save rbx and rsi in the trap frame */
731    mov [rbp + KTRAP_FRAME_Rbx], rbx
732    mov [rbp + KTRAP_FRAME_Rsi], rsi
733
734    /* Load the address of the dispatch code into rbx */
735    mov rbx, [rbp + KTRAP_FRAME_ErrorCode]
736
737    /* Substract offset of the DispatchCode member plus 6 for the call instruction */
738    sub rbx, KINTERRUPT_DispatchCode + 6
739
740    /* Save the address of the InterruptListEntry in rsi */
741    lea rsi, [rbx + KINTERRUPT_InterruptListEntry]
742
743.DoDispatchInterrupt:
744    /* Raise IRQL to SynchronizeIrql */
745    movzx rax, byte ptr [rbx + KINTERRUPT_SynchronizeIrql]
746    mov cr8, rax
747
748    /* Enable interrupts */
749    sti
750
751#ifdef CONFIG_SMP
752    /* Acquire interrupt lock */
753    mov rcx, [rbx + KINTERRUPT_ActualLock]
754    call KeAcquireSpinLockAtDpcLevel
755#endif
756
757    /* Call the ISR */
758    mov rcx, rbx
759    mov rdx, [rbx + KINTERRUPT_ServiceContext]
760    call qword ptr [rbx + KINTERRUPT_ServiceRoutine]
761
762#ifdef CONFIG_SMP
763    /* Release interrupt lock */
764    mov rcx, [rbx + KINTERRUPT_ActualLock]
765    call KeReleaseSpinLockFromDpcLevel
766#endif
767
768    /* Disable interrupts and go back to old irql */
769    cli
770    movzx rax, byte ptr [rbp + KTRAP_FRAME_PreviousIrql]
771    mov cr8, rax
772
773    /* Check for chained interrupts */
774    mov rax, [rbx + KINTERRUPT_InterruptListEntry]
775    cmp rax, rsi
776    je .Done
777
778    /* Load the next interrupt object into rbx and repeat */
779    lea rbx, [rax - KINTERRUPT_InterruptListEntry]
780    jmp .DoDispatchInterrupt
781
782.Done:
783    /* Restore rbx and rsi */
784    mov rbx, [rbp + KTRAP_FRAME_Rbx]
785    mov rsi, [rbp + KTRAP_FRAME_Rsi]
786
787    /* Return */
788    ExitTrap (TF_SAVE_ALL or TF_SEND_EOI)
789ENDFUNC
790
791EXTERN KiSystemCallHandler:PROC
792
793/*! \name KiSystemCallEntry64
794 *
795 *  \brief This is the entrypoint for syscalls from 64bit user mode
796 *
797 *  \param rax - The system call number
798 *  \param rcx - User mode return address, set by the syscall instruction
799 *  \param rdx,r8,r9 - Parameters 2-4 to the service function
800 *  \param r10 - Parameter 1 to the service function
801 *  \param r11 - RFLAGS saved by the syscall instruction
802 *--*/
803PUBLIC KiSystemCallEntry64
804.PROC KiSystemCallEntry64
805    /* The unwind info pretends we have a machine frame */
806    .PUSHFRAME
807    .ALLOCSTACK (KTRAP_FRAME_LENGTH + MAX_SYSCALL_PARAM_SIZE - MachineFrameLength)
808    .SAVEREG rbp, MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rbp
809    .ENDPROLOG
810
811    /* Swap gs to kernel, so we can access the PCR */
812    swapgs
813
814    /* Save the user mode rsp in the PCR */
815    mov gs:[PcUserRsp], rsp
816
817    /* Get the kernel stack from the PCR */
818    mov rsp, gs:[PcRspBase]
819
820    /* Allocate a TRAP_FRAME and space for parameters */
821    sub rsp, (KTRAP_FRAME_LENGTH + MAX_SYSCALL_PARAM_SIZE)
822
823    /* Save volatile registers and rbp in the trap frame */
824    mov [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rax], rax
825    mov [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rip], rcx
826    mov [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rdx], rdx
827    mov [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_R8], r8
828    mov [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_R9], r9
829    mov [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rcx], r10
830    mov [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_EFlags], r11
831    mov [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rbp], rbp
832
833    /* Store user stack pointer in the trap frame */
834    mov rax, gs:[PcUserRsp]
835    mov [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rsp], rax
836
837    /* Set sane segments */
838    mov ax, (KGDT64_R3_DATA or RPL_MASK)
839    mov ds, ax
840    mov es, ax
841
842    /* Save MCXSR and set kernel value */
843    stmxcsr [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_MxCsr]
844    ldmxcsr gs:[PcMxCsr]
845
846    /* Get the current thread and the trap frame */
847    mov rax, gs:[PcCurrentThread]
848    mov rcx, [rax + ThTrapFrame]
849
850    /* Save the old trap frame */
851    lea rdx, [rsp + MAX_SYSCALL_PARAM_SIZE]
852    mov [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_TrapFrame], rcx
853    mov [rax + ThTrapFrame], rdx
854
855#if DBG
856    /* Check IRQL */
857    mov rax, cr8
858    test eax, eax
859    jz KiSystemCall64Again
860    int HEX(2C)
861#endif
862
863GLOBAL_LABEL KiSystemCall64Again
864
865    /* Call the C-handler (will enable interrupts) */
866    call KiSystemCallHandler
867
868    /* The return value from KiSystemCallHandler is the address of the Nt-function */
869    mov rcx, [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rcx]
870    mov rdx, [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rdx]
871    mov r8, [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_R8]
872    mov r9, [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_R9]
873    call rax
874
875GLOBAL_LABEL KiSystemServiceExit
876
877    ASSERT_TRAP_FRAME_INTS_ENABLED rsp + MAX_SYSCALL_PARAM_SIZE
878
879    /* Check for pending user APCs */
880    mov rcx, gs:[PcCurrentThread]
881    cmp byte ptr [rcx + ThApcState + AsUserApcPending], 0
882    jz NoUserApcPending
883
884    /* Save missing regs in the trap frame */
885    mov [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rax], rax
886    mov [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rbp], rbp
887    mov [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_R9], rbp
888    mov rax, [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rsp]
889    mov [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_R8], rax
890    mov rax, [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rip]
891    mov [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rcx], rax
892    mov rax, [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_EFlags]
893    mov [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_R11], rax
894    xor rax, rax
895    mov [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rdx], rax
896    mov [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_R10], rax
897    pxor xmm0, xmm0
898    movdqa [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Xmm0], xmm0
899    movdqa [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Xmm1], xmm0
900    movdqa [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Xmm2], xmm0
901    movdqa [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Xmm3], xmm0
902    movdqa [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Xmm4], xmm0
903    movdqa [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Xmm5], xmm0
904
905    lea rcx, [rsp + MAX_SYSCALL_PARAM_SIZE]
906    call KiInitiateUserApc
907
908NoUserApcPending:
909    /* Disable interrupts for return */
910    cli
911
912    /* Restore MCXSR */
913    ldmxcsr [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_MxCsr]
914
915    /* Restore old trap frame */
916    mov rcx, gs:[PcCurrentThread]
917    mov rdx, [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_TrapFrame]
918    mov [rcx + KTHREAD_TrapFrame], rdx
919
920    /* Prepare user mode return address (rcx) and eflags (r11) for sysret */
921    mov rcx, [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rip]
922    mov r11, [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_EFlags]
923
924    /* Load user mode stack (It was copied to the trap frame) */
925    mov rsp, [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rsp]
926
927    /* r8 points to the user stack */
928    mov r8, rsp
929
930    /* r9 matches rbp */
931    mov r9, rbp
932
933    /* Swap gs back to user */
934    swapgs
935
936    /* Zero out volatiles */
937    pxor xmm0, xmm0
938    pxor xmm1, xmm1
939    pxor xmm2, xmm2
940    pxor xmm3, xmm3
941    pxor xmm4, xmm4
942    pxor xmm5, xmm5
943    xor rdx, rdx
944    xor r10, r10
945
946    /* return to user mode */
947    sysretq
948
949.ENDP
950
951
952/*!
953 * VOID
954 * DECLSPEC_NORETURN
955 * KiExceptionExit(
956 *     _In_ PKTRAP_FRAME TrapFrame@<rcx>,
957 *     _In_ PKEXCEPTION_FRAME ExceptionFrame@<rdx>);
958 */
959PUBLIC KiExceptionExit
960KiExceptionExit:
961
962    /* Restore registers from exception frame */
963    movaps xmm6, [rdx + ExXmm6]
964    movaps xmm7, [rdx + ExXmm7]
965    movaps xmm8, [rdx + ExXmm8]
966    movaps xmm9, [rdx + ExXmm9]
967    movaps xmm10, [rdx + ExXmm10]
968    movaps xmm11, [rdx + ExXmm11]
969    movaps xmm12, [rdx + ExXmm12]
970    movaps xmm13, [rdx + ExXmm13]
971    movaps xmm14, [rdx + ExXmm14]
972    movaps xmm15, [rdx + ExXmm15]
973    mov rbx, [rdx + ExRbx]
974    mov rdi, [rdx + ExRdi]
975    mov rsi, [rdx + ExRsi]
976    mov r12, [rdx + ExR12]
977    mov r13, [rdx + ExR13]
978    mov r14, [rdx + ExR14]
979    mov r15, [rdx + ExR15]
980
981    /* Point rsp at the trap frame */
982    mov rsp, rcx
983    /* Fall through */
984
985/*
986 * Internal function. Exits to user-mode with rsp pointing to the trap frame.
987 * All non-volatile register context must be set up already.
988 * Used by KiInitializeContextThread to set up the init path for a new thread.
989 */
990PUBLIC KiServiceExit3
991.PROC KiServiceExit3
992    .PUSHFRAME
993    .ALLOCSTACK (KTRAP_FRAME_LENGTH - MachineFrameLength)
994    .ENDPROLOG
995
996    /* Return */
997    mov rbp, rsp
998    ExitTrap (TF_SEGMENTS or TF_CHECKUSERAPC)
999.ENDP
1000
1001
1002PUBLIC KiSystemCallEntry32
1003KiSystemCallEntry32:
1004    swapgs
1005    int 3
1006
1007
1008PUBLIC KiZwSystemService
1009FUNC KiZwSystemService
1010    push rbp
1011    .pushreg rbp
1012    sub rsp, KTRAP_FRAME_LENGTH
1013    .allocstack KTRAP_FRAME_LENGTH
1014    mov [rsp + KTRAP_FRAME_Rsi], rsi
1015    .savereg rsi, KTRAP_FRAME_Rsi
1016    mov [rsp + KTRAP_FRAME_Rdi], rdi
1017    .savereg rdi, KTRAP_FRAME_Rdi
1018    mov rbp, rsp
1019    .setframe rbp, 0
1020    .endprolog
1021
1022    /* Get current thread */
1023    mov r11, gs:[PcCurrentThread]
1024
1025    /* Save PreviousMode in the trap frame */
1026    mov dil, byte ptr [r11 + KTHREAD_PreviousMode]
1027    mov byte ptr [rbp + KTRAP_FRAME_PreviousMode], dil
1028
1029    /* Save the old trap frame in TrapFrame.Rdx */
1030    mov rdi, [r11 + KTHREAD_TrapFrame]
1031    mov [rbp + KTRAP_FRAME_Rdx], rdi
1032
1033    /* Set the new trap frame and previous mode */
1034    mov [r11 + ThTrapFrame], rbp
1035    mov byte ptr [r11 + KTHREAD_PreviousMode], 0
1036
1037    /* allocate space for parameters */
1038    sub rsp, r10
1039    and rsp, HEX(0fffffffffffffff0)
1040
1041    /* Save rcx */
1042    mov [rbp + KTRAP_FRAME_Rcx], rcx
1043
1044    /* copy parameters to the new location */
1045    lea rsi, [rbp + KTRAP_FRAME_LENGTH + 16]
1046    lea rdi, [rsp]
1047    mov rcx, r10
1048    shr rcx, 3
1049    rep movsq
1050
1051    /* Restore rcx */
1052    mov rcx, [rbp + KTRAP_FRAME_Rcx]
1053
1054    /* Call the service function */
1055    call rax
1056
1057    /* Restore the old trap frame */
1058    mov r11, gs:[PcCurrentThread]
1059    mov rsi, [rbp + KTRAP_FRAME_Rdx]
1060    mov [r11 + KTHREAD_TrapFrame], rsi
1061
1062    /* Restore PreviousMode from the trap frame */
1063    mov dil, byte ptr [rbp + KTRAP_FRAME_PreviousMode]
1064    mov byte ptr [r11 + KTHREAD_PreviousMode], dil
1065
1066    /* Restore rdi and rsi */
1067    mov rsi, [rbp + KTRAP_FRAME_Rsi]
1068    mov rdi, [rbp + KTRAP_FRAME_Rdi]
1069
1070    /* Cleanup the stack and return */
1071    lea rsp, [rbp + KTRAP_FRAME_LENGTH]
1072    pop rbp
1073    ret
1074
1075ENDFUNC
1076
1077PUBLIC KiConvertToGuiThread
1078FUNC KiConvertToGuiThread
1079
1080    sub rsp, 40
1081    .allocstack 40
1082    .endprolog
1083
1084    /* Check if we already have a large stack */
1085    mov rax, gs:[PcCurrentThread]
1086    cmp byte ptr [rax + KTHREAD_LargeStack], 0
1087    jnz AlreadyLargeStack
1088
1089    // NewStack = (ULONG_PTR)MmCreateKernelStack(TRUE, 0);
1090    mov cl, 1
1091    xor rdx, rdx
1092    call MmCreateKernelStack
1093
1094    /* Check for failure */
1095    test rax, rax
1096    jz KiConvertToGuiThreadFailed
1097
1098    /* OldStack = KeSwitchKernelStack((PVOID)NewStack, (PVOID)(NewStack - KERNEL_LARGE_STACK_COMMIT )); */
1099    mov rcx, rax
1100    mov rdx, rax
1101    sub rdx, KERNEL_LARGE_STACK_COMMIT
1102    call KeSwitchKernelStack
1103
1104    // MmDeleteKernelStack(OldStack, FALSE);
1105    mov rcx, rax
1106    xor rdx, rdx
1107    call MmDeleteKernelStack
1108
1109AlreadyLargeStack:
1110
1111    /* Call the worker function */
1112    call PsConvertToGuiThread
1113
1114    /* Check for failure */
1115    test eax, eax
1116    js KiConvertToGuiThreadFailed
1117
1118    /* Disable interrupts for return */
1119    cli
1120
1121    // Restore register parameters
1122    mov rcx, [rsp + 48 + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rip]
1123    mov rdx, [rsp + 48 + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rdx]
1124    mov r8, [rsp + 48 + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_R8]
1125    mov r9, [rsp + 48 + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_R9]
1126
1127    /* Run KiSystemCallHandler again */
1128    add rsp, 48
1129    jmp KiSystemCall64Again
1130
1131KiConvertToGuiThreadFailed:
1132
1133    /* Clean up the stack and return failure */
1134    add rsp, 40
1135    mov eax, HEX(C0000017) // STATUS_NO_MEMORY
1136    ret
1137
1138ENDFUNC
1139
1140
1141EXTERN KiSetTrapContextInternal:PROC
1142
1143/*
1144 * VOID
1145 * KiSetTrapContext(
1146 *     _Out_ PKTRAP_FRAME TrapFrame,
1147 *     _In_ PCONTEXT Context,
1148 *     _In_ KPROCESSOR_MODE RequestorMode);
1149 */
1150PUBLIC KiSetTrapContext
1151.PROC KiSetTrapContext
1152
1153    /* Generate a KEXCEPTION_FRAME on the stack */
1154    GENERATE_EXCEPTION_FRAME
1155
1156    call KiSetTrapContextInternal
1157
1158    /* Restore the registers from the KEXCEPTION_FRAME */
1159    RESTORE_EXCEPTION_STATE
1160
1161    /* Return */
1162    ret
1163
1164.ENDP
1165
1166
1167/*
1168 * VOID
1169 * KiDeliverApc(
1170 *     _In_ KPROCESSOR_MODE DeliveryMode,
1171 *     _In_ PKEXCEPTION_FRAME ExceptionFrame,
1172 *     _In_ PKTRAP_FRAME TrapFrame);
1173 *
1174 */
1175EXTERN KiDeliverApc:PROC
1176
1177/*
1178 * VOID
1179 * KiInitiateUserApc(
1180 *     _In_ PKTRAP_FRAME TrapFrame@<rcx>);
1181 *
1182 * This function is called to deliver user mode APCs.
1183 * It clobbers all non-volatile registers, except rax.
1184 */
1185PUBLIC KiInitiateUserApc
1186.PROC KiInitiateUserApc
1187
1188    /* Generate a KEXCEPTION_FRAME on the stack */
1189    GENERATE_EXCEPTION_FRAME
1190
1191    /* Raise IRQL to APC_LEVEL */
1192    mov rax, APC_LEVEL
1193    mov cr8, rax
1194
1195    /* Get the current thread */
1196    mov rbp, gs:[PcCurrentThread]
1197
1198    /* Save the trap frame in rsi */
1199    mov rsi, rcx
1200
1201    /* Enable interrupts */
1202    sti
1203
1204    /* Call the C function */
1205    mov ecx, 1
1206    mov rdx, rsp
1207    mov r8, rsi
1208    call KiDeliverApc
1209
1210    /* Disable interrupts again */
1211    cli
1212
1213    /* Go back to PASSIVE_LEVEL */
1214    mov rax, PASSIVE_LEVEL
1215    mov cr8, rax
1216
1217    /* Restore the registers from the KEXCEPTION_FRAME */
1218    RESTORE_EXCEPTION_STATE
1219
1220    /* Return */
1221    ret
1222
1223.ENDP
1224
1225
1226PUBLIC KiInitializeSegments
1227KiInitializeSegments:
1228    mov ax, KGDT64_R3_DATA or RPL_MASK
1229    mov gs, ax
1230    swapgs
1231    mov gs, ax
1232    ret
1233
1234/*!
1235 * VOID
1236 * KiSwitchKernelStackHelper(
1237 *     LONG_PTR StackOffset,
1238 *     PVOID OldStackBase);
1239 */
1240PUBLIC KiSwitchKernelStackHelper
1241KiSwitchKernelStackHelper:
1242
1243    /* Pop return address from the current stack */
1244    pop rax
1245
1246    /* Switch to new stack */
1247    lea rsp, [rsp + rcx]
1248
1249    /* Push return address on the new stack */
1250    push rax
1251
1252    /* Return on new stack */
1253    mov rax, rdx
1254    ret
1255
1256EXTERN KiSwitchKernelStack:PROC
1257
1258PUBLIC KeSwitchKernelStack
1259FUNC KeSwitchKernelStack
1260
1261    /* Save rcx and allocate callee home space */
1262    mov [rsp + P1Home], rcx
1263    .savereg rcx, P1Home
1264    sub rsp, 40
1265    .allocstack 40
1266    .endprolog
1267
1268    /* Call the C handler, which returns the old stack in rax */
1269    call KiSwitchKernelStack
1270
1271    /* Restore rcx (StackBase) */
1272    mov rcx, [rsp + 40 + P1Home]
1273
1274    /* Switch to new stack: RSP += (StackBase - OldStackBase) */
1275    sub rcx, rax
1276    add rsp, rcx
1277
1278    /* Deallocate the home frame */
1279    add rsp, 40
1280    ret
1281
1282ENDFUNC
1283
1284
1285
1286
1287#ifdef _MSC_VER
1288#undef lgdt
1289#undef lidt
1290
1291//void __lgdt(void *Source);
1292PUBLIC __lgdt
1293__lgdt:
1294    lgdt fword ptr [rcx]
1295    ret
1296
1297//void __sgdt(void *Destination);
1298PUBLIC __sgdt
1299__sgdt:
1300    sgdt fword ptr [rcx]
1301    ret
1302
1303// void __lldt(unsigned short Value)
1304PUBLIC __lldt
1305__lldt:
1306    lldt cx
1307    ret
1308
1309//void __sldt(void *Destination);
1310PUBLIC __sldt
1311__sldt:
1312    sldt word ptr [rcx]
1313    ret
1314
1315//void __ltr(unsigned short Source);
1316PUBLIC __ltr
1317__ltr:
1318    ltr cx
1319    ret
1320
1321//void __str(unsigned short *Destination);
1322PUBLIC __str
1323__str:
1324    str word ptr [rcx]
1325    ret
1326
1327PUBLIC __swapgs
1328__swapgs:
1329    swapgs
1330    ret
1331
1332#endif
1333
1334END
1335