1 /*
2 * PROJECT: ReactOS HAL
3 * LICENSE: GNU GPL - See COPYING in the top level directory
4 * FILE: hal/halx86/apic/apic.c
5 * PURPOSE: HAL APIC Management and Control Code
6 * PROGRAMMERS: Timo Kreuzer (timo.kreuzer@reactos.org)
7 * REFERENCES: https://web.archive.org/web/20190407074221/http://www.joseflores.com/docs/ExploringIrql.html
8 * http://www.codeproject.com/KB/system/soviet_kernel_hack.aspx
9 * http://bbs.unixmap.net/thread-2022-1-1.html
10 * https://www.codemachine.com/article_interruptdispatching.html
11 * https://www.osronline.com/article.cfm%5Earticle=211.htm
12 */
13
14 /* INCLUDES *******************************************************************/
15
16 #include <hal.h>
17 #include "apicp.h"
18 #define NDEBUG
19 #include <debug.h>
20
21 #ifndef _M_AMD64
22 #define APIC_LAZY_IRQL
23 #endif
24
25 /* GLOBALS ********************************************************************/
26
27 ULONG ApicVersion;
28 UCHAR HalpVectorToIndex[256];
29
30 #ifndef _M_AMD64
31 const UCHAR
32 HalpIRQLtoTPR[32] =
33 {
34 0x00, /* 0 PASSIVE_LEVEL */
35 0x3d, /* 1 APC_LEVEL */
36 0x41, /* 2 DISPATCH_LEVEL */
37 0x41, /* 3 \ */
38 0x51, /* 4 \ */
39 0x61, /* 5 | */
40 0x71, /* 6 | */
41 0x81, /* 7 | */
42 0x91, /* 8 | */
43 0xa1, /* 9 | */
44 0xb1, /* 10 | */
45 0xb1, /* 11 | */
46 0xb1, /* 12 | */
47 0xb1, /* 13 | */
48 0xb1, /* 14 | */
49 0xb1, /* 15 DEVICE IRQL */
50 0xb1, /* 16 | */
51 0xb1, /* 17 | */
52 0xb1, /* 18 | */
53 0xb1, /* 19 | */
54 0xb1, /* 20 | */
55 0xb1, /* 21 | */
56 0xb1, /* 22 | */
57 0xb1, /* 23 | */
58 0xb1, /* 24 | */
59 0xb1, /* 25 / */
60 0xb1, /* 26 / */
61 0xc1, /* 27 PROFILE_LEVEL */
62 0xd1, /* 28 CLOCK2_LEVEL */
63 0xe1, /* 29 IPI_LEVEL */
64 0xef, /* 30 POWER_LEVEL */
65 0xff, /* 31 HIGH_LEVEL */
66 };
67
68 const KIRQL
69 HalVectorToIRQL[16] =
70 {
71 0, /* 00 PASSIVE_LEVEL */
72 0xff, /* 10 */
73 0xff, /* 20 */
74 1, /* 3D APC_LEVEL */
75 2, /* 41 DISPATCH_LEVEL */
76 4, /* 50 \ */
77 5, /* 60 \ */
78 6, /* 70 | */
79 7, /* 80 DEVICE IRQL */
80 8, /* 90 | */
81 9, /* A0 / */
82 10, /* B0 / */
83 27, /* C1 PROFILE_LEVEL */
84 28, /* D1 CLOCK2_LEVEL */
85 29, /* E1 IPI_LEVEL / EF POWER_LEVEL */
86 31, /* FF HIGH_LEVEL */
87 };
88 #endif
89
90 /* PRIVATE FUNCTIONS **********************************************************/
91
92 FORCEINLINE
93 ULONG
IOApicRead(UCHAR Register)94 IOApicRead(UCHAR Register)
95 {
96 /* Select the register, then do the read */
97 ASSERT(Register <= 0x3F);
98 WRITE_REGISTER_ULONG((PULONG)(IOAPIC_BASE + IOAPIC_IOREGSEL), Register);
99 return READ_REGISTER_ULONG((PULONG)(IOAPIC_BASE + IOAPIC_IOWIN));
100 }
101
102 FORCEINLINE
103 VOID
IOApicWrite(UCHAR Register,ULONG Value)104 IOApicWrite(UCHAR Register, ULONG Value)
105 {
106 /* Select the register, then do the write */
107 ASSERT(Register <= 0x3F);
108 WRITE_REGISTER_ULONG((PULONG)(IOAPIC_BASE + IOAPIC_IOREGSEL), Register);
109 WRITE_REGISTER_ULONG((PULONG)(IOAPIC_BASE + IOAPIC_IOWIN), Value);
110 }
111
112 FORCEINLINE
113 VOID
ApicWriteIORedirectionEntry(UCHAR Index,IOAPIC_REDIRECTION_REGISTER ReDirReg)114 ApicWriteIORedirectionEntry(
115 UCHAR Index,
116 IOAPIC_REDIRECTION_REGISTER ReDirReg)
117 {
118 ASSERT(Index < APIC_MAX_IRQ);
119 IOApicWrite(IOAPIC_REDTBL + 2 * Index, ReDirReg.Long0);
120 IOApicWrite(IOAPIC_REDTBL + 2 * Index + 1, ReDirReg.Long1);
121 }
122
123 FORCEINLINE
124 IOAPIC_REDIRECTION_REGISTER
ApicReadIORedirectionEntry(UCHAR Index)125 ApicReadIORedirectionEntry(
126 UCHAR Index)
127 {
128 IOAPIC_REDIRECTION_REGISTER ReDirReg;
129
130 ASSERT(Index < APIC_MAX_IRQ);
131 ReDirReg.Long0 = IOApicRead(IOAPIC_REDTBL + 2 * Index);
132 ReDirReg.Long1 = IOApicRead(IOAPIC_REDTBL + 2 * Index + 1);
133
134 return ReDirReg;
135 }
136
137 FORCEINLINE
138 VOID
ApicRequestSelfInterrupt(IN UCHAR Vector,UCHAR TriggerMode)139 ApicRequestSelfInterrupt(IN UCHAR Vector, UCHAR TriggerMode)
140 {
141 ULONG Flags;
142 APIC_INTERRUPT_COMMAND_REGISTER Icr;
143 APIC_INTERRUPT_COMMAND_REGISTER IcrStatus;
144
145 /*
146 * The IRR registers are spaced 16 bytes apart and hold 32 status bits each.
147 * Pre-compute the register and bit that match our vector.
148 */
149 ULONG VectorHigh = Vector / 32;
150 ULONG VectorLow = Vector % 32;
151 ULONG Irr = APIC_IRR + 0x10 * VectorHigh;
152 ULONG IrrBit = 1UL << VectorLow;
153
154 /* Setup the command register */
155 Icr.Long0 = 0;
156 Icr.Vector = Vector;
157 Icr.MessageType = APIC_MT_Fixed;
158 Icr.TriggerMode = TriggerMode;
159 Icr.DestinationShortHand = APIC_DSH_Self;
160
161 /* Disable interrupts so that we can change IRR without being interrupted */
162 Flags = __readeflags();
163 _disable();
164
165 /* Wait for the APIC to be idle */
166 do
167 {
168 IcrStatus.Long0 = ApicRead(APIC_ICR0);
169 } while (IcrStatus.DeliveryStatus);
170
171 /* Write the low dword to send the interrupt */
172 ApicWrite(APIC_ICR0, Icr.Long0);
173
174 /* Wait until we see the interrupt request.
175 * It will stay in requested state until we re-enable interrupts.
176 */
177 while (!(ApicRead(Irr) & IrrBit))
178 {
179 NOTHING;
180 }
181
182 /* Finally, restore the original interrupt state */
183 if (Flags & EFLAGS_INTERRUPT_MASK)
184 {
185 _enable();
186 }
187 }
188
189 FORCEINLINE
190 VOID
ApicSendEOI(void)191 ApicSendEOI(void)
192 {
193 ApicWrite(APIC_EOI, 0);
194 }
195
196 FORCEINLINE
197 KIRQL
ApicGetProcessorIrql(VOID)198 ApicGetProcessorIrql(VOID)
199 {
200 /* Read the TPR and convert it to an IRQL */
201 return TprToIrql(ApicRead(APIC_PPR));
202 }
203
204 FORCEINLINE
205 KIRQL
ApicGetCurrentIrql(VOID)206 ApicGetCurrentIrql(VOID)
207 {
208 #ifdef _M_AMD64
209 return (KIRQL)__readcr8();
210 #elif defined(APIC_LAZY_IRQL)
211 /* Return the field in the PCR */
212 return (KIRQL)__readfsbyte(FIELD_OFFSET(KPCR, Irql));
213 #else
214 /* Read the TPR and convert it to an IRQL */
215 return TprToIrql(ApicRead(APIC_TPR));
216 #endif
217 }
218
219 FORCEINLINE
220 VOID
ApicSetIrql(KIRQL Irql)221 ApicSetIrql(KIRQL Irql)
222 {
223 #ifdef _M_AMD64
224 __writecr8(Irql);
225 #elif defined(APIC_LAZY_IRQL)
226 __writefsbyte(FIELD_OFFSET(KPCR, Irql), Irql);
227 #else
228 /* Convert IRQL and write the TPR */
229 ApicWrite(APIC_TPR, IrqlToTpr(Irql));
230 #endif
231 }
232 #define ApicRaiseIrql ApicSetIrql
233
234 #ifdef APIC_LAZY_IRQL
235 FORCEINLINE
236 VOID
ApicLowerIrql(KIRQL Irql)237 ApicLowerIrql(KIRQL Irql)
238 {
239 __writefsbyte(FIELD_OFFSET(KPCR, Irql), Irql);
240
241 /* Is the new Irql lower than set in the TPR? */
242 if (Irql < KeGetPcr()->IRR)
243 {
244 /* Save the new hard IRQL in the IRR field */
245 KeGetPcr()->IRR = Irql;
246
247 /* Need to lower it back */
248 ApicWrite(APIC_TPR, IrqlToTpr(Irql));
249 }
250 }
251 #else
252 #define ApicLowerIrql ApicSetIrql
253 #endif
254
255 UCHAR
256 FASTCALL
HalpIrqToVector(UCHAR Irq)257 HalpIrqToVector(UCHAR Irq)
258 {
259 IOAPIC_REDIRECTION_REGISTER ReDirReg;
260
261 /* Read low dword of the redirection entry */
262 ReDirReg.Long0 = IOApicRead(IOAPIC_REDTBL + 2 * Irq);
263
264 /* Return the vector */
265 return (UCHAR)ReDirReg.Vector;
266 }
267
268 KIRQL
269 FASTCALL
HalpVectorToIrql(UCHAR Vector)270 HalpVectorToIrql(UCHAR Vector)
271 {
272 return TprToIrql(Vector);
273 }
274
275 UCHAR
276 FASTCALL
HalpVectorToIrq(UCHAR Vector)277 HalpVectorToIrq(UCHAR Vector)
278 {
279 return HalpVectorToIndex[Vector];
280 }
281
282 VOID
283 NTAPI
HalpSendEOI(VOID)284 HalpSendEOI(VOID)
285 {
286 ApicSendEOI();
287 }
288
289 VOID
290 NTAPI
ApicInitializeLocalApic(ULONG Cpu)291 ApicInitializeLocalApic(ULONG Cpu)
292 {
293 APIC_BASE_ADDRESS_REGISTER BaseRegister;
294 APIC_SPURIOUS_INERRUPT_REGISTER SpIntRegister;
295 LVT_REGISTER LvtEntry;
296
297 /* Enable the APIC if it wasn't yet */
298 BaseRegister.LongLong = __readmsr(MSR_APIC_BASE);
299 BaseRegister.Enable = 1;
300 BaseRegister.BootStrapCPUCore = (Cpu == 0);
301 __writemsr(MSR_APIC_BASE, BaseRegister.LongLong);
302
303 /* Set spurious vector and SoftwareEnable to 1 */
304 SpIntRegister.Long = ApicRead(APIC_SIVR);
305 SpIntRegister.Vector = APIC_SPURIOUS_VECTOR;
306 SpIntRegister.SoftwareEnable = 1;
307 SpIntRegister.FocusCPUCoreChecking = 0;
308 ApicWrite(APIC_SIVR, SpIntRegister.Long);
309
310 /* Read the version and save it globally */
311 if (Cpu == 0) ApicVersion = ApicRead(APIC_VER);
312
313 /* Set the mode to flat (max 8 CPUs supported!) */
314 ApicWrite(APIC_DFR, APIC_DF_Flat);
315
316 /* Set logical apic ID */
317 ApicWrite(APIC_LDR, ApicLogicalId(Cpu) << 24);
318
319 /* Set the spurious ISR */
320 KeRegisterInterruptHandler(APIC_SPURIOUS_VECTOR, ApicSpuriousService);
321
322 /* Create a template LVT */
323 LvtEntry.Long = 0;
324 LvtEntry.Vector = APIC_FREE_VECTOR;
325 LvtEntry.MessageType = APIC_MT_Fixed;
326 LvtEntry.DeliveryStatus = 0;
327 LvtEntry.RemoteIRR = 0;
328 LvtEntry.TriggerMode = APIC_TGM_Edge;
329 LvtEntry.Mask = 1;
330 LvtEntry.TimerMode = 0;
331
332 /* Initialize and mask LVTs */
333 ApicWrite(APIC_TMRLVTR, LvtEntry.Long);
334 ApicWrite(APIC_THRMLVTR, LvtEntry.Long);
335 ApicWrite(APIC_PCLVTR, LvtEntry.Long);
336 ApicWrite(APIC_EXT0LVTR, LvtEntry.Long);
337 ApicWrite(APIC_EXT1LVTR, LvtEntry.Long);
338 ApicWrite(APIC_EXT2LVTR, LvtEntry.Long);
339 ApicWrite(APIC_EXT3LVTR, LvtEntry.Long);
340
341 /* LINT0 */
342 LvtEntry.Vector = APIC_SPURIOUS_VECTOR;
343 LvtEntry.MessageType = APIC_MT_ExtInt;
344 ApicWrite(APIC_LINT0, LvtEntry.Long);
345
346 /* Enable LINT1 (NMI) */
347 LvtEntry.Mask = 0;
348 LvtEntry.Vector = APIC_NMI_VECTOR;
349 LvtEntry.MessageType = APIC_MT_NMI;
350 LvtEntry.TriggerMode = APIC_TGM_Level;
351 ApicWrite(APIC_LINT1, LvtEntry.Long);
352
353 /* Enable error LVTR */
354 LvtEntry.Vector = APIC_ERROR_VECTOR;
355 LvtEntry.MessageType = APIC_MT_Fixed;
356 ApicWrite(APIC_ERRLVTR, LvtEntry.Long);
357
358 /* Set the IRQL from the PCR */
359 ApicSetIrql(KeGetPcr()->Irql);
360 #ifdef APIC_LAZY_IRQL
361 /* Save the new hard IRQL in the IRR field */
362 KeGetPcr()->IRR = KeGetPcr()->Irql;
363 #endif
364 }
365
366 UCHAR
367 NTAPI
HalpAllocateSystemInterrupt(_In_ UCHAR Irq,_In_ UCHAR Vector)368 HalpAllocateSystemInterrupt(
369 _In_ UCHAR Irq,
370 _In_ UCHAR Vector)
371 {
372 IOAPIC_REDIRECTION_REGISTER ReDirReg;
373
374 ASSERT(Irq < APIC_MAX_IRQ);
375 ASSERT(HalpVectorToIndex[Vector] == APIC_FREE_VECTOR);
376
377 /* Setup a redirection entry */
378 ReDirReg.Vector = Vector;
379 ReDirReg.MessageType = APIC_MT_LowestPriority;
380 ReDirReg.DestinationMode = APIC_DM_Logical;
381 ReDirReg.DeliveryStatus = 0;
382 ReDirReg.Polarity = 0;
383 ReDirReg.RemoteIRR = 0;
384 ReDirReg.TriggerMode = APIC_TGM_Edge;
385 ReDirReg.Mask = 1;
386 ReDirReg.Reserved = 0;
387 ReDirReg.Destination = 0;
388
389 /* Initialize entry */
390 ApicWriteIORedirectionEntry(Irq, ReDirReg);
391
392 /* Save irq in the table */
393 HalpVectorToIndex[Vector] = Irq;
394
395 return Vector;
396 }
397
398 ULONG
399 NTAPI
HalpGetRootInterruptVector(_In_ ULONG BusInterruptLevel,_In_ ULONG BusInterruptVector,_Out_ PKIRQL OutIrql,_Out_ PKAFFINITY OutAffinity)400 HalpGetRootInterruptVector(
401 _In_ ULONG BusInterruptLevel,
402 _In_ ULONG BusInterruptVector,
403 _Out_ PKIRQL OutIrql,
404 _Out_ PKAFFINITY OutAffinity)
405 {
406 UCHAR Vector;
407 KIRQL Irql;
408
409 /* Get the vector currently registered */
410 Vector = HalpIrqToVector(BusInterruptLevel);
411
412 /* Check if it's used */
413 if (Vector != APIC_FREE_VECTOR)
414 {
415 /* Calculate IRQL */
416 NT_ASSERT(HalpVectorToIndex[Vector] == BusInterruptLevel);
417 *OutIrql = HalpVectorToIrql(Vector);
418 }
419 else
420 {
421 ULONG Offset;
422
423 /* Outer loop to find alternative slots, when all IRQLs are in use */
424 for (Offset = 0; Offset < 15; Offset++)
425 {
426 /* Loop allowed IRQL range */
427 for (Irql = CLOCK_LEVEL - 1; Irql >= CMCI_LEVEL; Irql--)
428 {
429 /* Calculate the vactor */
430 Vector = IrqlToTpr(Irql) + Offset;
431
432 /* Check if the vector is free */
433 if (HalpVectorToIrq(Vector) == APIC_FREE_VECTOR)
434 {
435 /* Found one, allocate the interrupt */
436 Vector = HalpAllocateSystemInterrupt(BusInterruptLevel, Vector);
437 *OutIrql = Irql;
438 goto Exit;
439 }
440 }
441 }
442
443 DPRINT1("Failed to get an interrupt vector for IRQ %lu\n", BusInterruptLevel);
444 *OutAffinity = 0;
445 *OutIrql = 0;
446 return 0;
447 }
448
449 Exit:
450
451 *OutAffinity = HalpDefaultInterruptAffinity;
452 ASSERT(HalpDefaultInterruptAffinity);
453
454 return Vector;
455 }
456
457 VOID
458 NTAPI
ApicInitializeIOApic(VOID)459 ApicInitializeIOApic(VOID)
460 {
461 PHARDWARE_PTE Pte;
462 IOAPIC_REDIRECTION_REGISTER ReDirReg;
463 UCHAR Index;
464 ULONG Vector;
465
466 /* Map the I/O Apic page */
467 Pte = HalAddressToPte(IOAPIC_BASE);
468 Pte->PageFrameNumber = IOAPIC_PHYS_BASE / PAGE_SIZE;
469 Pte->Valid = 1;
470 Pte->Write = 1;
471 Pte->Owner = 1;
472 Pte->CacheDisable = 1;
473 Pte->Global = 1;
474 _ReadWriteBarrier();
475
476 /* Setup a redirection entry */
477 ReDirReg.Vector = APIC_FREE_VECTOR;
478 ReDirReg.MessageType = APIC_MT_Fixed;
479 ReDirReg.DestinationMode = APIC_DM_Physical;
480 ReDirReg.DeliveryStatus = 0;
481 ReDirReg.Polarity = 0;
482 ReDirReg.RemoteIRR = 0;
483 ReDirReg.TriggerMode = APIC_TGM_Edge;
484 ReDirReg.Mask = 1;
485 ReDirReg.Reserved = 0;
486 ReDirReg.Destination = 0;
487
488 /* Loop all table entries */
489 for (Index = 0; Index < APIC_MAX_IRQ; Index++)
490 {
491 /* Initialize entry */
492 ApicWriteIORedirectionEntry(Index, ReDirReg);
493 }
494
495 /* Init the vactor to index table */
496 for (Vector = 0; Vector <= 255; Vector++)
497 {
498 HalpVectorToIndex[Vector] = APIC_FREE_VECTOR;
499 }
500
501 /* Enable the timer interrupt (but keep it masked) */
502 ReDirReg.Vector = APIC_CLOCK_VECTOR;
503 ReDirReg.MessageType = APIC_MT_Fixed;
504 ReDirReg.DestinationMode = APIC_DM_Physical;
505 ReDirReg.TriggerMode = APIC_TGM_Edge;
506 ReDirReg.Mask = 1;
507 ReDirReg.Destination = ApicRead(APIC_ID);
508 ApicWriteIORedirectionEntry(APIC_CLOCK_INDEX, ReDirReg);
509 }
510
511 VOID
512 NTAPI
HalpInitializePICs(IN BOOLEAN EnableInterrupts)513 HalpInitializePICs(IN BOOLEAN EnableInterrupts)
514 {
515 ULONG_PTR EFlags;
516
517 /* Save EFlags and disable interrupts */
518 EFlags = __readeflags();
519 _disable();
520
521 /* Initialize and mask the PIC */
522 HalpInitializeLegacyPICs();
523
524 /* Initialize the I/O APIC */
525 ApicInitializeIOApic();
526
527 /* Manually reserve some vectors */
528 HalpVectorToIndex[APC_VECTOR] = APIC_RESERVED_VECTOR;
529 HalpVectorToIndex[DISPATCH_VECTOR] = APIC_RESERVED_VECTOR;
530 HalpVectorToIndex[APIC_CLOCK_VECTOR] = 8;
531 HalpVectorToIndex[CLOCK_IPI_VECTOR] = APIC_RESERVED_VECTOR;
532 HalpVectorToIndex[APIC_SPURIOUS_VECTOR] = APIC_RESERVED_VECTOR;
533
534 /* Set interrupt handlers in the IDT */
535 KeRegisterInterruptHandler(APIC_CLOCK_VECTOR, HalpClockInterrupt);
536 KeRegisterInterruptHandler(CLOCK_IPI_VECTOR, HalpClockIpi);
537 #ifndef _M_AMD64
538 KeRegisterInterruptHandler(APC_VECTOR, HalpApcInterrupt);
539 KeRegisterInterruptHandler(DISPATCH_VECTOR, HalpDispatchInterrupt);
540 #endif
541
542 /* Register the vectors for APC and dispatch interrupts */
543 HalpRegisterVector(IDT_INTERNAL, 0, APC_VECTOR, APC_LEVEL);
544 HalpRegisterVector(IDT_INTERNAL, 0, DISPATCH_VECTOR, DISPATCH_LEVEL);
545
546 /* Restore interrupt state */
547 if (EnableInterrupts) EFlags |= EFLAGS_INTERRUPT_MASK;
548 __writeeflags(EFlags);
549 }
550
551
552 /* SOFTWARE INTERRUPT TRAPS ***************************************************/
553
554 #ifndef _M_AMD64
555 VOID
556 DECLSPEC_NORETURN
557 FASTCALL
HalpApcInterruptHandler(IN PKTRAP_FRAME TrapFrame)558 HalpApcInterruptHandler(IN PKTRAP_FRAME TrapFrame)
559 {
560 KPROCESSOR_MODE ProcessorMode;
561 KIRQL OldIrql;
562 ASSERT(ApicGetProcessorIrql() == APC_LEVEL);
563
564 /* Enter trap */
565 KiEnterInterruptTrap(TrapFrame);
566
567 #ifdef APIC_LAZY_IRQL
568 if (!HalBeginSystemInterrupt(APC_LEVEL, APC_VECTOR, &OldIrql))
569 {
570 /* "Spurious" interrupt, exit the interrupt */
571 KiEoiHelper(TrapFrame);
572 }
573 #else
574 /* Save the old IRQL */
575 OldIrql = ApicGetCurrentIrql();
576 ASSERT(OldIrql < APC_LEVEL);
577 #endif
578
579 /* Raise to APC_LEVEL */
580 ApicRaiseIrql(APC_LEVEL);
581
582 /* End the interrupt */
583 ApicSendEOI();
584
585 /* Kernel or user APC? */
586 if (KiUserTrap(TrapFrame)) ProcessorMode = UserMode;
587 else if (TrapFrame->EFlags & EFLAGS_V86_MASK) ProcessorMode = UserMode;
588 else ProcessorMode = KernelMode;
589
590 /* Enable interrupts and call the kernel's APC interrupt handler */
591 _enable();
592 KiDeliverApc(ProcessorMode, NULL, TrapFrame);
593
594 /* Disable interrupts */
595 _disable();
596
597 /* Restore the old IRQL */
598 ApicLowerIrql(OldIrql);
599
600 /* Exit the interrupt */
601 KiEoiHelper(TrapFrame);
602 }
603
604 VOID
605 DECLSPEC_NORETURN
606 FASTCALL
HalpDispatchInterruptHandler(IN PKTRAP_FRAME TrapFrame)607 HalpDispatchInterruptHandler(IN PKTRAP_FRAME TrapFrame)
608 {
609 KIRQL OldIrql;
610 ASSERT(ApicGetProcessorIrql() == DISPATCH_LEVEL);
611
612 /* Enter trap */
613 KiEnterInterruptTrap(TrapFrame);
614
615 #ifdef APIC_LAZY_IRQL
616 if (!HalBeginSystemInterrupt(DISPATCH_LEVEL, DISPATCH_VECTOR, &OldIrql))
617 {
618 /* "Spurious" interrupt, exit the interrupt */
619 KiEoiHelper(TrapFrame);
620 }
621 #else
622 /* Get the current IRQL */
623 OldIrql = ApicGetCurrentIrql();
624 ASSERT(OldIrql < DISPATCH_LEVEL);
625 #endif
626
627 /* Raise to DISPATCH_LEVEL */
628 ApicRaiseIrql(DISPATCH_LEVEL);
629
630 /* End the interrupt */
631 ApicSendEOI();
632
633 /* Enable interrupts and call the kernel's DPC interrupt handler */
634 _enable();
635 KiDispatchInterrupt();
636 _disable();
637
638 /* Restore the old IRQL */
639 ApicLowerIrql(OldIrql);
640
641 /* Exit the interrupt */
642 KiEoiHelper(TrapFrame);
643 }
644 #endif
645
646
647 /* SOFTWARE INTERRUPTS ********************************************************/
648
649
650 VOID
651 FASTCALL
HalRequestSoftwareInterrupt(IN KIRQL Irql)652 HalRequestSoftwareInterrupt(IN KIRQL Irql)
653 {
654 /* Convert irql to vector and request an interrupt */
655 ApicRequestSelfInterrupt(IrqlToSoftVector(Irql), APIC_TGM_Edge);
656 }
657
658 VOID
659 FASTCALL
HalClearSoftwareInterrupt(IN KIRQL Irql)660 HalClearSoftwareInterrupt(
661 IN KIRQL Irql)
662 {
663 /* Nothing to do */
664 }
665
666
667 /* SYSTEM INTERRUPTS **********************************************************/
668
669 BOOLEAN
670 NTAPI
HalEnableSystemInterrupt(IN ULONG Vector,IN KIRQL Irql,IN KINTERRUPT_MODE InterruptMode)671 HalEnableSystemInterrupt(
672 IN ULONG Vector,
673 IN KIRQL Irql,
674 IN KINTERRUPT_MODE InterruptMode)
675 {
676 IOAPIC_REDIRECTION_REGISTER ReDirReg;
677 PKPRCB Prcb = KeGetCurrentPrcb();
678 UCHAR Index;
679 ASSERT(Irql <= HIGH_LEVEL);
680 ASSERT((IrqlToTpr(Irql) & 0xF0) == (Vector & 0xF0));
681
682 /* Get the irq for this vector */
683 Index = HalpVectorToIndex[Vector];
684
685 /* Check if its valid */
686 if (Index == APIC_FREE_VECTOR)
687 {
688 /* Interrupt is not in use */
689 return FALSE;
690 }
691
692 /* Read the redirection entry */
693 ReDirReg = ApicReadIORedirectionEntry(Index);
694
695 /* Check if the interrupt was unused */
696 if (ReDirReg.Vector != Vector)
697 {
698 ReDirReg.Vector = Vector;
699 ReDirReg.MessageType = APIC_MT_LowestPriority;
700 ReDirReg.DestinationMode = APIC_DM_Logical;
701 ReDirReg.Destination = 0;
702 }
703
704 /* Check if the destination is logical */
705 if (ReDirReg.DestinationMode == APIC_DM_Logical)
706 {
707 /* Set the bit for this cpu */
708 ReDirReg.Destination |= ApicLogicalId(Prcb->Number);
709 }
710
711 /* Set the trigger mode */
712 ReDirReg.TriggerMode = 1 - InterruptMode;
713
714 /* Now unmask it */
715 ReDirReg.Mask = FALSE;
716
717 /* Write back the entry */
718 ApicWriteIORedirectionEntry(Index, ReDirReg);
719
720 return TRUE;
721 }
722
723 VOID
724 NTAPI
HalDisableSystemInterrupt(IN ULONG Vector,IN KIRQL Irql)725 HalDisableSystemInterrupt(
726 IN ULONG Vector,
727 IN KIRQL Irql)
728 {
729 IOAPIC_REDIRECTION_REGISTER ReDirReg;
730 UCHAR Index;
731 ASSERT(Irql <= HIGH_LEVEL);
732 ASSERT(Vector < RTL_NUMBER_OF(HalpVectorToIndex));
733
734 Index = HalpVectorToIndex[Vector];
735
736 /* Read lower dword of redirection entry */
737 ReDirReg.Long0 = IOApicRead(IOAPIC_REDTBL + 2 * Index);
738
739 /* Mask it */
740 ReDirReg.Mask = 1;
741
742 /* Write back lower dword */
743 IOApicWrite(IOAPIC_REDTBL + 2 * Index, ReDirReg.Long0);
744 }
745
746 #ifndef _M_AMD64
747 BOOLEAN
748 NTAPI
HalBeginSystemInterrupt(IN KIRQL Irql,IN ULONG Vector,OUT PKIRQL OldIrql)749 HalBeginSystemInterrupt(
750 IN KIRQL Irql,
751 IN ULONG Vector,
752 OUT PKIRQL OldIrql)
753 {
754 KIRQL CurrentIrql;
755
756 /* Get the current IRQL */
757 CurrentIrql = ApicGetCurrentIrql();
758
759 #ifdef APIC_LAZY_IRQL
760 /* Check if this interrupt is allowed */
761 if (CurrentIrql >= Irql)
762 {
763 IOAPIC_REDIRECTION_REGISTER RedirReg;
764 UCHAR Index;
765
766 /* It is not, set the real Irql in the TPR! */
767 ApicWrite(APIC_TPR, IrqlToTpr(CurrentIrql));
768
769 /* Save the new hard IRQL in the IRR field */
770 KeGetPcr()->IRR = CurrentIrql;
771
772 /* End this interrupt */
773 ApicSendEOI();
774
775 /* Get the irq for this vector */
776 Index = HalpVectorToIndex[Vector];
777
778 /* Check if it's valid */
779 if (Index < APIC_MAX_IRQ)
780 {
781 /* Read the I/O redirection entry */
782 RedirReg = ApicReadIORedirectionEntry(Index);
783
784 /* Re-request the interrupt to be handled later */
785 ApicRequestSelfInterrupt(Vector, (UCHAR)RedirReg.TriggerMode);
786 }
787 else
788 {
789 /* This should be a reserved vector! */
790 ASSERT(Index == APIC_RESERVED_VECTOR);
791
792 /* Re-request the interrupt to be handled later */
793 ApicRequestSelfInterrupt(Vector, APIC_TGM_Edge);
794 }
795
796 /* Pretend it was a spurious interrupt */
797 return FALSE;
798 }
799 #endif
800 /* Save the current IRQL */
801 *OldIrql = CurrentIrql;
802
803 /* Set the new IRQL */
804 ApicRaiseIrql(Irql);
805
806 /* Turn on interrupts */
807 _enable();
808
809 /* Success */
810 return TRUE;
811 }
812
813 VOID
814 NTAPI
HalEndSystemInterrupt(IN KIRQL OldIrql,IN PKTRAP_FRAME TrapFrame)815 HalEndSystemInterrupt(
816 IN KIRQL OldIrql,
817 IN PKTRAP_FRAME TrapFrame)
818 {
819 /* Send an EOI */
820 ApicSendEOI();
821
822 /* Restore the old IRQL */
823 ApicLowerIrql(OldIrql);
824 }
825
826
827 /* IRQL MANAGEMENT ************************************************************/
828
829 KIRQL
830 NTAPI
KeGetCurrentIrql(VOID)831 KeGetCurrentIrql(VOID)
832 {
833 /* Read the current TPR and convert it to an IRQL */
834 return ApicGetCurrentIrql();
835 }
836
837 VOID
838 FASTCALL
KfLowerIrql(IN KIRQL OldIrql)839 KfLowerIrql(
840 IN KIRQL OldIrql)
841 {
842 #if DBG
843 /* Validate correct lower */
844 if (OldIrql > ApicGetCurrentIrql())
845 {
846 /* Crash system */
847 KeBugCheck(IRQL_NOT_LESS_OR_EQUAL);
848 }
849 #endif
850 /* Set the new IRQL */
851 ApicLowerIrql(OldIrql);
852 }
853
854 KIRQL
855 FASTCALL
KfRaiseIrql(IN KIRQL NewIrql)856 KfRaiseIrql(
857 IN KIRQL NewIrql)
858 {
859 KIRQL OldIrql;
860
861 /* Read the current IRQL */
862 OldIrql = ApicGetCurrentIrql();
863 #if DBG
864 /* Validate correct raise */
865 if (OldIrql > NewIrql)
866 {
867 /* Crash system */
868 KeBugCheck(IRQL_NOT_GREATER_OR_EQUAL);
869 }
870 #endif
871 /* Convert the new IRQL to a TPR value and write the register */
872 ApicRaiseIrql(NewIrql);
873
874 /* Return old IRQL */
875 return OldIrql;
876 }
877
878 KIRQL
879 NTAPI
KeRaiseIrqlToDpcLevel(VOID)880 KeRaiseIrqlToDpcLevel(VOID)
881 {
882 return KfRaiseIrql(DISPATCH_LEVEL);
883 }
884
885 KIRQL
886 NTAPI
KeRaiseIrqlToSynchLevel(VOID)887 KeRaiseIrqlToSynchLevel(VOID)
888 {
889 return KfRaiseIrql(SYNCH_LEVEL);
890 }
891
892 #endif /* !_M_AMD64 */
893
894