xref: /reactos/ntoskrnl/ke/spinlock.c (revision d6eebaa4) 
1 /*
2  * PROJECT:         ReactOS Kernel
3  * LICENSE:         GPL - See COPYING in the top level directory
4  * FILE:            ntoskrnl/ke/spinlock.c
5  * PURPOSE:         Spinlock and Queued Spinlock Support
6  * PROGRAMMERS:     Alex Ionescu (alex.ionescu@reactos.org)
7  */
8 
9 /* INCLUDES ******************************************************************/
10 
11 #include <ntoskrnl.h>
12 #define NDEBUG
13 #include <debug.h>
14 
15 #define LQ_WAIT     1
16 #define LQ_OWN      2
17 
18 /* PRIVATE FUNCTIONS *********************************************************/
19 
20 #if 0
21 //
22 // FIXME: The queued spinlock routines are broken.
23 //
24 
25 VOID
26 FASTCALL
27 KeAcquireQueuedSpinLockAtDpcLevel(IN PKSPIN_LOCK_QUEUE LockHandle)
28 {
29 #ifdef CONFIG_SMP
30     PKSPIN_LOCK_QUEUE Prev;
31 
32     /* Set the new lock */
33     Prev = (PKSPIN_LOCK_QUEUE)
34            InterlockedExchange((PLONG)LockHandle->Next,
35                                (LONG)LockHandle);
36     if (!Prev)
37     {
38         /* There was nothing there before. We now own it */
39          *LockHandle->Lock |= LQ_OWN;
40         return;
41     }
42 
43     /* Set the wait flag */
44      *LockHandle->Lock |= LQ_WAIT;
45 
46     /* Link us */
47     Prev->Next = (PKSPIN_LOCK_QUEUE)LockHandle;
48 
49     /* Loop and wait */
50     while (*LockHandle->Lock & LQ_WAIT)
51         YieldProcessor();
52 #endif
53 }
54 
55 VOID
56 FASTCALL
57 KeReleaseQueuedSpinLockFromDpcLevel(IN PKSPIN_LOCK_QUEUE LockHandle)
58 {
59 #ifdef CONFIG_SMP
60     KSPIN_LOCK LockVal;
61     PKSPIN_LOCK_QUEUE Waiter;
62 
63     /* Remove own and wait flags */
64     *LockHandle->Lock &= ~(LQ_OWN | LQ_WAIT);
65     LockVal = *LockHandle->Lock;
66 
67     /* Check if we already own it */
68     if (LockVal == (KSPIN_LOCK)LockHandle)
69     {
70         /* Disown it */
71         LockVal = (KSPIN_LOCK)
72                   InterlockedCompareExchangePointer(LockHandle->Lock,
73                                                     NULL,
74                                                     LockHandle);
75     }
76     if (LockVal == (KSPIN_LOCK)LockHandle) return;
77 
78     /* Need to wait for it */
79     Waiter = LockHandle->Next;
80     while (!Waiter)
81     {
82         YieldProcessor();
83         Waiter = LockHandle->Next;
84     }
85 
86     /* It's gone */
87     *(ULONG_PTR*)&Waiter->Lock ^= (LQ_OWN | LQ_WAIT);
88     LockHandle->Next = NULL;
89 #endif
90 }
91 
92 #else
93 //
94 // HACK: Hacked to work like normal spinlocks
95 //
96 
97 _IRQL_requires_min_(DISPATCH_LEVEL)
98 _Acquires_nonreentrant_lock_(*LockHandle->Lock)
99 _Acquires_exclusive_lock_(*LockHandle->Lock)
100 VOID
101 FASTCALL
102 KeAcquireQueuedSpinLockAtDpcLevel(_Inout_ PKSPIN_LOCK_QUEUE LockHandle)
103 {
104 #if defined(CONFIG_SMP) || DBG
105     /* Make sure we are at DPC or above! */
106     if (KeGetCurrentIrql() < DISPATCH_LEVEL)
107     {
108         /* We aren't -- bugcheck */
109         KeBugCheckEx(IRQL_NOT_GREATER_OR_EQUAL,
110                      (ULONG_PTR)LockHandle->Lock,
111                      KeGetCurrentIrql(),
112                      0,
113                      0);
114     }
115 #endif
116 
117     /* Do the inlined function */
118     KxAcquireSpinLock(LockHandle->Lock);
119 }
120 
121 _IRQL_requires_min_(DISPATCH_LEVEL)
122 _Releases_nonreentrant_lock_(*LockHandle->Lock)
123 _Releases_exclusive_lock_(*LockHandle->Lock)
124 VOID
125 FASTCALL
126 KeReleaseQueuedSpinLockFromDpcLevel(_Inout_ PKSPIN_LOCK_QUEUE LockHandle)
127 {
128 #if defined(CONFIG_SMP) || DBG
129     /* Make sure we are at DPC or above! */
130     if (KeGetCurrentIrql() < DISPATCH_LEVEL)
131     {
132         /* We aren't -- bugcheck */
133         KeBugCheckEx(IRQL_NOT_GREATER_OR_EQUAL,
134                      (ULONG_PTR)LockHandle->Lock,
135                      KeGetCurrentIrql(),
136                      0,
137                      0);
138     }
139 #endif
140 
141     /* Do the inlined function */
142     KxReleaseSpinLock(LockHandle->Lock);
143 }
144 
145 #endif
146 
147 /* PUBLIC FUNCTIONS **********************************************************/
148 
149 /*
150  * @implemented
151  */
152 KIRQL
153 NTAPI
154 KeAcquireInterruptSpinLock(IN PKINTERRUPT Interrupt)
155 {
156     KIRQL OldIrql;
157 
158     /* Raise IRQL */
159     KeRaiseIrql(Interrupt->SynchronizeIrql, &OldIrql);
160 
161     /* Acquire spinlock on MP */
162     KeAcquireSpinLockAtDpcLevel(Interrupt->ActualLock);
163     return OldIrql;
164 }
165 
166 /*
167  * @implemented
168  */
169 VOID
170 NTAPI
171 KeReleaseInterruptSpinLock(IN PKINTERRUPT Interrupt,
172                            IN KIRQL OldIrql)
173 {
174     /* Release lock on MP */
175     KeReleaseSpinLockFromDpcLevel(Interrupt->ActualLock);
176 
177     /* Lower IRQL */
178     KeLowerIrql(OldIrql);
179 }
180 
181 /*
182  * @implemented
183  */
184 VOID
185 NTAPI
186 _KeInitializeSpinLock(IN PKSPIN_LOCK SpinLock)
187 {
188     /* Clear it */
189     *SpinLock = 0;
190 }
191 
192 /*
193  * @implemented
194  */
195 #undef KeAcquireSpinLockAtDpcLevel
196 VOID
197 NTAPI
198 KeAcquireSpinLockAtDpcLevel(IN PKSPIN_LOCK SpinLock)
199 {
200     /* Make sure we are at DPC or above! */
201     if (KeGetCurrentIrql() < DISPATCH_LEVEL)
202     {
203         /* We aren't -- bugcheck */
204         KeBugCheckEx(IRQL_NOT_GREATER_OR_EQUAL,
205                      (ULONG_PTR)SpinLock,
206                      KeGetCurrentIrql(),
207                      0,
208                      0);
209     }
210 
211     /* Do the inlined function */
212     KxAcquireSpinLock(SpinLock);
213 }
214 
215 /*
216  * @implemented
217  */
218 #undef KeReleaseSpinLockFromDpcLevel
219 VOID
220 NTAPI
221 KeReleaseSpinLockFromDpcLevel(IN PKSPIN_LOCK SpinLock)
222 {
223     /* Make sure we are at DPC or above! */
224     if (KeGetCurrentIrql() < DISPATCH_LEVEL)
225     {
226         /* We aren't -- bugcheck */
227         KeBugCheckEx(IRQL_NOT_GREATER_OR_EQUAL,
228                      (ULONG_PTR)SpinLock,
229                      KeGetCurrentIrql(),
230                      0,
231                      0);
232     }
233 
234     /* Do the inlined function */
235     KxReleaseSpinLock(SpinLock);
236 }
237 
238 /*
239  * @implemented
240  */
241 VOID
242 FASTCALL
243 KefAcquireSpinLockAtDpcLevel(IN PKSPIN_LOCK SpinLock)
244 {
245     /* Make sure we are at DPC or above! */
246     if (KeGetCurrentIrql() < DISPATCH_LEVEL)
247     {
248         /* We aren't -- bugcheck */
249         KeBugCheckEx(IRQL_NOT_GREATER_OR_EQUAL,
250                      (ULONG_PTR)SpinLock,
251                      KeGetCurrentIrql(),
252                      0,
253                      0);
254     }
255 
256     /* Do the inlined function */
257     KxAcquireSpinLock(SpinLock);
258 }
259 
260 /*
261  * @implemented
262  */
263 VOID
264 FASTCALL
265 KefReleaseSpinLockFromDpcLevel(IN PKSPIN_LOCK SpinLock)
266 {
267     /* Make sure we are at DPC or above! */
268     if (KeGetCurrentIrql() < DISPATCH_LEVEL)
269     {
270         /* We aren't -- bugcheck */
271         KeBugCheckEx(IRQL_NOT_GREATER_OR_EQUAL,
272                      (ULONG_PTR)SpinLock,
273                      KeGetCurrentIrql(),
274                      0,
275                      0);
276     }
277 
278     /* Do the inlined function */
279     KxReleaseSpinLock(SpinLock);
280 }
281 
282 /*
283  * @implemented
284  */
285 VOID
286 FASTCALL
287 KiAcquireSpinLock(IN PKSPIN_LOCK SpinLock)
288 {
289     /* Do the inlined function */
290     KxAcquireSpinLock(SpinLock);
291 }
292 
293 /*
294  * @implemented
295  */
296 VOID
297 FASTCALL
298 KiReleaseSpinLock(IN PKSPIN_LOCK SpinLock)
299 {
300     /* Do the inlined function */
301     KxReleaseSpinLock(SpinLock);
302 }
303 
304 /*
305  * @implemented
306  */
307 BOOLEAN
308 FASTCALL
309 KeTryToAcquireSpinLockAtDpcLevel(IN OUT PKSPIN_LOCK SpinLock)
310 {
311 #if DBG
312     /* Make sure we are at DPC or above! */
313     if (KeGetCurrentIrql() < DISPATCH_LEVEL)
314     {
315         /* We aren't -- bugcheck */
316         KeBugCheckEx(IRQL_NOT_GREATER_OR_EQUAL,
317                      (ULONG_PTR)SpinLock,
318                      KeGetCurrentIrql(),
319                      0,
320                      0);
321     }
322 
323     /* Make sure that we don't own the lock already */
324     if (((KSPIN_LOCK)KeGetCurrentThread() | 1) == *SpinLock)
325     {
326         /* We do, bugcheck! */
327         KeBugCheckEx(SPIN_LOCK_ALREADY_OWNED, (ULONG_PTR)SpinLock, 0, 0, 0);
328     }
329 #endif
330 
331 #ifdef CONFIG_SMP
332     /* Check if it's already acquired */
333     if (!(*SpinLock))
334     {
335         /* Try to acquire it */
336         if (InterlockedBitTestAndSet((PLONG)SpinLock, 0))
337         {
338             /* Someone else acquired it */
339             return FALSE;
340         }
341     }
342     else
343     {
344         /* It was already acquired */
345         return FALSE;
346     }
347 #endif
348 
349 #if DBG
350     /* On debug builds, we OR in the KTHREAD */
351     *SpinLock = (ULONG_PTR)KeGetCurrentThread() | 1;
352 #endif
353 
354     /* All is well, return TRUE */
355     return TRUE;
356 }
357 
358 /*
359  * @implemented
360  */
361 VOID
362 FASTCALL
363 KeAcquireInStackQueuedSpinLockAtDpcLevel(IN PKSPIN_LOCK SpinLock,
364                                          IN PKLOCK_QUEUE_HANDLE LockHandle)
365 {
366     /* Set it up properly */
367     LockHandle->LockQueue.Next = NULL;
368     LockHandle->LockQueue.Lock = SpinLock;
369 #ifdef CONFIG_SMP
370 #if 0
371     KeAcquireQueuedSpinLockAtDpcLevel(LockHandle->LockQueue.Next);
372 #else
373     /* Make sure we are at DPC or above! */
374     if (KeGetCurrentIrql() < DISPATCH_LEVEL)
375     {
376         /* We aren't -- bugcheck */
377         KeBugCheckEx(IRQL_NOT_GREATER_OR_EQUAL,
378                      (ULONG_PTR)LockHandle->LockQueue.Lock,
379                      KeGetCurrentIrql(),
380                      0,
381                      0);
382     }
383 #endif
384 #endif
385 
386     /* Acquire the lock */
387     KxAcquireSpinLock(LockHandle->LockQueue.Lock); // HACK
388 }
389 
390 /*
391  * @implemented
392  */
393 VOID
394 FASTCALL
395 KeReleaseInStackQueuedSpinLockFromDpcLevel(IN PKLOCK_QUEUE_HANDLE LockHandle)
396 {
397 #ifdef CONFIG_SMP
398 #if 0
399     /* Call the internal function */
400     KeReleaseQueuedSpinLockFromDpcLevel(LockHandle->LockQueue.Next);
401 #else
402     /* Make sure we are at DPC or above! */
403     if (KeGetCurrentIrql() < DISPATCH_LEVEL)
404     {
405         /* We aren't -- bugcheck */
406         KeBugCheckEx(IRQL_NOT_GREATER_OR_EQUAL,
407                      (ULONG_PTR)LockHandle->LockQueue.Lock,
408                      KeGetCurrentIrql(),
409                      0,
410                      0);
411     }
412 #endif
413 #endif
414 
415     /* Release the lock */
416     KxReleaseSpinLock(LockHandle->LockQueue.Lock); // HACK
417 }
418 
419 /*
420  * @unimplemented
421  */
422 KIRQL
423 FASTCALL
424 KeAcquireSpinLockForDpc(IN PKSPIN_LOCK SpinLock)
425 {
426     UNIMPLEMENTED;
427     return 0;
428 }
429 
430 /*
431  * @unimplemented
432  */
433 VOID
434 FASTCALL
435 KeReleaseSpinLockForDpc(IN PKSPIN_LOCK SpinLock,
436                         IN KIRQL OldIrql)
437 {
438     UNIMPLEMENTED;
439 }
440 
441 /*
442  * @implemented
443  */
444 VOID
445 FASTCALL
446 KeAcquireInStackQueuedSpinLockForDpc(IN PKSPIN_LOCK SpinLock,
447                                      IN PKLOCK_QUEUE_HANDLE LockHandle)
448 {
449     LockHandle->OldIrql = KeGetCurrentIrql();
450     if (LockHandle->OldIrql >= DISPATCH_LEVEL)
451         KeAcquireInStackQueuedSpinLockAtDpcLevel(SpinLock, LockHandle);
452     else
453         KeAcquireInStackQueuedSpinLock(SpinLock, LockHandle);
454 }
455 
456 /*
457  * @implemented
458  */
459 VOID
460 FASTCALL
461 KeReleaseInStackQueuedSpinLockForDpc(IN PKLOCK_QUEUE_HANDLE LockHandle)
462 {
463     if (LockHandle->OldIrql >= DISPATCH_LEVEL)
464         KeReleaseInStackQueuedSpinLockFromDpcLevel(LockHandle);
465     else
466         KeReleaseInStackQueuedSpinLock(LockHandle);
467 
468 }
469 
470 /*
471  * @implemented
472  */
473 BOOLEAN
474 FASTCALL
475 KeTestSpinLock(IN PKSPIN_LOCK SpinLock)
476 {
477     /* Test this spinlock */
478     if (*SpinLock)
479     {
480         /* Spinlock is busy, yield execution */
481         YieldProcessor();
482 
483         /* Return busy flag */
484         return FALSE;
485     }
486 
487     /* Spinlock appears to be free */
488     return TRUE;
489 }
490 
491 #ifdef _M_IX86
492 VOID
493 NTAPI
494 Kii386SpinOnSpinLock(PKSPIN_LOCK SpinLock, ULONG Flags)
495 {
496     // FIXME: Handle flags
497     UNREFERENCED_PARAMETER(Flags);
498 
499     /* Spin until it's unlocked */
500     while (*(volatile KSPIN_LOCK *)SpinLock & 1)
501     {
502         // FIXME: Check for timeout
503 
504         /* Yield and keep looping */
505         YieldProcessor();
506     }
507 }
508 #endif
509