1 /*
2 * PROJECT: ReactOS Kernel
3 * LICENSE: GPL - See COPYING in the top level directory
4 * FILE: ntoskrnl/ex/interlocked.c
5 * PURPOSE: Interlocked functions
6 * PROGRAMMERS: Timo Kreuzer (timo.kreuzer@reactos.org)
7 */
8
9 /* INCLUDES *****************************************************************/
10
11 #include <ntoskrnl.h>
12
13 #define NDEBUG
14 #include <debug.h>
15
16 #undef ExInterlockedAddUlong
17 #undef ExInterlockedInsertHeadList
18 #undef ExInterlockedInsertTailList
19 #undef ExInterlockedRemoveHeadList
20 #undef ExInterlockedPopEntryList
21 #undef ExInterlockedPushEntryList
22 #undef ExInterlockedIncrementLong
23 #undef ExInterlockedDecrementLong
24 #undef ExInterlockedExchangeUlong
25 #undef ExInterlockedCompareExchange64
26
27
28 /* FUNCTIONS ****************************************************************/
29
30 FORCEINLINE
31 BOOLEAN
_ExiDisableInterruptsAndAcquireSpinlock(IN OUT PKSPIN_LOCK Lock)32 _ExiDisableInterruptsAndAcquireSpinlock(
33 IN OUT PKSPIN_LOCK Lock)
34 {
35 BOOLEAN Enabled;
36
37 /* Disable interrupts */
38 Enabled = KeDisableInterrupts();
39
40 /* Acquire the spinlock (inline) */
41 KxAcquireSpinLock(Lock);
42
43 return Enabled;
44 }
45
46 FORCEINLINE
47 VOID
_ExiReleaseSpinLockAndRestoreInterrupts(IN OUT PKSPIN_LOCK Lock,BOOLEAN Enable)48 _ExiReleaseSpinLockAndRestoreInterrupts(
49 IN OUT PKSPIN_LOCK Lock,
50 BOOLEAN Enable)
51 {
52 /* Release the spinlock */
53 KxReleaseSpinLock(Lock);
54
55 /* Restore interrupts */
56 KeRestoreInterrupts(Enable);
57 }
58
59
60 LARGE_INTEGER
61 NTAPI
ExInterlockedAddLargeInteger(IN OUT PLARGE_INTEGER Addend,IN LARGE_INTEGER Increment,IN OUT PKSPIN_LOCK Lock)62 ExInterlockedAddLargeInteger(
63 IN OUT PLARGE_INTEGER Addend,
64 IN LARGE_INTEGER Increment,
65 IN OUT PKSPIN_LOCK Lock)
66 {
67 LARGE_INTEGER OldValue;
68 BOOLEAN Enable;
69
70 /* Disable interrupts and acquire the spinlock */
71 Enable = _ExiDisableInterruptsAndAcquireSpinlock(Lock);
72
73 /* Save the old value */
74 OldValue.QuadPart = Addend->QuadPart;
75
76 /* Do the operation */
77 Addend->QuadPart += Increment.QuadPart;
78
79 /* Release the spinlock and restore interrupts */
80 _ExiReleaseSpinLockAndRestoreInterrupts(Lock, Enable);
81
82 /* Return the old value */
83 return OldValue;
84 }
85
86 ULONG
87 NTAPI
ExInterlockedAddUlong(IN OUT PULONG Addend,IN ULONG Increment,IN OUT PKSPIN_LOCK Lock)88 ExInterlockedAddUlong(
89 IN OUT PULONG Addend,
90 IN ULONG Increment,
91 IN OUT PKSPIN_LOCK Lock)
92 {
93 BOOLEAN Enable;
94 ULONG OldValue;
95
96 /* Disable interrupts and acquire the spinlock */
97 Enable = _ExiDisableInterruptsAndAcquireSpinlock(Lock);
98
99 /* Save the old value */
100 OldValue = *Addend;
101
102 /* Do the operation */
103 *Addend += Increment;
104
105 /* Release the spinlock and restore interrupts */
106 _ExiReleaseSpinLockAndRestoreInterrupts(Lock, Enable);
107
108 /* Return the old value */
109 return OldValue;
110 }
111
112 PLIST_ENTRY
113 NTAPI
ExInterlockedInsertHeadList(IN OUT PLIST_ENTRY ListHead,IN OUT PLIST_ENTRY ListEntry,IN OUT PKSPIN_LOCK Lock)114 ExInterlockedInsertHeadList(
115 IN OUT PLIST_ENTRY ListHead,
116 IN OUT PLIST_ENTRY ListEntry,
117 IN OUT PKSPIN_LOCK Lock)
118 {
119 BOOLEAN Enable;
120 PLIST_ENTRY FirstEntry;
121
122 /* Disable interrupts and acquire the spinlock */
123 Enable = _ExiDisableInterruptsAndAcquireSpinlock(Lock);
124
125 /* Save the first entry */
126 FirstEntry = ListHead->Flink;
127
128 /* Insert the new entry */
129 InsertHeadList(ListHead, ListEntry);
130
131 /* Release the spinlock and restore interrupts */
132 _ExiReleaseSpinLockAndRestoreInterrupts(Lock, Enable);
133
134 /* Return the old first entry or NULL for empty list */
135 return (FirstEntry == ListHead) ? NULL : FirstEntry;
136 }
137
138 PLIST_ENTRY
139 NTAPI
ExInterlockedInsertTailList(IN OUT PLIST_ENTRY ListHead,IN OUT PLIST_ENTRY ListEntry,IN OUT PKSPIN_LOCK Lock)140 ExInterlockedInsertTailList(
141 IN OUT PLIST_ENTRY ListHead,
142 IN OUT PLIST_ENTRY ListEntry,
143 IN OUT PKSPIN_LOCK Lock)
144 {
145 BOOLEAN Enable;
146 PLIST_ENTRY LastEntry;
147
148 /* Disable interrupts and acquire the spinlock */
149 Enable = _ExiDisableInterruptsAndAcquireSpinlock(Lock);
150
151 /* Save the last entry */
152 LastEntry = ListHead->Blink;
153
154 /* Insert the new entry */
155 InsertTailList(ListHead, ListEntry);
156
157 /* Release the spinlock and restore interrupts */
158 _ExiReleaseSpinLockAndRestoreInterrupts(Lock, Enable);
159
160 /* Return the old last entry or NULL for empty list */
161 return (LastEntry == ListHead) ? NULL : LastEntry;
162 }
163
164 PLIST_ENTRY
165 NTAPI
ExInterlockedRemoveHeadList(IN OUT PLIST_ENTRY ListHead,IN OUT PKSPIN_LOCK Lock)166 ExInterlockedRemoveHeadList(
167 IN OUT PLIST_ENTRY ListHead,
168 IN OUT PKSPIN_LOCK Lock)
169 {
170 BOOLEAN Enable;
171 PLIST_ENTRY ListEntry;
172
173 /* Disable interrupts and acquire the spinlock */
174 Enable = _ExiDisableInterruptsAndAcquireSpinlock(Lock);
175
176 /* Check if the list is empty */
177 if (IsListEmpty(ListHead))
178 {
179 /* Return NULL */
180 ListEntry = NULL;
181 }
182 else
183 {
184 /* Remove the first entry from the list head */
185 ListEntry = RemoveHeadList(ListHead);
186 #if DBG
187 ListEntry->Flink = (PLIST_ENTRY)(ULONG_PTR)0xBADDD0FFBADDD0FFULL;
188 ListEntry->Blink = (PLIST_ENTRY)(ULONG_PTR)0xBADDD0FFBADDD0FFULL;
189 #endif
190 }
191
192 /* Release the spinlock and restore interrupts */
193 _ExiReleaseSpinLockAndRestoreInterrupts(Lock, Enable);
194
195 /* Return the entry */
196 return ListEntry;
197 }
198
199 PSINGLE_LIST_ENTRY
200 NTAPI
ExInterlockedPopEntryList(IN OUT PSINGLE_LIST_ENTRY ListHead,IN OUT PKSPIN_LOCK Lock)201 ExInterlockedPopEntryList(
202 IN OUT PSINGLE_LIST_ENTRY ListHead,
203 IN OUT PKSPIN_LOCK Lock)
204 {
205 BOOLEAN Enable;
206 PSINGLE_LIST_ENTRY ListEntry;
207
208 /* Disable interrupts and acquire the spinlock */
209 Enable = _ExiDisableInterruptsAndAcquireSpinlock(Lock);
210
211 /* Pop the first entry from the list */
212 ListEntry = PopEntryList(ListHead);
213 #if DBG
214 if (ListEntry)
215 ListEntry->Next = (PSINGLE_LIST_ENTRY)(ULONG_PTR)0xBADDD0FFBADDD0FFULL;
216 #endif
217
218 /* Release the spinlock and restore interrupts */
219 _ExiReleaseSpinLockAndRestoreInterrupts(Lock, Enable);
220
221 /* Return the entry */
222 return ListEntry;
223 }
224
225 PSINGLE_LIST_ENTRY
226 NTAPI
ExInterlockedPushEntryList(IN OUT PSINGLE_LIST_ENTRY ListHead,IN OUT PSINGLE_LIST_ENTRY ListEntry,IN OUT PKSPIN_LOCK Lock)227 ExInterlockedPushEntryList(
228 IN OUT PSINGLE_LIST_ENTRY ListHead,
229 IN OUT PSINGLE_LIST_ENTRY ListEntry,
230 IN OUT PKSPIN_LOCK Lock)
231 {
232 BOOLEAN Enable;
233 PSINGLE_LIST_ENTRY OldListEntry;
234
235 /* Disable interrupts and acquire the spinlock */
236 Enable = _ExiDisableInterruptsAndAcquireSpinlock(Lock);
237
238 /* Save the old top entry */
239 OldListEntry = ListHead->Next;
240
241 /* Push a new entry on the list */
242 PushEntryList(ListHead, ListEntry);
243
244 /* Release the spinlock and restore interrupts */
245 _ExiReleaseSpinLockAndRestoreInterrupts(Lock, Enable);
246
247 /* Return the entry */
248 return OldListEntry;
249 }
250
251 INTERLOCKED_RESULT
252 NTAPI
ExInterlockedIncrementLong(IN PLONG Addend,IN PKSPIN_LOCK Lock)253 ExInterlockedIncrementLong(
254 IN PLONG Addend,
255 IN PKSPIN_LOCK Lock)
256 {
257 LONG Result;
258
259 Result = _InterlockedIncrement(Addend);
260 return (Result < 0) ? ResultNegative :
261 (Result > 0) ? ResultPositive :
262 ResultZero;
263 }
264
265 INTERLOCKED_RESULT
266 NTAPI
ExInterlockedDecrementLong(IN PLONG Addend,IN PKSPIN_LOCK Lock)267 ExInterlockedDecrementLong(
268 IN PLONG Addend,
269 IN PKSPIN_LOCK Lock)
270 {
271 LONG Result;
272
273 Result = _InterlockedDecrement(Addend);
274 return (Result < 0) ? ResultNegative :
275 (Result > 0) ? ResultPositive :
276 ResultZero;
277 }
278
279 ULONG
280 NTAPI
ExInterlockedExchangeUlong(IN PULONG Target,IN ULONG Value,IN PKSPIN_LOCK Lock)281 ExInterlockedExchangeUlong(
282 IN PULONG Target,
283 IN ULONG Value,
284 IN PKSPIN_LOCK Lock)
285 {
286 return (ULONG)_InterlockedExchange((PLONG)Target, (LONG)Value);
287 }
288
289 #ifdef _M_IX86
290
291 ULONG
292 FASTCALL
ExfInterlockedAddUlong(IN OUT PULONG Addend,IN ULONG Increment,IN OUT PKSPIN_LOCK Lock)293 ExfInterlockedAddUlong(
294 IN OUT PULONG Addend,
295 IN ULONG Increment,
296 IN OUT PKSPIN_LOCK Lock)
297 {
298 BOOLEAN Enable;
299 ULONG OldValue;
300
301 /* Disable interrupts and acquire the spinlock */
302 Enable = _ExiDisableInterruptsAndAcquireSpinlock(Lock);
303
304 /* Save the old value */
305 OldValue = *Addend;
306
307 /* Do the operation */
308 *Addend += Increment;
309
310 /* Release the spinlock and restore interrupts */
311 _ExiReleaseSpinLockAndRestoreInterrupts(Lock, Enable);
312
313 /* Return the old value */
314 return OldValue;
315 }
316
317 PLIST_ENTRY
318 FASTCALL
ExfInterlockedInsertHeadList(IN OUT PLIST_ENTRY ListHead,IN PLIST_ENTRY ListEntry,IN OUT PKSPIN_LOCK Lock)319 ExfInterlockedInsertHeadList(
320 IN OUT PLIST_ENTRY ListHead,
321 IN PLIST_ENTRY ListEntry,
322 IN OUT PKSPIN_LOCK Lock)
323 {
324 BOOLEAN Enable;
325 PLIST_ENTRY FirstEntry;
326
327 /* Disable interrupts and acquire the spinlock */
328 Enable = _ExiDisableInterruptsAndAcquireSpinlock(Lock);
329
330 /* Save the first entry */
331 FirstEntry = ListHead->Flink;
332
333 /* Insert the new entry */
334 InsertHeadList(ListHead, ListEntry);
335
336 /* Release the spinlock and restore interrupts */
337 _ExiReleaseSpinLockAndRestoreInterrupts(Lock, Enable);
338
339 /* Return the old first entry or NULL for empty list */
340 return (FirstEntry == ListHead) ? NULL : FirstEntry;
341 }
342
343 PLIST_ENTRY
344 FASTCALL
ExfInterlockedInsertTailList(IN OUT PLIST_ENTRY ListHead,IN PLIST_ENTRY ListEntry,IN OUT PKSPIN_LOCK Lock)345 ExfInterlockedInsertTailList(
346 IN OUT PLIST_ENTRY ListHead,
347 IN PLIST_ENTRY ListEntry,
348 IN OUT PKSPIN_LOCK Lock)
349 {
350 BOOLEAN Enable;
351 PLIST_ENTRY LastEntry;
352
353 /* Disable interrupts and acquire the spinlock */
354 Enable = _ExiDisableInterruptsAndAcquireSpinlock(Lock);
355
356 /* Save the last entry */
357 LastEntry = ListHead->Blink;
358
359 /* Insert the new entry */
360 InsertTailList(ListHead, ListEntry);
361
362 /* Release the spinlock and restore interrupts */
363 _ExiReleaseSpinLockAndRestoreInterrupts(Lock, Enable);
364
365 /* Return the old last entry or NULL for empty list */
366 return (LastEntry == ListHead) ? NULL : LastEntry;
367 }
368
369
370 PLIST_ENTRY
371 FASTCALL
ExfInterlockedRemoveHeadList(IN OUT PLIST_ENTRY ListHead,IN OUT PKSPIN_LOCK Lock)372 ExfInterlockedRemoveHeadList(
373 IN OUT PLIST_ENTRY ListHead,
374 IN OUT PKSPIN_LOCK Lock)
375 {
376 BOOLEAN Enable;
377 PLIST_ENTRY ListEntry;
378
379 /* Disable interrupts and acquire the spinlock */
380 Enable = _ExiDisableInterruptsAndAcquireSpinlock(Lock);
381
382 /* Check if the list is empty */
383 if (IsListEmpty(ListHead))
384 {
385 /* Return NULL */
386 ListEntry = NULL;
387 }
388 else
389 {
390 /* Remove the first entry from the list head */
391 ListEntry = RemoveHeadList(ListHead);
392 #if DBG
393 ListEntry->Flink = (PLIST_ENTRY)0x0BADD0FF;
394 ListEntry->Blink = (PLIST_ENTRY)0x0BADD0FF;
395 #endif
396 }
397
398 /* Release the spinlock and restore interrupts */
399 _ExiReleaseSpinLockAndRestoreInterrupts(Lock, Enable);
400
401 /* return the entry */
402 return ListEntry;
403 }
404
405 PSINGLE_LIST_ENTRY
406 FASTCALL
ExfInterlockedPopEntryList(IN OUT PSINGLE_LIST_ENTRY ListHead,IN OUT PKSPIN_LOCK Lock)407 ExfInterlockedPopEntryList(
408 IN OUT PSINGLE_LIST_ENTRY ListHead,
409 IN OUT PKSPIN_LOCK Lock)
410 {
411 BOOLEAN Enable;
412 PSINGLE_LIST_ENTRY ListEntry;
413
414 /* Disable interrupts and acquire the spinlock */
415 Enable = _ExiDisableInterruptsAndAcquireSpinlock(Lock);
416
417 /* Pop the first entry from the list */
418 ListEntry = PopEntryList(ListHead);
419 #if DBG
420 if (ListEntry)
421 ListEntry->Next = (PSINGLE_LIST_ENTRY)(ULONG_PTR)0xBADDD0FFBADDD0FFULL;
422 #endif
423
424 /* Release the spinlock and restore interrupts */
425 _ExiReleaseSpinLockAndRestoreInterrupts(Lock, Enable);
426
427 /* return the entry */
428 return ListEntry;
429 }
430
431 PSINGLE_LIST_ENTRY
432 FASTCALL
ExfInterlockedPushEntryList(IN OUT PSINGLE_LIST_ENTRY ListHead,IN PSINGLE_LIST_ENTRY ListEntry,IN OUT PKSPIN_LOCK Lock)433 ExfInterlockedPushEntryList(
434 IN OUT PSINGLE_LIST_ENTRY ListHead,
435 IN PSINGLE_LIST_ENTRY ListEntry,
436 IN OUT PKSPIN_LOCK Lock)
437 {
438 BOOLEAN Enable;
439 PSINGLE_LIST_ENTRY OldListEntry;
440
441 /* Disable interrupts and acquire the spinlock */
442 Enable = _ExiDisableInterruptsAndAcquireSpinlock(Lock);
443
444 /* Save the old top entry */
445 OldListEntry = ListHead->Next;
446
447 /* Push a new entry on the list */
448 PushEntryList(ListHead, ListEntry);
449
450 /* Release the spinlock and restore interrupts */
451 _ExiReleaseSpinLockAndRestoreInterrupts(Lock, Enable);
452
453 /* return the entry */
454 return OldListEntry;
455 }
456
457 INTERLOCKED_RESULT
458 NTAPI
Exi386InterlockedIncrementLong(IN PLONG Addend)459 Exi386InterlockedIncrementLong(
460 IN PLONG Addend)
461 {
462 LONG Result;
463
464 Result = _InterlockedIncrement(Addend);
465 return (Result < 0) ? ResultNegative :
466 (Result > 0) ? ResultPositive :
467 ResultZero;
468 }
469
470 INTERLOCKED_RESULT
471 FASTCALL
Exfi386InterlockedIncrementLong(IN OUT LONG volatile * Addend)472 Exfi386InterlockedIncrementLong(
473 IN OUT LONG volatile *Addend)
474 {
475 LONG Result;
476
477 Result = _InterlockedIncrement(Addend);
478 return (Result < 0) ? ResultNegative :
479 (Result > 0) ? ResultPositive :
480 ResultZero;
481 }
482
483 INTERLOCKED_RESULT
484 NTAPI
Exi386InterlockedDecrementLong(IN PLONG Addend)485 Exi386InterlockedDecrementLong(
486 IN PLONG Addend)
487 {
488 LONG Result;
489
490 Result = _InterlockedDecrement(Addend);
491 return (Result < 0) ? ResultNegative :
492 (Result > 0) ? ResultPositive :
493 ResultZero;
494 }
495
496 INTERLOCKED_RESULT
497 FASTCALL
Exfi386InterlockedDecrementLong(IN OUT PLONG Addend)498 Exfi386InterlockedDecrementLong(
499 IN OUT PLONG Addend)
500 {
501 LONG Result;
502
503 Result = _InterlockedDecrement(Addend);
504 return (Result < 0) ? ResultNegative :
505 (Result > 0) ? ResultPositive :
506 ResultZero;
507 }
508
509 LONG
510 NTAPI
Exi386InterlockedExchangeUlong(PLONG Target,LONG Exchange)511 Exi386InterlockedExchangeUlong(
512 PLONG Target,
513 LONG Exchange)
514 {
515 return _InterlockedExchange(Target, Exchange);
516 }
517
518 ULONG
519 FASTCALL
Exfi386InterlockedExchangeUlong(IN OUT PULONG Target,IN ULONG Exchange)520 Exfi386InterlockedExchangeUlong(
521 IN OUT PULONG Target,
522 IN ULONG Exchange)
523 {
524 return _InterlockedExchange((PLONG)Target, Exchange);
525 }
526
527 LONGLONG
528 FASTCALL
ExInterlockedCompareExchange64(IN OUT LONGLONG volatile * Destination,IN PLONGLONG Exchange,IN PLONGLONG Comparand,IN PKSPIN_LOCK Lock)529 ExInterlockedCompareExchange64(
530 IN OUT LONGLONG volatile *Destination,
531 IN PLONGLONG Exchange,
532 IN PLONGLONG Comparand,
533 IN PKSPIN_LOCK Lock)
534 {
535 return _InterlockedCompareExchange64(Destination, *Exchange, *Comparand);
536 }
537
538 LONGLONG
539 FASTCALL
ExfInterlockedCompareExchange64(IN OUT LONGLONG volatile * Destination,IN PLONGLONG Exchange,IN PLONGLONG Comparand)540 ExfInterlockedCompareExchange64(
541 IN OUT LONGLONG volatile *Destination,
542 IN PLONGLONG Exchange,
543 IN PLONGLONG Comparand)
544 {
545 return _InterlockedCompareExchange64(Destination, *Exchange, *Comparand);
546 }
547 #endif
548
549 #if 0
550
551 VOID
552 FASTCALL
553 ExInterlockedAddLargeStatistic(
554 IN OUT PLARGE_INTEGER Addend,
555 IN ULONG Increment)
556 {
557 }
558
559
560 #endif
561
562