1 /*
2 * COPYRIGHT: See COPYING in the top level directory
3 * PROJECT: ReactOS system libraries
4 * PURPOSE: Slim Reader/Writer (SRW) Routines
5 * PROGRAMMER: Thomas Weidenmueller <w3seek@reactos.com>
6 *
7 * NOTES: The algorithms used in this implementation
8 * may be different from Vista's implementation.
9 * Since applications should treat the RTL_SRWLOCK
10 * structure as opaque data, it should not matter.
11 * The algorithms are probably not as optimized.
12 */
13
14 /* INCLUDES *****************************************************************/
15
16 #include <rtl_vista.h>
17
18 #define NDEBUG
19 #include <debug.h>
20
21 /* FUNCTIONS *****************************************************************/
22
23 #ifdef _WIN64
24 #define InterlockedBitTestAndSetPointer(ptr,val) InterlockedBitTestAndSet64((PLONGLONG)ptr,(LONGLONG)val)
25 #define InterlockedAddPointer(ptr,val) InterlockedAdd64((PLONGLONG)ptr,(LONGLONG)val)
26 #define InterlockedAndPointer(ptr,val) InterlockedAnd64((PLONGLONG)ptr,(LONGLONG)val)
27 #define InterlockedOrPointer(ptr,val) InterlockedOr64((PLONGLONG)ptr,(LONGLONG)val)
28 #define _ONE 1LL
29 #else
30 #define InterlockedBitTestAndSetPointer(ptr,val) InterlockedBitTestAndSet((PLONG)ptr,(LONG)val)
31 #define InterlockedAddPointer(ptr,val) InterlockedAdd((PLONG)ptr,(LONG)val)
32 #define InterlockedAndPointer(ptr,val) InterlockedAnd((PLONG)ptr,(LONG)val)
33 #define InterlockedOrPointer(ptr,val) InterlockedOr((PLONG)ptr,(LONG)val)
34 #define _ONE 1L
35 #endif
36
37 #define RTL_SRWLOCK_OWNED_BIT 0
38 #define RTL_SRWLOCK_CONTENDED_BIT 1
39 #define RTL_SRWLOCK_SHARED_BIT 2
40 #define RTL_SRWLOCK_CONTENTION_LOCK_BIT 3
41 #define RTL_SRWLOCK_OWNED (_ONE << RTL_SRWLOCK_OWNED_BIT)
42 #define RTL_SRWLOCK_CONTENDED (_ONE << RTL_SRWLOCK_CONTENDED_BIT)
43 #define RTL_SRWLOCK_SHARED (_ONE << RTL_SRWLOCK_SHARED_BIT)
44 #define RTL_SRWLOCK_CONTENTION_LOCK (_ONE << RTL_SRWLOCK_CONTENTION_LOCK_BIT)
45 #define RTL_SRWLOCK_MASK (RTL_SRWLOCK_OWNED | RTL_SRWLOCK_CONTENDED | \
46 RTL_SRWLOCK_SHARED | RTL_SRWLOCK_CONTENTION_LOCK)
47 #define RTL_SRWLOCK_BITS 4
48
49 typedef struct _RTLP_SRWLOCK_SHARED_WAKE
50 {
51 LONG Wake;
52 volatile struct _RTLP_SRWLOCK_SHARED_WAKE *Next;
53 } volatile RTLP_SRWLOCK_SHARED_WAKE, *PRTLP_SRWLOCK_SHARED_WAKE;
54
55 typedef struct _RTLP_SRWLOCK_WAITBLOCK
56 {
57 /* SharedCount is the number of shared acquirers. */
58 LONG SharedCount;
59
60 /* Last points to the last wait block in the chain. The value
61 is only valid when read from the first wait block. */
62 volatile struct _RTLP_SRWLOCK_WAITBLOCK *Last;
63
64 /* Next points to the next wait block in the chain. */
65 volatile struct _RTLP_SRWLOCK_WAITBLOCK *Next;
66
67 union
68 {
69 /* Wake is only valid for exclusive wait blocks */
70 LONG Wake;
71 /* The wake chain is only valid for shared wait blocks */
72 struct
73 {
74 PRTLP_SRWLOCK_SHARED_WAKE SharedWakeChain;
75 PRTLP_SRWLOCK_SHARED_WAKE LastSharedWake;
76 };
77 };
78
79 BOOLEAN Exclusive;
80 } volatile RTLP_SRWLOCK_WAITBLOCK, *PRTLP_SRWLOCK_WAITBLOCK;
81
82
83 static VOID
84 NTAPI
RtlpReleaseWaitBlockLockExclusive(IN OUT PRTL_SRWLOCK SRWLock,IN PRTLP_SRWLOCK_WAITBLOCK FirstWaitBlock)85 RtlpReleaseWaitBlockLockExclusive(IN OUT PRTL_SRWLOCK SRWLock,
86 IN PRTLP_SRWLOCK_WAITBLOCK FirstWaitBlock)
87 {
88 PRTLP_SRWLOCK_WAITBLOCK Next;
89 LONG_PTR NewValue;
90
91 /* NOTE: We're currently in an exclusive lock in contended mode. */
92
93 Next = FirstWaitBlock->Next;
94 if (Next != NULL)
95 {
96 /* There's more blocks chained, we need to update the pointers
97 in the next wait block and update the wait block pointer. */
98 NewValue = (LONG_PTR)Next | RTL_SRWLOCK_OWNED | RTL_SRWLOCK_CONTENDED;
99 if (!FirstWaitBlock->Exclusive)
100 {
101 /* The next wait block has to be an exclusive lock! */
102 ASSERT(Next->Exclusive);
103
104 /* Save the shared count */
105 Next->SharedCount = FirstWaitBlock->SharedCount;
106
107 NewValue |= RTL_SRWLOCK_SHARED;
108 }
109
110 Next->Last = FirstWaitBlock->Last;
111 }
112 else
113 {
114 /* Convert the lock to a simple lock. */
115 if (FirstWaitBlock->Exclusive)
116 NewValue = RTL_SRWLOCK_OWNED;
117 else
118 {
119 ASSERT(FirstWaitBlock->SharedCount > 0);
120
121 NewValue = ((LONG_PTR)FirstWaitBlock->SharedCount << RTL_SRWLOCK_BITS) |
122 RTL_SRWLOCK_SHARED | RTL_SRWLOCK_OWNED;
123 }
124 }
125
126 (void)InterlockedExchangePointer(&SRWLock->Ptr, (PVOID)NewValue);
127
128 if (FirstWaitBlock->Exclusive)
129 {
130 (void)InterlockedOr(&FirstWaitBlock->Wake,
131 TRUE);
132 }
133 else
134 {
135 PRTLP_SRWLOCK_SHARED_WAKE WakeChain, NextWake;
136
137 /* If we were the first one to acquire the shared
138 lock, we now need to wake all others... */
139 WakeChain = FirstWaitBlock->SharedWakeChain;
140 do
141 {
142 NextWake = WakeChain->Next;
143
144 (void)InterlockedOr((PLONG)&WakeChain->Wake,
145 TRUE);
146
147 WakeChain = NextWake;
148 } while (WakeChain != NULL);
149 }
150 }
151
152
153 static VOID
154 NTAPI
RtlpReleaseWaitBlockLockLastShared(IN OUT PRTL_SRWLOCK SRWLock,IN PRTLP_SRWLOCK_WAITBLOCK FirstWaitBlock)155 RtlpReleaseWaitBlockLockLastShared(IN OUT PRTL_SRWLOCK SRWLock,
156 IN PRTLP_SRWLOCK_WAITBLOCK FirstWaitBlock)
157 {
158 PRTLP_SRWLOCK_WAITBLOCK Next;
159 LONG_PTR NewValue;
160
161 /* NOTE: We're currently in a shared lock in contended mode. */
162
163 /* The next acquirer to be unwaited *must* be an exclusive lock! */
164 ASSERT(FirstWaitBlock->Exclusive);
165
166 Next = FirstWaitBlock->Next;
167 if (Next != NULL)
168 {
169 /* There's more blocks chained, we need to update the pointers
170 in the next wait block and update the wait block pointer. */
171 NewValue = (LONG_PTR)Next | RTL_SRWLOCK_OWNED | RTL_SRWLOCK_CONTENDED;
172
173 Next->Last = FirstWaitBlock->Last;
174 }
175 else
176 {
177 /* Convert the lock to a simple exclusive lock. */
178 NewValue = RTL_SRWLOCK_OWNED;
179 }
180
181 (void)InterlockedExchangePointer(&SRWLock->Ptr, (PVOID)NewValue);
182
183 (void)InterlockedOr(&FirstWaitBlock->Wake,
184 TRUE);
185 }
186
187
188 static VOID
189 NTAPI
RtlpReleaseWaitBlockLock(IN OUT PRTL_SRWLOCK SRWLock)190 RtlpReleaseWaitBlockLock(IN OUT PRTL_SRWLOCK SRWLock)
191 {
192 InterlockedAndPointer(&SRWLock->Ptr,
193 ~RTL_SRWLOCK_CONTENTION_LOCK);
194 }
195
196
197 static PRTLP_SRWLOCK_WAITBLOCK
198 NTAPI
RtlpAcquireWaitBlockLock(IN OUT PRTL_SRWLOCK SRWLock)199 RtlpAcquireWaitBlockLock(IN OUT PRTL_SRWLOCK SRWLock)
200 {
201 LONG_PTR PrevValue;
202 PRTLP_SRWLOCK_WAITBLOCK WaitBlock;
203
204 while (1)
205 {
206 PrevValue = InterlockedOrPointer(&SRWLock->Ptr,
207 RTL_SRWLOCK_CONTENTION_LOCK);
208
209 if (!(PrevValue & RTL_SRWLOCK_CONTENTION_LOCK))
210 break;
211
212 YieldProcessor();
213 }
214
215 if (!(PrevValue & RTL_SRWLOCK_CONTENDED) ||
216 (PrevValue & ~RTL_SRWLOCK_MASK) == 0)
217 {
218 /* Too bad, looks like the wait block was removed in the
219 meanwhile, unlock again */
220 RtlpReleaseWaitBlockLock(SRWLock);
221 return NULL;
222 }
223
224 WaitBlock = (PRTLP_SRWLOCK_WAITBLOCK)(PrevValue & ~RTL_SRWLOCK_MASK);
225
226 return WaitBlock;
227 }
228
229
230 static VOID
231 NTAPI
RtlpAcquireSRWLockExclusiveWait(IN OUT PRTL_SRWLOCK SRWLock,IN PRTLP_SRWLOCK_WAITBLOCK WaitBlock)232 RtlpAcquireSRWLockExclusiveWait(IN OUT PRTL_SRWLOCK SRWLock,
233 IN PRTLP_SRWLOCK_WAITBLOCK WaitBlock)
234 {
235 LONG_PTR CurrentValue;
236
237 while (1)
238 {
239 CurrentValue = *(volatile LONG_PTR *)&SRWLock->Ptr;
240 if (!(CurrentValue & RTL_SRWLOCK_SHARED))
241 {
242 if (CurrentValue & RTL_SRWLOCK_CONTENDED)
243 {
244 if (WaitBlock->Wake != 0)
245 {
246 /* Our wait block became the first one
247 in the chain, we own the lock now! */
248 break;
249 }
250 }
251 else
252 {
253 /* The last wait block was removed and/or we're
254 finally a simple exclusive lock. This means we
255 don't need to wait anymore, we acquired the lock! */
256 break;
257 }
258 }
259
260 YieldProcessor();
261 }
262 }
263
264
265 static VOID
266 NTAPI
RtlpAcquireSRWLockSharedWait(IN OUT PRTL_SRWLOCK SRWLock,IN OUT PRTLP_SRWLOCK_WAITBLOCK FirstWait OPTIONAL,IN OUT PRTLP_SRWLOCK_SHARED_WAKE WakeChain)267 RtlpAcquireSRWLockSharedWait(IN OUT PRTL_SRWLOCK SRWLock,
268 IN OUT PRTLP_SRWLOCK_WAITBLOCK FirstWait OPTIONAL,
269 IN OUT PRTLP_SRWLOCK_SHARED_WAKE WakeChain)
270 {
271 if (FirstWait != NULL)
272 {
273 while (WakeChain->Wake == 0)
274 {
275 YieldProcessor();
276 }
277 }
278 else
279 {
280 LONG_PTR CurrentValue;
281
282 while (1)
283 {
284 CurrentValue = *(volatile LONG_PTR *)&SRWLock->Ptr;
285 if (CurrentValue & RTL_SRWLOCK_SHARED)
286 {
287 /* The RTL_SRWLOCK_OWNED bit always needs to be set when
288 RTL_SRWLOCK_SHARED is set! */
289 ASSERT(CurrentValue & RTL_SRWLOCK_OWNED);
290
291 if (CurrentValue & RTL_SRWLOCK_CONTENDED)
292 {
293 if (WakeChain->Wake != 0)
294 {
295 /* Our wait block became the first one
296 in the chain, we own the lock now! */
297 break;
298 }
299 }
300 else
301 {
302 /* The last wait block was removed and/or we're
303 finally a simple shared lock. This means we
304 don't need to wait anymore, we acquired the lock! */
305 break;
306 }
307 }
308
309 YieldProcessor();
310 }
311 }
312 }
313
314
315 VOID
316 NTAPI
RtlInitializeSRWLock(OUT PRTL_SRWLOCK SRWLock)317 RtlInitializeSRWLock(OUT PRTL_SRWLOCK SRWLock)
318 {
319 SRWLock->Ptr = NULL;
320 }
321
322
323 VOID
324 NTAPI
RtlAcquireSRWLockShared(IN OUT PRTL_SRWLOCK SRWLock)325 RtlAcquireSRWLockShared(IN OUT PRTL_SRWLOCK SRWLock)
326 {
327 __ALIGNED(16) RTLP_SRWLOCK_WAITBLOCK StackWaitBlock;
328 RTLP_SRWLOCK_SHARED_WAKE SharedWake;
329 LONG_PTR CurrentValue, NewValue;
330 PRTLP_SRWLOCK_WAITBLOCK First, Shared, FirstWait;
331
332 while (1)
333 {
334 CurrentValue = *(volatile LONG_PTR *)&SRWLock->Ptr;
335
336 if (CurrentValue & RTL_SRWLOCK_SHARED)
337 {
338 /* NOTE: It is possible that the RTL_SRWLOCK_OWNED bit is set! */
339
340 if (CurrentValue & RTL_SRWLOCK_CONTENDED)
341 {
342 /* There's other waiters already, lock the wait blocks and
343 increment the shared count */
344 First = RtlpAcquireWaitBlockLock(SRWLock);
345 if (First != NULL)
346 {
347 FirstWait = NULL;
348
349 if (First->Exclusive)
350 {
351 /* We need to setup a new wait block! Although
352 we're currently in a shared lock and we're acquiring
353 a shared lock, there are exclusive locks queued. We need
354 to wait until those are released. */
355 Shared = First->Last;
356
357 if (Shared->Exclusive)
358 {
359 StackWaitBlock.Exclusive = FALSE;
360 StackWaitBlock.SharedCount = 1;
361 StackWaitBlock.Next = NULL;
362 StackWaitBlock.Last = &StackWaitBlock;
363 StackWaitBlock.SharedWakeChain = &SharedWake;
364
365 Shared->Next = &StackWaitBlock;
366 First->Last = &StackWaitBlock;
367
368 Shared = &StackWaitBlock;
369 FirstWait = &StackWaitBlock;
370 }
371 else
372 {
373 Shared->LastSharedWake->Next = &SharedWake;
374 Shared->SharedCount++;
375 }
376 }
377 else
378 {
379 Shared = First;
380 Shared->LastSharedWake->Next = &SharedWake;
381 Shared->SharedCount++;
382 }
383
384 SharedWake.Next = NULL;
385 SharedWake.Wake = 0;
386
387 Shared->LastSharedWake = &SharedWake;
388
389 RtlpReleaseWaitBlockLock(SRWLock);
390
391 RtlpAcquireSRWLockSharedWait(SRWLock,
392 FirstWait,
393 &SharedWake);
394
395 /* Successfully incremented the shared count, we acquired the lock */
396 break;
397 }
398 }
399 else
400 {
401 /* This is a fastest path, just increment the number of
402 current shared locks */
403
404 /* Since the RTL_SRWLOCK_SHARED bit is set, the RTL_SRWLOCK_OWNED bit also has
405 to be set! */
406
407 ASSERT(CurrentValue & RTL_SRWLOCK_OWNED);
408
409 NewValue = (CurrentValue >> RTL_SRWLOCK_BITS) + 1;
410 NewValue = (NewValue << RTL_SRWLOCK_BITS) | (CurrentValue & RTL_SRWLOCK_MASK);
411
412 if ((LONG_PTR)InterlockedCompareExchangePointer(&SRWLock->Ptr,
413 (PVOID)NewValue,
414 (PVOID)CurrentValue) == CurrentValue)
415 {
416 /* Successfully incremented the shared count, we acquired the lock */
417 break;
418 }
419 }
420 }
421 else
422 {
423 if (CurrentValue & RTL_SRWLOCK_OWNED)
424 {
425 /* The resource is currently acquired exclusively */
426 if (CurrentValue & RTL_SRWLOCK_CONTENDED)
427 {
428 SharedWake.Next = NULL;
429 SharedWake.Wake = 0;
430
431 /* There's other waiters already, lock the wait blocks and
432 increment the shared count. If the last block in the chain
433 is an exclusive lock, add another block. */
434
435 StackWaitBlock.Exclusive = FALSE;
436 StackWaitBlock.SharedCount = 0;
437 StackWaitBlock.Next = NULL;
438 StackWaitBlock.Last = &StackWaitBlock;
439 StackWaitBlock.SharedWakeChain = &SharedWake;
440
441 First = RtlpAcquireWaitBlockLock(SRWLock);
442 if (First != NULL)
443 {
444 Shared = First->Last;
445 if (Shared->Exclusive)
446 {
447 Shared->Next = &StackWaitBlock;
448 First->Last = &StackWaitBlock;
449
450 Shared = &StackWaitBlock;
451 FirstWait = &StackWaitBlock;
452 }
453 else
454 {
455 FirstWait = NULL;
456 Shared->LastSharedWake->Next = &SharedWake;
457 }
458
459 Shared->SharedCount++;
460 Shared->LastSharedWake = &SharedWake;
461
462 RtlpReleaseWaitBlockLock(SRWLock);
463
464 RtlpAcquireSRWLockSharedWait(SRWLock,
465 FirstWait,
466 &SharedWake);
467
468 /* Successfully incremented the shared count, we acquired the lock */
469 break;
470 }
471 }
472 else
473 {
474 SharedWake.Next = NULL;
475 SharedWake.Wake = 0;
476
477 /* We need to setup the first wait block. Currently an exclusive lock is
478 held, change the lock to contended mode. */
479 StackWaitBlock.Exclusive = FALSE;
480 StackWaitBlock.SharedCount = 1;
481 StackWaitBlock.Next = NULL;
482 StackWaitBlock.Last = &StackWaitBlock;
483 StackWaitBlock.SharedWakeChain = &SharedWake;
484 StackWaitBlock.LastSharedWake = &SharedWake;
485
486 NewValue = (ULONG_PTR)&StackWaitBlock | RTL_SRWLOCK_OWNED | RTL_SRWLOCK_CONTENDED;
487 if ((LONG_PTR)InterlockedCompareExchangePointer(&SRWLock->Ptr,
488 (PVOID)NewValue,
489 (PVOID)CurrentValue) == CurrentValue)
490 {
491 RtlpAcquireSRWLockSharedWait(SRWLock,
492 &StackWaitBlock,
493 &SharedWake);
494
495 /* Successfully set the shared count, we acquired the lock */
496 break;
497 }
498 }
499 }
500 else
501 {
502 /* This is a fast path, we can simply try to set the shared count to 1 */
503 NewValue = (1 << RTL_SRWLOCK_BITS) | RTL_SRWLOCK_SHARED | RTL_SRWLOCK_OWNED;
504
505 /* The RTL_SRWLOCK_CONTENDED bit should never be set if neither the
506 RTL_SRWLOCK_SHARED nor the RTL_SRWLOCK_OWNED bit is set */
507 ASSERT(!(CurrentValue & RTL_SRWLOCK_CONTENDED));
508
509 if ((LONG_PTR)InterlockedCompareExchangePointer(&SRWLock->Ptr,
510 (PVOID)NewValue,
511 (PVOID)CurrentValue) == CurrentValue)
512 {
513 /* Successfully set the shared count, we acquired the lock */
514 break;
515 }
516 }
517 }
518
519 YieldProcessor();
520 }
521 }
522
523
524 VOID
525 NTAPI
RtlReleaseSRWLockShared(IN OUT PRTL_SRWLOCK SRWLock)526 RtlReleaseSRWLockShared(IN OUT PRTL_SRWLOCK SRWLock)
527 {
528 LONG_PTR CurrentValue, NewValue;
529 PRTLP_SRWLOCK_WAITBLOCK WaitBlock;
530 BOOLEAN LastShared;
531
532 while (1)
533 {
534 CurrentValue = *(volatile LONG_PTR *)&SRWLock->Ptr;
535
536 if (CurrentValue & RTL_SRWLOCK_SHARED)
537 {
538 if (CurrentValue & RTL_SRWLOCK_CONTENDED)
539 {
540 /* There's a wait block, we need to wake a pending
541 exclusive acquirer if this is the last shared release */
542 WaitBlock = RtlpAcquireWaitBlockLock(SRWLock);
543 if (WaitBlock != NULL)
544 {
545 LastShared = (--WaitBlock->SharedCount == 0);
546
547 if (LastShared)
548 RtlpReleaseWaitBlockLockLastShared(SRWLock,
549 WaitBlock);
550 else
551 RtlpReleaseWaitBlockLock(SRWLock);
552
553 /* We released the lock */
554 break;
555 }
556 }
557 else
558 {
559 /* This is a fast path, we can simply decrement the shared
560 count and store the pointer */
561 NewValue = CurrentValue >> RTL_SRWLOCK_BITS;
562
563 if (--NewValue != 0)
564 {
565 NewValue = (NewValue << RTL_SRWLOCK_BITS) | RTL_SRWLOCK_SHARED | RTL_SRWLOCK_OWNED;
566 }
567
568 if ((LONG_PTR)InterlockedCompareExchangePointer(&SRWLock->Ptr,
569 (PVOID)NewValue,
570 (PVOID)CurrentValue) == CurrentValue)
571 {
572 /* Successfully released the lock */
573 break;
574 }
575 }
576 }
577 else
578 {
579 /* The RTL_SRWLOCK_SHARED bit has to be present now,
580 even in the contended case! */
581 RtlRaiseStatus(STATUS_RESOURCE_NOT_OWNED);
582 }
583
584 YieldProcessor();
585 }
586 }
587
588
589 VOID
590 NTAPI
RtlAcquireSRWLockExclusive(IN OUT PRTL_SRWLOCK SRWLock)591 RtlAcquireSRWLockExclusive(IN OUT PRTL_SRWLOCK SRWLock)
592 {
593 __ALIGNED(16) RTLP_SRWLOCK_WAITBLOCK StackWaitBlock;
594 PRTLP_SRWLOCK_WAITBLOCK First, Last;
595
596 if (InterlockedBitTestAndSetPointer(&SRWLock->Ptr,
597 RTL_SRWLOCK_OWNED_BIT))
598 {
599 LONG_PTR CurrentValue, NewValue;
600
601 while (1)
602 {
603 CurrentValue = *(volatile LONG_PTR *)&SRWLock->Ptr;
604
605 if (CurrentValue & RTL_SRWLOCK_SHARED)
606 {
607 /* A shared lock is being held right now. We need to add a wait block! */
608
609 if (CurrentValue & RTL_SRWLOCK_CONTENDED)
610 {
611 goto AddWaitBlock;
612 }
613 else
614 {
615 /* There are no wait blocks so far, we need to add ourselves as the first
616 wait block. We need to keep the shared count! */
617 StackWaitBlock.Exclusive = TRUE;
618 StackWaitBlock.SharedCount = (LONG)(CurrentValue >> RTL_SRWLOCK_BITS);
619 StackWaitBlock.Next = NULL;
620 StackWaitBlock.Last = &StackWaitBlock;
621 StackWaitBlock.Wake = 0;
622
623 NewValue = (ULONG_PTR)&StackWaitBlock | RTL_SRWLOCK_SHARED | RTL_SRWLOCK_CONTENDED | RTL_SRWLOCK_OWNED;
624
625 if ((LONG_PTR)InterlockedCompareExchangePointer(&SRWLock->Ptr,
626 (PVOID)NewValue,
627 (PVOID)CurrentValue) == CurrentValue)
628 {
629 RtlpAcquireSRWLockExclusiveWait(SRWLock,
630 &StackWaitBlock);
631
632 /* Successfully acquired the exclusive lock */
633 break;
634 }
635 }
636 }
637 else
638 {
639 if (CurrentValue & RTL_SRWLOCK_OWNED)
640 {
641 /* An exclusive lock is being held right now. We need to add a wait block! */
642
643 if (CurrentValue & RTL_SRWLOCK_CONTENDED)
644 {
645 AddWaitBlock:
646 StackWaitBlock.Exclusive = TRUE;
647 StackWaitBlock.SharedCount = 0;
648 StackWaitBlock.Next = NULL;
649 StackWaitBlock.Last = &StackWaitBlock;
650 StackWaitBlock.Wake = 0;
651
652 First = RtlpAcquireWaitBlockLock(SRWLock);
653 if (First != NULL)
654 {
655 Last = First->Last;
656 Last->Next = &StackWaitBlock;
657 First->Last = &StackWaitBlock;
658
659 RtlpReleaseWaitBlockLock(SRWLock);
660
661 RtlpAcquireSRWLockExclusiveWait(SRWLock,
662 &StackWaitBlock);
663
664 /* Successfully acquired the exclusive lock */
665 break;
666 }
667 }
668 else
669 {
670 /* There are no wait blocks so far, we need to add ourselves as the first
671 wait block. We need to keep the shared count! */
672 StackWaitBlock.Exclusive = TRUE;
673 StackWaitBlock.SharedCount = 0;
674 StackWaitBlock.Next = NULL;
675 StackWaitBlock.Last = &StackWaitBlock;
676 StackWaitBlock.Wake = 0;
677
678 NewValue = (ULONG_PTR)&StackWaitBlock | RTL_SRWLOCK_OWNED | RTL_SRWLOCK_CONTENDED;
679 if ((LONG_PTR)InterlockedCompareExchangePointer(&SRWLock->Ptr,
680 (PVOID)NewValue,
681 (PVOID)CurrentValue) == CurrentValue)
682 {
683 RtlpAcquireSRWLockExclusiveWait(SRWLock,
684 &StackWaitBlock);
685
686 /* Successfully acquired the exclusive lock */
687 break;
688 }
689 }
690 }
691 else
692 {
693 if (!InterlockedBitTestAndSetPointer(&SRWLock->Ptr,
694 RTL_SRWLOCK_OWNED_BIT))
695 {
696 /* We managed to get hold of a simple exclusive lock! */
697 break;
698 }
699 }
700 }
701
702 YieldProcessor();
703 }
704 }
705 }
706
707
708 VOID
709 NTAPI
RtlReleaseSRWLockExclusive(IN OUT PRTL_SRWLOCK SRWLock)710 RtlReleaseSRWLockExclusive(IN OUT PRTL_SRWLOCK SRWLock)
711 {
712 LONG_PTR CurrentValue, NewValue;
713 PRTLP_SRWLOCK_WAITBLOCK WaitBlock;
714
715 while (1)
716 {
717 CurrentValue = *(volatile LONG_PTR *)&SRWLock->Ptr;
718
719 if (!(CurrentValue & RTL_SRWLOCK_OWNED))
720 {
721 RtlRaiseStatus(STATUS_RESOURCE_NOT_OWNED);
722 }
723
724 if (!(CurrentValue & RTL_SRWLOCK_SHARED))
725 {
726 if (CurrentValue & RTL_SRWLOCK_CONTENDED)
727 {
728 /* There's a wait block, we need to wake the next pending
729 acquirer (exclusive or shared) */
730 WaitBlock = RtlpAcquireWaitBlockLock(SRWLock);
731 if (WaitBlock != NULL)
732 {
733 RtlpReleaseWaitBlockLockExclusive(SRWLock,
734 WaitBlock);
735
736 /* We released the lock */
737 break;
738 }
739 }
740 else
741 {
742 /* This is a fast path, we can simply clear the RTL_SRWLOCK_OWNED
743 bit. All other bits should be 0 now because this is a simple
744 exclusive lock and no one is waiting. */
745
746 ASSERT(!(CurrentValue & ~RTL_SRWLOCK_OWNED));
747
748 NewValue = 0;
749 if ((LONG_PTR)InterlockedCompareExchangePointer(&SRWLock->Ptr,
750 (PVOID)NewValue,
751 (PVOID)CurrentValue) == CurrentValue)
752 {
753 /* We released the lock */
754 break;
755 }
756 }
757 }
758 else
759 {
760 /* The RTL_SRWLOCK_SHARED bit must not be present now,
761 not even in the contended case! */
762 RtlRaiseStatus(STATUS_RESOURCE_NOT_OWNED);
763 }
764
765 YieldProcessor();
766 }
767 }
768
769 BOOLEAN
770 NTAPI
RtlTryAcquireSRWLockShared(PRTL_SRWLOCK SRWLock)771 RtlTryAcquireSRWLockShared(PRTL_SRWLOCK SRWLock)
772 {
773
774 LONG_PTR CompareValue, NewValue, GotValue;
775
776 do
777 {
778 CompareValue = *(volatile LONG_PTR *)&SRWLock->Ptr;
779 NewValue = ((CompareValue >> RTL_SRWLOCK_BITS) + 1) | RTL_SRWLOCK_SHARED | RTL_SRWLOCK_OWNED;
780
781 /* Only increment shared count if there is no waiter */
782 CompareValue &= ~RTL_SRWLOCK_MASK | RTL_SRWLOCK_SHARED | RTL_SRWLOCK_OWNED;
783 } while (
784 ((GotValue = (LONG_PTR)InterlockedCompareExchangePointer(&SRWLock->Ptr, (LONG_PTR*)NewValue, (LONG_PTR*)CompareValue)) != CompareValue)
785 && (((GotValue & RTL_SRWLOCK_MASK) == (RTL_SRWLOCK_SHARED | RTL_SRWLOCK_OWNED)) || (GotValue == 0)));
786
787 return ((GotValue & RTL_SRWLOCK_MASK) == (RTL_SRWLOCK_SHARED | RTL_SRWLOCK_OWNED)) || (GotValue == 0);
788 }
789
790 BOOLEAN
791 NTAPI
RtlTryAcquireSRWLockExclusive(PRTL_SRWLOCK SRWLock)792 RtlTryAcquireSRWLockExclusive(PRTL_SRWLOCK SRWLock)
793 {
794 return InterlockedCompareExchangePointer(&SRWLock->Ptr, (ULONG_PTR*)RTL_SRWLOCK_OWNED, 0) == 0;
795 }
796