xref: /reactos/sdk/lib/rtl/amd64/slist.S (revision 34593d93) 
1/*
2 * COPYRIGHT:       See COPYING in the top level directory
3 * PROJECT:         ReactOS system libraries
4 * FILE:            lib/rtl/amd64/slist.S
5 * PURPOSE:         Rtl Interlocked Functions for amd64
6 * PROGRAMMERS:     Timo Kreuzer
7 */
8
9#include <asm.inc>
10#include <ksamd64.inc>
11
12EXTERN RtlpUse16ByteSLists:BYTE
13
14/*
15typedef union DECLSPEC_ALIGN(16) _SLIST_HEADER
16{
17    struct
18    {
19        ULONGLONG Alignment;
20        ULONGLONG Region;
21    };
22    struct
23    {
24        ULONGLONG Depth:16;
25        ULONGLONG Sequence:9;
26        ULONGLONG NextEntry:39;
27        ULONGLONG HeaderType:1;
28        ULONGLONG Init:1;
29        ULONGLONG Reserved:59;
30        ULONGLONG Region:3;
31    } Header8;
32    struct
33    {
34        ULONGLONG Depth:16;
35        ULONGLONG Sequence:48;
36        ULONGLONG HeaderType:1;
37        ULONGLONG Init:1;
38        ULONGLONG Reserved:2;
39        ULONGLONG NextEntry:60;
40    } Header16;
41    struct
42    {
43        ULONGLONG Depth:16;
44        ULONGLONG Sequence:48;
45        ULONGLONG HeaderType:1;
46        ULONGLONG Reserved:3;
47        ULONGLONG NextEntry:60;
48    } HeaderX64;
49} SLIST_HEADER, *PSLIST_HEADER;
50*/
51
52#define SLIST8A_DEPTH_MASK       HEX(000000000000FFFF)
53#define SLIST8A_DEPTH_INC        HEX(0000000000000001)
54#define SLIST8A_SEQUENCE_MASK    HEX(0000000001FF0000)
55#define SLIST8A_SEQUENCE_INC     HEX(0000000000010000)
56#define SLIST8A_NEXTENTRY_MASK   HEX(FFFFFFFFFE000000)
57#define SLIST8A_NEXTENTRY_SHIFT  21
58#define SLIST8B_HEADERTYPE_MASK  HEX(0000000000000001)
59#define SLIST8B_INIT_MASK        HEX(0000000000000002)
60#define SLIST8B_REGION_MASK      HEX(E000000000000000)
61#define SLIST8_POINTER_MASK      HEX(000007FFFFFFFFFF)
62
63#define SLIST16A_DEPTH_MASK      HEX(000000000000FFFF)
64#define SLIST16A_DEPTH_INC       HEX(0000000000000001)
65#define SLIST16A_SEQUENCE_MASK   HEX(FFFFFFFFFFFF0000)
66#define SLIST16A_SEQUENCE_INC    HEX(0000000000010000)
67#define SLIST16B_HEADERTYPE_MASK HEX(0000000000000001)
68#define SLIST16B_INIT_MASK       HEX(0000000000000002)
69#define SLIST16B_NEXTENTRY_MASK  HEX(FFFFFFFFFFFFFFF0)
70
71
72/* FUNCTIONS ****************************************************************/
73
74.code64
75
76PUBLIC ExpInterlockedPopEntrySList
77PUBLIC ExpInterlockedPopEntrySListResume
78PUBLIC ExpInterlockedPopEntrySListFault
79PUBLIC ExpInterlockedPopEntrySListEnd
80PUBLIC ExpInterlockedPopEntrySListResume16
81PUBLIC ExpInterlockedPopEntrySListFault16
82PUBLIC ExpInterlockedPopEntrySListEnd16
83PUBLIC ExpInterlockedPushEntrySList
84PUBLIC ExpInterlockedFlushSList
85
86PUBLIC RtlInterlockedPopEntrySList
87PUBLIC RtlInterlockedPushEntrySList
88PUBLIC RtlInterlockedFlushSList
89
90/* PSLIST_ENTRY
91 * NTAPI
92 * RtlInterlockedPopEntrySList(
93 *     IN PSLIST_HEADER ListHead);
94 */
95ExpInterlockedPopEntrySList:
96FUNC RtlInterlockedPopEntrySList
97    .ENDPROLOG
98
99    /* Load ListHead->Region into rdx */
100    mov rdx, [rcx + 8]
101
102    /* Load ListHead->Alignment into rax */
103    mov rax, [rcx]
104
105    /* Check for 16 byte SList support */
106    cmp byte ptr [rip+RtlpUse16ByteSLists], 0
107    jne RtlInterlockedPopEntrySList16
108
109    /* Use the 8 byte header */
110GLOBAL_LABEL ExpInterlockedPopEntrySListResume
111
112    /* Check if ListHead->NextEntry is NULL */
113    mov r9, rax
114    and r9, SLIST8A_NEXTENTRY_MASK
115    jz RtlInterlockedPopEntrySListEmpty
116
117    /* Copy Depth and Sequence number and adjust Depth */
118    lea r8, [rax - SLIST8A_DEPTH_INC]
119    and r8, (SLIST8A_SEQUENCE_MASK OR SLIST8A_DEPTH_MASK)
120
121    /* Create a pointer template from rcx in rdx */
122    mov rdx, (NOT SLIST8_POINTER_MASK)
123    and rdx, rcx
124
125    /* Shift the NextEntry pointer */
126    shr r9, SLIST8A_NEXTENTRY_SHIFT
127
128    /* Combine to new pointer in rdx */
129    or rdx, r9
130
131GLOBAL_LABEL ExpInterlockedPopEntrySListFault
132
133    /* Load the next NextEntry pointer to r9 */
134    mov r9, [rdx]
135
136    /* Shift bits in place */
137    shl r9, SLIST8A_NEXTENTRY_SHIFT
138
139    /* Combine into r8 */
140    or r8, r9
141
142GLOBAL_LABEL ExpInterlockedPopEntrySListEnd
143
144    /* If [rcx] equals rax, exchange it with r8 */
145    lock cmpxchg [rcx], r8
146
147    /* If not equal, retry with rax, being the content of [rcx] now */
148    jne ExpInterlockedPopEntrySListResume
149
150    /* Shift the pointer bits in place */
151    and rax, SLIST8A_NEXTENTRY_MASK
152    shr rax, SLIST8A_NEXTENTRY_SHIFT
153
154    /* Use rcx as pointer template */
155    mov rdx, (NOT SLIST8_POINTER_MASK)
156    and rdx, rcx
157
158    /* Combine result and return */
159    or rax, rdx
160    ret
161
162RtlInterlockedPopEntrySListEmpty:
163
164    xor rax, rax
165    ret
166
167ENDFUNC
168
169
170FUNC RtlInterlockedPopEntrySList16
171    mov [rsp + P3Home], rbx
172    .SAVEREG rbx, P3Home
173    .ENDPROLOG
174
175    /* This is a 16 byte header
176       rcx == ListHead
177       rdx == ListHead->Region
178       rax == ListHead->Alignment */
179
180    /* Copy rcx to r8, as we need rcx for the exchange */
181    mov r8, rcx
182
183GLOBAL_LABEL ExpInterlockedPopEntrySListResume16
184
185    /* Set r9 = ListHead->NextEntry and check if it is NULL */
186    mov r9, rdx
187    and r9, SLIST16B_NEXTENTRY_MASK
188    jz RtlInterlockedPopEntrySListEmpty16
189
190GLOBAL_LABEL ExpInterlockedPopEntrySListFault16
191
192    /* Set NewListHead.Next = ListHead->NextEntry->Next */
193    mov rcx, [r9]
194
195    /* Set NewListHead.HeaderType = 1 and  NewListHead.Init = 1 */
196    or rcx, (SLIST16B_HEADERTYPE_MASK or SLIST16B_INIT_MASK)
197
198    /* Copy Depth and Sequence number and adjust Depth */
199    lea rbx, [rax - SLIST16A_DEPTH_INC]
200
201GLOBAL_LABEL ExpInterlockedPopEntrySListEnd16
202
203    /* If [r8] equals rdx:rax, exchange it with rcx:rbx */
204    lock cmpxchg16b [r8]
205
206    /* If not equal, retry with rdx:rax, being the content of [r8] now */
207    jne ExpInterlockedPopEntrySListResume16
208
209    /* Copy the old NextEntry pointer to rax */
210    mov rax, rdx
211    and rax, SLIST16B_NEXTENTRY_MASK
212
213    /* Return */
214    mov rbx, [rsp + P3Home]
215    ret
216
217RtlInterlockedPopEntrySListEmpty16:
218
219    xor rax, rax
220    mov rbx, [rsp + P3Home]
221    ret
222ENDFUNC
223
224
225/* PSLIST_ENTRY
226 * NTAPI
227 * RtlInterlockedPushEntrySList(
228 *     IN PSLIST_HEADER ListHead,
229 *     IN PSLIST_ENTRY ListEntry);
230 */
231ExpInterlockedPushEntrySList:
232FUNC RtlInterlockedPushEntrySList
233    .ENDPROLOG
234
235#if DBG
236    /* Make sure the ListEntry is 16 bytes aligned */
237    test rdx, HEX(0F)
238    jz ExpInterlockedPushEntrySListChecked
239    /* Not aligned, raise an assertion */
240    int HEX(2C)
241ExpInterlockedPushEntrySListChecked:
242
243    /* Make sure RtlpUse16ByteSLists is initialized */
244    cmp byte ptr [rip+RtlpUse16ByteSLists], HEX(FF)
245    jne ExpInterlockedPushEntrySListChecked2
246    /* Not initialized, raise an assertion */
247    int HEX(2C)
248ExpInterlockedPushEntrySListChecked2:
249#endif
250
251    /* Load ListHead->Alignment into rax */
252    mov rax, [rcx]
253
254    /* Load ListHead->Region into r9 */
255    mov r9, [rcx + 8]
256
257    /* Check for 16 byte SList support */
258    cmp byte ptr [rip+RtlpUse16ByteSLists], 0
259    jne RtlInterlockedPushEntrySList16
260
261    /* Use the 8 byte header */
262
263RtlInterlockedPushEntrySListLoop:
264
265    /* Get ListHead->NextEntry */
266    mov r8, rax
267    and r8, SLIST8A_NEXTENTRY_MASK
268    jz RtlInterlockedPushEntrySListEmpty
269
270    /* Shift the NextEntry pointer */
271    shr r8, SLIST8A_NEXTENTRY_SHIFT
272
273    /* Create a pointer template from rcx in rdx */
274    mov r9, (NOT SLIST8_POINTER_MASK)
275    and r9, rcx
276
277    /* Combine to new pointer and save as ListEntry->NextEntry */
278    or r8, r9
279
280RtlInterlockedPushEntrySListEmpty:
281    /* Store the NextEntry pointer in the new ListEntry */
282    mov [rdx], r8
283
284    /* Shift and mask the new ListEntry pointer */
285    mov r8, rdx
286    shl r8, SLIST8A_NEXTENTRY_SHIFT
287    and r8, SLIST8A_NEXTENTRY_MASK
288
289    /* Copy and adjust depth and sequence number */
290    lea r9, [rax + SLIST8A_DEPTH_INC + SLIST8A_SEQUENCE_INC]
291    and r9, SLIST8A_SEQUENCE_MASK OR SLIST8A_DEPTH_MASK
292
293    /* Combine to exchange value in r8 */
294    or r8, r9
295
296    /* Save the NextEntry in r9 */
297    mov r9, [rdx]
298
299    /* If [rcx] equals rax, exchange it with r8 */
300    lock cmpxchg [rcx], r8
301
302    /* If not equal, retry with rax, being the content of [rcx] now */
303    jne RtlInterlockedPushEntrySListLoop
304
305    /* Return the old NextEntry pointer */
306    mov rax, r9
307    ret
308
309ENDFUNC
310
311FUNC RtlInterlockedPushEntrySList16
312    mov [rsp + P3Home], rbx
313    .SAVEREG rbx, P3Home
314    .ENDPROLOG
315
316    /* This is a 16 byte header
317       rcx = ListHead
318       rdx = ListEntry
319       rax = ListHead->Alignment
320       r9 = ListHead->Region */
321
322    /* Copy rcx/rdx to r8/r9, as we need rcx/rdx for the exchange */
323    mov r8, rcx
324    mov r9, rdx
325
326    /* Set NewListHead.NextEntry = ListEntry */
327    mov rcx, rdx
328
329    /* Set NewListHead.HeaderType = 1 and NewListHead.Init = 1 */
330    or rcx, (SLIST16B_HEADERTYPE_MASK or SLIST16B_INIT_MASK)
331
332    /* Set rdx = ListHead->Region */
333    mov rdx, [r8 + 8]
334
335RtlInterlockedPushEntrySListLoop16:
336    /* r8 = ListHead
337       r9 = ListEntry
338       rax = ListHead->Alignment
339       rdx = ListHead->Region
340    */
341
342    /* Move ListHead->NextEntry to rbx */
343    mov rbx, rdx
344    and rbx, SLIST16B_NEXTENTRY_MASK
345
346    /* Store next pointer in ListEntry->Next */
347    mov [r9], rbx
348
349    /* Copy and increment Depth and Sequence number to rbx */
350    lea rbx, [rax + SLIST16A_DEPTH_INC + SLIST16A_SEQUENCE_INC]
351
352    /* If [r8] equals rdx:rax, exchange it with rcx:rbx */
353    lock cmpxchg16b [r8]
354
355    /* If not equal, retry with rdx:rax, being the content of [r8] now */
356    jne RtlInterlockedPushEntrySListLoop16
357
358    /* Copy the old NextEntry pointer to rax */
359    mov rax, rdx
360    and rax, SLIST16B_NEXTENTRY_MASK
361
362    /* Return */
363    mov rbx, [rsp + P3Home]
364    ret
365
366ENDFUNC
367
368
369/* PSLIST_ENTRY
370 * NTAPI
371 * RtlInterlockedFlushSList(
372 *     IN PSLIST_HEADER ListHead);
373 */
374ExpInterlockedFlushSList:
375FUNC RtlInterlockedFlushSList
376
377    mov [rsp + P3Home], rbx
378    .SAVEREG rbx, P3Home
379    .ENDPROLOG
380
381    /* Load ListHead->Region into rdx */
382    mov rdx, [rcx + 8]
383
384    /* Load ListHead->Alignment into rax */
385    mov rax, [rcx]
386
387    /* Check for 16 byte SList support */
388    cmp byte ptr [rip+RtlpUse16ByteSLists], 0
389    jne RtlInterlockedFlushSList16
390
391    /* Use the 8 byte header */
392
393RtlInterlockedFlushSListLoop:
394
395    /* Zero NewListHead.Alignment */
396    xor r8, r8
397
398    /* If [rcx] equals rax, exchange it with r8 */
399    lock cmpxchg [rcx], r8
400
401    /* If not equal, retry with rax, being the content of [rcx] now */
402    jne RtlInterlockedFlushSListLoop
403
404    /* Create a pointer template from rcx in rdx */
405    mov rdx, (NOT SLIST8_POINTER_MASK)
406    and rdx, rcx
407
408    /* Load the old NextEntry pointer into rax */
409    and rax, SLIST8A_NEXTENTRY_MASK
410    shr rax, SLIST8A_NEXTENTRY_SHIFT
411
412    /* Combine result and return */
413    or rax, rdx
414    ret
415
416RtlInterlockedFlushSList16:
417    /* We have a 16 byte header
418        rcx = ListHead
419        rax = ListHead->Alignment
420        rdx = ListHead->Region
421    */
422
423    /* Load ListHead into r8, as we need rcx for the exchange */
424    mov r8, rcx
425
426    /* Initialize an ampty NewListHead in rcx:rbx */
427    xor rbx, rbx
428    mov rcx, (SLIST16B_HEADERTYPE_MASK or SLIST16B_INIT_MASK)
429
430RtlInterlockedFlushSListLoop16:
431
432    /* If [r8] equals rdx:rax, exchange it with rcx:rbx */
433    lock cmpxchg16b [r8]
434
435    /* If not equal, retry with rdx:rax, being the content of [r8] now */
436    jne RtlInterlockedFlushSListLoop16
437
438    /* Copy the old NextEntry pointer to rax */
439    mov rax, rdx
440    and rax, SLIST16B_NEXTENTRY_MASK
441
442    /* Return */
443    mov rbx, [rsp + P3Home]
444    ret
445
446ENDFUNC
447
448END
449
450