xref: /reactos/sdk/lib/rtl/amd64/slist.S (revision 5100859e) 
1/*
2 * COPYRIGHT:       See COPYING in the top level directory
3 * PROJECT:         ReactOS system libraries
4 * FILE:            lib/rtl/amd64/slist.S
5 * PURPOSE:         Rtl Interlocked Functions for amd64
6 * PROGRAMMERS:     Timo Kreuzer
7 */
8
9#include <asm.inc>
10#include <ksamd64.inc>
11
12EXTERN RtlpUse16ByteSLists:BYTE
13
14/*
15typedef union DECLSPEC_ALIGN(16) _SLIST_HEADER
16{
17    struct
18    {
19        ULONGLONG Alignment;
20        ULONGLONG Region;
21    };
22    struct
23    {
24        ULONGLONG Depth:16;
25        ULONGLONG Sequence:9;
26        ULONGLONG NextEntry:39;
27        ULONGLONG HeaderType:1;
28        ULONGLONG Init:1;
29        ULONGLONG Reserved:59;
30        ULONGLONG Region:3;
31    } Header8;
32    struct
33    {
34        ULONGLONG Depth:16;
35        ULONGLONG Sequence:48;
36        ULONGLONG HeaderType:1;
37        ULONGLONG Init:1;
38        ULONGLONG Reserved:2;
39        ULONGLONG NextEntry:60;
40    } Header16;
41    struct
42    {
43        ULONGLONG Depth:16;
44        ULONGLONG Sequence:48;
45        ULONGLONG HeaderType:1;
46        ULONGLONG Reserved:3;
47        ULONGLONG NextEntry:60;
48    } HeaderX64;
49} SLIST_HEADER, *PSLIST_HEADER;
50*/
51
52#define SLIST8A_DEPTH_MASK       HEX(000000000000FFFF)
53#define SLIST8A_DEPTH_INC        HEX(0000000000000001)
54#define SLIST8A_SEQUENCE_MASK    HEX(0000000001FF0000)
55#define SLIST8A_SEQUENCE_INC     HEX(0000000000010000)
56#define SLIST8A_NEXTENTRY_MASK   HEX(FFFFFFFFFE000000)
57#define SLIST8A_NEXTENTRY_SHIFT  21
58#define SLIST8B_HEADERTYPE_MASK  HEX(0000000000000001)
59#define SLIST8B_INIT_MASK        HEX(0000000000000002)
60#define SLIST8B_REGION_MASK      HEX(E000000000000000)
61#define SLIST8_POINTER_MASK      HEX(000007FFFFFFFFFF)
62
63#define SLIST16A_DEPTH_MASK      HEX(000000000000FFFF)
64#define SLIST16A_DEPTH_INC       HEX(0000000000000001)
65#define SLIST16A_SEQUENCE_MASK   HEX(FFFFFFFFFFFF0000)
66#define SLIST16A_SEQUENCE_INC    HEX(0000000000010000)
67#define SLIST16B_HEADERTYPE_MASK HEX(0000000000000001)
68#define SLIST16B_INIT_MASK       HEX(0000000000000002)
69#define SLIST16B_NEXTENTRY_MASK  HEX(FFFFFFFFFFFFFFF0)
70
71
72/* FUNCTIONS ****************************************************************/
73
74.code64
75
76PUBLIC ExpInterlockedPopEntrySList
77PUBLIC ExpInterlockedPopEntrySListResume
78PUBLIC ExpInterlockedPopEntrySListFault
79PUBLIC ExpInterlockedPopEntrySListEnd
80PUBLIC ExpInterlockedPopEntrySListResume16
81PUBLIC ExpInterlockedPopEntrySListFault16
82PUBLIC ExpInterlockedPopEntrySListEnd16
83PUBLIC ExpInterlockedPushEntrySList
84PUBLIC ExpInterlockedFlushSList
85
86PUBLIC RtlInterlockedFlushSList
87PUBLIC RtlInterlockedPopEntrySList
88PUBLIC RtlInterlockedPushEntrySList
89
90/* PSLIST_ENTRY
91 * NTAPI
92 * RtlInterlockedPopEntrySList(
93 *     IN PSLIST_HEADER ListHead);
94 */
95RtlInterlockedPopEntrySList:
96ExpInterlockedPopEntrySList:
97
98    /* Load ListHead->Region into rdx */
99    mov rdx, [rcx + 8]
100
101    /* Load ListHead->Alignment into rax */
102    mov rax, [rcx]
103
104    /* Check for 16 byte SList support */
105    cmp byte ptr [RtlpUse16ByteSLists], 0
106    jne RtlInterlockedPopEntrySList16
107
108    /* Use the 8 byte header */
109
110ExpInterlockedPopEntrySListResume:
111
112    /* Check if ListHead->NextEntry is NULL */
113    mov r9, rax
114    and r9, SLIST8A_NEXTENTRY_MASK
115    jz RtlInterlockedPopEntrySListEmpty
116
117    /* Copy Depth and Sequence number and adjust Depth */
118    lea r8, [rax - SLIST8A_DEPTH_INC]
119    and r8, (SLIST8A_SEQUENCE_MASK OR SLIST8A_DEPTH_MASK)
120
121    /* Create a pointer template from rcx in rdx */
122    mov rdx, (NOT SLIST8_POINTER_MASK)
123    and rdx, rcx
124
125    /* Shift the NextEntry pointer */
126    shr r9, SLIST8A_NEXTENTRY_SHIFT
127
128    /* Combine to new pointer in rdx */
129    or rdx, r9
130
131ExpInterlockedPopEntrySListFault:
132
133    /* Load the next NextEntry pointer to r9 */
134    mov r9, [rdx]
135
136    /* Shift bits in place */
137    shl r9, SLIST8A_NEXTENTRY_SHIFT
138
139    /* Combine into r8 */
140    or r8, r9
141
142ExpInterlockedPopEntrySListEnd:
143
144    /* If [rcx] equals rax, exchange it with r8 */
145    lock cmpxchg [rcx], r8
146
147    /* If not equal, retry with rax, being the content of [rcx] now */
148    jne ExpInterlockedPopEntrySListResume
149
150    /* Shift the pointer bits in place */
151    and rax, SLIST8A_NEXTENTRY_MASK
152    shr rax, SLIST8A_NEXTENTRY_SHIFT
153
154    /* Use rcx as pointer template */
155    mov rdx, (NOT SLIST8_POINTER_MASK)
156    and rdx, rcx
157
158    /* Combine result and return */
159    or rax, rdx
160    ret
161
162RtlInterlockedPopEntrySListEmpty:
163    xor rax, rax
164    ret
165
166RtlInterlockedPopEntrySList16:
167    /* This is a 16 byte header
168       rcx == ListHead
169       rdx == ListHead->Region
170       rax == ListHead->Alignment */
171
172    /* Save rbx */
173    push rbx
174
175    /* Copy rcx to r8, as we need rcx for the exchange */
176    mov r8, rcx
177
178ExpInterlockedPopEntrySListResume16:
179
180    /* Set r9 = ListHead->NextEntry and check if it is NULL */
181    mov r9, rdx
182    and r9, SLIST16B_NEXTENTRY_MASK
183    jz RtlInterlockedPopEntrySListEmpty16
184
185ExpInterlockedPopEntrySListFault16:
186
187    /* Set NewListHead.Next = ListHead->NextEntry->Next */
188    mov rcx, [r9]
189
190    /* Set NewListHead.HeaderType = 1 and  NewListHead.Init = 1 */
191    or rcx, (SLIST16B_HEADERTYPE_MASK or SLIST16B_INIT_MASK)
192
193    /* Copy Depth and Sequence number and adjust Depth */
194    lea rbx, [rax - SLIST16A_DEPTH_INC]
195
196ExpInterlockedPopEntrySListEnd16:
197
198    /* If [r8] equals rdx:rax, exchange it with rcx:rbx */
199    lock cmpxchg16b [r8]
200
201    /* If not equal, retry with rdx:rax, being the content of [r8] now */
202    jne ExpInterlockedPopEntrySListResume16
203
204    /* Copy the old NextEntry pointer to rax */
205    mov rax, rdx
206    and rax, SLIST16B_NEXTENTRY_MASK
207
208    /* Return */
209    pop rbx
210    ret
211
212RtlInterlockedPopEntrySListEmpty16:
213    xor rax, rax
214    pop rbx
215    ret
216
217
218/* PSLIST_ENTRY
219 * NTAPI
220 * RtlInterlockedPushEntrySList(
221 *     IN PSLIST_HEADER ListHead,
222 *     IN PSLIST_ENTRY ListEntry);
223 */
224RtlInterlockedPushEntrySList:
225ExpInterlockedPushEntrySList:
226
227#if DBG
228    /* Make sure the ListEntry is 16 bytes aligned */
229    test rdx, HEX(0F)
230    jz ExpInterlockedPushEntrySListChecked
231    /* Not aligned, raise an assertion */
232    int HEX(2C)
233ExpInterlockedPushEntrySListChecked:
234
235    /* Make sure RtlpUse16ByteSLists is initialized */
236    cmp byte ptr [RtlpUse16ByteSLists], HEX(FF)
237    jne ExpInterlockedPushEntrySListChecked2
238    /* Not initialized, raise an assertion */
239    int HEX(2C)
240ExpInterlockedPushEntrySListChecked2:
241#endif
242
243    /* Load ListHead->Alignment into rax */
244    mov rax, [rcx]
245
246    /* Load ListHead->Region into r9 */
247    mov r9, [rcx + 8]
248
249    /* Check for 16 byte SList support */
250    cmp byte ptr [RtlpUse16ByteSLists], 0
251    jne RtlInterlockedPushEntrySList16
252
253    /* Use the 8 byte header */
254
255RtlInterlockedPushEntrySListLoop:
256
257    /* Get ListHead->NextEntry */
258    mov r8, rax
259    and r8, SLIST8A_NEXTENTRY_MASK
260    jz RtlInterlockedPushEntrySListEmpty
261
262    /* Shift the NextEntry pointer */
263    shr r8, SLIST8A_NEXTENTRY_SHIFT
264
265    /* Create a pointer template from rcx in rdx */
266    mov r9, (NOT SLIST8_POINTER_MASK)
267    and r9, rcx
268
269    /* Combine to new pointer and save as ListEntry->NextEntry */
270    or r8, r9
271
272RtlInterlockedPushEntrySListEmpty:
273    /* Store the NextEntry pointer in the new ListEntry */
274    mov [rdx], r8
275
276    /* Shift and mask the new ListEntry pointer */
277    mov r8, rdx
278    shl r8, SLIST8A_NEXTENTRY_SHIFT
279    and r8, SLIST8A_NEXTENTRY_MASK
280
281    /* Copy and adjust depth and sequence number */
282    lea r9, [rax + SLIST8A_DEPTH_INC + SLIST8A_SEQUENCE_INC]
283    and r9, SLIST8A_SEQUENCE_MASK OR SLIST8A_DEPTH_MASK
284
285    /* Combine to exchange value in r8 */
286    or r8, r9
287
288    /* Save the NextEntry in r9 */
289    mov r9, [rdx]
290
291    /* If [rcx] equals rax, exchange it with r8 */
292    lock cmpxchg [rcx], r8
293
294    /* If not equal, retry with rax, being the content of [rcx] now */
295    jne RtlInterlockedPushEntrySListLoop
296
297    /* Return the old NextEntry pointer */
298    mov rax, r9
299    ret
300
301RtlInterlockedPushEntrySList16:
302    /* This is a 16 byte header
303       rcx = ListHead
304       rdx = ListEntry
305       rax = ListHead->Alignment
306       r9 = ListHead->Region */
307
308    /* Save rbx */
309    push rbx
310
311    /* Copy rcx/rdx to r8/r9, as we need rcx/rdx for the exchange */
312    mov r8, rcx
313    mov r9, rdx
314
315    /* Set NewListHead.NextEntry = ListEntry */
316    mov rcx, rdx
317
318    /* Set NewListHead.HeaderType = 1 and NewListHead.Init = 1 */
319    or rcx, (SLIST16B_HEADERTYPE_MASK or SLIST16B_INIT_MASK)
320
321    /* Set rdx = ListHead->Region */
322    mov rdx, [r8 + 8]
323
324RtlInterlockedPushEntrySListLoop16:
325    /* r8 = ListHead
326       r9 = ListEntry
327       rax = ListHead->Alignment
328       rdx = ListHead->Region
329    */
330
331    /* Move ListHead->NextEntry to rbx */
332    mov rbx, rdx
333    and rbx, SLIST16B_NEXTENTRY_MASK
334
335    /* Store next pointer in ListEntry->Next */
336    mov [r9], rbx
337
338    /* Copy and increment Depth and Sequence number to rbx */
339    lea rbx, [rax + SLIST16A_DEPTH_INC + SLIST16A_SEQUENCE_INC]
340
341    /* If [r8] equals rdx:rax, exchange it with rcx:rbx */
342    lock cmpxchg16b [r8]
343
344    /* If not equal, retry with rdx:rax, being the content of [r8] now */
345    jne RtlInterlockedPushEntrySListLoop16
346
347    /* Copy the old NextEntry pointer to rax */
348    mov rax, rdx
349    and rax, SLIST16B_NEXTENTRY_MASK
350
351    /* Return */
352    pop rbx
353    ret
354
355
356/* PSLIST_ENTRY
357 * NTAPI
358 * RtlInterlockedFlushSList(
359 *     IN PSLIST_HEADER ListHead);
360 */
361RtlInterlockedFlushSList:
362ExpInterlockedFlushSList:
363
364    /* Load ListHead->Region into rdx */
365    mov rdx, [rcx + 8]
366
367    /* Load ListHead->Alignment into rax */
368    mov rax, [rcx]
369
370    /* Check for 16 byte SList support */
371    cmp byte ptr [RtlpUse16ByteSLists], 0
372    jne RtlInterlockedFlushSList16
373
374    /* Use the 8 byte header */
375
376RtlInterlockedFlushSListLoop:
377
378    /* Zero NewListHead.Alignment */
379    xor r8, r8
380
381    /* If [rcx] equals rax, exchange it with r8 */
382    lock cmpxchg [rcx], r8
383
384    /* If not equal, retry with rax, being the content of [rcx] now */
385    jne RtlInterlockedFlushSListLoop
386
387    /* Create a pointer template from rcx in rdx */
388    mov rdx, (NOT SLIST8_POINTER_MASK)
389    and rdx, rcx
390
391    /* Load the old NextEntry pointer into rax */
392    and rax, SLIST8A_NEXTENTRY_MASK
393    shr rax, SLIST8A_NEXTENTRY_SHIFT
394
395    /* Combine result and return */
396    or rax, rdx
397    ret
398
399RtlInterlockedFlushSList16:
400    /* We have a 16 byte header
401        rcx = ListHead
402        rax = ListHead->Alignment
403        rdx = ListHead->Region
404    */
405
406    /* Save rbx */
407    push rbx
408
409    /* Load ListHead into r8, as we need rcx for the exchange */
410    mov r8, rcx
411
412    /* Initialize an ampty NewListHead in rcx:rbx */
413    xor rbx, rbx
414    mov rcx, (SLIST16B_HEADERTYPE_MASK or SLIST16B_INIT_MASK)
415
416RtlInterlockedFlushSListLoop16:
417
418    /* If [r8] equals rdx:rax, exchange it with rcx:rbx */
419    lock cmpxchg16b [r8]
420
421    /* If not equal, retry with rdx:rax, being the content of [r8] now */
422    jne RtlInterlockedFlushSListLoop16
423
424    /* Copy the old NextEntry pointer to rax */
425    mov rax, rdx
426    and rax, SLIST16B_NEXTENTRY_MASK
427
428    /* Return */
429    pop rbx
430    ret
431
432END
433
434