1// The content of this file is x86_64-only:
2#if defined(__x86_64__)
3
4#include "sanitizer_common/sanitizer_asm.h"
5
6#if !defined(__APPLE__)
7.section .text
8#else
9.section __TEXT,__text
10#endif
11
12ASM_HIDDEN(__tsan_trace_switch)
13.globl ASM_SYMBOL(__tsan_trace_switch_thunk)
14ASM_SYMBOL(__tsan_trace_switch_thunk):
15  CFI_STARTPROC
16  # Save scratch registers.
17  push %rax
18  CFI_ADJUST_CFA_OFFSET(8)
19  CFI_REL_OFFSET(%rax, 0)
20  push %rcx
21  CFI_ADJUST_CFA_OFFSET(8)
22  CFI_REL_OFFSET(%rcx, 0)
23  push %rdx
24  CFI_ADJUST_CFA_OFFSET(8)
25  CFI_REL_OFFSET(%rdx, 0)
26  push %rsi
27  CFI_ADJUST_CFA_OFFSET(8)
28  CFI_REL_OFFSET(%rsi, 0)
29  push %rdi
30  CFI_ADJUST_CFA_OFFSET(8)
31  CFI_REL_OFFSET(%rdi, 0)
32  push %r8
33  CFI_ADJUST_CFA_OFFSET(8)
34  CFI_REL_OFFSET(%r8, 0)
35  push %r9
36  CFI_ADJUST_CFA_OFFSET(8)
37  CFI_REL_OFFSET(%r9, 0)
38  push %r10
39  CFI_ADJUST_CFA_OFFSET(8)
40  CFI_REL_OFFSET(%r10, 0)
41  push %r11
42  CFI_ADJUST_CFA_OFFSET(8)
43  CFI_REL_OFFSET(%r11, 0)
44  # Align stack frame.
45  push %rbx  # non-scratch
46  CFI_ADJUST_CFA_OFFSET(8)
47  CFI_REL_OFFSET(%rbx, 0)
48  mov %rsp, %rbx  # save current rsp
49  CFI_DEF_CFA_REGISTER(%rbx)
50  shr $4, %rsp  # clear 4 lsb, align to 16
51  shl $4, %rsp
52
53  call ASM_SYMBOL(__tsan_trace_switch)
54
55  # Unalign stack frame back.
56  mov %rbx, %rsp  # restore the original rsp
57  CFI_DEF_CFA_REGISTER(%rsp)
58  pop %rbx
59  CFI_ADJUST_CFA_OFFSET(-8)
60  # Restore scratch registers.
61  pop %r11
62  CFI_ADJUST_CFA_OFFSET(-8)
63  pop %r10
64  CFI_ADJUST_CFA_OFFSET(-8)
65  pop %r9
66  CFI_ADJUST_CFA_OFFSET(-8)
67  pop %r8
68  CFI_ADJUST_CFA_OFFSET(-8)
69  pop %rdi
70  CFI_ADJUST_CFA_OFFSET(-8)
71  pop %rsi
72  CFI_ADJUST_CFA_OFFSET(-8)
73  pop %rdx
74  CFI_ADJUST_CFA_OFFSET(-8)
75  pop %rcx
76  CFI_ADJUST_CFA_OFFSET(-8)
77  pop %rax
78  CFI_ADJUST_CFA_OFFSET(-8)
79  CFI_RESTORE(%rax)
80  CFI_RESTORE(%rbx)
81  CFI_RESTORE(%rcx)
82  CFI_RESTORE(%rdx)
83  CFI_RESTORE(%rsi)
84  CFI_RESTORE(%rdi)
85  CFI_RESTORE(%r8)
86  CFI_RESTORE(%r9)
87  CFI_RESTORE(%r10)
88  CFI_RESTORE(%r11)
89  ret
90  CFI_ENDPROC
91
92ASM_HIDDEN(__tsan_report_race)
93.globl ASM_SYMBOL(__tsan_report_race_thunk)
94ASM_SYMBOL(__tsan_report_race_thunk):
95  CFI_STARTPROC
96  # Save scratch registers.
97  push %rax
98  CFI_ADJUST_CFA_OFFSET(8)
99  CFI_REL_OFFSET(%rax, 0)
100  push %rcx
101  CFI_ADJUST_CFA_OFFSET(8)
102  CFI_REL_OFFSET(%rcx, 0)
103  push %rdx
104  CFI_ADJUST_CFA_OFFSET(8)
105  CFI_REL_OFFSET(%rdx, 0)
106  push %rsi
107  CFI_ADJUST_CFA_OFFSET(8)
108  CFI_REL_OFFSET(%rsi, 0)
109  push %rdi
110  CFI_ADJUST_CFA_OFFSET(8)
111  CFI_REL_OFFSET(%rdi, 0)
112  push %r8
113  CFI_ADJUST_CFA_OFFSET(8)
114  CFI_REL_OFFSET(%r8, 0)
115  push %r9
116  CFI_ADJUST_CFA_OFFSET(8)
117  CFI_REL_OFFSET(%r9, 0)
118  push %r10
119  CFI_ADJUST_CFA_OFFSET(8)
120  CFI_REL_OFFSET(%r10, 0)
121  push %r11
122  CFI_ADJUST_CFA_OFFSET(8)
123  CFI_REL_OFFSET(%r11, 0)
124  # Align stack frame.
125  push %rbx  # non-scratch
126  CFI_ADJUST_CFA_OFFSET(8)
127  CFI_REL_OFFSET(%rbx, 0)
128  mov %rsp, %rbx  # save current rsp
129  CFI_DEF_CFA_REGISTER(%rbx)
130  shr $4, %rsp  # clear 4 lsb, align to 16
131  shl $4, %rsp
132
133  call ASM_SYMBOL(__tsan_report_race)
134
135  # Unalign stack frame back.
136  mov %rbx, %rsp  # restore the original rsp
137  CFI_DEF_CFA_REGISTER(%rsp)
138  pop %rbx
139  CFI_ADJUST_CFA_OFFSET(-8)
140  # Restore scratch registers.
141  pop %r11
142  CFI_ADJUST_CFA_OFFSET(-8)
143  pop %r10
144  CFI_ADJUST_CFA_OFFSET(-8)
145  pop %r9
146  CFI_ADJUST_CFA_OFFSET(-8)
147  pop %r8
148  CFI_ADJUST_CFA_OFFSET(-8)
149  pop %rdi
150  CFI_ADJUST_CFA_OFFSET(-8)
151  pop %rsi
152  CFI_ADJUST_CFA_OFFSET(-8)
153  pop %rdx
154  CFI_ADJUST_CFA_OFFSET(-8)
155  pop %rcx
156  CFI_ADJUST_CFA_OFFSET(-8)
157  pop %rax
158  CFI_ADJUST_CFA_OFFSET(-8)
159  CFI_RESTORE(%rax)
160  CFI_RESTORE(%rbx)
161  CFI_RESTORE(%rcx)
162  CFI_RESTORE(%rdx)
163  CFI_RESTORE(%rsi)
164  CFI_RESTORE(%rdi)
165  CFI_RESTORE(%r8)
166  CFI_RESTORE(%r9)
167  CFI_RESTORE(%r10)
168  CFI_RESTORE(%r11)
169  ret
170  CFI_ENDPROC
171
172ASM_HIDDEN(__tsan_setjmp)
173#if defined(__NetBSD__)
174.comm _ZN14__interception15real___setjmp14E,8,8
175#elif !defined(__APPLE__)
176.comm _ZN14__interception11real_setjmpE,8,8
177#endif
178#if defined(__NetBSD__)
179.globl ASM_SYMBOL_INTERCEPTOR(__setjmp14)
180ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(__setjmp14))
181ASM_SYMBOL_INTERCEPTOR(__setjmp14):
182#else
183.globl ASM_SYMBOL_INTERCEPTOR(setjmp)
184ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(setjmp))
185ASM_SYMBOL_INTERCEPTOR(setjmp):
186#endif
187  CFI_STARTPROC
188  // save env parameter
189  push %rdi
190  CFI_ADJUST_CFA_OFFSET(8)
191  CFI_REL_OFFSET(%rdi, 0)
192  // obtain SP, store in %rdi, first argument to `void __tsan_setjmp(uptr sp)`
193#if defined(__FreeBSD__) || defined(__NetBSD__)
194  lea 8(%rsp), %rdi
195#elif defined(__linux__) || defined(__APPLE__)
196  lea 16(%rsp), %rdi
197#else
198# error "Unknown platform"
199#endif
200  // call tsan interceptor
201  call ASM_SYMBOL(__tsan_setjmp)
202  // restore env parameter
203  pop %rdi
204  CFI_ADJUST_CFA_OFFSET(-8)
205  CFI_RESTORE(%rdi)
206  // tail jump to libc setjmp
207  movl $0, %eax
208#if defined(__NetBSD__)
209  movq _ZN14__interception15real___setjmp14E@GOTPCREL(%rip), %rdx
210  jmp *(%rdx)
211#elif !defined(__APPLE__)
212  movq _ZN14__interception11real_setjmpE@GOTPCREL(%rip), %rdx
213  jmp *(%rdx)
214#else
215  jmp ASM_SYMBOL(setjmp)
216#endif
217  CFI_ENDPROC
218#if defined(__NetBSD__)
219ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(__setjmp14))
220#else
221ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(setjmp))
222#endif
223
224.comm _ZN14__interception12real__setjmpE,8,8
225.globl ASM_SYMBOL_INTERCEPTOR(_setjmp)
226ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(_setjmp))
227ASM_SYMBOL_INTERCEPTOR(_setjmp):
228  CFI_STARTPROC
229  // save env parameter
230  push %rdi
231  CFI_ADJUST_CFA_OFFSET(8)
232  CFI_REL_OFFSET(%rdi, 0)
233  // obtain SP, store in %rdi, first argument to `void __tsan_setjmp(uptr sp)`
234#if defined(__FreeBSD__) || defined(__NetBSD__)
235  lea 8(%rsp), %rdi
236#elif defined(__linux__) || defined(__APPLE__)
237  lea 16(%rsp), %rdi
238#else
239# error "Unknown platform"
240#endif
241  // call tsan interceptor
242  call ASM_SYMBOL(__tsan_setjmp)
243  // restore env parameter
244  pop %rdi
245  CFI_ADJUST_CFA_OFFSET(-8)
246  CFI_RESTORE(%rdi)
247  // tail jump to libc setjmp
248  movl $0, %eax
249#if !defined(__APPLE__)
250  movq _ZN14__interception12real__setjmpE@GOTPCREL(%rip), %rdx
251  jmp *(%rdx)
252#else
253  jmp ASM_SYMBOL(_setjmp)
254#endif
255  CFI_ENDPROC
256ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(_setjmp))
257
258#if defined(__NetBSD__)
259.comm _ZN14__interception18real___sigsetjmp14E,8,8
260.globl ASM_SYMBOL_INTERCEPTOR(__sigsetjmp14)
261ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(__sigsetjmp14))
262ASM_SYMBOL_INTERCEPTOR(__sigsetjmp14):
263#else
264.comm _ZN14__interception14real_sigsetjmpE,8,8
265.globl ASM_SYMBOL_INTERCEPTOR(sigsetjmp)
266ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(sigsetjmp))
267ASM_SYMBOL_INTERCEPTOR(sigsetjmp):
268#endif
269  CFI_STARTPROC
270  // save env parameter
271  push %rdi
272  CFI_ADJUST_CFA_OFFSET(8)
273  CFI_REL_OFFSET(%rdi, 0)
274  // save savesigs parameter
275  push %rsi
276  CFI_ADJUST_CFA_OFFSET(8)
277  CFI_REL_OFFSET(%rsi, 0)
278  // align stack frame
279  sub $8, %rsp
280  CFI_ADJUST_CFA_OFFSET(8)
281  // obtain SP, store in %rdi, first argument to `void __tsan_setjmp(uptr sp)`
282#if defined(__FreeBSD__) || defined(__NetBSD__)
283  lea 24(%rsp), %rdi
284#elif defined(__linux__) || defined(__APPLE__)
285  lea 32(%rsp), %rdi
286#else
287# error "Unknown platform"
288#endif
289  // call tsan interceptor
290  call ASM_SYMBOL(__tsan_setjmp)
291  // unalign stack frame
292  add $8, %rsp
293  CFI_ADJUST_CFA_OFFSET(-8)
294  // restore savesigs parameter
295  pop %rsi
296  CFI_ADJUST_CFA_OFFSET(-8)
297  CFI_RESTORE(%rsi)
298  // restore env parameter
299  pop %rdi
300  CFI_ADJUST_CFA_OFFSET(-8)
301  CFI_RESTORE(%rdi)
302  // tail jump to libc sigsetjmp
303  movl $0, %eax
304#if defined(__NetBSD__)
305  movq _ZN14__interception18real___sigsetjmp14E@GOTPCREL(%rip), %rdx
306  jmp *(%rdx)
307#elif !defined(__APPLE__)
308  movq _ZN14__interception14real_sigsetjmpE@GOTPCREL(%rip), %rdx
309  jmp *(%rdx)
310#else
311  jmp ASM_SYMBOL(sigsetjmp)
312#endif
313  CFI_ENDPROC
314#if defined(__NetBSD__)
315ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(__sigsetjmp14))
316#else
317ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(sigsetjmp))
318#endif
319
320#if !defined(__APPLE__) && !defined(__NetBSD__)
321.comm _ZN14__interception16real___sigsetjmpE,8,8
322.globl ASM_SYMBOL_INTERCEPTOR(__sigsetjmp)
323ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(__sigsetjmp))
324ASM_SYMBOL_INTERCEPTOR(__sigsetjmp):
325  CFI_STARTPROC
326  // save env parameter
327  push %rdi
328  CFI_ADJUST_CFA_OFFSET(8)
329  CFI_REL_OFFSET(%rdi, 0)
330  // save savesigs parameter
331  push %rsi
332  CFI_ADJUST_CFA_OFFSET(8)
333  CFI_REL_OFFSET(%rsi, 0)
334  // align stack frame
335  sub $8, %rsp
336  CFI_ADJUST_CFA_OFFSET(8)
337  // obtain SP, store in %rdi, first argument to `void __tsan_setjmp(uptr sp)`
338#if defined(__FreeBSD__)
339  lea 24(%rsp), %rdi
340#else
341  lea 32(%rsp), %rdi
342#endif
343  // call tsan interceptor
344  call ASM_SYMBOL(__tsan_setjmp)
345  // unalign stack frame
346  add $8, %rsp
347  CFI_ADJUST_CFA_OFFSET(-8)
348  // restore savesigs parameter
349  pop %rsi
350  CFI_ADJUST_CFA_OFFSET(-8)
351  CFI_RESTORE(%rsi)
352  // restore env parameter
353  pop %rdi
354  CFI_ADJUST_CFA_OFFSET(-8)
355  CFI_RESTORE(%rdi)
356  // tail jump to libc sigsetjmp
357  movl $0, %eax
358  movq _ZN14__interception16real___sigsetjmpE@GOTPCREL(%rip), %rdx
359  jmp *(%rdx)
360  CFI_ENDPROC
361ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(__sigsetjmp))
362#endif  // !defined(__APPLE__) && !defined(__NetBSD__)
363
364NO_EXEC_STACK_DIRECTIVE
365
366#endif
367