1/* Copyright (C) 2008-2021 Free Software Foundation, Inc.
2   Contributed by Richard Henderson <rth@redhat.com>.
3
4   This file is part of the GNU Transactional Memory Library (libitm).
5
6   Libitm is free software; you can redistribute it and/or modify it
7   under the terms of the GNU General Public License as published by
8   the Free Software Foundation; either version 3 of the License, or
9   (at your option) any later version.
10
11   Libitm is distributed in the hope that it will be useful, but WITHOUT ANY
12   WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
13   FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
14   more details.
15
16   Under Section 7 of GPL version 3, you are granted additional
17   permissions described in the GCC Runtime Library Exception, version
18   3.1, as published by the Free Software Foundation.
19
20   You should have received a copy of the GNU General Public License and
21   a copy of the GCC Runtime Library Exception along with this program;
22   see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
23   <http://www.gnu.org/licenses/>.  */
24
25
26#include "asmcfi.h"
27#include "config.h"
28#include "cet.h"
29
30#define CONCAT1(a, b) CONCAT2(a, b)
31#define CONCAT2(a, b) a ## b
32
33#ifdef __USER_LABEL_PREFIX__
34#  define SYM(x) CONCAT1 (__USER_LABEL_PREFIX__, x)
35#else
36#  define SYM(x) x
37#endif
38
39#ifdef __ELF__
40#  define TYPE(x) .type SYM(x), @function
41#  define SIZE(x) .size SYM(x), . - SYM(x)
42#  ifdef HAVE_ATTRIBUTE_VISIBILITY
43#    define HIDDEN(x) .hidden SYM(x)
44#  else
45#    define HIDDEN(x)
46#  endif
47#else
48#  define TYPE(x)
49#  define SIZE(x)
50#  ifdef __MACH__
51#    define HIDDEN(x) .private_extern SYM(x)
52#  else
53#    define HIDDEN(x)
54#  endif
55#endif
56
57/* These are duplicates of the canonical definitions in libitm.h.  Note that
58   the code relies on pr_uninstrumentedCode == a_runUninstrumentedCode.  */
59#define pr_uninstrumentedCode	0x02
60#define pr_hasNoAbort		0x08
61#define pr_HTMRetryableAbort	0x800000
62#define pr_HTMRetriedAfterAbort	0x1000000
63#define a_runInstrumentedCode	0x01
64#define a_runUninstrumentedCode	0x02
65#define a_tryHTMFastPath	0x20
66
67#define _XABORT_EXPLICIT	(1 << 0)
68#define _XABORT_RETRY		(1 << 1)
69
70	.text
71
72	.align 4
73	.globl	SYM(_ITM_beginTransaction)
74
75SYM(_ITM_beginTransaction):
76	cfi_startproc
77	_CET_ENDBR
78#ifdef __x86_64__
79#ifdef HAVE_AS_RTM
80	/* Custom HTM fast path.  We start the HW transaction here and let
81	   gtm_thread::begin_transaction (aka GTM_begin_transaction) decide
82	   how to proceed on aborts: We either retry the fast path, or fall
83	   back to another execution method.  RTM restores all registers after
84	   a HW transaction abort, so we can do the SW setjmp after aborts,
85	   and we have to because we might choose a SW fall back.  However,
86	   we have to explicitly save/restore the first argument (edi).
87	   The htm_fastpath field is the second int in gtm_rwlock.  */
88	cmpl	$0, (SYM(gtm_serial_lock)+4)(%rip)
89	jz	.Lno_htm
90	testl	$pr_hasNoAbort, %edi
91	jz	.Lno_htm
92.Lhtm_fastpath:
93	xbegin	.Ltxn_abort
94	/* Monitor the serial lock (specifically, the 32b writer/summary field
95	   at its start), and only continue if there is no serial-mode
96	   transaction.  Note that we might be just a nested transaction and
97	   our outermost transaction might be in serial mode; we check for
98	   this case in the retry policy implementation.  */
99	cmpl	$0, SYM(gtm_serial_lock)(%rip)
100	jnz	1f
101	/* Now also check that HW transactions are still allowed to run (see
102	   gtm_thread::begin_transaction for why this is necessary).  */
103	cmpl	$0, (SYM(gtm_serial_lock)+4)(%rip)
104	jz	1f
105	/* Everything is good.  Run the transaction, preferably using the
106	   uninstrumented code path.  Note that the following works because
107	   pr_uninstrumentedCode == a_runUninstrumentedCode.  */
108	andl	$pr_uninstrumentedCode, %edi
109	mov	$a_runInstrumentedCode, %eax
110	cmovnz	%edi, %eax
111	ret
112	/* There is a serial-mode transaction or HW transactions are not
113	   allowed anymore, so abort (see htm_abort() regarding the abort
114	   code).  */
1151:	xabort	$0xff
116.Ltxn_abort:
117	/* If it might make sense to retry the HTM fast path, let the C++
118	   code decide.  */
119	testl	$(_XABORT_RETRY|_XABORT_EXPLICIT), %eax
120	jz	.Lno_htm
121	orl	$pr_HTMRetryableAbort, %edi
122	/* Let the C++ code handle the retry policy.  */
123.Lno_htm:
124#endif
125	leaq	8(%rsp), %rax
126	subq	$72, %rsp
127	cfi_adjust_cfa_offset(72)
128	/* Store edi for future HTM fast path retries.  We use a stack slot
129	   lower than the jmpbuf so that the jmpbuf's rip field will overlap
130	   with the proper return address on the stack.  */
131	movl	%edi, (%rsp)
132	/* Save the jmpbuf for any non-HTM-fastpath execution method.
133	   Because rsp-based addressing is 1 byte larger and we've got rax
134	   handy, use it.  */
135	movq	%rax, -72(%rax)
136	movq	%rbx, -64(%rax)
137	movq	%rbp, -56(%rax)
138	movq	%r12, -48(%rax)
139	movq	%r13, -40(%rax)
140	movq	%r14, -32(%rax)
141	movq	%r15, -24(%rax)
142	xorq	%rdx, %rdx
143	/* Save zero or shadow stack pointer in the new field.  */
144#if defined __SHSTK__ && defined __CET__ && (__CET__ & 2) != 0
145	rdsspq	%rdx
146#endif
147	movq	%rdx, -16(%rax)
148	leaq	-72(%rax), %rsi
149	call	SYM(GTM_begin_transaction)
150	movl	(%rsp), %edi
151	addq	$72, %rsp
152	cfi_adjust_cfa_offset(-72)
153#ifdef HAVE_AS_RTM
154	/* If a_tryHTMFastPath was returned, then we need to retry the
155	   fast path.  We also restore edi and set pr_HTMRetriedAfterAbort
156	   to state that we have retried the fast path already (it's harmless
157	   if this bit is set even if we don't retry the fast path because it
158	   is checked iff pr_HTMRetryableAbort is set).  We clear
159	   pr_HTMRetryableAbort because it applies to a previous HW
160	   transaction attempt.  */
161	cmpl	$a_tryHTMFastPath, %eax
162	jnz	2f
163	andl	$(0xffffffff-pr_HTMRetryableAbort), %edi
164	orl	$pr_HTMRetriedAfterAbort, %edi
165	jmp	.Lhtm_fastpath
1662:
167#endif
168#else
169	leal	4(%esp), %ecx
170	movl	4(%esp), %eax
171	subl	$28, %esp
172	cfi_def_cfa_offset(32)
173	movl	%ecx, 4(%esp)
174	movl	%ebx, 8(%esp)
175	movl	%esi, 12(%esp)
176	movl	%edi, 16(%esp)
177	movl	%ebp, 20(%esp)
178	xorl	%edx, %edx
179	/* Save zero or shadow stack pointer in the new field.  */
180#if defined __SHSTK__ && defined __CET__ && (__CET__ & 2) != 0
181	rdsspd	%edx
182#endif
183	movl	%edx, 24(%esp)
184	leal	4(%esp), %edx
185#if defined HAVE_ATTRIBUTE_VISIBILITY || !defined __PIC__
186	call	SYM(GTM_begin_transaction)
187#elif defined __ELF__
188	call	1f
1891:	popl	%ebx
190	addl	$_GLOBAL_OFFSET_TABLE_+[.-1b], %ebx
191	call	SYM(GTM_begin_transaction)@PLT
192	movl	8(%esp), %ebx
193#else
194# error "Unsupported PIC sequence"
195#endif
196	addl	$28, %esp
197	cfi_def_cfa_offset(4)
198#endif
199	ret
200	cfi_endproc
201
202	TYPE(_ITM_beginTransaction)
203	SIZE(_ITM_beginTransaction)
204
205	.align 4
206	.globl	SYM(GTM_longjmp)
207
208SYM(GTM_longjmp):
209	cfi_startproc
210	_CET_ENDBR
211#ifdef __x86_64__
212	movq	(%rsi), %rcx
213	movq	8(%rsi), %rbx
214	movq	16(%rsi), %rbp
215	movq	24(%rsi), %r12
216	movq	32(%rsi), %r13
217	movq	40(%rsi), %r14
218	movq	48(%rsi), %r15
219	movl	%edi, %eax
220	cfi_def_cfa(%rsi, 0)
221	cfi_offset(%rip, 64)
222	cfi_register(%rsp, %rcx)
223	movq	%rcx, %rsp
224#if defined __SHSTK__ && defined __CET__ && (__CET__ & 2) != 0
225	/* Check if Shadow Stack is enabled.  */
226	xorq	%rcx, %rcx
227	rdsspq	%rcx
228	testq	%rcx, %rcx
229	je	.L1
230	/* Calculate number of frames to skip.  */
231	subq	56(%rsi), %rcx
232	negq	%rcx
233	shrq	$3, %rcx
234	incq	%rcx
235	/* If # of frames is greater 255 then loop
236	   and adjust.  */
237	cmpq	$255, %rcx
238	jbe	.L3
239	movl	$255, %edi
240	.p2align 4,,10
241	.p2align 3
242.L4:
243	incsspq	%rdi
244	subq	$255, %rcx
245	cmpq	$255, %rcx
246	ja	.L4
247.L3:
248	incsspq	%rcx
249.L1:
250#endif
251	jmp	*64(%rsi)
252#else
253	movl	(%edx), %ecx
254	movl	4(%edx), %ebx
255	movl	8(%edx), %esi
256	movl	12(%edx), %edi
257	movl	16(%edx), %ebp
258	cfi_def_cfa(%edx, 0)
259	cfi_offset(%eip, 24)
260	cfi_register(%esp, %ecx)
261	movl	%ecx, %esp
262#if defined __SHSTK__ && defined __CET__ && (__CET__ & 2) != 0
263	/* Check if Shadow Stack is enabled.  */
264	xorl	%ecx, %ecx
265	rdsspd	%ecx
266	testl	%ecx, %ecx
267	je	.L1
268	/* Calculate # of frames to skip.  */
269	subl	20(%edx), %ecx
270	negl	%ecx
271	shrl	$2, %ecx
272	incl	%ecx
273	/* If # of frames is greater 255 then loop
274	   and adjust.  */
275	cmpl	$255, %ecx
276	jbe	.L3
277	pushl	%eax
278	movl	$255, %eax
279	.p2align 4,,10
280	.p2align 3
281.L4:
282	incsspd	%eax
283	subl	$255, %ecx
284	cmpl	$255, %ecx
285	ja	.L4
286	popl	%eax
287.L3:
288	incsspd	%ecx
289.L1:
290#endif
291	jmp	*24(%edx)
292#endif
293	cfi_endproc
294
295	TYPE(GTM_longjmp)
296	HIDDEN(GTM_longjmp)
297	SIZE(GTM_longjmp)
298
299#ifdef __linux__
300.section .note.GNU-stack, "", @progbits
301#endif
302