1 /* SPDX-License-Identifier: LGPL-2.1 OR MIT */
2 /*
3  * rseq-arm64.h
4  *
5  * (C) Copyright 2016-2018 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
6  * (C) Copyright 2018 - Will Deacon <will.deacon@arm.com>
7  */
8 
9 /*
10  * aarch64 -mbig-endian generates mixed endianness code vs data:
11  * little-endian code and big-endian data. Ensure the RSEQ_SIG signature
12  * matches code endianness.
13  */
14 #define RSEQ_SIG_CODE	0xd428bc00	/* BRK #0x45E0.  */
15 
16 #ifdef __AARCH64EB__
17 #define RSEQ_SIG_DATA	0x00bc28d4	/* BRK #0x45E0.  */
18 #else
19 #define RSEQ_SIG_DATA	RSEQ_SIG_CODE
20 #endif
21 
22 #define RSEQ_SIG	RSEQ_SIG_DATA
23 
24 #define rseq_smp_mb()	__asm__ __volatile__ ("dmb ish" ::: "memory")
25 #define rseq_smp_rmb()	__asm__ __volatile__ ("dmb ishld" ::: "memory")
26 #define rseq_smp_wmb()	__asm__ __volatile__ ("dmb ishst" ::: "memory")
27 
28 #define rseq_smp_load_acquire(p)						\
29 __extension__ ({								\
30 	__typeof(*p) ____p1;							\
31 	switch (sizeof(*p)) {							\
32 	case 1:									\
33 		asm volatile ("ldarb %w0, %1"					\
34 			: "=r" (*(__u8 *)p)					\
35 			: "Q" (*p) : "memory");					\
36 		break;								\
37 	case 2:									\
38 		asm volatile ("ldarh %w0, %1"					\
39 			: "=r" (*(__u16 *)p)					\
40 			: "Q" (*p) : "memory");					\
41 		break;								\
42 	case 4:									\
43 		asm volatile ("ldar %w0, %1"					\
44 			: "=r" (*(__u32 *)p)					\
45 			: "Q" (*p) : "memory");					\
46 		break;								\
47 	case 8:									\
48 		asm volatile ("ldar %0, %1"					\
49 			: "=r" (*(__u64 *)p)					\
50 			: "Q" (*p) : "memory");					\
51 		break;								\
52 	}									\
53 	____p1;									\
54 })
55 
56 #define rseq_smp_acquire__after_ctrl_dep()	rseq_smp_rmb()
57 
58 #define rseq_smp_store_release(p, v)						\
59 do {										\
60 	switch (sizeof(*p)) {							\
61 	case 1:									\
62 		asm volatile ("stlrb %w1, %0"					\
63 				: "=Q" (*p)					\
64 				: "r" ((__u8)v)					\
65 				: "memory");					\
66 		break;								\
67 	case 2:									\
68 		asm volatile ("stlrh %w1, %0"					\
69 				: "=Q" (*p)					\
70 				: "r" ((__u16)v)				\
71 				: "memory");					\
72 		break;								\
73 	case 4:									\
74 		asm volatile ("stlr %w1, %0"					\
75 				: "=Q" (*p)					\
76 				: "r" ((__u32)v)				\
77 				: "memory");					\
78 		break;								\
79 	case 8:									\
80 		asm volatile ("stlr %1, %0"					\
81 				: "=Q" (*p)					\
82 				: "r" ((__u64)v)				\
83 				: "memory");					\
84 		break;								\
85 	}									\
86 } while (0)
87 
88 #ifdef RSEQ_SKIP_FASTPATH
89 #include "rseq-skip.h"
90 #else /* !RSEQ_SKIP_FASTPATH */
91 
92 #define RSEQ_ASM_TMP_REG32	"w15"
93 #define RSEQ_ASM_TMP_REG	"x15"
94 #define RSEQ_ASM_TMP_REG_2	"x14"
95 
96 #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, start_ip,		\
97 				post_commit_offset, abort_ip)			\
98 	"	.pushsection	__rseq_cs, \"aw\"\n"				\
99 	"	.balign	32\n"							\
100 	__rseq_str(label) ":\n"							\
101 	"	.long	" __rseq_str(version) ", " __rseq_str(flags) "\n"	\
102 	"	.quad	" __rseq_str(start_ip) ", "				\
103 			  __rseq_str(post_commit_offset) ", "			\
104 			  __rseq_str(abort_ip) "\n"				\
105 	"	.popsection\n\t"						\
106 	"	.pushsection __rseq_cs_ptr_array, \"aw\"\n"				\
107 	"	.quad " __rseq_str(label) "b\n"					\
108 	"	.popsection\n"
109 
110 #define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip)	\
111 	__RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip,			\
112 				(post_commit_ip - start_ip), abort_ip)
113 
114 /*
115  * Exit points of a rseq critical section consist of all instructions outside
116  * of the critical section where a critical section can either branch to or
117  * reach through the normal course of its execution. The abort IP and the
118  * post-commit IP are already part of the __rseq_cs section and should not be
119  * explicitly defined as additional exit points. Knowing all exit points is
120  * useful to assist debuggers stepping over the critical section.
121  */
122 #define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip)				\
123 	"	.pushsection __rseq_exit_point_array, \"aw\"\n"			\
124 	"	.quad " __rseq_str(start_ip) ", " __rseq_str(exit_ip) "\n"	\
125 	"	.popsection\n"
126 
127 #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs)			\
128 	RSEQ_INJECT_ASM(1)							\
129 	"	adrp	" RSEQ_ASM_TMP_REG ", " __rseq_str(cs_label) "\n"	\
130 	"	add	" RSEQ_ASM_TMP_REG ", " RSEQ_ASM_TMP_REG		\
131 			", :lo12:" __rseq_str(cs_label) "\n"			\
132 	"	str	" RSEQ_ASM_TMP_REG ", %[" __rseq_str(rseq_cs) "]\n"	\
133 	__rseq_str(label) ":\n"
134 
135 #define RSEQ_ASM_DEFINE_ABORT(label, abort_label)				\
136 	"	b	222f\n"							\
137 	"	.inst 	"	__rseq_str(RSEQ_SIG_CODE) "\n"			\
138 	__rseq_str(label) ":\n"							\
139 	"	b	%l[" __rseq_str(abort_label) "]\n"			\
140 	"222:\n"
141 
142 #define RSEQ_ASM_OP_STORE(value, var)						\
143 	"	str	%[" __rseq_str(value) "], %[" __rseq_str(var) "]\n"
144 
145 #define RSEQ_ASM_OP_STORE_RELEASE(value, var)					\
146 	"	stlr	%[" __rseq_str(value) "], %[" __rseq_str(var) "]\n"
147 
148 #define RSEQ_ASM_OP_FINAL_STORE(value, var, post_commit_label)			\
149 	RSEQ_ASM_OP_STORE(value, var)						\
150 	__rseq_str(post_commit_label) ":\n"
151 
152 #define RSEQ_ASM_OP_FINAL_STORE_RELEASE(value, var, post_commit_label)		\
153 	RSEQ_ASM_OP_STORE_RELEASE(value, var)					\
154 	__rseq_str(post_commit_label) ":\n"
155 
156 #define RSEQ_ASM_OP_CMPEQ(var, expect, label)					\
157 	"	ldr	" RSEQ_ASM_TMP_REG ", %[" __rseq_str(var) "]\n"		\
158 	"	sub	" RSEQ_ASM_TMP_REG ", " RSEQ_ASM_TMP_REG		\
159 			", %[" __rseq_str(expect) "]\n"				\
160 	"	cbnz	" RSEQ_ASM_TMP_REG ", " __rseq_str(label) "\n"
161 
162 #define RSEQ_ASM_OP_CMPEQ32(var, expect, label)					\
163 	"	ldr	" RSEQ_ASM_TMP_REG32 ", %[" __rseq_str(var) "]\n"	\
164 	"	sub	" RSEQ_ASM_TMP_REG32 ", " RSEQ_ASM_TMP_REG32		\
165 			", %w[" __rseq_str(expect) "]\n"			\
166 	"	cbnz	" RSEQ_ASM_TMP_REG32 ", " __rseq_str(label) "\n"
167 
168 #define RSEQ_ASM_OP_CMPNE(var, expect, label)					\
169 	"	ldr	" RSEQ_ASM_TMP_REG ", %[" __rseq_str(var) "]\n"		\
170 	"	sub	" RSEQ_ASM_TMP_REG ", " RSEQ_ASM_TMP_REG		\
171 			", %[" __rseq_str(expect) "]\n"				\
172 	"	cbz	" RSEQ_ASM_TMP_REG ", " __rseq_str(label) "\n"
173 
174 #define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label)			\
175 	RSEQ_INJECT_ASM(2)							\
176 	RSEQ_ASM_OP_CMPEQ32(current_cpu_id, cpu_id, label)
177 
178 #define RSEQ_ASM_OP_R_LOAD(var)							\
179 	"	ldr	" RSEQ_ASM_TMP_REG ", %[" __rseq_str(var) "]\n"
180 
181 #define RSEQ_ASM_OP_R_STORE(var)						\
182 	"	str	" RSEQ_ASM_TMP_REG ", %[" __rseq_str(var) "]\n"
183 
184 #define RSEQ_ASM_OP_R_LOAD_OFF(offset)						\
185 	"	ldr	" RSEQ_ASM_TMP_REG ", [" RSEQ_ASM_TMP_REG		\
186 			", %[" __rseq_str(offset) "]]\n"
187 
188 #define RSEQ_ASM_OP_R_ADD(count)						\
189 	"	add	" RSEQ_ASM_TMP_REG ", " RSEQ_ASM_TMP_REG		\
190 			", %[" __rseq_str(count) "]\n"
191 
192 #define RSEQ_ASM_OP_R_FINAL_STORE(var, post_commit_label)			\
193 	"	str	" RSEQ_ASM_TMP_REG ", %[" __rseq_str(var) "]\n"		\
194 	__rseq_str(post_commit_label) ":\n"
195 
196 #define RSEQ_ASM_OP_R_BAD_MEMCPY(dst, src, len)					\
197 	"	cbz	%[" __rseq_str(len) "], 333f\n"				\
198 	"	mov	" RSEQ_ASM_TMP_REG_2 ", %[" __rseq_str(len) "]\n"	\
199 	"222:	sub	" RSEQ_ASM_TMP_REG_2 ", " RSEQ_ASM_TMP_REG_2 ", #1\n"	\
200 	"	ldrb	" RSEQ_ASM_TMP_REG32 ", [%[" __rseq_str(src) "]"	\
201 			", " RSEQ_ASM_TMP_REG_2 "]\n"				\
202 	"	strb	" RSEQ_ASM_TMP_REG32 ", [%[" __rseq_str(dst) "]"	\
203 			", " RSEQ_ASM_TMP_REG_2 "]\n"				\
204 	"	cbnz	" RSEQ_ASM_TMP_REG_2 ", 222b\n"				\
205 	"333:\n"
206 
207 static inline __attribute__((always_inline))
208 int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
209 {
210 	RSEQ_INJECT_C(9)
211 
212 	__asm__ __volatile__ goto (
213 		RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
214 		RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail])
215 #ifdef RSEQ_COMPARE_TWICE
216 		RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1])
217 		RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2])
218 #endif
219 		RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
220 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
221 		RSEQ_INJECT_ASM(3)
222 		RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
223 		RSEQ_INJECT_ASM(4)
224 #ifdef RSEQ_COMPARE_TWICE
225 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
226 		RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
227 #endif
228 		RSEQ_ASM_OP_FINAL_STORE(newv, v, 3)
229 		RSEQ_INJECT_ASM(5)
230 		RSEQ_ASM_DEFINE_ABORT(4, abort)
231 		: /* gcc asm goto does not allow outputs */
232 		: [cpu_id]		"r" (cpu),
233 		  [current_cpu_id]	"Qo" (__rseq_abi.cpu_id),
234 		  [rseq_cs]		"m" (__rseq_abi.rseq_cs),
235 		  [v]			"Qo" (*v),
236 		  [expect]		"r" (expect),
237 		  [newv]		"r" (newv)
238 		  RSEQ_INJECT_INPUT
239 		: "memory", RSEQ_ASM_TMP_REG
240 		: abort, cmpfail
241 #ifdef RSEQ_COMPARE_TWICE
242 		  , error1, error2
243 #endif
244 	);
245 
246 	return 0;
247 abort:
248 	RSEQ_INJECT_FAILED
249 	return -1;
250 cmpfail:
251 	return 1;
252 #ifdef RSEQ_COMPARE_TWICE
253 error1:
254 	rseq_bug("cpu_id comparison failed");
255 error2:
256 	rseq_bug("expected value comparison failed");
257 #endif
258 }
259 
260 static inline __attribute__((always_inline))
261 int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
262 			       off_t voffp, intptr_t *load, int cpu)
263 {
264 	RSEQ_INJECT_C(9)
265 
266 	__asm__ __volatile__ goto (
267 		RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
268 		RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail])
269 #ifdef RSEQ_COMPARE_TWICE
270 		RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1])
271 		RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2])
272 #endif
273 		RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
274 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
275 		RSEQ_INJECT_ASM(3)
276 		RSEQ_ASM_OP_CMPNE(v, expectnot, %l[cmpfail])
277 		RSEQ_INJECT_ASM(4)
278 #ifdef RSEQ_COMPARE_TWICE
279 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
280 		RSEQ_ASM_OP_CMPNE(v, expectnot, %l[error2])
281 #endif
282 		RSEQ_ASM_OP_R_LOAD(v)
283 		RSEQ_ASM_OP_R_STORE(load)
284 		RSEQ_ASM_OP_R_LOAD_OFF(voffp)
285 		RSEQ_ASM_OP_R_FINAL_STORE(v, 3)
286 		RSEQ_INJECT_ASM(5)
287 		RSEQ_ASM_DEFINE_ABORT(4, abort)
288 		: /* gcc asm goto does not allow outputs */
289 		: [cpu_id]		"r" (cpu),
290 		  [current_cpu_id]	"Qo" (__rseq_abi.cpu_id),
291 		  [rseq_cs]		"m" (__rseq_abi.rseq_cs),
292 		  [v]			"Qo" (*v),
293 		  [expectnot]		"r" (expectnot),
294 		  [load]		"Qo" (*load),
295 		  [voffp]		"r" (voffp)
296 		  RSEQ_INJECT_INPUT
297 		: "memory", RSEQ_ASM_TMP_REG
298 		: abort, cmpfail
299 #ifdef RSEQ_COMPARE_TWICE
300 		  , error1, error2
301 #endif
302 	);
303 	return 0;
304 abort:
305 	RSEQ_INJECT_FAILED
306 	return -1;
307 cmpfail:
308 	return 1;
309 #ifdef RSEQ_COMPARE_TWICE
310 error1:
311 	rseq_bug("cpu_id comparison failed");
312 error2:
313 	rseq_bug("expected value comparison failed");
314 #endif
315 }
316 
317 static inline __attribute__((always_inline))
318 int rseq_addv(intptr_t *v, intptr_t count, int cpu)
319 {
320 	RSEQ_INJECT_C(9)
321 
322 	__asm__ __volatile__ goto (
323 		RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
324 #ifdef RSEQ_COMPARE_TWICE
325 		RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1])
326 #endif
327 		RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
328 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
329 		RSEQ_INJECT_ASM(3)
330 #ifdef RSEQ_COMPARE_TWICE
331 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
332 #endif
333 		RSEQ_ASM_OP_R_LOAD(v)
334 		RSEQ_ASM_OP_R_ADD(count)
335 		RSEQ_ASM_OP_R_FINAL_STORE(v, 3)
336 		RSEQ_INJECT_ASM(4)
337 		RSEQ_ASM_DEFINE_ABORT(4, abort)
338 		: /* gcc asm goto does not allow outputs */
339 		: [cpu_id]		"r" (cpu),
340 		  [current_cpu_id]	"Qo" (__rseq_abi.cpu_id),
341 		  [rseq_cs]		"m" (__rseq_abi.rseq_cs),
342 		  [v]			"Qo" (*v),
343 		  [count]		"r" (count)
344 		  RSEQ_INJECT_INPUT
345 		: "memory", RSEQ_ASM_TMP_REG
346 		: abort
347 #ifdef RSEQ_COMPARE_TWICE
348 		  , error1
349 #endif
350 	);
351 	return 0;
352 abort:
353 	RSEQ_INJECT_FAILED
354 	return -1;
355 #ifdef RSEQ_COMPARE_TWICE
356 error1:
357 	rseq_bug("cpu_id comparison failed");
358 #endif
359 }
360 
361 static inline __attribute__((always_inline))
362 int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
363 				 intptr_t *v2, intptr_t newv2,
364 				 intptr_t newv, int cpu)
365 {
366 	RSEQ_INJECT_C(9)
367 
368 	__asm__ __volatile__ goto (
369 		RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
370 		RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail])
371 #ifdef RSEQ_COMPARE_TWICE
372 		RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1])
373 		RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2])
374 #endif
375 		RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
376 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
377 		RSEQ_INJECT_ASM(3)
378 		RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
379 		RSEQ_INJECT_ASM(4)
380 #ifdef RSEQ_COMPARE_TWICE
381 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
382 		RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
383 #endif
384 		RSEQ_ASM_OP_STORE(newv2, v2)
385 		RSEQ_INJECT_ASM(5)
386 		RSEQ_ASM_OP_FINAL_STORE(newv, v, 3)
387 		RSEQ_INJECT_ASM(6)
388 		RSEQ_ASM_DEFINE_ABORT(4, abort)
389 		: /* gcc asm goto does not allow outputs */
390 		: [cpu_id]		"r" (cpu),
391 		  [current_cpu_id]	"Qo" (__rseq_abi.cpu_id),
392 		  [rseq_cs]		"m" (__rseq_abi.rseq_cs),
393 		  [expect]		"r" (expect),
394 		  [v]			"Qo" (*v),
395 		  [newv]		"r" (newv),
396 		  [v2]			"Qo" (*v2),
397 		  [newv2]		"r" (newv2)
398 		  RSEQ_INJECT_INPUT
399 		: "memory", RSEQ_ASM_TMP_REG
400 		: abort, cmpfail
401 #ifdef RSEQ_COMPARE_TWICE
402 		  , error1, error2
403 #endif
404 	);
405 
406 	return 0;
407 abort:
408 	RSEQ_INJECT_FAILED
409 	return -1;
410 cmpfail:
411 	return 1;
412 #ifdef RSEQ_COMPARE_TWICE
413 error1:
414 	rseq_bug("cpu_id comparison failed");
415 error2:
416 	rseq_bug("expected value comparison failed");
417 #endif
418 }
419 
420 static inline __attribute__((always_inline))
421 int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect,
422 					 intptr_t *v2, intptr_t newv2,
423 					 intptr_t newv, int cpu)
424 {
425 	RSEQ_INJECT_C(9)
426 
427 	__asm__ __volatile__ goto (
428 		RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
429 		RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail])
430 #ifdef RSEQ_COMPARE_TWICE
431 		RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1])
432 		RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2])
433 #endif
434 		RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
435 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
436 		RSEQ_INJECT_ASM(3)
437 		RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
438 		RSEQ_INJECT_ASM(4)
439 #ifdef RSEQ_COMPARE_TWICE
440 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
441 		RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
442 #endif
443 		RSEQ_ASM_OP_STORE(newv2, v2)
444 		RSEQ_INJECT_ASM(5)
445 		RSEQ_ASM_OP_FINAL_STORE_RELEASE(newv, v, 3)
446 		RSEQ_INJECT_ASM(6)
447 		RSEQ_ASM_DEFINE_ABORT(4, abort)
448 		: /* gcc asm goto does not allow outputs */
449 		: [cpu_id]		"r" (cpu),
450 		  [current_cpu_id]	"Qo" (__rseq_abi.cpu_id),
451 		  [rseq_cs]		"m" (__rseq_abi.rseq_cs),
452 		  [expect]		"r" (expect),
453 		  [v]			"Qo" (*v),
454 		  [newv]		"r" (newv),
455 		  [v2]			"Qo" (*v2),
456 		  [newv2]		"r" (newv2)
457 		  RSEQ_INJECT_INPUT
458 		: "memory", RSEQ_ASM_TMP_REG
459 		: abort, cmpfail
460 #ifdef RSEQ_COMPARE_TWICE
461 		  , error1, error2
462 #endif
463 	);
464 
465 	return 0;
466 abort:
467 	RSEQ_INJECT_FAILED
468 	return -1;
469 cmpfail:
470 	return 1;
471 #ifdef RSEQ_COMPARE_TWICE
472 error1:
473 	rseq_bug("cpu_id comparison failed");
474 error2:
475 	rseq_bug("expected value comparison failed");
476 #endif
477 }
478 
479 static inline __attribute__((always_inline))
480 int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
481 			      intptr_t *v2, intptr_t expect2,
482 			      intptr_t newv, int cpu)
483 {
484 	RSEQ_INJECT_C(9)
485 
486 	__asm__ __volatile__ goto (
487 		RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
488 		RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail])
489 #ifdef RSEQ_COMPARE_TWICE
490 		RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1])
491 		RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2])
492 		RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error3])
493 #endif
494 		RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
495 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
496 		RSEQ_INJECT_ASM(3)
497 		RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
498 		RSEQ_INJECT_ASM(4)
499 		RSEQ_ASM_OP_CMPEQ(v2, expect2, %l[cmpfail])
500 		RSEQ_INJECT_ASM(5)
501 #ifdef RSEQ_COMPARE_TWICE
502 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
503 		RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
504 		RSEQ_ASM_OP_CMPEQ(v2, expect2, %l[error3])
505 #endif
506 		RSEQ_ASM_OP_FINAL_STORE(newv, v, 3)
507 		RSEQ_INJECT_ASM(6)
508 		RSEQ_ASM_DEFINE_ABORT(4, abort)
509 		: /* gcc asm goto does not allow outputs */
510 		: [cpu_id]		"r" (cpu),
511 		  [current_cpu_id]	"Qo" (__rseq_abi.cpu_id),
512 		  [rseq_cs]		"m" (__rseq_abi.rseq_cs),
513 		  [v]			"Qo" (*v),
514 		  [expect]		"r" (expect),
515 		  [v2]			"Qo" (*v2),
516 		  [expect2]		"r" (expect2),
517 		  [newv]		"r" (newv)
518 		  RSEQ_INJECT_INPUT
519 		: "memory", RSEQ_ASM_TMP_REG
520 		: abort, cmpfail
521 #ifdef RSEQ_COMPARE_TWICE
522 		  , error1, error2, error3
523 #endif
524 	);
525 
526 	return 0;
527 abort:
528 	RSEQ_INJECT_FAILED
529 	return -1;
530 cmpfail:
531 	return 1;
532 #ifdef RSEQ_COMPARE_TWICE
533 error1:
534 	rseq_bug("cpu_id comparison failed");
535 error2:
536 	rseq_bug("expected value comparison failed");
537 error3:
538 	rseq_bug("2nd expected value comparison failed");
539 #endif
540 }
541 
542 static inline __attribute__((always_inline))
543 int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,
544 				 void *dst, void *src, size_t len,
545 				 intptr_t newv, int cpu)
546 {
547 	RSEQ_INJECT_C(9)
548 
549 	__asm__ __volatile__ goto (
550 		RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
551 		RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail])
552 #ifdef RSEQ_COMPARE_TWICE
553 		RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1])
554 		RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2])
555 #endif
556 		RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
557 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
558 		RSEQ_INJECT_ASM(3)
559 		RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
560 		RSEQ_INJECT_ASM(4)
561 #ifdef RSEQ_COMPARE_TWICE
562 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
563 		RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
564 #endif
565 		RSEQ_ASM_OP_R_BAD_MEMCPY(dst, src, len)
566 		RSEQ_INJECT_ASM(5)
567 		RSEQ_ASM_OP_FINAL_STORE(newv, v, 3)
568 		RSEQ_INJECT_ASM(6)
569 		RSEQ_ASM_DEFINE_ABORT(4, abort)
570 		: /* gcc asm goto does not allow outputs */
571 		: [cpu_id]		"r" (cpu),
572 		  [current_cpu_id]	"Qo" (__rseq_abi.cpu_id),
573 		  [rseq_cs]		"m" (__rseq_abi.rseq_cs),
574 		  [expect]		"r" (expect),
575 		  [v]			"Qo" (*v),
576 		  [newv]		"r" (newv),
577 		  [dst]			"r" (dst),
578 		  [src]			"r" (src),
579 		  [len]			"r" (len)
580 		  RSEQ_INJECT_INPUT
581 		: "memory", RSEQ_ASM_TMP_REG, RSEQ_ASM_TMP_REG_2
582 		: abort, cmpfail
583 #ifdef RSEQ_COMPARE_TWICE
584 		  , error1, error2
585 #endif
586 	);
587 
588 	return 0;
589 abort:
590 	RSEQ_INJECT_FAILED
591 	return -1;
592 cmpfail:
593 	return 1;
594 #ifdef RSEQ_COMPARE_TWICE
595 error1:
596 	rseq_bug("cpu_id comparison failed");
597 error2:
598 	rseq_bug("expected value comparison failed");
599 #endif
600 }
601 
602 static inline __attribute__((always_inline))
603 int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect,
604 					 void *dst, void *src, size_t len,
605 					 intptr_t newv, int cpu)
606 {
607 	RSEQ_INJECT_C(9)
608 
609 	__asm__ __volatile__ goto (
610 		RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
611 		RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail])
612 #ifdef RSEQ_COMPARE_TWICE
613 		RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1])
614 		RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2])
615 #endif
616 		RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
617 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
618 		RSEQ_INJECT_ASM(3)
619 		RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
620 		RSEQ_INJECT_ASM(4)
621 #ifdef RSEQ_COMPARE_TWICE
622 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
623 		RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
624 #endif
625 		RSEQ_ASM_OP_R_BAD_MEMCPY(dst, src, len)
626 		RSEQ_INJECT_ASM(5)
627 		RSEQ_ASM_OP_FINAL_STORE_RELEASE(newv, v, 3)
628 		RSEQ_INJECT_ASM(6)
629 		RSEQ_ASM_DEFINE_ABORT(4, abort)
630 		: /* gcc asm goto does not allow outputs */
631 		: [cpu_id]		"r" (cpu),
632 		  [current_cpu_id]	"Qo" (__rseq_abi.cpu_id),
633 		  [rseq_cs]		"m" (__rseq_abi.rseq_cs),
634 		  [expect]		"r" (expect),
635 		  [v]			"Qo" (*v),
636 		  [newv]		"r" (newv),
637 		  [dst]			"r" (dst),
638 		  [src]			"r" (src),
639 		  [len]			"r" (len)
640 		  RSEQ_INJECT_INPUT
641 		: "memory", RSEQ_ASM_TMP_REG, RSEQ_ASM_TMP_REG_2
642 		: abort, cmpfail
643 #ifdef RSEQ_COMPARE_TWICE
644 		  , error1, error2
645 #endif
646 	);
647 
648 	return 0;
649 abort:
650 	RSEQ_INJECT_FAILED
651 	return -1;
652 cmpfail:
653 	return 1;
654 #ifdef RSEQ_COMPARE_TWICE
655 error1:
656 	rseq_bug("cpu_id comparison failed");
657 error2:
658 	rseq_bug("expected value comparison failed");
659 #endif
660 }
661 
662 #endif /* !RSEQ_SKIP_FASTPATH */
663