1 /* SPDX-License-Identifier: LGPL-2.1 OR MIT */
2
3 /*
4 * RSEQ_SIG uses the trap4 instruction. As Linux does not make use of the
5 * access-register mode nor the linkage stack this instruction will always
6 * cause a special-operation exception (the trap-enabled bit in the DUCT
7 * is and will stay 0). The instruction pattern is
8 * b2 ff 0f ff trap4 4095(%r0)
9 */
10 #define RSEQ_SIG 0xB2FF0FFF
11
12 #define rseq_smp_mb() __asm__ __volatile__ ("bcr 15,0" ::: "memory")
13 #define rseq_smp_rmb() rseq_smp_mb()
14 #define rseq_smp_wmb() rseq_smp_mb()
15
16 #define rseq_smp_load_acquire(p) \
17 __extension__ ({ \
18 __typeof(*p) ____p1 = RSEQ_READ_ONCE(*p); \
19 rseq_barrier(); \
20 ____p1; \
21 })
22
23 #define rseq_smp_acquire__after_ctrl_dep() rseq_smp_rmb()
24
25 #define rseq_smp_store_release(p, v) \
26 do { \
27 rseq_barrier(); \
28 RSEQ_WRITE_ONCE(*p, v); \
29 } while (0)
30
31 #ifdef RSEQ_SKIP_FASTPATH
32 #include "rseq-skip.h"
33 #else /* !RSEQ_SKIP_FASTPATH */
34
35 #ifdef __s390x__
36
37 #define LONG_L "lg"
38 #define LONG_S "stg"
39 #define LONG_LT_R "ltgr"
40 #define LONG_CMP "cg"
41 #define LONG_CMP_R "cgr"
42 #define LONG_ADDI "aghi"
43 #define LONG_ADD_R "agr"
44
45 #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \
46 start_ip, post_commit_offset, abort_ip) \
47 ".pushsection __rseq_cs, \"aw\"\n\t" \
48 ".balign 32\n\t" \
49 __rseq_str(label) ":\n\t" \
50 ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
51 ".quad " __rseq_str(start_ip) ", " __rseq_str(post_commit_offset) ", " __rseq_str(abort_ip) "\n\t" \
52 ".popsection\n\t" \
53 ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t" \
54 ".quad " __rseq_str(label) "b\n\t" \
55 ".popsection\n\t"
56
57 /*
58 * Exit points of a rseq critical section consist of all instructions outside
59 * of the critical section where a critical section can either branch to or
60 * reach through the normal course of its execution. The abort IP and the
61 * post-commit IP are already part of the __rseq_cs section and should not be
62 * explicitly defined as additional exit points. Knowing all exit points is
63 * useful to assist debuggers stepping over the critical section.
64 */
65 #define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \
66 ".pushsection __rseq_exit_point_array, \"aw\"\n\t" \
67 ".quad " __rseq_str(start_ip) ", " __rseq_str(exit_ip) "\n\t" \
68 ".popsection\n\t"
69
70 #elif __s390__
71
72 #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \
73 start_ip, post_commit_offset, abort_ip) \
74 ".pushsection __rseq_cs, \"aw\"\n\t" \
75 ".balign 32\n\t" \
76 __rseq_str(label) ":\n\t" \
77 ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
78 ".long 0x0, " __rseq_str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) "\n\t" \
79 ".popsection\n\t" \
80 ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t" \
81 ".long 0x0, " __rseq_str(label) "b\n\t" \
82 ".popsection\n\t"
83
84 /*
85 * Exit points of a rseq critical section consist of all instructions outside
86 * of the critical section where a critical section can either branch to or
87 * reach through the normal course of its execution. The abort IP and the
88 * post-commit IP are already part of the __rseq_cs section and should not be
89 * explicitly defined as additional exit points. Knowing all exit points is
90 * useful to assist debuggers stepping over the critical section.
91 */
92 #define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \
93 ".pushsection __rseq_exit_point_array, \"aw\"\n\t" \
94 ".long 0x0, " __rseq_str(start_ip) ", 0x0, " __rseq_str(exit_ip) "\n\t" \
95 ".popsection\n\t"
96
97 #define LONG_L "l"
98 #define LONG_S "st"
99 #define LONG_LT_R "ltr"
100 #define LONG_CMP "c"
101 #define LONG_CMP_R "cr"
102 #define LONG_ADDI "ahi"
103 #define LONG_ADD_R "ar"
104
105 #endif
106
107 #define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \
108 __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip, \
109 (post_commit_ip - start_ip), abort_ip)
110
111 #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \
112 RSEQ_INJECT_ASM(1) \
113 "larl %%r0, " __rseq_str(cs_label) "\n\t" \
114 LONG_S " %%r0, %[" __rseq_str(rseq_cs) "]\n\t" \
115 __rseq_str(label) ":\n\t"
116
117 #define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label) \
118 RSEQ_INJECT_ASM(2) \
119 "c %[" __rseq_str(cpu_id) "], %[" __rseq_str(current_cpu_id) "]\n\t" \
120 "jnz " __rseq_str(label) "\n\t"
121
122 #define RSEQ_ASM_DEFINE_ABORT(label, teardown, abort_label) \
123 ".pushsection __rseq_failure, \"ax\"\n\t" \
124 ".long " __rseq_str(RSEQ_SIG) "\n\t" \
125 __rseq_str(label) ":\n\t" \
126 teardown \
127 "jg %l[" __rseq_str(abort_label) "]\n\t" \
128 ".popsection\n\t"
129
130 #define RSEQ_ASM_DEFINE_CMPFAIL(label, teardown, cmpfail_label) \
131 ".pushsection __rseq_failure, \"ax\"\n\t" \
132 __rseq_str(label) ":\n\t" \
133 teardown \
134 "jg %l[" __rseq_str(cmpfail_label) "]\n\t" \
135 ".popsection\n\t"
136
137 static inline __attribute__((always_inline))
rseq_cmpeqv_storev(intptr_t * v,intptr_t expect,intptr_t newv,int cpu)138 int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
139 {
140 RSEQ_INJECT_C(9)
141
142 __asm__ __volatile__ goto (
143 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
144 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
145 #ifdef RSEQ_COMPARE_TWICE
146 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
147 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
148 #endif
149 /* Start rseq by storing table entry pointer into rseq_cs. */
150 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
151 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
152 RSEQ_INJECT_ASM(3)
153 LONG_CMP " %[expect], %[v]\n\t"
154 "jnz %l[cmpfail]\n\t"
155 RSEQ_INJECT_ASM(4)
156 #ifdef RSEQ_COMPARE_TWICE
157 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
158 LONG_CMP " %[expect], %[v]\n\t"
159 "jnz %l[error2]\n\t"
160 #endif
161 /* final store */
162 LONG_S " %[newv], %[v]\n\t"
163 "2:\n\t"
164 RSEQ_INJECT_ASM(5)
165 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
166 : /* gcc asm goto does not allow outputs */
167 : [cpu_id] "r" (cpu),
168 [current_cpu_id] "m" (__rseq_abi.cpu_id),
169 [rseq_cs] "m" (__rseq_abi.rseq_cs),
170 [v] "m" (*v),
171 [expect] "r" (expect),
172 [newv] "r" (newv)
173 RSEQ_INJECT_INPUT
174 : "memory", "cc", "r0"
175 RSEQ_INJECT_CLOBBER
176 : abort, cmpfail
177 #ifdef RSEQ_COMPARE_TWICE
178 , error1, error2
179 #endif
180 );
181 return 0;
182 abort:
183 RSEQ_INJECT_FAILED
184 return -1;
185 cmpfail:
186 return 1;
187 #ifdef RSEQ_COMPARE_TWICE
188 error1:
189 rseq_bug("cpu_id comparison failed");
190 error2:
191 rseq_bug("expected value comparison failed");
192 #endif
193 }
194
195 /*
196 * Compare @v against @expectnot. When it does _not_ match, load @v
197 * into @load, and store the content of *@v + voffp into @v.
198 */
199 static inline __attribute__((always_inline))
rseq_cmpnev_storeoffp_load(intptr_t * v,intptr_t expectnot,off_t voffp,intptr_t * load,int cpu)200 int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
201 off_t voffp, intptr_t *load, int cpu)
202 {
203 RSEQ_INJECT_C(9)
204
205 __asm__ __volatile__ goto (
206 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
207 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
208 #ifdef RSEQ_COMPARE_TWICE
209 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
210 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
211 #endif
212 /* Start rseq by storing table entry pointer into rseq_cs. */
213 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
214 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
215 RSEQ_INJECT_ASM(3)
216 LONG_L " %%r1, %[v]\n\t"
217 LONG_CMP_R " %%r1, %[expectnot]\n\t"
218 "je %l[cmpfail]\n\t"
219 RSEQ_INJECT_ASM(4)
220 #ifdef RSEQ_COMPARE_TWICE
221 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
222 LONG_L " %%r1, %[v]\n\t"
223 LONG_CMP_R " %%r1, %[expectnot]\n\t"
224 "je %l[error2]\n\t"
225 #endif
226 LONG_S " %%r1, %[load]\n\t"
227 LONG_ADD_R " %%r1, %[voffp]\n\t"
228 LONG_L " %%r1, 0(%%r1)\n\t"
229 /* final store */
230 LONG_S " %%r1, %[v]\n\t"
231 "2:\n\t"
232 RSEQ_INJECT_ASM(5)
233 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
234 : /* gcc asm goto does not allow outputs */
235 : [cpu_id] "r" (cpu),
236 [current_cpu_id] "m" (__rseq_abi.cpu_id),
237 [rseq_cs] "m" (__rseq_abi.rseq_cs),
238 /* final store input */
239 [v] "m" (*v),
240 [expectnot] "r" (expectnot),
241 [voffp] "r" (voffp),
242 [load] "m" (*load)
243 RSEQ_INJECT_INPUT
244 : "memory", "cc", "r0", "r1"
245 RSEQ_INJECT_CLOBBER
246 : abort, cmpfail
247 #ifdef RSEQ_COMPARE_TWICE
248 , error1, error2
249 #endif
250 );
251 return 0;
252 abort:
253 RSEQ_INJECT_FAILED
254 return -1;
255 cmpfail:
256 return 1;
257 #ifdef RSEQ_COMPARE_TWICE
258 error1:
259 rseq_bug("cpu_id comparison failed");
260 error2:
261 rseq_bug("expected value comparison failed");
262 #endif
263 }
264
265 static inline __attribute__((always_inline))
rseq_addv(intptr_t * v,intptr_t count,int cpu)266 int rseq_addv(intptr_t *v, intptr_t count, int cpu)
267 {
268 RSEQ_INJECT_C(9)
269
270 __asm__ __volatile__ goto (
271 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
272 #ifdef RSEQ_COMPARE_TWICE
273 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
274 #endif
275 /* Start rseq by storing table entry pointer into rseq_cs. */
276 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
277 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
278 RSEQ_INJECT_ASM(3)
279 #ifdef RSEQ_COMPARE_TWICE
280 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
281 #endif
282 LONG_L " %%r0, %[v]\n\t"
283 LONG_ADD_R " %%r0, %[count]\n\t"
284 /* final store */
285 LONG_S " %%r0, %[v]\n\t"
286 "2:\n\t"
287 RSEQ_INJECT_ASM(4)
288 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
289 : /* gcc asm goto does not allow outputs */
290 : [cpu_id] "r" (cpu),
291 [current_cpu_id] "m" (__rseq_abi.cpu_id),
292 [rseq_cs] "m" (__rseq_abi.rseq_cs),
293 /* final store input */
294 [v] "m" (*v),
295 [count] "r" (count)
296 RSEQ_INJECT_INPUT
297 : "memory", "cc", "r0"
298 RSEQ_INJECT_CLOBBER
299 : abort
300 #ifdef RSEQ_COMPARE_TWICE
301 , error1
302 #endif
303 );
304 return 0;
305 abort:
306 RSEQ_INJECT_FAILED
307 return -1;
308 #ifdef RSEQ_COMPARE_TWICE
309 error1:
310 rseq_bug("cpu_id comparison failed");
311 #endif
312 }
313
314 static inline __attribute__((always_inline))
rseq_cmpeqv_trystorev_storev(intptr_t * v,intptr_t expect,intptr_t * v2,intptr_t newv2,intptr_t newv,int cpu)315 int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
316 intptr_t *v2, intptr_t newv2,
317 intptr_t newv, int cpu)
318 {
319 RSEQ_INJECT_C(9)
320
321 __asm__ __volatile__ goto (
322 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
323 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
324 #ifdef RSEQ_COMPARE_TWICE
325 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
326 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
327 #endif
328 /* Start rseq by storing table entry pointer into rseq_cs. */
329 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
330 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
331 RSEQ_INJECT_ASM(3)
332 LONG_CMP " %[expect], %[v]\n\t"
333 "jnz %l[cmpfail]\n\t"
334 RSEQ_INJECT_ASM(4)
335 #ifdef RSEQ_COMPARE_TWICE
336 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
337 LONG_CMP " %[expect], %[v]\n\t"
338 "jnz %l[error2]\n\t"
339 #endif
340 /* try store */
341 LONG_S " %[newv2], %[v2]\n\t"
342 RSEQ_INJECT_ASM(5)
343 /* final store */
344 LONG_S " %[newv], %[v]\n\t"
345 "2:\n\t"
346 RSEQ_INJECT_ASM(6)
347 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
348 : /* gcc asm goto does not allow outputs */
349 : [cpu_id] "r" (cpu),
350 [current_cpu_id] "m" (__rseq_abi.cpu_id),
351 [rseq_cs] "m" (__rseq_abi.rseq_cs),
352 /* try store input */
353 [v2] "m" (*v2),
354 [newv2] "r" (newv2),
355 /* final store input */
356 [v] "m" (*v),
357 [expect] "r" (expect),
358 [newv] "r" (newv)
359 RSEQ_INJECT_INPUT
360 : "memory", "cc", "r0"
361 RSEQ_INJECT_CLOBBER
362 : abort, cmpfail
363 #ifdef RSEQ_COMPARE_TWICE
364 , error1, error2
365 #endif
366 );
367 return 0;
368 abort:
369 RSEQ_INJECT_FAILED
370 return -1;
371 cmpfail:
372 return 1;
373 #ifdef RSEQ_COMPARE_TWICE
374 error1:
375 rseq_bug("cpu_id comparison failed");
376 error2:
377 rseq_bug("expected value comparison failed");
378 #endif
379 }
380
381 /* s390 is TSO. */
382 static inline __attribute__((always_inline))
rseq_cmpeqv_trystorev_storev_release(intptr_t * v,intptr_t expect,intptr_t * v2,intptr_t newv2,intptr_t newv,int cpu)383 int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect,
384 intptr_t *v2, intptr_t newv2,
385 intptr_t newv, int cpu)
386 {
387 return rseq_cmpeqv_trystorev_storev(v, expect, v2, newv2, newv, cpu);
388 }
389
390 static inline __attribute__((always_inline))
rseq_cmpeqv_cmpeqv_storev(intptr_t * v,intptr_t expect,intptr_t * v2,intptr_t expect2,intptr_t newv,int cpu)391 int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
392 intptr_t *v2, intptr_t expect2,
393 intptr_t newv, int cpu)
394 {
395 RSEQ_INJECT_C(9)
396
397 __asm__ __volatile__ goto (
398 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
399 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
400 #ifdef RSEQ_COMPARE_TWICE
401 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
402 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
403 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3])
404 #endif
405 /* Start rseq by storing table entry pointer into rseq_cs. */
406 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
407 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
408 RSEQ_INJECT_ASM(3)
409 LONG_CMP " %[expect], %[v]\n\t"
410 "jnz %l[cmpfail]\n\t"
411 RSEQ_INJECT_ASM(4)
412 LONG_CMP " %[expect2], %[v2]\n\t"
413 "jnz %l[cmpfail]\n\t"
414 RSEQ_INJECT_ASM(5)
415 #ifdef RSEQ_COMPARE_TWICE
416 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
417 LONG_CMP " %[expect], %[v]\n\t"
418 "jnz %l[error2]\n\t"
419 LONG_CMP " %[expect2], %[v2]\n\t"
420 "jnz %l[error3]\n\t"
421 #endif
422 /* final store */
423 LONG_S " %[newv], %[v]\n\t"
424 "2:\n\t"
425 RSEQ_INJECT_ASM(6)
426 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
427 : /* gcc asm goto does not allow outputs */
428 : [cpu_id] "r" (cpu),
429 [current_cpu_id] "m" (__rseq_abi.cpu_id),
430 [rseq_cs] "m" (__rseq_abi.rseq_cs),
431 /* cmp2 input */
432 [v2] "m" (*v2),
433 [expect2] "r" (expect2),
434 /* final store input */
435 [v] "m" (*v),
436 [expect] "r" (expect),
437 [newv] "r" (newv)
438 RSEQ_INJECT_INPUT
439 : "memory", "cc", "r0"
440 RSEQ_INJECT_CLOBBER
441 : abort, cmpfail
442 #ifdef RSEQ_COMPARE_TWICE
443 , error1, error2, error3
444 #endif
445 );
446 return 0;
447 abort:
448 RSEQ_INJECT_FAILED
449 return -1;
450 cmpfail:
451 return 1;
452 #ifdef RSEQ_COMPARE_TWICE
453 error1:
454 rseq_bug("cpu_id comparison failed");
455 error2:
456 rseq_bug("1st expected value comparison failed");
457 error3:
458 rseq_bug("2nd expected value comparison failed");
459 #endif
460 }
461
462 static inline __attribute__((always_inline))
rseq_cmpeqv_trymemcpy_storev(intptr_t * v,intptr_t expect,void * dst,void * src,size_t len,intptr_t newv,int cpu)463 int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,
464 void *dst, void *src, size_t len,
465 intptr_t newv, int cpu)
466 {
467 uint64_t rseq_scratch[3];
468
469 RSEQ_INJECT_C(9)
470
471 __asm__ __volatile__ goto (
472 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
473 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
474 #ifdef RSEQ_COMPARE_TWICE
475 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
476 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
477 #endif
478 LONG_S " %[src], %[rseq_scratch0]\n\t"
479 LONG_S " %[dst], %[rseq_scratch1]\n\t"
480 LONG_S " %[len], %[rseq_scratch2]\n\t"
481 /* Start rseq by storing table entry pointer into rseq_cs. */
482 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
483 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
484 RSEQ_INJECT_ASM(3)
485 LONG_CMP " %[expect], %[v]\n\t"
486 "jnz 5f\n\t"
487 RSEQ_INJECT_ASM(4)
488 #ifdef RSEQ_COMPARE_TWICE
489 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 6f)
490 LONG_CMP " %[expect], %[v]\n\t"
491 "jnz 7f\n\t"
492 #endif
493 /* try memcpy */
494 LONG_LT_R " %[len], %[len]\n\t"
495 "jz 333f\n\t"
496 "222:\n\t"
497 "ic %%r0,0(%[src])\n\t"
498 "stc %%r0,0(%[dst])\n\t"
499 LONG_ADDI " %[src], 1\n\t"
500 LONG_ADDI " %[dst], 1\n\t"
501 LONG_ADDI " %[len], -1\n\t"
502 "jnz 222b\n\t"
503 "333:\n\t"
504 RSEQ_INJECT_ASM(5)
505 /* final store */
506 LONG_S " %[newv], %[v]\n\t"
507 "2:\n\t"
508 RSEQ_INJECT_ASM(6)
509 /* teardown */
510 LONG_L " %[len], %[rseq_scratch2]\n\t"
511 LONG_L " %[dst], %[rseq_scratch1]\n\t"
512 LONG_L " %[src], %[rseq_scratch0]\n\t"
513 RSEQ_ASM_DEFINE_ABORT(4,
514 LONG_L " %[len], %[rseq_scratch2]\n\t"
515 LONG_L " %[dst], %[rseq_scratch1]\n\t"
516 LONG_L " %[src], %[rseq_scratch0]\n\t",
517 abort)
518 RSEQ_ASM_DEFINE_CMPFAIL(5,
519 LONG_L " %[len], %[rseq_scratch2]\n\t"
520 LONG_L " %[dst], %[rseq_scratch1]\n\t"
521 LONG_L " %[src], %[rseq_scratch0]\n\t",
522 cmpfail)
523 #ifdef RSEQ_COMPARE_TWICE
524 RSEQ_ASM_DEFINE_CMPFAIL(6,
525 LONG_L " %[len], %[rseq_scratch2]\n\t"
526 LONG_L " %[dst], %[rseq_scratch1]\n\t"
527 LONG_L " %[src], %[rseq_scratch0]\n\t",
528 error1)
529 RSEQ_ASM_DEFINE_CMPFAIL(7,
530 LONG_L " %[len], %[rseq_scratch2]\n\t"
531 LONG_L " %[dst], %[rseq_scratch1]\n\t"
532 LONG_L " %[src], %[rseq_scratch0]\n\t",
533 error2)
534 #endif
535 : /* gcc asm goto does not allow outputs */
536 : [cpu_id] "r" (cpu),
537 [current_cpu_id] "m" (__rseq_abi.cpu_id),
538 [rseq_cs] "m" (__rseq_abi.rseq_cs),
539 /* final store input */
540 [v] "m" (*v),
541 [expect] "r" (expect),
542 [newv] "r" (newv),
543 /* try memcpy input */
544 [dst] "r" (dst),
545 [src] "r" (src),
546 [len] "r" (len),
547 [rseq_scratch0] "m" (rseq_scratch[0]),
548 [rseq_scratch1] "m" (rseq_scratch[1]),
549 [rseq_scratch2] "m" (rseq_scratch[2])
550 RSEQ_INJECT_INPUT
551 : "memory", "cc", "r0"
552 RSEQ_INJECT_CLOBBER
553 : abort, cmpfail
554 #ifdef RSEQ_COMPARE_TWICE
555 , error1, error2
556 #endif
557 );
558 return 0;
559 abort:
560 RSEQ_INJECT_FAILED
561 return -1;
562 cmpfail:
563 return 1;
564 #ifdef RSEQ_COMPARE_TWICE
565 error1:
566 rseq_bug("cpu_id comparison failed");
567 error2:
568 rseq_bug("expected value comparison failed");
569 #endif
570 }
571
572 /* s390 is TSO. */
573 static inline __attribute__((always_inline))
rseq_cmpeqv_trymemcpy_storev_release(intptr_t * v,intptr_t expect,void * dst,void * src,size_t len,intptr_t newv,int cpu)574 int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect,
575 void *dst, void *src, size_t len,
576 intptr_t newv, int cpu)
577 {
578 return rseq_cmpeqv_trymemcpy_storev(v, expect, dst, src, len,
579 newv, cpu);
580 }
581 #endif /* !RSEQ_SKIP_FASTPATH */
582