1 /* SPDX-License-Identifier: LGPL-2.1 OR MIT */
2 /*
3 * rseq-x86.h
4 *
5 * (C) Copyright 2016-2018 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
6 */
7
8 #include <stdint.h>
9
10 /*
11 * RSEQ_SIG is used with the following reserved undefined instructions, which
12 * trap in user-space:
13 *
14 * x86-32: 0f b9 3d 53 30 05 53 ud1 0x53053053,%edi
15 * x86-64: 0f b9 3d 53 30 05 53 ud1 0x53053053(%rip),%edi
16 */
17 #define RSEQ_SIG 0x53053053
18
19 /*
20 * Due to a compiler optimization bug in gcc-8 with asm goto and TLS asm input
21 * operands, we cannot use "m" input operands, and rather pass the __rseq_abi
22 * address through a "r" input operand.
23 */
24
25 /* Offset of cpu_id and rseq_cs fields in struct rseq. */
26 #define RSEQ_CPU_ID_OFFSET 4
27 #define RSEQ_CS_OFFSET 8
28
29 #ifdef __x86_64__
30
31 #define rseq_smp_mb() \
32 __asm__ __volatile__ ("lock; addl $0,-128(%%rsp)" ::: "memory", "cc")
33 #define rseq_smp_rmb() rseq_barrier()
34 #define rseq_smp_wmb() rseq_barrier()
35
36 #define rseq_smp_load_acquire(p) \
37 __extension__ ({ \
38 __typeof(*p) ____p1 = RSEQ_READ_ONCE(*p); \
39 rseq_barrier(); \
40 ____p1; \
41 })
42
43 #define rseq_smp_acquire__after_ctrl_dep() rseq_smp_rmb()
44
45 #define rseq_smp_store_release(p, v) \
46 do { \
47 rseq_barrier(); \
48 RSEQ_WRITE_ONCE(*p, v); \
49 } while (0)
50
51 #ifdef RSEQ_SKIP_FASTPATH
52 #include "rseq-skip.h"
53 #else /* !RSEQ_SKIP_FASTPATH */
54
55 #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \
56 start_ip, post_commit_offset, abort_ip) \
57 ".pushsection __rseq_cs, \"aw\"\n\t" \
58 ".balign 32\n\t" \
59 __rseq_str(label) ":\n\t" \
60 ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
61 ".quad " __rseq_str(start_ip) ", " __rseq_str(post_commit_offset) ", " __rseq_str(abort_ip) "\n\t" \
62 ".popsection\n\t" \
63 ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t" \
64 ".quad " __rseq_str(label) "b\n\t" \
65 ".popsection\n\t"
66
67
68 #define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \
69 __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip, \
70 (post_commit_ip - start_ip), abort_ip)
71
72 /*
73 * Exit points of a rseq critical section consist of all instructions outside
74 * of the critical section where a critical section can either branch to or
75 * reach through the normal course of its execution. The abort IP and the
76 * post-commit IP are already part of the __rseq_cs section and should not be
77 * explicitly defined as additional exit points. Knowing all exit points is
78 * useful to assist debuggers stepping over the critical section.
79 */
80 #define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \
81 ".pushsection __rseq_exit_point_array, \"aw\"\n\t" \
82 ".quad " __rseq_str(start_ip) ", " __rseq_str(exit_ip) "\n\t" \
83 ".popsection\n\t"
84
85 #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \
86 RSEQ_INJECT_ASM(1) \
87 "leaq " __rseq_str(cs_label) "(%%rip), %%rax\n\t" \
88 "movq %%rax, " __rseq_str(rseq_cs) "\n\t" \
89 __rseq_str(label) ":\n\t"
90
91 #define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label) \
92 RSEQ_INJECT_ASM(2) \
93 "cmpl %[" __rseq_str(cpu_id) "], " __rseq_str(current_cpu_id) "\n\t" \
94 "jnz " __rseq_str(label) "\n\t"
95
96 #define RSEQ_ASM_DEFINE_ABORT(label, teardown, abort_label) \
97 ".pushsection __rseq_failure, \"ax\"\n\t" \
98 /* Disassembler-friendly signature: ud1 <sig>(%rip),%edi. */ \
99 ".byte 0x0f, 0xb9, 0x3d\n\t" \
100 ".long " __rseq_str(RSEQ_SIG) "\n\t" \
101 __rseq_str(label) ":\n\t" \
102 teardown \
103 "jmp %l[" __rseq_str(abort_label) "]\n\t" \
104 ".popsection\n\t"
105
106 #define RSEQ_ASM_DEFINE_CMPFAIL(label, teardown, cmpfail_label) \
107 ".pushsection __rseq_failure, \"ax\"\n\t" \
108 __rseq_str(label) ":\n\t" \
109 teardown \
110 "jmp %l[" __rseq_str(cmpfail_label) "]\n\t" \
111 ".popsection\n\t"
112
113 static inline __attribute__((always_inline))
rseq_cmpeqv_storev(intptr_t * v,intptr_t expect,intptr_t newv,int cpu)114 int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
115 {
116 RSEQ_INJECT_C(9)
117
118 __asm__ __volatile__ goto (
119 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
120 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
121 #ifdef RSEQ_COMPARE_TWICE
122 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
123 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
124 #endif
125 /* Start rseq by storing table entry pointer into rseq_cs. */
126 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
127 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
128 RSEQ_INJECT_ASM(3)
129 "cmpq %[v], %[expect]\n\t"
130 "jnz %l[cmpfail]\n\t"
131 RSEQ_INJECT_ASM(4)
132 #ifdef RSEQ_COMPARE_TWICE
133 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
134 "cmpq %[v], %[expect]\n\t"
135 "jnz %l[error2]\n\t"
136 #endif
137 /* final store */
138 "movq %[newv], %[v]\n\t"
139 "2:\n\t"
140 RSEQ_INJECT_ASM(5)
141 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
142 : /* gcc asm goto does not allow outputs */
143 : [cpu_id] "r" (cpu),
144 [rseq_abi] "r" (&__rseq_abi),
145 [v] "m" (*v),
146 [expect] "r" (expect),
147 [newv] "r" (newv)
148 : "memory", "cc", "rax"
149 RSEQ_INJECT_CLOBBER
150 : abort, cmpfail
151 #ifdef RSEQ_COMPARE_TWICE
152 , error1, error2
153 #endif
154 );
155 return 0;
156 abort:
157 RSEQ_INJECT_FAILED
158 return -1;
159 cmpfail:
160 return 1;
161 #ifdef RSEQ_COMPARE_TWICE
162 error1:
163 rseq_bug("cpu_id comparison failed");
164 error2:
165 rseq_bug("expected value comparison failed");
166 #endif
167 }
168
169 /*
170 * Compare @v against @expectnot. When it does _not_ match, load @v
171 * into @load, and store the content of *@v + voffp into @v.
172 */
173 static inline __attribute__((always_inline))
rseq_cmpnev_storeoffp_load(intptr_t * v,intptr_t expectnot,off_t voffp,intptr_t * load,int cpu)174 int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
175 off_t voffp, intptr_t *load, int cpu)
176 {
177 RSEQ_INJECT_C(9)
178
179 __asm__ __volatile__ goto (
180 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
181 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
182 #ifdef RSEQ_COMPARE_TWICE
183 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
184 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
185 #endif
186 /* Start rseq by storing table entry pointer into rseq_cs. */
187 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
188 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
189 RSEQ_INJECT_ASM(3)
190 "movq %[v], %%rbx\n\t"
191 "cmpq %%rbx, %[expectnot]\n\t"
192 "je %l[cmpfail]\n\t"
193 RSEQ_INJECT_ASM(4)
194 #ifdef RSEQ_COMPARE_TWICE
195 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
196 "movq %[v], %%rbx\n\t"
197 "cmpq %%rbx, %[expectnot]\n\t"
198 "je %l[error2]\n\t"
199 #endif
200 "movq %%rbx, %[load]\n\t"
201 "addq %[voffp], %%rbx\n\t"
202 "movq (%%rbx), %%rbx\n\t"
203 /* final store */
204 "movq %%rbx, %[v]\n\t"
205 "2:\n\t"
206 RSEQ_INJECT_ASM(5)
207 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
208 : /* gcc asm goto does not allow outputs */
209 : [cpu_id] "r" (cpu),
210 [rseq_abi] "r" (&__rseq_abi),
211 /* final store input */
212 [v] "m" (*v),
213 [expectnot] "r" (expectnot),
214 [voffp] "er" (voffp),
215 [load] "m" (*load)
216 : "memory", "cc", "rax", "rbx"
217 RSEQ_INJECT_CLOBBER
218 : abort, cmpfail
219 #ifdef RSEQ_COMPARE_TWICE
220 , error1, error2
221 #endif
222 );
223 return 0;
224 abort:
225 RSEQ_INJECT_FAILED
226 return -1;
227 cmpfail:
228 return 1;
229 #ifdef RSEQ_COMPARE_TWICE
230 error1:
231 rseq_bug("cpu_id comparison failed");
232 error2:
233 rseq_bug("expected value comparison failed");
234 #endif
235 }
236
237 static inline __attribute__((always_inline))
rseq_addv(intptr_t * v,intptr_t count,int cpu)238 int rseq_addv(intptr_t *v, intptr_t count, int cpu)
239 {
240 RSEQ_INJECT_C(9)
241
242 __asm__ __volatile__ goto (
243 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
244 #ifdef RSEQ_COMPARE_TWICE
245 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
246 #endif
247 /* Start rseq by storing table entry pointer into rseq_cs. */
248 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
249 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
250 RSEQ_INJECT_ASM(3)
251 #ifdef RSEQ_COMPARE_TWICE
252 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
253 #endif
254 /* final store */
255 "addq %[count], %[v]\n\t"
256 "2:\n\t"
257 RSEQ_INJECT_ASM(4)
258 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
259 : /* gcc asm goto does not allow outputs */
260 : [cpu_id] "r" (cpu),
261 [rseq_abi] "r" (&__rseq_abi),
262 /* final store input */
263 [v] "m" (*v),
264 [count] "er" (count)
265 : "memory", "cc", "rax"
266 RSEQ_INJECT_CLOBBER
267 : abort
268 #ifdef RSEQ_COMPARE_TWICE
269 , error1
270 #endif
271 );
272 return 0;
273 abort:
274 RSEQ_INJECT_FAILED
275 return -1;
276 #ifdef RSEQ_COMPARE_TWICE
277 error1:
278 rseq_bug("cpu_id comparison failed");
279 #endif
280 }
281
282 #define RSEQ_ARCH_HAS_OFFSET_DEREF_ADDV
283
284 /*
285 * pval = *(ptr+off)
286 * *pval += inc;
287 */
288 static inline __attribute__((always_inline))
rseq_offset_deref_addv(intptr_t * ptr,off_t off,intptr_t inc,int cpu)289 int rseq_offset_deref_addv(intptr_t *ptr, off_t off, intptr_t inc, int cpu)
290 {
291 RSEQ_INJECT_C(9)
292
293 __asm__ __volatile__ goto (
294 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
295 #ifdef RSEQ_COMPARE_TWICE
296 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
297 #endif
298 /* Start rseq by storing table entry pointer into rseq_cs. */
299 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
300 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
301 RSEQ_INJECT_ASM(3)
302 #ifdef RSEQ_COMPARE_TWICE
303 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
304 #endif
305 /* get p+v */
306 "movq %[ptr], %%rbx\n\t"
307 "addq %[off], %%rbx\n\t"
308 /* get pv */
309 "movq (%%rbx), %%rcx\n\t"
310 /* *pv += inc */
311 "addq %[inc], (%%rcx)\n\t"
312 "2:\n\t"
313 RSEQ_INJECT_ASM(4)
314 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
315 : /* gcc asm goto does not allow outputs */
316 : [cpu_id] "r" (cpu),
317 [rseq_abi] "r" (&__rseq_abi),
318 /* final store input */
319 [ptr] "m" (*ptr),
320 [off] "er" (off),
321 [inc] "er" (inc)
322 : "memory", "cc", "rax", "rbx", "rcx"
323 RSEQ_INJECT_CLOBBER
324 : abort
325 #ifdef RSEQ_COMPARE_TWICE
326 , error1
327 #endif
328 );
329 return 0;
330 abort:
331 RSEQ_INJECT_FAILED
332 return -1;
333 #ifdef RSEQ_COMPARE_TWICE
334 error1:
335 rseq_bug("cpu_id comparison failed");
336 #endif
337 }
338
339 static inline __attribute__((always_inline))
rseq_cmpeqv_trystorev_storev(intptr_t * v,intptr_t expect,intptr_t * v2,intptr_t newv2,intptr_t newv,int cpu)340 int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
341 intptr_t *v2, intptr_t newv2,
342 intptr_t newv, int cpu)
343 {
344 RSEQ_INJECT_C(9)
345
346 __asm__ __volatile__ goto (
347 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
348 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
349 #ifdef RSEQ_COMPARE_TWICE
350 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
351 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
352 #endif
353 /* Start rseq by storing table entry pointer into rseq_cs. */
354 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
355 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
356 RSEQ_INJECT_ASM(3)
357 "cmpq %[v], %[expect]\n\t"
358 "jnz %l[cmpfail]\n\t"
359 RSEQ_INJECT_ASM(4)
360 #ifdef RSEQ_COMPARE_TWICE
361 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
362 "cmpq %[v], %[expect]\n\t"
363 "jnz %l[error2]\n\t"
364 #endif
365 /* try store */
366 "movq %[newv2], %[v2]\n\t"
367 RSEQ_INJECT_ASM(5)
368 /* final store */
369 "movq %[newv], %[v]\n\t"
370 "2:\n\t"
371 RSEQ_INJECT_ASM(6)
372 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
373 : /* gcc asm goto does not allow outputs */
374 : [cpu_id] "r" (cpu),
375 [rseq_abi] "r" (&__rseq_abi),
376 /* try store input */
377 [v2] "m" (*v2),
378 [newv2] "r" (newv2),
379 /* final store input */
380 [v] "m" (*v),
381 [expect] "r" (expect),
382 [newv] "r" (newv)
383 : "memory", "cc", "rax"
384 RSEQ_INJECT_CLOBBER
385 : abort, cmpfail
386 #ifdef RSEQ_COMPARE_TWICE
387 , error1, error2
388 #endif
389 );
390 return 0;
391 abort:
392 RSEQ_INJECT_FAILED
393 return -1;
394 cmpfail:
395 return 1;
396 #ifdef RSEQ_COMPARE_TWICE
397 error1:
398 rseq_bug("cpu_id comparison failed");
399 error2:
400 rseq_bug("expected value comparison failed");
401 #endif
402 }
403
404 /* x86-64 is TSO. */
405 static inline __attribute__((always_inline))
rseq_cmpeqv_trystorev_storev_release(intptr_t * v,intptr_t expect,intptr_t * v2,intptr_t newv2,intptr_t newv,int cpu)406 int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect,
407 intptr_t *v2, intptr_t newv2,
408 intptr_t newv, int cpu)
409 {
410 return rseq_cmpeqv_trystorev_storev(v, expect, v2, newv2, newv, cpu);
411 }
412
413 static inline __attribute__((always_inline))
rseq_cmpeqv_cmpeqv_storev(intptr_t * v,intptr_t expect,intptr_t * v2,intptr_t expect2,intptr_t newv,int cpu)414 int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
415 intptr_t *v2, intptr_t expect2,
416 intptr_t newv, int cpu)
417 {
418 RSEQ_INJECT_C(9)
419
420 __asm__ __volatile__ goto (
421 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
422 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
423 #ifdef RSEQ_COMPARE_TWICE
424 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
425 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
426 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3])
427 #endif
428 /* Start rseq by storing table entry pointer into rseq_cs. */
429 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
430 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
431 RSEQ_INJECT_ASM(3)
432 "cmpq %[v], %[expect]\n\t"
433 "jnz %l[cmpfail]\n\t"
434 RSEQ_INJECT_ASM(4)
435 "cmpq %[v2], %[expect2]\n\t"
436 "jnz %l[cmpfail]\n\t"
437 RSEQ_INJECT_ASM(5)
438 #ifdef RSEQ_COMPARE_TWICE
439 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
440 "cmpq %[v], %[expect]\n\t"
441 "jnz %l[error2]\n\t"
442 "cmpq %[v2], %[expect2]\n\t"
443 "jnz %l[error3]\n\t"
444 #endif
445 /* final store */
446 "movq %[newv], %[v]\n\t"
447 "2:\n\t"
448 RSEQ_INJECT_ASM(6)
449 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
450 : /* gcc asm goto does not allow outputs */
451 : [cpu_id] "r" (cpu),
452 [rseq_abi] "r" (&__rseq_abi),
453 /* cmp2 input */
454 [v2] "m" (*v2),
455 [expect2] "r" (expect2),
456 /* final store input */
457 [v] "m" (*v),
458 [expect] "r" (expect),
459 [newv] "r" (newv)
460 : "memory", "cc", "rax"
461 RSEQ_INJECT_CLOBBER
462 : abort, cmpfail
463 #ifdef RSEQ_COMPARE_TWICE
464 , error1, error2, error3
465 #endif
466 );
467 return 0;
468 abort:
469 RSEQ_INJECT_FAILED
470 return -1;
471 cmpfail:
472 return 1;
473 #ifdef RSEQ_COMPARE_TWICE
474 error1:
475 rseq_bug("cpu_id comparison failed");
476 error2:
477 rseq_bug("1st expected value comparison failed");
478 error3:
479 rseq_bug("2nd expected value comparison failed");
480 #endif
481 }
482
483 static inline __attribute__((always_inline))
rseq_cmpeqv_trymemcpy_storev(intptr_t * v,intptr_t expect,void * dst,void * src,size_t len,intptr_t newv,int cpu)484 int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,
485 void *dst, void *src, size_t len,
486 intptr_t newv, int cpu)
487 {
488 uint64_t rseq_scratch[3];
489
490 RSEQ_INJECT_C(9)
491
492 __asm__ __volatile__ goto (
493 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
494 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
495 #ifdef RSEQ_COMPARE_TWICE
496 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
497 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
498 #endif
499 "movq %[src], %[rseq_scratch0]\n\t"
500 "movq %[dst], %[rseq_scratch1]\n\t"
501 "movq %[len], %[rseq_scratch2]\n\t"
502 /* Start rseq by storing table entry pointer into rseq_cs. */
503 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
504 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
505 RSEQ_INJECT_ASM(3)
506 "cmpq %[v], %[expect]\n\t"
507 "jnz 5f\n\t"
508 RSEQ_INJECT_ASM(4)
509 #ifdef RSEQ_COMPARE_TWICE
510 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 6f)
511 "cmpq %[v], %[expect]\n\t"
512 "jnz 7f\n\t"
513 #endif
514 /* try memcpy */
515 "test %[len], %[len]\n\t" \
516 "jz 333f\n\t" \
517 "222:\n\t" \
518 "movb (%[src]), %%al\n\t" \
519 "movb %%al, (%[dst])\n\t" \
520 "inc %[src]\n\t" \
521 "inc %[dst]\n\t" \
522 "dec %[len]\n\t" \
523 "jnz 222b\n\t" \
524 "333:\n\t" \
525 RSEQ_INJECT_ASM(5)
526 /* final store */
527 "movq %[newv], %[v]\n\t"
528 "2:\n\t"
529 RSEQ_INJECT_ASM(6)
530 /* teardown */
531 "movq %[rseq_scratch2], %[len]\n\t"
532 "movq %[rseq_scratch1], %[dst]\n\t"
533 "movq %[rseq_scratch0], %[src]\n\t"
534 RSEQ_ASM_DEFINE_ABORT(4,
535 "movq %[rseq_scratch2], %[len]\n\t"
536 "movq %[rseq_scratch1], %[dst]\n\t"
537 "movq %[rseq_scratch0], %[src]\n\t",
538 abort)
539 RSEQ_ASM_DEFINE_CMPFAIL(5,
540 "movq %[rseq_scratch2], %[len]\n\t"
541 "movq %[rseq_scratch1], %[dst]\n\t"
542 "movq %[rseq_scratch0], %[src]\n\t",
543 cmpfail)
544 #ifdef RSEQ_COMPARE_TWICE
545 RSEQ_ASM_DEFINE_CMPFAIL(6,
546 "movq %[rseq_scratch2], %[len]\n\t"
547 "movq %[rseq_scratch1], %[dst]\n\t"
548 "movq %[rseq_scratch0], %[src]\n\t",
549 error1)
550 RSEQ_ASM_DEFINE_CMPFAIL(7,
551 "movq %[rseq_scratch2], %[len]\n\t"
552 "movq %[rseq_scratch1], %[dst]\n\t"
553 "movq %[rseq_scratch0], %[src]\n\t",
554 error2)
555 #endif
556 : /* gcc asm goto does not allow outputs */
557 : [cpu_id] "r" (cpu),
558 [rseq_abi] "r" (&__rseq_abi),
559 /* final store input */
560 [v] "m" (*v),
561 [expect] "r" (expect),
562 [newv] "r" (newv),
563 /* try memcpy input */
564 [dst] "r" (dst),
565 [src] "r" (src),
566 [len] "r" (len),
567 [rseq_scratch0] "m" (rseq_scratch[0]),
568 [rseq_scratch1] "m" (rseq_scratch[1]),
569 [rseq_scratch2] "m" (rseq_scratch[2])
570 : "memory", "cc", "rax"
571 RSEQ_INJECT_CLOBBER
572 : abort, cmpfail
573 #ifdef RSEQ_COMPARE_TWICE
574 , error1, error2
575 #endif
576 );
577 return 0;
578 abort:
579 RSEQ_INJECT_FAILED
580 return -1;
581 cmpfail:
582 return 1;
583 #ifdef RSEQ_COMPARE_TWICE
584 error1:
585 rseq_bug("cpu_id comparison failed");
586 error2:
587 rseq_bug("expected value comparison failed");
588 #endif
589 }
590
591 /* x86-64 is TSO. */
592 static inline __attribute__((always_inline))
rseq_cmpeqv_trymemcpy_storev_release(intptr_t * v,intptr_t expect,void * dst,void * src,size_t len,intptr_t newv,int cpu)593 int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect,
594 void *dst, void *src, size_t len,
595 intptr_t newv, int cpu)
596 {
597 return rseq_cmpeqv_trymemcpy_storev(v, expect, dst, src, len,
598 newv, cpu);
599 }
600
601 #endif /* !RSEQ_SKIP_FASTPATH */
602
603 #elif __i386__
604
605 #define rseq_smp_mb() \
606 __asm__ __volatile__ ("lock; addl $0,-128(%%esp)" ::: "memory", "cc")
607 #define rseq_smp_rmb() \
608 __asm__ __volatile__ ("lock; addl $0,-128(%%esp)" ::: "memory", "cc")
609 #define rseq_smp_wmb() \
610 __asm__ __volatile__ ("lock; addl $0,-128(%%esp)" ::: "memory", "cc")
611
612 #define rseq_smp_load_acquire(p) \
613 __extension__ ({ \
614 __typeof(*p) ____p1 = RSEQ_READ_ONCE(*p); \
615 rseq_smp_mb(); \
616 ____p1; \
617 })
618
619 #define rseq_smp_acquire__after_ctrl_dep() rseq_smp_rmb()
620
621 #define rseq_smp_store_release(p, v) \
622 do { \
623 rseq_smp_mb(); \
624 RSEQ_WRITE_ONCE(*p, v); \
625 } while (0)
626
627 #ifdef RSEQ_SKIP_FASTPATH
628 #include "rseq-skip.h"
629 #else /* !RSEQ_SKIP_FASTPATH */
630
631 /*
632 * Use eax as scratch register and take memory operands as input to
633 * lessen register pressure. Especially needed when compiling in O0.
634 */
635 #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \
636 start_ip, post_commit_offset, abort_ip) \
637 ".pushsection __rseq_cs, \"aw\"\n\t" \
638 ".balign 32\n\t" \
639 __rseq_str(label) ":\n\t" \
640 ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
641 ".long " __rseq_str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) ", 0x0\n\t" \
642 ".popsection\n\t" \
643 ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t" \
644 ".long " __rseq_str(label) "b, 0x0\n\t" \
645 ".popsection\n\t"
646
647 #define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \
648 __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip, \
649 (post_commit_ip - start_ip), abort_ip)
650
651 /*
652 * Exit points of a rseq critical section consist of all instructions outside
653 * of the critical section where a critical section can either branch to or
654 * reach through the normal course of its execution. The abort IP and the
655 * post-commit IP are already part of the __rseq_cs section and should not be
656 * explicitly defined as additional exit points. Knowing all exit points is
657 * useful to assist debuggers stepping over the critical section.
658 */
659 #define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \
660 ".pushsection __rseq_exit_point_array, \"aw\"\n\t" \
661 ".long " __rseq_str(start_ip) ", 0x0, " __rseq_str(exit_ip) ", 0x0\n\t" \
662 ".popsection\n\t"
663
664 #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \
665 RSEQ_INJECT_ASM(1) \
666 "movl $" __rseq_str(cs_label) ", " __rseq_str(rseq_cs) "\n\t" \
667 __rseq_str(label) ":\n\t"
668
669 #define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label) \
670 RSEQ_INJECT_ASM(2) \
671 "cmpl %[" __rseq_str(cpu_id) "], " __rseq_str(current_cpu_id) "\n\t" \
672 "jnz " __rseq_str(label) "\n\t"
673
674 #define RSEQ_ASM_DEFINE_ABORT(label, teardown, abort_label) \
675 ".pushsection __rseq_failure, \"ax\"\n\t" \
676 /* Disassembler-friendly signature: ud1 <sig>,%edi. */ \
677 ".byte 0x0f, 0xb9, 0x3d\n\t" \
678 ".long " __rseq_str(RSEQ_SIG) "\n\t" \
679 __rseq_str(label) ":\n\t" \
680 teardown \
681 "jmp %l[" __rseq_str(abort_label) "]\n\t" \
682 ".popsection\n\t"
683
684 #define RSEQ_ASM_DEFINE_CMPFAIL(label, teardown, cmpfail_label) \
685 ".pushsection __rseq_failure, \"ax\"\n\t" \
686 __rseq_str(label) ":\n\t" \
687 teardown \
688 "jmp %l[" __rseq_str(cmpfail_label) "]\n\t" \
689 ".popsection\n\t"
690
691 static inline __attribute__((always_inline))
rseq_cmpeqv_storev(intptr_t * v,intptr_t expect,intptr_t newv,int cpu)692 int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
693 {
694 RSEQ_INJECT_C(9)
695
696 __asm__ __volatile__ goto (
697 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
698 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
699 #ifdef RSEQ_COMPARE_TWICE
700 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
701 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
702 #endif
703 /* Start rseq by storing table entry pointer into rseq_cs. */
704 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
705 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
706 RSEQ_INJECT_ASM(3)
707 "cmpl %[v], %[expect]\n\t"
708 "jnz %l[cmpfail]\n\t"
709 RSEQ_INJECT_ASM(4)
710 #ifdef RSEQ_COMPARE_TWICE
711 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
712 "cmpl %[v], %[expect]\n\t"
713 "jnz %l[error2]\n\t"
714 #endif
715 /* final store */
716 "movl %[newv], %[v]\n\t"
717 "2:\n\t"
718 RSEQ_INJECT_ASM(5)
719 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
720 : /* gcc asm goto does not allow outputs */
721 : [cpu_id] "r" (cpu),
722 [rseq_abi] "r" (&__rseq_abi),
723 [v] "m" (*v),
724 [expect] "r" (expect),
725 [newv] "r" (newv)
726 : "memory", "cc", "eax"
727 RSEQ_INJECT_CLOBBER
728 : abort, cmpfail
729 #ifdef RSEQ_COMPARE_TWICE
730 , error1, error2
731 #endif
732 );
733 return 0;
734 abort:
735 RSEQ_INJECT_FAILED
736 return -1;
737 cmpfail:
738 return 1;
739 #ifdef RSEQ_COMPARE_TWICE
740 error1:
741 rseq_bug("cpu_id comparison failed");
742 error2:
743 rseq_bug("expected value comparison failed");
744 #endif
745 }
746
747 /*
748 * Compare @v against @expectnot. When it does _not_ match, load @v
749 * into @load, and store the content of *@v + voffp into @v.
750 */
751 static inline __attribute__((always_inline))
rseq_cmpnev_storeoffp_load(intptr_t * v,intptr_t expectnot,off_t voffp,intptr_t * load,int cpu)752 int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
753 off_t voffp, intptr_t *load, int cpu)
754 {
755 RSEQ_INJECT_C(9)
756
757 __asm__ __volatile__ goto (
758 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
759 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
760 #ifdef RSEQ_COMPARE_TWICE
761 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
762 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
763 #endif
764 /* Start rseq by storing table entry pointer into rseq_cs. */
765 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
766 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
767 RSEQ_INJECT_ASM(3)
768 "movl %[v], %%ebx\n\t"
769 "cmpl %%ebx, %[expectnot]\n\t"
770 "je %l[cmpfail]\n\t"
771 RSEQ_INJECT_ASM(4)
772 #ifdef RSEQ_COMPARE_TWICE
773 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
774 "movl %[v], %%ebx\n\t"
775 "cmpl %%ebx, %[expectnot]\n\t"
776 "je %l[error2]\n\t"
777 #endif
778 "movl %%ebx, %[load]\n\t"
779 "addl %[voffp], %%ebx\n\t"
780 "movl (%%ebx), %%ebx\n\t"
781 /* final store */
782 "movl %%ebx, %[v]\n\t"
783 "2:\n\t"
784 RSEQ_INJECT_ASM(5)
785 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
786 : /* gcc asm goto does not allow outputs */
787 : [cpu_id] "r" (cpu),
788 [rseq_abi] "r" (&__rseq_abi),
789 /* final store input */
790 [v] "m" (*v),
791 [expectnot] "r" (expectnot),
792 [voffp] "ir" (voffp),
793 [load] "m" (*load)
794 : "memory", "cc", "eax", "ebx"
795 RSEQ_INJECT_CLOBBER
796 : abort, cmpfail
797 #ifdef RSEQ_COMPARE_TWICE
798 , error1, error2
799 #endif
800 );
801 return 0;
802 abort:
803 RSEQ_INJECT_FAILED
804 return -1;
805 cmpfail:
806 return 1;
807 #ifdef RSEQ_COMPARE_TWICE
808 error1:
809 rseq_bug("cpu_id comparison failed");
810 error2:
811 rseq_bug("expected value comparison failed");
812 #endif
813 }
814
815 static inline __attribute__((always_inline))
rseq_addv(intptr_t * v,intptr_t count,int cpu)816 int rseq_addv(intptr_t *v, intptr_t count, int cpu)
817 {
818 RSEQ_INJECT_C(9)
819
820 __asm__ __volatile__ goto (
821 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
822 #ifdef RSEQ_COMPARE_TWICE
823 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
824 #endif
825 /* Start rseq by storing table entry pointer into rseq_cs. */
826 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
827 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
828 RSEQ_INJECT_ASM(3)
829 #ifdef RSEQ_COMPARE_TWICE
830 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
831 #endif
832 /* final store */
833 "addl %[count], %[v]\n\t"
834 "2:\n\t"
835 RSEQ_INJECT_ASM(4)
836 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
837 : /* gcc asm goto does not allow outputs */
838 : [cpu_id] "r" (cpu),
839 [rseq_abi] "r" (&__rseq_abi),
840 /* final store input */
841 [v] "m" (*v),
842 [count] "ir" (count)
843 : "memory", "cc", "eax"
844 RSEQ_INJECT_CLOBBER
845 : abort
846 #ifdef RSEQ_COMPARE_TWICE
847 , error1
848 #endif
849 );
850 return 0;
851 abort:
852 RSEQ_INJECT_FAILED
853 return -1;
854 #ifdef RSEQ_COMPARE_TWICE
855 error1:
856 rseq_bug("cpu_id comparison failed");
857 #endif
858 }
859
860 static inline __attribute__((always_inline))
rseq_cmpeqv_trystorev_storev(intptr_t * v,intptr_t expect,intptr_t * v2,intptr_t newv2,intptr_t newv,int cpu)861 int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
862 intptr_t *v2, intptr_t newv2,
863 intptr_t newv, int cpu)
864 {
865 RSEQ_INJECT_C(9)
866
867 __asm__ __volatile__ goto (
868 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
869 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
870 #ifdef RSEQ_COMPARE_TWICE
871 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
872 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
873 #endif
874 /* Start rseq by storing table entry pointer into rseq_cs. */
875 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
876 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
877 RSEQ_INJECT_ASM(3)
878 "cmpl %[v], %[expect]\n\t"
879 "jnz %l[cmpfail]\n\t"
880 RSEQ_INJECT_ASM(4)
881 #ifdef RSEQ_COMPARE_TWICE
882 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
883 "cmpl %[v], %[expect]\n\t"
884 "jnz %l[error2]\n\t"
885 #endif
886 /* try store */
887 "movl %[newv2], %%eax\n\t"
888 "movl %%eax, %[v2]\n\t"
889 RSEQ_INJECT_ASM(5)
890 /* final store */
891 "movl %[newv], %[v]\n\t"
892 "2:\n\t"
893 RSEQ_INJECT_ASM(6)
894 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
895 : /* gcc asm goto does not allow outputs */
896 : [cpu_id] "r" (cpu),
897 [rseq_abi] "r" (&__rseq_abi),
898 /* try store input */
899 [v2] "m" (*v2),
900 [newv2] "m" (newv2),
901 /* final store input */
902 [v] "m" (*v),
903 [expect] "r" (expect),
904 [newv] "r" (newv)
905 : "memory", "cc", "eax"
906 RSEQ_INJECT_CLOBBER
907 : abort, cmpfail
908 #ifdef RSEQ_COMPARE_TWICE
909 , error1, error2
910 #endif
911 );
912 return 0;
913 abort:
914 RSEQ_INJECT_FAILED
915 return -1;
916 cmpfail:
917 return 1;
918 #ifdef RSEQ_COMPARE_TWICE
919 error1:
920 rseq_bug("cpu_id comparison failed");
921 error2:
922 rseq_bug("expected value comparison failed");
923 #endif
924 }
925
926 static inline __attribute__((always_inline))
rseq_cmpeqv_trystorev_storev_release(intptr_t * v,intptr_t expect,intptr_t * v2,intptr_t newv2,intptr_t newv,int cpu)927 int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect,
928 intptr_t *v2, intptr_t newv2,
929 intptr_t newv, int cpu)
930 {
931 RSEQ_INJECT_C(9)
932
933 __asm__ __volatile__ goto (
934 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
935 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
936 #ifdef RSEQ_COMPARE_TWICE
937 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
938 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
939 #endif
940 /* Start rseq by storing table entry pointer into rseq_cs. */
941 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
942 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
943 RSEQ_INJECT_ASM(3)
944 "movl %[expect], %%eax\n\t"
945 "cmpl %[v], %%eax\n\t"
946 "jnz %l[cmpfail]\n\t"
947 RSEQ_INJECT_ASM(4)
948 #ifdef RSEQ_COMPARE_TWICE
949 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
950 "movl %[expect], %%eax\n\t"
951 "cmpl %[v], %%eax\n\t"
952 "jnz %l[error2]\n\t"
953 #endif
954 /* try store */
955 "movl %[newv2], %[v2]\n\t"
956 RSEQ_INJECT_ASM(5)
957 "lock; addl $0,-128(%%esp)\n\t"
958 /* final store */
959 "movl %[newv], %[v]\n\t"
960 "2:\n\t"
961 RSEQ_INJECT_ASM(6)
962 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
963 : /* gcc asm goto does not allow outputs */
964 : [cpu_id] "r" (cpu),
965 [rseq_abi] "r" (&__rseq_abi),
966 /* try store input */
967 [v2] "m" (*v2),
968 [newv2] "r" (newv2),
969 /* final store input */
970 [v] "m" (*v),
971 [expect] "m" (expect),
972 [newv] "r" (newv)
973 : "memory", "cc", "eax"
974 RSEQ_INJECT_CLOBBER
975 : abort, cmpfail
976 #ifdef RSEQ_COMPARE_TWICE
977 , error1, error2
978 #endif
979 );
980 return 0;
981 abort:
982 RSEQ_INJECT_FAILED
983 return -1;
984 cmpfail:
985 return 1;
986 #ifdef RSEQ_COMPARE_TWICE
987 error1:
988 rseq_bug("cpu_id comparison failed");
989 error2:
990 rseq_bug("expected value comparison failed");
991 #endif
992
993 }
994
995 static inline __attribute__((always_inline))
rseq_cmpeqv_cmpeqv_storev(intptr_t * v,intptr_t expect,intptr_t * v2,intptr_t expect2,intptr_t newv,int cpu)996 int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
997 intptr_t *v2, intptr_t expect2,
998 intptr_t newv, int cpu)
999 {
1000 RSEQ_INJECT_C(9)
1001
1002 __asm__ __volatile__ goto (
1003 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
1004 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
1005 #ifdef RSEQ_COMPARE_TWICE
1006 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
1007 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
1008 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3])
1009 #endif
1010 /* Start rseq by storing table entry pointer into rseq_cs. */
1011 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
1012 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
1013 RSEQ_INJECT_ASM(3)
1014 "cmpl %[v], %[expect]\n\t"
1015 "jnz %l[cmpfail]\n\t"
1016 RSEQ_INJECT_ASM(4)
1017 "cmpl %[expect2], %[v2]\n\t"
1018 "jnz %l[cmpfail]\n\t"
1019 RSEQ_INJECT_ASM(5)
1020 #ifdef RSEQ_COMPARE_TWICE
1021 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
1022 "cmpl %[v], %[expect]\n\t"
1023 "jnz %l[error2]\n\t"
1024 "cmpl %[expect2], %[v2]\n\t"
1025 "jnz %l[error3]\n\t"
1026 #endif
1027 "movl %[newv], %%eax\n\t"
1028 /* final store */
1029 "movl %%eax, %[v]\n\t"
1030 "2:\n\t"
1031 RSEQ_INJECT_ASM(6)
1032 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
1033 : /* gcc asm goto does not allow outputs */
1034 : [cpu_id] "r" (cpu),
1035 [rseq_abi] "r" (&__rseq_abi),
1036 /* cmp2 input */
1037 [v2] "m" (*v2),
1038 [expect2] "r" (expect2),
1039 /* final store input */
1040 [v] "m" (*v),
1041 [expect] "r" (expect),
1042 [newv] "m" (newv)
1043 : "memory", "cc", "eax"
1044 RSEQ_INJECT_CLOBBER
1045 : abort, cmpfail
1046 #ifdef RSEQ_COMPARE_TWICE
1047 , error1, error2, error3
1048 #endif
1049 );
1050 return 0;
1051 abort:
1052 RSEQ_INJECT_FAILED
1053 return -1;
1054 cmpfail:
1055 return 1;
1056 #ifdef RSEQ_COMPARE_TWICE
1057 error1:
1058 rseq_bug("cpu_id comparison failed");
1059 error2:
1060 rseq_bug("1st expected value comparison failed");
1061 error3:
1062 rseq_bug("2nd expected value comparison failed");
1063 #endif
1064 }
1065
1066 /* TODO: implement a faster memcpy. */
1067 static inline __attribute__((always_inline))
rseq_cmpeqv_trymemcpy_storev(intptr_t * v,intptr_t expect,void * dst,void * src,size_t len,intptr_t newv,int cpu)1068 int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,
1069 void *dst, void *src, size_t len,
1070 intptr_t newv, int cpu)
1071 {
1072 uint32_t rseq_scratch[3];
1073
1074 RSEQ_INJECT_C(9)
1075
1076 __asm__ __volatile__ goto (
1077 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
1078 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
1079 #ifdef RSEQ_COMPARE_TWICE
1080 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
1081 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
1082 #endif
1083 "movl %[src], %[rseq_scratch0]\n\t"
1084 "movl %[dst], %[rseq_scratch1]\n\t"
1085 "movl %[len], %[rseq_scratch2]\n\t"
1086 /* Start rseq by storing table entry pointer into rseq_cs. */
1087 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
1088 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
1089 RSEQ_INJECT_ASM(3)
1090 "movl %[expect], %%eax\n\t"
1091 "cmpl %%eax, %[v]\n\t"
1092 "jnz 5f\n\t"
1093 RSEQ_INJECT_ASM(4)
1094 #ifdef RSEQ_COMPARE_TWICE
1095 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 6f)
1096 "movl %[expect], %%eax\n\t"
1097 "cmpl %%eax, %[v]\n\t"
1098 "jnz 7f\n\t"
1099 #endif
1100 /* try memcpy */
1101 "test %[len], %[len]\n\t" \
1102 "jz 333f\n\t" \
1103 "222:\n\t" \
1104 "movb (%[src]), %%al\n\t" \
1105 "movb %%al, (%[dst])\n\t" \
1106 "inc %[src]\n\t" \
1107 "inc %[dst]\n\t" \
1108 "dec %[len]\n\t" \
1109 "jnz 222b\n\t" \
1110 "333:\n\t" \
1111 RSEQ_INJECT_ASM(5)
1112 "movl %[newv], %%eax\n\t"
1113 /* final store */
1114 "movl %%eax, %[v]\n\t"
1115 "2:\n\t"
1116 RSEQ_INJECT_ASM(6)
1117 /* teardown */
1118 "movl %[rseq_scratch2], %[len]\n\t"
1119 "movl %[rseq_scratch1], %[dst]\n\t"
1120 "movl %[rseq_scratch0], %[src]\n\t"
1121 RSEQ_ASM_DEFINE_ABORT(4,
1122 "movl %[rseq_scratch2], %[len]\n\t"
1123 "movl %[rseq_scratch1], %[dst]\n\t"
1124 "movl %[rseq_scratch0], %[src]\n\t",
1125 abort)
1126 RSEQ_ASM_DEFINE_CMPFAIL(5,
1127 "movl %[rseq_scratch2], %[len]\n\t"
1128 "movl %[rseq_scratch1], %[dst]\n\t"
1129 "movl %[rseq_scratch0], %[src]\n\t",
1130 cmpfail)
1131 #ifdef RSEQ_COMPARE_TWICE
1132 RSEQ_ASM_DEFINE_CMPFAIL(6,
1133 "movl %[rseq_scratch2], %[len]\n\t"
1134 "movl %[rseq_scratch1], %[dst]\n\t"
1135 "movl %[rseq_scratch0], %[src]\n\t",
1136 error1)
1137 RSEQ_ASM_DEFINE_CMPFAIL(7,
1138 "movl %[rseq_scratch2], %[len]\n\t"
1139 "movl %[rseq_scratch1], %[dst]\n\t"
1140 "movl %[rseq_scratch0], %[src]\n\t",
1141 error2)
1142 #endif
1143 : /* gcc asm goto does not allow outputs */
1144 : [cpu_id] "r" (cpu),
1145 [rseq_abi] "r" (&__rseq_abi),
1146 /* final store input */
1147 [v] "m" (*v),
1148 [expect] "m" (expect),
1149 [newv] "m" (newv),
1150 /* try memcpy input */
1151 [dst] "r" (dst),
1152 [src] "r" (src),
1153 [len] "r" (len),
1154 [rseq_scratch0] "m" (rseq_scratch[0]),
1155 [rseq_scratch1] "m" (rseq_scratch[1]),
1156 [rseq_scratch2] "m" (rseq_scratch[2])
1157 : "memory", "cc", "eax"
1158 RSEQ_INJECT_CLOBBER
1159 : abort, cmpfail
1160 #ifdef RSEQ_COMPARE_TWICE
1161 , error1, error2
1162 #endif
1163 );
1164 return 0;
1165 abort:
1166 RSEQ_INJECT_FAILED
1167 return -1;
1168 cmpfail:
1169 return 1;
1170 #ifdef RSEQ_COMPARE_TWICE
1171 error1:
1172 rseq_bug("cpu_id comparison failed");
1173 error2:
1174 rseq_bug("expected value comparison failed");
1175 #endif
1176 }
1177
1178 /* TODO: implement a faster memcpy. */
1179 static inline __attribute__((always_inline))
rseq_cmpeqv_trymemcpy_storev_release(intptr_t * v,intptr_t expect,void * dst,void * src,size_t len,intptr_t newv,int cpu)1180 int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect,
1181 void *dst, void *src, size_t len,
1182 intptr_t newv, int cpu)
1183 {
1184 uint32_t rseq_scratch[3];
1185
1186 RSEQ_INJECT_C(9)
1187
1188 __asm__ __volatile__ goto (
1189 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
1190 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
1191 #ifdef RSEQ_COMPARE_TWICE
1192 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
1193 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
1194 #endif
1195 "movl %[src], %[rseq_scratch0]\n\t"
1196 "movl %[dst], %[rseq_scratch1]\n\t"
1197 "movl %[len], %[rseq_scratch2]\n\t"
1198 /* Start rseq by storing table entry pointer into rseq_cs. */
1199 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
1200 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
1201 RSEQ_INJECT_ASM(3)
1202 "movl %[expect], %%eax\n\t"
1203 "cmpl %%eax, %[v]\n\t"
1204 "jnz 5f\n\t"
1205 RSEQ_INJECT_ASM(4)
1206 #ifdef RSEQ_COMPARE_TWICE
1207 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 6f)
1208 "movl %[expect], %%eax\n\t"
1209 "cmpl %%eax, %[v]\n\t"
1210 "jnz 7f\n\t"
1211 #endif
1212 /* try memcpy */
1213 "test %[len], %[len]\n\t" \
1214 "jz 333f\n\t" \
1215 "222:\n\t" \
1216 "movb (%[src]), %%al\n\t" \
1217 "movb %%al, (%[dst])\n\t" \
1218 "inc %[src]\n\t" \
1219 "inc %[dst]\n\t" \
1220 "dec %[len]\n\t" \
1221 "jnz 222b\n\t" \
1222 "333:\n\t" \
1223 RSEQ_INJECT_ASM(5)
1224 "lock; addl $0,-128(%%esp)\n\t"
1225 "movl %[newv], %%eax\n\t"
1226 /* final store */
1227 "movl %%eax, %[v]\n\t"
1228 "2:\n\t"
1229 RSEQ_INJECT_ASM(6)
1230 /* teardown */
1231 "movl %[rseq_scratch2], %[len]\n\t"
1232 "movl %[rseq_scratch1], %[dst]\n\t"
1233 "movl %[rseq_scratch0], %[src]\n\t"
1234 RSEQ_ASM_DEFINE_ABORT(4,
1235 "movl %[rseq_scratch2], %[len]\n\t"
1236 "movl %[rseq_scratch1], %[dst]\n\t"
1237 "movl %[rseq_scratch0], %[src]\n\t",
1238 abort)
1239 RSEQ_ASM_DEFINE_CMPFAIL(5,
1240 "movl %[rseq_scratch2], %[len]\n\t"
1241 "movl %[rseq_scratch1], %[dst]\n\t"
1242 "movl %[rseq_scratch0], %[src]\n\t",
1243 cmpfail)
1244 #ifdef RSEQ_COMPARE_TWICE
1245 RSEQ_ASM_DEFINE_CMPFAIL(6,
1246 "movl %[rseq_scratch2], %[len]\n\t"
1247 "movl %[rseq_scratch1], %[dst]\n\t"
1248 "movl %[rseq_scratch0], %[src]\n\t",
1249 error1)
1250 RSEQ_ASM_DEFINE_CMPFAIL(7,
1251 "movl %[rseq_scratch2], %[len]\n\t"
1252 "movl %[rseq_scratch1], %[dst]\n\t"
1253 "movl %[rseq_scratch0], %[src]\n\t",
1254 error2)
1255 #endif
1256 : /* gcc asm goto does not allow outputs */
1257 : [cpu_id] "r" (cpu),
1258 [rseq_abi] "r" (&__rseq_abi),
1259 /* final store input */
1260 [v] "m" (*v),
1261 [expect] "m" (expect),
1262 [newv] "m" (newv),
1263 /* try memcpy input */
1264 [dst] "r" (dst),
1265 [src] "r" (src),
1266 [len] "r" (len),
1267 [rseq_scratch0] "m" (rseq_scratch[0]),
1268 [rseq_scratch1] "m" (rseq_scratch[1]),
1269 [rseq_scratch2] "m" (rseq_scratch[2])
1270 : "memory", "cc", "eax"
1271 RSEQ_INJECT_CLOBBER
1272 : abort, cmpfail
1273 #ifdef RSEQ_COMPARE_TWICE
1274 , error1, error2
1275 #endif
1276 );
1277 return 0;
1278 abort:
1279 RSEQ_INJECT_FAILED
1280 return -1;
1281 cmpfail:
1282 return 1;
1283 #ifdef RSEQ_COMPARE_TWICE
1284 error1:
1285 rseq_bug("cpu_id comparison failed");
1286 error2:
1287 rseq_bug("expected value comparison failed");
1288 #endif
1289 }
1290
1291 #endif /* !RSEQ_SKIP_FASTPATH */
1292
1293 #endif
1294