1 /* SPDX-License-Identifier: LGPL-2.1 OR MIT */
2 /*
3 * rseq-arm.h
4 *
5 * (C) Copyright 2016-2018 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
6 */
7
8 /*
9 * - ARM little endian
10 *
11 * RSEQ_SIG uses the udf A32 instruction with an uncommon immediate operand
12 * value 0x5de3. This traps if user-space reaches this instruction by mistake,
13 * and the uncommon operand ensures the kernel does not move the instruction
14 * pointer to attacker-controlled code on rseq abort.
15 *
16 * The instruction pattern in the A32 instruction set is:
17 *
18 * e7f5def3 udf #24035 ; 0x5de3
19 *
20 * This translates to the following instruction pattern in the T16 instruction
21 * set:
22 *
23 * little endian:
24 * def3 udf #243 ; 0xf3
25 * e7f5 b.n <7f5>
26 *
27 * - ARMv6+ big endian (BE8):
28 *
29 * ARMv6+ -mbig-endian generates mixed endianness code vs data: little-endian
30 * code and big-endian data. The data value of the signature needs to have its
31 * byte order reversed to generate the trap instruction:
32 *
33 * Data: 0xf3def5e7
34 *
35 * Translates to this A32 instruction pattern:
36 *
37 * e7f5def3 udf #24035 ; 0x5de3
38 *
39 * Translates to this T16 instruction pattern:
40 *
41 * def3 udf #243 ; 0xf3
42 * e7f5 b.n <7f5>
43 *
44 * - Prior to ARMv6 big endian (BE32):
45 *
46 * Prior to ARMv6, -mbig-endian generates big-endian code and data
47 * (which match), so the endianness of the data representation of the
48 * signature should not be reversed. However, the choice between BE32
49 * and BE8 is done by the linker, so we cannot know whether code and
50 * data endianness will be mixed before the linker is invoked. So rather
51 * than try to play tricks with the linker, the rseq signature is simply
52 * data (not a trap instruction) prior to ARMv6 on big endian. This is
53 * why the signature is expressed as data (.word) rather than as
54 * instruction (.inst) in assembler.
55 */
56
57 #ifdef __ARMEB__
58 #define RSEQ_SIG 0xf3def5e7 /* udf #24035 ; 0x5de3 (ARMv6+) */
59 #else
60 #define RSEQ_SIG 0xe7f5def3 /* udf #24035 ; 0x5de3 */
61 #endif
62
63 #define rseq_smp_mb() __asm__ __volatile__ ("dmb" ::: "memory", "cc")
64 #define rseq_smp_rmb() __asm__ __volatile__ ("dmb" ::: "memory", "cc")
65 #define rseq_smp_wmb() __asm__ __volatile__ ("dmb" ::: "memory", "cc")
66
67 #define rseq_smp_load_acquire(p) \
68 __extension__ ({ \
69 __typeof(*p) ____p1 = RSEQ_READ_ONCE(*p); \
70 rseq_smp_mb(); \
71 ____p1; \
72 })
73
74 #define rseq_smp_acquire__after_ctrl_dep() rseq_smp_rmb()
75
76 #define rseq_smp_store_release(p, v) \
77 do { \
78 rseq_smp_mb(); \
79 RSEQ_WRITE_ONCE(*p, v); \
80 } while (0)
81
82 #ifdef RSEQ_SKIP_FASTPATH
83 #include "rseq-skip.h"
84 #else /* !RSEQ_SKIP_FASTPATH */
85
86 #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, start_ip, \
87 post_commit_offset, abort_ip) \
88 ".pushsection __rseq_cs, \"aw\"\n\t" \
89 ".balign 32\n\t" \
90 __rseq_str(label) ":\n\t" \
91 ".word " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
92 ".word " __rseq_str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) ", 0x0\n\t" \
93 ".popsection\n\t" \
94 ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t" \
95 ".word " __rseq_str(label) "b, 0x0\n\t" \
96 ".popsection\n\t"
97
98 #define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \
99 __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip, \
100 (post_commit_ip - start_ip), abort_ip)
101
102 /*
103 * Exit points of a rseq critical section consist of all instructions outside
104 * of the critical section where a critical section can either branch to or
105 * reach through the normal course of its execution. The abort IP and the
106 * post-commit IP are already part of the __rseq_cs section and should not be
107 * explicitly defined as additional exit points. Knowing all exit points is
108 * useful to assist debuggers stepping over the critical section.
109 */
110 #define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \
111 ".pushsection __rseq_exit_point_array, \"aw\"\n\t" \
112 ".word " __rseq_str(start_ip) ", 0x0, " __rseq_str(exit_ip) ", 0x0\n\t" \
113 ".popsection\n\t"
114
115 #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \
116 RSEQ_INJECT_ASM(1) \
117 "adr r0, " __rseq_str(cs_label) "\n\t" \
118 "str r0, %[" __rseq_str(rseq_cs) "]\n\t" \
119 __rseq_str(label) ":\n\t"
120
121 #define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label) \
122 RSEQ_INJECT_ASM(2) \
123 "ldr r0, %[" __rseq_str(current_cpu_id) "]\n\t" \
124 "cmp %[" __rseq_str(cpu_id) "], r0\n\t" \
125 "bne " __rseq_str(label) "\n\t"
126
127 #define __RSEQ_ASM_DEFINE_ABORT(table_label, label, teardown, \
128 abort_label, version, flags, \
129 start_ip, post_commit_offset, abort_ip) \
130 ".balign 32\n\t" \
131 __rseq_str(table_label) ":\n\t" \
132 ".word " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
133 ".word " __rseq_str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) ", 0x0\n\t" \
134 ".word " __rseq_str(RSEQ_SIG) "\n\t" \
135 __rseq_str(label) ":\n\t" \
136 teardown \
137 "b %l[" __rseq_str(abort_label) "]\n\t"
138
139 #define RSEQ_ASM_DEFINE_ABORT(table_label, label, teardown, abort_label, \
140 start_ip, post_commit_ip, abort_ip) \
141 __RSEQ_ASM_DEFINE_ABORT(table_label, label, teardown, \
142 abort_label, 0x0, 0x0, start_ip, \
143 (post_commit_ip - start_ip), abort_ip)
144
145 #define RSEQ_ASM_DEFINE_CMPFAIL(label, teardown, cmpfail_label) \
146 __rseq_str(label) ":\n\t" \
147 teardown \
148 "b %l[" __rseq_str(cmpfail_label) "]\n\t"
149
150 #define rseq_workaround_gcc_asm_size_guess() __asm__ __volatile__("")
151
152 static inline __attribute__((always_inline))
rseq_cmpeqv_storev(intptr_t * v,intptr_t expect,intptr_t newv,int cpu)153 int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
154 {
155 RSEQ_INJECT_C(9)
156
157 rseq_workaround_gcc_asm_size_guess();
158 __asm__ __volatile__ goto (
159 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
160 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
161 #ifdef RSEQ_COMPARE_TWICE
162 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
163 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
164 #endif
165 /* Start rseq by storing table entry pointer into rseq_cs. */
166 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
167 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
168 RSEQ_INJECT_ASM(3)
169 "ldr r0, %[v]\n\t"
170 "cmp %[expect], r0\n\t"
171 "bne %l[cmpfail]\n\t"
172 RSEQ_INJECT_ASM(4)
173 #ifdef RSEQ_COMPARE_TWICE
174 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
175 "ldr r0, %[v]\n\t"
176 "cmp %[expect], r0\n\t"
177 "bne %l[error2]\n\t"
178 #endif
179 /* final store */
180 "str %[newv], %[v]\n\t"
181 "2:\n\t"
182 RSEQ_INJECT_ASM(5)
183 "b 5f\n\t"
184 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
185 "5:\n\t"
186 : /* gcc asm goto does not allow outputs */
187 : [cpu_id] "r" (cpu),
188 [current_cpu_id] "m" (__rseq_abi.cpu_id),
189 [rseq_cs] "m" (__rseq_abi.rseq_cs),
190 [v] "m" (*v),
191 [expect] "r" (expect),
192 [newv] "r" (newv)
193 RSEQ_INJECT_INPUT
194 : "r0", "memory", "cc"
195 RSEQ_INJECT_CLOBBER
196 : abort, cmpfail
197 #ifdef RSEQ_COMPARE_TWICE
198 , error1, error2
199 #endif
200 );
201 rseq_workaround_gcc_asm_size_guess();
202 return 0;
203 abort:
204 rseq_workaround_gcc_asm_size_guess();
205 RSEQ_INJECT_FAILED
206 return -1;
207 cmpfail:
208 rseq_workaround_gcc_asm_size_guess();
209 return 1;
210 #ifdef RSEQ_COMPARE_TWICE
211 error1:
212 rseq_bug("cpu_id comparison failed");
213 error2:
214 rseq_bug("expected value comparison failed");
215 #endif
216 }
217
218 static inline __attribute__((always_inline))
rseq_cmpnev_storeoffp_load(intptr_t * v,intptr_t expectnot,off_t voffp,intptr_t * load,int cpu)219 int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
220 off_t voffp, intptr_t *load, int cpu)
221 {
222 RSEQ_INJECT_C(9)
223
224 rseq_workaround_gcc_asm_size_guess();
225 __asm__ __volatile__ goto (
226 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
227 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
228 #ifdef RSEQ_COMPARE_TWICE
229 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
230 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
231 #endif
232 /* Start rseq by storing table entry pointer into rseq_cs. */
233 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
234 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
235 RSEQ_INJECT_ASM(3)
236 "ldr r0, %[v]\n\t"
237 "cmp %[expectnot], r0\n\t"
238 "beq %l[cmpfail]\n\t"
239 RSEQ_INJECT_ASM(4)
240 #ifdef RSEQ_COMPARE_TWICE
241 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
242 "ldr r0, %[v]\n\t"
243 "cmp %[expectnot], r0\n\t"
244 "beq %l[error2]\n\t"
245 #endif
246 "str r0, %[load]\n\t"
247 "add r0, %[voffp]\n\t"
248 "ldr r0, [r0]\n\t"
249 /* final store */
250 "str r0, %[v]\n\t"
251 "2:\n\t"
252 RSEQ_INJECT_ASM(5)
253 "b 5f\n\t"
254 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
255 "5:\n\t"
256 : /* gcc asm goto does not allow outputs */
257 : [cpu_id] "r" (cpu),
258 [current_cpu_id] "m" (__rseq_abi.cpu_id),
259 [rseq_cs] "m" (__rseq_abi.rseq_cs),
260 /* final store input */
261 [v] "m" (*v),
262 [expectnot] "r" (expectnot),
263 [voffp] "Ir" (voffp),
264 [load] "m" (*load)
265 RSEQ_INJECT_INPUT
266 : "r0", "memory", "cc"
267 RSEQ_INJECT_CLOBBER
268 : abort, cmpfail
269 #ifdef RSEQ_COMPARE_TWICE
270 , error1, error2
271 #endif
272 );
273 rseq_workaround_gcc_asm_size_guess();
274 return 0;
275 abort:
276 rseq_workaround_gcc_asm_size_guess();
277 RSEQ_INJECT_FAILED
278 return -1;
279 cmpfail:
280 rseq_workaround_gcc_asm_size_guess();
281 return 1;
282 #ifdef RSEQ_COMPARE_TWICE
283 error1:
284 rseq_bug("cpu_id comparison failed");
285 error2:
286 rseq_bug("expected value comparison failed");
287 #endif
288 }
289
290 static inline __attribute__((always_inline))
rseq_addv(intptr_t * v,intptr_t count,int cpu)291 int rseq_addv(intptr_t *v, intptr_t count, int cpu)
292 {
293 RSEQ_INJECT_C(9)
294
295 rseq_workaround_gcc_asm_size_guess();
296 __asm__ __volatile__ goto (
297 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
298 #ifdef RSEQ_COMPARE_TWICE
299 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
300 #endif
301 /* Start rseq by storing table entry pointer into rseq_cs. */
302 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
303 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
304 RSEQ_INJECT_ASM(3)
305 #ifdef RSEQ_COMPARE_TWICE
306 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
307 #endif
308 "ldr r0, %[v]\n\t"
309 "add r0, %[count]\n\t"
310 /* final store */
311 "str r0, %[v]\n\t"
312 "2:\n\t"
313 RSEQ_INJECT_ASM(4)
314 "b 5f\n\t"
315 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
316 "5:\n\t"
317 : /* gcc asm goto does not allow outputs */
318 : [cpu_id] "r" (cpu),
319 [current_cpu_id] "m" (__rseq_abi.cpu_id),
320 [rseq_cs] "m" (__rseq_abi.rseq_cs),
321 [v] "m" (*v),
322 [count] "Ir" (count)
323 RSEQ_INJECT_INPUT
324 : "r0", "memory", "cc"
325 RSEQ_INJECT_CLOBBER
326 : abort
327 #ifdef RSEQ_COMPARE_TWICE
328 , error1
329 #endif
330 );
331 rseq_workaround_gcc_asm_size_guess();
332 return 0;
333 abort:
334 rseq_workaround_gcc_asm_size_guess();
335 RSEQ_INJECT_FAILED
336 return -1;
337 #ifdef RSEQ_COMPARE_TWICE
338 error1:
339 rseq_bug("cpu_id comparison failed");
340 #endif
341 }
342
343 static inline __attribute__((always_inline))
rseq_cmpeqv_trystorev_storev(intptr_t * v,intptr_t expect,intptr_t * v2,intptr_t newv2,intptr_t newv,int cpu)344 int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
345 intptr_t *v2, intptr_t newv2,
346 intptr_t newv, int cpu)
347 {
348 RSEQ_INJECT_C(9)
349
350 rseq_workaround_gcc_asm_size_guess();
351 __asm__ __volatile__ goto (
352 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
353 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
354 #ifdef RSEQ_COMPARE_TWICE
355 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
356 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
357 #endif
358 /* Start rseq by storing table entry pointer into rseq_cs. */
359 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
360 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
361 RSEQ_INJECT_ASM(3)
362 "ldr r0, %[v]\n\t"
363 "cmp %[expect], r0\n\t"
364 "bne %l[cmpfail]\n\t"
365 RSEQ_INJECT_ASM(4)
366 #ifdef RSEQ_COMPARE_TWICE
367 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
368 "ldr r0, %[v]\n\t"
369 "cmp %[expect], r0\n\t"
370 "bne %l[error2]\n\t"
371 #endif
372 /* try store */
373 "str %[newv2], %[v2]\n\t"
374 RSEQ_INJECT_ASM(5)
375 /* final store */
376 "str %[newv], %[v]\n\t"
377 "2:\n\t"
378 RSEQ_INJECT_ASM(6)
379 "b 5f\n\t"
380 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
381 "5:\n\t"
382 : /* gcc asm goto does not allow outputs */
383 : [cpu_id] "r" (cpu),
384 [current_cpu_id] "m" (__rseq_abi.cpu_id),
385 [rseq_cs] "m" (__rseq_abi.rseq_cs),
386 /* try store input */
387 [v2] "m" (*v2),
388 [newv2] "r" (newv2),
389 /* final store input */
390 [v] "m" (*v),
391 [expect] "r" (expect),
392 [newv] "r" (newv)
393 RSEQ_INJECT_INPUT
394 : "r0", "memory", "cc"
395 RSEQ_INJECT_CLOBBER
396 : abort, cmpfail
397 #ifdef RSEQ_COMPARE_TWICE
398 , error1, error2
399 #endif
400 );
401 rseq_workaround_gcc_asm_size_guess();
402 return 0;
403 abort:
404 rseq_workaround_gcc_asm_size_guess();
405 RSEQ_INJECT_FAILED
406 return -1;
407 cmpfail:
408 rseq_workaround_gcc_asm_size_guess();
409 return 1;
410 #ifdef RSEQ_COMPARE_TWICE
411 error1:
412 rseq_bug("cpu_id comparison failed");
413 error2:
414 rseq_bug("expected value comparison failed");
415 #endif
416 }
417
418 static inline __attribute__((always_inline))
rseq_cmpeqv_trystorev_storev_release(intptr_t * v,intptr_t expect,intptr_t * v2,intptr_t newv2,intptr_t newv,int cpu)419 int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect,
420 intptr_t *v2, intptr_t newv2,
421 intptr_t newv, int cpu)
422 {
423 RSEQ_INJECT_C(9)
424
425 rseq_workaround_gcc_asm_size_guess();
426 __asm__ __volatile__ goto (
427 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
428 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
429 #ifdef RSEQ_COMPARE_TWICE
430 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
431 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
432 #endif
433 /* Start rseq by storing table entry pointer into rseq_cs. */
434 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
435 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
436 RSEQ_INJECT_ASM(3)
437 "ldr r0, %[v]\n\t"
438 "cmp %[expect], r0\n\t"
439 "bne %l[cmpfail]\n\t"
440 RSEQ_INJECT_ASM(4)
441 #ifdef RSEQ_COMPARE_TWICE
442 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
443 "ldr r0, %[v]\n\t"
444 "cmp %[expect], r0\n\t"
445 "bne %l[error2]\n\t"
446 #endif
447 /* try store */
448 "str %[newv2], %[v2]\n\t"
449 RSEQ_INJECT_ASM(5)
450 "dmb\n\t" /* full mb provides store-release */
451 /* final store */
452 "str %[newv], %[v]\n\t"
453 "2:\n\t"
454 RSEQ_INJECT_ASM(6)
455 "b 5f\n\t"
456 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
457 "5:\n\t"
458 : /* gcc asm goto does not allow outputs */
459 : [cpu_id] "r" (cpu),
460 [current_cpu_id] "m" (__rseq_abi.cpu_id),
461 [rseq_cs] "m" (__rseq_abi.rseq_cs),
462 /* try store input */
463 [v2] "m" (*v2),
464 [newv2] "r" (newv2),
465 /* final store input */
466 [v] "m" (*v),
467 [expect] "r" (expect),
468 [newv] "r" (newv)
469 RSEQ_INJECT_INPUT
470 : "r0", "memory", "cc"
471 RSEQ_INJECT_CLOBBER
472 : abort, cmpfail
473 #ifdef RSEQ_COMPARE_TWICE
474 , error1, error2
475 #endif
476 );
477 rseq_workaround_gcc_asm_size_guess();
478 return 0;
479 abort:
480 rseq_workaround_gcc_asm_size_guess();
481 RSEQ_INJECT_FAILED
482 return -1;
483 cmpfail:
484 rseq_workaround_gcc_asm_size_guess();
485 return 1;
486 #ifdef RSEQ_COMPARE_TWICE
487 error1:
488 rseq_bug("cpu_id comparison failed");
489 error2:
490 rseq_bug("expected value comparison failed");
491 #endif
492 }
493
494 static inline __attribute__((always_inline))
rseq_cmpeqv_cmpeqv_storev(intptr_t * v,intptr_t expect,intptr_t * v2,intptr_t expect2,intptr_t newv,int cpu)495 int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
496 intptr_t *v2, intptr_t expect2,
497 intptr_t newv, int cpu)
498 {
499 RSEQ_INJECT_C(9)
500
501 rseq_workaround_gcc_asm_size_guess();
502 __asm__ __volatile__ goto (
503 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
504 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
505 #ifdef RSEQ_COMPARE_TWICE
506 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
507 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
508 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3])
509 #endif
510 /* Start rseq by storing table entry pointer into rseq_cs. */
511 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
512 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
513 RSEQ_INJECT_ASM(3)
514 "ldr r0, %[v]\n\t"
515 "cmp %[expect], r0\n\t"
516 "bne %l[cmpfail]\n\t"
517 RSEQ_INJECT_ASM(4)
518 "ldr r0, %[v2]\n\t"
519 "cmp %[expect2], r0\n\t"
520 "bne %l[cmpfail]\n\t"
521 RSEQ_INJECT_ASM(5)
522 #ifdef RSEQ_COMPARE_TWICE
523 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
524 "ldr r0, %[v]\n\t"
525 "cmp %[expect], r0\n\t"
526 "bne %l[error2]\n\t"
527 "ldr r0, %[v2]\n\t"
528 "cmp %[expect2], r0\n\t"
529 "bne %l[error3]\n\t"
530 #endif
531 /* final store */
532 "str %[newv], %[v]\n\t"
533 "2:\n\t"
534 RSEQ_INJECT_ASM(6)
535 "b 5f\n\t"
536 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
537 "5:\n\t"
538 : /* gcc asm goto does not allow outputs */
539 : [cpu_id] "r" (cpu),
540 [current_cpu_id] "m" (__rseq_abi.cpu_id),
541 [rseq_cs] "m" (__rseq_abi.rseq_cs),
542 /* cmp2 input */
543 [v2] "m" (*v2),
544 [expect2] "r" (expect2),
545 /* final store input */
546 [v] "m" (*v),
547 [expect] "r" (expect),
548 [newv] "r" (newv)
549 RSEQ_INJECT_INPUT
550 : "r0", "memory", "cc"
551 RSEQ_INJECT_CLOBBER
552 : abort, cmpfail
553 #ifdef RSEQ_COMPARE_TWICE
554 , error1, error2, error3
555 #endif
556 );
557 rseq_workaround_gcc_asm_size_guess();
558 return 0;
559 abort:
560 rseq_workaround_gcc_asm_size_guess();
561 RSEQ_INJECT_FAILED
562 return -1;
563 cmpfail:
564 rseq_workaround_gcc_asm_size_guess();
565 return 1;
566 #ifdef RSEQ_COMPARE_TWICE
567 error1:
568 rseq_bug("cpu_id comparison failed");
569 error2:
570 rseq_bug("1st expected value comparison failed");
571 error3:
572 rseq_bug("2nd expected value comparison failed");
573 #endif
574 }
575
576 static inline __attribute__((always_inline))
rseq_cmpeqv_trymemcpy_storev(intptr_t * v,intptr_t expect,void * dst,void * src,size_t len,intptr_t newv,int cpu)577 int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,
578 void *dst, void *src, size_t len,
579 intptr_t newv, int cpu)
580 {
581 uint32_t rseq_scratch[3];
582
583 RSEQ_INJECT_C(9)
584
585 rseq_workaround_gcc_asm_size_guess();
586 __asm__ __volatile__ goto (
587 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
588 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
589 #ifdef RSEQ_COMPARE_TWICE
590 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
591 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
592 #endif
593 "str %[src], %[rseq_scratch0]\n\t"
594 "str %[dst], %[rseq_scratch1]\n\t"
595 "str %[len], %[rseq_scratch2]\n\t"
596 /* Start rseq by storing table entry pointer into rseq_cs. */
597 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
598 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
599 RSEQ_INJECT_ASM(3)
600 "ldr r0, %[v]\n\t"
601 "cmp %[expect], r0\n\t"
602 "bne 5f\n\t"
603 RSEQ_INJECT_ASM(4)
604 #ifdef RSEQ_COMPARE_TWICE
605 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 6f)
606 "ldr r0, %[v]\n\t"
607 "cmp %[expect], r0\n\t"
608 "bne 7f\n\t"
609 #endif
610 /* try memcpy */
611 "cmp %[len], #0\n\t" \
612 "beq 333f\n\t" \
613 "222:\n\t" \
614 "ldrb %%r0, [%[src]]\n\t" \
615 "strb %%r0, [%[dst]]\n\t" \
616 "adds %[src], #1\n\t" \
617 "adds %[dst], #1\n\t" \
618 "subs %[len], #1\n\t" \
619 "bne 222b\n\t" \
620 "333:\n\t" \
621 RSEQ_INJECT_ASM(5)
622 /* final store */
623 "str %[newv], %[v]\n\t"
624 "2:\n\t"
625 RSEQ_INJECT_ASM(6)
626 /* teardown */
627 "ldr %[len], %[rseq_scratch2]\n\t"
628 "ldr %[dst], %[rseq_scratch1]\n\t"
629 "ldr %[src], %[rseq_scratch0]\n\t"
630 "b 8f\n\t"
631 RSEQ_ASM_DEFINE_ABORT(3, 4,
632 /* teardown */
633 "ldr %[len], %[rseq_scratch2]\n\t"
634 "ldr %[dst], %[rseq_scratch1]\n\t"
635 "ldr %[src], %[rseq_scratch0]\n\t",
636 abort, 1b, 2b, 4f)
637 RSEQ_ASM_DEFINE_CMPFAIL(5,
638 /* teardown */
639 "ldr %[len], %[rseq_scratch2]\n\t"
640 "ldr %[dst], %[rseq_scratch1]\n\t"
641 "ldr %[src], %[rseq_scratch0]\n\t",
642 cmpfail)
643 #ifdef RSEQ_COMPARE_TWICE
644 RSEQ_ASM_DEFINE_CMPFAIL(6,
645 /* teardown */
646 "ldr %[len], %[rseq_scratch2]\n\t"
647 "ldr %[dst], %[rseq_scratch1]\n\t"
648 "ldr %[src], %[rseq_scratch0]\n\t",
649 error1)
650 RSEQ_ASM_DEFINE_CMPFAIL(7,
651 /* teardown */
652 "ldr %[len], %[rseq_scratch2]\n\t"
653 "ldr %[dst], %[rseq_scratch1]\n\t"
654 "ldr %[src], %[rseq_scratch0]\n\t",
655 error2)
656 #endif
657 "8:\n\t"
658 : /* gcc asm goto does not allow outputs */
659 : [cpu_id] "r" (cpu),
660 [current_cpu_id] "m" (__rseq_abi.cpu_id),
661 [rseq_cs] "m" (__rseq_abi.rseq_cs),
662 /* final store input */
663 [v] "m" (*v),
664 [expect] "r" (expect),
665 [newv] "r" (newv),
666 /* try memcpy input */
667 [dst] "r" (dst),
668 [src] "r" (src),
669 [len] "r" (len),
670 [rseq_scratch0] "m" (rseq_scratch[0]),
671 [rseq_scratch1] "m" (rseq_scratch[1]),
672 [rseq_scratch2] "m" (rseq_scratch[2])
673 RSEQ_INJECT_INPUT
674 : "r0", "memory", "cc"
675 RSEQ_INJECT_CLOBBER
676 : abort, cmpfail
677 #ifdef RSEQ_COMPARE_TWICE
678 , error1, error2
679 #endif
680 );
681 rseq_workaround_gcc_asm_size_guess();
682 return 0;
683 abort:
684 rseq_workaround_gcc_asm_size_guess();
685 RSEQ_INJECT_FAILED
686 return -1;
687 cmpfail:
688 rseq_workaround_gcc_asm_size_guess();
689 return 1;
690 #ifdef RSEQ_COMPARE_TWICE
691 error1:
692 rseq_workaround_gcc_asm_size_guess();
693 rseq_bug("cpu_id comparison failed");
694 error2:
695 rseq_workaround_gcc_asm_size_guess();
696 rseq_bug("expected value comparison failed");
697 #endif
698 }
699
700 static inline __attribute__((always_inline))
rseq_cmpeqv_trymemcpy_storev_release(intptr_t * v,intptr_t expect,void * dst,void * src,size_t len,intptr_t newv,int cpu)701 int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect,
702 void *dst, void *src, size_t len,
703 intptr_t newv, int cpu)
704 {
705 uint32_t rseq_scratch[3];
706
707 RSEQ_INJECT_C(9)
708
709 rseq_workaround_gcc_asm_size_guess();
710 __asm__ __volatile__ goto (
711 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
712 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
713 #ifdef RSEQ_COMPARE_TWICE
714 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
715 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
716 #endif
717 "str %[src], %[rseq_scratch0]\n\t"
718 "str %[dst], %[rseq_scratch1]\n\t"
719 "str %[len], %[rseq_scratch2]\n\t"
720 /* Start rseq by storing table entry pointer into rseq_cs. */
721 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
722 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
723 RSEQ_INJECT_ASM(3)
724 "ldr r0, %[v]\n\t"
725 "cmp %[expect], r0\n\t"
726 "bne 5f\n\t"
727 RSEQ_INJECT_ASM(4)
728 #ifdef RSEQ_COMPARE_TWICE
729 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 6f)
730 "ldr r0, %[v]\n\t"
731 "cmp %[expect], r0\n\t"
732 "bne 7f\n\t"
733 #endif
734 /* try memcpy */
735 "cmp %[len], #0\n\t" \
736 "beq 333f\n\t" \
737 "222:\n\t" \
738 "ldrb %%r0, [%[src]]\n\t" \
739 "strb %%r0, [%[dst]]\n\t" \
740 "adds %[src], #1\n\t" \
741 "adds %[dst], #1\n\t" \
742 "subs %[len], #1\n\t" \
743 "bne 222b\n\t" \
744 "333:\n\t" \
745 RSEQ_INJECT_ASM(5)
746 "dmb\n\t" /* full mb provides store-release */
747 /* final store */
748 "str %[newv], %[v]\n\t"
749 "2:\n\t"
750 RSEQ_INJECT_ASM(6)
751 /* teardown */
752 "ldr %[len], %[rseq_scratch2]\n\t"
753 "ldr %[dst], %[rseq_scratch1]\n\t"
754 "ldr %[src], %[rseq_scratch0]\n\t"
755 "b 8f\n\t"
756 RSEQ_ASM_DEFINE_ABORT(3, 4,
757 /* teardown */
758 "ldr %[len], %[rseq_scratch2]\n\t"
759 "ldr %[dst], %[rseq_scratch1]\n\t"
760 "ldr %[src], %[rseq_scratch0]\n\t",
761 abort, 1b, 2b, 4f)
762 RSEQ_ASM_DEFINE_CMPFAIL(5,
763 /* teardown */
764 "ldr %[len], %[rseq_scratch2]\n\t"
765 "ldr %[dst], %[rseq_scratch1]\n\t"
766 "ldr %[src], %[rseq_scratch0]\n\t",
767 cmpfail)
768 #ifdef RSEQ_COMPARE_TWICE
769 RSEQ_ASM_DEFINE_CMPFAIL(6,
770 /* teardown */
771 "ldr %[len], %[rseq_scratch2]\n\t"
772 "ldr %[dst], %[rseq_scratch1]\n\t"
773 "ldr %[src], %[rseq_scratch0]\n\t",
774 error1)
775 RSEQ_ASM_DEFINE_CMPFAIL(7,
776 /* teardown */
777 "ldr %[len], %[rseq_scratch2]\n\t"
778 "ldr %[dst], %[rseq_scratch1]\n\t"
779 "ldr %[src], %[rseq_scratch0]\n\t",
780 error2)
781 #endif
782 "8:\n\t"
783 : /* gcc asm goto does not allow outputs */
784 : [cpu_id] "r" (cpu),
785 [current_cpu_id] "m" (__rseq_abi.cpu_id),
786 [rseq_cs] "m" (__rseq_abi.rseq_cs),
787 /* final store input */
788 [v] "m" (*v),
789 [expect] "r" (expect),
790 [newv] "r" (newv),
791 /* try memcpy input */
792 [dst] "r" (dst),
793 [src] "r" (src),
794 [len] "r" (len),
795 [rseq_scratch0] "m" (rseq_scratch[0]),
796 [rseq_scratch1] "m" (rseq_scratch[1]),
797 [rseq_scratch2] "m" (rseq_scratch[2])
798 RSEQ_INJECT_INPUT
799 : "r0", "memory", "cc"
800 RSEQ_INJECT_CLOBBER
801 : abort, cmpfail
802 #ifdef RSEQ_COMPARE_TWICE
803 , error1, error2
804 #endif
805 );
806 rseq_workaround_gcc_asm_size_guess();
807 return 0;
808 abort:
809 rseq_workaround_gcc_asm_size_guess();
810 RSEQ_INJECT_FAILED
811 return -1;
812 cmpfail:
813 rseq_workaround_gcc_asm_size_guess();
814 return 1;
815 #ifdef RSEQ_COMPARE_TWICE
816 error1:
817 rseq_workaround_gcc_asm_size_guess();
818 rseq_bug("cpu_id comparison failed");
819 error2:
820 rseq_workaround_gcc_asm_size_guess();
821 rseq_bug("expected value comparison failed");
822 #endif
823 }
824
825 #endif /* !RSEQ_SKIP_FASTPATH */
826