1 /*
2  * SPDX-License-Identifier: GPL-2.0-or-later
3  * Load/store for 128-bit atomic operations, AArch64 version.
4  *
5  * Copyright (C) 2018, 2023 Linaro, Ltd.
6  *
7  * See docs/devel/atomics.rst for discussion about the guarantees each
8  * atomic primitive is meant to provide.
9  */
10 
11 #ifndef AARCH64_ATOMIC128_LDST_H
12 #define AARCH64_ATOMIC128_LDST_H
13 
14 #include "host/cpuinfo.h"
15 #include "tcg/debug-assert.h"
16 
17 /*
18  * Through gcc 10, aarch64 has no support for 128-bit atomics.
19  * Through clang 16, without -march=armv8.4-a, __atomic_load_16
20  * is incorrectly expanded to a read-write operation.
21  *
22  * Anyway, this method allows runtime detection of FEAT_LSE2.
23  */
24 
25 #define HAVE_ATOMIC128_RO (cpuinfo & CPUINFO_LSE2)
26 #define HAVE_ATOMIC128_RW 1
27 
28 static inline Int128 atomic16_read_ro(const Int128 *ptr)
29 {
30     uint64_t l, h;
31 
32     tcg_debug_assert(HAVE_ATOMIC128_RO);
33     /* With FEAT_LSE2, 16-byte aligned LDP is atomic. */
34     asm("ldp %[l], %[h], %[mem]"
35         : [l] "=r"(l), [h] "=r"(h) : [mem] "m"(*ptr));
36 
37     return int128_make128(l, h);
38 }
39 
40 static inline Int128 atomic16_read_rw(Int128 *ptr)
41 {
42     uint64_t l, h;
43     uint32_t tmp;
44 
45     if (cpuinfo & CPUINFO_LSE2) {
46         /* With FEAT_LSE2, 16-byte aligned LDP is atomic. */
47         asm("ldp %[l], %[h], %[mem]"
48             : [l] "=r"(l), [h] "=r"(h) : [mem] "m"(*ptr));
49     } else {
50         /* The load must be paired with the store to guarantee not tearing.  */
51         asm("0: ldxp %[l], %[h], %[mem]\n\t"
52             "stxp %w[tmp], %[l], %[h], %[mem]\n\t"
53             "cbnz %w[tmp], 0b"
54             : [mem] "+m"(*ptr), [tmp] "=&r"(tmp), [l] "=&r"(l), [h] "=&r"(h));
55     }
56 
57     return int128_make128(l, h);
58 }
59 
60 static inline void atomic16_set(Int128 *ptr, Int128 val)
61 {
62     uint64_t l = int128_getlo(val), h = int128_gethi(val);
63     uint64_t t1, t2;
64 
65     if (cpuinfo & CPUINFO_LSE2) {
66         /* With FEAT_LSE2, 16-byte aligned STP is atomic. */
67         asm("stp %[l], %[h], %[mem]"
68             : [mem] "=m"(*ptr) : [l] "r"(l), [h] "r"(h));
69     } else {
70         /* Load into temporaries to acquire the exclusive access lock.  */
71         asm("0: ldxp %[t1], %[t2], %[mem]\n\t"
72             "stxp %w[t1], %[l], %[h], %[mem]\n\t"
73             "cbnz %w[t1], 0b"
74             : [mem] "+m"(*ptr), [t1] "=&r"(t1), [t2] "=&r"(t2)
75             : [l] "r"(l), [h] "r"(h));
76     }
77 }
78 
79 #endif /* AARCH64_ATOMIC128_LDST_H */
80