1 /* 2 * Simple interface for 128-bit atomic operations. 3 * 4 * Copyright (C) 2018 Linaro, Ltd. 5 * 6 * This work is licensed under the terms of the GNU GPL, version 2 or later. 7 * See the COPYING file in the top-level directory. 8 * 9 * See docs/devel/atomics.rst for discussion about the guarantees each 10 * atomic primitive is meant to provide. 11 */ 12 13 #ifndef QEMU_ATOMIC128_H 14 #define QEMU_ATOMIC128_H 15 16 #include "qemu/int128.h" 17 18 /* 19 * GCC is a house divided about supporting large atomic operations. 20 * 21 * For hosts that only have large compare-and-swap, a legalistic reading 22 * of the C++ standard means that one cannot implement __atomic_read on 23 * read-only memory, and thus all atomic operations must synchronize 24 * through libatomic. 25 * 26 * See https://gcc.gnu.org/bugzilla/show_bug.cgi?id=80878 27 * 28 * This interpretation is not especially helpful for QEMU. 29 * For softmmu, all RAM is always read/write from the hypervisor. 30 * For user-only, if the guest doesn't implement such an __atomic_read 31 * then the host need not worry about it either. 32 * 33 * Moreover, using libatomic is not an option, because its interface is 34 * built for std::atomic<T>, and requires that *all* accesses to such an 35 * object go through the library. In our case we do not have an object 36 * in the C/C++ sense, but a view of memory as seen by the guest. 37 * The guest may issue a large atomic operation and then access those 38 * pieces using word-sized accesses. From the hypervisor, we have no 39 * way to connect those two actions. 40 * 41 * Therefore, special case each platform. 42 */ 43 44 #if defined(CONFIG_ATOMIC128) 45 static inline Int128 atomic16_cmpxchg(Int128 *ptr, Int128 cmp, Int128 new) 46 { 47 Int128Alias r, c, n; 48 49 c.s = cmp; 50 n.s = new; 51 r.i = qatomic_cmpxchg__nocheck((__int128_t *)ptr, c.i, n.i); 52 return r.s; 53 } 54 # define HAVE_CMPXCHG128 1 55 #elif defined(CONFIG_CMPXCHG128) 56 static inline Int128 atomic16_cmpxchg(Int128 *ptr, Int128 cmp, Int128 new) 57 { 58 Int128Alias r, c, n; 59 60 c.s = cmp; 61 n.s = new; 62 r.i = __sync_val_compare_and_swap_16((__int128_t *)ptr, c.i, n.i); 63 return r.s; 64 } 65 # define HAVE_CMPXCHG128 1 66 #elif defined(__aarch64__) 67 /* Through gcc 8, aarch64 has no support for 128-bit at all. */ 68 static inline Int128 atomic16_cmpxchg(Int128 *ptr, Int128 cmp, Int128 new) 69 { 70 uint64_t cmpl = int128_getlo(cmp), cmph = int128_gethi(cmp); 71 uint64_t newl = int128_getlo(new), newh = int128_gethi(new); 72 uint64_t oldl, oldh; 73 uint32_t tmp; 74 75 asm("0: ldaxp %[oldl], %[oldh], %[mem]\n\t" 76 "cmp %[oldl], %[cmpl]\n\t" 77 "ccmp %[oldh], %[cmph], #0, eq\n\t" 78 "b.ne 1f\n\t" 79 "stlxp %w[tmp], %[newl], %[newh], %[mem]\n\t" 80 "cbnz %w[tmp], 0b\n" 81 "1:" 82 : [mem] "+m"(*ptr), [tmp] "=&r"(tmp), 83 [oldl] "=&r"(oldl), [oldh] "=&r"(oldh) 84 : [cmpl] "r"(cmpl), [cmph] "r"(cmph), 85 [newl] "r"(newl), [newh] "r"(newh) 86 : "memory", "cc"); 87 88 return int128_make128(oldl, oldh); 89 } 90 # define HAVE_CMPXCHG128 1 91 #else 92 /* Fallback definition that must be optimized away, or error. */ 93 Int128 QEMU_ERROR("unsupported atomic") 94 atomic16_cmpxchg(Int128 *ptr, Int128 cmp, Int128 new); 95 # define HAVE_CMPXCHG128 0 96 #endif /* Some definition for HAVE_CMPXCHG128 */ 97 98 99 #if defined(CONFIG_ATOMIC128) 100 static inline Int128 atomic16_read(Int128 *ptr) 101 { 102 Int128Alias r; 103 104 r.i = qatomic_read__nocheck((__int128_t *)ptr); 105 return r.s; 106 } 107 108 static inline void atomic16_set(Int128 *ptr, Int128 val) 109 { 110 Int128Alias v; 111 112 v.s = val; 113 qatomic_set__nocheck((__int128_t *)ptr, v.i); 114 } 115 116 # define HAVE_ATOMIC128 1 117 #elif !defined(CONFIG_USER_ONLY) && defined(__aarch64__) 118 /* We can do better than cmpxchg for AArch64. */ 119 static inline Int128 atomic16_read(Int128 *ptr) 120 { 121 uint64_t l, h; 122 uint32_t tmp; 123 124 /* The load must be paired with the store to guarantee not tearing. */ 125 asm("0: ldxp %[l], %[h], %[mem]\n\t" 126 "stxp %w[tmp], %[l], %[h], %[mem]\n\t" 127 "cbnz %w[tmp], 0b" 128 : [mem] "+m"(*ptr), [tmp] "=r"(tmp), [l] "=r"(l), [h] "=r"(h)); 129 130 return int128_make128(l, h); 131 } 132 133 static inline void atomic16_set(Int128 *ptr, Int128 val) 134 { 135 uint64_t l = int128_getlo(val), h = int128_gethi(val); 136 uint64_t t1, t2; 137 138 /* Load into temporaries to acquire the exclusive access lock. */ 139 asm("0: ldxp %[t1], %[t2], %[mem]\n\t" 140 "stxp %w[t1], %[l], %[h], %[mem]\n\t" 141 "cbnz %w[t1], 0b" 142 : [mem] "+m"(*ptr), [t1] "=&r"(t1), [t2] "=&r"(t2) 143 : [l] "r"(l), [h] "r"(h)); 144 } 145 146 # define HAVE_ATOMIC128 1 147 #elif !defined(CONFIG_USER_ONLY) && HAVE_CMPXCHG128 148 static inline Int128 atomic16_read(Int128 *ptr) 149 { 150 /* Maybe replace 0 with 0, returning the old value. */ 151 Int128 z = int128_make64(0); 152 return atomic16_cmpxchg(ptr, z, z); 153 } 154 155 static inline void atomic16_set(Int128 *ptr, Int128 val) 156 { 157 Int128 old = *ptr, cmp; 158 do { 159 cmp = old; 160 old = atomic16_cmpxchg(ptr, cmp, val); 161 } while (int128_ne(old, cmp)); 162 } 163 164 # define HAVE_ATOMIC128 1 165 #else 166 /* Fallback definitions that must be optimized away, or error. */ 167 Int128 QEMU_ERROR("unsupported atomic") atomic16_read(Int128 *ptr); 168 void QEMU_ERROR("unsupported atomic") atomic16_set(Int128 *ptr, Int128 val); 169 # define HAVE_ATOMIC128 0 170 #endif /* Some definition for HAVE_ATOMIC128 */ 171 172 #endif /* QEMU_ATOMIC128_H */ 173