1 /* 2 * Simple interface for 128-bit atomic operations. 3 * 4 * Copyright (C) 2018 Linaro, Ltd. 5 * 6 * This work is licensed under the terms of the GNU GPL, version 2 or later. 7 * See the COPYING file in the top-level directory. 8 * 9 * See docs/devel/atomics.txt for discussion about the guarantees each 10 * atomic primitive is meant to provide. 11 */ 12 13 #ifndef QEMU_ATOMIC128_H 14 #define QEMU_ATOMIC128_H 15 16 #include "qemu/int128.h" 17 18 /* 19 * GCC is a house divided about supporting large atomic operations. 20 * 21 * For hosts that only have large compare-and-swap, a legalistic reading 22 * of the C++ standard means that one cannot implement __atomic_read on 23 * read-only memory, and thus all atomic operations must synchronize 24 * through libatomic. 25 * 26 * See https://gcc.gnu.org/bugzilla/show_bug.cgi?id=80878 27 * 28 * This interpretation is not especially helpful for QEMU. 29 * For softmmu, all RAM is always read/write from the hypervisor. 30 * For user-only, if the guest doesn't implement such an __atomic_read 31 * then the host need not worry about it either. 32 * 33 * Moreover, using libatomic is not an option, because its interface is 34 * built for std::atomic<T>, and requires that *all* accesses to such an 35 * object go through the library. In our case we do not have an object 36 * in the C/C++ sense, but a view of memory as seen by the guest. 37 * The guest may issue a large atomic operation and then access those 38 * pieces using word-sized accesses. From the hypervisor, we have no 39 * way to connect those two actions. 40 * 41 * Therefore, special case each platform. 42 */ 43 44 #if defined(CONFIG_ATOMIC128) 45 static inline Int128 atomic16_cmpxchg(Int128 *ptr, Int128 cmp, Int128 new) 46 { 47 return atomic_cmpxchg__nocheck(ptr, cmp, new); 48 } 49 # define HAVE_CMPXCHG128 1 50 #elif defined(CONFIG_CMPXCHG128) 51 static inline Int128 atomic16_cmpxchg(Int128 *ptr, Int128 cmp, Int128 new) 52 { 53 return __sync_val_compare_and_swap_16(ptr, cmp, new); 54 } 55 # define HAVE_CMPXCHG128 1 56 #elif defined(__aarch64__) 57 /* Through gcc 8, aarch64 has no support for 128-bit at all. */ 58 static inline Int128 atomic16_cmpxchg(Int128 *ptr, Int128 cmp, Int128 new) 59 { 60 uint64_t cmpl = int128_getlo(cmp), cmph = int128_gethi(cmp); 61 uint64_t newl = int128_getlo(new), newh = int128_gethi(new); 62 uint64_t oldl, oldh; 63 uint32_t tmp; 64 65 asm("0: ldaxp %[oldl], %[oldh], %[mem]\n\t" 66 "cmp %[oldl], %[cmpl]\n\t" 67 "ccmp %[oldh], %[cmph], #0, eq\n\t" 68 "b.ne 1f\n\t" 69 "stlxp %w[tmp], %[newl], %[newh], %[mem]\n\t" 70 "cbnz %w[tmp], 0b\n" 71 "1:" 72 : [mem] "+m"(*ptr), [tmp] "=&r"(tmp), 73 [oldl] "=&r"(oldl), [oldh] "=&r"(oldh) 74 : [cmpl] "r"(cmpl), [cmph] "r"(cmph), 75 [newl] "r"(newl), [newh] "r"(newh) 76 : "memory", "cc"); 77 78 return int128_make128(oldl, oldh); 79 } 80 # define HAVE_CMPXCHG128 1 81 #else 82 /* Fallback definition that must be optimized away, or error. */ 83 Int128 QEMU_ERROR("unsupported atomic") 84 atomic16_cmpxchg(Int128 *ptr, Int128 cmp, Int128 new); 85 # define HAVE_CMPXCHG128 0 86 #endif /* Some definition for HAVE_CMPXCHG128 */ 87 88 89 #if defined(CONFIG_ATOMIC128) 90 static inline Int128 atomic16_read(Int128 *ptr) 91 { 92 return atomic_read__nocheck(ptr); 93 } 94 95 static inline void atomic16_set(Int128 *ptr, Int128 val) 96 { 97 atomic_set__nocheck(ptr, val); 98 } 99 100 # define HAVE_ATOMIC128 1 101 #elif !defined(CONFIG_USER_ONLY) && defined(__aarch64__) 102 /* We can do better than cmpxchg for AArch64. */ 103 static inline Int128 atomic16_read(Int128 *ptr) 104 { 105 uint64_t l, h; 106 uint32_t tmp; 107 108 /* The load must be paired with the store to guarantee not tearing. */ 109 asm("0: ldxp %[l], %[h], %[mem]\n\t" 110 "stxp %w[tmp], %[l], %[h], %[mem]\n\t" 111 "cbnz %w[tmp], 0b" 112 : [mem] "+m"(*ptr), [tmp] "=r"(tmp), [l] "=r"(l), [h] "=r"(h)); 113 114 return int128_make128(l, h); 115 } 116 117 static inline void atomic16_set(Int128 *ptr, Int128 val) 118 { 119 uint64_t l = int128_getlo(val), h = int128_gethi(val); 120 uint64_t t1, t2; 121 122 /* Load into temporaries to acquire the exclusive access lock. */ 123 asm("0: ldxp %[t1], %[t2], %[mem]\n\t" 124 "stxp %w[t1], %[l], %[h], %[mem]\n\t" 125 "cbnz %w[t1], 0b" 126 : [mem] "+m"(*ptr), [t1] "=&r"(t1), [t2] "=&r"(t2) 127 : [l] "r"(l), [h] "r"(h)); 128 } 129 130 # define HAVE_ATOMIC128 1 131 #elif !defined(CONFIG_USER_ONLY) && HAVE_CMPXCHG128 132 static inline Int128 atomic16_read(Int128 *ptr) 133 { 134 /* Maybe replace 0 with 0, returning the old value. */ 135 return atomic16_cmpxchg(ptr, 0, 0); 136 } 137 138 static inline void atomic16_set(Int128 *ptr, Int128 val) 139 { 140 Int128 old = *ptr, cmp; 141 do { 142 cmp = old; 143 old = atomic16_cmpxchg(ptr, cmp, val); 144 } while (old != cmp); 145 } 146 147 # define HAVE_ATOMIC128 1 148 #else 149 /* Fallback definitions that must be optimized away, or error. */ 150 Int128 QEMU_ERROR("unsupported atomic") atomic16_read(Int128 *ptr); 151 void QEMU_ERROR("unsupported atomic") atomic16_set(Int128 *ptr, Int128 val); 152 # define HAVE_ATOMIC128 0 153 #endif /* Some definition for HAVE_ATOMIC128 */ 154 155 #endif /* QEMU_ATOMIC128_H */ 156