1 /* 2 * Atomic helper templates 3 * Included from tcg-runtime.c and cputlb.c. 4 * 5 * Copyright (c) 2016 Red Hat, Inc 6 * 7 * This library is free software; you can redistribute it and/or 8 * modify it under the terms of the GNU Lesser General Public 9 * License as published by the Free Software Foundation; either 10 * version 2.1 of the License, or (at your option) any later version. 11 * 12 * This library is distributed in the hope that it will be useful, 13 * but WITHOUT ANY WARRANTY; without even the implied warranty of 14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 15 * Lesser General Public License for more details. 16 * 17 * You should have received a copy of the GNU Lesser General Public 18 * License along with this library; if not, see <http://www.gnu.org/licenses/>. 19 */ 20 21 #include "qemu/plugin.h" 22 23 #if DATA_SIZE == 16 24 # define SUFFIX o 25 # define DATA_TYPE Int128 26 # define BSWAP bswap128 27 # define SHIFT 4 28 #elif DATA_SIZE == 8 29 # define SUFFIX q 30 # define DATA_TYPE aligned_uint64_t 31 # define SDATA_TYPE aligned_int64_t 32 # define BSWAP bswap64 33 # define SHIFT 3 34 #elif DATA_SIZE == 4 35 # define SUFFIX l 36 # define DATA_TYPE uint32_t 37 # define SDATA_TYPE int32_t 38 # define BSWAP bswap32 39 # define SHIFT 2 40 #elif DATA_SIZE == 2 41 # define SUFFIX w 42 # define DATA_TYPE uint16_t 43 # define SDATA_TYPE int16_t 44 # define BSWAP bswap16 45 # define SHIFT 1 46 #elif DATA_SIZE == 1 47 # define SUFFIX b 48 # define DATA_TYPE uint8_t 49 # define SDATA_TYPE int8_t 50 # define BSWAP 51 # define SHIFT 0 52 #else 53 # error unsupported data size 54 #endif 55 56 #if DATA_SIZE >= 4 57 # define ABI_TYPE DATA_TYPE 58 #else 59 # define ABI_TYPE uint32_t 60 #endif 61 62 /* Define host-endian atomic operations. Note that END is used within 63 the ATOMIC_NAME macro, and redefined below. */ 64 #if DATA_SIZE == 1 65 # define END 66 #elif defined(HOST_WORDS_BIGENDIAN) 67 # define END _be 68 #else 69 # define END _le 70 #endif 71 72 ABI_TYPE ATOMIC_NAME(cmpxchg)(CPUArchState *env, target_ulong addr, 73 ABI_TYPE cmpv, ABI_TYPE newv, 74 MemOpIdx oi, uintptr_t retaddr) 75 { 76 DATA_TYPE *haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE, 77 PAGE_READ | PAGE_WRITE, retaddr); 78 DATA_TYPE ret; 79 80 atomic_trace_rmw_pre(env, addr, oi); 81 #if DATA_SIZE == 16 82 ret = atomic16_cmpxchg(haddr, cmpv, newv); 83 #else 84 ret = qatomic_cmpxchg__nocheck(haddr, cmpv, newv); 85 #endif 86 ATOMIC_MMU_CLEANUP; 87 atomic_trace_rmw_post(env, addr, oi); 88 return ret; 89 } 90 91 #if DATA_SIZE >= 16 92 #if HAVE_ATOMIC128 93 ABI_TYPE ATOMIC_NAME(ld)(CPUArchState *env, target_ulong addr, 94 MemOpIdx oi, uintptr_t retaddr) 95 { 96 DATA_TYPE *haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE, 97 PAGE_READ, retaddr); 98 DATA_TYPE val; 99 100 atomic_trace_ld_pre(env, addr, oi); 101 val = atomic16_read(haddr); 102 ATOMIC_MMU_CLEANUP; 103 atomic_trace_ld_post(env, addr, oi); 104 return val; 105 } 106 107 void ATOMIC_NAME(st)(CPUArchState *env, target_ulong addr, ABI_TYPE val, 108 MemOpIdx oi, uintptr_t retaddr) 109 { 110 DATA_TYPE *haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE, 111 PAGE_WRITE, retaddr); 112 113 atomic_trace_st_pre(env, addr, oi); 114 atomic16_set(haddr, val); 115 ATOMIC_MMU_CLEANUP; 116 atomic_trace_st_post(env, addr, oi); 117 } 118 #endif 119 #else 120 ABI_TYPE ATOMIC_NAME(xchg)(CPUArchState *env, target_ulong addr, ABI_TYPE val, 121 MemOpIdx oi, uintptr_t retaddr) 122 { 123 DATA_TYPE *haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE, 124 PAGE_READ | PAGE_WRITE, retaddr); 125 DATA_TYPE ret; 126 127 atomic_trace_rmw_pre(env, addr, oi); 128 ret = qatomic_xchg__nocheck(haddr, val); 129 ATOMIC_MMU_CLEANUP; 130 atomic_trace_rmw_post(env, addr, oi); 131 return ret; 132 } 133 134 #define GEN_ATOMIC_HELPER(X) \ 135 ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr, \ 136 ABI_TYPE val, MemOpIdx oi, uintptr_t retaddr) \ 137 { \ 138 DATA_TYPE *haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE, \ 139 PAGE_READ | PAGE_WRITE, retaddr); \ 140 DATA_TYPE ret; \ 141 atomic_trace_rmw_pre(env, addr, oi); \ 142 ret = qatomic_##X(haddr, val); \ 143 ATOMIC_MMU_CLEANUP; \ 144 atomic_trace_rmw_post(env, addr, oi); \ 145 return ret; \ 146 } 147 148 GEN_ATOMIC_HELPER(fetch_add) 149 GEN_ATOMIC_HELPER(fetch_and) 150 GEN_ATOMIC_HELPER(fetch_or) 151 GEN_ATOMIC_HELPER(fetch_xor) 152 GEN_ATOMIC_HELPER(add_fetch) 153 GEN_ATOMIC_HELPER(and_fetch) 154 GEN_ATOMIC_HELPER(or_fetch) 155 GEN_ATOMIC_HELPER(xor_fetch) 156 157 #undef GEN_ATOMIC_HELPER 158 159 /* 160 * These helpers are, as a whole, full barriers. Within the helper, 161 * the leading barrier is explicit and the trailing barrier is within 162 * cmpxchg primitive. 163 * 164 * Trace this load + RMW loop as a single RMW op. This way, regardless 165 * of CF_PARALLEL's value, we'll trace just a read and a write. 166 */ 167 #define GEN_ATOMIC_HELPER_FN(X, FN, XDATA_TYPE, RET) \ 168 ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr, \ 169 ABI_TYPE xval, MemOpIdx oi, uintptr_t retaddr) \ 170 { \ 171 XDATA_TYPE *haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE, \ 172 PAGE_READ | PAGE_WRITE, retaddr); \ 173 XDATA_TYPE cmp, old, new, val = xval; \ 174 atomic_trace_rmw_pre(env, addr, oi); \ 175 smp_mb(); \ 176 cmp = qatomic_read__nocheck(haddr); \ 177 do { \ 178 old = cmp; new = FN(old, val); \ 179 cmp = qatomic_cmpxchg__nocheck(haddr, old, new); \ 180 } while (cmp != old); \ 181 ATOMIC_MMU_CLEANUP; \ 182 atomic_trace_rmw_post(env, addr, oi); \ 183 return RET; \ 184 } 185 186 GEN_ATOMIC_HELPER_FN(fetch_smin, MIN, SDATA_TYPE, old) 187 GEN_ATOMIC_HELPER_FN(fetch_umin, MIN, DATA_TYPE, old) 188 GEN_ATOMIC_HELPER_FN(fetch_smax, MAX, SDATA_TYPE, old) 189 GEN_ATOMIC_HELPER_FN(fetch_umax, MAX, DATA_TYPE, old) 190 191 GEN_ATOMIC_HELPER_FN(smin_fetch, MIN, SDATA_TYPE, new) 192 GEN_ATOMIC_HELPER_FN(umin_fetch, MIN, DATA_TYPE, new) 193 GEN_ATOMIC_HELPER_FN(smax_fetch, MAX, SDATA_TYPE, new) 194 GEN_ATOMIC_HELPER_FN(umax_fetch, MAX, DATA_TYPE, new) 195 196 #undef GEN_ATOMIC_HELPER_FN 197 #endif /* DATA SIZE >= 16 */ 198 199 #undef END 200 201 #if DATA_SIZE > 1 202 203 /* Define reverse-host-endian atomic operations. Note that END is used 204 within the ATOMIC_NAME macro. */ 205 #ifdef HOST_WORDS_BIGENDIAN 206 # define END _le 207 #else 208 # define END _be 209 #endif 210 211 ABI_TYPE ATOMIC_NAME(cmpxchg)(CPUArchState *env, target_ulong addr, 212 ABI_TYPE cmpv, ABI_TYPE newv, 213 MemOpIdx oi, uintptr_t retaddr) 214 { 215 DATA_TYPE *haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE, 216 PAGE_READ | PAGE_WRITE, retaddr); 217 DATA_TYPE ret; 218 219 atomic_trace_rmw_pre(env, addr, oi); 220 #if DATA_SIZE == 16 221 ret = atomic16_cmpxchg(haddr, BSWAP(cmpv), BSWAP(newv)); 222 #else 223 ret = qatomic_cmpxchg__nocheck(haddr, BSWAP(cmpv), BSWAP(newv)); 224 #endif 225 ATOMIC_MMU_CLEANUP; 226 atomic_trace_rmw_post(env, addr, oi); 227 return BSWAP(ret); 228 } 229 230 #if DATA_SIZE >= 16 231 #if HAVE_ATOMIC128 232 ABI_TYPE ATOMIC_NAME(ld)(CPUArchState *env, target_ulong addr, 233 MemOpIdx oi, uintptr_t retaddr) 234 { 235 DATA_TYPE *haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE, 236 PAGE_READ, retaddr); 237 DATA_TYPE val; 238 239 atomic_trace_ld_pre(env, addr, oi); 240 val = atomic16_read(haddr); 241 ATOMIC_MMU_CLEANUP; 242 atomic_trace_ld_post(env, addr, oi); 243 return BSWAP(val); 244 } 245 246 void ATOMIC_NAME(st)(CPUArchState *env, target_ulong addr, ABI_TYPE val, 247 MemOpIdx oi, uintptr_t retaddr) 248 { 249 DATA_TYPE *haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE, 250 PAGE_WRITE, retaddr); 251 252 atomic_trace_st_pre(env, addr, oi); 253 val = BSWAP(val); 254 atomic16_set(haddr, val); 255 ATOMIC_MMU_CLEANUP; 256 atomic_trace_st_post(env, addr, oi); 257 } 258 #endif 259 #else 260 ABI_TYPE ATOMIC_NAME(xchg)(CPUArchState *env, target_ulong addr, ABI_TYPE val, 261 MemOpIdx oi, uintptr_t retaddr) 262 { 263 DATA_TYPE *haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE, 264 PAGE_READ | PAGE_WRITE, retaddr); 265 ABI_TYPE ret; 266 267 atomic_trace_rmw_pre(env, addr, oi); 268 ret = qatomic_xchg__nocheck(haddr, BSWAP(val)); 269 ATOMIC_MMU_CLEANUP; 270 atomic_trace_rmw_post(env, addr, oi); 271 return BSWAP(ret); 272 } 273 274 #define GEN_ATOMIC_HELPER(X) \ 275 ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr, \ 276 ABI_TYPE val, MemOpIdx oi, uintptr_t retaddr) \ 277 { \ 278 DATA_TYPE *haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE, \ 279 PAGE_READ | PAGE_WRITE, retaddr); \ 280 DATA_TYPE ret; \ 281 atomic_trace_rmw_pre(env, addr, oi); \ 282 ret = qatomic_##X(haddr, BSWAP(val)); \ 283 ATOMIC_MMU_CLEANUP; \ 284 atomic_trace_rmw_post(env, addr, oi); \ 285 return BSWAP(ret); \ 286 } 287 288 GEN_ATOMIC_HELPER(fetch_and) 289 GEN_ATOMIC_HELPER(fetch_or) 290 GEN_ATOMIC_HELPER(fetch_xor) 291 GEN_ATOMIC_HELPER(and_fetch) 292 GEN_ATOMIC_HELPER(or_fetch) 293 GEN_ATOMIC_HELPER(xor_fetch) 294 295 #undef GEN_ATOMIC_HELPER 296 297 /* These helpers are, as a whole, full barriers. Within the helper, 298 * the leading barrier is explicit and the trailing barrier is within 299 * cmpxchg primitive. 300 * 301 * Trace this load + RMW loop as a single RMW op. This way, regardless 302 * of CF_PARALLEL's value, we'll trace just a read and a write. 303 */ 304 #define GEN_ATOMIC_HELPER_FN(X, FN, XDATA_TYPE, RET) \ 305 ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr, \ 306 ABI_TYPE xval, MemOpIdx oi, uintptr_t retaddr) \ 307 { \ 308 XDATA_TYPE *haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE, \ 309 PAGE_READ | PAGE_WRITE, retaddr); \ 310 XDATA_TYPE ldo, ldn, old, new, val = xval; \ 311 atomic_trace_rmw_pre(env, addr, oi); \ 312 smp_mb(); \ 313 ldn = qatomic_read__nocheck(haddr); \ 314 do { \ 315 ldo = ldn; old = BSWAP(ldo); new = FN(old, val); \ 316 ldn = qatomic_cmpxchg__nocheck(haddr, ldo, BSWAP(new)); \ 317 } while (ldo != ldn); \ 318 ATOMIC_MMU_CLEANUP; \ 319 atomic_trace_rmw_post(env, addr, oi); \ 320 return RET; \ 321 } 322 323 GEN_ATOMIC_HELPER_FN(fetch_smin, MIN, SDATA_TYPE, old) 324 GEN_ATOMIC_HELPER_FN(fetch_umin, MIN, DATA_TYPE, old) 325 GEN_ATOMIC_HELPER_FN(fetch_smax, MAX, SDATA_TYPE, old) 326 GEN_ATOMIC_HELPER_FN(fetch_umax, MAX, DATA_TYPE, old) 327 328 GEN_ATOMIC_HELPER_FN(smin_fetch, MIN, SDATA_TYPE, new) 329 GEN_ATOMIC_HELPER_FN(umin_fetch, MIN, DATA_TYPE, new) 330 GEN_ATOMIC_HELPER_FN(smax_fetch, MAX, SDATA_TYPE, new) 331 GEN_ATOMIC_HELPER_FN(umax_fetch, MAX, DATA_TYPE, new) 332 333 /* Note that for addition, we need to use a separate cmpxchg loop instead 334 of bswaps for the reverse-host-endian helpers. */ 335 #define ADD(X, Y) (X + Y) 336 GEN_ATOMIC_HELPER_FN(fetch_add, ADD, DATA_TYPE, old) 337 GEN_ATOMIC_HELPER_FN(add_fetch, ADD, DATA_TYPE, new) 338 #undef ADD 339 340 #undef GEN_ATOMIC_HELPER_FN 341 #endif /* DATA_SIZE >= 16 */ 342 343 #undef END 344 #endif /* DATA_SIZE > 1 */ 345 346 #undef BSWAP 347 #undef ABI_TYPE 348 #undef DATA_TYPE 349 #undef SDATA_TYPE 350 #undef SUFFIX 351 #undef DATA_SIZE 352 #undef SHIFT 353