1 /* 2 * Atomic helper templates 3 * Included from tcg-runtime.c and cputlb.c. 4 * 5 * Copyright (c) 2016 Red Hat, Inc 6 * 7 * This library is free software; you can redistribute it and/or 8 * modify it under the terms of the GNU Lesser General Public 9 * License as published by the Free Software Foundation; either 10 * version 2.1 of the License, or (at your option) any later version. 11 * 12 * This library is distributed in the hope that it will be useful, 13 * but WITHOUT ANY WARRANTY; without even the implied warranty of 14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 15 * Lesser General Public License for more details. 16 * 17 * You should have received a copy of the GNU Lesser General Public 18 * License along with this library; if not, see <http://www.gnu.org/licenses/>. 19 */ 20 21 #include "qemu/plugin.h" 22 #include "trace/mem.h" 23 24 #if DATA_SIZE == 16 25 # define SUFFIX o 26 # define DATA_TYPE Int128 27 # define BSWAP bswap128 28 # define SHIFT 4 29 #elif DATA_SIZE == 8 30 # define SUFFIX q 31 # define DATA_TYPE aligned_uint64_t 32 # define SDATA_TYPE aligned_int64_t 33 # define BSWAP bswap64 34 # define SHIFT 3 35 #elif DATA_SIZE == 4 36 # define SUFFIX l 37 # define DATA_TYPE uint32_t 38 # define SDATA_TYPE int32_t 39 # define BSWAP bswap32 40 # define SHIFT 2 41 #elif DATA_SIZE == 2 42 # define SUFFIX w 43 # define DATA_TYPE uint16_t 44 # define SDATA_TYPE int16_t 45 # define BSWAP bswap16 46 # define SHIFT 1 47 #elif DATA_SIZE == 1 48 # define SUFFIX b 49 # define DATA_TYPE uint8_t 50 # define SDATA_TYPE int8_t 51 # define BSWAP 52 # define SHIFT 0 53 #else 54 # error unsupported data size 55 #endif 56 57 #if DATA_SIZE >= 4 58 # define ABI_TYPE DATA_TYPE 59 #else 60 # define ABI_TYPE uint32_t 61 #endif 62 63 /* Define host-endian atomic operations. Note that END is used within 64 the ATOMIC_NAME macro, and redefined below. */ 65 #if DATA_SIZE == 1 66 # define END 67 #elif defined(HOST_WORDS_BIGENDIAN) 68 # define END _be 69 #else 70 # define END _le 71 #endif 72 73 ABI_TYPE ATOMIC_NAME(cmpxchg)(CPUArchState *env, target_ulong addr, 74 ABI_TYPE cmpv, ABI_TYPE newv, 75 TCGMemOpIdx oi, uintptr_t retaddr) 76 { 77 DATA_TYPE *haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE, 78 PAGE_READ | PAGE_WRITE, retaddr); 79 DATA_TYPE ret; 80 uint16_t info = trace_mem_build_info(SHIFT, false, 0, false, 81 ATOMIC_MMU_IDX); 82 83 atomic_trace_rmw_pre(env, addr, info); 84 #if DATA_SIZE == 16 85 ret = atomic16_cmpxchg(haddr, cmpv, newv); 86 #else 87 ret = qatomic_cmpxchg__nocheck(haddr, cmpv, newv); 88 #endif 89 ATOMIC_MMU_CLEANUP; 90 atomic_trace_rmw_post(env, addr, info); 91 return ret; 92 } 93 94 #if DATA_SIZE >= 16 95 #if HAVE_ATOMIC128 96 ABI_TYPE ATOMIC_NAME(ld)(CPUArchState *env, target_ulong addr, 97 TCGMemOpIdx oi, uintptr_t retaddr) 98 { 99 DATA_TYPE *haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE, 100 PAGE_READ, retaddr); 101 DATA_TYPE val; 102 uint16_t info = trace_mem_build_info(SHIFT, false, 0, false, 103 ATOMIC_MMU_IDX); 104 105 atomic_trace_ld_pre(env, addr, info); 106 val = atomic16_read(haddr); 107 ATOMIC_MMU_CLEANUP; 108 atomic_trace_ld_post(env, addr, info); 109 return val; 110 } 111 112 void ATOMIC_NAME(st)(CPUArchState *env, target_ulong addr, ABI_TYPE val, 113 TCGMemOpIdx oi, uintptr_t retaddr) 114 { 115 DATA_TYPE *haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE, 116 PAGE_WRITE, retaddr); 117 uint16_t info = trace_mem_build_info(SHIFT, false, 0, true, 118 ATOMIC_MMU_IDX); 119 120 atomic_trace_st_pre(env, addr, info); 121 atomic16_set(haddr, val); 122 ATOMIC_MMU_CLEANUP; 123 atomic_trace_st_post(env, addr, info); 124 } 125 #endif 126 #else 127 ABI_TYPE ATOMIC_NAME(xchg)(CPUArchState *env, target_ulong addr, ABI_TYPE val, 128 TCGMemOpIdx oi, uintptr_t retaddr) 129 { 130 DATA_TYPE *haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE, 131 PAGE_READ | PAGE_WRITE, retaddr); 132 DATA_TYPE ret; 133 uint16_t info = trace_mem_build_info(SHIFT, false, 0, false, 134 ATOMIC_MMU_IDX); 135 136 atomic_trace_rmw_pre(env, addr, info); 137 ret = qatomic_xchg__nocheck(haddr, val); 138 ATOMIC_MMU_CLEANUP; 139 atomic_trace_rmw_post(env, addr, info); 140 return ret; 141 } 142 143 #define GEN_ATOMIC_HELPER(X) \ 144 ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr, \ 145 ABI_TYPE val, TCGMemOpIdx oi, uintptr_t retaddr) \ 146 { \ 147 DATA_TYPE *haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE, \ 148 PAGE_READ | PAGE_WRITE, retaddr); \ 149 DATA_TYPE ret; \ 150 uint16_t info = trace_mem_build_info(SHIFT, false, 0, false, \ 151 ATOMIC_MMU_IDX); \ 152 atomic_trace_rmw_pre(env, addr, info); \ 153 ret = qatomic_##X(haddr, val); \ 154 ATOMIC_MMU_CLEANUP; \ 155 atomic_trace_rmw_post(env, addr, info); \ 156 return ret; \ 157 } 158 159 GEN_ATOMIC_HELPER(fetch_add) 160 GEN_ATOMIC_HELPER(fetch_and) 161 GEN_ATOMIC_HELPER(fetch_or) 162 GEN_ATOMIC_HELPER(fetch_xor) 163 GEN_ATOMIC_HELPER(add_fetch) 164 GEN_ATOMIC_HELPER(and_fetch) 165 GEN_ATOMIC_HELPER(or_fetch) 166 GEN_ATOMIC_HELPER(xor_fetch) 167 168 #undef GEN_ATOMIC_HELPER 169 170 /* 171 * These helpers are, as a whole, full barriers. Within the helper, 172 * the leading barrier is explicit and the trailing barrier is within 173 * cmpxchg primitive. 174 * 175 * Trace this load + RMW loop as a single RMW op. This way, regardless 176 * of CF_PARALLEL's value, we'll trace just a read and a write. 177 */ 178 #define GEN_ATOMIC_HELPER_FN(X, FN, XDATA_TYPE, RET) \ 179 ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr, \ 180 ABI_TYPE xval, TCGMemOpIdx oi, uintptr_t retaddr) \ 181 { \ 182 XDATA_TYPE *haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE, \ 183 PAGE_READ | PAGE_WRITE, retaddr); \ 184 XDATA_TYPE cmp, old, new, val = xval; \ 185 uint16_t info = trace_mem_build_info(SHIFT, false, 0, false, \ 186 ATOMIC_MMU_IDX); \ 187 atomic_trace_rmw_pre(env, addr, info); \ 188 smp_mb(); \ 189 cmp = qatomic_read__nocheck(haddr); \ 190 do { \ 191 old = cmp; new = FN(old, val); \ 192 cmp = qatomic_cmpxchg__nocheck(haddr, old, new); \ 193 } while (cmp != old); \ 194 ATOMIC_MMU_CLEANUP; \ 195 atomic_trace_rmw_post(env, addr, info); \ 196 return RET; \ 197 } 198 199 GEN_ATOMIC_HELPER_FN(fetch_smin, MIN, SDATA_TYPE, old) 200 GEN_ATOMIC_HELPER_FN(fetch_umin, MIN, DATA_TYPE, old) 201 GEN_ATOMIC_HELPER_FN(fetch_smax, MAX, SDATA_TYPE, old) 202 GEN_ATOMIC_HELPER_FN(fetch_umax, MAX, DATA_TYPE, old) 203 204 GEN_ATOMIC_HELPER_FN(smin_fetch, MIN, SDATA_TYPE, new) 205 GEN_ATOMIC_HELPER_FN(umin_fetch, MIN, DATA_TYPE, new) 206 GEN_ATOMIC_HELPER_FN(smax_fetch, MAX, SDATA_TYPE, new) 207 GEN_ATOMIC_HELPER_FN(umax_fetch, MAX, DATA_TYPE, new) 208 209 #undef GEN_ATOMIC_HELPER_FN 210 #endif /* DATA SIZE >= 16 */ 211 212 #undef END 213 214 #if DATA_SIZE > 1 215 216 /* Define reverse-host-endian atomic operations. Note that END is used 217 within the ATOMIC_NAME macro. */ 218 #ifdef HOST_WORDS_BIGENDIAN 219 # define END _le 220 #else 221 # define END _be 222 #endif 223 224 ABI_TYPE ATOMIC_NAME(cmpxchg)(CPUArchState *env, target_ulong addr, 225 ABI_TYPE cmpv, ABI_TYPE newv, 226 TCGMemOpIdx oi, uintptr_t retaddr) 227 { 228 DATA_TYPE *haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE, 229 PAGE_READ | PAGE_WRITE, retaddr); 230 DATA_TYPE ret; 231 uint16_t info = trace_mem_build_info(SHIFT, false, MO_BSWAP, false, 232 ATOMIC_MMU_IDX); 233 234 atomic_trace_rmw_pre(env, addr, info); 235 #if DATA_SIZE == 16 236 ret = atomic16_cmpxchg(haddr, BSWAP(cmpv), BSWAP(newv)); 237 #else 238 ret = qatomic_cmpxchg__nocheck(haddr, BSWAP(cmpv), BSWAP(newv)); 239 #endif 240 ATOMIC_MMU_CLEANUP; 241 atomic_trace_rmw_post(env, addr, info); 242 return BSWAP(ret); 243 } 244 245 #if DATA_SIZE >= 16 246 #if HAVE_ATOMIC128 247 ABI_TYPE ATOMIC_NAME(ld)(CPUArchState *env, target_ulong addr, 248 TCGMemOpIdx oi, uintptr_t retaddr) 249 { 250 DATA_TYPE *haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE, 251 PAGE_READ, retaddr); 252 DATA_TYPE val; 253 uint16_t info = trace_mem_build_info(SHIFT, false, MO_BSWAP, false, 254 ATOMIC_MMU_IDX); 255 256 atomic_trace_ld_pre(env, addr, info); 257 val = atomic16_read(haddr); 258 ATOMIC_MMU_CLEANUP; 259 atomic_trace_ld_post(env, addr, info); 260 return BSWAP(val); 261 } 262 263 void ATOMIC_NAME(st)(CPUArchState *env, target_ulong addr, ABI_TYPE val, 264 TCGMemOpIdx oi, uintptr_t retaddr) 265 { 266 DATA_TYPE *haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE, 267 PAGE_WRITE, retaddr); 268 uint16_t info = trace_mem_build_info(SHIFT, false, MO_BSWAP, true, 269 ATOMIC_MMU_IDX); 270 271 val = BSWAP(val); 272 atomic_trace_st_pre(env, addr, info); 273 val = BSWAP(val); 274 atomic16_set(haddr, val); 275 ATOMIC_MMU_CLEANUP; 276 atomic_trace_st_post(env, addr, info); 277 } 278 #endif 279 #else 280 ABI_TYPE ATOMIC_NAME(xchg)(CPUArchState *env, target_ulong addr, ABI_TYPE val, 281 TCGMemOpIdx oi, uintptr_t retaddr) 282 { 283 DATA_TYPE *haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE, 284 PAGE_READ | PAGE_WRITE, retaddr); 285 ABI_TYPE ret; 286 uint16_t info = trace_mem_build_info(SHIFT, false, MO_BSWAP, false, 287 ATOMIC_MMU_IDX); 288 289 atomic_trace_rmw_pre(env, addr, info); 290 ret = qatomic_xchg__nocheck(haddr, BSWAP(val)); 291 ATOMIC_MMU_CLEANUP; 292 atomic_trace_rmw_post(env, addr, info); 293 return BSWAP(ret); 294 } 295 296 #define GEN_ATOMIC_HELPER(X) \ 297 ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr, \ 298 ABI_TYPE val, TCGMemOpIdx oi, uintptr_t retaddr) \ 299 { \ 300 DATA_TYPE *haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE, \ 301 PAGE_READ | PAGE_WRITE, retaddr); \ 302 DATA_TYPE ret; \ 303 uint16_t info = trace_mem_build_info(SHIFT, false, MO_BSWAP, \ 304 false, ATOMIC_MMU_IDX); \ 305 atomic_trace_rmw_pre(env, addr, info); \ 306 ret = qatomic_##X(haddr, BSWAP(val)); \ 307 ATOMIC_MMU_CLEANUP; \ 308 atomic_trace_rmw_post(env, addr, info); \ 309 return BSWAP(ret); \ 310 } 311 312 GEN_ATOMIC_HELPER(fetch_and) 313 GEN_ATOMIC_HELPER(fetch_or) 314 GEN_ATOMIC_HELPER(fetch_xor) 315 GEN_ATOMIC_HELPER(and_fetch) 316 GEN_ATOMIC_HELPER(or_fetch) 317 GEN_ATOMIC_HELPER(xor_fetch) 318 319 #undef GEN_ATOMIC_HELPER 320 321 /* These helpers are, as a whole, full barriers. Within the helper, 322 * the leading barrier is explicit and the trailing barrier is within 323 * cmpxchg primitive. 324 * 325 * Trace this load + RMW loop as a single RMW op. This way, regardless 326 * of CF_PARALLEL's value, we'll trace just a read and a write. 327 */ 328 #define GEN_ATOMIC_HELPER_FN(X, FN, XDATA_TYPE, RET) \ 329 ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr, \ 330 ABI_TYPE xval, TCGMemOpIdx oi, uintptr_t retaddr) \ 331 { \ 332 XDATA_TYPE *haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE, \ 333 PAGE_READ | PAGE_WRITE, retaddr); \ 334 XDATA_TYPE ldo, ldn, old, new, val = xval; \ 335 uint16_t info = trace_mem_build_info(SHIFT, false, MO_BSWAP, \ 336 false, ATOMIC_MMU_IDX); \ 337 atomic_trace_rmw_pre(env, addr, info); \ 338 smp_mb(); \ 339 ldn = qatomic_read__nocheck(haddr); \ 340 do { \ 341 ldo = ldn; old = BSWAP(ldo); new = FN(old, val); \ 342 ldn = qatomic_cmpxchg__nocheck(haddr, ldo, BSWAP(new)); \ 343 } while (ldo != ldn); \ 344 ATOMIC_MMU_CLEANUP; \ 345 atomic_trace_rmw_post(env, addr, info); \ 346 return RET; \ 347 } 348 349 GEN_ATOMIC_HELPER_FN(fetch_smin, MIN, SDATA_TYPE, old) 350 GEN_ATOMIC_HELPER_FN(fetch_umin, MIN, DATA_TYPE, old) 351 GEN_ATOMIC_HELPER_FN(fetch_smax, MAX, SDATA_TYPE, old) 352 GEN_ATOMIC_HELPER_FN(fetch_umax, MAX, DATA_TYPE, old) 353 354 GEN_ATOMIC_HELPER_FN(smin_fetch, MIN, SDATA_TYPE, new) 355 GEN_ATOMIC_HELPER_FN(umin_fetch, MIN, DATA_TYPE, new) 356 GEN_ATOMIC_HELPER_FN(smax_fetch, MAX, SDATA_TYPE, new) 357 GEN_ATOMIC_HELPER_FN(umax_fetch, MAX, DATA_TYPE, new) 358 359 /* Note that for addition, we need to use a separate cmpxchg loop instead 360 of bswaps for the reverse-host-endian helpers. */ 361 #define ADD(X, Y) (X + Y) 362 GEN_ATOMIC_HELPER_FN(fetch_add, ADD, DATA_TYPE, old) 363 GEN_ATOMIC_HELPER_FN(add_fetch, ADD, DATA_TYPE, new) 364 #undef ADD 365 366 #undef GEN_ATOMIC_HELPER_FN 367 #endif /* DATA_SIZE >= 16 */ 368 369 #undef END 370 #endif /* DATA_SIZE > 1 */ 371 372 #undef BSWAP 373 #undef ABI_TYPE 374 #undef DATA_TYPE 375 #undef SDATA_TYPE 376 #undef SUFFIX 377 #undef DATA_SIZE 378 #undef SHIFT 379