1 /* 2 * Atomic helper templates 3 * Included from tcg-runtime.c and cputlb.c. 4 * 5 * Copyright (c) 2016 Red Hat, Inc 6 * 7 * This library is free software; you can redistribute it and/or 8 * modify it under the terms of the GNU Lesser General Public 9 * License as published by the Free Software Foundation; either 10 * version 2.1 of the License, or (at your option) any later version. 11 * 12 * This library is distributed in the hope that it will be useful, 13 * but WITHOUT ANY WARRANTY; without even the implied warranty of 14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 15 * Lesser General Public License for more details. 16 * 17 * You should have received a copy of the GNU Lesser General Public 18 * License along with this library; if not, see <http://www.gnu.org/licenses/>. 19 */ 20 21 #include "qemu/plugin.h" 22 #include "trace/mem.h" 23 24 #if DATA_SIZE == 16 25 # define SUFFIX o 26 # define DATA_TYPE Int128 27 # define BSWAP bswap128 28 # define SHIFT 4 29 #elif DATA_SIZE == 8 30 # define SUFFIX q 31 # define DATA_TYPE aligned_uint64_t 32 # define SDATA_TYPE aligned_int64_t 33 # define BSWAP bswap64 34 # define SHIFT 3 35 #elif DATA_SIZE == 4 36 # define SUFFIX l 37 # define DATA_TYPE uint32_t 38 # define SDATA_TYPE int32_t 39 # define BSWAP bswap32 40 # define SHIFT 2 41 #elif DATA_SIZE == 2 42 # define SUFFIX w 43 # define DATA_TYPE uint16_t 44 # define SDATA_TYPE int16_t 45 # define BSWAP bswap16 46 # define SHIFT 1 47 #elif DATA_SIZE == 1 48 # define SUFFIX b 49 # define DATA_TYPE uint8_t 50 # define SDATA_TYPE int8_t 51 # define BSWAP 52 # define SHIFT 0 53 #else 54 # error unsupported data size 55 #endif 56 57 #if DATA_SIZE >= 4 58 # define ABI_TYPE DATA_TYPE 59 #else 60 # define ABI_TYPE uint32_t 61 #endif 62 63 /* Define host-endian atomic operations. Note that END is used within 64 the ATOMIC_NAME macro, and redefined below. */ 65 #if DATA_SIZE == 1 66 # define END 67 #elif defined(HOST_WORDS_BIGENDIAN) 68 # define END _be 69 #else 70 # define END _le 71 #endif 72 73 ABI_TYPE ATOMIC_NAME(cmpxchg)(CPUArchState *env, target_ulong addr, 74 ABI_TYPE cmpv, ABI_TYPE newv, 75 TCGMemOpIdx oi, uintptr_t retaddr) 76 { 77 DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP_RW; 78 DATA_TYPE ret; 79 uint16_t info = trace_mem_build_info(SHIFT, false, 0, false, 80 ATOMIC_MMU_IDX); 81 82 atomic_trace_rmw_pre(env, addr, info); 83 #if DATA_SIZE == 16 84 ret = atomic16_cmpxchg(haddr, cmpv, newv); 85 #else 86 ret = qatomic_cmpxchg__nocheck(haddr, cmpv, newv); 87 #endif 88 ATOMIC_MMU_CLEANUP; 89 atomic_trace_rmw_post(env, addr, info); 90 return ret; 91 } 92 93 #if DATA_SIZE >= 16 94 #if HAVE_ATOMIC128 95 ABI_TYPE ATOMIC_NAME(ld)(CPUArchState *env, target_ulong addr, 96 TCGMemOpIdx oi, uintptr_t retaddr) 97 { 98 DATA_TYPE val, *haddr = ATOMIC_MMU_LOOKUP_R; 99 uint16_t info = trace_mem_build_info(SHIFT, false, 0, false, 100 ATOMIC_MMU_IDX); 101 102 atomic_trace_ld_pre(env, addr, info); 103 val = atomic16_read(haddr); 104 ATOMIC_MMU_CLEANUP; 105 atomic_trace_ld_post(env, addr, info); 106 return val; 107 } 108 109 void ATOMIC_NAME(st)(CPUArchState *env, target_ulong addr, ABI_TYPE val, 110 TCGMemOpIdx oi, uintptr_t retaddr) 111 { 112 DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP_W; 113 uint16_t info = trace_mem_build_info(SHIFT, false, 0, true, 114 ATOMIC_MMU_IDX); 115 116 atomic_trace_st_pre(env, addr, info); 117 atomic16_set(haddr, val); 118 ATOMIC_MMU_CLEANUP; 119 atomic_trace_st_post(env, addr, info); 120 } 121 #endif 122 #else 123 ABI_TYPE ATOMIC_NAME(xchg)(CPUArchState *env, target_ulong addr, ABI_TYPE val, 124 TCGMemOpIdx oi, uintptr_t retaddr) 125 { 126 DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP_RW; 127 DATA_TYPE ret; 128 uint16_t info = trace_mem_build_info(SHIFT, false, 0, false, 129 ATOMIC_MMU_IDX); 130 131 atomic_trace_rmw_pre(env, addr, info); 132 ret = qatomic_xchg__nocheck(haddr, val); 133 ATOMIC_MMU_CLEANUP; 134 atomic_trace_rmw_post(env, addr, info); 135 return ret; 136 } 137 138 #define GEN_ATOMIC_HELPER(X) \ 139 ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr, \ 140 ABI_TYPE val, TCGMemOpIdx oi, uintptr_t retaddr) \ 141 { \ 142 DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP_RW; \ 143 DATA_TYPE ret; \ 144 uint16_t info = trace_mem_build_info(SHIFT, false, 0, false, \ 145 ATOMIC_MMU_IDX); \ 146 atomic_trace_rmw_pre(env, addr, info); \ 147 ret = qatomic_##X(haddr, val); \ 148 ATOMIC_MMU_CLEANUP; \ 149 atomic_trace_rmw_post(env, addr, info); \ 150 return ret; \ 151 } 152 153 GEN_ATOMIC_HELPER(fetch_add) 154 GEN_ATOMIC_HELPER(fetch_and) 155 GEN_ATOMIC_HELPER(fetch_or) 156 GEN_ATOMIC_HELPER(fetch_xor) 157 GEN_ATOMIC_HELPER(add_fetch) 158 GEN_ATOMIC_HELPER(and_fetch) 159 GEN_ATOMIC_HELPER(or_fetch) 160 GEN_ATOMIC_HELPER(xor_fetch) 161 162 #undef GEN_ATOMIC_HELPER 163 164 /* These helpers are, as a whole, full barriers. Within the helper, 165 * the leading barrier is explicit and the trailing barrier is within 166 * cmpxchg primitive. 167 * 168 * Trace this load + RMW loop as a single RMW op. This way, regardless 169 * of CF_PARALLEL's value, we'll trace just a read and a write. 170 */ 171 #define GEN_ATOMIC_HELPER_FN(X, FN, XDATA_TYPE, RET) \ 172 ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr, \ 173 ABI_TYPE xval, TCGMemOpIdx oi, uintptr_t retaddr) \ 174 { \ 175 XDATA_TYPE *haddr = ATOMIC_MMU_LOOKUP_RW; \ 176 XDATA_TYPE cmp, old, new, val = xval; \ 177 uint16_t info = trace_mem_build_info(SHIFT, false, 0, false, \ 178 ATOMIC_MMU_IDX); \ 179 atomic_trace_rmw_pre(env, addr, info); \ 180 smp_mb(); \ 181 cmp = qatomic_read__nocheck(haddr); \ 182 do { \ 183 old = cmp; new = FN(old, val); \ 184 cmp = qatomic_cmpxchg__nocheck(haddr, old, new); \ 185 } while (cmp != old); \ 186 ATOMIC_MMU_CLEANUP; \ 187 atomic_trace_rmw_post(env, addr, info); \ 188 return RET; \ 189 } 190 191 GEN_ATOMIC_HELPER_FN(fetch_smin, MIN, SDATA_TYPE, old) 192 GEN_ATOMIC_HELPER_FN(fetch_umin, MIN, DATA_TYPE, old) 193 GEN_ATOMIC_HELPER_FN(fetch_smax, MAX, SDATA_TYPE, old) 194 GEN_ATOMIC_HELPER_FN(fetch_umax, MAX, DATA_TYPE, old) 195 196 GEN_ATOMIC_HELPER_FN(smin_fetch, MIN, SDATA_TYPE, new) 197 GEN_ATOMIC_HELPER_FN(umin_fetch, MIN, DATA_TYPE, new) 198 GEN_ATOMIC_HELPER_FN(smax_fetch, MAX, SDATA_TYPE, new) 199 GEN_ATOMIC_HELPER_FN(umax_fetch, MAX, DATA_TYPE, new) 200 201 #undef GEN_ATOMIC_HELPER_FN 202 #endif /* DATA SIZE >= 16 */ 203 204 #undef END 205 206 #if DATA_SIZE > 1 207 208 /* Define reverse-host-endian atomic operations. Note that END is used 209 within the ATOMIC_NAME macro. */ 210 #ifdef HOST_WORDS_BIGENDIAN 211 # define END _le 212 #else 213 # define END _be 214 #endif 215 216 ABI_TYPE ATOMIC_NAME(cmpxchg)(CPUArchState *env, target_ulong addr, 217 ABI_TYPE cmpv, ABI_TYPE newv, 218 TCGMemOpIdx oi, uintptr_t retaddr) 219 { 220 DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP_RW; 221 DATA_TYPE ret; 222 uint16_t info = trace_mem_build_info(SHIFT, false, MO_BSWAP, false, 223 ATOMIC_MMU_IDX); 224 225 atomic_trace_rmw_pre(env, addr, info); 226 #if DATA_SIZE == 16 227 ret = atomic16_cmpxchg(haddr, BSWAP(cmpv), BSWAP(newv)); 228 #else 229 ret = qatomic_cmpxchg__nocheck(haddr, BSWAP(cmpv), BSWAP(newv)); 230 #endif 231 ATOMIC_MMU_CLEANUP; 232 atomic_trace_rmw_post(env, addr, info); 233 return BSWAP(ret); 234 } 235 236 #if DATA_SIZE >= 16 237 #if HAVE_ATOMIC128 238 ABI_TYPE ATOMIC_NAME(ld)(CPUArchState *env, target_ulong addr, 239 TCGMemOpIdx oi, uintptr_t retaddr) 240 { 241 DATA_TYPE val, *haddr = ATOMIC_MMU_LOOKUP_R; 242 uint16_t info = trace_mem_build_info(SHIFT, false, MO_BSWAP, false, 243 ATOMIC_MMU_IDX); 244 245 atomic_trace_ld_pre(env, addr, info); 246 val = atomic16_read(haddr); 247 ATOMIC_MMU_CLEANUP; 248 atomic_trace_ld_post(env, addr, info); 249 return BSWAP(val); 250 } 251 252 void ATOMIC_NAME(st)(CPUArchState *env, target_ulong addr, ABI_TYPE val, 253 TCGMemOpIdx oi, uintptr_t retaddr) 254 { 255 DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP_W; 256 uint16_t info = trace_mem_build_info(SHIFT, false, MO_BSWAP, true, 257 ATOMIC_MMU_IDX); 258 259 val = BSWAP(val); 260 atomic_trace_st_pre(env, addr, info); 261 val = BSWAP(val); 262 atomic16_set(haddr, val); 263 ATOMIC_MMU_CLEANUP; 264 atomic_trace_st_post(env, addr, info); 265 } 266 #endif 267 #else 268 ABI_TYPE ATOMIC_NAME(xchg)(CPUArchState *env, target_ulong addr, ABI_TYPE val, 269 TCGMemOpIdx oi, uintptr_t retaddr) 270 { 271 DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP_RW; 272 ABI_TYPE ret; 273 uint16_t info = trace_mem_build_info(SHIFT, false, MO_BSWAP, false, 274 ATOMIC_MMU_IDX); 275 276 atomic_trace_rmw_pre(env, addr, info); 277 ret = qatomic_xchg__nocheck(haddr, BSWAP(val)); 278 ATOMIC_MMU_CLEANUP; 279 atomic_trace_rmw_post(env, addr, info); 280 return BSWAP(ret); 281 } 282 283 #define GEN_ATOMIC_HELPER(X) \ 284 ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr, \ 285 ABI_TYPE val, TCGMemOpIdx oi, uintptr_t retaddr) \ 286 { \ 287 DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP_RW; \ 288 DATA_TYPE ret; \ 289 uint16_t info = trace_mem_build_info(SHIFT, false, MO_BSWAP, \ 290 false, ATOMIC_MMU_IDX); \ 291 atomic_trace_rmw_pre(env, addr, info); \ 292 ret = qatomic_##X(haddr, BSWAP(val)); \ 293 ATOMIC_MMU_CLEANUP; \ 294 atomic_trace_rmw_post(env, addr, info); \ 295 return BSWAP(ret); \ 296 } 297 298 GEN_ATOMIC_HELPER(fetch_and) 299 GEN_ATOMIC_HELPER(fetch_or) 300 GEN_ATOMIC_HELPER(fetch_xor) 301 GEN_ATOMIC_HELPER(and_fetch) 302 GEN_ATOMIC_HELPER(or_fetch) 303 GEN_ATOMIC_HELPER(xor_fetch) 304 305 #undef GEN_ATOMIC_HELPER 306 307 /* These helpers are, as a whole, full barriers. Within the helper, 308 * the leading barrier is explicit and the trailing barrier is within 309 * cmpxchg primitive. 310 * 311 * Trace this load + RMW loop as a single RMW op. This way, regardless 312 * of CF_PARALLEL's value, we'll trace just a read and a write. 313 */ 314 #define GEN_ATOMIC_HELPER_FN(X, FN, XDATA_TYPE, RET) \ 315 ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr, \ 316 ABI_TYPE xval, TCGMemOpIdx oi, uintptr_t retaddr) \ 317 { \ 318 XDATA_TYPE *haddr = ATOMIC_MMU_LOOKUP_RW; \ 319 XDATA_TYPE ldo, ldn, old, new, val = xval; \ 320 uint16_t info = trace_mem_build_info(SHIFT, false, MO_BSWAP, \ 321 false, ATOMIC_MMU_IDX); \ 322 atomic_trace_rmw_pre(env, addr, info); \ 323 smp_mb(); \ 324 ldn = qatomic_read__nocheck(haddr); \ 325 do { \ 326 ldo = ldn; old = BSWAP(ldo); new = FN(old, val); \ 327 ldn = qatomic_cmpxchg__nocheck(haddr, ldo, BSWAP(new)); \ 328 } while (ldo != ldn); \ 329 ATOMIC_MMU_CLEANUP; \ 330 atomic_trace_rmw_post(env, addr, info); \ 331 return RET; \ 332 } 333 334 GEN_ATOMIC_HELPER_FN(fetch_smin, MIN, SDATA_TYPE, old) 335 GEN_ATOMIC_HELPER_FN(fetch_umin, MIN, DATA_TYPE, old) 336 GEN_ATOMIC_HELPER_FN(fetch_smax, MAX, SDATA_TYPE, old) 337 GEN_ATOMIC_HELPER_FN(fetch_umax, MAX, DATA_TYPE, old) 338 339 GEN_ATOMIC_HELPER_FN(smin_fetch, MIN, SDATA_TYPE, new) 340 GEN_ATOMIC_HELPER_FN(umin_fetch, MIN, DATA_TYPE, new) 341 GEN_ATOMIC_HELPER_FN(smax_fetch, MAX, SDATA_TYPE, new) 342 GEN_ATOMIC_HELPER_FN(umax_fetch, MAX, DATA_TYPE, new) 343 344 /* Note that for addition, we need to use a separate cmpxchg loop instead 345 of bswaps for the reverse-host-endian helpers. */ 346 #define ADD(X, Y) (X + Y) 347 GEN_ATOMIC_HELPER_FN(fetch_add, ADD, DATA_TYPE, old) 348 GEN_ATOMIC_HELPER_FN(add_fetch, ADD, DATA_TYPE, new) 349 #undef ADD 350 351 #undef GEN_ATOMIC_HELPER_FN 352 #endif /* DATA_SIZE >= 16 */ 353 354 #undef END 355 #endif /* DATA_SIZE > 1 */ 356 357 #undef BSWAP 358 #undef ABI_TYPE 359 #undef DATA_TYPE 360 #undef SDATA_TYPE 361 #undef SUFFIX 362 #undef DATA_SIZE 363 #undef SHIFT 364