1 /* 2 * Atomic helper templates 3 * Included from tcg-runtime.c and cputlb.c. 4 * 5 * Copyright (c) 2016 Red Hat, Inc 6 * 7 * This library is free software; you can redistribute it and/or 8 * modify it under the terms of the GNU Lesser General Public 9 * License as published by the Free Software Foundation; either 10 * version 2.1 of the License, or (at your option) any later version. 11 * 12 * This library is distributed in the hope that it will be useful, 13 * but WITHOUT ANY WARRANTY; without even the implied warranty of 14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 15 * Lesser General Public License for more details. 16 * 17 * You should have received a copy of the GNU Lesser General Public 18 * License along with this library; if not, see <http://www.gnu.org/licenses/>. 19 */ 20 21 #include "trace/mem.h" 22 23 #if DATA_SIZE == 16 24 # define SUFFIX o 25 # define DATA_TYPE Int128 26 # define BSWAP bswap128 27 # define SHIFT 4 28 #elif DATA_SIZE == 8 29 # define SUFFIX q 30 # define DATA_TYPE uint64_t 31 # define SDATA_TYPE int64_t 32 # define BSWAP bswap64 33 # define SHIFT 3 34 #elif DATA_SIZE == 4 35 # define SUFFIX l 36 # define DATA_TYPE uint32_t 37 # define SDATA_TYPE int32_t 38 # define BSWAP bswap32 39 # define SHIFT 2 40 #elif DATA_SIZE == 2 41 # define SUFFIX w 42 # define DATA_TYPE uint16_t 43 # define SDATA_TYPE int16_t 44 # define BSWAP bswap16 45 # define SHIFT 1 46 #elif DATA_SIZE == 1 47 # define SUFFIX b 48 # define DATA_TYPE uint8_t 49 # define SDATA_TYPE int8_t 50 # define BSWAP 51 # define SHIFT 0 52 #else 53 # error unsupported data size 54 #endif 55 56 #if DATA_SIZE >= 4 57 # define ABI_TYPE DATA_TYPE 58 #else 59 # define ABI_TYPE uint32_t 60 #endif 61 62 /* Define host-endian atomic operations. Note that END is used within 63 the ATOMIC_NAME macro, and redefined below. */ 64 #if DATA_SIZE == 1 65 # define END 66 # define MEND _be /* either le or be would be fine */ 67 #elif defined(HOST_WORDS_BIGENDIAN) 68 # define END _be 69 # define MEND _be 70 #else 71 # define END _le 72 # define MEND _le 73 #endif 74 75 ABI_TYPE ATOMIC_NAME(cmpxchg)(CPUArchState *env, target_ulong addr, 76 ABI_TYPE cmpv, ABI_TYPE newv EXTRA_ARGS) 77 { 78 ATOMIC_MMU_DECLS; 79 DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP; 80 DATA_TYPE ret; 81 uint16_t info = glue(trace_mem_build_info_no_se, MEND)(SHIFT, false, 82 ATOMIC_MMU_IDX); 83 84 atomic_trace_rmw_pre(env, addr, info); 85 #if DATA_SIZE == 16 86 ret = atomic16_cmpxchg(haddr, cmpv, newv); 87 #else 88 ret = atomic_cmpxchg__nocheck(haddr, cmpv, newv); 89 #endif 90 ATOMIC_MMU_CLEANUP; 91 atomic_trace_rmw_post(env, addr, info); 92 return ret; 93 } 94 95 #if DATA_SIZE >= 16 96 #if HAVE_ATOMIC128 97 ABI_TYPE ATOMIC_NAME(ld)(CPUArchState *env, target_ulong addr EXTRA_ARGS) 98 { 99 ATOMIC_MMU_DECLS; 100 DATA_TYPE val, *haddr = ATOMIC_MMU_LOOKUP; 101 uint16_t info = glue(trace_mem_build_info_no_se, MEND)(SHIFT, false, 102 ATOMIC_MMU_IDX); 103 104 atomic_trace_ld_pre(env, addr, info); 105 val = atomic16_read(haddr); 106 ATOMIC_MMU_CLEANUP; 107 atomic_trace_ld_post(env, addr, info); 108 return val; 109 } 110 111 void ATOMIC_NAME(st)(CPUArchState *env, target_ulong addr, 112 ABI_TYPE val EXTRA_ARGS) 113 { 114 ATOMIC_MMU_DECLS; 115 DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP; 116 uint16_t info = glue(trace_mem_build_info_no_se, MEND)(SHIFT, true, 117 ATOMIC_MMU_IDX); 118 119 atomic_trace_st_pre(env, addr, info); 120 atomic16_set(haddr, val); 121 ATOMIC_MMU_CLEANUP; 122 atomic_trace_st_post(env, addr, info); 123 } 124 #endif 125 #else 126 ABI_TYPE ATOMIC_NAME(xchg)(CPUArchState *env, target_ulong addr, 127 ABI_TYPE val EXTRA_ARGS) 128 { 129 ATOMIC_MMU_DECLS; 130 DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP; 131 DATA_TYPE ret; 132 uint16_t info = glue(trace_mem_build_info_no_se, MEND)(SHIFT, false, 133 ATOMIC_MMU_IDX); 134 135 atomic_trace_rmw_pre(env, addr, info); 136 ret = atomic_xchg__nocheck(haddr, val); 137 ATOMIC_MMU_CLEANUP; 138 atomic_trace_rmw_post(env, addr, info); 139 return ret; 140 } 141 142 #define GEN_ATOMIC_HELPER(X) \ 143 ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr, \ 144 ABI_TYPE val EXTRA_ARGS) \ 145 { \ 146 ATOMIC_MMU_DECLS; \ 147 DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP; \ 148 DATA_TYPE ret; \ 149 uint16_t info = glue(trace_mem_build_info_no_se, MEND)(SHIFT, \ 150 false, \ 151 ATOMIC_MMU_IDX); \ 152 \ 153 atomic_trace_rmw_pre(env, addr, info); \ 154 ret = atomic_##X(haddr, val); \ 155 ATOMIC_MMU_CLEANUP; \ 156 atomic_trace_rmw_post(env, addr, info); \ 157 return ret; \ 158 } 159 160 GEN_ATOMIC_HELPER(fetch_add) 161 GEN_ATOMIC_HELPER(fetch_and) 162 GEN_ATOMIC_HELPER(fetch_or) 163 GEN_ATOMIC_HELPER(fetch_xor) 164 GEN_ATOMIC_HELPER(add_fetch) 165 GEN_ATOMIC_HELPER(and_fetch) 166 GEN_ATOMIC_HELPER(or_fetch) 167 GEN_ATOMIC_HELPER(xor_fetch) 168 169 #undef GEN_ATOMIC_HELPER 170 171 /* These helpers are, as a whole, full barriers. Within the helper, 172 * the leading barrier is explicit and the trailing barrier is within 173 * cmpxchg primitive. 174 * 175 * Trace this load + RMW loop as a single RMW op. This way, regardless 176 * of CF_PARALLEL's value, we'll trace just a read and a write. 177 */ 178 #define GEN_ATOMIC_HELPER_FN(X, FN, XDATA_TYPE, RET) \ 179 ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr, \ 180 ABI_TYPE xval EXTRA_ARGS) \ 181 { \ 182 ATOMIC_MMU_DECLS; \ 183 XDATA_TYPE *haddr = ATOMIC_MMU_LOOKUP; \ 184 XDATA_TYPE cmp, old, new, val = xval; \ 185 uint16_t info = glue(trace_mem_build_info_no_se, MEND)(SHIFT, \ 186 false, \ 187 ATOMIC_MMU_IDX); \ 188 \ 189 atomic_trace_rmw_pre(env, addr, info); \ 190 smp_mb(); \ 191 cmp = atomic_read__nocheck(haddr); \ 192 do { \ 193 old = cmp; new = FN(old, val); \ 194 cmp = atomic_cmpxchg__nocheck(haddr, old, new); \ 195 } while (cmp != old); \ 196 ATOMIC_MMU_CLEANUP; \ 197 atomic_trace_rmw_post(env, addr, info); \ 198 return RET; \ 199 } 200 201 GEN_ATOMIC_HELPER_FN(fetch_smin, MIN, SDATA_TYPE, old) 202 GEN_ATOMIC_HELPER_FN(fetch_umin, MIN, DATA_TYPE, old) 203 GEN_ATOMIC_HELPER_FN(fetch_smax, MAX, SDATA_TYPE, old) 204 GEN_ATOMIC_HELPER_FN(fetch_umax, MAX, DATA_TYPE, old) 205 206 GEN_ATOMIC_HELPER_FN(smin_fetch, MIN, SDATA_TYPE, new) 207 GEN_ATOMIC_HELPER_FN(umin_fetch, MIN, DATA_TYPE, new) 208 GEN_ATOMIC_HELPER_FN(smax_fetch, MAX, SDATA_TYPE, new) 209 GEN_ATOMIC_HELPER_FN(umax_fetch, MAX, DATA_TYPE, new) 210 211 #undef GEN_ATOMIC_HELPER_FN 212 #endif /* DATA SIZE >= 16 */ 213 214 #undef END 215 #undef MEND 216 217 #if DATA_SIZE > 1 218 219 /* Define reverse-host-endian atomic operations. Note that END is used 220 within the ATOMIC_NAME macro. */ 221 #ifdef HOST_WORDS_BIGENDIAN 222 # define END _le 223 # define MEND _le 224 #else 225 # define END _be 226 # define MEND _be 227 #endif 228 229 ABI_TYPE ATOMIC_NAME(cmpxchg)(CPUArchState *env, target_ulong addr, 230 ABI_TYPE cmpv, ABI_TYPE newv EXTRA_ARGS) 231 { 232 ATOMIC_MMU_DECLS; 233 DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP; 234 DATA_TYPE ret; 235 uint16_t info = glue(trace_mem_build_info_no_se, MEND)(SHIFT, 236 false, 237 ATOMIC_MMU_IDX); 238 239 atomic_trace_rmw_pre(env, addr, info); 240 #if DATA_SIZE == 16 241 ret = atomic16_cmpxchg(haddr, BSWAP(cmpv), BSWAP(newv)); 242 #else 243 ret = atomic_cmpxchg__nocheck(haddr, BSWAP(cmpv), BSWAP(newv)); 244 #endif 245 ATOMIC_MMU_CLEANUP; 246 atomic_trace_rmw_post(env, addr, info); 247 return BSWAP(ret); 248 } 249 250 #if DATA_SIZE >= 16 251 #if HAVE_ATOMIC128 252 ABI_TYPE ATOMIC_NAME(ld)(CPUArchState *env, target_ulong addr EXTRA_ARGS) 253 { 254 ATOMIC_MMU_DECLS; 255 DATA_TYPE val, *haddr = ATOMIC_MMU_LOOKUP; 256 uint16_t info = glue(trace_mem_build_info_no_se, MEND)(SHIFT, 257 false, 258 ATOMIC_MMU_IDX); 259 260 atomic_trace_ld_pre(env, addr, info); 261 val = atomic16_read(haddr); 262 ATOMIC_MMU_CLEANUP; 263 atomic_trace_ld_post(env, addr, info); 264 return BSWAP(val); 265 } 266 267 void ATOMIC_NAME(st)(CPUArchState *env, target_ulong addr, 268 ABI_TYPE val EXTRA_ARGS) 269 { 270 ATOMIC_MMU_DECLS; 271 DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP; 272 uint16_t info = glue(trace_mem_build_info_no_se, MEND)(SHIFT, 273 true, 274 ATOMIC_MMU_IDX); 275 276 val = BSWAP(val); 277 atomic_trace_st_pre(env, addr, info); 278 val = BSWAP(val); 279 atomic16_set(haddr, val); 280 ATOMIC_MMU_CLEANUP; 281 atomic_trace_st_post(env, addr, info); 282 } 283 #endif 284 #else 285 ABI_TYPE ATOMIC_NAME(xchg)(CPUArchState *env, target_ulong addr, 286 ABI_TYPE val EXTRA_ARGS) 287 { 288 ATOMIC_MMU_DECLS; 289 DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP; 290 ABI_TYPE ret; 291 uint16_t info = glue(trace_mem_build_info_no_se, MEND)(SHIFT, 292 false, 293 ATOMIC_MMU_IDX); 294 295 atomic_trace_rmw_pre(env, addr, info); 296 ret = atomic_xchg__nocheck(haddr, BSWAP(val)); 297 ATOMIC_MMU_CLEANUP; 298 atomic_trace_rmw_post(env, addr, info); 299 return BSWAP(ret); 300 } 301 302 #define GEN_ATOMIC_HELPER(X) \ 303 ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr, \ 304 ABI_TYPE val EXTRA_ARGS) \ 305 { \ 306 ATOMIC_MMU_DECLS; \ 307 DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP; \ 308 DATA_TYPE ret; \ 309 uint16_t info = glue(trace_mem_build_info_no_se, MEND)(SHIFT, \ 310 false, \ 311 ATOMIC_MMU_IDX); \ 312 \ 313 atomic_trace_rmw_pre(env, addr, info); \ 314 ret = atomic_##X(haddr, BSWAP(val)); \ 315 ATOMIC_MMU_CLEANUP; \ 316 atomic_trace_rmw_post(env, addr, info); \ 317 return BSWAP(ret); \ 318 } 319 320 GEN_ATOMIC_HELPER(fetch_and) 321 GEN_ATOMIC_HELPER(fetch_or) 322 GEN_ATOMIC_HELPER(fetch_xor) 323 GEN_ATOMIC_HELPER(and_fetch) 324 GEN_ATOMIC_HELPER(or_fetch) 325 GEN_ATOMIC_HELPER(xor_fetch) 326 327 #undef GEN_ATOMIC_HELPER 328 329 /* These helpers are, as a whole, full barriers. Within the helper, 330 * the leading barrier is explicit and the trailing barrier is within 331 * cmpxchg primitive. 332 * 333 * Trace this load + RMW loop as a single RMW op. This way, regardless 334 * of CF_PARALLEL's value, we'll trace just a read and a write. 335 */ 336 #define GEN_ATOMIC_HELPER_FN(X, FN, XDATA_TYPE, RET) \ 337 ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr, \ 338 ABI_TYPE xval EXTRA_ARGS) \ 339 { \ 340 ATOMIC_MMU_DECLS; \ 341 XDATA_TYPE *haddr = ATOMIC_MMU_LOOKUP; \ 342 XDATA_TYPE ldo, ldn, old, new, val = xval; \ 343 uint16_t info = glue(trace_mem_build_info_no_se, MEND)(SHIFT, \ 344 false, \ 345 ATOMIC_MMU_IDX); \ 346 \ 347 atomic_trace_rmw_pre(env, addr, info); \ 348 smp_mb(); \ 349 ldn = atomic_read__nocheck(haddr); \ 350 do { \ 351 ldo = ldn; old = BSWAP(ldo); new = FN(old, val); \ 352 ldn = atomic_cmpxchg__nocheck(haddr, ldo, BSWAP(new)); \ 353 } while (ldo != ldn); \ 354 ATOMIC_MMU_CLEANUP; \ 355 atomic_trace_rmw_post(env, addr, info); \ 356 return RET; \ 357 } 358 359 GEN_ATOMIC_HELPER_FN(fetch_smin, MIN, SDATA_TYPE, old) 360 GEN_ATOMIC_HELPER_FN(fetch_umin, MIN, DATA_TYPE, old) 361 GEN_ATOMIC_HELPER_FN(fetch_smax, MAX, SDATA_TYPE, old) 362 GEN_ATOMIC_HELPER_FN(fetch_umax, MAX, DATA_TYPE, old) 363 364 GEN_ATOMIC_HELPER_FN(smin_fetch, MIN, SDATA_TYPE, new) 365 GEN_ATOMIC_HELPER_FN(umin_fetch, MIN, DATA_TYPE, new) 366 GEN_ATOMIC_HELPER_FN(smax_fetch, MAX, SDATA_TYPE, new) 367 GEN_ATOMIC_HELPER_FN(umax_fetch, MAX, DATA_TYPE, new) 368 369 /* Note that for addition, we need to use a separate cmpxchg loop instead 370 of bswaps for the reverse-host-endian helpers. */ 371 #define ADD(X, Y) (X + Y) 372 GEN_ATOMIC_HELPER_FN(fetch_add, ADD, DATA_TYPE, old) 373 GEN_ATOMIC_HELPER_FN(add_fetch, ADD, DATA_TYPE, new) 374 #undef ADD 375 376 #undef GEN_ATOMIC_HELPER_FN 377 #endif /* DATA_SIZE >= 16 */ 378 379 #undef END 380 #undef MEND 381 #endif /* DATA_SIZE > 1 */ 382 383 #undef BSWAP 384 #undef ABI_TYPE 385 #undef DATA_TYPE 386 #undef SDATA_TYPE 387 #undef SUFFIX 388 #undef DATA_SIZE 389 #undef SHIFT 390