1 /*
2 * Atomic helper templates
3 * Included from tcg-runtime.c and cputlb.c.
4 *
5 * Copyright (c) 2016 Red Hat, Inc
6 *
7 * This library is free software; you can redistribute it and/or
8 * modify it under the terms of the GNU Lesser General Public
9 * License as published by the Free Software Foundation; either
10 * version 2.1 of the License, or (at your option) any later version.
11 *
12 * This library is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 * Lesser General Public License for more details.
16 *
17 * You should have received a copy of the GNU Lesser General Public
18 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
19 */
20
21 #include "qemu/plugin.h"
22 #include "trace/mem.h"
23
24 #if DATA_SIZE == 16
25 # define SUFFIX o
26 # define DATA_TYPE Int128
27 # define BSWAP bswap128
28 # define SHIFT 4
29 #elif DATA_SIZE == 8
30 # define SUFFIX q
31 # define DATA_TYPE uint64_t
32 # define SDATA_TYPE int64_t
33 # define BSWAP bswap64
34 # define SHIFT 3
35 #elif DATA_SIZE == 4
36 # define SUFFIX l
37 # define DATA_TYPE uint32_t
38 # define SDATA_TYPE int32_t
39 # define BSWAP bswap32
40 # define SHIFT 2
41 #elif DATA_SIZE == 2
42 # define SUFFIX w
43 # define DATA_TYPE uint16_t
44 # define SDATA_TYPE int16_t
45 # define BSWAP bswap16
46 # define SHIFT 1
47 #elif DATA_SIZE == 1
48 # define SUFFIX b
49 # define DATA_TYPE uint8_t
50 # define SDATA_TYPE int8_t
51 # define BSWAP
52 # define SHIFT 0
53 #else
54 # error unsupported data size
55 #endif
56
57 #if DATA_SIZE >= 4
58 # define ABI_TYPE DATA_TYPE
59 #else
60 # define ABI_TYPE uint32_t
61 #endif
62
63 /* Define host-endian atomic operations. Note that END is used within
64 the ATOMIC_NAME macro, and redefined below. */
65 #if DATA_SIZE == 1
66 # define END
67 # define MEND _be /* either le or be would be fine */
68 #elif defined(HOST_WORDS_BIGENDIAN)
69 # define END _be
70 # define MEND _be
71 #else
72 # define END _le
73 # define MEND _le
74 #endif
75
ATOMIC_NAME(cmpxchg)76 ABI_TYPE ATOMIC_NAME(cmpxchg)(CPUArchState *env, target_ulong addr,
77 ABI_TYPE cmpv, ABI_TYPE newv EXTRA_ARGS)
78 {
79 ATOMIC_MMU_DECLS;
80 DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
81 DATA_TYPE ret;
82 uint16_t info = glue(trace_mem_build_info_no_se, MEND)(SHIFT, false,
83 ATOMIC_MMU_IDX);
84
85 atomic_trace_rmw_pre(env, addr, info);
86 #if DATA_SIZE == 16
87 ret = atomic16_cmpxchg(haddr, cmpv, newv);
88 #else
89 ret = atomic_cmpxchg__nocheck(haddr, cmpv, newv);
90 #endif
91 ATOMIC_MMU_CLEANUP;
92 atomic_trace_rmw_post(env, addr, info);
93 return ret;
94 }
95
96 #if DATA_SIZE >= 16
97 #if HAVE_ATOMIC128
ATOMIC_NAME(ld)98 ABI_TYPE ATOMIC_NAME(ld)(CPUArchState *env, target_ulong addr EXTRA_ARGS)
99 {
100 ATOMIC_MMU_DECLS;
101 DATA_TYPE val, *haddr = ATOMIC_MMU_LOOKUP;
102 uint16_t info = glue(trace_mem_build_info_no_se, MEND)(SHIFT, false,
103 ATOMIC_MMU_IDX);
104
105 atomic_trace_ld_pre(env, addr, info);
106 val = atomic16_read(haddr);
107 ATOMIC_MMU_CLEANUP;
108 atomic_trace_ld_post(env, addr, info);
109 return val;
110 }
111
ATOMIC_NAME(st)112 void ATOMIC_NAME(st)(CPUArchState *env, target_ulong addr,
113 ABI_TYPE val EXTRA_ARGS)
114 {
115 ATOMIC_MMU_DECLS;
116 DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
117 uint16_t info = glue(trace_mem_build_info_no_se, MEND)(SHIFT, true,
118 ATOMIC_MMU_IDX);
119
120 atomic_trace_st_pre(env, addr, info);
121 atomic16_set(haddr, val);
122 ATOMIC_MMU_CLEANUP;
123 atomic_trace_st_post(env, addr, info);
124 }
125 #endif
126 #else
ATOMIC_NAME(xchg)127 ABI_TYPE ATOMIC_NAME(xchg)(CPUArchState *env, target_ulong addr,
128 ABI_TYPE val EXTRA_ARGS)
129 {
130 ATOMIC_MMU_DECLS;
131 DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
132 DATA_TYPE ret;
133 uint16_t info = glue(trace_mem_build_info_no_se, MEND)(SHIFT, false,
134 ATOMIC_MMU_IDX);
135
136 atomic_trace_rmw_pre(env, addr, info);
137 ret = atomic_xchg__nocheck(haddr, val);
138 ATOMIC_MMU_CLEANUP;
139 atomic_trace_rmw_post(env, addr, info);
140 return ret;
141 }
142
143 #define GEN_ATOMIC_HELPER(X) \
144 ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr, \
145 ABI_TYPE val EXTRA_ARGS) \
146 { \
147 ATOMIC_MMU_DECLS; \
148 DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP; \
149 DATA_TYPE ret; \
150 uint16_t info = glue(trace_mem_build_info_no_se, MEND)(SHIFT, \
151 false, \
152 ATOMIC_MMU_IDX); \
153 \
154 atomic_trace_rmw_pre(env, addr, info); \
155 ret = atomic_##X(haddr, val); \
156 ATOMIC_MMU_CLEANUP; \
157 atomic_trace_rmw_post(env, addr, info); \
158 return ret; \
159 }
160
161 GEN_ATOMIC_HELPER(fetch_add)
GEN_ATOMIC_HELPER(fetch_and)162 GEN_ATOMIC_HELPER(fetch_and)
163 GEN_ATOMIC_HELPER(fetch_or)
164 GEN_ATOMIC_HELPER(fetch_xor)
165 GEN_ATOMIC_HELPER(add_fetch)
166 GEN_ATOMIC_HELPER(and_fetch)
167 GEN_ATOMIC_HELPER(or_fetch)
168 GEN_ATOMIC_HELPER(xor_fetch)
169
170 #undef GEN_ATOMIC_HELPER
171
172 /* These helpers are, as a whole, full barriers. Within the helper,
173 * the leading barrier is explicit and the trailing barrier is within
174 * cmpxchg primitive.
175 *
176 * Trace this load + RMW loop as a single RMW op. This way, regardless
177 * of CF_PARALLEL's value, we'll trace just a read and a write.
178 */
179 #define GEN_ATOMIC_HELPER_FN(X, FN, XDATA_TYPE, RET) \
180 ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr, \
181 ABI_TYPE xval EXTRA_ARGS) \
182 { \
183 ATOMIC_MMU_DECLS; \
184 XDATA_TYPE *haddr = ATOMIC_MMU_LOOKUP; \
185 XDATA_TYPE cmp, old, new, val = xval; \
186 uint16_t info = glue(trace_mem_build_info_no_se, MEND)(SHIFT, \
187 false, \
188 ATOMIC_MMU_IDX); \
189 \
190 atomic_trace_rmw_pre(env, addr, info); \
191 smp_mb(); \
192 cmp = atomic_read__nocheck(haddr); \
193 do { \
194 old = cmp; new = FN(old, val); \
195 cmp = atomic_cmpxchg__nocheck(haddr, old, new); \
196 } while (cmp != old); \
197 ATOMIC_MMU_CLEANUP; \
198 atomic_trace_rmw_post(env, addr, info); \
199 return RET; \
200 }
201
202 GEN_ATOMIC_HELPER_FN(fetch_smin, MIN, SDATA_TYPE, old)
203 GEN_ATOMIC_HELPER_FN(fetch_umin, MIN, DATA_TYPE, old)
204 GEN_ATOMIC_HELPER_FN(fetch_smax, MAX, SDATA_TYPE, old)
205 GEN_ATOMIC_HELPER_FN(fetch_umax, MAX, DATA_TYPE, old)
206
207 GEN_ATOMIC_HELPER_FN(smin_fetch, MIN, SDATA_TYPE, new)
208 GEN_ATOMIC_HELPER_FN(umin_fetch, MIN, DATA_TYPE, new)
209 GEN_ATOMIC_HELPER_FN(smax_fetch, MAX, SDATA_TYPE, new)
210 GEN_ATOMIC_HELPER_FN(umax_fetch, MAX, DATA_TYPE, new)
211
212 #undef GEN_ATOMIC_HELPER_FN
213 #endif /* DATA SIZE >= 16 */
214
215 #undef END
216 #undef MEND
217
218 #if DATA_SIZE > 1
219
220 /* Define reverse-host-endian atomic operations. Note that END is used
221 within the ATOMIC_NAME macro. */
222 #ifdef HOST_WORDS_BIGENDIAN
223 # define END _le
224 # define MEND _le
225 #else
226 # define END _be
227 # define MEND _be
228 #endif
229
230 ABI_TYPE ATOMIC_NAME(cmpxchg)(CPUArchState *env, target_ulong addr,
231 ABI_TYPE cmpv, ABI_TYPE newv EXTRA_ARGS)
232 {
233 ATOMIC_MMU_DECLS;
234 DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
235 DATA_TYPE ret;
236 uint16_t info = glue(trace_mem_build_info_no_se, MEND)(SHIFT,
237 false,
238 ATOMIC_MMU_IDX);
239
240 atomic_trace_rmw_pre(env, addr, info);
241 #if DATA_SIZE == 16
242 ret = atomic16_cmpxchg(haddr, BSWAP(cmpv), BSWAP(newv));
243 #else
244 ret = atomic_cmpxchg__nocheck(haddr, BSWAP(cmpv), BSWAP(newv));
245 #endif
246 ATOMIC_MMU_CLEANUP;
247 atomic_trace_rmw_post(env, addr, info);
248 return BSWAP(ret);
249 }
250
251 #if DATA_SIZE >= 16
252 #if HAVE_ATOMIC128
ATOMIC_NAME(ld)253 ABI_TYPE ATOMIC_NAME(ld)(CPUArchState *env, target_ulong addr EXTRA_ARGS)
254 {
255 ATOMIC_MMU_DECLS;
256 DATA_TYPE val, *haddr = ATOMIC_MMU_LOOKUP;
257 uint16_t info = glue(trace_mem_build_info_no_se, MEND)(SHIFT,
258 false,
259 ATOMIC_MMU_IDX);
260
261 atomic_trace_ld_pre(env, addr, info);
262 val = atomic16_read(haddr);
263 ATOMIC_MMU_CLEANUP;
264 atomic_trace_ld_post(env, addr, info);
265 return BSWAP(val);
266 }
267
ATOMIC_NAME(st)268 void ATOMIC_NAME(st)(CPUArchState *env, target_ulong addr,
269 ABI_TYPE val EXTRA_ARGS)
270 {
271 ATOMIC_MMU_DECLS;
272 DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
273 uint16_t info = glue(trace_mem_build_info_no_se, MEND)(SHIFT,
274 true,
275 ATOMIC_MMU_IDX);
276
277 val = BSWAP(val);
278 atomic_trace_st_pre(env, addr, info);
279 val = BSWAP(val);
280 atomic16_set(haddr, val);
281 ATOMIC_MMU_CLEANUP;
282 atomic_trace_st_post(env, addr, info);
283 }
284 #endif
285 #else
ATOMIC_NAME(xchg)286 ABI_TYPE ATOMIC_NAME(xchg)(CPUArchState *env, target_ulong addr,
287 ABI_TYPE val EXTRA_ARGS)
288 {
289 ATOMIC_MMU_DECLS;
290 DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
291 ABI_TYPE ret;
292 uint16_t info = glue(trace_mem_build_info_no_se, MEND)(SHIFT,
293 false,
294 ATOMIC_MMU_IDX);
295
296 atomic_trace_rmw_pre(env, addr, info);
297 ret = atomic_xchg__nocheck(haddr, BSWAP(val));
298 ATOMIC_MMU_CLEANUP;
299 atomic_trace_rmw_post(env, addr, info);
300 return BSWAP(ret);
301 }
302
303 #define GEN_ATOMIC_HELPER(X) \
304 ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr, \
305 ABI_TYPE val EXTRA_ARGS) \
306 { \
307 ATOMIC_MMU_DECLS; \
308 DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP; \
309 DATA_TYPE ret; \
310 uint16_t info = glue(trace_mem_build_info_no_se, MEND)(SHIFT, \
311 false, \
312 ATOMIC_MMU_IDX); \
313 \
314 atomic_trace_rmw_pre(env, addr, info); \
315 ret = atomic_##X(haddr, BSWAP(val)); \
316 ATOMIC_MMU_CLEANUP; \
317 atomic_trace_rmw_post(env, addr, info); \
318 return BSWAP(ret); \
319 }
320
321 GEN_ATOMIC_HELPER(fetch_and)
322 GEN_ATOMIC_HELPER(fetch_or)
323 GEN_ATOMIC_HELPER(fetch_xor)
324 GEN_ATOMIC_HELPER(and_fetch)
325 GEN_ATOMIC_HELPER(or_fetch)
326 GEN_ATOMIC_HELPER(xor_fetch)
327
328 #undef GEN_ATOMIC_HELPER
329
330 /* These helpers are, as a whole, full barriers. Within the helper,
331 * the leading barrier is explicit and the trailing barrier is within
332 * cmpxchg primitive.
333 *
334 * Trace this load + RMW loop as a single RMW op. This way, regardless
335 * of CF_PARALLEL's value, we'll trace just a read and a write.
336 */
337 #define GEN_ATOMIC_HELPER_FN(X, FN, XDATA_TYPE, RET) \
338 ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr, \
339 ABI_TYPE xval EXTRA_ARGS) \
340 { \
341 ATOMIC_MMU_DECLS; \
342 XDATA_TYPE *haddr = ATOMIC_MMU_LOOKUP; \
343 XDATA_TYPE ldo, ldn, old, new, val = xval; \
344 uint16_t info = glue(trace_mem_build_info_no_se, MEND)(SHIFT, \
345 false, \
346 ATOMIC_MMU_IDX); \
347 \
348 atomic_trace_rmw_pre(env, addr, info); \
349 smp_mb(); \
350 ldn = atomic_read__nocheck(haddr); \
351 do { \
352 ldo = ldn; old = BSWAP(ldo); new = FN(old, val); \
353 ldn = atomic_cmpxchg__nocheck(haddr, ldo, BSWAP(new)); \
354 } while (ldo != ldn); \
355 ATOMIC_MMU_CLEANUP; \
356 atomic_trace_rmw_post(env, addr, info); \
357 return RET; \
358 }
359
360 GEN_ATOMIC_HELPER_FN(fetch_smin, MIN, SDATA_TYPE, old)
361 GEN_ATOMIC_HELPER_FN(fetch_umin, MIN, DATA_TYPE, old)
362 GEN_ATOMIC_HELPER_FN(fetch_smax, MAX, SDATA_TYPE, old)
363 GEN_ATOMIC_HELPER_FN(fetch_umax, MAX, DATA_TYPE, old)
364
365 GEN_ATOMIC_HELPER_FN(smin_fetch, MIN, SDATA_TYPE, new)
366 GEN_ATOMIC_HELPER_FN(umin_fetch, MIN, DATA_TYPE, new)
367 GEN_ATOMIC_HELPER_FN(smax_fetch, MAX, SDATA_TYPE, new)
368 GEN_ATOMIC_HELPER_FN(umax_fetch, MAX, DATA_TYPE, new)
369
370 /* Note that for addition, we need to use a separate cmpxchg loop instead
371 of bswaps for the reverse-host-endian helpers. */
372 #define ADD(X, Y) (X + Y)
373 GEN_ATOMIC_HELPER_FN(fetch_add, ADD, DATA_TYPE, old)
374 GEN_ATOMIC_HELPER_FN(add_fetch, ADD, DATA_TYPE, new)
375 #undef ADD
376
377 #undef GEN_ATOMIC_HELPER_FN
378 #endif /* DATA_SIZE >= 16 */
379
380 #undef END
381 #undef MEND
382 #endif /* DATA_SIZE > 1 */
383
384 #undef BSWAP
385 #undef ABI_TYPE
386 #undef DATA_TYPE
387 #undef SDATA_TYPE
388 #undef SUFFIX
389 #undef DATA_SIZE
390 #undef SHIFT
391