1 /* cairo - a vector graphics library with display and print output
2 *
3 * Copyright © 2007 Chris Wilson
4 * Copyright © 2010 Andrea Canciani
5 *
6 * This library is free software; you can redistribute it and/or
7 * modify it either under the terms of the GNU Lesser General Public
8 * License version 2.1 as published by the Free Software Foundation
9 * (the "LGPL") or, at your option, under the terms of the Mozilla
10 * Public License Version 1.1 (the "MPL"). If you do not alter this
11 * notice, a recipient may use your version of this file under either
12 * the MPL or the LGPL.
13 *
14 * You should have received a copy of the LGPL along with this library
15 * in the file COPYING-LGPL-2.1; if not, write to the Free Software
16 * Foundation, Inc., 51 Franklin Street, Suite 500, Boston, MA 02110-1335, USA
17 * You should have received a copy of the MPL along with this library
18 * in the file COPYING-MPL-1.1
19 *
20 * The contents of this file are subject to the Mozilla Public License
21 * Version 1.1 (the "License"); you may not use this file except in
22 * compliance with the License. You may obtain a copy of the License at
23 * http://www.mozilla.org/MPL/
24 *
25 * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY
26 * OF ANY KIND, either express or implied. See the LGPL or the MPL for
27 * the specific language governing rights and limitations.
28 *
29 * The Original Code is the cairo graphics library.
30 *
31 * The Initial Developer of the Original Code is University of Southern
32 * California.
33 *
34 * Contributor(s):
35 * Chris Wilson <chris@chris-wilson.co.uk>
36 * Andrea Canciani <ranma42@gmail.com>
37 */
38
39 #ifndef CAIRO_ATOMIC_PRIVATE_H
40 #define CAIRO_ATOMIC_PRIVATE_H
41
42 # include "cairo-compiler-private.h"
43
44 #if HAVE_CONFIG_H
45 #include "config.h"
46 #endif
47
48 #if HAVE_WIN32_ATOMIC_PRIMITIVES
49 #include <Windows.h>
50 #endif
51
52 /* The autoconf on OpenBSD 4.5 produces the malformed constant name
53 * SIZEOF_VOID__ rather than SIZEOF_VOID_P. Work around that here. */
54 #if !defined(SIZEOF_VOID_P) && defined(SIZEOF_VOID__)
55 # define SIZEOF_VOID_P SIZEOF_VOID__
56 #endif
57
58 CAIRO_BEGIN_DECLS
59
60 /* C++11 atomic primitives were designed to be more flexible than the
61 * __sync_* family of primitives. Despite the name, they are available
62 * in C as well as C++. The motivating reason for using them is that
63 * for _cairo_atomic_{int,ptr}_get, the compiler is able to see that
64 * the load is intended to be atomic, as opposed to the __sync_*
65 * version, below, where the load looks like a plain load. Having
66 * the load appear atomic to the compiler is particular important for
67 * tools like ThreadSanitizer so they don't report false positives on
68 * memory operations that we intend to be atomic.
69 */
70 #if HAVE_CXX11_ATOMIC_PRIMITIVES
71
72 #define HAS_ATOMIC_OPS 1
73
74 typedef int cairo_atomic_int_t;
75
76 static cairo_always_inline cairo_atomic_int_t
_cairo_atomic_int_get(cairo_atomic_int_t * x)77 _cairo_atomic_int_get (cairo_atomic_int_t *x)
78 {
79 return __atomic_load_n(x, __ATOMIC_SEQ_CST);
80 }
81
82 static cairo_always_inline cairo_atomic_int_t
_cairo_atomic_int_get_relaxed(cairo_atomic_int_t * x)83 _cairo_atomic_int_get_relaxed (cairo_atomic_int_t *x)
84 {
85 return __atomic_load_n(x, __ATOMIC_RELAXED);
86 }
87
88 static cairo_always_inline void
_cairo_atomic_int_set_relaxed(cairo_atomic_int_t * x,cairo_atomic_int_t val)89 _cairo_atomic_int_set_relaxed (cairo_atomic_int_t *x, cairo_atomic_int_t val)
90 {
91 __atomic_store_n(x, val, __ATOMIC_RELAXED);
92 }
93
94 static cairo_always_inline void *
_cairo_atomic_ptr_get(void ** x)95 _cairo_atomic_ptr_get (void **x)
96 {
97 return __atomic_load_n(x, __ATOMIC_SEQ_CST);
98 }
99
100 # define _cairo_atomic_int_inc(x) ((void) __atomic_fetch_add(x, 1, __ATOMIC_SEQ_CST))
101 # define _cairo_atomic_int_dec(x) ((void) __atomic_fetch_sub(x, 1, __ATOMIC_SEQ_CST))
102 # define _cairo_atomic_int_dec_and_test(x) (__atomic_fetch_sub(x, 1, __ATOMIC_SEQ_CST) == 1)
103
104 #if SIZEOF_VOID_P==SIZEOF_INT
105 typedef int cairo_atomic_intptr_t;
106 #elif SIZEOF_VOID_P==SIZEOF_LONG
107 typedef long cairo_atomic_intptr_t;
108 #elif SIZEOF_VOID_P==SIZEOF_LONG_LONG
109 typedef long long cairo_atomic_intptr_t;
110 #else
111 #error No matching integer pointer type
112 #endif
113
114 static cairo_always_inline cairo_bool_t
_cairo_atomic_int_cmpxchg_impl(cairo_atomic_int_t * x,cairo_atomic_int_t oldv,cairo_atomic_int_t newv)115 _cairo_atomic_int_cmpxchg_impl(cairo_atomic_int_t *x,
116 cairo_atomic_int_t oldv,
117 cairo_atomic_int_t newv)
118 {
119 cairo_atomic_int_t expected = oldv;
120 return __atomic_compare_exchange_n(x, &expected, newv, 0, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST);
121 }
122
123 #define _cairo_atomic_int_cmpxchg(x, oldv, newv) \
124 _cairo_atomic_int_cmpxchg_impl(x, oldv, newv)
125
126 static cairo_always_inline cairo_atomic_int_t
_cairo_atomic_int_cmpxchg_return_old_impl(cairo_atomic_int_t * x,cairo_atomic_int_t oldv,cairo_atomic_int_t newv)127 _cairo_atomic_int_cmpxchg_return_old_impl(cairo_atomic_int_t *x,
128 cairo_atomic_int_t oldv,
129 cairo_atomic_int_t newv)
130 {
131 cairo_atomic_int_t expected = oldv;
132 (void) __atomic_compare_exchange_n(x, &expected, newv, 0, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST);
133 return expected;
134 }
135
136 #define _cairo_atomic_int_cmpxchg_return_old(x, oldv, newv) \
137 _cairo_atomic_int_cmpxchg_return_old_impl(x, oldv, newv)
138
139 static cairo_always_inline cairo_bool_t
_cairo_atomic_ptr_cmpxchg_impl(void ** x,void * oldv,void * newv)140 _cairo_atomic_ptr_cmpxchg_impl(void **x, void *oldv, void *newv)
141 {
142 void *expected = oldv;
143 return __atomic_compare_exchange_n(x, &expected, newv, 0, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST);
144 }
145
146 #define _cairo_atomic_ptr_cmpxchg(x, oldv, newv) \
147 _cairo_atomic_ptr_cmpxchg_impl(x, oldv, newv)
148
149 static cairo_always_inline void *
_cairo_atomic_ptr_cmpxchg_return_old_impl(void ** x,void * oldv,void * newv)150 _cairo_atomic_ptr_cmpxchg_return_old_impl(void **x, void *oldv, void *newv)
151 {
152 void *expected = oldv;
153 (void) __atomic_compare_exchange_n(x, &expected, newv, 0, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST);
154 return expected;
155 }
156
157 #define _cairo_atomic_ptr_cmpxchg_return_old(x, oldv, newv) \
158 _cairo_atomic_ptr_cmpxchg_return_old_impl(x, oldv, newv)
159
160 #endif
161
162 #if HAVE_WIN32_ATOMIC_PRIMITIVES
163
164 #define HAS_ATOMIC_OPS 1
165
166 typedef volatile long cairo_atomic_int_t;
167
168 # define _cairo_atomic_int_get(x) ((int)*x)
169 # define _cairo_atomic_int_get_relaxed(x) ((int)*(x))
170 # define _cairo_atomic_int_set_relaxed(x, val) (*(x) = (val))
171 # define _cairo_atomic_ptr_get(x) ((void*)*x)
172
173 # define _cairo_atomic_int_inc(x) ((void) InterlockedIncrement(x))
174 # define _cairo_atomic_int_dec(x) ((void) InterlockedDecrement(x))
175 # define _cairo_atomic_int_dec_and_test(x) (InterlockedDecrement(x) == 0)
176 # define _cairo_atomic_int_cmpxchg(x, oldv, newv) (InterlockedCompareExchange(x, newv, oldv) == oldv)
177 # define _cairo_atomic_int_cmpxchg_return_old(x, oldv, newv) InterlockedCompareExchange(x, newv, oldv)
178
179 typedef volatile void* cairo_atomic_intptr_t;
180
181 #define _cairo_atomic_ptr_cmpxchg(x, oldv, newv) (InterlockedCompareExchangePointer(x, newv, oldv) == oldv)
182 #define _cairo_atomic_ptr_cmpxchg_return_old(x, oldv, newv) (InterlockedCompareExchangePointer(x, newv, oldv))
183
184 #endif
185
186 #if HAVE_INTEL_ATOMIC_PRIMITIVES
187
188 #define HAS_ATOMIC_OPS 1
189
190 typedef int cairo_atomic_int_t;
191
192 #ifdef ATOMIC_OP_NEEDS_MEMORY_BARRIER
193 static cairo_always_inline cairo_atomic_int_t
_cairo_atomic_int_get(cairo_atomic_int_t * x)194 _cairo_atomic_int_get (cairo_atomic_int_t *x)
195 {
196 __sync_synchronize ();
197 return *x;
198 }
199
200 static cairo_always_inline cairo_atomic_int_t
_cairo_atomic_int_get_relaxed(cairo_atomic_int_t * x)201 _cairo_atomic_int_get_relaxed (cairo_atomic_int_t *x)
202 {
203 return *x;
204 }
205
206 static cairo_always_inline void
_cairo_atomic_int_set_relaxed(cairo_atomic_int_t * x,cairo_atomic_int_t val)207 _cairo_atomic_int_set_relaxed (cairo_atomic_int_t *x, cairo_atomic_int_t val)
208 {
209 *x = val;
210 }
211
212 static cairo_always_inline void *
_cairo_atomic_ptr_get(void ** x)213 _cairo_atomic_ptr_get (void **x)
214 {
215 __sync_synchronize ();
216 return *x;
217 }
218 #else
219 # define _cairo_atomic_int_get(x) (*x)
220 # define _cairo_atomic_int_get_relaxed(x) (*(x))
221 # define _cairo_atomic_int_set_relaxed(x, val) (*(x) = (val))
222 # define _cairo_atomic_ptr_get(x) (*x)
223 #endif
224
225 # define _cairo_atomic_int_inc(x) ((void) __sync_fetch_and_add(x, 1))
226 # define _cairo_atomic_int_dec_and_test(x) (__sync_fetch_and_add(x, -1) == 1)
227 # define _cairo_atomic_int_cmpxchg(x, oldv, newv) __sync_bool_compare_and_swap (x, oldv, newv)
228 # define _cairo_atomic_int_cmpxchg_return_old(x, oldv, newv) __sync_val_compare_and_swap (x, oldv, newv)
229
230 #if SIZEOF_VOID_P==SIZEOF_INT
231 typedef int cairo_atomic_intptr_t;
232 #elif SIZEOF_VOID_P==SIZEOF_LONG
233 typedef long cairo_atomic_intptr_t;
234 #elif SIZEOF_VOID_P==SIZEOF_LONG_LONG
235 typedef long long cairo_atomic_intptr_t;
236 #else
237 #error No matching integer pointer type
238 #endif
239
240 # define _cairo_atomic_ptr_cmpxchg(x, oldv, newv) \
241 __sync_bool_compare_and_swap ((cairo_atomic_intptr_t*)x, (cairo_atomic_intptr_t)oldv, (cairo_atomic_intptr_t)newv)
242
243 # define _cairo_atomic_ptr_cmpxchg_return_old(x, oldv, newv) \
244 _cairo_atomic_intptr_to_voidptr (__sync_val_compare_and_swap ((cairo_atomic_intptr_t*)x, (cairo_atomic_intptr_t)oldv, (cairo_atomic_intptr_t)newv))
245
246 #endif
247
248 #if HAVE_LIB_ATOMIC_OPS
249 #include <atomic_ops.h>
250
251 #define HAS_ATOMIC_OPS 1
252
253 typedef AO_t cairo_atomic_int_t;
254
255 # define _cairo_atomic_int_get(x) (AO_load_full (x))
256 # define _cairo_atomic_int_get_relaxed(x) (AO_load_full (x))
257 # define _cairo_atomic_int_set_relaxed(x, val) (AO_store_full ((x), (val)))
258
259 # define _cairo_atomic_int_inc(x) ((void) AO_fetch_and_add1_full(x))
260 # define _cairo_atomic_int_dec_and_test(x) (AO_fetch_and_sub1_full(x) == 1)
261 # define _cairo_atomic_int_cmpxchg(x, oldv, newv) AO_compare_and_swap_full(x, oldv, newv)
262
263 #if SIZEOF_VOID_P==SIZEOF_INT
264 typedef unsigned int cairo_atomic_intptr_t;
265 #elif SIZEOF_VOID_P==SIZEOF_LONG
266 typedef unsigned long cairo_atomic_intptr_t;
267 #elif SIZEOF_VOID_P==SIZEOF_LONG_LONG
268 typedef unsigned long long cairo_atomic_intptr_t;
269 #else
270 #error No matching integer pointer type
271 #endif
272
273 # define _cairo_atomic_ptr_get(x) _cairo_atomic_intptr_to_voidptr (AO_load_full (x))
274 # define _cairo_atomic_ptr_cmpxchg(x, oldv, newv) \
275 _cairo_atomic_int_cmpxchg ((cairo_atomic_intptr_t*)(x), (cairo_atomic_intptr_t)oldv, (cairo_atomic_intptr_t)newv)
276
277 #endif
278
279 #if HAVE_OS_ATOMIC_OPS
280 #include <libkern/OSAtomic.h>
281
282 #define HAS_ATOMIC_OPS 1
283
284 typedef int32_t cairo_atomic_int_t;
285
286 # define _cairo_atomic_int_get(x) (OSMemoryBarrier(), *(x))
287 # define _cairo_atomic_int_get_relaxed(x) (*(x))
288 # define _cairo_atomic_int_set_relaxed(x, val) (*(x) = (val))
289
290 # define _cairo_atomic_int_inc(x) ((void) OSAtomicIncrement32Barrier (x))
291 # define _cairo_atomic_int_dec_and_test(x) (OSAtomicDecrement32Barrier (x) == 0)
292 # define _cairo_atomic_int_cmpxchg(x, oldv, newv) OSAtomicCompareAndSwap32Barrier(oldv, newv, x)
293
294 #if SIZEOF_VOID_P==4
295 typedef int32_t cairo_atomic_intptr_t;
296 # define _cairo_atomic_ptr_cmpxchg(x, oldv, newv) \
297 OSAtomicCompareAndSwap32Barrier((cairo_atomic_intptr_t)oldv, (cairo_atomic_intptr_t)newv, (cairo_atomic_intptr_t *)x)
298
299 #elif SIZEOF_VOID_P==8
300 typedef int64_t cairo_atomic_intptr_t;
301 # define _cairo_atomic_ptr_cmpxchg(x, oldv, newv) \
302 OSAtomicCompareAndSwap64Barrier((cairo_atomic_intptr_t)oldv, (cairo_atomic_intptr_t)newv, (cairo_atomic_intptr_t *)x)
303
304 #else
305 #error No matching integer pointer type
306 #endif
307
308 # define _cairo_atomic_ptr_get(x) (OSMemoryBarrier(), *(x))
309
310 #endif
311
312 #ifndef HAS_ATOMIC_OPS
313
314 #if SIZEOF_VOID_P==SIZEOF_INT
315 typedef unsigned int cairo_atomic_intptr_t;
316 #elif SIZEOF_VOID_P==SIZEOF_LONG
317 typedef unsigned long cairo_atomic_intptr_t;
318 #elif SIZEOF_VOID_P==SIZEOF_LONG_LONG
319 typedef unsigned long long cairo_atomic_intptr_t;
320 #else
321 #error No matching integer pointer type
322 #endif
323
324 typedef cairo_atomic_intptr_t cairo_atomic_int_t;
325
326 cairo_private void
327 _cairo_atomic_int_inc (cairo_atomic_int_t *x);
328
329 cairo_private cairo_bool_t
330 _cairo_atomic_int_dec_and_test (cairo_atomic_int_t *x);
331
332 cairo_private cairo_atomic_int_t
333 _cairo_atomic_int_cmpxchg_return_old_impl (cairo_atomic_int_t *x, cairo_atomic_int_t oldv, cairo_atomic_int_t newv);
334
335 cairo_private void *
336 _cairo_atomic_ptr_cmpxchg_return_old_impl (void **x, void *oldv, void *newv);
337
338 #define _cairo_atomic_int_cmpxchg_return_old(x, oldv, newv) _cairo_atomic_int_cmpxchg_return_old_impl (x, oldv, newv)
339 #define _cairo_atomic_ptr_cmpxchg_return_old(x, oldv, newv) _cairo_atomic_ptr_cmpxchg_return_old_impl (x, oldv, newv)
340
341 #ifdef ATOMIC_OP_NEEDS_MEMORY_BARRIER
342 cairo_private cairo_atomic_int_t
343 _cairo_atomic_int_get (cairo_atomic_int_t *x);
344 cairo_private cairo_atomic_int_t
345 _cairo_atomic_int_get_relaxed (cairo_atomic_int_t *x);
346 void
347 _cairo_atomic_int_set_relaxed (cairo_atomic_int_t *x, cairo_atomic_int_t val);
348 # define _cairo_atomic_ptr_get(x) (void *) _cairo_atomic_int_get((cairo_atomic_int_t *) x)
349 #else
350 # define _cairo_atomic_int_get(x) (*x)
351 # define _cairo_atomic_int_get_relaxed(x) (*(x))
352 # define _cairo_atomic_int_set_relaxed(x, val) (*(x) = (val))
353 # define _cairo_atomic_ptr_get(x) (*x)
354 #endif
355
356 #else
357
358 /* Workaround GCC complaining about casts */
359 static cairo_always_inline void *
_cairo_atomic_intptr_to_voidptr(cairo_atomic_intptr_t x)360 _cairo_atomic_intptr_to_voidptr (cairo_atomic_intptr_t x)
361 {
362 return (void *) x;
363 }
364
365 static cairo_always_inline cairo_atomic_int_t
_cairo_atomic_int_cmpxchg_return_old_fallback(cairo_atomic_int_t * x,cairo_atomic_int_t oldv,cairo_atomic_int_t newv)366 _cairo_atomic_int_cmpxchg_return_old_fallback(cairo_atomic_int_t *x, cairo_atomic_int_t oldv, cairo_atomic_int_t newv)
367 {
368 cairo_atomic_int_t curr;
369
370 do {
371 curr = _cairo_atomic_int_get (x);
372 } while (curr == oldv && !_cairo_atomic_int_cmpxchg (x, oldv, newv));
373
374 return curr;
375 }
376
377 static cairo_always_inline void *
_cairo_atomic_ptr_cmpxchg_return_old_fallback(void ** x,void * oldv,void * newv)378 _cairo_atomic_ptr_cmpxchg_return_old_fallback(void **x, void *oldv, void *newv)
379 {
380 void *curr;
381
382 do {
383 curr = _cairo_atomic_ptr_get (x);
384 } while (curr == oldv && !_cairo_atomic_ptr_cmpxchg (x, oldv, newv));
385
386 return curr;
387 }
388 #endif
389
390 #ifndef _cairo_atomic_int_cmpxchg_return_old
391 #define _cairo_atomic_int_cmpxchg_return_old(x, oldv, newv) _cairo_atomic_int_cmpxchg_return_old_fallback (x, oldv, newv)
392 #endif
393
394 #ifndef _cairo_atomic_ptr_cmpxchg_return_old
395 #define _cairo_atomic_ptr_cmpxchg_return_old(x, oldv, newv) _cairo_atomic_ptr_cmpxchg_return_old_fallback (x, oldv, newv)
396 #endif
397
398 #ifndef _cairo_atomic_int_cmpxchg
399 #define _cairo_atomic_int_cmpxchg(x, oldv, newv) (_cairo_atomic_int_cmpxchg_return_old (x, oldv, newv) == oldv)
400 #endif
401
402 #ifndef _cairo_atomic_ptr_cmpxchg
403 #define _cairo_atomic_ptr_cmpxchg(x, oldv, newv) (_cairo_atomic_ptr_cmpxchg_return_old (x, oldv, newv) == oldv)
404 #endif
405
406 #define _cairo_atomic_uint_get(x) _cairo_atomic_int_get(x)
407 #define _cairo_atomic_uint_cmpxchg(x, oldv, newv) \
408 _cairo_atomic_int_cmpxchg((cairo_atomic_int_t *)x, oldv, newv)
409
410 #define _cairo_status_set_error(status, err) do { \
411 /* hide compiler warnings about cairo_status_t != int (gcc treats its as \
412 * an unsigned integer instead, and about ignoring the return value. */ \
413 int ret__ = _cairo_atomic_int_cmpxchg ((cairo_atomic_int_t *) status, CAIRO_STATUS_SUCCESS, err); \
414 (void) ret__; \
415 } while (0)
416
417 CAIRO_END_DECLS
418
419 #endif
420