1 /* cairo - a vector graphics library with display and print output
2 *
3 * Copyright © 2007 Chris Wilson
4 * Copyright © 2010 Andrea Canciani
5 *
6 * This library is free software; you can redistribute it and/or
7 * modify it either under the terms of the GNU Lesser General Public
8 * License version 2.1 as published by the Free Software Foundation
9 * (the "LGPL") or, at your option, under the terms of the Mozilla
10 * Public License Version 1.1 (the "MPL"). If you do not alter this
11 * notice, a recipient may use your version of this file under either
12 * the MPL or the LGPL.
13 *
14 * You should have received a copy of the LGPL along with this library
15 * in the file COPYING-LGPL-2.1; if not, write to the Free Software
16 * Foundation, Inc., 51 Franklin Street, Suite 500, Boston, MA 02110-1335, USA
17 * You should have received a copy of the MPL along with this library
18 * in the file COPYING-MPL-1.1
19 *
20 * The contents of this file are subject to the Mozilla Public License
21 * Version 1.1 (the "License"); you may not use this file except in
22 * compliance with the License. You may obtain a copy of the License at
23 * http://www.mozilla.org/MPL/
24 *
25 * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY
26 * OF ANY KIND, either express or implied. See the LGPL or the MPL for
27 * the specific language governing rights and limitations.
28 *
29 * The Original Code is the cairo graphics library.
30 *
31 * The Initial Developer of the Original Code is University of Southern
32 * California.
33 *
34 * Contributor(s):
35 * Chris Wilson <chris@chris-wilson.co.uk>
36 * Andrea Canciani <ranma42@gmail.com>
37 */
38
39 #ifndef CAIRO_ATOMIC_PRIVATE_H
40 #define CAIRO_ATOMIC_PRIVATE_H
41
42 # include "cairo-compiler-private.h"
43
44 #if HAVE_CONFIG_H
45 #include "config.h"
46 #endif
47
48 #include <assert.h>
49
50 /* The autoconf on OpenBSD 4.5 produces the malformed constant name
51 * SIZEOF_VOID__ rather than SIZEOF_VOID_P. Work around that here. */
52 #if !defined(SIZEOF_VOID_P) && defined(SIZEOF_VOID__)
53 # define SIZEOF_VOID_P SIZEOF_VOID__
54 #endif
55
56 CAIRO_BEGIN_DECLS
57
58 /* C++11 atomic primitives were designed to be more flexible than the
59 * __sync_* family of primitives. Despite the name, they are available
60 * in C as well as C++. The motivating reason for using them is that
61 * for _cairo_atomic_{int,ptr}_get, the compiler is able to see that
62 * the load is intended to be atomic, as opposed to the __sync_*
63 * version, below, where the load looks like a plain load. Having
64 * the load appear atomic to the compiler is particular important for
65 * tools like ThreadSanitizer so they don't report false positives on
66 * memory operations that we intend to be atomic.
67 */
68 #if HAVE_CXX11_ATOMIC_PRIMITIVES
69
70 #define HAS_ATOMIC_OPS 1
71
72 typedef int cairo_atomic_int_t;
73
74 static cairo_always_inline cairo_atomic_int_t
_cairo_atomic_int_get(cairo_atomic_int_t * x)75 _cairo_atomic_int_get (cairo_atomic_int_t *x)
76 {
77 return __atomic_load_n(x, __ATOMIC_SEQ_CST);
78 }
79
80 static cairo_always_inline cairo_atomic_int_t
_cairo_atomic_int_get_relaxed(cairo_atomic_int_t * x)81 _cairo_atomic_int_get_relaxed (cairo_atomic_int_t *x)
82 {
83 return __atomic_load_n(x, __ATOMIC_RELAXED);
84 }
85
86 static cairo_always_inline void
_cairo_atomic_int_set_relaxed(cairo_atomic_int_t * x,cairo_atomic_int_t val)87 _cairo_atomic_int_set_relaxed (cairo_atomic_int_t *x, cairo_atomic_int_t val)
88 {
89 __atomic_store_n(x, val, __ATOMIC_RELAXED);
90 }
91
92 static cairo_always_inline void *
_cairo_atomic_ptr_get(void ** x)93 _cairo_atomic_ptr_get (void **x)
94 {
95 return __atomic_load_n(x, __ATOMIC_SEQ_CST);
96 }
97
98 # define _cairo_atomic_int_inc(x) ((void) __atomic_fetch_add(x, 1, __ATOMIC_SEQ_CST))
99 # define _cairo_atomic_int_dec(x) ((void) __atomic_fetch_sub(x, 1, __ATOMIC_SEQ_CST))
100 # define _cairo_atomic_int_dec_and_test(x) (__atomic_fetch_sub(x, 1, __ATOMIC_SEQ_CST) == 1)
101
102 #if SIZEOF_VOID_P==SIZEOF_INT
103 typedef int cairo_atomic_intptr_t;
104 #elif SIZEOF_VOID_P==SIZEOF_LONG
105 typedef long cairo_atomic_intptr_t;
106 #elif SIZEOF_VOID_P==SIZEOF_LONG_LONG
107 typedef long long cairo_atomic_intptr_t;
108 #else
109 #error No matching integer pointer type
110 #endif
111
112 static cairo_always_inline cairo_bool_t
_cairo_atomic_int_cmpxchg_impl(cairo_atomic_int_t * x,cairo_atomic_int_t oldv,cairo_atomic_int_t newv)113 _cairo_atomic_int_cmpxchg_impl(cairo_atomic_int_t *x,
114 cairo_atomic_int_t oldv,
115 cairo_atomic_int_t newv)
116 {
117 cairo_atomic_int_t expected = oldv;
118 return __atomic_compare_exchange_n(x, &expected, newv, 0, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST);
119 }
120
121 #define _cairo_atomic_int_cmpxchg(x, oldv, newv) \
122 _cairo_atomic_int_cmpxchg_impl(x, oldv, newv)
123
124 static cairo_always_inline cairo_atomic_int_t
_cairo_atomic_int_cmpxchg_return_old_impl(cairo_atomic_int_t * x,cairo_atomic_int_t oldv,cairo_atomic_int_t newv)125 _cairo_atomic_int_cmpxchg_return_old_impl(cairo_atomic_int_t *x,
126 cairo_atomic_int_t oldv,
127 cairo_atomic_int_t newv)
128 {
129 cairo_atomic_int_t expected = oldv;
130 (void) __atomic_compare_exchange_n(x, &expected, newv, 0, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST);
131 return expected;
132 }
133
134 #define _cairo_atomic_int_cmpxchg_return_old(x, oldv, newv) \
135 _cairo_atomic_int_cmpxchg_return_old_impl(x, oldv, newv)
136
137 static cairo_always_inline cairo_bool_t
_cairo_atomic_ptr_cmpxchg_impl(void ** x,void * oldv,void * newv)138 _cairo_atomic_ptr_cmpxchg_impl(void **x, void *oldv, void *newv)
139 {
140 void *expected = oldv;
141 return __atomic_compare_exchange_n(x, &expected, newv, 0, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST);
142 }
143
144 #define _cairo_atomic_ptr_cmpxchg(x, oldv, newv) \
145 _cairo_atomic_ptr_cmpxchg_impl(x, oldv, newv)
146
147 static cairo_always_inline void *
_cairo_atomic_ptr_cmpxchg_return_old_impl(void ** x,void * oldv,void * newv)148 _cairo_atomic_ptr_cmpxchg_return_old_impl(void **x, void *oldv, void *newv)
149 {
150 void *expected = oldv;
151 (void) __atomic_compare_exchange_n(x, &expected, newv, 0, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST);
152 return expected;
153 }
154
155 #define _cairo_atomic_ptr_cmpxchg_return_old(x, oldv, newv) \
156 _cairo_atomic_ptr_cmpxchg_return_old_impl(x, oldv, newv)
157
158 #endif
159
160 #if HAVE_GCC_LEGACY_ATOMICS
161
162 #define HAS_ATOMIC_OPS 1
163
164 typedef int cairo_atomic_int_t;
165
166 static cairo_always_inline cairo_atomic_int_t
_cairo_atomic_int_get(cairo_atomic_int_t * x)167 _cairo_atomic_int_get (cairo_atomic_int_t *x)
168 {
169 __sync_synchronize ();
170 return *x;
171 }
172
173 static cairo_always_inline cairo_atomic_int_t
_cairo_atomic_int_get_relaxed(cairo_atomic_int_t * x)174 _cairo_atomic_int_get_relaxed (cairo_atomic_int_t *x)
175 {
176 return *x;
177 }
178
179 static cairo_always_inline void
_cairo_atomic_int_set_relaxed(cairo_atomic_int_t * x,cairo_atomic_int_t val)180 _cairo_atomic_int_set_relaxed (cairo_atomic_int_t *x, cairo_atomic_int_t val)
181 {
182 *x = val;
183 }
184
185 static cairo_always_inline void *
_cairo_atomic_ptr_get(void ** x)186 _cairo_atomic_ptr_get (void **x)
187 {
188 __sync_synchronize ();
189 return *x;
190 }
191
192 # define _cairo_atomic_int_inc(x) ((void) __sync_fetch_and_add(x, 1))
193 # define _cairo_atomic_int_dec(x) ((void) __sync_fetch_and_add(x, -1))
194 # define _cairo_atomic_int_dec_and_test(x) (__sync_fetch_and_add(x, -1) == 1)
195 # define _cairo_atomic_int_cmpxchg(x, oldv, newv) __sync_bool_compare_and_swap (x, oldv, newv)
196 # define _cairo_atomic_int_cmpxchg_return_old(x, oldv, newv) __sync_val_compare_and_swap (x, oldv, newv)
197
198 #if SIZEOF_VOID_P==SIZEOF_INT
199 typedef int cairo_atomic_intptr_t;
200 #elif SIZEOF_VOID_P==SIZEOF_LONG
201 typedef long cairo_atomic_intptr_t;
202 #elif SIZEOF_VOID_P==SIZEOF_LONG_LONG
203 typedef long long cairo_atomic_intptr_t;
204 #else
205 #error No matching integer pointer type
206 #endif
207
208 # define _cairo_atomic_ptr_cmpxchg(x, oldv, newv) \
209 __sync_bool_compare_and_swap ((cairo_atomic_intptr_t*)x, (cairo_atomic_intptr_t)oldv, (cairo_atomic_intptr_t)newv)
210
211 # define _cairo_atomic_ptr_cmpxchg_return_old(x, oldv, newv) \
212 _cairo_atomic_intptr_to_voidptr (__sync_val_compare_and_swap ((cairo_atomic_intptr_t*)x, (cairo_atomic_intptr_t)oldv, (cairo_atomic_intptr_t)newv))
213
214 #endif
215
216 #if HAVE_LIB_ATOMIC_OPS
217 #include <atomic_ops.h>
218
219 #define HAS_ATOMIC_OPS 1
220
221 typedef AO_t cairo_atomic_int_t;
222
223 # define _cairo_atomic_int_get(x) (AO_load_full (x))
224 # define _cairo_atomic_int_get_relaxed(x) (AO_load_full (x))
225 # define _cairo_atomic_int_set_relaxed(x, val) (AO_store_full ((x), (val)))
226
227 # define _cairo_atomic_int_inc(x) ((void) AO_fetch_and_add1_full(x))
228 # define _cairo_atomic_int_dec(x) ((void) AO_fetch_and_sub1_full(x))
229 # define _cairo_atomic_int_dec_and_test(x) (AO_fetch_and_sub1_full(x) == 1)
230 # define _cairo_atomic_int_cmpxchg(x, oldv, newv) AO_compare_and_swap_full(x, oldv, newv)
231
232 #if SIZEOF_VOID_P==SIZEOF_INT
233 typedef unsigned int cairo_atomic_intptr_t;
234 #elif SIZEOF_VOID_P==SIZEOF_LONG
235 typedef unsigned long cairo_atomic_intptr_t;
236 #elif SIZEOF_VOID_P==SIZEOF_LONG_LONG
237 typedef unsigned long long cairo_atomic_intptr_t;
238 #else
239 #error No matching integer pointer type
240 #endif
241
242 # define _cairo_atomic_ptr_get(x) _cairo_atomic_intptr_to_voidptr (AO_load_full (x))
243 # define _cairo_atomic_ptr_cmpxchg(x, oldv, newv) \
244 _cairo_atomic_int_cmpxchg ((cairo_atomic_intptr_t*)(x), (cairo_atomic_intptr_t)oldv, (cairo_atomic_intptr_t)newv)
245
246 #endif
247
248 #if HAVE_OS_ATOMIC_OPS
249 #include <libkern/OSAtomic.h>
250
251 #define HAS_ATOMIC_OPS 1
252
253 typedef int32_t cairo_atomic_int_t;
254
255 # define _cairo_atomic_int_get(x) (OSMemoryBarrier(), *(x))
256 # define _cairo_atomic_int_get_relaxed(x) *(x)
257 # define _cairo_atomic_int_set_relaxed(x, val) *(x) = (val)
258
259 # define _cairo_atomic_int_inc(x) ((void) OSAtomicIncrement32Barrier (x))
260 # define _cairo_atomic_int_dec(x) ((void) OSAtomicDecrement32Barrier (x))
261 # define _cairo_atomic_int_dec_and_test(x) (OSAtomicDecrement32Barrier (x) == 0)
262 # define _cairo_atomic_int_cmpxchg(x, oldv, newv) OSAtomicCompareAndSwap32Barrier(oldv, newv, x)
263
264 #if SIZEOF_VOID_P==4
265 typedef int32_t cairo_atomic_intptr_t;
266 # define _cairo_atomic_ptr_cmpxchg(x, oldv, newv) \
267 OSAtomicCompareAndSwap32Barrier((cairo_atomic_intptr_t)oldv, (cairo_atomic_intptr_t)newv, (cairo_atomic_intptr_t *)x)
268
269 #elif SIZEOF_VOID_P==8
270 typedef int64_t cairo_atomic_intptr_t;
271 # define _cairo_atomic_ptr_cmpxchg(x, oldv, newv) \
272 OSAtomicCompareAndSwap64Barrier((cairo_atomic_intptr_t)oldv, (cairo_atomic_intptr_t)newv, (cairo_atomic_intptr_t *)x)
273
274 #else
275 #error No matching integer pointer type
276 #endif
277
278 # define _cairo_atomic_ptr_get(x) (OSMemoryBarrier(), *(x))
279
280 #endif
281
282 #ifndef HAS_ATOMIC_OPS
283
284 #if SIZEOF_VOID_P==SIZEOF_INT
285 typedef unsigned int cairo_atomic_intptr_t;
286 #elif SIZEOF_VOID_P==SIZEOF_LONG
287 typedef unsigned long cairo_atomic_intptr_t;
288 #elif SIZEOF_VOID_P==SIZEOF_LONG_LONG
289 typedef unsigned long long cairo_atomic_intptr_t;
290 #else
291 #error No matching integer pointer type
292 #endif
293
294 typedef cairo_atomic_intptr_t cairo_atomic_int_t;
295
296 cairo_private void
297 _cairo_atomic_int_inc (cairo_atomic_int_t *x);
298
299 #define _cairo_atomic_int_dec(x) _cairo_atomic_int_dec_and_test(x)
300
301 cairo_private cairo_bool_t
302 _cairo_atomic_int_dec_and_test (cairo_atomic_int_t *x);
303
304 cairo_private cairo_atomic_int_t
305 _cairo_atomic_int_cmpxchg_return_old_impl (cairo_atomic_int_t *x, cairo_atomic_int_t oldv, cairo_atomic_int_t newv);
306
307 cairo_private void *
308 _cairo_atomic_ptr_cmpxchg_return_old_impl (void **x, void *oldv, void *newv);
309
310 #define _cairo_atomic_int_cmpxchg_return_old(x, oldv, newv) _cairo_atomic_int_cmpxchg_return_old_impl (x, oldv, newv)
311 #define _cairo_atomic_ptr_cmpxchg_return_old(x, oldv, newv) _cairo_atomic_ptr_cmpxchg_return_old_impl (x, oldv, newv)
312
313 #ifdef ATOMIC_OP_NEEDS_MEMORY_BARRIER
314 cairo_private cairo_atomic_int_t
315 _cairo_atomic_int_get (cairo_atomic_int_t *x);
316 cairo_private cairo_atomic_int_t
317 _cairo_atomic_int_get_relaxed (cairo_atomic_int_t *x);
318 void
319 _cairo_atomic_int_set_relaxed (cairo_atomic_int_t *x, cairo_atomic_int_t val);
320 # define _cairo_atomic_ptr_get(x) (void *) _cairo_atomic_int_get((cairo_atomic_int_t *) x)
321 #else
322 # define _cairo_atomic_int_get(x) (*x)
323 # define _cairo_atomic_int_get_relaxed(x) (*x)
324 # define _cairo_atomic_int_set_relaxed(x, val) (*x) = (val)
325 # define _cairo_atomic_ptr_get(x) (*x)
326 #endif
327
328 #else
329
330 /* Workaround GCC complaining about casts */
331 static cairo_always_inline void *
_cairo_atomic_intptr_to_voidptr(cairo_atomic_intptr_t x)332 _cairo_atomic_intptr_to_voidptr (cairo_atomic_intptr_t x)
333 {
334 return (void *) x;
335 }
336
337 static cairo_always_inline cairo_atomic_int_t
_cairo_atomic_int_cmpxchg_return_old_fallback(cairo_atomic_int_t * x,cairo_atomic_int_t oldv,cairo_atomic_int_t newv)338 _cairo_atomic_int_cmpxchg_return_old_fallback(cairo_atomic_int_t *x, cairo_atomic_int_t oldv, cairo_atomic_int_t newv)
339 {
340 cairo_atomic_int_t curr;
341
342 do {
343 curr = _cairo_atomic_int_get (x);
344 } while (curr == oldv && !_cairo_atomic_int_cmpxchg (x, oldv, newv));
345
346 return curr;
347 }
348
349 static cairo_always_inline void *
_cairo_atomic_ptr_cmpxchg_return_old_fallback(void ** x,void * oldv,void * newv)350 _cairo_atomic_ptr_cmpxchg_return_old_fallback(void **x, void *oldv, void *newv)
351 {
352 void *curr;
353
354 do {
355 curr = _cairo_atomic_ptr_get (x);
356 } while (curr == oldv && !_cairo_atomic_ptr_cmpxchg (x, oldv, newv));
357
358 return curr;
359 }
360 #endif
361
362 #ifndef _cairo_atomic_int_cmpxchg_return_old
363 #define _cairo_atomic_int_cmpxchg_return_old(x, oldv, newv) _cairo_atomic_int_cmpxchg_return_old_fallback (x, oldv, newv)
364 #endif
365
366 #ifndef _cairo_atomic_ptr_cmpxchg_return_old
367 #define _cairo_atomic_ptr_cmpxchg_return_old(x, oldv, newv) _cairo_atomic_ptr_cmpxchg_return_old_fallback (x, oldv, newv)
368 #endif
369
370 #ifndef _cairo_atomic_int_cmpxchg
371 #define _cairo_atomic_int_cmpxchg(x, oldv, newv) (_cairo_atomic_int_cmpxchg_return_old (x, oldv, newv) == oldv)
372 #endif
373
374 #ifndef _cairo_atomic_ptr_cmpxchg
375 #define _cairo_atomic_ptr_cmpxchg(x, oldv, newv) (_cairo_atomic_ptr_cmpxchg_return_old (x, oldv, newv) == oldv)
376 #endif
377
378 #define _cairo_atomic_uint_get(x) _cairo_atomic_int_get(x)
379 #define _cairo_atomic_uint_cmpxchg(x, oldv, newv) \
380 _cairo_atomic_int_cmpxchg((cairo_atomic_int_t *)x, oldv, newv)
381
382 #define _cairo_status_set_error(status, err) do { \
383 int ret__; \
384 assert (err < CAIRO_STATUS_LAST_STATUS); \
385 /* hide compiler warnings about cairo_status_t != int (gcc treats its as \
386 * an unsigned integer instead, and about ignoring the return value. */ \
387 ret__ = _cairo_atomic_int_cmpxchg ((cairo_atomic_int_t *) status, CAIRO_STATUS_SUCCESS, err); \
388 (void) ret__; \
389 } while (0)
390
391 typedef cairo_atomic_int_t cairo_atomic_once_t;
392
393 #define CAIRO_ATOMIC_ONCE_UNINITIALIZED (0)
394 #define CAIRO_ATOMIC_ONCE_INITIALIZING (1)
395 #define CAIRO_ATOMIC_ONCE_INITIALIZED (2)
396 #define CAIRO_ATOMIC_ONCE_INIT CAIRO_ATOMIC_ONCE_UNINITIALIZED
397
398 static cairo_always_inline cairo_bool_t
_cairo_atomic_init_once_enter(cairo_atomic_once_t * once)399 _cairo_atomic_init_once_enter(cairo_atomic_once_t *once)
400 {
401 if (likely(_cairo_atomic_int_get(once) == CAIRO_ATOMIC_ONCE_INITIALIZED))
402 return 0;
403
404 if (_cairo_atomic_int_cmpxchg(once,
405 CAIRO_ATOMIC_ONCE_UNINITIALIZED,
406 CAIRO_ATOMIC_ONCE_INITIALIZING))
407 return 1;
408
409 while (_cairo_atomic_int_get(once) != CAIRO_ATOMIC_ONCE_INITIALIZED) {}
410 return 0;
411 }
412
413 static cairo_always_inline void
_cairo_atomic_init_once_leave(cairo_atomic_once_t * once)414 _cairo_atomic_init_once_leave(cairo_atomic_once_t *once)
415 {
416 if (unlikely(!_cairo_atomic_int_cmpxchg(once,
417 CAIRO_ATOMIC_ONCE_INITIALIZING,
418 CAIRO_ATOMIC_ONCE_INITIALIZED)))
419 assert (0 && "incorrect use of _cairo_atomic_init_once API (once != CAIRO_ATOMIC_ONCE_INITIALIZING)");
420 }
421
422 CAIRO_END_DECLS
423
424 #endif
425