1 #ifndef JEMALLOC_INTERNAL_ATOMIC_GCC_ATOMIC_H 2 #define JEMALLOC_INTERNAL_ATOMIC_GCC_ATOMIC_H 3 4 #include "jemalloc/internal/assert.h" 5 6 #define ATOMIC_INIT(...) {__VA_ARGS__} 7 8 typedef enum { 9 atomic_memory_order_relaxed, 10 atomic_memory_order_acquire, 11 atomic_memory_order_release, 12 atomic_memory_order_acq_rel, 13 atomic_memory_order_seq_cst 14 } atomic_memory_order_t; 15 16 ATOMIC_INLINE int 17 atomic_enum_to_builtin(atomic_memory_order_t mo) { 18 switch (mo) { 19 case atomic_memory_order_relaxed: 20 return __ATOMIC_RELAXED; 21 case atomic_memory_order_acquire: 22 return __ATOMIC_ACQUIRE; 23 case atomic_memory_order_release: 24 return __ATOMIC_RELEASE; 25 case atomic_memory_order_acq_rel: 26 return __ATOMIC_ACQ_REL; 27 case atomic_memory_order_seq_cst: 28 return __ATOMIC_SEQ_CST; 29 } 30 /* Can't happen; the switch is exhaustive. */ 31 not_reached(); 32 } 33 34 ATOMIC_INLINE void 35 atomic_fence(atomic_memory_order_t mo) { 36 __atomic_thread_fence(atomic_enum_to_builtin(mo)); 37 } 38 39 #define JEMALLOC_GENERATE_ATOMICS(type, short_type, \ 40 /* unused */ lg_size) \ 41 typedef struct { \ 42 type repr; \ 43 } atomic_##short_type##_t; \ 44 \ 45 ATOMIC_INLINE type \ 46 atomic_load_##short_type(const atomic_##short_type##_t *a, \ 47 atomic_memory_order_t mo) { \ 48 type result; \ 49 __atomic_load(&a->repr, &result, atomic_enum_to_builtin(mo)); \ 50 return result; \ 51 } \ 52 \ 53 ATOMIC_INLINE void \ 54 atomic_store_##short_type(atomic_##short_type##_t *a, type val, \ 55 atomic_memory_order_t mo) { \ 56 __atomic_store(&a->repr, &val, atomic_enum_to_builtin(mo)); \ 57 } \ 58 \ 59 ATOMIC_INLINE type \ 60 atomic_exchange_##short_type(atomic_##short_type##_t *a, type val, \ 61 atomic_memory_order_t mo) { \ 62 type result; \ 63 __atomic_exchange(&a->repr, &val, &result, \ 64 atomic_enum_to_builtin(mo)); \ 65 return result; \ 66 } \ 67 \ 68 ATOMIC_INLINE bool \ 69 atomic_compare_exchange_weak_##short_type(atomic_##short_type##_t *a, \ 70 UNUSED type *expected, type desired, \ 71 atomic_memory_order_t success_mo, \ 72 atomic_memory_order_t failure_mo) { \ 73 return __atomic_compare_exchange(&a->repr, expected, &desired, \ 74 true, atomic_enum_to_builtin(success_mo), \ 75 atomic_enum_to_builtin(failure_mo)); \ 76 } \ 77 \ 78 ATOMIC_INLINE bool \ 79 atomic_compare_exchange_strong_##short_type(atomic_##short_type##_t *a, \ 80 UNUSED type *expected, type desired, \ 81 atomic_memory_order_t success_mo, \ 82 atomic_memory_order_t failure_mo) { \ 83 return __atomic_compare_exchange(&a->repr, expected, &desired, \ 84 false, \ 85 atomic_enum_to_builtin(success_mo), \ 86 atomic_enum_to_builtin(failure_mo)); \ 87 } 88 89 90 #define JEMALLOC_GENERATE_INT_ATOMICS(type, short_type, \ 91 /* unused */ lg_size) \ 92 JEMALLOC_GENERATE_ATOMICS(type, short_type, /* unused */ lg_size) \ 93 \ 94 ATOMIC_INLINE type \ 95 atomic_fetch_add_##short_type(atomic_##short_type##_t *a, type val, \ 96 atomic_memory_order_t mo) { \ 97 return __atomic_fetch_add(&a->repr, val, \ 98 atomic_enum_to_builtin(mo)); \ 99 } \ 100 \ 101 ATOMIC_INLINE type \ 102 atomic_fetch_sub_##short_type(atomic_##short_type##_t *a, type val, \ 103 atomic_memory_order_t mo) { \ 104 return __atomic_fetch_sub(&a->repr, val, \ 105 atomic_enum_to_builtin(mo)); \ 106 } \ 107 \ 108 ATOMIC_INLINE type \ 109 atomic_fetch_and_##short_type(atomic_##short_type##_t *a, type val, \ 110 atomic_memory_order_t mo) { \ 111 return __atomic_fetch_and(&a->repr, val, \ 112 atomic_enum_to_builtin(mo)); \ 113 } \ 114 \ 115 ATOMIC_INLINE type \ 116 atomic_fetch_or_##short_type(atomic_##short_type##_t *a, type val, \ 117 atomic_memory_order_t mo) { \ 118 return __atomic_fetch_or(&a->repr, val, \ 119 atomic_enum_to_builtin(mo)); \ 120 } \ 121 \ 122 ATOMIC_INLINE type \ 123 atomic_fetch_xor_##short_type(atomic_##short_type##_t *a, type val, \ 124 atomic_memory_order_t mo) { \ 125 return __atomic_fetch_xor(&a->repr, val, \ 126 atomic_enum_to_builtin(mo)); \ 127 } 128 129 #endif /* JEMALLOC_INTERNAL_ATOMIC_GCC_ATOMIC_H */ 130