1// -*- C++ -*-
2//===----------------------------------------------------------------------===//
3//
4// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
5// See https://llvm.org/LICENSE.txt for license information.
6// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7//
8//===----------------------------------------------------------------------===//
9
10#ifndef _LIBCPP_ATOMIC
11#define _LIBCPP_ATOMIC
12
13/*
14    atomic synopsis
15
16namespace std
17{
18
19// feature test macro [version.syn]
20
21#define __cpp_lib_atomic_is_always_lock_free
22#define __cpp_lib_atomic_flag_test
23#define __cpp_lib_atomic_lock_free_type_aliases
24#define __cpp_lib_atomic_wait
25
26 // order and consistency
27
28 enum memory_order: unspecified // enum class in C++20
29 {
30    relaxed,
31    consume, // load-consume
32    acquire, // load-acquire
33    release, // store-release
34    acq_rel, // store-release load-acquire
35    seq_cst // store-release load-acquire
36 };
37
38 inline constexpr auto memory_order_relaxed = memory_order::relaxed;
39 inline constexpr auto memory_order_consume = memory_order::consume;
40 inline constexpr auto memory_order_acquire = memory_order::acquire;
41 inline constexpr auto memory_order_release = memory_order::release;
42 inline constexpr auto memory_order_acq_rel = memory_order::acq_rel;
43 inline constexpr auto memory_order_seq_cst = memory_order::seq_cst;
44
45template <class T> T kill_dependency(T y) noexcept;
46
47// lock-free property
48
49#define ATOMIC_BOOL_LOCK_FREE unspecified
50#define ATOMIC_CHAR_LOCK_FREE unspecified
51#define ATOMIC_CHAR8_T_LOCK_FREE unspecified // C++20
52#define ATOMIC_CHAR16_T_LOCK_FREE unspecified
53#define ATOMIC_CHAR32_T_LOCK_FREE unspecified
54#define ATOMIC_WCHAR_T_LOCK_FREE unspecified
55#define ATOMIC_SHORT_LOCK_FREE unspecified
56#define ATOMIC_INT_LOCK_FREE unspecified
57#define ATOMIC_LONG_LOCK_FREE unspecified
58#define ATOMIC_LLONG_LOCK_FREE unspecified
59#define ATOMIC_POINTER_LOCK_FREE unspecified
60
61template <class T>
62struct atomic
63{
64    using value_type = T;
65
66    static constexpr bool is_always_lock_free;
67    bool is_lock_free() const volatile noexcept;
68    bool is_lock_free() const noexcept;
69
70    atomic() noexcept = default; // until C++20
71    constexpr atomic() noexcept(is_nothrow_default_constructible_v<T>); // since C++20
72    constexpr atomic(T desr) noexcept;
73    atomic(const atomic&) = delete;
74    atomic& operator=(const atomic&) = delete;
75    atomic& operator=(const atomic&) volatile = delete;
76
77    T load(memory_order m = memory_order_seq_cst) const volatile noexcept;
78    T load(memory_order m = memory_order_seq_cst) const noexcept;
79    operator T() const volatile noexcept;
80    operator T() const noexcept;
81    void store(T desr, memory_order m = memory_order_seq_cst) volatile noexcept;
82    void store(T desr, memory_order m = memory_order_seq_cst) noexcept;
83    T operator=(T) volatile noexcept;
84    T operator=(T) noexcept;
85
86    T exchange(T desr, memory_order m = memory_order_seq_cst) volatile noexcept;
87    T exchange(T desr, memory_order m = memory_order_seq_cst) noexcept;
88    bool compare_exchange_weak(T& expc, T desr,
89                               memory_order s, memory_order f) volatile noexcept;
90    bool compare_exchange_weak(T& expc, T desr, memory_order s, memory_order f) noexcept;
91    bool compare_exchange_strong(T& expc, T desr,
92                                 memory_order s, memory_order f) volatile noexcept;
93    bool compare_exchange_strong(T& expc, T desr,
94                                 memory_order s, memory_order f) noexcept;
95    bool compare_exchange_weak(T& expc, T desr,
96                               memory_order m = memory_order_seq_cst) volatile noexcept;
97    bool compare_exchange_weak(T& expc, T desr,
98                               memory_order m = memory_order_seq_cst) noexcept;
99    bool compare_exchange_strong(T& expc, T desr,
100                                memory_order m = memory_order_seq_cst) volatile noexcept;
101    bool compare_exchange_strong(T& expc, T desr,
102                                 memory_order m = memory_order_seq_cst) noexcept;
103
104    void wait(T, memory_order = memory_order::seq_cst) const volatile noexcept;
105    void wait(T, memory_order = memory_order::seq_cst) const noexcept;
106    void notify_one() volatile noexcept;
107    void notify_one() noexcept;
108    void notify_all() volatile noexcept;
109    void notify_all() noexcept;
110};
111
112template <>
113struct atomic<integral>
114{
115    using value_type = integral;
116    using difference_type = value_type;
117
118    static constexpr bool is_always_lock_free;
119    bool is_lock_free() const volatile noexcept;
120    bool is_lock_free() const noexcept;
121
122    atomic() noexcept = default;
123    constexpr atomic(integral desr) noexcept;
124    atomic(const atomic&) = delete;
125    atomic& operator=(const atomic&) = delete;
126    atomic& operator=(const atomic&) volatile = delete;
127
128    integral load(memory_order m = memory_order_seq_cst) const volatile noexcept;
129    integral load(memory_order m = memory_order_seq_cst) const noexcept;
130    operator integral() const volatile noexcept;
131    operator integral() const noexcept;
132    void store(integral desr, memory_order m = memory_order_seq_cst) volatile noexcept;
133    void store(integral desr, memory_order m = memory_order_seq_cst) noexcept;
134    integral operator=(integral desr) volatile noexcept;
135    integral operator=(integral desr) noexcept;
136
137    integral exchange(integral desr,
138                      memory_order m = memory_order_seq_cst) volatile noexcept;
139    integral exchange(integral desr, memory_order m = memory_order_seq_cst) noexcept;
140    bool compare_exchange_weak(integral& expc, integral desr,
141                               memory_order s, memory_order f) volatile noexcept;
142    bool compare_exchange_weak(integral& expc, integral desr,
143                               memory_order s, memory_order f) noexcept;
144    bool compare_exchange_strong(integral& expc, integral desr,
145                                 memory_order s, memory_order f) volatile noexcept;
146    bool compare_exchange_strong(integral& expc, integral desr,
147                                 memory_order s, memory_order f) noexcept;
148    bool compare_exchange_weak(integral& expc, integral desr,
149                               memory_order m = memory_order_seq_cst) volatile noexcept;
150    bool compare_exchange_weak(integral& expc, integral desr,
151                               memory_order m = memory_order_seq_cst) noexcept;
152    bool compare_exchange_strong(integral& expc, integral desr,
153                                memory_order m = memory_order_seq_cst) volatile noexcept;
154    bool compare_exchange_strong(integral& expc, integral desr,
155                                 memory_order m = memory_order_seq_cst) noexcept;
156
157    integral fetch_add(integral op, memory_order m = memory_order_seq_cst) volatile noexcept;
158    integral fetch_add(integral op, memory_order m = memory_order_seq_cst) noexcept;
159    integral fetch_sub(integral op, memory_order m = memory_order_seq_cst) volatile noexcept;
160    integral fetch_sub(integral op, memory_order m = memory_order_seq_cst) noexcept;
161    integral fetch_and(integral op, memory_order m = memory_order_seq_cst) volatile noexcept;
162    integral fetch_and(integral op, memory_order m = memory_order_seq_cst) noexcept;
163    integral fetch_or(integral op, memory_order m = memory_order_seq_cst) volatile noexcept;
164    integral fetch_or(integral op, memory_order m = memory_order_seq_cst) noexcept;
165    integral fetch_xor(integral op, memory_order m = memory_order_seq_cst) volatile noexcept;
166    integral fetch_xor(integral op, memory_order m = memory_order_seq_cst) noexcept;
167
168    integral operator++(int) volatile noexcept;
169    integral operator++(int) noexcept;
170    integral operator--(int) volatile noexcept;
171    integral operator--(int) noexcept;
172    integral operator++() volatile noexcept;
173    integral operator++() noexcept;
174    integral operator--() volatile noexcept;
175    integral operator--() noexcept;
176    integral operator+=(integral op) volatile noexcept;
177    integral operator+=(integral op) noexcept;
178    integral operator-=(integral op) volatile noexcept;
179    integral operator-=(integral op) noexcept;
180    integral operator&=(integral op) volatile noexcept;
181    integral operator&=(integral op) noexcept;
182    integral operator|=(integral op) volatile noexcept;
183    integral operator|=(integral op) noexcept;
184    integral operator^=(integral op) volatile noexcept;
185    integral operator^=(integral op) noexcept;
186
187    void wait(integral, memory_order = memory_order::seq_cst) const volatile noexcept;
188    void wait(integral, memory_order = memory_order::seq_cst) const noexcept;
189    void notify_one() volatile noexcept;
190    void notify_one() noexcept;
191    void notify_all() volatile noexcept;
192    void notify_all() noexcept;
193};
194
195template <class T>
196struct atomic<T*>
197{
198    using value_type = T*;
199    using difference_type = ptrdiff_t;
200
201    static constexpr bool is_always_lock_free;
202    bool is_lock_free() const volatile noexcept;
203    bool is_lock_free() const noexcept;
204
205    atomic() noexcept = default; // until C++20
206    constexpr atomic() noexcept; // since C++20
207    constexpr atomic(T* desr) noexcept;
208    atomic(const atomic&) = delete;
209    atomic& operator=(const atomic&) = delete;
210    atomic& operator=(const atomic&) volatile = delete;
211
212    T* load(memory_order m = memory_order_seq_cst) const volatile noexcept;
213    T* load(memory_order m = memory_order_seq_cst) const noexcept;
214    operator T*() const volatile noexcept;
215    operator T*() const noexcept;
216    void store(T* desr, memory_order m = memory_order_seq_cst) volatile noexcept;
217    void store(T* desr, memory_order m = memory_order_seq_cst) noexcept;
218    T* operator=(T*) volatile noexcept;
219    T* operator=(T*) noexcept;
220
221    T* exchange(T* desr, memory_order m = memory_order_seq_cst) volatile noexcept;
222    T* exchange(T* desr, memory_order m = memory_order_seq_cst) noexcept;
223    bool compare_exchange_weak(T*& expc, T* desr,
224                               memory_order s, memory_order f) volatile noexcept;
225    bool compare_exchange_weak(T*& expc, T* desr,
226                               memory_order s, memory_order f) noexcept;
227    bool compare_exchange_strong(T*& expc, T* desr,
228                                 memory_order s, memory_order f) volatile noexcept;
229    bool compare_exchange_strong(T*& expc, T* desr,
230                                 memory_order s, memory_order f) noexcept;
231    bool compare_exchange_weak(T*& expc, T* desr,
232                               memory_order m = memory_order_seq_cst) volatile noexcept;
233    bool compare_exchange_weak(T*& expc, T* desr,
234                               memory_order m = memory_order_seq_cst) noexcept;
235    bool compare_exchange_strong(T*& expc, T* desr,
236                                memory_order m = memory_order_seq_cst) volatile noexcept;
237    bool compare_exchange_strong(T*& expc, T* desr,
238                                 memory_order m = memory_order_seq_cst) noexcept;
239    T* fetch_add(ptrdiff_t op, memory_order m = memory_order_seq_cst) volatile noexcept;
240    T* fetch_add(ptrdiff_t op, memory_order m = memory_order_seq_cst) noexcept;
241    T* fetch_sub(ptrdiff_t op, memory_order m = memory_order_seq_cst) volatile noexcept;
242    T* fetch_sub(ptrdiff_t op, memory_order m = memory_order_seq_cst) noexcept;
243
244    T* operator++(int) volatile noexcept;
245    T* operator++(int) noexcept;
246    T* operator--(int) volatile noexcept;
247    T* operator--(int) noexcept;
248    T* operator++() volatile noexcept;
249    T* operator++() noexcept;
250    T* operator--() volatile noexcept;
251    T* operator--() noexcept;
252    T* operator+=(ptrdiff_t op) volatile noexcept;
253    T* operator+=(ptrdiff_t op) noexcept;
254    T* operator-=(ptrdiff_t op) volatile noexcept;
255    T* operator-=(ptrdiff_t op) noexcept;
256
257    void wait(T*, memory_order = memory_order::seq_cst) const volatile noexcept;
258    void wait(T*, memory_order = memory_order::seq_cst) const noexcept;
259    void notify_one() volatile noexcept;
260    void notify_one() noexcept;
261    void notify_all() volatile noexcept;
262    void notify_all() noexcept;
263};
264
265
266// [atomics.nonmembers], non-member functions
267template<class T>
268  bool atomic_is_lock_free(const volatile atomic<T>*) noexcept;
269template<class T>
270  bool atomic_is_lock_free(const atomic<T>*) noexcept;
271template<class T>
272  void atomic_store(volatile atomic<T>*, atomic<T>::value_type) noexcept;
273template<class T>
274  void atomic_store(atomic<T>*, atomic<T>::value_type) noexcept;
275template<class T>
276  void atomic_store_explicit(volatile atomic<T>*, atomic<T>::value_type,
277                             memory_order) noexcept;
278template<class T>
279  void atomic_store_explicit(atomic<T>*, atomic<T>::value_type,
280                             memory_order) noexcept;
281template<class T>
282  T atomic_load(const volatile atomic<T>*) noexcept;
283template<class T>
284  T atomic_load(const atomic<T>*) noexcept;
285template<class T>
286  T atomic_load_explicit(const volatile atomic<T>*, memory_order) noexcept;
287template<class T>
288  T atomic_load_explicit(const atomic<T>*, memory_order) noexcept;
289template<class T>
290  T atomic_exchange(volatile atomic<T>*, atomic<T>::value_type) noexcept;
291template<class T>
292  T atomic_exchange(atomic<T>*, atomic<T>::value_type) noexcept;
293template<class T>
294  T atomic_exchange_explicit(volatile atomic<T>*, atomic<T>::value_type,
295                             memory_order) noexcept;
296template<class T>
297  T atomic_exchange_explicit(atomic<T>*, atomic<T>::value_type,
298                             memory_order) noexcept;
299template<class T>
300  bool atomic_compare_exchange_weak(volatile atomic<T>*, atomic<T>::value_type*,
301                                    atomic<T>::value_type) noexcept;
302template<class T>
303  bool atomic_compare_exchange_weak(atomic<T>*, atomic<T>::value_type*,
304                                    atomic<T>::value_type) noexcept;
305template<class T>
306  bool atomic_compare_exchange_strong(volatile atomic<T>*, atomic<T>::value_type*,
307                                      atomic<T>::value_type) noexcept;
308template<class T>
309  bool atomic_compare_exchange_strong(atomic<T>*, atomic<T>::value_type*,
310                                      atomic<T>::value_type) noexcept;
311template<class T>
312  bool atomic_compare_exchange_weak_explicit(volatile atomic<T>*, atomic<T>::value_type*,
313                                             atomic<T>::value_type,
314                                             memory_order, memory_order) noexcept;
315template<class T>
316  bool atomic_compare_exchange_weak_explicit(atomic<T>*, atomic<T>::value_type*,
317                                             atomic<T>::value_type,
318                                             memory_order, memory_order) noexcept;
319template<class T>
320  bool atomic_compare_exchange_strong_explicit(volatile atomic<T>*, atomic<T>::value_type*,
321                                               atomic<T>::value_type,
322                                               memory_order, memory_order) noexcept;
323template<class T>
324  bool atomic_compare_exchange_strong_explicit(atomic<T>*, atomic<T>::value_type*,
325                                               atomic<T>::value_type,
326                                               memory_order, memory_order) noexcept;
327
328template<class T>
329  T atomic_fetch_add(volatile atomic<T>*, atomic<T>::difference_type) noexcept;
330template<class T>
331  T atomic_fetch_add(atomic<T>*, atomic<T>::difference_type) noexcept;
332template<class T>
333  T atomic_fetch_add_explicit(volatile atomic<T>*, atomic<T>::difference_type,
334                              memory_order) noexcept;
335template<class T>
336  T atomic_fetch_add_explicit(atomic<T>*, atomic<T>::difference_type,
337                              memory_order) noexcept;
338template<class T>
339  T atomic_fetch_sub(volatile atomic<T>*, atomic<T>::difference_type) noexcept;
340template<class T>
341  T atomic_fetch_sub(atomic<T>*, atomic<T>::difference_type) noexcept;
342template<class T>
343  T atomic_fetch_sub_explicit(volatile atomic<T>*, atomic<T>::difference_type,
344                              memory_order) noexcept;
345template<class T>
346  T atomic_fetch_sub_explicit(atomic<T>*, atomic<T>::difference_type,
347                              memory_order) noexcept;
348template<class T>
349  T atomic_fetch_and(volatile atomic<T>*, atomic<T>::value_type) noexcept;
350template<class T>
351  T atomic_fetch_and(atomic<T>*, atomic<T>::value_type) noexcept;
352template<class T>
353  T atomic_fetch_and_explicit(volatile atomic<T>*, atomic<T>::value_type,
354                              memory_order) noexcept;
355template<class T>
356  T atomic_fetch_and_explicit(atomic<T>*, atomic<T>::value_type,
357                              memory_order) noexcept;
358template<class T>
359  T atomic_fetch_or(volatile atomic<T>*, atomic<T>::value_type) noexcept;
360template<class T>
361  T atomic_fetch_or(atomic<T>*, atomic<T>::value_type) noexcept;
362template<class T>
363  T atomic_fetch_or_explicit(volatile atomic<T>*, atomic<T>::value_type,
364                             memory_order) noexcept;
365template<class T>
366  T atomic_fetch_or_explicit(atomic<T>*, atomic<T>::value_type,
367                             memory_order) noexcept;
368template<class T>
369  T atomic_fetch_xor(volatile atomic<T>*, atomic<T>::value_type) noexcept;
370template<class T>
371  T atomic_fetch_xor(atomic<T>*, atomic<T>::value_type) noexcept;
372template<class T>
373  T atomic_fetch_xor_explicit(volatile atomic<T>*, atomic<T>::value_type,
374                              memory_order) noexcept;
375template<class T>
376  T atomic_fetch_xor_explicit(atomic<T>*, atomic<T>::value_type,
377                              memory_order) noexcept;
378
379template<class T>
380  void atomic_wait(const volatile atomic<T>*, atomic<T>::value_type) noexcept;
381template<class T>
382  void atomic_wait(const atomic<T>*, atomic<T>::value_type) noexcept;
383template<class T>
384  void atomic_wait_explicit(const volatile atomic<T>*, atomic<T>::value_type,
385                            memory_order) noexcept;
386template<class T>
387  void atomic_wait_explicit(const atomic<T>*, atomic<T>::value_type,
388                            memory_order) noexcept;
389template<class T>
390  void atomic_notify_one(volatile atomic<T>*) noexcept;
391template<class T>
392  void atomic_notify_one(atomic<T>*) noexcept;
393template<class T>
394  void atomic_notify_all(volatile atomic<T>*) noexcept;
395template<class T>
396  void atomic_notify_all(atomic<T>*) noexcept;
397
398// Atomics for standard typedef types
399
400typedef atomic<bool>               atomic_bool;
401typedef atomic<char>               atomic_char;
402typedef atomic<signed char>        atomic_schar;
403typedef atomic<unsigned char>      atomic_uchar;
404typedef atomic<short>              atomic_short;
405typedef atomic<unsigned short>     atomic_ushort;
406typedef atomic<int>                atomic_int;
407typedef atomic<unsigned int>       atomic_uint;
408typedef atomic<long>               atomic_long;
409typedef atomic<unsigned long>      atomic_ulong;
410typedef atomic<long long>          atomic_llong;
411typedef atomic<unsigned long long> atomic_ullong;
412typedef atomic<char8_t>            atomic_char8_t; // C++20
413typedef atomic<char16_t>           atomic_char16_t;
414typedef atomic<char32_t>           atomic_char32_t;
415typedef atomic<wchar_t>            atomic_wchar_t;
416
417typedef atomic<int_least8_t>   atomic_int_least8_t;
418typedef atomic<uint_least8_t>  atomic_uint_least8_t;
419typedef atomic<int_least16_t>  atomic_int_least16_t;
420typedef atomic<uint_least16_t> atomic_uint_least16_t;
421typedef atomic<int_least32_t>  atomic_int_least32_t;
422typedef atomic<uint_least32_t> atomic_uint_least32_t;
423typedef atomic<int_least64_t>  atomic_int_least64_t;
424typedef atomic<uint_least64_t> atomic_uint_least64_t;
425
426typedef atomic<int_fast8_t>   atomic_int_fast8_t;
427typedef atomic<uint_fast8_t>  atomic_uint_fast8_t;
428typedef atomic<int_fast16_t>  atomic_int_fast16_t;
429typedef atomic<uint_fast16_t> atomic_uint_fast16_t;
430typedef atomic<int_fast32_t>  atomic_int_fast32_t;
431typedef atomic<uint_fast32_t> atomic_uint_fast32_t;
432typedef atomic<int_fast64_t>  atomic_int_fast64_t;
433typedef atomic<uint_fast64_t> atomic_uint_fast64_t;
434
435typedef atomic<int8_t>   atomic_int8_t;
436typedef atomic<uint8_t>  atomic_uint8_t;
437typedef atomic<int16_t>  atomic_int16_t;
438typedef atomic<uint16_t> atomic_uint16_t;
439typedef atomic<int32_t>  atomic_int32_t;
440typedef atomic<uint32_t> atomic_uint32_t;
441typedef atomic<int64_t>  atomic_int64_t;
442typedef atomic<uint64_t> atomic_uint64_t;
443
444typedef atomic<intptr_t>  atomic_intptr_t;
445typedef atomic<uintptr_t> atomic_uintptr_t;
446typedef atomic<size_t>    atomic_size_t;
447typedef atomic<ptrdiff_t> atomic_ptrdiff_t;
448typedef atomic<intmax_t>  atomic_intmax_t;
449typedef atomic<uintmax_t> atomic_uintmax_t;
450
451// flag type and operations
452
453typedef struct atomic_flag
454{
455    atomic_flag() noexcept = default; // until C++20
456    constexpr atomic_flag() noexcept; // since C++20
457    atomic_flag(const atomic_flag&) = delete;
458    atomic_flag& operator=(const atomic_flag&) = delete;
459    atomic_flag& operator=(const atomic_flag&) volatile = delete;
460
461    bool test(memory_order m = memory_order_seq_cst) volatile noexcept;
462    bool test(memory_order m = memory_order_seq_cst) noexcept;
463    bool test_and_set(memory_order m = memory_order_seq_cst) volatile noexcept;
464    bool test_and_set(memory_order m = memory_order_seq_cst) noexcept;
465    void clear(memory_order m = memory_order_seq_cst) volatile noexcept;
466    void clear(memory_order m = memory_order_seq_cst) noexcept;
467
468    void wait(bool, memory_order = memory_order::seq_cst) const volatile noexcept;
469    void wait(bool, memory_order = memory_order::seq_cst) const noexcept;
470    void notify_one() volatile noexcept;
471    void notify_one() noexcept;
472    void notify_all() volatile noexcept;
473    void notify_all() noexcept;
474} atomic_flag;
475
476bool atomic_flag_test(volatile atomic_flag* obj) noexcept;
477bool atomic_flag_test(atomic_flag* obj) noexcept;
478bool atomic_flag_test_explicit(volatile atomic_flag* obj,
479                               memory_order m) noexcept;
480bool atomic_flag_test_explicit(atomic_flag* obj, memory_order m) noexcept;
481bool atomic_flag_test_and_set(volatile atomic_flag* obj) noexcept;
482bool atomic_flag_test_and_set(atomic_flag* obj) noexcept;
483bool atomic_flag_test_and_set_explicit(volatile atomic_flag* obj,
484                                       memory_order m) noexcept;
485bool atomic_flag_test_and_set_explicit(atomic_flag* obj, memory_order m) noexcept;
486void atomic_flag_clear(volatile atomic_flag* obj) noexcept;
487void atomic_flag_clear(atomic_flag* obj) noexcept;
488void atomic_flag_clear_explicit(volatile atomic_flag* obj, memory_order m) noexcept;
489void atomic_flag_clear_explicit(atomic_flag* obj, memory_order m) noexcept;
490
491void atomic_wait(const volatile atomic_flag* obj, T old) noexcept;
492void atomic_wait(const atomic_flag* obj, T old) noexcept;
493void atomic_wait_explicit(const volatile atomic_flag* obj, T old, memory_order m) noexcept;
494void atomic_wait_explicit(const atomic_flag* obj, T old, memory_order m) noexcept;
495void atomic_one(volatile atomic_flag* obj) noexcept;
496void atomic_one(atomic_flag* obj) noexcept;
497void atomic_all(volatile atomic_flag* obj) noexcept;
498void atomic_all(atomic_flag* obj) noexcept;
499
500// fences
501
502void atomic_thread_fence(memory_order m) noexcept;
503void atomic_signal_fence(memory_order m) noexcept;
504
505// deprecated
506
507template <class T>
508  void atomic_init(volatile atomic<T>* obj, atomic<T>::value_type desr) noexcept;
509
510template <class T>
511  void atomic_init(atomic<T>* obj, atomic<T>::value_type desr) noexcept;
512
513#define ATOMIC_VAR_INIT(value) see below
514
515#define ATOMIC_FLAG_INIT see below
516
517}  // std
518
519*/
520
521#include <__assert> // all public C++ headers provide the assertion handler
522#include <__availability>
523#include <__chrono/duration.h>
524#include <__config>
525#include <__thread/poll_with_backoff.h>
526#include <__thread/timed_backoff_policy.h>
527#include <__type_traits/conditional.h>
528#include <__type_traits/decay.h>
529#include <__type_traits/is_assignable.h>
530#include <__type_traits/is_function.h>
531#include <__type_traits/is_nothrow_default_constructible.h>
532#include <__type_traits/is_same.h>
533#include <__type_traits/is_trivially_copyable.h>
534#include <__type_traits/remove_const.h>
535#include <__type_traits/remove_pointer.h>
536#include <__type_traits/underlying_type.h>
537#include <cstddef>
538#include <cstdint>
539#include <cstring>
540#include <version>
541
542#ifndef _LIBCPP_HAS_NO_THREADS
543# include <__threading_support>
544#endif
545
546#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
547#  pragma GCC system_header
548#endif
549
550#ifdef _LIBCPP_HAS_NO_ATOMIC_HEADER
551# error <atomic> is not implemented
552#endif
553#ifdef kill_dependency
554# error <atomic> is incompatible with <stdatomic.h> before C++23. Please compile with -std=c++23.
555#endif
556
557#define _LIBCPP_CHECK_STORE_MEMORY_ORDER(__m) \
558  _LIBCPP_DIAGNOSE_WARNING(__m == memory_order_consume || \
559                           __m == memory_order_acquire || \
560                           __m == memory_order_acq_rel,   \
561                        "memory order argument to atomic operation is invalid")
562
563#define _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) \
564  _LIBCPP_DIAGNOSE_WARNING(__m == memory_order_release || \
565                           __m == memory_order_acq_rel,   \
566                        "memory order argument to atomic operation is invalid")
567
568#define _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__m, __f) \
569  _LIBCPP_DIAGNOSE_WARNING(__f == memory_order_release || \
570                           __f == memory_order_acq_rel,   \
571                        "memory order argument to atomic operation is invalid")
572
573_LIBCPP_BEGIN_NAMESPACE_STD
574
575// Figure out what the underlying type for `memory_order` would be if it were
576// declared as an unscoped enum (accounting for -fshort-enums). Use this result
577// to pin the underlying type in C++20.
578enum __legacy_memory_order {
579    __mo_relaxed,
580    __mo_consume,
581    __mo_acquire,
582    __mo_release,
583    __mo_acq_rel,
584    __mo_seq_cst
585};
586
587typedef underlying_type<__legacy_memory_order>::type __memory_order_underlying_t;
588
589#if _LIBCPP_STD_VER > 17
590
591enum class memory_order : __memory_order_underlying_t {
592  relaxed = __mo_relaxed,
593  consume = __mo_consume,
594  acquire = __mo_acquire,
595  release = __mo_release,
596  acq_rel = __mo_acq_rel,
597  seq_cst = __mo_seq_cst
598};
599
600inline constexpr auto memory_order_relaxed = memory_order::relaxed;
601inline constexpr auto memory_order_consume = memory_order::consume;
602inline constexpr auto memory_order_acquire = memory_order::acquire;
603inline constexpr auto memory_order_release = memory_order::release;
604inline constexpr auto memory_order_acq_rel = memory_order::acq_rel;
605inline constexpr auto memory_order_seq_cst = memory_order::seq_cst;
606
607#else
608
609typedef enum memory_order {
610  memory_order_relaxed = __mo_relaxed,
611  memory_order_consume = __mo_consume,
612  memory_order_acquire = __mo_acquire,
613  memory_order_release = __mo_release,
614  memory_order_acq_rel = __mo_acq_rel,
615  memory_order_seq_cst = __mo_seq_cst,
616} memory_order;
617
618#endif // _LIBCPP_STD_VER > 17
619
620template <typename _Tp> _LIBCPP_INLINE_VISIBILITY
621bool __cxx_nonatomic_compare_equal(_Tp const& __lhs, _Tp const& __rhs) {
622    return _VSTD::memcmp(&__lhs, &__rhs, sizeof(_Tp)) == 0;
623}
624
625static_assert((is_same<underlying_type<memory_order>::type, __memory_order_underlying_t>::value),
626  "unexpected underlying type for std::memory_order");
627
628#if defined(_LIBCPP_HAS_GCC_ATOMIC_IMP) || \
629    defined(_LIBCPP_ATOMIC_ONLY_USE_BUILTINS)
630
631// [atomics.types.generic]p1 guarantees _Tp is trivially copyable. Because
632// the default operator= in an object is not volatile, a byte-by-byte copy
633// is required.
634template <typename _Tp, typename _Tv> _LIBCPP_INLINE_VISIBILITY
635typename enable_if<is_assignable<_Tp&, _Tv>::value>::type
636__cxx_atomic_assign_volatile(_Tp& __a_value, _Tv const& __val) {
637  __a_value = __val;
638}
639template <typename _Tp, typename _Tv> _LIBCPP_INLINE_VISIBILITY
640typename enable_if<is_assignable<_Tp&, _Tv>::value>::type
641__cxx_atomic_assign_volatile(_Tp volatile& __a_value, _Tv volatile const& __val) {
642  volatile char* __to = reinterpret_cast<volatile char*>(&__a_value);
643  volatile char* __end = __to + sizeof(_Tp);
644  volatile const char* __from = reinterpret_cast<volatile const char*>(&__val);
645  while (__to != __end)
646    *__to++ = *__from++;
647}
648
649#endif
650
651#if defined(_LIBCPP_HAS_GCC_ATOMIC_IMP)
652
653template <typename _Tp>
654struct __cxx_atomic_base_impl {
655
656  _LIBCPP_INLINE_VISIBILITY
657#ifndef _LIBCPP_CXX03_LANG
658    __cxx_atomic_base_impl() _NOEXCEPT = default;
659#else
660    __cxx_atomic_base_impl() _NOEXCEPT : __a_value() {}
661#endif // _LIBCPP_CXX03_LANG
662  _LIBCPP_CONSTEXPR explicit __cxx_atomic_base_impl(_Tp value) _NOEXCEPT
663    : __a_value(value) {}
664  _Tp __a_value;
665};
666
667_LIBCPP_INLINE_VISIBILITY inline _LIBCPP_CONSTEXPR int __to_gcc_order(memory_order __order) {
668  // Avoid switch statement to make this a constexpr.
669  return __order == memory_order_relaxed ? __ATOMIC_RELAXED:
670         (__order == memory_order_acquire ? __ATOMIC_ACQUIRE:
671          (__order == memory_order_release ? __ATOMIC_RELEASE:
672           (__order == memory_order_seq_cst ? __ATOMIC_SEQ_CST:
673            (__order == memory_order_acq_rel ? __ATOMIC_ACQ_REL:
674              __ATOMIC_CONSUME))));
675}
676
677_LIBCPP_INLINE_VISIBILITY inline _LIBCPP_CONSTEXPR int __to_gcc_failure_order(memory_order __order) {
678  // Avoid switch statement to make this a constexpr.
679  return __order == memory_order_relaxed ? __ATOMIC_RELAXED:
680         (__order == memory_order_acquire ? __ATOMIC_ACQUIRE:
681          (__order == memory_order_release ? __ATOMIC_RELAXED:
682           (__order == memory_order_seq_cst ? __ATOMIC_SEQ_CST:
683            (__order == memory_order_acq_rel ? __ATOMIC_ACQUIRE:
684              __ATOMIC_CONSUME))));
685}
686
687template <typename _Tp>
688_LIBCPP_INLINE_VISIBILITY
689void __cxx_atomic_init(volatile __cxx_atomic_base_impl<_Tp>* __a,  _Tp __val) {
690  __cxx_atomic_assign_volatile(__a->__a_value, __val);
691}
692
693template <typename _Tp>
694_LIBCPP_INLINE_VISIBILITY
695void __cxx_atomic_init(__cxx_atomic_base_impl<_Tp>* __a,  _Tp __val) {
696  __a->__a_value = __val;
697}
698
699_LIBCPP_INLINE_VISIBILITY inline
700void __cxx_atomic_thread_fence(memory_order __order) {
701  __atomic_thread_fence(__to_gcc_order(__order));
702}
703
704_LIBCPP_INLINE_VISIBILITY inline
705void __cxx_atomic_signal_fence(memory_order __order) {
706  __atomic_signal_fence(__to_gcc_order(__order));
707}
708
709template <typename _Tp>
710_LIBCPP_INLINE_VISIBILITY
711void __cxx_atomic_store(volatile __cxx_atomic_base_impl<_Tp>* __a,  _Tp __val,
712                        memory_order __order) {
713  __atomic_store(&__a->__a_value, &__val,
714                 __to_gcc_order(__order));
715}
716
717template <typename _Tp>
718_LIBCPP_INLINE_VISIBILITY
719void __cxx_atomic_store(__cxx_atomic_base_impl<_Tp>* __a,  _Tp __val,
720                        memory_order __order) {
721  __atomic_store(&__a->__a_value, &__val,
722                 __to_gcc_order(__order));
723}
724
725template <typename _Tp>
726_LIBCPP_INLINE_VISIBILITY
727_Tp __cxx_atomic_load(const volatile __cxx_atomic_base_impl<_Tp>* __a,
728                      memory_order __order) {
729  _Tp __ret;
730  __atomic_load(&__a->__a_value, &__ret,
731                __to_gcc_order(__order));
732  return __ret;
733}
734
735template <typename _Tp>
736_LIBCPP_INLINE_VISIBILITY
737_Tp __cxx_atomic_load(const __cxx_atomic_base_impl<_Tp>* __a, memory_order __order) {
738  _Tp __ret;
739  __atomic_load(&__a->__a_value, &__ret,
740                __to_gcc_order(__order));
741  return __ret;
742}
743
744template <typename _Tp>
745_LIBCPP_INLINE_VISIBILITY
746_Tp __cxx_atomic_exchange(volatile __cxx_atomic_base_impl<_Tp>* __a,
747                          _Tp __value, memory_order __order) {
748  _Tp __ret;
749  __atomic_exchange(&__a->__a_value, &__value, &__ret,
750                    __to_gcc_order(__order));
751  return __ret;
752}
753
754template <typename _Tp>
755_LIBCPP_INLINE_VISIBILITY
756_Tp __cxx_atomic_exchange(__cxx_atomic_base_impl<_Tp>* __a, _Tp __value,
757                          memory_order __order) {
758  _Tp __ret;
759  __atomic_exchange(&__a->__a_value, &__value, &__ret,
760                    __to_gcc_order(__order));
761  return __ret;
762}
763
764template <typename _Tp>
765_LIBCPP_INLINE_VISIBILITY
766bool __cxx_atomic_compare_exchange_strong(
767    volatile __cxx_atomic_base_impl<_Tp>* __a, _Tp* __expected, _Tp __value,
768    memory_order __success, memory_order __failure) {
769  return __atomic_compare_exchange(&__a->__a_value, __expected, &__value,
770                                   false,
771                                   __to_gcc_order(__success),
772                                   __to_gcc_failure_order(__failure));
773}
774
775template <typename _Tp>
776_LIBCPP_INLINE_VISIBILITY
777bool __cxx_atomic_compare_exchange_strong(
778    __cxx_atomic_base_impl<_Tp>* __a, _Tp* __expected, _Tp __value, memory_order __success,
779    memory_order __failure) {
780  return __atomic_compare_exchange(&__a->__a_value, __expected, &__value,
781                                   false,
782                                   __to_gcc_order(__success),
783                                   __to_gcc_failure_order(__failure));
784}
785
786template <typename _Tp>
787_LIBCPP_INLINE_VISIBILITY
788bool __cxx_atomic_compare_exchange_weak(
789    volatile __cxx_atomic_base_impl<_Tp>* __a, _Tp* __expected, _Tp __value,
790    memory_order __success, memory_order __failure) {
791  return __atomic_compare_exchange(&__a->__a_value, __expected, &__value,
792                                   true,
793                                   __to_gcc_order(__success),
794                                   __to_gcc_failure_order(__failure));
795}
796
797template <typename _Tp>
798_LIBCPP_INLINE_VISIBILITY
799bool __cxx_atomic_compare_exchange_weak(
800    __cxx_atomic_base_impl<_Tp>* __a, _Tp* __expected, _Tp __value, memory_order __success,
801    memory_order __failure) {
802  return __atomic_compare_exchange(&__a->__a_value, __expected, &__value,
803                                   true,
804                                   __to_gcc_order(__success),
805                                   __to_gcc_failure_order(__failure));
806}
807
808template <typename _Tp>
809struct __skip_amt { enum {value = 1}; };
810
811template <typename _Tp>
812struct __skip_amt<_Tp*> { enum {value = sizeof(_Tp)}; };
813
814// FIXME: Haven't figured out what the spec says about using arrays with
815// atomic_fetch_add. Force a failure rather than creating bad behavior.
816template <typename _Tp>
817struct __skip_amt<_Tp[]> { };
818template <typename _Tp, int n>
819struct __skip_amt<_Tp[n]> { };
820
821template <typename _Tp, typename _Td>
822_LIBCPP_INLINE_VISIBILITY
823_Tp __cxx_atomic_fetch_add(volatile __cxx_atomic_base_impl<_Tp>* __a,
824                           _Td __delta, memory_order __order) {
825  return __atomic_fetch_add(&__a->__a_value, __delta * __skip_amt<_Tp>::value,
826                            __to_gcc_order(__order));
827}
828
829template <typename _Tp, typename _Td>
830_LIBCPP_INLINE_VISIBILITY
831_Tp __cxx_atomic_fetch_add(__cxx_atomic_base_impl<_Tp>* __a, _Td __delta,
832                           memory_order __order) {
833  return __atomic_fetch_add(&__a->__a_value, __delta * __skip_amt<_Tp>::value,
834                            __to_gcc_order(__order));
835}
836
837template <typename _Tp, typename _Td>
838_LIBCPP_INLINE_VISIBILITY
839_Tp __cxx_atomic_fetch_sub(volatile __cxx_atomic_base_impl<_Tp>* __a,
840                           _Td __delta, memory_order __order) {
841  return __atomic_fetch_sub(&__a->__a_value, __delta * __skip_amt<_Tp>::value,
842                            __to_gcc_order(__order));
843}
844
845template <typename _Tp, typename _Td>
846_LIBCPP_INLINE_VISIBILITY
847_Tp __cxx_atomic_fetch_sub(__cxx_atomic_base_impl<_Tp>* __a, _Td __delta,
848                           memory_order __order) {
849  return __atomic_fetch_sub(&__a->__a_value, __delta * __skip_amt<_Tp>::value,
850                            __to_gcc_order(__order));
851}
852
853template <typename _Tp>
854_LIBCPP_INLINE_VISIBILITY
855_Tp __cxx_atomic_fetch_and(volatile __cxx_atomic_base_impl<_Tp>* __a,
856                           _Tp __pattern, memory_order __order) {
857  return __atomic_fetch_and(&__a->__a_value, __pattern,
858                            __to_gcc_order(__order));
859}
860
861template <typename _Tp>
862_LIBCPP_INLINE_VISIBILITY
863_Tp __cxx_atomic_fetch_and(__cxx_atomic_base_impl<_Tp>* __a,
864                           _Tp __pattern, memory_order __order) {
865  return __atomic_fetch_and(&__a->__a_value, __pattern,
866                            __to_gcc_order(__order));
867}
868
869template <typename _Tp>
870_LIBCPP_INLINE_VISIBILITY
871_Tp __cxx_atomic_fetch_or(volatile __cxx_atomic_base_impl<_Tp>* __a,
872                          _Tp __pattern, memory_order __order) {
873  return __atomic_fetch_or(&__a->__a_value, __pattern,
874                           __to_gcc_order(__order));
875}
876
877template <typename _Tp>
878_LIBCPP_INLINE_VISIBILITY
879_Tp __cxx_atomic_fetch_or(__cxx_atomic_base_impl<_Tp>* __a, _Tp __pattern,
880                          memory_order __order) {
881  return __atomic_fetch_or(&__a->__a_value, __pattern,
882                           __to_gcc_order(__order));
883}
884
885template <typename _Tp>
886_LIBCPP_INLINE_VISIBILITY
887_Tp __cxx_atomic_fetch_xor(volatile __cxx_atomic_base_impl<_Tp>* __a,
888                           _Tp __pattern, memory_order __order) {
889  return __atomic_fetch_xor(&__a->__a_value, __pattern,
890                            __to_gcc_order(__order));
891}
892
893template <typename _Tp>
894_LIBCPP_INLINE_VISIBILITY
895_Tp __cxx_atomic_fetch_xor(__cxx_atomic_base_impl<_Tp>* __a, _Tp __pattern,
896                           memory_order __order) {
897  return __atomic_fetch_xor(&__a->__a_value, __pattern,
898                            __to_gcc_order(__order));
899}
900
901#define __cxx_atomic_is_lock_free(__s) __atomic_is_lock_free(__s, 0)
902
903#elif defined(_LIBCPP_HAS_C_ATOMIC_IMP)
904
905template <typename _Tp>
906struct __cxx_atomic_base_impl {
907
908  _LIBCPP_INLINE_VISIBILITY
909#ifndef _LIBCPP_CXX03_LANG
910    __cxx_atomic_base_impl() _NOEXCEPT = default;
911#else
912    __cxx_atomic_base_impl() _NOEXCEPT : __a_value() {}
913#endif // _LIBCPP_CXX03_LANG
914  _LIBCPP_CONSTEXPR explicit __cxx_atomic_base_impl(_Tp __value) _NOEXCEPT
915    : __a_value(__value) {}
916  _LIBCPP_DISABLE_EXTENSION_WARNING _Atomic(_Tp) __a_value;
917};
918
919#define __cxx_atomic_is_lock_free(__s) __c11_atomic_is_lock_free(__s)
920
921_LIBCPP_INLINE_VISIBILITY inline
922void __cxx_atomic_thread_fence(memory_order __order) _NOEXCEPT {
923    __c11_atomic_thread_fence(static_cast<__memory_order_underlying_t>(__order));
924}
925
926_LIBCPP_INLINE_VISIBILITY inline
927void __cxx_atomic_signal_fence(memory_order __order) _NOEXCEPT {
928    __c11_atomic_signal_fence(static_cast<__memory_order_underlying_t>(__order));
929}
930
931template<class _Tp>
932_LIBCPP_INLINE_VISIBILITY
933void __cxx_atomic_init(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __val) _NOEXCEPT {
934    __c11_atomic_init(&__a->__a_value, __val);
935}
936template<class _Tp>
937_LIBCPP_INLINE_VISIBILITY
938void __cxx_atomic_init(__cxx_atomic_base_impl<_Tp> * __a, _Tp __val) _NOEXCEPT {
939    __c11_atomic_init(&__a->__a_value, __val);
940}
941
942template<class _Tp>
943_LIBCPP_INLINE_VISIBILITY
944void __cxx_atomic_store(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __val, memory_order __order) _NOEXCEPT {
945    __c11_atomic_store(&__a->__a_value, __val, static_cast<__memory_order_underlying_t>(__order));
946}
947template<class _Tp>
948_LIBCPP_INLINE_VISIBILITY
949void __cxx_atomic_store(__cxx_atomic_base_impl<_Tp> * __a, _Tp __val, memory_order __order) _NOEXCEPT {
950    __c11_atomic_store(&__a->__a_value, __val, static_cast<__memory_order_underlying_t>(__order));
951}
952
953template<class _Tp>
954_LIBCPP_INLINE_VISIBILITY
955_Tp __cxx_atomic_load(__cxx_atomic_base_impl<_Tp> const volatile* __a, memory_order __order) _NOEXCEPT {
956    using __ptr_type = __remove_const_t<decltype(__a->__a_value)>*;
957    return __c11_atomic_load(const_cast<__ptr_type>(&__a->__a_value), static_cast<__memory_order_underlying_t>(__order));
958}
959template<class _Tp>
960_LIBCPP_INLINE_VISIBILITY
961_Tp __cxx_atomic_load(__cxx_atomic_base_impl<_Tp> const* __a, memory_order __order) _NOEXCEPT {
962    using __ptr_type = __remove_const_t<decltype(__a->__a_value)>*;
963    return __c11_atomic_load(const_cast<__ptr_type>(&__a->__a_value), static_cast<__memory_order_underlying_t>(__order));
964}
965
966template<class _Tp>
967_LIBCPP_INLINE_VISIBILITY
968_Tp __cxx_atomic_exchange(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __value, memory_order __order) _NOEXCEPT {
969    return __c11_atomic_exchange(&__a->__a_value, __value, static_cast<__memory_order_underlying_t>(__order));
970}
971template<class _Tp>
972_LIBCPP_INLINE_VISIBILITY
973_Tp __cxx_atomic_exchange(__cxx_atomic_base_impl<_Tp> * __a, _Tp __value, memory_order __order) _NOEXCEPT {
974    return __c11_atomic_exchange(&__a->__a_value, __value, static_cast<__memory_order_underlying_t>(__order));
975}
976
977_LIBCPP_INLINE_VISIBILITY inline _LIBCPP_CONSTEXPR memory_order __to_failure_order(memory_order __order) {
978  // Avoid switch statement to make this a constexpr.
979  return __order == memory_order_release ? memory_order_relaxed:
980         (__order == memory_order_acq_rel ? memory_order_acquire:
981             __order);
982}
983
984template<class _Tp>
985_LIBCPP_INLINE_VISIBILITY
986bool __cxx_atomic_compare_exchange_strong(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp* __expected, _Tp __value, memory_order __success, memory_order __failure) _NOEXCEPT {
987    return __c11_atomic_compare_exchange_strong(&__a->__a_value, __expected, __value, static_cast<__memory_order_underlying_t>(__success), static_cast<__memory_order_underlying_t>(__to_failure_order(__failure)));
988}
989template<class _Tp>
990_LIBCPP_INLINE_VISIBILITY
991bool __cxx_atomic_compare_exchange_strong(__cxx_atomic_base_impl<_Tp> * __a, _Tp* __expected, _Tp __value, memory_order __success, memory_order __failure) _NOEXCEPT {
992    return __c11_atomic_compare_exchange_strong(&__a->__a_value, __expected, __value, static_cast<__memory_order_underlying_t>(__success), static_cast<__memory_order_underlying_t>(__to_failure_order(__failure)));
993}
994
995template<class _Tp>
996_LIBCPP_INLINE_VISIBILITY
997bool __cxx_atomic_compare_exchange_weak(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp* __expected, _Tp __value, memory_order __success, memory_order __failure) _NOEXCEPT {
998    return __c11_atomic_compare_exchange_weak(&__a->__a_value, __expected, __value, static_cast<__memory_order_underlying_t>(__success), static_cast<__memory_order_underlying_t>(__to_failure_order(__failure)));
999}
1000template<class _Tp>
1001_LIBCPP_INLINE_VISIBILITY
1002bool __cxx_atomic_compare_exchange_weak(__cxx_atomic_base_impl<_Tp> * __a, _Tp* __expected, _Tp __value, memory_order __success, memory_order __failure) _NOEXCEPT {
1003    return __c11_atomic_compare_exchange_weak(&__a->__a_value, __expected, __value,  static_cast<__memory_order_underlying_t>(__success), static_cast<__memory_order_underlying_t>(__to_failure_order(__failure)));
1004}
1005
1006template<class _Tp>
1007_LIBCPP_INLINE_VISIBILITY
1008_Tp __cxx_atomic_fetch_add(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __delta, memory_order __order) _NOEXCEPT {
1009    return __c11_atomic_fetch_add(&__a->__a_value, __delta, static_cast<__memory_order_underlying_t>(__order));
1010}
1011template<class _Tp>
1012_LIBCPP_INLINE_VISIBILITY
1013_Tp __cxx_atomic_fetch_add(__cxx_atomic_base_impl<_Tp> * __a, _Tp __delta, memory_order __order) _NOEXCEPT {
1014    return __c11_atomic_fetch_add(&__a->__a_value, __delta, static_cast<__memory_order_underlying_t>(__order));
1015}
1016
1017template<class _Tp>
1018_LIBCPP_INLINE_VISIBILITY
1019_Tp* __cxx_atomic_fetch_add(__cxx_atomic_base_impl<_Tp*> volatile* __a, ptrdiff_t __delta, memory_order __order) _NOEXCEPT {
1020    return __c11_atomic_fetch_add(&__a->__a_value, __delta, static_cast<__memory_order_underlying_t>(__order));
1021}
1022template<class _Tp>
1023_LIBCPP_INLINE_VISIBILITY
1024_Tp* __cxx_atomic_fetch_add(__cxx_atomic_base_impl<_Tp*> * __a, ptrdiff_t __delta, memory_order __order) _NOEXCEPT {
1025    return __c11_atomic_fetch_add(&__a->__a_value, __delta, static_cast<__memory_order_underlying_t>(__order));
1026}
1027
1028template<class _Tp>
1029_LIBCPP_INLINE_VISIBILITY
1030_Tp __cxx_atomic_fetch_sub(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __delta, memory_order __order) _NOEXCEPT {
1031    return __c11_atomic_fetch_sub(&__a->__a_value, __delta, static_cast<__memory_order_underlying_t>(__order));
1032}
1033template<class _Tp>
1034_LIBCPP_INLINE_VISIBILITY
1035_Tp __cxx_atomic_fetch_sub(__cxx_atomic_base_impl<_Tp> * __a, _Tp __delta, memory_order __order) _NOEXCEPT {
1036    return __c11_atomic_fetch_sub(&__a->__a_value, __delta, static_cast<__memory_order_underlying_t>(__order));
1037}
1038template<class _Tp>
1039_LIBCPP_INLINE_VISIBILITY
1040_Tp* __cxx_atomic_fetch_sub(__cxx_atomic_base_impl<_Tp*> volatile* __a, ptrdiff_t __delta, memory_order __order) _NOEXCEPT {
1041    return __c11_atomic_fetch_sub(&__a->__a_value, __delta, static_cast<__memory_order_underlying_t>(__order));
1042}
1043template<class _Tp>
1044_LIBCPP_INLINE_VISIBILITY
1045_Tp* __cxx_atomic_fetch_sub(__cxx_atomic_base_impl<_Tp*> * __a, ptrdiff_t __delta, memory_order __order) _NOEXCEPT {
1046    return __c11_atomic_fetch_sub(&__a->__a_value, __delta, static_cast<__memory_order_underlying_t>(__order));
1047}
1048
1049template<class _Tp>
1050_LIBCPP_INLINE_VISIBILITY
1051_Tp __cxx_atomic_fetch_and(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __pattern, memory_order __order) _NOEXCEPT {
1052    return __c11_atomic_fetch_and(&__a->__a_value, __pattern, static_cast<__memory_order_underlying_t>(__order));
1053}
1054template<class _Tp>
1055_LIBCPP_INLINE_VISIBILITY
1056_Tp __cxx_atomic_fetch_and(__cxx_atomic_base_impl<_Tp> * __a, _Tp __pattern, memory_order __order) _NOEXCEPT {
1057    return __c11_atomic_fetch_and(&__a->__a_value, __pattern, static_cast<__memory_order_underlying_t>(__order));
1058}
1059
1060template<class _Tp>
1061_LIBCPP_INLINE_VISIBILITY
1062_Tp __cxx_atomic_fetch_or(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __pattern, memory_order __order) _NOEXCEPT {
1063    return __c11_atomic_fetch_or(&__a->__a_value, __pattern, static_cast<__memory_order_underlying_t>(__order));
1064}
1065template<class _Tp>
1066_LIBCPP_INLINE_VISIBILITY
1067_Tp __cxx_atomic_fetch_or(__cxx_atomic_base_impl<_Tp> * __a, _Tp __pattern, memory_order __order) _NOEXCEPT {
1068    return __c11_atomic_fetch_or(&__a->__a_value, __pattern, static_cast<__memory_order_underlying_t>(__order));
1069}
1070
1071template<class _Tp>
1072_LIBCPP_INLINE_VISIBILITY
1073_Tp __cxx_atomic_fetch_xor(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __pattern, memory_order __order) _NOEXCEPT {
1074    return __c11_atomic_fetch_xor(&__a->__a_value, __pattern, static_cast<__memory_order_underlying_t>(__order));
1075}
1076template<class _Tp>
1077_LIBCPP_INLINE_VISIBILITY
1078_Tp __cxx_atomic_fetch_xor(__cxx_atomic_base_impl<_Tp> * __a, _Tp __pattern, memory_order __order) _NOEXCEPT {
1079    return __c11_atomic_fetch_xor(&__a->__a_value, __pattern, static_cast<__memory_order_underlying_t>(__order));
1080}
1081
1082#endif // _LIBCPP_HAS_GCC_ATOMIC_IMP, _LIBCPP_HAS_C_ATOMIC_IMP
1083
1084template <class _Tp>
1085_LIBCPP_INLINE_VISIBILITY
1086_Tp kill_dependency(_Tp __y) _NOEXCEPT
1087{
1088    return __y;
1089}
1090
1091#if defined(__CLANG_ATOMIC_BOOL_LOCK_FREE)
1092# define ATOMIC_BOOL_LOCK_FREE      __CLANG_ATOMIC_BOOL_LOCK_FREE
1093# define ATOMIC_CHAR_LOCK_FREE      __CLANG_ATOMIC_CHAR_LOCK_FREE
1094#ifndef _LIBCPP_HAS_NO_CHAR8_T
1095# define ATOMIC_CHAR8_T_LOCK_FREE   __CLANG_ATOMIC_CHAR8_T_LOCK_FREE
1096#endif
1097# define ATOMIC_CHAR16_T_LOCK_FREE  __CLANG_ATOMIC_CHAR16_T_LOCK_FREE
1098# define ATOMIC_CHAR32_T_LOCK_FREE  __CLANG_ATOMIC_CHAR32_T_LOCK_FREE
1099# define ATOMIC_WCHAR_T_LOCK_FREE   __CLANG_ATOMIC_WCHAR_T_LOCK_FREE
1100# define ATOMIC_SHORT_LOCK_FREE     __CLANG_ATOMIC_SHORT_LOCK_FREE
1101# define ATOMIC_INT_LOCK_FREE       __CLANG_ATOMIC_INT_LOCK_FREE
1102# define ATOMIC_LONG_LOCK_FREE      __CLANG_ATOMIC_LONG_LOCK_FREE
1103# define ATOMIC_LLONG_LOCK_FREE     __CLANG_ATOMIC_LLONG_LOCK_FREE
1104# define ATOMIC_POINTER_LOCK_FREE   __CLANG_ATOMIC_POINTER_LOCK_FREE
1105#elif defined(__GCC_ATOMIC_BOOL_LOCK_FREE)
1106# define ATOMIC_BOOL_LOCK_FREE      __GCC_ATOMIC_BOOL_LOCK_FREE
1107# define ATOMIC_CHAR_LOCK_FREE      __GCC_ATOMIC_CHAR_LOCK_FREE
1108#ifndef _LIBCPP_HAS_NO_CHAR8_T
1109# define ATOMIC_CHAR8_T_LOCK_FREE   __GCC_ATOMIC_CHAR8_T_LOCK_FREE
1110#endif
1111# define ATOMIC_CHAR16_T_LOCK_FREE  __GCC_ATOMIC_CHAR16_T_LOCK_FREE
1112# define ATOMIC_CHAR32_T_LOCK_FREE  __GCC_ATOMIC_CHAR32_T_LOCK_FREE
1113# define ATOMIC_WCHAR_T_LOCK_FREE   __GCC_ATOMIC_WCHAR_T_LOCK_FREE
1114# define ATOMIC_SHORT_LOCK_FREE     __GCC_ATOMIC_SHORT_LOCK_FREE
1115# define ATOMIC_INT_LOCK_FREE       __GCC_ATOMIC_INT_LOCK_FREE
1116# define ATOMIC_LONG_LOCK_FREE      __GCC_ATOMIC_LONG_LOCK_FREE
1117# define ATOMIC_LLONG_LOCK_FREE     __GCC_ATOMIC_LLONG_LOCK_FREE
1118# define ATOMIC_POINTER_LOCK_FREE   __GCC_ATOMIC_POINTER_LOCK_FREE
1119#endif
1120
1121template <class _Tp>
1122struct __libcpp_is_always_lock_free {
1123  // __atomic_always_lock_free is available in all Standard modes
1124  static const bool __value = __atomic_always_lock_free(sizeof(_Tp), 0);
1125};
1126
1127#ifdef _LIBCPP_ATOMIC_ONLY_USE_BUILTINS
1128
1129template<typename _Tp>
1130struct __cxx_atomic_lock_impl {
1131
1132  _LIBCPP_INLINE_VISIBILITY
1133  __cxx_atomic_lock_impl() _NOEXCEPT
1134    : __a_value(), __a_lock(0) {}
1135  _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR explicit
1136  __cxx_atomic_lock_impl(_Tp value) _NOEXCEPT
1137    : __a_value(value), __a_lock(0) {}
1138
1139  _Tp __a_value;
1140  mutable __cxx_atomic_base_impl<_LIBCPP_ATOMIC_FLAG_TYPE> __a_lock;
1141
1142  _LIBCPP_INLINE_VISIBILITY void __lock() const volatile {
1143    while(1 == __cxx_atomic_exchange(&__a_lock, _LIBCPP_ATOMIC_FLAG_TYPE(true), memory_order_acquire))
1144        /*spin*/;
1145  }
1146  _LIBCPP_INLINE_VISIBILITY void __lock() const {
1147    while(1 == __cxx_atomic_exchange(&__a_lock, _LIBCPP_ATOMIC_FLAG_TYPE(true), memory_order_acquire))
1148        /*spin*/;
1149  }
1150  _LIBCPP_INLINE_VISIBILITY void __unlock() const volatile {
1151    __cxx_atomic_store(&__a_lock, _LIBCPP_ATOMIC_FLAG_TYPE(false), memory_order_release);
1152  }
1153  _LIBCPP_INLINE_VISIBILITY void __unlock() const {
1154    __cxx_atomic_store(&__a_lock, _LIBCPP_ATOMIC_FLAG_TYPE(false), memory_order_release);
1155  }
1156  _LIBCPP_INLINE_VISIBILITY _Tp __read() const volatile {
1157    __lock();
1158    _Tp __old;
1159    __cxx_atomic_assign_volatile(__old, __a_value);
1160    __unlock();
1161    return __old;
1162  }
1163  _LIBCPP_INLINE_VISIBILITY _Tp __read() const {
1164    __lock();
1165    _Tp __old = __a_value;
1166    __unlock();
1167    return __old;
1168  }
1169};
1170
1171template <typename _Tp>
1172_LIBCPP_INLINE_VISIBILITY
1173void __cxx_atomic_init(volatile __cxx_atomic_lock_impl<_Tp>* __a,  _Tp __val) {
1174  __cxx_atomic_assign_volatile(__a->__a_value, __val);
1175}
1176template <typename _Tp>
1177_LIBCPP_INLINE_VISIBILITY
1178void __cxx_atomic_init(__cxx_atomic_lock_impl<_Tp>* __a,  _Tp __val) {
1179  __a->__a_value = __val;
1180}
1181
1182template <typename _Tp>
1183_LIBCPP_INLINE_VISIBILITY
1184void __cxx_atomic_store(volatile __cxx_atomic_lock_impl<_Tp>* __a,  _Tp __val, memory_order) {
1185  __a->__lock();
1186  __cxx_atomic_assign_volatile(__a->__a_value, __val);
1187  __a->__unlock();
1188}
1189template <typename _Tp>
1190_LIBCPP_INLINE_VISIBILITY
1191void __cxx_atomic_store(__cxx_atomic_lock_impl<_Tp>* __a,  _Tp __val, memory_order) {
1192  __a->__lock();
1193  __a->__a_value = __val;
1194  __a->__unlock();
1195}
1196
1197template <typename _Tp>
1198_LIBCPP_INLINE_VISIBILITY
1199_Tp __cxx_atomic_load(const volatile __cxx_atomic_lock_impl<_Tp>* __a, memory_order) {
1200  return __a->__read();
1201}
1202template <typename _Tp>
1203_LIBCPP_INLINE_VISIBILITY
1204_Tp __cxx_atomic_load(const __cxx_atomic_lock_impl<_Tp>* __a, memory_order) {
1205  return __a->__read();
1206}
1207
1208template <typename _Tp>
1209_LIBCPP_INLINE_VISIBILITY
1210_Tp __cxx_atomic_exchange(volatile __cxx_atomic_lock_impl<_Tp>* __a, _Tp __value, memory_order) {
1211  __a->__lock();
1212  _Tp __old;
1213  __cxx_atomic_assign_volatile(__old, __a->__a_value);
1214  __cxx_atomic_assign_volatile(__a->__a_value, __value);
1215  __a->__unlock();
1216  return __old;
1217}
1218template <typename _Tp>
1219_LIBCPP_INLINE_VISIBILITY
1220_Tp __cxx_atomic_exchange(__cxx_atomic_lock_impl<_Tp>* __a, _Tp __value, memory_order) {
1221  __a->__lock();
1222  _Tp __old = __a->__a_value;
1223  __a->__a_value = __value;
1224  __a->__unlock();
1225  return __old;
1226}
1227
1228template <typename _Tp>
1229_LIBCPP_INLINE_VISIBILITY
1230bool __cxx_atomic_compare_exchange_strong(volatile __cxx_atomic_lock_impl<_Tp>* __a,
1231                                          _Tp* __expected, _Tp __value, memory_order, memory_order) {
1232  _Tp __temp;
1233  __a->__lock();
1234  __cxx_atomic_assign_volatile(__temp, __a->__a_value);
1235  bool __ret = (_VSTD::memcmp(&__temp, __expected, sizeof(_Tp)) == 0);
1236  if(__ret)
1237    __cxx_atomic_assign_volatile(__a->__a_value, __value);
1238  else
1239    __cxx_atomic_assign_volatile(*__expected, __a->__a_value);
1240  __a->__unlock();
1241  return __ret;
1242}
1243template <typename _Tp>
1244_LIBCPP_INLINE_VISIBILITY
1245bool __cxx_atomic_compare_exchange_strong(__cxx_atomic_lock_impl<_Tp>* __a,
1246                                          _Tp* __expected, _Tp __value, memory_order, memory_order) {
1247  __a->__lock();
1248  bool __ret = (_VSTD::memcmp(&__a->__a_value, __expected, sizeof(_Tp)) == 0);
1249  if(__ret)
1250    _VSTD::memcpy(&__a->__a_value, &__value, sizeof(_Tp));
1251  else
1252    _VSTD::memcpy(__expected, &__a->__a_value, sizeof(_Tp));
1253  __a->__unlock();
1254  return __ret;
1255}
1256
1257template <typename _Tp>
1258_LIBCPP_INLINE_VISIBILITY
1259bool __cxx_atomic_compare_exchange_weak(volatile __cxx_atomic_lock_impl<_Tp>* __a,
1260                                        _Tp* __expected, _Tp __value, memory_order, memory_order) {
1261  _Tp __temp;
1262  __a->__lock();
1263  __cxx_atomic_assign_volatile(__temp, __a->__a_value);
1264  bool __ret = (_VSTD::memcmp(&__temp, __expected, sizeof(_Tp)) == 0);
1265  if(__ret)
1266    __cxx_atomic_assign_volatile(__a->__a_value, __value);
1267  else
1268    __cxx_atomic_assign_volatile(*__expected, __a->__a_value);
1269  __a->__unlock();
1270  return __ret;
1271}
1272template <typename _Tp>
1273_LIBCPP_INLINE_VISIBILITY
1274bool __cxx_atomic_compare_exchange_weak(__cxx_atomic_lock_impl<_Tp>* __a,
1275                                        _Tp* __expected, _Tp __value, memory_order, memory_order) {
1276  __a->__lock();
1277  bool __ret = (_VSTD::memcmp(&__a->__a_value, __expected, sizeof(_Tp)) == 0);
1278  if(__ret)
1279    _VSTD::memcpy(&__a->__a_value, &__value, sizeof(_Tp));
1280  else
1281    _VSTD::memcpy(__expected, &__a->__a_value, sizeof(_Tp));
1282  __a->__unlock();
1283  return __ret;
1284}
1285
1286template <typename _Tp, typename _Td>
1287_LIBCPP_INLINE_VISIBILITY
1288_Tp __cxx_atomic_fetch_add(volatile __cxx_atomic_lock_impl<_Tp>* __a,
1289                           _Td __delta, memory_order) {
1290  __a->__lock();
1291  _Tp __old;
1292  __cxx_atomic_assign_volatile(__old, __a->__a_value);
1293  __cxx_atomic_assign_volatile(__a->__a_value, _Tp(__old + __delta));
1294  __a->__unlock();
1295  return __old;
1296}
1297template <typename _Tp, typename _Td>
1298_LIBCPP_INLINE_VISIBILITY
1299_Tp __cxx_atomic_fetch_add(__cxx_atomic_lock_impl<_Tp>* __a,
1300                           _Td __delta, memory_order) {
1301  __a->__lock();
1302  _Tp __old = __a->__a_value;
1303  __a->__a_value += __delta;
1304  __a->__unlock();
1305  return __old;
1306}
1307
1308template <typename _Tp, typename _Td>
1309_LIBCPP_INLINE_VISIBILITY
1310_Tp* __cxx_atomic_fetch_add(volatile __cxx_atomic_lock_impl<_Tp*>* __a,
1311                           ptrdiff_t __delta, memory_order) {
1312  __a->__lock();
1313  _Tp* __old;
1314  __cxx_atomic_assign_volatile(__old, __a->__a_value);
1315  __cxx_atomic_assign_volatile(__a->__a_value, __old + __delta);
1316  __a->__unlock();
1317  return __old;
1318}
1319template <typename _Tp, typename _Td>
1320_LIBCPP_INLINE_VISIBILITY
1321_Tp* __cxx_atomic_fetch_add(__cxx_atomic_lock_impl<_Tp*>* __a,
1322                           ptrdiff_t __delta, memory_order) {
1323  __a->__lock();
1324  _Tp* __old = __a->__a_value;
1325  __a->__a_value += __delta;
1326  __a->__unlock();
1327  return __old;
1328}
1329
1330template <typename _Tp, typename _Td>
1331_LIBCPP_INLINE_VISIBILITY
1332_Tp __cxx_atomic_fetch_sub(volatile __cxx_atomic_lock_impl<_Tp>* __a,
1333                           _Td __delta, memory_order) {
1334  __a->__lock();
1335  _Tp __old;
1336  __cxx_atomic_assign_volatile(__old, __a->__a_value);
1337  __cxx_atomic_assign_volatile(__a->__a_value, _Tp(__old - __delta));
1338  __a->__unlock();
1339  return __old;
1340}
1341template <typename _Tp, typename _Td>
1342_LIBCPP_INLINE_VISIBILITY
1343_Tp __cxx_atomic_fetch_sub(__cxx_atomic_lock_impl<_Tp>* __a,
1344                           _Td __delta, memory_order) {
1345  __a->__lock();
1346  _Tp __old = __a->__a_value;
1347  __a->__a_value -= __delta;
1348  __a->__unlock();
1349  return __old;
1350}
1351
1352template <typename _Tp>
1353_LIBCPP_INLINE_VISIBILITY
1354_Tp __cxx_atomic_fetch_and(volatile __cxx_atomic_lock_impl<_Tp>* __a,
1355                           _Tp __pattern, memory_order) {
1356  __a->__lock();
1357  _Tp __old;
1358  __cxx_atomic_assign_volatile(__old, __a->__a_value);
1359  __cxx_atomic_assign_volatile(__a->__a_value, _Tp(__old & __pattern));
1360  __a->__unlock();
1361  return __old;
1362}
1363template <typename _Tp>
1364_LIBCPP_INLINE_VISIBILITY
1365_Tp __cxx_atomic_fetch_and(__cxx_atomic_lock_impl<_Tp>* __a,
1366                           _Tp __pattern, memory_order) {
1367  __a->__lock();
1368  _Tp __old = __a->__a_value;
1369  __a->__a_value &= __pattern;
1370  __a->__unlock();
1371  return __old;
1372}
1373
1374template <typename _Tp>
1375_LIBCPP_INLINE_VISIBILITY
1376_Tp __cxx_atomic_fetch_or(volatile __cxx_atomic_lock_impl<_Tp>* __a,
1377                          _Tp __pattern, memory_order) {
1378  __a->__lock();
1379  _Tp __old;
1380  __cxx_atomic_assign_volatile(__old, __a->__a_value);
1381  __cxx_atomic_assign_volatile(__a->__a_value, _Tp(__old | __pattern));
1382  __a->__unlock();
1383  return __old;
1384}
1385template <typename _Tp>
1386_LIBCPP_INLINE_VISIBILITY
1387_Tp __cxx_atomic_fetch_or(__cxx_atomic_lock_impl<_Tp>* __a,
1388                          _Tp __pattern, memory_order) {
1389  __a->__lock();
1390  _Tp __old = __a->__a_value;
1391  __a->__a_value |= __pattern;
1392  __a->__unlock();
1393  return __old;
1394}
1395
1396template <typename _Tp>
1397_LIBCPP_INLINE_VISIBILITY
1398_Tp __cxx_atomic_fetch_xor(volatile __cxx_atomic_lock_impl<_Tp>* __a,
1399                           _Tp __pattern, memory_order) {
1400  __a->__lock();
1401  _Tp __old;
1402  __cxx_atomic_assign_volatile(__old, __a->__a_value);
1403  __cxx_atomic_assign_volatile(__a->__a_value, _Tp(__old ^ __pattern));
1404  __a->__unlock();
1405  return __old;
1406}
1407template <typename _Tp>
1408_LIBCPP_INLINE_VISIBILITY
1409_Tp __cxx_atomic_fetch_xor(__cxx_atomic_lock_impl<_Tp>* __a,
1410                           _Tp __pattern, memory_order) {
1411  __a->__lock();
1412  _Tp __old = __a->__a_value;
1413  __a->__a_value ^= __pattern;
1414  __a->__unlock();
1415  return __old;
1416}
1417
1418template <typename _Tp,
1419          typename _Base = typename conditional<__libcpp_is_always_lock_free<_Tp>::__value,
1420                                                __cxx_atomic_base_impl<_Tp>,
1421                                                __cxx_atomic_lock_impl<_Tp> >::type>
1422#else
1423template <typename _Tp,
1424          typename _Base = __cxx_atomic_base_impl<_Tp> >
1425#endif //_LIBCPP_ATOMIC_ONLY_USE_BUILTINS
1426struct __cxx_atomic_impl : public _Base {
1427    static_assert(is_trivially_copyable<_Tp>::value,
1428      "std::atomic<T> requires that 'T' be a trivially copyable type");
1429
1430  _LIBCPP_INLINE_VISIBILITY __cxx_atomic_impl() _NOEXCEPT = default;
1431  _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR explicit __cxx_atomic_impl(_Tp __value) _NOEXCEPT
1432    : _Base(__value) {}
1433};
1434
1435#if defined(__linux__) || (defined(_AIX) && !defined(__64BIT__)) || \
1436    (defined(__FreeBSD__) && defined(__mips__))
1437    using __cxx_contention_t = int32_t;
1438#else
1439    using __cxx_contention_t = int64_t;
1440#endif // __linux__ || (_AIX && !__64BIT__) || (__FreeBSD__ && __mips__)
1441
1442using __cxx_atomic_contention_t = __cxx_atomic_impl<__cxx_contention_t>;
1443
1444#ifndef _LIBCPP_HAS_NO_THREADS
1445
1446_LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI void __cxx_atomic_notify_one(void const volatile*);
1447_LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI void __cxx_atomic_notify_all(void const volatile*);
1448_LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI __cxx_contention_t __libcpp_atomic_monitor(void const volatile*);
1449_LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI void __libcpp_atomic_wait(void const volatile*, __cxx_contention_t);
1450
1451_LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI void __cxx_atomic_notify_one(__cxx_atomic_contention_t const volatile*);
1452_LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI void __cxx_atomic_notify_all(__cxx_atomic_contention_t const volatile*);
1453_LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI __cxx_contention_t __libcpp_atomic_monitor(__cxx_atomic_contention_t const volatile*);
1454_LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI void __libcpp_atomic_wait(__cxx_atomic_contention_t const volatile*, __cxx_contention_t);
1455
1456template <class _Atp, class _Fn>
1457struct __libcpp_atomic_wait_backoff_impl {
1458    _Atp* __a;
1459    _Fn __test_fn;
1460    _LIBCPP_AVAILABILITY_SYNC
1461    _LIBCPP_INLINE_VISIBILITY bool operator()(chrono::nanoseconds __elapsed) const
1462    {
1463        if(__elapsed > chrono::microseconds(64))
1464        {
1465            auto const __monitor = std::__libcpp_atomic_monitor(__a);
1466            if(__test_fn())
1467                return true;
1468            std::__libcpp_atomic_wait(__a, __monitor);
1469        }
1470        else if(__elapsed > chrono::microseconds(4))
1471            __libcpp_thread_yield();
1472        else
1473            {} // poll
1474        return false;
1475    }
1476};
1477
1478template <class _Atp, class _Fn>
1479_LIBCPP_AVAILABILITY_SYNC
1480_LIBCPP_INLINE_VISIBILITY bool __cxx_atomic_wait(_Atp* __a, _Fn && __test_fn)
1481{
1482    __libcpp_atomic_wait_backoff_impl<_Atp, typename decay<_Fn>::type> __backoff_fn = {__a, __test_fn};
1483    return std::__libcpp_thread_poll_with_backoff(__test_fn, __backoff_fn);
1484}
1485
1486#else // _LIBCPP_HAS_NO_THREADS
1487
1488template <class _Tp>
1489_LIBCPP_INLINE_VISIBILITY void __cxx_atomic_notify_all(__cxx_atomic_impl<_Tp> const volatile*) { }
1490template <class _Tp>
1491_LIBCPP_INLINE_VISIBILITY void __cxx_atomic_notify_one(__cxx_atomic_impl<_Tp> const volatile*) { }
1492template <class _Atp, class _Fn>
1493_LIBCPP_INLINE_VISIBILITY bool __cxx_atomic_wait(_Atp*, _Fn && __test_fn)
1494{
1495    return __libcpp_thread_poll_with_backoff(__test_fn, __spinning_backoff_policy());
1496}
1497
1498#endif // _LIBCPP_HAS_NO_THREADS
1499
1500template <class _Atp, class _Tp>
1501struct __cxx_atomic_wait_test_fn_impl {
1502    _Atp* __a;
1503    _Tp __val;
1504    memory_order __order;
1505    _LIBCPP_INLINE_VISIBILITY bool operator()() const
1506    {
1507        return !std::__cxx_nonatomic_compare_equal(std::__cxx_atomic_load(__a, __order), __val);
1508    }
1509};
1510
1511template <class _Atp, class _Tp>
1512_LIBCPP_AVAILABILITY_SYNC
1513_LIBCPP_INLINE_VISIBILITY bool __cxx_atomic_wait(_Atp* __a, _Tp const __val, memory_order __order)
1514{
1515    __cxx_atomic_wait_test_fn_impl<_Atp, _Tp> __test_fn = {__a, __val, __order};
1516    return std::__cxx_atomic_wait(__a, __test_fn);
1517}
1518
1519// general atomic<T>
1520
1521template <class _Tp, bool = is_integral<_Tp>::value && !is_same<_Tp, bool>::value>
1522struct __atomic_base  // false
1523{
1524    mutable __cxx_atomic_impl<_Tp> __a_;
1525
1526#if defined(__cpp_lib_atomic_is_always_lock_free)
1527  static _LIBCPP_CONSTEXPR bool is_always_lock_free = __libcpp_is_always_lock_free<__cxx_atomic_impl<_Tp> >::__value;
1528#endif
1529
1530    _LIBCPP_INLINE_VISIBILITY
1531    bool is_lock_free() const volatile _NOEXCEPT
1532        {return __cxx_atomic_is_lock_free(sizeof(_Tp));}
1533    _LIBCPP_INLINE_VISIBILITY
1534    bool is_lock_free() const _NOEXCEPT
1535        {return static_cast<__atomic_base const volatile*>(this)->is_lock_free();}
1536    _LIBCPP_INLINE_VISIBILITY
1537    void store(_Tp __d, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT
1538      _LIBCPP_CHECK_STORE_MEMORY_ORDER(__m)
1539        {std::__cxx_atomic_store(&__a_, __d, __m);}
1540    _LIBCPP_INLINE_VISIBILITY
1541    void store(_Tp __d, memory_order __m = memory_order_seq_cst) _NOEXCEPT
1542      _LIBCPP_CHECK_STORE_MEMORY_ORDER(__m)
1543        {std::__cxx_atomic_store(&__a_, __d, __m);}
1544    _LIBCPP_INLINE_VISIBILITY
1545    _Tp load(memory_order __m = memory_order_seq_cst) const volatile _NOEXCEPT
1546      _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m)
1547        {return std::__cxx_atomic_load(&__a_, __m);}
1548    _LIBCPP_INLINE_VISIBILITY
1549    _Tp load(memory_order __m = memory_order_seq_cst) const _NOEXCEPT
1550      _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m)
1551        {return std::__cxx_atomic_load(&__a_, __m);}
1552    _LIBCPP_INLINE_VISIBILITY
1553    operator _Tp() const volatile _NOEXCEPT {return load();}
1554    _LIBCPP_INLINE_VISIBILITY
1555    operator _Tp() const _NOEXCEPT          {return load();}
1556    _LIBCPP_INLINE_VISIBILITY
1557    _Tp exchange(_Tp __d, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT
1558        {return std::__cxx_atomic_exchange(&__a_, __d, __m);}
1559    _LIBCPP_INLINE_VISIBILITY
1560    _Tp exchange(_Tp __d, memory_order __m = memory_order_seq_cst) _NOEXCEPT
1561        {return std::__cxx_atomic_exchange(&__a_, __d, __m);}
1562    _LIBCPP_INLINE_VISIBILITY
1563    bool compare_exchange_weak(_Tp& __e, _Tp __d,
1564                               memory_order __s, memory_order __f) volatile _NOEXCEPT
1565      _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f)
1566        {return std::__cxx_atomic_compare_exchange_weak(&__a_, &__e, __d, __s, __f);}
1567    _LIBCPP_INLINE_VISIBILITY
1568    bool compare_exchange_weak(_Tp& __e, _Tp __d,
1569                               memory_order __s, memory_order __f) _NOEXCEPT
1570      _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f)
1571        {return std::__cxx_atomic_compare_exchange_weak(&__a_, &__e, __d, __s, __f);}
1572    _LIBCPP_INLINE_VISIBILITY
1573    bool compare_exchange_strong(_Tp& __e, _Tp __d,
1574                                 memory_order __s, memory_order __f) volatile _NOEXCEPT
1575      _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f)
1576        {return std::__cxx_atomic_compare_exchange_strong(&__a_, &__e, __d, __s, __f);}
1577    _LIBCPP_INLINE_VISIBILITY
1578    bool compare_exchange_strong(_Tp& __e, _Tp __d,
1579                                 memory_order __s, memory_order __f) _NOEXCEPT
1580      _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f)
1581        {return std::__cxx_atomic_compare_exchange_strong(&__a_, &__e, __d, __s, __f);}
1582    _LIBCPP_INLINE_VISIBILITY
1583    bool compare_exchange_weak(_Tp& __e, _Tp __d,
1584                              memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT
1585        {return std::__cxx_atomic_compare_exchange_weak(&__a_, &__e, __d, __m, __m);}
1586    _LIBCPP_INLINE_VISIBILITY
1587    bool compare_exchange_weak(_Tp& __e, _Tp __d,
1588                               memory_order __m = memory_order_seq_cst) _NOEXCEPT
1589        {return std::__cxx_atomic_compare_exchange_weak(&__a_, &__e, __d, __m, __m);}
1590    _LIBCPP_INLINE_VISIBILITY
1591    bool compare_exchange_strong(_Tp& __e, _Tp __d,
1592                              memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT
1593        {return std::__cxx_atomic_compare_exchange_strong(&__a_, &__e, __d, __m, __m);}
1594    _LIBCPP_INLINE_VISIBILITY
1595    bool compare_exchange_strong(_Tp& __e, _Tp __d,
1596                                 memory_order __m = memory_order_seq_cst) _NOEXCEPT
1597        {return std::__cxx_atomic_compare_exchange_strong(&__a_, &__e, __d, __m, __m);}
1598
1599    _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY void wait(_Tp __v, memory_order __m = memory_order_seq_cst) const volatile _NOEXCEPT
1600        {std::__cxx_atomic_wait(&__a_, __v, __m);}
1601    _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY void wait(_Tp __v, memory_order __m = memory_order_seq_cst) const _NOEXCEPT
1602        {std::__cxx_atomic_wait(&__a_, __v, __m);}
1603    _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY void notify_one() volatile _NOEXCEPT
1604        {std::__cxx_atomic_notify_one(&__a_);}
1605    _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY void notify_one() _NOEXCEPT
1606        {std::__cxx_atomic_notify_one(&__a_);}
1607    _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY void notify_all() volatile _NOEXCEPT
1608        {std::__cxx_atomic_notify_all(&__a_);}
1609    _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY void notify_all() _NOEXCEPT
1610        {std::__cxx_atomic_notify_all(&__a_);}
1611
1612#if _LIBCPP_STD_VER > 17
1613    _LIBCPP_INLINE_VISIBILITY constexpr
1614    __atomic_base() noexcept(is_nothrow_default_constructible_v<_Tp>) : __a_(_Tp()) {}
1615#else
1616    _LIBCPP_INLINE_VISIBILITY
1617    __atomic_base() _NOEXCEPT = default;
1618#endif
1619
1620    _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR
1621    __atomic_base(_Tp __d) _NOEXCEPT : __a_(__d) {}
1622
1623    __atomic_base(const __atomic_base&) = delete;
1624};
1625
1626#if defined(__cpp_lib_atomic_is_always_lock_free)
1627template <class _Tp, bool __b>
1628_LIBCPP_CONSTEXPR bool __atomic_base<_Tp, __b>::is_always_lock_free;
1629#endif
1630
1631// atomic<Integral>
1632
1633template <class _Tp>
1634struct __atomic_base<_Tp, true>
1635    : public __atomic_base<_Tp, false>
1636{
1637    typedef __atomic_base<_Tp, false> __base;
1638
1639    _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_SINCE_CXX20
1640    __atomic_base() _NOEXCEPT = default;
1641
1642    _LIBCPP_INLINE_VISIBILITY
1643    _LIBCPP_CONSTEXPR __atomic_base(_Tp __d) _NOEXCEPT : __base(__d) {}
1644
1645    _LIBCPP_INLINE_VISIBILITY
1646    _Tp fetch_add(_Tp __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT
1647        {return std::__cxx_atomic_fetch_add(&this->__a_, __op, __m);}
1648    _LIBCPP_INLINE_VISIBILITY
1649    _Tp fetch_add(_Tp __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT
1650        {return std::__cxx_atomic_fetch_add(&this->__a_, __op, __m);}
1651    _LIBCPP_INLINE_VISIBILITY
1652    _Tp fetch_sub(_Tp __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT
1653        {return std::__cxx_atomic_fetch_sub(&this->__a_, __op, __m);}
1654    _LIBCPP_INLINE_VISIBILITY
1655    _Tp fetch_sub(_Tp __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT
1656        {return std::__cxx_atomic_fetch_sub(&this->__a_, __op, __m);}
1657    _LIBCPP_INLINE_VISIBILITY
1658    _Tp fetch_and(_Tp __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT
1659        {return std::__cxx_atomic_fetch_and(&this->__a_, __op, __m);}
1660    _LIBCPP_INLINE_VISIBILITY
1661    _Tp fetch_and(_Tp __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT
1662        {return std::__cxx_atomic_fetch_and(&this->__a_, __op, __m);}
1663    _LIBCPP_INLINE_VISIBILITY
1664    _Tp fetch_or(_Tp __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT
1665        {return std::__cxx_atomic_fetch_or(&this->__a_, __op, __m);}
1666    _LIBCPP_INLINE_VISIBILITY
1667    _Tp fetch_or(_Tp __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT
1668        {return std::__cxx_atomic_fetch_or(&this->__a_, __op, __m);}
1669    _LIBCPP_INLINE_VISIBILITY
1670    _Tp fetch_xor(_Tp __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT
1671        {return std::__cxx_atomic_fetch_xor(&this->__a_, __op, __m);}
1672    _LIBCPP_INLINE_VISIBILITY
1673    _Tp fetch_xor(_Tp __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT
1674        {return std::__cxx_atomic_fetch_xor(&this->__a_, __op, __m);}
1675
1676    _LIBCPP_INLINE_VISIBILITY
1677    _Tp operator++(int) volatile _NOEXCEPT      {return fetch_add(_Tp(1));}
1678    _LIBCPP_INLINE_VISIBILITY
1679    _Tp operator++(int) _NOEXCEPT               {return fetch_add(_Tp(1));}
1680    _LIBCPP_INLINE_VISIBILITY
1681    _Tp operator--(int) volatile _NOEXCEPT      {return fetch_sub(_Tp(1));}
1682    _LIBCPP_INLINE_VISIBILITY
1683    _Tp operator--(int) _NOEXCEPT               {return fetch_sub(_Tp(1));}
1684    _LIBCPP_INLINE_VISIBILITY
1685    _Tp operator++() volatile _NOEXCEPT         {return fetch_add(_Tp(1)) + _Tp(1);}
1686    _LIBCPP_INLINE_VISIBILITY
1687    _Tp operator++() _NOEXCEPT                  {return fetch_add(_Tp(1)) + _Tp(1);}
1688    _LIBCPP_INLINE_VISIBILITY
1689    _Tp operator--() volatile _NOEXCEPT         {return fetch_sub(_Tp(1)) - _Tp(1);}
1690    _LIBCPP_INLINE_VISIBILITY
1691    _Tp operator--() _NOEXCEPT                  {return fetch_sub(_Tp(1)) - _Tp(1);}
1692    _LIBCPP_INLINE_VISIBILITY
1693    _Tp operator+=(_Tp __op) volatile _NOEXCEPT {return fetch_add(__op) + __op;}
1694    _LIBCPP_INLINE_VISIBILITY
1695    _Tp operator+=(_Tp __op) _NOEXCEPT          {return fetch_add(__op) + __op;}
1696    _LIBCPP_INLINE_VISIBILITY
1697    _Tp operator-=(_Tp __op) volatile _NOEXCEPT {return fetch_sub(__op) - __op;}
1698    _LIBCPP_INLINE_VISIBILITY
1699    _Tp operator-=(_Tp __op) _NOEXCEPT          {return fetch_sub(__op) - __op;}
1700    _LIBCPP_INLINE_VISIBILITY
1701    _Tp operator&=(_Tp __op) volatile _NOEXCEPT {return fetch_and(__op) & __op;}
1702    _LIBCPP_INLINE_VISIBILITY
1703    _Tp operator&=(_Tp __op) _NOEXCEPT          {return fetch_and(__op) & __op;}
1704    _LIBCPP_INLINE_VISIBILITY
1705    _Tp operator|=(_Tp __op) volatile _NOEXCEPT {return fetch_or(__op) | __op;}
1706    _LIBCPP_INLINE_VISIBILITY
1707    _Tp operator|=(_Tp __op) _NOEXCEPT          {return fetch_or(__op) | __op;}
1708    _LIBCPP_INLINE_VISIBILITY
1709    _Tp operator^=(_Tp __op) volatile _NOEXCEPT {return fetch_xor(__op) ^ __op;}
1710    _LIBCPP_INLINE_VISIBILITY
1711    _Tp operator^=(_Tp __op) _NOEXCEPT          {return fetch_xor(__op) ^ __op;}
1712};
1713
1714// atomic<T>
1715
1716template <class _Tp>
1717struct atomic
1718    : public __atomic_base<_Tp>
1719{
1720    typedef __atomic_base<_Tp> __base;
1721    typedef _Tp value_type;
1722    typedef value_type difference_type;
1723
1724#if _LIBCPP_STD_VER > 17
1725    _LIBCPP_INLINE_VISIBILITY
1726    atomic() = default;
1727#else
1728    _LIBCPP_INLINE_VISIBILITY
1729    atomic() _NOEXCEPT = default;
1730#endif
1731
1732    _LIBCPP_INLINE_VISIBILITY
1733    _LIBCPP_CONSTEXPR atomic(_Tp __d) _NOEXCEPT : __base(__d) {}
1734
1735    _LIBCPP_INLINE_VISIBILITY
1736    _Tp operator=(_Tp __d) volatile _NOEXCEPT
1737        {__base::store(__d); return __d;}
1738    _LIBCPP_INLINE_VISIBILITY
1739    _Tp operator=(_Tp __d) _NOEXCEPT
1740        {__base::store(__d); return __d;}
1741
1742    atomic& operator=(const atomic&) = delete;
1743    atomic& operator=(const atomic&) volatile = delete;
1744};
1745
1746// atomic<T*>
1747
1748template <class _Tp>
1749struct atomic<_Tp*>
1750    : public __atomic_base<_Tp*>
1751{
1752    typedef __atomic_base<_Tp*> __base;
1753    typedef _Tp* value_type;
1754    typedef ptrdiff_t difference_type;
1755
1756    _LIBCPP_INLINE_VISIBILITY
1757    atomic() _NOEXCEPT = default;
1758
1759    _LIBCPP_INLINE_VISIBILITY
1760    _LIBCPP_CONSTEXPR atomic(_Tp* __d) _NOEXCEPT : __base(__d) {}
1761
1762    _LIBCPP_INLINE_VISIBILITY
1763    _Tp* operator=(_Tp* __d) volatile _NOEXCEPT
1764        {__base::store(__d); return __d;}
1765    _LIBCPP_INLINE_VISIBILITY
1766    _Tp* operator=(_Tp* __d) _NOEXCEPT
1767        {__base::store(__d); return __d;}
1768
1769    _LIBCPP_INLINE_VISIBILITY
1770    _Tp* fetch_add(ptrdiff_t __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT {
1771        // __atomic_fetch_add accepts function pointers, guard against them.
1772        static_assert(!is_function<__remove_pointer_t<_Tp> >::value, "Pointer to function isn't allowed");
1773        return std::__cxx_atomic_fetch_add(&this->__a_, __op, __m);
1774    }
1775
1776    _LIBCPP_INLINE_VISIBILITY
1777    _Tp* fetch_add(ptrdiff_t __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT {
1778        // __atomic_fetch_add accepts function pointers, guard against them.
1779        static_assert(!is_function<__remove_pointer_t<_Tp> >::value, "Pointer to function isn't allowed");
1780        return std::__cxx_atomic_fetch_add(&this->__a_, __op, __m);
1781    }
1782
1783    _LIBCPP_INLINE_VISIBILITY
1784    _Tp* fetch_sub(ptrdiff_t __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT {
1785        // __atomic_fetch_add accepts function pointers, guard against them.
1786        static_assert(!is_function<__remove_pointer_t<_Tp> >::value, "Pointer to function isn't allowed");
1787        return std::__cxx_atomic_fetch_sub(&this->__a_, __op, __m);
1788    }
1789
1790    _LIBCPP_INLINE_VISIBILITY
1791    _Tp* fetch_sub(ptrdiff_t __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT {
1792        // __atomic_fetch_add accepts function pointers, guard against them.
1793        static_assert(!is_function<__remove_pointer_t<_Tp> >::value, "Pointer to function isn't allowed");
1794        return std::__cxx_atomic_fetch_sub(&this->__a_, __op, __m);
1795    }
1796
1797    _LIBCPP_INLINE_VISIBILITY
1798    _Tp* operator++(int) volatile _NOEXCEPT            {return fetch_add(1);}
1799    _LIBCPP_INLINE_VISIBILITY
1800    _Tp* operator++(int) _NOEXCEPT                     {return fetch_add(1);}
1801    _LIBCPP_INLINE_VISIBILITY
1802    _Tp* operator--(int) volatile _NOEXCEPT            {return fetch_sub(1);}
1803    _LIBCPP_INLINE_VISIBILITY
1804    _Tp* operator--(int) _NOEXCEPT                     {return fetch_sub(1);}
1805    _LIBCPP_INLINE_VISIBILITY
1806    _Tp* operator++() volatile _NOEXCEPT               {return fetch_add(1) + 1;}
1807    _LIBCPP_INLINE_VISIBILITY
1808    _Tp* operator++() _NOEXCEPT                        {return fetch_add(1) + 1;}
1809    _LIBCPP_INLINE_VISIBILITY
1810    _Tp* operator--() volatile _NOEXCEPT               {return fetch_sub(1) - 1;}
1811    _LIBCPP_INLINE_VISIBILITY
1812    _Tp* operator--() _NOEXCEPT                        {return fetch_sub(1) - 1;}
1813    _LIBCPP_INLINE_VISIBILITY
1814    _Tp* operator+=(ptrdiff_t __op) volatile _NOEXCEPT {return fetch_add(__op) + __op;}
1815    _LIBCPP_INLINE_VISIBILITY
1816    _Tp* operator+=(ptrdiff_t __op) _NOEXCEPT          {return fetch_add(__op) + __op;}
1817    _LIBCPP_INLINE_VISIBILITY
1818    _Tp* operator-=(ptrdiff_t __op) volatile _NOEXCEPT {return fetch_sub(__op) - __op;}
1819    _LIBCPP_INLINE_VISIBILITY
1820    _Tp* operator-=(ptrdiff_t __op) _NOEXCEPT          {return fetch_sub(__op) - __op;}
1821
1822    atomic& operator=(const atomic&) = delete;
1823    atomic& operator=(const atomic&) volatile = delete;
1824};
1825
1826// atomic_is_lock_free
1827
1828template <class _Tp>
1829_LIBCPP_INLINE_VISIBILITY
1830bool
1831atomic_is_lock_free(const volatile atomic<_Tp>* __o) _NOEXCEPT
1832{
1833    return __o->is_lock_free();
1834}
1835
1836template <class _Tp>
1837_LIBCPP_INLINE_VISIBILITY
1838bool
1839atomic_is_lock_free(const atomic<_Tp>* __o) _NOEXCEPT
1840{
1841    return __o->is_lock_free();
1842}
1843
1844// atomic_init
1845
1846template <class _Tp>
1847_LIBCPP_DEPRECATED_IN_CXX20 _LIBCPP_INLINE_VISIBILITY
1848void
1849atomic_init(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d) _NOEXCEPT
1850{
1851    std::__cxx_atomic_init(&__o->__a_, __d);
1852}
1853
1854template <class _Tp>
1855_LIBCPP_DEPRECATED_IN_CXX20 _LIBCPP_INLINE_VISIBILITY
1856void
1857atomic_init(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d) _NOEXCEPT
1858{
1859    std::__cxx_atomic_init(&__o->__a_, __d);
1860}
1861
1862// atomic_store
1863
1864template <class _Tp>
1865_LIBCPP_INLINE_VISIBILITY
1866void
1867atomic_store(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d) _NOEXCEPT
1868{
1869    __o->store(__d);
1870}
1871
1872template <class _Tp>
1873_LIBCPP_INLINE_VISIBILITY
1874void
1875atomic_store(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d) _NOEXCEPT
1876{
1877    __o->store(__d);
1878}
1879
1880// atomic_store_explicit
1881
1882template <class _Tp>
1883_LIBCPP_INLINE_VISIBILITY
1884void
1885atomic_store_explicit(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d, memory_order __m) _NOEXCEPT
1886  _LIBCPP_CHECK_STORE_MEMORY_ORDER(__m)
1887{
1888    __o->store(__d, __m);
1889}
1890
1891template <class _Tp>
1892_LIBCPP_INLINE_VISIBILITY
1893void
1894atomic_store_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d, memory_order __m) _NOEXCEPT
1895  _LIBCPP_CHECK_STORE_MEMORY_ORDER(__m)
1896{
1897    __o->store(__d, __m);
1898}
1899
1900// atomic_load
1901
1902template <class _Tp>
1903_LIBCPP_INLINE_VISIBILITY
1904_Tp
1905atomic_load(const volatile atomic<_Tp>* __o) _NOEXCEPT
1906{
1907    return __o->load();
1908}
1909
1910template <class _Tp>
1911_LIBCPP_INLINE_VISIBILITY
1912_Tp
1913atomic_load(const atomic<_Tp>* __o) _NOEXCEPT
1914{
1915    return __o->load();
1916}
1917
1918// atomic_load_explicit
1919
1920template <class _Tp>
1921_LIBCPP_INLINE_VISIBILITY
1922_Tp
1923atomic_load_explicit(const volatile atomic<_Tp>* __o, memory_order __m) _NOEXCEPT
1924  _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m)
1925{
1926    return __o->load(__m);
1927}
1928
1929template <class _Tp>
1930_LIBCPP_INLINE_VISIBILITY
1931_Tp
1932atomic_load_explicit(const atomic<_Tp>* __o, memory_order __m) _NOEXCEPT
1933  _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m)
1934{
1935    return __o->load(__m);
1936}
1937
1938// atomic_exchange
1939
1940template <class _Tp>
1941_LIBCPP_INLINE_VISIBILITY
1942_Tp
1943atomic_exchange(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d) _NOEXCEPT
1944{
1945    return __o->exchange(__d);
1946}
1947
1948template <class _Tp>
1949_LIBCPP_INLINE_VISIBILITY
1950_Tp
1951atomic_exchange(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d) _NOEXCEPT
1952{
1953    return __o->exchange(__d);
1954}
1955
1956// atomic_exchange_explicit
1957
1958template <class _Tp>
1959_LIBCPP_INLINE_VISIBILITY
1960_Tp
1961atomic_exchange_explicit(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d, memory_order __m) _NOEXCEPT
1962{
1963    return __o->exchange(__d, __m);
1964}
1965
1966template <class _Tp>
1967_LIBCPP_INLINE_VISIBILITY
1968_Tp
1969atomic_exchange_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d, memory_order __m) _NOEXCEPT
1970{
1971    return __o->exchange(__d, __m);
1972}
1973
1974// atomic_compare_exchange_weak
1975
1976template <class _Tp>
1977_LIBCPP_INLINE_VISIBILITY
1978bool
1979atomic_compare_exchange_weak(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type* __e, typename atomic<_Tp>::value_type __d) _NOEXCEPT
1980{
1981    return __o->compare_exchange_weak(*__e, __d);
1982}
1983
1984template <class _Tp>
1985_LIBCPP_INLINE_VISIBILITY
1986bool
1987atomic_compare_exchange_weak(atomic<_Tp>* __o, typename atomic<_Tp>::value_type* __e, typename atomic<_Tp>::value_type __d) _NOEXCEPT
1988{
1989    return __o->compare_exchange_weak(*__e, __d);
1990}
1991
1992// atomic_compare_exchange_strong
1993
1994template <class _Tp>
1995_LIBCPP_INLINE_VISIBILITY
1996bool
1997atomic_compare_exchange_strong(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type* __e, typename atomic<_Tp>::value_type __d) _NOEXCEPT
1998{
1999    return __o->compare_exchange_strong(*__e, __d);
2000}
2001
2002template <class _Tp>
2003_LIBCPP_INLINE_VISIBILITY
2004bool
2005atomic_compare_exchange_strong(atomic<_Tp>* __o, typename atomic<_Tp>::value_type* __e, typename atomic<_Tp>::value_type __d) _NOEXCEPT
2006{
2007    return __o->compare_exchange_strong(*__e, __d);
2008}
2009
2010// atomic_compare_exchange_weak_explicit
2011
2012template <class _Tp>
2013_LIBCPP_INLINE_VISIBILITY
2014bool
2015atomic_compare_exchange_weak_explicit(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type* __e,
2016                                      typename atomic<_Tp>::value_type __d,
2017                                      memory_order __s, memory_order __f) _NOEXCEPT
2018  _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f)
2019{
2020    return __o->compare_exchange_weak(*__e, __d, __s, __f);
2021}
2022
2023template <class _Tp>
2024_LIBCPP_INLINE_VISIBILITY
2025bool
2026atomic_compare_exchange_weak_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::value_type* __e, typename atomic<_Tp>::value_type __d,
2027                                      memory_order __s, memory_order __f) _NOEXCEPT
2028  _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f)
2029{
2030    return __o->compare_exchange_weak(*__e, __d, __s, __f);
2031}
2032
2033// atomic_compare_exchange_strong_explicit
2034
2035template <class _Tp>
2036_LIBCPP_INLINE_VISIBILITY
2037bool
2038atomic_compare_exchange_strong_explicit(volatile atomic<_Tp>* __o,
2039                                        typename atomic<_Tp>::value_type* __e, typename atomic<_Tp>::value_type __d,
2040                                        memory_order __s, memory_order __f) _NOEXCEPT
2041  _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f)
2042{
2043    return __o->compare_exchange_strong(*__e, __d, __s, __f);
2044}
2045
2046template <class _Tp>
2047_LIBCPP_INLINE_VISIBILITY
2048bool
2049atomic_compare_exchange_strong_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::value_type* __e,
2050                                        typename atomic<_Tp>::value_type __d,
2051                                        memory_order __s, memory_order __f) _NOEXCEPT
2052  _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f)
2053{
2054    return __o->compare_exchange_strong(*__e, __d, __s, __f);
2055}
2056
2057// atomic_wait
2058
2059template <class _Tp>
2060_LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY
2061void atomic_wait(const volatile atomic<_Tp>* __o,
2062                 typename atomic<_Tp>::value_type __v) _NOEXCEPT
2063{
2064    return __o->wait(__v);
2065}
2066
2067template <class _Tp>
2068_LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY
2069void atomic_wait(const atomic<_Tp>* __o,
2070                 typename atomic<_Tp>::value_type __v) _NOEXCEPT
2071{
2072    return __o->wait(__v);
2073}
2074
2075// atomic_wait_explicit
2076
2077template <class _Tp>
2078_LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY
2079void atomic_wait_explicit(const volatile atomic<_Tp>* __o,
2080                          typename atomic<_Tp>::value_type __v,
2081                          memory_order __m) _NOEXCEPT
2082  _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m)
2083{
2084    return __o->wait(__v, __m);
2085}
2086
2087template <class _Tp>
2088_LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY
2089void atomic_wait_explicit(const atomic<_Tp>* __o,
2090                          typename atomic<_Tp>::value_type __v,
2091                          memory_order __m) _NOEXCEPT
2092  _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m)
2093{
2094    return __o->wait(__v, __m);
2095}
2096
2097// atomic_notify_one
2098
2099template <class _Tp>
2100_LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY
2101void atomic_notify_one(volatile atomic<_Tp>* __o) _NOEXCEPT
2102{
2103    __o->notify_one();
2104}
2105template <class _Tp>
2106_LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY
2107void atomic_notify_one(atomic<_Tp>* __o) _NOEXCEPT
2108{
2109    __o->notify_one();
2110}
2111
2112// atomic_notify_all
2113
2114template <class _Tp>
2115_LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY
2116void atomic_notify_all(volatile atomic<_Tp>* __o) _NOEXCEPT
2117{
2118    __o->notify_all();
2119}
2120template <class _Tp>
2121_LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY
2122void atomic_notify_all(atomic<_Tp>* __o) _NOEXCEPT
2123{
2124    __o->notify_all();
2125}
2126
2127// atomic_fetch_add
2128
2129template <class _Tp>
2130_LIBCPP_INLINE_VISIBILITY
2131_Tp
2132atomic_fetch_add(volatile atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op) _NOEXCEPT
2133{
2134    return __o->fetch_add(__op);
2135}
2136
2137template <class _Tp>
2138_LIBCPP_INLINE_VISIBILITY
2139_Tp
2140atomic_fetch_add(atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op) _NOEXCEPT
2141{
2142    return __o->fetch_add(__op);
2143}
2144
2145// atomic_fetch_add_explicit
2146
2147template <class _Tp>
2148_LIBCPP_INLINE_VISIBILITY
2149_Tp atomic_fetch_add_explicit(volatile atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op, memory_order __m) _NOEXCEPT
2150{
2151    return __o->fetch_add(__op, __m);
2152}
2153
2154template <class _Tp>
2155_LIBCPP_INLINE_VISIBILITY
2156_Tp atomic_fetch_add_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op, memory_order __m) _NOEXCEPT
2157{
2158    return __o->fetch_add(__op, __m);
2159}
2160
2161// atomic_fetch_sub
2162
2163template <class _Tp>
2164_LIBCPP_INLINE_VISIBILITY
2165_Tp atomic_fetch_sub(volatile atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op) _NOEXCEPT
2166{
2167    return __o->fetch_sub(__op);
2168}
2169
2170template <class _Tp>
2171_LIBCPP_INLINE_VISIBILITY
2172_Tp atomic_fetch_sub(atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op) _NOEXCEPT
2173{
2174    return __o->fetch_sub(__op);
2175}
2176
2177// atomic_fetch_sub_explicit
2178
2179template <class _Tp>
2180_LIBCPP_INLINE_VISIBILITY
2181_Tp atomic_fetch_sub_explicit(volatile atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op, memory_order __m) _NOEXCEPT
2182{
2183    return __o->fetch_sub(__op, __m);
2184}
2185
2186template <class _Tp>
2187_LIBCPP_INLINE_VISIBILITY
2188_Tp atomic_fetch_sub_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op, memory_order __m) _NOEXCEPT
2189{
2190    return __o->fetch_sub(__op, __m);
2191}
2192
2193// atomic_fetch_and
2194
2195template <class _Tp>
2196_LIBCPP_INLINE_VISIBILITY
2197typename enable_if
2198<
2199    is_integral<_Tp>::value && !is_same<_Tp, bool>::value,
2200    _Tp
2201>::type
2202atomic_fetch_and(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op) _NOEXCEPT
2203{
2204    return __o->fetch_and(__op);
2205}
2206
2207template <class _Tp>
2208_LIBCPP_INLINE_VISIBILITY
2209typename enable_if
2210<
2211    is_integral<_Tp>::value && !is_same<_Tp, bool>::value,
2212    _Tp
2213>::type
2214atomic_fetch_and(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op) _NOEXCEPT
2215{
2216    return __o->fetch_and(__op);
2217}
2218
2219// atomic_fetch_and_explicit
2220
2221template <class _Tp>
2222_LIBCPP_INLINE_VISIBILITY
2223typename enable_if
2224<
2225    is_integral<_Tp>::value && !is_same<_Tp, bool>::value,
2226    _Tp
2227>::type
2228atomic_fetch_and_explicit(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op, memory_order __m) _NOEXCEPT
2229{
2230    return __o->fetch_and(__op, __m);
2231}
2232
2233template <class _Tp>
2234_LIBCPP_INLINE_VISIBILITY
2235typename enable_if
2236<
2237    is_integral<_Tp>::value && !is_same<_Tp, bool>::value,
2238    _Tp
2239>::type
2240atomic_fetch_and_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op, memory_order __m) _NOEXCEPT
2241{
2242    return __o->fetch_and(__op, __m);
2243}
2244
2245// atomic_fetch_or
2246
2247template <class _Tp>
2248_LIBCPP_INLINE_VISIBILITY
2249typename enable_if
2250<
2251    is_integral<_Tp>::value && !is_same<_Tp, bool>::value,
2252    _Tp
2253>::type
2254atomic_fetch_or(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op) _NOEXCEPT
2255{
2256    return __o->fetch_or(__op);
2257}
2258
2259template <class _Tp>
2260_LIBCPP_INLINE_VISIBILITY
2261typename enable_if
2262<
2263    is_integral<_Tp>::value && !is_same<_Tp, bool>::value,
2264    _Tp
2265>::type
2266atomic_fetch_or(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op) _NOEXCEPT
2267{
2268    return __o->fetch_or(__op);
2269}
2270
2271// atomic_fetch_or_explicit
2272
2273template <class _Tp>
2274_LIBCPP_INLINE_VISIBILITY
2275typename enable_if
2276<
2277    is_integral<_Tp>::value && !is_same<_Tp, bool>::value,
2278    _Tp
2279>::type
2280atomic_fetch_or_explicit(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op, memory_order __m) _NOEXCEPT
2281{
2282    return __o->fetch_or(__op, __m);
2283}
2284
2285template <class _Tp>
2286_LIBCPP_INLINE_VISIBILITY
2287typename enable_if
2288<
2289    is_integral<_Tp>::value && !is_same<_Tp, bool>::value,
2290    _Tp
2291>::type
2292atomic_fetch_or_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op, memory_order __m) _NOEXCEPT
2293{
2294    return __o->fetch_or(__op, __m);
2295}
2296
2297// atomic_fetch_xor
2298
2299template <class _Tp>
2300_LIBCPP_INLINE_VISIBILITY
2301typename enable_if
2302<
2303    is_integral<_Tp>::value && !is_same<_Tp, bool>::value,
2304    _Tp
2305>::type
2306atomic_fetch_xor(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op) _NOEXCEPT
2307{
2308    return __o->fetch_xor(__op);
2309}
2310
2311template <class _Tp>
2312_LIBCPP_INLINE_VISIBILITY
2313typename enable_if
2314<
2315    is_integral<_Tp>::value && !is_same<_Tp, bool>::value,
2316    _Tp
2317>::type
2318atomic_fetch_xor(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op) _NOEXCEPT
2319{
2320    return __o->fetch_xor(__op);
2321}
2322
2323// atomic_fetch_xor_explicit
2324
2325template <class _Tp>
2326_LIBCPP_INLINE_VISIBILITY
2327typename enable_if
2328<
2329    is_integral<_Tp>::value && !is_same<_Tp, bool>::value,
2330    _Tp
2331>::type
2332atomic_fetch_xor_explicit(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op, memory_order __m) _NOEXCEPT
2333{
2334    return __o->fetch_xor(__op, __m);
2335}
2336
2337template <class _Tp>
2338_LIBCPP_INLINE_VISIBILITY
2339typename enable_if
2340<
2341    is_integral<_Tp>::value && !is_same<_Tp, bool>::value,
2342    _Tp
2343>::type
2344atomic_fetch_xor_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op, memory_order __m) _NOEXCEPT
2345{
2346    return __o->fetch_xor(__op, __m);
2347}
2348
2349// flag type and operations
2350
2351typedef struct atomic_flag
2352{
2353    __cxx_atomic_impl<_LIBCPP_ATOMIC_FLAG_TYPE> __a_;
2354
2355    _LIBCPP_INLINE_VISIBILITY
2356    bool test(memory_order __m = memory_order_seq_cst) const volatile _NOEXCEPT
2357        {return _LIBCPP_ATOMIC_FLAG_TYPE(true) == __cxx_atomic_load(&__a_, __m);}
2358    _LIBCPP_INLINE_VISIBILITY
2359    bool test(memory_order __m = memory_order_seq_cst) const _NOEXCEPT
2360        {return _LIBCPP_ATOMIC_FLAG_TYPE(true) == __cxx_atomic_load(&__a_, __m);}
2361
2362    _LIBCPP_INLINE_VISIBILITY
2363    bool test_and_set(memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT
2364        {return __cxx_atomic_exchange(&__a_, _LIBCPP_ATOMIC_FLAG_TYPE(true), __m);}
2365    _LIBCPP_INLINE_VISIBILITY
2366    bool test_and_set(memory_order __m = memory_order_seq_cst) _NOEXCEPT
2367        {return __cxx_atomic_exchange(&__a_, _LIBCPP_ATOMIC_FLAG_TYPE(true), __m);}
2368    _LIBCPP_INLINE_VISIBILITY
2369    void clear(memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT
2370        {__cxx_atomic_store(&__a_, _LIBCPP_ATOMIC_FLAG_TYPE(false), __m);}
2371    _LIBCPP_INLINE_VISIBILITY
2372    void clear(memory_order __m = memory_order_seq_cst) _NOEXCEPT
2373        {__cxx_atomic_store(&__a_, _LIBCPP_ATOMIC_FLAG_TYPE(false), __m);}
2374
2375    _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY
2376    void wait(bool __v, memory_order __m = memory_order_seq_cst) const volatile _NOEXCEPT
2377        {__cxx_atomic_wait(&__a_, _LIBCPP_ATOMIC_FLAG_TYPE(__v), __m);}
2378    _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY
2379    void wait(bool __v, memory_order __m = memory_order_seq_cst) const _NOEXCEPT
2380        {__cxx_atomic_wait(&__a_, _LIBCPP_ATOMIC_FLAG_TYPE(__v), __m);}
2381    _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY
2382    void notify_one() volatile _NOEXCEPT
2383        {__cxx_atomic_notify_one(&__a_);}
2384    _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY
2385    void notify_one() _NOEXCEPT
2386        {__cxx_atomic_notify_one(&__a_);}
2387    _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY
2388    void notify_all() volatile _NOEXCEPT
2389        {__cxx_atomic_notify_all(&__a_);}
2390    _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY
2391    void notify_all() _NOEXCEPT
2392        {__cxx_atomic_notify_all(&__a_);}
2393
2394#if _LIBCPP_STD_VER > 17
2395    _LIBCPP_INLINE_VISIBILITY constexpr
2396    atomic_flag() _NOEXCEPT : __a_(false) {}
2397#else
2398    _LIBCPP_INLINE_VISIBILITY
2399    atomic_flag() _NOEXCEPT = default;
2400#endif
2401
2402    _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR
2403    atomic_flag(bool __b) _NOEXCEPT : __a_(__b) {} // EXTENSION
2404
2405    atomic_flag(const atomic_flag&) = delete;
2406    atomic_flag& operator=(const atomic_flag&) = delete;
2407    atomic_flag& operator=(const atomic_flag&) volatile = delete;
2408
2409} atomic_flag;
2410
2411
2412inline _LIBCPP_INLINE_VISIBILITY
2413bool
2414atomic_flag_test(const volatile atomic_flag* __o) _NOEXCEPT
2415{
2416    return __o->test();
2417}
2418
2419inline _LIBCPP_INLINE_VISIBILITY
2420bool
2421atomic_flag_test(const atomic_flag* __o) _NOEXCEPT
2422{
2423    return __o->test();
2424}
2425
2426inline _LIBCPP_INLINE_VISIBILITY
2427bool
2428atomic_flag_test_explicit(const volatile atomic_flag* __o, memory_order __m) _NOEXCEPT
2429{
2430    return __o->test(__m);
2431}
2432
2433inline _LIBCPP_INLINE_VISIBILITY
2434bool
2435atomic_flag_test_explicit(const atomic_flag* __o, memory_order __m) _NOEXCEPT
2436{
2437    return __o->test(__m);
2438}
2439
2440inline _LIBCPP_INLINE_VISIBILITY
2441bool
2442atomic_flag_test_and_set(volatile atomic_flag* __o) _NOEXCEPT
2443{
2444    return __o->test_and_set();
2445}
2446
2447inline _LIBCPP_INLINE_VISIBILITY
2448bool
2449atomic_flag_test_and_set(atomic_flag* __o) _NOEXCEPT
2450{
2451    return __o->test_and_set();
2452}
2453
2454inline _LIBCPP_INLINE_VISIBILITY
2455bool
2456atomic_flag_test_and_set_explicit(volatile atomic_flag* __o, memory_order __m) _NOEXCEPT
2457{
2458    return __o->test_and_set(__m);
2459}
2460
2461inline _LIBCPP_INLINE_VISIBILITY
2462bool
2463atomic_flag_test_and_set_explicit(atomic_flag* __o, memory_order __m) _NOEXCEPT
2464{
2465    return __o->test_and_set(__m);
2466}
2467
2468inline _LIBCPP_INLINE_VISIBILITY
2469void
2470atomic_flag_clear(volatile atomic_flag* __o) _NOEXCEPT
2471{
2472    __o->clear();
2473}
2474
2475inline _LIBCPP_INLINE_VISIBILITY
2476void
2477atomic_flag_clear(atomic_flag* __o) _NOEXCEPT
2478{
2479    __o->clear();
2480}
2481
2482inline _LIBCPP_INLINE_VISIBILITY
2483void
2484atomic_flag_clear_explicit(volatile atomic_flag* __o, memory_order __m) _NOEXCEPT
2485{
2486    __o->clear(__m);
2487}
2488
2489inline _LIBCPP_INLINE_VISIBILITY
2490void
2491atomic_flag_clear_explicit(atomic_flag* __o, memory_order __m) _NOEXCEPT
2492{
2493    __o->clear(__m);
2494}
2495
2496inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_AVAILABILITY_SYNC
2497void
2498atomic_flag_wait(const volatile atomic_flag* __o, bool __v) _NOEXCEPT
2499{
2500    __o->wait(__v);
2501}
2502
2503inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_AVAILABILITY_SYNC
2504void
2505atomic_flag_wait(const atomic_flag* __o, bool __v) _NOEXCEPT
2506{
2507    __o->wait(__v);
2508}
2509
2510inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_AVAILABILITY_SYNC
2511void
2512atomic_flag_wait_explicit(const volatile atomic_flag* __o,
2513                          bool __v, memory_order __m) _NOEXCEPT
2514{
2515    __o->wait(__v, __m);
2516}
2517
2518inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_AVAILABILITY_SYNC
2519void
2520atomic_flag_wait_explicit(const atomic_flag* __o,
2521                          bool __v, memory_order __m) _NOEXCEPT
2522{
2523    __o->wait(__v, __m);
2524}
2525
2526inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_AVAILABILITY_SYNC
2527void
2528atomic_flag_notify_one(volatile atomic_flag* __o) _NOEXCEPT
2529{
2530    __o->notify_one();
2531}
2532
2533inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_AVAILABILITY_SYNC
2534void
2535atomic_flag_notify_one(atomic_flag* __o) _NOEXCEPT
2536{
2537    __o->notify_one();
2538}
2539
2540inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_AVAILABILITY_SYNC
2541void
2542atomic_flag_notify_all(volatile atomic_flag* __o) _NOEXCEPT
2543{
2544    __o->notify_all();
2545}
2546
2547inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_AVAILABILITY_SYNC
2548void
2549atomic_flag_notify_all(atomic_flag* __o) _NOEXCEPT
2550{
2551    __o->notify_all();
2552}
2553
2554// fences
2555
2556inline _LIBCPP_INLINE_VISIBILITY
2557void
2558atomic_thread_fence(memory_order __m) _NOEXCEPT
2559{
2560    __cxx_atomic_thread_fence(__m);
2561}
2562
2563inline _LIBCPP_INLINE_VISIBILITY
2564void
2565atomic_signal_fence(memory_order __m) _NOEXCEPT
2566{
2567    __cxx_atomic_signal_fence(__m);
2568}
2569
2570// Atomics for standard typedef types
2571
2572typedef atomic<bool>               atomic_bool;
2573typedef atomic<char>               atomic_char;
2574typedef atomic<signed char>        atomic_schar;
2575typedef atomic<unsigned char>      atomic_uchar;
2576typedef atomic<short>              atomic_short;
2577typedef atomic<unsigned short>     atomic_ushort;
2578typedef atomic<int>                atomic_int;
2579typedef atomic<unsigned int>       atomic_uint;
2580typedef atomic<long>               atomic_long;
2581typedef atomic<unsigned long>      atomic_ulong;
2582typedef atomic<long long>          atomic_llong;
2583typedef atomic<unsigned long long> atomic_ullong;
2584#ifndef _LIBCPP_HAS_NO_CHAR8_T
2585typedef atomic<char8_t>            atomic_char8_t;
2586#endif
2587typedef atomic<char16_t>           atomic_char16_t;
2588typedef atomic<char32_t>           atomic_char32_t;
2589#ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
2590typedef atomic<wchar_t>            atomic_wchar_t;
2591#endif
2592
2593typedef atomic<int_least8_t>   atomic_int_least8_t;
2594typedef atomic<uint_least8_t>  atomic_uint_least8_t;
2595typedef atomic<int_least16_t>  atomic_int_least16_t;
2596typedef atomic<uint_least16_t> atomic_uint_least16_t;
2597typedef atomic<int_least32_t>  atomic_int_least32_t;
2598typedef atomic<uint_least32_t> atomic_uint_least32_t;
2599typedef atomic<int_least64_t>  atomic_int_least64_t;
2600typedef atomic<uint_least64_t> atomic_uint_least64_t;
2601
2602typedef atomic<int_fast8_t>   atomic_int_fast8_t;
2603typedef atomic<uint_fast8_t>  atomic_uint_fast8_t;
2604typedef atomic<int_fast16_t>  atomic_int_fast16_t;
2605typedef atomic<uint_fast16_t> atomic_uint_fast16_t;
2606typedef atomic<int_fast32_t>  atomic_int_fast32_t;
2607typedef atomic<uint_fast32_t> atomic_uint_fast32_t;
2608typedef atomic<int_fast64_t>  atomic_int_fast64_t;
2609typedef atomic<uint_fast64_t> atomic_uint_fast64_t;
2610
2611typedef atomic< int8_t>  atomic_int8_t;
2612typedef atomic<uint8_t>  atomic_uint8_t;
2613typedef atomic< int16_t> atomic_int16_t;
2614typedef atomic<uint16_t> atomic_uint16_t;
2615typedef atomic< int32_t> atomic_int32_t;
2616typedef atomic<uint32_t> atomic_uint32_t;
2617typedef atomic< int64_t> atomic_int64_t;
2618typedef atomic<uint64_t> atomic_uint64_t;
2619
2620typedef atomic<intptr_t>  atomic_intptr_t;
2621typedef atomic<uintptr_t> atomic_uintptr_t;
2622typedef atomic<size_t>    atomic_size_t;
2623typedef atomic<ptrdiff_t> atomic_ptrdiff_t;
2624typedef atomic<intmax_t>  atomic_intmax_t;
2625typedef atomic<uintmax_t> atomic_uintmax_t;
2626
2627// atomic_*_lock_free : prefer the contention type most highly, then the largest lock-free type
2628
2629#ifdef __cpp_lib_atomic_is_always_lock_free
2630# define _LIBCPP_CONTENTION_LOCK_FREE ::std::__libcpp_is_always_lock_free<__cxx_contention_t>::__value
2631#else
2632# define _LIBCPP_CONTENTION_LOCK_FREE false
2633#endif
2634
2635#if ATOMIC_LLONG_LOCK_FREE == 2
2636typedef __conditional_t<_LIBCPP_CONTENTION_LOCK_FREE, __cxx_contention_t, long long>          __libcpp_signed_lock_free;
2637typedef __conditional_t<_LIBCPP_CONTENTION_LOCK_FREE, __cxx_contention_t, unsigned long long> __libcpp_unsigned_lock_free;
2638#elif ATOMIC_INT_LOCK_FREE == 2
2639typedef __conditional_t<_LIBCPP_CONTENTION_LOCK_FREE, __cxx_contention_t, int>                __libcpp_signed_lock_free;
2640typedef __conditional_t<_LIBCPP_CONTENTION_LOCK_FREE, __cxx_contention_t, unsigned int>       __libcpp_unsigned_lock_free;
2641#elif ATOMIC_SHORT_LOCK_FREE == 2
2642typedef __conditional_t<_LIBCPP_CONTENTION_LOCK_FREE, __cxx_contention_t, short>              __libcpp_signed_lock_free;
2643typedef __conditional_t<_LIBCPP_CONTENTION_LOCK_FREE, __cxx_contention_t, unsigned short>     __libcpp_unsigned_lock_free;
2644#elif ATOMIC_CHAR_LOCK_FREE == 2
2645typedef __conditional_t<_LIBCPP_CONTENTION_LOCK_FREE, __cxx_contention_t, char>               __libcpp_signed_lock_free;
2646typedef __conditional_t<_LIBCPP_CONTENTION_LOCK_FREE, __cxx_contention_t, unsigned char>      __libcpp_unsigned_lock_free;
2647#else
2648    // No signed/unsigned lock-free types
2649#define _LIBCPP_NO_LOCK_FREE_TYPES
2650#endif
2651
2652#if !defined(_LIBCPP_NO_LOCK_FREE_TYPES)
2653typedef atomic<__libcpp_signed_lock_free> atomic_signed_lock_free;
2654typedef atomic<__libcpp_unsigned_lock_free> atomic_unsigned_lock_free;
2655#endif
2656
2657#define ATOMIC_FLAG_INIT {false}
2658#define ATOMIC_VAR_INIT(__v) {__v}
2659
2660#if _LIBCPP_STD_VER > 17 && !defined(_LIBCPP_DISABLE_DEPRECATION_WARNINGS)
2661# if defined(_LIBCPP_CLANG_VER) && _LIBCPP_CLANG_VER >= 1400
2662#  pragma clang deprecated(ATOMIC_VAR_INIT)
2663# endif
2664#endif // _LIBCPP_STD_VER > 17 && !defined(_LIBCPP_DISABLE_DEPRECATION_WARNINGS)
2665
2666_LIBCPP_END_NAMESPACE_STD
2667
2668#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
2669#  include <cmath>
2670#  include <compare>
2671#  include <type_traits>
2672#endif
2673
2674#endif // _LIBCPP_ATOMIC
2675