1 // Copyright 2017 The Abseil Authors.
2 //
3 // Licensed under the Apache License, Version 2.0 (the "License");
4 // you may not use this file except in compliance with the License.
5 // You may obtain a copy of the License at
6 //
7 // http://www.apache.org/licenses/LICENSE-2.0
8 //
9 // Unless required by applicable law or agreed to in writing, software
10 // distributed under the License is distributed on an "AS IS" BASIS,
11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 // See the License for the specific language governing permissions and
13 // limitations under the License.
14 //
15 // -----------------------------------------------------------------------------
16 // File: call_once.h
17 // -----------------------------------------------------------------------------
18 //
19 // This header file provides an Abseil version of `std::call_once` for invoking
20 // a given function at most once, across all threads. This Abseil version is
21 // faster than the C++11 version and incorporates the C++17 argument-passing
22 // fix, so that (for example) non-const references may be passed to the invoked
23 // function.
24
25 #ifndef ABSL_BASE_CALL_ONCE_H_
26 #define ABSL_BASE_CALL_ONCE_H_
27
28 #include <algorithm>
29 #include <atomic>
30 #include <cstdint>
31 #include <type_traits>
32
33 #include "absl/base/internal/invoke.h"
34 #include "absl/base/internal/low_level_scheduling.h"
35 #include "absl/base/internal/raw_logging.h"
36 #include "absl/base/internal/scheduling_mode.h"
37 #include "absl/base/internal/spinlock_wait.h"
38 #include "absl/base/macros.h"
39 #include "absl/base/port.h"
40
41 namespace absl {
42
43 class once_flag;
44
45 namespace base_internal {
46 std::atomic<uint32_t>* ControlWord(absl::once_flag* flag);
47 } // namespace base_internal
48
49 // call_once()
50 //
51 // For all invocations using a given `once_flag`, invokes a given `fn` exactly
52 // once across all threads. The first call to `call_once()` with a particular
53 // `once_flag` argument (that does not throw an exception) will run the
54 // specified function with the provided `args`; other calls with the same
55 // `once_flag` argument will not run the function, but will wait
56 // for the provided function to finish running (if it is still running).
57 //
58 // This mechanism provides a safe, simple, and fast mechanism for one-time
59 // initialization in a multi-threaded process.
60 //
61 // Example:
62 //
63 // class MyInitClass {
64 // public:
65 // ...
66 // mutable absl::once_flag once_;
67 //
68 // MyInitClass* init() const {
69 // absl::call_once(once_, &MyInitClass::Init, this);
70 // return ptr_;
71 // }
72 //
73 template <typename Callable, typename... Args>
74 void call_once(absl::once_flag& flag, Callable&& fn, Args&&... args);
75
76 // once_flag
77 //
78 // Objects of this type are used to distinguish calls to `call_once()` and
79 // ensure the provided function is only invoked once across all threads. This
80 // type is not copyable or movable. However, it has a `constexpr`
81 // constructor, and is safe to use as a namespace-scoped global variable.
82 class once_flag {
83 public:
once_flag()84 constexpr once_flag() : control_(0) {}
85 once_flag(const once_flag&) = delete;
86 once_flag& operator=(const once_flag&) = delete;
87
88 private:
89 friend std::atomic<uint32_t>* base_internal::ControlWord(once_flag* flag);
90 std::atomic<uint32_t> control_;
91 };
92
93 //------------------------------------------------------------------------------
94 // End of public interfaces.
95 // Implementation details follow.
96 //------------------------------------------------------------------------------
97
98 namespace base_internal {
99
100 // Like call_once, but uses KERNEL_ONLY scheduling. Intended to be used to
101 // initialize entities used by the scheduler implementation.
102 template <typename Callable, typename... Args>
103 void LowLevelCallOnce(absl::once_flag* flag, Callable&& fn, Args&&... args);
104
105 // Disables scheduling while on stack when scheduling mode is non-cooperative.
106 // No effect for cooperative scheduling modes.
107 class SchedulingHelper {
108 public:
SchedulingHelper(base_internal::SchedulingMode mode)109 explicit SchedulingHelper(base_internal::SchedulingMode mode) : mode_(mode) {
110 if (mode_ == base_internal::SCHEDULE_KERNEL_ONLY) {
111 guard_result_ = base_internal::SchedulingGuard::DisableRescheduling();
112 }
113 }
114
~SchedulingHelper()115 ~SchedulingHelper() {
116 if (mode_ == base_internal::SCHEDULE_KERNEL_ONLY) {
117 base_internal::SchedulingGuard::EnableRescheduling(guard_result_);
118 }
119 }
120
121 private:
122 base_internal::SchedulingMode mode_;
123 bool guard_result_;
124 };
125
126 // Bit patterns for call_once state machine values. Internal implementation
127 // detail, not for use by clients.
128 //
129 // The bit patterns are arbitrarily chosen from unlikely values, to aid in
130 // debugging. However, kOnceInit must be 0, so that a zero-initialized
131 // once_flag will be valid for immediate use.
132 enum {
133 kOnceInit = 0,
134 kOnceRunning = 0x65C2937B,
135 kOnceWaiter = 0x05A308D2,
136 // A very small constant is chosen for kOnceDone so that it fit in a single
137 // compare with immediate instruction for most common ISAs. This is verified
138 // for x86, POWER and ARM.
139 kOnceDone = 221, // Random Number
140 };
141
142 template <typename Callable, typename... Args>
CallOnceImpl(std::atomic<uint32_t> * control,base_internal::SchedulingMode scheduling_mode,Callable && fn,Args &&...args)143 void CallOnceImpl(std::atomic<uint32_t>* control,
144 base_internal::SchedulingMode scheduling_mode, Callable&& fn,
145 Args&&... args) {
146 #ifndef NDEBUG
147 {
148 uint32_t old_control = control->load(std::memory_order_acquire);
149 if (old_control != kOnceInit &&
150 old_control != kOnceRunning &&
151 old_control != kOnceWaiter &&
152 old_control != kOnceDone) {
153 ABSL_RAW_LOG(
154 FATAL,
155 "Unexpected value for control word: %lx. Either the control word "
156 "has non-static storage duration (where GoogleOnceDynamic might "
157 "be appropriate), or there's been a memory corruption.",
158 static_cast<unsigned long>(old_control)); // NOLINT
159 }
160 }
161 #endif // NDEBUG
162 static const base_internal::SpinLockWaitTransition trans[] = {
163 {kOnceInit, kOnceRunning, true},
164 {kOnceRunning, kOnceWaiter, false},
165 {kOnceDone, kOnceDone, true}};
166
167 // Must do this before potentially modifying control word's state.
168 base_internal::SchedulingHelper maybe_disable_scheduling(scheduling_mode);
169 // Short circuit the simplest case to avoid procedure call overhead.
170 uint32_t old_control = kOnceInit;
171 if (control->compare_exchange_strong(old_control, kOnceRunning,
172 std::memory_order_acquire,
173 std::memory_order_relaxed) ||
174 base_internal::SpinLockWait(control, ABSL_ARRAYSIZE(trans), trans,
175 scheduling_mode) == kOnceInit) {
176 base_internal::Invoke(std::forward<Callable>(fn),
177 std::forward<Args>(args)...);
178 old_control = control->load(std::memory_order_relaxed);
179 control->store(base_internal::kOnceDone, std::memory_order_release);
180 if (old_control == base_internal::kOnceWaiter) {
181 base_internal::SpinLockWake(control, true);
182 }
183 } // else *control is already kOnceDone
184 }
185
ControlWord(once_flag * flag)186 inline std::atomic<uint32_t>* ControlWord(once_flag* flag) {
187 return &flag->control_;
188 }
189
190 template <typename Callable, typename... Args>
LowLevelCallOnce(absl::once_flag * flag,Callable && fn,Args &&...args)191 void LowLevelCallOnce(absl::once_flag* flag, Callable&& fn, Args&&... args) {
192 std::atomic<uint32_t>* once = base_internal::ControlWord(flag);
193 uint32_t s = once->load(std::memory_order_acquire);
194 if (ABSL_PREDICT_FALSE(s != base_internal::kOnceDone)) {
195 base_internal::CallOnceImpl(once, base_internal::SCHEDULE_KERNEL_ONLY,
196 std::forward<Callable>(fn),
197 std::forward<Args>(args)...);
198 }
199 }
200
201 } // namespace base_internal
202
203 template <typename Callable, typename... Args>
call_once(absl::once_flag & flag,Callable && fn,Args &&...args)204 void call_once(absl::once_flag& flag, Callable&& fn, Args&&... args) {
205 std::atomic<uint32_t>* once = base_internal::ControlWord(&flag);
206 uint32_t s = once->load(std::memory_order_acquire);
207 if (ABSL_PREDICT_FALSE(s != base_internal::kOnceDone)) {
208 base_internal::CallOnceImpl(
209 once, base_internal::SCHEDULE_COOPERATIVE_AND_KERNEL,
210 std::forward<Callable>(fn), std::forward<Args>(args)...);
211 }
212 }
213
214 } // namespace absl
215
216 #endif // ABSL_BASE_CALL_ONCE_H_
217