1 /*
2 * %CopyrightBegin%
3 *
4 * Copyright Ericsson AB 2011-2018. All Rights Reserved.
5 *
6 * Licensed under the Apache License, Version 2.0 (the "License");
7 * you may not use this file except in compliance with the License.
8 * You may obtain a copy of the License at
9 *
10 * http://www.apache.org/licenses/LICENSE-2.0
11 *
12 * Unless required by applicable law or agreed to in writing, software
13 * distributed under the License is distributed on an "AS IS" BASIS,
14 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 * See the License for the specific language governing permissions and
16 * limitations under the License.
17 *
18 * %CopyrightEnd%
19 */
20
21 /*
22 * Description: Thread progress information. Used by lock free algorithms
23 * to determine when all involved threads are guaranteed to
24 * have passed a specific point of execution.
25 *
26 * Usage instructions can be found in ert_thr_progress.c
27 *
28 * Author: Rickard Green
29 */
30
31 #ifndef ERL_THR_PROGRESS_H__TSD_TYPE__
32 #define ERL_THR_PROGRESS_H__TSD_TYPE__
33
34 #include "sys.h"
35
36 void erts_thr_progress_block(void);
37 void erts_thr_progress_unblock(void);
38 int erts_thr_progress_is_blocking(void);
39
40 typedef Uint64 ErtsThrPrgrVal;
41
42 #define ERTS_THR_PRGR_WAKEUP_DATA_SIZE 4 /* Need to be an even power of 2. */
43
44 typedef struct {
45 ErtsThrPrgrVal next;
46 ErtsThrPrgrVal current;
47 int chk_next_ix;
48 struct {
49 int current;
50 int waiting;
51 } umrefc_ix;
52 } ErtsThrPrgrLeaderState;
53
54 typedef struct {
55 int id;
56 int is_managed;
57 int is_blocking;
58 #ifdef ERTS_ENABLE_LOCK_CHECK
59 int is_delaying; /* managed is always delaying */
60 #endif
61 int is_temporary;
62
63 /* --- Part below only for registered threads --- */
64
65 ErtsThrPrgrVal wakeup_request[ERTS_THR_PRGR_WAKEUP_DATA_SIZE];
66
67 /* --- Part below only for managed threads --- */
68
69 int leader; /* Needs to be first in the managed threads part */
70 int active;
71 int is_deep_sleeper;
72 ErtsThrPrgrVal confirmed;
73 ErtsThrPrgrLeaderState leader_state;
74 } ErtsThrPrgrData;
75
76 int erts_thr_progress_fatal_error_block(ErtsThrPrgrData *tmp_tpd_bufp);
77 void erts_thr_progress_fatal_error_wait(SWord timeout);
78
79
80 typedef struct ErtsThrPrgrLaterOp_ ErtsThrPrgrLaterOp;
81 struct ErtsThrPrgrLaterOp_ {
82 ErtsThrPrgrVal later;
83 void (*func)(void *);
84 void *data;
85 ErtsThrPrgrLaterOp *next;
86 };
87
88 #endif
89
90 #if !defined(ERL_THR_PROGRESS_H__) && !defined(ERL_THR_PROGRESS_TSD_TYPE_ONLY)
91 #define ERL_THR_PROGRESS_H__
92
93 #include "erl_threads.h"
94 #include "erl_process.h"
95
96
97 /* ERTS_THR_PRGR_VAL_FIRST should only be used when initializing... */
98 #define ERTS_THR_PRGR_VAL_FIRST ((ErtsThrPrgrVal) 0)
99 #define ERTS_THR_PRGR_VAL_WAITING (~((ErtsThrPrgrVal) 0))
100 #define ERTS_THR_PRGR_INVALID (~((ErtsThrPrgrVal) 0))
101
102 extern erts_tsd_key_t erts_thr_prgr_data_key__;
103
104 #define ERTS_THR_PRGR_ATOMIC erts_atomic64_t
105
106 typedef struct {
107 void *arg;
108 void (*wakeup)(void *);
109 void (*prepare_wait)(void *);
110 void (*wait)(void *);
111 void (*finalize_wait)(void *);
112 } ErtsThrPrgrCallbacks;
113
114 typedef struct {
115 ERTS_THR_PRGR_ATOMIC current;
116 } ErtsThrPrgr;
117
118 typedef int ErtsThrPrgrDelayHandle;
119 #define ERTS_THR_PRGR_DHANDLE_MANAGED ((ErtsThrPrgrDelayHandle) -1)
120 /* ERTS_THR_PRGR_DHANDLE_MANAGED implies managed thread */
121 #define ERTS_THR_PRGR_DHANDLE_INVALID ((ErtsThrPrgrDelayHandle) -2)
122
123 extern ErtsThrPrgr erts_thr_prgr__;
124
125 void erts_thr_progress_pre_init(void);
126 void erts_thr_progress_init(int no_schedulers, int managed, int unmanaged);
127 ErtsThrPrgrData *erts_thr_progress_register_managed_thread(
128 ErtsSchedulerData *esdp, ErtsThrPrgrCallbacks *, int, int);
129 void erts_thr_progress_register_unmanaged_thread(ErtsThrPrgrCallbacks *);
130 void erts_thr_progress_active(ErtsThrPrgrData *, int on);
131 void erts_thr_progress_wakeup(ErtsThrPrgrData *,
132 ErtsThrPrgrVal value);
133 int erts_thr_progress_update(ErtsThrPrgrData *);
134 int erts_thr_progress_leader_update(ErtsThrPrgrData *);
135 void erts_thr_progress_prepare_wait(ErtsThrPrgrData *);
136 void erts_thr_progress_finalize_wait(ErtsThrPrgrData *);
137 ErtsThrPrgrDelayHandle erts_thr_progress_unmanaged_delay__(void);
138 void erts_thr_progress_unmanaged_continue__(int umrefc_ix);
139 ErtsThrPrgrData *erts_thr_progress_data(void);
140
141 void erts_thr_progress_dbg_print_state(void);
142
143 ERTS_GLB_INLINE ErtsThrPrgrData *erts_thr_prgr_data(ErtsSchedulerData *esdp);
144
145 ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_prgr_read_nob__(ERTS_THR_PRGR_ATOMIC *atmc);
146 ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_prgr_read_acqb__(ERTS_THR_PRGR_ATOMIC *atmc);
147 ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_prgr_read_mb__(ERTS_THR_PRGR_ATOMIC *atmc);
148
149 ERTS_GLB_INLINE int erts_thr_progress_is_managed_thread(void);
150 ERTS_GLB_INLINE ErtsThrPrgrDelayHandle erts_thr_progress_unmanaged_delay(void);
151 ERTS_GLB_INLINE void erts_thr_progress_unmanaged_continue(ErtsThrPrgrDelayHandle handle);
152 #ifdef ERTS_ENABLE_LOCK_CHECK
153 ERTS_GLB_INLINE int erts_thr_progress_lc_is_delaying(void);
154 #endif
155 ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_progress_current_to_later__(ErtsThrPrgrVal val);
156 ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_progress_later(ErtsSchedulerData *);
157 ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_progress_current(void);
158 ERTS_GLB_INLINE int erts_thr_progress_has_passed__(ErtsThrPrgrVal val1, ErtsThrPrgrVal val2);
159 ERTS_GLB_INLINE int erts_thr_progress_has_reached_this(ErtsThrPrgrVal this, ErtsThrPrgrVal val);
160 ERTS_GLB_INLINE int erts_thr_progress_equal(ErtsThrPrgrVal val1,
161 ErtsThrPrgrVal val2);
162 ERTS_GLB_INLINE int erts_thr_progress_cmp(ErtsThrPrgrVal val1, ErtsThrPrgrVal val2);
163 ERTS_GLB_INLINE int erts_thr_progress_has_reached(ErtsThrPrgrVal val);
164
165 #if ERTS_GLB_INLINE_INCL_FUNC_DEF
166
167 ERTS_GLB_INLINE ErtsThrPrgrData *
erts_thr_prgr_data(ErtsSchedulerData * esdp)168 erts_thr_prgr_data(ErtsSchedulerData *esdp) {
169 if (esdp) {
170 return &esdp->thr_progress_data;
171 } else {
172 return erts_thr_progress_data();
173 }
174 }
175
176 ERTS_GLB_INLINE ErtsThrPrgrVal
erts_thr_prgr_read_nob__(ERTS_THR_PRGR_ATOMIC * atmc)177 erts_thr_prgr_read_nob__(ERTS_THR_PRGR_ATOMIC *atmc)
178 {
179 return (ErtsThrPrgrVal) erts_atomic64_read_nob(atmc);
180 }
181
182 ERTS_GLB_INLINE ErtsThrPrgrVal
erts_thr_prgr_read_acqb__(ERTS_THR_PRGR_ATOMIC * atmc)183 erts_thr_prgr_read_acqb__(ERTS_THR_PRGR_ATOMIC *atmc)
184 {
185 return (ErtsThrPrgrVal) erts_atomic64_read_acqb(atmc);
186 }
187
188 ERTS_GLB_INLINE ErtsThrPrgrVal
erts_thr_prgr_read_mb__(ERTS_THR_PRGR_ATOMIC * atmc)189 erts_thr_prgr_read_mb__(ERTS_THR_PRGR_ATOMIC *atmc)
190 {
191 return (ErtsThrPrgrVal) erts_atomic64_read_mb(atmc);
192 }
193
194 ERTS_GLB_INLINE int
erts_thr_progress_is_managed_thread(void)195 erts_thr_progress_is_managed_thread(void)
196 {
197 ErtsThrPrgrData *tpd = erts_tsd_get(erts_thr_prgr_data_key__);
198 return tpd && tpd->is_managed;
199 }
200
201 ERTS_GLB_INLINE ErtsThrPrgrDelayHandle
erts_thr_progress_unmanaged_delay(void)202 erts_thr_progress_unmanaged_delay(void)
203 {
204 if (erts_thr_progress_is_managed_thread())
205 return ERTS_THR_PRGR_DHANDLE_MANAGED; /* Nothing to do */
206 else
207 return erts_thr_progress_unmanaged_delay__();
208 }
209
210 ERTS_GLB_INLINE void
erts_thr_progress_unmanaged_continue(ErtsThrPrgrDelayHandle handle)211 erts_thr_progress_unmanaged_continue(ErtsThrPrgrDelayHandle handle)
212 {
213 ASSERT(handle != ERTS_THR_PRGR_DHANDLE_MANAGED
214 || erts_thr_progress_is_managed_thread());
215 if (handle != ERTS_THR_PRGR_DHANDLE_MANAGED)
216 erts_thr_progress_unmanaged_continue__(handle);
217 }
218
219 #ifdef ERTS_ENABLE_LOCK_CHECK
220
221 ERTS_GLB_INLINE int
erts_thr_progress_lc_is_delaying(void)222 erts_thr_progress_lc_is_delaying(void)
223 {
224 ErtsThrPrgrData *tpd = erts_tsd_get(erts_thr_prgr_data_key__);
225 return tpd && tpd->is_delaying;
226 }
227
228 #endif
229
230 ERTS_GLB_INLINE ErtsThrPrgrVal
erts_thr_progress_current_to_later__(ErtsThrPrgrVal val)231 erts_thr_progress_current_to_later__(ErtsThrPrgrVal val)
232 {
233 if (val == (ERTS_THR_PRGR_VAL_WAITING-((ErtsThrPrgrVal)2)))
234 return ((ErtsThrPrgrVal) 0);
235 else if (val == (ERTS_THR_PRGR_VAL_WAITING-((ErtsThrPrgrVal)1)))
236 return ((ErtsThrPrgrVal) 1);
237 else
238 return val + ((ErtsThrPrgrVal) 2);
239 }
240
241 ERTS_GLB_INLINE ErtsThrPrgrVal
erts_thr_progress_later(ErtsSchedulerData * esdp)242 erts_thr_progress_later(ErtsSchedulerData *esdp)
243 {
244 ErtsThrPrgrData *tpd;
245 ErtsThrPrgrVal val;
246 if (esdp) {
247 tpd = &esdp->thr_progress_data;
248 managed_thread:
249 val = tpd->confirmed;
250 ERTS_THR_MEMORY_BARRIER;
251 }
252 else {
253 tpd = erts_tsd_get(erts_thr_prgr_data_key__);
254 if (tpd && tpd->is_managed)
255 goto managed_thread;
256 val = erts_thr_prgr_read_mb__(&erts_thr_prgr__.current);
257 }
258 ASSERT(val != ERTS_THR_PRGR_VAL_WAITING);
259 return erts_thr_progress_current_to_later__(val);
260 }
261
262 ERTS_GLB_INLINE ErtsThrPrgrVal
erts_thr_progress_current(void)263 erts_thr_progress_current(void)
264 {
265 if (erts_thr_progress_is_managed_thread())
266 return erts_thr_prgr_read_nob__(&erts_thr_prgr__.current);
267 else
268 return erts_thr_prgr_read_acqb__(&erts_thr_prgr__.current);
269 }
270
271 ERTS_GLB_INLINE int
erts_thr_progress_has_passed__(ErtsThrPrgrVal val1,ErtsThrPrgrVal val0)272 erts_thr_progress_has_passed__(ErtsThrPrgrVal val1, ErtsThrPrgrVal val0)
273 {
274 if ((((((ErtsThrPrgrVal) 1) << 63) & val1)
275 ^ ((((ErtsThrPrgrVal) 1) << 63) & val0)) != 0) {
276 /* May have wrapped... */
277 if (val1 < (((ErtsThrPrgrVal) 1) << 62)
278 && val0 > (((ErtsThrPrgrVal) 3) << 62)) {
279 /*
280 * 'val1' has wrapped but 'val0' has not yet wrapped. While in
281 * these ranges 'current' is considered later than 'val0'.
282 */
283 return 1;
284 }
285 }
286 return val1 > val0;
287 }
288
289 ERTS_GLB_INLINE int
erts_thr_progress_has_reached_this(ErtsThrPrgrVal this,ErtsThrPrgrVal val)290 erts_thr_progress_has_reached_this(ErtsThrPrgrVal this, ErtsThrPrgrVal val)
291 {
292 if (this == val)
293 return 1;
294 return erts_thr_progress_has_passed__(this, val);
295 }
296
297 ERTS_GLB_INLINE int
erts_thr_progress_equal(ErtsThrPrgrVal val1,ErtsThrPrgrVal val2)298 erts_thr_progress_equal(ErtsThrPrgrVal val1, ErtsThrPrgrVal val2)
299 {
300 return val1 == val2 && val1 != ERTS_THR_PRGR_INVALID;
301 }
302
303 ERTS_GLB_INLINE int
erts_thr_progress_cmp(ErtsThrPrgrVal val1,ErtsThrPrgrVal val2)304 erts_thr_progress_cmp(ErtsThrPrgrVal val1, ErtsThrPrgrVal val2)
305 {
306 if (val1 == val2)
307 return 0;
308 if (erts_thr_progress_has_passed__(val1, val2))
309 return 1;
310 else
311 return -1;
312 }
313
314 ERTS_GLB_INLINE int
erts_thr_progress_has_reached(ErtsThrPrgrVal val)315 erts_thr_progress_has_reached(ErtsThrPrgrVal val)
316 {
317 ErtsThrPrgrVal current = erts_thr_progress_current();
318 return erts_thr_progress_has_reached_this(current, val);
319 }
320
321 #endif
322
323
324 #endif
325