1 /*
2  * Implementation of the Global Interpreter Lock (GIL).
3  */
4 
5 #include <stdlib.h>
6 #include <errno.h>
7 
8 
9 /* First some general settings */
10 
11 #define INTERVAL (_PyRuntime.ceval.gil.interval >= 1 ? _PyRuntime.ceval.gil.interval : 1)
12 
13 
14 /*
15    Notes about the implementation:
16 
17    - The GIL is just a boolean variable (locked) whose access is protected
18      by a mutex (gil_mutex), and whose changes are signalled by a condition
19      variable (gil_cond). gil_mutex is taken for short periods of time,
20      and therefore mostly uncontended.
21 
22    - In the GIL-holding thread, the main loop (PyEval_EvalFrameEx) must be
23      able to release the GIL on demand by another thread. A volatile boolean
24      variable (gil_drop_request) is used for that purpose, which is checked
25      at every turn of the eval loop. That variable is set after a wait of
26      `interval` microseconds on `gil_cond` has timed out.
27 
28       [Actually, another volatile boolean variable (eval_breaker) is used
29        which ORs several conditions into one. Volatile booleans are
30        sufficient as inter-thread signalling means since Python is run
31        on cache-coherent architectures only.]
32 
33    - A thread wanting to take the GIL will first let pass a given amount of
34      time (`interval` microseconds) before setting gil_drop_request. This
35      encourages a defined switching period, but doesn't enforce it since
36      opcodes can take an arbitrary time to execute.
37 
38      The `interval` value is available for the user to read and modify
39      using the Python API `sys.{get,set}switchinterval()`.
40 
41    - When a thread releases the GIL and gil_drop_request is set, that thread
42      ensures that another GIL-awaiting thread gets scheduled.
43      It does so by waiting on a condition variable (switch_cond) until
44      the value of last_holder is changed to something else than its
45      own thread state pointer, indicating that another thread was able to
46      take the GIL.
47 
48      This is meant to prohibit the latency-adverse behaviour on multi-core
49      machines where one thread would speculatively release the GIL, but still
50      run and end up being the first to re-acquire it, making the "timeslices"
51      much longer than expected.
52      (Note: this mechanism is enabled with FORCE_SWITCHING above)
53 */
54 
55 #include "condvar.h"
56 
57 #define MUTEX_INIT(mut) \
58     if (PyMUTEX_INIT(&(mut))) { \
59         Py_FatalError("PyMUTEX_INIT(" #mut ") failed"); };
60 #define MUTEX_FINI(mut) \
61     if (PyMUTEX_FINI(&(mut))) { \
62         Py_FatalError("PyMUTEX_FINI(" #mut ") failed"); };
63 #define MUTEX_LOCK(mut) \
64     if (PyMUTEX_LOCK(&(mut))) { \
65         Py_FatalError("PyMUTEX_LOCK(" #mut ") failed"); };
66 #define MUTEX_UNLOCK(mut) \
67     if (PyMUTEX_UNLOCK(&(mut))) { \
68         Py_FatalError("PyMUTEX_UNLOCK(" #mut ") failed"); };
69 
70 #define COND_INIT(cond) \
71     if (PyCOND_INIT(&(cond))) { \
72         Py_FatalError("PyCOND_INIT(" #cond ") failed"); };
73 #define COND_FINI(cond) \
74     if (PyCOND_FINI(&(cond))) { \
75         Py_FatalError("PyCOND_FINI(" #cond ") failed"); };
76 #define COND_SIGNAL(cond) \
77     if (PyCOND_SIGNAL(&(cond))) { \
78         Py_FatalError("PyCOND_SIGNAL(" #cond ") failed"); };
79 #define COND_WAIT(cond, mut) \
80     if (PyCOND_WAIT(&(cond), &(mut))) { \
81         Py_FatalError("PyCOND_WAIT(" #cond ") failed"); };
82 #define COND_TIMED_WAIT(cond, mut, microseconds, timeout_result) \
83     { \
84         int r = PyCOND_TIMEDWAIT(&(cond), &(mut), (microseconds)); \
85         if (r < 0) \
86             Py_FatalError("PyCOND_WAIT(" #cond ") failed"); \
87         if (r) /* 1 == timeout, 2 == impl. can't say, so assume timeout */ \
88             timeout_result = 1; \
89         else \
90             timeout_result = 0; \
91     } \
92 
93 
94 #define DEFAULT_INTERVAL 5000
95 
_gil_initialize(struct _gil_runtime_state * state)96 static void _gil_initialize(struct _gil_runtime_state *state)
97 {
98     _Py_atomic_int uninitialized = {-1};
99     state->locked = uninitialized;
100     state->interval = DEFAULT_INTERVAL;
101 }
102 
gil_created(void)103 static int gil_created(void)
104 {
105     return (_Py_atomic_load_explicit(&_PyRuntime.ceval.gil.locked,
106                                      _Py_memory_order_acquire)
107             ) >= 0;
108 }
109 
create_gil(void)110 static void create_gil(void)
111 {
112     MUTEX_INIT(_PyRuntime.ceval.gil.mutex);
113 #ifdef FORCE_SWITCHING
114     MUTEX_INIT(_PyRuntime.ceval.gil.switch_mutex);
115 #endif
116     COND_INIT(_PyRuntime.ceval.gil.cond);
117 #ifdef FORCE_SWITCHING
118     COND_INIT(_PyRuntime.ceval.gil.switch_cond);
119 #endif
120     _Py_atomic_store_relaxed(&_PyRuntime.ceval.gil.last_holder, 0);
121     _Py_ANNOTATE_RWLOCK_CREATE(&_PyRuntime.ceval.gil.locked);
122     _Py_atomic_store_explicit(&_PyRuntime.ceval.gil.locked, 0,
123                               _Py_memory_order_release);
124 }
125 
destroy_gil(void)126 static void destroy_gil(void)
127 {
128     /* some pthread-like implementations tie the mutex to the cond
129      * and must have the cond destroyed first.
130      */
131     COND_FINI(_PyRuntime.ceval.gil.cond);
132     MUTEX_FINI(_PyRuntime.ceval.gil.mutex);
133 #ifdef FORCE_SWITCHING
134     COND_FINI(_PyRuntime.ceval.gil.switch_cond);
135     MUTEX_FINI(_PyRuntime.ceval.gil.switch_mutex);
136 #endif
137     _Py_atomic_store_explicit(&_PyRuntime.ceval.gil.locked, -1,
138                               _Py_memory_order_release);
139     _Py_ANNOTATE_RWLOCK_DESTROY(&_PyRuntime.ceval.gil.locked);
140 }
141 
recreate_gil(void)142 static void recreate_gil(void)
143 {
144     _Py_ANNOTATE_RWLOCK_DESTROY(&_PyRuntime.ceval.gil.locked);
145     /* XXX should we destroy the old OS resources here? */
146     create_gil();
147 }
148 
drop_gil(PyThreadState * tstate)149 static void drop_gil(PyThreadState *tstate)
150 {
151     if (!_Py_atomic_load_relaxed(&_PyRuntime.ceval.gil.locked))
152         Py_FatalError("drop_gil: GIL is not locked");
153     /* tstate is allowed to be NULL (early interpreter init) */
154     if (tstate != NULL) {
155         /* Sub-interpreter support: threads might have been switched
156            under our feet using PyThreadState_Swap(). Fix the GIL last
157            holder variable so that our heuristics work. */
158         _Py_atomic_store_relaxed(&_PyRuntime.ceval.gil.last_holder,
159                                  (uintptr_t)tstate);
160     }
161 
162     MUTEX_LOCK(_PyRuntime.ceval.gil.mutex);
163     _Py_ANNOTATE_RWLOCK_RELEASED(&_PyRuntime.ceval.gil.locked, /*is_write=*/1);
164     _Py_atomic_store_relaxed(&_PyRuntime.ceval.gil.locked, 0);
165     COND_SIGNAL(_PyRuntime.ceval.gil.cond);
166     MUTEX_UNLOCK(_PyRuntime.ceval.gil.mutex);
167 
168 #ifdef FORCE_SWITCHING
169     if (_Py_atomic_load_relaxed(&_PyRuntime.ceval.gil_drop_request) &&
170         tstate != NULL)
171     {
172         MUTEX_LOCK(_PyRuntime.ceval.gil.switch_mutex);
173         /* Not switched yet => wait */
174         if (((PyThreadState*)_Py_atomic_load_relaxed(
175                     &_PyRuntime.ceval.gil.last_holder)
176             ) == tstate)
177         {
178         RESET_GIL_DROP_REQUEST();
179             /* NOTE: if COND_WAIT does not atomically start waiting when
180                releasing the mutex, another thread can run through, take
181                the GIL and drop it again, and reset the condition
182                before we even had a chance to wait for it. */
183             COND_WAIT(_PyRuntime.ceval.gil.switch_cond,
184                       _PyRuntime.ceval.gil.switch_mutex);
185     }
186         MUTEX_UNLOCK(_PyRuntime.ceval.gil.switch_mutex);
187     }
188 #endif
189 }
190 
take_gil(PyThreadState * tstate)191 static void take_gil(PyThreadState *tstate)
192 {
193     int err;
194     if (tstate == NULL)
195         Py_FatalError("take_gil: NULL tstate");
196 
197     err = errno;
198     MUTEX_LOCK(_PyRuntime.ceval.gil.mutex);
199 
200     if (!_Py_atomic_load_relaxed(&_PyRuntime.ceval.gil.locked))
201         goto _ready;
202 
203     while (_Py_atomic_load_relaxed(&_PyRuntime.ceval.gil.locked)) {
204         int timed_out = 0;
205         unsigned long saved_switchnum;
206 
207         saved_switchnum = _PyRuntime.ceval.gil.switch_number;
208         COND_TIMED_WAIT(_PyRuntime.ceval.gil.cond, _PyRuntime.ceval.gil.mutex,
209                         INTERVAL, timed_out);
210         /* If we timed out and no switch occurred in the meantime, it is time
211            to ask the GIL-holding thread to drop it. */
212         if (timed_out &&
213             _Py_atomic_load_relaxed(&_PyRuntime.ceval.gil.locked) &&
214             _PyRuntime.ceval.gil.switch_number == saved_switchnum) {
215             SET_GIL_DROP_REQUEST();
216         }
217     }
218 _ready:
219 #ifdef FORCE_SWITCHING
220     /* This mutex must be taken before modifying
221        _PyRuntime.ceval.gil.last_holder (see drop_gil()). */
222     MUTEX_LOCK(_PyRuntime.ceval.gil.switch_mutex);
223 #endif
224     /* We now hold the GIL */
225     _Py_atomic_store_relaxed(&_PyRuntime.ceval.gil.locked, 1);
226     _Py_ANNOTATE_RWLOCK_ACQUIRED(&_PyRuntime.ceval.gil.locked, /*is_write=*/1);
227 
228     if (tstate != (PyThreadState*)_Py_atomic_load_relaxed(
229                     &_PyRuntime.ceval.gil.last_holder))
230     {
231         _Py_atomic_store_relaxed(&_PyRuntime.ceval.gil.last_holder,
232                                  (uintptr_t)tstate);
233         ++_PyRuntime.ceval.gil.switch_number;
234     }
235 
236 #ifdef FORCE_SWITCHING
237     COND_SIGNAL(_PyRuntime.ceval.gil.switch_cond);
238     MUTEX_UNLOCK(_PyRuntime.ceval.gil.switch_mutex);
239 #endif
240     if (_Py_atomic_load_relaxed(&_PyRuntime.ceval.gil_drop_request)) {
241         RESET_GIL_DROP_REQUEST();
242     }
243     if (tstate->async_exc != NULL) {
244         _PyEval_SignalAsyncExc();
245     }
246 
247     MUTEX_UNLOCK(_PyRuntime.ceval.gil.mutex);
248     errno = err;
249 }
250 
_PyEval_SetSwitchInterval(unsigned long microseconds)251 void _PyEval_SetSwitchInterval(unsigned long microseconds)
252 {
253     _PyRuntime.ceval.gil.interval = microseconds;
254 }
255 
_PyEval_GetSwitchInterval()256 unsigned long _PyEval_GetSwitchInterval()
257 {
258     return _PyRuntime.ceval.gil.interval;
259 }
260