Line data Source code
1 : /*
2 : * Implementation of the Global Interpreter Lock (GIL).
3 : */
4 :
5 : #include <stdlib.h>
6 : #include <errno.h>
7 :
8 : #include "pycore_atomic.h"
9 :
10 :
11 : /*
12 : Notes about the implementation:
13 :
14 : - The GIL is just a boolean variable (locked) whose access is protected
15 : by a mutex (gil_mutex), and whose changes are signalled by a condition
16 : variable (gil_cond). gil_mutex is taken for short periods of time,
17 : and therefore mostly uncontended.
18 :
19 : - In the GIL-holding thread, the main loop (PyEval_EvalFrameEx) must be
20 : able to release the GIL on demand by another thread. A volatile boolean
21 : variable (gil_drop_request) is used for that purpose, which is checked
22 : at every turn of the eval loop. That variable is set after a wait of
23 : `interval` microseconds on `gil_cond` has timed out.
24 :
25 : [Actually, another volatile boolean variable (eval_breaker) is used
26 : which ORs several conditions into one. Volatile booleans are
27 : sufficient as inter-thread signalling means since Python is run
28 : on cache-coherent architectures only.]
29 :
30 : - A thread wanting to take the GIL will first let pass a given amount of
31 : time (`interval` microseconds) before setting gil_drop_request. This
32 : encourages a defined switching period, but doesn't enforce it since
33 : opcodes can take an arbitrary time to execute.
34 :
35 : The `interval` value is available for the user to read and modify
36 : using the Python API `sys.{get,set}switchinterval()`.
37 :
38 : - When a thread releases the GIL and gil_drop_request is set, that thread
39 : ensures that another GIL-awaiting thread gets scheduled.
40 : It does so by waiting on a condition variable (switch_cond) until
41 : the value of last_holder is changed to something else than its
42 : own thread state pointer, indicating that another thread was able to
43 : take the GIL.
44 :
45 : This is meant to prohibit the latency-adverse behaviour on multi-core
46 : machines where one thread would speculatively release the GIL, but still
47 : run and end up being the first to re-acquire it, making the "timeslices"
48 : much longer than expected.
49 : (Note: this mechanism is enabled with FORCE_SWITCHING above)
50 : */
51 :
52 : #include "condvar.h"
53 :
54 : #define MUTEX_INIT(mut) \
55 : if (PyMUTEX_INIT(&(mut))) { \
56 : Py_FatalError("PyMUTEX_INIT(" #mut ") failed"); };
57 : #define MUTEX_FINI(mut) \
58 : if (PyMUTEX_FINI(&(mut))) { \
59 : Py_FatalError("PyMUTEX_FINI(" #mut ") failed"); };
60 : #define MUTEX_LOCK(mut) \
61 : if (PyMUTEX_LOCK(&(mut))) { \
62 : Py_FatalError("PyMUTEX_LOCK(" #mut ") failed"); };
63 : #define MUTEX_UNLOCK(mut) \
64 : if (PyMUTEX_UNLOCK(&(mut))) { \
65 : Py_FatalError("PyMUTEX_UNLOCK(" #mut ") failed"); };
66 :
67 : #define COND_INIT(cond) \
68 : if (PyCOND_INIT(&(cond))) { \
69 : Py_FatalError("PyCOND_INIT(" #cond ") failed"); };
70 : #define COND_FINI(cond) \
71 : if (PyCOND_FINI(&(cond))) { \
72 : Py_FatalError("PyCOND_FINI(" #cond ") failed"); };
73 : #define COND_SIGNAL(cond) \
74 : if (PyCOND_SIGNAL(&(cond))) { \
75 : Py_FatalError("PyCOND_SIGNAL(" #cond ") failed"); };
76 : #define COND_WAIT(cond, mut) \
77 : if (PyCOND_WAIT(&(cond), &(mut))) { \
78 : Py_FatalError("PyCOND_WAIT(" #cond ") failed"); };
79 : #define COND_TIMED_WAIT(cond, mut, microseconds, timeout_result) \
80 : { \
81 : int r = PyCOND_TIMEDWAIT(&(cond), &(mut), (microseconds)); \
82 : if (r < 0) \
83 : Py_FatalError("PyCOND_WAIT(" #cond ") failed"); \
84 : if (r) /* 1 == timeout, 2 == impl. can't say, so assume timeout */ \
85 : timeout_result = 1; \
86 : else \
87 : timeout_result = 0; \
88 : } \
89 :
90 :
91 : #define DEFAULT_INTERVAL 5000
92 :
93 2984 : static void _gil_initialize(struct _gil_runtime_state *gil)
94 : {
95 2984 : _Py_atomic_int uninitialized = {-1};
96 2984 : gil->locked = uninitialized;
97 2984 : gil->interval = DEFAULT_INTERVAL;
98 2984 : }
99 :
100 20047900 : static int gil_created(struct _gil_runtime_state *gil)
101 : {
102 20047900 : return (_Py_atomic_load_explicit(&gil->locked, _Py_memory_order_acquire) >= 0);
103 : }
104 :
105 2971 : static void create_gil(struct _gil_runtime_state *gil)
106 : {
107 2971 : MUTEX_INIT(gil->mutex);
108 : #ifdef FORCE_SWITCHING
109 2971 : MUTEX_INIT(gil->switch_mutex);
110 : #endif
111 2971 : COND_INIT(gil->cond);
112 : #ifdef FORCE_SWITCHING
113 2971 : COND_INIT(gil->switch_cond);
114 : #endif
115 2971 : _Py_atomic_store_relaxed(&gil->last_holder, 0);
116 : _Py_ANNOTATE_RWLOCK_CREATE(&gil->locked);
117 2971 : _Py_atomic_store_explicit(&gil->locked, 0, _Py_memory_order_release);
118 2971 : }
119 :
120 0 : static void destroy_gil(struct _gil_runtime_state *gil)
121 : {
122 : /* some pthread-like implementations tie the mutex to the cond
123 : * and must have the cond destroyed first.
124 : */
125 0 : COND_FINI(gil->cond);
126 0 : MUTEX_FINI(gil->mutex);
127 : #ifdef FORCE_SWITCHING
128 0 : COND_FINI(gil->switch_cond);
129 0 : MUTEX_FINI(gil->switch_mutex);
130 : #endif
131 0 : _Py_atomic_store_explicit(&gil->locked, -1,
132 : _Py_memory_order_release);
133 : _Py_ANNOTATE_RWLOCK_DESTROY(&gil->locked);
134 0 : }
135 :
136 8 : static void recreate_gil(struct _gil_runtime_state *gil)
137 : {
138 : _Py_ANNOTATE_RWLOCK_DESTROY(&gil->locked);
139 : /* XXX should we destroy the old OS resources here? */
140 8 : create_gil(gil);
141 8 : }
142 :
143 : static void
144 10031700 : drop_gil(struct _ceval_runtime_state *ceval, struct _ceval_state *ceval2,
145 : PyThreadState *tstate)
146 : {
147 10031700 : struct _gil_runtime_state *gil = &ceval->gil;
148 10031700 : if (!_Py_atomic_load_relaxed(&gil->locked)) {
149 0 : Py_FatalError("drop_gil: GIL is not locked");
150 : }
151 :
152 : /* tstate is allowed to be NULL (early interpreter init) */
153 10031700 : if (tstate != NULL) {
154 : /* Sub-interpreter support: threads might have been switched
155 : under our feet using PyThreadState_Swap(). Fix the GIL last
156 : holder variable so that our heuristics work. */
157 10031700 : _Py_atomic_store_relaxed(&gil->last_holder, (uintptr_t)tstate);
158 : }
159 :
160 10031700 : MUTEX_LOCK(gil->mutex);
161 : _Py_ANNOTATE_RWLOCK_RELEASED(&gil->locked, /*is_write=*/1);
162 10031700 : _Py_atomic_store_relaxed(&gil->locked, 0);
163 10031700 : COND_SIGNAL(gil->cond);
164 10031700 : MUTEX_UNLOCK(gil->mutex);
165 :
166 : #ifdef FORCE_SWITCHING
167 10031700 : if (_Py_atomic_load_relaxed(&ceval2->gil_drop_request) && tstate != NULL) {
168 13170 : MUTEX_LOCK(gil->switch_mutex);
169 : /* Not switched yet => wait */
170 13170 : if (((PyThreadState*)_Py_atomic_load_relaxed(&gil->last_holder)) == tstate)
171 : {
172 12291 : assert(is_tstate_valid(tstate));
173 12291 : RESET_GIL_DROP_REQUEST(tstate->interp);
174 : /* NOTE: if COND_WAIT does not atomically start waiting when
175 : releasing the mutex, another thread can run through, take
176 : the GIL and drop it again, and reset the condition
177 : before we even had a chance to wait for it. */
178 12291 : COND_WAIT(gil->switch_cond, gil->switch_mutex);
179 : }
180 13170 : MUTEX_UNLOCK(gil->switch_mutex);
181 : }
182 : #endif
183 10031700 : }
184 :
185 :
186 : /* Check if a Python thread must exit immediately, rather than taking the GIL
187 : if Py_Finalize() has been called.
188 :
189 : When this function is called by a daemon thread after Py_Finalize() has been
190 : called, the GIL does no longer exist.
191 :
192 : tstate must be non-NULL. */
193 : static inline int
194 20191700 : tstate_must_exit(PyThreadState *tstate)
195 : {
196 : /* bpo-39877: Access _PyRuntime directly rather than using
197 : tstate->interp->runtime to support calls from Python daemon threads.
198 : After Py_Finalize() has been called, tstate can be a dangling pointer:
199 : point to PyThreadState freed memory. */
200 20191700 : PyThreadState *finalizing = _PyRuntimeState_GetFinalizing(&_PyRuntime);
201 20191700 : return (finalizing != NULL && finalizing != tstate);
202 : }
203 :
204 :
205 : /* Take the GIL.
206 :
207 : The function saves errno at entry and restores its value at exit.
208 :
209 : tstate must be non-NULL. */
210 : static void
211 10034800 : take_gil(PyThreadState *tstate)
212 : {
213 10034800 : int err = errno;
214 :
215 10034800 : assert(tstate != NULL);
216 :
217 10034800 : if (tstate_must_exit(tstate)) {
218 : /* bpo-39877: If Py_Finalize() has been called and tstate is not the
219 : thread which called Py_Finalize(), exit immediately the thread.
220 :
221 : This code path can be reached by a daemon thread after Py_Finalize()
222 : completes. In this case, tstate is a dangling pointer: points to
223 : PyThreadState freed memory. */
224 37 : PyThread_exit_thread();
225 : }
226 :
227 10034700 : assert(is_tstate_valid(tstate));
228 10034700 : PyInterpreterState *interp = tstate->interp;
229 10034700 : struct _ceval_runtime_state *ceval = &interp->runtime->ceval;
230 10034700 : struct _ceval_state *ceval2 = &interp->ceval;
231 10034700 : struct _gil_runtime_state *gil = &ceval->gil;
232 :
233 : /* Check that _PyEval_InitThreads() was called to create the lock */
234 10034700 : assert(gil_created(gil));
235 :
236 10034700 : MUTEX_LOCK(gil->mutex);
237 :
238 10034700 : if (!_Py_atomic_load_relaxed(&gil->locked)) {
239 9932020 : goto _ready;
240 : }
241 :
242 422422 : while (_Py_atomic_load_relaxed(&gil->locked)) {
243 319760 : unsigned long saved_switchnum = gil->switch_number;
244 :
245 319760 : unsigned long interval = (gil->interval >= 1 ? gil->interval : 1);
246 319760 : int timed_out = 0;
247 319760 : COND_TIMED_WAIT(gil->cond, gil->mutex, interval, timed_out);
248 :
249 : /* If we timed out and no switch occurred in the meantime, it is time
250 : to ask the GIL-holding thread to drop it. */
251 319758 : if (timed_out &&
252 124778 : _Py_atomic_load_relaxed(&gil->locked) &&
253 124424 : gil->switch_number == saved_switchnum)
254 : {
255 122229 : if (tstate_must_exit(tstate)) {
256 45 : MUTEX_UNLOCK(gil->mutex);
257 45 : PyThread_exit_thread();
258 : }
259 122184 : assert(is_tstate_valid(tstate));
260 :
261 122184 : SET_GIL_DROP_REQUEST(interp);
262 : }
263 : }
264 :
265 102662 : _ready:
266 : #ifdef FORCE_SWITCHING
267 : /* This mutex must be taken before modifying gil->last_holder:
268 : see drop_gil(). */
269 10034700 : MUTEX_LOCK(gil->switch_mutex);
270 : #endif
271 : /* We now hold the GIL */
272 10034700 : _Py_atomic_store_relaxed(&gil->locked, 1);
273 : _Py_ANNOTATE_RWLOCK_ACQUIRED(&gil->locked, /*is_write=*/1);
274 :
275 10034700 : if (tstate != (PyThreadState*)_Py_atomic_load_relaxed(&gil->last_holder)) {
276 182024 : _Py_atomic_store_relaxed(&gil->last_holder, (uintptr_t)tstate);
277 182024 : ++gil->switch_number;
278 : }
279 :
280 : #ifdef FORCE_SWITCHING
281 10034700 : COND_SIGNAL(gil->switch_cond);
282 10034700 : MUTEX_UNLOCK(gil->switch_mutex);
283 : #endif
284 :
285 10034700 : if (tstate_must_exit(tstate)) {
286 : /* bpo-36475: If Py_Finalize() has been called and tstate is not
287 : the thread which called Py_Finalize(), exit immediately the
288 : thread.
289 :
290 : This code path can be reached by a daemon thread which was waiting
291 : in take_gil() while the main thread called
292 : wait_for_thread_shutdown() from Py_Finalize(). */
293 0 : MUTEX_UNLOCK(gil->mutex);
294 0 : drop_gil(ceval, ceval2, tstate);
295 0 : PyThread_exit_thread();
296 : }
297 10034700 : assert(is_tstate_valid(tstate));
298 :
299 10034700 : if (_Py_atomic_load_relaxed(&ceval2->gil_drop_request)) {
300 10767 : RESET_GIL_DROP_REQUEST(interp);
301 : }
302 : else {
303 : /* bpo-40010: eval_breaker should be recomputed to be set to 1 if there
304 : is a pending signal: signal received by another thread which cannot
305 : handle signals.
306 :
307 : Note: RESET_GIL_DROP_REQUEST() calls COMPUTE_EVAL_BREAKER(). */
308 10023900 : COMPUTE_EVAL_BREAKER(interp, ceval, ceval2);
309 : }
310 :
311 : /* Don't access tstate if the thread must exit */
312 10034700 : if (tstate->async_exc != NULL) {
313 1 : _PyEval_SignalAsyncExc(tstate->interp);
314 : }
315 :
316 10034700 : MUTEX_UNLOCK(gil->mutex);
317 :
318 10034700 : errno = err;
319 10034700 : }
320 :
321 133 : void _PyEval_SetSwitchInterval(unsigned long microseconds)
322 : {
323 133 : struct _gil_runtime_state *gil = &_PyRuntime.ceval.gil;
324 133 : gil->interval = microseconds;
325 133 : }
326 :
327 20 : unsigned long _PyEval_GetSwitchInterval()
328 : {
329 20 : struct _gil_runtime_state *gil = &_PyRuntime.ceval.gil;
330 20 : return gil->interval;
331 : }
|
