/*

 * Implementation of the Global Interpreter Lock (GIL).

 */

#include <stdlib.h>

#include <errno.h>

#include "pycore_atomic.h"

/*

   Notes about the implementation:

   - The GIL is just a boolean variable (locked) whose access is protected

     by a mutex (gil_mutex), and whose changes are signalled by a condition

     variable (gil_cond). gil_mutex is taken for short periods of time,

     and therefore mostly uncontended.

   - In the GIL-holding thread, the main loop (PyEval_EvalFrameEx) must be

     able to release the GIL on demand by another thread. A volatile boolean

     variable (gil_drop_request) is used for that purpose, which is checked

     at every turn of the eval loop. That variable is set after a wait of

     `interval` microseconds on `gil_cond` has timed out.

      [Actually, another volatile boolean variable (eval_breaker) is used

       which ORs several conditions into one. Volatile booleans are

       sufficient as inter-thread signalling means since Python is run

       on cache-coherent architectures only.]

   - A thread wanting to take the GIL will first let pass a given amount of

     time (`interval` microseconds) before setting gil_drop_request. This

     encourages a defined switching period, but doesn't enforce it since

     opcodes can take an arbitrary time to execute.

     The `interval` value is available for the user to read and modify

     using the Python API `sys.{get,set}switchinterval()`.

   - When a thread releases the GIL and gil_drop_request is set, that thread

     ensures that another GIL-awaiting thread gets scheduled.

     It does so by waiting on a condition variable (switch_cond) until

     the value of last_holder is changed to something else than its

     own thread state pointer, indicating that another thread was able to

     take the GIL.

     This is meant to prohibit the latency-adverse behaviour on multi-core

     machines where one thread would speculatively release the GIL, but still

     run and end up being the first to re-acquire it, making the "timeslices"

     much longer than expected.

     (Note: this mechanism is enabled with FORCE_SWITCHING above)

*/

#include "condvar.h"

#define MUTEX_INIT(mut) \

    if (PyMUTEX_INIT(&(mut))) { \

        
Py_FatalError0
("PyMUTEX_INIT(" #mut ") failed"); };

#define MUTEX_FINI(mut) \

    if (PyMUTEX_FINI(&(mut))) { \

        Py_FatalError("PyMUTEX_FINI(" #mut ") failed"); };

#define MUTEX_LOCK(mut) \

    if (PyMUTEX_LOCK(&(mut))) { \

        
Py_FatalError0
("PyMUTEX_LOCK(" #mut ") failed"); };

#define MUTEX_UNLOCK(mut) \

    if (PyMUTEX_UNLOCK(&(mut))) { \

        
Py_FatalError0
("PyMUTEX_UNLOCK(" #mut ") failed"); };

#define COND_INIT(cond) \

    if (PyCOND_INIT(&(cond))) { \

        
Py_FatalError0
("PyCOND_INIT(" #cond ") failed"); };

#define COND_FINI(cond) \

    if (PyCOND_FINI(&(cond))) { \

        Py_FatalError("PyCOND_FINI(" #cond ") failed"); };

#define COND_SIGNAL(cond) \

    if (PyCOND_SIGNAL(&(cond))) { \

        
Py_FatalError0
("PyCOND_SIGNAL(" #cond ") failed"); };

#define COND_WAIT(cond, mut) \

    if (PyCOND_WAIT(&(cond), &(mut))) { \

        
Py_FatalError0
("PyCOND_WAIT(" #cond ") failed"); };

#define COND_TIMED_WAIT(cond, mut, microseconds, timeout_result) \

    { \

        int r = PyCOND_TIMEDWAIT(&(cond), &(mut), (microseconds)); \

        if (r < 0) \

            
Py_FatalError0
("PyCOND_WAIT(" #cond ") failed"); \

        if (r) /* 1 == timeout, 2 == impl. can't say, so assume timeout */ \

            
timeout_result = 1123k
; \

        else \

            
timeout_result = 0171k
; \

    } \

#define DEFAULT_INTERVAL 5000

static void _gil_initialize(struct _gil_runtime_state *gil)

{

    _Py_atomic_int uninitialized = {-1};

    gil->locked = uninitialized;

    gil->interval = DEFAULT_INTERVAL;

}

static int gil_created(struct _gil_runtime_state *gil)

{

    return (_Py_atomic_load_explicit(&gil->locked, _Py_memory_order_acquire) >= 0);

}

static void create_gil(struct _gil_runtime_state *gil)

{

    MUTEX_INIT(gil->mutex);

#ifdef FORCE_SWITCHING

    MUTEX_INIT(gil->switch_mutex);

#endif

    COND_INIT(gil->cond);

#ifdef FORCE_SWITCHING

    COND_INIT(gil->switch_cond);

#endif

    _Py_atomic_store_relaxed(&gil->last_holder, 0);

    _Py_ANNOTATE_RWLOCK_CREATE(&gil->locked);

    _Py_atomic_store_explicit(&gil->locked, 0, _Py_memory_order_release);

}

static void destroy_gil(struct _gil_runtime_state *gil)

{

    /* some pthread-like implementations tie the mutex to the cond

     * and must have the cond destroyed first.

     */

    COND_FINI(gil->cond);

    MUTEX_FINI(gil->mutex);

#ifdef FORCE_SWITCHING

    COND_FINI(gil->switch_cond);

    MUTEX_FINI(gil->switch_mutex);

#endif

    _Py_atomic_store_explicit(&gil->locked, -1,

                              _Py_memory_order_release);

    _Py_ANNOTATE_RWLOCK_DESTROY(&gil->locked);

}

static void recreate_gil(struct _gil_runtime_state *gil)

{

    _Py_ANNOTATE_RWLOCK_DESTROY(&gil->locked);

    /* XXX should we destroy the old OS resources here? */

    create_gil(gil);

}

static void

drop_gil(struct _ceval_runtime_state *ceval, struct _ceval_state *ceval2,

         PyThreadState *tstate)

{

    struct _gil_runtime_state *gil = &ceval->gil;

    if (!_Py_atomic_load_relaxed(&gil->locked)) {

  Branch (148:9): [True: 0, False: 7.77M]

        Py_FatalError("drop_gil: GIL is not locked");

    }

    /* tstate is allowed to be NULL (early interpreter init) */

    if (tstate != NULL) {

  Branch (153:9): [True: 7.77M, False: 0]

        /* Sub-interpreter support: threads might have been switched

           under our feet using PyThreadState_Swap(). Fix the GIL last

           holder variable so that our heuristics work. */

        _Py_atomic_store_relaxed(&gil->last_holder, (uintptr_t)tstate);

    }

    MUTEX_LOCK(gil->mutex);

    _Py_ANNOTATE_RWLOCK_RELEASED(&gil->locked, /*is_write=*/1);

    _Py_atomic_store_relaxed(&gil->locked, 0);

    COND_SIGNAL(gil->cond);

    MUTEX_UNLOCK(gil->mutex);

#ifdef FORCE_SWITCHING

    if (_Py_atomic_load_relaxed(&ceval2->gil_drop_request) && 
tstate != NULL39.5k
) {

  Branch (167:63): [True: 39.5k, False: 0]

        MUTEX_LOCK(gil->switch_mutex);

        /* Not switched yet => wait */

        if (((PyThreadState*)_Py_atomic_load_relaxed(&gil->last_holder)) == tstate)

  Branch (170:13): [True: 22.3k, False: 17.2k]

        {

            assert(is_tstate_valid(tstate));

            RESET_GIL_DROP_REQUEST(tstate->interp);

            /* NOTE: if COND_WAIT does not atomically start waiting when

               releasing the mutex, another thread can run through, take

               the GIL and drop it again, and reset the condition

               before we even had a chance to wait for it. */

            COND_WAIT(gil->switch_cond, gil->switch_mutex);

        }

        
MUTEX_UNLOCK39.5k
(gil->switch_mutex);

    }

#endif

}

/* Check if a Python thread must exit immediately, rather than taking the GIL

   if Py_Finalize() has been called.

   When this function is called by a daemon thread after Py_Finalize() has been

   called, the GIL does no longer exist.

   tstate must be non-NULL. */

static inline int

tstate_must_exit(PyThreadState *tstate)

{

    /* bpo-39877: Access _PyRuntime directly rather than using

       tstate->interp->runtime to support calls from Python daemon threads.

       After Py_Finalize() has been called, tstate can be a dangling pointer:

       point to PyThreadState freed memory. */

    PyThreadState *finalizing = _PyRuntimeState_GetFinalizing(&_PyRuntime);

    return (finalizing != NULL && 
finalizing != tstate1.10k
);

  Branch (201:13): [True: 1.10k, False: 15.6M]
  Branch (201:35): [True: 0, False: 1.10k]

}

/* Take the GIL.

   The function saves errno at entry and restores its value at exit.

   tstate must be non-NULL. */

static void

take_gil(PyThreadState *tstate)

{

    int err = errno;

    assert(tstate != NULL);

    if (tstate_must_exit(tstate)) {

  Branch (217:9): [True: 0, False: 7.77M]

        /* bpo-39877: If Py_Finalize() has been called and tstate is not the

           thread which called Py_Finalize(), exit immediately the thread.

           This code path can be reached by a daemon thread after Py_Finalize()

           completes. In this case, tstate is a dangling pointer: points to

           PyThreadState freed memory. */

        PyThread_exit_thread();

    }

    assert(is_tstate_valid(tstate));

    PyInterpreterState *interp = tstate->interp;

    struct _ceval_runtime_state *ceval = &interp->runtime->ceval;

    struct _ceval_state *ceval2 = &interp->ceval;

    struct _gil_runtime_state *gil = &ceval->gil;

    /* Check that _PyEval_InitThreads() was called to create the lock */

    assert(gil_created(gil));

    MUTEX_LOCK(gil->mutex);

    if (!_Py_atomic_load_relaxed(&gil->locked)) {

  Branch (238:9): [True: 7.64M, False: 122k]

        goto _ready;

    }

    
while (_Py_atomic_load_relaxed(&gil->locked))122k
 {

        unsigned long saved_switchnum = gil->switch_number;

        unsigned long interval = (gil->interval >= 1 ? gil->interval : 
10
);

  Branch (245:35): [True: 294k, False: 0]

        int timed_out = 0;

        COND_TIMED_WAIT(gil->cond, gil->mutex, interval, timed_out);

        /* If we timed out and no switch occurred in the meantime, it is time

           to ask the GIL-holding thread to drop it. */

        if (timed_out &&

  Branch (251:13): [True: 123k, False: 171k]

            _Py_atomic_load_relaxed(&gil->locked) &&

            
gil->switch_number == saved_switchnum123k
)

  Branch (253:13): [True: 120k, False: 2.06k]

        {

            if (tstate_must_exit(tstate)) {

  Branch (255:17): [True: 0, False: 120k]

                MUTEX_UNLOCK(gil->mutex);

                PyThread_exit_thread();

            }

            assert(is_tstate_valid(tstate));

            SET_GIL_DROP_REQUEST(interp);

        }

    }

_ready:

#ifdef FORCE_SWITCHING

    /* This mutex must be taken before modifying gil->last_holder:

       see drop_gil(). */

    MUTEX_LOCK(gil->switch_mutex);

#endif

    /* We now hold the GIL */

    _Py_atomic_store_relaxed(&gil->locked, 1);

    _Py_ANNOTATE_RWLOCK_ACQUIRED(&gil->locked, /*is_write=*/1);

    if (tstate != (PyThreadState*)_Py_atomic_load_relaxed(&gil->last_holder)) {

  Branch (275:9): [True: 185k, False: 7.58M]

        _Py_atomic_store_relaxed(&gil->last_holder, (uintptr_t)tstate);

        ++gil->switch_number;

    }

#ifdef FORCE_SWITCHING

    COND_SIGNAL(gil->switch_cond);

    MUTEX_UNLOCK(gil->switch_mutex);

#endif

    if (tstate_must_exit(tstate)) {

  Branch (285:9): [True: 0, False: 7.77M]

        /* bpo-36475: If Py_Finalize() has been called and tstate is not

           the thread which called Py_Finalize(), exit immediately the

           thread.

           This code path can be reached by a daemon thread which was waiting

           in take_gil() while the main thread called

           wait_for_thread_shutdown() from Py_Finalize(). */

        MUTEX_UNLOCK(gil->mutex);

        drop_gil(ceval, ceval2, tstate);

        PyThread_exit_thread();

    }

    assert(is_tstate_valid(tstate));

    if (_Py_atomic_load_relaxed(&ceval2->gil_drop_request)) {

        RESET_GIL_DROP_REQUEST(interp);

    }

    else {

        /* bpo-40010: eval_breaker should be recomputed to be set to 1 if there

           is a pending signal: signal received by another thread which cannot

           handle signals.

           Note: RESET_GIL_DROP_REQUEST() calls COMPUTE_EVAL_BREAKER(). */

        COMPUTE_EVAL_BREAKER(interp, ceval, ceval2);

    }

    /* Don't access tstate if the thread must exit */

    if (tstate->async_exc != NULL) {

  Branch (312:9): [True: 1, False: 7.77M]

        _PyEval_SignalAsyncExc(tstate->interp);

    }

    MUTEX_UNLOCK(gil->mutex);

    errno = err;

}

void _PyEval_SetSwitchInterval(unsigned long microseconds)

{

    struct _gil_runtime_state *gil = &_PyRuntime.ceval.gil;

    gil->interval = microseconds;

}

unsigned long _PyEval_GetSwitchInterval()

{

    struct _gil_runtime_state *gil = &_PyRuntime.ceval.gil;

    return gil->interval;

}