Line data Source code
1 : /* Execute compiled code */
2 :
3 : /* XXX TO DO:
4 : XXX speed up searching for keywords by using a dictionary
5 : XXX document it!
6 : */
7 :
8 : #define _PY_INTERPRETER
9 :
10 : #include "Python.h"
11 : #include "pycore_abstract.h" // _PyIndex_Check()
12 : #include "pycore_call.h" // _PyObject_FastCallDictTstate()
13 : #include "pycore_ceval.h" // _PyEval_SignalAsyncExc()
14 : #include "pycore_code.h"
15 : #include "pycore_function.h"
16 : #include "pycore_initconfig.h" // _PyStatus_OK()
17 : #include "pycore_long.h" // _PyLong_GetZero()
18 : #include "pycore_object.h" // _PyObject_GC_TRACK()
19 : #include "pycore_moduleobject.h" // PyModuleObject
20 : #include "pycore_opcode.h" // EXTRA_CASES
21 : #include "pycore_pyerrors.h" // _PyErr_Fetch()
22 : #include "pycore_pylifecycle.h" // _PyErr_Print()
23 : #include "pycore_pymem.h" // _PyMem_IsPtrFreed()
24 : #include "pycore_pystate.h" // _PyInterpreterState_GET()
25 : #include "pycore_range.h" // _PyRangeIterObject
26 : #include "pycore_sliceobject.h" // _PyBuildSlice_ConsumeRefs
27 : #include "pycore_sysmodule.h" // _PySys_Audit()
28 : #include "pycore_tuple.h" // _PyTuple_ITEMS()
29 : #include "pycore_emscripten_signal.h" // _Py_CHECK_EMSCRIPTEN_SIGNALS
30 :
31 : #include "pycore_dict.h"
32 : #include "dictobject.h"
33 : #include "pycore_frame.h"
34 : #include "opcode.h"
35 : #include "pydtrace.h"
36 : #include "setobject.h"
37 : #include "structmember.h" // struct PyMemberDef, T_OFFSET_EX
38 :
39 : #include <ctype.h>
40 : #include <stdbool.h>
41 :
42 : #ifdef Py_DEBUG
43 : /* For debugging the interpreter: */
44 : # define LLTRACE 1 /* Low-level trace feature */
45 : #endif
46 :
47 : #if !defined(Py_BUILD_CORE)
48 : # error "ceval.c must be build with Py_BUILD_CORE define for best performance"
49 : #endif
50 :
51 : #ifndef Py_DEBUG
52 : // GH-89279: The MSVC compiler does not inline these static inline functions
53 : // in PGO build in _PyEval_EvalFrameDefault(), because this function is over
54 : // the limit of PGO, and that limit cannot be configured.
55 : // Define them as macros to make sure that they are always inlined by the
56 : // preprocessor.
57 :
58 : #undef Py_DECREF
59 : #define Py_DECREF(arg) \
60 : do { \
61 : _Py_DECREF_STAT_INC(); \
62 : PyObject *op = _PyObject_CAST(arg); \
63 : if (--op->ob_refcnt == 0) { \
64 : destructor dealloc = Py_TYPE(op)->tp_dealloc; \
65 : (*dealloc)(op); \
66 : } \
67 : } while (0)
68 :
69 : #undef Py_XDECREF
70 : #define Py_XDECREF(arg) \
71 : do { \
72 : PyObject *xop = _PyObject_CAST(arg); \
73 : if (xop != NULL) { \
74 : Py_DECREF(xop); \
75 : } \
76 : } while (0)
77 :
78 : #undef Py_IS_TYPE
79 : #define Py_IS_TYPE(ob, type) \
80 : (_PyObject_CAST(ob)->ob_type == (type))
81 :
82 : #undef _Py_DECREF_SPECIALIZED
83 : #define _Py_DECREF_SPECIALIZED(arg, dealloc) \
84 : do { \
85 : _Py_DECREF_STAT_INC(); \
86 : PyObject *op = _PyObject_CAST(arg); \
87 : if (--op->ob_refcnt == 0) { \
88 : destructor d = (destructor)(dealloc); \
89 : d(op); \
90 : } \
91 : } while (0)
92 : #endif
93 :
94 : // GH-89279: Similar to above, force inlining by using a macro.
95 : #if defined(_MSC_VER) && SIZEOF_INT == 4
96 : #define _Py_atomic_load_relaxed_int32(ATOMIC_VAL) (assert(sizeof((ATOMIC_VAL)->_value) == 4), *((volatile int*)&((ATOMIC_VAL)->_value)))
97 : #else
98 : #define _Py_atomic_load_relaxed_int32(ATOMIC_VAL) _Py_atomic_load_relaxed(ATOMIC_VAL)
99 : #endif
100 :
101 :
102 : /* Forward declarations */
103 : static PyObject *trace_call_function(
104 : PyThreadState *tstate, PyObject *callable, PyObject **stack,
105 : Py_ssize_t oparg, PyObject *kwnames);
106 : static PyObject * do_call_core(
107 : PyThreadState *tstate, PyObject *func,
108 : PyObject *callargs, PyObject *kwdict, int use_tracing);
109 :
110 : #ifdef LLTRACE
111 : static void
112 72 : dump_stack(_PyInterpreterFrame *frame, PyObject **stack_pointer)
113 : {
114 72 : PyObject **stack_base = _PyFrame_Stackbase(frame);
115 : PyObject *type, *value, *traceback;
116 72 : PyErr_Fetch(&type, &value, &traceback);
117 72 : printf(" stack=[");
118 160 : for (PyObject **ptr = stack_base; ptr < stack_pointer; ptr++) {
119 88 : if (ptr != stack_base) {
120 34 : printf(", ");
121 : }
122 88 : if (PyObject_Print(*ptr, stdout, 0) != 0) {
123 0 : PyErr_Clear();
124 0 : printf("<%s object at %p>",
125 0 : Py_TYPE(*ptr)->tp_name, (void *)(*ptr));
126 : }
127 : }
128 72 : printf("]\n");
129 72 : fflush(stdout);
130 72 : PyErr_Restore(type, value, traceback);
131 72 : }
132 :
133 : static void
134 72 : lltrace_instruction(_PyInterpreterFrame *frame,
135 : PyObject **stack_pointer,
136 : _Py_CODEUNIT *next_instr)
137 : {
138 72 : dump_stack(frame, stack_pointer);
139 72 : int oparg = _Py_OPARG(*next_instr);
140 72 : int opcode = _Py_OPCODE(*next_instr);
141 72 : const char *opname = _PyOpcode_OpName[opcode];
142 72 : assert(opname != NULL);
143 72 : int offset = (int)(next_instr - _PyCode_CODE(frame->f_code));
144 72 : if (HAS_ARG(opcode)) {
145 55 : printf("%d: %s %d\n", offset * 2, opname, oparg);
146 : }
147 : else {
148 17 : printf("%d: %s\n", offset * 2, opname);
149 : }
150 72 : fflush(stdout);
151 72 : }
152 : static void
153 5 : lltrace_resume_frame(_PyInterpreterFrame *frame)
154 : {
155 5 : PyFunctionObject *f = frame->f_func;
156 5 : if (f == NULL) {
157 0 : printf("\nResuming frame.");
158 0 : return;
159 : }
160 : PyObject *type, *value, *traceback;
161 5 : PyErr_Fetch(&type, &value, &traceback);
162 5 : PyObject *name = f->func_qualname;
163 5 : if (name == NULL) {
164 0 : name = f->func_name;
165 : }
166 5 : printf("\nResuming frame");
167 5 : if (name) {
168 5 : printf(" for ");
169 5 : if (PyObject_Print(name, stdout, 0) < 0) {
170 0 : PyErr_Clear();
171 : }
172 : }
173 5 : if (f->func_module) {
174 2 : printf(" in module ");
175 2 : if (PyObject_Print(f->func_module, stdout, 0) < 0) {
176 0 : PyErr_Clear();
177 : }
178 : }
179 5 : printf("\n");
180 5 : fflush(stdout);
181 5 : PyErr_Restore(type, value, traceback);
182 : }
183 : #endif
184 : static int call_trace(Py_tracefunc, PyObject *,
185 : PyThreadState *, _PyInterpreterFrame *,
186 : int, PyObject *);
187 : static int call_trace_protected(Py_tracefunc, PyObject *,
188 : PyThreadState *, _PyInterpreterFrame *,
189 : int, PyObject *);
190 : static void call_exc_trace(Py_tracefunc, PyObject *,
191 : PyThreadState *, _PyInterpreterFrame *);
192 : static int maybe_call_line_trace(Py_tracefunc, PyObject *,
193 : PyThreadState *, _PyInterpreterFrame *, int);
194 : static void maybe_dtrace_line(_PyInterpreterFrame *, PyTraceInfo *, int);
195 : static void dtrace_function_entry(_PyInterpreterFrame *);
196 : static void dtrace_function_return(_PyInterpreterFrame *);
197 :
198 : static PyObject * import_name(PyThreadState *, _PyInterpreterFrame *,
199 : PyObject *, PyObject *, PyObject *);
200 : static PyObject * import_from(PyThreadState *, PyObject *, PyObject *);
201 : static int import_all_from(PyThreadState *, PyObject *, PyObject *);
202 : static void format_exc_check_arg(PyThreadState *, PyObject *, const char *, PyObject *);
203 : static void format_exc_unbound(PyThreadState *tstate, PyCodeObject *co, int oparg);
204 : static int check_args_iterable(PyThreadState *, PyObject *func, PyObject *vararg);
205 : static int check_except_type_valid(PyThreadState *tstate, PyObject* right);
206 : static int check_except_star_type_valid(PyThreadState *tstate, PyObject* right);
207 : static void format_kwargs_error(PyThreadState *, PyObject *func, PyObject *kwargs);
208 : static void format_awaitable_error(PyThreadState *, PyTypeObject *, int);
209 : static int get_exception_handler(PyCodeObject *, int, int*, int*, int*);
210 : static _PyInterpreterFrame *
211 : _PyEvalFramePushAndInit(PyThreadState *tstate, PyFunctionObject *func,
212 : PyObject *locals, PyObject* const* args,
213 : size_t argcount, PyObject *kwnames);
214 : static void
215 : _PyEvalFrameClearAndPop(PyThreadState *tstate, _PyInterpreterFrame *frame);
216 :
217 : #define NAME_ERROR_MSG \
218 : "name '%.200s' is not defined"
219 : #define UNBOUNDLOCAL_ERROR_MSG \
220 : "cannot access local variable '%s' where it is not associated with a value"
221 : #define UNBOUNDFREE_ERROR_MSG \
222 : "cannot access free variable '%s' where it is not associated with a" \
223 : " value in enclosing scope"
224 :
225 : #ifndef NDEBUG
226 : /* Ensure that tstate is valid: sanity check for PyEval_AcquireThread() and
227 : PyEval_RestoreThread(). Detect if tstate memory was freed. It can happen
228 : when a thread continues to run after Python finalization, especially
229 : daemon threads. */
230 : static int
231 20249800 : is_tstate_valid(PyThreadState *tstate)
232 : {
233 20249800 : assert(!_PyMem_IsPtrFreed(tstate));
234 20249800 : assert(!_PyMem_IsPtrFreed(tstate->interp));
235 20249800 : return 1;
236 : }
237 : #endif
238 :
239 :
240 : /* This can set eval_breaker to 0 even though gil_drop_request became
241 : 1. We believe this is all right because the eval loop will release
242 : the GIL eventually anyway. */
243 : static inline void
244 10110600 : COMPUTE_EVAL_BREAKER(PyInterpreterState *interp,
245 : struct _ceval_runtime_state *ceval,
246 : struct _ceval_state *ceval2)
247 : {
248 10110600 : _Py_atomic_store_relaxed(&ceval2->eval_breaker,
249 : _Py_atomic_load_relaxed_int32(&ceval2->gil_drop_request)
250 : | (_Py_atomic_load_relaxed_int32(&ceval->signals_pending)
251 : && _Py_ThreadCanHandleSignals(interp))
252 : | (_Py_atomic_load_relaxed_int32(&ceval2->pending.calls_to_do)
253 : && _Py_ThreadCanHandlePendingCalls())
254 : | ceval2->pending.async_exc);
255 10110600 : }
256 :
257 :
258 : static inline void
259 122184 : SET_GIL_DROP_REQUEST(PyInterpreterState *interp)
260 : {
261 122184 : struct _ceval_state *ceval2 = &interp->ceval;
262 122184 : _Py_atomic_store_relaxed(&ceval2->gil_drop_request, 1);
263 122184 : _Py_atomic_store_relaxed(&ceval2->eval_breaker, 1);
264 122184 : }
265 :
266 :
267 : static inline void
268 23058 : RESET_GIL_DROP_REQUEST(PyInterpreterState *interp)
269 : {
270 23058 : struct _ceval_runtime_state *ceval = &interp->runtime->ceval;
271 23058 : struct _ceval_state *ceval2 = &interp->ceval;
272 23058 : _Py_atomic_store_relaxed(&ceval2->gil_drop_request, 0);
273 23058 : COMPUTE_EVAL_BREAKER(interp, ceval, ceval2);
274 23058 : }
275 :
276 :
277 : static inline void
278 101 : SIGNAL_PENDING_CALLS(PyInterpreterState *interp)
279 : {
280 101 : struct _ceval_runtime_state *ceval = &interp->runtime->ceval;
281 101 : struct _ceval_state *ceval2 = &interp->ceval;
282 101 : _Py_atomic_store_relaxed(&ceval2->pending.calls_to_do, 1);
283 101 : COMPUTE_EVAL_BREAKER(interp, ceval, ceval2);
284 101 : }
285 :
286 :
287 : static inline void
288 486 : UNSIGNAL_PENDING_CALLS(PyInterpreterState *interp)
289 : {
290 486 : struct _ceval_runtime_state *ceval = &interp->runtime->ceval;
291 486 : struct _ceval_state *ceval2 = &interp->ceval;
292 486 : _Py_atomic_store_relaxed(&ceval2->pending.calls_to_do, 0);
293 486 : COMPUTE_EVAL_BREAKER(interp, ceval, ceval2);
294 486 : }
295 :
296 :
297 : static inline void
298 31467 : SIGNAL_PENDING_SIGNALS(PyInterpreterState *interp, int force)
299 : {
300 31467 : struct _ceval_runtime_state *ceval = &interp->runtime->ceval;
301 31467 : struct _ceval_state *ceval2 = &interp->ceval;
302 31467 : _Py_atomic_store_relaxed(&ceval->signals_pending, 1);
303 31467 : if (force) {
304 0 : _Py_atomic_store_relaxed(&ceval2->eval_breaker, 1);
305 : }
306 : else {
307 : /* eval_breaker is not set to 1 if thread_can_handle_signals() is false */
308 31467 : COMPUTE_EVAL_BREAKER(interp, ceval, ceval2);
309 : }
310 31467 : }
311 :
312 :
313 : static inline void
314 31528 : UNSIGNAL_PENDING_SIGNALS(PyInterpreterState *interp)
315 : {
316 31528 : struct _ceval_runtime_state *ceval = &interp->runtime->ceval;
317 31528 : struct _ceval_state *ceval2 = &interp->ceval;
318 31528 : _Py_atomic_store_relaxed(&ceval->signals_pending, 0);
319 31528 : COMPUTE_EVAL_BREAKER(interp, ceval, ceval2);
320 31528 : }
321 :
322 :
323 : static inline void
324 3 : SIGNAL_ASYNC_EXC(PyInterpreterState *interp)
325 : {
326 3 : struct _ceval_state *ceval2 = &interp->ceval;
327 3 : ceval2->pending.async_exc = 1;
328 3 : _Py_atomic_store_relaxed(&ceval2->eval_breaker, 1);
329 3 : }
330 :
331 :
332 : static inline void
333 2 : UNSIGNAL_ASYNC_EXC(PyInterpreterState *interp)
334 : {
335 2 : struct _ceval_runtime_state *ceval = &interp->runtime->ceval;
336 2 : struct _ceval_state *ceval2 = &interp->ceval;
337 2 : ceval2->pending.async_exc = 0;
338 2 : COMPUTE_EVAL_BREAKER(interp, ceval, ceval2);
339 2 : }
340 :
341 :
342 : #ifdef HAVE_ERRNO_H
343 : #include <errno.h>
344 : #endif
345 : #include "ceval_gil.h"
346 :
347 : void _Py_NO_RETURN
348 0 : _Py_FatalError_TstateNULL(const char *func)
349 : {
350 0 : _Py_FatalErrorFunc(func,
351 : "the function must be called with the GIL held, "
352 : "after Python initialization and before Python finalization, "
353 : "but the GIL is released (the current Python thread state is NULL)");
354 : }
355 :
356 : int
357 4099 : _PyEval_ThreadsInitialized(_PyRuntimeState *runtime)
358 : {
359 4099 : return gil_created(&runtime->ceval.gil);
360 : }
361 :
362 : int
363 0 : PyEval_ThreadsInitialized(void)
364 : {
365 0 : _PyRuntimeState *runtime = &_PyRuntime;
366 0 : return _PyEval_ThreadsInitialized(runtime);
367 : }
368 :
369 : PyStatus
370 3134 : _PyEval_InitGIL(PyThreadState *tstate)
371 : {
372 3134 : if (!_Py_IsMainInterpreter(tstate->interp)) {
373 : /* Currently, the GIL is shared by all interpreters,
374 : and only the main interpreter is responsible to create
375 : and destroy it. */
376 171 : return _PyStatus_OK();
377 : }
378 :
379 2963 : struct _gil_runtime_state *gil = &tstate->interp->runtime->ceval.gil;
380 2963 : assert(!gil_created(gil));
381 :
382 2963 : PyThread_init_thread();
383 2963 : create_gil(gil);
384 :
385 2963 : take_gil(tstate);
386 :
387 2963 : assert(gil_created(gil));
388 2963 : return _PyStatus_OK();
389 : }
390 :
391 : void
392 3134 : _PyEval_FiniGIL(PyInterpreterState *interp)
393 : {
394 3134 : if (!_Py_IsMainInterpreter(interp)) {
395 : /* Currently, the GIL is shared by all interpreters,
396 : and only the main interpreter is responsible to create
397 : and destroy it. */
398 171 : return;
399 : }
400 :
401 2963 : struct _gil_runtime_state *gil = &interp->runtime->ceval.gil;
402 2963 : if (!gil_created(gil)) {
403 : /* First Py_InitializeFromConfig() call: the GIL doesn't exist
404 : yet: do nothing. */
405 2963 : return;
406 : }
407 :
408 0 : destroy_gil(gil);
409 0 : assert(!gil_created(gil));
410 : }
411 :
412 : void
413 0 : PyEval_InitThreads(void)
414 : {
415 : /* Do nothing: kept for backward compatibility */
416 0 : }
417 :
418 : void
419 2952 : _PyEval_Fini(void)
420 : {
421 : #ifdef Py_STATS
422 : _Py_PrintSpecializationStats(1);
423 : #endif
424 2952 : }
425 :
426 : void
427 0 : PyEval_AcquireLock(void)
428 : {
429 0 : _PyRuntimeState *runtime = &_PyRuntime;
430 0 : PyThreadState *tstate = _PyRuntimeState_GetThreadState(runtime);
431 0 : _Py_EnsureTstateNotNULL(tstate);
432 :
433 0 : take_gil(tstate);
434 0 : }
435 :
436 : void
437 0 : PyEval_ReleaseLock(void)
438 : {
439 0 : _PyRuntimeState *runtime = &_PyRuntime;
440 0 : PyThreadState *tstate = _PyRuntimeState_GetThreadState(runtime);
441 : /* This function must succeed when the current thread state is NULL.
442 : We therefore avoid PyThreadState_Get() which dumps a fatal error
443 : in debug mode. */
444 0 : struct _ceval_runtime_state *ceval = &runtime->ceval;
445 0 : struct _ceval_state *ceval2 = &tstate->interp->ceval;
446 0 : drop_gil(ceval, ceval2, tstate);
447 0 : }
448 :
449 : void
450 8683 : _PyEval_ReleaseLock(PyThreadState *tstate)
451 : {
452 8683 : struct _ceval_runtime_state *ceval = &tstate->interp->runtime->ceval;
453 8683 : struct _ceval_state *ceval2 = &tstate->interp->ceval;
454 8683 : drop_gil(ceval, ceval2, tstate);
455 8683 : }
456 :
457 : void
458 8804 : PyEval_AcquireThread(PyThreadState *tstate)
459 : {
460 8804 : _Py_EnsureTstateNotNULL(tstate);
461 :
462 8804 : take_gil(tstate);
463 :
464 8804 : struct _gilstate_runtime_state *gilstate = &tstate->interp->runtime->gilstate;
465 8804 : if (_PyThreadState_Swap(gilstate, tstate) != NULL) {
466 0 : Py_FatalError("non-NULL old thread state");
467 : }
468 8804 : }
469 :
470 : void
471 12 : PyEval_ReleaseThread(PyThreadState *tstate)
472 : {
473 12 : assert(is_tstate_valid(tstate));
474 :
475 12 : _PyRuntimeState *runtime = tstate->interp->runtime;
476 12 : PyThreadState *new_tstate = _PyThreadState_Swap(&runtime->gilstate, NULL);
477 12 : if (new_tstate != tstate) {
478 0 : Py_FatalError("wrong thread state");
479 : }
480 12 : struct _ceval_runtime_state *ceval = &runtime->ceval;
481 12 : struct _ceval_state *ceval2 = &tstate->interp->ceval;
482 12 : drop_gil(ceval, ceval2, tstate);
483 12 : }
484 :
485 : #ifdef HAVE_FORK
486 : /* This function is called from PyOS_AfterFork_Child to destroy all threads
487 : which are not running in the child process, and clear internal locks
488 : which might be held by those threads. */
489 : PyStatus
490 8 : _PyEval_ReInitThreads(PyThreadState *tstate)
491 : {
492 8 : _PyRuntimeState *runtime = tstate->interp->runtime;
493 :
494 8 : struct _gil_runtime_state *gil = &runtime->ceval.gil;
495 8 : if (!gil_created(gil)) {
496 0 : return _PyStatus_OK();
497 : }
498 8 : recreate_gil(gil);
499 :
500 8 : take_gil(tstate);
501 :
502 8 : struct _pending_calls *pending = &tstate->interp->ceval.pending;
503 8 : if (_PyThread_at_fork_reinit(&pending->lock) < 0) {
504 0 : return _PyStatus_ERR("Can't reinitialize pending calls lock");
505 : }
506 :
507 : /* Destroy all threads except the current one */
508 8 : _PyThreadState_DeleteExcept(runtime, tstate);
509 8 : return _PyStatus_OK();
510 : }
511 : #endif
512 :
513 : /* This function is used to signal that async exceptions are waiting to be
514 : raised. */
515 :
516 : void
517 3 : _PyEval_SignalAsyncExc(PyInterpreterState *interp)
518 : {
519 3 : SIGNAL_ASYNC_EXC(interp);
520 3 : }
521 :
522 : PyThreadState *
523 10000200 : PyEval_SaveThread(void)
524 : {
525 10000200 : _PyRuntimeState *runtime = &_PyRuntime;
526 10000200 : PyThreadState *tstate = _PyThreadState_Swap(&runtime->gilstate, NULL);
527 10000200 : _Py_EnsureTstateNotNULL(tstate);
528 :
529 10000200 : struct _ceval_runtime_state *ceval = &runtime->ceval;
530 10000200 : struct _ceval_state *ceval2 = &tstate->interp->ceval;
531 10000200 : assert(gil_created(&ceval->gil));
532 10000200 : drop_gil(ceval, ceval2, tstate);
533 10000200 : return tstate;
534 : }
535 :
536 : void
537 10000200 : PyEval_RestoreThread(PyThreadState *tstate)
538 : {
539 10000200 : _Py_EnsureTstateNotNULL(tstate);
540 :
541 10000200 : take_gil(tstate);
542 :
543 10000100 : struct _gilstate_runtime_state *gilstate = &tstate->interp->runtime->gilstate;
544 10000100 : _PyThreadState_Swap(gilstate, tstate);
545 10000100 : }
546 :
547 :
548 : /* Mechanism whereby asynchronously executing callbacks (e.g. UNIX
549 : signal handlers or Mac I/O completion routines) can schedule calls
550 : to a function to be called synchronously.
551 : The synchronous function is called with one void* argument.
552 : It should return 0 for success or -1 for failure -- failure should
553 : be accompanied by an exception.
554 :
555 : If registry succeeds, the registry function returns 0; if it fails
556 : (e.g. due to too many pending calls) it returns -1 (without setting
557 : an exception condition).
558 :
559 : Note that because registry may occur from within signal handlers,
560 : or other asynchronous events, calling malloc() is unsafe!
561 :
562 : Any thread can schedule pending calls, but only the main thread
563 : will execute them.
564 : There is no facility to schedule calls to a particular thread, but
565 : that should be easy to change, should that ever be required. In
566 : that case, the static variables here should go into the python
567 : threadstate.
568 : */
569 :
570 : void
571 31454 : _PyEval_SignalReceived(PyInterpreterState *interp)
572 : {
573 : #ifdef MS_WINDOWS
574 : // bpo-42296: On Windows, _PyEval_SignalReceived() is called from a signal
575 : // handler which can run in a thread different than the Python thread, in
576 : // which case _Py_ThreadCanHandleSignals() is wrong. Ignore
577 : // _Py_ThreadCanHandleSignals() and always set eval_breaker to 1.
578 : //
579 : // The next eval_frame_handle_pending() call will call
580 : // _Py_ThreadCanHandleSignals() to recompute eval_breaker.
581 : int force = 1;
582 : #else
583 31454 : int force = 0;
584 : #endif
585 : /* bpo-30703: Function called when the C signal handler of Python gets a
586 : signal. We cannot queue a callback using _PyEval_AddPendingCall() since
587 : that function is not async-signal-safe. */
588 31454 : SIGNAL_PENDING_SIGNALS(interp, force);
589 31454 : }
590 :
591 : /* Push one item onto the queue while holding the lock. */
592 : static int
593 101 : _push_pending_call(struct _pending_calls *pending,
594 : int (*func)(void *), void *arg)
595 : {
596 101 : int i = pending->last;
597 101 : int j = (i + 1) % NPENDINGCALLS;
598 101 : if (j == pending->first) {
599 0 : return -1; /* Queue full */
600 : }
601 101 : pending->calls[i].func = func;
602 101 : pending->calls[i].arg = arg;
603 101 : pending->last = j;
604 101 : return 0;
605 : }
606 :
607 : /* Pop one item off the queue while holding the lock. */
608 : static void
609 587 : _pop_pending_call(struct _pending_calls *pending,
610 : int (**func)(void *), void **arg)
611 : {
612 587 : int i = pending->first;
613 587 : if (i == pending->last) {
614 486 : return; /* Queue empty */
615 : }
616 :
617 101 : *func = pending->calls[i].func;
618 101 : *arg = pending->calls[i].arg;
619 101 : pending->first = (i + 1) % NPENDINGCALLS;
620 : }
621 :
622 : /* This implementation is thread-safe. It allows
623 : scheduling to be made from any thread, and even from an executing
624 : callback.
625 : */
626 :
627 : int
628 101 : _PyEval_AddPendingCall(PyInterpreterState *interp,
629 : int (*func)(void *), void *arg)
630 : {
631 101 : struct _pending_calls *pending = &interp->ceval.pending;
632 :
633 : /* Ensure that _PyEval_InitPendingCalls() was called
634 : and that _PyEval_FiniPendingCalls() is not called yet. */
635 101 : assert(pending->lock != NULL);
636 :
637 101 : PyThread_acquire_lock(pending->lock, WAIT_LOCK);
638 101 : int result = _push_pending_call(pending, func, arg);
639 101 : PyThread_release_lock(pending->lock);
640 :
641 : /* signal main loop */
642 101 : SIGNAL_PENDING_CALLS(interp);
643 101 : return result;
644 : }
645 :
646 : int
647 96 : Py_AddPendingCall(int (*func)(void *), void *arg)
648 : {
649 : /* Best-effort to support subinterpreters and calls with the GIL released.
650 :
651 : First attempt _PyThreadState_GET() since it supports subinterpreters.
652 :
653 : If the GIL is released, _PyThreadState_GET() returns NULL . In this
654 : case, use PyGILState_GetThisThreadState() which works even if the GIL
655 : is released.
656 :
657 : Sadly, PyGILState_GetThisThreadState() doesn't support subinterpreters:
658 : see bpo-10915 and bpo-15751.
659 :
660 : Py_AddPendingCall() doesn't require the caller to hold the GIL. */
661 96 : PyThreadState *tstate = _PyThreadState_GET();
662 96 : if (tstate == NULL) {
663 96 : tstate = PyGILState_GetThisThreadState();
664 : }
665 :
666 : PyInterpreterState *interp;
667 96 : if (tstate != NULL) {
668 96 : interp = tstate->interp;
669 : }
670 : else {
671 : /* Last resort: use the main interpreter */
672 0 : interp = _PyInterpreterState_Main();
673 : }
674 96 : return _PyEval_AddPendingCall(interp, func, arg);
675 : }
676 :
677 : static int
678 31544 : handle_signals(PyThreadState *tstate)
679 : {
680 31544 : assert(is_tstate_valid(tstate));
681 31544 : if (!_Py_ThreadCanHandleSignals(tstate->interp)) {
682 16 : return 0;
683 : }
684 :
685 31528 : UNSIGNAL_PENDING_SIGNALS(tstate->interp);
686 31528 : if (_PyErr_CheckSignalsTstate(tstate) < 0) {
687 : /* On failure, re-schedule a call to handle_signals(). */
688 13 : SIGNAL_PENDING_SIGNALS(tstate->interp, 0);
689 13 : return -1;
690 : }
691 31515 : return 0;
692 : }
693 :
694 : static int
695 486 : make_pending_calls(PyInterpreterState *interp)
696 : {
697 : /* only execute pending calls on main thread */
698 486 : if (!_Py_ThreadCanHandlePendingCalls()) {
699 0 : return 0;
700 : }
701 :
702 : /* don't perform recursive pending calls */
703 : static int busy = 0;
704 486 : if (busy) {
705 0 : return 0;
706 : }
707 486 : busy = 1;
708 :
709 : /* unsignal before starting to call callbacks, so that any callback
710 : added in-between re-signals */
711 486 : UNSIGNAL_PENDING_CALLS(interp);
712 486 : int res = 0;
713 :
714 : /* perform a bounded number of calls, in case of recursion */
715 486 : struct _pending_calls *pending = &interp->ceval.pending;
716 587 : for (int i=0; i<NPENDINGCALLS; i++) {
717 587 : int (*func)(void *) = NULL;
718 587 : void *arg = NULL;
719 :
720 : /* pop one item off the queue while holding the lock */
721 587 : PyThread_acquire_lock(pending->lock, WAIT_LOCK);
722 587 : _pop_pending_call(pending, &func, &arg);
723 587 : PyThread_release_lock(pending->lock);
724 :
725 : /* having released the lock, perform the callback */
726 587 : if (func == NULL) {
727 486 : break;
728 : }
729 101 : res = func(arg);
730 101 : if (res) {
731 0 : goto error;
732 : }
733 : }
734 :
735 486 : busy = 0;
736 486 : return res;
737 :
738 0 : error:
739 0 : busy = 0;
740 0 : SIGNAL_PENDING_CALLS(interp);
741 0 : return res;
742 : }
743 :
744 : void
745 2952 : _Py_FinishPendingCalls(PyThreadState *tstate)
746 : {
747 2952 : assert(PyGILState_Check());
748 2952 : assert(is_tstate_valid(tstate));
749 :
750 2952 : struct _pending_calls *pending = &tstate->interp->ceval.pending;
751 :
752 2952 : if (!_Py_atomic_load_relaxed_int32(&(pending->calls_to_do))) {
753 2952 : return;
754 : }
755 :
756 0 : if (make_pending_calls(tstate->interp) < 0) {
757 : PyObject *exc, *val, *tb;
758 0 : _PyErr_Fetch(tstate, &exc, &val, &tb);
759 0 : PyErr_BadInternalCall();
760 0 : _PyErr_ChainExceptions(exc, val, tb);
761 0 : _PyErr_Print(tstate);
762 : }
763 : }
764 :
765 : /* Py_MakePendingCalls() is a simple wrapper for the sake
766 : of backward-compatibility. */
767 : int
768 405 : Py_MakePendingCalls(void)
769 : {
770 405 : assert(PyGILState_Check());
771 :
772 405 : PyThreadState *tstate = _PyThreadState_GET();
773 405 : assert(is_tstate_valid(tstate));
774 :
775 : /* Python signal handler doesn't really queue a callback: it only signals
776 : that a signal was received, see _PyEval_SignalReceived(). */
777 405 : int res = handle_signals(tstate);
778 405 : if (res != 0) {
779 2 : return res;
780 : }
781 :
782 403 : res = make_pending_calls(tstate->interp);
783 403 : if (res != 0) {
784 0 : return res;
785 : }
786 :
787 403 : return 0;
788 : }
789 :
790 : /* The interpreter's recursion limit */
791 :
792 : void
793 2984 : _PyEval_InitRuntimeState(struct _ceval_runtime_state *ceval)
794 : {
795 2984 : _gil_initialize(&ceval->gil);
796 2984 : }
797 :
798 : void
799 3134 : _PyEval_InitState(struct _ceval_state *ceval, PyThread_type_lock pending_lock)
800 : {
801 3134 : struct _pending_calls *pending = &ceval->pending;
802 3134 : assert(pending->lock == NULL);
803 :
804 3134 : pending->lock = pending_lock;
805 3134 : }
806 :
807 : void
808 3120 : _PyEval_FiniState(struct _ceval_state *ceval)
809 : {
810 3120 : struct _pending_calls *pending = &ceval->pending;
811 3120 : if (pending->lock != NULL) {
812 3120 : PyThread_free_lock(pending->lock);
813 3120 : pending->lock = NULL;
814 : }
815 3120 : }
816 :
817 : int
818 365941 : Py_GetRecursionLimit(void)
819 : {
820 365941 : PyInterpreterState *interp = _PyInterpreterState_GET();
821 365941 : return interp->ceval.recursion_limit;
822 : }
823 :
824 : void
825 96 : Py_SetRecursionLimit(int new_limit)
826 : {
827 96 : PyInterpreterState *interp = _PyInterpreterState_GET();
828 96 : interp->ceval.recursion_limit = new_limit;
829 192 : for (PyThreadState *p = interp->threads.head; p != NULL; p = p->next) {
830 96 : int depth = p->recursion_limit - p->recursion_remaining;
831 96 : p->recursion_limit = new_limit;
832 96 : p->recursion_remaining = new_limit - depth;
833 : }
834 96 : }
835 :
836 : /* The function _Py_EnterRecursiveCallTstate() only calls _Py_CheckRecursiveCall()
837 : if the recursion_depth reaches recursion_limit. */
838 : int
839 16547 : _Py_CheckRecursiveCall(PyThreadState *tstate, const char *where)
840 : {
841 : /* Check against global limit first. */
842 16547 : int depth = tstate->recursion_limit - tstate->recursion_remaining;
843 16547 : if (depth < tstate->interp->ceval.recursion_limit) {
844 0 : tstate->recursion_limit = tstate->interp->ceval.recursion_limit;
845 0 : tstate->recursion_remaining = tstate->recursion_limit - depth;
846 0 : assert(tstate->recursion_remaining > 0);
847 0 : return 0;
848 : }
849 : #ifdef USE_STACKCHECK
850 : if (PyOS_CheckStack()) {
851 : ++tstate->recursion_remaining;
852 : _PyErr_SetString(tstate, PyExc_MemoryError, "Stack overflow");
853 : return -1;
854 : }
855 : #endif
856 16547 : if (tstate->recursion_headroom) {
857 8205 : if (tstate->recursion_remaining < -50) {
858 : /* Overflowing while handling an overflow. Give up. */
859 0 : Py_FatalError("Cannot recover from stack overflow.");
860 : }
861 : }
862 : else {
863 8342 : if (tstate->recursion_remaining <= 0) {
864 8342 : tstate->recursion_headroom++;
865 8342 : _PyErr_Format(tstate, PyExc_RecursionError,
866 : "maximum recursion depth exceeded%s",
867 : where);
868 8342 : tstate->recursion_headroom--;
869 8342 : ++tstate->recursion_remaining;
870 8342 : return -1;
871 : }
872 : }
873 8205 : return 0;
874 : }
875 :
876 :
877 : static const binaryfunc binary_ops[] = {
878 : [NB_ADD] = PyNumber_Add,
879 : [NB_AND] = PyNumber_And,
880 : [NB_FLOOR_DIVIDE] = PyNumber_FloorDivide,
881 : [NB_LSHIFT] = PyNumber_Lshift,
882 : [NB_MATRIX_MULTIPLY] = PyNumber_MatrixMultiply,
883 : [NB_MULTIPLY] = PyNumber_Multiply,
884 : [NB_REMAINDER] = PyNumber_Remainder,
885 : [NB_OR] = PyNumber_Or,
886 : [NB_POWER] = _PyNumber_PowerNoMod,
887 : [NB_RSHIFT] = PyNumber_Rshift,
888 : [NB_SUBTRACT] = PyNumber_Subtract,
889 : [NB_TRUE_DIVIDE] = PyNumber_TrueDivide,
890 : [NB_XOR] = PyNumber_Xor,
891 : [NB_INPLACE_ADD] = PyNumber_InPlaceAdd,
892 : [NB_INPLACE_AND] = PyNumber_InPlaceAnd,
893 : [NB_INPLACE_FLOOR_DIVIDE] = PyNumber_InPlaceFloorDivide,
894 : [NB_INPLACE_LSHIFT] = PyNumber_InPlaceLshift,
895 : [NB_INPLACE_MATRIX_MULTIPLY] = PyNumber_InPlaceMatrixMultiply,
896 : [NB_INPLACE_MULTIPLY] = PyNumber_InPlaceMultiply,
897 : [NB_INPLACE_REMAINDER] = PyNumber_InPlaceRemainder,
898 : [NB_INPLACE_OR] = PyNumber_InPlaceOr,
899 : [NB_INPLACE_POWER] = _PyNumber_InPlacePowerNoMod,
900 : [NB_INPLACE_RSHIFT] = PyNumber_InPlaceRshift,
901 : [NB_INPLACE_SUBTRACT] = PyNumber_InPlaceSubtract,
902 : [NB_INPLACE_TRUE_DIVIDE] = PyNumber_InPlaceTrueDivide,
903 : [NB_INPLACE_XOR] = PyNumber_InPlaceXor,
904 : };
905 :
906 :
907 : // PEP 634: Structural Pattern Matching
908 :
909 :
910 : // Return a tuple of values corresponding to keys, with error checks for
911 : // duplicate/missing keys.
912 : static PyObject*
913 47 : match_keys(PyThreadState *tstate, PyObject *map, PyObject *keys)
914 : {
915 47 : assert(PyTuple_CheckExact(keys));
916 47 : Py_ssize_t nkeys = PyTuple_GET_SIZE(keys);
917 47 : if (!nkeys) {
918 : // No keys means no items.
919 3 : return PyTuple_New(0);
920 : }
921 44 : PyObject *seen = NULL;
922 44 : PyObject *dummy = NULL;
923 44 : PyObject *values = NULL;
924 44 : PyObject *get = NULL;
925 : // We use the two argument form of map.get(key, default) for two reasons:
926 : // - Atomically check for a key and get its value without error handling.
927 : // - Don't cause key creation or resizing in dict subclasses like
928 : // collections.defaultdict that define __missing__ (or similar).
929 44 : int meth_found = _PyObject_GetMethod(map, &_Py_ID(get), &get);
930 44 : if (get == NULL) {
931 0 : goto fail;
932 : }
933 44 : seen = PySet_New(NULL);
934 44 : if (seen == NULL) {
935 0 : goto fail;
936 : }
937 : // dummy = object()
938 44 : dummy = _PyObject_CallNoArgs((PyObject *)&PyBaseObject_Type);
939 44 : if (dummy == NULL) {
940 0 : goto fail;
941 : }
942 44 : values = PyTuple_New(nkeys);
943 44 : if (values == NULL) {
944 0 : goto fail;
945 : }
946 92 : for (Py_ssize_t i = 0; i < nkeys; i++) {
947 57 : PyObject *key = PyTuple_GET_ITEM(keys, i);
948 57 : if (PySet_Contains(seen, key) || PySet_Add(seen, key)) {
949 1 : if (!_PyErr_Occurred(tstate)) {
950 : // Seen it before!
951 1 : _PyErr_Format(tstate, PyExc_ValueError,
952 : "mapping pattern checks duplicate key (%R)", key);
953 : }
954 1 : goto fail;
955 : }
956 56 : PyObject *args[] = { map, key, dummy };
957 56 : PyObject *value = NULL;
958 56 : if (meth_found) {
959 55 : value = PyObject_Vectorcall(get, args, 3, NULL);
960 : }
961 : else {
962 1 : value = PyObject_Vectorcall(get, &args[1], 2, NULL);
963 : }
964 56 : if (value == NULL) {
965 0 : goto fail;
966 : }
967 56 : if (value == dummy) {
968 : // key not in map!
969 8 : Py_DECREF(value);
970 8 : Py_DECREF(values);
971 : // Return None:
972 8 : Py_INCREF(Py_None);
973 8 : values = Py_None;
974 8 : goto done;
975 : }
976 48 : PyTuple_SET_ITEM(values, i, value);
977 : }
978 : // Success:
979 35 : done:
980 43 : Py_DECREF(get);
981 43 : Py_DECREF(seen);
982 43 : Py_DECREF(dummy);
983 43 : return values;
984 1 : fail:
985 1 : Py_XDECREF(get);
986 1 : Py_XDECREF(seen);
987 1 : Py_XDECREF(dummy);
988 1 : Py_XDECREF(values);
989 1 : return NULL;
990 : }
991 :
992 : // Extract a named attribute from the subject, with additional bookkeeping to
993 : // raise TypeErrors for repeated lookups. On failure, return NULL (with no
994 : // error set). Use _PyErr_Occurred(tstate) to disambiguate.
995 : static PyObject*
996 4918 : match_class_attr(PyThreadState *tstate, PyObject *subject, PyObject *type,
997 : PyObject *name, PyObject *seen)
998 : {
999 4918 : assert(PyUnicode_CheckExact(name));
1000 4918 : assert(PySet_CheckExact(seen));
1001 4918 : if (PySet_Contains(seen, name) || PySet_Add(seen, name)) {
1002 2 : if (!_PyErr_Occurred(tstate)) {
1003 : // Seen it before!
1004 2 : _PyErr_Format(tstate, PyExc_TypeError,
1005 : "%s() got multiple sub-patterns for attribute %R",
1006 : ((PyTypeObject*)type)->tp_name, name);
1007 : }
1008 2 : return NULL;
1009 : }
1010 4916 : PyObject *attr = PyObject_GetAttr(subject, name);
1011 4916 : if (attr == NULL && _PyErr_ExceptionMatches(tstate, PyExc_AttributeError)) {
1012 0 : _PyErr_Clear(tstate);
1013 : }
1014 4916 : return attr;
1015 : }
1016 :
1017 : // On success (match), return a tuple of extracted attributes. On failure (no
1018 : // match), return NULL. Use _PyErr_Occurred(tstate) to disambiguate.
1019 : static PyObject*
1020 13960 : match_class(PyThreadState *tstate, PyObject *subject, PyObject *type,
1021 : Py_ssize_t nargs, PyObject *kwargs)
1022 : {
1023 13960 : if (!PyType_Check(type)) {
1024 0 : const char *e = "called match pattern must be a type";
1025 0 : _PyErr_Format(tstate, PyExc_TypeError, e);
1026 0 : return NULL;
1027 : }
1028 13960 : assert(PyTuple_CheckExact(kwargs));
1029 : // First, an isinstance check:
1030 13960 : if (PyObject_IsInstance(subject, type) <= 0) {
1031 8027 : return NULL;
1032 : }
1033 : // So far so good:
1034 5933 : PyObject *seen = PySet_New(NULL);
1035 5933 : if (seen == NULL) {
1036 0 : return NULL;
1037 : }
1038 5933 : PyObject *attrs = PyList_New(0);
1039 5933 : if (attrs == NULL) {
1040 0 : Py_DECREF(seen);
1041 0 : return NULL;
1042 : }
1043 : // NOTE: From this point on, goto fail on failure:
1044 5933 : PyObject *match_args = NULL;
1045 : // First, the positional subpatterns:
1046 5933 : if (nargs) {
1047 4888 : int match_self = 0;
1048 4888 : match_args = PyObject_GetAttrString(type, "__match_args__");
1049 4888 : if (match_args) {
1050 4870 : if (!PyTuple_CheckExact(match_args)) {
1051 3 : const char *e = "%s.__match_args__ must be a tuple (got %s)";
1052 3 : _PyErr_Format(tstate, PyExc_TypeError, e,
1053 : ((PyTypeObject *)type)->tp_name,
1054 3 : Py_TYPE(match_args)->tp_name);
1055 3 : goto fail;
1056 : }
1057 : }
1058 18 : else if (_PyErr_ExceptionMatches(tstate, PyExc_AttributeError)) {
1059 18 : _PyErr_Clear(tstate);
1060 : // _Py_TPFLAGS_MATCH_SELF is only acknowledged if the type does not
1061 : // define __match_args__. This is natural behavior for subclasses:
1062 : // it's as if __match_args__ is some "magic" value that is lost as
1063 : // soon as they redefine it.
1064 18 : match_args = PyTuple_New(0);
1065 18 : match_self = PyType_HasFeature((PyTypeObject*)type,
1066 : _Py_TPFLAGS_MATCH_SELF);
1067 : }
1068 : else {
1069 0 : goto fail;
1070 : }
1071 4885 : assert(PyTuple_CheckExact(match_args));
1072 4885 : Py_ssize_t allowed = match_self ? 1 : PyTuple_GET_SIZE(match_args);
1073 4885 : if (allowed < nargs) {
1074 2 : const char *plural = (allowed == 1) ? "" : "s";
1075 2 : _PyErr_Format(tstate, PyExc_TypeError,
1076 : "%s() accepts %d positional sub-pattern%s (%d given)",
1077 : ((PyTypeObject*)type)->tp_name,
1078 : allowed, plural, nargs);
1079 2 : goto fail;
1080 : }
1081 4883 : if (match_self) {
1082 : // Easy. Copy the subject itself, and move on to kwargs.
1083 17 : PyList_Append(attrs, subject);
1084 : }
1085 : else {
1086 9766 : for (Py_ssize_t i = 0; i < nargs; i++) {
1087 4902 : PyObject *name = PyTuple_GET_ITEM(match_args, i);
1088 4902 : if (!PyUnicode_CheckExact(name)) {
1089 1 : _PyErr_Format(tstate, PyExc_TypeError,
1090 : "__match_args__ elements must be strings "
1091 1 : "(got %s)", Py_TYPE(name)->tp_name);
1092 1 : goto fail;
1093 : }
1094 4901 : PyObject *attr = match_class_attr(tstate, subject, type, name,
1095 : seen);
1096 4901 : if (attr == NULL) {
1097 1 : goto fail;
1098 : }
1099 4900 : PyList_Append(attrs, attr);
1100 4900 : Py_DECREF(attr);
1101 : }
1102 : }
1103 4881 : Py_CLEAR(match_args);
1104 : }
1105 : // Finally, the keyword subpatterns:
1106 5942 : for (Py_ssize_t i = 0; i < PyTuple_GET_SIZE(kwargs); i++) {
1107 17 : PyObject *name = PyTuple_GET_ITEM(kwargs, i);
1108 17 : PyObject *attr = match_class_attr(tstate, subject, type, name, seen);
1109 17 : if (attr == NULL) {
1110 1 : goto fail;
1111 : }
1112 16 : PyList_Append(attrs, attr);
1113 16 : Py_DECREF(attr);
1114 : }
1115 5925 : Py_SETREF(attrs, PyList_AsTuple(attrs));
1116 5925 : Py_DECREF(seen);
1117 5925 : return attrs;
1118 8 : fail:
1119 : // We really don't care whether an error was raised or not... that's our
1120 : // caller's problem. All we know is that the match failed.
1121 8 : Py_XDECREF(match_args);
1122 8 : Py_DECREF(seen);
1123 8 : Py_DECREF(attrs);
1124 8 : return NULL;
1125 : }
1126 :
1127 :
1128 : static int do_raise(PyThreadState *tstate, PyObject *exc, PyObject *cause);
1129 : static int exception_group_match(
1130 : PyObject* exc_value, PyObject *match_type,
1131 : PyObject **match, PyObject **rest);
1132 :
1133 : static int unpack_iterable(PyThreadState *, PyObject *, int, int, PyObject **);
1134 :
1135 : PyObject *
1136 319938 : PyEval_EvalCode(PyObject *co, PyObject *globals, PyObject *locals)
1137 : {
1138 319938 : PyThreadState *tstate = _PyThreadState_GET();
1139 319938 : if (locals == NULL) {
1140 0 : locals = globals;
1141 : }
1142 319938 : PyObject *builtins = _PyEval_BuiltinsFromGlobals(tstate, globals); // borrowed ref
1143 319938 : if (builtins == NULL) {
1144 0 : return NULL;
1145 : }
1146 319938 : PyFrameConstructor desc = {
1147 : .fc_globals = globals,
1148 : .fc_builtins = builtins,
1149 319938 : .fc_name = ((PyCodeObject *)co)->co_name,
1150 319938 : .fc_qualname = ((PyCodeObject *)co)->co_name,
1151 : .fc_code = co,
1152 : .fc_defaults = NULL,
1153 : .fc_kwdefaults = NULL,
1154 : .fc_closure = NULL
1155 : };
1156 319938 : PyFunctionObject *func = _PyFunction_FromConstructor(&desc);
1157 319938 : if (func == NULL) {
1158 0 : return NULL;
1159 : }
1160 : EVAL_CALL_STAT_INC(EVAL_CALL_LEGACY);
1161 319938 : PyObject *res = _PyEval_Vector(tstate, func, locals, NULL, 0, NULL);
1162 319936 : Py_DECREF(func);
1163 319936 : return res;
1164 : }
1165 :
1166 :
1167 : /* Interpreter main loop */
1168 :
1169 : PyObject *
1170 0 : PyEval_EvalFrame(PyFrameObject *f)
1171 : {
1172 : /* Function kept for backward compatibility */
1173 0 : PyThreadState *tstate = _PyThreadState_GET();
1174 0 : return _PyEval_EvalFrame(tstate, f->f_frame, 0);
1175 : }
1176 :
1177 : PyObject *
1178 0 : PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
1179 : {
1180 0 : PyThreadState *tstate = _PyThreadState_GET();
1181 0 : return _PyEval_EvalFrame(tstate, f->f_frame, throwflag);
1182 : }
1183 :
1184 :
1185 : /* Handle signals, pending calls, GIL drop request
1186 : and asynchronous exception */
1187 : static int
1188 54068 : eval_frame_handle_pending(PyThreadState *tstate)
1189 : {
1190 54068 : _PyRuntimeState * const runtime = &_PyRuntime;
1191 54068 : struct _ceval_runtime_state *ceval = &runtime->ceval;
1192 :
1193 : /* Pending signals */
1194 54068 : if (_Py_atomic_load_relaxed_int32(&ceval->signals_pending)) {
1195 31139 : if (handle_signals(tstate) != 0) {
1196 11 : return -1;
1197 : }
1198 : }
1199 :
1200 : /* Pending calls */
1201 54057 : struct _ceval_state *ceval2 = &tstate->interp->ceval;
1202 54057 : if (_Py_atomic_load_relaxed_int32(&ceval2->pending.calls_to_do)) {
1203 83 : if (make_pending_calls(tstate->interp) != 0) {
1204 0 : return -1;
1205 : }
1206 : }
1207 :
1208 : /* GIL drop request */
1209 54057 : if (_Py_atomic_load_relaxed_int32(&ceval2->gil_drop_request)) {
1210 : /* Give another thread a chance */
1211 22833 : if (_PyThreadState_Swap(&runtime->gilstate, NULL) != tstate) {
1212 0 : Py_FatalError("tstate mix-up");
1213 : }
1214 22833 : drop_gil(ceval, ceval2, tstate);
1215 :
1216 : /* Other threads may run now */
1217 :
1218 22833 : take_gil(tstate);
1219 :
1220 22833 : if (_PyThreadState_Swap(&runtime->gilstate, tstate) != NULL) {
1221 0 : Py_FatalError("orphan tstate");
1222 : }
1223 : }
1224 :
1225 : /* Check for asynchronous exception. */
1226 54057 : if (tstate->async_exc != NULL) {
1227 2 : PyObject *exc = tstate->async_exc;
1228 2 : tstate->async_exc = NULL;
1229 2 : UNSIGNAL_ASYNC_EXC(tstate->interp);
1230 2 : _PyErr_SetNone(tstate, exc);
1231 2 : Py_DECREF(exc);
1232 2 : return -1;
1233 : }
1234 :
1235 : #ifdef MS_WINDOWS
1236 : // bpo-42296: On Windows, _PyEval_SignalReceived() can be called in a
1237 : // different thread than the Python thread, in which case
1238 : // _Py_ThreadCanHandleSignals() is wrong. Recompute eval_breaker in the
1239 : // current Python thread with the correct _Py_ThreadCanHandleSignals()
1240 : // value. It prevents to interrupt the eval loop at every instruction if
1241 : // the current Python thread cannot handle signals (if
1242 : // _Py_ThreadCanHandleSignals() is false).
1243 : COMPUTE_EVAL_BREAKER(tstate->interp, ceval, ceval2);
1244 : #endif
1245 :
1246 54055 : return 0;
1247 : }
1248 :
1249 :
1250 : /* Computed GOTOs, or
1251 : the-optimization-commonly-but-improperly-known-as-"threaded code"
1252 : using gcc's labels-as-values extension
1253 : (http://gcc.gnu.org/onlinedocs/gcc/Labels-as-Values.html).
1254 :
1255 : The traditional bytecode evaluation loop uses a "switch" statement, which
1256 : decent compilers will optimize as a single indirect branch instruction
1257 : combined with a lookup table of jump addresses. However, since the
1258 : indirect jump instruction is shared by all opcodes, the CPU will have a
1259 : hard time making the right prediction for where to jump next (actually,
1260 : it will be always wrong except in the uncommon case of a sequence of
1261 : several identical opcodes).
1262 :
1263 : "Threaded code" in contrast, uses an explicit jump table and an explicit
1264 : indirect jump instruction at the end of each opcode. Since the jump
1265 : instruction is at a different address for each opcode, the CPU will make a
1266 : separate prediction for each of these instructions, which is equivalent to
1267 : predicting the second opcode of each opcode pair. These predictions have
1268 : a much better chance to turn out valid, especially in small bytecode loops.
1269 :
1270 : A mispredicted branch on a modern CPU flushes the whole pipeline and
1271 : can cost several CPU cycles (depending on the pipeline depth),
1272 : and potentially many more instructions (depending on the pipeline width).
1273 : A correctly predicted branch, however, is nearly free.
1274 :
1275 : At the time of this writing, the "threaded code" version is up to 15-20%
1276 : faster than the normal "switch" version, depending on the compiler and the
1277 : CPU architecture.
1278 :
1279 : NOTE: care must be taken that the compiler doesn't try to "optimize" the
1280 : indirect jumps by sharing them between all opcodes. Such optimizations
1281 : can be disabled on gcc by using the -fno-gcse flag (or possibly
1282 : -fno-crossjumping).
1283 : */
1284 :
1285 : /* Use macros rather than inline functions, to make it as clear as possible
1286 : * to the C compiler that the tracing check is a simple test then branch.
1287 : * We want to be sure that the compiler knows this before it generates
1288 : * the CFG.
1289 : */
1290 :
1291 : #ifdef WITH_DTRACE
1292 : #define OR_DTRACE_LINE | (PyDTrace_LINE_ENABLED() ? 255 : 0)
1293 : #else
1294 : #define OR_DTRACE_LINE
1295 : #endif
1296 :
1297 : #ifdef HAVE_COMPUTED_GOTOS
1298 : #ifndef USE_COMPUTED_GOTOS
1299 : #define USE_COMPUTED_GOTOS 1
1300 : #endif
1301 : #else
1302 : #if defined(USE_COMPUTED_GOTOS) && USE_COMPUTED_GOTOS
1303 : #error "Computed gotos are not supported on this compiler."
1304 : #endif
1305 : #undef USE_COMPUTED_GOTOS
1306 : #define USE_COMPUTED_GOTOS 0
1307 : #endif
1308 :
1309 : #ifdef Py_STATS
1310 : #define INSTRUCTION_START(op) \
1311 : do { \
1312 : frame->prev_instr = next_instr++; \
1313 : OPCODE_EXE_INC(op); \
1314 : if (_py_stats) _py_stats->opcode_stats[lastopcode].pair_count[op]++; \
1315 : lastopcode = op; \
1316 : } while (0)
1317 : #else
1318 : #define INSTRUCTION_START(op) (frame->prev_instr = next_instr++)
1319 : #endif
1320 :
1321 : #if USE_COMPUTED_GOTOS
1322 : #define TARGET(op) TARGET_##op: INSTRUCTION_START(op);
1323 : #define DISPATCH_GOTO() goto *opcode_targets[opcode]
1324 : #else
1325 : #define TARGET(op) case op: INSTRUCTION_START(op);
1326 : #define DISPATCH_GOTO() goto dispatch_opcode
1327 : #endif
1328 :
1329 : /* PRE_DISPATCH_GOTO() does lltrace if enabled. Normally a no-op */
1330 : #ifdef LLTRACE
1331 : #define PRE_DISPATCH_GOTO() if (lltrace) { \
1332 : lltrace_instruction(frame, stack_pointer, next_instr); }
1333 : #else
1334 : #define PRE_DISPATCH_GOTO() ((void)0)
1335 : #endif
1336 :
1337 : #define NOTRACE_DISPATCH() \
1338 : { \
1339 : NEXTOPARG(); \
1340 : PRE_DISPATCH_GOTO(); \
1341 : DISPATCH_GOTO(); \
1342 : }
1343 :
1344 : /* Do interpreter dispatch accounting for tracing and instrumentation */
1345 : #define DISPATCH() \
1346 : { \
1347 : NEXTOPARG(); \
1348 : PRE_DISPATCH_GOTO(); \
1349 : assert(cframe.use_tracing == 0 || cframe.use_tracing == 255); \
1350 : opcode |= cframe.use_tracing OR_DTRACE_LINE; \
1351 : DISPATCH_GOTO(); \
1352 : }
1353 :
1354 : #define NOTRACE_DISPATCH_SAME_OPARG() \
1355 : { \
1356 : opcode = _Py_OPCODE(*next_instr); \
1357 : PRE_DISPATCH_GOTO(); \
1358 : DISPATCH_GOTO(); \
1359 : }
1360 :
1361 : #define CHECK_EVAL_BREAKER() \
1362 : _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); \
1363 : if (_Py_atomic_load_relaxed_int32(eval_breaker)) { \
1364 : goto handle_eval_breaker; \
1365 : }
1366 :
1367 :
1368 : /* Tuple access macros */
1369 :
1370 : #ifndef Py_DEBUG
1371 : #define GETITEM(v, i) PyTuple_GET_ITEM((PyTupleObject *)(v), (i))
1372 : #else
1373 : #define GETITEM(v, i) PyTuple_GetItem((v), (i))
1374 : #endif
1375 :
1376 : /* Code access macros */
1377 :
1378 : /* The integer overflow is checked by an assertion below. */
1379 : #define INSTR_OFFSET() ((int)(next_instr - first_instr))
1380 : #define NEXTOPARG() do { \
1381 : _Py_CODEUNIT word = *next_instr; \
1382 : opcode = _Py_OPCODE(word); \
1383 : oparg = _Py_OPARG(word); \
1384 : } while (0)
1385 : #define JUMPTO(x) (next_instr = first_instr + (x))
1386 : #define JUMPBY(x) (next_instr += (x))
1387 :
1388 : /* Get opcode and oparg from original instructions, not quickened form. */
1389 : #define TRACING_NEXTOPARG() do { \
1390 : NEXTOPARG(); \
1391 : opcode = _PyOpcode_Deopt[opcode]; \
1392 : } while (0)
1393 :
1394 : /* OpCode prediction macros
1395 : Some opcodes tend to come in pairs thus making it possible to
1396 : predict the second code when the first is run. For example,
1397 : COMPARE_OP is often followed by POP_JUMP_IF_FALSE or POP_JUMP_IF_TRUE.
1398 :
1399 : Verifying the prediction costs a single high-speed test of a register
1400 : variable against a constant. If the pairing was good, then the
1401 : processor's own internal branch predication has a high likelihood of
1402 : success, resulting in a nearly zero-overhead transition to the
1403 : next opcode. A successful prediction saves a trip through the eval-loop
1404 : including its unpredictable switch-case branch. Combined with the
1405 : processor's internal branch prediction, a successful PREDICT has the
1406 : effect of making the two opcodes run as if they were a single new opcode
1407 : with the bodies combined.
1408 :
1409 : If collecting opcode statistics, your choices are to either keep the
1410 : predictions turned-on and interpret the results as if some opcodes
1411 : had been combined or turn-off predictions so that the opcode frequency
1412 : counter updates for both opcodes.
1413 :
1414 : Opcode prediction is disabled with threaded code, since the latter allows
1415 : the CPU to record separate branch prediction information for each
1416 : opcode.
1417 :
1418 : */
1419 :
1420 : #define PREDICT_ID(op) PRED_##op
1421 :
1422 : #if USE_COMPUTED_GOTOS
1423 : #define PREDICT(op) if (0) goto PREDICT_ID(op)
1424 : #else
1425 : #define PREDICT(op) \
1426 : do { \
1427 : _Py_CODEUNIT word = *next_instr; \
1428 : opcode = _Py_OPCODE(word) | cframe.use_tracing OR_DTRACE_LINE; \
1429 : if (opcode == op) { \
1430 : oparg = _Py_OPARG(word); \
1431 : INSTRUCTION_START(op); \
1432 : goto PREDICT_ID(op); \
1433 : } \
1434 : } while(0)
1435 : #endif
1436 : #define PREDICTED(op) PREDICT_ID(op):
1437 :
1438 :
1439 : /* Stack manipulation macros */
1440 :
1441 : /* The stack can grow at most MAXINT deep, as co_nlocals and
1442 : co_stacksize are ints. */
1443 : #define STACK_LEVEL() ((int)(stack_pointer - _PyFrame_Stackbase(frame)))
1444 : #define STACK_SIZE() (frame->f_code->co_stacksize)
1445 : #define EMPTY() (STACK_LEVEL() == 0)
1446 : #define TOP() (stack_pointer[-1])
1447 : #define SECOND() (stack_pointer[-2])
1448 : #define THIRD() (stack_pointer[-3])
1449 : #define FOURTH() (stack_pointer[-4])
1450 : #define PEEK(n) (stack_pointer[-(n)])
1451 : #define SET_TOP(v) (stack_pointer[-1] = (v))
1452 : #define SET_SECOND(v) (stack_pointer[-2] = (v))
1453 : #define BASIC_STACKADJ(n) (stack_pointer += n)
1454 : #define BASIC_PUSH(v) (*stack_pointer++ = (v))
1455 : #define BASIC_POP() (*--stack_pointer)
1456 :
1457 : #ifdef Py_DEBUG
1458 : #define PUSH(v) do { \
1459 : BASIC_PUSH(v); \
1460 : assert(STACK_LEVEL() <= STACK_SIZE()); \
1461 : } while (0)
1462 : #define POP() (assert(STACK_LEVEL() > 0), BASIC_POP())
1463 : #define STACK_GROW(n) do { \
1464 : assert(n >= 0); \
1465 : BASIC_STACKADJ(n); \
1466 : assert(STACK_LEVEL() <= STACK_SIZE()); \
1467 : } while (0)
1468 : #define STACK_SHRINK(n) do { \
1469 : assert(n >= 0); \
1470 : assert(STACK_LEVEL() >= n); \
1471 : BASIC_STACKADJ(-(n)); \
1472 : } while (0)
1473 : #else
1474 : #define PUSH(v) BASIC_PUSH(v)
1475 : #define POP() BASIC_POP()
1476 : #define STACK_GROW(n) BASIC_STACKADJ(n)
1477 : #define STACK_SHRINK(n) BASIC_STACKADJ(-(n))
1478 : #endif
1479 :
1480 : /* Local variable macros */
1481 :
1482 : #define GETLOCAL(i) (frame->localsplus[i])
1483 :
1484 : /* The SETLOCAL() macro must not DECREF the local variable in-place and
1485 : then store the new value; it must copy the old value to a temporary
1486 : value, then store the new value, and then DECREF the temporary value.
1487 : This is because it is possible that during the DECREF the frame is
1488 : accessed by other code (e.g. a __del__ method or gc.collect()) and the
1489 : variable would be pointing to already-freed memory. */
1490 : #define SETLOCAL(i, value) do { PyObject *tmp = GETLOCAL(i); \
1491 : GETLOCAL(i) = value; \
1492 : Py_XDECREF(tmp); } while (0)
1493 :
1494 : #define JUMP_TO_INSTRUCTION(op) goto PREDICT_ID(op)
1495 :
1496 :
1497 : #define DEOPT_IF(cond, instname) if (cond) { goto miss; }
1498 :
1499 :
1500 : #define GLOBALS() frame->f_globals
1501 : #define BUILTINS() frame->f_builtins
1502 : #define LOCALS() frame->f_locals
1503 :
1504 : /* Shared opcode macros */
1505 :
1506 : #define TRACE_FUNCTION_EXIT() \
1507 : if (cframe.use_tracing) { \
1508 : if (trace_function_exit(tstate, frame, retval)) { \
1509 : Py_DECREF(retval); \
1510 : goto exit_unwind; \
1511 : } \
1512 : }
1513 :
1514 : #define DTRACE_FUNCTION_EXIT() \
1515 : if (PyDTrace_FUNCTION_RETURN_ENABLED()) { \
1516 : dtrace_function_return(frame); \
1517 : }
1518 :
1519 : #define TRACE_FUNCTION_UNWIND() \
1520 : if (cframe.use_tracing) { \
1521 : /* Since we are already unwinding, \
1522 : * we don't care if this raises */ \
1523 : trace_function_exit(tstate, frame, NULL); \
1524 : }
1525 :
1526 : #define TRACE_FUNCTION_ENTRY() \
1527 : if (cframe.use_tracing) { \
1528 : _PyFrame_SetStackPointer(frame, stack_pointer); \
1529 : int err = trace_function_entry(tstate, frame); \
1530 : stack_pointer = _PyFrame_GetStackPointer(frame); \
1531 : if (err) { \
1532 : goto error; \
1533 : } \
1534 : }
1535 :
1536 : #define TRACE_FUNCTION_THROW_ENTRY() \
1537 : if (cframe.use_tracing) { \
1538 : assert(frame->stacktop >= 0); \
1539 : if (trace_function_entry(tstate, frame)) { \
1540 : goto exit_unwind; \
1541 : } \
1542 : }
1543 :
1544 : #define DTRACE_FUNCTION_ENTRY() \
1545 : if (PyDTrace_FUNCTION_ENTRY_ENABLED()) { \
1546 : dtrace_function_entry(frame); \
1547 : }
1548 :
1549 : #define ADAPTIVE_COUNTER_IS_ZERO(cache) \
1550 : (cache)->counter < (1<<ADAPTIVE_BACKOFF_BITS)
1551 :
1552 : #define DECREMENT_ADAPTIVE_COUNTER(cache) \
1553 : (cache)->counter -= (1<<ADAPTIVE_BACKOFF_BITS)
1554 :
1555 : static int
1556 137403 : trace_function_entry(PyThreadState *tstate, _PyInterpreterFrame *frame)
1557 : {
1558 137403 : if (tstate->c_tracefunc != NULL) {
1559 : /* tstate->c_tracefunc, if defined, is a
1560 : function that will be called on *every* entry
1561 : to a code block. Its return value, if not
1562 : None, is a function that will be called at
1563 : the start of each executed line of code.
1564 : (Actually, the function must return itself
1565 : in order to continue tracing.) The trace
1566 : functions are called with three arguments:
1567 : a pointer to the current frame, a string
1568 : indicating why the function is called, and
1569 : an argument which depends on the situation.
1570 : The global trace function is also called
1571 : whenever an exception is detected. */
1572 134564 : if (call_trace_protected(tstate->c_tracefunc,
1573 : tstate->c_traceobj,
1574 : tstate, frame,
1575 : PyTrace_CALL, Py_None)) {
1576 : /* Trace function raised an error */
1577 1003 : return -1;
1578 : }
1579 : }
1580 136400 : if (tstate->c_profilefunc != NULL) {
1581 : /* Similar for c_profilefunc, except it needn't
1582 : return itself and isn't called for "line" events */
1583 2839 : if (call_trace_protected(tstate->c_profilefunc,
1584 : tstate->c_profileobj,
1585 : tstate, frame,
1586 : PyTrace_CALL, Py_None)) {
1587 : /* Profile function raised an error */
1588 0 : return -1;
1589 : }
1590 : }
1591 136400 : return 0;
1592 : }
1593 :
1594 : static int
1595 2478760 : trace_function_exit(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject *retval)
1596 : {
1597 2478760 : if (tstate->c_tracefunc) {
1598 2455050 : if (call_trace_protected(tstate->c_tracefunc, tstate->c_traceobj,
1599 : tstate, frame, PyTrace_RETURN, retval)) {
1600 1008 : return -1;
1601 : }
1602 : }
1603 2477760 : if (tstate->c_profilefunc) {
1604 23715 : if (call_trace_protected(tstate->c_profilefunc, tstate->c_profileobj,
1605 : tstate, frame, PyTrace_RETURN, retval)) {
1606 0 : return -1;
1607 : }
1608 : }
1609 2477760 : return 0;
1610 : }
1611 :
1612 : static _PyInterpreterFrame *
1613 186182000 : pop_frame(PyThreadState *tstate, _PyInterpreterFrame *frame)
1614 : {
1615 186182000 : _PyInterpreterFrame *prev_frame = frame->previous;
1616 186182000 : _PyEvalFrameClearAndPop(tstate, frame);
1617 186182000 : return prev_frame;
1618 : }
1619 :
1620 : /* It is only between the KW_NAMES instruction and the following CALL,
1621 : * that this has any meaning.
1622 : */
1623 : typedef struct {
1624 : PyObject *kwnames;
1625 : } CallShape;
1626 :
1627 : // GH-89279: Must be a macro to be sure it's inlined by MSVC.
1628 : #define is_method(stack_pointer, args) (PEEK((args)+2) != NULL)
1629 :
1630 : #define KWNAMES_LEN() \
1631 : (call_shape.kwnames == NULL ? 0 : ((int)PyTuple_GET_SIZE(call_shape.kwnames)))
1632 :
1633 : PyObject* _Py_HOT_FUNCTION
1634 133336000 : _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int throwflag)
1635 : {
1636 133336000 : _Py_EnsureTstateNotNULL(tstate);
1637 : CALL_STAT_INC(pyeval_calls);
1638 :
1639 : #if USE_COMPUTED_GOTOS
1640 : /* Import the static jump table */
1641 : #include "opcode_targets.h"
1642 : #endif
1643 :
1644 : #ifdef Py_STATS
1645 : int lastopcode = 0;
1646 : #endif
1647 : // opcode is an 8-bit value to improve the code generated by MSVC
1648 : // for the big switch below (in combination with the EXTRA_CASES macro).
1649 : uint8_t opcode; /* Current opcode */
1650 : int oparg; /* Current opcode argument, if any */
1651 133336000 : _Py_atomic_int * const eval_breaker = &tstate->interp->ceval.eval_breaker;
1652 : #ifdef LLTRACE
1653 133336000 : int lltrace = 0;
1654 : #endif
1655 :
1656 : _PyCFrame cframe;
1657 : CallShape call_shape;
1658 133336000 : call_shape.kwnames = NULL; // Borrowed reference. Reset by CALL instructions.
1659 :
1660 : /* WARNING: Because the _PyCFrame lives on the C stack,
1661 : * but can be accessed from a heap allocated object (tstate)
1662 : * strict stack discipline must be maintained.
1663 : */
1664 133336000 : _PyCFrame *prev_cframe = tstate->cframe;
1665 133336000 : cframe.use_tracing = prev_cframe->use_tracing;
1666 133336000 : cframe.previous = prev_cframe;
1667 133336000 : tstate->cframe = &cframe;
1668 :
1669 133336000 : frame->is_entry = true;
1670 : /* Push frame */
1671 133336000 : frame->previous = prev_cframe->current_frame;
1672 133336000 : cframe.current_frame = frame;
1673 :
1674 : /* support for generator.throw() */
1675 133336000 : if (throwflag) {
1676 609408 : if (_Py_EnterRecursiveCallTstate(tstate, "")) {
1677 0 : tstate->recursion_remaining--;
1678 0 : goto exit_unwind;
1679 : }
1680 609408 : TRACE_FUNCTION_THROW_ENTRY();
1681 609408 : DTRACE_FUNCTION_ENTRY();
1682 609408 : goto resume_with_error;
1683 : }
1684 :
1685 : /* Local "register" variables.
1686 : * These are cached values from the frame and code object. */
1687 :
1688 : PyObject *names;
1689 : PyObject *consts;
1690 : _Py_CODEUNIT *first_instr;
1691 : _Py_CODEUNIT *next_instr;
1692 : PyObject **stack_pointer;
1693 :
1694 : /* Sets the above local variables from the frame */
1695 : #define SET_LOCALS_FROM_FRAME() \
1696 : { \
1697 : PyCodeObject *co = frame->f_code; \
1698 : names = co->co_names; \
1699 : consts = co->co_consts; \
1700 : first_instr = _PyCode_CODE(co); \
1701 : } \
1702 : assert(_PyInterpreterFrame_LASTI(frame) >= -1); \
1703 : /* Jump back to the last instruction executed... */ \
1704 : next_instr = frame->prev_instr + 1; \
1705 : stack_pointer = _PyFrame_GetStackPointer(frame); \
1706 : /* Set stackdepth to -1. \
1707 : Update when returning or calling trace function. \
1708 : Having stackdepth <= 0 ensures that invalid \
1709 : values are not visible to the cycle GC. \
1710 : We choose -1 rather than 0 to assist debugging. \
1711 : */ \
1712 : frame->stacktop = -1;
1713 :
1714 :
1715 132727000 : start_frame:
1716 334935000 : if (_Py_EnterRecursiveCallTstate(tstate, "")) {
1717 8263 : tstate->recursion_remaining--;
1718 8263 : goto exit_unwind;
1719 : }
1720 :
1721 334927000 : resume_frame:
1722 536636000 : SET_LOCALS_FROM_FRAME();
1723 :
1724 : #ifdef LLTRACE
1725 : {
1726 536636000 : int r = PyDict_Contains(GLOBALS(), &_Py_ID(__lltrace__));
1727 536636000 : if (r < 0) {
1728 0 : goto exit_unwind;
1729 : }
1730 536636000 : lltrace = r;
1731 : }
1732 536636000 : if (lltrace) {
1733 5 : lltrace_resume_frame(frame);
1734 : }
1735 : #endif
1736 :
1737 : #ifdef Py_DEBUG
1738 : /* _PyEval_EvalFrameDefault() must not be called with an exception set,
1739 : because it can clear it (directly or indirectly) and so the
1740 : caller loses its exception */
1741 536636000 : assert(!_PyErr_Occurred(tstate));
1742 : #endif
1743 :
1744 536636000 : DISPATCH();
1745 :
1746 54068 : handle_eval_breaker:
1747 :
1748 : /* Do periodic things, like check for signals and async I/0.
1749 : * We need to do reasonably frequently, but not too frequently.
1750 : * All loops should include a check of the eval breaker.
1751 : * We also check on return from any builtin function.
1752 : */
1753 54068 : if (eval_frame_handle_pending(tstate) != 0) {
1754 13 : goto error;
1755 : }
1756 54055 : DISPATCH();
1757 :
1758 : {
1759 : /* Start instructions */
1760 : #if USE_COMPUTED_GOTOS
1761 : {
1762 : #else
1763 : dispatch_opcode:
1764 : switch (opcode) {
1765 : #endif
1766 :
1767 : /* BEWARE!
1768 : It is essential that any operation that fails must goto error
1769 : and that all operation that succeed call DISPATCH() ! */
1770 :
1771 44494500 : TARGET(NOP) {
1772 44494500 : DISPATCH();
1773 : }
1774 :
1775 9105580 : TARGET(RESUME) {
1776 9105580 : _PyCode_Warmup(frame->f_code);
1777 9105580 : JUMP_TO_INSTRUCTION(RESUME_QUICK);
1778 : }
1779 :
1780 308634000 : TARGET(RESUME_QUICK) {
1781 317739000 : PREDICTED(RESUME_QUICK);
1782 317739000 : assert(tstate->cframe == &cframe);
1783 317739000 : assert(frame == cframe.current_frame);
1784 317739000 : if (_Py_atomic_load_relaxed_int32(eval_breaker) && oparg < 2) {
1785 28939 : goto handle_eval_breaker;
1786 : }
1787 317710000 : DISPATCH();
1788 : }
1789 :
1790 4015190 : TARGET(LOAD_CLOSURE) {
1791 : /* We keep LOAD_CLOSURE so that the bytecode stays more readable. */
1792 4015190 : PyObject *value = GETLOCAL(oparg);
1793 4015190 : if (value == NULL) {
1794 0 : goto unbound_local_error;
1795 : }
1796 4015190 : Py_INCREF(value);
1797 4015190 : PUSH(value);
1798 4015190 : DISPATCH();
1799 : }
1800 :
1801 3145430 : TARGET(LOAD_FAST_CHECK) {
1802 3145430 : PyObject *value = GETLOCAL(oparg);
1803 3145430 : if (value == NULL) {
1804 6 : goto unbound_local_error;
1805 : }
1806 3145420 : Py_INCREF(value);
1807 3145420 : PUSH(value);
1808 3145420 : DISPATCH();
1809 : }
1810 :
1811 1107920000 : TARGET(LOAD_FAST) {
1812 1107920000 : PyObject *value = GETLOCAL(oparg);
1813 1107920000 : assert(value != NULL);
1814 1107920000 : Py_INCREF(value);
1815 1107920000 : PUSH(value);
1816 1107920000 : DISPATCH();
1817 : }
1818 :
1819 512932000 : TARGET(LOAD_CONST) {
1820 512932000 : PREDICTED(LOAD_CONST);
1821 512932000 : PyObject *value = GETITEM(consts, oparg);
1822 512932000 : Py_INCREF(value);
1823 512932000 : PUSH(value);
1824 512932000 : DISPATCH();
1825 : }
1826 :
1827 172772000 : TARGET(STORE_FAST) {
1828 172772000 : PyObject *value = POP();
1829 172772000 : SETLOCAL(oparg, value);
1830 172772000 : DISPATCH();
1831 : }
1832 :
1833 380527000 : TARGET(LOAD_FAST__LOAD_FAST) {
1834 380527000 : PyObject *value = GETLOCAL(oparg);
1835 380527000 : assert(value != NULL);
1836 380527000 : NEXTOPARG();
1837 380527000 : next_instr++;
1838 380527000 : Py_INCREF(value);
1839 380527000 : PUSH(value);
1840 380527000 : value = GETLOCAL(oparg);
1841 380527000 : assert(value != NULL);
1842 380527000 : Py_INCREF(value);
1843 380527000 : PUSH(value);
1844 380527000 : NOTRACE_DISPATCH();
1845 : }
1846 :
1847 173713000 : TARGET(LOAD_FAST__LOAD_CONST) {
1848 173713000 : PyObject *value = GETLOCAL(oparg);
1849 173713000 : assert(value != NULL);
1850 173713000 : NEXTOPARG();
1851 173713000 : next_instr++;
1852 173713000 : Py_INCREF(value);
1853 173713000 : PUSH(value);
1854 173713000 : value = GETITEM(consts, oparg);
1855 173713000 : Py_INCREF(value);
1856 173713000 : PUSH(value);
1857 173713000 : NOTRACE_DISPATCH();
1858 : }
1859 :
1860 257632000 : TARGET(STORE_FAST__LOAD_FAST) {
1861 257632000 : PyObject *value = POP();
1862 257632000 : SETLOCAL(oparg, value);
1863 257632000 : NEXTOPARG();
1864 257632000 : next_instr++;
1865 257632000 : value = GETLOCAL(oparg);
1866 257632000 : assert(value != NULL);
1867 257632000 : Py_INCREF(value);
1868 257632000 : PUSH(value);
1869 257632000 : NOTRACE_DISPATCH();
1870 : }
1871 :
1872 107856000 : TARGET(STORE_FAST__STORE_FAST) {
1873 107856000 : PyObject *value = POP();
1874 107856000 : SETLOCAL(oparg, value);
1875 107856000 : NEXTOPARG();
1876 107856000 : next_instr++;
1877 107856000 : value = POP();
1878 107856000 : SETLOCAL(oparg, value);
1879 107856000 : NOTRACE_DISPATCH();
1880 : }
1881 :
1882 71175300 : TARGET(LOAD_CONST__LOAD_FAST) {
1883 71175300 : PyObject *value = GETITEM(consts, oparg);
1884 71175300 : NEXTOPARG();
1885 71175300 : next_instr++;
1886 71175300 : Py_INCREF(value);
1887 71175300 : PUSH(value);
1888 71175300 : value = GETLOCAL(oparg);
1889 71175300 : assert(value != NULL);
1890 71175300 : Py_INCREF(value);
1891 71175300 : PUSH(value);
1892 71175300 : NOTRACE_DISPATCH();
1893 : }
1894 :
1895 189038000 : TARGET(POP_TOP) {
1896 189038000 : PyObject *value = POP();
1897 189038000 : Py_DECREF(value);
1898 189038000 : DISPATCH();
1899 : }
1900 :
1901 99896200 : TARGET(PUSH_NULL) {
1902 : /* Use BASIC_PUSH as NULL is not a valid object pointer */
1903 99896200 : BASIC_PUSH(NULL);
1904 99896200 : DISPATCH();
1905 : }
1906 :
1907 40 : TARGET(UNARY_POSITIVE) {
1908 40 : PyObject *value = TOP();
1909 40 : PyObject *res = PyNumber_Positive(value);
1910 40 : Py_DECREF(value);
1911 40 : SET_TOP(res);
1912 40 : if (res == NULL)
1913 3 : goto error;
1914 37 : DISPATCH();
1915 : }
1916 :
1917 1207700 : TARGET(UNARY_NEGATIVE) {
1918 1207700 : PyObject *value = TOP();
1919 1207700 : PyObject *res = PyNumber_Negative(value);
1920 1207700 : Py_DECREF(value);
1921 1207700 : SET_TOP(res);
1922 1207700 : if (res == NULL)
1923 6 : goto error;
1924 1207700 : DISPATCH();
1925 : }
1926 :
1927 911114 : TARGET(UNARY_NOT) {
1928 911114 : PyObject *value = TOP();
1929 911114 : int err = PyObject_IsTrue(value);
1930 911114 : Py_DECREF(value);
1931 911114 : if (err == 0) {
1932 554670 : Py_INCREF(Py_True);
1933 554670 : SET_TOP(Py_True);
1934 554670 : DISPATCH();
1935 : }
1936 356444 : else if (err > 0) {
1937 356443 : Py_INCREF(Py_False);
1938 356443 : SET_TOP(Py_False);
1939 356443 : DISPATCH();
1940 : }
1941 1 : STACK_SHRINK(1);
1942 1 : goto error;
1943 : }
1944 :
1945 824841 : TARGET(UNARY_INVERT) {
1946 824841 : PyObject *value = TOP();
1947 824841 : PyObject *res = PyNumber_Invert(value);
1948 824841 : Py_DECREF(value);
1949 824841 : SET_TOP(res);
1950 824841 : if (res == NULL)
1951 5 : goto error;
1952 824836 : DISPATCH();
1953 : }
1954 :
1955 5753700 : TARGET(BINARY_OP_MULTIPLY_INT) {
1956 5753700 : assert(cframe.use_tracing == 0);
1957 5753700 : PyObject *left = SECOND();
1958 5753700 : PyObject *right = TOP();
1959 5753700 : DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP);
1960 5747090 : DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP);
1961 : STAT_INC(BINARY_OP, hit);
1962 5694550 : PyObject *prod = _PyLong_Multiply((PyLongObject *)left, (PyLongObject *)right);
1963 5694550 : SET_SECOND(prod);
1964 5694550 : _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free);
1965 5694550 : _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free);
1966 5694550 : STACK_SHRINK(1);
1967 5694550 : if (prod == NULL) {
1968 0 : goto error;
1969 : }
1970 5694550 : JUMPBY(INLINE_CACHE_ENTRIES_BINARY_OP);
1971 5694550 : NOTRACE_DISPATCH();
1972 : }
1973 :
1974 34357400 : TARGET(BINARY_OP_MULTIPLY_FLOAT) {
1975 34357400 : assert(cframe.use_tracing == 0);
1976 34357400 : PyObject *left = SECOND();
1977 34357400 : PyObject *right = TOP();
1978 34357400 : DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP);
1979 34357200 : DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP);
1980 : STAT_INC(BINARY_OP, hit);
1981 34355600 : double dprod = ((PyFloatObject *)left)->ob_fval *
1982 34355600 : ((PyFloatObject *)right)->ob_fval;
1983 34355600 : PyObject *prod = PyFloat_FromDouble(dprod);
1984 34355600 : SET_SECOND(prod);
1985 34355600 : _Py_DECREF_SPECIALIZED(right, _PyFloat_ExactDealloc);
1986 34355600 : _Py_DECREF_SPECIALIZED(left, _PyFloat_ExactDealloc);
1987 34355600 : STACK_SHRINK(1);
1988 34355600 : if (prod == NULL) {
1989 0 : goto error;
1990 : }
1991 34355600 : JUMPBY(INLINE_CACHE_ENTRIES_BINARY_OP);
1992 34355600 : NOTRACE_DISPATCH();
1993 : }
1994 :
1995 20557000 : TARGET(BINARY_OP_SUBTRACT_INT) {
1996 20557000 : assert(cframe.use_tracing == 0);
1997 20557000 : PyObject *left = SECOND();
1998 20557000 : PyObject *right = TOP();
1999 20557000 : DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP);
2000 20556700 : DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP);
2001 : STAT_INC(BINARY_OP, hit);
2002 20556700 : PyObject *sub = _PyLong_Subtract((PyLongObject *)left, (PyLongObject *)right);
2003 20556700 : SET_SECOND(sub);
2004 20556700 : _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free);
2005 20556700 : _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free);
2006 20556700 : STACK_SHRINK(1);
2007 20556700 : if (sub == NULL) {
2008 0 : goto error;
2009 : }
2010 20556700 : JUMPBY(INLINE_CACHE_ENTRIES_BINARY_OP);
2011 20556700 : NOTRACE_DISPATCH();
2012 : }
2013 :
2014 7376710 : TARGET(BINARY_OP_SUBTRACT_FLOAT) {
2015 7376710 : assert(cframe.use_tracing == 0);
2016 7376710 : PyObject *left = SECOND();
2017 7376710 : PyObject *right = TOP();
2018 7376710 : DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP);
2019 7376230 : DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP);
2020 : STAT_INC(BINARY_OP, hit);
2021 7375070 : double dsub = ((PyFloatObject *)left)->ob_fval - ((PyFloatObject *)right)->ob_fval;
2022 7375070 : PyObject *sub = PyFloat_FromDouble(dsub);
2023 7375070 : SET_SECOND(sub);
2024 7375070 : _Py_DECREF_SPECIALIZED(right, _PyFloat_ExactDealloc);
2025 7375070 : _Py_DECREF_SPECIALIZED(left, _PyFloat_ExactDealloc);
2026 7375070 : STACK_SHRINK(1);
2027 7375070 : if (sub == NULL) {
2028 0 : goto error;
2029 : }
2030 7375070 : JUMPBY(INLINE_CACHE_ENTRIES_BINARY_OP);
2031 7375070 : NOTRACE_DISPATCH();
2032 : }
2033 :
2034 26952500 : TARGET(BINARY_OP_ADD_UNICODE) {
2035 26952500 : assert(cframe.use_tracing == 0);
2036 26952500 : PyObject *left = SECOND();
2037 26952500 : PyObject *right = TOP();
2038 26952500 : DEOPT_IF(!PyUnicode_CheckExact(left), BINARY_OP);
2039 26949300 : DEOPT_IF(Py_TYPE(right) != Py_TYPE(left), BINARY_OP);
2040 : STAT_INC(BINARY_OP, hit);
2041 26949300 : PyObject *res = PyUnicode_Concat(left, right);
2042 26949300 : STACK_SHRINK(1);
2043 26949300 : SET_TOP(res);
2044 26949300 : _Py_DECREF_SPECIALIZED(left, _PyUnicode_ExactDealloc);
2045 26949300 : _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc);
2046 26949300 : if (TOP() == NULL) {
2047 0 : goto error;
2048 : }
2049 26949300 : JUMPBY(INLINE_CACHE_ENTRIES_BINARY_OP);
2050 26949300 : NOTRACE_DISPATCH();
2051 : }
2052 :
2053 3595980 : TARGET(BINARY_OP_INPLACE_ADD_UNICODE) {
2054 3595980 : assert(cframe.use_tracing == 0);
2055 3595980 : PyObject *left = SECOND();
2056 3595980 : PyObject *right = TOP();
2057 3595980 : DEOPT_IF(!PyUnicode_CheckExact(left), BINARY_OP);
2058 3592860 : DEOPT_IF(Py_TYPE(right) != Py_TYPE(left), BINARY_OP);
2059 3592860 : _Py_CODEUNIT true_next = next_instr[INLINE_CACHE_ENTRIES_BINARY_OP];
2060 3592860 : assert(_Py_OPCODE(true_next) == STORE_FAST ||
2061 : _Py_OPCODE(true_next) == STORE_FAST__LOAD_FAST);
2062 3592860 : PyObject **target_local = &GETLOCAL(_Py_OPARG(true_next));
2063 3592860 : DEOPT_IF(*target_local != left, BINARY_OP);
2064 : STAT_INC(BINARY_OP, hit);
2065 : /* Handle `left = left + right` or `left += right` for str.
2066 : *
2067 : * When possible, extend `left` in place rather than
2068 : * allocating a new PyUnicodeObject. This attempts to avoid
2069 : * quadratic behavior when one neglects to use str.join().
2070 : *
2071 : * If `left` has only two references remaining (one from
2072 : * the stack, one in the locals), DECREFing `left` leaves
2073 : * only the locals reference, so PyUnicode_Append knows
2074 : * that the string is safe to mutate.
2075 : */
2076 3592490 : assert(Py_REFCNT(left) >= 2);
2077 3592490 : _Py_DECREF_NO_DEALLOC(left);
2078 3592490 : STACK_SHRINK(2);
2079 3592490 : PyUnicode_Append(target_local, right);
2080 3592490 : _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc);
2081 3592490 : if (*target_local == NULL) {
2082 0 : goto error;
2083 : }
2084 : // The STORE_FAST is already done.
2085 3592490 : JUMPBY(INLINE_CACHE_ENTRIES_BINARY_OP + 1);
2086 3592490 : NOTRACE_DISPATCH();
2087 : }
2088 :
2089 18846300 : TARGET(BINARY_OP_ADD_FLOAT) {
2090 18846300 : assert(cframe.use_tracing == 0);
2091 18846300 : PyObject *left = SECOND();
2092 18846300 : PyObject *right = TOP();
2093 18846300 : DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP);
2094 18843700 : DEOPT_IF(Py_TYPE(right) != Py_TYPE(left), BINARY_OP);
2095 : STAT_INC(BINARY_OP, hit);
2096 18843100 : double dsum = ((PyFloatObject *)left)->ob_fval +
2097 18843100 : ((PyFloatObject *)right)->ob_fval;
2098 18843100 : PyObject *sum = PyFloat_FromDouble(dsum);
2099 18843100 : SET_SECOND(sum);
2100 18843100 : _Py_DECREF_SPECIALIZED(right, _PyFloat_ExactDealloc);
2101 18843100 : _Py_DECREF_SPECIALIZED(left, _PyFloat_ExactDealloc);
2102 18843100 : STACK_SHRINK(1);
2103 18843100 : if (sum == NULL) {
2104 0 : goto error;
2105 : }
2106 18843100 : JUMPBY(INLINE_CACHE_ENTRIES_BINARY_OP);
2107 18843100 : NOTRACE_DISPATCH();
2108 : }
2109 :
2110 45105500 : TARGET(BINARY_OP_ADD_INT) {
2111 45105500 : assert(cframe.use_tracing == 0);
2112 45105500 : PyObject *left = SECOND();
2113 45105500 : PyObject *right = TOP();
2114 45105500 : DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP);
2115 45104600 : DEOPT_IF(Py_TYPE(right) != Py_TYPE(left), BINARY_OP);
2116 : STAT_INC(BINARY_OP, hit);
2117 45090000 : PyObject *sum = _PyLong_Add((PyLongObject *)left, (PyLongObject *)right);
2118 45090000 : SET_SECOND(sum);
2119 45090000 : _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free);
2120 45090000 : _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free);
2121 45090000 : STACK_SHRINK(1);
2122 45090000 : if (sum == NULL) {
2123 0 : goto error;
2124 : }
2125 45090000 : JUMPBY(INLINE_CACHE_ENTRIES_BINARY_OP);
2126 45090000 : NOTRACE_DISPATCH();
2127 : }
2128 :
2129 9468810 : TARGET(BINARY_SUBSCR) {
2130 76575000 : PREDICTED(BINARY_SUBSCR);
2131 76575000 : PyObject *sub = POP();
2132 76575000 : PyObject *container = TOP();
2133 76575000 : PyObject *res = PyObject_GetItem(container, sub);
2134 76575000 : Py_DECREF(container);
2135 76575000 : Py_DECREF(sub);
2136 76575000 : SET_TOP(res);
2137 76575000 : if (res == NULL)
2138 493839 : goto error;
2139 76081200 : JUMPBY(INLINE_CACHE_ENTRIES_BINARY_SUBSCR);
2140 76081200 : DISPATCH();
2141 : }
2142 :
2143 48203700 : TARGET(BINARY_SLICE) {
2144 48203700 : PyObject *stop = POP();
2145 48203700 : PyObject *start = POP();
2146 48203700 : PyObject *container = TOP();
2147 :
2148 48203700 : PyObject *slice = _PyBuildSlice_ConsumeRefs(start, stop);
2149 48203700 : if (slice == NULL) {
2150 0 : goto error;
2151 : }
2152 48203700 : PyObject *res = PyObject_GetItem(container, slice);
2153 48203700 : Py_DECREF(slice);
2154 48203700 : if (res == NULL) {
2155 31 : goto error;
2156 : }
2157 48203700 : SET_TOP(res);
2158 48203700 : Py_DECREF(container);
2159 48203700 : DISPATCH();
2160 : }
2161 :
2162 235603 : TARGET(STORE_SLICE) {
2163 235603 : PyObject *stop = POP();
2164 235603 : PyObject *start = POP();
2165 235603 : PyObject *container = TOP();
2166 235603 : PyObject *v = SECOND();
2167 :
2168 235603 : PyObject *slice = _PyBuildSlice_ConsumeRefs(start, stop);
2169 235603 : if (slice == NULL) {
2170 0 : goto error;
2171 : }
2172 235603 : int err = PyObject_SetItem(container, slice, v);
2173 235603 : Py_DECREF(slice);
2174 235603 : if (err) {
2175 23 : goto error;
2176 : }
2177 235580 : STACK_SHRINK(2);
2178 235580 : Py_DECREF(v);
2179 235580 : Py_DECREF(container);
2180 235580 : DISPATCH();
2181 : }
2182 :
2183 67706200 : TARGET(BINARY_SUBSCR_ADAPTIVE) {
2184 67706200 : _PyBinarySubscrCache *cache = (_PyBinarySubscrCache *)next_instr;
2185 67706200 : if (ADAPTIVE_COUNTER_IS_ZERO(cache)) {
2186 599947 : PyObject *sub = TOP();
2187 599947 : PyObject *container = SECOND();
2188 599947 : next_instr--;
2189 599947 : if (_Py_Specialize_BinarySubscr(container, sub, next_instr) < 0) {
2190 0 : goto error;
2191 : }
2192 599947 : NOTRACE_DISPATCH_SAME_OPARG();
2193 : }
2194 : else {
2195 : STAT_INC(BINARY_SUBSCR, deferred);
2196 67106200 : DECREMENT_ADAPTIVE_COUNTER(cache);
2197 67106200 : JUMP_TO_INSTRUCTION(BINARY_SUBSCR);
2198 : }
2199 : }
2200 :
2201 33710900 : TARGET(BINARY_SUBSCR_LIST_INT) {
2202 33710900 : assert(cframe.use_tracing == 0);
2203 33710900 : PyObject *sub = TOP();
2204 33710900 : PyObject *list = SECOND();
2205 33710900 : DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR);
2206 33710800 : DEOPT_IF(!PyList_CheckExact(list), BINARY_SUBSCR);
2207 :
2208 : // Deopt unless 0 <= sub < PyList_Size(list)
2209 33620800 : Py_ssize_t signed_magnitude = Py_SIZE(sub);
2210 33620800 : DEOPT_IF(((size_t)signed_magnitude) > 1, BINARY_SUBSCR);
2211 27667700 : assert(((PyLongObject *)_PyLong_GetZero())->ob_digit[0] == 0);
2212 27667700 : Py_ssize_t index = ((PyLongObject*)sub)->ob_digit[0];
2213 27667700 : DEOPT_IF(index >= PyList_GET_SIZE(list), BINARY_SUBSCR);
2214 : STAT_INC(BINARY_SUBSCR, hit);
2215 27346300 : PyObject *res = PyList_GET_ITEM(list, index);
2216 27346300 : assert(res != NULL);
2217 27346300 : Py_INCREF(res);
2218 27346300 : STACK_SHRINK(1);
2219 27346300 : _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free);
2220 27346300 : SET_TOP(res);
2221 27346300 : Py_DECREF(list);
2222 27346300 : JUMPBY(INLINE_CACHE_ENTRIES_BINARY_SUBSCR);
2223 27346300 : NOTRACE_DISPATCH();
2224 : }
2225 :
2226 16368200 : TARGET(BINARY_SUBSCR_TUPLE_INT) {
2227 16368200 : assert(cframe.use_tracing == 0);
2228 16368200 : PyObject *sub = TOP();
2229 16368200 : PyObject *tuple = SECOND();
2230 16368200 : DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR);
2231 16368200 : DEOPT_IF(!PyTuple_CheckExact(tuple), BINARY_SUBSCR);
2232 :
2233 : // Deopt unless 0 <= sub < PyTuple_Size(list)
2234 15721700 : Py_ssize_t signed_magnitude = Py_SIZE(sub);
2235 15721700 : DEOPT_IF(((size_t)signed_magnitude) > 1, BINARY_SUBSCR);
2236 14540000 : assert(((PyLongObject *)_PyLong_GetZero())->ob_digit[0] == 0);
2237 14540000 : Py_ssize_t index = ((PyLongObject*)sub)->ob_digit[0];
2238 14540000 : DEOPT_IF(index >= PyTuple_GET_SIZE(tuple), BINARY_SUBSCR);
2239 : STAT_INC(BINARY_SUBSCR, hit);
2240 14539700 : PyObject *res = PyTuple_GET_ITEM(tuple, index);
2241 14539700 : assert(res != NULL);
2242 14539700 : Py_INCREF(res);
2243 14539700 : STACK_SHRINK(1);
2244 14539700 : _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free);
2245 14539700 : SET_TOP(res);
2246 14539700 : Py_DECREF(tuple);
2247 14539700 : JUMPBY(INLINE_CACHE_ENTRIES_BINARY_SUBSCR);
2248 14539700 : NOTRACE_DISPATCH();
2249 : }
2250 :
2251 14939800 : TARGET(BINARY_SUBSCR_DICT) {
2252 14939800 : assert(cframe.use_tracing == 0);
2253 14939800 : PyObject *dict = SECOND();
2254 14939800 : DEOPT_IF(!PyDict_CheckExact(SECOND()), BINARY_SUBSCR);
2255 : STAT_INC(BINARY_SUBSCR, hit);
2256 14937300 : PyObject *sub = TOP();
2257 14937300 : PyObject *res = PyDict_GetItemWithError(dict, sub);
2258 14937300 : if (res == NULL) {
2259 743847 : goto binary_subscr_dict_error;
2260 : }
2261 14193500 : Py_INCREF(res);
2262 14193500 : STACK_SHRINK(1);
2263 14193500 : Py_DECREF(sub);
2264 14193500 : SET_TOP(res);
2265 14193500 : Py_DECREF(dict);
2266 14193500 : JUMPBY(INLINE_CACHE_ENTRIES_BINARY_SUBSCR);
2267 14193500 : DISPATCH();
2268 : }
2269 :
2270 1741580 : TARGET(BINARY_SUBSCR_GETITEM) {
2271 1741580 : PyObject *sub = TOP();
2272 1741580 : PyObject *container = SECOND();
2273 1741580 : _PyBinarySubscrCache *cache = (_PyBinarySubscrCache *)next_instr;
2274 1741580 : uint32_t type_version = read_u32(cache->type_version);
2275 1741580 : PyTypeObject *tp = Py_TYPE(container);
2276 1741580 : DEOPT_IF(tp->tp_version_tag != type_version, BINARY_SUBSCR);
2277 1738240 : assert(tp->tp_flags & Py_TPFLAGS_HEAPTYPE);
2278 1738240 : PyObject *cached = ((PyHeapTypeObject *)tp)->_spec_cache.getitem;
2279 1738240 : assert(PyFunction_Check(cached));
2280 1738240 : PyFunctionObject *getitem = (PyFunctionObject *)cached;
2281 1738240 : DEOPT_IF(getitem->func_version != cache->func_version, BINARY_SUBSCR);
2282 1738240 : PyCodeObject *code = (PyCodeObject *)getitem->func_code;
2283 1738240 : assert(code->co_argcount == 2);
2284 1738240 : DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), BINARY_SUBSCR);
2285 : STAT_INC(BINARY_SUBSCR, hit);
2286 1738220 : Py_INCREF(getitem);
2287 1738220 : _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, getitem);
2288 : CALL_STAT_INC(inlined_py_calls);
2289 1738220 : STACK_SHRINK(2);
2290 1738220 : new_frame->localsplus[0] = container;
2291 1738220 : new_frame->localsplus[1] = sub;
2292 1978920 : for (int i = 2; i < code->co_nlocalsplus; i++) {
2293 240707 : new_frame->localsplus[i] = NULL;
2294 : }
2295 1738220 : _PyFrame_SetStackPointer(frame, stack_pointer);
2296 1738220 : JUMPBY(INLINE_CACHE_ENTRIES_BINARY_SUBSCR);
2297 1738220 : frame->prev_instr = next_instr - 1;
2298 1738220 : new_frame->previous = frame;
2299 1738220 : frame = cframe.current_frame = new_frame;
2300 : CALL_STAT_INC(inlined_py_calls);
2301 1738220 : goto start_frame;
2302 : }
2303 :
2304 30195000 : TARGET(LIST_APPEND) {
2305 30195000 : PyObject *v = POP();
2306 30195000 : PyObject *list = PEEK(oparg);
2307 30195000 : if (_PyList_AppendTakeRef((PyListObject *)list, v) < 0)
2308 0 : goto error;
2309 : PREDICT(JUMP_BACKWARD_QUICK);
2310 30195000 : DISPATCH();
2311 : }
2312 :
2313 1238510 : TARGET(SET_ADD) {
2314 1238510 : PyObject *v = POP();
2315 1238510 : PyObject *set = PEEK(oparg);
2316 : int err;
2317 1238510 : err = PySet_Add(set, v);
2318 1238510 : Py_DECREF(v);
2319 1238510 : if (err != 0)
2320 0 : goto error;
2321 : PREDICT(JUMP_BACKWARD_QUICK);
2322 1238510 : DISPATCH();
2323 : }
2324 :
2325 1277120 : TARGET(STORE_SUBSCR) {
2326 9400590 : PREDICTED(STORE_SUBSCR);
2327 9400590 : PyObject *sub = TOP();
2328 9400590 : PyObject *container = SECOND();
2329 9400590 : PyObject *v = THIRD();
2330 : int err;
2331 9400590 : STACK_SHRINK(3);
2332 : /* container[sub] = v */
2333 9400590 : err = PyObject_SetItem(container, sub, v);
2334 9400590 : Py_DECREF(v);
2335 9400590 : Py_DECREF(container);
2336 9400590 : Py_DECREF(sub);
2337 9400590 : if (err != 0) {
2338 9120 : goto error;
2339 : }
2340 9391470 : JUMPBY(INLINE_CACHE_ENTRIES_STORE_SUBSCR);
2341 9391470 : DISPATCH();
2342 : }
2343 :
2344 8401270 : TARGET(STORE_SUBSCR_ADAPTIVE) {
2345 8401270 : _PyStoreSubscrCache *cache = (_PyStoreSubscrCache *)next_instr;
2346 8401270 : if (ADAPTIVE_COUNTER_IS_ZERO(cache)) {
2347 277798 : PyObject *sub = TOP();
2348 277798 : PyObject *container = SECOND();
2349 277798 : next_instr--;
2350 277798 : if (_Py_Specialize_StoreSubscr(container, sub, next_instr) < 0) {
2351 0 : goto error;
2352 : }
2353 277798 : NOTRACE_DISPATCH_SAME_OPARG();
2354 : }
2355 : else {
2356 : STAT_INC(STORE_SUBSCR, deferred);
2357 8123470 : DECREMENT_ADAPTIVE_COUNTER(cache);
2358 8123470 : JUMP_TO_INSTRUCTION(STORE_SUBSCR);
2359 : }
2360 : }
2361 :
2362 4879500 : TARGET(STORE_SUBSCR_LIST_INT) {
2363 4879500 : assert(cframe.use_tracing == 0);
2364 4879500 : PyObject *sub = TOP();
2365 4879500 : PyObject *list = SECOND();
2366 4879500 : PyObject *value = THIRD();
2367 4879500 : DEOPT_IF(!PyLong_CheckExact(sub), STORE_SUBSCR);
2368 4879500 : DEOPT_IF(!PyList_CheckExact(list), STORE_SUBSCR);
2369 :
2370 : // Ensure nonnegative, zero-or-one-digit ints.
2371 4879500 : DEOPT_IF(((size_t)Py_SIZE(sub)) > 1, STORE_SUBSCR);
2372 4879130 : Py_ssize_t index = ((PyLongObject*)sub)->ob_digit[0];
2373 : // Ensure index < len(list)
2374 4879130 : DEOPT_IF(index >= PyList_GET_SIZE(list), STORE_SUBSCR);
2375 : STAT_INC(STORE_SUBSCR, hit);
2376 :
2377 4879130 : PyObject *old_value = PyList_GET_ITEM(list, index);
2378 4879130 : PyList_SET_ITEM(list, index, value);
2379 4879130 : STACK_SHRINK(3);
2380 4879130 : assert(old_value != NULL);
2381 4879130 : Py_DECREF(old_value);
2382 4879130 : _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free);
2383 4879130 : Py_DECREF(list);
2384 4879130 : JUMPBY(INLINE_CACHE_ENTRIES_STORE_SUBSCR);
2385 4879130 : NOTRACE_DISPATCH();
2386 : }
2387 :
2388 11523500 : TARGET(STORE_SUBSCR_DICT) {
2389 11523500 : assert(cframe.use_tracing == 0);
2390 11523500 : PyObject *sub = TOP();
2391 11523500 : PyObject *dict = SECOND();
2392 11523500 : PyObject *value = THIRD();
2393 11523500 : DEOPT_IF(!PyDict_CheckExact(dict), STORE_SUBSCR);
2394 11522900 : STACK_SHRINK(3);
2395 : STAT_INC(STORE_SUBSCR, hit);
2396 11522900 : int err = _PyDict_SetItem_Take2((PyDictObject *)dict, sub, value);
2397 11522900 : Py_DECREF(dict);
2398 11522900 : if (err != 0) {
2399 3 : goto error;
2400 : }
2401 11522900 : JUMPBY(INLINE_CACHE_ENTRIES_STORE_SUBSCR);
2402 11522900 : DISPATCH();
2403 : }
2404 :
2405 1653160 : TARGET(DELETE_SUBSCR) {
2406 1653160 : PyObject *sub = TOP();
2407 1653160 : PyObject *container = SECOND();
2408 : int err;
2409 1653160 : STACK_SHRINK(2);
2410 : /* del container[sub] */
2411 1653160 : err = PyObject_DelItem(container, sub);
2412 1653160 : Py_DECREF(container);
2413 1653160 : Py_DECREF(sub);
2414 1653160 : if (err != 0)
2415 2429 : goto error;
2416 1650730 : DISPATCH();
2417 : }
2418 :
2419 3070 : TARGET(PRINT_EXPR) {
2420 3070 : PyObject *value = POP();
2421 3070 : PyObject *hook = _PySys_GetAttr(tstate, &_Py_ID(displayhook));
2422 : PyObject *res;
2423 3070 : if (hook == NULL) {
2424 1 : _PyErr_SetString(tstate, PyExc_RuntimeError,
2425 : "lost sys.displayhook");
2426 1 : Py_DECREF(value);
2427 1 : goto error;
2428 : }
2429 3069 : res = PyObject_CallOneArg(hook, value);
2430 3069 : Py_DECREF(value);
2431 3069 : if (res == NULL)
2432 1 : goto error;
2433 3068 : Py_DECREF(res);
2434 3068 : DISPATCH();
2435 : }
2436 :
2437 382971 : TARGET(RAISE_VARARGS) {
2438 382971 : PyObject *cause = NULL, *exc = NULL;
2439 382971 : switch (oparg) {
2440 122065 : case 2:
2441 122065 : cause = POP(); /* cause */
2442 : /* fall through */
2443 368990 : case 1:
2444 368990 : exc = POP(); /* exc */
2445 : /* fall through */
2446 382971 : case 0:
2447 382971 : if (do_raise(tstate, exc, cause)) {
2448 13979 : goto exception_unwind;
2449 : }
2450 368992 : break;
2451 0 : default:
2452 0 : _PyErr_SetString(tstate, PyExc_SystemError,
2453 : "bad RAISE_VARARGS oparg");
2454 0 : break;
2455 : }
2456 368992 : goto error;
2457 : }
2458 :
2459 271709000 : TARGET(RETURN_VALUE) {
2460 271709000 : PyObject *retval = POP();
2461 271709000 : assert(EMPTY());
2462 271709000 : _PyFrame_SetStackPointer(frame, stack_pointer);
2463 271709000 : TRACE_FUNCTION_EXIT();
2464 271708000 : DTRACE_FUNCTION_EXIT();
2465 271708000 : _Py_LeaveRecursiveCallTstate(tstate);
2466 271708000 : if (!frame->is_entry) {
2467 185684000 : frame = cframe.current_frame = pop_frame(tstate, frame);
2468 185684000 : _PyFrame_StackPush(frame, retval);
2469 185684000 : goto resume_frame;
2470 : }
2471 : /* Restore previous cframe and return. */
2472 86024200 : tstate->cframe = cframe.previous;
2473 86024200 : tstate->cframe->use_tracing = cframe.use_tracing;
2474 86024200 : assert(tstate->cframe->current_frame == frame->previous);
2475 86024200 : assert(!_PyErr_Occurred(tstate));
2476 86024200 : return retval;
2477 : }
2478 :
2479 158 : TARGET(GET_AITER) {
2480 158 : unaryfunc getter = NULL;
2481 158 : PyObject *iter = NULL;
2482 158 : PyObject *obj = TOP();
2483 158 : PyTypeObject *type = Py_TYPE(obj);
2484 :
2485 158 : if (type->tp_as_async != NULL) {
2486 157 : getter = type->tp_as_async->am_aiter;
2487 : }
2488 :
2489 158 : if (getter != NULL) {
2490 157 : iter = (*getter)(obj);
2491 157 : Py_DECREF(obj);
2492 157 : if (iter == NULL) {
2493 2 : SET_TOP(NULL);
2494 2 : goto error;
2495 : }
2496 : }
2497 : else {
2498 1 : SET_TOP(NULL);
2499 1 : _PyErr_Format(tstate, PyExc_TypeError,
2500 : "'async for' requires an object with "
2501 : "__aiter__ method, got %.100s",
2502 : type->tp_name);
2503 1 : Py_DECREF(obj);
2504 1 : goto error;
2505 : }
2506 :
2507 155 : if (Py_TYPE(iter)->tp_as_async == NULL ||
2508 155 : Py_TYPE(iter)->tp_as_async->am_anext == NULL) {
2509 :
2510 1 : SET_TOP(NULL);
2511 1 : _PyErr_Format(tstate, PyExc_TypeError,
2512 : "'async for' received an object from __aiter__ "
2513 : "that does not implement __anext__: %.100s",
2514 1 : Py_TYPE(iter)->tp_name);
2515 1 : Py_DECREF(iter);
2516 1 : goto error;
2517 : }
2518 :
2519 154 : SET_TOP(iter);
2520 154 : DISPATCH();
2521 : }
2522 :
2523 763 : TARGET(GET_ANEXT) {
2524 763 : unaryfunc getter = NULL;
2525 763 : PyObject *next_iter = NULL;
2526 763 : PyObject *awaitable = NULL;
2527 763 : PyObject *aiter = TOP();
2528 763 : PyTypeObject *type = Py_TYPE(aiter);
2529 :
2530 763 : if (PyAsyncGen_CheckExact(aiter)) {
2531 434 : awaitable = type->tp_as_async->am_anext(aiter);
2532 434 : if (awaitable == NULL) {
2533 0 : goto error;
2534 : }
2535 : } else {
2536 329 : if (type->tp_as_async != NULL){
2537 329 : getter = type->tp_as_async->am_anext;
2538 : }
2539 :
2540 329 : if (getter != NULL) {
2541 329 : next_iter = (*getter)(aiter);
2542 329 : if (next_iter == NULL) {
2543 0 : goto error;
2544 : }
2545 : }
2546 : else {
2547 0 : _PyErr_Format(tstate, PyExc_TypeError,
2548 : "'async for' requires an iterator with "
2549 : "__anext__ method, got %.100s",
2550 : type->tp_name);
2551 0 : goto error;
2552 : }
2553 :
2554 329 : awaitable = _PyCoro_GetAwaitableIter(next_iter);
2555 329 : if (awaitable == NULL) {
2556 2 : _PyErr_FormatFromCause(
2557 : PyExc_TypeError,
2558 : "'async for' received an invalid object "
2559 : "from __anext__: %.100s",
2560 2 : Py_TYPE(next_iter)->tp_name);
2561 :
2562 2 : Py_DECREF(next_iter);
2563 2 : goto error;
2564 : } else {
2565 327 : Py_DECREF(next_iter);
2566 : }
2567 : }
2568 :
2569 761 : PUSH(awaitable);
2570 : PREDICT(LOAD_CONST);
2571 761 : DISPATCH();
2572 : }
2573 :
2574 31052 : TARGET(GET_AWAITABLE) {
2575 31052 : PREDICTED(GET_AWAITABLE);
2576 31052 : PyObject *iterable = TOP();
2577 31052 : PyObject *iter = _PyCoro_GetAwaitableIter(iterable);
2578 :
2579 31052 : if (iter == NULL) {
2580 28 : format_awaitable_error(tstate, Py_TYPE(iterable), oparg);
2581 : }
2582 :
2583 31052 : Py_DECREF(iterable);
2584 :
2585 31052 : if (iter != NULL && PyCoro_CheckExact(iter)) {
2586 17086 : PyObject *yf = _PyGen_yf((PyGenObject*)iter);
2587 17086 : if (yf != NULL) {
2588 : /* `iter` is a coroutine object that is being
2589 : awaited, `yf` is a pointer to the current awaitable
2590 : being awaited on. */
2591 2 : Py_DECREF(yf);
2592 2 : Py_CLEAR(iter);
2593 2 : _PyErr_SetString(tstate, PyExc_RuntimeError,
2594 : "coroutine is being awaited already");
2595 : /* The code below jumps to `error` if `iter` is NULL. */
2596 : }
2597 : }
2598 :
2599 31052 : SET_TOP(iter); /* Even if it's NULL */
2600 :
2601 31052 : if (iter == NULL) {
2602 30 : goto error;
2603 : }
2604 :
2605 : PREDICT(LOAD_CONST);
2606 31022 : DISPATCH();
2607 : }
2608 :
2609 6788070 : TARGET(SEND) {
2610 6788070 : assert(frame->is_entry);
2611 6788070 : assert(STACK_LEVEL() >= 2);
2612 6788070 : PyObject *v = POP();
2613 6788070 : PyObject *receiver = TOP();
2614 : PySendResult gen_status;
2615 : PyObject *retval;
2616 6788070 : if (tstate->c_tracefunc == NULL) {
2617 6787050 : gen_status = PyIter_Send(receiver, v, &retval);
2618 : } else {
2619 1022 : if (Py_IsNone(v) && PyIter_Check(receiver)) {
2620 962 : retval = Py_TYPE(receiver)->tp_iternext(receiver);
2621 : }
2622 : else {
2623 60 : retval = PyObject_CallMethodOneArg(receiver, &_Py_ID(send), v);
2624 : }
2625 1022 : if (retval == NULL) {
2626 198 : if (tstate->c_tracefunc != NULL
2627 198 : && _PyErr_ExceptionMatches(tstate, PyExc_StopIteration))
2628 81 : call_exc_trace(tstate->c_tracefunc, tstate->c_traceobj, tstate, frame);
2629 198 : if (_PyGen_FetchStopIterationValue(&retval) == 0) {
2630 187 : gen_status = PYGEN_RETURN;
2631 : }
2632 : else {
2633 11 : gen_status = PYGEN_ERROR;
2634 : }
2635 : }
2636 : else {
2637 824 : gen_status = PYGEN_NEXT;
2638 : }
2639 : }
2640 6788070 : Py_DECREF(v);
2641 6788070 : if (gen_status == PYGEN_ERROR) {
2642 728 : assert(retval == NULL);
2643 728 : goto error;
2644 : }
2645 6787340 : if (gen_status == PYGEN_RETURN) {
2646 786852 : assert(retval != NULL);
2647 786852 : Py_DECREF(receiver);
2648 786852 : SET_TOP(retval);
2649 786852 : JUMPBY(oparg);
2650 786852 : DISPATCH();
2651 : }
2652 6000490 : assert(gen_status == PYGEN_NEXT);
2653 6000490 : assert(retval != NULL);
2654 6000490 : PUSH(retval);
2655 6000490 : DISPATCH();
2656 : }
2657 :
2658 434 : TARGET(ASYNC_GEN_WRAP) {
2659 434 : PyObject *v = TOP();
2660 434 : assert(frame->f_code->co_flags & CO_ASYNC_GENERATOR);
2661 434 : PyObject *w = _PyAsyncGenValueWrapperNew(v);
2662 434 : if (w == NULL) {
2663 0 : goto error;
2664 : }
2665 434 : SET_TOP(w);
2666 434 : Py_DECREF(v);
2667 434 : DISPATCH();
2668 : }
2669 :
2670 44958600 : TARGET(YIELD_VALUE) {
2671 44958600 : assert(oparg == STACK_LEVEL());
2672 44958600 : assert(frame->is_entry);
2673 44958600 : PyObject *retval = POP();
2674 44958600 : _PyFrame_GetGenerator(frame)->gi_frame_state = FRAME_SUSPENDED;
2675 44958600 : _PyFrame_SetStackPointer(frame, stack_pointer);
2676 44958600 : TRACE_FUNCTION_EXIT();
2677 44958600 : DTRACE_FUNCTION_EXIT();
2678 44958600 : _Py_LeaveRecursiveCallTstate(tstate);
2679 : /* Restore previous cframe and return. */
2680 44958600 : tstate->cframe = cframe.previous;
2681 44958600 : tstate->cframe->use_tracing = cframe.use_tracing;
2682 44958600 : assert(tstate->cframe->current_frame == frame->previous);
2683 44958600 : assert(!_PyErr_Occurred(tstate));
2684 44958600 : return retval;
2685 : }
2686 :
2687 4923040 : TARGET(POP_EXCEPT) {
2688 4923040 : _PyErr_StackItem *exc_info = tstate->exc_info;
2689 4923040 : PyObject *value = exc_info->exc_value;
2690 4923040 : exc_info->exc_value = POP();
2691 4923040 : Py_XDECREF(value);
2692 4923040 : DISPATCH();
2693 : }
2694 :
2695 670243 : TARGET(RERAISE) {
2696 670243 : if (oparg) {
2697 449115 : PyObject *lasti = PEEK(oparg + 1);
2698 449115 : if (PyLong_Check(lasti)) {
2699 449115 : frame->prev_instr = first_instr + PyLong_AsLong(lasti);
2700 449115 : assert(!_PyErr_Occurred(tstate));
2701 : }
2702 : else {
2703 0 : assert(PyLong_Check(lasti));
2704 0 : _PyErr_SetString(tstate, PyExc_SystemError, "lasti is not an int");
2705 0 : goto error;
2706 : }
2707 : }
2708 670243 : PyObject *val = POP();
2709 670243 : assert(val && PyExceptionInstance_Check(val));
2710 670243 : PyObject *exc = Py_NewRef(PyExceptionInstance_Class(val));
2711 670243 : PyObject *tb = PyException_GetTraceback(val);
2712 670243 : _PyErr_Restore(tstate, exc, val, tb);
2713 670243 : goto exception_unwind;
2714 : }
2715 :
2716 150 : TARGET(PREP_RERAISE_STAR) {
2717 150 : PyObject *excs = POP();
2718 150 : assert(PyList_Check(excs));
2719 150 : PyObject *orig = POP();
2720 :
2721 150 : PyObject *val = _PyExc_PrepReraiseStar(orig, excs);
2722 150 : Py_DECREF(excs);
2723 150 : Py_DECREF(orig);
2724 :
2725 150 : if (val == NULL) {
2726 0 : goto error;
2727 : }
2728 :
2729 150 : PUSH(val);
2730 150 : DISPATCH();
2731 : }
2732 :
2733 140 : TARGET(END_ASYNC_FOR) {
2734 140 : PyObject *val = POP();
2735 140 : assert(val && PyExceptionInstance_Check(val));
2736 140 : if (PyErr_GivenExceptionMatches(val, PyExc_StopAsyncIteration)) {
2737 130 : Py_DECREF(val);
2738 130 : Py_DECREF(POP());
2739 130 : DISPATCH();
2740 : }
2741 : else {
2742 10 : PyObject *exc = Py_NewRef(PyExceptionInstance_Class(val));
2743 10 : PyObject *tb = PyException_GetTraceback(val);
2744 10 : _PyErr_Restore(tstate, exc, val, tb);
2745 10 : goto exception_unwind;
2746 : }
2747 : }
2748 :
2749 27 : TARGET(LOAD_ASSERTION_ERROR) {
2750 27 : PyObject *value = PyExc_AssertionError;
2751 27 : Py_INCREF(value);
2752 27 : PUSH(value);
2753 27 : DISPATCH();
2754 : }
2755 :
2756 950091 : TARGET(LOAD_BUILD_CLASS) {
2757 : PyObject *bc;
2758 950091 : if (PyDict_CheckExact(BUILTINS())) {
2759 950091 : bc = _PyDict_GetItemWithError(BUILTINS(),
2760 : &_Py_ID(__build_class__));
2761 950091 : if (bc == NULL) {
2762 1 : if (!_PyErr_Occurred(tstate)) {
2763 1 : _PyErr_SetString(tstate, PyExc_NameError,
2764 : "__build_class__ not found");
2765 : }
2766 1 : goto error;
2767 : }
2768 950090 : Py_INCREF(bc);
2769 : }
2770 : else {
2771 0 : bc = PyObject_GetItem(BUILTINS(), &_Py_ID(__build_class__));
2772 0 : if (bc == NULL) {
2773 0 : if (_PyErr_ExceptionMatches(tstate, PyExc_KeyError))
2774 0 : _PyErr_SetString(tstate, PyExc_NameError,
2775 : "__build_class__ not found");
2776 0 : goto error;
2777 : }
2778 : }
2779 950090 : PUSH(bc);
2780 950090 : DISPATCH();
2781 : }
2782 :
2783 19883400 : TARGET(STORE_NAME) {
2784 19883400 : PyObject *name = GETITEM(names, oparg);
2785 19883400 : PyObject *v = POP();
2786 19883400 : PyObject *ns = LOCALS();
2787 : int err;
2788 19883400 : if (ns == NULL) {
2789 0 : _PyErr_Format(tstate, PyExc_SystemError,
2790 : "no locals found when storing %R", name);
2791 0 : Py_DECREF(v);
2792 0 : goto error;
2793 : }
2794 19883400 : if (PyDict_CheckExact(ns))
2795 19721900 : err = PyDict_SetItem(ns, name, v);
2796 : else
2797 161520 : err = PyObject_SetItem(ns, name, v);
2798 19883400 : Py_DECREF(v);
2799 19883400 : if (err != 0)
2800 55 : goto error;
2801 19883400 : DISPATCH();
2802 : }
2803 :
2804 1165800 : TARGET(DELETE_NAME) {
2805 1165800 : PyObject *name = GETITEM(names, oparg);
2806 1165800 : PyObject *ns = LOCALS();
2807 : int err;
2808 1165800 : if (ns == NULL) {
2809 0 : _PyErr_Format(tstate, PyExc_SystemError,
2810 : "no locals when deleting %R", name);
2811 0 : goto error;
2812 : }
2813 1165800 : err = PyObject_DelItem(ns, name);
2814 1165800 : if (err != 0) {
2815 0 : format_exc_check_arg(tstate, PyExc_NameError,
2816 : NAME_ERROR_MSG,
2817 : name);
2818 0 : goto error;
2819 : }
2820 1165800 : DISPATCH();
2821 : }
2822 :
2823 1063250 : TARGET(UNPACK_SEQUENCE) {
2824 3739570 : PREDICTED(UNPACK_SEQUENCE);
2825 3739570 : PyObject *seq = POP();
2826 3739570 : PyObject **top = stack_pointer + oparg;
2827 3739570 : if (!unpack_iterable(tstate, seq, oparg, -1, top)) {
2828 1426 : Py_DECREF(seq);
2829 1426 : goto error;
2830 : }
2831 3738140 : STACK_GROW(oparg);
2832 3738140 : Py_DECREF(seq);
2833 3738140 : JUMPBY(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE);
2834 3738140 : DISPATCH();
2835 : }
2836 :
2837 2786200 : TARGET(UNPACK_SEQUENCE_ADAPTIVE) {
2838 2786200 : assert(cframe.use_tracing == 0);
2839 2786200 : _PyUnpackSequenceCache *cache = (_PyUnpackSequenceCache *)next_instr;
2840 2786200 : if (ADAPTIVE_COUNTER_IS_ZERO(cache)) {
2841 109875 : PyObject *seq = TOP();
2842 109875 : next_instr--;
2843 109875 : _Py_Specialize_UnpackSequence(seq, next_instr, oparg);
2844 109875 : NOTRACE_DISPATCH_SAME_OPARG();
2845 : }
2846 : else {
2847 : STAT_INC(UNPACK_SEQUENCE, deferred);
2848 2676320 : DECREMENT_ADAPTIVE_COUNTER(cache);
2849 2676320 : JUMP_TO_INSTRUCTION(UNPACK_SEQUENCE);
2850 : }
2851 : }
2852 :
2853 64347900 : TARGET(UNPACK_SEQUENCE_TWO_TUPLE) {
2854 64347900 : PyObject *seq = TOP();
2855 64347900 : DEOPT_IF(!PyTuple_CheckExact(seq), UNPACK_SEQUENCE);
2856 64347200 : DEOPT_IF(PyTuple_GET_SIZE(seq) != 2, UNPACK_SEQUENCE);
2857 : STAT_INC(UNPACK_SEQUENCE, hit);
2858 64347100 : SET_TOP(Py_NewRef(PyTuple_GET_ITEM(seq, 1)));
2859 64347100 : PUSH(Py_NewRef(PyTuple_GET_ITEM(seq, 0)));
2860 64347100 : Py_DECREF(seq);
2861 64347100 : JUMPBY(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE);
2862 64347100 : NOTRACE_DISPATCH();
2863 : }
2864 :
2865 15608400 : TARGET(UNPACK_SEQUENCE_TUPLE) {
2866 15608400 : PyObject *seq = TOP();
2867 15608400 : DEOPT_IF(!PyTuple_CheckExact(seq), UNPACK_SEQUENCE);
2868 15608400 : DEOPT_IF(PyTuple_GET_SIZE(seq) != oparg, UNPACK_SEQUENCE);
2869 : STAT_INC(UNPACK_SEQUENCE, hit);
2870 15608400 : STACK_SHRINK(1);
2871 15608400 : PyObject **items = _PyTuple_ITEMS(seq);
2872 65665300 : while (oparg--) {
2873 50056900 : PUSH(Py_NewRef(items[oparg]));
2874 : }
2875 15608400 : Py_DECREF(seq);
2876 15608400 : JUMPBY(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE);
2877 15608400 : NOTRACE_DISPATCH();
2878 : }
2879 :
2880 578939 : TARGET(UNPACK_SEQUENCE_LIST) {
2881 578939 : PyObject *seq = TOP();
2882 578939 : DEOPT_IF(!PyList_CheckExact(seq), UNPACK_SEQUENCE);
2883 578600 : DEOPT_IF(PyList_GET_SIZE(seq) != oparg, UNPACK_SEQUENCE);
2884 : STAT_INC(UNPACK_SEQUENCE, hit);
2885 577863 : STACK_SHRINK(1);
2886 577863 : PyObject **items = _PyList_ITEMS(seq);
2887 1882930 : while (oparg--) {
2888 1305070 : PUSH(Py_NewRef(items[oparg]));
2889 : }
2890 577863 : Py_DECREF(seq);
2891 577863 : JUMPBY(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE);
2892 577863 : NOTRACE_DISPATCH();
2893 : }
2894 :
2895 105333 : TARGET(UNPACK_EX) {
2896 105333 : int totalargs = 1 + (oparg & 0xFF) + (oparg >> 8);
2897 105333 : PyObject *seq = POP();
2898 105333 : PyObject **top = stack_pointer + totalargs;
2899 105333 : if (!unpack_iterable(tstate, seq, oparg & 0xFF, oparg >> 8, top)) {
2900 4 : Py_DECREF(seq);
2901 4 : goto error;
2902 : }
2903 105329 : STACK_GROW(totalargs);
2904 105329 : Py_DECREF(seq);
2905 105329 : DISPATCH();
2906 : }
2907 :
2908 7393780 : TARGET(STORE_ATTR) {
2909 26191300 : PREDICTED(STORE_ATTR);
2910 26191300 : PyObject *name = GETITEM(names, oparg);
2911 26191300 : PyObject *owner = TOP();
2912 26191300 : PyObject *v = SECOND();
2913 : int err;
2914 26191300 : STACK_SHRINK(2);
2915 26191300 : err = PyObject_SetAttr(owner, name, v);
2916 26191300 : Py_DECREF(v);
2917 26191300 : Py_DECREF(owner);
2918 26191300 : if (err != 0) {
2919 1100 : goto error;
2920 : }
2921 26190200 : JUMPBY(INLINE_CACHE_ENTRIES_STORE_ATTR);
2922 26190200 : DISPATCH();
2923 : }
2924 :
2925 3381190 : TARGET(DELETE_ATTR) {
2926 3381190 : PyObject *name = GETITEM(names, oparg);
2927 3381190 : PyObject *owner = POP();
2928 : int err;
2929 3381190 : err = PyObject_SetAttr(owner, name, (PyObject *)NULL);
2930 3381190 : Py_DECREF(owner);
2931 3381190 : if (err != 0)
2932 241 : goto error;
2933 3380950 : DISPATCH();
2934 : }
2935 :
2936 127209 : TARGET(STORE_GLOBAL) {
2937 127209 : PyObject *name = GETITEM(names, oparg);
2938 127209 : PyObject *v = POP();
2939 : int err;
2940 127209 : err = PyDict_SetItem(GLOBALS(), name, v);
2941 127209 : Py_DECREF(v);
2942 127209 : if (err != 0)
2943 0 : goto error;
2944 127209 : DISPATCH();
2945 : }
2946 :
2947 11 : TARGET(DELETE_GLOBAL) {
2948 11 : PyObject *name = GETITEM(names, oparg);
2949 : int err;
2950 11 : err = PyDict_DelItem(GLOBALS(), name);
2951 11 : if (err != 0) {
2952 0 : if (_PyErr_ExceptionMatches(tstate, PyExc_KeyError)) {
2953 0 : format_exc_check_arg(tstate, PyExc_NameError,
2954 : NAME_ERROR_MSG, name);
2955 : }
2956 0 : goto error;
2957 : }
2958 11 : DISPATCH();
2959 : }
2960 :
2961 16318500 : TARGET(LOAD_NAME) {
2962 16318500 : PyObject *name = GETITEM(names, oparg);
2963 16318500 : PyObject *locals = LOCALS();
2964 : PyObject *v;
2965 16318500 : if (locals == NULL) {
2966 0 : _PyErr_Format(tstate, PyExc_SystemError,
2967 : "no locals when loading %R", name);
2968 0 : goto error;
2969 : }
2970 16318500 : if (PyDict_CheckExact(locals)) {
2971 16257500 : v = PyDict_GetItemWithError(locals, name);
2972 16257500 : if (v != NULL) {
2973 11892100 : Py_INCREF(v);
2974 : }
2975 4365360 : else if (_PyErr_Occurred(tstate)) {
2976 0 : goto error;
2977 : }
2978 : }
2979 : else {
2980 61041 : v = PyObject_GetItem(locals, name);
2981 61041 : if (v == NULL) {
2982 48245 : if (!_PyErr_ExceptionMatches(tstate, PyExc_KeyError))
2983 1 : goto error;
2984 48244 : _PyErr_Clear(tstate);
2985 : }
2986 : }
2987 16318500 : if (v == NULL) {
2988 4413600 : v = PyDict_GetItemWithError(GLOBALS(), name);
2989 4413600 : if (v != NULL) {
2990 1670880 : Py_INCREF(v);
2991 : }
2992 2742720 : else if (_PyErr_Occurred(tstate)) {
2993 0 : goto error;
2994 : }
2995 : else {
2996 2742720 : if (PyDict_CheckExact(BUILTINS())) {
2997 2742720 : v = PyDict_GetItemWithError(BUILTINS(), name);
2998 2742720 : if (v == NULL) {
2999 853 : if (!_PyErr_Occurred(tstate)) {
3000 853 : format_exc_check_arg(
3001 : tstate, PyExc_NameError,
3002 : NAME_ERROR_MSG, name);
3003 : }
3004 853 : goto error;
3005 : }
3006 2741870 : Py_INCREF(v);
3007 : }
3008 : else {
3009 1 : v = PyObject_GetItem(BUILTINS(), name);
3010 1 : if (v == NULL) {
3011 1 : if (_PyErr_ExceptionMatches(tstate, PyExc_KeyError)) {
3012 0 : format_exc_check_arg(
3013 : tstate, PyExc_NameError,
3014 : NAME_ERROR_MSG, name);
3015 : }
3016 1 : goto error;
3017 : }
3018 : }
3019 : }
3020 : }
3021 16317600 : PUSH(v);
3022 16317600 : DISPATCH();
3023 : }
3024 :
3025 30671800 : TARGET(LOAD_GLOBAL) {
3026 34919100 : PREDICTED(LOAD_GLOBAL);
3027 34919100 : int push_null = oparg & 1;
3028 34919100 : PEEK(0) = NULL;
3029 34919100 : PyObject *name = GETITEM(names, oparg>>1);
3030 : PyObject *v;
3031 34919100 : if (PyDict_CheckExact(GLOBALS())
3032 34888900 : && PyDict_CheckExact(BUILTINS()))
3033 : {
3034 34888900 : v = _PyDict_LoadGlobal((PyDictObject *)GLOBALS(),
3035 34888900 : (PyDictObject *)BUILTINS(),
3036 : name);
3037 34888900 : if (v == NULL) {
3038 99 : if (!_PyErr_Occurred(tstate)) {
3039 : /* _PyDict_LoadGlobal() returns NULL without raising
3040 : * an exception if the key doesn't exist */
3041 99 : format_exc_check_arg(tstate, PyExc_NameError,
3042 : NAME_ERROR_MSG, name);
3043 : }
3044 99 : goto error;
3045 : }
3046 34888800 : Py_INCREF(v);
3047 : }
3048 : else {
3049 : /* Slow-path if globals or builtins is not a dict */
3050 :
3051 : /* namespace 1: globals */
3052 30229 : v = PyObject_GetItem(GLOBALS(), name);
3053 30229 : if (v == NULL) {
3054 78 : if (!_PyErr_ExceptionMatches(tstate, PyExc_KeyError)) {
3055 0 : goto error;
3056 : }
3057 78 : _PyErr_Clear(tstate);
3058 :
3059 : /* namespace 2: builtins */
3060 78 : v = PyObject_GetItem(BUILTINS(), name);
3061 78 : if (v == NULL) {
3062 0 : if (_PyErr_ExceptionMatches(tstate, PyExc_KeyError)) {
3063 0 : format_exc_check_arg(
3064 : tstate, PyExc_NameError,
3065 : NAME_ERROR_MSG, name);
3066 : }
3067 0 : goto error;
3068 : }
3069 : }
3070 : }
3071 : /* Skip over inline cache */
3072 34919000 : JUMPBY(INLINE_CACHE_ENTRIES_LOAD_GLOBAL);
3073 34919000 : STACK_GROW(push_null);
3074 34919000 : PUSH(v);
3075 34919000 : DISPATCH();
3076 : }
3077 :
3078 6190140 : TARGET(LOAD_GLOBAL_ADAPTIVE) {
3079 6190140 : assert(cframe.use_tracing == 0);
3080 6190140 : _PyLoadGlobalCache *cache = (_PyLoadGlobalCache *)next_instr;
3081 6190140 : if (ADAPTIVE_COUNTER_IS_ZERO(cache)) {
3082 1942840 : PyObject *name = GETITEM(names, oparg>>1);
3083 1942840 : next_instr--;
3084 1942840 : if (_Py_Specialize_LoadGlobal(GLOBALS(), BUILTINS(), next_instr, name) < 0) {
3085 0 : goto error;
3086 : }
3087 1942840 : NOTRACE_DISPATCH_SAME_OPARG();
3088 : }
3089 : else {
3090 : STAT_INC(LOAD_GLOBAL, deferred);
3091 4247300 : DECREMENT_ADAPTIVE_COUNTER(cache);
3092 4247300 : JUMP_TO_INSTRUCTION(LOAD_GLOBAL);
3093 : }
3094 : }
3095 :
3096 297630000 : TARGET(LOAD_GLOBAL_MODULE) {
3097 297630000 : assert(cframe.use_tracing == 0);
3098 297630000 : DEOPT_IF(!PyDict_CheckExact(GLOBALS()), LOAD_GLOBAL);
3099 297630000 : PyDictObject *dict = (PyDictObject *)GLOBALS();
3100 297630000 : _PyLoadGlobalCache *cache = (_PyLoadGlobalCache *)next_instr;
3101 297630000 : uint32_t version = read_u32(cache->module_keys_version);
3102 297630000 : DEOPT_IF(dict->ma_keys->dk_version != version, LOAD_GLOBAL);
3103 290908000 : assert(DK_IS_UNICODE(dict->ma_keys));
3104 290908000 : PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(dict->ma_keys);
3105 290908000 : PyObject *res = entries[cache->index].me_value;
3106 290908000 : DEOPT_IF(res == NULL, LOAD_GLOBAL);
3107 290908000 : int push_null = oparg & 1;
3108 290908000 : PEEK(0) = NULL;
3109 290908000 : JUMPBY(INLINE_CACHE_ENTRIES_LOAD_GLOBAL);
3110 : STAT_INC(LOAD_GLOBAL, hit);
3111 290908000 : STACK_GROW(push_null+1);
3112 290908000 : Py_INCREF(res);
3113 290908000 : SET_TOP(res);
3114 290908000 : NOTRACE_DISPATCH();
3115 : }
3116 :
3117 218149000 : TARGET(LOAD_GLOBAL_BUILTIN) {
3118 218149000 : assert(cframe.use_tracing == 0);
3119 218149000 : DEOPT_IF(!PyDict_CheckExact(GLOBALS()), LOAD_GLOBAL);
3120 218149000 : DEOPT_IF(!PyDict_CheckExact(BUILTINS()), LOAD_GLOBAL);
3121 218149000 : PyDictObject *mdict = (PyDictObject *)GLOBALS();
3122 218149000 : PyDictObject *bdict = (PyDictObject *)BUILTINS();
3123 218149000 : _PyLoadGlobalCache *cache = (_PyLoadGlobalCache *)next_instr;
3124 218149000 : uint32_t mod_version = read_u32(cache->module_keys_version);
3125 218149000 : uint16_t bltn_version = cache->builtin_keys_version;
3126 218149000 : DEOPT_IF(mdict->ma_keys->dk_version != mod_version, LOAD_GLOBAL);
3127 214231000 : DEOPT_IF(bdict->ma_keys->dk_version != bltn_version, LOAD_GLOBAL);
3128 211638000 : assert(DK_IS_UNICODE(bdict->ma_keys));
3129 211638000 : PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(bdict->ma_keys);
3130 211638000 : PyObject *res = entries[cache->index].me_value;
3131 211638000 : DEOPT_IF(res == NULL, LOAD_GLOBAL);
3132 211638000 : int push_null = oparg & 1;
3133 211638000 : PEEK(0) = NULL;
3134 211638000 : JUMPBY(INLINE_CACHE_ENTRIES_LOAD_GLOBAL);
3135 : STAT_INC(LOAD_GLOBAL, hit);
3136 211638000 : STACK_GROW(push_null+1);
3137 211638000 : Py_INCREF(res);
3138 211638000 : SET_TOP(res);
3139 211638000 : NOTRACE_DISPATCH();
3140 : }
3141 :
3142 785085 : TARGET(DELETE_FAST) {
3143 785085 : PyObject *v = GETLOCAL(oparg);
3144 785085 : if (v != NULL) {
3145 785085 : SETLOCAL(oparg, NULL);
3146 785085 : DISPATCH();
3147 : }
3148 0 : goto unbound_local_error;
3149 : }
3150 :
3151 4638440 : TARGET(MAKE_CELL) {
3152 : // "initial" is probably NULL but not if it's an arg (or set
3153 : // via PyFrame_LocalsToFast() before MAKE_CELL has run).
3154 4638440 : PyObject *initial = GETLOCAL(oparg);
3155 4638440 : PyObject *cell = PyCell_New(initial);
3156 4638440 : if (cell == NULL) {
3157 0 : goto resume_with_error;
3158 : }
3159 4638440 : SETLOCAL(oparg, cell);
3160 4638440 : DISPATCH();
3161 : }
3162 :
3163 38 : TARGET(DELETE_DEREF) {
3164 38 : PyObject *cell = GETLOCAL(oparg);
3165 38 : PyObject *oldobj = PyCell_GET(cell);
3166 38 : if (oldobj != NULL) {
3167 38 : PyCell_SET(cell, NULL);
3168 38 : Py_DECREF(oldobj);
3169 38 : DISPATCH();
3170 : }
3171 0 : format_exc_unbound(tstate, frame->f_code, oparg);
3172 0 : goto error;
3173 : }
3174 :
3175 6049 : TARGET(LOAD_CLASSDEREF) {
3176 6049 : PyObject *name, *value, *locals = LOCALS();
3177 6049 : assert(locals);
3178 6049 : assert(oparg >= 0 && oparg < frame->f_code->co_nlocalsplus);
3179 6049 : name = PyTuple_GET_ITEM(frame->f_code->co_localsplusnames, oparg);
3180 6049 : if (PyDict_CheckExact(locals)) {
3181 5610 : value = PyDict_GetItemWithError(locals, name);
3182 5610 : if (value != NULL) {
3183 1 : Py_INCREF(value);
3184 : }
3185 5609 : else if (_PyErr_Occurred(tstate)) {
3186 0 : goto error;
3187 : }
3188 : }
3189 : else {
3190 439 : value = PyObject_GetItem(locals, name);
3191 439 : if (value == NULL) {
3192 439 : if (!_PyErr_ExceptionMatches(tstate, PyExc_KeyError)) {
3193 0 : goto error;
3194 : }
3195 439 : _PyErr_Clear(tstate);
3196 : }
3197 : }
3198 6049 : if (!value) {
3199 6048 : PyObject *cell = GETLOCAL(oparg);
3200 6048 : value = PyCell_GET(cell);
3201 6048 : if (value == NULL) {
3202 0 : format_exc_unbound(tstate, frame->f_code, oparg);
3203 0 : goto error;
3204 : }
3205 6048 : Py_INCREF(value);
3206 : }
3207 6049 : PUSH(value);
3208 6049 : DISPATCH();
3209 : }
3210 :
3211 91632800 : TARGET(LOAD_DEREF) {
3212 91632800 : PyObject *cell = GETLOCAL(oparg);
3213 91632800 : PyObject *value = PyCell_GET(cell);
3214 91632800 : if (value == NULL) {
3215 8 : format_exc_unbound(tstate, frame->f_code, oparg);
3216 8 : goto error;
3217 : }
3218 91632800 : Py_INCREF(value);
3219 91632800 : PUSH(value);
3220 91632800 : DISPATCH();
3221 : }
3222 :
3223 1820990 : TARGET(STORE_DEREF) {
3224 1820990 : PyObject *v = POP();
3225 1820990 : PyObject *cell = GETLOCAL(oparg);
3226 1820990 : PyObject *oldobj = PyCell_GET(cell);
3227 1820990 : PyCell_SET(cell, v);
3228 1820990 : Py_XDECREF(oldobj);
3229 1820990 : DISPATCH();
3230 : }
3231 :
3232 35324500 : TARGET(COPY_FREE_VARS) {
3233 : /* Copy closure variables to free variables */
3234 35324500 : PyCodeObject *co = frame->f_code;
3235 35324500 : PyObject *closure = frame->f_func->func_closure;
3236 35324500 : int offset = co->co_nlocals + co->co_nplaincellvars;
3237 35324500 : assert(oparg == co->co_nfreevars);
3238 93499700 : for (int i = 0; i < oparg; ++i) {
3239 58175200 : PyObject *o = PyTuple_GET_ITEM(closure, i);
3240 58175200 : Py_INCREF(o);
3241 58175200 : frame->localsplus[offset + i] = o;
3242 : }
3243 35324500 : DISPATCH();
3244 : }
3245 :
3246 2238120 : TARGET(BUILD_STRING) {
3247 : PyObject *str;
3248 2238120 : str = _PyUnicode_JoinArray(&_Py_STR(empty),
3249 2238120 : stack_pointer - oparg, oparg);
3250 2238120 : if (str == NULL)
3251 0 : goto error;
3252 8670430 : while (--oparg >= 0) {
3253 6432310 : PyObject *item = POP();
3254 6432310 : Py_DECREF(item);
3255 : }
3256 2238120 : PUSH(str);
3257 2238120 : DISPATCH();
3258 : }
3259 :
3260 76696900 : TARGET(BUILD_TUPLE) {
3261 76696900 : PyObject *tup = PyTuple_New(oparg);
3262 76696900 : if (tup == NULL)
3263 0 : goto error;
3264 263624000 : while (--oparg >= 0) {
3265 186927000 : PyObject *item = POP();
3266 186927000 : PyTuple_SET_ITEM(tup, oparg, item);
3267 : }
3268 76696900 : PUSH(tup);
3269 76696900 : DISPATCH();
3270 : }
3271 :
3272 16990800 : TARGET(BUILD_LIST) {
3273 16990800 : PyObject *list = PyList_New(oparg);
3274 16990800 : if (list == NULL)
3275 0 : goto error;
3276 59344100 : while (--oparg >= 0) {
3277 42353300 : PyObject *item = POP();
3278 42353300 : PyList_SET_ITEM(list, oparg, item);
3279 : }
3280 16990800 : PUSH(list);
3281 16990800 : DISPATCH();
3282 : }
3283 :
3284 1106410 : TARGET(LIST_TO_TUPLE) {
3285 1106410 : PyObject *list = POP();
3286 1106410 : PyObject *tuple = PyList_AsTuple(list);
3287 1106410 : Py_DECREF(list);
3288 1106410 : if (tuple == NULL) {
3289 0 : goto error;
3290 : }
3291 1106410 : PUSH(tuple);
3292 1106410 : DISPATCH();
3293 : }
3294 :
3295 1332490 : TARGET(LIST_EXTEND) {
3296 1332490 : PyObject *iterable = POP();
3297 1332490 : PyObject *list = PEEK(oparg);
3298 1332490 : PyObject *none_val = _PyList_Extend((PyListObject *)list, iterable);
3299 1332490 : if (none_val == NULL) {
3300 20 : if (_PyErr_ExceptionMatches(tstate, PyExc_TypeError) &&
3301 20 : (Py_TYPE(iterable)->tp_iter == NULL && !PySequence_Check(iterable)))
3302 : {
3303 16 : _PyErr_Clear(tstate);
3304 16 : _PyErr_Format(tstate, PyExc_TypeError,
3305 : "Value after * must be an iterable, not %.200s",
3306 16 : Py_TYPE(iterable)->tp_name);
3307 : }
3308 20 : Py_DECREF(iterable);
3309 20 : goto error;
3310 : }
3311 1332470 : Py_DECREF(none_val);
3312 1332470 : Py_DECREF(iterable);
3313 1332470 : DISPATCH();
3314 : }
3315 :
3316 6646 : TARGET(SET_UPDATE) {
3317 6646 : PyObject *iterable = POP();
3318 6646 : PyObject *set = PEEK(oparg);
3319 6646 : int err = _PySet_Update(set, iterable);
3320 6646 : Py_DECREF(iterable);
3321 6646 : if (err < 0) {
3322 1 : goto error;
3323 : }
3324 6645 : DISPATCH();
3325 : }
3326 :
3327 137289 : TARGET(BUILD_SET) {
3328 137289 : PyObject *set = PySet_New(NULL);
3329 137289 : int err = 0;
3330 : int i;
3331 137289 : if (set == NULL)
3332 0 : goto error;
3333 507651 : for (i = oparg; i > 0; i--) {
3334 370362 : PyObject *item = PEEK(i);
3335 370362 : if (err == 0)
3336 370362 : err = PySet_Add(set, item);
3337 370362 : Py_DECREF(item);
3338 : }
3339 137289 : STACK_SHRINK(oparg);
3340 137289 : if (err != 0) {
3341 1 : Py_DECREF(set);
3342 1 : goto error;
3343 : }
3344 137288 : PUSH(set);
3345 137288 : DISPATCH();
3346 : }
3347 :
3348 16927000 : TARGET(BUILD_MAP) {
3349 16927000 : PyObject *map = _PyDict_FromItems(
3350 16927000 : &PEEK(2*oparg), 2,
3351 16927000 : &PEEK(2*oparg - 1), 2,
3352 : oparg);
3353 16927000 : if (map == NULL)
3354 0 : goto error;
3355 :
3356 17795200 : while (oparg--) {
3357 868185 : Py_DECREF(POP());
3358 868185 : Py_DECREF(POP());
3359 : }
3360 16927000 : PUSH(map);
3361 16927000 : DISPATCH();
3362 : }
3363 :
3364 2311 : TARGET(SETUP_ANNOTATIONS) {
3365 : int err;
3366 : PyObject *ann_dict;
3367 2311 : if (LOCALS() == NULL) {
3368 0 : _PyErr_Format(tstate, PyExc_SystemError,
3369 : "no locals found when setting up annotations");
3370 0 : goto error;
3371 : }
3372 : /* check if __annotations__ in locals()... */
3373 2311 : if (PyDict_CheckExact(LOCALS())) {
3374 2309 : ann_dict = _PyDict_GetItemWithError(LOCALS(),
3375 : &_Py_ID(__annotations__));
3376 2309 : if (ann_dict == NULL) {
3377 2306 : if (_PyErr_Occurred(tstate)) {
3378 0 : goto error;
3379 : }
3380 : /* ...if not, create a new one */
3381 2306 : ann_dict = PyDict_New();
3382 2306 : if (ann_dict == NULL) {
3383 0 : goto error;
3384 : }
3385 2306 : err = PyDict_SetItem(LOCALS(), &_Py_ID(__annotations__),
3386 : ann_dict);
3387 2306 : Py_DECREF(ann_dict);
3388 2306 : if (err != 0) {
3389 0 : goto error;
3390 : }
3391 : }
3392 : }
3393 : else {
3394 : /* do the same if locals() is not a dict */
3395 2 : ann_dict = PyObject_GetItem(LOCALS(), &_Py_ID(__annotations__));
3396 2 : if (ann_dict == NULL) {
3397 1 : if (!_PyErr_ExceptionMatches(tstate, PyExc_KeyError)) {
3398 0 : goto error;
3399 : }
3400 1 : _PyErr_Clear(tstate);
3401 1 : ann_dict = PyDict_New();
3402 1 : if (ann_dict == NULL) {
3403 0 : goto error;
3404 : }
3405 1 : err = PyObject_SetItem(LOCALS(), &_Py_ID(__annotations__),
3406 : ann_dict);
3407 1 : Py_DECREF(ann_dict);
3408 1 : if (err != 0) {
3409 0 : goto error;
3410 : }
3411 : }
3412 : else {
3413 1 : Py_DECREF(ann_dict);
3414 : }
3415 : }
3416 2311 : DISPATCH();
3417 : }
3418 :
3419 837384 : TARGET(BUILD_CONST_KEY_MAP) {
3420 : PyObject *map;
3421 837384 : PyObject *keys = TOP();
3422 837384 : if (!PyTuple_CheckExact(keys) ||
3423 837384 : PyTuple_GET_SIZE(keys) != (Py_ssize_t)oparg) {
3424 0 : _PyErr_SetString(tstate, PyExc_SystemError,
3425 : "bad BUILD_CONST_KEY_MAP keys argument");
3426 0 : goto error;
3427 : }
3428 1674770 : map = _PyDict_FromItems(
3429 837384 : &PyTuple_GET_ITEM(keys, 0), 1,
3430 837384 : &PEEK(oparg + 1), 1, oparg);
3431 837384 : if (map == NULL) {
3432 1 : goto error;
3433 : }
3434 :
3435 837383 : Py_DECREF(POP());
3436 3289990 : while (oparg--) {
3437 2452600 : Py_DECREF(POP());
3438 : }
3439 837383 : PUSH(map);
3440 837383 : DISPATCH();
3441 : }
3442 :
3443 244842 : TARGET(DICT_UPDATE) {
3444 244842 : PyObject *update = POP();
3445 244842 : PyObject *dict = PEEK(oparg);
3446 244842 : if (PyDict_Update(dict, update) < 0) {
3447 3 : if (_PyErr_ExceptionMatches(tstate, PyExc_AttributeError)) {
3448 2 : _PyErr_Format(tstate, PyExc_TypeError,
3449 : "'%.200s' object is not a mapping",
3450 2 : Py_TYPE(update)->tp_name);
3451 : }
3452 3 : Py_DECREF(update);
3453 3 : goto error;
3454 : }
3455 244839 : Py_DECREF(update);
3456 244839 : DISPATCH();
3457 : }
3458 :
3459 3412470 : TARGET(DICT_MERGE) {
3460 3412470 : PyObject *update = POP();
3461 3412470 : PyObject *dict = PEEK(oparg);
3462 :
3463 3412470 : if (_PyDict_MergeEx(dict, update, 2) < 0) {
3464 23 : format_kwargs_error(tstate, PEEK(2 + oparg), update);
3465 23 : Py_DECREF(update);
3466 23 : goto error;
3467 : }
3468 3412450 : Py_DECREF(update);
3469 : PREDICT(CALL_FUNCTION_EX);
3470 3412450 : DISPATCH();
3471 : }
3472 :
3473 4826760 : TARGET(MAP_ADD) {
3474 4826760 : PyObject *value = TOP();
3475 4826760 : PyObject *key = SECOND();
3476 : PyObject *map;
3477 4826760 : STACK_SHRINK(2);
3478 4826760 : map = PEEK(oparg); /* dict */
3479 4826760 : assert(PyDict_CheckExact(map));
3480 : /* map[key] = value */
3481 4826760 : if (_PyDict_SetItem_Take2((PyDictObject *)map, key, value) != 0) {
3482 1 : goto error;
3483 : }
3484 : PREDICT(JUMP_BACKWARD_QUICK);
3485 4826760 : DISPATCH();
3486 : }
3487 :
3488 49112400 : TARGET(LOAD_ATTR) {
3489 156790000 : PREDICTED(LOAD_ATTR);
3490 156790000 : PyObject *name = GETITEM(names, oparg >> 1);
3491 156790000 : PyObject *owner = TOP();
3492 156790000 : if (oparg & 1) {
3493 : /* Designed to work in tandem with CALL. */
3494 36584700 : PyObject* meth = NULL;
3495 :
3496 36584700 : int meth_found = _PyObject_GetMethod(owner, name, &meth);
3497 :
3498 36584700 : if (meth == NULL) {
3499 : /* Most likely attribute wasn't found. */
3500 7287 : goto error;
3501 : }
3502 :
3503 36577400 : if (meth_found) {
3504 : /* We can bypass temporary bound method object.
3505 : meth is unbound method and obj is self.
3506 :
3507 : meth | self | arg1 | ... | argN
3508 : */
3509 23220900 : SET_TOP(meth);
3510 23220900 : PUSH(owner); // self
3511 : }
3512 : else {
3513 : /* meth is not an unbound method (but a regular attr, or
3514 : something was returned by a descriptor protocol). Set
3515 : the second element of the stack to NULL, to signal
3516 : CALL that it's not a method call.
3517 :
3518 : NULL | meth | arg1 | ... | argN
3519 : */
3520 13356500 : SET_TOP(NULL);
3521 13356500 : Py_DECREF(owner);
3522 13356500 : PUSH(meth);
3523 : }
3524 36577400 : JUMPBY(INLINE_CACHE_ENTRIES_LOAD_ATTR);
3525 36577400 : DISPATCH();
3526 : }
3527 120205000 : PyObject *res = PyObject_GetAttr(owner, name);
3528 120205000 : if (res == NULL) {
3529 172732 : goto error;
3530 : }
3531 120033000 : Py_DECREF(owner);
3532 120033000 : SET_TOP(res);
3533 120033000 : JUMPBY(INLINE_CACHE_ENTRIES_LOAD_ATTR);
3534 120033000 : DISPATCH();
3535 : }
3536 :
3537 110600000 : TARGET(LOAD_ATTR_ADAPTIVE) {
3538 110600000 : assert(cframe.use_tracing == 0);
3539 110600000 : _PyAttrCache *cache = (_PyAttrCache *)next_instr;
3540 110600000 : if (ADAPTIVE_COUNTER_IS_ZERO(cache)) {
3541 2922590 : PyObject *owner = TOP();
3542 2922590 : PyObject *name = GETITEM(names, oparg>>1);
3543 2922590 : next_instr--;
3544 2922590 : if (_Py_Specialize_LoadAttr(owner, next_instr, name) < 0) {
3545 0 : goto error;
3546 : }
3547 2922590 : NOTRACE_DISPATCH_SAME_OPARG();
3548 : }
3549 : else {
3550 : STAT_INC(LOAD_ATTR, deferred);
3551 107678000 : DECREMENT_ADAPTIVE_COUNTER(cache);
3552 107678000 : JUMP_TO_INSTRUCTION(LOAD_ATTR);
3553 : }
3554 : }
3555 :
3556 203620000 : TARGET(LOAD_ATTR_INSTANCE_VALUE) {
3557 203620000 : assert(cframe.use_tracing == 0);
3558 203620000 : PyObject *owner = TOP();
3559 : PyObject *res;
3560 203620000 : PyTypeObject *tp = Py_TYPE(owner);
3561 203620000 : _PyAttrCache *cache = (_PyAttrCache *)next_instr;
3562 203620000 : uint32_t type_version = read_u32(cache->version);
3563 203620000 : assert(type_version != 0);
3564 203620000 : DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR);
3565 200036000 : assert(tp->tp_dictoffset < 0);
3566 200036000 : assert(tp->tp_flags & Py_TPFLAGS_MANAGED_DICT);
3567 200036000 : PyDictValues *values = *_PyObject_ValuesPointer(owner);
3568 200036000 : DEOPT_IF(values == NULL, LOAD_ATTR);
3569 199854000 : res = values->values[cache->index];
3570 199854000 : DEOPT_IF(res == NULL, LOAD_ATTR);
3571 : STAT_INC(LOAD_ATTR, hit);
3572 199308000 : Py_INCREF(res);
3573 199308000 : SET_TOP(NULL);
3574 199308000 : STACK_GROW((oparg & 1));
3575 199308000 : SET_TOP(res);
3576 199308000 : Py_DECREF(owner);
3577 199308000 : JUMPBY(INLINE_CACHE_ENTRIES_LOAD_ATTR);
3578 199308000 : NOTRACE_DISPATCH();
3579 : }
3580 :
3581 66438200 : TARGET(LOAD_ATTR_MODULE) {
3582 66438200 : assert(cframe.use_tracing == 0);
3583 66438200 : PyObject *owner = TOP();
3584 : PyObject *res;
3585 66438200 : _PyAttrCache *cache = (_PyAttrCache *)next_instr;
3586 66438200 : DEOPT_IF(!PyModule_CheckExact(owner), LOAD_ATTR);
3587 66437700 : PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner)->md_dict;
3588 66437700 : assert(dict != NULL);
3589 66437700 : DEOPT_IF(dict->ma_keys->dk_version != read_u32(cache->version),
3590 : LOAD_ATTR);
3591 63707900 : assert(dict->ma_keys->dk_kind == DICT_KEYS_UNICODE);
3592 63707900 : assert(cache->index < dict->ma_keys->dk_nentries);
3593 63707900 : PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + cache->index;
3594 63707900 : res = ep->me_value;
3595 63707900 : DEOPT_IF(res == NULL, LOAD_ATTR);
3596 : STAT_INC(LOAD_ATTR, hit);
3597 63707900 : Py_INCREF(res);
3598 63707900 : SET_TOP(NULL);
3599 63707900 : STACK_GROW((oparg & 1));
3600 63707900 : SET_TOP(res);
3601 63707900 : Py_DECREF(owner);
3602 63707900 : JUMPBY(INLINE_CACHE_ENTRIES_LOAD_ATTR);
3603 63707900 : NOTRACE_DISPATCH();
3604 : }
3605 :
3606 11406000 : TARGET(LOAD_ATTR_WITH_HINT) {
3607 11406000 : assert(cframe.use_tracing == 0);
3608 11406000 : PyObject *owner = TOP();
3609 : PyObject *res;
3610 11406000 : PyTypeObject *tp = Py_TYPE(owner);
3611 11406000 : _PyAttrCache *cache = (_PyAttrCache *)next_instr;
3612 11406000 : uint32_t type_version = read_u32(cache->version);
3613 11406000 : assert(type_version != 0);
3614 11406000 : DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR);
3615 10917400 : assert(tp->tp_flags & Py_TPFLAGS_MANAGED_DICT);
3616 10917400 : PyDictObject *dict = *(PyDictObject **)_PyObject_ManagedDictPointer(owner);
3617 10917400 : DEOPT_IF(dict == NULL, LOAD_ATTR);
3618 10796600 : assert(PyDict_CheckExact((PyObject *)dict));
3619 10796600 : PyObject *name = GETITEM(names, oparg>>1);
3620 10796600 : uint16_t hint = cache->index;
3621 10796600 : DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries, LOAD_ATTR);
3622 10781500 : if (DK_IS_UNICODE(dict->ma_keys)) {
3623 10781500 : PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint;
3624 10781500 : DEOPT_IF(ep->me_key != name, LOAD_ATTR);
3625 10724500 : res = ep->me_value;
3626 : }
3627 : else {
3628 0 : PyDictKeyEntry *ep = DK_ENTRIES(dict->ma_keys) + hint;
3629 0 : DEOPT_IF(ep->me_key != name, LOAD_ATTR);
3630 0 : res = ep->me_value;
3631 : }
3632 10724500 : DEOPT_IF(res == NULL, LOAD_ATTR);
3633 : STAT_INC(LOAD_ATTR, hit);
3634 6489630 : Py_INCREF(res);
3635 6489630 : SET_TOP(NULL);
3636 6489630 : STACK_GROW((oparg & 1));
3637 6489630 : SET_TOP(res);
3638 6489630 : Py_DECREF(owner);
3639 6489630 : JUMPBY(INLINE_CACHE_ENTRIES_LOAD_ATTR);
3640 6489630 : NOTRACE_DISPATCH();
3641 : }
3642 :
3643 46410600 : TARGET(LOAD_ATTR_SLOT) {
3644 46410600 : assert(cframe.use_tracing == 0);
3645 46410600 : PyObject *owner = TOP();
3646 : PyObject *res;
3647 46410600 : PyTypeObject *tp = Py_TYPE(owner);
3648 46410600 : _PyAttrCache *cache = (_PyAttrCache *)next_instr;
3649 46410600 : uint32_t type_version = read_u32(cache->version);
3650 46410600 : assert(type_version != 0);
3651 46410600 : DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR);
3652 43818000 : char *addr = (char *)owner + cache->index;
3653 43818000 : res = *(PyObject **)addr;
3654 43818000 : DEOPT_IF(res == NULL, LOAD_ATTR);
3655 : STAT_INC(LOAD_ATTR, hit);
3656 43789500 : Py_INCREF(res);
3657 43789500 : SET_TOP(NULL);
3658 43789500 : STACK_GROW((oparg & 1));
3659 43789500 : SET_TOP(res);
3660 43789500 : Py_DECREF(owner);
3661 43789500 : JUMPBY(INLINE_CACHE_ENTRIES_LOAD_ATTR);
3662 43789500 : NOTRACE_DISPATCH();
3663 : }
3664 :
3665 12886300 : TARGET(LOAD_ATTR_CLASS) {
3666 12886300 : assert(cframe.use_tracing == 0);
3667 12886300 : _PyLoadMethodCache *cache = (_PyLoadMethodCache *)next_instr;
3668 :
3669 12886300 : PyObject *cls = TOP();
3670 12886300 : DEOPT_IF(!PyType_Check(cls), LOAD_ATTR);
3671 12884700 : uint32_t type_version = read_u32(cache->type_version);
3672 12884700 : DEOPT_IF(((PyTypeObject *)cls)->tp_version_tag != type_version,
3673 : LOAD_ATTR);
3674 6930660 : assert(type_version != 0);
3675 :
3676 : STAT_INC(LOAD_ATTR, hit);
3677 6930660 : PyObject *res = read_obj(cache->descr);
3678 6930660 : assert(res != NULL);
3679 6930660 : Py_INCREF(res);
3680 6930660 : SET_TOP(NULL);
3681 6930660 : STACK_GROW((oparg & 1));
3682 6930660 : SET_TOP(res);
3683 6930660 : Py_DECREF(cls);
3684 6930660 : JUMPBY(INLINE_CACHE_ENTRIES_LOAD_ATTR);
3685 6930660 : NOTRACE_DISPATCH();
3686 : }
3687 :
3688 5947070 : TARGET(LOAD_ATTR_PROPERTY) {
3689 5947070 : assert(cframe.use_tracing == 0);
3690 5947070 : DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR);
3691 5947070 : _PyLoadMethodCache *cache = (_PyLoadMethodCache *)next_instr;
3692 :
3693 5947070 : PyObject *owner = TOP();
3694 5947070 : PyTypeObject *cls = Py_TYPE(owner);
3695 5947070 : uint32_t type_version = read_u32(cache->type_version);
3696 5947070 : DEOPT_IF(cls->tp_version_tag != type_version, LOAD_ATTR);
3697 5807480 : assert(type_version != 0);
3698 5807480 : PyObject *fget = read_obj(cache->descr);
3699 5807480 : PyFunctionObject *f = (PyFunctionObject *)fget;
3700 5807480 : uint32_t func_version = read_u32(cache->keys_version);
3701 5807480 : assert(func_version != 0);
3702 5807480 : DEOPT_IF(f->func_version != func_version, LOAD_ATTR);
3703 5807480 : PyCodeObject *code = (PyCodeObject *)f->func_code;
3704 5807480 : assert(code->co_argcount == 1);
3705 5807480 : DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), LOAD_ATTR);
3706 : STAT_INC(LOAD_ATTR, hit);
3707 5807440 : Py_INCREF(fget);
3708 5807440 : _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, f);
3709 5807440 : SET_TOP(NULL);
3710 5807440 : int push_null = !(oparg & 1);
3711 5807440 : STACK_SHRINK(push_null);
3712 5807440 : new_frame->localsplus[0] = owner;
3713 5941880 : for (int i = 1; i < code->co_nlocalsplus; i++) {
3714 134447 : new_frame->localsplus[i] = NULL;
3715 : }
3716 5807440 : _PyFrame_SetStackPointer(frame, stack_pointer);
3717 5807440 : JUMPBY(INLINE_CACHE_ENTRIES_LOAD_ATTR);
3718 5807440 : frame->prev_instr = next_instr - 1;
3719 5807440 : new_frame->previous = frame;
3720 5807440 : frame = cframe.current_frame = new_frame;
3721 : CALL_STAT_INC(inlined_py_calls);
3722 5807440 : goto start_frame;
3723 : }
3724 :
3725 19502300 : TARGET(STORE_ATTR_ADAPTIVE) {
3726 19502300 : assert(cframe.use_tracing == 0);
3727 19502300 : _PyAttrCache *cache = (_PyAttrCache *)next_instr;
3728 19502300 : if (ADAPTIVE_COUNTER_IS_ZERO(cache)) {
3729 704814 : PyObject *owner = TOP();
3730 704814 : PyObject *name = GETITEM(names, oparg);
3731 704814 : next_instr--;
3732 704814 : if (_Py_Specialize_StoreAttr(owner, next_instr, name) < 0) {
3733 0 : goto error;
3734 : }
3735 704814 : NOTRACE_DISPATCH_SAME_OPARG();
3736 : }
3737 : else {
3738 : STAT_INC(STORE_ATTR, deferred);
3739 18797500 : DECREMENT_ADAPTIVE_COUNTER(cache);
3740 18797500 : JUMP_TO_INSTRUCTION(STORE_ATTR);
3741 : }
3742 : }
3743 :
3744 53478000 : TARGET(STORE_ATTR_INSTANCE_VALUE) {
3745 53478000 : assert(cframe.use_tracing == 0);
3746 53478000 : PyObject *owner = TOP();
3747 53478000 : PyTypeObject *tp = Py_TYPE(owner);
3748 53478000 : _PyAttrCache *cache = (_PyAttrCache *)next_instr;
3749 53478000 : uint32_t type_version = read_u32(cache->version);
3750 53478000 : assert(type_version != 0);
3751 53478000 : DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR);
3752 49733000 : assert(tp->tp_flags & Py_TPFLAGS_MANAGED_DICT);
3753 49733000 : PyDictValues *values = *_PyObject_ValuesPointer(owner);
3754 49733000 : DEOPT_IF(values == NULL, STORE_ATTR);
3755 : STAT_INC(STORE_ATTR, hit);
3756 49635700 : Py_ssize_t index = cache->index;
3757 49635700 : STACK_SHRINK(1);
3758 49635700 : PyObject *value = POP();
3759 49635700 : PyObject *old_value = values->values[index];
3760 49635700 : values->values[index] = value;
3761 49635700 : if (old_value == NULL) {
3762 27059500 : _PyDictValues_AddToInsertionOrder(values, index);
3763 : }
3764 : else {
3765 22576200 : Py_DECREF(old_value);
3766 : }
3767 49635700 : Py_DECREF(owner);
3768 49635700 : JUMPBY(INLINE_CACHE_ENTRIES_STORE_ATTR);
3769 49635700 : NOTRACE_DISPATCH();
3770 : }
3771 :
3772 656522 : TARGET(STORE_ATTR_WITH_HINT) {
3773 656522 : assert(cframe.use_tracing == 0);
3774 656522 : PyObject *owner = TOP();
3775 656522 : PyTypeObject *tp = Py_TYPE(owner);
3776 656522 : _PyAttrCache *cache = (_PyAttrCache *)next_instr;
3777 656522 : uint32_t type_version = read_u32(cache->version);
3778 656522 : assert(type_version != 0);
3779 656522 : DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR);
3780 399336 : assert(tp->tp_flags & Py_TPFLAGS_MANAGED_DICT);
3781 399336 : PyDictObject *dict = *(PyDictObject **)_PyObject_ManagedDictPointer(owner);
3782 399336 : DEOPT_IF(dict == NULL, STORE_ATTR);
3783 385985 : assert(PyDict_CheckExact((PyObject *)dict));
3784 385985 : PyObject *name = GETITEM(names, oparg);
3785 385985 : uint16_t hint = cache->index;
3786 385985 : DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries, STORE_ATTR);
3787 : PyObject *value, *old_value;
3788 382954 : if (DK_IS_UNICODE(dict->ma_keys)) {
3789 382954 : PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint;
3790 382954 : DEOPT_IF(ep->me_key != name, STORE_ATTR);
3791 365094 : old_value = ep->me_value;
3792 365094 : DEOPT_IF(old_value == NULL, STORE_ATTR);
3793 72269 : STACK_SHRINK(1);
3794 72269 : value = POP();
3795 72269 : ep->me_value = value;
3796 : }
3797 : else {
3798 0 : PyDictKeyEntry *ep = DK_ENTRIES(dict->ma_keys) + hint;
3799 0 : DEOPT_IF(ep->me_key != name, STORE_ATTR);
3800 0 : old_value = ep->me_value;
3801 0 : DEOPT_IF(old_value == NULL, STORE_ATTR);
3802 0 : STACK_SHRINK(1);
3803 0 : value = POP();
3804 0 : ep->me_value = value;
3805 : }
3806 72269 : Py_DECREF(old_value);
3807 : STAT_INC(STORE_ATTR, hit);
3808 : /* Ensure dict is GC tracked if it needs to be */
3809 72269 : if (!_PyObject_GC_IS_TRACKED(dict) && _PyObject_GC_MAY_BE_TRACKED(value)) {
3810 0 : _PyObject_GC_TRACK(dict);
3811 : }
3812 : /* PEP 509 */
3813 72269 : dict->ma_version_tag = DICT_NEXT_VERSION();
3814 72269 : Py_DECREF(owner);
3815 72269 : JUMPBY(INLINE_CACHE_ENTRIES_STORE_ATTR);
3816 72269 : NOTRACE_DISPATCH();
3817 : }
3818 :
3819 8421460 : TARGET(STORE_ATTR_SLOT) {
3820 8421460 : assert(cframe.use_tracing == 0);
3821 8421460 : PyObject *owner = TOP();
3822 8421460 : PyTypeObject *tp = Py_TYPE(owner);
3823 8421460 : _PyAttrCache *cache = (_PyAttrCache *)next_instr;
3824 8421460 : uint32_t type_version = read_u32(cache->version);
3825 8421460 : assert(type_version != 0);
3826 8421460 : DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR);
3827 8331190 : char *addr = (char *)owner + cache->index;
3828 : STAT_INC(STORE_ATTR, hit);
3829 8331190 : STACK_SHRINK(1);
3830 8331190 : PyObject *value = POP();
3831 8331190 : PyObject *old_value = *(PyObject **)addr;
3832 8331190 : *(PyObject **)addr = value;
3833 8331190 : Py_XDECREF(old_value);
3834 8331190 : Py_DECREF(owner);
3835 8331190 : JUMPBY(INLINE_CACHE_ENTRIES_STORE_ATTR);
3836 8331190 : NOTRACE_DISPATCH();
3837 : }
3838 :
3839 32286700 : TARGET(COMPARE_OP) {
3840 55978800 : PREDICTED(COMPARE_OP);
3841 55978800 : assert(oparg <= Py_GE);
3842 55978800 : PyObject *right = POP();
3843 55978800 : PyObject *left = TOP();
3844 55978800 : PyObject *res = PyObject_RichCompare(left, right, oparg);
3845 55978800 : SET_TOP(res);
3846 55978800 : Py_DECREF(left);
3847 55978800 : Py_DECREF(right);
3848 55978800 : if (res == NULL) {
3849 6678 : goto error;
3850 : }
3851 55972100 : JUMPBY(INLINE_CACHE_ENTRIES_COMPARE_OP);
3852 55972100 : DISPATCH();
3853 : }
3854 :
3855 24116400 : TARGET(COMPARE_OP_ADAPTIVE) {
3856 24116400 : assert(cframe.use_tracing == 0);
3857 24116400 : _PyCompareOpCache *cache = (_PyCompareOpCache *)next_instr;
3858 24116400 : if (ADAPTIVE_COUNTER_IS_ZERO(cache)) {
3859 424297 : PyObject *right = TOP();
3860 424297 : PyObject *left = SECOND();
3861 424297 : next_instr--;
3862 424297 : _Py_Specialize_CompareOp(left, right, next_instr, oparg);
3863 424297 : NOTRACE_DISPATCH_SAME_OPARG();
3864 : }
3865 : else {
3866 : STAT_INC(COMPARE_OP, deferred);
3867 23692100 : DECREMENT_ADAPTIVE_COUNTER(cache);
3868 23692100 : JUMP_TO_INSTRUCTION(COMPARE_OP);
3869 : }
3870 : }
3871 :
3872 11258000 : TARGET(COMPARE_OP_FLOAT_JUMP) {
3873 11258000 : assert(cframe.use_tracing == 0);
3874 : // Combined: COMPARE_OP (float ? float) + POP_JUMP_(direction)_IF_(true/false)
3875 11258000 : _PyCompareOpCache *cache = (_PyCompareOpCache *)next_instr;
3876 11258000 : int when_to_jump_mask = cache->mask;
3877 11258000 : PyObject *right = TOP();
3878 11258000 : PyObject *left = SECOND();
3879 11258000 : DEOPT_IF(!PyFloat_CheckExact(left), COMPARE_OP);
3880 11252900 : DEOPT_IF(!PyFloat_CheckExact(right), COMPARE_OP);
3881 11230000 : double dleft = PyFloat_AS_DOUBLE(left);
3882 11230000 : double dright = PyFloat_AS_DOUBLE(right);
3883 11230000 : int sign = (dleft > dright) - (dleft < dright);
3884 11230000 : DEOPT_IF(isnan(dleft), COMPARE_OP);
3885 11229900 : DEOPT_IF(isnan(dright), COMPARE_OP);
3886 : STAT_INC(COMPARE_OP, hit);
3887 11229900 : JUMPBY(INLINE_CACHE_ENTRIES_COMPARE_OP);
3888 11229900 : NEXTOPARG();
3889 11229900 : STACK_SHRINK(2);
3890 11229900 : _Py_DECREF_SPECIALIZED(left, _PyFloat_ExactDealloc);
3891 11229900 : _Py_DECREF_SPECIALIZED(right, _PyFloat_ExactDealloc);
3892 11229900 : assert(opcode == POP_JUMP_FORWARD_IF_FALSE ||
3893 : opcode == POP_JUMP_BACKWARD_IF_FALSE ||
3894 : opcode == POP_JUMP_FORWARD_IF_TRUE ||
3895 : opcode == POP_JUMP_BACKWARD_IF_TRUE);
3896 11229900 : int jump = (9 << (sign + 1)) & when_to_jump_mask;
3897 11229900 : if (!jump) {
3898 4369390 : next_instr++;
3899 : }
3900 6860460 : else if (jump >= 8) {
3901 63191 : assert(opcode == POP_JUMP_BACKWARD_IF_TRUE ||
3902 : opcode == POP_JUMP_BACKWARD_IF_FALSE);
3903 63191 : JUMPBY(1 - oparg);
3904 63191 : CHECK_EVAL_BREAKER();
3905 : }
3906 : else {
3907 6797270 : assert(opcode == POP_JUMP_FORWARD_IF_TRUE ||
3908 : opcode == POP_JUMP_FORWARD_IF_FALSE);
3909 6797270 : JUMPBY(1 + oparg);
3910 : }
3911 11229900 : NOTRACE_DISPATCH();
3912 : }
3913 :
3914 126671000 : TARGET(COMPARE_OP_INT_JUMP) {
3915 126671000 : assert(cframe.use_tracing == 0);
3916 : // Combined: COMPARE_OP (int ? int) + POP_JUMP_(direction)_IF_(true/false)
3917 126671000 : _PyCompareOpCache *cache = (_PyCompareOpCache *)next_instr;
3918 126671000 : int when_to_jump_mask = cache->mask;
3919 126671000 : PyObject *right = TOP();
3920 126671000 : PyObject *left = SECOND();
3921 126671000 : DEOPT_IF(!PyLong_CheckExact(left), COMPARE_OP);
3922 126481000 : DEOPT_IF(!PyLong_CheckExact(right), COMPARE_OP);
3923 126473000 : DEOPT_IF((size_t)(Py_SIZE(left) + 1) > 2, COMPARE_OP);
3924 126149000 : DEOPT_IF((size_t)(Py_SIZE(right) + 1) > 2, COMPARE_OP);
3925 : STAT_INC(COMPARE_OP, hit);
3926 125958000 : assert(Py_ABS(Py_SIZE(left)) <= 1 && Py_ABS(Py_SIZE(right)) <= 1);
3927 125958000 : Py_ssize_t ileft = Py_SIZE(left) * ((PyLongObject *)left)->ob_digit[0];
3928 125958000 : Py_ssize_t iright = Py_SIZE(right) * ((PyLongObject *)right)->ob_digit[0];
3929 125958000 : int sign = (ileft > iright) - (ileft < iright);
3930 125958000 : JUMPBY(INLINE_CACHE_ENTRIES_COMPARE_OP);
3931 125958000 : NEXTOPARG();
3932 125958000 : STACK_SHRINK(2);
3933 125958000 : _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free);
3934 125958000 : _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free);
3935 125958000 : assert(opcode == POP_JUMP_FORWARD_IF_FALSE ||
3936 : opcode == POP_JUMP_BACKWARD_IF_FALSE ||
3937 : opcode == POP_JUMP_FORWARD_IF_TRUE ||
3938 : opcode == POP_JUMP_BACKWARD_IF_TRUE);
3939 125958000 : int jump = (9 << (sign + 1)) & when_to_jump_mask;
3940 125958000 : if (!jump) {
3941 36976300 : next_instr++;
3942 : }
3943 88981900 : else if (jump >= 8) {
3944 15129700 : assert(opcode == POP_JUMP_BACKWARD_IF_TRUE ||
3945 : opcode == POP_JUMP_BACKWARD_IF_FALSE);
3946 15129700 : JUMPBY(1 - oparg);
3947 15129700 : CHECK_EVAL_BREAKER();
3948 : }
3949 : else {
3950 73852200 : assert(opcode == POP_JUMP_FORWARD_IF_TRUE ||
3951 : opcode == POP_JUMP_FORWARD_IF_FALSE);
3952 73852200 : JUMPBY(1 + oparg);
3953 : }
3954 125958000 : NOTRACE_DISPATCH();
3955 : }
3956 :
3957 52532500 : TARGET(COMPARE_OP_STR_JUMP) {
3958 52532500 : assert(cframe.use_tracing == 0);
3959 : // Combined: COMPARE_OP (str == str or str != str) + POP_JUMP_(direction)_IF_(true/false)
3960 52532500 : _PyCompareOpCache *cache = (_PyCompareOpCache *)next_instr;
3961 52532500 : int when_to_jump_mask = cache->mask;
3962 52532500 : PyObject *right = TOP();
3963 52532500 : PyObject *left = SECOND();
3964 52532500 : DEOPT_IF(!PyUnicode_CheckExact(left), COMPARE_OP);
3965 52480000 : DEOPT_IF(!PyUnicode_CheckExact(right), COMPARE_OP);
3966 : STAT_INC(COMPARE_OP, hit);
3967 52424100 : int res = _PyUnicode_Equal(left, right);
3968 52424100 : if (res < 0) {
3969 0 : goto error;
3970 : }
3971 52424100 : assert(oparg == Py_EQ || oparg == Py_NE);
3972 52424100 : JUMPBY(INLINE_CACHE_ENTRIES_COMPARE_OP);
3973 52424100 : NEXTOPARG();
3974 52424100 : assert(opcode == POP_JUMP_FORWARD_IF_FALSE ||
3975 : opcode == POP_JUMP_BACKWARD_IF_FALSE ||
3976 : opcode == POP_JUMP_FORWARD_IF_TRUE ||
3977 : opcode == POP_JUMP_BACKWARD_IF_TRUE);
3978 52424100 : STACK_SHRINK(2);
3979 52424100 : _Py_DECREF_SPECIALIZED(left, _PyUnicode_ExactDealloc);
3980 52424100 : _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc);
3981 52424100 : assert(res == 0 || res == 1);
3982 52424100 : int sign = 1 - res;
3983 52424100 : int jump = (9 << (sign + 1)) & when_to_jump_mask;
3984 52424100 : if (!jump) {
3985 13121600 : next_instr++;
3986 : }
3987 39302500 : else if (jump >= 8) {
3988 4560160 : assert(opcode == POP_JUMP_BACKWARD_IF_TRUE ||
3989 : opcode == POP_JUMP_BACKWARD_IF_FALSE);
3990 4560160 : JUMPBY(1 - oparg);
3991 4560160 : CHECK_EVAL_BREAKER();
3992 : }
3993 : else {
3994 34742400 : assert(opcode == POP_JUMP_FORWARD_IF_TRUE ||
3995 : opcode == POP_JUMP_FORWARD_IF_FALSE);
3996 34742400 : JUMPBY(1 + oparg);
3997 : }
3998 52424100 : NOTRACE_DISPATCH();
3999 : }
4000 :
4001 34718600 : TARGET(IS_OP) {
4002 34718600 : PyObject *right = POP();
4003 34718600 : PyObject *left = TOP();
4004 34718600 : int res = Py_Is(left, right) ^ oparg;
4005 34718600 : PyObject *b = res ? Py_True : Py_False;
4006 34718600 : Py_INCREF(b);
4007 34718600 : SET_TOP(b);
4008 34718600 : Py_DECREF(left);
4009 34718600 : Py_DECREF(right);
4010 34718600 : DISPATCH();
4011 : }
4012 :
4013 91809200 : TARGET(CONTAINS_OP) {
4014 91809200 : PyObject *right = POP();
4015 91809200 : PyObject *left = POP();
4016 91809200 : int res = PySequence_Contains(right, left);
4017 91809200 : Py_DECREF(left);
4018 91809200 : Py_DECREF(right);
4019 91809200 : if (res < 0) {
4020 60 : goto error;
4021 : }
4022 91809200 : PyObject *b = (res^oparg) ? Py_True : Py_False;
4023 91809200 : Py_INCREF(b);
4024 91809200 : PUSH(b);
4025 91809200 : DISPATCH();
4026 : }
4027 :
4028 179 : TARGET(CHECK_EG_MATCH) {
4029 179 : PyObject *match_type = POP();
4030 179 : if (check_except_star_type_valid(tstate, match_type) < 0) {
4031 4 : Py_DECREF(match_type);
4032 4 : goto error;
4033 : }
4034 :
4035 175 : PyObject *exc_value = TOP();
4036 175 : PyObject *match = NULL, *rest = NULL;
4037 175 : int res = exception_group_match(exc_value, match_type,
4038 : &match, &rest);
4039 175 : Py_DECREF(match_type);
4040 175 : if (res < 0) {
4041 0 : goto error;
4042 : }
4043 :
4044 175 : if (match == NULL || rest == NULL) {
4045 0 : assert(match == NULL);
4046 0 : assert(rest == NULL);
4047 0 : goto error;
4048 : }
4049 175 : if (Py_IsNone(match)) {
4050 27 : PUSH(match);
4051 27 : Py_XDECREF(rest);
4052 : }
4053 : else {
4054 : /* Total or partial match - update the stack from
4055 : * [val]
4056 : * to
4057 : * [rest, match]
4058 : * (rest can be Py_None)
4059 : */
4060 :
4061 148 : SET_TOP(rest);
4062 148 : PUSH(match);
4063 148 : PyErr_SetExcInfo(NULL, Py_NewRef(match), NULL);
4064 148 : Py_DECREF(exc_value);
4065 : }
4066 175 : DISPATCH();
4067 : }
4068 :
4069 4606700 : TARGET(CHECK_EXC_MATCH) {
4070 4606700 : PyObject *right = POP();
4071 4606700 : PyObject *left = TOP();
4072 4606700 : assert(PyExceptionInstance_Check(left));
4073 4606700 : if (check_except_type_valid(tstate, right) < 0) {
4074 10 : Py_DECREF(right);
4075 10 : goto error;
4076 : }
4077 :
4078 4606700 : int res = PyErr_GivenExceptionMatches(left, right);
4079 4606700 : Py_DECREF(right);
4080 4606700 : PUSH(Py_NewRef(res ? Py_True : Py_False));
4081 4606700 : DISPATCH();
4082 : }
4083 :
4084 1450810 : TARGET(IMPORT_NAME) {
4085 1450810 : PyObject *name = GETITEM(names, oparg);
4086 1450810 : PyObject *fromlist = POP();
4087 1450810 : PyObject *level = TOP();
4088 : PyObject *res;
4089 1450810 : res = import_name(tstate, frame, name, fromlist, level);
4090 1450810 : Py_DECREF(level);
4091 1450810 : Py_DECREF(fromlist);
4092 1450810 : SET_TOP(res);
4093 1450810 : if (res == NULL)
4094 14744 : goto error;
4095 1436070 : DISPATCH();
4096 : }
4097 :
4098 41176 : TARGET(IMPORT_STAR) {
4099 41176 : PyObject *from = POP(), *locals;
4100 : int err;
4101 41176 : if (_PyFrame_FastToLocalsWithError(frame) < 0) {
4102 0 : Py_DECREF(from);
4103 0 : goto error;
4104 : }
4105 :
4106 41176 : locals = LOCALS();
4107 41176 : if (locals == NULL) {
4108 0 : _PyErr_SetString(tstate, PyExc_SystemError,
4109 : "no locals found during 'import *'");
4110 0 : Py_DECREF(from);
4111 0 : goto error;
4112 : }
4113 41176 : err = import_all_from(tstate, locals, from);
4114 41176 : _PyFrame_LocalsToFast(frame, 0);
4115 41176 : Py_DECREF(from);
4116 41176 : if (err != 0)
4117 2 : goto error;
4118 41174 : DISPATCH();
4119 : }
4120 :
4121 809909 : TARGET(IMPORT_FROM) {
4122 809909 : PyObject *name = GETITEM(names, oparg);
4123 809909 : PyObject *from = TOP();
4124 : PyObject *res;
4125 809909 : res = import_from(tstate, from, name);
4126 809909 : PUSH(res);
4127 809909 : if (res == NULL)
4128 4768 : goto error;
4129 805141 : DISPATCH();
4130 : }
4131 :
4132 33102900 : TARGET(JUMP_FORWARD) {
4133 33102900 : JUMPBY(oparg);
4134 33102900 : DISPATCH();
4135 : }
4136 :
4137 1131840 : TARGET(JUMP_BACKWARD) {
4138 1131840 : _PyCode_Warmup(frame->f_code);
4139 1131840 : JUMP_TO_INSTRUCTION(JUMP_BACKWARD_QUICK);
4140 : }
4141 :
4142 39243000 : TARGET(POP_JUMP_BACKWARD_IF_FALSE) {
4143 39243000 : PREDICTED(POP_JUMP_BACKWARD_IF_FALSE);
4144 39243000 : PyObject *cond = POP();
4145 39243000 : if (Py_IsTrue(cond)) {
4146 7661950 : _Py_DECREF_NO_DEALLOC(cond);
4147 7661950 : DISPATCH();
4148 : }
4149 31581000 : if (Py_IsFalse(cond)) {
4150 21176600 : _Py_DECREF_NO_DEALLOC(cond);
4151 21176600 : JUMPBY(-oparg);
4152 21176600 : CHECK_EVAL_BREAKER();
4153 21176600 : DISPATCH();
4154 : }
4155 10404400 : int err = PyObject_IsTrue(cond);
4156 10404400 : Py_DECREF(cond);
4157 10404400 : if (err > 0)
4158 : ;
4159 2772180 : else if (err == 0) {
4160 2772180 : JUMPBY(-oparg);
4161 2772180 : CHECK_EVAL_BREAKER();
4162 : }
4163 : else
4164 1 : goto error;
4165 10404400 : DISPATCH();
4166 : }
4167 :
4168 228366000 : TARGET(POP_JUMP_FORWARD_IF_FALSE) {
4169 228366000 : PREDICTED(POP_JUMP_FORWARD_IF_FALSE);
4170 228366000 : PyObject *cond = POP();
4171 228366000 : if (Py_IsTrue(cond)) {
4172 64593000 : _Py_DECREF_NO_DEALLOC(cond);
4173 : }
4174 163773000 : else if (Py_IsFalse(cond)) {
4175 114193000 : _Py_DECREF_NO_DEALLOC(cond);
4176 114193000 : JUMPBY(oparg);
4177 : }
4178 : else {
4179 49579400 : int err = PyObject_IsTrue(cond);
4180 49579400 : Py_DECREF(cond);
4181 49579400 : if (err > 0)
4182 : ;
4183 23382200 : else if (err == 0) {
4184 23382200 : JUMPBY(oparg);
4185 : }
4186 : else
4187 5 : goto error;
4188 : }
4189 228366000 : DISPATCH();
4190 : }
4191 :
4192 10870200 : TARGET(POP_JUMP_BACKWARD_IF_TRUE) {
4193 10870200 : PyObject *cond = POP();
4194 10870200 : if (Py_IsFalse(cond)) {
4195 1558290 : _Py_DECREF_NO_DEALLOC(cond);
4196 1558290 : DISPATCH();
4197 : }
4198 9311920 : if (Py_IsTrue(cond)) {
4199 8103920 : _Py_DECREF_NO_DEALLOC(cond);
4200 8103920 : JUMPBY(-oparg);
4201 8103920 : CHECK_EVAL_BREAKER();
4202 8103920 : DISPATCH();
4203 : }
4204 1208000 : int err = PyObject_IsTrue(cond);
4205 1208000 : Py_DECREF(cond);
4206 1208000 : if (err > 0) {
4207 935610 : JUMPBY(-oparg);
4208 935610 : CHECK_EVAL_BREAKER();
4209 : }
4210 272392 : else if (err == 0)
4211 : ;
4212 : else
4213 0 : goto error;
4214 1207810 : DISPATCH();
4215 : }
4216 :
4217 87224000 : TARGET(POP_JUMP_FORWARD_IF_TRUE) {
4218 87224000 : PyObject *cond = POP();
4219 87224000 : if (Py_IsFalse(cond)) {
4220 35372200 : _Py_DECREF_NO_DEALLOC(cond);
4221 : }
4222 51851900 : else if (Py_IsTrue(cond)) {
4223 24273900 : _Py_DECREF_NO_DEALLOC(cond);
4224 24273900 : JUMPBY(oparg);
4225 : }
4226 : else {
4227 27578000 : int err = PyObject_IsTrue(cond);
4228 27578000 : Py_DECREF(cond);
4229 27578000 : if (err > 0) {
4230 23244200 : JUMPBY(oparg);
4231 : }
4232 4333740 : else if (err == 0)
4233 : ;
4234 : else
4235 1 : goto error;
4236 : }
4237 87224000 : DISPATCH();
4238 : }
4239 :
4240 506123 : TARGET(POP_JUMP_BACKWARD_IF_NOT_NONE) {
4241 506123 : PyObject *value = POP();
4242 506123 : if (!Py_IsNone(value)) {
4243 283164 : Py_DECREF(value);
4244 283164 : JUMPBY(-oparg);
4245 283164 : CHECK_EVAL_BREAKER();
4246 283164 : DISPATCH();
4247 : }
4248 222959 : _Py_DECREF_NO_DEALLOC(value);
4249 222959 : DISPATCH();
4250 : }
4251 :
4252 38849300 : TARGET(POP_JUMP_FORWARD_IF_NOT_NONE) {
4253 38849300 : PyObject *value = POP();
4254 38849300 : if (!Py_IsNone(value)) {
4255 24318800 : JUMPBY(oparg);
4256 : }
4257 38849300 : Py_DECREF(value);
4258 38849300 : DISPATCH();
4259 : }
4260 :
4261 2473990 : TARGET(POP_JUMP_BACKWARD_IF_NONE) {
4262 2473990 : PyObject *value = POP();
4263 2473990 : if (Py_IsNone(value)) {
4264 1055660 : _Py_DECREF_NO_DEALLOC(value);
4265 1055660 : JUMPBY(-oparg);
4266 1055660 : CHECK_EVAL_BREAKER();
4267 : }
4268 : else {
4269 1418340 : Py_DECREF(value);
4270 : }
4271 2473990 : DISPATCH();
4272 : }
4273 :
4274 38553200 : TARGET(POP_JUMP_FORWARD_IF_NONE) {
4275 38553200 : PyObject *value = POP();
4276 38553200 : if (Py_IsNone(value)) {
4277 18720700 : _Py_DECREF_NO_DEALLOC(value);
4278 18720700 : JUMPBY(oparg);
4279 : }
4280 : else {
4281 19832500 : Py_DECREF(value);
4282 : }
4283 38553200 : DISPATCH();
4284 : }
4285 :
4286 11092300 : TARGET(JUMP_IF_FALSE_OR_POP) {
4287 11092300 : PyObject *cond = TOP();
4288 : int err;
4289 11092300 : if (Py_IsTrue(cond)) {
4290 8189730 : STACK_SHRINK(1);
4291 8189730 : _Py_DECREF_NO_DEALLOC(cond);
4292 8189730 : DISPATCH();
4293 : }
4294 2902620 : if (Py_IsFalse(cond)) {
4295 2686430 : JUMPBY(oparg);
4296 2686430 : DISPATCH();
4297 : }
4298 216186 : err = PyObject_IsTrue(cond);
4299 216186 : if (err > 0) {
4300 16371 : STACK_SHRINK(1);
4301 16371 : Py_DECREF(cond);
4302 : }
4303 199815 : else if (err == 0)
4304 199815 : JUMPBY(oparg);
4305 : else
4306 0 : goto error;
4307 216186 : DISPATCH();
4308 : }
4309 :
4310 5394310 : TARGET(JUMP_IF_TRUE_OR_POP) {
4311 5394310 : PyObject *cond = TOP();
4312 : int err;
4313 5394310 : if (Py_IsFalse(cond)) {
4314 2450810 : STACK_SHRINK(1);
4315 2450810 : _Py_DECREF_NO_DEALLOC(cond);
4316 2450810 : DISPATCH();
4317 : }
4318 2943500 : if (Py_IsTrue(cond)) {
4319 169987 : JUMPBY(oparg);
4320 169987 : DISPATCH();
4321 : }
4322 2773510 : err = PyObject_IsTrue(cond);
4323 2773510 : if (err > 0) {
4324 1850440 : JUMPBY(oparg);
4325 : }
4326 923068 : else if (err == 0) {
4327 923068 : STACK_SHRINK(1);
4328 923068 : Py_DECREF(cond);
4329 : }
4330 : else
4331 0 : goto error;
4332 2773510 : DISPATCH();
4333 : }
4334 :
4335 5985620 : TARGET(JUMP_BACKWARD_NO_INTERRUPT) {
4336 : /* This bytecode is used in the `yield from` or `await` loop.
4337 : * If there is an interrupt, we want it handled in the innermost
4338 : * generator or coroutine, so we deliberately do not check it here.
4339 : * (see bpo-30039).
4340 : */
4341 5985620 : JUMPBY(-oparg);
4342 5985620 : DISPATCH();
4343 : }
4344 :
4345 133432000 : TARGET(JUMP_BACKWARD_QUICK) {
4346 134564000 : PREDICTED(JUMP_BACKWARD_QUICK);
4347 134564000 : assert(oparg < INSTR_OFFSET());
4348 134564000 : JUMPBY(-oparg);
4349 134564000 : CHECK_EVAL_BREAKER();
4350 134563000 : DISPATCH();
4351 : }
4352 :
4353 221 : TARGET(GET_LEN) {
4354 : // PUSH(len(TOS))
4355 221 : Py_ssize_t len_i = PyObject_Length(TOP());
4356 221 : if (len_i < 0) {
4357 0 : goto error;
4358 : }
4359 221 : PyObject *len_o = PyLong_FromSsize_t(len_i);
4360 221 : if (len_o == NULL) {
4361 0 : goto error;
4362 : }
4363 221 : PUSH(len_o);
4364 221 : DISPATCH();
4365 : }
4366 :
4367 13960 : TARGET(MATCH_CLASS) {
4368 : // Pop TOS and TOS1. Set TOS to a tuple of attributes on success, or
4369 : // None on failure.
4370 13960 : PyObject *names = POP();
4371 13960 : PyObject *type = POP();
4372 13960 : PyObject *subject = TOP();
4373 13960 : assert(PyTuple_CheckExact(names));
4374 13960 : PyObject *attrs = match_class(tstate, subject, type, oparg, names);
4375 13960 : Py_DECREF(names);
4376 13960 : Py_DECREF(type);
4377 13960 : if (attrs) {
4378 : // Success!
4379 5925 : assert(PyTuple_CheckExact(attrs));
4380 5925 : SET_TOP(attrs);
4381 : }
4382 8035 : else if (_PyErr_Occurred(tstate)) {
4383 : // Error!
4384 8 : goto error;
4385 : }
4386 : else {
4387 : // Failure!
4388 8027 : Py_INCREF(Py_None);
4389 8027 : SET_TOP(Py_None);
4390 : }
4391 13952 : Py_DECREF(subject);
4392 13952 : DISPATCH();
4393 : }
4394 :
4395 92 : TARGET(MATCH_MAPPING) {
4396 92 : PyObject *subject = TOP();
4397 92 : int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_MAPPING;
4398 92 : PyObject *res = match ? Py_True : Py_False;
4399 92 : Py_INCREF(res);
4400 92 : PUSH(res);
4401 : PREDICT(POP_JUMP_FORWARD_IF_FALSE);
4402 : PREDICT(POP_JUMP_BACKWARD_IF_FALSE);
4403 92 : DISPATCH();
4404 : }
4405 :
4406 235 : TARGET(MATCH_SEQUENCE) {
4407 235 : PyObject *subject = TOP();
4408 235 : int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_SEQUENCE;
4409 235 : PyObject *res = match ? Py_True : Py_False;
4410 235 : Py_INCREF(res);
4411 235 : PUSH(res);
4412 : PREDICT(POP_JUMP_FORWARD_IF_FALSE);
4413 : PREDICT(POP_JUMP_BACKWARD_IF_FALSE);
4414 235 : DISPATCH();
4415 : }
4416 :
4417 47 : TARGET(MATCH_KEYS) {
4418 : // On successful match, PUSH(values). Otherwise, PUSH(None).
4419 47 : PyObject *keys = TOP();
4420 47 : PyObject *subject = SECOND();
4421 47 : PyObject *values_or_none = match_keys(tstate, subject, keys);
4422 47 : if (values_or_none == NULL) {
4423 1 : goto error;
4424 : }
4425 46 : PUSH(values_or_none);
4426 46 : DISPATCH();
4427 : }
4428 :
4429 43638900 : TARGET(GET_ITER) {
4430 : /* before: [obj]; after [getiter(obj)] */
4431 43638900 : PyObject *iterable = TOP();
4432 43638900 : PyObject *iter = PyObject_GetIter(iterable);
4433 43638900 : Py_DECREF(iterable);
4434 43638900 : SET_TOP(iter);
4435 43638900 : if (iter == NULL)
4436 86 : goto error;
4437 43638800 : DISPATCH();
4438 : }
4439 :
4440 770664 : TARGET(GET_YIELD_FROM_ITER) {
4441 : /* before: [obj]; after [getiter(obj)] */
4442 770664 : PyObject *iterable = TOP();
4443 : PyObject *iter;
4444 770664 : if (PyCoro_CheckExact(iterable)) {
4445 : /* `iterable` is a coroutine */
4446 2 : if (!(frame->f_code->co_flags & (CO_COROUTINE | CO_ITERABLE_COROUTINE))) {
4447 : /* and it is used in a 'yield from' expression of a
4448 : regular generator. */
4449 1 : Py_DECREF(iterable);
4450 1 : SET_TOP(NULL);
4451 1 : _PyErr_SetString(tstate, PyExc_TypeError,
4452 : "cannot 'yield from' a coroutine object "
4453 : "in a non-coroutine generator");
4454 1 : goto error;
4455 : }
4456 : }
4457 770662 : else if (!PyGen_CheckExact(iterable)) {
4458 : /* `iterable` is not a generator. */
4459 14059 : iter = PyObject_GetIter(iterable);
4460 14059 : Py_DECREF(iterable);
4461 14059 : SET_TOP(iter);
4462 14059 : if (iter == NULL)
4463 0 : goto error;
4464 : }
4465 : PREDICT(LOAD_CONST);
4466 770663 : DISPATCH();
4467 : }
4468 :
4469 5148860 : TARGET(FOR_ITER) {
4470 109425000 : PREDICTED(FOR_ITER);
4471 : /* before: [iter]; after: [iter, iter()] *or* [] */
4472 109425000 : PyObject *iter = TOP();
4473 109425000 : PyObject *next = (*Py_TYPE(iter)->tp_iternext)(iter);
4474 109425000 : if (next != NULL) {
4475 78505200 : PUSH(next);
4476 78505200 : JUMPBY(INLINE_CACHE_ENTRIES_FOR_ITER);
4477 78505200 : DISPATCH();
4478 : }
4479 30919700 : if (_PyErr_Occurred(tstate)) {
4480 6347 : if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) {
4481 1140 : goto error;
4482 : }
4483 5207 : else if (tstate->c_tracefunc != NULL) {
4484 5 : call_exc_trace(tstate->c_tracefunc, tstate->c_traceobj, tstate, frame);
4485 : }
4486 5207 : _PyErr_Clear(tstate);
4487 : }
4488 30913300 : iterator_exhausted_no_error:
4489 : /* iterator ended normally */
4490 39902400 : assert(!_PyErr_Occurred(tstate));
4491 39902400 : Py_DECREF(POP());
4492 39902400 : JUMPBY(INLINE_CACHE_ENTRIES_FOR_ITER + oparg);
4493 39902400 : DISPATCH();
4494 : }
4495 :
4496 105090000 : TARGET(FOR_ITER_ADAPTIVE) {
4497 105090000 : assert(cframe.use_tracing == 0);
4498 105090000 : _PyForIterCache *cache = (_PyForIterCache *)next_instr;
4499 105090000 : if (ADAPTIVE_COUNTER_IS_ZERO(cache)) {
4500 813756 : next_instr--;
4501 813756 : _Py_Specialize_ForIter(TOP(), next_instr);
4502 813756 : NOTRACE_DISPATCH_SAME_OPARG();
4503 : }
4504 : else {
4505 : STAT_INC(FOR_ITER, deferred);
4506 104276000 : DECREMENT_ADAPTIVE_COUNTER(cache);
4507 104276000 : JUMP_TO_INSTRUCTION(FOR_ITER);
4508 : }
4509 : }
4510 :
4511 44946100 : TARGET(FOR_ITER_LIST) {
4512 44946100 : assert(cframe.use_tracing == 0);
4513 44946100 : _PyListIterObject *it = (_PyListIterObject *)TOP();
4514 44946100 : DEOPT_IF(Py_TYPE(it) != &PyListIter_Type, FOR_ITER);
4515 : STAT_INC(FOR_ITER, hit);
4516 44608400 : PyListObject *seq = it->it_seq;
4517 44608400 : if (seq == NULL) {
4518 1589 : goto iterator_exhausted_no_error;
4519 : }
4520 44606800 : if (it->it_index < PyList_GET_SIZE(seq)) {
4521 36495600 : PyObject *next = PyList_GET_ITEM(seq, it->it_index++);
4522 36495600 : Py_INCREF(next);
4523 36495600 : PUSH(next);
4524 36495600 : JUMPBY(INLINE_CACHE_ENTRIES_FOR_ITER);
4525 36495600 : NOTRACE_DISPATCH();
4526 : }
4527 8111220 : it->it_seq = NULL;
4528 8111220 : Py_DECREF(seq);
4529 8111220 : goto iterator_exhausted_no_error;
4530 : }
4531 :
4532 45817900 : TARGET(FOR_ITER_RANGE) {
4533 45817900 : assert(cframe.use_tracing == 0);
4534 45817900 : _PyRangeIterObject *r = (_PyRangeIterObject *)TOP();
4535 45817900 : DEOPT_IF(Py_TYPE(r) != &PyRangeIter_Type, FOR_ITER);
4536 : STAT_INC(FOR_ITER, hit);
4537 45812000 : _Py_CODEUNIT next = next_instr[INLINE_CACHE_ENTRIES_FOR_ITER];
4538 45812000 : assert(_PyOpcode_Deopt[_Py_OPCODE(next)] == STORE_FAST);
4539 45812000 : if (r->index >= r->len) {
4540 871067 : goto iterator_exhausted_no_error;
4541 : }
4542 44940900 : long value = (long)(r->start +
4543 44940900 : (unsigned long)(r->index++) * r->step);
4544 44940900 : if (_PyLong_AssignValue(&GETLOCAL(_Py_OPARG(next)), value) < 0) {
4545 0 : goto error;
4546 : }
4547 : // The STORE_FAST is already done.
4548 44940900 : JUMPBY(INLINE_CACHE_ENTRIES_FOR_ITER + 1);
4549 44940900 : NOTRACE_DISPATCH();
4550 : }
4551 :
4552 378 : TARGET(BEFORE_ASYNC_WITH) {
4553 378 : PyObject *mgr = TOP();
4554 : PyObject *res;
4555 378 : PyObject *enter = _PyObject_LookupSpecial(mgr, &_Py_ID(__aenter__));
4556 378 : if (enter == NULL) {
4557 2 : if (!_PyErr_Occurred(tstate)) {
4558 2 : _PyErr_Format(tstate, PyExc_TypeError,
4559 : "'%.200s' object does not support the "
4560 : "asynchronous context manager protocol",
4561 2 : Py_TYPE(mgr)->tp_name);
4562 : }
4563 2 : goto error;
4564 : }
4565 376 : PyObject *exit = _PyObject_LookupSpecial(mgr, &_Py_ID(__aexit__));
4566 376 : if (exit == NULL) {
4567 1 : if (!_PyErr_Occurred(tstate)) {
4568 1 : _PyErr_Format(tstate, PyExc_TypeError,
4569 : "'%.200s' object does not support the "
4570 : "asynchronous context manager protocol "
4571 : "(missed __aexit__ method)",
4572 1 : Py_TYPE(mgr)->tp_name);
4573 : }
4574 1 : Py_DECREF(enter);
4575 1 : goto error;
4576 : }
4577 375 : SET_TOP(exit);
4578 375 : Py_DECREF(mgr);
4579 375 : res = _PyObject_CallNoArgs(enter);
4580 375 : Py_DECREF(enter);
4581 375 : if (res == NULL)
4582 0 : goto error;
4583 375 : PUSH(res);
4584 : PREDICT(GET_AWAITABLE);
4585 375 : DISPATCH();
4586 : }
4587 :
4588 4456220 : TARGET(BEFORE_WITH) {
4589 4456220 : PyObject *mgr = TOP();
4590 : PyObject *res;
4591 4456220 : PyObject *enter = _PyObject_LookupSpecial(mgr, &_Py_ID(__enter__));
4592 4456220 : if (enter == NULL) {
4593 4 : if (!_PyErr_Occurred(tstate)) {
4594 3 : _PyErr_Format(tstate, PyExc_TypeError,
4595 : "'%.200s' object does not support the "
4596 : "context manager protocol",
4597 3 : Py_TYPE(mgr)->tp_name);
4598 : }
4599 4 : goto error;
4600 : }
4601 4456220 : PyObject *exit = _PyObject_LookupSpecial(mgr, &_Py_ID(__exit__));
4602 4456220 : if (exit == NULL) {
4603 3 : if (!_PyErr_Occurred(tstate)) {
4604 2 : _PyErr_Format(tstate, PyExc_TypeError,
4605 : "'%.200s' object does not support the "
4606 : "context manager protocol "
4607 : "(missed __exit__ method)",
4608 2 : Py_TYPE(mgr)->tp_name);
4609 : }
4610 3 : Py_DECREF(enter);
4611 3 : goto error;
4612 : }
4613 4456220 : SET_TOP(exit);
4614 4456220 : Py_DECREF(mgr);
4615 4456220 : res = _PyObject_CallNoArgs(enter);
4616 4456220 : Py_DECREF(enter);
4617 4456220 : if (res == NULL) {
4618 76 : goto error;
4619 : }
4620 4456140 : PUSH(res);
4621 4456140 : DISPATCH();
4622 : }
4623 :
4624 218474 : TARGET(WITH_EXCEPT_START) {
4625 : /* At the top of the stack are 4 values:
4626 : - TOP = exc_info()
4627 : - SECOND = previous exception
4628 : - THIRD: lasti of exception in exc_info()
4629 : - FOURTH: the context.__exit__ bound method
4630 : We call FOURTH(type(TOP), TOP, GetTraceback(TOP)).
4631 : Then we push the __exit__ return value.
4632 : */
4633 : PyObject *exit_func;
4634 : PyObject *exc, *val, *tb, *res;
4635 :
4636 218474 : val = TOP();
4637 218474 : assert(val && PyExceptionInstance_Check(val));
4638 218474 : exc = PyExceptionInstance_Class(val);
4639 218474 : tb = PyException_GetTraceback(val);
4640 218474 : Py_XDECREF(tb);
4641 218474 : assert(PyLong_Check(PEEK(3)));
4642 218474 : exit_func = PEEK(4);
4643 218474 : PyObject *stack[4] = {NULL, exc, val, tb};
4644 218474 : res = PyObject_Vectorcall(exit_func, stack + 1,
4645 : 3 | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL);
4646 218474 : if (res == NULL)
4647 41 : goto error;
4648 :
4649 218433 : PUSH(res);
4650 218433 : DISPATCH();
4651 : }
4652 :
4653 4923050 : TARGET(PUSH_EXC_INFO) {
4654 4923050 : PyObject *value = TOP();
4655 :
4656 4923050 : _PyErr_StackItem *exc_info = tstate->exc_info;
4657 4923050 : if (exc_info->exc_value != NULL) {
4658 4587190 : SET_TOP(exc_info->exc_value);
4659 : }
4660 : else {
4661 335857 : Py_INCREF(Py_None);
4662 335857 : SET_TOP(Py_None);
4663 : }
4664 :
4665 4923050 : Py_INCREF(value);
4666 4923050 : PUSH(value);
4667 4923050 : assert(PyExceptionInstance_Check(value));
4668 4923050 : exc_info->exc_value = value;
4669 :
4670 4923050 : DISPATCH();
4671 : }
4672 :
4673 74063100 : TARGET(LOAD_ATTR_METHOD_WITH_VALUES) {
4674 : /* Cached method object */
4675 74063100 : assert(cframe.use_tracing == 0);
4676 74063100 : PyObject *self = TOP();
4677 74063100 : PyTypeObject *self_cls = Py_TYPE(self);
4678 74063100 : _PyLoadMethodCache *cache = (_PyLoadMethodCache *)next_instr;
4679 74063100 : uint32_t type_version = read_u32(cache->type_version);
4680 74063100 : assert(type_version != 0);
4681 74063100 : DEOPT_IF(self_cls->tp_version_tag != type_version, LOAD_ATTR);
4682 69510500 : assert(self_cls->tp_flags & Py_TPFLAGS_MANAGED_DICT);
4683 69510500 : PyDictObject *dict = *(PyDictObject**)_PyObject_ManagedDictPointer(self);
4684 69510500 : DEOPT_IF(dict != NULL, LOAD_ATTR);
4685 69491200 : PyHeapTypeObject *self_heap_type = (PyHeapTypeObject *)self_cls;
4686 69491200 : DEOPT_IF(self_heap_type->ht_cached_keys->dk_version !=
4687 : read_u32(cache->keys_version), LOAD_ATTR);
4688 : STAT_INC(LOAD_ATTR, hit);
4689 69361800 : PyObject *res = read_obj(cache->descr);
4690 69361800 : assert(res != NULL);
4691 69361800 : assert(_PyType_HasFeature(Py_TYPE(res), Py_TPFLAGS_METHOD_DESCRIPTOR));
4692 69361800 : Py_INCREF(res);
4693 69361800 : SET_TOP(res);
4694 69361800 : PUSH(self);
4695 69361800 : JUMPBY(INLINE_CACHE_ENTRIES_LOAD_ATTR);
4696 69361800 : NOTRACE_DISPATCH();
4697 : }
4698 :
4699 897004 : TARGET(LOAD_ATTR_METHOD_WITH_DICT) {
4700 : /* Can be either a managed dict, or a tp_dictoffset offset.*/
4701 897004 : assert(cframe.use_tracing == 0);
4702 897004 : PyObject *self = TOP();
4703 897004 : PyTypeObject *self_cls = Py_TYPE(self);
4704 897004 : _PyLoadMethodCache *cache = (_PyLoadMethodCache *)next_instr;
4705 :
4706 897004 : DEOPT_IF(self_cls->tp_version_tag != read_u32(cache->type_version),
4707 : LOAD_ATTR);
4708 : /* Treat index as a signed 16 bit value */
4709 886022 : Py_ssize_t dictoffset = self_cls->tp_dictoffset;
4710 886022 : assert(dictoffset > 0);
4711 886022 : PyDictObject **dictptr = (PyDictObject**)(((char *)self)+dictoffset);
4712 886022 : PyDictObject *dict = *dictptr;
4713 886022 : DEOPT_IF(dict == NULL, LOAD_ATTR);
4714 885815 : DEOPT_IF(dict->ma_keys->dk_version != read_u32(cache->keys_version),
4715 : LOAD_ATTR);
4716 : STAT_INC(LOAD_ATTR, hit);
4717 768198 : PyObject *res = read_obj(cache->descr);
4718 768198 : assert(res != NULL);
4719 768198 : assert(_PyType_HasFeature(Py_TYPE(res), Py_TPFLAGS_METHOD_DESCRIPTOR));
4720 768198 : Py_INCREF(res);
4721 768198 : SET_TOP(res);
4722 768198 : PUSH(self);
4723 768198 : JUMPBY(INLINE_CACHE_ENTRIES_LOAD_ATTR);
4724 768198 : NOTRACE_DISPATCH();
4725 : }
4726 :
4727 173618000 : TARGET(LOAD_ATTR_METHOD_NO_DICT) {
4728 173618000 : assert(cframe.use_tracing == 0);
4729 173618000 : PyObject *self = TOP();
4730 173618000 : PyTypeObject *self_cls = Py_TYPE(self);
4731 173618000 : _PyLoadMethodCache *cache = (_PyLoadMethodCache *)next_instr;
4732 173618000 : uint32_t type_version = read_u32(cache->type_version);
4733 173618000 : DEOPT_IF(self_cls->tp_version_tag != type_version, LOAD_ATTR);
4734 173493000 : assert(self_cls->tp_dictoffset == 0);
4735 : STAT_INC(LOAD_ATTR, hit);
4736 173493000 : PyObject *res = read_obj(cache->descr);
4737 173493000 : assert(res != NULL);
4738 173493000 : assert(_PyType_HasFeature(Py_TYPE(res), Py_TPFLAGS_METHOD_DESCRIPTOR));
4739 173493000 : Py_INCREF(res);
4740 173493000 : SET_TOP(res);
4741 173493000 : PUSH(self);
4742 173493000 : JUMPBY(INLINE_CACHE_ENTRIES_LOAD_ATTR);
4743 173493000 : NOTRACE_DISPATCH();
4744 : }
4745 :
4746 12911700 : TARGET(LOAD_ATTR_METHOD_LAZY_DICT) {
4747 12911700 : assert(cframe.use_tracing == 0);
4748 12911700 : PyObject *self = TOP();
4749 12911700 : PyTypeObject *self_cls = Py_TYPE(self);
4750 12911700 : _PyLoadMethodCache *cache = (_PyLoadMethodCache *)next_instr;
4751 12911700 : uint32_t type_version = read_u32(cache->type_version);
4752 12911700 : DEOPT_IF(self_cls->tp_version_tag != type_version, LOAD_ATTR);
4753 12852000 : Py_ssize_t dictoffset = self_cls->tp_dictoffset;
4754 12852000 : assert(dictoffset > 0);
4755 12852000 : PyObject *dict = *(PyObject **)((char *)self + dictoffset);
4756 : /* This object has a __dict__, just not yet created */
4757 12852000 : DEOPT_IF(dict != NULL, LOAD_ATTR);
4758 : STAT_INC(LOAD_ATTR, hit);
4759 12851800 : PyObject *res = read_obj(cache->descr);
4760 12851800 : assert(res != NULL);
4761 12851800 : assert(_PyType_HasFeature(Py_TYPE(res), Py_TPFLAGS_METHOD_DESCRIPTOR));
4762 12851800 : Py_INCREF(res);
4763 12851800 : SET_TOP(res);
4764 12851800 : PUSH(self);
4765 12851800 : JUMPBY(INLINE_CACHE_ENTRIES_LOAD_ATTR);
4766 12851800 : NOTRACE_DISPATCH();
4767 : }
4768 :
4769 21072800 : TARGET(CALL_BOUND_METHOD_EXACT_ARGS) {
4770 21072800 : DEOPT_IF(is_method(stack_pointer, oparg), CALL);
4771 19822600 : PyObject *function = PEEK(oparg + 1);
4772 19822600 : DEOPT_IF(Py_TYPE(function) != &PyMethod_Type, CALL);
4773 : STAT_INC(CALL, hit);
4774 19809600 : PyObject *meth = ((PyMethodObject *)function)->im_func;
4775 19809600 : PyObject *self = ((PyMethodObject *)function)->im_self;
4776 19809600 : Py_INCREF(meth);
4777 19809600 : Py_INCREF(self);
4778 19809600 : PEEK(oparg + 1) = self;
4779 19809600 : PEEK(oparg + 2) = meth;
4780 19809600 : Py_DECREF(function);
4781 19809600 : goto call_exact_args;
4782 : }
4783 :
4784 17169000 : TARGET(KW_NAMES) {
4785 17169000 : assert(call_shape.kwnames == NULL);
4786 17169000 : assert(oparg < PyTuple_GET_SIZE(consts));
4787 17169000 : call_shape.kwnames = GETITEM(consts, oparg);
4788 17169000 : DISPATCH();
4789 : }
4790 :
4791 41820800 : TARGET(CALL) {
4792 : int total_args, is_meth;
4793 192184000 : call_function:
4794 192184000 : is_meth = is_method(stack_pointer, oparg);
4795 192184000 : PyObject *function = PEEK(oparg + 1);
4796 192184000 : if (!is_meth && Py_TYPE(function) == &PyMethod_Type) {
4797 12102700 : PyObject *meth = ((PyMethodObject *)function)->im_func;
4798 12102700 : PyObject *self = ((PyMethodObject *)function)->im_self;
4799 12102700 : Py_INCREF(meth);
4800 12102700 : Py_INCREF(self);
4801 12102700 : PEEK(oparg+1) = self;
4802 12102700 : PEEK(oparg+2) = meth;
4803 12102700 : Py_DECREF(function);
4804 12102700 : is_meth = 1;
4805 : }
4806 192184000 : total_args = oparg + is_meth;
4807 192184000 : function = PEEK(total_args + 1);
4808 192184000 : int positional_args = total_args - KWNAMES_LEN();
4809 : // Check if the call can be inlined or not
4810 192184000 : if (Py_TYPE(function) == &PyFunction_Type && tstate->interp->eval_frame == NULL) {
4811 50961500 : int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(function))->co_flags;
4812 50961500 : PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(function));
4813 50961500 : STACK_SHRINK(total_args);
4814 50961500 : _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit(
4815 : tstate, (PyFunctionObject *)function, locals,
4816 : stack_pointer, positional_args, call_shape.kwnames
4817 : );
4818 50961500 : call_shape.kwnames = NULL;
4819 50961500 : STACK_SHRINK(2-is_meth);
4820 : // The frame has stolen all the arguments from the stack,
4821 : // so there is no need to clean them up.
4822 50961500 : if (new_frame == NULL) {
4823 312 : goto error;
4824 : }
4825 50961200 : _PyFrame_SetStackPointer(frame, stack_pointer);
4826 50961200 : JUMPBY(INLINE_CACHE_ENTRIES_CALL);
4827 50961200 : frame->prev_instr = next_instr - 1;
4828 50961200 : new_frame->previous = frame;
4829 50961200 : cframe.current_frame = frame = new_frame;
4830 : CALL_STAT_INC(inlined_py_calls);
4831 50961200 : goto start_frame;
4832 : }
4833 : /* Callable is not a normal Python function */
4834 : PyObject *res;
4835 141222000 : if (cframe.use_tracing) {
4836 2005420 : res = trace_call_function(
4837 2005420 : tstate, function, stack_pointer-total_args,
4838 : positional_args, call_shape.kwnames);
4839 : }
4840 : else {
4841 139217000 : res = PyObject_Vectorcall(
4842 139217000 : function, stack_pointer-total_args,
4843 : positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET,
4844 : call_shape.kwnames);
4845 : }
4846 141222000 : call_shape.kwnames = NULL;
4847 141222000 : assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
4848 141222000 : Py_DECREF(function);
4849 : /* Clear the stack */
4850 141222000 : STACK_SHRINK(total_args);
4851 383710000 : for (int i = 0; i < total_args; i++) {
4852 242487000 : Py_DECREF(stack_pointer[i]);
4853 : }
4854 141222000 : STACK_SHRINK(2-is_meth);
4855 141222000 : PUSH(res);
4856 141222000 : if (res == NULL) {
4857 216079 : goto error;
4858 : }
4859 141006000 : JUMPBY(INLINE_CACHE_ENTRIES_CALL);
4860 141006000 : CHECK_EVAL_BREAKER();
4861 140989000 : DISPATCH();
4862 : }
4863 :
4864 153830000 : TARGET(CALL_ADAPTIVE) {
4865 153830000 : _PyCallCache *cache = (_PyCallCache *)next_instr;
4866 153830000 : if (ADAPTIVE_COUNTER_IS_ZERO(cache)) {
4867 3467280 : next_instr--;
4868 3467280 : int is_meth = is_method(stack_pointer, oparg);
4869 3467280 : int nargs = oparg + is_meth;
4870 3467280 : PyObject *callable = PEEK(nargs + 1);
4871 3467280 : int err = _Py_Specialize_Call(callable, next_instr, nargs,
4872 : call_shape.kwnames);
4873 3467280 : if (err < 0) {
4874 0 : goto error;
4875 : }
4876 3467280 : NOTRACE_DISPATCH_SAME_OPARG();
4877 : }
4878 : else {
4879 : STAT_INC(CALL, deferred);
4880 150363000 : DECREMENT_ADAPTIVE_COUNTER(cache);
4881 150363000 : goto call_function;
4882 : }
4883 : }
4884 :
4885 114653000 : TARGET(CALL_PY_EXACT_ARGS) {
4886 134463000 : call_exact_args:
4887 134463000 : assert(call_shape.kwnames == NULL);
4888 134463000 : DEOPT_IF(tstate->interp->eval_frame, CALL);
4889 134463000 : _PyCallCache *cache = (_PyCallCache *)next_instr;
4890 134463000 : int is_meth = is_method(stack_pointer, oparg);
4891 134463000 : int argcount = oparg + is_meth;
4892 134463000 : PyObject *callable = PEEK(argcount + 1);
4893 134463000 : DEOPT_IF(!PyFunction_Check(callable), CALL);
4894 134414000 : PyFunctionObject *func = (PyFunctionObject *)callable;
4895 134414000 : DEOPT_IF(func->func_version != read_u32(cache->func_version), CALL);
4896 123299000 : PyCodeObject *code = (PyCodeObject *)func->func_code;
4897 123299000 : DEOPT_IF(code->co_argcount != argcount, CALL);
4898 123299000 : DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL);
4899 : STAT_INC(CALL, hit);
4900 123295000 : _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, func);
4901 : CALL_STAT_INC(inlined_py_calls);
4902 123295000 : STACK_SHRINK(argcount);
4903 367394000 : for (int i = 0; i < argcount; i++) {
4904 244100000 : new_frame->localsplus[i] = stack_pointer[i];
4905 : }
4906 374126000 : for (int i = argcount; i < code->co_nlocalsplus; i++) {
4907 250831000 : new_frame->localsplus[i] = NULL;
4908 : }
4909 123295000 : STACK_SHRINK(2-is_meth);
4910 123295000 : _PyFrame_SetStackPointer(frame, stack_pointer);
4911 123295000 : JUMPBY(INLINE_CACHE_ENTRIES_CALL);
4912 123295000 : frame->prev_instr = next_instr - 1;
4913 123295000 : new_frame->previous = frame;
4914 123295000 : frame = cframe.current_frame = new_frame;
4915 123295000 : goto start_frame;
4916 : }
4917 :
4918 20538300 : TARGET(CALL_PY_WITH_DEFAULTS) {
4919 20538300 : assert(call_shape.kwnames == NULL);
4920 20538300 : DEOPT_IF(tstate->interp->eval_frame, CALL);
4921 20538300 : _PyCallCache *cache = (_PyCallCache *)next_instr;
4922 20538300 : int is_meth = is_method(stack_pointer, oparg);
4923 20538300 : int argcount = oparg + is_meth;
4924 20538300 : PyObject *callable = PEEK(argcount + 1);
4925 20538300 : DEOPT_IF(!PyFunction_Check(callable), CALL);
4926 20536100 : PyFunctionObject *func = (PyFunctionObject *)callable;
4927 20536100 : DEOPT_IF(func->func_version != read_u32(cache->func_version), CALL);
4928 20406500 : PyCodeObject *code = (PyCodeObject *)func->func_code;
4929 20406500 : DEOPT_IF(argcount > code->co_argcount, CALL);
4930 20406500 : int minargs = cache->min_args;
4931 20406500 : DEOPT_IF(argcount < minargs, CALL);
4932 20406500 : DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL);
4933 : STAT_INC(CALL, hit);
4934 20406500 : _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, func);
4935 : CALL_STAT_INC(inlined_py_calls);
4936 20406500 : STACK_SHRINK(argcount);
4937 71736200 : for (int i = 0; i < argcount; i++) {
4938 51329800 : new_frame->localsplus[i] = stack_pointer[i];
4939 : }
4940 45998800 : for (int i = argcount; i < code->co_argcount; i++) {
4941 25592400 : PyObject *def = PyTuple_GET_ITEM(func->func_defaults,
4942 : i - minargs);
4943 25592400 : Py_INCREF(def);
4944 25592400 : new_frame->localsplus[i] = def;
4945 : }
4946 82002100 : for (int i = code->co_argcount; i < code->co_nlocalsplus; i++) {
4947 61595600 : new_frame->localsplus[i] = NULL;
4948 : }
4949 20406500 : STACK_SHRINK(2-is_meth);
4950 20406500 : _PyFrame_SetStackPointer(frame, stack_pointer);
4951 20406500 : JUMPBY(INLINE_CACHE_ENTRIES_CALL);
4952 20406500 : frame->prev_instr = next_instr - 1;
4953 20406500 : new_frame->previous = frame;
4954 20406500 : frame = cframe.current_frame = new_frame;
4955 20406500 : goto start_frame;
4956 : }
4957 :
4958 30724000 : TARGET(CALL_NO_KW_TYPE_1) {
4959 30724000 : assert(call_shape.kwnames == NULL);
4960 30724000 : assert(cframe.use_tracing == 0);
4961 30724000 : assert(oparg == 1);
4962 30724000 : DEOPT_IF(is_method(stack_pointer, 1), CALL);
4963 30724000 : PyObject *obj = TOP();
4964 30724000 : PyObject *callable = SECOND();
4965 30724000 : DEOPT_IF(callable != (PyObject *)&PyType_Type, CALL);
4966 : STAT_INC(CALL, hit);
4967 30724000 : JUMPBY(INLINE_CACHE_ENTRIES_CALL);
4968 30724000 : PyObject *res = Py_NewRef(Py_TYPE(obj));
4969 30724000 : Py_DECREF(callable);
4970 30724000 : Py_DECREF(obj);
4971 30724000 : STACK_SHRINK(2);
4972 30724000 : SET_TOP(res);
4973 30724000 : NOTRACE_DISPATCH();
4974 : }
4975 :
4976 4672930 : TARGET(CALL_NO_KW_STR_1) {
4977 4672930 : assert(call_shape.kwnames == NULL);
4978 4672930 : assert(cframe.use_tracing == 0);
4979 4672930 : assert(oparg == 1);
4980 4672930 : DEOPT_IF(is_method(stack_pointer, 1), CALL);
4981 4672930 : PyObject *callable = PEEK(2);
4982 4672930 : DEOPT_IF(callable != (PyObject *)&PyUnicode_Type, CALL);
4983 : STAT_INC(CALL, hit);
4984 4672580 : JUMPBY(INLINE_CACHE_ENTRIES_CALL);
4985 4672580 : PyObject *arg = TOP();
4986 4672580 : PyObject *res = PyObject_Str(arg);
4987 4672580 : Py_DECREF(arg);
4988 4672580 : Py_DECREF(&PyUnicode_Type);
4989 4672580 : STACK_SHRINK(2);
4990 4672580 : SET_TOP(res);
4991 4672580 : if (res == NULL) {
4992 6 : goto error;
4993 : }
4994 4672570 : CHECK_EVAL_BREAKER();
4995 4672570 : DISPATCH();
4996 : }
4997 :
4998 832923 : TARGET(CALL_NO_KW_TUPLE_1) {
4999 832923 : assert(call_shape.kwnames == NULL);
5000 832923 : assert(oparg == 1);
5001 832923 : DEOPT_IF(is_method(stack_pointer, 1), CALL);
5002 832923 : PyObject *callable = PEEK(2);
5003 832923 : DEOPT_IF(callable != (PyObject *)&PyTuple_Type, CALL);
5004 : STAT_INC(CALL, hit);
5005 832742 : JUMPBY(INLINE_CACHE_ENTRIES_CALL);
5006 832742 : PyObject *arg = TOP();
5007 832742 : PyObject *res = PySequence_Tuple(arg);
5008 832742 : Py_DECREF(arg);
5009 832742 : Py_DECREF(&PyTuple_Type);
5010 832742 : STACK_SHRINK(2);
5011 832742 : SET_TOP(res);
5012 832742 : if (res == NULL) {
5013 53 : goto error;
5014 : }
5015 832689 : CHECK_EVAL_BREAKER();
5016 832688 : DISPATCH();
5017 : }
5018 :
5019 16566800 : TARGET(CALL_BUILTIN_CLASS) {
5020 16566800 : int is_meth = is_method(stack_pointer, oparg);
5021 16566800 : int total_args = oparg + is_meth;
5022 16566800 : int kwnames_len = KWNAMES_LEN();
5023 16566800 : PyObject *callable = PEEK(total_args + 1);
5024 16566800 : DEOPT_IF(!PyType_Check(callable), CALL);
5025 16566200 : PyTypeObject *tp = (PyTypeObject *)callable;
5026 16566200 : DEOPT_IF(tp->tp_vectorcall == NULL, CALL);
5027 : STAT_INC(CALL, hit);
5028 16563200 : JUMPBY(INLINE_CACHE_ENTRIES_CALL);
5029 16563200 : STACK_SHRINK(total_args);
5030 16563200 : PyObject *res = tp->tp_vectorcall((PyObject *)tp, stack_pointer,
5031 16563200 : total_args-kwnames_len, call_shape.kwnames);
5032 16563200 : call_shape.kwnames = NULL;
5033 : /* Free the arguments. */
5034 26305800 : for (int i = 0; i < total_args; i++) {
5035 9742640 : Py_DECREF(stack_pointer[i]);
5036 : }
5037 16563200 : Py_DECREF(tp);
5038 16563200 : STACK_SHRINK(1-is_meth);
5039 16563200 : SET_TOP(res);
5040 16563200 : if (res == NULL) {
5041 1429 : goto error;
5042 : }
5043 16561800 : CHECK_EVAL_BREAKER();
5044 16561500 : DISPATCH();
5045 : }
5046 :
5047 62454700 : TARGET(CALL_NO_KW_BUILTIN_O) {
5048 62454700 : assert(cframe.use_tracing == 0);
5049 : /* Builtin METH_O functions */
5050 62454700 : assert(call_shape.kwnames == NULL);
5051 62454700 : int is_meth = is_method(stack_pointer, oparg);
5052 62454700 : int total_args = oparg + is_meth;
5053 62454700 : DEOPT_IF(total_args != 1, CALL);
5054 62454700 : PyObject *callable = PEEK(total_args + 1);
5055 62454700 : DEOPT_IF(!PyCFunction_CheckExact(callable), CALL);
5056 62452500 : DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_O, CALL);
5057 : STAT_INC(CALL, hit);
5058 62135600 : JUMPBY(INLINE_CACHE_ENTRIES_CALL);
5059 62135600 : PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable);
5060 : // This is slower but CPython promises to check all non-vectorcall
5061 : // function calls.
5062 62135600 : if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) {
5063 2 : goto error;
5064 : }
5065 62135600 : PyObject *arg = TOP();
5066 62135600 : PyObject *res = cfunc(PyCFunction_GET_SELF(callable), arg);
5067 62135600 : _Py_LeaveRecursiveCallTstate(tstate);
5068 62135600 : assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
5069 :
5070 62135600 : Py_DECREF(arg);
5071 62135600 : Py_DECREF(callable);
5072 62135600 : STACK_SHRINK(2-is_meth);
5073 62135600 : SET_TOP(res);
5074 62135600 : if (res == NULL) {
5075 8573 : goto error;
5076 : }
5077 62127000 : CHECK_EVAL_BREAKER();
5078 62126200 : DISPATCH();
5079 : }
5080 :
5081 60563400 : TARGET(CALL_NO_KW_BUILTIN_FAST) {
5082 60563400 : assert(cframe.use_tracing == 0);
5083 : /* Builtin METH_FASTCALL functions, without keywords */
5084 60563400 : assert(call_shape.kwnames == NULL);
5085 60563400 : int is_meth = is_method(stack_pointer, oparg);
5086 60563400 : int total_args = oparg + is_meth;
5087 60563400 : PyObject *callable = PEEK(total_args + 1);
5088 60563400 : DEOPT_IF(!PyCFunction_CheckExact(callable), CALL);
5089 60558700 : DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_FASTCALL,
5090 : CALL);
5091 : STAT_INC(CALL, hit);
5092 60484000 : JUMPBY(INLINE_CACHE_ENTRIES_CALL);
5093 60484000 : PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable);
5094 60484000 : STACK_SHRINK(total_args);
5095 : /* res = func(self, args, nargs) */
5096 60484000 : PyObject *res = ((_PyCFunctionFast)(void(*)(void))cfunc)(
5097 : PyCFunction_GET_SELF(callable),
5098 : stack_pointer,
5099 : total_args);
5100 60484000 : assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
5101 :
5102 : /* Free the arguments. */
5103 197074000 : for (int i = 0; i < total_args; i++) {
5104 136590000 : Py_DECREF(stack_pointer[i]);
5105 : }
5106 60484000 : STACK_SHRINK(2-is_meth);
5107 60484000 : PUSH(res);
5108 60484000 : Py_DECREF(callable);
5109 60484000 : if (res == NULL) {
5110 : /* Not deopting because this doesn't mean our optimization was
5111 : wrong. `res` can be NULL for valid reasons. Eg. getattr(x,
5112 : 'invalid'). In those cases an exception is set, so we must
5113 : handle it.
5114 : */
5115 1256500 : goto error;
5116 : }
5117 59227500 : CHECK_EVAL_BREAKER();
5118 59224000 : DISPATCH();
5119 : }
5120 :
5121 9910280 : TARGET(CALL_BUILTIN_FAST_WITH_KEYWORDS) {
5122 9910280 : assert(cframe.use_tracing == 0);
5123 : /* Builtin METH_FASTCALL | METH_KEYWORDS functions */
5124 9910280 : int is_meth = is_method(stack_pointer, oparg);
5125 9910280 : int total_args = oparg + is_meth;
5126 9910280 : PyObject *callable = PEEK(total_args + 1);
5127 9910280 : DEOPT_IF(!PyCFunction_CheckExact(callable), CALL);
5128 9906780 : DEOPT_IF(PyCFunction_GET_FLAGS(callable) !=
5129 : (METH_FASTCALL | METH_KEYWORDS), CALL);
5130 : STAT_INC(CALL, hit);
5131 9373330 : JUMPBY(INLINE_CACHE_ENTRIES_CALL);
5132 9373330 : STACK_SHRINK(total_args);
5133 : /* res = func(self, args, nargs, kwnames) */
5134 : _PyCFunctionFastWithKeywords cfunc =
5135 : (_PyCFunctionFastWithKeywords)(void(*)(void))
5136 9373330 : PyCFunction_GET_FUNCTION(callable);
5137 9373330 : PyObject *res = cfunc(
5138 : PyCFunction_GET_SELF(callable),
5139 : stack_pointer,
5140 9373330 : total_args - KWNAMES_LEN(),
5141 : call_shape.kwnames
5142 : );
5143 9373310 : assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
5144 9373310 : call_shape.kwnames = NULL;
5145 :
5146 : /* Free the arguments. */
5147 22632800 : for (int i = 0; i < total_args; i++) {
5148 13259500 : Py_DECREF(stack_pointer[i]);
5149 : }
5150 9373310 : STACK_SHRINK(2-is_meth);
5151 9373310 : PUSH(res);
5152 9373310 : Py_DECREF(callable);
5153 9373310 : if (res == NULL) {
5154 1298400 : goto error;
5155 : }
5156 8074910 : CHECK_EVAL_BREAKER();
5157 8074880 : DISPATCH();
5158 : }
5159 :
5160 31072300 : TARGET(CALL_NO_KW_LEN) {
5161 31072300 : assert(cframe.use_tracing == 0);
5162 31072300 : assert(call_shape.kwnames == NULL);
5163 : /* len(o) */
5164 31072300 : int is_meth = is_method(stack_pointer, oparg);
5165 31072300 : int total_args = oparg + is_meth;
5166 31072300 : DEOPT_IF(total_args != 1, CALL);
5167 31072300 : PyObject *callable = PEEK(total_args + 1);
5168 31072300 : PyInterpreterState *interp = _PyInterpreterState_GET();
5169 31072300 : DEOPT_IF(callable != interp->callable_cache.len, CALL);
5170 : STAT_INC(CALL, hit);
5171 31072300 : JUMPBY(INLINE_CACHE_ENTRIES_CALL);
5172 31072300 : PyObject *arg = TOP();
5173 31072300 : Py_ssize_t len_i = PyObject_Length(arg);
5174 31072300 : if (len_i < 0) {
5175 319 : goto error;
5176 : }
5177 31072000 : PyObject *res = PyLong_FromSsize_t(len_i);
5178 31072000 : assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
5179 :
5180 31072000 : STACK_SHRINK(2-is_meth);
5181 31072000 : SET_TOP(res);
5182 31072000 : Py_DECREF(callable);
5183 31072000 : Py_DECREF(arg);
5184 31072000 : if (res == NULL) {
5185 0 : goto error;
5186 : }
5187 31072000 : DISPATCH();
5188 : }
5189 :
5190 35812400 : TARGET(CALL_NO_KW_ISINSTANCE) {
5191 35812400 : assert(cframe.use_tracing == 0);
5192 35812400 : assert(call_shape.kwnames == NULL);
5193 : /* isinstance(o, o2) */
5194 35812400 : int is_meth = is_method(stack_pointer, oparg);
5195 35812400 : int total_args = oparg + is_meth;
5196 35812400 : PyObject *callable = PEEK(total_args + 1);
5197 35812400 : DEOPT_IF(total_args != 2, CALL);
5198 35812400 : PyInterpreterState *interp = _PyInterpreterState_GET();
5199 35812400 : DEOPT_IF(callable != interp->callable_cache.isinstance, CALL);
5200 : STAT_INC(CALL, hit);
5201 35812400 : JUMPBY(INLINE_CACHE_ENTRIES_CALL);
5202 35812400 : PyObject *cls = POP();
5203 35812400 : PyObject *inst = TOP();
5204 35812400 : int retval = PyObject_IsInstance(inst, cls);
5205 35812400 : if (retval < 0) {
5206 16 : Py_DECREF(cls);
5207 16 : goto error;
5208 : }
5209 35812400 : PyObject *res = PyBool_FromLong(retval);
5210 35812400 : assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
5211 :
5212 35812400 : STACK_SHRINK(2-is_meth);
5213 35812400 : SET_TOP(res);
5214 35812400 : Py_DECREF(inst);
5215 35812400 : Py_DECREF(cls);
5216 35812400 : Py_DECREF(callable);
5217 35812400 : if (res == NULL) {
5218 0 : goto error;
5219 : }
5220 35812400 : DISPATCH();
5221 : }
5222 :
5223 14998900 : TARGET(CALL_NO_KW_LIST_APPEND) {
5224 14998900 : assert(cframe.use_tracing == 0);
5225 14998900 : assert(call_shape.kwnames == NULL);
5226 14998900 : assert(oparg == 1);
5227 14998900 : PyObject *callable = PEEK(3);
5228 14998900 : PyInterpreterState *interp = _PyInterpreterState_GET();
5229 14998900 : DEOPT_IF(callable != interp->callable_cache.list_append, CALL);
5230 14998800 : PyObject *list = SECOND();
5231 14998800 : DEOPT_IF(!PyList_Check(list), CALL);
5232 : STAT_INC(CALL, hit);
5233 : // CALL + POP_TOP
5234 14998800 : JUMPBY(INLINE_CACHE_ENTRIES_CALL + 1);
5235 14998800 : assert(_Py_OPCODE(next_instr[-1]) == POP_TOP);
5236 14998800 : PyObject *arg = POP();
5237 14998800 : if (_PyList_AppendTakeRef((PyListObject *)list, arg) < 0) {
5238 0 : goto error;
5239 : }
5240 14998800 : STACK_SHRINK(2);
5241 14998800 : Py_DECREF(list);
5242 14998800 : Py_DECREF(callable);
5243 14998800 : NOTRACE_DISPATCH();
5244 : }
5245 :
5246 19916300 : TARGET(CALL_NO_KW_METHOD_DESCRIPTOR_O) {
5247 19916300 : assert(call_shape.kwnames == NULL);
5248 19916300 : int is_meth = is_method(stack_pointer, oparg);
5249 19916300 : int total_args = oparg + is_meth;
5250 19916300 : PyMethodDescrObject *callable =
5251 19916300 : (PyMethodDescrObject *)PEEK(total_args + 1);
5252 19916300 : DEOPT_IF(total_args != 2, CALL);
5253 19916200 : DEOPT_IF(!Py_IS_TYPE(callable, &PyMethodDescr_Type), CALL);
5254 19914200 : PyMethodDef *meth = callable->d_method;
5255 19914200 : DEOPT_IF(meth->ml_flags != METH_O, CALL);
5256 19912900 : PyObject *arg = TOP();
5257 19912900 : PyObject *self = SECOND();
5258 19912900 : DEOPT_IF(!Py_IS_TYPE(self, callable->d_common.d_type), CALL);
5259 : STAT_INC(CALL, hit);
5260 19659800 : JUMPBY(INLINE_CACHE_ENTRIES_CALL);
5261 19659800 : PyCFunction cfunc = meth->ml_meth;
5262 : // This is slower but CPython promises to check all non-vectorcall
5263 : // function calls.
5264 19659800 : if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) {
5265 0 : goto error;
5266 : }
5267 19659800 : PyObject *res = cfunc(self, arg);
5268 19659800 : _Py_LeaveRecursiveCallTstate(tstate);
5269 19659800 : assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
5270 19659800 : Py_DECREF(self);
5271 19659800 : Py_DECREF(arg);
5272 19659800 : STACK_SHRINK(oparg + 1);
5273 19659800 : SET_TOP(res);
5274 19659800 : Py_DECREF(callable);
5275 19659800 : if (res == NULL) {
5276 10038 : goto error;
5277 : }
5278 19649700 : CHECK_EVAL_BREAKER();
5279 19649700 : DISPATCH();
5280 : }
5281 :
5282 16207000 : TARGET(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS) {
5283 16207000 : int is_meth = is_method(stack_pointer, oparg);
5284 16207000 : int total_args = oparg + is_meth;
5285 16207000 : PyMethodDescrObject *callable =
5286 16207000 : (PyMethodDescrObject *)PEEK(total_args + 1);
5287 16207000 : DEOPT_IF(!Py_IS_TYPE(callable, &PyMethodDescr_Type), CALL);
5288 16204800 : PyMethodDef *meth = callable->d_method;
5289 16204800 : DEOPT_IF(meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS), CALL);
5290 16203400 : PyTypeObject *d_type = callable->d_common.d_type;
5291 16203400 : PyObject *self = PEEK(total_args);
5292 16203400 : DEOPT_IF(!Py_IS_TYPE(self, d_type), CALL);
5293 : STAT_INC(CALL, hit);
5294 16142200 : JUMPBY(INLINE_CACHE_ENTRIES_CALL);
5295 16142200 : int nargs = total_args-1;
5296 16142200 : STACK_SHRINK(nargs);
5297 16142200 : _PyCFunctionFastWithKeywords cfunc =
5298 : (_PyCFunctionFastWithKeywords)(void(*)(void))meth->ml_meth;
5299 16142200 : PyObject *res = cfunc(self, stack_pointer, nargs - KWNAMES_LEN(),
5300 : call_shape.kwnames);
5301 16142200 : assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
5302 16142200 : call_shape.kwnames = NULL;
5303 :
5304 : /* Free the arguments. */
5305 36597800 : for (int i = 0; i < nargs; i++) {
5306 20455600 : Py_DECREF(stack_pointer[i]);
5307 : }
5308 16142200 : Py_DECREF(self);
5309 16142200 : STACK_SHRINK(2-is_meth);
5310 16142200 : SET_TOP(res);
5311 16142200 : Py_DECREF(callable);
5312 16142200 : if (res == NULL) {
5313 2732 : goto error;
5314 : }
5315 16139500 : CHECK_EVAL_BREAKER();
5316 16139300 : DISPATCH();
5317 : }
5318 :
5319 47971200 : TARGET(CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS) {
5320 47971200 : assert(call_shape.kwnames == NULL);
5321 47971200 : assert(oparg == 0 || oparg == 1);
5322 47971200 : int is_meth = is_method(stack_pointer, oparg);
5323 47971200 : int total_args = oparg + is_meth;
5324 47971200 : DEOPT_IF(total_args != 1, CALL);
5325 47970900 : PyMethodDescrObject *callable = (PyMethodDescrObject *)SECOND();
5326 47970900 : DEOPT_IF(!Py_IS_TYPE(callable, &PyMethodDescr_Type), CALL);
5327 47963500 : PyMethodDef *meth = callable->d_method;
5328 47963500 : PyObject *self = TOP();
5329 47963500 : DEOPT_IF(!Py_IS_TYPE(self, callable->d_common.d_type), CALL);
5330 47588000 : DEOPT_IF(meth->ml_flags != METH_NOARGS, CALL);
5331 : STAT_INC(CALL, hit);
5332 47587800 : JUMPBY(INLINE_CACHE_ENTRIES_CALL);
5333 47587800 : PyCFunction cfunc = meth->ml_meth;
5334 : // This is slower but CPython promises to check all non-vectorcall
5335 : // function calls.
5336 47587800 : if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) {
5337 0 : goto error;
5338 : }
5339 47587800 : PyObject *res = cfunc(self, NULL);
5340 47587800 : _Py_LeaveRecursiveCallTstate(tstate);
5341 47587800 : assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
5342 47587800 : Py_DECREF(self);
5343 47587800 : STACK_SHRINK(oparg + 1);
5344 47587800 : SET_TOP(res);
5345 47587800 : Py_DECREF(callable);
5346 47587800 : if (res == NULL) {
5347 189484 : goto error;
5348 : }
5349 47398400 : CHECK_EVAL_BREAKER();
5350 47397600 : DISPATCH();
5351 : }
5352 :
5353 49312800 : TARGET(CALL_NO_KW_METHOD_DESCRIPTOR_FAST) {
5354 49312800 : assert(call_shape.kwnames == NULL);
5355 49312800 : int is_meth = is_method(stack_pointer, oparg);
5356 49312800 : int total_args = oparg + is_meth;
5357 49312800 : PyMethodDescrObject *callable =
5358 49312800 : (PyMethodDescrObject *)PEEK(total_args + 1);
5359 : /* Builtin METH_FASTCALL methods, without keywords */
5360 49312800 : DEOPT_IF(!Py_IS_TYPE(callable, &PyMethodDescr_Type), CALL);
5361 49306900 : PyMethodDef *meth = callable->d_method;
5362 49306900 : DEOPT_IF(meth->ml_flags != METH_FASTCALL, CALL);
5363 49306700 : PyObject *self = PEEK(total_args);
5364 49306700 : DEOPT_IF(!Py_IS_TYPE(self, callable->d_common.d_type), CALL);
5365 : STAT_INC(CALL, hit);
5366 49235700 : JUMPBY(INLINE_CACHE_ENTRIES_CALL);
5367 49235700 : _PyCFunctionFast cfunc =
5368 : (_PyCFunctionFast)(void(*)(void))meth->ml_meth;
5369 49235700 : int nargs = total_args-1;
5370 49235700 : STACK_SHRINK(nargs);
5371 49235700 : PyObject *res = cfunc(self, stack_pointer, nargs);
5372 49235700 : assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
5373 : /* Clear the stack of the arguments. */
5374 102235000 : for (int i = 0; i < nargs; i++) {
5375 52999200 : Py_DECREF(stack_pointer[i]);
5376 : }
5377 49235700 : Py_DECREF(self);
5378 49235700 : STACK_SHRINK(2-is_meth);
5379 49235700 : SET_TOP(res);
5380 49235700 : Py_DECREF(callable);
5381 49235700 : if (res == NULL) {
5382 282388 : goto error;
5383 : }
5384 48953400 : CHECK_EVAL_BREAKER();
5385 48953000 : DISPATCH();
5386 : }
5387 :
5388 9524250 : TARGET(CALL_FUNCTION_EX) {
5389 9524250 : PREDICTED(CALL_FUNCTION_EX);
5390 9524250 : PyObject *func, *callargs, *kwargs = NULL, *result;
5391 9524250 : if (oparg & 0x01) {
5392 3452250 : kwargs = POP();
5393 3452250 : if (!PyDict_CheckExact(kwargs)) {
5394 0 : PyObject *d = PyDict_New();
5395 0 : if (d == NULL)
5396 0 : goto error;
5397 0 : if (_PyDict_MergeEx(d, kwargs, 2) < 0) {
5398 0 : Py_DECREF(d);
5399 0 : format_kwargs_error(tstate, SECOND(), kwargs);
5400 0 : Py_DECREF(kwargs);
5401 0 : goto error;
5402 : }
5403 0 : Py_DECREF(kwargs);
5404 0 : kwargs = d;
5405 : }
5406 3452250 : assert(PyDict_CheckExact(kwargs));
5407 : }
5408 9524250 : callargs = POP();
5409 9524250 : func = TOP();
5410 9524250 : if (!PyTuple_CheckExact(callargs)) {
5411 363037 : if (check_args_iterable(tstate, func, callargs) < 0) {
5412 6 : Py_DECREF(callargs);
5413 6 : goto error;
5414 : }
5415 363031 : Py_SETREF(callargs, PySequence_Tuple(callargs));
5416 363031 : if (callargs == NULL) {
5417 4 : goto error;
5418 : }
5419 : }
5420 9524240 : assert(PyTuple_CheckExact(callargs));
5421 :
5422 9524240 : result = do_call_core(tstate, func, callargs, kwargs, cframe.use_tracing);
5423 9524100 : Py_DECREF(func);
5424 9524100 : Py_DECREF(callargs);
5425 9524100 : Py_XDECREF(kwargs);
5426 :
5427 9524100 : STACK_SHRINK(1);
5428 9524100 : assert(TOP() == NULL);
5429 9524100 : SET_TOP(result);
5430 9524100 : if (result == NULL) {
5431 128702 : goto error;
5432 : }
5433 9395400 : CHECK_EVAL_BREAKER();
5434 9393950 : DISPATCH();
5435 : }
5436 :
5437 15405500 : TARGET(MAKE_FUNCTION) {
5438 15405500 : PyObject *codeobj = POP();
5439 : PyFunctionObject *func = (PyFunctionObject *)
5440 15405500 : PyFunction_New(codeobj, GLOBALS());
5441 :
5442 15405500 : Py_DECREF(codeobj);
5443 15405500 : if (func == NULL) {
5444 0 : goto error;
5445 : }
5446 :
5447 15405500 : if (oparg & 0x08) {
5448 2515330 : assert(PyTuple_CheckExact(TOP()));
5449 2515330 : func->func_closure = POP();
5450 : }
5451 15405500 : if (oparg & 0x04) {
5452 121295 : assert(PyTuple_CheckExact(TOP()));
5453 121295 : func->func_annotations = POP();
5454 : }
5455 15405500 : if (oparg & 0x02) {
5456 251630 : assert(PyDict_CheckExact(TOP()));
5457 251630 : func->func_kwdefaults = POP();
5458 : }
5459 15405500 : if (oparg & 0x01) {
5460 1632910 : assert(PyTuple_CheckExact(TOP()));
5461 1632910 : func->func_defaults = POP();
5462 : }
5463 :
5464 15405500 : PUSH((PyObject *)func);
5465 15405500 : DISPATCH();
5466 : }
5467 :
5468 17186100 : TARGET(RETURN_GENERATOR) {
5469 17186100 : PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(frame->f_func);
5470 17186100 : if (gen == NULL) {
5471 0 : goto error;
5472 : }
5473 17186100 : assert(EMPTY());
5474 17186100 : _PyFrame_SetStackPointer(frame, stack_pointer);
5475 17186100 : _PyInterpreterFrame *gen_frame = (_PyInterpreterFrame *)gen->gi_iframe;
5476 17186100 : _PyFrame_Copy(frame, gen_frame);
5477 17186100 : assert(frame->frame_obj == NULL);
5478 17186100 : gen->gi_frame_state = FRAME_CREATED;
5479 17186100 : gen_frame->owner = FRAME_OWNED_BY_GENERATOR;
5480 17186100 : _Py_LeaveRecursiveCallTstate(tstate);
5481 17186100 : if (!frame->is_entry) {
5482 16025700 : _PyInterpreterFrame *prev = frame->previous;
5483 16025700 : _PyThreadState_PopFrame(tstate, frame);
5484 16025700 : frame = cframe.current_frame = prev;
5485 16025700 : _PyFrame_StackPush(frame, (PyObject *)gen);
5486 16025700 : goto resume_frame;
5487 : }
5488 : /* Make sure that frame is in a valid state */
5489 1160390 : frame->stacktop = 0;
5490 1160390 : frame->f_locals = NULL;
5491 1160390 : Py_INCREF(frame->f_func);
5492 1160390 : Py_INCREF(frame->f_code);
5493 : /* Restore previous cframe and return. */
5494 1160390 : tstate->cframe = cframe.previous;
5495 1160390 : tstate->cframe->use_tracing = cframe.use_tracing;
5496 1160390 : assert(tstate->cframe->current_frame == frame->previous);
5497 1160390 : assert(!_PyErr_Occurred(tstate));
5498 1160390 : return (PyObject *)gen;
5499 : }
5500 :
5501 586723 : TARGET(BUILD_SLICE) {
5502 : PyObject *start, *stop, *step, *slice;
5503 586723 : if (oparg == 3)
5504 512214 : step = POP();
5505 : else
5506 74509 : step = NULL;
5507 586723 : stop = POP();
5508 586723 : start = TOP();
5509 586723 : slice = PySlice_New(start, stop, step);
5510 586723 : Py_DECREF(start);
5511 586723 : Py_DECREF(stop);
5512 586723 : Py_XDECREF(step);
5513 586723 : SET_TOP(slice);
5514 586723 : if (slice == NULL)
5515 0 : goto error;
5516 586723 : DISPATCH();
5517 : }
5518 :
5519 2962920 : TARGET(FORMAT_VALUE) {
5520 : /* Handles f-string value formatting. */
5521 : PyObject *result;
5522 : PyObject *fmt_spec;
5523 : PyObject *value;
5524 : PyObject *(*conv_fn)(PyObject *);
5525 2962920 : int which_conversion = oparg & FVC_MASK;
5526 2962920 : int have_fmt_spec = (oparg & FVS_MASK) == FVS_HAVE_SPEC;
5527 :
5528 2962920 : fmt_spec = have_fmt_spec ? POP() : NULL;
5529 2962920 : value = POP();
5530 :
5531 : /* See if any conversion is specified. */
5532 2962920 : switch (which_conversion) {
5533 1865020 : case FVC_NONE: conv_fn = NULL; break;
5534 617775 : case FVC_STR: conv_fn = PyObject_Str; break;
5535 475971 : case FVC_REPR: conv_fn = PyObject_Repr; break;
5536 4153 : case FVC_ASCII: conv_fn = PyObject_ASCII; break;
5537 0 : default:
5538 0 : _PyErr_Format(tstate, PyExc_SystemError,
5539 : "unexpected conversion flag %d",
5540 : which_conversion);
5541 0 : goto error;
5542 : }
5543 :
5544 : /* If there's a conversion function, call it and replace
5545 : value with that result. Otherwise, just use value,
5546 : without conversion. */
5547 2962920 : if (conv_fn != NULL) {
5548 1097900 : result = conv_fn(value);
5549 1097900 : Py_DECREF(value);
5550 1097900 : if (result == NULL) {
5551 483 : Py_XDECREF(fmt_spec);
5552 483 : goto error;
5553 : }
5554 1097420 : value = result;
5555 : }
5556 :
5557 : /* If value is a unicode object, and there's no fmt_spec,
5558 : then we know the result of format(value) is value
5559 : itself. In that case, skip calling format(). I plan to
5560 : move this optimization in to PyObject_Format()
5561 : itself. */
5562 2962440 : if (PyUnicode_CheckExact(value) && fmt_spec == NULL) {
5563 : /* Do nothing, just transfer ownership to result. */
5564 2286230 : result = value;
5565 : } else {
5566 : /* Actually call format(). */
5567 676212 : result = PyObject_Format(value, fmt_spec);
5568 676212 : Py_DECREF(value);
5569 676212 : Py_XDECREF(fmt_spec);
5570 676212 : if (result == NULL) {
5571 16 : goto error;
5572 : }
5573 : }
5574 :
5575 2962420 : PUSH(result);
5576 2962420 : DISPATCH();
5577 : }
5578 :
5579 22056000 : TARGET(COPY) {
5580 22056000 : assert(oparg != 0);
5581 22056000 : PyObject *peek = PEEK(oparg);
5582 22056000 : Py_INCREF(peek);
5583 22056000 : PUSH(peek);
5584 22056000 : DISPATCH();
5585 : }
5586 :
5587 33989900 : TARGET(BINARY_OP) {
5588 41379900 : PREDICTED(BINARY_OP);
5589 41379900 : PyObject *rhs = POP();
5590 41379900 : PyObject *lhs = TOP();
5591 41379900 : assert(0 <= oparg);
5592 41379900 : assert((unsigned)oparg < Py_ARRAY_LENGTH(binary_ops));
5593 41379900 : assert(binary_ops[oparg]);
5594 41379900 : PyObject *res = binary_ops[oparg](lhs, rhs);
5595 41379900 : Py_DECREF(lhs);
5596 41379900 : Py_DECREF(rhs);
5597 41379900 : SET_TOP(res);
5598 41379900 : if (res == NULL) {
5599 4764 : goto error;
5600 : }
5601 41375200 : JUMPBY(INLINE_CACHE_ENTRIES_BINARY_OP);
5602 41375200 : DISPATCH();
5603 : }
5604 :
5605 7779400 : TARGET(BINARY_OP_ADAPTIVE) {
5606 7779400 : assert(cframe.use_tracing == 0);
5607 7779400 : _PyBinaryOpCache *cache = (_PyBinaryOpCache *)next_instr;
5608 7779400 : if (ADAPTIVE_COUNTER_IS_ZERO(cache)) {
5609 389375 : PyObject *lhs = SECOND();
5610 389375 : PyObject *rhs = TOP();
5611 389375 : next_instr--;
5612 389375 : _Py_Specialize_BinaryOp(lhs, rhs, next_instr, oparg, &GETLOCAL(0));
5613 389375 : NOTRACE_DISPATCH_SAME_OPARG();
5614 : }
5615 : else {
5616 : STAT_INC(BINARY_OP, deferred);
5617 7390020 : DECREMENT_ADAPTIVE_COUNTER(cache);
5618 7390020 : JUMP_TO_INSTRUCTION(BINARY_OP);
5619 : }
5620 : }
5621 :
5622 24596400 : TARGET(SWAP) {
5623 24596400 : assert(oparg != 0);
5624 24596400 : PyObject *top = TOP();
5625 24596400 : SET_TOP(PEEK(oparg));
5626 24596400 : PEEK(oparg) = top;
5627 24596400 : DISPATCH();
5628 : }
5629 :
5630 29684 : TARGET(EXTENDED_ARG) {
5631 29684 : assert(oparg);
5632 29684 : oparg <<= 8;
5633 29684 : oparg |= _Py_OPARG(*next_instr);
5634 29684 : opcode = _PyOpcode_Deopt[_Py_OPCODE(*next_instr)];
5635 29684 : PRE_DISPATCH_GOTO();
5636 29684 : DISPATCH_GOTO();
5637 : }
5638 :
5639 61894100 : TARGET(EXTENDED_ARG_QUICK) {
5640 61894100 : assert(oparg);
5641 61894100 : oparg <<= 8;
5642 61894100 : oparg |= _Py_OPARG(*next_instr);
5643 61894100 : NOTRACE_DISPATCH_SAME_OPARG();
5644 : }
5645 :
5646 0 : TARGET(CACHE) {
5647 0 : Py_UNREACHABLE();
5648 : }
5649 :
5650 : #if USE_COMPUTED_GOTOS
5651 47779500 : TARGET_DO_TRACING:
5652 : #else
5653 : case DO_TRACING:
5654 : #endif
5655 : {
5656 47779500 : if (tstate->tracing == 0 &&
5657 6053230 : INSTR_OFFSET() >= frame->f_code->_co_firsttraceable
5658 : ) {
5659 6045850 : int instr_prev = _PyInterpreterFrame_LASTI(frame);
5660 6045850 : frame->prev_instr = next_instr;
5661 6045850 : TRACING_NEXTOPARG();
5662 6045850 : if (opcode == RESUME) {
5663 137377 : if (oparg < 2) {
5664 136559 : CHECK_EVAL_BREAKER();
5665 : }
5666 : /* Call tracing */
5667 137355 : TRACE_FUNCTION_ENTRY();
5668 136352 : DTRACE_FUNCTION_ENTRY();
5669 : }
5670 : else {
5671 : /* line-by-line tracing support */
5672 5908480 : if (PyDTrace_LINE_ENABLED()) {
5673 0 : maybe_dtrace_line(frame, &tstate->trace_info, instr_prev);
5674 : }
5675 :
5676 5908480 : if (cframe.use_tracing &&
5677 5908480 : tstate->c_tracefunc != NULL && !tstate->tracing) {
5678 : int err;
5679 : /* see maybe_call_line_trace()
5680 : for expository comments */
5681 5843280 : _PyFrame_SetStackPointer(frame, stack_pointer);
5682 :
5683 5843280 : err = maybe_call_line_trace(tstate->c_tracefunc,
5684 : tstate->c_traceobj,
5685 : tstate, frame, instr_prev);
5686 5843280 : if (err) {
5687 : /* trace function raised an exception */
5688 1087 : next_instr++;
5689 1087 : goto error;
5690 : }
5691 : /* Reload possibly changed frame fields */
5692 5842190 : next_instr = frame->prev_instr;
5693 :
5694 5842190 : stack_pointer = _PyFrame_GetStackPointer(frame);
5695 5842190 : frame->stacktop = -1;
5696 : }
5697 : }
5698 : }
5699 47777400 : TRACING_NEXTOPARG();
5700 47777400 : PRE_DISPATCH_GOTO();
5701 47777400 : DISPATCH_GOTO();
5702 : }
5703 :
5704 : #if USE_COMPUTED_GOTOS
5705 0 : _unknown_opcode:
5706 : #else
5707 : EXTRA_CASES // From opcode.h, a 'case' for each unused opcode
5708 : #endif
5709 : /* Tell C compilers not to hold the opcode variable in the loop.
5710 : next_instr points the current instruction without TARGET(). */
5711 0 : opcode = _Py_OPCODE(*next_instr);
5712 0 : fprintf(stderr, "XXX lineno: %d, opcode: %d\n",
5713 : _PyInterpreterFrame_GetLine(frame), opcode);
5714 0 : _PyErr_SetString(tstate, PyExc_SystemError, "unknown opcode");
5715 0 : goto error;
5716 :
5717 : } /* End instructions */
5718 :
5719 : /* This should never be reached. Every opcode should end with DISPATCH()
5720 : or goto error. */
5721 : Py_UNREACHABLE();
5722 :
5723 : /* Specialization misses */
5724 :
5725 67207900 : miss:
5726 : {
5727 : STAT_INC(opcode, miss);
5728 67207900 : opcode = _PyOpcode_Deopt[opcode];
5729 : STAT_INC(opcode, miss);
5730 : /* The counter is always the first cache entry: */
5731 67207900 : _Py_CODEUNIT *counter = (_Py_CODEUNIT *)next_instr;
5732 67207900 : *counter -= 1;
5733 67207900 : if (*counter == 0) {
5734 1108020 : int adaptive_opcode = _PyOpcode_Adaptive[opcode];
5735 1108020 : assert(adaptive_opcode);
5736 1108020 : _Py_SET_OPCODE(next_instr[-1], adaptive_opcode);
5737 : STAT_INC(opcode, deopt);
5738 1108020 : *counter = adaptive_counter_start();
5739 : }
5740 67207900 : next_instr--;
5741 67207900 : DISPATCH_GOTO();
5742 : }
5743 :
5744 743847 : binary_subscr_dict_error:
5745 : {
5746 743847 : PyObject *sub = POP();
5747 743847 : if (!_PyErr_Occurred(tstate)) {
5748 743839 : _PyErr_SetKeyError(sub);
5749 : }
5750 743847 : Py_DECREF(sub);
5751 743847 : goto error;
5752 : }
5753 :
5754 6 : unbound_local_error:
5755 : {
5756 6 : format_exc_check_arg(tstate, PyExc_UnboundLocalError,
5757 : UNBOUNDLOCAL_ERROR_MSG,
5758 6 : PyTuple_GetItem(frame->f_code->co_localsplusnames, oparg)
5759 : );
5760 6 : goto error;
5761 : }
5762 :
5763 6340260 : error:
5764 6340260 : call_shape.kwnames = NULL;
5765 : /* Double-check exception status. */
5766 : #ifdef NDEBUG
5767 : if (!_PyErr_Occurred(tstate)) {
5768 : _PyErr_SetString(tstate, PyExc_SystemError,
5769 : "error return without exception set");
5770 : }
5771 : #else
5772 6340260 : assert(_PyErr_Occurred(tstate));
5773 : #endif
5774 :
5775 : /* Log traceback info. */
5776 6340260 : PyFrameObject *f = _PyFrame_GetFrameObject(frame);
5777 6340260 : if (f != NULL) {
5778 6340220 : PyTraceBack_Here(f);
5779 : }
5780 :
5781 6340260 : if (tstate->c_tracefunc != NULL) {
5782 : /* Make sure state is set to FRAME_UNWINDING for tracing */
5783 8687 : call_exc_trace(tstate->c_tracefunc, tstate->c_traceobj,
5784 : tstate, frame);
5785 : }
5786 :
5787 6340260 : exception_unwind:
5788 : {
5789 : /* We can't use frame->f_lasti here, as RERAISE may have set it */
5790 7024490 : int offset = INSTR_OFFSET()-1;
5791 : int level, handler, lasti;
5792 7024490 : if (get_exception_handler(frame->f_code, offset, &level, &handler, &lasti) == 0) {
5793 : // No handlers, so exit.
5794 1681470 : assert(_PyErr_Occurred(tstate));
5795 :
5796 : /* Pop remaining stack entries. */
5797 1681470 : PyObject **stackbase = _PyFrame_Stackbase(frame);
5798 3933020 : while (stack_pointer > stackbase) {
5799 2251550 : PyObject *o = POP();
5800 2251550 : Py_XDECREF(o);
5801 : }
5802 1681470 : assert(STACK_LEVEL() == 0);
5803 1681470 : _PyFrame_SetStackPointer(frame, stack_pointer);
5804 1681470 : TRACE_FUNCTION_UNWIND();
5805 1681470 : DTRACE_FUNCTION_EXIT();
5806 1681470 : goto exit_unwind;
5807 : }
5808 :
5809 5343020 : assert(STACK_LEVEL() >= level);
5810 5343020 : PyObject **new_top = _PyFrame_Stackbase(frame) + level;
5811 10552000 : while (stack_pointer > new_top) {
5812 5208980 : PyObject *v = POP();
5813 5208980 : Py_XDECREF(v);
5814 : }
5815 : PyObject *exc, *val, *tb;
5816 5343020 : if (lasti) {
5817 638308 : int frame_lasti = _PyInterpreterFrame_LASTI(frame);
5818 638308 : PyObject *lasti = PyLong_FromLong(frame_lasti);
5819 638308 : if (lasti == NULL) {
5820 0 : goto exception_unwind;
5821 : }
5822 638308 : PUSH(lasti);
5823 : }
5824 5343020 : _PyErr_Fetch(tstate, &exc, &val, &tb);
5825 : /* Make the raw exception data
5826 : available to the handler,
5827 : so a program can emulate the
5828 : Python main loop. */
5829 5343020 : _PyErr_NormalizeException(tstate, &exc, &val, &tb);
5830 5343020 : if (tb != NULL)
5831 5343000 : PyException_SetTraceback(val, tb);
5832 : else
5833 20 : PyException_SetTraceback(val, Py_None);
5834 5343020 : Py_XDECREF(tb);
5835 5343020 : Py_XDECREF(exc);
5836 5343020 : PUSH(val);
5837 5343020 : JUMPTO(handler);
5838 : /* Resume normal execution */
5839 5343020 : DISPATCH();
5840 : }
5841 : }
5842 :
5843 1690740 : exit_unwind:
5844 1690740 : assert(_PyErr_Occurred(tstate));
5845 1690740 : _Py_LeaveRecursiveCallTstate(tstate);
5846 1690740 : if (frame->is_entry) {
5847 : /* Restore previous cframe and exit */
5848 1192860 : tstate->cframe = cframe.previous;
5849 1192860 : tstate->cframe->use_tracing = cframe.use_tracing;
5850 1192860 : assert(tstate->cframe->current_frame == frame->previous);
5851 1192860 : return NULL;
5852 : }
5853 497881 : frame = cframe.current_frame = pop_frame(tstate, frame);
5854 :
5855 1107290 : resume_with_error:
5856 1107290 : SET_LOCALS_FROM_FRAME();
5857 1107290 : goto error;
5858 :
5859 : }
5860 :
5861 : static void
5862 639 : format_missing(PyThreadState *tstate, const char *kind,
5863 : PyCodeObject *co, PyObject *names, PyObject *qualname)
5864 : {
5865 : int err;
5866 639 : Py_ssize_t len = PyList_GET_SIZE(names);
5867 : PyObject *name_str, *comma, *tail, *tmp;
5868 :
5869 639 : assert(PyList_CheckExact(names));
5870 639 : assert(len >= 1);
5871 : /* Deal with the joys of natural language. */
5872 639 : switch (len) {
5873 554 : case 1:
5874 554 : name_str = PyList_GET_ITEM(names, 0);
5875 554 : Py_INCREF(name_str);
5876 554 : break;
5877 79 : case 2:
5878 158 : name_str = PyUnicode_FromFormat("%U and %U",
5879 79 : PyList_GET_ITEM(names, len - 2),
5880 79 : PyList_GET_ITEM(names, len - 1));
5881 79 : break;
5882 6 : default:
5883 12 : tail = PyUnicode_FromFormat(", %U, and %U",
5884 6 : PyList_GET_ITEM(names, len - 2),
5885 6 : PyList_GET_ITEM(names, len - 1));
5886 6 : if (tail == NULL)
5887 0 : return;
5888 : /* Chop off the last two objects in the list. This shouldn't actually
5889 : fail, but we can't be too careful. */
5890 6 : err = PyList_SetSlice(names, len - 2, len, NULL);
5891 6 : if (err == -1) {
5892 0 : Py_DECREF(tail);
5893 0 : return;
5894 : }
5895 : /* Stitch everything up into a nice comma-separated list. */
5896 6 : comma = PyUnicode_FromString(", ");
5897 6 : if (comma == NULL) {
5898 0 : Py_DECREF(tail);
5899 0 : return;
5900 : }
5901 6 : tmp = PyUnicode_Join(comma, names);
5902 6 : Py_DECREF(comma);
5903 6 : if (tmp == NULL) {
5904 0 : Py_DECREF(tail);
5905 0 : return;
5906 : }
5907 6 : name_str = PyUnicode_Concat(tmp, tail);
5908 6 : Py_DECREF(tmp);
5909 6 : Py_DECREF(tail);
5910 6 : break;
5911 : }
5912 639 : if (name_str == NULL)
5913 0 : return;
5914 639 : _PyErr_Format(tstate, PyExc_TypeError,
5915 : "%U() missing %i required %s argument%s: %U",
5916 : qualname,
5917 : len,
5918 : kind,
5919 : len == 1 ? "" : "s",
5920 : name_str);
5921 639 : Py_DECREF(name_str);
5922 : }
5923 :
5924 : static void
5925 639 : missing_arguments(PyThreadState *tstate, PyCodeObject *co,
5926 : Py_ssize_t missing, Py_ssize_t defcount,
5927 : PyObject **localsplus, PyObject *qualname)
5928 : {
5929 639 : Py_ssize_t i, j = 0;
5930 : Py_ssize_t start, end;
5931 639 : int positional = (defcount != -1);
5932 639 : const char *kind = positional ? "positional" : "keyword-only";
5933 : PyObject *missing_names;
5934 :
5935 : /* Compute the names of the arguments that are missing. */
5936 639 : missing_names = PyList_New(missing);
5937 639 : if (missing_names == NULL)
5938 0 : return;
5939 639 : if (positional) {
5940 621 : start = 0;
5941 621 : end = co->co_argcount - defcount;
5942 : }
5943 : else {
5944 18 : start = co->co_argcount;
5945 18 : end = start + co->co_kwonlyargcount;
5946 : }
5947 1863 : for (i = start; i < end; i++) {
5948 1224 : if (localsplus[i] == NULL) {
5949 734 : PyObject *raw = PyTuple_GET_ITEM(co->co_localsplusnames, i);
5950 734 : PyObject *name = PyObject_Repr(raw);
5951 734 : if (name == NULL) {
5952 0 : Py_DECREF(missing_names);
5953 0 : return;
5954 : }
5955 734 : PyList_SET_ITEM(missing_names, j++, name);
5956 : }
5957 : }
5958 639 : assert(j == missing);
5959 639 : format_missing(tstate, kind, co, missing_names, qualname);
5960 639 : Py_DECREF(missing_names);
5961 : }
5962 :
5963 : static void
5964 294 : too_many_positional(PyThreadState *tstate, PyCodeObject *co,
5965 : Py_ssize_t given, PyObject *defaults,
5966 : PyObject **localsplus, PyObject *qualname)
5967 : {
5968 : int plural;
5969 294 : Py_ssize_t kwonly_given = 0;
5970 : Py_ssize_t i;
5971 : PyObject *sig, *kwonly_sig;
5972 294 : Py_ssize_t co_argcount = co->co_argcount;
5973 :
5974 294 : assert((co->co_flags & CO_VARARGS) == 0);
5975 : /* Count missing keyword-only args. */
5976 373 : for (i = co_argcount; i < co_argcount + co->co_kwonlyargcount; i++) {
5977 79 : if (localsplus[i] != NULL) {
5978 7 : kwonly_given++;
5979 : }
5980 : }
5981 294 : Py_ssize_t defcount = defaults == NULL ? 0 : PyTuple_GET_SIZE(defaults);
5982 294 : if (defcount) {
5983 85 : Py_ssize_t atleast = co_argcount - defcount;
5984 85 : plural = 1;
5985 85 : sig = PyUnicode_FromFormat("from %zd to %zd", atleast, co_argcount);
5986 : }
5987 : else {
5988 209 : plural = (co_argcount != 1);
5989 209 : sig = PyUnicode_FromFormat("%zd", co_argcount);
5990 : }
5991 294 : if (sig == NULL)
5992 0 : return;
5993 294 : if (kwonly_given) {
5994 5 : const char *format = " positional argument%s (and %zd keyword-only argument%s)";
5995 5 : kwonly_sig = PyUnicode_FromFormat(format,
5996 : given != 1 ? "s" : "",
5997 : kwonly_given,
5998 : kwonly_given != 1 ? "s" : "");
5999 5 : if (kwonly_sig == NULL) {
6000 0 : Py_DECREF(sig);
6001 0 : return;
6002 : }
6003 : }
6004 : else {
6005 : /* This will not fail. */
6006 289 : kwonly_sig = PyUnicode_FromString("");
6007 289 : assert(kwonly_sig != NULL);
6008 : }
6009 304 : _PyErr_Format(tstate, PyExc_TypeError,
6010 : "%U() takes %U positional argument%s but %zd%U %s given",
6011 : qualname,
6012 : sig,
6013 : plural ? "s" : "",
6014 : given,
6015 : kwonly_sig,
6016 10 : given == 1 && !kwonly_given ? "was" : "were");
6017 294 : Py_DECREF(sig);
6018 294 : Py_DECREF(kwonly_sig);
6019 : }
6020 :
6021 : static int
6022 18 : positional_only_passed_as_keyword(PyThreadState *tstate, PyCodeObject *co,
6023 : Py_ssize_t kwcount, PyObject* kwnames,
6024 : PyObject *qualname)
6025 : {
6026 18 : int posonly_conflicts = 0;
6027 18 : PyObject* posonly_names = PyList_New(0);
6028 :
6029 47 : for(int k=0; k < co->co_posonlyargcount; k++){
6030 29 : PyObject* posonly_name = PyTuple_GET_ITEM(co->co_localsplusnames, k);
6031 :
6032 77 : for (int k2=0; k2<kwcount; k2++){
6033 : /* Compare the pointers first and fallback to PyObject_RichCompareBool*/
6034 48 : PyObject* kwname = PyTuple_GET_ITEM(kwnames, k2);
6035 48 : if (kwname == posonly_name){
6036 19 : if(PyList_Append(posonly_names, kwname) != 0) {
6037 0 : goto fail;
6038 : }
6039 19 : posonly_conflicts++;
6040 19 : continue;
6041 : }
6042 :
6043 29 : int cmp = PyObject_RichCompareBool(posonly_name, kwname, Py_EQ);
6044 :
6045 29 : if ( cmp > 0) {
6046 0 : if(PyList_Append(posonly_names, kwname) != 0) {
6047 0 : goto fail;
6048 : }
6049 0 : posonly_conflicts++;
6050 29 : } else if (cmp < 0) {
6051 0 : goto fail;
6052 : }
6053 :
6054 : }
6055 : }
6056 18 : if (posonly_conflicts) {
6057 17 : PyObject* comma = PyUnicode_FromString(", ");
6058 17 : if (comma == NULL) {
6059 0 : goto fail;
6060 : }
6061 17 : PyObject* error_names = PyUnicode_Join(comma, posonly_names);
6062 17 : Py_DECREF(comma);
6063 17 : if (error_names == NULL) {
6064 0 : goto fail;
6065 : }
6066 17 : _PyErr_Format(tstate, PyExc_TypeError,
6067 : "%U() got some positional-only arguments passed"
6068 : " as keyword arguments: '%U'",
6069 : qualname, error_names);
6070 17 : Py_DECREF(error_names);
6071 17 : goto fail;
6072 : }
6073 :
6074 1 : Py_DECREF(posonly_names);
6075 1 : return 0;
6076 :
6077 17 : fail:
6078 17 : Py_XDECREF(posonly_names);
6079 17 : return 1;
6080 :
6081 : }
6082 :
6083 : /* Exception table parsing code.
6084 : * See Objects/exception_table_notes.txt for details.
6085 : */
6086 :
6087 : static inline unsigned char *
6088 39695200 : parse_varint(unsigned char *p, int *result) {
6089 39695200 : int val = p[0] & 63;
6090 54743300 : while (p[0] & 64) {
6091 15048100 : p++;
6092 15048100 : val = (val << 6) | (p[0] & 63);
6093 : }
6094 39695200 : *result = val;
6095 39695200 : return p+1;
6096 : }
6097 :
6098 : static inline unsigned char *
6099 4480620 : scan_back_to_entry_start(unsigned char *p) {
6100 14843400 : for (; (p[0]&128) == 0; p--);
6101 4480620 : return p;
6102 : }
6103 :
6104 : static inline unsigned char *
6105 4868060 : skip_to_next_entry(unsigned char *p, unsigned char *end) {
6106 19122000 : while (p < end && ((p[0] & 128) == 0)) {
6107 14254000 : p++;
6108 : }
6109 4868060 : return p;
6110 : }
6111 :
6112 :
6113 : #define MAX_LINEAR_SEARCH 40
6114 :
6115 : static int
6116 7024490 : get_exception_handler(PyCodeObject *code, int index, int *level, int *handler, int *lasti)
6117 : {
6118 7024490 : unsigned char *start = (unsigned char *)PyBytes_AS_STRING(code->co_exceptiontable);
6119 7024490 : unsigned char *end = start + PyBytes_GET_SIZE(code->co_exceptiontable);
6120 : /* Invariants:
6121 : * start_table == end_table OR
6122 : * start_table points to a legal entry and end_table points
6123 : * beyond the table or to a legal entry that is after index.
6124 : */
6125 7024490 : if (end - start > MAX_LINEAR_SEARCH) {
6126 : int offset;
6127 3943000 : parse_varint(start, &offset);
6128 3943000 : if (offset > index) {
6129 5288 : return 0;
6130 : }
6131 : do {
6132 4480620 : unsigned char * mid = start + ((end-start)>>1);
6133 4480620 : mid = scan_back_to_entry_start(mid);
6134 4480620 : parse_varint(mid, &offset);
6135 4480620 : if (offset > index) {
6136 3847680 : end = mid;
6137 : }
6138 : else {
6139 632935 : start = mid;
6140 : }
6141 :
6142 4480620 : } while (end - start > MAX_LINEAR_SEARCH);
6143 : }
6144 7019200 : unsigned char *scan = start;
6145 11887300 : while (scan < end) {
6146 : int start_offset, size;
6147 10374500 : scan = parse_varint(scan, &start_offset);
6148 10374500 : if (start_offset > index) {
6149 163387 : break;
6150 : }
6151 10211100 : scan = parse_varint(scan, &size);
6152 10211100 : if (start_offset + size > index) {
6153 5343020 : scan = parse_varint(scan, handler);
6154 : int depth_and_lasti;
6155 5343020 : parse_varint(scan, &depth_and_lasti);
6156 5343020 : *level = depth_and_lasti >> 1;
6157 5343020 : *lasti = depth_and_lasti & 1;
6158 5343020 : return 1;
6159 : }
6160 4868060 : scan = skip_to_next_entry(scan, end);
6161 : }
6162 1676180 : return 0;
6163 : }
6164 :
6165 : static int
6166 122154000 : initialize_locals(PyThreadState *tstate, PyFunctionObject *func,
6167 : PyObject **localsplus, PyObject *const *args,
6168 : Py_ssize_t argcount, PyObject *kwnames)
6169 : {
6170 122154000 : PyCodeObject *co = (PyCodeObject*)func->func_code;
6171 122154000 : const Py_ssize_t total_args = co->co_argcount + co->co_kwonlyargcount;
6172 :
6173 : /* Create a dictionary for keyword parameters (**kwags) */
6174 : PyObject *kwdict;
6175 : Py_ssize_t i;
6176 122154000 : if (co->co_flags & CO_VARKEYWORDS) {
6177 5244180 : kwdict = PyDict_New();
6178 5244180 : if (kwdict == NULL) {
6179 0 : goto fail_pre_positional;
6180 : }
6181 5244180 : i = total_args;
6182 5244180 : if (co->co_flags & CO_VARARGS) {
6183 4333660 : i++;
6184 : }
6185 5244180 : assert(localsplus[i] == NULL);
6186 5244180 : localsplus[i] = kwdict;
6187 : }
6188 : else {
6189 116910000 : kwdict = NULL;
6190 : }
6191 :
6192 : /* Copy all positional arguments into local variables */
6193 : Py_ssize_t j, n;
6194 122154000 : if (argcount > co->co_argcount) {
6195 19363500 : n = co->co_argcount;
6196 : }
6197 : else {
6198 102790000 : n = argcount;
6199 : }
6200 375399000 : for (j = 0; j < n; j++) {
6201 253245000 : PyObject *x = args[j];
6202 253245000 : assert(localsplus[j] == NULL);
6203 253245000 : localsplus[j] = x;
6204 : }
6205 :
6206 : /* Pack other positional arguments into the *args argument */
6207 122154000 : if (co->co_flags & CO_VARARGS) {
6208 20426600 : PyObject *u = NULL;
6209 20426600 : u = _PyTuple_FromArraySteal(args + n, argcount - n);
6210 20426600 : if (u == NULL) {
6211 0 : goto fail_post_positional;
6212 : }
6213 20426600 : assert(localsplus[total_args] == NULL);
6214 20426600 : localsplus[total_args] = u;
6215 : }
6216 101727000 : else if (argcount > n) {
6217 : /* Too many postional args. Error is reported later */
6218 687 : for (j = n; j < argcount; j++) {
6219 371 : Py_DECREF(args[j]);
6220 : }
6221 : }
6222 :
6223 : /* Handle keyword arguments */
6224 122154000 : if (kwnames != NULL) {
6225 16020300 : Py_ssize_t kwcount = PyTuple_GET_SIZE(kwnames);
6226 35645700 : for (i = 0; i < kwcount; i++) {
6227 : PyObject **co_varnames;
6228 19625600 : PyObject *keyword = PyTuple_GET_ITEM(kwnames, i);
6229 19625600 : PyObject *value = args[i+argcount];
6230 : Py_ssize_t j;
6231 :
6232 19625600 : if (keyword == NULL || !PyUnicode_Check(keyword)) {
6233 0 : _PyErr_Format(tstate, PyExc_TypeError,
6234 : "%U() keywords must be strings",
6235 : func->func_qualname);
6236 0 : goto kw_fail;
6237 : }
6238 :
6239 : /* Speed hack: do raw pointer compares. As names are
6240 : normally interned this should almost always hit. */
6241 19625600 : co_varnames = ((PyTupleObject *)(co->co_localsplusnames))->ob_item;
6242 72477400 : for (j = co->co_posonlyargcount; j < total_args; j++) {
6243 70379300 : PyObject *varname = co_varnames[j];
6244 70379300 : if (varname == keyword) {
6245 17527600 : goto kw_found;
6246 : }
6247 : }
6248 :
6249 : /* Slow fallback, just in case */
6250 4063170 : for (j = co->co_posonlyargcount; j < total_args; j++) {
6251 1997650 : PyObject *varname = co_varnames[j];
6252 1997650 : int cmp = PyObject_RichCompareBool( keyword, varname, Py_EQ);
6253 1997650 : if (cmp > 0) {
6254 32520 : goto kw_found;
6255 : }
6256 1965130 : else if (cmp < 0) {
6257 0 : goto kw_fail;
6258 : }
6259 : }
6260 :
6261 2065520 : assert(j >= total_args);
6262 2065520 : if (kwdict == NULL) {
6263 :
6264 177 : if (co->co_posonlyargcount
6265 18 : && positional_only_passed_as_keyword(tstate, co,
6266 : kwcount, kwnames,
6267 : func->func_qualname))
6268 : {
6269 17 : goto kw_fail;
6270 : }
6271 :
6272 160 : _PyErr_Format(tstate, PyExc_TypeError,
6273 : "%U() got an unexpected keyword argument '%S'",
6274 : func->func_qualname, keyword);
6275 160 : goto kw_fail;
6276 : }
6277 :
6278 2065340 : if (PyDict_SetItem(kwdict, keyword, value) == -1) {
6279 0 : goto kw_fail;
6280 : }
6281 2065340 : Py_DECREF(value);
6282 2065340 : continue;
6283 :
6284 238 : kw_fail:
6285 624 : for (;i < kwcount; i++) {
6286 386 : PyObject *value = args[i+argcount];
6287 386 : Py_DECREF(value);
6288 : }
6289 238 : goto fail_post_args;
6290 :
6291 17560100 : kw_found:
6292 17560100 : if (localsplus[j] != NULL) {
6293 61 : _PyErr_Format(tstate, PyExc_TypeError,
6294 : "%U() got multiple values for argument '%S'",
6295 : func->func_qualname, keyword);
6296 61 : goto kw_fail;
6297 : }
6298 17560100 : localsplus[j] = value;
6299 : }
6300 : }
6301 :
6302 : /* Check the number of positional arguments */
6303 122154000 : if ((argcount > co->co_argcount) && !(co->co_flags & CO_VARARGS)) {
6304 294 : too_many_positional(tstate, co, argcount, func->func_defaults, localsplus,
6305 : func->func_qualname);
6306 294 : goto fail_post_args;
6307 : }
6308 :
6309 : /* Add missing positional arguments (copy default values from defs) */
6310 122153000 : if (argcount < co->co_argcount) {
6311 18722700 : Py_ssize_t defcount = func->func_defaults == NULL ? 0 : PyTuple_GET_SIZE(func->func_defaults);
6312 18722700 : Py_ssize_t m = co->co_argcount - defcount;
6313 18722700 : Py_ssize_t missing = 0;
6314 19119600 : for (i = argcount; i < m; i++) {
6315 396856 : if (localsplus[i] == NULL) {
6316 711 : missing++;
6317 : }
6318 : }
6319 18722700 : if (missing) {
6320 621 : missing_arguments(tstate, co, missing, defcount, localsplus,
6321 : func->func_qualname);
6322 621 : goto fail_post_args;
6323 : }
6324 18722100 : if (n > m)
6325 2230410 : i = n - m;
6326 : else
6327 16491700 : i = 0;
6328 18722100 : if (defcount) {
6329 18650100 : PyObject **defs = &PyTuple_GET_ITEM(func->func_defaults, 0);
6330 44973000 : for (; i < defcount; i++) {
6331 26322900 : if (localsplus[m+i] == NULL) {
6332 14846800 : PyObject *def = defs[i];
6333 14846800 : Py_INCREF(def);
6334 14846800 : localsplus[m+i] = def;
6335 : }
6336 : }
6337 : }
6338 : }
6339 :
6340 : /* Add missing keyword arguments (copy default values from kwdefs) */
6341 122153000 : if (co->co_kwonlyargcount > 0) {
6342 7013090 : Py_ssize_t missing = 0;
6343 18456600 : for (i = co->co_argcount; i < total_args; i++) {
6344 11443500 : if (localsplus[i] != NULL)
6345 5687820 : continue;
6346 5755700 : PyObject *varname = PyTuple_GET_ITEM(co->co_localsplusnames, i);
6347 5755700 : if (func->func_kwdefaults != NULL) {
6348 5755700 : PyObject *def = PyDict_GetItemWithError(func->func_kwdefaults, varname);
6349 5755700 : if (def) {
6350 5755680 : Py_INCREF(def);
6351 5755680 : localsplus[i] = def;
6352 5755680 : continue;
6353 : }
6354 13 : else if (_PyErr_Occurred(tstate)) {
6355 0 : goto fail_post_args;
6356 : }
6357 : }
6358 23 : missing++;
6359 : }
6360 7013090 : if (missing) {
6361 18 : missing_arguments(tstate, co, missing, -1, localsplus,
6362 : func->func_qualname);
6363 18 : goto fail_post_args;
6364 : }
6365 : }
6366 122153000 : return 0;
6367 :
6368 0 : fail_pre_positional:
6369 0 : for (j = 0; j < argcount; j++) {
6370 0 : Py_DECREF(args[j]);
6371 : }
6372 : /* fall through */
6373 0 : fail_post_positional:
6374 0 : if (kwnames) {
6375 0 : Py_ssize_t kwcount = PyTuple_GET_SIZE(kwnames);
6376 0 : for (j = argcount; j < argcount+kwcount; j++) {
6377 0 : Py_DECREF(args[j]);
6378 : }
6379 : }
6380 : /* fall through */
6381 0 : fail_post_args:
6382 1171 : return -1;
6383 : }
6384 :
6385 : /* Consumes references to func, locals and all the args */
6386 : static _PyInterpreterFrame *
6387 122154000 : _PyEvalFramePushAndInit(PyThreadState *tstate, PyFunctionObject *func,
6388 : PyObject *locals, PyObject* const* args,
6389 : size_t argcount, PyObject *kwnames)
6390 : {
6391 122154000 : PyCodeObject * code = (PyCodeObject *)func->func_code;
6392 : CALL_STAT_INC(frames_pushed);
6393 122154000 : _PyInterpreterFrame *frame = _PyThreadState_PushFrame(tstate, code->co_framesize);
6394 122154000 : if (frame == NULL) {
6395 0 : goto fail;
6396 : }
6397 122154000 : _PyFrame_InitializeSpecials(frame, func, locals, code);
6398 122154000 : PyObject **localsarray = &frame->localsplus[0];
6399 598709000 : for (int i = 0; i < code->co_nlocalsplus; i++) {
6400 476555000 : localsarray[i] = NULL;
6401 : }
6402 122154000 : if (initialize_locals(tstate, func, localsarray, args, argcount, kwnames)) {
6403 1171 : assert(frame->owner != FRAME_OWNED_BY_GENERATOR);
6404 1171 : _PyFrame_Clear(frame);
6405 1171 : return NULL;
6406 : }
6407 122153000 : return frame;
6408 0 : fail:
6409 : /* Consume the references */
6410 0 : for (size_t i = 0; i < argcount; i++) {
6411 0 : Py_DECREF(args[i]);
6412 : }
6413 0 : if (kwnames) {
6414 0 : Py_ssize_t kwcount = PyTuple_GET_SIZE(kwnames);
6415 0 : for (Py_ssize_t i = 0; i < kwcount; i++) {
6416 0 : Py_DECREF(args[i+argcount]);
6417 : }
6418 : }
6419 0 : PyErr_NoMemory();
6420 0 : return NULL;
6421 : }
6422 :
6423 : static void
6424 257373000 : _PyEvalFrameClearAndPop(PyThreadState *tstate, _PyInterpreterFrame * frame)
6425 : {
6426 257373000 : tstate->recursion_remaining--;
6427 257373000 : assert(frame->frame_obj == NULL || frame->frame_obj->f_frame == frame);
6428 257373000 : assert(frame->owner == FRAME_OWNED_BY_THREAD);
6429 257373000 : _PyFrame_Clear(frame);
6430 257373000 : tstate->recursion_remaining++;
6431 257373000 : _PyThreadState_PopFrame(tstate, frame);
6432 257373000 : }
6433 :
6434 : PyObject *
6435 71192500 : _PyEval_Vector(PyThreadState *tstate, PyFunctionObject *func,
6436 : PyObject *locals,
6437 : PyObject* const* args, size_t argcount,
6438 : PyObject *kwnames)
6439 : {
6440 : /* _PyEvalFramePushAndInit consumes the references
6441 : * to func, locals and all its arguments */
6442 71192500 : Py_INCREF(func);
6443 71192500 : Py_XINCREF(locals);
6444 248435000 : for (size_t i = 0; i < argcount; i++) {
6445 177242000 : Py_INCREF(args[i]);
6446 : }
6447 71192500 : if (kwnames) {
6448 3368260 : Py_ssize_t kwcount = PyTuple_GET_SIZE(kwnames);
6449 8425220 : for (Py_ssize_t i = 0; i < kwcount; i++) {
6450 5056960 : Py_INCREF(args[i+argcount]);
6451 : }
6452 : }
6453 71192500 : _PyInterpreterFrame *frame = _PyEvalFramePushAndInit(
6454 : tstate, func, locals, args, argcount, kwnames);
6455 71192500 : if (frame == NULL) {
6456 859 : return NULL;
6457 : }
6458 : EVAL_CALL_STAT_INC(EVAL_CALL_VECTOR);
6459 71191700 : PyObject *retval = _PyEval_EvalFrame(tstate, frame, 0);
6460 71191400 : assert(
6461 : _PyFrame_GetStackPointer(frame) == _PyFrame_Stackbase(frame) ||
6462 : _PyFrame_GetStackPointer(frame) == frame->localsplus
6463 : );
6464 71191400 : _PyEvalFrameClearAndPop(tstate, frame);
6465 71191400 : return retval;
6466 : }
6467 :
6468 : /* Legacy API */
6469 : PyObject *
6470 2 : PyEval_EvalCodeEx(PyObject *_co, PyObject *globals, PyObject *locals,
6471 : PyObject *const *args, int argcount,
6472 : PyObject *const *kws, int kwcount,
6473 : PyObject *const *defs, int defcount,
6474 : PyObject *kwdefs, PyObject *closure)
6475 : {
6476 2 : PyThreadState *tstate = _PyThreadState_GET();
6477 2 : PyObject *res = NULL;
6478 2 : PyObject *defaults = _PyTuple_FromArray(defs, defcount);
6479 2 : if (defaults == NULL) {
6480 0 : return NULL;
6481 : }
6482 2 : PyObject *builtins = _PyEval_BuiltinsFromGlobals(tstate, globals); // borrowed ref
6483 2 : if (builtins == NULL) {
6484 0 : Py_DECREF(defaults);
6485 0 : return NULL;
6486 : }
6487 2 : if (locals == NULL) {
6488 0 : locals = globals;
6489 : }
6490 2 : PyObject *kwnames = NULL;
6491 : PyObject *const *allargs;
6492 2 : PyObject **newargs = NULL;
6493 2 : PyFunctionObject *func = NULL;
6494 2 : if (kwcount == 0) {
6495 2 : allargs = args;
6496 : }
6497 : else {
6498 0 : kwnames = PyTuple_New(kwcount);
6499 0 : if (kwnames == NULL) {
6500 0 : goto fail;
6501 : }
6502 0 : newargs = PyMem_Malloc(sizeof(PyObject *)*(kwcount+argcount));
6503 0 : if (newargs == NULL) {
6504 0 : goto fail;
6505 : }
6506 0 : for (int i = 0; i < argcount; i++) {
6507 0 : newargs[i] = args[i];
6508 : }
6509 0 : for (int i = 0; i < kwcount; i++) {
6510 0 : Py_INCREF(kws[2*i]);
6511 0 : PyTuple_SET_ITEM(kwnames, i, kws[2*i]);
6512 0 : newargs[argcount+i] = kws[2*i+1];
6513 : }
6514 0 : allargs = newargs;
6515 : }
6516 2 : for (int i = 0; i < kwcount; i++) {
6517 0 : Py_INCREF(kws[2*i]);
6518 0 : PyTuple_SET_ITEM(kwnames, i, kws[2*i]);
6519 : }
6520 2 : PyFrameConstructor constr = {
6521 : .fc_globals = globals,
6522 : .fc_builtins = builtins,
6523 2 : .fc_name = ((PyCodeObject *)_co)->co_name,
6524 2 : .fc_qualname = ((PyCodeObject *)_co)->co_name,
6525 : .fc_code = _co,
6526 : .fc_defaults = defaults,
6527 : .fc_kwdefaults = kwdefs,
6528 : .fc_closure = closure
6529 : };
6530 2 : func = _PyFunction_FromConstructor(&constr);
6531 2 : if (func == NULL) {
6532 0 : goto fail;
6533 : }
6534 : EVAL_CALL_STAT_INC(EVAL_CALL_LEGACY);
6535 2 : res = _PyEval_Vector(tstate, func, locals,
6536 : allargs, argcount,
6537 : kwnames);
6538 2 : fail:
6539 2 : Py_XDECREF(func);
6540 2 : Py_XDECREF(kwnames);
6541 2 : PyMem_Free(newargs);
6542 2 : Py_DECREF(defaults);
6543 2 : return res;
6544 : }
6545 :
6546 :
6547 : /* Logic for the raise statement (too complicated for inlining).
6548 : This *consumes* a reference count to each of its arguments. */
6549 : static int
6550 382971 : do_raise(PyThreadState *tstate, PyObject *exc, PyObject *cause)
6551 : {
6552 382971 : PyObject *type = NULL, *value = NULL;
6553 :
6554 382971 : if (exc == NULL) {
6555 : /* Reraise */
6556 13981 : _PyErr_StackItem *exc_info = _PyErr_GetTopmostException(tstate);
6557 13981 : value = exc_info->exc_value;
6558 13981 : if (Py_IsNone(value) || value == NULL) {
6559 2 : _PyErr_SetString(tstate, PyExc_RuntimeError,
6560 : "No active exception to reraise");
6561 2 : return 0;
6562 : }
6563 13979 : assert(PyExceptionInstance_Check(value));
6564 13979 : type = PyExceptionInstance_Class(value);
6565 13979 : Py_XINCREF(type);
6566 13979 : Py_XINCREF(value);
6567 13979 : PyObject *tb = PyException_GetTraceback(value); /* new ref */
6568 13979 : _PyErr_Restore(tstate, type, value, tb);
6569 13979 : return 1;
6570 : }
6571 :
6572 : /* We support the following forms of raise:
6573 : raise
6574 : raise <instance>
6575 : raise <type> */
6576 :
6577 368990 : if (PyExceptionClass_Check(exc)) {
6578 29552 : type = exc;
6579 29552 : value = _PyObject_CallNoArgs(exc);
6580 29552 : if (value == NULL)
6581 2 : goto raise_error;
6582 29550 : if (!PyExceptionInstance_Check(value)) {
6583 1 : _PyErr_Format(tstate, PyExc_TypeError,
6584 : "calling %R should have returned an instance of "
6585 : "BaseException, not %R",
6586 : type, Py_TYPE(value));
6587 1 : goto raise_error;
6588 : }
6589 : }
6590 339438 : else if (PyExceptionInstance_Check(exc)) {
6591 339433 : value = exc;
6592 339433 : type = PyExceptionInstance_Class(exc);
6593 339433 : Py_INCREF(type);
6594 : }
6595 : else {
6596 : /* Not something you can raise. You get an exception
6597 : anyway, just not what you specified :-) */
6598 5 : Py_DECREF(exc);
6599 5 : _PyErr_SetString(tstate, PyExc_TypeError,
6600 : "exceptions must derive from BaseException");
6601 5 : goto raise_error;
6602 : }
6603 :
6604 368982 : assert(type != NULL);
6605 368982 : assert(value != NULL);
6606 :
6607 368982 : if (cause) {
6608 : PyObject *fixed_cause;
6609 122065 : if (PyExceptionClass_Check(cause)) {
6610 3 : fixed_cause = _PyObject_CallNoArgs(cause);
6611 3 : if (fixed_cause == NULL)
6612 1 : goto raise_error;
6613 2 : Py_DECREF(cause);
6614 : }
6615 122062 : else if (PyExceptionInstance_Check(cause)) {
6616 1663 : fixed_cause = cause;
6617 : }
6618 120399 : else if (Py_IsNone(cause)) {
6619 120398 : Py_DECREF(cause);
6620 120398 : fixed_cause = NULL;
6621 : }
6622 : else {
6623 1 : _PyErr_SetString(tstate, PyExc_TypeError,
6624 : "exception causes must derive from "
6625 : "BaseException");
6626 1 : goto raise_error;
6627 : }
6628 122063 : PyException_SetCause(value, fixed_cause);
6629 : }
6630 :
6631 368980 : _PyErr_SetObject(tstate, type, value);
6632 : /* _PyErr_SetObject incref's its arguments */
6633 368980 : Py_DECREF(value);
6634 368980 : Py_DECREF(type);
6635 368980 : return 0;
6636 :
6637 10 : raise_error:
6638 10 : Py_XDECREF(value);
6639 10 : Py_XDECREF(type);
6640 10 : Py_XDECREF(cause);
6641 10 : return 0;
6642 : }
6643 :
6644 : /* Logic for matching an exception in an except* clause (too
6645 : complicated for inlining).
6646 : */
6647 :
6648 : static int
6649 175 : exception_group_match(PyObject* exc_value, PyObject *match_type,
6650 : PyObject **match, PyObject **rest)
6651 : {
6652 175 : if (Py_IsNone(exc_value)) {
6653 2 : *match = Py_NewRef(Py_None);
6654 2 : *rest = Py_NewRef(Py_None);
6655 2 : return 0;
6656 : }
6657 173 : assert(PyExceptionInstance_Check(exc_value));
6658 :
6659 173 : if (PyErr_GivenExceptionMatches(exc_value, match_type)) {
6660 : /* Full match of exc itself */
6661 56 : bool is_eg = _PyBaseExceptionGroup_Check(exc_value);
6662 56 : if (is_eg) {
6663 18 : *match = Py_NewRef(exc_value);
6664 : }
6665 : else {
6666 : /* naked exception - wrap it */
6667 38 : PyObject *excs = PyTuple_Pack(1, exc_value);
6668 38 : if (excs == NULL) {
6669 0 : return -1;
6670 : }
6671 38 : PyObject *wrapped = _PyExc_CreateExceptionGroup("", excs);
6672 38 : Py_DECREF(excs);
6673 38 : if (wrapped == NULL) {
6674 0 : return -1;
6675 : }
6676 38 : *match = wrapped;
6677 : }
6678 56 : *rest = Py_NewRef(Py_None);
6679 56 : return 0;
6680 : }
6681 :
6682 : /* exc_value does not match match_type.
6683 : * Check for partial match if it's an exception group.
6684 : */
6685 117 : if (_PyBaseExceptionGroup_Check(exc_value)) {
6686 101 : PyObject *pair = PyObject_CallMethod(exc_value, "split", "(O)",
6687 : match_type);
6688 101 : if (pair == NULL) {
6689 0 : return -1;
6690 : }
6691 101 : assert(PyTuple_CheckExact(pair));
6692 101 : assert(PyTuple_GET_SIZE(pair) == 2);
6693 101 : *match = Py_NewRef(PyTuple_GET_ITEM(pair, 0));
6694 101 : *rest = Py_NewRef(PyTuple_GET_ITEM(pair, 1));
6695 101 : Py_DECREF(pair);
6696 101 : return 0;
6697 : }
6698 : /* no match */
6699 16 : *match = Py_NewRef(Py_None);
6700 16 : *rest = Py_NewRef(Py_None);
6701 16 : return 0;
6702 : }
6703 :
6704 : /* Iterate v argcnt times and store the results on the stack (via decreasing
6705 : sp). Return 1 for success, 0 if error.
6706 :
6707 : If argcntafter == -1, do a simple unpack. If it is >= 0, do an unpack
6708 : with a variable target.
6709 : */
6710 :
6711 : static int
6712 3844900 : unpack_iterable(PyThreadState *tstate, PyObject *v,
6713 : int argcnt, int argcntafter, PyObject **sp)
6714 : {
6715 3844900 : int i = 0, j = 0;
6716 3844900 : Py_ssize_t ll = 0;
6717 : PyObject *it; /* iter(v) */
6718 : PyObject *w;
6719 3844900 : PyObject *l = NULL; /* variable list */
6720 :
6721 3844900 : assert(v != NULL);
6722 :
6723 3844900 : it = PyObject_GetIter(v);
6724 3844900 : if (it == NULL) {
6725 12 : if (_PyErr_ExceptionMatches(tstate, PyExc_TypeError) &&
6726 9 : Py_TYPE(v)->tp_iter == NULL && !PySequence_Check(v))
6727 : {
6728 9 : _PyErr_Format(tstate, PyExc_TypeError,
6729 : "cannot unpack non-iterable %.200s object",
6730 9 : Py_TYPE(v)->tp_name);
6731 : }
6732 12 : return 0;
6733 : }
6734 :
6735 19812000 : for (; i < argcnt; i++) {
6736 15968500 : w = PyIter_Next(it);
6737 15968500 : if (w == NULL) {
6738 : /* Iterator done, via error or exhaustion. */
6739 1360 : if (!_PyErr_Occurred(tstate)) {
6740 1338 : if (argcntafter == -1) {
6741 1337 : _PyErr_Format(tstate, PyExc_ValueError,
6742 : "not enough values to unpack "
6743 : "(expected %d, got %d)",
6744 : argcnt, i);
6745 : }
6746 : else {
6747 1 : _PyErr_Format(tstate, PyExc_ValueError,
6748 : "not enough values to unpack "
6749 : "(expected at least %d, got %d)",
6750 : argcnt + argcntafter, i);
6751 : }
6752 : }
6753 1360 : goto Error;
6754 : }
6755 15967200 : *--sp = w;
6756 : }
6757 :
6758 3843530 : if (argcntafter == -1) {
6759 : /* We better have exhausted the iterator now. */
6760 3738200 : w = PyIter_Next(it);
6761 3738200 : if (w == NULL) {
6762 3738140 : if (_PyErr_Occurred(tstate))
6763 1 : goto Error;
6764 3738140 : Py_DECREF(it);
6765 3738140 : return 1;
6766 : }
6767 55 : Py_DECREF(w);
6768 55 : _PyErr_Format(tstate, PyExc_ValueError,
6769 : "too many values to unpack (expected %d)",
6770 : argcnt);
6771 55 : goto Error;
6772 : }
6773 :
6774 105331 : l = PySequence_List(it);
6775 105331 : if (l == NULL)
6776 1 : goto Error;
6777 105330 : *--sp = l;
6778 105330 : i++;
6779 :
6780 105330 : ll = PyList_GET_SIZE(l);
6781 105330 : if (ll < argcntafter) {
6782 1 : _PyErr_Format(tstate, PyExc_ValueError,
6783 : "not enough values to unpack (expected at least %d, got %zd)",
6784 : argcnt + argcntafter, argcnt + ll);
6785 1 : goto Error;
6786 : }
6787 :
6788 : /* Pop the "after-variable" args off the list. */
6789 105543 : for (j = argcntafter; j > 0; j--, i++) {
6790 214 : *--sp = PyList_GET_ITEM(l, ll - j);
6791 : }
6792 : /* Resize the list. */
6793 105329 : Py_SET_SIZE(l, ll - argcntafter);
6794 105329 : Py_DECREF(it);
6795 105329 : return 1;
6796 :
6797 1418 : Error:
6798 2882 : for (; i > 0; i--, sp++)
6799 1464 : Py_DECREF(*sp);
6800 1418 : Py_XDECREF(it);
6801 1418 : return 0;
6802 : }
6803 :
6804 : static void
6805 8773 : call_exc_trace(Py_tracefunc func, PyObject *self,
6806 : PyThreadState *tstate,
6807 : _PyInterpreterFrame *f)
6808 : {
6809 : PyObject *type, *value, *traceback, *orig_traceback, *arg;
6810 : int err;
6811 8773 : _PyErr_Fetch(tstate, &type, &value, &orig_traceback);
6812 8773 : if (value == NULL) {
6813 1142 : value = Py_None;
6814 1142 : Py_INCREF(value);
6815 : }
6816 8773 : _PyErr_NormalizeException(tstate, &type, &value, &orig_traceback);
6817 8773 : traceback = (orig_traceback != NULL) ? orig_traceback : Py_None;
6818 8773 : arg = PyTuple_Pack(3, type, value, traceback);
6819 8773 : if (arg == NULL) {
6820 0 : _PyErr_Restore(tstate, type, value, orig_traceback);
6821 0 : return;
6822 : }
6823 8773 : err = call_trace(func, self, tstate, f, PyTrace_EXCEPTION, arg);
6824 8773 : Py_DECREF(arg);
6825 8773 : if (err == 0) {
6826 7771 : _PyErr_Restore(tstate, type, value, orig_traceback);
6827 : }
6828 : else {
6829 1002 : Py_XDECREF(type);
6830 1002 : Py_XDECREF(value);
6831 1002 : Py_XDECREF(orig_traceback);
6832 : }
6833 : }
6834 :
6835 : static int
6836 2616240 : call_trace_protected(Py_tracefunc func, PyObject *obj,
6837 : PyThreadState *tstate, _PyInterpreterFrame *frame,
6838 : int what, PyObject *arg)
6839 : {
6840 : PyObject *type, *value, *traceback;
6841 : int err;
6842 2616240 : _PyErr_Fetch(tstate, &type, &value, &traceback);
6843 2616240 : err = call_trace(func, obj, tstate, frame, what, arg);
6844 2616240 : if (err == 0)
6845 : {
6846 2614230 : _PyErr_Restore(tstate, type, value, traceback);
6847 2614230 : return 0;
6848 : }
6849 : else {
6850 2011 : Py_XDECREF(type);
6851 2011 : Py_XDECREF(value);
6852 2011 : Py_XDECREF(traceback);
6853 2011 : return -1;
6854 : }
6855 : }
6856 :
6857 : static void
6858 0 : initialize_trace_info(PyTraceInfo *trace_info, _PyInterpreterFrame *frame)
6859 : {
6860 0 : PyCodeObject *code = frame->f_code;
6861 0 : if (trace_info->code != code) {
6862 0 : trace_info->code = code;
6863 0 : _PyCode_InitAddressRange(code, &trace_info->bounds);
6864 : }
6865 0 : }
6866 :
6867 : void
6868 5979180 : PyThreadState_EnterTracing(PyThreadState *tstate)
6869 : {
6870 5979180 : tstate->tracing++;
6871 5979180 : }
6872 :
6873 : void
6874 5979180 : PyThreadState_LeaveTracing(PyThreadState *tstate)
6875 : {
6876 5979180 : tstate->tracing--;
6877 5979180 : }
6878 :
6879 : static int
6880 3934640 : call_trace(Py_tracefunc func, PyObject *obj,
6881 : PyThreadState *tstate, _PyInterpreterFrame *frame,
6882 : int what, PyObject *arg)
6883 : {
6884 : int result;
6885 3934640 : if (tstate->tracing) {
6886 2383730 : return 0;
6887 : }
6888 1550910 : PyFrameObject *f = _PyFrame_GetFrameObject(frame);
6889 1550910 : if (f == NULL) {
6890 0 : return -1;
6891 : }
6892 1550910 : int old_what = tstate->tracing_what;
6893 1550910 : tstate->tracing_what = what;
6894 1550910 : PyThreadState_EnterTracing(tstate);
6895 1550910 : assert(_PyInterpreterFrame_LASTI(frame) >= 0);
6896 1550910 : if (_PyCode_InitLineArray(frame->f_code)) {
6897 0 : return -1;
6898 : }
6899 1550910 : f->f_lineno = _PyCode_LineNumberFromArray(frame->f_code, _PyInterpreterFrame_LASTI(frame));
6900 1550910 : result = func(obj, f, what, arg);
6901 1550910 : f->f_lineno = 0;
6902 1550910 : PyThreadState_LeaveTracing(tstate);
6903 1550910 : tstate->tracing_what = old_what;
6904 1550910 : return result;
6905 : }
6906 :
6907 : PyObject*
6908 2 : _PyEval_CallTracing(PyObject *func, PyObject *args)
6909 : {
6910 : // Save and disable tracing
6911 2 : PyThreadState *tstate = _PyThreadState_GET();
6912 2 : int save_tracing = tstate->tracing;
6913 2 : int save_use_tracing = tstate->cframe->use_tracing;
6914 2 : tstate->tracing = 0;
6915 :
6916 : // Call the tracing function
6917 2 : PyObject *result = PyObject_Call(func, args, NULL);
6918 :
6919 : // Restore tracing
6920 2 : tstate->tracing = save_tracing;
6921 2 : tstate->cframe->use_tracing = save_use_tracing;
6922 2 : return result;
6923 : }
6924 :
6925 : /* See Objects/lnotab_notes.txt for a description of how tracing works. */
6926 : static int
6927 5843280 : maybe_call_line_trace(Py_tracefunc func, PyObject *obj,
6928 : PyThreadState *tstate, _PyInterpreterFrame *frame, int instr_prev)
6929 : {
6930 5843280 : int result = 0;
6931 :
6932 : /* If the last instruction falls at the start of a line or if it
6933 : represents a jump backwards, update the frame's line number and
6934 : then call the trace function if we're tracing source lines.
6935 : */
6936 5843280 : if (_PyCode_InitLineArray(frame->f_code)) {
6937 0 : return -1;
6938 : }
6939 : int lastline;
6940 5843280 : if (instr_prev <= frame->f_code->_co_firsttraceable) {
6941 127553 : lastline = -1;
6942 : }
6943 : else {
6944 5715730 : lastline = _PyCode_LineNumberFromArray(frame->f_code, instr_prev);
6945 : }
6946 5843280 : int line = _PyCode_LineNumberFromArray(frame->f_code, _PyInterpreterFrame_LASTI(frame));
6947 5843280 : PyFrameObject *f = _PyFrame_GetFrameObject(frame);
6948 5843280 : if (f == NULL) {
6949 0 : return -1;
6950 : }
6951 5843280 : if (line != -1 && f->f_trace_lines) {
6952 : /* Trace backward edges (except in 'yield from') or if line number has changed */
6953 10413100 : int trace = line != lastline ||
6954 4574120 : (_PyInterpreterFrame_LASTI(frame) < instr_prev &&
6955 : // SEND has no quickened forms, so no need to use _PyOpcode_Deopt
6956 : // here:
6957 4102 : _Py_OPCODE(*frame->prev_instr) != SEND);
6958 5839020 : if (trace) {
6959 1268180 : result = call_trace(func, obj, tstate, frame, PyTrace_LINE, Py_None);
6960 : }
6961 : }
6962 : /* Always emit an opcode event if we're tracing all opcodes. */
6963 5843280 : if (f->f_trace_opcodes) {
6964 1381 : result = call_trace(func, obj, tstate, frame, PyTrace_OPCODE, Py_None);
6965 : }
6966 5843280 : return result;
6967 : }
6968 :
6969 : int
6970 170 : _PyEval_SetProfile(PyThreadState *tstate, Py_tracefunc func, PyObject *arg)
6971 : {
6972 170 : assert(is_tstate_valid(tstate));
6973 : /* The caller must hold the GIL */
6974 170 : assert(PyGILState_Check());
6975 :
6976 : static int reentrant = 0;
6977 170 : if (reentrant) {
6978 2 : _PyErr_SetString(tstate, PyExc_RuntimeError, "Cannot install a profile function "
6979 : "while another profile function is being installed");
6980 2 : reentrant = 0;
6981 2 : return -1;
6982 : }
6983 168 : reentrant = 1;
6984 :
6985 : /* Call _PySys_Audit() in the context of the current thread state,
6986 : even if tstate is not the current thread state. */
6987 168 : PyThreadState *current_tstate = _PyThreadState_GET();
6988 168 : if (_PySys_Audit(current_tstate, "sys.setprofile", NULL) < 0) {
6989 0 : reentrant = 0;
6990 0 : return -1;
6991 : }
6992 :
6993 168 : PyObject *profileobj = tstate->c_profileobj;
6994 :
6995 168 : tstate->c_profilefunc = NULL;
6996 168 : tstate->c_profileobj = NULL;
6997 : /* Must make sure that tracing is not ignored if 'profileobj' is freed */
6998 168 : _PyThreadState_UpdateTracingState(tstate);
6999 168 : Py_XDECREF(profileobj);
7000 :
7001 168 : Py_XINCREF(arg);
7002 168 : tstate->c_profileobj = arg;
7003 168 : tstate->c_profilefunc = func;
7004 :
7005 : /* Flag that tracing or profiling is turned on */
7006 168 : _PyThreadState_UpdateTracingState(tstate);
7007 168 : reentrant = 0;
7008 168 : return 0;
7009 : }
7010 :
7011 : void
7012 0 : PyEval_SetProfile(Py_tracefunc func, PyObject *arg)
7013 : {
7014 0 : PyThreadState *tstate = _PyThreadState_GET();
7015 0 : if (_PyEval_SetProfile(tstate, func, arg) < 0) {
7016 : /* Log _PySys_Audit() error */
7017 0 : _PyErr_WriteUnraisableMsg("in PyEval_SetProfile", NULL);
7018 : }
7019 0 : }
7020 :
7021 : int
7022 10887 : _PyEval_SetTrace(PyThreadState *tstate, Py_tracefunc func, PyObject *arg)
7023 : {
7024 10887 : assert(is_tstate_valid(tstate));
7025 : /* The caller must hold the GIL */
7026 10887 : assert(PyGILState_Check());
7027 :
7028 : static int reentrant = 0;
7029 :
7030 10887 : if (reentrant) {
7031 1 : _PyErr_SetString(tstate, PyExc_RuntimeError, "Cannot install a trace function "
7032 : "while another trace function is being installed");
7033 1 : reentrant = 0;
7034 1 : return -1;
7035 : }
7036 10886 : reentrant = 1;
7037 :
7038 : /* Call _PySys_Audit() in the context of the current thread state,
7039 : even if tstate is not the current thread state. */
7040 10886 : PyThreadState *current_tstate = _PyThreadState_GET();
7041 10886 : if (_PySys_Audit(current_tstate, "sys.settrace", NULL) < 0) {
7042 0 : reentrant = 0;
7043 0 : return -1;
7044 : }
7045 :
7046 10886 : PyObject *traceobj = tstate->c_traceobj;
7047 :
7048 10886 : tstate->c_tracefunc = NULL;
7049 10886 : tstate->c_traceobj = NULL;
7050 : /* Must make sure that profiling is not ignored if 'traceobj' is freed */
7051 10886 : _PyThreadState_UpdateTracingState(tstate);
7052 10886 : Py_XINCREF(arg);
7053 10886 : Py_XDECREF(traceobj);
7054 10886 : tstate->c_traceobj = arg;
7055 10886 : tstate->c_tracefunc = func;
7056 :
7057 : /* Flag that tracing or profiling is turned on */
7058 10886 : _PyThreadState_UpdateTracingState(tstate);
7059 :
7060 10886 : reentrant = 0;
7061 10886 : return 0;
7062 : }
7063 :
7064 : void
7065 3 : PyEval_SetTrace(Py_tracefunc func, PyObject *arg)
7066 : {
7067 3 : PyThreadState *tstate = _PyThreadState_GET();
7068 3 : if (_PyEval_SetTrace(tstate, func, arg) < 0) {
7069 : /* Log _PySys_Audit() error */
7070 0 : _PyErr_WriteUnraisableMsg("in PyEval_SetTrace", NULL);
7071 : }
7072 3 : }
7073 :
7074 :
7075 : int
7076 2458 : _PyEval_SetCoroutineOriginTrackingDepth(int depth)
7077 : {
7078 2458 : PyThreadState *tstate = _PyThreadState_GET();
7079 2458 : if (depth < 0) {
7080 1 : _PyErr_SetString(tstate, PyExc_ValueError, "depth must be >= 0");
7081 1 : return -1;
7082 : }
7083 2457 : tstate->coroutine_origin_tracking_depth = depth;
7084 2457 : return 0;
7085 : }
7086 :
7087 :
7088 : int
7089 1233 : _PyEval_GetCoroutineOriginTrackingDepth(void)
7090 : {
7091 1233 : PyThreadState *tstate = _PyThreadState_GET();
7092 1233 : return tstate->coroutine_origin_tracking_depth;
7093 : }
7094 :
7095 : int
7096 10508 : _PyEval_SetAsyncGenFirstiter(PyObject *firstiter)
7097 : {
7098 10508 : PyThreadState *tstate = _PyThreadState_GET();
7099 :
7100 10508 : if (_PySys_Audit(tstate, "sys.set_asyncgen_hook_firstiter", NULL) < 0) {
7101 0 : return -1;
7102 : }
7103 :
7104 10508 : Py_XINCREF(firstiter);
7105 10508 : Py_XSETREF(tstate->async_gen_firstiter, firstiter);
7106 10508 : return 0;
7107 : }
7108 :
7109 : PyObject *
7110 5261 : _PyEval_GetAsyncGenFirstiter(void)
7111 : {
7112 5261 : PyThreadState *tstate = _PyThreadState_GET();
7113 5261 : return tstate->async_gen_firstiter;
7114 : }
7115 :
7116 : int
7117 10508 : _PyEval_SetAsyncGenFinalizer(PyObject *finalizer)
7118 : {
7119 10508 : PyThreadState *tstate = _PyThreadState_GET();
7120 :
7121 10508 : if (_PySys_Audit(tstate, "sys.set_asyncgen_hook_finalizer", NULL) < 0) {
7122 0 : return -1;
7123 : }
7124 :
7125 10508 : Py_XINCREF(finalizer);
7126 10508 : Py_XSETREF(tstate->async_gen_finalizer, finalizer);
7127 10508 : return 0;
7128 : }
7129 :
7130 : PyObject *
7131 5261 : _PyEval_GetAsyncGenFinalizer(void)
7132 : {
7133 5261 : PyThreadState *tstate = _PyThreadState_GET();
7134 5261 : return tstate->async_gen_finalizer;
7135 : }
7136 :
7137 : _PyInterpreterFrame *
7138 6428 : _PyEval_GetFrame(void)
7139 : {
7140 6428 : PyThreadState *tstate = _PyThreadState_GET();
7141 6428 : return tstate->cframe->current_frame;
7142 : }
7143 :
7144 : PyFrameObject *
7145 0 : PyEval_GetFrame(void)
7146 : {
7147 0 : PyThreadState *tstate = _PyThreadState_GET();
7148 0 : if (tstate->cframe->current_frame == NULL) {
7149 0 : return NULL;
7150 : }
7151 0 : PyFrameObject *f = _PyFrame_GetFrameObject(tstate->cframe->current_frame);
7152 0 : if (f == NULL) {
7153 0 : PyErr_Clear();
7154 : }
7155 0 : return f;
7156 : }
7157 :
7158 : PyObject *
7159 251833 : _PyEval_GetBuiltins(PyThreadState *tstate)
7160 : {
7161 251833 : _PyInterpreterFrame *frame = tstate->cframe->current_frame;
7162 251833 : if (frame != NULL) {
7163 245572 : return frame->f_builtins;
7164 : }
7165 6261 : return tstate->interp->builtins;
7166 : }
7167 :
7168 : PyObject *
7169 251762 : PyEval_GetBuiltins(void)
7170 : {
7171 251762 : PyThreadState *tstate = _PyThreadState_GET();
7172 251762 : return _PyEval_GetBuiltins(tstate);
7173 : }
7174 :
7175 : /* Convenience function to get a builtin from its name */
7176 : PyObject *
7177 4291 : _PyEval_GetBuiltin(PyObject *name)
7178 : {
7179 4291 : PyThreadState *tstate = _PyThreadState_GET();
7180 4291 : PyObject *attr = PyDict_GetItemWithError(PyEval_GetBuiltins(), name);
7181 4291 : if (attr) {
7182 4291 : Py_INCREF(attr);
7183 : }
7184 0 : else if (!_PyErr_Occurred(tstate)) {
7185 0 : _PyErr_SetObject(tstate, PyExc_AttributeError, name);
7186 : }
7187 4291 : return attr;
7188 : }
7189 :
7190 : PyObject *
7191 0 : _PyEval_GetBuiltinId(_Py_Identifier *name)
7192 : {
7193 0 : return _PyEval_GetBuiltin(_PyUnicode_FromId(name));
7194 : }
7195 :
7196 : PyObject *
7197 21379 : PyEval_GetLocals(void)
7198 : {
7199 21379 : PyThreadState *tstate = _PyThreadState_GET();
7200 21379 : _PyInterpreterFrame *current_frame = tstate->cframe->current_frame;
7201 21379 : if (current_frame == NULL) {
7202 0 : _PyErr_SetString(tstate, PyExc_SystemError, "frame does not exist");
7203 0 : return NULL;
7204 : }
7205 :
7206 21379 : if (_PyFrame_FastToLocalsWithError(current_frame) < 0) {
7207 0 : return NULL;
7208 : }
7209 :
7210 21379 : PyObject *locals = current_frame->f_locals;
7211 21379 : assert(locals != NULL);
7212 21379 : return locals;
7213 : }
7214 :
7215 : PyObject *
7216 729369 : PyEval_GetGlobals(void)
7217 : {
7218 729369 : PyThreadState *tstate = _PyThreadState_GET();
7219 729369 : _PyInterpreterFrame *current_frame = tstate->cframe->current_frame;
7220 729369 : if (current_frame == NULL) {
7221 28068 : return NULL;
7222 : }
7223 701301 : return current_frame->f_globals;
7224 : }
7225 :
7226 : int
7227 110071 : PyEval_MergeCompilerFlags(PyCompilerFlags *cf)
7228 : {
7229 110071 : PyThreadState *tstate = _PyThreadState_GET();
7230 110071 : _PyInterpreterFrame *current_frame = tstate->cframe->current_frame;
7231 110071 : int result = cf->cf_flags != 0;
7232 :
7233 110071 : if (current_frame != NULL) {
7234 110071 : const int codeflags = current_frame->f_code->co_flags;
7235 110071 : const int compilerflags = codeflags & PyCF_MASK;
7236 110071 : if (compilerflags) {
7237 0 : result = 1;
7238 0 : cf->cf_flags |= compilerflags;
7239 : }
7240 : }
7241 110071 : return result;
7242 : }
7243 :
7244 :
7245 : const char *
7246 0 : PyEval_GetFuncName(PyObject *func)
7247 : {
7248 0 : if (PyMethod_Check(func))
7249 0 : return PyEval_GetFuncName(PyMethod_GET_FUNCTION(func));
7250 0 : else if (PyFunction_Check(func))
7251 0 : return PyUnicode_AsUTF8(((PyFunctionObject*)func)->func_name);
7252 0 : else if (PyCFunction_Check(func))
7253 0 : return ((PyCFunctionObject*)func)->m_ml->ml_name;
7254 : else
7255 0 : return Py_TYPE(func)->tp_name;
7256 : }
7257 :
7258 : const char *
7259 0 : PyEval_GetFuncDesc(PyObject *func)
7260 : {
7261 0 : if (PyMethod_Check(func))
7262 0 : return "()";
7263 0 : else if (PyFunction_Check(func))
7264 0 : return "()";
7265 0 : else if (PyCFunction_Check(func))
7266 0 : return "()";
7267 : else
7268 0 : return " object";
7269 : }
7270 :
7271 : #define C_TRACE(x, call) \
7272 : if (use_tracing && tstate->c_profilefunc) { \
7273 : if (call_trace(tstate->c_profilefunc, tstate->c_profileobj, \
7274 : tstate, tstate->cframe->current_frame, \
7275 : PyTrace_C_CALL, func)) { \
7276 : x = NULL; \
7277 : } \
7278 : else { \
7279 : x = call; \
7280 : if (tstate->c_profilefunc != NULL) { \
7281 : if (x == NULL) { \
7282 : call_trace_protected(tstate->c_profilefunc, \
7283 : tstate->c_profileobj, \
7284 : tstate, tstate->cframe->current_frame, \
7285 : PyTrace_C_EXCEPTION, func); \
7286 : /* XXX should pass (type, value, tb) */ \
7287 : } else { \
7288 : if (call_trace(tstate->c_profilefunc, \
7289 : tstate->c_profileobj, \
7290 : tstate, tstate->cframe->current_frame, \
7291 : PyTrace_C_RETURN, func)) { \
7292 : Py_DECREF(x); \
7293 : x = NULL; \
7294 : } \
7295 : } \
7296 : } \
7297 : } \
7298 : } else { \
7299 : x = call; \
7300 : }
7301 :
7302 :
7303 : static PyObject *
7304 2005420 : trace_call_function(PyThreadState *tstate,
7305 : PyObject *func,
7306 : PyObject **args, Py_ssize_t nargs,
7307 : PyObject *kwnames)
7308 : {
7309 2005420 : int use_tracing = 1;
7310 : PyObject *x;
7311 2005420 : if (PyCFunction_CheckExact(func) || PyCMethod_CheckExact(func)) {
7312 817712 : C_TRACE(x, PyObject_Vectorcall(func, args, nargs, kwnames));
7313 817712 : return x;
7314 : }
7315 1187710 : else if (Py_IS_TYPE(func, &PyMethodDescr_Type) && nargs > 0) {
7316 : /* We need to create a temporary bound method as argument
7317 : for profiling.
7318 :
7319 : If nargs == 0, then this cannot work because we have no
7320 : "self". In any case, the call itself would raise
7321 : TypeError (foo needs an argument), so we just skip
7322 : profiling. */
7323 1118590 : PyObject *self = args[0];
7324 1118590 : func = Py_TYPE(func)->tp_descr_get(func, self, (PyObject*)Py_TYPE(self));
7325 1118590 : if (func == NULL) {
7326 1 : return NULL;
7327 : }
7328 1118590 : C_TRACE(x, PyObject_Vectorcall(func,
7329 : args+1, nargs-1,
7330 : kwnames));
7331 1118590 : Py_DECREF(func);
7332 1118590 : return x;
7333 : }
7334 69117 : return PyObject_Vectorcall(func, args, nargs | PY_VECTORCALL_ARGUMENTS_OFFSET, kwnames);
7335 : }
7336 :
7337 : static PyObject *
7338 9524240 : do_call_core(PyThreadState *tstate,
7339 : PyObject *func,
7340 : PyObject *callargs,
7341 : PyObject *kwdict,
7342 : int use_tracing
7343 : )
7344 : {
7345 : PyObject *result;
7346 9524240 : if (PyCFunction_CheckExact(func) || PyCMethod_CheckExact(func)) {
7347 1638600 : C_TRACE(result, PyObject_Call(func, callargs, kwdict));
7348 1638600 : return result;
7349 : }
7350 7885640 : else if (Py_IS_TYPE(func, &PyMethodDescr_Type)) {
7351 311 : Py_ssize_t nargs = PyTuple_GET_SIZE(callargs);
7352 311 : if (nargs > 0 && use_tracing) {
7353 : /* We need to create a temporary bound method as argument
7354 : for profiling.
7355 :
7356 : If nargs == 0, then this cannot work because we have no
7357 : "self". In any case, the call itself would raise
7358 : TypeError (foo needs an argument), so we just skip
7359 : profiling. */
7360 2 : PyObject *self = PyTuple_GET_ITEM(callargs, 0);
7361 2 : func = Py_TYPE(func)->tp_descr_get(func, self, (PyObject*)Py_TYPE(self));
7362 2 : if (func == NULL) {
7363 1 : return NULL;
7364 : }
7365 :
7366 1 : C_TRACE(result, _PyObject_FastCallDictTstate(
7367 : tstate, func,
7368 : &_PyTuple_ITEMS(callargs)[1],
7369 : nargs - 1,
7370 : kwdict));
7371 1 : Py_DECREF(func);
7372 1 : return result;
7373 : }
7374 : }
7375 : EVAL_CALL_STAT_INC_IF_FUNCTION(EVAL_CALL_FUNCTION_EX, func);
7376 7885630 : return PyObject_Call(func, callargs, kwdict);
7377 : }
7378 :
7379 : /* Extract a slice index from a PyLong or an object with the
7380 : nb_index slot defined, and store in *pi.
7381 : Silently reduce values larger than PY_SSIZE_T_MAX to PY_SSIZE_T_MAX,
7382 : and silently boost values less than PY_SSIZE_T_MIN to PY_SSIZE_T_MIN.
7383 : Return 0 on error, 1 on success.
7384 : */
7385 : int
7386 64461800 : _PyEval_SliceIndex(PyObject *v, Py_ssize_t *pi)
7387 : {
7388 64461800 : PyThreadState *tstate = _PyThreadState_GET();
7389 64461800 : if (!Py_IsNone(v)) {
7390 : Py_ssize_t x;
7391 64461800 : if (_PyIndex_Check(v)) {
7392 64461800 : x = PyNumber_AsSsize_t(v, NULL);
7393 64461800 : if (x == -1 && _PyErr_Occurred(tstate))
7394 12 : return 0;
7395 : }
7396 : else {
7397 1 : _PyErr_SetString(tstate, PyExc_TypeError,
7398 : "slice indices must be integers or "
7399 : "None or have an __index__ method");
7400 1 : return 0;
7401 : }
7402 64461800 : *pi = x;
7403 : }
7404 64461800 : return 1;
7405 : }
7406 :
7407 : int
7408 272872 : _PyEval_SliceIndexNotNone(PyObject *v, Py_ssize_t *pi)
7409 : {
7410 272872 : PyThreadState *tstate = _PyThreadState_GET();
7411 : Py_ssize_t x;
7412 272872 : if (_PyIndex_Check(v)) {
7413 272872 : x = PyNumber_AsSsize_t(v, NULL);
7414 272872 : if (x == -1 && _PyErr_Occurred(tstate))
7415 0 : return 0;
7416 : }
7417 : else {
7418 0 : _PyErr_SetString(tstate, PyExc_TypeError,
7419 : "slice indices must be integers or "
7420 : "have an __index__ method");
7421 0 : return 0;
7422 : }
7423 272872 : *pi = x;
7424 272872 : return 1;
7425 : }
7426 :
7427 : static PyObject *
7428 1450810 : import_name(PyThreadState *tstate, _PyInterpreterFrame *frame,
7429 : PyObject *name, PyObject *fromlist, PyObject *level)
7430 : {
7431 : PyObject *import_func, *res;
7432 : PyObject* stack[5];
7433 :
7434 1450810 : import_func = _PyDict_GetItemWithError(frame->f_builtins, &_Py_ID(__import__));
7435 1450810 : if (import_func == NULL) {
7436 1 : if (!_PyErr_Occurred(tstate)) {
7437 1 : _PyErr_SetString(tstate, PyExc_ImportError, "__import__ not found");
7438 : }
7439 1 : return NULL;
7440 : }
7441 1450810 : PyObject *locals = frame->f_locals;
7442 : /* Fast path for not overloaded __import__. */
7443 1450810 : if (import_func == tstate->interp->import_func) {
7444 1450810 : int ilevel = _PyLong_AsInt(level);
7445 1450810 : if (ilevel == -1 && _PyErr_Occurred(tstate)) {
7446 0 : return NULL;
7447 : }
7448 1450810 : res = PyImport_ImportModuleLevelObject(
7449 : name,
7450 : frame->f_globals,
7451 : locals == NULL ? Py_None :locals,
7452 : fromlist,
7453 : ilevel);
7454 1450810 : return res;
7455 : }
7456 :
7457 2 : Py_INCREF(import_func);
7458 :
7459 2 : stack[0] = name;
7460 2 : stack[1] = frame->f_globals;
7461 2 : stack[2] = locals == NULL ? Py_None : locals;
7462 2 : stack[3] = fromlist;
7463 2 : stack[4] = level;
7464 2 : res = _PyObject_FastCall(import_func, stack, 5);
7465 2 : Py_DECREF(import_func);
7466 2 : return res;
7467 : }
7468 :
7469 : static PyObject *
7470 809909 : import_from(PyThreadState *tstate, PyObject *v, PyObject *name)
7471 : {
7472 : PyObject *x;
7473 : PyObject *fullmodname, *pkgname, *pkgpath, *pkgname_or_unknown, *errmsg;
7474 :
7475 809909 : if (_PyObject_LookupAttr(v, name, &x) != 0) {
7476 804461 : return x;
7477 : }
7478 : /* Issue #17636: in case this failed because of a circular relative
7479 : import, try to fallback on reading the module directly from
7480 : sys.modules. */
7481 5448 : pkgname = PyObject_GetAttr(v, &_Py_ID(__name__));
7482 5448 : if (pkgname == NULL) {
7483 1 : goto error;
7484 : }
7485 5447 : if (!PyUnicode_Check(pkgname)) {
7486 1 : Py_CLEAR(pkgname);
7487 1 : goto error;
7488 : }
7489 5446 : fullmodname = PyUnicode_FromFormat("%U.%U", pkgname, name);
7490 5446 : if (fullmodname == NULL) {
7491 0 : Py_DECREF(pkgname);
7492 0 : return NULL;
7493 : }
7494 5446 : x = PyImport_GetModule(fullmodname);
7495 5446 : Py_DECREF(fullmodname);
7496 5446 : if (x == NULL && !_PyErr_Occurred(tstate)) {
7497 4766 : goto error;
7498 : }
7499 680 : Py_DECREF(pkgname);
7500 680 : return x;
7501 4768 : error:
7502 4768 : pkgpath = PyModule_GetFilenameObject(v);
7503 4768 : if (pkgname == NULL) {
7504 2 : pkgname_or_unknown = PyUnicode_FromString("<unknown module name>");
7505 2 : if (pkgname_or_unknown == NULL) {
7506 0 : Py_XDECREF(pkgpath);
7507 0 : return NULL;
7508 : }
7509 : } else {
7510 4766 : pkgname_or_unknown = pkgname;
7511 : }
7512 :
7513 4768 : if (pkgpath == NULL || !PyUnicode_Check(pkgpath)) {
7514 3130 : _PyErr_Clear(tstate);
7515 3130 : errmsg = PyUnicode_FromFormat(
7516 : "cannot import name %R from %R (unknown location)",
7517 : name, pkgname_or_unknown
7518 : );
7519 : /* NULL checks for errmsg and pkgname done by PyErr_SetImportError. */
7520 3130 : PyErr_SetImportError(errmsg, pkgname, NULL);
7521 : }
7522 : else {
7523 1638 : PyObject *spec = PyObject_GetAttr(v, &_Py_ID(__spec__));
7524 1638 : const char *fmt =
7525 1638 : _PyModuleSpec_IsInitializing(spec) ?
7526 : "cannot import name %R from partially initialized module %R "
7527 1638 : "(most likely due to a circular import) (%S)" :
7528 : "cannot import name %R from %R (%S)";
7529 1638 : Py_XDECREF(spec);
7530 :
7531 1638 : errmsg = PyUnicode_FromFormat(fmt, name, pkgname_or_unknown, pkgpath);
7532 : /* NULL checks for errmsg and pkgname done by PyErr_SetImportError. */
7533 1638 : PyErr_SetImportError(errmsg, pkgname, pkgpath);
7534 : }
7535 :
7536 4768 : Py_XDECREF(errmsg);
7537 4768 : Py_XDECREF(pkgname_or_unknown);
7538 4768 : Py_XDECREF(pkgpath);
7539 4768 : return NULL;
7540 : }
7541 :
7542 : static int
7543 41176 : import_all_from(PyThreadState *tstate, PyObject *locals, PyObject *v)
7544 : {
7545 : PyObject *all, *dict, *name, *value;
7546 41176 : int skip_leading_underscores = 0;
7547 : int pos, err;
7548 :
7549 41176 : if (_PyObject_LookupAttr(v, &_Py_ID(__all__), &all) < 0) {
7550 0 : return -1; /* Unexpected error */
7551 : }
7552 41176 : if (all == NULL) {
7553 26037 : if (_PyObject_LookupAttr(v, &_Py_ID(__dict__), &dict) < 0) {
7554 0 : return -1;
7555 : }
7556 26037 : if (dict == NULL) {
7557 0 : _PyErr_SetString(tstate, PyExc_ImportError,
7558 : "from-import-* object has no __dict__ and no __all__");
7559 0 : return -1;
7560 : }
7561 26037 : all = PyMapping_Keys(dict);
7562 26037 : Py_DECREF(dict);
7563 26037 : if (all == NULL)
7564 0 : return -1;
7565 26037 : skip_leading_underscores = 1;
7566 : }
7567 :
7568 41176 : for (pos = 0, err = 0; ; pos++) {
7569 3040380 : name = PySequence_GetItem(all, pos);
7570 3040380 : if (name == NULL) {
7571 41174 : if (!_PyErr_ExceptionMatches(tstate, PyExc_IndexError)) {
7572 0 : err = -1;
7573 : }
7574 : else {
7575 41174 : _PyErr_Clear(tstate);
7576 : }
7577 41174 : break;
7578 : }
7579 2999200 : if (!PyUnicode_Check(name)) {
7580 2 : PyObject *modname = PyObject_GetAttr(v, &_Py_ID(__name__));
7581 2 : if (modname == NULL) {
7582 0 : Py_DECREF(name);
7583 0 : err = -1;
7584 0 : break;
7585 : }
7586 2 : if (!PyUnicode_Check(modname)) {
7587 0 : _PyErr_Format(tstate, PyExc_TypeError,
7588 : "module __name__ must be a string, not %.100s",
7589 0 : Py_TYPE(modname)->tp_name);
7590 : }
7591 : else {
7592 2 : _PyErr_Format(tstate, PyExc_TypeError,
7593 : "%s in %U.%s must be str, not %.100s",
7594 : skip_leading_underscores ? "Key" : "Item",
7595 : modname,
7596 : skip_leading_underscores ? "__dict__" : "__all__",
7597 2 : Py_TYPE(name)->tp_name);
7598 : }
7599 2 : Py_DECREF(modname);
7600 2 : Py_DECREF(name);
7601 2 : err = -1;
7602 2 : break;
7603 : }
7604 2999200 : if (skip_leading_underscores) {
7605 2749890 : if (PyUnicode_READY(name) == -1) {
7606 0 : Py_DECREF(name);
7607 0 : err = -1;
7608 0 : break;
7609 : }
7610 2749890 : if (PyUnicode_READ_CHAR(name, 0) == '_') {
7611 190027 : Py_DECREF(name);
7612 190027 : continue;
7613 : }
7614 : }
7615 2809180 : value = PyObject_GetAttr(v, name);
7616 2809180 : if (value == NULL)
7617 0 : err = -1;
7618 2809180 : else if (PyDict_CheckExact(locals))
7619 2809180 : err = PyDict_SetItem(locals, name, value);
7620 : else
7621 0 : err = PyObject_SetItem(locals, name, value);
7622 2809180 : Py_DECREF(name);
7623 2809180 : Py_XDECREF(value);
7624 2809180 : if (err != 0)
7625 0 : break;
7626 : }
7627 41176 : Py_DECREF(all);
7628 41176 : return err;
7629 : }
7630 :
7631 : #define CANNOT_CATCH_MSG "catching classes that do not inherit from "\
7632 : "BaseException is not allowed"
7633 :
7634 : #define CANNOT_EXCEPT_STAR_EG "catching ExceptionGroup with except* "\
7635 : "is not allowed. Use except instead."
7636 :
7637 : static int
7638 4606880 : check_except_type_valid(PyThreadState *tstate, PyObject* right)
7639 : {
7640 4606880 : if (PyTuple_Check(right)) {
7641 : Py_ssize_t i, length;
7642 177888 : length = PyTuple_GET_SIZE(right);
7643 551541 : for (i = 0; i < length; i++) {
7644 373659 : PyObject *exc = PyTuple_GET_ITEM(right, i);
7645 373659 : if (!PyExceptionClass_Check(exc)) {
7646 6 : _PyErr_SetString(tstate, PyExc_TypeError,
7647 : CANNOT_CATCH_MSG);
7648 6 : return -1;
7649 : }
7650 : }
7651 : }
7652 : else {
7653 4429000 : if (!PyExceptionClass_Check(right)) {
7654 6 : _PyErr_SetString(tstate, PyExc_TypeError,
7655 : CANNOT_CATCH_MSG);
7656 6 : return -1;
7657 : }
7658 : }
7659 4606870 : return 0;
7660 : }
7661 :
7662 : static int
7663 179 : check_except_star_type_valid(PyThreadState *tstate, PyObject* right)
7664 : {
7665 179 : if (check_except_type_valid(tstate, right) < 0) {
7666 2 : return -1;
7667 : }
7668 :
7669 : /* reject except *ExceptionGroup */
7670 :
7671 177 : int is_subclass = 0;
7672 177 : if (PyTuple_Check(right)) {
7673 13 : Py_ssize_t length = PyTuple_GET_SIZE(right);
7674 40 : for (Py_ssize_t i = 0; i < length; i++) {
7675 28 : PyObject *exc = PyTuple_GET_ITEM(right, i);
7676 28 : is_subclass = PyObject_IsSubclass(exc, PyExc_BaseExceptionGroup);
7677 28 : if (is_subclass < 0) {
7678 0 : return -1;
7679 : }
7680 28 : if (is_subclass) {
7681 1 : break;
7682 : }
7683 : }
7684 : }
7685 : else {
7686 164 : is_subclass = PyObject_IsSubclass(right, PyExc_BaseExceptionGroup);
7687 164 : if (is_subclass < 0) {
7688 0 : return -1;
7689 : }
7690 : }
7691 177 : if (is_subclass) {
7692 2 : _PyErr_SetString(tstate, PyExc_TypeError,
7693 : CANNOT_EXCEPT_STAR_EG);
7694 2 : return -1;
7695 : }
7696 175 : return 0;
7697 : }
7698 :
7699 : static int
7700 363037 : check_args_iterable(PyThreadState *tstate, PyObject *func, PyObject *args)
7701 : {
7702 363037 : if (Py_TYPE(args)->tp_iter == NULL && !PySequence_Check(args)) {
7703 : /* check_args_iterable() may be called with a live exception:
7704 : * clear it to prevent calling _PyObject_FunctionStr() with an
7705 : * exception set. */
7706 6 : _PyErr_Clear(tstate);
7707 6 : PyObject *funcstr = _PyObject_FunctionStr(func);
7708 6 : if (funcstr != NULL) {
7709 6 : _PyErr_Format(tstate, PyExc_TypeError,
7710 : "%U argument after * must be an iterable, not %.200s",
7711 6 : funcstr, Py_TYPE(args)->tp_name);
7712 6 : Py_DECREF(funcstr);
7713 : }
7714 6 : return -1;
7715 : }
7716 363031 : return 0;
7717 : }
7718 :
7719 : static void
7720 23 : format_kwargs_error(PyThreadState *tstate, PyObject *func, PyObject *kwargs)
7721 : {
7722 : /* _PyDict_MergeEx raises attribute
7723 : * error (percolated from an attempt
7724 : * to get 'keys' attribute) instead of
7725 : * a type error if its second argument
7726 : * is not a mapping.
7727 : */
7728 23 : if (_PyErr_ExceptionMatches(tstate, PyExc_AttributeError)) {
7729 9 : _PyErr_Clear(tstate);
7730 9 : PyObject *funcstr = _PyObject_FunctionStr(func);
7731 9 : if (funcstr != NULL) {
7732 9 : _PyErr_Format(
7733 : tstate, PyExc_TypeError,
7734 : "%U argument after ** must be a mapping, not %.200s",
7735 9 : funcstr, Py_TYPE(kwargs)->tp_name);
7736 9 : Py_DECREF(funcstr);
7737 : }
7738 : }
7739 14 : else if (_PyErr_ExceptionMatches(tstate, PyExc_KeyError)) {
7740 : PyObject *exc, *val, *tb;
7741 13 : _PyErr_Fetch(tstate, &exc, &val, &tb);
7742 26 : if (val && PyTuple_Check(val) && PyTuple_GET_SIZE(val) == 1) {
7743 13 : _PyErr_Clear(tstate);
7744 13 : PyObject *funcstr = _PyObject_FunctionStr(func);
7745 13 : if (funcstr != NULL) {
7746 13 : PyObject *key = PyTuple_GET_ITEM(val, 0);
7747 13 : _PyErr_Format(
7748 : tstate, PyExc_TypeError,
7749 : "%U got multiple values for keyword argument '%S'",
7750 : funcstr, key);
7751 13 : Py_DECREF(funcstr);
7752 : }
7753 13 : Py_XDECREF(exc);
7754 13 : Py_XDECREF(val);
7755 13 : Py_XDECREF(tb);
7756 : }
7757 : else {
7758 0 : _PyErr_Restore(tstate, exc, val, tb);
7759 : }
7760 : }
7761 23 : }
7762 :
7763 : static void
7764 966 : format_exc_check_arg(PyThreadState *tstate, PyObject *exc,
7765 : const char *format_str, PyObject *obj)
7766 : {
7767 : const char *obj_str;
7768 :
7769 966 : if (!obj)
7770 0 : return;
7771 :
7772 966 : obj_str = PyUnicode_AsUTF8(obj);
7773 966 : if (!obj_str)
7774 0 : return;
7775 :
7776 966 : _PyErr_Format(tstate, exc, format_str, obj_str);
7777 :
7778 966 : if (exc == PyExc_NameError) {
7779 : // Include the name in the NameError exceptions to offer suggestions later.
7780 : PyObject *type, *value, *traceback;
7781 957 : PyErr_Fetch(&type, &value, &traceback);
7782 957 : PyErr_NormalizeException(&type, &value, &traceback);
7783 957 : if (PyErr_GivenExceptionMatches(value, PyExc_NameError)) {
7784 957 : PyNameErrorObject* exc = (PyNameErrorObject*) value;
7785 957 : if (exc->name == NULL) {
7786 : // We do not care if this fails because we are going to restore the
7787 : // NameError anyway.
7788 957 : (void)PyObject_SetAttr(value, &_Py_ID(name), obj);
7789 : }
7790 : }
7791 957 : PyErr_Restore(type, value, traceback);
7792 : }
7793 : }
7794 :
7795 : static void
7796 8 : format_exc_unbound(PyThreadState *tstate, PyCodeObject *co, int oparg)
7797 : {
7798 : PyObject *name;
7799 : /* Don't stomp existing exception */
7800 8 : if (_PyErr_Occurred(tstate))
7801 0 : return;
7802 8 : name = PyTuple_GET_ITEM(co->co_localsplusnames, oparg);
7803 8 : if (oparg < co->co_nplaincellvars + co->co_nlocals) {
7804 3 : format_exc_check_arg(tstate, PyExc_UnboundLocalError,
7805 : UNBOUNDLOCAL_ERROR_MSG, name);
7806 : } else {
7807 5 : format_exc_check_arg(tstate, PyExc_NameError,
7808 : UNBOUNDFREE_ERROR_MSG, name);
7809 : }
7810 : }
7811 :
7812 : static void
7813 28 : format_awaitable_error(PyThreadState *tstate, PyTypeObject *type, int oparg)
7814 : {
7815 28 : if (type->tp_as_async == NULL || type->tp_as_async->am_await == NULL) {
7816 23 : if (oparg == 1) {
7817 1 : _PyErr_Format(tstate, PyExc_TypeError,
7818 : "'async with' received an object from __aenter__ "
7819 : "that does not implement __await__: %.100s",
7820 : type->tp_name);
7821 : }
7822 22 : else if (oparg == 2) {
7823 5 : _PyErr_Format(tstate, PyExc_TypeError,
7824 : "'async with' received an object from __aexit__ "
7825 : "that does not implement __await__: %.100s",
7826 : type->tp_name);
7827 : }
7828 : }
7829 28 : }
7830 :
7831 : #ifdef Py_STATS
7832 :
7833 : static PyObject *
7834 : getarray(uint64_t a[256])
7835 : {
7836 : int i;
7837 : PyObject *l = PyList_New(256);
7838 : if (l == NULL) return NULL;
7839 : for (i = 0; i < 256; i++) {
7840 : PyObject *x = PyLong_FromUnsignedLongLong(a[i]);
7841 : if (x == NULL) {
7842 : Py_DECREF(l);
7843 : return NULL;
7844 : }
7845 : PyList_SET_ITEM(l, i, x);
7846 : }
7847 : for (i = 0; i < 256; i++)
7848 : a[i] = 0;
7849 : return l;
7850 : }
7851 :
7852 : PyObject *
7853 : _Py_GetDXProfile(PyObject *self, PyObject *args)
7854 : {
7855 : int i;
7856 : PyObject *l = PyList_New(257);
7857 : if (l == NULL) return NULL;
7858 : for (i = 0; i < 256; i++) {
7859 : PyObject *x = getarray(_py_stats_struct.opcode_stats[i].pair_count);
7860 : if (x == NULL) {
7861 : Py_DECREF(l);
7862 : return NULL;
7863 : }
7864 : PyList_SET_ITEM(l, i, x);
7865 : }
7866 : PyObject *counts = PyList_New(256);
7867 : if (counts == NULL) {
7868 : Py_DECREF(l);
7869 : return NULL;
7870 : }
7871 : for (i = 0; i < 256; i++) {
7872 : PyObject *x = PyLong_FromUnsignedLongLong(
7873 : _py_stats_struct.opcode_stats[i].execution_count);
7874 : if (x == NULL) {
7875 : Py_DECREF(counts);
7876 : Py_DECREF(l);
7877 : return NULL;
7878 : }
7879 : PyList_SET_ITEM(counts, i, x);
7880 : }
7881 : PyList_SET_ITEM(l, 256, counts);
7882 : return l;
7883 : }
7884 :
7885 : #endif
7886 :
7887 : Py_ssize_t
7888 1 : _PyEval_RequestCodeExtraIndex(freefunc free)
7889 : {
7890 1 : PyInterpreterState *interp = _PyInterpreterState_GET();
7891 : Py_ssize_t new_index;
7892 :
7893 1 : if (interp->co_extra_user_count == MAX_CO_EXTRA_USERS - 1) {
7894 0 : return -1;
7895 : }
7896 1 : new_index = interp->co_extra_user_count++;
7897 1 : interp->co_extra_freefuncs[new_index] = free;
7898 1 : return new_index;
7899 : }
7900 :
7901 : static void
7902 0 : dtrace_function_entry(_PyInterpreterFrame *frame)
7903 : {
7904 : const char *filename;
7905 : const char *funcname;
7906 : int lineno;
7907 :
7908 0 : PyCodeObject *code = frame->f_code;
7909 0 : filename = PyUnicode_AsUTF8(code->co_filename);
7910 0 : funcname = PyUnicode_AsUTF8(code->co_name);
7911 0 : lineno = _PyInterpreterFrame_GetLine(frame);
7912 :
7913 0 : PyDTrace_FUNCTION_ENTRY(filename, funcname, lineno);
7914 0 : }
7915 :
7916 : static void
7917 0 : dtrace_function_return(_PyInterpreterFrame *frame)
7918 : {
7919 : const char *filename;
7920 : const char *funcname;
7921 : int lineno;
7922 :
7923 0 : PyCodeObject *code = frame->f_code;
7924 0 : filename = PyUnicode_AsUTF8(code->co_filename);
7925 0 : funcname = PyUnicode_AsUTF8(code->co_name);
7926 0 : lineno = _PyInterpreterFrame_GetLine(frame);
7927 :
7928 0 : PyDTrace_FUNCTION_RETURN(filename, funcname, lineno);
7929 0 : }
7930 :
7931 : /* DTrace equivalent of maybe_call_line_trace. */
7932 : static void
7933 0 : maybe_dtrace_line(_PyInterpreterFrame *frame,
7934 : PyTraceInfo *trace_info,
7935 : int instr_prev)
7936 : {
7937 : const char *co_filename, *co_name;
7938 :
7939 : /* If the last instruction executed isn't in the current
7940 : instruction window, reset the window.
7941 : */
7942 0 : initialize_trace_info(trace_info, frame);
7943 0 : int lastline = _PyCode_CheckLineNumber(instr_prev*sizeof(_Py_CODEUNIT), &trace_info->bounds);
7944 0 : int addr = _PyInterpreterFrame_LASTI(frame) * sizeof(_Py_CODEUNIT);
7945 0 : int line = _PyCode_CheckLineNumber(addr, &trace_info->bounds);
7946 0 : if (line != -1) {
7947 : /* Trace backward edges or first instruction of a new line */
7948 0 : if (_PyInterpreterFrame_LASTI(frame) < instr_prev ||
7949 0 : (line != lastline && addr == trace_info->bounds.ar_start))
7950 : {
7951 0 : co_filename = PyUnicode_AsUTF8(frame->f_code->co_filename);
7952 0 : if (!co_filename) {
7953 0 : co_filename = "?";
7954 : }
7955 0 : co_name = PyUnicode_AsUTF8(frame->f_code->co_name);
7956 0 : if (!co_name) {
7957 0 : co_name = "?";
7958 : }
7959 0 : PyDTrace_LINE(co_filename, co_name, line);
7960 : }
7961 : }
7962 0 : }
7963 :
7964 : /* Implement Py_EnterRecursiveCall() and Py_LeaveRecursiveCall() as functions
7965 : for the limited API. */
7966 :
7967 : #undef Py_EnterRecursiveCall
7968 :
7969 0 : int Py_EnterRecursiveCall(const char *where)
7970 : {
7971 0 : return _Py_EnterRecursiveCall(where);
7972 : }
7973 :
7974 : #undef Py_LeaveRecursiveCall
7975 :
7976 0 : void Py_LeaveRecursiveCall(void)
7977 : {
7978 0 : _Py_LeaveRecursiveCall();
7979 0 : }
|
