Line data Source code
1 : #include "Python.h"
2 : #include "pycore_ast.h" // identifier, stmt_ty
3 : #include "pycore_compile.h" // _Py_Mangle(), _PyFuture_FromAST()
4 : #include "pycore_parser.h" // _PyParser_ASTFromString()
5 : #include "pycore_pystate.h" // _PyThreadState_GET()
6 : #include "pycore_symtable.h" // PySTEntryObject
7 : #include "structmember.h" // PyMemberDef
8 :
9 : /* error strings used for warnings */
10 : #define GLOBAL_PARAM \
11 : "name '%U' is parameter and global"
12 :
13 : #define NONLOCAL_PARAM \
14 : "name '%U' is parameter and nonlocal"
15 :
16 : #define GLOBAL_AFTER_ASSIGN \
17 : "name '%U' is assigned to before global declaration"
18 :
19 : #define NONLOCAL_AFTER_ASSIGN \
20 : "name '%U' is assigned to before nonlocal declaration"
21 :
22 : #define GLOBAL_AFTER_USE \
23 : "name '%U' is used prior to global declaration"
24 :
25 : #define NONLOCAL_AFTER_USE \
26 : "name '%U' is used prior to nonlocal declaration"
27 :
28 : #define GLOBAL_ANNOT \
29 : "annotated name '%U' can't be global"
30 :
31 : #define NONLOCAL_ANNOT \
32 : "annotated name '%U' can't be nonlocal"
33 :
34 : #define IMPORT_STAR_WARNING "import * only allowed at module level"
35 :
36 : #define NAMED_EXPR_COMP_IN_CLASS \
37 : "assignment expression within a comprehension cannot be used in a class body"
38 :
39 : #define NAMED_EXPR_COMP_CONFLICT \
40 : "assignment expression cannot rebind comprehension iteration variable '%U'"
41 :
42 : #define NAMED_EXPR_COMP_INNER_LOOP_CONFLICT \
43 : "comprehension inner loop cannot rebind assignment expression target '%U'"
44 :
45 : #define NAMED_EXPR_COMP_ITER_EXPR \
46 : "assignment expression cannot be used in a comprehension iterable expression"
47 :
48 : #define ANNOTATION_NOT_ALLOWED \
49 : "'%s' can not be used within an annotation"
50 :
51 :
52 : #define LOCATION(x) \
53 : (x)->lineno, (x)->col_offset, (x)->end_lineno, (x)->end_col_offset
54 :
55 : #define ST_LOCATION(x) \
56 : (x)->ste_lineno, (x)->ste_col_offset, (x)->ste_end_lineno, (x)->ste_end_col_offset
57 :
58 : static PySTEntryObject *
59 470536 : ste_new(struct symtable *st, identifier name, _Py_block_ty block,
60 : void *key, int lineno, int col_offset,
61 : int end_lineno, int end_col_offset)
62 : {
63 470536 : PySTEntryObject *ste = NULL;
64 470536 : PyObject *k = NULL;
65 :
66 470536 : k = PyLong_FromVoidPtr(key);
67 470536 : if (k == NULL)
68 0 : goto fail;
69 470536 : ste = PyObject_New(PySTEntryObject, &PySTEntry_Type);
70 470536 : if (ste == NULL) {
71 0 : Py_DECREF(k);
72 0 : goto fail;
73 : }
74 470536 : ste->ste_table = st;
75 470536 : ste->ste_id = k; /* ste owns reference to k */
76 :
77 470536 : Py_INCREF(name);
78 470536 : ste->ste_name = name;
79 :
80 470536 : ste->ste_symbols = NULL;
81 470536 : ste->ste_varnames = NULL;
82 470536 : ste->ste_children = NULL;
83 :
84 470536 : ste->ste_directives = NULL;
85 :
86 470536 : ste->ste_type = block;
87 470536 : ste->ste_nested = 0;
88 470536 : ste->ste_free = 0;
89 470536 : ste->ste_varargs = 0;
90 470536 : ste->ste_varkeywords = 0;
91 470536 : ste->ste_opt_lineno = 0;
92 470536 : ste->ste_opt_col_offset = 0;
93 470536 : ste->ste_lineno = lineno;
94 470536 : ste->ste_col_offset = col_offset;
95 470536 : ste->ste_end_lineno = end_lineno;
96 470536 : ste->ste_end_col_offset = end_col_offset;
97 :
98 470536 : if (st->st_cur != NULL &&
99 355845 : (st->st_cur->ste_nested ||
100 346044 : st->st_cur->ste_type == FunctionBlock))
101 47663 : ste->ste_nested = 1;
102 470536 : ste->ste_child_free = 0;
103 470536 : ste->ste_generator = 0;
104 470536 : ste->ste_coroutine = 0;
105 470536 : ste->ste_comprehension = NoComprehension;
106 470536 : ste->ste_returns_value = 0;
107 470536 : ste->ste_needs_class_closure = 0;
108 470536 : ste->ste_comp_iter_target = 0;
109 470536 : ste->ste_comp_iter_expr = 0;
110 :
111 470536 : ste->ste_symbols = PyDict_New();
112 470536 : ste->ste_varnames = PyList_New(0);
113 470536 : ste->ste_children = PyList_New(0);
114 470536 : if (ste->ste_symbols == NULL
115 470536 : || ste->ste_varnames == NULL
116 470536 : || ste->ste_children == NULL)
117 0 : goto fail;
118 :
119 470536 : if (PyDict_SetItem(st->st_blocks, ste->ste_id, (PyObject *)ste) < 0)
120 0 : goto fail;
121 :
122 470536 : return ste;
123 0 : fail:
124 0 : Py_XDECREF(ste);
125 0 : return NULL;
126 : }
127 :
128 : static PyObject *
129 0 : ste_repr(PySTEntryObject *ste)
130 : {
131 0 : return PyUnicode_FromFormat("<symtable entry %U(%ld), line %d>",
132 : ste->ste_name,
133 : PyLong_AS_LONG(ste->ste_id), ste->ste_lineno);
134 : }
135 :
136 : static void
137 470536 : ste_dealloc(PySTEntryObject *ste)
138 : {
139 470536 : ste->ste_table = NULL;
140 470536 : Py_XDECREF(ste->ste_id);
141 470536 : Py_XDECREF(ste->ste_name);
142 470536 : Py_XDECREF(ste->ste_symbols);
143 470536 : Py_XDECREF(ste->ste_varnames);
144 470536 : Py_XDECREF(ste->ste_children);
145 470536 : Py_XDECREF(ste->ste_directives);
146 470536 : PyObject_Free(ste);
147 470536 : }
148 :
149 : #define OFF(x) offsetof(PySTEntryObject, x)
150 :
151 : static PyMemberDef ste_memberlist[] = {
152 : {"id", T_OBJECT, OFF(ste_id), READONLY},
153 : {"name", T_OBJECT, OFF(ste_name), READONLY},
154 : {"symbols", T_OBJECT, OFF(ste_symbols), READONLY},
155 : {"varnames", T_OBJECT, OFF(ste_varnames), READONLY},
156 : {"children", T_OBJECT, OFF(ste_children), READONLY},
157 : {"nested", T_INT, OFF(ste_nested), READONLY},
158 : {"type", T_INT, OFF(ste_type), READONLY},
159 : {"lineno", T_INT, OFF(ste_lineno), READONLY},
160 : {NULL}
161 : };
162 :
163 : PyTypeObject PySTEntry_Type = {
164 : PyVarObject_HEAD_INIT(&PyType_Type, 0)
165 : "symtable entry",
166 : sizeof(PySTEntryObject),
167 : 0,
168 : (destructor)ste_dealloc, /* tp_dealloc */
169 : 0, /* tp_vectorcall_offset */
170 : 0, /* tp_getattr */
171 : 0, /* tp_setattr */
172 : 0, /* tp_as_async */
173 : (reprfunc)ste_repr, /* tp_repr */
174 : 0, /* tp_as_number */
175 : 0, /* tp_as_sequence */
176 : 0, /* tp_as_mapping */
177 : 0, /* tp_hash */
178 : 0, /* tp_call */
179 : 0, /* tp_str */
180 : PyObject_GenericGetAttr, /* tp_getattro */
181 : 0, /* tp_setattro */
182 : 0, /* tp_as_buffer */
183 : Py_TPFLAGS_DEFAULT, /* tp_flags */
184 : 0, /* tp_doc */
185 : 0, /* tp_traverse */
186 : 0, /* tp_clear */
187 : 0, /* tp_richcompare */
188 : 0, /* tp_weaklistoffset */
189 : 0, /* tp_iter */
190 : 0, /* tp_iternext */
191 : 0, /* tp_methods */
192 : ste_memberlist, /* tp_members */
193 : 0, /* tp_getset */
194 : 0, /* tp_base */
195 : 0, /* tp_dict */
196 : 0, /* tp_descr_get */
197 : 0, /* tp_descr_set */
198 : 0, /* tp_dictoffset */
199 : 0, /* tp_init */
200 : 0, /* tp_alloc */
201 : 0, /* tp_new */
202 : };
203 :
204 : static int symtable_analyze(struct symtable *st);
205 : static int symtable_enter_block(struct symtable *st, identifier name,
206 : _Py_block_ty block, void *ast,
207 : int lineno, int col_offset,
208 : int end_lineno, int end_col_offset);
209 : static int symtable_exit_block(struct symtable *st);
210 : static int symtable_visit_stmt(struct symtable *st, stmt_ty s);
211 : static int symtable_visit_expr(struct symtable *st, expr_ty s);
212 : static int symtable_visit_genexp(struct symtable *st, expr_ty s);
213 : static int symtable_visit_listcomp(struct symtable *st, expr_ty s);
214 : static int symtable_visit_setcomp(struct symtable *st, expr_ty s);
215 : static int symtable_visit_dictcomp(struct symtable *st, expr_ty s);
216 : static int symtable_visit_arguments(struct symtable *st, arguments_ty);
217 : static int symtable_visit_excepthandler(struct symtable *st, excepthandler_ty);
218 : static int symtable_visit_alias(struct symtable *st, alias_ty);
219 : static int symtable_visit_comprehension(struct symtable *st, comprehension_ty);
220 : static int symtable_visit_keyword(struct symtable *st, keyword_ty);
221 : static int symtable_visit_params(struct symtable *st, asdl_arg_seq *args);
222 : static int symtable_visit_annotation(struct symtable *st, expr_ty annotation);
223 : static int symtable_visit_argannotations(struct symtable *st, asdl_arg_seq *args);
224 : static int symtable_implicit_arg(struct symtable *st, int pos);
225 : static int symtable_visit_annotations(struct symtable *st, stmt_ty, arguments_ty, expr_ty);
226 : static int symtable_visit_withitem(struct symtable *st, withitem_ty item);
227 : static int symtable_visit_match_case(struct symtable *st, match_case_ty m);
228 : static int symtable_visit_pattern(struct symtable *st, pattern_ty s);
229 : static int symtable_raise_if_annotation_block(struct symtable *st, const char *, expr_ty);
230 : static int symtable_raise_if_comprehension_block(struct symtable *st, expr_ty);
231 :
232 :
233 : #define DUPLICATE_ARGUMENT \
234 : "duplicate argument '%U' in function definition"
235 :
236 : static struct symtable *
237 114691 : symtable_new(void)
238 : {
239 : struct symtable *st;
240 :
241 114691 : st = (struct symtable *)PyMem_Malloc(sizeof(struct symtable));
242 114691 : if (st == NULL) {
243 0 : PyErr_NoMemory();
244 0 : return NULL;
245 : }
246 :
247 114691 : st->st_filename = NULL;
248 114691 : st->st_blocks = NULL;
249 :
250 114691 : if ((st->st_stack = PyList_New(0)) == NULL)
251 0 : goto fail;
252 114691 : if ((st->st_blocks = PyDict_New()) == NULL)
253 0 : goto fail;
254 114691 : st->st_cur = NULL;
255 114691 : st->st_private = NULL;
256 114691 : return st;
257 0 : fail:
258 0 : _PySymtable_Free(st);
259 0 : return NULL;
260 : }
261 :
262 : /* When compiling the use of C stack is probably going to be a lot
263 : lighter than when executing Python code but still can overflow
264 : and causing a Python crash if not checked (e.g. eval("()"*300000)).
265 : Using the current recursion limit for the compiler seems too
266 : restrictive (it caused at least one test to fail) so a factor is
267 : used to allow deeper recursion when compiling an expression.
268 :
269 : Using a scaling factor means this should automatically adjust when
270 : the recursion limit is adjusted for small or large C stack allocations.
271 : */
272 : #define COMPILER_STACK_FRAME_SCALE 3
273 :
274 : struct symtable *
275 114691 : _PySymtable_Build(mod_ty mod, PyObject *filename, PyFutureFeatures *future)
276 : {
277 114691 : struct symtable *st = symtable_new();
278 : asdl_stmt_seq *seq;
279 : int i;
280 : PyThreadState *tstate;
281 114691 : int recursion_limit = Py_GetRecursionLimit();
282 : int starting_recursion_depth;
283 :
284 114691 : if (st == NULL)
285 0 : return NULL;
286 114691 : if (filename == NULL) {
287 0 : _PySymtable_Free(st);
288 0 : return NULL;
289 : }
290 114691 : Py_INCREF(filename);
291 114691 : st->st_filename = filename;
292 114691 : st->st_future = future;
293 :
294 : /* Setup recursion depth check counters */
295 114691 : tstate = _PyThreadState_GET();
296 114691 : if (!tstate) {
297 0 : _PySymtable_Free(st);
298 0 : return NULL;
299 : }
300 : /* Be careful here to prevent overflow. */
301 114691 : int recursion_depth = tstate->recursion_limit - tstate->recursion_remaining;
302 114691 : starting_recursion_depth = (recursion_depth < INT_MAX / COMPILER_STACK_FRAME_SCALE) ?
303 114691 : recursion_depth * COMPILER_STACK_FRAME_SCALE : recursion_depth;
304 114691 : st->recursion_depth = starting_recursion_depth;
305 114691 : st->recursion_limit = (recursion_limit < INT_MAX / COMPILER_STACK_FRAME_SCALE) ?
306 114691 : recursion_limit * COMPILER_STACK_FRAME_SCALE : recursion_limit;
307 :
308 : /* Make the initial symbol information gathering pass */
309 114691 : if (!symtable_enter_block(st, &_Py_ID(top), ModuleBlock, (void *)mod, 0, 0, 0, 0)) {
310 0 : _PySymtable_Free(st);
311 0 : return NULL;
312 : }
313 :
314 114691 : st->st_top = st->st_cur;
315 114691 : switch (mod->kind) {
316 38120 : case Module_kind:
317 38120 : seq = mod->v.Module.body;
318 489593 : for (i = 0; i < asdl_seq_LEN(seq); i++)
319 451606 : if (!symtable_visit_stmt(st,
320 451606 : (stmt_ty)asdl_seq_GET(seq, i)))
321 133 : goto error;
322 37987 : break;
323 72424 : case Expression_kind:
324 72424 : if (!symtable_visit_expr(st, mod->v.Expression.body))
325 3 : goto error;
326 72421 : break;
327 4147 : case Interactive_kind:
328 4147 : seq = mod->v.Interactive.body;
329 8344 : for (i = 0; i < asdl_seq_LEN(seq); i++)
330 4208 : if (!symtable_visit_stmt(st,
331 4208 : (stmt_ty)asdl_seq_GET(seq, i)))
332 11 : goto error;
333 4136 : break;
334 0 : case FunctionType_kind:
335 0 : PyErr_SetString(PyExc_RuntimeError,
336 : "this compiler does not handle FunctionTypes");
337 0 : goto error;
338 : }
339 114544 : if (!symtable_exit_block(st)) {
340 0 : _PySymtable_Free(st);
341 0 : return NULL;
342 : }
343 : /* Check that the recursion depth counting balanced correctly */
344 114544 : if (st->recursion_depth != starting_recursion_depth) {
345 0 : PyErr_Format(PyExc_SystemError,
346 : "symtable analysis recursion depth mismatch (before=%d, after=%d)",
347 : starting_recursion_depth, st->recursion_depth);
348 0 : _PySymtable_Free(st);
349 0 : return NULL;
350 : }
351 : /* Make the second symbol analysis pass */
352 114544 : if (symtable_analyze(st))
353 114538 : return st;
354 6 : _PySymtable_Free(st);
355 6 : return NULL;
356 147 : error:
357 147 : (void) symtable_exit_block(st);
358 147 : _PySymtable_Free(st);
359 147 : return NULL;
360 : }
361 :
362 :
363 : void
364 114691 : _PySymtable_Free(struct symtable *st)
365 : {
366 114691 : Py_XDECREF(st->st_filename);
367 114691 : Py_XDECREF(st->st_blocks);
368 114691 : Py_XDECREF(st->st_stack);
369 114691 : PyMem_Free((void *)st);
370 114691 : }
371 :
372 : PySTEntryObject *
373 465532 : PySymtable_Lookup(struct symtable *st, void *key)
374 : {
375 : PyObject *k, *v;
376 :
377 465532 : k = PyLong_FromVoidPtr(key);
378 465532 : if (k == NULL)
379 0 : return NULL;
380 465532 : v = PyDict_GetItemWithError(st->st_blocks, k);
381 465532 : if (v) {
382 465532 : assert(PySTEntry_Check(v));
383 465532 : Py_INCREF(v);
384 : }
385 0 : else if (!PyErr_Occurred()) {
386 0 : PyErr_SetString(PyExc_KeyError,
387 : "unknown symbol table entry");
388 : }
389 :
390 465532 : Py_DECREF(k);
391 465532 : return (PySTEntryObject *)v;
392 : }
393 :
394 : long
395 6002690 : _PyST_GetSymbol(PySTEntryObject *ste, PyObject *name)
396 : {
397 6002690 : PyObject *v = PyDict_GetItemWithError(ste->ste_symbols, name);
398 6002690 : if (!v)
399 514740 : return 0;
400 5487950 : assert(PyLong_Check(v));
401 5487950 : return PyLong_AS_LONG(v);
402 : }
403 :
404 : int
405 5501800 : _PyST_GetScope(PySTEntryObject *ste, PyObject *name)
406 : {
407 5501800 : long symbol = _PyST_GetSymbol(ste, name);
408 5501800 : return (symbol >> SCOPE_OFFSET) & SCOPE_MASK;
409 : }
410 :
411 : static int
412 6 : error_at_directive(PySTEntryObject *ste, PyObject *name)
413 : {
414 : Py_ssize_t i;
415 : PyObject *data;
416 6 : assert(ste->ste_directives);
417 6 : for (i = 0; i < PyList_GET_SIZE(ste->ste_directives); i++) {
418 6 : data = PyList_GET_ITEM(ste->ste_directives, i);
419 6 : assert(PyTuple_CheckExact(data));
420 6 : assert(PyUnicode_CheckExact(PyTuple_GET_ITEM(data, 0)));
421 6 : if (PyUnicode_Compare(PyTuple_GET_ITEM(data, 0), name) == 0) {
422 6 : PyErr_RangedSyntaxLocationObject(ste->ste_table->st_filename,
423 6 : PyLong_AsLong(PyTuple_GET_ITEM(data, 1)),
424 6 : PyLong_AsLong(PyTuple_GET_ITEM(data, 2)) + 1,
425 6 : PyLong_AsLong(PyTuple_GET_ITEM(data, 3)),
426 6 : PyLong_AsLong(PyTuple_GET_ITEM(data, 4)) + 1);
427 :
428 6 : return 0;
429 : }
430 : }
431 0 : PyErr_SetString(PyExc_RuntimeError,
432 : "BUG: internal directive bookkeeping broken");
433 0 : return 0;
434 : }
435 :
436 :
437 : /* Analyze raw symbol information to determine scope of each name.
438 :
439 : The next several functions are helpers for symtable_analyze(),
440 : which determines whether a name is local, global, or free. In addition,
441 : it determines which local variables are cell variables; they provide
442 : bindings that are used for free variables in enclosed blocks.
443 :
444 : There are also two kinds of global variables, implicit and explicit. An
445 : explicit global is declared with the global statement. An implicit
446 : global is a free variable for which the compiler has found no binding
447 : in an enclosing function scope. The implicit global is either a global
448 : or a builtin. Python's module and class blocks use the xxx_NAME opcodes
449 : to handle these names to implement slightly odd semantics. In such a
450 : block, the name is treated as global until it is assigned to; then it
451 : is treated as a local.
452 :
453 : The symbol table requires two passes to determine the scope of each name.
454 : The first pass collects raw facts from the AST via the symtable_visit_*
455 : functions: the name is a parameter here, the name is used but not defined
456 : here, etc. The second pass analyzes these facts during a pass over the
457 : PySTEntryObjects created during pass 1.
458 :
459 : When a function is entered during the second pass, the parent passes
460 : the set of all name bindings visible to its children. These bindings
461 : are used to determine if non-local variables are free or implicit globals.
462 : Names which are explicitly declared nonlocal must exist in this set of
463 : visible names - if they do not, a syntax error is raised. After doing
464 : the local analysis, it analyzes each of its child blocks using an
465 : updated set of name bindings.
466 :
467 : The children update the free variable set. If a local variable is added to
468 : the free variable set by the child, the variable is marked as a cell. The
469 : function object being defined must provide runtime storage for the variable
470 : that may outlive the function's frame. Cell variables are removed from the
471 : free set before the analyze function returns to its parent.
472 :
473 : During analysis, the names are:
474 : symbols: dict mapping from symbol names to flag values (including offset scope values)
475 : scopes: dict mapping from symbol names to scope values (no offset)
476 : local: set of all symbol names local to the current scope
477 : bound: set of all symbol names local to a containing function scope
478 : free: set of all symbol names referenced but not bound in child scopes
479 : global: set of all symbol names explicitly declared as global
480 : */
481 :
482 : #define SET_SCOPE(DICT, NAME, I) { \
483 : PyObject *o = PyLong_FromLong(I); \
484 : if (!o) \
485 : return 0; \
486 : if (PyDict_SetItem((DICT), (NAME), o) < 0) { \
487 : Py_DECREF(o); \
488 : return 0; \
489 : } \
490 : Py_DECREF(o); \
491 : }
492 :
493 : /* Decide on scope of name, given flags.
494 :
495 : The namespace dictionaries may be modified to record information
496 : about the new name. For example, a new global will add an entry to
497 : global. A name that was global can be changed to local.
498 : */
499 :
500 : static int
501 2234850 : analyze_name(PySTEntryObject *ste, PyObject *scopes, PyObject *name, long flags,
502 : PyObject *bound, PyObject *local, PyObject *free,
503 : PyObject *global)
504 : {
505 2234850 : if (flags & DEF_GLOBAL) {
506 3690 : if (flags & DEF_NONLOCAL) {
507 2 : PyErr_Format(PyExc_SyntaxError,
508 : "name '%U' is nonlocal and global",
509 : name);
510 2 : return error_at_directive(ste, name);
511 : }
512 3688 : SET_SCOPE(scopes, name, GLOBAL_EXPLICIT);
513 3688 : if (PySet_Add(global, name) < 0)
514 0 : return 0;
515 3688 : if (bound && (PySet_Discard(bound, name) < 0))
516 0 : return 0;
517 3688 : return 1;
518 : }
519 2231160 : if (flags & DEF_NONLOCAL) {
520 794 : if (!bound) {
521 2 : PyErr_Format(PyExc_SyntaxError,
522 : "nonlocal declaration not allowed at module level");
523 2 : return error_at_directive(ste, name);
524 : }
525 792 : if (!PySet_Contains(bound, name)) {
526 2 : PyErr_Format(PyExc_SyntaxError,
527 : "no binding for nonlocal '%U' found",
528 : name);
529 :
530 2 : return error_at_directive(ste, name);
531 : }
532 790 : SET_SCOPE(scopes, name, FREE);
533 790 : ste->ste_free = 1;
534 790 : return PySet_Add(free, name) >= 0;
535 : }
536 2230360 : if (flags & DEF_BOUND) {
537 1566410 : SET_SCOPE(scopes, name, LOCAL);
538 1566410 : if (PySet_Add(local, name) < 0)
539 0 : return 0;
540 1566410 : if (PySet_Discard(global, name) < 0)
541 0 : return 0;
542 1566410 : return 1;
543 : }
544 : /* If an enclosing block has a binding for this name, it
545 : is a free variable rather than a global variable.
546 : Note that having a non-NULL bound implies that the block
547 : is nested.
548 : */
549 663953 : if (bound && PySet_Contains(bound, name)) {
550 32593 : SET_SCOPE(scopes, name, FREE);
551 32593 : ste->ste_free = 1;
552 32593 : return PySet_Add(free, name) >= 0;
553 : }
554 : /* If a parent has a global statement, then call it global
555 : explicit? It could also be global implicit.
556 : */
557 631360 : if (global && PySet_Contains(global, name)) {
558 2081 : SET_SCOPE(scopes, name, GLOBAL_IMPLICIT);
559 2081 : return 1;
560 : }
561 629279 : if (ste->ste_nested)
562 31197 : ste->ste_free = 1;
563 629279 : SET_SCOPE(scopes, name, GLOBAL_IMPLICIT);
564 629279 : return 1;
565 : }
566 :
567 : #undef SET_SCOPE
568 :
569 : /* If a name is defined in free and also in locals, then this block
570 : provides the binding for the free variable. The name should be
571 : marked CELL in this block and removed from the free list.
572 :
573 : Note that the current block's free variables are included in free.
574 : That's safe because no name can be free and local in the same scope.
575 : */
576 :
577 : static int
578 313315 : analyze_cells(PyObject *scopes, PyObject *free)
579 : {
580 : PyObject *name, *v, *v_cell;
581 313315 : int success = 0;
582 313315 : Py_ssize_t pos = 0;
583 :
584 313315 : v_cell = PyLong_FromLong(CELL);
585 313315 : if (!v_cell)
586 0 : return 0;
587 1966230 : while (PyDict_Next(scopes, &pos, &name, &v)) {
588 : long scope;
589 1652920 : assert(PyLong_Check(v));
590 1652920 : scope = PyLong_AS_LONG(v);
591 1652920 : if (scope != LOCAL)
592 554260 : continue;
593 1098660 : if (!PySet_Contains(free, name))
594 1077050 : continue;
595 : /* Replace LOCAL with CELL for this name, and remove
596 : from free. It is safe to replace the value of name
597 : in the dict, because it will not cause a resize.
598 : */
599 21612 : if (PyDict_SetItem(scopes, name, v_cell) < 0)
600 0 : goto error;
601 21612 : if (PySet_Discard(free, name) < 0)
602 0 : goto error;
603 : }
604 313315 : success = 1;
605 313315 : error:
606 313315 : Py_DECREF(v_cell);
607 313315 : return success;
608 : }
609 :
610 : static int
611 37744 : drop_class_free(PySTEntryObject *ste, PyObject *free)
612 : {
613 : int res;
614 37744 : res = PySet_Discard(free, &_Py_ID(__class__));
615 37744 : if (res < 0)
616 0 : return 0;
617 37744 : if (res)
618 4795 : ste->ste_needs_class_closure = 1;
619 37744 : return 1;
620 : }
621 :
622 : /* Enter the final scope information into the ste_symbols dict.
623 : *
624 : * All arguments are dicts. Modifies symbols, others are read-only.
625 : */
626 : static int
627 465597 : update_symbols(PyObject *symbols, PyObject *scopes,
628 : PyObject *bound, PyObject *free, int classflag)
629 : {
630 465597 : PyObject *name = NULL, *itr = NULL;
631 465597 : PyObject *v = NULL, *v_scope = NULL, *v_new = NULL, *v_free = NULL;
632 465597 : Py_ssize_t pos = 0;
633 :
634 : /* Update scope information for all symbols in this scope */
635 2700430 : while (PyDict_Next(symbols, &pos, &name, &v)) {
636 : long scope, flags;
637 2234830 : assert(PyLong_Check(v));
638 2234830 : flags = PyLong_AS_LONG(v);
639 2234830 : v_scope = PyDict_GetItemWithError(scopes, name);
640 2234830 : assert(v_scope && PyLong_Check(v_scope));
641 2234830 : scope = PyLong_AS_LONG(v_scope);
642 2234830 : flags |= (scope << SCOPE_OFFSET);
643 2234830 : v_new = PyLong_FromLong(flags);
644 2234830 : if (!v_new)
645 0 : return 0;
646 2234830 : if (PyDict_SetItem(symbols, name, v_new) < 0) {
647 0 : Py_DECREF(v_new);
648 0 : return 0;
649 : }
650 2234830 : Py_DECREF(v_new);
651 : }
652 :
653 : /* Record not yet resolved free variables from children (if any) */
654 465597 : v_free = PyLong_FromLong(FREE << SCOPE_OFFSET);
655 465597 : if (!v_free)
656 0 : return 0;
657 :
658 465597 : itr = PyObject_GetIter(free);
659 465597 : if (itr == NULL) {
660 0 : Py_DECREF(v_free);
661 0 : return 0;
662 : }
663 :
664 467413 : while ((name = PyIter_Next(itr))) {
665 1816 : v = PyDict_GetItemWithError(symbols, name);
666 :
667 : /* Handle symbol that already exists in this scope */
668 1816 : if (v) {
669 : /* Handle a free variable in a method of
670 : the class that has the same name as a local
671 : or global in the class scope.
672 : */
673 164 : if (classflag &&
674 20 : PyLong_AS_LONG(v) & (DEF_BOUND | DEF_GLOBAL)) {
675 13 : long flags = PyLong_AS_LONG(v) | DEF_FREE_CLASS;
676 13 : v_new = PyLong_FromLong(flags);
677 13 : if (!v_new) {
678 0 : goto error;
679 : }
680 13 : if (PyDict_SetItem(symbols, name, v_new) < 0) {
681 0 : Py_DECREF(v_new);
682 0 : goto error;
683 : }
684 13 : Py_DECREF(v_new);
685 : }
686 : /* It's a cell, or already free in this scope */
687 164 : Py_DECREF(name);
688 164 : continue;
689 : }
690 1652 : else if (PyErr_Occurred()) {
691 0 : goto error;
692 : }
693 : /* Handle global symbol */
694 1652 : if (bound && !PySet_Contains(bound, name)) {
695 0 : Py_DECREF(name);
696 0 : continue; /* it's a global */
697 : }
698 : /* Propagate new free symbol up the lexical stack */
699 1652 : if (PyDict_SetItem(symbols, name, v_free) < 0) {
700 0 : goto error;
701 : }
702 1652 : Py_DECREF(name);
703 : }
704 465597 : Py_DECREF(itr);
705 465597 : Py_DECREF(v_free);
706 465597 : return 1;
707 0 : error:
708 0 : Py_XDECREF(v_free);
709 0 : Py_XDECREF(itr);
710 0 : Py_XDECREF(name);
711 0 : return 0;
712 : }
713 :
714 : /* Make final symbol table decisions for block of ste.
715 :
716 : Arguments:
717 : ste -- current symtable entry (input/output)
718 : bound -- set of variables bound in enclosing scopes (input). bound
719 : is NULL for module blocks.
720 : free -- set of free variables in enclosed scopes (output)
721 : globals -- set of declared global variables in enclosing scopes (input)
722 :
723 : The implementation uses two mutually recursive functions,
724 : analyze_block() and analyze_child_block(). analyze_block() is
725 : responsible for analyzing the individual names defined in a block.
726 : analyze_child_block() prepares temporary namespace dictionaries
727 : used to evaluated nested blocks.
728 :
729 : The two functions exist because a child block should see the name
730 : bindings of its enclosing blocks, but those bindings should not
731 : propagate back to a parent block.
732 : */
733 :
734 : static int
735 : analyze_child_block(PySTEntryObject *entry, PyObject *bound, PyObject *free,
736 : PyObject *global, PyObject* child_free);
737 :
738 : static int
739 465608 : analyze_block(PySTEntryObject *ste, PyObject *bound, PyObject *free,
740 : PyObject *global)
741 : {
742 465608 : PyObject *name, *v, *local = NULL, *scopes = NULL, *newbound = NULL;
743 465608 : PyObject *newglobal = NULL, *newfree = NULL, *allfree = NULL;
744 : PyObject *temp;
745 465608 : int i, success = 0;
746 465608 : Py_ssize_t pos = 0;
747 :
748 465608 : local = PySet_New(NULL); /* collect new names bound in block */
749 465608 : if (!local)
750 0 : goto error;
751 465608 : scopes = PyDict_New(); /* collect scopes defined for each name */
752 465608 : if (!scopes)
753 0 : goto error;
754 :
755 : /* Allocate new global and bound variable dictionaries. These
756 : dictionaries hold the names visible in nested blocks. For
757 : ClassBlocks, the bound and global names are initialized
758 : before analyzing names, because class bindings aren't
759 : visible in methods. For other blocks, they are initialized
760 : after names are analyzed.
761 : */
762 :
763 : /* TODO(jhylton): Package these dicts in a struct so that we
764 : can write reasonable helper functions?
765 : */
766 465608 : newglobal = PySet_New(NULL);
767 465608 : if (!newglobal)
768 0 : goto error;
769 465608 : newfree = PySet_New(NULL);
770 465608 : if (!newfree)
771 0 : goto error;
772 465608 : newbound = PySet_New(NULL);
773 465608 : if (!newbound)
774 0 : goto error;
775 :
776 : /* Class namespace has no effect on names visible in
777 : nested functions, so populate the global and bound
778 : sets to be passed to child blocks before analyzing
779 : this one.
780 : */
781 465608 : if (ste->ste_type == ClassBlock) {
782 : /* Pass down known globals */
783 37745 : temp = PyNumber_InPlaceOr(newglobal, global);
784 37745 : if (!temp)
785 0 : goto error;
786 37745 : Py_DECREF(temp);
787 : /* Pass down previously bound symbols */
788 37745 : if (bound) {
789 37745 : temp = PyNumber_InPlaceOr(newbound, bound);
790 37745 : if (!temp)
791 0 : goto error;
792 37745 : Py_DECREF(temp);
793 : }
794 : }
795 :
796 2700450 : while (PyDict_Next(ste->ste_symbols, &pos, &name, &v)) {
797 2234850 : long flags = PyLong_AS_LONG(v);
798 2234850 : if (!analyze_name(ste, scopes, name, flags,
799 : bound, local, free, global))
800 6 : goto error;
801 : }
802 :
803 : /* Populate global and bound sets to be passed to children. */
804 465602 : if (ste->ste_type != ClassBlock) {
805 : /* Add function locals to bound set */
806 427857 : if (ste->ste_type == FunctionBlock) {
807 313315 : temp = PyNumber_InPlaceOr(newbound, local);
808 313315 : if (!temp)
809 0 : goto error;
810 313315 : Py_DECREF(temp);
811 : }
812 : /* Pass down previously bound symbols */
813 427857 : if (bound) {
814 313315 : temp = PyNumber_InPlaceOr(newbound, bound);
815 313315 : if (!temp)
816 0 : goto error;
817 313315 : Py_DECREF(temp);
818 : }
819 : /* Pass down known globals */
820 427857 : temp = PyNumber_InPlaceOr(newglobal, global);
821 427857 : if (!temp)
822 0 : goto error;
823 427857 : Py_DECREF(temp);
824 : }
825 : else {
826 : /* Special-case __class__ */
827 37745 : if (PySet_Add(newbound, &_Py_ID(__class__)) < 0)
828 0 : goto error;
829 : }
830 :
831 : /* Recursively call analyze_child_block() on each child block.
832 :
833 : newbound, newglobal now contain the names visible in
834 : nested blocks. The free variables in the children will
835 : be collected in allfree.
836 : */
837 465602 : allfree = PySet_New(NULL);
838 465602 : if (!allfree)
839 0 : goto error;
840 816661 : for (i = 0; i < PyList_GET_SIZE(ste->ste_children); ++i) {
841 351064 : PyObject *c = PyList_GET_ITEM(ste->ste_children, i);
842 : PySTEntryObject* entry;
843 351064 : assert(c && PySTEntry_Check(c));
844 351064 : entry = (PySTEntryObject*)c;
845 351064 : if (!analyze_child_block(entry, newbound, newfree, newglobal,
846 : allfree))
847 5 : goto error;
848 : /* Check if any children have free variables */
849 351059 : if (entry->ste_free || entry->ste_child_free)
850 60560 : ste->ste_child_free = 1;
851 : }
852 :
853 465597 : temp = PyNumber_InPlaceOr(newfree, allfree);
854 465597 : if (!temp)
855 0 : goto error;
856 465597 : Py_DECREF(temp);
857 :
858 : /* Check if any local variables must be converted to cell variables */
859 465597 : if (ste->ste_type == FunctionBlock && !analyze_cells(scopes, newfree))
860 0 : goto error;
861 465597 : else if (ste->ste_type == ClassBlock && !drop_class_free(ste, newfree))
862 0 : goto error;
863 : /* Records the results of the analysis in the symbol table entry */
864 465597 : if (!update_symbols(ste->ste_symbols, scopes, bound, newfree,
865 465597 : ste->ste_type == ClassBlock))
866 0 : goto error;
867 :
868 465597 : temp = PyNumber_InPlaceOr(free, newfree);
869 465597 : if (!temp)
870 0 : goto error;
871 465597 : Py_DECREF(temp);
872 465597 : success = 1;
873 465608 : error:
874 465608 : Py_XDECREF(scopes);
875 465608 : Py_XDECREF(local);
876 465608 : Py_XDECREF(newbound);
877 465608 : Py_XDECREF(newglobal);
878 465608 : Py_XDECREF(newfree);
879 465608 : Py_XDECREF(allfree);
880 465608 : if (!success)
881 11 : assert(PyErr_Occurred());
882 465608 : return success;
883 : }
884 :
885 : static int
886 351064 : analyze_child_block(PySTEntryObject *entry, PyObject *bound, PyObject *free,
887 : PyObject *global, PyObject* child_free)
888 : {
889 351064 : PyObject *temp_bound = NULL, *temp_global = NULL, *temp_free = NULL;
890 : PyObject *temp;
891 :
892 : /* Copy the bound and global dictionaries.
893 :
894 : These dictionaries are used by all blocks enclosed by the
895 : current block. The analyze_block() call modifies these
896 : dictionaries.
897 :
898 : */
899 351064 : temp_bound = PySet_New(bound);
900 351064 : if (!temp_bound)
901 0 : goto error;
902 351064 : temp_free = PySet_New(free);
903 351064 : if (!temp_free)
904 0 : goto error;
905 351064 : temp_global = PySet_New(global);
906 351064 : if (!temp_global)
907 0 : goto error;
908 :
909 351064 : if (!analyze_block(entry, temp_bound, temp_free, temp_global))
910 5 : goto error;
911 351059 : temp = PyNumber_InPlaceOr(child_free, temp_free);
912 351059 : if (!temp)
913 0 : goto error;
914 351059 : Py_DECREF(temp);
915 351059 : Py_DECREF(temp_bound);
916 351059 : Py_DECREF(temp_free);
917 351059 : Py_DECREF(temp_global);
918 351059 : return 1;
919 5 : error:
920 5 : Py_XDECREF(temp_bound);
921 5 : Py_XDECREF(temp_free);
922 5 : Py_XDECREF(temp_global);
923 5 : return 0;
924 : }
925 :
926 : static int
927 114544 : symtable_analyze(struct symtable *st)
928 : {
929 : PyObject *free, *global;
930 : int r;
931 :
932 114544 : free = PySet_New(NULL);
933 114544 : if (!free)
934 0 : return 0;
935 114544 : global = PySet_New(NULL);
936 114544 : if (!global) {
937 0 : Py_DECREF(free);
938 0 : return 0;
939 : }
940 114544 : r = analyze_block(st->st_top, NULL, free, global);
941 114544 : Py_DECREF(free);
942 114544 : Py_DECREF(global);
943 114544 : return r;
944 : }
945 :
946 : /* symtable_enter_block() gets a reference via ste_new.
947 : This reference is released when the block is exited, via the DECREF
948 : in symtable_exit_block().
949 : */
950 :
951 : static int
952 470383 : symtable_exit_block(struct symtable *st)
953 : {
954 : Py_ssize_t size;
955 :
956 470383 : st->st_cur = NULL;
957 470383 : size = PyList_GET_SIZE(st->st_stack);
958 470383 : if (size) {
959 470383 : if (PyList_SetSlice(st->st_stack, size - 1, size, NULL) < 0)
960 0 : return 0;
961 470383 : if (--size)
962 355816 : st->st_cur = (PySTEntryObject *)PyList_GET_ITEM(st->st_stack, size - 1);
963 : }
964 470383 : return 1;
965 : }
966 :
967 : static int
968 470536 : symtable_enter_block(struct symtable *st, identifier name, _Py_block_ty block,
969 : void *ast, int lineno, int col_offset,
970 : int end_lineno, int end_col_offset)
971 : {
972 470536 : PySTEntryObject *prev = NULL, *ste;
973 :
974 470536 : ste = ste_new(st, name, block, ast, lineno, col_offset, end_lineno, end_col_offset);
975 470536 : if (ste == NULL)
976 0 : return 0;
977 470536 : if (PyList_Append(st->st_stack, (PyObject *)ste) < 0) {
978 0 : Py_DECREF(ste);
979 0 : return 0;
980 : }
981 470536 : prev = st->st_cur;
982 : /* bpo-37757: For now, disallow *all* assignment expressions in the
983 : * outermost iterator expression of a comprehension, even those inside
984 : * a nested comprehension or a lambda expression.
985 : */
986 470536 : if (prev) {
987 355845 : ste->ste_comp_iter_expr = prev->ste_comp_iter_expr;
988 : }
989 : /* The entry is owned by the stack. Borrow it for st_cur. */
990 470536 : Py_DECREF(ste);
991 470536 : st->st_cur = ste;
992 :
993 : /* Annotation blocks shouldn't have any affect on the symbol table since in
994 : * the compilation stage, they will all be transformed to strings. They are
995 : * only created if future 'annotations' feature is activated. */
996 470536 : if (block == AnnotationBlock) {
997 4232 : return 1;
998 : }
999 :
1000 466304 : if (block == ModuleBlock)
1001 114691 : st->st_global = st->st_cur->ste_symbols;
1002 :
1003 466304 : if (prev) {
1004 351613 : if (PyList_Append(prev->ste_children, (PyObject *)ste) < 0) {
1005 0 : return 0;
1006 : }
1007 : }
1008 466304 : return 1;
1009 : }
1010 :
1011 : static long
1012 10660 : symtable_lookup(struct symtable *st, PyObject *name)
1013 : {
1014 10660 : PyObject *mangled = _Py_Mangle(st->st_private, name);
1015 10660 : if (!mangled)
1016 0 : return 0;
1017 10660 : long ret = _PyST_GetSymbol(st->st_cur, mangled);
1018 10660 : Py_DECREF(mangled);
1019 10660 : return ret;
1020 : }
1021 :
1022 : static int
1023 5798920 : symtable_add_def_helper(struct symtable *st, PyObject *name, int flag, struct _symtable_entry *ste,
1024 : int lineno, int col_offset, int end_lineno, int end_col_offset)
1025 : {
1026 : PyObject *o;
1027 : PyObject *dict;
1028 : long val;
1029 5798920 : PyObject *mangled = _Py_Mangle(st->st_private, name);
1030 :
1031 :
1032 5798920 : if (!mangled)
1033 0 : return 0;
1034 5798920 : dict = ste->ste_symbols;
1035 5798920 : if ((o = PyDict_GetItemWithError(dict, mangled))) {
1036 3559280 : val = PyLong_AS_LONG(o);
1037 3559280 : if ((flag & DEF_PARAM) && (val & DEF_PARAM)) {
1038 : /* Is it better to use 'mangled' or 'name' here? */
1039 16 : PyErr_Format(PyExc_SyntaxError, DUPLICATE_ARGUMENT, name);
1040 16 : PyErr_RangedSyntaxLocationObject(st->st_filename,
1041 : lineno, col_offset + 1,
1042 : end_lineno, end_col_offset + 1);
1043 16 : goto error;
1044 : }
1045 3559260 : val |= flag;
1046 : }
1047 2239640 : else if (PyErr_Occurred()) {
1048 3 : goto error;
1049 : }
1050 : else {
1051 2239640 : val = flag;
1052 : }
1053 5798900 : if (ste->ste_comp_iter_target) {
1054 : /* This name is an iteration variable in a comprehension,
1055 : * so check for a binding conflict with any named expressions.
1056 : * Otherwise, mark it as an iteration variable so subsequent
1057 : * named expressions can check for conflicts.
1058 : */
1059 22381 : if (val & (DEF_GLOBAL | DEF_NONLOCAL)) {
1060 13 : PyErr_Format(PyExc_SyntaxError,
1061 : NAMED_EXPR_COMP_INNER_LOOP_CONFLICT, name);
1062 13 : PyErr_RangedSyntaxLocationObject(st->st_filename,
1063 : lineno, col_offset + 1,
1064 : end_lineno, end_col_offset + 1);
1065 13 : goto error;
1066 : }
1067 22368 : val |= DEF_COMP_ITER;
1068 : }
1069 5798890 : o = PyLong_FromLong(val);
1070 5798890 : if (o == NULL)
1071 0 : goto error;
1072 5798890 : if (PyDict_SetItem(dict, mangled, o) < 0) {
1073 0 : Py_DECREF(o);
1074 0 : goto error;
1075 : }
1076 5798890 : Py_DECREF(o);
1077 :
1078 5798890 : if (flag & DEF_PARAM) {
1079 701082 : if (PyList_Append(ste->ste_varnames, mangled) < 0)
1080 0 : goto error;
1081 5097810 : } else if (flag & DEF_GLOBAL) {
1082 : /* XXX need to update DEF_GLOBAL for other flags too;
1083 : perhaps only DEF_FREE_GLOBAL */
1084 1996 : val = flag;
1085 1996 : if ((o = PyDict_GetItemWithError(st->st_global, mangled))) {
1086 1649 : val |= PyLong_AS_LONG(o);
1087 : }
1088 347 : else if (PyErr_Occurred()) {
1089 0 : goto error;
1090 : }
1091 1996 : o = PyLong_FromLong(val);
1092 1996 : if (o == NULL)
1093 0 : goto error;
1094 1996 : if (PyDict_SetItem(st->st_global, mangled, o) < 0) {
1095 0 : Py_DECREF(o);
1096 0 : goto error;
1097 : }
1098 1996 : Py_DECREF(o);
1099 : }
1100 5798890 : Py_DECREF(mangled);
1101 5798890 : return 1;
1102 :
1103 32 : error:
1104 32 : Py_DECREF(mangled);
1105 32 : return 0;
1106 : }
1107 :
1108 : static int
1109 5798830 : symtable_add_def(struct symtable *st, PyObject *name, int flag,
1110 : int lineno, int col_offset, int end_lineno, int end_col_offset)
1111 : {
1112 5798830 : return symtable_add_def_helper(st, name, flag, st->st_cur,
1113 : lineno, col_offset, end_lineno, end_col_offset);
1114 : }
1115 :
1116 : /* VISIT, VISIT_SEQ and VIST_SEQ_TAIL take an ASDL type as their second argument.
1117 : They use the ASDL name to synthesize the name of the C type and the visit
1118 : function.
1119 :
1120 : VISIT_SEQ_TAIL permits the start of an ASDL sequence to be skipped, which is
1121 : useful if the first node in the sequence requires special treatment.
1122 :
1123 : VISIT_QUIT macro returns the specified value exiting from the function but
1124 : first adjusts current recursion counter depth.
1125 : */
1126 :
1127 : #define VISIT_QUIT(ST, X) \
1128 : return --(ST)->recursion_depth,(X)
1129 :
1130 : #define VISIT(ST, TYPE, V) \
1131 : if (!symtable_visit_ ## TYPE((ST), (V))) \
1132 : VISIT_QUIT((ST), 0);
1133 :
1134 : #define VISIT_SEQ(ST, TYPE, SEQ) { \
1135 : int i; \
1136 : asdl_ ## TYPE ## _seq *seq = (SEQ); /* avoid variable capture */ \
1137 : for (i = 0; i < asdl_seq_LEN(seq); i++) { \
1138 : TYPE ## _ty elt = (TYPE ## _ty)asdl_seq_GET(seq, i); \
1139 : if (!symtable_visit_ ## TYPE((ST), elt)) \
1140 : VISIT_QUIT((ST), 0); \
1141 : } \
1142 : }
1143 :
1144 : #define VISIT_SEQ_TAIL(ST, TYPE, SEQ, START) { \
1145 : int i; \
1146 : asdl_ ## TYPE ## _seq *seq = (SEQ); /* avoid variable capture */ \
1147 : for (i = (START); i < asdl_seq_LEN(seq); i++) { \
1148 : TYPE ## _ty elt = (TYPE ## _ty)asdl_seq_GET(seq, i); \
1149 : if (!symtable_visit_ ## TYPE((ST), elt)) \
1150 : VISIT_QUIT((ST), 0); \
1151 : } \
1152 : }
1153 :
1154 : #define VISIT_SEQ_WITH_NULL(ST, TYPE, SEQ) { \
1155 : int i = 0; \
1156 : asdl_ ## TYPE ## _seq *seq = (SEQ); /* avoid variable capture */ \
1157 : for (i = 0; i < asdl_seq_LEN(seq); i++) { \
1158 : TYPE ## _ty elt = (TYPE ## _ty)asdl_seq_GET(seq, i); \
1159 : if (!elt) continue; /* can be NULL */ \
1160 : if (!symtable_visit_ ## TYPE((ST), elt)) \
1161 : VISIT_QUIT((ST), 0); \
1162 : } \
1163 : }
1164 :
1165 : static int
1166 2786 : symtable_record_directive(struct symtable *st, identifier name, int lineno,
1167 : int col_offset, int end_lineno, int end_col_offset)
1168 : {
1169 : PyObject *data, *mangled;
1170 : int res;
1171 2786 : if (!st->st_cur->ste_directives) {
1172 1971 : st->st_cur->ste_directives = PyList_New(0);
1173 1971 : if (!st->st_cur->ste_directives)
1174 0 : return 0;
1175 : }
1176 2786 : mangled = _Py_Mangle(st->st_private, name);
1177 2786 : if (!mangled)
1178 0 : return 0;
1179 2786 : data = Py_BuildValue("(Niiii)", mangled, lineno, col_offset, end_lineno, end_col_offset);
1180 2786 : if (!data)
1181 0 : return 0;
1182 2786 : res = PyList_Append(st->st_cur->ste_directives, data);
1183 2786 : Py_DECREF(data);
1184 2786 : return res == 0;
1185 : }
1186 :
1187 :
1188 : static int
1189 2610870 : symtable_visit_stmt(struct symtable *st, stmt_ty s)
1190 : {
1191 2610870 : if (++st->recursion_depth > st->recursion_limit) {
1192 0 : PyErr_SetString(PyExc_RecursionError,
1193 : "maximum recursion depth exceeded during compilation");
1194 0 : VISIT_QUIT(st, 0);
1195 : }
1196 2610870 : switch (s->kind) {
1197 258420 : case FunctionDef_kind:
1198 258420 : if (!symtable_add_def(st, s->v.FunctionDef.name, DEF_LOCAL, LOCATION(s)))
1199 0 : VISIT_QUIT(st, 0);
1200 258420 : if (s->v.FunctionDef.args->defaults)
1201 323986 : VISIT_SEQ(st, expr, s->v.FunctionDef.args->defaults);
1202 258420 : if (s->v.FunctionDef.args->kw_defaults)
1203 274554 : VISIT_SEQ_WITH_NULL(st, expr, s->v.FunctionDef.args->kw_defaults);
1204 258420 : if (!symtable_visit_annotations(st, s, s->v.FunctionDef.args,
1205 : s->v.FunctionDef.returns))
1206 3 : VISIT_QUIT(st, 0);
1207 258417 : if (s->v.FunctionDef.decorator_list)
1208 54217 : VISIT_SEQ(st, expr, s->v.FunctionDef.decorator_list);
1209 258417 : if (!symtable_enter_block(st, s->v.FunctionDef.name,
1210 : FunctionBlock, (void *)s,
1211 : LOCATION(s)))
1212 0 : VISIT_QUIT(st, 0);
1213 258417 : VISIT(st, arguments, s->v.FunctionDef.args);
1214 1146620 : VISIT_SEQ(st, stmt, s->v.FunctionDef.body);
1215 258369 : if (!symtable_exit_block(st))
1216 0 : VISIT_QUIT(st, 0);
1217 258369 : break;
1218 37747 : case ClassDef_kind: {
1219 : PyObject *tmp;
1220 37747 : if (!symtable_add_def(st, s->v.ClassDef.name, DEF_LOCAL, LOCATION(s)))
1221 0 : VISIT_QUIT(st, 0);
1222 70052 : VISIT_SEQ(st, expr, s->v.ClassDef.bases);
1223 38890 : VISIT_SEQ(st, keyword, s->v.ClassDef.keywords);
1224 37747 : if (s->v.ClassDef.decorator_list)
1225 3933 : VISIT_SEQ(st, expr, s->v.ClassDef.decorator_list);
1226 37747 : if (!symtable_enter_block(st, s->v.ClassDef.name, ClassBlock,
1227 : (void *)s, s->lineno, s->col_offset,
1228 : s->end_lineno, s->end_col_offset))
1229 0 : VISIT_QUIT(st, 0);
1230 37747 : tmp = st->st_private;
1231 37747 : st->st_private = s->v.ClassDef.name;
1232 270791 : VISIT_SEQ(st, stmt, s->v.ClassDef.body);
1233 37745 : st->st_private = tmp;
1234 37745 : if (!symtable_exit_block(st))
1235 0 : VISIT_QUIT(st, 0);
1236 37745 : break;
1237 : }
1238 212342 : case Return_kind:
1239 212342 : if (s->v.Return.value) {
1240 203679 : VISIT(st, expr, s->v.Return.value);
1241 203679 : st->st_cur->ste_returns_value = 1;
1242 : }
1243 212342 : break;
1244 6454 : case Delete_kind:
1245 13763 : VISIT_SEQ(st, expr, s->v.Delete.targets);
1246 6454 : break;
1247 635594 : case Assign_kind:
1248 1277580 : VISIT_SEQ(st, expr, s->v.Assign.targets);
1249 635594 : VISIT(st, expr, s->v.Assign.value);
1250 635593 : break;
1251 9876 : case AnnAssign_kind:
1252 9876 : if (s->v.AnnAssign.target->kind == Name_kind) {
1253 7946 : expr_ty e_name = s->v.AnnAssign.target;
1254 7946 : long cur = symtable_lookup(st, e_name->v.Name.id);
1255 7946 : if (cur < 0) {
1256 0 : VISIT_QUIT(st, 0);
1257 : }
1258 7946 : if ((cur & (DEF_GLOBAL | DEF_NONLOCAL))
1259 2 : && (st->st_cur->ste_symbols != st->st_global)
1260 1 : && s->v.AnnAssign.simple) {
1261 1 : PyErr_Format(PyExc_SyntaxError,
1262 1 : cur & DEF_GLOBAL ? GLOBAL_ANNOT : NONLOCAL_ANNOT,
1263 : e_name->v.Name.id);
1264 1 : PyErr_RangedSyntaxLocationObject(st->st_filename,
1265 : s->lineno,
1266 1 : s->col_offset + 1,
1267 : s->end_lineno,
1268 1 : s->end_col_offset + 1);
1269 1 : VISIT_QUIT(st, 0);
1270 : }
1271 15885 : if (s->v.AnnAssign.simple &&
1272 7940 : !symtable_add_def(st, e_name->v.Name.id,
1273 : DEF_ANNOT | DEF_LOCAL, LOCATION(e_name))) {
1274 0 : VISIT_QUIT(st, 0);
1275 : }
1276 : else {
1277 7945 : if (s->v.AnnAssign.value
1278 5543 : && !symtable_add_def(st, e_name->v.Name.id, DEF_LOCAL, LOCATION(e_name))) {
1279 0 : VISIT_QUIT(st, 0);
1280 : }
1281 : }
1282 : }
1283 : else {
1284 1930 : VISIT(st, expr, s->v.AnnAssign.target);
1285 : }
1286 9875 : if (!symtable_visit_annotation(st, s->v.AnnAssign.annotation)) {
1287 5 : VISIT_QUIT(st, 0);
1288 : }
1289 :
1290 9870 : if (s->v.AnnAssign.value) {
1291 7098 : VISIT(st, expr, s->v.AnnAssign.value);
1292 : }
1293 9870 : break;
1294 21479 : case AugAssign_kind:
1295 21479 : VISIT(st, expr, s->v.AugAssign.target);
1296 21479 : VISIT(st, expr, s->v.AugAssign.value);
1297 21479 : break;
1298 46298 : case For_kind:
1299 46298 : VISIT(st, expr, s->v.For.target);
1300 46298 : VISIT(st, expr, s->v.For.iter);
1301 141555 : VISIT_SEQ(st, stmt, s->v.For.body);
1302 46297 : if (s->v.For.orelse)
1303 4175 : VISIT_SEQ(st, stmt, s->v.For.orelse);
1304 46297 : break;
1305 9951 : case While_kind:
1306 9951 : VISIT(st, expr, s->v.While.test);
1307 42284 : VISIT_SEQ(st, stmt, s->v.While.body);
1308 9951 : if (s->v.While.orelse)
1309 820 : VISIT_SEQ(st, stmt, s->v.While.orelse);
1310 9951 : break;
1311 484759 : case If_kind:
1312 : /* XXX if 0: and lookup_yield() hacks */
1313 484759 : VISIT(st, expr, s->v.If.test);
1314 1109190 : VISIT_SEQ(st, stmt, s->v.If.body);
1315 484753 : if (s->v.If.orelse)
1316 200919 : VISIT_SEQ(st, stmt, s->v.If.orelse);
1317 484753 : break;
1318 682 : case Match_kind:
1319 682 : VISIT(st, expr, s->v.Match.subject);
1320 1663 : VISIT_SEQ(st, match_case, s->v.Match.cases);
1321 679 : break;
1322 52599 : case Raise_kind:
1323 52599 : if (s->v.Raise.exc) {
1324 48611 : VISIT(st, expr, s->v.Raise.exc);
1325 48611 : if (s->v.Raise.cause) {
1326 2501 : VISIT(st, expr, s->v.Raise.cause);
1327 : }
1328 : }
1329 52599 : break;
1330 39926 : case Try_kind:
1331 98716 : VISIT_SEQ(st, stmt, s->v.Try.body);
1332 47347 : VISIT_SEQ(st, stmt, s->v.Try.orelse);
1333 76768 : VISIT_SEQ(st, excepthandler, s->v.Try.handlers);
1334 47069 : VISIT_SEQ(st, stmt, s->v.Try.finalbody);
1335 39926 : break;
1336 108 : case TryStar_kind:
1337 218 : VISIT_SEQ(st, stmt, s->v.TryStar.body);
1338 152 : VISIT_SEQ(st, stmt, s->v.TryStar.orelse);
1339 236 : VISIT_SEQ(st, excepthandler, s->v.TryStar.handlers);
1340 133 : VISIT_SEQ(st, stmt, s->v.TryStar.finalbody);
1341 108 : break;
1342 8436 : case Assert_kind:
1343 8436 : VISIT(st, expr, s->v.Assert.test);
1344 8436 : if (s->v.Assert.msg)
1345 1981 : VISIT(st, expr, s->v.Assert.msg);
1346 8436 : break;
1347 36428 : case Import_kind:
1348 73920 : VISIT_SEQ(st, alias, s->v.Import.names);
1349 36428 : break;
1350 41631 : case ImportFrom_kind:
1351 113336 : VISIT_SEQ(st, alias, s->v.ImportFrom.names);
1352 41624 : break;
1353 1520 : case Global_kind: {
1354 : int i;
1355 1520 : asdl_identifier_seq *seq = s->v.Global.names;
1356 3487 : for (i = 0; i < asdl_seq_LEN(seq); i++) {
1357 1980 : identifier name = (identifier)asdl_seq_GET(seq, i);
1358 1980 : long cur = symtable_lookup(st, name);
1359 1980 : if (cur < 0)
1360 0 : VISIT_QUIT(st, 0);
1361 1980 : if (cur & (DEF_PARAM | DEF_LOCAL | USE | DEF_ANNOT)) {
1362 : const char* msg;
1363 13 : if (cur & DEF_PARAM) {
1364 7 : msg = GLOBAL_PARAM;
1365 6 : } else if (cur & USE) {
1366 3 : msg = GLOBAL_AFTER_USE;
1367 3 : } else if (cur & DEF_ANNOT) {
1368 1 : msg = GLOBAL_ANNOT;
1369 : } else { /* DEF_LOCAL */
1370 2 : msg = GLOBAL_AFTER_ASSIGN;
1371 : }
1372 13 : PyErr_Format(PyExc_SyntaxError,
1373 : msg, name);
1374 13 : PyErr_RangedSyntaxLocationObject(st->st_filename,
1375 : s->lineno,
1376 13 : s->col_offset + 1,
1377 : s->end_lineno,
1378 13 : s->end_col_offset + 1);
1379 13 : VISIT_QUIT(st, 0);
1380 : }
1381 1967 : if (!symtable_add_def(st, name, DEF_GLOBAL, LOCATION(s)))
1382 0 : VISIT_QUIT(st, 0);
1383 1967 : if (!symtable_record_directive(st, name, s->lineno, s->col_offset,
1384 : s->end_lineno, s->end_col_offset))
1385 0 : VISIT_QUIT(st, 0);
1386 : }
1387 1507 : break;
1388 : }
1389 581 : case Nonlocal_kind: {
1390 : int i;
1391 581 : asdl_identifier_seq *seq = s->v.Nonlocal.names;
1392 1310 : for (i = 0; i < asdl_seq_LEN(seq); i++) {
1393 734 : identifier name = (identifier)asdl_seq_GET(seq, i);
1394 734 : long cur = symtable_lookup(st, name);
1395 734 : if (cur < 0)
1396 0 : VISIT_QUIT(st, 0);
1397 734 : if (cur & (DEF_PARAM | DEF_LOCAL | USE | DEF_ANNOT)) {
1398 : const char* msg;
1399 5 : if (cur & DEF_PARAM) {
1400 3 : msg = NONLOCAL_PARAM;
1401 2 : } else if (cur & USE) {
1402 1 : msg = NONLOCAL_AFTER_USE;
1403 1 : } else if (cur & DEF_ANNOT) {
1404 0 : msg = NONLOCAL_ANNOT;
1405 : } else { /* DEF_LOCAL */
1406 1 : msg = NONLOCAL_AFTER_ASSIGN;
1407 : }
1408 5 : PyErr_Format(PyExc_SyntaxError, msg, name);
1409 5 : PyErr_RangedSyntaxLocationObject(st->st_filename,
1410 : s->lineno,
1411 5 : s->col_offset + 1,
1412 : s->end_lineno,
1413 5 : s->end_col_offset + 1);
1414 5 : VISIT_QUIT(st, 0);
1415 : }
1416 729 : if (!symtable_add_def(st, name, DEF_NONLOCAL, LOCATION(s)))
1417 0 : VISIT_QUIT(st, 0);
1418 729 : if (!symtable_record_directive(st, name, s->lineno, s->col_offset,
1419 : s->end_lineno, s->end_col_offset))
1420 0 : VISIT_QUIT(st, 0);
1421 : }
1422 576 : break;
1423 : }
1424 650006 : case Expr_kind:
1425 650006 : VISIT(st, expr, s->v.Expr.value);
1426 649912 : break;
1427 32757 : case Pass_kind:
1428 : case Break_kind:
1429 : case Continue_kind:
1430 : /* nothing to do here */
1431 32757 : break;
1432 20632 : case With_kind:
1433 41642 : VISIT_SEQ(st, withitem, s->v.With.items);
1434 60171 : VISIT_SEQ(st, stmt, s->v.With.body);
1435 20632 : break;
1436 2337 : case AsyncFunctionDef_kind:
1437 2337 : if (!symtable_add_def(st, s->v.AsyncFunctionDef.name, DEF_LOCAL, LOCATION(s)))
1438 0 : VISIT_QUIT(st, 0);
1439 2337 : if (s->v.AsyncFunctionDef.args->defaults)
1440 2511 : VISIT_SEQ(st, expr, s->v.AsyncFunctionDef.args->defaults);
1441 2337 : if (s->v.AsyncFunctionDef.args->kw_defaults)
1442 2629 : VISIT_SEQ_WITH_NULL(st, expr,
1443 : s->v.AsyncFunctionDef.args->kw_defaults);
1444 2337 : if (!symtable_visit_annotations(st, s, s->v.AsyncFunctionDef.args,
1445 : s->v.AsyncFunctionDef.returns))
1446 1 : VISIT_QUIT(st, 0);
1447 2336 : if (s->v.AsyncFunctionDef.decorator_list)
1448 463 : VISIT_SEQ(st, expr, s->v.AsyncFunctionDef.decorator_list);
1449 2336 : if (!symtable_enter_block(st, s->v.AsyncFunctionDef.name,
1450 : FunctionBlock, (void *)s,
1451 : s->lineno, s->col_offset,
1452 : s->end_lineno, s->end_col_offset))
1453 0 : VISIT_QUIT(st, 0);
1454 2336 : st->st_cur->ste_coroutine = 1;
1455 2336 : VISIT(st, arguments, s->v.AsyncFunctionDef.args);
1456 8832 : VISIT_SEQ(st, stmt, s->v.AsyncFunctionDef.body);
1457 2334 : if (!symtable_exit_block(st))
1458 0 : VISIT_QUIT(st, 0);
1459 2334 : break;
1460 225 : case AsyncWith_kind:
1461 458 : VISIT_SEQ(st, withitem, s->v.AsyncWith.items);
1462 598 : VISIT_SEQ(st, stmt, s->v.AsyncWith.body);
1463 225 : break;
1464 81 : case AsyncFor_kind:
1465 81 : VISIT(st, expr, s->v.AsyncFor.target);
1466 81 : VISIT(st, expr, s->v.AsyncFor.iter);
1467 167 : VISIT_SEQ(st, stmt, s->v.AsyncFor.body);
1468 81 : if (s->v.AsyncFor.orelse)
1469 37 : VISIT_SEQ(st, stmt, s->v.AsyncFor.orelse);
1470 81 : break;
1471 : }
1472 2610680 : VISIT_QUIT(st, 1);
1473 : }
1474 :
1475 : static int
1476 105 : symtable_extend_namedexpr_scope(struct symtable *st, expr_ty e)
1477 : {
1478 105 : assert(st->st_stack);
1479 105 : assert(e->kind == Name_kind);
1480 :
1481 105 : PyObject *target_name = e->v.Name.id;
1482 : Py_ssize_t i, size;
1483 : struct _symtable_entry *ste;
1484 105 : size = PyList_GET_SIZE(st->st_stack);
1485 105 : assert(size);
1486 :
1487 : /* Iterate over the stack in reverse and add to the nearest adequate scope */
1488 200 : for (i = size - 1; i >= 0; i--) {
1489 200 : ste = (struct _symtable_entry *) PyList_GET_ITEM(st->st_stack, i);
1490 :
1491 : /* If we find a comprehension scope, check for a target
1492 : * binding conflict with iteration variables, otherwise skip it
1493 : */
1494 200 : if (ste->ste_comprehension) {
1495 109 : long target_in_scope = _PyST_GetSymbol(ste, target_name);
1496 109 : if (target_in_scope & DEF_COMP_ITER) {
1497 14 : PyErr_Format(PyExc_SyntaxError, NAMED_EXPR_COMP_CONFLICT, target_name);
1498 14 : PyErr_RangedSyntaxLocationObject(st->st_filename,
1499 : e->lineno,
1500 14 : e->col_offset + 1,
1501 : e->end_lineno,
1502 14 : e->end_col_offset + 1);
1503 14 : VISIT_QUIT(st, 0);
1504 : }
1505 95 : continue;
1506 : }
1507 :
1508 : /* If we find a FunctionBlock entry, add as GLOBAL/LOCAL or NONLOCAL/LOCAL */
1509 91 : if (ste->ste_type == FunctionBlock) {
1510 76 : long target_in_scope = _PyST_GetSymbol(ste, target_name);
1511 76 : if (target_in_scope & DEF_GLOBAL) {
1512 1 : if (!symtable_add_def(st, target_name, DEF_GLOBAL, LOCATION(e)))
1513 0 : VISIT_QUIT(st, 0);
1514 : } else {
1515 75 : if (!symtable_add_def(st, target_name, DEF_NONLOCAL, LOCATION(e)))
1516 0 : VISIT_QUIT(st, 0);
1517 : }
1518 76 : if (!symtable_record_directive(st, target_name, LOCATION(e)))
1519 0 : VISIT_QUIT(st, 0);
1520 :
1521 76 : return symtable_add_def_helper(st, target_name, DEF_LOCAL, ste, LOCATION(e));
1522 : }
1523 : /* If we find a ModuleBlock entry, add as GLOBAL */
1524 15 : if (ste->ste_type == ModuleBlock) {
1525 14 : if (!symtable_add_def(st, target_name, DEF_GLOBAL, LOCATION(e)))
1526 0 : VISIT_QUIT(st, 0);
1527 14 : if (!symtable_record_directive(st, target_name, LOCATION(e)))
1528 0 : VISIT_QUIT(st, 0);
1529 :
1530 14 : return symtable_add_def_helper(st, target_name, DEF_GLOBAL, ste, LOCATION(e));
1531 : }
1532 : /* Disallow usage in ClassBlock */
1533 1 : if (ste->ste_type == ClassBlock) {
1534 1 : PyErr_Format(PyExc_SyntaxError, NAMED_EXPR_COMP_IN_CLASS);
1535 1 : PyErr_RangedSyntaxLocationObject(st->st_filename,
1536 : e->lineno,
1537 1 : e->col_offset + 1,
1538 : e->end_lineno,
1539 1 : e->end_col_offset + 1);
1540 1 : VISIT_QUIT(st, 0);
1541 : }
1542 : }
1543 :
1544 : /* We should always find either a FunctionBlock, ModuleBlock or ClassBlock
1545 : and should never fall to this case
1546 : */
1547 0 : assert(0);
1548 : return 0;
1549 : }
1550 :
1551 : static int
1552 799 : symtable_handle_namedexpr(struct symtable *st, expr_ty e)
1553 : {
1554 799 : if (st->st_cur->ste_comp_iter_expr > 0) {
1555 : /* Assignment isn't allowed in a comprehension iterable expression */
1556 54 : PyErr_Format(PyExc_SyntaxError, NAMED_EXPR_COMP_ITER_EXPR);
1557 54 : PyErr_RangedSyntaxLocationObject(st->st_filename,
1558 : e->lineno,
1559 54 : e->col_offset + 1,
1560 : e->end_lineno,
1561 54 : e->end_col_offset + 1);
1562 54 : return 0;
1563 : }
1564 745 : if (st->st_cur->ste_comprehension) {
1565 : /* Inside a comprehension body, so find the right target scope */
1566 105 : if (!symtable_extend_namedexpr_scope(st, e->v.NamedExpr.target))
1567 15 : return 0;
1568 : }
1569 730 : VISIT(st, expr, e->v.NamedExpr.value);
1570 730 : VISIT(st, expr, e->v.NamedExpr.target);
1571 730 : return 1;
1572 : }
1573 :
1574 : static int
1575 11336900 : symtable_visit_expr(struct symtable *st, expr_ty e)
1576 : {
1577 11336900 : if (++st->recursion_depth > st->recursion_limit) {
1578 0 : PyErr_SetString(PyExc_RecursionError,
1579 : "maximum recursion depth exceeded during compilation");
1580 0 : VISIT_QUIT(st, 0);
1581 : }
1582 11336900 : switch (e->kind) {
1583 801 : case NamedExpr_kind:
1584 801 : if (!symtable_raise_if_annotation_block(st, "named expression", e)) {
1585 2 : VISIT_QUIT(st, 0);
1586 : }
1587 799 : if(!symtable_handle_namedexpr(st, e))
1588 69 : VISIT_QUIT(st, 0);
1589 730 : break;
1590 58131 : case BoolOp_kind:
1591 182689 : VISIT_SEQ(st, expr, e->v.BoolOp.values);
1592 58127 : break;
1593 196676 : case BinOp_kind:
1594 196676 : VISIT(st, expr, e->v.BinOp.left);
1595 196676 : VISIT(st, expr, e->v.BinOp.right);
1596 196675 : break;
1597 55072 : case UnaryOp_kind:
1598 55072 : VISIT(st, expr, e->v.UnaryOp.operand);
1599 55071 : break;
1600 34998 : case Lambda_kind: {
1601 34998 : if (e->v.Lambda.args->defaults)
1602 35369 : VISIT_SEQ(st, expr, e->v.Lambda.args->defaults);
1603 34998 : if (e->v.Lambda.args->kw_defaults)
1604 35171 : VISIT_SEQ_WITH_NULL(st, expr, e->v.Lambda.args->kw_defaults);
1605 34998 : if (!symtable_enter_block(st, &_Py_ID(lambda),
1606 : FunctionBlock, (void *)e,
1607 : e->lineno, e->col_offset,
1608 : e->end_lineno, e->end_col_offset))
1609 0 : VISIT_QUIT(st, 0);
1610 34998 : VISIT(st, arguments, e->v.Lambda.args);
1611 34993 : VISIT(st, expr, e->v.Lambda.body);
1612 34965 : if (!symtable_exit_block(st))
1613 0 : VISIT_QUIT(st, 0);
1614 34965 : break;
1615 : }
1616 9909 : case IfExp_kind:
1617 9909 : VISIT(st, expr, e->v.IfExp.test);
1618 9909 : VISIT(st, expr, e->v.IfExp.body);
1619 9909 : VISIT(st, expr, e->v.IfExp.orelse);
1620 9909 : break;
1621 32282 : case Dict_kind:
1622 650049 : VISIT_SEQ_WITH_NULL(st, expr, e->v.Dict.keys);
1623 650049 : VISIT_SEQ(st, expr, e->v.Dict.values);
1624 32282 : break;
1625 1831 : case Set_kind:
1626 9073 : VISIT_SEQ(st, expr, e->v.Set.elts);
1627 1828 : break;
1628 7824 : case GeneratorExp_kind:
1629 7824 : if (!symtable_visit_genexp(st, e))
1630 1 : VISIT_QUIT(st, 0);
1631 7823 : break;
1632 8647 : case ListComp_kind:
1633 8647 : if (!symtable_visit_listcomp(st, e))
1634 58 : VISIT_QUIT(st, 0);
1635 8589 : break;
1636 507 : case SetComp_kind:
1637 507 : if (!symtable_visit_setcomp(st, e))
1638 52 : VISIT_QUIT(st, 0);
1639 455 : break;
1640 1191 : case DictComp_kind:
1641 1191 : if (!symtable_visit_dictcomp(st, e))
1642 2 : VISIT_QUIT(st, 0);
1643 1189 : break;
1644 11506 : case Yield_kind:
1645 11506 : if (!symtable_raise_if_annotation_block(st, "yield expression", e)) {
1646 3 : VISIT_QUIT(st, 0);
1647 : }
1648 11503 : if (e->v.Yield.value)
1649 10730 : VISIT(st, expr, e->v.Yield.value);
1650 11503 : st->st_cur->ste_generator = 1;
1651 11503 : if (st->st_cur->ste_comprehension) {
1652 10 : return symtable_raise_if_comprehension_block(st, e);
1653 : }
1654 11493 : break;
1655 1918 : case YieldFrom_kind:
1656 1918 : if (!symtable_raise_if_annotation_block(st, "yield expression", e)) {
1657 2 : VISIT_QUIT(st, 0);
1658 : }
1659 1916 : VISIT(st, expr, e->v.YieldFrom.value);
1660 1916 : st->st_cur->ste_generator = 1;
1661 1916 : if (st->st_cur->ste_comprehension) {
1662 1 : return symtable_raise_if_comprehension_block(st, e);
1663 : }
1664 1915 : break;
1665 1734 : case Await_kind:
1666 1734 : if (!symtable_raise_if_annotation_block(st, "await expression", e)) {
1667 2 : VISIT_QUIT(st, 0);
1668 : }
1669 1732 : VISIT(st, expr, e->v.Await.value);
1670 1732 : st->st_cur->ste_coroutine = 1;
1671 1732 : break;
1672 208265 : case Compare_kind:
1673 208265 : VISIT(st, expr, e->v.Compare.left);
1674 419256 : VISIT_SEQ(st, expr, e->v.Compare.comparators);
1675 208265 : break;
1676 1119430 : case Call_kind:
1677 1119430 : VISIT(st, expr, e->v.Call.func);
1678 2540150 : VISIT_SEQ(st, expr, e->v.Call.args);
1679 1253200 : VISIT_SEQ_WITH_NULL(st, keyword, e->v.Call.keywords);
1680 1119420 : break;
1681 118195 : case FormattedValue_kind:
1682 118195 : VISIT(st, expr, e->v.FormattedValue.value);
1683 118194 : if (e->v.FormattedValue.format_spec)
1684 4454 : VISIT(st, expr, e->v.FormattedValue.format_spec);
1685 118194 : break;
1686 30599 : case JoinedStr_kind:
1687 265935 : VISIT_SEQ(st, expr, e->v.JoinedStr.values);
1688 30598 : break;
1689 2993920 : case Constant_kind:
1690 : /* Nothing to do here. */
1691 2993920 : break;
1692 : /* The following exprs can be assignment targets. */
1693 1321750 : case Attribute_kind:
1694 1321750 : VISIT(st, expr, e->v.Attribute.value);
1695 1321750 : break;
1696 188599 : case Subscript_kind:
1697 188599 : VISIT(st, expr, e->v.Subscript.value);
1698 188599 : VISIT(st, expr, e->v.Subscript.slice);
1699 188599 : break;
1700 10286 : case Starred_kind:
1701 10286 : VISIT(st, expr, e->v.Starred.value);
1702 10286 : break;
1703 31029 : case Slice_kind:
1704 31029 : if (e->v.Slice.lower)
1705 17269 : VISIT(st, expr, e->v.Slice.lower)
1706 31029 : if (e->v.Slice.upper)
1707 17579 : VISIT(st, expr, e->v.Slice.upper)
1708 31029 : if (e->v.Slice.step)
1709 1005 : VISIT(st, expr, e->v.Slice.step)
1710 31029 : break;
1711 4659660 : case Name_kind:
1712 4659660 : if (!symtable_add_def(st, e->v.Name.id,
1713 4659660 : e->v.Name.ctx == Load ? USE : DEF_LOCAL, LOCATION(e)))
1714 16 : VISIT_QUIT(st, 0);
1715 : /* Special-case super: it counts as a use of __class__ */
1716 4659650 : if (e->v.Name.ctx == Load &&
1717 7136650 : st->st_cur->ste_type == FunctionBlock &&
1718 3129260 : _PyUnicode_EqualToASCIIString(e->v.Name.id, "super")) {
1719 7109 : if (!symtable_add_def(st, &_Py_ID(__class__), USE, LOCATION(e)))
1720 0 : VISIT_QUIT(st, 0);
1721 : }
1722 4659650 : break;
1723 : /* child nodes of List and Tuple will have expr_context set */
1724 63480 : case List_kind:
1725 577505 : VISIT_SEQ(st, expr, e->v.List.elts);
1726 63480 : break;
1727 168638 : case Tuple_kind:
1728 627035 : VISIT_SEQ(st, expr, e->v.Tuple.elts);
1729 168621 : break;
1730 : }
1731 11336600 : VISIT_QUIT(st, 1);
1732 : }
1733 :
1734 : static int
1735 2780 : symtable_visit_pattern(struct symtable *st, pattern_ty p)
1736 : {
1737 2780 : if (++st->recursion_depth > st->recursion_limit) {
1738 0 : PyErr_SetString(PyExc_RecursionError,
1739 : "maximum recursion depth exceeded during compilation");
1740 0 : VISIT_QUIT(st, 0);
1741 : }
1742 2780 : switch (p->kind) {
1743 770 : case MatchValue_kind:
1744 770 : VISIT(st, expr, p->v.MatchValue.value);
1745 770 : break;
1746 39 : case MatchSingleton_kind:
1747 : /* Nothing to do here. */
1748 39 : break;
1749 450 : case MatchSequence_kind:
1750 1317 : VISIT_SEQ(st, pattern, p->v.MatchSequence.patterns);
1751 450 : break;
1752 102 : case MatchStar_kind:
1753 102 : if (p->v.MatchStar.name) {
1754 68 : symtable_add_def(st, p->v.MatchStar.name, DEF_LOCAL, LOCATION(p));
1755 : }
1756 102 : break;
1757 271 : case MatchMapping_kind:
1758 536 : VISIT_SEQ(st, expr, p->v.MatchMapping.keys);
1759 536 : VISIT_SEQ(st, pattern, p->v.MatchMapping.patterns);
1760 271 : if (p->v.MatchMapping.rest) {
1761 16 : symtable_add_def(st, p->v.MatchMapping.rest, DEF_LOCAL, LOCATION(p));
1762 : }
1763 271 : break;
1764 184 : case MatchClass_kind:
1765 184 : VISIT(st, expr, p->v.MatchClass.cls);
1766 379 : VISIT_SEQ(st, pattern, p->v.MatchClass.patterns);
1767 261 : VISIT_SEQ(st, pattern, p->v.MatchClass.kwd_patterns);
1768 184 : break;
1769 853 : case MatchAs_kind:
1770 853 : if (p->v.MatchAs.pattern) {
1771 93 : VISIT(st, pattern, p->v.MatchAs.pattern);
1772 : }
1773 853 : if (p->v.MatchAs.name) {
1774 662 : symtable_add_def(st, p->v.MatchAs.name, DEF_LOCAL, LOCATION(p));
1775 : }
1776 853 : break;
1777 111 : case MatchOr_kind:
1778 410 : VISIT_SEQ(st, pattern, p->v.MatchOr.patterns);
1779 111 : break;
1780 : }
1781 2780 : VISIT_QUIT(st, 1);
1782 : }
1783 :
1784 : static int
1785 18115 : symtable_implicit_arg(struct symtable *st, int pos)
1786 : {
1787 18115 : PyObject *id = PyUnicode_FromFormat(".%d", pos);
1788 18115 : if (id == NULL)
1789 0 : return 0;
1790 18115 : if (!symtable_add_def(st, id, DEF_PARAM, ST_LOCATION(st->st_cur))) {
1791 0 : Py_DECREF(id);
1792 0 : return 0;
1793 : }
1794 18115 : Py_DECREF(id);
1795 18115 : return 1;
1796 : }
1797 :
1798 : static int
1799 887244 : symtable_visit_params(struct symtable *st, asdl_arg_seq *args)
1800 : {
1801 : int i;
1802 :
1803 887244 : if (!args)
1804 0 : return -1;
1805 :
1806 1522720 : for (i = 0; i < asdl_seq_LEN(args); i++) {
1807 635490 : arg_ty arg = (arg_ty)asdl_seq_GET(args, i);
1808 635490 : if (!symtable_add_def(st, arg->arg, DEF_PARAM, LOCATION(arg)))
1809 14 : return 0;
1810 : }
1811 :
1812 887230 : return 1;
1813 : }
1814 :
1815 : static int
1816 42849 : symtable_visit_annotation(struct symtable *st, expr_ty annotation)
1817 : {
1818 42849 : int future_annotations = st->st_future->ff_features & CO_FUTURE_ANNOTATIONS;
1819 45352 : if (future_annotations &&
1820 2503 : !symtable_enter_block(st, &_Py_ID(_annotation), AnnotationBlock,
1821 : (void *)annotation, annotation->lineno,
1822 : annotation->col_offset, annotation->end_lineno,
1823 : annotation->end_col_offset)) {
1824 0 : VISIT_QUIT(st, 0);
1825 : }
1826 42849 : VISIT(st, expr, annotation);
1827 42842 : if (future_annotations && !symtable_exit_block(st)) {
1828 0 : VISIT_QUIT(st, 0);
1829 : }
1830 42842 : return 1;
1831 : }
1832 :
1833 : static int
1834 782269 : symtable_visit_argannotations(struct symtable *st, asdl_arg_seq *args)
1835 : {
1836 : int i;
1837 :
1838 782269 : if (!args)
1839 0 : return -1;
1840 :
1841 1249660 : for (i = 0; i < asdl_seq_LEN(args); i++) {
1842 467397 : arg_ty arg = (arg_ty)asdl_seq_GET(args, i);
1843 467397 : if (arg->annotation)
1844 49100 : VISIT(st, expr, arg->annotation);
1845 : }
1846 :
1847 782267 : return 1;
1848 : }
1849 :
1850 : static int
1851 260757 : symtable_visit_annotations(struct symtable *st, stmt_ty o, arguments_ty a, expr_ty returns)
1852 : {
1853 260757 : int future_annotations = st->st_future->ff_features & CO_FUTURE_ANNOTATIONS;
1854 262486 : if (future_annotations &&
1855 1729 : !symtable_enter_block(st, &_Py_ID(_annotation), AnnotationBlock,
1856 : (void *)o, o->lineno, o->col_offset, o->end_lineno,
1857 : o->end_col_offset)) {
1858 0 : VISIT_QUIT(st, 0);
1859 : }
1860 260757 : if (a->posonlyargs && !symtable_visit_argannotations(st, a->posonlyargs))
1861 0 : return 0;
1862 260757 : if (a->args && !symtable_visit_argannotations(st, a->args))
1863 2 : return 0;
1864 260755 : if (a->vararg && a->vararg->annotation)
1865 777 : VISIT(st, expr, a->vararg->annotation);
1866 260755 : if (a->kwarg && a->kwarg->annotation)
1867 540 : VISIT(st, expr, a->kwarg->annotation);
1868 260755 : if (a->kwonlyargs && !symtable_visit_argannotations(st, a->kwonlyargs))
1869 0 : return 0;
1870 260755 : if (future_annotations && !symtable_exit_block(st)) {
1871 0 : VISIT_QUIT(st, 0);
1872 : }
1873 260755 : if (returns && !symtable_visit_annotation(st, returns)) {
1874 2 : VISIT_QUIT(st, 0);
1875 : }
1876 260753 : return 1;
1877 : }
1878 :
1879 : static int
1880 295751 : symtable_visit_arguments(struct symtable *st, arguments_ty a)
1881 : {
1882 : /* skip default arguments inside function block
1883 : XXX should ast be different?
1884 : */
1885 295751 : if (a->posonlyargs && !symtable_visit_params(st, a->posonlyargs))
1886 0 : return 0;
1887 295751 : if (a->args && !symtable_visit_params(st, a->args))
1888 9 : return 0;
1889 295742 : if (a->kwonlyargs && !symtable_visit_params(st, a->kwonlyargs))
1890 5 : return 0;
1891 295737 : if (a->vararg) {
1892 24295 : if (!symtable_add_def(st, a->vararg->arg, DEF_PARAM, LOCATION(a->vararg)))
1893 1 : return 0;
1894 24294 : st->st_cur->ste_varargs = 1;
1895 : }
1896 295736 : if (a->kwarg) {
1897 23198 : if (!symtable_add_def(st, a->kwarg->arg, DEF_PARAM, LOCATION(a->kwarg)))
1898 1 : return 0;
1899 23197 : st->st_cur->ste_varkeywords = 1;
1900 : }
1901 295735 : return 1;
1902 : }
1903 :
1904 :
1905 : static int
1906 36970 : symtable_visit_excepthandler(struct symtable *st, excepthandler_ty eh)
1907 : {
1908 36970 : if (eh->v.ExceptHandler.type)
1909 35364 : VISIT(st, expr, eh->v.ExceptHandler.type);
1910 36970 : if (eh->v.ExceptHandler.name)
1911 7263 : if (!symtable_add_def(st, eh->v.ExceptHandler.name, DEF_LOCAL, LOCATION(eh)))
1912 0 : return 0;
1913 86587 : VISIT_SEQ(st, stmt, eh->v.ExceptHandler.body);
1914 36970 : return 1;
1915 : }
1916 :
1917 : static int
1918 21243 : symtable_visit_withitem(struct symtable *st, withitem_ty item)
1919 : {
1920 21243 : VISIT(st, expr, item->context_expr);
1921 21243 : if (item->optional_vars) {
1922 9206 : VISIT(st, expr, item->optional_vars);
1923 : }
1924 21243 : return 1;
1925 : }
1926 :
1927 : static int
1928 984 : symtable_visit_match_case(struct symtable *st, match_case_ty m)
1929 : {
1930 984 : VISIT(st, pattern, m->pattern);
1931 984 : if (m->guard) {
1932 110 : VISIT(st, expr, m->guard);
1933 : }
1934 2078 : VISIT_SEQ(st, stmt, m->body);
1935 984 : return 1;
1936 : }
1937 :
1938 : static int
1939 109204 : symtable_visit_alias(struct symtable *st, alias_ty a)
1940 : {
1941 : /* Compute store_name, the name actually bound by the import
1942 : operation. It is different than a->name when a->name is a
1943 : dotted package name (e.g. spam.eggs)
1944 : */
1945 : PyObject *store_name;
1946 109204 : PyObject *name = (a->asname == NULL) ? a->name : a->asname;
1947 109204 : Py_ssize_t dot = PyUnicode_FindChar(name, '.', 0,
1948 : PyUnicode_GET_LENGTH(name), 1);
1949 109204 : if (dot != -1) {
1950 2257 : store_name = PyUnicode_Substring(name, 0, dot);
1951 2257 : if (!store_name)
1952 0 : return 0;
1953 : }
1954 : else {
1955 106947 : store_name = name;
1956 106947 : Py_INCREF(store_name);
1957 : }
1958 109204 : if (!_PyUnicode_EqualToASCIIString(name, "*")) {
1959 108178 : int r = symtable_add_def(st, store_name, DEF_IMPORT, LOCATION(a));
1960 108178 : Py_DECREF(store_name);
1961 108178 : return r;
1962 : }
1963 : else {
1964 1026 : if (st->st_cur->ste_type != ModuleBlock) {
1965 7 : int lineno = a->lineno;
1966 7 : int col_offset = a->col_offset;
1967 7 : int end_lineno = a->end_lineno;
1968 7 : int end_col_offset = a->end_col_offset;
1969 7 : PyErr_SetString(PyExc_SyntaxError, IMPORT_STAR_WARNING);
1970 7 : PyErr_RangedSyntaxLocationObject(st->st_filename,
1971 : lineno, col_offset + 1,
1972 : end_lineno, end_col_offset + 1);
1973 7 : Py_DECREF(store_name);
1974 7 : return 0;
1975 : }
1976 1019 : Py_DECREF(store_name);
1977 1019 : return 1;
1978 : }
1979 : }
1980 :
1981 :
1982 : static int
1983 651 : symtable_visit_comprehension(struct symtable *st, comprehension_ty lc)
1984 : {
1985 651 : st->st_cur->ste_comp_iter_target = 1;
1986 651 : VISIT(st, expr, lc->target);
1987 638 : st->st_cur->ste_comp_iter_target = 0;
1988 638 : st->st_cur->ste_comp_iter_expr++;
1989 638 : VISIT(st, expr, lc->iter);
1990 631 : st->st_cur->ste_comp_iter_expr--;
1991 723 : VISIT_SEQ(st, expr, lc->ifs);
1992 629 : if (lc->is_async) {
1993 13 : st->st_cur->ste_coroutine = 1;
1994 : }
1995 629 : return 1;
1996 : }
1997 :
1998 :
1999 : static int
2000 134926 : symtable_visit_keyword(struct symtable *st, keyword_ty k)
2001 : {
2002 134926 : VISIT(st, expr, k->value);
2003 134926 : return 1;
2004 : }
2005 :
2006 :
2007 : static int
2008 18169 : symtable_handle_comprehension(struct symtable *st, expr_ty e,
2009 : identifier scope_name, asdl_comprehension_seq *generators,
2010 : expr_ty elt, expr_ty value)
2011 : {
2012 18169 : int is_generator = (e->kind == GeneratorExp_kind);
2013 18169 : comprehension_ty outermost = ((comprehension_ty)
2014 : asdl_seq_GET(generators, 0));
2015 : /* Outermost iterator is evaluated in current scope */
2016 18169 : st->st_cur->ste_comp_iter_expr++;
2017 18169 : VISIT(st, expr, outermost->iter);
2018 18115 : st->st_cur->ste_comp_iter_expr--;
2019 : /* Create comprehension scope for the rest */
2020 36230 : if (!scope_name ||
2021 18115 : !symtable_enter_block(st, scope_name, FunctionBlock, (void *)e,
2022 : e->lineno, e->col_offset,
2023 : e->end_lineno, e->end_col_offset)) {
2024 0 : return 0;
2025 : }
2026 18115 : switch(e->kind) {
2027 8620 : case ListComp_kind:
2028 8620 : st->st_cur->ste_comprehension = ListComprehension;
2029 8620 : break;
2030 480 : case SetComp_kind:
2031 480 : st->st_cur->ste_comprehension = SetComprehension;
2032 480 : break;
2033 1191 : case DictComp_kind:
2034 1191 : st->st_cur->ste_comprehension = DictComprehension;
2035 1191 : break;
2036 7824 : default:
2037 7824 : st->st_cur->ste_comprehension = GeneratorExpression;
2038 7824 : break;
2039 : }
2040 18115 : if (outermost->is_async) {
2041 88 : st->st_cur->ste_coroutine = 1;
2042 : }
2043 :
2044 : /* Outermost iter is received as an argument */
2045 18115 : if (!symtable_implicit_arg(st, 0)) {
2046 0 : symtable_exit_block(st);
2047 0 : return 0;
2048 : }
2049 : /* Visit iteration variable target, and mark them as such */
2050 18115 : st->st_cur->ste_comp_iter_target = 1;
2051 18115 : VISIT(st, expr, outermost->target);
2052 18115 : st->st_cur->ste_comp_iter_target = 0;
2053 : /* Visit the rest of the comprehension body */
2054 22294 : VISIT_SEQ(st, expr, outermost->ifs);
2055 18735 : VISIT_SEQ_TAIL(st, comprehension, generators, 1);
2056 18084 : if (value)
2057 1191 : VISIT(st, expr, value);
2058 18083 : VISIT(st, expr, elt);
2059 18056 : st->st_cur->ste_generator = is_generator;
2060 18056 : int is_async = st->st_cur->ste_coroutine && !is_generator;
2061 18056 : if (!symtable_exit_block(st)) {
2062 0 : return 0;
2063 : }
2064 18056 : if (is_async) {
2065 111 : st->st_cur->ste_coroutine = 1;
2066 : }
2067 18056 : return 1;
2068 : }
2069 :
2070 : static int
2071 7824 : symtable_visit_genexp(struct symtable *st, expr_ty e)
2072 : {
2073 7824 : return symtable_handle_comprehension(st, e, &_Py_ID(genexpr),
2074 : e->v.GeneratorExp.generators,
2075 : e->v.GeneratorExp.elt, NULL);
2076 : }
2077 :
2078 : static int
2079 8647 : symtable_visit_listcomp(struct symtable *st, expr_ty e)
2080 : {
2081 8647 : return symtable_handle_comprehension(st, e, &_Py_ID(listcomp),
2082 : e->v.ListComp.generators,
2083 : e->v.ListComp.elt, NULL);
2084 : }
2085 :
2086 : static int
2087 507 : symtable_visit_setcomp(struct symtable *st, expr_ty e)
2088 : {
2089 507 : return symtable_handle_comprehension(st, e, &_Py_ID(setcomp),
2090 : e->v.SetComp.generators,
2091 : e->v.SetComp.elt, NULL);
2092 : }
2093 :
2094 : static int
2095 1191 : symtable_visit_dictcomp(struct symtable *st, expr_ty e)
2096 : {
2097 1191 : return symtable_handle_comprehension(st, e, &_Py_ID(dictcomp),
2098 : e->v.DictComp.generators,
2099 : e->v.DictComp.key,
2100 : e->v.DictComp.value);
2101 : }
2102 :
2103 : static int
2104 15959 : symtable_raise_if_annotation_block(struct symtable *st, const char *name, expr_ty e)
2105 : {
2106 15959 : if (st->st_cur->ste_type != AnnotationBlock) {
2107 15950 : return 1;
2108 : }
2109 :
2110 9 : PyErr_Format(PyExc_SyntaxError, ANNOTATION_NOT_ALLOWED, name);
2111 9 : PyErr_RangedSyntaxLocationObject(st->st_filename,
2112 : e->lineno,
2113 9 : e->col_offset + 1,
2114 : e->end_lineno,
2115 9 : e->end_col_offset + 1);
2116 9 : return 0;
2117 : }
2118 :
2119 : static int
2120 11 : symtable_raise_if_comprehension_block(struct symtable *st, expr_ty e) {
2121 11 : _Py_comprehension_ty type = st->st_cur->ste_comprehension;
2122 15 : PyErr_SetString(PyExc_SyntaxError,
2123 4 : (type == ListComprehension) ? "'yield' inside list comprehension" :
2124 : (type == SetComprehension) ? "'yield' inside set comprehension" :
2125 : (type == DictComprehension) ? "'yield' inside dict comprehension" :
2126 : "'yield' inside generator expression");
2127 11 : PyErr_RangedSyntaxLocationObject(st->st_filename,
2128 11 : e->lineno, e->col_offset + 1,
2129 11 : e->end_lineno, e->end_col_offset + 1);
2130 11 : VISIT_QUIT(st, 0);
2131 : }
2132 :
2133 : struct symtable *
2134 15 : _Py_SymtableStringObjectFlags(const char *str, PyObject *filename,
2135 : int start, PyCompilerFlags *flags)
2136 : {
2137 : struct symtable *st;
2138 : mod_ty mod;
2139 : PyArena *arena;
2140 :
2141 15 : arena = _PyArena_New();
2142 15 : if (arena == NULL)
2143 0 : return NULL;
2144 :
2145 15 : mod = _PyParser_ASTFromString(str, filename, start, flags, arena);
2146 15 : if (mod == NULL) {
2147 1 : _PyArena_Free(arena);
2148 1 : return NULL;
2149 : }
2150 14 : PyFutureFeatures *future = _PyFuture_FromAST(mod, filename);
2151 14 : if (future == NULL) {
2152 0 : _PyArena_Free(arena);
2153 0 : return NULL;
2154 : }
2155 14 : future->ff_features |= flags->cf_flags;
2156 14 : st = _PySymtable_Build(mod, filename, future);
2157 14 : PyObject_Free((void *)future);
2158 14 : _PyArena_Free(arena);
2159 14 : return st;
2160 : }
|
