LCOV - code coverage report
Current view: top level - Parser - pegen.h (source / functions) Hit Total Coverage
Test: CPython lcov report Lines: 37 45 82.2 %
Date: 2022-07-07 18:19:46 Functions: 6 6 100.0 %

          Line data    Source code
       1             : #ifndef PEGEN_H
       2             : #define PEGEN_H
       3             : 
       4             : #define PY_SSIZE_T_CLEAN
       5             : #include <Python.h>
       6             : #include <pycore_ast.h>
       7             : #include <pycore_token.h>
       8             : 
       9             : #if 0
      10             : #define PyPARSE_YIELD_IS_KEYWORD        0x0001
      11             : #endif
      12             : 
      13             : #define PyPARSE_DONT_IMPLY_DEDENT       0x0002
      14             : 
      15             : #if 0
      16             : #define PyPARSE_WITH_IS_KEYWORD         0x0003
      17             : #define PyPARSE_PRINT_IS_FUNCTION       0x0004
      18             : #define PyPARSE_UNICODE_LITERALS        0x0008
      19             : #endif
      20             : 
      21             : #define PyPARSE_IGNORE_COOKIE 0x0010
      22             : #define PyPARSE_BARRY_AS_BDFL 0x0020
      23             : #define PyPARSE_TYPE_COMMENTS 0x0040
      24             : #define PyPARSE_ASYNC_HACKS   0x0080
      25             : #define PyPARSE_ALLOW_INCOMPLETE_INPUT 0x0100
      26             : 
      27             : #define CURRENT_POS (-5)
      28             : 
      29             : typedef struct _memo {
      30             :     int type;
      31             :     void *node;
      32             :     int mark;
      33             :     struct _memo *next;
      34             : } Memo;
      35             : 
      36             : typedef struct {
      37             :     int type;
      38             :     PyObject *bytes;
      39             :     int level;
      40             :     int lineno, col_offset, end_lineno, end_col_offset;
      41             :     Memo *memo;
      42             : } Token;
      43             : 
      44             : typedef struct {
      45             :     char *str;
      46             :     int type;
      47             : } KeywordToken;
      48             : 
      49             : 
      50             : typedef struct {
      51             :     struct {
      52             :         int lineno;
      53             :         char *comment;  // The " <tag>" in "# type: ignore <tag>"
      54             :     } *items;
      55             :     size_t size;
      56             :     size_t num_items;
      57             : } growable_comment_array;
      58             : 
      59             : typedef struct {
      60             :     struct tok_state *tok;
      61             :     Token **tokens;
      62             :     int mark;
      63             :     int fill, size;
      64             :     PyArena *arena;
      65             :     KeywordToken **keywords;
      66             :     char **soft_keywords;
      67             :     int n_keyword_lists;
      68             :     int start_rule;
      69             :     int *errcode;
      70             :     int parsing_started;
      71             :     PyObject* normalize;
      72             :     int starting_lineno;
      73             :     int starting_col_offset;
      74             :     int error_indicator;
      75             :     int flags;
      76             :     int feature_version;
      77             :     growable_comment_array type_ignore_comments;
      78             :     Token *known_err_token;
      79             :     int level;
      80             :     int call_invalid_rules;
      81             :     int debug;
      82             : } Parser;
      83             : 
      84             : typedef struct {
      85             :     cmpop_ty cmpop;
      86             :     expr_ty expr;
      87             : } CmpopExprPair;
      88             : 
      89             : typedef struct {
      90             :     expr_ty key;
      91             :     expr_ty value;
      92             : } KeyValuePair;
      93             : 
      94             : typedef struct {
      95             :     expr_ty key;
      96             :     pattern_ty pattern;
      97             : } KeyPatternPair;
      98             : 
      99             : typedef struct {
     100             :     arg_ty arg;
     101             :     expr_ty value;
     102             : } NameDefaultPair;
     103             : 
     104             : typedef struct {
     105             :     asdl_arg_seq *plain_names;
     106             :     asdl_seq *names_with_defaults; // asdl_seq* of NameDefaultsPair's
     107             : } SlashWithDefault;
     108             : 
     109             : typedef struct {
     110             :     arg_ty vararg;
     111             :     asdl_seq *kwonlyargs; // asdl_seq* of NameDefaultsPair's
     112             :     arg_ty kwarg;
     113             : } StarEtc;
     114             : 
     115             : typedef struct { operator_ty kind; } AugOperator;
     116             : typedef struct {
     117             :     void *element;
     118             :     int is_keyword;
     119             : } KeywordOrStarred;
     120             : 
     121             : // Internal parser functions
     122             : #if defined(Py_DEBUG)
     123             : void _PyPegen_clear_memo_statistics(void);
     124             : PyObject *_PyPegen_get_memo_statistics(void);
     125             : #endif
     126             : 
     127             : int _PyPegen_insert_memo(Parser *p, int mark, int type, void *node);
     128             : int _PyPegen_update_memo(Parser *p, int mark, int type, void *node);
     129             : int _PyPegen_is_memoized(Parser *p, int type, void *pres);
     130             : 
     131             : int _PyPegen_lookahead_with_name(int, expr_ty (func)(Parser *), Parser *);
     132             : int _PyPegen_lookahead_with_int(int, Token *(func)(Parser *, int), Parser *, int);
     133             : int _PyPegen_lookahead_with_string(int , expr_ty (func)(Parser *, const char*), Parser *, const char*);
     134             : int _PyPegen_lookahead(int, void *(func)(Parser *), Parser *);
     135             : 
     136             : Token *_PyPegen_expect_token(Parser *p, int type);
     137             : void* _PyPegen_expect_forced_result(Parser *p, void* result, const char* expected);
     138             : Token *_PyPegen_expect_forced_token(Parser *p, int type, const char* expected);
     139             : expr_ty _PyPegen_expect_soft_keyword(Parser *p, const char *keyword);
     140             : expr_ty _PyPegen_soft_keyword_token(Parser *p);
     141             : Token *_PyPegen_get_last_nonnwhitespace_token(Parser *);
     142             : int _PyPegen_fill_token(Parser *p);
     143             : expr_ty _PyPegen_name_token(Parser *p);
     144             : expr_ty _PyPegen_number_token(Parser *p);
     145             : void *_PyPegen_string_token(Parser *p);
     146             : Py_ssize_t _PyPegen_byte_offset_to_character_offset(PyObject *line, Py_ssize_t col_offset);
     147             : 
     148             : // Error handling functions and APIs
     149             : typedef enum {
     150             :     STAR_TARGETS,
     151             :     DEL_TARGETS,
     152             :     FOR_TARGETS
     153             : } TARGETS_TYPE;
     154             : 
     155             : int _Pypegen_raise_decode_error(Parser *p);
     156             : void _PyPegen_raise_tokenizer_init_error(PyObject *filename);
     157             : int _Pypegen_tokenizer_error(Parser *p);
     158             : void *_PyPegen_raise_error(Parser *p, PyObject *errtype, const char *errmsg, ...);
     159             : void *_PyPegen_raise_error_known_location(Parser *p, PyObject *errtype,
     160             :                                           Py_ssize_t lineno, Py_ssize_t col_offset,
     161             :                                           Py_ssize_t end_lineno, Py_ssize_t end_col_offset,
     162             :                                           const char *errmsg, va_list va);
     163             : void _Pypegen_set_syntax_error(Parser* p, Token* last_token);
     164             : Py_LOCAL_INLINE(void *)
     165        1050 : RAISE_ERROR_KNOWN_LOCATION(Parser *p, PyObject *errtype,
     166             :                            Py_ssize_t lineno, Py_ssize_t col_offset,
     167             :                            Py_ssize_t end_lineno, Py_ssize_t end_col_offset,
     168             :                            const char *errmsg, ...)
     169             : {
     170             :     va_list va;
     171        1050 :     va_start(va, errmsg);
     172        1050 :     Py_ssize_t _col_offset = (col_offset == CURRENT_POS ? CURRENT_POS : col_offset + 1);
     173        1050 :     Py_ssize_t _end_col_offset = (end_col_offset == CURRENT_POS ? CURRENT_POS : end_col_offset + 1);
     174        1050 :     _PyPegen_raise_error_known_location(p, errtype, lineno, _col_offset, end_lineno, _end_col_offset, errmsg, va);
     175        1050 :     va_end(va);
     176        1050 :     return NULL;
     177             : }
     178             : #define RAISE_SYNTAX_ERROR(msg, ...) _PyPegen_raise_error(p, PyExc_SyntaxError, msg, ##__VA_ARGS__)
     179             : #define RAISE_INDENTATION_ERROR(msg, ...) _PyPegen_raise_error(p, PyExc_IndentationError, msg, ##__VA_ARGS__)
     180             : #define RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, msg, ...) \
     181             :     RAISE_ERROR_KNOWN_LOCATION(p, PyExc_SyntaxError, (a)->lineno, (a)->col_offset, (b)->end_lineno, (b)->end_col_offset, msg, ##__VA_ARGS__)
     182             : #define RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, msg, ...) \
     183             :     RAISE_ERROR_KNOWN_LOCATION(p, PyExc_SyntaxError, (a)->lineno, (a)->col_offset, (a)->end_lineno, (a)->end_col_offset, msg, ##__VA_ARGS__)
     184             : #define RAISE_SYNTAX_ERROR_STARTING_FROM(a, msg, ...) \
     185             :     RAISE_ERROR_KNOWN_LOCATION(p, PyExc_SyntaxError, (a)->lineno, (a)->col_offset, CURRENT_POS, CURRENT_POS, msg, ##__VA_ARGS__)
     186             : #define RAISE_SYNTAX_ERROR_INVALID_TARGET(type, e) _RAISE_SYNTAX_ERROR_INVALID_TARGET(p, type, e)
     187             : 
     188             : Py_LOCAL_INLINE(void *)
     189     1264110 : CHECK_CALL(Parser *p, void *result)
     190             : {
     191     1264110 :     if (result == NULL) {
     192           0 :         assert(PyErr_Occurred());
     193           0 :         p->error_indicator = 1;
     194             :     }
     195     1264110 :     return result;
     196             : }
     197             : 
     198             : /* This is needed for helper functions that are allowed to
     199             :    return NULL without an error. Example: _PyPegen_seq_extract_starred_exprs */
     200             : Py_LOCAL_INLINE(void *)
     201       77720 : CHECK_CALL_NULL_ALLOWED(Parser *p, void *result)
     202             : {
     203       77720 :     if (result == NULL && PyErr_Occurred()) {
     204           0 :         p->error_indicator = 1;
     205             :     }
     206       77720 :     return result;
     207             : }
     208             : 
     209             : #define CHECK(type, result) ((type) CHECK_CALL(p, result))
     210             : #define CHECK_NULL_ALLOWED(type, result) ((type) CHECK_CALL_NULL_ALLOWED(p, result))
     211             : 
     212             : expr_ty _PyPegen_get_invalid_target(expr_ty e, TARGETS_TYPE targets_type);
     213             : const char *_PyPegen_get_expr_name(expr_ty);
     214             : Py_LOCAL_INLINE(void *)
     215          84 : _RAISE_SYNTAX_ERROR_INVALID_TARGET(Parser *p, TARGETS_TYPE type, void *e)
     216             : {
     217          84 :     expr_ty invalid_target = CHECK_NULL_ALLOWED(expr_ty, _PyPegen_get_invalid_target(e, type));
     218          84 :     if (invalid_target != NULL) {
     219             :         const char *msg;
     220          75 :         if (type == STAR_TARGETS || type == FOR_TARGETS) {
     221          46 :             msg = "cannot assign to %s";
     222             :         }
     223             :         else {
     224          29 :             msg = "cannot delete %s";
     225             :         }
     226          75 :         return RAISE_SYNTAX_ERROR_KNOWN_LOCATION(
     227             :             invalid_target,
     228             :             msg,
     229             :             _PyPegen_get_expr_name(invalid_target)
     230             :         );
     231             :         return RAISE_SYNTAX_ERROR_KNOWN_LOCATION(invalid_target, "invalid syntax");
     232             :     }
     233           9 :     return NULL;
     234             : }
     235             : 
     236             : // Action utility functions
     237             : 
     238             : void *_PyPegen_dummy_name(Parser *p, ...);
     239             : void * _PyPegen_seq_last_item(asdl_seq *seq);
     240             : #define PyPegen_last_item(seq, type) ((type)_PyPegen_seq_last_item((asdl_seq*)seq))
     241             : void * _PyPegen_seq_first_item(asdl_seq *seq);
     242             : #define PyPegen_first_item(seq, type) ((type)_PyPegen_seq_first_item((asdl_seq*)seq))
     243             : #define UNUSED(expr) do { (void)(expr); } while (0)
     244             : #define EXTRA_EXPR(head, tail) head->lineno, (head)->col_offset, (tail)->end_lineno, (tail)->end_col_offset, p->arena
     245             : #define EXTRA _start_lineno, _start_col_offset, _end_lineno, _end_col_offset, p->arena
     246             : PyObject *_PyPegen_new_type_comment(Parser *, const char *);
     247             : 
     248             : Py_LOCAL_INLINE(PyObject *)
     249     1250870 : NEW_TYPE_COMMENT(Parser *p, Token *tc)
     250             : {
     251     1250870 :     if (tc == NULL) {
     252     1250770 :         return NULL;
     253             :     }
     254          92 :     const char *bytes = PyBytes_AsString(tc->bytes);
     255          92 :     if (bytes == NULL) {
     256           0 :         goto error;
     257             :     }
     258          92 :     PyObject *tco = _PyPegen_new_type_comment(p, bytes);
     259          92 :     if (tco == NULL) {
     260           0 :         goto error;
     261             :     }
     262          92 :     return tco;
     263           0 :  error:
     264           0 :     p->error_indicator = 1;  // Inline CHECK_CALL
     265           0 :     return NULL;
     266             : }
     267             : 
     268             : Py_LOCAL_INLINE(void *)
     269       19698 : INVALID_VERSION_CHECK(Parser *p, int version, char *msg, void *node)
     270             : {
     271       19698 :     if (node == NULL) {
     272           1 :         p->error_indicator = 1;  // Inline CHECK_CALL
     273           1 :         return NULL;
     274             :     }
     275       19697 :     if (p->feature_version < version) {
     276           4 :         p->error_indicator = 1;
     277           4 :         return RAISE_SYNTAX_ERROR("%s only supported in Python 3.%i and greater",
     278             :                                   msg, version);
     279             :     }
     280       19693 :     return node;
     281             : }
     282             : 
     283             : #define CHECK_VERSION(type, version, msg, node) ((type) INVALID_VERSION_CHECK(p, version, msg, node))
     284             : 
     285             : arg_ty _PyPegen_add_type_comment_to_arg(Parser *, arg_ty, Token *);
     286             : PyObject *_PyPegen_new_identifier(Parser *, const char *);
     287             : asdl_seq *_PyPegen_singleton_seq(Parser *, void *);
     288             : asdl_seq *_PyPegen_seq_insert_in_front(Parser *, void *, asdl_seq *);
     289             : asdl_seq *_PyPegen_seq_append_to_end(Parser *, asdl_seq *, void *);
     290             : asdl_seq *_PyPegen_seq_flatten(Parser *, asdl_seq *);
     291             : expr_ty _PyPegen_join_names_with_dot(Parser *, expr_ty, expr_ty);
     292             : int _PyPegen_seq_count_dots(asdl_seq *);
     293             : alias_ty _PyPegen_alias_for_star(Parser *, int, int, int, int, PyArena *);
     294             : asdl_identifier_seq *_PyPegen_map_names_to_ids(Parser *, asdl_expr_seq *);
     295             : CmpopExprPair *_PyPegen_cmpop_expr_pair(Parser *, cmpop_ty, expr_ty);
     296             : asdl_int_seq *_PyPegen_get_cmpops(Parser *p, asdl_seq *);
     297             : asdl_expr_seq *_PyPegen_get_exprs(Parser *, asdl_seq *);
     298             : expr_ty _PyPegen_set_expr_context(Parser *, expr_ty, expr_context_ty);
     299             : KeyValuePair *_PyPegen_key_value_pair(Parser *, expr_ty, expr_ty);
     300             : asdl_expr_seq *_PyPegen_get_keys(Parser *, asdl_seq *);
     301             : asdl_expr_seq *_PyPegen_get_values(Parser *, asdl_seq *);
     302             : KeyPatternPair *_PyPegen_key_pattern_pair(Parser *, expr_ty, pattern_ty);
     303             : asdl_expr_seq *_PyPegen_get_pattern_keys(Parser *, asdl_seq *);
     304             : asdl_pattern_seq *_PyPegen_get_patterns(Parser *, asdl_seq *);
     305             : NameDefaultPair *_PyPegen_name_default_pair(Parser *, arg_ty, expr_ty, Token *);
     306             : SlashWithDefault *_PyPegen_slash_with_default(Parser *, asdl_arg_seq *, asdl_seq *);
     307             : StarEtc *_PyPegen_star_etc(Parser *, arg_ty, asdl_seq *, arg_ty);
     308             : arguments_ty _PyPegen_make_arguments(Parser *, asdl_arg_seq *, SlashWithDefault *,
     309             :                                      asdl_arg_seq *, asdl_seq *, StarEtc *);
     310             : arguments_ty _PyPegen_empty_arguments(Parser *);
     311             : AugOperator *_PyPegen_augoperator(Parser*, operator_ty type);
     312             : stmt_ty _PyPegen_function_def_decorators(Parser *, asdl_expr_seq *, stmt_ty);
     313             : stmt_ty _PyPegen_class_def_decorators(Parser *, asdl_expr_seq *, stmt_ty);
     314             : KeywordOrStarred *_PyPegen_keyword_or_starred(Parser *, void *, int);
     315             : asdl_expr_seq *_PyPegen_seq_extract_starred_exprs(Parser *, asdl_seq *);
     316             : asdl_keyword_seq *_PyPegen_seq_delete_starred_exprs(Parser *, asdl_seq *);
     317             : expr_ty _PyPegen_collect_call_seqs(Parser *, asdl_expr_seq *, asdl_seq *,
     318             :                      int lineno, int col_offset, int end_lineno,
     319             :                      int end_col_offset, PyArena *arena);
     320             : expr_ty _PyPegen_concatenate_strings(Parser *p, asdl_seq *);
     321             : expr_ty _PyPegen_ensure_imaginary(Parser *p, expr_ty);
     322             : expr_ty _PyPegen_ensure_real(Parser *p, expr_ty);
     323             : asdl_seq *_PyPegen_join_sequences(Parser *, asdl_seq *, asdl_seq *);
     324             : int _PyPegen_check_barry_as_flufl(Parser *, Token *);
     325             : int _PyPegen_check_legacy_stmt(Parser *p, expr_ty t);
     326             : mod_ty _PyPegen_make_module(Parser *, asdl_stmt_seq *);
     327             : void *_PyPegen_arguments_parsing_error(Parser *, expr_ty);
     328             : expr_ty _PyPegen_get_last_comprehension_item(comprehension_ty comprehension);
     329             : void *_PyPegen_nonparen_genexp_in_call(Parser *p, expr_ty args, asdl_comprehension_seq *comprehensions);
     330             : 
     331             : // Parser API
     332             : 
     333             : Parser *_PyPegen_Parser_New(struct tok_state *, int, int, int, int *, PyArena *);
     334             : void _PyPegen_Parser_Free(Parser *);
     335             : mod_ty _PyPegen_run_parser_from_file_pointer(FILE *, int, PyObject *, const char *,
     336             :                                     const char *, const char *, PyCompilerFlags *, int *, PyArena *);
     337             : void *_PyPegen_run_parser(Parser *);
     338             : mod_ty _PyPegen_run_parser_from_string(const char *, int, PyObject *, PyCompilerFlags *, PyArena *);
     339             : asdl_stmt_seq *_PyPegen_interactive_exit(Parser *);
     340             : 
     341             : // Generated function in parse.c - function definition in python.gram
     342             : void *_PyPegen_parse(Parser *);
     343             : 
     344             : #endif

Generated by: LCOV version 1.14