Search results for query "top-pypi-packages"

21 messages

[Python-checkins] bpo-40334: PEP 617 implementation: New PEG parser for CPython (GH-19503)

by Pablo Galindo

https://github.com/python/cpython/commit/c5fc15685202cda73f7c3f5c6f299b0945… commit: c5fc15685202cda73f7c3f5c6f299b0945f58508 branch: master author: Pablo Galindo <Pablogsal(a)gmail.com> committer: GitHub <noreply(a)github.com> date: 2020-04-22T23:29:27+01:00 summary: bpo-40334: PEP 617 implementation: New PEG parser for CPython (GH-19503) Co-authored-by: Guido van Rossum <guido(a)python.org> Co-authored-by: Lysandros Nikolaou <lisandrosnik(a)gmail.com> files: A Grammar/python.gram A Include/pegen_interface.h A Lib/test/test_peg_generator/__init__.py A Lib/test/test_peg_generator/__main__.py A Lib/test/test_peg_generator/ast_dump.py A Lib/test/test_peg_generator/test_c_parser.py A Lib/test/test_peg_generator/test_first_sets.py A Lib/test/test_peg_generator/test_pegen.py A Lib/test/test_peg_parser.py A Misc/NEWS.d/next/Core and Builtins/2020-04-20-14-06-19.bpo-40334.CTLGEp.rst A Modules/_peg_parser.c A Parser/pegen/parse.c A Parser/pegen/parse_string.c A Parser/pegen/parse_string.h A Parser/pegen/peg_api.c A Parser/pegen/pegen.c A Parser/pegen/pegen.h A Tools/peg_generator/.clang-format A Tools/peg_generator/.gitignore A Tools/peg_generator/Makefile A Tools/peg_generator/data/cprog.py A Tools/peg_generator/data/xxl.zip A Tools/peg_generator/mypy.ini A Tools/peg_generator/peg_extension/peg_extension.c A Tools/peg_generator/pegen/__init__.py A Tools/peg_generator/pegen/__main__.py A Tools/peg_generator/pegen/build.py A Tools/peg_generator/pegen/c_generator.py A Tools/peg_generator/pegen/first_sets.py A Tools/peg_generator/pegen/grammar.py A Tools/peg_generator/pegen/grammar_parser.py A Tools/peg_generator/pegen/grammar_visualizer.py A Tools/peg_generator/pegen/metagrammar.gram A Tools/peg_generator/pegen/parser.py A Tools/peg_generator/pegen/parser_generator.py A Tools/peg_generator/pegen/python_generator.py A Tools/peg_generator/pegen/sccutils.py A Tools/peg_generator/pegen/testutil.py A Tools/peg_generator/pegen/tokenizer.py A Tools/peg_generator/pyproject.toml A Tools/peg_generator/requirements.pip A Tools/peg_generator/scripts/__init__.py A Tools/peg_generator/scripts/ast_timings.py A Tools/peg_generator/scripts/benchmark.py A Tools/peg_generator/scripts/download_pypi_packages.py A Tools/peg_generator/scripts/find_max_nesting.py A Tools/peg_generator/scripts/grammar_grapher.py A Tools/peg_generator/scripts/joinstats.py A Tools/peg_generator/scripts/show_parse.py A Tools/peg_generator/scripts/test_parse_directory.py A Tools/peg_generator/scripts/test_pypi_packages.py M .github/workflows/build.yml M .travis.yml M Doc/using/cmdline.rst M Include/compile.h M Include/cpython/initconfig.h M Lib/test/test_cmd_line_script.py M Lib/test/test_codeop.py M Lib/test/test_compile.py M Lib/test/test_embed.py M Lib/test/test_eof.py M Lib/test/test_exceptions.py M Lib/test/test_flufl.py M Lib/test/test_fstring.py M Lib/test/test_generators.py M Lib/test/test_parser.py M Lib/test/test_positional_only_arg.py M Lib/test/test_string_literals.py M Lib/test/test_syntax.py M Lib/test/test_sys.py M Lib/test/test_traceback.py M Lib/test/test_type_comments.py M Lib/test/test_unpack_ex.py M Lib/test/test_unparse.py M Makefile.pre.in M Modules/Setup M PC/config.c M PCbuild/pythoncore.vcxproj M PCbuild/pythoncore.vcxproj.filters M PCbuild/regen.vcxproj M Programs/_testembed.c M Python/ast_opt.c M Python/bltinmodule.c M Python/compile.c M Python/importlib.h M Python/importlib_external.h M Python/initconfig.c M Python/pythonrun.c M Python/sysmodule.c M Tools/README M Tools/scripts/run_tests.py diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 50d1561518bd8..c9e9c53da23fd 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -13,6 +13,7 @@ on: - '**/*.rst' pull_request: branches: + - pegen - master - 3.8 - 3.7 @@ -50,6 +51,22 @@ jobs: build_macos: name: 'macOS' runs-on: macos-latest + env: + PYTHONOLDPARSER: old + steps: + - uses: actions/checkout@v1 + - name: Configure CPython + run: ./configure --with-pydebug --with-openssl=/usr/local/opt/openssl --prefix=/opt/python-dev + - name: Build CPython + run: make -j4 + - name: Display build info + run: make pythoninfo + - name: Tests + run: make buildbottest TESTOPTS="-j4 -uall,-cpu" + + build_macos_pegen: + name: 'macOS - Pegen' + runs-on: macos-latest steps: - uses: actions/checkout@v1 - name: Configure CPython @@ -64,6 +81,34 @@ jobs: build_ubuntu: name: 'Ubuntu' runs-on: ubuntu-latest + env: + OPENSSL_VER: 1.1.1f + PYTHONOLDPARSER: old + steps: + - uses: actions/checkout@v1 + - name: Install Dependencies + run: sudo ./.github/workflows/posix-deps-apt.sh + - name: 'Restore OpenSSL build' + id: cache-openssl + uses: actions/cache@v1 + with: + path: ./multissl/openssl/${{ env.OPENSSL_VER }} + key: ${{ runner.os }}-multissl-openssl-${{ env.OPENSSL_VER }} + - name: Install OpenSSL + if: steps.cache-openssl.outputs.cache-hit != 'true' + run: python3 Tools/ssl/multissltests.py --steps=library --base-directory $PWD/multissl --openssl $OPENSSL_VER --system Linux + - name: Configure CPython + run: ./configure --with-pydebug --with-openssl=$PWD/multissl/openssl/$OPENSSL_VER + - name: Build CPython + run: make -j4 + - name: Display build info + run: make pythoninfo + - name: Tests + run: xvfb-run make buildbottest TESTOPTS="-j4 -uall,-cpu" + + build_ubuntu_pegen: + name: 'Ubuntu - Pegen' + runs-on: ubuntu-latest env: OPENSSL_VER: 1.1.1f steps: diff --git a/.travis.yml b/.travis.yml index c7fa9e3a7ca4f..80d7a16318adf 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,5 +1,5 @@ language: c -dist: xenial +dist: bionic # To cache doc-building dependencies and C compiler output. cache: @@ -22,6 +22,7 @@ env: branches: only: - master + - pegen - /^\d\.\d+$/ - buildbot-custom @@ -157,7 +158,9 @@ install: before_script: # -Og is much faster than -O0 - CFLAGS="${CFLAGS} -Og" ./configure --with-pydebug - - make -j4 regen-all + - eval "$(pyenv init -)" + - pyenv global 3.8 + - PYTHON_FOR_REGEN=python3.8 make -j4 regen-all - changes=`git status --porcelain` - | # Check for changes in regenerated files diff --git a/Doc/using/cmdline.rst b/Doc/using/cmdline.rst index 9b30c288211ed..a815436b3a8ee 100644 --- a/Doc/using/cmdline.rst +++ b/Doc/using/cmdline.rst @@ -426,6 +426,8 @@ Miscellaneous options defines the following possible values: * ``-X faulthandler`` to enable :mod:`faulthandler`; + * ``-X oldparser``: enable the traditional LL(1) parser. See also + :envvar:`PYTHONOLDPARSER`. * ``-X showrefcount`` to output the total reference count and number of used memory blocks when the program finishes or after each statement in the interactive interpreter. This only works on debug builds. @@ -574,6 +576,12 @@ conflict. :option:`-d` multiple times. +.. envvar:: PYTHONOLDPARSER + + If this is set it is equivalent to specifying the :option:`-X` + ``oldparser`` option. + + .. envvar:: PYTHONINSPECT If this is set to a non-empty string it is equivalent to specifying the diff --git a/Grammar/python.gram b/Grammar/python.gram new file mode 100644 index 0000000000000..40ca3dc8d12a5 --- /dev/null +++ b/Grammar/python.gram @@ -0,0 +1,555 @@ +# Simplified grammar for Python + +@bytecode True +@trailer ''' +void * +_PyPegen_parse(Parser *p) +{ + // Initialize keywords + p->keywords = reserved_keywords; + p->n_keyword_lists = n_keyword_lists; + + // Run parser + void *result = NULL; + if (p->start_rule == Py_file_input) { + result = file_rule(p); + } else if (p->start_rule == Py_single_input) { + result = interactive_rule(p); + } else if (p->start_rule == Py_eval_input) { + result = eval_rule(p); + } else if (p->start_rule == Py_fstring_input) { + result = fstring_rule(p); + } + + return result; +} + +// The end +''' +file[mod_ty]: a=[statements] ENDMARKER { Module(a, NULL, p->arena) } +interactive[mod_ty]: a=statement_newline { Interactive(a, p->arena) } +eval[mod_ty]: a=expressions NEWLINE* ENDMARKER { Expression(a, p->arena) } +fstring[expr_ty]: star_expressions + +statements[asdl_seq*]: a=statement+ { _PyPegen_seq_flatten(p, a) } +statement[asdl_seq*]: a=compound_stmt { _PyPegen_singleton_seq(p, a) } | simple_stmt +statement_newline[asdl_seq*]: + | a=compound_stmt NEWLINE { _PyPegen_singleton_seq(p, a) } + | simple_stmt + | NEWLINE { _PyPegen_singleton_seq(p, CHECK(_Py_Pass(EXTRA))) } + | ENDMARKER { _PyPegen_interactive_exit(p) } +simple_stmt[asdl_seq*]: + | a=small_stmt !';' NEWLINE { _PyPegen_singleton_seq(p, a) } # Not needed, there for speedup + | a=';'.small_stmt+ [';'] NEWLINE { a } +# NOTE: assignment MUST precede expression, else parsing a simple assignment +# will throw a SyntaxError. +small_stmt[stmt_ty] (memo): + | assignment + | e=star_expressions { _Py_Expr(e, EXTRA) } + | &'return' return_stmt + | &('import' | 'from') import_stmt + | &'raise' raise_stmt + | 'pass' { _Py_Pass(EXTRA) } + | &'del' del_stmt + | &'yield' yield_stmt + | &'assert' assert_stmt + | 'break' { _Py_Break(EXTRA) } + | 'continue' { _Py_Continue(EXTRA) } + | &'global' global_stmt + | &'nonlocal' nonlocal_stmt +compound_stmt[stmt_ty]: + | &('def' | '@' | ASYNC) function_def + | &'if' if_stmt + | &('class' | '@') class_def + | &('with' | ASYNC) with_stmt + | &('for' | ASYNC) for_stmt + | &'try' try_stmt + | &'while' while_stmt + +# NOTE: annotated_rhs may start with 'yield'; yield_expr must start with 'yield' +assignment: + | a=NAME ':' b=expression c=['=' d=annotated_rhs { d }] { + _Py_AnnAssign(CHECK(_PyPegen_set_expr_context(p, a, Store)), b, c, 1, EXTRA) } + | a=('(' b=inside_paren_ann_assign_target ')' { b } + | ann_assign_subscript_attribute_target) ':' b=expression c=['=' d=annotated_rhs { d }] { + _Py_AnnAssign(a, b, c, 0, EXTRA)} + | a=(z=star_targets '=' { z })+ b=(yield_expr | star_expressions) { + _Py_Assign(a, b, NULL, EXTRA) } + | a=target b=augassign c=(yield_expr | star_expressions) { + _Py_AugAssign(a, b->kind, c, EXTRA) } + | invalid_assignment + +augassign[AugOperator*]: + | '+=' {_PyPegen_augoperator(p, Add)} + | '-=' {_PyPegen_augoperator(p, Sub)} + | '*=' {_PyPegen_augoperator(p, Mult)} + | '@=' {_PyPegen_augoperator(p, MatMult)} + | '/=' {_PyPegen_augoperator(p, Div)} + | '%=' {_PyPegen_augoperator(p, Mod)} + | '&=' {_PyPegen_augoperator(p, BitAnd)} + | '|=' {_PyPegen_augoperator(p, BitOr)} + | '^=' {_PyPegen_augoperator(p, BitXor)} + | '<<=' {_PyPegen_augoperator(p, LShift)} + | '>>=' {_PyPegen_augoperator(p, RShift)} + | '**=' {_PyPegen_augoperator(p, Pow)} + | '//=' {_PyPegen_augoperator(p, FloorDiv)} + +global_stmt[stmt_ty]: 'global' a=','.NAME+ { + _Py_Global(CHECK(_PyPegen_map_names_to_ids(p, a)), EXTRA) } +nonlocal_stmt[stmt_ty]: 'nonlocal' a=','.NAME+ { + _Py_Nonlocal(CHECK(_PyPegen_map_names_to_ids(p, a)), EXTRA) } + +yield_stmt[stmt_ty]: y=yield_expr { _Py_Expr(y, EXTRA) } + +assert_stmt[stmt_ty]: 'assert' a=expression b=[',' z=expression { z }] { _Py_Assert(a, b, EXTRA) } + +del_stmt[stmt_ty]: 'del' a=del_targets { _Py_Delete(a, EXTRA) } + +import_stmt[stmt_ty]: import_name | import_from +import_name[stmt_ty]: 'import' a=dotted_as_names { _Py_Import(a, EXTRA) } +# note below: the ('.' | '...') is necessary because '...' is tokenized as ELLIPSIS +import_from[stmt_ty]: + | 'from' a=('.' | '...')* b=dotted_name 'import' c=import_from_targets { + _Py_ImportFrom(b->v.Name.id, c, _PyPegen_seq_count_dots(a), EXTRA) } + | 'from' a=('.' | '...')+ 'import' b=import_from_targets { + _Py_ImportFrom(NULL, b, _PyPegen_seq_count_dots(a), EXTRA) } +import_from_targets[asdl_seq*]: + | '(' a=import_from_as_names [','] ')' { a } + | import_from_as_names + | '*' { _PyPegen_singleton_seq(p, CHECK(_PyPegen_alias_for_star(p))) } +import_from_as_names[asdl_seq*]: + | a=','.import_from_as_name+ { a } +import_from_as_name[alias_ty]: + | a=NAME b=['as' z=NAME { z }] { _Py_alias(a->v.Name.id, + (b) ? ((expr_ty) b)->v.Name.id : NULL, + p->arena) } +dotted_as_names[asdl_seq*]: + | a=','.dotted_as_name+ { a } +dotted_as_name[alias_ty]: + | a=dotted_name b=['as' z=NAME { z }] { _Py_alias(a->v.Name.id, + (b) ? ((expr_ty) b)->v.Name.id : NULL, + p->arena) } +dotted_name[expr_ty]: + | a=dotted_name '.' b=NAME { _PyPegen_join_names_with_dot(p, a, b) } + | NAME + +if_stmt[stmt_ty]: + | 'if' a=named_expression ':' b=block c=elif_stmt { _Py_If(a, b, CHECK(_PyPegen_singleton_seq(p, c)), EXTRA) } + | 'if' a=named_expression ':' b=block c=[else_block] { _Py_If(a, b, c, EXTRA) } +elif_stmt[stmt_ty]: + | 'elif' a=named_expression ':' b=block c=elif_stmt { _Py_If(a, b, CHECK(_PyPegen_singleton_seq(p, c)), EXTRA) } + | 'elif' a=named_expression ':' b=block c=[else_block] { _Py_If(a, b, c, EXTRA) } +else_block[asdl_seq*]: 'else' ':' b=block { b } + +while_stmt[stmt_ty]: + | 'while' a=named_expression ':' b=block c=[else_block] { _Py_While(a, b, c, EXTRA) } + +for_stmt[stmt_ty]: + | is_async=[ASYNC] 'for' t=star_targets 'in' ex=star_expressions ':' b=block el=[else_block] { + (is_async ? _Py_AsyncFor : _Py_For)(t, ex, b, el, NULL, EXTRA) } + +with_stmt[stmt_ty]: + | is_async=[ASYNC] 'with' '(' a=','.with_item+ ')' ':' b=block { + (is_async ? _Py_AsyncWith : _Py_With)(a, b, NULL, EXTRA) } + | is_async=[ASYNC] 'with' a=','.with_item+ ':' b=block { + (is_async ? _Py_AsyncWith : _Py_With)(a, b, NULL, EXTRA) } +with_item[withitem_ty]: + | e=expression o=['as' t=target { t }] { _Py_withitem(e, o, p->arena) } + +try_stmt[stmt_ty]: + | 'try' ':' b=block f=finally_block { _Py_Try(b, NULL, NULL, f, EXTRA) } + | 'try' ':' b=block ex=except_block+ el=[else_block] f=[finally_block] { _Py_Try(b, ex, el, f, EXTRA) } +except_block[excepthandler_ty]: + | 'except' e=expression t=['as' z=target { z }] ':' b=block { + _Py_ExceptHandler(e, (t) ? ((expr_ty) t)->v.Name.id : NULL, b, EXTRA) } + | 'except' ':' b=block { _Py_ExceptHandler(NULL, NULL, b, EXTRA) } +finally_block[asdl_seq*]: 'finally' ':' a=block { a } + +return_stmt[stmt_ty]: + | 'return' a=[star_expressions] { _Py_Return(a, EXTRA) } + +raise_stmt[stmt_ty]: + | 'raise' a=expression b=['from' z=expression { z }] { _Py_Raise(a, b, EXTRA) } + | 'raise' { _Py_Raise(NULL, NULL, EXTRA) } + +function_def[stmt_ty]: + | d=decorators f=function_def_raw { _PyPegen_function_def_decorators(p, d, f) } + | function_def_raw + +function_def_raw[stmt_ty]: + | is_async=[ASYNC] 'def' n=NAME '(' params=[params] ')' a=['->' z=annotation { z }] ':' b=block { + (is_async ? _Py_AsyncFunctionDef : _Py_FunctionDef)(n->v.Name.id, + (params) ? params : CHECK(_PyPegen_empty_arguments(p)), + b, NULL, a, NULL, EXTRA) } + +params[arguments_ty]: + | invalid_parameters + | parameters +parameters[arguments_ty]: + | a=slash_without_default b=[',' x=plain_names { x }] c=[',' y=names_with_default { y }] d=[',' z=[star_etc] { z }] { + _PyPegen_make_arguments(p, a, NULL, b, c, d) } + | a=slash_with_default b=[',' y=names_with_default { y }] c=[',' z=[star_etc] { z }] { + _PyPegen_make_arguments(p, NULL, a, NULL, b, c) } + | a=plain_names b=[',' y=names_with_default { y }] c=[',' z=[star_etc] { z }] { + _PyPegen_make_arguments(p, NULL, NULL, a, b, c) } + | a=names_with_default b=[',' z=[star_etc] { z }] { _PyPegen_make_arguments(p, NULL, NULL, NULL, a, b)} + | a=star_etc { _PyPegen_make_arguments(p, NULL, NULL, NULL, NULL, a) } +slash_without_default[asdl_seq*]: a=plain_names ',' '/' { a } +slash_with_default[SlashWithDefault*]: a=[n=plain_names ',' { n }] b=names_with_default ',' '/' { + _PyPegen_slash_with_default(p, a, b) } +star_etc[StarEtc*]: + | '*' a=plain_name b=name_with_optional_default* c=[',' d=kwds { d }] [','] { + _PyPegen_star_etc(p, a, b, c) } + | '*' b=name_with_optional_default+ c=[',' d=kwds { d }] [','] { + _PyPegen_star_etc(p, NULL, b, c) } + | a=kwds [','] { _PyPegen_star_etc(p, NULL, NULL, a) } +name_with_optional_default[NameDefaultPair*]: + | ',' a=plain_name b=['=' e=expression { e }] { _PyPegen_name_default_pair(p, a, b) } +names_with_default[asdl_seq*]: a=','.name_with_default+ { a } +name_with_default[NameDefaultPair*]: + | n=plain_name '=' e=expression { _PyPegen_name_default_pair(p, n, e) } +plain_names[asdl_seq*] (memo): a=','.(plain_name !'=')+ { a } +plain_name[arg_ty]: + | a=NAME b=[':' z=annotation { z }] { _Py_arg(a->v.Name.id, b, NULL, EXTRA) } +kwds[arg_ty]: + | '**' a=plain_name { a } +annotation[expr_ty]: expression + +decorators[asdl_seq*]: a=('@' f=named_expression NEWLINE { f })+ { a } + +class_def[stmt_ty]: + | a=decorators b=class_def_raw { _PyPegen_class_def_decorators(p, a, b) } + | class_def_raw +class_def_raw[stmt_ty]: + | 'class' a=NAME b=['(' z=[arguments] ')' { z }] ':' c=block { + _Py_ClassDef(a->v.Name.id, + (b) ? ((expr_ty) b)->v.Call.args : NULL, + (b) ? ((expr_ty) b)->v.Call.keywords : NULL, + c, NULL, EXTRA) } + +block[asdl_seq*] (memo): + | NEWLINE INDENT a=statements DEDENT { a } + | simple_stmt + | invalid_block + +expressions_list[asdl_seq*]: a=','.star_expression+ [','] { a } +star_expressions[expr_ty]: + | a=star_expression b=(',' c=star_expression { c })+ [','] { + _Py_Tuple(CHECK(_PyPegen_seq_insert_in_front(p, a, b)), Load, EXTRA) } + | a=star_expression ',' { _Py_Tuple(CHECK(_PyPegen_singleton_seq(p, a)), Load, EXTRA) } + | star_expression +star_expression[expr_ty] (memo): + | '*' a=bitwise_or { _Py_Starred(a, Load, EXTRA) } + | expression + +star_named_expressions[asdl_seq*]: a=','.star_named_expression+ [','] { a } +star_named_expression[expr_ty]: + | '*' a=bitwise_or { _Py_Starred(a, Load, EXTRA) } + | named_expression +named_expression[expr_ty]: + | a=NAME ':=' b=expression { _Py_NamedExpr(CHECK(_PyPegen_set_expr_context(p, a, Store)), b, EXTRA) } + | expression !':=' + | invalid_named_expression + +annotated_rhs[expr_ty]: yield_expr | star_expressions + +expressions[expr_ty]: + | a=expression b=(',' c=expression { c })+ [','] { + _Py_Tuple(CHECK(_PyPegen_seq_insert_in_front(p, a, b)), Load, EXTRA) } + | a=expression ',' { _Py_Tuple(CHECK(_PyPegen_singleton_seq(p, a)), Load, EXTRA) } + | expression +expression[expr_ty] (memo): + | a=disjunction 'if' b=disjunction 'else' c=expression { _Py_IfExp(b, a, c, EXTRA) } + | disjunction + | lambdef + +lambdef[expr_ty]: + | 'lambda' a=[lambda_parameters] ':' b=expression { _Py_Lambda((a) ? a : CHECK(_PyPegen_empty_arguments(p)), b, EXTRA) } +lambda_parameters[arguments_ty]: + | a=lambda_slash_without_default b=[',' x=lambda_plain_names { x }] c=[',' y=lambda_names_with_default { y }] d=[',' z=[lambda_star_etc] { z }] { + _PyPegen_make_arguments(p, a, NULL, b, c, d) } + | a=lambda_slash_with_default b=[',' y=lambda_names_with_default { y }] c=[',' z=[lambda_star_etc] { z }] { + _PyPegen_make_arguments(p, NULL, a, NULL, b, c) } + | a=lambda_plain_names b=[',' y=lambda_names_with_default { y }] c=[',' z=[lambda_star_etc] { z }] { + _PyPegen_make_arguments(p, NULL, NULL, a, b, c) } + | a=lambda_names_with_default b=[',' z=[lambda_star_etc] { z }] { _PyPegen_make_arguments(p, NULL, NULL, NULL, a, b)} + | a=lambda_star_etc { _PyPegen_make_arguments(p, NULL, NULL, NULL, NULL, a) } +lambda_slash_without_default[asdl_seq*]: a=lambda_plain_names ',' '/' { a } +lambda_slash_with_default[SlashWithDefault*]: a=[n=lambda_plain_names ',' { n }] b=lambda_names_with_default ',' '/' { + _PyPegen_slash_with_default(p, a, b) } +lambda_star_etc[StarEtc*]: + | '*' a=lambda_plain_name b=lambda_name_with_optional_default* c=[',' d=lambda_kwds { d }] [','] { + _PyPegen_star_etc(p, a, b, c) } + | '*' b=lambda_name_with_optional_default+ c=[',' d=lambda_kwds { d }] [','] { + _PyPegen_star_etc(p, NULL, b, c) } + | a=lambda_kwds [','] { _PyPegen_star_etc(p, NULL, NULL, a) } +lambda_name_with_optional_default[NameDefaultPair*]: + | ',' a=lambda_plain_name b=['=' e=expression { e }] { _PyPegen_name_default_pair(p, a, b) } +lambda_names_with_default[asdl_seq*]: a=','.lambda_name_with_default+ { a } +lambda_name_with_default[NameDefaultPair*]: + | n=lambda_plain_name '=' e=expression { _PyPegen_name_default_pair(p, n, e) } +lambda_plain_names[asdl_seq*]: a=','.(lambda_plain_name !'=')+ { a } +lambda_plain_name[arg_ty]: a=NAME { _Py_arg(a->v.Name.id, NULL, NULL, EXTRA) } +lambda_kwds[arg_ty]: '**' a=lambda_plain_name { a } + +disjunction[expr_ty] (memo): + | a=conjunction b=('or' c=conjunction { c })+ { _Py_BoolOp( + Or, + CHECK(_PyPegen_seq_insert_in_front(p, a, b)), + EXTRA) } + | conjunction +conjunction[expr_ty] (memo): + | a=inversion b=('and' c=inversion { c })+ { _Py_BoolOp( + And, + CHECK(_PyPegen_seq_insert_in_front(p, a, b)), + EXTRA) } + | inversion +inversion[expr_ty] (memo): + | 'not' a=inversion { _Py_UnaryOp(Not, a, EXTRA) } + | comparison +comparison[expr_ty]: + | a=bitwise_or b=compare_op_bitwise_or_pair+ { + _Py_Compare(a, CHECK(_PyPegen_get_cmpops(p, b)), CHECK(_PyPegen_get_exprs(p, b)), EXTRA) } + | bitwise_or +compare_op_bitwise_or_pair[CmpopExprPair*]: + | eq_bitwise_or + | noteq_bitwise_or + | lte_bitwise_or + | lt_bitwise_or + | gte_bitwise_or + | gt_bitwise_or + | notin_bitwise_or + | in_bitwise_or + | isnot_bitwise_or + | is_bitwise_or +eq_bitwise_or[CmpopExprPair*]: '==' a=bitwise_or { _PyPegen_cmpop_expr_pair(p, Eq, a) } +noteq_bitwise_or[CmpopExprPair*]: '!=' a=bitwise_or { _PyPegen_cmpop_expr_pair(p, NotEq, a) } +lte_bitwise_or[CmpopExprPair*]: '<=' a=bitwise_or { _PyPegen_cmpop_expr_pair(p, LtE, a) } +lt_bitwise_or[CmpopExprPair*]: '<' a=bitwise_or { _PyPegen_cmpop_expr_pair(p, Lt, a) } +gte_bitwise_or[CmpopExprPair*]: '>=' a=bitwise_or { _PyPegen_cmpop_expr_pair(p, GtE, a) } +gt_bitwise_or[CmpopExprPair*]: '>' a=bitwise_or { _PyPegen_cmpop_expr_pair(p, Gt, a) } +notin_bitwise_or[CmpopExprPair*]: 'not' 'in' a=bitwise_or { _PyPegen_cmpop_expr_pair(p, NotIn, a) } +in_bitwise_or[CmpopExprPair*]: 'in' a=bitwise_or { _PyPegen_cmpop_expr_pair(p, In, a) } +isnot_bitwise_or[CmpopExprPair*]: 'is' 'not' a=bitwise_or { _PyPegen_cmpop_expr_pair(p, IsNot, a) } +is_bitwise_or[CmpopExprPair*]: 'is' a=bitwise_or { _PyPegen_cmpop_expr_pair(p, Is, a) } + +bitwise_or[expr_ty]: + | a=bitwise_or '|' b=bitwise_xor { _Py_BinOp(a, BitOr, b, EXTRA) } + | bitwise_xor +bitwise_xor[expr_ty]: + | a=bitwise_xor '^' b=bitwise_and { _Py_BinOp(a, BitXor, b, EXTRA) } + | bitwise_and +bitwise_and[expr_ty]: + | a=bitwise_and '&' b=shift_expr { _Py_BinOp(a, BitAnd, b, EXTRA) } + | shift_expr +shift_expr[expr_ty]: + | a=shift_expr '<<' b=sum { _Py_BinOp(a, LShift, b, EXTRA) } + | a=shift_expr '>>' b=sum { _Py_BinOp(a, RShift, b, EXTRA) } + | sum + +sum[expr_ty]: + | a=sum '+' b=term { _Py_BinOp(a, Add, b, EXTRA) } + | a=sum '-' b=term { _Py_BinOp(a, Sub, b, EXTRA) } + | term +term[expr_ty]: + | a=term '*' b=factor { _Py_BinOp(a, Mult, b, EXTRA) } + | a=term '/' b=factor { _Py_BinOp(a, Div, b, EXTRA) } + | a=term '//' b=factor { _Py_BinOp(a, FloorDiv, b, EXTRA) } + | a=term '%' b=factor { _Py_BinOp(a, Mod, b, EXTRA) } + | a=term '@' b=factor { _Py_BinOp(a, MatMult, b, EXTRA) } + | factor +factor[expr_ty] (memo): + | '+' a=factor { _Py_UnaryOp(UAdd, a, EXTRA) } + | '-' a=factor { _Py_UnaryOp(USub, a, EXTRA) } + | '~' a=factor { _Py_UnaryOp(Invert, a, EXTRA) } + | power +power[expr_ty]: + | a=await_primary '**' b=factor { _Py_BinOp(a, Pow, b, EXTRA) } + | await_primary +await_primary[expr_ty] (memo): + | AWAIT a=primary { _Py_Await(a, EXTRA) } + | primary +primary[expr_ty]: + | a=primary '.' b=NAME { _Py_Attribute(a, b->v.Name.id, Load, EXTRA) } + | a=primary b=genexp { _Py_Call(a, CHECK(_PyPegen_singleton_seq(p, b)), NULL, EXTRA) } + | a=primary '(' b=[arguments] ')' { + _Py_Call(a, + (b) ? ((expr_ty) b)->v.Call.args : NULL, + (b) ? ((expr_ty) b)->v.Call.keywords : NULL, + EXTRA) } + | a=primary '[' b=slices ']' { _Py_Subscript(a, b, Load, EXTRA) } + | atom + +slices[expr_ty]: + | a=slice !',' { a } + | a=','.slice+ [','] { _Py_Tuple(a, Load, EXTRA) } +slice[expr_ty]: + | a=[expression] ':' b=[expression] c=[':' d=[expression] { d }] { _Py_Slice(a, b, c, EXTRA) } + | a=expression { a } +atom[expr_ty]: + | NAME + | 'True' { _Py_Constant(Py_True, NULL, EXTRA) } + | 'False' { _Py_Constant(Py_False, NULL, EXTRA) } + | 'None' { _Py_Constant(Py_None, NULL, EXTRA) } + | '__new_parser__' { RAISE_SYNTAX_ERROR("You found it!") } + | &STRING strings + | NUMBER + | &'(' (tuple | group | genexp) + | &'[' (list | listcomp) + | &'{' (dict | set | dictcomp | setcomp) + | '...' { _Py_Constant(Py_Ellipsis, NULL, EXTRA) } + +strings[expr_ty] (memo): a=STRING+ { _PyPegen_concatenate_strings(p, a) } +list[expr_ty]: + | '[' a=[star_named_expressions] ']' { _Py_List(a, Load, EXTRA) } +listcomp[expr_ty]: + | '[' a=named_expression b=for_if_clauses ']' { _Py_ListComp(a, b, EXTRA) } + | invalid_comprehension +tuple[expr_ty]: + | '(' a=[y=star_named_expression ',' z=[star_named_expressions] { _PyPegen_seq_insert_in_front(p, y, z) } ] ')' { + _Py_Tuple(a, Load, EXTRA) } +group[expr_ty]: '(' a=(yield_expr | named_expression) ')' { a } +genexp[expr_ty]: + | '(' a=expression b=for_if_clauses ')' { _Py_GeneratorExp(a, b, EXTRA) } + | invalid_comprehension +set[expr_ty]: '{' a=expressions_list '}' { _Py_Set(a, EXTRA) } +setcomp[expr_ty]: + | '{' a=expression b=for_if_clauses '}' { _Py_SetComp(a, b, EXTRA) } + | invalid_comprehension +dict[expr_ty]: + | '{' a=[kvpairs] '}' { _Py_Dict(CHECK(_PyPegen_get_keys(p, a)), + CHECK(_PyPegen_get_values(p, a)), EXTRA) } +dictcomp[expr_ty]: + | '{' a=kvpair b=for_if_clauses '}' { _Py_DictComp(a->key, a->value, b, EXTRA) } +kvpairs[asdl_seq*]: a=','.kvpair+ [','] { a } +kvpair[KeyValuePair*]: + | '**' a=bitwise_or { _PyPegen_key_value_pair(p, NULL, a) } + | a=expression ':' b=expression { _PyPegen_key_value_pair(p, a, b) } +for_if_clauses[asdl_seq*]: + | a=(y=[ASYNC] 'for' a=star_targets 'in' b=disjunction c=('if' z=disjunction { z })* + { _Py_comprehension(a, b, c, y != NULL, p->arena) })+ { a } + +yield_expr[expr_ty]: + | 'yield' 'from' a=expression { _Py_YieldFrom(a, EXTRA) } + | 'yield' a=[star_expressions] { _Py_Yield(a, EXTRA) } + +arguments[expr_ty] (memo): + | a=args [','] &')' { a } + | incorrect_arguments +args[expr_ty]: + | a=starred_expression b=[',' c=args { c }] { + _Py_Call(_PyPegen_dummy_name(p), + (b) ? CHECK(_PyPegen_seq_insert_in_front(p, a, ((expr_ty) b)->v.Call.args)) + : CHECK(_PyPegen_singleton_seq(p, a)), + (b) ? ((expr_ty) b)->v.Call.keywords : NULL, + EXTRA) } + | a=kwargs { _Py_Call(_PyPegen_dummy_name(p), + CHECK_NULL_ALLOWED(_PyPegen_seq_extract_starred_exprs(p, a)), + CHECK_NULL_ALLOWED(_PyPegen_seq_delete_starred_exprs(p, a)), + EXTRA) } + | a=named_expression b=[',' c=args { c }] { + _Py_Call(_PyPegen_dummy_name(p), + (b) ? CHECK(_PyPegen_seq_insert_in_front(p, a, ((expr_ty) b)->v.Call.args)) + : CHECK(_PyPegen_singleton_seq(p, a)), + (b) ? ((expr_ty) b)->v.Call.keywords : NULL, + EXTRA) } +kwargs[asdl_seq*]: + | a=','.kwarg_or_starred+ ',' b=','.kwarg_or_double_starred+ { _PyPegen_join_sequences(p, a, b) } + | ','.kwarg_or_starred+ + | ','.kwarg_or_double_starred+ +starred_expression[expr_ty]: + | '*' a=expression { _Py_Starred(a, Load, EXTRA) } +kwarg_or_starred[KeywordOrStarred*]: + | a=NAME '=' b=expression { + _PyPegen_keyword_or_starred(p, CHECK(_Py_keyword(a->v.Name.id, b, EXTRA)), 1) } + | a=starred_expression { _PyPegen_keyword_or_starred(p, a, 0) } +kwarg_or_double_starred[KeywordOrStarred*]: + | a=NAME '=' b=expression { + _PyPegen_keyword_or_starred(p, CHECK(_Py_keyword(a->v.Name.id, b, EXTRA)), 1) } + | '**' a=expression { _PyPegen_keyword_or_starred(p, CHECK(_Py_keyword(NULL, a, EXTRA)), 1) } + +# NOTE: star_targets may contain *bitwise_or, targets may not. +star_targets[expr_ty]: + | a=star_target !',' { a } + | a=star_target b=(',' c=star_target { c })* [','] { + _Py_Tuple(CHECK(_PyPegen_seq_insert_in_front(p, a, b)), Store, EXTRA) } +star_targets_seq[asdl_seq*]: a=','.star_target+ [','] { a } +star_target[expr_ty] (memo): + | '*' a=(!'*' star_target) { + _Py_Starred(CHECK(_PyPegen_set_expr_context(p, a, Store)), Store, EXTRA) } + | a=t_primary '.' b=NAME !t_lookahead { _Py_Attribute(a, b->v.Name.id, Store, EXTRA) } + | a=t_primary '[' b=slices ']' !t_lookahead { _Py_Subscript(a, b, Store, EXTRA) } + | star_atom +star_atom[expr_ty]: + | a=NAME { _PyPegen_set_expr_context(p, a, Store) } + | '(' a=star_target ')' { _PyPegen_set_expr_context(p, a, Store) } + | '(' a=[star_targets_seq] ')' { _Py_Tuple(a, Store, EXTRA) } + | '[' a=[star_targets_seq] ']' { _Py_List(a, Store, EXTRA) } + +inside_paren_ann_assign_target[expr_ty]: + | ann_assign_subscript_attribute_target + | a=NAME { _PyPegen_set_expr_context(p, a, Store) } + | '(' a=inside_paren_ann_assign_target ')' { a } + +ann_assign_subscript_attribute_target[expr_ty]: + | a=t_primary '.' b=NAME !t_lookahead { _Py_Attribute(a, b->v.Name.id, Store, EXTRA) } + | a=t_primary '[' b=slices ']' !t_lookahead { _Py_Subscript(a, b, Store, EXTRA) } + +del_targets[asdl_seq*]: a=','.del_target+ [','] { a } +del_target[expr_ty] (memo): + | a=t_primary '.' b=NAME !t_lookahead { _Py_Attribute(a, b->v.Name.id, Del, EXTRA) } + | a=t_primary '[' b=slices ']' !t_lookahead { _Py_Subscript(a, b, Del, EXTRA) } + | del_t_atom +del_t_atom[expr_ty]: + | a=NAME { _PyPegen_set_expr_context(p, a, Del) } + | '(' a=del_target ')' { _PyPegen_set_expr_context(p, a, Del) } + | '(' a=[del_targets] ')' { _Py_Tuple(a, Del, EXTRA) } + | '[' a=[del_targets] ']' { _Py_List(a, Del, EXTRA) } + +targets[asdl_seq*]: a=','.target+ [','] { a } +target[expr_ty] (memo): + | a=t_primary '.' b=NAME !t_lookahead { _Py_Attribute(a, b->v.Name.id, Store, EXTRA) } + | a=t_primary '[' b=slices ']' !t_lookahead { _Py_Subscript(a, b, Store, EXTRA) } + | t_atom +t_primary[expr_ty]: + | a=t_primary '.' b=NAME &t_lookahead { _Py_Attribute(a, b->v.Name.id, Load, EXTRA) } + | a=t_primary '[' b=slices ']' &t_lookahead { _Py_Subscript(a, b, Load, EXTRA) } + | a=t_primary b=genexp &t_lookahead { _Py_Call(a, CHECK(_PyPegen_singleton_seq(p, b)), NULL, EXTRA) } + | a=t_primary '(' b=[arguments] ')' &t_lookahead { + _Py_Call(a, + (b) ? ((expr_ty) b)->v.Call.args : NULL, + (b) ? ((expr_ty) b)->v.Call.keywords : NULL, + EXTRA) } + | a=atom &t_lookahead { a } +t_lookahead: '(' | '[' | '.' +t_atom[expr_ty]: + | a=NAME { _PyPegen_set_expr_context(p, a, Store) } + | '(' a=target ')' { _PyPegen_set_expr_context(p, a, Store) } + | '(' b=[targets] ')' { _Py_Tuple(b, Store, EXTRA) } + | '[' b=[targets] ']' { _Py_List(b, Store, EXTRA) } + + +# From here on, there are rules for invalid syntax with specialised error messages +incorrect_arguments: + | args ',' '*' { RAISE_SYNTAX_ERROR("iterable argument unpacking follows keyword argument unpacking") } + | expression for_if_clauses ',' [args | expression for_if_clauses] { + RAISE_SYNTAX_ERROR("Generator expression must be parenthesized") } + | a=args ',' args { _PyPegen_arguments_parsing_error(p, a) } +invalid_named_expression: + | a=expression ':=' expression { + RAISE_SYNTAX_ERROR("cannot use assignment expressions with %s", _PyPegen_get_expr_name(a)) } +invalid_assignment: + | list ':' { RAISE_SYNTAX_ERROR("only single target (not list) can be annotated") } + | tuple ':' { RAISE_SYNTAX_ERROR("only single target (not tuple) can be annotated") } + | expression ':' expression ['=' annotated_rhs] { + RAISE_SYNTAX_ERROR("illegal target for annotation") } + | a=expression ('=' | augassign) (yield_expr | star_expressions) { + RAISE_SYNTAX_ERROR("cannot assign to %s", _PyPegen_get_expr_name(a)) } +invalid_block: + | NEWLINE !INDENT { RAISE_INDENTATION_ERROR("expected an indented block") } +invalid_comprehension: + | ('[' | '(' | '{') '*' expression for_if_clauses { + RAISE_SYNTAX_ERROR("iterable unpacking cannot be used in comprehension") } +invalid_parameters: + | [plain_names ','] (slash_with_default | names_with_default) ',' plain_names { + RAISE_SYNTAX_ERROR("non-default argument follows default argument") } diff --git a/Include/compile.h b/Include/compile.h index a2db65d47f001..dbba85bb5f653 100644 --- a/Include/compile.h +++ b/Include/compile.h @@ -108,4 +108,7 @@ PyAPI_FUNC(int) _PyAST_Optimize(struct _mod *, PyArena *arena, _PyASTOptimizeSta #define Py_eval_input 258 #define Py_func_type_input 345 +/* This doesn't need to match anything */ +#define Py_fstring_input 800 + #endif /* !Py_COMPILE_H */ diff --git a/Include/cpython/initconfig.h b/Include/cpython/initconfig.h index c5fa2b3857e7a..653959656f216 100644 --- a/Include/cpython/initconfig.h +++ b/Include/cpython/initconfig.h @@ -147,6 +147,10 @@ typedef struct { Set to 1 by -X faulthandler and PYTHONFAULTHANDLER. -1 means unset. */ int faulthandler; + /* Enable PEG parser? + 1 by default, set to 0 by -X oldparser and PYTHONOLDPARSER */ + int use_peg; + /* Enable tracemalloc? Set by -X tracemalloc=N and PYTHONTRACEMALLOC. -1 means unset */ int tracemalloc; diff --git a/Include/pegen_interface.h b/Include/pegen_interface.h new file mode 100644 index 0000000000000..bf5b29634ac33 --- /dev/null +++ b/Include/pegen_interface.h @@ -0,0 +1,32 @@ +#ifndef Py_LIMITED_API +#ifndef Py_PEGENINTERFACE +#define Py_PEGENINTERFACE +#ifdef __cplusplus +extern "C" { +#endif + +#include "Python.h" +#include "Python-ast.h" + +PyAPI_FUNC(mod_ty) PyPegen_ASTFromFile(const char *filename, int mode, PyArena *arena); +PyAPI_FUNC(mod_ty) PyPegen_ASTFromString(const char *str, int mode, PyCompilerFlags *flags, + PyArena *arena); +PyAPI_FUNC(mod_ty) PyPegen_ASTFromStringObject(const char *str, PyObject* filename, int mode, + PyCompilerFlags *flags, PyArena *arena); +PyAPI_FUNC(mod_ty) PyPegen_ASTFromFileObject(FILE *fp, PyObject *filename_ob, + int mode, const char *enc, const char *ps1, + const char *ps2, int *errcode, PyArena *arena); +PyAPI_FUNC(PyCodeObject *) PyPegen_CodeObjectFromFile(const char *filename, int mode); +PyAPI_FUNC(PyCodeObject *) PyPegen_CodeObjectFromString(const char *str, int mode, + PyCompilerFlags *flags); +PyAPI_FUNC(PyCodeObject *) PyPegen_CodeObjectFromFileObject(FILE *, PyObject *filename_ob, + int mode, const char *enc, + const char *ps1, + const char *ps2, + int *errcode); + +#ifdef __cplusplus +} +#endif +#endif /* !Py_PEGENINTERFACE*/ +#endif /* !Py_LIMITED_API */ diff --git a/Lib/test/test_cmd_line_script.py b/Lib/test/test_cmd_line_script.py index 44a5487d75dff..f0130e376aec4 100644 --- a/Lib/test/test_cmd_line_script.py +++ b/Lib/test/test_cmd_line_script.py @@ -599,7 +599,7 @@ def test_syntaxerror_unindented_caret_position(self): exitcode, stdout, stderr = assert_python_failure(script_name) text = io.TextIOWrapper(io.BytesIO(stderr), 'ascii').read() # Confirm that the caret is located under the first 1 character - self.assertIn("\n 1 + 1 = 2\n ^", text) + self.assertIn("\n 1 + 1 = 2\n ^", text) def test_syntaxerror_indented_caret_position(self): script = textwrap.dedent("""\ @@ -611,7 +611,7 @@ def test_syntaxerror_indented_caret_position(self): exitcode, stdout, stderr = assert_python_failure(script_name) text = io.TextIOWrapper(io.BytesIO(stderr), 'ascii').read() # Confirm that the caret is located under the first 1 character - self.assertIn("\n 1 + 1 = 2\n ^", text) + self.assertIn("\n 1 + 1 = 2\n ^", text) # Try the same with a form feed at the start of the indented line script = ( @@ -622,7 +622,7 @@ def test_syntaxerror_indented_caret_position(self): exitcode, stdout, stderr = assert_python_failure(script_name) text = io.TextIOWrapper(io.BytesIO(stderr), "ascii").read() self.assertNotIn("\f", text) - self.assertIn("\n 1 + 1 = 2\n ^", text) + self.assertIn("\n 1 + 1 = 2\n ^", text) def test_syntaxerror_multi_line_fstring(self): script = 'foo = f"""{}\nfoo"""\n' @@ -632,14 +632,14 @@ def test_syntaxerror_multi_line_fstring(self): self.assertEqual( stderr.splitlines()[-3:], [ - b' foo = f"""{}', - b' ^', + b' foo"""', + b' ^', b'SyntaxError: f-string: empty expression not allowed', ], ) def test_syntaxerror_invalid_escape_sequence_multi_line(self): - script = 'foo = """\\q\n"""\n' + script = 'foo = """\\q"""\n' with support.temp_dir() as script_dir: script_name = _make_test_script(script_dir, 'script', script) exitcode, stdout, stderr = assert_python_failure( @@ -647,10 +647,9 @@ def test_syntaxerror_invalid_escape_sequence_multi_line(self): ) self.assertEqual( stderr.splitlines()[-3:], - [ - b' foo = """\\q', - b' ^', - b'SyntaxError: invalid escape sequence \\q', + [ b' foo = """\\q"""', + b' ^', + b'SyntaxError: invalid escape sequence \\q' ], ) diff --git a/Lib/test/test_codeop.py b/Lib/test/test_codeop.py index 98da26fa5dab1..f1d74b1fb763e 100644 --- a/Lib/test/test_codeop.py +++ b/Lib/test/test_codeop.py @@ -2,6 +2,7 @@ Test cases for codeop.py Nick Mathewson """ +import sys import unittest from test.support import is_jython @@ -9,7 +10,6 @@ import io if is_jython: - import sys def unify_callables(d): for n,v in d.items(): @@ -122,6 +122,7 @@ def test_valid(self): av("def f():\n pass\n#foo\n") av("@a.b.c\ndef f():\n pass\n") + @unittest.skipIf(sys.flags.use_peg, "Pegen does not support PyCF_DONT_INPLY_DEDENT yet") def test_incomplete(self): ai = self.assertIncomplete diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py index 566ca27fca893..6535316dbea24 100644 --- a/Lib/test/test_compile.py +++ b/Lib/test/test_compile.py @@ -501,6 +501,7 @@ def test_single_statement(self): self.compile_single("if x:\n f(x)\nelse:\n g(x)") self.compile_single("class T:\n pass") + @unittest.skipIf(sys.flags.use_peg, 'Pegen does not disallow multiline single stmts') def test_bad_single_statement(self): self.assertInvalidSingle('1\n2') self.assertInvalidSingle('def f(): pass') diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index 444097b4f0817..24ebc5ca9ba25 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -347,6 +347,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): 'isolated': 0, 'use_environment': 1, 'dev_mode': 0, + 'use_peg': 1, 'install_signal_handlers': 1, 'use_hash_seed': 0, @@ -728,6 +729,7 @@ def test_init_from_config(self): 'import_time': 1, 'show_ref_count': 1, 'malloc_stats': 1, + 'use_peg': 0, 'stdio_encoding': 'iso8859-1', 'stdio_errors': 'replace', diff --git a/Lib/test/test_eof.py b/Lib/test/test_eof.py index 9ef8eb1187486..bb1300c7c24d4 100644 --- a/Lib/test/test_eof.py +++ b/Lib/test/test_eof.py @@ -26,6 +26,7 @@ def test_EOFS(self): else: raise support.TestFailed + @unittest.skipIf(sys.flags.use_peg, "TODO for PEG -- fails with new parser") def test_line_continuation_EOF(self): """A continuation at the end of input must be an error; bpo2180.""" expect = 'unexpected EOF while parsing (<string>, line 1)' @@ -36,6 +37,7 @@ def test_line_continuation_EOF(self): exec('\\') self.assertEqual(str(excinfo.exception), expect) + @unittest.skip("TODO for PEG -- fails even with old parser now") @unittest.skipIf(not sys.executable, "sys.executable required") def test_line_continuation_EOF_from_file_bpo2180(self): """Ensure tok_nextc() does not add too many ending newlines.""" diff --git a/Lib/test/test_exceptions.py b/Lib/test/test_exceptions.py index 8c4a2882bad82..c234c2b739c5e 100644 --- a/Lib/test/test_exceptions.py +++ b/Lib/test/test_exceptions.py @@ -178,6 +178,7 @@ def ckmsg(src, msg, exception=SyntaxError): s = '''if True:\n print()\n\texec "mixed tabs and spaces"''' ckmsg(s, "inconsistent use of tabs and spaces in indentation", TabError) + @unittest.skipIf(sys.flags.use_peg, "Pegen column offsets might be different") def testSyntaxErrorOffset(self): def check(src, lineno, offset, encoding='utf-8'): with self.assertRaises(SyntaxError) as cm: diff --git a/Lib/test/test_flufl.py b/Lib/test/test_flufl.py index 33e52e609a761..297a8aa90c96c 100644 --- a/Lib/test/test_flufl.py +++ b/Lib/test/test_flufl.py @@ -1,6 +1,9 @@ import __future__ import unittest +import sys + +(a)unittest.skipIf(sys.flags.use_peg, "Not supported by pegen yet") class FLUFLTests(unittest.TestCase): def test_barry_as_bdfl(self): diff --git a/Lib/test/test_fstring.py b/Lib/test/test_fstring.py index fe465b7e1d43d..802b08341e2b5 100644 --- a/Lib/test/test_fstring.py +++ b/Lib/test/test_fstring.py @@ -10,6 +10,7 @@ import ast import types import decimal +import sys import unittest a_global = 'global variable' @@ -205,7 +206,8 @@ def test_ast_line_numbers_nested(self): call = binop.right.values[1].value self.assertEqual(type(call), ast.Call) self.assertEqual(call.lineno, 3) - self.assertEqual(call.col_offset, 11) + if not sys.flags.use_peg: + self.assertEqual(call.col_offset, 11) def test_ast_line_numbers_duplicate_expression(self): """Duplicate expression diff --git a/Lib/test/test_generators.py b/Lib/test/test_generators.py index f8d86da5e2f5b..3e42bc6b69a81 100644 --- a/Lib/test/test_generators.py +++ b/Lib/test/test_generators.py @@ -1856,10 +1856,11 @@ def printsolution(self, x): ... SyntaxError: 'yield' outside function ->>> def f(): x = yield = y -Traceback (most recent call last): - ... -SyntaxError: assignment to yield expression not possible +# Pegen does not produce this error message yet +# >>> def f(): x = yield = y +# Traceback (most recent call last): +# ... +# SyntaxError: assignment to yield expression not possible >>> def f(): (yield bar) = y Traceback (most recent call last): diff --git a/Lib/test/test_parser.py b/Lib/test/test_parser.py index 73178f3e7759b..124a2790bf2bd 100644 --- a/Lib/test/test_parser.py +++ b/Lib/test/test_parser.py @@ -8,6 +8,7 @@ import unittest import operator import struct +import sys from test import support from test.support.script_helper import assert_python_failure from test.support.script_helper import assert_python_ok @@ -899,9 +900,10 @@ def test_deeply_nested_list(self): st = parser.expr(e) st.compile() + @unittest.skipIf(sys.flags.use_peg, "Pegen does not trigger memory error with this many parenthesis") def test_trigger_memory_error(self): e = self._nested_expression(100) - rc, out, err = assert_python_failure('-c', e) + rc, out, err = assert_python_failure('-Xoldparser', '-c', e) # parsing the expression will result in an error message # followed by a MemoryError (see #11963) self.assertIn(b's_push: parser stack overflow', err) diff --git a/Lib/test/test_peg_generator/__init__.py b/Lib/test/test_peg_generator/__init__.py new file mode 100644 index 0000000000000..fa855f2104c58 --- /dev/null +++ b/Lib/test/test_peg_generator/__init__.py @@ -0,0 +1,7 @@ +import os + +from test.support import load_package_tests + +# Load all tests in package +def load_tests(*args): + return load_package_tests(os.path.dirname(__file__), *args) diff --git a/Lib/test/test_peg_generator/__main__.py b/Lib/test/test_peg_generator/__main__.py new file mode 100644 index 0000000000000..1fab1fddb5744 --- /dev/null +++ b/Lib/test/test_peg_generator/__main__.py @@ -0,0 +1,4 @@ +import unittest +from . import load_tests + +unittest.main() diff --git a/Lib/test/test_peg_generator/ast_dump.py b/Lib/test/test_peg_generator/ast_dump.py new file mode 100644 index 0000000000000..22d2dde775597 --- /dev/null +++ b/Lib/test/test_peg_generator/ast_dump.py @@ -0,0 +1,62 @@ +""" +Copy-parse of ast.dump, removing the `isinstance` checks. This is needed, +because testing pegen requires generating a C extension module, which contains +a copy of the symbols defined in Python-ast.c. Thus, the isinstance check would +always fail. We rely on string comparison of the base classes instead. +TODO: Remove the above-described hack. +""" + +def ast_dump(node, annotate_fields=True, include_attributes=False, *, indent=None): + def _format(node, level=0): + if indent is not None: + level += 1 + prefix = '\n' + indent * level + sep = ',\n' + indent * level + else: + prefix = '' + sep = ', ' + if any(cls.__name__ == 'AST' for cls in node.__class__.__mro__): + cls = type(node) + args = [] + allsimple = True + keywords = annotate_fields + for name in node._fields: + try: + value = getattr(node, name) + except AttributeError: + keywords = True + continue + if value is None and getattr(cls, name, ...) is None: + keywords = True + continue + value, simple = _format(value, level) + allsimple = allsimple and simple + if keywords: + args.append('%s=%s' % (name, value)) + else: + args.append(value) + if include_attributes and node._attributes: + for name in node._attributes: + try: + value = getattr(node, name) + except AttributeError: + continue + if value is None and getattr(cls, name, ...) is None: + continue + value, simple = _format(value, level) + allsimple = allsimple and simple + args.append('%s=%s' % (name, value)) + if allsimple and len(args) <= 3: + return '%s(%s)' % (node.__class__.__name__, ', '.join(args)), not args + return '%s(%s%s)' % (node.__class__.__name__, prefix, sep.join(args)), False + elif isinstance(node, list): + if not node: + return '[]', True + return '[%s%s]' % (prefix, sep.join(_format(x, level)[0] for x in node)), False + return repr(node), True + + if all(cls.__name__ != 'AST' for cls in node.__class__.__mro__): + raise TypeError('expected AST, got %r' % node.__class__.__name__) + if indent is not None and not isinstance(indent, str): + indent = ' ' * indent + return _format(node)[0] diff --git a/Lib/test/test_peg_generator/test_c_parser.py b/Lib/test/test_peg_generator/test_c_parser.py new file mode 100644 index 0000000000000..f2f699c83df01 --- /dev/null +++ b/Lib/test/test_peg_generator/test_c_parser.py @@ -0,0 +1,333 @@ +import ast +import contextlib +import traceback +import tempfile +import shutil +import unittest +import sys + +from test import test_tools +from test.test_peg_generator.ast_dump import ast_dump +from pathlib import PurePath, Path +from typing import Sequence + +test_tools.skip_if_missing('peg_generator') +with test_tools.imports_under_tool('peg_generator'): + from pegen.grammar_parser import GeneratedParser as GrammarParser + from pegen.testutil import ( + parse_string, + generate_parser_c_extension, + generate_c_parser_source, + ) + + +class TestCParser(unittest.TestCase): + def setUp(self): + self.tmp_path = tempfile.mkdtemp() + + def tearDown(self): + with contextlib.suppress(PermissionError): + shutil.rmtree(self.tmp_path) + + def check_input_strings_for_grammar( + self, + source: str, + tmp_path: PurePath, + valid_cases: Sequence[str] = (), + invalid_cases: Sequence[str] = (), + ) -> None: + grammar = parse_string(source, GrammarParser) + extension = generate_parser_c_extension(grammar, Path(tmp_path)) + + if valid_cases: + for case in valid_cases: + extension.parse_string(case, mode=0) + + if invalid_cases: + for case in invalid_cases: + with self.assertRaises(SyntaxError): + extension.parse_string(case, mode=0) + + def verify_ast_generation(self, source: str, stmt: str, tmp_path: PurePath) -> None: + grammar = parse_string(source, GrammarParser) + extension = generate_parser_c_extension(grammar, Path(tmp_path)) + + expected_ast = ast.parse(stmt) + actual_ast = extension.parse_string(stmt, mode=1) + self.assertEqual(ast_dump(expected_ast), ast_dump(actual_ast)) + + def test_c_parser(self) -> None: + grammar_source = """ + start[mod_ty]: a=stmt* $ { Module(a, NULL, p->arena) } + stmt[stmt_ty]: a=expr_stmt { a } + expr_stmt[stmt_ty]: a=expression NEWLINE { _Py_Expr(a, EXTRA) } + expression[expr_ty]: ( l=expression '+' r=term { _Py_BinOp(l, Add, r, EXTRA) } + | l=expression '-' r=term { _Py_BinOp(l, Sub, r, EXTRA) } + | t=term { t } + ) + term[expr_ty]: ( l=term '*' r=factor { _Py_BinOp(l, Mult, r, EXTRA) } + | l=term '/' r=factor { _Py_BinOp(l, Div, r, EXTRA) } + | f=factor { f } + ) + factor[expr_ty]: ('(' e=expression ')' { e } + | a=atom { a } + ) + atom[expr_ty]: ( n=NAME { n } + | n=NUMBER { n } + | s=STRING { s } + ) + """ + grammar = parse_string(grammar_source, GrammarParser) + extension = generate_parser_c_extension(grammar, Path(self.tmp_path)) + + expressions = [ + "4+5", + "4-5", + "4*5", + "1+4*5", + "1+4/5", + "(1+1) + (1+1)", + "(1+1) - (1+1)", + "(1+1) * (1+1)", + "(1+1) / (1+1)", + ] + + for expr in expressions: + the_ast = extension.parse_string(expr, mode=1) + expected_ast = ast.parse(expr) + self.assertEqual(ast_dump(the_ast), ast_dump(expected_ast)) + + def test_lookahead(self) -> None: + grammar = """ + start: NAME &NAME expr NEWLINE? ENDMARKER + expr: NAME | NUMBER + """ + valid_cases = ["foo bar"] + invalid_cases = ["foo 34"] + self.check_input_strings_for_grammar(grammar, self.tmp_path, valid_cases, invalid_cases) + + def test_negative_lookahead(self) -> None: + grammar = """ + start: NAME !NAME expr NEWLINE? ENDMARKER + expr: NAME | NUMBER + """ + valid_cases = ["foo 34"] + invalid_cases = ["foo bar"] + self.check_input_strings_for_grammar(grammar, self.tmp_path, valid_cases, invalid_cases) + + def test_cut(self) -> None: + grammar = """ + start: X ~ Y Z | X Q S + X: 'x' + Y: 'y' + Z: 'z' + Q: 'q' + S: 's' + """ + valid_cases = ["x y z"] + invalid_cases = ["x q s"] + self.check_input_strings_for_grammar(grammar, self.tmp_path, valid_cases, invalid_cases) + + def test_gather(self) -> None: + grammar = """ + start: ';'.pass_stmt+ NEWLINE + pass_stmt: 'pass' + """ + valid_cases = ["pass", "pass; pass"] + invalid_cases = ["pass;", "pass; pass;"] + self.check_input_strings_for_grammar(grammar, self.tmp_path, valid_cases, invalid_cases) + + def test_left_recursion(self) -> None: + grammar = """ + start: expr NEWLINE + expr: ('-' term | expr '+' term | term) + term: NUMBER + """ + valid_cases = ["-34", "34", "34 + 12", "1 + 1 + 2 + 3"] + self.check_input_strings_for_grammar(grammar, self.tmp_path, valid_cases) + + def test_advanced_left_recursive(self) -> None: + grammar = """ + start: NUMBER | sign start + sign: ['-'] + """ + valid_cases = ["23", "-34"] + self.check_input_strings_for_grammar(grammar, self.tmp_path, valid_cases) + + def test_mutually_left_recursive(self) -> None: + grammar = """ + start: foo 'E' + foo: bar 'A' | 'B' + bar: foo 'C' | 'D' + """ + valid_cases = ["B E", "D A C A E"] + self.check_input_strings_for_grammar(grammar, self.tmp_path, valid_cases) + + def test_nasty_mutually_left_recursive(self) -> None: + grammar = """ + start: target '=' + target: maybe '+' | NAME + maybe: maybe '-' | target + """ + valid_cases = ["x ="] + invalid_cases = ["x - + ="] + self.check_input_strings_for_grammar(grammar, self.tmp_path, valid_cases, invalid_cases) + + def test_return_stmt_noexpr_action(self) -> None: + grammar = """ + start[mod_ty]: a=[statements] ENDMARKER { Module(a, NULL, p->arena) } + statements[asdl_seq*]: a=statement+ { a } + statement[stmt_ty]: simple_stmt + simple_stmt[stmt_ty]: small_stmt + small_stmt[stmt_ty]: return_stmt + return_stmt[stmt_ty]: a='return' NEWLINE { _Py_Return(NULL, EXTRA) } + """ + stmt = "return" + self.verify_ast_generation(grammar, stmt, self.tmp_path) + + def test_gather_action_ast(self) -> None: + grammar = """ + start[mod_ty]: a=';'.pass_stmt+ NEWLINE ENDMARKER { Module(a, NULL, p->arena) } + pass_stmt[stmt_ty]: a='pass' { _Py_Pass(EXTRA)} + """ + stmt = "pass; pass" + self.verify_ast_generation(grammar, stmt, self.tmp_path) + + def test_pass_stmt_action(self) -> None: + grammar = """ + start[mod_ty]: a=[statements] ENDMARKER { Module(a, NULL, p->arena) } + statements[asdl_seq*]: a=statement+ { a } + statement[stmt_ty]: simple_stmt + simple_stmt[stmt_ty]: small_stmt + small_stmt[stmt_ty]: pass_stmt + pass_stmt[stmt_ty]: a='pass' NEWLINE { _Py_Pass(EXTRA) } + """ + stmt = "pass" + self.verify_ast_generation(grammar, stmt, self.tmp_path) + + def test_if_stmt_action(self) -> None: + grammar = """ + start[mod_ty]: a=[statements] ENDMARKER { Module(a, NULL, p->arena) } + statements[asdl_seq*]: a=statement+ { _PyPegen_seq_flatten(p, a) } + statement[asdl_seq*]: a=compound_stmt { _PyPegen_singleton_seq(p, a) } | simple_stmt + + simple_stmt[asdl_seq*]: a=small_stmt b=further_small_stmt* [';'] NEWLINE { _PyPegen_seq_insert_in_front(p, a, b) } + further_small_stmt[stmt_ty]: ';' a=small_stmt { a } + + block: simple_stmt | NEWLINE INDENT a=statements DEDENT { a } + + compound_stmt: if_stmt + + if_stmt: 'if' a=full_expression ':' b=block { _Py_If(a, b, NULL, EXTRA) } + + small_stmt[stmt_ty]: pass_stmt + + pass_stmt[stmt_ty]: a='pass' { _Py_Pass(EXTRA) } + + full_expression: NAME + """ + stmt = "pass" + self.verify_ast_generation(grammar, stmt, self.tmp_path) + + def test_same_name_different_types(self) -> None: + source = """ + start[mod_ty]: a=import_from+ NEWLINE ENDMARKER { Module(a, NULL, p->arena)} + import_from[stmt_ty]: ( a='from' !'import' c=simple_name 'import' d=import_as_names_from { + _Py_ImportFrom(c->v.Name.id, d, 0, EXTRA) } + | a='from' '.' 'import' c=import_as_names_from { + _Py_ImportFrom(NULL, c, 1, EXTRA) } + ) + simple_name[expr_ty]: NAME + import_as_names_from[asdl_seq*]: a=','.import_as_name_from+ { a } + import_as_name_from[alias_ty]: a=NAME 'as' b=NAME { _Py_alias(((expr_ty) a)->v.Name.id, ((expr_ty) b)->v.Name.id, p->arena) } + """ + grammar = parse_string(source, GrammarParser) + extension = generate_parser_c_extension(grammar, Path(self.tmp_path)) + + for stmt in ("from a import b as c", "from . import a as b"): + expected_ast = ast.parse(stmt) + actual_ast = extension.parse_string(stmt, mode=1) + self.assertEqual(ast_dump(expected_ast), ast_dump(actual_ast)) + + def test_with_stmt_with_paren(self) -> None: + grammar_source = """ + start[mod_ty]: a=[statements] ENDMARKER { Module(a, NULL, p->arena) } + statements[asdl_seq*]: a=statement+ { _PyPegen_seq_flatten(p, a) } + statement[asdl_seq*]: a=compound_stmt { _PyPegen_singleton_seq(p, a) } + compound_stmt[stmt_ty]: with_stmt + with_stmt[stmt_ty]: ( + a='with' '(' b=','.with_item+ ')' ':' c=block { + _Py_With(b, _PyPegen_singleton_seq(p, c), NULL, EXTRA) } + ) + with_item[withitem_ty]: ( + e=NAME o=['as' t=NAME { t }] { _Py_withitem(e, _PyPegen_set_expr_context(p, o, Store), p->arena) } + ) + block[stmt_ty]: a=pass_stmt NEWLINE { a } | NEWLINE INDENT a=pass_stmt DEDENT { a } + pass_stmt[stmt_ty]: a='pass' { _Py_Pass(EXTRA) } + """ + stmt = "with (\n a as b,\n c as d\n): pass" + grammar = parse_string(grammar_source, GrammarParser) + extension = generate_parser_c_extension(grammar, Path(self.tmp_path)) + the_ast = extension.parse_string(stmt, mode=1) + self.assertTrue(ast_dump(the_ast).startswith( + "Module(body=[With(items=[withitem(context_expr=Name(id='a', ctx=Load()), optional_vars=Name(id='b', ctx=Store())), " + "withitem(context_expr=Name(id='c', ctx=Load()), optional_vars=Name(id='d', ctx=Store()))]" + )) + + def test_ternary_operator(self) -> None: + grammar_source = """ + start[mod_ty]: a=expr ENDMARKER { Module(a, NULL, p->arena) } + expr[asdl_seq*]: a=listcomp NEWLINE { _PyPegen_singleton_seq(p, _Py_Expr(a, EXTRA)) } + listcomp[expr_ty]: ( + a='[' b=NAME c=for_if_clauses d=']' { _Py_ListComp(b, c, EXTRA) } + ) + for_if_clauses[asdl_seq*]: ( + a=(y=[ASYNC] 'for' a=NAME 'in' b=NAME c=('if' z=NAME { z })* + { _Py_comprehension(_Py_Name(((expr_ty) a)->v.Name.id, Store, EXTRA), b, c, (y == NULL) ? 0 : 1, p->arena) })+ { a } + ) + """ + stmt = "[i for i in a if b]" + self.verify_ast_generation(grammar_source, stmt, self.tmp_path) + + def test_syntax_error_for_string(self) -> None: + grammar_source = """ + start: expr+ NEWLINE? ENDMARKER + expr: NAME + """ + grammar = parse_string(grammar_source, GrammarParser) + print(list(Path(self.tmp_path).iterdir())) + extension = generate_parser_c_extension(grammar, Path(self.tmp_path)) + for text in ("a b 42 b a", "名名 42 名名"): + try: + extension.parse_string(text, mode=0) + except SyntaxError as e: + tb = traceback.format_exc() + self.assertTrue('File "<string>", line 1' in tb) + self.assertTrue(f"SyntaxError: invalid syntax" in tb) + + def test_headers_and_trailer(self) -> None: + grammar_source = """ + @header 'SOME HEADER' + @subheader 'SOME SUBHEADER' + @trailer 'SOME TRAILER' + start: expr+ NEWLINE? ENDMARKER + expr: x=NAME + """ + grammar = parse_string(grammar_source, GrammarParser) + parser_source = generate_c_parser_source(grammar) + + self.assertTrue("SOME HEADER" in parser_source) + self.assertTrue("SOME SUBHEADER" in parser_source) + self.assertTrue("SOME TRAILER" in parser_source) + + + def test_error_in_rules(self) -> None: + grammar_source = """ + start: expr+ NEWLINE? ENDMARKER + expr: NAME {PyTuple_New(-1)} + """ + grammar = parse_string(grammar_source, GrammarParser) + extension = generate_parser_c_extension(grammar, Path(self.tmp_path)) + # PyTuple_New raises SystemError if an invalid argument was passed. + with self.assertRaises(SystemError): + extension.parse_string("a", mode=0) diff --git a/Lib/test/test_peg_generator/test_first_sets.py b/Lib/test/test_peg_generator/test_first_sets.py new file mode 100644 index 0000000000000..425ee23aa1ab5 --- /dev/null +++ b/Lib/test/test_peg_generator/test_first_sets.py @@ -0,0 +1,225 @@ +import unittest + +from test import test_tools +from typing import Dict, Set + +test_tools.skip_if_missing('peg_generator') +with test_tools.imports_under_tool('peg_generator'): + from pegen.grammar_parser import GeneratedParser as GrammarParser + from pegen.testutil import parse_string + from pegen.first_sets import FirstSetCalculator + from pegen.grammar import Grammar + + +class TestFirstSets(unittest.TestCase): + def calculate_first_sets(self, grammar_source: str) -> Dict[str, Set[str]]: + grammar: Grammar = parse_string(grammar_source, GrammarParser) + return FirstSetCalculator(grammar.rules).calculate() + + def test_alternatives(self) -> None: + grammar = """ + start: expr NEWLINE? ENDMARKER + expr: A | B + A: 'a' | '-' + B: 'b' | '+' + """ + self.assertEqual(self.calculate_first_sets(grammar), { + "A": {"'a'", "'-'"}, + "B": {"'+'", "'b'"}, + "expr": {"'+'", "'a'", "'b'", "'-'"}, + "start": {"'+'", "'a'", "'b'", "'-'"}, + }) + + def test_optionals(self) -> None: + grammar = """ + start: expr NEWLINE + expr: ['a'] ['b'] 'c' + """ + self.assertEqual(self.calculate_first_sets(grammar), { + "expr": {"'c'", "'a'", "'b'"}, + "start": {"'c'", "'a'", "'b'"}, + }) + + def test_repeat_with_separator(self) -> None: + grammar = """ + start: ','.thing+ NEWLINE + thing: NUMBER + """ + self.assertEqual(self.calculate_first_sets(grammar), {"thing": {"NUMBER"}, "start": {"NUMBER"}}) + + def test_optional_operator(self) -> None: + grammar = """ + start: sum NEWLINE + sum: (term)? 'b' + term: NUMBER + """ + self.assertEqual(self.calculate_first_sets(grammar), { + "term": {"NUMBER"}, + "sum": {"NUMBER", "'b'"}, + "start": {"'b'", "NUMBER"}, + }) + + def test_optional_literal(self) -> None: + grammar = """ + start: sum NEWLINE + sum: '+' ? term + term: NUMBER + """ + self.assertEqual(self.calculate_first_sets(grammar), { + "term": {"NUMBER"}, + "sum": {"'+'", "NUMBER"}, + "start": {"'+'", "NUMBER"}, + }) + + def test_optional_after(self) -> None: + grammar = """ + start: term NEWLINE + term: NUMBER ['+'] + """ + self.assertEqual(self.calculate_first_sets(grammar), {"term": {"NUMBER"}, "start": {"NUMBER"}}) + + def test_optional_before(self) -> None: + grammar = """ + start: term NEWLINE + term: ['+'] NUMBER + """ + self.assertEqual(self.calculate_first_sets(grammar), {"term": {"NUMBER", "'+'"}, "start": {"NUMBER", "'+'"}}) + + def test_repeat_0(self) -> None: + grammar = """ + start: thing* "+" NEWLINE + thing: NUMBER + """ + self.assertEqual(self.calculate_first_sets(grammar), {"thing": {"NUMBER"}, "start": {'"+"', "NUMBER"}}) + + def test_repeat_0_with_group(self) -> None: + grammar = """ + start: ('+' '-')* term NEWLINE + term: NUMBER + """ + self.assertEqual(self.calculate_first_sets(grammar), {"term": {"NUMBER"}, "start": {"'+'", "NUMBER"}}) + + def test_repeat_1(self) -> None: + grammar = """ + start: thing+ '-' NEWLINE + thing: NUMBER + """ + self.assertEqual(self.calculate_first_sets(grammar), {"thing": {"NUMBER"}, "start": {"NUMBER"}}) + + def test_repeat_1_with_group(self) -> None: + grammar = """ + start: ('+' term)+ term NEWLINE + term: NUMBER + """ + self.assertEqual(self.calculate_first_sets(grammar), {"term": {"NUMBER"}, "start": {"'+'"}}) + + def test_gather(self) -> None: + grammar = """ + start: ','.thing+ NEWLINE + thing: NUMBER + """ + self.assertEqual(self.calculate_first_sets(grammar), {"thing": {"NUMBER"}, "start": {"NUMBER"}}) + + def test_positive_lookahead(self) -> None: + grammar = """ + start: expr NEWLINE + expr: &'a' opt + opt: 'a' | 'b' | 'c' + """ + self.assertEqual(self.calculate_first_sets(grammar), { + "expr": {"'a'"}, + "start": {"'a'"}, + "opt": {"'b'", "'c'", "'a'"}, + }) + + def test_negative_lookahead(self) -> None: + grammar = """ + start: expr NEWLINE + expr: !'a' opt + opt: 'a' | 'b' | 'c' + """ + self.assertEqual(self.calculate_first_sets(grammar), { + "opt": {"'b'", "'a'", "'c'"}, + "expr": {"'b'", "'c'"}, + "start": {"'b'", "'c'"}, + }) + + def test_left_recursion(self) -> None: + grammar = """ + start: expr NEWLINE + expr: ('-' term | expr '+' term | term) + term: NUMBER + foo: 'foo' + bar: 'bar' + baz: 'baz' + """ + self.assertEqual(self.calculate_first_sets(grammar), { + "expr": {"NUMBER", "'-'"}, + "term": {"NUMBER"}, + "start": {"NUMBER", "'-'"}, + "foo": {"'foo'"}, + "bar": {"'bar'"}, + "baz": {"'baz'"}, + }) + + def test_advance_left_recursion(self) -> None: + grammar = """ + start: NUMBER | sign start + sign: ['-'] + """ + self.assertEqual(self.calculate_first_sets(grammar), {"sign": {"'-'", ""}, "start": {"'-'", "NUMBER"}}) + + def test_mutual_left_recursion(self) -> None: + grammar = """ + start: foo 'E' + foo: bar 'A' | 'B' + bar: foo 'C' | 'D' + """ + self.assertEqual(self.calculate_first_sets(grammar), { + "foo": {"'D'", "'B'"}, + "bar": {"'D'"}, + "start": {"'D'", "'B'"}, + }) + + def test_nasty_left_recursion(self) -> None: + # TODO: Validate this + grammar = """ + start: target '=' + target: maybe '+' | NAME + maybe: maybe '-' | target + """ + self.assertEqual(self.calculate_first_sets(grammar), {"maybe": set(), "target": {"NAME"}, "start": {"NAME"}}) + + def test_nullable_rule(self) -> None: + grammar = """ + start: sign thing $ + sign: ['-'] + thing: NUMBER + """ + self.assertEqual(self.calculate_first_sets(grammar), { + "sign": {"", "'-'"}, + "thing": {"NUMBER"}, + "start": {"NUMBER", "'-'"}, + }) + + def test_epsilon_production_in_start_rule(self) -> None: + grammar = """ + start: ['-'] $ + """ + self.assertEqual(self.calculate_first_sets(grammar), {"start": {"ENDMARKER", "'-'"}}) + + def test_multiple_nullable_rules(self) -> None: + grammar = """ + start: sign thing other another $ + sign: ['-'] + thing: ['+'] + other: '*' + another: '/' + """ + self.assertEqual(self.calculate_first_sets(grammar), { + "sign": {"", "'-'"}, + "thing": {"'+'", ""}, + "start": {"'+'", "'-'", "'*'"}, + "other": {"'*'"}, + "another": {"'/'"}, + }) diff --git a/Lib/test/test_peg_generator/test_pegen.py b/Lib/test/test_peg_generator/test_pegen.py new file mode 100644 index 0000000000000..581c7acd337e4 --- /dev/null +++ b/Lib/test/test_peg_generator/test_pegen.py @@ -0,0 +1,728 @@ +import io +import textwrap +import unittest + +from test import test_tools +from typing import Dict, Any +from tokenize import TokenInfo, NAME, NEWLINE, NUMBER, OP + +test_tools.skip_if_missing('peg_generator') +with test_tools.imports_under_tool('peg_generator'): + from pegen.grammar_parser import GeneratedParser as GrammarParser + from pegen.testutil import ( + parse_string, + generate_parser, + make_parser + ) + from pegen.grammar import GrammarVisitor, GrammarError, Grammar + from pegen.grammar_visualizer import ASTGrammarPrinter + from pegen.parser import Parser + from pegen.python_generator import PythonParserGenerator + + +class TestPegen(unittest.TestCase): + def test_parse_grammar(self) -> None: + grammar_source = """ + start: sum NEWLINE + sum: t1=term '+' t2=term { action } | term + term: NUMBER + """ + expected = """ + start: sum NEWLINE + sum: term '+' term | term + term: NUMBER + """ + grammar: Grammar = parse_string(grammar_source, GrammarParser) + rules = grammar.rules + self.assertEqual(str(grammar), textwrap.dedent(expected).strip()) + # Check the str() and repr() of a few rules; AST nodes don't support ==. + self.assertEqual(str(rules["start"]), "start: sum NEWLINE") + self.assertEqual(str(rules["sum"]), "sum: term '+' term | term") + expected_repr = "Rule('term', None, Rhs([Alt([NamedItem(None, NameLeaf('NUMBER'))])]))" + self.assertEqual(repr(rules["term"]), expected_repr) + + def test_long_rule_str(self) -> None: + grammar_source = """ + start: zero | one | one zero | one one | one zero zero | one zero one | one one zero | one one one + """ + expected = """ + start: + | zero + | one + | one zero + | one one + | one zero zero + | one zero one + | one one zero + | one one one + """ + grammar: Grammar = parse_string(grammar_source, GrammarParser) + self.assertEqual(str(grammar.rules["start"]), textwrap.dedent(expected).strip()) + + def test_typed_rules(self) -> None: + grammar = """ + start[int]: sum NEWLINE + sum[int]: t1=term '+' t2=term { action } | term + term[int]: NUMBER + """ + rules = parse_string(grammar, GrammarParser).rules + # Check the str() and repr() of a few rules; AST nodes don't support ==. + self.assertEqual(str(rules["start"]), "start: sum NEWLINE") + self.assertEqual(str(rules["sum"]), "sum: term '+' term | term") + self.assertEqual( + repr(rules["term"]), + "Rule('term', 'int', Rhs([Alt([NamedItem(None, NameLeaf('NUMBER'))])]))" + ) + + def test_repeat_with_separator_rules(self) -> None: + grammar = """ + start: ','.thing+ NEWLINE + thing: NUMBER + """ + rules = parse_string(grammar, GrammarParser).rules + self.assertEqual(str(rules["start"]), "start: ','.thing+ NEWLINE") + print(repr(rules["start"])) + self.assertTrue(repr(rules["start"]).startswith( + "Rule('start', None, Rhs([Alt([NamedItem(None, Gather(StringLeaf(\"','\"), NameLeaf('thing'" + )) + self.assertEqual(str(rules["thing"]), "thing: NUMBER") + + def test_expr_grammar(self) -> None: + grammar = """ + start: sum NEWLINE + sum: term '+' term | term + term: NUMBER + """ + parser_class = make_parser(grammar) + node = parse_string("42\n", parser_class) + self.assertEqual(node, [ + [[TokenInfo(NUMBER, string="42", start=(1, 0), end=(1, 2), line="42\n")]], + TokenInfo(NEWLINE, string="\n", start=(1, 2), end=(1, 3), line="42\n"), + ]) + + def test_optional_operator(self) -> None: + grammar = """ + start: sum NEWLINE + sum: term ('+' term)? + term: NUMBER + """ + parser_class = make_parser(grammar) + node = parse_string("1+2\n", parser_class) + self.assertEqual(node, [ + [ + [TokenInfo(NUMBER, string="1", start=(1, 0), end=(1, 1), line="1+2\n")], + [ + TokenInfo(OP, string="+", start=(1, 1), end=(1, 2), line="1+2\n"), + [TokenInfo(NUMBER, string="2", start=(1, 2), end=(1, 3), line="1+2\n")], + ], + ], + TokenInfo(NEWLINE, string="\n", start=(1, 3), end=(1, 4), line="1+2\n"), + ]) + node = parse_string("1\n", parser_class) + self.assertEqual(node, [ + [[TokenInfo(NUMBER, string="1", start=(1, 0), end=(1, 1), line="1\n")], None], + TokenInfo(NEWLINE, string="\n", start=(1, 1), end=(1, 2), line="1\n"), + ]) + + def test_optional_literal(self) -> None: + grammar = """ + start: sum NEWLINE + sum: term '+' ? + term: NUMBER + """ + parser_class = make_parser(grammar) + node = parse_string("1+\n", parser_class) + self.assertEqual(node, [ + [ + [TokenInfo(NUMBER, string="1", start=(1, 0), end=(1, 1), line="1+\n")], + TokenInfo(OP, string="+", start=(1, 1), end=(1, 2), line="1+\n"), + ], + TokenInfo(NEWLINE, string="\n", start=(1, 2), end=(1, 3), line="1+\n"), + ]) + node = parse_string("1\n", parser_class) + self.assertEqual(node, [ + [[TokenInfo(NUMBER, string="1", start=(1, 0), end=(1, 1), line="1\n")], None], + TokenInfo(NEWLINE, string="\n", start=(1, 1), end=(1, 2), line="1\n"), + ]) + + def test_alt_optional_operator(self) -> None: + grammar = """ + start: sum NEWLINE + sum: term ['+' term] + term: NUMBER + """ + parser_class = make_parser(grammar) + node = parse_string("1 + 2\n", parser_class) + self.assertEqual(node, [ + [ + [TokenInfo(NUMBER, string="1", start=(1, 0), end=(1, 1), line="1 + 2\n")], + [ + TokenInfo(OP, string="+", start=(1, 2), end=(1, 3), line="1 + 2\n"), + [TokenInfo(NUMBER, string="2", start=(1, 4), end=(1, 5), line="1 + 2\n")], + ], + ], + TokenInfo(NEWLINE, string="\n", start=(1, 5), end=(1, 6), line="1 + 2\n"), + ]) + node = parse_string("1\n", parser_class) + self.assertEqual(node, [ + [[TokenInfo(NUMBER, string="1", start=(1, 0), end=(1, 1), line="1\n")], None], + TokenInfo(NEWLINE, string="\n", start=(1, 1), end=(1, 2), line="1\n"), + ]) + + def test_repeat_0_simple(self) -> None: + grammar = """ + start: thing thing* NEWLINE + thing: NUMBER + """ + parser_class = make_parser(grammar) + node = parse_string("1 2 3\n", parser_class) + self.assertEqual(node, [ + [TokenInfo(NUMBER, string="1", start=(1, 0), end=(1, 1), line="1 2 3\n")], + [ + [[TokenInfo(NUMBER, string="2", start=(1, 2), end=(1, 3), line="1 2 3\n")]], + [[TokenInfo(NUMBER, string="3", start=(1, 4), end=(1, 5), line="1 2 3\n")]], + ], + TokenInfo(NEWLINE, string="\n", start=(1, 5), end=(1, 6), line="1 2 3\n"), + ]) + node = parse_string("1\n", parser_class) + self.assertEqual(node, [ + [TokenInfo(NUMBER, string="1", start=(1, 0), end=(1, 1), line="1\n")], + [], + TokenInfo(NEWLINE, string="\n", start=(1, 1), end=(1, 2), line="1\n"), + ]) + + def test_repeat_0_complex(self) -> None: + grammar = """ + start: term ('+' term)* NEWLINE + term: NUMBER + """ + parser_class = make_parser(grammar) + node = parse_string("1 + 2 + 3\n", parser_class) + self.assertEqual(node, [ + [TokenInfo(NUMBER, string="1", start=(1, 0), end=(1, 1), line="1 + 2 + 3\n")], + [ + [ + [ + TokenInfo(OP, string="+", start=(1, 2), end=(1, 3), line="1 + 2 + 3\n"), + [TokenInfo(NUMBER, string="2", start=(1, 4), end=(1, 5), line="1 + 2 + 3\n")], + ] + ], + [ + [ + TokenInfo(OP, string="+", start=(1, 6), end=(1, 7), line="1 + 2 + 3\n"), + [TokenInfo(NUMBER, string="3", start=(1, 8), end=(1, 9), line="1 + 2 + 3\n")], + ] + ], + ], + TokenInfo(NEWLINE, string="\n", start=(1, 9), end=(1, 10), line="1 + 2 + 3\n"), + ]) + + def test_repeat_1_simple(self) -> None: + grammar = """ + start: thing thing+ NEWLINE + thing: NUMBER + """ + parser_class = make_parser(grammar) + node = parse_string("1 2 3\n", parser_class) + self.assertEqual(node, [ + [TokenInfo(NUMBER, string="1", start=(1, 0), end=(1, 1), line="1 2 3\n")], + [ + [[TokenInfo(NUMBER, string="2", start=(1, 2), end=(1, 3), line="1 2 3\n")]], + [[TokenInfo(NUMBER, string="3", start=(1, 4), end=(1, 5), line="1 2 3\n")]], + ], + TokenInfo(NEWLINE, string="\n", start=(1, 5), end=(1, 6), line="1 2 3\n"), + ]) + with self.assertRaises(SyntaxError): + parse_string("1\n", parser_class) + + def test_repeat_1_complex(self) -> None: + grammar = """ + start: term ('+' term)+ NEWLINE + term: NUMBER + """ + parser_class = make_parser(grammar) + node = parse_string("1 + 2 + 3\n", parser_class) + self.assertEqual(node, [ + [TokenInfo(NUMBER, string="1", start=(1, 0), end=(1, 1), line="1 + 2 + 3\n")], + [ + [ + [ + TokenInfo(OP, string="+", start=(1, 2), end=(1, 3), line="1 + 2 + 3\n"), + [TokenInfo(NUMBER, string="2", start=(1, 4), end=(1, 5), line="1 + 2 + 3\n")], + ] + ], + [ + [ + TokenInfo(OP, string="+", start=(1, 6), end=(1, 7), line="1 + 2 + 3\n"), + [TokenInfo(NUMBER, string="3", start=(1, 8), end=(1, 9), line="1 + 2 + 3\n")], + ] + ], + ], + TokenInfo(NEWLINE, string="\n", start=(1, 9), end=(1, 10), line="1 + 2 + 3\n"), + ]) + with self.assertRaises(SyntaxError): + parse_string("1\n", parser_class) + + def test_repeat_with_sep_simple(self) -> None: + grammar = """ + start: ','.thing+ NEWLINE + thing: NUMBER + """ + parser_class = make_parser(grammar) + node = parse_string("1, 2, 3\n", parser_class) + self.assertEqual(node, [ + [ + [TokenInfo(NUMBER, string="1", start=(1, 0), end=(1, 1), line="1, 2, 3\n")], + [TokenInfo(NUMBER, string="2", start=(1, 3), end=(1, 4), line="1, 2, 3\n")], + [TokenInfo(NUMBER, string="3", start=(1, 6), end=(1, 7), line="1, 2, 3\n")], + ], + TokenInfo(NEWLINE, string="\n", start=(1, 7), end=(1, 8), line="1, 2, 3\n"), + ]) + + def test_left_recursive(self) -> None: + grammar_source = """ + start: expr NEWLINE + expr: ('-' term | expr '+' term | term) + term: NUMBER + foo: NAME+ + bar: NAME* + baz: NAME? + """ + grammar: Grammar = parse_string(grammar_source, GrammarParser) + parser_class = generate_parser(grammar) + rules = grammar.rules + self.assertFalse(rules["start"].left_recursive) + self.assertTrue(rules["expr"].left_recursive) + self.assertFalse(rules["term"].left_recursive) + self.assertFalse(rules["foo"].left_recursive) + self.assertFalse(rules["bar"].left_recursive) + self.assertFalse(rules["baz"].left_recursive) + node = parse_string("1 + 2 + 3\n", parser_class) + self.assertEqual(node, [ + [ + [ + [[TokenInfo(NUMBER, string="1", start=(1, 0), end=(1, 1), line="1 + 2 + 3\n")]], + TokenInfo(OP, string="+", start=(1, 2), end=(1, 3), line="1 + 2 + 3\n"), + [TokenInfo(NUMBER, string="2", start=(1, 4), end=(1, 5), line="1 + 2 + 3\n")], + ], + TokenInfo(OP, string="+", start=(1, 6), end=(1, 7), line="1 + 2 + 3\n"), + [TokenInfo(NUMBER, string="3", start=(1, 8), end=(1, 9), line="1 + 2 + 3\n")], + ], + TokenInfo(NEWLINE, string="\n", start=(1, 9), end=(1, 10), line="1 + 2 + 3\n"), + ]) + + def test_python_expr(self) -> None: + grammar = """ + start: expr NEWLINE? $ { ast.Expression(expr, lineno=1, col_offset=0) } + expr: ( expr '+' term { ast.BinOp(expr, ast.Add(), term, lineno=expr.lineno, col_offset=expr.col_offset, end_lineno=term.end_lineno, end_col_offset=term.end_col_offset) } + | expr '-' term { ast.BinOp(expr, ast.Sub(), term, lineno=expr.lineno, col_offset=expr.col_offset, end_lineno=term.end_lineno, end_col_offset=term.end_col_offset) } + | term { term } + ) + term: ( l=term '*' r=factor { ast.BinOp(l, ast.Mult(), r, lineno=l.lineno, col_offset=l.col_offset, end_lineno=r.end_lineno, end_col_offset=r.end_col_offset) } + | l=term '/' r=factor { ast.BinOp(l, ast.Div(), r, lineno=l.lineno, col_offset=l.col_offset, end_lineno=r.end_lineno, end_col_offset=r.end_col_offset) } + | factor { factor } + ) + factor: ( '(' expr ')' { expr } + | atom { atom } + ) + atom: ( n=NAME { ast.Name(id=n.string, ctx=ast.Load(), lineno=n.start[0], col_offset=n.start[1], end_lineno=n.end[0], end_col_offset=n.end[1]) } + | n=NUMBER { ast.Constant(value=ast.literal_eval(n.string), lineno=n.start[0], col_offset=n.start[1], end_lineno=n.end[0], end_col_offset=n.end[1]) } + ) + """ + parser_class = make_parser(grammar) + node = parse_string("(1 + 2*3 + 5)/(6 - 2)\n", parser_class) + code = compile(node, "", "eval") + val = eval(code) + self.assertEqual(val, 3.0) + + def test_nullable(self) -> None: + grammar_source = """ + start: sign NUMBER + sign: ['-' | '+'] + """ + grammar: Grammar = parse_string(grammar_source, GrammarParser) + out = io.StringIO() + genr = PythonParserGenerator(grammar, out) + rules = grammar.rules + self.assertFalse(rules["start"].nullable) # Not None! + self.assertTrue(rules["sign"].nullable) + + def test_advanced_left_recursive(self) -> None: + grammar_source = """ + start: NUMBER | sign start + sign: ['-'] + """ + grammar: Grammar = parse_string(grammar_source, GrammarParser) + out = io.StringIO() + genr = PythonParserGenerator(grammar, out) + rules = grammar.rules + self.assertFalse(rules["start"].nullable) # Not None! + self.assertTrue(rules["sign"].nullable) + self.assertTrue(rules["start"].left_recursive) + self.assertFalse(rules["sign"].left_recursive) + + def test_mutually_left_recursive(self) -> None: + grammar_source = """ + start: foo 'E' + foo: bar 'A' | 'B' + bar: foo 'C' | 'D' + """ + grammar: Grammar = parse_string(grammar_source, GrammarParser) + out = io.StringIO() + genr = PythonParserGenerator(grammar, out) + rules = grammar.rules + self.assertFalse(rules["start"].left_recursive) + self.assertTrue(rules["foo"].left_recursive) + self.assertTrue(rules["bar"].left_recursive) + genr.generate("<string>") + ns: Dict[str, Any] = {} + exec(out.getvalue(), ns) + parser_class: Type[Parser] = ns["GeneratedParser"] + node = parse_string("D A C A E", parser_class) + self.assertEqual(node, [ + [ + [ + [ + [TokenInfo(type=NAME, string="D", start=(1, 0), end=(1, 1), line="D A C A E")], + TokenInfo(type=NAME, string="A", start=(1, 2), end=(1, 3), line="D A C A E"), + ], + TokenInfo(type=NAME, string="C", start=(1, 4), end=(1, 5), line="D A C A E"), + ], + TokenInfo(type=NAME, string="A", start=(1, 6), end=(1, 7), line="D A C A E"), + ], + TokenInfo(type=NAME, string="E", start=(1, 8), end=(1, 9), line="D A C A E"), + ]) + node = parse_string("B C A E", parser_class) + self.assertIsNotNone(node) + self.assertEqual(node, [ + [ + [ + [TokenInfo(type=NAME, string="B", start=(1, 0), end=(1, 1), line="B C A E")], + TokenInfo(type=NAME, string="C", start=(1, 2), end=(1, 3), line="B C A E"), + ], + TokenInfo(type=NAME, string="A", start=(1, 4), end=(1, 5), line="B C A E"), + ], + TokenInfo(type=NAME, string="E", start=(1, 6), end=(1, 7), line="B C A E"), + ]) + + def test_nasty_mutually_left_recursive(self) -> None: + # This grammar does not recognize 'x - + =', much to my chagrin. + # But that's the way PEG works. + # [Breathlessly] + # The problem is that the toplevel target call + # recurses into maybe, which recognizes 'x - +', + # and then the toplevel target looks for another '+', + # which fails, so it retreats to NAME, + # which succeeds, so we end up just recognizing 'x', + # and then start fails because there's no '=' after that. + grammar_source = """ + start: target '=' + target: maybe '+' | NAME + maybe: maybe '-' | target + """ + grammar: Grammar = parse_string(grammar_source, GrammarParser) + out = io.StringIO() + genr = PythonParserGenerator(grammar, out) + genr.generate("<string>") + ns: Dict[str, Any] = {} + exec(out.getvalue(), ns) + parser_class = ns["GeneratedParser"] + with self.assertRaises(SyntaxError): + parse_string("x - + =", parser_class) + + def test_lookahead(self) -> None: + grammar = """ + start: (expr_stmt | assign_stmt) &'.' + expr_stmt: !(target '=') expr + assign_stmt: target '=' expr + expr: term ('+' term)* + target: NAME + term: NUMBER + """ + parser_class = make_parser(grammar) + node = parse_string("foo = 12 + 12 .", parser_class) + self.assertEqual(node, [ + [ + [ + [TokenInfo(NAME, string="foo", start=(1, 0), end=(1, 3), line="foo = 12 + 12 .")], + TokenInfo(OP, string="=", start=(1, 4), end=(1, 5), line="foo = 12 + 12 ."), + [ + [ + TokenInfo( + NUMBER, string="12", start=(1, 6), end=(1, 8), line="foo = 12 + 12 ." + ) + ], + [ + [ + [ + TokenInfo( + OP, + string="+", + start=(1, 9), + end=(1, 10), + line="foo = 12 + 12 .", + ), + [ + TokenInfo( + NUMBER, + string="12", + start=(1, 11), + end=(1, 13), + line="foo = 12 + 12 .", + ) + ], + ] + ] + ], + ], + ] + ] + ]) + + def test_named_lookahead_error(self) -> None: + grammar = """ + start: foo=!'x' NAME + """ + with self.assertRaises(SyntaxError): + make_parser(grammar) + + def test_start_leader(self) -> None: + grammar = """ + start: attr | NAME + attr: start '.' NAME + """ + # Would assert False without a special case in compute_left_recursives(). + make_parser(grammar) + + def test_left_recursion_too_complex(self) -> None: + grammar = """ + start: foo + foo: bar '+' | baz '+' | '+' + bar: baz '-' | foo '-' | '-' + baz: foo '*' | bar '*' | '*' + """ + with self.assertRaises(ValueError) as errinfo: + make_parser(grammar) + self.assertTrue("no leader" in str(errinfo.exception.value)) + + def test_cut(self) -> None: + grammar = """ + start: '(' ~ expr ')' + expr: NUMBER + """ + parser_class = make_parser(grammar) + node = parse_string("(1)", parser_class, verbose=True) + self.assertEqual(node, [ + TokenInfo(OP, string="(", start=(1, 0), end=(1, 1), line="(1)"), + [TokenInfo(NUMBER, string="1", start=(1, 1), end=(1, 2), line="(1)")], + TokenInfo(OP, string=")", start=(1, 2), end=(1, 3), line="(1)"), + ]) + + def test_dangling_reference(self) -> None: + grammar = """ + start: foo ENDMARKER + foo: bar NAME + """ + with self.assertRaises(GrammarError): + parser_class = make_parser(grammar) + + def test_bad_token_reference(self) -> None: + grammar = """ + start: foo + foo: NAMEE + """ + with self.assertRaises(GrammarError): + parser_class = make_parser(grammar) + + def test_missing_start(self) -> None: + grammar = """ + foo: NAME + """ + with self.assertRaises(GrammarError): + parser_class = make_parser(grammar) + + +class TestGrammarVisitor: + class Visitor(GrammarVisitor): + def __init__(self) -> None: + self.n_nodes = 0 + + def visit(self, node: Any, *args: Any, **kwargs: Any) -> None: + self.n_nodes += 1 + super().visit(node, *args, **kwargs) + + def test_parse_trivial_grammar(self) -> None: + grammar = """ + start: 'a' + """ + rules = parse_string(grammar, GrammarParser) + visitor = self.Visitor() + + visitor.visit(rules) + + self.assertEqual(visitor.n_nodes, 6) + + def test_parse_or_grammar(self) -> None: + grammar = """ + start: rule + rule: 'a' | 'b' + """ + rules = parse_string(grammar, GrammarParser) + visitor = self.Visitor() + + visitor.visit(rules) + + # Grammar/Rule/Rhs/Alt/NamedItem/NameLeaf -> 6 + # Rule/Rhs/ -> 2 + # Alt/NamedItem/StringLeaf -> 3 + # Alt/NamedItem/StringLeaf -> 3 + + self.assertEqual(visitor.n_nodes, 14) + + def test_parse_repeat1_grammar(self) -> None: + grammar = """ + start: 'a'+ + """ + rules = parse_string(grammar, GrammarParser) + visitor = self.Visitor() + + visitor.visit(rules) + + # Grammar/Rule/Rhs/Alt/NamedItem/Repeat1/StringLeaf -> 6 + self.assertEqual(visitor.n_nodes, 7) + + def test_parse_repeat0_grammar(self) -> None: + grammar = """ + start: 'a'* + """ + rules = parse_string(grammar, GrammarParser) + visitor = self.Visitor() + + visitor.visit(rules) + + # Grammar/Rule/Rhs/Alt/NamedItem/Repeat0/StringLeaf -> 6 + + self.assertEqual(visitor.n_nodes, 7) + + def test_parse_optional_grammar(self) -> None: + grammar = """ + start: 'a' ['b'] + """ + rules = parse_string(grammar, GrammarParser) + visitor = self.Visitor() + + visitor.visit(rules) + + # Grammar/Rule/Rhs/Alt/NamedItem/StringLeaf -> 6 + # NamedItem/Opt/Rhs/Alt/NamedItem/Stringleaf -> 6 + + self.assertEqual(visitor.n_nodes, 12) + + +class TestGrammarVisualizer(unittest.TestCase): + def test_simple_rule(self) -> None: + grammar = """ + start: 'a' 'b' + """ + rules = parse_string(grammar, GrammarParser) + + printer = ASTGrammarPrinter() + lines: List[str] = [] + printer.print_grammar_ast(rules, printer=lines.append) + + output = "\n".join(lines) + expected_output = textwrap.dedent( + """\ + └──Rule + └──Rhs + └──Alt + ├──NamedItem + │ └──StringLeaf("'a'") + └──NamedItem + └──StringLeaf("'b'") + """ + ) + + self.assertEqual(output, expected_output) + + def test_multiple_rules(self) -> None: + grammar = """ + start: a b + a: 'a' + b: 'b' + """ + rules = parse_string(grammar, GrammarParser) + + printer = ASTGrammarPrinter() + lines: List[str] = [] + printer.print_grammar_ast(rules, printer=lines.append) + + output = "\n".join(lines) + expected_output = textwrap.dedent( + """\ + └──Rule + └──Rhs + └──Alt + ├──NamedItem + │ └──NameLeaf('a') + └──NamedItem + └──NameLeaf('b') + + └──Rule + └──Rhs + └──Alt + └──NamedItem + └──StringLeaf("'a'") + + └──Rule + └──Rhs + └──Alt + └──NamedItem + └──StringLeaf("'b'") + """ + ) + + self.assertEqual(output, expected_output) + + def test_deep_nested_rule(self) -> None: + grammar = """ + start: 'a' ['b'['c'['d']]] + """ + rules = parse_string(grammar, GrammarParser) + + printer = ASTGrammarPrinter() + lines: List[str] = [] + printer.print_grammar_ast(rules, printer=lines.append) + + output = "\n".join(lines) + print() + print(output) + expected_output = textwrap.dedent( + """\ + └──Rule + └──Rhs + └──Alt + ├──NamedItem + │ └──StringLeaf("'a'") + └──NamedItem + └──Opt + └──Rhs + └──Alt + ├──NamedItem + │ └──StringLeaf("'b'") + └──NamedItem + └──Opt + └──Rhs + └──Alt + ├──NamedItem + │ └──StringLeaf("'c'") + └──NamedItem + └──Opt + └──Rhs + └──Alt + └──NamedItem + └──StringLeaf("'d'") + """ + ) + + self.assertEqual(output, expected_output) diff --git a/Lib/test/test_peg_parser.py b/Lib/test/test_peg_parser.py new file mode 100644 index 0000000000000..5aa6c0d8f4814 --- /dev/null +++ b/Lib/test/test_peg_parser.py @@ -0,0 +1,764 @@ +import ast +import os +import sys +import _peg_parser as peg_parser +import unittest +from pathlib import PurePath +from typing import Any, Union, Iterable, Tuple +from textwrap import dedent + + +TEST_CASES = [ + ('annotated_assignment', 'x: int = 42'), + ('annotated_assignment_with_tuple', 'x: tuple = 1, 2'), + ('annotated_assignment_with_parens', '(paren): int = 3+2'), + ('annotated_assignment_with_yield', 'x: int = yield 42'), + ('annotated_no_assignment', 'x: int'), + ('annotation_with_multiple_parens', '((parens)): int'), + ('annotation_with_parens', '(parens): int'), + ('annotated_assignment_with_attr', 'a.b: int'), + ('annotated_assignment_with_subscript', 'a[b]: int'), + ('annotated_assignment_with_attr_and_parens', '(a.b): int'), + ('annotated_assignment_with_subscript_and_parens', '(a[b]): int'), + ('assert', 'assert a'), + ('assert_message', 'assert a, b'), + ('assignment_false', 'a = False'), + ('assignment_none', 'a = None'), + ('assignment_true', 'a = True'), + ('assignment_paren', '(a) = 42'), + ('assignment_paren_multiple', '(a, b) = (0, 1)'), + ('asyncfor', + ''' + async for i in a: + pass + '''), + ('attribute_call', 'a.b()'), + ('attribute_multiple_names', 'abcd.efg.hij'), + ('attribute_simple', 'a.b'), + ('attributes_subscript', 'a.b[0]'), + ('augmented_assignment', 'x += 42'), + ('binop_add', '1 + 1'), + ('binop_add_multiple', '1 + 1 + 1 + 1'), + ('binop_all', '1 + 2 * 5 + 3 ** 2 - -3'), + ('binop_boolop_comp', '1 + 1 == 2 or 1 + 1 == 3 and not b'), + ('boolop_or', 'a or b'), + ('boolop_or_multiple', 'a or b or c'), + ('class_def_bases', + ''' + class C(A, B): + pass + '''), + ('class_def_decorators', + ''' + @a + class C: + pass + '''), + ('class_def_decorator_with_expression', + ''' + @lambda x: 42 + class C: + pass + '''), + ('class_def_decorator_with_expression_and_walrus', + ''' + @x:=lambda x: 42 + class C: + pass + '''), + + ('class_def_keywords', + ''' + class C(keyword=a+b, **c): + pass + '''), + ('class_def_mixed', + ''' + class C(A, B, keyword=0, **a): + pass + '''), + ('class_def_simple', + ''' + class C: + pass + '''), + ('class_def_starred_and_kwarg', + ''' + class C(A, B, *x, **y): + pass + '''), + ('class_def_starred_in_kwargs', + ''' + class C(A, x=2, *[B, C], y=3): + pass + '''), + ('call_attribute', 'f().b'), + ('call_genexp', 'f(i for i in a)'), + ('call_mixed_args', 'f(a, b, *c, **d)'), + ('call_mixed_args_named', 'f(a, b, *c, d=4, **v)'), + ('call_one_arg', 'f(a)'), + ('call_posarg_genexp', 'f(a, (i for i in a))'), + ('call_simple', 'f()'), + ('call_subscript', 'f()[0]'), + ('comp', 'a == b'), + ('comp_multiple', 'a == b == c'), + ('comp_paren_end', 'a == (b-1)'), + ('comp_paren_start', '(a-1) == b'), + ('decorator', + ''' + @a + def f(): + pass + '''), + ('decorator_async', + ''' + @a + async def d(): + pass + '''), + ('decorator_with_expression', + ''' + @lambda x: 42 + def f(): + pass + '''), + ('decorator_with_expression_and_walrus', + ''' + @x:=lambda x: 42 + def f(): + pass + '''), + ('del_attribute', 'del a.b'), + ('del_call_attribute', 'del a().c'), + ('del_call_genexp_attribute', 'del a(i for i in b).c'), + ('del_empty', 'del()'), + ('del_list', 'del a, [b, c]'), + ('del_mixed', 'del a[0].b().c'), + ('del_multiple', 'del a, b'), + ('del_multiple_calls_attribute', 'del a()().b'), + ('del_paren', 'del(a,b)'), + ('del_paren_single_target', 'del(a)'), + ('del_subscript_attribute', 'del a[0].b'), + ('del_tuple', 'del a, (b, c)'), + ('delete', 'del a'), + ('dict', + ''' + { + a: 1, + b: 2, + c: 3 + } + '''), + ('dict_comp', '{x:1 for x in a}'), + ('dict_comp_if', '{x:1+2 for x in a if b}'), + ('dict_empty', '{}'), + ('for', + ''' + for i in a: + pass + '''), + ('for_else', + ''' + for i in a: + pass + else: + pass + '''), + ('for_star_target_in_paren', 'for (a) in b: pass'), + ('for_star_targets_attribute', 'for a.b in c: pass'), + ('for_star_targets_call_attribute', 'for a().c in b: pass'), + ('for_star_targets_empty', 'for () in a: pass'), + ('for_star_targets_mixed', 'for a[0].b().c in d: pass'), + ('for_star_targets_mixed_starred', + ''' + for a, *b, (c, d) in e: + pass + '''), + ('for_star_targets_multiple', 'for a, b in c: pass'), + ('for_star_targets_nested_starred', 'for *[*a] in b: pass'), + ('for_star_targets_starred', 'for *a in b: pass'), + ('for_star_targets_subscript_attribute', 'for a[0].b in c: pass'), + ('for_star_targets_trailing_comma', + ''' + for a, (b, c), in d: + pass + '''), + ('for_star_targets_tuple', 'for a, (b, c) in d: pass'), + ('for_underscore', + ''' + for _ in a: + pass + '''), + ('function_return_type', + ''' + def f() -> Any: + pass + '''), + ('f-string_slice', "f'{x[2]}'"), + ('f-string_slice_upper', "f'{x[2:3]}'"), + ('f-string_slice_step', "f'{x[2:3:-2]}'"), + ('f-string_constant', "f'{42}'"), + ('f-string_boolop', "f'{x and y}'"), + ('f-string_named_expr', "f'{(x:=42)}'"), + ('f-string_binop', "f'{x+y}'"), + ('f-string_unaryop', "f'{not x}'"), + ('f-string_lambda', "f'{(lambda x, /, y, y2=42 , *z, k1, k2=34, **k3: 42)}'"), + ('f-string_lambda_call', "f'{(lambda: 2)(2)}'"), + ('f-string_ifexpr', "f'{x if y else z}'"), + ('f-string_dict', "f'{ {2:34, 3:34} }'"), + ('f-string_set', "f'{ {2,-45} }'"), + ('f-string_list', "f'{ [2,-45] }'"), + ('f-string_tuple', "f'{ (2,-45) }'"), + ('f-string_listcomp', "f'{[x for x in y if z]}'"), + ('f-string_setcomp', "f'{ {x for x in y if z} }'"), + ('f-string_dictcomp', "f'{ {x:x for x in y if z} }'"), + ('f-string_genexpr', "f'{ (x for x in y if z) }'"), + ('f-string_yield', "f'{ (yield x) }'"), + ('f-string_yieldfrom', "f'{ (yield from x) }'"), + ('f-string_await', "f'{ await x }'"), + ('f-string_compare', "f'{ x == y }'"), + ('f-string_call', "f'{ f(x,y,z) }'"), + ('f-string_attribute', "f'{ f.x.y.z }'"), + ('f-string_starred', "f'{ *x, }'"), + ('f-string_doublestarred', "f'{ {**x} }'"), + ('f-string_escape_brace', "f'{{Escape'"), + ('f-string_escape_closing_brace', "f'Escape}}'"), + ('f-string_repr', "f'{a!r}'"), + ('f-string_str', "f'{a!s}'"), + ('f-string_ascii', "f'{a!a}'"), + ('f-string_debug', "f'{a=}'"), + ('f-string_padding', "f'{a:03d}'"), + ('f-string_multiline', + """ + f''' + {hello} + ''' + """), + ('f-string_multiline_in_expr', + """ + f''' + { + hello + } + ''' + """), + ('f-string_multiline_in_call', + """ + f''' + {f( + a, b, c + )} + ''' + """), + ('global', 'global a, b'), + ('group', '(yield a)'), + ('if_elif', + ''' + if a: + pass + elif b: + pass + '''), + ('if_elif_elif', + ''' + if a: + pass + elif b: + pass + elif c: + pass + '''), + ('if_elif_else', + ''' + if a: + pass + elif b: + pass + else: + pass + '''), + ('if_else', + ''' + if a: + pass + else: + pass + '''), + ('if_simple', 'if a: pass'), + ('import', 'import a'), + ('import_alias', 'import a as b'), + ('import_dotted', 'import a.b'), + ('import_dotted_alias', 'import a.b as c'), + ('import_dotted_multichar', 'import ab.cd'), + ('import_from', 'from a import b'), + ('import_from_alias', 'from a import b as c'), + ('import_from_dotted', 'from a.b import c'), + ('import_from_dotted_alias', 'from a.b import c as d'), + ('import_from_multiple_aliases', 'from a import b as c, d as e'), + ('import_from_one_dot', 'from .a import b'), + ('import_from_one_dot_alias', 'from .a import b as c'), + ('import_from_star', 'from a import *'), + ('import_from_three_dots', 'from ...a import b'), + ('import_from_trailing_comma', 'from a import (b,)'), + ('kwarg', + ''' + def f(**a): + pass + '''), + ('kwonly_args', + ''' + def f(*, a, b): + pass + '''), + ('kwonly_args_with_default', + ''' + def f(*, a=2, b): + pass + '''), + ('lambda_kwarg', 'lambda **a: 42'), + ('lambda_kwonly_args', 'lambda *, a, b: 42'), + ('lambda_kwonly_args_with_default', 'lambda *, a=2, b: 42'), + ('lambda_mixed_args', 'lambda a, /, b, *, c: 42'), + ('lambda_mixed_args_with_default', 'lambda a, b=2, /, c=3, *e, f, **g: 42'), + ('lambda_no_args', 'lambda: 42'), + ('lambda_pos_args', 'lambda a,b: 42'), + ('lambda_pos_args_with_default', 'lambda a, b=2: 42'), + ('lambda_pos_only_args', 'lambda a, /: 42'), + ('lambda_pos_only_args_with_default', 'lambda a=0, /: 42'), + ('lambda_pos_posonly_args', 'lambda a, b, /, c, d: 42'), + ('lambda_pos_posonly_args_with_default', 'lambda a, b=0, /, c=2: 42'), + ('lambda_vararg', 'lambda *a: 42'), + ('lambda_vararg_kwonly_args', 'lambda *a, b: 42'), + ('list', '[1, 2, a]'), + ('list_comp', '[i for i in a]'), + ('list_comp_if', '[i for i in a if b]'), + ('list_trailing_comma', '[1+2, a, 3+4,]'), + ('mixed_args', + ''' + def f(a, /, b, *, c): + pass + '''), + ('mixed_args_with_default', + ''' + def f(a, b=2, /, c=3, *e, f, **g): + pass + '''), + ('multipart_string_bytes', 'b"Hola" b"Hello" b"Bye"'), + ('multipart_string_triple', '"""Something here""" "and now"'), + ('multipart_string_different_prefixes', 'u"Something" "Other thing" r"last thing"'), + ('multiple_assignments', 'x = y = z = 42'), + ('multiple_assignments_with_yield', 'x = y = z = yield 42'), + ('multiple_pass', + ''' + pass; pass + pass + '''), + ('namedexpr', '(x := [1, 2, 3])'), + ('namedexpr_false', '(x := False)'), + ('namedexpr_none', '(x := None)'), + ('namedexpr_true', '(x := True)'), + ('nonlocal', 'nonlocal a, b'), + ('number_complex', '-2.234+1j'), + ('number_float', '-34.2333'), + ('number_imaginary_literal', '1.1234j'), + ('number_integer', '-234'), + ('number_underscores', '1_234_567'), + ('pass', 'pass'), + ('pos_args', + ''' + def f(a, b): + pass + '''), + ('pos_args_with_default', + ''' + def f(a, b=2): + pass + '''), + ('pos_only_args', + ''' + def f(a, /): + pass + '''), + ('pos_only_args_with_default', + ''' + def f(a=0, /): + pass + '''), + ('pos_posonly_args', + ''' + def f(a, b, /, c, d): + pass + '''), + ('pos_posonly_args_with_default', + ''' + def f(a, b=0, /, c=2): + pass + '''), + ('primary_mixed', 'a.b.c().d[0]'), + ('raise', 'raise'), + ('raise_ellipsis', 'raise ...'), + ('raise_expr', 'raise a'), + ('raise_from', 'raise a from b'), + ('return', 'return'), + ('return_expr', 'return a'), + ('set', '{1, 2+4, 3+5}'), + ('set_comp', '{i for i in a}'), + ('set_trailing_comma', '{1, 2, 3,}'), + ('simple_assignment', 'x = 42'), + ('simple_assignment_with_yield', 'x = yield 42'), + ('string_bytes', 'b"hello"'), + ('string_concatenation_bytes', 'b"hello" b"world"'), + ('string_concatenation_simple', '"abcd" "efgh"'), + ('string_format_simple', 'f"hello"'), + ('string_format_with_formatted_value', 'f"hello {world}"'), + ('string_simple', '"hello"'), + ('string_unicode', 'u"hello"'), + ('subscript_attribute', 'a[0].b'), + ('subscript_call', 'a[b]()'), + ('subscript_multiple_slices', 'a[0:a:2, 1]'), + ('subscript_simple', 'a[0]'), + ('subscript_single_element_tuple', 'a[0,]'), + ('subscript_trailing_comma', 'a[0, 1, 2,]'), + ('subscript_tuple', 'a[0, 1, 2]'), + ('subscript_whole_slice', 'a[0+1:b:c]'), + ('try_except', + ''' + try: + pass + except: + pass + '''), + ('try_except_else', + ''' + try: + pass + except: + pass + else: + pass + '''), + ('try_except_else_finally', + ''' + try: + pass + except: + pass + else: + pass + finally: + pass + '''), + ('try_except_expr', + ''' + try: + pass + except a: + pass + '''), + ('try_except_expr_target', + ''' + try: + pass + except a as b: + pass + '''), + ('try_except_finally', + ''' + try: + pass + except: + pass + finally: + pass + '''), + ('try_finally', + ''' + try: + pass + finally: + pass + '''), + ('unpacking_binop', '[*([1, 2, 3] + [3, 4, 5])]'), + ('unpacking_call', '[*b()]'), + ('unpacking_compare', '[*(x < y)]'), + ('unpacking_constant', '[*3]'), + ('unpacking_dict', '[*{1: 2, 3: 4}]'), + ('unpacking_dict_comprehension', '[*{x:y for x,y in z}]'), + ('unpacking_ifexpr', '[*([1, 2, 3] if x else y)]'), + ('unpacking_list', '[*[1,2,3]]'), + ('unpacking_list_comprehension', '[*[x for x in y]]'), + ('unpacking_namedexpr', '[*(x:=[1, 2, 3])]'), + ('unpacking_set', '[*{1,2,3}]'), + ('unpacking_set_comprehension', '[*{x for x in y}]'), + ('unpacking_string', '[*"myvalue"]'), + ('unpacking_tuple', '[*(1,2,3)]'), + ('unpacking_unaryop', '[*(not [1, 2, 3])]'), + ('unpacking_yield', '[*(yield 42)]'), + ('unpacking_yieldfrom', '[*(yield from x)]'), + ('tuple', '(1, 2, 3)'), + ('vararg', + ''' + def f(*a): + pass + '''), + ('vararg_kwonly_args', + ''' + def f(*a, b): + pass + '''), + ('while', + ''' + while a: + pass + '''), + ('while_else', + ''' + while a: + pass + else: + pass + '''), + ('with', + ''' + with a: + pass + '''), + ('with_as', + ''' + with a as b: + pass + '''), + ('with_as_paren', + ''' + with a as (b): + pass + '''), + ('with_as_empty', 'with a as (): pass'), + ('with_list_recursive', + ''' + with a as [x, [y, z]]: + pass + '''), + ('with_tuple_recursive', + ''' + with a as ((x, y), z): + pass + '''), + ('with_tuple_target', + ''' + with a as (x, y): + pass + '''), + ('yield', 'yield'), + ('yield_expr', 'yield a'), + ('yield_from', 'yield from a'), +] + +FAIL_TEST_CASES = [ + ("annotation_multiple_targets", "(a, b): int = 42"), + ("annotation_nested_tuple", "((a, b)): int"), + ("annotation_list", "[a]: int"), + ("annotation_lambda", "lambda: int = 42"), + ("annotation_tuple", "(a,): int"), + ("annotation_tuple_without_paren", "a,: int"), + ("assignment_keyword", "a = if"), + ("comprehension_lambda", "(a for a in lambda: b)"), + ("comprehension_else", "(a for a in b if c else d"), + ("del_call", "del a()"), + ("del_call_genexp", "del a(i for i in b)"), + ("del_subscript_call", "del a[b]()"), + ("del_attribute_call", "del a.b()"), + ("del_mixed_call", "del a[0].b().c.d()"), + ("for_star_targets_call", "for a() in b: pass"), + ("for_star_targets_subscript_call", "for a[b]() in c: pass"), + ("for_star_targets_attribute_call", "for a.b() in c: pass"), + ("for_star_targets_mixed_call", "for a[0].b().c.d() in e: pass"), + ("for_star_targets_in", "for a, in in b: pass"), + ("f-string_assignment", "f'{x = 42}'"), + ("f-string_empty", "f'{}'"), + ("f-string_function_def", "f'{def f(): pass}'"), + ("f-string_lambda", "f'{lambda x: 42}'"), + ("f-string_singe_brace", "f'{'"), + ("f-string_single_closing_brace", "f'}'"), + ("from_import_invalid", "from import import a"), + ("from_import_trailing_comma", "from a import b,"), + # This test case checks error paths involving tokens with uninitialized + # values of col_offset and end_col_offset. + ("invalid indentation", + """ + def f(): + a + a + """), + ("not_terminated_string", "a = 'example"), +] + +FAIL_SPECIALIZED_MESSAGE_CASES = [ + ("f(x, y, z=1, **b, *a", "iterable argument unpacking follows keyword argument unpacking"), + ("f(x, y=1, *z, **a, b", "positional argument follows keyword argument unpacking"), + ("f(x, y, z=1, a=2, b", "positional argument follows keyword argument"), + ("True = 1", "cannot assign to True"), + ("a() = 1", "cannot assign to function call"), + ("(a, b): int", "only single target (not tuple) can be annotated"), + ("[a, b]: int", "only single target (not list) can be annotated"), + ("a(): int", "illegal target for annotation"), + ("1 += 1", "cannot assign to literal"), + ("pass\n pass", "unexpected indent"), + ("def f():\npass", "expected an indented block"), +] + +GOOD_BUT_FAIL_TEST_CASES = [ + ('string_concatenation_format', 'f"{hello} world" f"again {and_again}"'), + ('string_concatenation_multiple', + ''' + f"hello" f"{world} again" f"and_again" + '''), + ('f-string_multiline_comp', + """ + f''' + {(i for i in a + if b)} + ''' + """), +] + +FSTRINGS_TRACEBACKS = { + 'multiline_fstrings_same_line_with_brace': ( + """ + f''' + {a$b} + ''' + """, + '(a$b)', + ), + 'multiline_fstring_brace_on_next_line': ( + """ + f''' + {a$b + }''' + """, + '(a$b', + ), + 'multiline_fstring_brace_on_previous_line': ( + """ + f''' + { + a$b}''' + """, + 'a$b)', + ), +} + +EXPRESSIONS_TEST_CASES = [ + ("expression_add", "1+1"), + ("expression_add_2", "a+b"), + ("expression_call", "f(a, b=2, **kw)"), + ("expression_tuple", "1, 2, 3"), + ("expression_tuple_one_value", "1,") +] + + +def cleanup_source(source: Any) -> str: + if isinstance(source, str): + result = dedent(source) + elif not isinstance(source, (list, tuple)): + result = "\n".join(source) + else: + raise TypeError(f"Invalid type for test source: {source}") + return result + + +def prepare_test_cases( + test_cases: Iterable[Tuple[str, Union[str, Iterable[str]]]] +) -> Tuple[Iterable[str], Iterable[str]]: + + test_ids, _test_sources = zip(*test_cases) + test_sources = list(_test_sources) + for index, source in enumerate(test_sources): + result = cleanup_source(source) + test_sources[index] = result + return test_ids, test_sources + + +TEST_IDS, TEST_SOURCES = prepare_test_cases(TEST_CASES) + +GOOD_BUT_FAIL_TEST_IDS, GOOD_BUT_FAIL_SOURCES = prepare_test_cases( + GOOD_BUT_FAIL_TEST_CASES +) + +FAIL_TEST_IDS, FAIL_SOURCES = prepare_test_cases(FAIL_TEST_CASES) + +EXPRESSIONS_TEST_IDS, EXPRESSIONS_TEST_SOURCES = prepare_test_cases( + EXPRESSIONS_TEST_CASES +) + + +class ASTGenerationTest(unittest.TestCase): + def test_correct_ast_generation_on_source_files(self) -> None: + self.maxDiff = None + for source in TEST_SOURCES: + actual_ast = peg_parser.parse_string(source) + expected_ast = ast.parse(source) + self.assertEqual( + ast.dump(actual_ast, include_attributes=True), + ast.dump(expected_ast, include_attributes=True), + f"Wrong AST generation for source: {source}", + ) + + def test_incorrect_ast_generation_on_source_files(self) -> None: + for source in FAIL_SOURCES: + with self.assertRaises(SyntaxError, msg=f"Parsing {source} did not raise an exception"): + peg_parser.parse_string(source) + + def test_incorrect_ast_generation_with_specialized_errors(self) -> None: + for source, error_text in FAIL_SPECIALIZED_MESSAGE_CASES: + exc = IndentationError if "indent" in error_text else SyntaxError + with self.assertRaises(exc) as se: + peg_parser.parse_string(source) + self.assertTrue( + error_text in se.exception.msg, + f"Actual error message does not match expexted for {source}" + ) + + @unittest.skipIf(sys.flags.use_peg, "This tests nothing for now, since compile uses pegen as well") + @unittest.expectedFailure + def test_correct_but_known_to_fail_ast_generation_on_source_files(self) -> None: + for source in GOOD_BUT_FAIL_SOURCES: + actual_ast = peg_parser.parse_string(source) + expected_ast = ast.parse(source) + self.assertEqual( + ast.dump(actual_ast, include_attributes=True), + ast.dump(expected_ast, include_attributes=True), + f"Wrong AST generation for source: {source}", + ) + + def test_correct_ast_generation_without_pos_info(self) -> None: + for source in GOOD_BUT_FAIL_SOURCES: + actual_ast = peg_parser.parse_string(source) + expected_ast = ast.parse(source) + self.assertEqual( + ast.dump(actual_ast), + ast.dump(expected_ast), + f"Wrong AST generation for source: {source}", + ) + + def test_fstring_parse_error_tracebacks(self) -> None: + for source, error_text in FSTRINGS_TRACEBACKS.values(): + with self.assertRaises(SyntaxError) as se: + peg_parser.parse_string(dedent(source)) + self.assertEqual(error_text, se.exception.text) + + def test_correct_ast_generatrion_eval(self) -> None: + for source in EXPRESSIONS_TEST_SOURCES: + actual_ast = peg_parser.parse_string(source, mode='eval') + expected_ast = ast.parse(source, mode='eval') + self.assertEqual( + ast.dump(actual_ast, include_attributes=True), + ast.dump(expected_ast, include_attributes=True), + f"Wrong AST generation for source: {source}", + ) + + def test_tokenizer_errors_are_propagated(self) -> None: + n=201 + with self.assertRaisesRegex(SyntaxError, "too many nested parentheses"): + peg_parser.parse_string(n*'(' + ')'*n) diff --git a/Lib/test/test_positional_only_arg.py b/Lib/test/test_positional_only_arg.py index 0a9503e2025d6..332690051ed4d 100644 --- a/Lib/test/test_positional_only_arg.py +++ b/Lib/test/test_positional_only_arg.py @@ -3,6 +3,7 @@ import dis import pickle import unittest +import sys from test.support import check_syntax_error @@ -23,10 +24,12 @@ def assertRaisesSyntaxError(self, codestr, regex="invalid syntax"): compile(codestr + "\n", "<test>", "single") def test_invalid_syntax_errors(self): - check_syntax_error(self, "def f(a, b = 5, /, c): pass", "non-default argument follows default argument") - check_syntax_error(self, "def f(a = 5, b, /, c): pass", "non-default argument follows default argument") - check_syntax_error(self, "def f(a = 5, b=1, /, c, *, d=2): pass", "non-default argument follows default argument") - check_syntax_error(self, "def f(a = 5, b, /): pass", "non-default argument follows default argument") + if not sys.flags.use_peg: + check_syntax_error(self, "def f(a, b = 5, /, c): pass", "non-default argument follows default argument") + check_syntax_error(self, "def f(a = 5, b, /, c): pass", "non-default argument follows default argument") + check_syntax_error(self, "def f(a = 5, b=1, /, c, *, d=2): pass", "non-default argument follows default argument") + check_syntax_error(self, "def f(a = 5, b, /): pass", "non-default argument follows default argument") + check_syntax_error(self, "def f(*args, /): pass") check_syntax_error(self, "def f(*args, a, /): pass") check_syntax_error(self, "def f(**kwargs, /): pass") @@ -44,10 +47,12 @@ def test_invalid_syntax_errors(self): check_syntax_error(self, "def f(a, *, c, /, d, e): pass") def test_invalid_syntax_errors_async(self): - check_syntax_error(self, "async def f(a, b = 5, /, c): pass", "non-default argument follows default argument") - check_syntax_error(self, "async def f(a = 5, b, /, c): pass", "non-default argument follows default argument") - check_syntax_error(self, "async def f(a = 5, b=1, /, c, d=2): pass", "non-default argument follows default argument") - check_syntax_error(self, "async def f(a = 5, b, /): pass", "non-default argument follows default argument") + if not sys.flags.use_peg: + check_syntax_error(self, "async def f(a, b = 5, /, c): pass", "non-default argument follows default argument") + check_syntax_error(self, "async def f(a = 5, b, /, c): pass", "non-default argument follows default argument") + check_syntax_error(self, "async def f(a = 5, b=1, /, c, d=2): pass", "non-default argument follows default argument") + check_syntax_error(self, "async def f(a = 5, b, /): pass", "non-default argument follows default argument") + check_syntax_error(self, "async def f(*args, /): pass") check_syntax_error(self, "async def f(*args, a, /): pass") check_syntax_error(self, "async def f(**kwargs, /): pass") @@ -231,9 +236,11 @@ def test_lambdas(self): self.assertEqual(x(1, 2), 3) def test_invalid_syntax_lambda(self): - check_syntax_error(self, "lambda a, b = 5, /, c: None", "non-default argument follows default argument") - check_syntax_error(self, "lambda a = 5, b, /, c: None", "non-default argument follows default argument") - check_syntax_error(self, "lambda a = 5, b, /: None", "non-default argument follows default argument") + if not sys.flags.use_peg: + check_syntax_error(self, "lambda a, b = 5, /, c: None", "non-default argument follows default argument") + check_syntax_error(self, "lambda a = 5, b, /, c: None", "non-default argument follows default argument") + check_syntax_error(self, "lambda a = 5, b, /: None", "non-default argument follows default argument") + check_syntax_error(self, "lambda *args, /: None") check_syntax_error(self, "lambda *args, a, /: None") check_syntax_error(self, "lambda **kwargs, /: None") diff --git a/Lib/test/test_string_literals.py b/Lib/test/test_string_literals.py index 0cea2edc32afa..382c532df5e1e 100644 --- a/Lib/test/test_string_literals.py +++ b/Lib/test/test_string_literals.py @@ -119,7 +119,8 @@ def test_eval_str_invalid_escape(self): eval("'''\n\\z'''") self.assertEqual(len(w), 1) self.assertEqual(w[0].filename, '<string>') - self.assertEqual(w[0].lineno, 1) + if not sys.flags.use_peg: + self.assertEqual(w[0].lineno, 1) with warnings.catch_warnings(record=True) as w: warnings.simplefilter('error', category=DeprecationWarning) @@ -128,7 +129,8 @@ def test_eval_str_invalid_escape(self): exc = cm.exception self.assertEqual(w, []) self.assertEqual(exc.filename, '<string>') - self.assertEqual(exc.lineno, 1) + if not sys.flags.use_peg: + self.assertEqual(exc.lineno, 1) def test_eval_str_raw(self): self.assertEqual(eval(""" r'x' """), 'x') @@ -168,7 +170,8 @@ def test_eval_bytes_invalid_escape(self): eval("b'''\n\\z'''") self.assertEqual(len(w), 1) self.assertEqual(w[0].filename, '<string>') - self.assertEqual(w[0].lineno, 1) + if not sys.flags.use_peg: + self.assertEqual(w[0].lineno, 1) with warnings.catch_warnings(record=True) as w: warnings.simplefilter('error', category=DeprecationWarning) @@ -177,7 +180,8 @@ def test_eval_bytes_invalid_escape(self): exc = cm.exception self.assertEqual(w, []) self.assertEqual(exc.filename, '<string>') - self.assertEqual(exc.lineno, 1) + if not sys.flags.use_peg: + self.assertEqual(exc.lineno, 1) def test_eval_bytes_raw(self): self.assertEqual(eval(""" br'x' """), b'x') diff --git a/Lib/test/test_syntax.py b/Lib/test/test_syntax.py index a7e7e2c9e6f02..4798f22b2bb82 100644 --- a/Lib/test/test_syntax.py +++ b/Lib/test/test_syntax.py @@ -63,9 +63,10 @@ Traceback (most recent call last): SyntaxError: cannot assign to function call ->>> del f() -Traceback (most recent call last): -SyntaxError: cannot delete function call +# Pegen does not support this yet +# >>> del f() +# Traceback (most recent call last): +# SyntaxError: cannot delete function call >>> a + 1 = 2 Traceback (most recent call last): @@ -100,29 +101,30 @@ This test just checks a couple of cases rather than enumerating all of them. ->>> (a, "b", c) = (1, 2, 3) -Traceback (most recent call last): -SyntaxError: cannot assign to literal +# All of the following also produce different error messages with pegen +# >>> (a, "b", c) = (1, 2, 3) +# Traceback (most recent call last): +# SyntaxError: cannot assign to literal ->>> (a, True, c) = (1, 2, 3) -Traceback (most recent call last): -SyntaxError: cannot assign to True +# >>> (a, True, c) = (1, 2, 3) +# Traceback (most recent call last): +# SyntaxError: cannot assign to True >>> (a, __debug__, c) = (1, 2, 3) Traceback (most recent call last): SyntaxError: cannot assign to __debug__ ->>> (a, *True, c) = (1, 2, 3) -Traceback (most recent call last): -SyntaxError: cannot assign to True +# >>> (a, *True, c) = (1, 2, 3) +# Traceback (most recent call last): +# SyntaxError: cannot assign to True >>> (a, *__debug__, c) = (1, 2, 3) Traceback (most recent call last): SyntaxError: cannot assign to __debug__ ->>> [a, b, c + 1] = [1, 2, 3] -Traceback (most recent call last): -SyntaxError: cannot assign to operator +# >>> [a, b, c + 1] = [1, 2, 3] +# Traceback (most recent call last): +# SyntaxError: cannot assign to operator >>> a if 1 else b = 1 Traceback (most recent call last): @@ -186,9 +188,11 @@ >>> f(x for x in L, **{}) Traceback (most recent call last): SyntaxError: Generator expression must be parenthesized ->>> f(L, x for x in L) -Traceback (most recent call last): -SyntaxError: Generator expression must be parenthesized + +# >>> f(L, x for x in L) +# Traceback (most recent call last): +# SyntaxError: Generator expression must be parenthesized + >>> f(x for x in L, y for y in L) Traceback (most recent call last): SyntaxError: Generator expression must be parenthesized @@ -297,31 +301,34 @@ ... 290, 291, 292, 293, 294, 295, 296, 297, 298, 299) # doctest: +ELLIPSIS (0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, ..., 297, 298, 299) ->>> f(lambda x: x[0] = 3) -Traceback (most recent call last): -SyntaxError: expression cannot contain assignment, perhaps you meant "=="? +# >>> f(lambda x: x[0] = 3) +# Traceback (most recent call last): +# SyntaxError: expression cannot contain assignment, perhaps you meant "=="? The grammar accepts any test (basically, any expression) in the keyword slot of a call site. Test a few different options. ->>> f(x()=2) -Traceback (most recent call last): -SyntaxError: expression cannot contain assignment, perhaps you meant "=="? ->>> f(a or b=1) -Traceback (most recent call last): -SyntaxError: expression cannot contain assignment, perhaps you meant "=="? ->>> f(x.y=1) -Traceback (most recent call last): -SyntaxError: expression cannot contain assignment, perhaps you meant "=="? ->>> f((x)=2) -Traceback (most recent call last): -SyntaxError: expression cannot contain assignment, perhaps you meant "=="? ->>> f(True=2) -Traceback (most recent call last): -SyntaxError: cannot assign to True +# >>> f(x()=2) +# Traceback (most recent call last): +# SyntaxError: expression cannot contain assignment, perhaps you meant "=="? +# >>> f(a or b=1) +# Traceback (most recent call last): +# SyntaxError: expression cannot contain assignment, perhaps you meant "=="? +# >>> f(x.y=1) +# Traceback (most recent call last): +# SyntaxError: expression cannot contain assignment, perhaps you meant "=="? +# >>> f((x)=2) +# Traceback (most recent call last): +# SyntaxError: expression cannot contain assignment, perhaps you meant "=="? +# >>> f(True=2) +# Traceback (most recent call last): +# SyntaxError: cannot assign to True >>> f(__debug__=1) Traceback (most recent call last): SyntaxError: cannot assign to __debug__ +>>> __debug__: int +Traceback (most recent call last): +SyntaxError: cannot assign to __debug__ More set_context(): @@ -620,9 +627,9 @@ Traceback (most recent call last): SyntaxError: cannot assign to __debug__ - >>> with (lambda *:0): pass - Traceback (most recent call last): - SyntaxError: named arguments must follow bare * + # >>> with (lambda *:0): pass + # Traceback (most recent call last): + # SyntaxError: named arguments must follow bare * Corner-cases that used to crash: @@ -637,6 +644,7 @@ """ import re +import sys import unittest from test import support @@ -670,6 +678,8 @@ def _check_error(self, code, errtext, def test_assign_call(self): self._check_error("f() = 1", "assign") + @unittest.skipIf(sys.flags.use_peg, "Pegen does not produce a specialized error " + "message yet") def test_assign_del(self): self._check_error("del f()", "delete") diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index 329f7ddeb2c57..bd4ea4794426c 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -545,10 +545,10 @@ def __hash__(self): def test_sys_flags(self): self.assertTrue(sys.flags) attrs = ("debug", - "inspect", "interactive", "optimize", "dont_write_bytecode", - "no_user_site", "no_site", "ignore_environment", "verbose", - "bytes_warning", "quiet", "hash_randomization", "isolated", - "dev_mode", "utf8_mode") + "inspect", "interactive", "optimize", "use_peg", + "dont_write_bytecode", "no_user_site", "no_site", + "ignore_environment", "verbose", "bytes_warning", "quiet", + "hash_randomization", "isolated", "dev_mode", "utf8_mode") for attr in attrs: self.assertTrue(hasattr(sys.flags, attr), attr) attr_type = bool if attr == "dev_mode" else int diff --git a/Lib/test/test_traceback.py b/Lib/test/test_traceback.py index 60e0b582756b5..45f55e1f8ab6c 100644 --- a/Lib/test/test_traceback.py +++ b/Lib/test/test_traceback.py @@ -656,6 +656,8 @@ def outer_raise(): self.assertIn('inner_raise() # Marker', blocks[2]) self.check_zero_div(blocks[2]) + @unittest.skipIf(sys.flags.use_peg, + "Pegen is arguably better here, so no need to fix this") def test_syntax_error_offset_at_eol(self): # See #10186. def e(): diff --git a/Lib/test/test_type_comments.py b/Lib/test/test_type_comments.py index 017073a9f1d50..80506e4b12d03 100644 --- a/Lib/test/test_type_comments.py +++ b/Lib/test/test_type_comments.py @@ -218,6 +218,7 @@ def favk( """ +(a)unittest.skipIf(sys.flags.use_peg, "Pegen does not support type comments yet") class TypeCommentTests(unittest.TestCase): lowest = 4 # Lowest minor version supported diff --git a/Lib/test/test_unpack_ex.py b/Lib/test/test_unpack_ex.py index e333af78f1d2c..2f53457b232a6 100644 --- a/Lib/test/test_unpack_ex.py +++ b/Lib/test/test_unpack_ex.py @@ -158,14 +158,15 @@ ... SyntaxError: iterable unpacking cannot be used in comprehension -Generator expression in function arguments - - >>> list(*x for x in (range(5) for i in range(3))) - Traceback (most recent call last): - ... - list(*x for x in (range(5) for i in range(3))) - ^ - SyntaxError: invalid syntax +# Pegen is better here. +# Generator expression in function arguments + +# >>> list(*x for x in (range(5) for i in range(3))) +# Traceback (most recent call last): +# ... +# list(*x for x in (range(5) for i in range(3))) +# ^ +# SyntaxError: invalid syntax >>> dict(**x for x in [{1:2}]) Traceback (most recent call last): diff --git a/Lib/test/test_unparse.py b/Lib/test/test_unparse.py index d4089a3fc1cdf..f5441ed54eebf 100644 --- a/Lib/test/test_unparse.py +++ b/Lib/test/test_unparse.py @@ -6,6 +6,7 @@ import random import tokenize import ast +import sys def read_pyfile(filename): @@ -327,6 +328,7 @@ def test_constant_tuples(self): ast.Constant(value=(1, 2, 3), kind=None), "(1, 2, 3)" ) + @unittest.skipIf(sys.flags.use_peg, "Pegen does not support type annotation yet") def test_function_type(self): for function_type in ( "() -> int", diff --git a/Makefile.pre.in b/Makefile.pre.in index 4511e607d89aa..b34fa64ff4bdf 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -244,7 +244,7 @@ LIBOBJS= @LIBOBJS@ PYTHON= python$(EXE) BUILDPYTHON= python$(BUILDEXE) -PYTHON_FOR_REGEN=@PYTHON_FOR_REGEN@ +PYTHON_FOR_REGEN?=@PYTHON_FOR_REGEN@ UPDATE_FILE=@PYTHON_FOR_REGEN@ $(srcdir)/Tools/scripts/update_file.py PYTHON_FOR_BUILD=@PYTHON_FOR_BUILD@ _PYTHON_HOST_PLATFORM=@_PYTHON_HOST_PLATFORM@ @@ -295,6 +295,19 @@ LIBFFI_INCLUDEDIR= @LIBFFI_INCLUDEDIR@ ########################################################################## # Parser + +PEGEN_OBJS= \ + Parser/pegen/pegen.o \ + Parser/pegen/parse.o \ + Parser/pegen/parse_string.o \ + Parser/pegen/peg_api.o + + +PEGEN_HEADERS= \ + $(srcdir)/Include/pegen_interface.h \ + $(srcdir)/Parser/pegen/pegen.h \ + $(srcdir)/Parser/pegen/parse_string.h + POBJS= \ Parser/acceler.o \ Parser/grammar1.o \ @@ -303,9 +316,10 @@ POBJS= \ Parser/parser.o \ Parser/token.o \ -PARSER_OBJS= $(POBJS) Parser/myreadline.o Parser/parsetok.o Parser/tokenizer.o +PARSER_OBJS= $(POBJS) $(PEGEN_OBJS) Parser/myreadline.o Parser/parsetok.o Parser/tokenizer.o PARSER_HEADERS= \ + $(PEGEN_HEADERS) \ $(srcdir)/Include/grammar.h \ $(srcdir)/Include/parsetok.h \ $(srcdir)/Parser/parser.h \ @@ -731,7 +745,7 @@ regen-importlib: Programs/_freeze_importlib ############################################################################ # Regenerate all generated files -regen-all: regen-opcode regen-opcode-targets regen-typeslots regen-grammar \ +regen-all: regen-opcode regen-opcode-targets regen-typeslots regen-grammar regen-pegen \ regen-token regen-keyword regen-symbol regen-ast regen-importlib clinic ############################################################################ @@ -806,6 +820,12 @@ regen-grammar: regen-token $(UPDATE_FILE) $(srcdir)/Include/graminit.h $(srcdir)/Include/graminit.h.new $(UPDATE_FILE) $(srcdir)/Python/graminit.c $(srcdir)/Python/graminit.c.new +.PHONY: regen-pegen +regen-pegen: + PYTHONPATH=$(srcdir)/Tools/peg_generator $(PYTHON_FOR_REGEN) -m pegen -c -q $(srcdir)/Grammar/python.gram \ + -o $(srcdir)/Parser/pegen/parse.new.c + $(UPDATE_FILE) $(srcdir)/Parser/pegen/parse.c $(srcdir)/Parser/pegen/parse.new.c + .PHONY=regen-ast regen-ast: # Regenerate Include/Python-ast.h using Parser/asdl_c.py -h diff --git a/Misc/NEWS.d/next/Core and Builtins/2020-04-20-14-06-19.bpo-40334.CTLGEp.rst b/Misc/NEWS.d/next/Core and Builtins/2020-04-20-14-06-19.bpo-40334.CTLGEp.rst new file mode 100644 index 0000000000000..b52d310508a8a --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2020-04-20-14-06-19.bpo-40334.CTLGEp.rst @@ -0,0 +1,5 @@ +Switch to a new parser, based on PEG. For more details see PEP 617. To +temporarily switch back to the old parser, use ``-X oldparser`` or +``PYTHONOLDPARSER=1``. In Python 3.10 we will remove the old parser +completely, including the ``parser`` module (already deprecated) and +anything that depends on it. diff --git a/Modules/Setup b/Modules/Setup index 6f0374a206315..6bf142419de3d 100644 --- a/Modules/Setup +++ b/Modules/Setup @@ -134,6 +134,9 @@ faulthandler faulthandler.c # can call _PyTraceMalloc_NewReference(). _tracemalloc _tracemalloc.c hashtable.c +# PEG-based parser module -- slated to be *the* parser +_peg_parser _peg_parser.c + # The rest of the modules listed in this file are all commented out by # default. Usually they can be detected and built as dynamically # loaded modules by the new setup.py script added in Python 2.1. If diff --git a/Modules/_peg_parser.c b/Modules/_peg_parser.c new file mode 100644 index 0000000000000..0a84edcfc0082 --- /dev/null +++ b/Modules/_peg_parser.c @@ -0,0 +1,107 @@ +#include <Python.h> +#include <pegen_interface.h> + +PyObject * +_Py_parse_file(PyObject *self, PyObject *args, PyObject *kwds) +{ + static char *keywords[] = {"file", "mode", NULL}; + char *filename; + char *mode_str = "exec"; + + if (!PyArg_ParseTupleAndKeywords(args, kwds, "s|s", keywords, &filename, &mode_str)) { + return NULL; + } + + int mode; + if (strcmp(mode_str, "exec") == 0) { + mode = Py_file_input; + } + else if (strcmp(mode_str, "single") == 0) { + mode = Py_single_input; + } + else { + return PyErr_Format(PyExc_ValueError, "mode must be either 'exec' or 'single'"); + } + + PyArena *arena = PyArena_New(); + if (arena == NULL) { + return NULL; + } + + PyObject *result = NULL; + + mod_ty res = PyPegen_ASTFromFile(filename, mode, arena); + if (res == NULL) { + goto error; + } + result = PyAST_mod2obj(res); + +error: + PyArena_Free(arena); + return result; +} + +PyObject * +_Py_parse_string(PyObject *self, PyObject *args, PyObject *kwds) +{ + static char *keywords[] = {"string", "mode", NULL}; + char *the_string; + char *mode_str = "exec"; + + if (!PyArg_ParseTupleAndKeywords(args, kwds, "s|s", keywords, &the_string, &mode_str)) { + return NULL; + } + + int mode; + if (strcmp(mode_str, "exec") == 0) { + mode = Py_file_input; + } + else if (strcmp(mode_str, "eval") == 0) { + mode = Py_eval_input; + } + else if (strcmp(mode_str, "single") == 0) { + mode = Py_single_input; + } + else { + return PyErr_Format(PyExc_ValueError, "mode must be either 'exec' or 'eval' or 'single'"); + } + + PyArena *arena = PyArena_New(); + if (arena == NULL) { + return NULL; + } + + PyObject *result = NULL; + + PyCompilerFlags flags = _PyCompilerFlags_INIT; + flags.cf_flags = PyCF_IGNORE_COOKIE; + + mod_ty res = PyPegen_ASTFromString(the_string, mode, &flags, arena); + if (res == NULL) { + goto error; + } + result = PyAST_mod2obj(res); + +error: + PyArena_Free(arena); + return result; +} + +static PyMethodDef ParseMethods[] = { + {"parse_file", (PyCFunction)(void (*)(void))_Py_parse_file, METH_VARARGS|METH_KEYWORDS, "Parse a file."}, + {"parse_string", (PyCFunction)(void (*)(void))_Py_parse_string, METH_VARARGS|METH_KEYWORDS,"Parse a string."}, + {NULL, NULL, 0, NULL} /* Sentinel */ +}; + +static struct PyModuleDef parsemodule = { + PyModuleDef_HEAD_INIT, + .m_name = "peg_parser", + .m_doc = "A parser.", + .m_methods = ParseMethods, +}; + +PyMODINIT_FUNC +PyInit__peg_parser(void) +{ + return PyModule_Create(&parsemodule); +} diff --git a/PC/config.c b/PC/config.c index 8eaeb31c9b934..32af2a81aeb41 100644 --- a/PC/config.c +++ b/PC/config.c @@ -75,6 +75,8 @@ extern PyObject* PyInit__opcode(void); extern PyObject* PyInit__contextvars(void); +extern PyObject* PyInit__peg_parser(void); + /* tools/freeze/makeconfig.py marker for additional "extern" */ /* -- ADDMODULE MARKER 1 -- */ @@ -169,6 +171,7 @@ struct _inittab _PyImport_Inittab[] = { {"_opcode", PyInit__opcode}, {"_contextvars", PyInit__contextvars}, + {"_peg_parser", PyInit__peg_parser}, /* Sentinel */ {0, 0} diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index 862c5a821b1f9..d795c4d5a7d00 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -213,6 +213,8 @@ <ClInclude Include="..\Include\parsetok.h" /> <ClInclude Include="..\Include\patchlevel.h" /> <ClInclude Include="..\Include\picklebufobject.h" /> + <ClInclude Include="..\Include\pegen_interface.h" /> + <ClInclude Include="..\Include\pyhash.h" /> <ClInclude Include="..\Include\pyhash.h" /> <ClInclude Include="..\Include\py_curses.h" /> <ClInclude Include="..\Include\pyarena.h" /> @@ -276,6 +278,8 @@ <ClInclude Include="..\Objects\unicodetype_db.h" /> <ClInclude Include="..\Parser\parser.h" /> <ClInclude Include="..\Parser\tokenizer.h" /> + <ClInclude Include="..\Parser\pegen\parse_string.h" /> + <ClInclude Include="..\Parser\pegen\pegen.h" /> <ClInclude Include="..\PC\errmap.h" /> <ClInclude Include="..\PC\pyconfig.h" /> <ClInclude Include="..\Python\ceval_gil.h" /> @@ -338,6 +342,7 @@ <ClCompile Include="..\Modules\_opcode.c" /> <ClCompile Include="..\Modules\_operator.c" /> <ClCompile Include="..\Modules\parsermodule.c" /> + <ClCompile Include="..\Modules\_peg_parser.c" /> <ClCompile Include="..\Modules\posixmodule.c" /> <ClCompile Include="..\Modules\rotatingtree.c" /> <ClCompile Include="..\Modules\sha1module.c" /> @@ -419,6 +424,10 @@ <ClCompile Include="..\Parser\parsetok.c" /> <ClCompile Include="..\Parser\tokenizer.c" /> <ClCompile Include="..\Parser\token.c" /> + <ClCompile Include="..\Parser\pegen\pegen.c" /> + <ClCompile Include="..\Parser\pegen\parse.c" /> + <ClCompile Include="..\Parser\pegen\parse_string.c" /> + <ClCompile Include="..\Parser\pegen\peg_api.c" /> <ClCompile Include="..\PC\invalid_parameter_handler.c" /> <ClCompile Include="..\PC\winreg.c" /> <ClCompile Include="..\PC\config.c" /> diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index 9d6d997b5267e..8c02622fd552d 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -902,6 +902,18 @@ <ClCompile Include="..\Parser\grammar1.c"> <Filter>Parser</Filter> </ClCompile> + <ClCompile Include="..\Parser\pegen\pegen.c"> + <Filter>Parser</Filter> + </ClCompile> + <ClCompile Include="..\Parser\pegen\parse.c"> + <Filter>Parser</Filter> + </ClCompile> + <ClCompile Include="..\Parser\pegen\parse_string.c"> + <Filter>Parser</Filter> + </ClCompile> + <ClCompile Include="..\Parser\pegen\peg_api.c"> + <Filter>Parser</Filter> + </ClCompile> <ClCompile Include="..\Parser\listnode.c"> <Filter>Parser</Filter> </ClCompile> diff --git a/PCbuild/regen.vcxproj b/PCbuild/regen.vcxproj index 876b12bae9cdf..9fe8d6d0c3e11 100644 --- a/PCbuild/regen.vcxproj +++ b/PCbuild/regen.vcxproj @@ -166,6 +166,14 @@ </Copy> <Warning Text="Grammar updated. You will need to rebuild pythoncore to see the changes." Condition="'@(_UpdatedH)' != '' and '@(_UpdatedC)' != ''" /> </Target> + <Target Name="_RegenPegen" BeforeTargets="Build"> +  + <Exec Command=""$PYTHONPATH=$(srcdir)/Tools/peg_generator" "$(PythonExe)" -m pegen -c -q "$(PySourcePath)Grammar\python.gram" -o "$(IntDir)parse.c"" /> + <Copy SourceFiles="$(IntDir)parse.c" DestinationFiles="$(PySourcePath)Parser\pegen\parse.c"> + <Output TaskParameter="CopiedFiles" ItemName="_UpdatedParse" /> + </Copy> + <Warning Text="Pegen updated. You will need to rebuild pythoncore to see the changes." Condition="'@(_UpdatedParse)' != ''" /> + </Target> <Target Name="_RegenAST_H" AfterTargets="_RegenGrammar">  <Exec Command=""$(PythonExe)" "$(PySourcePath)Parser\asdl_c.py" -h "$(IntDir)Python-ast.h" "$(PySourcePath)Parser\Python.asdl"" /> @@ -222,4 +230,4 @@ <Clean Include="$(IntDir)graminit.c.new" /> </ItemGroup> </Target> -</Project> \ No newline at end of file +</Project> diff --git a/Parser/pegen/parse.c b/Parser/pegen/parse.c new file mode 100644 index 0000000000000..25607eaf73cdc --- /dev/null +++ b/Parser/pegen/parse.c @@ -0,0 +1,15391 @@ +// @generated by pegen.py from ./Grammar/python.gram +#include "pegen.h" +static const int n_keyword_lists = 15; +static KeywordToken *reserved_keywords[] = { + NULL, + NULL, + (KeywordToken[]) { + {"if", 510}, + {"in", 518}, + {"is", 526}, + {"as", 531}, + {"or", 532}, + {NULL, -1}, + }, + (KeywordToken[]) { + {"del", 503}, + {"try", 511}, + {"for", 517}, + {"def", 522}, + {"not", 525}, + {"and", 533}, + {NULL, -1}, + }, + (KeywordToken[]) { + {"pass", 502}, + {"from", 514}, + {"elif", 515}, + {"else", 516}, + {"with", 519}, + {"True", 527}, + {"None", 529}, + {NULL, -1}, + }, + (KeywordToken[]) { + {"raise", 501}, + {"yield", 504}, + {"break", 506}, + {"while", 512}, + {"class", 523}, + {"False", 528}, + {NULL, -1}, + }, + (KeywordToken[]) { + {"return", 500}, + {"assert", 505}, + {"global", 508}, + {"import", 513}, + {"except", 520}, + {"lambda", 524}, + {NULL, -1}, + }, + (KeywordToken[]) { + {"finally", 521}, + {NULL, -1}, + }, + (KeywordToken[]) { + {"continue", 507}, + {"nonlocal", 509}, + {NULL, -1}, + }, + NULL, + NULL, + NULL, + NULL, + NULL, + (KeywordToken[]) { + {"__new_parser__", 530}, + {NULL, -1}, + }, +}; +#define file_type 1000 +#define interactive_type 1001 +#define eval_type 1002 +#define fstring_type 1003 +#define statements_type 1004 +#define statement_type 1005 +#define statement_newline_type 1006 +#define simple_stmt_type 1007 +#define small_stmt_type 1008 +#define compound_stmt_type 1009 +#define assignment_type 1010 +#define augassign_type 1011 +#define global_stmt_type 1012 +#define nonlocal_stmt_type 1013 +#define yield_stmt_type 1014 +#define assert_stmt_type 1015 +#define del_stmt_type 1016 +#define import_stmt_type 1017 +#define import_name_type 1018 +#define import_from_type 1019 +#define import_from_targets_type 1020 +#define import_from_as_names_type 1021 +#define import_from_as_name_type 1022 +#define dotted_as_names_type 1023 +#define dotted_as_name_type 1024 +#define dotted_name_type 1025 // Left-recursive +#define if_stmt_type 1026 +#define elif_stmt_type 1027 +#define else_block_type 1028 +#define while_stmt_type 1029 +#define for_stmt_type 1030 +#define with_stmt_type 1031 +#define with_item_type 1032 +#define try_stmt_type 1033 +#define except_block_type 1034 +#define finally_block_type 1035 +#define return_stmt_type 1036 +#define raise_stmt_type 1037 +#define function_def_type 1038 +#define function_def_raw_type 1039 +#define params_type 1040 +#define parameters_type 1041 +#define slash_without_default_type 1042 +#define slash_with_default_type 1043 +#define star_etc_type 1044 +#define name_with_optional_default_type 1045 +#define names_with_default_type 1046 +#define name_with_default_type 1047 +#define plain_names_type 1048 +#define plain_name_type 1049 +#define kwds_type 1050 +#define annotation_type 1051 +#define decorators_type 1052 +#define class_def_type 1053 +#define class_def_raw_type 1054 +#define block_type 1055 +#define expressions_list_type 1056 +#define star_expressions_type 1057 +#define star_expression_type 1058 +#define star_named_expressions_type 1059 +#define star_named_expression_type 1060 +#define named_expression_type 1061 +#define annotated_rhs_type 1062 +#define expressions_type 1063 +#define expression_type 1064 +#define lambdef_type 1065 +#define lambda_parameters_type 1066 +#define lambda_slash_without_default_type 1067 +#define lambda_slash_with_default_type 1068 +#define lambda_star_etc_type 1069 +#define lambda_name_with_optional_default_type 1070 +#define lambda_names_with_default_type 1071 +#define lambda_name_with_default_type 1072 +#define lambda_plain_names_type 1073 +#define lambda_plain_name_type 1074 +#define lambda_kwds_type 1075 +#define disjunction_type 1076 +#define conjunction_type 1077 +#define inversion_type 1078 +#define comparison_type 1079 +#define compare_op_bitwise_or_pair_type 1080 +#define eq_bitwise_or_type 1081 +#define noteq_bitwise_or_type 1082 +#define lte_bitwise_or_type 1083 +#define lt_bitwise_or_type 1084 +#define gte_bitwise_or_type 1085 +#define gt_bitwise_or_type 1086 +#define notin_bitwise_or_type 1087 +#define in_bitwise_or_type 1088 +#define isnot_bitwise_or_type 1089 +#define is_bitwise_or_type 1090 +#define bitwise_or_type 1091 // Left-recursive +#define bitwise_xor_type 1092 // Left-recursive +#define bitwise_and_type 1093 // Left-recursive +#define shift_expr_type 1094 // Left-recursive +#define sum_type 1095 // Left-recursive +#define term_type 1096 // Left-recursive +#define factor_type 1097 +#define power_type 1098 +#define await_primary_type 1099 +#define primary_type 1100 // Left-recursive +#define slices_type 1101 +#define slice_type 1102 +#define atom_type 1103 +#define strings_type 1104 +#define list_type 1105 +#define listcomp_type 1106 +#define tuple_type 1107 +#define group_type 1108 +#define genexp_type 1109 +#define set_type 1110 +#define setcomp_type 1111 +#define dict_type 1112 +#define dictcomp_type 1113 +#define kvpairs_type 1114 +#define kvpair_type 1115 +#define for_if_clauses_type 1116 +#define yield_expr_type 1117 +#define arguments_type 1118 +#define args_type 1119 +#define kwargs_type 1120 +#define starred_expression_type 1121 +#define kwarg_or_starred_type 1122 +#define kwarg_or_double_starred_type 1123 +#define star_targets_type 1124 +#define star_targets_seq_type 1125 +#define star_target_type 1126 +#define star_atom_type 1127 +#define inside_paren_ann_assign_target_type 1128 +#define ann_assign_subscript_attribute_target_type 1129 +#define del_targets_type 1130 +#define del_target_type 1131 +#define del_t_atom_type 1132 +#define targets_type 1133 +#define target_type 1134 +#define t_primary_type 1135 // Left-recursive +#define t_lookahead_type 1136 +#define t_atom_type 1137 +#define incorrect_arguments_type 1138 +#define invalid_named_expression_type 1139 +#define invalid_assignment_type 1140 +#define invalid_block_type 1141 +#define invalid_comprehension_type 1142 +#define invalid_parameters_type 1143 +#define _loop0_1_type 1144 +#define _loop1_2_type 1145 +#define _loop0_4_type 1146 +#define _gather_3_type 1147 +#define _tmp_5_type 1148 +#define _tmp_6_type 1149 +#define _tmp_7_type 1150 +#define _tmp_8_type 1151 +#define _tmp_9_type 1152 +#define _tmp_10_type 1153 +#define _tmp_11_type 1154 +#define _tmp_12_type 1155 +#define _loop1_13_type 1156 +#define _tmp_14_type 1157 +#define _tmp_15_type 1158 +#define _loop0_17_type 1159 +#define _gather_16_type 1160 +#define _loop0_19_type 1161 +#define _gather_18_type 1162 +#define _tmp_20_type 1163 +#define _loop0_21_type 1164 +#define _loop1_22_type 1165 +#define _loop0_24_type 1166 +#define _gather_23_type 1167 +#define _tmp_25_type 1168 +#define _loop0_27_type 1169 +#define _gather_26_type 1170 +#define _tmp_28_type 1171 +#define _loop0_30_type 1172 +#define _gather_29_type 1173 +#define _loop0_32_type 1174 +#define _gather_31_type 1175 +#define _tmp_33_type 1176 +#define _loop1_34_type 1177 +#define _tmp_35_type 1178 +#define _tmp_36_type 1179 +#define _tmp_37_type 1180 +#define _tmp_38_type 1181 +#define _tmp_39_type 1182 +#define _tmp_40_type 1183 +#define _tmp_41_type 1184 +#define _tmp_42_type 1185 +#define _tmp_43_type 1186 +#define _tmp_44_type 1187 +#define _tmp_45_type 1188 +#define _tmp_46_type 1189 +#define _loop0_47_type 1190 +#define _tmp_48_type 1191 +#define _loop1_49_type 1192 +#define _tmp_50_type 1193 +#define _tmp_51_type 1194 +#define _loop0_53_type 1195 +#define _gather_52_type 1196 +#define _loop0_55_type 1197 +#define _gather_54_type 1198 +#define _tmp_56_type 1199 +#define _loop1_57_type 1200 +#define _tmp_58_type 1201 +#define _loop0_60_type 1202 +#define _gather_59_type 1203 +#define _loop1_61_type 1204 +#define _loop0_63_type 1205 +#define _gather_62_type 1206 +#define _loop1_64_type 1207 +#define _tmp_65_type 1208 +#define _tmp_66_type 1209 +#define _tmp_67_type 1210 +#define _tmp_68_type 1211 +#define _tmp_69_type 1212 +#define _tmp_70_type 1213 +#define _tmp_71_type 1214 +#define _tmp_72_type 1215 +#define _tmp_73_type 1216 +#define _loop0_74_type 1217 +#define _tmp_75_type 1218 +#define _loop1_76_type 1219 +#define _tmp_77_type 1220 +#define _tmp_78_type 1221 +#define _loop0_80_type 1222 +#define _gather_79_type 1223 +#define _loop0_82_type 1224 +#define _gather_81_type 1225 +#define _loop1_83_type 1226 +#define _loop1_84_type 1227 +#define _loop1_85_type 1228 +#define _loop0_87_type 1229 +#define _gather_86_type 1230 +#define _tmp_88_type 1231 +#define _tmp_89_type 1232 +#define _tmp_90_type 1233 +#define _tmp_91_type 1234 +#define _loop1_92_type 1235 +#define _tmp_93_type 1236 +#define _tmp_94_type 1237 +#define _loop0_96_type 1238 +#define _gather_95_type 1239 +#define _loop1_97_type 1240 +#define _tmp_98_type 1241 +#define _tmp_99_type 1242 +#define _loop0_101_type 1243 +#define _gather_100_type 1244 +#define _loop0_103_type 1245 +#define _gather_102_type 1246 +#define _loop0_105_type 1247 +#define _gather_104_type 1248 +#define _loop0_107_type 1249 +#define _gather_106_type 1250 +#define _loop0_108_type 1251 +#define _loop0_110_type 1252 +#define _gather_109_type 1253 +#define _tmp_111_type 1254 +#define _loop0_113_type 1255 +#define _gather_112_type 1256 +#define _loop0_115_type 1257 +#define _gather_114_type 1258 +#define _tmp_116_type 1259 +#define _tmp_117_type 1260 +#define _tmp_118_type 1261 +#define _tmp_119_type 1262 +#define _tmp_120_type 1263 +#define _tmp_121_type 1264 +#define _tmp_122_type 1265 +#define _tmp_123_type 1266 +#define _tmp_124_type 1267 +#define _tmp_125_type 1268 +#define _tmp_126_type 1269 +#define _tmp_127_type 1270 +#define _tmp_128_type 1271 +#define _tmp_129_type 1272 +#define _tmp_130_type 1273 +#define _tmp_131_type 1274 +#define _tmp_132_type 1275 +#define _tmp_133_type 1276 +#define _tmp_134_type 1277 +#define _loop0_135_type 1278 +#define _tmp_136_type 1279 + +static mod_ty file_rule(Parser *p); +static mod_ty interactive_rule(Parser *p); +static mod_ty eval_rule(Parser *p); +static expr_ty fstring_rule(Parser *p); +static asdl_seq* statements_rule(Parser *p); +static asdl_seq* statement_rule(Parser *p); +static asdl_seq* statement_newline_rule(Parser *p); +static asdl_seq* simple_stmt_rule(Parser *p); +static stmt_ty small_stmt_rule(Parser *p); +static stmt_ty compound_stmt_rule(Parser *p); +static void *assignment_rule(Parser *p); +static AugOperator* augassign_rule(Parser *p); +static stmt_ty global_stmt_rule(Parser *p); +static stmt_ty nonlocal_stmt_rule(Parser *p); +static stmt_ty yield_stmt_rule(Parser *p); +static stmt_ty assert_stmt_rule(Parser *p); +static stmt_ty del_stmt_rule(Parser *p); +static stmt_ty import_stmt_rule(Parser *p); +static stmt_ty import_name_rule(Parser *p); +static stmt_ty import_from_rule(Parser *p); +static asdl_seq* import_from_targets_rule(Parser *p); +static asdl_seq* import_from_as_names_rule(Parser *p); +static alias_ty import_from_as_name_rule(Parser *p); +static asdl_seq* dotted_as_names_rule(Parser *p); +static alias_ty dotted_as_name_rule(Parser *p); +static expr_ty dotted_name_rule(Parser *p); +static stmt_ty if_stmt_rule(Parser *p); +static stmt_ty elif_stmt_rule(Parser *p); +static asdl_seq* else_block_rule(Parser *p); +static stmt_ty while_stmt_rule(Parser *p); +static stmt_ty for_stmt_rule(Parser *p); +static stmt_ty with_stmt_rule(Parser *p); +static withitem_ty with_item_rule(Parser *p); +static stmt_ty try_stmt_rule(Parser *p); +static excepthandler_ty except_block_rule(Parser *p); +static asdl_seq* finally_block_rule(Parser *p); +static stmt_ty return_stmt_rule(Parser *p); +static stmt_ty raise_stmt_rule(Parser *p); +static stmt_ty function_def_rule(Parser *p); +static stmt_ty function_def_raw_rule(Parser *p); +static arguments_ty params_rule(Parser *p); +static arguments_ty parameters_rule(Parser *p); +static asdl_seq* slash_without_default_rule(Parser *p); +static SlashWithDefault* slash_with_default_rule(Parser *p); +static StarEtc* star_etc_rule(Parser *p); +static NameDefaultPair* name_with_optional_default_rule(Parser *p); +static asdl_seq* names_with_default_rule(Parser *p); +static NameDefaultPair* name_with_default_rule(Parser *p); +static asdl_seq* plain_names_rule(Parser *p); +static arg_ty plain_name_rule(Parser *p); +static arg_ty kwds_rule(Parser *p); +static expr_ty annotation_rule(Parser *p); +static asdl_seq* decorators_rule(Parser *p); +static stmt_ty class_def_rule(Parser *p); +static stmt_ty class_def_raw_rule(Parser *p); +static asdl_seq* block_rule(Parser *p); +static asdl_seq* expressions_list_rule(Parser *p); +static expr_ty star_expressions_rule(Parser *p); +static expr_ty star_expression_rule(Parser *p); +static asdl_seq* star_named_expressions_rule(Parser *p); +static expr_ty star_named_expression_rule(Parser *p); +static expr_ty named_expression_rule(Parser *p); +static expr_ty annotated_rhs_rule(Parser *p); +static expr_ty expressions_rule(Parser *p); +static expr_ty expression_rule(Parser *p); +static expr_ty lambdef_rule(Parser *p); +static arguments_ty lambda_parameters_rule(Parser *p); +static asdl_seq* lambda_slash_without_default_rule(Parser *p); +static SlashWithDefault* lambda_slash_with_default_rule(Parser *p); +static StarEtc* lambda_star_etc_rule(Parser *p); +static NameDefaultPair* lambda_name_with_optional_default_rule(Parser *p); +static asdl_seq* lambda_names_with_default_rule(Parser *p); +static NameDefaultPair* lambda_name_with_default_rule(Parser *p); +static asdl_seq* lambda_plain_names_rule(Parser *p); +static arg_ty lambda_plain_name_rule(Parser *p); +static arg_ty lambda_kwds_rule(Parser *p); +static expr_ty disjunction_rule(Parser *p); +static expr_ty conjunction_rule(Parser *p); +static expr_ty inversion_rule(Parser *p); +static expr_ty comparison_rule(Parser *p); +static CmpopExprPair* compare_op_bitwise_or_pair_rule(Parser *p); +static CmpopExprPair* eq_bitwise_or_rule(Parser *p); +static CmpopExprPair* noteq_bitwise_or_rule(Parser *p); +static CmpopExprPair* lte_bitwise_or_rule(Parser *p); +static CmpopExprPair* lt_bitwise_or_rule(Parser *p); +static CmpopExprPair* gte_bitwise_or_rule(Parser *p); +static CmpopExprPair* gt_bitwise_or_rule(Parser *p); +static CmpopExprPair* notin_bitwise_or_rule(Parser *p); +static CmpopExprPair* in_bitwise_or_rule(Parser *p); +static CmpopExprPair* isnot_bitwise_or_rule(Parser *p); +static CmpopExprPair* is_bitwise_or_rule(Parser *p); +static expr_ty bitwise_or_rule(Parser *p); +static expr_ty bitwise_xor_rule(Parser *p); +static expr_ty bitwise_and_rule(Parser *p); +static expr_ty shift_expr_rule(Parser *p); +static expr_ty sum_rule(Parser *p); +static expr_ty term_rule(Parser *p); +static expr_ty factor_rule(Parser *p); +static expr_ty power_rule(Parser *p); +static expr_ty await_primary_rule(Parser *p); +static expr_ty primary_rule(Parser *p); +static expr_ty slices_rule(Parser *p); +static expr_ty slice_rule(Parser *p); +static expr_ty atom_rule(Parser *p); +static expr_ty strings_rule(Parser *p); +static expr_ty list_rule(Parser *p); +static expr_ty listcomp_rule(Parser *p); +static expr_ty tuple_rule(Parser *p); +static expr_ty group_rule(Parser *p); +static expr_ty genexp_rule(Parser *p); +static expr_ty set_rule(Parser *p); +static expr_ty setcomp_rule(Parser *p); +static expr_ty dict_rule(Parser *p); +static expr_ty dictcomp_rule(Parser *p); +static asdl_seq* kvpairs_rule(Parser *p); +static KeyValuePair* kvpair_rule(Parser *p); +static asdl_seq* for_if_clauses_rule(Parser *p); +static expr_ty yield_expr_rule(Parser *p); +static expr_ty arguments_rule(Parser *p); +static expr_ty args_rule(Parser *p); +static asdl_seq* kwargs_rule(Parser *p); +static expr_ty starred_expression_rule(Parser *p); +static KeywordOrStarred* kwarg_or_starred_rule(Parser *p); +static KeywordOrStarred* kwarg_or_double_starred_rule(Parser *p); +static expr_ty star_targets_rule(Parser *p); +static asdl_seq* star_targets_seq_rule(Parser *p); +static expr_ty star_target_rule(Parser *p); +static expr_ty star_atom_rule(Parser *p); +static expr_ty inside_paren_ann_assign_target_rule(Parser *p); +static expr_ty ann_assign_subscript_attribute_target_rule(Parser *p); +static asdl_seq* del_targets_rule(Parser *p); +static expr_ty del_target_rule(Parser *p); +static expr_ty del_t_atom_rule(Parser *p); +static asdl_seq* targets_rule(Parser *p); +static expr_ty target_rule(Parser *p); +static expr_ty t_primary_rule(Parser *p); +static void *t_lookahead_rule(Parser *p); +static expr_ty t_atom_rule(Parser *p); +static void *incorrect_arguments_rule(Parser *p); +static void *invalid_named_expression_rule(Parser *p); +static void *invalid_assignment_rule(Parser *p); +static void *invalid_block_rule(Parser *p); +static void *invalid_comprehension_rule(Parser *p); +static void *invalid_parameters_rule(Parser *p); +static asdl_seq *_loop0_1_rule(Parser *p); +static asdl_seq *_loop1_2_rule(Parser *p); +static asdl_seq *_loop0_4_rule(Parser *p); +static asdl_seq *_gather_3_rule(Parser *p); +static void *_tmp_5_rule(Parser *p); +static void *_tmp_6_rule(Parser *p); +static void *_tmp_7_rule(Parser *p); +static void *_tmp_8_rule(Parser *p); +static void *_tmp_9_rule(Parser *p); +static void *_tmp_10_rule(Parser *p); +static void *_tmp_11_rule(Parser *p); +static void *_tmp_12_rule(Parser *p); +static asdl_seq *_loop1_13_rule(Parser *p); +static void *_tmp_14_rule(Parser *p); +static void *_tmp_15_rule(Parser *p); +static asdl_seq *_loop0_17_rule(Parser *p); +static asdl_seq *_gather_16_rule(Parser *p); +static asdl_seq *_loop0_19_rule(Parser *p); +static asdl_seq *_gather_18_rule(Parser *p); +static void *_tmp_20_rule(Parser *p); +static asdl_seq *_loop0_21_rule(Parser *p); +static asdl_seq *_loop1_22_rule(Parser *p); +static asdl_seq *_loop0_24_rule(Parser *p); +static asdl_seq *_gather_23_rule(Parser *p); +static void *_tmp_25_rule(Parser *p); +static asdl_seq *_loop0_27_rule(Parser *p); +static asdl_seq *_gather_26_rule(Parser *p); +static void *_tmp_28_rule(Parser *p); +static asdl_seq *_loop0_30_rule(Parser *p); +static asdl_seq *_gather_29_rule(Parser *p); +static asdl_seq *_loop0_32_rule(Parser *p); +static asdl_seq *_gather_31_rule(Parser *p); +static void *_tmp_33_rule(Parser *p); +static asdl_seq *_loop1_34_rule(Parser *p); +static void *_tmp_35_rule(Parser *p); +static void *_tmp_36_rule(Parser *p); +static void *_tmp_37_rule(Parser *p); +static void *_tmp_38_rule(Parser *p); +static void *_tmp_39_rule(Parser *p); +static void *_tmp_40_rule(Parser *p); +static void *_tmp_41_rule(Parser *p); +static void *_tmp_42_rule(Parser *p); +static void *_tmp_43_rule(Parser *p); +static void *_tmp_44_rule(Parser *p); +static void *_tmp_45_rule(Parser *p); +static void *_tmp_46_rule(Parser *p); +static asdl_seq *_loop0_47_rule(Parser *p); +static void *_tmp_48_rule(Parser *p); +static asdl_seq *_loop1_49_rule(Parser *p); +static void *_tmp_50_rule(Parser *p); +static void *_tmp_51_rule(Parser *p); +static asdl_seq *_loop0_53_rule(Parser *p); +static asdl_seq *_gather_52_rule(Parser *p); +static asdl_seq *_loop0_55_rule(Parser *p); +static asdl_seq *_gather_54_rule(Parser *p); +static void *_tmp_56_rule(Parser *p); +static asdl_seq *_loop1_57_rule(Parser *p); +static void *_tmp_58_rule(Parser *p); +static asdl_seq *_loop0_60_rule(Parser *p); +static asdl_seq *_gather_59_rule(Parser *p); +static asdl_seq *_loop1_61_rule(Parser *p); +static asdl_seq *_loop0_63_rule(Parser *p); +static asdl_seq *_gather_62_rule(Parser *p); +static asdl_seq *_loop1_64_rule(Parser *p); +static void *_tmp_65_rule(Parser *p); +static void *_tmp_66_rule(Parser *p); +static void *_tmp_67_rule(Parser *p); +static void *_tmp_68_rule(Parser *p); +static void *_tmp_69_rule(Parser *p); +static void *_tmp_70_rule(Parser *p); +static void *_tmp_71_rule(Parser *p); +static void *_tmp_72_rule(Parser *p); +static void *_tmp_73_rule(Parser *p); +static asdl_seq *_loop0_74_rule(Parser *p); +static void *_tmp_75_rule(Parser *p); +static asdl_seq *_loop1_76_rule(Parser *p); +static void *_tmp_77_rule(Parser *p); +static void *_tmp_78_rule(Parser *p); +static asdl_seq *_loop0_80_rule(Parser *p); +static asdl_seq *_gather_79_rule(Parser *p); +static asdl_seq *_loop0_82_rule(Parser *p); +static asdl_seq *_gather_81_rule(Parser *p); +static asdl_seq *_loop1_83_rule(Parser *p); +static asdl_seq *_loop1_84_rule(Parser *p); +static asdl_seq *_loop1_85_rule(Parser *p); +static asdl_seq *_loop0_87_rule(Parser *p); +static asdl_seq *_gather_86_rule(Parser *p); +static void *_tmp_88_rule(Parser *p); +static void *_tmp_89_rule(Parser *p); +static void *_tmp_90_rule(Parser *p); +static void *_tmp_91_rule(Parser *p); +static asdl_seq *_loop1_92_rule(Parser *p); +static void *_tmp_93_rule(Parser *p); +static void *_tmp_94_rule(Parser *p); +static asdl_seq *_loop0_96_rule(Parser *p); +static asdl_seq *_gather_95_rule(Parser *p); +static asdl_seq *_loop1_97_rule(Parser *p); +static void *_tmp_98_rule(Parser *p); +static void *_tmp_99_rule(Parser *p); +static asdl_seq *_loop0_101_rule(Parser *p); +static asdl_seq *_gather_100_rule(Parser *p); +static asdl_seq *_loop0_103_rule(Parser *p); +static asdl_seq *_gather_102_rule(Parser *p); +static asdl_seq *_loop0_105_rule(Parser *p); +static asdl_seq *_gather_104_rule(Parser *p); +static asdl_seq *_loop0_107_rule(Parser *p); +static asdl_seq *_gather_106_rule(Parser *p); +static asdl_seq *_loop0_108_rule(Parser *p); +static asdl_seq *_loop0_110_rule(Parser *p); +static asdl_seq *_gather_109_rule(Parser *p); +static void *_tmp_111_rule(Parser *p); +static asdl_seq *_loop0_113_rule(Parser *p); +static asdl_seq *_gather_112_rule(Parser *p); +static asdl_seq *_loop0_115_rule(Parser *p); +static asdl_seq *_gather_114_rule(Parser *p); +static void *_tmp_116_rule(Parser *p); +static void *_tmp_117_rule(Parser *p); +static void *_tmp_118_rule(Parser *p); +static void *_tmp_119_rule(Parser *p); +static void *_tmp_120_rule(Parser *p); +static void *_tmp_121_rule(Parser *p); +static void *_tmp_122_rule(Parser *p); +static void *_tmp_123_rule(Parser *p); +static void *_tmp_124_rule(Parser *p); +static void *_tmp_125_rule(Parser *p); +static void *_tmp_126_rule(Parser *p); +static void *_tmp_127_rule(Parser *p); +static void *_tmp_128_rule(Parser *p); +static void *_tmp_129_rule(Parser *p); +static void *_tmp_130_rule(Parser *p); +static void *_tmp_131_rule(Parser *p); +static void *_tmp_132_rule(Parser *p); +static void *_tmp_133_rule(Parser *p); +static void *_tmp_134_rule(Parser *p); +static asdl_seq *_loop0_135_rule(Parser *p); +static void *_tmp_136_rule(Parser *p); + + +// file: statements? $ +static mod_ty +file_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + mod_ty res = NULL; + int mark = p->mark; + { // statements? $ + void *a; + void *endmarker_var; + if ( + (a = statements_rule(p), 1) + && + (endmarker_var = _PyPegen_endmarker_token(p)) + ) + { + res = Module ( a , NULL , p -> arena ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// interactive: statement_newline +static mod_ty +interactive_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + mod_ty res = NULL; + int mark = p->mark; + { // statement_newline + asdl_seq* a; + if ( + (a = statement_newline_rule(p)) + ) + { + res = Interactive ( a , p -> arena ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// eval: expressions NEWLINE* $ +static mod_ty +eval_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + mod_ty res = NULL; + int mark = p->mark; + { // expressions NEWLINE* $ + asdl_seq * _loop0_1_var; + expr_ty a; + void *endmarker_var; + if ( + (a = expressions_rule(p)) + && + (_loop0_1_var = _loop0_1_rule(p)) + && + (endmarker_var = _PyPegen_endmarker_token(p)) + ) + { + res = Expression ( a , p -> arena ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// fstring: star_expressions +static expr_ty +fstring_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + expr_ty res = NULL; + int mark = p->mark; + { // star_expressions + expr_ty star_expressions_var; + if ( + (star_expressions_var = star_expressions_rule(p)) + ) + { + res = star_expressions_var; + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// statements: statement+ +static asdl_seq* +statements_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + asdl_seq* res = NULL; + int mark = p->mark; + { // statement+ + asdl_seq * a; + if ( + (a = _loop1_2_rule(p)) + ) + { + res = _PyPegen_seq_flatten ( p , a ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// statement: compound_stmt | simple_stmt +static asdl_seq* +statement_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + asdl_seq* res = NULL; + int mark = p->mark; + { // compound_stmt + stmt_ty a; + if ( + (a = compound_stmt_rule(p)) + ) + { + res = _PyPegen_singleton_seq ( p , a ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + { // simple_stmt + asdl_seq* simple_stmt_var; + if ( + (simple_stmt_var = simple_stmt_rule(p)) + ) + { + res = simple_stmt_var; + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// statement_newline: compound_stmt NEWLINE | simple_stmt | NEWLINE | $ +static asdl_seq* +statement_newline_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + asdl_seq* res = NULL; + int mark = p->mark; + if (p->mark == p->fill && _PyPegen_fill_token(p) < 0) { + p->error_indicator = 1; + return NULL; + } + int start_lineno = p->tokens[mark]->lineno; + UNUSED(start_lineno); // Only used by EXTRA macro + int start_col_offset = p->tokens[mark]->col_offset; + UNUSED(start_col_offset); // Only used by EXTRA macro + { // compound_stmt NEWLINE + stmt_ty a; + void *newline_var; + if ( + (a = compound_stmt_rule(p)) + && + (newline_var = _PyPegen_newline_token(p)) + ) + { + res = _PyPegen_singleton_seq ( p , a ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + { // simple_stmt + asdl_seq* simple_stmt_var; + if ( + (simple_stmt_var = simple_stmt_rule(p)) + ) + { + res = simple_stmt_var; + goto done; + } + p->mark = mark; + } + { // NEWLINE + void *newline_var; + if ( + (newline_var = _PyPegen_newline_token(p)) + ) + { + Token *token = _PyPegen_get_last_nonnwhitespace_token(p); + if (token == NULL) { + return NULL; + } + int end_lineno = token->end_lineno; + UNUSED(end_lineno); // Only used by EXTRA macro + int end_col_offset = token->end_col_offset; + UNUSED(end_col_offset); // Only used by EXTRA macro + res = _PyPegen_singleton_seq ( p , CHECK ( _Py_Pass ( EXTRA ) ) ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + { // $ + void *endmarker_var; + if ( + (endmarker_var = _PyPegen_endmarker_token(p)) + ) + { + res = _PyPegen_interactive_exit ( p ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// simple_stmt: small_stmt !';' NEWLINE | ';'.small_stmt+ ';'? NEWLINE +static asdl_seq* +simple_stmt_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + asdl_seq* res = NULL; + int mark = p->mark; + { // small_stmt !';' NEWLINE + stmt_ty a; + void *newline_var; + if ( + (a = small_stmt_rule(p)) + && + _PyPegen_lookahead_with_int(0, _PyPegen_expect_token, p, 13) + && + (newline_var = _PyPegen_newline_token(p)) + ) + { + res = _PyPegen_singleton_seq ( p , a ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + { // ';'.small_stmt+ ';'? NEWLINE + asdl_seq * a; + void *newline_var; + void *opt_var; + UNUSED(opt_var); // Silence compiler warnings + if ( + (a = _gather_3_rule(p)) + && + (opt_var = _PyPegen_expect_token(p, 13), 1) + && + (newline_var = _PyPegen_newline_token(p)) + ) + { + res = a; + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// small_stmt: +// | assignment +// | star_expressions +// | &'return' return_stmt +// | &('import' | 'from') import_stmt +// | &'raise' raise_stmt +// | 'pass' +// | &'del' del_stmt +// | &'yield' yield_stmt +// | &'assert' assert_stmt +// | 'break' +// | 'continue' +// | &'global' global_stmt +// | &'nonlocal' nonlocal_stmt +static stmt_ty +small_stmt_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + stmt_ty res = NULL; + if (_PyPegen_is_memoized(p, small_stmt_type, &res)) + return res; + int mark = p->mark; + if (p->mark == p->fill && _PyPegen_fill_token(p) < 0) { + p->error_indicator = 1; + return NULL; + } + int start_lineno = p->tokens[mark]->lineno; + UNUSED(start_lineno); // Only used by EXTRA macro + int start_col_offset = p->tokens[mark]->col_offset; + UNUSED(start_col_offset); // Only used by EXTRA macro + { // assignment + void *assignment_var; + if ( + (assignment_var = assignment_rule(p)) + ) + { + res = assignment_var; + goto done; + } + p->mark = mark; + } + { // star_expressions + expr_ty e; + if ( + (e = star_expressions_rule(p)) + ) + { + Token *token = _PyPegen_get_last_nonnwhitespace_token(p); + if (token == NULL) { + return NULL; + } + int end_lineno = token->end_lineno; + UNUSED(end_lineno); // Only used by EXTRA macro + int end_col_offset = token->end_col_offset; + UNUSED(end_col_offset); // Only used by EXTRA macro + res = _Py_Expr ( e , EXTRA ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + { // &'return' return_stmt + stmt_ty return_stmt_var; + if ( + _PyPegen_lookahead_with_int(1, _PyPegen_expect_token, p, 500) + && + (return_stmt_var = return_stmt_rule(p)) + ) + { + res = return_stmt_var; + goto done; + } + p->mark = mark; + } + { // &('import' | 'from') import_stmt + stmt_ty import_stmt_var; + if ( + _PyPegen_lookahead(1, _tmp_5_rule, p) + && + (import_stmt_var = import_stmt_rule(p)) + ) + { + res = import_stmt_var; + goto done; + } + p->mark = mark; + } + { // &'raise' raise_stmt + stmt_ty raise_stmt_var; + if ( + _PyPegen_lookahead_with_int(1, _PyPegen_expect_token, p, 501) + && + (raise_stmt_var = raise_stmt_rule(p)) + ) + { + res = raise_stmt_var; + goto done; + } + p->mark = mark; + } + { // 'pass' + void *keyword; + if ( + (keyword = _PyPegen_expect_token(p, 502)) + ) + { + Token *token = _PyPegen_get_last_nonnwhitespace_token(p); + if (token == NULL) { + return NULL; + } + int end_lineno = token->end_lineno; + UNUSED(end_lineno); // Only used by EXTRA macro + int end_col_offset = token->end_col_offset; + UNUSED(end_col_offset); // Only used by EXTRA macro + res = _Py_Pass ( EXTRA ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + { // &'del' del_stmt + stmt_ty del_stmt_var; + if ( + _PyPegen_lookahead_with_int(1, _PyPegen_expect_token, p, 503) + && + (del_stmt_var = del_stmt_rule(p)) + ) + { + res = del_stmt_var; + goto done; + } + p->mark = mark; + } + { // &'yield' yield_stmt + stmt_ty yield_stmt_var; + if ( + _PyPegen_lookahead_with_int(1, _PyPegen_expect_token, p, 504) + && + (yield_stmt_var = yield_stmt_rule(p)) + ) + { + res = yield_stmt_var; + goto done; + } + p->mark = mark; + } + { // &'assert' assert_stmt + stmt_ty assert_stmt_var; + if ( + _PyPegen_lookahead_with_int(1, _PyPegen_expect_token, p, 505) + && + (assert_stmt_var = assert_stmt_rule(p)) + ) + { + res = assert_stmt_var; + goto done; + } + p->mark = mark; + } + { // 'break' + void *keyword; + if ( + (keyword = _PyPegen_expect_token(p, 506)) + ) + { + Token *token = _PyPegen_get_last_nonnwhitespace_token(p); + if (token == NULL) { + return NULL; + } + int end_lineno = token->end_lineno; + UNUSED(end_lineno); // Only used by EXTRA macro + int end_col_offset = token->end_col_offset; + UNUSED(end_col_offset); // Only used by EXTRA macro + res = _Py_Break ( EXTRA ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + { // 'continue' + void *keyword; + if ( + (keyword = _PyPegen_expect_token(p, 507)) + ) + { + Token *token = _PyPegen_get_last_nonnwhitespace_token(p); + if (token == NULL) { + return NULL; + } + int end_lineno = token->end_lineno; + UNUSED(end_lineno); // Only used by EXTRA macro + int end_col_offset = token->end_col_offset; + UNUSED(end_col_offset); // Only used by EXTRA macro + res = _Py_Continue ( EXTRA ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + { // &'global' global_stmt + stmt_ty global_stmt_var; + if ( + _PyPegen_lookahead_with_int(1, _PyPegen_expect_token, p, 508) + && + (global_stmt_var = global_stmt_rule(p)) + ) + { + res = global_stmt_var; + goto done; + } + p->mark = mark; + } + { // &'nonlocal' nonlocal_stmt + stmt_ty nonlocal_stmt_var; + if ( + _PyPegen_lookahead_with_int(1, _PyPegen_expect_token, p, 509) + && + (nonlocal_stmt_var = nonlocal_stmt_rule(p)) + ) + { + res = nonlocal_stmt_var; + goto done; + } + p->mark = mark; + } + res = NULL; + done: + _PyPegen_insert_memo(p, mark, small_stmt_type, res); + return res; +} + +// compound_stmt: +// | &('def' | '@' | ASYNC) function_def +// | &'if' if_stmt +// | &('class' | '@') class_def +// | &('with' | ASYNC) with_stmt +// | &('for' | ASYNC) for_stmt +// | &'try' try_stmt +// | &'while' while_stmt +static stmt_ty +compound_stmt_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + stmt_ty res = NULL; + int mark = p->mark; + { // &('def' | '@' | ASYNC) function_def + stmt_ty function_def_var; + if ( + _PyPegen_lookahead(1, _tmp_6_rule, p) + && + (function_def_var = function_def_rule(p)) + ) + { + res = function_def_var; + goto done; + } + p->mark = mark; + } + { // &'if' if_stmt + stmt_ty if_stmt_var; + if ( + _PyPegen_lookahead_with_int(1, _PyPegen_expect_token, p, 510) + && + (if_stmt_var = if_stmt_rule(p)) + ) + { + res = if_stmt_var; + goto done; + } + p->mark = mark; + } + { // &('class' | '@') class_def + stmt_ty class_def_var; + if ( + _PyPegen_lookahead(1, _tmp_7_rule, p) + && + (class_def_var = class_def_rule(p)) + ) + { + res = class_def_var; + goto done; + } + p->mark = mark; + } + { // &('with' | ASYNC) with_stmt + stmt_ty with_stmt_var; + if ( + _PyPegen_lookahead(1, _tmp_8_rule, p) + && + (with_stmt_var = with_stmt_rule(p)) + ) + { + res = with_stmt_var; + goto done; + } + p->mark = mark; + } + { // &('for' | ASYNC) for_stmt + stmt_ty for_stmt_var; + if ( + _PyPegen_lookahead(1, _tmp_9_rule, p) + && + (for_stmt_var = for_stmt_rule(p)) + ) + { + res = for_stmt_var; + goto done; + } + p->mark = mark; + } + { // &'try' try_stmt + stmt_ty try_stmt_var; + if ( + _PyPegen_lookahead_with_int(1, _PyPegen_expect_token, p, 511) + && + (try_stmt_var = try_stmt_rule(p)) + ) + { + res = try_stmt_var; + goto done; + } + p->mark = mark; + } + { // &'while' while_stmt + stmt_ty while_stmt_var; + if ( + _PyPegen_lookahead_with_int(1, _PyPegen_expect_token, p, 512) + && + (while_stmt_var = while_stmt_rule(p)) + ) + { + res = while_stmt_var; + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// assignment: +// | NAME ':' expression ['=' annotated_rhs] +// | ('(' inside_paren_ann_assign_target ')' | ann_assign_subscript_attribute_target) ':' expression ['=' annotated_rhs] +// | ((star_targets '='))+ (yield_expr | star_expressions) +// | target augassign (yield_expr | star_expressions) +// | invalid_assignment +static void * +assignment_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + void * res = NULL; + int mark = p->mark; + if (p->mark == p->fill && _PyPegen_fill_token(p) < 0) { + p->error_indicator = 1; + return NULL; + } + int start_lineno = p->tokens[mark]->lineno; + UNUSED(start_lineno); // Only used by EXTRA macro + int start_col_offset = p->tokens[mark]->col_offset; + UNUSED(start_col_offset); // Only used by EXTRA macro + { // NAME ':' expression ['=' annotated_rhs] + expr_ty a; + expr_ty b; + void *c; + void *literal; + if ( + (a = _PyPegen_name_token(p)) + && + (literal = _PyPegen_expect_token(p, 11)) + && + (b = expression_rule(p)) + && + (c = _tmp_10_rule(p), 1) + ) + { + Token *token = _PyPegen_get_last_nonnwhitespace_token(p); + if (token == NULL) { + return NULL; + } + int end_lineno = token->end_lineno; + UNUSED(end_lineno); // Only used by EXTRA macro + int end_col_offset = token->end_col_offset; + UNUSED(end_col_offset); // Only used by EXTRA macro + res = _Py_AnnAssign ( CHECK ( _PyPegen_set_expr_context ( p , a , Store ) ) , b , c , 1 , EXTRA ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + { // ('(' inside_paren_ann_assign_target ')' | ann_assign_subscript_attribute_target) ':' expression ['=' annotated_rhs] + void *a; + expr_ty b; + void *c; + void *literal; + if ( + (a = _tmp_11_rule(p)) + && + (literal = _PyPegen_expect_token(p, 11)) + && + (b = expression_rule(p)) + && + (c = _tmp_12_rule(p), 1) + ) + { + Token *token = _PyPegen_get_last_nonnwhitespace_token(p); + if (token == NULL) { + return NULL; + } + int end_lineno = token->end_lineno; + UNUSED(end_lineno); // Only used by EXTRA macro + int end_col_offset = token->end_col_offset; + UNUSED(end_col_offset); // Only used by EXTRA macro + res = _Py_AnnAssign ( a , b , c , 0 , EXTRA ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + { // ((star_targets '='))+ (yield_expr | star_expressions) + asdl_seq * a; + void *b; + if ( + (a = _loop1_13_rule(p)) + && + (b = _tmp_14_rule(p)) + ) + { + Token *token = _PyPegen_get_last_nonnwhitespace_token(p); + if (token == NULL) { + return NULL; + } + int end_lineno = token->end_lineno; + UNUSED(end_lineno); // Only used by EXTRA macro + int end_col_offset = token->end_col_offset; + UNUSED(end_col_offset); // Only used by EXTRA macro + res = _Py_Assign ( a , b , NULL , EXTRA ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + { // target augassign (yield_expr | star_expressions) + expr_ty a; + AugOperator* b; + void *c; + if ( + (a = target_rule(p)) + && + (b = augassign_rule(p)) + && + (c = _tmp_15_rule(p)) + ) + { + Token *token = _PyPegen_get_last_nonnwhitespace_token(p); + if (token == NULL) { + return NULL; + } + int end_lineno = token->end_lineno; + UNUSED(end_lineno); // Only used by EXTRA macro + int end_col_offset = token->end_col_offset; + UNUSED(end_col_offset); // Only used by EXTRA macro + res = _Py_AugAssign ( a , b -> kind , c , EXTRA ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + { // invalid_assignment + void *invalid_assignment_var; + if ( + (invalid_assignment_var = invalid_assignment_rule(p)) + ) + { + res = invalid_assignment_var; + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// augassign: +// | '+=' +// | '-=' +// | '*=' +// | '@=' +// | '/=' +// | '%=' +// | '&=' +// | '|=' +// | '^=' +// | '<<=' +// | '>>=' +// | '**=' +// | '//=' +static AugOperator* +augassign_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + AugOperator* res = NULL; + int mark = p->mark; + { // '+=' + void *literal; + if ( + (literal = _PyPegen_expect_token(p, 36)) + ) + { + res = _PyPegen_augoperator ( p , Add ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + { // '-=' + void *literal; + if ( + (literal = _PyPegen_expect_token(p, 37)) + ) + { + res = _PyPegen_augoperator ( p , Sub ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + { // '*=' + void *literal; + if ( + (literal = _PyPegen_expect_token(p, 38)) + ) + { + res = _PyPegen_augoperator ( p , Mult ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + { // '@=' + void *literal; + if ( + (literal = _PyPegen_expect_token(p, 50)) + ) + { + res = _PyPegen_augoperator ( p , MatMult ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + { // '/=' + void *literal; + if ( + (literal = _PyPegen_expect_token(p, 39)) + ) + { + res = _PyPegen_augoperator ( p , Div ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + { // '%=' + void *literal; + if ( + (literal = _PyPegen_expect_token(p, 40)) + ) + { + res = _PyPegen_augoperator ( p , Mod ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + { // '&=' + void *literal; + if ( + (literal = _PyPegen_expect_token(p, 41)) + ) + { + res = _PyPegen_augoperator ( p , BitAnd ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + { // '|=' + void *literal; + if ( + (literal = _PyPegen_expect_token(p, 42)) + ) + { + res = _PyPegen_augoperator ( p , BitOr ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + { // '^=' + void *literal; + if ( + (literal = _PyPegen_expect_token(p, 43)) + ) + { + res = _PyPegen_augoperator ( p , BitXor ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + { // '<<=' + void *literal; + if ( + (literal = _PyPegen_expect_token(p, 44)) + ) + { + res = _PyPegen_augoperator ( p , LShift ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + { // '>>=' + void *literal; + if ( + (literal = _PyPegen_expect_token(p, 45)) + ) + { + res = _PyPegen_augoperator ( p , RShift ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + { // '**=' + void *literal; + if ( + (literal = _PyPegen_expect_token(p, 46)) + ) + { + res = _PyPegen_augoperator ( p , Pow ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + { // '//=' + void *literal; + if ( + (literal = _PyPegen_expect_token(p, 48)) + ) + { + res = _PyPegen_augoperator ( p , FloorDiv ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// global_stmt: 'global' ','.NAME+ +static stmt_ty +global_stmt_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + stmt_ty res = NULL; + int mark = p->mark; + if (p->mark == p->fill && _PyPegen_fill_token(p) < 0) { + p->error_indicator = 1; + return NULL; + } + int start_lineno = p->tokens[mark]->lineno; + UNUSED(start_lineno); // Only used by EXTRA macro + int start_col_offset = p->tokens[mark]->col_offset; + UNUSED(start_col_offset); // Only used by EXTRA macro + { // 'global' ','.NAME+ + asdl_seq * a; + void *keyword; + if ( + (keyword = _PyPegen_expect_token(p, 508)) + && + (a = _gather_16_rule(p)) + ) + { + Token *token = _PyPegen_get_last_nonnwhitespace_token(p); + if (token == NULL) { + return NULL; + } + int end_lineno = token->end_lineno; + UNUSED(end_lineno); // Only used by EXTRA macro + int end_col_offset = token->end_col_offset; + UNUSED(end_col_offset); // Only used by EXTRA macro + res = _Py_Global ( CHECK ( _PyPegen_map_names_to_ids ( p , a ) ) , EXTRA ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// nonlocal_stmt: 'nonlocal' ','.NAME+ +static stmt_ty +nonlocal_stmt_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + stmt_ty res = NULL; + int mark = p->mark; + if (p->mark == p->fill && _PyPegen_fill_token(p) < 0) { + p->error_indicator = 1; + return NULL; + } + int start_lineno = p->tokens[mark]->lineno; + UNUSED(start_lineno); // Only used by EXTRA macro + int start_col_offset = p->tokens[mark]->col_offset; + UNUSED(start_col_offset); // Only used by EXTRA macro + { // 'nonlocal' ','.NAME+ + asdl_seq * a; + void *keyword; + if ( + (keyword = _PyPegen_expect_token(p, 509)) + && + (a = _gather_18_rule(p)) + ) + { + Token *token = _PyPegen_get_last_nonnwhitespace_token(p); + if (token == NULL) { + return NULL; + } + int end_lineno = token->end_lineno; + UNUSED(end_lineno); // Only used by EXTRA macro + int end_col_offset = token->end_col_offset; + UNUSED(end_col_offset); // Only used by EXTRA macro + res = _Py_Nonlocal ( CHECK ( _PyPegen_map_names_to_ids ( p , a ) ) , EXTRA ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// yield_stmt: yield_expr +static stmt_ty +yield_stmt_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + stmt_ty res = NULL; + int mark = p->mark; + if (p->mark == p->fill && _PyPegen_fill_token(p) < 0) { + p->error_indicator = 1; + return NULL; + } + int start_lineno = p->tokens[mark]->lineno; + UNUSED(start_lineno); // Only used by EXTRA macro + int start_col_offset = p->tokens[mark]->col_offset; + UNUSED(start_col_offset); // Only used by EXTRA macro + { // yield_expr + expr_ty y; + if ( + (y = yield_expr_rule(p)) + ) + { + Token *token = _PyPegen_get_last_nonnwhitespace_token(p); + if (token == NULL) { + return NULL; + } + int end_lineno = token->end_lineno; + UNUSED(end_lineno); // Only used by EXTRA macro + int end_col_offset = token->end_col_offset; + UNUSED(end_col_offset); // Only used by EXTRA macro + res = _Py_Expr ( y , EXTRA ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// assert_stmt: 'assert' expression [',' expression] +static stmt_ty +assert_stmt_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + stmt_ty res = NULL; + int mark = p->mark; + if (p->mark == p->fill && _PyPegen_fill_token(p) < 0) { + p->error_indicator = 1; + return NULL; + } + int start_lineno = p->tokens[mark]->lineno; + UNUSED(start_lineno); // Only used by EXTRA macro + int start_col_offset = p->tokens[mark]->col_offset; + UNUSED(start_col_offset); // Only used by EXTRA macro + { // 'assert' expression [',' expression] + expr_ty a; + void *b; + void *keyword; + if ( + (keyword = _PyPegen_expect_token(p, 505)) + && + (a = expression_rule(p)) + && + (b = _tmp_20_rule(p), 1) + ) + { + Token *token = _PyPegen_get_last_nonnwhitespace_token(p); + if (token == NULL) { + return NULL; + } + int end_lineno = token->end_lineno; + UNUSED(end_lineno); // Only used by EXTRA macro + int end_col_offset = token->end_col_offset; + UNUSED(end_col_offset); // Only used by EXTRA macro + res = _Py_Assert ( a , b , EXTRA ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// del_stmt: 'del' del_targets +static stmt_ty +del_stmt_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + stmt_ty res = NULL; + int mark = p->mark; + if (p->mark == p->fill && _PyPegen_fill_token(p) < 0) { + p->error_indicator = 1; + return NULL; + } + int start_lineno = p->tokens[mark]->lineno; + UNUSED(start_lineno); // Only used by EXTRA macro + int start_col_offset = p->tokens[mark]->col_offset; + UNUSED(start_col_offset); // Only used by EXTRA macro + { // 'del' del_targets + asdl_seq* a; + void *keyword; + if ( + (keyword = _PyPegen_expect_token(p, 503)) + && + (a = del_targets_rule(p)) + ) + { + Token *token = _PyPegen_get_last_nonnwhitespace_token(p); + if (token == NULL) { + return NULL; + } + int end_lineno = token->end_lineno; + UNUSED(end_lineno); // Only used by EXTRA macro + int end_col_offset = token->end_col_offset; + UNUSED(end_col_offset); // Only used by EXTRA macro + res = _Py_Delete ( a , EXTRA ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// import_stmt: import_name | import_from +static stmt_ty +import_stmt_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + stmt_ty res = NULL; + int mark = p->mark; + { // import_name + stmt_ty import_name_var; + if ( + (import_name_var = import_name_rule(p)) + ) + { + res = import_name_var; + goto done; + } + p->mark = mark; + } + { // import_from + stmt_ty import_from_var; + if ( + (import_from_var = import_from_rule(p)) + ) + { + res = import_from_var; + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// import_name: 'import' dotted_as_names +static stmt_ty +import_name_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + stmt_ty res = NULL; + int mark = p->mark; + if (p->mark == p->fill && _PyPegen_fill_token(p) < 0) { + p->error_indicator = 1; + return NULL; + } + int start_lineno = p->tokens[mark]->lineno; + UNUSED(start_lineno); // Only used by EXTRA macro + int start_col_offset = p->tokens[mark]->col_offset; + UNUSED(start_col_offset); // Only used by EXTRA macro + { // 'import' dotted_as_names + asdl_seq* a; + void *keyword; + if ( + (keyword = _PyPegen_expect_token(p, 513)) + && + (a = dotted_as_names_rule(p)) + ) + { + Token *token = _PyPegen_get_last_nonnwhitespace_token(p); + if (token == NULL) { + return NULL; + } + int end_lineno = token->end_lineno; + UNUSED(end_lineno); // Only used by EXTRA macro + int end_col_offset = token->end_col_offset; + UNUSED(end_col_offset); // Only used by EXTRA macro + res = _Py_Import ( a , EXTRA ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// import_from: +// | 'from' (('.' | '...'))* dotted_name 'import' import_from_targets +// | 'from' (('.' | '...'))+ 'import' import_from_targets +static stmt_ty +import_from_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + stmt_ty res = NULL; + int mark = p->mark; + if (p->mark == p->fill && _PyPegen_fill_token(p) < 0) { + p->error_indicator = 1; + return NULL; + } + int start_lineno = p->tokens[mark]->lineno; + UNUSED(start_lineno); // Only used by EXTRA macro + int start_col_offset = p->tokens[mark]->col_offset; + UNUSED(start_col_offset); // Only used by EXTRA macro + { // 'from' (('.' | '...'))* dotted_name 'import' import_from_targets + asdl_seq * a; + expr_ty b; + asdl_seq* c; + void *keyword; + void *keyword_1; + if ( + (keyword = _PyPegen_expect_token(p, 514)) + && + (a = _loop0_21_rule(p)) + && + (b = dotted_name_rule(p)) + && + (keyword_1 = _PyPegen_expect_token(p, 513)) + && + (c = import_from_targets_rule(p)) + ) + { + Token *token = _PyPegen_get_last_nonnwhitespace_token(p); + if (token == NULL) { + return NULL; + } + int end_lineno = token->end_lineno; + UNUSED(end_lineno); // Only used by EXTRA macro + int end_col_offset = token->end_col_offset; + UNUSED(end_col_offset); // Only used by EXTRA macro + res = _Py_ImportFrom ( b -> v . Name . id , c , _PyPegen_seq_count_dots ( a ) , EXTRA ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + { // 'from' (('.' | '...'))+ 'import' import_from_targets + asdl_seq * a; + asdl_seq* b; + void *keyword; + void *keyword_1; + if ( + (keyword = _PyPegen_expect_token(p, 514)) + && + (a = _loop1_22_rule(p)) + && + (keyword_1 = _PyPegen_expect_token(p, 513)) + && + (b = import_from_targets_rule(p)) + ) + { + Token *token = _PyPegen_get_last_nonnwhitespace_token(p); + if (token == NULL) { + return NULL; + } + int end_lineno = token->end_lineno; + UNUSED(end_lineno); // Only used by EXTRA macro + int end_col_offset = token->end_col_offset; + UNUSED(end_col_offset); // Only used by EXTRA macro + res = _Py_ImportFrom ( NULL , b , _PyPegen_seq_count_dots ( a ) , EXTRA ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// import_from_targets: '(' import_from_as_names ','? ')' | import_from_as_names | '*' +static asdl_seq* +import_from_targets_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + asdl_seq* res = NULL; + int mark = p->mark; + { // '(' import_from_as_names ','? ')' + asdl_seq* a; + void *literal; + void *literal_1; + void *opt_var; + UNUSED(opt_var); // Silence compiler warnings + if ( + (literal = _PyPegen_expect_token(p, 7)) + && + (a = import_from_as_names_rule(p)) + && + (opt_var = _PyPegen_expect_token(p, 12), 1) + && + (literal_1 = _PyPegen_expect_token(p, 8)) + ) + { + res = a; + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + { // import_from_as_names + asdl_seq* import_from_as_names_var; + if ( + (import_from_as_names_var = import_from_as_names_rule(p)) + ) + { + res = import_from_as_names_var; + goto done; + } + p->mark = mark; + } + { // '*' + void *literal; + if ( + (literal = _PyPegen_expect_token(p, 16)) + ) + { + res = _PyPegen_singleton_seq ( p , CHECK ( _PyPegen_alias_for_star ( p ) ) ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// import_from_as_names: ','.import_from_as_name+ +static asdl_seq* +import_from_as_names_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + asdl_seq* res = NULL; + int mark = p->mark; + { // ','.import_from_as_name+ + asdl_seq * a; + if ( + (a = _gather_23_rule(p)) + ) + { + res = a; + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// import_from_as_name: NAME ['as' NAME] +static alias_ty +import_from_as_name_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + alias_ty res = NULL; + int mark = p->mark; + { // NAME ['as' NAME] + expr_ty a; + void *b; + if ( + (a = _PyPegen_name_token(p)) + && + (b = _tmp_25_rule(p), 1) + ) + { + res = _Py_alias ( a -> v . Name . id , ( b ) ? ( ( expr_ty ) b ) -> v . Name . id : NULL , p -> arena ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// dotted_as_names: ','.dotted_as_name+ +static asdl_seq* +dotted_as_names_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + asdl_seq* res = NULL; + int mark = p->mark; + { // ','.dotted_as_name+ + asdl_seq * a; + if ( + (a = _gather_26_rule(p)) + ) + { + res = a; + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// dotted_as_name: dotted_name ['as' NAME] +static alias_ty +dotted_as_name_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + alias_ty res = NULL; + int mark = p->mark; + { // dotted_name ['as' NAME] + expr_ty a; + void *b; + if ( + (a = dotted_name_rule(p)) + && + (b = _tmp_28_rule(p), 1) + ) + { + res = _Py_alias ( a -> v . Name . id , ( b ) ? ( ( expr_ty ) b ) -> v . Name . id : NULL , p -> arena ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// Left-recursive +// dotted_name: dotted_name '.' NAME | NAME +static expr_ty dotted_name_raw(Parser *); +static expr_ty +dotted_name_rule(Parser *p) +{ + expr_ty res = NULL; + if (_PyPegen_is_memoized(p, dotted_name_type, &res)) + return res; + int mark = p->mark; + int resmark = p->mark; + while (1) { + int tmpvar_0 = _PyPegen_update_memo(p, mark, dotted_name_type, res); + if (tmpvar_0) { + return res; + } + p->mark = mark; + void *raw = dotted_name_raw(p); + if (raw == NULL || p->mark <= resmark) + break; + resmark = p->mark; + res = raw; + } + p->mark = resmark; + return res; +} +static expr_ty +dotted_name_raw(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + expr_ty res = NULL; + int mark = p->mark; + { // dotted_name '.' NAME + expr_ty a; + expr_ty b; + void *literal; + if ( + (a = dotted_name_rule(p)) + && + (literal = _PyPegen_expect_token(p, 23)) + && + (b = _PyPegen_name_token(p)) + ) + { + res = _PyPegen_join_names_with_dot ( p , a , b ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + { // NAME + expr_ty name_var; + if ( + (name_var = _PyPegen_name_token(p)) + ) + { + res = name_var; + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// if_stmt: +// | 'if' named_expression ':' block elif_stmt +// | 'if' named_expression ':' block else_block? +static stmt_ty +if_stmt_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + stmt_ty res = NULL; + int mark = p->mark; + if (p->mark == p->fill && _PyPegen_fill_token(p) < 0) { + p->error_indicator = 1; + return NULL; + } + int start_lineno = p->tokens[mark]->lineno; + UNUSED(start_lineno); // Only used by EXTRA macro + int start_col_offset = p->tokens[mark]->col_offset; + UNUSED(start_col_offset); // Only used by EXTRA macro + { // 'if' named_expression ':' block elif_stmt + expr_ty a; + asdl_seq* b; + stmt_ty c; + void *keyword; + void *literal; + if ( + (keyword = _PyPegen_expect_token(p, 510)) + && + (a = named_expression_rule(p)) + && + (literal = _PyPegen_expect_token(p, 11)) + && + (b = block_rule(p)) + && + (c = elif_stmt_rule(p)) + ) + { + Token *token = _PyPegen_get_last_nonnwhitespace_token(p); + if (token == NULL) { + return NULL; + } + int end_lineno = token->end_lineno; + UNUSED(end_lineno); // Only used by EXTRA macro + int end_col_offset = token->end_col_offset; + UNUSED(end_col_offset); // Only used by EXTRA macro + res = _Py_If ( a , b , CHECK ( _PyPegen_singleton_seq ( p , c ) ) , EXTRA ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + { // 'if' named_expression ':' block else_block? + expr_ty a; + asdl_seq* b; + void *c; + void *keyword; + void *literal; + if ( + (keyword = _PyPegen_expect_token(p, 510)) + && + (a = named_expression_rule(p)) + && + (literal = _PyPegen_expect_token(p, 11)) + && + (b = block_rule(p)) + && + (c = else_block_rule(p), 1) + ) + { + Token *token = _PyPegen_get_last_nonnwhitespace_token(p); + if (token == NULL) { + return NULL; + } + int end_lineno = token->end_lineno; + UNUSED(end_lineno); // Only used by EXTRA macro + int end_col_offset = token->end_col_offset; + UNUSED(end_col_offset); // Only used by EXTRA macro + res = _Py_If ( a , b , c , EXTRA ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// elif_stmt: +// | 'elif' named_expression ':' block elif_stmt +// | 'elif' named_expression ':' block else_block? +static stmt_ty +elif_stmt_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + stmt_ty res = NULL; + int mark = p->mark; + if (p->mark == p->fill && _PyPegen_fill_token(p) < 0) { + p->error_indicator = 1; + return NULL; + } + int start_lineno = p->tokens[mark]->lineno; + UNUSED(start_lineno); // Only used by EXTRA macro + int start_col_offset = p->tokens[mark]->col_offset; + UNUSED(start_col_offset); // Only used by EXTRA macro + { // 'elif' named_expression ':' block elif_stmt + expr_ty a; + asdl_seq* b; + stmt_ty c; + void *keyword; + void *literal; + if ( + (keyword = _PyPegen_expect_token(p, 515)) + && + (a = named_expression_rule(p)) + && + (literal = _PyPegen_expect_token(p, 11)) + && + (b = block_rule(p)) + && + (c = elif_stmt_rule(p)) + ) + { + Token *token = _PyPegen_get_last_nonnwhitespace_token(p); + if (token == NULL) { + return NULL; + } + int end_lineno = token->end_lineno; + UNUSED(end_lineno); // Only used by EXTRA macro + int end_col_offset = token->end_col_offset; + UNUSED(end_col_offset); // Only used by EXTRA macro + res = _Py_If ( a , b , CHECK ( _PyPegen_singleton_seq ( p , c ) ) , EXTRA ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + { // 'elif' named_expression ':' block else_block? + expr_ty a; + asdl_seq* b; + void *c; + void *keyword; + void *literal; + if ( + (keyword = _PyPegen_expect_token(p, 515)) + && + (a = named_expression_rule(p)) + && + (literal = _PyPegen_expect_token(p, 11)) + && + (b = block_rule(p)) + && + (c = else_block_rule(p), 1) + ) + { + Token *token = _PyPegen_get_last_nonnwhitespace_token(p); + if (token == NULL) { + return NULL; + } + int end_lineno = token->end_lineno; + UNUSED(end_lineno); // Only used by EXTRA macro + int end_col_offset = token->end_col_offset; + UNUSED(end_col_offset); // Only used by EXTRA macro + res = _Py_If ( a , b , c , EXTRA ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// else_block: 'else' ':' block +static asdl_seq* +else_block_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + asdl_seq* res = NULL; + int mark = p->mark; + { // 'else' ':' block + asdl_seq* b; + void *keyword; + void *literal; + if ( + (keyword = _PyPegen_expect_token(p, 516)) + && + (literal = _PyPegen_expect_token(p, 11)) + && + (b = block_rule(p)) + ) + { + res = b; + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// while_stmt: 'while' named_expression ':' block else_block? +static stmt_ty +while_stmt_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + stmt_ty res = NULL; + int mark = p->mark; + if (p->mark == p->fill && _PyPegen_fill_token(p) < 0) { + p->error_indicator = 1; + return NULL; + } + int start_lineno = p->tokens[mark]->lineno; + UNUSED(start_lineno); // Only used by EXTRA macro + int start_col_offset = p->tokens[mark]->col_offset; + UNUSED(start_col_offset); // Only used by EXTRA macro + { // 'while' named_expression ':' block else_block? + expr_ty a; + asdl_seq* b; + void *c; + void *keyword; + void *literal; + if ( + (keyword = _PyPegen_expect_token(p, 512)) + && + (a = named_expression_rule(p)) + && + (literal = _PyPegen_expect_token(p, 11)) + && + (b = block_rule(p)) + && + (c = else_block_rule(p), 1) + ) + { + Token *token = _PyPegen_get_last_nonnwhitespace_token(p); + if (token == NULL) { + return NULL; + } + int end_lineno = token->end_lineno; + UNUSED(end_lineno); // Only used by EXTRA macro + int end_col_offset = token->end_col_offset; + UNUSED(end_col_offset); // Only used by EXTRA macro + res = _Py_While ( a , b , c , EXTRA ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// for_stmt: ASYNC? 'for' star_targets 'in' star_expressions ':' block else_block? +static stmt_ty +for_stmt_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + stmt_ty res = NULL; + int mark = p->mark; + if (p->mark == p->fill && _PyPegen_fill_token(p) < 0) { + p->error_indicator = 1; + return NULL; + } + int start_lineno = p->tokens[mark]->lineno; + UNUSED(start_lineno); // Only used by EXTRA macro + int start_col_offset = p->tokens[mark]->col_offset; + UNUSED(start_col_offset); // Only used by EXTRA macro + { // ASYNC? 'for' star_targets 'in' star_expressions ':' block else_block? + asdl_seq* b; + void *el; + expr_ty ex; + void *is_async; + void *keyword; + void *keyword_1; + void *literal; + expr_ty t; + if ( + (is_async = _PyPegen_async_token(p), 1) + && + (keyword = _PyPegen_expect_token(p, 517)) + && + (t = star_targets_rule(p)) + && + (keyword_1 = _PyPegen_expect_token(p, 518)) + && + (ex = star_expressions_rule(p)) + && + (literal = _PyPegen_expect_token(p, 11)) + && + (b = block_rule(p)) + && + (el = else_block_rule(p), 1) + ) + { + Token *token = _PyPegen_get_last_nonnwhitespace_token(p); + if (token == NULL) { + return NULL; + } + int end_lineno = token->end_lineno; + UNUSED(end_lineno); // Only used by EXTRA macro + int end_col_offset = token->end_col_offset; + UNUSED(end_col_offset); // Only used by EXTRA macro + res = ( is_async ? _Py_AsyncFor : _Py_For ) ( t , ex , b , el , NULL , EXTRA ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// with_stmt: +// | ASYNC? 'with' '(' ','.with_item+ ')' ':' block +// | ASYNC? 'with' ','.with_item+ ':' block +static stmt_ty +with_stmt_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + stmt_ty res = NULL; + int mark = p->mark; + if (p->mark == p->fill && _PyPegen_fill_token(p) < 0) { + p->error_indicator = 1; + return NULL; + } + int start_lineno = p->tokens[mark]->lineno; + UNUSED(start_lineno); // Only used by EXTRA macro + int start_col_offset = p->tokens[mark]->col_offset; + UNUSED(start_col_offset); // Only used by EXTRA macro + { // ASYNC? 'with' '(' ','.with_item+ ')' ':' block + asdl_seq * a; + asdl_seq* b; + void *is_async; + void *keyword; + void *literal; + void *literal_1; + void *literal_2; + if ( + (is_async = _PyPegen_async_token(p), 1) + && + (keyword = _PyPegen_expect_token(p, 519)) + && + (literal = _PyPegen_expect_token(p, 7)) + && + (a = _gather_29_rule(p)) + && + (literal_1 = _PyPegen_expect_token(p, 8)) + && + (literal_2 = _PyPegen_expect_token(p, 11)) + && + (b = block_rule(p)) + ) + { + Token *token = _PyPegen_get_last_nonnwhitespace_token(p); + if (token == NULL) { + return NULL; + } + int end_lineno = token->end_lineno; + UNUSED(end_lineno); // Only used by EXTRA macro + int end_col_offset = token->end_col_offset; + UNUSED(end_col_offset); // Only used by EXTRA macro + res = ( is_async ? _Py_AsyncWith : _Py_With ) ( a , b , NULL , EXTRA ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + { // ASYNC? 'with' ','.with_item+ ':' block + asdl_seq * a; + asdl_seq* b; + void *is_async; + void *keyword; + void *literal; + if ( + (is_async = _PyPegen_async_token(p), 1) + && + (keyword = _PyPegen_expect_token(p, 519)) + && + (a = _gather_31_rule(p)) + && + (literal = _PyPegen_expect_token(p, 11)) + && + (b = block_rule(p)) + ) + { + Token *token = _PyPegen_get_last_nonnwhitespace_token(p); + if (token == NULL) { + return NULL; + } + int end_lineno = token->end_lineno; + UNUSED(end_lineno); // Only used by EXTRA macro + int end_col_offset = token->end_col_offset; + UNUSED(end_col_offset); // Only used by EXTRA macro + res = ( is_async ? _Py_AsyncWith : _Py_With ) ( a , b , NULL , EXTRA ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// with_item: expression ['as' target] +static withitem_ty +with_item_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + withitem_ty res = NULL; + int mark = p->mark; + { // expression ['as' target] + expr_ty e; + void *o; + if ( + (e = expression_rule(p)) + && + (o = _tmp_33_rule(p), 1) + ) + { + res = _Py_withitem ( e , o , p -> arena ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// try_stmt: +// | 'try' ':' block finally_block +// | 'try' ':' block except_block+ else_block? finally_block? +static stmt_ty +try_stmt_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + stmt_ty res = NULL; + int mark = p->mark; + if (p->mark == p->fill && _PyPegen_fill_token(p) < 0) { + p->error_indicator = 1; + return NULL; + } + int start_lineno = p->tokens[mark]->lineno; + UNUSED(start_lineno); // Only used by EXTRA macro + int start_col_offset = p->tokens[mark]->col_offset; + UNUSED(start_col_offset); // Only used by EXTRA macro + { // 'try' ':' block finally_block + asdl_seq* b; + asdl_seq* f; + void *keyword; + void *literal; + if ( + (keyword = _PyPegen_expect_token(p, 511)) + && + (literal = _PyPegen_expect_token(p, 11)) + && + (b = block_rule(p)) + && + (f = finally_block_rule(p)) + ) + { + Token *token = _PyPegen_get_last_nonnwhitespace_token(p); + if (token == NULL) { + return NULL; + } + int end_lineno = token->end_lineno; + UNUSED(end_lineno); // Only used by EXTRA macro + int end_col_offset = token->end_col_offset; + UNUSED(end_col_offset); // Only used by EXTRA macro + res = _Py_Try ( b , NULL , NULL , f , EXTRA ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + { // 'try' ':' block except_block+ else_block? finally_block? + asdl_seq* b; + void *el; + asdl_seq * ex; + void *f; + void *keyword; + void *literal; + if ( + (keyword = _PyPegen_expect_token(p, 511)) + && + (literal = _PyPegen_expect_token(p, 11)) + && + (b = block_rule(p)) + && + (ex = _loop1_34_rule(p)) + && + (el = else_block_rule(p), 1) + && + (f = finally_block_rule(p), 1) + ) + { + Token *token = _PyPegen_get_last_nonnwhitespace_token(p); + if (token == NULL) { + return NULL; + } + int end_lineno = token->end_lineno; + UNUSED(end_lineno); // Only used by EXTRA macro + int end_col_offset = token->end_col_offset; + UNUSED(end_col_offset); // Only used by EXTRA macro + res = _Py_Try ( b , ex , el , f , EXTRA ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// except_block: 'except' expression ['as' target] ':' block | 'except' ':' block +static excepthandler_ty +except_block_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + excepthandler_ty res = NULL; + int mark = p->mark; + if (p->mark == p->fill && _PyPegen_fill_token(p) < 0) { + p->error_indicator = 1; + return NULL; + } + int start_lineno = p->tokens[mark]->lineno; + UNUSED(start_lineno); // Only used by EXTRA macro + int start_col_offset = p->tokens[mark]->col_offset; + UNUSED(start_col_offset); // Only used by EXTRA macro + { // 'except' expression ['as' target] ':' block + asdl_seq* b; + expr_ty e; + void *keyword; + void *literal; + void *t; + if ( + (keyword = _PyPegen_expect_token(p, 520)) + && + (e = expression_rule(p)) + && + (t = _tmp_35_rule(p), 1) + && + (literal = _PyPegen_expect_token(p, 11)) + && + (b = block_rule(p)) + ) + { + Token *token = _PyPegen_get_last_nonnwhitespace_token(p); + if (token == NULL) { + return NULL; + } + int end_lineno = token->end_lineno; + UNUSED(end_lineno); // Only used by EXTRA macro + int end_col_offset = token->end_col_offset; + UNUSED(end_col_offset); // Only used by EXTRA macro + res = _Py_ExceptHandler ( e , ( t ) ? ( ( expr_ty ) t ) -> v . Name . id : NULL , b , EXTRA ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + { // 'except' ':' block + asdl_seq* b; + void *keyword; + void *literal; + if ( + (keyword = _PyPegen_expect_token(p, 520)) + && + (literal = _PyPegen_expect_token(p, 11)) + && + (b = block_rule(p)) + ) + { + Token *token = _PyPegen_get_last_nonnwhitespace_token(p); + if (token == NULL) { + return NULL; + } + int end_lineno = token->end_lineno; + UNUSED(end_lineno); // Only used by EXTRA macro + int end_col_offset = token->end_col_offset; + UNUSED(end_col_offset); // Only used by EXTRA macro + res = _Py_ExceptHandler ( NULL , NULL , b , EXTRA ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// finally_block: 'finally' ':' block +static asdl_seq* +finally_block_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + asdl_seq* res = NULL; + int mark = p->mark; + { // 'finally' ':' block + asdl_seq* a; + void *keyword; + void *literal; + if ( + (keyword = _PyPegen_expect_token(p, 521)) + && + (literal = _PyPegen_expect_token(p, 11)) + && + (a = block_rule(p)) + ) + { + res = a; + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// return_stmt: 'return' star_expressions? +static stmt_ty +return_stmt_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + stmt_ty res = NULL; + int mark = p->mark; + if (p->mark == p->fill && _PyPegen_fill_token(p) < 0) { + p->error_indicator = 1; + return NULL; + } + int start_lineno = p->tokens[mark]->lineno; + UNUSED(start_lineno); // Only used by EXTRA macro + int start_col_offset = p->tokens[mark]->col_offset; + UNUSED(start_col_offset); // Only used by EXTRA macro + { // 'return' star_expressions? + void *a; + void *keyword; + if ( + (keyword = _PyPegen_expect_token(p, 500)) + && + (a = star_expressions_rule(p), 1) + ) + { + Token *token = _PyPegen_get_last_nonnwhitespace_token(p); + if (token == NULL) { + return NULL; + } + int end_lineno = token->end_lineno; + UNUSED(end_lineno); // Only used by EXTRA macro + int end_col_offset = token->end_col_offset; + UNUSED(end_col_offset); // Only used by EXTRA macro + res = _Py_Return ( a , EXTRA ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// raise_stmt: 'raise' expression ['from' expression] | 'raise' +static stmt_ty +raise_stmt_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + stmt_ty res = NULL; + int mark = p->mark; + if (p->mark == p->fill && _PyPegen_fill_token(p) < 0) { + p->error_indicator = 1; + return NULL; + } + int start_lineno = p->tokens[mark]->lineno; + UNUSED(start_lineno); // Only used by EXTRA macro + int start_col_offset = p->tokens[mark]->col_offset; + UNUSED(start_col_offset); // Only used by EXTRA macro + { // 'raise' expression ['from' expression] + expr_ty a; + void *b; + void *keyword; + if ( + (keyword = _PyPegen_expect_token(p, 501)) + && + (a = expression_rule(p)) + && + (b = _tmp_36_rule(p), 1) + ) + { + Token *token = _PyPegen_get_last_nonnwhitespace_token(p); + if (token == NULL) { + return NULL; + } + int end_lineno = token->end_lineno; + UNUSED(end_lineno); // Only used by EXTRA macro + int end_col_offset = token->end_col_offset; + UNUSED(end_col_offset); // Only used by EXTRA macro + res = _Py_Raise ( a , b , EXTRA ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + { // 'raise' + void *keyword; + if ( + (keyword = _PyPegen_expect_token(p, 501)) + ) + { + Token *token = _PyPegen_get_last_nonnwhitespace_token(p); + if (token == NULL) { + return NULL; + } + int end_lineno = token->end_lineno; + UNUSED(end_lineno); // Only used by EXTRA macro + int end_col_offset = token->end_col_offset; + UNUSED(end_col_offset); // Only used by EXTRA macro + res = _Py_Raise ( NULL , NULL , EXTRA ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// function_def: decorators function_def_raw | function_def_raw +static stmt_ty +function_def_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + stmt_ty res = NULL; + int mark = p->mark; + { // decorators function_def_raw + asdl_seq* d; + stmt_ty f; + if ( + (d = decorators_rule(p)) + && + (f = function_def_raw_rule(p)) + ) + { + res = _PyPegen_function_def_decorators ( p , d , f ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + { // function_def_raw + stmt_ty function_def_raw_var; + if ( + (function_def_raw_var = function_def_raw_rule(p)) + ) + { + res = function_def_raw_var; + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// function_def_raw: ASYNC? 'def' NAME '(' params? ')' ['->' annotation] ':' block +static stmt_ty +function_def_raw_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + stmt_ty res = NULL; + int mark = p->mark; + if (p->mark == p->fill && _PyPegen_fill_token(p) < 0) { + p->error_indicator = 1; + return NULL; + } + int start_lineno = p->tokens[mark]->lineno; + UNUSED(start_lineno); // Only used by EXTRA macro + int start_col_offset = p->tokens[mark]->col_offset; + UNUSED(start_col_offset); // Only used by EXTRA macro + { // ASYNC? 'def' NAME '(' params? ')' ['->' annotation] ':' block + void *a; + asdl_seq* b; + void *is_async; + void *keyword; + void *literal; + void *literal_1; + void *literal_2; + expr_ty n; + void *params; + if ( + (is_async = _PyPegen_async_token(p), 1) + && + (keyword = _PyPegen_expect_token(p, 522)) + && + (n = _PyPegen_name_token(p)) + && + (literal = _PyPegen_expect_token(p, 7)) + && + (params = params_rule(p), 1) + && + (literal_1 = _PyPegen_expect_token(p, 8)) + && + (a = _tmp_37_rule(p), 1) + && + (literal_2 = _PyPegen_expect_token(p, 11)) + && + (b = block_rule(p)) + ) + { + Token *token = _PyPegen_get_last_nonnwhitespace_token(p); + if (token == NULL) { + return NULL; + } + int end_lineno = token->end_lineno; + UNUSED(end_lineno); // Only used by EXTRA macro + int end_col_offset = token->end_col_offset; + UNUSED(end_col_offset); // Only used by EXTRA macro + res = ( is_async ? _Py_AsyncFunctionDef : _Py_FunctionDef ) ( n -> v . Name . id , ( params ) ? params : CHECK ( _PyPegen_empty_arguments ( p ) ) , b , NULL , a , NULL , EXTRA ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// params: invalid_parameters | parameters +static arguments_ty +params_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + arguments_ty res = NULL; + int mark = p->mark; + { // invalid_parameters + void *invalid_parameters_var; + if ( + (invalid_parameters_var = invalid_parameters_rule(p)) + ) + { + res = invalid_parameters_var; + goto done; + } + p->mark = mark; + } + { // parameters + arguments_ty parameters_var; + if ( + (parameters_var = parameters_rule(p)) + ) + { + res = parameters_var; + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// parameters: +// | slash_without_default [',' plain_names] [',' names_with_default] [',' star_etc?] +// | slash_with_default [',' names_with_default] [',' star_etc?] +// | plain_names [',' names_with_default] [',' star_etc?] +// | names_with_default [',' star_etc?] +// | star_etc +static arguments_ty +parameters_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + arguments_ty res = NULL; + int mark = p->mark; + { // slash_without_default [',' plain_names] [',' names_with_default] [',' star_etc?] + asdl_seq* a; + void *b; + void *c; + void *d; + if ( + (a = slash_without_default_rule(p)) + && + (b = _tmp_38_rule(p), 1) + && + (c = _tmp_39_rule(p), 1) + && + (d = _tmp_40_rule(p), 1) + ) + { + res = _PyPegen_make_arguments ( p , a , NULL , b , c , d ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + { // slash_with_default [',' names_with_default] [',' star_etc?] + SlashWithDefault* a; + void *b; + void *c; + if ( + (a = slash_with_default_rule(p)) + && + (b = _tmp_41_rule(p), 1) + && + (c = _tmp_42_rule(p), 1) + ) + { + res = _PyPegen_make_arguments ( p , NULL , a , NULL , b , c ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + { // plain_names [',' names_with_default] [',' star_etc?] + asdl_seq* a; + void *b; + void *c; + if ( + (a = plain_names_rule(p)) + && + (b = _tmp_43_rule(p), 1) + && + (c = _tmp_44_rule(p), 1) + ) + { + res = _PyPegen_make_arguments ( p , NULL , NULL , a , b , c ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + { // names_with_default [',' star_etc?] + asdl_seq* a; + void *b; + if ( + (a = names_with_default_rule(p)) + && + (b = _tmp_45_rule(p), 1) + ) + { + res = _PyPegen_make_arguments ( p , NULL , NULL , NULL , a , b ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + { // star_etc + StarEtc* a; + if ( + (a = star_etc_rule(p)) + ) + { + res = _PyPegen_make_arguments ( p , NULL , NULL , NULL , NULL , a ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// slash_without_default: plain_names ',' '/' +static asdl_seq* +slash_without_default_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + asdl_seq* res = NULL; + int mark = p->mark; + { // plain_names ',' '/' + asdl_seq* a; + void *literal; + void *literal_1; + if ( + (a = plain_names_rule(p)) + && + (literal = _PyPegen_expect_token(p, 12)) + && + (literal_1 = _PyPegen_expect_token(p, 17)) + ) + { + res = a; + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// slash_with_default: [plain_names ','] names_with_default ',' '/' +static SlashWithDefault* +slash_with_default_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + SlashWithDefault* res = NULL; + int mark = p->mark; + { // [plain_names ','] names_with_default ',' '/' + void *a; + asdl_seq* b; + void *literal; + void *literal_1; + if ( + (a = _tmp_46_rule(p), 1) + && + (b = names_with_default_rule(p)) + && + (literal = _PyPegen_expect_token(p, 12)) + && + (literal_1 = _PyPegen_expect_token(p, 17)) + ) + { + res = _PyPegen_slash_with_default ( p , a , b ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// star_etc: +// | '*' plain_name name_with_optional_default* [',' kwds] ','? +// | '*' name_with_optional_default+ [',' kwds] ','? +// | kwds ','? +static StarEtc* +star_etc_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + StarEtc* res = NULL; + int mark = p->mark; + { // '*' plain_name name_with_optional_default* [',' kwds] ','? + arg_ty a; + asdl_seq * b; + void *c; + void *literal; + void *opt_var; + UNUSED(opt_var); // Silence compiler warnings + if ( + (literal = _PyPegen_expect_token(p, 16)) + && + (a = plain_name_rule(p)) + && + (b = _loop0_47_rule(p)) + && + (c = _tmp_48_rule(p), 1) + && + (opt_var = _PyPegen_expect_token(p, 12), 1) + ) + { + res = _PyPegen_star_etc ( p , a , b , c ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + { // '*' name_with_optional_default+ [',' kwds] ','? + asdl_seq * b; + void *c; + void *literal; + void *opt_var; + UNUSED(opt_var); // Silence compiler warnings + if ( + (literal = _PyPegen_expect_token(p, 16)) + && + (b = _loop1_49_rule(p)) + && + (c = _tmp_50_rule(p), 1) + && + (opt_var = _PyPegen_expect_token(p, 12), 1) + ) + { + res = _PyPegen_star_etc ( p , NULL , b , c ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + { // kwds ','? + arg_ty a; + void *opt_var; + UNUSED(opt_var); // Silence compiler warnings + if ( + (a = kwds_rule(p)) + && + (opt_var = _PyPegen_expect_token(p, 12), 1) + ) + { + res = _PyPegen_star_etc ( p , NULL , NULL , a ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// name_with_optional_default: ',' plain_name ['=' expression] +static NameDefaultPair* +name_with_optional_default_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + NameDefaultPair* res = NULL; + int mark = p->mark; + { // ',' plain_name ['=' expression] + arg_ty a; + void *b; + void *literal; + if ( + (literal = _PyPegen_expect_token(p, 12)) + && + (a = plain_name_rule(p)) + && + (b = _tmp_51_rule(p), 1) + ) + { + res = _PyPegen_name_default_pair ( p , a , b ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// names_with_default: ','.name_with_default+ +static asdl_seq* +names_with_default_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + asdl_seq* res = NULL; + int mark = p->mark; + { // ','.name_with_default+ + asdl_seq * a; + if ( + (a = _gather_52_rule(p)) + ) + { + res = a; + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// name_with_default: plain_name '=' expression +static NameDefaultPair* +name_with_default_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + NameDefaultPair* res = NULL; + int mark = p->mark; + { // plain_name '=' expression + expr_ty e; + void *literal; + arg_ty n; + if ( + (n = plain_name_rule(p)) + && + (literal = _PyPegen_expect_token(p, 22)) + && + (e = expression_rule(p)) + ) + { + res = _PyPegen_name_default_pair ( p , n , e ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// plain_names: ','.(plain_name !'=')+ +static asdl_seq* +plain_names_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + asdl_seq* res = NULL; + if (_PyPegen_is_memoized(p, plain_names_type, &res)) + return res; + int mark = p->mark; + { // ','.(plain_name !'=')+ + asdl_seq * a; + if ( + (a = _gather_54_rule(p)) + ) + { + res = a; + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + res = NULL; + done: + _PyPegen_insert_memo(p, mark, plain_names_type, res); + return res; +} + +// plain_name: NAME [':' annotation] +static arg_ty +plain_name_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + arg_ty res = NULL; + int mark = p->mark; + if (p->mark == p->fill && _PyPegen_fill_token(p) < 0) { + p->error_indicator = 1; + return NULL; + } + int start_lineno = p->tokens[mark]->lineno; + UNUSED(start_lineno); // Only used by EXTRA macro + int start_col_offset = p->tokens[mark]->col_offset; + UNUSED(start_col_offset); // Only used by EXTRA macro + { // NAME [':' annotation] + expr_ty a; + void *b; + if ( + (a = _PyPegen_name_token(p)) + && + (b = _tmp_56_rule(p), 1) + ) + { + Token *token = _PyPegen_get_last_nonnwhitespace_token(p); + if (token == NULL) { + return NULL; + } + int end_lineno = token->end_lineno; + UNUSED(end_lineno); // Only used by EXTRA macro + int end_col_offset = token->end_col_offset; + UNUSED(end_col_offset); // Only used by EXTRA macro + res = _Py_arg ( a -> v . Name . id , b , NULL , EXTRA ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// kwds: '**' plain_name +static arg_ty +kwds_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + arg_ty res = NULL; + int mark = p->mark; + { // '**' plain_name + arg_ty a; + void *literal; + if ( + (literal = _PyPegen_expect_token(p, 35)) + && + (a = plain_name_rule(p)) + ) + { + res = a; + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// annotation: expression +static expr_ty +annotation_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + expr_ty res = NULL; + int mark = p->mark; + { // expression + expr_ty expression_var; + if ( + (expression_var = expression_rule(p)) + ) + { + res = expression_var; + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// decorators: (('@' named_expression NEWLINE))+ +static asdl_seq* +decorators_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + asdl_seq* res = NULL; + int mark = p->mark; + { // (('@' named_expression NEWLINE))+ + asdl_seq * a; + if ( + (a = _loop1_57_rule(p)) + ) + { + res = a; + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// class_def: decorators class_def_raw | class_def_raw +static stmt_ty +class_def_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + stmt_ty res = NULL; + int mark = p->mark; + { // decorators class_def_raw + asdl_seq* a; + stmt_ty b; + if ( + (a = decorators_rule(p)) + && + (b = class_def_raw_rule(p)) + ) + { + res = _PyPegen_class_def_decorators ( p , a , b ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + { // class_def_raw + stmt_ty class_def_raw_var; + if ( + (class_def_raw_var = class_def_raw_rule(p)) + ) + { + res = class_def_raw_var; + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// class_def_raw: 'class' NAME ['(' arguments? ')'] ':' block +static stmt_ty +class_def_raw_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + stmt_ty res = NULL; + int mark = p->mark; + if (p->mark == p->fill && _PyPegen_fill_token(p) < 0) { + p->error_indicator = 1; + return NULL; + } + int start_lineno = p->tokens[mark]->lineno; + UNUSED(start_lineno); // Only used by EXTRA macro + int start_col_offset = p->tokens[mark]->col_offset; + UNUSED(start_col_offset); // Only used by EXTRA macro + { // 'class' NAME ['(' arguments? ')'] ':' block + expr_ty a; + void *b; + asdl_seq* c; + void *keyword; + void *literal; + if ( + (keyword = _PyPegen_expect_token(p, 523)) + && + (a = _PyPegen_name_token(p)) + && + (b = _tmp_58_rule(p), 1) + && + (literal = _PyPegen_expect_token(p, 11)) + && + (c = block_rule(p)) + ) + { + Token *token = _PyPegen_get_last_nonnwhitespace_token(p); + if (token == NULL) { + return NULL; + } + int end_lineno = token->end_lineno; + UNUSED(end_lineno); // Only used by EXTRA macro + int end_col_offset = token->end_col_offset; + UNUSED(end_col_offset); // Only used by EXTRA macro + res = _Py_ClassDef ( a -> v . Name . id , ( b ) ? ( ( expr_ty ) b ) -> v . Call . args : NULL , ( b ) ? ( ( expr_ty ) b ) -> v . Call . keywords : NULL , c , NULL , EXTRA ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// block: NEWLINE INDENT statements DEDENT | simple_stmt | invalid_block +static asdl_seq* +block_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + asdl_seq* res = NULL; + if (_PyPegen_is_memoized(p, block_type, &res)) + return res; + int mark = p->mark; + { // NEWLINE INDENT statements DEDENT + asdl_seq* a; + void *dedent_var; + void *indent_var; + void *newline_var; + if ( + (newline_var = _PyPegen_newline_token(p)) + && + (indent_var = _PyPegen_indent_token(p)) + && + (a = statements_rule(p)) + && + (dedent_var = _PyPegen_dedent_token(p)) + ) + { + res = a; + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + { // simple_stmt + asdl_seq* simple_stmt_var; + if ( + (simple_stmt_var = simple_stmt_rule(p)) + ) + { + res = simple_stmt_var; + goto done; + } + p->mark = mark; + } + { // invalid_block + void *invalid_block_var; + if ( + (invalid_block_var = invalid_block_rule(p)) + ) + { + res = invalid_block_var; + goto done; + } + p->mark = mark; + } + res = NULL; + done: + _PyPegen_insert_memo(p, mark, block_type, res); + return res; +} + +// expressions_list: ','.star_expression+ ','? +static asdl_seq* +expressions_list_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + asdl_seq* res = NULL; + int mark = p->mark; + { // ','.star_expression+ ','? + asdl_seq * a; + void *opt_var; + UNUSED(opt_var); // Silence compiler warnings + if ( + (a = _gather_59_rule(p)) + && + (opt_var = _PyPegen_expect_token(p, 12), 1) + ) + { + res = a; + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// star_expressions: +// | star_expression ((',' star_expression))+ ','? +// | star_expression ',' +// | star_expression +static expr_ty +star_expressions_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + expr_ty res = NULL; + int mark = p->mark; + if (p->mark == p->fill && _PyPegen_fill_token(p) < 0) { + p->error_indicator = 1; + return NULL; + } + int start_lineno = p->tokens[mark]->lineno; + UNUSED(start_lineno); // Only used by EXTRA macro + int start_col_offset = p->tokens[mark]->col_offset; + UNUSED(start_col_offset); // Only used by EXTRA macro + { // star_expression ((',' star_expression))+ ','? + expr_ty a; + asdl_seq * b; + void *opt_var; + UNUSED(opt_var); // Silence compiler warnings + if ( + (a = star_expression_rule(p)) + && + (b = _loop1_61_rule(p)) + && + (opt_var = _PyPegen_expect_token(p, 12), 1) + ) + { + Token *token = _PyPegen_get_last_nonnwhitespace_token(p); + if (token == NULL) { + return NULL; + } + int end_lineno = token->end_lineno; + UNUSED(end_lineno); // Only used by EXTRA macro + int end_col_offset = token->end_col_offset; + UNUSED(end_col_offset); // Only used by EXTRA macro + res = _Py_Tuple ( CHECK ( _PyPegen_seq_insert_in_front ( p , a , b ) ) , Load , EXTRA ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + { // star_expression ',' + expr_ty a; + void *literal; + if ( + (a = star_expression_rule(p)) + && + (literal = _PyPegen_expect_token(p, 12)) + ) + { + Token *token = _PyPegen_get_last_nonnwhitespace_token(p); + if (token == NULL) { + return NULL; + } + int end_lineno = token->end_lineno; + UNUSED(end_lineno); // Only used by EXTRA macro + int end_col_offset = token->end_col_offset; + UNUSED(end_col_offset); // Only used by EXTRA macro + res = _Py_Tuple ( CHECK ( _PyPegen_singleton_seq ( p , a ) ) , Load , EXTRA ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + { // star_expression + expr_ty star_expression_var; + if ( + (star_expression_var = star_expression_rule(p)) + ) + { + res = star_expression_var; + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// star_expression: '*' bitwise_or | expression +static expr_ty +star_expression_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + expr_ty res = NULL; + if (_PyPegen_is_memoized(p, star_expression_type, &res)) + return res; + int mark = p->mark; + if (p->mark == p->fill && _PyPegen_fill_token(p) < 0) { + p->error_indicator = 1; + return NULL; + } + int start_lineno = p->tokens[mark]->lineno; + UNUSED(start_lineno); // Only used by EXTRA macro + int start_col_offset = p->tokens[mark]->col_offset; + UNUSED(start_col_offset); // Only used by EXTRA macro + { // '*' bitwise_or + expr_ty a; + void *literal; + if ( + (literal = _PyPegen_expect_token(p, 16)) + && + (a = bitwise_or_rule(p)) + ) + { + Token *token = _PyPegen_get_last_nonnwhitespace_token(p); + if (token == NULL) { + return NULL; + } + int end_lineno = token->end_lineno; + UNUSED(end_lineno); // Only used by EXTRA macro + int end_col_offset = token->end_col_offset; + UNUSED(end_col_offset); // Only used by EXTRA macro + res = _Py_Starred ( a , Load , EXTRA ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + { // expression + expr_ty expression_var; + if ( + (expression_var = expression_rule(p)) + ) + { + res = expression_var; + goto done; + } + p->mark = mark; + } + res = NULL; + done: + _PyPegen_insert_memo(p, mark, star_expression_type, res); + return res; +} + +// star_named_expressions: ','.star_named_expression+ ','? +static asdl_seq* +star_named_expressions_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + asdl_seq* res = NULL; + int mark = p->mark; + { // ','.star_named_expression+ ','? + asdl_seq * a; + void *opt_var; + UNUSED(opt_var); // Silence compiler warnings + if ( + (a = _gather_62_rule(p)) + && + (opt_var = _PyPegen_expect_token(p, 12), 1) + ) + { + res = a; + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// star_named_expression: '*' bitwise_or | named_expression +static expr_ty +star_named_expression_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + expr_ty res = NULL; + int mark = p->mark; + if (p->mark == p->fill && _PyPegen_fill_token(p) < 0) { + p->error_indicator = 1; + return NULL; + } + int start_lineno = p->tokens[mark]->lineno; + UNUSED(start_lineno); // Only used by EXTRA macro + int start_col_offset = p->tokens[mark]->col_offset; + UNUSED(start_col_offset); // Only used by EXTRA macro + { // '*' bitwise_or + expr_ty a; + void *literal; + if ( + (literal = _PyPegen_expect_token(p, 16)) + && + (a = bitwise_or_rule(p)) + ) + { + Token *token = _PyPegen_get_last_nonnwhitespace_token(p); + if (token == NULL) { + return NULL; + } + int end_lineno = token->end_lineno; + UNUSED(end_lineno); // Only used by EXTRA macro + int end_col_offset = token->end_col_offset; + UNUSED(end_col_offset); // Only used by EXTRA macro + res = _Py_Starred ( a , Load , EXTRA ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + { // named_expression + expr_ty named_expression_var; + if ( + (named_expression_var = named_expression_rule(p)) + ) + { + res = named_expression_var; + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// named_expression: NAME ':=' expression | expression !':=' | invalid_named_expression +static expr_ty +named_expression_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + expr_ty res = NULL; + int mark = p->mark; + if (p->mark == p->fill && _PyPegen_fill_token(p) < 0) { + p->error_indicator = 1; + return NULL; + } + int start_lineno = p->tokens[mark]->lineno; + UNUSED(start_lineno); // Only used by EXTRA macro + int start_col_offset = p->tokens[mark]->col_offset; + UNUSED(start_col_offset); // Only used by EXTRA macro + { // NAME ':=' expression + expr_ty a; + expr_ty b; + void *literal; + if ( + (a = _PyPegen_name_token(p)) + && + (literal = _PyPegen_expect_token(p, 53)) + && + (b = expression_rule(p)) + ) + { + Token *token = _PyPegen_get_last_nonnwhitespace_token(p); + if (token == NULL) { + return NULL; + } + int end_lineno = token->end_lineno; + UNUSED(end_lineno); // Only used by EXTRA macro + int end_col_offset = token->end_col_offset; + UNUSED(end_col_offset); // Only used by EXTRA macro + res = _Py_NamedExpr ( CHECK ( _PyPegen_set_expr_context ( p , a , Store ) ) , b , EXTRA ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + { // expression !':=' + expr_ty expression_var; + if ( + (expression_var = expression_rule(p)) + && + _PyPegen_lookahead_with_int(0, _PyPegen_expect_token, p, 53) + ) + { + res = expression_var; + goto done; + } + p->mark = mark; + } + { // invalid_named_expression + void *invalid_named_expression_var; + if ( + (invalid_named_expression_var = invalid_named_expression_rule(p)) + ) + { + res = invalid_named_expression_var; + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// annotated_rhs: yield_expr | star_expressions +static expr_ty +annotated_rhs_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + expr_ty res = NULL; + int mark = p->mark; + { // yield_expr + expr_ty yield_expr_var; + if ( + (yield_expr_var = yield_expr_rule(p)) + ) + { + res = yield_expr_var; + goto done; + } + p->mark = mark; + } + { // star_expressions + expr_ty star_expressions_var; + if ( + (star_expressions_var = star_expressions_rule(p)) + ) + { + res = star_expressions_var; + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// expressions: expression ((',' expression))+ ','? | expression ',' | expression +static expr_ty +expressions_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + expr_ty res = NULL; + int mark = p->mark; + if (p->mark == p->fill && _PyPegen_fill_token(p) < 0) { + p->error_indicator = 1; + return NULL; + } + int start_lineno = p->tokens[mark]->lineno; + UNUSED(start_lineno); // Only used by EXTRA macro + int start_col_offset = p->tokens[mark]->col_offset; + UNUSED(start_col_offset); // Only used by EXTRA macro + { // expression ((',' expression))+ ','? + expr_ty a; + asdl_seq * b; + void *opt_var; + UNUSED(opt_var); // Silence compiler warnings + if ( + (a = expression_rule(p)) + && + (b = _loop1_64_rule(p)) + && + (opt_var = _PyPegen_expect_token(p, 12), 1) + ) + { + Token *token = _PyPegen_get_last_nonnwhitespace_token(p); + if (token == NULL) { + return NULL; + } + int end_lineno = token->end_lineno; + UNUSED(end_lineno); // Only used by EXTRA macro + int end_col_offset = token->end_col_offset; + UNUSED(end_col_offset); // Only used by EXTRA macro + res = _Py_Tuple ( CHECK ( _PyPegen_seq_insert_in_front ( p , a , b ) ) , Load , EXTRA ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + { // expression ',' + expr_ty a; + void *literal; + if ( + (a = expression_rule(p)) + && + (literal = _PyPegen_expect_token(p, 12)) + ) + { + Token *token = _PyPegen_get_last_nonnwhitespace_token(p); + if (token == NULL) { + return NULL; + } + int end_lineno = token->end_lineno; + UNUSED(end_lineno); // Only used by EXTRA macro + int end_col_offset = token->end_col_offset; + UNUSED(end_col_offset); // Only used by EXTRA macro + res = _Py_Tuple ( CHECK ( _PyPegen_singleton_seq ( p , a ) ) , Load , EXTRA ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + { // expression + expr_ty expression_var; + if ( + (expression_var = expression_rule(p)) + ) + { + res = expression_var; + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// expression: disjunction 'if' disjunction 'else' expression | disjunction | lambdef +static expr_ty +expression_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + expr_ty res = NULL; + if (_PyPegen_is_memoized(p, expression_type, &res)) + return res; + int mark = p->mark; + if (p->mark == p->fill && _PyPegen_fill_token(p) < 0) { + p->error_indicator = 1; + return NULL; + } + int start_lineno = p->tokens[mark]->lineno; + UNUSED(start_lineno); // Only used by EXTRA macro + int start_col_offset = p->tokens[mark]->col_offset; + UNUSED(start_col_offset); // Only used by EXTRA macro + { // disjunction 'if' disjunction 'else' expression + expr_ty a; + expr_ty b; + expr_ty c; + void *keyword; + void *keyword_1; + if ( + (a = disjunction_rule(p)) + && + (keyword = _PyPegen_expect_token(p, 510)) + && + (b = disjunction_rule(p)) + && + (keyword_1 = _PyPegen_expect_token(p, 516)) + && + (c = expression_rule(p)) + ) + { + Token *token = _PyPegen_get_last_nonnwhitespace_token(p); + if (token == NULL) { + return NULL; + } + int end_lineno = token->end_lineno; + UNUSED(end_lineno); // Only used by EXTRA macro + int end_col_offset = token->end_col_offset; + UNUSED(end_col_offset); // Only used by EXTRA macro + res = _Py_IfExp ( b , a , c , EXTRA ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + { // disjunction + expr_ty disjunction_var; + if ( + (disjunction_var = disjunction_rule(p)) + ) + { + res = disjunction_var; + goto done; + } + p->mark = mark; + } + { // lambdef + expr_ty lambdef_var; + if ( + (lambdef_var = lambdef_rule(p)) + ) + { + res = lambdef_var; + goto done; + } + p->mark = mark; + } + res = NULL; + done: + _PyPegen_insert_memo(p, mark, expression_type, res); + return res; +} + +// lambdef: 'lambda' lambda_parameters? ':' expression +static expr_ty +lambdef_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + expr_ty res = NULL; + int mark = p->mark; + if (p->mark == p->fill && _PyPegen_fill_token(p) < 0) { + p->error_indicator = 1; + return NULL; + } + int start_lineno = p->tokens[mark]->lineno; + UNUSED(start_lineno); // Only used by EXTRA macro + int start_col_offset = p->tokens[mark]->col_offset; + UNUSED(start_col_offset); // Only used by EXTRA macro + { // 'lambda' lambda_parameters? ':' expression + void *a; + expr_ty b; + void *keyword; + void *literal; + if ( + (keyword = _PyPegen_expect_token(p, 524)) + && + (a = lambda_parameters_rule(p), 1) + && + (literal = _PyPegen_expect_token(p, 11)) + && + (b = expression_rule(p)) + ) + { + Token *token = _PyPegen_get_last_nonnwhitespace_token(p); + if (token == NULL) { + return NULL; + } + int end_lineno = token->end_lineno; + UNUSED(end_lineno); // Only used by EXTRA macro + int end_col_offset = token->end_col_offset; + UNUSED(end_col_offset); // Only used by EXTRA macro + res = _Py_Lambda ( ( a ) ? a : CHECK ( _PyPegen_empty_arguments ( p ) ) , b , EXTRA ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// lambda_parameters: +// | lambda_slash_without_default [',' lambda_plain_names] [',' lambda_names_with_default] [',' lambda_star_etc?] +// | lambda_slash_with_default [',' lambda_names_with_default] [',' lambda_star_etc?] +// | lambda_plain_names [',' lambda_names_with_default] [',' lambda_star_etc?] +// | lambda_names_with_default [',' lambda_star_etc?] +// | lambda_star_etc +static arguments_ty +lambda_parameters_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + arguments_ty res = NULL; + int mark = p->mark; + { // lambda_slash_without_default [',' lambda_plain_names] [',' lambda_names_with_default] [',' lambda_star_etc?] + asdl_seq* a; + void *b; + void *c; + void *d; + if ( + (a = lambda_slash_without_default_rule(p)) + && + (b = _tmp_65_rule(p), 1) + && + (c = _tmp_66_rule(p), 1) + && + (d = _tmp_67_rule(p), 1) + ) + { + res = _PyPegen_make_arguments ( p , a , NULL , b , c , d ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + { // lambda_slash_with_default [',' lambda_names_with_default] [',' lambda_star_etc?] + SlashWithDefault* a; + void *b; + void *c; + if ( + (a = lambda_slash_with_default_rule(p)) + && + (b = _tmp_68_rule(p), 1) + && + (c = _tmp_69_rule(p), 1) + ) + { + res = _PyPegen_make_arguments ( p , NULL , a , NULL , b , c ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + { // lambda_plain_names [',' lambda_names_with_default] [',' lambda_star_etc?] + asdl_seq* a; + void *b; + void *c; + if ( + (a = lambda_plain_names_rule(p)) + && + (b = _tmp_70_rule(p), 1) + && + (c = _tmp_71_rule(p), 1) + ) + { + res = _PyPegen_make_arguments ( p , NULL , NULL , a , b , c ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + { // lambda_names_with_default [',' lambda_star_etc?] + asdl_seq* a; + void *b; + if ( + (a = lambda_names_with_default_rule(p)) + && + (b = _tmp_72_rule(p), 1) + ) + { + res = _PyPegen_make_arguments ( p , NULL , NULL , NULL , a , b ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + { // lambda_star_etc + StarEtc* a; + if ( + (a = lambda_star_etc_rule(p)) + ) + { + res = _PyPegen_make_arguments ( p , NULL , NULL , NULL , NULL , a ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// lambda_slash_without_default: lambda_plain_names ',' '/' +static asdl_seq* +lambda_slash_without_default_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + asdl_seq* res = NULL; + int mark = p->mark; + { // lambda_plain_names ',' '/' + asdl_seq* a; + void *literal; + void *literal_1; + if ( + (a = lambda_plain_names_rule(p)) + && + (literal = _PyPegen_expect_token(p, 12)) + && + (literal_1 = _PyPegen_expect_token(p, 17)) + ) + { + res = a; + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// lambda_slash_with_default: [lambda_plain_names ','] lambda_names_with_default ',' '/' +static SlashWithDefault* +lambda_slash_with_default_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + SlashWithDefault* res = NULL; + int mark = p->mark; + { // [lambda_plain_names ','] lambda_names_with_default ',' '/' + void *a; + asdl_seq* b; + void *literal; + void *literal_1; + if ( + (a = _tmp_73_rule(p), 1) + && + (b = lambda_names_with_default_rule(p)) + && + (literal = _PyPegen_expect_token(p, 12)) + && + (literal_1 = _PyPegen_expect_token(p, 17)) + ) + { + res = _PyPegen_slash_with_default ( p , a , b ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// lambda_star_etc: +// | '*' lambda_plain_name lambda_name_with_optional_default* [',' lambda_kwds] ','? +// | '*' lambda_name_with_optional_default+ [',' lambda_kwds] ','? +// | lambda_kwds ','? +static StarEtc* +lambda_star_etc_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + StarEtc* res = NULL; + int mark = p->mark; + { // '*' lambda_plain_name lambda_name_with_optional_default* [',' lambda_kwds] ','? + arg_ty a; + asdl_seq * b; + void *c; + void *literal; + void *opt_var; + UNUSED(opt_var); // Silence compiler warnings + if ( + (literal = _PyPegen_expect_token(p, 16)) + && + (a = lambda_plain_name_rule(p)) + && + (b = _loop0_74_rule(p)) + && + (c = _tmp_75_rule(p), 1) + && + (opt_var = _PyPegen_expect_token(p, 12), 1) + ) + { + res = _PyPegen_star_etc ( p , a , b , c ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + { // '*' lambda_name_with_optional_default+ [',' lambda_kwds] ','? + asdl_seq * b; + void *c; + void *literal; + void *opt_var; + UNUSED(opt_var); // Silence compiler warnings + if ( + (literal = _PyPegen_expect_token(p, 16)) + && + (b = _loop1_76_rule(p)) + && + (c = _tmp_77_rule(p), 1) + && + (opt_var = _PyPegen_expect_token(p, 12), 1) + ) + { + res = _PyPegen_star_etc ( p , NULL , b , c ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + { // lambda_kwds ','? + arg_ty a; + void *opt_var; + UNUSED(opt_var); // Silence compiler warnings + if ( + (a = lambda_kwds_rule(p)) + && + (opt_var = _PyPegen_expect_token(p, 12), 1) + ) + { + res = _PyPegen_star_etc ( p , NULL , NULL , a ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// lambda_name_with_optional_default: ',' lambda_plain_name ['=' expression] +static NameDefaultPair* +lambda_name_with_optional_default_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + NameDefaultPair* res = NULL; + int mark = p->mark; + { // ',' lambda_plain_name ['=' expression] + arg_ty a; + void *b; + void *literal; + if ( + (literal = _PyPegen_expect_token(p, 12)) + && + (a = lambda_plain_name_rule(p)) + && + (b = _tmp_78_rule(p), 1) + ) + { + res = _PyPegen_name_default_pair ( p , a , b ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// lambda_names_with_default: ','.lambda_name_with_default+ +static asdl_seq* +lambda_names_with_default_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + asdl_seq* res = NULL; + int mark = p->mark; + { // ','.lambda_name_with_default+ + asdl_seq * a; + if ( + (a = _gather_79_rule(p)) + ) + { + res = a; + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// lambda_name_with_default: lambda_plain_name '=' expression +static NameDefaultPair* +lambda_name_with_default_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + NameDefaultPair* res = NULL; + int mark = p->mark; + { // lambda_plain_name '=' expression + expr_ty e; + void *literal; + arg_ty n; + if ( + (n = lambda_plain_name_rule(p)) + && + (literal = _PyPegen_expect_token(p, 22)) + && + (e = expression_rule(p)) + ) + { + res = _PyPegen_name_default_pair ( p , n , e ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// lambda_plain_names: ','.(lambda_plain_name !'=')+ +static asdl_seq* +lambda_plain_names_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + asdl_seq* res = NULL; + int mark = p->mark; + { // ','.(lambda_plain_name !'=')+ + asdl_seq * a; + if ( + (a = _gather_81_rule(p)) + ) + { + res = a; + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// lambda_plain_name: NAME +static arg_ty +lambda_plain_name_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + arg_ty res = NULL; + int mark = p->mark; + if (p->mark == p->fill && _PyPegen_fill_token(p) < 0) { + p->error_indicator = 1; + return NULL; + } + int start_lineno = p->tokens[mark]->lineno; + UNUSED(start_lineno); // Only used by EXTRA macro + int start_col_offset = p->tokens[mark]->col_offset; + UNUSED(start_col_offset); // Only used by EXTRA macro + { // NAME + expr_ty a; + if ( + (a = _PyPegen_name_token(p)) + ) + { + Token *token = _PyPegen_get_last_nonnwhitespace_token(p); + if (token == NULL) { + return NULL; + } + int end_lineno = token->end_lineno; + UNUSED(end_lineno); // Only used by EXTRA macro + int end_col_offset = token->end_col_offset; + UNUSED(end_col_offset); // Only used by EXTRA macro + res = _Py_arg ( a -> v . Name . id , NULL , NULL , EXTRA ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// lambda_kwds: '**' lambda_plain_name +static arg_ty +lambda_kwds_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + arg_ty res = NULL; + int mark = p->mark; + { // '**' lambda_plain_name + arg_ty a; + void *literal; + if ( + (literal = _PyPegen_expect_token(p, 35)) + && + (a = lambda_plain_name_rule(p)) + ) + { + res = a; + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// disjunction: conjunction (('or' conjunction))+ | conjunction +static expr_ty +disjunction_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + expr_ty res = NULL; + if (_PyPegen_is_memoized(p, disjunction_type, &res)) + return res; + int mark = p->mark; + if (p->mark == p->fill && _PyPegen_fill_token(p) < 0) { + p->error_indicator = 1; + return NULL; + } + int start_lineno = p->tokens[mark]->lineno; + UNUSED(start_lineno); // Only used by EXTRA macro + int start_col_offset = p->tokens[mark]->col_offset; + UNUSED(start_col_offset); // Only used by EXTRA macro + { // conjunction (('or' conjunction))+ + expr_ty a; + asdl_seq * b; + if ( + (a = conjunction_rule(p)) + && + (b = _loop1_83_rule(p)) + ) + { + Token *token = _PyPegen_get_last_nonnwhitespace_token(p); + if (token == NULL) { + return NULL; + } + int end_lineno = token->end_lineno; + UNUSED(end_lineno); // Only used by EXTRA macro + int end_col_offset = token->end_col_offset; + UNUSED(end_col_offset); // Only used by EXTRA macro + res = _Py_BoolOp ( Or , CHECK ( _PyPegen_seq_insert_in_front ( p , a , b ) ) , EXTRA ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + { // conjunction + expr_ty conjunction_var; + if ( + (conjunction_var = conjunction_rule(p)) + ) + { + res = conjunction_var; + goto done; + } + p->mark = mark; + } + res = NULL; + done: + _PyPegen_insert_memo(p, mark, disjunction_type, res); + return res; +} + +// conjunction: inversion (('and' inversion))+ | inversion +static expr_ty +conjunction_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + expr_ty res = NULL; + if (_PyPegen_is_memoized(p, conjunction_type, &res)) + return res; + int mark = p->mark; + if (p->mark == p->fill && _PyPegen_fill_token(p) < 0) { + p->error_indicator = 1; + return NULL; + } + int start_lineno = p->tokens[mark]->lineno; + UNUSED(start_lineno); // Only used by EXTRA macro + int start_col_offset = p->tokens[mark]->col_offset; + UNUSED(start_col_offset); // Only used by EXTRA macro + { // inversion (('and' inversion))+ + expr_ty a; + asdl_seq * b; + if ( + (a = inversion_rule(p)) + && + (b = _loop1_84_rule(p)) + ) + { + Token *token = _PyPegen_get_last_nonnwhitespace_token(p); + if (token == NULL) { + return NULL; + } + int end_lineno = token->end_lineno; + UNUSED(end_lineno); // Only used by EXTRA macro + int end_col_offset = token->end_col_offset; + UNUSED(end_col_offset); // Only used by EXTRA macro + res = _Py_BoolOp ( And , CHECK ( _PyPegen_seq_insert_in_front ( p , a , b ) ) , EXTRA ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + { // inversion + expr_ty inversion_var; + if ( + (inversion_var = inversion_rule(p)) + ) + { + res = inversion_var; + goto done; + } + p->mark = mark; + } + res = NULL; + done: + _PyPegen_insert_memo(p, mark, conjunction_type, res); + return res; +} + +// inversion: 'not' inversion | comparison +static expr_ty +inversion_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + expr_ty res = NULL; + if (_PyPegen_is_memoized(p, inversion_type, &res)) + return res; + int mark = p->mark; + if (p->mark == p->fill && _PyPegen_fill_token(p) < 0) { + p->error_indicator = 1; + return NULL; + } + int start_lineno = p->tokens[mark]->lineno; + UNUSED(start_lineno); // Only used by EXTRA macro + int start_col_offset = p->tokens[mark]->col_offset; + UNUSED(start_col_offset); // Only used by EXTRA macro + { // 'not' inversion + expr_ty a; + void *keyword; + if ( + (keyword = _PyPegen_expect_token(p, 525)) + && + (a = inversion_rule(p)) + ) + { + Token *token = _PyPegen_get_last_nonnwhitespace_token(p); + if (token == NULL) { + return NULL; + } + int end_lineno = token->end_lineno; + UNUSED(end_lineno); // Only used by EXTRA macro + int end_col_offset = token->end_col_offset; + UNUSED(end_col_offset); // Only used by EXTRA macro + res = _Py_UnaryOp ( Not , a , EXTRA ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + { // comparison + expr_ty comparison_var; + if ( + (comparison_var = comparison_rule(p)) + ) + { + res = comparison_var; + goto done; + } + p->mark = mark; + } + res = NULL; + done: + _PyPegen_insert_memo(p, mark, inversion_type, res); + return res; +} + +// comparison: bitwise_or compare_op_bitwise_or_pair+ | bitwise_or +static expr_ty +comparison_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + expr_ty res = NULL; + int mark = p->mark; + if (p->mark == p->fill && _PyPegen_fill_token(p) < 0) { + p->error_indicator = 1; + return NULL; + } + int start_lineno = p->tokens[mark]->lineno; + UNUSED(start_lineno); // Only used by EXTRA macro + int start_col_offset = p->tokens[mark]->col_offset; + UNUSED(start_col_offset); // Only used by EXTRA macro + { // bitwise_or compare_op_bitwise_or_pair+ + expr_ty a; + asdl_seq * b; + if ( + (a = bitwise_or_rule(p)) + && + (b = _loop1_85_rule(p)) + ) + { + Token *token = _PyPegen_get_last_nonnwhitespace_token(p); + if (token == NULL) { + return NULL; + } + int end_lineno = token->end_lineno; + UNUSED(end_lineno); // Only used by EXTRA macro + int end_col_offset = token->end_col_offset; + UNUSED(end_col_offset); // Only used by EXTRA macro + res = _Py_Compare ( a , CHECK ( _PyPegen_get_cmpops ( p , b ) ) , CHECK ( _PyPegen_get_exprs ( p , b ) ) , EXTRA ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + { // bitwise_or + expr_ty bitwise_or_var; + if ( + (bitwise_or_var = bitwise_or_rule(p)) + ) + { + res = bitwise_or_var; + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// compare_op_bitwise_or_pair: +// | eq_bitwise_or +// | noteq_bitwise_or +// | lte_bitwise_or +// | lt_bitwise_or +// | gte_bitwise_or +// | gt_bitwise_or +// | notin_bitwise_or +// | in_bitwise_or +// | isnot_bitwise_or +// | is_bitwise_or +static CmpopExprPair* +compare_op_bitwise_or_pair_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + CmpopExprPair* res = NULL; + int mark = p->mark; + { // eq_bitwise_or + CmpopExprPair* eq_bitwise_or_var; + if ( + (eq_bitwise_or_var = eq_bitwise_or_rule(p)) + ) + { + res = eq_bitwise_or_var; + goto done; + } + p->mark = mark; + } + { // noteq_bitwise_or + CmpopExprPair* noteq_bitwise_or_var; + if ( + (noteq_bitwise_or_var = noteq_bitwise_or_rule(p)) + ) + { + res = noteq_bitwise_or_var; + goto done; + } + p->mark = mark; + } + { // lte_bitwise_or + CmpopExprPair* lte_bitwise_or_var; + if ( + (lte_bitwise_or_var = lte_bitwise_or_rule(p)) + ) + { + res = lte_bitwise_or_var; + goto done; + } + p->mark = mark; + } + { // lt_bitwise_or + CmpopExprPair* lt_bitwise_or_var; + if ( + (lt_bitwise_or_var = lt_bitwise_or_rule(p)) + ) + { + res = lt_bitwise_or_var; + goto done; + } + p->mark = mark; + } + { // gte_bitwise_or + CmpopExprPair* gte_bitwise_or_var; + if ( + (gte_bitwise_or_var = gte_bitwise_or_rule(p)) + ) + { + res = gte_bitwise_or_var; + goto done; + } + p->mark = mark; + } + { // gt_bitwise_or + CmpopExprPair* gt_bitwise_or_var; + if ( + (gt_bitwise_or_var = gt_bitwise_or_rule(p)) + ) + { + res = gt_bitwise_or_var; + goto done; + } + p->mark = mark; + } + { // notin_bitwise_or + CmpopExprPair* notin_bitwise_or_var; + if ( + (notin_bitwise_or_var = notin_bitwise_or_rule(p)) + ) + { + res = notin_bitwise_or_var; + goto done; + } + p->mark = mark; + } + { // in_bitwise_or + CmpopExprPair* in_bitwise_or_var; + if ( + (in_bitwise_or_var = in_bitwise_or_rule(p)) + ) + { + res = in_bitwise_or_var; + goto done; + } + p->mark = mark; + } + { // isnot_bitwise_or + CmpopExprPair* isnot_bitwise_or_var; + if ( + (isnot_bitwise_or_var = isnot_bitwise_or_rule(p)) + ) + { + res = isnot_bitwise_or_var; + goto done; + } + p->mark = mark; + } + { // is_bitwise_or + CmpopExprPair* is_bitwise_or_var; + if ( + (is_bitwise_or_var = is_bitwise_or_rule(p)) + ) + { + res = is_bitwise_or_var; + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// eq_bitwise_or: '==' bitwise_or +static CmpopExprPair* +eq_bitwise_or_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + CmpopExprPair* res = NULL; + int mark = p->mark; + { // '==' bitwise_or + expr_ty a; + void *literal; + if ( + (literal = _PyPegen_expect_token(p, 27)) + && + (a = bitwise_or_rule(p)) + ) + { + res = _PyPegen_cmpop_expr_pair ( p , Eq , a ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// noteq_bitwise_or: '!=' bitwise_or +static CmpopExprPair* +noteq_bitwise_or_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + CmpopExprPair* res = NULL; + int mark = p->mark; + { // '!=' bitwise_or + expr_ty a; + void *literal; + if ( + (literal = _PyPegen_expect_token(p, 28)) + && + (a = bitwise_or_rule(p)) + ) + { + res = _PyPegen_cmpop_expr_pair ( p , NotEq , a ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// lte_bitwise_or: '<=' bitwise_or +static CmpopExprPair* +lte_bitwise_or_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + CmpopExprPair* res = NULL; + int mark = p->mark; + { // '<=' bitwise_or + expr_ty a; + void *literal; + if ( + (literal = _PyPegen_expect_token(p, 29)) + && + (a = bitwise_or_rule(p)) + ) + { + res = _PyPegen_cmpop_expr_pair ( p , LtE , a ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// lt_bitwise_or: '<' bitwise_or +static CmpopExprPair* +lt_bitwise_or_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + CmpopExprPair* res = NULL; + int mark = p->mark; + { // '<' bitwise_or + expr_ty a; + void *literal; + if ( + (literal = _PyPegen_expect_token(p, 20)) + && + (a = bitwise_or_rule(p)) + ) + { + res = _PyPegen_cmpop_expr_pair ( p , Lt , a ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// gte_bitwise_or: '>=' bitwise_or +static CmpopExprPair* +gte_bitwise_or_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + CmpopExprPair* res = NULL; + int mark = p->mark; + { // '>=' bitwise_or + expr_ty a; + void *literal; + if ( + (literal = _PyPegen_expect_token(p, 30)) + && + (a = bitwise_or_rule(p)) + ) + { + res = _PyPegen_cmpop_expr_pair ( p , GtE , a ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// gt_bitwise_or: '>' bitwise_or +static CmpopExprPair* +gt_bitwise_or_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + CmpopExprPair* res = NULL; + int mark = p->mark; + { // '>' bitwise_or + expr_ty a; + void *literal; + if ( + (literal = _PyPegen_expect_token(p, 21)) + && + (a = bitwise_or_rule(p)) + ) + { + res = _PyPegen_cmpop_expr_pair ( p , Gt , a ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// notin_bitwise_or: 'not' 'in' bitwise_or +static CmpopExprPair* +notin_bitwise_or_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + CmpopExprPair* res = NULL; + int mark = p->mark; + { // 'not' 'in' bitwise_or + expr_ty a; + void *keyword; + void *keyword_1; + if ( + (keyword = _PyPegen_expect_token(p, 525)) + && + (keyword_1 = _PyPegen_expect_token(p, 518)) + && + (a = bitwise_or_rule(p)) + ) + { + res = _PyPegen_cmpop_expr_pair ( p , NotIn , a ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// in_bitwise_or: 'in' bitwise_or +static CmpopExprPair* +in_bitwise_or_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + CmpopExprPair* res = NULL; + int mark = p->mark; + { // 'in' bitwise_or + expr_ty a; + void *keyword; + if ( + (keyword = _PyPegen_expect_token(p, 518)) + && + (a = bitwise_or_rule(p)) + ) + { + res = _PyPegen_cmpop_expr_pair ( p , In , a ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// isnot_bitwise_or: 'is' 'not' bitwise_or +static CmpopExprPair* +isnot_bitwise_or_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + CmpopExprPair* res = NULL; + int mark = p->mark; + { // 'is' 'not' bitwise_or + expr_ty a; + void *keyword; + void *keyword_1; + if ( + (keyword = _PyPegen_expect_token(p, 526)) + && + (keyword_1 = _PyPegen_expect_token(p, 525)) + && + (a = bitwise_or_rule(p)) + ) + { + res = _PyPegen_cmpop_expr_pair ( p , IsNot , a ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// is_bitwise_or: 'is' bitwise_or +static CmpopExprPair* +is_bitwise_or_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + CmpopExprPair* res = NULL; + int mark = p->mark; + { // 'is' bitwise_or + expr_ty a; + void *keyword; + if ( + (keyword = _PyPegen_expect_token(p, 526)) + && + (a = bitwise_or_rule(p)) + ) + { + res = _PyPegen_cmpop_expr_pair ( p , Is , a ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// Left-recursive +// bitwise_or: bitwise_or '|' bitwise_xor | bitwise_xor +static expr_ty bitwise_or_raw(Parser *); +static expr_ty +bitwise_or_rule(Parser *p) +{ + expr_ty res = NULL; + if (_PyPegen_is_memoized(p, bitwise_or_type, &res)) + return res; + int mark = p->mark; + int resmark = p->mark; + while (1) { + int tmpvar_1 = _PyPegen_update_memo(p, mark, bitwise_or_type, res); + if (tmpvar_1) { + return res; + } + p->mark = mark; + void *raw = bitwise_or_raw(p); + if (raw == NULL || p->mark <= resmark) + break; + resmark = p->mark; + res = raw; + } + p->mark = resmark; + return res; +} +static expr_ty +bitwise_or_raw(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + expr_ty res = NULL; + int mark = p->mark; + if (p->mark == p->fill && _PyPegen_fill_token(p) < 0) { + p->error_indicator = 1; + return NULL; + } + int start_lineno = p->tokens[mark]->lineno; + UNUSED(start_lineno); // Only used by EXTRA macro + int start_col_offset = p->tokens[mark]->col_offset; + UNUSED(start_col_offset); // Only used by EXTRA macro + { // bitwise_or '|' bitwise_xor + expr_ty a; + expr_ty b; + void *literal; + if ( + (a = bitwise_or_rule(p)) + && + (literal = _PyPegen_expect_token(p, 18)) + && + (b = bitwise_xor_rule(p)) + ) + { + Token *token = _PyPegen_get_last_nonnwhitespace_token(p); + if (token == NULL) { + return NULL; + } + int end_lineno = token->end_lineno; + UNUSED(end_lineno); // Only used by EXTRA macro + int end_col_offset = token->end_col_offset; + UNUSED(end_col_offset); // Only used by EXTRA macro + res = _Py_BinOp ( a , BitOr , b , EXTRA ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + { // bitwise_xor + expr_ty bitwise_xor_var; + if ( + (bitwise_xor_var = bitwise_xor_rule(p)) + ) + { + res = bitwise_xor_var; + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// Left-recursive +// bitwise_xor: bitwise_xor '^' bitwise_and | bitwise_and +static expr_ty bitwise_xor_raw(Parser *); +static expr_ty +bitwise_xor_rule(Parser *p) +{ + expr_ty res = NULL; + if (_PyPegen_is_memoized(p, bitwise_xor_type, &res)) + return res; + int mark = p->mark; + int resmark = p->mark; + while (1) { + int tmpvar_2 = _PyPegen_update_memo(p, mark, bitwise_xor_type, res); + if (tmpvar_2) { + return res; + } + p->mark = mark; + void *raw = bitwise_xor_raw(p); + if (raw == NULL || p->mark <= resmark) + break; + resmark = p->mark; + res = raw; + } + p->mark = resmark; + return res; +} +static expr_ty +bitwise_xor_raw(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + expr_ty res = NULL; + int mark = p->mark; + if (p->mark == p->fill && _PyPegen_fill_token(p) < 0) { + p->error_indicator = 1; + return NULL; + } + int start_lineno = p->tokens[mark]->lineno; + UNUSED(start_lineno); // Only used by EXTRA macro + int start_col_offset = p->tokens[mark]->col_offset; + UNUSED(start_col_offset); // Only used by EXTRA macro + { // bitwise_xor '^' bitwise_and + expr_ty a; + expr_ty b; + void *literal; + if ( + (a = bitwise_xor_rule(p)) + && + (literal = _PyPegen_expect_token(p, 32)) + && + (b = bitwise_and_rule(p)) + ) + { + Token *token = _PyPegen_get_last_nonnwhitespace_token(p); + if (token == NULL) { + return NULL; + } + int end_lineno = token->end_lineno; + UNUSED(end_lineno); // Only used by EXTRA macro + int end_col_offset = token->end_col_offset; + UNUSED(end_col_offset); // Only used by EXTRA macro + res = _Py_BinOp ( a , BitXor , b , EXTRA ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + { // bitwise_and + expr_ty bitwise_and_var; + if ( + (bitwise_and_var = bitwise_and_rule(p)) + ) + { + res = bitwise_and_var; + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// Left-recursive +// bitwise_and: bitwise_and '&' shift_expr | shift_expr +static expr_ty bitwise_and_raw(Parser *); +static expr_ty +bitwise_and_rule(Parser *p) +{ + expr_ty res = NULL; + if (_PyPegen_is_memoized(p, bitwise_and_type, &res)) + return res; + int mark = p->mark; + int resmark = p->mark; + while (1) { + int tmpvar_3 = _PyPegen_update_memo(p, mark, bitwise_and_type, res); + if (tmpvar_3) { + return res; + } + p->mark = mark; + void *raw = bitwise_and_raw(p); + if (raw == NULL || p->mark <= resmark) + break; + resmark = p->mark; + res = raw; + } + p->mark = resmark; + return res; +} +static expr_ty +bitwise_and_raw(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + expr_ty res = NULL; + int mark = p->mark; + if (p->mark == p->fill && _PyPegen_fill_token(p) < 0) { + p->error_indicator = 1; + return NULL; + } + int start_lineno = p->tokens[mark]->lineno; + UNUSED(start_lineno); // Only used by EXTRA macro + int start_col_offset = p->tokens[mark]->col_offset; + UNUSED(start_col_offset); // Only used by EXTRA macro + { // bitwise_and '&' shift_expr + expr_ty a; + expr_ty b; + void *literal; + if ( + (a = bitwise_and_rule(p)) + && + (literal = _PyPegen_expect_token(p, 19)) + && + (b = shift_expr_rule(p)) + ) + { + Token *token = _PyPegen_get_last_nonnwhitespace_token(p); + if (token == NULL) { + return NULL; + } + int end_lineno = token->end_lineno; + UNUSED(end_lineno); // Only used by EXTRA macro + int end_col_offset = token->end_col_offset; + UNUSED(end_col_offset); // Only used by EXTRA macro + res = _Py_BinOp ( a , BitAnd , b , EXTRA ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + { // shift_expr + expr_ty shift_expr_var; + if ( + (shift_expr_var = shift_expr_rule(p)) + ) + { + res = shift_expr_var; + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// Left-recursive +// shift_expr: shift_expr '<<' sum | shift_expr '>>' sum | sum +static expr_ty shift_expr_raw(Parser *); +static expr_ty +shift_expr_rule(Parser *p) +{ + expr_ty res = NULL; + if (_PyPegen_is_memoized(p, shift_expr_type, &res)) + return res; + int mark = p->mark; + int resmark = p->mark; + while (1) { + int tmpvar_4 = _PyPegen_update_memo(p, mark, shift_expr_type, res); + if (tmpvar_4) { + return res; + } + p->mark = mark; + void *raw = shift_expr_raw(p); + if (raw == NULL || p->mark <= resmark) + break; + resmark = p->mark; + res = raw; + } + p->mark = resmark; + return res; +} +static expr_ty +shift_expr_raw(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + expr_ty res = NULL; + int mark = p->mark; + if (p->mark == p->fill && _PyPegen_fill_token(p) < 0) { + p->error_indicator = 1; + return NULL; + } + int start_lineno = p->tokens[mark]->lineno; + UNUSED(start_lineno); // Only used by EXTRA macro + int start_col_offset = p->tokens[mark]->col_offset; + UNUSED(start_col_offset); // Only used by EXTRA macro + { // shift_expr '<<' sum + expr_ty a; + expr_ty b; + void *literal; + if ( + (a = shift_expr_rule(p)) + && + (literal = _PyPegen_expect_token(p, 33)) + && + (b = sum_rule(p)) + ) + { + Token *token = _PyPegen_get_last_nonnwhitespace_token(p); + if (token == NULL) { + return NULL; + } + int end_lineno = token->end_lineno; + UNUSED(end_lineno); // Only used by EXTRA macro + int end_col_offset = token->end_col_offset; + UNUSED(end_col_offset); // Only used by EXTRA macro + res = _Py_BinOp ( a , LShift , b , EXTRA ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + { // shift_expr '>>' sum + expr_ty a; + expr_ty b; + void *literal; + if ( + (a = shift_expr_rule(p)) + && + (literal = _PyPegen_expect_token(p, 34)) + && + (b = sum_rule(p)) + ) + { + Token *token = _PyPegen_get_last_nonnwhitespace_token(p); + if (token == NULL) { + return NULL; + } + int end_lineno = token->end_lineno; + UNUSED(end_lineno); // Only used by EXTRA macro + int end_col_offset = token->end_col_offset; + UNUSED(end_col_offset); // Only used by EXTRA macro + res = _Py_BinOp ( a , RShift , b , EXTRA ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + { // sum + expr_ty sum_var; + if ( + (sum_var = sum_rule(p)) + ) + { + res = sum_var; + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// Left-recursive +// sum: sum '+' term | sum '-' term | term +static expr_ty sum_raw(Parser *); +static expr_ty +sum_rule(Parser *p) +{ + expr_ty res = NULL; + if (_PyPegen_is_memoized(p, sum_type, &res)) + return res; + int mark = p->mark; + int resmark = p->mark; + while (1) { + int tmpvar_5 = _PyPegen_update_memo(p, mark, sum_type, res); + if (tmpvar_5) { + return res; + } + p->mark = mark; + void *raw = sum_raw(p); + if (raw == NULL || p->mark <= resmark) + break; + resmark = p->mark; + res = raw; + } + p->mark = resmark; + return res; +} +static expr_ty +sum_raw(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + expr_ty res = NULL; + int mark = p->mark; + if (p->mark == p->fill && _PyPegen_fill_token(p) < 0) { + p->error_indicator = 1; + return NULL; + } + int start_lineno = p->tokens[mark]->lineno; + UNUSED(start_lineno); // Only used by EXTRA macro + int start_col_offset = p->tokens[mark]->col_offset; + UNUSED(start_col_offset); // Only used by EXTRA macro + { // sum '+' term + expr_ty a; + expr_ty b; + void *literal; + if ( + (a = sum_rule(p)) + && + (literal = _PyPegen_expect_token(p, 14)) + && + (b = term_rule(p)) + ) + { + Token *token = _PyPegen_get_last_nonnwhitespace_token(p); + if (token == NULL) { + return NULL; + } + int end_lineno = token->end_lineno; + UNUSED(end_lineno); // Only used by EXTRA macro + int end_col_offset = token->end_col_offset; + UNUSED(end_col_offset); // Only used by EXTRA macro + res = _Py_BinOp ( a , Add , b , EXTRA ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + { // sum '-' term + expr_ty a; + expr_ty b; + void *literal; + if ( + (a = sum_rule(p)) + && + (literal = _PyPegen_expect_token(p, 15)) + && + (b = term_rule(p)) + ) + { + Token *token = _PyPegen_get_last_nonnwhitespace_token(p); + if (token == NULL) { + return NULL; + } + int end_lineno = token->end_lineno; + UNUSED(end_lineno); // Only used by EXTRA macro + int end_col_offset = token->end_col_offset; + UNUSED(end_col_offset); // Only used by EXTRA macro + res = _Py_BinOp ( a , Sub , b , EXTRA ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + { // term + expr_ty term_var; + if ( + (term_var = term_rule(p)) + ) + { + res = term_var; + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// Left-recursive +// term: +// | term '*' factor +// | term '/' factor +// | term '//' factor +// | term '%' factor +// | term '@' factor +// | factor +static expr_ty term_raw(Parser *); +static expr_ty +term_rule(Parser *p) +{ + expr_ty res = NULL; + if (_PyPegen_is_memoized(p, term_type, &res)) + return res; + int mark = p->mark; + int resmark = p->mark; + while (1) { + int tmpvar_6 = _PyPegen_update_memo(p, mark, term_type, res); + if (tmpvar_6) { + return res; + } + p->mark = mark; + void *raw = term_raw(p); + if (raw == NULL || p->mark <= resmark) + break; + resmark = p->mark; + res = raw; + } + p->mark = resmark; + return res; +} +static expr_ty +term_raw(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + expr_ty res = NULL; + int mark = p->mark; + if (p->mark == p->fill && _PyPegen_fill_token(p) < 0) { + p->error_indicator = 1; + return NULL; + } + int start_lineno = p->tokens[mark]->lineno; + UNUSED(start_lineno); // Only used by EXTRA macro + int start_col_offset = p->tokens[mark]->col_offset; + UNUSED(start_col_offset); // Only used by EXTRA macro + { // term '*' factor + expr_ty a; + expr_ty b; + void *literal; + if ( + (a = term_rule(p)) + && + (literal = _PyPegen_expect_token(p, 16)) + && + (b = factor_rule(p)) + ) + { + Token *token = _PyPegen_get_last_nonnwhitespace_token(p); + if (token == NULL) { + return NULL; + } + int end_lineno = token->end_lineno; + UNUSED(end_lineno); // Only used by EXTRA macro + int end_col_offset = token->end_col_offset; + UNUSED(end_col_offset); // Only used by EXTRA macro + res = _Py_BinOp ( a , Mult , b , EXTRA ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + { // term '/' factor + expr_ty a; + expr_ty b; + void *literal; + if ( + (a = term_rule(p)) + && + (literal = _PyPegen_expect_token(p, 17)) + && + (b = factor_rule(p)) + ) + { + Token *token = _PyPegen_get_last_nonnwhitespace_token(p); + if (token == NULL) { + return NULL; + } + int end_lineno = token->end_lineno; + UNUSED(end_lineno); // Only used by EXTRA macro + int end_col_offset = token->end_col_offset; + UNUSED(end_col_offset); // Only used by EXTRA macro + res = _Py_BinOp ( a , Div , b , EXTRA ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + { // term '//' factor + expr_ty a; + expr_ty b; + void *literal; + if ( + (a = term_rule(p)) + && + (literal = _PyPegen_expect_token(p, 47)) + && + (b = factor_rule(p)) + ) + { + Token *token = _PyPegen_get_last_nonnwhitespace_token(p); + if (token == NULL) { + return NULL; + } + int end_lineno = token->end_lineno; + UNUSED(end_lineno); // Only used by EXTRA macro + int end_col_offset = token->end_col_offset; + UNUSED(end_col_offset); // Only used by EXTRA macro + res = _Py_BinOp ( a , FloorDiv , b , EXTRA ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + { // term '%' factor + expr_ty a; + expr_ty b; + void *literal; + if ( + (a = term_rule(p)) + && + (literal = _PyPegen_expect_token(p, 24)) + && + (b = factor_rule(p)) + ) + { + Token *token = _PyPegen_get_last_nonnwhitespace_token(p); + if (token == NULL) { + return NULL; + } + int end_lineno = token->end_lineno; + UNUSED(end_lineno); // Only used by EXTRA macro + int end_col_offset = token->end_col_offset; + UNUSED(end_col_offset); // Only used by EXTRA macro + res = _Py_BinOp ( a , Mod , b , EXTRA ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + { // term '@' factor + expr_ty a; + expr_ty b; + void *literal; + if ( + (a = term_rule(p)) + && + (literal = _PyPegen_expect_token(p, 49)) + && + (b = factor_rule(p)) + ) + { + Token *token = _PyPegen_get_last_nonnwhitespace_token(p); + if (token == NULL) { + return NULL; + } + int end_lineno = token->end_lineno; + UNUSED(end_lineno); // Only used by EXTRA macro + int end_col_offset = token->end_col_offset; + UNUSED(end_col_offset); // Only used by EXTRA macro + res = _Py_BinOp ( a , MatMult , b , EXTRA ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + { // factor + expr_ty factor_var; + if ( + (factor_var = factor_rule(p)) + ) + { + res = factor_var; + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// factor: '+' factor | '-' factor | '~' factor | power +static expr_ty +factor_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + expr_ty res = NULL; + if (_PyPegen_is_memoized(p, factor_type, &res)) + return res; + int mark = p->mark; + if (p->mark == p->fill && _PyPegen_fill_token(p) < 0) { + p->error_indicator = 1; + return NULL; + } + int start_lineno = p->tokens[mark]->lineno; + UNUSED(start_lineno); // Only used by EXTRA macro + int start_col_offset = p->tokens[mark]->col_offset; + UNUSED(start_col_offset); // Only used by EXTRA macro + { // '+' factor + expr_ty a; + void *literal; + if ( + (literal = _PyPegen_expect_token(p, 14)) + && + (a = factor_rule(p)) + ) + { + Token *token = _PyPegen_get_last_nonnwhitespace_token(p); + if (token == NULL) { + return NULL; + } + int end_lineno = token->end_lineno; + UNUSED(end_lineno); // Only used by EXTRA macro + int end_col_offset = token->end_col_offset; + UNUSED(end_col_offset); // Only used by EXTRA macro + res = _Py_UnaryOp ( UAdd , a , EXTRA ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + { // '-' factor + expr_ty a; + void *literal; + if ( + (literal = _PyPegen_expect_token(p, 15)) + && + (a = factor_rule(p)) + ) + { + Token *token = _PyPegen_get_last_nonnwhitespace_token(p); + if (token == NULL) { + return NULL; + } + int end_lineno = token->end_lineno; + UNUSED(end_lineno); // Only used by EXTRA macro + int end_col_offset = token->end_col_offset; + UNUSED(end_col_offset); // Only used by EXTRA macro + res = _Py_UnaryOp ( USub , a , EXTRA ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + { // '~' factor + expr_ty a; + void *literal; + if ( + (literal = _PyPegen_expect_token(p, 31)) + && + (a = factor_rule(p)) + ) + { + Token *token = _PyPegen_get_last_nonnwhitespace_token(p); + if (token == NULL) { + return NULL; + } + int end_lineno = token->end_lineno; + UNUSED(end_lineno); // Only used by EXTRA macro + int end_col_offset = token->end_col_offset; + UNUSED(end_col_offset); // Only used by EXTRA macro + res = _Py_UnaryOp ( Invert , a , EXTRA ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + { // power + expr_ty power_var; + if ( + (power_var = power_rule(p)) + ) + { + res = power_var; + goto done; + } + p->mark = mark; + } + res = NULL; + done: + _PyPegen_insert_memo(p, mark, factor_type, res); + return res; +} + +// power: await_primary '**' factor | await_primary +static expr_ty +power_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + expr_ty res = NULL; + int mark = p->mark; + if (p->mark == p->fill && _PyPegen_fill_token(p) < 0) { + p->error_indicator = 1; + return NULL; + } + int start_lineno = p->tokens[mark]->lineno; + UNUSED(start_lineno); // Only used by EXTRA macro + int start_col_offset = p->tokens[mark]->col_offset; + UNUSED(start_col_offset); // Only used by EXTRA macro + { // await_primary '**' factor + expr_ty a; + expr_ty b; + void *literal; + if ( + (a = await_primary_rule(p)) + && + (literal = _PyPegen_expect_token(p, 35)) + && + (b = factor_rule(p)) + ) + { + Token *token = _PyPegen_get_last_nonnwhitespace_token(p); + if (token == NULL) { + return NULL; + } + int end_lineno = token->end_lineno; + UNUSED(end_lineno); // Only used by EXTRA macro + int end_col_offset = token->end_col_offset; + UNUSED(end_col_offset); // Only used by EXTRA macro + res = _Py_BinOp ( a , Pow , b , EXTRA ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + { // await_primary + expr_ty await_primary_var; + if ( + (await_primary_var = await_primary_rule(p)) + ) + { + res = await_primary_var; + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// await_primary: AWAIT primary | primary +static expr_ty +await_primary_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + expr_ty res = NULL; + if (_PyPegen_is_memoized(p, await_primary_type, &res)) + return res; + int mark = p->mark; + if (p->mark == p->fill && _PyPegen_fill_token(p) < 0) { + p->error_indicator = 1; + return NULL; + } + int start_lineno = p->tokens[mark]->lineno; + UNUSED(start_lineno); // Only used by EXTRA macro + int start_col_offset = p->tokens[mark]->col_offset; + UNUSED(start_col_offset); // Only used by EXTRA macro + { // AWAIT primary + expr_ty a; + void *await_var; + if ( + (await_var = _PyPegen_await_token(p)) + && + (a = primary_rule(p)) + ) + { + Token *token = _PyPegen_get_last_nonnwhitespace_token(p); + if (token == NULL) { + return NULL; + } + int end_lineno = token->end_lineno; + UNUSED(end_lineno); // Only used by EXTRA macro + int end_col_offset = token->end_col_offset; + UNUSED(end_col_offset); // Only used by EXTRA macro + res = _Py_Await ( a , EXTRA ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + { // primary + expr_ty primary_var; + if ( + (primary_var = primary_rule(p)) + ) + { + res = primary_var; + goto done; + } + p->mark = mark; + } + res = NULL; + done: + _PyPegen_insert_memo(p, mark, await_primary_type, res); + return res; +} + +// Left-recursive +// primary: +// | primary '.' NAME +// | primary genexp +// | primary '(' arguments? ')' +// | primary '[' slices ']' +// | atom +static expr_ty primary_raw(Parser *); +static expr_ty +primary_rule(Parser *p) +{ + expr_ty res = NULL; + if (_PyPegen_is_memoized(p, primary_type, &res)) + return res; + int mark = p->mark; + int resmark = p->mark; + while (1) { + int tmpvar_7 = _PyPegen_update_memo(p, mark, primary_type, res); + if (tmpvar_7) { + return res; + } + p->mark = mark; + void *raw = primary_raw(p); + if (raw == NULL || p->mark <= resmark) + break; + resmark = p->mark; + res = raw; + } + p->mark = resmark; + return res; +} +static expr_ty +primary_raw(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + expr_ty res = NULL; + int mark = p->mark; + if (p->mark == p->fill && _PyPegen_fill_token(p) < 0) { + p->error_indicator = 1; + return NULL; + } + int start_lineno = p->tokens[mark]->lineno; + UNUSED(start_lineno); // Only used by EXTRA macro + int start_col_offset = p->tokens[mark]->col_offset; + UNUSED(start_col_offset); // Only used by EXTRA macro + { // primary '.' NAME + expr_ty a; + expr_ty b; + void *literal; + if ( + (a = primary_rule(p)) + && + (literal = _PyPegen_expect_token(p, 23)) + && + (b = _PyPegen_name_token(p)) + ) + { + Token *token = _PyPegen_get_last_nonnwhitespace_token(p); + if (token == NULL) { + return NULL; + } + int end_lineno = token->end_lineno; + UNUSED(end_lineno); // Only used by EXTRA macro + int end_col_offset = token->end_col_offset; + UNUSED(end_col_offset); // Only used by EXTRA macro + res = _Py_Attribute ( a , b -> v . Name . id , Load , EXTRA ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + { // primary genexp + expr_ty a; + expr_ty b; + if ( + (a = primary_rule(p)) + && + (b = genexp_rule(p)) + ) + { + Token *token = _PyPegen_get_last_nonnwhitespace_token(p); + if (token == NULL) { + return NULL; + } + int end_lineno = token->end_lineno; + UNUSED(end_lineno); // Only used by EXTRA macro + int end_col_offset = token->end_col_offset; + UNUSED(end_col_offset); // Only used by EXTRA macro + res = _Py_Call ( a , CHECK ( _PyPegen_singleton_seq ( p , b ) ) , NULL , EXTRA ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + { // primary '(' arguments? ')' + expr_ty a; + void *b; + void *literal; + void *literal_1; + if ( + (a = primary_rule(p)) + && + (literal = _PyPegen_expect_token(p, 7)) + && + (b = arguments_rule(p), 1) + && + (literal_1 = _PyPegen_expect_token(p, 8)) + ) + { + Token *token = _PyPegen_get_last_nonnwhitespace_token(p); + if (token == NULL) { + return NULL; + } + int end_lineno = token->end_lineno; + UNUSED(end_lineno); // Only used by EXTRA macro + int end_col_offset = token->end_col_offset; + UNUSED(end_col_offset); // Only used by EXTRA macro + res = _Py_Call ( a , ( b ) ? ( ( expr_ty ) b ) -> v . Call . args : NULL , ( b ) ? ( ( expr_ty ) b ) -> v . Call . keywords : NULL , EXTRA ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + { // primary '[' slices ']' + expr_ty a; + expr_ty b; + void *literal; + void *literal_1; + if ( + (a = primary_rule(p)) + && + (literal = _PyPegen_expect_token(p, 9)) + && + (b = slices_rule(p)) + && + (literal_1 = _PyPegen_expect_token(p, 10)) + ) + { + Token *token = _PyPegen_get_last_nonnwhitespace_token(p); + if (token == NULL) { + return NULL; + } + int end_lineno = token->end_lineno; + UNUSED(end_lineno); // Only used by EXTRA macro + int end_col_offset = token->end_col_offset; + UNUSED(end_col_offset); // Only used by EXTRA macro + res = _Py_Subscript ( a , b , Load , EXTRA ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + { // atom + expr_ty atom_var; + if ( + (atom_var = atom_rule(p)) + ) + { + res = atom_var; + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// slices: slice !',' | ','.slice+ ','? +static expr_ty +slices_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + expr_ty res = NULL; + int mark = p->mark; + if (p->mark == p->fill && _PyPegen_fill_token(p) < 0) { + p->error_indicator = 1; + return NULL; + } + int start_lineno = p->tokens[mark]->lineno; + UNUSED(start_lineno); // Only used by EXTRA macro + int start_col_offset = p->tokens[mark]->col_offset; + UNUSED(start_col_offset); // Only used by EXTRA macro + { // slice !',' + expr_ty a; + if ( + (a = slice_rule(p)) + && + _PyPegen_lookahead_with_int(0, _PyPegen_expect_token, p, 12) + ) + { + res = a; + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + { // ','.slice+ ','? + asdl_seq * a; + void *opt_var; + UNUSED(opt_var); // Silence compiler warnings + if ( + (a = _gather_86_rule(p)) + && + (opt_var = _PyPegen_expect_token(p, 12), 1) + ) + { + Token *token = _PyPegen_get_last_nonnwhitespace_token(p); + if (token == NULL) { + return NULL; + } + int end_lineno = token->end_lineno; + UNUSED(end_lineno); // Only used by EXTRA macro + int end_col_offset = token->end_col_offset; + UNUSED(end_col_offset); // Only used by EXTRA macro + res = _Py_Tuple ( a , Load , EXTRA ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// slice: expression? ':' expression? [':' expression?] | expression +static expr_ty +slice_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + expr_ty res = NULL; + int mark = p->mark; + if (p->mark == p->fill && _PyPegen_fill_token(p) < 0) { + p->error_indicator = 1; + return NULL; + } + int start_lineno = p->tokens[mark]->lineno; + UNUSED(start_lineno); // Only used by EXTRA macro + int start_col_offset = p->tokens[mark]->col_offset; + UNUSED(start_col_offset); // Only used by EXTRA macro + { // expression? ':' expression? [':' expression?] + void *a; + void *b; + void *c; + void *literal; + if ( + (a = expression_rule(p), 1) + && + (literal = _PyPegen_expect_token(p, 11)) + && + (b = expression_rule(p), 1) + && + (c = _tmp_88_rule(p), 1) + ) + { + Token *token = _PyPegen_get_last_nonnwhitespace_token(p); + if (token == NULL) { + return NULL; + } + int end_lineno = token->end_lineno; + UNUSED(end_lineno); // Only used by EXTRA macro + int end_col_offset = token->end_col_offset; + UNUSED(end_col_offset); // Only used by EXTRA macro + res = _Py_Slice ( a , b , c , EXTRA ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + { // expression + expr_ty a; + if ( + (a = expression_rule(p)) + ) + { + res = a; + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// atom: +// | NAME +// | 'True' +// | 'False' +// | 'None' +// | '__new_parser__' +// | &STRING strings +// | NUMBER +// | &'(' (tuple | group | genexp) +// | &'[' (list | listcomp) +// | &'{' (dict | set | dictcomp | setcomp) +// | '...' +static expr_ty +atom_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + expr_ty res = NULL; + int mark = p->mark; + if (p->mark == p->fill && _PyPegen_fill_token(p) < 0) { + p->error_indicator = 1; + return NULL; + } + int start_lineno = p->tokens[mark]->lineno; + UNUSED(start_lineno); // Only used by EXTRA macro + int start_col_offset = p->tokens[mark]->col_offset; + UNUSED(start_col_offset); // Only used by EXTRA macro + { // NAME + expr_ty name_var; + if ( + (name_var = _PyPegen_name_token(p)) + ) + { + res = name_var; + goto done; + } + p->mark = mark; + } + { // 'True' + void *keyword; + if ( + (keyword = _PyPegen_expect_token(p, 527)) + ) + { + Token *token = _PyPegen_get_last_nonnwhitespace_token(p); + if (token == NULL) { + return NULL; + } + int end_lineno = token->end_lineno; + UNUSED(end_lineno); // Only used by EXTRA macro + int end_col_offset = token->end_col_offset; + UNUSED(end_col_offset); // Only used by EXTRA macro + res = _Py_Constant ( Py_True , NULL , EXTRA ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + { // 'False' + void *keyword; + if ( + (keyword = _PyPegen_expect_token(p, 528)) + ) + { + Token *token = _PyPegen_get_last_nonnwhitespace_token(p); + if (token == NULL) { + return NULL; + } + int end_lineno = token->end_lineno; + UNUSED(end_lineno); // Only used by EXTRA macro + int end_col_offset = token->end_col_offset; + UNUSED(end_col_offset); // Only used by EXTRA macro + res = _Py_Constant ( Py_False , NULL , EXTRA ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + { // 'None' + void *keyword; + if ( + (keyword = _PyPegen_expect_token(p, 529)) + ) + { + Token *token = _PyPegen_get_last_nonnwhitespace_token(p); + if (token == NULL) { + return NULL; + } + int end_lineno = token->end_lineno; + UNUSED(end_lineno); // Only used by EXTRA macro + int end_col_offset = token->end_col_offset; + UNUSED(end_col_offset); // Only used by EXTRA macro + res = _Py_Constant ( Py_None , NULL , EXTRA ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + { // '__new_parser__' + void *keyword; + if ( + (keyword = _PyPegen_expect_token(p, 530)) + ) + { + res = RAISE_SYNTAX_ERROR ( "You found it!" ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + { // &STRING strings + expr_ty strings_var; + if ( + _PyPegen_lookahead(1, _PyPegen_string_token, p) + && + (strings_var = strings_rule(p)) + ) + { + res = strings_var; + goto done; + } + p->mark = mark; + } + { // NUMBER + expr_ty number_var; + if ( + (number_var = _PyPegen_number_token(p)) + ) + { + res = number_var; + goto done; + } + p->mark = mark; + } + { // &'(' (tuple | group | genexp) + void *_tmp_89_var; + if ( + _PyPegen_lookahead_with_int(1, _PyPegen_expect_token, p, 7) + && + (_tmp_89_var = _tmp_89_rule(p)) + ) + { + res = _tmp_89_var; + goto done; + } + p->mark = mark; + } + { // &'[' (list | listcomp) + void *_tmp_90_var; + if ( + _PyPegen_lookahead_with_int(1, _PyPegen_expect_token, p, 9) + && + (_tmp_90_var = _tmp_90_rule(p)) + ) + { + res = _tmp_90_var; + goto done; + } + p->mark = mark; + } + { // &'{' (dict | set | dictcomp | setcomp) + void *_tmp_91_var; + if ( + _PyPegen_lookahead_with_int(1, _PyPegen_expect_token, p, 25) + && + (_tmp_91_var = _tmp_91_rule(p)) + ) + { + res = _tmp_91_var; + goto done; + } + p->mark = mark; + } + { // '...' + void *literal; + if ( + (literal = _PyPegen_expect_token(p, 52)) + ) + { + Token *token = _PyPegen_get_last_nonnwhitespace_token(p); + if (token == NULL) { + return NULL; + } + int end_lineno = token->end_lineno; + UNUSED(end_lineno); // Only used by EXTRA macro + int end_col_offset = token->end_col_offset; + UNUSED(end_col_offset); // Only used by EXTRA macro + res = _Py_Constant ( Py_Ellipsis , NULL , EXTRA ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// strings: STRING+ +static expr_ty +strings_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + expr_ty res = NULL; + if (_PyPegen_is_memoized(p, strings_type, &res)) + return res; + int mark = p->mark; + { // STRING+ + asdl_seq * a; + if ( + (a = _loop1_92_rule(p)) + ) + { + res = _PyPegen_concatenate_strings ( p , a ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + res = NULL; + done: + _PyPegen_insert_memo(p, mark, strings_type, res); + return res; +} + +// list: '[' star_named_expressions? ']' +static expr_ty +list_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + expr_ty res = NULL; + int mark = p->mark; + if (p->mark == p->fill && _PyPegen_fill_token(p) < 0) { + p->error_indicator = 1; + return NULL; + } + int start_lineno = p->tokens[mark]->lineno; + UNUSED(start_lineno); // Only used by EXTRA macro + int start_col_offset = p->tokens[mark]->col_offset; + UNUSED(start_col_offset); // Only used by EXTRA macro + { // '[' star_named_expressions? ']' + void *a; + void *literal; + void *literal_1; + if ( + (literal = _PyPegen_expect_token(p, 9)) + && + (a = star_named_expressions_rule(p), 1) + && + (literal_1 = _PyPegen_expect_token(p, 10)) + ) + { + Token *token = _PyPegen_get_last_nonnwhitespace_token(p); + if (token == NULL) { + return NULL; + } + int end_lineno = token->end_lineno; + UNUSED(end_lineno); // Only used by EXTRA macro + int end_col_offset = token->end_col_offset; + UNUSED(end_col_offset); // Only used by EXTRA macro + res = _Py_List ( a , Load , EXTRA ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// listcomp: '[' named_expression for_if_clauses ']' | invalid_comprehension +static expr_ty +listcomp_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + expr_ty res = NULL; + int mark = p->mark; + if (p->mark == p->fill && _PyPegen_fill_token(p) < 0) { + p->error_indicator = 1; + return NULL; + } + int start_lineno = p->tokens[mark]->lineno; + UNUSED(start_lineno); // Only used by EXTRA macro + int start_col_offset = p->tokens[mark]->col_offset; + UNUSED(start_col_offset); // Only used by EXTRA macro + { // '[' named_expression for_if_clauses ']' + expr_ty a; + asdl_seq* b; + void *literal; + void *literal_1; + if ( + (literal = _PyPegen_expect_token(p, 9)) + && + (a = named_expression_rule(p)) + && + (b = for_if_clauses_rule(p)) + && + (literal_1 = _PyPegen_expect_token(p, 10)) + ) + { + Token *token = _PyPegen_get_last_nonnwhitespace_token(p); + if (token == NULL) { + return NULL; + } + int end_lineno = token->end_lineno; + UNUSED(end_lineno); // Only used by EXTRA macro + int end_col_offset = token->end_col_offset; + UNUSED(end_col_offset); // Only used by EXTRA macro + res = _Py_ListComp ( a , b , EXTRA ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + { // invalid_comprehension + void *invalid_comprehension_var; + if ( + (invalid_comprehension_var = invalid_comprehension_rule(p)) + ) + { + res = invalid_comprehension_var; + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// tuple: '(' [star_named_expression ',' star_named_expressions?] ')' +static expr_ty +tuple_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + expr_ty res = NULL; + int mark = p->mark; + if (p->mark == p->fill && _PyPegen_fill_token(p) < 0) { + p->error_indicator = 1; + return NULL; + } + int start_lineno = p->tokens[mark]->lineno; + UNUSED(start_lineno); // Only used by EXTRA macro + int start_col_offset = p->tokens[mark]->col_offset; + UNUSED(start_col_offset); // Only used by EXTRA macro + { // '(' [star_named_expression ',' star_named_expressions?] ')' + void *a; + void *literal; + void *literal_1; + if ( + (literal = _PyPegen_expect_token(p, 7)) + && + (a = _tmp_93_rule(p), 1) + && + (literal_1 = _PyPegen_expect_token(p, 8)) + ) + { + Token *token = _PyPegen_get_last_nonnwhitespace_token(p); + if (token == NULL) { + return NULL; + } + int end_lineno = token->end_lineno; + UNUSED(end_lineno); // Only used by EXTRA macro + int end_col_offset = token->end_col_offset; + UNUSED(end_col_offset); // Only used by EXTRA macro + res = _Py_Tuple ( a , Load , EXTRA ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// group: '(' (yield_expr | named_expression) ')' +static expr_ty +group_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + expr_ty res = NULL; + int mark = p->mark; + { // '(' (yield_expr | named_expression) ')' + void *a; + void *literal; + void *literal_1; + if ( + (literal = _PyPegen_expect_token(p, 7)) + && + (a = _tmp_94_rule(p)) + && + (literal_1 = _PyPegen_expect_token(p, 8)) + ) + { + res = a; + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// genexp: '(' expression for_if_clauses ')' | invalid_comprehension +static expr_ty +genexp_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + expr_ty res = NULL; + int mark = p->mark; + if (p->mark == p->fill && _PyPegen_fill_token(p) < 0) { + p->error_indicator = 1; + return NULL; + } + int start_lineno = p->tokens[mark]->lineno; + UNUSED(start_lineno); // Only used by EXTRA macro + int start_col_offset = p->tokens[mark]->col_offset; + UNUSED(start_col_offset); // Only used by EXTRA macro + { // '(' expression for_if_clauses ')' + expr_ty a; + asdl_seq* b; + void *literal; + void *literal_1; + if ( + (literal = _PyPegen_expect_token(p, 7)) + && + (a = expression_rule(p)) + && + (b = for_if_clauses_rule(p)) + && + (literal_1 = _PyPegen_expect_token(p, 8)) + ) + { + Token *token = _PyPegen_get_last_nonnwhitespace_token(p); + if (token == NULL) { + return NULL; + } + int end_lineno = token->end_lineno; + UNUSED(end_lineno); // Only used by EXTRA macro + int end_col_offset = token->end_col_offset; + UNUSED(end_col_offset); // Only used by EXTRA macro + res = _Py_GeneratorExp ( a , b , EXTRA ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + { // invalid_comprehension + void *invalid_comprehension_var; + if ( + (invalid_comprehension_var = invalid_comprehension_rule(p)) + ) + { + res = invalid_comprehension_var; + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// set: '{' expressions_list '}' +static expr_ty +set_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + expr_ty res = NULL; + int mark = p->mark; + if (p->mark == p->fill && _PyPegen_fill_token(p) < 0) { + p->error_indicator = 1; + return NULL; + } + int start_lineno = p->tokens[mark]->lineno; + UNUSED(start_lineno); // Only used by EXTRA macro + int start_col_offset = p->tokens[mark]->col_offset; + UNUSED(start_col_offset); // Only used by EXTRA macro + { // '{' expressions_list '}' + asdl_seq* a; + void *literal; + void *literal_1; + if ( + (literal = _PyPegen_expect_token(p, 25)) + && + (a = expressions_list_rule(p)) + && + (literal_1 = _PyPegen_expect_token(p, 26)) + ) + { + Token *token = _PyPegen_get_last_nonnwhitespace_token(p); + if (token == NULL) { + return NULL; + } + int end_lineno = token->end_lineno; + UNUSED(end_lineno); // Only used by EXTRA macro + int end_col_offset = token->end_col_offset; + UNUSED(end_col_offset); // Only used by EXTRA macro + res = _Py_Set ( a , EXTRA ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// setcomp: '{' expression for_if_clauses '}' | invalid_comprehension +static expr_ty +setcomp_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + expr_ty res = NULL; + int mark = p->mark; + if (p->mark == p->fill && _PyPegen_fill_token(p) < 0) { + p->error_indicator = 1; + return NULL; + } + int start_lineno = p->tokens[mark]->lineno; + UNUSED(start_lineno); // Only used by EXTRA macro + int start_col_offset = p->tokens[mark]->col_offset; + UNUSED(start_col_offset); // Only used by EXTRA macro + { // '{' expression for_if_clauses '}' + expr_ty a; + asdl_seq* b; + void *literal; + void *literal_1; + if ( + (literal = _PyPegen_expect_token(p, 25)) + && + (a = expression_rule(p)) + && + (b = for_if_clauses_rule(p)) + && + (literal_1 = _PyPegen_expect_token(p, 26)) + ) + { + Token *token = _PyPegen_get_last_nonnwhitespace_token(p); + if (token == NULL) { + return NULL; + } + int end_lineno = token->end_lineno; + UNUSED(end_lineno); // Only used by EXTRA macro + int end_col_offset = token->end_col_offset; + UNUSED(end_col_offset); // Only used by EXTRA macro + res = _Py_SetComp ( a , b , EXTRA ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + { // invalid_comprehension + void *invalid_comprehension_var; + if ( + (invalid_comprehension_var = invalid_comprehension_rule(p)) + ) + { + res = invalid_comprehension_var; + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// dict: '{' kvpairs? '}' +static expr_ty +dict_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + expr_ty res = NULL; + int mark = p->mark; + if (p->mark == p->fill && _PyPegen_fill_token(p) < 0) { + p->error_indicator = 1; + return NULL; + } + int start_lineno = p->tokens[mark]->lineno; + UNUSED(start_lineno); // Only used by EXTRA macro + int start_col_offset = p->tokens[mark]->col_offset; + UNUSED(start_col_offset); // Only used by EXTRA macro + { // '{' kvpairs? '}' + void *a; + void *literal; + void *literal_1; + if ( + (literal = _PyPegen_expect_token(p, 25)) + && + (a = kvpairs_rule(p), 1) + && + (literal_1 = _PyPegen_expect_token(p, 26)) + ) + { + Token *token = _PyPegen_get_last_nonnwhitespace_token(p); + if (token == NULL) { + return NULL; + } + int end_lineno = token->end_lineno; + UNUSED(end_lineno); // Only used by EXTRA macro + int end_col_offset = token->end_col_offset; + UNUSED(end_col_offset); // Only used by EXTRA macro + res = _Py_Dict ( CHECK ( _PyPegen_get_keys ( p , a ) ) , CHECK ( _PyPegen_get_values ( p , a ) ) , EXTRA ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// dictcomp: '{' kvpair for_if_clauses '}' +static expr_ty +dictcomp_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + expr_ty res = NULL; + int mark = p->mark; + if (p->mark == p->fill && _PyPegen_fill_token(p) < 0) { + p->error_indicator = 1; + return NULL; + } + int start_lineno = p->tokens[mark]->lineno; + UNUSED(start_lineno); // Only used by EXTRA macro + int start_col_offset = p->tokens[mark]->col_offset; + UNUSED(start_col_offset); // Only used by EXTRA macro + { // '{' kvpair for_if_clauses '}' + KeyValuePair* a; + asdl_seq* b; + void *literal; + void *literal_1; + if ( + (literal = _PyPegen_expect_token(p, 25)) + && + (a = kvpair_rule(p)) + && + (b = for_if_clauses_rule(p)) + && + (literal_1 = _PyPegen_expect_token(p, 26)) + ) + { + Token *token = _PyPegen_get_last_nonnwhitespace_token(p); + if (token == NULL) { + return NULL; + } + int end_lineno = token->end_lineno; + UNUSED(end_lineno); // Only used by EXTRA macro + int end_col_offset = token->end_col_offset; + UNUSED(end_col_offset); // Only used by EXTRA macro + res = _Py_DictComp ( a -> key , a -> value , b , EXTRA ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// kvpairs: ','.kvpair+ ','? +static asdl_seq* +kvpairs_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + asdl_seq* res = NULL; + int mark = p->mark; + { // ','.kvpair+ ','? + asdl_seq * a; + void *opt_var; + UNUSED(opt_var); // Silence compiler warnings + if ( + (a = _gather_95_rule(p)) + && + (opt_var = _PyPegen_expect_token(p, 12), 1) + ) + { + res = a; + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// kvpair: '**' bitwise_or | expression ':' expression +static KeyValuePair* +kvpair_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + KeyValuePair* res = NULL; + int mark = p->mark; + { // '**' bitwise_or + expr_ty a; + void *literal; + if ( + (literal = _PyPegen_expect_token(p, 35)) + && + (a = bitwise_or_rule(p)) + ) + { + res = _PyPegen_key_value_pair ( p , NULL , a ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + { // expression ':' expression + expr_ty a; + expr_ty b; + void *literal; + if ( + (a = expression_rule(p)) + && + (literal = _PyPegen_expect_token(p, 11)) + && + (b = expression_rule(p)) + ) + { + res = _PyPegen_key_value_pair ( p , a , b ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// for_if_clauses: ((ASYNC? 'for' star_targets 'in' disjunction (('if' disjunction))*))+ +static asdl_seq* +for_if_clauses_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + asdl_seq* res = NULL; + int mark = p->mark; + { // ((ASYNC? 'for' star_targets 'in' disjunction (('if' disjunction))*))+ + asdl_seq * a; + if ( + (a = _loop1_97_rule(p)) + ) + { + res = a; + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// yield_expr: 'yield' 'from' expression | 'yield' star_expressions? +static expr_ty +yield_expr_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + expr_ty res = NULL; + int mark = p->mark; + if (p->mark == p->fill && _PyPegen_fill_token(p) < 0) { + p->error_indicator = 1; + return NULL; + } + int start_lineno = p->tokens[mark]->lineno; + UNUSED(start_lineno); // Only used by EXTRA macro + int start_col_offset = p->tokens[mark]->col_offset; + UNUSED(start_col_offset); // Only used by EXTRA macro + { // 'yield' 'from' expression + expr_ty a; + void *keyword; + void *keyword_1; + if ( + (keyword = _PyPegen_expect_token(p, 504)) + && + (keyword_1 = _PyPegen_expect_token(p, 514)) + && + (a = expression_rule(p)) + ) + { + Token *token = _PyPegen_get_last_nonnwhitespace_token(p); + if (token == NULL) { + return NULL; + } + int end_lineno = token->end_lineno; + UNUSED(end_lineno); // Only used by EXTRA macro + int end_col_offset = token->end_col_offset; + UNUSED(end_col_offset); // Only used by EXTRA macro + res = _Py_YieldFrom ( a , EXTRA ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + { // 'yield' star_expressions? + void *a; + void *keyword; + if ( + (keyword = _PyPegen_expect_token(p, 504)) + && + (a = star_expressions_rule(p), 1) + ) + { + Token *token = _PyPegen_get_last_nonnwhitespace_token(p); + if (token == NULL) { + return NULL; + } + int end_lineno = token->end_lineno; + UNUSED(end_lineno); // Only used by EXTRA macro + int end_col_offset = token->end_col_offset; + UNUSED(end_col_offset); // Only used by EXTRA macro + res = _Py_Yield ( a , EXTRA ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// arguments: args ','? &')' | incorrect_arguments +static expr_ty +arguments_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + expr_ty res = NULL; + if (_PyPegen_is_memoized(p, arguments_type, &res)) + return res; + int mark = p->mark; + { // args ','? &')' + expr_ty a; + void *opt_var; + UNUSED(opt_var); // Silence compiler warnings + if ( + (a = args_rule(p)) + && + (opt_var = _PyPegen_expect_token(p, 12), 1) + && + _PyPegen_lookahead_with_int(1, _PyPegen_expect_token, p, 8) + ) + { + res = a; + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + { // incorrect_arguments + void *incorrect_arguments_var; + if ( + (incorrect_arguments_var = incorrect_arguments_rule(p)) + ) + { + res = incorrect_arguments_var; + goto done; + } + p->mark = mark; + } + res = NULL; + done: + _PyPegen_insert_memo(p, mark, arguments_type, res); + return res; +} + +// args: starred_expression [',' args] | kwargs | named_expression [',' args] +static expr_ty +args_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + expr_ty res = NULL; + int mark = p->mark; + if (p->mark == p->fill && _PyPegen_fill_token(p) < 0) { + p->error_indicator = 1; + return NULL; + } + int start_lineno = p->tokens[mark]->lineno; + UNUSED(start_lineno); // Only used by EXTRA macro + int start_col_offset = p->tokens[mark]->col_offset; + UNUSED(start_col_offset); // Only used by EXTRA macro + { // starred_expression [',' args] + expr_ty a; + void *b; + if ( + (a = starred_expression_rule(p)) + && + (b = _tmp_98_rule(p), 1) + ) + { + Token *token = _PyPegen_get_last_nonnwhitespace_token(p); + if (token == NULL) { + return NULL; + } + int end_lineno = token->end_lineno; + UNUSED(end_lineno); // Only used by EXTRA macro + int end_col_offset = token->end_col_offset; + UNUSED(end_col_offset); // Only used by EXTRA macro + res = _Py_Call ( _PyPegen_dummy_name ( p ) , ( b ) ? CHECK ( _PyPegen_seq_insert_in_front ( p , a , ( ( expr_ty ) b ) -> v . Call . args ) ) : CHECK ( _PyPegen_singleton_seq ( p , a ) ) , ( b ) ? ( ( expr_ty ) b ) -> v . Call . keywords : NULL , EXTRA ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + { // kwargs + asdl_seq* a; + if ( + (a = kwargs_rule(p)) + ) + { + Token *token = _PyPegen_get_last_nonnwhitespace_token(p); + if (token == NULL) { + return NULL; + } + int end_lineno = token->end_lineno; + UNUSED(end_lineno); // Only used by EXTRA macro + int end_col_offset = token->end_col_offset; + UNUSED(end_col_offset); // Only used by EXTRA macro + res = _Py_Call ( _PyPegen_dummy_name ( p ) , CHECK_NULL_ALLOWED ( _PyPegen_seq_extract_starred_exprs ( p , a ) ) , CHECK_NULL_ALLOWED ( _PyPegen_seq_delete_starred_exprs ( p , a ) ) , EXTRA ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + { // named_expression [',' args] + expr_ty a; + void *b; + if ( + (a = named_expression_rule(p)) + && + (b = _tmp_99_rule(p), 1) + ) + { + Token *token = _PyPegen_get_last_nonnwhitespace_token(p); + if (token == NULL) { + return NULL; + } + int end_lineno = token->end_lineno; + UNUSED(end_lineno); // Only used by EXTRA macro + int end_col_offset = token->end_col_offset; + UNUSED(end_col_offset); // Only used by EXTRA macro + res = _Py_Call ( _PyPegen_dummy_name ( p ) , ( b ) ? CHECK ( _PyPegen_seq_insert_in_front ( p , a , ( ( expr_ty ) b ) -> v . Call . args ) ) : CHECK ( _PyPegen_singleton_seq ( p , a ) ) , ( b ) ? ( ( expr_ty ) b ) -> v . Call . keywords : NULL , EXTRA ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// kwargs: +// | ','.kwarg_or_starred+ ',' ','.kwarg_or_double_starred+ +// | ','.kwarg_or_starred+ +// | ','.kwarg_or_double_starred+ +static asdl_seq* +kwargs_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + asdl_seq* res = NULL; + int mark = p->mark; + { // ','.kwarg_or_starred+ ',' ','.kwarg_or_double_starred+ + asdl_seq * a; + asdl_seq * b; + void *literal; + if ( + (a = _gather_100_rule(p)) + && + (literal = _PyPegen_expect_token(p, 12)) + && + (b = _gather_102_rule(p)) + ) + { + res = _PyPegen_join_sequences ( p , a , b ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + { // ','.kwarg_or_starred+ + asdl_seq * _gather_104_var; + if ( + (_gather_104_var = _gather_104_rule(p)) + ) + { + res = _gather_104_var; + goto done; + } + p->mark = mark; + } + { // ','.kwarg_or_double_starred+ + asdl_seq * _gather_106_var; + if ( + (_gather_106_var = _gather_106_rule(p)) + ) + { + res = _gather_106_var; + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// starred_expression: '*' expression +static expr_ty +starred_expression_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + expr_ty res = NULL; + int mark = p->mark; + if (p->mark == p->fill && _PyPegen_fill_token(p) < 0) { + p->error_indicator = 1; + return NULL; + } + int start_lineno = p->tokens[mark]->lineno; + UNUSED(start_lineno); // Only used by EXTRA macro + int start_col_offset = p->tokens[mark]->col_offset; + UNUSED(start_col_offset); // Only used by EXTRA macro + { // '*' expression + expr_ty a; + void *literal; + if ( + (literal = _PyPegen_expect_token(p, 16)) + && + (a = expression_rule(p)) + ) + { + Token *token = _PyPegen_get_last_nonnwhitespace_token(p); + if (token == NULL) { + return NULL; + } + int end_lineno = token->end_lineno; + UNUSED(end_lineno); // Only used by EXTRA macro + int end_col_offset = token->end_col_offset; + UNUSED(end_col_offset); // Only used by EXTRA macro + res = _Py_Starred ( a , Load , EXTRA ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// kwarg_or_starred: NAME '=' expression | starred_expression +static KeywordOrStarred* +kwarg_or_starred_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + KeywordOrStarred* res = NULL; + int mark = p->mark; + if (p->mark == p->fill && _PyPegen_fill_token(p) < 0) { + p->error_indicator = 1; + return NULL; + } + int start_lineno = p->tokens[mark]->lineno; + UNUSED(start_lineno); // Only used by EXTRA macro + int start_col_offset = p->tokens[mark]->col_offset; + UNUSED(start_col_offset); // Only used by EXTRA macro + { // NAME '=' expression + expr_ty a; + expr_ty b; + void *literal; + if ( + (a = _PyPegen_name_token(p)) + && + (literal = _PyPegen_expect_token(p, 22)) + && + (b = expression_rule(p)) + ) + { + Token *token = _PyPegen_get_last_nonnwhitespace_token(p); + if (token == NULL) { + return NULL; + } + int end_lineno = token->end_lineno; + UNUSED(end_lineno); // Only used by EXTRA macro + int end_col_offset = token->end_col_offset; + UNUSED(end_col_offset); // Only used by EXTRA macro + res = _PyPegen_keyword_or_starred ( p , CHECK ( _Py_keyword ( a -> v . Name . id , b , EXTRA ) ) , 1 ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + { // starred_expression + expr_ty a; + if ( + (a = starred_expression_rule(p)) + ) + { + res = _PyPegen_keyword_or_starred ( p , a , 0 ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// kwarg_or_double_starred: NAME '=' expression | '**' expression +static KeywordOrStarred* +kwarg_or_double_starred_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + KeywordOrStarred* res = NULL; + int mark = p->mark; + if (p->mark == p->fill && _PyPegen_fill_token(p) < 0) { + p->error_indicator = 1; + return NULL; + } + int start_lineno = p->tokens[mark]->lineno; + UNUSED(start_lineno); // Only used by EXTRA macro + int start_col_offset = p->tokens[mark]->col_offset; + UNUSED(start_col_offset); // Only used by EXTRA macro + { // NAME '=' expression + expr_ty a; + expr_ty b; + void *literal; + if ( + (a = _PyPegen_name_token(p)) + && + (literal = _PyPegen_expect_token(p, 22)) + && + (b = expression_rule(p)) + ) + { + Token *token = _PyPegen_get_last_nonnwhitespace_token(p); + if (token == NULL) { + return NULL; + } + int end_lineno = token->end_lineno; + UNUSED(end_lineno); // Only used by EXTRA macro + int end_col_offset = token->end_col_offset; + UNUSED(end_col_offset); // Only used by EXTRA macro + res = _PyPegen_keyword_or_starred ( p , CHECK ( _Py_keyword ( a -> v . Name . id , b , EXTRA ) ) , 1 ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + { // '**' expression + expr_ty a; + void *literal; + if ( + (literal = _PyPegen_expect_token(p, 35)) + && + (a = expression_rule(p)) + ) + { + Token *token = _PyPegen_get_last_nonnwhitespace_token(p); + if (token == NULL) { + return NULL; + } + int end_lineno = token->end_lineno; + UNUSED(end_lineno); // Only used by EXTRA macro + int end_col_offset = token->end_col_offset; + UNUSED(end_col_offset); // Only used by EXTRA macro + res = _PyPegen_keyword_or_starred ( p , CHECK ( _Py_keyword ( NULL , a , EXTRA ) ) , 1 ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// star_targets: star_target !',' | star_target ((',' star_target))* ','? +static expr_ty +star_targets_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + expr_ty res = NULL; + int mark = p->mark; + if (p->mark == p->fill && _PyPegen_fill_token(p) < 0) { + p->error_indicator = 1; + return NULL; + } + int start_lineno = p->tokens[mark]->lineno; + UNUSED(start_lineno); // Only used by EXTRA macro + int start_col_offset = p->tokens[mark]->col_offset; + UNUSED(start_col_offset); // Only used by EXTRA macro + { // star_target !',' + expr_ty a; + if ( + (a = star_target_rule(p)) + && + _PyPegen_lookahead_with_int(0, _PyPegen_expect_token, p, 12) + ) + { + res = a; + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + { // star_target ((',' star_target))* ','? + expr_ty a; + asdl_seq * b; + void *opt_var; + UNUSED(opt_var); // Silence compiler warnings + if ( + (a = star_target_rule(p)) + && + (b = _loop0_108_rule(p)) + && + (opt_var = _PyPegen_expect_token(p, 12), 1) + ) + { + Token *token = _PyPegen_get_last_nonnwhitespace_token(p); + if (token == NULL) { + return NULL; + } + int end_lineno = token->end_lineno; + UNUSED(end_lineno); // Only used by EXTRA macro + int end_col_offset = token->end_col_offset; + UNUSED(end_col_offset); // Only used by EXTRA macro + res = _Py_Tuple ( CHECK ( _PyPegen_seq_insert_in_front ( p , a , b ) ) , Store , EXTRA ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// star_targets_seq: ','.star_target+ ','? +static asdl_seq* +star_targets_seq_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + asdl_seq* res = NULL; + int mark = p->mark; + { // ','.star_target+ ','? + asdl_seq * a; + void *opt_var; + UNUSED(opt_var); // Silence compiler warnings + if ( + (a = _gather_109_rule(p)) + && + (opt_var = _PyPegen_expect_token(p, 12), 1) + ) + { + res = a; + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// star_target: +// | '*' (!'*' star_target) +// | t_primary '.' NAME !t_lookahead +// | t_primary '[' slices ']' !t_lookahead +// | star_atom +static expr_ty +star_target_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + expr_ty res = NULL; + if (_PyPegen_is_memoized(p, star_target_type, &res)) + return res; + int mark = p->mark; + if (p->mark == p->fill && _PyPegen_fill_token(p) < 0) { + p->error_indicator = 1; + return NULL; + } + int start_lineno = p->tokens[mark]->lineno; + UNUSED(start_lineno); // Only used by EXTRA macro + int start_col_offset = p->tokens[mark]->col_offset; + UNUSED(start_col_offset); // Only used by EXTRA macro + { // '*' (!'*' star_target) + void *a; + void *literal; + if ( + (literal = _PyPegen_expect_token(p, 16)) + && + (a = _tmp_111_rule(p)) + ) + { + Token *token = _PyPegen_get_last_nonnwhitespace_token(p); + if (token == NULL) { + return NULL; + } + int end_lineno = token->end_lineno; + UNUSED(end_lineno); // Only used by EXTRA macro + int end_col_offset = token->end_col_offset; + UNUSED(end_col_offset); // Only used by EXTRA macro + res = _Py_Starred ( CHECK ( _PyPegen_set_expr_context ( p , a , Store ) ) , Store , EXTRA ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + { // t_primary '.' NAME !t_lookahead + expr_ty a; + expr_ty b; + void *literal; + if ( + (a = t_primary_rule(p)) + && + (literal = _PyPegen_expect_token(p, 23)) + && + (b = _PyPegen_name_token(p)) + && + _PyPegen_lookahead(0, t_lookahead_rule, p) + ) + { + Token *token = _PyPegen_get_last_nonnwhitespace_token(p); + if (token == NULL) { + return NULL; + } + int end_lineno = token->end_lineno; + UNUSED(end_lineno); // Only used by EXTRA macro + int end_col_offset = token->end_col_offset; + UNUSED(end_col_offset); // Only used by EXTRA macro + res = _Py_Attribute ( a , b -> v . Name . id , Store , EXTRA ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + { // t_primary '[' slices ']' !t_lookahead + expr_ty a; + expr_ty b; + void *literal; + void *literal_1; + if ( + (a = t_primary_rule(p)) + && + (literal = _PyPegen_expect_token(p, 9)) + && + (b = slices_rule(p)) + && + (literal_1 = _PyPegen_expect_token(p, 10)) + && + _PyPegen_lookahead(0, t_lookahead_rule, p) + ) + { + Token *token = _PyPegen_get_last_nonnwhitespace_token(p); + if (token == NULL) { + return NULL; + } + int end_lineno = token->end_lineno; + UNUSED(end_lineno); // Only used by EXTRA macro + int end_col_offset = token->end_col_offset; + UNUSED(end_col_offset); // Only used by EXTRA macro + res = _Py_Subscript ( a , b , Store , EXTRA ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + { // star_atom + expr_ty star_atom_var; + if ( + (star_atom_var = star_atom_rule(p)) + ) + { + res = star_atom_var; + goto done; + } + p->mark = mark; + } + res = NULL; + done: + _PyPegen_insert_memo(p, mark, star_target_type, res); + return res; +} + +// star_atom: +// | NAME +// | '(' star_target ')' +// | '(' star_targets_seq? ')' +// | '[' star_targets_seq? ']' +static expr_ty +star_atom_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + expr_ty res = NULL; + int mark = p->mark; + if (p->mark == p->fill && _PyPegen_fill_token(p) < 0) { + p->error_indicator = 1; + return NULL; + } + int start_lineno = p->tokens[mark]->lineno; + UNUSED(start_lineno); // Only used by EXTRA macro + int start_col_offset = p->tokens[mark]->col_offset; + UNUSED(start_col_offset); // Only used by EXTRA macro + { // NAME + expr_ty a; + if ( + (a = _PyPegen_name_token(p)) + ) + { + res = _PyPegen_set_expr_context ( p , a , Store ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + { // '(' star_target ')' + expr_ty a; + void *literal; + void *literal_1; + if ( + (literal = _PyPegen_expect_token(p, 7)) + && + (a = star_target_rule(p)) + && + (literal_1 = _PyPegen_expect_token(p, 8)) + ) + { + res = _PyPegen_set_expr_context ( p , a , Store ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + { // '(' star_targets_seq? ')' + void *a; + void *literal; + void *literal_1; + if ( + (literal = _PyPegen_expect_token(p, 7)) + && + (a = star_targets_seq_rule(p), 1) + && + (literal_1 = _PyPegen_expect_token(p, 8)) + ) + { + Token *token = _PyPegen_get_last_nonnwhitespace_token(p); + if (token == NULL) { + return NULL; + } + int end_lineno = token->end_lineno; + UNUSED(end_lineno); // Only used by EXTRA macro + int end_col_offset = token->end_col_offset; + UNUSED(end_col_offset); // Only used by EXTRA macro + res = _Py_Tuple ( a , Store , EXTRA ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + { // '[' star_targets_seq? ']' + void *a; + void *literal; + void *literal_1; + if ( + (literal = _PyPegen_expect_token(p, 9)) + && + (a = star_targets_seq_rule(p), 1) + && + (literal_1 = _PyPegen_expect_token(p, 10)) + ) + { + Token *token = _PyPegen_get_last_nonnwhitespace_token(p); + if (token == NULL) { + return NULL; + } + int end_lineno = token->end_lineno; + UNUSED(end_lineno); // Only used by EXTRA macro + int end_col_offset = token->end_col_offset; + UNUSED(end_col_offset); // Only used by EXTRA macro + res = _Py_List ( a , Store , EXTRA ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// inside_paren_ann_assign_target: +// | ann_assign_subscript_attribute_target +// | NAME +// | '(' inside_paren_ann_assign_target ')' +static expr_ty +inside_paren_ann_assign_target_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + expr_ty res = NULL; + int mark = p->mark; + { // ann_assign_subscript_attribute_target + expr_ty ann_assign_subscript_attribute_target_var; + if ( + (ann_assign_subscript_attribute_target_var = ann_assign_subscript_attribute_target_rule(p)) + ) + { + res = ann_assign_subscript_attribute_target_var; + goto done; + } + p->mark = mark; + } + { // NAME + expr_ty a; + if ( + (a = _PyPegen_name_token(p)) + ) + { + res = _PyPegen_set_expr_context ( p , a , Store ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + { // '(' inside_paren_ann_assign_target ')' + expr_ty a; + void *literal; + void *literal_1; + if ( + (literal = _PyPegen_expect_token(p, 7)) + && + (a = inside_paren_ann_assign_target_rule(p)) + && + (literal_1 = _PyPegen_expect_token(p, 8)) + ) + { + res = a; + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// ann_assign_subscript_attribute_target: +// | t_primary '.' NAME !t_lookahead +// | t_primary '[' slices ']' !t_lookahead +static expr_ty +ann_assign_subscript_attribute_target_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + expr_ty res = NULL; + int mark = p->mark; + if (p->mark == p->fill && _PyPegen_fill_token(p) < 0) { + p->error_indicator = 1; + return NULL; + } + int start_lineno = p->tokens[mark]->lineno; + UNUSED(start_lineno); // Only used by EXTRA macro + int start_col_offset = p->tokens[mark]->col_offset; + UNUSED(start_col_offset); // Only used by EXTRA macro + { // t_primary '.' NAME !t_lookahead + expr_ty a; + expr_ty b; + void *literal; + if ( + (a = t_primary_rule(p)) + && + (literal = _PyPegen_expect_token(p, 23)) + && + (b = _PyPegen_name_token(p)) + && + _PyPegen_lookahead(0, t_lookahead_rule, p) + ) + { + Token *token = _PyPegen_get_last_nonnwhitespace_token(p); + if (token == NULL) { + return NULL; + } + int end_lineno = token->end_lineno; + UNUSED(end_lineno); // Only used by EXTRA macro + int end_col_offset = token->end_col_offset; + UNUSED(end_col_offset); // Only used by EXTRA macro + res = _Py_Attribute ( a , b -> v . Name . id , Store , EXTRA ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + { // t_primary '[' slices ']' !t_lookahead + expr_ty a; + expr_ty b; + void *literal; + void *literal_1; + if ( + (a = t_primary_rule(p)) + && + (literal = _PyPegen_expect_token(p, 9)) + && + (b = slices_rule(p)) + && + (literal_1 = _PyPegen_expect_token(p, 10)) + && + _PyPegen_lookahead(0, t_lookahead_rule, p) + ) + { + Token *token = _PyPegen_get_last_nonnwhitespace_token(p); + if (token == NULL) { + return NULL; + } + int end_lineno = token->end_lineno; + UNUSED(end_lineno); // Only used by EXTRA macro + int end_col_offset = token->end_col_offset; + UNUSED(end_col_offset); // Only used by EXTRA macro + res = _Py_Subscript ( a , b , Store , EXTRA ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// del_targets: ','.del_target+ ','? +static asdl_seq* +del_targets_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + asdl_seq* res = NULL; + int mark = p->mark; + { // ','.del_target+ ','? + asdl_seq * a; + void *opt_var; + UNUSED(opt_var); // Silence compiler warnings + if ( + (a = _gather_112_rule(p)) + && + (opt_var = _PyPegen_expect_token(p, 12), 1) + ) + { + res = a; + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// del_target: +// | t_primary '.' NAME !t_lookahead +// | t_primary '[' slices ']' !t_lookahead +// | del_t_atom +static expr_ty +del_target_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + expr_ty res = NULL; + if (_PyPegen_is_memoized(p, del_target_type, &res)) + return res; + int mark = p->mark; + if (p->mark == p->fill && _PyPegen_fill_token(p) < 0) { + p->error_indicator = 1; + return NULL; + } + int start_lineno = p->tokens[mark]->lineno; + UNUSED(start_lineno); // Only used by EXTRA macro + int start_col_offset = p->tokens[mark]->col_offset; + UNUSED(start_col_offset); // Only used by EXTRA macro + { // t_primary '.' NAME !t_lookahead + expr_ty a; + expr_ty b; + void *literal; + if ( + (a = t_primary_rule(p)) + && + (literal = _PyPegen_expect_token(p, 23)) + && + (b = _PyPegen_name_token(p)) + && + _PyPegen_lookahead(0, t_lookahead_rule, p) + ) + { + Token *token = _PyPegen_get_last_nonnwhitespace_token(p); + if (token == NULL) { + return NULL; + } + int end_lineno = token->end_lineno; + UNUSED(end_lineno); // Only used by EXTRA macro + int end_col_offset = token->end_col_offset; + UNUSED(end_col_offset); // Only used by EXTRA macro + res = _Py_Attribute ( a , b -> v . Name . id , Del , EXTRA ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + { // t_primary '[' slices ']' !t_lookahead + expr_ty a; + expr_ty b; + void *literal; + void *literal_1; + if ( + (a = t_primary_rule(p)) + && + (literal = _PyPegen_expect_token(p, 9)) + && + (b = slices_rule(p)) + && + (literal_1 = _PyPegen_expect_token(p, 10)) + && + _PyPegen_lookahead(0, t_lookahead_rule, p) + ) + { + Token *token = _PyPegen_get_last_nonnwhitespace_token(p); + if (token == NULL) { + return NULL; + } + int end_lineno = token->end_lineno; + UNUSED(end_lineno); // Only used by EXTRA macro + int end_col_offset = token->end_col_offset; + UNUSED(end_col_offset); // Only used by EXTRA macro + res = _Py_Subscript ( a , b , Del , EXTRA ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + { // del_t_atom + expr_ty del_t_atom_var; + if ( + (del_t_atom_var = del_t_atom_rule(p)) + ) + { + res = del_t_atom_var; + goto done; + } + p->mark = mark; + } + res = NULL; + done: + _PyPegen_insert_memo(p, mark, del_target_type, res); + return res; +} + +// del_t_atom: NAME | '(' del_target ')' | '(' del_targets? ')' | '[' del_targets? ']' +static expr_ty +del_t_atom_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + expr_ty res = NULL; + int mark = p->mark; + if (p->mark == p->fill && _PyPegen_fill_token(p) < 0) { + p->error_indicator = 1; + return NULL; + } + int start_lineno = p->tokens[mark]->lineno; + UNUSED(start_lineno); // Only used by EXTRA macro + int start_col_offset = p->tokens[mark]->col_offset; + UNUSED(start_col_offset); // Only used by EXTRA macro + { // NAME + expr_ty a; + if ( + (a = _PyPegen_name_token(p)) + ) + { + res = _PyPegen_set_expr_context ( p , a , Del ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + { // '(' del_target ')' + expr_ty a; + void *literal; + void *literal_1; + if ( + (literal = _PyPegen_expect_token(p, 7)) + && + (a = del_target_rule(p)) + && + (literal_1 = _PyPegen_expect_token(p, 8)) + ) + { + res = _PyPegen_set_expr_context ( p , a , Del ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + { // '(' del_targets? ')' + void *a; + void *literal; + void *literal_1; + if ( + (literal = _PyPegen_expect_token(p, 7)) + && + (a = del_targets_rule(p), 1) + && + (literal_1 = _PyPegen_expect_token(p, 8)) + ) + { + Token *token = _PyPegen_get_last_nonnwhitespace_token(p); + if (token == NULL) { + return NULL; + } + int end_lineno = token->end_lineno; + UNUSED(end_lineno); // Only used by EXTRA macro + int end_col_offset = token->end_col_offset; + UNUSED(end_col_offset); // Only used by EXTRA macro + res = _Py_Tuple ( a , Del , EXTRA ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + { // '[' del_targets? ']' + void *a; + void *literal; + void *literal_1; + if ( + (literal = _PyPegen_expect_token(p, 9)) + && + (a = del_targets_rule(p), 1) + && + (literal_1 = _PyPegen_expect_token(p, 10)) + ) + { + Token *token = _PyPegen_get_last_nonnwhitespace_token(p); + if (token == NULL) { + return NULL; + } + int end_lineno = token->end_lineno; + UNUSED(end_lineno); // Only used by EXTRA macro + int end_col_offset = token->end_col_offset; + UNUSED(end_col_offset); // Only used by EXTRA macro + res = _Py_List ( a , Del , EXTRA ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// targets: ','.target+ ','? +static asdl_seq* +targets_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + asdl_seq* res = NULL; + int mark = p->mark; + { // ','.target+ ','? + asdl_seq * a; + void *opt_var; + UNUSED(opt_var); // Silence compiler warnings + if ( + (a = _gather_114_rule(p)) + && + (opt_var = _PyPegen_expect_token(p, 12), 1) + ) + { + res = a; + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// target: +// | t_primary '.' NAME !t_lookahead +// | t_primary '[' slices ']' !t_lookahead +// | t_atom +static expr_ty +target_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + expr_ty res = NULL; + if (_PyPegen_is_memoized(p, target_type, &res)) + return res; + int mark = p->mark; + if (p->mark == p->fill && _PyPegen_fill_token(p) < 0) { + p->error_indicator = 1; + return NULL; + } + int start_lineno = p->tokens[mark]->lineno; + UNUSED(start_lineno); // Only used by EXTRA macro + int start_col_offset = p->tokens[mark]->col_offset; + UNUSED(start_col_offset); // Only used by EXTRA macro + { // t_primary '.' NAME !t_lookahead + expr_ty a; + expr_ty b; + void *literal; + if ( + (a = t_primary_rule(p)) + && + (literal = _PyPegen_expect_token(p, 23)) + && + (b = _PyPegen_name_token(p)) + && + _PyPegen_lookahead(0, t_lookahead_rule, p) + ) + { + Token *token = _PyPegen_get_last_nonnwhitespace_token(p); + if (token == NULL) { + return NULL; + } + int end_lineno = token->end_lineno; + UNUSED(end_lineno); // Only used by EXTRA macro + int end_col_offset = token->end_col_offset; + UNUSED(end_col_offset); // Only used by EXTRA macro + res = _Py_Attribute ( a , b -> v . Name . id , Store , EXTRA ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + { // t_primary '[' slices ']' !t_lookahead + expr_ty a; + expr_ty b; + void *literal; + void *literal_1; + if ( + (a = t_primary_rule(p)) + && + (literal = _PyPegen_expect_token(p, 9)) + && + (b = slices_rule(p)) + && + (literal_1 = _PyPegen_expect_token(p, 10)) + && + _PyPegen_lookahead(0, t_lookahead_rule, p) + ) + { + Token *token = _PyPegen_get_last_nonnwhitespace_token(p); + if (token == NULL) { + return NULL; + } + int end_lineno = token->end_lineno; + UNUSED(end_lineno); // Only used by EXTRA macro + int end_col_offset = token->end_col_offset; + UNUSED(end_col_offset); // Only used by EXTRA macro + res = _Py_Subscript ( a , b , Store , EXTRA ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + { // t_atom + expr_ty t_atom_var; + if ( + (t_atom_var = t_atom_rule(p)) + ) + { + res = t_atom_var; + goto done; + } + p->mark = mark; + } + res = NULL; + done: + _PyPegen_insert_memo(p, mark, target_type, res); + return res; +} + +// Left-recursive +// t_primary: +// | t_primary '.' NAME &t_lookahead +// | t_primary '[' slices ']' &t_lookahead +// | t_primary genexp &t_lookahead +// | t_primary '(' arguments? ')' &t_lookahead +// | atom &t_lookahead +static expr_ty t_primary_raw(Parser *); +static expr_ty +t_primary_rule(Parser *p) +{ + expr_ty res = NULL; + if (_PyPegen_is_memoized(p, t_primary_type, &res)) + return res; + int mark = p->mark; + int resmark = p->mark; + while (1) { + int tmpvar_8 = _PyPegen_update_memo(p, mark, t_primary_type, res); + if (tmpvar_8) { + return res; + } + p->mark = mark; + void *raw = t_primary_raw(p); + if (raw == NULL || p->mark <= resmark) + break; + resmark = p->mark; + res = raw; + } + p->mark = resmark; + return res; +} +static expr_ty +t_primary_raw(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + expr_ty res = NULL; + int mark = p->mark; + if (p->mark == p->fill && _PyPegen_fill_token(p) < 0) { + p->error_indicator = 1; + return NULL; + } + int start_lineno = p->tokens[mark]->lineno; + UNUSED(start_lineno); // Only used by EXTRA macro + int start_col_offset = p->tokens[mark]->col_offset; + UNUSED(start_col_offset); // Only used by EXTRA macro + { // t_primary '.' NAME &t_lookahead + expr_ty a; + expr_ty b; + void *literal; + if ( + (a = t_primary_rule(p)) + && + (literal = _PyPegen_expect_token(p, 23)) + && + (b = _PyPegen_name_token(p)) + && + _PyPegen_lookahead(1, t_lookahead_rule, p) + ) + { + Token *token = _PyPegen_get_last_nonnwhitespace_token(p); + if (token == NULL) { + return NULL; + } + int end_lineno = token->end_lineno; + UNUSED(end_lineno); // Only used by EXTRA macro + int end_col_offset = token->end_col_offset; + UNUSED(end_col_offset); // Only used by EXTRA macro + res = _Py_Attribute ( a , b -> v . Name . id , Load , EXTRA ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + { // t_primary '[' slices ']' &t_lookahead + expr_ty a; + expr_ty b; + void *literal; + void *literal_1; + if ( + (a = t_primary_rule(p)) + && + (literal = _PyPegen_expect_token(p, 9)) + && + (b = slices_rule(p)) + && + (literal_1 = _PyPegen_expect_token(p, 10)) + && + _PyPegen_lookahead(1, t_lookahead_rule, p) + ) + { + Token *token = _PyPegen_get_last_nonnwhitespace_token(p); + if (token == NULL) { + return NULL; + } + int end_lineno = token->end_lineno; + UNUSED(end_lineno); // Only used by EXTRA macro + int end_col_offset = token->end_col_offset; + UNUSED(end_col_offset); // Only used by EXTRA macro + res = _Py_Subscript ( a , b , Load , EXTRA ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + { // t_primary genexp &t_lookahead + expr_ty a; + expr_ty b; + if ( + (a = t_primary_rule(p)) + && + (b = genexp_rule(p)) + && + _PyPegen_lookahead(1, t_lookahead_rule, p) + ) + { + Token *token = _PyPegen_get_last_nonnwhitespace_token(p); + if (token == NULL) { + return NULL; + } + int end_lineno = token->end_lineno; + UNUSED(end_lineno); // Only used by EXTRA macro + int end_col_offset = token->end_col_offset; + UNUSED(end_col_offset); // Only used by EXTRA macro + res = _Py_Call ( a , CHECK ( _PyPegen_singleton_seq ( p , b ) ) , NULL , EXTRA ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + { // t_primary '(' arguments? ')' &t_lookahead + expr_ty a; + void *b; + void *literal; + void *literal_1; + if ( + (a = t_primary_rule(p)) + && + (literal = _PyPegen_expect_token(p, 7)) + && + (b = arguments_rule(p), 1) + && + (literal_1 = _PyPegen_expect_token(p, 8)) + && + _PyPegen_lookahead(1, t_lookahead_rule, p) + ) + { + Token *token = _PyPegen_get_last_nonnwhitespace_token(p); + if (token == NULL) { + return NULL; + } + int end_lineno = token->end_lineno; + UNUSED(end_lineno); // Only used by EXTRA macro + int end_col_offset = token->end_col_offset; + UNUSED(end_col_offset); // Only used by EXTRA macro + res = _Py_Call ( a , ( b ) ? ( ( expr_ty ) b ) -> v . Call . args : NULL , ( b ) ? ( ( expr_ty ) b ) -> v . Call . keywords : NULL , EXTRA ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + { // atom &t_lookahead + expr_ty a; + if ( + (a = atom_rule(p)) + && + _PyPegen_lookahead(1, t_lookahead_rule, p) + ) + { + res = a; + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// t_lookahead: '(' | '[' | '.' +static void * +t_lookahead_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + void * res = NULL; + int mark = p->mark; + { // '(' + void *literal; + if ( + (literal = _PyPegen_expect_token(p, 7)) + ) + { + res = literal; + goto done; + } + p->mark = mark; + } + { // '[' + void *literal; + if ( + (literal = _PyPegen_expect_token(p, 9)) + ) + { + res = literal; + goto done; + } + p->mark = mark; + } + { // '.' + void *literal; + if ( + (literal = _PyPegen_expect_token(p, 23)) + ) + { + res = literal; + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// t_atom: NAME | '(' target ')' | '(' targets? ')' | '[' targets? ']' +static expr_ty +t_atom_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + expr_ty res = NULL; + int mark = p->mark; + if (p->mark == p->fill && _PyPegen_fill_token(p) < 0) { + p->error_indicator = 1; + return NULL; + } + int start_lineno = p->tokens[mark]->lineno; + UNUSED(start_lineno); // Only used by EXTRA macro + int start_col_offset = p->tokens[mark]->col_offset; + UNUSED(start_col_offset); // Only used by EXTRA macro + { // NAME + expr_ty a; + if ( + (a = _PyPegen_name_token(p)) + ) + { + res = _PyPegen_set_expr_context ( p , a , Store ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + { // '(' target ')' + expr_ty a; + void *literal; + void *literal_1; + if ( + (literal = _PyPegen_expect_token(p, 7)) + && + (a = target_rule(p)) + && + (literal_1 = _PyPegen_expect_token(p, 8)) + ) + { + res = _PyPegen_set_expr_context ( p , a , Store ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + { // '(' targets? ')' + void *b; + void *literal; + void *literal_1; + if ( + (literal = _PyPegen_expect_token(p, 7)) + && + (b = targets_rule(p), 1) + && + (literal_1 = _PyPegen_expect_token(p, 8)) + ) + { + Token *token = _PyPegen_get_last_nonnwhitespace_token(p); + if (token == NULL) { + return NULL; + } + int end_lineno = token->end_lineno; + UNUSED(end_lineno); // Only used by EXTRA macro + int end_col_offset = token->end_col_offset; + UNUSED(end_col_offset); // Only used by EXTRA macro + res = _Py_Tuple ( b , Store , EXTRA ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + { // '[' targets? ']' + void *b; + void *literal; + void *literal_1; + if ( + (literal = _PyPegen_expect_token(p, 9)) + && + (b = targets_rule(p), 1) + && + (literal_1 = _PyPegen_expect_token(p, 10)) + ) + { + Token *token = _PyPegen_get_last_nonnwhitespace_token(p); + if (token == NULL) { + return NULL; + } + int end_lineno = token->end_lineno; + UNUSED(end_lineno); // Only used by EXTRA macro + int end_col_offset = token->end_col_offset; + UNUSED(end_col_offset); // Only used by EXTRA macro + res = _Py_List ( b , Store , EXTRA ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// incorrect_arguments: +// | args ',' '*' +// | expression for_if_clauses ',' [args | expression for_if_clauses] +// | args ',' args +static void * +incorrect_arguments_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + void * res = NULL; + int mark = p->mark; + { // args ',' '*' + expr_ty args_var; + void *literal; + void *literal_1; + if ( + (args_var = args_rule(p)) + && + (literal = _PyPegen_expect_token(p, 12)) + && + (literal_1 = _PyPegen_expect_token(p, 16)) + ) + { + res = RAISE_SYNTAX_ERROR ( "iterable argument unpacking follows keyword argument unpacking" ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + { // expression for_if_clauses ',' [args | expression for_if_clauses] + expr_ty expression_var; + asdl_seq* for_if_clauses_var; + void *literal; + void *opt_var; + UNUSED(opt_var); // Silence compiler warnings + if ( + (expression_var = expression_rule(p)) + && + (for_if_clauses_var = for_if_clauses_rule(p)) + && + (literal = _PyPegen_expect_token(p, 12)) + && + (opt_var = _tmp_116_rule(p), 1) + ) + { + res = RAISE_SYNTAX_ERROR ( "Generator expression must be parenthesized" ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + { // args ',' args + expr_ty a; + expr_ty args_var; + void *literal; + if ( + (a = args_rule(p)) + && + (literal = _PyPegen_expect_token(p, 12)) + && + (args_var = args_rule(p)) + ) + { + res = _PyPegen_arguments_parsing_error ( p , a ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// invalid_named_expression: expression ':=' expression +static void * +invalid_named_expression_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + void * res = NULL; + int mark = p->mark; + { // expression ':=' expression + expr_ty a; + expr_ty expression_var; + void *literal; + if ( + (a = expression_rule(p)) + && + (literal = _PyPegen_expect_token(p, 53)) + && + (expression_var = expression_rule(p)) + ) + { + res = RAISE_SYNTAX_ERROR ( "cannot use assignment expressions with %s" , _PyPegen_get_expr_name ( a ) ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// invalid_assignment: +// | list ':' +// | tuple ':' +// | expression ':' expression ['=' annotated_rhs] +// | expression ('=' | augassign) (yield_expr | star_expressions) +static void * +invalid_assignment_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + void * res = NULL; + int mark = p->mark; + { // list ':' + expr_ty list_var; + void *literal; + if ( + (list_var = list_rule(p)) + && + (literal = _PyPegen_expect_token(p, 11)) + ) + { + res = RAISE_SYNTAX_ERROR ( "only single target (not list) can be annotated" ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + { // tuple ':' + void *literal; + expr_ty tuple_var; + if ( + (tuple_var = tuple_rule(p)) + && + (literal = _PyPegen_expect_token(p, 11)) + ) + { + res = RAISE_SYNTAX_ERROR ( "only single target (not tuple) can be annotated" ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + { // expression ':' expression ['=' annotated_rhs] + expr_ty expression_var; + expr_ty expression_var_1; + void *literal; + void *opt_var; + UNUSED(opt_var); // Silence compiler warnings + if ( + (expression_var = expression_rule(p)) + && + (literal = _PyPegen_expect_token(p, 11)) + && + (expression_var_1 = expression_rule(p)) + && + (opt_var = _tmp_117_rule(p), 1) + ) + { + res = RAISE_SYNTAX_ERROR ( "illegal target for annotation" ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + { // expression ('=' | augassign) (yield_expr | star_expressions) + void *_tmp_118_var; + void *_tmp_119_var; + expr_ty a; + if ( + (a = expression_rule(p)) + && + (_tmp_118_var = _tmp_118_rule(p)) + && + (_tmp_119_var = _tmp_119_rule(p)) + ) + { + res = RAISE_SYNTAX_ERROR ( "cannot assign to %s" , _PyPegen_get_expr_name ( a ) ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// invalid_block: NEWLINE !INDENT +static void * +invalid_block_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + void * res = NULL; + int mark = p->mark; + { // NEWLINE !INDENT + void *newline_var; + if ( + (newline_var = _PyPegen_newline_token(p)) + && + _PyPegen_lookahead(0, _PyPegen_indent_token, p) + ) + { + res = RAISE_INDENTATION_ERROR ( "expected an indented block" ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// invalid_comprehension: ('[' | '(' | '{') '*' expression for_if_clauses +static void * +invalid_comprehension_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + void * res = NULL; + int mark = p->mark; + { // ('[' | '(' | '{') '*' expression for_if_clauses + void *_tmp_120_var; + expr_ty expression_var; + asdl_seq* for_if_clauses_var; + void *literal; + if ( + (_tmp_120_var = _tmp_120_rule(p)) + && + (literal = _PyPegen_expect_token(p, 16)) + && + (expression_var = expression_rule(p)) + && + (for_if_clauses_var = for_if_clauses_rule(p)) + ) + { + res = RAISE_SYNTAX_ERROR ( "iterable unpacking cannot be used in comprehension" ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// invalid_parameters: +// | [plain_names ','] (slash_with_default | names_with_default) ',' plain_names +static void * +invalid_parameters_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + void * res = NULL; + int mark = p->mark; + { // [plain_names ','] (slash_with_default | names_with_default) ',' plain_names + void *_tmp_122_var; + void *literal; + void *opt_var; + UNUSED(opt_var); // Silence compiler warnings + asdl_seq* plain_names_var; + if ( + (opt_var = _tmp_121_rule(p), 1) + && + (_tmp_122_var = _tmp_122_rule(p)) + && + (literal = _PyPegen_expect_token(p, 12)) + && + (plain_names_var = plain_names_rule(p)) + ) + { + res = RAISE_SYNTAX_ERROR ( "non-default argument follows default argument" ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// _loop0_1: NEWLINE +static asdl_seq * +_loop0_1_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + void *res = NULL; + int mark = p->mark; + int start_mark = p->mark; + void **children = PyMem_Malloc(sizeof(void *)); + if (!children) { + PyErr_Format(PyExc_MemoryError, "Parser out of memory"); + return NULL; + } + ssize_t children_capacity = 1; + ssize_t n = 0; + { // NEWLINE + void *newline_var; + while ( + (newline_var = _PyPegen_newline_token(p)) + ) + { + res = newline_var; + if (n == children_capacity) { + children_capacity *= 2; + children = PyMem_Realloc(children, children_capacity*sizeof(void *)); + if (!children) { + PyErr_Format(PyExc_MemoryError, "realloc None"); + return NULL; + } + } + children[n++] = res; + mark = p->mark; + } + p->mark = mark; + } + asdl_seq *seq = _Py_asdl_seq_new(n, p->arena); + if (!seq) { + PyErr_Format(PyExc_MemoryError, "asdl_seq_new _loop0_1"); + PyMem_Free(children); + return NULL; + } + for (int i = 0; i < n; i++) asdl_seq_SET(seq, i, children[i]); + PyMem_Free(children); + _PyPegen_insert_memo(p, start_mark, _loop0_1_type, seq); + return seq; +} + +// _loop1_2: statement +static asdl_seq * +_loop1_2_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + void *res = NULL; + int mark = p->mark; + int start_mark = p->mark; + void **children = PyMem_Malloc(sizeof(void *)); + if (!children) { + PyErr_Format(PyExc_MemoryError, "Parser out of memory"); + return NULL; + } + ssize_t children_capacity = 1; + ssize_t n = 0; + { // statement + asdl_seq* statement_var; + while ( + (statement_var = statement_rule(p)) + ) + { + res = statement_var; + if (n == children_capacity) { + children_capacity *= 2; + children = PyMem_Realloc(children, children_capacity*sizeof(void *)); + if (!children) { + PyErr_Format(PyExc_MemoryError, "realloc None"); + return NULL; + } + } + children[n++] = res; + mark = p->mark; + } + p->mark = mark; + } + if (n == 0) { + PyMem_Free(children); + return NULL; + } + asdl_seq *seq = _Py_asdl_seq_new(n, p->arena); + if (!seq) { + PyErr_Format(PyExc_MemoryError, "asdl_seq_new _loop1_2"); + PyMem_Free(children); + return NULL; + } + for (int i = 0; i < n; i++) asdl_seq_SET(seq, i, children[i]); + PyMem_Free(children); + _PyPegen_insert_memo(p, start_mark, _loop1_2_type, seq); + return seq; +} + +// _loop0_4: ';' small_stmt +static asdl_seq * +_loop0_4_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + void *res = NULL; + int mark = p->mark; + int start_mark = p->mark; + void **children = PyMem_Malloc(sizeof(void *)); + if (!children) { + PyErr_Format(PyExc_MemoryError, "Parser out of memory"); + return NULL; + } + ssize_t children_capacity = 1; + ssize_t n = 0; + { // ';' small_stmt + stmt_ty elem; + void *literal; + while ( + (literal = _PyPegen_expect_token(p, 13)) + && + (elem = small_stmt_rule(p)) + ) + { + res = elem; + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + PyMem_Free(children); + return NULL; + } + if (n == children_capacity) { + children_capacity *= 2; + children = PyMem_Realloc(children, children_capacity*sizeof(void *)); + if (!children) { + PyErr_Format(PyExc_MemoryError, "realloc None"); + return NULL; + } + } + children[n++] = res; + mark = p->mark; + } + p->mark = mark; + } + asdl_seq *seq = _Py_asdl_seq_new(n, p->arena); + if (!seq) { + PyErr_Format(PyExc_MemoryError, "asdl_seq_new _loop0_4"); + PyMem_Free(children); + return NULL; + } + for (int i = 0; i < n; i++) asdl_seq_SET(seq, i, children[i]); + PyMem_Free(children); + _PyPegen_insert_memo(p, start_mark, _loop0_4_type, seq); + return seq; +} + +// _gather_3: small_stmt _loop0_4 +static asdl_seq * +_gather_3_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + asdl_seq * res = NULL; + int mark = p->mark; + { // small_stmt _loop0_4 + stmt_ty elem; + asdl_seq * seq; + if ( + (elem = small_stmt_rule(p)) + && + (seq = _loop0_4_rule(p)) + ) + { + res = _PyPegen_seq_insert_in_front(p, elem, seq); + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// _tmp_5: 'import' | 'from' +static void * +_tmp_5_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + void * res = NULL; + int mark = p->mark; + { // 'import' + void *keyword; + if ( + (keyword = _PyPegen_expect_token(p, 513)) + ) + { + res = keyword; + goto done; + } + p->mark = mark; + } + { // 'from' + void *keyword; + if ( + (keyword = _PyPegen_expect_token(p, 514)) + ) + { + res = keyword; + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// _tmp_6: 'def' | '@' | ASYNC +static void * +_tmp_6_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + void * res = NULL; + int mark = p->mark; + { // 'def' + void *keyword; + if ( + (keyword = _PyPegen_expect_token(p, 522)) + ) + { + res = keyword; + goto done; + } + p->mark = mark; + } + { // '@' + void *literal; + if ( + (literal = _PyPegen_expect_token(p, 49)) + ) + { + res = literal; + goto done; + } + p->mark = mark; + } + { // ASYNC + void *async_var; + if ( + (async_var = _PyPegen_async_token(p)) + ) + { + res = async_var; + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// _tmp_7: 'class' | '@' +static void * +_tmp_7_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + void * res = NULL; + int mark = p->mark; + { // 'class' + void *keyword; + if ( + (keyword = _PyPegen_expect_token(p, 523)) + ) + { + res = keyword; + goto done; + } + p->mark = mark; + } + { // '@' + void *literal; + if ( + (literal = _PyPegen_expect_token(p, 49)) + ) + { + res = literal; + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// _tmp_8: 'with' | ASYNC +static void * +_tmp_8_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + void * res = NULL; + int mark = p->mark; + { // 'with' + void *keyword; + if ( + (keyword = _PyPegen_expect_token(p, 519)) + ) + { + res = keyword; + goto done; + } + p->mark = mark; + } + { // ASYNC + void *async_var; + if ( + (async_var = _PyPegen_async_token(p)) + ) + { + res = async_var; + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// _tmp_9: 'for' | ASYNC +static void * +_tmp_9_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + void * res = NULL; + int mark = p->mark; + { // 'for' + void *keyword; + if ( + (keyword = _PyPegen_expect_token(p, 517)) + ) + { + res = keyword; + goto done; + } + p->mark = mark; + } + { // ASYNC + void *async_var; + if ( + (async_var = _PyPegen_async_token(p)) + ) + { + res = async_var; + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// _tmp_10: '=' annotated_rhs +static void * +_tmp_10_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + void * res = NULL; + int mark = p->mark; + { // '=' annotated_rhs + expr_ty d; + void *literal; + if ( + (literal = _PyPegen_expect_token(p, 22)) + && + (d = annotated_rhs_rule(p)) + ) + { + res = d; + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// _tmp_11: '(' inside_paren_ann_assign_target ')' | ann_assign_subscript_attribute_target +static void * +_tmp_11_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + void * res = NULL; + int mark = p->mark; + { // '(' inside_paren_ann_assign_target ')' + expr_ty b; + void *literal; + void *literal_1; + if ( + (literal = _PyPegen_expect_token(p, 7)) + && + (b = inside_paren_ann_assign_target_rule(p)) + && + (literal_1 = _PyPegen_expect_token(p, 8)) + ) + { + res = b; + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + { // ann_assign_subscript_attribute_target + expr_ty ann_assign_subscript_attribute_target_var; + if ( + (ann_assign_subscript_attribute_target_var = ann_assign_subscript_attribute_target_rule(p)) + ) + { + res = ann_assign_subscript_attribute_target_var; + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// _tmp_12: '=' annotated_rhs +static void * +_tmp_12_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + void * res = NULL; + int mark = p->mark; + { // '=' annotated_rhs + expr_ty d; + void *literal; + if ( + (literal = _PyPegen_expect_token(p, 22)) + && + (d = annotated_rhs_rule(p)) + ) + { + res = d; + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// _loop1_13: (star_targets '=') +static asdl_seq * +_loop1_13_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + void *res = NULL; + int mark = p->mark; + int start_mark = p->mark; + void **children = PyMem_Malloc(sizeof(void *)); + if (!children) { + PyErr_Format(PyExc_MemoryError, "Parser out of memory"); + return NULL; + } + ssize_t children_capacity = 1; + ssize_t n = 0; + { // (star_targets '=') + void *_tmp_123_var; + while ( + (_tmp_123_var = _tmp_123_rule(p)) + ) + { + res = _tmp_123_var; + if (n == children_capacity) { + children_capacity *= 2; + children = PyMem_Realloc(children, children_capacity*sizeof(void *)); + if (!children) { + PyErr_Format(PyExc_MemoryError, "realloc None"); + return NULL; + } + } + children[n++] = res; + mark = p->mark; + } + p->mark = mark; + } + if (n == 0) { + PyMem_Free(children); + return NULL; + } + asdl_seq *seq = _Py_asdl_seq_new(n, p->arena); + if (!seq) { + PyErr_Format(PyExc_MemoryError, "asdl_seq_new _loop1_13"); + PyMem_Free(children); + return NULL; + } + for (int i = 0; i < n; i++) asdl_seq_SET(seq, i, children[i]); + PyMem_Free(children); + _PyPegen_insert_memo(p, start_mark, _loop1_13_type, seq); + return seq; +} + +// _tmp_14: yield_expr | star_expressions +static void * +_tmp_14_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + void * res = NULL; + int mark = p->mark; + { // yield_expr + expr_ty yield_expr_var; + if ( + (yield_expr_var = yield_expr_rule(p)) + ) + { + res = yield_expr_var; + goto done; + } + p->mark = mark; + } + { // star_expressions + expr_ty star_expressions_var; + if ( + (star_expressions_var = star_expressions_rule(p)) + ) + { + res = star_expressions_var; + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// _tmp_15: yield_expr | star_expressions +static void * +_tmp_15_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + void * res = NULL; + int mark = p->mark; + { // yield_expr + expr_ty yield_expr_var; + if ( + (yield_expr_var = yield_expr_rule(p)) + ) + { + res = yield_expr_var; + goto done; + } + p->mark = mark; + } + { // star_expressions + expr_ty star_expressions_var; + if ( + (star_expressions_var = star_expressions_rule(p)) + ) + { + res = star_expressions_var; + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// _loop0_17: ',' NAME +static asdl_seq * +_loop0_17_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + void *res = NULL; + int mark = p->mark; + int start_mark = p->mark; + void **children = PyMem_Malloc(sizeof(void *)); + if (!children) { + PyErr_Format(PyExc_MemoryError, "Parser out of memory"); + return NULL; + } + ssize_t children_capacity = 1; + ssize_t n = 0; + { // ',' NAME + expr_ty elem; + void *literal; + while ( + (literal = _PyPegen_expect_token(p, 12)) + && + (elem = _PyPegen_name_token(p)) + ) + { + res = elem; + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + PyMem_Free(children); + return NULL; + } + if (n == children_capacity) { + children_capacity *= 2; + children = PyMem_Realloc(children, children_capacity*sizeof(void *)); + if (!children) { + PyErr_Format(PyExc_MemoryError, "realloc None"); + return NULL; + } + } + children[n++] = res; + mark = p->mark; + } + p->mark = mark; + } + asdl_seq *seq = _Py_asdl_seq_new(n, p->arena); + if (!seq) { + PyErr_Format(PyExc_MemoryError, "asdl_seq_new _loop0_17"); + PyMem_Free(children); + return NULL; + } + for (int i = 0; i < n; i++) asdl_seq_SET(seq, i, children[i]); + PyMem_Free(children); + _PyPegen_insert_memo(p, start_mark, _loop0_17_type, seq); + return seq; +} + +// _gather_16: NAME _loop0_17 +static asdl_seq * +_gather_16_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + asdl_seq * res = NULL; + int mark = p->mark; + { // NAME _loop0_17 + expr_ty elem; + asdl_seq * seq; + if ( + (elem = _PyPegen_name_token(p)) + && + (seq = _loop0_17_rule(p)) + ) + { + res = _PyPegen_seq_insert_in_front(p, elem, seq); + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// _loop0_19: ',' NAME +static asdl_seq * +_loop0_19_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + void *res = NULL; + int mark = p->mark; + int start_mark = p->mark; + void **children = PyMem_Malloc(sizeof(void *)); + if (!children) { + PyErr_Format(PyExc_MemoryError, "Parser out of memory"); + return NULL; + } + ssize_t children_capacity = 1; + ssize_t n = 0; + { // ',' NAME + expr_ty elem; + void *literal; + while ( + (literal = _PyPegen_expect_token(p, 12)) + && + (elem = _PyPegen_name_token(p)) + ) + { + res = elem; + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + PyMem_Free(children); + return NULL; + } + if (n == children_capacity) { + children_capacity *= 2; + children = PyMem_Realloc(children, children_capacity*sizeof(void *)); + if (!children) { + PyErr_Format(PyExc_MemoryError, "realloc None"); + return NULL; + } + } + children[n++] = res; + mark = p->mark; + } + p->mark = mark; + } + asdl_seq *seq = _Py_asdl_seq_new(n, p->arena); + if (!seq) { + PyErr_Format(PyExc_MemoryError, "asdl_seq_new _loop0_19"); + PyMem_Free(children); + return NULL; + } + for (int i = 0; i < n; i++) asdl_seq_SET(seq, i, children[i]); + PyMem_Free(children); + _PyPegen_insert_memo(p, start_mark, _loop0_19_type, seq); + return seq; +} + +// _gather_18: NAME _loop0_19 +static asdl_seq * +_gather_18_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + asdl_seq * res = NULL; + int mark = p->mark; + { // NAME _loop0_19 + expr_ty elem; + asdl_seq * seq; + if ( + (elem = _PyPegen_name_token(p)) + && + (seq = _loop0_19_rule(p)) + ) + { + res = _PyPegen_seq_insert_in_front(p, elem, seq); + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// _tmp_20: ',' expression +static void * +_tmp_20_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + void * res = NULL; + int mark = p->mark; + { // ',' expression + void *literal; + expr_ty z; + if ( + (literal = _PyPegen_expect_token(p, 12)) + && + (z = expression_rule(p)) + ) + { + res = z; + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// _loop0_21: ('.' | '...') +static asdl_seq * +_loop0_21_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + void *res = NULL; + int mark = p->mark; + int start_mark = p->mark; + void **children = PyMem_Malloc(sizeof(void *)); + if (!children) { + PyErr_Format(PyExc_MemoryError, "Parser out of memory"); + return NULL; + } + ssize_t children_capacity = 1; + ssize_t n = 0; + { // ('.' | '...') + void *_tmp_124_var; + while ( + (_tmp_124_var = _tmp_124_rule(p)) + ) + { + res = _tmp_124_var; + if (n == children_capacity) { + children_capacity *= 2; + children = PyMem_Realloc(children, children_capacity*sizeof(void *)); + if (!children) { + PyErr_Format(PyExc_MemoryError, "realloc None"); + return NULL; + } + } + children[n++] = res; + mark = p->mark; + } + p->mark = mark; + } + asdl_seq *seq = _Py_asdl_seq_new(n, p->arena); + if (!seq) { + PyErr_Format(PyExc_MemoryError, "asdl_seq_new _loop0_21"); + PyMem_Free(children); + return NULL; + } + for (int i = 0; i < n; i++) asdl_seq_SET(seq, i, children[i]); + PyMem_Free(children); + _PyPegen_insert_memo(p, start_mark, _loop0_21_type, seq); + return seq; +} + +// _loop1_22: ('.' | '...') +static asdl_seq * +_loop1_22_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + void *res = NULL; + int mark = p->mark; + int start_mark = p->mark; + void **children = PyMem_Malloc(sizeof(void *)); + if (!children) { + PyErr_Format(PyExc_MemoryError, "Parser out of memory"); + return NULL; + } + ssize_t children_capacity = 1; + ssize_t n = 0; + { // ('.' | '...') + void *_tmp_125_var; + while ( + (_tmp_125_var = _tmp_125_rule(p)) + ) + { + res = _tmp_125_var; + if (n == children_capacity) { + children_capacity *= 2; + children = PyMem_Realloc(children, children_capacity*sizeof(void *)); + if (!children) { + PyErr_Format(PyExc_MemoryError, "realloc None"); + return NULL; + } + } + children[n++] = res; + mark = p->mark; + } + p->mark = mark; + } + if (n == 0) { + PyMem_Free(children); + return NULL; + } + asdl_seq *seq = _Py_asdl_seq_new(n, p->arena); + if (!seq) { + PyErr_Format(PyExc_MemoryError, "asdl_seq_new _loop1_22"); + PyMem_Free(children); + return NULL; + } + for (int i = 0; i < n; i++) asdl_seq_SET(seq, i, children[i]); + PyMem_Free(children); + _PyPegen_insert_memo(p, start_mark, _loop1_22_type, seq); + return seq; +} + +// _loop0_24: ',' import_from_as_name +static asdl_seq * +_loop0_24_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + void *res = NULL; + int mark = p->mark; + int start_mark = p->mark; + void **children = PyMem_Malloc(sizeof(void *)); + if (!children) { + PyErr_Format(PyExc_MemoryError, "Parser out of memory"); + return NULL; + } + ssize_t children_capacity = 1; + ssize_t n = 0; + { // ',' import_from_as_name + alias_ty elem; + void *literal; + while ( + (literal = _PyPegen_expect_token(p, 12)) + && + (elem = import_from_as_name_rule(p)) + ) + { + res = elem; + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + PyMem_Free(children); + return NULL; + } + if (n == children_capacity) { + children_capacity *= 2; + children = PyMem_Realloc(children, children_capacity*sizeof(void *)); + if (!children) { + PyErr_Format(PyExc_MemoryError, "realloc None"); + return NULL; + } + } + children[n++] = res; + mark = p->mark; + } + p->mark = mark; + } + asdl_seq *seq = _Py_asdl_seq_new(n, p->arena); + if (!seq) { + PyErr_Format(PyExc_MemoryError, "asdl_seq_new _loop0_24"); + PyMem_Free(children); + return NULL; + } + for (int i = 0; i < n; i++) asdl_seq_SET(seq, i, children[i]); + PyMem_Free(children); + _PyPegen_insert_memo(p, start_mark, _loop0_24_type, seq); + return seq; +} + +// _gather_23: import_from_as_name _loop0_24 +static asdl_seq * +_gather_23_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + asdl_seq * res = NULL; + int mark = p->mark; + { // import_from_as_name _loop0_24 + alias_ty elem; + asdl_seq * seq; + if ( + (elem = import_from_as_name_rule(p)) + && + (seq = _loop0_24_rule(p)) + ) + { + res = _PyPegen_seq_insert_in_front(p, elem, seq); + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// _tmp_25: 'as' NAME +static void * +_tmp_25_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + void * res = NULL; + int mark = p->mark; + { // 'as' NAME + void *keyword; + expr_ty z; + if ( + (keyword = _PyPegen_expect_token(p, 531)) + && + (z = _PyPegen_name_token(p)) + ) + { + res = z; + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// _loop0_27: ',' dotted_as_name +static asdl_seq * +_loop0_27_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + void *res = NULL; + int mark = p->mark; + int start_mark = p->mark; + void **children = PyMem_Malloc(sizeof(void *)); + if (!children) { + PyErr_Format(PyExc_MemoryError, "Parser out of memory"); + return NULL; + } + ssize_t children_capacity = 1; + ssize_t n = 0; + { // ',' dotted_as_name + alias_ty elem; + void *literal; + while ( + (literal = _PyPegen_expect_token(p, 12)) + && + (elem = dotted_as_name_rule(p)) + ) + { + res = elem; + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + PyMem_Free(children); + return NULL; + } + if (n == children_capacity) { + children_capacity *= 2; + children = PyMem_Realloc(children, children_capacity*sizeof(void *)); + if (!children) { + PyErr_Format(PyExc_MemoryError, "realloc None"); + return NULL; + } + } + children[n++] = res; + mark = p->mark; + } + p->mark = mark; + } + asdl_seq *seq = _Py_asdl_seq_new(n, p->arena); + if (!seq) { + PyErr_Format(PyExc_MemoryError, "asdl_seq_new _loop0_27"); + PyMem_Free(children); + return NULL; + } + for (int i = 0; i < n; i++) asdl_seq_SET(seq, i, children[i]); + PyMem_Free(children); + _PyPegen_insert_memo(p, start_mark, _loop0_27_type, seq); + return seq; +} + +// _gather_26: dotted_as_name _loop0_27 +static asdl_seq * +_gather_26_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + asdl_seq * res = NULL; + int mark = p->mark; + { // dotted_as_name _loop0_27 + alias_ty elem; + asdl_seq * seq; + if ( + (elem = dotted_as_name_rule(p)) + && + (seq = _loop0_27_rule(p)) + ) + { + res = _PyPegen_seq_insert_in_front(p, elem, seq); + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// _tmp_28: 'as' NAME +static void * +_tmp_28_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + void * res = NULL; + int mark = p->mark; + { // 'as' NAME + void *keyword; + expr_ty z; + if ( + (keyword = _PyPegen_expect_token(p, 531)) + && + (z = _PyPegen_name_token(p)) + ) + { + res = z; + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// _loop0_30: ',' with_item +static asdl_seq * +_loop0_30_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + void *res = NULL; + int mark = p->mark; + int start_mark = p->mark; + void **children = PyMem_Malloc(sizeof(void *)); + if (!children) { + PyErr_Format(PyExc_MemoryError, "Parser out of memory"); + return NULL; + } + ssize_t children_capacity = 1; + ssize_t n = 0; + { // ',' with_item + withitem_ty elem; + void *literal; + while ( + (literal = _PyPegen_expect_token(p, 12)) + && + (elem = with_item_rule(p)) + ) + { + res = elem; + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + PyMem_Free(children); + return NULL; + } + if (n == children_capacity) { + children_capacity *= 2; + children = PyMem_Realloc(children, children_capacity*sizeof(void *)); + if (!children) { + PyErr_Format(PyExc_MemoryError, "realloc None"); + return NULL; + } + } + children[n++] = res; + mark = p->mark; + } + p->mark = mark; + } + asdl_seq *seq = _Py_asdl_seq_new(n, p->arena); + if (!seq) { + PyErr_Format(PyExc_MemoryError, "asdl_seq_new _loop0_30"); + PyMem_Free(children); + return NULL; + } + for (int i = 0; i < n; i++) asdl_seq_SET(seq, i, children[i]); + PyMem_Free(children); + _PyPegen_insert_memo(p, start_mark, _loop0_30_type, seq); + return seq; +} + +// _gather_29: with_item _loop0_30 +static asdl_seq * +_gather_29_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + asdl_seq * res = NULL; + int mark = p->mark; + { // with_item _loop0_30 + withitem_ty elem; + asdl_seq * seq; + if ( + (elem = with_item_rule(p)) + && + (seq = _loop0_30_rule(p)) + ) + { + res = _PyPegen_seq_insert_in_front(p, elem, seq); + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// _loop0_32: ',' with_item +static asdl_seq * +_loop0_32_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + void *res = NULL; + int mark = p->mark; + int start_mark = p->mark; + void **children = PyMem_Malloc(sizeof(void *)); + if (!children) { + PyErr_Format(PyExc_MemoryError, "Parser out of memory"); + return NULL; + } + ssize_t children_capacity = 1; + ssize_t n = 0; + { // ',' with_item + withitem_ty elem; + void *literal; + while ( + (literal = _PyPegen_expect_token(p, 12)) + && + (elem = with_item_rule(p)) + ) + { + res = elem; + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + PyMem_Free(children); + return NULL; + } + if (n == children_capacity) { + children_capacity *= 2; + children = PyMem_Realloc(children, children_capacity*sizeof(void *)); + if (!children) { + PyErr_Format(PyExc_MemoryError, "realloc None"); + return NULL; + } + } + children[n++] = res; + mark = p->mark; + } + p->mark = mark; + } + asdl_seq *seq = _Py_asdl_seq_new(n, p->arena); + if (!seq) { + PyErr_Format(PyExc_MemoryError, "asdl_seq_new _loop0_32"); + PyMem_Free(children); + return NULL; + } + for (int i = 0; i < n; i++) asdl_seq_SET(seq, i, children[i]); + PyMem_Free(children); + _PyPegen_insert_memo(p, start_mark, _loop0_32_type, seq); + return seq; +} + +// _gather_31: with_item _loop0_32 +static asdl_seq * +_gather_31_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + asdl_seq * res = NULL; + int mark = p->mark; + { // with_item _loop0_32 + withitem_ty elem; + asdl_seq * seq; + if ( + (elem = with_item_rule(p)) + && + (seq = _loop0_32_rule(p)) + ) + { + res = _PyPegen_seq_insert_in_front(p, elem, seq); + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// _tmp_33: 'as' target +static void * +_tmp_33_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + void * res = NULL; + int mark = p->mark; + { // 'as' target + void *keyword; + expr_ty t; + if ( + (keyword = _PyPegen_expect_token(p, 531)) + && + (t = target_rule(p)) + ) + { + res = t; + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// _loop1_34: except_block +static asdl_seq * +_loop1_34_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + void *res = NULL; + int mark = p->mark; + int start_mark = p->mark; + void **children = PyMem_Malloc(sizeof(void *)); + if (!children) { + PyErr_Format(PyExc_MemoryError, "Parser out of memory"); + return NULL; + } + ssize_t children_capacity = 1; + ssize_t n = 0; + { // except_block + excepthandler_ty except_block_var; + while ( + (except_block_var = except_block_rule(p)) + ) + { + res = except_block_var; + if (n == children_capacity) { + children_capacity *= 2; + children = PyMem_Realloc(children, children_capacity*sizeof(void *)); + if (!children) { + PyErr_Format(PyExc_MemoryError, "realloc None"); + return NULL; + } + } + children[n++] = res; + mark = p->mark; + } + p->mark = mark; + } + if (n == 0) { + PyMem_Free(children); + return NULL; + } + asdl_seq *seq = _Py_asdl_seq_new(n, p->arena); + if (!seq) { + PyErr_Format(PyExc_MemoryError, "asdl_seq_new _loop1_34"); + PyMem_Free(children); + return NULL; + } + for (int i = 0; i < n; i++) asdl_seq_SET(seq, i, children[i]); + PyMem_Free(children); + _PyPegen_insert_memo(p, start_mark, _loop1_34_type, seq); + return seq; +} + +// _tmp_35: 'as' target +static void * +_tmp_35_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + void * res = NULL; + int mark = p->mark; + { // 'as' target + void *keyword; + expr_ty z; + if ( + (keyword = _PyPegen_expect_token(p, 531)) + && + (z = target_rule(p)) + ) + { + res = z; + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// _tmp_36: 'from' expression +static void * +_tmp_36_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + void * res = NULL; + int mark = p->mark; + { // 'from' expression + void *keyword; + expr_ty z; + if ( + (keyword = _PyPegen_expect_token(p, 514)) + && + (z = expression_rule(p)) + ) + { + res = z; + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// _tmp_37: '->' annotation +static void * +_tmp_37_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + void * res = NULL; + int mark = p->mark; + { // '->' annotation + void *literal; + expr_ty z; + if ( + (literal = _PyPegen_expect_token(p, 51)) + && + (z = annotation_rule(p)) + ) + { + res = z; + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// _tmp_38: ',' plain_names +static void * +_tmp_38_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + void * res = NULL; + int mark = p->mark; + { // ',' plain_names + void *literal; + asdl_seq* x; + if ( + (literal = _PyPegen_expect_token(p, 12)) + && + (x = plain_names_rule(p)) + ) + { + res = x; + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// _tmp_39: ',' names_with_default +static void * +_tmp_39_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + void * res = NULL; + int mark = p->mark; + { // ',' names_with_default + void *literal; + asdl_seq* y; + if ( + (literal = _PyPegen_expect_token(p, 12)) + && + (y = names_with_default_rule(p)) + ) + { + res = y; + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// _tmp_40: ',' star_etc? +static void * +_tmp_40_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + void * res = NULL; + int mark = p->mark; + { // ',' star_etc? + void *literal; + void *z; + if ( + (literal = _PyPegen_expect_token(p, 12)) + && + (z = star_etc_rule(p), 1) + ) + { + res = z; + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// _tmp_41: ',' names_with_default +static void * +_tmp_41_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + void * res = NULL; + int mark = p->mark; + { // ',' names_with_default + void *literal; + asdl_seq* y; + if ( + (literal = _PyPegen_expect_token(p, 12)) + && + (y = names_with_default_rule(p)) + ) + { + res = y; + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// _tmp_42: ',' star_etc? +static void * +_tmp_42_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + void * res = NULL; + int mark = p->mark; + { // ',' star_etc? + void *literal; + void *z; + if ( + (literal = _PyPegen_expect_token(p, 12)) + && + (z = star_etc_rule(p), 1) + ) + { + res = z; + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// _tmp_43: ',' names_with_default +static void * +_tmp_43_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + void * res = NULL; + int mark = p->mark; + { // ',' names_with_default + void *literal; + asdl_seq* y; + if ( + (literal = _PyPegen_expect_token(p, 12)) + && + (y = names_with_default_rule(p)) + ) + { + res = y; + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// _tmp_44: ',' star_etc? +static void * +_tmp_44_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + void * res = NULL; + int mark = p->mark; + { // ',' star_etc? + void *literal; + void *z; + if ( + (literal = _PyPegen_expect_token(p, 12)) + && + (z = star_etc_rule(p), 1) + ) + { + res = z; + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// _tmp_45: ',' star_etc? +static void * +_tmp_45_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + void * res = NULL; + int mark = p->mark; + { // ',' star_etc? + void *literal; + void *z; + if ( + (literal = _PyPegen_expect_token(p, 12)) + && + (z = star_etc_rule(p), 1) + ) + { + res = z; + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// _tmp_46: plain_names ',' +static void * +_tmp_46_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + void * res = NULL; + int mark = p->mark; + { // plain_names ',' + void *literal; + asdl_seq* n; + if ( + (n = plain_names_rule(p)) + && + (literal = _PyPegen_expect_token(p, 12)) + ) + { + res = n; + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// _loop0_47: name_with_optional_default +static asdl_seq * +_loop0_47_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + void *res = NULL; + int mark = p->mark; + int start_mark = p->mark; + void **children = PyMem_Malloc(sizeof(void *)); + if (!children) { + PyErr_Format(PyExc_MemoryError, "Parser out of memory"); + return NULL; + } + ssize_t children_capacity = 1; + ssize_t n = 0; + { // name_with_optional_default + NameDefaultPair* name_with_optional_default_var; + while ( + (name_with_optional_default_var = name_with_optional_default_rule(p)) + ) + { + res = name_with_optional_default_var; + if (n == children_capacity) { + children_capacity *= 2; + children = PyMem_Realloc(children, children_capacity*sizeof(void *)); + if (!children) { + PyErr_Format(PyExc_MemoryError, "realloc None"); + return NULL; + } + } + children[n++] = res; + mark = p->mark; + } + p->mark = mark; + } + asdl_seq *seq = _Py_asdl_seq_new(n, p->arena); + if (!seq) { + PyErr_Format(PyExc_MemoryError, "asdl_seq_new _loop0_47"); + PyMem_Free(children); + return NULL; + } + for (int i = 0; i < n; i++) asdl_seq_SET(seq, i, children[i]); + PyMem_Free(children); + _PyPegen_insert_memo(p, start_mark, _loop0_47_type, seq); + return seq; +} + +// _tmp_48: ',' kwds +static void * +_tmp_48_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + void * res = NULL; + int mark = p->mark; + { // ',' kwds + arg_ty d; + void *literal; + if ( + (literal = _PyPegen_expect_token(p, 12)) + && + (d = kwds_rule(p)) + ) + { + res = d; + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// _loop1_49: name_with_optional_default +static asdl_seq * +_loop1_49_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + void *res = NULL; + int mark = p->mark; + int start_mark = p->mark; + void **children = PyMem_Malloc(sizeof(void *)); + if (!children) { + PyErr_Format(PyExc_MemoryError, "Parser out of memory"); + return NULL; + } + ssize_t children_capacity = 1; + ssize_t n = 0; + { // name_with_optional_default + NameDefaultPair* name_with_optional_default_var; + while ( + (name_with_optional_default_var = name_with_optional_default_rule(p)) + ) + { + res = name_with_optional_default_var; + if (n == children_capacity) { + children_capacity *= 2; + children = PyMem_Realloc(children, children_capacity*sizeof(void *)); + if (!children) { + PyErr_Format(PyExc_MemoryError, "realloc None"); + return NULL; + } + } + children[n++] = res; + mark = p->mark; + } + p->mark = mark; + } + if (n == 0) { + PyMem_Free(children); + return NULL; + } + asdl_seq *seq = _Py_asdl_seq_new(n, p->arena); + if (!seq) { + PyErr_Format(PyExc_MemoryError, "asdl_seq_new _loop1_49"); + PyMem_Free(children); + return NULL; + } + for (int i = 0; i < n; i++) asdl_seq_SET(seq, i, children[i]); + PyMem_Free(children); + _PyPegen_insert_memo(p, start_mark, _loop1_49_type, seq); + return seq; +} + +// _tmp_50: ',' kwds +static void * +_tmp_50_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + void * res = NULL; + int mark = p->mark; + { // ',' kwds + arg_ty d; + void *literal; + if ( + (literal = _PyPegen_expect_token(p, 12)) + && + (d = kwds_rule(p)) + ) + { + res = d; + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// _tmp_51: '=' expression +static void * +_tmp_51_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + void * res = NULL; + int mark = p->mark; + { // '=' expression + expr_ty e; + void *literal; + if ( + (literal = _PyPegen_expect_token(p, 22)) + && + (e = expression_rule(p)) + ) + { + res = e; + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// _loop0_53: ',' name_with_default +static asdl_seq * +_loop0_53_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + void *res = NULL; + int mark = p->mark; + int start_mark = p->mark; + void **children = PyMem_Malloc(sizeof(void *)); + if (!children) { + PyErr_Format(PyExc_MemoryError, "Parser out of memory"); + return NULL; + } + ssize_t children_capacity = 1; + ssize_t n = 0; + { // ',' name_with_default + NameDefaultPair* elem; + void *literal; + while ( + (literal = _PyPegen_expect_token(p, 12)) + && + (elem = name_with_default_rule(p)) + ) + { + res = elem; + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + PyMem_Free(children); + return NULL; + } + if (n == children_capacity) { + children_capacity *= 2; + children = PyMem_Realloc(children, children_capacity*sizeof(void *)); + if (!children) { + PyErr_Format(PyExc_MemoryError, "realloc None"); + return NULL; + } + } + children[n++] = res; + mark = p->mark; + } + p->mark = mark; + } + asdl_seq *seq = _Py_asdl_seq_new(n, p->arena); + if (!seq) { + PyErr_Format(PyExc_MemoryError, "asdl_seq_new _loop0_53"); + PyMem_Free(children); + return NULL; + } + for (int i = 0; i < n; i++) asdl_seq_SET(seq, i, children[i]); + PyMem_Free(children); + _PyPegen_insert_memo(p, start_mark, _loop0_53_type, seq); + return seq; +} + +// _gather_52: name_with_default _loop0_53 +static asdl_seq * +_gather_52_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + asdl_seq * res = NULL; + int mark = p->mark; + { // name_with_default _loop0_53 + NameDefaultPair* elem; + asdl_seq * seq; + if ( + (elem = name_with_default_rule(p)) + && + (seq = _loop0_53_rule(p)) + ) + { + res = _PyPegen_seq_insert_in_front(p, elem, seq); + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// _loop0_55: ',' (plain_name !'=') +static asdl_seq * +_loop0_55_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + void *res = NULL; + int mark = p->mark; + int start_mark = p->mark; + void **children = PyMem_Malloc(sizeof(void *)); + if (!children) { + PyErr_Format(PyExc_MemoryError, "Parser out of memory"); + return NULL; + } + ssize_t children_capacity = 1; + ssize_t n = 0; + { // ',' (plain_name !'=') + void *elem; + void *literal; + while ( + (literal = _PyPegen_expect_token(p, 12)) + && + (elem = _tmp_126_rule(p)) + ) + { + res = elem; + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + PyMem_Free(children); + return NULL; + } + if (n == children_capacity) { + children_capacity *= 2; + children = PyMem_Realloc(children, children_capacity*sizeof(void *)); + if (!children) { + PyErr_Format(PyExc_MemoryError, "realloc None"); + return NULL; + } + } + children[n++] = res; + mark = p->mark; + } + p->mark = mark; + } + asdl_seq *seq = _Py_asdl_seq_new(n, p->arena); + if (!seq) { + PyErr_Format(PyExc_MemoryError, "asdl_seq_new _loop0_55"); + PyMem_Free(children); + return NULL; + } + for (int i = 0; i < n; i++) asdl_seq_SET(seq, i, children[i]); + PyMem_Free(children); + _PyPegen_insert_memo(p, start_mark, _loop0_55_type, seq); + return seq; +} + +// _gather_54: (plain_name !'=') _loop0_55 +static asdl_seq * +_gather_54_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + asdl_seq * res = NULL; + int mark = p->mark; + { // (plain_name !'=') _loop0_55 + void *elem; + asdl_seq * seq; + if ( + (elem = _tmp_126_rule(p)) + && + (seq = _loop0_55_rule(p)) + ) + { + res = _PyPegen_seq_insert_in_front(p, elem, seq); + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// _tmp_56: ':' annotation +static void * +_tmp_56_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + void * res = NULL; + int mark = p->mark; + { // ':' annotation + void *literal; + expr_ty z; + if ( + (literal = _PyPegen_expect_token(p, 11)) + && + (z = annotation_rule(p)) + ) + { + res = z; + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// _loop1_57: ('@' named_expression NEWLINE) +static asdl_seq * +_loop1_57_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + void *res = NULL; + int mark = p->mark; + int start_mark = p->mark; + void **children = PyMem_Malloc(sizeof(void *)); + if (!children) { + PyErr_Format(PyExc_MemoryError, "Parser out of memory"); + return NULL; + } + ssize_t children_capacity = 1; + ssize_t n = 0; + { // ('@' named_expression NEWLINE) + void *_tmp_127_var; + while ( + (_tmp_127_var = _tmp_127_rule(p)) + ) + { + res = _tmp_127_var; + if (n == children_capacity) { + children_capacity *= 2; + children = PyMem_Realloc(children, children_capacity*sizeof(void *)); + if (!children) { + PyErr_Format(PyExc_MemoryError, "realloc None"); + return NULL; + } + } + children[n++] = res; + mark = p->mark; + } + p->mark = mark; + } + if (n == 0) { + PyMem_Free(children); + return NULL; + } + asdl_seq *seq = _Py_asdl_seq_new(n, p->arena); + if (!seq) { + PyErr_Format(PyExc_MemoryError, "asdl_seq_new _loop1_57"); + PyMem_Free(children); + return NULL; + } + for (int i = 0; i < n; i++) asdl_seq_SET(seq, i, children[i]); + PyMem_Free(children); + _PyPegen_insert_memo(p, start_mark, _loop1_57_type, seq); + return seq; +} + +// _tmp_58: '(' arguments? ')' +static void * +_tmp_58_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + void * res = NULL; + int mark = p->mark; + { // '(' arguments? ')' + void *literal; + void *literal_1; + void *z; + if ( + (literal = _PyPegen_expect_token(p, 7)) + && + (z = arguments_rule(p), 1) + && + (literal_1 = _PyPegen_expect_token(p, 8)) + ) + { + res = z; + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// _loop0_60: ',' star_expression +static asdl_seq * +_loop0_60_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + void *res = NULL; + int mark = p->mark; + int start_mark = p->mark; + void **children = PyMem_Malloc(sizeof(void *)); + if (!children) { + PyErr_Format(PyExc_MemoryError, "Parser out of memory"); + return NULL; + } + ssize_t children_capacity = 1; + ssize_t n = 0; + { // ',' star_expression + expr_ty elem; + void *literal; + while ( + (literal = _PyPegen_expect_token(p, 12)) + && + (elem = star_expression_rule(p)) + ) + { + res = elem; + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + PyMem_Free(children); + return NULL; + } + if (n == children_capacity) { + children_capacity *= 2; + children = PyMem_Realloc(children, children_capacity*sizeof(void *)); + if (!children) { + PyErr_Format(PyExc_MemoryError, "realloc None"); + return NULL; + } + } + children[n++] = res; + mark = p->mark; + } + p->mark = mark; + } + asdl_seq *seq = _Py_asdl_seq_new(n, p->arena); + if (!seq) { + PyErr_Format(PyExc_MemoryError, "asdl_seq_new _loop0_60"); + PyMem_Free(children); + return NULL; + } + for (int i = 0; i < n; i++) asdl_seq_SET(seq, i, children[i]); + PyMem_Free(children); + _PyPegen_insert_memo(p, start_mark, _loop0_60_type, seq); + return seq; +} + +// _gather_59: star_expression _loop0_60 +static asdl_seq * +_gather_59_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + asdl_seq * res = NULL; + int mark = p->mark; + { // star_expression _loop0_60 + expr_ty elem; + asdl_seq * seq; + if ( + (elem = star_expression_rule(p)) + && + (seq = _loop0_60_rule(p)) + ) + { + res = _PyPegen_seq_insert_in_front(p, elem, seq); + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// _loop1_61: (',' star_expression) +static asdl_seq * +_loop1_61_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + void *res = NULL; + int mark = p->mark; + int start_mark = p->mark; + void **children = PyMem_Malloc(sizeof(void *)); + if (!children) { + PyErr_Format(PyExc_MemoryError, "Parser out of memory"); + return NULL; + } + ssize_t children_capacity = 1; + ssize_t n = 0; + { // (',' star_expression) + void *_tmp_128_var; + while ( + (_tmp_128_var = _tmp_128_rule(p)) + ) + { + res = _tmp_128_var; + if (n == children_capacity) { + children_capacity *= 2; + children = PyMem_Realloc(children, children_capacity*sizeof(void *)); + if (!children) { + PyErr_Format(PyExc_MemoryError, "realloc None"); + return NULL; + } + } + children[n++] = res; + mark = p->mark; + } + p->mark = mark; + } + if (n == 0) { + PyMem_Free(children); + return NULL; + } + asdl_seq *seq = _Py_asdl_seq_new(n, p->arena); + if (!seq) { + PyErr_Format(PyExc_MemoryError, "asdl_seq_new _loop1_61"); + PyMem_Free(children); + return NULL; + } + for (int i = 0; i < n; i++) asdl_seq_SET(seq, i, children[i]); + PyMem_Free(children); + _PyPegen_insert_memo(p, start_mark, _loop1_61_type, seq); + return seq; +} + +// _loop0_63: ',' star_named_expression +static asdl_seq * +_loop0_63_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + void *res = NULL; + int mark = p->mark; + int start_mark = p->mark; + void **children = PyMem_Malloc(sizeof(void *)); + if (!children) { + PyErr_Format(PyExc_MemoryError, "Parser out of memory"); + return NULL; + } + ssize_t children_capacity = 1; + ssize_t n = 0; + { // ',' star_named_expression + expr_ty elem; + void *literal; + while ( + (literal = _PyPegen_expect_token(p, 12)) + && + (elem = star_named_expression_rule(p)) + ) + { + res = elem; + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + PyMem_Free(children); + return NULL; + } + if (n == children_capacity) { + children_capacity *= 2; + children = PyMem_Realloc(children, children_capacity*sizeof(void *)); + if (!children) { + PyErr_Format(PyExc_MemoryError, "realloc None"); + return NULL; + } + } + children[n++] = res; + mark = p->mark; + } + p->mark = mark; + } + asdl_seq *seq = _Py_asdl_seq_new(n, p->arena); + if (!seq) { + PyErr_Format(PyExc_MemoryError, "asdl_seq_new _loop0_63"); + PyMem_Free(children); + return NULL; + } + for (int i = 0; i < n; i++) asdl_seq_SET(seq, i, children[i]); + PyMem_Free(children); + _PyPegen_insert_memo(p, start_mark, _loop0_63_type, seq); + return seq; +} + +// _gather_62: star_named_expression _loop0_63 +static asdl_seq * +_gather_62_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + asdl_seq * res = NULL; + int mark = p->mark; + { // star_named_expression _loop0_63 + expr_ty elem; + asdl_seq * seq; + if ( + (elem = star_named_expression_rule(p)) + && + (seq = _loop0_63_rule(p)) + ) + { + res = _PyPegen_seq_insert_in_front(p, elem, seq); + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// _loop1_64: (',' expression) +static asdl_seq * +_loop1_64_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + void *res = NULL; + int mark = p->mark; + int start_mark = p->mark; + void **children = PyMem_Malloc(sizeof(void *)); + if (!children) { + PyErr_Format(PyExc_MemoryError, "Parser out of memory"); + return NULL; + } + ssize_t children_capacity = 1; + ssize_t n = 0; + { // (',' expression) + void *_tmp_129_var; + while ( + (_tmp_129_var = _tmp_129_rule(p)) + ) + { + res = _tmp_129_var; + if (n == children_capacity) { + children_capacity *= 2; + children = PyMem_Realloc(children, children_capacity*sizeof(void *)); + if (!children) { + PyErr_Format(PyExc_MemoryError, "realloc None"); + return NULL; + } + } + children[n++] = res; + mark = p->mark; + } + p->mark = mark; + } + if (n == 0) { + PyMem_Free(children); + return NULL; + } + asdl_seq *seq = _Py_asdl_seq_new(n, p->arena); + if (!seq) { + PyErr_Format(PyExc_MemoryError, "asdl_seq_new _loop1_64"); + PyMem_Free(children); + return NULL; + } + for (int i = 0; i < n; i++) asdl_seq_SET(seq, i, children[i]); + PyMem_Free(children); + _PyPegen_insert_memo(p, start_mark, _loop1_64_type, seq); + return seq; +} + +// _tmp_65: ',' lambda_plain_names +static void * +_tmp_65_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + void * res = NULL; + int mark = p->mark; + { // ',' lambda_plain_names + void *literal; + asdl_seq* x; + if ( + (literal = _PyPegen_expect_token(p, 12)) + && + (x = lambda_plain_names_rule(p)) + ) + { + res = x; + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// _tmp_66: ',' lambda_names_with_default +static void * +_tmp_66_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + void * res = NULL; + int mark = p->mark; + { // ',' lambda_names_with_default + void *literal; + asdl_seq* y; + if ( + (literal = _PyPegen_expect_token(p, 12)) + && + (y = lambda_names_with_default_rule(p)) + ) + { + res = y; + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// _tmp_67: ',' lambda_star_etc? +static void * +_tmp_67_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + void * res = NULL; + int mark = p->mark; + { // ',' lambda_star_etc? + void *literal; + void *z; + if ( + (literal = _PyPegen_expect_token(p, 12)) + && + (z = lambda_star_etc_rule(p), 1) + ) + { + res = z; + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// _tmp_68: ',' lambda_names_with_default +static void * +_tmp_68_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + void * res = NULL; + int mark = p->mark; + { // ',' lambda_names_with_default + void *literal; + asdl_seq* y; + if ( + (literal = _PyPegen_expect_token(p, 12)) + && + (y = lambda_names_with_default_rule(p)) + ) + { + res = y; + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// _tmp_69: ',' lambda_star_etc? +static void * +_tmp_69_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + void * res = NULL; + int mark = p->mark; + { // ',' lambda_star_etc? + void *literal; + void *z; + if ( + (literal = _PyPegen_expect_token(p, 12)) + && + (z = lambda_star_etc_rule(p), 1) + ) + { + res = z; + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// _tmp_70: ',' lambda_names_with_default +static void * +_tmp_70_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + void * res = NULL; + int mark = p->mark; + { // ',' lambda_names_with_default + void *literal; + asdl_seq* y; + if ( + (literal = _PyPegen_expect_token(p, 12)) + && + (y = lambda_names_with_default_rule(p)) + ) + { + res = y; + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// _tmp_71: ',' lambda_star_etc? +static void * +_tmp_71_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + void * res = NULL; + int mark = p->mark; + { // ',' lambda_star_etc? + void *literal; + void *z; + if ( + (literal = _PyPegen_expect_token(p, 12)) + && + (z = lambda_star_etc_rule(p), 1) + ) + { + res = z; + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// _tmp_72: ',' lambda_star_etc? +static void * +_tmp_72_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + void * res = NULL; + int mark = p->mark; + { // ',' lambda_star_etc? + void *literal; + void *z; + if ( + (literal = _PyPegen_expect_token(p, 12)) + && + (z = lambda_star_etc_rule(p), 1) + ) + { + res = z; + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// _tmp_73: lambda_plain_names ',' +static void * +_tmp_73_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + void * res = NULL; + int mark = p->mark; + { // lambda_plain_names ',' + void *literal; + asdl_seq* n; + if ( + (n = lambda_plain_names_rule(p)) + && + (literal = _PyPegen_expect_token(p, 12)) + ) + { + res = n; + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// _loop0_74: lambda_name_with_optional_default +static asdl_seq * +_loop0_74_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + void *res = NULL; + int mark = p->mark; + int start_mark = p->mark; + void **children = PyMem_Malloc(sizeof(void *)); + if (!children) { + PyErr_Format(PyExc_MemoryError, "Parser out of memory"); + return NULL; + } + ssize_t children_capacity = 1; + ssize_t n = 0; + { // lambda_name_with_optional_default + NameDefaultPair* lambda_name_with_optional_default_var; + while ( + (lambda_name_with_optional_default_var = lambda_name_with_optional_default_rule(p)) + ) + { + res = lambda_name_with_optional_default_var; + if (n == children_capacity) { + children_capacity *= 2; + children = PyMem_Realloc(children, children_capacity*sizeof(void *)); + if (!children) { + PyErr_Format(PyExc_MemoryError, "realloc None"); + return NULL; + } + } + children[n++] = res; + mark = p->mark; + } + p->mark = mark; + } + asdl_seq *seq = _Py_asdl_seq_new(n, p->arena); + if (!seq) { + PyErr_Format(PyExc_MemoryError, "asdl_seq_new _loop0_74"); + PyMem_Free(children); + return NULL; + } + for (int i = 0; i < n; i++) asdl_seq_SET(seq, i, children[i]); + PyMem_Free(children); + _PyPegen_insert_memo(p, start_mark, _loop0_74_type, seq); + return seq; +} + +// _tmp_75: ',' lambda_kwds +static void * +_tmp_75_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + void * res = NULL; + int mark = p->mark; + { // ',' lambda_kwds + arg_ty d; + void *literal; + if ( + (literal = _PyPegen_expect_token(p, 12)) + && + (d = lambda_kwds_rule(p)) + ) + { + res = d; + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// _loop1_76: lambda_name_with_optional_default +static asdl_seq * +_loop1_76_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + void *res = NULL; + int mark = p->mark; + int start_mark = p->mark; + void **children = PyMem_Malloc(sizeof(void *)); + if (!children) { + PyErr_Format(PyExc_MemoryError, "Parser out of memory"); + return NULL; + } + ssize_t children_capacity = 1; + ssize_t n = 0; + { // lambda_name_with_optional_default + NameDefaultPair* lambda_name_with_optional_default_var; + while ( + (lambda_name_with_optional_default_var = lambda_name_with_optional_default_rule(p)) + ) + { + res = lambda_name_with_optional_default_var; + if (n == children_capacity) { + children_capacity *= 2; + children = PyMem_Realloc(children, children_capacity*sizeof(void *)); + if (!children) { + PyErr_Format(PyExc_MemoryError, "realloc None"); + return NULL; + } + } + children[n++] = res; + mark = p->mark; + } + p->mark = mark; + } + if (n == 0) { + PyMem_Free(children); + return NULL; + } + asdl_seq *seq = _Py_asdl_seq_new(n, p->arena); + if (!seq) { + PyErr_Format(PyExc_MemoryError, "asdl_seq_new _loop1_76"); + PyMem_Free(children); + return NULL; + } + for (int i = 0; i < n; i++) asdl_seq_SET(seq, i, children[i]); + PyMem_Free(children); + _PyPegen_insert_memo(p, start_mark, _loop1_76_type, seq); + return seq; +} + +// _tmp_77: ',' lambda_kwds +static void * +_tmp_77_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + void * res = NULL; + int mark = p->mark; + { // ',' lambda_kwds + arg_ty d; + void *literal; + if ( + (literal = _PyPegen_expect_token(p, 12)) + && + (d = lambda_kwds_rule(p)) + ) + { + res = d; + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// _tmp_78: '=' expression +static void * +_tmp_78_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + void * res = NULL; + int mark = p->mark; + { // '=' expression + expr_ty e; + void *literal; + if ( + (literal = _PyPegen_expect_token(p, 22)) + && + (e = expression_rule(p)) + ) + { + res = e; + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// _loop0_80: ',' lambda_name_with_default +static asdl_seq * +_loop0_80_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + void *res = NULL; + int mark = p->mark; + int start_mark = p->mark; + void **children = PyMem_Malloc(sizeof(void *)); + if (!children) { + PyErr_Format(PyExc_MemoryError, "Parser out of memory"); + return NULL; + } + ssize_t children_capacity = 1; + ssize_t n = 0; + { // ',' lambda_name_with_default + NameDefaultPair* elem; + void *literal; + while ( + (literal = _PyPegen_expect_token(p, 12)) + && + (elem = lambda_name_with_default_rule(p)) + ) + { + res = elem; + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + PyMem_Free(children); + return NULL; + } + if (n == children_capacity) { + children_capacity *= 2; + children = PyMem_Realloc(children, children_capacity*sizeof(void *)); + if (!children) { + PyErr_Format(PyExc_MemoryError, "realloc None"); + return NULL; + } + } + children[n++] = res; + mark = p->mark; + } + p->mark = mark; + } + asdl_seq *seq = _Py_asdl_seq_new(n, p->arena); + if (!seq) { + PyErr_Format(PyExc_MemoryError, "asdl_seq_new _loop0_80"); + PyMem_Free(children); + return NULL; + } + for (int i = 0; i < n; i++) asdl_seq_SET(seq, i, children[i]); + PyMem_Free(children); + _PyPegen_insert_memo(p, start_mark, _loop0_80_type, seq); + return seq; +} + +// _gather_79: lambda_name_with_default _loop0_80 +static asdl_seq * +_gather_79_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + asdl_seq * res = NULL; + int mark = p->mark; + { // lambda_name_with_default _loop0_80 + NameDefaultPair* elem; + asdl_seq * seq; + if ( + (elem = lambda_name_with_default_rule(p)) + && + (seq = _loop0_80_rule(p)) + ) + { + res = _PyPegen_seq_insert_in_front(p, elem, seq); + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// _loop0_82: ',' (lambda_plain_name !'=') +static asdl_seq * +_loop0_82_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + void *res = NULL; + int mark = p->mark; + int start_mark = p->mark; + void **children = PyMem_Malloc(sizeof(void *)); + if (!children) { + PyErr_Format(PyExc_MemoryError, "Parser out of memory"); + return NULL; + } + ssize_t children_capacity = 1; + ssize_t n = 0; + { // ',' (lambda_plain_name !'=') + void *elem; + void *literal; + while ( + (literal = _PyPegen_expect_token(p, 12)) + && + (elem = _tmp_130_rule(p)) + ) + { + res = elem; + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + PyMem_Free(children); + return NULL; + } + if (n == children_capacity) { + children_capacity *= 2; + children = PyMem_Realloc(children, children_capacity*sizeof(void *)); + if (!children) { + PyErr_Format(PyExc_MemoryError, "realloc None"); + return NULL; + } + } + children[n++] = res; + mark = p->mark; + } + p->mark = mark; + } + asdl_seq *seq = _Py_asdl_seq_new(n, p->arena); + if (!seq) { + PyErr_Format(PyExc_MemoryError, "asdl_seq_new _loop0_82"); + PyMem_Free(children); + return NULL; + } + for (int i = 0; i < n; i++) asdl_seq_SET(seq, i, children[i]); + PyMem_Free(children); + _PyPegen_insert_memo(p, start_mark, _loop0_82_type, seq); + return seq; +} + +// _gather_81: (lambda_plain_name !'=') _loop0_82 +static asdl_seq * +_gather_81_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + asdl_seq * res = NULL; + int mark = p->mark; + { // (lambda_plain_name !'=') _loop0_82 + void *elem; + asdl_seq * seq; + if ( + (elem = _tmp_130_rule(p)) + && + (seq = _loop0_82_rule(p)) + ) + { + res = _PyPegen_seq_insert_in_front(p, elem, seq); + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// _loop1_83: ('or' conjunction) +static asdl_seq * +_loop1_83_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + void *res = NULL; + int mark = p->mark; + int start_mark = p->mark; + void **children = PyMem_Malloc(sizeof(void *)); + if (!children) { + PyErr_Format(PyExc_MemoryError, "Parser out of memory"); + return NULL; + } + ssize_t children_capacity = 1; + ssize_t n = 0; + { // ('or' conjunction) + void *_tmp_131_var; + while ( + (_tmp_131_var = _tmp_131_rule(p)) + ) + { + res = _tmp_131_var; + if (n == children_capacity) { + children_capacity *= 2; + children = PyMem_Realloc(children, children_capacity*sizeof(void *)); + if (!children) { + PyErr_Format(PyExc_MemoryError, "realloc None"); + return NULL; + } + } + children[n++] = res; + mark = p->mark; + } + p->mark = mark; + } + if (n == 0) { + PyMem_Free(children); + return NULL; + } + asdl_seq *seq = _Py_asdl_seq_new(n, p->arena); + if (!seq) { + PyErr_Format(PyExc_MemoryError, "asdl_seq_new _loop1_83"); + PyMem_Free(children); + return NULL; + } + for (int i = 0; i < n; i++) asdl_seq_SET(seq, i, children[i]); + PyMem_Free(children); + _PyPegen_insert_memo(p, start_mark, _loop1_83_type, seq); + return seq; +} + +// _loop1_84: ('and' inversion) +static asdl_seq * +_loop1_84_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + void *res = NULL; + int mark = p->mark; + int start_mark = p->mark; + void **children = PyMem_Malloc(sizeof(void *)); + if (!children) { + PyErr_Format(PyExc_MemoryError, "Parser out of memory"); + return NULL; + } + ssize_t children_capacity = 1; + ssize_t n = 0; + { // ('and' inversion) + void *_tmp_132_var; + while ( + (_tmp_132_var = _tmp_132_rule(p)) + ) + { + res = _tmp_132_var; + if (n == children_capacity) { + children_capacity *= 2; + children = PyMem_Realloc(children, children_capacity*sizeof(void *)); + if (!children) { + PyErr_Format(PyExc_MemoryError, "realloc None"); + return NULL; + } + } + children[n++] = res; + mark = p->mark; + } + p->mark = mark; + } + if (n == 0) { + PyMem_Free(children); + return NULL; + } + asdl_seq *seq = _Py_asdl_seq_new(n, p->arena); + if (!seq) { + PyErr_Format(PyExc_MemoryError, "asdl_seq_new _loop1_84"); + PyMem_Free(children); + return NULL; + } + for (int i = 0; i < n; i++) asdl_seq_SET(seq, i, children[i]); + PyMem_Free(children); + _PyPegen_insert_memo(p, start_mark, _loop1_84_type, seq); + return seq; +} + +// _loop1_85: compare_op_bitwise_or_pair +static asdl_seq * +_loop1_85_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + void *res = NULL; + int mark = p->mark; + int start_mark = p->mark; + void **children = PyMem_Malloc(sizeof(void *)); + if (!children) { + PyErr_Format(PyExc_MemoryError, "Parser out of memory"); + return NULL; + } + ssize_t children_capacity = 1; + ssize_t n = 0; + { // compare_op_bitwise_or_pair + CmpopExprPair* compare_op_bitwise_or_pair_var; + while ( + (compare_op_bitwise_or_pair_var = compare_op_bitwise_or_pair_rule(p)) + ) + { + res = compare_op_bitwise_or_pair_var; + if (n == children_capacity) { + children_capacity *= 2; + children = PyMem_Realloc(children, children_capacity*sizeof(void *)); + if (!children) { + PyErr_Format(PyExc_MemoryError, "realloc None"); + return NULL; + } + } + children[n++] = res; + mark = p->mark; + } + p->mark = mark; + } + if (n == 0) { + PyMem_Free(children); + return NULL; + } + asdl_seq *seq = _Py_asdl_seq_new(n, p->arena); + if (!seq) { + PyErr_Format(PyExc_MemoryError, "asdl_seq_new _loop1_85"); + PyMem_Free(children); + return NULL; + } + for (int i = 0; i < n; i++) asdl_seq_SET(seq, i, children[i]); + PyMem_Free(children); + _PyPegen_insert_memo(p, start_mark, _loop1_85_type, seq); + return seq; +} + +// _loop0_87: ',' slice +static asdl_seq * +_loop0_87_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + void *res = NULL; + int mark = p->mark; + int start_mark = p->mark; + void **children = PyMem_Malloc(sizeof(void *)); + if (!children) { + PyErr_Format(PyExc_MemoryError, "Parser out of memory"); + return NULL; + } + ssize_t children_capacity = 1; + ssize_t n = 0; + { // ',' slice + expr_ty elem; + void *literal; + while ( + (literal = _PyPegen_expect_token(p, 12)) + && + (elem = slice_rule(p)) + ) + { + res = elem; + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + PyMem_Free(children); + return NULL; + } + if (n == children_capacity) { + children_capacity *= 2; + children = PyMem_Realloc(children, children_capacity*sizeof(void *)); + if (!children) { + PyErr_Format(PyExc_MemoryError, "realloc None"); + return NULL; + } + } + children[n++] = res; + mark = p->mark; + } + p->mark = mark; + } + asdl_seq *seq = _Py_asdl_seq_new(n, p->arena); + if (!seq) { + PyErr_Format(PyExc_MemoryError, "asdl_seq_new _loop0_87"); + PyMem_Free(children); + return NULL; + } + for (int i = 0; i < n; i++) asdl_seq_SET(seq, i, children[i]); + PyMem_Free(children); + _PyPegen_insert_memo(p, start_mark, _loop0_87_type, seq); + return seq; +} + +// _gather_86: slice _loop0_87 +static asdl_seq * +_gather_86_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + asdl_seq * res = NULL; + int mark = p->mark; + { // slice _loop0_87 + expr_ty elem; + asdl_seq * seq; + if ( + (elem = slice_rule(p)) + && + (seq = _loop0_87_rule(p)) + ) + { + res = _PyPegen_seq_insert_in_front(p, elem, seq); + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// _tmp_88: ':' expression? +static void * +_tmp_88_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + void * res = NULL; + int mark = p->mark; + { // ':' expression? + void *d; + void *literal; + if ( + (literal = _PyPegen_expect_token(p, 11)) + && + (d = expression_rule(p), 1) + ) + { + res = d; + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// _tmp_89: tuple | group | genexp +static void * +_tmp_89_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + void * res = NULL; + int mark = p->mark; + { // tuple + expr_ty tuple_var; + if ( + (tuple_var = tuple_rule(p)) + ) + { + res = tuple_var; + goto done; + } + p->mark = mark; + } + { // group + expr_ty group_var; + if ( + (group_var = group_rule(p)) + ) + { + res = group_var; + goto done; + } + p->mark = mark; + } + { // genexp + expr_ty genexp_var; + if ( + (genexp_var = genexp_rule(p)) + ) + { + res = genexp_var; + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// _tmp_90: list | listcomp +static void * +_tmp_90_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + void * res = NULL; + int mark = p->mark; + { // list + expr_ty list_var; + if ( + (list_var = list_rule(p)) + ) + { + res = list_var; + goto done; + } + p->mark = mark; + } + { // listcomp + expr_ty listcomp_var; + if ( + (listcomp_var = listcomp_rule(p)) + ) + { + res = listcomp_var; + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// _tmp_91: dict | set | dictcomp | setcomp +static void * +_tmp_91_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + void * res = NULL; + int mark = p->mark; + { // dict + expr_ty dict_var; + if ( + (dict_var = dict_rule(p)) + ) + { + res = dict_var; + goto done; + } + p->mark = mark; + } + { // set + expr_ty set_var; + if ( + (set_var = set_rule(p)) + ) + { + res = set_var; + goto done; + } + p->mark = mark; + } + { // dictcomp + expr_ty dictcomp_var; + if ( + (dictcomp_var = dictcomp_rule(p)) + ) + { + res = dictcomp_var; + goto done; + } + p->mark = mark; + } + { // setcomp + expr_ty setcomp_var; + if ( + (setcomp_var = setcomp_rule(p)) + ) + { + res = setcomp_var; + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// _loop1_92: STRING +static asdl_seq * +_loop1_92_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + void *res = NULL; + int mark = p->mark; + int start_mark = p->mark; + void **children = PyMem_Malloc(sizeof(void *)); + if (!children) { + PyErr_Format(PyExc_MemoryError, "Parser out of memory"); + return NULL; + } + ssize_t children_capacity = 1; + ssize_t n = 0; + { // STRING + expr_ty string_var; + while ( + (string_var = _PyPegen_string_token(p)) + ) + { + res = string_var; + if (n == children_capacity) { + children_capacity *= 2; + children = PyMem_Realloc(children, children_capacity*sizeof(void *)); + if (!children) { + PyErr_Format(PyExc_MemoryError, "realloc None"); + return NULL; + } + } + children[n++] = res; + mark = p->mark; + } + p->mark = mark; + } + if (n == 0) { + PyMem_Free(children); + return NULL; + } + asdl_seq *seq = _Py_asdl_seq_new(n, p->arena); + if (!seq) { + PyErr_Format(PyExc_MemoryError, "asdl_seq_new _loop1_92"); + PyMem_Free(children); + return NULL; + } + for (int i = 0; i < n; i++) asdl_seq_SET(seq, i, children[i]); + PyMem_Free(children); + _PyPegen_insert_memo(p, start_mark, _loop1_92_type, seq); + return seq; +} + +// _tmp_93: star_named_expression ',' star_named_expressions? +static void * +_tmp_93_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + void * res = NULL; + int mark = p->mark; + { // star_named_expression ',' star_named_expressions? + void *literal; + expr_ty y; + void *z; + if ( + (y = star_named_expression_rule(p)) + && + (literal = _PyPegen_expect_token(p, 12)) + && + (z = star_named_expressions_rule(p), 1) + ) + { + res = _PyPegen_seq_insert_in_front ( p , y , z ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// _tmp_94: yield_expr | named_expression +static void * +_tmp_94_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + void * res = NULL; + int mark = p->mark; + { // yield_expr + expr_ty yield_expr_var; + if ( + (yield_expr_var = yield_expr_rule(p)) + ) + { + res = yield_expr_var; + goto done; + } + p->mark = mark; + } + { // named_expression + expr_ty named_expression_var; + if ( + (named_expression_var = named_expression_rule(p)) + ) + { + res = named_expression_var; + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// _loop0_96: ',' kvpair +static asdl_seq * +_loop0_96_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + void *res = NULL; + int mark = p->mark; + int start_mark = p->mark; + void **children = PyMem_Malloc(sizeof(void *)); + if (!children) { + PyErr_Format(PyExc_MemoryError, "Parser out of memory"); + return NULL; + } + ssize_t children_capacity = 1; + ssize_t n = 0; + { // ',' kvpair + KeyValuePair* elem; + void *literal; + while ( + (literal = _PyPegen_expect_token(p, 12)) + && + (elem = kvpair_rule(p)) + ) + { + res = elem; + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + PyMem_Free(children); + return NULL; + } + if (n == children_capacity) { + children_capacity *= 2; + children = PyMem_Realloc(children, children_capacity*sizeof(void *)); + if (!children) { + PyErr_Format(PyExc_MemoryError, "realloc None"); + return NULL; + } + } + children[n++] = res; + mark = p->mark; + } + p->mark = mark; + } + asdl_seq *seq = _Py_asdl_seq_new(n, p->arena); + if (!seq) { + PyErr_Format(PyExc_MemoryError, "asdl_seq_new _loop0_96"); + PyMem_Free(children); + return NULL; + } + for (int i = 0; i < n; i++) asdl_seq_SET(seq, i, children[i]); + PyMem_Free(children); + _PyPegen_insert_memo(p, start_mark, _loop0_96_type, seq); + return seq; +} + +// _gather_95: kvpair _loop0_96 +static asdl_seq * +_gather_95_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + asdl_seq * res = NULL; + int mark = p->mark; + { // kvpair _loop0_96 + KeyValuePair* elem; + asdl_seq * seq; + if ( + (elem = kvpair_rule(p)) + && + (seq = _loop0_96_rule(p)) + ) + { + res = _PyPegen_seq_insert_in_front(p, elem, seq); + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// _loop1_97: (ASYNC? 'for' star_targets 'in' disjunction (('if' disjunction))*) +static asdl_seq * +_loop1_97_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + void *res = NULL; + int mark = p->mark; + int start_mark = p->mark; + void **children = PyMem_Malloc(sizeof(void *)); + if (!children) { + PyErr_Format(PyExc_MemoryError, "Parser out of memory"); + return NULL; + } + ssize_t children_capacity = 1; + ssize_t n = 0; + { // (ASYNC? 'for' star_targets 'in' disjunction (('if' disjunction))*) + void *_tmp_133_var; + while ( + (_tmp_133_var = _tmp_133_rule(p)) + ) + { + res = _tmp_133_var; + if (n == children_capacity) { + children_capacity *= 2; + children = PyMem_Realloc(children, children_capacity*sizeof(void *)); + if (!children) { + PyErr_Format(PyExc_MemoryError, "realloc None"); + return NULL; + } + } + children[n++] = res; + mark = p->mark; + } + p->mark = mark; + } + if (n == 0) { + PyMem_Free(children); + return NULL; + } + asdl_seq *seq = _Py_asdl_seq_new(n, p->arena); + if (!seq) { + PyErr_Format(PyExc_MemoryError, "asdl_seq_new _loop1_97"); + PyMem_Free(children); + return NULL; + } + for (int i = 0; i < n; i++) asdl_seq_SET(seq, i, children[i]); + PyMem_Free(children); + _PyPegen_insert_memo(p, start_mark, _loop1_97_type, seq); + return seq; +} + +// _tmp_98: ',' args +static void * +_tmp_98_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + void * res = NULL; + int mark = p->mark; + { // ',' args + expr_ty c; + void *literal; + if ( + (literal = _PyPegen_expect_token(p, 12)) + && + (c = args_rule(p)) + ) + { + res = c; + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// _tmp_99: ',' args +static void * +_tmp_99_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + void * res = NULL; + int mark = p->mark; + { // ',' args + expr_ty c; + void *literal; + if ( + (literal = _PyPegen_expect_token(p, 12)) + && + (c = args_rule(p)) + ) + { + res = c; + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// _loop0_101: ',' kwarg_or_starred +static asdl_seq * +_loop0_101_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + void *res = NULL; + int mark = p->mark; + int start_mark = p->mark; + void **children = PyMem_Malloc(sizeof(void *)); + if (!children) { + PyErr_Format(PyExc_MemoryError, "Parser out of memory"); + return NULL; + } + ssize_t children_capacity = 1; + ssize_t n = 0; + { // ',' kwarg_or_starred + KeywordOrStarred* elem; + void *literal; + while ( + (literal = _PyPegen_expect_token(p, 12)) + && + (elem = kwarg_or_starred_rule(p)) + ) + { + res = elem; + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + PyMem_Free(children); + return NULL; + } + if (n == children_capacity) { + children_capacity *= 2; + children = PyMem_Realloc(children, children_capacity*sizeof(void *)); + if (!children) { + PyErr_Format(PyExc_MemoryError, "realloc None"); + return NULL; + } + } + children[n++] = res; + mark = p->mark; + } + p->mark = mark; + } + asdl_seq *seq = _Py_asdl_seq_new(n, p->arena); + if (!seq) { + PyErr_Format(PyExc_MemoryError, "asdl_seq_new _loop0_101"); + PyMem_Free(children); + return NULL; + } + for (int i = 0; i < n; i++) asdl_seq_SET(seq, i, children[i]); + PyMem_Free(children); + _PyPegen_insert_memo(p, start_mark, _loop0_101_type, seq); + return seq; +} + +// _gather_100: kwarg_or_starred _loop0_101 +static asdl_seq * +_gather_100_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + asdl_seq * res = NULL; + int mark = p->mark; + { // kwarg_or_starred _loop0_101 + KeywordOrStarred* elem; + asdl_seq * seq; + if ( + (elem = kwarg_or_starred_rule(p)) + && + (seq = _loop0_101_rule(p)) + ) + { + res = _PyPegen_seq_insert_in_front(p, elem, seq); + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// _loop0_103: ',' kwarg_or_double_starred +static asdl_seq * +_loop0_103_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + void *res = NULL; + int mark = p->mark; + int start_mark = p->mark; + void **children = PyMem_Malloc(sizeof(void *)); + if (!children) { + PyErr_Format(PyExc_MemoryError, "Parser out of memory"); + return NULL; + } + ssize_t children_capacity = 1; + ssize_t n = 0; + { // ',' kwarg_or_double_starred + KeywordOrStarred* elem; + void *literal; + while ( + (literal = _PyPegen_expect_token(p, 12)) + && + (elem = kwarg_or_double_starred_rule(p)) + ) + { + res = elem; + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + PyMem_Free(children); + return NULL; + } + if (n == children_capacity) { + children_capacity *= 2; + children = PyMem_Realloc(children, children_capacity*sizeof(void *)); + if (!children) { + PyErr_Format(PyExc_MemoryError, "realloc None"); + return NULL; + } + } + children[n++] = res; + mark = p->mark; + } + p->mark = mark; + } + asdl_seq *seq = _Py_asdl_seq_new(n, p->arena); + if (!seq) { + PyErr_Format(PyExc_MemoryError, "asdl_seq_new _loop0_103"); + PyMem_Free(children); + return NULL; + } + for (int i = 0; i < n; i++) asdl_seq_SET(seq, i, children[i]); + PyMem_Free(children); + _PyPegen_insert_memo(p, start_mark, _loop0_103_type, seq); + return seq; +} + +// _gather_102: kwarg_or_double_starred _loop0_103 +static asdl_seq * +_gather_102_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + asdl_seq * res = NULL; + int mark = p->mark; + { // kwarg_or_double_starred _loop0_103 + KeywordOrStarred* elem; + asdl_seq * seq; + if ( + (elem = kwarg_or_double_starred_rule(p)) + && + (seq = _loop0_103_rule(p)) + ) + { + res = _PyPegen_seq_insert_in_front(p, elem, seq); + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// _loop0_105: ',' kwarg_or_starred +static asdl_seq * +_loop0_105_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + void *res = NULL; + int mark = p->mark; + int start_mark = p->mark; + void **children = PyMem_Malloc(sizeof(void *)); + if (!children) { + PyErr_Format(PyExc_MemoryError, "Parser out of memory"); + return NULL; + } + ssize_t children_capacity = 1; + ssize_t n = 0; + { // ',' kwarg_or_starred + KeywordOrStarred* elem; + void *literal; + while ( + (literal = _PyPegen_expect_token(p, 12)) + && + (elem = kwarg_or_starred_rule(p)) + ) + { + res = elem; + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + PyMem_Free(children); + return NULL; + } + if (n == children_capacity) { + children_capacity *= 2; + children = PyMem_Realloc(children, children_capacity*sizeof(void *)); + if (!children) { + PyErr_Format(PyExc_MemoryError, "realloc None"); + return NULL; + } + } + children[n++] = res; + mark = p->mark; + } + p->mark = mark; + } + asdl_seq *seq = _Py_asdl_seq_new(n, p->arena); + if (!seq) { + PyErr_Format(PyExc_MemoryError, "asdl_seq_new _loop0_105"); + PyMem_Free(children); + return NULL; + } + for (int i = 0; i < n; i++) asdl_seq_SET(seq, i, children[i]); + PyMem_Free(children); + _PyPegen_insert_memo(p, start_mark, _loop0_105_type, seq); + return seq; +} + +// _gather_104: kwarg_or_starred _loop0_105 +static asdl_seq * +_gather_104_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + asdl_seq * res = NULL; + int mark = p->mark; + { // kwarg_or_starred _loop0_105 + KeywordOrStarred* elem; + asdl_seq * seq; + if ( + (elem = kwarg_or_starred_rule(p)) + && + (seq = _loop0_105_rule(p)) + ) + { + res = _PyPegen_seq_insert_in_front(p, elem, seq); + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// _loop0_107: ',' kwarg_or_double_starred +static asdl_seq * +_loop0_107_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + void *res = NULL; + int mark = p->mark; + int start_mark = p->mark; + void **children = PyMem_Malloc(sizeof(void *)); + if (!children) { + PyErr_Format(PyExc_MemoryError, "Parser out of memory"); + return NULL; + } + ssize_t children_capacity = 1; + ssize_t n = 0; + { // ',' kwarg_or_double_starred + KeywordOrStarred* elem; + void *literal; + while ( + (literal = _PyPegen_expect_token(p, 12)) + && + (elem = kwarg_or_double_starred_rule(p)) + ) + { + res = elem; + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + PyMem_Free(children); + return NULL; + } + if (n == children_capacity) { + children_capacity *= 2; + children = PyMem_Realloc(children, children_capacity*sizeof(void *)); + if (!children) { + PyErr_Format(PyExc_MemoryError, "realloc None"); + return NULL; + } + } + children[n++] = res; + mark = p->mark; + } + p->mark = mark; + } + asdl_seq *seq = _Py_asdl_seq_new(n, p->arena); + if (!seq) { + PyErr_Format(PyExc_MemoryError, "asdl_seq_new _loop0_107"); + PyMem_Free(children); + return NULL; + } + for (int i = 0; i < n; i++) asdl_seq_SET(seq, i, children[i]); + PyMem_Free(children); + _PyPegen_insert_memo(p, start_mark, _loop0_107_type, seq); + return seq; +} + +// _gather_106: kwarg_or_double_starred _loop0_107 +static asdl_seq * +_gather_106_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + asdl_seq * res = NULL; + int mark = p->mark; + { // kwarg_or_double_starred _loop0_107 + KeywordOrStarred* elem; + asdl_seq * seq; + if ( + (elem = kwarg_or_double_starred_rule(p)) + && + (seq = _loop0_107_rule(p)) + ) + { + res = _PyPegen_seq_insert_in_front(p, elem, seq); + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// _loop0_108: (',' star_target) +static asdl_seq * +_loop0_108_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + void *res = NULL; + int mark = p->mark; + int start_mark = p->mark; + void **children = PyMem_Malloc(sizeof(void *)); + if (!children) { + PyErr_Format(PyExc_MemoryError, "Parser out of memory"); + return NULL; + } + ssize_t children_capacity = 1; + ssize_t n = 0; + { // (',' star_target) + void *_tmp_134_var; + while ( + (_tmp_134_var = _tmp_134_rule(p)) + ) + { + res = _tmp_134_var; + if (n == children_capacity) { + children_capacity *= 2; + children = PyMem_Realloc(children, children_capacity*sizeof(void *)); + if (!children) { + PyErr_Format(PyExc_MemoryError, "realloc None"); + return NULL; + } + } + children[n++] = res; + mark = p->mark; + } + p->mark = mark; + } + asdl_seq *seq = _Py_asdl_seq_new(n, p->arena); + if (!seq) { + PyErr_Format(PyExc_MemoryError, "asdl_seq_new _loop0_108"); + PyMem_Free(children); + return NULL; + } + for (int i = 0; i < n; i++) asdl_seq_SET(seq, i, children[i]); + PyMem_Free(children); + _PyPegen_insert_memo(p, start_mark, _loop0_108_type, seq); + return seq; +} + +// _loop0_110: ',' star_target +static asdl_seq * +_loop0_110_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + void *res = NULL; + int mark = p->mark; + int start_mark = p->mark; + void **children = PyMem_Malloc(sizeof(void *)); + if (!children) { + PyErr_Format(PyExc_MemoryError, "Parser out of memory"); + return NULL; + } + ssize_t children_capacity = 1; + ssize_t n = 0; + { // ',' star_target + expr_ty elem; + void *literal; + while ( + (literal = _PyPegen_expect_token(p, 12)) + && + (elem = star_target_rule(p)) + ) + { + res = elem; + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + PyMem_Free(children); + return NULL; + } + if (n == children_capacity) { + children_capacity *= 2; + children = PyMem_Realloc(children, children_capacity*sizeof(void *)); + if (!children) { + PyErr_Format(PyExc_MemoryError, "realloc None"); + return NULL; + } + } + children[n++] = res; + mark = p->mark; + } + p->mark = mark; + } + asdl_seq *seq = _Py_asdl_seq_new(n, p->arena); + if (!seq) { + PyErr_Format(PyExc_MemoryError, "asdl_seq_new _loop0_110"); + PyMem_Free(children); + return NULL; + } + for (int i = 0; i < n; i++) asdl_seq_SET(seq, i, children[i]); + PyMem_Free(children); + _PyPegen_insert_memo(p, start_mark, _loop0_110_type, seq); + return seq; +} + +// _gather_109: star_target _loop0_110 +static asdl_seq * +_gather_109_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + asdl_seq * res = NULL; + int mark = p->mark; + { // star_target _loop0_110 + expr_ty elem; + asdl_seq * seq; + if ( + (elem = star_target_rule(p)) + && + (seq = _loop0_110_rule(p)) + ) + { + res = _PyPegen_seq_insert_in_front(p, elem, seq); + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// _tmp_111: !'*' star_target +static void * +_tmp_111_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + void * res = NULL; + int mark = p->mark; + { // !'*' star_target + expr_ty star_target_var; + if ( + _PyPegen_lookahead_with_int(0, _PyPegen_expect_token, p, 16) + && + (star_target_var = star_target_rule(p)) + ) + { + res = star_target_var; + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// _loop0_113: ',' del_target +static asdl_seq * +_loop0_113_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + void *res = NULL; + int mark = p->mark; + int start_mark = p->mark; + void **children = PyMem_Malloc(sizeof(void *)); + if (!children) { + PyErr_Format(PyExc_MemoryError, "Parser out of memory"); + return NULL; + } + ssize_t children_capacity = 1; + ssize_t n = 0; + { // ',' del_target + expr_ty elem; + void *literal; + while ( + (literal = _PyPegen_expect_token(p, 12)) + && + (elem = del_target_rule(p)) + ) + { + res = elem; + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + PyMem_Free(children); + return NULL; + } + if (n == children_capacity) { + children_capacity *= 2; + children = PyMem_Realloc(children, children_capacity*sizeof(void *)); + if (!children) { + PyErr_Format(PyExc_MemoryError, "realloc None"); + return NULL; + } + } + children[n++] = res; + mark = p->mark; + } + p->mark = mark; + } + asdl_seq *seq = _Py_asdl_seq_new(n, p->arena); + if (!seq) { + PyErr_Format(PyExc_MemoryError, "asdl_seq_new _loop0_113"); + PyMem_Free(children); + return NULL; + } + for (int i = 0; i < n; i++) asdl_seq_SET(seq, i, children[i]); + PyMem_Free(children); + _PyPegen_insert_memo(p, start_mark, _loop0_113_type, seq); + return seq; +} + +// _gather_112: del_target _loop0_113 +static asdl_seq * +_gather_112_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + asdl_seq * res = NULL; + int mark = p->mark; + { // del_target _loop0_113 + expr_ty elem; + asdl_seq * seq; + if ( + (elem = del_target_rule(p)) + && + (seq = _loop0_113_rule(p)) + ) + { + res = _PyPegen_seq_insert_in_front(p, elem, seq); + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// _loop0_115: ',' target +static asdl_seq * +_loop0_115_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + void *res = NULL; + int mark = p->mark; + int start_mark = p->mark; + void **children = PyMem_Malloc(sizeof(void *)); + if (!children) { + PyErr_Format(PyExc_MemoryError, "Parser out of memory"); + return NULL; + } + ssize_t children_capacity = 1; + ssize_t n = 0; + { // ',' target + expr_ty elem; + void *literal; + while ( + (literal = _PyPegen_expect_token(p, 12)) + && + (elem = target_rule(p)) + ) + { + res = elem; + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + PyMem_Free(children); + return NULL; + } + if (n == children_capacity) { + children_capacity *= 2; + children = PyMem_Realloc(children, children_capacity*sizeof(void *)); + if (!children) { + PyErr_Format(PyExc_MemoryError, "realloc None"); + return NULL; + } + } + children[n++] = res; + mark = p->mark; + } + p->mark = mark; + } + asdl_seq *seq = _Py_asdl_seq_new(n, p->arena); + if (!seq) { + PyErr_Format(PyExc_MemoryError, "asdl_seq_new _loop0_115"); + PyMem_Free(children); + return NULL; + } + for (int i = 0; i < n; i++) asdl_seq_SET(seq, i, children[i]); + PyMem_Free(children); + _PyPegen_insert_memo(p, start_mark, _loop0_115_type, seq); + return seq; +} + +// _gather_114: target _loop0_115 +static asdl_seq * +_gather_114_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + asdl_seq * res = NULL; + int mark = p->mark; + { // target _loop0_115 + expr_ty elem; + asdl_seq * seq; + if ( + (elem = target_rule(p)) + && + (seq = _loop0_115_rule(p)) + ) + { + res = _PyPegen_seq_insert_in_front(p, elem, seq); + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// _tmp_116: args | expression for_if_clauses +static void * +_tmp_116_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + void * res = NULL; + int mark = p->mark; + { // args + expr_ty args_var; + if ( + (args_var = args_rule(p)) + ) + { + res = args_var; + goto done; + } + p->mark = mark; + } + { // expression for_if_clauses + expr_ty expression_var; + asdl_seq* for_if_clauses_var; + if ( + (expression_var = expression_rule(p)) + && + (for_if_clauses_var = for_if_clauses_rule(p)) + ) + { + res = _PyPegen_dummy_name(p, expression_var, for_if_clauses_var); + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// _tmp_117: '=' annotated_rhs +static void * +_tmp_117_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + void * res = NULL; + int mark = p->mark; + { // '=' annotated_rhs + expr_ty annotated_rhs_var; + void *literal; + if ( + (literal = _PyPegen_expect_token(p, 22)) + && + (annotated_rhs_var = annotated_rhs_rule(p)) + ) + { + res = _PyPegen_dummy_name(p, literal, annotated_rhs_var); + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// _tmp_118: '=' | augassign +static void * +_tmp_118_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + void * res = NULL; + int mark = p->mark; + { // '=' + void *literal; + if ( + (literal = _PyPegen_expect_token(p, 22)) + ) + { + res = literal; + goto done; + } + p->mark = mark; + } + { // augassign + AugOperator* augassign_var; + if ( + (augassign_var = augassign_rule(p)) + ) + { + res = augassign_var; + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// _tmp_119: yield_expr | star_expressions +static void * +_tmp_119_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + void * res = NULL; + int mark = p->mark; + { // yield_expr + expr_ty yield_expr_var; + if ( + (yield_expr_var = yield_expr_rule(p)) + ) + { + res = yield_expr_var; + goto done; + } + p->mark = mark; + } + { // star_expressions + expr_ty star_expressions_var; + if ( + (star_expressions_var = star_expressions_rule(p)) + ) + { + res = star_expressions_var; + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// _tmp_120: '[' | '(' | '{' +static void * +_tmp_120_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + void * res = NULL; + int mark = p->mark; + { // '[' + void *literal; + if ( + (literal = _PyPegen_expect_token(p, 9)) + ) + { + res = literal; + goto done; + } + p->mark = mark; + } + { // '(' + void *literal; + if ( + (literal = _PyPegen_expect_token(p, 7)) + ) + { + res = literal; + goto done; + } + p->mark = mark; + } + { // '{' + void *literal; + if ( + (literal = _PyPegen_expect_token(p, 25)) + ) + { + res = literal; + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// _tmp_121: plain_names ',' +static void * +_tmp_121_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + void * res = NULL; + int mark = p->mark; + { // plain_names ',' + void *literal; + asdl_seq* plain_names_var; + if ( + (plain_names_var = plain_names_rule(p)) + && + (literal = _PyPegen_expect_token(p, 12)) + ) + { + res = _PyPegen_dummy_name(p, plain_names_var, literal); + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// _tmp_122: slash_with_default | names_with_default +static void * +_tmp_122_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + void * res = NULL; + int mark = p->mark; + { // slash_with_default + SlashWithDefault* slash_with_default_var; + if ( + (slash_with_default_var = slash_with_default_rule(p)) + ) + { + res = slash_with_default_var; + goto done; + } + p->mark = mark; + } + { // names_with_default + asdl_seq* names_with_default_var; + if ( + (names_with_default_var = names_with_default_rule(p)) + ) + { + res = names_with_default_var; + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// _tmp_123: star_targets '=' +static void * +_tmp_123_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + void * res = NULL; + int mark = p->mark; + { // star_targets '=' + void *literal; + expr_ty z; + if ( + (z = star_targets_rule(p)) + && + (literal = _PyPegen_expect_token(p, 22)) + ) + { + res = z; + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// _tmp_124: '.' | '...' +static void * +_tmp_124_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + void * res = NULL; + int mark = p->mark; + { // '.' + void *literal; + if ( + (literal = _PyPegen_expect_token(p, 23)) + ) + { + res = literal; + goto done; + } + p->mark = mark; + } + { // '...' + void *literal; + if ( + (literal = _PyPegen_expect_token(p, 52)) + ) + { + res = literal; + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// _tmp_125: '.' | '...' +static void * +_tmp_125_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + void * res = NULL; + int mark = p->mark; + { // '.' + void *literal; + if ( + (literal = _PyPegen_expect_token(p, 23)) + ) + { + res = literal; + goto done; + } + p->mark = mark; + } + { // '...' + void *literal; + if ( + (literal = _PyPegen_expect_token(p, 52)) + ) + { + res = literal; + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// _tmp_126: plain_name !'=' +static void * +_tmp_126_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + void * res = NULL; + int mark = p->mark; + { // plain_name !'=' + arg_ty plain_name_var; + if ( + (plain_name_var = plain_name_rule(p)) + && + _PyPegen_lookahead_with_int(0, _PyPegen_expect_token, p, 22) + ) + { + res = plain_name_var; + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// _tmp_127: '@' named_expression NEWLINE +static void * +_tmp_127_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + void * res = NULL; + int mark = p->mark; + { // '@' named_expression NEWLINE + expr_ty f; + void *literal; + void *newline_var; + if ( + (literal = _PyPegen_expect_token(p, 49)) + && + (f = named_expression_rule(p)) + && + (newline_var = _PyPegen_newline_token(p)) + ) + { + res = f; + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// _tmp_128: ',' star_expression +static void * +_tmp_128_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + void * res = NULL; + int mark = p->mark; + { // ',' star_expression + expr_ty c; + void *literal; + if ( + (literal = _PyPegen_expect_token(p, 12)) + && + (c = star_expression_rule(p)) + ) + { + res = c; + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// _tmp_129: ',' expression +static void * +_tmp_129_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + void * res = NULL; + int mark = p->mark; + { // ',' expression + expr_ty c; + void *literal; + if ( + (literal = _PyPegen_expect_token(p, 12)) + && + (c = expression_rule(p)) + ) + { + res = c; + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// _tmp_130: lambda_plain_name !'=' +static void * +_tmp_130_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + void * res = NULL; + int mark = p->mark; + { // lambda_plain_name !'=' + arg_ty lambda_plain_name_var; + if ( + (lambda_plain_name_var = lambda_plain_name_rule(p)) + && + _PyPegen_lookahead_with_int(0, _PyPegen_expect_token, p, 22) + ) + { + res = lambda_plain_name_var; + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// _tmp_131: 'or' conjunction +static void * +_tmp_131_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + void * res = NULL; + int mark = p->mark; + { // 'or' conjunction + expr_ty c; + void *keyword; + if ( + (keyword = _PyPegen_expect_token(p, 532)) + && + (c = conjunction_rule(p)) + ) + { + res = c; + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// _tmp_132: 'and' inversion +static void * +_tmp_132_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + void * res = NULL; + int mark = p->mark; + { // 'and' inversion + expr_ty c; + void *keyword; + if ( + (keyword = _PyPegen_expect_token(p, 533)) + && + (c = inversion_rule(p)) + ) + { + res = c; + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// _tmp_133: ASYNC? 'for' star_targets 'in' disjunction (('if' disjunction))* +static void * +_tmp_133_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + void * res = NULL; + int mark = p->mark; + { // ASYNC? 'for' star_targets 'in' disjunction (('if' disjunction))* + expr_ty a; + expr_ty b; + asdl_seq * c; + void *keyword; + void *keyword_1; + void *y; + if ( + (y = _PyPegen_async_token(p), 1) + && + (keyword = _PyPegen_expect_token(p, 517)) + && + (a = star_targets_rule(p)) + && + (keyword_1 = _PyPegen_expect_token(p, 518)) + && + (b = disjunction_rule(p)) + && + (c = _loop0_135_rule(p)) + ) + { + res = _Py_comprehension ( a , b , c , y != NULL , p -> arena ); + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// _tmp_134: ',' star_target +static void * +_tmp_134_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + void * res = NULL; + int mark = p->mark; + { // ',' star_target + expr_ty c; + void *literal; + if ( + (literal = _PyPegen_expect_token(p, 12)) + && + (c = star_target_rule(p)) + ) + { + res = c; + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// _loop0_135: ('if' disjunction) +static asdl_seq * +_loop0_135_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + void *res = NULL; + int mark = p->mark; + int start_mark = p->mark; + void **children = PyMem_Malloc(sizeof(void *)); + if (!children) { + PyErr_Format(PyExc_MemoryError, "Parser out of memory"); + return NULL; + } + ssize_t children_capacity = 1; + ssize_t n = 0; + { // ('if' disjunction) + void *_tmp_136_var; + while ( + (_tmp_136_var = _tmp_136_rule(p)) + ) + { + res = _tmp_136_var; + if (n == children_capacity) { + children_capacity *= 2; + children = PyMem_Realloc(children, children_capacity*sizeof(void *)); + if (!children) { + PyErr_Format(PyExc_MemoryError, "realloc None"); + return NULL; + } + } + children[n++] = res; + mark = p->mark; + } + p->mark = mark; + } + asdl_seq *seq = _Py_asdl_seq_new(n, p->arena); + if (!seq) { + PyErr_Format(PyExc_MemoryError, "asdl_seq_new _loop0_135"); + PyMem_Free(children); + return NULL; + } + for (int i = 0; i < n; i++) asdl_seq_SET(seq, i, children[i]); + PyMem_Free(children); + _PyPegen_insert_memo(p, start_mark, _loop0_135_type, seq); + return seq; +} + +// _tmp_136: 'if' disjunction +static void * +_tmp_136_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + void * res = NULL; + int mark = p->mark; + { // 'if' disjunction + void *keyword; + expr_ty z; + if ( + (keyword = _PyPegen_expect_token(p, 510)) + && + (z = disjunction_rule(p)) + ) + { + res = z; + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +void * +_PyPegen_parse(Parser *p) +{ + // Initialize keywords + p->keywords = reserved_keywords; + p->n_keyword_lists = n_keyword_lists; + + // Run parser + void *result = NULL; + if (p->start_rule == Py_file_input) { + result = file_rule(p); + } else if (p->start_rule == Py_single_input) { + result = interactive_rule(p); + } else if (p->start_rule == Py_eval_input) { + result = eval_rule(p); + } else if (p->start_rule == Py_fstring_input) { + result = fstring_rule(p); + } + + return result; +} + +// The end diff --git a/Parser/pegen/parse_string.c b/Parser/pegen/parse_string.c new file mode 100644 index 0000000000000..41485a9669d68 --- /dev/null +++ b/Parser/pegen/parse_string.c @@ -0,0 +1,1387 @@ +#include <Python.h> + +#include "../tokenizer.h" +#include "pegen.h" +#include "parse_string.h" + +//// STRING HANDLING FUNCTIONS //// + +// These functions are ported directly from Python/ast.c with some modifications +// to account for the use of "Parser *p", the fact that don't have parser nodes +// to pass around and the usage of some specialized APIs present only in this +// file (like "_PyPegen_raise_syntax_error"). + +static int +warn_invalid_escape_sequence(Parser *p, unsigned char first_invalid_escape_char) +{ + PyObject *msg = + PyUnicode_FromFormat("invalid escape sequence \\%c", first_invalid_escape_char); + if (msg == NULL) { + return -1; + } + if (PyErr_WarnExplicitObject(PyExc_DeprecationWarning, msg, p->tok->filename, + p->tok->lineno, NULL, NULL) < 0) { + if (PyErr_ExceptionMatches(PyExc_DeprecationWarning)) { + /* Replace the DeprecationWarning exception with a SyntaxError + to get a more accurate error report */ + PyErr_Clear(); + RAISE_SYNTAX_ERROR("invalid escape sequence \\%c", first_invalid_escape_char); + } + Py_DECREF(msg); + return -1; + } + Py_DECREF(msg); + return 0; +} + +static PyObject * +decode_utf8(const char **sPtr, const char *end) +{ + const char *s, *t; + t = s = *sPtr; + while (s < end && (*s & 0x80)) { + s++; + } + *sPtr = s; + return PyUnicode_DecodeUTF8(t, s - t, NULL); +} + +static PyObject * +decode_unicode_with_escapes(Parser *parser, const char *s, size_t len) +{ + PyObject *v, *u; + char *buf; + char *p; + const char *end; + + /* check for integer overflow */ + if (len > SIZE_MAX / 6) { + return NULL; + } + /* "ä" (2 bytes) may become "\U000000E4" (10 bytes), or 1:5 + "\ä" (3 bytes) may become "\u005c\U000000E4" (16 bytes), or ~1:6 */ + u = PyBytes_FromStringAndSize((char *)NULL, len * 6); + if (u == NULL) { + return NULL; + } + p = buf = PyBytes_AsString(u); + end = s + len; + while (s < end) { + if (*s == '\\') { + *p++ = *s++; + if (s >= end || *s & 0x80) { + strcpy(p, "u005c"); + p += 5; + if (s >= end) { + break; + } + } + } + if (*s & 0x80) { + PyObject *w; + int kind; + void *data; + Py_ssize_t len, i; + w = decode_utf8(&s, end); + if (w == NULL) { + Py_DECREF(u); + return NULL; + } + kind = PyUnicode_KIND(w); + data = PyUnicode_DATA(w); + len = PyUnicode_GET_LENGTH(w); + for (i = 0; i < len; i++) { + Py_UCS4 chr = PyUnicode_READ(kind, data, i); + sprintf(p, "\\U%08x", chr); + p += 10; + } + /* Should be impossible to overflow */ + assert(p - buf <= PyBytes_GET_SIZE(u)); + Py_DECREF(w); + } + else { + *p++ = *s++; + } + } + len = p - buf; + s = buf; + + const char *first_invalid_escape; + v = _PyUnicode_DecodeUnicodeEscape(s, len, NULL, &first_invalid_escape); + + if (v != NULL && first_invalid_escape != NULL) { + if (warn_invalid_escape_sequence(parser, *first_invalid_escape) < 0) { + /* We have not decref u before because first_invalid_escape points + inside u. */ + Py_XDECREF(u); + Py_DECREF(v); + return NULL; + } + } + Py_XDECREF(u); + return v; +} + +static PyObject * +decode_bytes_with_escapes(Parser *p, const char *s, Py_ssize_t len) +{ + const char *first_invalid_escape; + PyObject *result = _PyBytes_DecodeEscape(s, len, NULL, &first_invalid_escape); + if (result == NULL) { + return NULL; + } + + if (first_invalid_escape != NULL) { + if (warn_invalid_escape_sequence(p, *first_invalid_escape) < 0) { + Py_DECREF(result); + return NULL; + } + } + return result; +} + +/* s must include the bracketing quote characters, and r, b, u, + &/or f prefixes (if any), and embedded escape sequences (if any). + _PyPegen_parsestr parses it, and sets *result to decoded Python string object. + If the string is an f-string, set *fstr and *fstrlen to the unparsed + string object. Return 0 if no errors occurred. */ +int +_PyPegen_parsestr(Parser *p, const char *s, int *bytesmode, int *rawmode, PyObject **result, + const char **fstr, Py_ssize_t *fstrlen) +{ + size_t len; + int quote = Py_CHARMASK(*s); + int fmode = 0; + *bytesmode = 0; + *rawmode = 0; + *result = NULL; + *fstr = NULL; + if (Py_ISALPHA(quote)) { + while (!*bytesmode || !*rawmode) { + if (quote == 'b' || quote == 'B') { + quote = *++s; + *bytesmode = 1; + } + else if (quote == 'u' || quote == 'U') { + quote = *++s; + } + else if (quote == 'r' || quote == 'R') { + quote = *++s; + *rawmode = 1; + } + else if (quote == 'f' || quote == 'F') { + quote = *++s; + fmode = 1; + } + else { + break; + } + } + } + + if (fmode && *bytesmode) { + PyErr_BadInternalCall(); + return -1; + } + if (quote != '\'' && quote != '\"') { + PyErr_BadInternalCall(); + return -1; + } + /* Skip the leading quote char. */ + s++; + len = strlen(s); + if (len > INT_MAX) { + PyErr_SetString(PyExc_OverflowError, "string to parse is too long"); + return -1; + } + if (s[--len] != quote) { + /* Last quote char must match the first. */ + PyErr_BadInternalCall(); + return -1; + } + if (len >= 4 && s[0] == quote && s[1] == quote) { + /* A triple quoted string. We've already skipped one quote at + the start and one at the end of the string. Now skip the + two at the start. */ + s += 2; + len -= 2; + /* And check that the last two match. */ + if (s[--len] != quote || s[--len] != quote) { + PyErr_BadInternalCall(); + return -1; + } + } + + if (fmode) { + /* Just return the bytes. The caller will parse the resulting + string. */ + *fstr = s; + *fstrlen = len; + return 0; + } + + /* Not an f-string. */ + /* Avoid invoking escape decoding routines if possible. */ + *rawmode = *rawmode || strchr(s, '\\') == NULL; + if (*bytesmode) { + /* Disallow non-ASCII characters. */ + const char *ch; + for (ch = s; *ch; ch++) { + if (Py_CHARMASK(*ch) >= 0x80) { + RAISE_SYNTAX_ERROR( + "bytes can only contain ASCII " + "literal characters."); + return -1; + } + } + if (*rawmode) { + *result = PyBytes_FromStringAndSize(s, len); + } + else { + *result = decode_bytes_with_escapes(p, s, len); + } + } + else { + if (*rawmode) { + *result = PyUnicode_DecodeUTF8Stateful(s, len, NULL, NULL); + } + else { + *result = decode_unicode_with_escapes(p, s, len); + } + } + return *result == NULL ? -1 : 0; +} + + + +// FSTRING STUFF + +static void fstring_shift_expr_locations(expr_ty n, int lineno, int col_offset); +static void fstring_shift_argument(expr_ty parent, arg_ty args, int lineno, int col_offset); + + +static inline void shift_expr(expr_ty parent, expr_ty n, int line, int col) { + if (parent->lineno < n->lineno) { + col = 0; + } + fstring_shift_expr_locations(n, line, col); +} + +static inline void shift_arg(expr_ty parent, arg_ty n, int line, int col) { + if (parent->lineno < n->lineno) { + col = 0; + } + fstring_shift_argument(parent, n, line, col); +} + +static void fstring_shift_seq_locations(expr_ty parent, asdl_seq *seq, int lineno, int col_offset) { + for (Py_ssize_t i = 0, l = asdl_seq_LEN(seq); i < l; i++) { + expr_ty expr = asdl_seq_GET(seq, i); + if (expr == NULL){ + continue; + } + shift_expr(parent, expr, lineno, col_offset); + } +} + +static void fstring_shift_slice_locations(expr_ty parent, expr_ty slice, int lineno, int col_offset) { + switch (slice->kind) { + case Slice_kind: + if (slice->v.Slice.lower) { + shift_expr(parent, slice->v.Slice.lower, lineno, col_offset); + } + if (slice->v.Slice.upper) { + shift_expr(parent, slice->v.Slice.upper, lineno, col_offset); + } + if (slice->v.Slice.step) { + shift_expr(parent, slice->v.Slice.step, lineno, col_offset); + } + break; + case Tuple_kind: + fstring_shift_seq_locations(parent, slice->v.Tuple.elts, lineno, col_offset); + break; + default: + break; + } +} + +static void fstring_shift_comprehension(expr_ty parent, comprehension_ty comp, int lineno, int col_offset) { + shift_expr(parent, comp->target, lineno, col_offset); + shift_expr(parent, comp->iter, lineno, col_offset); + fstring_shift_seq_locations(parent, comp->ifs, lineno, col_offset); +} + +static void fstring_shift_argument(expr_ty parent, arg_ty arg, int lineno, int col_offset) { + if (arg->annotation != NULL){ + shift_expr(parent, arg->annotation, lineno, col_offset); + } + arg->col_offset = arg->col_offset + col_offset; + arg->end_col_offset = arg->end_col_offset + col_offset; + arg->lineno = arg->lineno + lineno; + arg->end_lineno = arg->end_lineno + lineno; +} + +static void fstring_shift_arguments(expr_ty parent, arguments_ty args, int lineno, int col_offset) { + for (Py_ssize_t i = 0, l = asdl_seq_LEN(args->posonlyargs); i < l; i++) { + arg_ty arg = asdl_seq_GET(args->posonlyargs, i); + shift_arg(parent, arg, lineno, col_offset); + } + + for (Py_ssize_t i = 0, l = asdl_seq_LEN(args->args); i < l; i++) { + arg_ty arg = asdl_seq_GET(args->args, i); + shift_arg(parent, arg, lineno, col_offset); + } + + if (args->vararg != NULL) { + shift_arg(parent, args->vararg, lineno, col_offset); + } + + for (Py_ssize_t i = 0, l = asdl_seq_LEN(args->kwonlyargs); i < l; i++) { + arg_ty arg = asdl_seq_GET(args->kwonlyargs, i); + shift_arg(parent, arg, lineno, col_offset); + } + + fstring_shift_seq_locations(parent, args->kw_defaults, lineno, col_offset); + + if (args->kwarg != NULL) { + shift_arg(parent, args->kwarg, lineno, col_offset); + } + + fstring_shift_seq_locations(parent, args->defaults, lineno, col_offset); +} + +static void fstring_shift_children_locations(expr_ty n, int lineno, int col_offset) { + switch (n->kind) { + case BoolOp_kind: + fstring_shift_seq_locations(n, n->v.BoolOp.values, lineno, col_offset); + break; + case NamedExpr_kind: + shift_expr(n, n->v.NamedExpr.target, lineno, col_offset); + shift_expr(n, n->v.NamedExpr.value, lineno, col_offset); + break; + case BinOp_kind: + shift_expr(n, n->v.BinOp.left, lineno, col_offset); + shift_expr(n, n->v.BinOp.right, lineno, col_offset); + break; + case UnaryOp_kind: + shift_expr(n, n->v.UnaryOp.operand, lineno, col_offset); + break; + case Lambda_kind: + fstring_shift_arguments(n, n->v.Lambda.args, lineno, col_offset); + shift_expr(n, n->v.Lambda.body, lineno, col_offset); + break; + case IfExp_kind: + shift_expr(n, n->v.IfExp.test, lineno, col_offset); + shift_expr(n, n->v.IfExp.body, lineno, col_offset); + shift_expr(n, n->v.IfExp.orelse, lineno, col_offset); + break; + case Dict_kind: + fstring_shift_seq_locations(n, n->v.Dict.keys, lineno, col_offset); + fstring_shift_seq_locations(n, n->v.Dict.values, lineno, col_offset); + break; + case Set_kind: + fstring_shift_seq_locations(n, n->v.Set.elts, lineno, col_offset); + break; + case ListComp_kind: + shift_expr(n, n->v.ListComp.elt, lineno, col_offset); + for (Py_ssize_t i = 0, l = asdl_seq_LEN(n->v.ListComp.generators); i < l; i++) { + comprehension_ty comp = asdl_seq_GET(n->v.ListComp.generators, i); + fstring_shift_comprehension(n, comp, lineno, col_offset); + } + break; + case SetComp_kind: + shift_expr(n, n->v.SetComp.elt, lineno, col_offset); + for (Py_ssize_t i = 0, l = asdl_seq_LEN(n->v.SetComp.generators); i < l; i++) { + comprehension_ty comp = asdl_seq_GET(n->v.SetComp.generators, i); + fstring_shift_comprehension(n, comp, lineno, col_offset); + } + break; + case DictComp_kind: + shift_expr(n, n->v.DictComp.key, lineno, col_offset); + shift_expr(n, n->v.DictComp.value, lineno, col_offset); + for (Py_ssize_t i = 0, l = asdl_seq_LEN(n->v.DictComp.generators); i < l; i++) { + comprehension_ty comp = asdl_seq_GET(n->v.DictComp.generators, i); + fstring_shift_comprehension(n, comp, lineno, col_offset); + } + break; + case GeneratorExp_kind: + shift_expr(n, n->v.GeneratorExp.elt, lineno, col_offset); + for (Py_ssize_t i = 0, l = asdl_seq_LEN(n->v.GeneratorExp.generators); i < l; i++) { + comprehension_ty comp = asdl_seq_GET(n->v.GeneratorExp.generators, i); + fstring_shift_comprehension(n, comp, lineno, col_offset); + } + break; + case Await_kind: + shift_expr(n, n->v.Await.value, lineno, col_offset); + break; + case Yield_kind: + shift_expr(n, n->v.Yield.value, lineno, col_offset); + break; + case YieldFrom_kind: + shift_expr(n, n->v.YieldFrom.value, lineno, col_offset); + break; + case Compare_kind: + shift_expr(n, n->v.Compare.left, lineno, col_offset); + fstring_shift_seq_locations(n, n->v.Compare.comparators, lineno, col_offset); + break; + case Call_kind: + shift_expr(n, n->v.Call.func, lineno, col_offset); + fstring_shift_seq_locations(n, n->v.Call.args, lineno, col_offset); + for (Py_ssize_t i = 0, l = asdl_seq_LEN(n->v.Call.keywords); i < l; i++) { + keyword_ty keyword = asdl_seq_GET(n->v.Call.keywords, i); + shift_expr(n, keyword->value, lineno, col_offset); + } + break; + case Attribute_kind: + shift_expr(n, n->v.Attribute.value, lineno, col_offset); + break; + case Subscript_kind: + shift_expr(n, n->v.Subscript.value, lineno, col_offset); + fstring_shift_slice_locations(n, n->v.Subscript.slice, lineno, col_offset); + shift_expr(n, n->v.Subscript.slice, lineno, col_offset); + break; + case Starred_kind: + shift_expr(n, n->v.Starred.value, lineno, col_offset); + break; + case List_kind: + fstring_shift_seq_locations(n, n->v.List.elts, lineno, col_offset); + break; + case Tuple_kind: + fstring_shift_seq_locations(n, n->v.Tuple.elts, lineno, col_offset); + break; + default: + return; + } +} + +/* Shift locations for the given node and all its children by adding `lineno` + and `col_offset` to existing locations. Note that n is the already parsed + expression. */ +static void fstring_shift_expr_locations(expr_ty n, int lineno, int col_offset) +{ + n->col_offset = n->col_offset + col_offset; + + // The following is needed, in order for nodes spanning across multiple lines + // to be shifted correctly. An example of such a node is a Call node, the closing + // parenthesis of which is not on the same line as its name. + if (n->lineno == n->end_lineno) { + n->end_col_offset = n->end_col_offset + col_offset; + } + + fstring_shift_children_locations(n, lineno, col_offset); + n->lineno = n->lineno + lineno; + n->end_lineno = n->end_lineno + lineno; +} + +/* Fix locations for the given node and its children. + + `parent` is the enclosing node. + `n` is the node which locations are going to be fixed relative to parent. + `expr_str` is the child node's string representation, including braces. +*/ +static void +fstring_fix_expr_location(Token *parent, expr_ty n, char *expr_str) +{ + char *substr = NULL; + char *start; + int lines = 0; + int cols = 0; + + if (parent && parent->bytes) { + char *parent_str = PyBytes_AsString(parent->bytes); + if (!parent_str) { + return; + } + substr = strstr(parent_str, expr_str); + if (substr) { + // The following is needed, in order to correctly shift the column + // offset, in the case that (disregarding any whitespace) a newline + // immediately follows the opening curly brace of the fstring expression. + int newline_after_brace = 1; + start = substr + 1; + while (start && *start != '}' && *start != '\n') { + if (*start != ' ' && *start != '\t' && *start != '\f') { + newline_after_brace = 0; + break; + } + start++; + } + + // Account for the characters from the last newline character to our + // left until the beginning of substr. + if (!newline_after_brace) { + start = substr; + while (start > parent_str && *start != '\n') { + start--; + } + cols += (int)(substr - start); + } + /* adjust the start based on the number of newlines encountered + before the f-string expression */ + for (char* p = parent_str; p < substr; p++) { + if (*p == '\n') { + lines++; + } + } + } + } + fstring_shift_expr_locations(n, lines, cols); +} + + +/* Compile this expression in to an expr_ty. Add parens around the + expression, in order to allow leading spaces in the expression. */ +static expr_ty +fstring_compile_expr(Parser *p, const char *expr_start, const char *expr_end, + Token *t) +{ + expr_ty expr = NULL; + char *str; + Py_ssize_t len; + const char *s; + expr_ty result = NULL; + + assert(expr_end >= expr_start); + assert(*(expr_start-1) == '{'); + assert(*expr_end == '}' || *expr_end == '!' || *expr_end == ':' || + *expr_end == '='); + + /* If the substring is all whitespace, it's an error. We need to catch this + here, and not when we call PyParser_SimpleParseStringFlagsFilename, + because turning the expression '' in to '()' would go from being invalid + to valid. */ + for (s = expr_start; s != expr_end; s++) { + char c = *s; + /* The Python parser ignores only the following whitespace + characters (\r already is converted to \n). */ + if (!(c == ' ' || c == '\t' || c == '\n' || c == '\f')) { + break; + } + } + if (s == expr_end) { + RAISE_SYNTAX_ERROR("f-string: empty expression not allowed"); + return NULL; + } + + len = expr_end - expr_start; + /* Allocate 3 extra bytes: open paren, close paren, null byte. */ + str = PyMem_RawMalloc(len + 3); + if (str == NULL) { + PyErr_NoMemory(); + return NULL; + } + + str[0] = '('; + memcpy(str+1, expr_start, len); + str[len+1] = ')'; + str[len+2] = 0; + + struct tok_state* tok = PyTokenizer_FromString(str, 1); + if (tok == NULL) { + return NULL; + } + tok->filename = PyUnicode_FromString("<fstring>"); + if (!tok->filename) { + PyTokenizer_Free(tok); + return NULL; + } + + Parser *p2 = _PyPegen_Parser_New(tok, Py_fstring_input, NULL, p->arena); + p2->starting_lineno = p->starting_lineno + p->tok->first_lineno - 1; + p2->starting_col_offset = p->tok->first_lineno == p->tok->lineno + ? p->starting_col_offset + t->col_offset : 0; + + expr = _PyPegen_run_parser(p2); + + if (expr == NULL) { + goto exit; + } + + /* Reuse str to find the correct column offset. */ + str[0] = '{'; + str[len+1] = '}'; + fstring_fix_expr_location(t, expr, str); + + result = expr; + +exit: + _PyPegen_Parser_Free(p2); + PyTokenizer_Free(tok); + return result; +} + +/* Return -1 on error. + + Return 0 if we reached the end of the literal. + + Return 1 if we haven't reached the end of the literal, but we want + the caller to process the literal up to this point. Used for + doubled braces. +*/ +static int +fstring_find_literal(Parser *p, const char **str, const char *end, int raw, + PyObject **literal, int recurse_lvl) +{ + /* Get any literal string. It ends when we hit an un-doubled left + brace (which isn't part of a unicode name escape such as + "\N{EULER CONSTANT}"), or the end of the string. */ + + const char *s = *str; + const char *literal_start = s; + int result = 0; + + assert(*literal == NULL); + while (s < end) { + char ch = *s++; + if (!raw && ch == '\\' && s < end) { + ch = *s++; + if (ch == 'N') { + if (s < end && *s++ == '{') { + while (s < end && *s++ != '}') { + } + continue; + } + break; + } + if (ch == '{' && warn_invalid_escape_sequence(p, ch) < 0) { + return -1; + } + } + if (ch == '{' || ch == '}') { + /* Check for doubled braces, but only at the top level. If + we checked at every level, then f'{0:{3}}' would fail + with the two closing braces. */ + if (recurse_lvl == 0) { + if (s < end && *s == ch) { + /* We're going to tell the caller that the literal ends + here, but that they should continue scanning. But also + skip over the second brace when we resume scanning. */ + *str = s + 1; + result = 1; + goto done; + } + + /* Where a single '{' is the start of a new expression, a + single '}' is not allowed. */ + if (ch == '}') { + *str = s - 1; + RAISE_SYNTAX_ERROR("f-string: single '}' is not allowed"); + return -1; + } + } + /* We're either at a '{', which means we're starting another + expression; or a '}', which means we're at the end of this + f-string (for a nested format_spec). */ + s--; + break; + } + } + *str = s; + assert(s <= end); + assert(s == end || *s == '{' || *s == '}'); +done: + if (literal_start != s) { + if (raw) + *literal = PyUnicode_DecodeUTF8Stateful(literal_start, + s - literal_start, + NULL, NULL); + else + *literal = decode_unicode_with_escapes(p, literal_start, + s - literal_start); + if (!*literal) + return -1; + } + return result; +} + +/* Forward declaration because parsing is recursive. */ +static expr_ty +fstring_parse(Parser *p, const char **str, const char *end, int raw, int recurse_lvl, + Token *first_token, Token* t, Token *last_token); + +/* Parse the f-string at *str, ending at end. We know *str starts an + expression (so it must be a '{'). Returns the FormattedValue node, which + includes the expression, conversion character, format_spec expression, and + optionally the text of the expression (if = is used). + + Note that I don't do a perfect job here: I don't make sure that a + closing brace doesn't match an opening paren, for example. It + doesn't need to error on all invalid expressions, just correctly + find the end of all valid ones. Any errors inside the expression + will be caught when we parse it later. + + *expression is set to the expression. For an '=' "debug" expression, + *expr_text is set to the debug text (the original text of the expression, + including the '=' and any whitespace around it, as a string object). If + not a debug expression, *expr_text set to NULL. */ +static int +fstring_find_expr(Parser *p, const char **str, const char *end, int raw, int recurse_lvl, + PyObject **expr_text, expr_ty *expression, Token *first_token, + Token *t, Token *last_token) +{ + /* Return -1 on error, else 0. */ + + const char *expr_start; + const char *expr_end; + expr_ty simple_expression; + expr_ty format_spec = NULL; /* Optional format specifier. */ + int conversion = -1; /* The conversion char. Use default if not + specified, or !r if using = and no format + spec. */ + + /* 0 if we're not in a string, else the quote char we're trying to + match (single or double quote). */ + char quote_char = 0; + + /* If we're inside a string, 1=normal, 3=triple-quoted. */ + int string_type = 0; + + /* Keep track of nesting level for braces/parens/brackets in + expressions. */ + Py_ssize_t nested_depth = 0; + char parenstack[MAXLEVEL]; + + *expr_text = NULL; + + /* Can only nest one level deep. */ + if (recurse_lvl >= 2) { + RAISE_SYNTAX_ERROR("f-string: expressions nested too deeply"); + goto error; + } + + /* The first char must be a left brace, or we wouldn't have gotten + here. Skip over it. */ + assert(**str == '{'); + *str += 1; + + expr_start = *str; + for (; *str < end; (*str)++) { + char ch; + + /* Loop invariants. */ + assert(nested_depth >= 0); + assert(*str >= expr_start && *str < end); + if (quote_char) + assert(string_type == 1 || string_type == 3); + else + assert(string_type == 0); + + ch = **str; + /* Nowhere inside an expression is a backslash allowed. */ + if (ch == '\\') { + /* Error: can't include a backslash character, inside + parens or strings or not. */ + RAISE_SYNTAX_ERROR( + "f-string expression part " + "cannot include a backslash"); + goto error; + } + if (quote_char) { + /* We're inside a string. See if we're at the end. */ + /* This code needs to implement the same non-error logic + as tok_get from tokenizer.c, at the letter_quote + label. To actually share that code would be a + nightmare. But, it's unlikely to change and is small, + so duplicate it here. Note we don't need to catch all + of the errors, since they'll be caught when parsing the + expression. We just need to match the non-error + cases. Thus we can ignore \n in single-quoted strings, + for example. Or non-terminated strings. */ + if (ch == quote_char) { + /* Does this match the string_type (single or triple + quoted)? */ + if (string_type == 3) { + if (*str+2 < end && *(*str+1) == ch && *(*str+2) == ch) { + /* We're at the end of a triple quoted string. */ + *str += 2; + string_type = 0; + quote_char = 0; + continue; + } + } else { + /* We're at the end of a normal string. */ + quote_char = 0; + string_type = 0; + continue; + } + } + } else if (ch == '\'' || ch == '"') { + /* Is this a triple quoted string? */ + if (*str+2 < end && *(*str+1) == ch && *(*str+2) == ch) { + string_type = 3; + *str += 2; + } else { + /* Start of a normal string. */ + string_type = 1; + } + /* Start looking for the end of the string. */ + quote_char = ch; + } else if (ch == '[' || ch == '{' || ch == '(') { + if (nested_depth >= MAXLEVEL) { + RAISE_SYNTAX_ERROR("f-string: too many nested parenthesis"); + goto error; + } + parenstack[nested_depth] = ch; + nested_depth++; + } else if (ch == '#') { + /* Error: can't include a comment character, inside parens + or not. */ + RAISE_SYNTAX_ERROR("f-string expression part cannot include '#'"); + goto error; + } else if (nested_depth == 0 && + (ch == '!' || ch == ':' || ch == '}' || + ch == '=' || ch == '>' || ch == '<')) { + /* See if there's a next character. */ + if (*str+1 < end) { + char next = *(*str+1); + + /* For "!=". since '=' is not an allowed conversion character, + nothing is lost in this test. */ + if ((ch == '!' && next == '=') || /* != */ + (ch == '=' && next == '=') || /* == */ + (ch == '<' && next == '=') || /* <= */ + (ch == '>' && next == '=') /* >= */ + ) { + *str += 1; + continue; + } + /* Don't get out of the loop for these, if they're single + chars (not part of 2-char tokens). If by themselves, they + don't end an expression (unlike say '!'). */ + if (ch == '>' || ch == '<') { + continue; + } + } + + /* Normal way out of this loop. */ + break; + } else if (ch == ']' || ch == '}' || ch == ')') { + if (!nested_depth) { + RAISE_SYNTAX_ERROR("f-string: unmatched '%c'", ch); + goto error; + } + nested_depth--; + int opening = parenstack[nested_depth]; + if (!((opening == '(' && ch == ')') || + (opening == '[' && ch == ']') || + (opening == '{' && ch == '}'))) + { + RAISE_SYNTAX_ERROR( + "f-string: closing parenthesis '%c' " + "does not match opening parenthesis '%c'", + ch, opening); + goto error; + } + } else { + /* Just consume this char and loop around. */ + } + } + expr_end = *str; + /* If we leave this loop in a string or with mismatched parens, we + don't care. We'll get a syntax error when compiling the + expression. But, we can produce a better error message, so + let's just do that.*/ + if (quote_char) { + RAISE_SYNTAX_ERROR("f-string: unterminated string"); + goto error; + } + if (nested_depth) { + int opening = parenstack[nested_depth - 1]; + RAISE_SYNTAX_ERROR("f-string: unmatched '%c'", opening); + goto error; + } + + if (*str >= end) + goto unexpected_end_of_string; + + /* Compile the expression as soon as possible, so we show errors + related to the expression before errors related to the + conversion or format_spec. */ + simple_expression = fstring_compile_expr(p, expr_start, expr_end, t); + if (!simple_expression) + goto error; + + /* Check for =, which puts the text value of the expression in + expr_text. */ + if (**str == '=') { + *str += 1; + + /* Skip over ASCII whitespace. No need to test for end of string + here, since we know there's at least a trailing quote somewhere + ahead. */ + while (Py_ISSPACE(**str)) { + *str += 1; + } + + /* Set *expr_text to the text of the expression. */ + *expr_text = PyUnicode_FromStringAndSize(expr_start, *str-expr_start); + if (!*expr_text) { + goto error; + } + } + + /* Check for a conversion char, if present. */ + if (**str == '!') { + *str += 1; + if (*str >= end) + goto unexpected_end_of_string; + + conversion = **str; + *str += 1; + + /* Validate the conversion. */ + if (!(conversion == 's' || conversion == 'r' || conversion == 'a')) { + RAISE_SYNTAX_ERROR( + "f-string: invalid conversion character: " + "expected 's', 'r', or 'a'"); + goto error; + } + + } + + /* Check for the format spec, if present. */ + if (*str >= end) + goto unexpected_end_of_string; + if (**str == ':') { + *str += 1; + if (*str >= end) + goto unexpected_end_of_string; + + /* Parse the format spec. */ + format_spec = fstring_parse(p, str, end, raw, recurse_lvl+1, + first_token, t, last_token); + if (!format_spec) + goto error; + } + + if (*str >= end || **str != '}') + goto unexpected_end_of_string; + + /* We're at a right brace. Consume it. */ + assert(*str < end); + assert(**str == '}'); + *str += 1; + + /* If we're in = mode (detected by non-NULL expr_text), and have no format + spec and no explicit conversion, set the conversion to 'r'. */ + if (*expr_text && format_spec == NULL && conversion == -1) { + conversion = 'r'; + } + + /* And now create the FormattedValue node that represents this + entire expression with the conversion and format spec. */ + //TODO: Fix this + *expression = FormattedValue(simple_expression, conversion, + format_spec, first_token->lineno, + first_token->col_offset, last_token->end_lineno, + last_token->end_col_offset, p->arena); + if (!*expression) + goto error; + + return 0; + +unexpected_end_of_string: + RAISE_SYNTAX_ERROR("f-string: expecting '}'"); + /* Falls through to error. */ + +error: + Py_XDECREF(*expr_text); + return -1; + +} + +/* Return -1 on error. + + Return 0 if we have a literal (possible zero length) and an + expression (zero length if at the end of the string. + + Return 1 if we have a literal, but no expression, and we want the + caller to call us again. This is used to deal with doubled + braces. + + When called multiple times on the string 'a{{b{0}c', this function + will return: + + 1. the literal 'a{' with no expression, and a return value + of 1. Despite the fact that there's no expression, the return + value of 1 means we're not finished yet. + + 2. the literal 'b' and the expression '0', with a return value of + 0. The fact that there's an expression means we're not finished. + + 3. literal 'c' with no expression and a return value of 0. The + combination of the return value of 0 with no expression means + we're finished. +*/ +static int +fstring_find_literal_and_expr(Parser *p, const char **str, const char *end, int raw, + int recurse_lvl, PyObject **literal, + PyObject **expr_text, expr_ty *expression, + Token *first_token, Token *t, Token *last_token) +{ + int result; + + assert(*literal == NULL && *expression == NULL); + + /* Get any literal string. */ + result = fstring_find_literal(p, str, end, raw, literal, recurse_lvl); + if (result < 0) + goto error; + + assert(result == 0 || result == 1); + + if (result == 1) + /* We have a literal, but don't look at the expression. */ + return 1; + + if (*str >= end || **str == '}') + /* We're at the end of the string or the end of a nested + f-string: no expression. The top-level error case where we + expect to be at the end of the string but we're at a '}' is + handled later. */ + return 0; + + /* We must now be the start of an expression, on a '{'. */ + assert(**str == '{'); + + if (fstring_find_expr(p, str, end, raw, recurse_lvl, expr_text, + expression, first_token, t, last_token) < 0) + goto error; + + return 0; + +error: + Py_CLEAR(*literal); + return -1; +} + +#ifdef NDEBUG +#define ExprList_check_invariants(l) +#else +static void +ExprList_check_invariants(ExprList *l) +{ + /* Check our invariants. Make sure this object is "live", and + hasn't been deallocated. */ + assert(l->size >= 0); + assert(l->p != NULL); + if (l->size <= EXPRLIST_N_CACHED) + assert(l->data == l->p); +} +#endif + +static void +ExprList_Init(ExprList *l) +{ + l->allocated = EXPRLIST_N_CACHED; + l->size = 0; + + /* Until we start allocating dynamically, p points to data. */ + l->p = l->data; + + ExprList_check_invariants(l); +} + +static int +ExprList_Append(ExprList *l, expr_ty exp) +{ + ExprList_check_invariants(l); + if (l->size >= l->allocated) { + /* We need to alloc (or realloc) the memory. */ + Py_ssize_t new_size = l->allocated * 2; + + /* See if we've ever allocated anything dynamically. */ + if (l->p == l->data) { + Py_ssize_t i; + /* We're still using the cached data. Switch to + alloc-ing. */ + l->p = PyMem_RawMalloc(sizeof(expr_ty) * new_size); + if (!l->p) + return -1; + /* Copy the cached data into the new buffer. */ + for (i = 0; i < l->size; i++) + l->p[i] = l->data[i]; + } else { + /* Just realloc. */ + expr_ty *tmp = PyMem_RawRealloc(l->p, sizeof(expr_ty) * new_size); + if (!tmp) { + PyMem_RawFree(l->p); + l->p = NULL; + return -1; + } + l->p = tmp; + } + + l->allocated = new_size; + assert(l->allocated == 2 * l->size); + } + + l->p[l->size++] = exp; + + ExprList_check_invariants(l); + return 0; +} + +static void +ExprList_Dealloc(ExprList *l) +{ + ExprList_check_invariants(l); + + /* If there's been an error, or we've never dynamically allocated, + do nothing. */ + if (!l->p || l->p == l->data) { + /* Do nothing. */ + } else { + /* We have dynamically allocated. Free the memory. */ + PyMem_RawFree(l->p); + } + l->p = NULL; + l->size = -1; +} + +static asdl_seq * +ExprList_Finish(ExprList *l, PyArena *arena) +{ + asdl_seq *seq; + + ExprList_check_invariants(l); + + /* Allocate the asdl_seq and copy the expressions in to it. */ + seq = _Py_asdl_seq_new(l->size, arena); + if (seq) { + Py_ssize_t i; + for (i = 0; i < l->size; i++) + asdl_seq_SET(seq, i, l->p[i]); + } + ExprList_Dealloc(l); + return seq; +} + +#ifdef NDEBUG +#define FstringParser_check_invariants(state) +#else +static void +FstringParser_check_invariants(FstringParser *state) +{ + if (state->last_str) + assert(PyUnicode_CheckExact(state->last_str)); + ExprList_check_invariants(&state->expr_list); +} +#endif + +void +_PyPegen_FstringParser_Init(FstringParser *state) +{ + state->last_str = NULL; + state->fmode = 0; + ExprList_Init(&state->expr_list); + FstringParser_check_invariants(state); +} + +void +_PyPegen_FstringParser_Dealloc(FstringParser *state) +{ + FstringParser_check_invariants(state); + + Py_XDECREF(state->last_str); + ExprList_Dealloc(&state->expr_list); +} + +/* Make a Constant node, but decref the PyUnicode object being added. */ +static expr_ty +make_str_node_and_del(Parser *p, PyObject **str, Token* first_token, Token *last_token) +{ + PyObject *s = *str; + PyObject *kind = NULL; + *str = NULL; + assert(PyUnicode_CheckExact(s)); + if (PyArena_AddPyObject(p->arena, s) < 0) { + Py_DECREF(s); + return NULL; + } + const char* the_str = PyBytes_AsString(first_token->bytes); + if (the_str && the_str[0] == 'u') { + kind = _PyPegen_new_identifier(p, "u"); + } + + if (kind == NULL && PyErr_Occurred()) { + return NULL; + } + + return Constant(s, kind, first_token->lineno, first_token->col_offset, + last_token->end_lineno, last_token->end_col_offset, p->arena); + +} + + +/* Add a non-f-string (that is, a regular literal string). str is + decref'd. */ +int +_PyPegen_FstringParser_ConcatAndDel(FstringParser *state, PyObject *str) +{ + FstringParser_check_invariants(state); + + assert(PyUnicode_CheckExact(str)); + + if (PyUnicode_GET_LENGTH(str) == 0) { + Py_DECREF(str); + return 0; + } + + if (!state->last_str) { + /* We didn't have a string before, so just remember this one. */ + state->last_str = str; + } else { + /* Concatenate this with the previous string. */ + PyUnicode_AppendAndDel(&state->last_str, str); + if (!state->last_str) + return -1; + } + FstringParser_check_invariants(state); + return 0; +} + +/* Parse an f-string. The f-string is in *str to end, with no + 'f' or quotes. */ +int +_PyPegen_FstringParser_ConcatFstring(Parser *p, FstringParser *state, const char **str, + const char *end, int raw, int recurse_lvl, + Token *first_token, Token* t, Token *last_token) +{ + FstringParser_check_invariants(state); + state->fmode = 1; + + /* Parse the f-string. */ + while (1) { + PyObject *literal = NULL; + PyObject *expr_text = NULL; + expr_ty expression = NULL; + + /* If there's a zero length literal in front of the + expression, literal will be NULL. If we're at the end of + the f-string, expression will be NULL (unless result == 1, + see below). */ + int result = fstring_find_literal_and_expr(p, str, end, raw, recurse_lvl, + &literal, &expr_text, + &expression, first_token, t, last_token); + if (result < 0) + return -1; + + /* Add the literal, if any. */ + if (literal && _PyPegen_FstringParser_ConcatAndDel(state, literal) < 0) { + Py_XDECREF(expr_text); + return -1; + } + /* Add the expr_text, if any. */ + if (expr_text && _PyPegen_FstringParser_ConcatAndDel(state, expr_text) < 0) { + return -1; + } + + /* We've dealt with the literal and expr_text, their ownership has + been transferred to the state object. Don't look at them again. */ + + /* See if we should just loop around to get the next literal + and expression, while ignoring the expression this + time. This is used for un-doubling braces, as an + optimization. */ + if (result == 1) + continue; + + if (!expression) + /* We're done with this f-string. */ + break; + + /* We know we have an expression. Convert any existing string + to a Constant node. */ + if (!state->last_str) { + /* Do nothing. No previous literal. */ + } else { + /* Convert the existing last_str literal to a Constant node. */ + expr_ty str = make_str_node_and_del(p, &state->last_str, first_token, last_token); + if (!str || ExprList_Append(&state->expr_list, str) < 0) + return -1; + } + + if (ExprList_Append(&state->expr_list, expression) < 0) + return -1; + } + + /* If recurse_lvl is zero, then we must be at the end of the + string. Otherwise, we must be at a right brace. */ + + if (recurse_lvl == 0 && *str < end-1) { + RAISE_SYNTAX_ERROR("f-string: unexpected end of string"); + return -1; + } + if (recurse_lvl != 0 && **str != '}') { + RAISE_SYNTAX_ERROR("f-string: expecting '}'"); + return -1; + } + + FstringParser_check_invariants(state); + return 0; +} + +/* Convert the partial state reflected in last_str and expr_list to an + expr_ty. The expr_ty can be a Constant, or a JoinedStr. */ +expr_ty +_PyPegen_FstringParser_Finish(Parser *p, FstringParser *state, Token* first_token, + Token *last_token) +{ + asdl_seq *seq; + + FstringParser_check_invariants(state); + + /* If we're just a constant string with no expressions, return + that. */ + if (!state->fmode) { + assert(!state->expr_list.size); + if (!state->last_str) { + /* Create a zero length string. */ + state->last_str = PyUnicode_FromStringAndSize(NULL, 0); + if (!state->last_str) + goto error; + } + return make_str_node_and_del(p, &state->last_str, first_token, last_token); + } + + /* Create a Constant node out of last_str, if needed. It will be the + last node in our expression list. */ + if (state->last_str) { + expr_ty str = make_str_node_and_del(p, &state->last_str, first_token, last_token); + if (!str || ExprList_Append(&state->expr_list, str) < 0) + goto error; + } + /* This has already been freed. */ + assert(state->last_str == NULL); + + seq = ExprList_Finish(&state->expr_list, p->arena); + if (!seq) + goto error; + + return _Py_JoinedStr(seq, first_token->lineno, first_token->col_offset, + last_token->end_lineno, last_token->end_col_offset, p->arena); + +error: + _PyPegen_FstringParser_Dealloc(state); + return NULL; +} + +/* Given an f-string (with no 'f' or quotes) that's in *str and ends + at end, parse it into an expr_ty. Return NULL on error. Adjust + str to point past the parsed portion. */ +static expr_ty +fstring_parse(Parser *p, const char **str, const char *end, int raw, + int recurse_lvl, Token *first_token, Token* t, Token *last_token) +{ + FstringParser state; + + _PyPegen_FstringParser_Init(&state); + if (_PyPegen_FstringParser_ConcatFstring(p, &state, str, end, raw, recurse_lvl, + first_token, t, last_token) < 0) { + _PyPegen_FstringParser_Dealloc(&state); + return NULL; + } + + return _PyPegen_FstringParser_Finish(p, &state, t, t); +} diff --git a/Parser/pegen/parse_string.h b/Parser/pegen/parse_string.h new file mode 100644 index 0000000000000..4f2aa94fc19b0 --- /dev/null +++ b/Parser/pegen/parse_string.h @@ -0,0 +1,46 @@ +#ifndef STRINGS_H +#define STRINGS_H + +#include <Python.h> +#include <Python-ast.h> +#include "pegen.h" + +#define EXPRLIST_N_CACHED 64 + +typedef struct { + /* Incrementally build an array of expr_ty, so be used in an + asdl_seq. Cache some small but reasonably sized number of + expr_ty's, and then after that start dynamically allocating, + doubling the number allocated each time. Note that the f-string + f'{0}a{1}' contains 3 expr_ty's: 2 FormattedValue's, and one + Constant for the literal 'a'. So you add expr_ty's about twice as + fast as you add expressions in an f-string. */ + + Py_ssize_t allocated; /* Number we've allocated. */ + Py_ssize_t size; /* Number we've used. */ + expr_ty *p; /* Pointer to the memory we're actually + using. Will point to 'data' until we + start dynamically allocating. */ + expr_ty data[EXPRLIST_N_CACHED]; +} ExprList; + +/* The FstringParser is designed to add a mix of strings and + f-strings, and concat them together as needed. Ultimately, it + generates an expr_ty. */ +typedef struct { + PyObject *last_str; + ExprList expr_list; + int fmode; +} FstringParser; + +void _PyPegen_FstringParser_Init(FstringParser *); +int _PyPegen_parsestr(Parser *, const char *, int *, int *, PyObject **, + const char **, Py_ssize_t *); +int _PyPegen_FstringParser_ConcatFstring(Parser *, FstringParser *, const char **, + const char *, int, int, Token *, Token *, + Token *); +int _PyPegen_FstringParser_ConcatAndDel(FstringParser *, PyObject *); +expr_ty _PyPegen_FstringParser_Finish(Parser *, FstringParser *, Token *, Token *); +void _PyPegen_FstringParser_Dealloc(FstringParser *); + +#endif diff --git a/Parser/pegen/peg_api.c b/Parser/pegen/peg_api.c new file mode 100644 index 0000000000000..7c6903cdd9334 --- /dev/null +++ b/Parser/pegen/peg_api.c @@ -0,0 +1,134 @@ +#include <pegen_interface.h> + +#include "../tokenizer.h" +#include "pegen.h" + +mod_ty +PyPegen_ASTFromString(const char *str, int mode, PyCompilerFlags *flags, PyArena *arena) +{ + PyObject *filename_ob = PyUnicode_FromString("<string>"); + if (filename_ob == NULL) { + return NULL; + } + mod_ty result = PyPegen_ASTFromStringObject(str, filename_ob, mode, flags, arena); + Py_XDECREF(filename_ob); + return result; +} + +mod_ty +PyPegen_ASTFromStringObject(const char *str, PyObject* filename, int mode, PyCompilerFlags *flags, PyArena *arena) +{ + if (PySys_Audit("compile", "yO", str, filename) < 0) { + return NULL; + } + + int iflags = flags != NULL ? flags->cf_flags : PyCF_IGNORE_COOKIE; + mod_ty result = _PyPegen_run_parser_from_string(str, mode, filename, iflags, arena); + return result; +} + +mod_ty +PyPegen_ASTFromFile(const char *filename, int mode, PyArena *arena) +{ + PyObject *filename_ob = PyUnicode_FromString(filename); + if (filename_ob == NULL) { + return NULL; + } + + mod_ty result = _PyPegen_run_parser_from_file(filename, mode, filename_ob, arena); + Py_XDECREF(filename_ob); + return result; +} + +mod_ty +PyPegen_ASTFromFileObject(FILE *fp, PyObject *filename_ob, int mode, + const char *enc, const char *ps1, const char* ps2, + int *errcode, PyArena *arena) +{ + if (PySys_Audit("compile", "OO", Py_None, filename_ob) < 0) { + return NULL; + } + return _PyPegen_run_parser_from_file_pointer(fp, mode, filename_ob, enc, ps1, ps2, + errcode, arena); +} + +PyCodeObject * +PyPegen_CodeObjectFromString(const char *str, int mode, PyCompilerFlags *flags) +{ + PyArena *arena = PyArena_New(); + if (arena == NULL) { + return NULL; + } + + PyCodeObject *result = NULL; + + PyObject *filename_ob = PyUnicode_FromString("<string>"); + if (filename_ob == NULL) { + goto error; + } + + mod_ty res = PyPegen_ASTFromString(str, mode, flags, arena); + if (res == NULL) { + goto error; + } + + result = PyAST_CompileObject(res, filename_ob, NULL, -1, arena); + +error: + Py_XDECREF(filename_ob); + PyArena_Free(arena); + return result; +} + +PyCodeObject * +PyPegen_CodeObjectFromFile(const char *filename, int mode) +{ + PyArena *arena = PyArena_New(); + if (arena == NULL) { + return NULL; + } + + PyCodeObject *result = NULL; + + PyObject *filename_ob = PyUnicode_FromString(filename); + if (filename_ob == NULL) { + goto error; + } + + mod_ty res = PyPegen_ASTFromFile(filename, mode, arena); + if (res == NULL) { + goto error; + } + + result = PyAST_CompileObject(res, filename_ob, NULL, -1, arena); + +error: + Py_XDECREF(filename_ob); + PyArena_Free(arena); + return result; +} + +PyCodeObject * +PyPegen_CodeObjectFromFileObject(FILE *fp, PyObject *filename_ob, int mode, + const char *ps1, const char *ps2, const char *enc, + int *errcode) +{ + PyArena *arena = PyArena_New(); + if (arena == NULL) { + return NULL; + } + + PyCodeObject *result = NULL; + + mod_ty res = PyPegen_ASTFromFileObject(fp, filename_ob, mode, enc, ps1, ps2, + errcode, arena); + if (res == NULL) { + goto error; + } + + result = PyAST_CompileObject(res, filename_ob, NULL, -1, arena); + +error: + PyArena_Free(arena); + return result; +} diff --git a/Parser/pegen/pegen.c b/Parser/pegen/pegen.c new file mode 100644 index 0000000000000..47b712f262c46 --- /dev/null +++ b/Parser/pegen/pegen.c @@ -0,0 +1,1865 @@ +#include <Python.h> +#include <errcode.h> +#include "../tokenizer.h" + +#include "pegen.h" +#include "parse_string.h" + +static int +init_normalization(Parser *p) +{ + PyObject *m = PyImport_ImportModuleNoBlock("unicodedata"); + if (!m) + { + return 0; + } + p->normalize = PyObject_GetAttrString(m, "normalize"); + Py_DECREF(m); + if (!p->normalize) + { + return 0; + } + return 1; +} + +PyObject * +_PyPegen_new_identifier(Parser *p, char *n) +{ + PyObject *id = PyUnicode_DecodeUTF8(n, strlen(n), NULL); + if (!id) { + goto error; + } + /* PyUnicode_DecodeUTF8 should always return a ready string. */ + assert(PyUnicode_IS_READY(id)); + /* Check whether there are non-ASCII characters in the + identifier; if so, normalize to NFKC. */ + if (!PyUnicode_IS_ASCII(id)) + { + PyObject *id2; + if (!p->normalize && !init_normalization(p)) + { + Py_DECREF(id); + goto error; + } + PyObject *form = PyUnicode_InternFromString("NFKC"); + if (form == NULL) + { + Py_DECREF(id); + goto error; + } + PyObject *args[2] = {form, id}; + id2 = _PyObject_FastCall(p->normalize, args, 2); + Py_DECREF(id); + Py_DECREF(form); + if (!id2) { + goto error; + } + if (!PyUnicode_Check(id2)) + { + PyErr_Format(PyExc_TypeError, + "unicodedata.normalize() must return a string, not " + "%.200s", + _PyType_Name(Py_TYPE(id2))); + Py_DECREF(id2); + goto error; + } + id = id2; + } + PyUnicode_InternInPlace(&id); + if (PyArena_AddPyObject(p->arena, id) < 0) + { + Py_DECREF(id); + goto error; + } + return id; + +error: + p->error_indicator = 1; + return NULL; +} + +static PyObject * +_create_dummy_identifier(Parser *p) +{ + return _PyPegen_new_identifier(p, ""); +} + +static inline Py_ssize_t +byte_offset_to_character_offset(PyObject *line, int col_offset) +{ + const char *str = PyUnicode_AsUTF8(line); + PyObject *text = PyUnicode_DecodeUTF8(str, col_offset, NULL); + if (!text) { + return 0; + } + Py_ssize_t size = PyUnicode_GET_LENGTH(text); + Py_DECREF(text); + return size; +} + +const char * +_PyPegen_get_expr_name(expr_ty e) +{ + switch (e->kind) { + case Attribute_kind: + return "attribute"; + case Subscript_kind: + return "subscript"; + case Starred_kind: + return "starred"; + case Name_kind: + return "name"; + case List_kind: + return "list"; + case Tuple_kind: + return "tuple"; + case Lambda_kind: + return "lambda"; + case Call_kind: + return "function call"; + case BoolOp_kind: + case BinOp_kind: + case UnaryOp_kind: + return "operator"; + case GeneratorExp_kind: + return "generator expression"; + case Yield_kind: + case YieldFrom_kind: + return "yield expression"; + case Await_kind: + return "await expression"; + case ListComp_kind: + return "list comprehension"; + case SetComp_kind: + return "set comprehension"; + case DictComp_kind: + return "dict comprehension"; + case Dict_kind: + return "dict display"; + case Set_kind: + return "set display"; + case JoinedStr_kind: + case FormattedValue_kind: + return "f-string expression"; + case Constant_kind: { + PyObject *value = e->v.Constant.value; + if (value == Py_None) { + return "None"; + } + if (value == Py_False) { + return "False"; + } + if (value == Py_True) { + return "True"; + } + if (value == Py_Ellipsis) { + return "Ellipsis"; + } + return "literal"; + } + case Compare_kind: + return "comparison"; + case IfExp_kind: + return "conditional expression"; + case NamedExpr_kind: + return "named expression"; + default: + PyErr_Format(PyExc_SystemError, + "unexpected expression in assignment %d (line %d)", + e->kind, e->lineno); + return NULL; + } +} + +static void +raise_decode_error(Parser *p) +{ + const char *errtype = NULL; + if (PyErr_ExceptionMatches(PyExc_UnicodeError)) { + errtype = "unicode error"; + } + else if (PyErr_ExceptionMatches(PyExc_ValueError)) { + errtype = "value error"; + } + if (errtype) { + PyObject *type, *value, *tback, *errstr; + PyErr_Fetch(&type, &value, &tback); + errstr = PyObject_Str(value); + if (errstr) { + RAISE_SYNTAX_ERROR("(%s) %U", errtype, errstr); + Py_DECREF(errstr); + } + else { + PyErr_Clear(); + RAISE_SYNTAX_ERROR("(%s) unknown error", errtype); + } + Py_XDECREF(type); + Py_XDECREF(value); + Py_XDECREF(tback); + } +} + +static void +raise_tokenizer_init_error(PyObject *filename) +{ + if (!(PyErr_ExceptionMatches(PyExc_LookupError) + || PyErr_ExceptionMatches(PyExc_ValueError) + || PyErr_ExceptionMatches(PyExc_UnicodeDecodeError))) { + return; + } + PyObject *type, *value, *tback, *errstr; + PyErr_Fetch(&type, &value, &tback); + errstr = PyObject_Str(value); + + Py_INCREF(Py_None); + PyObject *tmp = Py_BuildValue("(OiiN)", filename, 0, -1, Py_None); + if (!tmp) { + goto error; + } + + value = PyTuple_Pack(2, errstr, tmp); + Py_DECREF(tmp); + if (!value) { + goto error; + } + PyErr_SetObject(PyExc_SyntaxError, value); + +error: + Py_XDECREF(type); + Py_XDECREF(value); + Py_XDECREF(tback); +} + +static inline PyObject * +get_error_line(char *buffer) +{ + char *newline = strchr(buffer, '\n'); + if (newline) { + return PyUnicode_FromStringAndSize(buffer, newline - buffer); + } + else { + return PyUnicode_FromString(buffer); + } +} + +static int +tokenizer_error_with_col_offset(Parser *p, PyObject *errtype, const char *errmsg) +{ + PyObject *errstr = NULL; + PyObject *value = NULL; + int col_number = -1; + + errstr = PyUnicode_FromString(errmsg); + if (!errstr) { + return -1; + } + + PyObject *loc = NULL; + if (p->start_rule == Py_file_input) { + loc = PyErr_ProgramTextObject(p->tok->filename, p->tok->lineno); + } + if (!loc) { + loc = get_error_line(p->tok->buf); + } + + if (loc) { + col_number = p->tok->cur - p->tok->buf; + } + else { + Py_INCREF(Py_None); + loc = Py_None; + } + + PyObject *tmp = Py_BuildValue("(OiiN)", p->tok->filename, p->tok->lineno, + col_number, loc); + if (!tmp) { + goto error; + } + + value = PyTuple_Pack(2, errstr, tmp); + Py_DECREF(tmp); + if (!value) { + goto error; + } + PyErr_SetObject(errtype, value); + + Py_XDECREF(value); + Py_XDECREF(errstr); + return -1; + +error: + Py_XDECREF(errstr); + Py_XDECREF(loc); + return -1; +} + +static int +tokenizer_error(Parser *p) +{ + if (PyErr_Occurred()) { + return -1; + } + + const char *msg = NULL; + PyObject* errtype = PyExc_SyntaxError; + switch (p->tok->done) { + case E_TOKEN: + msg = "invalid token"; + break; + case E_IDENTIFIER: + msg = "invalid character in identifier"; + break; + case E_BADPREFIX: + return tokenizer_error_with_col_offset(p, + PyExc_SyntaxError, "invalid string prefix"); + case E_EOFS: + return tokenizer_error_with_col_offset(p, + PyExc_SyntaxError, "EOF while scanning triple-quoted string literal"); + case E_EOLS: + return tokenizer_error_with_col_offset(p, + PyExc_SyntaxError, "EOL while scanning string literal"); + case E_DEDENT: + return tokenizer_error_with_col_offset(p, + PyExc_IndentationError, "unindent does not match any outer indentation level"); + case E_INTR: + if (!PyErr_Occurred()) { + PyErr_SetNone(PyExc_KeyboardInterrupt); + } + return -1; + case E_NOMEM: + PyErr_NoMemory(); + return -1; + case E_TABSPACE: + errtype = PyExc_TabError; + msg = "inconsistent use of tabs and spaces in indentation"; + break; + case E_TOODEEP: + errtype = PyExc_IndentationError; + msg = "too many levels of indentation"; + break; + case E_DECODE: + raise_decode_error(p); + return -1; + case E_LINECONT: + msg = "unexpected character after line continuation character"; + break; + default: + msg = "unknown parsing error"; + } + + PyErr_Format(errtype, msg); + // There is no reliable column information for this error + PyErr_SyntaxLocationObject(p->tok->filename, p->tok->lineno, 0); + + return -1; +} + +void * +_PyPegen_raise_error(Parser *p, PyObject *errtype, const char *errmsg, ...) +{ + PyObject *value = NULL; + PyObject *errstr = NULL; + PyObject *loc = NULL; + PyObject *tmp = NULL; + Token *t = p->tokens[p->fill - 1]; + Py_ssize_t col_number = 0; + va_list va; + + va_start(va, errmsg); + errstr = PyUnicode_FromFormatV(errmsg, va); + va_end(va); + if (!errstr) { + goto error; + } + + if (p->start_rule == Py_file_input) { + loc = PyErr_ProgramTextObject(p->tok->filename, t->lineno); + } + + if (!loc) { + loc = get_error_line(p->tok->buf); + } + + if (loc) { + int col_offset = t->col_offset == -1 ? 0 : t->col_offset; + col_number = byte_offset_to_character_offset(loc, col_offset) + 1; + } + else { + Py_INCREF(Py_None); + loc = Py_None; + } + + + tmp = Py_BuildValue("(OiiN)", p->tok->filename, t->lineno, col_number, loc); + if (!tmp) { + goto error; + } + value = PyTuple_Pack(2, errstr, tmp); + Py_DECREF(tmp); + if (!value) { + goto error; + } + PyErr_SetObject(errtype, value); + + Py_DECREF(errstr); + Py_DECREF(value); + return NULL; + +error: + Py_XDECREF(errstr); + Py_XDECREF(loc); + return NULL; +} + +void *_PyPegen_arguments_parsing_error(Parser *p, expr_ty e) { + int kwarg_unpacking = 0; + for (Py_ssize_t i = 0, l = asdl_seq_LEN(e->v.Call.keywords); i < l; i++) { + keyword_ty keyword = asdl_seq_GET(e->v.Call.keywords, i); + if (!keyword->arg) { + kwarg_unpacking = 1; + } + } + + const char *msg = NULL; + if (kwarg_unpacking) { + msg = "positional argument follows keyword argument unpacking"; + } else { + msg = "positional argument follows keyword argument"; + } + + return RAISE_SYNTAX_ERROR(msg); +} + +#if 0 +static const char * +token_name(int type) +{ + if (0 <= type && type <= N_TOKENS) { + return _PyParser_TokenNames[type]; + } + return "<Huh?>"; +} +#endif + +// Here, mark is the start of the node, while p->mark is the end. +// If node==NULL, they should be the same. +int +_PyPegen_insert_memo(Parser *p, int mark, int type, void *node) +{ + // Insert in front + Memo *m = PyArena_Malloc(p->arena, sizeof(Memo)); + if (m == NULL) { + return -1; + } + m->type = type; + m->node = node; + m->mark = p->mark; + m->next = p->tokens[mark]->memo; + p->tokens[mark]->memo = m; + return 0; +} + +// Like _PyPegen_insert_memo(), but updates an existing node if found. +int +_PyPegen_update_memo(Parser *p, int mark, int type, void *node) +{ + for (Memo *m = p->tokens[mark]->memo; m != NULL; m = m->next) { + if (m->type == type) { + // Update existing node. + m->node = node; + m->mark = p->mark; + return 0; + } + } + // Insert new node. + return _PyPegen_insert_memo(p, mark, type, node); +} + +// Return dummy NAME. +void * +_PyPegen_dummy_name(Parser *p, ...) +{ + static void *cache = NULL; + + if (cache != NULL) { + return cache; + } + + PyObject *id = _create_dummy_identifier(p); + if (!id) { + return NULL; + } + cache = Name(id, Load, 1, 0, 1, 0, p->arena); + return cache; +} + +static int +_get_keyword_or_name_type(Parser *p, const char *name, int name_len) +{ + if (name_len >= p->n_keyword_lists || p->keywords[name_len] == NULL) { + return NAME; + } + for (KeywordToken *k = p->keywords[name_len]; k->type != -1; k++) { + if (strncmp(k->str, name, name_len) == 0) { + return k->type; + } + } + return NAME; +} + +int +_PyPegen_fill_token(Parser *p) +{ + const char *start, *end; + int type = PyTokenizer_Get(p->tok, &start, &end); + if (type == ERRORTOKEN) { + return tokenizer_error(p); + } + if (type == ENDMARKER && p->start_rule == Py_single_input && p->parsing_started) { + type = NEWLINE; /* Add an extra newline */ + p->parsing_started = 0; + + if (p->tok->indent) { + p->tok->pendin = -p->tok->indent; + p->tok->indent = 0; + } + } + else { + p->parsing_started = 1; + } + + if (p->fill == p->size) { + int newsize = p->size * 2; + p->tokens = PyMem_Realloc(p->tokens, newsize * sizeof(Token *)); + if (p->tokens == NULL) { + PyErr_Format(PyExc_MemoryError, "Realloc tokens failed"); + return -1; + } + for (int i = p->size; i < newsize; i++) { + p->tokens[i] = PyMem_Malloc(sizeof(Token)); + memset(p->tokens[i], '\0', sizeof(Token)); + } + p->size = newsize; + } + + Token *t = p->tokens[p->fill]; + t->type = (type == NAME) ? _get_keyword_or_name_type(p, start, (int)(end - start)) : type; + t->bytes = PyBytes_FromStringAndSize(start, end - start); + if (t->bytes == NULL) { + return -1; + } + PyArena_AddPyObject(p->arena, t->bytes); + + int lineno = type == STRING ? p->tok->first_lineno : p->tok->lineno; + const char *line_start = type == STRING ? p->tok->multi_line_start : p->tok->line_start; + int end_lineno = p->tok->lineno; + int col_offset = -1, end_col_offset = -1; + if (start != NULL && start >= line_start) { + col_offset = start - line_start; + } + if (end != NULL && end >= p->tok->line_start) { + end_col_offset = end - p->tok->line_start; + } + + t->lineno = p->starting_lineno + lineno; + t->col_offset = p->tok->lineno == 1 ? p->starting_col_offset + col_offset : col_offset; + t->end_lineno = p->starting_lineno + end_lineno; + t->end_col_offset = p->tok->lineno == 1 ? p->starting_col_offset + end_col_offset : end_col_offset; + + // if (p->fill % 100 == 0) fprintf(stderr, "Filled at %d: %s \"%s\"\n", p->fill, + // token_name(type), PyBytes_AsString(t->bytes)); + p->fill += 1; + return 0; +} + +// Instrumentation to count the effectiveness of memoization. +// The array counts the number of tokens skipped by memoization, +// indexed by type. + +#define NSTATISTICS 2000 +static long memo_statistics[NSTATISTICS]; + +void +_PyPegen_clear_memo_statistics() +{ + for (int i = 0; i < NSTATISTICS; i++) { + memo_statistics[i] = 0; + } +} + +PyObject * +_PyPegen_get_memo_statistics() +{ + PyObject *ret = PyList_New(NSTATISTICS); + if (ret == NULL) { + return NULL; + } + for (int i = 0; i < NSTATISTICS; i++) { + PyObject *value = PyLong_FromLong(memo_statistics[i]); + if (value == NULL) { + Py_DECREF(ret); + return NULL; + } + // PyList_SetItem borrows a reference to value. + if (PyList_SetItem(ret, i, value) < 0) { + Py_DECREF(ret); + return NULL; + } + } + return ret; +} + +int // bool +_PyPegen_is_memoized(Parser *p, int type, void *pres) +{ + if (p->mark == p->fill) { + if (_PyPegen_fill_token(p) < 0) { + return -1; + } + } + + Token *t = p->tokens[p->mark]; + + for (Memo *m = t->memo; m != NULL; m = m->next) { + if (m->type == type) { + if (0 <= type && type < NSTATISTICS) { + long count = m->mark - p->mark; + // A memoized negative result counts for one. + if (count <= 0) { + count = 1; + } + memo_statistics[type] += count; + } + p->mark = m->mark; + *(void **)(pres) = m->node; + // fprintf(stderr, "%d < %d: memoized!\n", p->mark, p->fill); + return 1; + } + } + // fprintf(stderr, "%d < %d: not memoized\n", p->mark, p->fill); + return 0; +} + +int +_PyPegen_lookahead_with_string(int positive, void *(func)(Parser *, const char *), Parser *p, + const char *arg) +{ + int mark = p->mark; + void *res = func(p, arg); + p->mark = mark; + return (res != NULL) == positive; +} + +int +_PyPegen_lookahead_with_int(int positive, Token *(func)(Parser *, int), Parser *p, int arg) +{ + int mark = p->mark; + void *res = func(p, arg); + p->mark = mark; + return (res != NULL) == positive; +} + +int +_PyPegen_lookahead(int positive, void *(func)(Parser *), Parser *p) +{ + int mark = p->mark; + void *res = func(p); + p->mark = mark; + return (res != NULL) == positive; +} + +Token * +_PyPegen_expect_token(Parser *p, int type) +{ + if (p->mark == p->fill) { + if (_PyPegen_fill_token(p) < 0) { + return NULL; + } + } + Token *t = p->tokens[p->mark]; + if (t->type != type) { + // fprintf(stderr, "No %s at %d\n", token_name(type), p->mark); + return NULL; + } + p->mark += 1; + // fprintf(stderr, "Got %s at %d: %s\n", token_name(type), p->mark, + // PyBytes_AsString(t->bytes)); + + return t; +} + +Token * +_PyPegen_get_last_nonnwhitespace_token(Parser *p) +{ + assert(p->mark >= 0); + Token *token = NULL; + for (int m = p->mark - 1; m >= 0; m--) { + token = p->tokens[m]; + if (token->type != ENDMARKER && (token->type < NEWLINE || token->type > DEDENT)) { + break; + } + } + return token; +} + +void * +_PyPegen_async_token(Parser *p) +{ + return _PyPegen_expect_token(p, ASYNC); +} + +void * +_PyPegen_await_token(Parser *p) +{ + return _PyPegen_expect_token(p, AWAIT); +} + +void * +_PyPegen_endmarker_token(Parser *p) +{ + return _PyPegen_expect_token(p, ENDMARKER); +} + +expr_ty +_PyPegen_name_token(Parser *p) +{ + Token *t = _PyPegen_expect_token(p, NAME); + if (t == NULL) { + return NULL; + } + char* s = PyBytes_AsString(t->bytes); + if (!s) { + return NULL; + } + PyObject *id = _PyPegen_new_identifier(p, s); + if (id == NULL) { + return NULL; + } + return Name(id, Load, t->lineno, t->col_offset, t->end_lineno, t->end_col_offset, + p->arena); +} + +void * +_PyPegen_string_token(Parser *p) +{ + return _PyPegen_expect_token(p, STRING); +} + +void * +_PyPegen_newline_token(Parser *p) +{ + return _PyPegen_expect_token(p, NEWLINE); +} + +void * +_PyPegen_indent_token(Parser *p) +{ + return _PyPegen_expect_token(p, INDENT); +} + +void * +_PyPegen_dedent_token(Parser *p) +{ + return _PyPegen_expect_token(p, DEDENT); +} + +static PyObject * +parsenumber_raw(const char *s) +{ + const char *end; + long x; + double dx; + Py_complex compl; + int imflag; + + assert(s != NULL); + errno = 0; + end = s + strlen(s) - 1; + imflag = *end == 'j' || *end == 'J'; + if (s[0] == '0') { + x = (long)PyOS_strtoul(s, (char **)&end, 0); + if (x < 0 && errno == 0) { + return PyLong_FromString(s, (char **)0, 0); + } + } + else + x = PyOS_strtol(s, (char **)&end, 0); + if (*end == '\0') { + if (errno != 0) + return PyLong_FromString(s, (char **)0, 0); + return PyLong_FromLong(x); + } + /* XXX Huge floats may silently fail */ + if (imflag) { + compl.real = 0.; + compl.imag = PyOS_string_to_double(s, (char **)&end, NULL); + if (compl.imag == -1.0 && PyErr_Occurred()) + return NULL; + return PyComplex_FromCComplex(compl); + } + else { + dx = PyOS_string_to_double(s, NULL, NULL); + if (dx == -1.0 && PyErr_Occurred()) + return NULL; + return PyFloat_FromDouble(dx); + } +} + +static PyObject * +parsenumber(const char *s) +{ + char *dup, *end; + PyObject *res = NULL; + + assert(s != NULL); + + if (strchr(s, '_') == NULL) { + return parsenumber_raw(s); + } + /* Create a duplicate without underscores. */ + dup = PyMem_Malloc(strlen(s) + 1); + if (dup == NULL) { + return PyErr_NoMemory(); + } + end = dup; + for (; *s; s++) { + if (*s != '_') { + *end++ = *s; + } + } + *end = '\0'; + res = parsenumber_raw(dup); + PyMem_Free(dup); + return res; +} + +expr_ty +_PyPegen_number_token(Parser *p) +{ + Token *t = _PyPegen_expect_token(p, NUMBER); + if (t == NULL) { + return NULL; + } + + char *num_raw = PyBytes_AsString(t->bytes); + + if (num_raw == NULL) { + return NULL; + } + + PyObject *c = parsenumber(num_raw); + + if (c == NULL) { + return NULL; + } + + if (PyArena_AddPyObject(p->arena, c) < 0) { + Py_DECREF(c); + return NULL; + } + + return Constant(c, NULL, t->lineno, t->col_offset, t->end_lineno, t->end_col_offset, + p->arena); +} + +void +_PyPegen_Parser_Free(Parser *p) +{ + Py_XDECREF(p->normalize); + for (int i = 0; i < p->size; i++) { + PyMem_Free(p->tokens[i]); + } + PyMem_Free(p->tokens); + PyMem_Free(p); +} + +Parser * +_PyPegen_Parser_New(struct tok_state *tok, int start_rule, int *errcode, PyArena *arena) +{ + Parser *p = PyMem_Malloc(sizeof(Parser)); + if (p == NULL) { + PyErr_Format(PyExc_MemoryError, "Out of memory for Parser"); + return NULL; + } + assert(tok != NULL); + p->tok = tok; + p->keywords = NULL; + p->n_keyword_lists = -1; + p->tokens = PyMem_Malloc(sizeof(Token *)); + if (!p->tokens) { + PyMem_Free(p); + PyErr_Format(PyExc_MemoryError, "Out of memory for tokens"); + return NULL; + } + p->tokens[0] = PyMem_Malloc(sizeof(Token)); + memset(p->tokens[0], '\0', sizeof(Token)); + p->mark = 0; + p->fill = 0; + p->size = 1; + + p->errcode = errcode; + p->arena = arena; + p->start_rule = start_rule; + p->parsing_started = 0; + p->normalize = NULL; + p->error_indicator = 0; + + p->starting_lineno = 0; + p->starting_col_offset = 0; + + return p; +} + +void * +_PyPegen_run_parser(Parser *p) +{ + void *res = _PyPegen_parse(p); + if (res == NULL) { + if (PyErr_Occurred()) { + return NULL; + } + if (p->fill == 0) { + RAISE_SYNTAX_ERROR("error at start before reading any input"); + } + else if (p->tok->done == E_EOF) { + RAISE_SYNTAX_ERROR("unexpected EOF while parsing"); + } + else { + if (p->tokens[p->fill-1]->type == INDENT) { + RAISE_INDENTATION_ERROR("unexpected indent"); + } + else if (p->tokens[p->fill-1]->type == DEDENT) { + RAISE_INDENTATION_ERROR("unexpected unindent"); + } + else { + RAISE_SYNTAX_ERROR("invalid syntax"); + } + } + return NULL; + } + + return res; +} + +mod_ty +_PyPegen_run_parser_from_file_pointer(FILE *fp, int start_rule, PyObject *filename_ob, + const char *enc, const char *ps1, const char *ps2, + int *errcode, PyArena *arena) +{ + struct tok_state *tok = PyTokenizer_FromFile(fp, enc, ps1, ps2); + if (tok == NULL) { + if (PyErr_Occurred()) { + raise_tokenizer_init_error(filename_ob); + return NULL; + } + return NULL; + } + // This transfers the ownership to the tokenizer + tok->filename = filename_ob; + Py_INCREF(filename_ob); + + // From here on we need to clean up even if there's an error + mod_ty result = NULL; + + Parser *p = _PyPegen_Parser_New(tok, start_rule, errcode, arena); + if (p == NULL) { + goto error; + } + + result = _PyPegen_run_parser(p); + _PyPegen_Parser_Free(p); + +error: + PyTokenizer_Free(tok); + return result; +} + +mod_ty +_PyPegen_run_parser_from_file(const char *filename, int start_rule, + PyObject *filename_ob, PyArena *arena) +{ + FILE *fp = fopen(filename, "rb"); + if (fp == NULL) { + PyErr_SetFromErrnoWithFilename(PyExc_OSError, filename); + return NULL; + } + + mod_ty result = _PyPegen_run_parser_from_file_pointer(fp, start_rule, filename_ob, + NULL, NULL, NULL, NULL, arena); + + fclose(fp); + return result; +} + +mod_ty +_PyPegen_run_parser_from_string(const char *str, int start_rule, PyObject *filename_ob, + int iflags, PyArena *arena) +{ + int exec_input = start_rule == Py_file_input; + + struct tok_state *tok; + if (iflags & PyCF_IGNORE_COOKIE) { + tok = PyTokenizer_FromUTF8(str, exec_input); + } else { + tok = PyTokenizer_FromString(str, exec_input); + } + if (tok == NULL) { + if (PyErr_Occurred()) { + raise_tokenizer_init_error(filename_ob); + } + return NULL; + } + // This transfers the ownership to the tokenizer + tok->filename = filename_ob; + Py_INCREF(filename_ob); + + // We need to clear up from here on + mod_ty result = NULL; + + Parser *p = _PyPegen_Parser_New(tok, start_rule, NULL, arena); + if (p == NULL) { + goto error; + } + + result = _PyPegen_run_parser(p); + _PyPegen_Parser_Free(p); + +error: + PyTokenizer_Free(tok); + return result; +} + +void * +_PyPegen_interactive_exit(Parser *p) +{ + if (p->errcode) { + *(p->errcode) = E_EOF; + } + return NULL; +} + +/* Creates a single-element asdl_seq* that contains a */ +asdl_seq * +_PyPegen_singleton_seq(Parser *p, void *a) +{ + assert(a != NULL); + asdl_seq *seq = _Py_asdl_seq_new(1, p->arena); + if (!seq) { + return NULL; + } + asdl_seq_SET(seq, 0, a); + return seq; +} + +/* Creates a copy of seq and prepends a to it */ +asdl_seq * +_PyPegen_seq_insert_in_front(Parser *p, void *a, asdl_seq *seq) +{ + assert(a != NULL); + if (!seq) { + return _PyPegen_singleton_seq(p, a); + } + + asdl_seq *new_seq = _Py_asdl_seq_new(asdl_seq_LEN(seq) + 1, p->arena); + if (!new_seq) { + return NULL; + } + + asdl_seq_SET(new_seq, 0, a); + for (int i = 1, l = asdl_seq_LEN(new_seq); i < l; i++) { + asdl_seq_SET(new_seq, i, asdl_seq_GET(seq, i - 1)); + } + return new_seq; +} + +static int +_get_flattened_seq_size(asdl_seq *seqs) +{ + int size = 0; + for (Py_ssize_t i = 0, l = asdl_seq_LEN(seqs); i < l; i++) { + asdl_seq *inner_seq = asdl_seq_GET(seqs, i); + size += asdl_seq_LEN(inner_seq); + } + return size; +} + +/* Flattens an asdl_seq* of asdl_seq*s */ +asdl_seq * +_PyPegen_seq_flatten(Parser *p, asdl_seq *seqs) +{ + int flattened_seq_size = _get_flattened_seq_size(seqs); + assert(flattened_seq_size > 0); + + asdl_seq *flattened_seq = _Py_asdl_seq_new(flattened_seq_size, p->arena); + if (!flattened_seq) { + return NULL; + } + + int flattened_seq_idx = 0; + for (Py_ssize_t i = 0, l = asdl_seq_LEN(seqs); i < l; i++) { + asdl_seq *inner_seq = asdl_seq_GET(seqs, i); + for (int j = 0, li = asdl_seq_LEN(inner_seq); j < li; j++) { + asdl_seq_SET(flattened_seq, flattened_seq_idx++, asdl_seq_GET(inner_seq, j)); + } + } + assert(flattened_seq_idx == flattened_seq_size); + + return flattened_seq; +} + +/* Creates a new name of the form <first_name>.<second_name> */ +expr_ty +_PyPegen_join_names_with_dot(Parser *p, expr_ty first_name, expr_ty second_name) +{ + assert(first_name != NULL && second_name != NULL); + PyObject *first_identifier = first_name->v.Name.id; + PyObject *second_identifier = second_name->v.Name.id; + + if (PyUnicode_READY(first_identifier) == -1) { + return NULL; + } + if (PyUnicode_READY(second_identifier) == -1) { + return NULL; + } + const char *first_str = PyUnicode_AsUTF8(first_identifier); + if (!first_str) { + return NULL; + } + const char *second_str = PyUnicode_AsUTF8(second_identifier); + if (!second_str) { + return NULL; + } + ssize_t len = strlen(first_str) + strlen(second_str) + 1; // +1 for the dot + + PyObject *str = PyBytes_FromStringAndSize(NULL, len); + if (!str) { + return NULL; + } + + char *s = PyBytes_AS_STRING(str); + if (!s) { + return NULL; + } + + strcpy(s, first_str); + s += strlen(first_str); + *s++ = '.'; + strcpy(s, second_str); + s += strlen(second_str); + *s = '\0'; + + PyObject *uni = PyUnicode_DecodeUTF8(PyBytes_AS_STRING(str), PyBytes_GET_SIZE(str), NULL); + Py_DECREF(str); + if (!uni) { + return NULL; + } + PyUnicode_InternInPlace(&uni); + if (PyArena_AddPyObject(p->arena, uni) < 0) { + Py_DECREF(uni); + return NULL; + } + + return _Py_Name(uni, Load, EXTRA_EXPR(first_name, second_name)); +} + +/* Counts the total number of dots in seq's tokens */ +int +_PyPegen_seq_count_dots(asdl_seq *seq) +{ + int number_of_dots = 0; + for (Py_ssize_t i = 0, l = asdl_seq_LEN(seq); i < l; i++) { + Token *current_expr = asdl_seq_GET(seq, i); + switch (current_expr->type) { + case ELLIPSIS: + number_of_dots += 3; + break; + case DOT: + number_of_dots += 1; + break; + default: + assert(current_expr->type == ELLIPSIS || current_expr->type == DOT); + } + } + + return number_of_dots; +} + +/* Creates an alias with '*' as the identifier name */ +alias_ty +_PyPegen_alias_for_star(Parser *p) +{ + PyObject *str = PyUnicode_InternFromString("*"); + if (!str) { + return NULL; + } + if (PyArena_AddPyObject(p->arena, str) < 0) { + Py_DECREF(str); + return NULL; + } + return alias(str, NULL, p->arena); +} + +/* Creates a new asdl_seq* with the identifiers of all the names in seq */ +asdl_seq * +_PyPegen_map_names_to_ids(Parser *p, asdl_seq *seq) +{ + int len = asdl_seq_LEN(seq); + assert(len > 0); + + asdl_seq *new_seq = _Py_asdl_seq_new(len, p->arena); + if (!new_seq) { + return NULL; + } + for (Py_ssize_t i = 0; i < len; i++) { + expr_ty e = asdl_seq_GET(seq, i); + asdl_seq_SET(new_seq, i, e->v.Name.id); + } + return new_seq; +} + +/* Constructs a CmpopExprPair */ +CmpopExprPair * +_PyPegen_cmpop_expr_pair(Parser *p, cmpop_ty cmpop, expr_ty expr) +{ + assert(expr != NULL); + CmpopExprPair *a = PyArena_Malloc(p->arena, sizeof(CmpopExprPair)); + if (!a) { + return NULL; + } + a->cmpop = cmpop; + a->expr = expr; + return a; +} + +asdl_int_seq * +_PyPegen_get_cmpops(Parser *p, asdl_seq *seq) +{ + int len = asdl_seq_LEN(seq); + assert(len > 0); + + asdl_int_seq *new_seq = _Py_asdl_int_seq_new(len, p->arena); + if (!new_seq) { + return NULL; + } + for (Py_ssize_t i = 0; i < len; i++) { + CmpopExprPair *pair = asdl_seq_GET(seq, i); + asdl_seq_SET(new_seq, i, pair->cmpop); + } + return new_seq; +} + +asdl_seq * +_PyPegen_get_exprs(Parser *p, asdl_seq *seq) +{ + int len = asdl_seq_LEN(seq); + assert(len > 0); + + asdl_seq *new_seq = _Py_asdl_seq_new(len, p->arena); + if (!new_seq) { + return NULL; + } + for (Py_ssize_t i = 0; i < len; i++) { + CmpopExprPair *pair = asdl_seq_GET(seq, i); + asdl_seq_SET(new_seq, i, pair->expr); + } + return new_seq; +} + +/* Creates an asdl_seq* where all the elements have been changed to have ctx as context */ +static asdl_seq * +_set_seq_context(Parser *p, asdl_seq *seq, expr_context_ty ctx) +{ + int len = asdl_seq_LEN(seq); + if (len == 0) { + return NULL; + } + + asdl_seq *new_seq = _Py_asdl_seq_new(len, p->arena); + if (!new_seq) { + return NULL; + } + for (Py_ssize_t i = 0; i < len; i++) { + expr_ty e = asdl_seq_GET(seq, i); + asdl_seq_SET(new_seq, i, _PyPegen_set_expr_context(p, e, ctx)); + } + return new_seq; +} + +static expr_ty +_set_name_context(Parser *p, expr_ty e, expr_context_ty ctx) +{ + return _Py_Name(e->v.Name.id, ctx, EXTRA_EXPR(e, e)); +} + +static expr_ty +_set_tuple_context(Parser *p, expr_ty e, expr_context_ty ctx) +{ + return _Py_Tuple(_set_seq_context(p, e->v.Tuple.elts, ctx), ctx, EXTRA_EXPR(e, e)); +} + +static expr_ty +_set_list_context(Parser *p, expr_ty e, expr_context_ty ctx) +{ + return _Py_List(_set_seq_context(p, e->v.List.elts, ctx), ctx, EXTRA_EXPR(e, e)); +} + +static expr_ty +_set_subscript_context(Parser *p, expr_ty e, expr_context_ty ctx) +{ + return _Py_Subscript(e->v.Subscript.value, e->v.Subscript.slice, ctx, EXTRA_EXPR(e, e)); +} + +static expr_ty +_set_attribute_context(Parser *p, expr_ty e, expr_context_ty ctx) +{ + return _Py_Attribute(e->v.Attribute.value, e->v.Attribute.attr, ctx, EXTRA_EXPR(e, e)); +} + +static expr_ty +_set_starred_context(Parser *p, expr_ty e, expr_context_ty ctx) +{ + return _Py_Starred(_PyPegen_set_expr_context(p, e->v.Starred.value, ctx), ctx, EXTRA_EXPR(e, e)); +} + +/* Creates an `expr_ty` equivalent to `expr` but with `ctx` as context */ +expr_ty +_PyPegen_set_expr_context(Parser *p, expr_ty expr, expr_context_ty ctx) +{ + assert(expr != NULL); + + expr_ty new = NULL; + switch (expr->kind) { + case Name_kind: + new = _set_name_context(p, expr, ctx); + break; + case Tuple_kind: + new = _set_tuple_context(p, expr, ctx); + break; + case List_kind: + new = _set_list_context(p, expr, ctx); + break; + case Subscript_kind: + new = _set_subscript_context(p, expr, ctx); + break; + case Attribute_kind: + new = _set_attribute_context(p, expr, ctx); + break; + case Starred_kind: + new = _set_starred_context(p, expr, ctx); + break; + default: + new = expr; + } + return new; +} + +/* Constructs a KeyValuePair that is used when parsing a dict's key value pairs */ +KeyValuePair * +_PyPegen_key_value_pair(Parser *p, expr_ty key, expr_ty value) +{ + KeyValuePair *a = PyArena_Malloc(p->arena, sizeof(KeyValuePair)); + if (!a) { + return NULL; + } + a->key = key; + a->value = value; + return a; +} + +/* Extracts all keys from an asdl_seq* of KeyValuePair*'s */ +asdl_seq * +_PyPegen_get_keys(Parser *p, asdl_seq *seq) +{ + int len = asdl_seq_LEN(seq); + asdl_seq *new_seq = _Py_asdl_seq_new(len, p->arena); + if (!new_seq) { + return NULL; + } + for (Py_ssize_t i = 0; i < len; i++) { + KeyValuePair *pair = asdl_seq_GET(seq, i); + asdl_seq_SET(new_seq, i, pair->key); + } + return new_seq; +} + +/* Extracts all values from an asdl_seq* of KeyValuePair*'s */ +asdl_seq * +_PyPegen_get_values(Parser *p, asdl_seq *seq) +{ + int len = asdl_seq_LEN(seq); + asdl_seq *new_seq = _Py_asdl_seq_new(len, p->arena); + if (!new_seq) { + return NULL; + } + for (Py_ssize_t i = 0; i < len; i++) { + KeyValuePair *pair = asdl_seq_GET(seq, i); + asdl_seq_SET(new_seq, i, pair->value); + } + return new_seq; +} + +/* Constructs a NameDefaultPair */ +NameDefaultPair * +_PyPegen_name_default_pair(Parser *p, arg_ty arg, expr_ty value) +{ + NameDefaultPair *a = PyArena_Malloc(p->arena, sizeof(NameDefaultPair)); + if (!a) { + return NULL; + } + a->arg = arg; + a->value = value; + return a; +} + +/* Constructs a SlashWithDefault */ +SlashWithDefault * +_PyPegen_slash_with_default(Parser *p, asdl_seq *plain_names, asdl_seq *names_with_defaults) +{ + SlashWithDefault *a = PyArena_Malloc(p->arena, sizeof(SlashWithDefault)); + if (!a) { + return NULL; + } + a->plain_names = plain_names; + a->names_with_defaults = names_with_defaults; + return a; +} + +/* Constructs a StarEtc */ +StarEtc * +_PyPegen_star_etc(Parser *p, arg_ty vararg, asdl_seq *kwonlyargs, arg_ty kwarg) +{ + StarEtc *a = PyArena_Malloc(p->arena, sizeof(StarEtc)); + if (!a) { + return NULL; + } + a->vararg = vararg; + a->kwonlyargs = kwonlyargs; + a->kwarg = kwarg; + return a; +} + +asdl_seq * +_PyPegen_join_sequences(Parser *p, asdl_seq *a, asdl_seq *b) +{ + int first_len = asdl_seq_LEN(a); + int second_len = asdl_seq_LEN(b); + asdl_seq *new_seq = _Py_asdl_seq_new(first_len + second_len, p->arena); + if (!new_seq) { + return NULL; + } + + int k = 0; + for (Py_ssize_t i = 0; i < first_len; i++) { + asdl_seq_SET(new_seq, k++, asdl_seq_GET(a, i)); + } + for (Py_ssize_t i = 0; i < second_len; i++) { + asdl_seq_SET(new_seq, k++, asdl_seq_GET(b, i)); + } + + return new_seq; +} + +static asdl_seq * +_get_names(Parser *p, asdl_seq *names_with_defaults) +{ + int len = asdl_seq_LEN(names_with_defaults); + asdl_seq *seq = _Py_asdl_seq_new(len, p->arena); + if (!seq) { + return NULL; + } + for (Py_ssize_t i = 0; i < len; i++) { + NameDefaultPair *pair = asdl_seq_GET(names_with_defaults, i); + asdl_seq_SET(seq, i, pair->arg); + } + return seq; +} + +static asdl_seq * +_get_defaults(Parser *p, asdl_seq *names_with_defaults) +{ + int len = asdl_seq_LEN(names_with_defaults); + asdl_seq *seq = _Py_asdl_seq_new(len, p->arena); + if (!seq) { + return NULL; + } + for (Py_ssize_t i = 0; i < len; i++) { + NameDefaultPair *pair = asdl_seq_GET(names_with_defaults, i); + asdl_seq_SET(seq, i, pair->value); + } + return seq; +} + +/* Constructs an arguments_ty object out of all the parsed constructs in the parameters rule */ +arguments_ty +_PyPegen_make_arguments(Parser *p, asdl_seq *slash_without_default, + SlashWithDefault *slash_with_default, asdl_seq *plain_names, + asdl_seq *names_with_default, StarEtc *star_etc) +{ + asdl_seq *posonlyargs; + if (slash_without_default != NULL) { + posonlyargs = slash_without_default; + } + else if (slash_with_default != NULL) { + asdl_seq *slash_with_default_names = + _get_names(p, slash_with_default->names_with_defaults); + if (!slash_with_default_names) { + return NULL; + } + posonlyargs = _PyPegen_join_sequences(p, slash_with_default->plain_names, slash_with_default_names); + if (!posonlyargs) { + return NULL; + } + } + else { + posonlyargs = _Py_asdl_seq_new(0, p->arena); + if (!posonlyargs) { + return NULL; + } + } + + asdl_seq *posargs; + if (plain_names != NULL && names_with_default != NULL) { + asdl_seq *names_with_default_names = _get_names(p, names_with_default); + if (!names_with_default_names) { + return NULL; + } + posargs = _PyPegen_join_sequences(p, plain_names, names_with_default_names); + if (!posargs) { + return NULL; + } + } + else if (plain_names == NULL && names_with_default != NULL) { + posargs = _get_names(p, names_with_default); + if (!posargs) { + return NULL; + } + } + else if (plain_names != NULL && names_with_default == NULL) { + posargs = plain_names; + } + else { + posargs = _Py_asdl_seq_new(0, p->arena); + if (!posargs) { + return NULL; + } + } + + asdl_seq *posdefaults; + if (slash_with_default != NULL && names_with_default != NULL) { + asdl_seq *slash_with_default_values = + _get_defaults(p, slash_with_default->names_with_defaults); + if (!slash_with_default_values) { + return NULL; + } + asdl_seq *names_with_default_values = _get_defaults(p, names_with_default); + if (!names_with_default_values) { + return NULL; + } + posdefaults = _PyPegen_join_sequences(p, slash_with_default_values, names_with_default_values); + if (!posdefaults) { + return NULL; + } + } + else if (slash_with_default == NULL && names_with_default != NULL) { + posdefaults = _get_defaults(p, names_with_default); + if (!posdefaults) { + return NULL; + } + } + else if (slash_with_default != NULL && names_with_default == NULL) { + posdefaults = _get_defaults(p, slash_with_default->names_with_defaults); + if (!posdefaults) { + return NULL; + } + } + else { + posdefaults = _Py_asdl_seq_new(0, p->arena); + if (!posdefaults) { + return NULL; + } + } + + arg_ty vararg = NULL; + if (star_etc != NULL && star_etc->vararg != NULL) { + vararg = star_etc->vararg; + } + + asdl_seq *kwonlyargs; + if (star_etc != NULL && star_etc->kwonlyargs != NULL) { + kwonlyargs = _get_names(p, star_etc->kwonlyargs); + if (!kwonlyargs) { + return NULL; + } + } + else { + kwonlyargs = _Py_asdl_seq_new(0, p->arena); + if (!kwonlyargs) { + return NULL; + } + } + + asdl_seq *kwdefaults; + if (star_etc != NULL && star_etc->kwonlyargs != NULL) { + kwdefaults = _get_defaults(p, star_etc->kwonlyargs); + if (!kwdefaults) { + return NULL; + } + } + else { + kwdefaults = _Py_asdl_seq_new(0, p->arena); + if (!kwdefaults) { + return NULL; + } + } + + arg_ty kwarg = NULL; + if (star_etc != NULL && star_etc->kwarg != NULL) { + kwarg = star_etc->kwarg; + } + + return _Py_arguments(posonlyargs, posargs, vararg, kwonlyargs, kwdefaults, kwarg, + posdefaults, p->arena); +} + +/* Constructs an empty arguments_ty object, that gets used when a function accepts no + * arguments. */ +arguments_ty +_PyPegen_empty_arguments(Parser *p) +{ + asdl_seq *posonlyargs = _Py_asdl_seq_new(0, p->arena); + if (!posonlyargs) { + return NULL; + } + asdl_seq *posargs = _Py_asdl_seq_new(0, p->arena); + if (!posargs) { + return NULL; + } + asdl_seq *posdefaults = _Py_asdl_seq_new(0, p->arena); + if (!posdefaults) { + return NULL; + } + asdl_seq *kwonlyargs = _Py_asdl_seq_new(0, p->arena); + if (!kwonlyargs) { + return NULL; + } + asdl_seq *kwdefaults = _Py_asdl_seq_new(0, p->arena); + if (!kwdefaults) { + return NULL; + } + + return _Py_arguments(posonlyargs, posargs, NULL, kwonlyargs, kwdefaults, NULL, kwdefaults, + p->arena); +} + +/* Encapsulates the value of an operator_ty into an AugOperator struct */ +AugOperator * +_PyPegen_augoperator(Parser *p, operator_ty kind) +{ + AugOperator *a = PyArena_Malloc(p->arena, sizeof(AugOperator)); + if (!a) { + return NULL; + } + a->kind = kind; + return a; +} + +/* Construct a FunctionDef equivalent to function_def, but with decorators */ +stmt_ty +_PyPegen_function_def_decorators(Parser *p, asdl_seq *decorators, stmt_ty function_def) +{ + assert(function_def != NULL); + if (function_def->kind == AsyncFunctionDef_kind) { + return _Py_AsyncFunctionDef( + function_def->v.FunctionDef.name, function_def->v.FunctionDef.args, + function_def->v.FunctionDef.body, decorators, function_def->v.FunctionDef.returns, + function_def->v.FunctionDef.type_comment, function_def->lineno, + function_def->col_offset, function_def->end_lineno, function_def->end_col_offset, + p->arena); + } + + return _Py_FunctionDef(function_def->v.FunctionDef.name, function_def->v.FunctionDef.args, + function_def->v.FunctionDef.body, decorators, + function_def->v.FunctionDef.returns, + function_def->v.FunctionDef.type_comment, function_def->lineno, + function_def->col_offset, function_def->end_lineno, + function_def->end_col_offset, p->arena); +} + +/* Construct a ClassDef equivalent to class_def, but with decorators */ +stmt_ty +_PyPegen_class_def_decorators(Parser *p, asdl_seq *decorators, stmt_ty class_def) +{ + assert(class_def != NULL); + return _Py_ClassDef(class_def->v.ClassDef.name, class_def->v.ClassDef.bases, + class_def->v.ClassDef.keywords, class_def->v.ClassDef.body, decorators, + class_def->lineno, class_def->col_offset, class_def->end_lineno, + class_def->end_col_offset, p->arena); +} + +/* Construct a KeywordOrStarred */ +KeywordOrStarred * +_PyPegen_keyword_or_starred(Parser *p, void *element, int is_keyword) +{ + KeywordOrStarred *a = PyArena_Malloc(p->arena, sizeof(KeywordOrStarred)); + if (!a) { + return NULL; + } + a->element = element; + a->is_keyword = is_keyword; + return a; +} + +/* Get the number of starred expressions in an asdl_seq* of KeywordOrStarred*s */ +static int +_seq_number_of_starred_exprs(asdl_seq *seq) +{ + int n = 0; + for (Py_ssize_t i = 0, l = asdl_seq_LEN(seq); i < l; i++) { + KeywordOrStarred *k = asdl_seq_GET(seq, i); + if (!k->is_keyword) { + n++; + } + } + return n; +} + +/* Extract the starred expressions of an asdl_seq* of KeywordOrStarred*s */ +asdl_seq * +_PyPegen_seq_extract_starred_exprs(Parser *p, asdl_seq *kwargs) +{ + int new_len = _seq_number_of_starred_exprs(kwargs); + if (new_len == 0) { + return NULL; + } + asdl_seq *new_seq = _Py_asdl_seq_new(new_len, p->arena); + if (!new_seq) { + return NULL; + } + + int idx = 0; + for (Py_ssize_t i = 0, len = asdl_seq_LEN(kwargs); i < len; i++) { + KeywordOrStarred *k = asdl_seq_GET(kwargs, i); + if (!k->is_keyword) { + asdl_seq_SET(new_seq, idx++, k->element); + } + } + return new_seq; +} + +/* Return a new asdl_seq* with only the keywords in kwargs */ +asdl_seq * +_PyPegen_seq_delete_starred_exprs(Parser *p, asdl_seq *kwargs) +{ + int len = asdl_seq_LEN(kwargs); + int new_len = len - _seq_number_of_starred_exprs(kwargs); + if (new_len == 0) { + return NULL; + } + asdl_seq *new_seq = _Py_asdl_seq_new(new_len, p->arena); + if (!new_seq) { + return NULL; + } + + int idx = 0; + for (Py_ssize_t i = 0; i < len; i++) { + KeywordOrStarred *k = asdl_seq_GET(kwargs, i); + if (k->is_keyword) { + asdl_seq_SET(new_seq, idx++, k->element); + } + } + return new_seq; +} + +expr_ty +_PyPegen_concatenate_strings(Parser *p, asdl_seq *strings) +{ + int len = asdl_seq_LEN(strings); + assert(len > 0); + + Token *first = asdl_seq_GET(strings, 0); + Token *last = asdl_seq_GET(strings, len - 1); + + int bytesmode = 0; + PyObject *bytes_str = NULL; + + FstringParser state; + _PyPegen_FstringParser_Init(&state); + + for (Py_ssize_t i = 0; i < len; i++) { + Token *t = asdl_seq_GET(strings, i); + + int this_bytesmode; + int this_rawmode; + PyObject *s; + const char *fstr; + Py_ssize_t fstrlen = -1; + + char *this_str = PyBytes_AsString(t->bytes); + if (!this_str) { + goto error; + } + + if (_PyPegen_parsestr(p, this_str, &this_bytesmode, &this_rawmode, &s, &fstr, &fstrlen) != 0) { + goto error; + } + + /* Check that we are not mixing bytes with unicode. */ + if (i != 0 && bytesmode != this_bytesmode) { + RAISE_SYNTAX_ERROR("cannot mix bytes and nonbytes literals"); + Py_XDECREF(s); + goto error; + } + bytesmode = this_bytesmode; + + if (fstr != NULL) { + assert(s == NULL && !bytesmode); + + int result = _PyPegen_FstringParser_ConcatFstring(p, &state, &fstr, fstr + fstrlen, + this_rawmode, 0, first, t, last); + if (result < 0) { + goto error; + } + } + else { + /* String or byte string. */ + assert(s != NULL && fstr == NULL); + assert(bytesmode ? PyBytes_CheckExact(s) : PyUnicode_CheckExact(s)); + + if (bytesmode) { + if (i == 0) { + bytes_str = s; + } + else { + PyBytes_ConcatAndDel(&bytes_str, s); + if (!bytes_str) { + goto error; + } + } + } + else { + /* This is a regular string. Concatenate it. */ + if (_PyPegen_FstringParser_ConcatAndDel(&state, s) < 0) { + goto error; + } + } + } + } + + if (bytesmode) { + if (PyArena_AddPyObject(p->arena, bytes_str) < 0) { + goto error; + } + return Constant(bytes_str, NULL, first->lineno, first->col_offset, last->end_lineno, + last->end_col_offset, p->arena); + } + + return _PyPegen_FstringParser_Finish(p, &state, first, last); + +error: + Py_XDECREF(bytes_str); + _PyPegen_FstringParser_Dealloc(&state); + if (PyErr_Occurred()) { + raise_decode_error(p); + } + return NULL; +} diff --git a/Parser/pegen/pegen.h b/Parser/pegen/pegen.h new file mode 100644 index 0000000000000..5acd9883f3fd8 --- /dev/null +++ b/Parser/pegen/pegen.h @@ -0,0 +1,179 @@ +#ifndef PEGEN_H +#define PEGEN_H + +#define PY_SSIZE_T_CLEAN +#include <Python.h> +#include <token.h> +#include <Python-ast.h> +#include <pyarena.h> + +typedef struct _memo { + int type; + void *node; + int mark; + struct _memo *next; +} Memo; + +typedef struct { + int type; + PyObject *bytes; + int lineno, col_offset, end_lineno, end_col_offset; + Memo *memo; +} Token; + +typedef struct { + char *str; + int type; +} KeywordToken; + +typedef struct { + struct tok_state *tok; + Token **tokens; + int mark; + int fill, size; + PyArena *arena; + KeywordToken **keywords; + int n_keyword_lists; + int start_rule; + int *errcode; + int parsing_started; + PyObject* normalize; + int starting_lineno; + int starting_col_offset; + int error_indicator; +} Parser; + +typedef struct { + cmpop_ty cmpop; + expr_ty expr; +} CmpopExprPair; + +typedef struct { + expr_ty key; + expr_ty value; +} KeyValuePair; + +typedef struct { + arg_ty arg; + expr_ty value; +} NameDefaultPair; + +typedef struct { + asdl_seq *plain_names; + asdl_seq *names_with_defaults; // asdl_seq* of NameDefaultsPair's +} SlashWithDefault; + +typedef struct { + arg_ty vararg; + asdl_seq *kwonlyargs; // asdl_seq* of NameDefaultsPair's + arg_ty kwarg; +} StarEtc; + +typedef struct { + operator_ty kind; +} AugOperator; + +typedef struct { + void *element; + int is_keyword; +} KeywordOrStarred; + +void _PyPegen_clear_memo_statistics(void); +PyObject *_PyPegen_get_memo_statistics(void); + +int _PyPegen_insert_memo(Parser *p, int mark, int type, void *node); +int _PyPegen_update_memo(Parser *p, int mark, int type, void *node); +int _PyPegen_is_memoized(Parser *p, int type, void *pres); + +int _PyPegen_lookahead_with_string(int, void *(func)(Parser *, const char *), Parser *, const char *); +int _PyPegen_lookahead_with_int(int, Token *(func)(Parser *, int), Parser *, int); +int _PyPegen_lookahead(int, void *(func)(Parser *), Parser *); + +Token *_PyPegen_expect_token(Parser *p, int type); +Token *_PyPegen_get_last_nonnwhitespace_token(Parser *); +int _PyPegen_fill_token(Parser *p); +void *_PyPegen_async_token(Parser *p); +void *_PyPegen_await_token(Parser *p); +void *_PyPegen_endmarker_token(Parser *p); +expr_ty _PyPegen_name_token(Parser *p); +void *_PyPegen_newline_token(Parser *p); +void *_PyPegen_indent_token(Parser *p); +void *_PyPegen_dedent_token(Parser *p); +expr_ty _PyPegen_number_token(Parser *p); +void *_PyPegen_string_token(Parser *p); +const char *_PyPegen_get_expr_name(expr_ty); +void *_PyPegen_raise_error(Parser *p, PyObject *, const char *errmsg, ...); +void *_PyPegen_dummy_name(Parser *p, ...); + +#define UNUSED(expr) do { (void)(expr); } while (0) +#define EXTRA_EXPR(head, tail) head->lineno, head->col_offset, tail->end_lineno, tail->end_col_offset, p->arena +#define EXTRA start_lineno, start_col_offset, end_lineno, end_col_offset, p->arena +#define RAISE_SYNTAX_ERROR(msg, ...) _PyPegen_raise_error(p, PyExc_SyntaxError, msg, ##__VA_ARGS__) +#define RAISE_INDENTATION_ERROR(msg, ...) _PyPegen_raise_error(p, PyExc_IndentationError, msg, ##__VA_ARGS__) + +Py_LOCAL_INLINE(void *) +CHECK_CALL(Parser *p, void *result) +{ + if (result == NULL) { + assert(PyErr_Occurred()); + p->error_indicator = 1; + } + return result; +} + +/* This is needed for helper functions that are allowed to + return NULL without an error. Example: _PyPegen_seq_extract_starred_exprs */ +Py_LOCAL_INLINE(void *) +CHECK_CALL_NULL_ALLOWED(Parser *p, void *result) +{ + if (result == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + } + return result; +} + +#define CHECK(result) CHECK_CALL(p, result) +#define CHECK_NULL_ALLOWED(result) CHECK_CALL_NULL_ALLOWED(p, result) + +PyObject *_PyPegen_new_identifier(Parser *, char *); +Parser *_PyPegen_Parser_New(struct tok_state *, int, int *, PyArena *); +void _PyPegen_Parser_Free(Parser *); +mod_ty _PyPegen_run_parser_from_file_pointer(FILE *, int, PyObject *, const char *, + const char *, const char *, int *, PyArena *); +void *_PyPegen_run_parser(Parser *); +mod_ty _PyPegen_run_parser_from_file(const char *, int, PyObject *, PyArena *); +mod_ty _PyPegen_run_parser_from_string(const char *, int, PyObject *, int, PyArena *); +void *_PyPegen_interactive_exit(Parser *); +asdl_seq *_PyPegen_singleton_seq(Parser *, void *); +asdl_seq *_PyPegen_seq_insert_in_front(Parser *, void *, asdl_seq *); +asdl_seq *_PyPegen_seq_flatten(Parser *, asdl_seq *); +expr_ty _PyPegen_join_names_with_dot(Parser *, expr_ty, expr_ty); +int _PyPegen_seq_count_dots(asdl_seq *); +alias_ty _PyPegen_alias_for_star(Parser *); +asdl_seq *_PyPegen_map_names_to_ids(Parser *, asdl_seq *); +CmpopExprPair *_PyPegen_cmpop_expr_pair(Parser *, cmpop_ty, expr_ty); +asdl_int_seq *_PyPegen_get_cmpops(Parser *p, asdl_seq *); +asdl_seq *_PyPegen_get_exprs(Parser *, asdl_seq *); +expr_ty _PyPegen_set_expr_context(Parser *, expr_ty, expr_context_ty); +KeyValuePair *_PyPegen_key_value_pair(Parser *, expr_ty, expr_ty); +asdl_seq *_PyPegen_get_keys(Parser *, asdl_seq *); +asdl_seq *_PyPegen_get_values(Parser *, asdl_seq *); +NameDefaultPair *_PyPegen_name_default_pair(Parser *, arg_ty, expr_ty); +SlashWithDefault *_PyPegen_slash_with_default(Parser *, asdl_seq *, asdl_seq *); +StarEtc *_PyPegen_star_etc(Parser *, arg_ty, asdl_seq *, arg_ty); +arguments_ty _PyPegen_make_arguments(Parser *, asdl_seq *, SlashWithDefault *, + asdl_seq *, asdl_seq *, StarEtc *); +arguments_ty _PyPegen_empty_arguments(Parser *); +AugOperator *_PyPegen_augoperator(Parser*, operator_ty type); +stmt_ty _PyPegen_function_def_decorators(Parser *, asdl_seq *, stmt_ty); +stmt_ty _PyPegen_class_def_decorators(Parser *, asdl_seq *, stmt_ty); +KeywordOrStarred *_PyPegen_keyword_or_starred(Parser *, void *, int); +asdl_seq *_PyPegen_seq_extract_starred_exprs(Parser *, asdl_seq *); +asdl_seq *_PyPegen_seq_delete_starred_exprs(Parser *, asdl_seq *); +expr_ty _PyPegen_concatenate_strings(Parser *p, asdl_seq *); +asdl_seq *_PyPegen_join_sequences(Parser *, asdl_seq *, asdl_seq *); +void *_PyPegen_arguments_parsing_error(Parser *, expr_ty); + +void *_PyPegen_parse(Parser *); + +#endif diff --git a/Programs/_testembed.c b/Programs/_testembed.c index 249f7e2304f92..8165aa74df1a0 100644 --- a/Programs/_testembed.c +++ b/Programs/_testembed.c @@ -485,6 +485,9 @@ static int test_init_from_config(void) config.install_signal_handlers = 0; + putenv("PYTHONOLDPARSER="); + config.use_peg = 0; + /* FIXME: test use_environment */ putenv("PYTHONHASHSEED=42"); diff --git a/Python/ast_opt.c b/Python/ast_opt.c index 1766321a11a5b..ff786d6f8d63e 100644 --- a/Python/ast_opt.c +++ b/Python/ast_opt.c @@ -563,7 +563,8 @@ astfold_expr(expr_ty node_, PyArena *ctx_, _PyASTOptimizeState *state) CALL(fold_tuple, expr_ty, node_); break; case Name_kind: - if (_PyUnicode_EqualToASCIIString(node_->v.Name.id, "__debug__")) { + if (node_->v.Name.ctx == Load && + _PyUnicode_EqualToASCIIString(node_->v.Name.id, "__debug__")) { return make_const(node_, PyBool_FromLong(!state->optimize), ctx_); } break; diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index 22ee5969473f5..18883353575f3 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -816,7 +816,12 @@ builtin_compile_impl(PyObject *module, PyObject *source, PyObject *filename, if (str == NULL) goto error; + int current_use_peg = PyInterpreterState_Get()->config.use_peg; + if (flags & PyCF_TYPE_COMMENTS || feature_version >= 0) { + PyInterpreterState_Get()->config.use_peg = 0; + } result = Py_CompileStringObject(str, filename, start[compile_mode], &cf, optimize); + PyInterpreterState_Get()->config.use_peg = current_use_peg; Py_XDECREF(source_copy); goto finally; diff --git a/Python/compile.c b/Python/compile.c index 54e6516b3ad01..3c21fbabf663f 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -2152,6 +2152,55 @@ compiler_default_arguments(struct compiler *c, arguments_ty args) return funcflags; } +static int +forbidden_name(struct compiler *c, identifier name, expr_context_ty ctx) +{ + + if (ctx == Store && _PyUnicode_EqualToASCIIString(name, "__debug__")) { + compiler_error(c, "cannot assign to __debug__"); + return 1; + } + return 0; +} + +static int +compiler_check_debug_one_arg(struct compiler *c, arg_ty arg) +{ + if (arg != NULL) { + if (forbidden_name(c, arg->arg, Store)) + return 0; + } + return 1; +} + +static int +compiler_check_debug_args_seq(struct compiler *c, asdl_seq *args) +{ + if (args != NULL) { + for (int i = 0, n = asdl_seq_LEN(args); i < n; i++) { + if (!compiler_check_debug_one_arg(c, asdl_seq_GET(args, i))) + return 0; + } + } + return 1; +} + +static int +compiler_check_debug_args(struct compiler *c, arguments_ty args) +{ + if (!compiler_check_debug_args_seq(c, args->posonlyargs)) + return 0; + if (!compiler_check_debug_args_seq(c, args->args)) + return 0; + if (!compiler_check_debug_one_arg(c, args->vararg)) + return 0; + if (!compiler_check_debug_args_seq(c, args->kwonlyargs)) + return 0; + if (!compiler_check_debug_one_arg(c, args->kwarg)) + return 0; + return 1; +} + static int compiler_function(struct compiler *c, stmt_ty s, int is_async) { @@ -2189,6 +2238,9 @@ compiler_function(struct compiler *c, stmt_ty s, int is_async) scope_type = COMPILER_SCOPE_FUNCTION; } + if (!compiler_check_debug_args(c, args)) + return 0; + if (!compiler_decorators(c, decos)) return 0; @@ -2596,6 +2648,9 @@ compiler_lambda(struct compiler *c, expr_ty e) arguments_ty args = e->v.Lambda.args; assert(e->kind == Lambda_kind); + if (!compiler_check_debug_args(c, args)) + return 0; + if (!name) { name = PyUnicode_InternFromString("<lambda>"); if (!name) @@ -3505,6 +3560,9 @@ compiler_nameop(struct compiler *c, identifier name, expr_context_ty ctx) !_PyUnicode_EqualToASCIIString(name, "True") && !_PyUnicode_EqualToASCIIString(name, "False")); + if (forbidden_name(c, name, ctx)) + return 0; + mangled = _Py_Mangle(c->u->u_private, name); if (!mangled) return 0; @@ -4056,6 +4114,9 @@ validate_keywords(struct compiler *c, asdl_seq *keywords) if (key->arg == NULL) { continue; } + if (forbidden_name(c, key->arg, Store)) { + return -1; + } for (Py_ssize_t j = i + 1; j < nkeywords; j++) { keyword_ty other = ((keyword_ty)asdl_seq_GET(keywords, j)); if (other->arg && !PyUnicode_Compare(key->arg, other->arg)) { @@ -5013,6 +5074,8 @@ compiler_visit_expr1(struct compiler *c, expr_ty e) ADDOP_NAME(c, LOAD_ATTR, e->v.Attribute.attr, names); break; case Store: + if (forbidden_name(c, e->v.Attribute.attr, e->v.Attribute.ctx)) + return 0; ADDOP_NAME(c, STORE_ATTR, e->v.Attribute.attr, names); break; case Del: @@ -5183,6 +5246,8 @@ compiler_annassign(struct compiler *c, stmt_ty s) } switch (targ->kind) { case Name_kind: + if (forbidden_name(c, targ->v.Name.id, Store)) + return 0; /* If we have a simple name in a module or class, store annotation. */ if (s->v.AnnAssign.simple && (c->u->u_scope_type == COMPILER_SCOPE_MODULE || @@ -5200,6 +5265,8 @@ compiler_annassign(struct compiler *c, stmt_ty s) } break; case Attribute_kind: + if (forbidden_name(c, targ->v.Attribute.attr, Store)) + return 0; if (!s->v.AnnAssign.value && !check_ann_expr(c, targ->v.Attribute.value)) { return 0; diff --git a/Python/importlib.h b/Python/importlib.h index 4bd8b62d809ee..59e0272b61dab 100644 --- a/Python/importlib.h +++ b/Python/importlib.h @@ -1594,50 +1594,51 @@ const unsigned char _Py_M__importlib_bootstrap[] = { 0,218,1,120,90,5,119,104,101,114,101,90,9,102,114,111, 109,95,110,97,109,101,90,3,101,120,99,114,10,0,0,0, 114,10,0,0,0,114,11,0,0,0,114,215,0,0,0,9, - 4,0,0,115,44,0,0,0,0,10,8,1,10,1,4,1, - 12,2,4,1,28,2,8,1,14,1,10,1,2,255,8,2, - 10,1,14,1,2,1,14,1,14,4,10,1,16,255,2,2, - 12,1,26,1,114,215,0,0,0,99,1,0,0,0,0,0, - 0,0,0,0,0,0,3,0,0,0,6,0,0,0,67,0, - 0,0,115,146,0,0,0,124,0,160,0,100,1,161,1,125, - 1,124,0,160,0,100,2,161,1,125,2,124,1,100,3,117, - 1,114,82,124,2,100,3,117,1,114,78,124,1,124,2,106, - 1,107,3,114,78,116,2,106,3,100,4,124,1,155,2,100, - 5,124,2,106,1,155,2,100,6,157,5,116,4,100,7,100, - 8,141,3,1,0,124,1,83,0,124,2,100,3,117,1,114, - 96,124,2,106,1,83,0,116,2,106,3,100,9,116,4,100, - 7,100,8,141,3,1,0,124,0,100,10,25,0,125,1,100, - 11,124,0,118,1,114,142,124,1,160,5,100,12,161,1,100, - 13,25,0,125,1,124,1,83,0,41,14,122,167,67,97,108, - 99,117,108,97,116,101,32,119,104,97,116,32,95,95,112,97, - 99,107,97,103,101,95,95,32,115,104,111,117,108,100,32,98, - 101,46,10,10,32,32,32,32,95,95,112,97,99,107,97,103, - 101,95,95,32,105,115,32,110,111,116,32,103,117,97,114,97, - 110,116,101,101,100,32,116,111,32,98,101,32,100,101,102,105, - 110,101,100,32,111,114,32,99,111,117,108,100,32,98,101,32, - 115,101,116,32,116,111,32,78,111,110,101,10,32,32,32,32, - 116,111,32,114,101,112,114,101,115,101,110,116,32,116,104,97, - 116,32,105,116,115,32,112,114,111,112,101,114,32,118,97,108, - 117,101,32,105,115,32,117,110,107,110,111,119,110,46,10,10, - 32,32,32,32,114,146,0,0,0,114,106,0,0,0,78,122, - 32,95,95,112,97,99,107,97,103,101,95,95,32,33,61,32, - 95,95,115,112,101,99,95,95,46,112,97,114,101,110,116,32, - 40,122,4,32,33,61,32,250,1,41,233,3,0,0,0,41, - 1,90,10,115,116,97,99,107,108,101,118,101,108,122,89,99, - 97,110,39,116,32,114,101,115,111,108,118,101,32,112,97,99, - 107,97,103,101,32,102,114,111,109,32,95,95,115,112,101,99, - 95,95,32,111,114,32,95,95,112,97,99,107,97,103,101,95, - 95,44,32,102,97,108,108,105,110,103,32,98,97,99,107,32, - 111,110,32,95,95,110,97,109,101,95,95,32,97,110,100,32, - 95,95,112,97,116,104,95,95,114,1,0,0,0,114,142,0, - 0,0,114,129,0,0,0,114,22,0,0,0,41,6,114,35, - 0,0,0,114,131,0,0,0,114,193,0,0,0,114,194,0, - 0,0,114,195,0,0,0,114,130,0,0,0,41,3,218,7, - 103,108,111,98,97,108,115,114,187,0,0,0,114,96,0,0, - 0,114,10,0,0,0,114,10,0,0,0,114,11,0,0,0, - 218,17,95,99,97,108,99,95,95,95,112,97,99,107,97,103, - 101,95,95,46,4,0,0,115,34,0,0,0,0,7,10,1, - 10,1,8,1,18,1,22,2,4,254,6,3,4,1,8,1, + 4,0,0,115,52,0,0,0,0,10,8,1,10,1,4,1, + 12,2,4,1,4,1,2,255,4,1,8,255,10,2,8,1, + 14,1,10,1,2,255,8,2,10,1,14,1,2,1,14,1, + 14,4,10,1,16,255,2,2,12,1,26,1,114,215,0,0, + 0,99,1,0,0,0,0,0,0,0,0,0,0,0,3,0, + 0,0,6,0,0,0,67,0,0,0,115,146,0,0,0,124, + 0,160,0,100,1,161,1,125,1,124,0,160,0,100,2,161, + 1,125,2,124,1,100,3,117,1,114,82,124,2,100,3,117, + 1,114,78,124,1,124,2,106,1,107,3,114,78,116,2,106, + 3,100,4,124,1,155,2,100,5,124,2,106,1,155,2,100, + 6,157,5,116,4,100,7,100,8,141,3,1,0,124,1,83, + 0,124,2,100,3,117,1,114,96,124,2,106,1,83,0,116, + 2,106,3,100,9,116,4,100,7,100,8,141,3,1,0,124, + 0,100,10,25,0,125,1,100,11,124,0,118,1,114,142,124, + 1,160,5,100,12,161,1,100,13,25,0,125,1,124,1,83, + 0,41,14,122,167,67,97,108,99,117,108,97,116,101,32,119, + 104,97,116,32,95,95,112,97,99,107,97,103,101,95,95,32, + 115,104,111,117,108,100,32,98,101,46,10,10,32,32,32,32, + 95,95,112,97,99,107,97,103,101,95,95,32,105,115,32,110, + 111,116,32,103,117,97,114,97,110,116,101,101,100,32,116,111, + 32,98,101,32,100,101,102,105,110,101,100,32,111,114,32,99, + 111,117,108,100,32,98,101,32,115,101,116,32,116,111,32,78, + 111,110,101,10,32,32,32,32,116,111,32,114,101,112,114,101, + 115,101,110,116,32,116,104,97,116,32,105,116,115,32,112,114, + 111,112,101,114,32,118,97,108,117,101,32,105,115,32,117,110, + 107,110,111,119,110,46,10,10,32,32,32,32,114,146,0,0, + 0,114,106,0,0,0,78,122,32,95,95,112,97,99,107,97, + 103,101,95,95,32,33,61,32,95,95,115,112,101,99,95,95, + 46,112,97,114,101,110,116,32,40,122,4,32,33,61,32,250, + 1,41,233,3,0,0,0,41,1,90,10,115,116,97,99,107, + 108,101,118,101,108,122,89,99,97,110,39,116,32,114,101,115, + 111,108,118,101,32,112,97,99,107,97,103,101,32,102,114,111, + 109,32,95,95,115,112,101,99,95,95,32,111,114,32,95,95, + 112,97,99,107,97,103,101,95,95,44,32,102,97,108,108,105, + 110,103,32,98,97,99,107,32,111,110,32,95,95,110,97,109, + 101,95,95,32,97,110,100,32,95,95,112,97,116,104,95,95, + 114,1,0,0,0,114,142,0,0,0,114,129,0,0,0,114, + 22,0,0,0,41,6,114,35,0,0,0,114,131,0,0,0, + 114,193,0,0,0,114,194,0,0,0,114,195,0,0,0,114, + 130,0,0,0,41,3,218,7,103,108,111,98,97,108,115,114, + 187,0,0,0,114,96,0,0,0,114,10,0,0,0,114,10, + 0,0,0,114,11,0,0,0,218,17,95,99,97,108,99,95, + 95,95,112,97,99,107,97,103,101,95,95,46,4,0,0,115, + 42,0,0,0,0,7,10,1,10,1,8,1,18,1,6,1, + 2,255,4,1,4,255,6,2,4,254,6,3,4,1,8,1, 6,2,6,2,4,254,6,3,8,1,8,1,14,1,114,221, 0,0,0,114,10,0,0,0,99,5,0,0,0,0,0,0, 0,0,0,0,0,9,0,0,0,5,0,0,0,67,0,0, diff --git a/Python/importlib_external.h b/Python/importlib_external.h index 9618f9f0f6a8f..dd237428867ba 100644 --- a/Python/importlib_external.h +++ b/Python/importlib_external.h @@ -481,10 +481,11 @@ const unsigned char _Py_M__importlib_bootstrap_external[] = { 108,101,118,101,108,90,13,98,97,115,101,95,102,105,108,101, 110,97,109,101,114,5,0,0,0,114,5,0,0,0,114,8, 0,0,0,218,17,115,111,117,114,99,101,95,102,114,111,109, - 95,99,97,99,104,101,116,1,0,0,115,52,0,0,0,0, + 95,99,97,99,104,101,116,1,0,0,115,68,0,0,0,0, 9,12,1,8,1,10,1,12,1,4,1,10,1,12,1,14, - 1,16,1,4,1,4,1,12,1,8,1,18,2,10,1,8, - 1,16,1,10,1,16,1,10,1,14,2,16,1,10,1,16, + 1,16,1,4,1,4,1,12,1,8,1,2,1,2,255,4, + 1,2,255,8,2,10,1,8,1,16,1,10,1,16,1,10, + 1,4,1,2,255,8,2,16,1,10,1,4,1,2,255,10, 2,14,1,114,102,0,0,0,99,1,0,0,0,0,0,0, 0,0,0,0,0,5,0,0,0,9,0,0,0,67,0,0, 0,115,124,0,0,0,116,0,124,0,131,1,100,1,107,2, diff --git a/Python/initconfig.c b/Python/initconfig.c index c313d91ac7309..7662d6192686d 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -68,6 +68,7 @@ static const char usage_3[] = "\ -X opt : set implementation-specific option. The following options are available:\n\ \n\ -X faulthandler: enable faulthandler\n\ + -X oldparser: enable the traditional LL(1) parser; also PYTHONOLDPARSER\n\ -X showrefcount: output the total reference count and number of used\n\ memory blocks when the program finishes or after each statement in the\n\ interactive interpreter. This only works on debug builds\n\ @@ -634,6 +635,7 @@ _PyConfig_InitCompatConfig(PyConfig *config) #ifdef MS_WINDOWS config->legacy_windows_stdio = -1; #endif + config->use_peg = 1; } @@ -791,6 +793,7 @@ _PyConfig_Copy(PyConfig *config, const PyConfig *config2) COPY_ATTR(isolated); COPY_ATTR(use_environment); COPY_ATTR(dev_mode); + COPY_ATTR(use_peg); COPY_ATTR(install_signal_handlers); COPY_ATTR(use_hash_seed); COPY_ATTR(hash_seed); @@ -894,6 +897,7 @@ config_as_dict(const PyConfig *config) SET_ITEM_INT(isolated); SET_ITEM_INT(use_environment); SET_ITEM_INT(dev_mode); + SET_ITEM_INT(use_peg); SET_ITEM_INT(install_signal_handlers); SET_ITEM_INT(use_hash_seed); SET_ITEM_UINT(hash_seed); @@ -1428,6 +1432,11 @@ config_read_complex_options(PyConfig *config) config->import_time = 1; } + if (config_get_env(config, "PYTHONOLDPARSER") + || config_get_xoption(config, L"oldparser")) { + config->use_peg = 0; + } + PyStatus status; if (config->tracemalloc < 0) { status = config_init_tracemalloc(config); @@ -2507,6 +2516,7 @@ PyConfig_Read(PyConfig *config) assert(config->isolated >= 0); assert(config->use_environment >= 0); assert(config->dev_mode >= 0); + assert(config->use_peg >= 0); assert(config->install_signal_handlers >= 0); assert(config->use_hash_seed >= 0); assert(config->faulthandler >= 0); diff --git a/Python/pythonrun.c b/Python/pythonrun.c index 0a25ebc854ff5..6199f0c66104a 100644 --- a/Python/pythonrun.c +++ b/Python/pythonrun.c @@ -29,6 +29,8 @@ #include "ast.h" // PyAST_FromNodeObject() #include "marshal.h" // PyMarshal_ReadLongFromFile() +#include <pegen_interface.h> // PyPegen_ASTFrom* + #ifdef MS_WINDOWS # include "malloc.h" // alloca() #endif @@ -183,6 +185,7 @@ PyRun_InteractiveOneObjectEx(FILE *fp, PyObject *filename, PyArena *arena; const char *ps1 = "", *ps2 = "", *enc = NULL; int errcode = 0; + int use_peg = _PyInterpreterState_GET()->config.use_peg; _Py_IDENTIFIER(encoding); _Py_IDENTIFIER(__main__); @@ -235,9 +238,17 @@ PyRun_InteractiveOneObjectEx(FILE *fp, PyObject *filename, Py_XDECREF(oenc); return -1; } - mod = PyParser_ASTFromFileObject(fp, filename, enc, - Py_single_input, ps1, ps2, - flags, &errcode, arena); + + if (use_peg) { + mod = PyPegen_ASTFromFileObject(fp, filename, Py_single_input, + enc, ps1, ps2, &errcode, arena); + } + else { + mod = PyParser_ASTFromFileObject(fp, filename, enc, + Py_single_input, ps1, ps2, + flags, &errcode, arena); + } + Py_XDECREF(v); Py_XDECREF(w); Py_XDECREF(oenc); @@ -1019,6 +1030,7 @@ PyRun_StringFlags(const char *str, int start, PyObject *globals, mod_ty mod; PyArena *arena; PyObject *filename; + int use_peg = _PyInterpreterState_GET()->config.use_peg; filename = _PyUnicode_FromId(&PyId_string); /* borrowed */ if (filename == NULL) @@ -1028,7 +1040,13 @@ PyRun_StringFlags(const char *str, int start, PyObject *globals, if (arena == NULL) return NULL; - mod = PyParser_ASTFromStringObject(str, filename, start, flags, arena); + if (use_peg) { + mod = PyPegen_ASTFromStringObject(str, filename, start, flags, arena); + } + else { + mod = PyParser_ASTFromStringObject(str, filename, start, flags, arena); + } + if (mod != NULL) ret = run_mod(mod, filename, globals, locals, flags, arena); PyArena_Free(arena); @@ -1043,6 +1061,7 @@ PyRun_FileExFlags(FILE *fp, const char *filename_str, int start, PyObject *globa mod_ty mod; PyArena *arena = NULL; PyObject *filename; + int use_peg = _PyInterpreterState_GET()->config.use_peg; filename = PyUnicode_DecodeFSDefault(filename_str); if (filename == NULL) @@ -1052,8 +1071,15 @@ PyRun_FileExFlags(FILE *fp, const char *filename_str, int start, PyObject *globa if (arena == NULL) goto exit; - mod = PyParser_ASTFromFileObject(fp, filename, NULL, start, 0, 0, - flags, NULL, arena); + if (use_peg) { + mod = PyPegen_ASTFromFileObject(fp, filename, start, NULL, NULL, NULL, + NULL, arena); + } + else { + mod = PyParser_ASTFromFileObject(fp, filename, NULL, start, 0, 0, + flags, NULL, arena); + } + if (closeit) fclose(fp); if (mod == NULL) { @@ -1196,11 +1222,17 @@ Py_CompileStringObject(const char *str, PyObject *filename, int start, { PyCodeObject *co; mod_ty mod; + int use_peg = _PyInterpreterState_GET()->config.use_peg; PyArena *arena = PyArena_New(); if (arena == NULL) return NULL; - mod = PyParser_ASTFromStringObject(str, filename, start, flags, arena); + if (use_peg) { + mod = PyPegen_ASTFromStringObject(str, filename, start, flags, arena); + } + else { + mod = PyParser_ASTFromStringObject(str, filename, start, flags, arena); + } if (mod == NULL) { PyArena_Free(arena); return NULL; @@ -1297,13 +1329,19 @@ _Py_SymtableStringObjectFlags(const char *str, PyObject *filename, int start, Py { struct symtable *st; mod_ty mod; + int use_peg = _PyInterpreterState_GET()->config.use_peg; PyArena *arena; arena = PyArena_New(); if (arena == NULL) return NULL; - mod = PyParser_ASTFromStringObject(str, filename, start, flags, arena); + if (use_peg) { + mod = PyPegen_ASTFromStringObject(str, filename, start, flags, arena); + } + else { + mod = PyParser_ASTFromStringObject(str, filename, start, flags, arena); + } if (mod == NULL) { PyArena_Free(arena); return NULL; diff --git a/Python/sysmodule.c b/Python/sysmodule.c index 92ea5e7d637b9..cf3ddff44d19e 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -2427,6 +2427,7 @@ static PyStructSequence_Field flags_fields[] = { {"inspect", "-i"}, {"interactive", "-i"}, {"optimize", "-O or -OO"}, + {"use_peg", "-p old or -p new"}, {"dont_write_bytecode", "-B"}, {"no_user_site", "-s"}, {"no_site", "-S"}, @@ -2447,7 +2448,7 @@ static PyStructSequence_Desc flags_desc = { "sys.flags", /* name */ flags__doc__, /* doc */ flags_fields, /* fields */ - 15 + 16 }; static PyObject* @@ -2470,6 +2471,7 @@ make_flags(PyThreadState *tstate) SetFlag(config->inspect); SetFlag(config->interactive); SetFlag(config->optimization_level); + SetFlag(config->use_peg); SetFlag(!config->write_bytecode); SetFlag(!config->user_site_directory); SetFlag(!config->site_import); diff --git a/Tools/README b/Tools/README index 6c5fb20818120..b6d0b18e5a5c2 100644 --- a/Tools/README +++ b/Tools/README @@ -23,6 +23,8 @@ msi Support for packaging Python as an MSI package on Windows. parser Un-parsing tool to generate code from an AST. +peg_generator PEG-based parser generator (pegen) used for new parser. + pynche A Tkinter-based color editor. scripts A number of useful single-file programs, e.g. tabnanny.py diff --git a/Tools/peg_generator/.clang-format b/Tools/peg_generator/.clang-format new file mode 100644 index 0000000000000..b2bb93dbd059b --- /dev/null +++ b/Tools/peg_generator/.clang-format @@ -0,0 +1,17 @@ +# A clang-format style that approximates Python's PEP 7 +BasedOnStyle: Google +AlwaysBreakAfterReturnType: All +AllowShortIfStatementsOnASingleLine: false +AlignAfterOpenBracket: Align +BreakBeforeBraces: Stroustrup +ColumnLimit: 95 +DerivePointerAlignment: false +IndentWidth: 4 +Language: Cpp +PointerAlignment: Right +ReflowComments: true +SpaceBeforeParens: ControlStatements +SpacesInParentheses: false +TabWidth: 4 +UseTab: Never +SortIncludes: false diff --git a/Tools/peg_generator/.gitignore b/Tools/peg_generator/.gitignore new file mode 100644 index 0000000000000..91c41f89e8cb5 --- /dev/null +++ b/Tools/peg_generator/.gitignore @@ -0,0 +1,3 @@ +peg_extension/parse.c +data/xxl.py +@data diff --git a/Tools/peg_generator/Makefile b/Tools/peg_generator/Makefile new file mode 100644 index 0000000000000..fb67a21b67b6e --- /dev/null +++ b/Tools/peg_generator/Makefile @@ -0,0 +1,116 @@ +UNAME_S := $(shell uname -s) +ifeq ($(UNAME_S),Linux) + PYTHON ?= ../../python +endif +ifeq ($(UNAME_S),Darwin) + PYTHON ?= ../../python.exe +endif + +CPYTHON ?= ../../Lib +MYPY ?= mypy + +GRAMMAR = ../../Grammar/python.gram +TESTFILE = data/cprog.py +TIMEFILE = data/xxl.py +TESTDIR = . +TESTFLAGS = --short + +data/xxl.py: + $(PYTHON) -m zipfile -e data/xxl.zip data + +build: peg_extension/parse.c + +peg_extension/parse.c: $(GRAMMAR) pegen/*.py peg_extension/peg_extension.c ../../Parser/pegen/pegen.c ../../Parser/pegen/parse_string.c ../../Parser/pegen/*.h pegen/grammar_parser.py + $(PYTHON) -m pegen -q -c $(GRAMMAR) -o peg_extension/parse.c --compile-extension + +clean: + -rm -f peg_extension/*.o peg_extension/*.so peg_extension/parse.c + -rm -f data/xxl.py + +dump: peg_extension/parse.c + cat -n $(TESTFILE) + $(PYTHON) -c "from peg_extension import parse; import ast; t = parse.parse_file('$(TESTFILE)', mode=1); print(ast.dump(t))" + +regen-metaparser: pegen/metagrammar.gram pegen/*.py + $(PYTHON) -m pegen -q -c pegen/metagrammar.gram -o pegen/grammar_parser.py + +# Note: These targets really depend on the generated shared object in peg_extension/parse.*.so but +# this has different names in different systems so we are abusing the implicit dependency on +# parse.c by the use of --compile-extension. + +.PHONY: test + +test: run + +run: peg_extension/parse.c + $(PYTHON) -c "from peg_extension import parse; t = parse.parse_file('$(TESTFILE)'); exec(t)" + +compile: peg_extension/parse.c + $(PYTHON) -c "from peg_extension import parse; t = parse.parse_file('$(TESTFILE)', mode=2)" + +parse: peg_extension/parse.c + $(PYTHON) -c "from peg_extension import parse; t = parse.parse_file('$(TESTFILE)', mode=1)" + +check: peg_extension/parse.c + $(PYTHON) -c "from peg_extension import parse; t = parse.parse_file('$(TESTFILE)', mode=0)" + +stats: peg_extension/parse.c data/xxl.py + $(PYTHON) -c "from peg_extension import parse; t = parse.parse_file('$(TIMEFILE)', mode=0); parse.dump_memo_stats()" >@data + $(PYTHON) scripts/joinstats.py @data + +time: time_compile + +time_compile: peg_extension/parse.c data/xxl.py + $(PYTHON) scripts/benchmark.py --parser=pegen --target=xxl compile + +time_parse: peg_extension/parse.c data/xxl.py + $(PYTHON) scripts/benchmark.py --parser=pegen --target=xxl parse + +time_check: peg_extension/parse.c data/xxl.py + $(PYTHON) scripts/benchmark.py --parser=pegen --target=xxl check + +time_stdlib: time_stdlib_compile + +time_stdlib_compile: data/xxl.py + $(PYTHON) scripts/benchmark.py --parser=cpython --target=xxl compile + +time_stdlib_parse: data/xxl.py + $(PYTHON) scripts/benchmark.py --parser=cpython --target=xxl parse + +test_local: + $(PYTHON) scripts/test_parse_directory.py \ + -g $(GRAMMAR) \ + -d $(TESTDIR) \ + $(TESTFLAGS) \ + --exclude "*/failset/*" \ + --exclude "*/failset/**" \ + --exclude "*/failset/**/*" + +test_global: $(CPYTHON) + $(PYTHON) scripts/test_parse_directory.py \ + -g $(GRAMMAR) \ + -d $(CPYTHON) \ + $(TESTFLAGS) \ + --exclude "*/test2to3/*" \ + --exclude "*/test2to3/**/*" \ + --exclude "*/bad*" \ + --exclude "*/lib2to3/tests/data/*" + +mypy: regen-metaparser + $(MYPY) # For list of files, see mypy.ini + +format-python: + black pegen scripts + +bench: + $(PYTHON) scripts/benchmark.py --parser=pegen --target=stdlib check + +format: format-python + +find_max_nesting: + $(PYTHON) scripts/find_max_nesting.py + +tags: TAGS + +TAGS: pegen/*.py test/test_pegen.py + etags pegen/*.py test/test_pegen.py diff --git a/Tools/peg_generator/data/cprog.py b/Tools/peg_generator/data/cprog.py new file mode 100644 index 0000000000000..07b96f0753a98 --- /dev/null +++ b/Tools/peg_generator/data/cprog.py @@ -0,0 +1,10 @@ +if 1: + print("Hello " + "world") + if 0: + print("then") + print("clause") + elif 1: + pass + elif 1: + pass + else: print("else-clause") diff --git a/Tools/peg_generator/data/xxl.zip b/Tools/peg_generator/data/xxl.zip new file mode 100644 index 0000000000000..5421408809b0d Binary files /dev/null and b/Tools/peg_generator/data/xxl.zip differ diff --git a/Tools/peg_generator/mypy.ini b/Tools/peg_generator/mypy.ini new file mode 100644 index 0000000000000..80d5c057ca1a9 --- /dev/null +++ b/Tools/peg_generator/mypy.ini @@ -0,0 +1,26 @@ +[mypy] +files = pegen, scripts + +follow_imports = error +no_implicit_optional = True +strict_optional = True + +#check_untyped_defs = True +disallow_untyped_calls = True +disallow_untyped_defs = True + +disallow_any_generics = true +disallow_any_unimported = True +disallow_incomplete_defs = True +disallow_subclassing_any = True + +warn_unused_configs = True +warn_unused_ignores = true +warn_redundant_casts = true +warn_no_return = True + +show_traceback = True +show_error_codes = True + +[mypy-pegen.grammar_parser] +strict_optional = False diff --git a/Tools/peg_generator/peg_extension/peg_extension.c b/Tools/peg_generator/peg_extension/peg_extension.c new file mode 100644 index 0000000000000..d8d36a0a1a5b0 --- /dev/null +++ b/Tools/peg_generator/peg_extension/peg_extension.c @@ -0,0 +1,153 @@ +#include "pegen.h" + +PyObject * +_build_return_object(mod_ty module, int mode, PyObject *filename_ob, PyArena *arena) +{ + PyObject *result = NULL; + + if (mode == 2) { + result = (PyObject *)PyAST_CompileObject(module, filename_ob, NULL, -1, arena); + } else if (mode == 1) { + result = PyAST_mod2obj(module); + } else { + result = Py_None; + Py_INCREF(result); + + } + + return result; +} + +static PyObject * +parse_file(PyObject *self, PyObject *args, PyObject *kwds) +{ + static char *keywords[] = {"file", "mode", NULL}; + const char *filename; + int mode = 2; + if (!PyArg_ParseTupleAndKeywords(args, kwds, "s|i", keywords, &filename, &mode)) { + return NULL; + } + if (mode < 0 || mode > 2) { + return PyErr_Format(PyExc_ValueError, "Bad mode, must be 0 <= mode <= 2"); + } + + PyArena *arena = PyArena_New(); + if (arena == NULL) { + return NULL; + } + + PyObject *result = NULL; + + PyObject *filename_ob = PyUnicode_FromString(filename); + if (filename_ob == NULL) { + goto error; + } + + mod_ty res = _PyPegen_run_parser_from_file(filename, Py_file_input, filename_ob, arena); + if (res == NULL) { + goto error; + } + + result = _build_return_object(res, mode, filename_ob, arena); + +error: + Py_XDECREF(filename_ob); + PyArena_Free(arena); + return result; +} + +static PyObject * +parse_string(PyObject *self, PyObject *args, PyObject *kwds) +{ + static char *keywords[] = {"str", "mode", NULL}; + const char *the_string; + int mode = 2; + if (!PyArg_ParseTupleAndKeywords(args, kwds, "s|i", keywords, &the_string, &mode)) { + return NULL; + } + if (mode < 0 || mode > 2) { + return PyErr_Format(PyExc_ValueError, "Bad mode, must be 0 <= mode <= 2"); + } + + PyArena *arena = PyArena_New(); + if (arena == NULL) { + return NULL; + } + + PyObject *result = NULL; + + PyObject *filename_ob = PyUnicode_FromString("<string>"); + if (filename_ob == NULL) { + goto error; + } + + mod_ty res = _PyPegen_run_parser_from_string(the_string, Py_file_input, filename_ob, + PyCF_IGNORE_COOKIE, arena); + if (res == NULL) { + goto error; + } + result = _build_return_object(res, mode, filename_ob, arena); + +error: + Py_XDECREF(filename_ob); + PyArena_Free(arena); + return result; +} + +static PyObject * +clear_memo_stats() +{ + _PyPegen_clear_memo_statistics(); + Py_RETURN_NONE; +} + +static PyObject * +get_memo_stats() +{ + return _PyPegen_get_memo_statistics(); +} + +// TODO: Write to Python's sys.stdout instead of C's stdout. +static PyObject * +dump_memo_stats() +{ + PyObject *list = _PyPegen_get_memo_statistics(); + if (list == NULL) { + return NULL; + } + Py_ssize_t len = PyList_Size(list); + for (Py_ssize_t i = 0; i < len; i++) { + PyObject *value = PyList_GetItem(list, i); // Borrowed reference. + long count = PyLong_AsLong(value); + if (count < 0) { + break; + } + if (count > 0) { + printf("%4ld %9ld\n", i, count); + } + } + Py_DECREF(list); + Py_RETURN_NONE; +} + +static PyMethodDef ParseMethods[] = { + {"parse_file", (PyCFunction)(void(*)(void))parse_file, METH_VARARGS|METH_KEYWORDS, "Parse a file."}, + {"parse_string", (PyCFunction)(void(*)(void))parse_string, METH_VARARGS|METH_KEYWORDS, "Parse a string."}, + {"clear_memo_stats", clear_memo_stats, METH_NOARGS}, + {"dump_memo_stats", dump_memo_stats, METH_NOARGS}, + {"get_memo_stats", get_memo_stats, METH_NOARGS}, + {NULL, NULL, 0, NULL} /* Sentinel */ +}; + +static struct PyModuleDef parsemodule = { + PyModuleDef_HEAD_INIT, + .m_name = "parse", + .m_doc = "A parser.", + .m_methods = ParseMethods, +}; + +PyMODINIT_FUNC +PyInit_parse(void) +{ + return PyModule_Create(&parsemodule); +} diff --git a/Tools/peg_generator/pegen/__init__.py b/Tools/peg_generator/pegen/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/Tools/peg_generator/pegen/__main__.py b/Tools/peg_generator/pegen/__main__.py new file mode 100755 index 0000000000000..874b307ab5c18 --- /dev/null +++ b/Tools/peg_generator/pegen/__main__.py @@ -0,0 +1,136 @@ +#!/usr/bin/env python3.8 + +"""pegen -- PEG Generator. + +Search the web for PEG Parsers for reference. +""" + +import argparse +import sys +import time +import token +import traceback + +from typing import Final + +from pegen.build import build_parser_and_generator +from pegen.testutil import print_memstats + + +argparser = argparse.ArgumentParser( + prog="pegen", description="Experimental PEG-like parser generator" +) +argparser.add_argument("-q", "--quiet", action="store_true", help="Don't print the parsed grammar") +argparser.add_argument( + "-v", + "--verbose", + action="count", + default=0, + help="Print timing stats; repeat for more debug output", +) +argparser.add_argument( + "-c", "--cpython", action="store_true", help="Generate C code for inclusion into CPython" +) +argparser.add_argument( + "--compile-extension", + action="store_true", + help="Compile generated C code into an extension module", +) +argparser.add_argument( + "-o", + "--output", + metavar="OUT", + help="Where to write the generated parser (default parse.py or parse.c)", +) +argparser.add_argument("filename", help="Grammar description") +argparser.add_argument( + "--optimized", action="store_true", help="Compile the extension in optimized mode" +) +argparser.add_argument( + "--skip-actions", action="store_true", help="Suppress code emission for rule actions", +) + + +def main() -> None: + args = argparser.parse_args() + verbose = args.verbose + verbose_tokenizer = verbose >= 3 + verbose_parser = verbose == 2 or verbose >= 4 + t0 = time.time() + + output_file = args.output + if not output_file: + if args.cpython: + output_file = "parse.c" + else: + output_file = "parse.py" + + try: + grammar, parser, tokenizer, gen = build_parser_and_generator( + args.filename, + output_file, + args.compile_extension, + verbose_tokenizer, + verbose_parser, + args.verbose, + keep_asserts_in_extension=False if args.optimized else True, + skip_actions=args.skip_actions, + ) + except Exception as err: + if args.verbose: + raise # Show traceback + traceback.print_exception(err.__class__, err, None) + sys.stderr.write("For full traceback, use -v\n") + sys.exit(1) + + if not args.quiet: + if args.verbose: + print("Raw Grammar:") + for line in repr(grammar).splitlines(): + print(" ", line) + + print("Clean Grammar:") + for line in str(grammar).splitlines(): + print(" ", line) + + if args.verbose: + print("First Graph:") + for src, dsts in gen.first_graph.items(): + print(f" {src} -> {', '.join(dsts)}") + print("First SCCS:") + for scc in gen.first_sccs: + print(" ", scc, end="") + if len(scc) > 1: + print( + " # Indirectly left-recursive; leaders:", + {name for name in scc if grammar.rules[name].leader}, + ) + else: + name = next(iter(scc)) + if name in gen.first_graph[name]: + print(" # Left-recursive") + else: + print() + + t1 = time.time() + + if args.verbose: + dt = t1 - t0 + diag = tokenizer.diagnose() + nlines = diag.end[0] + if diag.type == token.ENDMARKER: + nlines -= 1 + print(f"Total time: {dt:.3f} sec; {nlines} lines", end="") + if dt: + print(f"; {nlines / dt:.0f} lines/sec") + else: + print() + print("Caches sizes:") + print(f" token array : {len(tokenizer._tokens):10}") + print(f" cache : {len(parser._cache):10}") + if not print_memstats(): + print("(Can't find psutil; install it for memory stats.)") + + +if __name__ == "__main__": + main() diff --git a/Tools/peg_generator/pegen/build.py b/Tools/peg_generator/pegen/build.py new file mode 100644 index 0000000000000..623b4aeb66069 --- /dev/null +++ b/Tools/peg_generator/pegen/build.py @@ -0,0 +1,169 @@ +import pathlib +import shutil +import tokenize + +from typing import Optional, Tuple + +import distutils.log +from distutils.core import Distribution, Extension +from distutils.command.clean import clean # type: ignore +from distutils.command.build_ext import build_ext # type: ignore + +from pegen.c_generator import CParserGenerator +from pegen.grammar import Grammar +from pegen.grammar_parser import GeneratedParser as GrammarParser +from pegen.parser import Parser +from pegen.parser_generator import ParserGenerator +from pegen.python_generator import PythonParserGenerator +from pegen.tokenizer import Tokenizer + +MOD_DIR = pathlib.Path(__file__).parent + + +def compile_c_extension( + generated_source_path: str, + build_dir: Optional[str] = None, + verbose: bool = False, + keep_asserts: bool = True, +) -> str: + """Compile the generated source for a parser generator into an extension module. + + The extension module will be generated in the same directory as the provided path + for the generated source, with the same basename (in addition to extension module + metadata). For example, for the source mydir/parser.c the generated extension + in a darwin system with python 3.8 will be mydir/parser.cpython-38-darwin.so. + + If *build_dir* is provided, that path will be used as the temporary build directory + of distutils (this is useful in case you want to use a temporary directory). + """ + if verbose: + distutils.log.set_verbosity(distutils.log.DEBUG) + + source_file_path = pathlib.Path(generated_source_path) + extension_name = source_file_path.stem + extra_compile_args = [] + if keep_asserts: + extra_compile_args.append("-UNDEBUG") + extension = [ + Extension( + extension_name, + sources=[ + str(MOD_DIR.parent.parent.parent / "Python" / "Python-ast.c"), + str(MOD_DIR.parent.parent.parent / "Python" / "asdl.c"), + str(MOD_DIR.parent.parent.parent / "Parser" / "tokenizer.c"), + str(MOD_DIR.parent.parent.parent / "Parser" / "pegen" / "pegen.c"), + str(MOD_DIR.parent.parent.parent / "Parser" / "pegen" / "parse_string.c"), + str(MOD_DIR.parent / "peg_extension" / "peg_extension.c"), + generated_source_path, + ], + include_dirs=[ + str(MOD_DIR.parent.parent.parent / "Include" / "internal"), + str(MOD_DIR.parent.parent.parent / "Parser"), + str(MOD_DIR.parent.parent.parent / "Parser" / "pegen"), + ], + extra_compile_args=extra_compile_args, + ) + ] + dist = Distribution({"name": extension_name, "ext_modules": extension}) + cmd = build_ext(dist) + cmd.inplace = True + if build_dir: + cmd.build_temp = build_dir + cmd.ensure_finalized() + cmd.run() + + extension_path = source_file_path.parent / cmd.get_ext_filename(extension_name) + shutil.move(cmd.get_ext_fullpath(extension_name), extension_path) + + cmd = clean(dist) + cmd.finalize_options() + cmd.run() + + return extension_path + + +def build_parser( + grammar_file: str, verbose_tokenizer: bool = False, verbose_parser: bool = False +) -> Tuple[Grammar, Parser, Tokenizer]: + with open(grammar_file) as file: + tokenizer = Tokenizer(tokenize.generate_tokens(file.readline), verbose=verbose_tokenizer) + parser = GrammarParser(tokenizer, verbose=verbose_parser) + grammar = parser.start() + + if not grammar: + raise parser.make_syntax_error(grammar_file) + + return grammar, parser, tokenizer + + +def build_generator( + tokenizer: Tokenizer, + grammar: Grammar, + grammar_file: str, + output_file: str, + compile_extension: bool = False, + verbose_c_extension: bool = False, + keep_asserts_in_extension: bool = True, + skip_actions: bool = False, +) -> ParserGenerator: + # TODO: Allow other extensions; pass the output type as an argument. + if not output_file.endswith((".c", ".py")): + raise RuntimeError("Your output file must either be a .c or .py file") + with open(output_file, "w") as file: + gen: ParserGenerator + if output_file.endswith(".c"): + gen = CParserGenerator(grammar, file, skip_actions=skip_actions) + elif output_file.endswith(".py"): + gen = PythonParserGenerator(grammar, file) # TODO: skip_actions + else: + assert False # Should have been checked above + gen.generate(grammar_file) + + if compile_extension and output_file.endswith(".c"): + compile_c_extension( + output_file, verbose=verbose_c_extension, keep_asserts=keep_asserts_in_extension + ) + + return gen + + +def build_parser_and_generator( + grammar_file: str, + output_file: str, + compile_extension: bool = False, + verbose_tokenizer: bool = False, + verbose_parser: bool = False, + verbose_c_extension: bool = False, + keep_asserts_in_extension: bool = True, + skip_actions: bool = False, +) -> Tuple[Grammar, Parser, Tokenizer, ParserGenerator]: + """Generate rules, parser, tokenizer, parser generator for a given grammar + + Args: + grammar_file (string): Path for the grammar file + output_file (string): Path for the output file + compile_extension (bool, optional): Whether to compile the C extension. + Defaults to False. + verbose_tokenizer (bool, optional): Whether to display additional output + when generating the tokenizer. Defaults to False. + verbose_parser (bool, optional): Whether to display additional output + when generating the parser. Defaults to False. + verbose_c_extension (bool, optional): Whether to display additional + output when compiling the C extension . Defaults to False. + keep_asserts_in_extension (bool, optional): Whether to keep the assert statements + when compiling the extension module. Defaults to True. + skip_actions (bool, optional): Whether to pretend no rule has any actions. + """ + grammar, parser, tokenizer = build_parser(grammar_file, verbose_tokenizer, verbose_parser) + gen = build_generator( + tokenizer, + grammar, + grammar_file, + output_file, + compile_extension, + verbose_c_extension, + keep_asserts_in_extension, + skip_actions=skip_actions, + ) + + return grammar, parser, tokenizer, gen diff --git a/Tools/peg_generator/pegen/c_generator.py b/Tools/peg_generator/pegen/c_generator.py new file mode 100644 index 0000000000000..ce732a09f096b --- /dev/null +++ b/Tools/peg_generator/pegen/c_generator.py @@ -0,0 +1,605 @@ +import ast +import re +from typing import Any, cast, Dict, IO, Optional, List, Text, Tuple + +from pegen.grammar import ( + Cut, + GrammarVisitor, + Rhs, + Alt, + NamedItem, + NameLeaf, + StringLeaf, + Lookahead, + PositiveLookahead, + NegativeLookahead, + Opt, + Repeat0, + Repeat1, + Gather, + Group, + Rule, +) +from pegen import grammar +from pegen.parser_generator import dedupe, ParserGenerator +from pegen.tokenizer import exact_token_types + +EXTENSION_PREFIX = """\ +#include "pegen.h" + +""" + +EXTENSION_SUFFIX = """ +void * +_PyPegen_parse(Parser *p) +{ + // Initialize keywords + p->keywords = reserved_keywords; + p->n_keyword_lists = n_keyword_lists; + + return start_rule(p); +} +""" + + +class CCallMakerVisitor(GrammarVisitor): + def __init__(self, parser_generator: ParserGenerator): + self.gen = parser_generator + self.cache: Dict[Any, Any] = {} + self.keyword_cache: Dict[str, int] = {} + + def keyword_helper(self, keyword: str) -> Tuple[str, str]: + if keyword not in self.keyword_cache: + self.keyword_cache[keyword] = self.gen.keyword_type() + return "keyword", f"_PyPegen_expect_token(p, {self.keyword_cache[keyword]})" + + def visit_NameLeaf(self, node: NameLeaf) -> Tuple[str, str]: + name = node.value + if name in ("NAME", "NUMBER", "STRING"): + name = name.lower() + return f"{name}_var", f"_PyPegen_{name}_token(p)" + if name in ("NEWLINE", "DEDENT", "INDENT", "ENDMARKER", "ASYNC", "AWAIT"): + name = name.lower() + return f"{name}_var", f"_PyPegen_{name}_token(p)" + return f"{name}_var", f"{name}_rule(p)" + + def visit_StringLeaf(self, node: StringLeaf) -> Tuple[str, str]: + val = ast.literal_eval(node.value) + if re.match(r"[a-zA-Z_]\w*\Z", val): # This is a keyword + return self.keyword_helper(val) + else: + assert val in exact_token_types, f"{node.value} is not a known literal" + type = exact_token_types[val] + return "literal", f"_PyPegen_expect_token(p, {type})" + + def visit_Rhs(self, node: Rhs) -> Tuple[Optional[str], str]: + if node in self.cache: + return self.cache[node] + if len(node.alts) == 1 and len(node.alts[0].items) == 1: + self.cache[node] = self.visit(node.alts[0].items[0]) + else: + name = self.gen.name_node(node) + self.cache[node] = f"{name}_var", f"{name}_rule(p)" + return self.cache[node] + + def visit_NamedItem(self, node: NamedItem) -> Tuple[Optional[str], str]: + name, call = self.visit(node.item) + if node.name: + name = node.name + return name, call + + def lookahead_call_helper(self, node: Lookahead, positive: int) -> Tuple[None, str]: + name, call = self.visit(node.node) + func, args = call.split("(", 1) + assert args[-1] == ")" + args = args[:-1] + if not args.startswith("p,"): + return None, f"_PyPegen_lookahead({positive}, {func}, {args})" + elif args[2:].strip().isalnum(): + return None, f"_PyPegen_lookahead_with_int({positive}, {func}, {args})" + else: + return None, f"_PyPegen_lookahead_with_string({positive}, {func}, {args})" + + def visit_PositiveLookahead(self, node: PositiveLookahead) -> Tuple[None, str]: + return self.lookahead_call_helper(node, 1) + + def visit_NegativeLookahead(self, node: NegativeLookahead) -> Tuple[None, str]: + return self.lookahead_call_helper(node, 0) + + def visit_Opt(self, node: Opt) -> Tuple[str, str]: + name, call = self.visit(node.node) + return "opt_var", f"{call}, 1" # Using comma operator! + + def visit_Repeat0(self, node: Repeat0) -> Tuple[str, str]: + if node in self.cache: + return self.cache[node] + name = self.gen.name_loop(node.node, False) + self.cache[node] = f"{name}_var", f"{name}_rule(p)" + return self.cache[node] + + def visit_Repeat1(self, node: Repeat1) -> Tuple[str, str]: + if node in self.cache: + return self.cache[node] + name = self.gen.name_loop(node.node, True) + self.cache[node] = f"{name}_var", f"{name}_rule(p)" + return self.cache[node] + + def visit_Gather(self, node: Gather) -> Tuple[str, str]: + if node in self.cache: + return self.cache[node] + name = self.gen.name_gather(node) + self.cache[node] = f"{name}_var", f"{name}_rule(p)" + return self.cache[node] + + def visit_Group(self, node: Group) -> Tuple[Optional[str], str]: + return self.visit(node.rhs) + + def visit_Cut(self, node: Cut) -> Tuple[str, str]: + return "cut_var", "1" + + +class CParserGenerator(ParserGenerator, GrammarVisitor): + def __init__( + self, + grammar: grammar.Grammar, + file: Optional[IO[Text]], + debug: bool = False, + skip_actions: bool = False, + ): + super().__init__(grammar, file) + self.callmakervisitor: CCallMakerVisitor = CCallMakerVisitor(self) + self._varname_counter = 0 + self.debug = debug + self.skip_actions = skip_actions + + def unique_varname(self, name: str = "tmpvar") -> str: + new_var = name + "_" + str(self._varname_counter) + self._varname_counter += 1 + return new_var + + def call_with_errorcheck_return(self, call_text: str, returnval: str) -> None: + error_var = self.unique_varname() + self.print(f"int {error_var} = {call_text};") + self.print(f"if ({error_var}) {{") + with self.indent(): + self.print(f"return {returnval};") + self.print(f"}}") + + def call_with_errorcheck_goto(self, call_text: str, goto_target: str) -> None: + error_var = self.unique_varname() + self.print(f"int {error_var} = {call_text};") + self.print(f"if ({error_var}) {{") + with self.indent(): + self.print(f"goto {goto_target};") + self.print(f"}}") + + def out_of_memory_return( + self, expr: str, returnval: str, message: str = "Parser out of memory", cleanup_code=None + ) -> None: + self.print(f"if ({expr}) {{") + with self.indent(): + self.print(f'PyErr_Format(PyExc_MemoryError, "{message}");') + if cleanup_code is not None: + self.print(cleanup_code) + self.print(f"return {returnval};") + self.print(f"}}") + + def out_of_memory_goto( + self, expr: str, goto_target: str, message: str = "Parser out of memory" + ) -> None: + self.print(f"if ({expr}) {{") + with self.indent(): + self.print(f'PyErr_Format(PyExc_MemoryError, "{message}");') + self.print(f"goto {goto_target};") + self.print(f"}}") + + def generate(self, filename: str) -> None: + self.collect_todo() + self.print(f"// @generated by pegen.py from {filename}") + header = self.grammar.metas.get("header", EXTENSION_PREFIX) + if header: + self.print(header.rstrip("\n")) + subheader = self.grammar.metas.get("subheader", "") + if subheader: + self.print(subheader) + self._setup_keywords() + for i, (rulename, rule) in enumerate(self.todo.items(), 1000): + comment = " // Left-recursive" if rule.left_recursive else "" + self.print(f"#define {rulename}_type {i}{comment}") + self.print() + for rulename, rule in self.todo.items(): + if rule.is_loop() or rule.is_gather(): + type = "asdl_seq *" + elif rule.type: + type = rule.type + " " + else: + type = "void *" + self.print(f"static {type}{rulename}_rule(Parser *p);") + self.print() + while self.todo: + for rulename, rule in list(self.todo.items()): + del self.todo[rulename] + self.print() + if rule.left_recursive: + self.print("// Left-recursive") + self.visit(rule) + if self.skip_actions: + mode = 0 + else: + mode = int(self.rules["start"].type == "mod_ty") if "start" in self.rules else 1 + if mode == 1 and self.grammar.metas.get("bytecode"): + mode += 1 + modulename = self.grammar.metas.get("modulename", "parse") + trailer = self.grammar.metas.get("trailer", EXTENSION_SUFFIX) + keyword_cache = self.callmakervisitor.keyword_cache + if trailer: + self.print(trailer.rstrip("\n") % dict(mode=mode, modulename=modulename)) + + def _group_keywords_by_length(self) -> Dict[int, List[Tuple[str, int]]]: + groups: Dict[int, List[Tuple[str, int]]] = {} + for keyword_str, keyword_type in self.callmakervisitor.keyword_cache.items(): + length = len(keyword_str) + if length in groups: + groups[length].append((keyword_str, keyword_type)) + else: + groups[length] = [(keyword_str, keyword_type)] + return groups + + def _setup_keywords(self) -> None: + keyword_cache = self.callmakervisitor.keyword_cache + n_keyword_lists = ( + len(max(keyword_cache.keys(), key=len)) + 1 if len(keyword_cache) > 0 else 0 + ) + self.print(f"static const int n_keyword_lists = {n_keyword_lists};") + groups = self._group_keywords_by_length() + self.print("static KeywordToken *reserved_keywords[] = {") + with self.indent(): + num_groups = max(groups) + 1 if groups else 1 + for keywords_length in range(num_groups): + if keywords_length not in groups.keys(): + self.print("NULL,") + else: + self.print("(KeywordToken[]) {") + with self.indent(): + for keyword_str, keyword_type in groups[keywords_length]: + self.print(f'{{"{keyword_str}", {keyword_type}}},') + self.print("{NULL, -1},") + self.print("},") + self.print("};") + + def _set_up_token_start_metadata_extraction(self) -> None: + self.print("if (p->mark == p->fill && _PyPegen_fill_token(p) < 0) {") + with self.indent(): + self.print("p->error_indicator = 1;") + self.print("return NULL;") + self.print("}") + self.print("int start_lineno = p->tokens[mark]->lineno;") + self.print("UNUSED(start_lineno); // Only used by EXTRA macro") + self.print("int start_col_offset = p->tokens[mark]->col_offset;") + self.print("UNUSED(start_col_offset); // Only used by EXTRA macro") + + def _set_up_token_end_metadata_extraction(self) -> None: + self.print("Token *token = _PyPegen_get_last_nonnwhitespace_token(p);") + self.print("if (token == NULL) {") + with self.indent(): + self.print("return NULL;") + self.print("}") + self.print(f"int end_lineno = token->end_lineno;") + self.print("UNUSED(end_lineno); // Only used by EXTRA macro") + self.print(f"int end_col_offset = token->end_col_offset;") + self.print("UNUSED(end_col_offset); // Only used by EXTRA macro") + + def _set_up_rule_memoization(self, node: Rule, result_type: str) -> None: + self.print("{") + with self.indent(): + self.print(f"{result_type} res = NULL;") + self.print(f"if (_PyPegen_is_memoized(p, {node.name}_type, &res))") + with self.indent(): + self.print("return res;") + self.print("int mark = p->mark;") + self.print("int resmark = p->mark;") + self.print("while (1) {") + with self.indent(): + self.call_with_errorcheck_return( + f"_PyPegen_update_memo(p, mark, {node.name}_type, res)", "res" + ) + self.print("p->mark = mark;") + self.print(f"void *raw = {node.name}_raw(p);") + self.print("if (raw == NULL || p->mark <= resmark)") + with self.indent(): + self.print("break;") + self.print("resmark = p->mark;") + self.print("res = raw;") + self.print("}") + self.print("p->mark = resmark;") + self.print("return res;") + self.print("}") + self.print(f"static {result_type}") + self.print(f"{node.name}_raw(Parser *p)") + + def _should_memoize(self, node: Rule) -> bool: + return node.memo and not node.left_recursive + + def _handle_default_rule_body(self, node: Rule, rhs: Rhs, result_type: str) -> None: + memoize = self._should_memoize(node) + + with self.indent(): + self.print("if (p->error_indicator) {") + with self.indent(): + self.print("return NULL;") + self.print("}") + self.print(f"{result_type} res = NULL;") + if memoize: + self.print(f"if (_PyPegen_is_memoized(p, {node.name}_type, &res))") + with self.indent(): + self.print("return res;") + self.print("int mark = p->mark;") + if any(alt.action and "EXTRA" in alt.action for alt in rhs.alts): + self._set_up_token_start_metadata_extraction() + self.visit( + rhs, + is_loop=False, + is_gather=node.is_gather(), + rulename=node.name if memoize else None, + ) + if self.debug: + self.print(f'fprintf(stderr, "Fail at %d: {node.name}\\n", p->mark);') + self.print("res = NULL;") + self.print(" done:") + with self.indent(): + if memoize: + self.print(f"_PyPegen_insert_memo(p, mark, {node.name}_type, res);") + self.print("return res;") + + def _handle_loop_rule_body(self, node: Rule, rhs: Rhs) -> None: + memoize = self._should_memoize(node) + is_repeat1 = node.name.startswith("_loop1") + + with self.indent(): + self.print("if (p->error_indicator) {") + with self.indent(): + self.print("return NULL;") + self.print("}") + self.print(f"void *res = NULL;") + if memoize: + self.print(f"if (_PyPegen_is_memoized(p, {node.name}_type, &res))") + with self.indent(): + self.print("return res;") + self.print("int mark = p->mark;") + self.print("int start_mark = p->mark;") + self.print("void **children = PyMem_Malloc(sizeof(void *));") + self.out_of_memory_return(f"!children", "NULL") + self.print("ssize_t children_capacity = 1;") + self.print("ssize_t n = 0;") + if any(alt.action and "EXTRA" in alt.action for alt in rhs.alts): + self._set_up_token_start_metadata_extraction() + self.visit( + rhs, + is_loop=True, + is_gather=node.is_gather(), + rulename=node.name if memoize else None, + ) + if is_repeat1: + self.print("if (n == 0) {") + with self.indent(): + self.print("PyMem_Free(children);") + self.print("return NULL;") + self.print("}") + self.print("asdl_seq *seq = _Py_asdl_seq_new(n, p->arena);") + self.out_of_memory_return( + f"!seq", + "NULL", + message=f"asdl_seq_new {node.name}", + cleanup_code="PyMem_Free(children);", + ) + self.print("for (int i = 0; i < n; i++) asdl_seq_SET(seq, i, children[i]);") + self.print("PyMem_Free(children);") + if node.name: + self.print(f"_PyPegen_insert_memo(p, start_mark, {node.name}_type, seq);") + self.print("return seq;") + + def visit_Rule(self, node: Rule) -> None: + is_loop = node.is_loop() + is_gather = node.is_gather() + rhs = node.flatten() + if is_loop or is_gather: + result_type = "asdl_seq *" + elif node.type: + result_type = node.type + else: + result_type = "void *" + + for line in str(node).splitlines(): + self.print(f"// {line}") + if node.left_recursive and node.leader: + self.print(f"static {result_type} {node.name}_raw(Parser *);") + + self.print(f"static {result_type}") + self.print(f"{node.name}_rule(Parser *p)") + + if node.left_recursive and node.leader: + self._set_up_rule_memoization(node, result_type) + + self.print("{") + if is_loop: + self._handle_loop_rule_body(node, rhs) + else: + self._handle_default_rule_body(node, rhs, result_type) + self.print("}") + + def visit_NamedItem(self, node: NamedItem, names: List[str]) -> None: + name, call = self.callmakervisitor.visit(node) + if not name: + self.print(call) + else: + name = dedupe(name, names) + self.print(f"({name} = {call})") + + def visit_Rhs( + self, node: Rhs, is_loop: bool, is_gather: bool, rulename: Optional[str] + ) -> None: + if is_loop: + assert len(node.alts) == 1 + for alt in node.alts: + self.visit(alt, is_loop=is_loop, is_gather=is_gather, rulename=rulename) + + def join_conditions(self, keyword: str, node: Any, names: List[str]) -> None: + self.print(f"{keyword} (") + with self.indent(): + first = True + for item in node.items: + if first: + first = False + else: + self.print("&&") + self.visit(item, names=names) + self.print(")") + + def emit_action(self, node: Alt, cleanup_code=None) -> None: + self.print(f"res = {node.action};") + + self.print("if (res == NULL && PyErr_Occurred()) {") + with self.indent(): + self.print("p->error_indicator = 1;") + if cleanup_code: + self.print(cleanup_code) + self.print("return NULL;") + self.print("}") + + if self.debug: + self.print( + f'fprintf(stderr, "Hit with action [%d-%d]: %s\\n", mark, p->mark, "{node}");' + ) + + def emit_default_action(self, is_gather: bool, names: List[str], node: Alt) -> None: + if len(names) > 1: + if is_gather: + assert len(names) == 2 + self.print(f"res = _PyPegen_seq_insert_in_front(p, {names[0]}, {names[1]});") + else: + if self.debug: + self.print( + f'fprintf(stderr, "Hit without action [%d:%d]: %s\\n", mark, p->mark, "{node}");' + ) + self.print(f"res = _PyPegen_dummy_name(p, {', '.join(names)});") + else: + if self.debug: + self.print( + f'fprintf(stderr, "Hit with default action [%d:%d]: %s\\n", mark, p->mark, "{node}");' + ) + self.print(f"res = {names[0]};") + + def emit_dummy_action(self) -> None: + self.print(f"res = _PyPegen_dummy_name(p);") + + def handle_alt_normal(self, node: Alt, is_gather: bool, names: List[str]) -> None: + self.join_conditions(keyword="if", node=node, names=names) + self.print("{") + # We have parsed successfully all the conditions for the option. + with self.indent(): + # Prepare to emmit the rule action and do so + if node.action and "EXTRA" in node.action: + self._set_up_token_end_metadata_extraction() + if self.skip_actions: + self.emit_dummy_action() + elif node.action: + self.emit_action(node) + else: + self.emit_default_action(is_gather, names, node) + + # As the current option has parsed correctly, do not continue with the rest. + self.print(f"goto done;") + self.print("}") + + def handle_alt_loop( + self, node: Alt, is_gather: bool, rulename: Optional[str], names: List[str] + ) -> None: + # Condition of the main body of the alternative + self.join_conditions(keyword="while", node=node, names=names) + self.print("{") + # We have parsed successfully one item! + with self.indent(): + # Prepare to emit the rule action and do so + if node.action and "EXTRA" in node.action: + self._set_up_token_end_metadata_extraction() + if self.skip_actions: + self.emit_dummy_action() + elif node.action: + self.emit_action(node, cleanup_code="PyMem_Free(children);") + else: + self.emit_default_action(is_gather, names, node) + + # Add the result of rule to the temporary buffer of children. This buffer + # will populate later an asdl_seq with all elements to return. + self.print("if (n == children_capacity) {") + with self.indent(): + self.print("children_capacity *= 2;") + self.print("children = PyMem_Realloc(children, children_capacity*sizeof(void *));") + self.out_of_memory_return(f"!children", "NULL", message=f"realloc {rulename}") + self.print("}") + self.print(f"children[n++] = res;") + self.print("mark = p->mark;") + self.print("}") + + def visit_Alt( + self, node: Alt, is_loop: bool, is_gather: bool, rulename: Optional[str] + ) -> None: + self.print(f"{{ // {node}") + with self.indent(): + # Prepare variable declarations for the alternative + vars = self.collect_vars(node) + for v, var_type in sorted(item for item in vars.items() if item[0] is not None): + if not var_type: + var_type = "void *" + else: + var_type += " " + if v == "cut_var": + v += " = 0" # cut_var must be initialized + self.print(f"{var_type}{v};") + if v == "opt_var": + self.print("UNUSED(opt_var); // Silence compiler warnings") + + names: List[str] = [] + if is_loop: + self.handle_alt_loop(node, is_gather, rulename, names) + else: + self.handle_alt_normal(node, is_gather, names) + + self.print("p->mark = mark;") + if "cut_var" in names: + self.print("if (cut_var) return NULL;") + self.print("}") + + def collect_vars(self, node: Alt) -> Dict[str, Optional[str]]: + names: List[str] = [] + types = {} + for item in node.items: + name, type = self.add_var(item, names) + types[name] = type + return types + + def add_var(self, node: NamedItem, names: List[str]) -> Tuple[str, Optional[str]]: + name: str + call: str + name, call = self.callmakervisitor.visit(node.item) + type = None + if not name: + return name, type + if name.startswith("cut"): + return name, "int" + if name.endswith("_var"): + rulename = name[:-4] + rule = self.rules.get(rulename) + if rule is not None: + if rule.is_loop() or rule.is_gather(): + type = "asdl_seq *" + else: + type = rule.type + elif name.startswith("_loop") or name.startswith("_gather"): + type = "asdl_seq *" + elif name in ("name_var", "string_var", "number_var"): + type = "expr_ty" + if node.name: + name = node.name + name = dedupe(name, names) + return name, type diff --git a/Tools/peg_generator/pegen/first_sets.py b/Tools/peg_generator/pegen/first_sets.py new file mode 100755 index 0000000000000..da30eba99ce5a --- /dev/null +++ b/Tools/peg_generator/pegen/first_sets.py @@ -0,0 +1,153 @@ +#!/usr/bin/env python3.8 + +import argparse +import collections +import pprint +import sys +from typing import Optional, Set, Dict + +from pegen.build import build_parser +from pegen.grammar import ( + Alt, + Cut, + Gather, + Grammar, + GrammarVisitor, + Group, + Leaf, + Lookahead, + NamedItem, + NameLeaf, + NegativeLookahead, + Opt, + Repeat, + Repeat0, + Repeat1, + Rhs, + Rule, + StringLeaf, + PositiveLookahead, +) + +argparser = argparse.ArgumentParser( + prog="calculate_first_sets", description="Calculate the first sets of a grammar", +) +argparser.add_argument("grammar_file", help="The grammar file") + + +class FirstSetCalculator(GrammarVisitor): + def __init__(self, rules: Dict[str, Rule]) -> None: + self.rules = rules + for rule in rules.values(): + rule.nullable_visit(rules) + self.first_sets: Dict[str, Set[str]] = dict() + self.in_process: Set[str] = set() + + def calculate(self) -> Dict[str, Set[str]]: + for name, rule in self.rules.items(): + self.visit(rule) + return self.first_sets + + def visit_Alt(self, item: Alt) -> Set[str]: + result: Set[str] = set() + to_remove: Set[str] = set() + for other in item.items: + new_terminals = self.visit(other) + if isinstance(other.item, NegativeLookahead): + to_remove |= new_terminals + result |= new_terminals + if to_remove: + result -= to_remove + + # If the set of new terminals can start with the empty string, + # it means that the item is completelly nullable and we should + # also considering at least the next item in case the current + # one fails to parse. + + if "" in new_terminals: + continue + + if not isinstance(other.item, (Opt, NegativeLookahead, Repeat0)): + break + + # Do not allow the empty string to propagate. + result.discard("") + + return result + + def visit_Cut(self, item: Cut) -> Set[str]: + return set() + + def visit_Group(self, item: Group) -> Set[str]: + return self.visit(item.rhs) + + def visit_PositiveLookahead(self, item: Lookahead) -> Set[str]: + return self.visit(item.node) + + def visit_NegativeLookahead(self, item: NegativeLookahead) -> Set[str]: + return self.visit(item.node) + + def visit_NamedItem(self, item: NamedItem) -> Set[str]: + return self.visit(item.item) + + def visit_Opt(self, item: Opt) -> Set[str]: + return self.visit(item.node) + + def visit_Gather(self, item: Gather) -> Set[str]: + return self.visit(item.node) + + def visit_Repeat0(self, item: Repeat0) -> Set[str]: + return self.visit(item.node) + + def visit_Repeat1(self, item: Repeat1) -> Set[str]: + return self.visit(item.node) + + def visit_NameLeaf(self, item: NameLeaf) -> Set[str]: + if item.value not in self.rules: + return {item.value} + + if item.value not in self.first_sets: + self.first_sets[item.value] = self.visit(self.rules[item.value]) + return self.first_sets[item.value] + elif item.value in self.in_process: + return set() + + return self.first_sets[item.value] + + def visit_StringLeaf(self, item: StringLeaf) -> Set[str]: + return {item.value} + + def visit_Rhs(self, item: Rhs) -> Set[str]: + result: Set[str] = set() + for alt in item.alts: + result |= self.visit(alt) + return result + + def visit_Rule(self, item: Rule) -> Set[str]: + if item.name in self.in_process: + return set() + elif item.name not in self.first_sets: + self.in_process.add(item.name) + terminals = self.visit(item.rhs) + if item.nullable: + terminals.add("") + self.first_sets[item.name] = terminals + self.in_process.remove(item.name) + return self.first_sets[item.name] + + +def main() -> None: + args = argparser.parse_args() + + try: + grammar, parser, tokenizer = build_parser(args.grammar_file) + except Exception as err: + print("ERROR: Failed to parse grammar file", file=sys.stderr) + sys.exit(1) + + firs_sets = FirstSetCalculator(grammar.rules).calculate() + pprint.pprint(firs_sets) + + +if __name__ == "__main__": + main() diff --git a/Tools/peg_generator/pegen/grammar.py b/Tools/peg_generator/pegen/grammar.py new file mode 100644 index 0000000000000..67039d5a032ab --- /dev/null +++ b/Tools/peg_generator/pegen/grammar.py @@ -0,0 +1,470 @@ +from __future__ import annotations + +from abc import abstractmethod +from typing import ( + AbstractSet, + Any, + Callable, + Dict, + Iterable, + Iterator, + List, + Optional, + Set, + Tuple, + TYPE_CHECKING, + TypeVar, + Union, +) + +from pegen.parser import memoize, Parser + +if TYPE_CHECKING: + from pegen.parser_generator import ParserGenerator + + +class GrammarError(Exception): + pass + + +class GrammarVisitor: + def visit(self, node: Any, *args: Any, **kwargs: Any) -> Any: + """Visit a node.""" + method = "visit_" + node.__class__.__name__ + visitor = getattr(self, method, self.generic_visit) + return visitor(node, *args, **kwargs) + + def generic_visit(self, node: Iterable[Any], *args: Any, **kwargs: Any) -> None: + """Called if no explicit visitor function exists for a node.""" + for value in node: + if isinstance(value, list): + for item in value: + self.visit(item, *args, **kwargs) + else: + self.visit(value, *args, **kwargs) + + +class Grammar: + def __init__(self, rules: Iterable[Rule], metas: Iterable[Tuple[str, Optional[str]]]): + self.rules = {rule.name: rule for rule in rules} + self.metas = dict(metas) + + def __str__(self) -> str: + return "\n".join(str(rule) for name, rule in self.rules.items()) + + def __repr__(self) -> str: + lines = ["Grammar("] + lines.append(" [") + for rule in self.rules.values(): + lines.append(f" {repr(rule)},") + lines.append(" ],") + lines.append(" {repr(list(self.metas.items()))}") + lines.append(")") + return "\n".join(lines) + + def __iter__(self) -> Iterator[Rule]: + yield from self.rules.values() + + +# Global flag whether we want actions in __str__() -- default off. +SIMPLE_STR = True + + +class Rule: + def __init__(self, name: str, type: Optional[str], rhs: Rhs, memo: Optional[object] = None): + self.name = name + self.type = type + self.rhs = rhs + self.memo = bool(memo) + self.visited = False + self.nullable = False + self.left_recursive = False + self.leader = False + + def is_loop(self) -> bool: + return self.name.startswith("_loop") + + def is_gather(self) -> bool: + return self.name.startswith("_gather") + + def __str__(self) -> str: + if SIMPLE_STR or self.type is None: + res = f"{self.name}: {self.rhs}" + else: + res = f"{self.name}[{self.type}]: {self.rhs}" + if len(res) < 88: + return res + lines = [res.split(":")[0] + ":"] + lines += [f" | {alt}" for alt in self.rhs.alts] + return "\n".join(lines) + + def __repr__(self) -> str: + return f"Rule({self.name!r}, {self.type!r}, {self.rhs!r})" + + def __iter__(self) -> Iterator[Rhs]: + yield self.rhs + + def nullable_visit(self, rules: Dict[str, Rule]) -> bool: + if self.visited: + # A left-recursive rule is considered non-nullable. + return False + self.visited = True + self.nullable = self.rhs.nullable_visit(rules) + return self.nullable + + def initial_names(self) -> AbstractSet[str]: + return self.rhs.initial_names() + + def flatten(self) -> Rhs: + # If it's a single parenthesized group, flatten it. + rhs = self.rhs + if ( + not self.is_loop() + and len(rhs.alts) == 1 + and len(rhs.alts[0].items) == 1 + and isinstance(rhs.alts[0].items[0].item, Group) + ): + rhs = rhs.alts[0].items[0].item.rhs + return rhs + + def collect_todo(self, gen: ParserGenerator) -> None: + rhs = self.flatten() + rhs.collect_todo(gen) + + +class Leaf: + def __init__(self, value: str): + self.value = value + + def __str__(self) -> str: + return self.value + + def __iter__(self) -> Iterable[str]: + if False: + yield + + @abstractmethod + def nullable_visit(self, rules: Dict[str, Rule]) -> bool: + raise NotImplementedError + + @abstractmethod + def initial_names(self) -> AbstractSet[str]: + raise NotImplementedError + + +class NameLeaf(Leaf): + """The value is the name.""" + + def __str__(self) -> str: + if self.value == "ENDMARKER": + return "$" + return super().__str__() + + def __repr__(self) -> str: + return f"NameLeaf({self.value!r})" + + def nullable_visit(self, rules: Dict[str, Rule]) -> bool: + if self.value in rules: + return rules[self.value].nullable_visit(rules) + # Token or unknown; never empty. + return False + + def initial_names(self) -> AbstractSet[str]: + return {self.value} + + +class StringLeaf(Leaf): + """The value is a string literal, including quotes.""" + + def __repr__(self) -> str: + return f"StringLeaf({self.value!r})" + + def nullable_visit(self, rules: Dict[str, Rule]) -> bool: + # The string token '' is considered empty. + return not self.value + + def initial_names(self) -> AbstractSet[str]: + return set() + + +class Rhs: + def __init__(self, alts: List[Alt]): + self.alts = alts + self.memo: Optional[Tuple[Optional[str], str]] = None + + def __str__(self) -> str: + return " | ".join(str(alt) for alt in self.alts) + + def __repr__(self) -> str: + return f"Rhs({self.alts!r})" + + def __iter__(self) -> Iterator[List[Alt]]: + yield self.alts + + def nullable_visit(self, rules: Dict[str, Rule]) -> bool: + for alt in self.alts: + if alt.nullable_visit(rules): + return True + return False + + def initial_names(self) -> AbstractSet[str]: + names: Set[str] = set() + for alt in self.alts: + names |= alt.initial_names() + return names + + def collect_todo(self, gen: ParserGenerator) -> None: + for alt in self.alts: + alt.collect_todo(gen) + + +class Alt: + def __init__(self, items: List[NamedItem], *, icut: int = -1, action: Optional[str] = None): + self.items = items + self.icut = icut + self.action = action + + def __str__(self) -> str: + core = " ".join(str(item) for item in self.items) + if not SIMPLE_STR and self.action: + return f"{core} {{ {self.action} }}" + else: + return core + + def __repr__(self) -> str: + args = [repr(self.items)] + if self.icut >= 0: + args.append(f"icut={self.icut}") + if self.action: + args.append(f"action={self.action!r}") + return f"Alt({', '.join(args)})" + + def __iter__(self) -> Iterator[List[NamedItem]]: + yield self.items + + def nullable_visit(self, rules: Dict[str, Rule]) -> bool: + for item in self.items: + if not item.nullable_visit(rules): + return False + return True + + def initial_names(self) -> AbstractSet[str]: + names: Set[str] = set() + for item in self.items: + names |= item.initial_names() + if not item.nullable: + break + return names + + def collect_todo(self, gen: ParserGenerator) -> None: + for item in self.items: + item.collect_todo(gen) + + +class NamedItem: + def __init__(self, name: Optional[str], item: Item): + self.name = name + self.item = item + self.nullable = False + + def __str__(self) -> str: + if not SIMPLE_STR and self.name: + return f"{self.name}={self.item}" + else: + return str(self.item) + + def __repr__(self) -> str: + return f"NamedItem({self.name!r}, {self.item!r})" + + def __iter__(self) -> Iterator[Item]: + yield self.item + + def nullable_visit(self, rules: Dict[str, Rule]) -> bool: + self.nullable = self.item.nullable_visit(rules) + return self.nullable + + def initial_names(self) -> AbstractSet[str]: + return self.item.initial_names() + + def collect_todo(self, gen: ParserGenerator) -> None: + gen.callmakervisitor.visit(self.item) + + +class Lookahead: + def __init__(self, node: Plain, sign: str): + self.node = node + self.sign = sign + + def __str__(self) -> str: + return f"{self.sign}{self.node}" + + def __iter__(self) -> Iterator[Plain]: + yield self.node + + def nullable_visit(self, rules: Dict[str, Rule]) -> bool: + return True + + def initial_names(self) -> AbstractSet[str]: + return set() + + +class PositiveLookahead(Lookahead): + def __init__(self, node: Plain): + super().__init__(node, "&") + + def __repr__(self) -> str: + return f"PositiveLookahead({self.node!r})" + + +class NegativeLookahead(Lookahead): + def __init__(self, node: Plain): + super().__init__(node, "!") + + def __repr__(self) -> str: + return f"NegativeLookahead({self.node!r})" + + +class Opt: + def __init__(self, node: Item): + self.node = node + + def __str__(self) -> str: + s = str(self.node) + # TODO: Decide whether to use [X] or X? based on type of X + if " " in s: + return f"[{s}]" + else: + return f"{s}?" + + def __repr__(self) -> str: + return f"Opt({self.node!r})" + + def __iter__(self) -> Iterator[Item]: + yield self.node + + def nullable_visit(self, rules: Dict[str, Rule]) -> bool: + return True + + def initial_names(self) -> AbstractSet[str]: + return self.node.initial_names() + + +class Repeat: + """Shared base class for x* and x+.""" + + def __init__(self, node: Plain): + self.node = node + self.memo: Optional[Tuple[Optional[str], str]] = None + + @abstractmethod + def nullable_visit(self, rules: Dict[str, Rule]) -> bool: + raise NotImplementedError + + def __iter__(self) -> Iterator[Plain]: + yield self.node + + def initial_names(self) -> AbstractSet[str]: + return self.node.initial_names() + + +class Repeat0(Repeat): + def __str__(self) -> str: + s = str(self.node) + # TODO: Decide whether to use (X)* or X* based on type of X + if " " in s: + return f"({s})*" + else: + return f"{s}*" + + def __repr__(self) -> str: + return f"Repeat0({self.node!r})" + + def nullable_visit(self, rules: Dict[str, Rule]) -> bool: + return True + + +class Repeat1(Repeat): + def __str__(self) -> str: + s = str(self.node) + # TODO: Decide whether to use (X)+ or X+ based on type of X + if " " in s: + return f"({s})+" + else: + return f"{s}+" + + def __repr__(self) -> str: + return f"Repeat1({self.node!r})" + + def nullable_visit(self, rules: Dict[str, Rule]) -> bool: + return False + + +class Gather(Repeat): + def __init__(self, separator: Plain, node: Plain): + self.separator = separator + self.node = node + + def __str__(self) -> str: + return f"{self.separator!s}.{self.node!s}+" + + def __repr__(self) -> str: + return f"Gather({self.separator!r}, {self.node!r})" + + def nullable_visit(self, rules: Dict[str, Rule]) -> bool: + return False + + +class Group: + def __init__(self, rhs: Rhs): + self.rhs = rhs + + def __str__(self) -> str: + return f"({self.rhs})" + + def __repr__(self) -> str: + return f"Group({self.rhs!r})" + + def __iter__(self) -> Iterator[Rhs]: + yield self.rhs + + def nullable_visit(self, rules: Dict[str, Rule]) -> bool: + return self.rhs.nullable_visit(rules) + + def initial_names(self) -> AbstractSet[str]: + return self.rhs.initial_names() + + +class Cut: + def __init__(self) -> None: + pass + + def __repr__(self) -> str: + return f"Cut()" + + def __str__(self) -> str: + return f"~" + + def __iter__(self) -> Iterator[Tuple[str, str]]: + if False: + yield + + def __eq__(self, other: object) -> bool: + if not isinstance(other, Cut): + return NotImplemented + return True + + def nullable_visit(self, rules: Dict[str, Rule]) -> bool: + return True + + def initial_names(self) -> AbstractSet[str]: + return set() + + +Plain = Union[Leaf, Group] +Item = Union[Plain, Opt, Repeat, Lookahead, Rhs, Cut] +RuleName = Tuple[str, str] +MetaTuple = Tuple[str, Optional[str]] +MetaList = List[MetaTuple] +RuleList = List[Rule] +NamedItemList = List[NamedItem] +LookaheadOrCut = Union[Lookahead, Cut] diff --git a/Tools/peg_generator/pegen/grammar_parser.py b/Tools/peg_generator/pegen/grammar_parser.py new file mode 100644 index 0000000000000..0e206ee9cd5e4 --- /dev/null +++ b/Tools/peg_generator/pegen/grammar_parser.py @@ -0,0 +1,677 @@ +#!/usr/bin/env python3.8 +# @generated by pegen from pegen/metagrammar.gram + +import ast +import sys +import tokenize + +from typing import Any, Optional + +from pegen.parser import memoize, memoize_left_rec, logger, Parser +from ast import literal_eval + +from pegen.grammar import ( + Alt, + Cut, + Gather, + Group, + Item, + Lookahead, + LookaheadOrCut, + MetaTuple, + MetaList, + NameLeaf, + NamedItem, + NamedItemList, + NegativeLookahead, + Opt, + Plain, + PositiveLookahead, + Repeat0, + Repeat1, + Rhs, + Rule, + RuleList, + RuleName, + Grammar, + StringLeaf, +) + +class GeneratedParser(Parser): + + @memoize + def start(self) -> Optional[Grammar]: + # start: grammar $ + mark = self.mark() + cut = False + if ( + (grammar := self.grammar()) + and + (endmarker := self.expect('ENDMARKER')) + ): + return grammar + self.reset(mark) + if cut: return None + return None + + @memoize + def grammar(self) -> Optional[Grammar]: + # grammar: metas rules | rules + mark = self.mark() + cut = False + if ( + (metas := self.metas()) + and + (rules := self.rules()) + ): + return Grammar ( rules , metas ) + self.reset(mark) + if cut: return None + cut = False + if ( + (rules := self.rules()) + ): + return Grammar ( rules , [ ] ) + self.reset(mark) + if cut: return None + return None + + @memoize + def metas(self) -> Optional[MetaList]: + # metas: meta metas | meta + mark = self.mark() + cut = False + if ( + (meta := self.meta()) + and + (metas := self.metas()) + ): + return [ meta ] + metas + self.reset(mark) + if cut: return None + cut = False + if ( + (meta := self.meta()) + ): + return [ meta ] + self.reset(mark) + if cut: return None + return None + + @memoize + def meta(self) -> Optional[MetaTuple]: + # meta: "@" NAME NEWLINE | "@" NAME NAME NEWLINE | "@" NAME STRING NEWLINE + mark = self.mark() + cut = False + if ( + (literal := self.expect("@")) + and + (name := self.name()) + and + (newline := self.expect('NEWLINE')) + ): + return ( name . string , None ) + self.reset(mark) + if cut: return None + cut = False + if ( + (literal := self.expect("@")) + and + (a := self.name()) + and + (b := self.name()) + and + (newline := self.expect('NEWLINE')) + ): + return ( a . string , b . string ) + self.reset(mark) + if cut: return None + cut = False + if ( + (literal := self.expect("@")) + and + (name := self.name()) + and + (string := self.string()) + and + (newline := self.expect('NEWLINE')) + ): + return ( name . string , literal_eval ( string . string ) ) + self.reset(mark) + if cut: return None + return None + + @memoize + def rules(self) -> Optional[RuleList]: + # rules: rule rules | rule + mark = self.mark() + cut = False + if ( + (rule := self.rule()) + and + (rules := self.rules()) + ): + return [ rule ] + rules + self.reset(mark) + if cut: return None + cut = False + if ( + (rule := self.rule()) + ): + return [ rule ] + self.reset(mark) + if cut: return None + return None + + @memoize + def rule(self) -> Optional[Rule]: + # rule: rulename memoflag? ":" alts NEWLINE INDENT more_alts DEDENT | rulename memoflag? ":" NEWLINE INDENT more_alts DEDENT | rulename memoflag? ":" alts NEWLINE + mark = self.mark() + cut = False + if ( + (rulename := self.rulename()) + and + (opt := self.memoflag(),) + and + (literal := self.expect(":")) + and + (alts := self.alts()) + and + (newline := self.expect('NEWLINE')) + and + (indent := self.expect('INDENT')) + and + (more_alts := self.more_alts()) + and + (dedent := self.expect('DEDENT')) + ): + return Rule ( rulename [ 0 ] , rulename [ 1 ] , Rhs ( alts . alts + more_alts . alts ) , memo = opt ) + self.reset(mark) + if cut: return None + cut = False + if ( + (rulename := self.rulename()) + and + (opt := self.memoflag(),) + and + (literal := self.expect(":")) + and + (newline := self.expect('NEWLINE')) + and + (indent := self.expect('INDENT')) + and + (more_alts := self.more_alts()) + and + (dedent := self.expect('DEDENT')) + ): + return Rule ( rulename [ 0 ] , rulename [ 1 ] , more_alts , memo = opt ) + self.reset(mark) + if cut: return None + cut = False + if ( + (rulename := self.rulename()) + and + (opt := self.memoflag(),) + and + (literal := self.expect(":")) + and + (alts := self.alts()) + and + (newline := self.expect('NEWLINE')) + ): + return Rule ( rulename [ 0 ] , rulename [ 1 ] , alts , memo = opt ) + self.reset(mark) + if cut: return None + return None + + @memoize + def rulename(self) -> Optional[RuleName]: + # rulename: NAME '[' NAME '*' ']' | NAME '[' NAME ']' | NAME + mark = self.mark() + cut = False + if ( + (name := self.name()) + and + (literal := self.expect('[')) + and + (type := self.name()) + and + (literal_1 := self.expect('*')) + and + (literal_2 := self.expect(']')) + ): + return ( name . string , type . string + "*" ) + self.reset(mark) + if cut: return None + cut = False + if ( + (name := self.name()) + and + (literal := self.expect('[')) + and + (type := self.name()) + and + (literal_1 := self.expect(']')) + ): + return ( name . string , type . string ) + self.reset(mark) + if cut: return None + cut = False + if ( + (name := self.name()) + ): + return ( name . string , None ) + self.reset(mark) + if cut: return None + return None + + @memoize + def memoflag(self) -> Optional[str]: + # memoflag: '(' 'memo' ')' + mark = self.mark() + cut = False + if ( + (literal := self.expect('(')) + and + (literal_1 := self.expect('memo')) + and + (literal_2 := self.expect(')')) + ): + return "memo" + self.reset(mark) + if cut: return None + return None + + @memoize + def alts(self) -> Optional[Rhs]: + # alts: alt "|" alts | alt + mark = self.mark() + cut = False + if ( + (alt := self.alt()) + and + (literal := self.expect("|")) + and + (alts := self.alts()) + ): + return Rhs ( [ alt ] + alts . alts ) + self.reset(mark) + if cut: return None + cut = False + if ( + (alt := self.alt()) + ): + return Rhs ( [ alt ] ) + self.reset(mark) + if cut: return None + return None + + @memoize + def more_alts(self) -> Optional[Rhs]: + # more_alts: "|" alts NEWLINE more_alts | "|" alts NEWLINE + mark = self.mark() + cut = False + if ( + (literal := self.expect("|")) + and + (alts := self.alts()) + and + (newline := self.expect('NEWLINE')) + and + (more_alts := self.more_alts()) + ): + return Rhs ( alts . alts + more_alts . alts ) + self.reset(mark) + if cut: return None + cut = False + if ( + (literal := self.expect("|")) + and + (alts := self.alts()) + and + (newline := self.expect('NEWLINE')) + ): + return Rhs ( alts . alts ) + self.reset(mark) + if cut: return None + return None + + @memoize + def alt(self) -> Optional[Alt]: + # alt: items '$' action | items '$' | items action | items + mark = self.mark() + cut = False + if ( + (items := self.items()) + and + (literal := self.expect('$')) + and + (action := self.action()) + ): + return Alt ( items + [ NamedItem ( None , NameLeaf ( 'ENDMARKER' ) ) ] , action = action ) + self.reset(mark) + if cut: return None + cut = False + if ( + (items := self.items()) + and + (literal := self.expect('$')) + ): + return Alt ( items + [ NamedItem ( None , NameLeaf ( 'ENDMARKER' ) ) ] , action = None ) + self.reset(mark) + if cut: return None + cut = False + if ( + (items := self.items()) + and + (action := self.action()) + ): + return Alt ( items , action = action ) + self.reset(mark) + if cut: return None + cut = False + if ( + (items := self.items()) + ): + return Alt ( items , action = None ) + self.reset(mark) + if cut: return None + return None + + @memoize + def items(self) -> Optional[NamedItemList]: + # items: named_item items | named_item + mark = self.mark() + cut = False + if ( + (named_item := self.named_item()) + and + (items := self.items()) + ): + return [ named_item ] + items + self.reset(mark) + if cut: return None + cut = False + if ( + (named_item := self.named_item()) + ): + return [ named_item ] + self.reset(mark) + if cut: return None + return None + + @memoize + def named_item(self) -> Optional[NamedItem]: + # named_item: NAME '=' ~ item | item | lookahead + mark = self.mark() + cut = False + if ( + (name := self.name()) + and + (literal := self.expect('=')) + and + (cut := True) + and + (item := self.item()) + ): + return NamedItem ( name . string , item ) + self.reset(mark) + if cut: return None + cut = False + if ( + (item := self.item()) + ): + return NamedItem ( None , item ) + self.reset(mark) + if cut: return None + cut = False + if ( + (it := self.lookahead()) + ): + return NamedItem ( None , it ) + self.reset(mark) + if cut: return None + return None + + @memoize + def lookahead(self) -> Optional[LookaheadOrCut]: + # lookahead: '&' ~ atom | '!' ~ atom | '~' + mark = self.mark() + cut = False + if ( + (literal := self.expect('&')) + and + (cut := True) + and + (atom := self.atom()) + ): + return PositiveLookahead ( atom ) + self.reset(mark) + if cut: return None + cut = False + if ( + (literal := self.expect('!')) + and + (cut := True) + and + (atom := self.atom()) + ): + return NegativeLookahead ( atom ) + self.reset(mark) + if cut: return None + cut = False + if ( + (literal := self.expect('~')) + ): + return Cut ( ) + self.reset(mark) + if cut: return None + return None + + @memoize + def item(self) -> Optional[Item]: + # item: '[' ~ alts ']' | atom '?' | atom '*' | atom '+' | atom '.' atom '+' | atom + mark = self.mark() + cut = False + if ( + (literal := self.expect('[')) + and + (cut := True) + and + (alts := self.alts()) + and + (literal_1 := self.expect(']')) + ): + return Opt ( alts ) + self.reset(mark) + if cut: return None + cut = False + if ( + (atom := self.atom()) + and + (literal := self.expect('?')) + ): + return Opt ( atom ) + self.reset(mark) + if cut: return None + cut = False + if ( + (atom := self.atom()) + and + (literal := self.expect('*')) + ): + return Repeat0 ( atom ) + self.reset(mark) + if cut: return None + cut = False + if ( + (atom := self.atom()) + and + (literal := self.expect('+')) + ): + return Repeat1 ( atom ) + self.reset(mark) + if cut: return None + cut = False + if ( + (sep := self.atom()) + and + (literal := self.expect('.')) + and + (node := self.atom()) + and + (literal_1 := self.expect('+')) + ): + return Gather ( sep , node ) + self.reset(mark) + if cut: return None + cut = False + if ( + (atom := self.atom()) + ): + return atom + self.reset(mark) + if cut: return None + return None + + @memoize + def atom(self) -> Optional[Plain]: + # atom: '(' ~ alts ')' | NAME | STRING + mark = self.mark() + cut = False + if ( + (literal := self.expect('(')) + and + (cut := True) + and + (alts := self.alts()) + and + (literal_1 := self.expect(')')) + ): + return Group ( alts ) + self.reset(mark) + if cut: return None + cut = False + if ( + (name := self.name()) + ): + return NameLeaf ( name . string ) + self.reset(mark) + if cut: return None + cut = False + if ( + (string := self.string()) + ): + return StringLeaf ( string . string ) + self.reset(mark) + if cut: return None + return None + + @memoize + def action(self) -> Optional[str]: + # action: "{" ~ target_atoms "}" + mark = self.mark() + cut = False + if ( + (literal := self.expect("{")) + and + (cut := True) + and + (target_atoms := self.target_atoms()) + and + (literal_1 := self.expect("}")) + ): + return target_atoms + self.reset(mark) + if cut: return None + return None + + @memoize + def target_atoms(self) -> Optional[str]: + # target_atoms: target_atom target_atoms | target_atom + mark = self.mark() + cut = False + if ( + (target_atom := self.target_atom()) + and + (target_atoms := self.target_atoms()) + ): + return target_atom + " " + target_atoms + self.reset(mark) + if cut: return None + cut = False + if ( + (target_atom := self.target_atom()) + ): + return target_atom + self.reset(mark) + if cut: return None + return None + + @memoize + def target_atom(self) -> Optional[str]: + # target_atom: "{" ~ target_atoms "}" | NAME | NUMBER | STRING | "?" | ":" | !"}" OP + mark = self.mark() + cut = False + if ( + (literal := self.expect("{")) + and + (cut := True) + and + (target_atoms := self.target_atoms()) + and + (literal_1 := self.expect("}")) + ): + return "{" + target_atoms + "}" + self.reset(mark) + if cut: return None + cut = False + if ( + (name := self.name()) + ): + return name . string + self.reset(mark) + if cut: return None + cut = False + if ( + (number := self.number()) + ): + return number . string + self.reset(mark) + if cut: return None + cut = False + if ( + (string := self.string()) + ): + return string . string + self.reset(mark) + if cut: return None + cut = False + if ( + (literal := self.expect("?")) + ): + return "?" + self.reset(mark) + if cut: return None + cut = False + if ( + (literal := self.expect(":")) + ): + return ":" + self.reset(mark) + if cut: return None + cut = False + if ( + self.negative_lookahead(self.expect, "}") + and + (op := self.op()) + ): + return op . string + self.reset(mark) + if cut: return None + return None + + +if __name__ == '__main__': + from pegen.parser import simple_parser_main + simple_parser_main(GeneratedParser) diff --git a/Tools/peg_generator/pegen/grammar_visualizer.py b/Tools/peg_generator/pegen/grammar_visualizer.py new file mode 100644 index 0000000000000..b1d51d2cdb250 --- /dev/null +++ b/Tools/peg_generator/pegen/grammar_visualizer.py @@ -0,0 +1,65 @@ +import argparse +import sys + +from typing import Any, Iterator, Iterable, Callable + +from pegen.build import build_parser +from pegen.grammar import Grammar, Rule + +argparser = argparse.ArgumentParser( + prog="pegen", description="Pretty print the AST for a given PEG grammar" +) +argparser.add_argument("filename", help="Grammar description") + + +class ASTGrammarPrinter: + def children(self, node: Rule) -> Iterator[Any]: + for value in node: + if isinstance(value, list): + yield from value + else: + yield value + + def name(self, node: Rule) -> str: + if not list(self.children(node)): + return repr(node) + return node.__class__.__name__ + + def print_grammar_ast(self, grammar: Grammar, printer: Callable[..., None] = print) -> None: + for rule in grammar.rules.values(): + printer(self.print_nodes_recursively(rule)) + + def print_nodes_recursively(self, node: Rule, prefix: str = "", istail: bool = True) -> str: + + children = list(self.children(node)) + value = self.name(node) + + line = prefix + ("└──" if istail else "├──") + value + "\n" + sufix = " " if istail else "│ " + + if not children: + return line + + *children, last = children + for child in children: + line += self.print_nodes_recursively(child, prefix + sufix, False) + line += self.print_nodes_recursively(last, prefix + sufix, True) + + return line + + +def main() -> None: + args = argparser.parse_args() + + try: + grammar, parser, tokenizer = build_parser(args.filename) + except Exception as err: + print("ERROR: Failed to parse grammar file", file=sys.stderr) + sys.exit(1) + + visitor = ASTGrammarPrinter() + visitor.print_grammar_ast(grammar) + + +if __name__ == "__main__": + main() diff --git a/Tools/peg_generator/pegen/metagrammar.gram b/Tools/peg_generator/pegen/metagrammar.gram new file mode 100644 index 0000000000000..f0c5ac3ab390f --- /dev/null +++ b/Tools/peg_generator/pegen/metagrammar.gram @@ -0,0 +1,123 @@ +@subheader """\ +from ast import literal_eval + +from pegen.grammar import ( + Alt, + Cut, + Gather, + Group, + Item, + Lookahead, + LookaheadOrCut, + MetaTuple, + MetaList, + NameLeaf, + NamedItem, + NamedItemList, + NegativeLookahead, + Opt, + Plain, + PositiveLookahead, + Repeat0, + Repeat1, + Rhs, + Rule, + RuleList, + RuleName, + Grammar, + StringLeaf, +) +""" + +start[Grammar]: grammar ENDMARKER { grammar } + +grammar[Grammar]: + | metas rules { Grammar(rules, metas) } + | rules { Grammar(rules, []) } + +metas[MetaList]: + | meta metas { [meta] + metas } + | meta { [meta] } + +meta[MetaTuple]: + | "@" NAME NEWLINE { (name.string, None) } + | "@" a=NAME b=NAME NEWLINE { (a.string, b.string) } + | "@" NAME STRING NEWLINE { (name.string, literal_eval(string.string)) } + +rules[RuleList]: + | rule rules { [rule] + rules } + | rule { [rule] } + +rule[Rule]: + | rulename memoflag? ":" alts NEWLINE INDENT more_alts DEDENT { + Rule(rulename[0], rulename[1], Rhs(alts.alts + more_alts.alts), memo=opt) } + | rulename memoflag? ":" NEWLINE INDENT more_alts DEDENT { + Rule(rulename[0], rulename[1], more_alts, memo=opt) } + | rulename memoflag? ":" alts NEWLINE { Rule(rulename[0], rulename[1], alts, memo=opt) } + +rulename[RuleName]: + | NAME '[' type=NAME '*' ']' { (name.string, type.string+"*") } + | NAME '[' type=NAME ']' { (name.string, type.string) } + | NAME { (name.string, None) } + +# In the future this may return something more complicated +memoflag[str]: + | '(' 'memo' ')' { "memo" } + +alts[Rhs]: + | alt "|" alts { Rhs([alt] + alts.alts)} + | alt { Rhs([alt]) } + +more_alts[Rhs]: + | "|" alts NEWLINE more_alts { Rhs(alts.alts + more_alts.alts) } + | "|" alts NEWLINE { Rhs(alts.alts) } + +alt[Alt]: + | items '$' action { Alt(items + [NamedItem(None, NameLeaf('ENDMARKER'))], action=action) } + | items '$' { Alt(items + [NamedItem(None, NameLeaf('ENDMARKER'))], action=None) } + | items action { Alt(items, action=action) } + | items { Alt(items, action=None) } + +items[NamedItemList]: + | named_item items { [named_item] + items } + | named_item { [named_item] } + +named_item[NamedItem]: + | NAME '=' ~ item {NamedItem(name.string, item)} + | item {NamedItem(None, item)} + | it=lookahead {NamedItem(None, it)} + +lookahead[LookaheadOrCut]: + | '&' ~ atom {PositiveLookahead(atom)} + | '!' ~ atom {NegativeLookahead(atom)} + | '~' {Cut()} + +item[Item]: + | '[' ~ alts ']' {Opt(alts)} + | atom '?' {Opt(atom)} + | atom '*' {Repeat0(atom)} + | atom '+' {Repeat1(atom)} + | sep=atom '.' node=atom '+' {Gather(sep, node)} + | atom {atom} + +atom[Plain]: + | '(' ~ alts ')' {Group(alts)} + | NAME {NameLeaf(name.string) } + | STRING {StringLeaf(string.string)} + +# Mini-grammar for the actions + +action[str]: "{" ~ target_atoms "}" { target_atoms } + +target_atoms[str]: + | target_atom target_atoms { target_atom + " " + target_atoms } + | target_atom { target_atom } + +target_atom[str]: + | "{" ~ target_atoms "}" { "{" + target_atoms + "}" } + | NAME { name.string } + | NUMBER { number.string } + | STRING { string.string } + | "?" { "?" } + | ":" { ":" } + | !"}" OP { op.string } diff --git a/Tools/peg_generator/pegen/parser.py b/Tools/peg_generator/pegen/parser.py new file mode 100644 index 0000000000000..16d954dce89f7 --- /dev/null +++ b/Tools/peg_generator/pegen/parser.py @@ -0,0 +1,310 @@ +import argparse +import sys +import time +import token +import tokenize +import traceback + +from abc import abstractmethod +from typing import Any, Callable, cast, Dict, Optional, Tuple, Type, TypeVar + +from pegen.tokenizer import exact_token_types +from pegen.tokenizer import Mark +from pegen.tokenizer import Tokenizer + +T = TypeVar("T") +P = TypeVar("P", bound="Parser") +F = TypeVar("F", bound=Callable[..., Any]) + + +def logger(method: F) -> F: + """For non-memoized functions that we want to be logged. + + (In practice this is only non-leader left-recursive functions.) + """ + method_name = method.__name__ + + def logger_wrapper(self: P, *args: object) -> T: + if not self._verbose: + return method(self, *args) + argsr = ",".join(repr(arg) for arg in args) + fill = " " * self._level + print(f"{fill}{method_name}({argsr}) .... (looking at {self.showpeek()})") + self._level += 1 + tree = method(self, *args) + self._level -= 1 + print(f"{fill}... {method_name}({argsr}) --> {tree!s:.200}") + return tree + + logger_wrapper.__wrapped__ = method # type: ignore + return cast(F, logger_wrapper) + + +def memoize(method: F) -> F: + """Memoize a symbol method.""" + method_name = method.__name__ + + def memoize_wrapper(self: P, *args: object) -> T: + mark = self.mark() + key = mark, method_name, args + # Fast path: cache hit, and not verbose. + if key in self._cache and not self._verbose: + tree, endmark = self._cache[key] + self.reset(endmark) + return tree + # Slow path: no cache hit, or verbose. + verbose = self._verbose + argsr = ",".join(repr(arg) for arg in args) + fill = " " * self._level + if key not in self._cache: + if verbose: + print(f"{fill}{method_name}({argsr}) ... (looking at {self.showpeek()})") + self._level += 1 + tree = method(self, *args) + self._level -= 1 + if verbose: + print(f"{fill}... {method_name}({argsr}) -> {tree!s:.200}") + endmark = self.mark() + self._cache[key] = tree, endmark + else: + tree, endmark = self._cache[key] + if verbose: + print(f"{fill}{method_name}({argsr}) -> {tree!s:.200}") + self.reset(endmark) + return tree + + memoize_wrapper.__wrapped__ = method # type: ignore + return cast(F, memoize_wrapper) + + +def memoize_left_rec(method: Callable[[P], Optional[T]]) -> Callable[[P], Optional[T]]: + """Memoize a left-recursive symbol method.""" + method_name = method.__name__ + + def memoize_left_rec_wrapper(self: P) -> Optional[T]: + mark = self.mark() + key = mark, method_name, () + # Fast path: cache hit, and not verbose. + if key in self._cache and not self._verbose: + tree, endmark = self._cache[key] + self.reset(endmark) + return tree + # Slow path: no cache hit, or verbose. + verbose = self._verbose + fill = " " * self._level + if key not in self._cache: + if verbose: + print(f"{fill}{method_name} ... (looking at {self.showpeek()})") + self._level += 1 + + # For left-recursive rules we manipulate the cache and + # loop until the rule shows no progress, then pick the + # previous result. For an explanation why this works, see + # https://github.com/PhilippeSigaud/Pegged/wiki/Left-Recursion + # (But we use the memoization cache instead of a static + # variable; perhaps this is similar to a paper by Warth et al. + # (http://web.cs.ucla.edu/~todd/research/pub.php?id=pepm08). + + # Prime the cache with a failure. + self._cache[key] = None, mark + lastresult, lastmark = None, mark + depth = 0 + if verbose: + print(f"{fill}Recursive {method_name} at {mark} depth {depth}") + + while True: + self.reset(mark) + result = method(self) + endmark = self.mark() + depth += 1 + if verbose: + print( + f"{fill}Recursive {method_name} at {mark} depth {depth}: {result!s:.200} to {endmark}" + ) + if not result: + if verbose: + print(f"{fill}Fail with {lastresult!s:.200} to {lastmark}") + break + if endmark <= lastmark: + if verbose: + print(f"{fill}Bailing with {lastresult!s:.200} to {lastmark}") + break + self._cache[key] = lastresult, lastmark = result, endmark + + self.reset(lastmark) + tree = lastresult + + self._level -= 1 + if verbose: + print(f"{fill}{method_name}() -> {tree!s:.200} [cached]") + if tree: + endmark = self.mark() + else: + endmark = mark + self.reset(endmark) + self._cache[key] = tree, endmark + else: + tree, endmark = self._cache[key] + if verbose: + print(f"{fill}{method_name}() -> {tree!s:.200} [fresh]") + if tree: + self.reset(endmark) + return tree + + memoize_left_rec_wrapper.__wrapped__ = method # type: ignore + return memoize_left_rec_wrapper + + +class Parser: + """Parsing base class.""" + + def __init__(self, tokenizer: Tokenizer, *, verbose: bool = False): + self._tokenizer = tokenizer + self._verbose = verbose + self._level = 0 + self._cache: Dict[Tuple[Mark, str, Tuple[Any, ...]], Tuple[Any, Mark]] = {} + # Pass through common tokenizer methods. + # TODO: Rename to _mark and _reset. + self.mark = self._tokenizer.mark + self.reset = self._tokenizer.reset + + @abstractmethod + def start(self) -> Any: + pass + + def showpeek(self) -> str: + tok = self._tokenizer.peek() + return f"{tok.start[0]}.{tok.start[1]}: {token.tok_name[tok.type]}:{tok.string!r}" + + @memoize + def name(self) -> Optional[tokenize.TokenInfo]: + tok = self._tokenizer.peek() + if tok.type == token.NAME: + return self._tokenizer.getnext() + return None + + @memoize + def number(self) -> Optional[tokenize.TokenInfo]: + tok = self._tokenizer.peek() + if tok.type == token.NUMBER: + return self._tokenizer.getnext() + return None + + @memoize + def string(self) -> Optional[tokenize.TokenInfo]: + tok = self._tokenizer.peek() + if tok.type == token.STRING: + return self._tokenizer.getnext() + return None + + @memoize + def op(self) -> Optional[tokenize.TokenInfo]: + tok = self._tokenizer.peek() + if tok.type == token.OP: + return self._tokenizer.getnext() + return None + + @memoize + def expect(self, type: str) -> Optional[tokenize.TokenInfo]: + tok = self._tokenizer.peek() + if tok.string == type: + return self._tokenizer.getnext() + if type in exact_token_types: + if tok.type == exact_token_types[type]: + return self._tokenizer.getnext() + if type in token.__dict__: + if tok.type == token.__dict__[type]: + return self._tokenizer.getnext() + if tok.type == token.OP and tok.string == type: + return self._tokenizer.getnext() + return None + + def positive_lookahead(self, func: Callable[..., T], *args: object) -> T: + mark = self.mark() + ok = func(*args) + self.reset(mark) + return ok + + def negative_lookahead(self, func: Callable[..., object], *args: object) -> bool: + mark = self.mark() + ok = func(*args) + self.reset(mark) + return not ok + + def make_syntax_error(self, filename: str = "<unknown>") -> SyntaxError: + tok = self._tokenizer.diagnose() + return SyntaxError( + "pegen parse failure", (filename, tok.start[0], 1 + tok.start[1], tok.line) + ) + + +def simple_parser_main(parser_class: Type[Parser]) -> None: + argparser = argparse.ArgumentParser() + argparser.add_argument( + "-v", + "--verbose", + action="count", + default=0, + help="Print timing stats; repeat for more debug output", + ) + argparser.add_argument( + "-q", "--quiet", action="store_true", help="Don't print the parsed program" + ) + argparser.add_argument("filename", help="Input file ('-' to use stdin)") + + args = argparser.parse_args() + verbose = args.verbose + verbose_tokenizer = verbose >= 3 + verbose_parser = verbose == 2 or verbose >= 4 + + t0 = time.time() + + filename = args.filename + if filename == "" or filename == "-": + filename = "<stdin>" + file = sys.stdin + else: + file = open(args.filename) + try: + tokengen = tokenize.generate_tokens(file.readline) + tokenizer = Tokenizer(tokengen, verbose=verbose_tokenizer) + parser = parser_class(tokenizer, verbose=verbose_parser) + tree = parser.start() + try: + if file.isatty(): + endpos = 0 + else: + endpos = file.tell() + except IOError: + endpos = 0 + finally: + if file is not sys.stdin: + file.close() + + t1 = time.time() + + if not tree: + err = parser.make_syntax_error(filename) + traceback.print_exception(err.__class__, err, None) + sys.exit(1) + + if not args.quiet: + print(tree) + + if verbose: + dt = t1 - t0 + diag = tokenizer.diagnose() + nlines = diag.end[0] + if diag.type == token.ENDMARKER: + nlines -= 1 + print(f"Total time: {dt:.3f} sec; {nlines} lines", end="") + if endpos: + print(f" ({endpos} bytes)", end="") + if dt: + print(f"; {nlines / dt:.0f} lines/sec") + else: + print() + print("Caches sizes:") + print(f" token array : {len(tokenizer._tokens):10}") + print(f" cache : {len(parser._cache):10}") + ## print_memstats() diff --git a/Tools/peg_generator/pegen/parser_generator.py b/Tools/peg_generator/pegen/parser_generator.py new file mode 100644 index 0000000000000..7851a7c90f4d5 --- /dev/null +++ b/Tools/peg_generator/pegen/parser_generator.py @@ -0,0 +1,188 @@ +import contextlib +import token +from abc import abstractmethod + +from typing import AbstractSet, Dict, IO, Iterator, List, Optional, Set, Text, Tuple + +from pegen import sccutils +from pegen.grammar import ( + Grammar, + Rule, + Rhs, + Alt, + NamedItem, + Plain, + NameLeaf, + StringLeaf, + Gather, +) +from pegen.grammar import GrammarError, GrammarVisitor + + +class RuleCheckingVisitor(GrammarVisitor): + def __init__(self, rules: Dict[str, Rule]): + self.rules = rules + + def visit_NameLeaf(self, node: NameLeaf) -> None: + if node.value not in self.rules and node.value not in token.tok_name.values(): + # TODO: Add line/col info to (leaf) nodes + raise GrammarError(f"Dangling reference to rule {node.value!r}") + + +class ParserGenerator: + + callmakervisitor: GrammarVisitor + + def __init__(self, grammar: Grammar, file: Optional[IO[Text]]): + self.grammar = grammar + self.rules = grammar.rules + if "trailer" not in grammar.metas and "start" not in self.rules: + raise GrammarError("Grammar without a trailer must have a 'start' rule") + checker = RuleCheckingVisitor(self.rules) + for rule in self.rules.values(): + checker.visit(rule) + self.file = file + self.level = 0 + compute_nullables(self.rules) + self.first_graph, self.first_sccs = compute_left_recursives(self.rules) + self.todo = self.rules.copy() # Rules to generate + self.counter = 0 # For name_rule()/name_loop() + self.keyword_counter = 499 # For keyword_type() + + @abstractmethod + def generate(self, filename: str) -> None: + raise NotImplementedError + + @contextlib.contextmanager + def indent(self) -> Iterator[None]: + self.level += 1 + try: + yield + finally: + self.level -= 1 + + def print(self, *args: object) -> None: + if not args: + print(file=self.file) + else: + print(" " * self.level, end="", file=self.file) + print(*args, file=self.file) + + def printblock(self, lines: str) -> None: + for line in lines.splitlines(): + self.print(line) + + def collect_todo(self) -> None: + done: Set[str] = set() + while True: + alltodo = list(self.todo) + todo = [i for i in alltodo if i not in done] + if not todo: + break + for rulename in todo: + self.todo[rulename].collect_todo(self) + done = set(alltodo) + + def keyword_type(self) -> int: + self.keyword_counter += 1 + return self.keyword_counter + + def name_node(self, rhs: Rhs) -> str: + self.counter += 1 + name = f"_tmp_{self.counter}" # TODO: Pick a nicer name. + self.todo[name] = Rule(name, None, rhs) + return name + + def name_loop(self, node: Plain, is_repeat1: bool) -> str: + self.counter += 1 + if is_repeat1: + prefix = "_loop1_" + else: + prefix = "_loop0_" + name = f"{prefix}{self.counter}" # TODO: It's ugly to signal via the name. + self.todo[name] = Rule(name, None, Rhs([Alt([NamedItem(None, node)])])) + return name + + def name_gather(self, node: Gather) -> str: + self.counter += 1 + name = f"_gather_{self.counter}" + self.counter += 1 + extra_function_name = f"_loop0_{self.counter}" + extra_function_alt = Alt( + [NamedItem(None, node.separator), NamedItem("elem", node.node),], action="elem", + ) + self.todo[extra_function_name] = Rule( + extra_function_name, None, Rhs([extra_function_alt]), + ) + alt = Alt( + [NamedItem("elem", node.node), NamedItem("seq", NameLeaf(extra_function_name)),], + ) + self.todo[name] = Rule(name, None, Rhs([alt]),) + return name + + +def dedupe(name: str, names: List[str]) -> str: + origname = name + counter = 0 + while name in names: + counter += 1 + name = f"{origname}_{counter}" + names.append(name) + return name + + +def compute_nullables(rules: Dict[str, Rule]) -> None: + """Compute which rules in a grammar are nullable. + + Thanks to TatSu (tatsu/leftrec.py) for inspiration. + """ + for rule in rules.values(): + rule.nullable_visit(rules) + + +def compute_left_recursives( + rules: Dict[str, Rule] +) -> Tuple[Dict[str, AbstractSet[str]], List[AbstractSet[str]]]: + graph = make_first_graph(rules) + sccs = list(sccutils.strongly_connected_components(graph.keys(), graph)) + for scc in sccs: + if len(scc) > 1: + for name in scc: + rules[name].left_recursive = True + # Try to find a leader such that all cycles go through it. + leaders = set(scc) + for start in scc: + for cycle in sccutils.find_cycles_in_scc(graph, scc, start): + ## print("Cycle:", " -> ".join(cycle)) + leaders -= scc - set(cycle) + if not leaders: + raise ValueError( + f"SCC {scc} has no leadership candidate (no element is included in all cycles)" + ) + ## print("Leaders:", leaders) + leader = min(leaders) # Pick an arbitrary leader from the candidates. + rules[leader].leader = True + else: + name = min(scc) # The only element. + if name in graph[name]: + rules[name].left_recursive = True + rules[name].leader = True + return graph, sccs + + +def make_first_graph(rules: Dict[str, Rule]) -> Dict[str, AbstractSet[str]]: + """Compute the graph of left-invocations. + + There's an edge from A to B if A may invoke B at its initial + position. + + Note that this requires the nullable flags to have been computed. + """ + graph = {} + vertices: Set[str] = set() + for rulename, rhs in rules.items(): + graph[rulename] = names = rhs.initial_names() + vertices |= names + for vertex in vertices: + graph.setdefault(vertex, set()) + return graph diff --git a/Tools/peg_generator/pegen/python_generator.py b/Tools/peg_generator/pegen/python_generator.py new file mode 100644 index 0000000000000..b2891885f957e --- /dev/null +++ b/Tools/peg_generator/pegen/python_generator.py @@ -0,0 +1,224 @@ +from typing import Any, Dict, List, Optional, IO, Text, Tuple + +from pegen.grammar import ( + Cut, + GrammarVisitor, + NameLeaf, + StringLeaf, + Rhs, + NamedItem, + Lookahead, + PositiveLookahead, + NegativeLookahead, + Opt, + Repeat0, + Repeat1, + Gather, + Group, + Rule, + Alt, +) +from pegen import grammar +from pegen.parser_generator import dedupe, ParserGenerator + +MODULE_PREFIX = """\ +#!/usr/bin/env python3.8 +# @generated by pegen from {filename} + +import ast +import sys +import tokenize + +from typing import Any, Optional + +from pegen.parser import memoize, memoize_left_rec, logger, Parser + +""" +MODULE_SUFFIX = """ + +if __name__ == '__main__': + from pegen.parser import simple_parser_main + simple_parser_main(GeneratedParser) +""" + + +class PythonCallMakerVisitor(GrammarVisitor): + def __init__(self, parser_generator: ParserGenerator): + self.gen = parser_generator + self.cache: Dict[Any, Any] = {} + + def visit_NameLeaf(self, node: NameLeaf) -> Tuple[Optional[str], str]: + name = node.value + if name in ("NAME", "NUMBER", "STRING", "OP"): + name = name.lower() + return name, f"self.{name}()" + if name in ("NEWLINE", "DEDENT", "INDENT", "ENDMARKER", "ASYNC", "AWAIT"): + return name.lower(), f"self.expect({name!r})" + return name, f"self.{name}()" + + def visit_StringLeaf(self, node: StringLeaf) -> Tuple[str, str]: + return "literal", f"self.expect({node.value})" + + def visit_Rhs(self, node: Rhs) -> Tuple[Optional[str], str]: + if node in self.cache: + return self.cache[node] + if len(node.alts) == 1 and len(node.alts[0].items) == 1: + self.cache[node] = self.visit(node.alts[0].items[0]) + else: + name = self.gen.name_node(node) + self.cache[node] = name, f"self.{name}()" + return self.cache[node] + + def visit_NamedItem(self, node: NamedItem) -> Tuple[Optional[str], str]: + name, call = self.visit(node.item) + if node.name: + name = node.name + return name, call + + def lookahead_call_helper(self, node: Lookahead) -> Tuple[str, str]: + name, call = self.visit(node.node) + head, tail = call.split("(", 1) + assert tail[-1] == ")" + tail = tail[:-1] + return head, tail + + def visit_PositiveLookahead(self, node: PositiveLookahead) -> Tuple[None, str]: + head, tail = self.lookahead_call_helper(node) + return None, f"self.positive_lookahead({head}, {tail})" + + def visit_NegativeLookahead(self, node: NegativeLookahead) -> Tuple[None, str]: + head, tail = self.lookahead_call_helper(node) + return None, f"self.negative_lookahead({head}, {tail})" + + def visit_Opt(self, node: Opt) -> Tuple[str, str]: + name, call = self.visit(node.node) + return "opt", f"{call}," # Note trailing comma! + + def visit_Repeat0(self, node: Repeat0) -> Tuple[str, str]: + if node in self.cache: + return self.cache[node] + name = self.gen.name_loop(node.node, False) + self.cache[node] = name, f"self.{name}()," # Also a trailing comma! + return self.cache[node] + + def visit_Repeat1(self, node: Repeat1) -> Tuple[str, str]: + if node in self.cache: + return self.cache[node] + name = self.gen.name_loop(node.node, True) + self.cache[node] = name, f"self.{name}()" # But no trailing comma here! + return self.cache[node] + + def visit_Gather(self, node: Gather) -> Tuple[str, str]: + if node in self.cache: + return self.cache[node] + name = self.gen.name_gather(node) + self.cache[node] = name, f"self.{name}()" # No trailing comma here either! + return self.cache[node] + + def visit_Group(self, node: Group) -> Tuple[Optional[str], str]: + return self.visit(node.rhs) + + def visit_Cut(self, node: Cut) -> Tuple[str, str]: + return "cut", "True" + + +class PythonParserGenerator(ParserGenerator, GrammarVisitor): + def __init__(self, grammar: grammar.Grammar, file: Optional[IO[Text]]): + super().__init__(grammar, file) + self.callmakervisitor = PythonCallMakerVisitor(self) + + def generate(self, filename: str) -> None: + header = self.grammar.metas.get("header", MODULE_PREFIX) + if header is not None: + self.print(header.rstrip("\n").format(filename=filename)) + subheader = self.grammar.metas.get("subheader", "") + if subheader: + self.print(subheader.format(filename=filename)) + self.print("class GeneratedParser(Parser):") + while self.todo: + for rulename, rule in list(self.todo.items()): + del self.todo[rulename] + self.print() + with self.indent(): + self.visit(rule) + trailer = self.grammar.metas.get("trailer", MODULE_SUFFIX) + if trailer is not None: + self.print(trailer.rstrip("\n")) + + def visit_Rule(self, node: Rule) -> None: + is_loop = node.is_loop() + is_gather = node.is_gather() + rhs = node.flatten() + if node.left_recursive: + if node.leader: + self.print("@memoize_left_rec") + else: + # Non-leader rules in a cycle are not memoized, + # but they must still be logged. + self.print("@logger") + else: + self.print("@memoize") + node_type = node.type or "Any" + self.print(f"def {node.name}(self) -> Optional[{node_type}]:") + with self.indent(): + self.print(f"# {node.name}: {rhs}") + if node.nullable: + self.print(f"# nullable={node.nullable}") + self.print("mark = self.mark()") + if is_loop: + self.print("children = []") + self.visit(rhs, is_loop=is_loop, is_gather=is_gather) + if is_loop: + self.print("return children") + else: + self.print("return None") + + def visit_NamedItem(self, node: NamedItem, names: List[str]) -> None: + name, call = self.callmakervisitor.visit(node.item) + if node.name: + name = node.name + if not name: + self.print(call) + else: + if name != "cut": + name = dedupe(name, names) + self.print(f"({name} := {call})") + + def visit_Rhs(self, node: Rhs, is_loop: bool = False, is_gather: bool = False) -> None: + if is_loop: + assert len(node.alts) == 1 + for alt in node.alts: + self.visit(alt, is_loop=is_loop, is_gather=is_gather) + + def visit_Alt(self, node: Alt, is_loop: bool, is_gather: bool) -> None: + names: List[str] = [] + self.print("cut = False") # TODO: Only if needed. + if is_loop: + self.print("while (") + else: + self.print("if (") + with self.indent(): + first = True + for item in node.items: + if first: + first = False + else: + self.print("and") + self.visit(item, names=names) + self.print("):") + with self.indent(): + action = node.action + if not action: + if is_gather: + assert len(names) == 2 + action = f"[{names[0]}] + {names[1]}" + else: + action = f"[{', '.join(names)}]" + if is_loop: + self.print(f"children.append({action})") + self.print(f"mark = self.mark()") + else: + self.print(f"return {action}") + self.print("self.reset(mark)") + # Skip remaining alternatives if a cut was reached. + self.print("if cut: return None") # TODO: Only if needed. diff --git a/Tools/peg_generator/pegen/sccutils.py b/Tools/peg_generator/pegen/sccutils.py new file mode 100644 index 0000000000000..1f0586bb2f7d6 --- /dev/null +++ b/Tools/peg_generator/pegen/sccutils.py @@ -0,0 +1,128 @@ +# Adapted from mypy (mypy/build.py) under the MIT license. + +from typing import * + + +def strongly_connected_components( + vertices: AbstractSet[str], edges: Dict[str, AbstractSet[str]] +) -> Iterator[AbstractSet[str]]: + """Compute Strongly Connected Components of a directed graph. + + Args: + vertices: the labels for the vertices + edges: for each vertex, gives the target vertices of its outgoing edges + + Returns: + An iterator yielding strongly connected components, each + represented as a set of vertices. Each input vertex will occur + exactly once; vertices not part of a SCC are returned as + singleton sets. + + From http://code.activestate.com/recipes/578507/. + """ + identified: Set[str] = set() + stack: List[str] = [] + index: Dict[str, int] = {} + boundaries: List[int] = [] + + def dfs(v: str) -> Iterator[Set[str]]: + index[v] = len(stack) + stack.append(v) + boundaries.append(index[v]) + + for w in edges[v]: + if w not in index: + yield from dfs(w) + elif w not in identified: + while index[w] < boundaries[-1]: + boundaries.pop() + + if boundaries[-1] == index[v]: + boundaries.pop() + scc = set(stack[index[v] :]) + del stack[index[v] :] + identified.update(scc) + yield scc + + for v in vertices: + if v not in index: + yield from dfs(v) + + +def topsort( + data: Dict[AbstractSet[str], Set[AbstractSet[str]]] +) -> Iterable[AbstractSet[AbstractSet[str]]]: + """Topological sort. + + Args: + data: A map from SCCs (represented as frozen sets of strings) to + sets of SCCs, its dependencies. NOTE: This data structure + is modified in place -- for normalization purposes, + self-dependencies are removed and entries representing + orphans are added. + + Returns: + An iterator yielding sets of SCCs that have an equivalent + ordering. NOTE: The algorithm doesn't care about the internal + structure of SCCs. + + Example: + Suppose the input has the following structure: + + {A: {B, C}, B: {D}, C: {D}} + + This is normalized to: + + {A: {B, C}, B: {D}, C: {D}, D: {}} + + The algorithm will yield the following values: + + {D} + {B, C} + {A} + + From http://code.activestate.com/recipes/577413/. + """ + # TODO: Use a faster algorithm? + for k, v in data.items(): + v.discard(k) # Ignore self dependencies. + for item in set.union(*data.values()) - set(data.keys()): + data[item] = set() + while True: + ready = {item for item, dep in data.items() if not dep} + if not ready: + break + yield ready + data = {item: (dep - ready) for item, dep in data.items() if item not in ready} + assert not data, "A cyclic dependency exists amongst %r" % data + + +def find_cycles_in_scc( + graph: Dict[str, AbstractSet[str]], scc: AbstractSet[str], start: str +) -> Iterable[List[str]]: + """Find cycles in SCC emanating from start. + + Yields lists of the form ['A', 'B', 'C', 'A'], which means there's + a path from A -> B -> C -> A. The first item is always the start + argument, but the last item may be another element, e.g. ['A', + 'B', 'C', 'B'] means there's a path from A to B and there's a + cycle from B to C and back. + """ + # Basic input checks. + assert start in scc, (start, scc) + assert scc <= graph.keys(), scc - graph.keys() + + # Reduce the graph to nodes in the SCC. + graph = {src: {dst for dst in dsts if dst in scc} for src, dsts in graph.items() if src in scc} + assert start in graph + + # Recursive helper that yields cycles. + def dfs(node: str, path: List[str]) -> Iterator[List[str]]: + if node in path: + yield path + [node] + return + path = path + [node] # TODO: Make this not quadratic. + for child in graph[node]: + yield from dfs(child, path) + + yield from dfs(start, []) diff --git a/Tools/peg_generator/pegen/testutil.py b/Tools/peg_generator/pegen/testutil.py new file mode 100644 index 0000000000000..3616effe6b4f9 --- /dev/null +++ b/Tools/peg_generator/pegen/testutil.py @@ -0,0 +1,126 @@ +import importlib.util +import io +import os +import pathlib +import sys +import textwrap +import tokenize + +from typing import Any, cast, Dict, IO, Type, Final + +from pegen.build import compile_c_extension +from pegen.c_generator import CParserGenerator +from pegen.grammar import Grammar +from pegen.grammar_parser import GeneratedParser as GrammarParser +from pegen.parser import Parser +from pegen.python_generator import PythonParserGenerator +from pegen.tokenizer import Tokenizer + + +def generate_parser(grammar: Grammar) -> Type[Parser]: + # Generate a parser. + out = io.StringIO() + genr = PythonParserGenerator(grammar, out) + genr.generate("<string>") + + # Load the generated parser class. + ns: Dict[str, Any] = {} + exec(out.getvalue(), ns) + return ns["GeneratedParser"] + + +def run_parser(file: IO[bytes], parser_class: Type[Parser], *, verbose: bool = False) -> Any: + # Run a parser on a file (stream). + tokenizer = Tokenizer(tokenize.generate_tokens(file.readline)) # type: ignore # typeshed issue #3515 + parser = parser_class(tokenizer, verbose=verbose) + result = parser.start() + if result is None: + raise parser.make_syntax_error() + return result + + +def parse_string( + source: str, parser_class: Type[Parser], *, dedent: bool = True, verbose: bool = False +) -> Any: + # Run the parser on a string. + if dedent: + source = textwrap.dedent(source) + file = io.StringIO(source) + return run_parser(file, parser_class, verbose=verbose) # type: ignore # typeshed issue #3515 + + +def make_parser(source: str) -> Type[Parser]: + # Combine parse_string() and generate_parser(). + grammar = parse_string(source, GrammarParser) + return generate_parser(grammar) + + +def import_file(full_name: str, path: str) -> Any: + """Import a python module from a path""" + + spec = importlib.util.spec_from_file_location(full_name, path) + mod = importlib.util.module_from_spec(spec) + + # We assume this is not None and has an exec_module() method. + # See https://docs.python.org/3/reference/import.html?highlight=exec_module#loadi… + loader = cast(Any, spec.loader) + loader.exec_module(mod) + return mod + + +def generate_c_parser_source(grammar: Grammar) -> str: + out = io.StringIO() + genr = CParserGenerator(grammar, out) + genr.generate("<string>") + return out.getvalue() + + +def generate_parser_c_extension( + grammar: Grammar, path: pathlib.PurePath, debug: bool = False +) -> Any: + """Generate a parser c extension for the given grammar in the given path + + Returns a module object with a parse_string() method. + TODO: express that using a Protocol. + """ + # Make sure that the working directory is empty: reusing non-empty temporary + # directories when generating extensions can lead to segmentation faults. + # Check issue #95 (https://github.com/gvanrossum/pegen/issues/95) for more + # context. + assert not os.listdir(path) + source = path / "parse.c" + with open(source, "w") as file: + genr = CParserGenerator(grammar, file, debug=debug) + genr.generate("parse.c") + extension_path = compile_c_extension(str(source), build_dir=str(path / "build")) + extension = import_file("parse", extension_path) + return extension + + +def print_memstats() -> bool: + MiB: Final = 2 ** 20 + try: + import psutil # type: ignore + except ImportError: + return False + print("Memory stats:") + process = psutil.Process() + meminfo = process.memory_info() + res = {} + res["rss"] = meminfo.rss / MiB + res["vms"] = meminfo.vms / MiB + if sys.platform == "win32": + res["maxrss"] = meminfo.peak_wset / MiB + else: + # See https://stackoverflow.com/questions/938733/total-memory-used-by-python-proc… + import resource # Since it doesn't exist on Windows. + + rusage = resource.getrusage(resource.RUSAGE_SELF) + if sys.platform == "darwin": + factor = 1 + else: + factor = 1024 # Linux + res["maxrss"] = rusage.ru_maxrss * factor / MiB + for key, value in res.items(): + print(f" {key:12.12s}: {value:10.0f} MiB") + return True diff --git a/Tools/peg_generator/pegen/tokenizer.py b/Tools/peg_generator/pegen/tokenizer.py new file mode 100644 index 0000000000000..61a28efc84b62 --- /dev/null +++ b/Tools/peg_generator/pegen/tokenizer.py @@ -0,0 +1,86 @@ +import token +import tokenize +from typing import List, Iterator + +Mark = int # NewType('Mark', int) + +exact_token_types = token.EXACT_TOKEN_TYPES # type: ignore + + +def shorttok(tok: tokenize.TokenInfo) -> str: + return "%-25.25s" % f"{tok.start[0]}.{tok.start[1]}: {token.tok_name[tok.type]}:{tok.string!r}" + + +class Tokenizer: + """Caching wrapper for the tokenize module. + + This is pretty tied to Python's syntax. + """ + + _tokens: List[tokenize.TokenInfo] + + def __init__(self, tokengen: Iterator[tokenize.TokenInfo], *, verbose: bool = False): + self._tokengen = tokengen + self._tokens = [] + self._index = 0 + self._verbose = verbose + if verbose: + self.report(False, False) + + def getnext(self) -> tokenize.TokenInfo: + """Return the next token and updates the index.""" + cached = True + while self._index == len(self._tokens): + tok = next(self._tokengen) + if tok.type in (tokenize.NL, tokenize.COMMENT): + continue + if tok.type == token.ERRORTOKEN and tok.string.isspace(): + continue + self._tokens.append(tok) + cached = False + tok = self._tokens[self._index] + self._index += 1 + if self._verbose: + self.report(cached, False) + return tok + + def peek(self) -> tokenize.TokenInfo: + """Return the next token *without* updating the index.""" + while self._index == len(self._tokens): + tok = next(self._tokengen) + if tok.type in (tokenize.NL, tokenize.COMMENT): + continue + if tok.type == token.ERRORTOKEN and tok.string.isspace(): + continue + self._tokens.append(tok) + return self._tokens[self._index] + + def diagnose(self) -> tokenize.TokenInfo: + if not self._tokens: + self.getnext() + return self._tokens[-1] + + def mark(self) -> Mark: + return self._index + + def reset(self, index: Mark) -> None: + if index == self._index: + return + assert 0 <= index <= len(self._tokens), (index, len(self._tokens)) + old_index = self._index + self._index = index + if self._verbose: + self.report(True, index < old_index) + + def report(self, cached: bool, back: bool) -> None: + if back: + fill = "-" * self._index + "-" + elif cached: + fill = "-" * self._index + ">" + else: + fill = "-" * self._index + "*" + if self._index == 0: + print(f"{fill} (Bof)") + else: + tok = self._tokens[self._index - 1] + print(f"{fill} {shorttok(tok)}") diff --git a/Tools/peg_generator/pyproject.toml b/Tools/peg_generator/pyproject.toml new file mode 100644 index 0000000000000..f69c5b5e82faf --- /dev/null +++ b/Tools/peg_generator/pyproject.toml @@ -0,0 +1,9 @@ +[tool.black] +line-length = 99 +target_version = ['py38'] +exclude = ''' +( + /pegen/grammar_parser.py # generated file + | /test/test_data/ # test files +) +''' diff --git a/Tools/peg_generator/requirements.pip b/Tools/peg_generator/requirements.pip new file mode 100644 index 0000000000000..190b3488f7bc1 --- /dev/null +++ b/Tools/peg_generator/requirements.pip @@ -0,0 +1,2 @@ +memory-profiler==0.57.0 +psutil==5.7.0 diff --git a/Tools/peg_generator/scripts/__init__.py b/Tools/peg_generator/scripts/__init__.py new file mode 100644 index 0000000000000..1e423f483844b --- /dev/null +++ b/Tools/peg_generator/scripts/__init__.py @@ -0,0 +1 @@ +# This exists to let mypy find modules here diff --git a/Tools/peg_generator/scripts/ast_timings.py b/Tools/peg_generator/scripts/ast_timings.py new file mode 100644 index 0000000000000..7ebd46fdac685 --- /dev/null +++ b/Tools/peg_generator/scripts/ast_timings.py @@ -0,0 +1,28 @@ +import ast +import sys +import time +import token +import tokenize + +from pegen.testutil import print_memstats + + +def main() -> None: + t0 = time.time() + for filename in sys.argv[1:]: + print(filename, end="\r") + try: + with open(filename) as file: + source = file.read() + tree = ast.parse(source, filename) + except Exception as err: + print(f"{filename}: {err.__class__.__name__}: {err}", file=sys.stderr) + tok = None + t1 = time.time() + dt = t1 - t0 + print(f"Parsed in {dt:.3f} secs", file=sys.stderr) + print_memstats() + + +if __name__ == "__main__": + main() diff --git a/Tools/peg_generator/scripts/benchmark.py b/Tools/peg_generator/scripts/benchmark.py new file mode 100644 index 0000000000000..bc751156e8972 --- /dev/null +++ b/Tools/peg_generator/scripts/benchmark.py @@ -0,0 +1,140 @@ +#!/usr/bin/env python3.9 + +import argparse +import ast +import sys +import os +import resource +from time import time + +import memory_profiler + +sys.path.insert(0, os.getcwd()) +from peg_extension import parse +from pegen.build import build_parser_and_generator +from scripts.test_parse_directory import parse_directory + +argparser = argparse.ArgumentParser( + prog="benchmark", description="Reproduce the various pegen benchmarks" +) +argparser.add_argument( + "--parser", + action="store", + choices=["pegen", "cpython"], + default="pegen", + help="Which parser to benchmark (default is pegen)", +) +argparser.add_argument( + "--target", + action="store", + choices=["xxl", "stdlib"], + default="xxl", + help="Which target to use for the benchmark (default is xxl.py)", +) + +subcommands = argparser.add_subparsers(title="Benchmarks", dest="subcommand") +command_compile = subcommands.add_parser( + "compile", help="Benchmark parsing and compiling to bytecode" +) +command_parse = subcommands.add_parser("parse", help="Benchmark parsing and generating an ast.AST") +command_check = subcommands.add_parser( + "check", help="Benchmark parsing and throwing the tree away" +) + + +def benchmark(func): + def wrapper(*args): + times = list() + for _ in range(3): + start = time() + result = func(*args) + end = time() + times.append(end - start) + memory = memory_profiler.memory_usage((func, args)) + print(f"{func.__name__}") + print(f"\tTime: {sum(times)/3:.3f} seconds on an average of 3 runs") + print(f"\tMemory: {max(memory)} MiB on an average of 3 runs") + return result + + return wrapper + + +@benchmark +def time_compile(source, parser): + if parser == "cpython": + return compile(source, os.path.join("data", "xxl.py"), "exec") + else: + return parse.parse_string(source, mode=2) + + +@benchmark +def time_parse(source, parser): + if parser == "cpython": + return ast.parse(source, os.path.join("data", "xxl.py"), "exec") + else: + return parse.parse_string(source, mode=1) + + +@benchmark +def time_check(source): + return parse.parse_string(source, mode=0) + + +def run_benchmark_xxl(subcommand, parser, source): + if subcommand == "compile": + time_compile(source, parser) + elif subcommand == "parse": + time_parse(source, parser) + elif subcommand == "check": + time_check(source) + + +def run_benchmark_stdlib(subcommand, parser): + modes = {"compile": 2, "parse": 1, "check": 0} + extension = None + if parser == "pegen": + extension = build_parser_and_generator( + "../../Grammar/python.gram", + "peg_extension/parse.c", + compile_extension=True, + skip_actions=False, + ) + for _ in range(3): + parse_directory( + "../../Lib", + "../../Grammar/python.gram", + verbose=False, + excluded_files=[ + "*/bad*", + "*/lib2to3/tests/data/*", + ], + skip_actions=False, + tree_arg=0, + short=True, + extension=extension, + mode=modes[subcommand], + parser=parser, + ) + + +def main(): + args = argparser.parse_args() + subcommand = args.subcommand + parser = args.parser + target = args.target + + if subcommand is None: + argparser.error("A benchmark to run is required") + if subcommand == "check" and parser == "cpython": + argparser.error("Cannot use check target with the CPython parser") + + if target == "xxl": + with open(os.path.join("data", "xxl.py"), "r") as f: + source = f.read() + run_benchmark_xxl(subcommand, parser, source) + elif target == "stdlib": + run_benchmark_stdlib(subcommand, parser) + + +if __name__ == "__main__": + main() diff --git a/Tools/peg_generator/scripts/download_pypi_packages.py b/Tools/peg_generator/scripts/download_pypi_packages.py new file mode 100755 index 0000000000000..9874202d379ee --- /dev/null +++ b/Tools/peg_generator/scripts/download_pypi_packages.py @@ -0,0 +1,86 @@ +#!/usr/bin/env python3.8 + +import argparse +import os +import json + +from typing import Dict, Any +from urllib.request import urlretrieve + +argparser = argparse.ArgumentParser( + prog="download_pypi_packages", description="Helper program to download PyPI packages", +) +argparser.add_argument( + "-n", "--number", type=int, default=100, help="Number of packages to download" +) +argparser.add_argument( + "-a", "--all", action="store_true", help="Download all packages listed in the json file" +) + + +def load_json(filename: str) -> Dict[Any, Any]: + with open(os.path.join("data", f"{filename}.json"), "r") as f: + j = json.loads(f.read()) + return j + + +def remove_json(filename: str) -> None: + path = os.path.join("data", f"{filename}.json") + os.remove(path) + + +def download_package_json(package_name: str) -> None: + url = f"https://pypi.org/pypi/{package_name}/json" + urlretrieve(url, os.path.join("data", f"{package_name}.json")) + + +def download_package_code(name: str, package_json: Dict[Any, Any]) -> None: + source_index = -1 + for idx, url_info in enumerate(package_json["urls"]): + if url_info["python_version"] == "source": + source_index = idx + break + filename = package_json["urls"][source_index]["filename"] + url = package_json["urls"][source_index]["url"] + urlretrieve(url, os.path.join("data", "pypi", filename)) + + +def main() -> None: + args = argparser.parse_args() + number_packages = args.number + all_packages = args.all + + top_pypi_packages = load_json("top-pypi-packages-365-days") + if all_packages: + top_pypi_packages = top_pypi_packages["rows"] + elif number_packages >= 0 and number_packages <= 4000: + top_pypi_packages = top_pypi_packages["rows"][:number_packages] + else: + raise AssertionError("Unknown value for NUMBER_OF_PACKAGES") + + try: + os.mkdir(os.path.join("data", "pypi")) + except FileExistsError: + pass + + for package in top_pypi_packages: + package_name = package["project"] + + print(f"Downloading JSON Data for {package_name}... ", end="") + download_package_json(package_name) + print("Done") + + package_json = load_json(package_name) + try: + print(f"Dowloading and compressing package {package_name} ... ", end="") + download_package_code(package_name, package_json) + print("Done") + except (IndexError, KeyError): + print(f"Could not locate source for {package_name}") + continue + finally: + remove_json(package_name) + + +if __name__ == "__main__": + main() diff --git a/Tools/peg_generator/scripts/find_max_nesting.py b/Tools/peg_generator/scripts/find_max_nesting.py new file mode 100755 index 0000000000000..a2c41a821342a --- /dev/null +++ b/Tools/peg_generator/scripts/find_max_nesting.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python3.8 +"""Find the maximum amount of nesting for an expression that can be parsed +without causing a parse error. + +Starting at the INITIAL_NESTING_DEPTH, an expression containing n parenthesis +around a 0 is generated then tested with both the C and Python parsers. We +continue incrementing the number of parenthesis by 10 until both parsers have +failed. As soon as a single parser fails, we stop testing that parser. + +The grammar file, initial nesting size, and amount by which the nested size is +incremented on each success can be controlled by changing the GRAMMAR_FILE, +INITIAL_NESTING_DEPTH, or NESTED_INCR_AMT variables. + +Usage: python -m scripts.find_max_nesting +""" +import os +import sys +from tempfile import TemporaryDirectory +from pathlib import Path +from typing import Any + +from _peg_parser import parse_string + +GRAMMAR_FILE = "data/python.gram" +INITIAL_NESTING_DEPTH = 10 +NESTED_INCR_AMT = 10 + + +FAIL = "\033[91m" +ENDC = "\033[0m" + + +def check_nested_expr(nesting_depth: int) -> bool: + expr = f"{'(' * nesting_depth}0{')' * nesting_depth}" + + try: + parse_string(expr) + print(f"Nesting depth of {nesting_depth} is successful") + return True + except Exception as err: + print(f"{FAIL}(Failed with nesting depth of {nesting_depth}{ENDC}") + print(f"{FAIL}\t{err}{ENDC}") + return False + + +def main() -> None: + print(f"Testing {GRAMMAR_FILE} starting at nesting depth of {INITIAL_NESTING_DEPTH}...") + + nesting_depth = INITIAL_NESTING_DEPTH + succeeded = True + while succeeded: + expr = f"{'(' * nesting_depth}0{')' * nesting_depth}" + if succeeded: + succeeded = check_nested_expr(nesting_depth) + nesting_depth += NESTED_INCR_AMT + + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/Tools/peg_generator/scripts/grammar_grapher.py b/Tools/peg_generator/scripts/grammar_grapher.py new file mode 100755 index 0000000000000..3aa25466c70d4 --- /dev/null +++ b/Tools/peg_generator/scripts/grammar_grapher.py @@ -0,0 +1,111 @@ +#!/usr/bin/env python3.8 + +""" Convert a grammar into a dot-file suitable for use with GraphViz + + For example: + Generate the GraphViz file: + # scripts/grammar_grapher.py data/python.gram > python.gv + + Then generate the graph... + + # twopi python.gv -Tpng > python_twopi.png + + or + + # dot python.gv -Tpng > python_dot.png + + NOTE: The _dot_ and _twopi_ tools seem to produce the most useful results. + The _circo_ tool is the worst of the bunch. Don't even bother. +""" + +import argparse +import sys + +from typing import Any, List + +sys.path.insert(0, ".") + +from pegen.build import build_parser +from pegen.grammar import ( + Alt, + Cut, + Grammar, + Group, + Leaf, + Lookahead, + Rule, + NameLeaf, + NamedItem, + Opt, + Repeat, + Rhs, +) + +argparser = argparse.ArgumentParser(prog="graph_grammar", description="Graph a grammar tree",) +argparser.add_argument("grammar_file", help="The grammar file to graph") + + +def references_for_item(item: Any) -> List[Any]: + if isinstance(item, Alt): + return [_ref for _item in item.items for _ref in references_for_item(_item)] + elif isinstance(item, Cut): + return [] + elif isinstance(item, Group): + return references_for_item(item.rhs) + elif isinstance(item, Lookahead): + return references_for_item(item.node) + elif isinstance(item, NamedItem): + return references_for_item(item.item) + + # NOTE NameLeaf must be before Leaf + elif isinstance(item, NameLeaf): + if item.value == "ENDMARKER": + return [] + return [item.value] + elif isinstance(item, Leaf): + return [] + + elif isinstance(item, Opt): + return references_for_item(item.node) + elif isinstance(item, Repeat): + return references_for_item(item.node) + elif isinstance(item, Rhs): + return [_ref for alt in item.alts for _ref in references_for_item(alt)] + elif isinstance(item, Rule): + return references_for_item(item.rhs) + else: + raise RuntimeError(f"Unknown item: {type(item)}") + + +def main() -> None: + args = argparser.parse_args() + + try: + grammar, parser, tokenizer = build_parser(args.grammar_file) + except Exception as err: + print("ERROR: Failed to parse grammar file", file=sys.stderr) + sys.exit(1) + + references = {} + for name, rule in grammar.rules.items(): + references[name] = set(references_for_item(rule)) + + # Flatten the start node if has only a single reference + root_node = "start" + if start := references["start"]: + if len(start) == 1: + root_node = list(start)[0] + del references["start"] + + print("digraph g1 {") + print('\toverlap="scale";') # Force twopi to scale the graph to avoid overlaps + print(f'\troot="{root_node}";') + print(f"\t{root_node} [color=green, shape=circle]") + for name, refs in references.items(): + if refs: # Ignore empty sets + print(f"\t{name} -> {','.join(refs)};") + print("}") + + +if __name__ == "__main__": + main() diff --git a/Tools/peg_generator/scripts/joinstats.py b/Tools/peg_generator/scripts/joinstats.py new file mode 100644 index 0000000000000..b2d762b3d4286 --- /dev/null +++ b/Tools/peg_generator/scripts/joinstats.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python3.8 + +"""Produce a report about the most-memoable types. + +Reads a list of statistics from stdin. Each line must be two numbers, +being a type and a count. We then read some other files and produce a +list sorted by most frequent type. + +There should also be something to recognize left-recursive rules. +""" + +import os +import re +import sys + +from typing import Dict + +reporoot = os.path.dirname(os.path.dirname(__file__)) +parse_c = os.path.join(reporoot, "peg_extension", "parse.c") + + +class TypeMapper: + """State used to map types to names.""" + + def __init__(self, filename: str) -> None: + self.table: Dict[int, str] = {} + with open(filename) as f: + for line in f: + match = re.match(r"#define (\w+)_type (\d+)", line) + if match: + name, type = match.groups() + if "left" in line.lower(): + name += " // Left-recursive" + self.table[int(type)] = name + + def lookup(self, type: int) -> str: + return self.table.get(type, str(type)) + + +def main() -> None: + mapper = TypeMapper(parse_c) + table = [] + filename = sys.argv[1] + with open(filename) as f: + for lineno, line in enumerate(f, 1): + line = line.strip() + if not line or line.startswith("#"): + continue + parts = line.split() + # Extra fields ignored + if len(parts) < 2: + print(f"{lineno}: bad input ({line!r})") + continue + try: + type, count = map(int, parts[:2]) + except ValueError as err: + print(f"{lineno}: non-integer input ({line!r})") + continue + table.append((type, count)) + table.sort(key=lambda values: -values[1]) + for type, count in table: + print(f"{type:4d} {count:9d} {mapper.lookup(type)}") + + +if __name__ == "__main__": + main() diff --git a/Tools/peg_generator/scripts/show_parse.py b/Tools/peg_generator/scripts/show_parse.py new file mode 100755 index 0000000000000..f5f92fdaf755d --- /dev/null +++ b/Tools/peg_generator/scripts/show_parse.py @@ -0,0 +1,117 @@ +#!/usr/bin/env python3.8 + +"""Show the parse tree for a given program, nicely formatted. + +Example: + +$ scripts/show_parse.py a+b +Module( + body=[ + Expr( + value=BinOp( + left=Name(id="a", ctx=Load()), op=Add(), right=Name(id="b", ctx=Load()) + ) + ) + ], + type_ignores=[], +) +$ + +Use -v to show line numbers and column offsets. + +The formatting is done using black. You can also import this module +and call one of its functions. +""" + +import argparse +import ast +import difflib +import os +import sys +import tempfile + +from typing import List + +parser = argparse.ArgumentParser() +parser.add_argument( + "-d", "--diff", action="store_true", help="show diff between grammar and ast (requires -g)" +) +parser.add_argument("-g", "--grammar-file", help="grammar to use (default: use the ast module)") +parser.add_argument( + "-m", + "--multiline", + action="store_true", + help="concatenate program arguments using newline instead of space", +) +parser.add_argument("-v", "--verbose", action="store_true", help="show line/column numbers") +parser.add_argument("program", nargs="+", help="program to parse (will be concatenated)") + + +def format_tree(tree: ast.AST, verbose: bool = False) -> str: + with tempfile.NamedTemporaryFile("w+") as tf: + tf.write(ast.dump(tree, include_attributes=verbose)) + tf.write("\n") + tf.flush() + cmd = f"black -q {tf.name}" + sts = os.system(cmd) + if sts: + raise RuntimeError(f"Command {cmd!r} failed with status 0x{sts:x}") + tf.seek(0) + return tf.read() + + +def diff_trees(a: ast.AST, b: ast.AST, verbose: bool = False) -> List[str]: + sa = format_tree(a, verbose) + sb = format_tree(b, verbose) + la = sa.splitlines() + lb = sb.splitlines() + return list(difflib.unified_diff(la, lb, "a", "b", lineterm="")) + + +def show_parse(source: str, verbose: bool = False) -> str: + tree = ast.parse(source) + return format_tree(tree, verbose).rstrip("\n") + + +def print_parse(source: str, verbose: bool = False) -> None: + print(show_parse(source, verbose)) + + +def main() -> None: + args = parser.parse_args() + if args.diff and not args.grammar_file: + parser.error("-d/--diff requires -g/--grammar-file") + if args.multiline: + sep = "\n" + else: + sep = " " + program = sep.join(args.program) + if args.grammar_file: + sys.path.insert(0, os.curdir) + from pegen.build import build_parser_and_generator + + build_parser_and_generator(args.grammar_file, "peg_parser/parse.c", compile_extension=True) + from pegen.parse import parse_string # type: ignore[import] + + tree = parse_string(program, mode=1) + + if args.diff: + a = tree + b = ast.parse(program) + diff = diff_trees(a, b, args.verbose) + if diff: + for line in diff: + print(line) + else: + print("# Trees are the same") + else: + print(f"# Parsed using {args.grammar_file}") + print(format_tree(tree, args.verbose)) + else: + tree = ast.parse(program) + print("# Parse using ast.parse()") + print(format_tree(tree, args.verbose)) + + +if __name__ == "__main__": + main() diff --git a/Tools/peg_generator/scripts/test_parse_directory.py b/Tools/peg_generator/scripts/test_parse_directory.py new file mode 100755 index 0000000000000..06a38fca67a86 --- /dev/null +++ b/Tools/peg_generator/scripts/test_parse_directory.py @@ -0,0 +1,289 @@ +#!/usr/bin/env python3.8 + +import argparse +import ast +import os +import sys +import tempfile +import time +import traceback +from glob import glob +from pathlib import PurePath + +from typing import List, Optional, Any + +sys.path.insert(0, os.getcwd()) +from pegen.build import build_parser_and_generator +from pegen.testutil import print_memstats +from scripts import show_parse + +SUCCESS = "\033[92m" +FAIL = "\033[91m" +ENDC = "\033[0m" + +argparser = argparse.ArgumentParser( + prog="test_parse_directory", + description="Helper program to test directories or files for pegen", +) +argparser.add_argument("-d", "--directory", help="Directory path containing files to test") +argparser.add_argument("-g", "--grammar-file", help="Grammar file path") +argparser.add_argument( + "-e", "--exclude", action="append", default=[], help="Glob(s) for matching files to exclude" +) +argparser.add_argument( + "-s", "--short", action="store_true", help="Only show errors, in a more Emacs-friendly format" +) +argparser.add_argument( + "-v", "--verbose", action="store_true", help="Display detailed errors for failures" +) +argparser.add_argument( + "--skip-actions", action="store_true", help="Suppress code emission for rule actions", +) +argparser.add_argument( + "-t", "--tree", action="count", help="Compare parse tree to official AST", default=0 +) + + +def report_status( + succeeded: bool, + file: str, + verbose: bool, + error: Optional[Exception] = None, + short: bool = False, +) -> None: + if short and succeeded: + return + + if succeeded is True: + status = "OK" + COLOR = SUCCESS + else: + status = "Fail" + COLOR = FAIL + + if short: + lineno = 0 + offset = 0 + if isinstance(error, SyntaxError): + lineno = error.lineno or 1 + offset = error.offset or 1 + message = error.args[0] + else: + message = f"{error.__class__.__name__}: {error}" + print(f"{file}:{lineno}:{offset}: {message}") + else: + print(f"{COLOR}{file:60} {status}{ENDC}") + + if error and verbose: + print(f" {str(error.__class__.__name__)}: {error}") + + +def compare_trees( + actual_tree: ast.AST, file: str, verbose: bool, include_attributes: bool = False, +) -> int: + with open(file) as f: + expected_tree = ast.parse(f.read()) + + expected_text = ast.dump(expected_tree, include_attributes=include_attributes) + actual_text = ast.dump(actual_tree, include_attributes=include_attributes) + if actual_text == expected_text: + if verbose: + print("Tree for {file}:") + print(show_parse.format_tree(actual_tree, include_attributes)) + return 0 + + print(f"Diffing ASTs for {file} ...") + + expected = show_parse.format_tree(expected_tree, include_attributes) + actual = show_parse.format_tree(actual_tree, include_attributes) + + if verbose: + print("Expected for {file}:") + print(expected) + print("Actual for {file}:") + print(actual) + print(f"Diff for {file}:") + + diff = show_parse.diff_trees(expected_tree, actual_tree, include_attributes) + for line in diff: + print(line) + + return 1 + + +def parse_directory( + directory: str, + grammar_file: str, + verbose: bool, + excluded_files: List[str], + skip_actions: bool, + tree_arg: int, + short: bool, + extension: Any, + mode: int, + parser: str, +) -> int: + if parser == "cpython" and (tree_arg or mode == 0): + print("Cannot specify tree argument or mode=0 with the cpython parser.", file=sys.stderr) + return 1 + + if not directory: + print("You must specify a directory of files to test.", file=sys.stderr) + return 1 + + if grammar_file: + if not os.path.exists(grammar_file): + print(f"The specified grammar file, {grammar_file}, does not exist.", file=sys.stderr) + return 1 + + try: + if not extension and parser == "pegen": + build_parser_and_generator( + grammar_file, + "peg_extension/parse.c", + compile_extension=True, + skip_actions=skip_actions, + ) + except Exception as err: + print( + f"{FAIL}The following error occurred when generating the parser. Please check your grammar file.\n{ENDC}", + file=sys.stderr, + ) + traceback.print_exception(err.__class__, err, None) + + return 1 + + else: + print("A grammar file was not provided - attempting to use existing file...\n") + + if parser == "pegen": + try: + from peg_extension import parse # type: ignore + except: + print( + "An existing parser was not found. Please run `make` or specify a grammar file with the `-g` flag.", + file=sys.stderr, + ) + return 1 + + # For a given directory, traverse files and attempt to parse each one + # - Output success/failure for each file + errors = 0 + files = [] + trees = {} # Trees to compare (after everything else is done) + + t0 = time.time() + for file in sorted(glob(f"{directory}/**/*.py", recursive=True)): + # Only attempt to parse Python files and files that are not excluded + should_exclude_file = False + for pattern in excluded_files: + if PurePath(file).match(pattern): + should_exclude_file = True + break + + if not should_exclude_file: + try: + if tree_arg: + mode = 1 + if parser == "cpython": + with open(file, "r") as f: + source = f.read() + if mode == 2: + compile(source, file, "exec") + elif mode == 1: + ast.parse(source, file, "exec") + else: + tree = parse.parse_file(file, mode=mode) + if tree_arg: + trees[file] = tree + if not short: + report_status(succeeded=True, file=file, verbose=verbose) + except Exception as error: + try: + ast.parse(file) + except Exception: + if not short: + print(f"File {file} cannot be parsed by either pegen or the ast module.") + else: + report_status( + succeeded=False, file=file, verbose=verbose, error=error, short=short + ) + errors += 1 + files.append(file) + t1 = time.time() + + total_seconds = t1 - t0 + total_files = len(files) + + total_bytes = 0 + total_lines = 0 + for file in files: + # Count lines and bytes separately + with open(file, "rb") as f: + total_lines += sum(1 for _ in f) + total_bytes += f.tell() + + print( + f"Checked {total_files:,} files, {total_lines:,} lines,", + f"{total_bytes:,} bytes in {total_seconds:,.3f} seconds.", + ) + if total_seconds > 0: + print( + f"That's {total_lines / total_seconds :,.0f} lines/sec,", + f"or {total_bytes / total_seconds :,.0f} bytes/sec.", + ) + + if parser == "pegen": + # Dump memo stats to @data. + with open("@data", "w") as datafile: + for i, count in enumerate(parse.get_memo_stats()): + if count: + datafile.write(f"{i:4d} {count:9d}\n") + + if short: + print_memstats() + + if errors: + print(f"Encountered {errors} failures.", file=sys.stderr) + + # Compare trees (the dict is empty unless -t is given) + compare_trees_errors = 0 + for file, tree in trees.items(): + if not short: + print("Comparing ASTs for", file) + if compare_trees(tree, file, verbose, tree_arg >= 2) == 1: + compare_trees_errors += 1 + + if errors or compare_trees_errors: + return 1 + + return 0 + + +def main() -> None: + args = argparser.parse_args() + directory = args.directory + grammar_file = args.grammar_file + verbose = args.verbose + excluded_files = args.exclude + skip_actions = args.skip_actions + tree = args.tree + short = args.short + sys.exit( + parse_directory( + directory, + grammar_file, + verbose, + excluded_files, + skip_actions, + tree, + short, + None, + 0, + "pegen", + ) + ) + + +if __name__ == "__main__": + main() diff --git a/Tools/peg_generator/scripts/test_pypi_packages.py b/Tools/peg_generator/scripts/test_pypi_packages.py new file mode 100755 index 0000000000000..90490330fef1d --- /dev/null +++ b/Tools/peg_generator/scripts/test_pypi_packages.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python3.8 + +import argparse +import os +import glob +import tarfile +import zipfile +import shutil +import sys + +from typing import Generator, Any + +sys.path.insert(0, ".") +from pegen import build +from scripts import test_parse_directory + +argparser = argparse.ArgumentParser( + prog="test_pypi_packages", description="Helper program to test parsing PyPI packages", +) +argparser.add_argument( + "-t", "--tree", action="count", help="Compare parse tree to official AST", default=0 +) + + +def get_packages() -> Generator[str, None, None]: + all_packages = ( + glob.glob("./data/pypi/*.tar.gz") + + glob.glob("./data/pypi/*.zip") + + glob.glob("./data/pypi/*.tgz") + ) + for package in all_packages: + yield package + + +def extract_files(filename: str) -> None: + savedir = os.path.join("data", "pypi") + if tarfile.is_tarfile(filename): + tarfile.open(filename).extractall(savedir) + elif zipfile.is_zipfile(filename): + zipfile.ZipFile(filename).extractall(savedir) + else: + raise ValueError(f"Could not identify type of compressed file {filename}") + + +def find_dirname(package_name: str) -> str: + for name in os.listdir(os.path.join("data", "pypi")): + full_path = os.path.join("data", "pypi", name) + if os.path.isdir(full_path) and name in package_name: + return full_path + assert False # This is to fix mypy, should never be reached + + +def run_tests(dirname: str, tree: int, extension: Any) -> int: + return test_parse_directory.parse_directory( + dirname, + "data/python.gram", + verbose=False, + excluded_files=[ + "*/failset/*", + "*/failset/**", + "*/failset/**/*", + "*/test2to3/*", + "*/test2to3/**/*", + "*/bad*", + "*/lib2to3/tests/data/*", + ], + skip_actions=False, + tree_arg=tree, + short=True, + extension=extension, + ) + + +def main() -> None: + args = argparser.parse_args() + tree = args.tree + + extension = build.build_parser_and_generator( + "data/python.gram", "peg_parser/parse.c", compile_extension=True + ) + for package in get_packages(): + print(f"Extracting files from {package}... ", end="") + try: + extract_files(package) + print("Done") + except ValueError as e: + print(e) + continue + + print(f"Trying to parse all python files ... ") + dirname = find_dirname(package) + status = run_tests(dirname, tree, extension) + if status == 0: + print("Done") + shutil.rmtree(dirname) + else: + print(f"Failed to parse {dirname}") + + +if __name__ == "__main__": + main() diff --git a/Tools/scripts/run_tests.py b/Tools/scripts/run_tests.py index 3c1c3bd060bff..bcfa5e943b347 100644 --- a/Tools/scripts/run_tests.py +++ b/Tools/scripts/run_tests.py @@ -25,8 +25,10 @@ def main(regrtest_args): '-u', # Unbuffered stdout and stderr '-W', 'default', # Warnings set to 'default' '-bb', # Warnings about bytes/bytearray - '-E', # Ignore environment variables ] + if 'PYTHONOLDPARSER' not in os.environ: + args.append('-E') # Ignore environment variables + # Allow user-specified interpreter options to override our defaults. args.extend(test.support.args_from_interpreter_flags())

4 years, 7 months