[Python-checkins] bpo-35766: Merge typed_ast back into CPython (GH-11645)

https://github.com/python/cpython/commit/dcfcd146f8e6fc5c2fc16a4c192a0c5f5ca... commit: dcfcd146f8e6fc5c2fc16a4c192a0c5f5ca8c53c branch: master author: Guido van Rossum <guido@python.org> committer: Łukasz Langa <lukasz@langa.pl> date: 2019-01-31T12:40:27+01:00 summary: bpo-35766: Merge typed_ast back into CPython (GH-11645) files: A Lib/test/test_type_comments.py A Misc/NEWS.d/next/Core and Builtins/2019-01-22-19-17-27.bpo-35766.gh1tHZ.rst M Doc/library/ast.rst M Doc/library/token-list.inc M Doc/library/token.rst M Grammar/Grammar M Grammar/Tokens M Include/Python-ast.h M Include/compile.h M Include/graminit.h M Include/parsetok.h M Include/token.h M Lib/ast.py M Lib/symbol.py M Lib/test/test_asdl_parser.py M Lib/test/test_ast.py M Lib/token.py M Modules/parsermodule.c M Parser/Python.asdl M Parser/asdl_c.py M Parser/parser.c M Parser/parsetok.c M Parser/token.c M Parser/tokenizer.c M Parser/tokenizer.h M Python/Python-ast.c M Python/ast.c M Python/bltinmodule.c M Python/graminit.c M Python/pythonrun.c diff --git a/Doc/library/ast.rst b/Doc/library/ast.rst index 7715a28ce1b8..3df7f9ebc70c 100644 --- a/Doc/library/ast.rst +++ b/Doc/library/ast.rst @@ -126,16 +126,33 @@ The abstract grammar is currently defined as follows: Apart from the node classes, the :mod:`ast` module defines these utility functions and classes for traversing abstract syntax trees: -.. function:: parse(source, filename='<unknown>', mode='exec') +.. function:: parse(source, filename='<unknown>', mode='exec', *, type_comments=False) Parse the source into an AST node. Equivalent to ``compile(source, filename, mode, ast.PyCF_ONLY_AST)``. + If ``type_comments=True`` is given, the parser is modified to check + and return type comments as specified by :pep:`484` and :pep:`526`. + This is equivalent to adding :data:`ast.PyCF_TYPE_COMMENTS` to the + flags passed to :func:`compile()`. This will report syntax errors + for misplaced type comments. Without this flag, type comments will + be ignored, and the ``type_comment`` field on selected AST nodes + will always be ``None``. In addition, the locations of ``# type: + ignore`` comments will be returned as the ``type_ignores`` + attribute of :class:`Module` (otherwise it is always an empty list). + + In addition, if ``mode`` is ``'func_type'``, the input syntax is + modified to correspond to :pep:`484` "signature type comments", + e.g. ``(str, int) -> List[str]``. + .. warning:: It is possible to crash the Python interpreter with a sufficiently large/complex string due to stack depth limitations in Python's AST compiler. + .. versionchanged:: 3.8 + Added ``type_comments=True`` and ``mode='func_type'``. + .. function:: literal_eval(node_or_string) diff --git a/Doc/library/token-list.inc b/Doc/library/token-list.inc index 3ea9439be859..cb9fcd79effc 100644 --- a/Doc/library/token-list.inc +++ b/Doc/library/token-list.inc @@ -203,6 +203,10 @@ .. data:: OP +.. data:: TYPE_IGNORE + +.. data:: TYPE_COMMENT + .. data:: ERRORTOKEN .. data:: N_TOKENS diff --git a/Doc/library/token.rst b/Doc/library/token.rst index 5358eb5a291e..4936e9aa08f4 100644 --- a/Doc/library/token.rst +++ b/Doc/library/token.rst @@ -69,6 +69,13 @@ the :mod:`tokenize` module. always be an ``ENCODING`` token. +.. data:: TYPE_COMMENT + + Token value indicating that a type comment was recognized. Such + tokens are only produced when :func:`ast.parse()` is invoked with + ``type_comments=True``. + + .. versionchanged:: 3.5 Added :data:`AWAIT` and :data:`ASYNC` tokens. @@ -78,3 +85,6 @@ the :mod:`tokenize` module. .. versionchanged:: 3.7 Removed :data:`AWAIT` and :data:`ASYNC` tokens. "async" and "await" are now tokenized as :data:`NAME` tokens. + +.. versionchanged:: 3.8 + Added :data:`TYPE_COMMENT`. diff --git a/Grammar/Grammar b/Grammar/Grammar index 8455c1259259..e65a688e4cd8 100644 --- a/Grammar/Grammar +++ b/Grammar/Grammar @@ -7,7 +7,9 @@ # single_input is a single interactive statement; # file_input is a module or sequence of commands read from an input file; # eval_input is the input for the eval() functions. +# func_type_input is a PEP 484 Python 2 function type comment # NB: compound_stmt in single_input is followed by extra NEWLINE! +# NB: due to the way TYPE_COMMENT is tokenized it will always be followed by a NEWLINE single_input: NEWLINE | simple_stmt | compound_stmt NEWLINE file_input: (NEWLINE | stmt)* ENDMARKER eval_input: testlist NEWLINE* ENDMARKER @@ -17,14 +19,14 @@ decorators: decorator+ decorated: decorators (classdef | funcdef | async_funcdef) async_funcdef: 'async' funcdef -funcdef: 'def' NAME parameters ['->' test] ':' suite +funcdef: 'def' NAME parameters ['->' test] ':' [TYPE_COMMENT] func_body_suite parameters: '(' [typedargslist] ')' -typedargslist: (tfpdef ['=' test] (',' tfpdef ['=' test])* [',' [ - '*' [tfpdef] (',' tfpdef ['=' test])* [',' ['**' tfpdef [',']]] - | '**' tfpdef [',']]] - | '*' [tfpdef] (',' tfpdef ['=' test])* [',' ['**' tfpdef [',']]] - | '**' tfpdef [',']) +typedargslist: (tfpdef ['=' test] (',' [TYPE_COMMENT] tfpdef ['=' test])* (TYPE_COMMENT | [',' [TYPE_COMMENT] [ + '*' [tfpdef] (',' [TYPE_COMMENT] tfpdef ['=' test])* (TYPE_COMMENT | [',' [TYPE_COMMENT] ['**' tfpdef [','] [TYPE_COMMENT]]]) + | '**' tfpdef [','] [TYPE_COMMENT]]]) + | '*' [tfpdef] (',' [TYPE_COMMENT] tfpdef ['=' test])* (TYPE_COMMENT | [',' [TYPE_COMMENT] ['**' tfpdef [','] [TYPE_COMMENT]]]) + | '**' tfpdef [','] [TYPE_COMMENT]) tfpdef: NAME [':' test] varargslist: (vfpdef ['=' test] (',' vfpdef ['=' test])* [',' [ '*' [vfpdef] (',' vfpdef ['=' test])* [',' ['**' vfpdef [',']]] @@ -39,7 +41,7 @@ simple_stmt: small_stmt (';' small_stmt)* [';'] NEWLINE small_stmt: (expr_stmt | del_stmt | pass_stmt | flow_stmt | import_stmt | global_stmt | nonlocal_stmt | assert_stmt) expr_stmt: testlist_star_expr (annassign | augassign (yield_expr|testlist) | - ('=' (yield_expr|testlist_star_expr))*) + [('=' (yield_expr|testlist_star_expr))+ [TYPE_COMMENT]] ) annassign: ':' test ['=' (yield_expr|testlist)] testlist_star_expr: (test|star_expr) (',' (test|star_expr))* [','] augassign: ('+=' | '-=' | '*=' | '@=' | '/=' | '%=' | '&=' | '|=' | '^=' | @@ -71,13 +73,13 @@ compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | with_stmt | funcdef async_stmt: 'async' (funcdef | with_stmt | for_stmt) if_stmt: 'if' namedexpr_test ':' suite ('elif' namedexpr_test ':' suite)* ['else' ':' suite] while_stmt: 'while' test ':' suite ['else' ':' suite] -for_stmt: 'for' exprlist 'in' testlist ':' suite ['else' ':' suite] +for_stmt: 'for' exprlist 'in' testlist ':' [TYPE_COMMENT] suite ['else' ':' suite] try_stmt: ('try' ':' suite ((except_clause ':' suite)+ ['else' ':' suite] ['finally' ':' suite] | 'finally' ':' suite)) -with_stmt: 'with' with_item (',' with_item)* ':' suite +with_stmt: 'with' with_item (',' with_item)* ':' [TYPE_COMMENT] suite with_item: test ['as' expr] # NB compile.c makes sure that the default except clause is last except_clause: 'except' [test ['as' NAME]] @@ -150,3 +152,14 @@ encoding_decl: NAME yield_expr: 'yield' [yield_arg] yield_arg: 'from' test | testlist_star_expr + +# the TYPE_COMMENT in suites is only parsed for funcdefs, +# but can't go elsewhere due to ambiguity +func_body_suite: simple_stmt | NEWLINE [TYPE_COMMENT NEWLINE] INDENT stmt+ DEDENT + +func_type_input: func_type NEWLINE* ENDMARKER +func_type: '(' [typelist] ')' '->' test +# typelist is a modified typedargslist (see above) +typelist: (test (',' test)* [',' + ['*' [test] (',' test)* [',' '**' test] | '**' test]] + | '*' [test] (',' test)* [',' '**' test] | '**' test) diff --git a/Grammar/Tokens b/Grammar/Tokens index f6f303bd5292..1d45e05ea21d 100644 --- a/Grammar/Tokens +++ b/Grammar/Tokens @@ -55,6 +55,8 @@ ELLIPSIS '...' COLONEQUAL ':=' OP +TYPE_IGNORE +TYPE_COMMENT ERRORTOKEN # These aren't used by the C tokenizer but are needed for tokenize.py diff --git a/Include/Python-ast.h b/Include/Python-ast.h index 3527ae8949e7..106cecff887e 100644 --- a/Include/Python-ast.h +++ b/Include/Python-ast.h @@ -46,14 +46,17 @@ typedef struct _alias *alias_ty; typedef struct _withitem *withitem_ty; +typedef struct _type_ignore *type_ignore_ty; + enum _mod_kind {Module_kind=1, Interactive_kind=2, Expression_kind=3, - Suite_kind=4}; + FunctionType_kind=4, Suite_kind=5}; struct _mod { enum _mod_kind kind; union { struct { asdl_seq *body; + asdl_seq *type_ignores; } Module; struct { @@ -64,6 +67,11 @@ struct _mod { expr_ty body; } Expression; + struct { + asdl_seq *argtypes; + expr_ty returns; + } FunctionType; + struct { asdl_seq *body; } Suite; @@ -88,6 +96,7 @@ struct _stmt { asdl_seq *body; asdl_seq *decorator_list; expr_ty returns; + string type_comment; } FunctionDef; struct { @@ -96,6 +105,7 @@ struct _stmt { asdl_seq *body; asdl_seq *decorator_list; expr_ty returns; + string type_comment; } AsyncFunctionDef; struct { @@ -117,6 +127,7 @@ struct _stmt { struct { asdl_seq *targets; expr_ty value; + string type_comment; } Assign; struct { @@ -137,6 +148,7 @@ struct _stmt { expr_ty iter; asdl_seq *body; asdl_seq *orelse; + string type_comment; } For; struct { @@ -144,6 +156,7 @@ struct _stmt { expr_ty iter; asdl_seq *body; asdl_seq *orelse; + string type_comment; } AsyncFor; struct { @@ -161,11 +174,13 @@ struct _stmt { struct { asdl_seq *items; asdl_seq *body; + string type_comment; } With; struct { asdl_seq *items; asdl_seq *body; + string type_comment; } AsyncWith; struct { @@ -421,6 +436,7 @@ struct _arguments { struct _arg { identifier arg; expr_ty annotation; + string type_comment; int lineno; int col_offset; int end_lineno; @@ -442,26 +458,40 @@ struct _withitem { expr_ty optional_vars; }; +enum _type_ignore_kind {TypeIgnore_kind=1}; +struct _type_ignore { + enum _type_ignore_kind kind; + union { + struct { + int lineno; + } TypeIgnore; + + } v; +}; + // Note: these macros affect function definitions, not only call sites. -#define Module(a0, a1) _Py_Module(a0, a1) -mod_ty _Py_Module(asdl_seq * body, PyArena *arena); +#define Module(a0, a1, a2) _Py_Module(a0, a1, a2) +mod_ty _Py_Module(asdl_seq * body, asdl_seq * type_ignores, PyArena *arena); #define Interactive(a0, a1) _Py_Interactive(a0, a1) mod_ty _Py_Interactive(asdl_seq * body, PyArena *arena); #define Expression(a0, a1) _Py_Expression(a0, a1) mod_ty _Py_Expression(expr_ty body, PyArena *arena); +#define FunctionType(a0, a1, a2) _Py_FunctionType(a0, a1, a2) +mod_ty _Py_FunctionType(asdl_seq * argtypes, expr_ty returns, PyArena *arena); #define Suite(a0, a1) _Py_Suite(a0, a1) mod_ty _Py_Suite(asdl_seq * body, PyArena *arena); -#define FunctionDef(a0, a1, a2, a3, a4, a5, a6, a7, a8, a9) _Py_FunctionDef(a0, a1, a2, a3, a4, a5, a6, a7, a8, a9) +#define FunctionDef(a0, a1, a2, a3, a4, a5, a6, a7, a8, a9, a10) _Py_FunctionDef(a0, a1, a2, a3, a4, a5, a6, a7, a8, a9, a10) stmt_ty _Py_FunctionDef(identifier name, arguments_ty args, asdl_seq * body, - asdl_seq * decorator_list, expr_ty returns, int lineno, - int col_offset, int end_lineno, int end_col_offset, - PyArena *arena); -#define AsyncFunctionDef(a0, a1, a2, a3, a4, a5, a6, a7, a8, a9) _Py_AsyncFunctionDef(a0, a1, a2, a3, a4, a5, a6, a7, a8, a9) + asdl_seq * decorator_list, expr_ty returns, string + type_comment, int lineno, int col_offset, int + end_lineno, int end_col_offset, PyArena *arena); +#define AsyncFunctionDef(a0, a1, a2, a3, a4, a5, a6, a7, a8, a9, a10) _Py_AsyncFunctionDef(a0, a1, a2, a3, a4, a5, a6, a7, a8, a9, a10) stmt_ty _Py_AsyncFunctionDef(identifier name, arguments_ty args, asdl_seq * body, asdl_seq * decorator_list, expr_ty returns, - int lineno, int col_offset, int end_lineno, int - end_col_offset, PyArena *arena); + string type_comment, int lineno, int col_offset, + int end_lineno, int end_col_offset, PyArena + *arena); #define ClassDef(a0, a1, a2, a3, a4, a5, a6, a7, a8, a9) _Py_ClassDef(a0, a1, a2, a3, a4, a5, a6, a7, a8, a9) stmt_ty _Py_ClassDef(identifier name, asdl_seq * bases, asdl_seq * keywords, asdl_seq * body, asdl_seq * decorator_list, int lineno, @@ -473,10 +503,10 @@ stmt_ty _Py_Return(expr_ty value, int lineno, int col_offset, int end_lineno, #define Delete(a0, a1, a2, a3, a4, a5) _Py_Delete(a0, a1, a2, a3, a4, a5) stmt_ty _Py_Delete(asdl_seq * targets, int lineno, int col_offset, int end_lineno, int end_col_offset, PyArena *arena); -#define Assign(a0, a1, a2, a3, a4, a5, a6) _Py_Assign(a0, a1, a2, a3, a4, a5, a6) -stmt_ty _Py_Assign(asdl_seq * targets, expr_ty value, int lineno, int - col_offset, int end_lineno, int end_col_offset, PyArena - *arena); +#define Assign(a0, a1, a2, a3, a4, a5, a6, a7) _Py_Assign(a0, a1, a2, a3, a4, a5, a6, a7) +stmt_ty _Py_Assign(asdl_seq * targets, expr_ty value, string type_comment, int + lineno, int col_offset, int end_lineno, int end_col_offset, + PyArena *arena); #define AugAssign(a0, a1, a2, a3, a4, a5, a6, a7) _Py_AugAssign(a0, a1, a2, a3, a4, a5, a6, a7) stmt_ty _Py_AugAssign(expr_ty target, operator_ty op, expr_ty value, int lineno, int col_offset, int end_lineno, int @@ -485,14 +515,14 @@ stmt_ty _Py_AugAssign(expr_ty target, operator_ty op, expr_ty value, int stmt_ty _Py_AnnAssign(expr_ty target, expr_ty annotation, expr_ty value, int simple, int lineno, int col_offset, int end_lineno, int end_col_offset, PyArena *arena); -#define For(a0, a1, a2, a3, a4, a5, a6, a7, a8) _Py_For(a0, a1, a2, a3, a4, a5, a6, a7, a8) +#define For(a0, a1, a2, a3, a4, a5, a6, a7, a8, a9) _Py_For(a0, a1, a2, a3, a4, a5, a6, a7, a8, a9) stmt_ty _Py_For(expr_ty target, expr_ty iter, asdl_seq * body, asdl_seq * - orelse, int lineno, int col_offset, int end_lineno, int - end_col_offset, PyArena *arena); -#define AsyncFor(a0, a1, a2, a3, a4, a5, a6, a7, a8) _Py_AsyncFor(a0, a1, a2, a3, a4, a5, a6, a7, a8) + orelse, string type_comment, int lineno, int col_offset, int + end_lineno, int end_col_offset, PyArena *arena); +#define AsyncFor(a0, a1, a2, a3, a4, a5, a6, a7, a8, a9) _Py_AsyncFor(a0, a1, a2, a3, a4, a5, a6, a7, a8, a9) stmt_ty _Py_AsyncFor(expr_ty target, expr_ty iter, asdl_seq * body, asdl_seq * - orelse, int lineno, int col_offset, int end_lineno, int - end_col_offset, PyArena *arena); + orelse, string type_comment, int lineno, int col_offset, + int end_lineno, int end_col_offset, PyArena *arena); #define While(a0, a1, a2, a3, a4, a5, a6, a7) _Py_While(a0, a1, a2, a3, a4, a5, a6, a7) stmt_ty _Py_While(expr_ty test, asdl_seq * body, asdl_seq * orelse, int lineno, int col_offset, int end_lineno, int end_col_offset, PyArena @@ -501,13 +531,14 @@ stmt_ty _Py_While(expr_ty test, asdl_seq * body, asdl_seq * orelse, int lineno, stmt_ty _Py_If(expr_ty test, asdl_seq * body, asdl_seq * orelse, int lineno, int col_offset, int end_lineno, int end_col_offset, PyArena *arena); -#define With(a0, a1, a2, a3, a4, a5, a6) _Py_With(a0, a1, a2, a3, a4, a5, a6) -stmt_ty _Py_With(asdl_seq * items, asdl_seq * body, int lineno, int col_offset, - int end_lineno, int end_col_offset, PyArena *arena); -#define AsyncWith(a0, a1, a2, a3, a4, a5, a6) _Py_AsyncWith(a0, a1, a2, a3, a4, a5, a6) -stmt_ty _Py_AsyncWith(asdl_seq * items, asdl_seq * body, int lineno, int - col_offset, int end_lineno, int end_col_offset, PyArena - *arena); +#define With(a0, a1, a2, a3, a4, a5, a6, a7) _Py_With(a0, a1, a2, a3, a4, a5, a6, a7) +stmt_ty _Py_With(asdl_seq * items, asdl_seq * body, string type_comment, int + lineno, int col_offset, int end_lineno, int end_col_offset, + PyArena *arena); +#define AsyncWith(a0, a1, a2, a3, a4, a5, a6, a7) _Py_AsyncWith(a0, a1, a2, a3, a4, a5, a6, a7) +stmt_ty _Py_AsyncWith(asdl_seq * items, asdl_seq * body, string type_comment, + int lineno, int col_offset, int end_lineno, int + end_col_offset, PyArena *arena); #define Raise(a0, a1, a2, a3, a4, a5, a6) _Py_Raise(a0, a1, a2, a3, a4, a5, a6) stmt_ty _Py_Raise(expr_ty exc, expr_ty cause, int lineno, int col_offset, int end_lineno, int end_col_offset, PyArena *arena); @@ -656,9 +687,10 @@ excepthandler_ty _Py_ExceptHandler(expr_ty type, identifier name, asdl_seq * arguments_ty _Py_arguments(asdl_seq * args, arg_ty vararg, asdl_seq * kwonlyargs, asdl_seq * kw_defaults, arg_ty kwarg, asdl_seq * defaults, PyArena *arena); -#define arg(a0, a1, a2, a3, a4, a5, a6) _Py_arg(a0, a1, a2, a3, a4, a5, a6) -arg_ty _Py_arg(identifier arg, expr_ty annotation, int lineno, int col_offset, - int end_lineno, int end_col_offset, PyArena *arena); +#define arg(a0, a1, a2, a3, a4, a5, a6, a7) _Py_arg(a0, a1, a2, a3, a4, a5, a6, a7) +arg_ty _Py_arg(identifier arg, expr_ty annotation, string type_comment, int + lineno, int col_offset, int end_lineno, int end_col_offset, + PyArena *arena); #define keyword(a0, a1, a2) _Py_keyword(a0, a1, a2) keyword_ty _Py_keyword(identifier arg, expr_ty value, PyArena *arena); #define alias(a0, a1, a2) _Py_alias(a0, a1, a2) @@ -666,6 +698,8 @@ alias_ty _Py_alias(identifier name, identifier asname, PyArena *arena); #define withitem(a0, a1, a2) _Py_withitem(a0, a1, a2) withitem_ty _Py_withitem(expr_ty context_expr, expr_ty optional_vars, PyArena *arena); +#define TypeIgnore(a0, a1) _Py_TypeIgnore(a0, a1) +type_ignore_ty _Py_TypeIgnore(int lineno, PyArena *arena); PyObject* PyAST_mod2obj(mod_ty t); mod_ty PyAST_obj2mod(PyObject* ast, PyArena* arena, int mode); diff --git a/Include/compile.h b/Include/compile.h index 2dacfff37f8c..d0bbed8f558b 100644 --- a/Include/compile.h +++ b/Include/compile.h @@ -22,6 +22,7 @@ PyAPI_FUNC(PyCodeObject *) PyNode_Compile(struct _node *, const char *); #define PyCF_DONT_IMPLY_DEDENT 0x0200 #define PyCF_ONLY_AST 0x0400 #define PyCF_IGNORE_COOKIE 0x0800 +#define PyCF_TYPE_COMMENTS 0x1000 #ifndef Py_LIMITED_API typedef struct { @@ -85,10 +86,10 @@ PyAPI_FUNC(int) _PyAST_Optimize(struct _mod *, PyArena *arena, int optimize); #endif /* !Py_LIMITED_API */ -/* These definitions must match corresponding definitions in graminit.h. - There's code in compile.c that checks that they are the same. */ +/* These definitions must match corresponding definitions in graminit.h. */ #define Py_single_input 256 #define Py_file_input 257 #define Py_eval_input 258 +#define Py_func_type_input 345 #endif /* !Py_COMPILE_H */ diff --git a/Include/graminit.h b/Include/graminit.h index e3acff8a1e83..d1027b7a743f 100644 --- a/Include/graminit.h +++ b/Include/graminit.h @@ -88,3 +88,7 @@ #define encoding_decl 341 #define yield_expr 342 #define yield_arg 343 +#define func_body_suite 344 +#define func_type_input 345 +#define func_type 346 +#define typelist 347 diff --git a/Include/parsetok.h b/Include/parsetok.h index 1217c46d0ed7..e95dd31fb79a 100644 --- a/Include/parsetok.h +++ b/Include/parsetok.h @@ -37,6 +37,7 @@ typedef struct { #define PyPARSE_IGNORE_COOKIE 0x0010 #define PyPARSE_BARRY_AS_BDFL 0x0020 +#define PyPARSE_TYPE_COMMENTS 0x0040 PyAPI_FUNC(node *) PyParser_ParseString(const char *, grammar *, int, perrdetail *); diff --git a/Include/token.h b/Include/token.h index b87b84cd966d..ef6bf969b7c9 100644 --- a/Include/token.h +++ b/Include/token.h @@ -65,8 +65,10 @@ extern "C" { #define ELLIPSIS 52 #define COLONEQUAL 53 #define OP 54 -#define ERRORTOKEN 55 -#define N_TOKENS 59 +#define TYPE_IGNORE 55 +#define TYPE_COMMENT 56 +#define ERRORTOKEN 57 +#define N_TOKENS 61 #define NT_OFFSET 256 /* Special definitions for cooperation with parser */ diff --git a/Lib/ast.py b/Lib/ast.py index 6c1e978b0586..470a74b3b5ff 100644 --- a/Lib/ast.py +++ b/Lib/ast.py @@ -27,12 +27,16 @@ from _ast import * -def parse(source, filename='<unknown>', mode='exec'): +def parse(source, filename='<unknown>', mode='exec', *, type_comments=False): """ Parse the source into an AST node. Equivalent to compile(source, filename, mode, PyCF_ONLY_AST). + Pass type_comments=True to get back type comments where the syntax allows. """ - return compile(source, filename, mode, PyCF_ONLY_AST) + flags = PyCF_ONLY_AST + if type_comments: + flags |= PyCF_TYPE_COMMENTS + return compile(source, filename, mode, flags) def literal_eval(node_or_string): diff --git a/Lib/symbol.py b/Lib/symbol.py index b3fa08984d87..36e0eec7ac1f 100644 --- a/Lib/symbol.py +++ b/Lib/symbol.py @@ -100,6 +100,10 @@ encoding_decl = 341 yield_expr = 342 yield_arg = 343 +func_body_suite = 344 +func_type_input = 345 +func_type = 346 +typelist = 347 #--end constants-- sym_name = {} diff --git a/Lib/test/test_asdl_parser.py b/Lib/test/test_asdl_parser.py index 30e6466dbb32..9eaceecd50db 100644 --- a/Lib/test/test_asdl_parser.py +++ b/Lib/test/test_asdl_parser.py @@ -117,7 +117,8 @@ def visitConstructor(self, cons): v = CustomVisitor() v.visit(self.types['mod']) - self.assertEqual(v.names_with_seq, ['Module', 'Interactive', 'Suite']) + self.assertEqual(v.names_with_seq, + ['Module', 'Module', 'Interactive', 'FunctionType', 'Suite']) if __name__ == '__main__': diff --git a/Lib/test/test_ast.py b/Lib/test/test_ast.py index 09e425de2c30..609c8b2c6a7f 100644 --- a/Lib/test/test_ast.py +++ b/Lib/test/test_ast.py @@ -455,7 +455,7 @@ class N2(ast.Num): def test_module(self): body = [ast.Num(42)] - x = ast.Module(body) + x = ast.Module(body, []) self.assertEqual(x.body, body) def test_nodeclasses(self): @@ -524,13 +524,13 @@ def test_pickling(self): def test_invalid_sum(self): pos = dict(lineno=2, col_offset=3) - m = ast.Module([ast.Expr(ast.expr(**pos), **pos)]) + m = ast.Module([ast.Expr(ast.expr(**pos), **pos)], []) with self.assertRaises(TypeError) as cm: compile(m, "<test>", "exec") self.assertIn("but got <_ast.expr", str(cm.exception)) def test_invalid_identitifer(self): - m = ast.Module([ast.Expr(ast.Name(42, ast.Load()))]) + m = ast.Module([ast.Expr(ast.Name(42, ast.Load()))], []) ast.fix_missing_locations(m) with self.assertRaises(TypeError) as cm: compile(m, "<test>", "exec") @@ -575,11 +575,11 @@ def test_dump(self): self.assertEqual(ast.dump(node), "Module(body=[Expr(value=Call(func=Name(id='spam', ctx=Load()), " "args=[Name(id='eggs', ctx=Load()), Constant(value='and cheese')], " - "keywords=[]))])" + "keywords=[]))], type_ignores=[])" ) self.assertEqual(ast.dump(node, annotate_fields=False), "Module([Expr(Call(Name('spam', Load()), [Name('eggs', Load()), " - "Constant('and cheese')], []))])" + "Constant('and cheese')], []))], [])" ) self.assertEqual(ast.dump(node, include_attributes=True), "Module(body=[Expr(value=Call(func=Name(id='spam', ctx=Load(), " @@ -588,7 +588,7 @@ def test_dump(self): "end_lineno=1, end_col_offset=9), Constant(value='and cheese', " "lineno=1, col_offset=11, end_lineno=1, end_col_offset=23)], keywords=[], " "lineno=1, col_offset=0, end_lineno=1, end_col_offset=24), " - "lineno=1, col_offset=0, end_lineno=1, end_col_offset=24)])" + "lineno=1, col_offset=0, end_lineno=1, end_col_offset=24)], type_ignores=[])" ) def test_copy_location(self): @@ -617,7 +617,8 @@ def test_fix_missing_locations(self): "lineno=1, col_offset=0, end_lineno=1, end_col_offset=0), " "args=[Constant(value='eggs', lineno=1, col_offset=0, end_lineno=1, " "end_col_offset=0)], keywords=[], lineno=1, col_offset=0, end_lineno=1, " - "end_col_offset=0), lineno=1, col_offset=0, end_lineno=1, end_col_offset=0)])" + "end_col_offset=0), lineno=1, col_offset=0, end_lineno=1, end_col_offset=0)], " + "type_ignores=[])" ) def test_increment_lineno(self): @@ -760,7 +761,7 @@ def test_bad_integer(self): names=[ast.alias(name='sleep')], level=None, lineno=None, col_offset=None)] - mod = ast.Module(body) + mod = ast.Module(body, []) with self.assertRaises(ValueError) as cm: compile(mod, 'test', 'exec') self.assertIn("invalid integer value: None", str(cm.exception)) @@ -770,7 +771,7 @@ def test_level_as_none(self): names=[ast.alias(name='sleep')], level=None, lineno=0, col_offset=0)] - mod = ast.Module(body) + mod = ast.Module(body, []) code = compile(mod, 'test', 'exec') ns = {} exec(code, ns) @@ -790,11 +791,11 @@ def mod(self, mod, msg=None, mode="exec", *, exc=ValueError): self.assertIn(msg, str(cm.exception)) def expr(self, node, msg=None, *, exc=ValueError): - mod = ast.Module([ast.Expr(node)]) + mod = ast.Module([ast.Expr(node)], []) self.mod(mod, msg, exc=exc) def stmt(self, stmt, msg=None): - mod = ast.Module([stmt]) + mod = ast.Module([stmt], []) self.mod(mod, msg) def test_module(self): @@ -1603,61 +1604,61 @@ def main(): raise SystemExit unittest.main() -#### EVERYTHING BELOW IS GENERATED ##### +#### EVERYTHING BELOW IS GENERATED BY python Lib/test/test_ast.py -g ##### exec_results = [ -('Module', [('Expr', (1, 0), ('Constant', (1, 0), None))]), -('Module', [('Expr', (1, 0), ('Constant', (1, 0), 'module docstring'))]), -('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [], None, [], [], None, []), [('Pass', (1, 9))], [], None)]), -('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [], None, [], [], None, []), [('Expr', (1, 9), ('Constant', (1, 9), 'function docstring'))], [], None)]), -('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [('arg', (1, 6), 'a', None)], None, [], [], None, []), [('Pass', (1, 10))], [], None)]), -('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [('arg', (1, 6), 'a', None)], None, [], [], None, [('Constant', (1, 8), 0)]), [('Pass', (1, 12))], [], None)]), -('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [], ('arg', (1, 7), 'args', None), [], [], None, []), [('Pass', (1, 14))], [], None)]), -('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [], None, [], [], ('arg', (1, 8), 'kwargs', None), []), [('Pass', (1, 17))], [], None)]), -('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [('arg', (1, 6), 'a', None), ('arg', (1, 9), 'b', None), ('arg', (1, 14), 'c', None), ('arg', (1, 22), 'd', None), ('arg', (1, 28), 'e', None)], ('arg', (1, 35), 'args', None), [('arg', (1, 41), 'f', None)], [('Constant', (1, 43), 42)], ('arg', (1, 49), 'kwargs', None), [('Constant', (1, 11), 1), ('Constant', (1, 16), None), ('List', (1, 24), [], ('Load',)), ('Dict', (1, 30), [], [])]), [('Expr', (1, 58), ('Constant', (1, 58), 'doc for f()'))], [], None)]), -('Module', [('ClassDef', (1, 0), 'C', [], [], [('Pass', (1, 8))], [])]), -('Module', [('ClassDef', (1, 0), 'C', [], [], [('Expr', (1, 9), ('Constant', (1, 9), 'docstring for class C'))], [])]), -('Module', [('ClassDef', (1, 0), 'C', [('Name', (1, 8), 'object', ('Load',))], [], [('Pass', (1, 17))], [])]), -('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [], None, [], [], None, []), [('Return', (1, 8), ('Constant', (1, 15), 1))], [], None)]), -('Module', [('Delete', (1, 0), [('Name', (1, 4), 'v', ('Del',))])]), -('Module', [('Assign', (1, 0), [('Name', (1, 0), 'v', ('Store',))], ('Constant', (1, 4), 1))]), -('Module', [('Assign', (1, 0), [('Tuple', (1, 0), [('Name', (1, 0), 'a', ('Store',)), ('Name', (1, 2), 'b', ('Store',))], ('Store',))], ('Name', (1, 6), 'c', ('Load',)))]), -('Module', [('Assign', (1, 0), [('Tuple', (1, 0), [('Name', (1, 1), 'a', ('Store',)), ('Name', (1, 3), 'b', ('Store',))], ('Store',))], ('Name', (1, 8), 'c', ('Load',)))]), -('Module', [('Assign', (1, 0), [('List', (1, 0), [('Name', (1, 1), 'a', ('Store',)), ('Name', (1, 3), 'b', ('Store',))], ('Store',))], ('Name', (1, 8), 'c', ('Load',)))]), -('Module', [('AugAssign', (1, 0), ('Name', (1, 0), 'v', ('Store',)), ('Add',), ('Constant', (1, 5), 1))]), -('Module', [('For', (1, 0), ('Name', (1, 4), 'v', ('Store',)), ('Name', (1, 9), 'v', ('Load',)), [('Pass', (1, 11))], [])]), -('Module', [('While', (1, 0), ('Name', (1, 6), 'v', ('Load',)), [('Pass', (1, 8))], [])]), -('Module', [('If', (1, 0), ('Name', (1, 3), 'v', ('Load',)), [('Pass', (1, 5))], [])]), -('Module', [('With', (1, 0), [('withitem', ('Name', (1, 5), 'x', ('Load',)), ('Name', (1, 10), 'y', ('Store',)))], [('Pass', (1, 13))])]), -('Module', [('With', (1, 0), [('withitem', ('Name', (1, 5), 'x', ('Load',)), ('Name', (1, 10), 'y', ('Store',))), ('withitem', ('Name', (1, 13), 'z', ('Load',)), ('Name', (1, 18), 'q', ('Store',)))], [('Pass', (1, 21))])]), -('Module', [('Raise', (1, 0), ('Call', (1, 6), ('Name', (1, 6), 'Exception', ('Load',)), [('Constant', (1, 16), 'string')], []), None)]), -('Module', [('Try', (1, 0), [('Pass', (2, 2))], [('ExceptHandler', (3, 0), ('Name', (3, 7), 'Exception', ('Load',)), None, [('Pass', (4, 2))])], [], [])]), -('Module', [('Try', (1, 0), [('Pass', (2, 2))], [], [], [('Pass', (4, 2))])]), -('Module', [('Assert', (1, 0), ('Name', (1, 7), 'v', ('Load',)), None)]), -('Module', [('Import', (1, 0), [('alias', 'sys', None)])]), -('Module', [('ImportFrom', (1, 0), 'sys', [('alias', 'v', None)], 0)]), -('Module', [('Global', (1, 0), ['v'])]), -('Module', [('Expr', (1, 0), ('Constant', (1, 0), 1))]), -('Module', [('Pass', (1, 0))]), -('Module', [('For', (1, 0), ('Name', (1, 4), 'v', ('Store',)), ('Name', (1, 9), 'v', ('Load',)), [('Break', (1, 11))], [])]), -('Module', [('For', (1, 0), ('Name', (1, 4), 'v', ('Store',)), ('Name', (1, 9), 'v', ('Load',)), [('Continue', (1, 11))], [])]), -('Module', [('For', (1, 0), ('Tuple', (1, 4), [('Name', (1, 4), 'a', ('Store',)), ('Name', (1, 6), 'b', ('Store',))], ('Store',)), ('Name', (1, 11), 'c', ('Load',)), [('Pass', (1, 14))], [])]), -('Module', [('For', (1, 0), ('Tuple', (1, 4), [('Name', (1, 5), 'a', ('Store',)), ('Name', (1, 7), 'b', ('Store',))], ('Store',)), ('Name', (1, 13), 'c', ('Load',)), [('Pass', (1, 16))], [])]), -('Module', [('For', (1, 0), ('List', (1, 4), [('Name', (1, 5), 'a', ('Store',)), ('Name', (1, 7), 'b', ('Store',))], ('Store',)), ('Name', (1, 13), 'c', ('Load',)), [('Pass', (1, 16))], [])]), -('Module', [('Expr', (1, 0), ('GeneratorExp', (1, 0), ('Tuple', (2, 4), [('Name', (3, 4), 'Aa', ('Load',)), ('Name', (5, 7), 'Bb', ('Load',))], ('Load',)), [('comprehension', ('Tuple', (8, 4), [('Name', (8, 4), 'Aa', ('Store',)), ('Name', (10, 4), 'Bb', ('Store',))], ('Store',)), ('Name', (10, 10), 'Cc', ('Load',)), [], 0)]))]), -('Module', [('Expr', (1, 0), ('DictComp', (1, 0), ('Name', (1, 1), 'a', ('Load',)), ('Name', (1, 5), 'b', ('Load',)), [('comprehension', ('Name', (1, 11), 'w', ('Store',)), ('Name', (1, 16), 'x', ('Load',)), [], 0), ('comprehension', ('Name', (1, 22), 'm', ('Store',)), ('Name', (1, 27), 'p', ('Load',)), [('Name', (1, 32), 'g', ('Load',))], 0)]))]), -('Module', [('Expr', (1, 0), ('DictComp', (1, 0), ('Name', (1, 1), 'a', ('Load',)), ('Name', (1, 5), 'b', ('Load',)), [('comprehension', ('Tuple', (1, 11), [('Name', (1, 11), 'v', ('Store',)), ('Name', (1, 13), 'w', ('Store',))], ('Store',)), ('Name', (1, 18), 'x', ('Load',)), [], 0)]))]), -('Module', [('Expr', (1, 0), ('SetComp', (1, 0), ('Name', (1, 1), 'r', ('Load',)), [('comprehension', ('Name', (1, 7), 'l', ('Store',)), ('Name', (1, 12), 'x', ('Load',)), [('Name', (1, 17), 'g', ('Load',))], 0)]))]), -('Module', [('Expr', (1, 0), ('SetComp', (1, 0), ('Name', (1, 1), 'r', ('Load',)), [('comprehension', ('Tuple', (1, 7), [('Name', (1, 7), 'l', ('Store',)), ('Name', (1, 9), 'm', ('Store',))], ('Store',)), ('Name', (1, 14), 'x', ('Load',)), [], 0)]))]), -('Module', [('AsyncFunctionDef', (1, 0), 'f', ('arguments', [], None, [], [], None, []), [('Expr', (2, 1), ('Constant', (2, 1), 'async function')), ('Expr', (3, 1), ('Await', (3, 1), ('Call', (3, 7), ('Name', (3, 7), 'something', ('Load',)), [], [])))], [], None)]), -('Module', [('AsyncFunctionDef', (1, 0), 'f', ('arguments', [], None, [], [], None, []), [('AsyncFor', (2, 1), ('Name', (2, 11), 'e', ('Store',)), ('Name', (2, 16), 'i', ('Load',)), [('Expr', (2, 19), ('Constant', (2, 19), 1))], [('Expr', (3, 7), ('Constant', (3, 7), 2))])], [], None)]), -('Module', [('AsyncFunctionDef', (1, 0), 'f', ('arguments', [], None, [], [], None, []), [('AsyncWith', (2, 1), [('withitem', ('Name', (2, 12), 'a', ('Load',)), ('Name', (2, 17), 'b', ('Store',)))], [('Expr', (2, 20), ('Constant', (2, 20), 1))])], [], None)]), -('Module', [('Expr', (1, 0), ('Dict', (1, 0), [None, ('Constant', (1, 10), 2)], [('Dict', (1, 3), [('Constant', (1, 4), 1)], [('Constant', (1, 6), 2)]), ('Constant', (1, 12), 3)]))]), -('Module', [('Expr', (1, 0), ('Set', (1, 0), [('Starred', (1, 1), ('Set', (1, 2), [('Constant', (1, 3), 1), ('Constant', (1, 6), 2)]), ('Load',)), ('Constant', (1, 10), 3)]))]), -('Module', [('AsyncFunctionDef', (1, 0), 'f', ('arguments', [], None, [], [], None, []), [('Expr', (2, 1), ('ListComp', (2, 1), ('Name', (2, 2), 'i', ('Load',)), [('comprehension', ('Name', (2, 14), 'b', ('Store',)), ('Name', (2, 19), 'c', ('Load',)), [], 1)]))], [], None)]), -('Module', [('FunctionDef', (3, 0), 'f', ('arguments', [], None, [], [], None, []), [('Pass', (3, 9))], [('Name', (1, 1), 'deco1', ('Load',)), ('Call', (2, 0), ('Name', (2, 1), 'deco2', ('Load',)), [], [])], None)]), -('Module', [('AsyncFunctionDef', (3, 0), 'f', ('arguments', [], None, [], [], None, []), [('Pass', (3, 15))], [('Name', (1, 1), 'deco1', ('Load',)), ('Call', (2, 0), ('Name', (2, 1), 'deco2', ('Load',)), [], [])], None)]), -('Module', [('ClassDef', (3, 0), 'C', [], [], [('Pass', (3, 9))], [('Name', (1, 1), 'deco1', ('Load',)), ('Call', (2, 0), ('Name', (2, 1), 'deco2', ('Load',)), [], [])])]), -('Module', [('FunctionDef', (2, 0), 'f', ('arguments', [], None, [], [], None, []), [('Pass', (2, 9))], [('Call', (1, 1), ('Name', (1, 1), 'deco', ('Load',)), [('GeneratorExp', (1, 5), ('Name', (1, 6), 'a', ('Load',)), [('comprehension', ('Name', (1, 12), 'a', ('Store',)), ('Name', (1, 17), 'b', ('Load',)), [], 0)])], [])], None)]), +('Module', [('Expr', (1, 0), ('Constant', (1, 0), None))], []), +('Module', [('Expr', (1, 0), ('Constant', (1, 0), 'module docstring'))], []), +('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [], None, [], [], None, []), [('Pass', (1, 9))], [], None, None)], []), +('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [], None, [], [], None, []), [('Expr', (1, 9), ('Constant', (1, 9), 'function docstring'))], [], None, None)], []), +('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [('arg', (1, 6), 'a', None, None)], None, [], [], None, []), [('Pass', (1, 10))], [], None, None)], []), +('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [('arg', (1, 6), 'a', None, None)], None, [], [], None, [('Constant', (1, 8), 0)]), [('Pass', (1, 12))], [], None, None)], []), +('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [], ('arg', (1, 7), 'args', None, None), [], [], None, []), [('Pass', (1, 14))], [], None, None)], []), +('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [], None, [], [], ('arg', (1, 8), 'kwargs', None, None), []), [('Pass', (1, 17))], [], None, None)], []), +('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [('arg', (1, 6), 'a', None, None), ('arg', (1, 9), 'b', None, None), ('arg', (1, 14), 'c', None, None), ('arg', (1, 22), 'd', None, None), ('arg', (1, 28), 'e', None, None)], ('arg', (1, 35), 'args', None, None), [('arg', (1, 41), 'f', None, None)], [('Constant', (1, 43), 42)], ('arg', (1, 49), 'kwargs', None, None), [('Constant', (1, 11), 1), ('Constant', (1, 16), None), ('List', (1, 24), [], ('Load',)), ('Dict', (1, 30), [], [])]), [('Expr', (1, 58), ('Constant', (1, 58), 'doc for f()'))], [], None, None)], []), +('Module', [('ClassDef', (1, 0), 'C', [], [], [('Pass', (1, 8))], [])], []), +('Module', [('ClassDef', (1, 0), 'C', [], [], [('Expr', (1, 9), ('Constant', (1, 9), 'docstring for class C'))], [])], []), +('Module', [('ClassDef', (1, 0), 'C', [('Name', (1, 8), 'object', ('Load',))], [], [('Pass', (1, 17))], [])], []), +('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [], None, [], [], None, []), [('Return', (1, 8), ('Constant', (1, 15), 1))], [], None, None)], []), +('Module', [('Delete', (1, 0), [('Name', (1, 4), 'v', ('Del',))])], []), +('Module', [('Assign', (1, 0), [('Name', (1, 0), 'v', ('Store',))], ('Constant', (1, 4), 1), None)], []), +('Module', [('Assign', (1, 0), [('Tuple', (1, 0), [('Name', (1, 0), 'a', ('Store',)), ('Name', (1, 2), 'b', ('Store',))], ('Store',))], ('Name', (1, 6), 'c', ('Load',)), None)], []), +('Module', [('Assign', (1, 0), [('Tuple', (1, 0), [('Name', (1, 1), 'a', ('Store',)), ('Name', (1, 3), 'b', ('Store',))], ('Store',))], ('Name', (1, 8), 'c', ('Load',)), None)], []), +('Module', [('Assign', (1, 0), [('List', (1, 0), [('Name', (1, 1), 'a', ('Store',)), ('Name', (1, 3), 'b', ('Store',))], ('Store',))], ('Name', (1, 8), 'c', ('Load',)), None)], []), +('Module', [('AugAssign', (1, 0), ('Name', (1, 0), 'v', ('Store',)), ('Add',), ('Constant', (1, 5), 1))], []), +('Module', [('For', (1, 0), ('Name', (1, 4), 'v', ('Store',)), ('Name', (1, 9), 'v', ('Load',)), [('Pass', (1, 11))], [], None)], []), +('Module', [('While', (1, 0), ('Name', (1, 6), 'v', ('Load',)), [('Pass', (1, 8))], [])], []), +('Module', [('If', (1, 0), ('Name', (1, 3), 'v', ('Load',)), [('Pass', (1, 5))], [])], []), +('Module', [('With', (1, 0), [('withitem', ('Name', (1, 5), 'x', ('Load',)), ('Name', (1, 10), 'y', ('Store',)))], [('Pass', (1, 13))], None)], []), +('Module', [('With', (1, 0), [('withitem', ('Name', (1, 5), 'x', ('Load',)), ('Name', (1, 10), 'y', ('Store',))), ('withitem', ('Name', (1, 13), 'z', ('Load',)), ('Name', (1, 18), 'q', ('Store',)))], [('Pass', (1, 21))], None)], []), +('Module', [('Raise', (1, 0), ('Call', (1, 6), ('Name', (1, 6), 'Exception', ('Load',)), [('Constant', (1, 16), 'string')], []), None)], []), +('Module', [('Try', (1, 0), [('Pass', (2, 2))], [('ExceptHandler', (3, 0), ('Name', (3, 7), 'Exception', ('Load',)), None, [('Pass', (4, 2))])], [], [])], []), +('Module', [('Try', (1, 0), [('Pass', (2, 2))], [], [], [('Pass', (4, 2))])], []), +('Module', [('Assert', (1, 0), ('Name', (1, 7), 'v', ('Load',)), None)], []), +('Module', [('Import', (1, 0), [('alias', 'sys', None)])], []), +('Module', [('ImportFrom', (1, 0), 'sys', [('alias', 'v', None)], 0)], []), +('Module', [('Global', (1, 0), ['v'])], []), +('Module', [('Expr', (1, 0), ('Constant', (1, 0), 1))], []), +('Module', [('Pass', (1, 0))], []), +('Module', [('For', (1, 0), ('Name', (1, 4), 'v', ('Store',)), ('Name', (1, 9), 'v', ('Load',)), [('Break', (1, 11))], [], None)], []), +('Module', [('For', (1, 0), ('Name', (1, 4), 'v', ('Store',)), ('Name', (1, 9), 'v', ('Load',)), [('Continue', (1, 11))], [], None)], []), +('Module', [('For', (1, 0), ('Tuple', (1, 4), [('Name', (1, 4), 'a', ('Store',)), ('Name', (1, 6), 'b', ('Store',))], ('Store',)), ('Name', (1, 11), 'c', ('Load',)), [('Pass', (1, 14))], [], None)], []), +('Module', [('For', (1, 0), ('Tuple', (1, 4), [('Name', (1, 5), 'a', ('Store',)), ('Name', (1, 7), 'b', ('Store',))], ('Store',)), ('Name', (1, 13), 'c', ('Load',)), [('Pass', (1, 16))], [], None)], []), +('Module', [('For', (1, 0), ('List', (1, 4), [('Name', (1, 5), 'a', ('Store',)), ('Name', (1, 7), 'b', ('Store',))], ('Store',)), ('Name', (1, 13), 'c', ('Load',)), [('Pass', (1, 16))], [], None)], []), +('Module', [('Expr', (1, 0), ('GeneratorExp', (1, 0), ('Tuple', (2, 4), [('Name', (3, 4), 'Aa', ('Load',)), ('Name', (5, 7), 'Bb', ('Load',))], ('Load',)), [('comprehension', ('Tuple', (8, 4), [('Name', (8, 4), 'Aa', ('Store',)), ('Name', (10, 4), 'Bb', ('Store',))], ('Store',)), ('Name', (10, 10), 'Cc', ('Load',)), [], 0)]))], []), +('Module', [('Expr', (1, 0), ('DictComp', (1, 0), ('Name', (1, 1), 'a', ('Load',)), ('Name', (1, 5), 'b', ('Load',)), [('comprehension', ('Name', (1, 11), 'w', ('Store',)), ('Name', (1, 16), 'x', ('Load',)), [], 0), ('comprehension', ('Name', (1, 22), 'm', ('Store',)), ('Name', (1, 27), 'p', ('Load',)), [('Name', (1, 32), 'g', ('Load',))], 0)]))], []), +('Module', [('Expr', (1, 0), ('DictComp', (1, 0), ('Name', (1, 1), 'a', ('Load',)), ('Name', (1, 5), 'b', ('Load',)), [('comprehension', ('Tuple', (1, 11), [('Name', (1, 11), 'v', ('Store',)), ('Name', (1, 13), 'w', ('Store',))], ('Store',)), ('Name', (1, 18), 'x', ('Load',)), [], 0)]))], []), +('Module', [('Expr', (1, 0), ('SetComp', (1, 0), ('Name', (1, 1), 'r', ('Load',)), [('comprehension', ('Name', (1, 7), 'l', ('Store',)), ('Name', (1, 12), 'x', ('Load',)), [('Name', (1, 17), 'g', ('Load',))], 0)]))], []), +('Module', [('Expr', (1, 0), ('SetComp', (1, 0), ('Name', (1, 1), 'r', ('Load',)), [('comprehension', ('Tuple', (1, 7), [('Name', (1, 7), 'l', ('Store',)), ('Name', (1, 9), 'm', ('Store',))], ('Store',)), ('Name', (1, 14), 'x', ('Load',)), [], 0)]))], []), +('Module', [('AsyncFunctionDef', (1, 0), 'f', ('arguments', [], None, [], [], None, []), [('Expr', (2, 1), ('Constant', (2, 1), 'async function')), ('Expr', (3, 1), ('Await', (3, 1), ('Call', (3, 7), ('Name', (3, 7), 'something', ('Load',)), [], [])))], [], None, None)], []), +('Module', [('AsyncFunctionDef', (1, 0), 'f', ('arguments', [], None, [], [], None, []), [('AsyncFor', (2, 1), ('Name', (2, 11), 'e', ('Store',)), ('Name', (2, 16), 'i', ('Load',)), [('Expr', (2, 19), ('Constant', (2, 19), 1))], [('Expr', (3, 7), ('Constant', (3, 7), 2))], None)], [], None, None)], []), +('Module', [('AsyncFunctionDef', (1, 0), 'f', ('arguments', [], None, [], [], None, []), [('AsyncWith', (2, 1), [('withitem', ('Name', (2, 12), 'a', ('Load',)), ('Name', (2, 17), 'b', ('Store',)))], [('Expr', (2, 20), ('Constant', (2, 20), 1))], None)], [], None, None)], []), +('Module', [('Expr', (1, 0), ('Dict', (1, 0), [None, ('Constant', (1, 10), 2)], [('Dict', (1, 3), [('Constant', (1, 4), 1)], [('Constant', (1, 6), 2)]), ('Constant', (1, 12), 3)]))], []), +('Module', [('Expr', (1, 0), ('Set', (1, 0), [('Starred', (1, 1), ('Set', (1, 2), [('Constant', (1, 3), 1), ('Constant', (1, 6), 2)]), ('Load',)), ('Constant', (1, 10), 3)]))], []), +('Module', [('AsyncFunctionDef', (1, 0), 'f', ('arguments', [], None, [], [], None, []), [('Expr', (2, 1), ('ListComp', (2, 1), ('Name', (2, 2), 'i', ('Load',)), [('comprehension', ('Name', (2, 14), 'b', ('Store',)), ('Name', (2, 19), 'c', ('Load',)), [], 1)]))], [], None, None)], []), +('Module', [('FunctionDef', (3, 0), 'f', ('arguments', [], None, [], [], None, []), [('Pass', (3, 9))], [('Name', (1, 1), 'deco1', ('Load',)), ('Call', (2, 0), ('Name', (2, 1), 'deco2', ('Load',)), [], [])], None, None)], []), +('Module', [('AsyncFunctionDef', (3, 0), 'f', ('arguments', [], None, [], [], None, []), [('Pass', (3, 15))], [('Name', (1, 1), 'deco1', ('Load',)), ('Call', (2, 0), ('Name', (2, 1), 'deco2', ('Load',)), [], [])], None, None)], []), +('Module', [('ClassDef', (3, 0), 'C', [], [], [('Pass', (3, 9))], [('Name', (1, 1), 'deco1', ('Load',)), ('Call', (2, 0), ('Name', (2, 1), 'deco2', ('Load',)), [], [])])], []), +('Module', [('FunctionDef', (2, 0), 'f', ('arguments', [], None, [], [], None, []), [('Pass', (2, 9))], [('Call', (1, 1), ('Name', (1, 1), 'deco', ('Load',)), [('GeneratorExp', (1, 5), ('Name', (1, 6), 'a', ('Load',)), [('comprehension', ('Name', (1, 12), 'a', ('Store',)), ('Name', (1, 17), 'b', ('Load',)), [], 0)])], [])], None, None)], []), ] single_results = [ ('Interactive', [('Expr', (1, 0), ('BinOp', (1, 0), ('Constant', (1, 0), 1), ('Add',), ('Constant', (1, 2), 2)))]), diff --git a/Lib/test/test_type_comments.py b/Lib/test/test_type_comments.py new file mode 100644 index 000000000000..3065ddca2d9a --- /dev/null +++ b/Lib/test/test_type_comments.py @@ -0,0 +1,295 @@ +import ast +import unittest + + +funcdef = """\ +def foo(): + # type: () -> int + pass + +def bar(): # type: () -> None + pass +""" + +asyncdef = """\ +async def foo(): + # type: () -> int + return await bar() + +async def bar(): # type: () -> int + return await bar() +""" + +redundantdef = """\ +def foo(): # type: () -> int + # type: () -> str + return '' +""" + +nonasciidef = """\ +def foo(): + # type: () -> àçčéñt + pass +""" + +forstmt = """\ +for a in []: # type: int + pass +""" + +withstmt = """\ +with context() as a: # type: int + pass +""" + +vardecl = """\ +a = 0 # type: int +""" + +ignores = """\ +def foo(): + pass # type: ignore + +def bar(): + x = 1 # type: ignore +""" + +# Test for long-form type-comments in arguments. A test function +# named 'fabvk' would have two positional args, a and b, plus a +# var-arg *v, plus a kw-arg **k. It is verified in test_longargs() +# that it has exactly these arguments, no more, no fewer. +longargs = """\ +def fa( + a = 1, # type: A +): + pass + +def fa( + a = 1 # type: A +): + pass + +def fab( + a, # type: A + b, # type: B +): + pass + +def fab( + a, # type: A + b # type: B +): + pass + +def fv( + *v, # type: V +): + pass + +def fv( + *v # type: V +): + pass + +def fk( + **k, # type: K +): + pass + +def fk( + **k # type: K +): + pass + +def fvk( + *v, # type: V + **k, # type: K +): + pass + +def fvk( + *v, # type: V + **k # type: K +): + pass + +def fav( + a, # type: A + *v, # type: V +): + pass + +def fav( + a, # type: A + *v # type: V +): + pass + +def fak( + a, # type: A + **k, # type: K +): + pass + +def fak( + a, # type: A + **k # type: K +): + pass + +def favk( + a, # type: A + *v, # type: V + **k, # type: K +): + pass + +def favk( + a, # type: A + *v, # type: V + **k # type: K +): + pass +""" + + +class TypeCommentTests(unittest.TestCase): + + def parse(self, source): + return ast.parse(source, type_comments=True) + + def classic_parse(self, source): + return ast.parse(source) + + def test_funcdef(self): + tree = self.parse(funcdef) + self.assertEqual(tree.body[0].type_comment, "() -> int") + self.assertEqual(tree.body[1].type_comment, "() -> None") + tree = self.classic_parse(funcdef) + self.assertEqual(tree.body[0].type_comment, None) + self.assertEqual(tree.body[1].type_comment, None) + + def test_asyncdef(self): + tree = self.parse(asyncdef) + self.assertEqual(tree.body[0].type_comment, "() -> int") + self.assertEqual(tree.body[1].type_comment, "() -> int") + tree = self.classic_parse(asyncdef) + self.assertEqual(tree.body[0].type_comment, None) + self.assertEqual(tree.body[1].type_comment, None) + + def test_redundantdef(self): + with self.assertRaisesRegex(SyntaxError, "^Cannot have two type comments on def"): + tree = self.parse(redundantdef) + + def test_nonasciidef(self): + tree = self.parse(nonasciidef) + self.assertEqual(tree.body[0].type_comment, "() -> àçčéñt") + + def test_forstmt(self): + tree = self.parse(forstmt) + self.assertEqual(tree.body[0].type_comment, "int") + tree = self.classic_parse(forstmt) + self.assertEqual(tree.body[0].type_comment, None) + + def test_withstmt(self): + tree = self.parse(withstmt) + self.assertEqual(tree.body[0].type_comment, "int") + tree = self.classic_parse(withstmt) + self.assertEqual(tree.body[0].type_comment, None) + + def test_vardecl(self): + tree = self.parse(vardecl) + self.assertEqual(tree.body[0].type_comment, "int") + tree = self.classic_parse(vardecl) + self.assertEqual(tree.body[0].type_comment, None) + + def test_ignores(self): + tree = self.parse(ignores) + self.assertEqual([ti.lineno for ti in tree.type_ignores], [2, 5]) + tree = self.classic_parse(ignores) + self.assertEqual(tree.type_ignores, []) + + def test_longargs(self): + tree = self.parse(longargs) + for t in tree.body: + # The expected args are encoded in the function name + todo = set(t.name[1:]) + self.assertEqual(len(t.args.args), + len(todo) - bool(t.args.vararg) - bool(t.args.kwarg)) + self.assertTrue(t.name.startswith('f'), t.name) + for c in t.name[1:]: + todo.remove(c) + if c == 'v': + arg = t.args.vararg + elif c == 'k': + arg = t.args.kwarg + else: + assert 0 <= ord(c) - ord('a') < len(t.args.args) + arg = t.args.args[ord(c) - ord('a')] + self.assertEqual(arg.arg, c) # That's the argument name + self.assertEqual(arg.type_comment, arg.arg.upper()) + assert not todo + tree = self.classic_parse(longargs) + for t in tree.body: + for arg in t.args.args + [t.args.vararg, t.args.kwarg]: + if arg is not None: + self.assertIsNone(arg.type_comment, "%s(%s:%r)" % + (t.name, arg.arg, arg.type_comment)) + + def test_inappropriate_type_comments(self): + """Tests for inappropriately-placed type comments. + + These should be silently ignored with type comments off, + but raise SyntaxError with type comments on. + + This is not meant to be exhaustive. + """ + + def check_both_ways(source): + ast.parse(source, type_comments=False) + with self.assertRaises(SyntaxError): + ast.parse(source, type_comments=True) + + check_both_ways("pass # type: int\n") + check_both_ways("foo() # type: int\n") + check_both_ways("x += 1 # type: int\n") + check_both_ways("while True: # type: int\n continue\n") + check_both_ways("while True:\n continue # type: int\n") + check_both_ways("try: # type: int\n pass\nfinally:\n pass\n") + check_both_ways("try:\n pass\nfinally: # type: int\n pass\n") + + def test_func_type_input(self): + + def parse_func_type_input(source): + return ast.parse(source, "<unknown>", "func_type") + + # Some checks below will crash if the returned structure is wrong + tree = parse_func_type_input("() -> int") + self.assertEqual(tree.argtypes, []) + self.assertEqual(tree.returns.id, "int") + + tree = parse_func_type_input("(int) -> List[str]") + self.assertEqual(len(tree.argtypes), 1) + arg = tree.argtypes[0] + self.assertEqual(arg.id, "int") + self.assertEqual(tree.returns.value.id, "List") + self.assertEqual(tree.returns.slice.value.id, "str") + + tree = parse_func_type_input("(int, *str, **Any) -> float") + self.assertEqual(tree.argtypes[0].id, "int") + self.assertEqual(tree.argtypes[1].id, "str") + self.assertEqual(tree.argtypes[2].id, "Any") + self.assertEqual(tree.returns.id, "float") + + with self.assertRaises(SyntaxError): + tree = parse_func_type_input("(int, *str, *Any) -> float") + + with self.assertRaises(SyntaxError): + tree = parse_func_type_input("(int, **str, Any) -> float") + + with self.assertRaises(SyntaxError): + tree = parse_func_type_input("(**int, **str) -> float") + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/token.py b/Lib/token.py index 7224eca32fe0..9bf80a5950a2 100644 --- a/Lib/token.py +++ b/Lib/token.py @@ -58,12 +58,14 @@ ELLIPSIS = 52 COLONEQUAL = 53 OP = 54 +TYPE_IGNORE = 55 +TYPE_COMMENT = 56 # These aren't used by the C tokenizer but are needed for tokenize.py -ERRORTOKEN = 55 -COMMENT = 56 -NL = 57 -ENCODING = 58 -N_TOKENS = 59 +ERRORTOKEN = 57 +COMMENT = 58 +NL = 59 +ENCODING = 60 +N_TOKENS = 61 # Special definitions for cooperation with parser NT_OFFSET = 256 diff --git a/Misc/NEWS.d/next/Core and Builtins/2019-01-22-19-17-27.bpo-35766.gh1tHZ.rst b/Misc/NEWS.d/next/Core and Builtins/2019-01-22-19-17-27.bpo-35766.gh1tHZ.rst new file mode 100644 index 000000000000..29c5f34d6a3e --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2019-01-22-19-17-27.bpo-35766.gh1tHZ.rst @@ -0,0 +1 @@ +Add the option to parse PEP 484 type comments in the ast module. (Off by default.) This is merging the key functionality of the third party fork thereof, [typed_ast](https://github.com/python/typed_ast). \ No newline at end of file diff --git a/Modules/parsermodule.c b/Modules/parsermodule.c index eabc5c810f6c..87f58d340c2d 100644 --- a/Modules/parsermodule.c +++ b/Modules/parsermodule.c @@ -663,6 +663,12 @@ validate_node(node *tree) for (pos = 0; pos < nch; ++pos) { node *ch = CHILD(tree, pos); int ch_type = TYPE(ch); + if (ch_type == suite && TYPE(tree) == funcdef) { + /* This is the opposite hack of what we do in parser.c + (search for func_body_suite), except we don't ever + support type comments here. */ + ch_type = func_body_suite; + } for (arc = 0; arc < dfa_state->s_narcs; ++arc) { short a_label = dfa_state->s_arc[arc].a_lbl; assert(a_label < _PyParser_Grammar.g_ll.ll_nlabels); diff --git a/Parser/Python.asdl b/Parser/Python.asdl index 7b2a8737ab80..85b686d78a3b 100644 --- a/Parser/Python.asdl +++ b/Parser/Python.asdl @@ -3,17 +3,20 @@ module Python { - mod = Module(stmt* body) + mod = Module(stmt* body, type_ignore *type_ignores) | Interactive(stmt* body) | Expression(expr body) + | FunctionType(expr* argtypes, expr returns) -- not really an actual node but useful in Jython's typesystem. | Suite(stmt* body) stmt = FunctionDef(identifier name, arguments args, - stmt* body, expr* decorator_list, expr? returns) + stmt* body, expr* decorator_list, expr? returns, + string? type_comment) | AsyncFunctionDef(identifier name, arguments args, - stmt* body, expr* decorator_list, expr? returns) + stmt* body, expr* decorator_list, expr? returns, + string? type_comment) | ClassDef(identifier name, expr* bases, @@ -23,18 +26,18 @@ module Python | Return(expr? value) | Delete(expr* targets) - | Assign(expr* targets, expr value) + | Assign(expr* targets, expr value, string? type_comment) | AugAssign(expr target, operator op, expr value) -- 'simple' indicates that we annotate simple name without parens | AnnAssign(expr target, expr annotation, expr? value, int simple) -- use 'orelse' because else is a keyword in target languages - | For(expr target, expr iter, stmt* body, stmt* orelse) - | AsyncFor(expr target, expr iter, stmt* body, stmt* orelse) + | For(expr target, expr iter, stmt* body, stmt* orelse, string? type_comment) + | AsyncFor(expr target, expr iter, stmt* body, stmt* orelse, string? type_comment) | While(expr test, stmt* body, stmt* orelse) | If(expr test, stmt* body, stmt* orelse) - | With(withitem* items, stmt* body) - | AsyncWith(withitem* items, stmt* body) + | With(withitem* items, stmt* body, string? type_comment) + | AsyncWith(withitem* items, stmt* body, string? type_comment) | Raise(expr? exc, expr? cause) | Try(stmt* body, excepthandler* handlers, stmt* orelse, stmt* finalbody) @@ -111,7 +114,7 @@ module Python arguments = (arg* args, arg? vararg, arg* kwonlyargs, expr* kw_defaults, arg? kwarg, expr* defaults) - arg = (identifier arg, expr? annotation) + arg = (identifier arg, expr? annotation, string? type_comment) attributes (int lineno, int col_offset, int? end_lineno, int? end_col_offset) -- keyword arguments supplied to call (NULL identifier for **kwargs) @@ -121,5 +124,7 @@ module Python alias = (identifier name, identifier? asname) withitem = (expr context_expr, expr? optional_vars) + + type_ignore = TypeIgnore(int lineno) } diff --git a/Parser/asdl_c.py b/Parser/asdl_c.py index 8640b29b8f10..a51a5db73904 100644 --- a/Parser/asdl_c.py +++ b/Parser/asdl_c.py @@ -890,6 +890,15 @@ def visitModule(self, mod): return obj2ast_object(obj, out, arena); } +static int obj2ast_string(PyObject* obj, PyObject** out, PyArena* arena) +{ + if (!PyUnicode_CheckExact(obj) && !PyBytes_CheckExact(obj)) { + PyErr_SetString(PyExc_TypeError, "AST string must be of type str"); + return 1; + } + return obj2ast_object(obj, out, arena); +} + static int obj2ast_int(PyObject* obj, int* out, PyArena* arena) { int i; @@ -993,6 +1002,8 @@ def visitModule(self, mod): self.emit('if (PyDict_SetItemString(d, "AST", (PyObject*)&AST_type) < 0) return NULL;', 1) self.emit('if (PyModule_AddIntMacro(m, PyCF_ONLY_AST) < 0)', 1) self.emit("return NULL;", 2) + self.emit('if (PyModule_AddIntMacro(m, PyCF_TYPE_COMMENTS) < 0)', 1) + self.emit("return NULL;", 2) for dfn in mod.dfns: self.visit(dfn) self.emit("return m;", 1) @@ -1176,18 +1187,19 @@ class PartingShots(StaticVisitor): } /* mode is 0 for "exec", 1 for "eval" and 2 for "single" input */ +/* and 3 for "func_type" */ mod_ty PyAST_obj2mod(PyObject* ast, PyArena* arena, int mode) { mod_ty res; PyObject *req_type[3]; - char *req_name[] = {"Module", "Expression", "Interactive"}; + char *req_name[] = {"Module", "Expression", "Interactive", "FunctionType"}; int isinstance; req_type[0] = (PyObject*)Module_type; req_type[1] = (PyObject*)Expression_type; req_type[2] = (PyObject*)Interactive_type; - assert(0 <= mode && mode <= 2); + assert(0 <= mode && mode <= 3); if (!init_types()) return NULL; diff --git a/Parser/parser.c b/Parser/parser.c index a9916d392aab..fa4a8f011ff5 100644 --- a/Parser/parser.c +++ b/Parser/parser.c @@ -12,6 +12,7 @@ #include "node.h" #include "parser.h" #include "errcode.h" +#include "graminit.h" #ifdef Py_DEBUG @@ -260,7 +261,15 @@ PyParser_AddToken(parser_state *ps, int type, char *str, /* Push non-terminal */ int nt = (x >> 8) + NT_OFFSET; int arrow = x & ((1<<7)-1); - dfa *d1 = PyGrammar_FindDFA( + dfa *d1; + if (nt == func_body_suite && !(ps->p_flags & PyCF_TYPE_COMMENTS)) { + /* When parsing type comments is not requested, + we can provide better errors about bad indentation + by using 'suite' for the body of a funcdef */ + D(printf(" [switch func_body_suite to suite]")); + nt = suite; + } + d1 = PyGrammar_FindDFA( ps->p_grammar, nt); if ((err = push(&ps->p_stack, nt, d1, arrow, lineno, col_offset, @@ -268,7 +277,7 @@ PyParser_AddToken(parser_state *ps, int type, char *str, D(printf(" MemError: push\n")); return err; } - D(printf(" Push ...\n")); + D(printf(" Push '%s'\n", d1->d_name)); continue; } diff --git a/Parser/parsetok.c b/Parser/parsetok.c index 2b5254a8be67..7fddc5a0e897 100644 --- a/Parser/parsetok.c +++ b/Parser/parsetok.c @@ -15,6 +15,42 @@ static node *parsetok(struct tok_state *, grammar *, int, perrdetail *, int *); static int initerr(perrdetail *err_ret, PyObject * filename); +typedef struct { + int *items; + size_t size; + size_t num_items; +} growable_int_array; + +static int +growable_int_array_init(growable_int_array *arr, size_t initial_size) { + assert(initial_size > 0); + arr->items = malloc(initial_size * sizeof(*arr->items)); + arr->size = initial_size; + arr->num_items = 0; + + return arr->items != NULL; +} + +static int +growable_int_array_add(growable_int_array *arr, int item) { + if (arr->num_items >= arr->size) { + arr->size *= 2; + arr->items = realloc(arr->items, arr->size * sizeof(*arr->items)); + if (!arr->items) { + return 0; + } + } + + arr->items[arr->num_items] = item; + arr->num_items++; + return 1; +} + +static void +growable_int_array_deallocate(growable_int_array *arr) { + free(arr->items); +} + /* Parse input coming from a string. Return error code, print some errors. */ node * PyParser_ParseString(const char *s, grammar *g, int start, perrdetail *err_ret) @@ -59,6 +95,9 @@ PyParser_ParseStringObject(const char *s, PyObject *filename, err_ret->error = PyErr_Occurred() ? E_DECODE : E_NOMEM; return NULL; } + if (*flags & PyPARSE_TYPE_COMMENTS) { + tok->type_comments = 1; + } #ifndef PGEN Py_INCREF(err_ret->filename); @@ -127,6 +166,9 @@ PyParser_ParseFileObject(FILE *fp, PyObject *filename, err_ret->error = E_NOMEM; return NULL; } + if (*flags & PyPARSE_TYPE_COMMENTS) { + tok->type_comments = 1; + } #ifndef PGEN Py_INCREF(err_ret->filename); tok->filename = err_ret->filename; @@ -188,6 +230,13 @@ parsetok(struct tok_state *tok, grammar *g, int start, perrdetail *err_ret, node *n; int started = 0; int col_offset, end_col_offset; + growable_int_array type_ignores; + + if (!growable_int_array_init(&type_ignores, 10)) { + err_ret->error = E_NOMEM; + PyTokenizer_Free(tok); + return NULL; + } if ((ps = PyParser_New(g, start)) == NULL) { err_ret->error = E_NOMEM; @@ -197,6 +246,8 @@ parsetok(struct tok_state *tok, grammar *g, int start, perrdetail *err_ret, #ifdef PY_PARSER_REQUIRES_FUTURE_KEYWORD if (*flags & PyPARSE_BARRY_AS_BDFL) ps->p_flags |= CO_FUTURE_BARRY_AS_BDFL; + if (*flags & PyPARSE_TYPE_COMMENTS) + ps->p_flags |= PyCF_TYPE_COMMENTS; #endif for (;;) { @@ -277,6 +328,15 @@ parsetok(struct tok_state *tok, grammar *g, int start, perrdetail *err_ret, else { end_col_offset = -1; } + + if (type == TYPE_IGNORE) { + if (!growable_int_array_add(&type_ignores, tok->lineno)) { + err_ret->error = E_NOMEM; + break; + } + continue; + } + if ((err_ret->error = PyParser_AddToken(ps, (int)type, str, lineno, col_offset, tok->lineno, end_col_offset, @@ -293,6 +353,24 @@ parsetok(struct tok_state *tok, grammar *g, int start, perrdetail *err_ret, n = ps->p_tree; ps->p_tree = NULL; + if (n->n_type == file_input) { + /* Put type_ignore nodes in the ENDMARKER of file_input. */ + int num; + node *ch; + size_t i; + + num = NCH(n); + ch = CHILD(n, num - 1); + REQ(ch, ENDMARKER); + + for (i = 0; i < type_ignores.num_items; i++) { + PyNode_AddChild(ch, TYPE_IGNORE, NULL, + type_ignores.items[i], 0, + type_ignores.items[i], 0); + } + } + growable_int_array_deallocate(&type_ignores); + #ifndef PGEN /* Check that the source for a single input statement really is a single statement by looking at what is left in the diff --git a/Parser/token.c b/Parser/token.c index d27f98a34d55..228ecffc8415 100644 --- a/Parser/token.c +++ b/Parser/token.c @@ -61,6 +61,8 @@ const char * const _PyParser_TokenNames[] = { "ELLIPSIS", "COLONEQUAL", "OP", + "TYPE_IGNORE", + "TYPE_COMMENT", "<ERRORTOKEN>", "<COMMENT>", "<NL>", diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c index 3e3cf2cd7f58..1ded9ade3771 100644 --- a/Parser/tokenizer.c +++ b/Parser/tokenizer.c @@ -48,6 +48,10 @@ static int tok_nextc(struct tok_state *tok); static void tok_backup(struct tok_state *tok, int c); +/* Spaces in this constant are treated as "zero or more spaces or tabs" when + tokenizing. */ +static const char* type_comment_prefix = "# type: "; + /* Create and initialize a new tok_state structure */ static struct tok_state * @@ -82,6 +86,7 @@ tok_new(void) tok->decoding_readline = NULL; tok->decoding_buffer = NULL; #endif + tok->type_comments = 0; return tok; } @@ -1245,11 +1250,61 @@ tok_get(struct tok_state *tok, char **p_start, char **p_end) /* Set start of current token */ tok->start = tok->cur - 1; - /* Skip comment */ + /* Skip comment, unless it's a type comment */ if (c == '#') { + const char *prefix, *p, *type_start; + while (c != EOF && c != '\n') { c = tok_nextc(tok); } + + if (tok->type_comments) { + p = tok->start; + prefix = type_comment_prefix; + while (*prefix && p < tok->cur) { + if (*prefix == ' ') { + while (*p == ' ' || *p == '\t') { + p++; + } + } else if (*prefix == *p) { + p++; + } else { + break; + } + + prefix++; + } + + /* This is a type comment if we matched all of type_comment_prefix. */ + if (!*prefix) { + int is_type_ignore = 1; + tok_backup(tok, c); /* don't eat the newline or EOF */ + + type_start = p; + + is_type_ignore = tok->cur >= p + 6 && memcmp(p, "ignore", 6) == 0; + p += 6; + while (is_type_ignore && p < tok->cur) { + if (*p == '#') + break; + is_type_ignore = is_type_ignore && (*p == ' ' || *p == '\t'); + p++; + } + + if (is_type_ignore) { + /* If this type ignore is the only thing on the line, consume the newline also. */ + if (blankline) { + tok_nextc(tok); + tok->atbol = 1; + } + return TYPE_IGNORE; + } else { + *p_start = (char *) type_start; /* after type_comment_prefix */ + *p_end = tok->cur; + return TYPE_COMMENT; + } + } + } } /* Check for EOF and errors now */ diff --git a/Parser/tokenizer.h b/Parser/tokenizer.h index 096ce687ec54..9639c658b1c2 100644 --- a/Parser/tokenizer.h +++ b/Parser/tokenizer.h @@ -70,6 +70,8 @@ struct tok_state { const char* enc; /* Encoding for the current str. */ const char* str; const char* input; /* Tokenizer's newline translated copy of the string. */ + + int type_comments; /* Whether to look for type comments */ }; extern struct tok_state *PyTokenizer_FromString(const char *, int); diff --git a/Python/Python-ast.c b/Python/Python-ast.c index a333ff95b110..1a56e90bca09 100644 --- a/Python/Python-ast.c +++ b/Python/Python-ast.c @@ -10,8 +10,10 @@ static PyTypeObject *mod_type; static PyObject* ast2obj_mod(void*); static PyTypeObject *Module_type; _Py_IDENTIFIER(body); +_Py_IDENTIFIER(type_ignores); static char *Module_fields[]={ "body", + "type_ignores", }; static PyTypeObject *Interactive_type; static char *Interactive_fields[]={ @@ -21,6 +23,13 @@ static PyTypeObject *Expression_type; static char *Expression_fields[]={ "body", }; +static PyTypeObject *FunctionType_type; +_Py_IDENTIFIER(argtypes); +_Py_IDENTIFIER(returns); +static char *FunctionType_fields[]={ + "argtypes", + "returns", +}; static PyTypeObject *Suite_type; static char *Suite_fields[]={ "body", @@ -41,13 +50,14 @@ static PyTypeObject *FunctionDef_type; _Py_IDENTIFIER(name); _Py_IDENTIFIER(args); _Py_IDENTIFIER(decorator_list); -_Py_IDENTIFIER(returns); +_Py_IDENTIFIER(type_comment); static char *FunctionDef_fields[]={ "name", "args", "body", "decorator_list", "returns", + "type_comment", }; static PyTypeObject *AsyncFunctionDef_type; static char *AsyncFunctionDef_fields[]={ @@ -56,6 +66,7 @@ static char *AsyncFunctionDef_fields[]={ "body", "decorator_list", "returns", + "type_comment", }; static PyTypeObject *ClassDef_type; _Py_IDENTIFIER(bases); @@ -81,6 +92,7 @@ static PyTypeObject *Assign_type; static char *Assign_fields[]={ "targets", "value", + "type_comment", }; static PyTypeObject *AugAssign_type; _Py_IDENTIFIER(target); @@ -107,6 +119,7 @@ static char *For_fields[]={ "iter", "body", "orelse", + "type_comment", }; static PyTypeObject *AsyncFor_type; static char *AsyncFor_fields[]={ @@ -114,6 +127,7 @@ static char *AsyncFor_fields[]={ "iter", "body", "orelse", + "type_comment", }; static PyTypeObject *While_type; _Py_IDENTIFIER(test); @@ -133,11 +147,13 @@ _Py_IDENTIFIER(items); static char *With_fields[]={ "items", "body", + "type_comment", }; static PyTypeObject *AsyncWith_type; static char *AsyncWith_fields[]={ "items", "body", + "type_comment", }; static PyTypeObject *Raise_type; _Py_IDENTIFIER(exc); @@ -478,6 +494,7 @@ _Py_IDENTIFIER(arg); static char *arg_fields[]={ "arg", "annotation", + "type_comment", }; static PyTypeObject *keyword_type; static PyObject* ast2obj_keyword(void*); @@ -500,6 +517,12 @@ static char *withitem_fields[]={ "context_expr", "optional_vars", }; +static PyTypeObject *type_ignore_type; +static PyObject* ast2obj_type_ignore(void*); +static PyTypeObject *TypeIgnore_type; +static char *TypeIgnore_fields[]={ + "lineno", +}; _Py_IDENTIFIER(_fields); @@ -769,6 +792,15 @@ static int obj2ast_identifier(PyObject* obj, PyObject** out, PyArena* arena) return obj2ast_object(obj, out, arena); } +static int obj2ast_string(PyObject* obj, PyObject** out, PyArena* arena) +{ + if (!PyUnicode_CheckExact(obj) && !PyBytes_CheckExact(obj)) { + PyErr_SetString(PyExc_TypeError, "AST string must be of type str"); + return 1; + } + return obj2ast_object(obj, out, arena); +} + static int obj2ast_int(PyObject* obj, int* out, PyArena* arena) { int i; @@ -810,23 +842,26 @@ static int init_types(void) mod_type = make_type("mod", &AST_type, NULL, 0); if (!mod_type) return 0; if (!add_attributes(mod_type, NULL, 0)) return 0; - Module_type = make_type("Module", mod_type, Module_fields, 1); + Module_type = make_type("Module", mod_type, Module_fields, 2); if (!Module_type) return 0; Interactive_type = make_type("Interactive", mod_type, Interactive_fields, 1); if (!Interactive_type) return 0; Expression_type = make_type("Expression", mod_type, Expression_fields, 1); if (!Expression_type) return 0; + FunctionType_type = make_type("FunctionType", mod_type, + FunctionType_fields, 2); + if (!FunctionType_type) return 0; Suite_type = make_type("Suite", mod_type, Suite_fields, 1); if (!Suite_type) return 0; stmt_type = make_type("stmt", &AST_type, NULL, 0); if (!stmt_type) return 0; if (!add_attributes(stmt_type, stmt_attributes, 4)) return 0; FunctionDef_type = make_type("FunctionDef", stmt_type, FunctionDef_fields, - 5); + 6); if (!FunctionDef_type) return 0; AsyncFunctionDef_type = make_type("AsyncFunctionDef", stmt_type, - AsyncFunctionDef_fields, 5); + AsyncFunctionDef_fields, 6); if (!AsyncFunctionDef_type) return 0; ClassDef_type = make_type("ClassDef", stmt_type, ClassDef_fields, 5); if (!ClassDef_type) return 0; @@ -834,23 +869,23 @@ static int init_types(void) if (!Return_type) return 0; Delete_type = make_type("Delete", stmt_type, Delete_fields, 1); if (!Delete_type) return 0; - Assign_type = make_type("Assign", stmt_type, Assign_fields, 2); + Assign_type = make_type("Assign", stmt_type, Assign_fields, 3); if (!Assign_type) return 0; AugAssign_type = make_type("AugAssign", stmt_type, AugAssign_fields, 3); if (!AugAssign_type) return 0; AnnAssign_type = make_type("AnnAssign", stmt_type, AnnAssign_fields, 4); if (!AnnAssign_type) return 0; - For_type = make_type("For", stmt_type, For_fields, 4); + For_type = make_type("For", stmt_type, For_fields, 5); if (!For_type) return 0; - AsyncFor_type = make_type("AsyncFor", stmt_type, AsyncFor_fields, 4); + AsyncFor_type = make_type("AsyncFor", stmt_type, AsyncFor_fields, 5); if (!AsyncFor_type) return 0; While_type = make_type("While", stmt_type, While_fields, 3); if (!While_type) return 0; If_type = make_type("If", stmt_type, If_fields, 3); if (!If_type) return 0; - With_type = make_type("With", stmt_type, With_fields, 2); + With_type = make_type("With", stmt_type, With_fields, 3); if (!With_type) return 0; - AsyncWith_type = make_type("AsyncWith", stmt_type, AsyncWith_fields, 2); + AsyncWith_type = make_type("AsyncWith", stmt_type, AsyncWith_fields, 3); if (!AsyncWith_type) return 0; Raise_type = make_type("Raise", stmt_type, Raise_fields, 2); if (!Raise_type) return 0; @@ -1113,7 +1148,7 @@ static int init_types(void) arguments_type = make_type("arguments", &AST_type, arguments_fields, 6); if (!arguments_type) return 0; if (!add_attributes(arguments_type, NULL, 0)) return 0; - arg_type = make_type("arg", &AST_type, arg_fields, 2); + arg_type = make_type("arg", &AST_type, arg_fields, 3); if (!arg_type) return 0; if (!add_attributes(arg_type, arg_attributes, 4)) return 0; keyword_type = make_type("keyword", &AST_type, keyword_fields, 2); @@ -1125,6 +1160,12 @@ static int init_types(void) withitem_type = make_type("withitem", &AST_type, withitem_fields, 2); if (!withitem_type) return 0; if (!add_attributes(withitem_type, NULL, 0)) return 0; + type_ignore_type = make_type("type_ignore", &AST_type, NULL, 0); + if (!type_ignore_type) return 0; + if (!add_attributes(type_ignore_type, NULL, 0)) return 0; + TypeIgnore_type = make_type("TypeIgnore", type_ignore_type, + TypeIgnore_fields, 1); + if (!TypeIgnore_type) return 0; initialized = 1; return 1; } @@ -1148,9 +1189,11 @@ static int obj2ast_arg(PyObject* obj, arg_ty* out, PyArena* arena); static int obj2ast_keyword(PyObject* obj, keyword_ty* out, PyArena* arena); static int obj2ast_alias(PyObject* obj, alias_ty* out, PyArena* arena); static int obj2ast_withitem(PyObject* obj, withitem_ty* out, PyArena* arena); +static int obj2ast_type_ignore(PyObject* obj, type_ignore_ty* out, PyArena* + arena); mod_ty -Module(asdl_seq * body, PyArena *arena) +Module(asdl_seq * body, asdl_seq * type_ignores, PyArena *arena) { mod_ty p; p = (mod_ty)PyArena_Malloc(arena, sizeof(*p)); @@ -1158,6 +1201,7 @@ Module(asdl_seq * body, PyArena *arena) return NULL; p->kind = Module_kind; p->v.Module.body = body; + p->v.Module.type_ignores = type_ignores; return p; } @@ -1190,6 +1234,24 @@ Expression(expr_ty body, PyArena *arena) return p; } +mod_ty +FunctionType(asdl_seq * argtypes, expr_ty returns, PyArena *arena) +{ + mod_ty p; + if (!returns) { + PyErr_SetString(PyExc_ValueError, + "field returns is required for FunctionType"); + return NULL; + } + p = (mod_ty)PyArena_Malloc(arena, sizeof(*p)); + if (!p) + return NULL; + p->kind = FunctionType_kind; + p->v.FunctionType.argtypes = argtypes; + p->v.FunctionType.returns = returns; + return p; +} + mod_ty Suite(asdl_seq * body, PyArena *arena) { @@ -1204,8 +1266,8 @@ Suite(asdl_seq * body, PyArena *arena) stmt_ty FunctionDef(identifier name, arguments_ty args, asdl_seq * body, asdl_seq * - decorator_list, expr_ty returns, int lineno, int col_offset, int - end_lineno, int end_col_offset, PyArena *arena) + decorator_list, expr_ty returns, string type_comment, int lineno, + int col_offset, int end_lineno, int end_col_offset, PyArena *arena) { stmt_ty p; if (!name) { @@ -1227,6 +1289,7 @@ FunctionDef(identifier name, arguments_ty args, asdl_seq * body, asdl_seq * p->v.FunctionDef.body = body; p->v.FunctionDef.decorator_list = decorator_list; p->v.FunctionDef.returns = returns; + p->v.FunctionDef.type_comment = type_comment; p->lineno = lineno; p->col_offset = col_offset; p->end_lineno = end_lineno; @@ -1236,8 +1299,9 @@ FunctionDef(identifier name, arguments_ty args, asdl_seq * body, asdl_seq * stmt_ty AsyncFunctionDef(identifier name, arguments_ty args, asdl_seq * body, asdl_seq - * decorator_list, expr_ty returns, int lineno, int col_offset, - int end_lineno, int end_col_offset, PyArena *arena) + * decorator_list, expr_ty returns, string type_comment, int + lineno, int col_offset, int end_lineno, int end_col_offset, + PyArena *arena) { stmt_ty p; if (!name) { @@ -1259,6 +1323,7 @@ AsyncFunctionDef(identifier name, arguments_ty args, asdl_seq * body, asdl_seq p->v.AsyncFunctionDef.body = body; p->v.AsyncFunctionDef.decorator_list = decorator_list; p->v.AsyncFunctionDef.returns = returns; + p->v.AsyncFunctionDef.type_comment = type_comment; p->lineno = lineno; p->col_offset = col_offset; p->end_lineno = end_lineno; @@ -1328,8 +1393,8 @@ Delete(asdl_seq * targets, int lineno, int col_offset, int end_lineno, int } stmt_ty -Assign(asdl_seq * targets, expr_ty value, int lineno, int col_offset, int - end_lineno, int end_col_offset, PyArena *arena) +Assign(asdl_seq * targets, expr_ty value, string type_comment, int lineno, int + col_offset, int end_lineno, int end_col_offset, PyArena *arena) { stmt_ty p; if (!value) { @@ -1343,6 +1408,7 @@ Assign(asdl_seq * targets, expr_ty value, int lineno, int col_offset, int p->kind = Assign_kind; p->v.Assign.targets = targets; p->v.Assign.value = value; + p->v.Assign.type_comment = type_comment; p->lineno = lineno; p->col_offset = col_offset; p->end_lineno = end_lineno; @@ -1416,8 +1482,9 @@ AnnAssign(expr_ty target, expr_ty annotation, expr_ty value, int simple, int } stmt_ty -For(expr_ty target, expr_ty iter, asdl_seq * body, asdl_seq * orelse, int - lineno, int col_offset, int end_lineno, int end_col_offset, PyArena *arena) +For(expr_ty target, expr_ty iter, asdl_seq * body, asdl_seq * orelse, string + type_comment, int lineno, int col_offset, int end_lineno, int + end_col_offset, PyArena *arena) { stmt_ty p; if (!target) { @@ -1438,6 +1505,7 @@ For(expr_ty target, expr_ty iter, asdl_seq * body, asdl_seq * orelse, int p->v.For.iter = iter; p->v.For.body = body; p->v.For.orelse = orelse; + p->v.For.type_comment = type_comment; p->lineno = lineno; p->col_offset = col_offset; p->end_lineno = end_lineno; @@ -1446,9 +1514,9 @@ For(expr_ty target, expr_ty iter, asdl_seq * body, asdl_seq * orelse, int } stmt_ty -AsyncFor(expr_ty target, expr_ty iter, asdl_seq * body, asdl_seq * orelse, int - lineno, int col_offset, int end_lineno, int end_col_offset, PyArena - *arena) +AsyncFor(expr_ty target, expr_ty iter, asdl_seq * body, asdl_seq * orelse, + string type_comment, int lineno, int col_offset, int end_lineno, int + end_col_offset, PyArena *arena) { stmt_ty p; if (!target) { @@ -1469,6 +1537,7 @@ AsyncFor(expr_ty target, expr_ty iter, asdl_seq * body, asdl_seq * orelse, int p->v.AsyncFor.iter = iter; p->v.AsyncFor.body = body; p->v.AsyncFor.orelse = orelse; + p->v.AsyncFor.type_comment = type_comment; p->lineno = lineno; p->col_offset = col_offset; p->end_lineno = end_lineno; @@ -1525,8 +1594,8 @@ If(expr_ty test, asdl_seq * body, asdl_seq * orelse, int lineno, int } stmt_ty -With(asdl_seq * items, asdl_seq * body, int lineno, int col_offset, int - end_lineno, int end_col_offset, PyArena *arena) +With(asdl_seq * items, asdl_seq * body, string type_comment, int lineno, int + col_offset, int end_lineno, int end_col_offset, PyArena *arena) { stmt_ty p; p = (stmt_ty)PyArena_Malloc(arena, sizeof(*p)); @@ -1535,6 +1604,7 @@ With(asdl_seq * items, asdl_seq * body, int lineno, int col_offset, int p->kind = With_kind; p->v.With.items = items; p->v.With.body = body; + p->v.With.type_comment = type_comment; p->lineno = lineno; p->col_offset = col_offset; p->end_lineno = end_lineno; @@ -1543,8 +1613,8 @@ With(asdl_seq * items, asdl_seq * body, int lineno, int col_offset, int } stmt_ty -AsyncWith(asdl_seq * items, asdl_seq * body, int lineno, int col_offset, int - end_lineno, int end_col_offset, PyArena *arena) +AsyncWith(asdl_seq * items, asdl_seq * body, string type_comment, int lineno, + int col_offset, int end_lineno, int end_col_offset, PyArena *arena) { stmt_ty p; p = (stmt_ty)PyArena_Malloc(arena, sizeof(*p)); @@ -1553,6 +1623,7 @@ AsyncWith(asdl_seq * items, asdl_seq * body, int lineno, int col_offset, int p->kind = AsyncWith_kind; p->v.AsyncWith.items = items; p->v.AsyncWith.body = body; + p->v.AsyncWith.type_comment = type_comment; p->lineno = lineno; p->col_offset = col_offset; p->end_lineno = end_lineno; @@ -2518,8 +2589,8 @@ arguments(asdl_seq * args, arg_ty vararg, asdl_seq * kwonlyargs, asdl_seq * } arg_ty -arg(identifier arg, expr_ty annotation, int lineno, int col_offset, int - end_lineno, int end_col_offset, PyArena *arena) +arg(identifier arg, expr_ty annotation, string type_comment, int lineno, int + col_offset, int end_lineno, int end_col_offset, PyArena *arena) { arg_ty p; if (!arg) { @@ -2532,6 +2603,7 @@ arg(identifier arg, expr_ty annotation, int lineno, int col_offset, int return NULL; p->arg = arg; p->annotation = annotation; + p->type_comment = type_comment; p->lineno = lineno; p->col_offset = col_offset; p->end_lineno = end_lineno; @@ -2590,6 +2662,18 @@ withitem(expr_ty context_expr, expr_ty optional_vars, PyArena *arena) return p; } +type_ignore_ty +TypeIgnore(int lineno, PyArena *arena) +{ + type_ignore_ty p; + p = (type_ignore_ty)PyArena_Malloc(arena, sizeof(*p)); + if (!p) + return NULL; + p->kind = TypeIgnore_kind; + p->v.TypeIgnore.lineno = lineno; + return p; +} + PyObject* ast2obj_mod(void* _o) @@ -2609,6 +2693,11 @@ ast2obj_mod(void* _o) if (_PyObject_SetAttrId(result, &PyId_body, value) == -1) goto failed; Py_DECREF(value); + value = ast2obj_list(o->v.Module.type_ignores, ast2obj_type_ignore); + if (!value) goto failed; + if (_PyObject_SetAttrId(result, &PyId_type_ignores, value) == -1) + goto failed; + Py_DECREF(value); break; case Interactive_kind: result = PyType_GenericNew(Interactive_type, NULL, NULL); @@ -2628,6 +2717,20 @@ ast2obj_mod(void* _o) goto failed; Py_DECREF(value); break; + case FunctionType_kind: + result = PyType_GenericNew(FunctionType_type, NULL, NULL); + if (!result) goto failed; + value = ast2obj_list(o->v.FunctionType.argtypes, ast2obj_expr); + if (!value) goto failed; + if (_PyObject_SetAttrId(result, &PyId_argtypes, value) == -1) + goto failed; + Py_DECREF(value); + value = ast2obj_expr(o->v.FunctionType.returns); + if (!value) goto failed; + if (_PyObject_SetAttrId(result, &PyId_returns, value) == -1) + goto failed; + Py_DECREF(value); + break; case Suite_kind: result = PyType_GenericNew(Suite_type, NULL, NULL); if (!result) goto failed; @@ -2683,6 +2786,11 @@ ast2obj_stmt(void* _o) if (_PyObject_SetAttrId(result, &PyId_returns, value) == -1) goto failed; Py_DECREF(value); + value = ast2obj_string(o->v.FunctionDef.type_comment); + if (!value) goto failed; + if (_PyObject_SetAttrId(result, &PyId_type_comment, value) == -1) + goto failed; + Py_DECREF(value); break; case AsyncFunctionDef_kind: result = PyType_GenericNew(AsyncFunctionDef_type, NULL, NULL); @@ -2713,6 +2821,11 @@ ast2obj_stmt(void* _o) if (_PyObject_SetAttrId(result, &PyId_returns, value) == -1) goto failed; Py_DECREF(value); + value = ast2obj_string(o->v.AsyncFunctionDef.type_comment); + if (!value) goto failed; + if (_PyObject_SetAttrId(result, &PyId_type_comment, value) == -1) + goto failed; + Py_DECREF(value); break; case ClassDef_kind: result = PyType_GenericNew(ClassDef_type, NULL, NULL); @@ -2774,6 +2887,11 @@ ast2obj_stmt(void* _o) if (_PyObject_SetAttrId(result, &PyId_value, value) == -1) goto failed; Py_DECREF(value); + value = ast2obj_string(o->v.Assign.type_comment); + if (!value) goto failed; + if (_PyObject_SetAttrId(result, &PyId_type_comment, value) == -1) + goto failed; + Py_DECREF(value); break; case AugAssign_kind: result = PyType_GenericNew(AugAssign_type, NULL, NULL); @@ -2841,6 +2959,11 @@ ast2obj_stmt(void* _o) if (_PyObject_SetAttrId(result, &PyId_orelse, value) == -1) goto failed; Py_DECREF(value); + value = ast2obj_string(o->v.For.type_comment); + if (!value) goto failed; + if (_PyObject_SetAttrId(result, &PyId_type_comment, value) == -1) + goto failed; + Py_DECREF(value); break; case AsyncFor_kind: result = PyType_GenericNew(AsyncFor_type, NULL, NULL); @@ -2865,6 +2988,11 @@ ast2obj_stmt(void* _o) if (_PyObject_SetAttrId(result, &PyId_orelse, value) == -1) goto failed; Py_DECREF(value); + value = ast2obj_string(o->v.AsyncFor.type_comment); + if (!value) goto failed; + if (_PyObject_SetAttrId(result, &PyId_type_comment, value) == -1) + goto failed; + Py_DECREF(value); break; case While_kind: result = PyType_GenericNew(While_type, NULL, NULL); @@ -2917,6 +3045,11 @@ ast2obj_stmt(void* _o) if (_PyObject_SetAttrId(result, &PyId_body, value) == -1) goto failed; Py_DECREF(value); + value = ast2obj_string(o->v.With.type_comment); + if (!value) goto failed; + if (_PyObject_SetAttrId(result, &PyId_type_comment, value) == -1) + goto failed; + Py_DECREF(value); break; case AsyncWith_kind: result = PyType_GenericNew(AsyncWith_type, NULL, NULL); @@ -2931,6 +3064,11 @@ ast2obj_stmt(void* _o) if (_PyObject_SetAttrId(result, &PyId_body, value) == -1) goto failed; Py_DECREF(value); + value = ast2obj_string(o->v.AsyncWith.type_comment); + if (!value) goto failed; + if (_PyObject_SetAttrId(result, &PyId_type_comment, value) == -1) + goto failed; + Py_DECREF(value); break; case Raise_kind: result = PyType_GenericNew(Raise_type, NULL, NULL); @@ -3870,6 +4008,11 @@ ast2obj_arg(void* _o) if (_PyObject_SetAttrId(result, &PyId_annotation, value) == -1) goto failed; Py_DECREF(value); + value = ast2obj_string(o->type_comment); + if (!value) goto failed; + if (_PyObject_SetAttrId(result, &PyId_type_comment, value) == -1) + goto failed; + Py_DECREF(value); value = ast2obj_int(o->lineno); if (!value) goto failed; if (_PyObject_SetAttrId(result, &PyId_lineno, value) < 0) @@ -3981,6 +4124,33 @@ ast2obj_withitem(void* _o) return NULL; } +PyObject* +ast2obj_type_ignore(void* _o) +{ + type_ignore_ty o = (type_ignore_ty)_o; + PyObject *result = NULL, *value = NULL; + if (!o) { + Py_RETURN_NONE; + } + + switch (o->kind) { + case TypeIgnore_kind: + result = PyType_GenericNew(TypeIgnore_type, NULL, NULL); + if (!result) goto failed; + value = ast2obj_int(o->v.TypeIgnore.lineno); + if (!value) goto failed; + if (_PyObject_SetAttrId(result, &PyId_lineno, value) == -1) + goto failed; + Py_DECREF(value); + break; + } + return result; +failed: + Py_XDECREF(value); + Py_XDECREF(result); + return NULL; +} + int obj2ast_mod(PyObject* obj, mod_ty* out, PyArena* arena) @@ -3999,6 +4169,7 @@ obj2ast_mod(PyObject* obj, mod_ty* out, PyArena* arena) } if (isinstance) { asdl_seq* body; + asdl_seq* type_ignores; if (_PyObject_LookupAttrId(obj, &PyId_body, &tmp) < 0) { return 1; @@ -4030,7 +4201,37 @@ obj2ast_mod(PyObject* obj, mod_ty* out, PyArena* arena) } Py_CLEAR(tmp); } - *out = Module(body, arena); + if (_PyObject_LookupAttrId(obj, &PyId_type_ignores, &tmp) < 0) { + return 1; + } + if (tmp == NULL) { + PyErr_SetString(PyExc_TypeError, "required field \"type_ignores\" missing from Module"); + return 1; + } + else { + int res; + Py_ssize_t len; + Py_ssize_t i; + if (!PyList_Check(tmp)) { + PyErr_Format(PyExc_TypeError, "Module field \"type_ignores\" must be a list, not a %.200s", tmp->ob_type->tp_name); + goto failed; + } + len = PyList_GET_SIZE(tmp); + type_ignores = _Py_asdl_seq_new(len, arena); + if (type_ignores == NULL) goto failed; + for (i = 0; i < len; i++) { + type_ignore_ty val; + res = obj2ast_type_ignore(PyList_GET_ITEM(tmp, i), &val, arena); + if (res != 0) goto failed; + if (len != PyList_GET_SIZE(tmp)) { + PyErr_SetString(PyExc_RuntimeError, "Module field \"type_ignores\" changed size during iteration"); + goto failed; + } + asdl_seq_SET(type_ignores, i, val); + } + Py_CLEAR(tmp); + } + *out = Module(body, type_ignores, arena); if (*out == NULL) goto failed; return 0; } @@ -4099,6 +4300,61 @@ obj2ast_mod(PyObject* obj, mod_ty* out, PyArena* arena) if (*out == NULL) goto failed; return 0; } + isinstance = PyObject_IsInstance(obj, (PyObject*)FunctionType_type); + if (isinstance == -1) { + return 1; + } + if (isinstance) { + asdl_seq* argtypes; + expr_ty returns; + + if (_PyObject_LookupAttrId(obj, &PyId_argtypes, &tmp) < 0) { + return 1; + } + if (tmp == NULL) { + PyErr_SetString(PyExc_TypeError, "required field \"argtypes\" missing from FunctionType"); + return 1; + } + else { + int res; + Py_ssize_t len; + Py_ssize_t i; + if (!PyList_Check(tmp)) { + PyErr_Format(PyExc_TypeError, "FunctionType field \"argtypes\" must be a list, not a %.200s", tmp->ob_type->tp_name); + goto failed; + } + len = PyList_GET_SIZE(tmp); + argtypes = _Py_asdl_seq_new(len, arena); + if (argtypes == NULL) goto failed; + for (i = 0; i < len; i++) { + expr_ty val; + res = obj2ast_expr(PyList_GET_ITEM(tmp, i), &val, arena); + if (res != 0) goto failed; + if (len != PyList_GET_SIZE(tmp)) { + PyErr_SetString(PyExc_RuntimeError, "FunctionType field \"argtypes\" changed size during iteration"); + goto failed; + } + asdl_seq_SET(argtypes, i, val); + } + Py_CLEAR(tmp); + } + if (_PyObject_LookupAttrId(obj, &PyId_returns, &tmp) < 0) { + return 1; + } + if (tmp == NULL) { + PyErr_SetString(PyExc_TypeError, "required field \"returns\" missing from FunctionType"); + return 1; + } + else { + int res; + res = obj2ast_expr(tmp, &returns, arena); + if (res != 0) goto failed; + Py_CLEAR(tmp); + } + *out = FunctionType(argtypes, returns, arena); + if (*out == NULL) goto failed; + return 0; + } isinstance = PyObject_IsInstance(obj, (PyObject*)Suite_type); if (isinstance == -1) { return 1; @@ -4224,6 +4480,7 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) asdl_seq* body; asdl_seq* decorator_list; expr_ty returns; + string type_comment; if (_PyObject_LookupAttrId(obj, &PyId_name, &tmp) < 0) { return 1; @@ -4324,8 +4581,22 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) if (res != 0) goto failed; Py_CLEAR(tmp); } - *out = FunctionDef(name, args, body, decorator_list, returns, lineno, - col_offset, end_lineno, end_col_offset, arena); + if (_PyObject_LookupAttrId(obj, &PyId_type_comment, &tmp) < 0) { + return 1; + } + if (tmp == NULL || tmp == Py_None) { + Py_CLEAR(tmp); + type_comment = NULL; + } + else { + int res; + res = obj2ast_string(tmp, &type_comment, arena); + if (res != 0) goto failed; + Py_CLEAR(tmp); + } + *out = FunctionDef(name, args, body, decorator_list, returns, + type_comment, lineno, col_offset, end_lineno, + end_col_offset, arena); if (*out == NULL) goto failed; return 0; } @@ -4339,6 +4610,7 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) asdl_seq* body; asdl_seq* decorator_list; expr_ty returns; + string type_comment; if (_PyObject_LookupAttrId(obj, &PyId_name, &tmp) < 0) { return 1; @@ -4439,9 +4711,22 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) if (res != 0) goto failed; Py_CLEAR(tmp); } + if (_PyObject_LookupAttrId(obj, &PyId_type_comment, &tmp) < 0) { + return 1; + } + if (tmp == NULL || tmp == Py_None) { + Py_CLEAR(tmp); + type_comment = NULL; + } + else { + int res; + res = obj2ast_string(tmp, &type_comment, arena); + if (res != 0) goto failed; + Py_CLEAR(tmp); + } *out = AsyncFunctionDef(name, args, body, decorator_list, returns, - lineno, col_offset, end_lineno, end_col_offset, - arena); + type_comment, lineno, col_offset, end_lineno, + end_col_offset, arena); if (*out == NULL) goto failed; return 0; } @@ -4668,6 +4953,7 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) if (isinstance) { asdl_seq* targets; expr_ty value; + string type_comment; if (_PyObject_LookupAttrId(obj, &PyId_targets, &tmp) < 0) { return 1; @@ -4712,8 +4998,21 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) if (res != 0) goto failed; Py_CLEAR(tmp); } - *out = Assign(targets, value, lineno, col_offset, end_lineno, - end_col_offset, arena); + if (_PyObject_LookupAttrId(obj, &PyId_type_comment, &tmp) < 0) { + return 1; + } + if (tmp == NULL || tmp == Py_None) { + Py_CLEAR(tmp); + type_comment = NULL; + } + else { + int res; + res = obj2ast_string(tmp, &type_comment, arena); + if (res != 0) goto failed; + Py_CLEAR(tmp); + } + *out = Assign(targets, value, type_comment, lineno, col_offset, + end_lineno, end_col_offset, arena); if (*out == NULL) goto failed; return 0; } @@ -4846,6 +5145,7 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) expr_ty iter; asdl_seq* body; asdl_seq* orelse; + string type_comment; if (_PyObject_LookupAttrId(obj, &PyId_target, &tmp) < 0) { return 1; @@ -4933,8 +5233,21 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) } Py_CLEAR(tmp); } - *out = For(target, iter, body, orelse, lineno, col_offset, end_lineno, - end_col_offset, arena); + if (_PyObject_LookupAttrId(obj, &PyId_type_comment, &tmp) < 0) { + return 1; + } + if (tmp == NULL || tmp == Py_None) { + Py_CLEAR(tmp); + type_comment = NULL; + } + else { + int res; + res = obj2ast_string(tmp, &type_comment, arena); + if (res != 0) goto failed; + Py_CLEAR(tmp); + } + *out = For(target, iter, body, orelse, type_comment, lineno, + col_offset, end_lineno, end_col_offset, arena); if (*out == NULL) goto failed; return 0; } @@ -4947,6 +5260,7 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) expr_ty iter; asdl_seq* body; asdl_seq* orelse; + string type_comment; if (_PyObject_LookupAttrId(obj, &PyId_target, &tmp) < 0) { return 1; @@ -5034,8 +5348,21 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) } Py_CLEAR(tmp); } - *out = AsyncFor(target, iter, body, orelse, lineno, col_offset, - end_lineno, end_col_offset, arena); + if (_PyObject_LookupAttrId(obj, &PyId_type_comment, &tmp) < 0) { + return 1; + } + if (tmp == NULL || tmp == Py_None) { + Py_CLEAR(tmp); + type_comment = NULL; + } + else { + int res; + res = obj2ast_string(tmp, &type_comment, arena); + if (res != 0) goto failed; + Py_CLEAR(tmp); + } + *out = AsyncFor(target, iter, body, orelse, type_comment, lineno, + col_offset, end_lineno, end_col_offset, arena); if (*out == NULL) goto failed; return 0; } @@ -5220,6 +5547,7 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) if (isinstance) { asdl_seq* items; asdl_seq* body; + string type_comment; if (_PyObject_LookupAttrId(obj, &PyId_items, &tmp) < 0) { return 1; @@ -5281,7 +5609,20 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) } Py_CLEAR(tmp); } - *out = With(items, body, lineno, col_offset, end_lineno, + if (_PyObject_LookupAttrId(obj, &PyId_type_comment, &tmp) < 0) { + return 1; + } + if (tmp == NULL || tmp == Py_None) { + Py_CLEAR(tmp); + type_comment = NULL; + } + else { + int res; + res = obj2ast_string(tmp, &type_comment, arena); + if (res != 0) goto failed; + Py_CLEAR(tmp); + } + *out = With(items, body, type_comment, lineno, col_offset, end_lineno, end_col_offset, arena); if (*out == NULL) goto failed; return 0; @@ -5293,6 +5634,7 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) if (isinstance) { asdl_seq* items; asdl_seq* body; + string type_comment; if (_PyObject_LookupAttrId(obj, &PyId_items, &tmp) < 0) { return 1; @@ -5354,8 +5696,21 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) } Py_CLEAR(tmp); } - *out = AsyncWith(items, body, lineno, col_offset, end_lineno, - end_col_offset, arena); + if (_PyObject_LookupAttrId(obj, &PyId_type_comment, &tmp) < 0) { + return 1; + } + if (tmp == NULL || tmp == Py_None) { + Py_CLEAR(tmp); + type_comment = NULL; + } + else { + int res; + res = obj2ast_string(tmp, &type_comment, arena); + if (res != 0) goto failed; + Py_CLEAR(tmp); + } + *out = AsyncWith(items, body, type_comment, lineno, col_offset, + end_lineno, end_col_offset, arena); if (*out == NULL) goto failed; return 0; } @@ -8073,6 +8428,7 @@ obj2ast_arg(PyObject* obj, arg_ty* out, PyArena* arena) PyObject* tmp = NULL; identifier arg; expr_ty annotation; + string type_comment; int lineno; int col_offset; int end_lineno; @@ -8104,6 +8460,19 @@ obj2ast_arg(PyObject* obj, arg_ty* out, PyArena* arena) if (res != 0) goto failed; Py_CLEAR(tmp); } + if (_PyObject_LookupAttrId(obj, &PyId_type_comment, &tmp) < 0) { + return 1; + } + if (tmp == NULL || tmp == Py_None) { + Py_CLEAR(tmp); + type_comment = NULL; + } + else { + int res; + res = obj2ast_string(tmp, &type_comment, arena); + if (res != 0) goto failed; + Py_CLEAR(tmp); + } if (_PyObject_LookupAttrId(obj, &PyId_lineno, &tmp) < 0) { return 1; } @@ -8156,8 +8525,8 @@ obj2ast_arg(PyObject* obj, arg_ty* out, PyArena* arena) if (res != 0) goto failed; Py_CLEAR(tmp); } - *out = arg(arg, annotation, lineno, col_offset, end_lineno, end_col_offset, - arena); + *out = arg(arg, annotation, type_comment, lineno, col_offset, end_lineno, + end_col_offset, arena); return 0; failed: Py_XDECREF(tmp); @@ -8284,6 +8653,48 @@ obj2ast_withitem(PyObject* obj, withitem_ty* out, PyArena* arena) return 1; } +int +obj2ast_type_ignore(PyObject* obj, type_ignore_ty* out, PyArena* arena) +{ + int isinstance; + + PyObject *tmp = NULL; + + if (obj == Py_None) { + *out = NULL; + return 0; + } + isinstance = PyObject_IsInstance(obj, (PyObject*)TypeIgnore_type); + if (isinstance == -1) { + return 1; + } + if (isinstance) { + int lineno; + + if (_PyObject_LookupAttrId(obj, &PyId_lineno, &tmp) < 0) { + return 1; + } + if (tmp == NULL) { + PyErr_SetString(PyExc_TypeError, "required field \"lineno\" missing from TypeIgnore"); + return 1; + } + else { + int res; + res = obj2ast_int(tmp, &lineno, arena); + if (res != 0) goto failed; + Py_CLEAR(tmp); + } + *out = TypeIgnore(lineno, arena); + if (*out == NULL) goto failed; + return 0; + } + + PyErr_Format(PyExc_TypeError, "expected some sort of type_ignore, but got %R", obj); + failed: + Py_XDECREF(tmp); + return 1; +} + static struct PyModuleDef _astmodule = { PyModuleDef_HEAD_INIT, "_ast" @@ -8299,6 +8710,8 @@ PyInit__ast(void) if (PyDict_SetItemString(d, "AST", (PyObject*)&AST_type) < 0) return NULL; if (PyModule_AddIntMacro(m, PyCF_ONLY_AST) < 0) return NULL; + if (PyModule_AddIntMacro(m, PyCF_TYPE_COMMENTS) < 0) + return NULL; if (PyDict_SetItemString(d, "mod", (PyObject*)mod_type) < 0) return NULL; if (PyDict_SetItemString(d, "Module", (PyObject*)Module_type) < 0) return NULL; @@ -8306,6 +8719,8 @@ PyInit__ast(void) 0) return NULL; if (PyDict_SetItemString(d, "Expression", (PyObject*)Expression_type) < 0) return NULL; + if (PyDict_SetItemString(d, "FunctionType", (PyObject*)FunctionType_type) < + 0) return NULL; if (PyDict_SetItemString(d, "Suite", (PyObject*)Suite_type) < 0) return NULL; if (PyDict_SetItemString(d, "stmt", (PyObject*)stmt_type) < 0) return NULL; @@ -8486,6 +8901,10 @@ PyInit__ast(void) NULL; if (PyDict_SetItemString(d, "withitem", (PyObject*)withitem_type) < 0) return NULL; + if (PyDict_SetItemString(d, "type_ignore", (PyObject*)type_ignore_type) < + 0) return NULL; + if (PyDict_SetItemString(d, "TypeIgnore", (PyObject*)TypeIgnore_type) < 0) + return NULL; return m; } @@ -8498,18 +8917,19 @@ PyObject* PyAST_mod2obj(mod_ty t) } /* mode is 0 for "exec", 1 for "eval" and 2 for "single" input */ +/* and 3 for "func_type" */ mod_ty PyAST_obj2mod(PyObject* ast, PyArena* arena, int mode) { mod_ty res; PyObject *req_type[3]; - char *req_name[] = {"Module", "Expression", "Interactive"}; + char *req_name[] = {"Module", "Expression", "Interactive", "FunctionType"}; int isinstance; req_type[0] = (PyObject*)Module_type; req_type[1] = (PyObject*)Expression_type; req_type[2] = (PyObject*)Interactive_type; - assert(0 <= mode && mode <= 2); + assert(0 <= mode && mode <= 3); if (!init_types()) return NULL; diff --git a/Python/ast.c b/Python/ast.c index e10e63f539c3..c422e9651ced 100644 --- a/Python/ast.c +++ b/Python/ast.c @@ -698,6 +698,13 @@ ast_error(struct compiling *c, const node *n, const char *errmsg, ...) small_stmt elements is returned. */ +static string +new_type_comment(const char *s) +{ + return PyUnicode_DecodeUTF8(s, strlen(s), NULL); +} +#define NEW_TYPE_COMMENT(n) new_type_comment(STR(n)) + static int num_stmts(const node *n) { @@ -725,11 +732,17 @@ num_stmts(const node *n) case simple_stmt: return NCH(n) / 2; /* Divide by 2 to remove count of semi-colons */ case suite: + case func_body_suite: + /* func_body_suite: simple_stmt | NEWLINE [TYPE_COMMENT NEWLINE] INDENT stmt+ DEDENT */ + /* suite: simple_stmt | NEWLINE INDENT stmt+ DEDENT */ if (NCH(n) == 1) return num_stmts(CHILD(n, 0)); else { + i = 2; l = 0; - for (i = 2; i < (NCH(n) - 1); i++) + if (TYPE(CHILD(n, 1)) == TYPE_COMMENT) + i += 2; + for (; i < (NCH(n) - 1); i++) l += num_stmts(CHILD(n, i)); return l; } @@ -753,10 +766,13 @@ PyAST_FromNodeObject(const node *n, PyCompilerFlags *flags, { int i, j, k, num; asdl_seq *stmts = NULL; + asdl_seq *type_ignores = NULL; stmt_ty s; node *ch; struct compiling c; mod_ty res = NULL; + asdl_seq *argtypes = NULL; + expr_ty ret, arg; c.c_arena = arena; /* borrowed reference */ @@ -795,7 +811,23 @@ PyAST_FromNodeObject(const node *n, PyCompilerFlags *flags, } } } - res = Module(stmts, arena); + + /* Type ignores are stored under the ENDMARKER in file_input. */ + ch = CHILD(n, NCH(n) - 1); + REQ(ch, ENDMARKER); + num = NCH(ch); + type_ignores = _Py_asdl_seq_new(num, arena); + if (!type_ignores) + goto out; + + for (i = 0; i < num; i++) { + type_ignore_ty ti = TypeIgnore(LINENO(CHILD(ch, i)), arena); + if (!ti) + goto out; + asdl_seq_SET(type_ignores, i, ti); + } + + res = Module(stmts, type_ignores, arena); break; case eval_input: { expr_ty testlist_ast; @@ -847,6 +879,46 @@ PyAST_FromNodeObject(const node *n, PyCompilerFlags *flags, res = Interactive(stmts, arena); } break; + case func_type_input: + n = CHILD(n, 0); + REQ(n, func_type); + + if (TYPE(CHILD(n, 1)) == typelist) { + ch = CHILD(n, 1); + /* this is overly permissive -- we don't pay any attention to + * stars on the args -- just parse them into an ordered list */ + num = 0; + for (i = 0; i < NCH(ch); i++) { + if (TYPE(CHILD(ch, i)) == test) { + num++; + } + } + + argtypes = _Py_asdl_seq_new(num, arena); + if (!argtypes) + goto out; + + j = 0; + for (i = 0; i < NCH(ch); i++) { + if (TYPE(CHILD(ch, i)) == test) { + arg = ast_for_expr(&c, CHILD(ch, i)); + if (!arg) + goto out; + asdl_seq_SET(argtypes, j++, arg); + } + } + } + else { + argtypes = _Py_asdl_seq_new(0, arena); + if (!argtypes) + goto out; + } + + ret = ast_for_expr(&c, CHILD(n, NCH(n) - 1)); + if (!ret) + goto out; + res = FunctionType(argtypes, ret, arena); + break; default: PyErr_Format(PyExc_SystemError, "invalid node %d for PyAST_FromNode", TYPE(n)); @@ -1269,7 +1341,7 @@ ast_for_arg(struct compiling *c, const node *n) return NULL; } - ret = arg(name, annotation, LINENO(n), n->n_col_offset, + ret = arg(name, annotation, NULL, LINENO(n), n->n_col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena); if (!ret) return NULL; @@ -1328,13 +1400,22 @@ handle_keywordonly_args(struct compiling *c, const node *n, int start, goto error; if (forbidden_name(c, argname, ch, 0)) goto error; - arg = arg(argname, annotation, LINENO(ch), ch->n_col_offset, + arg = arg(argname, annotation, NULL, LINENO(ch), ch->n_col_offset, ch->n_end_lineno, ch->n_end_col_offset, c->c_arena); if (!arg) goto error; asdl_seq_SET(kwonlyargs, j++, arg); - i += 2; /* the name and the comma */ + i += 1; /* the name */ + if (TYPE(CHILD(n, i)) == COMMA) + i += 1; /* the comma, if present */ + break; + case TYPE_COMMENT: + /* arg will be equal to the last argument processed */ + arg->type_comment = NEW_TYPE_COMMENT(ch); + if (!arg->type_comment) + goto error; + i += 1; break; case DOUBLESTAR: return i; @@ -1464,19 +1545,29 @@ ast_for_arguments(struct compiling *c, const node *n) if (!arg) return NULL; asdl_seq_SET(posargs, k++, arg); - i += 2; /* the name and the comma */ + i += 1; /* the name */ + if (i < NCH(n) && TYPE(CHILD(n, i)) == COMMA) + i += 1; /* the comma, if present */ break; case STAR: if (i+1 >= NCH(n) || - (i+2 == NCH(n) && TYPE(CHILD(n, i+1)) == COMMA)) { + (i+2 == NCH(n) && (TYPE(CHILD(n, i+1)) == COMMA + || TYPE(CHILD(n, i+1)) == TYPE_COMMENT))) { ast_error(c, CHILD(n, i), - "named arguments must follow bare *"); + "named arguments must follow bare *"); return NULL; } ch = CHILD(n, i+1); /* tfpdef or COMMA */ if (TYPE(ch) == COMMA) { int res = 0; i += 2; /* now follows keyword only arguments */ + + if (i < NCH(n) && TYPE(CHILD(n, i)) == TYPE_COMMENT) { + ast_error(c, CHILD(n, i), + "bare * has associated type comment"); + return NULL; + } + res = handle_keywordonly_args(c, n, i, kwonlyargs, kwdefaults); if (res == -1) return NULL; @@ -1487,7 +1578,17 @@ ast_for_arguments(struct compiling *c, const node *n) if (!vararg) return NULL; - i += 3; + i += 2; /* the star and the name */ + if (i < NCH(n) && TYPE(CHILD(n, i)) == COMMA) + i += 1; /* the comma, if present */ + + if (i < NCH(n) && TYPE(CHILD(n, i)) == TYPE_COMMENT) { + vararg->type_comment = NEW_TYPE_COMMENT(CHILD(n, i)); + if (!vararg->type_comment) + return NULL; + i += 1; + } + if (i < NCH(n) && (TYPE(CHILD(n, i)) == tfpdef || TYPE(CHILD(n, i)) == vfpdef)) { int res = 0; @@ -1504,7 +1605,21 @@ ast_for_arguments(struct compiling *c, const node *n) kwarg = ast_for_arg(c, ch); if (!kwarg) return NULL; - i += 3; + i += 2; /* the double star and the name */ + if (TYPE(CHILD(n, i)) == COMMA) + i += 1; /* the comma, if present */ + break; + case TYPE_COMMENT: + assert(i); + + if (kwarg) + arg = kwarg; + + /* arg will be equal to the last argument processed */ + arg->type_comment = NEW_TYPE_COMMENT(ch); + if (!arg->type_comment) + return NULL; + i += 1; break; default: PyErr_Format(PyExc_SystemError, @@ -1613,7 +1728,7 @@ static stmt_ty ast_for_funcdef_impl(struct compiling *c, const node *n0, asdl_seq *decorator_seq, bool is_async) { - /* funcdef: 'def' NAME parameters ['->' test] ':' suite */ + /* funcdef: 'def' NAME parameters ['->' test] ':' [TYPE_COMMENT] suite */ const node * const n = is_async ? CHILD(n0, 1) : n0; identifier name; arguments_ty args; @@ -1621,6 +1736,8 @@ ast_for_funcdef_impl(struct compiling *c, const node *n0, expr_ty returns = NULL; int name_i = 1; int end_lineno, end_col_offset; + node *tc; + string type_comment = NULL; REQ(n, funcdef); @@ -1638,16 +1755,37 @@ ast_for_funcdef_impl(struct compiling *c, const node *n0, return NULL; name_i += 2; } + if (TYPE(CHILD(n, name_i + 3)) == TYPE_COMMENT) { + type_comment = NEW_TYPE_COMMENT(CHILD(n, name_i + 3)); + if (!type_comment) + return NULL; + name_i += 1; + } body = ast_for_suite(c, CHILD(n, name_i + 3)); if (!body) return NULL; get_last_end_pos(body, &end_lineno, &end_col_offset); + if (NCH(CHILD(n, name_i + 3)) > 1) { + /* Check if the suite has a type comment in it. */ + tc = CHILD(CHILD(n, name_i + 3), 1); + + if (TYPE(tc) == TYPE_COMMENT) { + if (type_comment != NULL) { + ast_error(c, n, "Cannot have two type comments on def"); + return NULL; + } + type_comment = NEW_TYPE_COMMENT(tc); + if (!type_comment) + return NULL; + } + } + if (is_async) - return AsyncFunctionDef(name, args, body, decorator_seq, returns, + return AsyncFunctionDef(name, args, body, decorator_seq, returns, type_comment, LINENO(n0), n0->n_col_offset, end_lineno, end_col_offset, c->c_arena); else - return FunctionDef(name, args, body, decorator_seq, returns, + return FunctionDef(name, args, body, decorator_seq, returns, type_comment, LINENO(n), n->n_col_offset, end_lineno, end_col_offset, c->c_arena); } @@ -2295,7 +2433,7 @@ ast_for_atom(struct compiling *c, const node *n) /* It's a dictionary comprehension. */ if (is_dict) { ast_error(c, n, "dict unpacking cannot be used in " - "dict comprehension"); + "dict comprehension"); return NULL; } res = ast_for_dictcomp(c, ch); @@ -2870,13 +3008,13 @@ ast_for_call(struct compiling *c, const node *n, expr_ty func, if (nkeywords) { if (ndoublestars) { ast_error(c, chch, - "positional argument follows " - "keyword argument unpacking"); + "positional argument follows " + "keyword argument unpacking"); } else { ast_error(c, chch, - "positional argument follows " - "keyword argument"); + "positional argument follows " + "keyword argument"); } return NULL; } @@ -2890,8 +3028,8 @@ ast_for_call(struct compiling *c, const node *n, expr_ty func, expr_ty starred; if (ndoublestars) { ast_error(c, chch, - "iterable argument unpacking follows " - "keyword argument unpacking"); + "iterable argument unpacking follows " + "keyword argument unpacking"); return NULL; } e = ast_for_expr(c, CHILD(ch, 1)); @@ -2929,13 +3067,13 @@ ast_for_call(struct compiling *c, const node *n, expr_ty func, if (nkeywords) { if (ndoublestars) { ast_error(c, chch, - "positional argument follows " - "keyword argument unpacking"); + "positional argument follows " + "keyword argument unpacking"); } else { ast_error(c, chch, - "positional argument follows " - "keyword argument"); + "positional argument follows " + "keyword argument"); } return NULL; } @@ -2996,7 +3134,7 @@ ast_for_call(struct compiling *c, const node *n, expr_ty func, tmp = ((keyword_ty)asdl_seq_GET(keywords, k))->arg; if (tmp && !PyUnicode_Compare(tmp, key)) { ast_error(c, chch, - "keyword argument repeated"); + "keyword argument repeated"); return NULL; } } @@ -3045,15 +3183,16 @@ ast_for_expr_stmt(struct compiling *c, const node *n) { REQ(n, expr_stmt); /* expr_stmt: testlist_star_expr (annassign | augassign (yield_expr|testlist) | - ('=' (yield_expr|testlist_star_expr))*) - annassign: ':' test ['=' test] - testlist_star_expr: (test|star_expr) (',' test|star_expr)* [','] - augassign: '+=' | '-=' | '*=' | '@=' | '/=' | '%=' | '&=' | '|=' | '^=' - | '<<=' | '>>=' | '**=' | '//=' + [('=' (yield_expr|testlist_star_expr))+ [TYPE_COMMENT]] ) + annassign: ':' test ['=' (yield_expr|testlist)] + testlist_star_expr: (test|star_expr) (',' (test|star_expr))* [','] + augassign: ('+=' | '-=' | '*=' | '@=' | '/=' | '%=' | '&=' | '|=' | '^=' | + '<<=' | '>>=' | '**=' | '//=') test: ... here starts the operator precedence dance */ + int num = NCH(n); - if (NCH(n) == 1) { + if (num == 1) { expr_ty e = ast_for_testlist(c, CHILD(n, 0)); if (!e) return NULL; @@ -3178,17 +3317,22 @@ ast_for_expr_stmt(struct compiling *c, const node *n) } } else { - int i; + int i, nch_minus_type, has_type_comment; asdl_seq *targets; node *value; expr_ty expression; + string type_comment; /* a normal assignment */ REQ(CHILD(n, 1), EQUAL); - targets = _Py_asdl_seq_new(NCH(n) / 2, c->c_arena); + + has_type_comment = TYPE(CHILD(n, num - 1)) == TYPE_COMMENT; + nch_minus_type = num - has_type_comment; + + targets = _Py_asdl_seq_new(nch_minus_type / 2, c->c_arena); if (!targets) return NULL; - for (i = 0; i < NCH(n) - 2; i += 2) { + for (i = 0; i < nch_minus_type - 2; i += 2) { expr_ty e; node *ch = CHILD(n, i); if (TYPE(ch) == yield_expr) { @@ -3205,14 +3349,21 @@ ast_for_expr_stmt(struct compiling *c, const node *n) asdl_seq_SET(targets, i / 2, e); } - value = CHILD(n, NCH(n) - 1); + value = CHILD(n, nch_minus_type - 1); if (TYPE(value) == testlist_star_expr) expression = ast_for_testlist(c, value); else expression = ast_for_expr(c, value); if (!expression) return NULL; - return Assign(targets, expression, LINENO(n), n->n_col_offset, + if (has_type_comment) { + type_comment = NEW_TYPE_COMMENT(CHILD(n, nch_minus_type)); + if (!type_comment) + return NULL; + } + else + type_comment = NULL; + return Assign(targets, expression, type_comment, LINENO(n), n->n_col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena); } } @@ -3520,8 +3671,9 @@ ast_for_import_stmt(struct compiling *c, const node *n) n = CHILD(n, idx); n_children = NCH(n); if (n_children % 2 == 0) { - ast_error(c, n, "trailing comma not allowed without" - " surrounding parentheses"); + ast_error(c, n, + "trailing comma not allowed without" + " surrounding parentheses"); return NULL; } break; @@ -3639,13 +3791,15 @@ ast_for_assert_stmt(struct compiling *c, const node *n) static asdl_seq * ast_for_suite(struct compiling *c, const node *n) { - /* suite: simple_stmt | NEWLINE INDENT stmt+ DEDENT */ + /* suite: simple_stmt | NEWLINE [TYPE_COMMENT NEWLINE] INDENT stmt+ DEDENT */ asdl_seq *seq; stmt_ty s; int i, total, num, end, pos = 0; node *ch; - REQ(n, suite); + if (TYPE(n) != func_body_suite) { + REQ(n, suite); + } total = num_stmts(n); seq = _Py_asdl_seq_new(total, c->c_arena); @@ -3669,7 +3823,13 @@ ast_for_suite(struct compiling *c, const node *n) } } else { - for (i = 2; i < (NCH(n) - 1); i++) { + i = 2; + if (TYPE(CHILD(n, 1)) == TYPE_COMMENT) { + i += 2; + REQ(CHILD(n, 2), NEWLINE); + } + + for (; i < (NCH(n) - 1); i++) { ch = CHILD(n, i); REQ(ch, stmt); num = num_stmts(ch); @@ -3903,11 +4063,15 @@ ast_for_for_stmt(struct compiling *c, const node *n0, bool is_async) expr_ty target, first; const node *node_target; int end_lineno, end_col_offset; - /* for_stmt: 'for' exprlist 'in' testlist ':' suite ['else' ':' suite] */ + int has_type_comment; + string type_comment; + /* for_stmt: 'for' exprlist 'in' testlist ':' [TYPE_COMMENT] suite ['else' ':' suite] */ REQ(n, for_stmt); - if (NCH(n) == 9) { - seq = ast_for_suite(c, CHILD(n, 8)); + has_type_comment = TYPE(CHILD(n, 5)) == TYPE_COMMENT; + + if (NCH(n) == 9 + has_type_comment) { + seq = ast_for_suite(c, CHILD(n, 8 + has_type_comment)); if (!seq) return NULL; } @@ -3929,7 +4093,7 @@ ast_for_for_stmt(struct compiling *c, const node *n0, bool is_async) expression = ast_for_testlist(c, CHILD(n, 3)); if (!expression) return NULL; - suite_seq = ast_for_suite(c, CHILD(n, 5)); + suite_seq = ast_for_suite(c, CHILD(n, 5 + has_type_comment)); if (!suite_seq) return NULL; @@ -3938,12 +4102,21 @@ ast_for_for_stmt(struct compiling *c, const node *n0, bool is_async) } else { get_last_end_pos(suite_seq, &end_lineno, &end_col_offset); } + + if (has_type_comment) { + type_comment = NEW_TYPE_COMMENT(CHILD(n, 5)); + if (!type_comment) + return NULL; + } + else + type_comment = NULL; + if (is_async) - return AsyncFor(target, expression, suite_seq, seq, + return AsyncFor(target, expression, suite_seq, seq, type_comment, LINENO(n0), n0->n_col_offset, end_lineno, end_col_offset, c->c_arena); else - return For(target, expression, suite_seq, seq, + return For(target, expression, suite_seq, seq, type_comment, LINENO(n), n->n_col_offset, end_lineno, end_col_offset, c->c_arena); } @@ -4111,21 +4284,25 @@ ast_for_with_item(struct compiling *c, const node *n) return withitem(context_expr, optional_vars, c->c_arena); } -/* with_stmt: 'with' with_item (',' with_item)* ':' suite */ +/* with_stmt: 'with' with_item (',' with_item)* ':' [TYPE_COMMENT] suite */ static stmt_ty ast_for_with_stmt(struct compiling *c, const node *n0, bool is_async) { const node * const n = is_async ? CHILD(n0, 1) : n0; - int i, n_items, end_lineno, end_col_offset; + int i, n_items, nch_minus_type, has_type_comment, end_lineno, end_col_offset; asdl_seq *items, *body; + string type_comment; REQ(n, with_stmt); - n_items = (NCH(n) - 2) / 2; + has_type_comment = TYPE(CHILD(n, NCH(n) - 2)) == TYPE_COMMENT; + nch_minus_type = NCH(n) - has_type_comment; + + n_items = (nch_minus_type - 2) / 2; items = _Py_asdl_seq_new(n_items, c->c_arena); if (!items) return NULL; - for (i = 1; i < NCH(n) - 2; i += 2) { + for (i = 1; i < nch_minus_type - 2; i += 2) { withitem_ty item = ast_for_with_item(c, CHILD(n, i)); if (!item) return NULL; @@ -4137,11 +4314,19 @@ ast_for_with_stmt(struct compiling *c, const node *n0, bool is_async) return NULL; get_last_end_pos(body, &end_lineno, &end_col_offset); + if (has_type_comment) { + type_comment = NEW_TYPE_COMMENT(CHILD(n, NCH(n) - 2)); + if (!type_comment) + return NULL; + } + else + type_comment = NULL; + if (is_async) - return AsyncWith(items, body, LINENO(n0), n0->n_col_offset, + return AsyncWith(items, body, type_comment, LINENO(n0), n0->n_col_offset, end_lineno, end_col_offset, c->c_arena); else - return With(items, body, LINENO(n), n->n_col_offset, + return With(items, body, type_comment, LINENO(n), n->n_col_offset, end_lineno, end_col_offset, c->c_arena); } @@ -4768,8 +4953,9 @@ fstring_find_expr(const char **str, const char *end, int raw, int recurse_lvl, if (ch == '\\') { /* Error: can't include a backslash character, inside parens or strings or not. */ - ast_error(c, n, "f-string expression part " - "cannot include a backslash"); + ast_error(c, n, + "f-string expression part " + "cannot include a backslash"); return -1; } if (quote_char) { @@ -4893,8 +5079,9 @@ fstring_find_expr(const char **str, const char *end, int raw, int recurse_lvl, /* Validate the conversion. */ if (!(conversion == 's' || conversion == 'r' || conversion == 'a')) { - ast_error(c, n, "f-string: invalid conversion character: " - "expected 's', 'r', or 'a'"); + ast_error(c, n, + "f-string: invalid conversion character: " + "expected 's', 'r', or 'a'"); return -1; } } @@ -5446,7 +5633,8 @@ parsestr(struct compiling *c, const node *n, int *bytesmode, int *rawmode, const char *ch; for (ch = s; *ch; ch++) { if (Py_CHARMASK(*ch) >= 0x80) { - ast_error(c, n, "bytes can only contain ASCII " + ast_error(c, n, + "bytes can only contain ASCII " "literal characters."); return -1; } diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index 332142fc6ffc..f9b901f7e59f 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -765,13 +765,13 @@ builtin_compile_impl(PyObject *module, PyObject *source, PyObject *filename, int compile_mode = -1; int is_ast; PyCompilerFlags cf; - int start[] = {Py_file_input, Py_eval_input, Py_single_input}; + int start[] = {Py_file_input, Py_eval_input, Py_single_input, Py_func_type_input}; PyObject *result; cf.cf_flags = flags | PyCF_SOURCE_IS_UTF8; if (flags & - ~(PyCF_MASK | PyCF_MASK_OBSOLETE | PyCF_DONT_IMPLY_DEDENT | PyCF_ONLY_AST)) + ~(PyCF_MASK | PyCF_MASK_OBSOLETE | PyCF_DONT_IMPLY_DEDENT | PyCF_ONLY_AST | PyCF_TYPE_COMMENTS)) { PyErr_SetString(PyExc_ValueError, "compile(): unrecognised flags"); @@ -795,9 +795,21 @@ builtin_compile_impl(PyObject *module, PyObject *source, PyObject *filename, compile_mode = 1; else if (strcmp(mode, "single") == 0) compile_mode = 2; + else if (strcmp(mode, "func_type") == 0) { + if (!(flags & PyCF_ONLY_AST)) { + PyErr_SetString(PyExc_ValueError, + "compile() mode 'func_type' requires flag PyCF_ONLY_AST"); + goto error; + } + compile_mode = 3; + } else { - PyErr_SetString(PyExc_ValueError, - "compile() mode must be 'exec', 'eval' or 'single'"); + const char *msg; + if (flags & PyCF_ONLY_AST) + msg = "compile() mode must be 'exec', 'eval', 'single' or 'func_type'"; + else + msg = "compile() mode must be 'exec', 'eval' or 'single'"; + PyErr_SetString(PyExc_ValueError, msg); goto error; } diff --git a/Python/graminit.c b/Python/graminit.c index 225d32793906..6e0f19891baa 100644 --- a/Python/graminit.c +++ b/Python/graminit.c @@ -135,30 +135,35 @@ static arc arcs_7_3[2] = { static arc arcs_7_4[1] = { {26, 6}, }; -static arc arcs_7_5[1] = { +static arc arcs_7_5[2] = { {28, 7}, + {29, 8}, }; static arc arcs_7_6[1] = { {27, 5}, }; static arc arcs_7_7[1] = { - {0, 7}, + {29, 8}, +}; +static arc arcs_7_8[1] = { + {0, 8}, }; -static state states_7[8] = { +static state states_7[9] = { {1, arcs_7_0}, {1, arcs_7_1}, {1, arcs_7_2}, {2, arcs_7_3}, {1, arcs_7_4}, - {1, arcs_7_5}, + {2, arcs_7_5}, {1, arcs_7_6}, {1, arcs_7_7}, + {1, arcs_7_8}, }; static arc arcs_8_0[1] = { {13, 1}, }; static arc arcs_8_1[2] = { - {29, 2}, + {30, 2}, {15, 3}, }; static arc arcs_8_2[1] = { @@ -174,107 +179,144 @@ static state states_8[4] = { {1, arcs_8_3}, }; static arc arcs_9_0[3] = { - {30, 1}, - {33, 2}, - {34, 3}, + {31, 1}, + {34, 2}, + {35, 3}, }; -static arc arcs_9_1[3] = { - {31, 4}, - {32, 5}, +static arc arcs_9_1[4] = { + {32, 4}, + {33, 5}, + {28, 6}, {0, 1}, }; -static arc arcs_9_2[3] = { - {30, 6}, - {32, 7}, +static arc arcs_9_2[4] = { + {31, 7}, + {33, 8}, + {28, 6}, {0, 2}, }; static arc arcs_9_3[1] = { - {30, 8}, + {31, 9}, }; static arc arcs_9_4[1] = { - {26, 9}, + {26, 10}, }; -static arc arcs_9_5[4] = { - {30, 10}, - {33, 11}, - {34, 3}, +static arc arcs_9_5[5] = { + {28, 11}, + {31, 12}, + {34, 13}, + {35, 3}, {0, 5}, }; -static arc arcs_9_6[2] = { - {32, 7}, +static arc arcs_9_6[1] = { {0, 6}, }; static arc arcs_9_7[3] = { - {30, 12}, - {34, 3}, + {33, 8}, + {28, 6}, {0, 7}, }; -static arc arcs_9_8[2] = { - {32, 13}, +static arc arcs_9_8[4] = { + {28, 14}, + {31, 15}, + {35, 3}, {0, 8}, }; -static arc arcs_9_9[2] = { - {32, 5}, +static arc arcs_9_9[3] = { + {33, 16}, + {28, 6}, {0, 9}, }; static arc arcs_9_10[3] = { - {32, 5}, - {31, 4}, + {33, 5}, + {28, 6}, {0, 10}, }; -static arc arcs_9_11[3] = { - {30, 14}, - {32, 15}, +static arc arcs_9_11[4] = { + {31, 12}, + {34, 13}, + {35, 3}, {0, 11}, }; -static arc arcs_9_12[3] = { - {32, 7}, - {31, 16}, +static arc arcs_9_12[4] = { + {33, 5}, + {32, 4}, + {28, 6}, {0, 12}, }; -static arc arcs_9_13[1] = { +static arc arcs_9_13[4] = { + {31, 17}, + {33, 18}, + {28, 6}, {0, 13}, }; -static arc arcs_9_14[2] = { - {32, 15}, +static arc arcs_9_14[3] = { + {31, 15}, + {35, 3}, {0, 14}, }; -static arc arcs_9_15[3] = { - {30, 17}, - {34, 3}, +static arc arcs_9_15[4] = { + {33, 8}, + {32, 19}, + {28, 6}, {0, 15}, }; -static arc arcs_9_16[1] = { - {26, 6}, +static arc arcs_9_16[2] = { + {28, 6}, + {0, 16}, }; static arc arcs_9_17[3] = { - {32, 15}, - {31, 18}, + {33, 18}, + {28, 6}, {0, 17}, }; -static arc arcs_9_18[1] = { - {26, 14}, +static arc arcs_9_18[4] = { + {28, 20}, + {31, 21}, + {35, 3}, + {0, 18}, +}; +static arc arcs_9_19[1] = { + {26, 7}, }; -static state states_9[19] = { +static arc arcs_9_20[3] = { + {31, 21}, + {35, 3}, + {0, 20}, +}; +static arc arcs_9_21[4] = { + {33, 18}, + {32, 22}, + {28, 6}, + {0, 21}, +}; +static arc arcs_9_22[1] = { + {26, 17}, +}; +static state states_9[23] = { {3, arcs_9_0}, - {3, arcs_9_1}, - {3, arcs_9_2}, + {4, arcs_9_1}, + {4, arcs_9_2}, {1, arcs_9_3}, {1, arcs_9_4}, - {4, arcs_9_5}, - {2, arcs_9_6}, + {5, arcs_9_5}, + {1, arcs_9_6}, {3, arcs_9_7}, - {2, arcs_9_8}, - {2, arcs_9_9}, + {4, arcs_9_8}, + {3, arcs_9_9}, {3, arcs_9_10}, - {3, arcs_9_11}, - {3, arcs_9_12}, - {1, arcs_9_13}, - {2, arcs_9_14}, - {3, arcs_9_15}, - {1, arcs_9_16}, + {4, arcs_9_11}, + {4, arcs_9_12}, + {4, arcs_9_13}, + {3, arcs_9_14}, + {4, arcs_9_15}, + {2, arcs_9_16}, {3, arcs_9_17}, - {1, arcs_9_18}, + {4, arcs_9_18}, + {1, arcs_9_19}, + {3, arcs_9_20}, + {4, arcs_9_21}, + {1, arcs_9_22}, }; static arc arcs_10_0[1] = { {23, 1}, @@ -296,82 +338,82 @@ static state states_10[4] = { {1, arcs_10_3}, }; static arc arcs_11_0[3] = { - {36, 1}, - {33, 2}, - {34, 3}, + {37, 1}, + {34, 2}, + {35, 3}, }; static arc arcs_11_1[3] = { - {31, 4}, - {32, 5}, + {32, 4}, + {33, 5}, {0, 1}, }; static arc arcs_11_2[3] = { - {36, 6}, - {32, 7}, + {37, 6}, + {33, 7}, {0, 2}, }; static arc arcs_11_3[1] = { - {36, 8}, + {37, 8}, }; static arc arcs_11_4[1] = { {26, 9}, }; static arc arcs_11_5[4] = { - {36, 10}, - {33, 11}, - {34, 3}, + {37, 10}, + {34, 11}, + {35, 3}, {0, 5}, }; static arc arcs_11_6[2] = { - {32, 7}, + {33, 7}, {0, 6}, }; static arc arcs_11_7[3] = { - {36, 12}, - {34, 3}, + {37, 12}, + {35, 3}, {0, 7}, }; static arc arcs_11_8[2] = { - {32, 13}, + {33, 13}, {0, 8}, }; static arc arcs_11_9[2] = { - {32, 5}, + {33, 5}, {0, 9}, }; static arc arcs_11_10[3] = { - {32, 5}, - {31, 4}, + {33, 5}, + {32, 4}, {0, 10}, }; static arc arcs_11_11[3] = { - {36, 14}, - {32, 15}, + {37, 14}, + {33, 15}, {0, 11}, }; static arc arcs_11_12[3] = { - {32, 7}, - {31, 16}, + {33, 7}, + {32, 16}, {0, 12}, }; static arc arcs_11_13[1] = { {0, 13}, }; static arc arcs_11_14[2] = { - {32, 15}, + {33, 15}, {0, 14}, }; static arc arcs_11_15[3] = { - {36, 17}, - {34, 3}, + {37, 17}, + {35, 3}, {0, 15}, }; static arc arcs_11_16[1] = { {26, 6}, }; static arc arcs_11_17[3] = { - {32, 15}, - {31, 18}, + {33, 15}, + {32, 18}, {0, 17}, }; static arc arcs_11_18[1] = { @@ -420,14 +462,14 @@ static state states_13[2] = { {1, arcs_13_1}, }; static arc arcs_14_0[1] = { - {37, 1}, + {38, 1}, }; static arc arcs_14_1[2] = { - {38, 2}, + {39, 2}, {2, 3}, }; static arc arcs_14_2[2] = { - {37, 1}, + {38, 1}, {2, 3}, }; static arc arcs_14_3[1] = { @@ -440,7 +482,6 @@ static state states_14[4] = { {1, arcs_14_3}, }; static arc arcs_15_0[8] = { - {39, 1}, {40, 1}, {41, 1}, {42, 1}, @@ -448,6 +489,7 @@ static arc arcs_15_0[8] = { {44, 1}, {45, 1}, {46, 1}, + {47, 1}, }; static arc arcs_15_1[1] = { {0, 1}, @@ -457,27 +499,28 @@ static state states_15[2] = { {1, arcs_15_1}, }; static arc arcs_16_0[1] = { - {47, 1}, + {48, 1}, }; static arc arcs_16_1[4] = { - {48, 2}, - {49, 3}, - {31, 4}, + {49, 2}, + {50, 3}, + {32, 4}, {0, 1}, }; static arc arcs_16_2[1] = { {0, 2}, }; static arc arcs_16_3[2] = { - {50, 2}, + {51, 2}, {9, 2}, }; static arc arcs_16_4[2] = { - {50, 5}, - {47, 5}, + {51, 5}, + {48, 5}, }; -static arc arcs_16_5[2] = { - {31, 4}, +static arc arcs_16_5[3] = { + {32, 4}, + {28, 2}, {0, 5}, }; static state states_16[6] = { @@ -486,7 +529,7 @@ static state states_16[6] = { {1, arcs_16_2}, {2, arcs_16_3}, {2, arcs_16_4}, - {2, arcs_16_5}, + {3, arcs_16_5}, }; static arc arcs_17_0[1] = { {27, 1}, @@ -495,11 +538,11 @@ static arc arcs_17_1[1] = { {26, 2}, }; static arc arcs_17_2[2] = { - {31, 3}, + {32, 3}, {0, 2}, }; static arc arcs_17_3[2] = { - {50, 4}, + {51, 4}, {9, 4}, }; static arc arcs_17_4[1] = { @@ -514,15 +557,15 @@ static state states_17[5] = { }; static arc arcs_18_0[2] = { {26, 1}, - {51, 1}, + {52, 1}, }; static arc arcs_18_1[2] = { - {32, 2}, + {33, 2}, {0, 1}, }; static arc arcs_18_2[3] = { {26, 1}, - {51, 1}, + {52, 1}, {0, 2}, }; static state states_18[3] = { @@ -531,7 +574,6 @@ static state states_18[3] = { {3, arcs_18_2}, }; static arc arcs_19_0[13] = { - {52, 1}, {53, 1}, {54, 1}, {55, 1}, @@ -544,6 +586,7 @@ static arc arcs_19_0[13] = { {62, 1}, {63, 1}, {64, 1}, + {65, 1}, }; static arc arcs_19_1[1] = { {0, 1}, @@ -553,10 +596,10 @@ static state states_19[2] = { {1, arcs_19_1}, }; static arc arcs_20_0[1] = { - {65, 1}, + {66, 1}, }; static arc arcs_20_1[1] = { - {66, 2}, + {67, 2}, }; static arc arcs_20_2[1] = { {0, 2}, @@ -567,7 +610,7 @@ static state states_20[3] = { {1, arcs_20_2}, }; static arc arcs_21_0[1] = { - {67, 1}, + {68, 1}, }; static arc arcs_21_1[1] = { {0, 1}, @@ -577,11 +620,11 @@ static state states_21[2] = { {1, arcs_21_1}, }; static arc arcs_22_0[5] = { - {68, 1}, {69, 1}, {70, 1}, {71, 1}, {72, 1}, + {73, 1}, }; static arc arcs_22_1[1] = { {0, 1}, @@ -591,7 +634,7 @@ static state states_22[2] = { {1, arcs_22_1}, }; static arc arcs_23_0[1] = { - {73, 1}, + {74, 1}, }; static arc arcs_23_1[1] = { {0, 1}, @@ -601,7 +644,7 @@ static state states_23[2] = { {1, arcs_23_1}, }; static arc arcs_24_0[1] = { - {74, 1}, + {75, 1}, }; static arc arcs_24_1[1] = { {0, 1}, @@ -611,10 +654,10 @@ static state states_24[2] = { {1, arcs_24_1}, }; static arc arcs_25_0[1] = { - {75, 1}, + {76, 1}, }; static arc arcs_25_1[2] = { - {47, 2}, + {48, 2}, {0, 1}, }; static arc arcs_25_2[1] = { @@ -626,7 +669,7 @@ static state states_25[3] = { {1, arcs_25_2}, }; static arc arcs_26_0[1] = { - {50, 1}, + {51, 1}, }; static arc arcs_26_1[1] = { {0, 1}, @@ -636,14 +679,14 @@ static state states_26[2] = { {1, arcs_26_1}, }; static arc arcs_27_0[1] = { - {76, 1}, + {77, 1}, }; static arc arcs_27_1[2] = { {26, 2}, {0, 1}, }; static arc arcs_27_2[2] = { - {77, 3}, + {78, 3}, {0, 2}, }; static arc arcs_27_3[1] = { @@ -660,8 +703,8 @@ static state states_27[5] = { {1, arcs_27_4}, }; static arc arcs_28_0[2] = { - {78, 1}, {79, 1}, + {80, 1}, }; static arc arcs_28_1[1] = { {0, 1}, @@ -671,10 +714,10 @@ static state states_28[2] = { {1, arcs_28_1}, }; static arc arcs_29_0[1] = { - {80, 1}, + {81, 1}, }; static arc arcs_29_1[1] = { - {81, 2}, + {82, 2}, }; static arc arcs_29_2[1] = { {0, 2}, @@ -685,32 +728,32 @@ static state states_29[3] = { {1, arcs_29_2}, }; static arc arcs_30_0[1] = { - {77, 1}, + {78, 1}, }; static arc arcs_30_1[3] = { - {82, 2}, {83, 2}, + {84, 2}, {12, 3}, }; static arc arcs_30_2[4] = { - {82, 2}, {83, 2}, + {84, 2}, {12, 3}, - {80, 4}, + {81, 4}, }; static arc arcs_30_3[1] = { - {80, 4}, + {81, 4}, }; static arc arcs_30_4[3] = { - {33, 5}, + {34, 5}, {13, 6}, - {84, 5}, + {85, 5}, }; static arc arcs_30_5[1] = { {0, 5}, }; static arc arcs_30_6[1] = { - {84, 7}, + {85, 7}, }; static arc arcs_30_7[1] = { {15, 5}, @@ -729,7 +772,7 @@ static arc arcs_31_0[1] = { {23, 1}, }; static arc arcs_31_1[2] = { - {86, 2}, + {87, 2}, {0, 1}, }; static arc arcs_31_2[1] = { @@ -748,7 +791,7 @@ static arc arcs_32_0[1] = { {12, 1}, }; static arc arcs_32_1[2] = { - {86, 2}, + {87, 2}, {0, 1}, }; static arc arcs_32_2[1] = { @@ -764,14 +807,14 @@ static state states_32[4] = { {1, arcs_32_3}, }; static arc arcs_33_0[1] = { - {85, 1}, + {86, 1}, }; static arc arcs_33_1[2] = { - {32, 2}, + {33, 2}, {0, 1}, }; static arc arcs_33_2[2] = { - {85, 1}, + {86, 1}, {0, 2}, }; static state states_33[3] = { @@ -780,10 +823,10 @@ static state states_33[3] = { {2, arcs_33_2}, }; static arc arcs_34_0[1] = { - {87, 1}, + {88, 1}, }; static arc arcs_34_1[2] = { - {32, 0}, + {33, 0}, {0, 1}, }; static state states_34[2] = { @@ -794,7 +837,7 @@ static arc arcs_35_0[1] = { {23, 1}, }; static arc arcs_35_1[2] = { - {82, 0}, + {83, 0}, {0, 1}, }; static state states_35[2] = { @@ -802,13 +845,13 @@ static state states_35[2] = { {2, arcs_35_1}, }; static arc arcs_36_0[1] = { - {88, 1}, + {89, 1}, }; static arc arcs_36_1[1] = { {23, 2}, }; static arc arcs_36_2[2] = { - {32, 1}, + {33, 1}, {0, 2}, }; static state states_36[3] = { @@ -817,13 +860,13 @@ static state states_36[3] = { {2, arcs_36_2}, }; static arc arcs_37_0[1] = { - {89, 1}, + {90, 1}, }; static arc arcs_37_1[1] = { {23, 2}, }; static arc arcs_37_2[2] = { - {32, 1}, + {33, 1}, {0, 2}, }; static state states_37[3] = { @@ -832,13 +875,13 @@ static state states_37[3] = { {2, arcs_37_2}, }; static arc arcs_38_0[1] = { - {90, 1}, + {91, 1}, }; static arc arcs_38_1[1] = { {26, 2}, }; static arc arcs_38_2[2] = { - {32, 3}, + {33, 3}, {0, 2}, }; static arc arcs_38_3[1] = { @@ -855,15 +898,15 @@ static state states_38[5] = { {1, arcs_38_4}, }; static arc arcs_39_0[9] = { - {91, 1}, {92, 1}, {93, 1}, {94, 1}, {95, 1}, + {96, 1}, {19, 1}, {18, 1}, {17, 1}, - {96, 1}, + {97, 1}, }; static arc arcs_39_1[1] = { {0, 1}, @@ -877,8 +920,8 @@ static arc arcs_40_0[1] = { }; static arc arcs_40_1[3] = { {19, 2}, - {95, 2}, - {93, 2}, + {96, 2}, + {94, 2}, }; static arc arcs_40_2[1] = { {0, 2}, @@ -889,27 +932,27 @@ static state states_40[3] = { {1, arcs_40_2}, }; static arc arcs_41_0[1] = { - {97, 1}, + {98, 1}, }; static arc arcs_41_1[1] = { - {98, 2}, + {99, 2}, }; static arc arcs_41_2[1] = { {27, 3}, }; static arc arcs_41_3[1] = { - {28, 4}, + {100, 4}, }; static arc arcs_41_4[3] = { - {99, 1}, - {100, 5}, + {101, 1}, + {102, 5}, {0, 4}, }; static arc arcs_41_5[1] = { {27, 6}, }; static arc arcs_41_6[1] = { - {28, 7}, + {100, 7}, }; static arc arcs_41_7[1] = { {0, 7}, @@ -925,7 +968,7 @@ static state states_41[8] = { {1, arcs_41_7}, }; static arc arcs_42_0[1] = { - {101, 1}, + {103, 1}, }; static arc arcs_42_1[1] = { {26, 2}, @@ -934,17 +977,17 @@ static arc arcs_42_2[1] = { {27, 3}, }; static arc arcs_42_3[1] = { - {28, 4}, + {100, 4}, }; static arc arcs_42_4[2] = { - {100, 5}, + {102, 5}, {0, 4}, }; static arc arcs_42_5[1] = { {27, 6}, }; static arc arcs_42_6[1] = { - {28, 7}, + {100, 7}, }; static arc arcs_42_7[1] = { {0, 7}, @@ -960,13 +1003,13 @@ static state states_42[8] = { {1, arcs_42_7}, }; static arc arcs_43_0[1] = { - {102, 1}, + {104, 1}, }; static arc arcs_43_1[1] = { - {66, 2}, + {67, 2}, }; static arc arcs_43_2[1] = { - {103, 3}, + {105, 3}, }; static arc arcs_43_3[1] = { {9, 4}, @@ -974,46 +1017,51 @@ static arc arcs_43_3[1] = { static arc arcs_43_4[1] = { {27, 5}, }; -static arc arcs_43_5[1] = { +static arc arcs_43_5[2] = { {28, 6}, + {100, 7}, }; -static arc arcs_43_6[2] = { +static arc arcs_43_6[1] = { {100, 7}, - {0, 6}, }; -static arc arcs_43_7[1] = { - {27, 8}, +static arc arcs_43_7[2] = { + {102, 8}, + {0, 7}, }; static arc arcs_43_8[1] = { - {28, 9}, + {27, 9}, }; static arc arcs_43_9[1] = { - {0, 9}, + {100, 10}, +}; +static arc arcs_43_10[1] = { + {0, 10}, }; -static state states_43[10] = { +static state states_43[11] = { {1, arcs_43_0}, {1, arcs_43_1}, {1, arcs_43_2}, {1, arcs_43_3}, {1, arcs_43_4}, - {1, arcs_43_5}, - {2, arcs_43_6}, - {1, arcs_43_7}, + {2, arcs_43_5}, + {1, arcs_43_6}, + {2, arcs_43_7}, {1, arcs_43_8}, {1, arcs_43_9}, + {1, arcs_43_10}, }; static arc arcs_44_0[1] = { - {104, 1}, + {106, 1}, }; static arc arcs_44_1[1] = { {27, 2}, }; static arc arcs_44_2[1] = { - {28, 3}, + {100, 3}, }; static arc arcs_44_3[2] = { - {105, 4}, - {106, 5}, + {107, 4}, + {108, 5}, }; static arc arcs_44_4[1] = { {27, 6}, @@ -1022,15 +1070,15 @@ static arc arcs_44_5[1] = { {27, 7}, }; static arc arcs_44_6[1] = { - {28, 8}, + {100, 8}, }; static arc arcs_44_7[1] = { - {28, 9}, + {100, 9}, }; static arc arcs_44_8[4] = { - {105, 4}, - {100, 10}, - {106, 5}, + {107, 4}, + {102, 10}, + {108, 5}, {0, 8}, }; static arc arcs_44_9[1] = { @@ -1040,10 +1088,10 @@ static arc arcs_44_10[1] = { {27, 11}, }; static arc arcs_44_11[1] = { - {28, 12}, + {100, 12}, }; static arc arcs_44_12[2] = { - {106, 5}, + {108, 5}, {0, 12}, }; static state states_44[13] = { @@ -1062,37 +1110,42 @@ static state states_44[13] = { {2, arcs_44_12}, }; static arc arcs_45_0[1] = { - {107, 1}, + {109, 1}, }; static arc arcs_45_1[1] = { - {108, 2}, + {110, 2}, }; static arc arcs_45_2[2] = { - {32, 1}, + {33, 1}, {27, 3}, }; -static arc arcs_45_3[1] = { +static arc arcs_45_3[2] = { {28, 4}, + {100, 5}, }; static arc arcs_45_4[1] = { - {0, 4}, + {100, 5}, }; -static state states_45[5] = { +static arc arcs_45_5[1] = { + {0, 5}, +}; +static state states_45[6] = { {1, arcs_45_0}, {1, arcs_45_1}, {2, arcs_45_2}, - {1, arcs_45_3}, + {2, arcs_45_3}, {1, arcs_45_4}, + {1, arcs_45_5}, }; static arc arcs_46_0[1] = { {26, 1}, }; static arc arcs_46_1[2] = { - {86, 2}, + {87, 2}, {0, 1}, }; static arc arcs_46_2[1] = { - {109, 3}, + {111, 3}, }; static arc arcs_46_3[1] = { {0, 3}, @@ -1104,14 +1157,14 @@ static state states_46[4] = { {1, arcs_46_3}, }; static arc arcs_47_0[1] = { - {110, 1}, + {112, 1}, }; static arc arcs_47_1[2] = { {26, 2}, {0, 1}, }; static arc arcs_47_2[2] = { - {86, 3}, + {87, 3}, {0, 2}, }; static arc arcs_47_3[1] = { @@ -1135,14 +1188,14 @@ static arc arcs_48_1[1] = { {0, 1}, }; static arc arcs_48_2[1] = { - {111, 3}, + {113, 3}, }; static arc arcs_48_3[1] = { {6, 4}, }; static arc arcs_48_4[2] = { {6, 4}, - {112, 1}, + {114, 1}, }; static state states_48[5] = { {2, arcs_48_0}, @@ -1155,7 +1208,7 @@ static arc arcs_49_0[1] = { {26, 1}, }; static arc arcs_49_1[2] = { - {113, 2}, + {115, 2}, {0, 1}, }; static arc arcs_49_2[1] = { @@ -1171,21 +1224,21 @@ static state states_49[4] = { {1, arcs_49_3}, }; static arc arcs_50_0[2] = { - {114, 1}, - {115, 2}, + {116, 1}, + {117, 2}, }; static arc arcs_50_1[2] = { - {97, 3}, + {98, 3}, {0, 1}, }; static arc arcs_50_2[1] = { {0, 2}, }; static arc arcs_50_3[1] = { - {114, 4}, + {116, 4}, }; static arc arcs_50_4[1] = { - {100, 5}, + {102, 5}, }; static arc arcs_50_5[1] = { {26, 2}, @@ -1199,8 +1252,8 @@ static state states_50[6] = { {1, arcs_50_5}, }; static arc arcs_51_0[2] = { - {114, 1}, - {117, 1}, + {116, 1}, + {119, 1}, }; static arc arcs_51_1[1] = { {0, 1}, @@ -1210,10 +1263,10 @@ static state states_51[2] = { {1, arcs_51_1}, }; static arc arcs_52_0[1] = { - {118, 1}, + {120, 1}, }; static arc arcs_52_1[2] = { - {35, 2}, + {36, 2}, {27, 3}, }; static arc arcs_52_2[1] = { @@ -1233,17 +1286,17 @@ static state states_52[5] = { {1, arcs_52_4}, }; static arc arcs_53_0[1] = { - {118, 1}, + {120, 1}, }; static arc arcs_53_1[2] = { - {35, 2}, + {36, 2}, {27, 3}, }; static arc arcs_53_2[1] = { {27, 3}, }; static arc arcs_53_3[1] = { - {116, 4}, + {118, 4}, }; static arc arcs_53_4[1] = { {0, 4}, @@ -1256,10 +1309,10 @@ static state states_53[5] = { {1, arcs_53_4}, }; static arc arcs_54_0[1] = { - {119, 1}, + {121, 1}, }; static arc arcs_54_1[2] = { - {120, 0}, + {122, 0}, {0, 1}, }; static state states_54[2] = { @@ -1267,10 +1320,10 @@ static state states_54[2] = { {2, arcs_54_1}, }; static arc arcs_55_0[1] = { - {121, 1}, + {123, 1}, }; static arc arcs_55_1[2] = { - {122, 0}, + {124, 0}, {0, 1}, }; static state states_55[2] = { @@ -1278,11 +1331,11 @@ static state states_55[2] = { {2, arcs_55_1}, }; static arc arcs_56_0[2] = { - {123, 1}, - {124, 2}, + {125, 1}, + {126, 2}, }; static arc arcs_56_1[1] = { - {121, 2}, + {123, 2}, }; static arc arcs_56_2[1] = { {0, 2}, @@ -1293,10 +1346,10 @@ static state states_56[3] = { {1, arcs_56_2}, }; static arc arcs_57_0[1] = { - {109, 1}, + {111, 1}, }; static arc arcs_57_1[2] = { - {125, 0}, + {127, 0}, {0, 1}, }; static state states_57[2] = { @@ -1304,25 +1357,25 @@ static state states_57[2] = { {2, arcs_57_1}, }; static arc arcs_58_0[10] = { - {126, 1}, - {127, 1}, {128, 1}, {129, 1}, {130, 1}, {131, 1}, {132, 1}, - {103, 1}, - {123, 2}, - {133, 3}, + {133, 1}, + {134, 1}, + {105, 1}, + {125, 2}, + {135, 3}, }; static arc arcs_58_1[1] = { {0, 1}, }; static arc arcs_58_2[1] = { - {103, 1}, + {105, 1}, }; static arc arcs_58_3[2] = { - {123, 1}, + {125, 1}, {0, 3}, }; static state states_58[4] = { @@ -1332,10 +1385,10 @@ static state states_58[4] = { {2, arcs_58_3}, }; static arc arcs_59_0[1] = { - {33, 1}, + {34, 1}, }; static arc arcs_59_1[1] = { - {109, 2}, + {111, 2}, }; static arc arcs_59_2[1] = { {0, 2}, @@ -1346,10 +1399,10 @@ static state states_59[3] = { {1, arcs_59_2}, }; static arc arcs_60_0[1] = { - {134, 1}, + {136, 1}, }; static arc arcs_60_1[2] = { - {135, 0}, + {137, 0}, {0, 1}, }; static state states_60[2] = { @@ -1357,10 +1410,10 @@ static state states_60[2] = { {2, arcs_60_1}, }; static arc arcs_61_0[1] = { - {136, 1}, + {138, 1}, }; static arc arcs_61_1[2] = { - {137, 0}, + {139, 0}, {0, 1}, }; static state states_61[2] = { @@ -1368,10 +1421,10 @@ static state states_61[2] = { {2, arcs_61_1}, }; static arc arcs_62_0[1] = { - {138, 1}, + {140, 1}, }; static arc arcs_62_1[2] = { - {139, 0}, + {141, 0}, {0, 1}, }; static state states_62[2] = { @@ -1379,11 +1432,11 @@ static state states_62[2] = { {2, arcs_62_1}, }; static arc arcs_63_0[1] = { - {140, 1}, + {142, 1}, }; static arc arcs_63_1[3] = { - {141, 0}, - {142, 0}, + {143, 0}, + {144, 0}, {0, 1}, }; static state states_63[2] = { @@ -1391,11 +1444,11 @@ static state states_63[2] = { {3, arcs_63_1}, }; static arc arcs_64_0[1] = { - {143, 1}, + {145, 1}, }; static arc arcs_64_1[3] = { - {144, 0}, - {145, 0}, + {146, 0}, + {147, 0}, {0, 1}, }; static state states_64[2] = { @@ -1403,14 +1456,14 @@ static state states_64[2] = { {3, arcs_64_1}, }; static arc arcs_65_0[1] = { - {146, 1}, + {148, 1}, }; static arc arcs_65_1[6] = { - {33, 0}, + {34, 0}, {11, 0}, - {147, 0}, - {148, 0}, {149, 0}, + {150, 0}, + {151, 0}, {0, 1}, }; static state states_65[2] = { @@ -1418,13 +1471,13 @@ static state states_65[2] = { {6, arcs_65_1}, }; static arc arcs_66_0[4] = { - {144, 1}, - {145, 1}, - {150, 1}, - {151, 2}, + {146, 1}, + {147, 1}, + {152, 1}, + {153, 2}, }; static arc arcs_66_1[1] = { - {146, 2}, + {148, 2}, }; static arc arcs_66_2[1] = { {0, 2}, @@ -1435,14 +1488,14 @@ static state states_66[3] = { {1, arcs_66_2}, }; static arc arcs_67_0[1] = { - {152, 1}, + {154, 1}, }; static arc arcs_67_1[2] = { - {34, 2}, + {35, 2}, {0, 1}, }; static arc arcs_67_2[1] = { - {146, 3}, + {148, 3}, }; static arc arcs_67_3[1] = { {0, 3}, @@ -1454,14 +1507,14 @@ static state states_67[4] = { {1, arcs_67_3}, }; static arc arcs_68_0[2] = { - {153, 1}, - {154, 2}, + {155, 1}, + {156, 2}, }; static arc arcs_68_1[1] = { - {154, 2}, + {156, 2}, }; static arc arcs_68_2[2] = { - {155, 2}, + {157, 2}, {0, 2}, }; static state states_68[3] = { @@ -1471,44 +1524,44 @@ static state states_68[3] = { }; static arc arcs_69_0[10] = { {13, 1}, - {157, 2}, - {159, 3}, + {159, 2}, + {161, 3}, {23, 4}, - {162, 4}, - {163, 5}, - {83, 4}, {164, 4}, - {165, 4}, + {165, 5}, + {84, 4}, {166, 4}, + {167, 4}, + {168, 4}, }; static arc arcs_69_1[3] = { - {50, 6}, - {156, 6}, + {51, 6}, + {158, 6}, {15, 4}, }; static arc arcs_69_2[2] = { - {156, 7}, - {158, 4}, + {158, 7}, + {160, 4}, }; static arc arcs_69_3[2] = { - {160, 8}, - {161, 4}, + {162, 8}, + {163, 4}, }; static arc arcs_69_4[1] = { {0, 4}, }; static arc arcs_69_5[2] = { - {163, 5}, + {165, 5}, {0, 5}, }; static arc arcs_69_6[1] = { {15, 4}, }; static arc arcs_69_7[1] = { - {158, 4}, + {160, 4}, }; static arc arcs_69_8[1] = { - {161, 4}, + {163, 4}, }; static state states_69[9] = { {10, arcs_69_0}, @@ -1522,24 +1575,24 @@ static state states_69[9] = { {1, arcs_69_8}, }; static arc arcs_70_0[2] = { - {98, 1}, - {51, 1}, + {99, 1}, + {52, 1}, }; static arc arcs_70_1[3] = { - {167, 2}, - {32, 3}, + {169, 2}, + {33, 3}, {0, 1}, }; static arc arcs_70_2[1] = { {0, 2}, }; static arc arcs_70_3[3] = { - {98, 4}, - {51, 4}, + {99, 4}, + {52, 4}, {0, 3}, }; static arc arcs_70_4[2] = { - {32, 3}, + {33, 3}, {0, 4}, }; static state states_70[5] = { @@ -1551,15 +1604,15 @@ static state states_70[5] = { }; static arc arcs_71_0[3] = { {13, 1}, - {157, 2}, - {82, 3}, + {159, 2}, + {83, 3}, }; static arc arcs_71_1[2] = { {14, 4}, {15, 5}, }; static arc arcs_71_2[1] = { - {168, 6}, + {170, 6}, }; static arc arcs_71_3[1] = { {23, 5}, @@ -1571,7 +1624,7 @@ static arc arcs_71_5[1] = { {0, 5}, }; static arc arcs_71_6[1] = { - {158, 5}, + {160, 5}, }; static state states_71[7] = { {3, arcs_71_0}, @@ -1583,14 +1636,14 @@ static state states_71[7] = { {1, arcs_71_6}, }; static arc arcs_72_0[1] = { - {169, 1}, + {171, 1}, }; static arc arcs_72_1[2] = { - {32, 2}, + {33, 2}, {0, 1}, }; static arc arcs_72_2[2] = { - {169, 1}, + {171, 1}, {0, 2}, }; static state states_72[3] = { @@ -1608,11 +1661,11 @@ static arc arcs_73_1[2] = { }; static arc arcs_73_2[3] = { {26, 3}, - {170, 4}, + {172, 4}, {0, 2}, }; static arc arcs_73_3[2] = { - {170, 4}, + {172, 4}, {0, 3}, }; static arc arcs_73_4[1] = { @@ -1641,16 +1694,16 @@ static state states_74[3] = { {1, arcs_74_2}, }; static arc arcs_75_0[2] = { - {109, 1}, - {51, 1}, + {111, 1}, + {52, 1}, }; static arc arcs_75_1[2] = { - {32, 2}, + {33, 2}, {0, 1}, }; static arc arcs_75_2[3] = { - {109, 1}, - {51, 1}, + {111, 1}, + {52, 1}, {0, 2}, }; static state states_75[3] = { @@ -1662,7 +1715,7 @@ static arc arcs_76_0[1] = { {26, 1}, }; static arc arcs_76_1[2] = { - {32, 2}, + {33, 2}, {0, 1}, }; static arc arcs_76_2[2] = { @@ -1676,21 +1729,21 @@ static state states_76[3] = { }; static arc arcs_77_0[3] = { {26, 1}, - {34, 2}, - {51, 3}, + {35, 2}, + {52, 3}, }; static arc arcs_77_1[4] = { {27, 4}, - {167, 5}, - {32, 6}, + {169, 5}, + {33, 6}, {0, 1}, }; static arc arcs_77_2[1] = { - {109, 7}, + {111, 7}, }; static arc arcs_77_3[3] = { - {167, 5}, - {32, 6}, + {169, 5}, + {33, 6}, {0, 3}, }; static arc arcs_77_4[1] = { @@ -1701,34 +1754,34 @@ static arc arcs_77_5[1] = { }; static arc arcs_77_6[3] = { {26, 8}, - {51, 8}, + {52, 8}, {0, 6}, }; static arc arcs_77_7[3] = { - {167, 5}, - {32, 9}, + {169, 5}, + {33, 9}, {0, 7}, }; static arc arcs_77_8[2] = { - {32, 6}, + {33, 6}, {0, 8}, }; static arc arcs_77_9[3] = { {26, 10}, - {34, 11}, + {35, 11}, {0, 9}, }; static arc arcs_77_10[1] = { {27, 12}, }; static arc arcs_77_11[1] = { - {109, 13}, + {111, 13}, }; static arc arcs_77_12[1] = { {26, 13}, }; static arc arcs_77_13[2] = { - {32, 9}, + {33, 9}, {0, 13}, }; static state states_77[14] = { @@ -1748,7 +1801,7 @@ static state states_77[14] = { {2, arcs_77_13}, }; static arc arcs_78_0[1] = { - {171, 1}, + {173, 1}, }; static arc arcs_78_1[1] = { {23, 2}, @@ -1762,7 +1815,7 @@ static arc arcs_78_3[2] = { {15, 6}, }; static arc arcs_78_4[1] = { - {28, 7}, + {100, 7}, }; static arc arcs_78_5[1] = { {15, 6}, @@ -1784,14 +1837,14 @@ static state states_78[8] = { {1, arcs_78_7}, }; static arc arcs_79_0[1] = { - {172, 1}, + {174, 1}, }; static arc arcs_79_1[2] = { - {32, 2}, + {33, 2}, {0, 1}, }; static arc arcs_79_2[2] = { - {172, 1}, + {174, 1}, {0, 2}, }; static state states_79[3] = { @@ -1801,13 +1854,13 @@ static state states_79[3] = { }; static arc arcs_80_0[3] = { {26, 1}, + {35, 2}, {34, 2}, - {33, 2}, }; static arc arcs_80_1[4] = { - {167, 3}, - {113, 2}, - {31, 2}, + {169, 3}, + {115, 2}, + {32, 2}, {0, 1}, }; static arc arcs_80_2[1] = { @@ -1823,8 +1876,8 @@ static state states_80[4] = { {1, arcs_80_3}, }; static arc arcs_81_0[2] = { - {167, 1}, - {174, 1}, + {169, 1}, + {176, 1}, }; static arc arcs_81_1[1] = { {0, 1}, @@ -1834,19 +1887,19 @@ static state states_81[2] = { {1, arcs_81_1}, }; static arc arcs_82_0[1] = { - {102, 1}, + {104, 1}, }; static arc arcs_82_1[1] = { - {66, 2}, + {67, 2}, }; static arc arcs_82_2[1] = { - {103, 3}, + {105, 3}, }; static arc arcs_82_3[1] = { - {114, 4}, + {116, 4}, }; static arc arcs_82_4[2] = { - {173, 5}, + {175, 5}, {0, 4}, }; static arc arcs_82_5[1] = { @@ -1862,10 +1915,10 @@ static state states_82[6] = { }; static arc arcs_83_0[2] = { {21, 1}, - {175, 2}, + {177, 2}, }; static arc arcs_83_1[1] = { - {175, 2}, + {177, 2}, }; static arc arcs_83_2[1] = { {0, 2}, @@ -1876,13 +1929,13 @@ static state states_83[3] = { {1, arcs_83_2}, }; static arc arcs_84_0[1] = { - {97, 1}, + {98, 1}, }; static arc arcs_84_1[1] = { - {116, 2}, + {118, 2}, }; static arc arcs_84_2[2] = { - {173, 3}, + {175, 3}, {0, 2}, }; static arc arcs_84_3[1] = { @@ -1905,10 +1958,10 @@ static state states_85[2] = { {1, arcs_85_1}, }; static arc arcs_86_0[1] = { - {177, 1}, + {179, 1}, }; static arc arcs_86_1[2] = { - {178, 2}, + {180, 2}, {0, 1}, }; static arc arcs_86_2[1] = { @@ -1920,8 +1973,8 @@ static state states_86[3] = { {1, arcs_86_2}, }; static arc arcs_87_0[2] = { - {77, 1}, - {47, 2}, + {78, 1}, + {48, 2}, }; static arc arcs_87_1[1] = { {26, 2}, @@ -1934,13 +1987,148 @@ static state states_87[3] = { {1, arcs_87_1}, {1, arcs_87_2}, }; -static dfa dfas[88] = { +static arc arcs_88_0[2] = { + {3, 1}, + {2, 2}, +}; +static arc arcs_88_1[1] = { + {0, 1}, +}; +static arc arcs_88_2[2] = { + {28, 3}, + {113, 4}, +}; +static arc arcs_88_3[1] = { + {2, 5}, +}; +static arc arcs_88_4[1] = { + {6, 6}, +}; +static arc arcs_88_5[1] = { + {113, 4}, +}; +static arc arcs_88_6[2] = { + {6, 6}, + {114, 1}, +}; +static state states_88[7] = { + {2, arcs_88_0}, + {1, arcs_88_1}, + {2, arcs_88_2}, + {1, arcs_88_3}, + {1, arcs_88_4}, + {1, arcs_88_5}, + {2, arcs_88_6}, +}; +static arc arcs_89_0[1] = { + {182, 1}, +}; +static arc arcs_89_1[2] = { + {2, 1}, + {7, 2}, +}; +static arc arcs_89_2[1] = { + {0, 2}, +}; +static state states_89[3] = { + {1, arcs_89_0}, + {2, arcs_89_1}, + {1, arcs_89_2}, +}; +static arc arcs_90_0[1] = { + {13, 1}, +}; +static arc arcs_90_1[2] = { + {183, 2}, + {15, 3}, +}; +static arc arcs_90_2[1] = { + {15, 3}, +}; +static arc arcs_90_3[1] = { + {25, 4}, +}; +static arc arcs_90_4[1] = { + {26, 5}, +}; +static arc arcs_90_5[1] = { + {0, 5}, +}; +static state states_90[6] = { + {1, arcs_90_0}, + {2, arcs_90_1}, + {1, arcs_90_2}, + {1, arcs_90_3}, + {1, arcs_90_4}, + {1, arcs_90_5}, +}; +static arc arcs_91_0[3] = { + {26, 1}, + {34, 2}, + {35, 3}, +}; +static arc arcs_91_1[2] = { + {33, 4}, + {0, 1}, +}; +static arc arcs_91_2[3] = { + {26, 5}, + {33, 6}, + {0, 2}, +}; +static arc arcs_91_3[1] = { + {26, 7}, +}; +static arc arcs_91_4[4] = { + {26, 1}, + {34, 8}, + {35, 3}, + {0, 4}, +}; +static arc arcs_91_5[2] = { + {33, 6}, + {0, 5}, +}; +static arc arcs_91_6[2] = { + {26, 5}, + {35, 3}, +}; +static arc arcs_91_7[1] = { + {0, 7}, +}; +static arc arcs_91_8[3] = { + {26, 9}, + {33, 10}, + {0, 8}, +}; +static arc arcs_91_9[2] = { + {33, 10}, + {0, 9}, +}; +static arc arcs_91_10[2] = { + {26, 9}, + {35, 3}, +}; +static state states_91[11] = { + {3, arcs_91_0}, + {2, arcs_91_1}, + {3, arcs_91_2}, + {1, arcs_91_3}, + {4, arcs_91_4}, + {2, arcs_91_5}, + {2, arcs_91_6}, + {1, arcs_91_7}, + {3, arcs_91_8}, + {2, arcs_91_9}, + {2, arcs_91_10}, +}; +static dfa dfas[92] = { {256, "single_input", 0, 3, states_0, - "\004\050\340\000\002\000\000\000\012\076\011\007\142\011\100\010\000\000\103\242\174\010\002"}, + "\004\050\340\000\004\000\000\000\024\174\022\016\204\045\000\041\000\000\014\211\362\041\010"}, {257, "file_input", 0, 2, states_1, - "\204\050\340\000\002\000\000\000\012\076\011\007\142\011\100\010\000\000\103\242\174\010\002"}, + "\204\050\340\000\004\000\000\000\024\174\022\016\204\045\000\041\000\000\014\211\362\041\010"}, {258, "eval_input", 0, 3, states_2, - "\000\040\200\000\000\000\000\000\000\000\010\000\000\000\100\010\000\000\103\242\174\000\000"}, + "\000\040\200\000\000\000\000\000\000\000\020\000\000\000\000\041\000\000\014\211\362\001\000"}, {259, "decorator", 0, 7, states_3, "\000\010\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {260, "decorators", 0, 2, states_4, @@ -1949,54 +2137,54 @@ static dfa dfas[88] = { "\000\010\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {262, "async_funcdef", 0, 3, states_6, "\000\000\040\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, - {263, "funcdef", 0, 8, states_7, + {263, "funcdef", 0, 9, states_7, "\000\000\100\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {264, "parameters", 0, 4, states_8, "\000\040\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, - {265, "typedargslist", 0, 19, states_9, - "\000\000\200\000\006\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + {265, "typedargslist", 0, 23, states_9, + "\000\000\200\000\014\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {266, "tfpdef", 0, 4, states_10, "\000\000\200\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {267, "varargslist", 0, 19, states_11, - "\000\000\200\000\006\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + "\000\000\200\000\014\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {268, "vfpdef", 0, 2, states_12, "\000\000\200\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {269, "stmt", 0, 2, states_13, - "\000\050\340\000\002\000\000\000\012\076\011\007\142\011\100\010\000\000\103\242\174\010\002"}, + "\000\050\340\000\004\000\000\000\024\174\022\016\204\045\000\041\000\000\014\211\362\041\010"}, {270, "simple_stmt", 0, 4, states_14, - "\000\040\200\000\002\000\000\000\012\076\011\007\000\000\100\010\000\000\103\242\174\000\002"}, + "\000\040\200\000\004\000\000\000\024\174\022\016\000\000\000\041\000\000\014\211\362\001\010"}, {271, "small_stmt", 0, 2, states_15, - "\000\040\200\000\002\000\000\000\012\076\011\007\000\000\100\010\000\000\103\242\174\000\002"}, + "\000\040\200\000\004\000\000\000\024\174\022\016\000\000\000\041\000\000\014\211\362\001\010"}, {272, "expr_stmt", 0, 6, states_16, - "\000\040\200\000\002\000\000\000\000\000\010\000\000\000\100\010\000\000\103\242\174\000\000"}, + "\000\040\200\000\004\000\000\000\000\000\020\000\000\000\000\041\000\000\014\211\362\001\000"}, {273, "annassign", 0, 5, states_17, "\000\000\000\010\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {274, "testlist_star_expr", 0, 3, states_18, - "\000\040\200\000\002\000\000\000\000\000\010\000\000\000\100\010\000\000\103\242\174\000\000"}, + "\000\040\200\000\004\000\000\000\000\000\020\000\000\000\000\041\000\000\014\211\362\001\000"}, {275, "augassign", 0, 2, states_19, - "\000\000\000\000\000\000\360\377\001\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + "\000\000\000\000\000\000\340\377\003\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {276, "del_stmt", 0, 3, states_20, - "\000\000\000\000\000\000\000\000\002\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + "\000\000\000\000\000\000\000\000\004\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {277, "pass_stmt", 0, 2, states_21, - "\000\000\000\000\000\000\000\000\010\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + "\000\000\000\000\000\000\000\000\020\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {278, "flow_stmt", 0, 2, states_22, - "\000\000\000\000\000\000\000\000\000\036\000\000\000\000\000\000\000\000\000\000\000\000\002"}, + "\000\000\000\000\000\000\000\000\000\074\000\000\000\000\000\000\000\000\000\000\000\000\010"}, {279, "break_stmt", 0, 2, states_23, - "\000\000\000\000\000\000\000\000\000\002\000\000\000\000\000\000\000\000\000\000\000\000\000"}, - {280, "continue_stmt", 0, 2, states_24, "\000\000\000\000\000\000\000\000\000\004\000\000\000\000\000\000\000\000\000\000\000\000\000"}, - {281, "return_stmt", 0, 3, states_25, + {280, "continue_stmt", 0, 2, states_24, "\000\000\000\000\000\000\000\000\000\010\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + {281, "return_stmt", 0, 3, states_25, + "\000\000\000\000\000\000\000\000\000\020\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {282, "yield_stmt", 0, 2, states_26, - "\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\002"}, + "\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\010"}, {283, "raise_stmt", 0, 5, states_27, - "\000\000\000\000\000\000\000\000\000\020\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + "\000\000\000\000\000\000\000\000\000\040\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {284, "import_stmt", 0, 2, states_28, - "\000\000\000\000\000\000\000\000\000\040\001\000\000\000\000\000\000\000\000\000\000\000\000"}, + "\000\000\000\000\000\000\000\000\000\100\002\000\000\000\000\000\000\000\000\000\000\000\000"}, {285, "import_name", 0, 3, states_29, - "\000\000\000\000\000\000\000\000\000\000\001\000\000\000\000\000\000\000\000\000\000\000\000"}, + "\000\000\000\000\000\000\000\000\000\000\002\000\000\000\000\000\000\000\000\000\000\000\000"}, {286, "import_from", 0, 8, states_30, - "\000\000\000\000\000\000\000\000\000\040\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + "\000\000\000\000\000\000\000\000\000\100\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {287, "import_as_name", 0, 4, states_31, "\000\000\200\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {288, "dotted_as_name", 0, 4, states_32, @@ -2008,111 +2196,119 @@ static dfa dfas[88] = { {291, "dotted_name", 0, 2, states_35, "\000\000\200\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {292, "global_stmt", 0, 3, states_36, - "\000\000\000\000\000\000\000\000\000\000\000\001\000\000\000\000\000\000\000\000\000\000\000"}, - {293, "nonlocal_stmt", 0, 3, states_37, "\000\000\000\000\000\000\000\000\000\000\000\002\000\000\000\000\000\000\000\000\000\000\000"}, - {294, "assert_stmt", 0, 5, states_38, + {293, "nonlocal_stmt", 0, 3, states_37, "\000\000\000\000\000\000\000\000\000\000\000\004\000\000\000\000\000\000\000\000\000\000\000"}, + {294, "assert_stmt", 0, 5, states_38, + "\000\000\000\000\000\000\000\000\000\000\000\010\000\000\000\000\000\000\000\000\000\000\000"}, {295, "compound_stmt", 0, 2, states_39, - "\000\010\140\000\000\000\000\000\000\000\000\000\142\011\000\000\000\000\000\000\000\010\000"}, + "\000\010\140\000\000\000\000\000\000\000\000\000\204\045\000\000\000\000\000\000\000\040\000"}, {296, "async_stmt", 0, 3, states_40, "\000\000\040\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {297, "if_stmt", 0, 8, states_41, - "\000\000\000\000\000\000\000\000\000\000\000\000\002\000\000\000\000\000\000\000\000\000\000"}, + "\000\000\000\000\000\000\000\000\000\000\000\000\004\000\000\000\000\000\000\000\000\000\000"}, {298, "while_stmt", 0, 8, states_42, - "\000\000\000\000\000\000\000\000\000\000\000\000\040\000\000\000\000\000\000\000\000\000\000"}, - {299, "for_stmt", 0, 10, states_43, - "\000\000\000\000\000\000\000\000\000\000\000\000\100\000\000\000\000\000\000\000\000\000\000"}, - {300, "try_stmt", 0, 13, states_44, + "\000\000\000\000\000\000\000\000\000\000\000\000\200\000\000\000\000\000\000\000\000\000\000"}, + {299, "for_stmt", 0, 11, states_43, "\000\000\000\000\000\000\000\000\000\000\000\000\000\001\000\000\000\000\000\000\000\000\000"}, - {301, "with_stmt", 0, 5, states_45, - "\000\000\000\000\000\000\000\000\000\000\000\000\000\010\000\000\000\000\000\000\000\000\000"}, + {300, "try_stmt", 0, 13, states_44, + "\000\000\000\000\000\000\000\000\000\000\000\000\000\004\000\000\000\000\000\000\000\000\000"}, + {301, "with_stmt", 0, 6, states_45, + "\000\000\000\000\000\000\000\000\000\000\000\000\000\040\000\000\000\000\000\000\000\000\000"}, {302, "with_item", 0, 4, states_46, - "\000\040\200\000\000\000\000\000\000\000\010\000\000\000\100\010\000\000\103\242\174\000\000"}, + "\000\040\200\000\000\000\000\000\000\000\020\000\000\000\000\041\000\000\014\211\362\001\000"}, {303, "except_clause", 0, 5, states_47, - "\000\000\000\000\000\000\000\000\000\000\000\000\000\100\000\000\000\000\000\000\000\000\000"}, + "\000\000\000\000\000\000\000\000\000\000\000\000\000\000\001\000\000\000\000\000\000\000\000"}, {304, "suite", 0, 5, states_48, - "\004\040\200\000\002\000\000\000\012\076\011\007\000\000\100\010\000\000\103\242\174\000\002"}, + "\004\040\200\000\004\000\000\000\024\174\022\016\000\000\000\041\000\000\014\211\362\001\010"}, {305, "namedexpr_test", 0, 4, states_49, - "\000\040\200\000\000\000\000\000\000\000\010\000\000\000\100\010\000\000\103\242\174\000\000"}, + "\000\040\200\000\000\000\000\000\000\000\020\000\000\000\000\041\000\000\014\211\362\001\000"}, {306, "test", 0, 6, states_50, - "\000\040\200\000\000\000\000\000\000\000\010\000\000\000\100\010\000\000\103\242\174\000\000"}, + "\000\040\200\000\000\000\000\000\000\000\020\000\000\000\000\041\000\000\014\211\362\001\000"}, {307, "test_nocond", 0, 2, states_51, - "\000\040\200\000\000\000\000\000\000\000\010\000\000\000\100\010\000\000\103\242\174\000\000"}, + "\000\040\200\000\000\000\000\000\000\000\020\000\000\000\000\041\000\000\014\211\362\001\000"}, {308, "lambdef", 0, 5, states_52, - "\000\000\000\000\000\000\000\000\000\000\000\000\000\000\100\000\000\000\000\000\000\000\000"}, + "\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\001\000\000\000\000\000\000\000"}, {309, "lambdef_nocond", 0, 5, states_53, - "\000\000\000\000\000\000\000\000\000\000\000\000\000\000\100\000\000\000\000\000\000\000\000"}, + "\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\001\000\000\000\000\000\000\000"}, {310, "or_test", 0, 2, states_54, - "\000\040\200\000\000\000\000\000\000\000\010\000\000\000\000\010\000\000\103\242\174\000\000"}, + "\000\040\200\000\000\000\000\000\000\000\020\000\000\000\000\040\000\000\014\211\362\001\000"}, {311, "and_test", 0, 2, states_55, - "\000\040\200\000\000\000\000\000\000\000\010\000\000\000\000\010\000\000\103\242\174\000\000"}, + "\000\040\200\000\000\000\000\000\000\000\020\000\000\000\000\040\000\000\014\211\362\001\000"}, {312, "not_test", 0, 3, states_56, - "\000\040\200\000\000\000\000\000\000\000\010\000\000\000\000\010\000\000\103\242\174\000\000"}, + "\000\040\200\000\000\000\000\000\000\000\020\000\000\000\000\040\000\000\014\211\362\001\000"}, {313, "comparison", 0, 2, states_57, - "\000\040\200\000\000\000\000\000\000\000\010\000\000\000\000\000\000\000\103\242\174\000\000"}, + "\000\040\200\000\000\000\000\000\000\000\020\000\000\000\000\000\000\000\014\211\362\001\000"}, {314, "comp_op", 0, 4, states_58, - "\000\000\000\000\000\000\000\000\000\000\000\000\200\000\000\310\077\000\000\000\000\000\000"}, + "\000\000\000\000\000\000\000\000\000\000\000\000\000\002\000\040\377\000\000\000\000\000\000"}, {315, "star_expr", 0, 3, states_59, - "\000\000\000\000\002\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + "\000\000\000\000\004\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {316, "expr", 0, 2, states_60, - "\000\040\200\000\000\000\000\000\000\000\010\000\000\000\000\000\000\000\103\242\174\000\000"}, + "\000\040\200\000\000\000\000\000\000\000\020\000\000\000\000\000\000\000\014\211\362\001\000"}, {317, "xor_expr", 0, 2, states_61, - "\000\040\200\000\000\000\000\000\000\000\010\000\000\000\000\000\000\000\103\242\174\000\000"}, + "\000\040\200\000\000\000\000\000\000\000\020\000\000\000\000\000\000\000\014\211\362\001\000"}, {318, "and_expr", 0, 2, states_62, - "\000\040\200\000\000\000\000\000\000\000\010\000\000\000\000\000\000\000\103\242\174\000\000"}, + "\000\040\200\000\000\000\000\000\000\000\020\000\000\000\000\000\000\000\014\211\362\001\000"}, {319, "shift_expr", 0, 2, states_63, - "\000\040\200\000\000\000\000\000\000\000\010\000\000\000\000\000\000\000\103\242\174\000\000"}, + "\000\040\200\000\000\000\000\000\000\000\020\000\000\000\000\000\000\000\014\211\362\001\000"}, {320, "arith_expr", 0, 2, states_64, - "\000\040\200\000\000\000\000\000\000\000\010\000\000\000\000\000\000\000\103\242\174\000\000"}, + "\000\040\200\000\000\000\000\000\000\000\020\000\000\000\000\000\000\000\014\211\362\001\000"}, {321, "term", 0, 2, states_65, - "\000\040\200\000\000\000\000\000\000\000\010\000\000\000\000\000\000\000\103\242\174\000\000"}, + "\000\040\200\000\000\000\000\000\000\000\020\000\000\000\000\000\000\000\014\211\362\001\000"}, {322, "factor", 0, 3, states_66, - "\000\040\200\000\000\000\000\000\000\000\010\000\000\000\000\000\000\000\103\242\174\000\000"}, + "\000\040\200\000\000\000\000\000\000\000\020\000\000\000\000\000\000\000\014\211\362\001\000"}, {323, "power", 0, 4, states_67, - "\000\040\200\000\000\000\000\000\000\000\010\000\000\000\000\000\000\000\000\242\174\000\000"}, + "\000\040\200\000\000\000\000\000\000\000\020\000\000\000\000\000\000\000\000\210\362\001\000"}, {324, "atom_expr", 0, 3, states_68, - "\000\040\200\000\000\000\000\000\000\000\010\000\000\000\000\000\000\000\000\242\174\000\000"}, + "\000\040\200\000\000\000\000\000\000\000\020\000\000\000\000\000\000\000\000\210\362\001\000"}, {325, "atom", 0, 9, states_69, - "\000\040\200\000\000\000\000\000\000\000\010\000\000\000\000\000\000\000\000\240\174\000\000"}, + "\000\040\200\000\000\000\000\000\000\000\020\000\000\000\000\000\000\000\000\200\362\001\000"}, {326, "testlist_comp", 0, 5, states_70, - "\000\040\200\000\002\000\000\000\000\000\010\000\000\000\100\010\000\000\103\242\174\000\000"}, + "\000\040\200\000\004\000\000\000\000\000\020\000\000\000\000\041\000\000\014\211\362\001\000"}, {327, "trailer", 0, 7, states_71, - "\000\040\000\000\000\000\000\000\000\000\004\000\000\000\000\000\000\000\000\040\000\000\000"}, + "\000\040\000\000\000\000\000\000\000\000\010\000\000\000\000\000\000\000\000\200\000\000\000"}, {328, "subscriptlist", 0, 3, states_72, - "\000\040\200\010\000\000\000\000\000\000\010\000\000\000\100\010\000\000\103\242\174\000\000"}, + "\000\040\200\010\000\000\000\000\000\000\020\000\000\000\000\041\000\000\014\211\362\001\000"}, {329, "subscript", 0, 5, states_73, - "\000\040\200\010\000\000\000\000\000\000\010\000\000\000\100\010\000\000\103\242\174\000\000"}, + "\000\040\200\010\000\000\000\000\000\000\020\000\000\000\000\041\000\000\014\211\362\001\000"}, {330, "sliceop", 0, 3, states_74, "\000\000\000\010\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {331, "exprlist", 0, 3, states_75, - "\000\040\200\000\002\000\000\000\000\000\010\000\000\000\000\000\000\000\103\242\174\000\000"}, + "\000\040\200\000\004\000\000\000\000\000\020\000\000\000\000\000\000\000\014\211\362\001\000"}, {332, "testlist", 0, 3, states_76, - "\000\040\200\000\000\000\000\000\000\000\010\000\000\000\100\010\000\000\103\242\174\000\000"}, + "\000\040\200\000\000\000\000\000\000\000\020\000\000\000\000\041\000\000\014\211\362\001\000"}, {333, "dictorsetmaker", 0, 14, states_77, - "\000\040\200\000\006\000\000\000\000\000\010\000\000\000\100\010\000\000\103\242\174\000\000"}, + "\000\040\200\000\014\000\000\000\000\000\020\000\000\000\000\041\000\000\014\211\362\001\000"}, {334, "classdef", 0, 8, states_78, - "\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\010\000"}, + "\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\040\000"}, {335, "arglist", 0, 3, states_79, - "\000\040\200\000\006\000\000\000\000\000\010\000\000\000\100\010\000\000\103\242\174\000\000"}, + "\000\040\200\000\014\000\000\000\000\000\020\000\000\000\000\041\000\000\014\211\362\001\000"}, {336, "argument", 0, 4, states_80, - "\000\040\200\000\006\000\000\000\000\000\010\000\000\000\100\010\000\000\103\242\174\000\000"}, + "\000\040\200\000\014\000\000\000\000\000\020\000\000\000\000\041\000\000\014\211\362\001\000"}, {337, "comp_iter", 0, 2, states_81, - "\000\000\040\000\000\000\000\000\000\000\000\000\102\000\000\000\000\000\000\000\000\000\000"}, + "\000\000\040\000\000\000\000\000\000\000\000\000\004\001\000\000\000\000\000\000\000\000\000"}, {338, "sync_comp_for", 0, 6, states_82, - "\000\000\000\000\000\000\000\000\000\000\000\000\100\000\000\000\000\000\000\000\000\000\000"}, + "\000\000\000\000\000\000\000\000\000\000\000\000\000\001\000\000\000\000\000\000\000\000\000"}, {339, "comp_for", 0, 3, states_83, - "\000\000\040\000\000\000\000\000\000\000\000\000\100\000\000\000\000\000\000\000\000\000\000"}, + "\000\000\040\000\000\000\000\000\000\000\000\000\000\001\000\000\000\000\000\000\000\000\000"}, {340, "comp_if", 0, 4, states_84, - "\000\000\000\000\000\000\000\000\000\000\000\000\002\000\000\000\000\000\000\000\000\000\000"}, + "\000\000\000\000\000\000\000\000\000\000\000\000\004\000\000\000\000\000\000\000\000\000\000"}, {341, "encoding_decl", 0, 2, states_85, "\000\000\200\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {342, "yield_expr", 0, 3, states_86, - "\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\002"}, + "\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\010"}, {343, "yield_arg", 0, 3, states_87, - "\000\040\200\000\002\000\000\000\000\040\010\000\000\000\100\010\000\000\103\242\174\000\000"}, + "\000\040\200\000\004\000\000\000\000\100\020\000\000\000\000\041\000\000\014\211\362\001\000"}, + {344, "func_body_suite", 0, 7, states_88, + "\004\040\200\000\004\000\000\000\024\174\022\016\000\000\000\041\000\000\014\211\362\001\010"}, + {345, "func_type_input", 0, 3, states_89, + "\000\040\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + {346, "func_type", 0, 6, states_90, + "\000\040\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + {347, "typelist", 0, 11, states_91, + "\000\040\200\000\014\000\000\000\000\000\020\000\000\000\000\041\000\000\014\211\362\001\000"}, }; -static label labels[179] = { +static label labels[184] = { {0, "EMPTY"}, {256, 0}, {4, 0}, @@ -2141,7 +2337,8 @@ static label labels[179] = { {51, 0}, {306, 0}, {11, 0}, - {304, 0}, + {56, 0}, + {344, 0}, {265, 0}, {266, 0}, {22, 0}, @@ -2212,6 +2409,7 @@ static label labels[179] = { {296, 0}, {1, "if"}, {305, 0}, + {304, 0}, {1, "elif"}, {1, "else"}, {1, "while"}, @@ -2292,10 +2490,13 @@ static label labels[179] = { {341, 0}, {1, "yield"}, {343, 0}, + {345, 0}, + {346, 0}, + {347, 0}, }; grammar _PyParser_Grammar = { - 88, + 92, dfas, - {179, labels}, + {184, labels}, 256 }; diff --git a/Python/pythonrun.c b/Python/pythonrun.c index 9b6371d9c0da..c7a622c83d37 100644 --- a/Python/pythonrun.c +++ b/Python/pythonrun.c @@ -158,6 +158,8 @@ static int PARSER_FLAGS(PyCompilerFlags *flags) parser_flags |= PyPARSE_IGNORE_COOKIE; if (flags->cf_flags & CO_FUTURE_BARRY_AS_BDFL) parser_flags |= PyPARSE_BARRY_AS_BDFL; + if (flags->cf_flags & PyCF_TYPE_COMMENTS) + parser_flags |= PyPARSE_TYPE_COMMENTS; return parser_flags; }