https://github.com/python/cpython/commit/2f8c22f1d6c22f018c78264937db66d52fb... commit: 2f8c22f1d6c22f018c78264937db66d52fb18869 branch: 3.12 author: Miss Islington (bot) <31488909+miss-islington@users.noreply.github.com> committer: pablogsal <Pablogsal@gmail.com> date: 2023-05-31T11:11:39+01:00 summary: [3.12] gh-105042: Disable unmatched parens syntax error in python tokenize (GH-105061) (#105120) gh-105042: Disable unmatched parens syntax error in python tokenize (GH-105061) (cherry picked from commit 70f315c2d6de87b0514ce16cc00a91a5b60a6098) Co-authored-by: Lysandros Nikolaou <lisandrosnik@gmail.com> files: M Lib/test/inspect_fodder.py M Lib/test/test_inspect.py M Lib/test/test_tokenize.py M Parser/tokenizer.c M Python/Python-tokenize.c diff --git a/Lib/test/inspect_fodder.py b/Lib/test/inspect_fodder.py index 567dfbab80486..60ba7aa78394e 100644 --- a/Lib/test/inspect_fodder.py +++ b/Lib/test/inspect_fodder.py @@ -113,3 +113,8 @@ async def asyncf(self): # after asyncf - line 113 # end of WhichComments - line 114 # after WhichComments - line 115 + +# Test that getsource works on a line that includes +# a closing parenthesis with the opening paren being in another line +( +); after_closing = lambda: 1 diff --git a/Lib/test/test_inspect.py b/Lib/test/test_inspect.py index a7bd680d0f5bc..6a49e3b5530e1 100644 --- a/Lib/test/test_inspect.py +++ b/Lib/test/test_inspect.py @@ -557,7 +557,8 @@ def test_getclasses(self): def test_getfunctions(self): functions = inspect.getmembers(mod, inspect.isfunction) - self.assertEqual(functions, [('eggs', mod.eggs), + self.assertEqual(functions, [('after_closing', mod.after_closing), + ('eggs', mod.eggs), ('lobbest', mod.lobbest), ('spam', mod.spam)]) @@ -641,6 +642,7 @@ def test_getsource(self): self.assertSourceEqual(git.abuse, 29, 39) self.assertSourceEqual(mod.StupidGit, 21, 51) self.assertSourceEqual(mod.lobbest, 75, 76) + self.assertSourceEqual(mod.after_closing, 120, 120) def test_getsourcefile(self): self.assertEqual(normcase(inspect.getsourcefile(mod.spam)), modfile) diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py index cd11dddd0fe51..3adcc4e420671 100644 --- a/Lib/test/test_tokenize.py +++ b/Lib/test/test_tokenize.py @@ -1100,6 +1100,13 @@ def test_newline_after_parenthesized_block_with_comment(self): NEWLINE '\\n' (4, 1) (4, 2) """) + def test_closing_parenthesis_from_different_line(self): + self.check_tokenize("); x", """\ + OP ')' (1, 0) (1, 1) + OP ';' (1, 1) (1, 2) + NAME 'x' (1, 3) (1, 4) + """) + class GenerateTokensTest(TokenizeTest): def check_tokenize(self, s, expected): # Format the tokens in s in a table format. diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c index 9058c67ada648..b6d63e150cfbc 100644 --- a/Parser/tokenizer.c +++ b/Parser/tokenizer.c @@ -2496,41 +2496,42 @@ tok_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct t case ')': case ']': case '}': - if (!tok->level) { - if (INSIDE_FSTRING(tok) && !current_tok->curly_bracket_depth && c == '}') { - return MAKE_TOKEN(syntaxerror(tok, "f-string: single '}' is not allowed")); - } + if (INSIDE_FSTRING(tok) && !current_tok->curly_bracket_depth && c == '}') { + return MAKE_TOKEN(syntaxerror(tok, "f-string: single '}' is not allowed")); + } + if (!tok->tok_extra_tokens && !tok->level) { return MAKE_TOKEN(syntaxerror(tok, "unmatched '%c'", c)); } - tok->level--; - int opening = tok->parenstack[tok->level]; - if (!((opening == '(' && c == ')') || - (opening == '[' && c == ']') || - (opening == '{' && c == '}'))) - { - /* If the opening bracket belongs to an f-string's expression - part (e.g. f"{)}") and the closing bracket is an arbitrary - nested expression, then instead of matching a different - syntactical construct with it; we'll throw an unmatched - parentheses error. */ - if (INSIDE_FSTRING(tok) && opening == '{') { - assert(current_tok->curly_bracket_depth >= 0); - int previous_bracket = current_tok->curly_bracket_depth - 1; - if (previous_bracket == current_tok->curly_bracket_expr_start_depth) { - return MAKE_TOKEN(syntaxerror(tok, "f-string: unmatched '%c'", c)); + if (tok->level > 0) { + tok->level--; + int opening = tok->parenstack[tok->level]; + if (!tok->tok_extra_tokens && !((opening == '(' && c == ')') || + (opening == '[' && c == ']') || + (opening == '{' && c == '}'))) { + /* If the opening bracket belongs to an f-string's expression + part (e.g. f"{)}") and the closing bracket is an arbitrary + nested expression, then instead of matching a different + syntactical construct with it; we'll throw an unmatched + parentheses error. */ + if (INSIDE_FSTRING(tok) && opening == '{') { + assert(current_tok->curly_bracket_depth >= 0); + int previous_bracket = current_tok->curly_bracket_depth - 1; + if (previous_bracket == current_tok->curly_bracket_expr_start_depth) { + return MAKE_TOKEN(syntaxerror(tok, "f-string: unmatched '%c'", c)); + } + } + if (tok->parenlinenostack[tok->level] != tok->lineno) { + return MAKE_TOKEN(syntaxerror(tok, + "closing parenthesis '%c' does not match " + "opening parenthesis '%c' on line %d", + c, opening, tok->parenlinenostack[tok->level])); + } + else { + return MAKE_TOKEN(syntaxerror(tok, + "closing parenthesis '%c' does not match " + "opening parenthesis '%c'", + c, opening)); } - } - if (tok->parenlinenostack[tok->level] != tok->lineno) { - return MAKE_TOKEN(syntaxerror(tok, - "closing parenthesis '%c' does not match " - "opening parenthesis '%c' on line %d", - c, opening, tok->parenlinenostack[tok->level])); - } - else { - return MAKE_TOKEN(syntaxerror(tok, - "closing parenthesis '%c' does not match " - "opening parenthesis '%c'", - c, opening)); } } diff --git a/Python/Python-tokenize.c b/Python/Python-tokenize.c index 4eced66b61770..2de1daae8c0dd 100644 --- a/Python/Python-tokenize.c +++ b/Python/Python-tokenize.c @@ -82,7 +82,7 @@ _tokenizer_error(struct tok_state *tok) msg = "invalid token"; break; case E_EOF: - if (tok->level) { + if (tok->level > 0) { PyErr_Format(PyExc_SyntaxError, "parenthesis '%c' was never closed", tok->parenstack[tok->level-1]);