[Python-checkins] bpo-44201: Avoid side effects of "invalid_*" rules in the REPL (GH-26298) (GH-26313)

[pablogsal]

https://github.com/python/cpython/commit/1fb6b9e91d8685f7eca0fc33402589c65723bd94
commit: 1fb6b9e91d8685f7eca0fc33402589c65723bd94
branch: 3.10
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: pablogsal <Pablogsal at gmail.com>
date: 2021-05-22T23:23:26+01:00
summary:

bpo-44201: Avoid side effects of "invalid_*" rules in the REPL (GH-26298) (GH-26313)

When the parser does a second pass to check for errors, these rules can
have some small side-effects as they may advance the parser more than
the point reached in the first pass. This can cause the tokenizer to ask
for extra tokens in interactive mode causing the tokenizer to show the
prompt instead of failing instantly.

To avoid this, add a new mode to the tokenizer that is activated in the
second pass and deactivates asking for new tokens when the interactive
line is finished. As the parsing should have reached the last line in
the first pass, the second pass should not need to ask for more tokens.

(cherry picked from commit bd7476dae337e905e7b1bbf33ddb96cc270fdc84)

Co-authored-by: Pablo Galindo <Pablogsal at gmail.com>

files:
A Misc/NEWS.d/next/Core and Builtins/2021-05-21-21-16-03.bpo-44201.bGaSjt.rst
M Include/errcode.h
M Parser/pegen.c
M Parser/tokenizer.c
M Parser/tokenizer.h

diff --git a/Include/errcode.h b/Include/errcode.h
index f2671d6c9b30b..2e07fc2c96328 100644
--- a/Include/errcode.h
+++ b/Include/errcode.h
@@ -28,6 +28,7 @@ extern "C" {
 #define E_DECODE        22      /* Error in decoding into Unicode */
 #define E_LINECONT      25      /* Unexpected characters after a line continuation */
 #define E_BADSINGLE     27      /* Ill-formed single statement input */
+#define E_INTERACT_STOP 28      /* Interactive mode stopped tokenization */
 
 #ifdef __cplusplus
 }
diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-05-21-21-16-03.bpo-44201.bGaSjt.rst b/Misc/NEWS.d/next/Core and Builtins/2021-05-21-21-16-03.bpo-44201.bGaSjt.rst
new file mode 100644
index 0000000000000..6f61aac7817b2
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and Builtins/2021-05-21-21-16-03.bpo-44201.bGaSjt.rst	
@@ -0,0 +1,3 @@
+Avoid side effects of checking for specialized syntax errors in the REPL
+that was causing it to ask for extra tokens after a syntax error had been
+detected. Patch by Pablo Galindo
diff --git a/Parser/pegen.c b/Parser/pegen.c
index 3c25e4dbae0ff..548a64788dec4 100644
--- a/Parser/pegen.c
+++ b/Parser/pegen.c
@@ -1234,6 +1234,9 @@ reset_parser_state(Parser *p)
     }
     p->mark = 0;
     p->call_invalid_rules = 1;
+    // Don't try to get extra tokens in interactive mode when trying to
+    // raise specialized errors in the second pass.
+    p->tok->interactive_underflow = IUNDERFLOW_STOP;
 }
 
 static int
diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c
index ad32293d70b78..a86af9bc0620c 100644
--- a/Parser/tokenizer.c
+++ b/Parser/tokenizer.c
@@ -85,6 +85,7 @@ tok_new(void)
     tok->async_def = 0;
     tok->async_def_indent = 0;
     tok->async_def_nl = 0;
+    tok->interactive_underflow = IUNDERFLOW_NORMAL;
 
     return tok;
 }
@@ -845,6 +846,10 @@ tok_underflow_string(struct tok_state *tok) {
 
 static int
 tok_underflow_interactive(struct tok_state *tok) {
+    if (tok->interactive_underflow == IUNDERFLOW_STOP) {
+        tok->done = E_INTERACT_STOP;
+        return 1;
+    }
     char *newtok = PyOS_Readline(stdin, stdout, tok->prompt);
     if (newtok != NULL) {
         char *translated = translate_newlines(newtok, 0, tok);
@@ -1399,6 +1404,10 @@ tok_get(struct tok_state *tok, const char **p_start, const char **p_end)
         }
     }
 
+    if (tok->done == E_INTERACT_STOP) {
+        return ENDMARKER;
+    }
+
     /* Check for EOF and errors now */
     if (c == EOF) {
         if (tok->level) {
diff --git a/Parser/tokenizer.h b/Parser/tokenizer.h
index aaa31f3796207..ff563d57fa8b1 100644
--- a/Parser/tokenizer.h
+++ b/Parser/tokenizer.h
@@ -19,6 +19,14 @@ enum decoding_state {
     STATE_NORMAL
 };
 
+enum interactive_underflow_t {
+    /* Normal mode of operation: return a new token when asked in interactie mode */
+    IUNDERFLOW_NORMAL,
+    /* Forcefully return ENDMARKER when asked for a new token in interactive mode. This
+     * can be used to prevent the tokenizer to promt the user for new tokens */
+    IUNDERFLOW_STOP,
+};
+
 /* Tokenizer state */
 struct tok_state {
     /* Input state; buf <= cur <= inp <= end */
@@ -74,6 +82,8 @@ struct tok_state {
     int async_def_indent; /* Indentation level of the outermost 'async def'. */
     int async_def_nl;     /* =1 if the outermost 'async def' had at least one
                              NEWLINE token after it. */
+    /* How to proceed when asked for a new token in interactive mode */
+    enum interactive_underflow_t interactive_underflow; 
 };
 
 extern struct tok_state *PyTokenizer_FromString(const char *, int);