[Python-checkins] bpo-39209: Manage correctly multi-line tokens in interactive mode (GH-17860)

Miss Islington (bot) webhook-mailer at python.org
Mon Jan 6 11:26:18 EST 2020


https://github.com/python/cpython/commit/b2e281aaa2e4a1f24671d293dfd06b23bb052e47
commit: b2e281aaa2e4a1f24671d293dfd06b23bb052e47
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2020-01-06T08:26:13-08:00
summary:

bpo-39209: Manage correctly multi-line tokens in interactive mode (GH-17860)

(cherry picked from commit 5ec91f78d59d9c39b984f284e00cd04b96ddb5db)

Co-authored-by: Pablo Galindo <Pablogsal at gmail.com>

files:
A Misc/NEWS.d/next/Core and Builtins/2020-01-06-10-29-16.bpo-39209.QHAONe.rst
M Lib/test/test_repl.py
M Parser/tokenizer.c

diff --git a/Lib/test/test_repl.py b/Lib/test/test_repl.py
index 9efd459a6f076..71f192f90d9a1 100644
--- a/Lib/test/test_repl.py
+++ b/Lib/test/test_repl.py
@@ -58,5 +58,41 @@ def test_no_memory(self):
         # Exit code 120: Py_FinalizeEx() failed to flush stdout and stderr.
         self.assertIn(p.returncode, (1, 120))
 
+    @cpython_only
+    def test_multiline_string_parsing(self):
+        # bpo-39209: Multiline string tokens need to be handled in the tokenizer
+        # in two places: the interactive path and the non-interactive path.
+        user_input = '''\
+        x = """<?xml version="1.0" encoding="iso-8859-1"?>
+        <test>
+            <Users>
+                <fun25>
+                    <limits>
+                        <total>0KiB</total>
+                        <kbps>0</kbps>
+                        <rps>1.3</rps>
+                        <connections>0</connections>
+                    </limits>
+                    <usages>
+                        <total>16738211KiB</total>
+                        <kbps>237.15</kbps>
+                        <rps>1.3</rps>
+                        <connections>0</connections>
+                    </usages>
+                    <time_to_refresh>never</time_to_refresh>
+                    <limit_exceeded_URL>none</limit_exceeded_URL>
+                </fun25>
+            </Users>
+        </test>"""
+        '''
+        user_input = dedent(user_input)
+        user_input = user_input.encode()
+        p = spawn_repl()
+        with SuppressCrashReport():
+            p.stdin.write(user_input)
+        output = kill_python(p)
+        self.assertEqual(p.returncode, 0)
+
+
 if __name__ == "__main__":
     unittest.main()
diff --git a/Misc/NEWS.d/next/Core and Builtins/2020-01-06-10-29-16.bpo-39209.QHAONe.rst b/Misc/NEWS.d/next/Core and Builtins/2020-01-06-10-29-16.bpo-39209.QHAONe.rst
new file mode 100644
index 0000000000000..c05b3f8dfa4d4
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and Builtins/2020-01-06-10-29-16.bpo-39209.QHAONe.rst	
@@ -0,0 +1,2 @@
+Correctly handle multi-line tokens in interactive mode. Patch by Pablo
+Galindo.
diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c
index f84093dae5b62..f73c32684c7b7 100644
--- a/Parser/tokenizer.c
+++ b/Parser/tokenizer.c
@@ -886,6 +886,7 @@ tok_nextc(struct tok_state *tok)
                 size_t start = tok->start - tok->buf;
                 size_t oldlen = tok->cur - tok->buf;
                 size_t newlen = oldlen + strlen(newtok);
+                Py_ssize_t cur_multi_line_start = tok->multi_line_start - tok->buf;
                 char *buf = tok->buf;
                 buf = (char *)PyMem_REALLOC(buf, newlen+1);
                 tok->lineno++;
@@ -898,6 +899,7 @@ tok_nextc(struct tok_state *tok)
                 }
                 tok->buf = buf;
                 tok->cur = tok->buf + oldlen;
+                tok->multi_line_start = tok->buf + cur_multi_line_start;
                 tok->line_start = tok->cur;
                 strcpy(tok->buf + oldlen, newtok);
                 PyMem_FREE(newtok);



More information about the Python-checkins mailing list