Mailman 3 May 2022 - Python-checkins

gh-92597: Ensure that AST nodes without explicit end positions can be compiled (GH-93359)
by miss-islington May 31, 2022

May 31, 2022

https://github.com/python/cpython/commit/b425d887aa51c8e7900b08cb8df457f450… commit: b425d887aa51c8e7900b08cb8df457f450f6fbfd branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: miss-islington <31488909+miss-islington(a)users.noreply.github.com> date: 2022-05-31T16:26:16-07:00 summary: gh-92597: Ensure that AST nodes without explicit end positions can be compiled (GH-93359) (cherry picked from commit 705eaec28f7bee530b1c1635ba385a49a1feaf32) Co-authored-by: Pablo Galindo Salgado <Pablogsal(a)gmail.com> files: A Misc/NEWS.d/next/Core and Builtins/2022-05-30-19-00-38.gh-issue-93359.zXV3A0.rst M Lib/test/test_ast.py M Parser/asdl_c.py M Python/Python-ast.c diff --git a/Lib/test/test_ast.py b/Lib/test/test_ast.py index 896eb5bedd7f3..33df22cb35a9e 100644 --- a/Lib/test/test_ast.py +++ b/Lib/test/test_ast.py @@ -362,6 +362,14 @@ def test_invalid_position_information(self): with self.assertRaises(ValueError): compile(tree, '<string>', 'exec') + def test_compilation_of_ast_nodes_with_default_end_position_values(self): + tree = ast.Module(body=[ + ast.Import(names=[ast.alias(name='builtins', lineno=1, col_offset=0)], lineno=1, col_offset=0), + ast.Import(names=[ast.alias(name='traceback', lineno=0, col_offset=0)], lineno=0, col_offset=1) + ], type_ignores=[]) + + # Check that compilation doesn't crash. Note: this may crash explicitly only on debug mode. + compile(tree, "<string>", "exec") def test_slice(self): slc = ast.parse("x[::]").body[0].value.slice diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-05-30-19-00-38.gh-issue-93359.zXV3A0.rst b/Misc/NEWS.d/next/Core and Builtins/2022-05-30-19-00-38.gh-issue-93359.zXV3A0.rst new file mode 100644 index 0000000000000..36e5e52390d7b --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2022-05-30-19-00-38.gh-issue-93359.zXV3A0.rst @@ -0,0 +1,2 @@ +Ensure that custom :mod:`ast` nodes without explicit end positions can be +compiled. Patch by Pablo Galindo. diff --git a/Parser/asdl_c.py b/Parser/asdl_c.py index 3bfe320fe3b70..1101a3593dfe2 100755 --- a/Parser/asdl_c.py +++ b/Parser/asdl_c.py @@ -488,6 +488,12 @@ def visitProduct(self, prod, name): class Obj2ModVisitor(PickleVisitor): + + attribute_special_defaults = { + "end_lineno": "lineno", + "end_col_offset": "col_offset", + } + @contextmanager def recursive_call(self, node, level): self.emit('if (_Py_EnterRecursiveCall(" while traversing \'%s\' node")) {' % node, level, reflow=False) @@ -637,7 +643,13 @@ def visitField(self, field, name, sum=None, prod=None, depth=0): self.emit("if (tmp == NULL || tmp == Py_None) {", depth) self.emit("Py_CLEAR(tmp);", depth+1) if self.isNumeric(field): - self.emit("%s = 0;" % field.name, depth+1) + if field.name in self.attribute_special_defaults: + self.emit( + "%s = %s;" % (field.name, self.attribute_special_defaults[field.name]), + depth+1, + ) + else: + self.emit("%s = 0;" % field.name, depth+1) elif not self.isSimpleType(field): self.emit("%s = NULL;" % field.name, depth+1) else: diff --git a/Python/Python-ast.c b/Python/Python-ast.c index 3861eaf978a38..e52a72d43bcbd 100644 --- a/Python/Python-ast.c +++ b/Python/Python-ast.c @@ -5697,7 +5697,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* } if (tmp == NULL || tmp == Py_None) { Py_CLEAR(tmp); - end_lineno = 0; + end_lineno = lineno; } else { int res; @@ -5714,7 +5714,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* } if (tmp == NULL || tmp == Py_None) { Py_CLEAR(tmp); - end_col_offset = 0; + end_col_offset = col_offset; } else { int res; @@ -8114,7 +8114,7 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* } if (tmp == NULL || tmp == Py_None) { Py_CLEAR(tmp); - end_lineno = 0; + end_lineno = lineno; } else { int res; @@ -8131,7 +8131,7 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* } if (tmp == NULL || tmp == Py_None) { Py_CLEAR(tmp); - end_col_offset = 0; + end_col_offset = col_offset; } else { int res; @@ -10291,7 +10291,7 @@ obj2ast_excepthandler(struct ast_state *state, PyObject* obj, excepthandler_ty* } if (tmp == NULL || tmp == Py_None) { Py_CLEAR(tmp); - end_lineno = 0; + end_lineno = lineno; } else { int res; @@ -10308,7 +10308,7 @@ obj2ast_excepthandler(struct ast_state *state, PyObject* obj, excepthandler_ty* } if (tmp == NULL || tmp == Py_None) { Py_CLEAR(tmp); - end_col_offset = 0; + end_col_offset = col_offset; } else { int res; @@ -10755,7 +10755,7 @@ obj2ast_arg(struct ast_state *state, PyObject* obj, arg_ty* out, PyArena* arena) } if (tmp == NULL || tmp == Py_None) { Py_CLEAR(tmp); - end_lineno = 0; + end_lineno = lineno; } else { int res; @@ -10772,7 +10772,7 @@ obj2ast_arg(struct ast_state *state, PyObject* obj, arg_ty* out, PyArena* arena) } if (tmp == NULL || tmp == Py_None) { Py_CLEAR(tmp); - end_col_offset = 0; + end_col_offset = col_offset; } else { int res; @@ -10877,7 +10877,7 @@ obj2ast_keyword(struct ast_state *state, PyObject* obj, keyword_ty* out, } if (tmp == NULL || tmp == Py_None) { Py_CLEAR(tmp); - end_lineno = 0; + end_lineno = lineno; } else { int res; @@ -10894,7 +10894,7 @@ obj2ast_keyword(struct ast_state *state, PyObject* obj, keyword_ty* out, } if (tmp == NULL || tmp == Py_None) { Py_CLEAR(tmp); - end_col_offset = 0; + end_col_offset = col_offset; } else { int res; @@ -10999,7 +10999,7 @@ obj2ast_alias(struct ast_state *state, PyObject* obj, alias_ty* out, PyArena* } if (tmp == NULL || tmp == Py_None) { Py_CLEAR(tmp); - end_lineno = 0; + end_lineno = lineno; } else { int res; @@ -11016,7 +11016,7 @@ obj2ast_alias(struct ast_state *state, PyObject* obj, alias_ty* out, PyArena* } if (tmp == NULL || tmp == Py_None) { Py_CLEAR(tmp); - end_col_offset = 0; + end_col_offset = col_offset; } else { int res;

1 0

gh-92597: Ensure that AST nodes without explicit end positions can be compiled (GH-93359)
by pablogsal May 31, 2022

May 31, 2022

https://github.com/python/cpython/commit/705eaec28f7bee530b1c1635ba385a49a1… commit: 705eaec28f7bee530b1c1635ba385a49a1feaf32 branch: main author: Pablo Galindo Salgado <Pablogsal(a)gmail.com> committer: pablogsal <Pablogsal(a)gmail.com> date: 2022-06-01T00:00:47+01:00 summary: gh-92597: Ensure that AST nodes without explicit end positions can be compiled (GH-93359) files: A Misc/NEWS.d/next/Core and Builtins/2022-05-30-19-00-38.gh-issue-93359.zXV3A0.rst M Lib/test/test_ast.py M Parser/asdl_c.py M Python/Python-ast.c diff --git a/Lib/test/test_ast.py b/Lib/test/test_ast.py index 896eb5bedd7f3..33df22cb35a9e 100644 --- a/Lib/test/test_ast.py +++ b/Lib/test/test_ast.py @@ -362,6 +362,14 @@ def test_invalid_position_information(self): with self.assertRaises(ValueError): compile(tree, '<string>', 'exec') + def test_compilation_of_ast_nodes_with_default_end_position_values(self): + tree = ast.Module(body=[ + ast.Import(names=[ast.alias(name='builtins', lineno=1, col_offset=0)], lineno=1, col_offset=0), + ast.Import(names=[ast.alias(name='traceback', lineno=0, col_offset=0)], lineno=0, col_offset=1) + ], type_ignores=[]) + + # Check that compilation doesn't crash. Note: this may crash explicitly only on debug mode. + compile(tree, "<string>", "exec") def test_slice(self): slc = ast.parse("x[::]").body[0].value.slice diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-05-30-19-00-38.gh-issue-93359.zXV3A0.rst b/Misc/NEWS.d/next/Core and Builtins/2022-05-30-19-00-38.gh-issue-93359.zXV3A0.rst new file mode 100644 index 0000000000000..36e5e52390d7b --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2022-05-30-19-00-38.gh-issue-93359.zXV3A0.rst @@ -0,0 +1,2 @@ +Ensure that custom :mod:`ast` nodes without explicit end positions can be +compiled. Patch by Pablo Galindo. diff --git a/Parser/asdl_c.py b/Parser/asdl_c.py index 3bfe320fe3b70..1101a3593dfe2 100755 --- a/Parser/asdl_c.py +++ b/Parser/asdl_c.py @@ -488,6 +488,12 @@ def visitProduct(self, prod, name): class Obj2ModVisitor(PickleVisitor): + + attribute_special_defaults = { + "end_lineno": "lineno", + "end_col_offset": "col_offset", + } + @contextmanager def recursive_call(self, node, level): self.emit('if (_Py_EnterRecursiveCall(" while traversing \'%s\' node")) {' % node, level, reflow=False) @@ -637,7 +643,13 @@ def visitField(self, field, name, sum=None, prod=None, depth=0): self.emit("if (tmp == NULL || tmp == Py_None) {", depth) self.emit("Py_CLEAR(tmp);", depth+1) if self.isNumeric(field): - self.emit("%s = 0;" % field.name, depth+1) + if field.name in self.attribute_special_defaults: + self.emit( + "%s = %s;" % (field.name, self.attribute_special_defaults[field.name]), + depth+1, + ) + else: + self.emit("%s = 0;" % field.name, depth+1) elif not self.isSimpleType(field): self.emit("%s = NULL;" % field.name, depth+1) else: diff --git a/Python/Python-ast.c b/Python/Python-ast.c index 3861eaf978a38..e52a72d43bcbd 100644 --- a/Python/Python-ast.c +++ b/Python/Python-ast.c @@ -5697,7 +5697,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* } if (tmp == NULL || tmp == Py_None) { Py_CLEAR(tmp); - end_lineno = 0; + end_lineno = lineno; } else { int res; @@ -5714,7 +5714,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* } if (tmp == NULL || tmp == Py_None) { Py_CLEAR(tmp); - end_col_offset = 0; + end_col_offset = col_offset; } else { int res; @@ -8114,7 +8114,7 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* } if (tmp == NULL || tmp == Py_None) { Py_CLEAR(tmp); - end_lineno = 0; + end_lineno = lineno; } else { int res; @@ -8131,7 +8131,7 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* } if (tmp == NULL || tmp == Py_None) { Py_CLEAR(tmp); - end_col_offset = 0; + end_col_offset = col_offset; } else { int res; @@ -10291,7 +10291,7 @@ obj2ast_excepthandler(struct ast_state *state, PyObject* obj, excepthandler_ty* } if (tmp == NULL || tmp == Py_None) { Py_CLEAR(tmp); - end_lineno = 0; + end_lineno = lineno; } else { int res; @@ -10308,7 +10308,7 @@ obj2ast_excepthandler(struct ast_state *state, PyObject* obj, excepthandler_ty* } if (tmp == NULL || tmp == Py_None) { Py_CLEAR(tmp); - end_col_offset = 0; + end_col_offset = col_offset; } else { int res; @@ -10755,7 +10755,7 @@ obj2ast_arg(struct ast_state *state, PyObject* obj, arg_ty* out, PyArena* arena) } if (tmp == NULL || tmp == Py_None) { Py_CLEAR(tmp); - end_lineno = 0; + end_lineno = lineno; } else { int res; @@ -10772,7 +10772,7 @@ obj2ast_arg(struct ast_state *state, PyObject* obj, arg_ty* out, PyArena* arena) } if (tmp == NULL || tmp == Py_None) { Py_CLEAR(tmp); - end_col_offset = 0; + end_col_offset = col_offset; } else { int res; @@ -10877,7 +10877,7 @@ obj2ast_keyword(struct ast_state *state, PyObject* obj, keyword_ty* out, } if (tmp == NULL || tmp == Py_None) { Py_CLEAR(tmp); - end_lineno = 0; + end_lineno = lineno; } else { int res; @@ -10894,7 +10894,7 @@ obj2ast_keyword(struct ast_state *state, PyObject* obj, keyword_ty* out, } if (tmp == NULL || tmp == Py_None) { Py_CLEAR(tmp); - end_col_offset = 0; + end_col_offset = col_offset; } else { int res; @@ -10999,7 +10999,7 @@ obj2ast_alias(struct ast_state *state, PyObject* obj, alias_ty* out, PyArena* } if (tmp == NULL || tmp == Py_None) { Py_CLEAR(tmp); - end_lineno = 0; + end_lineno = lineno; } else { int res; @@ -11016,7 +11016,7 @@ obj2ast_alias(struct ast_state *state, PyObject* obj, alias_ty* out, PyArena* } if (tmp == NULL || tmp == Py_None) { Py_CLEAR(tmp); - end_col_offset = 0; + end_col_offset = col_offset; } else { int res;

1 0

gh-93391: fix typo in `array` docs (GH-93392)
by miss-islington May 31, 2022

May 31, 2022

https://github.com/python/cpython/commit/fc694364cc6745eefba0afc3ea2c5283bb… commit: fc694364cc6745eefba0afc3ea2c5283bbb64a3b branch: main author: ynfle <23086821+ynfle(a)users.noreply.github.com> committer: miss-islington <31488909+miss-islington(a)users.noreply.github.com> date: 2022-05-31T14:52:03-07:00 summary: gh-93391: fix typo in `array` docs (GH-93392) Fixes #93391 Automerge-Triggered-By: GH:rhettinger files: M Doc/library/array.rst diff --git a/Doc/library/array.rst b/Doc/library/array.rst index c7f137d15b4b8..975670cc81a20 100644 --- a/Doc/library/array.rst +++ b/Doc/library/array.rst @@ -52,7 +52,7 @@ Notes: .. versionchanged:: 3.9 ``array('u')`` now uses ``wchar_t`` as C type instead of deprecated - ``Py_UNICODE``. This change doesn't affect to its behavior because + ``Py_UNICODE``. This change doesn't affect its behavior because ``Py_UNICODE`` is alias of ``wchar_t`` since Python 3.3. .. deprecated-removed:: 3.3 4.0

1 0

[doc] logging.rst - Change link to point directly to the Google Group. (GH-93390)
by vsajip May 31, 2022

May 31, 2022

https://github.com/python/cpython/commit/e6e81602f49a9bff51e2049c7bad60d1ac… commit: e6e81602f49a9bff51e2049c7bad60d1acb18d3f branch: main author: Moshe Kaplan <mosheekaplan(a)gmail.com> committer: vsajip <vinay_sajip(a)yahoo.co.uk> date: 2022-05-31T22:46:52+01:00 summary: [doc] logging.rst - Change link to point directly to the Google Group. (GH-93390) files: M Doc/howto/logging.rst diff --git a/Doc/howto/logging.rst b/Doc/howto/logging.rst index 125bde2915227..cc4e278a7fe2e 100644 --- a/Doc/howto/logging.rst +++ b/Doc/howto/logging.rst @@ -336,7 +336,7 @@ favourite beverage and carry on. If your logging needs are simple, then use the above examples to incorporate logging into your own scripts, and if you run into problems or don't understand something, please post a question on the comp.lang.python Usenet -group (available at https://groups.google.com/forum/#!forum/comp.lang.python) and you +group (available at https://groups.google.com/g/comp.lang.python) and you should receive help before too long. Still here? You can carry on reading the next few sections, which provide a

1 0

gh-93143: Avoid NULL check in LOAD_FAST based on analysis in the compiler (GH-93144)
by sweeneyde May 31, 2022

May 31, 2022

https://github.com/python/cpython/commit/f425f3bb27e826d25aac05139360cc6aa2… commit: f425f3bb27e826d25aac05139360cc6aa279126e branch: main author: Dennis Sweeney <36520290+sweeneyde(a)users.noreply.github.com> committer: sweeneyde <36520290+sweeneyde(a)users.noreply.github.com> date: 2022-05-31T16:32:30-04:00 summary: gh-93143: Avoid NULL check in LOAD_FAST based on analysis in the compiler (GH-93144) files: A Misc/NEWS.d/next/Core and Builtins/2022-05-23-18-36-07.gh-issue-93143.X1Yqxm.rst M Include/internal/pycore_opcode.h M Include/opcode.h M Lib/importlib/_bootstrap_external.py M Lib/opcode.py M Lib/test/test_dis.py M Lib/test/test_peepholer.py M Objects/frameobject.c M Python/ceval.c M Python/compile.c M Python/opcode_targets.h diff --git a/Include/internal/pycore_opcode.h b/Include/internal/pycore_opcode.h index c693fe3c5eca6..e5d948dd696a2 100644 --- a/Include/internal/pycore_opcode.h +++ b/Include/internal/pycore_opcode.h @@ -158,6 +158,7 @@ const uint8_t _PyOpcode_Deopt[256] = { [LOAD_CONST__LOAD_FAST] = LOAD_CONST, [LOAD_DEREF] = LOAD_DEREF, [LOAD_FAST] = LOAD_FAST, + [LOAD_FAST_CHECK] = LOAD_FAST_CHECK, [LOAD_FAST__LOAD_CONST] = LOAD_FAST, [LOAD_FAST__LOAD_FAST] = LOAD_FAST, [LOAD_GLOBAL] = LOAD_GLOBAL, @@ -340,6 +341,7 @@ const uint8_t _PyOpcode_Original[256] = { [LOAD_CONST__LOAD_FAST] = LOAD_CONST, [LOAD_DEREF] = LOAD_DEREF, [LOAD_FAST] = LOAD_FAST, + [LOAD_FAST_CHECK] = LOAD_FAST_CHECK, [LOAD_FAST__LOAD_CONST] = LOAD_FAST, [LOAD_FAST__LOAD_FAST] = LOAD_FAST, [LOAD_GLOBAL] = LOAD_GLOBAL, @@ -547,7 +549,7 @@ static const char *const _PyOpcode_OpName[256] = { [LOAD_FAST] = "LOAD_FAST", [STORE_FAST] = "STORE_FAST", [DELETE_FAST] = "DELETE_FAST", - [LOAD_METHOD_WITH_VALUES] = "LOAD_METHOD_WITH_VALUES", + [LOAD_FAST_CHECK] = "LOAD_FAST_CHECK", [POP_JUMP_FORWARD_IF_NOT_NONE] = "POP_JUMP_FORWARD_IF_NOT_NONE", [POP_JUMP_FORWARD_IF_NONE] = "POP_JUMP_FORWARD_IF_NONE", [RAISE_VARARGS] = "RAISE_VARARGS", @@ -561,9 +563,9 @@ static const char *const _PyOpcode_OpName[256] = { [STORE_DEREF] = "STORE_DEREF", [DELETE_DEREF] = "DELETE_DEREF", [JUMP_BACKWARD] = "JUMP_BACKWARD", - [RESUME_QUICK] = "RESUME_QUICK", + [LOAD_METHOD_WITH_VALUES] = "LOAD_METHOD_WITH_VALUES", [CALL_FUNCTION_EX] = "CALL_FUNCTION_EX", - [STORE_ATTR_ADAPTIVE] = "STORE_ATTR_ADAPTIVE", + [RESUME_QUICK] = "RESUME_QUICK", [EXTENDED_ARG] = "EXTENDED_ARG", [LIST_APPEND] = "LIST_APPEND", [SET_ADD] = "SET_ADD", @@ -573,33 +575,33 @@ static const char *const _PyOpcode_OpName[256] = { [YIELD_VALUE] = "YIELD_VALUE", [RESUME] = "RESUME", [MATCH_CLASS] = "MATCH_CLASS", + [STORE_ATTR_ADAPTIVE] = "STORE_ATTR_ADAPTIVE", [STORE_ATTR_INSTANCE_VALUE] = "STORE_ATTR_INSTANCE_VALUE", - [STORE_ATTR_SLOT] = "STORE_ATTR_SLOT", [FORMAT_VALUE] = "FORMAT_VALUE", [BUILD_CONST_KEY_MAP] = "BUILD_CONST_KEY_MAP", [BUILD_STRING] = "BUILD_STRING", + [STORE_ATTR_SLOT] = "STORE_ATTR_SLOT", [STORE_ATTR_WITH_HINT] = "STORE_ATTR_WITH_HINT", - [STORE_FAST__LOAD_FAST] = "STORE_FAST__LOAD_FAST", [LOAD_METHOD] = "LOAD_METHOD", - [STORE_FAST__STORE_FAST] = "STORE_FAST__STORE_FAST", + [STORE_FAST__LOAD_FAST] = "STORE_FAST__LOAD_FAST", [LIST_EXTEND] = "LIST_EXTEND", [SET_UPDATE] = "SET_UPDATE", [DICT_MERGE] = "DICT_MERGE", [DICT_UPDATE] = "DICT_UPDATE", + [STORE_FAST__STORE_FAST] = "STORE_FAST__STORE_FAST", [STORE_SUBSCR_ADAPTIVE] = "STORE_SUBSCR_ADAPTIVE", [STORE_SUBSCR_DICT] = "STORE_SUBSCR_DICT", [STORE_SUBSCR_LIST_INT] = "STORE_SUBSCR_LIST_INT", [UNPACK_SEQUENCE_ADAPTIVE] = "UNPACK_SEQUENCE_ADAPTIVE", - [UNPACK_SEQUENCE_LIST] = "UNPACK_SEQUENCE_LIST", [CALL] = "CALL", [KW_NAMES] = "KW_NAMES", [POP_JUMP_BACKWARD_IF_NOT_NONE] = "POP_JUMP_BACKWARD_IF_NOT_NONE", [POP_JUMP_BACKWARD_IF_NONE] = "POP_JUMP_BACKWARD_IF_NONE", [POP_JUMP_BACKWARD_IF_FALSE] = "POP_JUMP_BACKWARD_IF_FALSE", [POP_JUMP_BACKWARD_IF_TRUE] = "POP_JUMP_BACKWARD_IF_TRUE", + [UNPACK_SEQUENCE_LIST] = "UNPACK_SEQUENCE_LIST", [UNPACK_SEQUENCE_TUPLE] = "UNPACK_SEQUENCE_TUPLE", [UNPACK_SEQUENCE_TWO_TUPLE] = "UNPACK_SEQUENCE_TWO_TUPLE", - [179] = "<179>", [180] = "<180>", [181] = "<181>", [182] = "<182>", @@ -680,7 +682,6 @@ static const char *const _PyOpcode_OpName[256] = { #endif #define EXTRA_CASES \ - case 179: \ case 180: \ case 181: \ case 182: \ diff --git a/Include/opcode.h b/Include/opcode.h index f76ca946be3ad..e7713013cd308 100644 --- a/Include/opcode.h +++ b/Include/opcode.h @@ -81,6 +81,7 @@ extern "C" { #define LOAD_FAST 124 #define STORE_FAST 125 #define DELETE_FAST 126 +#define LOAD_FAST_CHECK 127 #define POP_JUMP_FORWARD_IF_NOT_NONE 128 #define POP_JUMP_FORWARD_IF_NONE 129 #define RAISE_VARARGS 130 @@ -173,21 +174,21 @@ extern "C" { #define LOAD_METHOD_MODULE 86 #define LOAD_METHOD_NO_DICT 113 #define LOAD_METHOD_WITH_DICT 121 -#define LOAD_METHOD_WITH_VALUES 127 -#define RESUME_QUICK 141 -#define STORE_ATTR_ADAPTIVE 143 -#define STORE_ATTR_INSTANCE_VALUE 153 -#define STORE_ATTR_SLOT 154 -#define STORE_ATTR_WITH_HINT 158 -#define STORE_FAST__LOAD_FAST 159 -#define STORE_FAST__STORE_FAST 161 -#define STORE_SUBSCR_ADAPTIVE 166 -#define STORE_SUBSCR_DICT 167 -#define STORE_SUBSCR_LIST_INT 168 -#define UNPACK_SEQUENCE_ADAPTIVE 169 -#define UNPACK_SEQUENCE_LIST 170 -#define UNPACK_SEQUENCE_TUPLE 177 -#define UNPACK_SEQUENCE_TWO_TUPLE 178 +#define LOAD_METHOD_WITH_VALUES 141 +#define RESUME_QUICK 143 +#define STORE_ATTR_ADAPTIVE 153 +#define STORE_ATTR_INSTANCE_VALUE 154 +#define STORE_ATTR_SLOT 158 +#define STORE_ATTR_WITH_HINT 159 +#define STORE_FAST__LOAD_FAST 161 +#define STORE_FAST__STORE_FAST 166 +#define STORE_SUBSCR_ADAPTIVE 167 +#define STORE_SUBSCR_DICT 168 +#define STORE_SUBSCR_LIST_INT 169 +#define UNPACK_SEQUENCE_ADAPTIVE 170 +#define UNPACK_SEQUENCE_LIST 177 +#define UNPACK_SEQUENCE_TUPLE 178 +#define UNPACK_SEQUENCE_TWO_TUPLE 179 #define DO_TRACING 255 #define HAS_CONST(op) (false\ diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py index eac371fdefc78..894848a15d97d 100644 --- a/Lib/importlib/_bootstrap_external.py +++ b/Lib/importlib/_bootstrap_external.py @@ -406,6 +406,7 @@ def _write_atomic(path, data, mode=0o666): # Python 3.12a1 3500 (Remove PRECALL opcode) # Python 3.12a1 3501 (YIELD_VALUE oparg == stack_depth) +# Python 3.12a1 3502 (LOAD_FAST_CHECK, no NULL-check in LOAD_FAST) # Python 3.13 will start with 3550 @@ -419,7 +420,7 @@ def _write_atomic(path, data, mode=0o666): # Whenever MAGIC_NUMBER is changed, the ranges in the magic_values array # in PC/launcher.c must also be updated. -MAGIC_NUMBER = (3501).to_bytes(2, 'little') + b'\r\n' +MAGIC_NUMBER = (3502).to_bytes(2, 'little') + b'\r\n' _RAW_MAGIC_NUMBER = int.from_bytes(MAGIC_NUMBER, 'little') # For import.c diff --git a/Lib/opcode.py b/Lib/opcode.py index 573e461666d4d..5cbb5c57521ae 100644 --- a/Lib/opcode.py +++ b/Lib/opcode.py @@ -139,12 +139,14 @@ def jabs_op(name, op): def_op('COPY', 120) def_op('BINARY_OP', 122) jrel_op('SEND', 123) # Number of bytes to skip -def_op('LOAD_FAST', 124) # Local variable number +def_op('LOAD_FAST', 124) # Local variable number, no null check haslocal.append(124) def_op('STORE_FAST', 125) # Local variable number haslocal.append(125) def_op('DELETE_FAST', 126) # Local variable number haslocal.append(126) +def_op('LOAD_FAST_CHECK', 127) # Local variable number +haslocal.append(127) jrel_op('POP_JUMP_FORWARD_IF_NOT_NONE', 128) jrel_op('POP_JUMP_FORWARD_IF_NONE', 129) def_op('RAISE_VARARGS', 130) # Number of raise arguments (1, 2, or 3) diff --git a/Lib/test/test_dis.py b/Lib/test/test_dis.py index c5b80b72f8395..5aca019e74596 100644 --- a/Lib/test/test_dis.py +++ b/Lib/test/test_dis.py @@ -360,7 +360,7 @@ def bug42562(): --> BINARY_OP 11 (/) POP_TOP -%3d LOAD_FAST 1 (tb) +%3d LOAD_FAST_CHECK 1 (tb) RETURN_VALUE >> PUSH_EXC_INFO @@ -1399,7 +1399,7 @@ def _prepare_test_cases(): Instruction(opname='LOAD_CONST', opcode=100, arg=4, argval='I can haz else clause?', argrepr="'I can haz else clause?'", offset=100, starts_line=None, is_jump_target=False, positions=None), Instruction(opname='CALL', opcode=171, arg=1, argval=1, argrepr='', offset=102, starts_line=None, is_jump_target=False, positions=None), Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=112, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=114, starts_line=11, is_jump_target=True, positions=None), + Instruction(opname='LOAD_FAST_CHECK', opcode=127, arg=0, argval='i', argrepr='i', offset=114, starts_line=11, is_jump_target=True, positions=None), Instruction(opname='POP_JUMP_FORWARD_IF_FALSE', opcode=114, arg=34, argval=186, argrepr='to 186', offset=116, starts_line=None, is_jump_target=False, positions=None), Instruction(opname='LOAD_GLOBAL', opcode=116, arg=3, argval='print', argrepr='NULL + print', offset=118, starts_line=12, is_jump_target=True, positions=None), Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=130, starts_line=None, is_jump_target=False, positions=None), diff --git a/Lib/test/test_peepholer.py b/Lib/test/test_peepholer.py index dd993851bee92..2c3b1ab65a8f9 100644 --- a/Lib/test/test_peepholer.py +++ b/Lib/test/test_peepholer.py @@ -1,5 +1,6 @@ import dis from itertools import combinations, product +import sys import textwrap import unittest @@ -682,5 +683,184 @@ def test_bpo_45773_pop_jump_if_false(self): compile("while True or not spam: pass", "<test>", "exec") +class TestMarkingVariablesAsUnKnown(BytecodeTestCase): + + def setUp(self): + self.addCleanup(sys.settrace, sys.gettrace()) + sys.settrace(None) + + def test_load_fast_known_simple(self): + def f(): + x = 1 + y = x + x + self.assertInBytecode(f, 'LOAD_FAST') + + def test_load_fast_unknown_simple(self): + def f(): + if condition(): + x = 1 + print(x) + self.assertInBytecode(f, 'LOAD_FAST_CHECK') + self.assertNotInBytecode(f, 'LOAD_FAST') + + def test_load_fast_unknown_because_del(self): + def f(): + x = 1 + del x + print(x) + self.assertInBytecode(f, 'LOAD_FAST_CHECK') + self.assertNotInBytecode(f, 'LOAD_FAST') + + def test_load_fast_known_because_parameter(self): + def f1(x): + print(x) + self.assertInBytecode(f1, 'LOAD_FAST') + self.assertNotInBytecode(f1, 'LOAD_FAST_CHECK') + + def f2(*, x): + print(x) + self.assertInBytecode(f2, 'LOAD_FAST') + self.assertNotInBytecode(f2, 'LOAD_FAST_CHECK') + + def f3(*args): + print(args) + self.assertInBytecode(f3, 'LOAD_FAST') + self.assertNotInBytecode(f3, 'LOAD_FAST_CHECK') + + def f4(**kwargs): + print(kwargs) + self.assertInBytecode(f4, 'LOAD_FAST') + self.assertNotInBytecode(f4, 'LOAD_FAST_CHECK') + + def f5(x=0): + print(x) + self.assertInBytecode(f5, 'LOAD_FAST') + self.assertNotInBytecode(f5, 'LOAD_FAST_CHECK') + + def test_load_fast_known_because_already_loaded(self): + def f(): + if condition(): + x = 1 + print(x) + print(x) + self.assertInBytecode(f, 'LOAD_FAST_CHECK') + self.assertInBytecode(f, 'LOAD_FAST') + + def test_load_fast_known_multiple_branches(self): + def f(): + if condition(): + x = 1 + else: + x = 2 + print(x) + self.assertInBytecode(f, 'LOAD_FAST') + self.assertNotInBytecode(f, 'LOAD_FAST_CHECK') + + def test_load_fast_unknown_after_error(self): + def f(): + try: + res = 1 / 0 + except ZeroDivisionError: + pass + return res + # LOAD_FAST (known) still occurs in the no-exception branch. + # Assert that it doesn't occur in the LOAD_FAST_CHECK branch. + self.assertInBytecode(f, 'LOAD_FAST_CHECK') + + def test_load_fast_unknown_after_error_2(self): + def f(): + try: + 1 / 0 + except: + print(a, b, c, d, e, f, g) + a = b = c = d = e = f = g = 1 + self.assertInBytecode(f, 'LOAD_FAST_CHECK') + self.assertNotInBytecode(f, 'LOAD_FAST') + + def test_setting_lineno_adds_check(self): + code = textwrap.dedent("""\ + def f(): + x = 2 + L = 3 + L = 4 + for i in range(55): + x + 6 + del x + L = 8 + L = 9 + L = 10 + """) + ns = {} + exec(code, ns) + f = ns['f'] + self.assertInBytecode(f, "LOAD_FAST") + def trace(frame, event, arg): + if event == 'line' and frame.f_lineno == 9: + frame.f_lineno = 2 + sys.settrace(None) + return None + return trace + sys.settrace(trace) + f() + self.assertNotInBytecode(f, "LOAD_FAST") + + def make_function_with_no_checks(self): + code = textwrap.dedent("""\ + def f(): + x = 2 + L = 3 + L = 4 + L = 5 + if not L: + x + 7 + y = 2 + """) + ns = {} + exec(code, ns) + f = ns['f'] + self.assertInBytecode(f, "LOAD_FAST") + self.assertNotInBytecode(f, "LOAD_FAST_CHECK") + return f + + def test_deleting_local_adds_check(self): + f = self.make_function_with_no_checks() + def trace(frame, event, arg): + if event == 'line' and frame.f_lineno == 4: + del frame.f_locals["x"] + sys.settrace(None) + return None + return trace + sys.settrace(trace) + f() + self.assertNotInBytecode(f, "LOAD_FAST") + self.assertInBytecode(f, "LOAD_FAST_CHECK") + + def test_modifying_local_does_not_add_check(self): + f = self.make_function_with_no_checks() + def trace(frame, event, arg): + if event == 'line' and frame.f_lineno == 4: + frame.f_locals["x"] = 42 + sys.settrace(None) + return None + return trace + sys.settrace(trace) + f() + self.assertInBytecode(f, "LOAD_FAST") + self.assertNotInBytecode(f, "LOAD_FAST_CHECK") + + def test_initializing_local_does_not_add_check(self): + f = self.make_function_with_no_checks() + def trace(frame, event, arg): + if event == 'line' and frame.f_lineno == 4: + frame.f_locals["y"] = 42 + sys.settrace(None) + return None + return trace + sys.settrace(trace) + f() + self.assertInBytecode(f, "LOAD_FAST") + self.assertNotInBytecode(f, "LOAD_FAST_CHECK") + + if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-05-23-18-36-07.gh-issue-93143.X1Yqxm.rst b/Misc/NEWS.d/next/Core and Builtins/2022-05-23-18-36-07.gh-issue-93143.X1Yqxm.rst new file mode 100644 index 0000000000000..03994bcfdb56f --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2022-05-23-18-36-07.gh-issue-93143.X1Yqxm.rst @@ -0,0 +1 @@ +Avoid ``NULL`` checks for uninitialized local variables by determining at compile time which variables must be initialized. diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 60f0f2f4edd38..a448ba3a6c6f9 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -455,6 +455,26 @@ _PyFrame_GetState(PyFrameObject *frame) Py_UNREACHABLE(); } +static void +add_load_fast_null_checks(PyCodeObject *co) +{ + _Py_CODEUNIT *instructions = _PyCode_CODE(co); + for (Py_ssize_t i = 0; i < Py_SIZE(co); i++) { + switch (_Py_OPCODE(instructions[i])) { + case LOAD_FAST: + case LOAD_FAST__LOAD_FAST: + case LOAD_FAST__LOAD_CONST: + _Py_SET_OPCODE(instructions[i], LOAD_FAST_CHECK); + break; + case LOAD_CONST__LOAD_FAST: + _Py_SET_OPCODE(instructions[i], LOAD_CONST); + break; + case STORE_FAST__LOAD_FAST: + _Py_SET_OPCODE(instructions[i], STORE_FAST); + break; + } + } +} /* Setter for f_lineno - you can set f_lineno from within a trace function in * order to jump to a given line of code, subject to some restrictions. Most @@ -545,6 +565,8 @@ frame_setlineno(PyFrameObject *f, PyObject* p_new_lineno, void *Py_UNUSED(ignore return -1; } + add_load_fast_null_checks(f->f_frame->f_code); + /* PyCode_NewWithPosOnlyArgs limits co_code to be under INT_MAX so this * should never overflow. */ int len = (int)Py_SIZE(f->f_frame->f_code); @@ -1047,6 +1069,7 @@ _PyFrame_LocalsToFast(_PyInterpreterFrame *frame, int clear) } fast = _PyFrame_GetLocalsArray(frame); co = frame->f_code; + bool added_null_checks = false; PyErr_Fetch(&error_type, &error_value, &error_traceback); for (int i = 0; i < co->co_nlocalsplus; i++) { @@ -1066,6 +1089,10 @@ _PyFrame_LocalsToFast(_PyInterpreterFrame *frame, int clear) } } PyObject *oldvalue = fast[i]; + if (!added_null_checks && oldvalue != NULL && value == NULL) { + add_load_fast_null_checks(co); + added_null_checks = true; + } PyObject *cell = NULL; if (kind == CO_FAST_FREE) { // The cell was set when the frame was created from diff --git a/Python/ceval.c b/Python/ceval.c index 36546929e19c4..ec86c70673ca9 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -1813,7 +1813,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int DISPATCH(); } - TARGET(LOAD_FAST) { + TARGET(LOAD_FAST_CHECK) { PyObject *value = GETLOCAL(oparg); if (value == NULL) { goto unbound_local_error; @@ -1823,6 +1823,14 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int DISPATCH(); } + TARGET(LOAD_FAST) { + PyObject *value = GETLOCAL(oparg); + assert(value != NULL); + Py_INCREF(value); + PUSH(value); + DISPATCH(); + } + TARGET(LOAD_CONST) { PREDICTED(LOAD_CONST); PyObject *value = GETITEM(consts, oparg); @@ -1840,17 +1848,13 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int TARGET(LOAD_FAST__LOAD_FAST) { PyObject *value = GETLOCAL(oparg); - if (value == NULL) { - goto unbound_local_error; - } + assert(value != NULL); NEXTOPARG(); next_instr++; Py_INCREF(value); PUSH(value); value = GETLOCAL(oparg); - if (value == NULL) { - goto unbound_local_error; - } + assert(value != NULL); Py_INCREF(value); PUSH(value); NOTRACE_DISPATCH(); @@ -1858,9 +1862,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int TARGET(LOAD_FAST__LOAD_CONST) { PyObject *value = GETLOCAL(oparg); - if (value == NULL) { - goto unbound_local_error; - } + assert(value != NULL); NEXTOPARG(); next_instr++; Py_INCREF(value); @@ -1877,9 +1879,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int NEXTOPARG(); next_instr++; value = GETLOCAL(oparg); - if (value == NULL) { - goto unbound_local_error; - } + assert(value != NULL); Py_INCREF(value); PUSH(value); NOTRACE_DISPATCH(); @@ -1902,9 +1902,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int Py_INCREF(value); PUSH(value); value = GETLOCAL(oparg); - if (value == NULL) { - goto unbound_local_error; - } + assert(value != NULL); Py_INCREF(value); PUSH(value); NOTRACE_DISPATCH(); diff --git a/Python/compile.c b/Python/compile.c index 0920cebfe7c13..a6788b732831f 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -1109,6 +1109,7 @@ stack_effect(int opcode, int oparg, int jump) return 1; case LOAD_FAST: + case LOAD_FAST_CHECK: return 1; case STORE_FAST: return -1; @@ -7746,6 +7747,109 @@ assemble_jump_offsets(struct assembler *a, struct compiler *c) } while (extended_arg_recompile); } + +// Ensure each basicblock is only put onto the stack once. +#define MAYBE_PUSH(B) do { \ + if ((B)->b_visited == 0) { \ + *(*stack_top)++ = (B); \ + (B)->b_visited = 1; \ + } \ + } while (0) + +static void +scan_block_for_local(int target, basicblock *b, bool unsafe_to_start, + basicblock ***stack_top) +{ + bool unsafe = unsafe_to_start; + for (int i = 0; i < b->b_iused; i++) { + struct instr *instr = &b->b_instr[i]; + assert(instr->i_opcode != EXTENDED_ARG); + assert(instr->i_opcode != EXTENDED_ARG_QUICK); + assert(instr->i_opcode != LOAD_FAST__LOAD_FAST); + assert(instr->i_opcode != STORE_FAST__LOAD_FAST); + assert(instr->i_opcode != LOAD_CONST__LOAD_FAST); + assert(instr->i_opcode != STORE_FAST__STORE_FAST); + assert(instr->i_opcode != LOAD_FAST__LOAD_CONST); + if (unsafe && instr->i_except != NULL) { + MAYBE_PUSH(instr->i_except); + } + if (instr->i_oparg != target) { + continue; + } + switch (instr->i_opcode) { + case LOAD_FAST_CHECK: + // if this doesn't raise, then var is defined + unsafe = false; + break; + case LOAD_FAST: + if (unsafe) { + instr->i_opcode = LOAD_FAST_CHECK; + } + unsafe = false; + break; + case STORE_FAST: + unsafe = false; + break; + case DELETE_FAST: + unsafe = true; + break; + } + } + if (unsafe) { + // unsafe at end of this block, + // so unsafe at start of next blocks + if (b->b_next && !b->b_nofallthrough) { + MAYBE_PUSH(b->b_next); + } + if (b->b_iused > 0) { + struct instr *last = &b->b_instr[b->b_iused-1]; + if (is_jump(last)) { + assert(last->i_target != NULL); + MAYBE_PUSH(last->i_target); + } + } + } +} +#undef MAYBE_PUSH + +static int +add_checks_for_loads_of_unknown_variables(struct assembler *a, + struct compiler *c) +{ + basicblock **stack = make_cfg_traversal_stack(a->a_entry); + if (stack == NULL) { + return -1; + } + Py_ssize_t nparams = PyList_GET_SIZE(c->u->u_ste->ste_varnames); + int nlocals = (int)PyDict_GET_SIZE(c->u->u_varnames); + for (int target = 0; target < nlocals; target++) { + for (basicblock *b = a->a_entry; b != NULL; b = b->b_next) { + b->b_visited = 0; + } + basicblock **stack_top = stack; + + // First pass: find the relevant DFS starting points: + // the places where "being uninitialized" originates, + // which are the entry block and any DELETE_FAST statements. + if (target >= nparams) { + // only non-parameter locals start out uninitialized. + *(stack_top++) = a->a_entry; + a->a_entry->b_visited = 1; + } + for (basicblock *b = a->a_entry; b != NULL; b = b->b_next) { + scan_block_for_local(target, b, false, &stack_top); + } + + // Second pass: Depth-first search to propagate uncertainty + while (stack_top > stack) { + basicblock *b = *--stack_top; + scan_block_for_local(target, b, true, &stack_top); + } + } + PyMem_Free(stack); + return 0; +} + static PyObject * dict_keys_inorder(PyObject *dict, Py_ssize_t offset) { @@ -8385,6 +8489,10 @@ assemble(struct compiler *c, int addNone) /* Order of basic blocks must have been determined by now */ normalize_jumps(&a); + if (add_checks_for_loads_of_unknown_variables(&a, c) < 0) { + goto error; + } + /* Can't modify the bytecode after computing jump offsets. */ assemble_jump_offsets(&a, c); diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h index 71b7a76a01aa8..1009b3a93458e 100644 --- a/Python/opcode_targets.h +++ b/Python/opcode_targets.h @@ -126,7 +126,7 @@ static void *opcode_targets[256] = { &&TARGET_LOAD_FAST, &&TARGET_STORE_FAST, &&TARGET_DELETE_FAST, - &&TARGET_LOAD_METHOD_WITH_VALUES, + &&TARGET_LOAD_FAST_CHECK, &&TARGET_POP_JUMP_FORWARD_IF_NOT_NONE, &&TARGET_POP_JUMP_FORWARD_IF_NONE, &&TARGET_RAISE_VARARGS, @@ -140,9 +140,9 @@ static void *opcode_targets[256] = { &&TARGET_STORE_DEREF, &&TARGET_DELETE_DEREF, &&TARGET_JUMP_BACKWARD, - &&TARGET_RESUME_QUICK, + &&TARGET_LOAD_METHOD_WITH_VALUES, &&TARGET_CALL_FUNCTION_EX, - &&TARGET_STORE_ATTR_ADAPTIVE, + &&TARGET_RESUME_QUICK, &&TARGET_EXTENDED_ARG, &&TARGET_LIST_APPEND, &&TARGET_SET_ADD, @@ -152,30 +152,31 @@ static void *opcode_targets[256] = { &&TARGET_YIELD_VALUE, &&TARGET_RESUME, &&TARGET_MATCH_CLASS, + &&TARGET_STORE_ATTR_ADAPTIVE, &&TARGET_STORE_ATTR_INSTANCE_VALUE, - &&TARGET_STORE_ATTR_SLOT, &&TARGET_FORMAT_VALUE, &&TARGET_BUILD_CONST_KEY_MAP, &&TARGET_BUILD_STRING, + &&TARGET_STORE_ATTR_SLOT, &&TARGET_STORE_ATTR_WITH_HINT, - &&TARGET_STORE_FAST__LOAD_FAST, &&TARGET_LOAD_METHOD, - &&TARGET_STORE_FAST__STORE_FAST, + &&TARGET_STORE_FAST__LOAD_FAST, &&TARGET_LIST_EXTEND, &&TARGET_SET_UPDATE, &&TARGET_DICT_MERGE, &&TARGET_DICT_UPDATE, + &&TARGET_STORE_FAST__STORE_FAST, &&TARGET_STORE_SUBSCR_ADAPTIVE, &&TARGET_STORE_SUBSCR_DICT, &&TARGET_STORE_SUBSCR_LIST_INT, &&TARGET_UNPACK_SEQUENCE_ADAPTIVE, - &&TARGET_UNPACK_SEQUENCE_LIST, &&TARGET_CALL, &&TARGET_KW_NAMES, &&TARGET_POP_JUMP_BACKWARD_IF_NOT_NONE, &&TARGET_POP_JUMP_BACKWARD_IF_NONE, &&TARGET_POP_JUMP_BACKWARD_IF_FALSE, &&TARGET_POP_JUMP_BACKWARD_IF_TRUE, + &&TARGET_UNPACK_SEQUENCE_LIST, &&TARGET_UNPACK_SEQUENCE_TUPLE, &&TARGET_UNPACK_SEQUENCE_TWO_TUPLE, &&_unknown_opcode, @@ -253,6 +254,5 @@ static void *opcode_targets[256] = { &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, - &&_unknown_opcode, &&TARGET_DO_TRACING };

1 0

gh-69093: Fix Setup.local.in rule for _sqlite3 (GH-93380)
by miss-islington May 31, 2022

May 31, 2022

https://github.com/python/cpython/commit/29bfabe9709d4439cacb0806ba0e9f1dfd… commit: 29bfabe9709d4439cacb0806ba0e9f1dfd688157 branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: miss-islington <31488909+miss-islington(a)users.noreply.github.com> date: 2022-05-31T12:53:27-07:00 summary: gh-69093: Fix Setup.local.in rule for _sqlite3 (GH-93380) (cherry picked from commit 8a5e3c2ec6254b2ce06d17545f58a6719e0c8fdb) Co-authored-by: Christian Heimes <christian(a)python.org> files: A Misc/NEWS.d/next/Build/2022-05-31-18-04-58.gh-issue-69093.6lSa0C.rst M Modules/Setup.stdlib.in diff --git a/Misc/NEWS.d/next/Build/2022-05-31-18-04-58.gh-issue-69093.6lSa0C.rst b/Misc/NEWS.d/next/Build/2022-05-31-18-04-58.gh-issue-69093.6lSa0C.rst new file mode 100644 index 0000000000000..b061d6c2186c1 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2022-05-31-18-04-58.gh-issue-69093.6lSa0C.rst @@ -0,0 +1 @@ +Fix ``Modules/Setup.stdlib.in`` rule for ``_sqlite3`` extension. diff --git a/Modules/Setup.stdlib.in b/Modules/Setup.stdlib.in index 22c0b147c1b89..2730030a15650 100644 --- a/Modules/Setup.stdlib.in +++ b/Modules/Setup.stdlib.in @@ -143,7 +143,7 @@ # needs -lncurses and -lpanel #@MODULE__CURSES_PANEL_TRUE@_curses_panel _curses_panel.c -@MODULE__SQLITE3_TRUE@_sqlite3 _sqlite/connection.c _sqlite/cursor.c _sqlite/microprotocols.c _sqlite/module.c _sqlite/prepare_protocol.c _sqlite/row.c _sqlite/statement.c _sqlite/util.c +@MODULE__SQLITE3_TRUE@_sqlite3 _sqlite/blob.c _sqlite/connection.c _sqlite/cursor.c _sqlite/microprotocols.c _sqlite/module.c _sqlite/prepare_protocol.c _sqlite/row.c _sqlite/statement.c _sqlite/util.c # needs -lssl and -lcrypt @MODULE__SSL_TRUE@_ssl _ssl.c

1 0

gh-69093: Fix Setup.local.in rule for _sqlite3 (GH-93380)
by tiran May 31, 2022

May 31, 2022

https://github.com/python/cpython/commit/8a5e3c2ec6254b2ce06d17545f58a6719e… commit: 8a5e3c2ec6254b2ce06d17545f58a6719e0c8fdb branch: main author: Christian Heimes <christian(a)python.org> committer: tiran <christian(a)python.org> date: 2022-05-31T21:23:49+02:00 summary: gh-69093: Fix Setup.local.in rule for _sqlite3 (GH-93380) files: A Misc/NEWS.d/next/Build/2022-05-31-18-04-58.gh-issue-69093.6lSa0C.rst M Modules/Setup.stdlib.in diff --git a/Misc/NEWS.d/next/Build/2022-05-31-18-04-58.gh-issue-69093.6lSa0C.rst b/Misc/NEWS.d/next/Build/2022-05-31-18-04-58.gh-issue-69093.6lSa0C.rst new file mode 100644 index 0000000000000..b061d6c2186c1 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2022-05-31-18-04-58.gh-issue-69093.6lSa0C.rst @@ -0,0 +1 @@ +Fix ``Modules/Setup.stdlib.in`` rule for ``_sqlite3`` extension. diff --git a/Modules/Setup.stdlib.in b/Modules/Setup.stdlib.in index 22c0b147c1b89..2730030a15650 100644 --- a/Modules/Setup.stdlib.in +++ b/Modules/Setup.stdlib.in @@ -143,7 +143,7 @@ # needs -lncurses and -lpanel #@MODULE__CURSES_PANEL_TRUE@_curses_panel _curses_panel.c -@MODULE__SQLITE3_TRUE@_sqlite3 _sqlite/connection.c _sqlite/cursor.c _sqlite/microprotocols.c _sqlite/module.c _sqlite/prepare_protocol.c _sqlite/row.c _sqlite/statement.c _sqlite/util.c +@MODULE__SQLITE3_TRUE@_sqlite3 _sqlite/blob.c _sqlite/connection.c _sqlite/cursor.c _sqlite/microprotocols.c _sqlite/module.c _sqlite/prepare_protocol.c _sqlite/row.c _sqlite/statement.c _sqlite/util.c # needs -lssl and -lcrypt @MODULE__SSL_TRUE@_ssl _ssl.c

1 0

Remove VOC reference (93333)
by miss-islington May 31, 2022

May 31, 2022

https://github.com/python/cpython/commit/694455d5eddaee6b8a0a1e47fc0de56c02… commit: 694455d5eddaee6b8a0a1e47fc0de56c02a489fe branch: 3.10 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: miss-islington <31488909+miss-islington(a)users.noreply.github.com> date: 2022-05-31T10:49:14-07:00 summary: Remove VOC reference (93333) VOC has been archived by the BeeWare project, and they are instead embedding CPython, rather than transpiling to Java bytecode. (cherry picked from commit bb900712a5511ba82ef64105fe28d2a6886a8fed) Co-authored-by: Carl Bordum Hansen <carl(a)bordum.dk> files: M Doc/faq/design.rst diff --git a/Doc/faq/design.rst b/Doc/faq/design.rst index ff83a1b8134b7..a624fdb07a17d 100644 --- a/Doc/faq/design.rst +++ b/Doc/faq/design.rst @@ -324,8 +324,7 @@ Can Python be compiled to machine code, C or some other language? `Cython <http://cython.org/>`_ compiles a modified version of Python with optional annotations into C extensions. `Nuitka <http://www.nuitka.net/>`_ is an up-and-coming compiler of Python into C++ code, aiming to support the full -Python language. For compiling to Java you can consider -`VOC <https://voc.readthedocs.io>`_. +Python language. How does Python manage memory?

1 0

Remove VOC reference (93333)
by miss-islington May 31, 2022

May 31, 2022

https://github.com/python/cpython/commit/3a627014527fa5c7289b1c83c5f0607c56… commit: 3a627014527fa5c7289b1c83c5f0607c56df660b branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: miss-islington <31488909+miss-islington(a)users.noreply.github.com> date: 2022-05-31T10:46:37-07:00 summary: Remove VOC reference (93333) VOC has been archived by the BeeWare project, and they are instead embedding CPython, rather than transpiling to Java bytecode. (cherry picked from commit bb900712a5511ba82ef64105fe28d2a6886a8fed) Co-authored-by: Carl Bordum Hansen <carl(a)bordum.dk> files: M Doc/faq/design.rst diff --git a/Doc/faq/design.rst b/Doc/faq/design.rst index ff83a1b8134b7..a624fdb07a17d 100644 --- a/Doc/faq/design.rst +++ b/Doc/faq/design.rst @@ -324,8 +324,7 @@ Can Python be compiled to machine code, C or some other language? `Cython <http://cython.org/>`_ compiles a modified version of Python with optional annotations into C extensions. `Nuitka <http://www.nuitka.net/>`_ is an up-and-coming compiler of Python into C++ code, aiming to support the full -Python language. For compiling to Java you can consider -`VOC <https://voc.readthedocs.io>`_. +Python language. How does Python manage memory?

1 0

gh-93283: Improve error message for f-string with invalid conversion character (GH-93349)
by serhiy-storchaka May 31, 2022

May 31, 2022

https://github.com/python/cpython/commit/07df8d5b2c715c9aa770c47986d59e272b… commit: 07df8d5b2c715c9aa770c47986d59e272bf049f0 branch: main author: Serhiy Storchaka <storchaka(a)gmail.com> committer: serhiy-storchaka <storchaka(a)gmail.com> date: 2022-05-31T20:38:29+03:00 summary: gh-93283: Improve error message for f-string with invalid conversion character (GH-93349) files: A Misc/NEWS.d/next/Core and Builtins/2022-05-30-14-50-03.gh-issue-93283.XDO2ZQ.rst M Lib/test/test_fstring.py M Parser/string_parser.c diff --git a/Lib/test/test_fstring.py b/Lib/test/test_fstring.py index 0c3372f033551..e8bf420d699ed 100644 --- a/Lib/test/test_fstring.py +++ b/Lib/test/test_fstring.py @@ -590,7 +590,9 @@ def test_format_specifier_expressions(self): self.assertEqual(f'{-10:{"-"}#{1}0{"x"}}', ' -0xa') self.assertEqual(f'{10:#{3 != {4:5} and width}x}', ' 0xa') - self.assertAllRaise(SyntaxError, "f-string: expecting '}'", + self.assertAllRaise(SyntaxError, + """f-string: invalid conversion character 'r{"': """ + """expected 's', 'r', or 'a'""", ["""f'{"s"!r{":10"}}'""", # This looks like a nested format spec. @@ -1012,19 +1014,28 @@ def test_conversions(self): # Not a conversion, but show that ! is allowed in a format spec. self.assertEqual(f'{3.14:!<10.10}', '3.14!!!!!!') - self.assertAllRaise(SyntaxError, 'f-string: invalid conversion character', - ["f'{3!g}'", - "f'{3!A}'", - "f'{3!3}'", - "f'{3!G}'", - "f'{3!!}'", + self.assertAllRaise(SyntaxError, "f-string: expecting '}'", + ["f'{3!'", + "f'{3!s'", + "f'{3!g'", + ]) + + self.assertAllRaise(SyntaxError, 'f-string: missed conversion character', + ["f'{3!}'", + "f'{3!:'", "f'{3!:}'", - "f'{3! s}'", # no space before conversion char ]) - self.assertAllRaise(SyntaxError, "f-string: expecting '}'", - ["f'{x!s{y}}'", - "f'{3!ss}'", + for conv in 'g', 'A', '3', 'G', '!', ' s', 's ', ' s ', 'ä', 'ɐ', 'ª': + self.assertAllRaise(SyntaxError, + "f-string: invalid conversion character %r: " + "expected 's', 'r', or 'a'" % conv, + ["f'{3!" + conv + "}'"]) + + self.assertAllRaise(SyntaxError, + "f-string: invalid conversion character 'ss': " + "expected 's', 'r', or 'a'", + ["f'{3!ss}'", "f'{3!ss:}'", "f'{3!ss:s}'", ]) diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-05-30-14-50-03.gh-issue-93283.XDO2ZQ.rst b/Misc/NEWS.d/next/Core and Builtins/2022-05-30-14-50-03.gh-issue-93283.XDO2ZQ.rst new file mode 100644 index 0000000000000..550e86988b25a --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2022-05-30-14-50-03.gh-issue-93283.XDO2ZQ.rst @@ -0,0 +1,2 @@ +Improve error message for invalid syntax of conversion character in f-string +expressions. diff --git a/Parser/string_parser.c b/Parser/string_parser.c index 9c12d8ca101d0..c56ed20ad4cfc 100644 --- a/Parser/string_parser.c +++ b/Parser/string_parser.c @@ -767,27 +767,43 @@ fstring_find_expr(Parser *p, const char **str, const char *end, int raw, int rec /* Check for a conversion char, if present. */ if (**str == '!') { *str += 1; - if (*str >= end) { - goto unexpected_end_of_string; + const char *conv_start = *str; + while (1) { + if (*str >= end) { + goto unexpected_end_of_string; + } + if (**str == '}' || **str == ':') { + break; + } + *str += 1; + } + if (*str == conv_start) { + RAISE_SYNTAX_ERROR( + "f-string: missed conversion character"); + goto error; } - conversion = (unsigned char)**str; - *str += 1; - + conversion = (unsigned char)*conv_start; /* Validate the conversion. */ - if (!(conversion == 's' || conversion == 'r' || conversion == 'a')) { - RAISE_SYNTAX_ERROR( - "f-string: invalid conversion character: " - "expected 's', 'r', or 'a'"); + if ((*str != conv_start + 1) || + !(conversion == 's' || conversion == 'r' || conversion == 'a')) + { + PyObject *conv_obj = PyUnicode_FromStringAndSize(conv_start, + *str-conv_start); + if (conv_obj) { + RAISE_SYNTAX_ERROR( + "f-string: invalid conversion character %R: " + "expected 's', 'r', or 'a'", + conv_obj); + Py_DECREF(conv_obj); + } goto error; } } /* Check for the format spec, if present. */ - if (*str >= end) { - goto unexpected_end_of_string; - } + assert(*str < end); if (**str == ':') { *str += 1; if (*str >= end) {

1 0