Mailman 3 July 2020 - Python-checkins

bpo-33864: Clarify the docs for typing.ByteString (GH-21311)
by Miss Islington (bot) July 4, 2020

July 4, 2020

https://github.com/python/cpython/commit/6857ebefc048e316f948091946d337b5ad… commit: 6857ebefc048e316f948091946d337b5ada807a4 branch: 3.8 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: GitHub <noreply(a)github.com> date: 2020-07-03T21:06:07-07:00 summary: bpo-33864: Clarify the docs for typing.ByteString (GH-21311) (cherry picked from commit b40e434386cd94a367d4a256e3364771140160e7) Co-authored-by: Zackery Spytz <zspytz(a)gmail.com> files: M Doc/library/typing.rst diff --git a/Doc/library/typing.rst b/Doc/library/typing.rst index 5fa89423d20bf..1467276d0141d 100644 --- a/Doc/library/typing.rst +++ b/Doc/library/typing.rst @@ -672,7 +672,7 @@ The module defines the following classes, functions and decorators: A generic version of :class:`collections.abc.ByteString`. This type represents the types :class:`bytes`, :class:`bytearray`, - and :class:`memoryview`. + and :class:`memoryview` of byte sequences. As a shorthand for this type, :class:`bytes` can be used to annotate arguments of any of the types mentioned above.

1 0

bpo-33864: Clarify the docs for typing.ByteString (GH-21311)
by Zackery Spytz July 4, 2020

July 4, 2020

https://github.com/python/cpython/commit/b40e434386cd94a367d4a256e336477114… commit: b40e434386cd94a367d4a256e3364771140160e7 branch: master author: Zackery Spytz <zspytz(a)gmail.com> committer: GitHub <noreply(a)github.com> date: 2020-07-03T20:58:21-07:00 summary: bpo-33864: Clarify the docs for typing.ByteString (GH-21311) files: M Doc/library/typing.rst diff --git a/Doc/library/typing.rst b/Doc/library/typing.rst index e85b6d697f79d..e5143c5f8d9e4 100644 --- a/Doc/library/typing.rst +++ b/Doc/library/typing.rst @@ -672,7 +672,7 @@ The module defines the following classes, functions and decorators: A generic version of :class:`collections.abc.ByteString`. This type represents the types :class:`bytes`, :class:`bytearray`, - and :class:`memoryview`. + and :class:`memoryview` of byte sequences. As a shorthand for this type, :class:`bytes` can be used to annotate arguments of any of the types mentioned above.

1 0

bpo-41162: Clear audit hooks later during finalization (GH-21222)
by Steve Dower July 3, 2020

July 3, 2020

https://github.com/python/cpython/commit/b9e288cc1bfd583e887f784e38d9c511b4… commit: b9e288cc1bfd583e887f784e38d9c511b43c0c3a branch: 3.8 author: Steve Dower <steve.dower(a)python.org> committer: GitHub <noreply(a)github.com> date: 2020-07-04T00:04:22+01:00 summary: bpo-41162: Clear audit hooks later during finalization (GH-21222) Co-authored-by: Konge <zkonge(a)outlook.com> files: A Misc/NEWS.d/next/Security/2020-07-03-20-41-29.bpo-41162.tb8pVj.rst M Lib/test/audit-tests.py M Lib/test/test_audit.py M Programs/_testembed.c M Python/pylifecycle.c diff --git a/Lib/test/audit-tests.py b/Lib/test/audit-tests.py index a58395b068b39..ee6fc93351b75 100644 --- a/Lib/test/audit-tests.py +++ b/Lib/test/audit-tests.py @@ -44,28 +44,6 @@ def __call__(self, event, args): raise self.exc_type("saw event " + event) -class TestFinalizeHook: - """Used in the test_finalize_hooks function to ensure that hooks - are correctly cleaned up, that they are notified about the cleanup, - and are unable to prevent it. - """ - - def __init__(self): - print("Created", id(self), file=sys.stdout, flush=True) - - def __call__(self, event, args): - # Avoid recursion when we call id() below - if event == "builtins.id": - return - - print(event, id(self), file=sys.stdout, flush=True) - - if event == "cpython._PySys_ClearAuditHooks": - raise RuntimeError("Should be ignored") - elif event == "cpython.PyInterpreterState_Clear": - raise RuntimeError("Should be ignored") - - # Simple helpers, since we are not in unittest here def assertEqual(x, y): if x != y: @@ -128,10 +106,6 @@ def test_block_add_hook_baseexception(): pass -def test_finalize_hooks(): - sys.addaudithook(TestFinalizeHook()) - - def test_pickle(): import pickle diff --git a/Lib/test/test_audit.py b/Lib/test/test_audit.py index f405c6923979c..f79edbc4bd0d9 100644 --- a/Lib/test/test_audit.py +++ b/Lib/test/test_audit.py @@ -51,22 +51,6 @@ def test_block_add_hook(self): def test_block_add_hook_baseexception(self): self.do_test("test_block_add_hook_baseexception") - def test_finalize_hooks(self): - returncode, events, stderr = self.run_python("test_finalize_hooks") - if stderr: - print(stderr, file=sys.stderr) - if returncode: - self.fail(stderr) - - firstId = events[0][2] - self.assertSequenceEqual( - [ - ("Created", " ", firstId), - ("cpython._PySys_ClearAuditHooks", " ", firstId), - ], - events, - ) - def test_pickle(self): support.import_module("pickle") diff --git a/Misc/NEWS.d/next/Security/2020-07-03-20-41-29.bpo-41162.tb8pVj.rst b/Misc/NEWS.d/next/Security/2020-07-03-20-41-29.bpo-41162.tb8pVj.rst new file mode 100644 index 0000000000000..f0333ac4a7bb3 --- /dev/null +++ b/Misc/NEWS.d/next/Security/2020-07-03-20-41-29.bpo-41162.tb8pVj.rst @@ -0,0 +1 @@ +Audit hooks are now cleared later during finalization to avoid missing events. \ No newline at end of file diff --git a/Programs/_testembed.c b/Programs/_testembed.c index b98a38a1ba678..460d70ccadefd 100644 --- a/Programs/_testembed.c +++ b/Programs/_testembed.c @@ -1106,8 +1106,11 @@ static int test_open_code_hook(void) return result; } +static int _audit_hook_clear_count = 0; + static int _audit_hook(const char *event, PyObject *args, void *userdata) { + assert(args && PyTuple_CheckExact(args)); if (strcmp(event, "_testembed.raise") == 0) { PyErr_SetString(PyExc_RuntimeError, "Intentional error"); return -1; @@ -1116,6 +1119,8 @@ static int _audit_hook(const char *event, PyObject *args, void *userdata) return -1; } return 0; + } else if (strcmp(event, "cpython._PySys_ClearAuditHooks") == 0) { + _audit_hook_clear_count += 1; } return 0; } @@ -1161,6 +1166,9 @@ static int test_audit(void) { int result = _test_audit(42); Py_Finalize(); + if (_audit_hook_clear_count != 1) { + return 0x1000 | _audit_hook_clear_count; + } return result; } diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 27cebf33da544..dc2d13db419e5 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -1229,13 +1229,6 @@ Py_FinalizeEx(void) /* nothing */; #endif - /* Clear all loghooks */ - /* We want minimal exposure of this function, so define the extern - * here. The linker should discover the correct function without - * exporting a symbol. */ - extern void _PySys_ClearAuditHooks(void); - _PySys_ClearAuditHooks(); - /* Destroy all modules */ PyImport_Cleanup(); @@ -1306,6 +1299,13 @@ Py_FinalizeEx(void) /* Clear interpreter state and all thread states. */ PyInterpreterState_Clear(interp); + /* Clear all loghooks */ + /* We want minimal exposure of this function, so define the extern + * here. The linker should discover the correct function without + * exporting a symbol. */ + extern void _PySys_ClearAuditHooks(void); + _PySys_ClearAuditHooks(); + /* Now we decref the exception classes. After this point nothing can raise an exception. That's okay, because each Fini() method below has been checked to make sure no exceptions are ever

1 0

bpo-41180: Audit code.__new__ when unmarshalling (GH-21271)
by Miss Islington (bot) July 3, 2020

July 3, 2020

https://github.com/python/cpython/commit/c1d916595eb6979d4d87cc3e5216e26b3c… commit: c1d916595eb6979d4d87cc3e5216e26b3c6fac25 branch: 3.8 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: GitHub <noreply(a)github.com> date: 2020-07-03T14:13:29-07:00 summary: bpo-41180: Audit code.__new__ when unmarshalling (GH-21271) (cherry picked from commit d160e0f8e283d0a8737644588b38e8c6a07c134f) Co-authored-by: tkmikan <36260601+tkmikan(a)users.noreply.github.com> files: M Python/marshal.c diff --git a/Python/marshal.c b/Python/marshal.c index d3fee32380b58..a9ba7a436fd19 100644 --- a/Python/marshal.c +++ b/Python/marshal.c @@ -1396,6 +1396,12 @@ r_object(RFILE *p) if (lnotab == NULL) goto code_error; + if (PySys_Audit("code.__new__", "OOOiiiiii", + code, filename, name, argcount, posonlyargcount, + kwonlyargcount, nlocals, stacksize, flags) < 0) { + goto code_error; + } + v = (PyObject *) PyCode_NewWithPosOnlyArgs( argcount, posonlyargcount, kwonlyargcount, nlocals, stacksize, flags,

1 0

bpo-41162: Clear audit hooks later during finalization (GH-21222)
by Konge July 3, 2020

July 3, 2020

https://github.com/python/cpython/commit/daa0fe03a517d335d48e65ace8e5da636e… commit: daa0fe03a517d335d48e65ace8e5da636e265a8f branch: master author: Konge <zkonge(a)outlook.com> committer: GitHub <noreply(a)github.com> date: 2020-07-03T22:06:46+01:00 summary: bpo-41162: Clear audit hooks later during finalization (GH-21222) files: A Misc/NEWS.d/next/Security/2020-07-03-20-41-29.bpo-41162.tb8pVj.rst M Lib/test/audit-tests.py M Lib/test/test_audit.py M Programs/_testembed.c M Python/pylifecycle.c diff --git a/Lib/test/audit-tests.py b/Lib/test/audit-tests.py index a58395b068b39..ee6fc93351b75 100644 --- a/Lib/test/audit-tests.py +++ b/Lib/test/audit-tests.py @@ -44,28 +44,6 @@ def __call__(self, event, args): raise self.exc_type("saw event " + event) -class TestFinalizeHook: - """Used in the test_finalize_hooks function to ensure that hooks - are correctly cleaned up, that they are notified about the cleanup, - and are unable to prevent it. - """ - - def __init__(self): - print("Created", id(self), file=sys.stdout, flush=True) - - def __call__(self, event, args): - # Avoid recursion when we call id() below - if event == "builtins.id": - return - - print(event, id(self), file=sys.stdout, flush=True) - - if event == "cpython._PySys_ClearAuditHooks": - raise RuntimeError("Should be ignored") - elif event == "cpython.PyInterpreterState_Clear": - raise RuntimeError("Should be ignored") - - # Simple helpers, since we are not in unittest here def assertEqual(x, y): if x != y: @@ -128,10 +106,6 @@ def test_block_add_hook_baseexception(): pass -def test_finalize_hooks(): - sys.addaudithook(TestFinalizeHook()) - - def test_pickle(): import pickle diff --git a/Lib/test/test_audit.py b/Lib/test/test_audit.py index f405c6923979c..f79edbc4bd0d9 100644 --- a/Lib/test/test_audit.py +++ b/Lib/test/test_audit.py @@ -51,22 +51,6 @@ def test_block_add_hook(self): def test_block_add_hook_baseexception(self): self.do_test("test_block_add_hook_baseexception") - def test_finalize_hooks(self): - returncode, events, stderr = self.run_python("test_finalize_hooks") - if stderr: - print(stderr, file=sys.stderr) - if returncode: - self.fail(stderr) - - firstId = events[0][2] - self.assertSequenceEqual( - [ - ("Created", " ", firstId), - ("cpython._PySys_ClearAuditHooks", " ", firstId), - ], - events, - ) - def test_pickle(self): support.import_module("pickle") diff --git a/Misc/NEWS.d/next/Security/2020-07-03-20-41-29.bpo-41162.tb8pVj.rst b/Misc/NEWS.d/next/Security/2020-07-03-20-41-29.bpo-41162.tb8pVj.rst new file mode 100644 index 0000000000000..f0333ac4a7bb3 --- /dev/null +++ b/Misc/NEWS.d/next/Security/2020-07-03-20-41-29.bpo-41162.tb8pVj.rst @@ -0,0 +1 @@ +Audit hooks are now cleared later during finalization to avoid missing events. \ No newline at end of file diff --git a/Programs/_testembed.c b/Programs/_testembed.c index b60d70be5f71e..5aad16a6f7c47 100644 --- a/Programs/_testembed.c +++ b/Programs/_testembed.c @@ -1112,8 +1112,11 @@ static int test_open_code_hook(void) return result; } +static int _audit_hook_clear_count = 0; + static int _audit_hook(const char *event, PyObject *args, void *userdata) { + assert(args && PyTuple_CheckExact(args)); if (strcmp(event, "_testembed.raise") == 0) { PyErr_SetString(PyExc_RuntimeError, "Intentional error"); return -1; @@ -1122,6 +1125,8 @@ static int _audit_hook(const char *event, PyObject *args, void *userdata) return -1; } return 0; + } else if (strcmp(event, "cpython._PySys_ClearAuditHooks") == 0) { + _audit_hook_clear_count += 1; } return 0; } @@ -1167,6 +1172,9 @@ static int test_audit(void) { int result = _test_audit(42); Py_Finalize(); + if (_audit_hook_clear_count != 1) { + return 0x1000 | _audit_hook_clear_count; + } return result; } diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 3ce2c41ef1ff6..2d219a4a3a8b0 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -1291,6 +1291,13 @@ finalize_interp_clear(PyThreadState *tstate) _PyGC_CollectNoFail(); } + /* Clear all loghooks */ + /* Both _PySys_Audit function and users still need PyObject, such as tuple. + Call _PySys_ClearAuditHooks when PyObject available. */ + if (is_main_interp) { + _PySys_ClearAuditHooks(tstate); + } + _PyGC_Fini(tstate); if (is_main_interp) { @@ -1405,9 +1412,6 @@ Py_FinalizeEx(void) */ _PyGC_CollectIfEnabled(); - /* Clear all loghooks */ - _PySys_ClearAuditHooks(tstate); - /* Destroy all modules */ _PyImport_Cleanup(tstate);

1 0

bpo-41180: Audit code.__new__ when unmarshalling (GH-21271)
by tkmikan July 3, 2020

July 3, 2020

https://github.com/python/cpython/commit/d160e0f8e283d0a8737644588b38e8c6a0… commit: d160e0f8e283d0a8737644588b38e8c6a07c134f branch: master author: tkmikan <36260601+tkmikan(a)users.noreply.github.com> committer: GitHub <noreply(a)github.com> date: 2020-07-03T21:56:30+01:00 summary: bpo-41180: Audit code.__new__ when unmarshalling (GH-21271) files: M Python/marshal.c diff --git a/Python/marshal.c b/Python/marshal.c index a0f6b9812601b..c4538bd373a82 100644 --- a/Python/marshal.c +++ b/Python/marshal.c @@ -1371,6 +1371,12 @@ r_object(RFILE *p) if (lnotab == NULL) goto code_error; + if (PySys_Audit("code.__new__", "OOOiiiiii", + code, filename, name, argcount, posonlyargcount, + kwonlyargcount, nlocals, stacksize, flags) < 0) { + goto code_error; + } + v = (PyObject *) PyCode_NewWithPosOnlyArgs( argcount, posonlyargcount, kwonlyargcount, nlocals, stacksize, flags,

1 0

bpo-41194: Convert _ast extension to PEP 489 (GH-21293)
by Victor Stinner July 3, 2020

July 3, 2020

https://github.com/python/cpython/commit/b1cc6ba73a51d5cc3aeb113b5e7378fb50… commit: b1cc6ba73a51d5cc3aeb113b5e7378fb50a0e20a branch: master author: Victor Stinner <vstinner(a)python.org> committer: GitHub <noreply(a)github.com> date: 2020-07-03T20:01:46+02:00 summary: bpo-41194: Convert _ast extension to PEP 489 (GH-21293) Convert the _ast extension module to PEP 489 "Multiphase initialization". Replace the global _ast state with a module state. files: M Parser/asdl_c.py M Python/Python-ast.c diff --git a/Parser/asdl_c.py b/Parser/asdl_c.py index f029ca6618f93..b93906ba8d45d 100755 --- a/Parser/asdl_c.py +++ b/Parser/asdl_c.py @@ -692,6 +692,9 @@ def visitModule(self, mod): int res = -1; PyObject *key, *value, *fields; astmodulestate *state = get_global_ast_state(); + if (state == NULL) { + goto cleanup; + } if (_PyObject_LookupAttr((PyObject*)Py_TYPE(self), state->_fields, &fields) < 0) { goto cleanup; } @@ -761,6 +764,10 @@ def visitModule(self, mod): ast_type_reduce(PyObject *self, PyObject *unused) { astmodulestate *state = get_global_ast_state(); + if (state == NULL) { + return NULL; + } + PyObject *dict; if (_PyObject_LookupAttr(self, state->__dict__, &dict) < 0) { return NULL; @@ -969,9 +976,8 @@ def visitModule(self, mod): """, 0, reflow=False) - self.emit("static int init_types(void)",0) + self.emit("static int init_types(astmodulestate *state)",0) self.emit("{", 0) - self.emit("astmodulestate *state = get_global_ast_state();", 1) self.emit("if (state->initialized) return 1;", 1) self.emit("if (init_identifiers(state) < 0) return 0;", 1) self.emit("state->AST_type = PyType_FromSpec(&AST_type_spec);", 1) @@ -1046,40 +1052,55 @@ def emit_defaults(self, name, fields, depth): class ASTModuleVisitor(PickleVisitor): def visitModule(self, mod): - self.emit("PyMODINIT_FUNC", 0) - self.emit("PyInit__ast(void)", 0) + self.emit("static int", 0) + self.emit("astmodule_exec(PyObject *m)", 0) self.emit("{", 0) - self.emit("PyObject *m = PyModule_Create(&_astmodule);", 1) - self.emit("if (!m) {", 1) - self.emit("return NULL;", 2) - self.emit("}", 1) self.emit('astmodulestate *state = get_ast_state(m);', 1) - self.emit('', 1) + self.emit("", 0) - self.emit("if (!init_types()) {", 1) - self.emit("goto error;", 2) + self.emit("if (!init_types(state)) {", 1) + self.emit("return -1;", 2) self.emit("}", 1) self.emit('if (PyModule_AddObject(m, "AST", state->AST_type) < 0) {', 1) - self.emit('goto error;', 2) + self.emit('return -1;', 2) self.emit('}', 1) self.emit('Py_INCREF(state->AST_type);', 1) self.emit('if (PyModule_AddIntMacro(m, PyCF_ALLOW_TOP_LEVEL_AWAIT) < 0) {', 1) - self.emit("goto error;", 2) + self.emit("return -1;", 2) self.emit('}', 1) self.emit('if (PyModule_AddIntMacro(m, PyCF_ONLY_AST) < 0) {', 1) - self.emit("goto error;", 2) + self.emit("return -1;", 2) self.emit('}', 1) self.emit('if (PyModule_AddIntMacro(m, PyCF_TYPE_COMMENTS) < 0) {', 1) - self.emit("goto error;", 2) + self.emit("return -1;", 2) self.emit('}', 1) for dfn in mod.dfns: self.visit(dfn) - self.emit("return m;", 1) - self.emit("", 0) - self.emit("error:", 0) - self.emit("Py_DECREF(m);", 1) - self.emit("return NULL;", 1) + self.emit("return 0;", 1) self.emit("}", 0) + self.emit("", 0) + self.emit(""" +static PyModuleDef_Slot astmodule_slots[] = { + {Py_mod_exec, astmodule_exec}, + {0, NULL} +}; + +static struct PyModuleDef _astmodule = { + PyModuleDef_HEAD_INIT, + .m_name = "_ast", + .m_size = sizeof(astmodulestate), + .m_slots = astmodule_slots, + .m_traverse = astmodule_traverse, + .m_clear = astmodule_clear, + .m_free = astmodule_free, +}; + +PyMODINIT_FUNC +PyInit__ast(void) +{ + return PyModuleDef_Init(&_astmodule); +} +""".strip(), 0, reflow=False) def visitProduct(self, prod, name): self.addObj(name) @@ -1095,7 +1116,7 @@ def visitConstructor(self, cons, name): def addObj(self, name): self.emit("if (PyModule_AddObject(m, \"%s\", " "state->%s_type) < 0) {" % (name, name), 1) - self.emit("goto error;", 2) + self.emit("return -1;", 2) self.emit('}', 1) self.emit("Py_INCREF(state->%s_type);" % name, 1) @@ -1255,11 +1276,10 @@ class PartingShots(StaticVisitor): CODE = """ PyObject* PyAST_mod2obj(mod_ty t) { - if (!init_types()) { + astmodulestate *state = get_global_ast_state(); + if (state == NULL) { return NULL; } - - astmodulestate *state = get_global_ast_state(); return ast2obj_mod(state, t); } @@ -1281,10 +1301,6 @@ class PartingShots(StaticVisitor): assert(0 <= mode && mode <= 2); - if (!init_types()) { - return NULL; - } - isinstance = PyObject_IsInstance(ast, req_type[mode]); if (isinstance == -1) return NULL; @@ -1303,11 +1319,10 @@ class PartingShots(StaticVisitor): int PyAST_Check(PyObject* obj) { - if (!init_types()) { + astmodulestate *state = get_global_ast_state(); + if (state == NULL) { return -1; } - - astmodulestate *state = get_global_ast_state(); return PyObject_IsInstance(obj, state->AST_type); } """ @@ -1358,12 +1373,35 @@ def generate_module_def(f, mod): f.write(' PyObject *' + s + ';\n') f.write('} astmodulestate;\n\n') f.write(""" -static astmodulestate global_ast_state; +static astmodulestate* +get_ast_state(PyObject *module) +{ + void *state = PyModule_GetState(module); + assert(state != NULL); + return (astmodulestate*)state; +} -static astmodulestate * -get_ast_state(PyObject *Py_UNUSED(module)) +static astmodulestate* +get_global_ast_state(void) { - return &global_ast_state; + _Py_IDENTIFIER(_ast); + PyObject *name = _PyUnicode_FromId(&PyId__ast); // borrowed reference + if (name == NULL) { + return NULL; + } + PyObject *module = PyImport_GetModule(name); + if (module == NULL) { + if (PyErr_Occurred()) { + return NULL; + } + module = PyImport_Import(name); + if (module == NULL) { + return NULL; + } + } + astmodulestate *state = get_ast_state(module); + Py_DECREF(module); + return state; } static int astmodule_clear(PyObject *module) @@ -1390,17 +1428,6 @@ def generate_module_def(f, mod): astmodule_clear((PyObject*)module); } -static struct PyModuleDef _astmodule = { - PyModuleDef_HEAD_INIT, - .m_name = "_ast", - .m_size = -1, - .m_traverse = astmodule_traverse, - .m_clear = astmodule_clear, - .m_free = astmodule_free, -}; - -#define get_global_ast_state() (&global_ast_state) - """) f.write('static int init_identifiers(astmodulestate *state)\n') f.write('{\n') diff --git a/Python/Python-ast.c b/Python/Python-ast.c index 844033ee37e28..b81b282a03965 100644 --- a/Python/Python-ast.c +++ b/Python/Python-ast.c @@ -224,12 +224,35 @@ typedef struct { } astmodulestate; -static astmodulestate global_ast_state; +static astmodulestate* +get_ast_state(PyObject *module) +{ + void *state = PyModule_GetState(module); + assert(state != NULL); + return (astmodulestate*)state; +} -static astmodulestate * -get_ast_state(PyObject *Py_UNUSED(module)) +static astmodulestate* +get_global_ast_state(void) { - return &global_ast_state; + _Py_IDENTIFIER(_ast); + PyObject *name = _PyUnicode_FromId(&PyId__ast); // borrowed reference + if (name == NULL) { + return NULL; + } + PyObject *module = PyImport_GetModule(name); + if (module == NULL) { + if (PyErr_Occurred()) { + return NULL; + } + module = PyImport_Import(name); + if (module == NULL) { + return NULL; + } + } + astmodulestate *state = get_ast_state(module); + Py_DECREF(module); + return state; } static int astmodule_clear(PyObject *module) @@ -676,17 +699,6 @@ static void astmodule_free(void* module) { astmodule_clear((PyObject*)module); } -static struct PyModuleDef _astmodule = { - PyModuleDef_HEAD_INIT, - .m_name = "_ast", - .m_size = -1, - .m_traverse = astmodule_traverse, - .m_clear = astmodule_clear, - .m_free = astmodule_free, -}; - -#define get_global_ast_state() (&global_ast_state) - static int init_identifiers(astmodulestate *state) { if ((state->__dict__ = PyUnicode_InternFromString("__dict__")) == NULL) return 0; @@ -1132,6 +1144,9 @@ ast_type_init(PyObject *self, PyObject *args, PyObject *kw) int res = -1; PyObject *key, *value, *fields; astmodulestate *state = get_global_ast_state(); + if (state == NULL) { + goto cleanup; + } if (_PyObject_LookupAttr((PyObject*)Py_TYPE(self), state->_fields, &fields) < 0) { goto cleanup; } @@ -1201,6 +1216,10 @@ static PyObject * ast_type_reduce(PyObject *self, PyObject *unused) { astmodulestate *state = get_global_ast_state(); + if (state == NULL) { + return NULL; + } + PyObject *dict; if (_PyObject_LookupAttr(self, state->__dict__, &dict) < 0) { return NULL; @@ -1408,9 +1427,8 @@ static int add_ast_fields(astmodulestate *state) } -static int init_types(void) +static int init_types(astmodulestate *state) { - astmodulestate *state = get_global_ast_state(); if (state->initialized) return 1; if (init_identifiers(state) < 0) return 0; state->AST_type = PyType_FromSpec(&AST_type_spec); @@ -9839,472 +9857,484 @@ obj2ast_type_ignore(astmodulestate *state, PyObject* obj, type_ignore_ty* out, } -PyMODINIT_FUNC -PyInit__ast(void) +static int +astmodule_exec(PyObject *m) { - PyObject *m = PyModule_Create(&_astmodule); - if (!m) { - return NULL; - } astmodulestate *state = get_ast_state(m); - if (!init_types()) { - goto error; + if (!init_types(state)) { + return -1; } if (PyModule_AddObject(m, "AST", state->AST_type) < 0) { - goto error; + return -1; } Py_INCREF(state->AST_type); if (PyModule_AddIntMacro(m, PyCF_ALLOW_TOP_LEVEL_AWAIT) < 0) { - goto error; + return -1; } if (PyModule_AddIntMacro(m, PyCF_ONLY_AST) < 0) { - goto error; + return -1; } if (PyModule_AddIntMacro(m, PyCF_TYPE_COMMENTS) < 0) { - goto error; + return -1; } if (PyModule_AddObject(m, "mod", state->mod_type) < 0) { - goto error; + return -1; } Py_INCREF(state->mod_type); if (PyModule_AddObject(m, "Module", state->Module_type) < 0) { - goto error; + return -1; } Py_INCREF(state->Module_type); if (PyModule_AddObject(m, "Interactive", state->Interactive_type) < 0) { - goto error; + return -1; } Py_INCREF(state->Interactive_type); if (PyModule_AddObject(m, "Expression", state->Expression_type) < 0) { - goto error; + return -1; } Py_INCREF(state->Expression_type); if (PyModule_AddObject(m, "FunctionType", state->FunctionType_type) < 0) { - goto error; + return -1; } Py_INCREF(state->FunctionType_type); if (PyModule_AddObject(m, "stmt", state->stmt_type) < 0) { - goto error; + return -1; } Py_INCREF(state->stmt_type); if (PyModule_AddObject(m, "FunctionDef", state->FunctionDef_type) < 0) { - goto error; + return -1; } Py_INCREF(state->FunctionDef_type); if (PyModule_AddObject(m, "AsyncFunctionDef", state->AsyncFunctionDef_type) < 0) { - goto error; + return -1; } Py_INCREF(state->AsyncFunctionDef_type); if (PyModule_AddObject(m, "ClassDef", state->ClassDef_type) < 0) { - goto error; + return -1; } Py_INCREF(state->ClassDef_type); if (PyModule_AddObject(m, "Return", state->Return_type) < 0) { - goto error; + return -1; } Py_INCREF(state->Return_type); if (PyModule_AddObject(m, "Delete", state->Delete_type) < 0) { - goto error; + return -1; } Py_INCREF(state->Delete_type); if (PyModule_AddObject(m, "Assign", state->Assign_type) < 0) { - goto error; + return -1; } Py_INCREF(state->Assign_type); if (PyModule_AddObject(m, "AugAssign", state->AugAssign_type) < 0) { - goto error; + return -1; } Py_INCREF(state->AugAssign_type); if (PyModule_AddObject(m, "AnnAssign", state->AnnAssign_type) < 0) { - goto error; + return -1; } Py_INCREF(state->AnnAssign_type); if (PyModule_AddObject(m, "For", state->For_type) < 0) { - goto error; + return -1; } Py_INCREF(state->For_type); if (PyModule_AddObject(m, "AsyncFor", state->AsyncFor_type) < 0) { - goto error; + return -1; } Py_INCREF(state->AsyncFor_type); if (PyModule_AddObject(m, "While", state->While_type) < 0) { - goto error; + return -1; } Py_INCREF(state->While_type); if (PyModule_AddObject(m, "If", state->If_type) < 0) { - goto error; + return -1; } Py_INCREF(state->If_type); if (PyModule_AddObject(m, "With", state->With_type) < 0) { - goto error; + return -1; } Py_INCREF(state->With_type); if (PyModule_AddObject(m, "AsyncWith", state->AsyncWith_type) < 0) { - goto error; + return -1; } Py_INCREF(state->AsyncWith_type); if (PyModule_AddObject(m, "Raise", state->Raise_type) < 0) { - goto error; + return -1; } Py_INCREF(state->Raise_type); if (PyModule_AddObject(m, "Try", state->Try_type) < 0) { - goto error; + return -1; } Py_INCREF(state->Try_type); if (PyModule_AddObject(m, "Assert", state->Assert_type) < 0) { - goto error; + return -1; } Py_INCREF(state->Assert_type); if (PyModule_AddObject(m, "Import", state->Import_type) < 0) { - goto error; + return -1; } Py_INCREF(state->Import_type); if (PyModule_AddObject(m, "ImportFrom", state->ImportFrom_type) < 0) { - goto error; + return -1; } Py_INCREF(state->ImportFrom_type); if (PyModule_AddObject(m, "Global", state->Global_type) < 0) { - goto error; + return -1; } Py_INCREF(state->Global_type); if (PyModule_AddObject(m, "Nonlocal", state->Nonlocal_type) < 0) { - goto error; + return -1; } Py_INCREF(state->Nonlocal_type); if (PyModule_AddObject(m, "Expr", state->Expr_type) < 0) { - goto error; + return -1; } Py_INCREF(state->Expr_type); if (PyModule_AddObject(m, "Pass", state->Pass_type) < 0) { - goto error; + return -1; } Py_INCREF(state->Pass_type); if (PyModule_AddObject(m, "Break", state->Break_type) < 0) { - goto error; + return -1; } Py_INCREF(state->Break_type); if (PyModule_AddObject(m, "Continue", state->Continue_type) < 0) { - goto error; + return -1; } Py_INCREF(state->Continue_type); if (PyModule_AddObject(m, "expr", state->expr_type) < 0) { - goto error; + return -1; } Py_INCREF(state->expr_type); if (PyModule_AddObject(m, "BoolOp", state->BoolOp_type) < 0) { - goto error; + return -1; } Py_INCREF(state->BoolOp_type); if (PyModule_AddObject(m, "NamedExpr", state->NamedExpr_type) < 0) { - goto error; + return -1; } Py_INCREF(state->NamedExpr_type); if (PyModule_AddObject(m, "BinOp", state->BinOp_type) < 0) { - goto error; + return -1; } Py_INCREF(state->BinOp_type); if (PyModule_AddObject(m, "UnaryOp", state->UnaryOp_type) < 0) { - goto error; + return -1; } Py_INCREF(state->UnaryOp_type); if (PyModule_AddObject(m, "Lambda", state->Lambda_type) < 0) { - goto error; + return -1; } Py_INCREF(state->Lambda_type); if (PyModule_AddObject(m, "IfExp", state->IfExp_type) < 0) { - goto error; + return -1; } Py_INCREF(state->IfExp_type); if (PyModule_AddObject(m, "Dict", state->Dict_type) < 0) { - goto error; + return -1; } Py_INCREF(state->Dict_type); if (PyModule_AddObject(m, "Set", state->Set_type) < 0) { - goto error; + return -1; } Py_INCREF(state->Set_type); if (PyModule_AddObject(m, "ListComp", state->ListComp_type) < 0) { - goto error; + return -1; } Py_INCREF(state->ListComp_type); if (PyModule_AddObject(m, "SetComp", state->SetComp_type) < 0) { - goto error; + return -1; } Py_INCREF(state->SetComp_type); if (PyModule_AddObject(m, "DictComp", state->DictComp_type) < 0) { - goto error; + return -1; } Py_INCREF(state->DictComp_type); if (PyModule_AddObject(m, "GeneratorExp", state->GeneratorExp_type) < 0) { - goto error; + return -1; } Py_INCREF(state->GeneratorExp_type); if (PyModule_AddObject(m, "Await", state->Await_type) < 0) { - goto error; + return -1; } Py_INCREF(state->Await_type); if (PyModule_AddObject(m, "Yield", state->Yield_type) < 0) { - goto error; + return -1; } Py_INCREF(state->Yield_type); if (PyModule_AddObject(m, "YieldFrom", state->YieldFrom_type) < 0) { - goto error; + return -1; } Py_INCREF(state->YieldFrom_type); if (PyModule_AddObject(m, "Compare", state->Compare_type) < 0) { - goto error; + return -1; } Py_INCREF(state->Compare_type); if (PyModule_AddObject(m, "Call", state->Call_type) < 0) { - goto error; + return -1; } Py_INCREF(state->Call_type); if (PyModule_AddObject(m, "FormattedValue", state->FormattedValue_type) < 0) { - goto error; + return -1; } Py_INCREF(state->FormattedValue_type); if (PyModule_AddObject(m, "JoinedStr", state->JoinedStr_type) < 0) { - goto error; + return -1; } Py_INCREF(state->JoinedStr_type); if (PyModule_AddObject(m, "Constant", state->Constant_type) < 0) { - goto error; + return -1; } Py_INCREF(state->Constant_type); if (PyModule_AddObject(m, "Attribute", state->Attribute_type) < 0) { - goto error; + return -1; } Py_INCREF(state->Attribute_type); if (PyModule_AddObject(m, "Subscript", state->Subscript_type) < 0) { - goto error; + return -1; } Py_INCREF(state->Subscript_type); if (PyModule_AddObject(m, "Starred", state->Starred_type) < 0) { - goto error; + return -1; } Py_INCREF(state->Starred_type); if (PyModule_AddObject(m, "Name", state->Name_type) < 0) { - goto error; + return -1; } Py_INCREF(state->Name_type); if (PyModule_AddObject(m, "List", state->List_type) < 0) { - goto error; + return -1; } Py_INCREF(state->List_type); if (PyModule_AddObject(m, "Tuple", state->Tuple_type) < 0) { - goto error; + return -1; } Py_INCREF(state->Tuple_type); if (PyModule_AddObject(m, "Slice", state->Slice_type) < 0) { - goto error; + return -1; } Py_INCREF(state->Slice_type); if (PyModule_AddObject(m, "expr_context", state->expr_context_type) < 0) { - goto error; + return -1; } Py_INCREF(state->expr_context_type); if (PyModule_AddObject(m, "Load", state->Load_type) < 0) { - goto error; + return -1; } Py_INCREF(state->Load_type); if (PyModule_AddObject(m, "Store", state->Store_type) < 0) { - goto error; + return -1; } Py_INCREF(state->Store_type); if (PyModule_AddObject(m, "Del", state->Del_type) < 0) { - goto error; + return -1; } Py_INCREF(state->Del_type); if (PyModule_AddObject(m, "boolop", state->boolop_type) < 0) { - goto error; + return -1; } Py_INCREF(state->boolop_type); if (PyModule_AddObject(m, "And", state->And_type) < 0) { - goto error; + return -1; } Py_INCREF(state->And_type); if (PyModule_AddObject(m, "Or", state->Or_type) < 0) { - goto error; + return -1; } Py_INCREF(state->Or_type); if (PyModule_AddObject(m, "operator", state->operator_type) < 0) { - goto error; + return -1; } Py_INCREF(state->operator_type); if (PyModule_AddObject(m, "Add", state->Add_type) < 0) { - goto error; + return -1; } Py_INCREF(state->Add_type); if (PyModule_AddObject(m, "Sub", state->Sub_type) < 0) { - goto error; + return -1; } Py_INCREF(state->Sub_type); if (PyModule_AddObject(m, "Mult", state->Mult_type) < 0) { - goto error; + return -1; } Py_INCREF(state->Mult_type); if (PyModule_AddObject(m, "MatMult", state->MatMult_type) < 0) { - goto error; + return -1; } Py_INCREF(state->MatMult_type); if (PyModule_AddObject(m, "Div", state->Div_type) < 0) { - goto error; + return -1; } Py_INCREF(state->Div_type); if (PyModule_AddObject(m, "Mod", state->Mod_type) < 0) { - goto error; + return -1; } Py_INCREF(state->Mod_type); if (PyModule_AddObject(m, "Pow", state->Pow_type) < 0) { - goto error; + return -1; } Py_INCREF(state->Pow_type); if (PyModule_AddObject(m, "LShift", state->LShift_type) < 0) { - goto error; + return -1; } Py_INCREF(state->LShift_type); if (PyModule_AddObject(m, "RShift", state->RShift_type) < 0) { - goto error; + return -1; } Py_INCREF(state->RShift_type); if (PyModule_AddObject(m, "BitOr", state->BitOr_type) < 0) { - goto error; + return -1; } Py_INCREF(state->BitOr_type); if (PyModule_AddObject(m, "BitXor", state->BitXor_type) < 0) { - goto error; + return -1; } Py_INCREF(state->BitXor_type); if (PyModule_AddObject(m, "BitAnd", state->BitAnd_type) < 0) { - goto error; + return -1; } Py_INCREF(state->BitAnd_type); if (PyModule_AddObject(m, "FloorDiv", state->FloorDiv_type) < 0) { - goto error; + return -1; } Py_INCREF(state->FloorDiv_type); if (PyModule_AddObject(m, "unaryop", state->unaryop_type) < 0) { - goto error; + return -1; } Py_INCREF(state->unaryop_type); if (PyModule_AddObject(m, "Invert", state->Invert_type) < 0) { - goto error; + return -1; } Py_INCREF(state->Invert_type); if (PyModule_AddObject(m, "Not", state->Not_type) < 0) { - goto error; + return -1; } Py_INCREF(state->Not_type); if (PyModule_AddObject(m, "UAdd", state->UAdd_type) < 0) { - goto error; + return -1; } Py_INCREF(state->UAdd_type); if (PyModule_AddObject(m, "USub", state->USub_type) < 0) { - goto error; + return -1; } Py_INCREF(state->USub_type); if (PyModule_AddObject(m, "cmpop", state->cmpop_type) < 0) { - goto error; + return -1; } Py_INCREF(state->cmpop_type); if (PyModule_AddObject(m, "Eq", state->Eq_type) < 0) { - goto error; + return -1; } Py_INCREF(state->Eq_type); if (PyModule_AddObject(m, "NotEq", state->NotEq_type) < 0) { - goto error; + return -1; } Py_INCREF(state->NotEq_type); if (PyModule_AddObject(m, "Lt", state->Lt_type) < 0) { - goto error; + return -1; } Py_INCREF(state->Lt_type); if (PyModule_AddObject(m, "LtE", state->LtE_type) < 0) { - goto error; + return -1; } Py_INCREF(state->LtE_type); if (PyModule_AddObject(m, "Gt", state->Gt_type) < 0) { - goto error; + return -1; } Py_INCREF(state->Gt_type); if (PyModule_AddObject(m, "GtE", state->GtE_type) < 0) { - goto error; + return -1; } Py_INCREF(state->GtE_type); if (PyModule_AddObject(m, "Is", state->Is_type) < 0) { - goto error; + return -1; } Py_INCREF(state->Is_type); if (PyModule_AddObject(m, "IsNot", state->IsNot_type) < 0) { - goto error; + return -1; } Py_INCREF(state->IsNot_type); if (PyModule_AddObject(m, "In", state->In_type) < 0) { - goto error; + return -1; } Py_INCREF(state->In_type); if (PyModule_AddObject(m, "NotIn", state->NotIn_type) < 0) { - goto error; + return -1; } Py_INCREF(state->NotIn_type); if (PyModule_AddObject(m, "comprehension", state->comprehension_type) < 0) { - goto error; + return -1; } Py_INCREF(state->comprehension_type); if (PyModule_AddObject(m, "excepthandler", state->excepthandler_type) < 0) { - goto error; + return -1; } Py_INCREF(state->excepthandler_type); if (PyModule_AddObject(m, "ExceptHandler", state->ExceptHandler_type) < 0) { - goto error; + return -1; } Py_INCREF(state->ExceptHandler_type); if (PyModule_AddObject(m, "arguments", state->arguments_type) < 0) { - goto error; + return -1; } Py_INCREF(state->arguments_type); if (PyModule_AddObject(m, "arg", state->arg_type) < 0) { - goto error; + return -1; } Py_INCREF(state->arg_type); if (PyModule_AddObject(m, "keyword", state->keyword_type) < 0) { - goto error; + return -1; } Py_INCREF(state->keyword_type); if (PyModule_AddObject(m, "alias", state->alias_type) < 0) { - goto error; + return -1; } Py_INCREF(state->alias_type); if (PyModule_AddObject(m, "withitem", state->withitem_type) < 0) { - goto error; + return -1; } Py_INCREF(state->withitem_type); if (PyModule_AddObject(m, "type_ignore", state->type_ignore_type) < 0) { - goto error; + return -1; } Py_INCREF(state->type_ignore_type); if (PyModule_AddObject(m, "TypeIgnore", state->TypeIgnore_type) < 0) { - goto error; + return -1; } Py_INCREF(state->TypeIgnore_type); - return m; + return 0; +} -error: - Py_DECREF(m); - return NULL; +static PyModuleDef_Slot astmodule_slots[] = { + {Py_mod_exec, astmodule_exec}, + {0, NULL} +}; + +static struct PyModuleDef _astmodule = { + PyModuleDef_HEAD_INIT, + .m_name = "_ast", + .m_size = sizeof(astmodulestate), + .m_slots = astmodule_slots, + .m_traverse = astmodule_traverse, + .m_clear = astmodule_clear, + .m_free = astmodule_free, +}; + +PyMODINIT_FUNC +PyInit__ast(void) +{ + return PyModuleDef_Init(&_astmodule); } PyObject* PyAST_mod2obj(mod_ty t) { - if (!init_types()) { + astmodulestate *state = get_global_ast_state(); + if (state == NULL) { return NULL; } - - astmodulestate *state = get_global_ast_state(); return ast2obj_mod(state, t); } @@ -10326,10 +10356,6 @@ mod_ty PyAST_obj2mod(PyObject* ast, PyArena* arena, int mode) assert(0 <= mode && mode <= 2); - if (!init_types()) { - return NULL; - } - isinstance = PyObject_IsInstance(ast, req_type[mode]); if (isinstance == -1) return NULL; @@ -10348,11 +10374,10 @@ mod_ty PyAST_obj2mod(PyObject* ast, PyArena* arena, int mode) int PyAST_Check(PyObject* obj) { - if (!init_types()) { + astmodulestate *state = get_global_ast_state(); + if (state == NULL) { return -1; } - - astmodulestate *state = get_global_ast_state(); return PyObject_IsInstance(obj, state->AST_type); }

1 0

bpo-1635741: Port faulthandler module to multiphase initialization (GH-21294)
by Dong-hee Na July 3, 2020

July 3, 2020

https://github.com/python/cpython/commit/c0b214bc08f0da89e5b2e4b8cc9f077838… commit: c0b214bc08f0da89e5b2e4b8cc9f07783833d6b8 branch: master author: Dong-hee Na <donghee.na92(a)gmail.com> committer: GitHub <noreply(a)github.com> date: 2020-07-04T01:36:47+09:00 summary: bpo-1635741: Port faulthandler module to multiphase initialization (GH-21294) files: A Misc/NEWS.d/next/Core and Builtins/2020-07-03-23-10-02.bpo-1635741.F5coWe.rst M Modules/faulthandler.c diff --git a/Misc/NEWS.d/next/Core and Builtins/2020-07-03-23-10-02.bpo-1635741.F5coWe.rst b/Misc/NEWS.d/next/Core and Builtins/2020-07-03-23-10-02.bpo-1635741.F5coWe.rst new file mode 100644 index 0000000000000..927c8e5b7083f --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2020-07-03-23-10-02.bpo-1635741.F5coWe.rst @@ -0,0 +1 @@ +Port :mod:`faulthandler` to multiphase initialization. diff --git a/Modules/faulthandler.c b/Modules/faulthandler.c index e7a285033051d..67fe1ca9ffffd 100644 --- a/Modules/faulthandler.c +++ b/Modules/faulthandler.c @@ -1291,59 +1291,55 @@ static PyMethodDef module_methods[] = { {NULL, NULL} /* sentinel */ }; -static struct PyModuleDef module_def = { - PyModuleDef_HEAD_INIT, - "faulthandler", - module_doc, - 0, /* non-negative size to be able to unload the module */ - module_methods, - NULL, - faulthandler_traverse, - NULL, - NULL -}; - -PyMODINIT_FUNC -PyInit_faulthandler(void) -{ - PyObject *m = PyModule_Create(&module_def); - if (m == NULL) - return NULL; - +static int +PyExec_faulthandler(PyObject *module) { /* Add constants for unit tests */ #ifdef MS_WINDOWS /* RaiseException() codes (prefixed by an underscore) */ - if (PyModule_AddIntConstant(m, "_EXCEPTION_ACCESS_VIOLATION", + if (PyModule_AddIntConstant(module, "_EXCEPTION_ACCESS_VIOLATION", EXCEPTION_ACCESS_VIOLATION)) { - goto error; + return -1; } - if (PyModule_AddIntConstant(m, "_EXCEPTION_INT_DIVIDE_BY_ZERO", + if (PyModule_AddIntConstant(module, "_EXCEPTION_INT_DIVIDE_BY_ZERO", EXCEPTION_INT_DIVIDE_BY_ZERO)) { - goto error; + return -1; } - if (PyModule_AddIntConstant(m, "_EXCEPTION_STACK_OVERFLOW", + if (PyModule_AddIntConstant(module, "_EXCEPTION_STACK_OVERFLOW", EXCEPTION_STACK_OVERFLOW)) { - goto error; + return -1; } /* RaiseException() flags (prefixed by an underscore) */ - if (PyModule_AddIntConstant(m, "_EXCEPTION_NONCONTINUABLE", + if (PyModule_AddIntConstant(module, "_EXCEPTION_NONCONTINUABLE", EXCEPTION_NONCONTINUABLE)) { - goto error; + return -1; } - if (PyModule_AddIntConstant(m, "_EXCEPTION_NONCONTINUABLE_EXCEPTION", + if (PyModule_AddIntConstant(module, "_EXCEPTION_NONCONTINUABLE_EXCEPTION", EXCEPTION_NONCONTINUABLE_EXCEPTION)) { - goto error; + return -1; } #endif + return 0; +} - return m; +static PyModuleDef_Slot faulthandler_slots[] = { + {Py_mod_exec, PyExec_faulthandler}, + {0, NULL} +}; -#ifdef MS_WINDOWS -error: - Py_DECREF(m); - return NULL; -#endif +static struct PyModuleDef module_def = { + PyModuleDef_HEAD_INIT, + .m_name = "faulthandler", + .m_doc = module_doc, + .m_methods = module_methods, + .m_traverse = faulthandler_traverse, + .m_slots = faulthandler_slots +}; + +PyMODINIT_FUNC +PyInit_faulthandler(void) +{ + return PyModuleDef_Init(&module_def); } static int

1 0

bpo-1635741: Fix unicode_dealloc() for mortal interned string (GH-21270)
by Victor Stinner July 3, 2020

July 3, 2020

https://github.com/python/cpython/commit/3549ca313a6103a3adb281ef3a849298b7… commit: 3549ca313a6103a3adb281ef3a849298b7d7f72c branch: master author: Victor Stinner <vstinner(a)python.org> committer: GitHub <noreply(a)github.com> date: 2020-07-03T16:59:12+02:00 summary: bpo-1635741: Fix unicode_dealloc() for mortal interned string (GH-21270) When unicode_dealloc() is called on a mortal interned string, the string reference counter is now reset at zero. files: M Objects/unicodeobject.c diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 37e7fe5c0eff2..ca68c57534b22 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -1943,13 +1943,20 @@ unicode_dealloc(PyObject *unicode) break; case SSTATE_INTERNED_MORTAL: - /* revive dead object temporarily for DelItem */ - Py_SET_REFCNT(unicode, 3); #ifdef INTERNED_STRINGS + /* Revive the dead object temporarily. PyDict_DelItem() removes two + references (key and value) which were ignored by + PyUnicode_InternInPlace(). Use refcnt=3 rather than refcnt=2 + to prevent calling unicode_dealloc() again. Adjust refcnt after + PyDict_DelItem(). */ + assert(Py_REFCNT(unicode) == 0); + Py_SET_REFCNT(unicode, 3); if (PyDict_DelItem(interned, unicode) != 0) { _PyErr_WriteUnraisableMsg("deletion of interned string failed", NULL); } + assert(Py_REFCNT(unicode) == 1); + Py_SET_REFCNT(unicode, 0); #endif break; @@ -15710,8 +15717,9 @@ PyUnicode_InternInPlace(PyObject **p) return; } - /* The two references in interned are not counted by refcnt. - The deallocator will take care of this */ + /* The two references in interned dict (key and value) are not counted by + refcnt. unicode_dealloc() and _PyUnicode_ClearInterned() take care of + this. */ Py_SET_REFCNT(s, Py_REFCNT(s) - 2); _PyUnicode_STATE(s).interned = SSTATE_INTERNED_MORTAL; #endif @@ -15780,6 +15788,8 @@ _PyUnicode_ClearInterned(PyThreadState *tstate) #endif break; case SSTATE_INTERNED_MORTAL: + // Restore the two references (key and value) ignored + // by PyUnicode_InternInPlace(). Py_SET_REFCNT(s, Py_REFCNT(s) + 2); #ifdef INTERNED_STATS mortal_size += PyUnicode_GET_LENGTH(s);

1 0

bpo-41194: The _ast module cannot be loaded more than once (GH-21290)
by Victor Stinner July 3, 2020

July 3, 2020

https://github.com/python/cpython/commit/91e1bc18bd467a13bceb62e16fbc435b33… commit: 91e1bc18bd467a13bceb62e16fbc435b33381c82 branch: master author: Victor Stinner <vstinner(a)python.org> committer: GitHub <noreply(a)github.com> date: 2020-07-03T14:15:53+02:00 summary: bpo-41194: The _ast module cannot be loaded more than once (GH-21290) Fix a crash in the _ast module: it can no longer be loaded more than once. It now uses a global state rather than a module state. * Move _ast module state: use a global state instead. * Set _astmodule.m_size to -1, so the extension cannot be loaded more than once. files: A Misc/NEWS.d/next/Library/2020-07-03-13-15-08.bpo-41194.djrKjs.rst M Parser/asdl_c.py M Python/Python-ast.c diff --git a/Misc/NEWS.d/next/Library/2020-07-03-13-15-08.bpo-41194.djrKjs.rst b/Misc/NEWS.d/next/Library/2020-07-03-13-15-08.bpo-41194.djrKjs.rst new file mode 100644 index 0000000000000..d63a0e5222ba9 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2020-07-03-13-15-08.bpo-41194.djrKjs.rst @@ -0,0 +1,2 @@ +Fix a crash in the ``_ast`` module: it can no longer be loaded more than once. +It now uses a global state rather than a module state. diff --git a/Parser/asdl_c.py b/Parser/asdl_c.py index 39e216b8192b1..f029ca6618f93 100755 --- a/Parser/asdl_c.py +++ b/Parser/asdl_c.py @@ -691,7 +691,7 @@ def visitModule(self, mod): Py_ssize_t i, numfields = 0; int res = -1; PyObject *key, *value, *fields; - astmodulestate *state = astmodulestate_global; + astmodulestate *state = get_global_ast_state(); if (_PyObject_LookupAttr((PyObject*)Py_TYPE(self), state->_fields, &fields) < 0) { goto cleanup; } @@ -760,7 +760,7 @@ def visitModule(self, mod): static PyObject * ast_type_reduce(PyObject *self, PyObject *unused) { - astmodulestate *state = astmodulestate_global; + astmodulestate *state = get_global_ast_state(); PyObject *dict; if (_PyObject_LookupAttr(self, state->__dict__, &dict) < 0) { return NULL; @@ -971,19 +971,7 @@ def visitModule(self, mod): self.emit("static int init_types(void)",0) self.emit("{", 0) - self.emit("PyObject *module = PyState_FindModule(&_astmodule);", 1) - self.emit("if (module == NULL) {", 1) - self.emit("module = PyModule_Create(&_astmodule);", 2) - self.emit("if (!module) {", 2) - self.emit("return 0;", 3) - self.emit("}", 2) - self.emit("if (PyState_AddModule(module, &_astmodule) < 0) {", 2) - self.emit("return 0;", 3) - self.emit("}", 2) - self.emit("}", 1) - self.emit("", 0) - - self.emit("astmodulestate *state = get_ast_state(module);", 1) + self.emit("astmodulestate *state = get_global_ast_state();", 1) self.emit("if (state->initialized) return 1;", 1) self.emit("if (init_identifiers(state) < 0) return 0;", 1) self.emit("state->AST_type = PyType_FromSpec(&AST_type_spec);", 1) @@ -1061,13 +1049,16 @@ def visitModule(self, mod): self.emit("PyMODINIT_FUNC", 0) self.emit("PyInit__ast(void)", 0) self.emit("{", 0) - self.emit("PyObject *m;", 1) - self.emit("if (!init_types()) return NULL;", 1) - self.emit('m = PyState_FindModule(&_astmodule);', 1) - self.emit("if (!m) return NULL;", 1) + self.emit("PyObject *m = PyModule_Create(&_astmodule);", 1) + self.emit("if (!m) {", 1) + self.emit("return NULL;", 2) + self.emit("}", 1) self.emit('astmodulestate *state = get_ast_state(m);', 1) self.emit('', 1) + self.emit("if (!init_types()) {", 1) + self.emit("goto error;", 2) + self.emit("}", 1) self.emit('if (PyModule_AddObject(m, "AST", state->AST_type) < 0) {', 1) self.emit('goto error;', 2) self.emit('}', 1) @@ -1084,6 +1075,7 @@ def visitModule(self, mod): for dfn in mod.dfns: self.visit(dfn) self.emit("return m;", 1) + self.emit("", 0) self.emit("error:", 0) self.emit("Py_DECREF(m);", 1) self.emit("return NULL;", 1) @@ -1263,9 +1255,11 @@ class PartingShots(StaticVisitor): CODE = """ PyObject* PyAST_mod2obj(mod_ty t) { - if (!init_types()) + if (!init_types()) { return NULL; - astmodulestate *state = astmodulestate_global; + } + + astmodulestate *state = get_global_ast_state(); return ast2obj_mod(state, t); } @@ -1279,7 +1273,7 @@ class PartingShots(StaticVisitor): return NULL; } - astmodulestate *state = astmodulestate_global; + astmodulestate *state = get_global_ast_state(); PyObject *req_type[3]; req_type[0] = state->Module_type; req_type[1] = state->Expression_type; @@ -1287,8 +1281,9 @@ class PartingShots(StaticVisitor): assert(0 <= mode && mode <= 2); - if (!init_types()) + if (!init_types()) { return NULL; + } isinstance = PyObject_IsInstance(ast, req_type[mode]); if (isinstance == -1) @@ -1308,9 +1303,11 @@ class PartingShots(StaticVisitor): int PyAST_Check(PyObject* obj) { - if (!init_types()) + if (!init_types()) { return -1; - astmodulestate *state = astmodulestate_global; + } + + astmodulestate *state = get_global_ast_state(); return PyObject_IsInstance(obj, state->AST_type); } """ @@ -1361,13 +1358,12 @@ def generate_module_def(f, mod): f.write(' PyObject *' + s + ';\n') f.write('} astmodulestate;\n\n') f.write(""" +static astmodulestate global_ast_state; + static astmodulestate * -get_ast_state(PyObject *module) +get_ast_state(PyObject *Py_UNUSED(module)) { - assert(module != NULL); - void *state = PyModule_GetState(module); - assert(state != NULL); - return (astmodulestate *)state; + return &global_ast_state; } static int astmodule_clear(PyObject *module) @@ -1396,17 +1392,14 @@ def generate_module_def(f, mod): static struct PyModuleDef _astmodule = { PyModuleDef_HEAD_INIT, - "_ast", - NULL, - sizeof(astmodulestate), - NULL, - NULL, - astmodule_traverse, - astmodule_clear, - astmodule_free, + .m_name = "_ast", + .m_size = -1, + .m_traverse = astmodule_traverse, + .m_clear = astmodule_clear, + .m_free = astmodule_free, }; -#define astmodulestate_global get_ast_state(PyState_FindModule(&_astmodule)) +#define get_global_ast_state() (&global_ast_state) """) f.write('static int init_identifiers(astmodulestate *state)\n') diff --git a/Python/Python-ast.c b/Python/Python-ast.c index 3296b067584d0..844033ee37e28 100644 --- a/Python/Python-ast.c +++ b/Python/Python-ast.c @@ -224,13 +224,12 @@ typedef struct { } astmodulestate; +static astmodulestate global_ast_state; + static astmodulestate * -get_ast_state(PyObject *module) +get_ast_state(PyObject *Py_UNUSED(module)) { - assert(module != NULL); - void *state = PyModule_GetState(module); - assert(state != NULL); - return (astmodulestate *)state; + return &global_ast_state; } static int astmodule_clear(PyObject *module) @@ -679,17 +678,14 @@ static void astmodule_free(void* module) { static struct PyModuleDef _astmodule = { PyModuleDef_HEAD_INIT, - "_ast", - NULL, - sizeof(astmodulestate), - NULL, - NULL, - astmodule_traverse, - astmodule_clear, - astmodule_free, + .m_name = "_ast", + .m_size = -1, + .m_traverse = astmodule_traverse, + .m_clear = astmodule_clear, + .m_free = astmodule_free, }; -#define astmodulestate_global get_ast_state(PyState_FindModule(&_astmodule)) +#define get_global_ast_state() (&global_ast_state) static int init_identifiers(astmodulestate *state) { @@ -1135,7 +1131,7 @@ ast_type_init(PyObject *self, PyObject *args, PyObject *kw) Py_ssize_t i, numfields = 0; int res = -1; PyObject *key, *value, *fields; - astmodulestate *state = astmodulestate_global; + astmodulestate *state = get_global_ast_state(); if (_PyObject_LookupAttr((PyObject*)Py_TYPE(self), state->_fields, &fields) < 0) { goto cleanup; } @@ -1204,7 +1200,7 @@ ast_type_init(PyObject *self, PyObject *args, PyObject *kw) static PyObject * ast_type_reduce(PyObject *self, PyObject *unused) { - astmodulestate *state = astmodulestate_global; + astmodulestate *state = get_global_ast_state(); PyObject *dict; if (_PyObject_LookupAttr(self, state->__dict__, &dict) < 0) { return NULL; @@ -1414,18 +1410,7 @@ static int add_ast_fields(astmodulestate *state) static int init_types(void) { - PyObject *module = PyState_FindModule(&_astmodule); - if (module == NULL) { - module = PyModule_Create(&_astmodule); - if (!module) { - return 0; - } - if (PyState_AddModule(module, &_astmodule) < 0) { - return 0; - } - } - - astmodulestate *state = get_ast_state(module); + astmodulestate *state = get_global_ast_state(); if (state->initialized) return 1; if (init_identifiers(state) < 0) return 0; state->AST_type = PyType_FromSpec(&AST_type_spec); @@ -9857,12 +9842,15 @@ obj2ast_type_ignore(astmodulestate *state, PyObject* obj, type_ignore_ty* out, PyMODINIT_FUNC PyInit__ast(void) { - PyObject *m; - if (!init_types()) return NULL; - m = PyState_FindModule(&_astmodule); - if (!m) return NULL; + PyObject *m = PyModule_Create(&_astmodule); + if (!m) { + return NULL; + } astmodulestate *state = get_ast_state(m); + if (!init_types()) { + goto error; + } if (PyModule_AddObject(m, "AST", state->AST_type) < 0) { goto error; } @@ -10303,6 +10291,7 @@ PyInit__ast(void) } Py_INCREF(state->TypeIgnore_type); return m; + error: Py_DECREF(m); return NULL; @@ -10311,9 +10300,11 @@ PyInit__ast(void) PyObject* PyAST_mod2obj(mod_ty t) { - if (!init_types()) + if (!init_types()) { return NULL; - astmodulestate *state = astmodulestate_global; + } + + astmodulestate *state = get_global_ast_state(); return ast2obj_mod(state, t); } @@ -10327,7 +10318,7 @@ mod_ty PyAST_obj2mod(PyObject* ast, PyArena* arena, int mode) return NULL; } - astmodulestate *state = astmodulestate_global; + astmodulestate *state = get_global_ast_state(); PyObject *req_type[3]; req_type[0] = state->Module_type; req_type[1] = state->Expression_type; @@ -10335,8 +10326,9 @@ mod_ty PyAST_obj2mod(PyObject* ast, PyArena* arena, int mode) assert(0 <= mode && mode <= 2); - if (!init_types()) + if (!init_types()) { return NULL; + } isinstance = PyObject_IsInstance(ast, req_type[mode]); if (isinstance == -1) @@ -10356,9 +10348,11 @@ mod_ty PyAST_obj2mod(PyObject* ast, PyArena* arena, int mode) int PyAST_Check(PyObject* obj) { - if (!init_types()) + if (!init_types()) { return -1; - astmodulestate *state = astmodulestate_global; + } + + astmodulestate *state = get_global_ast_state(); return PyObject_IsInstance(obj, state->AST_type); }

1 0