Mailman 3 February 2020 - Python-checkins

bpo-39542: Make _Py_NewReference() opaque in C API (GH-18346)
by Victor Stinner Feb. 5, 2020

Feb. 5, 2020

https://github.com/python/cpython/commit/40e547dfbb9052ca0c667b242f6825ed1c… commit: 40e547dfbb9052ca0c667b242f6825ed1c23c195 branch: master author: Victor Stinner <vstinner(a)python.org> committer: GitHub <noreply(a)github.com> date: 2020-02-05T01:11:10+01:00 summary: bpo-39542: Make _Py_NewReference() opaque in C API (GH-18346) _Py_NewReference() becomes a regular opaque function, rather than a static inline function in the C API (object.h), to better hide implementation details. Move _Py_tracemalloc_config from public pymem.h to internal pycore_pymem.h header. Make _Py_AddToAllObjects() private. files: M Include/internal/pycore_pymem.h M Include/object.h M Include/pymem.h M Modules/_tracemalloc.c M Objects/object.c diff --git a/Include/internal/pycore_pymem.h b/Include/internal/pycore_pymem.h index 06d0d06c75c36..db153e0bd2d71 100644 --- a/Include/internal/pycore_pymem.h +++ b/Include/internal/pycore_pymem.h @@ -169,6 +169,40 @@ PyAPI_FUNC(int) _PyMem_GetAllocatorName( PYMEM_ALLOCATOR_NOT_SET does nothing. */ PyAPI_FUNC(int) _PyMem_SetupAllocators(PyMemAllocatorName allocator); +/* bpo-35053: Expose _Py_tracemalloc_config for _Py_NewReference() + which access directly _Py_tracemalloc_config.tracing for best + performances. */ +struct _PyTraceMalloc_Config { + /* Module initialized? + Variable protected by the GIL */ + enum { + TRACEMALLOC_NOT_INITIALIZED, + TRACEMALLOC_INITIALIZED, + TRACEMALLOC_FINALIZED + } initialized; + + /* Is tracemalloc tracing memory allocations? + Variable protected by the GIL */ + int tracing; + + /* limit of the number of frames in a traceback, 1 by default. + Variable protected by the GIL. */ + int max_nframe; + + /* use domain in trace key? + Variable protected by the GIL. */ + int use_domain; +}; + +#define _PyTraceMalloc_Config_INIT \ + {.initialized = TRACEMALLOC_NOT_INITIALIZED, \ + .tracing = 0, \ + .max_nframe = 1, \ + .use_domain = 0} + +PyAPI_DATA(struct _PyTraceMalloc_Config) _Py_tracemalloc_config; + + #ifdef __cplusplus } #endif diff --git a/Include/object.h b/Include/object.h index 175a208f0d2d7..8dccbf16971a7 100644 --- a/Include/object.h +++ b/Include/object.h @@ -1,8 +1,6 @@ #ifndef Py_OBJECT_H #define Py_OBJECT_H -#include "pymem.h" /* _Py_tracemalloc_config */ - #ifdef __cplusplus extern "C" { #endif @@ -390,28 +388,13 @@ PyAPI_FUNC(Py_ssize_t) _Py_GetRefTotal(void); when a memory block is reused from a free list. */ PyAPI_FUNC(int) _PyTraceMalloc_NewReference(PyObject *op); +PyAPI_FUNC(void) _Py_NewReference(PyObject *op); + #ifdef Py_TRACE_REFS /* Py_TRACE_REFS is such major surgery that we call external routines. */ PyAPI_FUNC(void) _Py_ForgetReference(PyObject *); -PyAPI_FUNC(void) _Py_AddToAllObjects(PyObject *, int force); #endif - -static inline void _Py_NewReference(PyObject *op) -{ - if (_Py_tracemalloc_config.tracing) { - _PyTraceMalloc_NewReference(op); - } -#ifdef Py_REF_DEBUG - _Py_RefTotal++; -#endif - Py_REFCNT(op) = 1; -#ifdef Py_TRACE_REFS - _Py_AddToAllObjects(op, 1); -#endif -} - - PyAPI_FUNC(void) _Py_Dealloc(PyObject *); static inline void _Py_INCREF(PyObject *op) diff --git a/Include/pymem.h b/Include/pymem.h index 07b380aa6e7fa..607feb9484f24 100644 --- a/Include/pymem.h +++ b/Include/pymem.h @@ -101,41 +101,6 @@ PyAPI_FUNC(void) PyMem_Free(void *ptr); #define PyMem_Del PyMem_Free #define PyMem_DEL PyMem_FREE -/* bpo-35053: expose _Py_tracemalloc_config for performance: - _Py_NewReference() needs an efficient check to test if tracemalloc is - tracing. - - It has to be defined in pymem.h, before object.h is included. */ -struct _PyTraceMalloc_Config { - /* Module initialized? - Variable protected by the GIL */ - enum { - TRACEMALLOC_NOT_INITIALIZED, - TRACEMALLOC_INITIALIZED, - TRACEMALLOC_FINALIZED - } initialized; - - /* Is tracemalloc tracing memory allocations? - Variable protected by the GIL */ - int tracing; - - /* limit of the number of frames in a traceback, 1 by default. - Variable protected by the GIL. */ - int max_nframe; - - /* use domain in trace key? - Variable protected by the GIL. */ - int use_domain; -}; - -PyAPI_DATA(struct _PyTraceMalloc_Config) _Py_tracemalloc_config; - -#define _PyTraceMalloc_Config_INIT \ - {.initialized = TRACEMALLOC_NOT_INITIALIZED, \ - .tracing = 0, \ - .max_nframe = 1, \ - .use_domain = 0} - #ifndef Py_LIMITED_API # define Py_CPYTHON_PYMEM_H diff --git a/Modules/_tracemalloc.c b/Modules/_tracemalloc.c index 70219721b51cd..ddf6ef4e11dd4 100644 --- a/Modules/_tracemalloc.c +++ b/Modules/_tracemalloc.c @@ -1,4 +1,5 @@ #include "Python.h" +#include "pycore_pymem.h" #include "pycore_traceback.h" #include "hashtable.h" #include "frameobject.h" diff --git a/Objects/object.c b/Objects/object.c index 085605ae36714..e6bfad4e7295e 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -93,7 +93,7 @@ static PyObject refchain = {&refchain, &refchain}; * way, though; exceptions include statically allocated type objects, and * statically allocated singletons (like Py_True and Py_None). */ -void +static void _Py_AddToAllObjects(PyObject *op, int force) { #ifdef Py_DEBUG @@ -1805,6 +1805,22 @@ _PyTypes_Init(void) } +void +_Py_NewReference(PyObject *op) +{ + if (_Py_tracemalloc_config.tracing) { + _PyTraceMalloc_NewReference(op); + } +#ifdef Py_REF_DEBUG + _Py_RefTotal++; +#endif + Py_REFCNT(op) = 1; +#ifdef Py_TRACE_REFS + _Py_AddToAllObjects(op, 1); +#endif +} + + #ifdef Py_TRACE_REFS void _Py_ForgetReference(PyObject *op)

1 0

Fix MinGW library generation command (GH-17917)
by Baljak Feb. 5, 2020

Feb. 5, 2020

https://github.com/python/cpython/commit/2545fa87628b4caca519da8aeb0eeef368… commit: 2545fa87628b4caca519da8aeb0eeef368b9dc0d branch: master author: Baljak <baljci(a)hotmail.com> committer: GitHub <noreply(a)github.com> date: 2020-02-05T11:10:16+11:00 summary: Fix MinGW library generation command (GH-17917) To print the exports to stdout, the gendef command requires the option "-". Without this option, no output is generated. files: M Doc/whatsnew/3.8.rst diff --git a/Doc/whatsnew/3.8.rst b/Doc/whatsnew/3.8.rst index fabc1c597eca7..cb4c518662d6b 100644 --- a/Doc/whatsnew/3.8.rst +++ b/Doc/whatsnew/3.8.rst @@ -2109,7 +2109,7 @@ Changes in the C API .. code-block:: shell - gendef python38.dll > tmp.def + gendef - python38.dll > tmp.def dlltool --dllname python38.dll --def tmp.def --output-lib libpython38.a The location of an installed :file:`pythonXY.dll` will depend on the

1 0

closes bpo-39510: Fix use-after-free in BufferedReader.readinto() (GH-18350)
by Miss Islington (bot) Feb. 4, 2020

Feb. 4, 2020

https://github.com/python/cpython/commit/c352e6c7446c894b13643f538db312092b… commit: c352e6c7446c894b13643f538db312092b351789 branch: 3.6 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: GitHub <noreply(a)github.com> date: 2020-02-04T14:06:42-08:00 summary: closes bpo-39510: Fix use-after-free in BufferedReader.readinto() (GH-18350) When called on a closed object, readinto() segfaults on account of a write to a freed buffer: ==220553== Process terminating with default action of signal 11 (SIGSEGV): dumping core ==220553== Access not within mapped region at address 0x2A ==220553== at 0x48408A0: memmove (vg_replace_strmem.c:1272) ==220553== by 0x58DB0C: _buffered_readinto_generic (bufferedio.c:972) ==220553== by 0x58DCBA: _io__Buffered_readinto_impl (bufferedio.c:1053) ==220553== by 0x58DCBA: _io__Buffered_readinto (bufferedio.c.h:253) Reproducer: reader = open ("/dev/zero", "rb") _void = reader.read (42) reader.close () reader.readinto (bytearray (42)) GH-GH-GH- BANG! The problem exists since 2012 when commit dc469454ec added code to free the read buffer on close(). Signed-off-by: Philipp Gesang <philipp.gesang(a)intra2net.com> (cherry picked from commit cb1c0746f277052e45a60d6c436a765e34722821) Co-authored-by: Philipp Gesang <phg(a)phi-gamma.net> Co-authored-by: Philipp Gesang <phg(a)phi-gamma.net> files: A Misc/NEWS.d/next/Core and Builtins/2020-02-04-10-27-41.bpo-39510.PMIh-f.rst M Lib/test/test_io.py M Modules/_io/bufferedio.c diff --git a/Lib/test/test_io.py b/Lib/test/test_io.py index f29c16cb47c56..4ba46563b9908 100644 --- a/Lib/test/test_io.py +++ b/Lib/test/test_io.py @@ -707,6 +707,11 @@ def test_read_closed(self): file.seek(0) file.close() self.assertRaises(ValueError, file.read) + with self.open(support.TESTFN, "rb") as f: + file = self.open(f.fileno(), "rb", closefd=False) + self.assertEqual(file.read()[:3], b"egg") + file.close() + self.assertRaises(ValueError, file.readinto, bytearray(1)) def test_no_closefd_with_filename(self): # can't use closefd in combination with a file name diff --git a/Misc/NEWS.d/next/Core and Builtins/2020-02-04-10-27-41.bpo-39510.PMIh-f.rst b/Misc/NEWS.d/next/Core and Builtins/2020-02-04-10-27-41.bpo-39510.PMIh-f.rst new file mode 100644 index 0000000000000..9a38e4ab76228 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2020-02-04-10-27-41.bpo-39510.PMIh-f.rst @@ -0,0 +1 @@ +Fix segfault in ``readinto()`` method on closed BufferedReader. diff --git a/Modules/_io/bufferedio.c b/Modules/_io/bufferedio.c index 32355813e88df..3e463098881fd 100644 --- a/Modules/_io/bufferedio.c +++ b/Modules/_io/bufferedio.c @@ -993,6 +993,7 @@ _buffered_readinto_generic(buffered *self, Py_buffer *buffer, char readinto1) PyObject *res = NULL; CHECK_INITIALIZED(self) + CHECK_CLOSED(self, "readinto of closed file") n = Py_SAFE_DOWNCAST(READAHEAD(self), Py_off_t, Py_ssize_t); if (n > 0) {

1 0

closes bpo-39510: Fix use-after-free in BufferedReader.readinto() (GH-18295)
by Miss Islington (bot) Feb. 4, 2020

Feb. 4, 2020

https://github.com/python/cpython/commit/97d2a9832494e4c85da124ceab18802eed… commit: 97d2a9832494e4c85da124ceab18802eed9a4ab1 branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: GitHub <noreply(a)github.com> date: 2020-02-04T13:43:07-08:00 summary: closes bpo-39510: Fix use-after-free in BufferedReader.readinto() (GH-18295) When called on a closed object, readinto() segfaults on account of a write to a freed buffer: ==220553== Process terminating with default action of signal 11 (SIGSEGV): dumping core ==220553== Access not within mapped region at address 0x2A ==220553== at 0x48408A0: memmove (vg_replace_strmem.c:1272) ==220553== by 0x58DB0C: _buffered_readinto_generic (bufferedio.c:972) ==220553== by 0x58DCBA: _io__Buffered_readinto_impl (bufferedio.c:1053) ==220553== by 0x58DCBA: _io__Buffered_readinto (bufferedio.c.h:253) Reproducer: reader = open ("/dev/zero", "rb") _void = reader.read (42) reader.close () reader.readinto (bytearray (42)) GH-GH-GH- BANG! The problem exists since 2012 when commit dc469454ec added code to free the read buffer on close(). Signed-off-by: Philipp Gesang <philipp.gesang(a)intra2net.com> (cherry picked from commit cb1c0746f277052e45a60d6c436a765e34722821) Co-authored-by: Philipp Gesang <phg(a)phi-gamma.net> files: A Misc/NEWS.d/next/Core and Builtins/2020-02-04-10-27-41.bpo-39510.PMIh-f.rst M Lib/test/test_io.py M Modules/_io/bufferedio.c diff --git a/Lib/test/test_io.py b/Lib/test/test_io.py index 42002b9acfb76..7d50037bd7e0c 100644 --- a/Lib/test/test_io.py +++ b/Lib/test/test_io.py @@ -726,6 +726,11 @@ def test_read_closed(self): file.seek(0) file.close() self.assertRaises(ValueError, file.read) + with self.open(support.TESTFN, "rb") as f: + file = self.open(f.fileno(), "rb", closefd=False) + self.assertEqual(file.read()[:3], b"egg") + file.close() + self.assertRaises(ValueError, file.readinto, bytearray(1)) def test_no_closefd_with_filename(self): # can't use closefd in combination with a file name diff --git a/Misc/NEWS.d/next/Core and Builtins/2020-02-04-10-27-41.bpo-39510.PMIh-f.rst b/Misc/NEWS.d/next/Core and Builtins/2020-02-04-10-27-41.bpo-39510.PMIh-f.rst new file mode 100644 index 0000000000000..9a38e4ab76228 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2020-02-04-10-27-41.bpo-39510.PMIh-f.rst @@ -0,0 +1 @@ +Fix segfault in ``readinto()`` method on closed BufferedReader. diff --git a/Modules/_io/bufferedio.c b/Modules/_io/bufferedio.c index 358a654a98ca5..9c814c342cd81 100644 --- a/Modules/_io/bufferedio.c +++ b/Modules/_io/bufferedio.c @@ -974,6 +974,7 @@ _buffered_readinto_generic(buffered *self, Py_buffer *buffer, char readinto1) PyObject *res = NULL; CHECK_INITIALIZED(self) + CHECK_CLOSED(self, "readinto of closed file") n = Py_SAFE_DOWNCAST(READAHEAD(self), Py_off_t, Py_ssize_t); if (n > 0) {

1 0

closes bpo-39510: Fix use-after-free in BufferedReader.readinto() (GH-18295)
by Miss Islington (bot) Feb. 4, 2020

Feb. 4, 2020

https://github.com/python/cpython/commit/30e769382dfb67a68fe8e6bfe8509addb4… commit: 30e769382dfb67a68fe8e6bfe8509addb4aa9514 branch: 3.8 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: GitHub <noreply(a)github.com> date: 2020-02-04T13:41:55-08:00 summary: closes bpo-39510: Fix use-after-free in BufferedReader.readinto() (GH-18295) When called on a closed object, readinto() segfaults on account of a write to a freed buffer: ==220553== Process terminating with default action of signal 11 (SIGSEGV): dumping core ==220553== Access not within mapped region at address 0x2A ==220553== at 0x48408A0: memmove (vg_replace_strmem.c:1272) ==220553== by 0x58DB0C: _buffered_readinto_generic (bufferedio.c:972) ==220553== by 0x58DCBA: _io__Buffered_readinto_impl (bufferedio.c:1053) ==220553== by 0x58DCBA: _io__Buffered_readinto (bufferedio.c.h:253) Reproducer: reader = open ("/dev/zero", "rb") _void = reader.read (42) reader.close () reader.readinto (bytearray (42)) GH-GH-GH- BANG! The problem exists since 2012 when commit dc469454ec added code to free the read buffer on close(). Signed-off-by: Philipp Gesang <philipp.gesang(a)intra2net.com> (cherry picked from commit cb1c0746f277052e45a60d6c436a765e34722821) Co-authored-by: Philipp Gesang <phg(a)phi-gamma.net> files: A Misc/NEWS.d/next/Core and Builtins/2020-02-04-10-27-41.bpo-39510.PMIh-f.rst M Lib/test/test_io.py M Modules/_io/bufferedio.c diff --git a/Lib/test/test_io.py b/Lib/test/test_io.py index 50459e0792962..bea4342e7f0f9 100644 --- a/Lib/test/test_io.py +++ b/Lib/test/test_io.py @@ -735,6 +735,11 @@ def test_read_closed(self): file.seek(0) file.close() self.assertRaises(ValueError, file.read) + with self.open(support.TESTFN, "rb") as f: + file = self.open(f.fileno(), "rb", closefd=False) + self.assertEqual(file.read()[:3], b"egg") + file.close() + self.assertRaises(ValueError, file.readinto, bytearray(1)) def test_no_closefd_with_filename(self): # can't use closefd in combination with a file name diff --git a/Misc/NEWS.d/next/Core and Builtins/2020-02-04-10-27-41.bpo-39510.PMIh-f.rst b/Misc/NEWS.d/next/Core and Builtins/2020-02-04-10-27-41.bpo-39510.PMIh-f.rst new file mode 100644 index 0000000000000..9a38e4ab76228 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2020-02-04-10-27-41.bpo-39510.PMIh-f.rst @@ -0,0 +1 @@ +Fix segfault in ``readinto()`` method on closed BufferedReader. diff --git a/Modules/_io/bufferedio.c b/Modules/_io/bufferedio.c index 8e8ff97ff8c67..ad7a8c9d26427 100644 --- a/Modules/_io/bufferedio.c +++ b/Modules/_io/bufferedio.c @@ -965,6 +965,7 @@ _buffered_readinto_generic(buffered *self, Py_buffer *buffer, char readinto1) PyObject *res = NULL; CHECK_INITIALIZED(self) + CHECK_CLOSED(self, "readinto of closed file") n = Py_SAFE_DOWNCAST(READAHEAD(self), Py_off_t, Py_ssize_t); if (n > 0) {

1 0

closes bpo-39510: Fix use-after-free in BufferedReader.readinto() (GH-18295)
by Philipp Gesang Feb. 4, 2020

Feb. 4, 2020

https://github.com/python/cpython/commit/cb1c0746f277052e45a60d6c436a765e34… commit: cb1c0746f277052e45a60d6c436a765e34722821 branch: master author: Philipp Gesang <phg(a)phi-gamma.net> committer: GitHub <noreply(a)github.com> date: 2020-02-04T13:25:16-08:00 summary: closes bpo-39510: Fix use-after-free in BufferedReader.readinto() (GH-18295) When called on a closed object, readinto() segfaults on account of a write to a freed buffer: ==220553== Process terminating with default action of signal 11 (SIGSEGV): dumping core ==220553== Access not within mapped region at address 0x2A ==220553== at 0x48408A0: memmove (vg_replace_strmem.c:1272) ==220553== by 0x58DB0C: _buffered_readinto_generic (bufferedio.c:972) ==220553== by 0x58DCBA: _io__Buffered_readinto_impl (bufferedio.c:1053) ==220553== by 0x58DCBA: _io__Buffered_readinto (bufferedio.c.h:253) Reproducer: reader = open ("/dev/zero", "rb") _void = reader.read (42) reader.close () reader.readinto (bytearray (42)) ### BANG! The problem exists since 2012 when commit dc469454ec added code to free the read buffer on close(). Signed-off-by: Philipp Gesang <philipp.gesang(a)intra2net.com> files: A Misc/NEWS.d/next/Core and Builtins/2020-02-04-10-27-41.bpo-39510.PMIh-f.rst M Lib/test/test_io.py M Modules/_io/bufferedio.c diff --git a/Lib/test/test_io.py b/Lib/test/test_io.py index a66726e9a790a..0bfa4d249e9a6 100644 --- a/Lib/test/test_io.py +++ b/Lib/test/test_io.py @@ -737,6 +737,11 @@ def test_read_closed(self): file.seek(0) file.close() self.assertRaises(ValueError, file.read) + with self.open(support.TESTFN, "rb") as f: + file = self.open(f.fileno(), "rb", closefd=False) + self.assertEqual(file.read()[:3], b"egg") + file.close() + self.assertRaises(ValueError, file.readinto, bytearray(1)) def test_no_closefd_with_filename(self): # can't use closefd in combination with a file name diff --git a/Misc/NEWS.d/next/Core and Builtins/2020-02-04-10-27-41.bpo-39510.PMIh-f.rst b/Misc/NEWS.d/next/Core and Builtins/2020-02-04-10-27-41.bpo-39510.PMIh-f.rst new file mode 100644 index 0000000000000..9a38e4ab76228 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2020-02-04-10-27-41.bpo-39510.PMIh-f.rst @@ -0,0 +1 @@ +Fix segfault in ``readinto()`` method on closed BufferedReader. diff --git a/Modules/_io/bufferedio.c b/Modules/_io/bufferedio.c index 586e93f25e28f..ddd17a2863c24 100644 --- a/Modules/_io/bufferedio.c +++ b/Modules/_io/bufferedio.c @@ -965,6 +965,7 @@ _buffered_readinto_generic(buffered *self, Py_buffer *buffer, char readinto1) PyObject *res = NULL; CHECK_INITIALIZED(self) + CHECK_CLOSED(self, "readinto of closed file") n = Py_SAFE_DOWNCAST(READAHEAD(self), Py_off_t, Py_ssize_t); if (n > 0) {

1 0

bpo-37224: Improve test__xxsubinterpreters.DestroyTests (GH-18058)
by Miss Islington (bot) Feb. 4, 2020

Feb. 4, 2020

https://github.com/python/cpython/commit/9a740b6c7e7a88185d79128b8a1993ac38… commit: 9a740b6c7e7a88185d79128b8a1993ac387d5091 branch: 3.8 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: GitHub <noreply(a)github.com> date: 2020-02-04T07:31:19-08:00 summary: bpo-37224: Improve test__xxsubinterpreters.DestroyTests (GH-18058) Adds an additional assertion check based on a race condition for `test__xxsubinterpreters.DestroyTests.test_still_running` discovered in the bpo issue. https://bugs.python.org/issue37224 (cherry picked from commit f03a8f8d5001963ad5b5b28dbd95497e9cc15596) Co-authored-by: Kyle Stanley <aeros167(a)gmail.com> files: M Lib/test/test__xxsubinterpreters.py diff --git a/Lib/test/test__xxsubinterpreters.py b/Lib/test/test__xxsubinterpreters.py index 207b5db5d8fb9..30f8f98acc9dd 100644 --- a/Lib/test/test__xxsubinterpreters.py +++ b/Lib/test/test__xxsubinterpreters.py @@ -759,7 +759,11 @@ def test_still_running(self): main, = interpreters.list_all() interp = interpreters.create() with _running(interp): - with self.assertRaises(RuntimeError): + self.assertTrue(interpreters.is_running(interp), + msg=f"Interp {interp} should be running before destruction.") + + with self.assertRaises(RuntimeError, + msg=f"Should not be able to destroy interp {interp} while it's still running."): interpreters.destroy(interp) self.assertTrue(interpreters.is_running(interp))

1 0

bpo-39432: Implement PEP-489 algorithm for non-ascii "PyInit_*" symbol names in distutils (GH-18150)
by Stefan Behnel Feb. 4, 2020

Feb. 4, 2020

https://github.com/python/cpython/commit/9538bc9185e934bee2bd5ae2cda2b2e92a… commit: 9538bc9185e934bee2bd5ae2cda2b2e92a61906d branch: master author: Stefan Behnel <stefan_ml(a)behnel.de> committer: GitHub <noreply(a)github.com> date: 2020-02-04T07:24:30-08:00 summary: bpo-39432: Implement PEP-489 algorithm for non-ascii "PyInit_*" symbol names in distutils (GH-18150) Make it export the correct init symbol also on Windows. https://bugs.python.org/issue39432 files: A Misc/NEWS.d/next/Library/2020-01-23-16-08-58.bpo-39432.Cee6mi.rst M Lib/distutils/command/build_ext.py M Lib/distutils/tests/test_build_ext.py diff --git a/Lib/distutils/command/build_ext.py b/Lib/distutils/command/build_ext.py index 38bb8fd93c278..1a9bd1200f823 100644 --- a/Lib/distutils/command/build_ext.py +++ b/Lib/distutils/command/build_ext.py @@ -689,7 +689,15 @@ def get_export_symbols(self, ext): provided, "PyInit_" + module_name. Only relevant on Windows, where the .pyd file (DLL) must export the module "PyInit_" function. """ - initfunc_name = "PyInit_" + ext.name.split('.')[-1] + suffix = '_' + ext.name.split('.')[-1] + try: + # Unicode module name support as defined in PEP-489 + # https://www.python.org/dev/peps/pep-0489/#export-hook-name + suffix.encode('ascii') + except UnicodeEncodeError: + suffix = 'U' + suffix.encode('punycode').replace(b'-', b'_').decode('ascii') + + initfunc_name = "PyInit" + suffix if initfunc_name not in ext.export_symbols: ext.export_symbols.append(initfunc_name) return ext.export_symbols diff --git a/Lib/distutils/tests/test_build_ext.py b/Lib/distutils/tests/test_build_ext.py index 52d36b2484f41..7e3eafa8ef231 100644 --- a/Lib/distutils/tests/test_build_ext.py +++ b/Lib/distutils/tests/test_build_ext.py @@ -304,6 +304,19 @@ def test_get_source_files(self): cmd.ensure_finalized() self.assertEqual(cmd.get_source_files(), ['xxx']) + def test_unicode_module_names(self): + modules = [ + Extension('foo', ['aaa'], optional=False), + Extension('föö', ['uuu'], optional=False), + ] + dist = Distribution({'name': 'xx', 'ext_modules': modules}) + cmd = self.build_ext(dist) + cmd.ensure_finalized() + self.assertRegex(cmd.get_ext_filename(modules[0].name), r'foo\..*') + self.assertRegex(cmd.get_ext_filename(modules[1].name), r'föö\..*') + self.assertEqual(cmd.get_export_symbols(modules[0]), ['PyInit_foo']) + self.assertEqual(cmd.get_export_symbols(modules[1]), ['PyInitU_f_gkaa']) + def test_compiler_option(self): # cmd.compiler is an option and # should not be overridden by a compiler instance diff --git a/Misc/NEWS.d/next/Library/2020-01-23-16-08-58.bpo-39432.Cee6mi.rst b/Misc/NEWS.d/next/Library/2020-01-23-16-08-58.bpo-39432.Cee6mi.rst new file mode 100644 index 0000000000000..21c4ba8620755 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2020-01-23-16-08-58.bpo-39432.Cee6mi.rst @@ -0,0 +1 @@ +Implement PEP-489 algorithm for non-ascii "PyInit\_..." symbol names in distutils to make it export the correct init symbol also on Windows.

1 0

Restore PyObject_IsInstance() comment (GH-18345)
by Victor Stinner Feb. 4, 2020

Feb. 4, 2020

https://github.com/python/cpython/commit/850a4bd839ca11b59439e21dda2a3ebe91… commit: 850a4bd839ca11b59439e21dda2a3ebe917a9a16 branch: master author: Victor Stinner <vstinner(a)python.org> committer: GitHub <noreply(a)github.com> date: 2020-02-04T13:42:13+01:00 summary: Restore PyObject_IsInstance() comment (GH-18345) Restore PyObject_IsInstance() comment explaining why only tuples of types are accepted, but not general sequence. Comment written by Guido van Rossum in commit 03290ecbf1661c0192e6abdbe00ae163af461d77 which implements isinstance(x, (A, B, ...)). The comment was lost in a PyObject_IsInstance() optimization: commit ec569b794737be248671d0dfac11b664fc930eef. Cleanup also the code. recursive_isinstance() is no longer recursive, so rename it to object_isinstance(), whereas object_isinstance() is recursive and so rename it to object_recursive_isinstance(). files: M Objects/abstract.c diff --git a/Objects/abstract.c b/Objects/abstract.c index dc8ba10762de6..aca1a4e518919 100644 --- a/Objects/abstract.c +++ b/Objects/abstract.c @@ -2423,7 +2423,7 @@ check_class(PyObject *cls, const char *error) } static int -recursive_isinstance(PyObject *inst, PyObject *cls) +object_isinstance(PyObject *inst, PyObject *cls) { PyObject *icls; int retval; @@ -2461,21 +2461,23 @@ recursive_isinstance(PyObject *inst, PyObject *cls) } static int -object_isinstance(PyThreadState *tstate, PyObject *inst, PyObject *cls) +object_recursive_isinstance(PyThreadState *tstate, PyObject *inst, PyObject *cls) { _Py_IDENTIFIER(__instancecheck__); - PyObject *checker; /* Quick test for an exact match */ - if (Py_TYPE(inst) == (PyTypeObject *)cls) + if (Py_TYPE(inst) == (PyTypeObject *)cls) { return 1; + } /* We know what type's __instancecheck__ does. */ if (PyType_CheckExact(cls)) { - return recursive_isinstance(inst, cls); + return object_isinstance(inst, cls); } if (PyTuple_Check(cls)) { + /* Not a general sequence -- that opens up the road to + recursion and stack overflow. */ if (_Py_EnterRecursiveCall(tstate, " in __instancecheck__")) { return -1; } @@ -2483,37 +2485,41 @@ object_isinstance(PyThreadState *tstate, PyObject *inst, PyObject *cls) int r = 0; for (Py_ssize_t i = 0; i < n; ++i) { PyObject *item = PyTuple_GET_ITEM(cls, i); - r = object_isinstance(tstate, inst, item); - if (r != 0) + r = object_recursive_isinstance(tstate, inst, item); + if (r != 0) { /* either found it, or got an error */ break; + } } _Py_LeaveRecursiveCall(tstate); return r; } - checker = _PyObject_LookupSpecial(cls, &PyId___instancecheck__); + PyObject *checker = _PyObject_LookupSpecial(cls, &PyId___instancecheck__); if (checker != NULL) { - int ok = -1; if (_Py_EnterRecursiveCall(tstate, " in __instancecheck__")) { Py_DECREF(checker); - return ok; + return -1; } + PyObject *res = _PyObject_CallOneArg(checker, inst); _Py_LeaveRecursiveCall(tstate); Py_DECREF(checker); - if (res != NULL) { - ok = PyObject_IsTrue(res); - Py_DECREF(res); + + if (res == NULL) { + return -1; } + int ok = PyObject_IsTrue(res); + Py_DECREF(res); + return ok; } else if (_PyErr_Occurred(tstate)) { return -1; } - /* Probably never reached anymore. */ - return recursive_isinstance(inst, cls); + /* cls has no __instancecheck__() method */ + return object_isinstance(inst, cls); } @@ -2521,7 +2527,7 @@ int PyObject_IsInstance(PyObject *inst, PyObject *cls) { PyThreadState *tstate = _PyThreadState_GET(); - return object_isinstance(tstate, inst, cls); + return object_recursive_isinstance(tstate, inst, cls); } @@ -2611,7 +2617,7 @@ PyObject_IsSubclass(PyObject *derived, PyObject *cls) int _PyObject_RealIsInstance(PyObject *inst, PyObject *cls) { - return recursive_isinstance(inst, cls); + return object_isinstance(inst, cls); } int

1 0

bpo-38076 Clear the interpreter state only after clearing module globals (GH-18039)
by Eddie Elizondo Feb. 4, 2020

Feb. 4, 2020

https://github.com/python/cpython/commit/4590f72259ecbcea66e12a28af14d86725… commit: 4590f72259ecbcea66e12a28af14d867255d2e66 branch: master author: Eddie Elizondo <eelizondo(a)fb.com> committer: GitHub <noreply(a)github.com> date: 2020-02-04T02:29:25-08:00 summary: bpo-38076 Clear the interpreter state only after clearing module globals (GH-18039) Currently, during runtime destruction, `_PyImport_Cleanup` is clearing the interpreter state before clearing out the modules themselves. This leads to a segfault on modules that rely on the module state to clear themselves up. For example, let's take the small snippet added in the issue by @DinoV : ``` import _struct class C: def __init__(self): self.pack = _struct.pack def __del__(self): self.pack('I', -42) _struct.x = C() ``` The module `_struct` uses the module state to run `pack`. Therefore, the module state has to be alive until after the module has been cleared out to successfully run `C.__del__`. This happens at line 606, when `_PyImport_Cleanup` calls `_PyModule_Clear`. In fact, the loop that calls `_PyModule_Clear` has in its comments: > Now, if there are any modules left alive, clear their globals to minimize potential leaks. All C extension modules actually end up here, since they are kept alive in the interpreter state. That means that we can't clear the module state (which is used by C Extensions) before we run that loop. Moving `_PyInterpreterState_ClearModules` until after it, fixes the segfault in the code snippet. Finally, this updates a test in `io` to correctly assert the error that it now throws (since it now finds the io module state). The test that uses this is: `test_create_at_shutdown_without_encoding`. Given this test is now working is a proof that the module state now stays alive even when `__del__` is called at module destruction time. Thus, I didn't add a new tests for this. https://bugs.python.org/issue38076 files: A Misc/NEWS.d/next/C API/2020-01-17-11-37-05.bpo-38076.cxfw2x.rst M Lib/test/test_io.py M Lib/test/test_struct.py M Lib/test/test_sys.py M Python/import.c diff --git a/Lib/test/test_io.py b/Lib/test/test_io.py index 8a123fa1dc07a..a66726e9a790a 100644 --- a/Lib/test/test_io.py +++ b/Lib/test/test_io.py @@ -3683,7 +3683,7 @@ def _to_memoryview(buf): class CTextIOWrapperTest(TextIOWrapperTest): io = io - shutdown_error = "RuntimeError: could not find io module state" + shutdown_error = "LookupError: unknown encoding: ascii" def test_initialization(self): r = self.BytesIO(b"\xc3\xa9\n\n") diff --git a/Lib/test/test_struct.py b/Lib/test/test_struct.py index 157efa1347a9b..4829fbe1b975c 100644 --- a/Lib/test/test_struct.py +++ b/Lib/test/test_struct.py @@ -7,6 +7,7 @@ import sys from test import support +from test.support.script_helper import assert_python_ok ISBIGENDIAN = sys.byteorder == "big" @@ -652,6 +653,23 @@ def test_format_attr(self): s2 = struct.Struct(s.format.encode()) self.assertEqual(s2.format, s.format) + def test_struct_cleans_up_at_runtime_shutdown(self): + code = """if 1: + import struct + + class C: + def __init__(self): + self.pack = struct.pack + def __del__(self): + self.pack('I', -42) + + struct.x = C() + """ + rc, stdout, stderr = assert_python_ok("-c", code) + self.assertEqual(rc, 0) + self.assertEqual(stdout.rstrip(), b"") + self.assertIn(b"Exception ignored in:", stderr) + self.assertIn(b"C.__del__", stderr) class UnpackIteratorTest(unittest.TestCase): """ diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index 58701a11f919c..c5bd8a4b1ff96 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -855,6 +855,23 @@ def __del__(self, sys=sys): self.assertIn(b'sys.flags', out[0]) self.assertIn(b'sys.float_info', out[1]) + def test_sys_ignores_cleaning_up_user_data(self): + code = """if 1: + import struct, sys + + class C: + def __init__(self): + self.pack = struct.pack + def __del__(self): + self.pack('I', -42) + + sys.x = C() + """ + rc, stdout, stderr = assert_python_ok('-c', code) + self.assertEqual(rc, 0) + self.assertEqual(stdout.rstrip(), b"") + self.assertEqual(stderr.rstrip(), b"") + @unittest.skipUnless(hasattr(sys, 'getandroidapilevel'), 'need sys.getandroidapilevel()') def test_getandroidapilevel(self): diff --git a/Misc/NEWS.d/next/C API/2020-01-17-11-37-05.bpo-38076.cxfw2x.rst b/Misc/NEWS.d/next/C API/2020-01-17-11-37-05.bpo-38076.cxfw2x.rst new file mode 100644 index 0000000000000..d9f6dc31efd8d --- /dev/null +++ b/Misc/NEWS.d/next/C API/2020-01-17-11-37-05.bpo-38076.cxfw2x.rst @@ -0,0 +1,2 @@ +Fix to clear the interpreter state only after clearing module globals to +guarantee module state access from C Extensions during runtime destruction diff --git a/Python/import.c b/Python/import.c index 9838c3fa04538..8bf044827c6e9 100644 --- a/Python/import.c +++ b/Python/import.c @@ -568,8 +568,6 @@ _PyImport_Cleanup(PyThreadState *tstate) _PyErr_Clear(tstate); } Py_XDECREF(dict); - /* Clear module dict copies stored in the interpreter state */ - _PyInterpreterState_ClearModules(interp); /* Collect references */ _PyGC_CollectNoFail(); /* Dump GC stats before it's too late, since it uses the warnings @@ -621,6 +619,9 @@ _PyImport_Cleanup(PyThreadState *tstate) } _PyModule_ClearDict(interp->builtins); + /* Clear module dict copies stored in the interpreter state */ + _PyInterpreterState_ClearModules(interp); + /* Clear and delete the modules directory. Actual modules will still be there only if imported during the execution of some destructor. */

1 0