Mailman 3 November 2018 - Python-checkins

bpo-35337: Fix gettmarg(): use PyStructSequence_GET_ITEM() (GH-10765)
by Victor Stinner 28 Nov '18

28 Nov '18

https://github.com/python/cpython/commit/1cdfcfc9843d35ab2cb87387d3a79b2c8a… commit: 1cdfcfc9843d35ab2cb87387d3a79b2c8a585a38 branch: master author: Victor Stinner <vstinner(a)redhat.com> committer: GitHub <noreply(a)github.com> date: 2018-11-28T15:19:51+01:00 summary: bpo-35337: Fix gettmarg(): use PyStructSequence_GET_ITEM() (GH-10765) PyStructSequence_GET_ITEM() must be used instead of PyTuple_GET_ITEM() on a StructTimeType. files: M Modules/timemodule.c diff --git a/Modules/timemodule.c b/Modules/timemodule.c index c3ecd80a6083..55b82d74cb19 100644 --- a/Modules/timemodule.c +++ b/Modules/timemodule.c @@ -568,7 +568,7 @@ gettmarg(PyObject *args, struct tm *p, const char *format) #ifdef HAVE_STRUCT_TM_TM_ZONE if (Py_TYPE(args) == &StructTimeType) { PyObject *item; - item = PyTuple_GET_ITEM(args, 9); + item = PyStructSequence_GET_ITEM(args, 9); if (item != Py_None) { p->tm_zone = (char *)PyUnicode_AsUTF8(item); if (p->tm_zone == NULL) { @@ -589,7 +589,7 @@ gettmarg(PyObject *args, struct tm *p, const char *format) } #endif } - item = PyTuple_GET_ITEM(args, 10); + item = PyStructSequence_GET_ITEM(args, 10); if (item != Py_None) { p->tm_gmtoff = PyLong_AsLong(item); if (PyErr_Occurred())

1 0

bpo-35134: Create Include/cpython/tupleobject.h (GH-10764)
by Victor Stinner 28 Nov '18

28 Nov '18

https://github.com/python/cpython/commit/54ba556c6c7d8fd5504dc142c2e773890c… commit: 54ba556c6c7d8fd5504dc142c2e773890c55a774 branch: master author: Victor Stinner <vstinner(a)redhat.com> committer: GitHub <noreply(a)github.com> date: 2018-11-28T13:01:32+01:00 summary: bpo-35134: Create Include/cpython/tupleobject.h (GH-10764) Move tupleobject.h code surrounded by "#ifndef Py_LIMITED_API" to a new Include/cpython/tupleobject.h header file. Add cpython/ header files to Makefile.pre.in and pythoncore project of PCbuild. files: A Include/cpython/tupleobject.h M Include/tupleobject.h M Makefile.pre.in M PCbuild/pythoncore.vcxproj M PCbuild/pythoncore.vcxproj.filters diff --git a/Include/cpython/tupleobject.h b/Include/cpython/tupleobject.h new file mode 100644 index 000000000000..1565f2a5c3d9 --- /dev/null +++ b/Include/cpython/tupleobject.h @@ -0,0 +1,36 @@ +#ifndef Py_CPYTHON_TUPLEOBJECT_H +# error "this header file must not be included directly" +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct { + PyObject_VAR_HEAD + /* ob_item contains space for 'ob_size' elements. + Items must normally not be NULL, except during construction when + the tuple is not yet visible outside the function that builds it. */ + PyObject *ob_item[1]; +} PyTupleObject; + +PyAPI_FUNC(int) _PyTuple_Resize(PyObject **, Py_ssize_t); +PyAPI_FUNC(void) _PyTuple_MaybeUntrack(PyObject *); + +/* Macros trading safety for speed */ + +/* Cast argument to PyTupleObject* type. */ +#define _PyTuple_CAST(op) (assert(PyTuple_Check(op)), (PyTupleObject *)(op)) + +#define PyTuple_GET_SIZE(op) Py_SIZE(_PyTuple_CAST(op)) + +#define PyTuple_GET_ITEM(op, i) (_PyTuple_CAST(op)->ob_item[i]) + +/* Macro, *only* to be used to fill in brand new tuples */ +#define PyTuple_SET_ITEM(op, i, v) (_PyTuple_CAST(op)->ob_item[i] = v) + +PyAPI_FUNC(void) _PyTuple_DebugMallocStats(FILE *out); + +#ifdef __cplusplus +} +#endif diff --git a/Include/tupleobject.h b/Include/tupleobject.h index 75574bc43fb3..590902de9d02 100644 --- a/Include/tupleobject.h +++ b/Include/tupleobject.h @@ -1,4 +1,3 @@ - /* Tuple object interface */ #ifndef Py_TUPLEOBJECT_H @@ -21,18 +20,6 @@ inserted in the tuple. Similarly, PyTuple_GetItem does not increment the returned item's reference count. */ -#ifndef Py_LIMITED_API -typedef struct { - PyObject_VAR_HEAD - PyObject *ob_item[1]; - - /* ob_item contains space for 'ob_size' elements. - * Items must normally not be NULL, except during construction when - * the tuple is not yet visible outside the function that builds it. - */ -} PyTupleObject; -#endif - PyAPI_DATA(PyTypeObject) PyTuple_Type; PyAPI_DATA(PyTypeObject) PyTupleIter_Type; @@ -45,30 +32,15 @@ PyAPI_FUNC(Py_ssize_t) PyTuple_Size(PyObject *); PyAPI_FUNC(PyObject *) PyTuple_GetItem(PyObject *, Py_ssize_t); PyAPI_FUNC(int) PyTuple_SetItem(PyObject *, Py_ssize_t, PyObject *); PyAPI_FUNC(PyObject *) PyTuple_GetSlice(PyObject *, Py_ssize_t, Py_ssize_t); -#ifndef Py_LIMITED_API -PyAPI_FUNC(int) _PyTuple_Resize(PyObject **, Py_ssize_t); -#endif PyAPI_FUNC(PyObject *) PyTuple_Pack(Py_ssize_t, ...); -#ifndef Py_LIMITED_API -PyAPI_FUNC(void) _PyTuple_MaybeUntrack(PyObject *); -#endif - -/* Macro, trading safety for speed */ -#ifndef Py_LIMITED_API -/* Cast argument to PyTupleObject* type. */ -#define _PyTuple_CAST(op) (assert(PyTuple_Check(op)), (PyTupleObject *)(op)) - -#define PyTuple_GET_ITEM(op, i) (_PyTuple_CAST(op)->ob_item[i]) -#define PyTuple_GET_SIZE(op) Py_SIZE(_PyTuple_CAST(op)) - -/* Macro, *only* to be used to fill in brand new tuples */ -#define PyTuple_SET_ITEM(op, i, v) (_PyTuple_CAST(op)->ob_item[i] = v) -#endif PyAPI_FUNC(int) PyTuple_ClearFreeList(void); + #ifndef Py_LIMITED_API -PyAPI_FUNC(void) _PyTuple_DebugMallocStats(FILE *out); -#endif /* Py_LIMITED_API */ +# define Py_CPYTHON_TUPLEOBJECT_H +# include "cpython/tupleobject.h" +# undef Py_CPYTHON_TUPLEOBJECT_H +#endif #ifdef __cplusplus } diff --git a/Makefile.pre.in b/Makefile.pre.in index eee64a75f785..02ce0af55ce2 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1027,7 +1027,15 @@ PYTHON_HEADERS= \ $(PARSER_HEADERS) \ $(srcdir)/Include/Python-ast.h \ \ + $(srcdir)/Include/cpython/abstract.h \ + $(srcdir)/Include/cpython/dictobject.h \ + $(srcdir)/Include/cpython/object.h \ $(srcdir)/Include/cpython/objimpl.h \ + $(srcdir)/Include/cpython/pyerrors.h \ + $(srcdir)/Include/cpython/pylifecycle.h \ + $(srcdir)/Include/cpython/pystate.h \ + $(srcdir)/Include/cpython/tupleobject.h \ + $(srcdir)/Include/cpython/unicodeobject.h \ \ $(srcdir)/Include/internal/pycore_accu.h \ $(srcdir)/Include/internal/pycore_atomic.h \ diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index 8aa4c0610f60..4ae2d692eee1 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -95,7 +95,15 @@ <ClInclude Include="..\Include\complexobject.h" /> <ClInclude Include="..\Include\context.h" /> <ClInclude Include="..\Include\coreconfig.h" /> + <ClInclude Include="..\Include\cpython\abstract.h" /> + <ClInclude Include="..\Include\cpython\dictobject.h" /> + <ClInclude Include="..\Include\cpython\object.h" /> <ClInclude Include="..\Include\cpython\objimpl.h" /> + <ClInclude Include="..\Include\cpython\pyerrors.h" /> + <ClInclude Include="..\Include\cpython\pylifecycle.h" /> + <ClInclude Include="..\Include\cpython\pystate.h" /> + <ClInclude Include="..\Include\cpython\tupleobject.h" /> + <ClInclude Include="..\Include\cpython\unicodeobject.h" /> <ClInclude Include="..\Include\datetime.h" /> <ClInclude Include="..\Include\descrobject.h" /> <ClInclude Include="..\Include\dictobject.h" /> diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index 021a67efccbc..5a43a9951b63 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -84,9 +84,33 @@ <ClInclude Include="..\Include\coreconfig.h"> <Filter>Include</Filter> </ClInclude> + <ClInclude Include="..\Include\cpython\abstract.h"> + <Filter>Include</Filter> + </ClInclude> + <ClInclude Include="..\Include\cpython\dictobject.h"> + <Filter>Include</Filter> + </ClInclude> + <ClInclude Include="..\Include\cpython\object.h"> + <Filter>Include</Filter> + </ClInclude> <ClInclude Include="..\Include\cpython\objimpl.h"> <Filter>Include</Filter> </ClInclude> + <ClInclude Include="..\Include\cpython\pyerrors.h"> + <Filter>Include</Filter> + </ClInclude> + <ClInclude Include="..\Include\cpython\pylifecycle.h"> + <Filter>Include</Filter> + </ClInclude> + <ClInclude Include="..\Include\cpython\pystate.h"> + <Filter>Include</Filter> + </ClInclude> + <ClInclude Include="..\Include\cpython\tupleobject.h"> + <Filter>Include</Filter> + </ClInclude> + <ClInclude Include="..\Include\cpython\unicodeobject.h"> + <Filter>Include</Filter> + </ClInclude> <ClInclude Include="..\Include\datetime.h"> <Filter>Include</Filter> </ClInclude>

1 0

bpo-34523, bpo-35322: Fix unicode_encode_locale() (GH-10759) (GH-10761)
by Victor Stinner 28 Nov '18

28 Nov '18

https://github.com/python/cpython/commit/85ab974f78c0ebcfa611639864640d0273… commit: 85ab974f78c0ebcfa611639864640d0273eb5466 branch: 3.7 author: Victor Stinner <vstinner(a)redhat.com> committer: GitHub <noreply(a)github.com> date: 2018-11-28T12:42:40+01:00 summary: bpo-34523, bpo-35322: Fix unicode_encode_locale() (GH-10759) (GH-10761) Fix memory leak in PyUnicode_EncodeLocale() and PyUnicode_EncodeFSDefault() on error handling. Fix unicode_encode_locale() error handling. (cherry picked from commit bde9d6bbb46ca59bcee5d5060adaa33c3ffee3a6) files: A Misc/NEWS.d/next/C API/2018-11-28-03-20-36.bpo-35322.Qcqsag.rst M Objects/unicodeobject.c diff --git a/Misc/NEWS.d/next/C API/2018-11-28-03-20-36.bpo-35322.Qcqsag.rst b/Misc/NEWS.d/next/C API/2018-11-28-03-20-36.bpo-35322.Qcqsag.rst new file mode 100644 index 000000000000..f5b4796307f7 --- /dev/null +++ b/Misc/NEWS.d/next/C API/2018-11-28-03-20-36.bpo-35322.Qcqsag.rst @@ -0,0 +1,2 @@ +Fix memory leak in :c:func:`PyUnicode_EncodeLocale` and +:c:func:`PyUnicode_EncodeFSDefault` on error handling. diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index e6371d2337c3..bd3f151c6a50 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -3391,10 +3391,9 @@ unicode_encode_locale(PyObject *unicode, const char *errors, return NULL; } - Py_ssize_t wlen2 = wcslen(wstr); - if (wlen2 != wlen) { - PyMem_Free(wstr); + if ((size_t)wlen != wcslen(wstr)) { PyErr_SetString(PyExc_ValueError, "embedded null character"); + PyMem_Free(wstr); return NULL; } @@ -3403,6 +3402,8 @@ unicode_encode_locale(PyObject *unicode, const char *errors, const char *reason; int res = _Py_EncodeLocaleEx(wstr, &str, &error_pos, &reason, current_locale, surrogateescape); + PyMem_Free(wstr); + if (res != 0) { if (res == -2) { PyObject *exc; @@ -3415,15 +3416,12 @@ unicode_encode_locale(PyObject *unicode, const char *errors, PyCodec_StrictErrors(exc); Py_DECREF(exc); } - return NULL; } else { PyErr_NoMemory(); - PyMem_Free(wstr); - return NULL; } + return NULL; } - PyMem_Free(wstr); PyObject *bytes = PyBytes_FromString(str); PyMem_RawFree(str);

1 0

bpo-34523, bpo-35322: Fix unicode_encode_locale() (GH-10759)
by Victor Stinner 28 Nov '18

28 Nov '18

https://github.com/python/cpython/commit/bde9d6bbb46ca59bcee5d5060adaa33c3f… commit: bde9d6bbb46ca59bcee5d5060adaa33c3ffee3a6 branch: master author: Victor Stinner <vstinner(a)redhat.com> committer: GitHub <noreply(a)github.com> date: 2018-11-28T10:26:20+01:00 summary: bpo-34523, bpo-35322: Fix unicode_encode_locale() (GH-10759) Fix memory leak in PyUnicode_EncodeLocale() and PyUnicode_EncodeFSDefault() on error handling. Changes: * Fix unicode_encode_locale() error handling * Fix test_codecs.LocaleCodecTest files: A Misc/NEWS.d/next/C API/2018-11-28-03-20-36.bpo-35322.Qcqsag.rst M Lib/test/test_codecs.py M Objects/unicodeobject.c diff --git a/Lib/test/test_codecs.py b/Lib/test/test_codecs.py index 00b5d317c401..8c92556e8429 100644 --- a/Lib/test/test_codecs.py +++ b/Lib/test/test_codecs.py @@ -3290,9 +3290,9 @@ def check_encode_strings(self, errors): expected = text.encode(self.ENCODING, errors) except UnicodeEncodeError: with self.assertRaises(RuntimeError) as cm: - self.encode(self.SURROGATES) + self.encode(text, errors) errmsg = str(cm.exception) - self.assertTrue(errmsg.startswith("encode error: pos=0, reason="), errmsg) + self.assertRegex(errmsg, r"encode error: pos=[0-9]+, reason=") else: encoded = self.encode(text, errors) self.assertEqual(encoded, expected) @@ -3315,6 +3315,11 @@ def test_encode_surrogatepass(self): self.check_encode_strings("surrogatepass") + def test_encode_unsupported_error_handler(self): + with self.assertRaises(ValueError) as cm: + self.encode('', 'backslashreplace') + self.assertEqual(str(cm.exception), 'unsupported error handler') + def decode(self, encoded, errors="strict"): return _testcapi.DecodeLocaleEx(encoded, 0, errors) @@ -3370,6 +3375,11 @@ def test_decode_surrogatepass(self): self.check_decode_strings("surrogatepass") + def test_decode_unsupported_error_handler(self): + with self.assertRaises(ValueError) as cm: + self.decode(b'', 'backslashreplace') + self.assertEqual(str(cm.exception), 'unsupported error handler') + if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS.d/next/C API/2018-11-28-03-20-36.bpo-35322.Qcqsag.rst b/Misc/NEWS.d/next/C API/2018-11-28-03-20-36.bpo-35322.Qcqsag.rst new file mode 100644 index 000000000000..f5b4796307f7 --- /dev/null +++ b/Misc/NEWS.d/next/C API/2018-11-28-03-20-36.bpo-35322.Qcqsag.rst @@ -0,0 +1,2 @@ +Fix memory leak in :c:func:`PyUnicode_EncodeLocale` and +:c:func:`PyUnicode_EncodeFSDefault` on error handling. diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 2b1db918a154..bc98c44c7407 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -3449,10 +3449,9 @@ unicode_encode_locale(PyObject *unicode, const char *errors, return NULL; } - Py_ssize_t wlen2 = wcslen(wstr); - if (wlen2 != wlen) { - PyMem_Free(wstr); + if ((size_t)wlen != wcslen(wstr)) { PyErr_SetString(PyExc_ValueError, "embedded null character"); + PyMem_Free(wstr); return NULL; } @@ -3461,6 +3460,8 @@ unicode_encode_locale(PyObject *unicode, const char *errors, const char *reason; int res = _Py_EncodeLocaleEx(wstr, &str, &error_pos, &reason, current_locale, error_handler); + PyMem_Free(wstr); + if (res != 0) { if (res == -2) { PyObject *exc; @@ -3473,18 +3474,15 @@ unicode_encode_locale(PyObject *unicode, const char *errors, PyCodec_StrictErrors(exc); Py_DECREF(exc); } - return NULL; } else if (res == -3) { PyErr_SetString(PyExc_ValueError, "unsupported error handler"); } else { PyErr_NoMemory(); - PyMem_Free(wstr); - return NULL; } + return NULL; } - PyMem_Free(wstr); PyObject *bytes = PyBytes_FromString(str); PyMem_RawFree(str);

1 0

bpo-35240: Add "doctest" job to Travis CI (GH-10753)
by Victor Stinner 28 Nov '18

28 Nov '18

https://github.com/python/cpython/commit/a22df4896f6b83c8741203118790ae2817… commit: a22df4896f6b83c8741203118790ae281716bca5 branch: master author: Victor Stinner <vstinner(a)redhat.com> committer: GitHub <noreply(a)github.com> date: 2018-11-28T10:24:08+01:00 summary: bpo-35240: Add "doctest" job to Travis CI (GH-10753) Create a new "doctest" job in Travis CI to run "make doctest". files: M .travis.yml diff --git a/.travis.yml b/.travis.yml index ef3c2be5e580..c6e009291a2d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -57,6 +57,20 @@ matrix: - python -m pip install sphinx==1.8.2 blurb python-docs-theme script: - make check suspicious html SPHINXOPTS="-q -W -j4" + - os: linux + language: c + compiler: clang + env: TESTING=doctest + addons: + apt: + packages: + - xvfb + before_script: + - ./configure + - make -j4 + - make -C Doc/ PYTHON=../python venv + script: + xvfb-run make -C Doc/ PYTHON=../python SPHINXOPTS="-q -W -j4" doctest - os: osx language: c compiler: clang @@ -96,7 +110,7 @@ before_install: - set -e - | # Check short-circuit conditions - if [[ "${TESTING}" != "docs" ]] + if [[ "${TESTING}" != "docs" && "${TESTING}" != "doctest" ]] then if [[ "$TRAVIS_PULL_REQUEST" == "false" ]] then @@ -121,6 +135,7 @@ before_install: install: - | # Install OpenSSL as necessary + # Note: doctest needs OpenSSL if [[ "${TESTING}" != "docs" ]] then # clang complains about unused-parameter a lot, redirect stderr @@ -160,9 +175,6 @@ script: XVFB_RUN=xvfb-run; fi $XVFB_RUN make buildbottest TESTOPTS="-j4 -uall,-cpu" - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then - $XVFB_RUN make PYTHON=../python SPHINXOPTS="-q -W -j4" -C Doc/ venv doctest - fi notifications: email: false irc:

1 0

Daily reference leaks (4243df51fe43): sum=6
by solipsis＠pitrou.net 28 Nov '18

28 Nov '18

results for 4243df51fe43 on branch "default" -------------------------------------------- test_collections leaked [0, 7, -7] memory blocks, sum=0 test_functools leaked [0, 3, 1] memory blocks, sum=4 test_multiprocessing_fork leaked [1, -1, 2] memory blocks, sum=2 Command line was: ['./python', '-m', 'test.regrtest', '-uall', '-R', '3:3:/home/psf-users/antoine/refleaks/reflogDfh0jr', '--timeout', '7200']

1 0

bpo-33676: Fix dangling thread in _test_multiprocessing (GH-10755)
by Miss Islington (bot) 27 Nov '18

27 Nov '18

https://github.com/python/cpython/commit/dd5293871703e6a12ffdde14eeaa86a73b… commit: dd5293871703e6a12ffdde14eeaa86a73b7b0d99 branch: 3.6 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: GitHub <noreply(a)github.com> date: 2018-11-27T16:37:23-08:00 summary: bpo-33676: Fix dangling thread in _test_multiprocessing (GH-10755) Fix WithThreadsTestPool.test_wrapped_exception() of test_multiprocessing_fork: join the pool. WithThreadsTestPool.test_del_pool() is now also decorated with @support.reap_threads. (cherry picked from commit b7278736b3ae158a7738057e3045bc767ced019e) Co-authored-by: Victor Stinner <vstinner(a)redhat.com> files: M Lib/test/_test_multiprocessing.py diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index 6667f1178574..59f9a2e1e2eb 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -2250,6 +2250,7 @@ def test_wrapped_exception(self): with self.Pool(1) as p: with self.assertRaises(RuntimeError): p.apply(self._test_wrapped_exception) + p.join() def test_map_no_failfast(self): # Issue #23992: the fail-fast behaviour when an exception is raised @@ -2285,6 +2286,7 @@ def test_release_task_refs(self): # they were released too. self.assertEqual(CountedObject.n_instances, 0) + @support.reap_threads def test_del_pool(self): p = self.Pool(1) wr = weakref.ref(p)

1 0

bpo-33676: Fix dangling thread in _test_multiprocessing (GH-10755)
by Miss Islington (bot) 27 Nov '18

27 Nov '18

https://github.com/python/cpython/commit/80db40cdd6a58c9d66694a43ee9a7c06ab… commit: 80db40cdd6a58c9d66694a43ee9a7c06ab958373 branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: GitHub <noreply(a)github.com> date: 2018-11-27T16:30:34-08:00 summary: bpo-33676: Fix dangling thread in _test_multiprocessing (GH-10755) Fix WithThreadsTestPool.test_wrapped_exception() of test_multiprocessing_fork: join the pool. WithThreadsTestPool.test_del_pool() is now also decorated with @support.reap_threads. (cherry picked from commit b7278736b3ae158a7738057e3045bc767ced019e) Co-authored-by: Victor Stinner <vstinner(a)redhat.com> files: M Lib/test/_test_multiprocessing.py diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index 6e416a9855da..7a657d9d120c 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -2516,6 +2516,7 @@ def test_wrapped_exception(self): with self.Pool(1) as p: with self.assertRaises(RuntimeError): p.apply(self._test_wrapped_exception) + p.join() def test_map_no_failfast(self): # Issue #23992: the fail-fast behaviour when an exception is raised @@ -2551,6 +2552,7 @@ def test_release_task_refs(self): # they were released too. self.assertEqual(CountedObject.n_instances, 0) + @support.reap_threads def test_del_pool(self): p = self.Pool(1) wr = weakref.ref(p)

1 0

bpo-33676: Fix dangling thread in _test_multiprocessing (GH-10755)
by Victor Stinner 27 Nov '18

27 Nov '18

https://github.com/python/cpython/commit/b7278736b3ae158a7738057e3045bc767c… commit: b7278736b3ae158a7738057e3045bc767ced019e branch: master author: Victor Stinner <vstinner(a)redhat.com> committer: GitHub <noreply(a)github.com> date: 2018-11-28T01:14:31+01:00 summary: bpo-33676: Fix dangling thread in _test_multiprocessing (GH-10755) Fix WithThreadsTestPool.test_wrapped_exception() of test_multiprocessing_fork: join the pool. WithThreadsTestPool.test_del_pool() is now also decorated with @support.reap_threads. files: M Lib/test/_test_multiprocessing.py diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index 4302708c14f1..b62c119e9ae0 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -2522,6 +2522,7 @@ def test_wrapped_exception(self): with self.Pool(1) as p: with self.assertRaises(RuntimeError): p.apply(self._test_wrapped_exception) + p.join() def test_map_no_failfast(self): # Issue #23992: the fail-fast behaviour when an exception is raised @@ -2557,6 +2558,7 @@ def test_release_task_refs(self): # they were released too. self.assertEqual(CountedObject.n_instances, 0) + @support.reap_threads def test_del_pool(self): p = self.Pool(1) wr = weakref.ref(p)

1 0

bpo-35134: Don't define types twice in header files (GH-10754)
by Victor Stinner 27 Nov '18

27 Nov '18

https://github.com/python/cpython/commit/9bdd2de84c1af55fbc006d3f892313623b… commit: 9bdd2de84c1af55fbc006d3f892313623bd0195c branch: master author: Victor Stinner <vstinner(a)redhat.com> committer: GitHub <noreply(a)github.com> date: 2018-11-27T23:54:59+01:00 summary: bpo-35134: Don't define types twice in header files (GH-10754) Fix the following clang warning: Include/cpython/pystate.h:217:3: warning: redefinition of typedef 'PyThreadState' is a C11 feature [-Wtypedef-redefinition] files: M Include/object.h M Include/pystate.h diff --git a/Include/object.h b/Include/object.h index 21c29e7bdb00..a729335750c7 100644 --- a/Include/object.h +++ b/Include/object.h @@ -175,8 +175,12 @@ typedef int (*initproc)(PyObject *, PyObject *, PyObject *); typedef PyObject *(*newfunc)(struct _typeobject *, PyObject *, PyObject *); typedef PyObject *(*allocfunc)(struct _typeobject *, Py_ssize_t); +#ifdef Py_LIMITED_API /* In Py_LIMITED_API, PyTypeObject is an opaque structure. */ typedef struct _typeobject PyTypeObject; +#else +/* PyTypeObject is defined in cpython/object.h */ +#endif typedef struct{ int slot; /* slot id, see below */ @@ -196,30 +200,30 @@ PyAPI_FUNC(PyObject*) PyType_FromSpec(PyType_Spec*); PyAPI_FUNC(PyObject*) PyType_FromSpecWithBases(PyType_Spec*, PyObject*); #endif #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x03040000 -PyAPI_FUNC(void*) PyType_GetSlot(PyTypeObject*, int); +PyAPI_FUNC(void*) PyType_GetSlot(struct _typeobject*, int); #endif /* Generic type check */ -PyAPI_FUNC(int) PyType_IsSubtype(PyTypeObject *, PyTypeObject *); +PyAPI_FUNC(int) PyType_IsSubtype(struct _typeobject *, struct _typeobject *); #define PyObject_TypeCheck(ob, tp) \ (Py_TYPE(ob) == (tp) || PyType_IsSubtype(Py_TYPE(ob), (tp))) -PyAPI_DATA(PyTypeObject) PyType_Type; /* built-in 'type' */ -PyAPI_DATA(PyTypeObject) PyBaseObject_Type; /* built-in 'object' */ -PyAPI_DATA(PyTypeObject) PySuper_Type; /* built-in 'super' */ +PyAPI_DATA(struct _typeobject) PyType_Type; /* built-in 'type' */ +PyAPI_DATA(struct _typeobject) PyBaseObject_Type; /* built-in 'object' */ +PyAPI_DATA(struct _typeobject) PySuper_Type; /* built-in 'super' */ -PyAPI_FUNC(unsigned long) PyType_GetFlags(PyTypeObject*); +PyAPI_FUNC(unsigned long) PyType_GetFlags(struct _typeobject*); #define PyType_Check(op) \ PyType_FastSubclass(Py_TYPE(op), Py_TPFLAGS_TYPE_SUBCLASS) #define PyType_CheckExact(op) (Py_TYPE(op) == &PyType_Type) -PyAPI_FUNC(int) PyType_Ready(PyTypeObject *); -PyAPI_FUNC(PyObject *) PyType_GenericAlloc(PyTypeObject *, Py_ssize_t); -PyAPI_FUNC(PyObject *) PyType_GenericNew(PyTypeObject *, +PyAPI_FUNC(int) PyType_Ready(struct _typeobject *); +PyAPI_FUNC(PyObject *) PyType_GenericAlloc(struct _typeobject *, Py_ssize_t); +PyAPI_FUNC(PyObject *) PyType_GenericNew(struct _typeobject *, PyObject *, PyObject *); PyAPI_FUNC(unsigned int) PyType_ClearCache(void); -PyAPI_FUNC(void) PyType_Modified(PyTypeObject *); +PyAPI_FUNC(void) PyType_Modified(struct _typeobject *); /* Generic operations on objects */ PyAPI_FUNC(PyObject *) PyObject_Repr(PyObject *); @@ -397,8 +401,8 @@ PyAPI_FUNC(void) _PyDebug_PrintTotalRefs(void); #endif /* Py_REF_DEBUG */ #ifdef COUNT_ALLOCS -PyAPI_FUNC(void) _Py_inc_count(PyTypeObject *); -PyAPI_FUNC(void) _Py_dec_count(PyTypeObject *); +PyAPI_FUNC(void) _Py_inc_count(struct _typeobject *); +PyAPI_FUNC(void) _Py_dec_count(struct _typeobject *); #define _Py_INC_TPALLOCS(OP) _Py_inc_count(Py_TYPE(OP)) #define _Py_INC_TPFREES(OP) _Py_dec_count(Py_TYPE(OP)) #define _Py_DEC_TPFREES(OP) Py_TYPE(OP)->tp_frees-- diff --git a/Include/pystate.h b/Include/pystate.h index cc479ff867a4..ba5e9883306a 100644 --- a/Include/pystate.h +++ b/Include/pystate.h @@ -14,25 +14,28 @@ extern "C" { removed (with effort). */ #define MAX_CO_EXTRA_USERS 255 -/* State shared between threads */ - -struct _ts; /* Forward */ -struct _is; /* Forward */ -struct _frame; /* Forward declaration for PyFrameObject. */ +/* Forward declarations for PyFrameObject, PyThreadState + and PyInterpreterState */ +struct _frame; +struct _ts; +struct _is; +#ifdef Py_LIMITED_API +typedef struct _ts PyThreadState; typedef struct _is PyInterpreterState; +#else +/* PyThreadState and PyInterpreterState are defined in cpython/pystate.h */ +#endif /* State unique per thread */ -typedef struct _ts PyThreadState; - -PyAPI_FUNC(PyInterpreterState *) PyInterpreterState_New(void); -PyAPI_FUNC(void) PyInterpreterState_Clear(PyInterpreterState *); -PyAPI_FUNC(void) PyInterpreterState_Delete(PyInterpreterState *); +PyAPI_FUNC(struct _is *) PyInterpreterState_New(void); +PyAPI_FUNC(void) PyInterpreterState_Clear(struct _is *); +PyAPI_FUNC(void) PyInterpreterState_Delete(struct _is *); #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x03070000 /* New in 3.7 */ -PyAPI_FUNC(int64_t) PyInterpreterState_GetID(PyInterpreterState *); +PyAPI_FUNC(int64_t) PyInterpreterState_GetID(struct _is *); #endif #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x03030000 /* New in 3.3 */ @@ -41,9 +44,9 @@ PyAPI_FUNC(int) PyState_RemoveModule(struct PyModuleDef*); #endif PyAPI_FUNC(PyObject*) PyState_FindModule(struct PyModuleDef*); -PyAPI_FUNC(PyThreadState *) PyThreadState_New(PyInterpreterState *); -PyAPI_FUNC(void) PyThreadState_Clear(PyThreadState *); -PyAPI_FUNC(void) PyThreadState_Delete(PyThreadState *); +PyAPI_FUNC(struct _ts *) PyThreadState_New(struct _is *); +PyAPI_FUNC(void) PyThreadState_Clear(struct _ts *); +PyAPI_FUNC(void) PyThreadState_Delete(struct _ts *); PyAPI_FUNC(void) PyThreadState_DeleteCurrent(void); /* Get the current thread state. @@ -54,7 +57,7 @@ PyAPI_FUNC(void) PyThreadState_DeleteCurrent(void); The caller must hold the GIL. See also PyThreadState_GET() and _PyThreadState_GET(). */ -PyAPI_FUNC(PyThreadState *) PyThreadState_Get(void); +PyAPI_FUNC(struct _ts *) PyThreadState_Get(void); /* Get the current Python thread state. @@ -67,7 +70,7 @@ PyAPI_FUNC(PyThreadState *) PyThreadState_Get(void); See also PyThreadState_Get() and _PyThreadState_GET(). */ #define PyThreadState_GET() PyThreadState_Get() -PyAPI_FUNC(PyThreadState *) PyThreadState_Swap(PyThreadState *); +PyAPI_FUNC(struct _ts *) PyThreadState_Swap(struct _ts *); PyAPI_FUNC(PyObject *) PyThreadState_GetDict(void); PyAPI_FUNC(int) PyThreadState_SetAsyncExc(unsigned long, PyObject *); @@ -115,7 +118,7 @@ PyAPI_FUNC(void) PyGILState_Release(PyGILState_STATE); thread-state, even if no auto-thread-state call has been made on the main thread. */ -PyAPI_FUNC(PyThreadState *) PyGILState_GetThisThreadState(void); +PyAPI_FUNC(struct _ts *) PyGILState_GetThisThreadState(void); #ifndef Py_LIMITED_API

1 0