Mailman 3 November 2019 - Python-checkins

bpo-38866: Remove asyncore from test_pyclbr.py (GH-17316)
by Inada Naoki 22 Nov '19

22 Nov '19

https://github.com/python/cpython/commit/138e7bbb0a5ed44bdd54605e8c58c8f3d3… commit: 138e7bbb0a5ed44bdd54605e8c58c8f3d3865321 branch: master author: jacksonriley <52106215+jacksonriley(a)users.noreply.github.com> committer: Inada Naoki <songofacandy(a)gmail.com> date: 2019-11-22T21:51:58+09:00 summary: bpo-38866: Remove asyncore from test_pyclbr.py (GH-17316) Co-Authored-By: Kyle Stanley <aeros167(a)gmail.com> files: M Lib/test/test_pyclbr.py diff --git a/Lib/test/test_pyclbr.py b/Lib/test/test_pyclbr.py index 4385271cd0f2b..869799cfa9a66 100644 --- a/Lib/test/test_pyclbr.py +++ b/Lib/test/test_pyclbr.py @@ -247,7 +247,7 @@ def test_dotted_name_not_a_package(self): # not a package. # # Issue #14798. - self.assertRaises(ImportError, pyclbr.readmodule_ex, 'asyncore.foo') + self.assertRaises(ImportError, pyclbr.readmodule_ex, 'asyncio.foo') def test_module_has_no_spec(self): module_name = "doesnotexist"

1 0

bpo-36854: Fix reference counter in PyInit__testcapi() (GH-17338)
by Victor Stinner 22 Nov '19

22 Nov '19

https://github.com/python/cpython/commit/84c36c152a2bdf98f9cc7ce0e1db98e1f4… commit: 84c36c152a2bdf98f9cc7ce0e1db98e1f442a05e branch: 3.8 author: Victor Stinner <vstinner(a)python.org> committer: GitHub <noreply(a)github.com> date: 2019-11-22T13:39:36+01:00 summary: bpo-36854: Fix reference counter in PyInit__testcapi() (GH-17338) Increment properly Py_True/Py_False reference counter for _testcapi.WITH_PYMALLOC variable. files: M Modules/_testcapimodule.c diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 46d772cff608e..b54071669cf99 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -6283,11 +6283,14 @@ PyInit__testcapi(void) PyModule_AddObject(m, "instancemethod", (PyObject *)&PyInstanceMethod_Type); PyModule_AddIntConstant(m, "the_number_three", 3); + PyObject *v; #ifdef WITH_PYMALLOC - PyModule_AddObject(m, "WITH_PYMALLOC", Py_True); + v = Py_True; #else - PyModule_AddObject(m, "WITH_PYMALLOC", Py_False); + v = Py_False; #endif + Py_INCREF(v); + PyModule_AddObject(m, "WITH_PYMALLOC", v); TestError = PyErr_NewException("_testcapi.error", NULL, NULL); Py_INCREF(TestError);

1 0

bpo-38858: Call _PyUnicode_Fini() in Py_EndInterpreter() (GH-17330)
by Victor Stinner 22 Nov '19

22 Nov '19

https://github.com/python/cpython/commit/3d4833488a173c16446c3f94f58f05e2d1… commit: 3d4833488a173c16446c3f94f58f05e2d13c5dee branch: master author: Victor Stinner <vstinner(a)python.org> committer: GitHub <noreply(a)github.com> date: 2019-11-22T12:27:50+01:00 summary: bpo-38858: Call _PyUnicode_Fini() in Py_EndInterpreter() (GH-17330) Py_EndInterpreter() now clears the filesystem codec. files: M Include/internal/pycore_pylifecycle.h M Objects/unicodeobject.c M Python/pylifecycle.c diff --git a/Include/internal/pycore_pylifecycle.h b/Include/internal/pycore_pylifecycle.h index c837bcdbc03cc..73aa5ef1f6cab 100644 --- a/Include/internal/pycore_pylifecycle.h +++ b/Include/internal/pycore_pylifecycle.h @@ -77,7 +77,7 @@ extern void _PyImport_Fini2(void); extern void _PyGC_Fini(PyThreadState *tstate); extern void _PyType_Fini(void); extern void _Py_HashRandomization_Fini(void); -extern void _PyUnicode_Fini(void); +extern void _PyUnicode_Fini(PyThreadState *tstate); extern void _PyLong_Fini(void); extern void _PyFaulthandler_Fini(void); extern void _PyHash_Fini(void); diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 5ae0af8ea336a..89e45d01e3789 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -15929,34 +15929,37 @@ _PyUnicode_EnableLegacyWindowsFSEncoding(void) void -_PyUnicode_Fini(void) +_PyUnicode_Fini(PyThreadState *tstate) { + if (_Py_IsMainInterpreter(tstate)) { #if defined(WITH_VALGRIND) || defined(__INSURE__) - /* Insure++ is a memory analysis tool that aids in discovering - * memory leaks and other memory problems. On Python exit, the - * interned string dictionaries are flagged as being in use at exit - * (which it is). Under normal circumstances, this is fine because - * the memory will be automatically reclaimed by the system. Under - * memory debugging, it's a huge source of useless noise, so we - * trade off slower shutdown for less distraction in the memory - * reports. -baw - */ - unicode_release_interned(); + /* Insure++ is a memory analysis tool that aids in discovering + * memory leaks and other memory problems. On Python exit, the + * interned string dictionaries are flagged as being in use at exit + * (which it is). Under normal circumstances, this is fine because + * the memory will be automatically reclaimed by the system. Under + * memory debugging, it's a huge source of useless noise, so we + * trade off slower shutdown for less distraction in the memory + * reports. -baw + */ + unicode_release_interned(); #endif /* __INSURE__ */ - Py_CLEAR(unicode_empty); + Py_CLEAR(unicode_empty); - for (Py_ssize_t i = 0; i < 256; i++) { - Py_CLEAR(unicode_latin1[i]); + for (Py_ssize_t i = 0; i < 256; i++) { + Py_CLEAR(unicode_latin1[i]); + } + _PyUnicode_ClearStaticStrings(); + (void)PyUnicode_ClearFreeList(); } - _PyUnicode_ClearStaticStrings(); - (void)PyUnicode_ClearFreeList(); PyInterpreterState *interp = _PyInterpreterState_GET_UNSAFE(); PyMem_RawFree(interp->fs_codec.encoding); interp->fs_codec.encoding = NULL; PyMem_RawFree(interp->fs_codec.errors); interp->fs_codec.errors = NULL; + interp->config.filesystem_errors = _Py_ERROR_UNKNOWN; } diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 8c508e33800ef..5f3c49a680439 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -1182,9 +1182,6 @@ finalize_interp_types(PyThreadState *tstate, int is_main_interp) _PySet_Fini(); _PyBytes_Fini(); _PyLong_Fini(); - } - - if (is_main_interp) { _PyFloat_Fini(); _PyDict_Fini(); _PySlice_Fini(); @@ -1197,9 +1194,12 @@ finalize_interp_types(PyThreadState *tstate, int is_main_interp) _PyArg_Fini(); _PyAsyncGen_Fini(); _PyContext_Fini(); + } + + /* Cleanup Unicode implementation */ + _PyUnicode_Fini(tstate); - /* Cleanup Unicode implementation */ - _PyUnicode_Fini(); + if (is_main_interp) { _Py_ClearFileSystemEncoding(); } }

1 0

bpo-36854: Fix refleak in subinterpreter (GH-17331)
by Victor Stinner 22 Nov '19

22 Nov '19

https://github.com/python/cpython/commit/310e2d25170a88ef03f6fd31efcc899fe0… commit: 310e2d25170a88ef03f6fd31efcc899fe062da2c branch: master author: Victor Stinner <vstinner(a)python.org> committer: GitHub <noreply(a)github.com> date: 2019-11-22T10:58:00+01:00 summary: bpo-36854: Fix refleak in subinterpreter (GH-17331) finalize_interp_clear() now explicitly clears the codec registry and then trigger a GC collection to clear all references. files: M Modules/_testcapimodule.c M Python/pylifecycle.c diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index baa6907b7e4f6..0908f3457f580 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -6721,11 +6721,14 @@ PyInit__testcapi(void) PyModule_AddObject(m, "instancemethod", (PyObject *)&PyInstanceMethod_Type); PyModule_AddIntConstant(m, "the_number_three", 3); + PyObject *v; #ifdef WITH_PYMALLOC - PyModule_AddObject(m, "WITH_PYMALLOC", Py_True); + v = Py_True; #else - PyModule_AddObject(m, "WITH_PYMALLOC", Py_False); + v = Py_False; #endif + Py_INCREF(v); + PyModule_AddObject(m, "WITH_PYMALLOC", v); TestError = PyErr_NewException("_testcapi.error", NULL, NULL); Py_INCREF(TestError); diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 7591f069b455d..8c508e33800ef 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -1210,6 +1210,14 @@ finalize_interp_clear(PyThreadState *tstate) { int is_main_interp = _Py_IsMainInterpreter(tstate); + /* bpo-36854: Explicitly clear the codec registry + and trigger a GC collection */ + PyInterpreterState *interp = tstate->interp; + Py_CLEAR(interp->codec_search_path); + Py_CLEAR(interp->codec_search_cache); + Py_CLEAR(interp->codec_error_registry); + _PyGC_CollectNoFail(); + /* Clear interpreter state and all thread states */ PyInterpreterState_Clear(tstate->interp);

1 0

bpo-38863: Improve is_cgi() in http.server (GH-17312)
by Miss Islington (bot) 22 Nov '19

22 Nov '19

https://github.com/python/cpython/commit/91daa9d7224626dad4bb820924c01b3438… commit: 91daa9d7224626dad4bb820924c01b3438ca6e3f branch: master author: Siwon Kang <kkangshawn(a)gmail.com> committer: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> date: 2019-11-22T01:13:05-08:00 summary: bpo-38863: Improve is_cgi() in http.server (GH-17312) is_cgi() function of http.server library does not currently handle a cgi script if one of the cgi_directories is located at the sub-directory of given path. Since is_cgi() in CGIHTTPRequestHandler class separates given path into (dir, rest) based on the first seen '/', multi-level directories like /sub/dir/cgi-bin/hello.py is divided into head=/sub, rest=dir/cgi-bin/hello.py then check whether '/sub' exists in cgi_directories = [..., '/sub/dir/cgi-bin']. This patch makes the is_cgi() keep expanding dir part to the next '/' then checking if that expanded path exists in the cgi_directories. Signed-off-by: Siwon Kang <kkangshawn(a)gmail.com> https://bugs.python.org/issue38863 files: A Misc/NEWS.d/next/Library/2019-11-21-16-30-00.bpo-38863.RkdTjf.rst M Lib/http/server.py M Lib/test/test_httpservers.py diff --git a/Lib/http/server.py b/Lib/http/server.py index 005dd824f9a71..47a4fcf9a6518 100644 --- a/Lib/http/server.py +++ b/Lib/http/server.py @@ -1014,8 +1014,10 @@ def is_cgi(self): """ collapsed_path = _url_collapse_path(self.path) dir_sep = collapsed_path.find('/', 1) - head, tail = collapsed_path[:dir_sep], collapsed_path[dir_sep+1:] - if head in self.cgi_directories: + while dir_sep > 0 and not collapsed_path[:dir_sep] in self.cgi_directories: + dir_sep = collapsed_path.find('/', dir_sep+1) + if dir_sep > 0: + head, tail = collapsed_path[:dir_sep], collapsed_path[dir_sep+1:] self.cgi_info = head, tail return True return False diff --git a/Lib/test/test_httpservers.py b/Lib/test/test_httpservers.py index 1c980a2fa66ea..26da71e0b2701 100644 --- a/Lib/test/test_httpservers.py +++ b/Lib/test/test_httpservers.py @@ -601,13 +601,20 @@ def setUp(self): self.parent_dir = tempfile.mkdtemp() self.cgi_dir = os.path.join(self.parent_dir, 'cgi-bin') self.cgi_child_dir = os.path.join(self.cgi_dir, 'child-dir') + self.sub_dir_1 = os.path.join(self.parent_dir, 'sub') + self.sub_dir_2 = os.path.join(self.sub_dir_1, 'dir') + self.cgi_dir_in_sub_dir = os.path.join(self.sub_dir_2, 'cgi-bin') os.mkdir(self.cgi_dir) os.mkdir(self.cgi_child_dir) + os.mkdir(self.sub_dir_1) + os.mkdir(self.sub_dir_2) + os.mkdir(self.cgi_dir_in_sub_dir) self.nocgi_path = None self.file1_path = None self.file2_path = None self.file3_path = None self.file4_path = None + self.file5_path = None # The shebang line should be pure ASCII: use symlink if possible. # See issue #7668. @@ -652,6 +659,11 @@ def setUp(self): file4.write(cgi_file4 % (self.pythonexe, 'QUERY_STRING')) os.chmod(self.file4_path, 0o777) + self.file5_path = os.path.join(self.cgi_dir_in_sub_dir, 'file5.py') + with open(self.file5_path, 'w', encoding='utf-8') as file5: + file5.write(cgi_file1 % self.pythonexe) + os.chmod(self.file5_path, 0o777) + os.chdir(self.parent_dir) def tearDown(self): @@ -669,8 +681,13 @@ def tearDown(self): os.remove(self.file3_path) if self.file4_path: os.remove(self.file4_path) + if self.file5_path: + os.remove(self.file5_path) os.rmdir(self.cgi_child_dir) os.rmdir(self.cgi_dir) + os.rmdir(self.cgi_dir_in_sub_dir) + os.rmdir(self.sub_dir_2) + os.rmdir(self.sub_dir_1) os.rmdir(self.parent_dir) finally: BaseTestCase.tearDown(self) @@ -789,6 +806,13 @@ def test_query_with_continuous_slashes(self): 'text/html', HTTPStatus.OK), (res.read(), res.getheader('Content-type'), res.status)) + def test_cgi_path_in_sub_directories(self): + CGIHTTPRequestHandler.cgi_directories.append('/sub/dir/cgi-bin') + res = self.request('/sub/dir/cgi-bin/file5.py') + self.assertEqual( + (b'Hello World' + self.linesep, 'text/html', HTTPStatus.OK), + (res.read(), res.getheader('Content-type'), res.status)) + class SocketlessRequestHandler(SimpleHTTPRequestHandler): def __init__(self, directory=None): diff --git a/Misc/NEWS.d/next/Library/2019-11-21-16-30-00.bpo-38863.RkdTjf.rst b/Misc/NEWS.d/next/Library/2019-11-21-16-30-00.bpo-38863.RkdTjf.rst new file mode 100644 index 0000000000000..6b621eeea2fb6 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2019-11-21-16-30-00.bpo-38863.RkdTjf.rst @@ -0,0 +1,2 @@ +Improve :func:`is_cgi` function in :mod:`http.server`, which enables processing +the case that cgi directory is a child of another directory other than root.

1 0

Defer import of shutil which only needed for help and usage (GH-17334)
by Raymond Hettinger 21 Nov '19

21 Nov '19

https://github.com/python/cpython/commit/b4e5eeac267c436bb60776dc5be771d325… commit: b4e5eeac267c436bb60776dc5be771d3259bd298 branch: master author: Raymond Hettinger <rhettinger(a)users.noreply.github.com> committer: GitHub <noreply(a)github.com> date: 2019-11-21T22:51:45-08:00 summary: Defer import of shutil which only needed for help and usage (GH-17334) files: M Lib/argparse.py diff --git a/Lib/argparse.py b/Lib/argparse.py index 5a8eff2f4cc89..5d3ce2ad709f0 100644 --- a/Lib/argparse.py +++ b/Lib/argparse.py @@ -87,7 +87,6 @@ import os as _os import re as _re -import shutil as _shutil import sys as _sys from gettext import gettext as _, ngettext @@ -167,7 +166,8 @@ def __init__(self, # default setting for width if width is None: - width = _shutil.get_terminal_size().columns + import shutil + width = shutil.get_terminal_size().columns width -= 2 self._prog = prog @@ -264,7 +264,7 @@ def add_argument(self, action): invocations.append(get_invocation(subaction)) # update the maximum item length - invocation_length = max([len(s) for s in invocations]) + invocation_length = max(map(len, invocations)) action_length = invocation_length + self._current_indent self._action_max_length = max(self._action_max_length, action_length)

1 0

[3.7] bpo-16576: Add checks for bitfields passed by value to functions. (GH-17097) (GH-17224)
by Vinay Sajip 21 Nov '19

21 Nov '19

https://github.com/python/cpython/commit/91c15a542cb780377dcde8fc17ba93111b… commit: 91c15a542cb780377dcde8fc17ba93111bc1d1cf branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: Vinay Sajip <vinay_sajip(a)yahoo.co.uk> date: 2019-11-21T21:47:22Z summary: [3.7] bpo-16576: Add checks for bitfields passed by value to functions. (GH-17097) (GH-17224) (cherry picked from commit 106271568c58cfebae58f0c52b640dbe716ba2ce) files: M Lib/ctypes/test/test_structures.py M Modules/_ctypes/_ctypes.c M Modules/_ctypes/_ctypes_test.c diff --git a/Lib/ctypes/test/test_structures.py b/Lib/ctypes/test/test_structures.py index 8e2897ad51230..c129377041f03 100644 --- a/Lib/ctypes/test/test_structures.py +++ b/Lib/ctypes/test/test_structures.py @@ -612,6 +612,87 @@ class Test5(Structure): self.assertEqual(test5.nested.an_int, 0) self.assertEqual(test5.another_int, 0) + #(a)unittest.skipIf('s390' in MACHINE, 'Test causes segfault on S390') + def test_bitfield_by_value(self): + # See bpo-16576 + + # These should mirror the structures in Modules/_ctypes/_ctypes_test.c + + class Test6(Structure): + _fields_ = [ + ('A', c_int, 1), + ('B', c_int, 2), + ('C', c_int, 3), + ('D', c_int, 2), + ] + + test6 = Test6() + # As these are signed int fields, all are logically -1 due to sign + # extension. + test6.A = 1 + test6.B = 3 + test6.C = 7 + test6.D = 3 + dll = CDLL(_ctypes_test.__file__) + with self.assertRaises(TypeError) as ctx: + func = dll._testfunc_bitfield_by_value1 + func.restype = c_long + func.argtypes = (Test6,) + result = func(test6) + self.assertEqual(ctx.exception.args[0], 'item 1 in _argtypes_ passes ' + 'a struct/union with a bitfield by value, which is ' + 'unsupported.') + # passing by reference should be OK + func = dll._testfunc_bitfield_by_reference1 + func.restype = c_long + func.argtypes = (POINTER(Test6),) + result = func(byref(test6)) + self.assertEqual(result, -4) + self.assertEqual(test6.A, 0) + self.assertEqual(test6.B, 0) + self.assertEqual(test6.C, 0) + self.assertEqual(test6.D, 0) + + class Test7(Structure): + _fields_ = [ + ('A', c_uint, 1), + ('B', c_uint, 2), + ('C', c_uint, 3), + ('D', c_uint, 2), + ] + test7 = Test7() + test7.A = 1 + test7.B = 3 + test7.C = 7 + test7.D = 3 + func = dll._testfunc_bitfield_by_reference2 + func.restype = c_long + func.argtypes = (POINTER(Test7),) + result = func(byref(test7)) + self.assertEqual(result, 14) + self.assertEqual(test7.A, 0) + self.assertEqual(test7.B, 0) + self.assertEqual(test7.C, 0) + self.assertEqual(test7.D, 0) + + # for a union with bitfields, the union check happens first + class Test8(Union): + _fields_ = [ + ('A', c_int, 1), + ('B', c_int, 2), + ('C', c_int, 3), + ('D', c_int, 2), + ] + + test8 = Test8() + with self.assertRaises(TypeError) as ctx: + func = dll._testfunc_bitfield_by_value2 + func.restype = c_long + func.argtypes = (Test8,) + result = func(test8) + self.assertEqual(ctx.exception.args[0], 'item 1 in _argtypes_ passes ' + 'a union by value, which is unsupported.') + class PointerMemberTestCase(unittest.TestCase): def test(self): diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index 0e209c33a80af..c8fed44599543 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -2276,16 +2276,13 @@ converters_from_argtypes(PyObject *ob) for (i = 0; i < nArgs; ++i) { PyObject *tp = PyTuple_GET_ITEM(ob, i); - PyObject *cnv = PyObject_GetAttrString(tp, "from_param"); - if (!cnv) - goto argtypes_error_1; + PyObject *cnv; StgDictObject *stgdict = PyType_stgdict(tp); if (stgdict != NULL) { if (stgdict->flags & TYPEFLAG_HASUNION) { Py_DECREF(converters); Py_DECREF(ob); - Py_DECREF(cnv); if (!PyErr_Occurred()) { PyErr_Format(PyExc_TypeError, "item %zd in _argtypes_ passes a union by " @@ -2294,12 +2291,22 @@ converters_from_argtypes(PyObject *ob) } return NULL; } - /* if (stgdict->flags & TYPEFLAG_HASBITFIELD) { - printf("found stgdict with bitfield\n"); + Py_DECREF(converters); + Py_DECREF(ob); + if (!PyErr_Occurred()) { + PyErr_Format(PyExc_TypeError, + "item %zd in _argtypes_ passes a struct/" + "union with a bitfield by value, which is " + "unsupported.", + i + 1); + } + return NULL; } - */ } + cnv = PyObject_GetAttrString(tp, "from_param"); + if (!cnv) + goto argtypes_error_1; PyTuple_SET_ITEM(converters, i, cnv); } Py_DECREF(ob); diff --git a/Modules/_ctypes/_ctypes_test.c b/Modules/_ctypes/_ctypes_test.c index e06a045545578..d08a011e944ad 100644 --- a/Modules/_ctypes/_ctypes_test.c +++ b/Modules/_ctypes/_ctypes_test.c @@ -198,6 +198,56 @@ _testfunc_union_by_reference3(Test5 *in) { return result; } +typedef struct { + signed int A: 1, B:2, C:3, D:2; +} Test6; + +EXPORT(long) +_testfunc_bitfield_by_value1(Test6 in) { + long result = in.A + in.B + in.C + in.D; + + /* As the struct is passed by value, changes to it shouldn't be + * reflected in the caller. + */ + memset(&in, 0, sizeof(in)); + return result; +} + +EXPORT(long) +_testfunc_bitfield_by_reference1(Test6 *in) { + long result = in->A + in->B + in->C + in->D; + + memset(in, 0, sizeof(Test6)); + return result; +} + +typedef struct { + unsigned int A: 1, B:2, C:3, D:2; +} Test7; + +EXPORT(long) +_testfunc_bitfield_by_reference2(Test7 *in) { + long result = in->A + in->B + in->C + in->D; + + memset(in, 0, sizeof(Test7)); + return result; +} + +typedef union { + signed int A: 1, B:2, C:3, D:2; +} Test8; + +EXPORT(long) +_testfunc_bitfield_by_value2(Test8 in) { + long result = in.A + in.B + in.C + in.D; + + /* As the struct is passed by value, changes to it shouldn't be + * reflected in the caller. + */ + memset(&in, 0, sizeof(in)); + return result; +} + EXPORT(void)testfunc_array(int values[4]) { printf("testfunc_array %d %d %d %d\n",

1 0

bpo-38526: Fix zipfile.Path method name to be the correct one (GH-17317)
by Miss Islington (bot) 21 Nov '19

21 Nov '19

https://github.com/python/cpython/commit/107ed88cde3ae6f1cb01ae75575ea0f92c… commit: 107ed88cde3ae6f1cb01ae75575ea0f92c138464 branch: 3.8 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: GitHub <noreply(a)github.com> date: 2019-11-21T13:41:20-08:00 summary: bpo-38526: Fix zipfile.Path method name to be the correct one (GH-17317) (cherry picked from commit 65444cf7fe84d8ca1f9b51c7f5992210751e08bb) Co-authored-by: Claudiu Popa <pcmanticore(a)gmail.com> files: M Doc/library/zipfile.rst diff --git a/Doc/library/zipfile.rst b/Doc/library/zipfile.rst index 18d2e9a60a7ba..e8a2530fb8c17 100644 --- a/Doc/library/zipfile.rst +++ b/Doc/library/zipfile.rst @@ -494,7 +494,7 @@ Path objects are traversable using the ``/`` operator. Invoke :meth:`ZipFile.open` on the current path. Accepts the same arguments as :meth:`ZipFile.open`. -.. method:: Path.listdir() +.. method:: Path.iterdir() Enumerate the children of the current directory.

1 0

bpo-38526: Fix zipfile.Path method name to be the correct one (#17317)
by Jason R. Coombs 21 Nov '19

21 Nov '19

https://github.com/python/cpython/commit/65444cf7fe84d8ca1f9b51c7f599221075… commit: 65444cf7fe84d8ca1f9b51c7f5992210751e08bb branch: master author: Claudiu Popa <pcmanticore(a)gmail.com> committer: Jason R. Coombs <jaraco(a)jaraco.com> date: 2019-11-21T16:23:13-05:00 summary: bpo-38526: Fix zipfile.Path method name to be the correct one (#17317) files: M Doc/library/zipfile.rst diff --git a/Doc/library/zipfile.rst b/Doc/library/zipfile.rst index 18d2e9a60a7ba..e8a2530fb8c17 100644 --- a/Doc/library/zipfile.rst +++ b/Doc/library/zipfile.rst @@ -494,7 +494,7 @@ Path objects are traversable using the ``/`` operator. Invoke :meth:`ZipFile.open` on the current path. Accepts the same arguments as :meth:`ZipFile.open`. -.. method:: Path.listdir() +.. method:: Path.iterdir() Enumerate the children of the current directory.

1 0

[3.8] bpo-38857: AsyncMock fix for awaitable values and StopIteration fix [3.8] (GH-17269) (#17304)
by Andrew Svetlov 21 Nov '19

21 Nov '19

https://github.com/python/cpython/commit/b2744c1be73f5af0d2dc4b952389efc90c… commit: b2744c1be73f5af0d2dc4b952389efc90c8de94e branch: 3.8 author: Lisa Roach <lisaroach14(a)gmail.com> committer: Andrew Svetlov <andrew.svetlov(a)gmail.com> date: 2019-11-21T20:14:32+02:00 summary: [3.8] bpo-38857: AsyncMock fix for awaitable values and StopIteration fix [3.8] (GH-17269) (#17304) (cherry picked from commit 046442d02bcc6e848e71e93e47f6cde9e279e993) Co-authored-by: Jason Fried <fried(a)fb.com> files: A Misc/NEWS.d/next/Library/2019-11-19-16-28-25.bpo-38857.YPUkU9.rst A Misc/NEWS.d/next/Library/2019-11-19-16-30-46.bpo-38859.AZUzL8.rst M Doc/library/unittest.mock.rst M Lib/unittest/mock.py M Lib/unittest/test/testmock/testasync.py diff --git a/Doc/library/unittest.mock.rst b/Doc/library/unittest.mock.rst index 746cf765b077f..ebfaf7d643956 100644 --- a/Doc/library/unittest.mock.rst +++ b/Doc/library/unittest.mock.rst @@ -873,7 +873,7 @@ object:: exception, - if ``side_effect`` is an iterable, the async function will return the next value of the iterable, however, if the sequence of result is - exhausted, ``StopIteration`` is raised immediately, + exhausted, ``StopAsyncIteration`` is raised immediately, - if ``side_effect`` is not defined, the async function will return the value defined by ``return_value``, hence, by default, the async function returns a new :class:`AsyncMock` object. diff --git a/Lib/unittest/mock.py b/Lib/unittest/mock.py index 488ab1c23de42..d6e30679456a5 100644 --- a/Lib/unittest/mock.py +++ b/Lib/unittest/mock.py @@ -1125,8 +1125,8 @@ def _increment_mock_call(self, /, *args, **kwargs): _new_parent = _new_parent._mock_new_parent def _execute_mock_call(self, /, *args, **kwargs): - # seperate from _increment_mock_call so that awaited functions are - # executed seperately from their call + # separate from _increment_mock_call so that awaited functions are + # executed separately from their call, also AsyncMock overrides this method effect = self.side_effect if effect is not None: @@ -2122,29 +2122,45 @@ def __init__(self, /, *args, **kwargs): code_mock.co_flags = inspect.CO_COROUTINE self.__dict__['__code__'] = code_mock - async def _mock_call(self, /, *args, **kwargs): - try: - result = super()._mock_call(*args, **kwargs) - except (BaseException, StopIteration) as e: - side_effect = self.side_effect - if side_effect is not None and not callable(side_effect): - raise - return await _raise(e) + async def _execute_mock_call(self, /, *args, **kwargs): + # This is nearly just like super(), except for sepcial handling + # of coroutines _call = self.call_args + self.await_count += 1 + self.await_args = _call + self.await_args_list.append(_call) - async def proxy(): - try: - if inspect.isawaitable(result): - return await result - else: - return result - finally: - self.await_count += 1 - self.await_args = _call - self.await_args_list.append(_call) + effect = self.side_effect + if effect is not None: + if _is_exception(effect): + raise effect + elif not _callable(effect): + try: + result = next(effect) + except StopIteration: + # It is impossible to propogate a StopIteration + # through coroutines because of PEP 479 + raise StopAsyncIteration + if _is_exception(result): + raise result + elif asyncio.iscoroutinefunction(effect): + result = await effect(*args, **kwargs) + else: + result = effect(*args, **kwargs) - return await proxy() + if result is not DEFAULT: + return result + + if self._mock_return_value is not DEFAULT: + return self.return_value + + if self._mock_wraps is not None: + if asyncio.iscoroutinefunction(self._mock_wraps): + return await self._mock_wraps(*args, **kwargs) + return self._mock_wraps(*args, **kwargs) + + return self.return_value def assert_awaited(self): """ @@ -2852,10 +2868,6 @@ def seal(mock): seal(m) -async def _raise(exception): - raise exception - - class _AsyncIterator: """ Wraps an iterator in an asynchronous iterator. diff --git a/Lib/unittest/test/testmock/testasync.py b/Lib/unittest/test/testmock/testasync.py index 7671d55a8cfba..be8e3117c466b 100644 --- a/Lib/unittest/test/testmock/testasync.py +++ b/Lib/unittest/test/testmock/testasync.py @@ -353,42 +353,84 @@ def test_magicmock_lambda_spec(self): self.assertIsInstance(cm, MagicMock) -class AsyncArguments(unittest.TestCase): - def test_add_return_value(self): +class AsyncArguments(unittest.IsolatedAsyncioTestCase): + async def test_add_return_value(self): async def addition(self, var): return var + 1 mock = AsyncMock(addition, return_value=10) - output = asyncio.run(mock(5)) + output = await mock(5) self.assertEqual(output, 10) - def test_add_side_effect_exception(self): + async def test_add_side_effect_exception(self): async def addition(var): return var + 1 mock = AsyncMock(addition, side_effect=Exception('err')) with self.assertRaises(Exception): - asyncio.run(mock(5)) + await mock(5) - def test_add_side_effect_function(self): + async def test_add_side_effect_function(self): async def addition(var): return var + 1 mock = AsyncMock(side_effect=addition) - result = asyncio.run(mock(5)) + result = await mock(5) self.assertEqual(result, 6) - def test_add_side_effect_iterable(self): + async def test_add_side_effect_iterable(self): vals = [1, 2, 3] mock = AsyncMock(side_effect=vals) for item in vals: - self.assertEqual(item, asyncio.run(mock())) - - with self.assertRaises(RuntimeError) as e: - asyncio.run(mock()) - self.assertEqual( - e.exception, - RuntimeError('coroutine raised StopIteration') - ) + self.assertEqual(item, await mock()) + + with self.assertRaises(StopAsyncIteration) as e: + await mock() + + async def test_return_value_AsyncMock(self): + value = AsyncMock(return_value=10) + mock = AsyncMock(return_value=value) + result = await mock() + self.assertIs(result, value) + + async def test_return_value_awaitable(self): + fut = asyncio.Future() + fut.set_result(None) + mock = AsyncMock(return_value=fut) + result = await mock() + self.assertIsInstance(result, asyncio.Future) + + async def test_side_effect_awaitable_values(self): + fut = asyncio.Future() + fut.set_result(None) + + mock = AsyncMock(side_effect=[fut]) + result = await mock() + self.assertIsInstance(result, asyncio.Future) + + with self.assertRaises(StopAsyncIteration): + await mock() + + async def test_side_effect_is_AsyncMock(self): + effect = AsyncMock(return_value=10) + mock = AsyncMock(side_effect=effect) + + result = await mock() + self.assertEqual(result, 10) + + async def test_wraps_coroutine(self): + value = asyncio.Future() + + ran = False + async def inner(): + nonlocal ran + ran = True + return value + + mock = AsyncMock(wraps=inner) + result = await mock() + self.assertEqual(result, value) + mock.assert_awaited() + self.assertTrue(ran) class AsyncMagicMethods(unittest.TestCase): def test_async_magic_methods_return_async_mocks(self): diff --git a/Misc/NEWS.d/next/Library/2019-11-19-16-28-25.bpo-38857.YPUkU9.rst b/Misc/NEWS.d/next/Library/2019-11-19-16-28-25.bpo-38857.YPUkU9.rst new file mode 100644 index 0000000000000..f28df2811fb78 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2019-11-19-16-28-25.bpo-38857.YPUkU9.rst @@ -0,0 +1,4 @@ +AsyncMock fix for return values that are awaitable types. This also covers +side_effect iterable values that happend to be awaitable, and wraps +callables that return an awaitable type. Before these awaitables were being +awaited instead of being returned as is. diff --git a/Misc/NEWS.d/next/Library/2019-11-19-16-30-46.bpo-38859.AZUzL8.rst b/Misc/NEWS.d/next/Library/2019-11-19-16-30-46.bpo-38859.AZUzL8.rst new file mode 100644 index 0000000000000..c059539a1de60 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2019-11-19-16-30-46.bpo-38859.AZUzL8.rst @@ -0,0 +1,3 @@ +AsyncMock now returns StopAsyncIteration on the exaustion of a side_effects +iterable. Since PEP-479 its Impossible to raise a StopIteration exception +from a coroutine.

1 0