Mailman 3 May 2019 - Python-checkins

bpo-20602: Do not clear sys.flags and sys.float_info during shutdown (GH-8096)
by Petr Viktorin May 30, 2019

May 30, 2019

https://github.com/python/cpython/commit/249b7d59d8038f9017fc95dc28a3ce3494… commit: 249b7d59d8038f9017fc95dc28a3ce3494aaf832 branch: master author: Zackery Spytz <zspytz(a)gmail.com> committer: Petr Viktorin <pviktori(a)redhat.com> date: 2019-05-30T13:08:24+02:00 summary: bpo-20602: Do not clear sys.flags and sys.float_info during shutdown (GH-8096) There is no need to clear these immutable objects during shutdown. files: A Misc/NEWS.d/next/Core and Builtins/2018-07-04-16-57-59.bpo-20602.sDLElw.rst M Lib/test/test_sys.py M Python/import.c diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index c558d116e6fe..49f2722d9514 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -822,6 +822,22 @@ def __del__(self): rc, stdout, stderr = assert_python_ok('-c', code) self.assertEqual(stdout.rstrip(), b'True') + @test.support.requires_type_collecting + def test_issue20602(self): + # sys.flags and sys.float_info were wiped during shutdown. + code = """if 1: + import sys + class A: + def __del__(self, sys=sys): + print(sys.flags) + print(sys.float_info) + a = A() + """ + rc, out, err = assert_python_ok('-c', code) + out = out.splitlines() + self.assertIn(b'sys.flags', out[0]) + self.assertIn(b'sys.float_info', out[1]) + @unittest.skipUnless(hasattr(sys, 'getandroidapilevel'), 'need sys.getandroidapilevel()') def test_getandroidapilevel(self): diff --git a/Misc/NEWS.d/next/Core and Builtins/2018-07-04-16-57-59.bpo-20602.sDLElw.rst b/Misc/NEWS.d/next/Core and Builtins/2018-07-04-16-57-59.bpo-20602.sDLElw.rst new file mode 100644 index 000000000000..ab37a020d803 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2018-07-04-16-57-59.bpo-20602.sDLElw.rst @@ -0,0 +1,2 @@ +Do not clear :data:`sys.flags` and :data:`sys.float_info` during shutdown. +Patch by Zackery Spytz. diff --git a/Python/import.c b/Python/import.c index 41a5c01cadf3..ab7db6bc17f6 100644 --- a/Python/import.c +++ b/Python/import.c @@ -383,8 +383,6 @@ static const char * const sys_deletes[] = { "last_type", "last_value", "last_traceback", "path_hooks", "path_importer_cache", "meta_path", "__interactivehook__", - /* misc stuff */ - "flags", "float_info", NULL };

1 0

bpo-36974: remove _PyObject_HasFastCall (GH-13460)
by Petr Viktorin May 30, 2019

May 30, 2019

https://github.com/python/cpython/commit/c145f3bfbe80d498d40848450d4d33c14e… commit: c145f3bfbe80d498d40848450d4d33c14e2cf782 branch: master author: Jeroen Demeyer <J.Demeyer(a)UGent.be> committer: Petr Viktorin <encukou(a)gmail.com> date: 2019-05-30T12:43:58+02:00 summary: bpo-36974: remove _PyObject_HasFastCall (GH-13460) files: M Include/cpython/abstract.h M Modules/_functoolsmodule.c M Objects/call.c diff --git a/Include/cpython/abstract.h b/Include/cpython/abstract.h index 7099178f8208..7ab2045923d8 100644 --- a/Include/cpython/abstract.h +++ b/Include/cpython/abstract.h @@ -55,10 +55,6 @@ PyAPI_FUNC(int) _PyStack_UnpackDict( 40 bytes on the stack. */ #define _PY_FASTCALL_SMALL_STACK 5 -/* Return 1 if callable supports FASTCALL calling convention for positional - arguments: see _PyObject_Vectorcall() and _PyObject_FastCallDict() */ -PyAPI_FUNC(int) _PyObject_HasFastCall(PyObject *callable); - PyAPI_FUNC(PyObject *) _Py_CheckFunctionResult(PyObject *callable, PyObject *result, const char *where); diff --git a/Modules/_functoolsmodule.c b/Modules/_functoolsmodule.c index 13f2db939bb7..213fb3ea336c 100644 --- a/Modules/_functoolsmodule.c +++ b/Modules/_functoolsmodule.c @@ -107,7 +107,7 @@ partial_new(PyTypeObject *type, PyObject *args, PyObject *kw) return NULL; } - pto->use_fastcall = _PyObject_HasFastCall(func); + pto->use_fastcall = (_PyVectorcall_Function(func) != NULL); return (PyObject *)pto; } @@ -365,7 +365,7 @@ partial_setstate(partialobject *pto, PyObject *state) Py_INCREF(dict); Py_INCREF(fn); - pto->use_fastcall = _PyObject_HasFastCall(fn); + pto->use_fastcall = (_PyVectorcall_Function(fn) != NULL); Py_SETREF(pto->fn, fn); Py_SETREF(pto->args, fnargs); Py_SETREF(pto->kw, kw); diff --git a/Objects/call.c b/Objects/call.c index 183a5c2e5a24..55dfc520f1de 100644 --- a/Objects/call.c +++ b/Objects/call.c @@ -9,22 +9,6 @@ static PyObject * cfunction_call_varargs(PyObject *func, PyObject *args, PyObject *kwargs); -int -_PyObject_HasFastCall(PyObject *callable) -{ - if (PyFunction_Check(callable)) { - return 1; - } - else if (PyCFunction_Check(callable)) { - return !(PyCFunction_GET_FLAGS(callable) & METH_VARARGS); - } - else { - assert (PyCallable_Check(callable)); - return 0; - } -} - - static PyObject * null_error(void) {

1 0

bpo-36974: inherit the vectorcall protocol (GH-13498)
by Petr Viktorin May 30, 2019

May 30, 2019

https://github.com/python/cpython/commit/735e8afa9ee942367b5d0807633a2b9f66… commit: 735e8afa9ee942367b5d0807633a2b9f662cbdbf branch: master author: Jeroen Demeyer <J.Demeyer(a)UGent.be> committer: Petr Viktorin <encukou(a)gmail.com> date: 2019-05-30T12:43:19+02:00 summary: bpo-36974: inherit the vectorcall protocol (GH-13498) files: M Lib/test/test_capi.py M Modules/_testcapimodule.c M Objects/typeobject.c diff --git a/Lib/test/test_capi.py b/Lib/test/test_capi.py index 0813abb9a697..795aa78d8866 100644 --- a/Lib/test/test_capi.py +++ b/Lib/test/test_capi.py @@ -27,6 +27,7 @@ # Were we compiled --with-pydebug or with #define Py_DEBUG? Py_DEBUG = hasattr(sys, 'gettotalrefcount') +Py_TPFLAGS_HAVE_VECTORCALL = 1 << 11 Py_TPFLAGS_METHOD_DESCRIPTOR = 1 << 17 @@ -484,6 +485,27 @@ class MethodDescriptorHeap(_testcapi.MethodDescriptorBase): pass self.assertFalse(MethodDescriptorHeap.__flags__ & Py_TPFLAGS_METHOD_DESCRIPTOR) + def test_vectorcall_flag(self): + self.assertTrue(_testcapi.MethodDescriptorBase.__flags__ & Py_TPFLAGS_HAVE_VECTORCALL) + self.assertTrue(_testcapi.MethodDescriptorDerived.__flags__ & Py_TPFLAGS_HAVE_VECTORCALL) + self.assertFalse(_testcapi.MethodDescriptorNopGet.__flags__ & Py_TPFLAGS_HAVE_VECTORCALL) + self.assertTrue(_testcapi.MethodDescriptor2.__flags__ & Py_TPFLAGS_HAVE_VECTORCALL) + + # Heap type should not inherit Py_TPFLAGS_HAVE_VECTORCALL + class MethodDescriptorHeap(_testcapi.MethodDescriptorBase): + pass + self.assertFalse(MethodDescriptorHeap.__flags__ & Py_TPFLAGS_HAVE_VECTORCALL) + + def test_vectorcall_override(self): + # Check that tp_call can correctly override vectorcall. + # MethodDescriptorNopGet implements tp_call but it inherits from + # MethodDescriptorBase, which implements vectorcall. Since + # MethodDescriptorNopGet returns the args tuple when called, we check + # additionally that no new tuple is created for this call. + args = tuple(range(5)) + f = _testcapi.MethodDescriptorNopGet() + self.assertIs(f(*args), args) + def test_vectorcall(self): # Test a bunch of different ways to call objects: # 1. normal call @@ -498,7 +520,10 @@ def test_vectorcall(self): ([].append, (0,), {}, None), (sum, ([36],), {"start":6}, 42), (testfunction, (42,), {}, 42), - (testfunction_kw, (42,), {"kw":None}, 42)] + (testfunction_kw, (42,), {"kw":None}, 42), + (_testcapi.MethodDescriptorBase(), (0,), {}, True), + (_testcapi.MethodDescriptorDerived(), (0,), {}, True), + (_testcapi.MethodDescriptor2(), (0,), {}, False)] from _testcapi import pyobject_vectorcall, pyvectorcall_call from types import MethodType diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index f2f418c997ab..a7451c66359d 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -5814,6 +5814,29 @@ static PyTypeObject Generic_Type = { /* Test PEP 590 */ +typedef struct { + PyObject_HEAD + vectorcallfunc vectorcall; +} MethodDescriptorObject; + +static PyObject * +MethodDescriptor_vectorcall(PyObject *callable, PyObject *const *args, + size_t nargsf, PyObject *kwnames) +{ + /* True if using the vectorcall function in MethodDescriptorObject + * but False for MethodDescriptor2Object */ + MethodDescriptorObject *md = (MethodDescriptorObject *)callable; + return PyBool_FromLong(md->vectorcall != NULL); +} + +static PyObject * +MethodDescriptor_new(PyTypeObject* type, PyObject* args, PyObject *kw) +{ + MethodDescriptorObject *op = PyObject_New(MethodDescriptorObject, type); + op->vectorcall = MethodDescriptor_vectorcall; + return (PyObject *)op; +} + static PyObject * func_descr_get(PyObject *func, PyObject *obj, PyObject *type) { @@ -5831,10 +5854,22 @@ nop_descr_get(PyObject *func, PyObject *obj, PyObject *type) return func; } +static PyObject * +call_return_args(PyObject *self, PyObject *args, PyObject *kwargs) +{ + Py_INCREF(args); + return args; +} + static PyTypeObject MethodDescriptorBase_Type = { PyVarObject_HEAD_INIT(NULL, 0) "MethodDescriptorBase", - .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_METHOD_DESCRIPTOR, + sizeof(MethodDescriptorObject), + .tp_new = MethodDescriptor_new, + .tp_call = PyVectorcall_Call, + .tp_vectorcall_offset = offsetof(MethodDescriptorObject, vectorcall), + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | + Py_TPFLAGS_METHOD_DESCRIPTOR | _Py_TPFLAGS_HAVE_VECTORCALL, .tp_descr_get = func_descr_get, }; @@ -5848,9 +5883,34 @@ static PyTypeObject MethodDescriptorNopGet_Type = { PyVarObject_HEAD_INIT(NULL, 0) "MethodDescriptorNopGet", .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, + .tp_call = call_return_args, .tp_descr_get = nop_descr_get, }; +typedef struct { + MethodDescriptorObject base; + vectorcallfunc vectorcall; +} MethodDescriptor2Object; + +static PyObject * +MethodDescriptor2_new(PyTypeObject* type, PyObject* args, PyObject *kw) +{ + MethodDescriptor2Object *op = PyObject_New(MethodDescriptor2Object, type); + op->base.vectorcall = NULL; + op->vectorcall = MethodDescriptor_vectorcall; + return (PyObject *)op; +} + +static PyTypeObject MethodDescriptor2_Type = { + PyVarObject_HEAD_INIT(NULL, 0) + "MethodDescriptor2", + sizeof(MethodDescriptor2Object), + .tp_new = MethodDescriptor2_new, + .tp_call = PyVectorcall_Call, + .tp_vectorcall_offset = offsetof(MethodDescriptor2Object, vectorcall), + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | _Py_TPFLAGS_HAVE_VECTORCALL, +}; + static struct PyModuleDef _testcapimodule = { PyModuleDef_HEAD_INIT, @@ -5916,6 +5976,12 @@ PyInit__testcapi(void) Py_INCREF(&MethodDescriptorNopGet_Type); PyModule_AddObject(m, "MethodDescriptorNopGet", (PyObject *)&MethodDescriptorNopGet_Type); + MethodDescriptor2_Type.tp_base = &MethodDescriptorBase_Type; + if (PyType_Ready(&MethodDescriptor2_Type) < 0) + return NULL; + Py_INCREF(&MethodDescriptor2_Type); + PyModule_AddObject(m, "MethodDescriptor2", (PyObject *)&MethodDescriptor2_Type); + if (PyType_Ready(&GenericAlias_Type) < 0) return NULL; Py_INCREF(&GenericAlias_Type); diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 071ff27d5323..ac5a68681d15 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -5147,6 +5147,17 @@ inherit_slots(PyTypeObject *type, PyTypeObject *base) COPYSLOT(tp_repr); /* tp_hash see tp_richcompare */ COPYSLOT(tp_call); + /* Inherit tp_vectorcall_offset and _Py_TPFLAGS_HAVE_VECTORCALL if tp_call + * was inherited, but only for extension types */ + if ((base->tp_flags & _Py_TPFLAGS_HAVE_VECTORCALL) && + !(type->tp_flags & _Py_TPFLAGS_HAVE_VECTORCALL) && + !(type->tp_flags & Py_TPFLAGS_HEAPTYPE) && + base->tp_call && + type->tp_call == base->tp_call) + { + type->tp_vectorcall_offset = base->tp_vectorcall_offset; + type->tp_flags |= _Py_TPFLAGS_HAVE_VECTORCALL; + } COPYSLOT(tp_str); { /* Copy comparison-related slots only when

1 0

bpo-37015: Ensure tasks created by _accept_connection2 due to AsyncMock are completed (GH-13661)
by Miss Islington (bot) May 30, 2019

May 30, 2019

https://github.com/python/cpython/commit/0f39c2b1919727904f4fac2d79cb41dc6b… commit: 0f39c2b1919727904f4fac2d79cb41dc6bfe41fe branch: master author: Xtreak <tir.karthi(a)gmail.com> committer: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> date: 2019-05-30T03:00:29-07:00 summary: bpo-37015: Ensure tasks created by _accept_connection2 due to AsyncMock are completed (GH-13661) >From 3.8 async functions used with mock.patch return an `AsyncMock`. `_accept_connection2` is an async function where create_task is also mocked. Don't mock `create_task` so that tasks are created out of coroutine returned by `AsyncMock` and the tasks are completed. https://bugs.python.org/issue37015 files: M Lib/test/test_asyncio/test_selector_events.py diff --git a/Lib/test/test_asyncio/test_selector_events.py b/Lib/test/test_asyncio/test_selector_events.py index 2e52e9df5c3b..68b7853b2eba 100644 --- a/Lib/test/test_asyncio/test_selector_events.py +++ b/Lib/test/test_asyncio/test_selector_events.py @@ -363,14 +363,16 @@ def test_accept_connection_multiple(self): sock.accept.return_value = (mock.Mock(), mock.Mock()) backlog = 100 # Mock the coroutine generation for a connection to prevent - # warnings related to un-awaited coroutines. + # warnings related to un-awaited coroutines. _accept_connection2 + # is an async function that is patched with AsyncMock. create_task + # creates a task out of coroutine returned by AsyncMock, so use + # asyncio.sleep(0) to ensure created tasks are complete to avoid + # task pending warnings. mock_obj = mock.patch.object with mock_obj(self.loop, '_accept_connection2') as accept2_mock: - accept2_mock.return_value = None - with mock_obj(self.loop, 'create_task') as task_mock: - task_mock.return_value = None - self.loop._accept_connection( - mock.Mock(), sock, backlog=backlog) + self.loop._accept_connection( + mock.Mock(), sock, backlog=backlog) + self.loop.run_until_complete(asyncio.sleep(0)) self.assertEqual(sock.accept.call_count, backlog)

1 0

bpo-37098: Skip memfd_create test before Linux 3.17 (GH-13677)
by Christian Heimes May 30, 2019

May 30, 2019

https://github.com/python/cpython/commit/6eb814b8ce9a4fed8773a65501fb96aad8… commit: 6eb814b8ce9a4fed8773a65501fb96aad8b3ecf2 branch: master author: Christian Heimes <christian(a)python.org> committer: GitHub <noreply(a)github.com> date: 2019-05-30T11:27:06+02:00 summary: bpo-37098: Skip memfd_create test before Linux 3.17 (GH-13677) files: A Misc/NEWS.d/next/Tests/2019-05-30-10-57-39.bpo-37098.SfXt1M.rst M Lib/test/test_os.py diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index 820c99c7a07c..f17a19a7585d 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -3123,6 +3123,7 @@ def test_stty_match(self): @unittest.skipUnless(hasattr(os, 'memfd_create'), 'requires os.memfd_create') +(a)support.requires_linux_version(3, 17) class MemfdCreateTests(unittest.TestCase): def test_memfd_create(self): fd = os.memfd_create("Hi", os.MFD_CLOEXEC) diff --git a/Misc/NEWS.d/next/Tests/2019-05-30-10-57-39.bpo-37098.SfXt1M.rst b/Misc/NEWS.d/next/Tests/2019-05-30-10-57-39.bpo-37098.SfXt1M.rst new file mode 100644 index 000000000000..84e06e71869f --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2019-05-30-10-57-39.bpo-37098.SfXt1M.rst @@ -0,0 +1 @@ +Fix test_memfd_create on older Linux Kernels.

1 0

bpo-36935: Remove usage of the deprecated PyErr_SetFromWindowsErrWithUnicodeFilename() (GH-13355)
by Victor Stinner May 30, 2019

May 30, 2019

https://github.com/python/cpython/commit/eda385c0dca62f97a8ae80feb57c2a51df… commit: eda385c0dca62f97a8ae80feb57c2a51df3c807f branch: master author: Zackery Spytz <zspytz(a)gmail.com> committer: Victor Stinner <vstinner(a)redhat.com> date: 2019-05-30T09:58:50+02:00 summary: bpo-36935: Remove usage of the deprecated PyErr_SetFromWindowsErrWithUnicodeFilename() (GH-13355) In e895de3e7f3cc2f7213b87621cfe9812ea4343f0, the deprecated function PyErr_SetFromWindowsErrWithUnicodeFilename() was added in two functions in Modules/_winapi.c. This function was deprecated in 3.3. files: M Modules/_winapi.c diff --git a/Modules/_winapi.c b/Modules/_winapi.c index 1317fc9a172c..e9dcec6590b6 100644 --- a/Modules/_winapi.c +++ b/Modules/_winapi.c @@ -508,7 +508,9 @@ _winapi_CreateFileMapping_impl(PyObject *module, HANDLE file_handle, Py_END_ALLOW_THREADS if (handle == NULL) { - PyErr_SetFromWindowsErrWithUnicodeFilename(0, name); + PyObject *temp = PyUnicode_FromWideChar(name, -1); + PyErr_SetExcFromWindowsErrWithFilenameObject(PyExc_OSError, 0, temp); + Py_XDECREF(temp); handle = INVALID_HANDLE_VALUE; } @@ -1405,7 +1407,9 @@ _winapi_OpenFileMapping_impl(PyObject *module, DWORD desired_access, Py_END_ALLOW_THREADS if (handle == NULL) { - PyErr_SetFromWindowsErrWithUnicodeFilename(0, name); + PyObject *temp = PyUnicode_FromWideChar(name, -1); + PyErr_SetExcFromWindowsErrWithFilenameObject(PyExc_OSError, 0, temp); + Py_XDECREF(temp); handle = INVALID_HANDLE_VALUE; }

1 0

autoreconf (GH-13651)
by Inada Naoki May 30, 2019

May 30, 2019

https://github.com/python/cpython/commit/bee31ce775ec70013d360b353572bd1a01… commit: bee31ce775ec70013d360b353572bd1a01d78e67 branch: master author: Inada Naoki <songofacandy(a)gmail.com> committer: GitHub <noreply(a)github.com> date: 2019-05-30T16:35:41+09:00 summary: autoreconf (GH-13651) files: M configure diff --git a/configure b/configure index cc18322d7514..cacf9fc41894 100755 --- a/configure +++ b/configure @@ -11483,9 +11483,9 @@ for ac_func in alarm accept4 setitimer getitimer bind_textdomain_codeset chown \ getgrouplist getgroups getlogin getloadavg getpeername getpgid getpid \ getpriority getresuid getresgid getpwent getpwnam_r getpwuid_r getspnam getspent getsid getwd \ if_nameindex \ - initgroups kill killpg lchmod lchown lockf linkat lstat lutimes madvise mmap \ + initgroups kill killpg lchmod lchown lockf linkat lstat lutimes mmap \ memrchr mbrtowc mkdirat mkfifo \ - mkfifoat mknod mknodat mktime mremap nice openat pathconf pause pipe2 plock poll \ + madvise mkfifoat mknod mknodat mktime mremap nice openat pathconf pause pipe2 plock poll \ posix_fallocate posix_fadvise posix_spawn posix_spawnp pread preadv preadv2 \ pthread_condattr_setclock pthread_init pthread_kill putenv pwrite pwritev pwritev2 \ readlink readlinkat readv realpath renameat \

1 0

bpo-36610: shutil.copyfile(): use sendfile() on Linux only (GH-13675)
by Giampaolo Rodola May 30, 2019

May 30, 2019

https://github.com/python/cpython/commit/413d955f8ec88a7183f91d7ad8b0ff7def… commit: 413d955f8ec88a7183f91d7ad8b0ff7def803de3 branch: master author: Giampaolo Rodola <g.rodola(a)gmail.com> committer: GitHub <noreply(a)github.com> date: 2019-05-30T14:05:41+08:00 summary: bpo-36610: shutil.copyfile(): use sendfile() on Linux only (GH-13675) ...and avoid using it on Solaris as it can raise EINVAL if offset is equal or bigger than the size of the file files: M Doc/library/shutil.rst M Doc/whatsnew/3.8.rst M Lib/shutil.py M Lib/test/test_shutil.py M Misc/NEWS.d/3.8.0a1.rst diff --git a/Doc/library/shutil.rst b/Doc/library/shutil.rst index 4af5a1680608..dcb2a16cff98 100644 --- a/Doc/library/shutil.rst +++ b/Doc/library/shutil.rst @@ -420,8 +420,7 @@ the use of userspace buffers in Python as in "``outfd.write(infd.read())``". On macOS `fcopyfile`_ is used to copy the file content (not metadata). -On Linux, Solaris and other POSIX platforms where :func:`os.sendfile` supports -copies between 2 regular file descriptors :func:`os.sendfile` is used. +On Linux :func:`os.sendfile` is used. On Windows :func:`shutil.copyfile` uses a bigger default buffer size (1 MiB instead of 64 KiB) and a :func:`memoryview`-based variant of diff --git a/Doc/whatsnew/3.8.rst b/Doc/whatsnew/3.8.rst index 5ee9cf07ebaf..98f0c3474f26 100644 --- a/Doc/whatsnew/3.8.rst +++ b/Doc/whatsnew/3.8.rst @@ -772,7 +772,7 @@ Optimizations * :func:`shutil.copyfile`, :func:`shutil.copy`, :func:`shutil.copy2`, :func:`shutil.copytree` and :func:`shutil.move` use platform-specific - "fast-copy" syscalls on Linux, macOS and Solaris in order to copy the file + "fast-copy" syscalls on Linux and macOS in order to copy the file more efficiently. "fast-copy" means that the copying operation occurs within the kernel, avoiding the use of userspace buffers in Python as in diff --git a/Lib/shutil.py b/Lib/shutil.py index 2dfae87c9ce9..dae916b41605 100644 --- a/Lib/shutil.py +++ b/Lib/shutil.py @@ -50,7 +50,7 @@ import nt COPY_BUFSIZE = 1024 * 1024 if _WINDOWS else 64 * 1024 -_HAS_SENDFILE = posix and hasattr(os, "sendfile") +_USE_CP_SENDFILE = hasattr(os, "sendfile") and sys.platform.startswith("linux") _HAS_FCOPYFILE = posix and hasattr(posix, "_fcopyfile") # macOS __all__ = ["copyfileobj", "copyfile", "copymode", "copystat", "copy", "copy2", @@ -111,7 +111,7 @@ def _fastcopy_fcopyfile(fsrc, fdst, flags): def _fastcopy_sendfile(fsrc, fdst): """Copy data from one regular mmap-like fd to another by using high-performance sendfile(2) syscall. - This should work on Linux >= 2.6.33 and Solaris only. + This should work on Linux >= 2.6.33 only. """ # Note: copyfileobj() is left alone in order to not introduce any # unexpected breakage. Possible risks by using zero-copy calls @@ -122,7 +122,7 @@ def _fastcopy_sendfile(fsrc, fdst): # GzipFile (which decompresses data), HTTPResponse (which decodes # chunks). # - possibly others (e.g. encrypted fs/partition?) - global _HAS_SENDFILE + global _USE_CP_SENDFILE try: infd = fsrc.fileno() outfd = fdst.fileno() @@ -152,7 +152,7 @@ def _fastcopy_sendfile(fsrc, fdst): # sendfile() on this platform (probably Linux < 2.6.33) # does not support copies between regular files (only # sockets). - _HAS_SENDFILE = False + _USE_CP_SENDFILE = False raise _GiveupOnFastCopy(err) if err.errno == errno.ENOSPC: # filesystem is full @@ -260,8 +260,8 @@ def copyfile(src, dst, *, follow_symlinks=True): return dst except _GiveupOnFastCopy: pass - # Linux / Solaris - elif _HAS_SENDFILE: + # Linux + elif _USE_CP_SENDFILE: try: _fastcopy_sendfile(fsrc, fdst) return dst diff --git a/Lib/test/test_shutil.py b/Lib/test/test_shutil.py index eeebb97ff692..208718bb1281 100644 --- a/Lib/test/test_shutil.py +++ b/Lib/test/test_shutil.py @@ -2315,7 +2315,7 @@ def test_file2file_not_supported(self): # Emulate a case where sendfile() only support file->socket # fds. In such a case copyfile() is supposed to skip the # fast-copy attempt from then on. - assert shutil._HAS_SENDFILE + assert shutil._USE_CP_SENDFILE try: with unittest.mock.patch( self.PATCHPOINT, @@ -2324,13 +2324,13 @@ def test_file2file_not_supported(self): with self.assertRaises(_GiveupOnFastCopy): shutil._fastcopy_sendfile(src, dst) assert m.called - assert not shutil._HAS_SENDFILE + assert not shutil._USE_CP_SENDFILE with unittest.mock.patch(self.PATCHPOINT) as m: shutil.copyfile(TESTFN, TESTFN2) assert not m.called finally: - shutil._HAS_SENDFILE = True + shutil._USE_CP_SENDFILE = True @unittest.skipIf(not MACOS, 'macOS only') diff --git a/Misc/NEWS.d/3.8.0a1.rst b/Misc/NEWS.d/3.8.0a1.rst index 3d5e6336246e..f4b0a0483300 100644 --- a/Misc/NEWS.d/3.8.0a1.rst +++ b/Misc/NEWS.d/3.8.0a1.rst @@ -4450,7 +4450,7 @@ data_received() being called before connection_made(). :func:`shutil.copyfile`, :func:`shutil.copy`, :func:`shutil.copy2`, :func:`shutil.copytree` and :func:`shutil.move` use platform-specific -fast-copy syscalls on Linux, Solaris and macOS in order to copy the file +fast-copy syscalls on Linux and macOS in order to copy the file more efficiently. On Windows :func:`shutil.copyfile` uses a bigger default buffer size (1 MiB instead of 16 KiB) and a :func:`memoryview`-based variant of :func:`shutil.copyfileobj` is used. The speedup for copying a 512MiB file

1 0

bpo-24564: shutil.copystat(): ignore EINVAL on os.setxattr() (GH-13369)
by Giampaolo Rodola May 30, 2019

May 30, 2019

https://github.com/python/cpython/commit/f1487b323549e2360460383b4304f6592f… commit: f1487b323549e2360460383b4304f6592fb38e27 branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: Giampaolo Rodola <g.rodola(a)gmail.com> date: 2019-05-30T13:58:30+08:00 summary: bpo-24564: shutil.copystat(): ignore EINVAL on os.setxattr() (GH-13369) (cherry picked from commit a16387ab2d85f19665920bb6ff91a7e57f59dd2a) Co-authored-by: Ying Wang <me(a)yingw787.com> files: A Misc/NEWS.d/next/Library/2019-05-16-23-40-36.bpo-24564.lIwV_7.rst M Lib/shutil.py diff --git a/Lib/shutil.py b/Lib/shutil.py index 4c6fdd7d33d4..fc6fb4edd24c 100644 --- a/Lib/shutil.py +++ b/Lib/shutil.py @@ -156,7 +156,7 @@ def _copyxattr(src, dst, *, follow_symlinks=True): try: names = os.listxattr(src, follow_symlinks=follow_symlinks) except OSError as e: - if e.errno not in (errno.ENOTSUP, errno.ENODATA): + if e.errno not in (errno.ENOTSUP, errno.ENODATA, errno.EINVAL): raise return for name in names: @@ -164,7 +164,8 @@ def _copyxattr(src, dst, *, follow_symlinks=True): value = os.getxattr(src, name, follow_symlinks=follow_symlinks) os.setxattr(dst, name, value, follow_symlinks=follow_symlinks) except OSError as e: - if e.errno not in (errno.EPERM, errno.ENOTSUP, errno.ENODATA): + if e.errno not in (errno.EPERM, errno.ENOTSUP, errno.ENODATA, + errno.EINVAL): raise else: def _copyxattr(*args, **kwargs): diff --git a/Misc/NEWS.d/next/Library/2019-05-16-23-40-36.bpo-24564.lIwV_7.rst b/Misc/NEWS.d/next/Library/2019-05-16-23-40-36.bpo-24564.lIwV_7.rst new file mode 100644 index 000000000000..27cb6178b54e --- /dev/null +++ b/Misc/NEWS.d/next/Library/2019-05-16-23-40-36.bpo-24564.lIwV_7.rst @@ -0,0 +1,3 @@ +:func:`shutil.copystat` now ignores :const:`errno.EINVAL` on :func:`os.setxattr` which may occur when copying files on filesystems without extended attributes support. + +Original patch by Giampaolo Rodola, updated by Ying Wang.

1 0

[3.7] bpo-36983: Fix typing.__all__ and add test for exported names (GH-13456) (GH-13662)
by Miss Islington (bot) May 30, 2019

May 30, 2019

https://github.com/python/cpython/commit/3a98bbf7275903a0f84d1374abd0b7f3a8… commit: 3a98bbf7275903a0f84d1374abd0b7f3a85950a4 branch: 3.7 author: Anthony Sottile <asottile(a)umich.edu> committer: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> date: 2019-05-29T21:05:33-07:00 summary: [3.7] bpo-36983: Fix typing.__all__ and add test for exported names (GH-13456) (GH-13662) https://bugs.python.org/issue36983 Fixes issue 36983 files: A Misc/NEWS.d/next/Library/2019-05-20-20-41-30.bpo-36983.hz-fLr.rst M Lib/test/test_typing.py M Lib/typing.py diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index 0d66ebbd1845..ffd2007ee70d 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -2665,6 +2665,30 @@ def test_all(self): self.assertIn('SupportsBytes', a) self.assertIn('SupportsComplex', a) + def test_all_exported_names(self): + import typing + + actual_all = set(typing.__all__) + computed_all = { + k for k, v in vars(typing).items() + # explicitly exported, not a thing with __module__ + if k in actual_all or ( + # avoid private names + not k.startswith('_') and + # avoid things in the io / re typing submodules + k not in typing.io.__all__ and + k not in typing.re.__all__ and + k not in {'io', 're'} and + # there's a few types and metaclasses that aren't exported + not k.endswith(('Meta', '_contra', '_co')) and + not k.upper() == k and + # but export all things that have __module__ == 'typing' + getattr(v, '__module__', None) == typing.__name__ + ) + } + self.assertSetEqual(computed_all, actual_all) + + if __name__ == '__main__': main() diff --git a/Lib/typing.py b/Lib/typing.py index 8cf0d00bceaf..9851cb4c7ebd 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -36,6 +36,7 @@ 'Any', 'Callable', 'ClassVar', + 'ForwardRef', 'Generic', 'Optional', 'Tuple', @@ -79,11 +80,13 @@ 'SupportsRound', # Concrete collection types. + 'ChainMap', 'Counter', 'Deque', 'Dict', 'DefaultDict', 'List', + 'OrderedDict', 'Set', 'FrozenSet', 'NamedTuple', # Not really a type. diff --git a/Misc/NEWS.d/next/Library/2019-05-20-20-41-30.bpo-36983.hz-fLr.rst b/Misc/NEWS.d/next/Library/2019-05-20-20-41-30.bpo-36983.hz-fLr.rst new file mode 100644 index 000000000000..bd2d91ad9234 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2019-05-20-20-41-30.bpo-36983.hz-fLr.rst @@ -0,0 +1,2 @@ +Add missing names to ``typing.__all__``: ``ChainMap``, ``ForwardRef``, +``OrderedDict`` - by Anthony Sottile.

1 0