Mailman 3 November 2019 - Python-checkins

bpo-37523: Raise ValueError for I/O operations on a closed zipfile.ZipExtFile. (GH-14658)
by Serhiy Storchaka 30 Nov '19

30 Nov '19

https://github.com/python/cpython/commit/8d62df60d8733d0fa9aee14ef746d0009a… commit: 8d62df60d8733d0fa9aee14ef746d0009a7a9726 branch: master author: Daniel Hillier <daniel.hillier(a)gmail.com> committer: Serhiy Storchaka <storchaka(a)gmail.com> date: 2019-11-30T10:30:47+02:00 summary: bpo-37523: Raise ValueError for I/O operations on a closed zipfile.ZipExtFile. (GH-14658) Raises ValueError when calling the following on a closed zipfile.ZipExtFile: read, readable, seek, seekable, tell. files: A Misc/NEWS.d/next/Library/2019-10-02-02-55-37.bpo-37523.GguwJ6.rst M Lib/test/test_zipfile.py M Lib/zipfile.py diff --git a/Lib/test/test_zipfile.py b/Lib/test/test_zipfile.py index 1e1854be7109b..66f05ac1f3aef 100644 --- a/Lib/test/test_zipfile.py +++ b/Lib/test/test_zipfile.py @@ -571,6 +571,20 @@ def test_write_default_name(self): with open(TESTFN, "rb") as f: self.assertEqual(zipfp.read(TESTFN), f.read()) + def test_io_on_closed_zipextfile(self): + fname = "somefile.txt" + with zipfile.ZipFile(TESTFN2, mode="w") as zipfp: + zipfp.writestr(fname, "bogus") + + with zipfile.ZipFile(TESTFN2, mode="r") as zipfp: + with zipfp.open(fname) as fid: + fid.close() + self.assertRaises(ValueError, fid.read) + self.assertRaises(ValueError, fid.seek, 0) + self.assertRaises(ValueError, fid.tell) + self.assertRaises(ValueError, fid.readable) + self.assertRaises(ValueError, fid.seekable) + def test_write_to_readonly(self): """Check that trying to call write() on a readonly ZipFile object raises a ValueError.""" diff --git a/Lib/zipfile.py b/Lib/zipfile.py index b0afb9da942b1..e1d07f2a5237b 100644 --- a/Lib/zipfile.py +++ b/Lib/zipfile.py @@ -889,12 +889,16 @@ def peek(self, n=1): return self._readbuffer[self._offset: self._offset + 512] def readable(self): + if self.closed: + raise ValueError("I/O operation on closed file.") return True def read(self, n=-1): """Read and return up to n bytes. If the argument is omitted, None, or negative, data is read and returned until EOF is reached. """ + if self.closed: + raise ValueError("read from closed file.") if n is None or n < 0: buf = self._readbuffer[self._offset:] self._readbuffer = b'' @@ -1031,9 +1035,13 @@ def close(self): super().close() def seekable(self): + if self.closed: + raise ValueError("I/O operation on closed file.") return self._seekable def seek(self, offset, whence=0): + if self.closed: + raise ValueError("seek on closed file.") if not self._seekable: raise io.UnsupportedOperation("underlying stream is not seekable") curr_pos = self.tell() @@ -1082,6 +1090,8 @@ def seek(self, offset, whence=0): return self.tell() def tell(self): + if self.closed: + raise ValueError("tell on closed file.") if not self._seekable: raise io.UnsupportedOperation("underlying stream is not seekable") filepos = self._orig_file_size - self._left - len(self._readbuffer) + self._offset diff --git a/Misc/NEWS.d/next/Library/2019-10-02-02-55-37.bpo-37523.GguwJ6.rst b/Misc/NEWS.d/next/Library/2019-10-02-02-55-37.bpo-37523.GguwJ6.rst new file mode 100644 index 0000000000000..5711969ff38cf --- /dev/null +++ b/Misc/NEWS.d/next/Library/2019-10-02-02-55-37.bpo-37523.GguwJ6.rst @@ -0,0 +1 @@ +Change :class:`zipfile.ZipExtFile` to raise ``ValueError`` when trying to access the underlying file object after it has been closed. This new behavior is consistent with how accessing closed files is handled in other parts of Python. \ No newline at end of file

1 0

Fix old mention of virtualenv (GH-17417)
by Miss Islington (bot) 29 Nov '19

29 Nov '19

https://github.com/python/cpython/commit/1df65f7c6c00dfae9286c7a58e1b3803e3… commit: 1df65f7c6c00dfae9286c7a58e1b3803e3af33e5 branch: master author: Brett Cannon <54418+brettcannon(a)users.noreply.github.com> committer: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> date: 2019-11-29T15:37:08-08:00 summary: Fix old mention of virtualenv (GH-17417) Automerge-Triggered-By: @brettcannon files: M Lib/venv/scripts/posix/activate.fish diff --git a/Lib/venv/scripts/posix/activate.fish b/Lib/venv/scripts/posix/activate.fish index d213b9060a687..cb1ba1c301ede 100644 --- a/Lib/venv/scripts/posix/activate.fish +++ b/Lib/venv/scripts/posix/activate.fish @@ -1,7 +1,7 @@ # This file must be used with "source <venv>/bin/activate.fish" *from fish* # (http://fishshell.org); you cannot run it directly. -function deactivate -d "Exit virtualenv and return to normal shell environment" +function deactivate -d "Exit virtual environment and return to normal shell environment" # reset old environment variables if test -n "$_OLD_VIRTUAL_PATH" set -gx PATH $_OLD_VIRTUAL_PATH

1 0

[3.8] Added missing coma after end of list in subprocess.rst (GH-17389)
by Benjamin Peterson 29 Nov '19

29 Nov '19

https://github.com/python/cpython/commit/305189ecdc8322c22879a04564cad5989f… commit: 305189ecdc8322c22879a04564cad5989f937462 branch: 3.8 author: Jules Lasne (jlasne) <jules.lasne(a)gmail.com> committer: Benjamin Peterson <benjamin(a)python.org> date: 2019-11-28T22:47:45-06:00 summary: [3.8] Added missing coma after end of list in subprocess.rst (GH-17389) (cherry picked from commit f25875af425a3480e557aaedf49c3bb867bcbd5d) files: M Doc/library/subprocess.rst diff --git a/Doc/library/subprocess.rst b/Doc/library/subprocess.rst index 954e0fec11828..ea12cd133a6ff 100644 --- a/Doc/library/subprocess.rst +++ b/Doc/library/subprocess.rst @@ -596,7 +596,7 @@ functions. Popen and the other functions in this module that use it raise an :ref:`auditing event <auditing>` ``subprocess.Popen`` with arguments - ``executable``, ``args``, ``cwd``, ``env``. The value for ``args`` + ``executable``, ``args``, ``cwd``, and ``env``. The value for ``args`` may be a single string or a list of strings, depending on platform. .. versionchanged:: 3.2

1 0

bpo-38920: Add audit hooks for when sys.excepthook and sys.unraisablehook are invoked (GH-17392)
by Steve Dower 28 Nov '19

28 Nov '19

https://github.com/python/cpython/commit/b74a6f14b94d36fb72b1344663e81776bf… commit: b74a6f14b94d36fb72b1344663e81776bf450847 branch: 3.8 author: Steve Dower <steve.dower(a)python.org> committer: GitHub <noreply(a)github.com> date: 2019-11-28T08:46:23-08:00 summary: bpo-38920: Add audit hooks for when sys.excepthook and sys.unraisablehook are invoked (GH-17392) Also fixes some potential segfaults in unraisable hook handling. files: A Misc/NEWS.d/next/Core and Builtins/2019-11-26-09-16-47.bpo-38920.Vx__sT.rst M Doc/library/sys.rst M Lib/test/audit-tests.py M Lib/test/test_audit.py M Python/errors.c M Python/pythonrun.c M Python/sysmodule.c diff --git a/Doc/library/sys.rst b/Doc/library/sys.rst index 8d0de479e9cf5..1cf19b8ad6c02 100644 --- a/Doc/library/sys.rst +++ b/Doc/library/sys.rst @@ -36,13 +36,18 @@ always available. .. audit-event:: sys.addaudithook "" sys.addaudithook Raise an auditing event ``sys.addaudithook`` with no arguments. If any - existing hooks raise an exception derived from :class:`Exception`, the + existing hooks raise an exception derived from :class:`RuntimeError`, the new hook will not be added and the exception suppressed. As a result, callers cannot assume that their hook has been added unless they control all existing hooks. .. versionadded:: 3.8 + .. versionchanged:: 3.8.1 + + Exceptions derived from :class:`Exception` but not :class:`RuntimeError` + are no longer suppressed. + .. impl-detail:: When tracing is enabled (see :func:`settrace`), Python hooks are only @@ -308,6 +313,15 @@ always available. before the program exits. The handling of such top-level exceptions can be customized by assigning another three-argument function to ``sys.excepthook``. + .. audit-event:: sys.excepthook hook,type,value,traceback sys.excepthook + + Raise an auditing event ``sys.excepthook`` with arguments ``hook``, + ``type``, ``value``, ``traceback`` when an uncaught exception occurs. + If no hook has been set, ``hook`` may be ``None``. If any hook raises + an exception derived from :class:`RuntimeError` the call to the hook will + be suppressed. Otherwise, the audit hook exception will be reported as + unraisable and ``sys.excepthook`` will be called. + .. seealso:: The :func:`sys.unraisablehook` function handles unraisable exceptions @@ -1563,6 +1577,13 @@ always available. See also :func:`excepthook` which handles uncaught exceptions. + .. audit-event:: sys.unraisablehook hook,unraisable sys.unraisablehook + + Raise an auditing event ``sys.unraisablehook`` with arguments + ``hook``, ``unraisable`` when an exception that cannot be handled occurs. + The ``unraisable`` object is the same as what will be passed to the hook. + If no hook has been set, ``hook`` may be ``None``. + .. versionadded:: 3.8 .. data:: version diff --git a/Lib/test/audit-tests.py b/Lib/test/audit-tests.py index ddeff22030a4b..ed08612c0417f 100644 --- a/Lib/test/audit-tests.py +++ b/Lib/test/audit-tests.py @@ -263,13 +263,50 @@ def trace(frame, event, *args): def test_mmap(): import mmap + with TestHook() as hook: mmap.mmap(-1, 8) assertEqual(hook.seen[0][1][:2], (-1, 8)) +def test_excepthook(): + def excepthook(exc_type, exc_value, exc_tb): + if exc_type is not RuntimeError: + sys.__excepthook__(exc_type, exc_value, exc_tb) + + def hook(event, args): + if event == "sys.excepthook": + if not isinstance(args[2], args[1]): + raise TypeError(f"Expected isinstance({args[2]!r}, " f"{args[1]!r})") + if args[0] != excepthook: + raise ValueError(f"Expected {args[0]} == {excepthook}") + print(event, repr(args[2])) + + sys.addaudithook(hook) + sys.excepthook = excepthook + raise RuntimeError("fatal-error") + + +def test_unraisablehook(): + from _testcapi import write_unraisable_exc + + def unraisablehook(hookargs): + pass + + def hook(event, args): + if event == "sys.unraisablehook": + if args[0] != unraisablehook: + raise ValueError(f"Expected {args[0]} == {unraisablehook}") + print(event, repr(args[1].exc_value), args[1].err_msg) + + sys.addaudithook(hook) + sys.unraisablehook = unraisablehook + write_unraisable_exc(RuntimeError("nonfatal-error"), "for audit hook test", None) + + if __name__ == "__main__": from test.libregrtest.setup import suppress_msvcrt_asserts + suppress_msvcrt_asserts(False) test = sys.argv[1] diff --git a/Lib/test/test_audit.py b/Lib/test/test_audit.py index 41f9fae102231..31a08559273ee 100644 --- a/Lib/test/test_audit.py +++ b/Lib/test/test_audit.py @@ -24,7 +24,23 @@ def do_test(self, *args): sys.stdout.writelines(p.stdout) sys.stderr.writelines(p.stderr) if p.returncode: - self.fail(''.join(p.stderr)) + self.fail("".join(p.stderr)) + + def run_python(self, *args): + events = [] + with subprocess.Popen( + [sys.executable, "-X utf8", AUDIT_TESTS_PY, *args], + encoding="utf-8", + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) as p: + p.wait() + sys.stderr.writelines(p.stderr) + return ( + p.returncode, + [line.strip().partition(" ") for line in p.stdout], + "".join(p.stderr), + ) def test_basic(self): self.do_test("test_basic") @@ -36,19 +52,11 @@ def test_block_add_hook_baseexception(self): self.do_test("test_block_add_hook_baseexception") def test_finalize_hooks(self): - events = [] - with subprocess.Popen( - [sys.executable, "-X utf8", AUDIT_TESTS_PY, "test_finalize_hooks"], - encoding="utf-8", - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - ) as p: - p.wait() - for line in p.stdout: - events.append(line.strip().partition(" ")) - sys.stderr.writelines(p.stderr) - if p.returncode: - self.fail(''.join(p.stderr)) + returncode, events, stderr = self.run_python("test_finalize_hooks") + if stderr: + print(stderr, file=sys.stderr) + if returncode: + self.fail(stderr) firstId = events[0][2] self.assertSequenceEqual( @@ -76,6 +84,26 @@ def test_cantrace(self): def test_mmap(self): self.do_test("test_mmap") + def test_excepthook(self): + returncode, events, stderr = self.run_python("test_excepthook") + if not returncode: + self.fail(f"Expected fatal exception\n{stderr}") + + self.assertSequenceEqual( + [("sys.excepthook", " ", "RuntimeError('fatal-error')")], events + ) + + def test_unraisablehook(self): + returncode, events, stderr = self.run_python("test_unraisablehook") + if returncode: + self.fail(stderr) + + self.assertEqual(events[0][0], "sys.unraisablehook") + self.assertEqual( + events[0][2], + "RuntimeError('nonfatal-error') Exception ignored for audit hook test", + ) + if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS.d/next/Core and Builtins/2019-11-26-09-16-47.bpo-38920.Vx__sT.rst b/Misc/NEWS.d/next/Core and Builtins/2019-11-26-09-16-47.bpo-38920.Vx__sT.rst new file mode 100644 index 0000000000000..2e9e443dd999b --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2019-11-26-09-16-47.bpo-38920.Vx__sT.rst @@ -0,0 +1,2 @@ +Add audit hooks for when :func:`sys.excepthook` and +:func:`sys.unraisablehook` are invoked diff --git a/Python/errors.c b/Python/errors.c index 8a94afdd8c410..197d9779b390c 100644 --- a/Python/errors.c +++ b/Python/errors.c @@ -1367,44 +1367,54 @@ _PyErr_WriteUnraisableMsg(const char *err_msg_str, PyObject *obj) } } + PyObject *hook_args = make_unraisable_hook_args( + tstate, exc_type, exc_value, exc_tb, err_msg, obj); + if (hook_args == NULL) { + err_msg_str = ("Exception ignored on building " + "sys.unraisablehook arguments"); + goto error; + } + _Py_IDENTIFIER(unraisablehook); PyObject *hook = _PySys_GetObjectId(&PyId_unraisablehook); - if (hook != NULL && hook != Py_None) { - PyObject *hook_args; - - hook_args = make_unraisable_hook_args(tstate, exc_type, exc_value, - exc_tb, err_msg, obj); - if (hook_args != NULL) { - PyObject *args[1] = {hook_args}; - PyObject *res = _PyObject_FastCall(hook, args, 1); - Py_DECREF(hook_args); - if (res != NULL) { - Py_DECREF(res); - goto done; - } - - err_msg_str = "Exception ignored in sys.unraisablehook"; - } - else { - err_msg_str = ("Exception ignored on building " - "sys.unraisablehook arguments"); - } + if (hook == NULL) { + Py_DECREF(hook_args); + goto default_hook; + } - Py_XDECREF(err_msg); - err_msg = PyUnicode_FromString(err_msg_str); - if (err_msg == NULL) { - PyErr_Clear(); - } + if (PySys_Audit("sys.unraisablehook", "OO", hook, hook_args) < 0) { + Py_DECREF(hook_args); + err_msg_str = "Exception ignored in audit hook"; + obj = NULL; + goto error; + } - /* sys.unraisablehook failed: log its error using default hook */ - Py_XDECREF(exc_type); - Py_XDECREF(exc_value); - Py_XDECREF(exc_tb); - _PyErr_Fetch(tstate, &exc_type, &exc_value, &exc_tb); + if (hook == Py_None) { + Py_DECREF(hook_args); + goto default_hook; + } - obj = hook; + PyObject *args[1] = {hook_args}; + PyObject *res = _PyObject_FastCall(hook, args, 1); + Py_DECREF(hook_args); + if (res != NULL) { + Py_DECREF(res); + goto done; } + /* sys.unraisablehook failed: log its error using default hook */ + obj = hook; + err_msg_str = NULL; + +error: + /* err_msg_str and obj have been updated and we have a new exception */ + Py_XSETREF(err_msg, PyUnicode_FromString(err_msg_str ? + err_msg_str : "Exception ignored in sys.unraisablehook")); + Py_XDECREF(exc_type); + Py_XDECREF(exc_value); + Py_XDECREF(exc_tb); + _PyErr_Fetch(tstate, &exc_type, &exc_value, &exc_tb); + default_hook: /* Call the default unraisable hook (ignore failure) */ (void)write_unraisable_exc(tstate, exc_type, exc_value, exc_tb, diff --git a/Python/pythonrun.c b/Python/pythonrun.c index c9afa8f8baaf8..a7da143077a7a 100644 --- a/Python/pythonrun.c +++ b/Python/pythonrun.c @@ -695,6 +695,14 @@ _PyErr_PrintEx(PyThreadState *tstate, int set_sys_last_vars) } } hook = _PySys_GetObjectId(&PyId_excepthook); + if (PySys_Audit("sys.excepthook", "OOOO", hook ? hook : Py_None, + exception, v, tb) < 0) { + if (PyErr_ExceptionMatches(PyExc_RuntimeError)) { + PyErr_Clear(); + goto done; + } + _PyErr_WriteUnraisableMsg("in audit hook", NULL); + } if (hook) { PyObject* stack[3]; PyObject *result; diff --git a/Python/sysmodule.c b/Python/sysmodule.c index 5b0fb813b4566..1255665d024c3 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -293,8 +293,8 @@ PySys_AddAuditHook(Py_AuditHookFunction hook, void *userData) /* Cannot invoke hooks until we are initialized */ if (Py_IsInitialized()) { if (PySys_Audit("sys.addaudithook", NULL) < 0) { - if (PyErr_ExceptionMatches(PyExc_Exception)) { - /* We do not report errors derived from Exception */ + if (PyErr_ExceptionMatches(PyExc_RuntimeError)) { + /* We do not report errors derived from RuntimeError */ PyErr_Clear(); return 0; }

1 0

bpo-38920: Add audit hooks for when sys.excepthook and sys.unraisable hooks are invoked (GH-17392)
by Steve Dower 28 Nov '19

28 Nov '19

https://github.com/python/cpython/commit/bea33f5e1db6e4a554919a82894f445685… commit: bea33f5e1db6e4a554919a82894f44568576e979 branch: master author: Steve Dower <steve.dower(a)python.org> committer: GitHub <noreply(a)github.com> date: 2019-11-28T08:46:11-08:00 summary: bpo-38920: Add audit hooks for when sys.excepthook and sys.unraisable hooks are invoked (GH-17392) Also fixes some potential segfaults in unraisable hook handling. files: A Misc/NEWS.d/next/Core and Builtins/2019-11-26-09-16-47.bpo-38920.Vx__sT.rst M Doc/library/sys.rst M Lib/test/audit-tests.py M Lib/test/test_audit.py M Python/errors.c M Python/pythonrun.c M Python/sysmodule.c diff --git a/Doc/library/sys.rst b/Doc/library/sys.rst index 2f33445a6f9f5..a824fb95e8ecf 100644 --- a/Doc/library/sys.rst +++ b/Doc/library/sys.rst @@ -36,13 +36,18 @@ always available. .. audit-event:: sys.addaudithook "" sys.addaudithook Raise an auditing event ``sys.addaudithook`` with no arguments. If any - existing hooks raise an exception derived from :class:`Exception`, the + existing hooks raise an exception derived from :class:`RuntimeError`, the new hook will not be added and the exception suppressed. As a result, callers cannot assume that their hook has been added unless they control all existing hooks. .. versionadded:: 3.8 + .. versionchanged:: 3.8.1 + + Exceptions derived from :class:`Exception` but not :class:`RuntimeError` + are no longer suppressed. + .. impl-detail:: When tracing is enabled (see :func:`settrace`), Python hooks are only @@ -308,6 +313,15 @@ always available. before the program exits. The handling of such top-level exceptions can be customized by assigning another three-argument function to ``sys.excepthook``. + .. audit-event:: sys.excepthook hook,type,value,traceback sys.excepthook + + Raise an auditing event ``sys.excepthook`` with arguments ``hook``, + ``type``, ``value``, ``traceback`` when an uncaught exception occurs. + If no hook has been set, ``hook`` may be ``None``. If any hook raises + an exception derived from :class:`RuntimeError` the call to the hook will + be suppressed. Otherwise, the audit hook exception will be reported as + unraisable and ``sys.excepthook`` will be called. + .. seealso:: The :func:`sys.unraisablehook` function handles unraisable exceptions @@ -1540,6 +1554,13 @@ always available. See also :func:`excepthook` which handles uncaught exceptions. + .. audit-event:: sys.unraisablehook hook,unraisable sys.unraisablehook + + Raise an auditing event ``sys.unraisablehook`` with arguments + ``hook``, ``unraisable`` when an exception that cannot be handled occurs. + The ``unraisable`` object is the same as what will be passed to the hook. + If no hook has been set, ``hook`` may be ``None``. + .. versionadded:: 3.8 .. data:: version diff --git a/Lib/test/audit-tests.py b/Lib/test/audit-tests.py index ddeff22030a4b..ed08612c0417f 100644 --- a/Lib/test/audit-tests.py +++ b/Lib/test/audit-tests.py @@ -263,13 +263,50 @@ def trace(frame, event, *args): def test_mmap(): import mmap + with TestHook() as hook: mmap.mmap(-1, 8) assertEqual(hook.seen[0][1][:2], (-1, 8)) +def test_excepthook(): + def excepthook(exc_type, exc_value, exc_tb): + if exc_type is not RuntimeError: + sys.__excepthook__(exc_type, exc_value, exc_tb) + + def hook(event, args): + if event == "sys.excepthook": + if not isinstance(args[2], args[1]): + raise TypeError(f"Expected isinstance({args[2]!r}, " f"{args[1]!r})") + if args[0] != excepthook: + raise ValueError(f"Expected {args[0]} == {excepthook}") + print(event, repr(args[2])) + + sys.addaudithook(hook) + sys.excepthook = excepthook + raise RuntimeError("fatal-error") + + +def test_unraisablehook(): + from _testcapi import write_unraisable_exc + + def unraisablehook(hookargs): + pass + + def hook(event, args): + if event == "sys.unraisablehook": + if args[0] != unraisablehook: + raise ValueError(f"Expected {args[0]} == {unraisablehook}") + print(event, repr(args[1].exc_value), args[1].err_msg) + + sys.addaudithook(hook) + sys.unraisablehook = unraisablehook + write_unraisable_exc(RuntimeError("nonfatal-error"), "for audit hook test", None) + + if __name__ == "__main__": from test.libregrtest.setup import suppress_msvcrt_asserts + suppress_msvcrt_asserts(False) test = sys.argv[1] diff --git a/Lib/test/test_audit.py b/Lib/test/test_audit.py index 41f9fae102231..31a08559273ee 100644 --- a/Lib/test/test_audit.py +++ b/Lib/test/test_audit.py @@ -24,7 +24,23 @@ def do_test(self, *args): sys.stdout.writelines(p.stdout) sys.stderr.writelines(p.stderr) if p.returncode: - self.fail(''.join(p.stderr)) + self.fail("".join(p.stderr)) + + def run_python(self, *args): + events = [] + with subprocess.Popen( + [sys.executable, "-X utf8", AUDIT_TESTS_PY, *args], + encoding="utf-8", + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) as p: + p.wait() + sys.stderr.writelines(p.stderr) + return ( + p.returncode, + [line.strip().partition(" ") for line in p.stdout], + "".join(p.stderr), + ) def test_basic(self): self.do_test("test_basic") @@ -36,19 +52,11 @@ def test_block_add_hook_baseexception(self): self.do_test("test_block_add_hook_baseexception") def test_finalize_hooks(self): - events = [] - with subprocess.Popen( - [sys.executable, "-X utf8", AUDIT_TESTS_PY, "test_finalize_hooks"], - encoding="utf-8", - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - ) as p: - p.wait() - for line in p.stdout: - events.append(line.strip().partition(" ")) - sys.stderr.writelines(p.stderr) - if p.returncode: - self.fail(''.join(p.stderr)) + returncode, events, stderr = self.run_python("test_finalize_hooks") + if stderr: + print(stderr, file=sys.stderr) + if returncode: + self.fail(stderr) firstId = events[0][2] self.assertSequenceEqual( @@ -76,6 +84,26 @@ def test_cantrace(self): def test_mmap(self): self.do_test("test_mmap") + def test_excepthook(self): + returncode, events, stderr = self.run_python("test_excepthook") + if not returncode: + self.fail(f"Expected fatal exception\n{stderr}") + + self.assertSequenceEqual( + [("sys.excepthook", " ", "RuntimeError('fatal-error')")], events + ) + + def test_unraisablehook(self): + returncode, events, stderr = self.run_python("test_unraisablehook") + if returncode: + self.fail(stderr) + + self.assertEqual(events[0][0], "sys.unraisablehook") + self.assertEqual( + events[0][2], + "RuntimeError('nonfatal-error') Exception ignored for audit hook test", + ) + if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS.d/next/Core and Builtins/2019-11-26-09-16-47.bpo-38920.Vx__sT.rst b/Misc/NEWS.d/next/Core and Builtins/2019-11-26-09-16-47.bpo-38920.Vx__sT.rst new file mode 100644 index 0000000000000..2e9e443dd999b --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2019-11-26-09-16-47.bpo-38920.Vx__sT.rst @@ -0,0 +1,2 @@ +Add audit hooks for when :func:`sys.excepthook` and +:func:`sys.unraisablehook` are invoked diff --git a/Python/errors.c b/Python/errors.c index 1783084c336b1..d65707e7f97f5 100644 --- a/Python/errors.c +++ b/Python/errors.c @@ -1391,43 +1391,53 @@ _PyErr_WriteUnraisableMsg(const char *err_msg_str, PyObject *obj) } } + PyObject *hook_args = make_unraisable_hook_args( + tstate, exc_type, exc_value, exc_tb, err_msg, obj); + if (hook_args == NULL) { + err_msg_str = ("Exception ignored on building " + "sys.unraisablehook arguments"); + goto error; + } + _Py_IDENTIFIER(unraisablehook); PyObject *hook = _PySys_GetObjectId(&PyId_unraisablehook); - if (hook != NULL && hook != Py_None) { - PyObject *hook_args; - - hook_args = make_unraisable_hook_args(tstate, exc_type, exc_value, - exc_tb, err_msg, obj); - if (hook_args != NULL) { - PyObject *res = _PyObject_CallOneArg(hook, hook_args); - Py_DECREF(hook_args); - if (res != NULL) { - Py_DECREF(res); - goto done; - } - - err_msg_str = "Exception ignored in sys.unraisablehook"; - } - else { - err_msg_str = ("Exception ignored on building " - "sys.unraisablehook arguments"); - } + if (hook == NULL) { + Py_DECREF(hook_args); + goto default_hook; + } - Py_XDECREF(err_msg); - err_msg = PyUnicode_FromString(err_msg_str); - if (err_msg == NULL) { - PyErr_Clear(); - } + if (PySys_Audit("sys.unraisablehook", "OO", hook, hook_args) < 0) { + Py_DECREF(hook_args); + err_msg_str = "Exception ignored in audit hook"; + obj = NULL; + goto error; + } - /* sys.unraisablehook failed: log its error using default hook */ - Py_XDECREF(exc_type); - Py_XDECREF(exc_value); - Py_XDECREF(exc_tb); - _PyErr_Fetch(tstate, &exc_type, &exc_value, &exc_tb); + if (hook == Py_None) { + Py_DECREF(hook_args); + goto default_hook; + } - obj = hook; + PyObject *res = _PyObject_CallOneArg(hook, hook_args); + Py_DECREF(hook_args); + if (res != NULL) { + Py_DECREF(res); + goto done; } + /* sys.unraisablehook failed: log its error using default hook */ + obj = hook; + err_msg_str = NULL; + +error: + /* err_msg_str and obj have been updated and we have a new exception */ + Py_XSETREF(err_msg, PyUnicode_FromString(err_msg_str ? + err_msg_str : "Exception ignored in sys.unraisablehook")); + Py_XDECREF(exc_type); + Py_XDECREF(exc_value); + Py_XDECREF(exc_tb); + _PyErr_Fetch(tstate, &exc_type, &exc_value, &exc_tb); + default_hook: /* Call the default unraisable hook (ignore failure) */ (void)write_unraisable_exc(tstate, exc_type, exc_value, exc_tb, diff --git a/Python/pythonrun.c b/Python/pythonrun.c index 77a95ceb94bf0..b68a0b5572a1b 100644 --- a/Python/pythonrun.c +++ b/Python/pythonrun.c @@ -695,6 +695,14 @@ _PyErr_PrintEx(PyThreadState *tstate, int set_sys_last_vars) } } hook = _PySys_GetObjectId(&PyId_excepthook); + if (PySys_Audit("sys.excepthook", "OOOO", hook ? hook : Py_None, + exception, v, tb) < 0) { + if (PyErr_ExceptionMatches(PyExc_RuntimeError)) { + PyErr_Clear(); + goto done; + } + _PyErr_WriteUnraisableMsg("in audit hook", NULL); + } if (hook) { PyObject* stack[3]; PyObject *result; diff --git a/Python/sysmodule.c b/Python/sysmodule.c index 30c7e987461b2..78b9d22821fe3 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -323,8 +323,8 @@ PySys_AddAuditHook(Py_AuditHookFunction hook, void *userData) /* Cannot invoke hooks until we are initialized */ if (runtime->initialized) { if (PySys_Audit("sys.addaudithook", NULL) < 0) { - if (_PyErr_ExceptionMatches(tstate, PyExc_Exception)) { - /* We do not report errors derived from Exception */ + if (_PyErr_ExceptionMatches(tstate, PyExc_RuntimeError)) { + /* We do not report errors derived from RuntimeError */ _PyErr_Clear(tstate); return 0; }

1 0

bpo-38928: Remove upgrade_dependencies() from venv doc (GH-17410)
by Vinay Sajip 28 Nov '19

28 Nov '19

https://github.com/python/cpython/commit/18d8edbbb6626ac9cdf1152a720811beb2… commit: 18d8edbbb6626ac9cdf1152a720811beb2230b33 branch: 3.8 author: Tzu-ping Chung <uranusjr(a)gmail.com> committer: Vinay Sajip <vinay_sajip(a)yahoo.co.uk> date: 2019-11-28T15:44:08Z summary: bpo-38928: Remove upgrade_dependencies() from venv doc (GH-17410) files: M Doc/library/venv.rst diff --git a/Doc/library/venv.rst b/Doc/library/venv.rst index 3483ef36a2c33..31a3f41e4724c 100644 --- a/Doc/library/venv.rst +++ b/Doc/library/venv.rst @@ -186,14 +186,6 @@ creation according to their needs, the :class:`EnvBuilder` class. Installs activation scripts appropriate to the platform into the virtual environment. - .. method:: upgrade_dependencies(context) - - Upgrades the core venv dependency packages (currently ``pip`` and - ``setuptools``) in the environment. This is done by shelling out to the - ``pip`` executable in the environment. - - .. versionadded:: 3.8 - .. method:: post_setup(context) A placeholder method which can be overridden in third party

1 0

bpo-38524: clarify example a bit and improve formatting (GH-17406)
by Miss Islington (bot) 28 Nov '19

28 Nov '19

https://github.com/python/cpython/commit/c0db88f6abbace79644b2aca2290bf41b1… commit: c0db88f6abbace79644b2aca2290bf41b1a37174 branch: 3.8 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: GitHub <noreply(a)github.com> date: 2019-11-27T21:29:02-08:00 summary: bpo-38524: clarify example a bit and improve formatting (GH-17406) (cherry picked from commit 02519f75d15b063914a11351da30178ca4ceb54b) Co-authored-by: Tal Einat <taleinat+github(a)gmail.com> files: M Doc/reference/datamodel.rst diff --git a/Doc/reference/datamodel.rst b/Doc/reference/datamodel.rst index ae24adeacb167..46d50ad600ff5 100644 --- a/Doc/reference/datamodel.rst +++ b/Doc/reference/datamodel.rst @@ -1657,13 +1657,16 @@ class' :attr:`~object.__dict__`. .. note:: - ``__set_name__`` is only called implicitly as part of the ``type`` constructor, so - it will need to be called explicitly with the appropriate parameters when a - descriptor is added to a class after initial creation:: + :meth:`__set_name__` is only called implicitly as part of the + :class:`type` constructor, so it will need to be called explicitly with + the appropriate parameters when a descriptor is added to a class after + initial creation:: + class A: + pass descr = custom_descriptor() - cls.attr = descr - descr.__set_name__(cls, 'attr') + A.attr = descr + descr.__set_name__(A, 'attr') See :ref:`class-object-creation` for more details.

1 0

bpo-38524: clarify example a bit and improve formatting (GH-17406)
by Miss Islington (bot) 28 Nov '19

28 Nov '19

https://github.com/python/cpython/commit/7e9bbbe51e74e5928e6a6c3e70434d8249… commit: 7e9bbbe51e74e5928e6a6c3e70434d824970ef58 branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: GitHub <noreply(a)github.com> date: 2019-11-27T21:28:37-08:00 summary: bpo-38524: clarify example a bit and improve formatting (GH-17406) (cherry picked from commit 02519f75d15b063914a11351da30178ca4ceb54b) Co-authored-by: Tal Einat <taleinat+github(a)gmail.com> files: M Doc/reference/datamodel.rst diff --git a/Doc/reference/datamodel.rst b/Doc/reference/datamodel.rst index 7a99e1e5220bb..b4f9ddc1194cb 100644 --- a/Doc/reference/datamodel.rst +++ b/Doc/reference/datamodel.rst @@ -1643,13 +1643,16 @@ class' :attr:`~object.__dict__`. .. note:: - ``__set_name__`` is only called implicitly as part of the ``type`` constructor, so - it will need to be called explicitly with the appropriate parameters when a - descriptor is added to a class after initial creation:: + :meth:`__set_name__` is only called implicitly as part of the + :class:`type` constructor, so it will need to be called explicitly with + the appropriate parameters when a descriptor is added to a class after + initial creation:: + class A: + pass descr = custom_descriptor() - cls.attr = descr - descr.__set_name__(cls, 'attr') + A.attr = descr + descr.__set_name__(A, 'attr') See :ref:`class-object-creation` for more details.

1 0

bpo-26730: Fix SpooledTemporaryFile data corruption (GH-17400)
by Inada Naoki 28 Nov '19

28 Nov '19

https://github.com/python/cpython/commit/e65b3fa9f16537d20f5f37c25673ac899f… commit: e65b3fa9f16537d20f5f37c25673ac899fcd7099 branch: 3.7 author: Inada Naoki <songofacandy(a)gmail.com> committer: GitHub <noreply(a)github.com> date: 2019-11-28T14:23:58+09:00 summary: bpo-26730: Fix SpooledTemporaryFile data corruption (GH-17400) SpooledTemporaryFile.rollback() might cause data corruption when it is in text mode. Co-Authored-By: Serhiy Storchaka <storchaka(a)gmail.com>. (cherry picked from commit ea9835c5d154ab6a54eed627958473b6768b28cc) files: A Misc/NEWS.d/next/Library/2019-11-27-16-30-02.bpo-26730.56cdBn.rst M Doc/library/tempfile.rst M Lib/tempfile.py M Lib/test/test_tempfile.py diff --git a/Doc/library/tempfile.rst b/Doc/library/tempfile.rst index dd24a1c6f4ffd..00acf4b179237 100644 --- a/Doc/library/tempfile.rst +++ b/Doc/library/tempfile.rst @@ -95,8 +95,8 @@ The module defines the following user-callable items: causes the file to roll over to an on-disk file regardless of its size. The returned object is a file-like object whose :attr:`_file` attribute - is either an :class:`io.BytesIO` or :class:`io.StringIO` object (depending on - whether binary or text *mode* was specified) or a true file + is either an :class:`io.BytesIO` or :class:`io.TextIOWrapper` object + (depending on whether binary or text *mode* was specified) or a true file object, depending on whether :func:`rollover` has been called. This file-like object can be used in a :keyword:`with` statement, just like a normal file. diff --git a/Lib/tempfile.py b/Lib/tempfile.py index 214322416963c..24f673c64aa8d 100644 --- a/Lib/tempfile.py +++ b/Lib/tempfile.py @@ -637,10 +637,8 @@ def __init__(self, max_size=0, mode='w+b', buffering=-1, if 'b' in mode: self._file = _io.BytesIO() else: - # Setting newline="\n" avoids newline translation; - # this is important because otherwise on Windows we'd - # get double newline translation upon rollover(). - self._file = _io.StringIO(newline="\n") + self._file = _io.TextIOWrapper(_io.BytesIO(), + encoding=encoding, newline=newline) self._max_size = max_size self._rolled = False self._TemporaryFileArgs = {'mode': mode, 'buffering': buffering, @@ -660,8 +658,12 @@ def rollover(self): newfile = self._file = TemporaryFile(**self._TemporaryFileArgs) del self._TemporaryFileArgs - newfile.write(file.getvalue()) - newfile.seek(file.tell(), 0) + pos = file.tell() + if hasattr(newfile, 'buffer'): + newfile.buffer.write(file.detach().getvalue()) + else: + newfile.write(file.getvalue()) + newfile.seek(pos, 0) self._rolled = True diff --git a/Lib/test/test_tempfile.py b/Lib/test/test_tempfile.py index 931312831616c..c0464200a3aa3 100644 --- a/Lib/test/test_tempfile.py +++ b/Lib/test/test_tempfile.py @@ -1119,7 +1119,8 @@ def test_properties(self): def test_text_mode(self): # Creating a SpooledTemporaryFile with a text mode should produce # a file object reading and writing (Unicode) text strings. - f = tempfile.SpooledTemporaryFile(mode='w+', max_size=10) + f = tempfile.SpooledTemporaryFile(mode='w+', max_size=10, + encoding="utf-8") f.write("abc\n") f.seek(0) self.assertEqual(f.read(), "abc\n") @@ -1129,8 +1130,8 @@ def test_text_mode(self): self.assertFalse(f._rolled) self.assertEqual(f.mode, 'w+') self.assertIsNone(f.name) - self.assertIsNone(f.newlines) - self.assertIsNone(f.encoding) + self.assertEqual(f.newlines, os.linesep) + self.assertEqual(f.encoding, "utf-8") f.write("xyzzy\n") f.seek(0) @@ -1143,7 +1144,7 @@ def test_text_mode(self): self.assertEqual(f.mode, 'w+') self.assertIsNotNone(f.name) self.assertEqual(f.newlines, os.linesep) - self.assertIsNotNone(f.encoding) + self.assertEqual(f.encoding, "utf-8") def test_text_newline_and_encoding(self): f = tempfile.SpooledTemporaryFile(mode='w+', max_size=10, @@ -1154,12 +1155,14 @@ def test_text_newline_and_encoding(self): self.assertFalse(f._rolled) self.assertEqual(f.mode, 'w+') self.assertIsNone(f.name) - self.assertIsNone(f.newlines) - self.assertIsNone(f.encoding) + self.assertIsNotNone(f.newlines) + self.assertEqual(f.encoding, "utf-8") - f.write("\u039B" * 20 + "\r\n") + f.write("\u039C" * 10 + "\r\n") + f.write("\u039D" * 20) f.seek(0) - self.assertEqual(f.read(), "\u039B\r\n" + ("\u039B" * 20) + "\r\n") + self.assertEqual(f.read(), + "\u039B\r\n" + ("\u039C" * 10) + "\r\n" + ("\u039D" * 20)) self.assertTrue(f._rolled) self.assertEqual(f.mode, 'w+') self.assertIsNotNone(f.name) diff --git a/Misc/NEWS.d/next/Library/2019-11-27-16-30-02.bpo-26730.56cdBn.rst b/Misc/NEWS.d/next/Library/2019-11-27-16-30-02.bpo-26730.56cdBn.rst new file mode 100644 index 0000000000000..a92b90a495605 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2019-11-27-16-30-02.bpo-26730.56cdBn.rst @@ -0,0 +1,2 @@ +Fix ``SpooledTemporaryFile.rollover()`` might corrupt the file when it is in +text mode. Patch by Serhiy Storchaka.

1 0

bpo-26730: Fix SpooledTemporaryFile data corruption (GH-17400)
by Miss Islington (bot) 28 Nov '19

28 Nov '19

https://github.com/python/cpython/commit/d21b8e82dda3caf0989bb39bc0e0ce2bfe… commit: d21b8e82dda3caf0989bb39bc0e0ce2bfef97081 branch: 3.8 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: GitHub <noreply(a)github.com> date: 2019-11-27T21:23:14-08:00 summary: bpo-26730: Fix SpooledTemporaryFile data corruption (GH-17400) SpooledTemporaryFile.rollback() might cause data corruption when it is in text mode. Co-Authored-By: Serhiy Storchaka <storchaka(a)gmail.com> (cherry picked from commit ea9835c5d154ab6a54eed627958473b6768b28cc) Co-authored-by: Inada Naoki <songofacandy(a)gmail.com> files: A Misc/NEWS.d/next/Library/2019-11-27-16-30-02.bpo-26730.56cdBn.rst M Doc/library/tempfile.rst M Lib/tempfile.py M Lib/test/test_tempfile.py diff --git a/Doc/library/tempfile.rst b/Doc/library/tempfile.rst index fff7a7a03eb81..a59817c103921 100644 --- a/Doc/library/tempfile.rst +++ b/Doc/library/tempfile.rst @@ -105,8 +105,8 @@ The module defines the following user-callable items: causes the file to roll over to an on-disk file regardless of its size. The returned object is a file-like object whose :attr:`_file` attribute - is either an :class:`io.BytesIO` or :class:`io.StringIO` object (depending on - whether binary or text *mode* was specified) or a true file + is either an :class:`io.BytesIO` or :class:`io.TextIOWrapper` object + (depending on whether binary or text *mode* was specified) or a true file object, depending on whether :func:`rollover` has been called. This file-like object can be used in a :keyword:`with` statement, just like a normal file. diff --git a/Lib/tempfile.py b/Lib/tempfile.py index 45709cb06cf4e..62875540f8b92 100644 --- a/Lib/tempfile.py +++ b/Lib/tempfile.py @@ -633,10 +633,9 @@ def __init__(self, max_size=0, mode='w+b', buffering=-1, if 'b' in mode: self._file = _io.BytesIO() else: - # Setting newline="\n" avoids newline translation; - # this is important because otherwise on Windows we'd - # get double newline translation upon rollover(). - self._file = _io.StringIO(newline="\n") + self._file = _io.TextIOWrapper(_io.BytesIO(), + encoding=encoding, errors=errors, + newline=newline) self._max_size = max_size self._rolled = False self._TemporaryFileArgs = {'mode': mode, 'buffering': buffering, @@ -656,8 +655,12 @@ def rollover(self): newfile = self._file = TemporaryFile(**self._TemporaryFileArgs) del self._TemporaryFileArgs - newfile.write(file.getvalue()) - newfile.seek(file.tell(), 0) + pos = file.tell() + if hasattr(newfile, 'buffer'): + newfile.buffer.write(file.detach().getvalue()) + else: + newfile.write(file.getvalue()) + newfile.seek(pos, 0) self._rolled = True diff --git a/Lib/test/test_tempfile.py b/Lib/test/test_tempfile.py index f995f6c9bfaf0..232c5dae10fdf 100644 --- a/Lib/test/test_tempfile.py +++ b/Lib/test/test_tempfile.py @@ -1114,7 +1114,8 @@ def test_properties(self): def test_text_mode(self): # Creating a SpooledTemporaryFile with a text mode should produce # a file object reading and writing (Unicode) text strings. - f = tempfile.SpooledTemporaryFile(mode='w+', max_size=10) + f = tempfile.SpooledTemporaryFile(mode='w+', max_size=10, + encoding="utf-8") f.write("abc\n") f.seek(0) self.assertEqual(f.read(), "abc\n") @@ -1124,9 +1125,9 @@ def test_text_mode(self): self.assertFalse(f._rolled) self.assertEqual(f.mode, 'w+') self.assertIsNone(f.name) - self.assertIsNone(f.newlines) - self.assertIsNone(f.encoding) - self.assertIsNone(f.errors) + self.assertEqual(f.newlines, os.linesep) + self.assertEqual(f.encoding, "utf-8") + self.assertEqual(f.errors, "strict") f.write("xyzzy\n") f.seek(0) @@ -1139,8 +1140,8 @@ def test_text_mode(self): self.assertEqual(f.mode, 'w+') self.assertIsNotNone(f.name) self.assertEqual(f.newlines, os.linesep) - self.assertIsNotNone(f.encoding) - self.assertIsNotNone(f.errors) + self.assertEqual(f.encoding, "utf-8") + self.assertEqual(f.errors, "strict") def test_text_newline_and_encoding(self): f = tempfile.SpooledTemporaryFile(mode='w+', max_size=10, @@ -1152,13 +1153,15 @@ def test_text_newline_and_encoding(self): self.assertFalse(f._rolled) self.assertEqual(f.mode, 'w+') self.assertIsNone(f.name) - self.assertIsNone(f.newlines) - self.assertIsNone(f.encoding) - self.assertIsNone(f.errors) + self.assertIsNotNone(f.newlines) + self.assertEqual(f.encoding, "utf-8") + self.assertEqual(f.errors, "ignore") - f.write("\u039B" * 20 + "\r\n") + f.write("\u039C" * 10 + "\r\n") + f.write("\u039D" * 20) f.seek(0) - self.assertEqual(f.read(), "\u039B\r\n" + ("\u039B" * 20) + "\r\n") + self.assertEqual(f.read(), + "\u039B\r\n" + ("\u039C" * 10) + "\r\n" + ("\u039D" * 20)) self.assertTrue(f._rolled) self.assertEqual(f.mode, 'w+') self.assertIsNotNone(f.name) diff --git a/Misc/NEWS.d/next/Library/2019-11-27-16-30-02.bpo-26730.56cdBn.rst b/Misc/NEWS.d/next/Library/2019-11-27-16-30-02.bpo-26730.56cdBn.rst new file mode 100644 index 0000000000000..a92b90a495605 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2019-11-27-16-30-02.bpo-26730.56cdBn.rst @@ -0,0 +1,2 @@ +Fix ``SpooledTemporaryFile.rollover()`` might corrupt the file when it is in +text mode. Patch by Serhiy Storchaka.

1 0