Mailman 3 November 2017 - Python-checkins

bpo-28416: Break reference cycles in Pickler and Unpickler subclasses (#4080)
by Serhiy Storchaka Nov. 30, 2017

Nov. 30, 2017

https://github.com/python/cpython/commit/986375ebde0dd5ff2b7349e445a06bd28a… commit: 986375ebde0dd5ff2b7349e445a06bd28a3a8ee2 branch: master author: Serhiy Storchaka <storchaka(a)gmail.com> committer: GitHub <noreply(a)github.com> date: 2017-11-30T22:48:31+02:00 summary: bpo-28416: Break reference cycles in Pickler and Unpickler subclasses (#4080) with the persistent_id() and persistent_load() methods. files: A Misc/NEWS.d/next/Library/2017-10-23-12-05-33.bpo-28416.Ldnw8X.rst M Lib/test/test_pickle.py M Modules/_pickle.c diff --git a/Lib/test/test_pickle.py b/Lib/test/test_pickle.py index ee71c632733..895ed48df1d 100644 --- a/Lib/test/test_pickle.py +++ b/Lib/test/test_pickle.py @@ -6,6 +6,7 @@ import collections import struct import sys +import weakref import unittest from test import support @@ -117,6 +118,66 @@ class PyIdPersPicklerTests(AbstractIdentityPersistentPicklerTests, pickler = pickle._Pickler unpickler = pickle._Unpickler + @support.cpython_only + def test_pickler_reference_cycle(self): + def check(Pickler): + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + f = io.BytesIO() + pickler = Pickler(f, proto) + pickler.dump('abc') + self.assertEqual(self.loads(f.getvalue()), 'abc') + pickler = Pickler(io.BytesIO()) + self.assertEqual(pickler.persistent_id('def'), 'def') + r = weakref.ref(pickler) + del pickler + self.assertIsNone(r()) + + class PersPickler(self.pickler): + def persistent_id(subself, obj): + return obj + check(PersPickler) + + class PersPickler(self.pickler): + @classmethod + def persistent_id(cls, obj): + return obj + check(PersPickler) + + class PersPickler(self.pickler): + @staticmethod + def persistent_id(obj): + return obj + check(PersPickler) + + @support.cpython_only + def test_unpickler_reference_cycle(self): + def check(Unpickler): + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + unpickler = Unpickler(io.BytesIO(self.dumps('abc', proto))) + self.assertEqual(unpickler.load(), 'abc') + unpickler = Unpickler(io.BytesIO()) + self.assertEqual(unpickler.persistent_load('def'), 'def') + r = weakref.ref(unpickler) + del unpickler + self.assertIsNone(r()) + + class PersUnpickler(self.unpickler): + def persistent_load(subself, pid): + return pid + check(PersUnpickler) + + class PersUnpickler(self.unpickler): + @classmethod + def persistent_load(cls, pid): + return pid + check(PersUnpickler) + + class PersUnpickler(self.unpickler): + @staticmethod + def persistent_load(pid): + return pid + check(PersUnpickler) + class PyPicklerUnpicklerObjectTests(AbstractPicklerUnpicklerObjectTests): @@ -197,7 +258,7 @@ class SizeofTests(unittest.TestCase): check_sizeof = support.check_sizeof def test_pickler(self): - basesize = support.calcobjsize('5P2n3i2n3iP') + basesize = support.calcobjsize('6P2n3i2n3iP') p = _pickle.Pickler(io.BytesIO()) self.assertEqual(object.__sizeof__(p), basesize) MT_size = struct.calcsize('3nP0n') @@ -214,7 +275,7 @@ def test_pickler(self): 0) # Write buffer is cleared after every dump(). def test_unpickler(self): - basesize = support.calcobjsize('2Pn2P 2P2n2i5P 2P3n6P2n2i') + basesize = support.calcobjsize('2P2n2P 2P2n2i5P 2P3n6P2n2i') unpickler = _pickle.Unpickler P = struct.calcsize('P') # Size of memo table entry. n = struct.calcsize('n') # Size of mark table entry. diff --git a/Misc/NEWS.d/next/Library/2017-10-23-12-05-33.bpo-28416.Ldnw8X.rst b/Misc/NEWS.d/next/Library/2017-10-23-12-05-33.bpo-28416.Ldnw8X.rst new file mode 100644 index 00000000000..b1014827af6 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2017-10-23-12-05-33.bpo-28416.Ldnw8X.rst @@ -0,0 +1,3 @@ +Instances of pickle.Pickler subclass with the persistent_id() method and +pickle.Unpickler subclass with the persistent_load() method no longer create +reference cycles. diff --git a/Modules/_pickle.c b/Modules/_pickle.c index 943c70112b7..da915efd831 100644 --- a/Modules/_pickle.c +++ b/Modules/_pickle.c @@ -360,6 +360,69 @@ _Pickle_FastCall(PyObject *func, PyObject *obj) /*************************************************************************/ +/* Retrieve and deconstruct a method for avoiding a reference cycle + (pickler -> bound method of pickler -> pickler) */ +static int +init_method_ref(PyObject *self, _Py_Identifier *name, + PyObject **method_func, PyObject **method_self) +{ + PyObject *func, *func2; + + /* *method_func and *method_self should be consistent. All refcount decrements + should be occurred after setting *method_self and *method_func. */ + func = _PyObject_GetAttrId(self, name); + if (func == NULL) { + *method_self = NULL; + Py_CLEAR(*method_func); + if (!PyErr_ExceptionMatches(PyExc_AttributeError)) { + return -1; + } + PyErr_Clear(); + return 0; + } + + if (PyMethod_Check(func) && PyMethod_GET_SELF(func) == self) { + /* Deconstruct a bound Python method */ + func2 = PyMethod_GET_FUNCTION(func); + Py_INCREF(func2); + *method_self = self; /* borrowed */ + Py_XSETREF(*method_func, func2); + Py_DECREF(func); + return 0; + } + else { + *method_self = NULL; + Py_XSETREF(*method_func, func); + return 0; + } +} + +/* Bind a method if it was deconstructed */ +static PyObject * +reconstruct_method(PyObject *func, PyObject *self) +{ + if (self) { + return PyMethod_New(func, self); + } + else { + Py_INCREF(func); + return func; + } +} + +static PyObject * +call_method(PyObject *func, PyObject *self, PyObject *obj) +{ + if (self) { + return PyObject_CallFunctionObjArgs(func, self, obj, NULL); + } + else { + return PyObject_CallFunctionObjArgs(func, obj, NULL); + } +} + +/*************************************************************************/ + /* Internal data type used as the unpickling stack. */ typedef struct { PyObject_VAR_HEAD @@ -552,6 +615,8 @@ typedef struct PicklerObject { objects to support self-referential objects pickling. */ PyObject *pers_func; /* persistent_id() method, can be NULL */ + PyObject *pers_func_self; /* borrowed reference to self if pers_func + is an unbound method, NULL otherwise */ PyObject *dispatch_table; /* private dispatch_table, can be NULL */ PyObject *write; /* write() method of the output stream. */ @@ -590,6 +655,8 @@ typedef struct UnpicklerObject { Py_ssize_t memo_len; /* Number of objects in the memo */ PyObject *pers_func; /* persistent_load() method, can be NULL. */ + PyObject *pers_func_self; /* borrowed reference to self if pers_func + is an unbound method, NULL otherwise */ Py_buffer buffer; char *input_buffer; @@ -3444,7 +3511,7 @@ save_type(PicklerObject *self, PyObject *obj) } static int -save_pers(PicklerObject *self, PyObject *obj, PyObject *func) +save_pers(PicklerObject *self, PyObject *obj) { PyObject *pid = NULL; int status = 0; @@ -3452,8 +3519,7 @@ save_pers(PicklerObject *self, PyObject *obj, PyObject *func) const char persid_op = PERSID; const char binpersid_op = BINPERSID; - Py_INCREF(obj); - pid = _Pickle_FastCall(func, obj); + pid = call_method(self->pers_func, self->pers_func_self, obj); if (pid == NULL) return -1; @@ -3831,7 +3897,7 @@ save(PicklerObject *self, PyObject *obj, int pers_save) 0 if it did nothing successfully; 1 if a persistent id was saved. */ - if ((status = save_pers(self, obj, self->pers_func)) != 0) + if ((status = save_pers(self, obj)) != 0) goto done; } @@ -4246,13 +4312,10 @@ _pickle_Pickler___init___impl(PicklerObject *self, PyObject *file, self->fast_nesting = 0; self->fast_memo = NULL; - self->pers_func = _PyObject_GetAttrId((PyObject *)self, - &PyId_persistent_id); - if (self->pers_func == NULL) { - if (!PyErr_ExceptionMatches(PyExc_AttributeError)) { - return -1; - } - PyErr_Clear(); + if (init_method_ref((PyObject *)self, &PyId_persistent_id, + &self->pers_func, &self->pers_func_self) < 0) + { + return -1; } self->dispatch_table = _PyObject_GetAttrId((PyObject *)self, @@ -4519,11 +4582,11 @@ Pickler_set_memo(PicklerObject *self, PyObject *obj) static PyObject * Pickler_get_persid(PicklerObject *self) { - if (self->pers_func == NULL) + if (self->pers_func == NULL) { PyErr_SetString(PyExc_AttributeError, "persistent_id"); - else - Py_INCREF(self->pers_func); - return self->pers_func; + return NULL; + } + return reconstruct_method(self->pers_func, self->pers_func_self); } static int @@ -4540,6 +4603,7 @@ Pickler_set_persid(PicklerObject *self, PyObject *value) return -1; } + self->pers_func_self = NULL; Py_INCREF(value); Py_XSETREF(self->pers_func, value); @@ -5489,7 +5553,7 @@ load_stack_global(UnpicklerObject *self) static int load_persid(UnpicklerObject *self) { - PyObject *pid; + PyObject *pid, *obj; Py_ssize_t len; char *s; @@ -5509,13 +5573,12 @@ load_persid(UnpicklerObject *self) return -1; } - /* This does not leak since _Pickle_FastCall() steals the reference - to pid first. */ - pid = _Pickle_FastCall(self->pers_func, pid); - if (pid == NULL) + obj = call_method(self->pers_func, self->pers_func_self, pid); + Py_DECREF(pid); + if (obj == NULL) return -1; - PDATA_PUSH(self->stack, pid, -1); + PDATA_PUSH(self->stack, obj, -1); return 0; } else { @@ -5530,20 +5593,19 @@ load_persid(UnpicklerObject *self) static int load_binpersid(UnpicklerObject *self) { - PyObject *pid; + PyObject *pid, *obj; if (self->pers_func) { PDATA_POP(self->stack, pid); if (pid == NULL) return -1; - /* This does not leak since _Pickle_FastCall() steals the - reference to pid first. */ - pid = _Pickle_FastCall(self->pers_func, pid); - if (pid == NULL) + obj = call_method(self->pers_func, self->pers_func_self, pid); + Py_DECREF(pid); + if (obj == NULL) return -1; - PDATA_PUSH(self->stack, pid, -1); + PDATA_PUSH(self->stack, obj, -1); return 0; } else { @@ -6690,13 +6752,10 @@ _pickle_Unpickler___init___impl(UnpicklerObject *self, PyObject *file, self->fix_imports = fix_imports; - self->pers_func = _PyObject_GetAttrId((PyObject *)self, - &PyId_persistent_load); - if (self->pers_func == NULL) { - if (!PyErr_ExceptionMatches(PyExc_AttributeError)) { - return -1; - } - PyErr_Clear(); + if (init_method_ref((PyObject *)self, &PyId_persistent_load, + &self->pers_func, &self->pers_func_self) < 0) + { + return -1; } self->stack = (Pdata *)Pdata_New(); @@ -6983,11 +7042,11 @@ Unpickler_set_memo(UnpicklerObject *self, PyObject *obj) static PyObject * Unpickler_get_persload(UnpicklerObject *self) { - if (self->pers_func == NULL) + if (self->pers_func == NULL) { PyErr_SetString(PyExc_AttributeError, "persistent_load"); - else - Py_INCREF(self->pers_func); - return self->pers_func; + return NULL; + } + return reconstruct_method(self->pers_func, self->pers_func_self); } static int @@ -7005,6 +7064,7 @@ Unpickler_set_persload(UnpicklerObject *self, PyObject *value) return -1; } + self->pers_func_self = NULL; Py_INCREF(value); Py_XSETREF(self->pers_func, value);

1 0

bpo-32030: Fix _Py_InitializeEx_Private() (#4649)
by Victor Stinner Nov. 30, 2017

Nov. 30, 2017

https://github.com/python/cpython/commit/bc8ac6b00efcb3c601224b73f64071281f… commit: bc8ac6b00efcb3c601224b73f64071281f434bcd branch: master author: Victor Stinner <victor.stinner(a)gmail.com> committer: GitHub <noreply(a)github.com> date: 2017-11-30T18:03:55+01:00 summary: bpo-32030: Fix _Py_InitializeEx_Private() (#4649) _Py_InitializeEx_Private() now calls _PyMainInterpreterConfig_ReadEnv() to read environment variables PYTHONHOME and PYTHONPATH, and set the program name. files: M Python/pylifecycle.c diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 01f314e4bd1..f0a49f91fb8 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -961,28 +961,35 @@ _Py_InitializeEx_Private(int install_sigs, int install_importlib) _PyMainInterpreterConfig config = _PyMainInterpreterConfig_INIT; _PyInitError err; - /* TODO: Moar config options! */ core_config.ignore_environment = Py_IgnoreEnvironmentFlag; core_config._disable_importlib = !install_importlib; config.install_signal_handlers = install_sigs; err = _Py_InitializeCore(&core_config); if (_Py_INIT_FAILED(err)) { - return err; + goto done; + } + + err = _PyMainInterpreterConfig_ReadEnv(&config); + if (_Py_INIT_FAILED(err)) { + goto done; } - /* TODO: Print any exceptions raised by these operations */ err = _PyMainInterpreterConfig_Read(&config); if (_Py_INIT_FAILED(err)) { - return err; + goto done; } err = _Py_InitializeMainInterpreter(&config); if (_Py_INIT_FAILED(err)) { - return err; + goto done; } - return _Py_INIT_OK(); + err = _Py_INIT_OK(); + +done: + _PyMainInterpreterConfig_Clear(&config); + return err; }

1 0

Fix CID-1414686: PyInit_readline() handles errors (#4647)
by Victor Stinner Nov. 30, 2017

Nov. 30, 2017

https://github.com/python/cpython/commit/0efc0249ca1fd0675098957407fbba2c0f… commit: 0efc0249ca1fd0675098957407fbba2c0f6949ce branch: master author: Victor Stinner <victor.stinner(a)gmail.com> committer: GitHub <noreply(a)github.com> date: 2017-11-30T17:21:07+01:00 summary: Fix CID-1414686: PyInit_readline() handles errors (#4647) Handle PyModule_AddIntConstant() and PyModule_AddStringConstant() failures. Add also constants before calling setup_readline(), since setup_readline() registers callbacks which uses a reference to the module, whereas the module is destroyed if adding constants fails. Fix Coverity warning: CID 1414686: Unchecked return value (CHECKED_RETURN) 2. check_return: Calling PyModule_AddStringConstant without checking return value (as is done elsewhere 45 out of 55 times). files: M Modules/readline.c diff --git a/Modules/readline.c b/Modules/readline.c index d0e3b913e57..811fca8cd92 100644 --- a/Modules/readline.c +++ b/Modules/readline.c @@ -1352,13 +1352,27 @@ PyInit_readline(void) if (m == NULL) return NULL; + if (PyModule_AddIntConstant(m, "_READLINE_VERSION", + RL_READLINE_VERSION) < 0) { + goto error; + } + if (PyModule_AddIntConstant(m, "_READLINE_RUNTIME_VERSION", + rl_readline_version) < 0) { + goto error; + } + if (PyModule_AddStringConstant(m, "_READLINE_LIBRARY_VERSION", + rl_library_version) < 0) + { + goto error; + } + mod_state = (readlinestate *) PyModule_GetState(m); PyOS_ReadlineFunctionPointer = call_readline; setup_readline(mod_state); - PyModule_AddIntConstant(m, "_READLINE_VERSION", RL_READLINE_VERSION); - PyModule_AddIntConstant(m, "_READLINE_RUNTIME_VERSION", rl_readline_version); - PyModule_AddStringConstant(m, "_READLINE_LIBRARY_VERSION", rl_library_version); - return m; + +error: + Py_DECREF(m); + return NULL; }

1 0

Skip test_socket.test_sha256() on linux < 4.5 (GH-4643) (#4645)
by Victor Stinner Nov. 30, 2017

Nov. 30, 2017

https://github.com/python/cpython/commit/92a2c07b71aefc01f84ba4b0eda8e2a45c… commit: 92a2c07b71aefc01f84ba4b0eda8e2a45c1a6b65 branch: 3.6 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: Victor Stinner <victor.stinner(a)gmail.com> date: 2017-11-30T14:43:43+01:00 summary: Skip test_socket.test_sha256() on linux < 4.5 (GH-4643) (#4645) bpo-31705. (cherry picked from commit 86afc1f2a7fb3afe00779c6426bc141bc795d9a3) files: A Misc/NEWS.d/next/Tests/2017-11-30-12-27-10.bpo-31705.yULW7O.rst M Lib/test/test_socket.py diff --git a/Lib/test/test_socket.py b/Lib/test/test_socket.py index 034a3da43f8..a79ebfe9a34 100644 --- a/Lib/test/test_socket.py +++ b/Lib/test/test_socket.py @@ -5419,6 +5419,9 @@ def create_alg(self, typ, name): else: return sock + # bpo-31705: On kernel older than 4.5, sendto() failed with ENOKEY, + # at least on ppc64le architecture + @support.requires_linux_version(4, 5) def test_sha256(self): expected = bytes.fromhex("ba7816bf8f01cfea414140de5dae2223b00361a396" "177a9cb410ff61f20015ad") diff --git a/Misc/NEWS.d/next/Tests/2017-11-30-12-27-10.bpo-31705.yULW7O.rst b/Misc/NEWS.d/next/Tests/2017-11-30-12-27-10.bpo-31705.yULW7O.rst new file mode 100644 index 00000000000..aa2d30c3d6b --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2017-11-30-12-27-10.bpo-31705.yULW7O.rst @@ -0,0 +1,3 @@ +Skip test_socket.test_sha256() on Linux kernel older than 4.5. The test +fails with ENOKEY on kernel 3.10 (on ppc64le). A fix was merged into the +kernel 4.5.

1 0

Skip test_socket.test_sha256() on linux < 4.5 (#4643)
by Victor Stinner Nov. 30, 2017

Nov. 30, 2017

https://github.com/python/cpython/commit/86afc1f2a7fb3afe00779c6426bc141bc7… commit: 86afc1f2a7fb3afe00779c6426bc141bc795d9a3 branch: master author: Victor Stinner <victor.stinner(a)gmail.com> committer: GitHub <noreply(a)github.com> date: 2017-11-30T13:58:43+01:00 summary: Skip test_socket.test_sha256() on linux < 4.5 (#4643) bpo-31705. files: A Misc/NEWS.d/next/Tests/2017-11-30-12-27-10.bpo-31705.yULW7O.rst M Lib/test/test_socket.py diff --git a/Lib/test/test_socket.py b/Lib/test/test_socket.py index e70a8f69a7a..b97a7688a60 100644 --- a/Lib/test/test_socket.py +++ b/Lib/test/test_socket.py @@ -5573,6 +5573,9 @@ def create_alg(self, typ, name): else: return sock + # bpo-31705: On kernel older than 4.5, sendto() failed with ENOKEY, + # at least on ppc64le architecture + @support.requires_linux_version(4, 5) def test_sha256(self): expected = bytes.fromhex("ba7816bf8f01cfea414140de5dae2223b00361a396" "177a9cb410ff61f20015ad") diff --git a/Misc/NEWS.d/next/Tests/2017-11-30-12-27-10.bpo-31705.yULW7O.rst b/Misc/NEWS.d/next/Tests/2017-11-30-12-27-10.bpo-31705.yULW7O.rst new file mode 100644 index 00000000000..aa2d30c3d6b --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2017-11-30-12-27-10.bpo-31705.yULW7O.rst @@ -0,0 +1,3 @@ +Skip test_socket.test_sha256() on Linux kernel older than 4.5. The test +fails with ENOKEY on kernel 3.10 (on ppc64le). A fix was merged into the +kernel 4.5.

1 0

bpo-32101: Add PYTHONDEVMODE environment variable (#4624)
by Victor Stinner Nov. 30, 2017

Nov. 30, 2017

https://github.com/python/cpython/commit/5e3806f8cfd84722fc55d4299dc018ad9b… commit: 5e3806f8cfd84722fc55d4299dc018ad9b0f8401 branch: master author: Victor Stinner <victor.stinner(a)gmail.com> committer: GitHub <noreply(a)github.com> date: 2017-11-30T11:40:24+01:00 summary: bpo-32101: Add PYTHONDEVMODE environment variable (#4624) * bpo-32101: Add sys.flags.dev_mode flag Rename also the "Developer mode" to the "Development mode". * bpo-32101: Add PYTHONDEVMODE environment variable Mention it in the development chapiter. files: A Misc/NEWS.d/next/Library/2017-11-29-00-42-47.bpo-321010.-axD5l.rst M Doc/library/development.rst M Doc/library/sys.rst M Doc/using/cmdline.rst M Doc/whatsnew/3.7.rst M Lib/asyncio/coroutines.py M Lib/test/test_cmd_line.py M Lib/test/test_sys.py M Modules/main.c M Python/sysmodule.c diff --git a/Doc/library/development.rst b/Doc/library/development.rst index d2b5fa2aa4f..ab34e1f7ce5 100644 --- a/Doc/library/development.rst +++ b/Doc/library/development.rst @@ -24,3 +24,6 @@ The list of modules described in this chapter is: unittest.mock-examples.rst 2to3.rst test.rst + +See also the Python development mode: the :option:`-X` ``dev`` option and +:envvar:`PYTHONDEVMODE` environment variable. diff --git a/Doc/library/sys.rst b/Doc/library/sys.rst index faf540c4ea4..9e47681804b 100644 --- a/Doc/library/sys.rst +++ b/Doc/library/sys.rst @@ -334,6 +334,7 @@ always available. :const:`bytes_warning` :option:`-b` :const:`quiet` :option:`-q` :const:`hash_randomization` :option:`-R` + :const:`dev_mode` :option:`-X` ``dev`` ============================= ============================= .. versionchanged:: 3.2 @@ -345,6 +346,9 @@ always available. .. versionchanged:: 3.3 Removed obsolete ``division_warning`` attribute. + .. versionchanged:: 3.7 + Added ``dev_mode`` attribute for the new :option:`-X` ``dev`` flag. + .. data:: float_info diff --git a/Doc/using/cmdline.rst b/Doc/using/cmdline.rst index e6189fd8127..d110ae3baf2 100644 --- a/Doc/using/cmdline.rst +++ b/Doc/using/cmdline.rst @@ -411,7 +411,7 @@ Miscellaneous options nested imports). Note that its output may be broken in multi-threaded application. Typical usage is ``python3 -X importtime -c 'import asyncio'``. See also :envvar:`PYTHONPROFILEIMPORTTIME`. - * ``-X dev``: enable CPython's "developer mode", introducing additional + * ``-X dev``: enable CPython's "development mode", introducing additional runtime checks which are too expensive to be enabled by default. It should not be more verbose than the default if the code is correct: new warnings are only emitted when an issue is detected. Effect of the developer mode: @@ -426,6 +426,8 @@ Miscellaneous options * Enable the :mod:`faulthandler` module to dump the Python traceback on a crash. * Enable :ref:`asyncio debug mode <asyncio-debug-mode>`. + * Set the :attr:`~sys.flags.dev_mode` attribute of :attr:`sys.flags` to + ``True`` It also allows passing arbitrary values and retrieving them through the :data:`sys._xoptions` dictionary. @@ -796,6 +798,14 @@ conflict. .. versionadded:: 3.7 See :pep:`538` for more details. + +.. envvar:: PYTHONDEVMODE + + If this environment variable is set to a non-empty string, enable the + CPython "development mode". See the :option:`-X` ``dev`` option. + + .. versionadded:: 3.7 + Debug-mode variables ~~~~~~~~~~~~~~~~~~~~ diff --git a/Doc/whatsnew/3.7.rst b/Doc/whatsnew/3.7.rst index 5c001594dc8..6545a18a991 100644 --- a/Doc/whatsnew/3.7.rst +++ b/Doc/whatsnew/3.7.rst @@ -185,10 +185,10 @@ resolution on Linux and Windows. PEP written and implemented by Victor Stinner -New Developer Mode: -X dev --------------------------- +New Development Mode: -X dev +---------------------------- -Add a new "developer mode": ``-X dev`` command line option to enable debug +Add a new "development mode": ``-X dev`` command line option to enable debug checks at runtime. In short, ``python3 -X dev ...`` behaves as ``PYTHONMALLOC=debug python3 -W @@ -371,6 +371,11 @@ string expression pattern for braced placeholders and non-braced placeholders separately. (Contributed by Barry Warsaw in :issue:`1198569`.) +sys +--- + +Added :attr:`sys.flags.dev_mode` flag for the new development mode. + time ---- diff --git a/Lib/asyncio/coroutines.py b/Lib/asyncio/coroutines.py index b6f81a4b0f5..7d2ca05d2a0 100644 --- a/Lib/asyncio/coroutines.py +++ b/Lib/asyncio/coroutines.py @@ -27,11 +27,9 @@ def _is_debug_mode(): # before you define your coroutines. A downside of using this feature # is that tracebacks show entries for the CoroWrapper.__next__ method # when _DEBUG is true. - debug = (not sys.flags.ignore_environment and - bool(os.environ.get('PYTHONASYNCIODEBUG'))) - if hasattr(sys, '_xoptions') and 'dev' in sys._xoptions: - debug = True - return debug + return (sys.flags.dev_mode + or (not sys.flags.ignore_environment + and bool(os.environ.get('PYTHONASYNCIODEBUG')))) _DEBUG = _is_debug_mode() diff --git a/Lib/test/test_cmd_line.py b/Lib/test/test_cmd_line.py index 96405e70afc..383302bad8b 100644 --- a/Lib/test/test_cmd_line.py +++ b/Lib/test/test_cmd_line.py @@ -508,14 +508,18 @@ def test_sys_flags_set(self): with self.subTest(envar_value=value): assert_python_ok('-c', code, **env_vars) - def run_xdev(self, *args, check_exitcode=True): + def run_xdev(self, *args, check_exitcode=True, xdev=True): env = dict(os.environ) env.pop('PYTHONWARNINGS', None) + env.pop('PYTHONDEVMODE', None) # Force malloc() to disable the debug hooks which are enabled # by default for Python compiled in debug mode env['PYTHONMALLOC'] = 'malloc' - args = (sys.executable, '-X', 'dev', *args) + if xdev: + args = (sys.executable, '-X', 'dev', *args) + else: + args = (sys.executable, *args) proc = subprocess.run(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, @@ -526,6 +530,14 @@ def run_xdev(self, *args, check_exitcode=True): return proc.stdout.rstrip() def test_xdev(self): + # sys.flags.dev_mode + code = "import sys; print(sys.flags.dev_mode)" + out = self.run_xdev("-c", code, xdev=False) + self.assertEqual(out, "False") + out = self.run_xdev("-c", code) + self.assertEqual(out, "True") + + # Warnings code = ("import sys, warnings; " "print(' '.join('%s::%s' % (f[0], f[2].__name__) " "for f in warnings.filters))") @@ -555,6 +567,7 @@ def test_xdev(self): "default::ResourceWarning " "default::Warning") + # Memory allocator debug hooks try: import _testcapi except ImportError: @@ -569,6 +582,7 @@ def test_xdev(self): alloc_name = "malloc_debug" self.assertEqual(out, alloc_name) + # Faulthandler try: import faulthandler except ImportError: @@ -581,6 +595,7 @@ def test_xdev(self): def check_pythonmalloc(self, env_var, name): code = 'import _testcapi; print(_testcapi.pymem_getallocatorsname())' env = dict(os.environ) + env.pop('PYTHONDEVMODE', None) if env_var is not None: env['PYTHONMALLOC'] = env_var else: @@ -621,6 +636,24 @@ def test_pythonmalloc(self): with self.subTest(env_var=env_var, name=name): self.check_pythonmalloc(env_var, name) + def test_pythondevmode_env(self): + # Test the PYTHONDEVMODE environment variable + code = "import sys; print(sys.flags.dev_mode)" + env = dict(os.environ) + env.pop('PYTHONDEVMODE', None) + args = (sys.executable, '-c', code) + + proc = subprocess.run(args, stdout=subprocess.PIPE, + universal_newlines=True, env=env) + self.assertEqual(proc.stdout.rstrip(), 'False') + self.assertEqual(proc.returncode, 0, proc) + + env['PYTHONDEVMODE'] = '1' + proc = subprocess.run(args, stdout=subprocess.PIPE, + universal_newlines=True, env=env) + self.assertEqual(proc.stdout.rstrip(), 'True') + self.assertEqual(proc.returncode, 0, proc) + class IgnoreEnvironmentTest(unittest.TestCase): diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index 4b8fcb9540f..6346094ad08 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -526,10 +526,12 @@ def test_sys_flags(self): attrs = ("debug", "inspect", "interactive", "optimize", "dont_write_bytecode", "no_user_site", "no_site", "ignore_environment", "verbose", - "bytes_warning", "quiet", "hash_randomization", "isolated") + "bytes_warning", "quiet", "hash_randomization", "isolated", + "dev_mode") for attr in attrs: self.assertTrue(hasattr(sys.flags, attr), attr) - self.assertEqual(type(getattr(sys.flags, attr)), int, attr) + attr_type = bool if attr == "dev_mode" else int + self.assertEqual(type(getattr(sys.flags, attr)), attr_type, attr) self.assertTrue(repr(sys.flags)) self.assertEqual(len(sys.flags), len(attrs)) diff --git a/Misc/NEWS.d/next/Library/2017-11-29-00-42-47.bpo-321010.-axD5l.rst b/Misc/NEWS.d/next/Library/2017-11-29-00-42-47.bpo-321010.-axD5l.rst new file mode 100644 index 00000000000..715a269a5c0 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2017-11-29-00-42-47.bpo-321010.-axD5l.rst @@ -0,0 +1 @@ +Add :attr:`sys.flags.dev_mode` flag diff --git a/Modules/main.c b/Modules/main.c index ec33b5f086f..e9d524a1463 100644 --- a/Modules/main.c +++ b/Modules/main.c @@ -124,7 +124,8 @@ static const char usage_6[] = " hooks.\n" "PYTHONCOERCECLOCALE: if this variable is set to 0, it disables the locale\n" " coercion behavior. Use PYTHONCOERCECLOCALE=warn to request display of\n" -" locale coercion and locale compatibility warnings on stderr.\n"; +" locale coercion and locale compatibility warnings on stderr.\n" +"PYTHONDEVMODE: enable the development mode.\n"; static void pymain_usage(int error, const wchar_t* program) @@ -1520,7 +1521,9 @@ pymain_parse_envvars(_PyMain *pymain) if (pymain_init_tracemalloc(pymain) < 0) { return -1; } - if (pymain_get_xoption(pymain, L"dev")) { + if (pymain_get_xoption(pymain, L"dev" ) || + pymain_get_env_var("PYTHONDEVMODE")) + { core_config->dev_mode = 1; core_config->faulthandler = 1; core_config->allocator = "debug"; diff --git a/Python/sysmodule.c b/Python/sysmodule.c index d786739188a..64bc14e9c3d 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -1814,6 +1814,7 @@ static PyStructSequence_Field flags_fields[] = { {"quiet", "-q"}, {"hash_randomization", "-R"}, {"isolated", "-I"}, + {"dev_mode", "-X dev"}, {0} }; @@ -1821,7 +1822,7 @@ static PyStructSequence_Desc flags_desc = { "sys.flags", /* name */ flags__doc__, /* doc */ flags_fields, /* fields */ - 13 + 14 }; static PyObject* @@ -1829,6 +1830,7 @@ make_flags(void) { int pos = 0; PyObject *seq; + _PyCoreConfig *core_config = &_PyGILState_GetInterpreterStateUnsafe()->core_config; seq = PyStructSequence_New(&FlagsType); if (seq == NULL) @@ -1853,6 +1855,7 @@ make_flags(void) SetFlag(Py_HashRandomizationFlag); SetFlag(Py_IsolatedFlag); #undef SetFlag + PyStructSequence_SET_ITEM(seq, pos++, PyBool_FromLong(core_config->dev_mode)); if (PyErr_Occurred()) { Py_DECREF(seq);

1 0

Daily reference leaks (4243df51fe43): sum=3
by solipsis＠pitrou.net Nov. 30, 2017

Nov. 30, 2017

results for 4243df51fe43 on branch "default" -------------------------------------------- test_collections leaked [0, 7, -7] memory blocks, sum=0 test_functools leaked [0, 3, 1] memory blocks, sum=4 test_multiprocessing_fork leaked [-2, 2, -1] memory blocks, sum=-1 Command line was: ['./python', '-m', 'test.regrtest', '-uall', '-R', '3:3:/home/psf-users/antoine/refleaks/refloglrVXpw', '--timeout', '7200']

1 0

[65 flat] Results for Python (master branch) 2017-11-29
by lp_benchmark_robot＠intel.com Nov. 30, 2017

Nov. 30, 2017

Results for project python/master, build date: 2017-11-29 03:02:53-08:00. - commit: ef83806 - previous commit: a10dc3e - revision date: 2017-11-29 11:43:23+01:00 - environment: Broadwell-EP - cpu: Intel(R) Xeon(R) CPU E5-2699 v4 @ 2.20GHz 2x22 cores, stepping 1, LLC 55 MB - mem: 128 GB - os: Ubuntu 16.04.2 LTS - kernel: 4.4.0-62-generic x86_64 GNU/Linux Baseline results were generated using release v3.6.0, with hash 5c4568a from 2016-12-22 23:38:47+00:00. +-----+------------------------+--------+------------+------------+------------+ | | |relative|change since|change since|current rev | | | benchmark|std_dev*| last run | baseline |run with PGO| +-----+------------------------+--------+------------+------------+------------+ | :-| | 2to3| 0.874% | +0.310% | +5.488% | +7.090% | +-----+------------------------+--------+------------+------------+------------+ | :-| | call_method| 0.969% | -0.155% | +20.128% | +12.704% | +-----+------------------------+--------+------------+------------+------------+ | :-| | call_method_slots| 1.797% | -0.410% | +21.356% | +12.788% | +-----+------------------------+--------+------------+------------+------------+ | :-| | call_method_unknown| 0.693% | +0.127% | +18.742% | +11.975% | +-----+------------------------+--------+------------+------------+------------+ | :-| | call_simple| 3.464% | -0.169% | +6.768% | +10.279% | +-----+------------------------+--------+------------+------------+------------+ | :-| | chameleon| 1.762% | -0.017% | +10.501% | +10.843% | +-----+------------------------+--------+------------+------------+------------+ | :-| | chaos| 1.228% | +0.521% | +7.343% | +10.310% | +-----+------------------------+--------+------------+------------+------------+ | :-| | crypto_pyaes| 0.885% | +0.076% | +1.234% | +7.349% | +-----+------------------------+--------+------------+------------+------------+ | :-| | deltablue| 4.033% | +0.325% | +8.204% | +17.276% | +-----+------------------------+--------+------------+------------+------------+ | :-| | django_template| 2.902% | +1.187% | +11.856% | +14.267% | +-----+------------------------+--------+------------+------------+------------+ | :-| | dulwich_log| 1.275% | +0.166% | +4.167% | +6.179% | +-----+------------------------+--------+------------+------------+------------+ | :-| | fannkuch| 0.256% | +0.019% | +5.865% | +5.557% | +-----+------------------------+--------+------------+------------+------------+ | :-| | float| 0.915% | -0.378% | +2.713% | +6.567% | +-----+------------------------+--------+------------+------------+------------+ | :-| | genshi_text| 1.475% | +0.287% | +13.168% | +9.585% | +-----+------------------------+--------+------------+------------+------------+ | :-| | genshi_xml| 1.823% | -0.546% | +8.767% | +10.195% | +-----+------------------------+--------+------------+------------+------------+ | :-| | go| 0.912% | +0.054% | +4.648% | +11.832% | +-----+------------------------+--------+------------+------------+------------+ | :-| | hexiom| 0.556% | -0.024% | +11.017% | +10.800% | +-----+------------------------+--------+------------+------------+------------+ | :-| | html5lib| 2.953% | +0.259% | +8.778% | +8.456% | +-----+------------------------+--------+------------+------------+------------+ | :-| | json_dumps| 2.018% | +0.051% | +2.211% | +10.367% | +-----+------------------------+--------+------------+------------+------------+ | :-| | json_loads| 3.878% | -0.640% | -0.203% | +13.406% | +-----+------------------------+--------+------------+------------+------------+ | :-| | logging_format| 1.817% | +0.010% | +7.460% | +11.326% | +-----+------------------------+--------+------------+------------+------------+ | :-| | logging_silent| 2.908% | -0.463% | +46.928% | +12.664% | +-----+------------------------+--------+------------+------------+------------+ | :-| | logging_simple| 1.549% | -0.163% | +6.873% | +14.291% | +-----+------------------------+--------+------------+------------+------------+ | :-| | mako| 0.749% | -0.157% | +16.831% | +13.899% | +-----+------------------------+--------+------------+------------+------------+ | :-| | mdp| 2.022% | +0.698% | +8.747% | +8.600% | +-----+------------------------+--------+------------+------------+------------+ | :-| | meteor_contest| 3.941% | +0.620% | +6.376% | +3.135% | +-----+------------------------+--------+------------+------------+------------+ | :-| | nbody| 1.426% | +0.166% | +0.162% | +3.367% | +-----+------------------------+--------+------------+------------+------------+ | :-| | nqueens| 0.925% | -0.054% | +5.829% | +8.133% | +-----+------------------------+--------+------------+------------+------------+ | :-| | pathlib| 1.329% | -1.214% | +3.634% | +9.281% | +-----+------------------------+--------+------------+------------+------------+ | :-| | pickle| 1.143% | -0.213% | -0.259% | +22.434% | +-----+------------------------+--------+------------+------------+------------+ | :-| | pickle_dict| 0.243% | +0.474% | -2.496% | +22.908% | +-----+------------------------+--------+------------+------------+------------+ | :-| | pickle_list| 0.678% | -0.280% | +0.988% | +23.616% | +-----+------------------------+--------+------------+------------+------------+ | :-| | pickle_pure_python| 5.593% | -1.441% | +10.542% | +10.111% | +-----+------------------------+--------+------------+------------+------------+ | :-| | pidigits| 0.205% | +0.033% | +0.451% | +9.716% | +-----+------------------------+--------+------------+------------+------------+ | :-| | python_startup| 0.136% | -0.063% | +10.647% | +5.024% | +-----+------------------------+--------+------------+------------+------------+ | :-| | python_startup_no_site| 0.108% | -0.069% | +1.885% | +5.068% | +-----+------------------------+--------+------------+------------+------------+ | :-| | raytrace| 1.081% | +0.088% | +10.923% | +12.540% | +-----+------------------------+--------+------------+------------+------------+ | :-| | regex_compile| 5.243% | -0.294% | -1.645% | +11.654% | +-----+------------------------+--------+------------+------------+------------+ | :-| | regex_dna| 0.353% | +0.420% | +0.843% | +10.763% | +-----+------------------------+--------+------------+------------+------------+ | :-| | regex_effbot| 2.662% | +0.336% | -5.443% | +8.680% | +-----+------------------------+--------+------------+------------+------------+ | :-| | regex_v8| 2.454% | -0.773% | +5.254% | +10.387% | +-----+------------------------+--------+------------+------------+------------+ | :-| | richards| 1.423% | +0.123% | +8.472% | +13.785% | +-----+------------------------+--------+------------+------------+------------+ | :-| | scimark_fft| 2.447% | -0.089% | -1.771% | +4.875% | +-----+------------------------+--------+------------+------------+------------+ | :-| | scimark_lu| 1.606% | +0.293% | +26.977% | +8.386% | +-----+------------------------+--------+------------+------------+------------+ | :-| | scimark_monte_carlo| 1.956% | -0.203% | +4.487% | +3.809% | +-----+------------------------+--------+------------+------------+------------+ | :-| | scimark_sor| 0.559% | -0.137% | +13.474% | +8.650% | +-----+------------------------+--------+------------+------------+------------+ | :-| | scimark_sparse_mat_mult| 0.541% | -0.227% | +1.311% | +1.056% | +-----+------------------------+--------+------------+------------+------------+ | :-| | spectral_norm| 0.445% | +0.067% | +4.992% | +3.275% | +-----+------------------------+--------+------------+------------+------------+ | :-| | sqlalchemy_declarative| 1.126% | +0.074% | +6.849% | +6.359% | +-----+------------------------+--------+------------+------------+------------+ | :-| | sqlalchemy_imperative| 3.464% | +0.628% | +6.133% | +4.113% | +-----+------------------------+--------+------------+------------+------------+ | :-| | sqlite_synth| 3.734% | +0.916% | +19.015% | +8.878% | +-----+------------------------+--------+------------+------------+------------+ | :-| | sympy_expand| 2.480% | +0.129% | +11.078% | +8.123% | +-----+------------------------+--------+------------+------------+------------+ | :-| | sympy_integrate| 1.677% | +0.379% | +9.630% | +6.278% | +-----+------------------------+--------+------------+------------+------------+ | :-| | sympy_str| 3.248% | +0.259% | +11.656% | +8.371% | +-----+------------------------+--------+------------+------------+------------+ | :-| | sympy_sum| 4.714% | +0.138% | +10.085% | +9.857% | +-----+------------------------+--------+------------+------------+------------+ | :-| | telco| 3.274% | +0.481% | +21.590% | +12.419% | +-----+------------------------+--------+------------+------------+------------+ | :-| | tornado_http| 1.187% | -0.048% | +5.737% | +4.379% | +-----+------------------------+--------+------------+------------+------------+ | :-| | unpack_sequence| 1.139% | -0.676% | +1.880% | -0.558% | +-----+------------------------+--------+------------+------------+------------+ | :-| | unpickle| 5.306% | -0.140% | +8.501% | +20.051% | +-----+------------------------+--------+------------+------------+------------+ | :-| | unpickle_list| 0.625% | +0.868% | +1.667% | +15.848% | +-----+------------------------+--------+------------+------------+------------+ | :-| | unpickle_pure_python| 3.054% | +0.000% | +6.932% | +8.643% | +-----+------------------------+--------+------------+------------+------------+ | :-| | xml_etree_generate| 1.456% | -0.003% | +4.707% | +10.526% | +-----+------------------------+--------+------------+------------+------------+ | :-| | xml_etree_iterparse| 2.753% | +0.372% | +2.333% | +7.322% | +-----+------------------------+--------+------------+------------+------------+ | :-| | xml_etree_parse| 2.594% | +0.261% | -7.379% | +13.602% | +-----+------------------------+--------+------------+------------+------------+ | :-| | xml_etree_process| 1.235% | +0.044% | +6.544% | +9.640% | +-----+------------------------+--------+------------+------------+------------+ * Relative Standard Deviation (Standard Deviation/Average) If this is not displayed properly please visit our results page here: http://languagesperformance.intel.com/65-flat-results-for-python-master-bra… Our lab does a nightly source pull and build of the Python project and measures performance changes against the previous stable version and the previous nightly measurement. This is provided as a service to the community so that quality issues with current hardware can be identified quickly. Intel technologies' features and benefits depend on system configuration and may require enabled hardware, software or service activation. Performance varies depending on system configuration.

1 0

bpo-32121: Add most_recent_first parameter to tracemalloc.Traceback.format (#4534)
by Victor Stinner Nov. 29, 2017

Nov. 29, 2017

https://github.com/python/cpython/commit/706e10b186992e086e661a62d2c8ec9588… commit: 706e10b186992e086e661a62d2c8ec9588525b31 branch: master author: Jesse-Bakker <jessebakker00(a)gmail.com> committer: Victor Stinner <victor.stinner(a)gmail.com> date: 2017-11-30T00:05:07+01:00 summary: bpo-32121: Add most_recent_first parameter to tracemalloc.Traceback.format (#4534) * Add most_recent_first parameter to tracemalloc.Traceback.format to allow reversing the order of the frames in the output * Reversed default sorting of tracemalloc.Traceback frames * Allowed negative limit, truncating from the other side. files: A Misc/NEWS.d/next/Library/2017-11-24-00-59-12.bpo-32121.ePbmwC.rst M Doc/library/tracemalloc.rst M Doc/whatsnew/3.7.rst M Lib/test/test_tracemalloc.py M Lib/test/test_warnings/__init__.py M Lib/tracemalloc.py M Lib/warnings.py diff --git a/Doc/library/tracemalloc.rst b/Doc/library/tracemalloc.rst index 048ee64aac9..2d327c02540 100644 --- a/Doc/library/tracemalloc.rst +++ b/Doc/library/tracemalloc.rst @@ -650,8 +650,8 @@ Traceback .. class:: Traceback - Sequence of :class:`Frame` instances sorted from the most recent frame to - the oldest frame. + Sequence of :class:`Frame` instances sorted from the oldest frame to the + most recent frame. A traceback contains at least ``1`` frame. If the ``tracemalloc`` module failed to get a frame, the filename ``"<unknown>"`` at line number ``0`` is @@ -663,11 +663,17 @@ Traceback The :attr:`Trace.traceback` attribute is an instance of :class:`Traceback` instance. - .. method:: format(limit=None) + .. versionchanged:: 3.7 + Frames are now sorted from the oldest to the most recent, instead of most recent to oldest. - Format the traceback as a list of lines with newlines. Use the - :mod:`linecache` module to retrieve lines from the source code. If - *limit* is set, only format the *limit* most recent frames. + .. method:: format(limit=None, most_recent_first=False) + + Format the traceback as a list of lines with newlines. Use the + :mod:`linecache` module to retrieve lines from the source code. + If *limit* is set, format the *limit* most recent frames if *limit* + is positive. Otherwise, format the ``abs(limit)`` oldest frames. + If *most_recent_first* is ``True``, the order of the formatted frames + is reversed, returning the most recent frame first instead of last. Similar to the :func:`traceback.format_tb` function, except that :meth:`.format` does not include newlines. diff --git a/Doc/whatsnew/3.7.rst b/Doc/whatsnew/3.7.rst index e4600fe5ffd..5c001594dc8 100644 --- a/Doc/whatsnew/3.7.rst +++ b/Doc/whatsnew/3.7.rst @@ -751,6 +751,10 @@ Changes in the Python API avoid a warning escape them with a backslash. (Contributed by Serhiy Storchaka in :issue:`30349`.) +* :class:`tracemalloc.Traceback` frames are now sorted from oldest to most + recent to be more consistent with :mod:`traceback`. + (Contributed by Jesse Bakker in :issue:`32121`.) + .. _Unicode Technical Standard #18: https://unicode.org/reports/tr18/ diff --git a/Lib/test/test_tracemalloc.py b/Lib/test/test_tracemalloc.py index 533ba6dc460..b0a0e1b2d78 100644 --- a/Lib/test/test_tracemalloc.py +++ b/Lib/test/test_tracemalloc.py @@ -171,6 +171,9 @@ def allocate_bytes4(size): traces = tracemalloc._get_traces() + obj1_traceback._frames = tuple(reversed(obj1_traceback._frames)) + obj2_traceback._frames = tuple(reversed(obj2_traceback._frames)) + trace1 = self.find_trace(traces, obj1_traceback) trace2 = self.find_trace(traces, obj2_traceback) domain1, size1, traceback1 = trace1 @@ -537,11 +540,11 @@ def test_snapshot_group_by_cumulative(self): def test_trace_format(self): snapshot, snapshot2 = create_snapshots() trace = snapshot.traces[0] - self.assertEqual(str(trace), 'a.py:2: 10 B') + self.assertEqual(str(trace), 'b.py:4: 10 B') traceback = trace.traceback - self.assertEqual(str(traceback), 'a.py:2') + self.assertEqual(str(traceback), 'b.py:4') frame = traceback[0] - self.assertEqual(str(frame), 'a.py:2') + self.assertEqual(str(frame), 'b.py:4') def test_statistic_format(self): snapshot, snapshot2 = create_snapshots() @@ -574,17 +577,32 @@ def getline(filename, lineno): side_effect=getline): tb = snapshot.traces[0].traceback self.assertEqual(tb.format(), + [' File "b.py", line 4', + ' <b.py, 4>', + ' File "a.py", line 2', + ' <a.py, 2>']) + + self.assertEqual(tb.format(limit=1), + [' File "a.py", line 2', + ' <a.py, 2>']) + + self.assertEqual(tb.format(limit=-1), + [' File "b.py", line 4', + ' <b.py, 4>']) + + self.assertEqual(tb.format(most_recent_first=True), [' File "a.py", line 2', ' <a.py, 2>', ' File "b.py", line 4', ' <b.py, 4>']) - self.assertEqual(tb.format(limit=1), + self.assertEqual(tb.format(limit=1, most_recent_first=True), [' File "a.py", line 2', ' <a.py, 2>']) - self.assertEqual(tb.format(limit=-1), - []) + self.assertEqual(tb.format(limit=-1, most_recent_first=True), + [' File "b.py", line 4', + ' <b.py, 4>']) class TestFilters(unittest.TestCase): diff --git a/Lib/test/test_warnings/__init__.py b/Lib/test/test_warnings/__init__.py index f2fdaa5386b..e60bc4d27f6 100644 --- a/Lib/test/test_warnings/__init__.py +++ b/Lib/test/test_warnings/__init__.py @@ -941,11 +941,11 @@ def func(): expected = textwrap.dedent(''' {fname}:5: ResourceWarning: unclosed file <...> f = None - Object allocated at (most recent call first): - File "{fname}", lineno 3 - f = open(__file__) + Object allocated at (most recent call last): File "{fname}", lineno 7 func() + File "{fname}", lineno 3 + f = open(__file__) ''') expected = expected.format(fname=support.TESTFN).strip() self.assertEqual(stderr, expected) diff --git a/Lib/tracemalloc.py b/Lib/tracemalloc.py index 597a2978afe..2c1ac3b39b0 100644 --- a/Lib/tracemalloc.py +++ b/Lib/tracemalloc.py @@ -171,16 +171,18 @@ def __repr__(self): @total_ordering class Traceback(Sequence): """ - Sequence of Frame instances sorted from the most recent frame - to the oldest frame. + Sequence of Frame instances sorted from the oldest frame + to the most recent frame. """ __slots__ = ("_frames",) def __init__(self, frames): Sequence.__init__(self) # frames is a tuple of frame tuples: see Frame constructor for the - # format of a frame tuple - self._frames = frames + # format of a frame tuple; it is reversed, because _tracemalloc + # returns frames sorted from most recent to oldest, but the + # Python API expects oldest to most recent + self._frames = tuple(reversed(frames)) def __len__(self): return len(self._frames) @@ -209,11 +211,19 @@ def __str__(self): def __repr__(self): return "<Traceback %r>" % (tuple(self),) - def format(self, limit=None): + def format(self, limit=None, most_recent_first=False): lines = [] - if limit is not None and limit < 0: - return lines - for frame in self[:limit]: + if limit is not None: + if limit > 0: + frame_slice = self[-limit:] + else: + frame_slice = self[:limit] + else: + frame_slice = self + + if most_recent_first: + frame_slice = reversed(frame_slice) + for frame in frame_slice: lines.append(' File "%s", line %s' % (frame.filename, frame.lineno)) line = linecache.getline(frame.filename, frame.lineno).strip() diff --git a/Lib/warnings.py b/Lib/warnings.py index d7ea057a688..4e7241fe6ca 100644 --- a/Lib/warnings.py +++ b/Lib/warnings.py @@ -62,7 +62,7 @@ def _formatwarnmsg_impl(msg): tb = None if tb is not None: - s += 'Object allocated at (most recent call first):\n' + s += 'Object allocated at (most recent call last):\n' for frame in tb: s += (' File "%s", lineno %s\n' % (frame.filename, frame.lineno)) diff --git a/Misc/NEWS.d/next/Library/2017-11-24-00-59-12.bpo-32121.ePbmwC.rst b/Misc/NEWS.d/next/Library/2017-11-24-00-59-12.bpo-32121.ePbmwC.rst new file mode 100644 index 00000000000..7701c86aa47 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2017-11-24-00-59-12.bpo-32121.ePbmwC.rst @@ -0,0 +1,6 @@ +Made ``tracemalloc.Traceback`` behave more like the traceback module, +sorting the frames from oldest to most recent. ``Traceback.format()`` +now accepts negative *limit*, truncating the result to the ``abs(limit)`` +oldest frames. To get the old behaviour, one can use the new +*most_recent_first* argument to ``Traceback.format()``. +(Patch by Jesse Bakker.)

1 0

[3.6] bpo-30923, bpo-31279: Fix GCC warnings (#4620)
by Victor Stinner Nov. 29, 2017

Nov. 29, 2017

https://github.com/python/cpython/commit/dedcbee04cd52790027ecfb46cb3aa33ef… commit: dedcbee04cd52790027ecfb46cb3aa33efebdc84 branch: 3.6 author: Victor Stinner <victor.stinner(a)gmail.com> committer: GitHub <noreply(a)github.com> date: 2017-11-30T00:00:35+01:00 summary: [3.6] bpo-30923, bpo-31279: Fix GCC warnings (#4620) * bpo-30923: Silence fall-through warnings in libexpat build. (#3205) (cherry picked from commit 9e1e6f528f3fec16b9bd99f5ee38048ffec04a81) * bpo-31279: Silence -Wstringop-overflow warning. (#3207) (cherry picked from commit dce6502059f46a04f90938b9d832394c8215397b) files: M Objects/bytearrayobject.c M setup.py diff --git a/Objects/bytearrayobject.c b/Objects/bytearrayobject.c index 7653322b829..15524572929 100644 --- a/Objects/bytearrayobject.c +++ b/Objects/bytearrayobject.c @@ -244,7 +244,7 @@ PyByteArray_Resize(PyObject *self, Py_ssize_t requested_size) return -1; } memcpy(sval, PyByteArray_AS_STRING(self), - Py_MIN(requested_size, Py_SIZE(self))); + Py_MIN((size_t)requested_size, (size_t)Py_SIZE(self))); PyObject_Free(obj->ob_bytes); } else { diff --git a/setup.py b/setup.py index da4fcc31651..0d644db8573 100644 --- a/setup.py +++ b/setup.py @@ -1526,6 +1526,7 @@ class db_found(Exception): pass if '--with-system-expat' in sysconfig.get_config_var("CONFIG_ARGS"): expat_inc = [] define_macros = [] + extra_compile_args = [] expat_lib = ['expat'] expat_sources = [] expat_depends = [] @@ -1537,6 +1538,7 @@ class db_found(Exception): pass # call XML_SetHashSalt(), expat entropy sources are not needed ('XML_POOR_ENTROPY', '1'), ] + extra_compile_args = [] expat_lib = [] expat_sources = ['expat/xmlparse.c', 'expat/xmlrole.c', @@ -1554,8 +1556,15 @@ class db_found(Exception): pass 'expat/xmltok_impl.h' ] + cc = sysconfig.get_config_var('CC').split()[0] + ret = os.system( + '"%s" -Werror -Wimplicit-fallthrough -E -xc /dev/null >/dev/null 2>&1' % cc) + if ret >> 8 == 0: + extra_compile_args.append('-Wno-implicit-fallthrough') + exts.append(Extension('pyexpat', define_macros = define_macros, + extra_compile_args = extra_compile_args, include_dirs = expat_inc, libraries = expat_lib, sources = ['pyexpat.c'] + expat_sources,

1 0