Mailman 3 November 2015 - Python-checkins

cpython (2.7): Issue #25718: Fixed copying object with state with boolean value is false.
by serhiy.storchaka Nov. 30, 2015

Nov. 30, 2015

https://hg.python.org/cpython/rev/3d39e1e2dcb3 changeset: 99393:3d39e1e2dcb3 branch: 2.7 parent: 99390:436e58ae3bab user: Serhiy Storchaka <storchaka(a)gmail.com> date: Mon Nov 30 17:20:02 2015 +0200 summary: Issue #25718: Fixed copying object with state with boolean value is false. files: Lib/copy.py | 4 ++-- Lib/test/test_copy.py | 9 +++++++++ Misc/NEWS | 2 ++ 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/Lib/copy.py b/Lib/copy.py --- a/Lib/copy.py +++ b/Lib/copy.py @@ -315,7 +315,7 @@ if n > 2: state = info[2] else: - state = {} + state = None if n > 3: listiter = info[3] else: @@ -329,7 +329,7 @@ y = callable(*args) memo[id(x)] = y - if state: + if state is not None: if deep: state = deepcopy(state, memo) if hasattr(y, '__setstate__'): diff --git a/Lib/test/test_copy.py b/Lib/test/test_copy.py --- a/Lib/test/test_copy.py +++ b/Lib/test/test_copy.py @@ -165,6 +165,9 @@ return cmp(self.foo, other.foo) x = C(42) self.assertEqual(copy.copy(x), x) + # State with boolean value is false (issue #25718) + x = C(0.0) + self.assertEqual(copy.copy(x), x) # The deepcopy() method @@ -395,6 +398,12 @@ x = C([42]) y = copy.deepcopy(x) self.assertEqual(y, x) + self.assertIsNot(y, x) + self.assertIsNot(y.foo, x.foo) + # State with boolean value is false (issue #25718) + x = C([]) + y = copy.deepcopy(x) + self.assertEqual(y, x) self.assertTrue(y is not x) self.assertTrue(y.foo is not x.foo) diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -20,6 +20,8 @@ Library ------- +- Issue #25718: Fixed copying object with state with boolean value is false. + - Issue #25742: :func:`locale.setlocale` now accepts a Unicode string for its second parameter. -- Repository URL: https://hg.python.org/cpython

1 0

Daily reference leaks (6b08429a3932): sum=28
by solipsis＠pitrou.net Nov. 30, 2015

Nov. 30, 2015

results for 6b08429a3932 on branch "default" -------------------------------------------- test_functools leaked [0, 2, 2] memory blocks, sum=4 test_pickle leaked [4, 4, 4] references, sum=12 test_pickle leaked [4, 4, 4] memory blocks, sum=12 Command line was: ['./python', '-m', 'test.regrtest', '-uall', '-R', '3:3:/home/psf-users/antoine/refleaks/reflogFNst38', '--timeout', '7200']

1 0

cpython: Issue #5319: New Py_FinalizeEx() API to exit with status 120 on failure
by martin.panter Nov. 29, 2015

Nov. 29, 2015

https://hg.python.org/cpython/rev/6b08429a3932 changeset: 99391:6b08429a3932 parent: 99385:c852c7d8d681 user: Martin Panter <vadmium+py(a)gmail.com> date: Mon Nov 30 03:18:29 2015 +0000 summary: Issue #5319: New Py_FinalizeEx() API to exit with status 120 on failure files: Doc/c-api/init.rst | 33 +++++++++----- Doc/c-api/intro.rst | 4 +- Doc/c-api/sys.rst | 14 ++++-- Doc/extending/embedding.rst | 6 +- Doc/includes/run-func.c | 4 +- Doc/library/sys.rst | 7 ++- Doc/whatsnew/3.6.rst | 6 +- Include/pylifecycle.h | 1 + Lib/test/test_cmd_line.py | 3 +- Misc/NEWS | 3 + Misc/SpecialBuilds.txt | 8 +- Modules/main.c | 8 ++- PC/bdist_wininst/install.c | 18 +++++--- PC/python3.def | 1 + Python/frozenmain.c | 4 +- Python/pylifecycle.c | 52 +++++++++++++++++------ Python/pystate.c | 2 +- Tools/scripts/combinerefs.py | 4 +- 18 files changed, 120 insertions(+), 58 deletions(-) diff --git a/Doc/c-api/init.rst b/Doc/c-api/init.rst --- a/Doc/c-api/init.rst +++ b/Doc/c-api/init.rst @@ -25,7 +25,7 @@ triple: module; search; path single: PySys_SetArgv() single: PySys_SetArgvEx() - single: Py_Finalize() + single: Py_FinalizeEx() Initialize the Python interpreter. In an application embedding Python, this should be called before using any other Python/C API functions; with the @@ -34,7 +34,7 @@ modules :mod:`builtins`, :mod:`__main__` and :mod:`sys`. It also initializes the module search path (``sys.path``). It does not set ``sys.argv``; use :c:func:`PySys_SetArgvEx` for that. This is a no-op when called for a second time - (without calling :c:func:`Py_Finalize` first). There is no return value; it is a + (without calling :c:func:`Py_FinalizeEx` first). There is no return value; it is a fatal error if the initialization fails. @@ -48,19 +48,20 @@ .. c:function:: int Py_IsInitialized() Return true (nonzero) when the Python interpreter has been initialized, false - (zero) if not. After :c:func:`Py_Finalize` is called, this returns false until + (zero) if not. After :c:func:`Py_FinalizeEx` is called, this returns false until :c:func:`Py_Initialize` is called again. -.. c:function:: void Py_Finalize() +.. c:function:: int Py_FinalizeEx() Undo all initializations made by :c:func:`Py_Initialize` and subsequent use of Python/C API functions, and destroy all sub-interpreters (see :c:func:`Py_NewInterpreter` below) that were created and not yet destroyed since the last call to :c:func:`Py_Initialize`. Ideally, this frees all memory allocated by the Python interpreter. This is a no-op when called for a second - time (without calling :c:func:`Py_Initialize` again first). There is no return - value; errors during finalization are ignored. + time (without calling :c:func:`Py_Initialize` again first). Normally the + return value is 0. If there were errors during finalization + (flushing buffered data), -1 is returned. This function is provided for a number of reasons. An embedding application might want to restart Python without having to restart the application itself. @@ -79,7 +80,15 @@ freed. Some memory allocated by extension modules may not be freed. Some extensions may not work properly if their initialization routine is called more than once; this can happen if an application calls :c:func:`Py_Initialize` and - :c:func:`Py_Finalize` more than once. + :c:func:`Py_FinalizeEx` more than once. + + .. versionadded:: 3.6 + + +.. c:function:: void Py_Finalize() + + This is a backwards-compatible version of :c:func:`Py_FinalizeEx` that + disregards the return value. Process-wide parameters @@ -107,7 +116,7 @@ Note that :data:`sys.stderr` always uses the "backslashreplace" error handler, regardless of this (or any other) setting. - If :c:func:`Py_Finalize` is called, this function will need to be called + If :c:func:`Py_FinalizeEx` is called, this function will need to be called again in order to affect subsequent calls to :c:func:`Py_Initialize`. Returns 0 if successful, a nonzero value on error (e.g. calling after the @@ -918,7 +927,7 @@ entry.) .. index:: - single: Py_Finalize() + single: Py_FinalizeEx() single: Py_Initialize() Extension modules are shared between (sub-)interpreters as follows: the first @@ -928,7 +937,7 @@ and filled with the contents of this copy; the extension's ``init`` function is not called. Note that this is different from what happens when an extension is imported after the interpreter has been completely re-initialized by calling - :c:func:`Py_Finalize` and :c:func:`Py_Initialize`; in that case, the extension's + :c:func:`Py_FinalizeEx` and :c:func:`Py_Initialize`; in that case, the extension's ``initmodule`` function *is* called again. .. index:: single: close() (in module os) @@ -936,14 +945,14 @@ .. c:function:: void Py_EndInterpreter(PyThreadState *tstate) - .. index:: single: Py_Finalize() + .. index:: single: Py_FinalizeEx() Destroy the (sub-)interpreter represented by the given thread state. The given thread state must be the current thread state. See the discussion of thread states below. When the call returns, the current thread state is *NULL*. All thread states associated with this interpreter are destroyed. (The global interpreter lock must be held before calling this function and is still held - when it returns.) :c:func:`Py_Finalize` will destroy all sub-interpreters that + when it returns.) :c:func:`Py_FinalizeEx` will destroy all sub-interpreters that haven't been explicitly destroyed at that point. diff --git a/Doc/c-api/intro.rst b/Doc/c-api/intro.rst --- a/Doc/c-api/intro.rst +++ b/Doc/c-api/intro.rst @@ -578,9 +578,9 @@ application may want to start over (make another call to :c:func:`Py_Initialize`) or the application is simply done with its use of Python and wants to free memory allocated by Python. This can be accomplished -by calling :c:func:`Py_Finalize`. The function :c:func:`Py_IsInitialized` returns +by calling :c:func:`Py_FinalizeEx`. The function :c:func:`Py_IsInitialized` returns true if Python is currently in the initialized state. More information about -these functions is given in a later chapter. Notice that :c:func:`Py_Finalize` +these functions is given in a later chapter. Notice that :c:func:`Py_FinalizeEx` does *not* free all memory allocated by the Python interpreter, e.g. memory allocated by extension modules currently cannot be released. diff --git a/Doc/c-api/sys.rst b/Doc/c-api/sys.rst --- a/Doc/c-api/sys.rst +++ b/Doc/c-api/sys.rst @@ -212,20 +212,24 @@ .. c:function:: void Py_Exit(int status) .. index:: - single: Py_Finalize() + single: Py_FinalizeEx() single: exit() - Exit the current process. This calls :c:func:`Py_Finalize` and then calls the - standard C library function ``exit(status)``. + Exit the current process. This calls :c:func:`Py_FinalizeEx` and then calls the + standard C library function ``exit(status)``. If :c:func:`Py_FinalizeEx` + indicates an error, the exit status is set to 120. + + .. versionchanged:: 3.6 + Errors from finalization no longer ignored. .. c:function:: int Py_AtExit(void (*func) ()) .. index:: - single: Py_Finalize() + single: Py_FinalizeEx() single: cleanup functions - Register a cleanup function to be called by :c:func:`Py_Finalize`. The cleanup + Register a cleanup function to be called by :c:func:`Py_FinalizeEx`. The cleanup function will be called with no arguments and should return no value. At most 32 cleanup functions can be registered. When the registration is successful, :c:func:`Py_AtExit` returns ``0``; on failure, it returns ``-1``. The cleanup diff --git a/Doc/extending/embedding.rst b/Doc/extending/embedding.rst --- a/Doc/extending/embedding.rst +++ b/Doc/extending/embedding.rst @@ -67,7 +67,9 @@ Py_Initialize(); PyRun_SimpleString("from time import time,ctime\n" "print('Today is', ctime(time()))\n"); - Py_Finalize(); + if (Py_FinalizeEx() < 0) { + exit(120); + } PyMem_RawFree(program); return 0; } @@ -76,7 +78,7 @@ :c:func:`Py_Initialize` to inform the interpreter about paths to Python run-time libraries. Next, the Python interpreter is initialized with :c:func:`Py_Initialize`, followed by the execution of a hard-coded Python script -that prints the date and time. Afterwards, the :c:func:`Py_Finalize` call shuts +that prints the date and time. Afterwards, the :c:func:`Py_FinalizeEx` call shuts the interpreter down, followed by the end of the program. In a real program, you may want to get the Python script from another source, perhaps a text-editor routine, a file, or a database. Getting the Python code from a file can better diff --git a/Doc/includes/run-func.c b/Doc/includes/run-func.c --- a/Doc/includes/run-func.c +++ b/Doc/includes/run-func.c @@ -63,6 +63,8 @@ fprintf(stderr, "Failed to load \"%s\"\n", argv[1]); return 1; } - Py_Finalize(); + if (Py_FinalizeEx() < 0) { + return 120; + } return 0; } diff --git a/Doc/library/sys.rst b/Doc/library/sys.rst --- a/Doc/library/sys.rst +++ b/Doc/library/sys.rst @@ -255,7 +255,7 @@ (defaulting to zero), or another type of object. If it is an integer, zero is considered "successful termination" and any nonzero value is considered "abnormal termination" by shells and the like. Most systems require it to be - in the range 0-127, and produce undefined results otherwise. Some systems + in the range 0--127, and produce undefined results otherwise. Some systems have a convention for assigning specific meanings to specific exit codes, but these are generally underdeveloped; Unix programs generally use 2 for command line syntax errors and 1 for all other kind of errors. If another type of @@ -268,6 +268,11 @@ the process when called from the main thread, and the exception is not intercepted. + .. versionchanged:: 3.6 + If an error occurs in the cleanup after the Python interpreter + has caught :exc:`SystemExit` (such as an error flushing buffered data + in the standard streams), the exit status is changed to 120. + .. data:: flags diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -171,7 +171,8 @@ Build and C API Changes ======================= -* None yet. +* New :c:func:`Py_FinalizeEx` API which indicates if flushing buffered data + failed (:issue:`5319`). Deprecated @@ -247,4 +248,5 @@ Changes in the C API -------------------- -* None yet. +* :c:func:`Py_Exit` (and the main interpreter) now override the exit status + with 120 if flushing buffered data failed. See :issue:`5319`. diff --git a/Include/pylifecycle.h b/Include/pylifecycle.h --- a/Include/pylifecycle.h +++ b/Include/pylifecycle.h @@ -27,6 +27,7 @@ PyAPI_FUNC(void) _Py_InitializeEx_Private(int, int); #endif PyAPI_FUNC(void) Py_Finalize(void); +PyAPI_FUNC(int) Py_FinalizeEx(void); PyAPI_FUNC(int) Py_IsInitialized(void); PyAPI_FUNC(PyThreadState *) Py_NewInterpreter(void); PyAPI_FUNC(void) Py_EndInterpreter(PyThreadState *); diff --git a/Lib/test/test_cmd_line.py b/Lib/test/test_cmd_line.py --- a/Lib/test/test_cmd_line.py +++ b/Lib/test/test_cmd_line.py @@ -348,8 +348,9 @@ test.support.SuppressCrashReport().__enter__() sys.stdout.write('x') os.close(sys.stdout.fileno())""" - rc, out, err = assert_python_ok('-c', code) + rc, out, err = assert_python_failure('-c', code) self.assertEqual(b'', out) + self.assertEqual(120, rc) self.assertRegex(err.decode('ascii', 'ignore'), 'Exception ignored in.*\nOSError: .*') diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,6 +10,9 @@ Core and Builtins ----------------- +- Issue #5319: New Py_FinalizeEx() API allowing Python to set an exit status + of 120 on failure to flush buffered streams. + - Issue #25485: telnetlib.Telnet is now a context manager. - Issue #24097: Fixed crash in object.__reduce__() if slot name is freed inside diff --git a/Misc/SpecialBuilds.txt b/Misc/SpecialBuilds.txt --- a/Misc/SpecialBuilds.txt +++ b/Misc/SpecialBuilds.txt @@ -65,9 +65,9 @@ simply by virtue of being in the list. envvar PYTHONDUMPREFS - If this envvar exists, Py_Finalize() arranges to print a list of all + If this envvar exists, Py_FinalizeEx() arranges to print a list of all still-live heap objects. This is printed twice, in different formats, - before and after Py_Finalize has cleaned up everything it can clean up. The + before and after Py_FinalizeEx has cleaned up everything it can clean up. The first output block produces the repr() of each object so is more informative; however, a lot of stuff destined to die is still alive then. The second output block is much harder to work with (repr() can't be invoked @@ -144,7 +144,7 @@ envvar PYTHONMALLOCSTATS If this envvar exists, a report of pymalloc summary statistics is printed to - stderr whenever a new arena is allocated, and also by Py_Finalize(). + stderr whenever a new arena is allocated, and also by Py_FinalizeEx(). Changed in 2.5: The number of extra bytes allocated is 4*sizeof(size_t). Before it was 16 on all boxes, reflecting that Python couldn't make use of @@ -179,7 +179,7 @@ */ int tp_maxalloc; -Allocation and deallocation code keeps these counts up to date. Py_Finalize() +Allocation and deallocation code keeps these counts up to date. Py_FinalizeEx() displays a summary of the info returned by sys.getcounts() (see below), along with assorted other special allocation counts (like the number of tuple allocations satisfied by a tuple free-list, the number of 1-character strings diff --git a/Modules/main.c b/Modules/main.c --- a/Modules/main.c +++ b/Modules/main.c @@ -654,7 +654,7 @@ Py_SetProgramName(wbuf); /* Don't free wbuf, the argument to Py_SetProgramName - * must remain valid until the Py_Finalize is called. + * must remain valid until Py_FinalizeEx is called. */ } else { Py_SetProgramName(argv[0]); @@ -785,7 +785,11 @@ sts = PyRun_AnyFileFlags(stdin, "<stdin>", &cf) != 0; } - Py_Finalize(); + if (Py_FinalizeEx() < 0) { + /* Value unlikely to be confused with a non-error exit status or + other special meaning */ + sts = 120; + } #ifdef __INSURE__ /* Insure++ is a memory analysis tool that aids in discovering diff --git a/PC/bdist_wininst/install.c b/PC/bdist_wininst/install.c --- a/PC/bdist_wininst/install.c +++ b/PC/bdist_wininst/install.c @@ -709,7 +709,7 @@ * 1 if the Python-dll does not export the functions we need * 2 if no install-script is specified in pathname * 3 if the install-script file could not be opened - * the return value of PyRun_SimpleString() otherwise, + * the return value of PyRun_SimpleString() or Py_FinalizeEx() otherwise, * which is 0 if everything is ok, -1 if an exception had occurred * in the install-script. */ @@ -722,7 +722,7 @@ DECLPROC(hPython, void, Py_Initialize, (void)); DECLPROC(hPython, int, PySys_SetArgv, (int, wchar_t **)); DECLPROC(hPython, int, PyRun_SimpleString, (char *)); - DECLPROC(hPython, void, Py_Finalize, (void)); + DECLPROC(hPython, int, Py_FinalizeEx, (void)); DECLPROC(hPython, PyObject *, Py_BuildValue, (char *, ...)); DECLPROC(hPython, PyObject *, PyCFunction_New, (PyMethodDef *, PyObject *)); @@ -730,7 +730,7 @@ DECLPROC(hPython, PyObject *, PyErr_Format, (PyObject *, char *)); if (!Py_Initialize || !PySys_SetArgv - || !PyRun_SimpleString || !Py_Finalize) + || !PyRun_SimpleString || !Py_FinalizeEx) return 1; if (!Py_BuildValue || !PyArg_ParseTuple || !PyErr_Format) @@ -777,7 +777,9 @@ } } } - Py_Finalize(); + if (Py_FinalizeEx() < 0) { + result = -1; + } close(fh); return result; @@ -839,11 +841,11 @@ int rc; DECLPROC(hPython, void, Py_Initialize, (void)); DECLPROC(hPython, void, Py_SetProgramName, (wchar_t *)); - DECLPROC(hPython, void, Py_Finalize, (void)); + DECLPROC(hPython, int, Py_FinalizeEx, (void)); DECLPROC(hPython, int, PyRun_SimpleString, (char *)); DECLPROC(hPython, void, PyErr_Print, (void)); - if (!Py_Initialize || !Py_SetProgramName || !Py_Finalize || + if (!Py_Initialize || !Py_SetProgramName || !Py_FinalizeEx || !PyRun_SimpleString || !PyErr_Print) return -1; @@ -853,7 +855,9 @@ rc = PyRun_SimpleString(script); if (rc) PyErr_Print(); - Py_Finalize(); + if (Py_FinalizeEx() < 0) { + rc = -1; + } return rc; } diff --git a/PC/python3.def b/PC/python3.def --- a/PC/python3.def +++ b/PC/python3.def @@ -648,6 +648,7 @@ Py_FatalError=python36.Py_FatalError Py_FileSystemDefaultEncoding=python36.Py_FileSystemDefaultEncoding DATA Py_Finalize=python36.Py_Finalize + Py_FinalizeEx=python36.Py_FinalizeEx Py_GetBuildInfo=python36.Py_GetBuildInfo Py_GetCompiler=python36.Py_GetCompiler Py_GetCopyright=python36.Py_GetCopyright diff --git a/Python/frozenmain.c b/Python/frozenmain.c --- a/Python/frozenmain.c +++ b/Python/frozenmain.c @@ -99,7 +99,9 @@ #ifdef MS_WINDOWS PyWinFreeze_ExeTerm(); #endif - Py_Finalize(); + if (Py_FinalizeEx() < 0) { + sts = 120; + } error: PyMem_RawFree(argv_copy); diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -154,8 +154,8 @@ return 0; } -/* Global initializations. Can be undone by Py_Finalize(). Don't - call this twice without an intervening Py_Finalize() call. When +/* Global initializations. Can be undone by Py_FinalizeEx(). Don't + call this twice without an intervening Py_FinalizeEx() call. When initializations fail, a fatal error is issued and the function does not return. On return, the first thread and interpreter state have been created. @@ -327,11 +327,11 @@ (void) PyThreadState_Swap(tstate); #ifdef WITH_THREAD - /* We can't call _PyEval_FiniThreads() in Py_Finalize because + /* We can't call _PyEval_FiniThreads() in Py_FinalizeEx because destroying the GIL might fail when it is being referenced from another running thread (see issue #9901). Instead we destroy the previously created GIL here, which ensures - that we can call Py_Initialize / Py_Finalize multiple times. */ + that we can call Py_Initialize / Py_FinalizeEx multiple times. */ _PyEval_FiniThreads(); /* Auto-thread-state API */ @@ -477,28 +477,35 @@ return r > 0; } -static void +static int flush_std_files(void) { PyObject *fout = _PySys_GetObjectId(&PyId_stdout); PyObject *ferr = _PySys_GetObjectId(&PyId_stderr); PyObject *tmp; + int status = 0; if (fout != NULL && fout != Py_None && !file_is_closed(fout)) { tmp = _PyObject_CallMethodId(fout, &PyId_flush, ""); - if (tmp == NULL) + if (tmp == NULL) { PyErr_WriteUnraisable(fout); + status = -1; + } else Py_DECREF(tmp); } if (ferr != NULL && ferr != Py_None && !file_is_closed(ferr)) { tmp = _PyObject_CallMethodId(ferr, &PyId_flush, ""); - if (tmp == NULL) + if (tmp == NULL) { PyErr_Clear(); + status = -1; + } else Py_DECREF(tmp); } + + return status; } /* Undo the effect of Py_Initialize(). @@ -515,14 +522,15 @@ */ -void -Py_Finalize(void) +int +Py_FinalizeEx(void) { PyInterpreterState *interp; PyThreadState *tstate; + int status = 0; if (!initialized) - return; + return status; wait_for_thread_shutdown(); @@ -547,7 +555,9 @@ initialized = 0; /* Flush sys.stdout and sys.stderr */ - flush_std_files(); + if (flush_std_files() < 0) { + status = -1; + } /* Disable signal handling */ PyOS_FiniInterrupts(); @@ -576,7 +586,9 @@ PyImport_Cleanup(); /* Flush sys.stdout and sys.stderr (again, in case more was printed) */ - flush_std_files(); + if (flush_std_files() < 0) { + status = -1; + } /* Collect final garbage. This disposes of cycles created by * class definitions, for example. @@ -696,6 +708,13 @@ #endif call_ll_exitfuncs(); + return status; +} + +void +Py_Finalize(void) +{ + Py_FinalizeEx(); } /* Create and initialize a new interpreter and thread, and return the @@ -803,7 +822,7 @@ frames, and that it is its interpreter's only remaining thread. It is a fatal error to violate these constraints. - (Py_Finalize() doesn't have these constraints -- it zaps + (Py_FinalizeEx() doesn't have these constraints -- it zaps everything, regardless.) Locking: as above. @@ -1016,7 +1035,8 @@ mode = "rb"; buf = _PyObject_CallMethodId(io, &PyId_open, "isiOOOi", fd, mode, buffering, - Py_None, Py_None, Py_None, 0); + Py_None, Py_None, /* encoding, errors */ + Py_None, 0); /* newline, closefd */ if (buf == NULL) goto error; @@ -1450,7 +1470,9 @@ void Py_Exit(int sts) { - Py_Finalize(); + if (Py_FinalizeEx() < 0) { + sts = 120; + } exit(sts); } diff --git a/Python/pystate.c b/Python/pystate.c --- a/Python/pystate.c +++ b/Python/pystate.c @@ -686,7 +686,7 @@ } /* Internal initialization/finalization functions called by - Py_Initialize/Py_Finalize + Py_Initialize/Py_FinalizeEx */ void _PyGILState_Init(PyInterpreterState *i, PyThreadState *t) diff --git a/Tools/scripts/combinerefs.py b/Tools/scripts/combinerefs.py --- a/Tools/scripts/combinerefs.py +++ b/Tools/scripts/combinerefs.py @@ -6,7 +6,7 @@ A helper for analyzing PYTHONDUMPREFS output. When the PYTHONDUMPREFS envar is set in a debug build, at Python shutdown -time Py_Finalize() prints the list of all live objects twice: first it +time Py_FinalizeEx() prints the list of all live objects twice: first it prints the repr() of each object while the interpreter is still fully intact. After cleaning up everything it can, it prints all remaining live objects again, but the second time just prints their addresses, refcounts, and type @@ -41,7 +41,7 @@ objects shown in the repr: the repr was captured from the first output block, and some of the containees may have been released since then. For example, it's common for the line showing the dict of interned strings to display -strings that no longer exist at the end of Py_Finalize; this can be recognized +strings that no longer exist at the end of Py_FinalizeEx; this can be recognized (albeit painfully) because such containees don't have a line of their own. The objects are listed in allocation order, with most-recently allocated -- Repository URL: https://hg.python.org/cpython

1 0

cpython (2.7): Fixed yet one syntax error in test_xpickle in Python 2.5.
by serhiy.storchaka Nov. 29, 2015

Nov. 29, 2015

https://hg.python.org/cpython/rev/436e58ae3bab changeset: 99390:436e58ae3bab branch: 2.7 user: Serhiy Storchaka <storchaka(a)gmail.com> date: Sun Nov 29 20:18:27 2015 +0200 summary: Fixed yet one syntax error in test_xpickle in Python 2.5. files: Lib/test/pickletester.py | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/Lib/test/pickletester.py b/Lib/test/pickletester.py --- a/Lib/test/pickletester.py +++ b/Lib/test/pickletester.py @@ -746,7 +746,7 @@ # 'P', # PERSID # 'Pabc', 'S', # STRING - b"S'abc'", + "S'abc'", 'T', # BINSTRING 'T\x03\x00\x00', 'T\x03\x00\x00\x00', -- Repository URL: https://hg.python.org/cpython

1 0

cpython (2.7): Fixed Py3k warnings in tests for issue #24731.
by serhiy.storchaka Nov. 29, 2015

Nov. 29, 2015

https://hg.python.org/cpython/rev/2ea1a3bf448f changeset: 99389:2ea1a3bf448f branch: 2.7 user: Serhiy Storchaka <storchaka(a)gmail.com> date: Sun Nov 29 20:13:56 2015 +0200 summary: Fixed Py3k warnings in tests for issue #24731. files: Lib/test/test_compile.py | 17 +++++++++-------- Lib/test/test_float.py | 17 ++++++++++------- Lib/test/test_int.py | 17 ++++++++++------- 3 files changed, 29 insertions(+), 22 deletions(-) diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py --- a/Lib/test/test_compile.py +++ b/Lib/test/test_compile.py @@ -576,14 +576,15 @@ # objects are accepted, which could be not terminated. with self.assertRaisesRegexp(TypeError, "without null bytes"): compile(u"123\x00", "<dummy>", "eval") - with self.assertRaisesRegexp(TypeError, "without null bytes"): - compile(buffer("123\x00"), "<dummy>", "eval") - code = compile(buffer("123\x00", 1, 2), "<dummy>", "eval") - self.assertEqual(eval(code), 23) - code = compile(buffer("1234", 1, 2), "<dummy>", "eval") - self.assertEqual(eval(code), 23) - code = compile(buffer("$23$", 1, 2), "<dummy>", "eval") - self.assertEqual(eval(code), 23) + with test_support.check_py3k_warnings(): + with self.assertRaisesRegexp(TypeError, "without null bytes"): + compile(buffer("123\x00"), "<dummy>", "eval") + code = compile(buffer("123\x00", 1, 2), "<dummy>", "eval") + self.assertEqual(eval(code), 23) + code = compile(buffer("1234", 1, 2), "<dummy>", "eval") + self.assertEqual(eval(code), 23) + code = compile(buffer("$23$", 1, 2), "<dummy>", "eval") + self.assertEqual(eval(code), 23) class TestStackSize(unittest.TestCase): # These tests check that the computed stack size for a code object diff --git a/Lib/test/test_float.py b/Lib/test/test_float.py --- a/Lib/test/test_float.py +++ b/Lib/test/test_float.py @@ -71,7 +71,8 @@ factories += [unicode, CustomUnicode] for f in factories: - x = f(" 3.14 ") + with test_support.check_py3k_warnings(quiet=True): + x = f(" 3.14 ") msg = 'x has value %s and type %s' % (x, type(x).__name__) try: self.assertEqual(float(x), 3.14, msg=msg) @@ -79,15 +80,17 @@ raise AssertionError('For %s got TypeError: %s' % (type(x).__name__, err)) errmsg = "could not convert" - with self.assertRaisesRegexp(ValueError, errmsg, msg=msg): + with self.assertRaisesRegexp(ValueError, errmsg, msg=msg), \ + test_support.check_py3k_warnings(quiet=True): float(f('A' * 0x10)) def test_float_buffer(self): - self.assertEqual(float(buffer('12.3', 1, 3)), 2.3) - self.assertEqual(float(buffer('12.3\x00', 1, 3)), 2.3) - self.assertEqual(float(buffer('12.3 ', 1, 3)), 2.3) - self.assertEqual(float(buffer('12.3A', 1, 3)), 2.3) - self.assertEqual(float(buffer('12.34', 1, 3)), 2.3) + with test_support.check_py3k_warnings(): + self.assertEqual(float(buffer('12.3', 1, 3)), 2.3) + self.assertEqual(float(buffer('12.3\x00', 1, 3)), 2.3) + self.assertEqual(float(buffer('12.3 ', 1, 3)), 2.3) + self.assertEqual(float(buffer('12.3A', 1, 3)), 2.3) + self.assertEqual(float(buffer('12.34', 1, 3)), 2.3) def check_conversion_to_int(self, x): """Check that int(x) has the correct value and type, for a float x.""" diff --git a/Lib/test/test_int.py b/Lib/test/test_int.py --- a/Lib/test/test_int.py +++ b/Lib/test/test_int.py @@ -351,7 +351,8 @@ factories += [unicode, CustomUnicode] for f in factories: - x = f('100') + with test_support.check_py3k_warnings(quiet=True): + x = f('100') msg = 'x has value %s and type %s' % (x, type(x).__name__) try: self.assertEqual(int(x), 100, msg=msg) @@ -365,15 +366,17 @@ with self.assertRaisesRegexp(TypeError, errmsg, msg=msg): int(x, 2) errmsg = 'invalid literal' - with self.assertRaisesRegexp(ValueError, errmsg, msg=msg): + with self.assertRaisesRegexp(ValueError, errmsg, msg=msg), \ + test_support.check_py3k_warnings(quiet=True): int(f('A' * 0x10)) def test_int_buffer(self): - self.assertEqual(int(buffer('123', 1, 2)), 23) - self.assertEqual(int(buffer('123\x00', 1, 2)), 23) - self.assertEqual(int(buffer('123 ', 1, 2)), 23) - self.assertEqual(int(buffer('123A', 1, 2)), 23) - self.assertEqual(int(buffer('1234', 1, 2)), 23) + with test_support.check_py3k_warnings(): + self.assertEqual(int(buffer('123', 1, 2)), 23) + self.assertEqual(int(buffer('123\x00', 1, 2)), 23) + self.assertEqual(int(buffer('123 ', 1, 2)), 23) + self.assertEqual(int(buffer('123A', 1, 2)), 23) + self.assertEqual(int(buffer('1234', 1, 2)), 23) def test_error_on_string_float_for_x(self): self.assertRaises(ValueError, int, '1.2') -- Repository URL: https://hg.python.org/cpython

1 0

cpython (2.7): Trying to fix test_xpickle with python 2.4 and 2.5.
by serhiy.storchaka Nov. 29, 2015

Nov. 29, 2015

https://hg.python.org/cpython/rev/ee29e93ab3fb changeset: 99388:ee29e93ab3fb branch: 2.7 user: Serhiy Storchaka <storchaka(a)gmail.com> date: Sun Nov 29 19:20:11 2015 +0200 summary: Trying to fix test_xpickle with python 2.4 and 2.5. files: Lib/test/pickletester.py | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/test/pickletester.py b/Lib/test/pickletester.py --- a/Lib/test/pickletester.py +++ b/Lib/test/pickletester.py @@ -723,8 +723,8 @@ self.check_unpickling_error(self.bad_mark_errors, p) def test_truncated_data(self): - self.check_unpickling_error(EOFError, b'') - self.check_unpickling_error(EOFError, b'N') + self.check_unpickling_error(EOFError, '') + self.check_unpickling_error(EOFError, 'N') badpickles = [ 'F', # FLOAT 'F0.0', -- Repository URL: https://hg.python.org/cpython

1 0

cpython (2.7): Issue #25742: Try to fix test_locale on Windows
by victor.stinner Nov. 29, 2015

Nov. 29, 2015

https://hg.python.org/cpython/rev/d7481ebeaa4f changeset: 99387:d7481ebeaa4f branch: 2.7 user: Victor Stinner <victor.stinner(a)gmail.com> date: Sun Nov 29 16:33:18 2015 +0100 summary: Issue #25742: Try to fix test_locale on Windows files: Lib/test/test_locale.py | 16 ++++++++-------- 1 files changed, 8 insertions(+), 8 deletions(-) diff --git a/Lib/test/test_locale.py b/Lib/test/test_locale.py --- a/Lib/test/test_locale.py +++ b/Lib/test/test_locale.py @@ -494,14 +494,14 @@ self.assertEqual(locale.normalize(u'en_US'), 'en_US.ISO8859-1') def test_setlocale_unicode(self): - old_loc = locale.getlocale(locale.LC_ALL) - try: - user_locale = locale.setlocale(locale.LC_ALL, '') - unicode_locale = user_locale.decode('utf-8') - user_locale2 = locale.setlocale(locale.LC_ALL, unicode_locale) - self.assertEqual(user_locale, user_locale2) - finally: - locale.setlocale(locale.LC_ALL, old_loc) + oldlocale = locale.getlocale() + self.addCleanup(locale.setlocale, locale.LC_CTYPE, oldlocale) + + user_locale = locale.setlocale(locale.LC_CTYPE, '') + unicode_locale = user_locale.decode('utf-8') + + user_locale2 = locale.setlocale(locale.LC_CTYPE, unicode_locale) + self.assertEqual(user_locale, user_locale2) def test_main(): -- Repository URL: https://hg.python.org/cpython

1 0

cpython (2.7): Got rid of "with" for compatibility test_xpickle with Python 2.5.
by serhiy.storchaka Nov. 29, 2015

Nov. 29, 2015

https://hg.python.org/cpython/rev/2e864db39267 changeset: 99386:2e864db39267 branch: 2.7 parent: 99383:c7e7d77ef8bf user: Serhiy Storchaka <storchaka(a)gmail.com> date: Sun Nov 29 16:13:51 2015 +0200 summary: Got rid of "with" for compatibility test_xpickle with Python 2.5. files: Lib/test/pickletester.py | 15 +++++++++++---- 1 files changed, 11 insertions(+), 4 deletions(-) diff --git a/Lib/test/pickletester.py b/Lib/test/pickletester.py --- a/Lib/test/pickletester.py +++ b/Lib/test/pickletester.py @@ -476,15 +476,22 @@ getattr(objcopy, slot, None), msg=msg) def check_unpickling_error(self, errors, data): - with self.assertRaises(errors): + try: try: self.loads(data) except: if support.verbose > 1: - exc = sys.exc_info()[1] - print('%-32r - %s: %s' % - (data, exc.__class__.__name__, exc)) + exc_type, exc, tb = sys.exc_info() + print '%-32r - %s: %s' % (data, exc_type.__name__, exc) raise + except errors: + pass + else: + try: + exc_name = errors.__name__ + except AttributeError: + exc_name = str(errors) + raise self.failureException('%s not raised' % exc_name) def test_load_from_canned_string(self): expected = self._testdata -- Repository URL: https://hg.python.org/cpython

1 0

cpython (merge 3.4 -> 3.5): Issue #25761: Added more test cases for testing unpickling broken data.
by serhiy.storchaka Nov. 29, 2015

Nov. 29, 2015

https://hg.python.org/cpython/rev/4897438543da changeset: 99384:4897438543da branch: 3.5 parent: 99376:5d6b2dc7e3d0 parent: 99382:d47e5b162072 user: Serhiy Storchaka <storchaka(a)gmail.com> date: Sun Nov 29 13:13:24 2015 +0200 summary: Issue #25761: Added more test cases for testing unpickling broken data. Output raised exception at verbose level 2 (-vv). files: Lib/test/pickletester.py | 290 ++++++++++++++++++-------- Lib/test/test_pickle.py | 13 + 2 files changed, 210 insertions(+), 93 deletions(-) diff --git a/Lib/test/pickletester.py b/Lib/test/pickletester.py --- a/Lib/test/pickletester.py +++ b/Lib/test/pickletester.py @@ -11,8 +11,9 @@ import weakref from http.cookies import SimpleCookie +from test import support from test.support import ( - TestFailed, TESTFN, run_with_locale, no_tracing, captured_stdout, + TestFailed, TESTFN, run_with_locale, no_tracing, _2G, _4G, bigmemtest, ) @@ -679,6 +680,17 @@ self.assertEqual(getattr(obj, slot, None), getattr(objcopy, slot, None), msg=msg) + def check_unpickling_error(self, errors, data): + with self.subTest(data=data), \ + self.assertRaises(errors): + try: + self.loads(data) + except BaseException as exc: + if support.verbose > 1: + print('%-32r - %s: %s' % + (data, exc.__class__.__name__, exc)) + raise + def test_load_from_data0(self): self.assert_is_copy(self._testdata, self.loads(DATA0)) @@ -759,12 +771,7 @@ # Try too with a bogus literal. data = b'I' + str(maxint64).encode("ascii") + b'JUNK\n.' - self.assertRaises(ValueError, self.loads, data) - - def test_pop_empty_stack(self): - # Test issue7455 - s = b'0' - self.assertRaises((pickle.UnpicklingError, IndexError), self.loads, s) + self.check_unpickling_error(ValueError, data) def test_unpickle_from_2x(self): # Unpickle non-trivial data from Python 2.x. @@ -849,22 +856,22 @@ def test_negative_32b_binbytes(self): # On 32-bit builds, a BINBYTES of 2**31 or more is refused dumped = b'\x80\x03B\xff\xff\xff\xffxyzq\x00.' - with self.assertRaises((pickle.UnpicklingError, OverflowError)): - self.loads(dumped) + self.check_unpickling_error((pickle.UnpicklingError, OverflowError), + dumped) @requires_32b def test_negative_32b_binunicode(self): # On 32-bit builds, a BINUNICODE of 2**31 or more is refused dumped = b'\x80\x03X\xff\xff\xff\xffxyzq\x00.' - with self.assertRaises((pickle.UnpicklingError, OverflowError)): - self.loads(dumped) + self.check_unpickling_error((pickle.UnpicklingError, OverflowError), + dumped) def test_short_binunicode(self): dumped = b'\x80\x04\x8c\x04\xe2\x82\xac\x00.' self.assertEqual(self.loads(dumped), '\u20ac\x00') def test_misc_get(self): - self.assertRaises(KeyError, self.loads, b'g0\np0') + self.check_unpickling_error(KeyError, b'g0\np0') self.assert_is_copy([(100,), (100,)], self.loads(b'((Kdtp0\nh\x00l.))')) @@ -879,14 +886,14 @@ @requires_32b def test_large_32b_binbytes8(self): dumped = b'\x80\x04\x8e\4\0\0\0\1\0\0\0\xe2\x82\xac\x00.' - with self.assertRaises((pickle.UnpicklingError, OverflowError)): - self.loads(dumped) + self.check_unpickling_error((pickle.UnpicklingError, OverflowError), + dumped) @requires_32b def test_large_32b_binunicode8(self): dumped = b'\x80\x04\x8d\4\0\0\0\1\0\0\0\xe2\x82\xac\x00.' - with self.assertRaises((pickle.UnpicklingError, OverflowError)): - self.loads(dumped) + self.check_unpickling_error((pickle.UnpicklingError, OverflowError), + dumped) def test_get(self): pickled = b'((lp100000\ng100000\nt.' @@ -915,16 +922,16 @@ def test_negative_put(self): # Issue #12847 dumped = b'Va\np-1\n.' - self.assertRaises(ValueError, self.loads, dumped) + self.check_unpickling_error(ValueError, dumped) @requires_32b def test_negative_32b_binput(self): # Issue #12847 dumped = b'\x80\x03X\x01\x00\x00\x00ar\xff\xff\xff\xff.' - self.assertRaises(ValueError, self.loads, dumped) + self.check_unpickling_error(ValueError, dumped) def test_badly_escaped_string(self): - self.assertRaises(ValueError, self.loads, b"S'\\'\n.") + self.check_unpickling_error(ValueError, b"S'\\'\n.") def test_badly_quoted_string(self): # Issue #17710 @@ -942,7 +949,7 @@ b'S\n.', b'S.'] for p in badpickles: - self.assertRaises(pickle.UnpicklingError, self.loads, p) + self.check_unpickling_error(pickle.UnpicklingError, p) def test_correctly_quoted_string(self): goodpickles = [(b"S''\n.", ''), @@ -989,86 +996,183 @@ def test_bad_stack(self): badpickles = [ - b'0.', # POP - b'1.', # POP_MARK - b'2.', # DUP - # b'(2.', # PyUnpickler doesn't raise - b'R.', # REDUCE - b')R.', - b'a.', # APPEND - b'Na.', - b'b.', # BUILD - b'Nb.', - b'd.', # DICT - b'e.', # APPENDS - # b'(e.', # PyUnpickler raises AttributeError - b'ibuiltins\nlist\n.', # INST - b'l.', # LIST - b'o.', # OBJ - b'(o.', - b'p1\n.', # PUT - b'q\x00.', # BINPUT - b'r\x00\x00\x00\x00.', # LONG_BINPUT - b's.', # SETITEM - b'Ns.', - b'NNs.', - b't.', # TUPLE - b'u.', # SETITEMS - b'(u.', - b'}(Nu.', - b'\x81.', # NEWOBJ - b')\x81.', - b'\x85.', # TUPLE1 - b'\x86.', # TUPLE2 - b'N\x86.', - b'\x87.', # TUPLE3 - b'N\x87.', - b'NN\x87.', - b'\x90.', # ADDITEMS - # b'(\x90.', # PyUnpickler raises AttributeError - b'\x91.', # FROZENSET - b'\x92.', # NEWOBJ_EX - b')}\x92.', - b'\x93.', # STACK_GLOBAL - b'Vlist\n\x93.', - b'\x94.', # MEMOIZE + b'.', # STOP + b'0', # POP + b'1', # POP_MARK + b'2', # DUP + # b'(2', # PyUnpickler doesn't raise + b'R', # REDUCE + b')R', + b'a', # APPEND + b'Na', + b'b', # BUILD + b'Nb', + b'd', # DICT + b'e', # APPENDS + # b'(e', # PyUnpickler raises AttributeError + b'ibuiltins\nlist\n', # INST + b'l', # LIST + b'o', # OBJ + b'(o', + b'p1\n', # PUT + b'q\x00', # BINPUT + b'r\x00\x00\x00\x00', # LONG_BINPUT + b's', # SETITEM + b'Ns', + b'NNs', + b't', # TUPLE + b'u', # SETITEMS + # b'(u', # PyUnpickler doesn't raise + b'}(Nu', + b'\x81', # NEWOBJ + b')\x81', + b'\x85', # TUPLE1 + b'\x86', # TUPLE2 + b'N\x86', + b'\x87', # TUPLE3 + b'N\x87', + b'NN\x87', + b'\x90', # ADDITEMS + # b'(\x90', # PyUnpickler raises AttributeError + b'\x91', # FROZENSET + b'\x92', # NEWOBJ_EX + b')}\x92', + b'\x93', # STACK_GLOBAL + b'Vlist\n\x93', + b'\x94', # MEMOIZE ] for p in badpickles: - with self.subTest(p): - self.assertRaises(self.bad_stack_errors, self.loads, p) + self.check_unpickling_error(self.bad_stack_errors, p) def test_bad_mark(self): badpickles = [ - b'cbuiltins\nlist\n)(R.', # REDUCE - b'cbuiltins\nlist\n()R.', - b']N(a.', # APPEND - b'cbuiltins\nValueError\n)R}(b.', # BUILD - b'cbuiltins\nValueError\n)R(}b.', - b'(Nd.', # DICT - b'}NN(s.', # SETITEM - b'}N(Ns.', - b'cbuiltins\nlist\n)(\x81.', # NEWOBJ - b'cbuiltins\nlist\n()\x81.', - b'N(\x85.', # TUPLE1 - b'NN(\x86.', # TUPLE2 - b'N(N\x86.', - b'NNN(\x87.', # TUPLE3 - b'NN(N\x87.', - b'N(NN\x87.', - b'cbuiltins\nlist\n)}(\x92.', # NEWOBJ_EX - b'cbuiltins\nlist\n)(}\x92.', - b'cbuiltins\nlist\n()}\x92.', - b'Vbuiltins\n(Vlist\n\x93.', # STACK_GLOBAL - b'Vbuiltins\nVlist\n(\x93.', + # b'N(.', # STOP + b'N(2', # DUP + b'cbuiltins\nlist\n)(R', # REDUCE + b'cbuiltins\nlist\n()R', + b']N(a', # APPEND + # BUILD + b'cbuiltins\nValueError\n)R}(b', + b'cbuiltins\nValueError\n)R(}b', + b'(Nd', # DICT + b'N(p1\n', # PUT + b'N(q\x00', # BINPUT + b'N(r\x00\x00\x00\x00', # LONG_BINPUT + b'}NN(s', # SETITEM + b'}N(Ns', + b'}(NNs', + b'}((u', # SETITEMS + b'cbuiltins\nlist\n)(\x81', # NEWOBJ + b'cbuiltins\nlist\n()\x81', + b'N(\x85', # TUPLE1 + b'NN(\x86', # TUPLE2 + b'N(N\x86', + b'NNN(\x87', # TUPLE3 + b'NN(N\x87', + b'N(NN\x87', + b']((\x90', # ADDITEMS + # NEWOBJ_EX + b'cbuiltins\nlist\n)}(\x92', + b'cbuiltins\nlist\n)(}\x92', + b'cbuiltins\nlist\n()}\x92', + # STACK_GLOBAL + b'Vbuiltins\n(Vlist\n\x93', + b'Vbuiltins\nVlist\n(\x93', + b'N(\x94', # MEMOIZE ] for p in badpickles: - # PyUnpickler prints reduce errors to stdout - with self.subTest(p), captured_stdout(): - try: - self.loads(p) - except (IndexError, AttributeError, TypeError, - pickle.UnpicklingError): - pass + self.check_unpickling_error(self.bad_mark_errors, p) + + def test_truncated_data(self): + self.check_unpickling_error(EOFError, b'') + self.check_unpickling_error(EOFError, b'N') + badpickles = [ + b'B', # BINBYTES + b'B\x03\x00\x00', + b'B\x03\x00\x00\x00', + b'B\x03\x00\x00\x00ab', + b'C', # SHORT_BINBYTES + b'C\x03', + b'C\x03ab', + b'F', # FLOAT + b'F0.0', + b'F0.00', + b'G', # BINFLOAT + b'G\x00\x00\x00\x00\x00\x00\x00', + b'I', # INT + b'I0', + b'J', # BININT + b'J\x00\x00\x00', + b'K', # BININT1 + b'L', # LONG + b'L0', + b'L10', + b'L0L', + b'L10L', + b'M', # BININT2 + b'M\x00', + # b'P', # PERSID + # b'Pabc', + b'S', # STRING + b"S'abc'", + b'T', # BINSTRING + b'T\x03\x00\x00', + b'T\x03\x00\x00\x00', + b'T\x03\x00\x00\x00ab', + b'U', # SHORT_BINSTRING + b'U\x03', + b'U\x03ab', + b'V', # UNICODE + b'Vabc', + b'X', # BINUNICODE + b'X\x03\x00\x00', + b'X\x03\x00\x00\x00', + b'X\x03\x00\x00\x00ab', + b'(c', # GLOBAL + b'(cbuiltins', + b'(cbuiltins\n', + b'(cbuiltins\nlist', + b'Ng', # GET + b'Ng0', + b'(i', # INST + b'(ibuiltins', + b'(ibuiltins\n', + b'(ibuiltins\nlist', + b'Nh', # BINGET + b'Nj', # LONG_BINGET + b'Nj\x00\x00\x00', + b'Np', # PUT + b'Np0', + b'Nq', # BINPUT + b'Nr', # LONG_BINPUT + b'Nr\x00\x00\x00', + b'\x80', # PROTO + b'\x82', # EXT1 + b'\x83', # EXT2 + b'\x84\x01', + b'\x84', # EXT4 + b'\x84\x01\x00\x00', + b'\x8a', # LONG1 + b'\x8b', # LONG4 + b'\x8b\x00\x00\x00', + b'\x8c', # SHORT_BINUNICODE + b'\x8c\x03', + b'\x8c\x03ab', + b'\x8d', # BINUNICODE8 + b'\x8d\x03\x00\x00\x00\x00\x00\x00', + b'\x8d\x03\x00\x00\x00\x00\x00\x00\x00', + b'\x8d\x03\x00\x00\x00\x00\x00\x00\x00ab', + b'\x8e', # BINBYTES8 + b'\x8e\x03\x00\x00\x00\x00\x00\x00', + b'\x8e\x03\x00\x00\x00\x00\x00\x00\x00', + b'\x8e\x03\x00\x00\x00\x00\x00\x00\x00ab', + b'\x95', # FRAME + b'\x95\x02\x00\x00\x00\x00\x00\x00', + b'\x95\x02\x00\x00\x00\x00\x00\x00\x00', + b'\x95\x02\x00\x00\x00\x00\x00\x00\x00N', + ] + for p in badpickles: + self.check_unpickling_error(self.truncated_errors, p) class AbstractPickleTests(unittest.TestCase): diff --git a/Lib/test/test_pickle.py b/Lib/test/test_pickle.py --- a/Lib/test/test_pickle.py +++ b/Lib/test/test_pickle.py @@ -33,6 +33,11 @@ unpickler = pickle._Unpickler bad_stack_errors = (IndexError,) + bad_mark_errors = (IndexError, pickle.UnpicklingError, + TypeError, AttributeError, EOFError) + truncated_errors = (pickle.UnpicklingError, EOFError, + AttributeError, ValueError, + struct.error, IndexError, ImportError) def loads(self, buf, **kwds): f = io.BytesIO(buf) @@ -64,6 +69,11 @@ pickler = pickle._Pickler unpickler = pickle._Unpickler bad_stack_errors = (pickle.UnpicklingError, IndexError) + bad_mark_errors = (pickle.UnpicklingError, IndexError, + TypeError, AttributeError, EOFError) + truncated_errors = (pickle.UnpicklingError, EOFError, + AttributeError, ValueError, + struct.error, IndexError, ImportError) def dumps(self, arg, protocol=None): return pickle.dumps(arg, protocol) @@ -122,6 +132,9 @@ class CUnpicklerTests(PyUnpicklerTests): unpickler = _pickle.Unpickler bad_stack_errors = (pickle.UnpicklingError,) + bad_mark_errors = (EOFError,) + truncated_errors = (pickle.UnpicklingError, EOFError, + AttributeError, ValueError) class CPicklerTests(PyPicklerTests): pickler = _pickle.Pickler -- Repository URL: https://hg.python.org/cpython

1 0

cpython (merge 3.5 -> default): Issue #25761: Added more test cases for testing unpickling broken data.
by serhiy.storchaka Nov. 29, 2015

Nov. 29, 2015

https://hg.python.org/cpython/rev/c852c7d8d681 changeset: 99385:c852c7d8d681 parent: 99381:2e889344436e parent: 99384:4897438543da user: Serhiy Storchaka <storchaka(a)gmail.com> date: Sun Nov 29 13:14:00 2015 +0200 summary: Issue #25761: Added more test cases for testing unpickling broken data. Output raised exception at verbose level 2 (-vv). files: Lib/test/pickletester.py | 290 ++++++++++++++++++-------- Lib/test/test_pickle.py | 13 + 2 files changed, 210 insertions(+), 93 deletions(-) diff --git a/Lib/test/pickletester.py b/Lib/test/pickletester.py --- a/Lib/test/pickletester.py +++ b/Lib/test/pickletester.py @@ -11,8 +11,9 @@ import weakref from http.cookies import SimpleCookie +from test import support from test.support import ( - TestFailed, TESTFN, run_with_locale, no_tracing, captured_stdout, + TestFailed, TESTFN, run_with_locale, no_tracing, _2G, _4G, bigmemtest, ) @@ -679,6 +680,17 @@ self.assertEqual(getattr(obj, slot, None), getattr(objcopy, slot, None), msg=msg) + def check_unpickling_error(self, errors, data): + with self.subTest(data=data), \ + self.assertRaises(errors): + try: + self.loads(data) + except BaseException as exc: + if support.verbose > 1: + print('%-32r - %s: %s' % + (data, exc.__class__.__name__, exc)) + raise + def test_load_from_data0(self): self.assert_is_copy(self._testdata, self.loads(DATA0)) @@ -759,12 +771,7 @@ # Try too with a bogus literal. data = b'I' + str(maxint64).encode("ascii") + b'JUNK\n.' - self.assertRaises(ValueError, self.loads, data) - - def test_pop_empty_stack(self): - # Test issue7455 - s = b'0' - self.assertRaises((pickle.UnpicklingError, IndexError), self.loads, s) + self.check_unpickling_error(ValueError, data) def test_unpickle_from_2x(self): # Unpickle non-trivial data from Python 2.x. @@ -849,22 +856,22 @@ def test_negative_32b_binbytes(self): # On 32-bit builds, a BINBYTES of 2**31 or more is refused dumped = b'\x80\x03B\xff\xff\xff\xffxyzq\x00.' - with self.assertRaises((pickle.UnpicklingError, OverflowError)): - self.loads(dumped) + self.check_unpickling_error((pickle.UnpicklingError, OverflowError), + dumped) @requires_32b def test_negative_32b_binunicode(self): # On 32-bit builds, a BINUNICODE of 2**31 or more is refused dumped = b'\x80\x03X\xff\xff\xff\xffxyzq\x00.' - with self.assertRaises((pickle.UnpicklingError, OverflowError)): - self.loads(dumped) + self.check_unpickling_error((pickle.UnpicklingError, OverflowError), + dumped) def test_short_binunicode(self): dumped = b'\x80\x04\x8c\x04\xe2\x82\xac\x00.' self.assertEqual(self.loads(dumped), '\u20ac\x00') def test_misc_get(self): - self.assertRaises(KeyError, self.loads, b'g0\np0') + self.check_unpickling_error(KeyError, b'g0\np0') self.assert_is_copy([(100,), (100,)], self.loads(b'((Kdtp0\nh\x00l.))')) @@ -879,14 +886,14 @@ @requires_32b def test_large_32b_binbytes8(self): dumped = b'\x80\x04\x8e\4\0\0\0\1\0\0\0\xe2\x82\xac\x00.' - with self.assertRaises((pickle.UnpicklingError, OverflowError)): - self.loads(dumped) + self.check_unpickling_error((pickle.UnpicklingError, OverflowError), + dumped) @requires_32b def test_large_32b_binunicode8(self): dumped = b'\x80\x04\x8d\4\0\0\0\1\0\0\0\xe2\x82\xac\x00.' - with self.assertRaises((pickle.UnpicklingError, OverflowError)): - self.loads(dumped) + self.check_unpickling_error((pickle.UnpicklingError, OverflowError), + dumped) def test_get(self): pickled = b'((lp100000\ng100000\nt.' @@ -915,16 +922,16 @@ def test_negative_put(self): # Issue #12847 dumped = b'Va\np-1\n.' - self.assertRaises(ValueError, self.loads, dumped) + self.check_unpickling_error(ValueError, dumped) @requires_32b def test_negative_32b_binput(self): # Issue #12847 dumped = b'\x80\x03X\x01\x00\x00\x00ar\xff\xff\xff\xff.' - self.assertRaises(ValueError, self.loads, dumped) + self.check_unpickling_error(ValueError, dumped) def test_badly_escaped_string(self): - self.assertRaises(ValueError, self.loads, b"S'\\'\n.") + self.check_unpickling_error(ValueError, b"S'\\'\n.") def test_badly_quoted_string(self): # Issue #17710 @@ -942,7 +949,7 @@ b'S\n.', b'S.'] for p in badpickles: - self.assertRaises(pickle.UnpicklingError, self.loads, p) + self.check_unpickling_error(pickle.UnpicklingError, p) def test_correctly_quoted_string(self): goodpickles = [(b"S''\n.", ''), @@ -989,86 +996,183 @@ def test_bad_stack(self): badpickles = [ - b'0.', # POP - b'1.', # POP_MARK - b'2.', # DUP - # b'(2.', # PyUnpickler doesn't raise - b'R.', # REDUCE - b')R.', - b'a.', # APPEND - b'Na.', - b'b.', # BUILD - b'Nb.', - b'd.', # DICT - b'e.', # APPENDS - # b'(e.', # PyUnpickler raises AttributeError - b'ibuiltins\nlist\n.', # INST - b'l.', # LIST - b'o.', # OBJ - b'(o.', - b'p1\n.', # PUT - b'q\x00.', # BINPUT - b'r\x00\x00\x00\x00.', # LONG_BINPUT - b's.', # SETITEM - b'Ns.', - b'NNs.', - b't.', # TUPLE - b'u.', # SETITEMS - b'(u.', - b'}(Nu.', - b'\x81.', # NEWOBJ - b')\x81.', - b'\x85.', # TUPLE1 - b'\x86.', # TUPLE2 - b'N\x86.', - b'\x87.', # TUPLE3 - b'N\x87.', - b'NN\x87.', - b'\x90.', # ADDITEMS - # b'(\x90.', # PyUnpickler raises AttributeError - b'\x91.', # FROZENSET - b'\x92.', # NEWOBJ_EX - b')}\x92.', - b'\x93.', # STACK_GLOBAL - b'Vlist\n\x93.', - b'\x94.', # MEMOIZE + b'.', # STOP + b'0', # POP + b'1', # POP_MARK + b'2', # DUP + # b'(2', # PyUnpickler doesn't raise + b'R', # REDUCE + b')R', + b'a', # APPEND + b'Na', + b'b', # BUILD + b'Nb', + b'd', # DICT + b'e', # APPENDS + # b'(e', # PyUnpickler raises AttributeError + b'ibuiltins\nlist\n', # INST + b'l', # LIST + b'o', # OBJ + b'(o', + b'p1\n', # PUT + b'q\x00', # BINPUT + b'r\x00\x00\x00\x00', # LONG_BINPUT + b's', # SETITEM + b'Ns', + b'NNs', + b't', # TUPLE + b'u', # SETITEMS + # b'(u', # PyUnpickler doesn't raise + b'}(Nu', + b'\x81', # NEWOBJ + b')\x81', + b'\x85', # TUPLE1 + b'\x86', # TUPLE2 + b'N\x86', + b'\x87', # TUPLE3 + b'N\x87', + b'NN\x87', + b'\x90', # ADDITEMS + # b'(\x90', # PyUnpickler raises AttributeError + b'\x91', # FROZENSET + b'\x92', # NEWOBJ_EX + b')}\x92', + b'\x93', # STACK_GLOBAL + b'Vlist\n\x93', + b'\x94', # MEMOIZE ] for p in badpickles: - with self.subTest(p): - self.assertRaises(self.bad_stack_errors, self.loads, p) + self.check_unpickling_error(self.bad_stack_errors, p) def test_bad_mark(self): badpickles = [ - b'cbuiltins\nlist\n)(R.', # REDUCE - b'cbuiltins\nlist\n()R.', - b']N(a.', # APPEND - b'cbuiltins\nValueError\n)R}(b.', # BUILD - b'cbuiltins\nValueError\n)R(}b.', - b'(Nd.', # DICT - b'}NN(s.', # SETITEM - b'}N(Ns.', - b'cbuiltins\nlist\n)(\x81.', # NEWOBJ - b'cbuiltins\nlist\n()\x81.', - b'N(\x85.', # TUPLE1 - b'NN(\x86.', # TUPLE2 - b'N(N\x86.', - b'NNN(\x87.', # TUPLE3 - b'NN(N\x87.', - b'N(NN\x87.', - b'cbuiltins\nlist\n)}(\x92.', # NEWOBJ_EX - b'cbuiltins\nlist\n)(}\x92.', - b'cbuiltins\nlist\n()}\x92.', - b'Vbuiltins\n(Vlist\n\x93.', # STACK_GLOBAL - b'Vbuiltins\nVlist\n(\x93.', + # b'N(.', # STOP + b'N(2', # DUP + b'cbuiltins\nlist\n)(R', # REDUCE + b'cbuiltins\nlist\n()R', + b']N(a', # APPEND + # BUILD + b'cbuiltins\nValueError\n)R}(b', + b'cbuiltins\nValueError\n)R(}b', + b'(Nd', # DICT + b'N(p1\n', # PUT + b'N(q\x00', # BINPUT + b'N(r\x00\x00\x00\x00', # LONG_BINPUT + b'}NN(s', # SETITEM + b'}N(Ns', + b'}(NNs', + b'}((u', # SETITEMS + b'cbuiltins\nlist\n)(\x81', # NEWOBJ + b'cbuiltins\nlist\n()\x81', + b'N(\x85', # TUPLE1 + b'NN(\x86', # TUPLE2 + b'N(N\x86', + b'NNN(\x87', # TUPLE3 + b'NN(N\x87', + b'N(NN\x87', + b']((\x90', # ADDITEMS + # NEWOBJ_EX + b'cbuiltins\nlist\n)}(\x92', + b'cbuiltins\nlist\n)(}\x92', + b'cbuiltins\nlist\n()}\x92', + # STACK_GLOBAL + b'Vbuiltins\n(Vlist\n\x93', + b'Vbuiltins\nVlist\n(\x93', + b'N(\x94', # MEMOIZE ] for p in badpickles: - # PyUnpickler prints reduce errors to stdout - with self.subTest(p), captured_stdout(): - try: - self.loads(p) - except (IndexError, AttributeError, TypeError, - pickle.UnpicklingError): - pass + self.check_unpickling_error(self.bad_mark_errors, p) + + def test_truncated_data(self): + self.check_unpickling_error(EOFError, b'') + self.check_unpickling_error(EOFError, b'N') + badpickles = [ + b'B', # BINBYTES + b'B\x03\x00\x00', + b'B\x03\x00\x00\x00', + b'B\x03\x00\x00\x00ab', + b'C', # SHORT_BINBYTES + b'C\x03', + b'C\x03ab', + b'F', # FLOAT + b'F0.0', + b'F0.00', + b'G', # BINFLOAT + b'G\x00\x00\x00\x00\x00\x00\x00', + b'I', # INT + b'I0', + b'J', # BININT + b'J\x00\x00\x00', + b'K', # BININT1 + b'L', # LONG + b'L0', + b'L10', + b'L0L', + b'L10L', + b'M', # BININT2 + b'M\x00', + # b'P', # PERSID + # b'Pabc', + b'S', # STRING + b"S'abc'", + b'T', # BINSTRING + b'T\x03\x00\x00', + b'T\x03\x00\x00\x00', + b'T\x03\x00\x00\x00ab', + b'U', # SHORT_BINSTRING + b'U\x03', + b'U\x03ab', + b'V', # UNICODE + b'Vabc', + b'X', # BINUNICODE + b'X\x03\x00\x00', + b'X\x03\x00\x00\x00', + b'X\x03\x00\x00\x00ab', + b'(c', # GLOBAL + b'(cbuiltins', + b'(cbuiltins\n', + b'(cbuiltins\nlist', + b'Ng', # GET + b'Ng0', + b'(i', # INST + b'(ibuiltins', + b'(ibuiltins\n', + b'(ibuiltins\nlist', + b'Nh', # BINGET + b'Nj', # LONG_BINGET + b'Nj\x00\x00\x00', + b'Np', # PUT + b'Np0', + b'Nq', # BINPUT + b'Nr', # LONG_BINPUT + b'Nr\x00\x00\x00', + b'\x80', # PROTO + b'\x82', # EXT1 + b'\x83', # EXT2 + b'\x84\x01', + b'\x84', # EXT4 + b'\x84\x01\x00\x00', + b'\x8a', # LONG1 + b'\x8b', # LONG4 + b'\x8b\x00\x00\x00', + b'\x8c', # SHORT_BINUNICODE + b'\x8c\x03', + b'\x8c\x03ab', + b'\x8d', # BINUNICODE8 + b'\x8d\x03\x00\x00\x00\x00\x00\x00', + b'\x8d\x03\x00\x00\x00\x00\x00\x00\x00', + b'\x8d\x03\x00\x00\x00\x00\x00\x00\x00ab', + b'\x8e', # BINBYTES8 + b'\x8e\x03\x00\x00\x00\x00\x00\x00', + b'\x8e\x03\x00\x00\x00\x00\x00\x00\x00', + b'\x8e\x03\x00\x00\x00\x00\x00\x00\x00ab', + b'\x95', # FRAME + b'\x95\x02\x00\x00\x00\x00\x00\x00', + b'\x95\x02\x00\x00\x00\x00\x00\x00\x00', + b'\x95\x02\x00\x00\x00\x00\x00\x00\x00N', + ] + for p in badpickles: + self.check_unpickling_error(self.truncated_errors, p) class AbstractPickleTests(unittest.TestCase): diff --git a/Lib/test/test_pickle.py b/Lib/test/test_pickle.py --- a/Lib/test/test_pickle.py +++ b/Lib/test/test_pickle.py @@ -33,6 +33,11 @@ unpickler = pickle._Unpickler bad_stack_errors = (IndexError,) + bad_mark_errors = (IndexError, pickle.UnpicklingError, + TypeError, AttributeError, EOFError) + truncated_errors = (pickle.UnpicklingError, EOFError, + AttributeError, ValueError, + struct.error, IndexError, ImportError) def loads(self, buf, **kwds): f = io.BytesIO(buf) @@ -64,6 +69,11 @@ pickler = pickle._Pickler unpickler = pickle._Unpickler bad_stack_errors = (pickle.UnpicklingError, IndexError) + bad_mark_errors = (pickle.UnpicklingError, IndexError, + TypeError, AttributeError, EOFError) + truncated_errors = (pickle.UnpicklingError, EOFError, + AttributeError, ValueError, + struct.error, IndexError, ImportError) def dumps(self, arg, protocol=None): return pickle.dumps(arg, protocol) @@ -122,6 +132,9 @@ class CUnpicklerTests(PyUnpicklerTests): unpickler = _pickle.Unpickler bad_stack_errors = (pickle.UnpicklingError,) + bad_mark_errors = (EOFError,) + truncated_errors = (pickle.UnpicklingError, EOFError, + AttributeError, ValueError) class CPicklerTests(PyPicklerTests): pickler = _pickle.Pickler -- Repository URL: https://hg.python.org/cpython

1 0