Mailman 3 June 2014 - Python-checkins

cpython (2.7): don't overwrite the error from PyObject_GetAttrString (closes #4346)
by benjamin.peterson 27 Jun '14

27 Jun '14

http://hg.python.org/cpython/rev/aa4b4487c7ad changeset: 91434:aa4b4487c7ad branch: 2.7 parent: 91431:eb0921b2100b user: Benjamin Peterson <benjamin(a)python.org> date: Thu Jun 26 23:27:41 2014 -0700 summary: don't overwrite the error from PyObject_GetAttrString (closes #4346) files: Misc/NEWS | 3 +++ Objects/abstract.c | 12 ++++-------- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,6 +10,9 @@ Core and Builtins ----------------- +- Issue #4346: In PyObject_CallMethod and PyObject_CallMethodObjArgs, don't + overwrite the error set in PyObject_GetAttr. + - Issue #21831: Avoid integer overflow when large sizes and offsets are given to the buffer type. diff --git a/Objects/abstract.c b/Objects/abstract.c --- a/Objects/abstract.c +++ b/Objects/abstract.c @@ -2617,10 +2617,8 @@ return null_error(); func = PyObject_GetAttrString(o, name); - if (func == NULL) { - PyErr_SetString(PyExc_AttributeError, name); - return 0; - } + if (func == NULL) + return NULL; if (!PyCallable_Check(func)) { type_error("attribute of type '%.200s' is not callable", func); @@ -2656,10 +2654,8 @@ return null_error(); func = PyObject_GetAttrString(o, name); - if (func == NULL) { - PyErr_SetString(PyExc_AttributeError, name); - return 0; - } + if (func == NULL) + return NULL; if (!PyCallable_Check(func)) { type_error("attribute of type '%.200s' is not callable", func); -- Repository URL: http://hg.python.org/cpython

1 0

cpython (merge 3.4 -> default): (Merge 3.4) Issue #21858: Better handling of Python exceptions in the sqlite3
by victor.stinner 27 Jun '14

27 Jun '14

http://hg.python.org/cpython/rev/169171da66fa changeset: 91433:169171da66fa parent: 91430:bbb28082d7b4 parent: 91432:62612b195cb5 user: Victor Stinner <victor.stinner(a)gmail.com> date: Thu Jun 26 23:33:34 2014 +0200 summary: (Merge 3.4) Issue #21858: Better handling of Python exceptions in the sqlite3 module. files: Misc/NEWS | 2 + Modules/_sqlite/cursor.c | 42 +++++++++++++++++---------- 2 files changed, 28 insertions(+), 16 deletions(-) diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -103,6 +103,8 @@ Library ------- +- Issue #21858: Better handling of Python exceptions in the sqlite3 module. + - Issue #21476: Make sure the email.parser.BytesParser TextIOWrapper is discarded after parsing, so the input file isn't unexpectedly closed. diff --git a/Modules/_sqlite/cursor.c b/Modules/_sqlite/cursor.c --- a/Modules/_sqlite/cursor.c +++ b/Modules/_sqlite/cursor.c @@ -289,9 +289,8 @@ Py_END_ALLOW_THREADS row = PyTuple_New(numcols); - if (!row) { + if (!row) return NULL; - } for (i = 0; i < numcols; i++) { if (self->connection->detect_types) { @@ -311,14 +310,12 @@ converted = Py_None; } else { item = PyBytes_FromStringAndSize(val_str, nbytes); - if (!item) { - return NULL; - } + if (!item) + goto error; converted = PyObject_CallFunction(converter, "O", item); Py_DECREF(item); - if (!converted) { + if (!converted) break; - } } } else { Py_BEGIN_ALLOW_THREADS @@ -374,9 +371,8 @@ nbytes = sqlite3_column_bytes(self->statement->st, i); buffer = PyBytes_FromStringAndSize( sqlite3_column_blob(self->statement->st, i), nbytes); - if (!buffer) { + if (!buffer) break; - } converted = buffer; } } @@ -389,12 +385,14 @@ } } - if (PyErr_Occurred()) { - Py_DECREF(row); - row = NULL; - } + if (PyErr_Occurred()) + goto error; return row; + +error: + Py_DECREF(row); + return NULL; } /* @@ -612,6 +610,10 @@ while (1) { /* Actually execute the SQL statement. */ rc = pysqlite_step(self->statement->st, self->connection); + if (PyErr_Occurred()) { + (void)pysqlite_statement_reset(self->statement); + goto error; + } if (rc == SQLITE_DONE || rc == SQLITE_ROW) { /* If it worked, let's get out of the loop */ break; @@ -685,6 +687,8 @@ } self->next_row = _pysqlite_fetch_one_row(self); + if (self->next_row == NULL) + goto error; } else if (rc == SQLITE_DONE && !multiple) { pysqlite_statement_reset(self->statement); Py_CLEAR(self->statement); @@ -807,7 +811,10 @@ rc = SQLITE_ROW; while (rc == SQLITE_ROW) { rc = pysqlite_step(statement, self->connection); - /* TODO: we probably need more error handling here */ + if (PyErr_Occurred()) { + (void)sqlite3_finalize(statement); + goto error; + } } if (rc != SQLITE_DONE) { @@ -884,6 +891,11 @@ if (self->statement) { rc = pysqlite_step(self->statement->st, self->connection); + if (PyErr_Occurred()) { + (void)pysqlite_statement_reset(self->statement); + Py_DECREF(next_row); + return NULL; + } if (rc != SQLITE_DONE && rc != SQLITE_ROW) { (void)pysqlite_statement_reset(self->statement); Py_DECREF(next_row); @@ -895,8 +907,6 @@ self->next_row = _pysqlite_fetch_one_row(self); if (self->next_row == NULL) { (void)pysqlite_statement_reset(self->statement); - Py_DECREF(next_row); - _pysqlite_seterror(self->connection->db, NULL); return NULL; } } -- Repository URL: http://hg.python.org/cpython

1 0

cpython (3.4): Issue #21858: Better handling of Python exceptions in the sqlite3 module.
by victor.stinner 27 Jun '14

27 Jun '14

http://hg.python.org/cpython/rev/62612b195cb5 changeset: 91432:62612b195cb5 branch: 3.4 parent: 91429:ab708e4131dd user: Victor Stinner <victor.stinner(a)gmail.com> date: Thu Jun 26 23:32:00 2014 +0200 summary: Issue #21858: Better handling of Python exceptions in the sqlite3 module. files: Misc/NEWS | 2 + Modules/_sqlite/cursor.c | 42 +++++++++++++++++---------- 2 files changed, 28 insertions(+), 16 deletions(-) diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -27,6 +27,8 @@ Library ------- +- Issue #21858: Better handling of Python exceptions in the sqlite3 module. + - Issue #21476: Make sure the email.parser.BytesParser TextIOWrapper is discarded after parsing, so the input file isn't unexpectedly closed. diff --git a/Modules/_sqlite/cursor.c b/Modules/_sqlite/cursor.c --- a/Modules/_sqlite/cursor.c +++ b/Modules/_sqlite/cursor.c @@ -289,9 +289,8 @@ Py_END_ALLOW_THREADS row = PyTuple_New(numcols); - if (!row) { + if (!row) return NULL; - } for (i = 0; i < numcols; i++) { if (self->connection->detect_types) { @@ -311,14 +310,12 @@ converted = Py_None; } else { item = PyBytes_FromStringAndSize(val_str, nbytes); - if (!item) { - return NULL; - } + if (!item) + goto error; converted = PyObject_CallFunction(converter, "O", item); Py_DECREF(item); - if (!converted) { + if (!converted) break; - } } } else { Py_BEGIN_ALLOW_THREADS @@ -374,9 +371,8 @@ nbytes = sqlite3_column_bytes(self->statement->st, i); buffer = PyBytes_FromStringAndSize( sqlite3_column_blob(self->statement->st, i), nbytes); - if (!buffer) { + if (!buffer) break; - } converted = buffer; } } @@ -389,12 +385,14 @@ } } - if (PyErr_Occurred()) { - Py_DECREF(row); - row = NULL; - } + if (PyErr_Occurred()) + goto error; return row; + +error: + Py_DECREF(row); + return NULL; } /* @@ -612,6 +610,10 @@ while (1) { /* Actually execute the SQL statement. */ rc = pysqlite_step(self->statement->st, self->connection); + if (PyErr_Occurred()) { + (void)pysqlite_statement_reset(self->statement); + goto error; + } if (rc == SQLITE_DONE || rc == SQLITE_ROW) { /* If it worked, let's get out of the loop */ break; @@ -685,6 +687,8 @@ } self->next_row = _pysqlite_fetch_one_row(self); + if (self->next_row == NULL) + goto error; } else if (rc == SQLITE_DONE && !multiple) { pysqlite_statement_reset(self->statement); Py_CLEAR(self->statement); @@ -807,7 +811,10 @@ rc = SQLITE_ROW; while (rc == SQLITE_ROW) { rc = pysqlite_step(statement, self->connection); - /* TODO: we probably need more error handling here */ + if (PyErr_Occurred()) { + (void)sqlite3_finalize(statement); + goto error; + } } if (rc != SQLITE_DONE) { @@ -884,6 +891,11 @@ if (self->statement) { rc = pysqlite_step(self->statement->st, self->connection); + if (PyErr_Occurred()) { + (void)pysqlite_statement_reset(self->statement); + Py_DECREF(next_row); + return NULL; + } if (rc != SQLITE_DONE && rc != SQLITE_ROW) { (void)pysqlite_statement_reset(self->statement); Py_DECREF(next_row); @@ -895,8 +907,6 @@ self->next_row = _pysqlite_fetch_one_row(self); if (self->next_row == NULL) { (void)pysqlite_statement_reset(self->statement); - Py_DECREF(next_row); - _pysqlite_seterror(self->connection->db, NULL); return NULL; } } -- Repository URL: http://hg.python.org/cpython

1 0

peps: Add PEP 471: "os.scandir() function -- a better and faster directory iterator"
by victor.stinner 27 Jun '14

27 Jun '14

http://hg.python.org/peps/rev/2ff0e17443e4 changeset: 5490:2ff0e17443e4 user: Victor Stinner <victor.stinner(a)gmail.com> date: Thu Jun 26 23:14:01 2014 +0200 summary: Add PEP 471: "os.scandir() function -- a better and faster directory iterator" by Ben Hoyt files: pep-0471.txt | 376 +++++++++++++++++++++++++++++++++++++++ 1 files changed, 376 insertions(+), 0 deletions(-) diff --git a/pep-0471.txt b/pep-0471.txt new file mode 100644 --- /dev/null +++ b/pep-0471.txt @@ -0,0 +1,376 @@ +PEP: 471 +Title: os.scandir() function -- a better and faster directory iterator +Version: $Revision$ +Last-Modified: $Date$ +Author: Ben Hoyt <benhoyt(a)gmail.com> +Status: Draft +Type: Standards Track +Content-Type: text/x-rst +Created: 30-May-2014 +Python-Version: 3.5 + + +Abstract +======== + +This PEP proposes including a new directory iteration function, +``os.scandir()``, in the standard library. This new function adds +useful functionality and increases the speed of ``os.walk()`` by 2-10 +times (depending on the platform and file system) by significantly +reducing the number of times ``stat()`` needs to be called. + + +Rationale +========= + +Python's built-in ``os.walk()`` is significantly slower than it needs +to be, because -- in addition to calling ``os.listdir()`` on each +directory -- it executes the system call ``os.stat()`` or +``GetFileAttributes()`` on each file to determine whether the entry is +a directory or not. + +But the underlying system calls -- ``FindFirstFile`` / +``FindNextFile`` on Windows and ``readdir`` on Linux and OS X -- +already tell you whether the files returned are directories or not, so +no further system calls are needed. In short, you can reduce the +number of system calls from approximately 2N to N, where N is the +total number of files and directories in the tree. (And because +directory trees are usually much wider than they are deep, it's often +much better than this.) + +In practice, removing all those extra system calls makes ``os.walk()`` +about **8-9 times as fast on Windows**, and about **2-3 times as fast +on Linux and Mac OS X**. So we're not talking about micro- +optimizations. See more `benchmarks`_. + +.. _`benchmarks`: https://github.com/benhoyt/scandir#benchmarks + +Somewhat relatedly, many people (see Python `Issue 11406`_) are also +keen on a version of ``os.listdir()`` that yields filenames as it +iterates instead of returning them as one big list. This improves +memory efficiency for iterating very large directories. + +So as well as providing a ``scandir()`` iterator function for calling +directly, Python's existing ``os.walk()`` function could be sped up a +huge amount. + +.. _`Issue 11406`: http://bugs.python.org/issue11406 + + +Implementation +============== + +The implementation of this proposal was written by Ben Hoyt (initial +version) and Tim Golden (who helped a lot with the C extension +module). It lives on GitHub at `benhoyt/scandir`_. + +.. _`benhoyt/scandir`: https://github.com/benhoyt/scandir + +Note that this module has been used and tested (see "Use in the wild" +section in this PEP), so it's more than a proof-of-concept. However, +it is marked as beta software and is not extensively battle-tested. +It will need some cleanup and more thorough testing before going into +the standard library, as well as integration into `posixmodule.c`. + + + +Specifics of proposal +===================== + +Specifically, this PEP proposes adding a single function to the ``os`` +module in the standard library, ``scandir``, that takes a single, +optional string as its argument:: + + scandir(path='.') -> generator of DirEntry objects + +Like ``listdir``, ``scandir`` calls the operating system's directory +iteration system calls to get the names of the files in the ``path`` +directory, but it's different from ``listdir`` in two ways: + +* Instead of bare filename strings, it returns lightweight + ``DirEntry`` objects that hold the filename string and provide + simple methods that allow access to the stat-like data the operating + system returned. + +* It returns a generator instead of a list, so that ``scandir`` acts + as a true iterator instead of returning the full list immediately. + +``scandir()`` yields a ``DirEntry`` object for each file and directory +in ``path``. Just like ``listdir``, the ``'.'`` and ``'..'`` +pseudo-directories are skipped, and the entries are yielded in +system-dependent order. Each ``DirEntry`` object has the following +attributes and methods: + +* ``name``: the entry's filename, relative to ``path`` (corresponds to + the return values of ``os.listdir``) + +* ``is_dir()``: like ``os.path.isdir()``, but requires no system calls + on most systems (Linux, Windows, OS X) + +* ``is_file()``: like ``os.path.isfile()``, but requires no system + calls on most systems (Linux, Windows, OS X) + +* ``is_symlink()``: like ``os.path.islink()``, but requires no system + calls on most systems (Linux, Windows, OS X) + +* ``lstat()``: like ``os.lstat()``, but requires no system calls on + Windows + +The ``DirEntry`` attribute and method names were chosen to be the same +as those in the new ``pathlib`` module for consistency. + + +Notes on caching +---------------- + +The ``DirEntry`` objects are relatively dumb -- the ``name`` attribute +is obviously always cached, and the ``is_X`` and ``lstat`` methods +cache their values (immediately on Windows via ``FindNextFile``, and +on first use on Linux / OS X via a ``stat`` call) and never refetch +from the system. + +For this reason, ``DirEntry`` objects are intended to be used and +thrown away after iteration, not stored in long-lived data structured +and the methods called again and again. + +If a user wants to do that (for example, for watching a file's size +change), they'll need to call the regular ``os.lstat()`` or +``os.path.getsize()`` functions which force a new system call each +time. + + +Examples +======== + +Here's a good usage pattern for ``scandir``. This is in fact almost +exactly how the scandir module's faster ``os.walk()`` implementation +uses it:: + + dirs = [] + non_dirs = [] + for entry in scandir(path): + if entry.is_dir(): + dirs.append(entry) + else: + non_dirs.append(entry) + +The above ``os.walk()``-like code will be significantly using scandir +on both Windows and Linux or OS X. + +Or, for getting the total size of files in a directory tree -- showing +use of the ``DirEntry.lstat()`` method:: + + def get_tree_size(path): + """Return total size of files in path and subdirs.""" + size = 0 + for entry in scandir(path): + if entry.is_dir(): + sub_path = os.path.join(path, entry.name) + size += get_tree_size(sub_path) + else: + size += entry.lstat().st_size + return size + +Note that ``get_tree_size()`` will get a huge speed boost on Windows, +because no extra stat call are needed, but on Linux and OS X the size +information is not returned by the directory iteration functions, so +this function won't gain anything there. + + +Support +======= + +The scandir module on GitHub has been forked and used quite a bit (see +"Use in the wild" in this PEP), but there's also been a fair bit of +direct support for a scandir-like function from core developers and +others on the python-dev and python-ideas mailing lists. A sampling: + +* **Nick Coghlan**, a core Python developer: "I've had the local Red + Hat release engineering team express their displeasure at having to + stat every file in a network mounted directory tree for info that is + present in the dirent structure, so a definite +1 to os.scandir from + me, so long as it makes that info available." + [`source1 <http://bugs.python.org/issue11406>`_] + +* **Tim Golden**, a core Python developer, supports scandir enough to + have spent time refactoring and significantly improving scandir's C + extension module. + [`source2 <https://github.com/tjguk/scandir>`_] + +* **Christian Heimes**, a core Python developer: "+1 for something + like yielddir()" + [`source3 <https://mail.python.org/pipermail/python-ideas/2012-November/017772.html>`_] + and "Indeed! I'd like to see the feature in 3.4 so I can remove my + own hack from our code base." + [`source4 <http://bugs.python.org/issue11406>`_] + +* **Gregory P. Smith**, a core Python developer: "As 3.4beta1 happens + tonight, this isn't going to make 3.4 so i'm bumping this to 3.5. + I really like the proposed design outlined above." + [`source5 <http://bugs.python.org/issue11406>`_] + +* **Guido van Rossum** on the possibility of adding scandir to Python + 3.5 (as it was too late for 3.4): "The ship has likewise sailed for + adding scandir() (whether to os or pathlib). By all means experiment + and get it ready for consideration for 3.5, but I don't want to add + it to 3.4." + [`source6 <https://mail.python.org/pipermail/python-dev/2013-November/130583.html>`_] + +Support for this PEP itself (meta-support?) was given by Nick Coghlan +on python-dev: "A PEP reviewing all this for 3.5 and proposing a +specific os.scandir API would be a good thing." +[`source7 <https://mail.python.org/pipermail/python-dev/2013-November/130588.html>`_] + + +Use in the wild +=============== + +To date, ``scandir`` is definitely useful, but has been clearly marked +"beta", so it's uncertain how much use of it there is in the wild. Ben +Hoyt has had several reports from people using it. For example: + +* Chris F: "I am processing some pretty large directories and was half + expecting to have to modify getdents. So thanks for saving me the + effort." [via personal email] + +* bschollnick: "I wanted to let you know about this, since I am using + Scandir as a building block for this code. Here's a good example of + scandir making a radical performance improvement over os.listdir." + [`source8 <https://github.com/benhoyt/scandir/issues/19>`_] + +* Avram L: "I'm testing our scandir for a project I'm working on. + Seems pretty solid, so first thing, just want to say nice work!" + [via personal email] + +Others have `requested a PyPI package`_ for it, which has been +created. See `PyPI package`_. + +.. _`requested a PyPI package`: https://github.com/benhoyt/scandir/issues/12 +.. _`PyPI package`: https://pypi.python.org/pypi/scandir + +GitHub stats don't mean too much, but scandir does have several +watchers, issues, forks, etc. Here's the run-down as of the stats as +of June 5, 2014: + +* Watchers: 17 +* Stars: 48 +* Forks: 15 +* Issues: 2 open, 19 closed + +**However, the much larger point is this:**, if this PEP is accepted, +``os.walk()`` can easily be reimplemented using ``scandir`` rather +than ``listdir`` and ``stat``, increasing the speed of ``os.walk()`` +very significantly. There are thousands of developers, scripts, and +production code that would benefit from this large speedup of +``os.walk()``. For example, on GitHub, there are almost as many uses +of ``os.walk`` (194,000) as there are of ``os.mkdir`` (230,000). + + +Open issues and optional things +=============================== + +There are a few open issues or optional additions: + + +Should scandir be in its own module? +------------------------------------ + +Should the function be included in the standard library in a new +module, ``scandir.scandir()``, or just as ``os.scandir()`` as +discussed? The preference of this PEP's author (Ben Hoyt) would be +``os.scandir()``, as it's just a single function. + + +Should there be a way to access the full path? +---------------------------------------------- + +Should ``DirEntry``'s have a way to get the full path without using +``os.path.join(path, entry.name)``? This is a pretty common pattern, +and it may be useful to add pathlib-like ``str(entry)`` functionality. +This functionality has also been requested in `issue 13`_ on GitHub. + +.. _`issue 13`: https://github.com/benhoyt/scandir/issues/13 + + +Should it expose Windows wildcard functionality? +------------------------------------------------ + +Should ``scandir()`` have a way of exposing the wildcard functionality +in the Windows ``FindFirstFile`` / ``FindNextFile`` functions? The +scandir module on GitHub exposes this as a ``windows_wildcard`` +keyword argument, allowing Windows power users the option to pass a +custom wildcard to ``FindFirstFile``, which may avoid the need to use +``fnmatch`` or similar on the resulting names. It is named the +unwieldly ``windows_wildcard`` to remind you you're writing power- +user, Windows-only code if you use it. + +This boils down to whether ``scandir`` should be about exposing all of +the system's directory iteration features, or simply providing a fast, +simple, cross-platform directory iteration API. + +This PEP's author votes for not including ``windows_wildcard`` in the +standard library version, because even though it could be useful in +rare cases (say the Windows Dropbox client?), it'd be too easy to use +it just because you're a Windows developer, and create code that is +not cross-platform. + + +Possible improvements +===================== + +There are many possible improvements one could make to scandir, but +here is a short list of some this PEP's author has in mind: + +* scandir could potentially be further sped up by calling ``readdir`` + / ``FindNextFile`` say 50 times per ``Py_BEGIN_ALLOW_THREADS`` block + so that it stays in the C extension module for longer, and may be + somewhat faster as a result. This approach hasn't been tested, but + was suggested by on Issue 11406 by Antoine Pitrou. + [`source9 <http://bugs.python.org/msg130125>`_] + + +Previous discussion +=================== + +* `Original thread Ben Hoyt started on python-ideas`_ about speeding + up ``os.walk()`` + +* Python `Issue 11406`_, which includes the original proposal for a + scandir-like function + +* `Further thread Ben Hoyt started on python-dev`_ that refined the + ``scandir()`` API, including Nick Coghlan's suggestion of scandir + yielding ``DirEntry``-like objects + +* `Final thread Ben Hoyt started on python-dev`_ to discuss the + interaction between scandir and the new ``pathlib`` module + +* `Question on StackOverflow`_ about why ``os.walk()`` is slow and + pointers on how to fix it (this inspired the author of this PEP + early on) + +* `BetterWalk`_, this PEP's author's previous attempt at this, on + which the scandir code is based + +.. _`Original thread Ben Hoyt started on python-ideas`: https://mail.python.org/pipermail/python-ideas/2012-November/017770.html +.. _`Further thread Ben Hoyt started on python-dev`: https://mail.python.org/pipermail/python-dev/2013-May/126119.html +.. _`Final thread Ben Hoyt started on python-dev`: https://mail.python.org/pipermail/python-dev/2013-November/130572.html +.. _`Question on StackOverflow`: http://stackoverflow.com/questions/2485719/very-quickly-getting-total-size-… +.. _`BetterWalk`: https://github.com/benhoyt/betterwalk + + +Copyright +========= + +This document has been placed in the public domain. + + + +.. + Local Variables: + mode: indented-text + indent-tabs-mode: nil + sentence-end-double-space: t + fill-column: 70 + coding: utf-8 + End: -- Repository URL: http://hg.python.org/peps

1 0

cpython (2.7): Issue #19897: Use python as executable instead of python3.
by berker.peksag 27 Jun '14

27 Jun '14

http://hg.python.org/cpython/rev/eb0921b2100b changeset: 91431:eb0921b2100b branch: 2.7 parent: 91425:07eb04003839 user: Berker Peksag <berker.peksag(a)gmail.com> date: Thu Jun 26 23:27:26 2014 +0300 summary: Issue #19897: Use python as executable instead of python3. files: Doc/library/site.rst | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/library/site.rst b/Doc/library/site.rst --- a/Doc/library/site.rst +++ b/Doc/library/site.rst @@ -185,8 +185,8 @@ .. code-block:: sh - $ python3 -m site --user-site - /home/user/.local/lib/python3.3/site-packages + $ python -m site --user-site + /home/user/.local/lib/python2.7/site-packages .. program:: site -- Repository URL: http://hg.python.org/cpython

1 0

cpython (merge 3.4 -> default): Issue #21829: Merge with 3.4
by zach.ware 27 Jun '14

27 Jun '14

http://hg.python.org/cpython/rev/bbb28082d7b4 changeset: 91430:bbb28082d7b4 parent: 91428:a3ee325fd489 parent: 91429:ab708e4131dd user: Zachary Ware <zachary.ware(a)gmail.com> date: Thu Jun 26 15:22:16 2014 -0500 summary: Issue #21829: Merge with 3.4 files: Lib/ctypes/test/test_values.py | 15 +++------------ 1 files changed, 3 insertions(+), 12 deletions(-) diff --git a/Lib/ctypes/test/test_values.py b/Lib/ctypes/test/test_values.py --- a/Lib/ctypes/test/test_values.py +++ b/Lib/ctypes/test/test_values.py @@ -33,20 +33,11 @@ """This test only works when python itself is a dll/shared library""" def test_optimizeflag(self): - # This test accesses the Py_OptimizeFlag intger, which is - # exported by the Python dll. + # This test accesses the Py_OptimizeFlag integer, which is + # exported by the Python dll and should match the sys.flags value - # It's value is set depending on the -O and -OO flags: - # if not given, it is 0 and __debug__ is 1. - # If -O is given, the flag is 1, for -OO it is 2. - # docstrings are also removed in the latter case. opt = c_int.in_dll(pythonapi, "Py_OptimizeFlag").value - if __debug__: - self.assertEqual(opt, 0) - elif ValuesTestCase.__doc__ is not None: - self.assertEqual(opt, 1) - else: - self.assertEqual(opt, 2) + self.assertEqual(opt, sys.flags.optimize) def test_frozentable(self): # Python exports a PyImport_FrozenModules symbol. This is a -- Repository URL: http://hg.python.org/cpython

1 0

cpython (3.4): Issue #21829: Fix running test_ctypes on Windows with -O or -OO
by zach.ware 27 Jun '14

27 Jun '14

http://hg.python.org/cpython/rev/ab708e4131dd changeset: 91429:ab708e4131dd branch: 3.4 parent: 91427:0a16756dfcc0 user: Zachary Ware <zachary.ware(a)gmail.com> date: Thu Jun 26 15:20:44 2014 -0500 summary: Issue #21829: Fix running test_ctypes on Windows with -O or -OO files: Lib/ctypes/test/test_values.py | 15 +++------------ 1 files changed, 3 insertions(+), 12 deletions(-) diff --git a/Lib/ctypes/test/test_values.py b/Lib/ctypes/test/test_values.py --- a/Lib/ctypes/test/test_values.py +++ b/Lib/ctypes/test/test_values.py @@ -33,20 +33,11 @@ """This test only works when python itself is a dll/shared library""" def test_optimizeflag(self): - # This test accesses the Py_OptimizeFlag intger, which is - # exported by the Python dll. + # This test accesses the Py_OptimizeFlag integer, which is + # exported by the Python dll and should match the sys.flags value - # It's value is set depending on the -O and -OO flags: - # if not given, it is 0 and __debug__ is 1. - # If -O is given, the flag is 1, for -OO it is 2. - # docstrings are also removed in the latter case. opt = c_int.in_dll(pythonapi, "Py_OptimizeFlag").value - if __debug__: - self.assertEqual(opt, 0) - elif ValuesTestCase.__doc__ is not None: - self.assertEqual(opt, 1) - else: - self.assertEqual(opt, 2) + self.assertEqual(opt, sys.flags.optimize) def test_frozentable(self): # Python exports a PyImport_FrozenModules symbol. This is a -- Repository URL: http://hg.python.org/cpython

1 0

devguide: log the addition of Berker Peksag
by benjamin.peterson 26 Jun '14

26 Jun '14

http://hg.python.org/devguide/rev/31dded3eeae0 changeset: 705:31dded3eeae0 user: Benjamin Peterson <benjamin(a)python.org> date: Thu Jun 26 10:57:49 2014 -0700 summary: log the addition of Berker Peksag files: developers.rst | 3 +++ 1 files changed, 3 insertions(+), 0 deletions(-) diff --git a/developers.rst b/developers.rst --- a/developers.rst +++ b/developers.rst @@ -25,6 +25,9 @@ Permissions History ------------------- +- Berker Peksağ was given push privileges on June 26, 2014 by Benjamin Peterson, + on the recommendation of R. David Murray. + - Steve Dower was given push privileges on May 10, 2014 by Antoine Pitrou, on recommendation by Martin v. Loewis. -- Repository URL: http://hg.python.org/devguide

1 0

cpython (merge 3.4 -> default): Merge #21476: Unwrap fp in BytesParser so the file isn't unexpectedly closed.
by r.david.murray 26 Jun '14

26 Jun '14

http://hg.python.org/cpython/rev/a3ee325fd489 changeset: 91428:a3ee325fd489 parent: 91426:71b9a841119a parent: 91427:0a16756dfcc0 user: R David Murray <rdmurray(a)bitdance.com> date: Thu Jun 26 13:33:05 2014 -0400 summary: Merge #21476: Unwrap fp in BytesParser so the file isn't unexpectedly closed. files: Lib/email/parser.py | 4 ++- Lib/test/test_email/test_email.py | 25 +++++++++++++++++++ Misc/NEWS | 3 ++ 3 files changed, 31 insertions(+), 1 deletions(-) diff --git a/Lib/email/parser.py b/Lib/email/parser.py --- a/Lib/email/parser.py +++ b/Lib/email/parser.py @@ -106,8 +106,10 @@ meaning it parses the entire contents of the file. """ fp = TextIOWrapper(fp, encoding='ascii', errors='surrogateescape') - with fp: + try: return self.parser.parse(fp, headersonly) + finally: + fp.detach() def parsebytes(self, text, headersonly=False): diff --git a/Lib/test/test_email/test_email.py b/Lib/test/test_email/test_email.py --- a/Lib/test/test_email/test_email.py +++ b/Lib/test/test_email/test_email.py @@ -3390,6 +3390,31 @@ self.assertIsInstance(msg.get_payload(), str) self.assertIsInstance(msg.get_payload(decode=True), bytes) + def test_bytes_parser_does_not_close_file(self): + with openfile('msg_02.txt', 'rb') as fp: + email.parser.BytesParser().parse(fp) + self.assertFalse(fp.closed) + + def test_bytes_parser_on_exception_does_not_close_file(self): + with openfile('msg_15.txt', 'rb') as fp: + bytesParser = email.parser.BytesParser + self.assertRaises(email.errors.StartBoundaryNotFoundDefect, + bytesParser(policy=email.policy.strict).parse, + fp) + self.assertFalse(fp.closed) + + def test_parser_does_not_close_file(self): + with openfile('msg_02.txt', 'r') as fp: + email.parser.Parser().parse(fp) + self.assertFalse(fp.closed) + + def test_parser_on_exception_does_not_close_file(self): + with openfile('msg_15.txt', 'r') as fp: + parser = email.parser.Parser + self.assertRaises(email.errors.StartBoundaryNotFoundDefect, + parser(policy=email.policy.strict).parse, fp) + self.assertFalse(fp.closed) + def test_whitespace_continuation(self): eq = self.assertEqual # This message contains a line after the Subject: header that has only diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -103,6 +103,9 @@ Library ------- +- Issue #21476: Make sure the email.parser.BytesParser TextIOWrapper is + discarded after parsing, so the input file isn't unexpectedly closed. + - Issue #20295: imghdr now recognizes OpenEXR format images. - Issue #21729: Used the "with" statement in the dbm.dumb module to ensure -- Repository URL: http://hg.python.org/cpython

1 0

cpython (3.4): #21476: Unwrap fp in BytesParser so the file isn't unexpectedly closed.
by r.david.murray 26 Jun '14

26 Jun '14

http://hg.python.org/cpython/rev/0a16756dfcc0 changeset: 91427:0a16756dfcc0 branch: 3.4 parent: 91423:463f499ef591 user: R David Murray <rdmurray(a)bitdance.com> date: Thu Jun 26 13:31:43 2014 -0400 summary: #21476: Unwrap fp in BytesParser so the file isn't unexpectedly closed. This makes the behavior match that of Parser. Patch by Vajrasky Kok. files: Lib/email/parser.py | 4 ++- Lib/test/test_email/test_email.py | 25 +++++++++++++++++++ Misc/NEWS | 3 ++ 3 files changed, 31 insertions(+), 1 deletions(-) diff --git a/Lib/email/parser.py b/Lib/email/parser.py --- a/Lib/email/parser.py +++ b/Lib/email/parser.py @@ -106,8 +106,10 @@ meaning it parses the entire contents of the file. """ fp = TextIOWrapper(fp, encoding='ascii', errors='surrogateescape') - with fp: + try: return self.parser.parse(fp, headersonly) + finally: + fp.detach() def parsebytes(self, text, headersonly=False): diff --git a/Lib/test/test_email/test_email.py b/Lib/test/test_email/test_email.py --- a/Lib/test/test_email/test_email.py +++ b/Lib/test/test_email/test_email.py @@ -3390,6 +3390,31 @@ self.assertIsInstance(msg.get_payload(), str) self.assertIsInstance(msg.get_payload(decode=True), bytes) + def test_bytes_parser_does_not_close_file(self): + with openfile('msg_02.txt', 'rb') as fp: + email.parser.BytesParser().parse(fp) + self.assertFalse(fp.closed) + + def test_bytes_parser_on_exception_does_not_close_file(self): + with openfile('msg_15.txt', 'rb') as fp: + bytesParser = email.parser.BytesParser + self.assertRaises(email.errors.StartBoundaryNotFoundDefect, + bytesParser(policy=email.policy.strict).parse, + fp) + self.assertFalse(fp.closed) + + def test_parser_does_not_close_file(self): + with openfile('msg_02.txt', 'r') as fp: + email.parser.Parser().parse(fp) + self.assertFalse(fp.closed) + + def test_parser_on_exception_does_not_close_file(self): + with openfile('msg_15.txt', 'r') as fp: + parser = email.parser.Parser + self.assertRaises(email.errors.StartBoundaryNotFoundDefect, + parser(policy=email.policy.strict).parse, fp) + self.assertFalse(fp.closed) + def test_whitespace_continuation(self): eq = self.assertEqual # This message contains a line after the Subject: header that has only diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -27,6 +27,9 @@ Library ------- +- Issue #21476: Make sure the email.parser.BytesParser TextIOWrapper is + discarded after parsing, so the input file isn't unexpectedly closed. + - Issue #21729: Used the "with" statement in the dbm.dumb module to ensure files closing. Patch by Claudiu Popa. -- Repository URL: http://hg.python.org/cpython

1 0