From solipsis at pitrou.net Thu Mar 1 04:09:24 2018 From: solipsis at pitrou.net (solipsis at pitrou.net) Date: Thu, 01 Mar 2018 09:09:24 +0000 Subject: [Python-checkins] Daily reference leaks (4243df51fe43): sum=11 Message-ID: <20180301090924.1.B58AA01F01278E5A@psf.io> results for 4243df51fe43 on branch "default" -------------------------------------------- test_collections leaked [0, 0, 7] memory blocks, sum=7 test_functools leaked [0, 3, 1] memory blocks, sum=4 Command line was: ['./python', '-m', 'test.regrtest', '-uall', '-R', '3:3:/home/psf-users/antoine/refleaks/reflogiRNHxE', '--timeout', '7200'] From webhook-mailer at python.org Thu Mar 1 04:14:05 2018 From: webhook-mailer at python.org (Xiang Zhang) Date: Thu, 01 Mar 2018 09:14:05 -0000 Subject: [Python-checkins] bpo-32903: Fix a memory leak in os.chdir() on Windows (GH-5801) Message-ID: https://github.com/python/cpython/commit/3e197c7a6740d564ad52fb7901c07d5ff49460f5 commit: 3e197c7a6740d564ad52fb7901c07d5ff49460f5 branch: master author: Alexey Izbyshev committer: Xiang Zhang date: 2018-03-01T17:13:56+08:00 summary: bpo-32903: Fix a memory leak in os.chdir() on Windows (GH-5801) files: A Misc/NEWS.d/next/Windows/2018-02-28-11-03-24.bpo-32903.1SXY4t.rst M Modules/posixmodule.c diff --git a/Misc/NEWS.d/next/Windows/2018-02-28-11-03-24.bpo-32903.1SXY4t.rst b/Misc/NEWS.d/next/Windows/2018-02-28-11-03-24.bpo-32903.1SXY4t.rst new file mode 100644 index 000000000000..a20a414790f8 --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2018-02-28-11-03-24.bpo-32903.1SXY4t.rst @@ -0,0 +1,2 @@ +Fix a memory leak in os.chdir() on Windows if the current directory is set +to a UNC path. diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 4b592298834d..6bba8ee26e16 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -1529,15 +1529,15 @@ win32_wchdir(LPCWSTR path) return FALSE; } } - if (wcsncmp(new_path, L"\\\\", 2) == 0 || - wcsncmp(new_path, L"//", 2) == 0) - /* UNC path, nothing to do. */ - return TRUE; - env[1] = new_path[0]; - result = SetEnvironmentVariableW(env, new_path); + int is_unc_like_path = (wcsncmp(new_path, L"\\\\", 2) == 0 || + wcsncmp(new_path, L"//", 2) == 0); + if (!is_unc_like_path) { + env[1] = new_path[0]; + result = SetEnvironmentVariableW(env, new_path); + } if (new_path != path_buf) PyMem_RawFree(new_path); - return result; + return result ? TRUE : FALSE; } #endif From webhook-mailer at python.org Thu Mar 1 05:27:37 2018 From: webhook-mailer at python.org (Xiang Zhang) Date: Thu, 01 Mar 2018 10:27:37 -0000 Subject: [Python-checkins] [2.7] bpo-32903: Fix a memory leak in os.chdir() on Windows (GH-5801). (#5947) Message-ID: https://github.com/python/cpython/commit/aa40f92240adea7067c3add8e09cec09dcf24d7f commit: aa40f92240adea7067c3add8e09cec09dcf24d7f branch: 2.7 author: Alexey Izbyshev committer: Xiang Zhang date: 2018-03-01T18:27:34+08:00 summary: [2.7] bpo-32903: Fix a memory leak in os.chdir() on Windows (GH-5801). (#5947) (cherry picked from commit 3e197c7a6740d564ad52fb7901c07d5ff49460f5) Co-authored-by: Alexey Izbyshev files: A Misc/NEWS.d/next/Windows/2018-02-28-11-03-24.bpo-32903.1SXY4t.rst M Modules/posixmodule.c diff --git a/Misc/NEWS.d/next/Windows/2018-02-28-11-03-24.bpo-32903.1SXY4t.rst b/Misc/NEWS.d/next/Windows/2018-02-28-11-03-24.bpo-32903.1SXY4t.rst new file mode 100644 index 000000000000..a20a414790f8 --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2018-02-28-11-03-24.bpo-32903.1SXY4t.rst @@ -0,0 +1,2 @@ +Fix a memory leak in os.chdir() on Windows if the current directory is set +to a UNC path. diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index f8e081ed7d61..2baf9202c303 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -984,6 +984,7 @@ win32_wchdir(LPCWSTR path) wchar_t _new_path[MAX_PATH+1], *new_path = _new_path; int result; wchar_t env[4] = L"=x:"; + int is_unc_like_path; if(!SetCurrentDirectoryW(path)) return FALSE; @@ -1002,15 +1003,15 @@ win32_wchdir(LPCWSTR path) return FALSE; } } - if (wcsncmp(new_path, L"\\\\", 2) == 0 || - wcsncmp(new_path, L"//", 2) == 0) - /* UNC path, nothing to do. */ - return TRUE; - env[1] = new_path[0]; - result = SetEnvironmentVariableW(env, new_path); + is_unc_like_path = (wcsncmp(new_path, L"\\\\", 2) == 0 || + wcsncmp(new_path, L"//", 2) == 0); + if (!is_unc_like_path) { + env[1] = new_path[0]; + result = SetEnvironmentVariableW(env, new_path); + } if (new_path != _new_path) free(new_path); - return result; + return result ? TRUE : FALSE; } #endif From webhook-mailer at python.org Thu Mar 1 05:28:44 2018 From: webhook-mailer at python.org (Xiang Zhang) Date: Thu, 01 Mar 2018 10:28:44 -0000 Subject: [Python-checkins] bpo-32903: Fix a memory leak in os.chdir() on Windows (GH-5801) (#5945) Message-ID: https://github.com/python/cpython/commit/6ae75d9d1221459ab18c2599e42fcc45f9f65617 commit: 6ae75d9d1221459ab18c2599e42fcc45f9f65617 branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Xiang Zhang date: 2018-03-01T18:28:41+08:00 summary: bpo-32903: Fix a memory leak in os.chdir() on Windows (GH-5801) (#5945) (cherry picked from commit 3e197c7a6740d564ad52fb7901c07d5ff49460f5) Co-authored-by: Alexey Izbyshev files: A Misc/NEWS.d/next/Windows/2018-02-28-11-03-24.bpo-32903.1SXY4t.rst M Modules/posixmodule.c diff --git a/Misc/NEWS.d/next/Windows/2018-02-28-11-03-24.bpo-32903.1SXY4t.rst b/Misc/NEWS.d/next/Windows/2018-02-28-11-03-24.bpo-32903.1SXY4t.rst new file mode 100644 index 000000000000..a20a414790f8 --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2018-02-28-11-03-24.bpo-32903.1SXY4t.rst @@ -0,0 +1,2 @@ +Fix a memory leak in os.chdir() on Windows if the current directory is set +to a UNC path. diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 130e8c1b36f7..f6144a1d0527 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -1529,15 +1529,15 @@ win32_wchdir(LPCWSTR path) return FALSE; } } - if (wcsncmp(new_path, L"\\\\", 2) == 0 || - wcsncmp(new_path, L"//", 2) == 0) - /* UNC path, nothing to do. */ - return TRUE; - env[1] = new_path[0]; - result = SetEnvironmentVariableW(env, new_path); + int is_unc_like_path = (wcsncmp(new_path, L"\\\\", 2) == 0 || + wcsncmp(new_path, L"//", 2) == 0); + if (!is_unc_like_path) { + env[1] = new_path[0]; + result = SetEnvironmentVariableW(env, new_path); + } if (new_path != path_buf) PyMem_RawFree(new_path); - return result; + return result ? TRUE : FALSE; } #endif From webhook-mailer at python.org Thu Mar 1 08:01:49 2018 From: webhook-mailer at python.org (Eric V. Smith) Date: Thu, 01 Mar 2018 13:01:49 -0000 Subject: [Python-checkins] Fixed incorrect default value for dataclass unsafe_hash. (GH-5949) Message-ID: https://github.com/python/cpython/commit/5da8cfb838fa1bf3529c085c6dce1adf3d1eaf62 commit: 5da8cfb838fa1bf3529c085c6dce1adf3d1eaf62 branch: master author: Eric V. Smith committer: GitHub date: 2018-03-01T08:01:41-05:00 summary: Fixed incorrect default value for dataclass unsafe_hash. (GH-5949) files: M Lib/dataclasses.py diff --git a/Lib/dataclasses.py b/Lib/dataclasses.py index 54478fec93df..b55a497db302 100644 --- a/Lib/dataclasses.py +++ b/Lib/dataclasses.py @@ -745,7 +745,7 @@ def _process_class(cls, repr, eq, order, unsafe_hash, init, frozen): # underscore. The presence of _cls is used to detect if this # decorator is being called with parameters or not. def dataclass(_cls=None, *, init=True, repr=True, eq=True, order=False, - unsafe_hash=None, frozen=False): + unsafe_hash=False, frozen=False): """Returns the same class as was passed in, with dunder methods added based on the fields defined in the class. @@ -880,7 +880,7 @@ def _astuple_inner(obj, tuple_factory): def make_dataclass(cls_name, fields, *, bases=(), namespace=None, init=True, - repr=True, eq=True, order=False, unsafe_hash=None, + repr=True, eq=True, order=False, unsafe_hash=False, frozen=False): """Return a new dynamically created dataclass. From webhook-mailer at python.org Thu Mar 1 08:30:18 2018 From: webhook-mailer at python.org (Eric V. Smith) Date: Thu, 01 Mar 2018 13:30:18 -0000 Subject: [Python-checkins] Fixed incorrect default value for dataclass unsafe_hash. (GH-5949) (GH-5950) Message-ID: https://github.com/python/cpython/commit/398242a5b61067e3171b11964229fba87a251ef7 commit: 398242a5b61067e3171b11964229fba87a251ef7 branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Eric V. Smith date: 2018-03-01T08:30:13-05:00 summary: Fixed incorrect default value for dataclass unsafe_hash. (GH-5949) (GH-5950) (cherry picked from commit 5da8cfb838fa1bf3529c085c6dce1adf3d1eaf62) Co-authored-by: Eric V. Smith files: M Lib/dataclasses.py diff --git a/Lib/dataclasses.py b/Lib/dataclasses.py index 54478fec93df..b55a497db302 100644 --- a/Lib/dataclasses.py +++ b/Lib/dataclasses.py @@ -745,7 +745,7 @@ def _process_class(cls, repr, eq, order, unsafe_hash, init, frozen): # underscore. The presence of _cls is used to detect if this # decorator is being called with parameters or not. def dataclass(_cls=None, *, init=True, repr=True, eq=True, order=False, - unsafe_hash=None, frozen=False): + unsafe_hash=False, frozen=False): """Returns the same class as was passed in, with dunder methods added based on the fields defined in the class. @@ -880,7 +880,7 @@ def _astuple_inner(obj, tuple_factory): def make_dataclass(cls_name, fields, *, bases=(), namespace=None, init=True, - repr=True, eq=True, order=False, unsafe_hash=None, + repr=True, eq=True, order=False, unsafe_hash=False, frozen=False): """Return a new dynamically created dataclass. From webhook-mailer at python.org Thu Mar 1 13:48:13 2018 From: webhook-mailer at python.org (Alexander Belopolsky) Date: Thu, 01 Mar 2018 18:48:13 -0000 Subject: [Python-checkins] Add What's new entry for datetime.fromisoformat (GH-5559) (GH-5939) Message-ID: https://github.com/python/cpython/commit/0e06be836ca0d578cf9fc0c68979eb682c00f89c commit: 0e06be836ca0d578cf9fc0c68979eb682c00f89c branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Alexander Belopolsky date: 2018-03-01T13:48:10-05:00 summary: Add What's new entry for datetime.fromisoformat (GH-5559) (GH-5939) Documents bpo-15873 (cherry picked from commit 22864bc8e4a076bbac748ccda6c27f1ec41b53e7) Co-authored-by: Paul Ganssle files: M Doc/whatsnew/3.7.rst diff --git a/Doc/whatsnew/3.7.rst b/Doc/whatsnew/3.7.rst index 8fa39abc6ace..46f4f13aab47 100644 --- a/Doc/whatsnew/3.7.rst +++ b/Doc/whatsnew/3.7.rst @@ -502,6 +502,14 @@ Added support for the Blowfish method. The :func:`~crypt.mksalt` function now allows to specify the number of rounds for hashing. (Contributed by Serhiy Storchaka in :issue:`31702`.) +datetime +-------- + +Added the :func:`datetime.datetime.fromisoformat` method, which constructs a +:class:`datetime.datetime` object from a string in one of the formats output +by :func:`datetime.datetime.isoformat`. (Contributed by Paul Ganssle in +:issue:`15873`.) + dis --- From webhook-mailer at python.org Thu Mar 1 16:02:59 2018 From: webhook-mailer at python.org (Ned Deily) Date: Thu, 01 Mar 2018 21:02:59 -0000 Subject: [Python-checkins] bpo-30607: Use external python-doc-theme (GH-2017) Message-ID: https://github.com/python/cpython/commit/bf63e8d55fd2853df3bb99d66de7f428107aadb3 commit: bf63e8d55fd2853df3bb99d66de7f428107aadb3 branch: master author: Jon Wayne Parrott committer: Ned Deily date: 2018-03-01T16:02:50-05:00 summary: bpo-30607: Use external python-doc-theme (GH-2017) files: A Doc/tools/static/changelog_search.js A Misc/NEWS.d/next/Documentation/2018-01-25-13-58-49.bpo-30607.4dXxiq.rst D Doc/tools/pydoctheme/static/pydoctheme.css D Doc/tools/pydoctheme/theme.conf D Doc/tools/static/copybutton.js D Doc/tools/static/py.png D Doc/tools/static/sidebar.js M .travis.yml M Doc/Makefile M Doc/README.rst M Doc/conf.py M Doc/tools/templates/layout.html diff --git a/.travis.yml b/.travis.yml index 292b9c687638..af3cdf54654d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -49,7 +49,8 @@ matrix: - cd Doc # Sphinx is pinned so that new versions that introduce new warnings won't suddenly cause build failures. # (Updating the version is fine as long as no warnings are raised by doing so.) - - python -m pip install sphinx~=1.6.1 blurb + # The theme used by the docs is stored seperately, so we need to install that as well. + - python -m pip install sphinx~=1.6.1 blurb python-docs-theme script: - make check suspicious html SPHINXOPTS="-q -W -j4" - os: osx diff --git a/Doc/Makefile b/Doc/Makefile index 307d1e0e7de1..042f960b93e7 100644 --- a/Doc/Makefile +++ b/Doc/Makefile @@ -123,7 +123,7 @@ clean: venv: $(PYTHON) -m venv $(VENVDIR) - $(VENVDIR)/bin/python3 -m pip install -U Sphinx blurb + $(VENVDIR)/bin/python3 -m pip install -U Sphinx blurb python-docs-theme @echo "The venv has been created in the $(VENVDIR) directory" dist: diff --git a/Doc/README.rst b/Doc/README.rst index a29d1f3a708a..b34916040eb9 100644 --- a/Doc/README.rst +++ b/Doc/README.rst @@ -20,11 +20,11 @@ tree but are maintained separately and are available from * `Sphinx `_ * `blurb `_ +* `python-docs-theme `_ The easiest way to install these tools is to create a virtual environment and install the tools into there. - Using make ---------- diff --git a/Doc/conf.py b/Doc/conf.py index 19a2f7d67ff8..d8efce035c9c 100644 --- a/Doc/conf.py +++ b/Doc/conf.py @@ -46,9 +46,13 @@ # ----------------------- # Use our custom theme. -html_theme = 'pydoctheme' +html_theme = 'python_docs_theme' html_theme_path = ['tools'] -html_theme_options = {'collapsiblesidebar': True} +html_theme_options = { + 'collapsiblesidebar': True, + 'issues_url': 'https://docs.python.org/3/bugs.html', + 'root_include_title': False # We use the version switcher instead. +} # Short title used e.g. for HTML tags. html_short_title = '%s Documentation' % release diff --git a/Doc/tools/pydoctheme/static/pydoctheme.css b/Doc/tools/pydoctheme/static/pydoctheme.css deleted file mode 100644 index 1d5c18e5f623..000000000000 --- a/Doc/tools/pydoctheme/static/pydoctheme.css +++ /dev/null @@ -1,194 +0,0 @@ - at import url("default.css"); - -body { - background-color: white; - margin-left: 1em; - margin-right: 1em; -} - -div.related { - margin-bottom: 1.2em; - padding: 0.5em 0; - border-top: 1px solid #ccc; - margin-top: 0.5em; -} - -div.related a:hover { - color: #0095C4; -} - -div.related:first-child { - border-top: 0; - border-bottom: 1px solid #ccc; -} - -.inline-search { - display: inline; -} -form.inline-search input { - display: inline; -} -form.inline-search input[type="submit"] { - width: 30px; -} - -div.sphinxsidebar { - background-color: #eeeeee; - border-radius: 5px; - line-height: 130%; - font-size: smaller; -} - -div.sphinxsidebar h3, div.sphinxsidebar h4 { - margin-top: 1.5em; -} - -div.sphinxsidebarwrapper > h3:first-child { - margin-top: 0.2em; -} - -div.sphinxsidebarwrapper > ul > li > ul > li { - margin-bottom: 0.4em; -} - -div.sphinxsidebar a:hover { - color: #0095C4; -} - -form.inline-search input, -div.sphinxsidebar input { - font-family: 'Lucida Grande',Arial,sans-serif; - border: 1px solid #999999; - font-size: smaller; - border-radius: 3px; -} - -div.sphinxsidebar input[type=text] { - max-width: 150px; -} - -div.body { - padding: 0 0 0 1.2em; -} - -div.body p { - line-height: 140%; -} - -div.body h1, div.body h2, div.body h3, div.body h4, div.body h5, div.body h6 { - margin: 0; - border: 0; - padding: 0.3em 0; -} - -div.body hr { - border: 0; - background-color: #ccc; - height: 1px; -} - -div.body pre { - border-radius: 3px; - border: 1px solid #ac9; -} - -div.body div.admonition, div.body div.impl-detail { - border-radius: 3px; -} - -div.body div.impl-detail > p { - margin: 0; -} - -div.body div.seealso { - border: 1px solid #dddd66; -} - -div.body a { - color: #0072aa; -} - -div.body a:visited { - color: #6363bb; -} - -div.body a:hover { - color: #00B0E4; -} - -tt, code, pre { - font-family: monospace, sans-serif; - font-size: 96.5%; -} - -div.body tt, div.body code { - border-radius: 3px; -} - -div.body tt.descname, div.body code.descname { - font-size: 120%; -} - -div.body tt.xref, div.body a tt, div.body code.xref, div.body a code { - font-weight: normal; -} - -.deprecated { - border-radius: 3px; -} - -table.docutils { - border: 1px solid #ddd; - min-width: 20%; - border-radius: 3px; - margin-top: 10px; - margin-bottom: 10px; -} - -table.docutils td, table.docutils th { - border: 1px solid #ddd !important; - border-radius: 3px; -} - -table p, table li { - text-align: left !important; -} - -table.docutils th { - background-color: #eee; - padding: 0.3em 0.5em; -} - -table.docutils td { - background-color: white; - padding: 0.3em 0.5em; -} - -table.footnote, table.footnote td { - border: 0 !important; -} - -div.footer { - line-height: 150%; - margin-top: -2em; - text-align: right; - width: auto; - margin-right: 10px; -} - -div.footer a:hover { - color: #0095C4; -} - -.refcount { - color: #060; -} - -.stableabi { - color: #229; -} - -.highlight { - background: none !important; -} - diff --git a/Doc/tools/pydoctheme/theme.conf b/Doc/tools/pydoctheme/theme.conf deleted file mode 100644 index 0c438816740e..000000000000 --- a/Doc/tools/pydoctheme/theme.conf +++ /dev/null @@ -1,23 +0,0 @@ -[theme] -inherit = default -stylesheet = pydoctheme.css -pygments_style = sphinx - -[options] -bodyfont = 'Lucida Grande', Arial, sans-serif -headfont = 'Lucida Grande', Arial, sans-serif -footerbgcolor = white -footertextcolor = #555555 -relbarbgcolor = white -relbartextcolor = #666666 -relbarlinkcolor = #444444 -sidebarbgcolor = white -sidebartextcolor = #444444 -sidebarlinkcolor = #444444 -bgcolor = white -textcolor = #222222 -linkcolor = #0090c0 -visitedlinkcolor = #00608f -headtextcolor = #1a1a1a -headbgcolor = white -headlinkcolor = #aaaaaa diff --git a/Doc/tools/static/changelog_search.js b/Doc/tools/static/changelog_search.js new file mode 100644 index 000000000000..c881a9bd4c84 --- /dev/null +++ b/Doc/tools/static/changelog_search.js @@ -0,0 +1,53 @@ +$(document).ready(function() { + // add the search form and bind the events + $('h1').after([ + '<p>Filter entries by content:', + '<input type="text" value="" id="searchbox" style="width: 50%">', + '<input type="submit" id="searchbox-submit" value="Filter"></p>' + ].join('\n')); + + function dofilter() { + try { + var query = new RegExp($('#searchbox').val(), 'i'); + } + catch (e) { + return; // not a valid regex (yet) + } + // find headers for the versions (What's new in Python X.Y.Z?) + $('#changelog h2').each(function(index1, h2) { + var h2_parent = $(h2).parent(); + var sections_found = 0; + // find headers for the sections (Core, Library, etc.) + h2_parent.find('h3').each(function(index2, h3) { + var h3_parent = $(h3).parent(); + var entries_found = 0; + // find all the entries + h3_parent.find('li').each(function(index3, li) { + var li = $(li); + // check if the query matches the entry + if (query.test(li.text())) { + li.show(); + entries_found++; + } + else { + li.hide(); + } + }); + // if there are entries, show the section, otherwise hide it + if (entries_found > 0) { + h3_parent.show(); + sections_found++; + } + else { + h3_parent.hide(); + } + }); + if (sections_found > 0) + h2_parent.show(); + else + h2_parent.hide(); + }); + } + $('#searchbox').keyup(dofilter); + $('#searchbox-submit').click(dofilter); +}); diff --git a/Doc/tools/static/copybutton.js b/Doc/tools/static/copybutton.js deleted file mode 100644 index 716c9e472f4b..000000000000 --- a/Doc/tools/static/copybutton.js +++ /dev/null @@ -1,62 +0,0 @@ -$(document).ready(function() { - /* Add a [>>>] button on the top-right corner of code samples to hide - * the >>> and ... prompts and the output and thus make the code - * copyable. */ - var div = $('.highlight-python .highlight,' + - '.highlight-python3 .highlight') - var pre = div.find('pre'); - - // get the styles from the current theme - pre.parent().parent().css('position', 'relative'); - var hide_text = 'Hide the prompts and output'; - var show_text = 'Show the prompts and output'; - var border_width = pre.css('border-top-width'); - var border_style = pre.css('border-top-style'); - var border_color = pre.css('border-top-color'); - var button_styles = { - 'cursor':'pointer', 'position': 'absolute', 'top': '0', 'right': '0', - 'border-color': border_color, 'border-style': border_style, - 'border-width': border_width, 'color': border_color, 'text-size': '75%', - 'font-family': 'monospace', 'padding-left': '0.2em', 'padding-right': '0.2em', - 'border-radius': '0 3px 0 0' - } - - // create and add the button to all the code blocks that contain >>> - div.each(function(index) { - var jthis = $(this); - if (jthis.find('.gp').length > 0) { - var button = $('<span class="copybutton">>>></span>'); - button.css(button_styles) - button.attr('title', hide_text); - button.data('hidden', 'false'); - jthis.prepend(button); - } - // tracebacks (.gt) contain bare text elements that need to be - // wrapped in a span to work with .nextUntil() (see later) - jthis.find('pre:has(.gt)').contents().filter(function() { - return ((this.nodeType == 3) && (this.data.trim().length > 0)); - }).wrap('<span>'); - }); - - // define the behavior of the button when it's clicked - $('.copybutton').click(function(e){ - e.preventDefault(); - var button = $(this); - if (button.data('hidden') === 'false') { - // hide the code output - button.parent().find('.go, .gp, .gt').hide(); - button.next('pre').find('.gt').nextUntil('.gp, .go').css('visibility', 'hidden'); - button.css('text-decoration', 'line-through'); - button.attr('title', show_text); - button.data('hidden', 'true'); - } else { - // show the code output - button.parent().find('.go, .gp, .gt').show(); - button.next('pre').find('.gt').nextUntil('.gp, .go').css('visibility', 'visible'); - button.css('text-decoration', 'none'); - button.attr('title', hide_text); - button.data('hidden', 'false'); - } - }); -}); - diff --git a/Doc/tools/static/py.png b/Doc/tools/static/py.png deleted file mode 100644 index 93e4a02c3d32..000000000000 Binary files a/Doc/tools/static/py.png and /dev/null differ diff --git a/Doc/tools/static/sidebar.js b/Doc/tools/static/sidebar.js deleted file mode 100644 index e8d58f4bfaac..000000000000 --- a/Doc/tools/static/sidebar.js +++ /dev/null @@ -1,193 +0,0 @@ -/* - * sidebar.js - * ~~~~~~~~~~ - * - * This script makes the Sphinx sidebar collapsible and implements intelligent - * scrolling. - * - * .sphinxsidebar contains .sphinxsidebarwrapper. This script adds in - * .sphixsidebar, after .sphinxsidebarwrapper, the #sidebarbutton used to - * collapse and expand the sidebar. - * - * When the sidebar is collapsed the .sphinxsidebarwrapper is hidden and the - * width of the sidebar and the margin-left of the document are decreased. - * When the sidebar is expanded the opposite happens. This script saves a - * per-browser/per-session cookie used to remember the position of the sidebar - * among the pages. Once the browser is closed the cookie is deleted and the - * position reset to the default (expanded). - * - * :copyright: Copyright 2007-2011 by the Sphinx team, see AUTHORS. - * :license: BSD, see LICENSE for details. - * - */ - -$(function() { - // global elements used by the functions. - // the 'sidebarbutton' element is defined as global after its - // creation, in the add_sidebar_button function - var jwindow = $(window); - var jdocument = $(document); - var bodywrapper = $('.bodywrapper'); - var sidebar = $('.sphinxsidebar'); - var sidebarwrapper = $('.sphinxsidebarwrapper'); - - // original margin-left of the bodywrapper and width of the sidebar - // with the sidebar expanded - var bw_margin_expanded = bodywrapper.css('margin-left'); - var ssb_width_expanded = sidebar.width(); - - // margin-left of the bodywrapper and width of the sidebar - // with the sidebar collapsed - var bw_margin_collapsed = '.8em'; - var ssb_width_collapsed = '.8em'; - - // colors used by the current theme - var dark_color = '#AAAAAA'; - var light_color = '#CCCCCC'; - - function get_viewport_height() { - if (window.innerHeight) - return window.innerHeight; - else - return jwindow.height(); - } - - function sidebar_is_collapsed() { - return sidebarwrapper.is(':not(:visible)'); - } - - function toggle_sidebar() { - if (sidebar_is_collapsed()) - expand_sidebar(); - else - collapse_sidebar(); - // adjust the scrolling of the sidebar - scroll_sidebar(); - } - - function collapse_sidebar() { - sidebarwrapper.hide(); - sidebar.css('width', ssb_width_collapsed); - bodywrapper.css('margin-left', bw_margin_collapsed); - sidebarbutton.css({ - 'margin-left': '0', - 'height': bodywrapper.height(), - 'border-radius': '5px' - }); - sidebarbutton.find('span').text('?'); - sidebarbutton.attr('title', _('Expand sidebar')); - document.cookie = 'sidebar=collapsed'; - } - - function expand_sidebar() { - bodywrapper.css('margin-left', bw_margin_expanded); - sidebar.css('width', ssb_width_expanded); - sidebarwrapper.show(); - sidebarbutton.css({ - 'margin-left': ssb_width_expanded-12, - 'height': bodywrapper.height(), - 'border-radius': '0 5px 5px 0' - }); - sidebarbutton.find('span').text('?'); - sidebarbutton.attr('title', _('Collapse sidebar')); - //sidebarwrapper.css({'padding-top': - // Math.max(window.pageYOffset - sidebarwrapper.offset().top, 10)}); - document.cookie = 'sidebar=expanded'; - } - - function add_sidebar_button() { - sidebarwrapper.css({ - 'float': 'left', - 'margin-right': '0', - 'width': ssb_width_expanded - 28 - }); - // create the button - sidebar.append( - '<div id="sidebarbutton"><span>«</span></div>' - ); - var sidebarbutton = $('#sidebarbutton'); - // find the height of the viewport to center the '<<' in the page - var viewport_height = get_viewport_height(); - var sidebar_offset = sidebar.offset().top; - var sidebar_height = Math.max(bodywrapper.height(), sidebar.height()); - sidebarbutton.find('span').css({ - 'display': 'block', - 'position': 'fixed', - 'top': Math.min(viewport_height/2, sidebar_height/2 + sidebar_offset) - 10 - }); - - sidebarbutton.click(toggle_sidebar); - sidebarbutton.attr('title', _('Collapse sidebar')); - sidebarbutton.css({ - 'border-radius': '0 5px 5px 0', - 'color': '#444444', - 'background-color': '#CCCCCC', - 'font-size': '1.2em', - 'cursor': 'pointer', - 'height': sidebar_height, - 'padding-top': '1px', - 'padding-left': '1px', - 'margin-left': ssb_width_expanded - 12 - }); - - sidebarbutton.hover( - function () { - $(this).css('background-color', dark_color); - }, - function () { - $(this).css('background-color', light_color); - } - ); - } - - function set_position_from_cookie() { - if (!document.cookie) - return; - var items = document.cookie.split(';'); - for(var k=0; k<items.length; k++) { - var key_val = items[k].split('='); - var key = key_val[0]; - if (key == 'sidebar') { - var value = key_val[1]; - if ((value == 'collapsed') && (!sidebar_is_collapsed())) - collapse_sidebar(); - else if ((value == 'expanded') && (sidebar_is_collapsed())) - expand_sidebar(); - } - } - } - - add_sidebar_button(); - var sidebarbutton = $('#sidebarbutton'); - set_position_from_cookie(); - - - /* intelligent scrolling */ - function scroll_sidebar() { - var sidebar_height = sidebarwrapper.height(); - var viewport_height = get_viewport_height(); - var offset = sidebar.position()['top']; - var wintop = jwindow.scrollTop(); - var winbot = wintop + viewport_height; - var curtop = sidebarwrapper.position()['top']; - var curbot = curtop + sidebar_height; - // does sidebar fit in window? - if (sidebar_height < viewport_height) { - // yes: easy case -- always keep at the top - sidebarwrapper.css('top', $u.min([$u.max([0, wintop - offset - 10]), - jdocument.height() - sidebar_height - 200])); - } - else { - // no: only scroll if top/bottom edge of sidebar is at - // top/bottom edge of window - if (curtop > wintop && curbot > winbot) { - sidebarwrapper.css('top', $u.max([wintop - offset - 10, 0])); - } - else if (curtop < wintop && curbot < winbot) { - sidebarwrapper.css('top', $u.min([winbot - sidebar_height - offset - 20, - jdocument.height() - sidebar_height - 200])); - } - } - } - jwindow.scroll(scroll_sidebar); -}); diff --git a/Doc/tools/templates/layout.html b/Doc/tools/templates/layout.html index c2106678ac60..37811725d86b 100644 --- a/Doc/tools/templates/layout.html +++ b/Doc/tools/templates/layout.html @@ -1,118 +1,25 @@ {% extends "!layout.html" %} + {% block rootrellink %} - <li><img src="{{ pathto('_static/py.png', 1) }}" alt="" - style="vertical-align: middle; margin-top: -1px"/></li> - <li><a href="https://www.python.org/">Python</a>{{ reldelim1 }}</li> - <li> - {%- if switchers is defined %} - <span class="language_switcher_placeholder">{{ language or 'en' }}</span> - <span class="version_switcher_placeholder">{{ release }}</span> - <a href="{{ pathto('index') }}">{% trans %}Documentation {% endtrans %}</a>{{ reldelim1 }} - {%- else %} - <a href="{{ pathto('index') }}">{{ shorttitle }}</a>{{ reldelim1 }} - {%- endif %} - </li> -{% endblock %} -{%- macro searchbox() %} -{# modified from sphinx/themes/basic/searchbox.html #} - {%- if builder != "htmlhelp" %} - <div class="inline-search" style="display: none" role="search"> - <form class="inline-search" action="{{ pathto('search') }}" method="get"> - <input placeholder="{{ _('Quick search') }}" type="text" name="q" /> - <input type="submit" value="{{ _('Go') }}" /> - <input type="hidden" name="check_keywords" value="yes" /> - <input type="hidden" name="area" value="default" /> - </form> - </div> - <script type="text/javascript">$('.inline-search').show(0);</script> - {%- endif %} -{%- endmacro %} -{% block relbar1 %} {% if builder != 'qthelp' %} {{ relbar() }} {% endif %} {% endblock %} -{% block relbar2 %} {% if builder != 'qthelp' %} {{ relbar() }} {% endif %} {% endblock %} -{% block relbaritems %} - {%- if pagename != "search" and builder != "singlehtml" and builder != "htmlhelp" %} - <li class="right"> - {{ searchbox() }} - {{ reldelim2 }} +{{ super() }} + <li> + {%- if switchers is defined %} + <span class="language_switcher_placeholder">{{ language or 'en' }}</span> + <span class="version_switcher_placeholder">{{ release }}</span> + <a href="{{ pathto('index') }}">{% trans %}Documentation {% endtrans %}</a>{{ reldelim1 }} + {%- else %} + <a href="{{ pathto('index') }}">{{ shorttitle }}</a>{{ reldelim1 }} + {%- endif %} </li> - {%- endif %} {% endblock %} + {% block extrahead %} - <link rel="shortcut icon" type="image/png" href="{{ pathto('_static/py.png', 1) }}" /> <link rel="canonical" href="https://docs.python.org/3/{{pagename}}.html" /> {% if builder != "htmlhelp" %} - {% if not embedded %}<script type="text/javascript" src="{{ pathto('_static/copybutton.js', 1) }}"></script>{% endif %} - {% if switchers is defined and not embedded %}<script type="text/javascript" src="{{ pathto('_static/switchers.js', 1) }}"></script>{% endif %} - {% if pagename == 'whatsnew/changelog' and not embedded %} - <script type="text/javascript"> - $(document).ready(function() { - // add the search form and bind the events - $('h1').after([ - '<p>Filter entries by content:', - '<input type="text" value="" id="searchbox" style="width: 50%">', - '<input type="submit" id="searchbox-submit" value="Filter"></p>' - ].join('\n')); - - function dofilter() { - try { - var query = new RegExp($('#searchbox').val(), 'i'); - } - catch (e) { - return; // not a valid regex (yet) - } - // find headers for the versions (What's new in Python X.Y.Z?) - $('#changelog h2').each(function(index1, h2) { - var h2_parent = $(h2).parent(); - var sections_found = 0; - // find headers for the sections (Core, Library, etc.) - h2_parent.find('h3').each(function(index2, h3) { - var h3_parent = $(h3).parent(); - var entries_found = 0; - // find all the entries - h3_parent.find('li').each(function(index3, li) { - var li = $(li); - // check if the query matches the entry - if (query.test(li.text())) { - li.show(); - entries_found++; - } - else { - li.hide(); - } - }); - // if there are entries, show the section, otherwise hide it - if (entries_found > 0) { - h3_parent.show(); - sections_found++; - } - else { - h3_parent.hide(); - } - }); - if (sections_found > 0) - h2_parent.show(); - else - h2_parent.hide(); - }); - } - $('#searchbox').keyup(dofilter); - $('#searchbox-submit').click(dofilter); - }); - </script> - {% endif %} + {% if switchers is defined and not embedded %} + <script type="text/javascript" src="{{ pathto('_static/switchers.js', 1) }}"></script>{% endif %} + {% if pagename == 'whatsnew/changelog' and not embedded %} + <script type="text/javascript" src="{{ pathto('_static/changelog_search.js', 1) }}"></script>{% endif %} {% endif %} {{ super() }} {% endblock %} -{% block footer %} - <div class="footer"> - © <a href="{{ pathto('copyright') }}">{% trans %}Copyright{% endtrans %}</a> {{ copyright|e }}. - <br /> - {% trans %}The Python Software Foundation is a non-profit corporation.{% endtrans %} - <a href="https://www.python.org/psf/donations/">{% trans %}Please donate.{% endtrans %}</a> - <br /> - {% trans last_updated=last_updated|e %}Last updated on {{ last_updated }}.{% endtrans %} - {% trans pathto_bugs=pathto('bugs') %}<a href="{{ pathto_bugs }}">Found a bug</a>?{% endtrans %} - <br /> - {% trans sphinx_version=sphinx_version|e %}Created using <a href="http://sphinx.pocoo.org/">Sphinx</a> {{ sphinx_version }}.{% endtrans %} - </div> -{% endblock %} diff --git a/Misc/NEWS.d/next/Documentation/2018-01-25-13-58-49.bpo-30607.4dXxiq.rst b/Misc/NEWS.d/next/Documentation/2018-01-25-13-58-49.bpo-30607.4dXxiq.rst new file mode 100644 index 000000000000..8ff3b7887810 --- /dev/null +++ b/Misc/NEWS.d/next/Documentation/2018-01-25-13-58-49.bpo-30607.4dXxiq.rst @@ -0,0 +1,2 @@ +Use the externalized ``python-docs-theme`` package when building the +documenation. From tjreedy at udel.edu Thu Mar 1 16:56:25 2018 From: tjreedy at udel.edu (Terry Reedy) Date: Thu, 1 Mar 2018 16:56:25 -0500 Subject: [Python-checkins] bpo-30607: Use external python-doc-theme (GH-2017) In-Reply-To: <3zslJW70nHzFqs4@mail.python.org> References: <3zslJW70nHzFqs4@mail.python.org> Message-ID: <a0491a8b-47a1-5410-a9d7-872b57e6c1a6@udel.edu> On 3/1/2018 4:03 PM, Ned Deily wrote: > https://github.com/python/cpython/commit/bf63e8d55fd2853df3bb99d66de7f428107aadb3 > commit: bf63e8d55fd2853df3bb99d66de7f428107aadb3 > branch: master > author: Jon Wayne Parrott <jonwayne at google.com> > committer: Ned Deily <nad at python.org> > date: 2018-03-01T16:02:50-05:00 > summary: > diff --git a/.travis.yml b/.travis.yml > index 292b9c687638..af3cdf54654d 100644 > --- a/.travis.yml > +++ b/.travis.yml > @@ -49,7 +49,8 @@ matrix: > - cd Doc > # Sphinx is pinned so that new versions that introduce new warnings won't suddenly cause build failures. > # (Updating the version is fine as long as no warnings are raised by doing so.) > - - python -m pip install sphinx~=1.6.1 blurb > + # The theme used by the docs is stored seperately, so we need to install that as well. /seperately/separately/ From solipsis at pitrou.net Fri Mar 2 04:14:09 2018 From: solipsis at pitrou.net (solipsis at pitrou.net) Date: Fri, 02 Mar 2018 09:14:09 +0000 Subject: [Python-checkins] Daily reference leaks (4243df51fe43): sum=4 Message-ID: <20180302091409.1.8D0AEFF7FC9A1F4C@psf.io> results for 4243df51fe43 on branch "default" -------------------------------------------- test_functools leaked [0, 3, 1] memory blocks, sum=4 test_multiprocessing_fork leaked [0, -2, 2] memory blocks, sum=0 Command line was: ['./python', '-m', 'test.regrtest', '-uall', '-R', '3:3:/home/psf-users/antoine/refleaks/reflogPujVAB', '--timeout', '7200'] From webhook-mailer at python.org Fri Mar 2 04:53:59 2018 From: webhook-mailer at python.org (Serhiy Storchaka) Date: Fri, 02 Mar 2018 09:53:59 -0000 Subject: [Python-checkins] bpo-32964: Reuse a testing implementation of the path protocol in tests. (#5930) Message-ID: <mailman.103.1519984440.2831.python-checkins@python.org> https://github.com/python/cpython/commit/b21d155f57d284aecf9092a9bd24258293965c2f commit: b21d155f57d284aecf9092a9bd24258293965c2f branch: master author: Serhiy Storchaka <storchaka at gmail.com> committer: GitHub <noreply at github.com> date: 2018-03-02T11:53:51+02:00 summary: bpo-32964: Reuse a testing implementation of the path protocol in tests. (#5930) files: M Doc/library/test.rst M Lib/test/support/__init__.py M Lib/test/test_compile.py M Lib/test/test_genericpath.py M Lib/test/test_io.py M Lib/test/test_ntpath.py M Lib/test/test_os.py M Lib/test/test_pathlib.py M Lib/test/test_posixpath.py M Lib/test/test_shutil.py M Lib/test/test_subprocess.py diff --git a/Doc/library/test.rst b/Doc/library/test.rst index a366fe2556b5..6041f529c16f 100644 --- a/Doc/library/test.rst +++ b/Doc/library/test.rst @@ -1293,6 +1293,13 @@ The :mod:`test.support` module defines the following classes: Class for logging support. +.. class:: FakePath(path) + + Simple :term:`path-like object`. It implements the :meth:`__fspath__` + method which just returns the *path* argument. If *path* is an exception, + it will be raised in :meth:`!__fspath__`. + + :mod:`test.support.script_helper` --- Utilities for the Python execution tests ============================================================================== diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index b4269f4a5b70..2c2f888352e8 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -2840,3 +2840,21 @@ def restore(self): def with_pymalloc(): import _testcapi return _testcapi.WITH_PYMALLOC + + +class FakePath: + """Simple implementing of the path protocol. + """ + def __init__(self, path): + self.path = path + + def __repr__(self): + return f'<FakePath {self.path!r}>' + + def __fspath__(self): + if (isinstance(self.path, BaseException) or + isinstance(self.path, type) and + issubclass(self.path, BaseException)): + raise self.path + else: + return self.path diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py index 4617a126abee..acebdbdc4636 100644 --- a/Lib/test/test_compile.py +++ b/Lib/test/test_compile.py @@ -6,7 +6,7 @@ import tempfile import types from test import support -from test.support import script_helper +from test.support import script_helper, FakePath class TestSpecifics(unittest.TestCase): @@ -663,13 +663,7 @@ def check_different_constants(const1, const2): def test_path_like_objects(self): # An implicit test for PyUnicode_FSDecoder(). - class PathLike: - def __init__(self, path): - self._path = path - def __fspath__(self): - return self._path - - compile("42", PathLike("test_compile_pathlike"), "single") + compile("42", FakePath("test_compile_pathlike"), "single") def test_stack_overflow(self): # bpo-31113: Stack overflow when compile a long sequence of diff --git a/Lib/test/test_genericpath.py b/Lib/test/test_genericpath.py index ad5a59c44ed7..9ed53902dc12 100644 --- a/Lib/test/test_genericpath.py +++ b/Lib/test/test_genericpath.py @@ -9,6 +9,7 @@ import warnings from test import support from test.support.script_helper import assert_python_ok +from test.support import FakePath def create_file(filename, data=b'foo'): @@ -493,18 +494,9 @@ def test_import(self): class PathLikeTests(unittest.TestCase): - class PathLike: - def __init__(self, path=''): - self.path = path - def __fspath__(self): - if isinstance(self.path, BaseException): - raise self.path - else: - return self.path - def setUp(self): self.file_name = support.TESTFN.lower() - self.file_path = self.PathLike(support.TESTFN) + self.file_path = FakePath(support.TESTFN) self.addCleanup(support.unlink, self.file_name) create_file(self.file_name, b"test_genericpath.PathLikeTests") diff --git a/Lib/test/test_io.py b/Lib/test/test_io.py index 4be5631ab446..343c5a444072 100644 --- a/Lib/test/test_io.py +++ b/Lib/test/test_io.py @@ -37,6 +37,7 @@ from itertools import cycle, count from test import support from test.support.script_helper import assert_python_ok, run_python_until_end +from test.support import FakePath import codecs import io # C implementation of io @@ -891,13 +892,6 @@ def read(self, size): self.assertEqual(bytes(buffer), b"12345") def test_fspath_support(self): - class PathLike: - def __init__(self, path): - self.path = path - - def __fspath__(self): - return self.path - def check_path_succeeds(path): with self.open(path, "w") as f: f.write("egg\n") @@ -905,16 +899,25 @@ def check_path_succeeds(path): with self.open(path, "r") as f: self.assertEqual(f.read(), "egg\n") - check_path_succeeds(PathLike(support.TESTFN)) - check_path_succeeds(PathLike(support.TESTFN.encode('utf-8'))) + check_path_succeeds(FakePath(support.TESTFN)) + check_path_succeeds(FakePath(support.TESTFN.encode('utf-8'))) + + with self.open(support.TESTFN, "w") as f: + bad_path = FakePath(f.fileno()) + with self.assertRaises(TypeError): + self.open(bad_path, 'w') - bad_path = PathLike(TypeError) + bad_path = FakePath(None) with self.assertRaises(TypeError): self.open(bad_path, 'w') + bad_path = FakePath(FloatingPointError) + with self.assertRaises(FloatingPointError): + self.open(bad_path, 'w') + # ensure that refcounting is correct with some error conditions with self.assertRaisesRegex(ValueError, 'read/write/append mode'): - self.open(PathLike(support.TESTFN), 'rwxa') + self.open(FakePath(support.TESTFN), 'rwxa') def test_RawIOBase_readall(self): # Exercise the default unlimited RawIOBase.read() and readall() diff --git a/Lib/test/test_ntpath.py b/Lib/test/test_ntpath.py index 1eec26b20013..2d48be844242 100644 --- a/Lib/test/test_ntpath.py +++ b/Lib/test/test_ntpath.py @@ -3,7 +3,7 @@ import sys import unittest import warnings -from test.support import TestFailed +from test.support import TestFailed, FakePath from test import support, test_genericpath from tempfile import TemporaryFile @@ -459,18 +459,9 @@ class PathLikeTests(unittest.TestCase): path = ntpath - class PathLike: - def __init__(self, path=''): - self.path = path - def __fspath__(self): - if isinstance(self.path, BaseException): - raise self.path - else: - return self.path - def setUp(self): self.file_name = support.TESTFN.lower() - self.file_path = self.PathLike(support.TESTFN) + self.file_path = FakePath(support.TESTFN) self.addCleanup(support.unlink, self.file_name) with open(self.file_name, 'xb', 0) as file: file.write(b"test_ntpath.PathLikeTests") @@ -485,7 +476,7 @@ def test_path_isabs(self): self.assertPathEqual(self.path.isabs) def test_path_join(self): - self.assertEqual(self.path.join('a', self.PathLike('b'), 'c'), + self.assertEqual(self.path.join('a', FakePath('b'), 'c'), self.path.join('a', 'b', 'c')) def test_path_split(self): diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index 77e4a008ae61..4f8a2a7e19d5 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -61,7 +61,7 @@ INT_MAX = PY_SSIZE_T_MAX = sys.maxsize from test.support.script_helper import assert_python_ok -from test.support import unix_shell +from test.support import unix_shell, FakePath root_in_posix = False @@ -85,21 +85,6 @@ def requires_os_func(name): return unittest.skipUnless(hasattr(os, name), 'requires os.%s' % name) -class _PathLike(os.PathLike): - - def __init__(self, path=""): - self.path = path - - def __str__(self): - return str(self.path) - - def __fspath__(self): - if isinstance(self.path, BaseException): - raise self.path - else: - return self.path - - def create_file(filename, content=b'content'): with open(filename, "xb", 0) as fp: fp.write(content) @@ -942,15 +927,14 @@ def test_walk_prune(self, walk_path=None): dirs.remove('SUB1') self.assertEqual(len(all), 2) - self.assertEqual(all[0], - (str(walk_path), ["SUB2"], ["tmp1"])) + self.assertEqual(all[0], (self.walk_path, ["SUB2"], ["tmp1"])) all[1][-1].sort() all[1][1].sort() self.assertEqual(all[1], self.sub2_tree) def test_file_like_path(self): - self.test_walk_prune(_PathLike(self.walk_path)) + self.test_walk_prune(FakePath(self.walk_path)) def test_walk_bottom_up(self): # Walk bottom-up. @@ -2288,7 +2272,7 @@ def test_getppid(self): def test_waitpid(self): args = [sys.executable, '-c', 'pass'] # Add an implicit test for PyUnicode_FSConverter(). - pid = os.spawnv(os.P_NOWAIT, _PathLike(args[0]), args) + pid = os.spawnv(os.P_NOWAIT, FakePath(args[0]), args) status = os.waitpid(pid, 0) self.assertEqual(status, (pid, 0)) @@ -3129,13 +3113,13 @@ def test_path_t_converter(self): bytes_fspath = bytes_filename = None else: bytes_filename = support.TESTFN.encode('ascii') - bytes_fspath = _PathLike(bytes_filename) - fd = os.open(_PathLike(str_filename), os.O_WRONLY|os.O_CREAT) + bytes_fspath = FakePath(bytes_filename) + fd = os.open(FakePath(str_filename), os.O_WRONLY|os.O_CREAT) self.addCleanup(support.unlink, support.TESTFN) self.addCleanup(os.close, fd) - int_fspath = _PathLike(fd) - str_fspath = _PathLike(str_filename) + int_fspath = FakePath(fd) + str_fspath = FakePath(str_filename) for name, allow_fd, extra_args, cleanup_fn in self.functions: with self.subTest(name=name): @@ -3540,16 +3524,16 @@ def test_return_string(self): def test_fsencode_fsdecode(self): for p in "path/like/object", b"path/like/object": - pathlike = _PathLike(p) + pathlike = FakePath(p) self.assertEqual(p, self.fspath(pathlike)) self.assertEqual(b"path/like/object", os.fsencode(pathlike)) self.assertEqual("path/like/object", os.fsdecode(pathlike)) def test_pathlike(self): - self.assertEqual('#feelthegil', self.fspath(_PathLike('#feelthegil'))) - self.assertTrue(issubclass(_PathLike, os.PathLike)) - self.assertTrue(isinstance(_PathLike(), os.PathLike)) + self.assertEqual('#feelthegil', self.fspath(FakePath('#feelthegil'))) + self.assertTrue(issubclass(FakePath, os.PathLike)) + self.assertTrue(isinstance(FakePath('x'), os.PathLike)) def test_garbage_in_exception_out(self): vapor = type('blah', (), {}) @@ -3561,14 +3545,14 @@ def test_argument_required(self): def test_bad_pathlike(self): # __fspath__ returns a value other than str or bytes. - self.assertRaises(TypeError, self.fspath, _PathLike(42)) + self.assertRaises(TypeError, self.fspath, FakePath(42)) # __fspath__ attribute that is not callable. c = type('foo', (), {}) c.__fspath__ = 1 self.assertRaises(TypeError, self.fspath, c()) # __fspath__ raises an exception. self.assertRaises(ZeroDivisionError, self.fspath, - _PathLike(ZeroDivisionError())) + FakePath(ZeroDivisionError())) class TimesTests(unittest.TestCase): diff --git a/Lib/test/test_pathlib.py b/Lib/test/test_pathlib.py index e56e0d20844e..53215550b435 100644 --- a/Lib/test/test_pathlib.py +++ b/Lib/test/test_pathlib.py @@ -11,7 +11,7 @@ from unittest import mock from test import support -TESTFN = support.TESTFN +from test.support import TESTFN, FakePath try: import grp, pwd @@ -191,18 +191,15 @@ def test_constructor_common(self): P = self.cls p = P('a') self.assertIsInstance(p, P) - class PathLike: - def __fspath__(self): - return "a/b/c" P('a', 'b', 'c') P('/a', 'b', 'c') P('a/b/c') P('/a/b/c') - P(PathLike()) + P(FakePath("a/b/c")) self.assertEqual(P(P('a')), P('a')) self.assertEqual(P(P('a'), 'b'), P('a/b')) self.assertEqual(P(P('a'), P('b')), P('a/b')) - self.assertEqual(P(P('a'), P('b'), P('c')), P(PathLike())) + self.assertEqual(P(P('a'), P('b'), P('c')), P(FakePath("a/b/c"))) def _check_str_subclass(self, *args): # Issue #21127: it should be possible to construct a PurePath object diff --git a/Lib/test/test_posixpath.py b/Lib/test/test_posixpath.py index 8a1e33b0c899..96b267cd45fd 100644 --- a/Lib/test/test_posixpath.py +++ b/Lib/test/test_posixpath.py @@ -4,6 +4,7 @@ import warnings from posixpath import realpath, abspath, dirname, basename from test import support, test_genericpath +from test.support import FakePath try: import posix @@ -600,18 +601,9 @@ class PathLikeTests(unittest.TestCase): path = posixpath - class PathLike: - def __init__(self, path=''): - self.path = path - def __fspath__(self): - if isinstance(self.path, BaseException): - raise self.path - else: - return self.path - def setUp(self): self.file_name = support.TESTFN.lower() - self.file_path = self.PathLike(support.TESTFN) + self.file_path = FakePath(support.TESTFN) self.addCleanup(support.unlink, self.file_name) with open(self.file_name, 'xb', 0) as file: file.write(b"test_posixpath.PathLikeTests") @@ -626,7 +618,7 @@ def test_path_isabs(self): self.assertPathEqual(self.path.isabs) def test_path_join(self): - self.assertEqual(self.path.join('a', self.PathLike('b'), 'c'), + self.assertEqual(self.path.join('a', FakePath('b'), 'c'), self.path.join('a', 'b', 'c')) def test_path_split(self): diff --git a/Lib/test/test_shutil.py b/Lib/test/test_shutil.py index f3cf43e50516..2cb2f14643e1 100644 --- a/Lib/test/test_shutil.py +++ b/Lib/test/test_shutil.py @@ -22,7 +22,7 @@ import zipfile from test import support -from test.support import TESTFN +from test.support import TESTFN, FakePath TESTFN2 = TESTFN + "2" @@ -1232,14 +1232,7 @@ def test_register_archive_format(self): def check_unpack_archive(self, format): self.check_unpack_archive_with_converter(format, lambda path: path) self.check_unpack_archive_with_converter(format, pathlib.Path) - - class MyPath: - def __init__(self, path): - self.path = path - def __fspath__(self): - return self.path - - self.check_unpack_archive_with_converter(format, MyPath) + self.check_unpack_archive_with_converter(format, FakePath) def check_unpack_archive_with_converter(self, format, converter): root_dir, base_dir = self._create_files() diff --git a/Lib/test/test_subprocess.py b/Lib/test/test_subprocess.py index 46cb5f117e84..ddee3b94fed9 100644 --- a/Lib/test/test_subprocess.py +++ b/Lib/test/test_subprocess.py @@ -17,6 +17,7 @@ import threading import gc import textwrap +from test.support import FakePath try: import ctypes @@ -359,12 +360,7 @@ def test_cwd(self): def test_cwd_with_pathlike(self): temp_dir = tempfile.gettempdir() temp_dir = self._normalize_cwd(temp_dir) - - class _PathLikeObj: - def __fspath__(self): - return temp_dir - - self._assert_cwd(temp_dir, sys.executable, cwd=_PathLikeObj()) + self._assert_cwd(temp_dir, sys.executable, cwd=FakePath(temp_dir)) @unittest.skipIf(mswindows, "pending resolution of issue #15533") def test_cwd_with_relative_arg(self): From webhook-mailer at python.org Fri Mar 2 05:17:57 2018 From: webhook-mailer at python.org (Miss Islington (bot)) Date: Fri, 02 Mar 2018 10:17:57 -0000 Subject: [Python-checkins] bpo-32964: Reuse a testing implementation of the path protocol in tests. (GH-5930) Message-ID: <mailman.104.1519985879.2831.python-checkins@python.org> https://github.com/python/cpython/commit/a13b65422a1078104e9f53ad41945ea380a80798 commit: a13b65422a1078104e9f53ad41945ea380a80798 branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: GitHub <noreply at github.com> date: 2018-03-02T02:17:51-08:00 summary: bpo-32964: Reuse a testing implementation of the path protocol in tests. (GH-5930) (cherry picked from commit b21d155f57d284aecf9092a9bd24258293965c2f) Co-authored-by: Serhiy Storchaka <storchaka at gmail.com> files: M Doc/library/test.rst M Lib/test/support/__init__.py M Lib/test/test_compile.py M Lib/test/test_genericpath.py M Lib/test/test_io.py M Lib/test/test_ntpath.py M Lib/test/test_os.py M Lib/test/test_pathlib.py M Lib/test/test_posixpath.py M Lib/test/test_shutil.py M Lib/test/test_subprocess.py diff --git a/Doc/library/test.rst b/Doc/library/test.rst index a366fe2556b5..6041f529c16f 100644 --- a/Doc/library/test.rst +++ b/Doc/library/test.rst @@ -1293,6 +1293,13 @@ The :mod:`test.support` module defines the following classes: Class for logging support. +.. class:: FakePath(path) + + Simple :term:`path-like object`. It implements the :meth:`__fspath__` + method which just returns the *path* argument. If *path* is an exception, + it will be raised in :meth:`!__fspath__`. + + :mod:`test.support.script_helper` --- Utilities for the Python execution tests ============================================================================== diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index 78298ca434a1..16cee96108a3 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -2840,3 +2840,21 @@ def restore(self): def with_pymalloc(): import _testcapi return _testcapi.WITH_PYMALLOC + + +class FakePath: + """Simple implementing of the path protocol. + """ + def __init__(self, path): + self.path = path + + def __repr__(self): + return f'<FakePath {self.path!r}>' + + def __fspath__(self): + if (isinstance(self.path, BaseException) or + isinstance(self.path, type) and + issubclass(self.path, BaseException)): + raise self.path + else: + return self.path diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py index 4617a126abee..acebdbdc4636 100644 --- a/Lib/test/test_compile.py +++ b/Lib/test/test_compile.py @@ -6,7 +6,7 @@ import tempfile import types from test import support -from test.support import script_helper +from test.support import script_helper, FakePath class TestSpecifics(unittest.TestCase): @@ -663,13 +663,7 @@ def check_different_constants(const1, const2): def test_path_like_objects(self): # An implicit test for PyUnicode_FSDecoder(). - class PathLike: - def __init__(self, path): - self._path = path - def __fspath__(self): - return self._path - - compile("42", PathLike("test_compile_pathlike"), "single") + compile("42", FakePath("test_compile_pathlike"), "single") def test_stack_overflow(self): # bpo-31113: Stack overflow when compile a long sequence of diff --git a/Lib/test/test_genericpath.py b/Lib/test/test_genericpath.py index ad5a59c44ed7..9ed53902dc12 100644 --- a/Lib/test/test_genericpath.py +++ b/Lib/test/test_genericpath.py @@ -9,6 +9,7 @@ import warnings from test import support from test.support.script_helper import assert_python_ok +from test.support import FakePath def create_file(filename, data=b'foo'): @@ -493,18 +494,9 @@ def test_import(self): class PathLikeTests(unittest.TestCase): - class PathLike: - def __init__(self, path=''): - self.path = path - def __fspath__(self): - if isinstance(self.path, BaseException): - raise self.path - else: - return self.path - def setUp(self): self.file_name = support.TESTFN.lower() - self.file_path = self.PathLike(support.TESTFN) + self.file_path = FakePath(support.TESTFN) self.addCleanup(support.unlink, self.file_name) create_file(self.file_name, b"test_genericpath.PathLikeTests") diff --git a/Lib/test/test_io.py b/Lib/test/test_io.py index 4be5631ab446..343c5a444072 100644 --- a/Lib/test/test_io.py +++ b/Lib/test/test_io.py @@ -37,6 +37,7 @@ from itertools import cycle, count from test import support from test.support.script_helper import assert_python_ok, run_python_until_end +from test.support import FakePath import codecs import io # C implementation of io @@ -891,13 +892,6 @@ def read(self, size): self.assertEqual(bytes(buffer), b"12345") def test_fspath_support(self): - class PathLike: - def __init__(self, path): - self.path = path - - def __fspath__(self): - return self.path - def check_path_succeeds(path): with self.open(path, "w") as f: f.write("egg\n") @@ -905,16 +899,25 @@ def check_path_succeeds(path): with self.open(path, "r") as f: self.assertEqual(f.read(), "egg\n") - check_path_succeeds(PathLike(support.TESTFN)) - check_path_succeeds(PathLike(support.TESTFN.encode('utf-8'))) + check_path_succeeds(FakePath(support.TESTFN)) + check_path_succeeds(FakePath(support.TESTFN.encode('utf-8'))) + + with self.open(support.TESTFN, "w") as f: + bad_path = FakePath(f.fileno()) + with self.assertRaises(TypeError): + self.open(bad_path, 'w') - bad_path = PathLike(TypeError) + bad_path = FakePath(None) with self.assertRaises(TypeError): self.open(bad_path, 'w') + bad_path = FakePath(FloatingPointError) + with self.assertRaises(FloatingPointError): + self.open(bad_path, 'w') + # ensure that refcounting is correct with some error conditions with self.assertRaisesRegex(ValueError, 'read/write/append mode'): - self.open(PathLike(support.TESTFN), 'rwxa') + self.open(FakePath(support.TESTFN), 'rwxa') def test_RawIOBase_readall(self): # Exercise the default unlimited RawIOBase.read() and readall() diff --git a/Lib/test/test_ntpath.py b/Lib/test/test_ntpath.py index 1eec26b20013..2d48be844242 100644 --- a/Lib/test/test_ntpath.py +++ b/Lib/test/test_ntpath.py @@ -3,7 +3,7 @@ import sys import unittest import warnings -from test.support import TestFailed +from test.support import TestFailed, FakePath from test import support, test_genericpath from tempfile import TemporaryFile @@ -459,18 +459,9 @@ class PathLikeTests(unittest.TestCase): path = ntpath - class PathLike: - def __init__(self, path=''): - self.path = path - def __fspath__(self): - if isinstance(self.path, BaseException): - raise self.path - else: - return self.path - def setUp(self): self.file_name = support.TESTFN.lower() - self.file_path = self.PathLike(support.TESTFN) + self.file_path = FakePath(support.TESTFN) self.addCleanup(support.unlink, self.file_name) with open(self.file_name, 'xb', 0) as file: file.write(b"test_ntpath.PathLikeTests") @@ -485,7 +476,7 @@ def test_path_isabs(self): self.assertPathEqual(self.path.isabs) def test_path_join(self): - self.assertEqual(self.path.join('a', self.PathLike('b'), 'c'), + self.assertEqual(self.path.join('a', FakePath('b'), 'c'), self.path.join('a', 'b', 'c')) def test_path_split(self): diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index 77e4a008ae61..4f8a2a7e19d5 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -61,7 +61,7 @@ INT_MAX = PY_SSIZE_T_MAX = sys.maxsize from test.support.script_helper import assert_python_ok -from test.support import unix_shell +from test.support import unix_shell, FakePath root_in_posix = False @@ -85,21 +85,6 @@ def requires_os_func(name): return unittest.skipUnless(hasattr(os, name), 'requires os.%s' % name) -class _PathLike(os.PathLike): - - def __init__(self, path=""): - self.path = path - - def __str__(self): - return str(self.path) - - def __fspath__(self): - if isinstance(self.path, BaseException): - raise self.path - else: - return self.path - - def create_file(filename, content=b'content'): with open(filename, "xb", 0) as fp: fp.write(content) @@ -942,15 +927,14 @@ def test_walk_prune(self, walk_path=None): dirs.remove('SUB1') self.assertEqual(len(all), 2) - self.assertEqual(all[0], - (str(walk_path), ["SUB2"], ["tmp1"])) + self.assertEqual(all[0], (self.walk_path, ["SUB2"], ["tmp1"])) all[1][-1].sort() all[1][1].sort() self.assertEqual(all[1], self.sub2_tree) def test_file_like_path(self): - self.test_walk_prune(_PathLike(self.walk_path)) + self.test_walk_prune(FakePath(self.walk_path)) def test_walk_bottom_up(self): # Walk bottom-up. @@ -2288,7 +2272,7 @@ def test_getppid(self): def test_waitpid(self): args = [sys.executable, '-c', 'pass'] # Add an implicit test for PyUnicode_FSConverter(). - pid = os.spawnv(os.P_NOWAIT, _PathLike(args[0]), args) + pid = os.spawnv(os.P_NOWAIT, FakePath(args[0]), args) status = os.waitpid(pid, 0) self.assertEqual(status, (pid, 0)) @@ -3129,13 +3113,13 @@ def test_path_t_converter(self): bytes_fspath = bytes_filename = None else: bytes_filename = support.TESTFN.encode('ascii') - bytes_fspath = _PathLike(bytes_filename) - fd = os.open(_PathLike(str_filename), os.O_WRONLY|os.O_CREAT) + bytes_fspath = FakePath(bytes_filename) + fd = os.open(FakePath(str_filename), os.O_WRONLY|os.O_CREAT) self.addCleanup(support.unlink, support.TESTFN) self.addCleanup(os.close, fd) - int_fspath = _PathLike(fd) - str_fspath = _PathLike(str_filename) + int_fspath = FakePath(fd) + str_fspath = FakePath(str_filename) for name, allow_fd, extra_args, cleanup_fn in self.functions: with self.subTest(name=name): @@ -3540,16 +3524,16 @@ def test_return_string(self): def test_fsencode_fsdecode(self): for p in "path/like/object", b"path/like/object": - pathlike = _PathLike(p) + pathlike = FakePath(p) self.assertEqual(p, self.fspath(pathlike)) self.assertEqual(b"path/like/object", os.fsencode(pathlike)) self.assertEqual("path/like/object", os.fsdecode(pathlike)) def test_pathlike(self): - self.assertEqual('#feelthegil', self.fspath(_PathLike('#feelthegil'))) - self.assertTrue(issubclass(_PathLike, os.PathLike)) - self.assertTrue(isinstance(_PathLike(), os.PathLike)) + self.assertEqual('#feelthegil', self.fspath(FakePath('#feelthegil'))) + self.assertTrue(issubclass(FakePath, os.PathLike)) + self.assertTrue(isinstance(FakePath('x'), os.PathLike)) def test_garbage_in_exception_out(self): vapor = type('blah', (), {}) @@ -3561,14 +3545,14 @@ def test_argument_required(self): def test_bad_pathlike(self): # __fspath__ returns a value other than str or bytes. - self.assertRaises(TypeError, self.fspath, _PathLike(42)) + self.assertRaises(TypeError, self.fspath, FakePath(42)) # __fspath__ attribute that is not callable. c = type('foo', (), {}) c.__fspath__ = 1 self.assertRaises(TypeError, self.fspath, c()) # __fspath__ raises an exception. self.assertRaises(ZeroDivisionError, self.fspath, - _PathLike(ZeroDivisionError())) + FakePath(ZeroDivisionError())) class TimesTests(unittest.TestCase): diff --git a/Lib/test/test_pathlib.py b/Lib/test/test_pathlib.py index e56e0d20844e..53215550b435 100644 --- a/Lib/test/test_pathlib.py +++ b/Lib/test/test_pathlib.py @@ -11,7 +11,7 @@ from unittest import mock from test import support -TESTFN = support.TESTFN +from test.support import TESTFN, FakePath try: import grp, pwd @@ -191,18 +191,15 @@ def test_constructor_common(self): P = self.cls p = P('a') self.assertIsInstance(p, P) - class PathLike: - def __fspath__(self): - return "a/b/c" P('a', 'b', 'c') P('/a', 'b', 'c') P('a/b/c') P('/a/b/c') - P(PathLike()) + P(FakePath("a/b/c")) self.assertEqual(P(P('a')), P('a')) self.assertEqual(P(P('a'), 'b'), P('a/b')) self.assertEqual(P(P('a'), P('b')), P('a/b')) - self.assertEqual(P(P('a'), P('b'), P('c')), P(PathLike())) + self.assertEqual(P(P('a'), P('b'), P('c')), P(FakePath("a/b/c"))) def _check_str_subclass(self, *args): # Issue #21127: it should be possible to construct a PurePath object diff --git a/Lib/test/test_posixpath.py b/Lib/test/test_posixpath.py index 8a1e33b0c899..96b267cd45fd 100644 --- a/Lib/test/test_posixpath.py +++ b/Lib/test/test_posixpath.py @@ -4,6 +4,7 @@ import warnings from posixpath import realpath, abspath, dirname, basename from test import support, test_genericpath +from test.support import FakePath try: import posix @@ -600,18 +601,9 @@ class PathLikeTests(unittest.TestCase): path = posixpath - class PathLike: - def __init__(self, path=''): - self.path = path - def __fspath__(self): - if isinstance(self.path, BaseException): - raise self.path - else: - return self.path - def setUp(self): self.file_name = support.TESTFN.lower() - self.file_path = self.PathLike(support.TESTFN) + self.file_path = FakePath(support.TESTFN) self.addCleanup(support.unlink, self.file_name) with open(self.file_name, 'xb', 0) as file: file.write(b"test_posixpath.PathLikeTests") @@ -626,7 +618,7 @@ def test_path_isabs(self): self.assertPathEqual(self.path.isabs) def test_path_join(self): - self.assertEqual(self.path.join('a', self.PathLike('b'), 'c'), + self.assertEqual(self.path.join('a', FakePath('b'), 'c'), self.path.join('a', 'b', 'c')) def test_path_split(self): diff --git a/Lib/test/test_shutil.py b/Lib/test/test_shutil.py index f3cf43e50516..2cb2f14643e1 100644 --- a/Lib/test/test_shutil.py +++ b/Lib/test/test_shutil.py @@ -22,7 +22,7 @@ import zipfile from test import support -from test.support import TESTFN +from test.support import TESTFN, FakePath TESTFN2 = TESTFN + "2" @@ -1232,14 +1232,7 @@ def test_register_archive_format(self): def check_unpack_archive(self, format): self.check_unpack_archive_with_converter(format, lambda path: path) self.check_unpack_archive_with_converter(format, pathlib.Path) - - class MyPath: - def __init__(self, path): - self.path = path - def __fspath__(self): - return self.path - - self.check_unpack_archive_with_converter(format, MyPath) + self.check_unpack_archive_with_converter(format, FakePath) def check_unpack_archive_with_converter(self, format, converter): root_dir, base_dir = self._create_files() diff --git a/Lib/test/test_subprocess.py b/Lib/test/test_subprocess.py index 46cb5f117e84..ddee3b94fed9 100644 --- a/Lib/test/test_subprocess.py +++ b/Lib/test/test_subprocess.py @@ -17,6 +17,7 @@ import threading import gc import textwrap +from test.support import FakePath try: import ctypes @@ -359,12 +360,7 @@ def test_cwd(self): def test_cwd_with_pathlike(self): temp_dir = tempfile.gettempdir() temp_dir = self._normalize_cwd(temp_dir) - - class _PathLikeObj: - def __fspath__(self): - return temp_dir - - self._assert_cwd(temp_dir, sys.executable, cwd=_PathLikeObj()) + self._assert_cwd(temp_dir, sys.executable, cwd=FakePath(temp_dir)) @unittest.skipIf(mswindows, "pending resolution of issue #15533") def test_cwd_with_relative_arg(self): From webhook-mailer at python.org Fri Mar 2 17:10:24 2018 From: webhook-mailer at python.org (Brett Cannon) Date: Fri, 02 Mar 2018 22:10:24 -0000 Subject: [Python-checkins] bpo-32963: Fix the tutorial to state source has a default encoding of ASCII (GH-5961) Message-ID: <mailman.105.1520028626.2831.python-checkins@python.org> https://github.com/python/cpython/commit/20003f9162631c47b79202316036d13d74484e4c commit: 20003f9162631c47b79202316036d13d74484e4c branch: 2.7 author: Brett Cannon <brettcannon at users.noreply.github.com> committer: GitHub <noreply at github.com> date: 2018-03-02T14:10:21-08:00 summary: bpo-32963: Fix the tutorial to state source has a default encoding of ASCII (GH-5961) files: M Doc/tutorial/interpreter.rst diff --git a/Doc/tutorial/interpreter.rst b/Doc/tutorial/interpreter.rst index 3bf9c62ca87c..6c8609fabaeb 100644 --- a/Doc/tutorial/interpreter.rst +++ b/Doc/tutorial/interpreter.rst @@ -126,14 +126,7 @@ The Interpreter and Its Environment Source Code Encoding -------------------- -By default, Python source files are treated as encoded in UTF-8. In that -encoding, characters of most languages in the world can be used simultaneously -in string literals, identifiers and comments --- although the standard library -only uses ASCII characters for identifiers, a convention that any portable code -should follow. To display all these characters properly, your editor must -recognize that the file is UTF-8, and it must use a font that supports all the -characters in the file. - +By default, Python source files are treated as encoded in ASCII. To declare an encoding other than the default one, a special comment line should be added as the *first* line of the file. The syntax is as follows:: From solipsis at pitrou.net Sat Mar 3 04:12:36 2018 From: solipsis at pitrou.net (solipsis at pitrou.net) Date: Sat, 03 Mar 2018 09:12:36 +0000 Subject: [Python-checkins] Daily reference leaks (4243df51fe43): sum=3 Message-ID: <20180303091236.1.66CA96248A8A5664@psf.io> results for 4243df51fe43 on branch "default" -------------------------------------------- test_collections leaked [-7, 8, 0] memory blocks, sum=1 test_functools leaked [0, 3, 1] memory blocks, sum=4 test_multiprocessing_forkserver leaked [-1, 1, -2] memory blocks, sum=-2 Command line was: ['./python', '-m', 'test.regrtest', '-uall', '-R', '3:3:/home/psf-users/antoine/refleaks/reflogeUD0VY', '--timeout', '7200'] From webhook-mailer at python.org Sat Mar 3 07:19:35 2018 From: webhook-mailer at python.org (Serhiy Storchaka) Date: Sat, 03 Mar 2018 12:19:35 -0000 Subject: [Python-checkins] [3.6] bpo-32964: Reuse a testing implementation of the path protocol in tests. (GH-5930). (GH-5958) Message-ID: <mailman.106.1520079579.2831.python-checkins@python.org> https://github.com/python/cpython/commit/fbdd075c64a5229dfa26632cf1b2cf2361dc5003 commit: fbdd075c64a5229dfa26632cf1b2cf2361dc5003 branch: 3.6 author: Serhiy Storchaka <storchaka at gmail.com> committer: GitHub <noreply at github.com> date: 2018-03-03T14:19:29+02:00 summary: [3.6] bpo-32964: Reuse a testing implementation of the path protocol in tests. (GH-5930). (GH-5958) (cherry picked from commit b21d155f57d284aecf9092a9bd24258293965c2f) Co-authored-by: Serhiy Storchaka <storchaka at gmail.com> files: M Doc/library/test.rst M Lib/test/support/__init__.py M Lib/test/test_compile.py M Lib/test/test_genericpath.py M Lib/test/test_io.py M Lib/test/test_ntpath.py M Lib/test/test_os.py M Lib/test/test_pathlib.py M Lib/test/test_posixpath.py M Lib/test/test_shutil.py M Lib/test/test_subprocess.py diff --git a/Doc/library/test.rst b/Doc/library/test.rst index 1a3f8f9c5dce..04d6cd87eac5 100644 --- a/Doc/library/test.rst +++ b/Doc/library/test.rst @@ -677,3 +677,10 @@ The :mod:`test.support` module defines the following classes: Class used to record warnings for unit tests. See documentation of :func:`check_warnings` above for more details. + + +.. class:: FakePath(path) + + Simple :term:`path-like object`. It implements the :meth:`__fspath__` + method which just returns the *path* argument. If *path* is an exception, + it will be raised in :meth:`!__fspath__`. diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index e4840e6a9618..74d435511ae3 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -2708,3 +2708,21 @@ def save(self): def restore(self): for signum, handler in self.handlers.items(): self.signal.signal(signum, handler) + + +class FakePath: + """Simple implementing of the path protocol. + """ + def __init__(self, path): + self.path = path + + def __repr__(self): + return f'<FakePath {self.path!r}>' + + def __fspath__(self): + if (isinstance(self.path, BaseException) or + isinstance(self.path, type) and + issubclass(self.path, BaseException)): + raise self.path + else: + return self.path diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py index 7fe34658e04c..c9812e46b486 100644 --- a/Lib/test/test_compile.py +++ b/Lib/test/test_compile.py @@ -6,7 +6,7 @@ import tempfile import types from test import support -from test.support import script_helper +from test.support import script_helper, FakePath class TestSpecifics(unittest.TestCase): @@ -670,13 +670,7 @@ def check_different_constants(const1, const2): def test_path_like_objects(self): # An implicit test for PyUnicode_FSDecoder(). - class PathLike: - def __init__(self, path): - self._path = path - def __fspath__(self): - return self._path - - compile("42", PathLike("test_compile_pathlike"), "single") + compile("42", FakePath("test_compile_pathlike"), "single") class TestStackSize(unittest.TestCase): diff --git a/Lib/test/test_genericpath.py b/Lib/test/test_genericpath.py index ad5a59c44ed7..9ed53902dc12 100644 --- a/Lib/test/test_genericpath.py +++ b/Lib/test/test_genericpath.py @@ -9,6 +9,7 @@ import warnings from test import support from test.support.script_helper import assert_python_ok +from test.support import FakePath def create_file(filename, data=b'foo'): @@ -493,18 +494,9 @@ def test_import(self): class PathLikeTests(unittest.TestCase): - class PathLike: - def __init__(self, path=''): - self.path = path - def __fspath__(self): - if isinstance(self.path, BaseException): - raise self.path - else: - return self.path - def setUp(self): self.file_name = support.TESTFN.lower() - self.file_path = self.PathLike(support.TESTFN) + self.file_path = FakePath(support.TESTFN) self.addCleanup(support.unlink, self.file_name) create_file(self.file_name, b"test_genericpath.PathLikeTests") diff --git a/Lib/test/test_io.py b/Lib/test/test_io.py index 6c30a44e62a6..3f3b3902ea2d 100644 --- a/Lib/test/test_io.py +++ b/Lib/test/test_io.py @@ -36,6 +36,7 @@ from itertools import cycle, count from test import support from test.support.script_helper import assert_python_ok, run_python_until_end +from test.support import FakePath import codecs import io # C implementation of io @@ -880,13 +881,6 @@ def read(self, size): self.assertEqual(bytes(buffer), b"12345") def test_fspath_support(self): - class PathLike: - def __init__(self, path): - self.path = path - - def __fspath__(self): - return self.path - def check_path_succeeds(path): with self.open(path, "w") as f: f.write("egg\n") @@ -894,16 +888,25 @@ def check_path_succeeds(path): with self.open(path, "r") as f: self.assertEqual(f.read(), "egg\n") - check_path_succeeds(PathLike(support.TESTFN)) - check_path_succeeds(PathLike(support.TESTFN.encode('utf-8'))) + check_path_succeeds(FakePath(support.TESTFN)) + check_path_succeeds(FakePath(support.TESTFN.encode('utf-8'))) + + with self.open(support.TESTFN, "w") as f: + bad_path = FakePath(f.fileno()) + with self.assertRaises(TypeError): + self.open(bad_path, 'w') - bad_path = PathLike(TypeError) + bad_path = FakePath(None) with self.assertRaises(TypeError): self.open(bad_path, 'w') + bad_path = FakePath(FloatingPointError) + with self.assertRaises(FloatingPointError): + self.open(bad_path, 'w') + # ensure that refcounting is correct with some error conditions with self.assertRaisesRegex(ValueError, 'read/write/append mode'): - self.open(PathLike(support.TESTFN), 'rwxa') + self.open(FakePath(support.TESTFN), 'rwxa') def test_RawIOBase_readall(self): # Exercise the default unlimited RawIOBase.read() and readall() diff --git a/Lib/test/test_ntpath.py b/Lib/test/test_ntpath.py index 90edb6d0806d..40761843f34c 100644 --- a/Lib/test/test_ntpath.py +++ b/Lib/test/test_ntpath.py @@ -3,7 +3,7 @@ import sys import unittest import warnings -from test.support import TestFailed +from test.support import TestFailed, FakePath from test import support, test_genericpath from tempfile import TemporaryFile @@ -456,18 +456,9 @@ class PathLikeTests(unittest.TestCase): path = ntpath - class PathLike: - def __init__(self, path=''): - self.path = path - def __fspath__(self): - if isinstance(self.path, BaseException): - raise self.path - else: - return self.path - def setUp(self): self.file_name = support.TESTFN.lower() - self.file_path = self.PathLike(support.TESTFN) + self.file_path = FakePath(support.TESTFN) self.addCleanup(support.unlink, self.file_name) with open(self.file_name, 'xb', 0) as file: file.write(b"test_ntpath.PathLikeTests") @@ -482,7 +473,7 @@ def test_path_isabs(self): self.assertPathEqual(self.path.isabs) def test_path_join(self): - self.assertEqual(self.path.join('a', self.PathLike('b'), 'c'), + self.assertEqual(self.path.join('a', FakePath('b'), 'c'), self.path.join('a', 'b', 'c')) def test_path_split(self): diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index 033e544ffed5..26d544ca9925 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -64,7 +64,7 @@ INT_MAX = PY_SSIZE_T_MAX = sys.maxsize from test.support.script_helper import assert_python_ok -from test.support import unix_shell +from test.support import unix_shell, FakePath root_in_posix = False @@ -94,21 +94,6 @@ def requires_os_func(name): return unittest.skipUnless(hasattr(os, name), 'requires os.%s' % name) -class _PathLike(os.PathLike): - - def __init__(self, path=""): - self.path = path - - def __str__(self): - return str(self.path) - - def __fspath__(self): - if isinstance(self.path, BaseException): - raise self.path - else: - return self.path - - def create_file(filename, content=b'content'): with open(filename, "xb", 0) as fp: fp.write(content) @@ -970,15 +955,14 @@ def test_walk_prune(self, walk_path=None): dirs.remove('SUB1') self.assertEqual(len(all), 2) - self.assertEqual(all[0], - (str(walk_path), ["SUB2"], ["tmp1"])) + self.assertEqual(all[0], (self.walk_path, ["SUB2"], ["tmp1"])) all[1][-1].sort() all[1][1].sort() self.assertEqual(all[1], self.sub2_tree) def test_file_like_path(self): - self.test_walk_prune(_PathLike(self.walk_path)) + self.test_walk_prune(FakePath(self.walk_path)) def test_walk_bottom_up(self): # Walk bottom-up. @@ -2294,7 +2278,7 @@ def test_getppid(self): def test_waitpid(self): args = [sys.executable, '-c', 'pass'] # Add an implicit test for PyUnicode_FSConverter(). - pid = os.spawnv(os.P_NOWAIT, _PathLike(args[0]), args) + pid = os.spawnv(os.P_NOWAIT, FakePath(args[0]), args) status = os.waitpid(pid, 0) self.assertEqual(status, (pid, 0)) @@ -3140,13 +3124,13 @@ def test_path_t_converter(self): bytes_fspath = bytes_filename = None else: bytes_filename = support.TESTFN.encode('ascii') - bytes_fspath = _PathLike(bytes_filename) - fd = os.open(_PathLike(str_filename), os.O_WRONLY|os.O_CREAT) + bytes_fspath = FakePath(bytes_filename) + fd = os.open(FakePath(str_filename), os.O_WRONLY|os.O_CREAT) self.addCleanup(support.unlink, support.TESTFN) self.addCleanup(os.close, fd) - int_fspath = _PathLike(fd) - str_fspath = _PathLike(str_filename) + int_fspath = FakePath(fd) + str_fspath = FakePath(str_filename) for name, allow_fd, extra_args, cleanup_fn in self.functions: with self.subTest(name=name): @@ -3519,16 +3503,16 @@ def test_return_string(self): def test_fsencode_fsdecode(self): for p in "path/like/object", b"path/like/object": - pathlike = _PathLike(p) + pathlike = FakePath(p) self.assertEqual(p, self.fspath(pathlike)) self.assertEqual(b"path/like/object", os.fsencode(pathlike)) self.assertEqual("path/like/object", os.fsdecode(pathlike)) def test_pathlike(self): - self.assertEqual('#feelthegil', self.fspath(_PathLike('#feelthegil'))) - self.assertTrue(issubclass(_PathLike, os.PathLike)) - self.assertTrue(isinstance(_PathLike(), os.PathLike)) + self.assertEqual('#feelthegil', self.fspath(FakePath('#feelthegil'))) + self.assertTrue(issubclass(FakePath, os.PathLike)) + self.assertTrue(isinstance(FakePath('x'), os.PathLike)) def test_garbage_in_exception_out(self): vapor = type('blah', (), {}) @@ -3540,14 +3524,14 @@ def test_argument_required(self): def test_bad_pathlike(self): # __fspath__ returns a value other than str or bytes. - self.assertRaises(TypeError, self.fspath, _PathLike(42)) + self.assertRaises(TypeError, self.fspath, FakePath(42)) # __fspath__ attribute that is not callable. c = type('foo', (), {}) c.__fspath__ = 1 self.assertRaises(TypeError, self.fspath, c()) # __fspath__ raises an exception. self.assertRaises(ZeroDivisionError, self.fspath, - _PathLike(ZeroDivisionError())) + FakePath(ZeroDivisionError())) # Only test if the C version is provided, otherwise TestPEP519 already tested # the pure Python implementation. diff --git a/Lib/test/test_pathlib.py b/Lib/test/test_pathlib.py index 6672c448f4fb..a4d2f8d98333 100644 --- a/Lib/test/test_pathlib.py +++ b/Lib/test/test_pathlib.py @@ -11,7 +11,7 @@ from unittest import mock from test import support -TESTFN = support.TESTFN +from test.support import TESTFN, FakePath try: import grp, pwd @@ -191,18 +191,15 @@ def test_constructor_common(self): P = self.cls p = P('a') self.assertIsInstance(p, P) - class PathLike: - def __fspath__(self): - return "a/b/c" P('a', 'b', 'c') P('/a', 'b', 'c') P('a/b/c') P('/a/b/c') - P(PathLike()) + P(FakePath("a/b/c")) self.assertEqual(P(P('a')), P('a')) self.assertEqual(P(P('a'), 'b'), P('a/b')) self.assertEqual(P(P('a'), P('b')), P('a/b')) - self.assertEqual(P(P('a'), P('b'), P('c')), P(PathLike())) + self.assertEqual(P(P('a'), P('b'), P('c')), P(FakePath("a/b/c"))) def _check_str_subclass(self, *args): # Issue #21127: it should be possible to construct a PurePath object diff --git a/Lib/test/test_posixpath.py b/Lib/test/test_posixpath.py index 8a1e33b0c899..96b267cd45fd 100644 --- a/Lib/test/test_posixpath.py +++ b/Lib/test/test_posixpath.py @@ -4,6 +4,7 @@ import warnings from posixpath import realpath, abspath, dirname, basename from test import support, test_genericpath +from test.support import FakePath try: import posix @@ -600,18 +601,9 @@ class PathLikeTests(unittest.TestCase): path = posixpath - class PathLike: - def __init__(self, path=''): - self.path = path - def __fspath__(self): - if isinstance(self.path, BaseException): - raise self.path - else: - return self.path - def setUp(self): self.file_name = support.TESTFN.lower() - self.file_path = self.PathLike(support.TESTFN) + self.file_path = FakePath(support.TESTFN) self.addCleanup(support.unlink, self.file_name) with open(self.file_name, 'xb', 0) as file: file.write(b"test_posixpath.PathLikeTests") @@ -626,7 +618,7 @@ def test_path_isabs(self): self.assertPathEqual(self.path.isabs) def test_path_join(self): - self.assertEqual(self.path.join('a', self.PathLike('b'), 'c'), + self.assertEqual(self.path.join('a', FakePath('b'), 'c'), self.path.join('a', 'b', 'c')) def test_path_split(self): diff --git a/Lib/test/test_shutil.py b/Lib/test/test_shutil.py index 95717a48ff10..588e7d27941c 100644 --- a/Lib/test/test_shutil.py +++ b/Lib/test/test_shutil.py @@ -10,6 +10,7 @@ import os.path import errno import functools +import pathlib import subprocess from contextlib import ExitStack from shutil import (make_archive, @@ -21,9 +22,10 @@ import tarfile import zipfile import warnings +import pathlib from test import support -from test.support import TESTFN, check_warnings, captured_stdout +from test.support import TESTFN, FakePath TESTFN2 = TESTFN + "2" @@ -1231,6 +1233,11 @@ def test_register_archive_format(self): self.assertNotIn('xxx', formats) def check_unpack_archive(self, format): + self.check_unpack_archive_with_converter(format, lambda path: path) + self.check_unpack_archive_with_converter(format, pathlib.Path) + self.check_unpack_archive_with_converter(format, FakePath) + + def check_unpack_archive_with_converter(self, format, converter): root_dir, base_dir = self._create_files() expected = rlistdir(root_dir) expected.remove('outer') diff --git a/Lib/test/test_subprocess.py b/Lib/test/test_subprocess.py index 5c6429cc0227..aca1411cdfb5 100644 --- a/Lib/test/test_subprocess.py +++ b/Lib/test/test_subprocess.py @@ -16,6 +16,7 @@ import shutil import gc import textwrap +from test.support import FakePath try: import ctypes @@ -363,12 +364,7 @@ def test_cwd(self): def test_cwd_with_pathlike(self): temp_dir = tempfile.gettempdir() temp_dir = self._normalize_cwd(temp_dir) - - class _PathLikeObj: - def __fspath__(self): - return temp_dir - - self._assert_cwd(temp_dir, sys.executable, cwd=_PathLikeObj()) + self._assert_cwd(temp_dir, sys.executable, cwd=FakePath(temp_dir)) @unittest.skipIf(mswindows, "pending resolution of issue #15533") def test_cwd_with_relative_arg(self): From webhook-mailer at python.org Sat Mar 3 11:43:58 2018 From: webhook-mailer at python.org (Andrew Svetlov) Date: Sat, 03 Mar 2018 16:43:58 -0000 Subject: [Python-checkins] Fix missing coroutine declaration in the asyncio documentation. (#5964) Message-ID: <mailman.107.1520095440.2831.python-checkins@python.org> https://github.com/python/cpython/commit/13cfd57dcf58485d6242fd8118c6ea4b10e29aab commit: 13cfd57dcf58485d6242fd8118c6ea4b10e29aab branch: master author: Joongi Kim <me at daybreaker.info> committer: Andrew Svetlov <andrew.svetlov at gmail.com> date: 2018-03-03T18:43:54+02:00 summary: Fix missing coroutine declaration in the asyncio documentation. (#5964) files: M Doc/library/asyncio-task.rst diff --git a/Doc/library/asyncio-task.rst b/Doc/library/asyncio-task.rst index 71dbe06c899f..485e1b843d08 100644 --- a/Doc/library/asyncio-task.rst +++ b/Doc/library/asyncio-task.rst @@ -682,7 +682,7 @@ Task functions This function is a :ref:`coroutine <coroutine>`. -.. function:: shield(arg, \*, loop=None) +.. coroutinefunction:: shield(arg, \*, loop=None) Wait for a future, shielding it from cancellation. From webhook-mailer at python.org Sat Mar 3 16:23:11 2018 From: webhook-mailer at python.org (Miss Islington (bot)) Date: Sat, 03 Mar 2018 21:23:11 -0000 Subject: [Python-checkins] Fix missing coroutine declaration in the asyncio documentation. (GH-5964) Message-ID: <mailman.108.1520112192.2831.python-checkins@python.org> https://github.com/python/cpython/commit/0f7cf7eb8a2c9a1766780ebf225a2f4c0640788e commit: 0f7cf7eb8a2c9a1766780ebf225a2f4c0640788e branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: GitHub <noreply at github.com> date: 2018-03-03T13:22:49-08:00 summary: Fix missing coroutine declaration in the asyncio documentation. (GH-5964) (cherry picked from commit 13cfd57dcf58485d6242fd8118c6ea4b10e29aab) Co-authored-by: Joongi Kim <me at daybreaker.info> files: M Doc/library/asyncio-task.rst diff --git a/Doc/library/asyncio-task.rst b/Doc/library/asyncio-task.rst index 71dbe06c899f..485e1b843d08 100644 --- a/Doc/library/asyncio-task.rst +++ b/Doc/library/asyncio-task.rst @@ -682,7 +682,7 @@ Task functions This function is a :ref:`coroutine <coroutine>`. -.. function:: shield(arg, \*, loop=None) +.. coroutinefunction:: shield(arg, \*, loop=None) Wait for a future, shielding it from cancellation. From webhook-mailer at python.org Sat Mar 3 16:23:36 2018 From: webhook-mailer at python.org (Miss Islington (bot)) Date: Sat, 03 Mar 2018 21:23:36 -0000 Subject: [Python-checkins] Fix missing coroutine declaration in the asyncio documentation. (GH-5964) Message-ID: <mailman.109.1520112218.2831.python-checkins@python.org> https://github.com/python/cpython/commit/bd92cfe632228ea4d6a1c0dc9506baa18d353dc1 commit: bd92cfe632228ea4d6a1c0dc9506baa18d353dc1 branch: 3.6 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: GitHub <noreply at github.com> date: 2018-03-03T13:23:31-08:00 summary: Fix missing coroutine declaration in the asyncio documentation. (GH-5964) (cherry picked from commit 13cfd57dcf58485d6242fd8118c6ea4b10e29aab) Co-authored-by: Joongi Kim <me at daybreaker.info> files: M Doc/library/asyncio-task.rst diff --git a/Doc/library/asyncio-task.rst b/Doc/library/asyncio-task.rst index cc8fffb0659f..5b801aaf8ecc 100644 --- a/Doc/library/asyncio-task.rst +++ b/Doc/library/asyncio-task.rst @@ -630,7 +630,7 @@ Task functions This function is a :ref:`coroutine <coroutine>`. -.. function:: shield(arg, \*, loop=None) +.. coroutinefunction:: shield(arg, \*, loop=None) Wait for a future, shielding it from cancellation. From webhook-mailer at python.org Sun Mar 4 00:33:37 2018 From: webhook-mailer at python.org (Benjamin Peterson) Date: Sun, 04 Mar 2018 05:33:37 -0000 Subject: [Python-checkins] bpo-32981: Fix catastrophic backtracking vulns (#5955) Message-ID: <mailman.110.1520141619.2831.python-checkins@python.org> https://github.com/python/cpython/commit/0e6c8ee2358a2e23117501826c008842acb835ac commit: 0e6c8ee2358a2e23117501826c008842acb835ac branch: master author: Jamie Davis <davisjam at vt.edu> committer: Benjamin Peterson <benjamin at python.org> date: 2018-03-03T21:33:32-08:00 summary: bpo-32981: Fix catastrophic backtracking vulns (#5955) * Prevent low-grade poplib REDOS (CVE-2018-1060) The regex to test a mail server's timestamp is susceptible to catastrophic backtracking on long evil responses from the server. Happily, the maximum length of malicious inputs is 2K thanks to a limit introduced in the fix for CVE-2013-1752. A 2KB evil response from the mail server would result in small slowdowns (milliseconds vs. microseconds) accumulated over many apop calls. This is a potential DOS vector via accumulated slowdowns. Replace it with a similar non-vulnerable regex. The new regex is RFC compliant. The old regex was non-compliant in edge cases. * Prevent difflib REDOS (CVE-2018-1061) The default regex for IS_LINE_JUNK is susceptible to catastrophic backtracking. This is a potential DOS vector. Replace it with an equivalent non-vulnerable regex. Also introduce unit and REDOS tests for difflib. Co-authored-by: Tim Peters <tim.peters at gmail.com> Co-authored-by: Christian Heimes <christian at python.org> files: A Misc/NEWS.d/next/Security/2018-03-02-10-24-52.bpo-32981.O_qDyj.rst M Lib/difflib.py M Lib/poplib.py M Lib/test/test_difflib.py M Lib/test/test_poplib.py M Misc/ACKS diff --git a/Lib/difflib.py b/Lib/difflib.py index 82964719dc83..043a169c28be 100644 --- a/Lib/difflib.py +++ b/Lib/difflib.py @@ -1083,7 +1083,7 @@ def _qformat(self, aline, bline, atags, btags): import re -def IS_LINE_JUNK(line, pat=re.compile(r"\s*#?\s*$").match): +def IS_LINE_JUNK(line, pat=re.compile(r"\s*(?:#\s*)?$").match): r""" Return 1 for ignorable line: iff `line` is blank or contains a single '#'. diff --git a/Lib/poplib.py b/Lib/poplib.py index 6bcfa5cfeba3..d8a62c034327 100644 --- a/Lib/poplib.py +++ b/Lib/poplib.py @@ -308,7 +308,7 @@ def rpop(self, user): return self._shortcmd('RPOP %s' % user) - timestamp = re.compile(br'\+OK.*(<[^>]+>)') + timestamp = re.compile(br'\+OK.[^<]*(<.*>)') def apop(self, user, password): """Authorisation diff --git a/Lib/test/test_difflib.py b/Lib/test/test_difflib.py index 156b523c38c1..aaefe6db0291 100644 --- a/Lib/test/test_difflib.py +++ b/Lib/test/test_difflib.py @@ -466,13 +466,33 @@ def _assert_type_error(self, msg, generator, *args): list(generator(*args)) self.assertEqual(msg, str(ctx.exception)) +class TestJunkAPIs(unittest.TestCase): + def test_is_line_junk_true(self): + for line in ['#', ' ', ' #', '# ', ' # ', '']: + self.assertTrue(difflib.IS_LINE_JUNK(line), repr(line)) + + def test_is_line_junk_false(self): + for line in ['##', ' ##', '## ', 'abc ', 'abc #', 'Mr. Moose is up!']: + self.assertFalse(difflib.IS_LINE_JUNK(line), repr(line)) + + def test_is_line_junk_REDOS(self): + evil_input = ('\t' * 1000000) + '##' + self.assertFalse(difflib.IS_LINE_JUNK(evil_input)) + + def test_is_character_junk_true(self): + for char in [' ', '\t']: + self.assertTrue(difflib.IS_CHARACTER_JUNK(char), repr(char)) + + def test_is_character_junk_false(self): + for char in ['a', '#', '\n', '\f', '\r', '\v']: + self.assertFalse(difflib.IS_CHARACTER_JUNK(char), repr(char)) def test_main(): difflib.HtmlDiff._default_prefix = 0 Doctests = doctest.DocTestSuite(difflib) run_unittest( TestWithAscii, TestAutojunk, TestSFpatches, TestSFbugs, - TestOutputFormat, TestBytes, Doctests) + TestOutputFormat, TestBytes, TestJunkAPIs, Doctests) if __name__ == '__main__': test_main() diff --git a/Lib/test/test_poplib.py b/Lib/test/test_poplib.py index fd0db798ee43..bf568bd77bff 100644 --- a/Lib/test/test_poplib.py +++ b/Lib/test/test_poplib.py @@ -308,9 +308,19 @@ def test_noop(self): def test_rpop(self): self.assertOK(self.client.rpop('foo')) - def test_apop(self): + def test_apop_normal(self): self.assertOK(self.client.apop('foo', 'dummypassword')) + def test_apop_REDOS(self): + # Replace welcome with very long evil welcome. + # NB The upper bound on welcome length is currently 2048. + # At this length, evil input makes each apop call take + # on the order of milliseconds instead of microseconds. + evil_welcome = b'+OK' + (b'<' * 1000000) + with test_support.swap_attr(self.client, 'welcome', evil_welcome): + # The evil welcome is invalid, so apop should throw. + self.assertRaises(poplib.error_proto, self.client.apop, 'a', 'kb') + def test_top(self): expected = (b'+OK 116 bytes', [b'From: postmaster at python.org', b'Content-Type: text/plain', diff --git a/Misc/ACKS b/Misc/ACKS index e2addfc210b7..d8179c8b03ab 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -356,6 +356,7 @@ Jonathan Dasteel Pierre-Yves David A. Jesse Jiryu Davis Jake Davis +Jamie (James C.) Davis Ratnadeep Debnath Merlijn van Deen John DeGood diff --git a/Misc/NEWS.d/next/Security/2018-03-02-10-24-52.bpo-32981.O_qDyj.rst b/Misc/NEWS.d/next/Security/2018-03-02-10-24-52.bpo-32981.O_qDyj.rst new file mode 100644 index 000000000000..9ebabb44f91e --- /dev/null +++ b/Misc/NEWS.d/next/Security/2018-03-02-10-24-52.bpo-32981.O_qDyj.rst @@ -0,0 +1,4 @@ +Regexes in difflib and poplib were vulnerable to catastrophic backtracking. +These regexes formed potential DOS vectors (REDOS). They have been +refactored. This resolves CVE-2018-1060 and CVE-2018-1061. +Patch by Jamie Davis. From webhook-mailer at python.org Sun Mar 4 00:55:10 2018 From: webhook-mailer at python.org (Benjamin Peterson) Date: Sun, 04 Mar 2018 05:55:10 -0000 Subject: [Python-checkins] bpo-32981: Fix catastrophic backtracking vulns (GH-5955) Message-ID: <mailman.111.1520142911.2831.python-checkins@python.org> https://github.com/python/cpython/commit/0902a2d6b2d1d9dbde36aeaaccf1788ceaa97143 commit: 0902a2d6b2d1d9dbde36aeaaccf1788ceaa97143 branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Benjamin Peterson <benjamin at python.org> date: 2018-03-03T21:55:07-08:00 summary: bpo-32981: Fix catastrophic backtracking vulns (GH-5955) * Prevent low-grade poplib REDOS (CVE-2018-1060) The regex to test a mail server's timestamp is susceptible to catastrophic backtracking on long evil responses from the server. Happily, the maximum length of malicious inputs is 2K thanks to a limit introduced in the fix for CVE-2013-1752. A 2KB evil response from the mail server would result in small slowdowns (milliseconds vs. microseconds) accumulated over many apop calls. This is a potential DOS vector via accumulated slowdowns. Replace it with a similar non-vulnerable regex. The new regex is RFC compliant. The old regex was non-compliant in edge cases. * Prevent difflib REDOS (CVE-2018-1061) The default regex for IS_LINE_JUNK is susceptible to catastrophic backtracking. This is a potential DOS vector. Replace it with an equivalent non-vulnerable regex. Also introduce unit and REDOS tests for difflib. Co-authored-by: Tim Peters <tim.peters at gmail.com> Co-authored-by: Christian Heimes <christian at python.org> Co-authored-by: Jamie Davis <davisjam at vt.edu> (cherry picked from commit 0e6c8ee2358a2e23117501826c008842acb835ac) files: A Misc/NEWS.d/next/Security/2018-03-02-10-24-52.bpo-32981.O_qDyj.rst M Lib/difflib.py M Lib/poplib.py M Lib/test/test_difflib.py M Lib/test/test_poplib.py M Misc/ACKS diff --git a/Lib/difflib.py b/Lib/difflib.py index 82964719dc83..043a169c28be 100644 --- a/Lib/difflib.py +++ b/Lib/difflib.py @@ -1083,7 +1083,7 @@ def _qformat(self, aline, bline, atags, btags): import re -def IS_LINE_JUNK(line, pat=re.compile(r"\s*#?\s*$").match): +def IS_LINE_JUNK(line, pat=re.compile(r"\s*(?:#\s*)?$").match): r""" Return 1 for ignorable line: iff `line` is blank or contains a single '#'. diff --git a/Lib/poplib.py b/Lib/poplib.py index 6bcfa5cfeba3..d8a62c034327 100644 --- a/Lib/poplib.py +++ b/Lib/poplib.py @@ -308,7 +308,7 @@ def rpop(self, user): return self._shortcmd('RPOP %s' % user) - timestamp = re.compile(br'\+OK.*(<[^>]+>)') + timestamp = re.compile(br'\+OK.[^<]*(<.*>)') def apop(self, user, password): """Authorisation diff --git a/Lib/test/test_difflib.py b/Lib/test/test_difflib.py index 156b523c38c1..aaefe6db0291 100644 --- a/Lib/test/test_difflib.py +++ b/Lib/test/test_difflib.py @@ -466,13 +466,33 @@ def _assert_type_error(self, msg, generator, *args): list(generator(*args)) self.assertEqual(msg, str(ctx.exception)) +class TestJunkAPIs(unittest.TestCase): + def test_is_line_junk_true(self): + for line in ['#', ' ', ' #', '# ', ' # ', '']: + self.assertTrue(difflib.IS_LINE_JUNK(line), repr(line)) + + def test_is_line_junk_false(self): + for line in ['##', ' ##', '## ', 'abc ', 'abc #', 'Mr. Moose is up!']: + self.assertFalse(difflib.IS_LINE_JUNK(line), repr(line)) + + def test_is_line_junk_REDOS(self): + evil_input = ('\t' * 1000000) + '##' + self.assertFalse(difflib.IS_LINE_JUNK(evil_input)) + + def test_is_character_junk_true(self): + for char in [' ', '\t']: + self.assertTrue(difflib.IS_CHARACTER_JUNK(char), repr(char)) + + def test_is_character_junk_false(self): + for char in ['a', '#', '\n', '\f', '\r', '\v']: + self.assertFalse(difflib.IS_CHARACTER_JUNK(char), repr(char)) def test_main(): difflib.HtmlDiff._default_prefix = 0 Doctests = doctest.DocTestSuite(difflib) run_unittest( TestWithAscii, TestAutojunk, TestSFpatches, TestSFbugs, - TestOutputFormat, TestBytes, Doctests) + TestOutputFormat, TestBytes, TestJunkAPIs, Doctests) if __name__ == '__main__': test_main() diff --git a/Lib/test/test_poplib.py b/Lib/test/test_poplib.py index fd0db798ee43..bf568bd77bff 100644 --- a/Lib/test/test_poplib.py +++ b/Lib/test/test_poplib.py @@ -308,9 +308,19 @@ def test_noop(self): def test_rpop(self): self.assertOK(self.client.rpop('foo')) - def test_apop(self): + def test_apop_normal(self): self.assertOK(self.client.apop('foo', 'dummypassword')) + def test_apop_REDOS(self): + # Replace welcome with very long evil welcome. + # NB The upper bound on welcome length is currently 2048. + # At this length, evil input makes each apop call take + # on the order of milliseconds instead of microseconds. + evil_welcome = b'+OK' + (b'<' * 1000000) + with test_support.swap_attr(self.client, 'welcome', evil_welcome): + # The evil welcome is invalid, so apop should throw. + self.assertRaises(poplib.error_proto, self.client.apop, 'a', 'kb') + def test_top(self): expected = (b'+OK 116 bytes', [b'From: postmaster at python.org', b'Content-Type: text/plain', diff --git a/Misc/ACKS b/Misc/ACKS index 4c0b0e5720e1..d598520e9385 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -356,6 +356,7 @@ Jonathan Dasteel Pierre-Yves David A. Jesse Jiryu Davis Jake Davis +Jamie (James C.) Davis Ratnadeep Debnath Merlijn van Deen John DeGood diff --git a/Misc/NEWS.d/next/Security/2018-03-02-10-24-52.bpo-32981.O_qDyj.rst b/Misc/NEWS.d/next/Security/2018-03-02-10-24-52.bpo-32981.O_qDyj.rst new file mode 100644 index 000000000000..9ebabb44f91e --- /dev/null +++ b/Misc/NEWS.d/next/Security/2018-03-02-10-24-52.bpo-32981.O_qDyj.rst @@ -0,0 +1,4 @@ +Regexes in difflib and poplib were vulnerable to catastrophic backtracking. +These regexes formed potential DOS vectors (REDOS). They have been +refactored. This resolves CVE-2018-1060 and CVE-2018-1061. +Patch by Jamie Davis. From webhook-mailer at python.org Sun Mar 4 01:06:04 2018 From: webhook-mailer at python.org (Benjamin Peterson) Date: Sun, 04 Mar 2018 06:06:04 -0000 Subject: [Python-checkins] closes bpo-32980 Remove _PyFrame_Init (GH-5965) Message-ID: <mailman.112.1520143564.2831.python-checkins@python.org> https://github.com/python/cpython/commit/7023644e0c310a3006c984318c2c111c468735b4 commit: 7023644e0c310a3006c984318c2c111c468735b4 branch: master author: Thomas Nyberg <tomnyberg at gmail.com> committer: Benjamin Peterson <benjamin at python.org> date: 2018-03-03T22:06:01-08:00 summary: closes bpo-32980 Remove _PyFrame_Init (GH-5965) files: M Include/pylifecycle.h M Objects/frameobject.c M Python/pylifecycle.c diff --git a/Include/pylifecycle.h b/Include/pylifecycle.h index 659c6df644e3..95dd55b05173 100644 --- a/Include/pylifecycle.h +++ b/Include/pylifecycle.h @@ -148,7 +148,6 @@ PyAPI_FUNC(int) _PySys_EndInit(PyObject *sysdict, _PyMainInterpreterConfig *conf PyAPI_FUNC(_PyInitError) _PyImport_Init(PyInterpreterState *interp); PyAPI_FUNC(void) _PyExc_Init(PyObject * bltinmod); PyAPI_FUNC(_PyInitError) _PyImportHooks_Init(void); -PyAPI_FUNC(int) _PyFrame_Init(void); PyAPI_FUNC(int) _PyFloat_Init(void); PyAPI_FUNC(int) PyByteArray_Init(void); PyAPI_FUNC(_PyInitError) _Py_HashRandomization_Init(const _PyCoreConfig *); diff --git a/Objects/frameobject.c b/Objects/frameobject.c index d308457b7547..b9f380d7b60d 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -552,14 +552,6 @@ PyTypeObject PyFrame_Type = { _Py_IDENTIFIER(__builtins__); -int _PyFrame_Init() -{ - /* Before, PyId___builtins__ was a string created explicitly in - this function. Now there is nothing to initialize anymore, but - the function is kept for backward compatibility. */ - return 1; -} - PyFrameObject* _Py_HOT_FUNCTION _PyFrame_New_NoTrack(PyThreadState *tstate, PyCodeObject *code, PyObject *globals, PyObject *locals) diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 5db586e15dff..a9b9470c7265 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -690,9 +690,6 @@ _Py_InitializeCore(const _PyCoreConfig *core_config) _Py_ReadyTypes(); - if (!_PyFrame_Init()) - return _Py_INIT_ERR("can't init frames"); - if (!_PyLong_Init()) return _Py_INIT_ERR("can't init longs"); From webhook-mailer at python.org Sun Mar 4 01:18:20 2018 From: webhook-mailer at python.org (Benjamin Peterson) Date: Sun, 04 Mar 2018 06:18:20 -0000 Subject: [Python-checkins] [2.7] bpo-32981: Fix catastrophic backtracking vulns (GH-5955) Message-ID: <mailman.113.1520144301.2831.python-checkins@python.org> https://github.com/python/cpython/commit/e052d40cea15f582b50947f7d906b39744dc62a2 commit: e052d40cea15f582b50947f7d906b39744dc62a2 branch: 2.7 author: Benjamin Peterson <benjamin at python.org> committer: GitHub <noreply at github.com> date: 2018-03-03T22:18:17-08:00 summary: [2.7] bpo-32981: Fix catastrophic backtracking vulns (GH-5955) * Prevent low-grade poplib REDOS (CVE-2018-1060) The regex to test a mail server's timestamp is susceptible to catastrophic backtracking on long evil responses from the server. Happily, the maximum length of malicious inputs is 2K thanks to a limit introduced in the fix for CVE-2013-1752. A 2KB evil response from the mail server would result in small slowdowns (milliseconds vs. microseconds) accumulated over many apop calls. This is a potential DOS vector via accumulated slowdowns. Replace it with a similar non-vulnerable regex. The new regex is RFC compliant. The old regex was non-compliant in edge cases. * Prevent difflib REDOS (CVE-2018-1061) The default regex for IS_LINE_JUNK is susceptible to catastrophic backtracking. This is a potential DOS vector. Replace it with an equivalent non-vulnerable regex. Also introduce unit and REDOS tests for difflib. Co-authored-by: Tim Peters <tim.peters at gmail.com> Co-authored-by: Christian Heimes <christian at python.org>. (cherry picked from commit 0e6c8ee2358a2e23117501826c008842acb835ac) files: A Misc/NEWS.d/next/Security/2018-03-02-10-24-52.bpo-32981.O_qDyj.rst M Lib/difflib.py M Lib/poplib.py M Lib/test/test_difflib.py M Lib/test/test_poplib.py M Misc/ACKS diff --git a/Lib/difflib.py b/Lib/difflib.py index 1c6fbdbedcb7..788a92df3f89 100644 --- a/Lib/difflib.py +++ b/Lib/difflib.py @@ -1103,7 +1103,7 @@ def _qformat(self, aline, bline, atags, btags): import re -def IS_LINE_JUNK(line, pat=re.compile(r"\s*#?\s*$").match): +def IS_LINE_JUNK(line, pat=re.compile(r"\s*(?:#\s*)?$").match): r""" Return 1 for ignorable line: iff `line` is blank or contains a single '#'. diff --git a/Lib/poplib.py b/Lib/poplib.py index b91e5f72d2ca..a238510b38fc 100644 --- a/Lib/poplib.py +++ b/Lib/poplib.py @@ -274,7 +274,7 @@ def rpop(self, user): return self._shortcmd('RPOP %s' % user) - timestamp = re.compile(r'\+OK.*(<[^>]+>)') + timestamp = re.compile(br'\+OK.[^<]*(<.*>)') def apop(self, user, secret): """Authorisation diff --git a/Lib/test/test_difflib.py b/Lib/test/test_difflib.py index 35f2c36ca70a..d8277b79b880 100644 --- a/Lib/test/test_difflib.py +++ b/Lib/test/test_difflib.py @@ -269,13 +269,33 @@ def test_range_format_context(self): self.assertEqual(fmt(3,6), '4,6') self.assertEqual(fmt(0,0), '0') +class TestJunkAPIs(unittest.TestCase): + def test_is_line_junk_true(self): + for line in ['#', ' ', ' #', '# ', ' # ', '']: + self.assertTrue(difflib.IS_LINE_JUNK(line), repr(line)) + + def test_is_line_junk_false(self): + for line in ['##', ' ##', '## ', 'abc ', 'abc #', 'Mr. Moose is up!']: + self.assertFalse(difflib.IS_LINE_JUNK(line), repr(line)) + + def test_is_line_junk_REDOS(self): + evil_input = ('\t' * 1000000) + '##' + self.assertFalse(difflib.IS_LINE_JUNK(evil_input)) + + def test_is_character_junk_true(self): + for char in [' ', '\t']: + self.assertTrue(difflib.IS_CHARACTER_JUNK(char), repr(char)) + + def test_is_character_junk_false(self): + for char in ['a', '#', '\n', '\f', '\r', '\v']: + self.assertFalse(difflib.IS_CHARACTER_JUNK(char), repr(char)) def test_main(): difflib.HtmlDiff._default_prefix = 0 Doctests = doctest.DocTestSuite(difflib) run_unittest( TestWithAscii, TestAutojunk, TestSFpatches, TestSFbugs, - TestOutputFormat, Doctests) + TestOutputFormat, TestJunkAPIs) if __name__ == '__main__': test_main() diff --git a/Lib/test/test_poplib.py b/Lib/test/test_poplib.py index 23d688724b95..d2143759ba66 100644 --- a/Lib/test/test_poplib.py +++ b/Lib/test/test_poplib.py @@ -211,6 +211,16 @@ def test_noop(self): def test_rpop(self): self.assertOK(self.client.rpop('foo')) + def test_apop_REDOS(self): + # Replace welcome with very long evil welcome. + # NB The upper bound on welcome length is currently 2048. + # At this length, evil input makes each apop call take + # on the order of milliseconds instead of microseconds. + evil_welcome = b'+OK' + (b'<' * 1000000) + with test_support.swap_attr(self.client, 'welcome', evil_welcome): + # The evil welcome is invalid, so apop should throw. + self.assertRaises(poplib.error_proto, self.client.apop, 'a', 'kb') + def test_top(self): expected = ('+OK 116 bytes', ['From: postmaster at python.org', 'Content-Type: text/plain', diff --git a/Misc/ACKS b/Misc/ACKS index 28b01e8cc7c5..ee2e4fc19393 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -318,6 +318,8 @@ Jonathan Dasteel Pierre-Yves David A. Jesse Jiryu Davis Jake Davis +Jamie (James C.) Davis +Ratnadeep Debnath Merlijn van Deen John DeGood Ned Deily diff --git a/Misc/NEWS.d/next/Security/2018-03-02-10-24-52.bpo-32981.O_qDyj.rst b/Misc/NEWS.d/next/Security/2018-03-02-10-24-52.bpo-32981.O_qDyj.rst new file mode 100644 index 000000000000..9ebabb44f91e --- /dev/null +++ b/Misc/NEWS.d/next/Security/2018-03-02-10-24-52.bpo-32981.O_qDyj.rst @@ -0,0 +1,4 @@ +Regexes in difflib and poplib were vulnerable to catastrophic backtracking. +These regexes formed potential DOS vectors (REDOS). They have been +refactored. This resolves CVE-2018-1060 and CVE-2018-1061. +Patch by Jamie Davis. From webhook-mailer at python.org Sun Mar 4 01:59:15 2018 From: webhook-mailer at python.org (Benjamin Peterson) Date: Sun, 04 Mar 2018 06:59:15 -0000 Subject: [Python-checkins] [3.6] bpo-32981: Fix catastrophic backtracking vulns (GH-5955) Message-ID: <mailman.114.1520146757.2831.python-checkins@python.org> https://github.com/python/cpython/commit/c9516754067d71fd7429a25ccfcb2141fc583523 commit: c9516754067d71fd7429a25ccfcb2141fc583523 branch: 3.6 author: Benjamin Peterson <benjamin at python.org> committer: GitHub <noreply at github.com> date: 2018-03-03T22:59:12-08:00 summary: [3.6] bpo-32981: Fix catastrophic backtracking vulns (GH-5955) * Prevent low-grade poplib REDOS (CVE-2018-1060) The regex to test a mail server's timestamp is susceptible to catastrophic backtracking on long evil responses from the server. Happily, the maximum length of malicious inputs is 2K thanks to a limit introduced in the fix for CVE-2013-1752. A 2KB evil response from the mail server would result in small slowdowns (milliseconds vs. microseconds) accumulated over many apop calls. This is a potential DOS vector via accumulated slowdowns. Replace it with a similar non-vulnerable regex. The new regex is RFC compliant. The old regex was non-compliant in edge cases. * Prevent difflib REDOS (CVE-2018-1061) The default regex for IS_LINE_JUNK is susceptible to catastrophic backtracking. This is a potential DOS vector. Replace it with an equivalent non-vulnerable regex. Also introduce unit and REDOS tests for difflib. Co-authored-by: Tim Peters <tim.peters at gmail.com> Co-authored-by: Christian Heimes <christian at python.org>. (cherry picked from commit 0e6c8ee2358a2e23117501826c008842acb835ac) files: A Misc/NEWS.d/next/Security/2018-03-02-10-24-52.bpo-32981.O_qDyj.rst M Lib/difflib.py M Lib/poplib.py M Lib/test/test_difflib.py M Lib/test/test_poplib.py M Misc/ACKS diff --git a/Lib/difflib.py b/Lib/difflib.py index 2095a5e517cb..72971d5b87a9 100644 --- a/Lib/difflib.py +++ b/Lib/difflib.py @@ -1083,7 +1083,7 @@ def _qformat(self, aline, bline, atags, btags): import re -def IS_LINE_JUNK(line, pat=re.compile(r"\s*#?\s*$").match): +def IS_LINE_JUNK(line, pat=re.compile(r"\s*(?:#\s*)?$").match): r""" Return 1 for ignorable line: iff `line` is blank or contains a single '#'. diff --git a/Lib/poplib.py b/Lib/poplib.py index 6bcfa5cfeba3..d8a62c034327 100644 --- a/Lib/poplib.py +++ b/Lib/poplib.py @@ -308,7 +308,7 @@ def rpop(self, user): return self._shortcmd('RPOP %s' % user) - timestamp = re.compile(br'\+OK.*(<[^>]+>)') + timestamp = re.compile(br'\+OK.[^<]*(<.*>)') def apop(self, user, password): """Authorisation diff --git a/Lib/test/test_difflib.py b/Lib/test/test_difflib.py index 156b523c38c1..aaefe6db0291 100644 --- a/Lib/test/test_difflib.py +++ b/Lib/test/test_difflib.py @@ -466,13 +466,33 @@ def _assert_type_error(self, msg, generator, *args): list(generator(*args)) self.assertEqual(msg, str(ctx.exception)) +class TestJunkAPIs(unittest.TestCase): + def test_is_line_junk_true(self): + for line in ['#', ' ', ' #', '# ', ' # ', '']: + self.assertTrue(difflib.IS_LINE_JUNK(line), repr(line)) + + def test_is_line_junk_false(self): + for line in ['##', ' ##', '## ', 'abc ', 'abc #', 'Mr. Moose is up!']: + self.assertFalse(difflib.IS_LINE_JUNK(line), repr(line)) + + def test_is_line_junk_REDOS(self): + evil_input = ('\t' * 1000000) + '##' + self.assertFalse(difflib.IS_LINE_JUNK(evil_input)) + + def test_is_character_junk_true(self): + for char in [' ', '\t']: + self.assertTrue(difflib.IS_CHARACTER_JUNK(char), repr(char)) + + def test_is_character_junk_false(self): + for char in ['a', '#', '\n', '\f', '\r', '\v']: + self.assertFalse(difflib.IS_CHARACTER_JUNK(char), repr(char)) def test_main(): difflib.HtmlDiff._default_prefix = 0 Doctests = doctest.DocTestSuite(difflib) run_unittest( TestWithAscii, TestAutojunk, TestSFpatches, TestSFbugs, - TestOutputFormat, TestBytes, Doctests) + TestOutputFormat, TestBytes, TestJunkAPIs, Doctests) if __name__ == '__main__': test_main() diff --git a/Lib/test/test_poplib.py b/Lib/test/test_poplib.py index 608eac57f35c..ca9bc6217509 100644 --- a/Lib/test/test_poplib.py +++ b/Lib/test/test_poplib.py @@ -303,9 +303,19 @@ def test_noop(self): def test_rpop(self): self.assertOK(self.client.rpop('foo')) - def test_apop(self): + def test_apop_normal(self): self.assertOK(self.client.apop('foo', 'dummypassword')) + def test_apop_REDOS(self): + # Replace welcome with very long evil welcome. + # NB The upper bound on welcome length is currently 2048. + # At this length, evil input makes each apop call take + # on the order of milliseconds instead of microseconds. + evil_welcome = b'+OK' + (b'<' * 1000000) + with test_support.swap_attr(self.client, 'welcome', evil_welcome): + # The evil welcome is invalid, so apop should throw. + self.assertRaises(poplib.error_proto, self.client.apop, 'a', 'kb') + def test_top(self): expected = (b'+OK 116 bytes', [b'From: postmaster at python.org', b'Content-Type: text/plain', diff --git a/Misc/ACKS b/Misc/ACKS index 75c9b3f0563d..9403e110675a 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -351,6 +351,7 @@ Jonathan Dasteel Pierre-Yves David A. Jesse Jiryu Davis Jake Davis +Jamie (James C.) Davis Merlijn van Deen John DeGood Ned Deily diff --git a/Misc/NEWS.d/next/Security/2018-03-02-10-24-52.bpo-32981.O_qDyj.rst b/Misc/NEWS.d/next/Security/2018-03-02-10-24-52.bpo-32981.O_qDyj.rst new file mode 100644 index 000000000000..9ebabb44f91e --- /dev/null +++ b/Misc/NEWS.d/next/Security/2018-03-02-10-24-52.bpo-32981.O_qDyj.rst @@ -0,0 +1,4 @@ +Regexes in difflib and poplib were vulnerable to catastrophic backtracking. +These regexes formed potential DOS vectors (REDOS). They have been +refactored. This resolves CVE-2018-1060 and CVE-2018-1061. +Patch by Jamie Davis. From solipsis at pitrou.net Sun Mar 4 04:10:59 2018 From: solipsis at pitrou.net (solipsis at pitrou.net) Date: Sun, 04 Mar 2018 09:10:59 +0000 Subject: [Python-checkins] Daily reference leaks (4243df51fe43): sum=0 Message-ID: <20180304091059.1.7A2CF341C0B27086@psf.io> results for 4243df51fe43 on branch "default" -------------------------------------------- test_collections leaked [-7, 1, 0] memory blocks, sum=-6 test_functools leaked [0, 3, 1] memory blocks, sum=4 test_multiprocessing_fork leaked [2, -1, 1] memory blocks, sum=2 Command line was: ['./python', '-m', 'test.regrtest', '-uall', '-R', '3:3:/home/psf-users/antoine/refleaks/reflogUgVuuC', '--timeout', '7200'] From webhook-mailer at python.org Sun Mar 4 05:41:52 2018 From: webhook-mailer at python.org (Serhiy Storchaka) Date: Sun, 04 Mar 2018 10:41:52 -0000 Subject: [Python-checkins] bpo-32857: Raise error when tkinter after_cancel() is called with None. (GH-5701) Message-ID: <mailman.115.1520160113.2831.python-checkins@python.org> https://github.com/python/cpython/commit/74382a3f175ac285cc924a73fd758e8dc3cc41bb commit: 74382a3f175ac285cc924a73fd758e8dc3cc41bb branch: master author: Cheryl Sabella <cheryl.sabella at gmail.com> committer: Serhiy Storchaka <storchaka at gmail.com> date: 2018-03-04T12:41:47+02:00 summary: bpo-32857: Raise error when tkinter after_cancel() is called with None. (GH-5701) files: A Misc/NEWS.d/next/Library/2018-02-16-14-37-14.bpo-32857.-XljAx.rst M Lib/tkinter/__init__.py M Lib/tkinter/test/test_tkinter/test_misc.py diff --git a/Lib/tkinter/__init__.py b/Lib/tkinter/__init__.py index deea791831ed..53bad3fa95ae 100644 --- a/Lib/tkinter/__init__.py +++ b/Lib/tkinter/__init__.py @@ -739,6 +739,7 @@ def after(self, ms, func=None, *args): if not func: # I'd rather use time.sleep(ms*0.001) self.tk.call('after', ms) + return None else: def callit(): try: @@ -762,11 +763,13 @@ def after_cancel(self, id): """Cancel scheduling of function identified with ID. Identifier returned by after or after_idle must be - given as first parameter.""" + given as first parameter. + """ + if not id: + raise ValueError('id must be a valid identifier returned from ' + 'after or after_idle') try: data = self.tk.call('after', 'info', id) - # In Tk 8.3, splitlist returns: (script, type) - # In Tk 8.4, splitlist may return (script, type) or (script,) script = self.tk.splitlist(data)[0] self.deletecommand(script) except TclError: diff --git a/Lib/tkinter/test/test_tkinter/test_misc.py b/Lib/tkinter/test/test_tkinter/test_misc.py index 9dc1e37547fc..1d1a3c29f6bc 100644 --- a/Lib/tkinter/test/test_tkinter/test_misc.py +++ b/Lib/tkinter/test/test_tkinter/test_misc.py @@ -48,6 +48,114 @@ def test_tk_setPalette(self): '^must specify a background color$', root.tk_setPalette, highlightColor='blue') + def test_after(self): + root = self.root + + def callback(start=0, step=1): + nonlocal count + count = start + step + + # Without function, sleeps for ms. + self.assertIsNone(root.after(1)) + + # Set up with callback with no args. + count = 0 + timer1 = root.after(0, callback) + self.assertIn(timer1, root.tk.call('after', 'info')) + (script, _) = root.tk.splitlist(root.tk.call('after', 'info', timer1)) + root.update() # Process all pending events. + self.assertEqual(count, 1) + with self.assertRaises(tkinter.TclError): + root.tk.call(script) + + # Set up with callback with args. + count = 0 + timer1 = root.after(0, callback, 42, 11) + root.update() # Process all pending events. + self.assertEqual(count, 53) + + # Cancel before called. + timer1 = root.after(1000, callback) + self.assertIn(timer1, root.tk.call('after', 'info')) + (script, _) = root.tk.splitlist(root.tk.call('after', 'info', timer1)) + root.after_cancel(timer1) # Cancel this event. + self.assertEqual(count, 53) + with self.assertRaises(tkinter.TclError): + root.tk.call(script) + + def test_after_idle(self): + root = self.root + + def callback(start=0, step=1): + nonlocal count + count = start + step + + # Set up with callback with no args. + count = 0 + idle1 = root.after_idle(callback) + self.assertIn(idle1, root.tk.call('after', 'info')) + (script, _) = root.tk.splitlist(root.tk.call('after', 'info', idle1)) + root.update_idletasks() # Process all pending events. + self.assertEqual(count, 1) + with self.assertRaises(tkinter.TclError): + root.tk.call(script) + + # Set up with callback with args. + count = 0 + idle1 = root.after_idle(callback, 42, 11) + root.update_idletasks() # Process all pending events. + self.assertEqual(count, 53) + + # Cancel before called. + idle1 = root.after_idle(callback) + self.assertIn(idle1, root.tk.call('after', 'info')) + (script, _) = root.tk.splitlist(root.tk.call('after', 'info', idle1)) + root.after_cancel(idle1) # Cancel this event. + self.assertEqual(count, 53) + with self.assertRaises(tkinter.TclError): + root.tk.call(script) + + def test_after_cancel(self): + root = self.root + + def callback(): + nonlocal count + count += 1 + + timer1 = root.after(5000, callback) + idle1 = root.after_idle(callback) + + # No value for id raises a ValueError. + with self.assertRaises(ValueError): + root.after_cancel(None) + + # Cancel timer event. + count = 0 + (script, _) = root.tk.splitlist(root.tk.call('after', 'info', timer1)) + root.tk.call(script) + self.assertEqual(count, 1) + root.after_cancel(timer1) + with self.assertRaises(tkinter.TclError): + root.tk.call(script) + self.assertEqual(count, 1) + with self.assertRaises(tkinter.TclError): + root.tk.call('after', 'info', timer1) + + # Cancel same event - nothing happens. + root.after_cancel(timer1) + + # Cancel idle event. + count = 0 + (script, _) = root.tk.splitlist(root.tk.call('after', 'info', idle1)) + root.tk.call(script) + self.assertEqual(count, 1) + root.after_cancel(idle1) + with self.assertRaises(tkinter.TclError): + root.tk.call(script) + self.assertEqual(count, 1) + with self.assertRaises(tkinter.TclError): + root.tk.call('after', 'info', idle1) + tests_gui = (MiscTest, ) diff --git a/Misc/NEWS.d/next/Library/2018-02-16-14-37-14.bpo-32857.-XljAx.rst b/Misc/NEWS.d/next/Library/2018-02-16-14-37-14.bpo-32857.-XljAx.rst new file mode 100644 index 000000000000..4ebbde4d1946 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-02-16-14-37-14.bpo-32857.-XljAx.rst @@ -0,0 +1 @@ +In :mod:`tkinter`, ``after_cancel(None)`` now raises a :exc:`ValueError` instead of canceling the first scheduled function. Patch by Cheryl Sabella. From webhook-mailer at python.org Sun Mar 4 06:42:48 2018 From: webhook-mailer at python.org (Miss Islington (bot)) Date: Sun, 04 Mar 2018 11:42:48 -0000 Subject: [Python-checkins] bpo-32857: Raise error when tkinter after_cancel() is called with None. (GH-5701) Message-ID: <mailman.116.1520163768.2831.python-checkins@python.org> https://github.com/python/cpython/commit/73a43960c7be50e136c5482404980175cb99f611 commit: 73a43960c7be50e136c5482404980175cb99f611 branch: 3.6 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: GitHub <noreply at github.com> date: 2018-03-04T03:42:41-08:00 summary: bpo-32857: Raise error when tkinter after_cancel() is called with None. (GH-5701) (cherry picked from commit 74382a3f175ac285cc924a73fd758e8dc3cc41bb) Co-authored-by: Cheryl Sabella <cheryl.sabella at gmail.com> files: A Misc/NEWS.d/next/Library/2018-02-16-14-37-14.bpo-32857.-XljAx.rst M Lib/tkinter/__init__.py M Lib/tkinter/test/test_tkinter/test_misc.py diff --git a/Lib/tkinter/__init__.py b/Lib/tkinter/__init__.py index deea791831ed..53bad3fa95ae 100644 --- a/Lib/tkinter/__init__.py +++ b/Lib/tkinter/__init__.py @@ -739,6 +739,7 @@ def after(self, ms, func=None, *args): if not func: # I'd rather use time.sleep(ms*0.001) self.tk.call('after', ms) + return None else: def callit(): try: @@ -762,11 +763,13 @@ def after_cancel(self, id): """Cancel scheduling of function identified with ID. Identifier returned by after or after_idle must be - given as first parameter.""" + given as first parameter. + """ + if not id: + raise ValueError('id must be a valid identifier returned from ' + 'after or after_idle') try: data = self.tk.call('after', 'info', id) - # In Tk 8.3, splitlist returns: (script, type) - # In Tk 8.4, splitlist may return (script, type) or (script,) script = self.tk.splitlist(data)[0] self.deletecommand(script) except TclError: diff --git a/Lib/tkinter/test/test_tkinter/test_misc.py b/Lib/tkinter/test/test_tkinter/test_misc.py index 9dc1e37547fc..1d1a3c29f6bc 100644 --- a/Lib/tkinter/test/test_tkinter/test_misc.py +++ b/Lib/tkinter/test/test_tkinter/test_misc.py @@ -48,6 +48,114 @@ def test_tk_setPalette(self): '^must specify a background color$', root.tk_setPalette, highlightColor='blue') + def test_after(self): + root = self.root + + def callback(start=0, step=1): + nonlocal count + count = start + step + + # Without function, sleeps for ms. + self.assertIsNone(root.after(1)) + + # Set up with callback with no args. + count = 0 + timer1 = root.after(0, callback) + self.assertIn(timer1, root.tk.call('after', 'info')) + (script, _) = root.tk.splitlist(root.tk.call('after', 'info', timer1)) + root.update() # Process all pending events. + self.assertEqual(count, 1) + with self.assertRaises(tkinter.TclError): + root.tk.call(script) + + # Set up with callback with args. + count = 0 + timer1 = root.after(0, callback, 42, 11) + root.update() # Process all pending events. + self.assertEqual(count, 53) + + # Cancel before called. + timer1 = root.after(1000, callback) + self.assertIn(timer1, root.tk.call('after', 'info')) + (script, _) = root.tk.splitlist(root.tk.call('after', 'info', timer1)) + root.after_cancel(timer1) # Cancel this event. + self.assertEqual(count, 53) + with self.assertRaises(tkinter.TclError): + root.tk.call(script) + + def test_after_idle(self): + root = self.root + + def callback(start=0, step=1): + nonlocal count + count = start + step + + # Set up with callback with no args. + count = 0 + idle1 = root.after_idle(callback) + self.assertIn(idle1, root.tk.call('after', 'info')) + (script, _) = root.tk.splitlist(root.tk.call('after', 'info', idle1)) + root.update_idletasks() # Process all pending events. + self.assertEqual(count, 1) + with self.assertRaises(tkinter.TclError): + root.tk.call(script) + + # Set up with callback with args. + count = 0 + idle1 = root.after_idle(callback, 42, 11) + root.update_idletasks() # Process all pending events. + self.assertEqual(count, 53) + + # Cancel before called. + idle1 = root.after_idle(callback) + self.assertIn(idle1, root.tk.call('after', 'info')) + (script, _) = root.tk.splitlist(root.tk.call('after', 'info', idle1)) + root.after_cancel(idle1) # Cancel this event. + self.assertEqual(count, 53) + with self.assertRaises(tkinter.TclError): + root.tk.call(script) + + def test_after_cancel(self): + root = self.root + + def callback(): + nonlocal count + count += 1 + + timer1 = root.after(5000, callback) + idle1 = root.after_idle(callback) + + # No value for id raises a ValueError. + with self.assertRaises(ValueError): + root.after_cancel(None) + + # Cancel timer event. + count = 0 + (script, _) = root.tk.splitlist(root.tk.call('after', 'info', timer1)) + root.tk.call(script) + self.assertEqual(count, 1) + root.after_cancel(timer1) + with self.assertRaises(tkinter.TclError): + root.tk.call(script) + self.assertEqual(count, 1) + with self.assertRaises(tkinter.TclError): + root.tk.call('after', 'info', timer1) + + # Cancel same event - nothing happens. + root.after_cancel(timer1) + + # Cancel idle event. + count = 0 + (script, _) = root.tk.splitlist(root.tk.call('after', 'info', idle1)) + root.tk.call(script) + self.assertEqual(count, 1) + root.after_cancel(idle1) + with self.assertRaises(tkinter.TclError): + root.tk.call(script) + self.assertEqual(count, 1) + with self.assertRaises(tkinter.TclError): + root.tk.call('after', 'info', idle1) + tests_gui = (MiscTest, ) diff --git a/Misc/NEWS.d/next/Library/2018-02-16-14-37-14.bpo-32857.-XljAx.rst b/Misc/NEWS.d/next/Library/2018-02-16-14-37-14.bpo-32857.-XljAx.rst new file mode 100644 index 000000000000..4ebbde4d1946 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-02-16-14-37-14.bpo-32857.-XljAx.rst @@ -0,0 +1 @@ +In :mod:`tkinter`, ``after_cancel(None)`` now raises a :exc:`ValueError` instead of canceling the first scheduled function. Patch by Cheryl Sabella. From webhook-mailer at python.org Sun Mar 4 07:00:36 2018 From: webhook-mailer at python.org (Miss Islington (bot)) Date: Sun, 04 Mar 2018 12:00:36 -0000 Subject: [Python-checkins] bpo-32857: Raise error when tkinter after_cancel() is called with None. (GH-5701) Message-ID: <mailman.117.1520164837.2831.python-checkins@python.org> https://github.com/python/cpython/commit/a5303dd9c263b337f02dda0038f2f5a10208140c commit: a5303dd9c263b337f02dda0038f2f5a10208140c branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: GitHub <noreply at github.com> date: 2018-03-04T04:00:33-08:00 summary: bpo-32857: Raise error when tkinter after_cancel() is called with None. (GH-5701) (cherry picked from commit 74382a3f175ac285cc924a73fd758e8dc3cc41bb) Co-authored-by: Cheryl Sabella <cheryl.sabella at gmail.com> files: A Misc/NEWS.d/next/Library/2018-02-16-14-37-14.bpo-32857.-XljAx.rst M Lib/tkinter/__init__.py M Lib/tkinter/test/test_tkinter/test_misc.py diff --git a/Lib/tkinter/__init__.py b/Lib/tkinter/__init__.py index deea791831ed..53bad3fa95ae 100644 --- a/Lib/tkinter/__init__.py +++ b/Lib/tkinter/__init__.py @@ -739,6 +739,7 @@ def after(self, ms, func=None, *args): if not func: # I'd rather use time.sleep(ms*0.001) self.tk.call('after', ms) + return None else: def callit(): try: @@ -762,11 +763,13 @@ def after_cancel(self, id): """Cancel scheduling of function identified with ID. Identifier returned by after or after_idle must be - given as first parameter.""" + given as first parameter. + """ + if not id: + raise ValueError('id must be a valid identifier returned from ' + 'after or after_idle') try: data = self.tk.call('after', 'info', id) - # In Tk 8.3, splitlist returns: (script, type) - # In Tk 8.4, splitlist may return (script, type) or (script,) script = self.tk.splitlist(data)[0] self.deletecommand(script) except TclError: diff --git a/Lib/tkinter/test/test_tkinter/test_misc.py b/Lib/tkinter/test/test_tkinter/test_misc.py index 9dc1e37547fc..1d1a3c29f6bc 100644 --- a/Lib/tkinter/test/test_tkinter/test_misc.py +++ b/Lib/tkinter/test/test_tkinter/test_misc.py @@ -48,6 +48,114 @@ def test_tk_setPalette(self): '^must specify a background color$', root.tk_setPalette, highlightColor='blue') + def test_after(self): + root = self.root + + def callback(start=0, step=1): + nonlocal count + count = start + step + + # Without function, sleeps for ms. + self.assertIsNone(root.after(1)) + + # Set up with callback with no args. + count = 0 + timer1 = root.after(0, callback) + self.assertIn(timer1, root.tk.call('after', 'info')) + (script, _) = root.tk.splitlist(root.tk.call('after', 'info', timer1)) + root.update() # Process all pending events. + self.assertEqual(count, 1) + with self.assertRaises(tkinter.TclError): + root.tk.call(script) + + # Set up with callback with args. + count = 0 + timer1 = root.after(0, callback, 42, 11) + root.update() # Process all pending events. + self.assertEqual(count, 53) + + # Cancel before called. + timer1 = root.after(1000, callback) + self.assertIn(timer1, root.tk.call('after', 'info')) + (script, _) = root.tk.splitlist(root.tk.call('after', 'info', timer1)) + root.after_cancel(timer1) # Cancel this event. + self.assertEqual(count, 53) + with self.assertRaises(tkinter.TclError): + root.tk.call(script) + + def test_after_idle(self): + root = self.root + + def callback(start=0, step=1): + nonlocal count + count = start + step + + # Set up with callback with no args. + count = 0 + idle1 = root.after_idle(callback) + self.assertIn(idle1, root.tk.call('after', 'info')) + (script, _) = root.tk.splitlist(root.tk.call('after', 'info', idle1)) + root.update_idletasks() # Process all pending events. + self.assertEqual(count, 1) + with self.assertRaises(tkinter.TclError): + root.tk.call(script) + + # Set up with callback with args. + count = 0 + idle1 = root.after_idle(callback, 42, 11) + root.update_idletasks() # Process all pending events. + self.assertEqual(count, 53) + + # Cancel before called. + idle1 = root.after_idle(callback) + self.assertIn(idle1, root.tk.call('after', 'info')) + (script, _) = root.tk.splitlist(root.tk.call('after', 'info', idle1)) + root.after_cancel(idle1) # Cancel this event. + self.assertEqual(count, 53) + with self.assertRaises(tkinter.TclError): + root.tk.call(script) + + def test_after_cancel(self): + root = self.root + + def callback(): + nonlocal count + count += 1 + + timer1 = root.after(5000, callback) + idle1 = root.after_idle(callback) + + # No value for id raises a ValueError. + with self.assertRaises(ValueError): + root.after_cancel(None) + + # Cancel timer event. + count = 0 + (script, _) = root.tk.splitlist(root.tk.call('after', 'info', timer1)) + root.tk.call(script) + self.assertEqual(count, 1) + root.after_cancel(timer1) + with self.assertRaises(tkinter.TclError): + root.tk.call(script) + self.assertEqual(count, 1) + with self.assertRaises(tkinter.TclError): + root.tk.call('after', 'info', timer1) + + # Cancel same event - nothing happens. + root.after_cancel(timer1) + + # Cancel idle event. + count = 0 + (script, _) = root.tk.splitlist(root.tk.call('after', 'info', idle1)) + root.tk.call(script) + self.assertEqual(count, 1) + root.after_cancel(idle1) + with self.assertRaises(tkinter.TclError): + root.tk.call(script) + self.assertEqual(count, 1) + with self.assertRaises(tkinter.TclError): + root.tk.call('after', 'info', idle1) + tests_gui = (MiscTest, ) diff --git a/Misc/NEWS.d/next/Library/2018-02-16-14-37-14.bpo-32857.-XljAx.rst b/Misc/NEWS.d/next/Library/2018-02-16-14-37-14.bpo-32857.-XljAx.rst new file mode 100644 index 000000000000..4ebbde4d1946 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-02-16-14-37-14.bpo-32857.-XljAx.rst @@ -0,0 +1 @@ +In :mod:`tkinter`, ``after_cancel(None)`` now raises a :exc:`ValueError` instead of canceling the first scheduled function. Patch by Cheryl Sabella. From webhook-mailer at python.org Sun Mar 4 18:07:00 2018 From: webhook-mailer at python.org (Ned Deily) Date: Sun, 04 Mar 2018 23:07:00 -0000 Subject: [Python-checkins] bpo-30147: Add re.escape changes to 3.7 What's New (GH-5978) Message-ID: <mailman.118.1520204821.2831.python-checkins@python.org> https://github.com/python/cpython/commit/18fd89246333bfa1b76c1623df689214f3ce2bf3 commit: 18fd89246333bfa1b76c1623df689214f3ce2bf3 branch: master author: Cheryl Sabella <cheryl.sabella at gmail.com> committer: Ned Deily <nad at python.org> date: 2018-03-04T18:06:57-05:00 summary: bpo-30147: Add re.escape changes to 3.7 What's New (GH-5978) files: M Doc/whatsnew/3.7.rst diff --git a/Doc/whatsnew/3.7.rst b/Doc/whatsnew/3.7.rst index 46f4f13aab47..9e65488e37d6 100644 --- a/Doc/whatsnew/3.7.rst +++ b/Doc/whatsnew/3.7.rst @@ -1214,6 +1214,10 @@ Changes in the Python API (Contributed by Serhiy Storchaka in :issue:`25054` and :issue:`32308`.) +* Change :func:`re.escape` to only escape regex special characters instead + of escaping all characters other than ASCII letters, numbers, and ``'_'``. + (Contributed by Serhiy Storchaka in :issue:`29995`.) + * :class:`tracemalloc.Traceback` frames are now sorted from oldest to most recent to be more consistent with :mod:`traceback`. (Contributed by Jesse Bakker in :issue:`32121`.) From webhook-mailer at python.org Sun Mar 4 18:22:35 2018 From: webhook-mailer at python.org (Ned Deily) Date: Sun, 04 Mar 2018 23:22:35 -0000 Subject: [Python-checkins] [3.7] bpo-30147: Add re.escape changes to 3.7 What's New (GH-5978) (GH-5979) Message-ID: <mailman.119.1520205756.2831.python-checkins@python.org> https://github.com/python/cpython/commit/f92478d57a6b4014dcc0882d43b534fae1e7b929 commit: f92478d57a6b4014dcc0882d43b534fae1e7b929 branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Ned Deily <nad at python.org> date: 2018-03-04T18:22:32-05:00 summary: [3.7] bpo-30147: Add re.escape changes to 3.7 What's New (GH-5978) (GH-5979) (cherry picked from commit 18fd89246333bfa1b76c1623df689214f3ce2bf3) Co-authored-by: Cheryl Sabella <cheryl.sabella at gmail.com> files: M Doc/whatsnew/3.7.rst diff --git a/Doc/whatsnew/3.7.rst b/Doc/whatsnew/3.7.rst index 46f4f13aab47..9e65488e37d6 100644 --- a/Doc/whatsnew/3.7.rst +++ b/Doc/whatsnew/3.7.rst @@ -1214,6 +1214,10 @@ Changes in the Python API (Contributed by Serhiy Storchaka in :issue:`25054` and :issue:`32308`.) +* Change :func:`re.escape` to only escape regex special characters instead + of escaping all characters other than ASCII letters, numbers, and ``'_'``. + (Contributed by Serhiy Storchaka in :issue:`29995`.) + * :class:`tracemalloc.Traceback` frames are now sorted from oldest to most recent to be more consistent with :mod:`traceback`. (Contributed by Jesse Bakker in :issue:`32121`.) From solipsis at pitrou.net Mon Mar 5 04:11:02 2018 From: solipsis at pitrou.net (solipsis at pitrou.net) Date: Mon, 05 Mar 2018 09:11:02 +0000 Subject: [Python-checkins] Daily reference leaks (4243df51fe43): sum=4 Message-ID: <20180305091102.1.C253719D3C3EC870@psf.io> results for 4243df51fe43 on branch "default" -------------------------------------------- test_collections leaked [0, 7, -7] memory blocks, sum=0 test_functools leaked [0, 3, 1] memory blocks, sum=4 Command line was: ['./python', '-m', 'test.regrtest', '-uall', '-R', '3:3:/home/psf-users/antoine/refleaks/reflogy0oxKo', '--timeout', '7200'] From webhook-mailer at python.org Mon Mar 5 11:02:49 2018 From: webhook-mailer at python.org (Terry Jan Reedy) Date: Mon, 05 Mar 2018 16:02:49 -0000 Subject: [Python-checkins] bpo-32984: IDLE - set __file__ for startup files (GH-5981) Message-ID: <mailman.120.1520265771.2831.python-checkins@python.org> https://github.com/python/cpython/commit/22c82be5df70c3d51e3f89b54fe1d4fb84728c1e commit: 22c82be5df70c3d51e3f89b54fe1d4fb84728c1e branch: master author: Terry Jan Reedy <tjreedy at udel.edu> committer: GitHub <noreply at github.com> date: 2018-03-05T11:02:46-05:00 summary: bpo-32984: IDLE - set __file__ for startup files (GH-5981) Like Python, IDLE optionally runs one startup file in the Shell window before presenting the first interactive input prompt. For IDLE, option -s runs a file named in environmental variable IDLESTARTUP or PYTHONSTARTUP; -r file runs file. Python sets __file__ to the startup file name before running the file and unsets it before the first prompt. IDLE now does the same when run normally, without the -n option. files: A Misc/NEWS.d/next/IDLE/2018-03-05-01-29-05.bpo-32984.NGjgT4.rst M Lib/idlelib/pyshell.py diff --git a/Lib/idlelib/pyshell.py b/Lib/idlelib/pyshell.py index 8b07d52cc487..ee1313161da3 100755 --- a/Lib/idlelib/pyshell.py +++ b/Lib/idlelib/pyshell.py @@ -635,6 +635,9 @@ def execfile(self, filename, source=None): if source is None: with tokenize.open(filename) as fp: source = fp.read() + if use_subprocess: + source = (f"__file__ = r'''{os.path.abspath(filename)}'''\n" + + source + "\ndel __file__") try: code = compile(source, filename, "exec") except (OverflowError, SyntaxError): diff --git a/Misc/NEWS.d/next/IDLE/2018-03-05-01-29-05.bpo-32984.NGjgT4.rst b/Misc/NEWS.d/next/IDLE/2018-03-05-01-29-05.bpo-32984.NGjgT4.rst new file mode 100644 index 000000000000..15d40b72caaf --- /dev/null +++ b/Misc/NEWS.d/next/IDLE/2018-03-05-01-29-05.bpo-32984.NGjgT4.rst @@ -0,0 +1,7 @@ +Set ``__file__`` while running a startup file. Like Python, IDLE optionally +runs one startup file in the Shell window before presenting the first interactive +input prompt. For IDLE, ``-s`` runs a file named in environmental variable + :envvar:`IDLESTARTUP` or :envvar:`PYTHONSTARTUP`; ``-r file`` runs +``file``. Python sets ``__file__`` to the startup file name before running the +file and unsets it before the first prompt. IDLE now does the same when run +normally, without the ``-n`` option. From webhook-mailer at python.org Mon Mar 5 11:49:21 2018 From: webhook-mailer at python.org (Miss Islington (bot)) Date: Mon, 05 Mar 2018 16:49:21 -0000 Subject: [Python-checkins] bpo-32984: IDLE - set __file__ for startup files (GH-5981) Message-ID: <mailman.121.1520268562.2831.python-checkins@python.org> https://github.com/python/cpython/commit/fd340bf9e308130736c76257ff9a697edbeb082d commit: fd340bf9e308130736c76257ff9a697edbeb082d branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: GitHub <noreply at github.com> date: 2018-03-05T08:49:15-08:00 summary: bpo-32984: IDLE - set __file__ for startup files (GH-5981) Like Python, IDLE optionally runs one startup file in the Shell window before presenting the first interactive input prompt. For IDLE, option -s runs a file named in environmental variable IDLESTARTUP or PYTHONSTARTUP; -r file runs file. Python sets __file__ to the startup file name before running the file and unsets it before the first prompt. IDLE now does the same when run normally, without the -n option. (cherry picked from commit 22c82be5df70c3d51e3f89b54fe1d4fb84728c1e) Co-authored-by: Terry Jan Reedy <tjreedy at udel.edu> files: A Misc/NEWS.d/next/IDLE/2018-03-05-01-29-05.bpo-32984.NGjgT4.rst M Lib/idlelib/pyshell.py diff --git a/Lib/idlelib/pyshell.py b/Lib/idlelib/pyshell.py index 8b07d52cc487..ee1313161da3 100755 --- a/Lib/idlelib/pyshell.py +++ b/Lib/idlelib/pyshell.py @@ -635,6 +635,9 @@ def execfile(self, filename, source=None): if source is None: with tokenize.open(filename) as fp: source = fp.read() + if use_subprocess: + source = (f"__file__ = r'''{os.path.abspath(filename)}'''\n" + + source + "\ndel __file__") try: code = compile(source, filename, "exec") except (OverflowError, SyntaxError): diff --git a/Misc/NEWS.d/next/IDLE/2018-03-05-01-29-05.bpo-32984.NGjgT4.rst b/Misc/NEWS.d/next/IDLE/2018-03-05-01-29-05.bpo-32984.NGjgT4.rst new file mode 100644 index 000000000000..15d40b72caaf --- /dev/null +++ b/Misc/NEWS.d/next/IDLE/2018-03-05-01-29-05.bpo-32984.NGjgT4.rst @@ -0,0 +1,7 @@ +Set ``__file__`` while running a startup file. Like Python, IDLE optionally +runs one startup file in the Shell window before presenting the first interactive +input prompt. For IDLE, ``-s`` runs a file named in environmental variable + :envvar:`IDLESTARTUP` or :envvar:`PYTHONSTARTUP`; ``-r file`` runs +``file``. Python sets ``__file__`` to the startup file name before running the +file and unsets it before the first prompt. IDLE now does the same when run +normally, without the ``-n`` option. From webhook-mailer at python.org Mon Mar 5 14:23:11 2018 From: webhook-mailer at python.org (Miss Islington (bot)) Date: Mon, 05 Mar 2018 19:23:11 -0000 Subject: [Python-checkins] bpo-32984: IDLE - set __file__ for startup files (GH-5981) Message-ID: <mailman.122.1520277793.2831.python-checkins@python.org> https://github.com/python/cpython/commit/6935a511670797a3aaebdf96aad3dcff66baa76e commit: 6935a511670797a3aaebdf96aad3dcff66baa76e branch: 3.6 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: GitHub <noreply at github.com> date: 2018-03-05T11:23:08-08:00 summary: bpo-32984: IDLE - set __file__ for startup files (GH-5981) Like Python, IDLE optionally runs one startup file in the Shell window before presenting the first interactive input prompt. For IDLE, option -s runs a file named in environmental variable IDLESTARTUP or PYTHONSTARTUP; -r file runs file. Python sets __file__ to the startup file name before running the file and unsets it before the first prompt. IDLE now does the same when run normally, without the -n option. (cherry picked from commit 22c82be5df70c3d51e3f89b54fe1d4fb84728c1e) Co-authored-by: Terry Jan Reedy <tjreedy at udel.edu> files: A Misc/NEWS.d/next/IDLE/2018-03-05-01-29-05.bpo-32984.NGjgT4.rst M Lib/idlelib/pyshell.py diff --git a/Lib/idlelib/pyshell.py b/Lib/idlelib/pyshell.py index 8b07d52cc487..ee1313161da3 100755 --- a/Lib/idlelib/pyshell.py +++ b/Lib/idlelib/pyshell.py @@ -635,6 +635,9 @@ def execfile(self, filename, source=None): if source is None: with tokenize.open(filename) as fp: source = fp.read() + if use_subprocess: + source = (f"__file__ = r'''{os.path.abspath(filename)}'''\n" + + source + "\ndel __file__") try: code = compile(source, filename, "exec") except (OverflowError, SyntaxError): diff --git a/Misc/NEWS.d/next/IDLE/2018-03-05-01-29-05.bpo-32984.NGjgT4.rst b/Misc/NEWS.d/next/IDLE/2018-03-05-01-29-05.bpo-32984.NGjgT4.rst new file mode 100644 index 000000000000..15d40b72caaf --- /dev/null +++ b/Misc/NEWS.d/next/IDLE/2018-03-05-01-29-05.bpo-32984.NGjgT4.rst @@ -0,0 +1,7 @@ +Set ``__file__`` while running a startup file. Like Python, IDLE optionally +runs one startup file in the Shell window before presenting the first interactive +input prompt. For IDLE, ``-s`` runs a file named in environmental variable + :envvar:`IDLESTARTUP` or :envvar:`PYTHONSTARTUP`; ``-r file`` runs +``file``. Python sets ``__file__`` to the startup file name before running the +file and unsets it before the first prompt. IDLE now does the same when run +normally, without the ``-n`` option. From webhook-mailer at python.org Mon Mar 5 15:37:21 2018 From: webhook-mailer at python.org (Barry Warsaw) Date: Mon, 05 Mar 2018 20:37:21 -0000 Subject: [Python-checkins] Add What's New for issues 32303 and 32305 (GH-5994) Message-ID: <mailman.123.1520282242.2831.python-checkins@python.org> https://github.com/python/cpython/commit/4c19b95734faee4c390c1d0569dc876980c33d2c commit: 4c19b95734faee4c390c1d0569dc876980c33d2c branch: master author: Barry Warsaw <barry at python.org> committer: GitHub <noreply at github.com> date: 2018-03-05T12:37:12-08:00 summary: Add What's New for issues 32303 and 32305 (GH-5994) files: M Doc/whatsnew/3.7.rst diff --git a/Doc/whatsnew/3.7.rst b/Doc/whatsnew/3.7.rst index 9e65488e37d6..76e1f7b36b0b 100644 --- a/Doc/whatsnew/3.7.rst +++ b/Doc/whatsnew/3.7.rst @@ -958,6 +958,14 @@ Other CPython Implementation Changes by setting the new ``f_trace_opcodes`` attribute to :const:`True` on the frame being traced. (Contributed by Nick Coghlan in :issue:`31344`.) +* Fixed some consistency problems with namespace package module attributes. + Namespace module objects now have an ``__file__`` that is set to ``None`` + (previously unset), and their ``__spec__.origin`` is also set to ``None`` + (previously the string ``"namespace"``). See :issue:`32305`. Also, the + namespace module object's ``__spec__.loader`` is set to the same value as + ``__loader__`` (previously, the former was set to ``None``). See + :issue:`32303`. + Deprecated ========== From webhook-mailer at python.org Mon Mar 5 15:51:59 2018 From: webhook-mailer at python.org (Barry Warsaw) Date: Mon, 05 Mar 2018 20:51:59 -0000 Subject: [Python-checkins] Add What's New for issues 32303 and 32305 (GH-5994) (GH-5995) Message-ID: <mailman.124.1520283121.2831.python-checkins@python.org> https://github.com/python/cpython/commit/063c637cdbe4528b175f7d6122fa29faf66707ee commit: 063c637cdbe4528b175f7d6122fa29faf66707ee branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Barry Warsaw <barry at python.org> date: 2018-03-05T12:51:56-08:00 summary: Add What's New for issues 32303 and 32305 (GH-5994) (GH-5995) (cherry picked from commit 4c19b95734faee4c390c1d0569dc876980c33d2c) Co-authored-by: Barry Warsaw <barry at python.org> files: M Doc/whatsnew/3.7.rst diff --git a/Doc/whatsnew/3.7.rst b/Doc/whatsnew/3.7.rst index 9e65488e37d6..76e1f7b36b0b 100644 --- a/Doc/whatsnew/3.7.rst +++ b/Doc/whatsnew/3.7.rst @@ -958,6 +958,14 @@ Other CPython Implementation Changes by setting the new ``f_trace_opcodes`` attribute to :const:`True` on the frame being traced. (Contributed by Nick Coghlan in :issue:`31344`.) +* Fixed some consistency problems with namespace package module attributes. + Namespace module objects now have an ``__file__`` that is set to ``None`` + (previously unset), and their ``__spec__.origin`` is also set to ``None`` + (previously the string ``"namespace"``). See :issue:`32305`. Also, the + namespace module object's ``__spec__.loader`` is set to the same value as + ``__loader__`` (previously, the former was set to ``None``). See + :issue:`32303`. + Deprecated ========== From webhook-mailer at python.org Mon Mar 5 17:26:18 2018 From: webhook-mailer at python.org (Steve Dower) Date: Mon, 05 Mar 2018 22:26:18 -0000 Subject: [Python-checkins] bpo-33001: Prevent buffer overrun in os.symlink (GH-5989) Message-ID: <mailman.125.1520288779.2831.python-checkins@python.org> https://github.com/python/cpython/commit/6921e73e33edc3c61bc2d78ed558eaa22a89a564 commit: 6921e73e33edc3c61bc2d78ed558eaa22a89a564 branch: master author: Steve Dower <steve.dower at microsoft.com> committer: GitHub <noreply at github.com> date: 2018-03-05T14:26:08-08:00 summary: bpo-33001: Prevent buffer overrun in os.symlink (GH-5989) files: A Misc/NEWS.d/next/Security/2018-03-05-10-09-51.bpo-33001.elj4Aa.rst M Lib/test/test_os.py M Modules/posixmodule.c diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index 4f8a2a7e19d5..e509188243f6 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -2164,6 +2164,40 @@ def test_29248(self): target = os.readlink(r'C:\Users\All Users') self.assertTrue(os.path.samefile(target, r'C:\ProgramData')) + def test_buffer_overflow(self): + # Older versions would have a buffer overflow when detecting + # whether a link source was a directory. This test ensures we + # no longer crash, but does not otherwise validate the behavior + segment = 'X' * 27 + path = os.path.join(*[segment] * 10) + test_cases = [ + # overflow with absolute src + ('\\' + path, segment), + # overflow dest with relative src + (segment, path), + # overflow when joining src + (path[:180], path[:180]), + ] + for src, dest in test_cases: + try: + os.symlink(src, dest) + except FileNotFoundError: + pass + else: + try: + os.remove(dest) + except OSError: + pass + # Also test with bytes, since that is a separate code path. + try: + os.symlink(os.fsencode(src), os.fsencode(dest)) + except FileNotFoundError: + pass + else: + try: + os.remove(dest) + except OSError: + pass @unittest.skipUnless(sys.platform == "win32", "Win32 specific tests") class Win32JunctionTests(unittest.TestCase): diff --git a/Misc/NEWS.d/next/Security/2018-03-05-10-09-51.bpo-33001.elj4Aa.rst b/Misc/NEWS.d/next/Security/2018-03-05-10-09-51.bpo-33001.elj4Aa.rst new file mode 100644 index 000000000000..2acbac9e1af6 --- /dev/null +++ b/Misc/NEWS.d/next/Security/2018-03-05-10-09-51.bpo-33001.elj4Aa.rst @@ -0,0 +1 @@ +Minimal fix to prevent buffer overrun in os.symlink on Windows diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 6bba8ee26e16..f4c01048cdae 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -7471,7 +7471,7 @@ win_readlink(PyObject *self, PyObject *args, PyObject *kwargs) #if defined(MS_WINDOWS) /* Grab CreateSymbolicLinkW dynamically from kernel32 */ -static DWORD (CALLBACK *Py_CreateSymbolicLinkW)(LPCWSTR, LPCWSTR, DWORD) = NULL; +static BOOLEAN (CALLBACK *Py_CreateSymbolicLinkW)(LPCWSTR, LPCWSTR, DWORD) = NULL; static int check_CreateSymbolicLink(void) @@ -7486,47 +7486,51 @@ check_CreateSymbolicLink(void) return Py_CreateSymbolicLinkW != NULL; } -/* Remove the last portion of the path */ -static void +/* Remove the last portion of the path - return 0 on success */ +static int _dirnameW(WCHAR *path) { WCHAR *ptr; + size_t length = wcsnlen_s(path, MAX_PATH); + if (length == MAX_PATH) { + return -1; + } /* walk the path from the end until a backslash is encountered */ - for(ptr = path + wcslen(path); ptr != path; ptr--) { - if (*ptr == L'\\' || *ptr == L'/') + for(ptr = path + length; ptr != path; ptr--) { + if (*ptr == L'\\' || *ptr == L'/') { break; + } } *ptr = 0; + return 0; } /* Is this path absolute? */ static int _is_absW(const WCHAR *path) { - return path[0] == L'\\' || path[0] == L'/' || path[1] == L':'; - + return path[0] == L'\\' || path[0] == L'/' || + (path[0] && path[1] == L':'); } -/* join root and rest with a backslash */ -static void +/* join root and rest with a backslash - return 0 on success */ +static int _joinW(WCHAR *dest_path, const WCHAR *root, const WCHAR *rest) { - size_t root_len; - if (_is_absW(rest)) { - wcscpy(dest_path, rest); - return; + return wcscpy_s(dest_path, MAX_PATH, rest); } - root_len = wcslen(root); + if (wcscpy_s(dest_path, MAX_PATH, root)) { + return -1; + } - wcscpy(dest_path, root); - if(root_len) { - dest_path[root_len] = L'\\'; - root_len++; + if (dest_path[0] && wcscat_s(dest_path, MAX_PATH, L"\\")) { + return -1; } - wcscpy(dest_path+root_len, rest); + + return wcscat_s(dest_path, MAX_PATH, rest); } /* Return True if the path at src relative to dest is a directory */ @@ -7538,10 +7542,14 @@ _check_dirW(LPCWSTR src, LPCWSTR dest) WCHAR src_resolved[MAX_PATH] = L""; /* dest_parent = os.path.dirname(dest) */ - wcscpy(dest_parent, dest); - _dirnameW(dest_parent); + if (wcscpy_s(dest_parent, MAX_PATH, dest) || + _dirnameW(dest_parent)) { + return 0; + } /* src_resolved = os.path.join(dest_parent, src) */ - _joinW(src_resolved, dest_parent, src); + if (_joinW(src_resolved, dest_parent, src)) { + return 0; + } return ( GetFileAttributesExW(src_resolved, GetFileExInfoStandard, &src_info) && src_info.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY @@ -7597,19 +7605,15 @@ os_symlink_impl(PyObject *module, path_t *src, path_t *dst, } #endif - if ((src->narrow && dst->wide) || (src->wide && dst->narrow)) { - PyErr_SetString(PyExc_ValueError, - "symlink: src and dst must be the same type"); - return NULL; - } - #ifdef MS_WINDOWS Py_BEGIN_ALLOW_THREADS + _Py_BEGIN_SUPPRESS_IPH /* if src is a directory, ensure target_is_directory==1 */ target_is_directory |= _check_dirW(src->wide, dst->wide); result = Py_CreateSymbolicLinkW(dst->wide, src->wide, target_is_directory); + _Py_END_SUPPRESS_IPH Py_END_ALLOW_THREADS if (!result) @@ -7617,6 +7621,12 @@ os_symlink_impl(PyObject *module, path_t *src, path_t *dst, #else + if ((src->narrow && dst->wide) || (src->wide && dst->narrow)) { + PyErr_SetString(PyExc_ValueError, + "symlink: src and dst must be the same type"); + return NULL; + } + Py_BEGIN_ALLOW_THREADS #if HAVE_SYMLINKAT if (dir_fd != DEFAULT_DIR_FD) From webhook-mailer at python.org Mon Mar 5 17:26:31 2018 From: webhook-mailer at python.org (Steve Dower) Date: Mon, 05 Mar 2018 22:26:31 -0000 Subject: [Python-checkins] [3.6] bpo-33001: Prevent buffer overrun in os.symlink (GH-5989) (GH-5990) Message-ID: <mailman.126.1520288792.2831.python-checkins@python.org> https://github.com/python/cpython/commit/baa45079466eda1f5636a6d13f3a60c2c00fdcd3 commit: baa45079466eda1f5636a6d13f3a60c2c00fdcd3 branch: 3.6 author: Steve Dower <steve.dower at microsoft.com> committer: GitHub <noreply at github.com> date: 2018-03-05T14:26:28-08:00 summary: [3.6] bpo-33001: Prevent buffer overrun in os.symlink (GH-5989) (GH-5990) files: A Misc/NEWS.d/next/Security/2018-03-05-10-09-51.bpo-33001.elj4Aa.rst M Lib/test/test_os.py M Modules/posixmodule.c diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index 26d544ca9925..240b7c432ba9 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -2170,6 +2170,41 @@ def test_29248(self): target = os.readlink(r'C:\Users\All Users') self.assertTrue(os.path.samefile(target, r'C:\ProgramData')) + def test_buffer_overflow(self): + # Older versions would have a buffer overflow when detecting + # whether a link source was a directory. This test ensures we + # no longer crash, but does not otherwise validate the behavior + segment = 'X' * 27 + path = os.path.join(*[segment] * 10) + test_cases = [ + # overflow with absolute src + ('\\' + path, segment), + # overflow dest with relative src + (segment, path), + # overflow when joining src + (path[:180], path[:180]), + ] + for src, dest in test_cases: + try: + os.symlink(src, dest) + except FileNotFoundError: + pass + else: + try: + os.remove(dest) + except OSError: + pass + # Also test with bytes, since that is a separate code path. + try: + os.symlink(os.fsencode(src), os.fsencode(dest)) + except FileNotFoundError: + pass + else: + try: + os.remove(dest) + except OSError: + pass + @unittest.skipUnless(sys.platform == "win32", "Win32 specific tests") class Win32JunctionTests(unittest.TestCase): diff --git a/Misc/NEWS.d/next/Security/2018-03-05-10-09-51.bpo-33001.elj4Aa.rst b/Misc/NEWS.d/next/Security/2018-03-05-10-09-51.bpo-33001.elj4Aa.rst new file mode 100644 index 000000000000..2acbac9e1af6 --- /dev/null +++ b/Misc/NEWS.d/next/Security/2018-03-05-10-09-51.bpo-33001.elj4Aa.rst @@ -0,0 +1 @@ +Minimal fix to prevent buffer overrun in os.symlink on Windows diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 0837a4a4991e..39ba030b5191 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -7144,7 +7144,7 @@ win_readlink(PyObject *self, PyObject *args, PyObject *kwargs) #if defined(MS_WINDOWS) /* Grab CreateSymbolicLinkW dynamically from kernel32 */ -static DWORD (CALLBACK *Py_CreateSymbolicLinkW)(LPCWSTR, LPCWSTR, DWORD) = NULL; +static BOOLEAN (CALLBACK *Py_CreateSymbolicLinkW)(LPCWSTR, LPCWSTR, DWORD) = NULL; static int check_CreateSymbolicLink(void) @@ -7159,47 +7159,51 @@ check_CreateSymbolicLink(void) return Py_CreateSymbolicLinkW != NULL; } -/* Remove the last portion of the path */ -static void +/* Remove the last portion of the path - return 0 on success */ +static int _dirnameW(WCHAR *path) { WCHAR *ptr; + size_t length = wcsnlen_s(path, MAX_PATH); + if (length == MAX_PATH) { + return -1; + } /* walk the path from the end until a backslash is encountered */ - for(ptr = path + wcslen(path); ptr != path; ptr--) { - if (*ptr == L'\\' || *ptr == L'/') + for(ptr = path + length; ptr != path; ptr--) { + if (*ptr == L'\\' || *ptr == L'/') { break; + } } *ptr = 0; + return 0; } /* Is this path absolute? */ static int _is_absW(const WCHAR *path) { - return path[0] == L'\\' || path[0] == L'/' || path[1] == L':'; - + return path[0] == L'\\' || path[0] == L'/' || + (path[0] && path[1] == L':'); } -/* join root and rest with a backslash */ -static void +/* join root and rest with a backslash - return 0 on success */ +static int _joinW(WCHAR *dest_path, const WCHAR *root, const WCHAR *rest) { - size_t root_len; - if (_is_absW(rest)) { - wcscpy(dest_path, rest); - return; + return wcscpy_s(dest_path, MAX_PATH, rest); } - root_len = wcslen(root); + if (wcscpy_s(dest_path, MAX_PATH, root)) { + return -1; + } - wcscpy(dest_path, root); - if(root_len) { - dest_path[root_len] = L'\\'; - root_len++; + if (dest_path[0] && wcscat_s(dest_path, MAX_PATH, L"\\")) { + return -1; } - wcscpy(dest_path+root_len, rest); + + return wcscat_s(dest_path, MAX_PATH, rest); } /* Return True if the path at src relative to dest is a directory */ @@ -7211,10 +7215,14 @@ _check_dirW(LPCWSTR src, LPCWSTR dest) WCHAR src_resolved[MAX_PATH] = L""; /* dest_parent = os.path.dirname(dest) */ - wcscpy(dest_parent, dest); - _dirnameW(dest_parent); + if (wcscpy_s(dest_parent, MAX_PATH, dest) || + _dirnameW(dest_parent)) { + return 0; + } /* src_resolved = os.path.join(dest_parent, src) */ - _joinW(src_resolved, dest_parent, src); + if (_joinW(src_resolved, dest_parent, src)) { + return 0; + } return ( GetFileAttributesExW(src_resolved, GetFileExInfoStandard, &src_info) && src_info.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY @@ -7270,19 +7278,15 @@ os_symlink_impl(PyObject *module, path_t *src, path_t *dst, } #endif - if ((src->narrow && dst->wide) || (src->wide && dst->narrow)) { - PyErr_SetString(PyExc_ValueError, - "symlink: src and dst must be the same type"); - return NULL; - } - #ifdef MS_WINDOWS Py_BEGIN_ALLOW_THREADS + _Py_BEGIN_SUPPRESS_IPH /* if src is a directory, ensure target_is_directory==1 */ target_is_directory |= _check_dirW(src->wide, dst->wide); result = Py_CreateSymbolicLinkW(dst->wide, src->wide, target_is_directory); + _Py_END_SUPPRESS_IPH Py_END_ALLOW_THREADS if (!result) @@ -7290,6 +7294,12 @@ os_symlink_impl(PyObject *module, path_t *src, path_t *dst, #else + if ((src->narrow && dst->wide) || (src->wide && dst->narrow)) { + PyErr_SetString(PyExc_ValueError, + "symlink: src and dst must be the same type"); + return NULL; + } + Py_BEGIN_ALLOW_THREADS #if HAVE_SYMLINKAT if (dir_fd != DEFAULT_DIR_FD) From webhook-mailer at python.org Mon Mar 5 18:13:03 2018 From: webhook-mailer at python.org (Miss Islington (bot)) Date: Mon, 05 Mar 2018 23:13:03 -0000 Subject: [Python-checkins] bpo-33001: Prevent buffer overrun in os.symlink (GH-5989) Message-ID: <mailman.127.1520291584.2831.python-checkins@python.org> https://github.com/python/cpython/commit/96fdbacb7797a564249fd59ccf86ec153c4bb095 commit: 96fdbacb7797a564249fd59ccf86ec153c4bb095 branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: GitHub <noreply at github.com> date: 2018-03-05T15:12:56-08:00 summary: bpo-33001: Prevent buffer overrun in os.symlink (GH-5989) (cherry picked from commit 6921e73e33edc3c61bc2d78ed558eaa22a89a564) Co-authored-by: Steve Dower <steve.dower at microsoft.com> files: A Misc/NEWS.d/next/Security/2018-03-05-10-09-51.bpo-33001.elj4Aa.rst M Lib/test/test_os.py M Modules/posixmodule.c diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index 4f8a2a7e19d5..e509188243f6 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -2164,6 +2164,40 @@ def test_29248(self): target = os.readlink(r'C:\Users\All Users') self.assertTrue(os.path.samefile(target, r'C:\ProgramData')) + def test_buffer_overflow(self): + # Older versions would have a buffer overflow when detecting + # whether a link source was a directory. This test ensures we + # no longer crash, but does not otherwise validate the behavior + segment = 'X' * 27 + path = os.path.join(*[segment] * 10) + test_cases = [ + # overflow with absolute src + ('\\' + path, segment), + # overflow dest with relative src + (segment, path), + # overflow when joining src + (path[:180], path[:180]), + ] + for src, dest in test_cases: + try: + os.symlink(src, dest) + except FileNotFoundError: + pass + else: + try: + os.remove(dest) + except OSError: + pass + # Also test with bytes, since that is a separate code path. + try: + os.symlink(os.fsencode(src), os.fsencode(dest)) + except FileNotFoundError: + pass + else: + try: + os.remove(dest) + except OSError: + pass @unittest.skipUnless(sys.platform == "win32", "Win32 specific tests") class Win32JunctionTests(unittest.TestCase): diff --git a/Misc/NEWS.d/next/Security/2018-03-05-10-09-51.bpo-33001.elj4Aa.rst b/Misc/NEWS.d/next/Security/2018-03-05-10-09-51.bpo-33001.elj4Aa.rst new file mode 100644 index 000000000000..2acbac9e1af6 --- /dev/null +++ b/Misc/NEWS.d/next/Security/2018-03-05-10-09-51.bpo-33001.elj4Aa.rst @@ -0,0 +1 @@ +Minimal fix to prevent buffer overrun in os.symlink on Windows diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index f6144a1d0527..6bf8c6ba7b7a 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -7471,7 +7471,7 @@ win_readlink(PyObject *self, PyObject *args, PyObject *kwargs) #if defined(MS_WINDOWS) /* Grab CreateSymbolicLinkW dynamically from kernel32 */ -static DWORD (CALLBACK *Py_CreateSymbolicLinkW)(LPCWSTR, LPCWSTR, DWORD) = NULL; +static BOOLEAN (CALLBACK *Py_CreateSymbolicLinkW)(LPCWSTR, LPCWSTR, DWORD) = NULL; static int check_CreateSymbolicLink(void) @@ -7486,47 +7486,51 @@ check_CreateSymbolicLink(void) return Py_CreateSymbolicLinkW != NULL; } -/* Remove the last portion of the path */ -static void +/* Remove the last portion of the path - return 0 on success */ +static int _dirnameW(WCHAR *path) { WCHAR *ptr; + size_t length = wcsnlen_s(path, MAX_PATH); + if (length == MAX_PATH) { + return -1; + } /* walk the path from the end until a backslash is encountered */ - for(ptr = path + wcslen(path); ptr != path; ptr--) { - if (*ptr == L'\\' || *ptr == L'/') + for(ptr = path + length; ptr != path; ptr--) { + if (*ptr == L'\\' || *ptr == L'/') { break; + } } *ptr = 0; + return 0; } /* Is this path absolute? */ static int _is_absW(const WCHAR *path) { - return path[0] == L'\\' || path[0] == L'/' || path[1] == L':'; - + return path[0] == L'\\' || path[0] == L'/' || + (path[0] && path[1] == L':'); } -/* join root and rest with a backslash */ -static void +/* join root and rest with a backslash - return 0 on success */ +static int _joinW(WCHAR *dest_path, const WCHAR *root, const WCHAR *rest) { - size_t root_len; - if (_is_absW(rest)) { - wcscpy(dest_path, rest); - return; + return wcscpy_s(dest_path, MAX_PATH, rest); } - root_len = wcslen(root); + if (wcscpy_s(dest_path, MAX_PATH, root)) { + return -1; + } - wcscpy(dest_path, root); - if(root_len) { - dest_path[root_len] = L'\\'; - root_len++; + if (dest_path[0] && wcscat_s(dest_path, MAX_PATH, L"\\")) { + return -1; } - wcscpy(dest_path+root_len, rest); + + return wcscat_s(dest_path, MAX_PATH, rest); } /* Return True if the path at src relative to dest is a directory */ @@ -7538,10 +7542,14 @@ _check_dirW(LPCWSTR src, LPCWSTR dest) WCHAR src_resolved[MAX_PATH] = L""; /* dest_parent = os.path.dirname(dest) */ - wcscpy(dest_parent, dest); - _dirnameW(dest_parent); + if (wcscpy_s(dest_parent, MAX_PATH, dest) || + _dirnameW(dest_parent)) { + return 0; + } /* src_resolved = os.path.join(dest_parent, src) */ - _joinW(src_resolved, dest_parent, src); + if (_joinW(src_resolved, dest_parent, src)) { + return 0; + } return ( GetFileAttributesExW(src_resolved, GetFileExInfoStandard, &src_info) && src_info.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY @@ -7597,19 +7605,15 @@ os_symlink_impl(PyObject *module, path_t *src, path_t *dst, } #endif - if ((src->narrow && dst->wide) || (src->wide && dst->narrow)) { - PyErr_SetString(PyExc_ValueError, - "symlink: src and dst must be the same type"); - return NULL; - } - #ifdef MS_WINDOWS Py_BEGIN_ALLOW_THREADS + _Py_BEGIN_SUPPRESS_IPH /* if src is a directory, ensure target_is_directory==1 */ target_is_directory |= _check_dirW(src->wide, dst->wide); result = Py_CreateSymbolicLinkW(dst->wide, src->wide, target_is_directory); + _Py_END_SUPPRESS_IPH Py_END_ALLOW_THREADS if (!result) @@ -7617,6 +7621,12 @@ os_symlink_impl(PyObject *module, path_t *src, path_t *dst, #else + if ((src->narrow && dst->wide) || (src->wide && dst->narrow)) { + PyErr_SetString(PyExc_ValueError, + "symlink: src and dst must be the same type"); + return NULL; + } + Py_BEGIN_ALLOW_THREADS #if HAVE_SYMLINKAT if (dir_fd != DEFAULT_DIR_FD) From webhook-mailer at python.org Mon Mar 5 18:29:11 2018 From: webhook-mailer at python.org (Jason R. Coombs) Date: Mon, 05 Mar 2018 23:29:11 -0000 Subject: [Python-checkins] bpo-32991: Restore expectation that inspect.getfile raises TypeError on namespace package (GH-5980) Message-ID: <mailman.128.1520292553.2831.python-checkins@python.org> https://github.com/python/cpython/commit/b9650a04a81355c8a7dcd0464c28febfb4bfc0a9 commit: b9650a04a81355c8a7dcd0464c28febfb4bfc0a9 branch: master author: Jason R. Coombs <jaraco at jaraco.com> committer: GitHub <noreply at github.com> date: 2018-03-05T18:29:08-05:00 summary: bpo-32991: Restore expectation that inspect.getfile raises TypeError on namespace package (GH-5980) * bpo-32991: Add test capturing expectation. DocTestFinder.find should return an empty list for doctests in a namespace package. * bpo-32991: Restore expectation that inspect.getfile on a namespace package raises TypeError. files: M Lib/inspect.py M Lib/test/test_doctest.py diff --git a/Lib/inspect.py b/Lib/inspect.py index 109efc06b268..57c04877c743 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -642,13 +642,13 @@ def cleandoc(doc): def getfile(object): """Work out which source or compiled file an object was defined in.""" if ismodule(object): - if hasattr(object, '__file__'): + if getattr(object, '__file__', None): return object.__file__ raise TypeError('{!r} is a built-in module'.format(object)) if isclass(object): if hasattr(object, '__module__'): object = sys.modules.get(object.__module__) - if hasattr(object, '__file__'): + if getattr(object, '__file__', None): return object.__file__ raise TypeError('{!r} is a built-in class'.format(object)) if ismethod(object): diff --git a/Lib/test/test_doctest.py b/Lib/test/test_doctest.py index 5ad94aba6492..f0eb52881bcf 100644 --- a/Lib/test/test_doctest.py +++ b/Lib/test/test_doctest.py @@ -7,6 +7,8 @@ import functools import os import sys +import importlib +import unittest # NOTE: There are some additional tests relating to interaction with @@ -435,7 +437,7 @@ def basics(): r""" >>> tests = finder.find(sample_func) >>> print(tests) # doctest: +ELLIPSIS - [<DocTest sample_func from ...:19 (1 example)>] + [<DocTest sample_func from ...:21 (1 example)>] The exact name depends on how test_doctest was invoked, so allow for leading path components. @@ -681,6 +683,17 @@ def non_Python_modules(): r""" and 'int' is a type. """ + +class TestDocTestFinder(unittest.TestCase): + + def test_empty_namespace_package(self): + pkg_name = 'doctest_empty_pkg' + os.mkdir(pkg_name) + mod = importlib.import_module(pkg_name) + assert doctest.DocTestFinder().find(mod) == [] + os.rmdir(pkg_name) + + def test_DocTestParser(): r""" Unit tests for the `DocTestParser` class. @@ -2945,6 +2958,10 @@ def test_main(): from test import test_doctest support.run_doctest(test_doctest, verbosity=True) + # Run unittests + support.run_unittest(__name__) + + def test_coverage(coverdir): trace = support.import_module('trace') tracer = trace.Trace(ignoredirs=[sys.base_prefix, sys.base_exec_prefix,], From webhook-mailer at python.org Tue Mar 6 00:59:05 2018 From: webhook-mailer at python.org (Benjamin Peterson) Date: Tue, 06 Mar 2018 05:59:05 -0000 Subject: [Python-checkins] [2.7] closes bpo-32997: Fix REDOS in fpformat (GH-5984) Message-ID: <mailman.129.1520315946.2831.python-checkins@python.org> https://github.com/python/cpython/commit/55d5bfba9482d39080f7b9ec3e6257ecd23f264f commit: 55d5bfba9482d39080f7b9ec3e6257ecd23f264f branch: 2.7 author: Jamie Davis <davisjam at vt.edu> committer: Benjamin Peterson <benjamin at python.org> date: 2018-03-05T21:59:02-08:00 summary: [2.7] closes bpo-32997: Fix REDOS in fpformat (GH-5984) The regex to decode a number in fpformat is susceptible to catastrophic backtracking. This is a potential DOS vector if a server is using fpformat on untrusted number strings. Replace it with an equivalent non-vulnerable regex. The match behavior of the new regex is slightly different. It captures the whole integer part of the number in one group, Leading zeros are stripped off later. files: A Misc/NEWS.d/next/Security/2018-03-05-10-14-42.bpo-32997.hp2s8n.rst M Lib/fpformat.py M Lib/test/test_fpformat.py diff --git a/Lib/fpformat.py b/Lib/fpformat.py index 71cbb25f3c8b..0537a27b8820 100644 --- a/Lib/fpformat.py +++ b/Lib/fpformat.py @@ -19,7 +19,7 @@ __all__ = ["fix","sci","NotANumber"] # Compiled regular expression to "decode" a number -decoder = re.compile(r'^([-+]?)0*(\d*)((?:\.\d*)?)(([eE][-+]?\d+)?)$') +decoder = re.compile(r'^([-+]?)(\d*)((?:\.\d*)?)(([eE][-+]?\d+)?)$') # \0 the whole thing # \1 leading sign or empty # \2 digits left of decimal point @@ -41,6 +41,7 @@ def extract(s): res = decoder.match(s) if res is None: raise NotANumber, s sign, intpart, fraction, exppart = res.group(1,2,3,4) + intpart = intpart.lstrip('0'); if sign == '+': sign = '' if fraction: fraction = fraction[1:] if exppart: expo = int(exppart[1:]) diff --git a/Lib/test/test_fpformat.py b/Lib/test/test_fpformat.py index e6de3b0c11be..428623ebb35f 100644 --- a/Lib/test/test_fpformat.py +++ b/Lib/test/test_fpformat.py @@ -67,6 +67,16 @@ def test_failing_values(self): else: self.fail("No exception on non-numeric sci") + def test_REDOS(self): + # This attack string will hang on the old decoder pattern. + attack = '+0' + ('0' * 1000000) + '++' + digs = 5 # irrelevant + + # fix returns input if it does not decode + self.assertEqual(fpformat.fix(attack, digs), attack) + # sci raises NotANumber + with self.assertRaises(NotANumber): + fpformat.sci(attack, digs) def test_main(): run_unittest(FpformatTest) diff --git a/Misc/NEWS.d/next/Security/2018-03-05-10-14-42.bpo-32997.hp2s8n.rst b/Misc/NEWS.d/next/Security/2018-03-05-10-14-42.bpo-32997.hp2s8n.rst new file mode 100644 index 000000000000..3c78ba61ae34 --- /dev/null +++ b/Misc/NEWS.d/next/Security/2018-03-05-10-14-42.bpo-32997.hp2s8n.rst @@ -0,0 +1,4 @@ +A regex in fpformat was vulnerable to catastrophic backtracking. This regex +was a potential DOS vector (REDOS). Based on typical uses of fpformat the +risk seems low. The regex has been refactored and is now safe. Patch by +Jamie Davis. From solipsis at pitrou.net Tue Mar 6 04:09:25 2018 From: solipsis at pitrou.net (solipsis at pitrou.net) Date: Tue, 06 Mar 2018 09:09:25 +0000 Subject: [Python-checkins] Daily reference leaks (4243df51fe43): sum=4 Message-ID: <20180306090925.1.EEA84D906B3F2938@psf.io> results for 4243df51fe43 on branch "default" -------------------------------------------- test_functools leaked [0, 3, 1] memory blocks, sum=4 test_multiprocessing_fork leaked [2, -1, -1] memory blocks, sum=0 Command line was: ['./python', '-m', 'test.regrtest', '-uall', '-R', '3:3:/home/psf-users/antoine/refleaks/reflogQLD00B', '--timeout', '7200'] From webhook-mailer at python.org Tue Mar 6 05:34:39 2018 From: webhook-mailer at python.org (Xiang Zhang) Date: Tue, 06 Mar 2018 10:34:39 -0000 Subject: [Python-checkins] Fix strncpy warning with gcc 8 (#5840) Message-ID: <mailman.130.1520332480.2831.python-checkins@python.org> https://github.com/python/cpython/commit/efd2bac1564f8141a4eab1bf8779b412974b8d69 commit: efd2bac1564f8141a4eab1bf8779b412974b8d69 branch: master author: Siddhesh Poyarekar <siddhesh.poyarekar at gmail.com> committer: Xiang Zhang <angwerzx at 126.com> date: 2018-03-06T18:34:35+08:00 summary: Fix strncpy warning with gcc 8 (#5840) The length in strncpy is one char too short and as a result it leads to a build warning with gcc 8. Comment out the strncpy since the interpreter aborts immediately after anyway. files: M Python/pystrtod.c diff --git a/Python/pystrtod.c b/Python/pystrtod.c index 9bf936386210..601f7c691edf 100644 --- a/Python/pystrtod.c +++ b/Python/pystrtod.c @@ -1060,8 +1060,8 @@ format_float_short(double d, char format_code, else { /* shouldn't get here: Gay's code should always return something starting with a digit, an 'I', or 'N' */ - strncpy(p, "ERR", 3); - /* p += 3; */ + /* strncpy(p, "ERR", 3); + p += 3; */ Py_UNREACHABLE(); } goto exit; From webhook-mailer at python.org Tue Mar 6 08:31:41 2018 From: webhook-mailer at python.org (Victor Stinner) Date: Tue, 06 Mar 2018 13:31:41 -0000 Subject: [Python-checkins] bpo-33005: Fix _PyGILState_Reinit() (#6001) Message-ID: <mailman.131.1520343102.2831.python-checkins@python.org> https://github.com/python/cpython/commit/5d92647102fac9e116b98ab8bbc632eeed501c34 commit: 5d92647102fac9e116b98ab8bbc632eeed501c34 branch: master author: Victor Stinner <victor.stinner at gmail.com> committer: GitHub <noreply at github.com> date: 2018-03-06T14:31:37+01:00 summary: bpo-33005: Fix _PyGILState_Reinit() (#6001) Fix a crash on fork when using a custom memory allocator (ex: using PYTHONMALLOC env var). _PyGILState_Reinit() and _PyInterpreterState_Enable() now use the default RAW memory allocator to allocate a new interpreters mutex on fork. files: A Misc/NEWS.d/next/Core and Builtins/2018-03-06-12-19-19.bpo-33005.LP-V2U.rst M Python/pystate.c diff --git a/Misc/NEWS.d/next/Core and Builtins/2018-03-06-12-19-19.bpo-33005.LP-V2U.rst b/Misc/NEWS.d/next/Core and Builtins/2018-03-06-12-19-19.bpo-33005.LP-V2U.rst new file mode 100644 index 000000000000..6c8b99cbb897 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2018-03-06-12-19-19.bpo-33005.LP-V2U.rst @@ -0,0 +1,4 @@ +Fix a crash on fork when using a custom memory allocator (ex: using +PYTHONMALLOC env var). _PyGILState_Reinit() and _PyInterpreterState_Enable() +now use the default RAW memory allocator to allocate a new interpreters mutex +on fork. diff --git a/Python/pystate.c b/Python/pystate.c index a87801f56922..140d2fba8efd 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -103,15 +103,24 @@ _PyInitError _PyInterpreterState_Enable(_PyRuntimeState *runtime) { runtime->interpreters.next_id = 0; - /* Since we only call _PyRuntimeState_Init() once per process - (see _PyRuntime_Initialize()), we make sure the mutex is - initialized here. */ + + /* Py_Finalize() calls _PyRuntimeState_Fini() which clears the mutex. + Create a new mutex if needed. */ if (runtime->interpreters.mutex == NULL) { + /* Force default allocator, since _PyRuntimeState_Fini() must + use the same allocator than this function. */ + PyMemAllocatorEx old_alloc; + _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); + runtime->interpreters.mutex = PyThread_allocate_lock(); + + PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); + if (runtime->interpreters.mutex == NULL) { return _Py_INIT_ERR("Can't initialize threads for interpreter"); } } + return _Py_INIT_OK(); } @@ -933,9 +942,19 @@ _PyGILState_Fini(void) void _PyGILState_Reinit(void) { + /* Force default allocator, since _PyRuntimeState_Fini() must + use the same allocator than this function. */ + PyMemAllocatorEx old_alloc; + _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); + _PyRuntime.interpreters.mutex = PyThread_allocate_lock(); - if (_PyRuntime.interpreters.mutex == NULL) + + PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); + + if (_PyRuntime.interpreters.mutex == NULL) { Py_FatalError("Can't initialize threads for interpreter"); + } + PyThreadState *tstate = PyGILState_GetThisThreadState(); PyThread_tss_delete(&_PyRuntime.gilstate.autoTSSkey); if (PyThread_tss_create(&_PyRuntime.gilstate.autoTSSkey) != 0) { From webhook-mailer at python.org Tue Mar 6 08:52:29 2018 From: webhook-mailer at python.org (Miss Islington (bot)) Date: Tue, 06 Mar 2018 13:52:29 -0000 Subject: [Python-checkins] bpo-33005: Fix _PyGILState_Reinit() (GH-6001) Message-ID: <mailman.132.1520344351.2831.python-checkins@python.org> https://github.com/python/cpython/commit/31e2b76f7bbcb8278748565252767a8b7790ff27 commit: 31e2b76f7bbcb8278748565252767a8b7790ff27 branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: GitHub <noreply at github.com> date: 2018-03-06T05:52:27-08:00 summary: bpo-33005: Fix _PyGILState_Reinit() (GH-6001) Fix a crash on fork when using a custom memory allocator (ex: using PYTHONMALLOC env var). _PyGILState_Reinit() and _PyInterpreterState_Enable() now use the default RAW memory allocator to allocate a new interpreters mutex on fork. (cherry picked from commit 5d92647102fac9e116b98ab8bbc632eeed501c34) Co-authored-by: Victor Stinner <victor.stinner at gmail.com> files: A Misc/NEWS.d/next/Core and Builtins/2018-03-06-12-19-19.bpo-33005.LP-V2U.rst M Python/pystate.c diff --git a/Misc/NEWS.d/next/Core and Builtins/2018-03-06-12-19-19.bpo-33005.LP-V2U.rst b/Misc/NEWS.d/next/Core and Builtins/2018-03-06-12-19-19.bpo-33005.LP-V2U.rst new file mode 100644 index 000000000000..6c8b99cbb897 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2018-03-06-12-19-19.bpo-33005.LP-V2U.rst @@ -0,0 +1,4 @@ +Fix a crash on fork when using a custom memory allocator (ex: using +PYTHONMALLOC env var). _PyGILState_Reinit() and _PyInterpreterState_Enable() +now use the default RAW memory allocator to allocate a new interpreters mutex +on fork. diff --git a/Python/pystate.c b/Python/pystate.c index a87801f56922..140d2fba8efd 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -103,15 +103,24 @@ _PyInitError _PyInterpreterState_Enable(_PyRuntimeState *runtime) { runtime->interpreters.next_id = 0; - /* Since we only call _PyRuntimeState_Init() once per process - (see _PyRuntime_Initialize()), we make sure the mutex is - initialized here. */ + + /* Py_Finalize() calls _PyRuntimeState_Fini() which clears the mutex. + Create a new mutex if needed. */ if (runtime->interpreters.mutex == NULL) { + /* Force default allocator, since _PyRuntimeState_Fini() must + use the same allocator than this function. */ + PyMemAllocatorEx old_alloc; + _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); + runtime->interpreters.mutex = PyThread_allocate_lock(); + + PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); + if (runtime->interpreters.mutex == NULL) { return _Py_INIT_ERR("Can't initialize threads for interpreter"); } } + return _Py_INIT_OK(); } @@ -933,9 +942,19 @@ _PyGILState_Fini(void) void _PyGILState_Reinit(void) { + /* Force default allocator, since _PyRuntimeState_Fini() must + use the same allocator than this function. */ + PyMemAllocatorEx old_alloc; + _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); + _PyRuntime.interpreters.mutex = PyThread_allocate_lock(); - if (_PyRuntime.interpreters.mutex == NULL) + + PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); + + if (_PyRuntime.interpreters.mutex == NULL) { Py_FatalError("Can't initialize threads for interpreter"); + } + PyThreadState *tstate = PyGILState_GetThisThreadState(); PyThread_tss_delete(&_PyRuntime.gilstate.autoTSSkey); if (PyThread_tss_create(&_PyRuntime.gilstate.autoTSSkey) != 0) { From webhook-mailer at python.org Tue Mar 6 10:16:14 2018 From: webhook-mailer at python.org (Jason R. Coombs) Date: Tue, 06 Mar 2018 15:16:14 -0000 Subject: [Python-checkins] bpo-32991: Restore expectation that inspect.getfile raises TypeError on namespace package (GH-5980) (GH-5997) Message-ID: <mailman.133.1520349375.2831.python-checkins@python.org> https://github.com/python/cpython/commit/5a0c3987abd6a71b4fadeb525477eb5f560e8514 commit: 5a0c3987abd6a71b4fadeb525477eb5f560e8514 branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Jason R. Coombs <jaraco at jaraco.com> date: 2018-03-06T10:16:11-05:00 summary: bpo-32991: Restore expectation that inspect.getfile raises TypeError on namespace package (GH-5980) (GH-5997) * bpo-32991: Add test capturing expectation. DocTestFinder.find should return an empty list for doctests in a namespace package. * bpo-32991: Restore expectation that inspect.getfile on a namespace package raises TypeError. (cherry picked from commit b9650a04a81355c8a7dcd0464c28febfb4bfc0a9) Co-authored-by: Jason R. Coombs <jaraco at jaraco.com> files: M Lib/inspect.py M Lib/test/test_doctest.py diff --git a/Lib/inspect.py b/Lib/inspect.py index bc97efe179ca..d62968965462 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -642,13 +642,13 @@ def cleandoc(doc): def getfile(object): """Work out which source or compiled file an object was defined in.""" if ismodule(object): - if hasattr(object, '__file__'): + if getattr(object, '__file__', None): return object.__file__ raise TypeError('{!r} is a built-in module'.format(object)) if isclass(object): if hasattr(object, '__module__'): object = sys.modules.get(object.__module__) - if hasattr(object, '__file__'): + if getattr(object, '__file__', None): return object.__file__ raise TypeError('{!r} is a built-in class'.format(object)) if ismethod(object): diff --git a/Lib/test/test_doctest.py b/Lib/test/test_doctest.py index 5ad94aba6492..f0eb52881bcf 100644 --- a/Lib/test/test_doctest.py +++ b/Lib/test/test_doctest.py @@ -7,6 +7,8 @@ import functools import os import sys +import importlib +import unittest # NOTE: There are some additional tests relating to interaction with @@ -435,7 +437,7 @@ def basics(): r""" >>> tests = finder.find(sample_func) >>> print(tests) # doctest: +ELLIPSIS - [<DocTest sample_func from ...:19 (1 example)>] + [<DocTest sample_func from ...:21 (1 example)>] The exact name depends on how test_doctest was invoked, so allow for leading path components. @@ -681,6 +683,17 @@ def non_Python_modules(): r""" and 'int' is a type. """ + +class TestDocTestFinder(unittest.TestCase): + + def test_empty_namespace_package(self): + pkg_name = 'doctest_empty_pkg' + os.mkdir(pkg_name) + mod = importlib.import_module(pkg_name) + assert doctest.DocTestFinder().find(mod) == [] + os.rmdir(pkg_name) + + def test_DocTestParser(): r""" Unit tests for the `DocTestParser` class. @@ -2945,6 +2958,10 @@ def test_main(): from test import test_doctest support.run_doctest(test_doctest, verbosity=True) + # Run unittests + support.run_unittest(__name__) + + def test_coverage(coverdir): trace = support.import_module('trace') tracer = trace.Trace(ignoredirs=[sys.base_prefix, sys.base_exec_prefix,], From webhook-mailer at python.org Tue Mar 6 12:59:49 2018 From: webhook-mailer at python.org (Yury Selivanov) Date: Tue, 06 Mar 2018 17:59:49 -0000 Subject: [Python-checkins] bpo-33009: Fix inspect.signature() for single-parameter partialmethods. (GH-6004) Message-ID: <mailman.134.1520359190.2831.python-checkins@python.org> https://github.com/python/cpython/commit/8a387219bdfb6ee34928d6168ac42ca559f11c9a commit: 8a387219bdfb6ee34928d6168ac42ca559f11c9a branch: master author: Yury Selivanov <yury at magic.io> committer: GitHub <noreply at github.com> date: 2018-03-06T12:59:45-05:00 summary: bpo-33009: Fix inspect.signature() for single-parameter partialmethods. (GH-6004) files: A Misc/NEWS.d/next/Library/2018-03-06-11-54-59.bpo-33009.-Ekysb.rst M Lib/inspect.py M Lib/test/test_inspect.py diff --git a/Lib/inspect.py b/Lib/inspect.py index 57c04877c743..512785f9237e 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -2254,7 +2254,8 @@ def _signature_from_callable(obj, *, return sig else: sig_params = tuple(sig.parameters.values()) - assert first_wrapped_param is not sig_params[0] + assert (not sig_params or + first_wrapped_param is not sig_params[0]) new_params = (first_wrapped_param,) + sig_params return sig.replace(parameters=new_params) diff --git a/Lib/test/test_inspect.py b/Lib/test/test_inspect.py index 1a856f6387e8..3481a57ec833 100644 --- a/Lib/test/test_inspect.py +++ b/Lib/test/test_inspect.py @@ -2580,6 +2580,16 @@ def test(it, a, *, c) -> 'spam': ('c', 1, ..., 'keyword_only')), 'spam')) + class Spam: + def test(self: 'anno', x): + pass + + g = partialmethod(test, 1) + + self.assertEqual(self.signature(Spam.g), + ((('self', ..., 'anno', 'positional_or_keyword'),), + ...)) + def test_signature_on_fake_partialmethod(self): def foo(a): pass foo._partialmethod = 'spam' diff --git a/Misc/NEWS.d/next/Library/2018-03-06-11-54-59.bpo-33009.-Ekysb.rst b/Misc/NEWS.d/next/Library/2018-03-06-11-54-59.bpo-33009.-Ekysb.rst new file mode 100644 index 000000000000..96bc70a8c944 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-03-06-11-54-59.bpo-33009.-Ekysb.rst @@ -0,0 +1 @@ +Fix inspect.signature() for single-parameter partialmethods. From webhook-mailer at python.org Tue Mar 6 13:23:52 2018 From: webhook-mailer at python.org (Miss Islington (bot)) Date: Tue, 06 Mar 2018 18:23:52 -0000 Subject: [Python-checkins] bpo-33009: Fix inspect.signature() for single-parameter partialmethods. (GH-6004) Message-ID: <mailman.135.1520360633.2831.python-checkins@python.org> https://github.com/python/cpython/commit/112f799666bac1bdbb320840d5fda3132255eb5e commit: 112f799666bac1bdbb320840d5fda3132255eb5e branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: GitHub <noreply at github.com> date: 2018-03-06T10:23:48-08:00 summary: bpo-33009: Fix inspect.signature() for single-parameter partialmethods. (GH-6004) (cherry picked from commit 8a387219bdfb6ee34928d6168ac42ca559f11c9a) Co-authored-by: Yury Selivanov <yury at magic.io> files: A Misc/NEWS.d/next/Library/2018-03-06-11-54-59.bpo-33009.-Ekysb.rst M Lib/inspect.py M Lib/test/test_inspect.py diff --git a/Lib/inspect.py b/Lib/inspect.py index d62968965462..f2719923e1ae 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -2254,7 +2254,8 @@ def _signature_from_callable(obj, *, return sig else: sig_params = tuple(sig.parameters.values()) - assert first_wrapped_param is not sig_params[0] + assert (not sig_params or + first_wrapped_param is not sig_params[0]) new_params = (first_wrapped_param,) + sig_params return sig.replace(parameters=new_params) diff --git a/Lib/test/test_inspect.py b/Lib/test/test_inspect.py index 1a856f6387e8..3481a57ec833 100644 --- a/Lib/test/test_inspect.py +++ b/Lib/test/test_inspect.py @@ -2580,6 +2580,16 @@ def test(it, a, *, c) -> 'spam': ('c', 1, ..., 'keyword_only')), 'spam')) + class Spam: + def test(self: 'anno', x): + pass + + g = partialmethod(test, 1) + + self.assertEqual(self.signature(Spam.g), + ((('self', ..., 'anno', 'positional_or_keyword'),), + ...)) + def test_signature_on_fake_partialmethod(self): def foo(a): pass foo._partialmethod = 'spam' diff --git a/Misc/NEWS.d/next/Library/2018-03-06-11-54-59.bpo-33009.-Ekysb.rst b/Misc/NEWS.d/next/Library/2018-03-06-11-54-59.bpo-33009.-Ekysb.rst new file mode 100644 index 000000000000..96bc70a8c944 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-03-06-11-54-59.bpo-33009.-Ekysb.rst @@ -0,0 +1 @@ +Fix inspect.signature() for single-parameter partialmethods. From webhook-mailer at python.org Tue Mar 6 13:48:11 2018 From: webhook-mailer at python.org (Miss Islington (bot)) Date: Tue, 06 Mar 2018 18:48:11 -0000 Subject: [Python-checkins] bpo-33009: Fix inspect.signature() for single-parameter partialmethods. (GH-6004) Message-ID: <mailman.136.1520362092.2831.python-checkins@python.org> https://github.com/python/cpython/commit/387a055261267f5fafd2c12eafef49759c94704f commit: 387a055261267f5fafd2c12eafef49759c94704f branch: 3.6 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: GitHub <noreply at github.com> date: 2018-03-06T10:48:04-08:00 summary: bpo-33009: Fix inspect.signature() for single-parameter partialmethods. (GH-6004) (cherry picked from commit 8a387219bdfb6ee34928d6168ac42ca559f11c9a) Co-authored-by: Yury Selivanov <yury at magic.io> files: A Misc/NEWS.d/next/Library/2018-03-06-11-54-59.bpo-33009.-Ekysb.rst M Lib/inspect.py M Lib/test/test_inspect.py diff --git a/Lib/inspect.py b/Lib/inspect.py index e9c2dbd5c88d..dc2aab2826af 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -2251,7 +2251,8 @@ def _signature_from_callable(obj, *, return sig else: sig_params = tuple(sig.parameters.values()) - assert first_wrapped_param is not sig_params[0] + assert (not sig_params or + first_wrapped_param is not sig_params[0]) new_params = (first_wrapped_param,) + sig_params return sig.replace(parameters=new_params) diff --git a/Lib/test/test_inspect.py b/Lib/test/test_inspect.py index 55436748d6f5..126b26e49227 100644 --- a/Lib/test/test_inspect.py +++ b/Lib/test/test_inspect.py @@ -2526,6 +2526,16 @@ def test(it, a, *, c) -> 'spam': ('c', 1, ..., 'keyword_only')), 'spam')) + class Spam: + def test(self: 'anno', x): + pass + + g = partialmethod(test, 1) + + self.assertEqual(self.signature(Spam.g), + ((('self', ..., 'anno', 'positional_or_keyword'),), + ...)) + def test_signature_on_fake_partialmethod(self): def foo(a): pass foo._partialmethod = 'spam' diff --git a/Misc/NEWS.d/next/Library/2018-03-06-11-54-59.bpo-33009.-Ekysb.rst b/Misc/NEWS.d/next/Library/2018-03-06-11-54-59.bpo-33009.-Ekysb.rst new file mode 100644 index 000000000000..96bc70a8c944 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-03-06-11-54-59.bpo-33009.-Ekysb.rst @@ -0,0 +1 @@ +Fix inspect.signature() for single-parameter partialmethods. From webhook-mailer at python.org Wed Mar 7 00:05:40 2018 From: webhook-mailer at python.org (Xiang Zhang) Date: Wed, 07 Mar 2018 05:05:40 -0000 Subject: [Python-checkins] bpo-32969: Expose some missing constants in zlib and fix the doc (GH-5988) Message-ID: <mailman.137.1520399141.2831.python-checkins@python.org> https://github.com/python/cpython/commit/bc3f2289b9007396bfb7f986bee477b6176c1822 commit: bc3f2289b9007396bfb7f986bee477b6176c1822 branch: master author: Xiang Zhang <angwerzx at 126.com> committer: GitHub <noreply at github.com> date: 2018-03-07T13:05:37+08:00 summary: bpo-32969: Expose some missing constants in zlib and fix the doc (GH-5988) files: A Misc/NEWS.d/next/Library/2018-03-06-00-19-41.bpo-32969.rGTKa0.rst M Doc/library/zlib.rst M Lib/test/test_zlib.py M Modules/zlibmodule.c diff --git a/Doc/library/zlib.rst b/Doc/library/zlib.rst index 3d742ab35b9c..8a531c92b8ff 100644 --- a/Doc/library/zlib.rst +++ b/Doc/library/zlib.rst @@ -51,9 +51,9 @@ The available exception and functions in this module are: Compresses the bytes in *data*, returning a bytes object containing compressed data. *level* is an integer from ``0`` to ``9`` or ``-1`` controlling the level of compression; - ``1`` is fastest and produces the least compression, ``9`` is slowest and - produces the most. ``0`` is no compression. The default value is ``-1`` - (Z_DEFAULT_COMPRESSION). Z_DEFAULT_COMPRESSION represents a default + ``1`` (Z_BEST_SPEED) is fastest and produces the least compression, ``9`` (Z_BEST_COMPRESSION) + is slowest and produces the most. ``0`` (Z_NO_COMPRESSION) is no compression. + The default value is ``-1`` (Z_DEFAULT_COMPRESSION). Z_DEFAULT_COMPRESSION represents a default compromise between speed and compression (currently equivalent to level 6). Raises the :exc:`error` exception if any error occurs. @@ -61,23 +61,25 @@ The available exception and functions in this module are: *level* can now be used as a keyword parameter. -.. function:: compressobj(level=-1, method=DEFLATED, wbits=15, memLevel=8, strategy=Z_DEFAULT_STRATEGY[, zdict]) +.. function:: compressobj(level=-1, method=DEFLATED, wbits=MAX_WBITS, memLevel=DEF_MEM_LEVEL, strategy=Z_DEFAULT_STRATEGY[, zdict]) Returns a compression object, to be used for compressing data streams that won't fit into memory at once. *level* is the compression level -- an integer from ``0`` to ``9`` or ``-1``. - A value of ``1`` is fastest and produces the least compression, while a value of - ``9`` is slowest and produces the most. ``0`` is no compression. The default - value is ``-1`` (Z_DEFAULT_COMPRESSION). Z_DEFAULT_COMPRESSION represents a default - compromise between speed and compression (currently equivalent to level 6). + A value of ``1`` (Z_BEST_SPEED) is fastest and produces the least compression, + while a value of ``9`` (Z_BEST_COMPRESSION) is slowest and produces the most. + ``0`` (Z_NO_COMPRESSION) is no compression. The default value is ``-1`` (Z_DEFAULT_COMPRESSION). + Z_DEFAULT_COMPRESSION represents a default compromise between speed and compression + (currently equivalent to level 6). *method* is the compression algorithm. Currently, the only supported value is - ``DEFLATED``. + :const:`DEFLATED`. The *wbits* argument controls the size of the history buffer (or the "window size") used when compressing data, and whether a header and - trailer is included in the output. It can take several ranges of values: + trailer is included in the output. It can take several ranges of values, + defaulting to ``15`` (MAX_WBITS): * +9 to +15: The base-two logarithm of the window size, which therefore ranges between 512 and 32768. Larger values produce @@ -97,7 +99,8 @@ The available exception and functions in this module are: Higher values use more memory, but are faster and produce smaller output. *strategy* is used to tune the compression algorithm. Possible values are - ``Z_DEFAULT_STRATEGY``, ``Z_FILTERED``, and ``Z_HUFFMAN_ONLY``. + :const:`Z_DEFAULT_STRATEGY`, :const:`Z_FILTERED`, :const:`Z_HUFFMAN_ONLY`, + :const:`Z_RLE` (zlib 1.2.0.1) and :const:`Z_FIXED` (zlib 1.2.2.2). *zdict* is a predefined compression dictionary. This is a sequence of bytes (such as a :class:`bytes` object) containing subsequences that are expected @@ -175,7 +178,7 @@ The available exception and functions in this module are: .. versionchanged:: 3.6 *wbits* and *bufsize* can be used as keyword arguments. -.. function:: decompressobj(wbits=15[, zdict]) +.. function:: decompressobj(wbits=MAX_WBITS[, zdict]) Returns a decompression object, to be used for decompressing data streams that won't fit into memory at once. @@ -213,13 +216,13 @@ Compression objects support the following methods: All pending input is processed, and a bytes object containing the remaining compressed output is returned. *mode* can be selected from the constants - :const:`Z_SYNC_FLUSH`, :const:`Z_FULL_FLUSH`, or :const:`Z_FINISH`, - defaulting to :const:`Z_FINISH`. :const:`Z_SYNC_FLUSH` and - :const:`Z_FULL_FLUSH` allow compressing further bytestrings of data, while - :const:`Z_FINISH` finishes the compressed stream and prevents compressing any - more data. After calling :meth:`flush` with *mode* set to :const:`Z_FINISH`, - the :meth:`compress` method cannot be called again; the only realistic action is - to delete the object. + :const:`Z_NO_FLUSH`, :const:`Z_PARTIAL_FLUSH`, :const:`Z_SYNC_FLUSH`, + :const:`Z_FULL_FLUSH`, :const:`Z_BLOCK` (zlib 1.2.3.4), or :const:`Z_FINISH`, + defaulting to :const:`Z_FINISH`. Except :const:`Z_FINISH`, all constants + allow compressing further bytestrings of data, while :const:`Z_FINISH` finishes the + compressed stream and prevents compressing any more data. After calling :meth:`flush` + with *mode* set to :const:`Z_FINISH`, the :meth:`compress` method cannot be called again; + the only realistic action is to delete the object. .. method:: Compress.copy() diff --git a/Lib/test/test_zlib.py b/Lib/test/test_zlib.py index db950fc1907a..99aa89bdc8c9 100644 --- a/Lib/test/test_zlib.py +++ b/Lib/test/test_zlib.py @@ -434,7 +434,8 @@ def test_clear_unconsumed_tail(self): def test_flushes(self): # Test flush() with the various options, using all the # different levels in order to provide more variations. - sync_opt = ['Z_NO_FLUSH', 'Z_SYNC_FLUSH', 'Z_FULL_FLUSH'] + sync_opt = ['Z_NO_FLUSH', 'Z_SYNC_FLUSH', 'Z_FULL_FLUSH', + 'Z_PARTIAL_FLUSH', 'Z_BLOCK'] sync_opt = [getattr(zlib, opt) for opt in sync_opt if hasattr(zlib, opt)] data = HAMLET_SCENE * 8 diff --git a/Misc/NEWS.d/next/Library/2018-03-06-00-19-41.bpo-32969.rGTKa0.rst b/Misc/NEWS.d/next/Library/2018-03-06-00-19-41.bpo-32969.rGTKa0.rst new file mode 100644 index 000000000000..a92307e67bfa --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-03-06-00-19-41.bpo-32969.rGTKa0.rst @@ -0,0 +1,2 @@ +Expose several missing constants in zlib and fix corresponding +documentation. diff --git a/Modules/zlibmodule.c b/Modules/zlibmodule.c index cb2aae106944..cd587b4ac9c7 100644 --- a/Modules/zlibmodule.c +++ b/Modules/zlibmodule.c @@ -1361,18 +1361,33 @@ PyInit_zlib(void) PyModule_AddIntMacro(m, DEFLATED); PyModule_AddIntMacro(m, DEF_MEM_LEVEL); PyModule_AddIntMacro(m, DEF_BUF_SIZE); + // compression levels + PyModule_AddIntMacro(m, Z_NO_COMPRESSION); PyModule_AddIntMacro(m, Z_BEST_SPEED); PyModule_AddIntMacro(m, Z_BEST_COMPRESSION); PyModule_AddIntMacro(m, Z_DEFAULT_COMPRESSION); + // compression strategies PyModule_AddIntMacro(m, Z_FILTERED); PyModule_AddIntMacro(m, Z_HUFFMAN_ONLY); +#ifdef Z_RLE // 1.2.0.1 + PyModule_AddIntMacro(m, Z_RLE); +#endif +#ifdef Z_FIXED // 1.2.2.2 + PyModule_AddIntMacro(m, Z_FIXED); +#endif PyModule_AddIntMacro(m, Z_DEFAULT_STRATEGY); - - PyModule_AddIntMacro(m, Z_FINISH); + // allowed flush values PyModule_AddIntMacro(m, Z_NO_FLUSH); + PyModule_AddIntMacro(m, Z_PARTIAL_FLUSH); PyModule_AddIntMacro(m, Z_SYNC_FLUSH); PyModule_AddIntMacro(m, Z_FULL_FLUSH); - + PyModule_AddIntMacro(m, Z_FINISH); +#ifdef Z_BLOCK // 1.2.0.5 for inflate, 1.2.3.4 for deflate + PyModule_AddIntMacro(m, Z_BLOCK); +#endif +#ifdef Z_TREES // 1.2.3.4, only for inflate + PyModule_AddIntMacro(m, Z_TREES); +#endif ver = PyUnicode_FromString(ZLIB_VERSION); if (ver != NULL) PyModule_AddObject(m, "ZLIB_VERSION", ver); From webhook-mailer at python.org Wed Mar 7 00:26:22 2018 From: webhook-mailer at python.org (Miss Islington (bot)) Date: Wed, 07 Mar 2018 05:26:22 -0000 Subject: [Python-checkins] bpo-32969: Expose some missing constants in zlib and fix the doc (GH-5988) Message-ID: <mailman.138.1520400384.2831.python-checkins@python.org> https://github.com/python/cpython/commit/c4d77a661138debbbe584b8b08410afc8719a9b1 commit: c4d77a661138debbbe584b8b08410afc8719a9b1 branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: GitHub <noreply at github.com> date: 2018-03-06T21:26:19-08:00 summary: bpo-32969: Expose some missing constants in zlib and fix the doc (GH-5988) (cherry picked from commit bc3f2289b9007396bfb7f986bee477b6176c1822) Co-authored-by: Xiang Zhang <angwerzx at 126.com> files: A Misc/NEWS.d/next/Library/2018-03-06-00-19-41.bpo-32969.rGTKa0.rst M Doc/library/zlib.rst M Lib/test/test_zlib.py M Modules/zlibmodule.c diff --git a/Doc/library/zlib.rst b/Doc/library/zlib.rst index 3d742ab35b9c..8a531c92b8ff 100644 --- a/Doc/library/zlib.rst +++ b/Doc/library/zlib.rst @@ -51,9 +51,9 @@ The available exception and functions in this module are: Compresses the bytes in *data*, returning a bytes object containing compressed data. *level* is an integer from ``0`` to ``9`` or ``-1`` controlling the level of compression; - ``1`` is fastest and produces the least compression, ``9`` is slowest and - produces the most. ``0`` is no compression. The default value is ``-1`` - (Z_DEFAULT_COMPRESSION). Z_DEFAULT_COMPRESSION represents a default + ``1`` (Z_BEST_SPEED) is fastest and produces the least compression, ``9`` (Z_BEST_COMPRESSION) + is slowest and produces the most. ``0`` (Z_NO_COMPRESSION) is no compression. + The default value is ``-1`` (Z_DEFAULT_COMPRESSION). Z_DEFAULT_COMPRESSION represents a default compromise between speed and compression (currently equivalent to level 6). Raises the :exc:`error` exception if any error occurs. @@ -61,23 +61,25 @@ The available exception and functions in this module are: *level* can now be used as a keyword parameter. -.. function:: compressobj(level=-1, method=DEFLATED, wbits=15, memLevel=8, strategy=Z_DEFAULT_STRATEGY[, zdict]) +.. function:: compressobj(level=-1, method=DEFLATED, wbits=MAX_WBITS, memLevel=DEF_MEM_LEVEL, strategy=Z_DEFAULT_STRATEGY[, zdict]) Returns a compression object, to be used for compressing data streams that won't fit into memory at once. *level* is the compression level -- an integer from ``0`` to ``9`` or ``-1``. - A value of ``1`` is fastest and produces the least compression, while a value of - ``9`` is slowest and produces the most. ``0`` is no compression. The default - value is ``-1`` (Z_DEFAULT_COMPRESSION). Z_DEFAULT_COMPRESSION represents a default - compromise between speed and compression (currently equivalent to level 6). + A value of ``1`` (Z_BEST_SPEED) is fastest and produces the least compression, + while a value of ``9`` (Z_BEST_COMPRESSION) is slowest and produces the most. + ``0`` (Z_NO_COMPRESSION) is no compression. The default value is ``-1`` (Z_DEFAULT_COMPRESSION). + Z_DEFAULT_COMPRESSION represents a default compromise between speed and compression + (currently equivalent to level 6). *method* is the compression algorithm. Currently, the only supported value is - ``DEFLATED``. + :const:`DEFLATED`. The *wbits* argument controls the size of the history buffer (or the "window size") used when compressing data, and whether a header and - trailer is included in the output. It can take several ranges of values: + trailer is included in the output. It can take several ranges of values, + defaulting to ``15`` (MAX_WBITS): * +9 to +15: The base-two logarithm of the window size, which therefore ranges between 512 and 32768. Larger values produce @@ -97,7 +99,8 @@ The available exception and functions in this module are: Higher values use more memory, but are faster and produce smaller output. *strategy* is used to tune the compression algorithm. Possible values are - ``Z_DEFAULT_STRATEGY``, ``Z_FILTERED``, and ``Z_HUFFMAN_ONLY``. + :const:`Z_DEFAULT_STRATEGY`, :const:`Z_FILTERED`, :const:`Z_HUFFMAN_ONLY`, + :const:`Z_RLE` (zlib 1.2.0.1) and :const:`Z_FIXED` (zlib 1.2.2.2). *zdict* is a predefined compression dictionary. This is a sequence of bytes (such as a :class:`bytes` object) containing subsequences that are expected @@ -175,7 +178,7 @@ The available exception and functions in this module are: .. versionchanged:: 3.6 *wbits* and *bufsize* can be used as keyword arguments. -.. function:: decompressobj(wbits=15[, zdict]) +.. function:: decompressobj(wbits=MAX_WBITS[, zdict]) Returns a decompression object, to be used for decompressing data streams that won't fit into memory at once. @@ -213,13 +216,13 @@ Compression objects support the following methods: All pending input is processed, and a bytes object containing the remaining compressed output is returned. *mode* can be selected from the constants - :const:`Z_SYNC_FLUSH`, :const:`Z_FULL_FLUSH`, or :const:`Z_FINISH`, - defaulting to :const:`Z_FINISH`. :const:`Z_SYNC_FLUSH` and - :const:`Z_FULL_FLUSH` allow compressing further bytestrings of data, while - :const:`Z_FINISH` finishes the compressed stream and prevents compressing any - more data. After calling :meth:`flush` with *mode* set to :const:`Z_FINISH`, - the :meth:`compress` method cannot be called again; the only realistic action is - to delete the object. + :const:`Z_NO_FLUSH`, :const:`Z_PARTIAL_FLUSH`, :const:`Z_SYNC_FLUSH`, + :const:`Z_FULL_FLUSH`, :const:`Z_BLOCK` (zlib 1.2.3.4), or :const:`Z_FINISH`, + defaulting to :const:`Z_FINISH`. Except :const:`Z_FINISH`, all constants + allow compressing further bytestrings of data, while :const:`Z_FINISH` finishes the + compressed stream and prevents compressing any more data. After calling :meth:`flush` + with *mode* set to :const:`Z_FINISH`, the :meth:`compress` method cannot be called again; + the only realistic action is to delete the object. .. method:: Compress.copy() diff --git a/Lib/test/test_zlib.py b/Lib/test/test_zlib.py index db950fc1907a..99aa89bdc8c9 100644 --- a/Lib/test/test_zlib.py +++ b/Lib/test/test_zlib.py @@ -434,7 +434,8 @@ def test_clear_unconsumed_tail(self): def test_flushes(self): # Test flush() with the various options, using all the # different levels in order to provide more variations. - sync_opt = ['Z_NO_FLUSH', 'Z_SYNC_FLUSH', 'Z_FULL_FLUSH'] + sync_opt = ['Z_NO_FLUSH', 'Z_SYNC_FLUSH', 'Z_FULL_FLUSH', + 'Z_PARTIAL_FLUSH', 'Z_BLOCK'] sync_opt = [getattr(zlib, opt) for opt in sync_opt if hasattr(zlib, opt)] data = HAMLET_SCENE * 8 diff --git a/Misc/NEWS.d/next/Library/2018-03-06-00-19-41.bpo-32969.rGTKa0.rst b/Misc/NEWS.d/next/Library/2018-03-06-00-19-41.bpo-32969.rGTKa0.rst new file mode 100644 index 000000000000..a92307e67bfa --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-03-06-00-19-41.bpo-32969.rGTKa0.rst @@ -0,0 +1,2 @@ +Expose several missing constants in zlib and fix corresponding +documentation. diff --git a/Modules/zlibmodule.c b/Modules/zlibmodule.c index cb2aae106944..cd587b4ac9c7 100644 --- a/Modules/zlibmodule.c +++ b/Modules/zlibmodule.c @@ -1361,18 +1361,33 @@ PyInit_zlib(void) PyModule_AddIntMacro(m, DEFLATED); PyModule_AddIntMacro(m, DEF_MEM_LEVEL); PyModule_AddIntMacro(m, DEF_BUF_SIZE); + // compression levels + PyModule_AddIntMacro(m, Z_NO_COMPRESSION); PyModule_AddIntMacro(m, Z_BEST_SPEED); PyModule_AddIntMacro(m, Z_BEST_COMPRESSION); PyModule_AddIntMacro(m, Z_DEFAULT_COMPRESSION); + // compression strategies PyModule_AddIntMacro(m, Z_FILTERED); PyModule_AddIntMacro(m, Z_HUFFMAN_ONLY); +#ifdef Z_RLE // 1.2.0.1 + PyModule_AddIntMacro(m, Z_RLE); +#endif +#ifdef Z_FIXED // 1.2.2.2 + PyModule_AddIntMacro(m, Z_FIXED); +#endif PyModule_AddIntMacro(m, Z_DEFAULT_STRATEGY); - - PyModule_AddIntMacro(m, Z_FINISH); + // allowed flush values PyModule_AddIntMacro(m, Z_NO_FLUSH); + PyModule_AddIntMacro(m, Z_PARTIAL_FLUSH); PyModule_AddIntMacro(m, Z_SYNC_FLUSH); PyModule_AddIntMacro(m, Z_FULL_FLUSH); - + PyModule_AddIntMacro(m, Z_FINISH); +#ifdef Z_BLOCK // 1.2.0.5 for inflate, 1.2.3.4 for deflate + PyModule_AddIntMacro(m, Z_BLOCK); +#endif +#ifdef Z_TREES // 1.2.3.4, only for inflate + PyModule_AddIntMacro(m, Z_TREES); +#endif ver = PyUnicode_FromString(ZLIB_VERSION); if (ver != NULL) PyModule_AddObject(m, "ZLIB_VERSION", ver); From webhook-mailer at python.org Wed Mar 7 00:46:13 2018 From: webhook-mailer at python.org (Miss Islington (bot)) Date: Wed, 07 Mar 2018 05:46:13 -0000 Subject: [Python-checkins] bpo-32969: Expose some missing constants in zlib and fix the doc (GH-5988) Message-ID: <mailman.139.1520401575.2831.python-checkins@python.org> https://github.com/python/cpython/commit/7592c0a686a80b9fbe2e6d519a486aca58b3260b commit: 7592c0a686a80b9fbe2e6d519a486aca58b3260b branch: 3.6 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: GitHub <noreply at github.com> date: 2018-03-06T21:46:10-08:00 summary: bpo-32969: Expose some missing constants in zlib and fix the doc (GH-5988) (cherry picked from commit bc3f2289b9007396bfb7f986bee477b6176c1822) Co-authored-by: Xiang Zhang <angwerzx at 126.com> files: A Misc/NEWS.d/next/Library/2018-03-06-00-19-41.bpo-32969.rGTKa0.rst M Doc/library/zlib.rst M Lib/test/test_zlib.py M Modules/zlibmodule.c diff --git a/Doc/library/zlib.rst b/Doc/library/zlib.rst index 3d742ab35b9c..8a531c92b8ff 100644 --- a/Doc/library/zlib.rst +++ b/Doc/library/zlib.rst @@ -51,9 +51,9 @@ The available exception and functions in this module are: Compresses the bytes in *data*, returning a bytes object containing compressed data. *level* is an integer from ``0`` to ``9`` or ``-1`` controlling the level of compression; - ``1`` is fastest and produces the least compression, ``9`` is slowest and - produces the most. ``0`` is no compression. The default value is ``-1`` - (Z_DEFAULT_COMPRESSION). Z_DEFAULT_COMPRESSION represents a default + ``1`` (Z_BEST_SPEED) is fastest and produces the least compression, ``9`` (Z_BEST_COMPRESSION) + is slowest and produces the most. ``0`` (Z_NO_COMPRESSION) is no compression. + The default value is ``-1`` (Z_DEFAULT_COMPRESSION). Z_DEFAULT_COMPRESSION represents a default compromise between speed and compression (currently equivalent to level 6). Raises the :exc:`error` exception if any error occurs. @@ -61,23 +61,25 @@ The available exception and functions in this module are: *level* can now be used as a keyword parameter. -.. function:: compressobj(level=-1, method=DEFLATED, wbits=15, memLevel=8, strategy=Z_DEFAULT_STRATEGY[, zdict]) +.. function:: compressobj(level=-1, method=DEFLATED, wbits=MAX_WBITS, memLevel=DEF_MEM_LEVEL, strategy=Z_DEFAULT_STRATEGY[, zdict]) Returns a compression object, to be used for compressing data streams that won't fit into memory at once. *level* is the compression level -- an integer from ``0`` to ``9`` or ``-1``. - A value of ``1`` is fastest and produces the least compression, while a value of - ``9`` is slowest and produces the most. ``0`` is no compression. The default - value is ``-1`` (Z_DEFAULT_COMPRESSION). Z_DEFAULT_COMPRESSION represents a default - compromise between speed and compression (currently equivalent to level 6). + A value of ``1`` (Z_BEST_SPEED) is fastest and produces the least compression, + while a value of ``9`` (Z_BEST_COMPRESSION) is slowest and produces the most. + ``0`` (Z_NO_COMPRESSION) is no compression. The default value is ``-1`` (Z_DEFAULT_COMPRESSION). + Z_DEFAULT_COMPRESSION represents a default compromise between speed and compression + (currently equivalent to level 6). *method* is the compression algorithm. Currently, the only supported value is - ``DEFLATED``. + :const:`DEFLATED`. The *wbits* argument controls the size of the history buffer (or the "window size") used when compressing data, and whether a header and - trailer is included in the output. It can take several ranges of values: + trailer is included in the output. It can take several ranges of values, + defaulting to ``15`` (MAX_WBITS): * +9 to +15: The base-two logarithm of the window size, which therefore ranges between 512 and 32768. Larger values produce @@ -97,7 +99,8 @@ The available exception and functions in this module are: Higher values use more memory, but are faster and produce smaller output. *strategy* is used to tune the compression algorithm. Possible values are - ``Z_DEFAULT_STRATEGY``, ``Z_FILTERED``, and ``Z_HUFFMAN_ONLY``. + :const:`Z_DEFAULT_STRATEGY`, :const:`Z_FILTERED`, :const:`Z_HUFFMAN_ONLY`, + :const:`Z_RLE` (zlib 1.2.0.1) and :const:`Z_FIXED` (zlib 1.2.2.2). *zdict* is a predefined compression dictionary. This is a sequence of bytes (such as a :class:`bytes` object) containing subsequences that are expected @@ -175,7 +178,7 @@ The available exception and functions in this module are: .. versionchanged:: 3.6 *wbits* and *bufsize* can be used as keyword arguments. -.. function:: decompressobj(wbits=15[, zdict]) +.. function:: decompressobj(wbits=MAX_WBITS[, zdict]) Returns a decompression object, to be used for decompressing data streams that won't fit into memory at once. @@ -213,13 +216,13 @@ Compression objects support the following methods: All pending input is processed, and a bytes object containing the remaining compressed output is returned. *mode* can be selected from the constants - :const:`Z_SYNC_FLUSH`, :const:`Z_FULL_FLUSH`, or :const:`Z_FINISH`, - defaulting to :const:`Z_FINISH`. :const:`Z_SYNC_FLUSH` and - :const:`Z_FULL_FLUSH` allow compressing further bytestrings of data, while - :const:`Z_FINISH` finishes the compressed stream and prevents compressing any - more data. After calling :meth:`flush` with *mode* set to :const:`Z_FINISH`, - the :meth:`compress` method cannot be called again; the only realistic action is - to delete the object. + :const:`Z_NO_FLUSH`, :const:`Z_PARTIAL_FLUSH`, :const:`Z_SYNC_FLUSH`, + :const:`Z_FULL_FLUSH`, :const:`Z_BLOCK` (zlib 1.2.3.4), or :const:`Z_FINISH`, + defaulting to :const:`Z_FINISH`. Except :const:`Z_FINISH`, all constants + allow compressing further bytestrings of data, while :const:`Z_FINISH` finishes the + compressed stream and prevents compressing any more data. After calling :meth:`flush` + with *mode* set to :const:`Z_FINISH`, the :meth:`compress` method cannot be called again; + the only realistic action is to delete the object. .. method:: Compress.copy() diff --git a/Lib/test/test_zlib.py b/Lib/test/test_zlib.py index 8e29b1bcebff..5b58c77630b6 100644 --- a/Lib/test/test_zlib.py +++ b/Lib/test/test_zlib.py @@ -434,7 +434,8 @@ def test_clear_unconsumed_tail(self): def test_flushes(self): # Test flush() with the various options, using all the # different levels in order to provide more variations. - sync_opt = ['Z_NO_FLUSH', 'Z_SYNC_FLUSH', 'Z_FULL_FLUSH'] + sync_opt = ['Z_NO_FLUSH', 'Z_SYNC_FLUSH', 'Z_FULL_FLUSH', + 'Z_PARTIAL_FLUSH', 'Z_BLOCK'] sync_opt = [getattr(zlib, opt) for opt in sync_opt if hasattr(zlib, opt)] data = HAMLET_SCENE * 8 diff --git a/Misc/NEWS.d/next/Library/2018-03-06-00-19-41.bpo-32969.rGTKa0.rst b/Misc/NEWS.d/next/Library/2018-03-06-00-19-41.bpo-32969.rGTKa0.rst new file mode 100644 index 000000000000..a92307e67bfa --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-03-06-00-19-41.bpo-32969.rGTKa0.rst @@ -0,0 +1,2 @@ +Expose several missing constants in zlib and fix corresponding +documentation. diff --git a/Modules/zlibmodule.c b/Modules/zlibmodule.c index 29f55bdb19c6..a20dcd6de51e 100644 --- a/Modules/zlibmodule.c +++ b/Modules/zlibmodule.c @@ -1372,18 +1372,33 @@ PyInit_zlib(void) PyModule_AddIntMacro(m, DEFLATED); PyModule_AddIntMacro(m, DEF_MEM_LEVEL); PyModule_AddIntMacro(m, DEF_BUF_SIZE); + // compression levels + PyModule_AddIntMacro(m, Z_NO_COMPRESSION); PyModule_AddIntMacro(m, Z_BEST_SPEED); PyModule_AddIntMacro(m, Z_BEST_COMPRESSION); PyModule_AddIntMacro(m, Z_DEFAULT_COMPRESSION); + // compression strategies PyModule_AddIntMacro(m, Z_FILTERED); PyModule_AddIntMacro(m, Z_HUFFMAN_ONLY); +#ifdef Z_RLE // 1.2.0.1 + PyModule_AddIntMacro(m, Z_RLE); +#endif +#ifdef Z_FIXED // 1.2.2.2 + PyModule_AddIntMacro(m, Z_FIXED); +#endif PyModule_AddIntMacro(m, Z_DEFAULT_STRATEGY); - - PyModule_AddIntMacro(m, Z_FINISH); + // allowed flush values PyModule_AddIntMacro(m, Z_NO_FLUSH); + PyModule_AddIntMacro(m, Z_PARTIAL_FLUSH); PyModule_AddIntMacro(m, Z_SYNC_FLUSH); PyModule_AddIntMacro(m, Z_FULL_FLUSH); - + PyModule_AddIntMacro(m, Z_FINISH); +#ifdef Z_BLOCK // 1.2.0.5 for inflate, 1.2.3.4 for deflate + PyModule_AddIntMacro(m, Z_BLOCK); +#endif +#ifdef Z_TREES // 1.2.3.4, only for inflate + PyModule_AddIntMacro(m, Z_TREES); +#endif ver = PyUnicode_FromString(ZLIB_VERSION); if (ver != NULL) PyModule_AddObject(m, "ZLIB_VERSION", ver); From webhook-mailer at python.org Wed Mar 7 02:27:07 2018 From: webhook-mailer at python.org (INADA Naoki) Date: Wed, 07 Mar 2018 07:27:07 -0000 Subject: [Python-checkins] bpo-32999: Fix ABC.__subclasscheck__ crash (GH-6002) Message-ID: <mailman.140.1520407628.2831.python-checkins@python.org> https://github.com/python/cpython/commit/fc7df0e664198cb05cafd972f190a18ca422989c commit: fc7df0e664198cb05cafd972f190a18ca422989c branch: master author: INADA Naoki <methane at users.noreply.github.com> committer: GitHub <noreply at github.com> date: 2018-03-07T16:27:01+09:00 summary: bpo-32999: Fix ABC.__subclasscheck__ crash (GH-6002) files: A Misc/NEWS.d/next/Library/2018-03-06-20-30-20.bpo-32999.lgFXWl.rst M Lib/test/test_abc.py M Modules/_abc.c diff --git a/Lib/test/test_abc.py b/Lib/test/test_abc.py index af26c1d6b8c5..6fc3c95e4a64 100644 --- a/Lib/test/test_abc.py +++ b/Lib/test/test_abc.py @@ -392,6 +392,24 @@ class MyInt(int): self.assertIsInstance(42, A) self.assertIsInstance(42, (A,)) + def test_issubclass_bad_arguments(self): + class A(metaclass=abc_ABCMeta): + pass + + with self.assertRaises(TypeError): + issubclass({}, A) # unhashable + + with self.assertRaises(TypeError): + issubclass(42, A) # No __mro__ + + # Python version supports any iterable as __mro__. + # But it's implementation detail and don't emulate it in C version. + class C: + __mro__ = 42 # __mro__ is not tuple + + with self.assertRaises(TypeError): + issubclass(C(), A) + def test_all_new_methods_are_called(self): class A(metaclass=abc_ABCMeta): pass diff --git a/Misc/NEWS.d/next/Library/2018-03-06-20-30-20.bpo-32999.lgFXWl.rst b/Misc/NEWS.d/next/Library/2018-03-06-20-30-20.bpo-32999.lgFXWl.rst new file mode 100644 index 000000000000..45e75f939310 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-03-06-20-30-20.bpo-32999.lgFXWl.rst @@ -0,0 +1,2 @@ +Fix C implemetation of ``ABC.__subclasscheck__(cls, subclass)`` crashed when +``subclass`` is not a type object. diff --git a/Modules/_abc.c b/Modules/_abc.c index 504e23d9a74d..862883987fb7 100644 --- a/Modules/_abc.c +++ b/Modules/_abc.c @@ -16,6 +16,7 @@ _Py_IDENTIFIER(__abstractmethods__); _Py_IDENTIFIER(__class__); _Py_IDENTIFIER(__dict__); _Py_IDENTIFIER(__bases__); +_Py_IDENTIFIER(__mro__); _Py_IDENTIFIER(_abc_impl); _Py_IDENTIFIER(__subclasscheck__); _Py_IDENTIFIER(__subclasshook__); @@ -568,7 +569,7 @@ _abc__abc_subclasscheck_impl(PyObject *module, PyObject *self, PyObject *subclass) /*[clinic end generated code: output=b56c9e4a530e3894 input=1d947243409d10b8]*/ { - PyObject *ok, *mro, *subclasses = NULL, *result = NULL; + PyObject *ok, *mro = NULL, *subclasses = NULL, *result = NULL; Py_ssize_t pos; int incache; _abc_data *impl = _get_impl(self); @@ -637,20 +638,31 @@ _abc__abc_subclasscheck_impl(PyObject *module, PyObject *self, } Py_DECREF(ok); - /* 4. Check if it's a direct subclass. */ - mro = ((PyTypeObject *)subclass)->tp_mro; - assert(PyTuple_Check(mro)); - for (pos = 0; pos < PyTuple_GET_SIZE(mro); pos++) { - PyObject *mro_item = PyTuple_GET_ITEM(mro, pos); - if (mro_item == NULL) { + /* 4. Check if it's a direct subclass. + * + * if cls in getattr(subclass, '__mro__', ()): + * cls._abc_cache.add(subclass) + * return True + */ + if (_PyObject_LookupAttrId(subclass, &PyId___mro__, &mro) < 0) { + goto end; + } + if (mro != NULL) { + if (!PyTuple_Check(mro)) { + // Python version supports non-tuple iterable. Keep it as + // implementation detail. + PyErr_SetString(PyExc_TypeError, "__mro__ is not a tuple"); goto end; } - if ((PyObject *)self == mro_item) { - if (_add_to_weak_set(&impl->_abc_cache, subclass) < 0) { + for (pos = 0; pos < PyTuple_GET_SIZE(mro); pos++) { + PyObject *mro_item = PyTuple_GET_ITEM(mro, pos); + if ((PyObject *)self == mro_item) { + if (_add_to_weak_set(&impl->_abc_cache, subclass) < 0) { + goto end; + } + result = Py_True; goto end; } - result = Py_True; - goto end; } } @@ -690,7 +702,8 @@ _abc__abc_subclasscheck_impl(PyObject *module, PyObject *self, result = Py_False; end: - Py_XDECREF(impl); + Py_DECREF(impl); + Py_XDECREF(mro); Py_XDECREF(subclasses); Py_XINCREF(result); return result; From webhook-mailer at python.org Wed Mar 7 02:47:42 2018 From: webhook-mailer at python.org (Miss Islington (bot)) Date: Wed, 07 Mar 2018 07:47:42 -0000 Subject: [Python-checkins] bpo-32999: Fix ABC.__subclasscheck__ crash (GH-6002) Message-ID: <mailman.141.1520408864.2831.python-checkins@python.org> https://github.com/python/cpython/commit/d824b4e4afbe9cf02310e39b14402fb2aa271f8f commit: d824b4e4afbe9cf02310e39b14402fb2aa271f8f branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: GitHub <noreply at github.com> date: 2018-03-06T23:47:40-08:00 summary: bpo-32999: Fix ABC.__subclasscheck__ crash (GH-6002) (cherry picked from commit fc7df0e664198cb05cafd972f190a18ca422989c) Co-authored-by: INADA Naoki <methane at users.noreply.github.com> files: A Misc/NEWS.d/next/Library/2018-03-06-20-30-20.bpo-32999.lgFXWl.rst M Lib/test/test_abc.py M Modules/_abc.c diff --git a/Lib/test/test_abc.py b/Lib/test/test_abc.py index af26c1d6b8c5..6fc3c95e4a64 100644 --- a/Lib/test/test_abc.py +++ b/Lib/test/test_abc.py @@ -392,6 +392,24 @@ class MyInt(int): self.assertIsInstance(42, A) self.assertIsInstance(42, (A,)) + def test_issubclass_bad_arguments(self): + class A(metaclass=abc_ABCMeta): + pass + + with self.assertRaises(TypeError): + issubclass({}, A) # unhashable + + with self.assertRaises(TypeError): + issubclass(42, A) # No __mro__ + + # Python version supports any iterable as __mro__. + # But it's implementation detail and don't emulate it in C version. + class C: + __mro__ = 42 # __mro__ is not tuple + + with self.assertRaises(TypeError): + issubclass(C(), A) + def test_all_new_methods_are_called(self): class A(metaclass=abc_ABCMeta): pass diff --git a/Misc/NEWS.d/next/Library/2018-03-06-20-30-20.bpo-32999.lgFXWl.rst b/Misc/NEWS.d/next/Library/2018-03-06-20-30-20.bpo-32999.lgFXWl.rst new file mode 100644 index 000000000000..45e75f939310 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-03-06-20-30-20.bpo-32999.lgFXWl.rst @@ -0,0 +1,2 @@ +Fix C implemetation of ``ABC.__subclasscheck__(cls, subclass)`` crashed when +``subclass`` is not a type object. diff --git a/Modules/_abc.c b/Modules/_abc.c index 504e23d9a74d..862883987fb7 100644 --- a/Modules/_abc.c +++ b/Modules/_abc.c @@ -16,6 +16,7 @@ _Py_IDENTIFIER(__abstractmethods__); _Py_IDENTIFIER(__class__); _Py_IDENTIFIER(__dict__); _Py_IDENTIFIER(__bases__); +_Py_IDENTIFIER(__mro__); _Py_IDENTIFIER(_abc_impl); _Py_IDENTIFIER(__subclasscheck__); _Py_IDENTIFIER(__subclasshook__); @@ -568,7 +569,7 @@ _abc__abc_subclasscheck_impl(PyObject *module, PyObject *self, PyObject *subclass) /*[clinic end generated code: output=b56c9e4a530e3894 input=1d947243409d10b8]*/ { - PyObject *ok, *mro, *subclasses = NULL, *result = NULL; + PyObject *ok, *mro = NULL, *subclasses = NULL, *result = NULL; Py_ssize_t pos; int incache; _abc_data *impl = _get_impl(self); @@ -637,20 +638,31 @@ _abc__abc_subclasscheck_impl(PyObject *module, PyObject *self, } Py_DECREF(ok); - /* 4. Check if it's a direct subclass. */ - mro = ((PyTypeObject *)subclass)->tp_mro; - assert(PyTuple_Check(mro)); - for (pos = 0; pos < PyTuple_GET_SIZE(mro); pos++) { - PyObject *mro_item = PyTuple_GET_ITEM(mro, pos); - if (mro_item == NULL) { + /* 4. Check if it's a direct subclass. + * + * if cls in getattr(subclass, '__mro__', ()): + * cls._abc_cache.add(subclass) + * return True + */ + if (_PyObject_LookupAttrId(subclass, &PyId___mro__, &mro) < 0) { + goto end; + } + if (mro != NULL) { + if (!PyTuple_Check(mro)) { + // Python version supports non-tuple iterable. Keep it as + // implementation detail. + PyErr_SetString(PyExc_TypeError, "__mro__ is not a tuple"); goto end; } - if ((PyObject *)self == mro_item) { - if (_add_to_weak_set(&impl->_abc_cache, subclass) < 0) { + for (pos = 0; pos < PyTuple_GET_SIZE(mro); pos++) { + PyObject *mro_item = PyTuple_GET_ITEM(mro, pos); + if ((PyObject *)self == mro_item) { + if (_add_to_weak_set(&impl->_abc_cache, subclass) < 0) { + goto end; + } + result = Py_True; goto end; } - result = Py_True; - goto end; } } @@ -690,7 +702,8 @@ _abc__abc_subclasscheck_impl(PyObject *module, PyObject *self, result = Py_False; end: - Py_XDECREF(impl); + Py_DECREF(impl); + Py_XDECREF(mro); Py_XDECREF(subclasses); Py_XINCREF(result); return result; From solipsis at pitrou.net Wed Mar 7 04:14:00 2018 From: solipsis at pitrou.net (solipsis at pitrou.net) Date: Wed, 07 Mar 2018 09:14:00 +0000 Subject: [Python-checkins] Daily reference leaks (4243df51fe43): sum=6 Message-ID: <20180307091400.1.B121A3426EF090D2@psf.io> results for 4243df51fe43 on branch "default" -------------------------------------------- test_functools leaked [0, 3, 1] memory blocks, sum=4 test_multiprocessing_spawn leaked [2, 0, 0] memory blocks, sum=2 Command line was: ['./python', '-m', 'test.regrtest', '-uall', '-R', '3:3:/home/psf-users/antoine/refleaks/refloggulaMw', '--timeout', '7200'] From webhook-mailer at python.org Wed Mar 7 23:24:40 2018 From: webhook-mailer at python.org (Xiang Zhang) Date: Thu, 08 Mar 2018 04:24:40 -0000 Subject: [Python-checkins] Fix some ipaddress documentation errors (GH-6021) Message-ID: <mailman.1.1520483080.2865.python-checkins@python.org> https://github.com/python/cpython/commit/e405096ea91f516d411095b6fea4eec9668eac88 commit: e405096ea91f516d411095b6fea4eec9668eac88 branch: master author: Xiang Zhang <angwerzx at 126.com> committer: GitHub <noreply at github.com> date: 2018-03-08T12:24:36+08:00 summary: Fix some ipaddress documentation errors (GH-6021) * fix a typo: documention -> documentation * fix the type of IPv?Network.hostmask * add documentation about IPv?Network.netmask * fix IPv6Network constructor doc that extended netmasks are not supported files: M Doc/library/ipaddress.rst diff --git a/Doc/library/ipaddress.rst b/Doc/library/ipaddress.rst index 75c9107bd5eb..76177a0b23eb 100644 --- a/Doc/library/ipaddress.rst +++ b/Doc/library/ipaddress.rst @@ -236,7 +236,7 @@ write code that handles both IP versions correctly. groups consisting entirely of zeroes included. - For the following attributes, see the corresponding documention of the + For the following attributes, see the corresponding documentation of the :class:`IPv4Address` class: .. attribute:: packed @@ -442,7 +442,11 @@ so to avoid duplication they are only documented for :class:`IPv4Network`. .. attribute:: hostmask - The host mask, as a string. + The host mask, as an :class:`IPv4Address` object. + + .. attribute:: netmask + + The net mask, as an :class:`IPv4Address` object. .. attribute:: with_prefixlen .. attribute:: compressed @@ -588,13 +592,12 @@ so to avoid duplication they are only documented for :class:`IPv4Network`. 1. A string consisting of an IP address and an optional mask, separated by a slash (``/``). The IP address is the network address, and the mask - can be either a single number, which means it's a *prefix*, or a string - representation of an IPv6 address. If it's the latter, the mask is - interpreted as a *net mask*. If no mask is provided, it's considered to - be ``/128``. + is a single number, which represents a *prefix*. If no mask is provided, + it's considered to be ``/128``. - For example, the following *address* specifications are equivalent: - ``2001:db00::0/24`` and ``2001:db00::0/ffff:ff00::``. + Note that currently expanded netmasks are not supported. That means + ``2001:db00::0/24`` is a valid argument while ``2001:db00::0/ffff:ff00::`` + not. 2. An integer that fits into 128 bits. This is equivalent to a single-address network, with the network address being *address* and @@ -631,6 +634,7 @@ so to avoid duplication they are only documented for :class:`IPv4Network`. .. attribute:: network_address .. attribute:: broadcast_address .. attribute:: hostmask + .. attribute:: netmask .. attribute:: with_prefixlen .. attribute:: compressed .. attribute:: exploded From webhook-mailer at python.org Wed Mar 7 23:35:55 2018 From: webhook-mailer at python.org (Miss Islington (bot)) Date: Thu, 08 Mar 2018 04:35:55 -0000 Subject: [Python-checkins] Fix some ipaddress documentation errors (GH-6021) Message-ID: <mailman.2.1520483756.2865.python-checkins@python.org> https://github.com/python/cpython/commit/c072e258858bf3c125f1bce3ec27ddb176300d04 commit: c072e258858bf3c125f1bce3ec27ddb176300d04 branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: GitHub <noreply at github.com> date: 2018-03-07T20:35:52-08:00 summary: Fix some ipaddress documentation errors (GH-6021) * fix a typo: documention -> documentation * fix the type of IPv?Network.hostmask * add documentation about IPv?Network.netmask * fix IPv6Network constructor doc that extended netmasks are not supported (cherry picked from commit e405096ea91f516d411095b6fea4eec9668eac88) Co-authored-by: Xiang Zhang <angwerzx at 126.com> files: M Doc/library/ipaddress.rst diff --git a/Doc/library/ipaddress.rst b/Doc/library/ipaddress.rst index 75c9107bd5eb..76177a0b23eb 100644 --- a/Doc/library/ipaddress.rst +++ b/Doc/library/ipaddress.rst @@ -236,7 +236,7 @@ write code that handles both IP versions correctly. groups consisting entirely of zeroes included. - For the following attributes, see the corresponding documention of the + For the following attributes, see the corresponding documentation of the :class:`IPv4Address` class: .. attribute:: packed @@ -442,7 +442,11 @@ so to avoid duplication they are only documented for :class:`IPv4Network`. .. attribute:: hostmask - The host mask, as a string. + The host mask, as an :class:`IPv4Address` object. + + .. attribute:: netmask + + The net mask, as an :class:`IPv4Address` object. .. attribute:: with_prefixlen .. attribute:: compressed @@ -588,13 +592,12 @@ so to avoid duplication they are only documented for :class:`IPv4Network`. 1. A string consisting of an IP address and an optional mask, separated by a slash (``/``). The IP address is the network address, and the mask - can be either a single number, which means it's a *prefix*, or a string - representation of an IPv6 address. If it's the latter, the mask is - interpreted as a *net mask*. If no mask is provided, it's considered to - be ``/128``. + is a single number, which represents a *prefix*. If no mask is provided, + it's considered to be ``/128``. - For example, the following *address* specifications are equivalent: - ``2001:db00::0/24`` and ``2001:db00::0/ffff:ff00::``. + Note that currently expanded netmasks are not supported. That means + ``2001:db00::0/24`` is a valid argument while ``2001:db00::0/ffff:ff00::`` + not. 2. An integer that fits into 128 bits. This is equivalent to a single-address network, with the network address being *address* and @@ -631,6 +634,7 @@ so to avoid duplication they are only documented for :class:`IPv4Network`. .. attribute:: network_address .. attribute:: broadcast_address .. attribute:: hostmask + .. attribute:: netmask .. attribute:: with_prefixlen .. attribute:: compressed .. attribute:: exploded From webhook-mailer at python.org Wed Mar 7 23:41:30 2018 From: webhook-mailer at python.org (Miss Islington (bot)) Date: Thu, 08 Mar 2018 04:41:30 -0000 Subject: [Python-checkins] Fix some ipaddress documentation errors (GH-6021) Message-ID: <mailman.3.1520484091.2865.python-checkins@python.org> https://github.com/python/cpython/commit/15425cf2456d816ad25723471375199277887de6 commit: 15425cf2456d816ad25723471375199277887de6 branch: 3.6 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: GitHub <noreply at github.com> date: 2018-03-07T20:41:27-08:00 summary: Fix some ipaddress documentation errors (GH-6021) * fix a typo: documention -> documentation * fix the type of IPv?Network.hostmask * add documentation about IPv?Network.netmask * fix IPv6Network constructor doc that extended netmasks are not supported (cherry picked from commit e405096ea91f516d411095b6fea4eec9668eac88) Co-authored-by: Xiang Zhang <angwerzx at 126.com> files: M Doc/library/ipaddress.rst diff --git a/Doc/library/ipaddress.rst b/Doc/library/ipaddress.rst index 50fb778dfbad..b3c691e444a1 100644 --- a/Doc/library/ipaddress.rst +++ b/Doc/library/ipaddress.rst @@ -234,7 +234,7 @@ write code that handles both IP versions correctly. groups consisting entirely of zeroes included. - For the following attributes, see the corresponding documention of the + For the following attributes, see the corresponding documentation of the :class:`IPv4Address` class: .. attribute:: packed @@ -440,7 +440,11 @@ so to avoid duplication they are only documented for :class:`IPv4Network`. .. attribute:: hostmask - The host mask, as a string. + The host mask, as an :class:`IPv4Address` object. + + .. attribute:: netmask + + The net mask, as an :class:`IPv4Address` object. .. attribute:: with_prefixlen .. attribute:: compressed @@ -561,13 +565,12 @@ so to avoid duplication they are only documented for :class:`IPv4Network`. 1. A string consisting of an IP address and an optional mask, separated by a slash (``/``). The IP address is the network address, and the mask - can be either a single number, which means it's a *prefix*, or a string - representation of an IPv6 address. If it's the latter, the mask is - interpreted as a *net mask*. If no mask is provided, it's considered to - be ``/128``. + is a single number, which represents a *prefix*. If no mask is provided, + it's considered to be ``/128``. - For example, the following *address* specifications are equivalent: - ``2001:db00::0/24`` and ``2001:db00::0/ffff:ff00::``. + Note that currently expanded netmasks are not supported. That means + ``2001:db00::0/24`` is a valid argument while ``2001:db00::0/ffff:ff00::`` + not. 2. An integer that fits into 128 bits. This is equivalent to a single-address network, with the network address being *address* and @@ -604,6 +607,7 @@ so to avoid duplication they are only documented for :class:`IPv4Network`. .. attribute:: network_address .. attribute:: broadcast_address .. attribute:: hostmask + .. attribute:: netmask .. attribute:: with_prefixlen .. attribute:: compressed .. attribute:: exploded From webhook-mailer at python.org Thu Mar 8 00:59:49 2018 From: webhook-mailer at python.org (Xiang Zhang) Date: Thu, 08 Mar 2018 05:59:49 -0000 Subject: [Python-checkins] Add two missing error checks in hamt.c (GH-5851) Message-ID: <mailman.5.1520488790.2865.python-checkins@python.org> https://github.com/python/cpython/commit/3c7ac7ea2098c672e50402d1d1b5ee9f14586437 commit: 3c7ac7ea2098c672e50402d1d1b5ee9f14586437 branch: master author: Xiang Zhang <angwerzx at 126.com> committer: GitHub <noreply at github.com> date: 2018-03-08T13:59:46+08:00 summary: Add two missing error checks in hamt.c (GH-5851) files: M Python/hamt.c diff --git a/Python/hamt.c b/Python/hamt.c index e54d3a4c55f9..53a85723745e 100644 --- a/Python/hamt.c +++ b/Python/hamt.c @@ -1510,6 +1510,9 @@ hamt_node_collision_without(PyHamtNode_Collision *self, PyHamtNode_Collision *new = (PyHamtNode_Collision *) hamt_node_collision_new( self->c_hash, Py_SIZE(self) - 2); + if (new == NULL) { + return W_ERROR; + } /* Copy all other keys from `self` to `new` */ Py_ssize_t i; @@ -1742,7 +1745,10 @@ hamt_node_array_assoc(PyHamtNode_Array *self, Set the key to it./ */ child_node = hamt_node_assoc( node, shift + 5, hash, key, val, added_leaf); - if (child_node == (PyHamtNode *)self) { + if (child_node == NULL) { + return NULL; + } + else if (child_node == (PyHamtNode *)self) { Py_DECREF(child_node); return (PyHamtNode *)self; } From webhook-mailer at python.org Thu Mar 8 01:21:37 2018 From: webhook-mailer at python.org (Miss Islington (bot)) Date: Thu, 08 Mar 2018 06:21:37 -0000 Subject: [Python-checkins] Add two missing error checks in hamt.c (GH-5851) Message-ID: <mailman.6.1520490098.2865.python-checkins@python.org> https://github.com/python/cpython/commit/e724dd4e5e9bdb888169d7cd2ca415a2fc2c29ab commit: e724dd4e5e9bdb888169d7cd2ca415a2fc2c29ab branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: GitHub <noreply at github.com> date: 2018-03-07T22:21:34-08:00 summary: Add two missing error checks in hamt.c (GH-5851) (cherry picked from commit 3c7ac7ea2098c672e50402d1d1b5ee9f14586437) Co-authored-by: Xiang Zhang <angwerzx at 126.com> files: M Python/hamt.c diff --git a/Python/hamt.c b/Python/hamt.c index e54d3a4c55f9..53a85723745e 100644 --- a/Python/hamt.c +++ b/Python/hamt.c @@ -1510,6 +1510,9 @@ hamt_node_collision_without(PyHamtNode_Collision *self, PyHamtNode_Collision *new = (PyHamtNode_Collision *) hamt_node_collision_new( self->c_hash, Py_SIZE(self) - 2); + if (new == NULL) { + return W_ERROR; + } /* Copy all other keys from `self` to `new` */ Py_ssize_t i; @@ -1742,7 +1745,10 @@ hamt_node_array_assoc(PyHamtNode_Array *self, Set the key to it./ */ child_node = hamt_node_assoc( node, shift + 5, hash, key, val, added_leaf); - if (child_node == (PyHamtNode *)self) { + if (child_node == NULL) { + return NULL; + } + else if (child_node == (PyHamtNode *)self) { Py_DECREF(child_node); return (PyHamtNode *)self; } From solipsis at pitrou.net Thu Mar 8 04:12:16 2018 From: solipsis at pitrou.net (solipsis at pitrou.net) Date: Thu, 08 Mar 2018 09:12:16 +0000 Subject: [Python-checkins] Daily reference leaks (4243df51fe43): sum=7 Message-ID: <20180308091216.1.C09F5A28839DCEB0@psf.io> results for 4243df51fe43 on branch "default" -------------------------------------------- test_collections leaked [7, -7, 1] memory blocks, sum=1 test_functools leaked [0, 3, 1] memory blocks, sum=4 test_multiprocessing_fork leaked [0, -1, 2] memory blocks, sum=1 test_multiprocessing_spawn leaked [2, -1, 0] memory blocks, sum=1 Command line was: ['./python', '-m', 'test.regrtest', '-uall', '-R', '3:3:/home/psf-users/antoine/refleaks/reflogKrlbtM', '--timeout', '7200'] From webhook-mailer at python.org Thu Mar 8 10:28:59 2018 From: webhook-mailer at python.org (Ned Deily) Date: Thu, 08 Mar 2018 15:28:59 -0000 Subject: [Python-checkins] [3.6] bpo-30353: Fix pass by value for structs on 64-bit Cygwin/MinGW (GH-1559) (GH-5954) Message-ID: <mailman.7.1520522942.2865.python-checkins@python.org> https://github.com/python/cpython/commit/2f3ba27185a369bcb6b36b13aa3518ffcc970ffa commit: 2f3ba27185a369bcb6b36b13aa3518ffcc970ffa branch: 3.6 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Ned Deily <nad at python.org> date: 2018-03-08T10:28:53-05:00 summary: [3.6] bpo-30353: Fix pass by value for structs on 64-bit Cygwin/MinGW (GH-1559) (GH-5954) (cherry picked from commit 9ba3aa4d02a110d1a1ea464a8aff3be7dd9c63c3) Co-authored-by: Erik Bray <erik.m.bray at gmail.com> files: A Misc/NEWS.d/next/Library/2018-03-08-09-54-01.bpo-30353.XdE5aM.rst M Lib/ctypes/test/test_as_parameter.py M Lib/ctypes/test/test_structures.py M Modules/_ctypes/_ctypes_test.c M Modules/_ctypes/callproc.c diff --git a/Lib/ctypes/test/test_as_parameter.py b/Lib/ctypes/test/test_as_parameter.py index 2a3484bec01b..a2640575a074 100644 --- a/Lib/ctypes/test/test_as_parameter.py +++ b/Lib/ctypes/test/test_as_parameter.py @@ -169,6 +169,10 @@ class S2H(Structure): s2h = dll.ret_2h_func(self.wrap(inp)) self.assertEqual((s2h.x, s2h.y), (99*2, 88*3)) + # Test also that the original struct was unmodified (i.e. was passed by + # value) + self.assertEqual((inp.x, inp.y), (99, 88)) + def test_struct_return_8H(self): class S8I(Structure): _fields_ = [("a", c_int), diff --git a/Lib/ctypes/test/test_structures.py b/Lib/ctypes/test/test_structures.py index 4ad5915ba27a..d1ea43bc7e3b 100644 --- a/Lib/ctypes/test/test_structures.py +++ b/Lib/ctypes/test/test_structures.py @@ -417,6 +417,28 @@ class X(Structure): self.assertEqual(s.second, 0xcafebabe) self.assertEqual(s.third, 0x0bad1dea) + def test_pass_by_value_in_register(self): + class X(Structure): + _fields_ = [ + ('first', c_uint), + ('second', c_uint) + ] + + s = X() + s.first = 0xdeadbeef + s.second = 0xcafebabe + dll = CDLL(_ctypes_test.__file__) + func = dll._testfunc_reg_struct_update_value + func.argtypes = (X,) + func.restype = None + func(s) + self.assertEqual(s.first, 0xdeadbeef) + self.assertEqual(s.second, 0xcafebabe) + got = X.in_dll(dll, "last_tfrsuv_arg") + self.assertEqual(s.first, got.first) + self.assertEqual(s.second, got.second) + + class PointerMemberTestCase(unittest.TestCase): def test(self): diff --git a/Misc/NEWS.d/next/Library/2018-03-08-09-54-01.bpo-30353.XdE5aM.rst b/Misc/NEWS.d/next/Library/2018-03-08-09-54-01.bpo-30353.XdE5aM.rst new file mode 100644 index 000000000000..ddb625c71652 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-03-08-09-54-01.bpo-30353.XdE5aM.rst @@ -0,0 +1 @@ +Fix ctypes pass-by-value for structs on 64-bit Cygwin/MinGW. diff --git a/Modules/_ctypes/_ctypes_test.c b/Modules/_ctypes/_ctypes_test.c index 6119ecdaf90d..f6af49aea385 100644 --- a/Modules/_ctypes/_ctypes_test.c +++ b/Modules/_ctypes/_ctypes_test.c @@ -57,6 +57,24 @@ _testfunc_large_struct_update_value(Test in) ((volatile Test *)&in)->third = 0x0badf00d; } +typedef struct { + unsigned int first; + unsigned int second; +} TestReg; + + +EXPORT(TestReg) last_tfrsuv_arg; + + +EXPORT(void) +_testfunc_reg_struct_update_value(TestReg in) +{ + last_tfrsuv_arg = in; + ((volatile TestReg *)&in)->first = 0x0badf00d; + ((volatile TestReg *)&in)->second = 0x0badf00d; +} + + EXPORT(void)testfunc_array(int values[4]) { printf("testfunc_array %d %d %d %d\n", diff --git a/Modules/_ctypes/callproc.c b/Modules/_ctypes/callproc.c index ff95dffb82c2..8edffabcdf09 100644 --- a/Modules/_ctypes/callproc.c +++ b/Modules/_ctypes/callproc.c @@ -1040,6 +1040,13 @@ GetComError(HRESULT errcode, GUID *riid, IUnknown *pIunk) } #endif +#if (defined(__x86_64__) && (defined(__MINGW64__) || defined(__CYGWIN__))) || \ + defined(__aarch64__) +#define CTYPES_PASS_BY_REF_HACK +#define POW2(x) (((x & ~(x - 1)) == x) ? x : 0) +#define IS_PASS_BY_REF(x) (x > 8 || !POW2(x)) +#endif + /* * Requirements, must be ensured by the caller: * - argtuple is tuple of arguments @@ -1137,8 +1144,20 @@ PyObject *_ctypes_callproc(PPROC pProc, } for (i = 0; i < argcount; ++i) { atypes[i] = args[i].ffi_type; - if (atypes[i]->type == FFI_TYPE_STRUCT - ) +#ifdef CTYPES_PASS_BY_REF_HACK + size_t size = atypes[i]->size; + if (IS_PASS_BY_REF(size)) { + void *tmp = alloca(size); + if (atypes[i]->type == FFI_TYPE_STRUCT) + memcpy(tmp, args[i].value.p, size); + else + memcpy(tmp, (void*)&args[i].value, size); + + avalues[i] = tmp; + } + else +#endif + if (atypes[i]->type == FFI_TYPE_STRUCT) avalues[i] = (void *)args[i].value.p; else avalues[i] = (void *)&args[i].value; From webhook-mailer at python.org Thu Mar 8 11:03:34 2018 From: webhook-mailer at python.org (Steve Dower) Date: Thu, 08 Mar 2018 16:03:34 -0000 Subject: [Python-checkins] bpo-33016: Fix potential use of uninitialized memory in nt._getfinalpathname (#6010) Message-ID: <mailman.8.1520525015.2865.python-checkins@python.org> https://github.com/python/cpython/commit/3b20d3454e8082e07dba93617793de5dc9237828 commit: 3b20d3454e8082e07dba93617793de5dc9237828 branch: master author: Alexey Izbyshev <izbyshev at users.noreply.github.com> committer: Steve Dower <steve.dower at microsoft.com> date: 2018-03-08T08:03:25-08:00 summary: bpo-33016: Fix potential use of uninitialized memory in nt._getfinalpathname (#6010) files: A Misc/NEWS.d/next/Windows/2018-03-07-01-33-33.bpo-33016.Z_Med0.rst M Modules/posixmodule.c diff --git a/Misc/NEWS.d/next/Windows/2018-03-07-01-33-33.bpo-33016.Z_Med0.rst b/Misc/NEWS.d/next/Windows/2018-03-07-01-33-33.bpo-33016.Z_Med0.rst new file mode 100644 index 000000000000..f4f78d489bf1 --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2018-03-07-01-33-33.bpo-33016.Z_Med0.rst @@ -0,0 +1 @@ +Fix potential use of uninitialized memory in nt._getfinalpathname diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index f4c01048cdae..e6721032f211 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -303,12 +303,6 @@ extern int lstat(const char *, struct stat *); #ifdef HAVE_PROCESS_H #include <process.h> #endif -#ifndef VOLUME_NAME_DOS -#define VOLUME_NAME_DOS 0x0 -#endif -#ifndef VOLUME_NAME_NT -#define VOLUME_NAME_NT 0x2 -#endif #ifndef IO_REPARSE_TAG_SYMLINK #define IO_REPARSE_TAG_SYMLINK (0xA000000CL) #endif @@ -3731,11 +3725,10 @@ os__getfinalpathname_impl(PyObject *module, path_t *path) /*[clinic end generated code: output=621a3c79bc29ebfa input=2b6b6c7cbad5fb84]*/ { HANDLE hFile; - int buf_size; - wchar_t *target_path; + wchar_t buf[MAXPATHLEN], *target_path = buf; + int buf_size = Py_ARRAY_LENGTH(buf); int result_length; PyObject *result; - const char *err = NULL; Py_BEGIN_ALLOW_THREADS hFile = CreateFileW( @@ -3747,55 +3740,52 @@ os__getfinalpathname_impl(PyObject *module, path_t *path) /* FILE_FLAG_BACKUP_SEMANTICS is required to open a directory */ FILE_FLAG_BACKUP_SEMANTICS, NULL); + Py_END_ALLOW_THREADS if (hFile == INVALID_HANDLE_VALUE) { - err = "CreateFileW"; - goto done1; + return win32_error_object("CreateFileW", path->object); } /* We have a good handle to the target, use it to determine the target path name. */ - buf_size = GetFinalPathNameByHandleW(hFile, 0, 0, VOLUME_NAME_NT); + while (1) { + Py_BEGIN_ALLOW_THREADS + result_length = GetFinalPathNameByHandleW(hFile, target_path, + buf_size, VOLUME_NAME_DOS); + Py_END_ALLOW_THREADS - if (!buf_size) { - err = "GetFinalPathNameByHandle"; - goto done1; - } -done1: - Py_END_ALLOW_THREADS - if (err) - return win32_error_object(err, path->object); + if (!result_length) { + result = win32_error_object("GetFinalPathNameByHandleW", + path->object); + goto cleanup; + } - target_path = PyMem_New(wchar_t, buf_size+1); - if(!target_path) - return PyErr_NoMemory(); + if (result_length < buf_size) { + break; + } - Py_BEGIN_ALLOW_THREADS - result_length = GetFinalPathNameByHandleW(hFile, target_path, - buf_size, VOLUME_NAME_DOS); - if (!result_length) { - err = "GetFinalPathNameByHandle"; - goto done2; - } + wchar_t *tmp; + tmp = PyMem_Realloc(target_path != buf ? target_path : NULL, + result_length * sizeof(*tmp)); + if (!tmp) { + result = PyErr_NoMemory(); + goto cleanup; + } - if (!CloseHandle(hFile)) { - err = "CloseHandle"; - goto done2; - } -done2: - Py_END_ALLOW_THREADS - if (err) { - PyMem_Free(target_path); - return win32_error_object(err, path->object); + buf_size = result_length; + target_path = tmp; } - target_path[result_length] = 0; result = PyUnicode_FromWideChar(target_path, result_length); - PyMem_Free(target_path); if (path->narrow) Py_SETREF(result, PyUnicode_EncodeFSDefault(result)); - return result; +cleanup: + if (target_path != buf) { + PyMem_Free(target_path); + } + CloseHandle(hFile); + return result; } /*[clinic input] From webhook-mailer at python.org Thu Mar 8 11:26:46 2018 From: webhook-mailer at python.org (Miss Islington (bot)) Date: Thu, 08 Mar 2018 16:26:46 -0000 Subject: [Python-checkins] bpo-33016: Fix potential use of uninitialized memory in nt._getfinalpathname (GH-6010) Message-ID: <mailman.9.1520526407.2865.python-checkins@python.org> https://github.com/python/cpython/commit/8c163bbf370f6f6cedd2c07f7d54c9b36c97d8f2 commit: 8c163bbf370f6f6cedd2c07f7d54c9b36c97d8f2 branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: GitHub <noreply at github.com> date: 2018-03-08T08:26:43-08:00 summary: bpo-33016: Fix potential use of uninitialized memory in nt._getfinalpathname (GH-6010) (cherry picked from commit 3b20d3454e8082e07dba93617793de5dc9237828) Co-authored-by: Alexey Izbyshev <izbyshev at users.noreply.github.com> files: A Misc/NEWS.d/next/Windows/2018-03-07-01-33-33.bpo-33016.Z_Med0.rst M Modules/posixmodule.c diff --git a/Misc/NEWS.d/next/Windows/2018-03-07-01-33-33.bpo-33016.Z_Med0.rst b/Misc/NEWS.d/next/Windows/2018-03-07-01-33-33.bpo-33016.Z_Med0.rst new file mode 100644 index 000000000000..f4f78d489bf1 --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2018-03-07-01-33-33.bpo-33016.Z_Med0.rst @@ -0,0 +1 @@ +Fix potential use of uninitialized memory in nt._getfinalpathname diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 6bf8c6ba7b7a..e8964ce33901 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -303,12 +303,6 @@ extern int lstat(const char *, struct stat *); #ifdef HAVE_PROCESS_H #include <process.h> #endif -#ifndef VOLUME_NAME_DOS -#define VOLUME_NAME_DOS 0x0 -#endif -#ifndef VOLUME_NAME_NT -#define VOLUME_NAME_NT 0x2 -#endif #ifndef IO_REPARSE_TAG_SYMLINK #define IO_REPARSE_TAG_SYMLINK (0xA000000CL) #endif @@ -3731,11 +3725,10 @@ os__getfinalpathname_impl(PyObject *module, path_t *path) /*[clinic end generated code: output=621a3c79bc29ebfa input=2b6b6c7cbad5fb84]*/ { HANDLE hFile; - int buf_size; - wchar_t *target_path; + wchar_t buf[MAXPATHLEN], *target_path = buf; + int buf_size = Py_ARRAY_LENGTH(buf); int result_length; PyObject *result; - const char *err = NULL; Py_BEGIN_ALLOW_THREADS hFile = CreateFileW( @@ -3747,55 +3740,52 @@ os__getfinalpathname_impl(PyObject *module, path_t *path) /* FILE_FLAG_BACKUP_SEMANTICS is required to open a directory */ FILE_FLAG_BACKUP_SEMANTICS, NULL); + Py_END_ALLOW_THREADS if (hFile == INVALID_HANDLE_VALUE) { - err = "CreateFileW"; - goto done1; + return win32_error_object("CreateFileW", path->object); } /* We have a good handle to the target, use it to determine the target path name. */ - buf_size = GetFinalPathNameByHandleW(hFile, 0, 0, VOLUME_NAME_NT); + while (1) { + Py_BEGIN_ALLOW_THREADS + result_length = GetFinalPathNameByHandleW(hFile, target_path, + buf_size, VOLUME_NAME_DOS); + Py_END_ALLOW_THREADS - if (!buf_size) { - err = "GetFinalPathNameByHandle"; - goto done1; - } -done1: - Py_END_ALLOW_THREADS - if (err) - return win32_error_object(err, path->object); + if (!result_length) { + result = win32_error_object("GetFinalPathNameByHandleW", + path->object); + goto cleanup; + } - target_path = PyMem_New(wchar_t, buf_size+1); - if(!target_path) - return PyErr_NoMemory(); + if (result_length < buf_size) { + break; + } - Py_BEGIN_ALLOW_THREADS - result_length = GetFinalPathNameByHandleW(hFile, target_path, - buf_size, VOLUME_NAME_DOS); - if (!result_length) { - err = "GetFinalPathNameByHandle"; - goto done2; - } + wchar_t *tmp; + tmp = PyMem_Realloc(target_path != buf ? target_path : NULL, + result_length * sizeof(*tmp)); + if (!tmp) { + result = PyErr_NoMemory(); + goto cleanup; + } - if (!CloseHandle(hFile)) { - err = "CloseHandle"; - goto done2; - } -done2: - Py_END_ALLOW_THREADS - if (err) { - PyMem_Free(target_path); - return win32_error_object(err, path->object); + buf_size = result_length; + target_path = tmp; } - target_path[result_length] = 0; result = PyUnicode_FromWideChar(target_path, result_length); - PyMem_Free(target_path); if (path->narrow) Py_SETREF(result, PyUnicode_EncodeFSDefault(result)); - return result; +cleanup: + if (target_path != buf) { + PyMem_Free(target_path); + } + CloseHandle(hFile); + return result; } /*[clinic input] From webhook-mailer at python.org Thu Mar 8 17:50:40 2018 From: webhook-mailer at python.org (Steve Dower) Date: Thu, 08 Mar 2018 22:50:40 -0000 Subject: [Python-checkins] bpo-33016: Fix potential use of uninitialized memory in nt._getfinalpathname (GH-6032) Message-ID: <mailman.10.1520549441.2865.python-checkins@python.org> https://github.com/python/cpython/commit/32efcd13069a89abf007373274ee1bc0909d1996 commit: 32efcd13069a89abf007373274ee1bc0909d1996 branch: 3.6 author: Steve Dower <steve.dower at microsoft.com> committer: GitHub <noreply at github.com> date: 2018-03-08T14:50:30-08:00 summary: bpo-33016: Fix potential use of uninitialized memory in nt._getfinalpathname (GH-6032) files: A Misc/NEWS.d/next/Windows/2018-03-07-01-33-33.bpo-33016.Z_Med0.rst M Modules/posixmodule.c diff --git a/Misc/NEWS.d/next/Windows/2018-03-07-01-33-33.bpo-33016.Z_Med0.rst b/Misc/NEWS.d/next/Windows/2018-03-07-01-33-33.bpo-33016.Z_Med0.rst new file mode 100644 index 000000000000..f4f78d489bf1 --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2018-03-07-01-33-33.bpo-33016.Z_Med0.rst @@ -0,0 +1 @@ +Fix potential use of uninitialized memory in nt._getfinalpathname diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 39ba030b5191..9c05acb24f0c 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -306,12 +306,6 @@ extern int lstat(const char *, struct stat *); #ifdef HAVE_PROCESS_H #include <process.h> #endif -#ifndef VOLUME_NAME_DOS -#define VOLUME_NAME_DOS 0x0 -#endif -#ifndef VOLUME_NAME_NT -#define VOLUME_NAME_NT 0x2 -#endif #ifndef IO_REPARSE_TAG_SYMLINK #define IO_REPARSE_TAG_SYMLINK (0xA000000CL) #endif @@ -3672,8 +3666,8 @@ os__getfinalpathname_impl(PyObject *module, PyObject *path) /*[clinic end generated code: output=9bd78d0e52782e75 input=71d5e89334891bf4]*/ { HANDLE hFile; - int buf_size; - wchar_t *target_path; + wchar_t buf[MAXPATHLEN], *target_path = buf; + int buf_size = Py_ARRAY_LENGTH(buf); int result_length; PyObject *result; const wchar_t *path_wchar; @@ -3682,6 +3676,7 @@ os__getfinalpathname_impl(PyObject *module, PyObject *path) if (path_wchar == NULL) return NULL; + Py_BEGIN_ALLOW_THREADS hFile = CreateFileW( path_wchar, 0, /* desired access */ @@ -3691,32 +3686,47 @@ os__getfinalpathname_impl(PyObject *module, PyObject *path) /* FILE_FLAG_BACKUP_SEMANTICS is required to open a directory */ FILE_FLAG_BACKUP_SEMANTICS, NULL); + Py_END_ALLOW_THREADS if(hFile == INVALID_HANDLE_VALUE) return win32_error_object("CreateFileW", path); /* We have a good handle to the target, use it to determine the target path name. */ - buf_size = GetFinalPathNameByHandleW(hFile, 0, 0, VOLUME_NAME_NT); + while (1) { + Py_BEGIN_ALLOW_THREADS + result_length = GetFinalPathNameByHandleW(hFile, target_path, + buf_size, VOLUME_NAME_DOS); + Py_END_ALLOW_THREADS - if(!buf_size) - return win32_error_object("GetFinalPathNameByHandle", path); + if (!result_length) { + result = win32_error_object("GetFinalPathNameByHandleW", path); + goto cleanup; + } - target_path = PyMem_New(wchar_t, buf_size+1); - if(!target_path) - return PyErr_NoMemory(); + if (result_length < buf_size) { + break; + } - result_length = GetFinalPathNameByHandleW(hFile, target_path, - buf_size, VOLUME_NAME_DOS); - if(!result_length) - return win32_error_object("GetFinalPathNamyByHandle", path); + wchar_t *tmp; + tmp = PyMem_Realloc(target_path != buf ? target_path : NULL, + result_length * sizeof(*tmp)); + if (!tmp) { + result = PyErr_NoMemory(); + goto cleanup; + } - if(!CloseHandle(hFile)) - return win32_error_object("CloseHandle", path); + buf_size = result_length; + target_path = tmp; + } - target_path[result_length] = 0; result = PyUnicode_FromWideChar(target_path, result_length); - PyMem_Free(target_path); + +cleanup: + if (target_path != buf) { + PyMem_Free(target_path); + } + CloseHandle(hFile); return result; } From webhook-mailer at python.org Thu Mar 8 21:22:04 2018 From: webhook-mailer at python.org (Xiang Zhang) Date: Fri, 09 Mar 2018 02:22:04 -0000 Subject: [Python-checkins] Clear possible exception before calling PyTuple_Pack in IMPORT_NAME (GH-6033) Message-ID: <mailman.12.1520562125.2865.python-checkins@python.org> https://github.com/python/cpython/commit/34bb88dc5bc447832db8c7ccdc173311e0685eab commit: 34bb88dc5bc447832db8c7ccdc173311e0685eab branch: 2.7 author: Xiang Zhang <angwerzx at 126.com> committer: GitHub <noreply at github.com> date: 2018-03-09T10:21:58+08:00 summary: Clear possible exception before calling PyTuple_Pack in IMPORT_NAME (GH-6033) files: M Python/ceval.c diff --git a/Python/ceval.c b/Python/ceval.c index bae158dc1402..b55b4d66880c 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -2598,6 +2598,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) TARGET(IMPORT_NAME) { + long res; w = GETITEM(names, oparg); x = PyDict_GetItemString(f->f_builtins, "__import__"); if (x == NULL) { @@ -2608,7 +2609,12 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) Py_INCREF(x); v = POP(); u = TOP(); - if (PyInt_AsLong(u) != -1 || PyErr_Occurred()) + res = PyInt_AsLong(u); + if (res != -1 || PyErr_Occurred()) { + if (res == -1) { + assert(PyErr_Occurred()); + PyErr_Clear(); + } w = PyTuple_Pack(5, w, f->f_globals, @@ -2616,6 +2622,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) Py_None : f->f_locals, v, u); + } else w = PyTuple_Pack(4, w, From solipsis at pitrou.net Fri Mar 9 04:13:10 2018 From: solipsis at pitrou.net (solipsis at pitrou.net) Date: Fri, 09 Mar 2018 09:13:10 +0000 Subject: [Python-checkins] Daily reference leaks (4243df51fe43): sum=3 Message-ID: <20180309091310.1.F0B5818E18759A84@psf.io> results for 4243df51fe43 on branch "default" -------------------------------------------- test_collections leaked [0, 7, -7] memory blocks, sum=0 test_functools leaked [0, 3, 1] memory blocks, sum=4 test_multiprocessing_forkserver leaked [-1, 2, 0] memory blocks, sum=1 test_multiprocessing_spawn leaked [-1, 1, -2] memory blocks, sum=-2 Command line was: ['./python', '-m', 'test.regrtest', '-uall', '-R', '3:3:/home/psf-users/antoine/refleaks/reflog_jjzPq', '--timeout', '7200'] From webhook-mailer at python.org Fri Mar 9 12:43:47 2018 From: webhook-mailer at python.org (Yury Selivanov) Date: Fri, 09 Mar 2018 17:43:47 -0000 Subject: [Python-checkins] bpo-32517: fix test_read_pty_output() hangs on macOS 10.13.2+ (GH-6037) Message-ID: <mailman.0.1520617429.1871.python-checkins@python.org> https://github.com/python/cpython/commit/12f74d8608c15cacd9d5786524e2be9ca36f007e commit: 12f74d8608c15cacd9d5786524e2be9ca36f007e branch: 3.6 author: Nathan Henrie <n8henrie at users.noreply.github.com> committer: Yury Selivanov <yury at magic.io> date: 2018-03-09T12:43:45-05:00 summary: bpo-32517: fix test_read_pty_output() hangs on macOS 10.13.2+ (GH-6037) test_asyncio hangs indefinitely on macOS 10.13.2+ on `read_pty_output()` using the KqueueSelector. Closing `proto.transport` (as is done in `write_pty_output()`) seems to fix it. files: A Misc/NEWS.d/next/Tests/2018-03-09-07-05-12.bpo-32517.ugc1iW.rst M Lib/test/test_asyncio/test_events.py M Misc/ACKS diff --git a/Lib/test/test_asyncio/test_events.py b/Lib/test/test_asyncio/test_events.py index 27781a2d91b3..2c4629ab4924 100644 --- a/Lib/test/test_asyncio/test_events.py +++ b/Lib/test/test_asyncio/test_events.py @@ -1533,6 +1533,7 @@ def connect(): self.assertEqual(5, proto.nbytes) os.close(slave) + proto.transport.close() self.loop.run_until_complete(proto.done) self.assertEqual( ['INITIAL', 'CONNECTED', 'EOF', 'CLOSED'], proto.state) diff --git a/Misc/ACKS b/Misc/ACKS index 9403e110675a..b2033ee9d3f3 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -614,6 +614,7 @@ Thomas Heller Malte Helmert Lance Finn Helsten Jonathan Hendry +Nathan Henrie Michael Henry James Henstridge Kasun Herath diff --git a/Misc/NEWS.d/next/Tests/2018-03-09-07-05-12.bpo-32517.ugc1iW.rst b/Misc/NEWS.d/next/Tests/2018-03-09-07-05-12.bpo-32517.ugc1iW.rst new file mode 100644 index 000000000000..43f148f06ecb --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2018-03-09-07-05-12.bpo-32517.ugc1iW.rst @@ -0,0 +1,2 @@ +Fix failing ``test_asyncio`` on macOS 10.12.2+ due to transport of +``KqueueSelector`` loop was not being closed. From webhook-mailer at python.org Fri Mar 9 13:14:19 2018 From: webhook-mailer at python.org (Miss Islington (bot)) Date: Fri, 09 Mar 2018 18:14:19 -0000 Subject: [Python-checkins] bpo-32517: fix test_read_pty_output() hangs on macOS 10.13.2+ (GH-6037) Message-ID: <mailman.1.1520619260.1871.python-checkins@python.org> https://github.com/python/cpython/commit/3c39beb65d003b209a98a79af4aaf94ba16d0d29 commit: 3c39beb65d003b209a98a79af4aaf94ba16d0d29 branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: GitHub <noreply at github.com> date: 2018-03-09T10:14:14-08:00 summary: bpo-32517: fix test_read_pty_output() hangs on macOS 10.13.2+ (GH-6037) test_asyncio hangs indefinitely on macOS 10.13.2+ on `read_pty_output()` using the KqueueSelector. Closing `proto.transport` (as is done in `write_pty_output()`) seems to fix it. (cherry picked from commit 12f74d8608c15cacd9d5786524e2be9ca36f007e) Co-authored-by: Nathan Henrie <n8henrie at users.noreply.github.com> files: A Misc/NEWS.d/next/Tests/2018-03-09-07-05-12.bpo-32517.ugc1iW.rst M Lib/test/test_asyncio/test_events.py M Misc/ACKS diff --git a/Lib/test/test_asyncio/test_events.py b/Lib/test/test_asyncio/test_events.py index 6accbdae8b3e..f8bb01bd4e72 100644 --- a/Lib/test/test_asyncio/test_events.py +++ b/Lib/test/test_asyncio/test_events.py @@ -1502,6 +1502,7 @@ def test_read_pty_output(self): self.assertEqual(5, proto.nbytes) os.close(slave) + proto.transport.close() self.loop.run_until_complete(proto.done) self.assertEqual( ['INITIAL', 'CONNECTED', 'EOF', 'CLOSED'], proto.state) diff --git a/Misc/ACKS b/Misc/ACKS index d598520e9385..b0a9136170d6 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -628,6 +628,7 @@ Thomas Heller Malte Helmert Lance Finn Helsten Jonathan Hendry +Nathan Henrie Michael Henry James Henstridge Kasun Herath diff --git a/Misc/NEWS.d/next/Tests/2018-03-09-07-05-12.bpo-32517.ugc1iW.rst b/Misc/NEWS.d/next/Tests/2018-03-09-07-05-12.bpo-32517.ugc1iW.rst new file mode 100644 index 000000000000..43f148f06ecb --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2018-03-09-07-05-12.bpo-32517.ugc1iW.rst @@ -0,0 +1,2 @@ +Fix failing ``test_asyncio`` on macOS 10.12.2+ due to transport of +``KqueueSelector`` loop was not being closed. From webhook-mailer at python.org Fri Mar 9 14:57:31 2018 From: webhook-mailer at python.org (Mariatta) Date: Fri, 09 Mar 2018 19:57:31 -0000 Subject: [Python-checkins] controlflow: Use full example for "5 through 9" (GH-5907) Message-ID: <mailman.2.1520625452.1871.python-checkins@python.org> https://github.com/python/cpython/commit/83d7062d2dc5eacfef578e072bca4747c346fdae commit: 83d7062d2dc5eacfef578e072bca4747c346fdae branch: master author: Steven M. Vascellaro <S.Vascellaro at gmail.com> committer: Mariatta <Mariatta at users.noreply.github.com> date: 2018-03-09T11:57:21-08:00 summary: controlflow: Use full example for "5 through 9" (GH-5907) Replace example result of "5 through 9" with complete list: "5, 6, 7, 8, 9". This format is more consistent with the surrounding examples. files: M Doc/tutorial/controlflow.rst diff --git a/Doc/tutorial/controlflow.rst b/Doc/tutorial/controlflow.rst index 36b093950e10..30ef159f8b36 100644 --- a/Doc/tutorial/controlflow.rst +++ b/Doc/tutorial/controlflow.rst @@ -105,7 +105,7 @@ is possible to let the range start at another number, or to specify a different increment (even negative; sometimes this is called the 'step'):: range(5, 10) - 5 through 9 + 5, 6, 7, 8, 9 range(0, 10, 3) 0, 3, 6, 9 From webhook-mailer at python.org Fri Mar 9 15:03:26 2018 From: webhook-mailer at python.org (Brett Cannon) Date: Fri, 09 Mar 2018 20:03:26 -0000 Subject: [Python-checkins] bpo-32758: Warn that ast.parse() and ast.literal_eval() can segfault the interpreter (GH-5960) Message-ID: <mailman.3.1520625807.1871.python-checkins@python.org> https://github.com/python/cpython/commit/7a7f100eb352d08938ee0f5ba59c18f56dc4a7b5 commit: 7a7f100eb352d08938ee0f5ba59c18f56dc4a7b5 branch: master author: Brett Cannon <brettcannon at users.noreply.github.com> committer: GitHub <noreply at github.com> date: 2018-03-09T12:03:22-08:00 summary: bpo-32758: Warn that ast.parse() and ast.literal_eval() can segfault the interpreter (GH-5960) files: M Doc/library/ast.rst diff --git a/Doc/library/ast.rst b/Doc/library/ast.rst index b7f610ba8b2a..a75a6afbf2d7 100644 --- a/Doc/library/ast.rst +++ b/Doc/library/ast.rst @@ -113,6 +113,11 @@ and classes for traversing abstract syntax trees: Parse the source into an AST node. Equivalent to ``compile(source, filename, mode, ast.PyCF_ONLY_AST)``. + .. warning:: + It is possible to crash the Python interpreter with a + sufficiently large/complex string due to stack depth limitations + in Python's AST compiler. + .. function:: literal_eval(node_or_string) @@ -126,6 +131,11 @@ and classes for traversing abstract syntax trees: capable of evaluating arbitrarily complex expressions, for example involving operators or indexing. + .. warning:: + It is possible to crash the Python interpreter with a + sufficiently large/complex string due to stack depth limitations + in Python's AST compiler. + .. versionchanged:: 3.2 Now allows bytes and set literals. From webhook-mailer at python.org Fri Mar 9 15:11:33 2018 From: webhook-mailer at python.org (Miss Islington (bot)) Date: Fri, 09 Mar 2018 20:11:33 -0000 Subject: [Python-checkins] controlflow: Use full example for "5 through 9" (GH-5907) Message-ID: <mailman.4.1520626294.1871.python-checkins@python.org> https://github.com/python/cpython/commit/3f7d0b69f2175ff74157673148b2e45b7a2498b6 commit: 3f7d0b69f2175ff74157673148b2e45b7a2498b6 branch: 3.6 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: GitHub <noreply at github.com> date: 2018-03-09T12:11:30-08:00 summary: controlflow: Use full example for "5 through 9" (GH-5907) Replace example result of "5 through 9" with complete list: "5, 6, 7, 8, 9". This format is more consistent with the surrounding examples. (cherry picked from commit 83d7062d2dc5eacfef578e072bca4747c346fdae) Co-authored-by: Steven M. Vascellaro <S.Vascellaro at gmail.com> files: M Doc/tutorial/controlflow.rst diff --git a/Doc/tutorial/controlflow.rst b/Doc/tutorial/controlflow.rst index 36b093950e10..30ef159f8b36 100644 --- a/Doc/tutorial/controlflow.rst +++ b/Doc/tutorial/controlflow.rst @@ -105,7 +105,7 @@ is possible to let the range start at another number, or to specify a different increment (even negative; sometimes this is called the 'step'):: range(5, 10) - 5 through 9 + 5, 6, 7, 8, 9 range(0, 10, 3) 0, 3, 6, 9 From webhook-mailer at python.org Fri Mar 9 15:35:24 2018 From: webhook-mailer at python.org (Brett Cannon) Date: Fri, 09 Mar 2018 20:35:24 -0000 Subject: [Python-checkins] bpo-32758: Warn that ast.parse() and ast.literal_eval() can segfault the interpreter (GH-5960) (GH-6042) Message-ID: <mailman.5.1520627726.1871.python-checkins@python.org> https://github.com/python/cpython/commit/b316c44b0105d11a80ff971636143735f3655bbf commit: b316c44b0105d11a80ff971636143735f3655bbf branch: 3.6 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Brett Cannon <brettcannon at users.noreply.github.com> date: 2018-03-09T12:35:14-08:00 summary: bpo-32758: Warn that ast.parse() and ast.literal_eval() can segfault the interpreter (GH-5960) (GH-6042) (cherry picked from commit 7a7f100eb352d08938ee0f5ba59c18f56dc4a7b5) Co-authored-by: Brett Cannon <brettcannon at users.noreply.github.com> files: M Doc/library/ast.rst diff --git a/Doc/library/ast.rst b/Doc/library/ast.rst index 8d4ae2cc616f..6376f5fe4410 100644 --- a/Doc/library/ast.rst +++ b/Doc/library/ast.rst @@ -113,6 +113,11 @@ and classes for traversing abstract syntax trees: Parse the source into an AST node. Equivalent to ``compile(source, filename, mode, ast.PyCF_ONLY_AST)``. + .. warning:: + It is possible to crash the Python interpreter with a + sufficiently large/complex string due to stack depth limitations + in Python's AST compiler. + .. function:: literal_eval(node_or_string) @@ -126,6 +131,11 @@ and classes for traversing abstract syntax trees: capable of evaluating arbitrarily complex expressions, for example involving operators or indexing. + .. warning:: + It is possible to crash the Python interpreter with a + sufficiently large/complex string due to stack depth limitations + in Python's AST compiler. + .. versionchanged:: 3.2 Now allows bytes and set literals. From webhook-mailer at python.org Fri Mar 9 15:35:45 2018 From: webhook-mailer at python.org (Brett Cannon) Date: Fri, 09 Mar 2018 20:35:45 -0000 Subject: [Python-checkins] bpo-32758: Warn that ast.parse() and ast.literal_eval() can segfault the interpreter (GH-5960) (GH-6041) Message-ID: <mailman.6.1520627747.1871.python-checkins@python.org> https://github.com/python/cpython/commit/f2fffd41b42d88fe36b483852ae33d5a415b7082 commit: f2fffd41b42d88fe36b483852ae33d5a415b7082 branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Brett Cannon <brettcannon at users.noreply.github.com> date: 2018-03-09T12:35:42-08:00 summary: bpo-32758: Warn that ast.parse() and ast.literal_eval() can segfault the interpreter (GH-5960) (GH-6041) (cherry picked from commit 7a7f100eb352d08938ee0f5ba59c18f56dc4a7b5) Co-authored-by: Brett Cannon <brettcannon at users.noreply.github.com> files: M Doc/library/ast.rst diff --git a/Doc/library/ast.rst b/Doc/library/ast.rst index b7f610ba8b2a..a75a6afbf2d7 100644 --- a/Doc/library/ast.rst +++ b/Doc/library/ast.rst @@ -113,6 +113,11 @@ and classes for traversing abstract syntax trees: Parse the source into an AST node. Equivalent to ``compile(source, filename, mode, ast.PyCF_ONLY_AST)``. + .. warning:: + It is possible to crash the Python interpreter with a + sufficiently large/complex string due to stack depth limitations + in Python's AST compiler. + .. function:: literal_eval(node_or_string) @@ -126,6 +131,11 @@ and classes for traversing abstract syntax trees: capable of evaluating arbitrarily complex expressions, for example involving operators or indexing. + .. warning:: + It is possible to crash the Python interpreter with a + sufficiently large/complex string due to stack depth limitations + in Python's AST compiler. + .. versionchanged:: 3.2 Now allows bytes and set literals. From webhook-mailer at python.org Fri Mar 9 16:13:36 2018 From: webhook-mailer at python.org (Brett Cannon) Date: Fri, 09 Mar 2018 21:13:36 -0000 Subject: [Python-checkins] Warn that compile() can crash when compiling to an AST object (GH-6043) Message-ID: <mailman.7.1520630017.1871.python-checkins@python.org> https://github.com/python/cpython/commit/f7a6ff6fcab32a53f262ba3f8a072c27afc330d7 commit: f7a6ff6fcab32a53f262ba3f8a072c27afc330d7 branch: master author: Brett Cannon <brettcannon at users.noreply.github.com> committer: GitHub <noreply at github.com> date: 2018-03-09T13:13:32-08:00 summary: Warn that compile() can crash when compiling to an AST object (GH-6043) files: M Doc/library/functions.rst diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index bfb813cf3906..3ddd280f77eb 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -274,6 +274,12 @@ are always available. They are listed here in alphabetical order. character. This is to facilitate detection of incomplete and complete statements in the :mod:`code` module. + .. warning:: + + It is possible to crash the Python interpreter with a + sufficiently large/complex string when compiling to an AST + object due to stack depth limitations in Python's AST compiler. + .. versionchanged:: 3.2 Allowed use of Windows and Mac newlines. Also input in ``'exec'`` mode does not have to end in a newline anymore. Added the *optimize* parameter. From webhook-mailer at python.org Fri Mar 9 16:40:29 2018 From: webhook-mailer at python.org (Brett Cannon) Date: Fri, 09 Mar 2018 21:40:29 -0000 Subject: [Python-checkins] Warn that compile() can crash when compiling to an AST object (GH-6043) (GH-6045) Message-ID: <mailman.8.1520631630.1871.python-checkins@python.org> https://github.com/python/cpython/commit/fc5e0956fe492551c658ebe325ba77c0aa4f2adb commit: fc5e0956fe492551c658ebe325ba77c0aa4f2adb branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Brett Cannon <brettcannon at users.noreply.github.com> date: 2018-03-09T13:40:26-08:00 summary: Warn that compile() can crash when compiling to an AST object (GH-6043) (GH-6045) (cherry picked from commit f7a6ff6fcab32a53f262ba3f8a072c27afc330d7) Co-authored-by: Brett Cannon <brettcannon at users.noreply.github.com> files: M Doc/library/functions.rst diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index bfb813cf3906..3ddd280f77eb 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -274,6 +274,12 @@ are always available. They are listed here in alphabetical order. character. This is to facilitate detection of incomplete and complete statements in the :mod:`code` module. + .. warning:: + + It is possible to crash the Python interpreter with a + sufficiently large/complex string when compiling to an AST + object due to stack depth limitations in Python's AST compiler. + .. versionchanged:: 3.2 Allowed use of Windows and Mac newlines. Also input in ``'exec'`` mode does not have to end in a newline anymore. Added the *optimize* parameter. From webhook-mailer at python.org Fri Mar 9 16:40:41 2018 From: webhook-mailer at python.org (Brett Cannon) Date: Fri, 09 Mar 2018 21:40:41 -0000 Subject: [Python-checkins] Warn that compile() can crash when compiling to an AST object (GH-6043) (GH-6046) Message-ID: <mailman.9.1520631642.1871.python-checkins@python.org> https://github.com/python/cpython/commit/19b42fe97cc15ff123f7ccb61a26fed817e850f7 commit: 19b42fe97cc15ff123f7ccb61a26fed817e850f7 branch: 3.6 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Brett Cannon <brettcannon at users.noreply.github.com> date: 2018-03-09T13:40:39-08:00 summary: Warn that compile() can crash when compiling to an AST object (GH-6043) (GH-6046) (cherry picked from commit f7a6ff6fcab32a53f262ba3f8a072c27afc330d7) Co-authored-by: Brett Cannon <brettcannon at users.noreply.github.com> files: M Doc/library/functions.rst diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index 3b479bd64294..bc4203081ed0 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -260,6 +260,12 @@ are always available. They are listed here in alphabetical order. character. This is to facilitate detection of incomplete and complete statements in the :mod:`code` module. + .. warning:: + + It is possible to crash the Python interpreter with a + sufficiently large/complex string when compiling to an AST + object due to stack depth limitations in Python's AST compiler. + .. versionchanged:: 3.2 Allowed use of Windows and Mac newlines. Also input in ``'exec'`` mode does not have to end in a newline anymore. Added the *optimize* parameter. From webhook-mailer at python.org Fri Mar 9 18:58:43 2018 From: webhook-mailer at python.org (Brett Cannon) Date: Fri, 09 Mar 2018 23:58:43 -0000 Subject: [Python-checkins] Warn that dbm.dumb.open() can crash Python (GH-6047) Message-ID: <mailman.10.1520639924.1871.python-checkins@python.org> https://github.com/python/cpython/commit/10485ebd40669d3e17ab4f477c8c898543bcccd1 commit: 10485ebd40669d3e17ab4f477c8c898543bcccd1 branch: master author: Brett Cannon <brettcannon at users.noreply.github.com> committer: GitHub <noreply at github.com> date: 2018-03-09T15:58:40-08:00 summary: Warn that dbm.dumb.open() can crash Python (GH-6047) files: M Doc/library/dbm.rst diff --git a/Doc/library/dbm.rst b/Doc/library/dbm.rst index 1abc36c04a74..ab45cac830e2 100644 --- a/Doc/library/dbm.rst +++ b/Doc/library/dbm.rst @@ -361,6 +361,11 @@ The module defines the following: database has to be created. It defaults to octal ``0o666`` (and will be modified by the prevailing umask). + .. warning:: + It is possible to crash the Python interpreter when loading a database + with a sufficiently large/complex entry due to stack depth limitations in + Python's AST compiler. + .. versionchanged:: 3.5 :func:`.open` always creates a new database when the flag has the value ``'n'``. From webhook-mailer at python.org Fri Mar 9 19:12:57 2018 From: webhook-mailer at python.org (Brett Cannon) Date: Sat, 10 Mar 2018 00:12:57 -0000 Subject: [Python-checkins] Warn that dbm.dumb.open() can crash Python (GH-6047) (GH-6048) Message-ID: <mailman.11.1520640778.1871.python-checkins@python.org> https://github.com/python/cpython/commit/6826589415408e68e7872bfde30abbdff17772b2 commit: 6826589415408e68e7872bfde30abbdff17772b2 branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Brett Cannon <brettcannon at users.noreply.github.com> date: 2018-03-09T16:12:54-08:00 summary: Warn that dbm.dumb.open() can crash Python (GH-6047) (GH-6048) (cherry picked from commit 10485ebd40669d3e17ab4f477c8c898543bcccd1) Co-authored-by: Brett Cannon <brettcannon at users.noreply.github.com> files: M Doc/library/dbm.rst diff --git a/Doc/library/dbm.rst b/Doc/library/dbm.rst index 32e80b2cf6ed..0150f5d5c6e8 100644 --- a/Doc/library/dbm.rst +++ b/Doc/library/dbm.rst @@ -347,6 +347,11 @@ The module defines the following: database has to be created. It defaults to octal ``0o666`` (and will be modified by the prevailing umask). + .. warning:: + It is possible to crash the Python interpreter when loading a database + with a sufficiently large/complex entry due to stack depth limitations in + Python's AST compiler. + .. versionchanged:: 3.5 :func:`.open` always creates a new database when the flag has the value ``'n'``. From webhook-mailer at python.org Sat Mar 10 03:44:20 2018 From: webhook-mailer at python.org (Nick Coghlan) Date: Sat, 10 Mar 2018 08:44:20 -0000 Subject: [Python-checkins] bpo-26701: Add documentation for __trunc__ (GH-6022) Message-ID: <mailman.12.1520671461.1871.python-checkins@python.org> https://github.com/python/cpython/commit/308eab979d153f1ab934383dc08bc4546ced8b6c commit: 308eab979d153f1ab934383dc08bc4546ced8b6c branch: master author: Eric Appelt <eric.appelt at gmail.com> committer: Nick Coghlan <ncoghlan at gmail.com> date: 2018-03-10T18:44:12+10:00 summary: bpo-26701: Add documentation for __trunc__ (GH-6022) `int` fails back to `__trunc__` is `__int__` isn't defined, so cover that in the docs. files: M Doc/library/functions.rst M Doc/library/math.rst M Doc/reference/datamodel.rst diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index 3ddd280f77eb..c3b638572338 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -731,8 +731,11 @@ are always available. They are listed here in alphabetical order. Return an integer object constructed from a number or string *x*, or return ``0`` if no arguments are given. If *x* is a number, return - :meth:`x.__int__() <object.__int__>`. For floating point numbers, this - truncates towards zero. + :meth:`x.__int__() <object.__int__>`. If *x* defines + :meth:`x.__trunc__() <object.__trunc__>` but not + :meth:`x.__int__() <object.__int__>`, then return + if :meth:`x.__trunc__() <object.__trunc__>`. For floating point numbers, + this truncates towards zero. If *x* is not a number or if *base* is given, then *x* must be a string, :class:`bytes`, or :class:`bytearray` instance representing an :ref:`integer diff --git a/Doc/library/math.rst b/Doc/library/math.rst index 55eb41b86f5d..33aec573a1f2 100644 --- a/Doc/library/math.rst +++ b/Doc/library/math.rst @@ -203,7 +203,7 @@ Number-theoretic and representation functions Return the :class:`~numbers.Real` value *x* truncated to an :class:`~numbers.Integral` (usually an integer). Delegates to - ``x.__trunc__()``. + :meth:`x.__trunc__() <object.__trunc__>`. Note that :func:`frexp` and :func:`modf` have a different call/return pattern diff --git a/Doc/reference/datamodel.rst b/Doc/reference/datamodel.rst index 8b127a0b7e81..26ad7b8c050c 100644 --- a/Doc/reference/datamodel.rst +++ b/Doc/reference/datamodel.rst @@ -2377,6 +2377,15 @@ left undefined. of the appropriate type. +.. method:: object.__trunc__(self) + + Called to implement :meth:`math.trunc`. Should return the value of the + object truncated to a :class:`numbers.Integral` (typically an + :class:`int`). If a class defines :meth:`__trunc__` but not + :meth:`__int__`, then :meth:`__trunc__` is called to implement the + built-in function :func:`int`. + + .. method:: object.__index__(self) Called to implement :func:`operator.index`, and whenever Python needs to From solipsis at pitrou.net Sat Mar 10 04:08:34 2018 From: solipsis at pitrou.net (solipsis at pitrou.net) Date: Sat, 10 Mar 2018 09:08:34 +0000 Subject: [Python-checkins] Daily reference leaks (4243df51fe43): sum=11 Message-ID: <20180310090834.1.41C0C46E65B1583E@psf.io> results for 4243df51fe43 on branch "default" -------------------------------------------- test_collections leaked [0, 0, 7] memory blocks, sum=7 test_functools leaked [0, 3, 1] memory blocks, sum=4 Command line was: ['./python', '-m', 'test.regrtest', '-uall', '-R', '3:3:/home/psf-users/antoine/refleaks/reflogohDkjm', '--timeout', '7200'] From webhook-mailer at python.org Sat Mar 10 10:10:40 2018 From: webhook-mailer at python.org (Nick Coghlan) Date: Sat, 10 Mar 2018 15:10:40 -0000 Subject: [Python-checkins] bpo-26701: Add documentation for __trunc__ (GH-6049) Message-ID: <mailman.13.1520694641.1871.python-checkins@python.org> https://github.com/python/cpython/commit/f34e0d60e27acff3f9604ec63e9de36878c3743a commit: f34e0d60e27acff3f9604ec63e9de36878c3743a branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Nick Coghlan <ncoghlan at gmail.com> date: 2018-03-11T01:10:32+10:00 summary: bpo-26701: Add documentation for __trunc__ (GH-6049) `int` fails back to `__trunc__` is `__int__` isn't defined, so cover that in the docs. (cherry picked from commit 308eab979d153f1ab934383dc08bc4546ced8b6c) Co-authored-by: Eric Appelt <eric.appelt at gmail.com> files: M Doc/library/functions.rst M Doc/library/math.rst M Doc/reference/datamodel.rst diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index 3ddd280f77eb..c3b638572338 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -731,8 +731,11 @@ are always available. They are listed here in alphabetical order. Return an integer object constructed from a number or string *x*, or return ``0`` if no arguments are given. If *x* is a number, return - :meth:`x.__int__() <object.__int__>`. For floating point numbers, this - truncates towards zero. + :meth:`x.__int__() <object.__int__>`. If *x* defines + :meth:`x.__trunc__() <object.__trunc__>` but not + :meth:`x.__int__() <object.__int__>`, then return + if :meth:`x.__trunc__() <object.__trunc__>`. For floating point numbers, + this truncates towards zero. If *x* is not a number or if *base* is given, then *x* must be a string, :class:`bytes`, or :class:`bytearray` instance representing an :ref:`integer diff --git a/Doc/library/math.rst b/Doc/library/math.rst index 55eb41b86f5d..33aec573a1f2 100644 --- a/Doc/library/math.rst +++ b/Doc/library/math.rst @@ -203,7 +203,7 @@ Number-theoretic and representation functions Return the :class:`~numbers.Real` value *x* truncated to an :class:`~numbers.Integral` (usually an integer). Delegates to - ``x.__trunc__()``. + :meth:`x.__trunc__() <object.__trunc__>`. Note that :func:`frexp` and :func:`modf` have a different call/return pattern diff --git a/Doc/reference/datamodel.rst b/Doc/reference/datamodel.rst index 8b127a0b7e81..26ad7b8c050c 100644 --- a/Doc/reference/datamodel.rst +++ b/Doc/reference/datamodel.rst @@ -2377,6 +2377,15 @@ left undefined. of the appropriate type. +.. method:: object.__trunc__(self) + + Called to implement :meth:`math.trunc`. Should return the value of the + object truncated to a :class:`numbers.Integral` (typically an + :class:`int`). If a class defines :meth:`__trunc__` but not + :meth:`__int__`, then :meth:`__trunc__` is called to implement the + built-in function :func:`int`. + + .. method:: object.__index__(self) Called to implement :func:`operator.index`, and whenever Python needs to From webhook-mailer at python.org Sat Mar 10 10:10:48 2018 From: webhook-mailer at python.org (Nick Coghlan) Date: Sat, 10 Mar 2018 15:10:48 -0000 Subject: [Python-checkins] bpo-26701: Add documentation for __trunc__ (GH-6050) Message-ID: <mailman.14.1520694650.1871.python-checkins@python.org> https://github.com/python/cpython/commit/1028ca4f04c14cf40a8f3e7951623a96ec0143c2 commit: 1028ca4f04c14cf40a8f3e7951623a96ec0143c2 branch: 3.6 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Nick Coghlan <ncoghlan at gmail.com> date: 2018-03-11T01:10:45+10:00 summary: bpo-26701: Add documentation for __trunc__ (GH-6050) `int` fails back to `__trunc__` is `__int__` isn't defined, so cover that in the docs. (cherry picked from commit 308eab979d153f1ab934383dc08bc4546ced8b6c) Co-authored-by: Eric Appelt <eric.appelt at gmail.com> files: M Doc/library/functions.rst M Doc/library/math.rst M Doc/reference/datamodel.rst diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index bc4203081ed0..9cb6b0e1b5ca 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -717,8 +717,11 @@ are always available. They are listed here in alphabetical order. Return an integer object constructed from a number or string *x*, or return ``0`` if no arguments are given. If *x* is a number, return - :meth:`x.__int__() <object.__int__>`. For floating point numbers, this - truncates towards zero. + :meth:`x.__int__() <object.__int__>`. If *x* defines + :meth:`x.__trunc__() <object.__trunc__>` but not + :meth:`x.__int__() <object.__int__>`, then return + if :meth:`x.__trunc__() <object.__trunc__>`. For floating point numbers, + this truncates towards zero. If *x* is not a number or if *base* is given, then *x* must be a string, :class:`bytes`, or :class:`bytearray` instance representing an :ref:`integer diff --git a/Doc/library/math.rst b/Doc/library/math.rst index da2b8cc58627..0f36a3fa99eb 100644 --- a/Doc/library/math.rst +++ b/Doc/library/math.rst @@ -179,7 +179,7 @@ Number-theoretic and representation functions Return the :class:`~numbers.Real` value *x* truncated to an :class:`~numbers.Integral` (usually an integer). Delegates to - ``x.__trunc__()``. + :meth:`x.__trunc__() <object.__trunc__>`. Note that :func:`frexp` and :func:`modf` have a different call/return pattern diff --git a/Doc/reference/datamodel.rst b/Doc/reference/datamodel.rst index c95f4a970930..90f3a2aa0dcd 100644 --- a/Doc/reference/datamodel.rst +++ b/Doc/reference/datamodel.rst @@ -2312,6 +2312,15 @@ left undefined. of the appropriate type. +.. method:: object.__trunc__(self) + + Called to implement :meth:`math.trunc`. Should return the value of the + object truncated to a :class:`numbers.Integral` (typically an + :class:`int`). If a class defines :meth:`__trunc__` but not + :meth:`__int__`, then :meth:`__trunc__` is called to implement the + built-in function :func:`int`. + + .. method:: object.__index__(self) Called to implement :func:`operator.index`, and whenever Python needs to From webhook-mailer at python.org Sat Mar 10 10:18:35 2018 From: webhook-mailer at python.org (Serhiy Storchaka) Date: Sat, 10 Mar 2018 15:18:35 -0000 Subject: [Python-checkins] bpo-26701: Improve documentation for the rounding special methods. (#6054) Message-ID: <mailman.15.1520695117.1871.python-checkins@python.org> https://github.com/python/cpython/commit/496431ffb6c29719332bf2af773349e8dd85e45a commit: 496431ffb6c29719332bf2af773349e8dd85e45a branch: master author: Serhiy Storchaka <storchaka at gmail.com> committer: GitHub <noreply at github.com> date: 2018-03-10T17:18:32+02:00 summary: bpo-26701: Improve documentation for the rounding special methods. (#6054) files: M Doc/reference/datamodel.rst diff --git a/Doc/reference/datamodel.rst b/Doc/reference/datamodel.rst index 26ad7b8c050c..1e93ef4594a9 100644 --- a/Doc/reference/datamodel.rst +++ b/Doc/reference/datamodel.rst @@ -2364,28 +2364,17 @@ left undefined. .. method:: object.__complex__(self) object.__int__(self) object.__float__(self) - object.__round__(self, [,n]) .. index:: builtin: complex builtin: int builtin: float - builtin: round Called to implement the built-in functions :func:`complex`, - :func:`int`, :func:`float` and :func:`round`. Should return a value + :func:`int` and :func:`float`. Should return a value of the appropriate type. -.. method:: object.__trunc__(self) - - Called to implement :meth:`math.trunc`. Should return the value of the - object truncated to a :class:`numbers.Integral` (typically an - :class:`int`). If a class defines :meth:`__trunc__` but not - :meth:`__int__`, then :meth:`__trunc__` is called to implement the - built-in function :func:`int`. - - .. method:: object.__index__(self) Called to implement :func:`operator.index`, and whenever Python needs to @@ -2401,6 +2390,23 @@ left undefined. the same value. +.. method:: object.__round__(self, [,ndigits]) + object.__trunc__(self) + object.__floor__(self) + object.__ceil__(self) + + .. index:: builtin: round + + Called to implement the built-in function :func:`round` and :mod:`math` + functions :func:`~math.trunc`, :func:`~math.floor` and :func:`~math.ceil`. + Unless *ndigits* is passed to :meth:`!__round__` all these methods should + return the value of the object truncated to an :class:`~numbers.Integral` + (typically an :class:`int`). + + If :meth:`__int__` is not defined then the built-in function :func:`int` + falls back to :meth:`__trunc__`. + + .. _context-managers: With Statement Context Managers From webhook-mailer at python.org Sat Mar 10 10:48:38 2018 From: webhook-mailer at python.org (Andrew Svetlov) Date: Sat, 10 Mar 2018 15:48:38 -0000 Subject: [Python-checkins] bpo-33037: Skip sending/receiving after SSL transport closing (GH-6044) Message-ID: <mailman.16.1520696919.1871.python-checkins@python.org> https://github.com/python/cpython/commit/5e80a71ab67045fecec46573a1892e240b569ace commit: 5e80a71ab67045fecec46573a1892e240b569ace branch: master author: Andrew Svetlov <andrew.svetlov at gmail.com> committer: GitHub <noreply at github.com> date: 2018-03-10T17:48:35+02:00 summary: bpo-33037: Skip sending/receiving after SSL transport closing (GH-6044) * Skip write()/data_received() if sslpipe is destroyed files: A Misc/NEWS.d/next/Library/2018-03-09-23-07-07.bpo-33037.nAJ3at.rst M Lib/asyncio/sslproto.py M Lib/test/test_asyncio/test_sslproto.py diff --git a/Lib/asyncio/sslproto.py b/Lib/asyncio/sslproto.py index 863b54313cc6..2bbf134c0f7e 100644 --- a/Lib/asyncio/sslproto.py +++ b/Lib/asyncio/sslproto.py @@ -504,6 +504,10 @@ def data_received(self, data): The argument is a bytes object. """ + if self._sslpipe is None: + # transport closing, sslpipe is destroyed + return + try: ssldata, appdata = self._sslpipe.feed_ssldata(data) except ssl.SSLError as e: @@ -636,7 +640,7 @@ def _on_handshake_complete(self, handshake_exc): def _process_write_backlog(self): # Try to make progress on the write backlog. - if self._transport is None: + if self._transport is None or self._sslpipe is None: return try: diff --git a/Lib/test/test_asyncio/test_sslproto.py b/Lib/test/test_asyncio/test_sslproto.py index c4c30bedb4ec..7b27f4cfa322 100644 --- a/Lib/test/test_asyncio/test_sslproto.py +++ b/Lib/test/test_asyncio/test_sslproto.py @@ -26,16 +26,17 @@ def setUp(self): self.loop = asyncio.new_event_loop() self.set_event_loop(self.loop) - def ssl_protocol(self, waiter=None): + def ssl_protocol(self, *, waiter=None, proto=None): sslcontext = test_utils.dummy_ssl_context() - app_proto = asyncio.Protocol() - proto = sslproto.SSLProtocol(self.loop, app_proto, sslcontext, waiter, - ssl_handshake_timeout=0.1) - self.assertIs(proto._app_transport.get_protocol(), app_proto) - self.addCleanup(proto._app_transport.close) - return proto - - def connection_made(self, ssl_proto, do_handshake=None): + if proto is None: # app protocol + proto = asyncio.Protocol() + ssl_proto = sslproto.SSLProtocol(self.loop, proto, sslcontext, waiter, + ssl_handshake_timeout=0.1) + self.assertIs(ssl_proto._app_transport.get_protocol(), proto) + self.addCleanup(ssl_proto._app_transport.close) + return ssl_proto + + def connection_made(self, ssl_proto, *, do_handshake=None): transport = mock.Mock() sslpipe = mock.Mock() sslpipe.shutdown.return_value = b'' @@ -53,7 +54,7 @@ def test_cancel_handshake(self): # Python issue #23197: cancelling a handshake must not raise an # exception or log an error, even if the handshake failed waiter = asyncio.Future(loop=self.loop) - ssl_proto = self.ssl_protocol(waiter) + ssl_proto = self.ssl_protocol(waiter=waiter) handshake_fut = asyncio.Future(loop=self.loop) def do_handshake(callback): @@ -63,7 +64,7 @@ def do_handshake(callback): return [] waiter.cancel() - self.connection_made(ssl_proto, do_handshake) + self.connection_made(ssl_proto, do_handshake=do_handshake) with test_utils.disable_logger(): self.loop.run_until_complete(handshake_fut) @@ -96,7 +97,7 @@ def test_handshake_timeout_negative(self): def test_eof_received_waiter(self): waiter = asyncio.Future(loop=self.loop) - ssl_proto = self.ssl_protocol(waiter) + ssl_proto = self.ssl_protocol(waiter=waiter) self.connection_made(ssl_proto) ssl_proto.eof_received() test_utils.run_briefly(self.loop) @@ -107,7 +108,7 @@ def test_fatal_error_no_name_error(self): # _fatal_error() generates a NameError if sslproto.py # does not import base_events. waiter = asyncio.Future(loop=self.loop) - ssl_proto = self.ssl_protocol(waiter) + ssl_proto = self.ssl_protocol(waiter=waiter) # Temporarily turn off error logging so as not to spoil test output. log_level = log.logger.getEffectiveLevel() log.logger.setLevel(logging.FATAL) @@ -121,7 +122,7 @@ def test_connection_lost(self): # From issue #472. # yield from waiter hang if lost_connection was called. waiter = asyncio.Future(loop=self.loop) - ssl_proto = self.ssl_protocol(waiter) + ssl_proto = self.ssl_protocol(waiter=waiter) self.connection_made(ssl_proto) ssl_proto.connection_lost(ConnectionAbortedError) test_utils.run_briefly(self.loop) @@ -130,10 +131,7 @@ def test_connection_lost(self): def test_close_during_handshake(self): # bpo-29743 Closing transport during handshake process leaks socket waiter = asyncio.Future(loop=self.loop) - ssl_proto = self.ssl_protocol(waiter) - - def do_handshake(callback): - return [] + ssl_proto = self.ssl_protocol(waiter=waiter) transport = self.connection_made(ssl_proto) test_utils.run_briefly(self.loop) @@ -143,7 +141,7 @@ def do_handshake(callback): def test_get_extra_info_on_closed_connection(self): waiter = asyncio.Future(loop=self.loop) - ssl_proto = self.ssl_protocol(waiter) + ssl_proto = self.ssl_protocol(waiter=waiter) self.assertIsNone(ssl_proto._get_extra_info('socket')) default = object() self.assertIs(ssl_proto._get_extra_info('socket', default), default) @@ -154,12 +152,31 @@ def test_get_extra_info_on_closed_connection(self): def test_set_new_app_protocol(self): waiter = asyncio.Future(loop=self.loop) - ssl_proto = self.ssl_protocol(waiter) + ssl_proto = self.ssl_protocol(waiter=waiter) new_app_proto = asyncio.Protocol() ssl_proto._app_transport.set_protocol(new_app_proto) self.assertIs(ssl_proto._app_transport.get_protocol(), new_app_proto) self.assertIs(ssl_proto._app_protocol, new_app_proto) + def test_data_received_after_closing(self): + ssl_proto = self.ssl_protocol() + self.connection_made(ssl_proto) + transp = ssl_proto._app_transport + + transp.close() + + # should not raise + self.assertIsNone(ssl_proto.data_received(b'data')) + + def test_write_after_closing(self): + ssl_proto = self.ssl_protocol() + self.connection_made(ssl_proto) + transp = ssl_proto._app_transport + transp.close() + + # should not raise + self.assertIsNone(transp.write(b'data')) + ############################################################################## # Start TLS Tests diff --git a/Misc/NEWS.d/next/Library/2018-03-09-23-07-07.bpo-33037.nAJ3at.rst b/Misc/NEWS.d/next/Library/2018-03-09-23-07-07.bpo-33037.nAJ3at.rst new file mode 100644 index 000000000000..2732eeb4534b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-03-09-23-07-07.bpo-33037.nAJ3at.rst @@ -0,0 +1 @@ +Skip sending/receiving data after SSL transport closing. From webhook-mailer at python.org Sat Mar 10 10:49:27 2018 From: webhook-mailer at python.org (Miss Islington (bot)) Date: Sat, 10 Mar 2018 15:49:27 -0000 Subject: [Python-checkins] bpo-26701: Improve documentation for the rounding special methods. (GH-6054) Message-ID: <mailman.17.1520696968.1871.python-checkins@python.org> https://github.com/python/cpython/commit/89090789debb9d76892af566277cb71740808945 commit: 89090789debb9d76892af566277cb71740808945 branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: GitHub <noreply at github.com> date: 2018-03-10T07:49:24-08:00 summary: bpo-26701: Improve documentation for the rounding special methods. (GH-6054) (cherry picked from commit 496431ffb6c29719332bf2af773349e8dd85e45a) Co-authored-by: Serhiy Storchaka <storchaka at gmail.com> files: M Doc/reference/datamodel.rst diff --git a/Doc/reference/datamodel.rst b/Doc/reference/datamodel.rst index 26ad7b8c050c..1e93ef4594a9 100644 --- a/Doc/reference/datamodel.rst +++ b/Doc/reference/datamodel.rst @@ -2364,28 +2364,17 @@ left undefined. .. method:: object.__complex__(self) object.__int__(self) object.__float__(self) - object.__round__(self, [,n]) .. index:: builtin: complex builtin: int builtin: float - builtin: round Called to implement the built-in functions :func:`complex`, - :func:`int`, :func:`float` and :func:`round`. Should return a value + :func:`int` and :func:`float`. Should return a value of the appropriate type. -.. method:: object.__trunc__(self) - - Called to implement :meth:`math.trunc`. Should return the value of the - object truncated to a :class:`numbers.Integral` (typically an - :class:`int`). If a class defines :meth:`__trunc__` but not - :meth:`__int__`, then :meth:`__trunc__` is called to implement the - built-in function :func:`int`. - - .. method:: object.__index__(self) Called to implement :func:`operator.index`, and whenever Python needs to @@ -2401,6 +2390,23 @@ left undefined. the same value. +.. method:: object.__round__(self, [,ndigits]) + object.__trunc__(self) + object.__floor__(self) + object.__ceil__(self) + + .. index:: builtin: round + + Called to implement the built-in function :func:`round` and :mod:`math` + functions :func:`~math.trunc`, :func:`~math.floor` and :func:`~math.ceil`. + Unless *ndigits* is passed to :meth:`!__round__` all these methods should + return the value of the object truncated to an :class:`~numbers.Integral` + (typically an :class:`int`). + + If :meth:`__int__` is not defined then the built-in function :func:`int` + falls back to :meth:`__trunc__`. + + .. _context-managers: With Statement Context Managers From webhook-mailer at python.org Sat Mar 10 10:55:15 2018 From: webhook-mailer at python.org (Miss Islington (bot)) Date: Sat, 10 Mar 2018 15:55:15 -0000 Subject: [Python-checkins] bpo-26701: Improve documentation for the rounding special methods. (GH-6054) Message-ID: <mailman.18.1520697316.1871.python-checkins@python.org> https://github.com/python/cpython/commit/de8567e38c44b1509f0b906aec54437256848f14 commit: de8567e38c44b1509f0b906aec54437256848f14 branch: 3.6 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: GitHub <noreply at github.com> date: 2018-03-10T07:55:13-08:00 summary: bpo-26701: Improve documentation for the rounding special methods. (GH-6054) (cherry picked from commit 496431ffb6c29719332bf2af773349e8dd85e45a) Co-authored-by: Serhiy Storchaka <storchaka at gmail.com> files: M Doc/reference/datamodel.rst diff --git a/Doc/reference/datamodel.rst b/Doc/reference/datamodel.rst index 90f3a2aa0dcd..c08986f6c0a5 100644 --- a/Doc/reference/datamodel.rst +++ b/Doc/reference/datamodel.rst @@ -2299,28 +2299,17 @@ left undefined. .. method:: object.__complex__(self) object.__int__(self) object.__float__(self) - object.__round__(self, [,n]) .. index:: builtin: complex builtin: int builtin: float - builtin: round Called to implement the built-in functions :func:`complex`, - :func:`int`, :func:`float` and :func:`round`. Should return a value + :func:`int` and :func:`float`. Should return a value of the appropriate type. -.. method:: object.__trunc__(self) - - Called to implement :meth:`math.trunc`. Should return the value of the - object truncated to a :class:`numbers.Integral` (typically an - :class:`int`). If a class defines :meth:`__trunc__` but not - :meth:`__int__`, then :meth:`__trunc__` is called to implement the - built-in function :func:`int`. - - .. method:: object.__index__(self) Called to implement :func:`operator.index`, and whenever Python needs to @@ -2336,6 +2325,23 @@ left undefined. the same value. +.. method:: object.__round__(self, [,ndigits]) + object.__trunc__(self) + object.__floor__(self) + object.__ceil__(self) + + .. index:: builtin: round + + Called to implement the built-in function :func:`round` and :mod:`math` + functions :func:`~math.trunc`, :func:`~math.floor` and :func:`~math.ceil`. + Unless *ndigits* is passed to :meth:`!__round__` all these methods should + return the value of the object truncated to an :class:`~numbers.Integral` + (typically an :class:`int`). + + If :meth:`__int__` is not defined then the built-in function :func:`int` + falls back to :meth:`__trunc__`. + + .. _context-managers: With Statement Context Managers From webhook-mailer at python.org Sat Mar 10 11:22:36 2018 From: webhook-mailer at python.org (Serhiy Storchaka) Date: Sat, 10 Mar 2018 16:22:36 -0000 Subject: [Python-checkins] bpo-33041: Fixed bytecode generation for "async for" with a complex target. (#6052) Message-ID: <mailman.19.1520698958.1871.python-checkins@python.org> https://github.com/python/cpython/commit/24d3201eb7f0b39a7eaf2a5b2a2ceca10ad1f8eb commit: 24d3201eb7f0b39a7eaf2a5b2a2ceca10ad1f8eb branch: master author: Serhiy Storchaka <storchaka at gmail.com> committer: GitHub <noreply at github.com> date: 2018-03-10T18:22:34+02:00 summary: bpo-33041: Fixed bytecode generation for "async for" with a complex target. (#6052) A StopAsyncIteration raised on assigning or unpacking will be now propagated instead of stopping the iteration. files: A Misc/NEWS.d/next/Core and Builtins/2018-03-10-15-16-40.bpo-33041.-ak5Fk.rst M Lib/test/test_coroutines.py M Python/compile.c diff --git a/Lib/test/test_coroutines.py b/Lib/test/test_coroutines.py index b37b61b71959..f4a9d2aeb874 100644 --- a/Lib/test/test_coroutines.py +++ b/Lib/test/test_coroutines.py @@ -1949,6 +1949,71 @@ def test_fatal_coro_warning(self): support.gc_collect() self.assertIn("was never awaited", stderr.getvalue()) + def test_for_assign_raising_stop_async_iteration(self): + class BadTarget: + def __setitem__(self, key, value): + raise StopAsyncIteration(42) + tgt = BadTarget() + async def source(): + yield 10 + + async def run_for(): + with self.assertRaises(StopAsyncIteration) as cm: + async for tgt[0] in source(): + pass + self.assertEqual(cm.exception.args, (42,)) + return 'end' + self.assertEqual(run_async(run_for()), ([], 'end')) + + async def run_list(): + with self.assertRaises(StopAsyncIteration) as cm: + return [0 async for tgt[0] in source()] + self.assertEqual(cm.exception.args, (42,)) + return 'end' + self.assertEqual(run_async(run_list()), ([], 'end')) + + async def run_gen(): + gen = (0 async for tgt[0] in source()) + a = gen.asend(None) + with self.assertRaises(RuntimeError) as cm: + await a + self.assertIsInstance(cm.exception.__cause__, StopAsyncIteration) + self.assertEqual(cm.exception.__cause__.args, (42,)) + return 'end' + self.assertEqual(run_async(run_gen()), ([], 'end')) + + def test_for_assign_raising_stop_async_iteration_2(self): + class BadIterable: + def __iter__(self): + raise StopAsyncIteration(42) + async def badpairs(): + yield BadIterable() + + async def run_for(): + with self.assertRaises(StopAsyncIteration) as cm: + async for i, j in badpairs(): + pass + self.assertEqual(cm.exception.args, (42,)) + return 'end' + self.assertEqual(run_async(run_for()), ([], 'end')) + + async def run_list(): + with self.assertRaises(StopAsyncIteration) as cm: + return [0 async for i, j in badpairs()] + self.assertEqual(cm.exception.args, (42,)) + return 'end' + self.assertEqual(run_async(run_list()), ([], 'end')) + + async def run_gen(): + gen = (0 async for i, j in badpairs()) + a = gen.asend(None) + with self.assertRaises(RuntimeError) as cm: + await a + self.assertIsInstance(cm.exception.__cause__, StopAsyncIteration) + self.assertEqual(cm.exception.__cause__.args, (42,)) + return 'end' + self.assertEqual(run_async(run_gen()), ([], 'end')) + class CoroAsyncIOCompatTest(unittest.TestCase): diff --git a/Misc/NEWS.d/next/Core and Builtins/2018-03-10-15-16-40.bpo-33041.-ak5Fk.rst b/Misc/NEWS.d/next/Core and Builtins/2018-03-10-15-16-40.bpo-33041.-ak5Fk.rst new file mode 100644 index 000000000000..af9ccfd89f5d --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2018-03-10-15-16-40.bpo-33041.-ak5Fk.rst @@ -0,0 +1,3 @@ +Fixed bytecode generation for "async for" with a complex target. A +StopAsyncIteration raised on assigning or unpacking will be now propagated +instead of stopping the iteration. diff --git a/Python/compile.c b/Python/compile.c index f14647b4f641..6c9e7954d688 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -2473,8 +2473,8 @@ compiler_async_for(struct compiler *c, stmt_ty s) ADDOP(c, GET_ANEXT); ADDOP_O(c, LOAD_CONST, Py_None, consts); ADDOP(c, YIELD_FROM); - VISIT(c, expr, s->v.AsyncFor.target); ADDOP(c, POP_BLOCK); /* for SETUP_FINALLY */ + VISIT(c, expr, s->v.AsyncFor.target); compiler_pop_fblock(c, EXCEPT, try); ADDOP_JREL(c, JUMP_FORWARD, after_try); @@ -4060,8 +4060,8 @@ compiler_async_comprehension_generator(struct compiler *c, ADDOP(c, GET_ANEXT); ADDOP_O(c, LOAD_CONST, Py_None, consts); ADDOP(c, YIELD_FROM); - VISIT(c, expr, gen->target); ADDOP(c, POP_BLOCK); + VISIT(c, expr, gen->target); compiler_pop_fblock(c, EXCEPT, try); ADDOP_JREL(c, JUMP_FORWARD, after_try); From webhook-mailer at python.org Sat Mar 10 11:27:04 2018 From: webhook-mailer at python.org (Andrew Svetlov) Date: Sat, 10 Mar 2018 16:27:04 -0000 Subject: [Python-checkins] bpo-33037: Skip sending/receiving after SSL transport closing (GH-6044) (GH-6057) Message-ID: <mailman.20.1520699225.1871.python-checkins@python.org> https://github.com/python/cpython/commit/bf0d1165174e8347b4d3a731c4e47e8288f1d01b commit: bf0d1165174e8347b4d3a731c4e47e8288f1d01b branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Andrew Svetlov <andrew.svetlov at gmail.com> date: 2018-03-10T18:27:01+02:00 summary: bpo-33037: Skip sending/receiving after SSL transport closing (GH-6044) (GH-6057) * Skip write()/data_received() if sslpipe is destroyed (cherry picked from commit 5e80a71ab67045fecec46573a1892e240b569ace) Co-authored-by: Andrew Svetlov <andrew.svetlov at gmail.com> files: A Misc/NEWS.d/next/Library/2018-03-09-23-07-07.bpo-33037.nAJ3at.rst M Lib/asyncio/sslproto.py M Lib/test/test_asyncio/test_sslproto.py diff --git a/Lib/asyncio/sslproto.py b/Lib/asyncio/sslproto.py index 863b54313cc6..2bbf134c0f7e 100644 --- a/Lib/asyncio/sslproto.py +++ b/Lib/asyncio/sslproto.py @@ -504,6 +504,10 @@ def data_received(self, data): The argument is a bytes object. """ + if self._sslpipe is None: + # transport closing, sslpipe is destroyed + return + try: ssldata, appdata = self._sslpipe.feed_ssldata(data) except ssl.SSLError as e: @@ -636,7 +640,7 @@ def _on_handshake_complete(self, handshake_exc): def _process_write_backlog(self): # Try to make progress on the write backlog. - if self._transport is None: + if self._transport is None or self._sslpipe is None: return try: diff --git a/Lib/test/test_asyncio/test_sslproto.py b/Lib/test/test_asyncio/test_sslproto.py index c4c30bedb4ec..7b27f4cfa322 100644 --- a/Lib/test/test_asyncio/test_sslproto.py +++ b/Lib/test/test_asyncio/test_sslproto.py @@ -26,16 +26,17 @@ def setUp(self): self.loop = asyncio.new_event_loop() self.set_event_loop(self.loop) - def ssl_protocol(self, waiter=None): + def ssl_protocol(self, *, waiter=None, proto=None): sslcontext = test_utils.dummy_ssl_context() - app_proto = asyncio.Protocol() - proto = sslproto.SSLProtocol(self.loop, app_proto, sslcontext, waiter, - ssl_handshake_timeout=0.1) - self.assertIs(proto._app_transport.get_protocol(), app_proto) - self.addCleanup(proto._app_transport.close) - return proto - - def connection_made(self, ssl_proto, do_handshake=None): + if proto is None: # app protocol + proto = asyncio.Protocol() + ssl_proto = sslproto.SSLProtocol(self.loop, proto, sslcontext, waiter, + ssl_handshake_timeout=0.1) + self.assertIs(ssl_proto._app_transport.get_protocol(), proto) + self.addCleanup(ssl_proto._app_transport.close) + return ssl_proto + + def connection_made(self, ssl_proto, *, do_handshake=None): transport = mock.Mock() sslpipe = mock.Mock() sslpipe.shutdown.return_value = b'' @@ -53,7 +54,7 @@ def test_cancel_handshake(self): # Python issue #23197: cancelling a handshake must not raise an # exception or log an error, even if the handshake failed waiter = asyncio.Future(loop=self.loop) - ssl_proto = self.ssl_protocol(waiter) + ssl_proto = self.ssl_protocol(waiter=waiter) handshake_fut = asyncio.Future(loop=self.loop) def do_handshake(callback): @@ -63,7 +64,7 @@ def do_handshake(callback): return [] waiter.cancel() - self.connection_made(ssl_proto, do_handshake) + self.connection_made(ssl_proto, do_handshake=do_handshake) with test_utils.disable_logger(): self.loop.run_until_complete(handshake_fut) @@ -96,7 +97,7 @@ def test_handshake_timeout_negative(self): def test_eof_received_waiter(self): waiter = asyncio.Future(loop=self.loop) - ssl_proto = self.ssl_protocol(waiter) + ssl_proto = self.ssl_protocol(waiter=waiter) self.connection_made(ssl_proto) ssl_proto.eof_received() test_utils.run_briefly(self.loop) @@ -107,7 +108,7 @@ def test_fatal_error_no_name_error(self): # _fatal_error() generates a NameError if sslproto.py # does not import base_events. waiter = asyncio.Future(loop=self.loop) - ssl_proto = self.ssl_protocol(waiter) + ssl_proto = self.ssl_protocol(waiter=waiter) # Temporarily turn off error logging so as not to spoil test output. log_level = log.logger.getEffectiveLevel() log.logger.setLevel(logging.FATAL) @@ -121,7 +122,7 @@ def test_connection_lost(self): # From issue #472. # yield from waiter hang if lost_connection was called. waiter = asyncio.Future(loop=self.loop) - ssl_proto = self.ssl_protocol(waiter) + ssl_proto = self.ssl_protocol(waiter=waiter) self.connection_made(ssl_proto) ssl_proto.connection_lost(ConnectionAbortedError) test_utils.run_briefly(self.loop) @@ -130,10 +131,7 @@ def test_connection_lost(self): def test_close_during_handshake(self): # bpo-29743 Closing transport during handshake process leaks socket waiter = asyncio.Future(loop=self.loop) - ssl_proto = self.ssl_protocol(waiter) - - def do_handshake(callback): - return [] + ssl_proto = self.ssl_protocol(waiter=waiter) transport = self.connection_made(ssl_proto) test_utils.run_briefly(self.loop) @@ -143,7 +141,7 @@ def do_handshake(callback): def test_get_extra_info_on_closed_connection(self): waiter = asyncio.Future(loop=self.loop) - ssl_proto = self.ssl_protocol(waiter) + ssl_proto = self.ssl_protocol(waiter=waiter) self.assertIsNone(ssl_proto._get_extra_info('socket')) default = object() self.assertIs(ssl_proto._get_extra_info('socket', default), default) @@ -154,12 +152,31 @@ def test_get_extra_info_on_closed_connection(self): def test_set_new_app_protocol(self): waiter = asyncio.Future(loop=self.loop) - ssl_proto = self.ssl_protocol(waiter) + ssl_proto = self.ssl_protocol(waiter=waiter) new_app_proto = asyncio.Protocol() ssl_proto._app_transport.set_protocol(new_app_proto) self.assertIs(ssl_proto._app_transport.get_protocol(), new_app_proto) self.assertIs(ssl_proto._app_protocol, new_app_proto) + def test_data_received_after_closing(self): + ssl_proto = self.ssl_protocol() + self.connection_made(ssl_proto) + transp = ssl_proto._app_transport + + transp.close() + + # should not raise + self.assertIsNone(ssl_proto.data_received(b'data')) + + def test_write_after_closing(self): + ssl_proto = self.ssl_protocol() + self.connection_made(ssl_proto) + transp = ssl_proto._app_transport + transp.close() + + # should not raise + self.assertIsNone(transp.write(b'data')) + ############################################################################## # Start TLS Tests diff --git a/Misc/NEWS.d/next/Library/2018-03-09-23-07-07.bpo-33037.nAJ3at.rst b/Misc/NEWS.d/next/Library/2018-03-09-23-07-07.bpo-33037.nAJ3at.rst new file mode 100644 index 000000000000..2732eeb4534b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-03-09-23-07-07.bpo-33037.nAJ3at.rst @@ -0,0 +1 @@ +Skip sending/receiving data after SSL transport closing. From webhook-mailer at python.org Sat Mar 10 11:49:29 2018 From: webhook-mailer at python.org (Serhiy Storchaka) Date: Sat, 10 Mar 2018 16:49:29 -0000 Subject: [Python-checkins] bpo-33041: Add missed error checks when compile "async for" (#6053) Message-ID: <mailman.21.1520700571.1871.python-checkins@python.org> https://github.com/python/cpython/commit/67ee07795bcd84b679c000780212d4d81a1490a3 commit: 67ee07795bcd84b679c000780212d4d81a1490a3 branch: master author: Serhiy Storchaka <storchaka at gmail.com> committer: GitHub <noreply at github.com> date: 2018-03-10T18:49:26+02:00 summary: bpo-33041: Add missed error checks when compile "async for" (#6053) and remove redundant code. files: M Python/compile.c diff --git a/Python/compile.c b/Python/compile.c index 6c9e7954d688..fbd1fc960ab8 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -2437,7 +2437,7 @@ compiler_async_for(struct compiler *c, stmt_ty s) _Py_IDENTIFIER(StopAsyncIteration); basicblock *try, *except, *end, *after_try, *try_cleanup, - *after_loop, *after_loop_else; + *after_loop_else; PyObject *stop_aiter_error = _PyUnicode_FromId(&PyId_StopAsyncIteration); if (stop_aiter_error == NULL) { @@ -2449,14 +2449,14 @@ compiler_async_for(struct compiler *c, stmt_ty s) end = compiler_new_block(c); after_try = compiler_new_block(c); try_cleanup = compiler_new_block(c); - after_loop = compiler_new_block(c); after_loop_else = compiler_new_block(c); if (try == NULL || except == NULL || end == NULL - || after_try == NULL || try_cleanup == NULL) + || after_try == NULL || try_cleanup == NULL + || after_loop_else == NULL) return 0; - if (!compiler_push_fblock(c, FOR_LOOP, try, after_loop)) + if (!compiler_push_fblock(c, FOR_LOOP, try, end)) return 0; VISIT(c, expr, s->v.AsyncFor.iter); @@ -2504,10 +2504,6 @@ compiler_async_for(struct compiler *c, stmt_ty s) compiler_pop_fblock(c, FOR_LOOP, try); - /* Block reached after `break`ing from loop */ - compiler_use_next_block(c, after_loop); - ADDOP_JABS(c, JUMP_ABSOLUTE, end); - /* `else` block */ compiler_use_next_block(c, after_loop_else); VISIT_SEQ(c, stmt, s->v.For.orelse); @@ -4014,7 +4010,7 @@ compiler_async_comprehension_generator(struct compiler *c, _Py_IDENTIFIER(StopAsyncIteration); comprehension_ty gen; - basicblock *anchor, *skip, *if_cleanup, *try, + basicblock *anchor, *if_cleanup, *try, *after_try, *except, *try_cleanup; Py_ssize_t i, n; @@ -4027,13 +4023,12 @@ compiler_async_comprehension_generator(struct compiler *c, after_try = compiler_new_block(c); try_cleanup = compiler_new_block(c); except = compiler_new_block(c); - skip = compiler_new_block(c); if_cleanup = compiler_new_block(c); anchor = compiler_new_block(c); - if (skip == NULL || if_cleanup == NULL || anchor == NULL || + if (if_cleanup == NULL || anchor == NULL || try == NULL || after_try == NULL || - except == NULL || after_try == NULL) { + except == NULL || try_cleanup == NULL) { return 0; } @@ -4125,8 +4120,6 @@ compiler_async_comprehension_generator(struct compiler *c, default: return 0; } - - compiler_use_next_block(c, skip); } compiler_use_next_block(c, if_cleanup); ADDOP_JABS(c, JUMP_ABSOLUTE, try); From webhook-mailer at python.org Sat Mar 10 12:09:10 2018 From: webhook-mailer at python.org (Andrew Svetlov) Date: Sat, 10 Mar 2018 17:09:10 -0000 Subject: [Python-checkins] [3.6] bpo-33037: Skip sending/receiving after SSL transport closing (GH-6044) (GH-6058) Message-ID: <mailman.22.1520701752.1871.python-checkins@python.org> https://github.com/python/cpython/commit/017e9fda922a143ac9f1601cbde05e80214852d2 commit: 017e9fda922a143ac9f1601cbde05e80214852d2 branch: 3.6 author: Andrew Svetlov <andrew.svetlov at gmail.com> committer: GitHub <noreply at github.com> date: 2018-03-10T19:09:07+02:00 summary: [3.6] bpo-33037: Skip sending/receiving after SSL transport closing (GH-6044) (GH-6058) * Skip write()/data_received() if sslpipe is destroyed. (cherry picked from commit 5e80a71ab67045fecec46573a1892e240b569ace) files: A Misc/NEWS.d/next/Library/2018-03-09-23-07-07.bpo-33037.nAJ3at.rst M Lib/asyncio/sslproto.py M Lib/test/test_asyncio/test_sslproto.py diff --git a/Lib/asyncio/sslproto.py b/Lib/asyncio/sslproto.py index f4d8a4841823..a82babb6b9e2 100644 --- a/Lib/asyncio/sslproto.py +++ b/Lib/asyncio/sslproto.py @@ -497,6 +497,10 @@ def data_received(self, data): The argument is a bytes object. """ + if self._sslpipe is None: + # transport closing, sslpipe is destroyed + return + try: ssldata, appdata = self._sslpipe.feed_ssldata(data) except ssl.SSLError as e: @@ -626,7 +630,7 @@ def _on_handshake_complete(self, handshake_exc): def _process_write_backlog(self): # Try to make progress on the write backlog. - if self._transport is None: + if self._transport is None or self._sslpipe is None: return try: diff --git a/Lib/test/test_asyncio/test_sslproto.py b/Lib/test/test_asyncio/test_sslproto.py index f573ae8fe779..9b198bfd5307 100644 --- a/Lib/test/test_asyncio/test_sslproto.py +++ b/Lib/test/test_asyncio/test_sslproto.py @@ -22,15 +22,16 @@ def setUp(self): self.loop = asyncio.new_event_loop() self.set_event_loop(self.loop) - def ssl_protocol(self, waiter=None): + def ssl_protocol(self, *, waiter=None, proto=None): sslcontext = test_utils.dummy_ssl_context() - app_proto = asyncio.Protocol() - proto = sslproto.SSLProtocol(self.loop, app_proto, sslcontext, waiter) - self.assertIs(proto._app_transport.get_protocol(), app_proto) - self.addCleanup(proto._app_transport.close) - return proto - - def connection_made(self, ssl_proto, do_handshake=None): + if proto is None: # app protocol + proto = asyncio.Protocol() + ssl_proto = sslproto.SSLProtocol(self.loop, proto, sslcontext, waiter) + self.assertIs(ssl_proto._app_transport.get_protocol(), proto) + self.addCleanup(ssl_proto._app_transport.close) + return ssl_proto + + def connection_made(self, ssl_proto, *, do_handshake=None): transport = mock.Mock() sslpipe = mock.Mock() sslpipe.shutdown.return_value = b'' @@ -48,7 +49,7 @@ def test_cancel_handshake(self): # Python issue #23197: cancelling a handshake must not raise an # exception or log an error, even if the handshake failed waiter = asyncio.Future(loop=self.loop) - ssl_proto = self.ssl_protocol(waiter) + ssl_proto = self.ssl_protocol(waiter=waiter) handshake_fut = asyncio.Future(loop=self.loop) def do_handshake(callback): @@ -58,14 +59,14 @@ def do_handshake(callback): return [] waiter.cancel() - self.connection_made(ssl_proto, do_handshake) + self.connection_made(ssl_proto, do_handshake=do_handshake) with test_utils.disable_logger(): self.loop.run_until_complete(handshake_fut) def test_eof_received_waiter(self): waiter = asyncio.Future(loop=self.loop) - ssl_proto = self.ssl_protocol(waiter) + ssl_proto = self.ssl_protocol(waiter=waiter) self.connection_made(ssl_proto) ssl_proto.eof_received() test_utils.run_briefly(self.loop) @@ -76,7 +77,7 @@ def test_fatal_error_no_name_error(self): # _fatal_error() generates a NameError if sslproto.py # does not import base_events. waiter = asyncio.Future(loop=self.loop) - ssl_proto = self.ssl_protocol(waiter) + ssl_proto = self.ssl_protocol(waiter=waiter) # Temporarily turn off error logging so as not to spoil test output. log_level = log.logger.getEffectiveLevel() log.logger.setLevel(logging.FATAL) @@ -90,7 +91,7 @@ def test_connection_lost(self): # From issue #472. # yield from waiter hang if lost_connection was called. waiter = asyncio.Future(loop=self.loop) - ssl_proto = self.ssl_protocol(waiter) + ssl_proto = self.ssl_protocol(waiter=waiter) self.connection_made(ssl_proto) ssl_proto.connection_lost(ConnectionAbortedError) test_utils.run_briefly(self.loop) @@ -99,10 +100,7 @@ def test_connection_lost(self): def test_close_during_handshake(self): # bpo-29743 Closing transport during handshake process leaks socket waiter = asyncio.Future(loop=self.loop) - ssl_proto = self.ssl_protocol(waiter) - - def do_handshake(callback): - return [] + ssl_proto = self.ssl_protocol(waiter=waiter) transport = self.connection_made(ssl_proto) test_utils.run_briefly(self.loop) @@ -112,7 +110,7 @@ def do_handshake(callback): def test_get_extra_info_on_closed_connection(self): waiter = asyncio.Future(loop=self.loop) - ssl_proto = self.ssl_protocol(waiter) + ssl_proto = self.ssl_protocol(waiter=waiter) self.assertIsNone(ssl_proto._get_extra_info('socket')) default = object() self.assertIs(ssl_proto._get_extra_info('socket', default), default) @@ -123,12 +121,31 @@ def test_get_extra_info_on_closed_connection(self): def test_set_new_app_protocol(self): waiter = asyncio.Future(loop=self.loop) - ssl_proto = self.ssl_protocol(waiter) + ssl_proto = self.ssl_protocol(waiter=waiter) new_app_proto = asyncio.Protocol() ssl_proto._app_transport.set_protocol(new_app_proto) self.assertIs(ssl_proto._app_transport.get_protocol(), new_app_proto) self.assertIs(ssl_proto._app_protocol, new_app_proto) + def test_data_received_after_closing(self): + ssl_proto = self.ssl_protocol() + self.connection_made(ssl_proto) + transp = ssl_proto._app_transport + + transp.close() + + # should not raise + self.assertIsNone(ssl_proto.data_received(b'data')) + + def test_write_after_closing(self): + ssl_proto = self.ssl_protocol() + self.connection_made(ssl_proto) + transp = ssl_proto._app_transport + transp.close() + + # should not raise + self.assertIsNone(transp.write(b'data')) + if __name__ == '__main__': unittest.main() diff --git a/Misc/NEWS.d/next/Library/2018-03-09-23-07-07.bpo-33037.nAJ3at.rst b/Misc/NEWS.d/next/Library/2018-03-09-23-07-07.bpo-33037.nAJ3at.rst new file mode 100644 index 000000000000..2732eeb4534b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-03-09-23-07-07.bpo-33037.nAJ3at.rst @@ -0,0 +1 @@ +Skip sending/receiving data after SSL transport closing. From webhook-mailer at python.org Sat Mar 10 13:45:08 2018 From: webhook-mailer at python.org (Serhiy Storchaka) Date: Sat, 10 Mar 2018 18:45:08 -0000 Subject: [Python-checkins] [3.7] bpo-33041: Add missed error checks when compile "async for" (GH-6053) (GH-6060) Message-ID: <mailman.23.1520707509.1871.python-checkins@python.org> https://github.com/python/cpython/commit/9e94c0d3c78d1bc582c865240ed9353fe9689b2a commit: 9e94c0d3c78d1bc582c865240ed9353fe9689b2a branch: 3.7 author: Serhiy Storchaka <storchaka at gmail.com> committer: GitHub <noreply at github.com> date: 2018-03-10T20:45:05+02:00 summary: [3.7] bpo-33041: Add missed error checks when compile "async for" (GH-6053) (GH-6060) and remove redundant code. (cherry picked from commit 67ee07795bcd84b679c000780212d4d81a1490a3) Co-authored-by: Serhiy Storchaka <storchaka at gmail.com> files: M Python/compile.c diff --git a/Python/compile.c b/Python/compile.c index 0dc35662749b..8ab33613b524 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -2337,7 +2337,7 @@ compiler_async_for(struct compiler *c, stmt_ty s) _Py_IDENTIFIER(StopAsyncIteration); basicblock *try, *except, *end, *after_try, *try_cleanup, - *after_loop, *after_loop_else; + *after_loop_else; PyObject *stop_aiter_error = _PyUnicode_FromId(&PyId_StopAsyncIteration); if (stop_aiter_error == NULL) { @@ -2349,14 +2349,14 @@ compiler_async_for(struct compiler *c, stmt_ty s) end = compiler_new_block(c); after_try = compiler_new_block(c); try_cleanup = compiler_new_block(c); - after_loop = compiler_new_block(c); after_loop_else = compiler_new_block(c); if (try == NULL || except == NULL || end == NULL - || after_try == NULL || try_cleanup == NULL) + || after_try == NULL || try_cleanup == NULL + || after_loop_else == NULL) return 0; - ADDOP_JREL(c, SETUP_LOOP, after_loop); + ADDOP_JREL(c, SETUP_LOOP, end); if (!compiler_push_fblock(c, LOOP, try)) return 0; @@ -2404,9 +2404,6 @@ compiler_async_for(struct compiler *c, stmt_ty s) ADDOP(c, POP_BLOCK); /* for SETUP_LOOP */ compiler_pop_fblock(c, LOOP, try); - compiler_use_next_block(c, after_loop); - ADDOP_JABS(c, JUMP_ABSOLUTE, end); - compiler_use_next_block(c, after_loop_else); VISIT_SEQ(c, stmt, s->v.For.orelse); @@ -3893,7 +3890,7 @@ compiler_async_comprehension_generator(struct compiler *c, _Py_IDENTIFIER(StopAsyncIteration); comprehension_ty gen; - basicblock *anchor, *skip, *if_cleanup, *try, + basicblock *anchor, *if_cleanup, *try, *after_try, *except, *try_cleanup; Py_ssize_t i, n; @@ -3906,13 +3903,12 @@ compiler_async_comprehension_generator(struct compiler *c, after_try = compiler_new_block(c); try_cleanup = compiler_new_block(c); except = compiler_new_block(c); - skip = compiler_new_block(c); if_cleanup = compiler_new_block(c); anchor = compiler_new_block(c); - if (skip == NULL || if_cleanup == NULL || anchor == NULL || + if (if_cleanup == NULL || anchor == NULL || try == NULL || after_try == NULL || - except == NULL || after_try == NULL) { + except == NULL || try_cleanup == NULL) { return 0; } @@ -4004,8 +4000,6 @@ compiler_async_comprehension_generator(struct compiler *c, default: return 0; } - - compiler_use_next_block(c, skip); } compiler_use_next_block(c, if_cleanup); ADDOP_JABS(c, JUMP_ABSOLUTE, try); From webhook-mailer at python.org Sat Mar 10 13:58:55 2018 From: webhook-mailer at python.org (Xiang Zhang) Date: Sat, 10 Mar 2018 18:58:55 -0000 Subject: [Python-checkins] bpo-30249: Improve struct.unpack_from() error messages (GH-6059) Message-ID: <mailman.24.1520708337.1871.python-checkins@python.org> https://github.com/python/cpython/commit/c10b288f345aaef66d2c844924b9a576f9ea4f8b commit: c10b288f345aaef66d2c844924b9a576f9ea4f8b branch: master author: Xiang Zhang <angwerzx at 126.com> committer: GitHub <noreply at github.com> date: 2018-03-11T02:58:52+08:00 summary: bpo-30249: Improve struct.unpack_from() error messages (GH-6059) files: A Misc/NEWS.d/next/Library/2018-03-11-00-20-26.bpo-30249.KSkgLB.rst M Doc/library/struct.rst M Lib/test/test_struct.py M Modules/_struct.c M Modules/clinic/_struct.c.h diff --git a/Doc/library/struct.rst b/Doc/library/struct.rst index 2d0866c7e09e..d6a3cb721e83 100644 --- a/Doc/library/struct.rst +++ b/Doc/library/struct.rst @@ -74,8 +74,8 @@ The module defines the following exception and functions: Unpack from *buffer* starting at position *offset*, according to the format string *format*. The result is a tuple even if it contains exactly one - item. The buffer's size in bytes, minus *offset*, must be at least - the size required by the format, as reflected by :func:`calcsize`. + item. The buffer's size in bytes, starting at position *offset*, must be at + least the size required by the format, as reflected by :func:`calcsize`. .. function:: iter_unpack(format, buffer) @@ -428,7 +428,7 @@ The :mod:`struct` module also defines the following type: .. method:: unpack_from(buffer, offset=0) Identical to the :func:`unpack_from` function, using the compiled format. - The buffer's size in bytes, minus *offset*, must be at least + The buffer's size in bytes, starting at position *offset*, must be at least :attr:`size`. diff --git a/Lib/test/test_struct.py b/Lib/test/test_struct.py index 8fd56c91cb7a..454082e66d3f 100644 --- a/Lib/test/test_struct.py +++ b/Lib/test/test_struct.py @@ -579,14 +579,22 @@ def test__sizeof__(self): self.check_sizeof('0c', 0) def test_boundary_error_message(self): - regex = ( + regex1 = ( r'pack_into requires a buffer of at least 6 ' r'bytes for packing 1 bytes at offset 5 ' r'\(actual buffer size is 1\)' ) - with self.assertRaisesRegex(struct.error, regex): + with self.assertRaisesRegex(struct.error, regex1): struct.pack_into('b', bytearray(1), 5, 1) + regex2 = ( + r'unpack_from requires a buffer of at least 6 ' + r'bytes for unpacking 1 bytes at offset 5 ' + r'\(actual buffer size is 1\)' + ) + with self.assertRaisesRegex(struct.error, regex2): + struct.unpack_from('b', bytearray(1), 5) + def test_boundary_error_message_with_negative_offset(self): byte_list = bytearray(10) with self.assertRaisesRegex( @@ -599,16 +607,34 @@ def test_boundary_error_message_with_negative_offset(self): 'offset -11 out of range for 10-byte buffer'): struct.pack_into('<B', byte_list, -11, 123) + with self.assertRaisesRegex( + struct.error, + r'not enough data to unpack 4 bytes at offset -2'): + struct.unpack_from('<I', byte_list, -2) + + with self.assertRaisesRegex( + struct.error, + "offset -11 out of range for 10-byte buffer"): + struct.unpack_from('<B', byte_list, -11) + def test_boundary_error_message_with_large_offset(self): # Test overflows cause by large offset and value size (issue 30245) - regex = ( + regex1 = ( r'pack_into requires a buffer of at least ' + str(sys.maxsize + 4) + r' bytes for packing 4 bytes at offset ' + str(sys.maxsize) + r' \(actual buffer size is 10\)' ) - with self.assertRaisesRegex(struct.error, regex): + with self.assertRaisesRegex(struct.error, regex1): struct.pack_into('<I', bytearray(10), sys.maxsize, 1) + regex2 = ( + r'unpack_from requires a buffer of at least ' + str(sys.maxsize + 4) + + r' bytes for unpacking 4 bytes at offset ' + str(sys.maxsize) + + r' \(actual buffer size is 10\)' + ) + with self.assertRaisesRegex(struct.error, regex2): + struct.unpack_from('<I', bytearray(10), sys.maxsize) + def test_issue29802(self): # When the second argument of struct.unpack() was of wrong type # the Struct object was decrefed twice and the reference to diff --git a/Misc/NEWS.d/next/Library/2018-03-11-00-20-26.bpo-30249.KSkgLB.rst b/Misc/NEWS.d/next/Library/2018-03-11-00-20-26.bpo-30249.KSkgLB.rst new file mode 100644 index 000000000000..f30dff3124eb --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-03-11-00-20-26.bpo-30249.KSkgLB.rst @@ -0,0 +1,2 @@ +Improve struct.unpack_from() exception messages for problems with the buffer +size and offset. diff --git a/Modules/_struct.c b/Modules/_struct.c index 66f74d63b735..053970671f17 100644 --- a/Modules/_struct.c +++ b/Modules/_struct.c @@ -1553,7 +1553,8 @@ Return a tuple containing unpacked values. Values are unpacked according to the format string Struct.format. -The buffer's size in bytes, minus offset, must be at least Struct.size. +The buffer's size in bytes, starting at position offset, must be +at least Struct.size. See help(struct) for more on format strings. [clinic start generated code]*/ @@ -1561,16 +1562,38 @@ See help(struct) for more on format strings. static PyObject * Struct_unpack_from_impl(PyStructObject *self, Py_buffer *buffer, Py_ssize_t offset) -/*[clinic end generated code: output=57fac875e0977316 input=97ade52422f8962f]*/ +/*[clinic end generated code: output=57fac875e0977316 input=cafd4851d473c894]*/ { assert(self->s_codes != NULL); - if (offset < 0) + if (offset < 0) { + if (offset + self->s_size > 0) { + PyErr_Format(StructError, + "not enough data to unpack %zd bytes at offset %zd", + self->s_size, + offset); + return NULL; + } + + if (offset + buffer->len < 0) { + PyErr_Format(StructError, + "offset %zd out of range for %zd-byte buffer", + offset, + buffer->len); + return NULL; + } offset += buffer->len; - if (offset < 0 || buffer->len - offset < self->s_size) { + } + + if ((buffer->len - offset) < self->s_size) { PyErr_Format(StructError, - "unpack_from requires a buffer of at least %zd bytes", - self->s_size); + "unpack_from requires a buffer of at least %zu bytes for " + "unpacking %zd bytes at offset %zd " + "(actual buffer size is %zd)", + (size_t)self->s_size + (size_t)offset, + self->s_size, + offset, + buffer->len); return NULL; } return s_unpack_internal(self, (char*)buffer->buf + offset); diff --git a/Modules/clinic/_struct.c.h b/Modules/clinic/_struct.c.h index 6e43215b737a..2ecadfb1de97 100644 --- a/Modules/clinic/_struct.c.h +++ b/Modules/clinic/_struct.c.h @@ -79,7 +79,8 @@ PyDoc_STRVAR(Struct_unpack_from__doc__, "\n" "Values are unpacked according to the format string Struct.format.\n" "\n" -"The buffer\'s size in bytes, minus offset, must be at least Struct.size.\n" +"The buffer\'s size in bytes, starting at position offset, must be\n" +"at least Struct.size.\n" "\n" "See help(struct) for more on format strings."); @@ -302,4 +303,4 @@ iter_unpack(PyObject *module, PyObject *const *args, Py_ssize_t nargs) return return_value; } -/*[clinic end generated code: output=9119f213a951e4cc input=a9049054013a1b77]*/ +/*[clinic end generated code: output=d79b009652ae0b89 input=a9049054013a1b77]*/ From webhook-mailer at python.org Sat Mar 10 14:32:53 2018 From: webhook-mailer at python.org (Miss Islington (bot)) Date: Sat, 10 Mar 2018 19:32:53 -0000 Subject: [Python-checkins] [3.7] bpo-33041: Add missed error checks when compile "async for" (GH-6053) (GH-6060) Message-ID: <mailman.25.1520710374.1871.python-checkins@python.org> https://github.com/python/cpython/commit/d0826340d96e0953793b86d0b8475d2f43a280b6 commit: d0826340d96e0953793b86d0b8475d2f43a280b6 branch: 3.6 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: GitHub <noreply at github.com> date: 2018-03-10T11:32:49-08:00 summary: [3.7] bpo-33041: Add missed error checks when compile "async for" (GH-6053) (GH-6060) and remove redundant code. (cherry picked from commit 67ee07795bcd84b679c000780212d4d81a1490a3) Co-authored-by: Serhiy Storchaka <storchaka at gmail.com> (cherry picked from commit 9e94c0d3c78d1bc582c865240ed9353fe9689b2a) Co-authored-by: Serhiy Storchaka <storchaka at gmail.com> files: M Python/compile.c diff --git a/Python/compile.c b/Python/compile.c index 3b8f9bbd4426..aae3300febed 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -2189,7 +2189,7 @@ compiler_async_for(struct compiler *c, stmt_ty s) _Py_IDENTIFIER(StopAsyncIteration); basicblock *try, *except, *end, *after_try, *try_cleanup, - *after_loop, *after_loop_else; + *after_loop_else; PyObject *stop_aiter_error = _PyUnicode_FromId(&PyId_StopAsyncIteration); if (stop_aiter_error == NULL) { @@ -2201,14 +2201,14 @@ compiler_async_for(struct compiler *c, stmt_ty s) end = compiler_new_block(c); after_try = compiler_new_block(c); try_cleanup = compiler_new_block(c); - after_loop = compiler_new_block(c); after_loop_else = compiler_new_block(c); if (try == NULL || except == NULL || end == NULL - || after_try == NULL || try_cleanup == NULL) + || after_try == NULL || try_cleanup == NULL + || after_loop_else == NULL) return 0; - ADDOP_JREL(c, SETUP_LOOP, after_loop); + ADDOP_JREL(c, SETUP_LOOP, end); if (!compiler_push_fblock(c, LOOP, try)) return 0; @@ -2257,9 +2257,6 @@ compiler_async_for(struct compiler *c, stmt_ty s) ADDOP(c, POP_BLOCK); /* for SETUP_LOOP */ compiler_pop_fblock(c, LOOP, try); - compiler_use_next_block(c, after_loop); - ADDOP_JABS(c, JUMP_ABSOLUTE, end); - compiler_use_next_block(c, after_loop_else); VISIT_SEQ(c, stmt, s->v.For.orelse); @@ -3753,7 +3750,7 @@ compiler_async_comprehension_generator(struct compiler *c, _Py_IDENTIFIER(StopAsyncIteration); comprehension_ty gen; - basicblock *anchor, *skip, *if_cleanup, *try, + basicblock *anchor, *if_cleanup, *try, *after_try, *except, *try_cleanup; Py_ssize_t i, n; @@ -3766,13 +3763,12 @@ compiler_async_comprehension_generator(struct compiler *c, after_try = compiler_new_block(c); try_cleanup = compiler_new_block(c); except = compiler_new_block(c); - skip = compiler_new_block(c); if_cleanup = compiler_new_block(c); anchor = compiler_new_block(c); - if (skip == NULL || if_cleanup == NULL || anchor == NULL || + if (if_cleanup == NULL || anchor == NULL || try == NULL || after_try == NULL || - except == NULL || after_try == NULL) { + except == NULL || try_cleanup == NULL) { return 0; } @@ -3866,8 +3862,6 @@ compiler_async_comprehension_generator(struct compiler *c, default: return 0; } - - compiler_use_next_block(c, skip); } compiler_use_next_block(c, if_cleanup); ADDOP_JABS(c, JUMP_ABSOLUTE, try); From webhook-mailer at python.org Sat Mar 10 17:08:35 2018 From: webhook-mailer at python.org (Berker Peksag) Date: Sat, 10 Mar 2018 22:08:35 -0000 Subject: [Python-checkins] bpo-27645: Add support for native backup facility of SQLite (GH-4238) Message-ID: <mailman.26.1520719717.1871.python-checkins@python.org> https://github.com/python/cpython/commit/d7aed4102d2a40c74553240c7f03585624d27aea commit: d7aed4102d2a40c74553240c7f03585624d27aea branch: master author: Emanuele Gaifas <lelegaifax at gmail.com> committer: Berker Peksag <berker.peksag at gmail.com> date: 2018-03-11T01:08:31+03:00 summary: bpo-27645: Add support for native backup facility of SQLite (GH-4238) files: A Lib/sqlite3/test/backup.py A Misc/NEWS.d/next/Library/2017-10-05-20-41-48.bpo-27645.1Y_Wag.rst M Doc/library/sqlite3.rst M Doc/whatsnew/3.7.rst M Lib/test/test_sqlite.py M Modules/_sqlite/connection.c M Modules/_sqlite/module.c diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst index e7676a9f3a50..d7eaea638f82 100644 --- a/Doc/library/sqlite3.rst +++ b/Doc/library/sqlite3.rst @@ -532,6 +532,56 @@ Connection Objects f.write('%s\n' % line) + .. method:: backup(target, *, pages=0, progress=None, name="main", sleep=0.250) + + This method makes a backup of a SQLite database even while it's being accessed + by other clients, or concurrently by the same connection. The copy will be + written into the mandatory argument *target*, that must be another + :class:`Connection` instance. + + By default, or when *pages* is either ``0`` or a negative integer, the entire + database is copied in a single step; otherwise the method performs a loop + copying up to *pages* pages at a time. + + If *progress* is specified, it must either be ``None`` or a callable object that + will be executed at each iteration with three integer arguments, respectively + the *status* of the last iteration, the *remaining* number of pages still to be + copied and the *total* number of pages. + + The *name* argument specifies the database name that will be copied: it must be + a string containing either ``"main"``, the default, to indicate the main + database, ``"temp"`` to indicate the temporary database or the name specified + after the ``AS`` keyword in an ``ATTACH DATABASE`` statement for an attached + database. + + The *sleep* argument specifies the number of seconds to sleep by between + successive attempts to backup remaining pages, can be specified either as an + integer or a floating point value. + + Example 1, copy an existing database into another:: + + import sqlite3 + + def progress(status, remaining, total): + print(f'Copied {total-remaining} of {total} pages...') + + con = sqlite3.connect('existing_db.db') + with sqlite3.connect('backup.db') as bck: + con.backup(bck, pages=1, progress=progress) + + Example 2, copy an existing database into a transient copy:: + + import sqlite3 + + source = sqlite3.connect('existing_db.db') + dest = sqlite3.connect(':memory:') + source.backup(dest) + + Availability: SQLite 3.6.11 or higher + + .. versionadded:: 3.7 + + .. _sqlite3-cursor-objects: Cursor Objects diff --git a/Doc/whatsnew/3.7.rst b/Doc/whatsnew/3.7.rst index 76e1f7b36b0b..fc5f5ab77095 100644 --- a/Doc/whatsnew/3.7.rst +++ b/Doc/whatsnew/3.7.rst @@ -630,6 +630,15 @@ can be set within the scope of a group. ``'^$'`` or ``(?=-)`` that matches an empty string. (Contributed by Serhiy Storchaka in :issue:`25054`.) + +sqlite3 +------- + +:class:`sqlite3.Connection` now exposes a :class:`~sqlite3.Connection.backup` +method, if the underlying SQLite library is at version 3.6.11 or higher. +(Contributed by Lele Gaifax in :issue:`27645`.) + + ssl --- diff --git a/Lib/sqlite3/test/backup.py b/Lib/sqlite3/test/backup.py new file mode 100644 index 000000000000..784702fb46e4 --- /dev/null +++ b/Lib/sqlite3/test/backup.py @@ -0,0 +1,162 @@ +import sqlite3 as sqlite +import unittest + + + at unittest.skipIf(sqlite.sqlite_version_info < (3, 6, 11), "Backup API not supported") +class BackupTests(unittest.TestCase): + def setUp(self): + cx = self.cx = sqlite.connect(":memory:") + cx.execute('CREATE TABLE foo (key INTEGER)') + cx.executemany('INSERT INTO foo (key) VALUES (?)', [(3,), (4,)]) + cx.commit() + + def tearDown(self): + self.cx.close() + + def verify_backup(self, bckcx): + result = bckcx.execute("SELECT key FROM foo ORDER BY key").fetchall() + self.assertEqual(result[0][0], 3) + self.assertEqual(result[1][0], 4) + + def test_bad_target_none(self): + with self.assertRaises(TypeError): + self.cx.backup(None) + + def test_bad_target_filename(self): + with self.assertRaises(TypeError): + self.cx.backup('some_file_name.db') + + def test_bad_target_same_connection(self): + with self.assertRaises(ValueError): + self.cx.backup(self.cx) + + def test_bad_target_closed_connection(self): + bck = sqlite.connect(':memory:') + bck.close() + with self.assertRaises(sqlite.ProgrammingError): + self.cx.backup(bck) + + def test_bad_target_in_transaction(self): + bck = sqlite.connect(':memory:') + bck.execute('CREATE TABLE bar (key INTEGER)') + bck.executemany('INSERT INTO bar (key) VALUES (?)', [(3,), (4,)]) + with self.assertRaises(sqlite.OperationalError) as cm: + self.cx.backup(bck) + if sqlite.sqlite_version_info < (3, 8, 7): + self.assertEqual(str(cm.exception), 'target is in transaction') + + def test_keyword_only_args(self): + with self.assertRaises(TypeError): + with sqlite.connect(':memory:') as bck: + self.cx.backup(bck, 1) + + def test_simple(self): + with sqlite.connect(':memory:') as bck: + self.cx.backup(bck) + self.verify_backup(bck) + + def test_progress(self): + journal = [] + + def progress(status, remaining, total): + journal.append(status) + + with sqlite.connect(':memory:') as bck: + self.cx.backup(bck, pages=1, progress=progress) + self.verify_backup(bck) + + self.assertEqual(len(journal), 2) + self.assertEqual(journal[0], sqlite.SQLITE_OK) + self.assertEqual(journal[1], sqlite.SQLITE_DONE) + + def test_progress_all_pages_at_once_1(self): + journal = [] + + def progress(status, remaining, total): + journal.append(remaining) + + with sqlite.connect(':memory:') as bck: + self.cx.backup(bck, progress=progress) + self.verify_backup(bck) + + self.assertEqual(len(journal), 1) + self.assertEqual(journal[0], 0) + + def test_progress_all_pages_at_once_2(self): + journal = [] + + def progress(status, remaining, total): + journal.append(remaining) + + with sqlite.connect(':memory:') as bck: + self.cx.backup(bck, pages=-1, progress=progress) + self.verify_backup(bck) + + self.assertEqual(len(journal), 1) + self.assertEqual(journal[0], 0) + + def test_non_callable_progress(self): + with self.assertRaises(TypeError) as cm: + with sqlite.connect(':memory:') as bck: + self.cx.backup(bck, pages=1, progress='bar') + self.assertEqual(str(cm.exception), 'progress argument must be a callable') + + def test_modifying_progress(self): + journal = [] + + def progress(status, remaining, total): + if not journal: + self.cx.execute('INSERT INTO foo (key) VALUES (?)', (remaining+1000,)) + self.cx.commit() + journal.append(remaining) + + with sqlite.connect(':memory:') as bck: + self.cx.backup(bck, pages=1, progress=progress) + self.verify_backup(bck) + + result = bck.execute("SELECT key FROM foo" + " WHERE key >= 1000" + " ORDER BY key").fetchall() + self.assertEqual(result[0][0], 1001) + + self.assertEqual(len(journal), 3) + self.assertEqual(journal[0], 1) + self.assertEqual(journal[1], 1) + self.assertEqual(journal[2], 0) + + def test_failing_progress(self): + def progress(status, remaining, total): + raise SystemError('nearly out of space') + + with self.assertRaises(SystemError) as err: + with sqlite.connect(':memory:') as bck: + self.cx.backup(bck, progress=progress) + self.assertEqual(str(err.exception), 'nearly out of space') + + def test_database_source_name(self): + with sqlite.connect(':memory:') as bck: + self.cx.backup(bck, name='main') + with sqlite.connect(':memory:') as bck: + self.cx.backup(bck, name='temp') + with self.assertRaises(sqlite.OperationalError) as cm: + with sqlite.connect(':memory:') as bck: + self.cx.backup(bck, name='non-existing') + self.assertIn( + str(cm.exception), + ['SQL logic error', 'SQL logic error or missing database'] + ) + + self.cx.execute("ATTACH DATABASE ':memory:' AS attached_db") + self.cx.execute('CREATE TABLE attached_db.foo (key INTEGER)') + self.cx.executemany('INSERT INTO attached_db.foo (key) VALUES (?)', [(3,), (4,)]) + self.cx.commit() + with sqlite.connect(':memory:') as bck: + self.cx.backup(bck, name='attached_db') + self.verify_backup(bck) + + +def suite(): + return unittest.makeSuite(BackupTests) + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_sqlite.py b/Lib/test/test_sqlite.py index adfcd9994575..9564da35193f 100644 --- a/Lib/test/test_sqlite.py +++ b/Lib/test/test_sqlite.py @@ -7,7 +7,7 @@ import sqlite3 from sqlite3.test import (dbapi, types, userfunctions, factory, transactions, hooks, regression, - dump) + dump, backup) def load_tests(*args): if test.support.verbose: @@ -18,7 +18,8 @@ def load_tests(*args): userfunctions.suite(), factory.suite(), transactions.suite(), hooks.suite(), regression.suite(), - dump.suite()]) + dump.suite(), + backup.suite()]) if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS.d/next/Library/2017-10-05-20-41-48.bpo-27645.1Y_Wag.rst b/Misc/NEWS.d/next/Library/2017-10-05-20-41-48.bpo-27645.1Y_Wag.rst new file mode 100644 index 000000000000..c4b7185614a5 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2017-10-05-20-41-48.bpo-27645.1Y_Wag.rst @@ -0,0 +1,3 @@ +:class:`sqlite3.Connection` now exposes a :class:`~sqlite3.Connection.backup` +method, if the underlying SQLite library is at version 3.6.11 +or higher. Patch by Lele Gaifax. diff --git a/Modules/_sqlite/connection.c b/Modules/_sqlite/connection.c index 3e83fb662bba..14b6a2774e78 100644 --- a/Modules/_sqlite/connection.c +++ b/Modules/_sqlite/connection.c @@ -41,6 +41,10 @@ #endif #endif +#if SQLITE_VERSION_NUMBER >= 3006011 +#define HAVE_BACKUP_API +#endif + _Py_IDENTIFIER(cursor); static const char * const begin_statements[] = { @@ -1447,6 +1451,137 @@ pysqlite_connection_iterdump(pysqlite_Connection* self, PyObject* args) return retval; } +#ifdef HAVE_BACKUP_API +static PyObject * +pysqlite_connection_backup(pysqlite_Connection *self, PyObject *args, PyObject *kwds) +{ + PyObject *target = NULL; + int pages = -1; + PyObject *progress = Py_None; + const char *name = "main"; + int rc; + int callback_error = 0; + double sleep_secs = 0.250; + sqlite3 *bck_conn; + sqlite3_backup *bck_handle; + static char *keywords[] = {"target", "pages", "progress", "name", "sleep", NULL}; + + if (!PyArg_ParseTupleAndKeywords(args, kwds, "O!|$iOsd:backup", keywords, + &pysqlite_ConnectionType, &target, + &pages, &progress, &name, &sleep_secs)) { + return NULL; + } + + if (!pysqlite_check_connection((pysqlite_Connection *)target)) { + return NULL; + } + + if ((pysqlite_Connection *)target == self) { + PyErr_SetString(PyExc_ValueError, "target cannot be the same connection instance"); + return NULL; + } + +#if SQLITE_VERSION_NUMBER < 3008007 + /* Since 3.8.7 this is already done, per commit + https://www.sqlite.org/src/info/169b5505498c0a7e */ + if (!sqlite3_get_autocommit(((pysqlite_Connection *)target)->db)) { + PyErr_SetString(pysqlite_OperationalError, "target is in transaction"); + return NULL; + } +#endif + + if (progress != Py_None && !PyCallable_Check(progress)) { + PyErr_SetString(PyExc_TypeError, "progress argument must be a callable"); + return NULL; + } + + if (pages == 0) { + pages = -1; + } + + bck_conn = ((pysqlite_Connection *)target)->db; + + Py_BEGIN_ALLOW_THREADS + bck_handle = sqlite3_backup_init(bck_conn, "main", self->db, name); + Py_END_ALLOW_THREADS + + if (bck_handle) { + do { + Py_BEGIN_ALLOW_THREADS + rc = sqlite3_backup_step(bck_handle, pages); + Py_END_ALLOW_THREADS + + if (progress != Py_None) { + PyObject *res; + + res = PyObject_CallFunction(progress, "iii", rc, + sqlite3_backup_remaining(bck_handle), + sqlite3_backup_pagecount(bck_handle)); + if (res == NULL) { + /* User's callback raised an error: interrupt the loop and + propagate it. */ + callback_error = 1; + rc = -1; + } else { + Py_DECREF(res); + } + } + + /* Sleep for a while if there are still further pages to copy and + the engine could not make any progress */ + if (rc == SQLITE_BUSY || rc == SQLITE_LOCKED) { + Py_BEGIN_ALLOW_THREADS + sqlite3_sleep(sleep_secs * 1000.0); + Py_END_ALLOW_THREADS + } + } while (rc == SQLITE_OK || rc == SQLITE_BUSY || rc == SQLITE_LOCKED); + + Py_BEGIN_ALLOW_THREADS + rc = sqlite3_backup_finish(bck_handle); + Py_END_ALLOW_THREADS + } else { + rc = _pysqlite_seterror(bck_conn, NULL); + } + + if (!callback_error && rc != SQLITE_OK) { + /* We cannot use _pysqlite_seterror() here because the backup APIs do + not set the error status on the connection object, but rather on + the backup handle. */ + if (rc == SQLITE_NOMEM) { + (void)PyErr_NoMemory(); + } else { +#if SQLITE_VERSION_NUMBER > 3007015 + PyErr_SetString(pysqlite_OperationalError, sqlite3_errstr(rc)); +#else + switch (rc) { + case SQLITE_READONLY: + PyErr_SetString(pysqlite_OperationalError, + "attempt to write a readonly database"); + break; + case SQLITE_BUSY: + PyErr_SetString(pysqlite_OperationalError, "database is locked"); + break; + case SQLITE_LOCKED: + PyErr_SetString(pysqlite_OperationalError, + "database table is locked"); + break; + default: + PyErr_Format(pysqlite_OperationalError, + "unrecognized error code: %d", rc); + break; + } +#endif + } + } + + if (!callback_error && rc == SQLITE_OK) { + Py_RETURN_NONE; + } else { + return NULL; + } +} +#endif + static PyObject * pysqlite_connection_create_collation(pysqlite_Connection* self, PyObject* args) { @@ -1619,6 +1754,10 @@ static PyMethodDef connection_methods[] = { PyDoc_STR("Abort any pending database operation. Non-standard.")}, {"iterdump", (PyCFunction)pysqlite_connection_iterdump, METH_NOARGS, PyDoc_STR("Returns iterator to the dump of the database in an SQL text format. Non-standard.")}, + #ifdef HAVE_BACKUP_API + {"backup", (PyCFunction)pysqlite_connection_backup, METH_VARARGS | METH_KEYWORDS, + PyDoc_STR("Makes a backup of the database. Non-standard.")}, + #endif {"__enter__", (PyCFunction)pysqlite_connection_enter, METH_NOARGS, PyDoc_STR("For context manager. Non-standard.")}, {"__exit__", (PyCFunction)pysqlite_connection_exit, METH_VARARGS, diff --git a/Modules/_sqlite/module.c b/Modules/_sqlite/module.c index 879c66692bc9..6befa073dcf0 100644 --- a/Modules/_sqlite/module.c +++ b/Modules/_sqlite/module.c @@ -322,6 +322,9 @@ static const IntConstantPair _int_constants[] = { #endif #if SQLITE_VERSION_NUMBER >= 3008003 {"SQLITE_RECURSIVE", SQLITE_RECURSIVE}, +#endif +#if SQLITE_VERSION_NUMBER >= 3006011 + {"SQLITE_DONE", SQLITE_DONE}, #endif {(char*)NULL, 0} }; From webhook-mailer at python.org Sat Mar 10 17:29:22 2018 From: webhook-mailer at python.org (Berker Peksag) Date: Sat, 10 Mar 2018 22:29:22 -0000 Subject: [Python-checkins] bpo-27645: Add support for native backup facility of SQLite (GH-4238) Message-ID: <mailman.27.1520720963.1871.python-checkins@python.org> https://github.com/python/cpython/commit/e8a5a92037b1f27809806bb87c17976d2d48d3e9 commit: e8a5a92037b1f27809806bb87c17976d2d48d3e9 branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Berker Peksag <berker.peksag at gmail.com> date: 2018-03-11T01:29:19+03:00 summary: bpo-27645: Add support for native backup facility of SQLite (GH-4238) (cherry picked from commit d7aed4102d2a40c74553240c7f03585624d27aea) Co-authored-by: Emanuele Gaifas <lelegaifax at gmail.com> files: A Lib/sqlite3/test/backup.py A Misc/NEWS.d/next/Library/2017-10-05-20-41-48.bpo-27645.1Y_Wag.rst M Doc/library/sqlite3.rst M Doc/whatsnew/3.7.rst M Lib/test/test_sqlite.py M Modules/_sqlite/connection.c M Modules/_sqlite/module.c diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst index e7676a9f3a50..d7eaea638f82 100644 --- a/Doc/library/sqlite3.rst +++ b/Doc/library/sqlite3.rst @@ -532,6 +532,56 @@ Connection Objects f.write('%s\n' % line) + .. method:: backup(target, *, pages=0, progress=None, name="main", sleep=0.250) + + This method makes a backup of a SQLite database even while it's being accessed + by other clients, or concurrently by the same connection. The copy will be + written into the mandatory argument *target*, that must be another + :class:`Connection` instance. + + By default, or when *pages* is either ``0`` or a negative integer, the entire + database is copied in a single step; otherwise the method performs a loop + copying up to *pages* pages at a time. + + If *progress* is specified, it must either be ``None`` or a callable object that + will be executed at each iteration with three integer arguments, respectively + the *status* of the last iteration, the *remaining* number of pages still to be + copied and the *total* number of pages. + + The *name* argument specifies the database name that will be copied: it must be + a string containing either ``"main"``, the default, to indicate the main + database, ``"temp"`` to indicate the temporary database or the name specified + after the ``AS`` keyword in an ``ATTACH DATABASE`` statement for an attached + database. + + The *sleep* argument specifies the number of seconds to sleep by between + successive attempts to backup remaining pages, can be specified either as an + integer or a floating point value. + + Example 1, copy an existing database into another:: + + import sqlite3 + + def progress(status, remaining, total): + print(f'Copied {total-remaining} of {total} pages...') + + con = sqlite3.connect('existing_db.db') + with sqlite3.connect('backup.db') as bck: + con.backup(bck, pages=1, progress=progress) + + Example 2, copy an existing database into a transient copy:: + + import sqlite3 + + source = sqlite3.connect('existing_db.db') + dest = sqlite3.connect(':memory:') + source.backup(dest) + + Availability: SQLite 3.6.11 or higher + + .. versionadded:: 3.7 + + .. _sqlite3-cursor-objects: Cursor Objects diff --git a/Doc/whatsnew/3.7.rst b/Doc/whatsnew/3.7.rst index 76e1f7b36b0b..fc5f5ab77095 100644 --- a/Doc/whatsnew/3.7.rst +++ b/Doc/whatsnew/3.7.rst @@ -630,6 +630,15 @@ can be set within the scope of a group. ``'^$'`` or ``(?=-)`` that matches an empty string. (Contributed by Serhiy Storchaka in :issue:`25054`.) + +sqlite3 +------- + +:class:`sqlite3.Connection` now exposes a :class:`~sqlite3.Connection.backup` +method, if the underlying SQLite library is at version 3.6.11 or higher. +(Contributed by Lele Gaifax in :issue:`27645`.) + + ssl --- diff --git a/Lib/sqlite3/test/backup.py b/Lib/sqlite3/test/backup.py new file mode 100644 index 000000000000..784702fb46e4 --- /dev/null +++ b/Lib/sqlite3/test/backup.py @@ -0,0 +1,162 @@ +import sqlite3 as sqlite +import unittest + + + at unittest.skipIf(sqlite.sqlite_version_info < (3, 6, 11), "Backup API not supported") +class BackupTests(unittest.TestCase): + def setUp(self): + cx = self.cx = sqlite.connect(":memory:") + cx.execute('CREATE TABLE foo (key INTEGER)') + cx.executemany('INSERT INTO foo (key) VALUES (?)', [(3,), (4,)]) + cx.commit() + + def tearDown(self): + self.cx.close() + + def verify_backup(self, bckcx): + result = bckcx.execute("SELECT key FROM foo ORDER BY key").fetchall() + self.assertEqual(result[0][0], 3) + self.assertEqual(result[1][0], 4) + + def test_bad_target_none(self): + with self.assertRaises(TypeError): + self.cx.backup(None) + + def test_bad_target_filename(self): + with self.assertRaises(TypeError): + self.cx.backup('some_file_name.db') + + def test_bad_target_same_connection(self): + with self.assertRaises(ValueError): + self.cx.backup(self.cx) + + def test_bad_target_closed_connection(self): + bck = sqlite.connect(':memory:') + bck.close() + with self.assertRaises(sqlite.ProgrammingError): + self.cx.backup(bck) + + def test_bad_target_in_transaction(self): + bck = sqlite.connect(':memory:') + bck.execute('CREATE TABLE bar (key INTEGER)') + bck.executemany('INSERT INTO bar (key) VALUES (?)', [(3,), (4,)]) + with self.assertRaises(sqlite.OperationalError) as cm: + self.cx.backup(bck) + if sqlite.sqlite_version_info < (3, 8, 7): + self.assertEqual(str(cm.exception), 'target is in transaction') + + def test_keyword_only_args(self): + with self.assertRaises(TypeError): + with sqlite.connect(':memory:') as bck: + self.cx.backup(bck, 1) + + def test_simple(self): + with sqlite.connect(':memory:') as bck: + self.cx.backup(bck) + self.verify_backup(bck) + + def test_progress(self): + journal = [] + + def progress(status, remaining, total): + journal.append(status) + + with sqlite.connect(':memory:') as bck: + self.cx.backup(bck, pages=1, progress=progress) + self.verify_backup(bck) + + self.assertEqual(len(journal), 2) + self.assertEqual(journal[0], sqlite.SQLITE_OK) + self.assertEqual(journal[1], sqlite.SQLITE_DONE) + + def test_progress_all_pages_at_once_1(self): + journal = [] + + def progress(status, remaining, total): + journal.append(remaining) + + with sqlite.connect(':memory:') as bck: + self.cx.backup(bck, progress=progress) + self.verify_backup(bck) + + self.assertEqual(len(journal), 1) + self.assertEqual(journal[0], 0) + + def test_progress_all_pages_at_once_2(self): + journal = [] + + def progress(status, remaining, total): + journal.append(remaining) + + with sqlite.connect(':memory:') as bck: + self.cx.backup(bck, pages=-1, progress=progress) + self.verify_backup(bck) + + self.assertEqual(len(journal), 1) + self.assertEqual(journal[0], 0) + + def test_non_callable_progress(self): + with self.assertRaises(TypeError) as cm: + with sqlite.connect(':memory:') as bck: + self.cx.backup(bck, pages=1, progress='bar') + self.assertEqual(str(cm.exception), 'progress argument must be a callable') + + def test_modifying_progress(self): + journal = [] + + def progress(status, remaining, total): + if not journal: + self.cx.execute('INSERT INTO foo (key) VALUES (?)', (remaining+1000,)) + self.cx.commit() + journal.append(remaining) + + with sqlite.connect(':memory:') as bck: + self.cx.backup(bck, pages=1, progress=progress) + self.verify_backup(bck) + + result = bck.execute("SELECT key FROM foo" + " WHERE key >= 1000" + " ORDER BY key").fetchall() + self.assertEqual(result[0][0], 1001) + + self.assertEqual(len(journal), 3) + self.assertEqual(journal[0], 1) + self.assertEqual(journal[1], 1) + self.assertEqual(journal[2], 0) + + def test_failing_progress(self): + def progress(status, remaining, total): + raise SystemError('nearly out of space') + + with self.assertRaises(SystemError) as err: + with sqlite.connect(':memory:') as bck: + self.cx.backup(bck, progress=progress) + self.assertEqual(str(err.exception), 'nearly out of space') + + def test_database_source_name(self): + with sqlite.connect(':memory:') as bck: + self.cx.backup(bck, name='main') + with sqlite.connect(':memory:') as bck: + self.cx.backup(bck, name='temp') + with self.assertRaises(sqlite.OperationalError) as cm: + with sqlite.connect(':memory:') as bck: + self.cx.backup(bck, name='non-existing') + self.assertIn( + str(cm.exception), + ['SQL logic error', 'SQL logic error or missing database'] + ) + + self.cx.execute("ATTACH DATABASE ':memory:' AS attached_db") + self.cx.execute('CREATE TABLE attached_db.foo (key INTEGER)') + self.cx.executemany('INSERT INTO attached_db.foo (key) VALUES (?)', [(3,), (4,)]) + self.cx.commit() + with sqlite.connect(':memory:') as bck: + self.cx.backup(bck, name='attached_db') + self.verify_backup(bck) + + +def suite(): + return unittest.makeSuite(BackupTests) + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_sqlite.py b/Lib/test/test_sqlite.py index adfcd9994575..9564da35193f 100644 --- a/Lib/test/test_sqlite.py +++ b/Lib/test/test_sqlite.py @@ -7,7 +7,7 @@ import sqlite3 from sqlite3.test import (dbapi, types, userfunctions, factory, transactions, hooks, regression, - dump) + dump, backup) def load_tests(*args): if test.support.verbose: @@ -18,7 +18,8 @@ def load_tests(*args): userfunctions.suite(), factory.suite(), transactions.suite(), hooks.suite(), regression.suite(), - dump.suite()]) + dump.suite(), + backup.suite()]) if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS.d/next/Library/2017-10-05-20-41-48.bpo-27645.1Y_Wag.rst b/Misc/NEWS.d/next/Library/2017-10-05-20-41-48.bpo-27645.1Y_Wag.rst new file mode 100644 index 000000000000..c4b7185614a5 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2017-10-05-20-41-48.bpo-27645.1Y_Wag.rst @@ -0,0 +1,3 @@ +:class:`sqlite3.Connection` now exposes a :class:`~sqlite3.Connection.backup` +method, if the underlying SQLite library is at version 3.6.11 +or higher. Patch by Lele Gaifax. diff --git a/Modules/_sqlite/connection.c b/Modules/_sqlite/connection.c index 3e83fb662bba..14b6a2774e78 100644 --- a/Modules/_sqlite/connection.c +++ b/Modules/_sqlite/connection.c @@ -41,6 +41,10 @@ #endif #endif +#if SQLITE_VERSION_NUMBER >= 3006011 +#define HAVE_BACKUP_API +#endif + _Py_IDENTIFIER(cursor); static const char * const begin_statements[] = { @@ -1447,6 +1451,137 @@ pysqlite_connection_iterdump(pysqlite_Connection* self, PyObject* args) return retval; } +#ifdef HAVE_BACKUP_API +static PyObject * +pysqlite_connection_backup(pysqlite_Connection *self, PyObject *args, PyObject *kwds) +{ + PyObject *target = NULL; + int pages = -1; + PyObject *progress = Py_None; + const char *name = "main"; + int rc; + int callback_error = 0; + double sleep_secs = 0.250; + sqlite3 *bck_conn; + sqlite3_backup *bck_handle; + static char *keywords[] = {"target", "pages", "progress", "name", "sleep", NULL}; + + if (!PyArg_ParseTupleAndKeywords(args, kwds, "O!|$iOsd:backup", keywords, + &pysqlite_ConnectionType, &target, + &pages, &progress, &name, &sleep_secs)) { + return NULL; + } + + if (!pysqlite_check_connection((pysqlite_Connection *)target)) { + return NULL; + } + + if ((pysqlite_Connection *)target == self) { + PyErr_SetString(PyExc_ValueError, "target cannot be the same connection instance"); + return NULL; + } + +#if SQLITE_VERSION_NUMBER < 3008007 + /* Since 3.8.7 this is already done, per commit + https://www.sqlite.org/src/info/169b5505498c0a7e */ + if (!sqlite3_get_autocommit(((pysqlite_Connection *)target)->db)) { + PyErr_SetString(pysqlite_OperationalError, "target is in transaction"); + return NULL; + } +#endif + + if (progress != Py_None && !PyCallable_Check(progress)) { + PyErr_SetString(PyExc_TypeError, "progress argument must be a callable"); + return NULL; + } + + if (pages == 0) { + pages = -1; + } + + bck_conn = ((pysqlite_Connection *)target)->db; + + Py_BEGIN_ALLOW_THREADS + bck_handle = sqlite3_backup_init(bck_conn, "main", self->db, name); + Py_END_ALLOW_THREADS + + if (bck_handle) { + do { + Py_BEGIN_ALLOW_THREADS + rc = sqlite3_backup_step(bck_handle, pages); + Py_END_ALLOW_THREADS + + if (progress != Py_None) { + PyObject *res; + + res = PyObject_CallFunction(progress, "iii", rc, + sqlite3_backup_remaining(bck_handle), + sqlite3_backup_pagecount(bck_handle)); + if (res == NULL) { + /* User's callback raised an error: interrupt the loop and + propagate it. */ + callback_error = 1; + rc = -1; + } else { + Py_DECREF(res); + } + } + + /* Sleep for a while if there are still further pages to copy and + the engine could not make any progress */ + if (rc == SQLITE_BUSY || rc == SQLITE_LOCKED) { + Py_BEGIN_ALLOW_THREADS + sqlite3_sleep(sleep_secs * 1000.0); + Py_END_ALLOW_THREADS + } + } while (rc == SQLITE_OK || rc == SQLITE_BUSY || rc == SQLITE_LOCKED); + + Py_BEGIN_ALLOW_THREADS + rc = sqlite3_backup_finish(bck_handle); + Py_END_ALLOW_THREADS + } else { + rc = _pysqlite_seterror(bck_conn, NULL); + } + + if (!callback_error && rc != SQLITE_OK) { + /* We cannot use _pysqlite_seterror() here because the backup APIs do + not set the error status on the connection object, but rather on + the backup handle. */ + if (rc == SQLITE_NOMEM) { + (void)PyErr_NoMemory(); + } else { +#if SQLITE_VERSION_NUMBER > 3007015 + PyErr_SetString(pysqlite_OperationalError, sqlite3_errstr(rc)); +#else + switch (rc) { + case SQLITE_READONLY: + PyErr_SetString(pysqlite_OperationalError, + "attempt to write a readonly database"); + break; + case SQLITE_BUSY: + PyErr_SetString(pysqlite_OperationalError, "database is locked"); + break; + case SQLITE_LOCKED: + PyErr_SetString(pysqlite_OperationalError, + "database table is locked"); + break; + default: + PyErr_Format(pysqlite_OperationalError, + "unrecognized error code: %d", rc); + break; + } +#endif + } + } + + if (!callback_error && rc == SQLITE_OK) { + Py_RETURN_NONE; + } else { + return NULL; + } +} +#endif + static PyObject * pysqlite_connection_create_collation(pysqlite_Connection* self, PyObject* args) { @@ -1619,6 +1754,10 @@ static PyMethodDef connection_methods[] = { PyDoc_STR("Abort any pending database operation. Non-standard.")}, {"iterdump", (PyCFunction)pysqlite_connection_iterdump, METH_NOARGS, PyDoc_STR("Returns iterator to the dump of the database in an SQL text format. Non-standard.")}, + #ifdef HAVE_BACKUP_API + {"backup", (PyCFunction)pysqlite_connection_backup, METH_VARARGS | METH_KEYWORDS, + PyDoc_STR("Makes a backup of the database. Non-standard.")}, + #endif {"__enter__", (PyCFunction)pysqlite_connection_enter, METH_NOARGS, PyDoc_STR("For context manager. Non-standard.")}, {"__exit__", (PyCFunction)pysqlite_connection_exit, METH_VARARGS, diff --git a/Modules/_sqlite/module.c b/Modules/_sqlite/module.c index 879c66692bc9..6befa073dcf0 100644 --- a/Modules/_sqlite/module.c +++ b/Modules/_sqlite/module.c @@ -322,6 +322,9 @@ static const IntConstantPair _int_constants[] = { #endif #if SQLITE_VERSION_NUMBER >= 3008003 {"SQLITE_RECURSIVE", SQLITE_RECURSIVE}, +#endif +#if SQLITE_VERSION_NUMBER >= 3006011 + {"SQLITE_DONE", SQLITE_DONE}, #endif {(char*)NULL, 0} }; From webhook-mailer at python.org Sat Mar 10 18:11:50 2018 From: webhook-mailer at python.org (Miss Islington (bot)) Date: Sat, 10 Mar 2018 23:11:50 -0000 Subject: [Python-checkins] bpo-32836: Remove obsolete code from symtable pass (GH-5680) Message-ID: <mailman.28.1520723511.1871.python-checkins@python.org> https://github.com/python/cpython/commit/5506d603021518eaaa89e7037905f7a698c5e95c commit: 5506d603021518eaaa89e7037905f7a698c5e95c branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: GitHub <noreply at github.com> date: 2018-03-10T15:11:47-08:00 summary: bpo-32836: Remove obsolete code from symtable pass (GH-5680) When comprehensions switched to using a nested scope, the old code for generating a temporary name to hold the accumulation target became redundant, but was never actually removed. Patch by Nitish Chandra. (cherry picked from commit 3a087beddd9f0955eb9080a6fd1499ff89ca74bf) Co-authored-by: Nitish Chandra <nitishchandrachinta at gmail.com> files: A Misc/NEWS.d/next/Core and Builtins/2018-02-14-12-35-47.bpo-32836.bThJnx.rst M Include/symtable.h M Python/symtable.c diff --git a/Include/symtable.h b/Include/symtable.h index 86ae3c28e878..007f88db40e7 100644 --- a/Include/symtable.h +++ b/Include/symtable.h @@ -60,7 +60,6 @@ typedef struct _symtable_entry { int ste_col_offset; /* offset of first line of block */ int ste_opt_lineno; /* lineno of last exec or import * */ int ste_opt_col_offset; /* offset of last exec or import * */ - int ste_tmpname; /* counter for listcomp temp vars */ struct symtable *ste_table; } PySTEntryObject; diff --git a/Misc/NEWS.d/next/Core and Builtins/2018-02-14-12-35-47.bpo-32836.bThJnx.rst b/Misc/NEWS.d/next/Core and Builtins/2018-02-14-12-35-47.bpo-32836.bThJnx.rst new file mode 100644 index 000000000000..4eeb9aa2e52c --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2018-02-14-12-35-47.bpo-32836.bThJnx.rst @@ -0,0 +1 @@ +Don't use temporary variables in cases of list/dict/set comprehensions diff --git a/Python/symtable.c b/Python/symtable.c index bbac25cf3767..1c328a996140 100644 --- a/Python/symtable.c +++ b/Python/symtable.c @@ -69,7 +69,6 @@ ste_new(struct symtable *st, identifier name, _Py_block_ty block, ste->ste_varkeywords = 0; ste->ste_opt_lineno = 0; ste->ste_opt_col_offset = 0; - ste->ste_tmpname = 0; ste->ste_lineno = lineno; ste->ste_col_offset = col_offset; @@ -1082,24 +1081,6 @@ symtable_add_def(struct symtable *st, PyObject *name, int flag) } \ } -static int -symtable_new_tmpname(struct symtable *st) -{ - char tmpname[256]; - identifier tmp; - - PyOS_snprintf(tmpname, sizeof(tmpname), "_[%d]", - ++st->st_cur->ste_tmpname); - tmp = PyUnicode_InternFromString(tmpname); - if (!tmp) - return 0; - if (!symtable_add_def(st, tmp, DEF_LOCAL)) - return 0; - Py_DECREF(tmp); - return 1; -} - - static int symtable_record_directive(struct symtable *st, identifier name, stmt_ty s) { @@ -1723,7 +1704,6 @@ symtable_handle_comprehension(struct symtable *st, expr_ty e, expr_ty elt, expr_ty value) { int is_generator = (e->kind == GeneratorExp_kind); - int needs_tmp = !is_generator; comprehension_ty outermost = ((comprehension_ty) asdl_seq_GET(generators, 0)); /* Outermost iterator is evaluated in current scope */ @@ -1742,11 +1722,6 @@ symtable_handle_comprehension(struct symtable *st, expr_ty e, symtable_exit_block(st, (void *)e); return 0; } - /* Allocate temporary name if needed */ - if (needs_tmp && !symtable_new_tmpname(st)) { - symtable_exit_block(st, (void *)e); - return 0; - } VISIT(st, expr, outermost->target); VISIT_SEQ(st, expr, outermost->ifs); VISIT_SEQ_TAIL(st, comprehension, generators, 1); From webhook-mailer at python.org Sat Mar 10 18:25:17 2018 From: webhook-mailer at python.org (Miss Islington (bot)) Date: Sat, 10 Mar 2018 23:25:17 -0000 Subject: [Python-checkins] controlflow: Use full example for "5 through 9" (GH-5907) Message-ID: <mailman.29.1520724318.1871.python-checkins@python.org> https://github.com/python/cpython/commit/3ca5efcef53cdafa74f7acafd7bed7f878e7e584 commit: 3ca5efcef53cdafa74f7acafd7bed7f878e7e584 branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: GitHub <noreply at github.com> date: 2018-03-10T15:25:14-08:00 summary: controlflow: Use full example for "5 through 9" (GH-5907) Replace example result of "5 through 9" with complete list: "5, 6, 7, 8, 9". This format is more consistent with the surrounding examples. (cherry picked from commit 83d7062d2dc5eacfef578e072bca4747c346fdae) Co-authored-by: Steven M. Vascellaro <S.Vascellaro at gmail.com> files: M Doc/tutorial/controlflow.rst diff --git a/Doc/tutorial/controlflow.rst b/Doc/tutorial/controlflow.rst index 36b093950e10..30ef159f8b36 100644 --- a/Doc/tutorial/controlflow.rst +++ b/Doc/tutorial/controlflow.rst @@ -105,7 +105,7 @@ is possible to let the range start at another number, or to specify a different increment (even negative; sometimes this is called the 'step'):: range(5, 10) - 5 through 9 + 5, 6, 7, 8, 9 range(0, 10, 3) 0, 3, 6, 9 From webhook-mailer at python.org Sat Mar 10 19:38:53 2018 From: webhook-mailer at python.org (Berker Peksag) Date: Sun, 11 Mar 2018 00:38:53 -0000 Subject: [Python-checkins] bpo-27645: Skip test_bad_target_in_transaction if SQLite == 3.8.7.1 (GH-6067) Message-ID: <mailman.30.1520728734.1871.python-checkins@python.org> https://github.com/python/cpython/commit/7280a4eef5fbe17e8ac82afd71fe6e51843240a5 commit: 7280a4eef5fbe17e8ac82afd71fe6e51843240a5 branch: master author: Berker Peksag <berker.peksag at gmail.com> committer: GitHub <noreply at github.com> date: 2018-03-11T03:38:50+03:00 summary: bpo-27645: Skip test_bad_target_in_transaction if SQLite == 3.8.7.1 (GH-6067) files: M Lib/sqlite3/test/backup.py diff --git a/Lib/sqlite3/test/backup.py b/Lib/sqlite3/test/backup.py index 784702fb46e4..2a2148937413 100644 --- a/Lib/sqlite3/test/backup.py +++ b/Lib/sqlite3/test/backup.py @@ -37,6 +37,8 @@ def test_bad_target_closed_connection(self): self.cx.backup(bck) def test_bad_target_in_transaction(self): + if sqlite.sqlite_version_info == (3, 8, 7, 1): + self.skipTest('skip until we debug https://bugs.python.org/issue27645#msg313562') bck = sqlite.connect(':memory:') bck.execute('CREATE TABLE bar (key INTEGER)') bck.executemany('INSERT INTO bar (key) VALUES (?)', [(3,), (4,)]) From webhook-mailer at python.org Sat Mar 10 19:59:14 2018 From: webhook-mailer at python.org (Berker Peksag) Date: Sun, 11 Mar 2018 00:59:14 -0000 Subject: [Python-checkins] bpo-27645: Skip test_bad_target_in_transaction if SQLite == 3.8.7.1 (GH-6067) Message-ID: <mailman.31.1520729955.1871.python-checkins@python.org> https://github.com/python/cpython/commit/c546a62c4d7b37ead10f986554a01d6d593227a1 commit: c546a62c4d7b37ead10f986554a01d6d593227a1 branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Berker Peksag <berker.peksag at gmail.com> date: 2018-03-11T03:59:11+03:00 summary: bpo-27645: Skip test_bad_target_in_transaction if SQLite == 3.8.7.1 (GH-6067) (cherry picked from commit 7280a4eef5fbe17e8ac82afd71fe6e51843240a5) Co-authored-by: Berker Peksag <berker.peksag at gmail.com> files: M Lib/sqlite3/test/backup.py diff --git a/Lib/sqlite3/test/backup.py b/Lib/sqlite3/test/backup.py index 784702fb46e4..2a2148937413 100644 --- a/Lib/sqlite3/test/backup.py +++ b/Lib/sqlite3/test/backup.py @@ -37,6 +37,8 @@ def test_bad_target_closed_connection(self): self.cx.backup(bck) def test_bad_target_in_transaction(self): + if sqlite.sqlite_version_info == (3, 8, 7, 1): + self.skipTest('skip until we debug https://bugs.python.org/issue27645#msg313562') bck = sqlite.connect(':memory:') bck.execute('CREATE TABLE bar (key INTEGER)') bck.executemany('INSERT INTO bar (key) VALUES (?)', [(3,), (4,)]) From webhook-mailer at python.org Sat Mar 10 20:00:07 2018 From: webhook-mailer at python.org (Berker Peksag) Date: Sun, 11 Mar 2018 01:00:07 -0000 Subject: [Python-checkins] bpo-33045: Fix typos in SSL documentation (GH-6065) Message-ID: <mailman.32.1520730007.1871.python-checkins@python.org> https://github.com/python/cpython/commit/9cf8c42f3231d3f066670d087a997bd1278482a0 commit: 9cf8c42f3231d3f066670d087a997bd1278482a0 branch: master author: Matt Eaton <agnosticdev at gmail.com> committer: Berker Peksag <berker.peksag at gmail.com> date: 2018-03-11T04:00:04+03:00 summary: bpo-33045: Fix typos in SSL documentation (GH-6065) files: M Doc/library/ssl.rst diff --git a/Doc/library/ssl.rst b/Doc/library/ssl.rst index 2b4bed413985..8082a383d5ab 100644 --- a/Doc/library/ssl.rst +++ b/Doc/library/ssl.rst @@ -1688,7 +1688,7 @@ to speed up repeated connections from the same clients. .. method:: SSLContext.load_dh_params(dhfile) - Load the key generation parameters for Diffie-Helman (DH) key exchange. + Load the key generation parameters for Diffie-Hellman (DH) key exchange. Using DH key exchange improves forward secrecy at the expense of computational resources (both on the server and on the client). The *dhfile* parameter should be the path to a file containing DH @@ -2603,7 +2603,7 @@ with LibreSSL. Documentation of underlying :mod:`socket` class `SSL/TLS Strong Encryption: An Introduction <https://httpd.apache.org/docs/trunk/en/ssl/ssl_intro.html>`_ - Intro from the Apache webserver documentation + Intro from the Apache HTTP Server documentation `RFC 1422: Privacy Enhancement for Internet Electronic Mail: Part II: Certificate-Based Key Management <https://www.ietf.org/rfc/rfc1422>`_ Steve Kent From webhook-mailer at python.org Sat Mar 10 20:21:29 2018 From: webhook-mailer at python.org (Berker Peksag) Date: Sun, 11 Mar 2018 01:21:29 -0000 Subject: [Python-checkins] bpo-33045: Fix typos in SSL documentation (GH-6065) Message-ID: <mailman.33.1520731290.1871.python-checkins@python.org> https://github.com/python/cpython/commit/17b6c19d39229619de649f5a9e28a46779ef1c51 commit: 17b6c19d39229619de649f5a9e28a46779ef1c51 branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Berker Peksag <berker.peksag at gmail.com> date: 2018-03-11T04:21:27+03:00 summary: bpo-33045: Fix typos in SSL documentation (GH-6065) (cherry picked from commit 9cf8c42f3231d3f066670d087a997bd1278482a0) Co-authored-by: Matt Eaton <agnosticdev at gmail.com> files: M Doc/library/ssl.rst diff --git a/Doc/library/ssl.rst b/Doc/library/ssl.rst index 2b4bed413985..8082a383d5ab 100644 --- a/Doc/library/ssl.rst +++ b/Doc/library/ssl.rst @@ -1688,7 +1688,7 @@ to speed up repeated connections from the same clients. .. method:: SSLContext.load_dh_params(dhfile) - Load the key generation parameters for Diffie-Helman (DH) key exchange. + Load the key generation parameters for Diffie-Hellman (DH) key exchange. Using DH key exchange improves forward secrecy at the expense of computational resources (both on the server and on the client). The *dhfile* parameter should be the path to a file containing DH @@ -2603,7 +2603,7 @@ with LibreSSL. Documentation of underlying :mod:`socket` class `SSL/TLS Strong Encryption: An Introduction <https://httpd.apache.org/docs/trunk/en/ssl/ssl_intro.html>`_ - Intro from the Apache webserver documentation + Intro from the Apache HTTP Server documentation `RFC 1422: Privacy Enhancement for Internet Electronic Mail: Part II: Certificate-Based Key Management <https://www.ietf.org/rfc/rfc1422>`_ Steve Kent From webhook-mailer at python.org Sat Mar 10 20:26:04 2018 From: webhook-mailer at python.org (Berker Peksag) Date: Sun, 11 Mar 2018 01:26:04 -0000 Subject: [Python-checkins] bpo-33045: Fix typos in SSL documentation (GH-6065) Message-ID: <mailman.34.1520731565.1871.python-checkins@python.org> https://github.com/python/cpython/commit/3f439d14ad48f5d4f6017fc814326fb1f6237b53 commit: 3f439d14ad48f5d4f6017fc814326fb1f6237b53 branch: 3.6 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Berker Peksag <berker.peksag at gmail.com> date: 2018-03-11T04:26:01+03:00 summary: bpo-33045: Fix typos in SSL documentation (GH-6065) (cherry picked from commit 9cf8c42f3231d3f066670d087a997bd1278482a0) Co-authored-by: Matt Eaton <agnosticdev at gmail.com> files: M Doc/library/ssl.rst diff --git a/Doc/library/ssl.rst b/Doc/library/ssl.rst index fc68e3071436..677e94534702 100644 --- a/Doc/library/ssl.rst +++ b/Doc/library/ssl.rst @@ -1539,7 +1539,7 @@ to speed up repeated connections from the same clients. .. method:: SSLContext.load_dh_params(dhfile) - Load the key generation parameters for Diffie-Helman (DH) key exchange. + Load the key generation parameters for Diffie-Hellman (DH) key exchange. Using DH key exchange improves forward secrecy at the expense of computational resources (both on the server and on the client). The *dhfile* parameter should be the path to a file containing DH @@ -2343,7 +2343,7 @@ with LibreSSL. Documentation of underlying :mod:`socket` class `SSL/TLS Strong Encryption: An Introduction <https://httpd.apache.org/docs/trunk/en/ssl/ssl_intro.html>`_ - Intro from the Apache webserver documentation + Intro from the Apache HTTP Server documentation `RFC 1422: Privacy Enhancement for Internet Electronic Mail: Part II: Certificate-Based Key Management <https://www.ietf.org/rfc/rfc1422>`_ Steve Kent From webhook-mailer at python.org Sat Mar 10 20:48:17 2018 From: webhook-mailer at python.org (Serhiy Storchaka) Date: Sun, 11 Mar 2018 01:48:17 -0000 Subject: [Python-checkins] bpo-32996: Improve What's New in 3.7. (#5983) Message-ID: <mailman.35.1520732898.1871.python-checkins@python.org> https://github.com/python/cpython/commit/51302a5fcc557e6afc0bf1e3b371f5f37c76dc77 commit: 51302a5fcc557e6afc0bf1e3b371f5f37c76dc77 branch: master author: Serhiy Storchaka <storchaka at gmail.com> committer: GitHub <noreply at github.com> date: 2018-03-11T03:48:14+02:00 summary: bpo-32996: Improve What's New in 3.7. (#5983) files: M Doc/library/cgi.rst M Doc/whatsnew/3.7.rst diff --git a/Doc/library/cgi.rst b/Doc/library/cgi.rst index b60e1cc41e4b..17386b831833 100644 --- a/Doc/library/cgi.rst +++ b/Doc/library/cgi.rst @@ -289,11 +289,13 @@ algorithms implemented in this module in other circumstances. This function is deprecated in this module. Use :func:`urllib.parse.parse_qs` instead. It is maintained here only for backward compatibility. + .. function:: parse_qsl(qs, keep_blank_values=False, strict_parsing=False) This function is deprecated in this module. Use :func:`urllib.parse.parse_qsl` instead. It is maintained here only for backward compatibility. + .. function:: parse_multipart(fp, pdict, encoding="utf-8") Parse input of type :mimetype:`multipart/form-data` (for file uploads). @@ -309,6 +311,10 @@ algorithms implemented in this module in other circumstances. uploaded --- in that case, use the :class:`FieldStorage` class instead which is much more flexible. + .. versionchanged:: 3.7 + Added the *encoding* parameter. For non-file fields, the value is now + a list of strings, not bytes. + .. function:: parse_header(string) diff --git a/Doc/whatsnew/3.7.rst b/Doc/whatsnew/3.7.rst index fc5f5ab77095..a4ed481f67a5 100644 --- a/Doc/whatsnew/3.7.rst +++ b/Doc/whatsnew/3.7.rst @@ -91,17 +91,17 @@ rather than ASCII. The platform support definition in :pep:`11` has also been updated to limit full text handling support to suitably configured non-ASCII based locales. -As part of this change, the default error handler for ``stdin`` and ``stdout`` -is now ``surrogateescape`` (rather than ``strict``) when using any of the -defined coercion target locales (currently ``C.UTF-8``, ``C.utf8``, and -``UTF-8``). The default error handler for ``stderr`` continues to be -``backslashreplace``, regardless of locale. +As part of this change, the default error handler for :data:`~sys.stdin` and +:data:`~sys.stdout` is now ``surrogateescape`` (rather than ``strict``) when +using any of the defined coercion target locales (currently ``C.UTF-8``, +``C.utf8``, and ``UTF-8``). The default error handler for :data:`~sys.stderr` +continues to be ``backslashreplace``, regardless of locale. Locale coercion is silent by default, but to assist in debugging potentially locale related integration problems, explicit warnings (emitted directly on -``stderr`` can be requested by setting ``PYTHONCOERCECLOCALE=warn``. This -setting will also cause the Python runtime to emit a warning if the legacy C -locale remains active when the core interpreter is initialized. +:data:`~sys.stderr` can be requested by setting ``PYTHONCOERCECLOCALE=warn``. +This setting will also cause the Python runtime to emit a warning if the +legacy C locale remains active when the core interpreter is initialized. .. seealso:: @@ -114,9 +114,9 @@ locale remains active when the core interpreter is initialized. PEP 553: Built-in breakpoint() ------------------------------ -:pep:`553` describes a new built-in called ``breakpoint()`` which makes it +:pep:`553` describes a new built-in called :func:`breakpoint` which makes it easy and consistent to enter the Python debugger. Built-in ``breakpoint()`` -calls ``sys.breakpointhook()``. By default, this latter imports ``pdb`` and +calls :func:`sys.breakpointhook`. By default, this latter imports :mod:`pdb` and then calls ``pdb.set_trace()``, but by binding ``sys.breakpointhook()`` to the function of your choosing, ``breakpoint()`` can enter any debugger. Or, the environment variable :envvar:`PYTHONBREAKPOINT` can be set to the callable of @@ -165,13 +165,13 @@ PEP 562: Customization of access to module attributes ----------------------------------------------------- It is sometimes convenient to customize or otherwise have control over access -to module attributes. A typical example is managing deprecation warnings. -Typical workarounds are assigning ``__class__`` of a module object to -a custom subclass of :class:`types.ModuleType` or replacing the ``sys.modules`` -item with a custom wrapper instance. This procedure is now simplified by -recognizing ``__getattr__`` defined directly in a module that would act like -a normal ``__getattr__`` method, except that it will be defined on module -*instances*. +to module attributes. A typical example is managing deprecation warnings. +Typical workarounds are assigning :attr:`~instance.__class__` of a module +object to a custom subclass of :class:`types.ModuleType` or replacing the +:data:`sys.modules` item with a custom wrapper instance. This procedure is +now simplified by recognizing ``__getattr__`` defined directly in a module +that would act like a normal :meth:`__getattr__` method, except that it will +be defined on module *instances*. .. seealso:: @@ -217,7 +217,7 @@ following syntax valid:: ... Since this change breaks compatibility, the new behavior can be enabled -on a per-module basis in Python 3.7 using a ``__future__`` import, like +on a per-module basis in Python 3.7 using a :mod:`__future__` import, like this:: from __future__ import annotations @@ -263,7 +263,7 @@ PEP 565: Show DeprecationWarning in ``__main__`` The default handling of :exc:`DeprecationWarning` has been changed such that these warnings are once more shown by default, but only when the code -triggering them is running directly in the ``__main__`` module. As a result, +triggering them is running directly in the :mod:`__main__` module. As a result, developers of single file scripts and those using Python interactively should once again start seeing deprecation warnings for the APIs they use, but deprecation warnings triggered by imported application, library and framework @@ -275,7 +275,7 @@ between three different deprecation warning behaviours: * :exc:`FutureWarning`: always displayed by default, recommended for warnings intended to be seen by application end users (e.g. for deprecated application configuration settings). -* :exc:`DeprecationWarning`: displayed by default only in ``__main__`` and when +* :exc:`DeprecationWarning`: displayed by default only in :mod:`__main__` and when running tests, recommended for warnings intended to be seen by other Python developers where a version upgrade may result in changed behaviour or an error. @@ -316,11 +316,11 @@ environment variable are added to control the UTF-8 mode. PEP 557: Data Classes --------------------- -Adds a new module ``dataclasses``. It provides a class decorator -``dataclass`` which inspects the class's variable annotations (see +Adds a new module :mod:`dataclasses`. It provides a class decorator +:func:`~dataclasses.dataclass` which inspects the class's variable annotations (see :pep:`526`) and using them, adds methods such as ``__init__``, ``__repr__``, and ``__eq__`` to the class. It is similar to -``typing.NamedTuple``, but also works on classes with mutable +:class:`typing.NamedTuple`, but also works on classes with mutable instances, among other features. For example:: @@ -374,7 +374,7 @@ Hash-based pycs Python has traditionally checked the up-to-dateness of bytecode cache files (i.e., ``.pyc`` files) by comparing the source metadata (last-modified timestamp and size) with source metadata saved in the cache file header when it was -generated. While effective, this invalidation method has its drawbacks. When +generated. While effective, this invalidation method has its drawbacks. When filesystem timestamps are too coarse, Python can miss source updates, leading to user confusion. Additionally, having a timestamp in the cache file is problematic for `build reproduciblity <https://reproducible-builds.org/>`_ and @@ -407,7 +407,8 @@ Other Language Changes whitespace, not only spaces. (Contributed by Robert Xiao in :issue:`28927`.) * :exc:`ImportError` now displays module name and module ``__file__`` path when - ``from ... import ...`` fails. (Contributed by Matthias Bussonnier in :issue:`29546`.) + ``from ... import ...`` fails. (Contributed by Matthias Bussonnier in + :issue:`29546`.) * Circular imports involving absolute imports with binding a submodule to a name are now supported. @@ -453,7 +454,6 @@ the user intermix options and positional arguments on the command line, as is possible in many unix commands. It supports most but not all argparse features. (Contributed by paul.j3 in :issue:`14191`.) - binascii -------- @@ -461,7 +461,6 @@ The :func:`~binascii.b2a_uu` function now accepts an optional *backtick* keyword argument. When it's true, zeros are represented by ``'`'`` instead of spaces. (Contributed by Xiang Zhang in :issue:`30103`.) - calendar -------- @@ -469,14 +468,6 @@ The class :class:`~calendar.HTMLCalendar` has new class attributes which ease the customisation of the CSS classes in the produced HTML calendar. (Contributed by Oz Tiram in :issue:`30095`.) -cgi ---- - -:func:`~cgi.parse_multipart` returns the same results as -:class:`~FieldStorage` : for non-file fields, the value associated to a key -is a list of strings, not bytes. -(Contributed by Pierre Quentel in :issue:`29979`.) - contextlib ---------- @@ -490,8 +481,8 @@ Alexander Mohr and Ilya Kulakov in :issue:`29302`.) cProfile -------- -cProfile command line now accepts `-m module_name` as an alternative to -script path. (Contributed by Sanyam Khurana in :issue:`21862`.) +:mod:`cProfile` command line now accepts ``-m module_name`` as an alternative +to script path. (Contributed by Sanyam Khurana in :issue:`21862`.) crypt ----- @@ -505,10 +496,11 @@ for hashing. (Contributed by Serhiy Storchaka in :issue:`31702`.) datetime -------- -Added the :func:`datetime.datetime.fromisoformat` method, which constructs a -:class:`datetime.datetime` object from a string in one of the formats output -by :func:`datetime.datetime.isoformat`. (Contributed by Paul Ganssle in -:issue:`15873`.) +Added the :meth:`datetime.fromisoformat <datetime.datetime.fromisoformat>` +method, which constructs a :class:`~datetime.datetime` object from a string +in one of the formats output by +:meth:`datetime.isoformat <datetime.datetime.isoformat>`. +(Contributed by Paul Ganssle in :issue:`15873`.) dis --- @@ -521,7 +513,7 @@ classes). (Contributed by Serhiy Storchaka in :issue:`11822`.) distutils --------- -README.rst is now included in the list of distutils standard READMEs and +``README.rst`` is now included in the list of distutils standard READMEs and therefore included in source distributions. (Contributed by Ryan Gonzalez in :issue:`11913`.) @@ -537,28 +529,29 @@ equivalent to CR. http.client ----------- -Add Configurable *blocksize* to ``HTTPConnection`` and -``HTTPSConnection`` for improved upload throughput. +Add configurable *blocksize* to :class:`~http.client.HTTPConnection` and +:class:`~http.client.HTTPSConnection` for improved upload throughput. (Contributed by Nir Soffer in :issue:`31945`.) http.server ----------- :class:`~http.server.SimpleHTTPRequestHandler` supports the HTTP -If-Modified-Since header. The server returns the 304 response status if the +``If-Modified-Since`` header. The server returns the 304 response status if the target file was not modified after the time specified in the header. (Contributed by Pierre Quentel in :issue:`29654`.) -Add the parameter ``directory`` to the :class:`~http.server.SimpleHTTPRequestHandler` -and the ``--directory`` to the command line of the module :mod:`~http.server`. -With this parameter, the server serves the specified directory, by default it uses the current working directory. +Add the parameter *directory* to the :class:`~http.server.SimpleHTTPRequestHandler` +and the ``--directory`` to the command line of the module :mod:`http.server`. +With this parameter, the server serves the specified directory, by default it +uses the current working directory. (Contributed by St?phane Wirtel and Julien Palard in :issue:`28707`.) hmac ---- -The hmac module now has an optimized one-shot :func:`~hmac.digest` function, -which is up to three times faster than :func:`~hmac.HMAC`. +The :mod:`hmac` module now has an optimized one-shot :func:`~hmac.digest` +function, which is up to three times faster than :func:`~hmac.HMAC`. (Contributed by Christian Heimes in :issue:`32433`.) importlib @@ -570,11 +563,11 @@ support the loading of resource from packages. locale ------ -Added another argument *monetary* in :meth:`format_string` of :mod:`locale`. +Added another argument *monetary* in :func:`~locale.format_string` of :mod:`locale`. If *monetary* is true, the conversion uses monetary thousands separator and grouping strings. (Contributed by Garvit in :issue:`10379`.) -The :func:`locale.getpreferredencoding` function now always returns ``'UTF-8'`` +The :func:`~locale.getpreferredencoding` function now always returns ``'UTF-8'`` on Android or in the UTF-8 mode (:option:`-X` ``utf8`` option), the locale and the *do_setlocale* argument are ignored. @@ -593,7 +586,7 @@ Serhiy Storchaka in :issue:`28682`.) Added support for :ref:`file descriptors <path_fd>` in :func:`~os.scandir` on Unix. (Contributed by Serhiy Storchaka in :issue:`25996`.) -New function :func:`os.register_at_fork` allows registering Python callbacks +New function :func:`~os.register_at_fork` allows registering Python callbacks to be executed on a process fork. (Contributed by Antoine Pitrou in :issue:`16500`.) @@ -604,8 +597,8 @@ pdb argument. If given, this is printed to the console just before debugging begins. (Contributed by Barry Warsaw in :issue:`31389`.) -pdb command line now accepts `-m module_name` as an alternative to -script file. (Contributed by Mario Corchero in :issue:`32206`.) +:mod:`pdb` command line now accepts ``-m module_name`` as an alternative to +script file. (Contributed by Mario Corchero in :issue:`32206`.) py_compile ---------- @@ -618,7 +611,6 @@ This allows for guaranteeing files when they are created eagerly. (Contributed by Bernhard M. Wiedemann in :issue:`29708`.) - re -- @@ -642,7 +634,7 @@ method, if the underlying SQLite library is at version 3.6.11 or higher. ssl --- -The ssl module now uses OpenSSL's builtin API instead of +The :mod:`ssl` module now uses OpenSSL's builtin API instead of :func:`~ssl.match_hostname` to check host name or IP address. Values are validated during TLS handshake. Any cert validation error including a failing host name match now raises :exc:`~ssl.SSLCertVerificationError` and @@ -682,10 +674,6 @@ The ssl module has preliminary and experimental support for TLS 1.3 and OpenSSL 1.1.1. (Contributed by Christian Heimes in :issue:`32947`, :issue:`20995`, :issue:`29136`, and :issue:`30622`) -:func:`~ssl.wrap_socket` is deprecated. Documentation has been updated to -recommend :meth:`~ssl.SSLContext.wrap_socket` instead. -(Contributed by Christian Heimes in :issue:`28124`.) - :class:`~ssl.SSLSocket` and :class:`~ssl.SSLObject` no longer have a public constructor. Direct instantiation was never a documented and supported feature. Instances must be created with :class:`~ssl.SSLContext` methods @@ -708,27 +696,24 @@ separately. (Contributed by Barry Warsaw in :issue:`1198569`.) subprocess ---------- -On Windows the default for *close_fds* was changed from :const:`False` to -:const:`True` when redirecting the standard handles. It's now possible to set -*close_fds* to :const:`True` when redirecting the standard handles. See +On Windows the default for *close_fds* was changed from ``False`` to +``True`` when redirecting the standard handles. It's now possible to set +*close_fds* to ``True`` when redirecting the standard handles. See :class:`subprocess.Popen`. -This means that *close_fds* now defaults to :const:`True` on all supported -platforms. +This means that *close_fds* now defaults to ``True`` on all supported +platforms. (Contributed by Segev Finer in :issue:`19764`.) sys --- Added :attr:`sys.flags.dev_mode` flag for the new development mode. -Deprecated :func:`sys.set_coroutine_wrapper` and -:func:`sys.get_coroutine_wrapper`. - - tkinter ------- Added :class:`tkinter.ttk.Spinbox`. +(Contributed by Alan Moore in :issue:`32585`.) time ---- @@ -766,11 +751,12 @@ Peterson.) unittest -------- + Added new command-line option ``-k`` to filter tests to run with a substring or Unix shell-like pattern. For example, ``python -m unittest -k foo`` runs the tests ``foo_tests.SomeTest.test_something``, ``bar_tests.SomeTest.test_foo``, but not ``bar_tests.FooTest.test_something``. - +(Contributed by Jonas Haag in :issue:`32071`.) unittest.mock ------------- @@ -779,7 +765,7 @@ The :const:`~unittest.mock.sentinel` attributes now preserve their identity when they are :mod:`copied <copy>` or :mod:`pickled <pickle>`. (Contributed by Serhiy Storchaka in :issue:`20804`.) -New function :const:`~unittest.mock.seal` will disable the creation of mock +New function :func:`~unittest.mock.seal` will disable the creation of mock children by preventing to get or set any new attribute on the sealed mock. The sealing process is performed recursively. (Contributed by Mario Corchero in :issue:`30541`.) @@ -787,8 +773,8 @@ in :issue:`30541`.) urllib.parse ------------ -:func:`urllib.parse.quote` has been updated from RFC 2396 to RFC 3986, -adding `~` to the set of characters that is never quoted by default. +:func:`urllib.parse.quote` has been updated from :rfc:`2396` to :rfc:`3986`, +adding ``~`` to the set of characters that is never quoted by default. (Contributed by Christian Theune and Ratnadeep Debnath in :issue:`16285`.) uu @@ -832,26 +818,27 @@ better readability. (Contributed by Stefan Behnel in :issue:`31648`.) xmlrpc.server ------------- -:meth:`register_function` of :class:`xmlrpc.server.SimpleXMLRPCDispatcher` and -its subclasses can be used as a decorator. (Contributed by Xiang Zhang in +:meth:`register_function` of :class:`~xmlrpc.server.SimpleXMLRPCDispatcher` and +its subclasses can be used as a decorator. (Contributed by Xiang Zhang in :issue:`7769`.) zipapp ------ -Function :func:`zipapp.create_archive` now accepts an optional *filter* +Function :func:`~zipapp.create_archive` now accepts an optional *filter* argument to allow the user to select which files should be included in the -archive, and an optional *compressed* argument to generate a compressed -archive. +archive. (Contributed by Irmen de Jong in :issue:`31072`.) -A command line option ``--compress`` has also been added to support -compression. +Function :func:`~zipapp.create_archive` now accepts an optional *compressed* +argument to generate a compressed archive. A command line option +``--compress`` has also been added to support compression. +(Contributed by Zhiming Wang in :issue:`31638`.) Optimizations ============= -* Added two new opcodes: ``LOAD_METHOD`` and ``CALL_METHOD`` to avoid +* Added two new opcodes: :opcode:`LOAD_METHOD` and :opcode:`CALL_METHOD` to avoid instantiation of bound method objects for method calls, which results in method calls being faster up to 20%. (Contributed by Yury Selivanov and INADA Naoki in :issue:`26110`.) @@ -882,9 +869,10 @@ Optimizations Python 3.6 by about 10% depending on the pattern. (Contributed by INADA Naoki in :issue:`31671`.) -* :meth:`selectors.EpollSelector.modify`, :meth:`selectors.PollSelector.modify` - and :meth:`selectors.DevpollSelector.modify` may be around 10% faster under - heavy loads. (Contributed by Giampaolo Rodola' in :issue:`30014`) +* :meth:`~selectors.BaseSelector.modify` methods of classes + :class:`selectors.EpollSelector`, :class:`selectors.PollSelector` + and :class:`selectors.DevpollSelector` may be around 10% faster under + heavy loads. (Contributed by Giampaolo Rodola' in :issue:`30014`) * Constant folding is moved from peephole optimizer to new AST optimizer. (Contributed by Eugene Toder and INADA Naoki in :issue:`29469`) @@ -900,6 +888,7 @@ Optimizations constructors when not constructing subclasses. (Contributed by Paul Ganssle in :issue:`32403`) + Build and C API Changes ======================= @@ -955,6 +944,16 @@ Build and C API Changes and access to the UTC singleton with :c:data:`PyDateTime_TimeZone_UTC`. Contributed by Paul Ganssle in :issue:`10381`. +- The type of results of :c:func:`PyThread_start_new_thread` and + :c:func:`PyThread_get_thread_ident`, and the *id* parameter of + :c:func:`PyThreadState_SetAsyncExc` changed from :c:type:`long` to + :c:type:`unsigned long`. + (Contributed by Serhiy Storchaka in :issue:`6532`.) + +- :c:func:`PyUnicode_AsWideCharString` now raises a :exc:`ValueError` if the + second argument is *NULL* and the :c:type:`wchar_t*` string contains null + characters. (Contributed by Serhiy Storchaka in :issue:`30708`.) + Other CPython Implementation Changes ==================================== @@ -1007,13 +1006,13 @@ Deprecated - Methods :meth:`MetaPathFinder.find_module() <importlib.abc.MetaPathFinder.find_module>` (replaced by - :meth:`MetaPathFinder.find_spec() <importlib.abc.MetaPathFinder.find_spec>` - ) and + :meth:`MetaPathFinder.find_spec() <importlib.abc.MetaPathFinder.find_spec>`) + and :meth:`PathEntryFinder.find_loader() <importlib.abc.PathEntryFinder.find_loader>` (replaced by :meth:`PathEntryFinder.find_spec() <importlib.abc.PathEntryFinder.find_spec>`) - both deprecated in Python 3.4 now emit :exc:`DeprecationWarning`. (Contributed - by Matthias Bussonnier in :issue:`29576`) + both deprecated in Python 3.4 now emit :exc:`DeprecationWarning`. + (Contributed by Matthias Bussonnier in :issue:`29576`) - Using non-integer value for selecting a plural form in :mod:`gettext` is now deprecated. It never correctly worked. (Contributed by Serhiy Storchaka @@ -1024,23 +1023,17 @@ Deprecated - The :class:`importlib.abc.ResourceLoader` ABC has been deprecated in favour of :class:`importlib.abc.ResourceReader`. +- Deprecated :func:`sys.set_coroutine_wrapper` and + :func:`sys.get_coroutine_wrapper`. -Changes in the C API --------------------- - -- The type of results of :c:func:`PyThread_start_new_thread` and - :c:func:`PyThread_get_thread_ident`, and the *id* parameter of - :c:func:`PyThreadState_SetAsyncExc` changed from :c:type:`long` to - :c:type:`unsigned long`. - (Contributed by Serhiy Storchaka in :issue:`6532`.) - -- :c:func:`PyUnicode_AsWideCharString` now raises a :exc:`ValueError` if the - second argument is *NULL* and the :c:type:`wchar_t*` string contains null - characters. (Contributed by Serhiy Storchaka in :issue:`30708`.) +- :func:`ssl.wrap_socket` is deprecated. Use + :meth:`ssl.SSLContext.wrap_socket` instead. + (Contributed by Christian Heimes in :issue:`28124`.) Windows Only ------------ + - The python launcher, (py.exe), can accept 32 & 64 bit specifiers **without** having to specify a minor version as well. So ``py -3-32`` and ``py -3-64`` become valid as well as ``py -3.7-32``, also the -*m*-64 and -*m.n*-64 forms @@ -1054,6 +1047,7 @@ Windows Only the *short form* list of available specifiers. (Contributed by Steve Barnes in :issue:`30362`.) + Removed ======= @@ -1096,6 +1090,17 @@ API and Feature Removals :func:`~plistlib.readPlistFromBytes` are now normal dicts. You no longer can use attribute access to access items of these dictionaries. +* The ``asyncio.windows_utils.socketpair()`` function has been + removed: use directly :func:`socket.socketpair` which is available on all + platforms since Python 3.5 (before, it wasn't available on Windows). + ``asyncio.windows_utils.socketpair`` was just an alias to + ``socket.socketpair`` on Python 3.5 and newer. + +* :mod:`asyncio`: The module doesn't export :mod:`selectors` and + :mod:`_overlapped` modules as ``asyncio.selectors`` and + ``asyncio._overlapped``. Replace ``from asyncio import selectors`` with + ``import selectors`` for example. + Porting to Python 3.7 ===================== @@ -1132,29 +1137,20 @@ Changes in the Python API ------------------------- * :meth:`socketserver.ThreadingMixIn.server_close` now waits until all - non-daemon threads complete. Use daemonic threads by setting - :data:`ThreadingMixIn.daemon_threads` to ``True`` to not wait until threads - complete. (Contributed by Victor Stinner in :issue:`31233`.) + non-daemon threads complete. Use daemonic threads by setting + :data:`socketserver.ThreadingMixIn.daemon_threads` to ``True`` to not + wait until threads complete. + (Contributed by Victor Stinner in :issue:`31233`.) * :meth:`socketserver.ForkingMixIn.server_close` now waits until all child processes complete. (Contributed by Victor Stinner in :issue:`31151`.) * The :func:`locale.localeconv` function now sets temporarily the ``LC_CTYPE`` locale to the ``LC_NUMERIC`` locale in some cases. + (Contributed by Victor Stinner in :issue:`31900`.) -* The ``asyncio.windows_utils.socketpair()`` function has been - removed: use directly :func:`socket.socketpair` which is available on all - platforms since Python 3.5 (before, it wasn't available on Windows). - ``asyncio.windows_utils.socketpair()`` was just an alias to - ``socket.socketpair`` on Python 3.5 and newer. - -* :mod:`asyncio`: The module doesn't export :mod:`selectors` and - :mod:`_overlapped` modules as ``asyncio.selectors`` and - ``asyncio._overlapped``. Replace ``from asyncio import selectors`` with - ``import selectors`` for example. - -* :meth:`pkgutil.walk_packages` now raises ValueError if *path* is a string. - Previously an empty list was returned. (Contributed by Sanyam Khurana in +* :meth:`pkgutil.walk_packages` now raises :exc:`ValueError` if *path* is a string. + Previously an empty list was returned. (Contributed by Sanyam Khurana in :issue:`24744`.) * A format string argument for :meth:`string.Formatter.format` @@ -1186,6 +1182,11 @@ Changes in the Python API * The :attr:`struct.Struct.format` type is now :class:`str` instead of :class:`bytes`. (Contributed by Victor Stinner in :issue:`21071`.) +* :func:`~cgi.parse_multipart` returns now the same results as + :class:`~FieldStorage`: for non-file fields, the value associated to a key + is a list of strings, not bytes. + (Contributed by Pierre Quentel in :issue:`29979`.) + * Due to internal changes in :mod:`socket` you won't be able to :func:`socket.fromshare` a socket :func:`~socket.socket.share`-ed in older Python versions. @@ -1207,6 +1208,8 @@ Changes in the Python API avoid a warning escape them with a backslash. (Contributed by Serhiy Storchaka in :issue:`30349`.) + .. _Unicode Technical Standard #18: https://unicode.org/reports/tr18/ + * The result of splitting a string on a :mod:`regular expression <re>` that could match an empty string has been changed. For example splitting on ``r'\s*'`` will now split not only on whitespaces as it @@ -1246,8 +1249,6 @@ Changes in the Python API work as expected on all platforms. (Contributed by Yury Selivanov in :issue:`32331`.) -.. _Unicode Technical Standard #18: https://unicode.org/reports/tr18/ - * On Windows the default for the *close_fds* argument of :class:`subprocess.Popen` was changed from :const:`False` to :const:`True` when redirecting the standard handles. If you previously depended on handles @@ -1276,7 +1277,8 @@ CPython bytecode changes * Added two new opcodes: :opcode:`LOAD_METHOD` and :opcode:`CALL_METHOD`. (Contributed by Yury Selivanov and INADA Naoki in :issue:`26110`.) -* Removed the STORE_ANNOTATION opcode. +* Removed the :opcode:`STORE_ANNOTATION` opcode. + (Contributed by Mark Shannon in :issue:`32550`.) Other CPython implementation changes @@ -1293,9 +1295,9 @@ Other CPython implementation changes (Contributed by Nick Coghlan and Eric Snow as part of :issue:`22257`.) * Due to changes in the way the default warnings filters are configured, - setting ``Py_BytesWarningFlag`` to a value greater than one is no longer - sufficient to both emit ``BytesWarning`` messages and have them converted - to exceptions. Instead, the flag must be set (to cause the warnings to be + setting :c:data:`Py_BytesWarningFlag` to a value greater than one is no longer + sufficient to both emit :exc:`BytesWarning` messages and have them converted + to exceptions. Instead, the flag must be set (to cause the warnings to be emitted in the first place), and an explicit ``error::BytesWarning`` warnings filter added to convert them to exceptions. From webhook-mailer at python.org Sun Mar 11 01:29:36 2018 From: webhook-mailer at python.org (Miss Islington (bot)) Date: Sun, 11 Mar 2018 06:29:36 -0000 Subject: [Python-checkins] bpo-32996: Improve What's New in 3.7. (GH-5983) Message-ID: <mailman.36.1520749778.1871.python-checkins@python.org> https://github.com/python/cpython/commit/720a4b69c6b66ae5d6f447d5814bd204170c94b3 commit: 720a4b69c6b66ae5d6f447d5814bd204170c94b3 branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: GitHub <noreply at github.com> date: 2018-03-10T22:29:30-08:00 summary: bpo-32996: Improve What's New in 3.7. (GH-5983) (cherry picked from commit 51302a5fcc557e6afc0bf1e3b371f5f37c76dc77) Co-authored-by: Serhiy Storchaka <storchaka at gmail.com> files: M Doc/library/cgi.rst M Doc/whatsnew/3.7.rst diff --git a/Doc/library/cgi.rst b/Doc/library/cgi.rst index b60e1cc41e4b..17386b831833 100644 --- a/Doc/library/cgi.rst +++ b/Doc/library/cgi.rst @@ -289,11 +289,13 @@ algorithms implemented in this module in other circumstances. This function is deprecated in this module. Use :func:`urllib.parse.parse_qs` instead. It is maintained here only for backward compatibility. + .. function:: parse_qsl(qs, keep_blank_values=False, strict_parsing=False) This function is deprecated in this module. Use :func:`urllib.parse.parse_qsl` instead. It is maintained here only for backward compatibility. + .. function:: parse_multipart(fp, pdict, encoding="utf-8") Parse input of type :mimetype:`multipart/form-data` (for file uploads). @@ -309,6 +311,10 @@ algorithms implemented in this module in other circumstances. uploaded --- in that case, use the :class:`FieldStorage` class instead which is much more flexible. + .. versionchanged:: 3.7 + Added the *encoding* parameter. For non-file fields, the value is now + a list of strings, not bytes. + .. function:: parse_header(string) diff --git a/Doc/whatsnew/3.7.rst b/Doc/whatsnew/3.7.rst index fc5f5ab77095..a4ed481f67a5 100644 --- a/Doc/whatsnew/3.7.rst +++ b/Doc/whatsnew/3.7.rst @@ -91,17 +91,17 @@ rather than ASCII. The platform support definition in :pep:`11` has also been updated to limit full text handling support to suitably configured non-ASCII based locales. -As part of this change, the default error handler for ``stdin`` and ``stdout`` -is now ``surrogateescape`` (rather than ``strict``) when using any of the -defined coercion target locales (currently ``C.UTF-8``, ``C.utf8``, and -``UTF-8``). The default error handler for ``stderr`` continues to be -``backslashreplace``, regardless of locale. +As part of this change, the default error handler for :data:`~sys.stdin` and +:data:`~sys.stdout` is now ``surrogateescape`` (rather than ``strict``) when +using any of the defined coercion target locales (currently ``C.UTF-8``, +``C.utf8``, and ``UTF-8``). The default error handler for :data:`~sys.stderr` +continues to be ``backslashreplace``, regardless of locale. Locale coercion is silent by default, but to assist in debugging potentially locale related integration problems, explicit warnings (emitted directly on -``stderr`` can be requested by setting ``PYTHONCOERCECLOCALE=warn``. This -setting will also cause the Python runtime to emit a warning if the legacy C -locale remains active when the core interpreter is initialized. +:data:`~sys.stderr` can be requested by setting ``PYTHONCOERCECLOCALE=warn``. +This setting will also cause the Python runtime to emit a warning if the +legacy C locale remains active when the core interpreter is initialized. .. seealso:: @@ -114,9 +114,9 @@ locale remains active when the core interpreter is initialized. PEP 553: Built-in breakpoint() ------------------------------ -:pep:`553` describes a new built-in called ``breakpoint()`` which makes it +:pep:`553` describes a new built-in called :func:`breakpoint` which makes it easy and consistent to enter the Python debugger. Built-in ``breakpoint()`` -calls ``sys.breakpointhook()``. By default, this latter imports ``pdb`` and +calls :func:`sys.breakpointhook`. By default, this latter imports :mod:`pdb` and then calls ``pdb.set_trace()``, but by binding ``sys.breakpointhook()`` to the function of your choosing, ``breakpoint()`` can enter any debugger. Or, the environment variable :envvar:`PYTHONBREAKPOINT` can be set to the callable of @@ -165,13 +165,13 @@ PEP 562: Customization of access to module attributes ----------------------------------------------------- It is sometimes convenient to customize or otherwise have control over access -to module attributes. A typical example is managing deprecation warnings. -Typical workarounds are assigning ``__class__`` of a module object to -a custom subclass of :class:`types.ModuleType` or replacing the ``sys.modules`` -item with a custom wrapper instance. This procedure is now simplified by -recognizing ``__getattr__`` defined directly in a module that would act like -a normal ``__getattr__`` method, except that it will be defined on module -*instances*. +to module attributes. A typical example is managing deprecation warnings. +Typical workarounds are assigning :attr:`~instance.__class__` of a module +object to a custom subclass of :class:`types.ModuleType` or replacing the +:data:`sys.modules` item with a custom wrapper instance. This procedure is +now simplified by recognizing ``__getattr__`` defined directly in a module +that would act like a normal :meth:`__getattr__` method, except that it will +be defined on module *instances*. .. seealso:: @@ -217,7 +217,7 @@ following syntax valid:: ... Since this change breaks compatibility, the new behavior can be enabled -on a per-module basis in Python 3.7 using a ``__future__`` import, like +on a per-module basis in Python 3.7 using a :mod:`__future__` import, like this:: from __future__ import annotations @@ -263,7 +263,7 @@ PEP 565: Show DeprecationWarning in ``__main__`` The default handling of :exc:`DeprecationWarning` has been changed such that these warnings are once more shown by default, but only when the code -triggering them is running directly in the ``__main__`` module. As a result, +triggering them is running directly in the :mod:`__main__` module. As a result, developers of single file scripts and those using Python interactively should once again start seeing deprecation warnings for the APIs they use, but deprecation warnings triggered by imported application, library and framework @@ -275,7 +275,7 @@ between three different deprecation warning behaviours: * :exc:`FutureWarning`: always displayed by default, recommended for warnings intended to be seen by application end users (e.g. for deprecated application configuration settings). -* :exc:`DeprecationWarning`: displayed by default only in ``__main__`` and when +* :exc:`DeprecationWarning`: displayed by default only in :mod:`__main__` and when running tests, recommended for warnings intended to be seen by other Python developers where a version upgrade may result in changed behaviour or an error. @@ -316,11 +316,11 @@ environment variable are added to control the UTF-8 mode. PEP 557: Data Classes --------------------- -Adds a new module ``dataclasses``. It provides a class decorator -``dataclass`` which inspects the class's variable annotations (see +Adds a new module :mod:`dataclasses`. It provides a class decorator +:func:`~dataclasses.dataclass` which inspects the class's variable annotations (see :pep:`526`) and using them, adds methods such as ``__init__``, ``__repr__``, and ``__eq__`` to the class. It is similar to -``typing.NamedTuple``, but also works on classes with mutable +:class:`typing.NamedTuple`, but also works on classes with mutable instances, among other features. For example:: @@ -374,7 +374,7 @@ Hash-based pycs Python has traditionally checked the up-to-dateness of bytecode cache files (i.e., ``.pyc`` files) by comparing the source metadata (last-modified timestamp and size) with source metadata saved in the cache file header when it was -generated. While effective, this invalidation method has its drawbacks. When +generated. While effective, this invalidation method has its drawbacks. When filesystem timestamps are too coarse, Python can miss source updates, leading to user confusion. Additionally, having a timestamp in the cache file is problematic for `build reproduciblity <https://reproducible-builds.org/>`_ and @@ -407,7 +407,8 @@ Other Language Changes whitespace, not only spaces. (Contributed by Robert Xiao in :issue:`28927`.) * :exc:`ImportError` now displays module name and module ``__file__`` path when - ``from ... import ...`` fails. (Contributed by Matthias Bussonnier in :issue:`29546`.) + ``from ... import ...`` fails. (Contributed by Matthias Bussonnier in + :issue:`29546`.) * Circular imports involving absolute imports with binding a submodule to a name are now supported. @@ -453,7 +454,6 @@ the user intermix options and positional arguments on the command line, as is possible in many unix commands. It supports most but not all argparse features. (Contributed by paul.j3 in :issue:`14191`.) - binascii -------- @@ -461,7 +461,6 @@ The :func:`~binascii.b2a_uu` function now accepts an optional *backtick* keyword argument. When it's true, zeros are represented by ``'`'`` instead of spaces. (Contributed by Xiang Zhang in :issue:`30103`.) - calendar -------- @@ -469,14 +468,6 @@ The class :class:`~calendar.HTMLCalendar` has new class attributes which ease the customisation of the CSS classes in the produced HTML calendar. (Contributed by Oz Tiram in :issue:`30095`.) -cgi ---- - -:func:`~cgi.parse_multipart` returns the same results as -:class:`~FieldStorage` : for non-file fields, the value associated to a key -is a list of strings, not bytes. -(Contributed by Pierre Quentel in :issue:`29979`.) - contextlib ---------- @@ -490,8 +481,8 @@ Alexander Mohr and Ilya Kulakov in :issue:`29302`.) cProfile -------- -cProfile command line now accepts `-m module_name` as an alternative to -script path. (Contributed by Sanyam Khurana in :issue:`21862`.) +:mod:`cProfile` command line now accepts ``-m module_name`` as an alternative +to script path. (Contributed by Sanyam Khurana in :issue:`21862`.) crypt ----- @@ -505,10 +496,11 @@ for hashing. (Contributed by Serhiy Storchaka in :issue:`31702`.) datetime -------- -Added the :func:`datetime.datetime.fromisoformat` method, which constructs a -:class:`datetime.datetime` object from a string in one of the formats output -by :func:`datetime.datetime.isoformat`. (Contributed by Paul Ganssle in -:issue:`15873`.) +Added the :meth:`datetime.fromisoformat <datetime.datetime.fromisoformat>` +method, which constructs a :class:`~datetime.datetime` object from a string +in one of the formats output by +:meth:`datetime.isoformat <datetime.datetime.isoformat>`. +(Contributed by Paul Ganssle in :issue:`15873`.) dis --- @@ -521,7 +513,7 @@ classes). (Contributed by Serhiy Storchaka in :issue:`11822`.) distutils --------- -README.rst is now included in the list of distutils standard READMEs and +``README.rst`` is now included in the list of distutils standard READMEs and therefore included in source distributions. (Contributed by Ryan Gonzalez in :issue:`11913`.) @@ -537,28 +529,29 @@ equivalent to CR. http.client ----------- -Add Configurable *blocksize* to ``HTTPConnection`` and -``HTTPSConnection`` for improved upload throughput. +Add configurable *blocksize* to :class:`~http.client.HTTPConnection` and +:class:`~http.client.HTTPSConnection` for improved upload throughput. (Contributed by Nir Soffer in :issue:`31945`.) http.server ----------- :class:`~http.server.SimpleHTTPRequestHandler` supports the HTTP -If-Modified-Since header. The server returns the 304 response status if the +``If-Modified-Since`` header. The server returns the 304 response status if the target file was not modified after the time specified in the header. (Contributed by Pierre Quentel in :issue:`29654`.) -Add the parameter ``directory`` to the :class:`~http.server.SimpleHTTPRequestHandler` -and the ``--directory`` to the command line of the module :mod:`~http.server`. -With this parameter, the server serves the specified directory, by default it uses the current working directory. +Add the parameter *directory* to the :class:`~http.server.SimpleHTTPRequestHandler` +and the ``--directory`` to the command line of the module :mod:`http.server`. +With this parameter, the server serves the specified directory, by default it +uses the current working directory. (Contributed by St?phane Wirtel and Julien Palard in :issue:`28707`.) hmac ---- -The hmac module now has an optimized one-shot :func:`~hmac.digest` function, -which is up to three times faster than :func:`~hmac.HMAC`. +The :mod:`hmac` module now has an optimized one-shot :func:`~hmac.digest` +function, which is up to three times faster than :func:`~hmac.HMAC`. (Contributed by Christian Heimes in :issue:`32433`.) importlib @@ -570,11 +563,11 @@ support the loading of resource from packages. locale ------ -Added another argument *monetary* in :meth:`format_string` of :mod:`locale`. +Added another argument *monetary* in :func:`~locale.format_string` of :mod:`locale`. If *monetary* is true, the conversion uses monetary thousands separator and grouping strings. (Contributed by Garvit in :issue:`10379`.) -The :func:`locale.getpreferredencoding` function now always returns ``'UTF-8'`` +The :func:`~locale.getpreferredencoding` function now always returns ``'UTF-8'`` on Android or in the UTF-8 mode (:option:`-X` ``utf8`` option), the locale and the *do_setlocale* argument are ignored. @@ -593,7 +586,7 @@ Serhiy Storchaka in :issue:`28682`.) Added support for :ref:`file descriptors <path_fd>` in :func:`~os.scandir` on Unix. (Contributed by Serhiy Storchaka in :issue:`25996`.) -New function :func:`os.register_at_fork` allows registering Python callbacks +New function :func:`~os.register_at_fork` allows registering Python callbacks to be executed on a process fork. (Contributed by Antoine Pitrou in :issue:`16500`.) @@ -604,8 +597,8 @@ pdb argument. If given, this is printed to the console just before debugging begins. (Contributed by Barry Warsaw in :issue:`31389`.) -pdb command line now accepts `-m module_name` as an alternative to -script file. (Contributed by Mario Corchero in :issue:`32206`.) +:mod:`pdb` command line now accepts ``-m module_name`` as an alternative to +script file. (Contributed by Mario Corchero in :issue:`32206`.) py_compile ---------- @@ -618,7 +611,6 @@ This allows for guaranteeing files when they are created eagerly. (Contributed by Bernhard M. Wiedemann in :issue:`29708`.) - re -- @@ -642,7 +634,7 @@ method, if the underlying SQLite library is at version 3.6.11 or higher. ssl --- -The ssl module now uses OpenSSL's builtin API instead of +The :mod:`ssl` module now uses OpenSSL's builtin API instead of :func:`~ssl.match_hostname` to check host name or IP address. Values are validated during TLS handshake. Any cert validation error including a failing host name match now raises :exc:`~ssl.SSLCertVerificationError` and @@ -682,10 +674,6 @@ The ssl module has preliminary and experimental support for TLS 1.3 and OpenSSL 1.1.1. (Contributed by Christian Heimes in :issue:`32947`, :issue:`20995`, :issue:`29136`, and :issue:`30622`) -:func:`~ssl.wrap_socket` is deprecated. Documentation has been updated to -recommend :meth:`~ssl.SSLContext.wrap_socket` instead. -(Contributed by Christian Heimes in :issue:`28124`.) - :class:`~ssl.SSLSocket` and :class:`~ssl.SSLObject` no longer have a public constructor. Direct instantiation was never a documented and supported feature. Instances must be created with :class:`~ssl.SSLContext` methods @@ -708,27 +696,24 @@ separately. (Contributed by Barry Warsaw in :issue:`1198569`.) subprocess ---------- -On Windows the default for *close_fds* was changed from :const:`False` to -:const:`True` when redirecting the standard handles. It's now possible to set -*close_fds* to :const:`True` when redirecting the standard handles. See +On Windows the default for *close_fds* was changed from ``False`` to +``True`` when redirecting the standard handles. It's now possible to set +*close_fds* to ``True`` when redirecting the standard handles. See :class:`subprocess.Popen`. -This means that *close_fds* now defaults to :const:`True` on all supported -platforms. +This means that *close_fds* now defaults to ``True`` on all supported +platforms. (Contributed by Segev Finer in :issue:`19764`.) sys --- Added :attr:`sys.flags.dev_mode` flag for the new development mode. -Deprecated :func:`sys.set_coroutine_wrapper` and -:func:`sys.get_coroutine_wrapper`. - - tkinter ------- Added :class:`tkinter.ttk.Spinbox`. +(Contributed by Alan Moore in :issue:`32585`.) time ---- @@ -766,11 +751,12 @@ Peterson.) unittest -------- + Added new command-line option ``-k`` to filter tests to run with a substring or Unix shell-like pattern. For example, ``python -m unittest -k foo`` runs the tests ``foo_tests.SomeTest.test_something``, ``bar_tests.SomeTest.test_foo``, but not ``bar_tests.FooTest.test_something``. - +(Contributed by Jonas Haag in :issue:`32071`.) unittest.mock ------------- @@ -779,7 +765,7 @@ The :const:`~unittest.mock.sentinel` attributes now preserve their identity when they are :mod:`copied <copy>` or :mod:`pickled <pickle>`. (Contributed by Serhiy Storchaka in :issue:`20804`.) -New function :const:`~unittest.mock.seal` will disable the creation of mock +New function :func:`~unittest.mock.seal` will disable the creation of mock children by preventing to get or set any new attribute on the sealed mock. The sealing process is performed recursively. (Contributed by Mario Corchero in :issue:`30541`.) @@ -787,8 +773,8 @@ in :issue:`30541`.) urllib.parse ------------ -:func:`urllib.parse.quote` has been updated from RFC 2396 to RFC 3986, -adding `~` to the set of characters that is never quoted by default. +:func:`urllib.parse.quote` has been updated from :rfc:`2396` to :rfc:`3986`, +adding ``~`` to the set of characters that is never quoted by default. (Contributed by Christian Theune and Ratnadeep Debnath in :issue:`16285`.) uu @@ -832,26 +818,27 @@ better readability. (Contributed by Stefan Behnel in :issue:`31648`.) xmlrpc.server ------------- -:meth:`register_function` of :class:`xmlrpc.server.SimpleXMLRPCDispatcher` and -its subclasses can be used as a decorator. (Contributed by Xiang Zhang in +:meth:`register_function` of :class:`~xmlrpc.server.SimpleXMLRPCDispatcher` and +its subclasses can be used as a decorator. (Contributed by Xiang Zhang in :issue:`7769`.) zipapp ------ -Function :func:`zipapp.create_archive` now accepts an optional *filter* +Function :func:`~zipapp.create_archive` now accepts an optional *filter* argument to allow the user to select which files should be included in the -archive, and an optional *compressed* argument to generate a compressed -archive. +archive. (Contributed by Irmen de Jong in :issue:`31072`.) -A command line option ``--compress`` has also been added to support -compression. +Function :func:`~zipapp.create_archive` now accepts an optional *compressed* +argument to generate a compressed archive. A command line option +``--compress`` has also been added to support compression. +(Contributed by Zhiming Wang in :issue:`31638`.) Optimizations ============= -* Added two new opcodes: ``LOAD_METHOD`` and ``CALL_METHOD`` to avoid +* Added two new opcodes: :opcode:`LOAD_METHOD` and :opcode:`CALL_METHOD` to avoid instantiation of bound method objects for method calls, which results in method calls being faster up to 20%. (Contributed by Yury Selivanov and INADA Naoki in :issue:`26110`.) @@ -882,9 +869,10 @@ Optimizations Python 3.6 by about 10% depending on the pattern. (Contributed by INADA Naoki in :issue:`31671`.) -* :meth:`selectors.EpollSelector.modify`, :meth:`selectors.PollSelector.modify` - and :meth:`selectors.DevpollSelector.modify` may be around 10% faster under - heavy loads. (Contributed by Giampaolo Rodola' in :issue:`30014`) +* :meth:`~selectors.BaseSelector.modify` methods of classes + :class:`selectors.EpollSelector`, :class:`selectors.PollSelector` + and :class:`selectors.DevpollSelector` may be around 10% faster under + heavy loads. (Contributed by Giampaolo Rodola' in :issue:`30014`) * Constant folding is moved from peephole optimizer to new AST optimizer. (Contributed by Eugene Toder and INADA Naoki in :issue:`29469`) @@ -900,6 +888,7 @@ Optimizations constructors when not constructing subclasses. (Contributed by Paul Ganssle in :issue:`32403`) + Build and C API Changes ======================= @@ -955,6 +944,16 @@ Build and C API Changes and access to the UTC singleton with :c:data:`PyDateTime_TimeZone_UTC`. Contributed by Paul Ganssle in :issue:`10381`. +- The type of results of :c:func:`PyThread_start_new_thread` and + :c:func:`PyThread_get_thread_ident`, and the *id* parameter of + :c:func:`PyThreadState_SetAsyncExc` changed from :c:type:`long` to + :c:type:`unsigned long`. + (Contributed by Serhiy Storchaka in :issue:`6532`.) + +- :c:func:`PyUnicode_AsWideCharString` now raises a :exc:`ValueError` if the + second argument is *NULL* and the :c:type:`wchar_t*` string contains null + characters. (Contributed by Serhiy Storchaka in :issue:`30708`.) + Other CPython Implementation Changes ==================================== @@ -1007,13 +1006,13 @@ Deprecated - Methods :meth:`MetaPathFinder.find_module() <importlib.abc.MetaPathFinder.find_module>` (replaced by - :meth:`MetaPathFinder.find_spec() <importlib.abc.MetaPathFinder.find_spec>` - ) and + :meth:`MetaPathFinder.find_spec() <importlib.abc.MetaPathFinder.find_spec>`) + and :meth:`PathEntryFinder.find_loader() <importlib.abc.PathEntryFinder.find_loader>` (replaced by :meth:`PathEntryFinder.find_spec() <importlib.abc.PathEntryFinder.find_spec>`) - both deprecated in Python 3.4 now emit :exc:`DeprecationWarning`. (Contributed - by Matthias Bussonnier in :issue:`29576`) + both deprecated in Python 3.4 now emit :exc:`DeprecationWarning`. + (Contributed by Matthias Bussonnier in :issue:`29576`) - Using non-integer value for selecting a plural form in :mod:`gettext` is now deprecated. It never correctly worked. (Contributed by Serhiy Storchaka @@ -1024,23 +1023,17 @@ Deprecated - The :class:`importlib.abc.ResourceLoader` ABC has been deprecated in favour of :class:`importlib.abc.ResourceReader`. +- Deprecated :func:`sys.set_coroutine_wrapper` and + :func:`sys.get_coroutine_wrapper`. -Changes in the C API --------------------- - -- The type of results of :c:func:`PyThread_start_new_thread` and - :c:func:`PyThread_get_thread_ident`, and the *id* parameter of - :c:func:`PyThreadState_SetAsyncExc` changed from :c:type:`long` to - :c:type:`unsigned long`. - (Contributed by Serhiy Storchaka in :issue:`6532`.) - -- :c:func:`PyUnicode_AsWideCharString` now raises a :exc:`ValueError` if the - second argument is *NULL* and the :c:type:`wchar_t*` string contains null - characters. (Contributed by Serhiy Storchaka in :issue:`30708`.) +- :func:`ssl.wrap_socket` is deprecated. Use + :meth:`ssl.SSLContext.wrap_socket` instead. + (Contributed by Christian Heimes in :issue:`28124`.) Windows Only ------------ + - The python launcher, (py.exe), can accept 32 & 64 bit specifiers **without** having to specify a minor version as well. So ``py -3-32`` and ``py -3-64`` become valid as well as ``py -3.7-32``, also the -*m*-64 and -*m.n*-64 forms @@ -1054,6 +1047,7 @@ Windows Only the *short form* list of available specifiers. (Contributed by Steve Barnes in :issue:`30362`.) + Removed ======= @@ -1096,6 +1090,17 @@ API and Feature Removals :func:`~plistlib.readPlistFromBytes` are now normal dicts. You no longer can use attribute access to access items of these dictionaries. +* The ``asyncio.windows_utils.socketpair()`` function has been + removed: use directly :func:`socket.socketpair` which is available on all + platforms since Python 3.5 (before, it wasn't available on Windows). + ``asyncio.windows_utils.socketpair`` was just an alias to + ``socket.socketpair`` on Python 3.5 and newer. + +* :mod:`asyncio`: The module doesn't export :mod:`selectors` and + :mod:`_overlapped` modules as ``asyncio.selectors`` and + ``asyncio._overlapped``. Replace ``from asyncio import selectors`` with + ``import selectors`` for example. + Porting to Python 3.7 ===================== @@ -1132,29 +1137,20 @@ Changes in the Python API ------------------------- * :meth:`socketserver.ThreadingMixIn.server_close` now waits until all - non-daemon threads complete. Use daemonic threads by setting - :data:`ThreadingMixIn.daemon_threads` to ``True`` to not wait until threads - complete. (Contributed by Victor Stinner in :issue:`31233`.) + non-daemon threads complete. Use daemonic threads by setting + :data:`socketserver.ThreadingMixIn.daemon_threads` to ``True`` to not + wait until threads complete. + (Contributed by Victor Stinner in :issue:`31233`.) * :meth:`socketserver.ForkingMixIn.server_close` now waits until all child processes complete. (Contributed by Victor Stinner in :issue:`31151`.) * The :func:`locale.localeconv` function now sets temporarily the ``LC_CTYPE`` locale to the ``LC_NUMERIC`` locale in some cases. + (Contributed by Victor Stinner in :issue:`31900`.) -* The ``asyncio.windows_utils.socketpair()`` function has been - removed: use directly :func:`socket.socketpair` which is available on all - platforms since Python 3.5 (before, it wasn't available on Windows). - ``asyncio.windows_utils.socketpair()`` was just an alias to - ``socket.socketpair`` on Python 3.5 and newer. - -* :mod:`asyncio`: The module doesn't export :mod:`selectors` and - :mod:`_overlapped` modules as ``asyncio.selectors`` and - ``asyncio._overlapped``. Replace ``from asyncio import selectors`` with - ``import selectors`` for example. - -* :meth:`pkgutil.walk_packages` now raises ValueError if *path* is a string. - Previously an empty list was returned. (Contributed by Sanyam Khurana in +* :meth:`pkgutil.walk_packages` now raises :exc:`ValueError` if *path* is a string. + Previously an empty list was returned. (Contributed by Sanyam Khurana in :issue:`24744`.) * A format string argument for :meth:`string.Formatter.format` @@ -1186,6 +1182,11 @@ Changes in the Python API * The :attr:`struct.Struct.format` type is now :class:`str` instead of :class:`bytes`. (Contributed by Victor Stinner in :issue:`21071`.) +* :func:`~cgi.parse_multipart` returns now the same results as + :class:`~FieldStorage`: for non-file fields, the value associated to a key + is a list of strings, not bytes. + (Contributed by Pierre Quentel in :issue:`29979`.) + * Due to internal changes in :mod:`socket` you won't be able to :func:`socket.fromshare` a socket :func:`~socket.socket.share`-ed in older Python versions. @@ -1207,6 +1208,8 @@ Changes in the Python API avoid a warning escape them with a backslash. (Contributed by Serhiy Storchaka in :issue:`30349`.) + .. _Unicode Technical Standard #18: https://unicode.org/reports/tr18/ + * The result of splitting a string on a :mod:`regular expression <re>` that could match an empty string has been changed. For example splitting on ``r'\s*'`` will now split not only on whitespaces as it @@ -1246,8 +1249,6 @@ Changes in the Python API work as expected on all platforms. (Contributed by Yury Selivanov in :issue:`32331`.) -.. _Unicode Technical Standard #18: https://unicode.org/reports/tr18/ - * On Windows the default for the *close_fds* argument of :class:`subprocess.Popen` was changed from :const:`False` to :const:`True` when redirecting the standard handles. If you previously depended on handles @@ -1276,7 +1277,8 @@ CPython bytecode changes * Added two new opcodes: :opcode:`LOAD_METHOD` and :opcode:`CALL_METHOD`. (Contributed by Yury Selivanov and INADA Naoki in :issue:`26110`.) -* Removed the STORE_ANNOTATION opcode. +* Removed the :opcode:`STORE_ANNOTATION` opcode. + (Contributed by Mark Shannon in :issue:`32550`.) Other CPython implementation changes @@ -1293,9 +1295,9 @@ Other CPython implementation changes (Contributed by Nick Coghlan and Eric Snow as part of :issue:`22257`.) * Due to changes in the way the default warnings filters are configured, - setting ``Py_BytesWarningFlag`` to a value greater than one is no longer - sufficient to both emit ``BytesWarning`` messages and have them converted - to exceptions. Instead, the flag must be set (to cause the warnings to be + setting :c:data:`Py_BytesWarningFlag` to a value greater than one is no longer + sufficient to both emit :exc:`BytesWarning` messages and have them converted + to exceptions. Instead, the flag must be set (to cause the warnings to be emitted in the first place), and an explicit ``error::BytesWarning`` warnings filter added to convert them to exceptions. From webhook-mailer at python.org Sun Mar 11 01:32:50 2018 From: webhook-mailer at python.org (Serhiy Storchaka) Date: Sun, 11 Mar 2018 06:32:50 -0000 Subject: [Python-checkins] bpo-33026: Fix jumping out of "with" block by setting f_lineno. (#6026) Message-ID: <mailman.37.1520749971.1871.python-checkins@python.org> https://github.com/python/cpython/commit/26c9f565d016db21257a60d29ab2c99383dd5ac7 commit: 26c9f565d016db21257a60d29ab2c99383dd5ac7 branch: master author: Serhiy Storchaka <storchaka at gmail.com> committer: GitHub <noreply at github.com> date: 2018-03-11T08:32:47+02:00 summary: bpo-33026: Fix jumping out of "with" block by setting f_lineno. (#6026) files: A Misc/NEWS.d/next/Core and Builtins/2018-03-08-09-48-38.bpo-33026.QZA3Ba.rst M Lib/test/test_sys_settrace.py M Objects/frameobject.c diff --git a/Lib/test/test_sys_settrace.py b/Lib/test/test_sys_settrace.py index 7df10cb5c1f7..72cce33392dc 100644 --- a/Lib/test/test_sys_settrace.py +++ b/Lib/test/test_sys_settrace.py @@ -827,6 +827,34 @@ def test_jump_across_with(output): with tracecontext(output, 4): output.append(5) + @jump_test(4, 5, [1, 3, 5, 6]) + def test_jump_out_of_with_block_within_for_block(output): + output.append(1) + for i in [1]: + with tracecontext(output, 3): + output.append(4) + output.append(5) + output.append(6) + + @jump_test(4, 5, [1, 2, 3, 5, -2, 6]) + def test_jump_out_of_with_block_within_with_block(output): + output.append(1) + with tracecontext(output, 2): + with tracecontext(output, 3): + output.append(4) + output.append(5) + output.append(6) + + @jump_test(5, 6, [2, 4, 6, 7]) + def test_jump_out_of_with_block_within_finally_block(output): + try: + output.append(2) + finally: + with tracecontext(output, 4): + output.append(5) + output.append(6) + output.append(7) + @jump_test(8, 11, [1, 3, 5, 11, 12]) def test_jump_out_of_complex_nested_blocks(output): output.append(1) diff --git a/Misc/NEWS.d/next/Core and Builtins/2018-03-08-09-48-38.bpo-33026.QZA3Ba.rst b/Misc/NEWS.d/next/Core and Builtins/2018-03-08-09-48-38.bpo-33026.QZA3Ba.rst new file mode 100644 index 000000000000..dc166d1e5771 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2018-03-08-09-48-38.bpo-33026.QZA3Ba.rst @@ -0,0 +1 @@ +Fixed jumping out of "with" block by setting f_lineno. diff --git a/Objects/frameobject.c b/Objects/frameobject.c index b9f380d7b60d..31ad8d0be226 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -89,7 +89,6 @@ frame_setlineno(PyFrameObject *f, PyObject* p_new_lineno) long l_new_lineno; int overflow; int new_lasti = 0; /* The new value of f_lasti */ - int new_iblock = 0; /* The new value of f_iblock */ unsigned char *code = NULL; /* The bytecode for the frame... */ Py_ssize_t code_len = 0; /* ...and its length */ unsigned char *lnotab = NULL; /* Iterating over co_lnotab */ @@ -99,6 +98,7 @@ frame_setlineno(PyFrameObject *f, PyObject* p_new_lineno) int addr = 0; /* (ditto) */ int delta_iblock = 0; /* Scanning the SETUPs and POPs */ int for_loop_delta = 0; /* (ditto) */ + int delta; int blockstack[CO_MAXBLOCKS]; /* Walking the 'finally' blocks */ int blockstack_top = 0; /* (ditto) */ @@ -258,19 +258,25 @@ frame_setlineno(PyFrameObject *f, PyObject* p_new_lineno) assert(blockstack_top == 0); /* Pop any blocks that we're jumping out of. */ - new_iblock = f->f_iblock - delta_iblock; - while (f->f_iblock > new_iblock) { - PyTryBlock *b = &f->f_blockstack[--f->f_iblock]; - while ((f->f_stacktop - f->f_valuestack) > b->b_level) { - PyObject *v = (*--f->f_stacktop); - Py_DECREF(v); + delta = 0; + if (delta_iblock > 0) { + f->f_iblock -= delta_iblock; + PyTryBlock *b = &f->f_blockstack[f->f_iblock]; + delta = (f->f_stacktop - f->f_valuestack) - b->b_level; + if (b->b_type == SETUP_FINALLY && + code[b->b_handler] == WITH_CLEANUP_START) + { + /* Pop the exit function. */ + delta++; } } /* Pop the iterators of any 'for' loop we're jumping out of. */ - while (for_loop_delta > 0) { + delta += for_loop_delta; + + while (delta > 0) { PyObject *v = (*--f->f_stacktop); Py_DECREF(v); - for_loop_delta--; + delta--; } /* Finally set the new f_lineno and f_lasti and return OK. */ From webhook-mailer at python.org Sun Mar 11 01:38:16 2018 From: webhook-mailer at python.org (Serhiy Storchaka) Date: Sun, 11 Mar 2018 06:38:16 -0000 Subject: [Python-checkins] bpo-32338: OrderedDict import is no longer needed in re. (#4891) Message-ID: <mailman.38.1520750298.1871.python-checkins@python.org> https://github.com/python/cpython/commit/b931bd0a2fe7e9293339019352baf3317166b769 commit: b931bd0a2fe7e9293339019352baf3317166b769 branch: master author: Serhiy Storchaka <storchaka at gmail.com> committer: GitHub <noreply at github.com> date: 2018-03-11T08:38:13+02:00 summary: bpo-32338: OrderedDict import is no longer needed in re. (#4891) files: M Lib/re.py diff --git a/Lib/re.py b/Lib/re.py index a8b6753d3909..94d486579e08 100644 --- a/Lib/re.py +++ b/Lib/re.py @@ -128,12 +128,6 @@ except ImportError: _locale = None -# try _collections first to reduce startup cost -try: - from _collections import OrderedDict -except ImportError: - from collections import OrderedDict - # public symbols __all__ = [ @@ -271,7 +265,7 @@ def escape(pattern): # -------------------------------------------------------------------- # internals -_cache = OrderedDict() +_cache = {} # ordered! _MAXCACHE = 512 def _compile(pattern, flags): @@ -292,9 +286,10 @@ def _compile(pattern, flags): p = sre_compile.compile(pattern, flags) if not (flags & DEBUG): if len(_cache) >= _MAXCACHE: + # Drop the oldest item try: - _cache.popitem(last=False) - except KeyError: + del _cache[next(iter(_cache))] + except (StopIteration, RuntimeError, KeyError): pass _cache[type(pattern), pattern, flags] = p return p From webhook-mailer at python.org Sun Mar 11 03:02:01 2018 From: webhook-mailer at python.org (Miss Islington (bot)) Date: Sun, 11 Mar 2018 07:02:01 -0000 Subject: [Python-checkins] bpo-32338: OrderedDict import is no longer needed in re. (GH-4891) Message-ID: <mailman.39.1520751722.1871.python-checkins@python.org> https://github.com/python/cpython/commit/39441fce0218a3f51a80cf17aa179a32651a02f6 commit: 39441fce0218a3f51a80cf17aa179a32651a02f6 branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: GitHub <noreply at github.com> date: 2018-03-10T23:01:58-08:00 summary: bpo-32338: OrderedDict import is no longer needed in re. (GH-4891) (cherry picked from commit b931bd0a2fe7e9293339019352baf3317166b769) Co-authored-by: Serhiy Storchaka <storchaka at gmail.com> files: M Lib/re.py diff --git a/Lib/re.py b/Lib/re.py index a8b6753d3909..94d486579e08 100644 --- a/Lib/re.py +++ b/Lib/re.py @@ -128,12 +128,6 @@ except ImportError: _locale = None -# try _collections first to reduce startup cost -try: - from _collections import OrderedDict -except ImportError: - from collections import OrderedDict - # public symbols __all__ = [ @@ -271,7 +265,7 @@ def escape(pattern): # -------------------------------------------------------------------- # internals -_cache = OrderedDict() +_cache = {} # ordered! _MAXCACHE = 512 def _compile(pattern, flags): @@ -292,9 +286,10 @@ def _compile(pattern, flags): p = sre_compile.compile(pattern, flags) if not (flags & DEBUG): if len(_cache) >= _MAXCACHE: + # Drop the oldest item try: - _cache.popitem(last=False) - except KeyError: + del _cache[next(iter(_cache))] + except (StopIteration, RuntimeError, KeyError): pass _cache[type(pattern), pattern, flags] = p return p From webhook-mailer at python.org Sun Mar 11 03:30:16 2018 From: webhook-mailer at python.org (Serhiy Storchaka) Date: Sun, 11 Mar 2018 07:30:16 -0000 Subject: [Python-checkins] [3.7] bpo-33026: Fix jumping out of "with" block by setting f_lineno. (GH-6026). (#6074) Message-ID: <mailman.40.1520753418.1871.python-checkins@python.org> https://github.com/python/cpython/commit/04aadf23eac51fec2e436c5960c1362bbb7d03de commit: 04aadf23eac51fec2e436c5960c1362bbb7d03de branch: 3.7 author: Serhiy Storchaka <storchaka at gmail.com> committer: GitHub <noreply at github.com> date: 2018-03-11T09:30:13+02:00 summary: [3.7] bpo-33026: Fix jumping out of "with" block by setting f_lineno. (GH-6026). (#6074) (cherry picked from commit 26c9f565d016db21257a60d29ab2c99383dd5ac7) Co-authored-by: Serhiy Storchaka <storchaka at gmail.com> files: A Misc/NEWS.d/next/Core and Builtins/2018-03-08-09-48-38.bpo-33026.QZA3Ba.rst M Lib/test/test_sys_settrace.py M Objects/frameobject.c diff --git a/Lib/test/test_sys_settrace.py b/Lib/test/test_sys_settrace.py index e6eb80ad86a8..fa5b61af6172 100644 --- a/Lib/test/test_sys_settrace.py +++ b/Lib/test/test_sys_settrace.py @@ -797,6 +797,34 @@ def test_jump_across_with(output): with tracecontext(output, 4): output.append(5) + @jump_test(4, 5, [1, 3, 5, 6]) + def test_jump_out_of_with_block_within_for_block(output): + output.append(1) + for i in [1]: + with tracecontext(output, 3): + output.append(4) + output.append(5) + output.append(6) + + @jump_test(4, 5, [1, 2, 3, 5, -2, 6]) + def test_jump_out_of_with_block_within_with_block(output): + output.append(1) + with tracecontext(output, 2): + with tracecontext(output, 3): + output.append(4) + output.append(5) + output.append(6) + + @jump_test(5, 6, [2, 4, 6, 7]) + def test_jump_out_of_with_block_within_finally_block(output): + try: + output.append(2) + finally: + with tracecontext(output, 4): + output.append(5) + output.append(6) + output.append(7) + @jump_test(8, 11, [1, 3, 5, 11, 12]) def test_jump_out_of_complex_nested_blocks(output): output.append(1) diff --git a/Misc/NEWS.d/next/Core and Builtins/2018-03-08-09-48-38.bpo-33026.QZA3Ba.rst b/Misc/NEWS.d/next/Core and Builtins/2018-03-08-09-48-38.bpo-33026.QZA3Ba.rst new file mode 100644 index 000000000000..dc166d1e5771 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2018-03-08-09-48-38.bpo-33026.QZA3Ba.rst @@ -0,0 +1 @@ +Fixed jumping out of "with" block by setting f_lineno. diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 3083e5b6445c..2825041e6310 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -320,6 +320,13 @@ frame_setlineno(PyFrameObject *f, PyObject* p_new_lineno) PyObject *v = (*--f->f_stacktop); Py_DECREF(v); } + if (b->b_type == SETUP_FINALLY && + code[b->b_handler] == WITH_CLEANUP_START) + { + /* Pop the exit function. */ + PyObject *v = (*--f->f_stacktop); + Py_DECREF(v); + } } /* Finally set the new f_lineno and f_lasti and return OK. */ From webhook-mailer at python.org Sun Mar 11 04:21:16 2018 From: webhook-mailer at python.org (Serhiy Storchaka) Date: Sun, 11 Mar 2018 08:21:16 -0000 Subject: [Python-checkins] [3.6] bpo-33026: Fix jumping out of "with" block by setting f_lineno. (GH-6026). (GH-6074) (GH-6075) Message-ID: <mailman.41.1520756477.1871.python-checkins@python.org> https://github.com/python/cpython/commit/20ac11a9fb027f183914bbaaaed091b57c882e54 commit: 20ac11a9fb027f183914bbaaaed091b57c882e54 branch: 3.6 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Serhiy Storchaka <storchaka at gmail.com> date: 2018-03-11T10:21:13+02:00 summary: [3.6] bpo-33026: Fix jumping out of "with" block by setting f_lineno. (GH-6026). (GH-6074) (GH-6075) (cherry picked from commit 26c9f565d016db21257a60d29ab2c99383dd5ac7) (cherry picked from commit 04aadf23eac51fec2e436c5960c1362bbb7d03de) Co-authored-by: Serhiy Storchaka <storchaka at gmail.com> files: A Misc/NEWS.d/next/Core and Builtins/2018-03-08-09-48-38.bpo-33026.QZA3Ba.rst M Lib/test/test_sys_settrace.py M Objects/frameobject.c diff --git a/Lib/test/test_sys_settrace.py b/Lib/test/test_sys_settrace.py index 5b3b0660b892..a05fbbd04a84 100644 --- a/Lib/test/test_sys_settrace.py +++ b/Lib/test/test_sys_settrace.py @@ -751,6 +751,34 @@ def test_jump_across_with(output): with tracecontext(output, 4): output.append(5) + @jump_test(4, 5, [1, 3, 5, 6]) + def test_jump_out_of_with_block_within_for_block(output): + output.append(1) + for i in [1]: + with tracecontext(output, 3): + output.append(4) + output.append(5) + output.append(6) + + @jump_test(4, 5, [1, 2, 3, 5, -2, 6]) + def test_jump_out_of_with_block_within_with_block(output): + output.append(1) + with tracecontext(output, 2): + with tracecontext(output, 3): + output.append(4) + output.append(5) + output.append(6) + + @jump_test(5, 6, [2, 4, 6, 7]) + def test_jump_out_of_with_block_within_finally_block(output): + try: + output.append(2) + finally: + with tracecontext(output, 4): + output.append(5) + output.append(6) + output.append(7) + @jump_test(8, 11, [1, 3, 5, 11, 12]) def test_jump_out_of_complex_nested_blocks(output): output.append(1) diff --git a/Misc/NEWS.d/next/Core and Builtins/2018-03-08-09-48-38.bpo-33026.QZA3Ba.rst b/Misc/NEWS.d/next/Core and Builtins/2018-03-08-09-48-38.bpo-33026.QZA3Ba.rst new file mode 100644 index 000000000000..dc166d1e5771 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2018-03-08-09-48-38.bpo-33026.QZA3Ba.rst @@ -0,0 +1 @@ +Fixed jumping out of "with" block by setting f_lineno. diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 62f9f34c8ebf..5c73e8560f86 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -317,6 +317,13 @@ frame_setlineno(PyFrameObject *f, PyObject* p_new_lineno) PyObject *v = (*--f->f_stacktop); Py_DECREF(v); } + if (b->b_type == SETUP_FINALLY && + code[b->b_handler] == WITH_CLEANUP_START) + { + /* Pop the exit function. */ + PyObject *v = (*--f->f_stacktop); + Py_DECREF(v); + } } /* Finally set the new f_lineno and f_lasti and return OK. */ From webhook-mailer at python.org Sun Mar 11 04:52:42 2018 From: webhook-mailer at python.org (Serhiy Storchaka) Date: Sun, 11 Mar 2018 08:52:42 -0000 Subject: [Python-checkins] bpo-32946: Speed up "from ... import ..." from non-packages. (GH-5873) Message-ID: <mailman.42.1520758363.1871.python-checkins@python.org> https://github.com/python/cpython/commit/4e2442505c5e9eec396dcef4d2e6bdd2b6f92fc9 commit: 4e2442505c5e9eec396dcef4d2e6bdd2b6f92fc9 branch: master author: Serhiy Storchaka <storchaka at gmail.com> committer: GitHub <noreply at github.com> date: 2018-03-11T10:52:37+02:00 summary: bpo-32946: Speed up "from ... import ..." from non-packages. (GH-5873) files: A Misc/NEWS.d/next/Core and Builtins/2018-02-25-10-52-40.bpo-32946.Lo09rG.rst M Lib/importlib/_bootstrap.py M Python/import.c M Python/importlib.h diff --git a/Lib/importlib/_bootstrap.py b/Lib/importlib/_bootstrap.py index 2bdd1929b42f..ba5a053d6bfe 100644 --- a/Lib/importlib/_bootstrap.py +++ b/Lib/importlib/_bootstrap.py @@ -1016,31 +1016,30 @@ def _handle_fromlist(module, fromlist, import_, *, recursive=False): """ # The hell that is fromlist ... # If a package was imported, try to import stuff from fromlist. - if hasattr(module, '__path__'): - for x in fromlist: - if not isinstance(x, str): - if recursive: - where = module.__name__ + '.__all__' - else: - where = "``from list''" - raise TypeError(f"Item in {where} must be str, " - f"not {type(x).__name__}") - elif x == '*': - if not recursive and hasattr(module, '__all__'): - _handle_fromlist(module, module.__all__, import_, - recursive=True) - elif not hasattr(module, x): - from_name = '{}.{}'.format(module.__name__, x) - try: - _call_with_frames_removed(import_, from_name) - except ModuleNotFoundError as exc: - # Backwards-compatibility dictates we ignore failed - # imports triggered by fromlist for modules that don't - # exist. - if (exc.name == from_name and - sys.modules.get(from_name, _NEEDS_LOADING) is not None): - continue - raise + for x in fromlist: + if not isinstance(x, str): + if recursive: + where = module.__name__ + '.__all__' + else: + where = "``from list''" + raise TypeError(f"Item in {where} must be str, " + f"not {type(x).__name__}") + elif x == '*': + if not recursive and hasattr(module, '__all__'): + _handle_fromlist(module, module.__all__, import_, + recursive=True) + elif not hasattr(module, x): + from_name = '{}.{}'.format(module.__name__, x) + try: + _call_with_frames_removed(import_, from_name) + except ModuleNotFoundError as exc: + # Backwards-compatibility dictates we ignore failed + # imports triggered by fromlist for modules that don't + # exist. + if (exc.name == from_name and + sys.modules.get(from_name, _NEEDS_LOADING) is not None): + continue + raise return module @@ -1102,8 +1101,10 @@ def __import__(name, globals=None, locals=None, fromlist=(), level=0): # Slice end needs to be positive to alleviate need to special-case # when ``'.' not in name``. return sys.modules[module.__name__[:len(module.__name__)-cut_off]] - else: + elif hasattr(module, '__path__'): return _handle_fromlist(module, fromlist, _gcd_import) + else: + return module def _builtin_from_name(name): diff --git a/Misc/NEWS.d/next/Core and Builtins/2018-02-25-10-52-40.bpo-32946.Lo09rG.rst b/Misc/NEWS.d/next/Core and Builtins/2018-02-25-10-52-40.bpo-32946.Lo09rG.rst new file mode 100644 index 000000000000..cb1d2a7fb13c --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2018-02-25-10-52-40.bpo-32946.Lo09rG.rst @@ -0,0 +1,2 @@ +Importing names from already imported module with "from ... import ..." is +now 30% faster if the module is not a package. diff --git a/Python/import.c b/Python/import.c index eb5aeac55461..9f46da336234 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1800,10 +1800,21 @@ PyImport_ImportModuleLevelObject(PyObject *name, PyObject *globals, } } else { - final_mod = _PyObject_CallMethodIdObjArgs(interp->importlib, - &PyId__handle_fromlist, mod, - fromlist, interp->import_func, - NULL); + _Py_IDENTIFIER(__path__); + PyObject *path; + if (_PyObject_LookupAttrId(mod, &PyId___path__, &path) < 0) { + goto error; + } + if (path) { + Py_DECREF(path); + final_mod = _PyObject_CallMethodIdObjArgs( + interp->importlib, &PyId__handle_fromlist, + mod, fromlist, interp->import_func, NULL); + } + else { + final_mod = mod; + Py_INCREF(mod); + } } error: diff --git a/Python/importlib.h b/Python/importlib.h index 83c5dd5d650e..032ad45f96e9 100644 --- a/Python/importlib.h +++ b/Python/importlib.h @@ -1567,262 +1567,262 @@ const unsigned char _Py_M__importlib[] = { 0,115,8,0,0,0,0,9,12,1,8,1,12,1,114,181, 0,0,0,41,1,218,9,114,101,99,117,114,115,105,118,101, 99,3,0,0,0,1,0,0,0,8,0,0,0,11,0,0, - 0,67,0,0,0,115,236,0,0,0,116,0,124,0,100,1, - 131,2,114,232,124,1,68,0,93,216,125,4,116,1,124,4, - 116,2,131,2,115,76,124,3,114,44,124,0,106,3,100,2, - 23,0,125,5,110,4,100,3,125,5,116,4,100,4,124,5, - 155,0,100,5,116,5,124,4,131,1,106,3,155,0,157,4, - 131,1,130,1,113,14,124,4,100,6,107,2,114,118,124,3, - 115,230,116,0,124,0,100,7,131,2,114,230,116,6,124,0, - 124,0,106,7,124,2,100,8,100,9,141,4,1,0,113,14, - 116,0,124,0,124,4,131,2,115,14,100,10,160,8,124,0, - 106,3,124,4,161,2,125,6,122,14,116,9,124,2,124,6, - 131,2,1,0,87,0,113,14,4,0,116,10,107,10,114,228, - 1,0,125,7,1,0,122,42,124,7,106,11,124,6,107,2, - 114,210,116,12,106,13,160,14,124,6,116,15,161,2,100,11, - 107,9,114,210,87,0,89,0,162,8,113,14,130,0,87,0, - 53,0,100,11,125,7,126,7,88,0,89,0,113,14,88,0, - 113,14,124,0,83,0,41,12,122,238,70,105,103,117,114,101, - 32,111,117,116,32,119,104,97,116,32,95,95,105,109,112,111, - 114,116,95,95,32,115,104,111,117,108,100,32,114,101,116,117, - 114,110,46,10,10,32,32,32,32,84,104,101,32,105,109,112, - 111,114,116,95,32,112,97,114,97,109,101,116,101,114,32,105, - 115,32,97,32,99,97,108,108,97,98,108,101,32,119,104,105, - 99,104,32,116,97,107,101,115,32,116,104,101,32,110,97,109, - 101,32,111,102,32,109,111,100,117,108,101,32,116,111,10,32, - 32,32,32,105,109,112,111,114,116,46,32,73,116,32,105,115, - 32,114,101,113,117,105,114,101,100,32,116,111,32,100,101,99, - 111,117,112,108,101,32,116,104,101,32,102,117,110,99,116,105, - 111,110,32,102,114,111,109,32,97,115,115,117,109,105,110,103, - 32,105,109,112,111,114,116,108,105,98,39,115,10,32,32,32, - 32,105,109,112,111,114,116,32,105,109,112,108,101,109,101,110, - 116,97,116,105,111,110,32,105,115,32,100,101,115,105,114,101, - 100,46,10,10,32,32,32,32,114,127,0,0,0,122,8,46, - 95,95,97,108,108,95,95,122,13,96,96,102,114,111,109,32, - 108,105,115,116,39,39,122,8,73,116,101,109,32,105,110,32, - 122,18,32,109,117,115,116,32,98,101,32,115,116,114,44,32, - 110,111,116,32,250,1,42,218,7,95,95,97,108,108,95,95, - 84,41,1,114,182,0,0,0,122,5,123,125,46,123,125,78, - 41,16,114,4,0,0,0,114,170,0,0,0,114,171,0,0, - 0,114,1,0,0,0,114,172,0,0,0,114,13,0,0,0, - 218,16,95,104,97,110,100,108,101,95,102,114,111,109,108,105, - 115,116,114,184,0,0,0,114,38,0,0,0,114,59,0,0, - 0,114,176,0,0,0,114,15,0,0,0,114,14,0,0,0, - 114,79,0,0,0,114,30,0,0,0,114,179,0,0,0,41, - 8,114,83,0,0,0,218,8,102,114,111,109,108,105,115,116, - 114,177,0,0,0,114,182,0,0,0,218,1,120,90,5,119, - 104,101,114,101,90,9,102,114,111,109,95,110,97,109,101,90, - 3,101,120,99,114,10,0,0,0,114,10,0,0,0,114,11, - 0,0,0,114,185,0,0,0,241,3,0,0,115,42,0,0, - 0,0,10,10,1,8,1,10,1,4,1,12,2,4,1,28, - 2,8,1,14,1,10,1,10,1,10,1,14,1,2,1,14, - 1,16,4,10,1,18,1,8,1,22,1,114,185,0,0,0, - 99,1,0,0,0,0,0,0,0,3,0,0,0,6,0,0, - 0,67,0,0,0,115,146,0,0,0,124,0,160,0,100,1, - 161,1,125,1,124,0,160,0,100,2,161,1,125,2,124,1, - 100,3,107,9,114,82,124,2,100,3,107,9,114,78,124,1, - 124,2,106,1,107,3,114,78,116,2,106,3,100,4,124,1, - 155,2,100,5,124,2,106,1,155,2,100,6,157,5,116,4, - 100,7,100,8,141,3,1,0,124,1,83,0,124,2,100,3, - 107,9,114,96,124,2,106,1,83,0,116,2,106,3,100,9, - 116,4,100,7,100,8,141,3,1,0,124,0,100,10,25,0, - 125,1,100,11,124,0,107,7,114,142,124,1,160,5,100,12, - 161,1,100,13,25,0,125,1,124,1,83,0,41,14,122,167, - 67,97,108,99,117,108,97,116,101,32,119,104,97,116,32,95, - 95,112,97,99,107,97,103,101,95,95,32,115,104,111,117,108, - 100,32,98,101,46,10,10,32,32,32,32,95,95,112,97,99, - 107,97,103,101,95,95,32,105,115,32,110,111,116,32,103,117, - 97,114,97,110,116,101,101,100,32,116,111,32,98,101,32,100, - 101,102,105,110,101,100,32,111,114,32,99,111,117,108,100,32, - 98,101,32,115,101,116,32,116,111,32,78,111,110,101,10,32, - 32,32,32,116,111,32,114,101,112,114,101,115,101,110,116,32, - 116,104,97,116,32,105,116,115,32,112,114,111,112,101,114,32, - 118,97,108,117,101,32,105,115,32,117,110,107,110,111,119,110, - 46,10,10,32,32,32,32,114,130,0,0,0,114,89,0,0, - 0,78,122,32,95,95,112,97,99,107,97,103,101,95,95,32, - 33,61,32,95,95,115,112,101,99,95,95,46,112,97,114,101, - 110,116,32,40,122,4,32,33,61,32,250,1,41,233,3,0, - 0,0,41,1,90,10,115,116,97,99,107,108,101,118,101,108, - 122,89,99,97,110,39,116,32,114,101,115,111,108,118,101,32, - 112,97,99,107,97,103,101,32,102,114,111,109,32,95,95,115, - 112,101,99,95,95,32,111,114,32,95,95,112,97,99,107,97, - 103,101,95,95,44,32,102,97,108,108,105,110,103,32,98,97, - 99,107,32,111,110,32,95,95,110,97,109,101,95,95,32,97, - 110,100,32,95,95,112,97,116,104,95,95,114,1,0,0,0, - 114,127,0,0,0,114,117,0,0,0,114,19,0,0,0,41, - 6,114,30,0,0,0,114,119,0,0,0,114,166,0,0,0, - 114,167,0,0,0,114,168,0,0,0,114,118,0,0,0,41, - 3,218,7,103,108,111,98,97,108,115,114,160,0,0,0,114, - 82,0,0,0,114,10,0,0,0,114,10,0,0,0,114,11, - 0,0,0,218,17,95,99,97,108,99,95,95,95,112,97,99, - 107,97,103,101,95,95,23,4,0,0,115,30,0,0,0,0, - 7,10,1,10,1,8,1,18,1,22,2,10,1,4,1,8, - 1,6,2,6,2,10,1,8,1,8,1,14,1,114,191,0, - 0,0,114,10,0,0,0,99,5,0,0,0,0,0,0,0, - 9,0,0,0,5,0,0,0,67,0,0,0,115,166,0,0, - 0,124,4,100,1,107,2,114,18,116,0,124,0,131,1,125, - 5,110,36,124,1,100,2,107,9,114,30,124,1,110,2,105, - 0,125,6,116,1,124,6,131,1,125,7,116,0,124,0,124, - 7,124,4,131,3,125,5,124,3,115,150,124,4,100,1,107, - 2,114,84,116,0,124,0,160,2,100,3,161,1,100,1,25, - 0,131,1,83,0,124,0,115,92,124,5,83,0,116,3,124, - 0,131,1,116,3,124,0,160,2,100,3,161,1,100,1,25, - 0,131,1,24,0,125,8,116,4,106,5,124,5,106,6,100, - 2,116,3,124,5,106,6,131,1,124,8,24,0,133,2,25, - 0,25,0,83,0,110,12,116,7,124,5,124,3,116,0,131, - 3,83,0,100,2,83,0,41,4,97,215,1,0,0,73,109, - 112,111,114,116,32,97,32,109,111,100,117,108,101,46,10,10, - 32,32,32,32,84,104,101,32,39,103,108,111,98,97,108,115, - 39,32,97,114,103,117,109,101,110,116,32,105,115,32,117,115, - 101,100,32,116,111,32,105,110,102,101,114,32,119,104,101,114, - 101,32,116,104,101,32,105,109,112,111,114,116,32,105,115,32, - 111,99,99,117,114,114,105,110,103,32,102,114,111,109,10,32, - 32,32,32,116,111,32,104,97,110,100,108,101,32,114,101,108, - 97,116,105,118,101,32,105,109,112,111,114,116,115,46,32,84, - 104,101,32,39,108,111,99,97,108,115,39,32,97,114,103,117, - 109,101,110,116,32,105,115,32,105,103,110,111,114,101,100,46, - 32,84,104,101,10,32,32,32,32,39,102,114,111,109,108,105, - 115,116,39,32,97,114,103,117,109,101,110,116,32,115,112,101, - 99,105,102,105,101,115,32,119,104,97,116,32,115,104,111,117, - 108,100,32,101,120,105,115,116,32,97,115,32,97,116,116,114, - 105,98,117,116,101,115,32,111,110,32,116,104,101,32,109,111, - 100,117,108,101,10,32,32,32,32,98,101,105,110,103,32,105, - 109,112,111,114,116,101,100,32,40,101,46,103,46,32,96,96, - 102,114,111,109,32,109,111,100,117,108,101,32,105,109,112,111, - 114,116,32,60,102,114,111,109,108,105,115,116,62,96,96,41, - 46,32,32,84,104,101,32,39,108,101,118,101,108,39,10,32, - 32,32,32,97,114,103,117,109,101,110,116,32,114,101,112,114, - 101,115,101,110,116,115,32,116,104,101,32,112,97,99,107,97, - 103,101,32,108,111,99,97,116,105,111,110,32,116,111,32,105, - 109,112,111,114,116,32,102,114,111,109,32,105,110,32,97,32, - 114,101,108,97,116,105,118,101,10,32,32,32,32,105,109,112, - 111,114,116,32,40,101,46,103,46,32,96,96,102,114,111,109, - 32,46,46,112,107,103,32,105,109,112,111,114,116,32,109,111, - 100,96,96,32,119,111,117,108,100,32,104,97,118,101,32,97, - 32,39,108,101,118,101,108,39,32,111,102,32,50,41,46,10, - 10,32,32,32,32,114,19,0,0,0,78,114,117,0,0,0, - 41,8,114,181,0,0,0,114,191,0,0,0,218,9,112,97, - 114,116,105,116,105,111,110,114,158,0,0,0,114,14,0,0, - 0,114,79,0,0,0,114,1,0,0,0,114,185,0,0,0, - 41,9,114,15,0,0,0,114,190,0,0,0,218,6,108,111, - 99,97,108,115,114,186,0,0,0,114,161,0,0,0,114,83, - 0,0,0,90,8,103,108,111,98,97,108,115,95,114,160,0, - 0,0,90,7,99,117,116,95,111,102,102,114,10,0,0,0, - 114,10,0,0,0,114,11,0,0,0,218,10,95,95,105,109, - 112,111,114,116,95,95,50,4,0,0,115,26,0,0,0,0, - 11,8,1,10,2,16,1,8,1,12,1,4,3,8,1,18, - 1,4,1,4,4,26,3,32,2,114,194,0,0,0,99,1, - 0,0,0,0,0,0,0,2,0,0,0,3,0,0,0,67, - 0,0,0,115,38,0,0,0,116,0,160,1,124,0,161,1, - 125,1,124,1,100,0,107,8,114,30,116,2,100,1,124,0, - 23,0,131,1,130,1,116,3,124,1,131,1,83,0,41,2, - 78,122,25,110,111,32,98,117,105,108,116,45,105,110,32,109, - 111,100,117,108,101,32,110,97,109,101,100,32,41,4,114,141, - 0,0,0,114,145,0,0,0,114,70,0,0,0,114,140,0, - 0,0,41,2,114,15,0,0,0,114,82,0,0,0,114,10, - 0,0,0,114,10,0,0,0,114,11,0,0,0,218,18,95, - 98,117,105,108,116,105,110,95,102,114,111,109,95,110,97,109, - 101,85,4,0,0,115,8,0,0,0,0,1,10,1,8,1, - 12,1,114,195,0,0,0,99,2,0,0,0,0,0,0,0, - 10,0,0,0,5,0,0,0,67,0,0,0,115,166,0,0, - 0,124,1,97,0,124,0,97,1,116,2,116,1,131,1,125, - 2,116,1,106,3,160,4,161,0,68,0,93,72,92,2,125, - 3,125,4,116,5,124,4,124,2,131,2,114,26,124,3,116, - 1,106,6,107,6,114,60,116,7,125,5,110,18,116,0,160, - 8,124,3,161,1,114,26,116,9,125,5,110,2,113,26,116, - 10,124,4,124,5,131,2,125,6,116,11,124,6,124,4,131, - 2,1,0,113,26,116,1,106,3,116,12,25,0,125,7,100, - 1,68,0,93,46,125,8,124,8,116,1,106,3,107,7,114, - 138,116,13,124,8,131,1,125,9,110,10,116,1,106,3,124, - 8,25,0,125,9,116,14,124,7,124,8,124,9,131,3,1, - 0,113,114,100,2,83,0,41,3,122,250,83,101,116,117,112, - 32,105,109,112,111,114,116,108,105,98,32,98,121,32,105,109, - 112,111,114,116,105,110,103,32,110,101,101,100,101,100,32,98, - 117,105,108,116,45,105,110,32,109,111,100,117,108,101,115,32, - 97,110,100,32,105,110,106,101,99,116,105,110,103,32,116,104, - 101,109,10,32,32,32,32,105,110,116,111,32,116,104,101,32, - 103,108,111,98,97,108,32,110,97,109,101,115,112,97,99,101, - 46,10,10,32,32,32,32,65,115,32,115,121,115,32,105,115, - 32,110,101,101,100,101,100,32,102,111,114,32,115,121,115,46, - 109,111,100,117,108,101,115,32,97,99,99,101,115,115,32,97, - 110,100,32,95,105,109,112,32,105,115,32,110,101,101,100,101, - 100,32,116,111,32,108,111,97,100,32,98,117,105,108,116,45, - 105,110,10,32,32,32,32,109,111,100,117,108,101,115,44,32, - 116,104,111,115,101,32,116,119,111,32,109,111,100,117,108,101, - 115,32,109,117,115,116,32,98,101,32,101,120,112,108,105,99, - 105,116,108,121,32,112,97,115,115,101,100,32,105,110,46,10, - 10,32,32,32,32,41,3,114,20,0,0,0,114,166,0,0, - 0,114,56,0,0,0,78,41,15,114,49,0,0,0,114,14, - 0,0,0,114,13,0,0,0,114,79,0,0,0,218,5,105, - 116,101,109,115,114,170,0,0,0,114,69,0,0,0,114,141, - 0,0,0,114,75,0,0,0,114,151,0,0,0,114,128,0, - 0,0,114,133,0,0,0,114,1,0,0,0,114,195,0,0, - 0,114,5,0,0,0,41,10,218,10,115,121,115,95,109,111, - 100,117,108,101,218,11,95,105,109,112,95,109,111,100,117,108, - 101,90,11,109,111,100,117,108,101,95,116,121,112,101,114,15, - 0,0,0,114,83,0,0,0,114,93,0,0,0,114,82,0, - 0,0,90,11,115,101,108,102,95,109,111,100,117,108,101,90, - 12,98,117,105,108,116,105,110,95,110,97,109,101,90,14,98, - 117,105,108,116,105,110,95,109,111,100,117,108,101,114,10,0, - 0,0,114,10,0,0,0,114,11,0,0,0,218,6,95,115, - 101,116,117,112,92,4,0,0,115,36,0,0,0,0,9,4, - 1,4,3,8,1,18,1,10,1,10,1,6,1,10,1,6, - 2,2,1,10,1,12,3,10,1,8,1,10,1,10,2,10, - 1,114,199,0,0,0,99,2,0,0,0,0,0,0,0,2, - 0,0,0,3,0,0,0,67,0,0,0,115,38,0,0,0, - 116,0,124,0,124,1,131,2,1,0,116,1,106,2,160,3, - 116,4,161,1,1,0,116,1,106,2,160,3,116,5,161,1, - 1,0,100,1,83,0,41,2,122,48,73,110,115,116,97,108, - 108,32,105,109,112,111,114,116,101,114,115,32,102,111,114,32, - 98,117,105,108,116,105,110,32,97,110,100,32,102,114,111,122, - 101,110,32,109,111,100,117,108,101,115,78,41,6,114,199,0, - 0,0,114,14,0,0,0,114,165,0,0,0,114,109,0,0, - 0,114,141,0,0,0,114,151,0,0,0,41,2,114,197,0, - 0,0,114,198,0,0,0,114,10,0,0,0,114,10,0,0, - 0,114,11,0,0,0,218,8,95,105,110,115,116,97,108,108, - 127,4,0,0,115,6,0,0,0,0,2,10,2,12,1,114, - 200,0,0,0,99,0,0,0,0,0,0,0,0,1,0,0, - 0,4,0,0,0,67,0,0,0,115,32,0,0,0,100,1, - 100,2,108,0,125,0,124,0,97,1,124,0,160,2,116,3, - 106,4,116,5,25,0,161,1,1,0,100,2,83,0,41,3, - 122,57,73,110,115,116,97,108,108,32,105,109,112,111,114,116, - 101,114,115,32,116,104,97,116,32,114,101,113,117,105,114,101, - 32,101,120,116,101,114,110,97,108,32,102,105,108,101,115,121, - 115,116,101,109,32,97,99,99,101,115,115,114,19,0,0,0, - 78,41,6,218,26,95,102,114,111,122,101,110,95,105,109,112, - 111,114,116,108,105,98,95,101,120,116,101,114,110,97,108,114, - 115,0,0,0,114,200,0,0,0,114,14,0,0,0,114,79, - 0,0,0,114,1,0,0,0,41,1,114,201,0,0,0,114, - 10,0,0,0,114,10,0,0,0,114,11,0,0,0,218,27, - 95,105,110,115,116,97,108,108,95,101,120,116,101,114,110,97, - 108,95,105,109,112,111,114,116,101,114,115,135,4,0,0,115, - 6,0,0,0,0,3,8,1,4,1,114,202,0,0,0,41, - 2,78,78,41,1,78,41,2,78,114,19,0,0,0,41,4, - 78,78,114,10,0,0,0,114,19,0,0,0,41,51,114,3, - 0,0,0,114,115,0,0,0,114,12,0,0,0,114,16,0, - 0,0,114,51,0,0,0,114,29,0,0,0,114,36,0,0, - 0,114,17,0,0,0,114,18,0,0,0,114,41,0,0,0, - 114,42,0,0,0,114,45,0,0,0,114,57,0,0,0,114, - 59,0,0,0,114,68,0,0,0,114,74,0,0,0,114,77, - 0,0,0,114,84,0,0,0,114,95,0,0,0,114,96,0, - 0,0,114,102,0,0,0,114,78,0,0,0,114,128,0,0, - 0,114,133,0,0,0,114,136,0,0,0,114,91,0,0,0, - 114,80,0,0,0,114,139,0,0,0,114,140,0,0,0,114, - 81,0,0,0,114,141,0,0,0,114,151,0,0,0,114,156, - 0,0,0,114,162,0,0,0,114,164,0,0,0,114,169,0, - 0,0,114,173,0,0,0,90,15,95,69,82,82,95,77,83, - 71,95,80,82,69,70,73,88,114,175,0,0,0,114,178,0, - 0,0,218,6,111,98,106,101,99,116,114,179,0,0,0,114, - 180,0,0,0,114,181,0,0,0,114,185,0,0,0,114,191, - 0,0,0,114,194,0,0,0,114,195,0,0,0,114,199,0, - 0,0,114,200,0,0,0,114,202,0,0,0,114,10,0,0, - 0,114,10,0,0,0,114,10,0,0,0,114,11,0,0,0, - 218,8,60,109,111,100,117,108,101,62,25,0,0,0,115,96, - 0,0,0,4,0,4,2,8,8,8,8,4,2,4,3,16, - 4,14,68,14,21,14,16,8,37,8,17,8,11,14,8,8, - 11,8,12,8,16,8,36,14,27,14,101,16,26,10,45,14, - 72,8,17,8,17,8,24,8,29,8,23,8,15,14,73,14, - 77,14,13,8,9,8,9,10,47,8,16,4,1,8,2,8, - 27,6,3,8,16,10,15,14,38,8,27,10,35,8,7,8, - 35,8,8, + 0,67,0,0,0,115,226,0,0,0,124,1,68,0,93,216, + 125,4,116,0,124,4,116,1,131,2,115,66,124,3,114,34, + 124,0,106,2,100,1,23,0,125,5,110,4,100,2,125,5, + 116,3,100,3,124,5,155,0,100,4,116,4,124,4,131,1, + 106,2,155,0,157,4,131,1,130,1,113,4,124,4,100,5, + 107,2,114,108,124,3,115,220,116,5,124,0,100,6,131,2, + 114,220,116,6,124,0,124,0,106,7,124,2,100,7,100,8, + 141,4,1,0,113,4,116,5,124,0,124,4,131,2,115,4, + 100,9,160,8,124,0,106,2,124,4,161,2,125,6,122,14, + 116,9,124,2,124,6,131,2,1,0,87,0,113,4,4,0, + 116,10,107,10,114,218,1,0,125,7,1,0,122,42,124,7, + 106,11,124,6,107,2,114,200,116,12,106,13,160,14,124,6, + 116,15,161,2,100,10,107,9,114,200,87,0,89,0,162,8, + 113,4,130,0,87,0,53,0,100,10,125,7,126,7,88,0, + 89,0,113,4,88,0,113,4,124,0,83,0,41,11,122,238, + 70,105,103,117,114,101,32,111,117,116,32,119,104,97,116,32, + 95,95,105,109,112,111,114,116,95,95,32,115,104,111,117,108, + 100,32,114,101,116,117,114,110,46,10,10,32,32,32,32,84, + 104,101,32,105,109,112,111,114,116,95,32,112,97,114,97,109, + 101,116,101,114,32,105,115,32,97,32,99,97,108,108,97,98, + 108,101,32,119,104,105,99,104,32,116,97,107,101,115,32,116, + 104,101,32,110,97,109,101,32,111,102,32,109,111,100,117,108, + 101,32,116,111,10,32,32,32,32,105,109,112,111,114,116,46, + 32,73,116,32,105,115,32,114,101,113,117,105,114,101,100,32, + 116,111,32,100,101,99,111,117,112,108,101,32,116,104,101,32, + 102,117,110,99,116,105,111,110,32,102,114,111,109,32,97,115, + 115,117,109,105,110,103,32,105,109,112,111,114,116,108,105,98, + 39,115,10,32,32,32,32,105,109,112,111,114,116,32,105,109, + 112,108,101,109,101,110,116,97,116,105,111,110,32,105,115,32, + 100,101,115,105,114,101,100,46,10,10,32,32,32,32,122,8, + 46,95,95,97,108,108,95,95,122,13,96,96,102,114,111,109, + 32,108,105,115,116,39,39,122,8,73,116,101,109,32,105,110, + 32,122,18,32,109,117,115,116,32,98,101,32,115,116,114,44, + 32,110,111,116,32,250,1,42,218,7,95,95,97,108,108,95, + 95,84,41,1,114,182,0,0,0,122,5,123,125,46,123,125, + 78,41,16,114,170,0,0,0,114,171,0,0,0,114,1,0, + 0,0,114,172,0,0,0,114,13,0,0,0,114,4,0,0, + 0,218,16,95,104,97,110,100,108,101,95,102,114,111,109,108, + 105,115,116,114,184,0,0,0,114,38,0,0,0,114,59,0, + 0,0,114,176,0,0,0,114,15,0,0,0,114,14,0,0, + 0,114,79,0,0,0,114,30,0,0,0,114,179,0,0,0, + 41,8,114,83,0,0,0,218,8,102,114,111,109,108,105,115, + 116,114,177,0,0,0,114,182,0,0,0,218,1,120,90,5, + 119,104,101,114,101,90,9,102,114,111,109,95,110,97,109,101, + 90,3,101,120,99,114,10,0,0,0,114,10,0,0,0,114, + 11,0,0,0,114,185,0,0,0,241,3,0,0,115,40,0, + 0,0,0,10,8,1,10,1,4,1,12,2,4,1,28,2, + 8,1,14,1,10,1,10,1,10,1,14,1,2,1,14,1, + 16,4,10,1,18,1,8,1,22,1,114,185,0,0,0,99, + 1,0,0,0,0,0,0,0,3,0,0,0,6,0,0,0, + 67,0,0,0,115,146,0,0,0,124,0,160,0,100,1,161, + 1,125,1,124,0,160,0,100,2,161,1,125,2,124,1,100, + 3,107,9,114,82,124,2,100,3,107,9,114,78,124,1,124, + 2,106,1,107,3,114,78,116,2,106,3,100,4,124,1,155, + 2,100,5,124,2,106,1,155,2,100,6,157,5,116,4,100, + 7,100,8,141,3,1,0,124,1,83,0,124,2,100,3,107, + 9,114,96,124,2,106,1,83,0,116,2,106,3,100,9,116, + 4,100,7,100,8,141,3,1,0,124,0,100,10,25,0,125, + 1,100,11,124,0,107,7,114,142,124,1,160,5,100,12,161, + 1,100,13,25,0,125,1,124,1,83,0,41,14,122,167,67, + 97,108,99,117,108,97,116,101,32,119,104,97,116,32,95,95, + 112,97,99,107,97,103,101,95,95,32,115,104,111,117,108,100, + 32,98,101,46,10,10,32,32,32,32,95,95,112,97,99,107, + 97,103,101,95,95,32,105,115,32,110,111,116,32,103,117,97, + 114,97,110,116,101,101,100,32,116,111,32,98,101,32,100,101, + 102,105,110,101,100,32,111,114,32,99,111,117,108,100,32,98, + 101,32,115,101,116,32,116,111,32,78,111,110,101,10,32,32, + 32,32,116,111,32,114,101,112,114,101,115,101,110,116,32,116, + 104,97,116,32,105,116,115,32,112,114,111,112,101,114,32,118, + 97,108,117,101,32,105,115,32,117,110,107,110,111,119,110,46, + 10,10,32,32,32,32,114,130,0,0,0,114,89,0,0,0, + 78,122,32,95,95,112,97,99,107,97,103,101,95,95,32,33, + 61,32,95,95,115,112,101,99,95,95,46,112,97,114,101,110, + 116,32,40,122,4,32,33,61,32,250,1,41,233,3,0,0, + 0,41,1,90,10,115,116,97,99,107,108,101,118,101,108,122, + 89,99,97,110,39,116,32,114,101,115,111,108,118,101,32,112, + 97,99,107,97,103,101,32,102,114,111,109,32,95,95,115,112, + 101,99,95,95,32,111,114,32,95,95,112,97,99,107,97,103, + 101,95,95,44,32,102,97,108,108,105,110,103,32,98,97,99, + 107,32,111,110,32,95,95,110,97,109,101,95,95,32,97,110, + 100,32,95,95,112,97,116,104,95,95,114,1,0,0,0,114, + 127,0,0,0,114,117,0,0,0,114,19,0,0,0,41,6, + 114,30,0,0,0,114,119,0,0,0,114,166,0,0,0,114, + 167,0,0,0,114,168,0,0,0,114,118,0,0,0,41,3, + 218,7,103,108,111,98,97,108,115,114,160,0,0,0,114,82, + 0,0,0,114,10,0,0,0,114,10,0,0,0,114,11,0, + 0,0,218,17,95,99,97,108,99,95,95,95,112,97,99,107, + 97,103,101,95,95,22,4,0,0,115,30,0,0,0,0,7, + 10,1,10,1,8,1,18,1,22,2,10,1,4,1,8,1, + 6,2,6,2,10,1,8,1,8,1,14,1,114,191,0,0, + 0,114,10,0,0,0,99,5,0,0,0,0,0,0,0,9, + 0,0,0,5,0,0,0,67,0,0,0,115,180,0,0,0, + 124,4,100,1,107,2,114,18,116,0,124,0,131,1,125,5, + 110,36,124,1,100,2,107,9,114,30,124,1,110,2,105,0, + 125,6,116,1,124,6,131,1,125,7,116,0,124,0,124,7, + 124,4,131,3,125,5,124,3,115,150,124,4,100,1,107,2, + 114,84,116,0,124,0,160,2,100,3,161,1,100,1,25,0, + 131,1,83,0,124,0,115,92,124,5,83,0,116,3,124,0, + 131,1,116,3,124,0,160,2,100,3,161,1,100,1,25,0, + 131,1,24,0,125,8,116,4,106,5,124,5,106,6,100,2, + 116,3,124,5,106,6,131,1,124,8,24,0,133,2,25,0, + 25,0,83,0,110,26,116,7,124,5,100,4,131,2,114,172, + 116,8,124,5,124,3,116,0,131,3,83,0,124,5,83,0, + 100,2,83,0,41,5,97,215,1,0,0,73,109,112,111,114, + 116,32,97,32,109,111,100,117,108,101,46,10,10,32,32,32, + 32,84,104,101,32,39,103,108,111,98,97,108,115,39,32,97, + 114,103,117,109,101,110,116,32,105,115,32,117,115,101,100,32, + 116,111,32,105,110,102,101,114,32,119,104,101,114,101,32,116, + 104,101,32,105,109,112,111,114,116,32,105,115,32,111,99,99, + 117,114,114,105,110,103,32,102,114,111,109,10,32,32,32,32, + 116,111,32,104,97,110,100,108,101,32,114,101,108,97,116,105, + 118,101,32,105,109,112,111,114,116,115,46,32,84,104,101,32, + 39,108,111,99,97,108,115,39,32,97,114,103,117,109,101,110, + 116,32,105,115,32,105,103,110,111,114,101,100,46,32,84,104, + 101,10,32,32,32,32,39,102,114,111,109,108,105,115,116,39, + 32,97,114,103,117,109,101,110,116,32,115,112,101,99,105,102, + 105,101,115,32,119,104,97,116,32,115,104,111,117,108,100,32, + 101,120,105,115,116,32,97,115,32,97,116,116,114,105,98,117, + 116,101,115,32,111,110,32,116,104,101,32,109,111,100,117,108, + 101,10,32,32,32,32,98,101,105,110,103,32,105,109,112,111, + 114,116,101,100,32,40,101,46,103,46,32,96,96,102,114,111, + 109,32,109,111,100,117,108,101,32,105,109,112,111,114,116,32, + 60,102,114,111,109,108,105,115,116,62,96,96,41,46,32,32, + 84,104,101,32,39,108,101,118,101,108,39,10,32,32,32,32, + 97,114,103,117,109,101,110,116,32,114,101,112,114,101,115,101, + 110,116,115,32,116,104,101,32,112,97,99,107,97,103,101,32, + 108,111,99,97,116,105,111,110,32,116,111,32,105,109,112,111, + 114,116,32,102,114,111,109,32,105,110,32,97,32,114,101,108, + 97,116,105,118,101,10,32,32,32,32,105,109,112,111,114,116, + 32,40,101,46,103,46,32,96,96,102,114,111,109,32,46,46, + 112,107,103,32,105,109,112,111,114,116,32,109,111,100,96,96, + 32,119,111,117,108,100,32,104,97,118,101,32,97,32,39,108, + 101,118,101,108,39,32,111,102,32,50,41,46,10,10,32,32, + 32,32,114,19,0,0,0,78,114,117,0,0,0,114,127,0, + 0,0,41,9,114,181,0,0,0,114,191,0,0,0,218,9, + 112,97,114,116,105,116,105,111,110,114,158,0,0,0,114,14, + 0,0,0,114,79,0,0,0,114,1,0,0,0,114,4,0, + 0,0,114,185,0,0,0,41,9,114,15,0,0,0,114,190, + 0,0,0,218,6,108,111,99,97,108,115,114,186,0,0,0, + 114,161,0,0,0,114,83,0,0,0,90,8,103,108,111,98, + 97,108,115,95,114,160,0,0,0,90,7,99,117,116,95,111, + 102,102,114,10,0,0,0,114,10,0,0,0,114,11,0,0, + 0,218,10,95,95,105,109,112,111,114,116,95,95,49,4,0, + 0,115,30,0,0,0,0,11,8,1,10,2,16,1,8,1, + 12,1,4,3,8,1,18,1,4,1,4,4,26,3,32,1, + 10,1,12,2,114,194,0,0,0,99,1,0,0,0,0,0, + 0,0,2,0,0,0,3,0,0,0,67,0,0,0,115,38, + 0,0,0,116,0,160,1,124,0,161,1,125,1,124,1,100, + 0,107,8,114,30,116,2,100,1,124,0,23,0,131,1,130, + 1,116,3,124,1,131,1,83,0,41,2,78,122,25,110,111, + 32,98,117,105,108,116,45,105,110,32,109,111,100,117,108,101, + 32,110,97,109,101,100,32,41,4,114,141,0,0,0,114,145, + 0,0,0,114,70,0,0,0,114,140,0,0,0,41,2,114, + 15,0,0,0,114,82,0,0,0,114,10,0,0,0,114,10, + 0,0,0,114,11,0,0,0,218,18,95,98,117,105,108,116, + 105,110,95,102,114,111,109,95,110,97,109,101,86,4,0,0, + 115,8,0,0,0,0,1,10,1,8,1,12,1,114,195,0, + 0,0,99,2,0,0,0,0,0,0,0,10,0,0,0,5, + 0,0,0,67,0,0,0,115,166,0,0,0,124,1,97,0, + 124,0,97,1,116,2,116,1,131,1,125,2,116,1,106,3, + 160,4,161,0,68,0,93,72,92,2,125,3,125,4,116,5, + 124,4,124,2,131,2,114,26,124,3,116,1,106,6,107,6, + 114,60,116,7,125,5,110,18,116,0,160,8,124,3,161,1, + 114,26,116,9,125,5,110,2,113,26,116,10,124,4,124,5, + 131,2,125,6,116,11,124,6,124,4,131,2,1,0,113,26, + 116,1,106,3,116,12,25,0,125,7,100,1,68,0,93,46, + 125,8,124,8,116,1,106,3,107,7,114,138,116,13,124,8, + 131,1,125,9,110,10,116,1,106,3,124,8,25,0,125,9, + 116,14,124,7,124,8,124,9,131,3,1,0,113,114,100,2, + 83,0,41,3,122,250,83,101,116,117,112,32,105,109,112,111, + 114,116,108,105,98,32,98,121,32,105,109,112,111,114,116,105, + 110,103,32,110,101,101,100,101,100,32,98,117,105,108,116,45, + 105,110,32,109,111,100,117,108,101,115,32,97,110,100,32,105, + 110,106,101,99,116,105,110,103,32,116,104,101,109,10,32,32, + 32,32,105,110,116,111,32,116,104,101,32,103,108,111,98,97, + 108,32,110,97,109,101,115,112,97,99,101,46,10,10,32,32, + 32,32,65,115,32,115,121,115,32,105,115,32,110,101,101,100, + 101,100,32,102,111,114,32,115,121,115,46,109,111,100,117,108, + 101,115,32,97,99,99,101,115,115,32,97,110,100,32,95,105, + 109,112,32,105,115,32,110,101,101,100,101,100,32,116,111,32, + 108,111,97,100,32,98,117,105,108,116,45,105,110,10,32,32, + 32,32,109,111,100,117,108,101,115,44,32,116,104,111,115,101, + 32,116,119,111,32,109,111,100,117,108,101,115,32,109,117,115, + 116,32,98,101,32,101,120,112,108,105,99,105,116,108,121,32, + 112,97,115,115,101,100,32,105,110,46,10,10,32,32,32,32, + 41,3,114,20,0,0,0,114,166,0,0,0,114,56,0,0, + 0,78,41,15,114,49,0,0,0,114,14,0,0,0,114,13, + 0,0,0,114,79,0,0,0,218,5,105,116,101,109,115,114, + 170,0,0,0,114,69,0,0,0,114,141,0,0,0,114,75, + 0,0,0,114,151,0,0,0,114,128,0,0,0,114,133,0, + 0,0,114,1,0,0,0,114,195,0,0,0,114,5,0,0, + 0,41,10,218,10,115,121,115,95,109,111,100,117,108,101,218, + 11,95,105,109,112,95,109,111,100,117,108,101,90,11,109,111, + 100,117,108,101,95,116,121,112,101,114,15,0,0,0,114,83, + 0,0,0,114,93,0,0,0,114,82,0,0,0,90,11,115, + 101,108,102,95,109,111,100,117,108,101,90,12,98,117,105,108, + 116,105,110,95,110,97,109,101,90,14,98,117,105,108,116,105, + 110,95,109,111,100,117,108,101,114,10,0,0,0,114,10,0, + 0,0,114,11,0,0,0,218,6,95,115,101,116,117,112,93, + 4,0,0,115,36,0,0,0,0,9,4,1,4,3,8,1, + 18,1,10,1,10,1,6,1,10,1,6,2,2,1,10,1, + 12,3,10,1,8,1,10,1,10,2,10,1,114,199,0,0, + 0,99,2,0,0,0,0,0,0,0,2,0,0,0,3,0, + 0,0,67,0,0,0,115,38,0,0,0,116,0,124,0,124, + 1,131,2,1,0,116,1,106,2,160,3,116,4,161,1,1, + 0,116,1,106,2,160,3,116,5,161,1,1,0,100,1,83, + 0,41,2,122,48,73,110,115,116,97,108,108,32,105,109,112, + 111,114,116,101,114,115,32,102,111,114,32,98,117,105,108,116, + 105,110,32,97,110,100,32,102,114,111,122,101,110,32,109,111, + 100,117,108,101,115,78,41,6,114,199,0,0,0,114,14,0, + 0,0,114,165,0,0,0,114,109,0,0,0,114,141,0,0, + 0,114,151,0,0,0,41,2,114,197,0,0,0,114,198,0, + 0,0,114,10,0,0,0,114,10,0,0,0,114,11,0,0, + 0,218,8,95,105,110,115,116,97,108,108,128,4,0,0,115, + 6,0,0,0,0,2,10,2,12,1,114,200,0,0,0,99, + 0,0,0,0,0,0,0,0,1,0,0,0,4,0,0,0, + 67,0,0,0,115,32,0,0,0,100,1,100,2,108,0,125, + 0,124,0,97,1,124,0,160,2,116,3,106,4,116,5,25, + 0,161,1,1,0,100,2,83,0,41,3,122,57,73,110,115, + 116,97,108,108,32,105,109,112,111,114,116,101,114,115,32,116, + 104,97,116,32,114,101,113,117,105,114,101,32,101,120,116,101, + 114,110,97,108,32,102,105,108,101,115,121,115,116,101,109,32, + 97,99,99,101,115,115,114,19,0,0,0,78,41,6,218,26, + 95,102,114,111,122,101,110,95,105,109,112,111,114,116,108,105, + 98,95,101,120,116,101,114,110,97,108,114,115,0,0,0,114, + 200,0,0,0,114,14,0,0,0,114,79,0,0,0,114,1, + 0,0,0,41,1,114,201,0,0,0,114,10,0,0,0,114, + 10,0,0,0,114,11,0,0,0,218,27,95,105,110,115,116, + 97,108,108,95,101,120,116,101,114,110,97,108,95,105,109,112, + 111,114,116,101,114,115,136,4,0,0,115,6,0,0,0,0, + 3,8,1,4,1,114,202,0,0,0,41,2,78,78,41,1, + 78,41,2,78,114,19,0,0,0,41,4,78,78,114,10,0, + 0,0,114,19,0,0,0,41,51,114,3,0,0,0,114,115, + 0,0,0,114,12,0,0,0,114,16,0,0,0,114,51,0, + 0,0,114,29,0,0,0,114,36,0,0,0,114,17,0,0, + 0,114,18,0,0,0,114,41,0,0,0,114,42,0,0,0, + 114,45,0,0,0,114,57,0,0,0,114,59,0,0,0,114, + 68,0,0,0,114,74,0,0,0,114,77,0,0,0,114,84, + 0,0,0,114,95,0,0,0,114,96,0,0,0,114,102,0, + 0,0,114,78,0,0,0,114,128,0,0,0,114,133,0,0, + 0,114,136,0,0,0,114,91,0,0,0,114,80,0,0,0, + 114,139,0,0,0,114,140,0,0,0,114,81,0,0,0,114, + 141,0,0,0,114,151,0,0,0,114,156,0,0,0,114,162, + 0,0,0,114,164,0,0,0,114,169,0,0,0,114,173,0, + 0,0,90,15,95,69,82,82,95,77,83,71,95,80,82,69, + 70,73,88,114,175,0,0,0,114,178,0,0,0,218,6,111, + 98,106,101,99,116,114,179,0,0,0,114,180,0,0,0,114, + 181,0,0,0,114,185,0,0,0,114,191,0,0,0,114,194, + 0,0,0,114,195,0,0,0,114,199,0,0,0,114,200,0, + 0,0,114,202,0,0,0,114,10,0,0,0,114,10,0,0, + 0,114,10,0,0,0,114,11,0,0,0,218,8,60,109,111, + 100,117,108,101,62,25,0,0,0,115,96,0,0,0,4,0, + 4,2,8,8,8,8,4,2,4,3,16,4,14,68,14,21, + 14,16,8,37,8,17,8,11,14,8,8,11,8,12,8,16, + 8,36,14,27,14,101,16,26,10,45,14,72,8,17,8,17, + 8,24,8,29,8,23,8,15,14,73,14,77,14,13,8,9, + 8,9,10,47,8,16,4,1,8,2,8,27,6,3,8,16, + 10,15,14,37,8,27,10,37,8,7,8,35,8,8, }; From webhook-mailer at python.org Sun Mar 11 04:54:50 2018 From: webhook-mailer at python.org (Serhiy Storchaka) Date: Sun, 11 Mar 2018 08:54:50 -0000 Subject: [Python-checkins] bpo-32925: Optimized iterating and containing test for literal lists (GH-5842) Message-ID: <mailman.43.1520758491.1871.python-checkins@python.org> https://github.com/python/cpython/commit/3f7e9aa2ef215917b9f1521441f67f4ecd33a1bc commit: 3f7e9aa2ef215917b9f1521441f67f4ecd33a1bc branch: master author: Serhiy Storchaka <storchaka at gmail.com> committer: GitHub <noreply at github.com> date: 2018-03-11T10:54:47+02:00 summary: bpo-32925: Optimized iterating and containing test for literal lists (GH-5842) consisting of non-constants: `x in [a, b]` and `for x in [a, b]`. The case of all constant elements already was optimized. files: A Misc/NEWS.d/next/Core and Builtins/2018-02-24-00-07-05.bpo-32925.e-7Ufh.rst M Lib/test/test_peepholer.py M Python/ast_opt.c diff --git a/Lib/test/test_peepholer.py b/Lib/test/test_peepholer.py index c24cf77e7b2d..794d104d5919 100644 --- a/Lib/test/test_peepholer.py +++ b/Lib/test/test_peepholer.py @@ -3,6 +3,19 @@ from test.bytecode_helper import BytecodeTestCase +def count_instr_recursively(f, opname): + count = 0 + for instr in dis.get_instructions(f): + if instr.opname == opname: + count += 1 + if hasattr(f, '__code__'): + f = f.__code__ + for c in f.co_consts: + if hasattr(c, 'co_code'): + count += count_instr_recursively(c, opname) + return count + + class TestTranforms(BytecodeTestCase): def test_unot(self): @@ -311,6 +324,17 @@ def test_constant_folding(self): self.assertFalse(instr.opname.startswith('BINARY_')) self.assertFalse(instr.opname.startswith('BUILD_')) + def test_in_literal_list(self): + def containtest(): + return x in [a, b] + self.assertEqual(count_instr_recursively(containtest, 'BUILD_LIST'), 0) + + def test_iterate_literal_list(self): + def forloop(): + for x in [a, b]: + pass + self.assertEqual(count_instr_recursively(forloop, 'BUILD_LIST'), 0) + class TestBuglets(unittest.TestCase): diff --git a/Misc/NEWS.d/next/Core and Builtins/2018-02-24-00-07-05.bpo-32925.e-7Ufh.rst b/Misc/NEWS.d/next/Core and Builtins/2018-02-24-00-07-05.bpo-32925.e-7Ufh.rst new file mode 100644 index 000000000000..e9443e69e2a2 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2018-02-24-00-07-05.bpo-32925.e-7Ufh.rst @@ -0,0 +1,3 @@ +Optimized iterating and containing test for literal lists consisting of +non-constants: ``x in [a, b]`` and ``for x in [a, b]``. The case of all +constant elements already was optimized. diff --git a/Python/ast_opt.c b/Python/ast_opt.c index 65cf3c126417..a54f98c8c721 100644 --- a/Python/ast_opt.c +++ b/Python/ast_opt.c @@ -369,7 +369,8 @@ fold_subscr(expr_ty node, PyArena *arena, int optimize) } /* Change literal list or set of constants into constant - tuple or frozenset respectively. + tuple or frozenset respectively. Change literal list of + non-constants into tuple. Used for right operand of "in" and "not in" tests and for iterable in "for" loop and comprehensions. */ @@ -378,7 +379,21 @@ fold_iter(expr_ty arg, PyArena *arena, int optimize) { PyObject *newval; if (arg->kind == List_kind) { - newval = make_const_tuple(arg->v.List.elts); + /* First change a list into tuple. */ + asdl_seq *elts = arg->v.List.elts; + Py_ssize_t n = asdl_seq_LEN(elts); + for (Py_ssize_t i = 0; i < n; i++) { + expr_ty e = (expr_ty)asdl_seq_GET(elts, i); + if (e->kind == Starred_kind) { + return 1; + } + } + expr_context_ty ctx = arg->v.List.ctx; + arg->kind = Tuple_kind; + arg->v.Tuple.elts = elts; + arg->v.Tuple.ctx = ctx; + /* Try to create a constant tuple. */ + newval = make_const_tuple(elts); } else if (arg->kind == Set_kind) { newval = make_const_tuple(arg->v.Set.elts); From webhook-mailer at python.org Sun Mar 11 04:56:02 2018 From: webhook-mailer at python.org (Serhiy Storchaka) Date: Sun, 11 Mar 2018 08:56:02 -0000 Subject: [Python-checkins] [2.7] bpo-33026: Fix jumping out of "with" block by setting f_lineno. (GH-6026). (GH-6074) (GH-6076) Message-ID: <mailman.44.1520758564.1871.python-checkins@python.org> https://github.com/python/cpython/commit/3854f5885edc8dc67b1aba82fbb525604fbc625b commit: 3854f5885edc8dc67b1aba82fbb525604fbc625b branch: 2.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Serhiy Storchaka <storchaka at gmail.com> date: 2018-03-11T10:55:59+02:00 summary: [2.7] bpo-33026: Fix jumping out of "with" block by setting f_lineno. (GH-6026). (GH-6074) (GH-6076) (cherry picked from commit 26c9f565d016db21257a60d29ab2c99383dd5ac7) (cherry picked from commit 04aadf23eac51fec2e436c5960c1362bbb7d03de) Co-authored-by: Serhiy Storchaka <storchaka at gmail.com> files: A Misc/NEWS.d/next/Core and Builtins/2018-03-08-09-48-38.bpo-33026.QZA3Ba.rst M Lib/test/test_sys_settrace.py M Objects/frameobject.c diff --git a/Lib/test/test_sys_settrace.py b/Lib/test/test_sys_settrace.py index 7e1d7a131f5a..04b1ac339a49 100644 --- a/Lib/test/test_sys_settrace.py +++ b/Lib/test/test_sys_settrace.py @@ -723,6 +723,34 @@ def test_jump_across_with(output): with tracecontext(output, 4): output.append(5) + @jump_test(4, 5, [1, 3, 5, 6]) + def test_jump_out_of_with_block_within_for_block(output): + output.append(1) + for i in [1]: + with tracecontext(output, 3): + output.append(4) + output.append(5) + output.append(6) + + @jump_test(4, 5, [1, 2, 3, 5, -2, 6]) + def test_jump_out_of_with_block_within_with_block(output): + output.append(1) + with tracecontext(output, 2): + with tracecontext(output, 3): + output.append(4) + output.append(5) + output.append(6) + + @jump_test(5, 6, [2, 4, 6, 7]) + def test_jump_out_of_with_block_within_finally_block(output): + try: + output.append(2) + finally: + with tracecontext(output, 4): + output.append(5) + output.append(6) + output.append(7) + @jump_test(8, 11, [1, 3, 5, 11, 12]) def test_jump_out_of_complex_nested_blocks(output): output.append(1) diff --git a/Misc/NEWS.d/next/Core and Builtins/2018-03-08-09-48-38.bpo-33026.QZA3Ba.rst b/Misc/NEWS.d/next/Core and Builtins/2018-03-08-09-48-38.bpo-33026.QZA3Ba.rst new file mode 100644 index 000000000000..dc166d1e5771 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2018-03-08-09-48-38.bpo-33026.QZA3Ba.rst @@ -0,0 +1 @@ +Fixed jumping out of "with" block by setting f_lineno. diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 2c8fb017492e..bf1c7c52737d 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -340,6 +340,11 @@ frame_setlineno(PyFrameObject *f, PyObject* p_new_lineno) PyObject *v = (*--f->f_stacktop); Py_DECREF(v); } + if (b->b_type == SETUP_WITH) { + /* Pop the exit function. */ + PyObject *v = (*--f->f_stacktop); + Py_DECREF(v); + } } /* Finally set the new f_lineno and f_lasti and return OK. */ From webhook-mailer at python.org Sun Mar 11 05:07:09 2018 From: webhook-mailer at python.org (Serhiy Storchaka) Date: Sun, 11 Mar 2018 09:07:09 -0000 Subject: [Python-checkins] bpo-32970: Improve disassembly of the MAKE_FUNCTION instruction. (GH-5937) Message-ID: <mailman.45.1520759230.1871.python-checkins@python.org> https://github.com/python/cpython/commit/e2732d3e66eba9ec13f9d55c499f2437ead552db commit: e2732d3e66eba9ec13f9d55c499f2437ead552db branch: master author: Serhiy Storchaka <storchaka at gmail.com> committer: GitHub <noreply at github.com> date: 2018-03-11T11:07:06+02:00 summary: bpo-32970: Improve disassembly of the MAKE_FUNCTION instruction. (GH-5937) files: A Misc/NEWS.d/next/Library/2018-02-28-18-39-48.bpo-32970.IPWtbS.rst M Lib/dis.py M Lib/test/test_dis.py diff --git a/Lib/dis.py b/Lib/dis.py index 90ddf4f33603..b2b0003203a4 100644 --- a/Lib/dis.py +++ b/Lib/dis.py @@ -17,6 +17,15 @@ classmethod, staticmethod, type) FORMAT_VALUE = opmap['FORMAT_VALUE'] +FORMAT_VALUE_CONVERTERS = ( + (None, ''), + (str, 'str'), + (repr, 'repr'), + (ascii, 'ascii'), +) +MAKE_FUNCTION = opmap['MAKE_FUNCTION'] +MAKE_FUNCTION_FLAGS = ('defaults', 'kwdefaults', 'annotations', 'closure') + def _try_compile(source, name): """Attempts to compile the given source, first as an expression and @@ -339,12 +348,15 @@ def _get_instructions_bytes(code, varnames=None, names=None, constants=None, elif op in hasfree: argval, argrepr = _get_name_info(arg, cells) elif op == FORMAT_VALUE: - argval = ((None, str, repr, ascii)[arg & 0x3], bool(arg & 0x4)) - argrepr = ('', 'str', 'repr', 'ascii')[arg & 0x3] + argval, argrepr = FORMAT_VALUE_CONVERTERS[arg & 0x3] + argval = (argval, bool(arg & 0x4)) if argval[1]: if argrepr: argrepr += ', ' argrepr += 'with format' + elif op == MAKE_FUNCTION: + argrepr = ', '.join(s for i, s in enumerate(MAKE_FUNCTION_FLAGS) + if arg & (1<<i)) yield Instruction(opname[op], op, arg, argval, argrepr, offset, starts_line, is_jump_target) diff --git a/Lib/test/test_dis.py b/Lib/test/test_dis.py index 08393f734ba5..098367ca55d1 100644 --- a/Lib/test/test_dis.py +++ b/Lib/test/test_dis.py @@ -348,7 +348,7 @@ def foo(x): 2 BUILD_TUPLE 1 4 LOAD_CONST 1 (<code object foo at 0x..., file "%s", line %d>) 6 LOAD_CONST 2 ('_h.<locals>.foo') - 8 MAKE_FUNCTION 8 + 8 MAKE_FUNCTION 8 (closure) 10 STORE_FAST 1 (foo) %3d 12 LOAD_FAST 1 (foo) @@ -365,7 +365,7 @@ def foo(x): 2 BUILD_TUPLE 1 4 LOAD_CONST 1 (<code object <listcomp> at 0x..., file "%s", line %d>) 6 LOAD_CONST 2 ('_h.<locals>.foo.<locals>.<listcomp>') - 8 MAKE_FUNCTION 8 + 8 MAKE_FUNCTION 8 (closure) 10 LOAD_DEREF 1 (y) 12 GET_ITER 14 CALL_FUNCTION 1 @@ -862,7 +862,7 @@ def jumpy(): Instruction(opname='BUILD_TUPLE', opcode=102, arg=2, argval=2, argrepr='', offset=6, starts_line=None, is_jump_target=False), Instruction(opname='LOAD_CONST', opcode=100, arg=3, argval=code_object_f, argrepr=repr(code_object_f), offset=8, starts_line=None, is_jump_target=False), Instruction(opname='LOAD_CONST', opcode=100, arg=4, argval='outer.<locals>.f', argrepr="'outer.<locals>.f'", offset=10, starts_line=None, is_jump_target=False), - Instruction(opname='MAKE_FUNCTION', opcode=132, arg=9, argval=9, argrepr='', offset=12, starts_line=None, is_jump_target=False), + Instruction(opname='MAKE_FUNCTION', opcode=132, arg=9, argval=9, argrepr='defaults, closure', offset=12, starts_line=None, is_jump_target=False), Instruction(opname='STORE_FAST', opcode=125, arg=2, argval='f', argrepr='f', offset=14, starts_line=None, is_jump_target=False), Instruction(opname='LOAD_GLOBAL', opcode=116, arg=0, argval='print', argrepr='print', offset=16, starts_line=7, is_jump_target=False), Instruction(opname='LOAD_DEREF', opcode=136, arg=0, argval='a', argrepr='a', offset=18, starts_line=None, is_jump_target=False), @@ -887,7 +887,7 @@ def jumpy(): Instruction(opname='BUILD_TUPLE', opcode=102, arg=4, argval=4, argrepr='', offset=10, starts_line=None, is_jump_target=False), Instruction(opname='LOAD_CONST', opcode=100, arg=3, argval=code_object_inner, argrepr=repr(code_object_inner), offset=12, starts_line=None, is_jump_target=False), Instruction(opname='LOAD_CONST', opcode=100, arg=4, argval='outer.<locals>.f.<locals>.inner', argrepr="'outer.<locals>.f.<locals>.inner'", offset=14, starts_line=None, is_jump_target=False), - Instruction(opname='MAKE_FUNCTION', opcode=132, arg=9, argval=9, argrepr='', offset=16, starts_line=None, is_jump_target=False), + Instruction(opname='MAKE_FUNCTION', opcode=132, arg=9, argval=9, argrepr='defaults, closure', offset=16, starts_line=None, is_jump_target=False), Instruction(opname='STORE_FAST', opcode=125, arg=2, argval='inner', argrepr='inner', offset=18, starts_line=None, is_jump_target=False), Instruction(opname='LOAD_GLOBAL', opcode=116, arg=0, argval='print', argrepr='print', offset=20, starts_line=5, is_jump_target=False), Instruction(opname='LOAD_DEREF', opcode=136, arg=2, argval='a', argrepr='a', offset=22, starts_line=None, is_jump_target=False), diff --git a/Misc/NEWS.d/next/Library/2018-02-28-18-39-48.bpo-32970.IPWtbS.rst b/Misc/NEWS.d/next/Library/2018-02-28-18-39-48.bpo-32970.IPWtbS.rst new file mode 100644 index 000000000000..e97c16df5a0d --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-02-28-18-39-48.bpo-32970.IPWtbS.rst @@ -0,0 +1 @@ +Improved disassembly of the MAKE_FUNCTION instruction. From solipsis at pitrou.net Sun Mar 11 05:11:37 2018 From: solipsis at pitrou.net (solipsis at pitrou.net) Date: Sun, 11 Mar 2018 09:11:37 +0000 Subject: [Python-checkins] Daily reference leaks (4243df51fe43): sum=10 Message-ID: <20180311091137.1.62B4F46094D11A45@psf.io> results for 4243df51fe43 on branch "default" -------------------------------------------- test_collections leaked [7, -7, 8] memory blocks, sum=8 test_functools leaked [0, 3, 1] memory blocks, sum=4 test_multiprocessing_forkserver leaked [0, 0, -2] memory blocks, sum=-2 Command line was: ['./python', '-m', 'test.regrtest', '-uall', '-R', '3:3:/home/psf-users/antoine/refleaks/reflogCuZMFW', '--timeout', '7200'] From webhook-mailer at python.org Sun Mar 11 05:45:19 2018 From: webhook-mailer at python.org (Xiang Zhang) Date: Sun, 11 Mar 2018 09:45:19 -0000 Subject: [Python-checkins] Drop confusing commented out code in pystrtod.c (GH-6072) Message-ID: <mailman.46.1520761520.1871.python-checkins@python.org> https://github.com/python/cpython/commit/9fb84157595a385f15799e5d0729c1e1b0ba9d38 commit: 9fb84157595a385f15799e5d0729c1e1b0ba9d38 branch: master author: Siddhesh Poyarekar <siddhesh.poyarekar at gmail.com> committer: Xiang Zhang <angwerzx at 126.com> date: 2018-03-11T17:45:10+08:00 summary: Drop confusing commented out code in pystrtod.c (GH-6072) files: M Python/pystrtod.c diff --git a/Python/pystrtod.c b/Python/pystrtod.c index 601f7c691edf..3546d44c8424 100644 --- a/Python/pystrtod.c +++ b/Python/pystrtod.c @@ -1060,8 +1060,6 @@ format_float_short(double d, char format_code, else { /* shouldn't get here: Gay's code should always return something starting with a digit, an 'I', or 'N' */ - /* strncpy(p, "ERR", 3); - p += 3; */ Py_UNREACHABLE(); } goto exit; From webhook-mailer at python.org Sun Mar 11 14:21:41 2018 From: webhook-mailer at python.org (Antoine Pitrou) Date: Sun, 11 Mar 2018 18:21:41 -0000 Subject: [Python-checkins] bpo-31804: Fix multiprocessing.Process with broken standard streams (#6079) Message-ID: <mailman.47.1520792503.1871.python-checkins@python.org> https://github.com/python/cpython/commit/e756f66c83786ee82f5f7d45931ae50a6931dd7f commit: e756f66c83786ee82f5f7d45931ae50a6931dd7f branch: master author: Antoine Pitrou <pitrou at free.fr> committer: GitHub <noreply at github.com> date: 2018-03-11T19:21:38+01:00 summary: bpo-31804: Fix multiprocessing.Process with broken standard streams (#6079) In some conditions the standard streams will be None or closed in the child process (for example if using "pythonw" instead of "python" on Windows). Avoid failing with a non-0 exit code in those conditions. Report and initial patch by poxthegreat. files: A Misc/NEWS.d/next/Library/2018-03-11-19-03-52.bpo-31804.i8KUMp.rst M Lib/multiprocessing/popen_fork.py M Lib/multiprocessing/process.py M Lib/multiprocessing/util.py M Lib/test/_test_multiprocessing.py diff --git a/Lib/multiprocessing/popen_fork.py b/Lib/multiprocessing/popen_fork.py index b0fc01396eb2..008b97b36628 100644 --- a/Lib/multiprocessing/popen_fork.py +++ b/Lib/multiprocessing/popen_fork.py @@ -14,14 +14,7 @@ class Popen(object): method = 'fork' def __init__(self, process_obj): - try: - sys.stdout.flush() - except (AttributeError, ValueError): - pass - try: - sys.stderr.flush() - except (AttributeError, ValueError): - pass + util._flush_std_streams() self.returncode = None self.finalizer = None self._launch(process_obj) diff --git a/Lib/multiprocessing/process.py b/Lib/multiprocessing/process.py index 8fff3e105ead..cd592d0bdf09 100644 --- a/Lib/multiprocessing/process.py +++ b/Lib/multiprocessing/process.py @@ -314,8 +314,7 @@ def _bootstrap(self): finally: threading._shutdown() util.info('process exiting with exitcode %d' % exitcode) - sys.stdout.flush() - sys.stderr.flush() + util._flush_std_streams() return exitcode diff --git a/Lib/multiprocessing/util.py b/Lib/multiprocessing/util.py index f0827f0a9693..0c4eb2473273 100644 --- a/Lib/multiprocessing/util.py +++ b/Lib/multiprocessing/util.py @@ -391,6 +391,20 @@ def _close_stdin(): except (OSError, ValueError): pass +# +# Flush standard streams, if any +# + +def _flush_std_streams(): + try: + sys.stdout.flush() + except (AttributeError, ValueError): + pass + try: + sys.stderr.flush() + except (AttributeError, ValueError): + pass + # # Start a program with only specified fds kept open # diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index 1e497a572a76..940fe584e752 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -584,10 +584,19 @@ def test_wait_for_threads(self): self.assertTrue(evt.is_set()) @classmethod - def _test_error_on_stdio_flush(self, evt): + def _test_error_on_stdio_flush(self, evt, break_std_streams={}): + for stream_name, action in break_std_streams.items(): + if action == 'close': + stream = io.StringIO() + stream.close() + else: + assert action == 'remove' + stream = None + setattr(sys, stream_name, None) evt.set() - def test_error_on_stdio_flush(self): + def test_error_on_stdio_flush_1(self): + # Check that Process works with broken standard streams streams = [io.StringIO(), None] streams[0].close() for stream_name in ('stdout', 'stderr'): @@ -601,6 +610,24 @@ def test_error_on_stdio_flush(self): proc.start() proc.join() self.assertTrue(evt.is_set()) + self.assertEqual(proc.exitcode, 0) + finally: + setattr(sys, stream_name, old_stream) + + def test_error_on_stdio_flush_2(self): + # Same as test_error_on_stdio_flush_1(), but standard streams are + # broken by the child process + for stream_name in ('stdout', 'stderr'): + for action in ('close', 'remove'): + old_stream = getattr(sys, stream_name) + try: + evt = self.Event() + proc = self.Process(target=self._test_error_on_stdio_flush, + args=(evt, {stream_name: action})) + proc.start() + proc.join() + self.assertTrue(evt.is_set()) + self.assertEqual(proc.exitcode, 0) finally: setattr(sys, stream_name, old_stream) diff --git a/Misc/NEWS.d/next/Library/2018-03-11-19-03-52.bpo-31804.i8KUMp.rst b/Misc/NEWS.d/next/Library/2018-03-11-19-03-52.bpo-31804.i8KUMp.rst new file mode 100644 index 000000000000..7fcede297aca --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-03-11-19-03-52.bpo-31804.i8KUMp.rst @@ -0,0 +1,2 @@ +Avoid failing in multiprocessing.Process if the standard streams are closed +or None at exit. From webhook-mailer at python.org Sun Mar 11 14:29:08 2018 From: webhook-mailer at python.org (larryhastings) Date: Sun, 11 Mar 2018 18:29:08 -0000 Subject: [Python-checkins] [3.5] bpo-32981: Fix catastrophic backtracking vulns (GH-5955) (#6034) Message-ID: <mailman.48.1520792951.1871.python-checkins@python.org> https://github.com/python/cpython/commit/937ac1fe069a4dc8471dff205f553d82e724015b commit: 937ac1fe069a4dc8471dff205f553d82e724015b branch: 3.5 author: Ned Deily <nad at python.org> committer: larryhastings <larry at hastings.org> date: 2018-03-11T18:29:05Z summary: [3.5] bpo-32981: Fix catastrophic backtracking vulns (GH-5955) (#6034) * Prevent low-grade poplib REDOS (CVE-2018-1060) The regex to test a mail server's timestamp is susceptible to catastrophic backtracking on long evil responses from the server. Happily, the maximum length of malicious inputs is 2K thanks to a limit introduced in the fix for CVE-2013-1752. A 2KB evil response from the mail server would result in small slowdowns (milliseconds vs. microseconds) accumulated over many apop calls. This is a potential DOS vector via accumulated slowdowns. Replace it with a similar non-vulnerable regex. The new regex is RFC compliant. The old regex was non-compliant in edge cases. * Prevent difflib REDOS (CVE-2018-1061) The default regex for IS_LINE_JUNK is susceptible to catastrophic backtracking. This is a potential DOS vector. Replace it with an equivalent non-vulnerable regex. Also introduce unit and REDOS tests for difflib. Co-authored-by: Tim Peters <tim.peters at gmail.com> Co-authored-by: Christian Heimes <christian at python.org>. (cherry picked from commit 0e6c8ee2358a2e23117501826c008842acb835ac) files: A Misc/NEWS.d/next/Security/2018-03-02-10-24-52.bpo-32981.O_qDyj.rst M Lib/difflib.py M Lib/poplib.py M Lib/test/test_difflib.py M Lib/test/test_poplib.py M Misc/ACKS diff --git a/Lib/difflib.py b/Lib/difflib.py index 076bbac01dee..b4ec33505644 100644 --- a/Lib/difflib.py +++ b/Lib/difflib.py @@ -1083,7 +1083,7 @@ def _qformat(self, aline, bline, atags, btags): import re -def IS_LINE_JUNK(line, pat=re.compile(r"\s*#?\s*$").match): +def IS_LINE_JUNK(line, pat=re.compile(r"\s*(?:#\s*)?$").match): r""" Return 1 for ignorable line: iff `line` is blank or contains a single '#'. diff --git a/Lib/poplib.py b/Lib/poplib.py index 516b6f060d28..2437ea0e2717 100644 --- a/Lib/poplib.py +++ b/Lib/poplib.py @@ -308,7 +308,7 @@ def rpop(self, user): return self._shortcmd('RPOP %s' % user) - timestamp = re.compile(br'\+OK.*(<[^>]+>)') + timestamp = re.compile(br'\+OK.[^<]*(<.*>)') def apop(self, user, password): """Authorisation diff --git a/Lib/test/test_difflib.py b/Lib/test/test_difflib.py index ab9debf8e252..b6c8a7dd5bed 100644 --- a/Lib/test/test_difflib.py +++ b/Lib/test/test_difflib.py @@ -466,13 +466,33 @@ def _assert_type_error(self, msg, generator, *args): list(generator(*args)) self.assertEqual(msg, str(ctx.exception)) +class TestJunkAPIs(unittest.TestCase): + def test_is_line_junk_true(self): + for line in ['#', ' ', ' #', '# ', ' # ', '']: + self.assertTrue(difflib.IS_LINE_JUNK(line), repr(line)) + + def test_is_line_junk_false(self): + for line in ['##', ' ##', '## ', 'abc ', 'abc #', 'Mr. Moose is up!']: + self.assertFalse(difflib.IS_LINE_JUNK(line), repr(line)) + + def test_is_line_junk_REDOS(self): + evil_input = ('\t' * 1000000) + '##' + self.assertFalse(difflib.IS_LINE_JUNK(evil_input)) + + def test_is_character_junk_true(self): + for char in [' ', '\t']: + self.assertTrue(difflib.IS_CHARACTER_JUNK(char), repr(char)) + + def test_is_character_junk_false(self): + for char in ['a', '#', '\n', '\f', '\r', '\v']: + self.assertFalse(difflib.IS_CHARACTER_JUNK(char), repr(char)) def test_main(): difflib.HtmlDiff._default_prefix = 0 Doctests = doctest.DocTestSuite(difflib) run_unittest( TestWithAscii, TestAutojunk, TestSFpatches, TestSFbugs, - TestOutputFormat, TestBytes, Doctests) + TestOutputFormat, TestBytes, TestJunkAPIs, Doctests) if __name__ == '__main__': test_main() diff --git a/Lib/test/test_poplib.py b/Lib/test/test_poplib.py index bceeb93ad14a..799e40365214 100644 --- a/Lib/test/test_poplib.py +++ b/Lib/test/test_poplib.py @@ -300,9 +300,19 @@ def test_noop(self): def test_rpop(self): self.assertOK(self.client.rpop('foo')) - def test_apop(self): + def test_apop_normal(self): self.assertOK(self.client.apop('foo', 'dummypassword')) + def test_apop_REDOS(self): + # Replace welcome with very long evil welcome. + # NB The upper bound on welcome length is currently 2048. + # At this length, evil input makes each apop call take + # on the order of milliseconds instead of microseconds. + evil_welcome = b'+OK' + (b'<' * 1000000) + with test_support.swap_attr(self.client, 'welcome', evil_welcome): + # The evil welcome is invalid, so apop should throw. + self.assertRaises(poplib.error_proto, self.client.apop, 'a', 'kb') + def test_top(self): expected = (b'+OK 116 bytes', [b'From: postmaster at python.org', b'Content-Type: text/plain', diff --git a/Misc/ACKS b/Misc/ACKS index 1a35aad66ce7..72c5d740bdd2 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -341,6 +341,7 @@ Kushal Das Jonathan Dasteel Pierre-Yves David A. Jesse Jiryu Davis +Jamie (James C.) Davis Merlijn van Deen John DeGood Ned Deily diff --git a/Misc/NEWS.d/next/Security/2018-03-02-10-24-52.bpo-32981.O_qDyj.rst b/Misc/NEWS.d/next/Security/2018-03-02-10-24-52.bpo-32981.O_qDyj.rst new file mode 100644 index 000000000000..9ebabb44f91e --- /dev/null +++ b/Misc/NEWS.d/next/Security/2018-03-02-10-24-52.bpo-32981.O_qDyj.rst @@ -0,0 +1,4 @@ +Regexes in difflib and poplib were vulnerable to catastrophic backtracking. +These regexes formed potential DOS vectors (REDOS). They have been +refactored. This resolves CVE-2018-1060 and CVE-2018-1061. +Patch by Jamie Davis. From webhook-mailer at python.org Sun Mar 11 14:42:40 2018 From: webhook-mailer at python.org (Antoine Pitrou) Date: Sun, 11 Mar 2018 18:42:40 -0000 Subject: [Python-checkins] bpo-31804: Fix multiprocessing.Process with broken standard streams (GH-6079) (GH-6080) Message-ID: <mailman.49.1520793762.1871.python-checkins@python.org> https://github.com/python/cpython/commit/ff5d21331ec6cefec6ba5b78d256d8dbcd67a069 commit: ff5d21331ec6cefec6ba5b78d256d8dbcd67a069 branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Antoine Pitrou <pitrou at free.fr> date: 2018-03-11T19:42:37+01:00 summary: bpo-31804: Fix multiprocessing.Process with broken standard streams (GH-6079) (GH-6080) In some conditions the standard streams will be None or closed in the child process (for example if using "pythonw" instead of "python" on Windows). Avoid failing with a non-0 exit code in those conditions. Report and initial patch by poxthegreat. (cherry picked from commit e756f66c83786ee82f5f7d45931ae50a6931dd7f) Co-authored-by: Antoine Pitrou <pitrou at free.fr> files: A Misc/NEWS.d/next/Library/2018-03-11-19-03-52.bpo-31804.i8KUMp.rst M Lib/multiprocessing/popen_fork.py M Lib/multiprocessing/process.py M Lib/multiprocessing/util.py M Lib/test/_test_multiprocessing.py diff --git a/Lib/multiprocessing/popen_fork.py b/Lib/multiprocessing/popen_fork.py index b0fc01396eb2..008b97b36628 100644 --- a/Lib/multiprocessing/popen_fork.py +++ b/Lib/multiprocessing/popen_fork.py @@ -14,14 +14,7 @@ class Popen(object): method = 'fork' def __init__(self, process_obj): - try: - sys.stdout.flush() - except (AttributeError, ValueError): - pass - try: - sys.stderr.flush() - except (AttributeError, ValueError): - pass + util._flush_std_streams() self.returncode = None self.finalizer = None self._launch(process_obj) diff --git a/Lib/multiprocessing/process.py b/Lib/multiprocessing/process.py index 8fff3e105ead..cd592d0bdf09 100644 --- a/Lib/multiprocessing/process.py +++ b/Lib/multiprocessing/process.py @@ -314,8 +314,7 @@ def _bootstrap(self): finally: threading._shutdown() util.info('process exiting with exitcode %d' % exitcode) - sys.stdout.flush() - sys.stderr.flush() + util._flush_std_streams() return exitcode diff --git a/Lib/multiprocessing/util.py b/Lib/multiprocessing/util.py index f0827f0a9693..0c4eb2473273 100644 --- a/Lib/multiprocessing/util.py +++ b/Lib/multiprocessing/util.py @@ -391,6 +391,20 @@ def _close_stdin(): except (OSError, ValueError): pass +# +# Flush standard streams, if any +# + +def _flush_std_streams(): + try: + sys.stdout.flush() + except (AttributeError, ValueError): + pass + try: + sys.stderr.flush() + except (AttributeError, ValueError): + pass + # # Start a program with only specified fds kept open # diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index 1e497a572a76..940fe584e752 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -584,10 +584,19 @@ def test_wait_for_threads(self): self.assertTrue(evt.is_set()) @classmethod - def _test_error_on_stdio_flush(self, evt): + def _test_error_on_stdio_flush(self, evt, break_std_streams={}): + for stream_name, action in break_std_streams.items(): + if action == 'close': + stream = io.StringIO() + stream.close() + else: + assert action == 'remove' + stream = None + setattr(sys, stream_name, None) evt.set() - def test_error_on_stdio_flush(self): + def test_error_on_stdio_flush_1(self): + # Check that Process works with broken standard streams streams = [io.StringIO(), None] streams[0].close() for stream_name in ('stdout', 'stderr'): @@ -601,6 +610,24 @@ def test_error_on_stdio_flush(self): proc.start() proc.join() self.assertTrue(evt.is_set()) + self.assertEqual(proc.exitcode, 0) + finally: + setattr(sys, stream_name, old_stream) + + def test_error_on_stdio_flush_2(self): + # Same as test_error_on_stdio_flush_1(), but standard streams are + # broken by the child process + for stream_name in ('stdout', 'stderr'): + for action in ('close', 'remove'): + old_stream = getattr(sys, stream_name) + try: + evt = self.Event() + proc = self.Process(target=self._test_error_on_stdio_flush, + args=(evt, {stream_name: action})) + proc.start() + proc.join() + self.assertTrue(evt.is_set()) + self.assertEqual(proc.exitcode, 0) finally: setattr(sys, stream_name, old_stream) diff --git a/Misc/NEWS.d/next/Library/2018-03-11-19-03-52.bpo-31804.i8KUMp.rst b/Misc/NEWS.d/next/Library/2018-03-11-19-03-52.bpo-31804.i8KUMp.rst new file mode 100644 index 000000000000..7fcede297aca --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-03-11-19-03-52.bpo-31804.i8KUMp.rst @@ -0,0 +1,2 @@ +Avoid failing in multiprocessing.Process if the standard streams are closed +or None at exit. From webhook-mailer at python.org Sun Mar 11 15:09:23 2018 From: webhook-mailer at python.org (Antoine Pitrou) Date: Sun, 11 Mar 2018 19:09:23 -0000 Subject: [Python-checkins] [3.6] bpo-31804: Fix multiprocessing.Process with broken standard streams (GH-6079) (GH-6081) Message-ID: <mailman.50.1520795364.1871.python-checkins@python.org> https://github.com/python/cpython/commit/069b8d20be8018fbd49ed5aaf64c4caba311e48f commit: 069b8d20be8018fbd49ed5aaf64c4caba311e48f branch: 3.6 author: Antoine Pitrou <pitrou at free.fr> committer: GitHub <noreply at github.com> date: 2018-03-11T20:09:20+01:00 summary: [3.6] bpo-31804: Fix multiprocessing.Process with broken standard streams (GH-6079) (GH-6081) In some conditions the standard streams will be None or closed in the child process (for example if using "pythonw" instead of "python" on Windows). Avoid failing with a non-0 exit code in those conditions. Report and initial patch by poxthegreat.. (cherry picked from commit e756f66c83786ee82f5f7d45931ae50a6931dd7f) files: A Misc/NEWS.d/next/Library/2018-03-11-19-03-52.bpo-31804.i8KUMp.rst M Lib/multiprocessing/popen_fork.py M Lib/multiprocessing/process.py M Lib/multiprocessing/util.py M Lib/test/_test_multiprocessing.py diff --git a/Lib/multiprocessing/popen_fork.py b/Lib/multiprocessing/popen_fork.py index 5d0fa569f12e..6396db45b136 100644 --- a/Lib/multiprocessing/popen_fork.py +++ b/Lib/multiprocessing/popen_fork.py @@ -14,14 +14,7 @@ class Popen(object): method = 'fork' def __init__(self, process_obj): - try: - sys.stdout.flush() - except (AttributeError, ValueError): - pass - try: - sys.stderr.flush() - except (AttributeError, ValueError): - pass + util._flush_std_streams() self.returncode = None self._launch(process_obj) diff --git a/Lib/multiprocessing/process.py b/Lib/multiprocessing/process.py index 1d26b5e521e7..c6157b6046a0 100644 --- a/Lib/multiprocessing/process.py +++ b/Lib/multiprocessing/process.py @@ -274,8 +274,7 @@ def _bootstrap(self): traceback.print_exc() finally: util.info('process exiting with exitcode %d' % exitcode) - sys.stdout.flush() - sys.stderr.flush() + util._flush_std_streams() return exitcode diff --git a/Lib/multiprocessing/util.py b/Lib/multiprocessing/util.py index b490caa7e643..24573764fab1 100644 --- a/Lib/multiprocessing/util.py +++ b/Lib/multiprocessing/util.py @@ -388,6 +388,20 @@ def _close_stdin(): except (OSError, ValueError): pass +# +# Flush standard streams, if any +# + +def _flush_std_streams(): + try: + sys.stdout.flush() + except (AttributeError, ValueError): + pass + try: + sys.stderr.flush() + except (AttributeError, ValueError): + pass + # # Start a program with only specified fds kept open # diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index e4d60f880426..dd0a9d7a862a 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -427,10 +427,19 @@ def test_lose_target_ref(self): close_queue(q) @classmethod - def _test_error_on_stdio_flush(self, evt): + def _test_error_on_stdio_flush(self, evt, break_std_streams={}): + for stream_name, action in break_std_streams.items(): + if action == 'close': + stream = io.StringIO() + stream.close() + else: + assert action == 'remove' + stream = None + setattr(sys, stream_name, None) evt.set() - def test_error_on_stdio_flush(self): + def test_error_on_stdio_flush_1(self): + # Check that Process works with broken standard streams streams = [io.StringIO(), None] streams[0].close() for stream_name in ('stdout', 'stderr'): @@ -444,6 +453,24 @@ def test_error_on_stdio_flush(self): proc.start() proc.join() self.assertTrue(evt.is_set()) + self.assertEqual(proc.exitcode, 0) + finally: + setattr(sys, stream_name, old_stream) + + def test_error_on_stdio_flush_2(self): + # Same as test_error_on_stdio_flush_1(), but standard streams are + # broken by the child process + for stream_name in ('stdout', 'stderr'): + for action in ('close', 'remove'): + old_stream = getattr(sys, stream_name) + try: + evt = self.Event() + proc = self.Process(target=self._test_error_on_stdio_flush, + args=(evt, {stream_name: action})) + proc.start() + proc.join() + self.assertTrue(evt.is_set()) + self.assertEqual(proc.exitcode, 0) finally: setattr(sys, stream_name, old_stream) diff --git a/Misc/NEWS.d/next/Library/2018-03-11-19-03-52.bpo-31804.i8KUMp.rst b/Misc/NEWS.d/next/Library/2018-03-11-19-03-52.bpo-31804.i8KUMp.rst new file mode 100644 index 000000000000..7fcede297aca --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-03-11-19-03-52.bpo-31804.i8KUMp.rst @@ -0,0 +1,2 @@ +Avoid failing in multiprocessing.Process if the standard streams are closed +or None at exit. From webhook-mailer at python.org Sun Mar 11 19:39:25 2018 From: webhook-mailer at python.org (Antoine Pitrou) Date: Sun, 11 Mar 2018 23:39:25 -0000 Subject: [Python-checkins] bpo-33021: Release the GIL during fstat() calls (GH-6019) Message-ID: <mailman.51.1520811567.1871.python-checkins@python.org> https://github.com/python/cpython/commit/4484f9dca9149da135bbae035f10a50d20d1cbbb commit: 4484f9dca9149da135bbae035f10a50d20d1cbbb branch: master author: Nir Soffer <nirsof at gmail.com> committer: Antoine Pitrou <pitrou at free.fr> date: 2018-03-12T00:39:22+01:00 summary: bpo-33021: Release the GIL during fstat() calls (GH-6019) fstat may block for long time if the file descriptor is on a non-responsive NFS server, hanging all threads. Most fstat() calls are handled by _Py_fstat(), releasing the GIL internally, but but _Py_fstat_noraise() does not release the GIL, and most calls release the GIL explicitly around it. This patch fixes last 2 calls to _Py_fstat_no_raise(), avoiding hangs when calling: - mmap.mmap() - os.urandom() - random.seed() files: A Misc/NEWS.d/next/Library/2018-03-12-00-27-56.bpo-33021.m19B9T.rst M Modules/mmapmodule.c M Python/bootstrap_hash.c diff --git a/Misc/NEWS.d/next/Library/2018-03-12-00-27-56.bpo-33021.m19B9T.rst b/Misc/NEWS.d/next/Library/2018-03-12-00-27-56.bpo-33021.m19B9T.rst new file mode 100644 index 000000000000..50dfafe600d8 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-03-12-00-27-56.bpo-33021.m19B9T.rst @@ -0,0 +1,2 @@ +Release the GIL during fstat() calls, avoiding hang of all threads when +calling mmap.mmap(), os.urandom(), and random.seed(). Patch by Nir Soffer. diff --git a/Modules/mmapmodule.c b/Modules/mmapmodule.c index 6cf454573e93..6abdc71bbb82 100644 --- a/Modules/mmapmodule.c +++ b/Modules/mmapmodule.c @@ -1050,6 +1050,7 @@ static PyObject * new_mmap_object(PyTypeObject *type, PyObject *args, PyObject *kwdict) { struct _Py_stat_struct status; + int fstat_result; mmap_object *m_obj; Py_ssize_t map_size; off_t offset = 0; @@ -1115,8 +1116,14 @@ new_mmap_object(PyTypeObject *type, PyObject *args, PyObject *kwdict) if (fd != -1) (void)fcntl(fd, F_FULLFSYNC); #endif - if (fd != -1 && _Py_fstat_noraise(fd, &status) == 0 - && S_ISREG(status.st_mode)) { + + if (fd != -1) { + Py_BEGIN_ALLOW_THREADS + fstat_result = _Py_fstat_noraise(fd, &status); + Py_END_ALLOW_THREADS + } + + if (fd != -1 && fstat_result == 0 && S_ISREG(status.st_mode)) { if (map_size == 0) { if (status.st_size == 0) { PyErr_SetString(PyExc_ValueError, diff --git a/Python/bootstrap_hash.c b/Python/bootstrap_hash.c index 9fd5cfb200ea..073b2ea8dbda 100644 --- a/Python/bootstrap_hash.c +++ b/Python/bootstrap_hash.c @@ -301,10 +301,15 @@ dev_urandom(char *buffer, Py_ssize_t size, int raise) if (raise) { struct _Py_stat_struct st; + int fstat_result; if (urandom_cache.fd >= 0) { + Py_BEGIN_ALLOW_THREADS + fstat_result = _Py_fstat_noraise(urandom_cache.fd, &st); + Py_END_ALLOW_THREADS + /* Does the fd point to the same thing as before? (issue #21207) */ - if (_Py_fstat_noraise(urandom_cache.fd, &st) + if (fstat_result || st.st_dev != urandom_cache.st_dev || st.st_ino != urandom_cache.st_ino) { /* Something changed: forget the cached fd (but don't close it, From solipsis at pitrou.net Mon Mar 12 05:10:43 2018 From: solipsis at pitrou.net (solipsis at pitrou.net) Date: Mon, 12 Mar 2018 09:10:43 +0000 Subject: [Python-checkins] Daily reference leaks (4243df51fe43): sum=1 Message-ID: <20180312091043.1.276073873144E509@psf.io> results for 4243df51fe43 on branch "default" -------------------------------------------- test_collections leaked [-7, 1, 0] memory blocks, sum=-6 test_functools leaked [0, 3, 1] memory blocks, sum=4 test_multiprocessing_fork leaked [2, 0, 0] memory blocks, sum=2 test_multiprocessing_spawn leaked [2, 0, -1] memory blocks, sum=1 Command line was: ['./python', '-m', 'test.regrtest', '-uall', '-R', '3:3:/home/psf-users/antoine/refleaks/reflogMVru7n', '--timeout', '7200'] From webhook-mailer at python.org Mon Mar 12 09:42:44 2018 From: webhook-mailer at python.org (Antoine Pitrou) Date: Mon, 12 Mar 2018 13:42:44 -0000 Subject: [Python-checkins] signal: add strsignal() (#6017) Message-ID: <mailman.52.1520862165.1871.python-checkins@python.org> https://github.com/python/cpython/commit/5d2a27de625caf2fd5b763d8a8463790ccd954be commit: 5d2a27de625caf2fd5b763d8a8463790ccd954be branch: master author: Antoine Pietri <antoine.pietri1 at gmail.com> committer: Antoine Pitrou <pitrou at free.fr> date: 2018-03-12T14:42:34+01:00 summary: signal: add strsignal() (#6017) Co-authored-by: Vajrasky Kok <sky.kok at speaklikeaking.com> files: A Misc/NEWS.d/next/Library/2018-03-07-19-37-00.bpo-22674.2sIMmM.rst M Doc/library/signal.rst M Lib/test/test_signal.py M Modules/clinic/signalmodule.c.h M Modules/signalmodule.c diff --git a/Doc/library/signal.rst b/Doc/library/signal.rst index 67eaa2c63813..3f17e08f0f95 100644 --- a/Doc/library/signal.rst +++ b/Doc/library/signal.rst @@ -207,6 +207,15 @@ The :mod:`signal` module defines the following functions: installed from Python. +.. function:: strsignal(signalnum) + + Return the system description of the signal *signalnum*, such as + "Interrupt", "Segmentation fault", etc. Returns :const:`None` if the signal + is not recognized. + + .. versionadded:: 3.8 + + .. function:: pause() Cause the process to sleep until a signal is received; the appropriate handler diff --git a/Lib/test/test_signal.py b/Lib/test/test_signal.py index 48b7a392e50b..fbb12a5b67bf 100644 --- a/Lib/test/test_signal.py +++ b/Lib/test/test_signal.py @@ -43,6 +43,8 @@ def test_out_of_range_signal_number_raises_error(self): self.assertRaises(ValueError, signal.signal, 4242, self.trivial_signal_handler) + self.assertRaises(ValueError, signal.strsignal, 4242) + def test_setting_signal_handler_to_none_raises_error(self): self.assertRaises(TypeError, signal.signal, signal.SIGUSR1, None) @@ -55,6 +57,10 @@ def test_getsignal(self): signal.signal(signal.SIGHUP, hup) self.assertEqual(signal.getsignal(signal.SIGHUP), hup) + def test_strsignal(self): + self.assertEqual(signal.strsignal(signal.SIGINT), "Interrupt") + self.assertEqual(signal.strsignal(signal.SIGTERM), "Terminated") + # Issue 3864, unknown if this affects earlier versions of freebsd also def test_interprocess_signal(self): dirname = os.path.dirname(__file__) diff --git a/Misc/NEWS.d/next/Library/2018-03-07-19-37-00.bpo-22674.2sIMmM.rst b/Misc/NEWS.d/next/Library/2018-03-07-19-37-00.bpo-22674.2sIMmM.rst new file mode 100644 index 000000000000..a9af5da46ef1 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-03-07-19-37-00.bpo-22674.2sIMmM.rst @@ -0,0 +1,2 @@ +Add the strsignal() function in the signal module that returns the system +description of the given signal, as returned by strsignal(3). diff --git a/Modules/clinic/signalmodule.c.h b/Modules/clinic/signalmodule.c.h index dc3aadf878b2..1c439716c498 100644 --- a/Modules/clinic/signalmodule.c.h +++ b/Modules/clinic/signalmodule.c.h @@ -129,6 +129,36 @@ signal_getsignal(PyObject *module, PyObject *arg) return return_value; } +PyDoc_STRVAR(signal_strsignal__doc__, +"strsignal($module, signalnum, /)\n" +"--\n" +"\n" +"Return the system description of the given signal.\n" +"\n" +"The return values can be such as \"Interrupt\", \"Segmentation fault\", etc.\n" +"Returns None if the signal is not recognized."); + +#define SIGNAL_STRSIGNAL_METHODDEF \ + {"strsignal", (PyCFunction)signal_strsignal, METH_O, signal_strsignal__doc__}, + +static PyObject * +signal_strsignal_impl(PyObject *module, int signalnum); + +static PyObject * +signal_strsignal(PyObject *module, PyObject *arg) +{ + PyObject *return_value = NULL; + int signalnum; + + if (!PyArg_Parse(arg, "i:strsignal", &signalnum)) { + goto exit; + } + return_value = signal_strsignal_impl(module, signalnum); + +exit: + return return_value; +} + #if defined(HAVE_SIGINTERRUPT) PyDoc_STRVAR(signal_siginterrupt__doc__, @@ -440,4 +470,4 @@ signal_pthread_kill(PyObject *module, PyObject *const *args, Py_ssize_t nargs) #ifndef SIGNAL_PTHREAD_KILL_METHODDEF #define SIGNAL_PTHREAD_KILL_METHODDEF #endif /* !defined(SIGNAL_PTHREAD_KILL_METHODDEF) */ -/*[clinic end generated code: output=36132f4189381fe0 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=7b41486acf93aa8e input=a9049054013a1b77]*/ diff --git a/Modules/signalmodule.c b/Modules/signalmodule.c index b553eedc0f2c..791616014384 100644 --- a/Modules/signalmodule.c +++ b/Modules/signalmodule.c @@ -504,6 +504,66 @@ signal_getsignal_impl(PyObject *module, int signalnum) } } + +/*[clinic input] +signal.strsignal + + signalnum: int + / + +Return the system description of the given signal. + +The return values can be such as "Interrupt", "Segmentation fault", etc. +Returns None if the signal is not recognized. +[clinic start generated code]*/ + +static PyObject * +signal_strsignal_impl(PyObject *module, int signalnum) +/*[clinic end generated code: output=44e12e1e3b666261 input=b77914b03f856c74]*/ +{ + char *res; + + if (signalnum < 1 || signalnum >= NSIG) { + PyErr_SetString(PyExc_ValueError, + "signal number out of range"); + return NULL; + } + +#ifdef MS_WINDOWS + /* Custom redefinition of POSIX signals allowed on Windows */ + switch (signalnum) { + case SIGINT: + res = "Interrupt"; + break; + case SIGILL: + res = "Illegal instruction"; + break; + case SIGABRT: + res = "Aborted"; + break; + case SIGFPE: + res = "Floating point exception"; + break; + case SIGSEGV: + res = "Segmentation fault"; + break; + case SIGTERM: + res = "Terminated"; + break; + default: + Py_RETURN_NONE; + } +#else + errno = 0; + res = strsignal(signalnum); + + if (errno || res == NULL || strstr(res, "Unknown signal") != NULL) + Py_RETURN_NONE; +#endif + + return Py_BuildValue("s", res); +} + #ifdef HAVE_SIGINTERRUPT /*[clinic input] @@ -1152,6 +1212,7 @@ static PyMethodDef signal_methods[] = { SIGNAL_SETITIMER_METHODDEF SIGNAL_GETITIMER_METHODDEF SIGNAL_SIGNAL_METHODDEF + SIGNAL_STRSIGNAL_METHODDEF SIGNAL_GETSIGNAL_METHODDEF {"set_wakeup_fd", (PyCFunction)signal_set_wakeup_fd, METH_VARARGS | METH_KEYWORDS, set_wakeup_fd_doc}, SIGNAL_SIGINTERRUPT_METHODDEF From webhook-mailer at python.org Mon Mar 12 13:18:46 2018 From: webhook-mailer at python.org (Andrew Svetlov) Date: Mon, 12 Mar 2018 17:18:46 -0000 Subject: [Python-checkins] bpo-33056 FIX leaking fd in concurrent.futures.ProcessPoolExecutor (#6084) Message-ID: <mailman.53.1520875128.1871.python-checkins@python.org> https://github.com/python/cpython/commit/095ee415cee41bf24c3a1108c23307e5baf168dd commit: 095ee415cee41bf24c3a1108c23307e5baf168dd branch: master author: Thomas Moreau <thomas.moreau.2010 at gmail.com> committer: Andrew Svetlov <andrew.svetlov at gmail.com> date: 2018-03-12T19:18:41+02:00 summary: bpo-33056 FIX leaking fd in concurrent.futures.ProcessPoolExecutor (#6084) files: A Misc/NEWS.d/next/Library/2018-03-12-16-40-00.bpo-33056.lNN9Eh.rst M Lib/concurrent/futures/process.py diff --git a/Lib/concurrent/futures/process.py b/Lib/concurrent/futures/process.py index aaa5151e017c..63f22cfca325 100644 --- a/Lib/concurrent/futures/process.py +++ b/Lib/concurrent/futures/process.py @@ -78,11 +78,13 @@ class _ThreadWakeup: - __slot__ = ["_state"] - def __init__(self): self._reader, self._writer = mp.Pipe(duplex=False) + def close(self): + self._writer.close() + self._reader.close() + def wakeup(self): self._writer.send_bytes(b"") @@ -654,6 +656,11 @@ def shutdown(self, wait=True): self._call_queue = None self._result_queue = None self._processes = None + + if self._queue_management_thread_wakeup: + self._queue_management_thread_wakeup.close() + self._queue_management_thread_wakeup = None + shutdown.__doc__ = _base.Executor.shutdown.__doc__ atexit.register(_python_exit) diff --git a/Misc/NEWS.d/next/Library/2018-03-12-16-40-00.bpo-33056.lNN9Eh.rst b/Misc/NEWS.d/next/Library/2018-03-12-16-40-00.bpo-33056.lNN9Eh.rst new file mode 100644 index 000000000000..6acc19a36dc8 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-03-12-16-40-00.bpo-33056.lNN9Eh.rst @@ -0,0 +1 @@ +FIX properly close leaking fds in concurrent.futures.ProcessPoolExecutor. From webhook-mailer at python.org Mon Mar 12 14:50:53 2018 From: webhook-mailer at python.org (Andrew Svetlov) Date: Mon, 12 Mar 2018 18:50:53 -0000 Subject: [Python-checkins] Fix docs markup for asyncio current_task() and all_tasks() (#6089) Message-ID: <mailman.54.1520880655.1871.python-checkins@python.org> https://github.com/python/cpython/commit/b21505e7109bafe75d87ab63d524b4e749157dbd commit: b21505e7109bafe75d87ab63d524b4e749157dbd branch: master author: Andrew Svetlov <andrew.svetlov at gmail.com> committer: GitHub <noreply at github.com> date: 2018-03-12T20:50:50+02:00 summary: Fix docs markup for asyncio current_task() and all_tasks() (#6089) files: M Doc/library/asyncio-task.rst diff --git a/Doc/library/asyncio-task.rst b/Doc/library/asyncio-task.rst index 485e1b843d08..364323d5ca8b 100644 --- a/Doc/library/asyncio-task.rst +++ b/Doc/library/asyncio-task.rst @@ -535,7 +535,7 @@ Task functions not provided, the default event loop is used. -.. function:: current_task(loop=None): +.. function:: current_task(loop=None) Return the current running :class:`Task` instance or ``None``, if no task is running. @@ -546,7 +546,7 @@ Task functions .. versionadded:: 3.7 -.. function:: all_tasks(loop=None): +.. function:: all_tasks(loop=None) Return a set of :class:`Task` objects created for the loop. From webhook-mailer at python.org Mon Mar 12 15:03:17 2018 From: webhook-mailer at python.org (Antoine Pitrou) Date: Mon, 12 Mar 2018 19:03:17 -0000 Subject: [Python-checkins] bpo-22674: fix test_strsignal on OSX (GH-6085) Message-ID: <mailman.55.1520881400.1871.python-checkins@python.org> https://github.com/python/cpython/commit/019f5b3e9e4c2a1297580483c3d5a5a10bddb93b commit: 019f5b3e9e4c2a1297580483c3d5a5a10bddb93b branch: master author: Antoine Pietri <antoine.pietri1 at gmail.com> committer: Antoine Pitrou <pitrou at free.fr> date: 2018-03-12T20:03:14+01:00 summary: bpo-22674: fix test_strsignal on OSX (GH-6085) files: M Lib/test/test_signal.py diff --git a/Lib/test/test_signal.py b/Lib/test/test_signal.py index fbb12a5b67bf..7635eec148d8 100644 --- a/Lib/test/test_signal.py +++ b/Lib/test/test_signal.py @@ -58,8 +58,8 @@ def test_getsignal(self): self.assertEqual(signal.getsignal(signal.SIGHUP), hup) def test_strsignal(self): - self.assertEqual(signal.strsignal(signal.SIGINT), "Interrupt") - self.assertEqual(signal.strsignal(signal.SIGTERM), "Terminated") + self.assertIn("Interrupt", signal.strsignal(signal.SIGINT)) + self.assertIn("Terminated", signal.strsignal(signal.SIGTERM)) # Issue 3864, unknown if this affects earlier versions of freebsd also def test_interprocess_signal(self): From webhook-mailer at python.org Mon Mar 12 15:12:12 2018 From: webhook-mailer at python.org (Miss Islington (bot)) Date: Mon, 12 Mar 2018 19:12:12 -0000 Subject: [Python-checkins] Fix docs markup for asyncio current_task() and all_tasks() (GH-6089) Message-ID: <mailman.56.1520881933.1871.python-checkins@python.org> https://github.com/python/cpython/commit/ee28cd57df3b8fca66d41528bd4b8341aad26f57 commit: ee28cd57df3b8fca66d41528bd4b8341aad26f57 branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: GitHub <noreply at github.com> date: 2018-03-12T12:12:05-07:00 summary: Fix docs markup for asyncio current_task() and all_tasks() (GH-6089) (cherry picked from commit b21505e7109bafe75d87ab63d524b4e749157dbd) Co-authored-by: Andrew Svetlov <andrew.svetlov at gmail.com> files: M Doc/library/asyncio-task.rst diff --git a/Doc/library/asyncio-task.rst b/Doc/library/asyncio-task.rst index 485e1b843d08..364323d5ca8b 100644 --- a/Doc/library/asyncio-task.rst +++ b/Doc/library/asyncio-task.rst @@ -535,7 +535,7 @@ Task functions not provided, the default event loop is used. -.. function:: current_task(loop=None): +.. function:: current_task(loop=None) Return the current running :class:`Task` instance or ``None``, if no task is running. @@ -546,7 +546,7 @@ Task functions .. versionadded:: 3.7 -.. function:: all_tasks(loop=None): +.. function:: all_tasks(loop=None) Return a set of :class:`Task` objects created for the loop. From webhook-mailer at python.org Mon Mar 12 17:07:10 2018 From: webhook-mailer at python.org (Ned Deily) Date: Mon, 12 Mar 2018 21:07:10 -0000 Subject: [Python-checkins] Add macOS installer Conclusion file Message-ID: <mailman.57.1520888830.1871.python-checkins@python.org> https://github.com/python/cpython/commit/37ed6dfbbc2a18de46cad292f9b0fc5ce32bc01d commit: 37ed6dfbbc2a18de46cad292f9b0fc5ce32bc01d branch: master author: Ned Deily <nad at python.org> committer: Ned Deily <nad at python.org> date: 2018-03-12T16:49:59-04:00 summary: Add macOS installer Conclusion file files: A Mac/BuildScript/resources/Conclusion.rtf diff --git a/Mac/BuildScript/resources/Conclusion.rtf b/Mac/BuildScript/resources/Conclusion.rtf new file mode 100644 index 000000000000..9e0fa9fa6eeb --- /dev/null +++ b/Mac/BuildScript/resources/Conclusion.rtf @@ -0,0 +1,20 @@ +{\rtf1\ansi\ansicpg1252\cocoartf1561\cocoasubrtf200 +{\fonttbl\f0\fswiss\fcharset0 Helvetica;\f1\fnil\fcharset0 LucidaGrande-Bold;\f2\fnil\fcharset0 LucidaGrande; +\f3\fnil\fcharset0 Monaco;} +{\colortbl;\red255\green255\blue255;} +{\*\expandedcolortbl;;} +\margl1440\margr1440\vieww10540\viewh8400\viewkind0 +\pard\tx720\tx1440\tx2160\tx2880\tx3600\tx4320\tx5040\tx5760\tx6480\tx7200\tx7920\tx8640\pardirnatural\partightenfactor0 + +\f0\fs28 \cf0 Congratulations! +\fs24 +\f1\b\fs28 Python $FULL_VERSION for macOS $MACOSX_DEPLOYMENT_TARGET +\f2\b0 was successfully installed. +\fs24 \ +\ +One more thing: to verify the identity of secure network connections, this Python needs a set of SSL root certificates. You can download and install a current curated set from {\field{\*\fldinst{HYPERLINK "https://pypi.org/project/certifi/"}}{\fldrslt the Certifi project}} by double-clicking on the +\f3 Install Certificates +\f2 icon in {\field{\*\fldinst{HYPERLINK "file://localhost/Applications/Python%20$VERSION/"}}{\fldrslt the Finder window}}. See {\field{\*\fldinst{HYPERLINK "file://localhost/Applications/Python%20$VERSION/ReadMe.rtf"}}{\fldrslt the +\f3 ReadMe +\f2 file}} for more information.\ +} \ No newline at end of file From webhook-mailer at python.org Mon Mar 12 17:09:54 2018 From: webhook-mailer at python.org (Ned Deily) Date: Mon, 12 Mar 2018 21:09:54 -0000 Subject: [Python-checkins] Add macOS installer Conclusion file Message-ID: <mailman.58.1520888994.1871.python-checkins@python.org> https://github.com/python/cpython/commit/6f415c48197d2ce86bce2a1cf34d3e7faa4ceb98 commit: 6f415c48197d2ce86bce2a1cf34d3e7faa4ceb98 branch: 3.7 author: Ned Deily <nad at python.org> committer: Ned Deily <nad at python.org> date: 2018-03-12T17:08:32-04:00 summary: Add macOS installer Conclusion file files: A Mac/BuildScript/resources/Conclusion.rtf diff --git a/Mac/BuildScript/resources/Conclusion.rtf b/Mac/BuildScript/resources/Conclusion.rtf new file mode 100644 index 000000000000..9e0fa9fa6eeb --- /dev/null +++ b/Mac/BuildScript/resources/Conclusion.rtf @@ -0,0 +1,20 @@ +{\rtf1\ansi\ansicpg1252\cocoartf1561\cocoasubrtf200 +{\fonttbl\f0\fswiss\fcharset0 Helvetica;\f1\fnil\fcharset0 LucidaGrande-Bold;\f2\fnil\fcharset0 LucidaGrande; +\f3\fnil\fcharset0 Monaco;} +{\colortbl;\red255\green255\blue255;} +{\*\expandedcolortbl;;} +\margl1440\margr1440\vieww10540\viewh8400\viewkind0 +\pard\tx720\tx1440\tx2160\tx2880\tx3600\tx4320\tx5040\tx5760\tx6480\tx7200\tx7920\tx8640\pardirnatural\partightenfactor0 + +\f0\fs28 \cf0 Congratulations! +\fs24 +\f1\b\fs28 Python $FULL_VERSION for macOS $MACOSX_DEPLOYMENT_TARGET +\f2\b0 was successfully installed. +\fs24 \ +\ +One more thing: to verify the identity of secure network connections, this Python needs a set of SSL root certificates. You can download and install a current curated set from {\field{\*\fldinst{HYPERLINK "https://pypi.org/project/certifi/"}}{\fldrslt the Certifi project}} by double-clicking on the +\f3 Install Certificates +\f2 icon in {\field{\*\fldinst{HYPERLINK "file://localhost/Applications/Python%20$VERSION/"}}{\fldrslt the Finder window}}. See {\field{\*\fldinst{HYPERLINK "file://localhost/Applications/Python%20$VERSION/ReadMe.rtf"}}{\fldrslt the +\f3 ReadMe +\f2 file}} for more information.\ +} \ No newline at end of file From webhook-mailer at python.org Mon Mar 12 21:16:11 2018 From: webhook-mailer at python.org (Ned Deily) Date: Tue, 13 Mar 2018 01:16:11 -0000 Subject: [Python-checkins] bpo-29719: Remove Date and Release field in whatsnew/3.7 and 8 (GH-6093) Message-ID: <mailman.59.1520903772.1871.python-checkins@python.org> https://github.com/python/cpython/commit/a34510a4c562b4b23c7f9da6ff6e2318484f5f1a commit: a34510a4c562b4b23c7f9da6ff6e2318484f5f1a branch: master author: Ned Deily <nad at python.org> committer: GitHub <noreply at github.com> date: 2018-03-12T21:16:08-04:00 summary: bpo-29719: Remove Date and Release field in whatsnew/3.7 and 8 (GH-6093) files: M Doc/whatsnew/3.7.rst M Doc/whatsnew/3.8.rst diff --git a/Doc/whatsnew/3.7.rst b/Doc/whatsnew/3.7.rst index a4ed481f67a5..7eb19f0f82a6 100644 --- a/Doc/whatsnew/3.7.rst +++ b/Doc/whatsnew/3.7.rst @@ -2,9 +2,6 @@ What's New In Python 3.7 **************************** -:Release: |release| -:Date: |today| - .. Rules for maintenance: * Anyone can add text to this document. Do not spend very much time diff --git a/Doc/whatsnew/3.8.rst b/Doc/whatsnew/3.8.rst index 904a3584e67a..8569341055e1 100644 --- a/Doc/whatsnew/3.8.rst +++ b/Doc/whatsnew/3.8.rst @@ -2,9 +2,6 @@ What's New In Python 3.8 **************************** -:Release: |release| -:Date: |today| - .. Rules for maintenance: * Anyone can add text to this document. Do not spend very much time From webhook-mailer at python.org Mon Mar 12 21:16:57 2018 From: webhook-mailer at python.org (Ned Deily) Date: Tue, 13 Mar 2018 01:16:57 -0000 Subject: [Python-checkins] bpo-29719: Remove Date and Release field in whatsnew/3.7 (GH-6094) Message-ID: <mailman.60.1520903818.1871.python-checkins@python.org> https://github.com/python/cpython/commit/d461ed84b57aae0bdccd117dc38109c087444d50 commit: d461ed84b57aae0bdccd117dc38109c087444d50 branch: 3.7 author: Ned Deily <nad at python.org> committer: GitHub <noreply at github.com> date: 2018-03-12T21:16:54-04:00 summary: bpo-29719: Remove Date and Release field in whatsnew/3.7 (GH-6094) files: M Doc/whatsnew/3.7.rst diff --git a/Doc/whatsnew/3.7.rst b/Doc/whatsnew/3.7.rst index a4ed481f67a5..7eb19f0f82a6 100644 --- a/Doc/whatsnew/3.7.rst +++ b/Doc/whatsnew/3.7.rst @@ -2,9 +2,6 @@ What's New In Python 3.7 **************************** -:Release: |release| -:Date: |today| - .. Rules for maintenance: * Anyone can add text to this document. Do not spend very much time From webhook-mailer at python.org Tue Mar 13 03:44:54 2018 From: webhook-mailer at python.org (=?utf-8?q?=C5=81ukasz?= Langa) Date: Tue, 13 Mar 2018 07:44:54 -0000 Subject: [Python-checkins] bpo-33064: lib2to3: support trailing comma after *args and **kwargs (#6096) Message-ID: <mailman.61.1520927096.1871.python-checkins@python.org> https://github.com/python/cpython/commit/b51f5de71163f096d2d5229ede5379cdb284f651 commit: b51f5de71163f096d2d5229ede5379cdb284f651 branch: master author: ?ukasz Langa <lukasz at langa.pl> committer: GitHub <noreply at github.com> date: 2018-03-13T00:44:49-07:00 summary: bpo-33064: lib2to3: support trailing comma after *args and **kwargs (#6096) New tests also added. I also made the comments in line with the builtin Grammar/Grammar. PEP 306 was withdrawn, Kees Blom's railroad program has been lost to the sands of time for at least 16 years now (I found a python-dev post from people looking for it). files: A Misc/NEWS.d/next/Library/2018-03-12-19-58-25.bpo-33064.LO2KIY.rst M Lib/lib2to3/Grammar.txt M Lib/lib2to3/tests/test_parser.py diff --git a/Lib/lib2to3/Grammar.txt b/Lib/lib2to3/Grammar.txt index 0bdfcafcf3ca..b19b4a21fadd 100644 --- a/Lib/lib2to3/Grammar.txt +++ b/Lib/lib2to3/Grammar.txt @@ -1,26 +1,7 @@ # Grammar for 2to3. This grammar supports Python 2.x and 3.x. -# Note: Changing the grammar specified in this file will most likely -# require corresponding changes in the parser module -# (../Modules/parsermodule.c). If you can't make the changes to -# that module yourself, please co-ordinate the required changes -# with someone who can; ask around on python-dev for help. Fred -# Drake <fdrake at acm.org> will probably be listening there. - -# NOTE WELL: You should also follow all the steps listed in PEP 306, -# "How to Change Python's Grammar" - -# Commands for Kees Blom's railroad program -#diagram:token NAME -#diagram:token NUMBER -#diagram:token STRING -#diagram:token NEWLINE -#diagram:token ENDMARKER -#diagram:token INDENT -#diagram:output\input python.bla -#diagram:token DEDENT -#diagram:output\textwidth 20.04cm\oddsidemargin 0.0cm\evensidemargin 0.0cm -#diagram:rules +# NOTE WELL: You should also follow all the steps listed at +# https://devguide.python.org/grammar/ # Start symbols for the grammar: # file_input is a module or sequence of commands read from an input file; @@ -38,13 +19,13 @@ async_funcdef: 'async' funcdef funcdef: 'def' NAME parameters ['->' test] ':' suite parameters: '(' [typedargslist] ')' typedargslist: ((tfpdef ['=' test] ',')* - ('*' [tname] (',' tname ['=' test])* [',' '**' tname] | '**' tname) + ('*' [tname] (',' tname ['=' test])* [',' ['**' tname [',']]] | '**' tname [',']) | tfpdef ['=' test] (',' tfpdef ['=' test])* [',']) tname: NAME [':' test] tfpdef: tname | '(' tfplist ')' tfplist: tfpdef (',' tfpdef)* [','] varargslist: ((vfpdef ['=' test] ',')* - ('*' [vname] (',' vname ['=' test])* [',' '**' vname] | '**' vname) + ('*' [vname] (',' vname ['=' test])* [',' ['**' vname [',']]] | '**' vname [',']) | vfpdef ['=' test] (',' vfpdef ['=' test])* [',']) vname: NAME vfpdef: vname | '(' vfplist ')' diff --git a/Lib/lib2to3/tests/test_parser.py b/Lib/lib2to3/tests/test_parser.py index 0cbba26bec04..cec2f98e9665 100644 --- a/Lib/lib2to3/tests/test_parser.py +++ b/Lib/lib2to3/tests/test_parser.py @@ -9,7 +9,6 @@ # Testing imports from . import support from .support import driver, driver_no_print_statement -from test.support import verbose # Python imports import difflib @@ -22,7 +21,6 @@ import sys import tempfile import unittest -import warnings # Local imports from lib2to3.pgen2 import driver as pgen2_driver @@ -305,6 +303,38 @@ def test_8(self): *g:6, h:7, i=8, j:9=10, **k:11) -> 12: pass""" self.validate(s) + def test_9(self): + s = """def f( + a: str, + b: int, + *, + c: bool = False, + **kwargs, + ) -> None: + call(c=c, **kwargs,)""" + self.validate(s) + + def test_10(self): + s = """def f( + a: str, + ) -> None: + call(a,)""" + self.validate(s) + + def test_11(self): + s = """def f( + a: str = '', + ) -> None: + call(a=a,)""" + self.validate(s) + + def test_12(self): + s = """def f( + *args: str, + ) -> None: + call(*args,)""" + self.validate(s) + # Adapted from Python 3's Lib/test/test_grammar.py:GrammarTests.test_var_annot class TestVarAnnotations(GrammarTest): @@ -407,7 +437,7 @@ def test_new_syntax(self): self.validate("class B(t, *args): pass") self.validate("class B(t, **kwargs): pass") self.validate("class B(t, *args, **kwargs): pass") - self.validate("class B(t, y=9, *args, **kwargs): pass") + self.validate("class B(t, y=9, *args, **kwargs,): pass") class TestParserIdempotency(support.TestCase): diff --git a/Misc/NEWS.d/next/Library/2018-03-12-19-58-25.bpo-33064.LO2KIY.rst b/Misc/NEWS.d/next/Library/2018-03-12-19-58-25.bpo-33064.LO2KIY.rst new file mode 100644 index 000000000000..c8e955e335cb --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-03-12-19-58-25.bpo-33064.LO2KIY.rst @@ -0,0 +1,2 @@ +lib2to3 now properly supports trailing commas after ``*args`` and +``**kwargs`` in function signatures. From webhook-mailer at python.org Tue Mar 13 04:08:07 2018 From: webhook-mailer at python.org (=?utf-8?q?=C5=81ukasz?= Langa) Date: Tue, 13 Mar 2018 08:08:07 -0000 Subject: [Python-checkins] bpo-33064: lib2to3: support trailing comma after *args and **kwargs (GH-6096) (#6097) Message-ID: <mailman.62.1520928487.1871.python-checkins@python.org> https://github.com/python/cpython/commit/b4c8871ca43d37be167ef5dbe9fb341922c04a9f commit: b4c8871ca43d37be167ef5dbe9fb341922c04a9f branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: ?ukasz Langa <lukasz at langa.pl> date: 2018-03-13T01:08:02-07:00 summary: bpo-33064: lib2to3: support trailing comma after *args and **kwargs (GH-6096) (#6097) New tests also added. I also made the comments in line with the builtin Grammar/Grammar. PEP 306 was withdrawn, Kees Blom's railroad program has been lost to the sands of time for at least 16 years now (I found a python-dev post from people looking for it). (cherry picked from commit b51f5de71163f096d2d5229ede5379cdb284f651) Co-authored-by: ?ukasz Langa <lukasz at langa.pl> files: A Misc/NEWS.d/next/Library/2018-03-12-19-58-25.bpo-33064.LO2KIY.rst M Lib/lib2to3/Grammar.txt M Lib/lib2to3/tests/test_parser.py diff --git a/Lib/lib2to3/Grammar.txt b/Lib/lib2to3/Grammar.txt index 0bdfcafcf3ca..b19b4a21fadd 100644 --- a/Lib/lib2to3/Grammar.txt +++ b/Lib/lib2to3/Grammar.txt @@ -1,26 +1,7 @@ # Grammar for 2to3. This grammar supports Python 2.x and 3.x. -# Note: Changing the grammar specified in this file will most likely -# require corresponding changes in the parser module -# (../Modules/parsermodule.c). If you can't make the changes to -# that module yourself, please co-ordinate the required changes -# with someone who can; ask around on python-dev for help. Fred -# Drake <fdrake at acm.org> will probably be listening there. - -# NOTE WELL: You should also follow all the steps listed in PEP 306, -# "How to Change Python's Grammar" - -# Commands for Kees Blom's railroad program -#diagram:token NAME -#diagram:token NUMBER -#diagram:token STRING -#diagram:token NEWLINE -#diagram:token ENDMARKER -#diagram:token INDENT -#diagram:output\input python.bla -#diagram:token DEDENT -#diagram:output\textwidth 20.04cm\oddsidemargin 0.0cm\evensidemargin 0.0cm -#diagram:rules +# NOTE WELL: You should also follow all the steps listed at +# https://devguide.python.org/grammar/ # Start symbols for the grammar: # file_input is a module or sequence of commands read from an input file; @@ -38,13 +19,13 @@ async_funcdef: 'async' funcdef funcdef: 'def' NAME parameters ['->' test] ':' suite parameters: '(' [typedargslist] ')' typedargslist: ((tfpdef ['=' test] ',')* - ('*' [tname] (',' tname ['=' test])* [',' '**' tname] | '**' tname) + ('*' [tname] (',' tname ['=' test])* [',' ['**' tname [',']]] | '**' tname [',']) | tfpdef ['=' test] (',' tfpdef ['=' test])* [',']) tname: NAME [':' test] tfpdef: tname | '(' tfplist ')' tfplist: tfpdef (',' tfpdef)* [','] varargslist: ((vfpdef ['=' test] ',')* - ('*' [vname] (',' vname ['=' test])* [',' '**' vname] | '**' vname) + ('*' [vname] (',' vname ['=' test])* [',' ['**' vname [',']]] | '**' vname [',']) | vfpdef ['=' test] (',' vfpdef ['=' test])* [',']) vname: NAME vfpdef: vname | '(' vfplist ')' diff --git a/Lib/lib2to3/tests/test_parser.py b/Lib/lib2to3/tests/test_parser.py index 0cbba26bec04..cec2f98e9665 100644 --- a/Lib/lib2to3/tests/test_parser.py +++ b/Lib/lib2to3/tests/test_parser.py @@ -9,7 +9,6 @@ # Testing imports from . import support from .support import driver, driver_no_print_statement -from test.support import verbose # Python imports import difflib @@ -22,7 +21,6 @@ import sys import tempfile import unittest -import warnings # Local imports from lib2to3.pgen2 import driver as pgen2_driver @@ -305,6 +303,38 @@ def test_8(self): *g:6, h:7, i=8, j:9=10, **k:11) -> 12: pass""" self.validate(s) + def test_9(self): + s = """def f( + a: str, + b: int, + *, + c: bool = False, + **kwargs, + ) -> None: + call(c=c, **kwargs,)""" + self.validate(s) + + def test_10(self): + s = """def f( + a: str, + ) -> None: + call(a,)""" + self.validate(s) + + def test_11(self): + s = """def f( + a: str = '', + ) -> None: + call(a=a,)""" + self.validate(s) + + def test_12(self): + s = """def f( + *args: str, + ) -> None: + call(*args,)""" + self.validate(s) + # Adapted from Python 3's Lib/test/test_grammar.py:GrammarTests.test_var_annot class TestVarAnnotations(GrammarTest): @@ -407,7 +437,7 @@ def test_new_syntax(self): self.validate("class B(t, *args): pass") self.validate("class B(t, **kwargs): pass") self.validate("class B(t, *args, **kwargs): pass") - self.validate("class B(t, y=9, *args, **kwargs): pass") + self.validate("class B(t, y=9, *args, **kwargs,): pass") class TestParserIdempotency(support.TestCase): diff --git a/Misc/NEWS.d/next/Library/2018-03-12-19-58-25.bpo-33064.LO2KIY.rst b/Misc/NEWS.d/next/Library/2018-03-12-19-58-25.bpo-33064.LO2KIY.rst new file mode 100644 index 000000000000..c8e955e335cb --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-03-12-19-58-25.bpo-33064.LO2KIY.rst @@ -0,0 +1,2 @@ +lib2to3 now properly supports trailing commas after ``*args`` and +``**kwargs`` in function signatures. From webhook-mailer at python.org Tue Mar 13 04:32:37 2018 From: webhook-mailer at python.org (=?utf-8?q?=C5=81ukasz?= Langa) Date: Tue, 13 Mar 2018 08:32:37 -0000 Subject: [Python-checkins] bpo-33064: lib2to3: support trailing comma after *args and **kwargs (GH-6096) (#6098) Message-ID: <mailman.63.1520929958.1871.python-checkins@python.org> https://github.com/python/cpython/commit/6a526f673878677032c02f7800ee13d4769f391a commit: 6a526f673878677032c02f7800ee13d4769f391a branch: 3.6 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: ?ukasz Langa <lukasz at langa.pl> date: 2018-03-13T01:32:28-07:00 summary: bpo-33064: lib2to3: support trailing comma after *args and **kwargs (GH-6096) (#6098) New tests also added. I also made the comments in line with the builtin Grammar/Grammar. PEP 306 was withdrawn, Kees Blom's railroad program has been lost to the sands of time for at least 16 years now (I found a python-dev post from people looking for it). (cherry picked from commit b51f5de71163f096d2d5229ede5379cdb284f651) Co-authored-by: ?ukasz Langa <lukasz at langa.pl> files: A Misc/NEWS.d/next/Library/2018-03-12-19-58-25.bpo-33064.LO2KIY.rst M Lib/lib2to3/Grammar.txt M Lib/lib2to3/tests/test_parser.py diff --git a/Lib/lib2to3/Grammar.txt b/Lib/lib2to3/Grammar.txt index 2abd5ee65b5b..a7ddad3cf322 100644 --- a/Lib/lib2to3/Grammar.txt +++ b/Lib/lib2to3/Grammar.txt @@ -1,26 +1,7 @@ # Grammar for 2to3. This grammar supports Python 2.x and 3.x. -# Note: Changing the grammar specified in this file will most likely -# require corresponding changes in the parser module -# (../Modules/parsermodule.c). If you can't make the changes to -# that module yourself, please co-ordinate the required changes -# with someone who can; ask around on python-dev for help. Fred -# Drake <fdrake at acm.org> will probably be listening there. - -# NOTE WELL: You should also follow all the steps listed in PEP 306, -# "How to Change Python's Grammar" - -# Commands for Kees Blom's railroad program -#diagram:token NAME -#diagram:token NUMBER -#diagram:token STRING -#diagram:token NEWLINE -#diagram:token ENDMARKER -#diagram:token INDENT -#diagram:output\input python.bla -#diagram:token DEDENT -#diagram:output\textwidth 20.04cm\oddsidemargin 0.0cm\evensidemargin 0.0cm -#diagram:rules +# NOTE WELL: You should also follow all the steps listed at +# https://devguide.python.org/grammar/ # Start symbols for the grammar: # file_input is a module or sequence of commands read from an input file; @@ -38,13 +19,13 @@ async_funcdef: ASYNC funcdef funcdef: 'def' NAME parameters ['->' test] ':' suite parameters: '(' [typedargslist] ')' typedargslist: ((tfpdef ['=' test] ',')* - ('*' [tname] (',' tname ['=' test])* [',' '**' tname] | '**' tname) + ('*' [tname] (',' tname ['=' test])* [',' ['**' tname [',']]] | '**' tname [',']) | tfpdef ['=' test] (',' tfpdef ['=' test])* [',']) tname: NAME [':' test] tfpdef: tname | '(' tfplist ')' tfplist: tfpdef (',' tfpdef)* [','] varargslist: ((vfpdef ['=' test] ',')* - ('*' [vname] (',' vname ['=' test])* [',' '**' vname] | '**' vname) + ('*' [vname] (',' vname ['=' test])* [',' ['**' vname [',']]] | '**' vname [',']) | vfpdef ['=' test] (',' vfpdef ['=' test])* [',']) vname: NAME vfpdef: vname | '(' vfplist ')' diff --git a/Lib/lib2to3/tests/test_parser.py b/Lib/lib2to3/tests/test_parser.py index 3fc499522454..c4d062ff827e 100644 --- a/Lib/lib2to3/tests/test_parser.py +++ b/Lib/lib2to3/tests/test_parser.py @@ -9,7 +9,6 @@ # Testing imports from . import support from .support import driver, driver_no_print_statement -from test.support import verbose # Python imports import difflib @@ -22,7 +21,6 @@ import sys import tempfile import unittest -import warnings # Local imports from lib2to3.pgen2 import driver as pgen2_driver @@ -305,6 +303,38 @@ def test_8(self): *g:6, h:7, i=8, j:9=10, **k:11) -> 12: pass""" self.validate(s) + def test_9(self): + s = """def f( + a: str, + b: int, + *, + c: bool = False, + **kwargs, + ) -> None: + call(c=c, **kwargs,)""" + self.validate(s) + + def test_10(self): + s = """def f( + a: str, + ) -> None: + call(a,)""" + self.validate(s) + + def test_11(self): + s = """def f( + a: str = '', + ) -> None: + call(a=a,)""" + self.validate(s) + + def test_12(self): + s = """def f( + *args: str, + ) -> None: + call(*args,)""" + self.validate(s) + # Adapted from Python 3's Lib/test/test_grammar.py:GrammarTests.test_var_annot class TestVarAnnotations(GrammarTest): @@ -407,7 +437,7 @@ def test_new_syntax(self): self.validate("class B(t, *args): pass") self.validate("class B(t, **kwargs): pass") self.validate("class B(t, *args, **kwargs): pass") - self.validate("class B(t, y=9, *args, **kwargs): pass") + self.validate("class B(t, y=9, *args, **kwargs,): pass") class TestParserIdempotency(support.TestCase): diff --git a/Misc/NEWS.d/next/Library/2018-03-12-19-58-25.bpo-33064.LO2KIY.rst b/Misc/NEWS.d/next/Library/2018-03-12-19-58-25.bpo-33064.LO2KIY.rst new file mode 100644 index 000000000000..c8e955e335cb --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-03-12-19-58-25.bpo-33064.LO2KIY.rst @@ -0,0 +1,2 @@ +lib2to3 now properly supports trailing commas after ``*args`` and +``**kwargs`` in function signatures. From webhook-mailer at python.org Tue Mar 13 04:52:38 2018 From: webhook-mailer at python.org (Serhiy Storchaka) Date: Tue, 13 Mar 2018 08:52:38 -0000 Subject: [Python-checkins] [3.7] bpo-17288: Prevent jumps from 'return' and 'exception' trace events. (GH-5928) Message-ID: <mailman.64.1520931160.1871.python-checkins@python.org> https://github.com/python/cpython/commit/e32bbaf376a09c149fa7c7f2919d7c9ce4e2a055 commit: e32bbaf376a09c149fa7c7f2919d7c9ce4e2a055 branch: 3.7 author: xdegaye <xdegaye at gmail.com> committer: Serhiy Storchaka <storchaka at gmail.com> date: 2018-03-13T10:52:35+02:00 summary: [3.7] bpo-17288: Prevent jumps from 'return' and 'exception' trace events. (GH-5928) files: A Misc/NEWS.d/next/Core and Builtins/2018-02-27-13-36-21.bpo-17288.Gdj24S.rst M Lib/test/test_sys_settrace.py M Objects/frameobject.c diff --git a/Lib/test/test_sys_settrace.py b/Lib/test/test_sys_settrace.py index fa5b61af6172..44ea7d575b45 100644 --- a/Lib/test/test_sys_settrace.py +++ b/Lib/test/test_sys_settrace.py @@ -556,20 +556,35 @@ def g(frame, event, arg): class JumpTracer: """Defines a trace function that jumps from one place to another.""" - def __init__(self, function, jumpFrom, jumpTo): - self.function = function + def __init__(self, function, jumpFrom, jumpTo, event='line', + decorated=False): + self.code = function.__code__ self.jumpFrom = jumpFrom self.jumpTo = jumpTo + self.event = event + self.firstLine = None if decorated else self.code.co_firstlineno self.done = False def trace(self, frame, event, arg): - if not self.done and frame.f_code == self.function.__code__: - firstLine = frame.f_code.co_firstlineno - if event == 'line' and frame.f_lineno == firstLine + self.jumpFrom: + if self.done: + return + # frame.f_code.co_firstlineno is the first line of the decorator when + # 'function' is decorated and the decorator may be written using + # multiple physical lines when it is too long. Use the first line + # trace event in 'function' to find the first line of 'function'. + if (self.firstLine is None and frame.f_code == self.code and + event == 'line'): + self.firstLine = frame.f_lineno - 1 + if (event == self.event and self.firstLine and + frame.f_lineno == self.firstLine + self.jumpFrom): + f = frame + while f is not None and f.f_code != self.code: + f = f.f_back + if f is not None: # Cope with non-integer self.jumpTo (because of # no_jump_to_non_integers below). try: - frame.f_lineno = firstLine + self.jumpTo + frame.f_lineno = self.firstLine + self.jumpTo except TypeError: frame.f_lineno = self.jumpTo self.done = True @@ -609,8 +624,9 @@ def compare_jump_output(self, expected, received): "Expected: " + repr(expected) + "\n" + "Received: " + repr(received)) - def run_test(self, func, jumpFrom, jumpTo, expected, error=None): - tracer = JumpTracer(func, jumpFrom, jumpTo) + def run_test(self, func, jumpFrom, jumpTo, expected, error=None, + event='line', decorated=False): + tracer = JumpTracer(func, jumpFrom, jumpTo, event, decorated) sys.settrace(tracer.trace) output = [] if error is None: @@ -621,15 +637,15 @@ def run_test(self, func, jumpFrom, jumpTo, expected, error=None): sys.settrace(None) self.compare_jump_output(expected, output) - def jump_test(jumpFrom, jumpTo, expected, error=None): + def jump_test(jumpFrom, jumpTo, expected, error=None, event='line'): """Decorator that creates a test that makes a jump from one place to another in the following code. """ def decorator(func): @wraps(func) def test(self): - # +1 to compensate a decorator line - self.run_test(func, jumpFrom+1, jumpTo+1, expected, error) + self.run_test(func, jumpFrom, jumpTo, expected, + error=error, event=event, decorated=True) return test return decorator @@ -1104,6 +1120,36 @@ class fake_function: sys.settrace(None) self.compare_jump_output([2, 3, 2, 3, 4], namespace["output"]) + @jump_test(2, 3, [1], event='call', error=(ValueError, "can't jump from" + " the 'call' trace event of a new frame")) + def test_no_jump_from_call(output): + output.append(1) + def nested(): + output.append(3) + nested() + output.append(5) + + @jump_test(2, 1, [1], event='return', error=(ValueError, + "can only jump from a 'line' trace event")) + def test_no_jump_from_return_event(output): + output.append(1) + return + + @jump_test(2, 1, [1], event='exception', error=(ValueError, + "can only jump from a 'line' trace event")) + def test_no_jump_from_exception_event(output): + output.append(1) + 1 / 0 + + @jump_test(3, 2, [2], event='return', error=(ValueError, + "can't jump from a yield statement")) + def test_no_jump_from_yield(output): + def gen(): + output.append(2) + yield 3 + next(gen()) + output.append(5) + if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS.d/next/Core and Builtins/2018-02-27-13-36-21.bpo-17288.Gdj24S.rst b/Misc/NEWS.d/next/Core and Builtins/2018-02-27-13-36-21.bpo-17288.Gdj24S.rst new file mode 100644 index 000000000000..ce9e84c40313 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2018-02-27-13-36-21.bpo-17288.Gdj24S.rst @@ -0,0 +1 @@ +Prevent jumps from 'return' and 'exception' trace events. diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 2825041e6310..f518dc485616 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -59,6 +59,9 @@ frame_getlineno(PyFrameObject *f, void *closure) * o 'try'/'for'/'while' blocks can't be jumped into because the blockstack * needs to be set up before their code runs, and for 'for' loops the * iterator needs to be on the stack. + * o Jumps cannot be made from within a trace function invoked with a + * 'return' or 'exception' event since the eval loop has been exited at + * that time. */ static int frame_setlineno(PyFrameObject *f, PyObject* p_new_lineno) @@ -94,13 +97,32 @@ frame_setlineno(PyFrameObject *f, PyObject* p_new_lineno) return -1; } + /* Upon the 'call' trace event of a new frame, f->f_lasti is -1 and + * f->f_trace is NULL, check first on the first condition. + * Forbidding jumps from the 'call' event of a new frame is a side effect + * of allowing to set f_lineno only from trace functions. */ + if (f->f_lasti == -1) { + PyErr_Format(PyExc_ValueError, + "can't jump from the 'call' trace event of a new frame"); + return -1; + } + /* You can only do this from within a trace function, not via * _getframe or similar hackery. */ - if (!f->f_trace) - { + if (!f->f_trace) { PyErr_Format(PyExc_ValueError, - "f_lineno can only be set by a" - " line trace function"); + "f_lineno can only be set by a trace function"); + return -1; + } + + /* Forbid jumps upon a 'return' trace event (except after executing a + * YIELD_VALUE or YIELD_FROM opcode, f_stacktop is not NULL in that case) + * and upon an 'exception' trace event. + * Jumps from 'call' trace events have already been forbidden above for new + * frames, so this check does not change anything for 'call' events. */ + if (f->f_stacktop == NULL) { + PyErr_SetString(PyExc_ValueError, + "can only jump from a 'line' trace event"); return -1; } @@ -159,6 +181,16 @@ frame_setlineno(PyFrameObject *f, PyObject* p_new_lineno) /* We're now ready to look at the bytecode. */ PyBytes_AsStringAndSize(f->f_code->co_code, (char **)&code, &code_len); + + /* The trace function is called with a 'return' trace event after the + * execution of a yield statement. */ + assert(f->f_lasti != -1); + if (code[f->f_lasti] == YIELD_VALUE || code[f->f_lasti] == YIELD_FROM) { + PyErr_SetString(PyExc_ValueError, + "can't jump from a yield statement"); + return -1; + } + min_addr = Py_MIN(new_lasti, f->f_lasti); max_addr = Py_MAX(new_lasti, f->f_lasti); From webhook-mailer at python.org Tue Mar 13 05:11:00 2018 From: webhook-mailer at python.org (Antoine Pitrou) Date: Tue, 13 Mar 2018 09:11:00 -0000 Subject: [Python-checkins] bpo-33056 FIX leaking fd in concurrent.futures.ProcessPoolExecutor (GH-6084) (#6092) Message-ID: <mailman.65.1520932262.1871.python-checkins@python.org> https://github.com/python/cpython/commit/f216cbf9ab704da98146a25d57ff0e85aecb49da commit: f216cbf9ab704da98146a25d57ff0e85aecb49da branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Antoine Pitrou <pitrou at free.fr> date: 2018-03-13T10:10:57+01:00 summary: bpo-33056 FIX leaking fd in concurrent.futures.ProcessPoolExecutor (GH-6084) (#6092) (cherry picked from commit 095ee415cee41bf24c3a1108c23307e5baf168dd) Co-authored-by: Thomas Moreau <thomas.moreau.2010 at gmail.com> files: A Misc/NEWS.d/next/Library/2018-03-12-16-40-00.bpo-33056.lNN9Eh.rst M Lib/concurrent/futures/process.py diff --git a/Lib/concurrent/futures/process.py b/Lib/concurrent/futures/process.py index aaa5151e017c..63f22cfca325 100644 --- a/Lib/concurrent/futures/process.py +++ b/Lib/concurrent/futures/process.py @@ -78,11 +78,13 @@ class _ThreadWakeup: - __slot__ = ["_state"] - def __init__(self): self._reader, self._writer = mp.Pipe(duplex=False) + def close(self): + self._writer.close() + self._reader.close() + def wakeup(self): self._writer.send_bytes(b"") @@ -654,6 +656,11 @@ def shutdown(self, wait=True): self._call_queue = None self._result_queue = None self._processes = None + + if self._queue_management_thread_wakeup: + self._queue_management_thread_wakeup.close() + self._queue_management_thread_wakeup = None + shutdown.__doc__ = _base.Executor.shutdown.__doc__ atexit.register(_python_exit) diff --git a/Misc/NEWS.d/next/Library/2018-03-12-16-40-00.bpo-33056.lNN9Eh.rst b/Misc/NEWS.d/next/Library/2018-03-12-16-40-00.bpo-33056.lNN9Eh.rst new file mode 100644 index 000000000000..6acc19a36dc8 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-03-12-16-40-00.bpo-33056.lNN9Eh.rst @@ -0,0 +1 @@ +FIX properly close leaking fds in concurrent.futures.ProcessPoolExecutor. From solipsis at pitrou.net Tue Mar 13 05:12:50 2018 From: solipsis at pitrou.net (solipsis at pitrou.net) Date: Tue, 13 Mar 2018 09:12:50 +0000 Subject: [Python-checkins] Daily reference leaks (4243df51fe43): sum=1 Message-ID: <20180313091250.1.4F9660C55558EC80@psf.io> results for 4243df51fe43 on branch "default" -------------------------------------------- test_asyncio leaked [3, 0, 0] memory blocks, sum=3 test_collections leaked [-7, 8, -7] memory blocks, sum=-6 test_functools leaked [0, 3, 1] memory blocks, sum=4 test_multiprocessing_fork leaked [1, 0, -2] memory blocks, sum=-1 test_multiprocessing_forkserver leaked [2, 0, -1] memory blocks, sum=1 Command line was: ['./python', '-m', 'test.regrtest', '-uall', '-R', '3:3:/home/psf-users/antoine/refleaks/reflogqmRFmj', '--timeout', '7200'] From webhook-mailer at python.org Tue Mar 13 05:56:46 2018 From: webhook-mailer at python.org (Christian Heimes) Date: Tue, 13 Mar 2018 09:56:46 -0000 Subject: [Python-checkins] bpo-32885: Tools/scripts/pathfix.py: Add -n option for no backup~ (#5772) Message-ID: <mailman.66.1520935008.1871.python-checkins@python.org> https://github.com/python/cpython/commit/5affd5c29eb1493cb31ef3cfdde15538ac134689 commit: 5affd5c29eb1493cb31ef3cfdde15538ac134689 branch: master author: Miro Hron?ok <miro at hroncok.cz> committer: Christian Heimes <christian at python.org> date: 2018-03-13T05:56:43-04:00 summary: bpo-32885: Tools/scripts/pathfix.py: Add -n option for no backup~ (#5772) Creating backup files with ~ suffix can be undesirable in some environment, such as when building RPM packages. Instead of requiring the user to remove those files manually, option -n was added, that simply disables this feature. -n was selected because 2to3 has the same option with this behavior. files: A Misc/NEWS.d/next/Tools-Demos/2018-02-20-12-16-47.bpo-32885.dL5x7C.rst M Misc/ACKS M Tools/scripts/pathfix.py diff --git a/Misc/ACKS b/Misc/ACKS index d8179c8b03ab..d752d8a35434 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -687,6 +687,7 @@ Ken Howard Brad Howes Mike Hoy Ben Hoyt +Miro Hron?ok Chiu-Hsiang Hsu Chih-Hao Huang Christian Hudon diff --git a/Misc/NEWS.d/next/Tools-Demos/2018-02-20-12-16-47.bpo-32885.dL5x7C.rst b/Misc/NEWS.d/next/Tools-Demos/2018-02-20-12-16-47.bpo-32885.dL5x7C.rst new file mode 100644 index 000000000000..e003e1d84fd0 --- /dev/null +++ b/Misc/NEWS.d/next/Tools-Demos/2018-02-20-12-16-47.bpo-32885.dL5x7C.rst @@ -0,0 +1,2 @@ +Add an ``-n`` flag for ``Tools/scripts/pathfix.py`` to disbale automatic +backup creation (files with ``~`` suffix). diff --git a/Tools/scripts/pathfix.py b/Tools/scripts/pathfix.py index 562bbc737812..c5bf984306a3 100755 --- a/Tools/scripts/pathfix.py +++ b/Tools/scripts/pathfix.py @@ -7,8 +7,9 @@ # Directories are searched recursively for files whose name looks # like a python module. # Symbolic links are always ignored (except as explicit directory -# arguments). Of course, the original file is kept as a back-up -# (with a "~" attached to its name). +# arguments). +# The original file is kept as a back-up (with a "~" attached to its name), +# -n flag can be used to disable this. # # Undoubtedly you can do this using find and sed or perl, but this is # a nice example of Python code that recurses down a directory tree @@ -31,14 +32,17 @@ new_interpreter = None preserve_timestamps = False +create_backup = True + def main(): global new_interpreter global preserve_timestamps - usage = ('usage: %s -i /interpreter -p file-or-directory ...\n' % + global create_backup + usage = ('usage: %s -i /interpreter -p -n file-or-directory ...\n' % sys.argv[0]) try: - opts, args = getopt.getopt(sys.argv[1:], 'i:p') + opts, args = getopt.getopt(sys.argv[1:], 'i:pn') except getopt.error as msg: err(str(msg) + '\n') err(usage) @@ -48,6 +52,8 @@ def main(): new_interpreter = a.encode() if o == '-p': preserve_timestamps = True + if o == '-n': + create_backup = False if not new_interpreter or not new_interpreter.startswith(b'/') or \ not args: err('-i option or file-or-directory missing\n') @@ -134,10 +140,16 @@ def fix(filename): except OSError as msg: err('%s: warning: chmod failed (%r)\n' % (tempname, msg)) # Then make a backup of the original file as filename~ - try: - os.rename(filename, filename + '~') - except OSError as msg: - err('%s: warning: backup failed (%r)\n' % (filename, msg)) + if create_backup: + try: + os.rename(filename, filename + '~') + except OSError as msg: + err('%s: warning: backup failed (%r)\n' % (filename, msg)) + else: + try: + os.remove(filename) + except OSError as msg: + err('%s: warning: removing failed (%r)\n' % (filename, msg)) # Now move the temp file to the original file try: os.rename(tempname, filename) From webhook-mailer at python.org Tue Mar 13 06:12:19 2018 From: webhook-mailer at python.org (Miss Islington (bot)) Date: Tue, 13 Mar 2018 10:12:19 -0000 Subject: [Python-checkins] [3.7] bpo-17288: Prevent jumps from 'return' and 'exception' trace events. (GH-5928) Message-ID: <mailman.67.1520935942.1871.python-checkins@python.org> https://github.com/python/cpython/commit/cf61a81f1d600064be6dd43896afcf5f976de9b0 commit: cf61a81f1d600064be6dd43896afcf5f976de9b0 branch: 3.6 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: GitHub <noreply at github.com> date: 2018-03-13T03:12:16-07:00 summary: [3.7] bpo-17288: Prevent jumps from 'return' and 'exception' trace events. (GH-5928) (cherry picked from commit e32bbaf376a09c149fa7c7f2919d7c9ce4e2a055) Co-authored-by: xdegaye <xdegaye at gmail.com> files: A Misc/NEWS.d/next/Core and Builtins/2018-02-27-13-36-21.bpo-17288.Gdj24S.rst M Lib/test/test_sys_settrace.py M Objects/frameobject.c diff --git a/Lib/test/test_sys_settrace.py b/Lib/test/test_sys_settrace.py index a05fbbd04a84..46593cf54ad4 100644 --- a/Lib/test/test_sys_settrace.py +++ b/Lib/test/test_sys_settrace.py @@ -510,20 +510,35 @@ def g(frame, event, arg): class JumpTracer: """Defines a trace function that jumps from one place to another.""" - def __init__(self, function, jumpFrom, jumpTo): - self.function = function + def __init__(self, function, jumpFrom, jumpTo, event='line', + decorated=False): + self.code = function.__code__ self.jumpFrom = jumpFrom self.jumpTo = jumpTo + self.event = event + self.firstLine = None if decorated else self.code.co_firstlineno self.done = False def trace(self, frame, event, arg): - if not self.done and frame.f_code == self.function.__code__: - firstLine = frame.f_code.co_firstlineno - if event == 'line' and frame.f_lineno == firstLine + self.jumpFrom: + if self.done: + return + # frame.f_code.co_firstlineno is the first line of the decorator when + # 'function' is decorated and the decorator may be written using + # multiple physical lines when it is too long. Use the first line + # trace event in 'function' to find the first line of 'function'. + if (self.firstLine is None and frame.f_code == self.code and + event == 'line'): + self.firstLine = frame.f_lineno - 1 + if (event == self.event and self.firstLine and + frame.f_lineno == self.firstLine + self.jumpFrom): + f = frame + while f is not None and f.f_code != self.code: + f = f.f_back + if f is not None: # Cope with non-integer self.jumpTo (because of # no_jump_to_non_integers below). try: - frame.f_lineno = firstLine + self.jumpTo + frame.f_lineno = self.firstLine + self.jumpTo except TypeError: frame.f_lineno = self.jumpTo self.done = True @@ -563,8 +578,9 @@ def compare_jump_output(self, expected, received): "Expected: " + repr(expected) + "\n" + "Received: " + repr(received)) - def run_test(self, func, jumpFrom, jumpTo, expected, error=None): - tracer = JumpTracer(func, jumpFrom, jumpTo) + def run_test(self, func, jumpFrom, jumpTo, expected, error=None, + event='line', decorated=False): + tracer = JumpTracer(func, jumpFrom, jumpTo, event, decorated) sys.settrace(tracer.trace) output = [] if error is None: @@ -575,15 +591,15 @@ def run_test(self, func, jumpFrom, jumpTo, expected, error=None): sys.settrace(None) self.compare_jump_output(expected, output) - def jump_test(jumpFrom, jumpTo, expected, error=None): + def jump_test(jumpFrom, jumpTo, expected, error=None, event='line'): """Decorator that creates a test that makes a jump from one place to another in the following code. """ def decorator(func): @wraps(func) def test(self): - # +1 to compensate a decorator line - self.run_test(func, jumpFrom+1, jumpTo+1, expected, error) + self.run_test(func, jumpFrom, jumpTo, expected, + error=error, event=event, decorated=True) return test return decorator @@ -1058,6 +1074,36 @@ class fake_function: sys.settrace(None) self.compare_jump_output([2, 3, 2, 3, 4], namespace["output"]) + @jump_test(2, 3, [1], event='call', error=(ValueError, "can't jump from" + " the 'call' trace event of a new frame")) + def test_no_jump_from_call(output): + output.append(1) + def nested(): + output.append(3) + nested() + output.append(5) + + @jump_test(2, 1, [1], event='return', error=(ValueError, + "can only jump from a 'line' trace event")) + def test_no_jump_from_return_event(output): + output.append(1) + return + + @jump_test(2, 1, [1], event='exception', error=(ValueError, + "can only jump from a 'line' trace event")) + def test_no_jump_from_exception_event(output): + output.append(1) + 1 / 0 + + @jump_test(3, 2, [2], event='return', error=(ValueError, + "can't jump from a yield statement")) + def test_no_jump_from_yield(output): + def gen(): + output.append(2) + yield 3 + next(gen()) + output.append(5) + if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS.d/next/Core and Builtins/2018-02-27-13-36-21.bpo-17288.Gdj24S.rst b/Misc/NEWS.d/next/Core and Builtins/2018-02-27-13-36-21.bpo-17288.Gdj24S.rst new file mode 100644 index 000000000000..ce9e84c40313 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2018-02-27-13-36-21.bpo-17288.Gdj24S.rst @@ -0,0 +1 @@ +Prevent jumps from 'return' and 'exception' trace events. diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 5c73e8560f86..b7a16ad05966 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -56,6 +56,9 @@ frame_getlineno(PyFrameObject *f, void *closure) * o 'try'/'for'/'while' blocks can't be jumped into because the blockstack * needs to be set up before their code runs, and for 'for' loops the * iterator needs to be on the stack. + * o Jumps cannot be made from within a trace function invoked with a + * 'return' or 'exception' event since the eval loop has been exited at + * that time. */ static int frame_setlineno(PyFrameObject *f, PyObject* p_new_lineno) @@ -91,13 +94,32 @@ frame_setlineno(PyFrameObject *f, PyObject* p_new_lineno) return -1; } + /* Upon the 'call' trace event of a new frame, f->f_lasti is -1 and + * f->f_trace is NULL, check first on the first condition. + * Forbidding jumps from the 'call' event of a new frame is a side effect + * of allowing to set f_lineno only from trace functions. */ + if (f->f_lasti == -1) { + PyErr_Format(PyExc_ValueError, + "can't jump from the 'call' trace event of a new frame"); + return -1; + } + /* You can only do this from within a trace function, not via * _getframe or similar hackery. */ - if (!f->f_trace) - { + if (!f->f_trace) { PyErr_Format(PyExc_ValueError, - "f_lineno can only be set by a" - " line trace function"); + "f_lineno can only be set by a trace function"); + return -1; + } + + /* Forbid jumps upon a 'return' trace event (except after executing a + * YIELD_VALUE or YIELD_FROM opcode, f_stacktop is not NULL in that case) + * and upon an 'exception' trace event. + * Jumps from 'call' trace events have already been forbidden above for new + * frames, so this check does not change anything for 'call' events. */ + if (f->f_stacktop == NULL) { + PyErr_SetString(PyExc_ValueError, + "can only jump from a 'line' trace event"); return -1; } @@ -156,6 +178,16 @@ frame_setlineno(PyFrameObject *f, PyObject* p_new_lineno) /* We're now ready to look at the bytecode. */ PyBytes_AsStringAndSize(f->f_code->co_code, (char **)&code, &code_len); + + /* The trace function is called with a 'return' trace event after the + * execution of a yield statement. */ + assert(f->f_lasti != -1); + if (code[f->f_lasti] == YIELD_VALUE || code[f->f_lasti] == YIELD_FROM) { + PyErr_SetString(PyExc_ValueError, + "can't jump from a yield statement"); + return -1; + } + min_addr = Py_MIN(new_lasti, f->f_lasti); max_addr = Py_MAX(new_lasti, f->f_lasti); From webhook-mailer at python.org Tue Mar 13 13:31:36 2018 From: webhook-mailer at python.org (Serhiy Storchaka) Date: Tue, 13 Mar 2018 17:31:36 -0000 Subject: [Python-checkins] bpo-17288: Prevent jumps from 'return' and 'exception' trace events. (GH-6107) Message-ID: <mailman.68.1520962298.1871.python-checkins@python.org> https://github.com/python/cpython/commit/b8e9d6c5cd44ebc9c462fea9ad1bc5d0b970e28a commit: b8e9d6c5cd44ebc9c462fea9ad1bc5d0b970e28a branch: master author: xdegaye <xdegaye at gmail.com> committer: Serhiy Storchaka <storchaka at gmail.com> date: 2018-03-13T19:31:31+02:00 summary: bpo-17288: Prevent jumps from 'return' and 'exception' trace events. (GH-6107) (cherry picked from commit e32bbaf376a09c149fa7c7f2919d7c9ce4e2a055) files: A Misc/NEWS.d/next/Core and Builtins/2018-02-27-13-36-21.bpo-17288.Gdj24S.rst M Lib/test/test_sys_settrace.py M Objects/frameobject.c diff --git a/Lib/test/test_sys_settrace.py b/Lib/test/test_sys_settrace.py index 72cce33392dc..90d1e37a3916 100644 --- a/Lib/test/test_sys_settrace.py +++ b/Lib/test/test_sys_settrace.py @@ -555,20 +555,35 @@ def g(frame, event, arg): class JumpTracer: """Defines a trace function that jumps from one place to another.""" - def __init__(self, function, jumpFrom, jumpTo): - self.function = function + def __init__(self, function, jumpFrom, jumpTo, event='line', + decorated=False): + self.code = function.__code__ self.jumpFrom = jumpFrom self.jumpTo = jumpTo + self.event = event + self.firstLine = None if decorated else self.code.co_firstlineno self.done = False def trace(self, frame, event, arg): - if not self.done and frame.f_code == self.function.__code__: - firstLine = frame.f_code.co_firstlineno - if event == 'line' and frame.f_lineno == firstLine + self.jumpFrom: + if self.done: + return + # frame.f_code.co_firstlineno is the first line of the decorator when + # 'function' is decorated and the decorator may be written using + # multiple physical lines when it is too long. Use the first line + # trace event in 'function' to find the first line of 'function'. + if (self.firstLine is None and frame.f_code == self.code and + event == 'line'): + self.firstLine = frame.f_lineno - 1 + if (event == self.event and self.firstLine and + frame.f_lineno == self.firstLine + self.jumpFrom): + f = frame + while f is not None and f.f_code != self.code: + f = f.f_back + if f is not None: # Cope with non-integer self.jumpTo (because of # no_jump_to_non_integers below). try: - frame.f_lineno = firstLine + self.jumpTo + frame.f_lineno = self.firstLine + self.jumpTo except TypeError: frame.f_lineno = self.jumpTo self.done = True @@ -608,8 +623,9 @@ def compare_jump_output(self, expected, received): "Expected: " + repr(expected) + "\n" + "Received: " + repr(received)) - def run_test(self, func, jumpFrom, jumpTo, expected, error=None): - tracer = JumpTracer(func, jumpFrom, jumpTo) + def run_test(self, func, jumpFrom, jumpTo, expected, error=None, + event='line', decorated=False): + tracer = JumpTracer(func, jumpFrom, jumpTo, event, decorated) sys.settrace(tracer.trace) output = [] if error is None: @@ -620,15 +636,15 @@ def run_test(self, func, jumpFrom, jumpTo, expected, error=None): sys.settrace(None) self.compare_jump_output(expected, output) - def jump_test(jumpFrom, jumpTo, expected, error=None): + def jump_test(jumpFrom, jumpTo, expected, error=None, event='line'): """Decorator that creates a test that makes a jump from one place to another in the following code. """ def decorator(func): @wraps(func) def test(self): - # +1 to compensate a decorator line - self.run_test(func, jumpFrom+1, jumpTo+1, expected, error) + self.run_test(func, jumpFrom, jumpTo, expected, + error=error, event=event, decorated=True) return test return decorator @@ -1128,6 +1144,36 @@ class fake_function: sys.settrace(None) self.compare_jump_output([2, 3, 2, 3, 4], namespace["output"]) + @jump_test(2, 3, [1], event='call', error=(ValueError, "can't jump from" + " the 'call' trace event of a new frame")) + def test_no_jump_from_call(output): + output.append(1) + def nested(): + output.append(3) + nested() + output.append(5) + + @jump_test(2, 1, [1], event='return', error=(ValueError, + "can only jump from a 'line' trace event")) + def test_no_jump_from_return_event(output): + output.append(1) + return + + @jump_test(2, 1, [1], event='exception', error=(ValueError, + "can only jump from a 'line' trace event")) + def test_no_jump_from_exception_event(output): + output.append(1) + 1 / 0 + + @jump_test(3, 2, [2], event='return', error=(ValueError, + "can't jump from a yield statement")) + def test_no_jump_from_yield(output): + def gen(): + output.append(2) + yield 3 + next(gen()) + output.append(5) + if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS.d/next/Core and Builtins/2018-02-27-13-36-21.bpo-17288.Gdj24S.rst b/Misc/NEWS.d/next/Core and Builtins/2018-02-27-13-36-21.bpo-17288.Gdj24S.rst new file mode 100644 index 000000000000..ce9e84c40313 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2018-02-27-13-36-21.bpo-17288.Gdj24S.rst @@ -0,0 +1 @@ +Prevent jumps from 'return' and 'exception' trace events. diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 31ad8d0be226..643be08fa182 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -81,6 +81,9 @@ get_arg(const _Py_CODEUNIT *codestr, Py_ssize_t i) * the blockstack needs to be set up before their code runs. * o 'for' and 'async for' loops can't be jumped into because the * iterator needs to be on the stack. + * o Jumps cannot be made from within a trace function invoked with a + * 'return' or 'exception' event since the eval loop has been exited at + * that time. */ static int frame_setlineno(PyFrameObject *f, PyObject* p_new_lineno) @@ -109,13 +112,32 @@ frame_setlineno(PyFrameObject *f, PyObject* p_new_lineno) return -1; } + /* Upon the 'call' trace event of a new frame, f->f_lasti is -1 and + * f->f_trace is NULL, check first on the first condition. + * Forbidding jumps from the 'call' event of a new frame is a side effect + * of allowing to set f_lineno only from trace functions. */ + if (f->f_lasti == -1) { + PyErr_Format(PyExc_ValueError, + "can't jump from the 'call' trace event of a new frame"); + return -1; + } + /* You can only do this from within a trace function, not via * _getframe or similar hackery. */ - if (!f->f_trace) - { + if (!f->f_trace) { PyErr_Format(PyExc_ValueError, - "f_lineno can only be set by a" - " line trace function"); + "f_lineno can only be set by a trace function"); + return -1; + } + + /* Forbid jumps upon a 'return' trace event (except after executing a + * YIELD_VALUE or YIELD_FROM opcode, f_stacktop is not NULL in that case) + * and upon an 'exception' trace event. + * Jumps from 'call' trace events have already been forbidden above for new + * frames, so this check does not change anything for 'call' events. */ + if (f->f_stacktop == NULL) { + PyErr_SetString(PyExc_ValueError, + "can only jump from a 'line' trace event"); return -1; } @@ -175,6 +197,15 @@ frame_setlineno(PyFrameObject *f, PyObject* p_new_lineno) /* We're now ready to look at the bytecode. */ PyBytes_AsStringAndSize(f->f_code->co_code, (char **)&code, &code_len); + /* The trace function is called with a 'return' trace event after the + * execution of a yield statement. */ + assert(f->f_lasti != -1); + if (code[f->f_lasti] == YIELD_VALUE || code[f->f_lasti] == YIELD_FROM) { + PyErr_SetString(PyExc_ValueError, + "can't jump from a yield statement"); + return -1; + } + /* You can't jump onto a line with an 'except' statement on it - * they expect to have an exception on the top of the stack, which * won't be true if you jump to them. They always start with code From webhook-mailer at python.org Tue Mar 13 13:53:30 2018 From: webhook-mailer at python.org (=?utf-8?q?=C5=81ukasz?= Langa) Date: Tue, 13 Mar 2018 17:53:30 -0000 Subject: [Python-checkins] lib2to3: Add more tests (#6101) Message-ID: <mailman.69.1520963612.1871.python-checkins@python.org> https://github.com/python/cpython/commit/74f56878cdee18d485e4f8b485d55ce62e38f4c9 commit: 74f56878cdee18d485e4f8b485d55ce62e38f4c9 branch: master author: ?ukasz Langa <lukasz at langa.pl> committer: GitHub <noreply at github.com> date: 2018-03-13T10:53:22-07:00 summary: lib2to3: Add more tests (#6101) files: M Lib/lib2to3/tests/test_parser.py diff --git a/Lib/lib2to3/tests/test_parser.py b/Lib/lib2to3/tests/test_parser.py index cec2f98e9665..6813c65a7953 100644 --- a/Lib/lib2to3/tests/test_parser.py +++ b/Lib/lib2to3/tests/test_parser.py @@ -274,6 +274,78 @@ def test_argument_unpacking_2(self): def test_argument_unpacking_3(self): self.validate("""f(2, *a, *b, **b, **c, **d)""") + def test_trailing_commas_1(self): + self.validate("def f(a, b): call(a, b)") + self.validate("def f(a, b,): call(a, b,)") + + def test_trailing_commas_2(self): + self.validate("def f(a, *b): call(a, *b)") + self.validate("def f(a, *b,): call(a, *b,)") + + def test_trailing_commas_3(self): + self.validate("def f(a, b=1): call(a, b=1)") + self.validate("def f(a, b=1,): call(a, b=1,)") + + def test_trailing_commas_4(self): + self.validate("def f(a, **b): call(a, **b)") + self.validate("def f(a, **b,): call(a, **b,)") + + def test_trailing_commas_5(self): + self.validate("def f(*a, b=1): call(*a, b=1)") + self.validate("def f(*a, b=1,): call(*a, b=1,)") + + def test_trailing_commas_6(self): + self.validate("def f(*a, **b): call(*a, **b)") + self.validate("def f(*a, **b,): call(*a, **b,)") + + def test_trailing_commas_7(self): + self.validate("def f(*, b=1): call(*b)") + self.validate("def f(*, b=1,): call(*b,)") + + def test_trailing_commas_8(self): + self.validate("def f(a=1, b=2): call(a=1, b=2)") + self.validate("def f(a=1, b=2,): call(a=1, b=2,)") + + def test_trailing_commas_9(self): + self.validate("def f(a=1, **b): call(a=1, **b)") + self.validate("def f(a=1, **b,): call(a=1, **b,)") + + def test_trailing_commas_lambda_1(self): + self.validate("f = lambda a, b: call(a, b)") + self.validate("f = lambda a, b,: call(a, b,)") + + def test_trailing_commas_lambda_2(self): + self.validate("f = lambda a, *b: call(a, *b)") + self.validate("f = lambda a, *b,: call(a, *b,)") + + def test_trailing_commas_lambda_3(self): + self.validate("f = lambda a, b=1: call(a, b=1)") + self.validate("f = lambda a, b=1,: call(a, b=1,)") + + def test_trailing_commas_lambda_4(self): + self.validate("f = lambda a, **b: call(a, **b)") + self.validate("f = lambda a, **b,: call(a, **b,)") + + def test_trailing_commas_lambda_5(self): + self.validate("f = lambda *a, b=1: call(*a, b=1)") + self.validate("f = lambda *a, b=1,: call(*a, b=1,)") + + def test_trailing_commas_lambda_6(self): + self.validate("f = lambda *a, **b: call(*a, **b)") + self.validate("f = lambda *a, **b,: call(*a, **b,)") + + def test_trailing_commas_lambda_7(self): + self.validate("f = lambda *, b=1: call(*b)") + self.validate("f = lambda *, b=1,: call(*b,)") + + def test_trailing_commas_lambda_8(self): + self.validate("f = lambda a=1, b=2: call(a=1, b=2)") + self.validate("f = lambda a=1, b=2,: call(a=1, b=2,)") + + def test_trailing_commas_lambda_9(self): + self.validate("f = lambda a=1, **b: call(a=1, **b)") + self.validate("f = lambda a=1, **b,: call(a=1, **b,)") + # Adapted from Python 3's Lib/test/test_grammar.py:GrammarTests.testFuncdef class TestFunctionAnnotations(GrammarTest): @@ -335,6 +407,42 @@ def test_12(self): call(*args,)""" self.validate(s) + def test_13(self): + self.validate("def f(a: str, b: int) -> None: call(a, b)") + self.validate("def f(a: str, b: int,) -> None: call(a, b,)") + + def test_14(self): + self.validate("def f(a: str, *b: int) -> None: call(a, *b)") + self.validate("def f(a: str, *b: int,) -> None: call(a, *b,)") + + def test_15(self): + self.validate("def f(a: str, b: int=1) -> None: call(a, b=1)") + self.validate("def f(a: str, b: int=1,) -> None: call(a, b=1,)") + + def test_16(self): + self.validate("def f(a: str, **b: int) -> None: call(a, **b)") + self.validate("def f(a: str, **b: int,) -> None: call(a, **b,)") + + def test_17(self): + self.validate("def f(*a: str, b: int=1) -> None: call(*a, b=1)") + self.validate("def f(*a: str, b: int=1,) -> None: call(*a, b=1,)") + + def test_18(self): + self.validate("def f(*a: str, **b: int) -> None: call(*a, **b)") + self.validate("def f(*a: str, **b: int,) -> None: call(*a, **b,)") + + def test_19(self): + self.validate("def f(*, b: int=1) -> None: call(*b)") + self.validate("def f(*, b: int=1,) -> None: call(*b,)") + + def test_20(self): + self.validate("def f(a: str='', b: int=2) -> None: call(a=a, b=2)") + self.validate("def f(a: str='', b: int=2,) -> None: call(a=a, b=2,)") + + def test_21(self): + self.validate("def f(a: str='', **b: int) -> None: call(a=a, **b)") + self.validate("def f(a: str='', **b: int,) -> None: call(a=a, **b,)") + # Adapted from Python 3's Lib/test/test_grammar.py:GrammarTests.test_var_annot class TestVarAnnotations(GrammarTest): From webhook-mailer at python.org Tue Mar 13 14:20:41 2018 From: webhook-mailer at python.org (=?utf-8?q?=C5=81ukasz?= Langa) Date: Tue, 13 Mar 2018 18:20:41 -0000 Subject: [Python-checkins] lib2to3: Add more tests (GH-6101) (#6108) Message-ID: <mailman.70.1520965243.1871.python-checkins@python.org> https://github.com/python/cpython/commit/7d0528dd3f054b47d6689c1dd75ef267a72e9bb5 commit: 7d0528dd3f054b47d6689c1dd75ef267a72e9bb5 branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: ?ukasz Langa <lukasz at langa.pl> date: 2018-03-13T11:20:38-07:00 summary: lib2to3: Add more tests (GH-6101) (#6108) (cherry picked from commit 74f56878cdee18d485e4f8b485d55ce62e38f4c9) Co-authored-by: ?ukasz Langa <lukasz at langa.pl> files: M Lib/lib2to3/tests/test_parser.py diff --git a/Lib/lib2to3/tests/test_parser.py b/Lib/lib2to3/tests/test_parser.py index cec2f98e9665..6813c65a7953 100644 --- a/Lib/lib2to3/tests/test_parser.py +++ b/Lib/lib2to3/tests/test_parser.py @@ -274,6 +274,78 @@ def test_argument_unpacking_2(self): def test_argument_unpacking_3(self): self.validate("""f(2, *a, *b, **b, **c, **d)""") + def test_trailing_commas_1(self): + self.validate("def f(a, b): call(a, b)") + self.validate("def f(a, b,): call(a, b,)") + + def test_trailing_commas_2(self): + self.validate("def f(a, *b): call(a, *b)") + self.validate("def f(a, *b,): call(a, *b,)") + + def test_trailing_commas_3(self): + self.validate("def f(a, b=1): call(a, b=1)") + self.validate("def f(a, b=1,): call(a, b=1,)") + + def test_trailing_commas_4(self): + self.validate("def f(a, **b): call(a, **b)") + self.validate("def f(a, **b,): call(a, **b,)") + + def test_trailing_commas_5(self): + self.validate("def f(*a, b=1): call(*a, b=1)") + self.validate("def f(*a, b=1,): call(*a, b=1,)") + + def test_trailing_commas_6(self): + self.validate("def f(*a, **b): call(*a, **b)") + self.validate("def f(*a, **b,): call(*a, **b,)") + + def test_trailing_commas_7(self): + self.validate("def f(*, b=1): call(*b)") + self.validate("def f(*, b=1,): call(*b,)") + + def test_trailing_commas_8(self): + self.validate("def f(a=1, b=2): call(a=1, b=2)") + self.validate("def f(a=1, b=2,): call(a=1, b=2,)") + + def test_trailing_commas_9(self): + self.validate("def f(a=1, **b): call(a=1, **b)") + self.validate("def f(a=1, **b,): call(a=1, **b,)") + + def test_trailing_commas_lambda_1(self): + self.validate("f = lambda a, b: call(a, b)") + self.validate("f = lambda a, b,: call(a, b,)") + + def test_trailing_commas_lambda_2(self): + self.validate("f = lambda a, *b: call(a, *b)") + self.validate("f = lambda a, *b,: call(a, *b,)") + + def test_trailing_commas_lambda_3(self): + self.validate("f = lambda a, b=1: call(a, b=1)") + self.validate("f = lambda a, b=1,: call(a, b=1,)") + + def test_trailing_commas_lambda_4(self): + self.validate("f = lambda a, **b: call(a, **b)") + self.validate("f = lambda a, **b,: call(a, **b,)") + + def test_trailing_commas_lambda_5(self): + self.validate("f = lambda *a, b=1: call(*a, b=1)") + self.validate("f = lambda *a, b=1,: call(*a, b=1,)") + + def test_trailing_commas_lambda_6(self): + self.validate("f = lambda *a, **b: call(*a, **b)") + self.validate("f = lambda *a, **b,: call(*a, **b,)") + + def test_trailing_commas_lambda_7(self): + self.validate("f = lambda *, b=1: call(*b)") + self.validate("f = lambda *, b=1,: call(*b,)") + + def test_trailing_commas_lambda_8(self): + self.validate("f = lambda a=1, b=2: call(a=1, b=2)") + self.validate("f = lambda a=1, b=2,: call(a=1, b=2,)") + + def test_trailing_commas_lambda_9(self): + self.validate("f = lambda a=1, **b: call(a=1, **b)") + self.validate("f = lambda a=1, **b,: call(a=1, **b,)") + # Adapted from Python 3's Lib/test/test_grammar.py:GrammarTests.testFuncdef class TestFunctionAnnotations(GrammarTest): @@ -335,6 +407,42 @@ def test_12(self): call(*args,)""" self.validate(s) + def test_13(self): + self.validate("def f(a: str, b: int) -> None: call(a, b)") + self.validate("def f(a: str, b: int,) -> None: call(a, b,)") + + def test_14(self): + self.validate("def f(a: str, *b: int) -> None: call(a, *b)") + self.validate("def f(a: str, *b: int,) -> None: call(a, *b,)") + + def test_15(self): + self.validate("def f(a: str, b: int=1) -> None: call(a, b=1)") + self.validate("def f(a: str, b: int=1,) -> None: call(a, b=1,)") + + def test_16(self): + self.validate("def f(a: str, **b: int) -> None: call(a, **b)") + self.validate("def f(a: str, **b: int,) -> None: call(a, **b,)") + + def test_17(self): + self.validate("def f(*a: str, b: int=1) -> None: call(*a, b=1)") + self.validate("def f(*a: str, b: int=1,) -> None: call(*a, b=1,)") + + def test_18(self): + self.validate("def f(*a: str, **b: int) -> None: call(*a, **b)") + self.validate("def f(*a: str, **b: int,) -> None: call(*a, **b,)") + + def test_19(self): + self.validate("def f(*, b: int=1) -> None: call(*b)") + self.validate("def f(*, b: int=1,) -> None: call(*b,)") + + def test_20(self): + self.validate("def f(a: str='', b: int=2) -> None: call(a=a, b=2)") + self.validate("def f(a: str='', b: int=2,) -> None: call(a=a, b=2,)") + + def test_21(self): + self.validate("def f(a: str='', **b: int) -> None: call(a=a, **b)") + self.validate("def f(a: str='', **b: int,) -> None: call(a=a, **b,)") + # Adapted from Python 3's Lib/test/test_grammar.py:GrammarTests.test_var_annot class TestVarAnnotations(GrammarTest): From webhook-mailer at python.org Tue Mar 13 14:47:42 2018 From: webhook-mailer at python.org (=?utf-8?q?=C5=81ukasz?= Langa) Date: Tue, 13 Mar 2018 18:47:42 -0000 Subject: [Python-checkins] lib2to3: Add more tests (GH-6101) (#6109) Message-ID: <mailman.71.1520966863.1871.python-checkins@python.org> https://github.com/python/cpython/commit/cad3eb2026e5bf1a5001e1d21b8c4e33d5f89c4f commit: cad3eb2026e5bf1a5001e1d21b8c4e33d5f89c4f branch: 3.6 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: ?ukasz Langa <lukasz at langa.pl> date: 2018-03-13T11:47:32-07:00 summary: lib2to3: Add more tests (GH-6101) (#6109) (cherry picked from commit 74f56878cdee18d485e4f8b485d55ce62e38f4c9) Co-authored-by: ?ukasz Langa <lukasz at langa.pl> files: M Lib/lib2to3/tests/test_parser.py diff --git a/Lib/lib2to3/tests/test_parser.py b/Lib/lib2to3/tests/test_parser.py index c4d062ff827e..1c9a17e43abc 100644 --- a/Lib/lib2to3/tests/test_parser.py +++ b/Lib/lib2to3/tests/test_parser.py @@ -274,6 +274,78 @@ def test_argument_unpacking_2(self): def test_argument_unpacking_3(self): self.validate("""f(2, *a, *b, **b, **c, **d)""") + def test_trailing_commas_1(self): + self.validate("def f(a, b): call(a, b)") + self.validate("def f(a, b,): call(a, b,)") + + def test_trailing_commas_2(self): + self.validate("def f(a, *b): call(a, *b)") + self.validate("def f(a, *b,): call(a, *b,)") + + def test_trailing_commas_3(self): + self.validate("def f(a, b=1): call(a, b=1)") + self.validate("def f(a, b=1,): call(a, b=1,)") + + def test_trailing_commas_4(self): + self.validate("def f(a, **b): call(a, **b)") + self.validate("def f(a, **b,): call(a, **b,)") + + def test_trailing_commas_5(self): + self.validate("def f(*a, b=1): call(*a, b=1)") + self.validate("def f(*a, b=1,): call(*a, b=1,)") + + def test_trailing_commas_6(self): + self.validate("def f(*a, **b): call(*a, **b)") + self.validate("def f(*a, **b,): call(*a, **b,)") + + def test_trailing_commas_7(self): + self.validate("def f(*, b=1): call(*b)") + self.validate("def f(*, b=1,): call(*b,)") + + def test_trailing_commas_8(self): + self.validate("def f(a=1, b=2): call(a=1, b=2)") + self.validate("def f(a=1, b=2,): call(a=1, b=2,)") + + def test_trailing_commas_9(self): + self.validate("def f(a=1, **b): call(a=1, **b)") + self.validate("def f(a=1, **b,): call(a=1, **b,)") + + def test_trailing_commas_lambda_1(self): + self.validate("f = lambda a, b: call(a, b)") + self.validate("f = lambda a, b,: call(a, b,)") + + def test_trailing_commas_lambda_2(self): + self.validate("f = lambda a, *b: call(a, *b)") + self.validate("f = lambda a, *b,: call(a, *b,)") + + def test_trailing_commas_lambda_3(self): + self.validate("f = lambda a, b=1: call(a, b=1)") + self.validate("f = lambda a, b=1,: call(a, b=1,)") + + def test_trailing_commas_lambda_4(self): + self.validate("f = lambda a, **b: call(a, **b)") + self.validate("f = lambda a, **b,: call(a, **b,)") + + def test_trailing_commas_lambda_5(self): + self.validate("f = lambda *a, b=1: call(*a, b=1)") + self.validate("f = lambda *a, b=1,: call(*a, b=1,)") + + def test_trailing_commas_lambda_6(self): + self.validate("f = lambda *a, **b: call(*a, **b)") + self.validate("f = lambda *a, **b,: call(*a, **b,)") + + def test_trailing_commas_lambda_7(self): + self.validate("f = lambda *, b=1: call(*b)") + self.validate("f = lambda *, b=1,: call(*b,)") + + def test_trailing_commas_lambda_8(self): + self.validate("f = lambda a=1, b=2: call(a=1, b=2)") + self.validate("f = lambda a=1, b=2,: call(a=1, b=2,)") + + def test_trailing_commas_lambda_9(self): + self.validate("f = lambda a=1, **b: call(a=1, **b)") + self.validate("f = lambda a=1, **b,: call(a=1, **b,)") + # Adapted from Python 3's Lib/test/test_grammar.py:GrammarTests.testFuncdef class TestFunctionAnnotations(GrammarTest): @@ -335,6 +407,42 @@ def test_12(self): call(*args,)""" self.validate(s) + def test_13(self): + self.validate("def f(a: str, b: int) -> None: call(a, b)") + self.validate("def f(a: str, b: int,) -> None: call(a, b,)") + + def test_14(self): + self.validate("def f(a: str, *b: int) -> None: call(a, *b)") + self.validate("def f(a: str, *b: int,) -> None: call(a, *b,)") + + def test_15(self): + self.validate("def f(a: str, b: int=1) -> None: call(a, b=1)") + self.validate("def f(a: str, b: int=1,) -> None: call(a, b=1,)") + + def test_16(self): + self.validate("def f(a: str, **b: int) -> None: call(a, **b)") + self.validate("def f(a: str, **b: int,) -> None: call(a, **b,)") + + def test_17(self): + self.validate("def f(*a: str, b: int=1) -> None: call(*a, b=1)") + self.validate("def f(*a: str, b: int=1,) -> None: call(*a, b=1,)") + + def test_18(self): + self.validate("def f(*a: str, **b: int) -> None: call(*a, **b)") + self.validate("def f(*a: str, **b: int,) -> None: call(*a, **b,)") + + def test_19(self): + self.validate("def f(*, b: int=1) -> None: call(*b)") + self.validate("def f(*, b: int=1,) -> None: call(*b,)") + + def test_20(self): + self.validate("def f(a: str='', b: int=2) -> None: call(a=a, b=2)") + self.validate("def f(a: str='', b: int=2,) -> None: call(a=a, b=2,)") + + def test_21(self): + self.validate("def f(a: str='', **b: int) -> None: call(a=a, **b)") + self.validate("def f(a: str='', **b: int,) -> None: call(a=a, **b,)") + # Adapted from Python 3's Lib/test/test_grammar.py:GrammarTests.test_var_annot class TestVarAnnotations(GrammarTest): From webhook-mailer at python.org Tue Mar 13 17:06:17 2018 From: webhook-mailer at python.org (Serhiy Storchaka) Date: Tue, 13 Mar 2018 21:06:17 -0000 Subject: [Python-checkins] [2.7] bpo-17288: Prevent jumps from 'return' and 'exception' trace events. (GH-6111) Message-ID: <mailman.72.1520975179.1871.python-checkins@python.org> https://github.com/python/cpython/commit/baca85fcc7cdf70af4a76ea0966d4842c173de1a commit: baca85fcc7cdf70af4a76ea0966d4842c173de1a branch: 2.7 author: xdegaye <xdegaye at gmail.com> committer: Serhiy Storchaka <storchaka at gmail.com> date: 2018-03-13T23:06:14+02:00 summary: [2.7] bpo-17288: Prevent jumps from 'return' and 'exception' trace events. (GH-6111) (cherry picked from commit e32bbaf376a09c149fa7c7f2919d7c9ce4e2a055) files: A Misc/NEWS.d/next/Core and Builtins/2018-02-27-13-36-21.bpo-17288.Gdj24S.rst M Lib/test/test_sys_settrace.py M Objects/frameobject.c diff --git a/Lib/test/test_sys_settrace.py b/Lib/test/test_sys_settrace.py index 04b1ac339a49..d8737af3bc8d 100644 --- a/Lib/test/test_sys_settrace.py +++ b/Lib/test/test_sys_settrace.py @@ -482,20 +482,35 @@ def g(frame, why, extra): class JumpTracer: """Defines a trace function that jumps from one place to another.""" - def __init__(self, function, jumpFrom, jumpTo): - self.function = function + def __init__(self, function, jumpFrom, jumpTo, event='line', + decorated=False): + self.code = function.func_code self.jumpFrom = jumpFrom self.jumpTo = jumpTo + self.event = event + self.firstLine = None if decorated else self.code.co_firstlineno self.done = False def trace(self, frame, event, arg): - if not self.done and frame.f_code == self.function.func_code: - firstLine = frame.f_code.co_firstlineno - if event == 'line' and frame.f_lineno == firstLine + self.jumpFrom: + if self.done: + return + # frame.f_code.co_firstlineno is the first line of the decorator when + # 'function' is decorated and the decorator may be written using + # multiple physical lines when it is too long. Use the first line + # trace event in 'function' to find the first line of 'function'. + if (self.firstLine is None and frame.f_code == self.code and + event == 'line'): + self.firstLine = frame.f_lineno - 1 + if (event == self.event and self.firstLine and + frame.f_lineno == self.firstLine + self.jumpFrom): + f = frame + while f is not None and f.f_code != self.code: + f = f.f_back + if f is not None: # Cope with non-integer self.jumpTo (because of # no_jump_to_non_integers below). try: - frame.f_lineno = firstLine + self.jumpTo + frame.f_lineno = self.firstLine + self.jumpTo except TypeError: frame.f_lineno = self.jumpTo self.done = True @@ -535,8 +550,9 @@ def compare_jump_output(self, expected, received): "Expected: " + repr(expected) + "\n" + "Received: " + repr(received)) - def run_test(self, func, jumpFrom, jumpTo, expected, error=None): - tracer = JumpTracer(func, jumpFrom, jumpTo) + def run_test(self, func, jumpFrom, jumpTo, expected, error=None, + event='line', decorated=False): + tracer = JumpTracer(func, jumpFrom, jumpTo, event, decorated) sys.settrace(tracer.trace) output = [] if error is None: @@ -547,15 +563,15 @@ def run_test(self, func, jumpFrom, jumpTo, expected, error=None): sys.settrace(None) self.compare_jump_output(expected, output) - def jump_test(jumpFrom, jumpTo, expected, error=None): + def jump_test(jumpFrom, jumpTo, expected, error=None, event='line'): """Decorator that creates a test that makes a jump from one place to another in the following code. """ def decorator(func): @wraps(func) def test(self): - # +1 to compensate a decorator line - self.run_test(func, jumpFrom+1, jumpTo+1, expected, error) + self.run_test(func, jumpFrom, jumpTo, expected, + error=error, event=event, decorated=True) return test return decorator @@ -1018,6 +1034,36 @@ class fake_function: sys.settrace(None) self.compare_jump_output([2, 3, 2, 3, 4], namespace["output"]) + @jump_test(2, 3, [1], event='call', error=(ValueError, "can't jump from" + " the 'call' trace event of a new frame")) + def test_no_jump_from_call(output): + output.append(1) + def nested(): + output.append(3) + nested() + output.append(5) + + @jump_test(2, 1, [1], event='return', error=(ValueError, + "can only jump from a 'line' trace event")) + def test_no_jump_from_return_event(output): + output.append(1) + return + + @jump_test(2, 1, [1], event='exception', error=(ValueError, + "can only jump from a 'line' trace event")) + def test_no_jump_from_exception_event(output): + output.append(1) + 1 / 0 + + @jump_test(3, 2, [2], event='return', error=(ValueError, + "can't jump from a yield statement")) + def test_no_jump_from_yield(output): + def gen(): + output.append(2) + yield 3 + next(gen()) + output.append(5) + def test_main(): test_support.run_unittest( diff --git a/Misc/NEWS.d/next/Core and Builtins/2018-02-27-13-36-21.bpo-17288.Gdj24S.rst b/Misc/NEWS.d/next/Core and Builtins/2018-02-27-13-36-21.bpo-17288.Gdj24S.rst new file mode 100644 index 000000000000..ce9e84c40313 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2018-02-27-13-36-21.bpo-17288.Gdj24S.rst @@ -0,0 +1 @@ +Prevent jumps from 'return' and 'exception' trace events. diff --git a/Objects/frameobject.c b/Objects/frameobject.c index bf1c7c52737d..175874503bb2 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -89,6 +89,9 @@ frame_getlineno(PyFrameObject *f, void *closure) * o 'try'/'for'/'while' blocks can't be jumped into because the blockstack * needs to be set up before their code runs, and for 'for' loops the * iterator needs to be on the stack. + * o Jumps cannot be made from within a trace function invoked with a + * 'return' or 'exception' event since the eval loop has been exited at + * that time. */ static int frame_setlineno(PyFrameObject *f, PyObject* p_new_lineno) @@ -122,13 +125,32 @@ frame_setlineno(PyFrameObject *f, PyObject* p_new_lineno) return -1; } + /* Upon the 'call' trace event of a new frame, f->f_lasti is -1 and + * f->f_trace is NULL, check first on the first condition. + * Forbidding jumps from the 'call' event of a new frame is a side effect + * of allowing to set f_lineno only from trace functions. */ + if (f->f_lasti == -1) { + PyErr_Format(PyExc_ValueError, + "can't jump from the 'call' trace event of a new frame"); + return -1; + } + /* You can only do this from within a trace function, not via * _getframe or similar hackery. */ - if (!f->f_trace) - { + if (!f->f_trace) { PyErr_Format(PyExc_ValueError, - "f_lineno can only be set by a" - " line trace function"); + "f_lineno can only be set by a trace function"); + return -1; + } + + /* Forbid jumps upon a 'return' trace event (except after executing a + * YIELD_VALUE opcode, f_stacktop is not NULL in that case) and upon an + * 'exception' trace event. + * Jumps from 'call' trace events have already been forbidden above for new + * frames, so this check does not change anything for 'call' events. */ + if (f->f_stacktop == NULL) { + PyErr_SetString(PyExc_ValueError, + "can only jump from a 'line' trace event"); return -1; } @@ -178,6 +200,15 @@ frame_setlineno(PyFrameObject *f, PyObject* p_new_lineno) min_addr = MIN(new_lasti, f->f_lasti); max_addr = MAX(new_lasti, f->f_lasti); + /* The trace function is called with a 'return' trace event after the + * execution of a yield statement. */ + assert(f->f_lasti != -1); + if (code[f->f_lasti] == YIELD_VALUE) { + PyErr_SetString(PyExc_ValueError, + "can't jump from a yield statement"); + return -1; + } + /* You can't jump onto a line with an 'except' statement on it - * they expect to have an exception on the top of the stack, which * won't be true if you jump to them. They always start with code From webhook-mailer at python.org Tue Mar 13 23:40:28 2018 From: webhook-mailer at python.org (Ned Deily) Date: Wed, 14 Mar 2018 03:40:28 -0000 Subject: [Python-checkins] bpo-32726: macOS installer changes for 3.6.5 Message-ID: <mailman.73.1520998829.1871.python-checkins@python.org> https://github.com/python/cpython/commit/94552448d7bcc1eebc53b608e89d96e235054f2f commit: 94552448d7bcc1eebc53b608e89d96e235054f2f branch: 3.6 author: Ned Deily <nad at python.org> committer: Ned Deily <nad at python.org> date: 2018-03-13T21:01:08-04:00 summary: bpo-32726: macOS installer changes for 3.6.5 Backport the new 10.9+ installer variant from 3.7. This variant features more modern options; like 64-bit only (Apple is deprecating 32-bit support in future macOS releases); a built-in version of Tcl/Tk 8.6.8; built with clang rather than gcc-4.2. For 3.6.5, the 10.9+ variant will be offered as an additional alternative to the traditional 10.6+ variant in earlier 3.6.x releases. Binary extension modules (including wheels) built for earlier versions of 3.6.x with the 10.6 variant should continue to work with either 3.6.5 variant without recompilation. In addition, both installer variants have updated 3rd-party libraries: OpenSSL 1.0.2m -> 1.0.2n XZ 5.2.2 -> 5.2.3 SQLite 3.21.0 -> 3.22.0 Also the 10.6 variant now sets CC=gcc instead of CC=gcc-4.2 and does not search for the outdated 10.6 SDK. The variant is built with the same compiler as before. As before, for extension module builds, the CC can be overridden with the CC env variable and an SDK can be specified with the SDKROOT env variable (see man xcrun). These minor changes should be transparent to nearly all users. files: A Mac/BuildScript/resources/Conclusion.rtf A Mac/BuildScript/tk868_on_10_8_10_9.patch A Misc/NEWS.d/next/macOS/2018-03-13-21-00-20.bpo-32726.Mticyn.rst D Mac/BuildScript/issue19373_tk_8_5_15_source.patch D Mac/BuildScript/openssl_sdk_makedepend.patch M Mac/BuildScript/build-installer.py M Mac/BuildScript/resources/ReadMe.rtf M Mac/BuildScript/resources/Welcome.rtf M Mac/BuildScript/scripts/postflight.documentation M configure M configure.ac diff --git a/Mac/BuildScript/build-installer.py b/Mac/BuildScript/build-installer.py index ff02fb37126b..c33737e9d042 100755 --- a/Mac/BuildScript/build-installer.py +++ b/Mac/BuildScript/build-installer.py @@ -1,30 +1,36 @@ #!/usr/bin/env python """ -This script is used to build "official" universal installers on Mac OS X. -It requires at least Mac OS X 10.5, Xcode 3, and the 10.4u SDK for -32-bit builds. 64-bit or four-way universal builds require at least -OS X 10.5 and the 10.5 SDK. +This script is used to build "official" universal installers on macOS. + +NEW for 3.6.5: +- support Intel 64-bit-only () and 32-bit-only installer builds +- build and link with private Tcl/Tk 8.6 for 10.9+ builds +- deprecate use of explicit SDK (--sdk-path=) since all but the oldest + versions of Xcode support implicit setting of an SDK via environment + variables (SDKROOT and friends, see the xcrun man page for more info). + The SDK stuff was primarily needed for building universal installers + for 10.4; so as of 3.6.5, building installers for 10.4 is no longer + supported with build-installer. +- use generic "gcc" as compiler (CC env var) rather than "gcc-4.2" Please ensure that this script keeps working with Python 2.5, to avoid -bootstrap issues (/usr/bin/python is Python 2.5 on OSX 10.5). Sphinx, -which is used to build the documentation, currently requires at least -Python 2.4. However, as of Python 3.4.1, Doc builds require an external -sphinx-build and the current versions of Sphinx now require at least -Python 2.6. - -In addition to what is supplied with OS X 10.5+ and Xcode 3+, the script -requires an installed third-party version of -Tcl/Tk 8.4 (for OS X 10.4 and 10.5 deployment targets) or Tcl/TK 8.5 +bootstrap issues (/usr/bin/python is Python 2.5 on OSX 10.5). Doc builds +use current versions of Sphinx and require a reasonably current python3. +Sphinx and dependencies are installed into a venv using the python3's pip +so will fetch them from PyPI if necessary. Since python3 is now used for +Sphinx, build-installer.py should also be converted to use python3! + +For 10.9 or greater deployment targets, build-installer builds and links +with its own copy of Tcl/Tk 8.5 and the rest of this paragraph does not +apply. Otherwise, build-installer requires an installed third-party version +of Tcl/Tk 8.4 (for OS X 10.4 and 10.5 deployment targets) or Tcl/TK 8.5 (for 10.6 or later) installed in /Library/Frameworks. When installed, the Python built by this script will attempt to dynamically link first to Tcl and Tk frameworks in /Library/Frameworks if available otherwise fall back to the ones in /System/Library/Framework. For the build, we recommend -installing the most recent ActiveTcl 8.4 or 8.5 version. - -32-bit-only installer builds are still possible on OS X 10.4 with Xcode 2.5 -and the installation of additional components, such as a newer Python -(2.5 is needed for Python parser updates) and for the documentation -build either svn (pre-3.4.1) or sphinx-build (3.4.1 and later). +installing the most recent ActiveTcl 8.5 or 8.4 version, depending +on the deployment target. The actual version linked to depends on the +path of /Library/Frameworks/{Tcl,Tk}.framework/Versions/Current. Usage: see USAGE variable in the script. """ @@ -111,32 +117,19 @@ def getFullVersion(): DEPSRC = os.path.join(WORKDIR, 'third-party') DEPSRC = os.path.expanduser('~/Universal/other-sources') -# Location of the preferred SDK - -### There are some issues with the SDK selection below here, -### The resulting binary doesn't work on all platforms that -### it should. Always default to the 10.4u SDK until that -### issue is resolved. -### -##if int(os.uname()[2].split('.')[0]) == 8: -## # Explicitly use the 10.4u (universal) SDK when -## # building on 10.4, the system headers are not -## # useable for a universal build -## SDKPATH = "/Developer/SDKs/MacOSX10.4u.sdk" -##else: -## SDKPATH = "/" - -SDKPATH = "/Developer/SDKs/MacOSX10.4u.sdk" - universal_opts_map = { '32-bit': ('i386', 'ppc',), '64-bit': ('x86_64', 'ppc64',), 'intel': ('i386', 'x86_64'), + 'intel-32': ('i386',), + 'intel-64': ('x86_64',), '3-way': ('ppc', 'i386', 'x86_64'), 'all': ('i386', 'ppc', 'x86_64', 'ppc64',) } default_target_map = { '64-bit': '10.5', '3-way': '10.5', 'intel': '10.5', + 'intel-32': '10.4', + 'intel-64': '10.5', 'all': '10.5', } @@ -154,19 +147,18 @@ def getFullVersion(): )))) # $MACOSX_DEPLOYMENT_TARGET -> minimum OS X level -DEPTARGET = '10.3' +DEPTARGET = '10.5' def getDeptargetTuple(): return tuple([int(n) for n in DEPTARGET.split('.')[0:2]]) def getTargetCompilers(): target_cc_map = { - '10.3': ('gcc-4.0', 'g++-4.0'), '10.4': ('gcc-4.0', 'g++-4.0'), - '10.5': ('gcc-4.2', 'g++-4.2'), - '10.6': ('gcc-4.2', 'g++-4.2'), + '10.5': ('gcc', 'g++'), + '10.6': ('gcc', 'g++'), } - return target_cc_map.get(DEPTARGET, ('clang', 'clang++') ) + return target_cc_map.get(DEPTARGET, ('gcc', 'g++') ) CC, CXX = getTargetCompilers() @@ -180,9 +172,9 @@ def getTargetCompilers(): -b DIR --build-dir=DIR: Create build here (default: %(WORKDIR)r) --third-party=DIR: Store third-party sources here (default: %(DEPSRC)r) - --sdk-path=DIR: Location of the SDK (default: %(SDKPATH)r) + --sdk-path=DIR: Location of the SDK (deprecated, use SDKROOT env variable) --src-dir=DIR: Location of the Python sources (default: %(SRCDIR)r) - --dep-target=10.n OS X deployment target (default: %(DEPTARGET)r) + --dep-target=10.n macOS deployment target (default: %(DEPTARGET)r) --universal-archs=x universal architectures (options: %(UNIVERSALOPTS)r, default: %(UNIVERSALARCHS)r) """)% globals() @@ -194,6 +186,11 @@ def getTargetCompilers(): # '/Library/Frameworks/Tk.framework/Versions/8.5/Tk'] EXPECTED_SHARED_LIBS = {} +# Are we building and linking with our own copy of Tcl/TK? +# For now, do so if deployment target is 10.9+. +def internalTk(): + return getDeptargetTuple() >= (10, 9) + # List of names of third party software built with this installer. # The names will be inserted into the rtf version of the License. THIRD_PARTY_LIBS = [] @@ -213,25 +210,21 @@ def library_recipes(): result.extend([ dict( - name="OpenSSL 1.0.2m", - url="https://www.openssl.org/source/openssl-1.0.2m.tar.gz", - checksum='10e9e37f492094b9ef296f68f24a7666', - patches=[ - "openssl_sdk_makedepend.patch", - ], + name="OpenSSL 1.0.2n", + url="https://www.openssl.org/source/openssl-1.0.2n.tar.gz", + checksum='13bdc1b1d1ff39b6fd42a255e74676a4', buildrecipe=build_universal_openssl, configure=None, install=None, ), ]) -# Disable for now - if False: # if getDeptargetTuple() > (10, 5): + if internalTk(): result.extend([ dict( - name="Tcl 8.5.15", - url="ftp://ftp.tcl.tk/pub/tcl//tcl8_5/tcl8.5.15-src.tar.gz", - checksum='f3df162f92c69b254079c4d0af7a690f', + name="Tcl 8.6.8", + url="ftp://ftp.tcl.tk/pub/tcl//tcl8_6/tcl8.6.8-src.tar.gz", + checksum='81656d3367af032e0ae6157eff134f89', buildDir="unix", configure_pre=[ '--enable-shared', @@ -241,15 +234,15 @@ def library_recipes(): useLDFlags=False, install='make TCL_LIBRARY=%(TCL_LIBRARY)s && make install TCL_LIBRARY=%(TCL_LIBRARY)s DESTDIR=%(DESTDIR)s'%{ "DESTDIR": shellQuote(os.path.join(WORKDIR, 'libraries')), - "TCL_LIBRARY": shellQuote('/Library/Frameworks/Python.framework/Versions/%s/lib/tcl8.5'%(getVersion())), + "TCL_LIBRARY": shellQuote('/Library/Frameworks/Python.framework/Versions/%s/lib/tcl8.6'%(getVersion())), }, ), dict( - name="Tk 8.5.15", - url="ftp://ftp.tcl.tk/pub/tcl//tcl8_5/tk8.5.15-src.tar.gz", - checksum='55b8e33f903210a4e1c8bce0f820657f', + name="Tk 8.6.8", + url="ftp://ftp.tcl.tk/pub/tcl//tcl8_6/tk8.6.8-src.tar.gz", + checksum='5e0faecba458ee1386078fb228d008ba', patches=[ - "issue19373_tk_8_5_15_source.patch", + "tk868_on_10_8_10_9.patch", ], buildDir="unix", configure_pre=[ @@ -261,8 +254,8 @@ def library_recipes(): useLDFlags=False, install='make TCL_LIBRARY=%(TCL_LIBRARY)s TK_LIBRARY=%(TK_LIBRARY)s && make install TCL_LIBRARY=%(TCL_LIBRARY)s TK_LIBRARY=%(TK_LIBRARY)s DESTDIR=%(DESTDIR)s'%{ "DESTDIR": shellQuote(os.path.join(WORKDIR, 'libraries')), - "TCL_LIBRARY": shellQuote('/Library/Frameworks/Python.framework/Versions/%s/lib/tcl8.5'%(getVersion())), - "TK_LIBRARY": shellQuote('/Library/Frameworks/Python.framework/Versions/%s/lib/tk8.5'%(getVersion())), + "TCL_LIBRARY": shellQuote('/Library/Frameworks/Python.framework/Versions/%s/lib/tcl8.6'%(getVersion())), + "TK_LIBRARY": shellQuote('/Library/Frameworks/Python.framework/Versions/%s/lib/tk8.6'%(getVersion())), }, ), ]) @@ -270,9 +263,9 @@ def library_recipes(): if PYTHON_3: result.extend([ dict( - name="XZ 5.2.2", - url="http://tukaani.org/xz/xz-5.2.2.tar.gz", - checksum='7cf6a8544a7dae8e8106fdf7addfa28c', + name="XZ 5.2.3", + url="http://tukaani.org/xz/xz-5.2.3.tar.gz", + checksum='ef68674fb47a8b8e741b34e429d86e9d', configure_pre=[ '--disable-dependency-tracking', ] @@ -315,13 +308,14 @@ def library_recipes(): ), ), dict( - name="SQLite 3.21.0", - url="https://www.sqlite.org/2017/sqlite-autoconf-3210000.tar.gz", - checksum='7913de4c3126ba3c24689cb7a199ea31', + name="SQLite 3.22.0", + url="https://www.sqlite.org/2018/sqlite-autoconf-3220000.tar.gz", + checksum='96b5648d542e8afa6ab7ffb8db8ddc3d', extra_cflags=('-Os ' '-DSQLITE_ENABLE_FTS5 ' '-DSQLITE_ENABLE_FTS4 ' '-DSQLITE_ENABLE_FTS3_PARENTHESIS ' + '-DSQLITE_ENABLE_JSON1 ' '-DSQLITE_ENABLE_RTREE ' '-DSQLITE_TCL=0 ' '%s' % ('','-DSQLITE_WITHOUT_ZONEMALLOC ')[LT_10_5]), @@ -342,11 +336,10 @@ def library_recipes(): url="http://bzip.org/1.0.6/bzip2-1.0.6.tar.gz", checksum='00b516f4704d4a7cb50a1d97e6e8e15b', configure=None, - install='make install CC=%s CXX=%s, PREFIX=%s/usr/local/ CFLAGS="-arch %s -isysroot %s"'%( + install='make install CC=%s CXX=%s, PREFIX=%s/usr/local/ CFLAGS="-arch %s"'%( CC, CXX, shellQuote(os.path.join(WORKDIR, 'libraries')), ' -arch '.join(ARCHLIST), - SDKPATH, ), ), dict( @@ -354,11 +347,10 @@ def library_recipes(): url="http://www.gzip.org/zlib/zlib-1.2.3.tar.gz", checksum='debc62758716a169df9f62e6ab2bc634', configure=None, - install='make install CC=%s CXX=%s, prefix=%s/usr/local/ CFLAGS="-arch %s -isysroot %s"'%( + install='make install CC=%s CXX=%s, prefix=%s/usr/local/ CFLAGS="-arch %s"'%( CC, CXX, shellQuote(os.path.join(WORKDIR, 'libraries')), ' -arch '.join(ARCHLIST), - SDKPATH, ), ), dict( @@ -405,8 +397,7 @@ def pkg_recipes(): source="/Library/Frameworks/Python.framework", readme="""\ This package installs Python.framework, that is the python - interpreter and the standard library. This also includes Python - wrappers for lots of Mac OS X API's. + interpreter and the standard library. """, postflight="scripts/postflight.framework", selected='selected', @@ -483,24 +474,6 @@ def pkg_recipes(): ), ] - if getDeptargetTuple() < (10, 4) and not PYTHON_3: - result.append( - dict( - name="PythonSystemFixes", - long_name="Fix system Python", - readme="""\ - This package updates the system python installation on - Mac OS X 10.3 to ensure that you can build new python extensions - using that copy of python after installing this version. - """, - postflight="../Tools/fixapplepython23.py", - topdir="/Library/Frameworks/Python.framework", - source="/empty-dir", - required=False, - selected=unselected_for_python3, - ) - ) - return result def fatal(msg): @@ -565,55 +538,54 @@ def checkEnvironment(): Check that we're running on a supported system. """ - if sys.version_info[0:2] < (2, 4): - fatal("This script must be run with Python 2.4 or later") + if sys.version_info[0:2] < (2, 5): + fatal("This script must be run with Python 2.5 (or later)") if platform.system() != 'Darwin': - fatal("This script should be run on a Mac OS X 10.4 (or later) system") + fatal("This script should be run on a macOS 10.5 (or later) system") if int(platform.release().split('.')[0]) < 8: - fatal("This script should be run on a Mac OS X 10.4 (or later) system") - - if not os.path.exists(SDKPATH): - fatal("Please install the latest version of Xcode and the %s SDK"%( - os.path.basename(SDKPATH[:-4]))) + fatal("This script should be run on a macOS 10.5 (or later) system") # Because we only support dynamic load of only one major/minor version of + # Tcl/Tk, if we are not using building and using our own private copy of # Tcl/Tk, ensure: - # 1. there are no user-installed frameworks of Tcl/Tk with version - # higher than the Apple-supplied system version in - # SDKROOT/System/Library/Frameworks - # 2. there is a user-installed framework (usually ActiveTcl) in (or linked - # in) SDKROOT/Library/Frameworks with the same version as the system - # version. This allows users to choose to install a newer patch level. - - frameworks = {} - for framework in ['Tcl', 'Tk']: - fwpth = 'Library/Frameworks/%s.framework/Versions/Current' % framework - sysfw = os.path.join(SDKPATH, 'System', fwpth) - libfw = os.path.join(SDKPATH, fwpth) - usrfw = os.path.join(os.getenv('HOME'), fwpth) - frameworks[framework] = os.readlink(sysfw) - if not os.path.exists(libfw): - fatal("Please install a link to a current %s %s as %s so " - "the user can override the system framework." - % (framework, frameworks[framework], libfw)) - if os.readlink(libfw) != os.readlink(sysfw): - fatal("Version of %s must match %s" % (libfw, sysfw) ) - if os.path.exists(usrfw): - fatal("Please rename %s to avoid possible dynamic load issues." - % usrfw) - - if frameworks['Tcl'] != frameworks['Tk']: - fatal("The Tcl and Tk frameworks are not the same version.") - - # add files to check after build - EXPECTED_SHARED_LIBS['_tkinter.so'] = [ - "/Library/Frameworks/Tcl.framework/Versions/%s/Tcl" - % frameworks['Tcl'], - "/Library/Frameworks/Tk.framework/Versions/%s/Tk" - % frameworks['Tk'], - ] + # 1. there is a user-installed framework (usually ActiveTcl) in (or linked + # in) SDKROOT/Library/Frameworks. As of Python 3.6.5, we no longer + # enforce that the version of the user-installed framework also + # exists in the system-supplied Tcl/Tk frameworks. Time to support + # Tcl/Tk 8.6 even if Apple does not. + if not internalTk(): + frameworks = {} + for framework in ['Tcl', 'Tk']: + fwpth = 'Library/Frameworks/%s.framework/Versions/Current' % framework + libfw = os.path.join('/', fwpth) + usrfw = os.path.join(os.getenv('HOME'), fwpth) + frameworks[framework] = os.readlink(libfw) + if not os.path.exists(libfw): + fatal("Please install a link to a current %s %s as %s so " + "the user can override the system framework." + % (framework, frameworks[framework], libfw)) + if os.path.exists(usrfw): + fatal("Please rename %s to avoid possible dynamic load issues." + % usrfw) + + if frameworks['Tcl'] != frameworks['Tk']: + fatal("The Tcl and Tk frameworks are not the same version.") + + print(" -- Building with external Tcl/Tk %s frameworks" + % frameworks['Tk']) + + # add files to check after build + EXPECTED_SHARED_LIBS['_tkinter.so'] = [ + "/Library/Frameworks/Tcl.framework/Versions/%s/Tcl" + % frameworks['Tcl'], + "/Library/Frameworks/Tk.framework/Versions/%s/Tk" + % frameworks['Tk'], + ] + else: + print(" -- Building private copy of Tcl/Tk") + print("") # Remove inherited environment variables which might influence build environ_var_prefixes = ['CPATH', 'C_INCLUDE_', 'DYLD_', 'LANG', 'LC_', @@ -643,7 +615,7 @@ def parseOptions(args=None): """ Parse arguments and update global settings. """ - global WORKDIR, DEPSRC, SDKPATH, SRCDIR, DEPTARGET + global WORKDIR, DEPSRC, SRCDIR, DEPTARGET global UNIVERSALOPTS, UNIVERSALARCHS, ARCHLIST, CC, CXX global FW_VERSION_PREFIX global FW_SSL_DIRECTORY @@ -676,7 +648,7 @@ def parseOptions(args=None): DEPSRC=v elif k in ('--sdk-path',): - SDKPATH=v + print(" WARNING: --sdk-path is no longer supported") elif k in ('--src-dir',): SRCDIR=v @@ -692,7 +664,7 @@ def parseOptions(args=None): if deptarget is None: # Select alternate default deployment # target - DEPTARGET = default_target_map.get(v, '10.3') + DEPTARGET = default_target_map.get(v, '10.5') else: raise NotImplementedError(v) @@ -701,7 +673,6 @@ def parseOptions(args=None): SRCDIR=os.path.abspath(SRCDIR) WORKDIR=os.path.abspath(WORKDIR) - SDKPATH=os.path.abspath(SDKPATH) DEPSRC=os.path.abspath(DEPSRC) CC, CXX = getTargetCompilers() @@ -712,7 +683,6 @@ def parseOptions(args=None): print("-- Settings:") print(" * Source directory: %s" % SRCDIR) print(" * Build directory: %s" % WORKDIR) - print(" * SDK location: %s" % SDKPATH) print(" * Third-party source: %s" % DEPSRC) print(" * Deployment target: %s" % DEPTARGET) print(" * Universal archs: %s" % str(ARCHLIST)) @@ -854,9 +824,9 @@ def build_openssl_arch(archbase, arch): configure_opts.append("no-asm") runCommand(" ".join(["perl", "Configure"] + arch_opts[arch] + configure_opts)) - runCommand("make depend OSX_SDK=%s" % SDKPATH) - runCommand("make all OSX_SDK=%s" % SDKPATH) - runCommand("make install_sw OSX_SDK=%s" % SDKPATH) + runCommand("make depend") + runCommand("make all") + runCommand("make install_sw") # runCommand("make test") return @@ -1015,27 +985,24 @@ def buildRecipe(recipe, basedir, archList): if recipe.get('useLDFlags', 1): configure_args.extend([ - "CFLAGS=%s-mmacosx-version-min=%s -arch %s -isysroot %s " + "CFLAGS=%s-mmacosx-version-min=%s -arch %s " "-I%s/usr/local/include"%( recipe.get('extra_cflags', ''), DEPTARGET, ' -arch '.join(archList), - shellQuote(SDKPATH)[1:-1], shellQuote(basedir)[1:-1],), - "LDFLAGS=-mmacosx-version-min=%s -isysroot %s -L%s/usr/local/lib -arch %s"%( + "LDFLAGS=-mmacosx-version-min=%s -L%s/usr/local/lib -arch %s"%( DEPTARGET, - shellQuote(SDKPATH)[1:-1], shellQuote(basedir)[1:-1], ' -arch '.join(archList)), ]) else: configure_args.extend([ - "CFLAGS=%s-mmacosx-version-min=%s -arch %s -isysroot %s " + "CFLAGS=%s-mmacosx-version-min=%s -arch %s " "-I%s/usr/local/include"%( recipe.get('extra_cflags', ''), DEPTARGET, ' -arch '.join(archList), - shellQuote(SDKPATH)[1:-1], shellQuote(basedir)[1:-1],), ]) @@ -1113,10 +1080,6 @@ def buildPython(): curdir = os.getcwd() os.chdir(buildDir) - # Not sure if this is still needed, the original build script - # claims that parts of the install assume python.exe exists. - os.symlink('python', os.path.join(buildDir, 'python.exe')) - # Extract the version from the configure file, needed to calculate # several paths. version = getVersion() @@ -1127,16 +1090,22 @@ def buildPython(): os.environ['DYLD_LIBRARY_PATH'] = os.path.join(WORKDIR, 'libraries', 'usr', 'local', 'lib') print("Running configure...") - runCommand("%s -C --enable-framework --enable-universalsdk=%s " + runCommand("%s -C --enable-framework --enable-universalsdk=/ " "--with-universal-archs=%s " "%s " "%s " + "%s " + "%s " "LDFLAGS='-g -L%s/libraries/usr/local/lib' " "CFLAGS='-g -I%s/libraries/usr/local/include' 2>&1"%( - shellQuote(os.path.join(SRCDIR, 'configure')), shellQuote(SDKPATH), + shellQuote(os.path.join(SRCDIR, 'configure')), UNIVERSALARCHS, (' ', '--with-computed-gotos ')[PYTHON_3], (' ', '--without-ensurepip ')[PYTHON_3], + (' ', "--with-tcltk-includes='-I%s/libraries/usr/local/include'"%( + shellQuote(WORKDIR)[1:-1],))[internalTk()], + (' ', "--with-tcltk-libs='-L%s/libraries/usr/local/lib -ltcl8.6 -ltk8.6'"%( + shellQuote(WORKDIR)[1:-1],))[internalTk()], shellQuote(WORKDIR)[1:-1], shellQuote(WORKDIR)[1:-1])) @@ -1171,14 +1140,22 @@ def buildPython(): del os.environ['DYLD_LIBRARY_PATH'] print("Copying required shared libraries") if os.path.exists(os.path.join(WORKDIR, 'libraries', 'Library')): - runCommand("mv %s/* %s"%( - shellQuote(os.path.join( + build_lib_dir = os.path.join( WORKDIR, 'libraries', 'Library', 'Frameworks', - 'Python.framework', 'Versions', getVersion(), - 'lib')), - shellQuote(os.path.join(WORKDIR, '_root', 'Library', 'Frameworks', - 'Python.framework', 'Versions', getVersion(), - 'lib')))) + 'Python.framework', 'Versions', getVersion(), 'lib') + fw_lib_dir = os.path.join( + WORKDIR, '_root', 'Library', 'Frameworks', + 'Python.framework', 'Versions', getVersion(), 'lib') + if internalTk(): + # move Tcl and Tk pkgconfig files + runCommand("mv %s/pkgconfig/* %s/pkgconfig"%( + shellQuote(build_lib_dir), + shellQuote(fw_lib_dir) )) + runCommand("rm -r %s/pkgconfig"%( + shellQuote(build_lib_dir), )) + runCommand("mv %s/* %s"%( + shellQuote(build_lib_dir), + shellQuote(fw_lib_dir) )) frmDir = os.path.join(rootDir, 'Library', 'Frameworks', 'Python.framework') frmDirVersioned = os.path.join(frmDir, 'Versions', version) diff --git a/Mac/BuildScript/issue19373_tk_8_5_15_source.patch b/Mac/BuildScript/issue19373_tk_8_5_15_source.patch deleted file mode 100644 index de5d08e9d6c0..000000000000 --- a/Mac/BuildScript/issue19373_tk_8_5_15_source.patch +++ /dev/null @@ -1,13 +0,0 @@ -Issue #19373: Patch to Tk 8.5.15 to correct refresh problem on OS x 10.9. -From upstream checkin https://core.tcl.tk/tk/info/5a5abf71f9 - ---- tk8.5.15/macosx/tkMacOSXDraw.c 2013-09-16 09:41:21.000000000 -0700 -+++ Tk_Source_Code-5a5abf71f9fdb0da/macosx/tkMacOSXDraw.c 2013-10-27 13:27:00.000000000 -0700 -@@ -1688,6 +1688,7 @@ - { - if (dcPtr->context) { - CGContextSynchronize(dcPtr->context); -+ [[dcPtr->view window] setViewsNeedDisplay:YES]; - [[dcPtr->view window] enableFlushWindow]; - if (dcPtr->focusLocked) { - [dcPtr->view unlockFocus]; diff --git a/Mac/BuildScript/openssl_sdk_makedepend.patch b/Mac/BuildScript/openssl_sdk_makedepend.patch deleted file mode 100644 index 0caac0a64c1e..000000000000 --- a/Mac/BuildScript/openssl_sdk_makedepend.patch +++ /dev/null @@ -1,40 +0,0 @@ -# HG changeset patch -# -# using openssl 1.0.2k -# -# - support building with an OS X SDK - -diff Configure - -diff --git a/Configure b/Configure ---- a/Configure -+++ b/Configure -@@ -642,12 +642,12 @@ - - ##### MacOS X (a.k.a. Rhapsody or Darwin) setup - "rhapsody-ppc-cc","cc:-O3 -DB_ENDIAN::(unknown):MACOSX_RHAPSODY::BN_LLONG RC4_CHAR RC4_CHUNK DES_UNROLL BF_PTR:${no_asm}::", --"darwin-ppc-cc","cc:-arch ppc -O3 -DB_ENDIAN -Wa,-force_cpusubtype_ALL::-D_REENTRANT:MACOSX:-Wl,-search_paths_first%:BN_LLONG RC4_CHAR RC4_CHUNK DES_UNROLL BF_PTR:${ppc32_asm}:osx32:dlfcn:darwin-shared:-fPIC -fno-common:-arch ppc -dynamiclib:.\$(SHLIB_MAJOR).\$(SHLIB_MINOR).dylib", --"darwin64-ppc-cc","cc:-arch ppc64 -O3 -DB_ENDIAN::-D_REENTRANT:MACOSX:-Wl,-search_paths_first%:SIXTY_FOUR_BIT_LONG RC4_CHAR RC4_CHUNK DES_UNROLL BF_PTR:${ppc64_asm}:osx64:dlfcn:darwin-shared:-fPIC -fno-common:-arch ppc64 -dynamiclib:.\$(SHLIB_MAJOR).\$(SHLIB_MINOR).dylib", --"darwin-i386-cc","cc:-arch i386 -O3 -fomit-frame-pointer -DL_ENDIAN::-D_REENTRANT:MACOSX:-Wl,-search_paths_first%:BN_LLONG RC4_INT RC4_CHUNK DES_UNROLL BF_PTR:".eval{my $asm=$x86_asm;$asm=~s/cast\-586\.o//;$asm}.":macosx:dlfcn:darwin-shared:-fPIC -fno-common:-arch i386 -dynamiclib:.\$(SHLIB_MAJOR).\$(SHLIB_MINOR).dylib", --"debug-darwin-i386-cc","cc:-arch i386 -g3 -DL_ENDIAN::-D_REENTRANT:MACOSX:-Wl,-search_paths_first%:BN_LLONG RC4_INT RC4_CHUNK DES_UNROLL BF_PTR:${x86_asm}:macosx:dlfcn:darwin-shared:-fPIC -fno-common:-arch i386 -dynamiclib:.\$(SHLIB_MAJOR).\$(SHLIB_MINOR).dylib", --"darwin64-x86_64-cc","cc:-arch x86_64 -O3 -DL_ENDIAN -Wall::-D_REENTRANT:MACOSX:-Wl,-search_paths_first%:SIXTY_FOUR_BIT_LONG RC4_CHUNK DES_INT DES_UNROLL:".eval{my $asm=$x86_64_asm;$asm=~s/rc4\-[^:]+//;$asm}.":macosx:dlfcn:darwin-shared:-fPIC -fno-common:-arch x86_64 -dynamiclib:.\$(SHLIB_MAJOR).\$(SHLIB_MINOR).dylib", --"debug-darwin64-x86_64-cc","cc:-arch x86_64 -ggdb -g2 -O0 -DL_ENDIAN -Wall::-D_REENTRANT:MACOSX:-Wl,-search_paths_first%:SIXTY_FOUR_BIT_LONG RC4_CHUNK DES_INT DES_UNROLL:".eval{my $asm=$x86_64_asm;$asm=~s/rc4\-[^:]+//;$asm}.":macosx:dlfcn:darwin-shared:-fPIC -fno-common:-arch x86_64 -dynamiclib:.\$(SHLIB_MAJOR).\$(SHLIB_MINOR).dylib", -+"darwin-ppc-cc","cc:-arch ppc -isysroot \$(OSX_SDK) -O3 -DB_ENDIAN -Wa,-force_cpusubtype_ALL::-D_REENTRANT:MACOSX:-Wl,-search_paths_first%:BN_LLONG RC4_CHAR RC4_CHUNK DES_UNROLL BF_PTR:${ppc32_asm}:osx32:dlfcn:darwin-shared:-fPIC -fno-common:-arch ppc -dynamiclib:.\$(SHLIB_MAJOR).\$(SHLIB_MINOR).dylib", -+"darwin64-ppc-cc","cc:-arch ppc64 -isysroot \$(OSX_SDK) -O3 -DB_ENDIAN::-D_REENTRANT:MACOSX:-Wl,-search_paths_first%:SIXTY_FOUR_BIT_LONG RC4_CHAR RC4_CHUNK DES_UNROLL BF_PTR:${ppc64_asm}:osx64:dlfcn:darwin-shared:-fPIC -fno-common:-arch ppc64 -dynamiclib:.\$(SHLIB_MAJOR).\$(SHLIB_MINOR).dylib", -+"darwin-i386-cc","cc:-arch i386 -isysroot \$(OSX_SDK) -O3 -fomit-frame-pointer -DL_ENDIAN::-D_REENTRANT:MACOSX:-Wl,-search_paths_first%:BN_LLONG RC4_INT RC4_CHUNK DES_UNROLL BF_PTR:".eval{my $asm=$x86_asm;$asm=~s/cast\-586\.o//;$asm}.":macosx:dlfcn:darwin-shared:-fPIC -fno-common:-arch i386 -dynamiclib:.\$(SHLIB_MAJOR).\$(SHLIB_MINOR).dylib", -+"debug-darwin-i386-cc","cc:-arch i386 -isysroot \$(OSX_SDK) -g3 -DL_ENDIAN::-D_REENTRANT:MACOSX:-Wl,-search_paths_first%:BN_LLONG RC4_INT RC4_CHUNK DES_UNROLL BF_PTR:${x86_asm}:macosx:dlfcn:darwin-shared:-fPIC -fno-common:-arch i386 -dynamiclib:.\$(SHLIB_MAJOR).\$(SHLIB_MINOR).dylib", -+"darwin64-x86_64-cc","cc:-arch x86_64 -isysroot \$(OSX_SDK) -O3 -DL_ENDIAN -Wall::-D_REENTRANT:MACOSX:-Wl,-search_paths_first%:SIXTY_FOUR_BIT_LONG RC4_CHUNK DES_INT DES_UNROLL:".eval{my $asm=$x86_64_asm;$asm=~s/rc4\-[^:]+//;$asm}.":macosx:dlfcn:darwin-shared:-fPIC -fno-common:-arch x86_64 -dynamiclib:.\$(SHLIB_MAJOR).\$(SHLIB_MINOR).dylib", -+"debug-darwin64-x86_64-cc","cc:-arch x86_64 -isysroot \$(OSX_SDK) -ggdb -g2 -O0 -DL_ENDIAN -Wall::-D_REENTRANT:MACOSX:-Wl,-search_paths_first%:SIXTY_FOUR_BIT_LONG RC4_CHUNK DES_INT DES_UNROLL:".eval{my $asm=$x86_64_asm;$asm=~s/rc4\-[^:]+//;$asm}.":macosx:dlfcn:darwin-shared:-fPIC -fno-common:-arch x86_64 -dynamiclib:.\$(SHLIB_MAJOR).\$(SHLIB_MINOR).dylib", - "debug-darwin-ppc-cc","cc:-DBN_DEBUG -DREF_CHECK -DCONF_DEBUG -DCRYPTO_MDEBUG -DB_ENDIAN -g -Wall -O::-D_REENTRANT:MACOSX::BN_LLONG RC4_CHAR RC4_CHUNK DES_UNROLL BF_PTR:${ppc32_asm}:osx32:dlfcn:darwin-shared:-fPIC:-dynamiclib:.\$(SHLIB_MAJOR).\$(SHLIB_MINOR).dylib", - # iPhoneOS/iOS - "iphoneos-cross","llvm-gcc:-O3 -isysroot \$(CROSS_TOP)/SDKs/\$(CROSS_SDK) -fomit-frame-pointer -fno-common::-D_REENTRANT:iOS:-Wl,-search_paths_first%:BN_LLONG RC4_CHAR RC4_CHUNK DES_UNROLL BF_PTR:${no_asm}:dlfcn:darwin-shared:-fPIC -fno-common:-dynamiclib:.\$(SHLIB_MAJOR).\$(SHLIB_MINOR).dylib", -@@ -1728,8 +1728,7 @@ - s/^AR=\s*ar/AR= $ar/; - s/^RANLIB=.*/RANLIB= $ranlib/; - s/^RC=.*/RC= $windres/; -- s/^MAKEDEPPROG=.*$/MAKEDEPPROG= $cc/ if $cc eq "gcc"; -- s/^MAKEDEPPROG=.*$/MAKEDEPPROG= $cc/ if $ecc eq "gcc" || $ecc eq "clang"; -+ s/^MAKEDEPPROG=.*$/MAKEDEPPROG= $cc/; - } - s/^CFLAG=.*$/CFLAG= $cflags/; - s/^DEPFLAG=.*$/DEPFLAG=$depflags/; diff --git a/Mac/BuildScript/resources/Conclusion.rtf b/Mac/BuildScript/resources/Conclusion.rtf new file mode 100644 index 000000000000..9e0fa9fa6eeb --- /dev/null +++ b/Mac/BuildScript/resources/Conclusion.rtf @@ -0,0 +1,20 @@ +{\rtf1\ansi\ansicpg1252\cocoartf1561\cocoasubrtf200 +{\fonttbl\f0\fswiss\fcharset0 Helvetica;\f1\fnil\fcharset0 LucidaGrande-Bold;\f2\fnil\fcharset0 LucidaGrande; +\f3\fnil\fcharset0 Monaco;} +{\colortbl;\red255\green255\blue255;} +{\*\expandedcolortbl;;} +\margl1440\margr1440\vieww10540\viewh8400\viewkind0 +\pard\tx720\tx1440\tx2160\tx2880\tx3600\tx4320\tx5040\tx5760\tx6480\tx7200\tx7920\tx8640\pardirnatural\partightenfactor0 + +\f0\fs28 \cf0 Congratulations! +\fs24 +\f1\b\fs28 Python $FULL_VERSION for macOS $MACOSX_DEPLOYMENT_TARGET +\f2\b0 was successfully installed. +\fs24 \ +\ +One more thing: to verify the identity of secure network connections, this Python needs a set of SSL root certificates. You can download and install a current curated set from {\field{\*\fldinst{HYPERLINK "https://pypi.org/project/certifi/"}}{\fldrslt the Certifi project}} by double-clicking on the +\f3 Install Certificates +\f2 icon in {\field{\*\fldinst{HYPERLINK "file://localhost/Applications/Python%20$VERSION/"}}{\fldrslt the Finder window}}. See {\field{\*\fldinst{HYPERLINK "file://localhost/Applications/Python%20$VERSION/ReadMe.rtf"}}{\fldrslt the +\f3 ReadMe +\f2 file}} for more information.\ +} \ No newline at end of file diff --git a/Mac/BuildScript/resources/ReadMe.rtf b/Mac/BuildScript/resources/ReadMe.rtf index ac68786999df..e81465915380 100644 --- a/Mac/BuildScript/resources/ReadMe.rtf +++ b/Mac/BuildScript/resources/ReadMe.rtf @@ -1,43 +1,44 @@ -{\rtf1\ansi\ansicpg1252\cocoartf1504\cocoasubrtf750 +{\rtf1\ansi\ansicpg1252\cocoartf1561\cocoasubrtf200 {\fonttbl\f0\fswiss\fcharset0 Helvetica;\f1\fmodern\fcharset0 CourierNewPSMT;} {\colortbl;\red255\green255\blue255;} {\*\expandedcolortbl;;} \margl1440\margr1440\vieww13380\viewh14600\viewkind0 \pard\tx720\tx1440\tx2160\tx2880\tx3600\tx4320\tx5040\tx5760\tx6480\tx7200\tx7920\tx8640\pardirnatural\partightenfactor0 -\f0\fs24 \cf0 This package will install Python $FULL_VERSION for Mac OS X $MACOSX_DEPLOYMENT_TARGET for the following architecture(s): $ARCHITECTURES.\ +\f0\fs24 \cf0 This package will install Python $FULL_VERSION for macOS $MACOSX_DEPLOYMENT_TARGET for the following architecture(s): $ARCHITECTURES.\ \ \pard\tx720\tx1440\tx2160\tx2880\tx3600\tx4320\tx5040\tx5760\tx6480\tx7200\tx7920\tx8640\pardirnatural\partightenfactor0 -\b \cf0 \ul \ulc0 Which installer variant should I use? +\b \cf0 \ul \ulc0 Which installer variant should I use? [CHANGED in 3.6.5] \b0 \ulnone \ \ \pard\tx720\tx1440\tx2160\tx2880\tx3600\tx4320\tx5040\tx5760\tx6480\tx7200\tx7920\tx8640\pardirnatural\partightenfactor0 \b \cf0 **NEW** -\b0 For Python 3.6, the python.org website now provides only one installer variant for download: one that installs a +\b0 With Python 3.6.5, the python.org website now provides two installer variants for download: one that installs a +\i 64-bit-only +\i0 Python capable of running on +\i macOS 10.9 (Mavericks) +\i0 or later; and one that installs a \i 64-bit/32-bit Intel \i0 Python capable of running on -\i Mac OS X 10.6 (Snow Leopard) -\i0 or later. This ReadMe was installed with the +\i macOS 10.6 (Snow Leopard) +\i0 or later. (This ReadMe was installed with the \i $MACOSX_DEPLOYMENT_TARGET -\i0 variant. By default, Python will automatically run in 64-bit mode if your system supports it. The Python installed by this installer is built with private copies of some third-party libraries not included with or newer than those in OS X itself. The list of these libraries is included at the end of the License.rtf file. -\b \ul \ -\ -Certificate verification and OpenSSL\ +\i0 variant.) Previous Python 3.6.x releases only provided the 10.6 or later installer. If you are running on macOS 10.9 or later and if you have no need for compatibility with older systems, use the 10.9 variant. Use the 10.6 variant if you are running on macOS 10.6 through 10.8, if you need to maintain compatibility with previous 3.6.x releases, or if you want to produce standalone applications that can run on systems from 10.6. The Pythons installed by these installers are built with private copies of some third-party libraries not included with or newer than those in macOS itself. The list of these libraries varies by installer variant and is included at the end of the License.rtf file.\ \pard\tx720\tx1440\tx2160\tx2880\tx3600\tx4320\tx5040\tx5760\tx6480\tx7200\tx7920\tx8640\pardirnatural\partightenfactor0 -\b0 \cf0 \ulnone \ -\pard\tx720\tx1440\tx2160\tx2880\tx3600\tx4320\tx5040\tx5760\tx6480\tx7200\tx7920\tx8640\pardirnatural\partightenfactor0 +\b \cf0 \ul \ulc0 \ +Certificate verification and OpenSSL\ -\b \cf0 **NEW** -\b0 This variant of Python 3.6 now includes its own private copy of OpenSSL 1.0.2. Unlike previous releases, the deprecated Apple-supplied OpenSSL libraries are no longer used. This also means that the trust certificates in system and user keychains managed by the +\b0 \ulnone \ +This variant of Python 3.6 now includes its own private copy of OpenSSL 1.0.2. Unlike previous releases, the deprecated Apple-supplied OpenSSL libraries are no longer used. This also means that the trust certificates in system and user keychains managed by the \i Keychain Access \i0 application and the \i security \i0 command line utility are no longer used as defaults by the Python \f1 ssl -\f0 module. For 3.6.0, a sample command script is included in +\f0 module. A sample command script is included in \f1 /Applications/Python 3.6 \f0 to install a curated bundle of default root certificates from the third-party \f1 certifi @@ -49,16 +50,17 @@ The bundled \f1 pip \f0 included with the Python 3.6 installer has its own default certificate store for verifying download connections.\ \ -\pard\tx720\tx1440\tx2160\tx2880\tx3600\tx4320\tx5040\tx5760\tx6480\tx7200\tx7920\tx8640\pardirnatural\partightenfactor0 -\b \cf0 \ul Update your version of Tcl/Tk to use IDLE or other Tk applications +\b \ul Using IDLE or other Tk applications [NEW/CHANGED in 3.6.5] \b0 \ulnone \ \ -To use IDLE or other programs that use the Tkinter graphical user interface toolkit, you need to install a newer third-party version of the +The 10.9+ installer variant comes with its own private version of Tcl/Tk 8.6. It does not use system-supplied or third-party supplied versions of Tcl/Tk.\ +\ +For the 10.6+ variant, you continue to need to install a newer third-party version of the \i Tcl/Tk -\i0 frameworks. Visit {\field{\*\fldinst{HYPERLINK "https://www.python.org/download/mac/tcltk/"}}{\fldrslt https://www.python.org/download/mac/tcltk/}} for current information about supported and recommended versions of +\i0 8.5 (not 8.6) frameworks to use IDLE or other programs that use the Tkinter graphical user interface toolkit. Visit {\field{\*\fldinst{HYPERLINK "https://www.python.org/download/mac/tcltk/"}}{\fldrslt https://www.python.org/download/mac/tcltk/}} for current information about supported and recommended versions of \i Tcl/Tk -\i0 for this version of Python and of Mac OS X. For the initial release of Python 3.6, the installer is still linked with Tcl/Tk 8.5.\ +\i0 for this version of Python and of macOS.\ \b \ul \ Other changes\ diff --git a/Mac/BuildScript/resources/Welcome.rtf b/Mac/BuildScript/resources/Welcome.rtf index 3a9ab04454d8..cac9626693dc 100644 --- a/Mac/BuildScript/resources/Welcome.rtf +++ b/Mac/BuildScript/resources/Welcome.rtf @@ -1,34 +1,24 @@ -{\rtf1\ansi\ansicpg1252\cocoartf1504 +{\rtf1\ansi\ansicpg1252\cocoartf1561\cocoasubrtf200 \cocoascreenfonts1{\fonttbl\f0\fswiss\fcharset0 Helvetica;} {\colortbl;\red255\green255\blue255;} -{\*\expandedcolortbl;\csgray\c100000;} +{\*\expandedcolortbl;;} \paperw11905\paperh16837\margl1440\margr1440\vieww12200\viewh10880\viewkind0 -\pard\tx720\tx1440\tx2160\tx2880\tx3600\tx4320\tx5040\tx5760\tx6480\tx7200\tx7920\tx8640\partightenfactor0 +\pard\tx720\tx1440\tx2160\tx2880\tx3600\tx4320\tx5040\tx5760\tx6480\tx7200\tx7920\tx8640\pardirnatural\partightenfactor0 \f0\fs24 \cf0 This package will install \b Python $FULL_VERSION \b0 for -\b Mac OS X $MACOSX_DEPLOYMENT_TARGET +\b macOS $MACOSX_DEPLOYMENT_TARGET \b0 .\ -\ +\pard\tx720\tx1440\tx2160\tx2880\tx3600\tx4320\tx5040\tx5760\tx6480\tx7200\tx7920\tx8640\partightenfactor0 +\cf0 \ -\b Python for Mac OS X -\b0 consists of the Python programming language interpreter, plus a set of programs to allow easy access to it for Mac OS X users including an integrated development environment +\b Python for macOS +\b0 consists of the Python programming language interpreter, plus a set of programs to allow easy access to it for macOS users including an integrated development environment \b IDLE \b0 .\ \ -\pard\tx720\tx1440\tx2160\tx2880\tx3600\tx4320\tx5040\tx5760\tx6480\tx7200\tx7920\tx8640\partightenfactor0 - -\b \cf0 NEW: -\b0 There are important changes in this release regarding network security and trust certificates. Please see the ReadMe for more details.\ -\ -\pard\tx720\tx1440\tx2160\tx2880\tx3600\tx4320\tx5040\tx5760\tx6480\tx7200\tx7920\tx8640\partightenfactor0 -\b \cf0 IMPORTANT: -\b0 -\b IDLE -\b0 and other programs using the -\b tkinter -\b0 graphical user interface toolkit require specific versions of the -\b Tcl/Tk -\b0 platform independent windowing toolkit. Visit {\field{\*\fldinst{HYPERLINK "https://www.python.org/download/mac/tcltk/"}}{\fldrslt https://www.python.org/download/mac/tcltk/}} for current information on supported and recommended versions of Tcl/Tk for this version of Python and Mac OS X.} \ No newline at end of file +\b NEW in 3.6.5: +\b0 two installer variants (10.9+ 64-bit-only, 10.6+ 64-/32-bit), built-in Tcl/Tk 8.6 support in the 10.9+ variant (no additional third-party downloads!)\ +} \ No newline at end of file diff --git a/Mac/BuildScript/scripts/postflight.documentation b/Mac/BuildScript/scripts/postflight.documentation index b9f28a515215..3cbbc1bf10ca 100755 --- a/Mac/BuildScript/scripts/postflight.documentation +++ b/Mac/BuildScript/scripts/postflight.documentation @@ -12,6 +12,7 @@ SHARE_DOCDIR_TO_FWK="../../.." # make link in /Applications/Python m.n/ for Finder users if [ -d "${APPDIR}" ]; then ln -fhs "${FWK_DOCDIR}/index.html" "${APPDIR}/Python Documentation.html" + open "${APPDIR}" || true # open the applications folder fi # make share/doc link in framework for command line users diff --git a/Mac/BuildScript/tk868_on_10_8_10_9.patch b/Mac/BuildScript/tk868_on_10_8_10_9.patch new file mode 100644 index 000000000000..8fe10604a68c --- /dev/null +++ b/Mac/BuildScript/tk868_on_10_8_10_9.patch @@ -0,0 +1,18 @@ +Fix build failure with +quartz variant on OS X 10.8 and 10.9. +Even though Gestalt was deprecated in OS X 10.8, it should work fine +through OS X 10.9, and its replacement NSOperatingSystemVersion was +not introduced until OS X 10.10. + +Patch from MacPorts project and reported upstream: +https://trac.macports.org/ticket/55649 +--- tk8.6.8/macosx/tkMacOSXXStubs.c.orig 2017-12-06 09:25:08.000000000 -0600 ++++ tk8.6.8-patched/macosx/tkMacOSXXStubs.c 2018-01-06 19:34:17.000000000 -0600 +@@ -175,7 +175,7 @@ + { + int major, minor, patch; + +-#if MAC_OS_X_VERSION_MIN_REQUIRED < 1080 ++#if MAC_OS_X_VERSION_MIN_REQUIRED < 101000 + Gestalt(gestaltSystemVersionMajor, (SInt32*)&major); + Gestalt(gestaltSystemVersionMinor, (SInt32*)&minor); + Gestalt(gestaltSystemVersionBugFix, (SInt32*)&patch); diff --git a/Misc/NEWS.d/next/macOS/2018-03-13-21-00-20.bpo-32726.Mticyn.rst b/Misc/NEWS.d/next/macOS/2018-03-13-21-00-20.bpo-32726.Mticyn.rst new file mode 100644 index 000000000000..f2d096a063b7 --- /dev/null +++ b/Misc/NEWS.d/next/macOS/2018-03-13-21-00-20.bpo-32726.Mticyn.rst @@ -0,0 +1,4 @@ +Provide an additional, more modern macOS installer variant that supports +macOS 10.9+ systems in 64-bit mode only. Upgrade the supplied third-party +libraries to OpenSSL 1.0.2n, XZ 5.2.3, and SQLite 3.22.0. The 10.9+ +installer now links with and supplies its own copy of Tcl/Tk 8.6.8. diff --git a/configure b/configure index 1e66117fffbb..a707743b4966 100755 --- a/configure +++ b/configure @@ -1491,7 +1491,8 @@ Optional Packages: --without-PACKAGE do not use PACKAGE (same as --with-PACKAGE=no) --with-universal-archs=ARCH select architectures for universal build ("32-bit", - "64-bit", "3-way", "intel", "intel-32", or "all") + "64-bit", "3-way", "intel", "intel-32", "intel-64", + or "all") --with-framework-name=FRAMEWORK specify an alternate name of the framework built with --enable-framework @@ -7345,6 +7346,11 @@ $as_echo "$CC" >&6; } LIPO_32BIT_FLAGS="" ARCH_RUN_32BIT="" ;; + intel-64) + UNIVERSAL_ARCH_FLAGS="-arch x86_64" + LIPO_32BIT_FLAGS="" + ARCH_RUN_32BIT="true" + ;; 3-way) UNIVERSAL_ARCH_FLAGS="-arch i386 -arch ppc -arch x86_64" LIPO_32BIT_FLAGS="-extract ppc7400 -extract i386" @@ -7355,11 +7361,14 @@ $as_echo "$CC" >&6; } ;; esac - CFLAGS="${UNIVERSAL_ARCH_FLAGS} -isysroot ${UNIVERSALSDK} ${CFLAGS}" - LDFLAGS="${UNIVERSAL_ARCH_FLAGS} -isysroot ${UNIVERSALSDK} ${LDFLAGS}" if test "${UNIVERSALSDK}" != "/" then + CFLAGS="${UNIVERSAL_ARCH_FLAGS} -isysroot ${UNIVERSALSDK} ${CFLAGS}" + LDFLAGS="${UNIVERSAL_ARCH_FLAGS} -isysroot ${UNIVERSALSDK} ${LDFLAGS}" CPPFLAGS="-isysroot ${UNIVERSALSDK} ${CPPFLAGS}" + else + CFLAGS="${UNIVERSAL_ARCH_FLAGS} ${CFLAGS}" + LDFLAGS="${UNIVERSAL_ARCH_FLAGS} ${LDFLAGS}" fi fi diff --git a/configure.ac b/configure.ac index 2eb511bf7f1f..0ec09369a9a0 100644 --- a/configure.ac +++ b/configure.ac @@ -214,7 +214,7 @@ fi AC_SUBST(LIPO_32BIT_FLAGS) AC_MSG_CHECKING(for --with-universal-archs) AC_ARG_WITH(universal-archs, - AS_HELP_STRING([--with-universal-archs=ARCH], [select architectures for universal build ("32-bit", "64-bit", "3-way", "intel", "intel-32", or "all")]), + AS_HELP_STRING([--with-universal-archs=ARCH], [select architectures for universal build ("32-bit", "64-bit", "3-way", "intel", "intel-32", "intel-64", or "all")]), [ UNIVERSAL_ARCHS="$withval" ], @@ -1782,6 +1782,11 @@ yes) LIPO_32BIT_FLAGS="" ARCH_RUN_32BIT="" ;; + intel-64) + UNIVERSAL_ARCH_FLAGS="-arch x86_64" + LIPO_32BIT_FLAGS="" + ARCH_RUN_32BIT="true" + ;; 3-way) UNIVERSAL_ARCH_FLAGS="-arch i386 -arch ppc -arch x86_64" LIPO_32BIT_FLAGS="-extract ppc7400 -extract i386" @@ -1792,11 +1797,14 @@ yes) ;; esac - CFLAGS="${UNIVERSAL_ARCH_FLAGS} -isysroot ${UNIVERSALSDK} ${CFLAGS}" - LDFLAGS="${UNIVERSAL_ARCH_FLAGS} -isysroot ${UNIVERSALSDK} ${LDFLAGS}" if test "${UNIVERSALSDK}" != "/" then + CFLAGS="${UNIVERSAL_ARCH_FLAGS} -isysroot ${UNIVERSALSDK} ${CFLAGS}" + LDFLAGS="${UNIVERSAL_ARCH_FLAGS} -isysroot ${UNIVERSALSDK} ${LDFLAGS}" CPPFLAGS="-isysroot ${UNIVERSALSDK} ${CPPFLAGS}" + else + CFLAGS="${UNIVERSAL_ARCH_FLAGS} ${CFLAGS}" + LDFLAGS="${UNIVERSAL_ARCH_FLAGS} ${LDFLAGS}" fi fi From webhook-mailer at python.org Wed Mar 14 02:40:29 2018 From: webhook-mailer at python.org (Christian Heimes) Date: Wed, 14 Mar 2018 06:40:29 -0000 Subject: [Python-checkins] bpo-30622: Fix backport of NPN fix (#6102) Message-ID: <mailman.74.1521009630.1871.python-checkins@python.org> https://github.com/python/cpython/commit/0ec0290a075ad3ac7d9990efd701c81da16293c0 commit: 0ec0290a075ad3ac7d9990efd701c81da16293c0 branch: 3.6 author: Christian Heimes <christian at python.org> committer: GitHub <noreply at github.com> date: 2018-03-14T07:40:20+01:00 summary: bpo-30622: Fix backport of NPN fix (#6102) Fix backport a79591cf of bpo-30622 to 3.6 branch. Signed-off-by: Christian Heimes <christian at python.org> files: M Modules/_ssl.c diff --git a/Modules/_ssl.c b/Modules/_ssl.c index 2fe696daa5e1..c54e43c2b48a 100644 --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -2747,7 +2747,7 @@ _ssl__SSLContext_impl(PyTypeObject *type, int proto_version) return NULL; } self->ctx = ctx; -#ifdef HAVE_NPN +#if HAVE_NPN self->npn_protocols = NULL; #endif #if HAVE_ALPN From solipsis at pitrou.net Wed Mar 14 05:15:03 2018 From: solipsis at pitrou.net (solipsis at pitrou.net) Date: Wed, 14 Mar 2018 09:15:03 +0000 Subject: [Python-checkins] Daily reference leaks (4243df51fe43): sum=8 Message-ID: <20180314091503.1.D25BCB64D7430751@psf.io> results for 4243df51fe43 on branch "default" -------------------------------------------- test_asyncio leaked [0, 3, 0] memory blocks, sum=3 test_collections leaked [-7, 1, 7] memory blocks, sum=1 test_functools leaked [0, 3, 1] memory blocks, sum=4 test_multiprocessing_forkserver leaked [2, 0, -2] memory blocks, sum=0 Command line was: ['./python', '-m', 'test.regrtest', '-uall', '-R', '3:3:/home/psf-users/antoine/refleaks/reflogvdsJg2', '--timeout', '7200'] From webhook-mailer at python.org Wed Mar 14 13:52:26 2018 From: webhook-mailer at python.org (Christian Heimes) Date: Wed, 14 Mar 2018 17:52:26 -0000 Subject: [Python-checkins] [3.6] bpo-32885: Tools/scripts/pathfix.py: Add -n option for no backup~ (GH-5772) (#6104) Message-ID: <mailman.75.1521049948.1871.python-checkins@python.org> https://github.com/python/cpython/commit/a954919788f2130076e4f9dd91e9eccf69540f7a commit: a954919788f2130076e4f9dd91e9eccf69540f7a branch: 3.6 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Christian Heimes <christian at python.org> date: 2018-03-14T18:52:22+01:00 summary: [3.6] bpo-32885: Tools/scripts/pathfix.py: Add -n option for no backup~ (GH-5772) (#6104) Creating backup files with ~ suffix can be undesirable in some environment, such as when building RPM packages. Instead of requiring the user to remove those files manually, option -n was added, that simply disables this feature. -n was selected because 2to3 has the same option with this behavior. (cherry picked from commit 5affd5c29eb1493cb31ef3cfdde15538ac134689) Co-authored-by: Miro Hron?ok <miro at hroncok.cz> files: A Misc/NEWS.d/next/Tools-Demos/2018-02-20-12-16-47.bpo-32885.dL5x7C.rst M Misc/ACKS M Tools/scripts/pathfix.py diff --git a/Misc/ACKS b/Misc/ACKS index b2033ee9d3f3..ca8eaefc5b46 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -672,6 +672,7 @@ Ken Howard Brad Howes Mike Hoy Ben Hoyt +Miro Hron?ok Chiu-Hsiang Hsu Chih-Hao Huang Christian Hudon diff --git a/Misc/NEWS.d/next/Tools-Demos/2018-02-20-12-16-47.bpo-32885.dL5x7C.rst b/Misc/NEWS.d/next/Tools-Demos/2018-02-20-12-16-47.bpo-32885.dL5x7C.rst new file mode 100644 index 000000000000..e003e1d84fd0 --- /dev/null +++ b/Misc/NEWS.d/next/Tools-Demos/2018-02-20-12-16-47.bpo-32885.dL5x7C.rst @@ -0,0 +1,2 @@ +Add an ``-n`` flag for ``Tools/scripts/pathfix.py`` to disbale automatic +backup creation (files with ``~`` suffix). diff --git a/Tools/scripts/pathfix.py b/Tools/scripts/pathfix.py index 562bbc737812..c5bf984306a3 100755 --- a/Tools/scripts/pathfix.py +++ b/Tools/scripts/pathfix.py @@ -7,8 +7,9 @@ # Directories are searched recursively for files whose name looks # like a python module. # Symbolic links are always ignored (except as explicit directory -# arguments). Of course, the original file is kept as a back-up -# (with a "~" attached to its name). +# arguments). +# The original file is kept as a back-up (with a "~" attached to its name), +# -n flag can be used to disable this. # # Undoubtedly you can do this using find and sed or perl, but this is # a nice example of Python code that recurses down a directory tree @@ -31,14 +32,17 @@ new_interpreter = None preserve_timestamps = False +create_backup = True + def main(): global new_interpreter global preserve_timestamps - usage = ('usage: %s -i /interpreter -p file-or-directory ...\n' % + global create_backup + usage = ('usage: %s -i /interpreter -p -n file-or-directory ...\n' % sys.argv[0]) try: - opts, args = getopt.getopt(sys.argv[1:], 'i:p') + opts, args = getopt.getopt(sys.argv[1:], 'i:pn') except getopt.error as msg: err(str(msg) + '\n') err(usage) @@ -48,6 +52,8 @@ def main(): new_interpreter = a.encode() if o == '-p': preserve_timestamps = True + if o == '-n': + create_backup = False if not new_interpreter or not new_interpreter.startswith(b'/') or \ not args: err('-i option or file-or-directory missing\n') @@ -134,10 +140,16 @@ def fix(filename): except OSError as msg: err('%s: warning: chmod failed (%r)\n' % (tempname, msg)) # Then make a backup of the original file as filename~ - try: - os.rename(filename, filename + '~') - except OSError as msg: - err('%s: warning: backup failed (%r)\n' % (filename, msg)) + if create_backup: + try: + os.rename(filename, filename + '~') + except OSError as msg: + err('%s: warning: backup failed (%r)\n' % (filename, msg)) + else: + try: + os.remove(filename) + except OSError as msg: + err('%s: warning: removing failed (%r)\n' % (filename, msg)) # Now move the temp file to the original file try: os.rename(tempname, filename) From webhook-mailer at python.org Wed Mar 14 13:52:31 2018 From: webhook-mailer at python.org (Christian Heimes) Date: Wed, 14 Mar 2018 17:52:31 -0000 Subject: [Python-checkins] [3.7] bpo-32885: Tools/scripts/pathfix.py: Add -n option for no backup~ (GH-5772) (#6103) Message-ID: <mailman.76.1521049952.1871.python-checkins@python.org> https://github.com/python/cpython/commit/6e65e4462692cbf4461c307a411af7cf40a1ca4a commit: 6e65e4462692cbf4461c307a411af7cf40a1ca4a branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Christian Heimes <christian at python.org> date: 2018-03-14T18:52:28+01:00 summary: [3.7] bpo-32885: Tools/scripts/pathfix.py: Add -n option for no backup~ (GH-5772) (#6103) Creating backup files with ~ suffix can be undesirable in some environment, such as when building RPM packages. Instead of requiring the user to remove those files manually, option -n was added, that simply disables this feature. -n was selected because 2to3 has the same option with this behavior. (cherry picked from commit 5affd5c29eb1493cb31ef3cfdde15538ac134689) Co-authored-by: Miro Hron?ok <miro at hroncok.cz> files: A Misc/NEWS.d/next/Tools-Demos/2018-02-20-12-16-47.bpo-32885.dL5x7C.rst M Misc/ACKS M Tools/scripts/pathfix.py diff --git a/Misc/ACKS b/Misc/ACKS index b0a9136170d6..9c7cb6f0057d 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -686,6 +686,7 @@ Ken Howard Brad Howes Mike Hoy Ben Hoyt +Miro Hron?ok Chiu-Hsiang Hsu Chih-Hao Huang Christian Hudon diff --git a/Misc/NEWS.d/next/Tools-Demos/2018-02-20-12-16-47.bpo-32885.dL5x7C.rst b/Misc/NEWS.d/next/Tools-Demos/2018-02-20-12-16-47.bpo-32885.dL5x7C.rst new file mode 100644 index 000000000000..e003e1d84fd0 --- /dev/null +++ b/Misc/NEWS.d/next/Tools-Demos/2018-02-20-12-16-47.bpo-32885.dL5x7C.rst @@ -0,0 +1,2 @@ +Add an ``-n`` flag for ``Tools/scripts/pathfix.py`` to disbale automatic +backup creation (files with ``~`` suffix). diff --git a/Tools/scripts/pathfix.py b/Tools/scripts/pathfix.py index 562bbc737812..c5bf984306a3 100755 --- a/Tools/scripts/pathfix.py +++ b/Tools/scripts/pathfix.py @@ -7,8 +7,9 @@ # Directories are searched recursively for files whose name looks # like a python module. # Symbolic links are always ignored (except as explicit directory -# arguments). Of course, the original file is kept as a back-up -# (with a "~" attached to its name). +# arguments). +# The original file is kept as a back-up (with a "~" attached to its name), +# -n flag can be used to disable this. # # Undoubtedly you can do this using find and sed or perl, but this is # a nice example of Python code that recurses down a directory tree @@ -31,14 +32,17 @@ new_interpreter = None preserve_timestamps = False +create_backup = True + def main(): global new_interpreter global preserve_timestamps - usage = ('usage: %s -i /interpreter -p file-or-directory ...\n' % + global create_backup + usage = ('usage: %s -i /interpreter -p -n file-or-directory ...\n' % sys.argv[0]) try: - opts, args = getopt.getopt(sys.argv[1:], 'i:p') + opts, args = getopt.getopt(sys.argv[1:], 'i:pn') except getopt.error as msg: err(str(msg) + '\n') err(usage) @@ -48,6 +52,8 @@ def main(): new_interpreter = a.encode() if o == '-p': preserve_timestamps = True + if o == '-n': + create_backup = False if not new_interpreter or not new_interpreter.startswith(b'/') or \ not args: err('-i option or file-or-directory missing\n') @@ -134,10 +140,16 @@ def fix(filename): except OSError as msg: err('%s: warning: chmod failed (%r)\n' % (tempname, msg)) # Then make a backup of the original file as filename~ - try: - os.rename(filename, filename + '~') - except OSError as msg: - err('%s: warning: backup failed (%r)\n' % (filename, msg)) + if create_backup: + try: + os.rename(filename, filename + '~') + except OSError as msg: + err('%s: warning: backup failed (%r)\n' % (filename, msg)) + else: + try: + os.remove(filename) + except OSError as msg: + err('%s: warning: removing failed (%r)\n' % (filename, msg)) # Now move the temp file to the original file try: os.rename(tempname, filename) From webhook-mailer at python.org Wed Mar 14 16:08:11 2018 From: webhook-mailer at python.org (Antoine Pitrou) Date: Wed, 14 Mar 2018 20:08:11 -0000 Subject: [Python-checkins] bpo-33021: Fix GCC 7 warning (-Wmaybe-uninitialized) in mmapmodule.c (#6117) Message-ID: <mailman.77.1521058093.1871.python-checkins@python.org> https://github.com/python/cpython/commit/d6e140466142018ddbb7541185348be2b833a2ce commit: d6e140466142018ddbb7541185348be2b833a2ce branch: master author: Zackery Spytz <zspytz at gmail.com> committer: Antoine Pitrou <pitrou at free.fr> date: 2018-03-14T21:08:01+01:00 summary: bpo-33021: Fix GCC 7 warning (-Wmaybe-uninitialized) in mmapmodule.c (#6117) files: M Modules/mmapmodule.c diff --git a/Modules/mmapmodule.c b/Modules/mmapmodule.c index 6abdc71bbb82..95d42d6dc5e5 100644 --- a/Modules/mmapmodule.c +++ b/Modules/mmapmodule.c @@ -1050,7 +1050,7 @@ static PyObject * new_mmap_object(PyTypeObject *type, PyObject *args, PyObject *kwdict) { struct _Py_stat_struct status; - int fstat_result; + int fstat_result = -1; mmap_object *m_obj; Py_ssize_t map_size; off_t offset = 0; From solipsis at pitrou.net Thu Mar 15 05:11:28 2018 From: solipsis at pitrou.net (solipsis at pitrou.net) Date: Thu, 15 Mar 2018 09:11:28 +0000 Subject: [Python-checkins] Daily reference leaks (4243df51fe43): sum=11 Message-ID: <20180315091128.1.E40783936E8FBE77@psf.io> results for 4243df51fe43 on branch "default" -------------------------------------------- test_collections leaked [0, 0, 7] memory blocks, sum=7 test_functools leaked [0, 3, 1] memory blocks, sum=4 test_multiprocessing_forkserver leaked [1, 0, -2] memory blocks, sum=-1 test_multiprocessing_spawn leaked [-1, 2, 0] memory blocks, sum=1 Command line was: ['./python', '-m', 'test.regrtest', '-uall', '-R', '3:3:/home/psf-users/antoine/refleaks/reflogwKL1DM', '--timeout', '7200'] From solipsis at pitrou.net Fri Mar 16 05:10:00 2018 From: solipsis at pitrou.net (solipsis at pitrou.net) Date: Fri, 16 Mar 2018 09:10:00 +0000 Subject: [Python-checkins] Daily reference leaks (4243df51fe43): sum=1 Message-ID: <20180316091000.1.E3DDD742719EDF04@psf.io> results for 4243df51fe43 on branch "default" -------------------------------------------- test_asyncio leaked [3, 0, 0] memory blocks, sum=3 test_collections leaked [-7, 1, 0] memory blocks, sum=-6 test_functools leaked [0, 3, 1] memory blocks, sum=4 Command line was: ['./python', '-m', 'test.regrtest', '-uall', '-R', '3:3:/home/psf-users/antoine/refleaks/reflogLUdo0M', '--timeout', '7200'] From webhook-mailer at python.org Sat Mar 17 01:41:30 2018 From: webhook-mailer at python.org (Nick Coghlan) Date: Sat, 17 Mar 2018 05:41:30 -0000 Subject: [Python-checkins] bpo-32374: m_traverse may be called with m_state=NULL (GH-5140) Message-ID: <mailman.78.1521265292.1871.python-checkins@python.org> https://github.com/python/cpython/commit/c2b0b12d1a137ada1023ab7c10b8d9a0249d95f9 commit: c2b0b12d1a137ada1023ab7c10b8d9a0249d95f9 branch: master author: Marcel Plch <gmarcel.plch at gmail.com> committer: Nick Coghlan <ncoghlan at gmail.com> date: 2018-03-17T15:41:20+10:00 summary: bpo-32374: m_traverse may be called with m_state=NULL (GH-5140) Multi-phase initialized modules allow m_traverse to be called while the module is still being initialized, so module authors may need to account for that. files: A Misc/NEWS.d/next/C API/2018-01-09-17-03-54.bpo-32374.SwwLoz.rst M Doc/c-api/module.rst M Lib/test/test_importlib/extension/test_loader.py M Modules/_testmultiphase.c M Objects/moduleobject.c diff --git a/Doc/c-api/module.rst b/Doc/c-api/module.rst index 7efab28af724..017b656854a8 100644 --- a/Doc/c-api/module.rst +++ b/Doc/c-api/module.rst @@ -196,17 +196,23 @@ or request "multi-phase initialization" by returning the definition struct itsel .. c:member:: traverseproc m_traverse A traversal function to call during GC traversal of the module object, or - *NULL* if not needed. + *NULL* if not needed. This function may be called before module state + is allocated (:c:func:`PyModule_GetState()` may return `NULL`), + and before the :c:member:`Py_mod_exec` function is executed. .. c:member:: inquiry m_clear A clear function to call during GC clearing of the module object, or - *NULL* if not needed. + *NULL* if not needed. This function may be called before module state + is allocated (:c:func:`PyModule_GetState()` may return `NULL`), + and before the :c:member:`Py_mod_exec` function is executed. .. c:member:: freefunc m_free A function to call during deallocation of the module object, or *NULL* if - not needed. + not needed. This function may be called before module state + is allocated (:c:func:`PyModule_GetState()` may return `NULL`), + and before the :c:member:`Py_mod_exec` function is executed. Single-phase initialization ........................... diff --git a/Lib/test/test_importlib/extension/test_loader.py b/Lib/test/test_importlib/extension/test_loader.py index 8d20040aab8d..53ac3c71d4b5 100644 --- a/Lib/test/test_importlib/extension/test_loader.py +++ b/Lib/test/test_importlib/extension/test_loader.py @@ -9,7 +9,7 @@ import unittest import importlib.util import importlib - +from test.support.script_helper import assert_python_failure class LoaderTests(abc.LoaderTests): @@ -267,6 +267,20 @@ def test_nonascii(self): self.assertEqual(module.__name__, name) self.assertEqual(module.__doc__, "Module named in %s" % lang) + @unittest.skipIf(not hasattr(sys, 'gettotalrefcount'), + '--with-pydebug has to be enabled for this test') + def test_bad_traverse(self): + ''' Issue #32374: Test that traverse fails when accessing per-module + state before Py_mod_exec was executed. + (Multiphase initialization modules only) + ''' + script = """if True: + import importlib.util as util + spec = util.find_spec('_testmultiphase') + spec.name = '_testmultiphase_with_bad_traverse' + m = spec.loader.create_module(spec)""" + assert_python_failure("-c", script) + (Frozen_MultiPhaseExtensionModuleTests, Source_MultiPhaseExtensionModuleTests diff --git a/Misc/NEWS.d/next/C API/2018-01-09-17-03-54.bpo-32374.SwwLoz.rst b/Misc/NEWS.d/next/C API/2018-01-09-17-03-54.bpo-32374.SwwLoz.rst new file mode 100644 index 000000000000..f9cf6d6b99ce --- /dev/null +++ b/Misc/NEWS.d/next/C API/2018-01-09-17-03-54.bpo-32374.SwwLoz.rst @@ -0,0 +1,2 @@ +Document that m_traverse for multi-phase initialized modules can be called +with m_state=NULL, and add a sanity check diff --git a/Modules/_testmultiphase.c b/Modules/_testmultiphase.c index 9b04ebf15586..8090a485f4a9 100644 --- a/Modules/_testmultiphase.c +++ b/Modules/_testmultiphase.c @@ -10,6 +10,10 @@ typedef struct { PyObject *x_attr; /* Attributes dictionary */ } ExampleObject; +typedef struct { + PyObject *integer; +} testmultiphase_state; + /* Example methods */ static int @@ -218,18 +222,21 @@ static int execfunc(PyObject *m) } /* Helper for module definitions; there'll be a lot of them */ -#define TEST_MODULE_DEF(name, slots, methods) { \ + +#define TEST_MODULE_DEF_EX(name, slots, methods, statesize, traversefunc) { \ PyModuleDef_HEAD_INIT, /* m_base */ \ name, /* m_name */ \ PyDoc_STR("Test module " name), /* m_doc */ \ - 0, /* m_size */ \ + statesize, /* m_size */ \ methods, /* m_methods */ \ slots, /* m_slots */ \ - NULL, /* m_traverse */ \ + traversefunc, /* m_traverse */ \ NULL, /* m_clear */ \ NULL, /* m_free */ \ } +#define TEST_MODULE_DEF(name, slots, methods) TEST_MODULE_DEF_EX(name, slots, methods, 0, NULL) + PyModuleDef_Slot main_slots[] = { {Py_mod_exec, execfunc}, {0, NULL}, @@ -613,6 +620,44 @@ PyInit__testmultiphase_exec_unreported_exception(PyObject *spec) return PyModuleDef_Init(&def_exec_unreported_exception); } +static int +bad_traverse(PyObject *self, visitproc visit, void *arg) { + testmultiphase_state *m_state; + + m_state = PyModule_GetState(self); + Py_VISIT(m_state->integer); + return 0; +} + +static int +execfunc_with_bad_traverse(PyObject *mod) { + testmultiphase_state *m_state; + + m_state = PyModule_GetState(mod); + if (m_state == NULL) { + return -1; + } + + m_state->integer = PyLong_FromLong(0x7fffffff); + Py_INCREF(m_state->integer); + + return 0; +} + +static PyModuleDef_Slot slots_with_bad_traverse[] = { + {Py_mod_exec, execfunc_with_bad_traverse}, + {0, NULL} +}; + +static PyModuleDef def_with_bad_traverse = TEST_MODULE_DEF_EX( + "_testmultiphase_with_bad_traverse", slots_with_bad_traverse, NULL, + sizeof(testmultiphase_state), bad_traverse); + +PyMODINIT_FUNC +PyInit__testmultiphase_with_bad_traverse(PyObject *spec) { + return PyModuleDef_Init(&def_with_bad_traverse); +} + /*** Helper for imp test ***/ static PyModuleDef imp_dummy_def = TEST_MODULE_DEF("imp_dummy", main_slots, testexport_methods); @@ -622,3 +667,4 @@ PyInit_imp_dummy(PyObject *spec) { return PyModuleDef_Init(&imp_dummy_def); } + diff --git a/Objects/moduleobject.c b/Objects/moduleobject.c index d6cde4024336..8fb368e41474 100644 --- a/Objects/moduleobject.c +++ b/Objects/moduleobject.c @@ -21,6 +21,17 @@ static PyMemberDef module_members[] = { {0} }; + +/* Helper for sanity check for traverse not handling m_state == NULL + * Issue #32374 */ +#ifdef Py_DEBUG +static int +bad_traverse_test(PyObject *self, void *arg) { + assert(self != NULL); + return 0; +} +#endif + PyTypeObject PyModuleDef_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) "moduledef", /* tp_name */ @@ -345,6 +356,16 @@ PyModule_FromDefAndSpec2(struct PyModuleDef* def, PyObject *spec, int module_api } } + /* Sanity check for traverse not handling m_state == NULL + * This doesn't catch all possible cases, but in many cases it should + * make many cases of invalid code crash or raise Valgrind issues + * sooner than they would otherwise. + * Issue #32374 */ +#ifdef Py_DEBUG + if (def->m_traverse != NULL) { + def->m_traverse(m, bad_traverse_test, NULL); + } +#endif Py_DECREF(nameobj); return m; From webhook-mailer at python.org Sat Mar 17 02:03:59 2018 From: webhook-mailer at python.org (Miss Islington (bot)) Date: Sat, 17 Mar 2018 06:03:59 -0000 Subject: [Python-checkins] bpo-32374: m_traverse may be called with m_state=NULL (GH-5140) Message-ID: <mailman.79.1521266641.1871.python-checkins@python.org> https://github.com/python/cpython/commit/136905fffd5f77395f80e3409630c11756b5469c commit: 136905fffd5f77395f80e3409630c11756b5469c branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: GitHub <noreply at github.com> date: 2018-03-16T23:03:56-07:00 summary: bpo-32374: m_traverse may be called with m_state=NULL (GH-5140) Multi-phase initialized modules allow m_traverse to be called while the module is still being initialized, so module authors may need to account for that. (cherry picked from commit c2b0b12d1a137ada1023ab7c10b8d9a0249d95f9) Co-authored-by: Marcel Plch <gmarcel.plch at gmail.com> files: A Misc/NEWS.d/next/C API/2018-01-09-17-03-54.bpo-32374.SwwLoz.rst M Doc/c-api/module.rst M Lib/test/test_importlib/extension/test_loader.py M Modules/_testmultiphase.c M Objects/moduleobject.c diff --git a/Doc/c-api/module.rst b/Doc/c-api/module.rst index 7efab28af724..017b656854a8 100644 --- a/Doc/c-api/module.rst +++ b/Doc/c-api/module.rst @@ -196,17 +196,23 @@ or request "multi-phase initialization" by returning the definition struct itsel .. c:member:: traverseproc m_traverse A traversal function to call during GC traversal of the module object, or - *NULL* if not needed. + *NULL* if not needed. This function may be called before module state + is allocated (:c:func:`PyModule_GetState()` may return `NULL`), + and before the :c:member:`Py_mod_exec` function is executed. .. c:member:: inquiry m_clear A clear function to call during GC clearing of the module object, or - *NULL* if not needed. + *NULL* if not needed. This function may be called before module state + is allocated (:c:func:`PyModule_GetState()` may return `NULL`), + and before the :c:member:`Py_mod_exec` function is executed. .. c:member:: freefunc m_free A function to call during deallocation of the module object, or *NULL* if - not needed. + not needed. This function may be called before module state + is allocated (:c:func:`PyModule_GetState()` may return `NULL`), + and before the :c:member:`Py_mod_exec` function is executed. Single-phase initialization ........................... diff --git a/Lib/test/test_importlib/extension/test_loader.py b/Lib/test/test_importlib/extension/test_loader.py index 8d20040aab8d..53ac3c71d4b5 100644 --- a/Lib/test/test_importlib/extension/test_loader.py +++ b/Lib/test/test_importlib/extension/test_loader.py @@ -9,7 +9,7 @@ import unittest import importlib.util import importlib - +from test.support.script_helper import assert_python_failure class LoaderTests(abc.LoaderTests): @@ -267,6 +267,20 @@ def test_nonascii(self): self.assertEqual(module.__name__, name) self.assertEqual(module.__doc__, "Module named in %s" % lang) + @unittest.skipIf(not hasattr(sys, 'gettotalrefcount'), + '--with-pydebug has to be enabled for this test') + def test_bad_traverse(self): + ''' Issue #32374: Test that traverse fails when accessing per-module + state before Py_mod_exec was executed. + (Multiphase initialization modules only) + ''' + script = """if True: + import importlib.util as util + spec = util.find_spec('_testmultiphase') + spec.name = '_testmultiphase_with_bad_traverse' + m = spec.loader.create_module(spec)""" + assert_python_failure("-c", script) + (Frozen_MultiPhaseExtensionModuleTests, Source_MultiPhaseExtensionModuleTests diff --git a/Misc/NEWS.d/next/C API/2018-01-09-17-03-54.bpo-32374.SwwLoz.rst b/Misc/NEWS.d/next/C API/2018-01-09-17-03-54.bpo-32374.SwwLoz.rst new file mode 100644 index 000000000000..f9cf6d6b99ce --- /dev/null +++ b/Misc/NEWS.d/next/C API/2018-01-09-17-03-54.bpo-32374.SwwLoz.rst @@ -0,0 +1,2 @@ +Document that m_traverse for multi-phase initialized modules can be called +with m_state=NULL, and add a sanity check diff --git a/Modules/_testmultiphase.c b/Modules/_testmultiphase.c index 9b04ebf15586..8090a485f4a9 100644 --- a/Modules/_testmultiphase.c +++ b/Modules/_testmultiphase.c @@ -10,6 +10,10 @@ typedef struct { PyObject *x_attr; /* Attributes dictionary */ } ExampleObject; +typedef struct { + PyObject *integer; +} testmultiphase_state; + /* Example methods */ static int @@ -218,18 +222,21 @@ static int execfunc(PyObject *m) } /* Helper for module definitions; there'll be a lot of them */ -#define TEST_MODULE_DEF(name, slots, methods) { \ + +#define TEST_MODULE_DEF_EX(name, slots, methods, statesize, traversefunc) { \ PyModuleDef_HEAD_INIT, /* m_base */ \ name, /* m_name */ \ PyDoc_STR("Test module " name), /* m_doc */ \ - 0, /* m_size */ \ + statesize, /* m_size */ \ methods, /* m_methods */ \ slots, /* m_slots */ \ - NULL, /* m_traverse */ \ + traversefunc, /* m_traverse */ \ NULL, /* m_clear */ \ NULL, /* m_free */ \ } +#define TEST_MODULE_DEF(name, slots, methods) TEST_MODULE_DEF_EX(name, slots, methods, 0, NULL) + PyModuleDef_Slot main_slots[] = { {Py_mod_exec, execfunc}, {0, NULL}, @@ -613,6 +620,44 @@ PyInit__testmultiphase_exec_unreported_exception(PyObject *spec) return PyModuleDef_Init(&def_exec_unreported_exception); } +static int +bad_traverse(PyObject *self, visitproc visit, void *arg) { + testmultiphase_state *m_state; + + m_state = PyModule_GetState(self); + Py_VISIT(m_state->integer); + return 0; +} + +static int +execfunc_with_bad_traverse(PyObject *mod) { + testmultiphase_state *m_state; + + m_state = PyModule_GetState(mod); + if (m_state == NULL) { + return -1; + } + + m_state->integer = PyLong_FromLong(0x7fffffff); + Py_INCREF(m_state->integer); + + return 0; +} + +static PyModuleDef_Slot slots_with_bad_traverse[] = { + {Py_mod_exec, execfunc_with_bad_traverse}, + {0, NULL} +}; + +static PyModuleDef def_with_bad_traverse = TEST_MODULE_DEF_EX( + "_testmultiphase_with_bad_traverse", slots_with_bad_traverse, NULL, + sizeof(testmultiphase_state), bad_traverse); + +PyMODINIT_FUNC +PyInit__testmultiphase_with_bad_traverse(PyObject *spec) { + return PyModuleDef_Init(&def_with_bad_traverse); +} + /*** Helper for imp test ***/ static PyModuleDef imp_dummy_def = TEST_MODULE_DEF("imp_dummy", main_slots, testexport_methods); @@ -622,3 +667,4 @@ PyInit_imp_dummy(PyObject *spec) { return PyModuleDef_Init(&imp_dummy_def); } + diff --git a/Objects/moduleobject.c b/Objects/moduleobject.c index d6cde4024336..8fb368e41474 100644 --- a/Objects/moduleobject.c +++ b/Objects/moduleobject.c @@ -21,6 +21,17 @@ static PyMemberDef module_members[] = { {0} }; + +/* Helper for sanity check for traverse not handling m_state == NULL + * Issue #32374 */ +#ifdef Py_DEBUG +static int +bad_traverse_test(PyObject *self, void *arg) { + assert(self != NULL); + return 0; +} +#endif + PyTypeObject PyModuleDef_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) "moduledef", /* tp_name */ @@ -345,6 +356,16 @@ PyModule_FromDefAndSpec2(struct PyModuleDef* def, PyObject *spec, int module_api } } + /* Sanity check for traverse not handling m_state == NULL + * This doesn't catch all possible cases, but in many cases it should + * make many cases of invalid code crash or raise Valgrind issues + * sooner than they would otherwise. + * Issue #32374 */ +#ifdef Py_DEBUG + if (def->m_traverse != NULL) { + def->m_traverse(m, bad_traverse_test, NULL); + } +#endif Py_DECREF(nameobj); return m; From webhook-mailer at python.org Sat Mar 17 02:29:32 2018 From: webhook-mailer at python.org (Miss Islington (bot)) Date: Sat, 17 Mar 2018 06:29:32 -0000 Subject: [Python-checkins] bpo-32374: m_traverse may be called with m_state=NULL (GH-5140) Message-ID: <mailman.80.1521268175.1871.python-checkins@python.org> https://github.com/python/cpython/commit/1da0479f687613a43620430616f4db87e2ed4423 commit: 1da0479f687613a43620430616f4db87e2ed4423 branch: 3.6 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: GitHub <noreply at github.com> date: 2018-03-16T23:29:30-07:00 summary: bpo-32374: m_traverse may be called with m_state=NULL (GH-5140) Multi-phase initialized modules allow m_traverse to be called while the module is still being initialized, so module authors may need to account for that. (cherry picked from commit c2b0b12d1a137ada1023ab7c10b8d9a0249d95f9) Co-authored-by: Marcel Plch <gmarcel.plch at gmail.com> files: A Misc/NEWS.d/next/C API/2018-01-09-17-03-54.bpo-32374.SwwLoz.rst M Doc/c-api/module.rst M Lib/test/test_importlib/extension/test_loader.py M Modules/_testmultiphase.c M Objects/moduleobject.c diff --git a/Doc/c-api/module.rst b/Doc/c-api/module.rst index d3125b86f072..797a67e49c72 100644 --- a/Doc/c-api/module.rst +++ b/Doc/c-api/module.rst @@ -196,17 +196,23 @@ or request "multi-phase initialization" by returning the definition struct itsel .. c:member:: traverseproc m_traverse A traversal function to call during GC traversal of the module object, or - *NULL* if not needed. + *NULL* if not needed. This function may be called before module state + is allocated (:c:func:`PyModule_GetState()` may return `NULL`), + and before the :c:member:`Py_mod_exec` function is executed. .. c:member:: inquiry m_clear A clear function to call during GC clearing of the module object, or - *NULL* if not needed. + *NULL* if not needed. This function may be called before module state + is allocated (:c:func:`PyModule_GetState()` may return `NULL`), + and before the :c:member:`Py_mod_exec` function is executed. .. c:member:: freefunc m_free A function to call during deallocation of the module object, or *NULL* if - not needed. + not needed. This function may be called before module state + is allocated (:c:func:`PyModule_GetState()` may return `NULL`), + and before the :c:member:`Py_mod_exec` function is executed. Single-phase initialization ........................... diff --git a/Lib/test/test_importlib/extension/test_loader.py b/Lib/test/test_importlib/extension/test_loader.py index 8d20040aab8d..53ac3c71d4b5 100644 --- a/Lib/test/test_importlib/extension/test_loader.py +++ b/Lib/test/test_importlib/extension/test_loader.py @@ -9,7 +9,7 @@ import unittest import importlib.util import importlib - +from test.support.script_helper import assert_python_failure class LoaderTests(abc.LoaderTests): @@ -267,6 +267,20 @@ def test_nonascii(self): self.assertEqual(module.__name__, name) self.assertEqual(module.__doc__, "Module named in %s" % lang) + @unittest.skipIf(not hasattr(sys, 'gettotalrefcount'), + '--with-pydebug has to be enabled for this test') + def test_bad_traverse(self): + ''' Issue #32374: Test that traverse fails when accessing per-module + state before Py_mod_exec was executed. + (Multiphase initialization modules only) + ''' + script = """if True: + import importlib.util as util + spec = util.find_spec('_testmultiphase') + spec.name = '_testmultiphase_with_bad_traverse' + m = spec.loader.create_module(spec)""" + assert_python_failure("-c", script) + (Frozen_MultiPhaseExtensionModuleTests, Source_MultiPhaseExtensionModuleTests diff --git a/Misc/NEWS.d/next/C API/2018-01-09-17-03-54.bpo-32374.SwwLoz.rst b/Misc/NEWS.d/next/C API/2018-01-09-17-03-54.bpo-32374.SwwLoz.rst new file mode 100644 index 000000000000..f9cf6d6b99ce --- /dev/null +++ b/Misc/NEWS.d/next/C API/2018-01-09-17-03-54.bpo-32374.SwwLoz.rst @@ -0,0 +1,2 @@ +Document that m_traverse for multi-phase initialized modules can be called +with m_state=NULL, and add a sanity check diff --git a/Modules/_testmultiphase.c b/Modules/_testmultiphase.c index 4daa34e10614..8891d8d21dfc 100644 --- a/Modules/_testmultiphase.c +++ b/Modules/_testmultiphase.c @@ -10,6 +10,10 @@ typedef struct { PyObject *x_attr; /* Attributes dictionary */ } ExampleObject; +typedef struct { + PyObject *integer; +} testmultiphase_state; + /* Example methods */ static int @@ -219,18 +223,21 @@ static int execfunc(PyObject *m) } /* Helper for module definitions; there'll be a lot of them */ -#define TEST_MODULE_DEF(name, slots, methods) { \ + +#define TEST_MODULE_DEF_EX(name, slots, methods, statesize, traversefunc) { \ PyModuleDef_HEAD_INIT, /* m_base */ \ name, /* m_name */ \ PyDoc_STR("Test module " name), /* m_doc */ \ - 0, /* m_size */ \ + statesize, /* m_size */ \ methods, /* m_methods */ \ slots, /* m_slots */ \ - NULL, /* m_traverse */ \ + traversefunc, /* m_traverse */ \ NULL, /* m_clear */ \ NULL, /* m_free */ \ } +#define TEST_MODULE_DEF(name, slots, methods) TEST_MODULE_DEF_EX(name, slots, methods, 0, NULL) + PyModuleDef_Slot main_slots[] = { {Py_mod_exec, execfunc}, {0, NULL}, @@ -614,6 +621,44 @@ PyInit__testmultiphase_exec_unreported_exception(PyObject *spec) return PyModuleDef_Init(&def_exec_unreported_exception); } +static int +bad_traverse(PyObject *self, visitproc visit, void *arg) { + testmultiphase_state *m_state; + + m_state = PyModule_GetState(self); + Py_VISIT(m_state->integer); + return 0; +} + +static int +execfunc_with_bad_traverse(PyObject *mod) { + testmultiphase_state *m_state; + + m_state = PyModule_GetState(mod); + if (m_state == NULL) { + return -1; + } + + m_state->integer = PyLong_FromLong(0x7fffffff); + Py_INCREF(m_state->integer); + + return 0; +} + +static PyModuleDef_Slot slots_with_bad_traverse[] = { + {Py_mod_exec, execfunc_with_bad_traverse}, + {0, NULL} +}; + +static PyModuleDef def_with_bad_traverse = TEST_MODULE_DEF_EX( + "_testmultiphase_with_bad_traverse", slots_with_bad_traverse, NULL, + sizeof(testmultiphase_state), bad_traverse); + +PyMODINIT_FUNC +PyInit__testmultiphase_with_bad_traverse(PyObject *spec) { + return PyModuleDef_Init(&def_with_bad_traverse); +} + /*** Helper for imp test ***/ static PyModuleDef imp_dummy_def = TEST_MODULE_DEF("imp_dummy", main_slots, testexport_methods); @@ -623,3 +668,4 @@ PyInit_imp_dummy(PyObject *spec) { return PyModuleDef_Init(&imp_dummy_def); } + diff --git a/Objects/moduleobject.c b/Objects/moduleobject.c index f357af2b2118..c5979329647b 100644 --- a/Objects/moduleobject.c +++ b/Objects/moduleobject.c @@ -20,6 +20,17 @@ static PyMemberDef module_members[] = { {0} }; + +/* Helper for sanity check for traverse not handling m_state == NULL + * Issue #32374 */ +#ifdef Py_DEBUG +static int +bad_traverse_test(PyObject *self, void *arg) { + assert(self != NULL); + return 0; +} +#endif + PyTypeObject PyModuleDef_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) "moduledef", /* tp_name */ @@ -338,6 +349,16 @@ PyModule_FromDefAndSpec2(struct PyModuleDef* def, PyObject *spec, int module_api } } + /* Sanity check for traverse not handling m_state == NULL + * This doesn't catch all possible cases, but in many cases it should + * make many cases of invalid code crash or raise Valgrind issues + * sooner than they would otherwise. + * Issue #32374 */ +#ifdef Py_DEBUG + if (def->m_traverse != NULL) { + def->m_traverse(m, bad_traverse_test, NULL); + } +#endif Py_DECREF(nameobj); return m; From solipsis at pitrou.net Sat Mar 17 05:15:24 2018 From: solipsis at pitrou.net (solipsis at pitrou.net) Date: Sat, 17 Mar 2018 09:15:24 +0000 Subject: [Python-checkins] Daily reference leaks (4243df51fe43): sum=12 Message-ID: <20180317091524.1.DFC2FCA6FA8B5732@psf.io> results for 4243df51fe43 on branch "default" -------------------------------------------- test_collections leaked [7, -7, 8] memory blocks, sum=8 test_functools leaked [0, 3, 1] memory blocks, sum=4 Command line was: ['./python', '-m', 'test.regrtest', '-uall', '-R', '3:3:/home/psf-users/antoine/refleaks/reflog5_4zyE', '--timeout', '7200'] From webhook-mailer at python.org Sat Mar 17 11:13:51 2018 From: webhook-mailer at python.org (Donald Stufft) Date: Sat, 17 Mar 2018 15:13:51 -0000 Subject: [Python-checkins] Update pip to 9.0.2 and setuptools to 38.6.1 (#6133) Message-ID: <mailman.81.1521299632.1871.python-checkins@python.org> https://github.com/python/cpython/commit/7f81bb2addbbccfa45a2fc1aa6030f26dcf4bd78 commit: 7f81bb2addbbccfa45a2fc1aa6030f26dcf4bd78 branch: master author: Donald Stufft <donald at stufft.io> committer: GitHub <noreply at github.com> date: 2018-03-17T11:13:48-04:00 summary: Update pip to 9.0.2 and setuptools to 38.6.1 (#6133) files: A Lib/ensurepip/_bundled/pip-9.0.2-py2.py3-none-any.whl A Lib/ensurepip/_bundled/setuptools-38.6.1-py2.py3-none-any.whl D Lib/ensurepip/_bundled/pip-9.0.1-py2.py3-none-any.whl D Lib/ensurepip/_bundled/setuptools-28.8.0-py2.py3-none-any.whl M Lib/ensurepip/__init__.py diff --git a/Lib/ensurepip/__init__.py b/Lib/ensurepip/__init__.py index d69e09fab084..d7a6dcf6bc85 100644 --- a/Lib/ensurepip/__init__.py +++ b/Lib/ensurepip/__init__.py @@ -8,9 +8,9 @@ __all__ = ["version", "bootstrap"] -_SETUPTOOLS_VERSION = "28.8.0" +_SETUPTOOLS_VERSION = "38.6.1" -_PIP_VERSION = "9.0.1" +_PIP_VERSION = "9.0.2" _PROJECTS = [ ("setuptools", _SETUPTOOLS_VERSION), diff --git a/Lib/ensurepip/_bundled/pip-9.0.1-py2.py3-none-any.whl b/Lib/ensurepip/_bundled/pip-9.0.2-py2.py3-none-any.whl similarity index 54% rename from Lib/ensurepip/_bundled/pip-9.0.1-py2.py3-none-any.whl rename to Lib/ensurepip/_bundled/pip-9.0.2-py2.py3-none-any.whl index 4b8ecc69db7e..baf1f57e3d8b 100644 Binary files a/Lib/ensurepip/_bundled/pip-9.0.1-py2.py3-none-any.whl and b/Lib/ensurepip/_bundled/pip-9.0.2-py2.py3-none-any.whl differ diff --git a/Lib/ensurepip/_bundled/setuptools-28.8.0-py2.py3-none-any.whl b/Lib/ensurepip/_bundled/setuptools-38.6.1-py2.py3-none-any.whl similarity index 65% rename from Lib/ensurepip/_bundled/setuptools-28.8.0-py2.py3-none-any.whl rename to Lib/ensurepip/_bundled/setuptools-38.6.1-py2.py3-none-any.whl index 502e3cb418c1..2707d8be4a2d 100644 Binary files a/Lib/ensurepip/_bundled/setuptools-28.8.0-py2.py3-none-any.whl and b/Lib/ensurepip/_bundled/setuptools-38.6.1-py2.py3-none-any.whl differ From webhook-mailer at python.org Sat Mar 17 11:46:08 2018 From: webhook-mailer at python.org (Donald Stufft) Date: Sat, 17 Mar 2018 15:46:08 -0000 Subject: [Python-checkins] Update pip to 9.0.2 and setuptools to 38.6.1 (GH-6133) (#6134) Message-ID: <mailman.82.1521301569.1871.python-checkins@python.org> https://github.com/python/cpython/commit/d7b81037412291fb08d8dbdc5249d670e94c645a commit: d7b81037412291fb08d8dbdc5249d670e94c645a branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Donald Stufft <donald at stufft.io> date: 2018-03-17T11:46:05-04:00 summary: Update pip to 9.0.2 and setuptools to 38.6.1 (GH-6133) (#6134) (cherry picked from commit 7f81bb2addbbccfa45a2fc1aa6030f26dcf4bd78) Co-authored-by: Donald Stufft <donald at stufft.io> files: A Lib/ensurepip/_bundled/pip-9.0.2-py2.py3-none-any.whl A Lib/ensurepip/_bundled/setuptools-38.6.1-py2.py3-none-any.whl D Lib/ensurepip/_bundled/pip-9.0.1-py2.py3-none-any.whl D Lib/ensurepip/_bundled/setuptools-28.8.0-py2.py3-none-any.whl M Lib/ensurepip/__init__.py diff --git a/Lib/ensurepip/__init__.py b/Lib/ensurepip/__init__.py index d69e09fab084..d7a6dcf6bc85 100644 --- a/Lib/ensurepip/__init__.py +++ b/Lib/ensurepip/__init__.py @@ -8,9 +8,9 @@ __all__ = ["version", "bootstrap"] -_SETUPTOOLS_VERSION = "28.8.0" +_SETUPTOOLS_VERSION = "38.6.1" -_PIP_VERSION = "9.0.1" +_PIP_VERSION = "9.0.2" _PROJECTS = [ ("setuptools", _SETUPTOOLS_VERSION), diff --git a/Lib/ensurepip/_bundled/pip-9.0.1-py2.py3-none-any.whl b/Lib/ensurepip/_bundled/pip-9.0.2-py2.py3-none-any.whl similarity index 54% rename from Lib/ensurepip/_bundled/pip-9.0.1-py2.py3-none-any.whl rename to Lib/ensurepip/_bundled/pip-9.0.2-py2.py3-none-any.whl index 4b8ecc69db7e..baf1f57e3d8b 100644 Binary files a/Lib/ensurepip/_bundled/pip-9.0.1-py2.py3-none-any.whl and b/Lib/ensurepip/_bundled/pip-9.0.2-py2.py3-none-any.whl differ diff --git a/Lib/ensurepip/_bundled/setuptools-28.8.0-py2.py3-none-any.whl b/Lib/ensurepip/_bundled/setuptools-38.6.1-py2.py3-none-any.whl similarity index 65% rename from Lib/ensurepip/_bundled/setuptools-28.8.0-py2.py3-none-any.whl rename to Lib/ensurepip/_bundled/setuptools-38.6.1-py2.py3-none-any.whl index 502e3cb418c1..2707d8be4a2d 100644 Binary files a/Lib/ensurepip/_bundled/setuptools-28.8.0-py2.py3-none-any.whl and b/Lib/ensurepip/_bundled/setuptools-38.6.1-py2.py3-none-any.whl differ From webhook-mailer at python.org Sat Mar 17 11:54:49 2018 From: webhook-mailer at python.org (Donald Stufft) Date: Sat, 17 Mar 2018 15:54:49 -0000 Subject: [Python-checkins] Update pip to 9.0.2 and setuptools to 38.6.1 (GH-6133) (GH-6135) Message-ID: <mailman.83.1521302090.1871.python-checkins@python.org> https://github.com/python/cpython/commit/4e907d8fafde6b8df9b8b461d9a6fbbf3b8674bd commit: 4e907d8fafde6b8df9b8b461d9a6fbbf3b8674bd branch: 2.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Donald Stufft <donald at stufft.io> date: 2018-03-17T11:54:45-04:00 summary: Update pip to 9.0.2 and setuptools to 38.6.1 (GH-6133) (GH-6135) (cherry picked from commit 7f81bb2addbbccfa45a2fc1aa6030f26dcf4bd78) Co-authored-by: Donald Stufft <donald at stufft.io> files: A Lib/ensurepip/_bundled/pip-9.0.2-py2.py3-none-any.whl A Lib/ensurepip/_bundled/setuptools-38.6.1-py2.py3-none-any.whl D Lib/ensurepip/_bundled/pip-9.0.1-py2.py3-none-any.whl D Lib/ensurepip/_bundled/setuptools-28.8.0-py2.py3-none-any.whl M Lib/ensurepip/__init__.py diff --git a/Lib/ensurepip/__init__.py b/Lib/ensurepip/__init__.py index ea2a5e634822..167696720267 100644 --- a/Lib/ensurepip/__init__.py +++ b/Lib/ensurepip/__init__.py @@ -12,9 +12,9 @@ __all__ = ["version", "bootstrap"] -_SETUPTOOLS_VERSION = "28.8.0" +_SETUPTOOLS_VERSION = "38.6.1" -_PIP_VERSION = "9.0.1" +_PIP_VERSION = "9.0.2" _PROJECTS = [ ("setuptools", _SETUPTOOLS_VERSION), diff --git a/Lib/ensurepip/_bundled/pip-9.0.1-py2.py3-none-any.whl b/Lib/ensurepip/_bundled/pip-9.0.2-py2.py3-none-any.whl similarity index 54% rename from Lib/ensurepip/_bundled/pip-9.0.1-py2.py3-none-any.whl rename to Lib/ensurepip/_bundled/pip-9.0.2-py2.py3-none-any.whl index 4b8ecc69db7e..baf1f57e3d8b 100644 Binary files a/Lib/ensurepip/_bundled/pip-9.0.1-py2.py3-none-any.whl and b/Lib/ensurepip/_bundled/pip-9.0.2-py2.py3-none-any.whl differ diff --git a/Lib/ensurepip/_bundled/setuptools-28.8.0-py2.py3-none-any.whl b/Lib/ensurepip/_bundled/setuptools-38.6.1-py2.py3-none-any.whl similarity index 65% rename from Lib/ensurepip/_bundled/setuptools-28.8.0-py2.py3-none-any.whl rename to Lib/ensurepip/_bundled/setuptools-38.6.1-py2.py3-none-any.whl index 502e3cb418c1..2707d8be4a2d 100644 Binary files a/Lib/ensurepip/_bundled/setuptools-28.8.0-py2.py3-none-any.whl and b/Lib/ensurepip/_bundled/setuptools-38.6.1-py2.py3-none-any.whl differ From webhook-mailer at python.org Sat Mar 17 12:19:45 2018 From: webhook-mailer at python.org (Donald Stufft) Date: Sat, 17 Mar 2018 16:19:45 -0000 Subject: [Python-checkins] Update pip to 9.0.2 and setuptools to 38.6.1 (GH-6133) (#6136) Message-ID: <mailman.84.1521303586.1871.python-checkins@python.org> https://github.com/python/cpython/commit/5bd0fe6a00be51d76c9ed98bd38ccc0237861250 commit: 5bd0fe6a00be51d76c9ed98bd38ccc0237861250 branch: 3.6 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Donald Stufft <donald at stufft.io> date: 2018-03-17T12:19:41-04:00 summary: Update pip to 9.0.2 and setuptools to 38.6.1 (GH-6133) (#6136) (cherry picked from commit 7f81bb2addbbccfa45a2fc1aa6030f26dcf4bd78) Co-authored-by: Donald Stufft <donald at stufft.io> files: A Lib/ensurepip/_bundled/pip-9.0.2-py2.py3-none-any.whl A Lib/ensurepip/_bundled/setuptools-38.6.1-py2.py3-none-any.whl D Lib/ensurepip/_bundled/pip-9.0.1-py2.py3-none-any.whl D Lib/ensurepip/_bundled/setuptools-28.8.0-py2.py3-none-any.whl M Lib/ensurepip/__init__.py diff --git a/Lib/ensurepip/__init__.py b/Lib/ensurepip/__init__.py index d69e09fab084..d7a6dcf6bc85 100644 --- a/Lib/ensurepip/__init__.py +++ b/Lib/ensurepip/__init__.py @@ -8,9 +8,9 @@ __all__ = ["version", "bootstrap"] -_SETUPTOOLS_VERSION = "28.8.0" +_SETUPTOOLS_VERSION = "38.6.1" -_PIP_VERSION = "9.0.1" +_PIP_VERSION = "9.0.2" _PROJECTS = [ ("setuptools", _SETUPTOOLS_VERSION), diff --git a/Lib/ensurepip/_bundled/pip-9.0.1-py2.py3-none-any.whl b/Lib/ensurepip/_bundled/pip-9.0.2-py2.py3-none-any.whl similarity index 54% rename from Lib/ensurepip/_bundled/pip-9.0.1-py2.py3-none-any.whl rename to Lib/ensurepip/_bundled/pip-9.0.2-py2.py3-none-any.whl index 4b8ecc69db7e..baf1f57e3d8b 100644 Binary files a/Lib/ensurepip/_bundled/pip-9.0.1-py2.py3-none-any.whl and b/Lib/ensurepip/_bundled/pip-9.0.2-py2.py3-none-any.whl differ diff --git a/Lib/ensurepip/_bundled/setuptools-28.8.0-py2.py3-none-any.whl b/Lib/ensurepip/_bundled/setuptools-38.6.1-py2.py3-none-any.whl similarity index 65% rename from Lib/ensurepip/_bundled/setuptools-28.8.0-py2.py3-none-any.whl rename to Lib/ensurepip/_bundled/setuptools-38.6.1-py2.py3-none-any.whl index 502e3cb418c1..2707d8be4a2d 100644 Binary files a/Lib/ensurepip/_bundled/setuptools-28.8.0-py2.py3-none-any.whl and b/Lib/ensurepip/_bundled/setuptools-38.6.1-py2.py3-none-any.whl differ From webhook-mailer at python.org Sat Mar 17 20:49:04 2018 From: webhook-mailer at python.org (Berker Peksag) Date: Sun, 18 Mar 2018 00:49:04 -0000 Subject: [Python-checkins] bpo-27645: Fix version number in 'database in transaction' fallback (GH-6131) Message-ID: <mailman.85.1521334145.1871.python-checkins@python.org> https://github.com/python/cpython/commit/bbf7bb7a636b3112ef6f6b31df385606d52517ce commit: bbf7bb7a636b3112ef6f6b31df385606d52517ce branch: master author: Aviv Palivoda <palaviv at gmail.com> committer: Berker Peksag <berker.peksag at gmail.com> date: 2018-03-18T03:48:55+03:00 summary: bpo-27645: Fix version number in 'database in transaction' fallback (GH-6131) It was actually fixed in SQLite 3.8.8, not 3.8.7. files: M Lib/sqlite3/test/backup.py M Modules/_sqlite/connection.c diff --git a/Lib/sqlite3/test/backup.py b/Lib/sqlite3/test/backup.py index 2a2148937413..903bacf49030 100644 --- a/Lib/sqlite3/test/backup.py +++ b/Lib/sqlite3/test/backup.py @@ -37,14 +37,12 @@ def test_bad_target_closed_connection(self): self.cx.backup(bck) def test_bad_target_in_transaction(self): - if sqlite.sqlite_version_info == (3, 8, 7, 1): - self.skipTest('skip until we debug https://bugs.python.org/issue27645#msg313562') bck = sqlite.connect(':memory:') bck.execute('CREATE TABLE bar (key INTEGER)') bck.executemany('INSERT INTO bar (key) VALUES (?)', [(3,), (4,)]) with self.assertRaises(sqlite.OperationalError) as cm: self.cx.backup(bck) - if sqlite.sqlite_version_info < (3, 8, 7): + if sqlite.sqlite_version_info < (3, 8, 8): self.assertEqual(str(cm.exception), 'target is in transaction') def test_keyword_only_args(self): diff --git a/Modules/_sqlite/connection.c b/Modules/_sqlite/connection.c index 14b6a2774e78..dc5061c1d91d 100644 --- a/Modules/_sqlite/connection.c +++ b/Modules/_sqlite/connection.c @@ -1481,8 +1481,8 @@ pysqlite_connection_backup(pysqlite_Connection *self, PyObject *args, PyObject * return NULL; } -#if SQLITE_VERSION_NUMBER < 3008007 - /* Since 3.8.7 this is already done, per commit +#if SQLITE_VERSION_NUMBER < 3008008 + /* Since 3.8.8 this is already done, per commit https://www.sqlite.org/src/info/169b5505498c0a7e */ if (!sqlite3_get_autocommit(((pysqlite_Connection *)target)->db)) { PyErr_SetString(pysqlite_OperationalError, "target is in transaction"); From webhook-mailer at python.org Sun Mar 18 02:24:42 2018 From: webhook-mailer at python.org (Berker Peksag) Date: Sun, 18 Mar 2018 06:24:42 -0000 Subject: [Python-checkins] bpo-27645: Fix version number in 'database in transaction' fallback (GH-6131) Message-ID: <mailman.86.1521354284.1871.python-checkins@python.org> https://github.com/python/cpython/commit/429ca448d2a36040f229ad9edc67e31fc6d18bf4 commit: 429ca448d2a36040f229ad9edc67e31fc6d18bf4 branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Berker Peksag <berker.peksag at gmail.com> date: 2018-03-18T09:24:33+03:00 summary: bpo-27645: Fix version number in 'database in transaction' fallback (GH-6131) It was actually fixed in SQLite 3.8.8, not 3.8.7. (cherry picked from commit bbf7bb7a636b3112ef6f6b31df385606d52517ce) Co-authored-by: Aviv Palivoda <palaviv at gmail.com> files: M Lib/sqlite3/test/backup.py M Modules/_sqlite/connection.c diff --git a/Lib/sqlite3/test/backup.py b/Lib/sqlite3/test/backup.py index 2a2148937413..903bacf49030 100644 --- a/Lib/sqlite3/test/backup.py +++ b/Lib/sqlite3/test/backup.py @@ -37,14 +37,12 @@ def test_bad_target_closed_connection(self): self.cx.backup(bck) def test_bad_target_in_transaction(self): - if sqlite.sqlite_version_info == (3, 8, 7, 1): - self.skipTest('skip until we debug https://bugs.python.org/issue27645#msg313562') bck = sqlite.connect(':memory:') bck.execute('CREATE TABLE bar (key INTEGER)') bck.executemany('INSERT INTO bar (key) VALUES (?)', [(3,), (4,)]) with self.assertRaises(sqlite.OperationalError) as cm: self.cx.backup(bck) - if sqlite.sqlite_version_info < (3, 8, 7): + if sqlite.sqlite_version_info < (3, 8, 8): self.assertEqual(str(cm.exception), 'target is in transaction') def test_keyword_only_args(self): diff --git a/Modules/_sqlite/connection.c b/Modules/_sqlite/connection.c index 14b6a2774e78..dc5061c1d91d 100644 --- a/Modules/_sqlite/connection.c +++ b/Modules/_sqlite/connection.c @@ -1481,8 +1481,8 @@ pysqlite_connection_backup(pysqlite_Connection *self, PyObject *args, PyObject * return NULL; } -#if SQLITE_VERSION_NUMBER < 3008007 - /* Since 3.8.7 this is already done, per commit +#if SQLITE_VERSION_NUMBER < 3008008 + /* Since 3.8.8 this is already done, per commit https://www.sqlite.org/src/info/169b5505498c0a7e */ if (!sqlite3_get_autocommit(((pysqlite_Connection *)target)->db)) { PyErr_SetString(pysqlite_OperationalError, "target is in transaction"); From webhook-mailer at python.org Sun Mar 18 03:53:17 2018 From: webhook-mailer at python.org (Serhiy Storchaka) Date: Sun, 18 Mar 2018 07:53:17 -0000 Subject: [Python-checkins] bpo-33041: Add tests for jumps in/out of 'async with' blocks. (#6110) Message-ID: <mailman.87.1521359600.1871.python-checkins@python.org> https://github.com/python/cpython/commit/bc300ce205f99acb1ef92c37de06dc76147e073b commit: bc300ce205f99acb1ef92c37de06dc76147e073b branch: master author: Serhiy Storchaka <storchaka at gmail.com> committer: GitHub <noreply at github.com> date: 2018-03-18T09:53:08+02:00 summary: bpo-33041: Add tests for jumps in/out of 'async with' blocks. (#6110) files: M Lib/test/test_sys_settrace.py diff --git a/Lib/test/test_sys_settrace.py b/Lib/test/test_sys_settrace.py index 90d1e37a3916..2587794c69a3 100644 --- a/Lib/test/test_sys_settrace.py +++ b/Lib/test/test_sys_settrace.py @@ -6,6 +6,8 @@ import difflib import gc from functools import wraps +import asyncio + class tracecontext: """Context manager that traces its enter and exit.""" @@ -19,6 +21,20 @@ def __enter__(self): def __exit__(self, *exc_info): self.output.append(-self.value) +class asynctracecontext: + """Asynchronous context manager that traces its aenter and aexit.""" + def __init__(self, output, value): + self.output = output + self.value = value + + async def __aenter__(self): + self.output.append(self.value) + + async def __aexit__(self, *exc_info): + self.output.append(-self.value) + + + # A very basic example. If this fails, we're in deep trouble. def basic(): return 1 @@ -636,6 +652,19 @@ def run_test(self, func, jumpFrom, jumpTo, expected, error=None, sys.settrace(None) self.compare_jump_output(expected, output) + def run_async_test(self, func, jumpFrom, jumpTo, expected, error=None, + event='line', decorated=False): + tracer = JumpTracer(func, jumpFrom, jumpTo, event, decorated) + sys.settrace(tracer.trace) + output = [] + if error is None: + asyncio.run(func(output)) + else: + with self.assertRaisesRegex(*error): + asyncio.run(func(output)) + sys.settrace(None) + self.compare_jump_output(expected, output) + def jump_test(jumpFrom, jumpTo, expected, error=None, event='line'): """Decorator that creates a test that makes a jump from one place to another in the following code. @@ -648,6 +677,18 @@ def test(self): return test return decorator + def async_jump_test(jumpFrom, jumpTo, expected, error=None, event='line'): + """Decorator that creates a test that makes a jump + from one place to another in the following asynchronous code. + """ + def decorator(func): + @wraps(func) + def test(self): + self.run_async_test(func, jumpFrom, jumpTo, expected, + error=error, event=event, decorated=True) + return test + return decorator + ## The first set of 'jump' tests are for things that are allowed: @jump_test(1, 3, [3]) @@ -774,12 +815,24 @@ def test_jump_forwards_out_of_with_block(output): output.append(2) output.append(3) + @async_jump_test(2, 3, [1, 3]) + async def test_jump_forwards_out_of_async_with_block(output): + async with asynctracecontext(output, 1): + output.append(2) + output.append(3) + @jump_test(3, 1, [1, 2, 1, 2, 3, -2]) def test_jump_backwards_out_of_with_block(output): output.append(1) with tracecontext(output, 2): output.append(3) + @async_jump_test(3, 1, [1, 2, 1, 2, 3, -2]) + async def test_jump_backwards_out_of_async_with_block(output): + output.append(1) + async with asynctracecontext(output, 2): + output.append(3) + @jump_test(2, 5, [5]) def test_jump_forwards_out_of_try_finally_block(output): try: @@ -843,6 +896,14 @@ def test_jump_across_with(output): with tracecontext(output, 4): output.append(5) + @async_jump_test(2, 4, [1, 4, 5, -4]) + async def test_jump_across_async_with(output): + output.append(1) + async with asynctracecontext(output, 2): + output.append(3) + async with asynctracecontext(output, 4): + output.append(5) + @jump_test(4, 5, [1, 3, 5, 6]) def test_jump_out_of_with_block_within_for_block(output): output.append(1) @@ -852,6 +913,15 @@ def test_jump_out_of_with_block_within_for_block(output): output.append(5) output.append(6) + @async_jump_test(4, 5, [1, 3, 5, 6]) + async def test_jump_out_of_async_with_block_within_for_block(output): + output.append(1) + for i in [1]: + async with asynctracecontext(output, 3): + output.append(4) + output.append(5) + output.append(6) + @jump_test(4, 5, [1, 2, 3, 5, -2, 6]) def test_jump_out_of_with_block_within_with_block(output): output.append(1) @@ -861,6 +931,15 @@ def test_jump_out_of_with_block_within_with_block(output): output.append(5) output.append(6) + @async_jump_test(4, 5, [1, 2, 3, 5, -2, 6]) + async def test_jump_out_of_async_with_block_within_with_block(output): + output.append(1) + with tracecontext(output, 2): + async with asynctracecontext(output, 3): + output.append(4) + output.append(5) + output.append(6) + @jump_test(5, 6, [2, 4, 6, 7]) def test_jump_out_of_with_block_within_finally_block(output): try: @@ -871,6 +950,16 @@ def test_jump_out_of_with_block_within_finally_block(output): output.append(6) output.append(7) + @async_jump_test(5, 6, [2, 4, 6, 7]) + async def test_jump_out_of_async_with_block_within_finally_block(output): + try: + output.append(2) + finally: + async with asynctracecontext(output, 4): + output.append(5) + output.append(6) + output.append(7) + @jump_test(8, 11, [1, 3, 5, 11, 12]) def test_jump_out_of_complex_nested_blocks(output): output.append(1) @@ -894,6 +983,14 @@ def test_jump_out_of_with_assignment(output): output.append(4) output.append(5) + @async_jump_test(3, 5, [1, 2, 5]) + async def test_jump_out_of_async_with_assignment(output): + output.append(1) + async with asynctracecontext(output, 2) \ + as x: + output.append(4) + output.append(5) + @jump_test(3, 6, [1, 6, 8, 9]) def test_jump_over_return_in_try_finally_block(output): output.append(1) @@ -996,12 +1093,24 @@ def test_no_jump_forwards_into_with_block(output): with tracecontext(output, 2): output.append(3) + @async_jump_test(1, 3, [], (ValueError, 'into')) + async def test_no_jump_forwards_into_async_with_block(output): + output.append(1) + async with asynctracecontext(output, 2): + output.append(3) + @jump_test(3, 2, [1, 2, -1], (ValueError, 'into')) def test_no_jump_backwards_into_with_block(output): with tracecontext(output, 1): output.append(2) output.append(3) + @async_jump_test(3, 2, [1, 2, -1], (ValueError, 'into')) + async def test_no_jump_backwards_into_async_with_block(output): + async with asynctracecontext(output, 1): + output.append(2) + output.append(3) + @jump_test(1, 3, [], (ValueError, 'into')) def test_no_jump_forwards_into_try_finally_block(output): output.append(1) @@ -1082,6 +1191,14 @@ def test_no_jump_between_with_blocks(output): with tracecontext(output, 4): output.append(5) + @async_jump_test(3, 5, [1, 2, -2], (ValueError, 'into')) + async def test_no_jump_between_async_with_blocks(output): + output.append(1) + async with asynctracecontext(output, 2): + output.append(3) + async with asynctracecontext(output, 4): + output.append(5) + @jump_test(5, 7, [2, 4], (ValueError, 'finally')) def test_no_jump_over_return_out_of_finally_block(output): try: From webhook-mailer at python.org Sun Mar 18 03:55:56 2018 From: webhook-mailer at python.org (Serhiy Storchaka) Date: Sun, 18 Mar 2018 07:55:56 -0000 Subject: [Python-checkins] bpo-32056: Improve exceptions in aifc, wave and sunau. (GH-5951) Message-ID: <mailman.88.1521359757.1871.python-checkins@python.org> https://github.com/python/cpython/commit/134cb01cda50f02725575808130b05d2d776693f commit: 134cb01cda50f02725575808130b05d2d776693f branch: master author: Serhiy Storchaka <storchaka at gmail.com> committer: GitHub <noreply at github.com> date: 2018-03-18T09:55:53+02:00 summary: bpo-32056: Improve exceptions in aifc, wave and sunau. (GH-5951) files: A Misc/NEWS.d/next/Library/2018-03-01-17-49-56.bpo-32056.IlpfgE.rst M Lib/aifc.py M Lib/sunau.py M Lib/test/test_aifc.py M Lib/test/test_sunau.py M Lib/test/test_wave.py M Lib/wave.py diff --git a/Lib/aifc.py b/Lib/aifc.py index 3d2dc56de198..1916e7ef8e7e 100644 --- a/Lib/aifc.py +++ b/Lib/aifc.py @@ -467,6 +467,10 @@ def _read_comm_chunk(self, chunk): self._nframes = _read_long(chunk) self._sampwidth = (_read_short(chunk) + 7) // 8 self._framerate = int(_read_float(chunk)) + if self._sampwidth <= 0: + raise Error('bad sample width') + if self._nchannels <= 0: + raise Error('bad # of channels') self._framesize = self._nchannels * self._sampwidth if self._aifc: #DEBUG: SGI's soundeditor produces a bad size :-( diff --git a/Lib/sunau.py b/Lib/sunau.py index dbad3db8392d..129502b0b417 100644 --- a/Lib/sunau.py +++ b/Lib/sunau.py @@ -208,6 +208,8 @@ def initfp(self, file): raise Error('unknown encoding') self._framerate = int(_read_u32(file)) self._nchannels = int(_read_u32(file)) + if not self._nchannels: + raise Error('bad # of channels') self._framesize = self._framesize * self._nchannels if self._hdr_size > 24: self._info = file.read(self._hdr_size - 24) diff --git a/Lib/test/test_aifc.py b/Lib/test/test_aifc.py index 8fd306a36592..ff52f5b6feb8 100644 --- a/Lib/test/test_aifc.py +++ b/Lib/test/test_aifc.py @@ -268,7 +268,8 @@ def test_read_no_comm_chunk(self): def test_read_no_ssnd_chunk(self): b = b'FORM' + struct.pack('>L', 4) + b'AIFC' - b += b'COMM' + struct.pack('>LhlhhLL', 38, 0, 0, 0, 0, 0, 0) + b += b'COMM' + struct.pack('>LhlhhLL', 38, 1, 0, 8, + 0x4000 | 12, 11025<<18, 0) b += b'NONE' + struct.pack('B', 14) + b'not compressed' + b'\x00' with self.assertRaisesRegex(aifc.Error, 'COMM chunk and/or SSND chunk' ' missing'): @@ -276,13 +277,35 @@ def test_read_no_ssnd_chunk(self): def test_read_wrong_compression_type(self): b = b'FORM' + struct.pack('>L', 4) + b'AIFC' - b += b'COMM' + struct.pack('>LhlhhLL', 23, 0, 0, 0, 0, 0, 0) + b += b'COMM' + struct.pack('>LhlhhLL', 23, 1, 0, 8, + 0x4000 | 12, 11025<<18, 0) b += b'WRNG' + struct.pack('B', 0) self.assertRaises(aifc.Error, aifc.open, io.BytesIO(b)) + def test_read_wrong_number_of_channels(self): + for nchannels in 0, -1: + b = b'FORM' + struct.pack('>L', 4) + b'AIFC' + b += b'COMM' + struct.pack('>LhlhhLL', 38, nchannels, 0, 8, + 0x4000 | 12, 11025<<18, 0) + b += b'NONE' + struct.pack('B', 14) + b'not compressed' + b'\x00' + b += b'SSND' + struct.pack('>L', 8) + b'\x00' * 8 + with self.assertRaisesRegex(aifc.Error, 'bad # of channels'): + aifc.open(io.BytesIO(b)) + + def test_read_wrong_sample_width(self): + for sampwidth in 0, -1: + b = b'FORM' + struct.pack('>L', 4) + b'AIFC' + b += b'COMM' + struct.pack('>LhlhhLL', 38, 1, 0, sampwidth, + 0x4000 | 12, 11025<<18, 0) + b += b'NONE' + struct.pack('B', 14) + b'not compressed' + b'\x00' + b += b'SSND' + struct.pack('>L', 8) + b'\x00' * 8 + with self.assertRaisesRegex(aifc.Error, 'bad sample width'): + aifc.open(io.BytesIO(b)) + def test_read_wrong_marks(self): b = b'FORM' + struct.pack('>L', 4) + b'AIFF' - b += b'COMM' + struct.pack('>LhlhhLL', 18, 0, 0, 0, 0, 0, 0) + b += b'COMM' + struct.pack('>LhlhhLL', 18, 1, 0, 8, + 0x4000 | 12, 11025<<18, 0) b += b'SSND' + struct.pack('>L', 8) + b'\x00' * 8 b += b'MARK' + struct.pack('>LhB', 3, 1, 1) with self.assertWarns(UserWarning) as cm: @@ -293,7 +316,8 @@ def test_read_wrong_marks(self): def test_read_comm_kludge_compname_even(self): b = b'FORM' + struct.pack('>L', 4) + b'AIFC' - b += b'COMM' + struct.pack('>LhlhhLL', 18, 0, 0, 0, 0, 0, 0) + b += b'COMM' + struct.pack('>LhlhhLL', 18, 1, 0, 8, + 0x4000 | 12, 11025<<18, 0) b += b'NONE' + struct.pack('B', 4) + b'even' + b'\x00' b += b'SSND' + struct.pack('>L', 8) + b'\x00' * 8 with self.assertWarns(UserWarning) as cm: @@ -303,7 +327,8 @@ def test_read_comm_kludge_compname_even(self): def test_read_comm_kludge_compname_odd(self): b = b'FORM' + struct.pack('>L', 4) + b'AIFC' - b += b'COMM' + struct.pack('>LhlhhLL', 18, 0, 0, 0, 0, 0, 0) + b += b'COMM' + struct.pack('>LhlhhLL', 18, 1, 0, 8, + 0x4000 | 12, 11025<<18, 0) b += b'NONE' + struct.pack('B', 3) + b'odd' b += b'SSND' + struct.pack('>L', 8) + b'\x00' * 8 with self.assertWarns(UserWarning) as cm: diff --git a/Lib/test/test_sunau.py b/Lib/test/test_sunau.py index 966224b1df5a..470a1007b4d4 100644 --- a/Lib/test/test_sunau.py +++ b/Lib/test/test_sunau.py @@ -1,6 +1,8 @@ import unittest from test import audiotests from audioop import byteswap +import io +import struct import sys import sunau @@ -121,5 +123,40 @@ class SunauMiscTests(audiotests.AudioMiscTests, unittest.TestCase): module = sunau +class SunauLowLevelTest(unittest.TestCase): + + def test_read_bad_magic_number(self): + b = b'SPA' + with self.assertRaises(EOFError): + sunau.open(io.BytesIO(b)) + b = b'SPAM' + with self.assertRaisesRegex(sunau.Error, 'bad magic number'): + sunau.open(io.BytesIO(b)) + + def test_read_too_small_header(self): + b = struct.pack('>LLLLL', sunau.AUDIO_FILE_MAGIC, 20, 0, + sunau.AUDIO_FILE_ENCODING_LINEAR_8, 11025) + with self.assertRaisesRegex(sunau.Error, 'header size too small'): + sunau.open(io.BytesIO(b)) + + def test_read_too_large_header(self): + b = struct.pack('>LLLLLL', sunau.AUDIO_FILE_MAGIC, 124, 0, + sunau.AUDIO_FILE_ENCODING_LINEAR_8, 11025, 1) + b += b'\0' * 100 + with self.assertRaisesRegex(sunau.Error, 'header size ridiculously large'): + sunau.open(io.BytesIO(b)) + + def test_read_wrong_encoding(self): + b = struct.pack('>LLLLLL', sunau.AUDIO_FILE_MAGIC, 24, 0, 0, 11025, 1) + with self.assertRaisesRegex(sunau.Error, r'encoding not \(yet\) supported'): + sunau.open(io.BytesIO(b)) + + def test_read_wrong_number_of_channels(self): + b = struct.pack('>LLLLLL', sunau.AUDIO_FILE_MAGIC, 24, 0, + sunau.AUDIO_FILE_ENCODING_LINEAR_8, 11025, 0) + with self.assertRaisesRegex(sunau.Error, 'bad # of channels'): + sunau.open(io.BytesIO(b)) + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_wave.py b/Lib/test/test_wave.py index c5d2e02450ef..8a42f8e47105 100644 --- a/Lib/test/test_wave.py +++ b/Lib/test/test_wave.py @@ -2,6 +2,8 @@ from test import audiotests from test import support from audioop import byteswap +import io +import struct import sys import wave @@ -111,5 +113,65 @@ def test__all__(self): support.check__all__(self, wave, blacklist=blacklist) +class WaveLowLevelTest(unittest.TestCase): + + def test_read_no_chunks(self): + b = b'SPAM' + with self.assertRaises(EOFError): + wave.open(io.BytesIO(b)) + + def test_read_no_riff_chunk(self): + b = b'SPAM' + struct.pack('<L', 0) + with self.assertRaisesRegex(wave.Error, + 'file does not start with RIFF id'): + wave.open(io.BytesIO(b)) + + def test_read_not_wave(self): + b = b'RIFF' + struct.pack('<L', 4) + b'SPAM' + with self.assertRaisesRegex(wave.Error, + 'not a WAVE file'): + wave.open(io.BytesIO(b)) + + def test_read_no_fmt_no_data_chunk(self): + b = b'RIFF' + struct.pack('<L', 4) + b'WAVE' + with self.assertRaisesRegex(wave.Error, + 'fmt chunk and/or data chunk missing'): + wave.open(io.BytesIO(b)) + + def test_read_no_data_chunk(self): + b = b'RIFF' + struct.pack('<L', 28) + b'WAVE' + b += b'fmt ' + struct.pack('<LHHLLHH', 16, 1, 1, 11025, 11025, 1, 8) + with self.assertRaisesRegex(wave.Error, + 'fmt chunk and/or data chunk missing'): + wave.open(io.BytesIO(b)) + + def test_read_no_fmt_chunk(self): + b = b'RIFF' + struct.pack('<L', 12) + b'WAVE' + b += b'data' + struct.pack('<L', 0) + with self.assertRaisesRegex(wave.Error, 'data chunk before fmt chunk'): + wave.open(io.BytesIO(b)) + + def test_read_wrong_form(self): + b = b'RIFF' + struct.pack('<L', 36) + b'WAVE' + b += b'fmt ' + struct.pack('<LHHLLHH', 16, 2, 1, 11025, 11025, 1, 1) + b += b'data' + struct.pack('<L', 0) + with self.assertRaisesRegex(wave.Error, 'unknown format: 2'): + wave.open(io.BytesIO(b)) + + def test_read_wrong_number_of_channels(self): + b = b'RIFF' + struct.pack('<L', 36) + b'WAVE' + b += b'fmt ' + struct.pack('<LHHLLHH', 16, 1, 0, 11025, 11025, 1, 8) + b += b'data' + struct.pack('<L', 0) + with self.assertRaisesRegex(wave.Error, 'bad # of channels'): + wave.open(io.BytesIO(b)) + + def test_read_wrong_sample_width(self): + b = b'RIFF' + struct.pack('<L', 36) + b'WAVE' + b += b'fmt ' + struct.pack('<LHHLLHH', 16, 1, 1, 11025, 11025, 1, 0) + b += b'data' + struct.pack('<L', 0) + with self.assertRaisesRegex(wave.Error, 'bad sample width'): + wave.open(io.BytesIO(b)) + + if __name__ == '__main__': unittest.main() diff --git a/Lib/wave.py b/Lib/wave.py index cf94d5af72b4..f155879a9a76 100644 --- a/Lib/wave.py +++ b/Lib/wave.py @@ -253,12 +253,22 @@ def readframes(self, nframes): # def _read_fmt_chunk(self, chunk): - wFormatTag, self._nchannels, self._framerate, dwAvgBytesPerSec, wBlockAlign = struct.unpack_from('<HHLLH', chunk.read(14)) + try: + wFormatTag, self._nchannels, self._framerate, dwAvgBytesPerSec, wBlockAlign = struct.unpack_from('<HHLLH', chunk.read(14)) + except struct.error: + raise EOFError from None if wFormatTag == WAVE_FORMAT_PCM: - sampwidth = struct.unpack_from('<H', chunk.read(2))[0] + try: + sampwidth = struct.unpack_from('<H', chunk.read(2))[0] + except struct.error: + raise EOFError from None self._sampwidth = (sampwidth + 7) // 8 + if not self._sampwidth: + raise Error('bad sample width') else: raise Error('unknown format: %r' % (wFormatTag,)) + if not self._nchannels: + raise Error('bad # of channels') self._framesize = self._nchannels * self._sampwidth self._comptype = 'NONE' self._compname = 'not compressed' diff --git a/Misc/NEWS.d/next/Library/2018-03-01-17-49-56.bpo-32056.IlpfgE.rst b/Misc/NEWS.d/next/Library/2018-03-01-17-49-56.bpo-32056.IlpfgE.rst new file mode 100644 index 000000000000..421aa3767794 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-03-01-17-49-56.bpo-32056.IlpfgE.rst @@ -0,0 +1,3 @@ +Improved exceptions raised for invalid number of channels and sample width +when read an audio file in modules :mod:`aifc`, :mod:`wave` and +:mod:`sunau`. From webhook-mailer at python.org Sun Mar 18 03:56:55 2018 From: webhook-mailer at python.org (Serhiy Storchaka) Date: Sun, 18 Mar 2018 07:56:55 -0000 Subject: [Python-checkins] bpo-32489: Allow 'continue' in 'finally' clause. (GH-5822) Message-ID: <mailman.89.1521359817.1871.python-checkins@python.org> https://github.com/python/cpython/commit/fe2bbb1869b42222a3f331a3dfb8b304a19a5819 commit: fe2bbb1869b42222a3f331a3dfb8b304a19a5819 branch: master author: Serhiy Storchaka <storchaka at gmail.com> committer: GitHub <noreply at github.com> date: 2018-03-18T09:56:52+02:00 summary: bpo-32489: Allow 'continue' in 'finally' clause. (GH-5822) files: A Misc/NEWS.d/next/Core and Builtins/2018-01-03-23-12-43.bpo-32489.SDEPHB.rst M Doc/library/dis.rst M Doc/reference/compound_stmts.rst M Doc/reference/simple_stmts.rst M Doc/whatsnew/3.8.rst M Lib/test/test_compile.py M Lib/test/test_exceptions.py M Lib/test/test_grammar.py M Lib/test/test_syntax.py M Python/compile.c diff --git a/Doc/library/dis.rst b/Doc/library/dis.rst index aa66128d60dd..47f226bd1625 100644 --- a/Doc/library/dis.rst +++ b/Doc/library/dis.rst @@ -698,8 +698,8 @@ iterations of the loop. removed from the block stack. It is similar to :opcode:`END_FINALLY`, but doesn't change the bytecode - counter nor raise an exception. Used for implementing :keyword:`break` - and :keyword:`return` in the :keyword:`finally` block. + counter nor raise an exception. Used for implementing :keyword:`break`, + :keyword:`continue` and :keyword:`return` in the :keyword:`finally` block. .. versionadded:: 3.8 diff --git a/Doc/reference/compound_stmts.rst b/Doc/reference/compound_stmts.rst index d7792f1eaf82..153e85b39491 100644 --- a/Doc/reference/compound_stmts.rst +++ b/Doc/reference/compound_stmts.rst @@ -321,8 +321,8 @@ not handled, the exception is temporarily saved. The :keyword:`finally` clause is executed. If there is a saved exception it is re-raised at the end of the :keyword:`finally` clause. If the :keyword:`finally` clause raises another exception, the saved exception is set as the context of the new exception. -If the :keyword:`finally` clause executes a :keyword:`return` or :keyword:`break` -statement, the saved exception is discarded:: +If the :keyword:`finally` clause executes a :keyword:`return`, :keyword:`break` +or :keyword:`continue` statement, the saved exception is discarded:: >>> def f(): ... try: @@ -343,10 +343,7 @@ the :keyword:`finally` clause. When a :keyword:`return`, :keyword:`break` or :keyword:`continue` statement is executed in the :keyword:`try` suite of a :keyword:`try`...\ :keyword:`finally` -statement, the :keyword:`finally` clause is also executed 'on the way out.' A -:keyword:`continue` statement is illegal in the :keyword:`finally` clause. (The -reason is a problem with the current implementation --- this restriction may be -lifted in the future). +statement, the :keyword:`finally` clause is also executed 'on the way out.' The return value of a function is determined by the last :keyword:`return` statement executed. Since the :keyword:`finally` clause always executes, a @@ -366,6 +363,10 @@ Additional information on exceptions can be found in section :ref:`exceptions`, and information on using the :keyword:`raise` statement to generate exceptions may be found in section :ref:`raise`. +.. versionchanged:: 3.8 + Prior to Python 3.8, a :keyword:`continue` statement was illegal in the + :keyword:`finally` clause due to a problem with the implementation. + .. _with: .. _as: diff --git a/Doc/reference/simple_stmts.rst b/Doc/reference/simple_stmts.rst index ef9a5f0dc854..9186210d951f 100644 --- a/Doc/reference/simple_stmts.rst +++ b/Doc/reference/simple_stmts.rst @@ -686,9 +686,8 @@ The :keyword:`continue` statement continue_stmt: "continue" :keyword:`continue` may only occur syntactically nested in a :keyword:`for` or -:keyword:`while` loop, but not nested in a function or class definition or -:keyword:`finally` clause within that loop. It continues with the next -cycle of the nearest enclosing loop. +:keyword:`while` loop, but not nested in a function or class definition within +that loop. It continues with the next cycle of the nearest enclosing loop. When :keyword:`continue` passes control out of a :keyword:`try` statement with a :keyword:`finally` clause, that :keyword:`finally` clause is executed before diff --git a/Doc/whatsnew/3.8.rst b/Doc/whatsnew/3.8.rst index 8569341055e1..fcc868b041ad 100644 --- a/Doc/whatsnew/3.8.rst +++ b/Doc/whatsnew/3.8.rst @@ -72,6 +72,11 @@ New Features Other Language Changes ====================== +* A :keyword:`continue` statement was illegal in the :keyword:`finally` clause + due to a problem with the implementation. In Python 3.8 this restriction + was lifted. + (Contributed by Serhiy Storchaka in :issue:`32489`.) + * Added support of ``\N{name}`` escapes in :mod:`regular expressions <re>`. (Contributed by Jonathan Eunice and Serhiy Storchaka in :issue:`30688`.) diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py index acebdbdc4636..6b45a2433438 100644 --- a/Lib/test/test_compile.py +++ b/Lib/test/test_compile.py @@ -856,7 +856,7 @@ def test_for_break_continue_inside_try_finally_block(self): """ self.check_stack_size(snippet) - def test_for_break_inside_finally_block(self): + def test_for_break_continue_inside_finally_block(self): snippet = """ for x in y: try: @@ -864,6 +864,8 @@ def test_for_break_inside_finally_block(self): finally: if z: break + elif u: + continue else: a else: diff --git a/Lib/test/test_exceptions.py b/Lib/test/test_exceptions.py index 9d10df5f9425..2a9ec706467f 100644 --- a/Lib/test/test_exceptions.py +++ b/Lib/test/test_exceptions.py @@ -138,15 +138,6 @@ def ckmsg(src, msg): else: self.fail("failed to get expected SyntaxError") - s = '''while 1: - try: - pass - finally: - continue''' - - if not sys.platform.startswith('java'): - ckmsg(s, "'continue' not supported inside 'finally' clause") - s = '''if 1: try: continue diff --git a/Lib/test/test_grammar.py b/Lib/test/test_grammar.py index d89bfdc06335..ee4136286ba1 100644 --- a/Lib/test/test_grammar.py +++ b/Lib/test/test_grammar.py @@ -859,6 +859,59 @@ def test_break_in_finally(self): break self.assertEqual(count, 0) + def test_continue_in_finally(self): + count = 0 + while count < 2: + count += 1 + try: + pass + finally: + continue + break + self.assertEqual(count, 2) + + count = 0 + while count < 2: + count += 1 + try: + break + finally: + continue + self.assertEqual(count, 2) + + count = 0 + while count < 2: + count += 1 + try: + 1/0 + finally: + continue + break + self.assertEqual(count, 2) + + for count in [0, 1]: + try: + pass + finally: + continue + break + self.assertEqual(count, 1) + + for count in [0, 1]: + try: + break + finally: + continue + self.assertEqual(count, 1) + + for count in [0, 1]: + try: + 1/0 + finally: + continue + break + self.assertEqual(count, 1) + def test_return_in_finally(self): def g1(): try: diff --git a/Lib/test/test_syntax.py b/Lib/test/test_syntax.py index 2b96a94401a8..fa1e7aa5d4f2 100644 --- a/Lib/test/test_syntax.py +++ b/Lib/test/test_syntax.py @@ -298,7 +298,7 @@ >>> test() 9 -Start simple, a continue in a finally should not be allowed. +continue in a finally should be ok. >>> def test(): ... for abc in range(10): @@ -306,11 +306,9 @@ ... pass ... finally: ... continue - Traceback (most recent call last): - ... - SyntaxError: 'continue' not supported inside 'finally' clause - -This is essentially a continue in a finally which should not be allowed. + ... print(abc) + >>> test() + 9 >>> def test(): ... for abc in range(10): @@ -321,9 +319,24 @@ ... continue ... except: ... pass - Traceback (most recent call last): - ... - SyntaxError: 'continue' not supported inside 'finally' clause + ... print(abc) + >>> test() + 9 + + >>> def test(): + ... for abc in range(10): + ... try: + ... pass + ... finally: + ... try: + ... pass + ... except: + ... continue + ... print(abc) + >>> test() + 9 + +A continue outside loop should not be allowed. >>> def foo(): ... try: @@ -332,42 +345,7 @@ ... continue Traceback (most recent call last): ... - SyntaxError: 'continue' not supported inside 'finally' clause - - >>> def foo(): - ... for a in (): - ... try: - ... pass - ... finally: - ... continue - Traceback (most recent call last): - ... - SyntaxError: 'continue' not supported inside 'finally' clause - - >>> def foo(): - ... for a in (): - ... try: - ... pass - ... finally: - ... try: - ... continue - ... finally: - ... pass - Traceback (most recent call last): - ... - SyntaxError: 'continue' not supported inside 'finally' clause - - >>> def foo(): - ... for a in (): - ... try: pass - ... finally: - ... try: - ... pass - ... except: - ... continue - Traceback (most recent call last): - ... - SyntaxError: 'continue' not supported inside 'finally' clause + SyntaxError: 'continue' not properly in loop There is one test for a break that is not in a loop. The compiler uses a single data structure to keep track of try-finally and loops, diff --git a/Misc/NEWS.d/next/Core and Builtins/2018-01-03-23-12-43.bpo-32489.SDEPHB.rst b/Misc/NEWS.d/next/Core and Builtins/2018-01-03-23-12-43.bpo-32489.SDEPHB.rst new file mode 100644 index 000000000000..68babebdad3c --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2018-01-03-23-12-43.bpo-32489.SDEPHB.rst @@ -0,0 +1,2 @@ +A :keyword:`continue` statement is now allowed in the :keyword:`finally` +clause. diff --git a/Python/compile.c b/Python/compile.c index fbd1fc960ab8..c3ffaae8ce20 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -2625,10 +2625,6 @@ compiler_continue(struct compiler *c) ADDOP_JABS(c, JUMP_ABSOLUTE, info->fb_block); return 1; } - if (info->fb_type == FINALLY_END) { - return compiler_error(c, - "'continue' not supported inside 'finally' clause"); - } if (!compiler_unwind_fblock(c, info, 0)) return 0; } From solipsis at pitrou.net Sun Mar 18 05:10:31 2018 From: solipsis at pitrou.net (solipsis at pitrou.net) Date: Sun, 18 Mar 2018 09:10:31 +0000 Subject: [Python-checkins] Daily reference leaks (4243df51fe43): sum=71 Message-ID: <20180318091031.1.CA16D930CD8C285F@psf.io> results for 4243df51fe43 on branch "default" -------------------------------------------- test_collections leaked [0, -7, 8] memory blocks, sum=1 test_functools leaked [0, 3, 1] memory blocks, sum=4 test_multiprocessing_forkserver leaked [-2, 2, 0] memory blocks, sum=0 test_multiprocessing_spawn leaked [0, 44, 0] references, sum=44 test_multiprocessing_spawn leaked [0, 21, -1] memory blocks, sum=20 test_multiprocessing_spawn leaked [0, 2, 0] file descriptors, sum=2 Command line was: ['./python', '-m', 'test.regrtest', '-uall', '-R', '3:3:/home/psf-users/antoine/refleaks/reflogmXjf9O', '--timeout', '7200'] From webhook-mailer at python.org Sun Mar 18 06:31:42 2018 From: webhook-mailer at python.org (Serhiy Storchaka) Date: Sun, 18 Mar 2018 10:31:42 -0000 Subject: [Python-checkins] [3.7] bpo-33041: Add tests for jumps in/out of 'async with' blocks. (GH-6110). (GH-6140) Message-ID: <mailman.90.1521369103.1871.python-checkins@python.org> https://github.com/python/cpython/commit/773573e9ac654d4b5c6682e70360f75864acd80e commit: 773573e9ac654d4b5c6682e70360f75864acd80e branch: 3.7 author: Serhiy Storchaka <storchaka at gmail.com> committer: GitHub <noreply at github.com> date: 2018-03-18T12:31:37+02:00 summary: [3.7] bpo-33041: Add tests for jumps in/out of 'async with' blocks. (GH-6110). (GH-6140) (cherry picked from commit bc300ce205f99acb1ef92c37de06dc76147e073b) Co-authored-by: Serhiy Storchaka <storchaka at gmail.com> files: M Lib/test/test_sys_settrace.py diff --git a/Lib/test/test_sys_settrace.py b/Lib/test/test_sys_settrace.py index 44ea7d575b45..bf8c722d0cf7 100644 --- a/Lib/test/test_sys_settrace.py +++ b/Lib/test/test_sys_settrace.py @@ -6,6 +6,8 @@ import difflib import gc from functools import wraps +import asyncio + class tracecontext: """Context manager that traces its enter and exit.""" @@ -19,6 +21,20 @@ def __enter__(self): def __exit__(self, *exc_info): self.output.append(-self.value) +class asynctracecontext: + """Asynchronous context manager that traces its aenter and aexit.""" + def __init__(self, output, value): + self.output = output + self.value = value + + async def __aenter__(self): + self.output.append(self.value) + + async def __aexit__(self, *exc_info): + self.output.append(-self.value) + + + # A very basic example. If this fails, we're in deep trouble. def basic(): return 1 @@ -637,6 +653,19 @@ def run_test(self, func, jumpFrom, jumpTo, expected, error=None, sys.settrace(None) self.compare_jump_output(expected, output) + def run_async_test(self, func, jumpFrom, jumpTo, expected, error=None, + event='line', decorated=False): + tracer = JumpTracer(func, jumpFrom, jumpTo, event, decorated) + sys.settrace(tracer.trace) + output = [] + if error is None: + asyncio.run(func(output)) + else: + with self.assertRaisesRegex(*error): + asyncio.run(func(output)) + sys.settrace(None) + self.compare_jump_output(expected, output) + def jump_test(jumpFrom, jumpTo, expected, error=None, event='line'): """Decorator that creates a test that makes a jump from one place to another in the following code. @@ -649,6 +678,18 @@ def test(self): return test return decorator + def async_jump_test(jumpFrom, jumpTo, expected, error=None, event='line'): + """Decorator that creates a test that makes a jump + from one place to another in the following asynchronous code. + """ + def decorator(func): + @wraps(func) + def test(self): + self.run_async_test(func, jumpFrom, jumpTo, expected, + error=error, event=event, decorated=True) + return test + return decorator + ## The first set of 'jump' tests are for things that are allowed: @jump_test(1, 3, [3]) @@ -744,12 +785,24 @@ def test_jump_forwards_out_of_with_block(output): output.append(2) output.append(3) + @async_jump_test(2, 3, [1, 3]) + async def test_jump_forwards_out_of_async_with_block(output): + async with asynctracecontext(output, 1): + output.append(2) + output.append(3) + @jump_test(3, 1, [1, 2, 1, 2, 3, -2]) def test_jump_backwards_out_of_with_block(output): output.append(1) with tracecontext(output, 2): output.append(3) + @async_jump_test(3, 1, [1, 2, 1, 2, 3, -2]) + async def test_jump_backwards_out_of_async_with_block(output): + output.append(1) + async with asynctracecontext(output, 2): + output.append(3) + @jump_test(2, 5, [5]) def test_jump_forwards_out_of_try_finally_block(output): try: @@ -813,6 +866,14 @@ def test_jump_across_with(output): with tracecontext(output, 4): output.append(5) + @async_jump_test(2, 4, [1, 4, 5, -4]) + async def test_jump_across_async_with(output): + output.append(1) + async with asynctracecontext(output, 2): + output.append(3) + async with asynctracecontext(output, 4): + output.append(5) + @jump_test(4, 5, [1, 3, 5, 6]) def test_jump_out_of_with_block_within_for_block(output): output.append(1) @@ -822,6 +883,15 @@ def test_jump_out_of_with_block_within_for_block(output): output.append(5) output.append(6) + @async_jump_test(4, 5, [1, 3, 5, 6]) + async def test_jump_out_of_async_with_block_within_for_block(output): + output.append(1) + for i in [1]: + async with asynctracecontext(output, 3): + output.append(4) + output.append(5) + output.append(6) + @jump_test(4, 5, [1, 2, 3, 5, -2, 6]) def test_jump_out_of_with_block_within_with_block(output): output.append(1) @@ -831,6 +901,15 @@ def test_jump_out_of_with_block_within_with_block(output): output.append(5) output.append(6) + @async_jump_test(4, 5, [1, 2, 3, 5, -2, 6]) + async def test_jump_out_of_async_with_block_within_with_block(output): + output.append(1) + with tracecontext(output, 2): + async with asynctracecontext(output, 3): + output.append(4) + output.append(5) + output.append(6) + @jump_test(5, 6, [2, 4, 6, 7]) def test_jump_out_of_with_block_within_finally_block(output): try: @@ -841,6 +920,16 @@ def test_jump_out_of_with_block_within_finally_block(output): output.append(6) output.append(7) + @async_jump_test(5, 6, [2, 4, 6, 7]) + async def test_jump_out_of_async_with_block_within_finally_block(output): + try: + output.append(2) + finally: + async with asynctracecontext(output, 4): + output.append(5) + output.append(6) + output.append(7) + @jump_test(8, 11, [1, 3, 5, 11, 12]) def test_jump_out_of_complex_nested_blocks(output): output.append(1) @@ -864,6 +953,14 @@ def test_jump_out_of_with_assignment(output): output.append(4) output.append(5) + @async_jump_test(3, 5, [1, 2, 5]) + async def test_jump_out_of_async_with_assignment(output): + output.append(1) + async with asynctracecontext(output, 2) \ + as x: + output.append(4) + output.append(5) + @jump_test(3, 6, [1, 6, 8, 9]) def test_jump_over_return_in_try_finally_block(output): output.append(1) @@ -982,12 +1079,24 @@ def test_no_jump_forwards_into_with_block(output): with tracecontext(output, 2): output.append(3) + @async_jump_test(1, 3, [], (ValueError, 'into')) + async def test_no_jump_forwards_into_async_with_block(output): + output.append(1) + async with asynctracecontext(output, 2): + output.append(3) + @jump_test(3, 2, [1, 2, -1], (ValueError, 'into')) def test_no_jump_backwards_into_with_block(output): with tracecontext(output, 1): output.append(2) output.append(3) + @async_jump_test(3, 2, [1, 2, -1], (ValueError, 'into')) + async def test_no_jump_backwards_into_async_with_block(output): + async with asynctracecontext(output, 1): + output.append(2) + output.append(3) + @jump_test(1, 3, [], (ValueError, 'into')) def test_no_jump_forwards_into_try_finally_block(output): output.append(1) @@ -1068,6 +1177,14 @@ def test_no_jump_between_with_blocks(output): with tracecontext(output, 4): output.append(5) + @async_jump_test(3, 5, [1, 2, -2], (ValueError, 'into')) + async def test_no_jump_between_async_with_blocks(output): + output.append(1) + async with asynctracecontext(output, 2): + output.append(3) + async with asynctracecontext(output, 4): + output.append(5) + @jump_test(7, 4, [1, 6], (ValueError, 'into')) def test_no_jump_into_for_block_before_else(output): output.append(1) From webhook-mailer at python.org Sun Mar 18 06:32:35 2018 From: webhook-mailer at python.org (Serhiy Storchaka) Date: Sun, 18 Mar 2018 10:32:35 -0000 Subject: [Python-checkins] [3.6] bpo-33041: Add tests for jumps in/out of 'async with' blocks. (GH-6110). (GH-6141) Message-ID: <mailman.91.1521369157.1871.python-checkins@python.org> https://github.com/python/cpython/commit/193760f00aacb122698674ed15327dba412653a5 commit: 193760f00aacb122698674ed15327dba412653a5 branch: 3.6 author: Serhiy Storchaka <storchaka at gmail.com> committer: GitHub <noreply at github.com> date: 2018-03-18T12:32:32+02:00 summary: [3.6] bpo-33041: Add tests for jumps in/out of 'async with' blocks. (GH-6110). (GH-6141) (cherry picked from commit bc300ce205f99acb1ef92c37de06dc76147e073b) files: M Lib/test/test_sys_settrace.py diff --git a/Lib/test/test_sys_settrace.py b/Lib/test/test_sys_settrace.py index 46593cf54ad4..659790efe7cc 100644 --- a/Lib/test/test_sys_settrace.py +++ b/Lib/test/test_sys_settrace.py @@ -7,8 +7,9 @@ import gc from functools import wraps + class tracecontext: - """Contex manager that traces its enter and exit.""" + """Context manager that traces its enter and exit.""" def __init__(self, output, value): self.output = output self.value = value @@ -19,6 +20,36 @@ def __enter__(self): def __exit__(self, *exc_info): self.output.append(-self.value) +class asynctracecontext: + """Asynchronous context manager that traces its aenter and aexit.""" + def __init__(self, output, value): + self.output = output + self.value = value + + async def __aenter__(self): + self.output.append(self.value) + + async def __aexit__(self, *exc_info): + self.output.append(-self.value) + +def asyncio_run(main): + import asyncio + import asyncio.events + import asyncio.coroutines + assert asyncio.events._get_running_loop() is None + assert asyncio.coroutines.iscoroutine(main) + loop = asyncio.events.new_event_loop() + try: + asyncio.events.set_event_loop(loop) + return loop.run_until_complete(main) + finally: + try: + loop.run_until_complete(loop.shutdown_asyncgens()) + finally: + asyncio.events.set_event_loop(None) + loop.close() + + # A very basic example. If this fails, we're in deep trouble. def basic(): return 1 @@ -591,6 +622,19 @@ def run_test(self, func, jumpFrom, jumpTo, expected, error=None, sys.settrace(None) self.compare_jump_output(expected, output) + def run_async_test(self, func, jumpFrom, jumpTo, expected, error=None, + event='line', decorated=False): + tracer = JumpTracer(func, jumpFrom, jumpTo, event, decorated) + sys.settrace(tracer.trace) + output = [] + if error is None: + asyncio_run(func(output)) + else: + with self.assertRaisesRegex(*error): + asyncio_run(func(output)) + sys.settrace(None) + self.compare_jump_output(expected, output) + def jump_test(jumpFrom, jumpTo, expected, error=None, event='line'): """Decorator that creates a test that makes a jump from one place to another in the following code. @@ -603,6 +647,18 @@ def test(self): return test return decorator + def async_jump_test(jumpFrom, jumpTo, expected, error=None, event='line'): + """Decorator that creates a test that makes a jump + from one place to another in the following asynchronous code. + """ + def decorator(func): + @wraps(func) + def test(self): + self.run_async_test(func, jumpFrom, jumpTo, expected, + error=error, event=event, decorated=True) + return test + return decorator + ## The first set of 'jump' tests are for things that are allowed: @jump_test(1, 3, [3]) @@ -698,12 +754,24 @@ def test_jump_forwards_out_of_with_block(output): output.append(2) output.append(3) + @async_jump_test(2, 3, [1, 3]) + async def test_jump_forwards_out_of_async_with_block(output): + async with asynctracecontext(output, 1): + output.append(2) + output.append(3) + @jump_test(3, 1, [1, 2, 1, 2, 3, -2]) def test_jump_backwards_out_of_with_block(output): output.append(1) with tracecontext(output, 2): output.append(3) + @async_jump_test(3, 1, [1, 2, 1, 2, 3, -2]) + async def test_jump_backwards_out_of_async_with_block(output): + output.append(1) + async with asynctracecontext(output, 2): + output.append(3) + @jump_test(2, 5, [5]) def test_jump_forwards_out_of_try_finally_block(output): try: @@ -767,6 +835,14 @@ def test_jump_across_with(output): with tracecontext(output, 4): output.append(5) + @async_jump_test(2, 4, [1, 4, 5, -4]) + async def test_jump_across_async_with(output): + output.append(1) + async with asynctracecontext(output, 2): + output.append(3) + async with asynctracecontext(output, 4): + output.append(5) + @jump_test(4, 5, [1, 3, 5, 6]) def test_jump_out_of_with_block_within_for_block(output): output.append(1) @@ -776,6 +852,15 @@ def test_jump_out_of_with_block_within_for_block(output): output.append(5) output.append(6) + @async_jump_test(4, 5, [1, 3, 5, 6]) + async def test_jump_out_of_async_with_block_within_for_block(output): + output.append(1) + for i in [1]: + async with asynctracecontext(output, 3): + output.append(4) + output.append(5) + output.append(6) + @jump_test(4, 5, [1, 2, 3, 5, -2, 6]) def test_jump_out_of_with_block_within_with_block(output): output.append(1) @@ -785,6 +870,15 @@ def test_jump_out_of_with_block_within_with_block(output): output.append(5) output.append(6) + @async_jump_test(4, 5, [1, 2, 3, 5, -2, 6]) + async def test_jump_out_of_async_with_block_within_with_block(output): + output.append(1) + with tracecontext(output, 2): + async with asynctracecontext(output, 3): + output.append(4) + output.append(5) + output.append(6) + @jump_test(5, 6, [2, 4, 6, 7]) def test_jump_out_of_with_block_within_finally_block(output): try: @@ -795,6 +889,16 @@ def test_jump_out_of_with_block_within_finally_block(output): output.append(6) output.append(7) + @async_jump_test(5, 6, [2, 4, 6, 7]) + async def test_jump_out_of_async_with_block_within_finally_block(output): + try: + output.append(2) + finally: + async with asynctracecontext(output, 4): + output.append(5) + output.append(6) + output.append(7) + @jump_test(8, 11, [1, 3, 5, 11, 12]) def test_jump_out_of_complex_nested_blocks(output): output.append(1) @@ -818,6 +922,14 @@ def test_jump_out_of_with_assignment(output): output.append(4) output.append(5) + @async_jump_test(3, 5, [1, 2, 5]) + async def test_jump_out_of_async_with_assignment(output): + output.append(1) + async with asynctracecontext(output, 2) \ + as x: + output.append(4) + output.append(5) + @jump_test(3, 6, [1, 6, 8, 9]) def test_jump_over_return_in_try_finally_block(output): output.append(1) @@ -936,12 +1048,24 @@ def test_no_jump_forwards_into_with_block(output): with tracecontext(output, 2): output.append(3) + @async_jump_test(1, 3, [], (ValueError, 'into')) + async def test_no_jump_forwards_into_async_with_block(output): + output.append(1) + async with asynctracecontext(output, 2): + output.append(3) + @jump_test(3, 2, [1, 2, -1], (ValueError, 'into')) def test_no_jump_backwards_into_with_block(output): with tracecontext(output, 1): output.append(2) output.append(3) + @async_jump_test(3, 2, [1, 2, -1], (ValueError, 'into')) + async def test_no_jump_backwards_into_async_with_block(output): + async with asynctracecontext(output, 1): + output.append(2) + output.append(3) + @jump_test(1, 3, [], (ValueError, 'into')) def test_no_jump_forwards_into_try_finally_block(output): output.append(1) @@ -1022,6 +1146,14 @@ def test_no_jump_between_with_blocks(output): with tracecontext(output, 4): output.append(5) + @async_jump_test(3, 5, [1, 2, -2], (ValueError, 'into')) + async def test_no_jump_between_async_with_blocks(output): + output.append(1) + async with asynctracecontext(output, 2): + output.append(3) + async with asynctracecontext(output, 4): + output.append(5) + @jump_test(7, 4, [1, 6], (ValueError, 'into')) def test_no_jump_into_for_block_before_else(output): output.append(1) From webhook-mailer at python.org Sun Mar 18 12:54:37 2018 From: webhook-mailer at python.org (=?utf-8?q?=C5=81ukasz?= Langa) Date: Sun, 18 Mar 2018 16:54:37 -0000 Subject: [Python-checkins] Revert "bpo-30406: Make async and await proper keywords (#1669)" (GH-6143) Message-ID: <mailman.93.1521392078.1871.python-checkins@python.org> https://github.com/python/cpython/commit/f64aae46da292f71f6be750026cd052362e066bc commit: f64aae46da292f71f6be750026cd052362e066bc branch: master author: Jelle Zijlstra <jelle.zijlstra at gmail.com> committer: ?ukasz Langa <lukasz at langa.pl> date: 2018-03-18T09:54:33-07:00 summary: Revert "bpo-30406: Make async and await proper keywords (#1669)" (GH-6143) This reverts commit ac317700ce7439e38a8b420218d9a5035bba92ed. (Reverts only the lib2to3 part.) files: M Lib/lib2to3/Grammar.txt M Lib/lib2to3/pgen2/token.py M Lib/lib2to3/pgen2/tokenize.py M Lib/lib2to3/tests/test_parser.py diff --git a/Lib/lib2to3/Grammar.txt b/Lib/lib2to3/Grammar.txt index b19b4a21fadd..4905c91e635e 100644 --- a/Lib/lib2to3/Grammar.txt +++ b/Lib/lib2to3/Grammar.txt @@ -15,7 +15,7 @@ eval_input: testlist NEWLINE* ENDMARKER decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE decorators: decorator+ decorated: decorators (classdef | funcdef | async_funcdef) -async_funcdef: 'async' funcdef +async_funcdef: ASYNC funcdef funcdef: 'def' NAME parameters ['->' test] ':' suite parameters: '(' [typedargslist] ')' typedargslist: ((tfpdef ['=' test] ',')* @@ -66,7 +66,7 @@ exec_stmt: 'exec' expr ['in' test [',' test]] assert_stmt: 'assert' test [',' test] compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | with_stmt | funcdef | classdef | decorated | async_stmt -async_stmt: 'async' (funcdef | with_stmt | for_stmt) +async_stmt: ASYNC (funcdef | with_stmt | for_stmt) if_stmt: 'if' test ':' suite ('elif' test ':' suite)* ['else' ':' suite] while_stmt: 'while' test ':' suite ['else' ':' suite] for_stmt: 'for' exprlist 'in' testlist ':' suite ['else' ':' suite] @@ -105,7 +105,7 @@ shift_expr: arith_expr (('<<'|'>>') arith_expr)* arith_expr: term (('+'|'-') term)* term: factor (('*'|'@'|'/'|'%'|'//') factor)* factor: ('+'|'-'|'~') factor | power -power: ['await'] atom trailer* ['**' factor] +power: [AWAIT] atom trailer* ['**' factor] atom: ('(' [yield_expr|testlist_gexp] ')' | '[' [listmaker] ']' | '{' [dictsetmaker] '}' | @@ -142,7 +142,7 @@ argument: ( test [comp_for] | star_expr ) comp_iter: comp_for | comp_if -comp_for: ['async'] 'for' exprlist 'in' or_test [comp_iter] +comp_for: [ASYNC] 'for' exprlist 'in' or_test [comp_iter] comp_if: 'if' old_test [comp_iter] # As noted above, testlist_safe extends the syntax allowed in list @@ -161,7 +161,7 @@ comp_if: 'if' old_test [comp_iter] # # See https://bugs.python.org/issue27494 old_comp_iter: old_comp_for | old_comp_if -old_comp_for: ['async'] 'for' exprlist 'in' testlist_safe [old_comp_iter] +old_comp_for: [ASYNC] 'for' exprlist 'in' testlist_safe [old_comp_iter] old_comp_if: 'if' old_test [old_comp_iter] testlist1: test (',' test)* diff --git a/Lib/lib2to3/pgen2/token.py b/Lib/lib2to3/pgen2/token.py index 7599396611b2..1a679554d2db 100755 --- a/Lib/lib2to3/pgen2/token.py +++ b/Lib/lib2to3/pgen2/token.py @@ -62,8 +62,10 @@ COMMENT = 53 NL = 54 RARROW = 55 -ERRORTOKEN = 56 -N_TOKENS = 57 +AWAIT = 56 +ASYNC = 57 +ERRORTOKEN = 58 +N_TOKENS = 59 NT_OFFSET = 256 #--end constants-- diff --git a/Lib/lib2to3/pgen2/tokenize.py b/Lib/lib2to3/pgen2/tokenize.py index 14560e4fddff..45afc5f4e53f 100644 --- a/Lib/lib2to3/pgen2/tokenize.py +++ b/Lib/lib2to3/pgen2/tokenize.py @@ -234,7 +234,7 @@ def compat(self, token, iterable): for tok in iterable: toknum, tokval = tok[:2] - if toknum in (NAME, NUMBER): + if toknum in (NAME, NUMBER, ASYNC, AWAIT): tokval += ' ' if toknum == INDENT: @@ -380,6 +380,12 @@ def generate_tokens(readline): contline = None indents = [0] + # 'stashed' and 'async_*' are used for async/await parsing + stashed = None + async_def = False + async_def_indent = 0 + async_def_nl = False + while 1: # loop over lines in stream try: line = readline() @@ -420,6 +426,10 @@ def generate_tokens(readline): pos = pos + 1 if pos == max: break + if stashed: + yield stashed + stashed = None + if line[pos] in '#\r\n': # skip comments or blank lines if line[pos] == '#': comment_token = line[pos:].rstrip('\r\n') @@ -443,8 +453,18 @@ def generate_tokens(readline): ("<tokenize>", lnum, pos, line)) indents = indents[:-1] + if async_def and async_def_indent >= indents[-1]: + async_def = False + async_def_nl = False + async_def_indent = 0 + yield (DEDENT, '', (lnum, pos), (lnum, pos), line) + if async_def and async_def_nl and async_def_indent >= indents[-1]: + async_def = False + async_def_nl = False + async_def_indent = 0 + else: # continued statement if not line: raise TokenError("EOF in multi-line statement", (lnum, 0)) @@ -464,10 +484,18 @@ def generate_tokens(readline): newline = NEWLINE if parenlev > 0: newline = NL + elif async_def: + async_def_nl = True + if stashed: + yield stashed + stashed = None yield (newline, token, spos, epos, line) elif initial == '#': assert not token.endswith("\n") + if stashed: + yield stashed + stashed = None yield (COMMENT, token, spos, epos, line) elif token in triple_quoted: endprog = endprogs[token] @@ -475,6 +503,9 @@ def generate_tokens(readline): if endmatch: # all on one line pos = endmatch.end(0) token = line[start:pos] + if stashed: + yield stashed + stashed = None yield (STRING, token, spos, (lnum, pos), line) else: strstart = (lnum, start) # multiple lines @@ -492,22 +523,63 @@ def generate_tokens(readline): contline = line break else: # ordinary string + if stashed: + yield stashed + stashed = None yield (STRING, token, spos, epos, line) elif initial in namechars: # ordinary name - yield (NAME, token, spos, epos, line) + if token in ('async', 'await'): + if async_def: + yield (ASYNC if token == 'async' else AWAIT, + token, spos, epos, line) + continue + + tok = (NAME, token, spos, epos, line) + if token == 'async' and not stashed: + stashed = tok + continue + + if token == 'def': + if (stashed + and stashed[0] == NAME + and stashed[1] == 'async'): + + async_def = True + async_def_indent = indents[-1] + + yield (ASYNC, stashed[1], + stashed[2], stashed[3], + stashed[4]) + stashed = None + + if stashed: + yield stashed + stashed = None + + yield tok elif initial == '\\': # continued stmt # This yield is new; needed for better idempotency: + if stashed: + yield stashed + stashed = None yield (NL, token, spos, (lnum, pos), line) continued = 1 else: if initial in '([{': parenlev = parenlev + 1 elif initial in ')]}': parenlev = parenlev - 1 + if stashed: + yield stashed + stashed = None yield (OP, token, spos, epos, line) else: yield (ERRORTOKEN, line[pos], (lnum, pos), (lnum, pos+1), line) pos = pos + 1 + if stashed: + yield stashed + stashed = None + for indent in indents[1:]: # pop remaining indent levels yield (DEDENT, '', (lnum, 0), (lnum, 0), '') yield (ENDMARKER, '', (lnum, 0), (lnum, 0), '') diff --git a/Lib/lib2to3/tests/test_parser.py b/Lib/lib2to3/tests/test_parser.py index 6813c65a7953..74653ea80fef 100644 --- a/Lib/lib2to3/tests/test_parser.py +++ b/Lib/lib2to3/tests/test_parser.py @@ -181,34 +181,34 @@ def foo(): pass async def foo(): await x """) - self.validate("await x") - self.validate("""def foo(): - await x""") + self.invalid_syntax("await x") + self.invalid_syntax("""def foo(): + await x""") - self.validate("""def foo(): + self.invalid_syntax("""def foo(): def foo(): pass async def foo(): pass await x """) def test_async_var(self): - self.invalid_syntax("""async = 1""") - self.invalid_syntax("""await = 1""") - self.invalid_syntax("""def async(): pass""") + self.validate("""async = 1""") + self.validate("""await = 1""") + self.validate("""def async(): pass""") def test_async_with(self): self.validate("""async def foo(): async for a in b: pass""") - self.validate("""def foo(): - async for a in b: pass""") + self.invalid_syntax("""def foo(): + async for a in b: pass""") def test_async_for(self): self.validate("""async def foo(): async with a: pass""") - self.validate("""def foo(): - async with a: pass""") + self.invalid_syntax("""def foo(): + async with a: pass""") class TestRaiseChanges(GrammarTest): From webhook-mailer at python.org Sun Mar 18 15:15:55 2018 From: webhook-mailer at python.org (Miss Islington (bot)) Date: Sun, 18 Mar 2018 19:15:55 -0000 Subject: [Python-checkins] Revert "bpo-30406: Make async and await proper keywords (GH-1669)" (GH-6143) Message-ID: <mailman.94.1521400558.1871.python-checkins@python.org> https://github.com/python/cpython/commit/a90df5085b51e8bb9de1c04c408bbef42ce6cbc3 commit: a90df5085b51e8bb9de1c04c408bbef42ce6cbc3 branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: GitHub <noreply at github.com> date: 2018-03-18T12:15:52-07:00 summary: Revert "bpo-30406: Make async and await proper keywords (GH-1669)" (GH-6143) This reverts commit ac317700ce7439e38a8b420218d9a5035bba92ed. (Reverts only the lib2to3 part.) (cherry picked from commit f64aae46da292f71f6be750026cd052362e066bc) Co-authored-by: Jelle Zijlstra <jelle.zijlstra at gmail.com> files: M Lib/lib2to3/Grammar.txt M Lib/lib2to3/pgen2/token.py M Lib/lib2to3/pgen2/tokenize.py M Lib/lib2to3/tests/test_parser.py diff --git a/Lib/lib2to3/Grammar.txt b/Lib/lib2to3/Grammar.txt index b19b4a21fadd..4905c91e635e 100644 --- a/Lib/lib2to3/Grammar.txt +++ b/Lib/lib2to3/Grammar.txt @@ -15,7 +15,7 @@ eval_input: testlist NEWLINE* ENDMARKER decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE decorators: decorator+ decorated: decorators (classdef | funcdef | async_funcdef) -async_funcdef: 'async' funcdef +async_funcdef: ASYNC funcdef funcdef: 'def' NAME parameters ['->' test] ':' suite parameters: '(' [typedargslist] ')' typedargslist: ((tfpdef ['=' test] ',')* @@ -66,7 +66,7 @@ exec_stmt: 'exec' expr ['in' test [',' test]] assert_stmt: 'assert' test [',' test] compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | with_stmt | funcdef | classdef | decorated | async_stmt -async_stmt: 'async' (funcdef | with_stmt | for_stmt) +async_stmt: ASYNC (funcdef | with_stmt | for_stmt) if_stmt: 'if' test ':' suite ('elif' test ':' suite)* ['else' ':' suite] while_stmt: 'while' test ':' suite ['else' ':' suite] for_stmt: 'for' exprlist 'in' testlist ':' suite ['else' ':' suite] @@ -105,7 +105,7 @@ shift_expr: arith_expr (('<<'|'>>') arith_expr)* arith_expr: term (('+'|'-') term)* term: factor (('*'|'@'|'/'|'%'|'//') factor)* factor: ('+'|'-'|'~') factor | power -power: ['await'] atom trailer* ['**' factor] +power: [AWAIT] atom trailer* ['**' factor] atom: ('(' [yield_expr|testlist_gexp] ')' | '[' [listmaker] ']' | '{' [dictsetmaker] '}' | @@ -142,7 +142,7 @@ argument: ( test [comp_for] | star_expr ) comp_iter: comp_for | comp_if -comp_for: ['async'] 'for' exprlist 'in' or_test [comp_iter] +comp_for: [ASYNC] 'for' exprlist 'in' or_test [comp_iter] comp_if: 'if' old_test [comp_iter] # As noted above, testlist_safe extends the syntax allowed in list @@ -161,7 +161,7 @@ comp_if: 'if' old_test [comp_iter] # # See https://bugs.python.org/issue27494 old_comp_iter: old_comp_for | old_comp_if -old_comp_for: ['async'] 'for' exprlist 'in' testlist_safe [old_comp_iter] +old_comp_for: [ASYNC] 'for' exprlist 'in' testlist_safe [old_comp_iter] old_comp_if: 'if' old_test [old_comp_iter] testlist1: test (',' test)* diff --git a/Lib/lib2to3/pgen2/token.py b/Lib/lib2to3/pgen2/token.py index 7599396611b2..1a679554d2db 100755 --- a/Lib/lib2to3/pgen2/token.py +++ b/Lib/lib2to3/pgen2/token.py @@ -62,8 +62,10 @@ COMMENT = 53 NL = 54 RARROW = 55 -ERRORTOKEN = 56 -N_TOKENS = 57 +AWAIT = 56 +ASYNC = 57 +ERRORTOKEN = 58 +N_TOKENS = 59 NT_OFFSET = 256 #--end constants-- diff --git a/Lib/lib2to3/pgen2/tokenize.py b/Lib/lib2to3/pgen2/tokenize.py index 14560e4fddff..45afc5f4e53f 100644 --- a/Lib/lib2to3/pgen2/tokenize.py +++ b/Lib/lib2to3/pgen2/tokenize.py @@ -234,7 +234,7 @@ def compat(self, token, iterable): for tok in iterable: toknum, tokval = tok[:2] - if toknum in (NAME, NUMBER): + if toknum in (NAME, NUMBER, ASYNC, AWAIT): tokval += ' ' if toknum == INDENT: @@ -380,6 +380,12 @@ def generate_tokens(readline): contline = None indents = [0] + # 'stashed' and 'async_*' are used for async/await parsing + stashed = None + async_def = False + async_def_indent = 0 + async_def_nl = False + while 1: # loop over lines in stream try: line = readline() @@ -420,6 +426,10 @@ def generate_tokens(readline): pos = pos + 1 if pos == max: break + if stashed: + yield stashed + stashed = None + if line[pos] in '#\r\n': # skip comments or blank lines if line[pos] == '#': comment_token = line[pos:].rstrip('\r\n') @@ -443,8 +453,18 @@ def generate_tokens(readline): ("<tokenize>", lnum, pos, line)) indents = indents[:-1] + if async_def and async_def_indent >= indents[-1]: + async_def = False + async_def_nl = False + async_def_indent = 0 + yield (DEDENT, '', (lnum, pos), (lnum, pos), line) + if async_def and async_def_nl and async_def_indent >= indents[-1]: + async_def = False + async_def_nl = False + async_def_indent = 0 + else: # continued statement if not line: raise TokenError("EOF in multi-line statement", (lnum, 0)) @@ -464,10 +484,18 @@ def generate_tokens(readline): newline = NEWLINE if parenlev > 0: newline = NL + elif async_def: + async_def_nl = True + if stashed: + yield stashed + stashed = None yield (newline, token, spos, epos, line) elif initial == '#': assert not token.endswith("\n") + if stashed: + yield stashed + stashed = None yield (COMMENT, token, spos, epos, line) elif token in triple_quoted: endprog = endprogs[token] @@ -475,6 +503,9 @@ def generate_tokens(readline): if endmatch: # all on one line pos = endmatch.end(0) token = line[start:pos] + if stashed: + yield stashed + stashed = None yield (STRING, token, spos, (lnum, pos), line) else: strstart = (lnum, start) # multiple lines @@ -492,22 +523,63 @@ def generate_tokens(readline): contline = line break else: # ordinary string + if stashed: + yield stashed + stashed = None yield (STRING, token, spos, epos, line) elif initial in namechars: # ordinary name - yield (NAME, token, spos, epos, line) + if token in ('async', 'await'): + if async_def: + yield (ASYNC if token == 'async' else AWAIT, + token, spos, epos, line) + continue + + tok = (NAME, token, spos, epos, line) + if token == 'async' and not stashed: + stashed = tok + continue + + if token == 'def': + if (stashed + and stashed[0] == NAME + and stashed[1] == 'async'): + + async_def = True + async_def_indent = indents[-1] + + yield (ASYNC, stashed[1], + stashed[2], stashed[3], + stashed[4]) + stashed = None + + if stashed: + yield stashed + stashed = None + + yield tok elif initial == '\\': # continued stmt # This yield is new; needed for better idempotency: + if stashed: + yield stashed + stashed = None yield (NL, token, spos, (lnum, pos), line) continued = 1 else: if initial in '([{': parenlev = parenlev + 1 elif initial in ')]}': parenlev = parenlev - 1 + if stashed: + yield stashed + stashed = None yield (OP, token, spos, epos, line) else: yield (ERRORTOKEN, line[pos], (lnum, pos), (lnum, pos+1), line) pos = pos + 1 + if stashed: + yield stashed + stashed = None + for indent in indents[1:]: # pop remaining indent levels yield (DEDENT, '', (lnum, 0), (lnum, 0), '') yield (ENDMARKER, '', (lnum, 0), (lnum, 0), '') diff --git a/Lib/lib2to3/tests/test_parser.py b/Lib/lib2to3/tests/test_parser.py index 6813c65a7953..74653ea80fef 100644 --- a/Lib/lib2to3/tests/test_parser.py +++ b/Lib/lib2to3/tests/test_parser.py @@ -181,34 +181,34 @@ def foo(): pass async def foo(): await x """) - self.validate("await x") - self.validate("""def foo(): - await x""") + self.invalid_syntax("await x") + self.invalid_syntax("""def foo(): + await x""") - self.validate("""def foo(): + self.invalid_syntax("""def foo(): def foo(): pass async def foo(): pass await x """) def test_async_var(self): - self.invalid_syntax("""async = 1""") - self.invalid_syntax("""await = 1""") - self.invalid_syntax("""def async(): pass""") + self.validate("""async = 1""") + self.validate("""await = 1""") + self.validate("""def async(): pass""") def test_async_with(self): self.validate("""async def foo(): async for a in b: pass""") - self.validate("""def foo(): - async for a in b: pass""") + self.invalid_syntax("""def foo(): + async for a in b: pass""") def test_async_for(self): self.validate("""async def foo(): async with a: pass""") - self.validate("""def foo(): - async with a: pass""") + self.invalid_syntax("""def foo(): + async with a: pass""") class TestRaiseChanges(GrammarTest): From webhook-mailer at python.org Sun Mar 18 16:02:51 2018 From: webhook-mailer at python.org (Mariatta) Date: Sun, 18 Mar 2018 20:02:51 -0000 Subject: [Python-checkins] bpo-19417: Add test_bdb.py (GH-5217) Message-ID: <mailman.95.1521403373.1871.python-checkins@python.org> https://github.com/python/cpython/commit/3fe33043ee83d19e15551094fc1e0984617ded3c commit: 3fe33043ee83d19e15551094fc1e0984617ded3c branch: master author: xdegaye <xdegaye at gmail.com> committer: Mariatta <Mariatta at users.noreply.github.com> date: 2018-03-18T13:02:47-07:00 summary: bpo-19417: Add test_bdb.py (GH-5217) files: A Lib/test/test_bdb.py A Misc/NEWS.d/next/Tests/2018-01-08-13-33-47.bpo-19417.2asoXy.rst M Lib/test/test_sundry.py diff --git a/Lib/test/test_bdb.py b/Lib/test/test_bdb.py new file mode 100644 index 000000000000..abefe6c4e57a --- /dev/null +++ b/Lib/test/test_bdb.py @@ -0,0 +1,1151 @@ +""" Test the bdb module. + + A test defines a list of tuples that may be seen as paired tuples, each + pair being defined by 'expect_tuple, set_tuple' as follows: + + ([event, [lineno[, co_name[, eargs]]]]), (set_type, [sargs]) + + * 'expect_tuple' describes the expected current state of the Bdb instance. + It may be the empty tuple and no check is done in that case. + * 'set_tuple' defines the set_*() method to be invoked when the Bdb + instance reaches this state. + + Example of an 'expect_tuple, set_tuple' pair: + + ('line', 2, 'tfunc_main'), ('step', ) + + Definitions of the members of the 'expect_tuple': + event: + Name of the trace event. The set methods that do not give back + control to the tracer [1] do not trigger a tracer event and in + that case the next 'event' may be 'None' by convention, its value + is not checked. + [1] Methods that trigger a trace event are set_step(), set_next(), + set_return(), set_until() and set_continue(). + lineno: + Line number. Line numbers are relative to the start of the + function when tracing a function in the test_bdb module (i.e. this + module). + co_name: + Name of the function being currently traced. + eargs: + A tuple: + * On an 'exception' event the tuple holds a class object, the + current exception must be an instance of this class. + * On a 'line' event, the tuple holds a dictionary and a list. The + dictionary maps each breakpoint number that has been hit on this + line to its hits count. The list holds the list of breakpoint + number temporaries that are being deleted. + + Definitions of the members of the 'set_tuple': + set_type: + The type of the set method to be invoked. This may + be the type of one of the Bdb set methods: 'step', 'next', + 'until', 'return', 'continue', 'break', 'quit', or the type of one + of the set methods added by test_bdb.Bdb: 'ignore', 'enable', + 'disable', 'clear', 'up', 'down'. + sargs: + The arguments of the set method if any, packed in a tuple. +""" + +import bdb as _bdb +import sys +import os +import unittest +import textwrap +import importlib +import linecache +from contextlib import contextmanager +from itertools import islice, repeat +import test.support + +class BdbException(Exception): pass +class BdbError(BdbException): """Error raised by the Bdb instance.""" +class BdbSyntaxError(BdbException): """Syntax error in the test case.""" +class BdbNotExpectedError(BdbException): """Unexpected result.""" + +# When 'dry_run' is set to true, expect tuples are ignored and the actual +# state of the tracer is printed after running each set_*() method of the test +# case. The full list of breakpoints and their attributes is also printed +# after each 'line' event where a breakpoint has been hit. +dry_run = 0 + +def reset_Breakpoint(): + _bdb.Breakpoint.next = 1 + _bdb.Breakpoint.bplist = {} + _bdb.Breakpoint.bpbynumber = [None] + +def info_breakpoints(): + bp_list = [bp for bp in _bdb.Breakpoint.bpbynumber if bp] + if not bp_list: + return '' + + header_added = False + for bp in bp_list: + if not header_added: + info = 'BpNum Temp Enb Hits Ignore Where\n' + header_added = True + + disp = 'yes ' if bp.temporary else 'no ' + enab = 'yes' if bp.enabled else 'no ' + info += ('%-5d %s %s %-4d %-6d at %s:%d' % + (bp.number, disp, enab, bp.hits, bp.ignore, + os.path.basename(bp.file), bp.line)) + if bp.cond: + info += '\n\tstop only if %s' % (bp.cond,) + info += '\n' + return info + +class Bdb(_bdb.Bdb): + """Extend Bdb to enhance test coverage.""" + + def trace_dispatch(self, frame, event, arg): + self.currentbp = None + return super().trace_dispatch(frame, event, arg) + + def set_break(self, filename, lineno, temporary=False, cond=None, + funcname=None): + if isinstance(funcname, str): + if filename == __file__: + globals_ = globals() + else: + module = importlib.import_module(filename[:-3]) + globals_ = module.__dict__ + func = eval(funcname, globals_) + code = func.__code__ + filename = code.co_filename + lineno = code.co_firstlineno + funcname = code.co_name + + res = super().set_break(filename, lineno, temporary=temporary, + cond=cond, funcname=funcname) + if isinstance(res, str): + raise BdbError(res) + return res + + def get_stack(self, f, t): + self.stack, self.index = super().get_stack(f, t) + self.frame = self.stack[self.index][0] + return self.stack, self.index + + def set_ignore(self, bpnum): + """Increment the ignore count of Breakpoint number 'bpnum'.""" + bp = self.get_bpbynumber(bpnum) + bp.ignore += 1 + + def set_enable(self, bpnum): + bp = self.get_bpbynumber(bpnum) + bp.enabled = True + + def set_disable(self, bpnum): + bp = self.get_bpbynumber(bpnum) + bp.enabled = False + + def set_clear(self, fname, lineno): + err = self.clear_break(fname, lineno) + if err: + raise BdbError(err) + + def set_up(self): + """Move up in the frame stack.""" + if not self.index: + raise BdbError('Oldest frame') + self.index -= 1 + self.frame = self.stack[self.index][0] + + def set_down(self): + """Move down in the frame stack.""" + if self.index + 1 == len(self.stack): + raise BdbError('Newest frame') + self.index += 1 + self.frame = self.stack[self.index][0] + +class Tracer(Bdb): + """A tracer for testing the bdb module.""" + + def __init__(self, expect_set, skip=None, dry_run=False, test_case=None): + super().__init__(skip=skip) + self.expect_set = expect_set + self.dry_run = dry_run + self.header = ('Dry-run results for %s:' % test_case if + test_case is not None else None) + self.init_test() + + def init_test(self): + self.cur_except = None + self.expect_set_no = 0 + self.breakpoint_hits = None + self.expected_list = list(islice(self.expect_set, 0, None, 2)) + self.set_list = list(islice(self.expect_set, 1, None, 2)) + + def trace_dispatch(self, frame, event, arg): + # On an 'exception' event, call_exc_trace() in Python/ceval.c discards + # a BdbException raised by the Tracer instance, so we raise it on the + # next trace_dispatch() call that occurs unless the set_quit() or + # set_continue() method has been invoked on the 'exception' event. + if self.cur_except is not None: + raise self.cur_except + + if event == 'exception': + try: + res = super().trace_dispatch(frame, event, arg) + return res + except BdbException as e: + self.cur_except = e + return self.trace_dispatch + else: + return super().trace_dispatch(frame, event, arg) + + def user_call(self, frame, argument_list): + # Adopt the same behavior as pdb and, as a side effect, skip also the + # first 'call' event when the Tracer is started with Tracer.runcall() + # which may be possibly considered as a bug. + if not self.stop_here(frame): + return + self.process_event('call', frame, argument_list) + self.next_set_method() + + def user_line(self, frame): + self.process_event('line', frame) + + if self.dry_run and self.breakpoint_hits: + info = info_breakpoints().strip('\n') + # Indent each line. + for line in info.split('\n'): + print(' ' + line) + self.delete_temporaries() + self.breakpoint_hits = None + + self.next_set_method() + + def user_return(self, frame, return_value): + self.process_event('return', frame, return_value) + self.next_set_method() + + def user_exception(self, frame, exc_info): + self.exc_info = exc_info + self.process_event('exception', frame) + self.next_set_method() + + def do_clear(self, arg): + # The temporary breakpoints are deleted in user_line(). + bp_list = [self.currentbp] + self.breakpoint_hits = (bp_list, bp_list) + + def delete_temporaries(self): + if self.breakpoint_hits: + for n in self.breakpoint_hits[1]: + self.clear_bpbynumber(n) + + def pop_next(self): + self.expect_set_no += 1 + try: + self.expect = self.expected_list.pop(0) + except IndexError: + raise BdbNotExpectedError( + 'expect_set list exhausted, cannot pop item %d' % + self.expect_set_no) + self.set_tuple = self.set_list.pop(0) + + def process_event(self, event, frame, *args): + # Call get_stack() to enable walking the stack with set_up() and + # set_down(). + tb = None + if event == 'exception': + tb = self.exc_info[2] + self.get_stack(frame, tb) + + # A breakpoint has been hit and it is not a temporary. + if self.currentbp is not None and not self.breakpoint_hits: + bp_list = [self.currentbp] + self.breakpoint_hits = (bp_list, []) + + # Pop next event. + self.event= event + self.pop_next() + if self.dry_run: + self.print_state(self.header) + return + + # Validate the expected results. + if self.expect: + self.check_equal(self.expect[0], event, 'Wrong event type') + self.check_lno_name() + + if event in ('call', 'return'): + self.check_expect_max_size(3) + elif len(self.expect) > 3: + if event == 'line': + bps, temporaries = self.expect[3] + bpnums = sorted(bps.keys()) + if not self.breakpoint_hits: + self.raise_not_expected( + 'No breakpoints hit at expect_set item %d' % + self.expect_set_no) + self.check_equal(bpnums, self.breakpoint_hits[0], + 'Breakpoint numbers do not match') + self.check_equal([bps[n] for n in bpnums], + [self.get_bpbynumber(n).hits for + n in self.breakpoint_hits[0]], + 'Wrong breakpoint hit count') + self.check_equal(sorted(temporaries), self.breakpoint_hits[1], + 'Wrong temporary breakpoints') + + elif event == 'exception': + if not isinstance(self.exc_info[1], self.expect[3]): + self.raise_not_expected( + "Wrong exception at expect_set item %d, got '%s'" % + (self.expect_set_no, self.exc_info)) + + def check_equal(self, expected, result, msg): + if expected == result: + return + self.raise_not_expected("%s at expect_set item %d, got '%s'" % + (msg, self.expect_set_no, result)) + + def check_lno_name(self): + """Check the line number and function co_name.""" + s = len(self.expect) + if s > 1: + lineno = self.lno_abs2rel() + self.check_equal(self.expect[1], lineno, 'Wrong line number') + if s > 2: + self.check_equal(self.expect[2], self.frame.f_code.co_name, + 'Wrong function name') + + def check_expect_max_size(self, size): + if len(self.expect) > size: + raise BdbSyntaxError('Invalid size of the %s expect tuple: %s' % + (self.event, self.expect)) + + def lno_abs2rel(self): + fname = self.canonic(self.frame.f_code.co_filename) + lineno = self.frame.f_lineno + return ((lineno - self.frame.f_code.co_firstlineno + 1) + if fname == self.canonic(__file__) else lineno) + + def lno_rel2abs(self, fname, lineno): + return (self.frame.f_code.co_firstlineno + lineno - 1 + if (lineno and self.canonic(fname) == self.canonic(__file__)) + else lineno) + + def get_state(self): + lineno = self.lno_abs2rel() + co_name = self.frame.f_code.co_name + state = "('%s', %d, '%s'" % (self.event, lineno, co_name) + if self.breakpoint_hits: + bps = '{' + for n in self.breakpoint_hits[0]: + if bps != '{': + bps += ', ' + bps += '%s: %s' % (n, self.get_bpbynumber(n).hits) + bps += '}' + bps = '(' + bps + ', ' + str(self.breakpoint_hits[1]) + ')' + state += ', ' + bps + elif self.event == 'exception': + state += ', ' + self.exc_info[0].__name__ + state += '), ' + return state.ljust(32) + str(self.set_tuple) + ',' + + def print_state(self, header=None): + if header is not None and self.expect_set_no == 1: + print() + print(header) + print('%d: %s' % (self.expect_set_no, self.get_state())) + + def raise_not_expected(self, msg): + msg += '\n' + msg += ' Expected: %s\n' % str(self.expect) + msg += ' Got: ' + self.get_state() + raise BdbNotExpectedError(msg) + + def next_set_method(self): + set_type = self.set_tuple[0] + args = self.set_tuple[1] if len(self.set_tuple) == 2 else None + set_method = getattr(self, 'set_' + set_type) + + # The following set methods give back control to the tracer. + if set_type in ('step', 'continue', 'quit'): + set_method() + return + elif set_type in ('next', 'return'): + set_method(self.frame) + return + elif set_type == 'until': + lineno = None + if args: + lineno = self.lno_rel2abs(self.frame.f_code.co_filename, + args[0]) + set_method(self.frame, lineno) + return + + # The following set methods do not give back control to the tracer and + # next_set_method() is called recursively. + if (args and set_type in ('break', 'clear', 'ignore', 'enable', + 'disable')) or set_type in ('up', 'down'): + if set_type in ('break', 'clear'): + fname, lineno, *remain = args + lineno = self.lno_rel2abs(fname, lineno) + args = [fname, lineno] + args.extend(remain) + set_method(*args) + elif set_type in ('ignore', 'enable', 'disable'): + set_method(*args) + elif set_type in ('up', 'down'): + set_method() + + # Process the next expect_set item. + # It is not expected that a test may reach the recursion limit. + self.event= None + self.pop_next() + if self.dry_run: + self.print_state() + else: + if self.expect: + self.check_lno_name() + self.check_expect_max_size(3) + self.next_set_method() + else: + raise BdbSyntaxError('"%s" is an invalid set_tuple' % + self.set_tuple) + +class TracerRun(): + """Provide a context for running a Tracer instance with a test case.""" + + def __init__(self, test_case, skip=None): + self.test_case = test_case + self.dry_run = test_case.dry_run + self.tracer = Tracer(test_case.expect_set, skip=skip, + dry_run=self.dry_run, test_case=test_case.id()) + + def __enter__(self): + # test_pdb does not reset Breakpoint class attributes on exit :-( + reset_Breakpoint() + return self.tracer + + def __exit__(self, type_=None, value=None, traceback=None): + reset_Breakpoint() + sys.settrace(None) + + not_empty = '' + if self.tracer.set_list: + not_empty += 'All paired tuples have not been processed, ' + not_empty += ('the last one was number %d' % + self.tracer.expect_set_no) + + # Make a BdbNotExpectedError a unittest failure. + if type_ is not None and issubclass(BdbNotExpectedError, type_): + if isinstance(value, BaseException) and value.args: + err_msg = value.args[0] + if not_empty: + err_msg += '\n' + not_empty + if self.dry_run: + print(err_msg) + return True + else: + self.test_case.fail(err_msg) + else: + assert False, 'BdbNotExpectedError with empty args' + + if not_empty: + if self.dry_run: + print(not_empty) + else: + self.test_case.fail(not_empty) + +def run_test(modules, set_list, skip=None): + """Run a test and print the dry-run results. + + 'modules': A dictionary mapping module names to their source code as a + string. The dictionary MUST include one module named + 'test_module' with a main() function. + 'set_list': A list of set_type tuples to be run on the module. + + For example, running the following script outputs the following results: + + ***************************** SCRIPT ******************************** + + from test.test_bdb import run_test, break_in_func + + code = ''' + def func(): + lno = 3 + + def main(): + func() + lno = 7 + ''' + + set_list = [ + break_in_func('func', 'test_module.py'), + ('continue', ), + ('step', ), + ('step', ), + ('step', ), + ('quit', ), + ] + + modules = { 'test_module': code } + run_test(modules, set_list) + + **************************** results ******************************** + + 1: ('line', 2, 'tfunc_import'), ('next',), + 2: ('line', 3, 'tfunc_import'), ('step',), + 3: ('call', 5, 'main'), ('break', ('test_module.py', None, False, None, 'func')), + 4: ('None', 5, 'main'), ('continue',), + 5: ('line', 3, 'func', ({1: 1}, [])), ('step',), + BpNum Temp Enb Hits Ignore Where + 1 no yes 1 0 at test_module.py:2 + 6: ('return', 3, 'func'), ('step',), + 7: ('line', 7, 'main'), ('step',), + 8: ('return', 7, 'main'), ('quit',), + + ************************************************************************* + + """ + def gen(a, b): + try: + while 1: + x = next(a) + y = next(b) + yield x + yield y + except StopIteration: + return + + # Step over the import statement in tfunc_import using 'next' and step + # into main() in test_module. + sl = [('next', ), ('step', )] + sl.extend(set_list) + + test = BaseTestCase() + test.dry_run = True + test.id = lambda : None + test.expect_set = list(gen(repeat(()), iter(sl))) + with create_modules(modules): + sys.path.append(os.getcwd()) + with TracerRun(test, skip=skip) as tracer: + tracer.runcall(tfunc_import) + + at contextmanager +def create_modules(modules): + with test.support.temp_cwd(): + try: + for m in modules: + fname = m + '.py' + with open(fname, 'w') as f: + f.write(textwrap.dedent(modules[m])) + linecache.checkcache(fname) + importlib.invalidate_caches() + yield + finally: + for m in modules: + test.support.forget(m) + +def break_in_func(funcname, fname=__file__, temporary=False, cond=None): + return 'break', (fname, None, temporary, cond, funcname) + +TEST_MODULE = 'test_module' +TEST_MODULE_FNAME = TEST_MODULE + '.py' +def tfunc_import(): + import test_module + test_module.main() + +def tfunc_main(): + lno = 2 + tfunc_first() + tfunc_second() + lno = 5 + lno = 6 + lno = 7 + +def tfunc_first(): + lno = 2 + lno = 3 + lno = 4 + +def tfunc_second(): + lno = 2 + +class BaseTestCase(unittest.TestCase): + """Base class for all tests.""" + + dry_run = dry_run + + def fail(self, msg=None): + # Override fail() to use 'raise from None' to avoid repetition of the + # error message and traceback. + raise self.failureException(msg) from None + +class StateTestCase(BaseTestCase): + """Test the step, next, return, until and quit 'set_' methods.""" + + def test_step(self): + self.expect_set = [ + ('line', 2, 'tfunc_main'), ('step', ), + ('line', 3, 'tfunc_main'), ('step', ), + ('call', 1, 'tfunc_first'), ('step', ), + ('line', 2, 'tfunc_first'), ('quit', ), + ] + with TracerRun(self) as tracer: + tracer.runcall(tfunc_main) + + def test_step_next_on_last_statement(self): + for set_type in ('step', 'next'): + with self.subTest(set_type=set_type): + self.expect_set = [ + ('line', 2, 'tfunc_main'), ('step', ), + ('line', 3, 'tfunc_main'), ('step', ), + ('call', 1, 'tfunc_first'), ('break', (__file__, 3)), + ('None', 1, 'tfunc_first'), ('continue', ), + ('line', 3, 'tfunc_first', ({1:1}, [])), (set_type, ), + ('line', 4, 'tfunc_first'), ('quit', ), + ] + with TracerRun(self) as tracer: + tracer.runcall(tfunc_main) + + def test_next(self): + self.expect_set = [ + ('line', 2, 'tfunc_main'), ('step', ), + ('line', 3, 'tfunc_main'), ('next', ), + ('line', 4, 'tfunc_main'), ('step', ), + ('call', 1, 'tfunc_second'), ('step', ), + ('line', 2, 'tfunc_second'), ('quit', ), + ] + with TracerRun(self) as tracer: + tracer.runcall(tfunc_main) + + def test_next_over_import(self): + code = """ + def main(): + lno = 3 + """ + modules = { TEST_MODULE: code } + with create_modules(modules): + self.expect_set = [ + ('line', 2, 'tfunc_import'), ('next', ), + ('line', 3, 'tfunc_import'), ('quit', ), + ] + with TracerRun(self) as tracer: + tracer.runcall(tfunc_import) + + def test_next_on_plain_statement(self): + # Check that set_next() is equivalent to set_step() on a plain + # statement. + self.expect_set = [ + ('line', 2, 'tfunc_main'), ('step', ), + ('line', 3, 'tfunc_main'), ('step', ), + ('call', 1, 'tfunc_first'), ('next', ), + ('line', 2, 'tfunc_first'), ('quit', ), + ] + with TracerRun(self) as tracer: + tracer.runcall(tfunc_main) + + def test_next_in_caller_frame(self): + # Check that set_next() in the caller frame causes the tracer + # to stop next in the caller frame. + self.expect_set = [ + ('line', 2, 'tfunc_main'), ('step', ), + ('line', 3, 'tfunc_main'), ('step', ), + ('call', 1, 'tfunc_first'), ('up', ), + ('None', 3, 'tfunc_main'), ('next', ), + ('line', 4, 'tfunc_main'), ('quit', ), + ] + with TracerRun(self) as tracer: + tracer.runcall(tfunc_main) + + def test_return(self): + self.expect_set = [ + ('line', 2, 'tfunc_main'), ('step', ), + ('line', 3, 'tfunc_main'), ('step', ), + ('call', 1, 'tfunc_first'), ('step', ), + ('line', 2, 'tfunc_first'), ('return', ), + ('return', 4, 'tfunc_first'), ('step', ), + ('line', 4, 'tfunc_main'), ('quit', ), + ] + with TracerRun(self) as tracer: + tracer.runcall(tfunc_main) + + def test_return_in_caller_frame(self): + self.expect_set = [ + ('line', 2, 'tfunc_main'), ('step', ), + ('line', 3, 'tfunc_main'), ('step', ), + ('call', 1, 'tfunc_first'), ('up', ), + ('None', 3, 'tfunc_main'), ('return', ), + ('return', 7, 'tfunc_main'), ('quit', ), + ] + with TracerRun(self) as tracer: + tracer.runcall(tfunc_main) + + def test_until(self): + self.expect_set = [ + ('line', 2, 'tfunc_main'), ('step', ), + ('line', 3, 'tfunc_main'), ('step', ), + ('call', 1, 'tfunc_first'), ('step', ), + ('line', 2, 'tfunc_first'), ('until', (4, )), + ('line', 4, 'tfunc_first'), ('quit', ), + ] + with TracerRun(self) as tracer: + tracer.runcall(tfunc_main) + + def test_until_with_too_large_count(self): + self.expect_set = [ + ('line', 2, 'tfunc_main'), break_in_func('tfunc_first'), + ('None', 2, 'tfunc_main'), ('continue', ), + ('line', 2, 'tfunc_first', ({1:1}, [])), ('until', (9999, )), + ('return', 4, 'tfunc_first'), ('quit', ), + ] + with TracerRun(self) as tracer: + tracer.runcall(tfunc_main) + + def test_until_in_caller_frame(self): + self.expect_set = [ + ('line', 2, 'tfunc_main'), ('step', ), + ('line', 3, 'tfunc_main'), ('step', ), + ('call', 1, 'tfunc_first'), ('up', ), + ('None', 3, 'tfunc_main'), ('until', (6, )), + ('line', 6, 'tfunc_main'), ('quit', ), + ] + with TracerRun(self) as tracer: + tracer.runcall(tfunc_main) + + def test_skip(self): + # Check that tracing is skipped over the import statement in + # 'tfunc_import()'. + code = """ + def main(): + lno = 3 + """ + modules = { TEST_MODULE: code } + with create_modules(modules): + self.expect_set = [ + ('line', 2, 'tfunc_import'), ('step', ), + ('line', 3, 'tfunc_import'), ('quit', ), + ] + skip = ('importlib*', TEST_MODULE) + with TracerRun(self, skip=skip) as tracer: + tracer.runcall(tfunc_import) + + def test_down(self): + # Check that set_down() raises BdbError at the newest frame. + self.expect_set = [ + ('line', 2, 'tfunc_main'), ('down', ), + ] + with TracerRun(self) as tracer: + self.assertRaises(BdbError, tracer.runcall, tfunc_main) + + def test_up(self): + self.expect_set = [ + ('line', 2, 'tfunc_main'), ('step', ), + ('line', 3, 'tfunc_main'), ('step', ), + ('call', 1, 'tfunc_first'), ('up', ), + ('None', 3, 'tfunc_main'), ('quit', ), + ] + with TracerRun(self) as tracer: + tracer.runcall(tfunc_main) + +class BreakpointTestCase(BaseTestCase): + """Test the breakpoint set method.""" + + def test_bp_on_non_existent_module(self): + self.expect_set = [ + ('line', 2, 'tfunc_import'), ('break', ('/non/existent/module.py', 1)) + ] + with TracerRun(self) as tracer: + self.assertRaises(BdbError, tracer.runcall, tfunc_import) + + def test_bp_after_last_statement(self): + code = """ + def main(): + lno = 3 + """ + modules = { TEST_MODULE: code } + with create_modules(modules): + self.expect_set = [ + ('line', 2, 'tfunc_import'), ('break', (TEST_MODULE_FNAME, 4)) + ] + with TracerRun(self) as tracer: + self.assertRaises(BdbError, tracer.runcall, tfunc_import) + + def test_temporary_bp(self): + code = """ + def func(): + lno = 3 + + def main(): + for i in range(2): + func() + """ + modules = { TEST_MODULE: code } + with create_modules(modules): + self.expect_set = [ + ('line', 2, 'tfunc_import'), + break_in_func('func', TEST_MODULE_FNAME, True), + ('None', 2, 'tfunc_import'), + break_in_func('func', TEST_MODULE_FNAME, True), + ('None', 2, 'tfunc_import'), ('continue', ), + ('line', 3, 'func', ({1:1}, [1])), ('continue', ), + ('line', 3, 'func', ({2:1}, [2])), ('quit', ), + ] + with TracerRun(self) as tracer: + tracer.runcall(tfunc_import) + + def test_disabled_temporary_bp(self): + code = """ + def func(): + lno = 3 + + def main(): + for i in range(3): + func() + """ + modules = { TEST_MODULE: code } + with create_modules(modules): + self.expect_set = [ + ('line', 2, 'tfunc_import'), + break_in_func('func', TEST_MODULE_FNAME), + ('None', 2, 'tfunc_import'), + break_in_func('func', TEST_MODULE_FNAME, True), + ('None', 2, 'tfunc_import'), ('disable', (2, )), + ('None', 2, 'tfunc_import'), ('continue', ), + ('line', 3, 'func', ({1:1}, [])), ('enable', (2, )), + ('None', 3, 'func'), ('disable', (1, )), + ('None', 3, 'func'), ('continue', ), + ('line', 3, 'func', ({2:1}, [2])), ('enable', (1, )), + ('None', 3, 'func'), ('continue', ), + ('line', 3, 'func', ({1:2}, [])), ('quit', ), + ] + with TracerRun(self) as tracer: + tracer.runcall(tfunc_import) + + def test_bp_condition(self): + code = """ + def func(a): + lno = 3 + + def main(): + for i in range(3): + func(i) + """ + modules = { TEST_MODULE: code } + with create_modules(modules): + self.expect_set = [ + ('line', 2, 'tfunc_import'), + break_in_func('func', TEST_MODULE_FNAME, False, 'a == 2'), + ('None', 2, 'tfunc_import'), ('continue', ), + ('line', 3, 'func', ({1:3}, [])), ('quit', ), + ] + with TracerRun(self) as tracer: + tracer.runcall(tfunc_import) + + def test_bp_exception_on_condition_evaluation(self): + code = """ + def func(a): + lno = 3 + + def main(): + func(0) + """ + modules = { TEST_MODULE: code } + with create_modules(modules): + self.expect_set = [ + ('line', 2, 'tfunc_import'), + break_in_func('func', TEST_MODULE_FNAME, False, '1 / 0'), + ('None', 2, 'tfunc_import'), ('continue', ), + ('line', 3, 'func', ({1:1}, [])), ('quit', ), + ] + with TracerRun(self) as tracer: + tracer.runcall(tfunc_import) + + def test_bp_ignore_count(self): + code = """ + def func(): + lno = 3 + + def main(): + for i in range(2): + func() + """ + modules = { TEST_MODULE: code } + with create_modules(modules): + self.expect_set = [ + ('line', 2, 'tfunc_import'), + break_in_func('func', TEST_MODULE_FNAME), + ('None', 2, 'tfunc_import'), ('ignore', (1, )), + ('None', 2, 'tfunc_import'), ('continue', ), + ('line', 3, 'func', ({1:2}, [])), ('quit', ), + ] + with TracerRun(self) as tracer: + tracer.runcall(tfunc_import) + + def test_ignore_count_on_disabled_bp(self): + code = """ + def func(): + lno = 3 + + def main(): + for i in range(3): + func() + """ + modules = { TEST_MODULE: code } + with create_modules(modules): + self.expect_set = [ + ('line', 2, 'tfunc_import'), + break_in_func('func', TEST_MODULE_FNAME), + ('None', 2, 'tfunc_import'), + break_in_func('func', TEST_MODULE_FNAME), + ('None', 2, 'tfunc_import'), ('ignore', (1, )), + ('None', 2, 'tfunc_import'), ('disable', (1, )), + ('None', 2, 'tfunc_import'), ('continue', ), + ('line', 3, 'func', ({2:1}, [])), ('enable', (1, )), + ('None', 3, 'func'), ('continue', ), + ('line', 3, 'func', ({2:2}, [])), ('continue', ), + ('line', 3, 'func', ({1:2}, [])), ('quit', ), + ] + with TracerRun(self) as tracer: + tracer.runcall(tfunc_import) + + def test_clear_two_bp_on_same_line(self): + code = """ + def func(): + lno = 3 + lno = 4 + + def main(): + for i in range(3): + func() + """ + modules = { TEST_MODULE: code } + with create_modules(modules): + self.expect_set = [ + ('line', 2, 'tfunc_import'), ('break', (TEST_MODULE_FNAME, 3)), + ('None', 2, 'tfunc_import'), ('break', (TEST_MODULE_FNAME, 3)), + ('None', 2, 'tfunc_import'), ('break', (TEST_MODULE_FNAME, 4)), + ('None', 2, 'tfunc_import'), ('continue', ), + ('line', 3, 'func', ({1:1}, [])), ('continue', ), + ('line', 4, 'func', ({3:1}, [])), ('clear', (TEST_MODULE_FNAME, 3)), + ('None', 4, 'func'), ('continue', ), + ('line', 4, 'func', ({3:2}, [])), ('quit', ), + ] + with TracerRun(self) as tracer: + tracer.runcall(tfunc_import) + + def test_clear_at_no_bp(self): + self.expect_set = [ + ('line', 2, 'tfunc_import'), ('clear', (__file__, 1)) + ] + with TracerRun(self) as tracer: + self.assertRaises(BdbError, tracer.runcall, tfunc_import) + +class RunTestCase(BaseTestCase): + """Test run, runeval and set_trace.""" + + def test_run_step(self): + # Check that the bdb 'run' method stops at the first line event. + code = """ + lno = 2 + """ + self.expect_set = [ + ('line', 2, '<module>'), ('step', ), + ('return', 2, '<module>'), ('quit', ), + ] + with TracerRun(self) as tracer: + tracer.run(compile(textwrap.dedent(code), '<string>', 'exec')) + + def test_runeval_step(self): + # Test bdb 'runeval'. + code = """ + def main(): + lno = 3 + """ + modules = { TEST_MODULE: code } + with create_modules(modules): + self.expect_set = [ + ('line', 1, '<module>'), ('step', ), + ('call', 2, 'main'), ('step', ), + ('line', 3, 'main'), ('step', ), + ('return', 3, 'main'), ('step', ), + ('return', 1, '<module>'), ('quit', ), + ] + import test_module + with TracerRun(self) as tracer: + tracer.runeval('test_module.main()', globals(), locals()) + +class IssuesTestCase(BaseTestCase): + """Test fixed bdb issues.""" + + def test_step_at_return_with_no_trace_in_caller(self): + # Issue #13183. + # Check that the tracer does step into the caller frame when the + # trace function is not set in that frame. + code_1 = """ + from test_module_2 import func + def main(): + func() + lno = 5 + """ + code_2 = """ + def func(): + lno = 3 + """ + modules = { + TEST_MODULE: code_1, + 'test_module_2': code_2, + } + with create_modules(modules): + self.expect_set = [ + ('line', 2, 'tfunc_import'), + break_in_func('func', 'test_module_2.py'), + ('None', 2, 'tfunc_import'), ('continue', ), + ('line', 3, 'func', ({1:1}, [])), ('step', ), + ('return', 3, 'func'), ('step', ), + ('line', 5, 'main'), ('quit', ), + ] + with TracerRun(self) as tracer: + tracer.runcall(tfunc_import) + + def test_next_until_return_in_generator(self): + # Issue #16596. + # Check that set_next(), set_until() and set_return() do not treat the + # `yield` and `yield from` statements as if they were returns and stop + # instead in the current frame. + code = """ + def test_gen(): + yield 0 + lno = 4 + return 123 + + def main(): + it = test_gen() + next(it) + next(it) + lno = 11 + """ + modules = { TEST_MODULE: code } + for set_type in ('next', 'until', 'return'): + with self.subTest(set_type=set_type): + with create_modules(modules): + self.expect_set = [ + ('line', 2, 'tfunc_import'), + break_in_func('test_gen', TEST_MODULE_FNAME), + ('None', 2, 'tfunc_import'), ('continue', ), + ('line', 3, 'test_gen', ({1:1}, [])), (set_type, ), + ] + + if set_type == 'return': + self.expect_set.extend( + [('exception', 10, 'main', StopIteration), ('step',), + ('return', 10, 'main'), ('quit', ), + ] + ) + else: + self.expect_set.extend( + [('line', 4, 'test_gen'), ('quit', ),] + ) + with TracerRun(self) as tracer: + tracer.runcall(tfunc_import) + + def test_next_command_in_generator_for_loop(self): + # Issue #16596. + code = """ + def test_gen(): + yield 0 + lno = 4 + yield 1 + return 123 + + def main(): + for i in test_gen(): + lno = 10 + lno = 11 + """ + modules = { TEST_MODULE: code } + with create_modules(modules): + self.expect_set = [ + ('line', 2, 'tfunc_import'), + break_in_func('test_gen', TEST_MODULE_FNAME), + ('None', 2, 'tfunc_import'), ('continue', ), + ('line', 3, 'test_gen', ({1:1}, [])), ('next', ), + ('line', 4, 'test_gen'), ('next', ), + ('line', 5, 'test_gen'), ('next', ), + ('line', 6, 'test_gen'), ('next', ), + ('exception', 9, 'main', StopIteration), ('step', ), + ('line', 11, 'main'), ('quit', ), + + ] + with TracerRun(self) as tracer: + tracer.runcall(tfunc_import) + + def test_next_command_in_generator_with_subiterator(self): + # Issue #16596. + code = """ + def test_subgen(): + yield 0 + return 123 + + def test_gen(): + x = yield from test_subgen() + return 456 + + def main(): + for i in test_gen(): + lno = 12 + lno = 13 + """ + modules = { TEST_MODULE: code } + with create_modules(modules): + self.expect_set = [ + ('line', 2, 'tfunc_import'), + break_in_func('test_gen', TEST_MODULE_FNAME), + ('None', 2, 'tfunc_import'), ('continue', ), + ('line', 7, 'test_gen', ({1:1}, [])), ('next', ), + ('line', 8, 'test_gen'), ('next', ), + ('exception', 11, 'main', StopIteration), ('step', ), + ('line', 13, 'main'), ('quit', ), + + ] + with TracerRun(self) as tracer: + tracer.runcall(tfunc_import) + + def test_return_command_in_generator_with_subiterator(self): + # Issue #16596. + code = """ + def test_subgen(): + yield 0 + return 123 + + def test_gen(): + x = yield from test_subgen() + return 456 + + def main(): + for i in test_gen(): + lno = 12 + lno = 13 + """ + modules = { TEST_MODULE: code } + with create_modules(modules): + self.expect_set = [ + ('line', 2, 'tfunc_import'), + break_in_func('test_subgen', TEST_MODULE_FNAME), + ('None', 2, 'tfunc_import'), ('continue', ), + ('line', 3, 'test_subgen', ({1:1}, [])), ('return', ), + ('exception', 7, 'test_gen', StopIteration), ('return', ), + ('exception', 11, 'main', StopIteration), ('step', ), + ('line', 13, 'main'), ('quit', ), + + ] + with TracerRun(self) as tracer: + tracer.runcall(tfunc_import) + +def test_main(): + test.support.run_unittest( + StateTestCase, + RunTestCase, + BreakpointTestCase, + IssuesTestCase, + ) + +if __name__ == "__main__": + test_main() diff --git a/Lib/test/test_sundry.py b/Lib/test/test_sundry.py index dddf3265525a..4025c2354a6c 100644 --- a/Lib/test/test_sundry.py +++ b/Lib/test/test_sundry.py @@ -6,7 +6,7 @@ class TestUntestedModules(unittest.TestCase): def test_untested_modules_can_be_imported(self): - untested = ('bdb', 'encodings', 'formatter', 'tabnanny') + untested = ('encodings', 'formatter', 'tabnanny') with support.check_warnings(quiet=True): for name in untested: try: diff --git a/Misc/NEWS.d/next/Tests/2018-01-08-13-33-47.bpo-19417.2asoXy.rst b/Misc/NEWS.d/next/Tests/2018-01-08-13-33-47.bpo-19417.2asoXy.rst new file mode 100644 index 000000000000..739352fcdd67 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2018-01-08-13-33-47.bpo-19417.2asoXy.rst @@ -0,0 +1 @@ +Add test_bdb.py. From webhook-mailer at python.org Sun Mar 18 16:25:18 2018 From: webhook-mailer at python.org (Miss Islington (bot)) Date: Sun, 18 Mar 2018 20:25:18 -0000 Subject: [Python-checkins] bpo-19417: Add test_bdb.py (GH-5217) Message-ID: <mailman.96.1521404719.1871.python-checkins@python.org> https://github.com/python/cpython/commit/fdd8e8b4ffb68a4e8749bdc3b130fea7bbbf821e commit: fdd8e8b4ffb68a4e8749bdc3b130fea7bbbf821e branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: GitHub <noreply at github.com> date: 2018-03-18T13:25:15-07:00 summary: bpo-19417: Add test_bdb.py (GH-5217) (cherry picked from commit 3fe33043ee83d19e15551094fc1e0984617ded3c) Co-authored-by: xdegaye <xdegaye at gmail.com> files: A Lib/test/test_bdb.py A Misc/NEWS.d/next/Tests/2018-01-08-13-33-47.bpo-19417.2asoXy.rst M Lib/test/test_sundry.py diff --git a/Lib/test/test_bdb.py b/Lib/test/test_bdb.py new file mode 100644 index 000000000000..abefe6c4e57a --- /dev/null +++ b/Lib/test/test_bdb.py @@ -0,0 +1,1151 @@ +""" Test the bdb module. + + A test defines a list of tuples that may be seen as paired tuples, each + pair being defined by 'expect_tuple, set_tuple' as follows: + + ([event, [lineno[, co_name[, eargs]]]]), (set_type, [sargs]) + + * 'expect_tuple' describes the expected current state of the Bdb instance. + It may be the empty tuple and no check is done in that case. + * 'set_tuple' defines the set_*() method to be invoked when the Bdb + instance reaches this state. + + Example of an 'expect_tuple, set_tuple' pair: + + ('line', 2, 'tfunc_main'), ('step', ) + + Definitions of the members of the 'expect_tuple': + event: + Name of the trace event. The set methods that do not give back + control to the tracer [1] do not trigger a tracer event and in + that case the next 'event' may be 'None' by convention, its value + is not checked. + [1] Methods that trigger a trace event are set_step(), set_next(), + set_return(), set_until() and set_continue(). + lineno: + Line number. Line numbers are relative to the start of the + function when tracing a function in the test_bdb module (i.e. this + module). + co_name: + Name of the function being currently traced. + eargs: + A tuple: + * On an 'exception' event the tuple holds a class object, the + current exception must be an instance of this class. + * On a 'line' event, the tuple holds a dictionary and a list. The + dictionary maps each breakpoint number that has been hit on this + line to its hits count. The list holds the list of breakpoint + number temporaries that are being deleted. + + Definitions of the members of the 'set_tuple': + set_type: + The type of the set method to be invoked. This may + be the type of one of the Bdb set methods: 'step', 'next', + 'until', 'return', 'continue', 'break', 'quit', or the type of one + of the set methods added by test_bdb.Bdb: 'ignore', 'enable', + 'disable', 'clear', 'up', 'down'. + sargs: + The arguments of the set method if any, packed in a tuple. +""" + +import bdb as _bdb +import sys +import os +import unittest +import textwrap +import importlib +import linecache +from contextlib import contextmanager +from itertools import islice, repeat +import test.support + +class BdbException(Exception): pass +class BdbError(BdbException): """Error raised by the Bdb instance.""" +class BdbSyntaxError(BdbException): """Syntax error in the test case.""" +class BdbNotExpectedError(BdbException): """Unexpected result.""" + +# When 'dry_run' is set to true, expect tuples are ignored and the actual +# state of the tracer is printed after running each set_*() method of the test +# case. The full list of breakpoints and their attributes is also printed +# after each 'line' event where a breakpoint has been hit. +dry_run = 0 + +def reset_Breakpoint(): + _bdb.Breakpoint.next = 1 + _bdb.Breakpoint.bplist = {} + _bdb.Breakpoint.bpbynumber = [None] + +def info_breakpoints(): + bp_list = [bp for bp in _bdb.Breakpoint.bpbynumber if bp] + if not bp_list: + return '' + + header_added = False + for bp in bp_list: + if not header_added: + info = 'BpNum Temp Enb Hits Ignore Where\n' + header_added = True + + disp = 'yes ' if bp.temporary else 'no ' + enab = 'yes' if bp.enabled else 'no ' + info += ('%-5d %s %s %-4d %-6d at %s:%d' % + (bp.number, disp, enab, bp.hits, bp.ignore, + os.path.basename(bp.file), bp.line)) + if bp.cond: + info += '\n\tstop only if %s' % (bp.cond,) + info += '\n' + return info + +class Bdb(_bdb.Bdb): + """Extend Bdb to enhance test coverage.""" + + def trace_dispatch(self, frame, event, arg): + self.currentbp = None + return super().trace_dispatch(frame, event, arg) + + def set_break(self, filename, lineno, temporary=False, cond=None, + funcname=None): + if isinstance(funcname, str): + if filename == __file__: + globals_ = globals() + else: + module = importlib.import_module(filename[:-3]) + globals_ = module.__dict__ + func = eval(funcname, globals_) + code = func.__code__ + filename = code.co_filename + lineno = code.co_firstlineno + funcname = code.co_name + + res = super().set_break(filename, lineno, temporary=temporary, + cond=cond, funcname=funcname) + if isinstance(res, str): + raise BdbError(res) + return res + + def get_stack(self, f, t): + self.stack, self.index = super().get_stack(f, t) + self.frame = self.stack[self.index][0] + return self.stack, self.index + + def set_ignore(self, bpnum): + """Increment the ignore count of Breakpoint number 'bpnum'.""" + bp = self.get_bpbynumber(bpnum) + bp.ignore += 1 + + def set_enable(self, bpnum): + bp = self.get_bpbynumber(bpnum) + bp.enabled = True + + def set_disable(self, bpnum): + bp = self.get_bpbynumber(bpnum) + bp.enabled = False + + def set_clear(self, fname, lineno): + err = self.clear_break(fname, lineno) + if err: + raise BdbError(err) + + def set_up(self): + """Move up in the frame stack.""" + if not self.index: + raise BdbError('Oldest frame') + self.index -= 1 + self.frame = self.stack[self.index][0] + + def set_down(self): + """Move down in the frame stack.""" + if self.index + 1 == len(self.stack): + raise BdbError('Newest frame') + self.index += 1 + self.frame = self.stack[self.index][0] + +class Tracer(Bdb): + """A tracer for testing the bdb module.""" + + def __init__(self, expect_set, skip=None, dry_run=False, test_case=None): + super().__init__(skip=skip) + self.expect_set = expect_set + self.dry_run = dry_run + self.header = ('Dry-run results for %s:' % test_case if + test_case is not None else None) + self.init_test() + + def init_test(self): + self.cur_except = None + self.expect_set_no = 0 + self.breakpoint_hits = None + self.expected_list = list(islice(self.expect_set, 0, None, 2)) + self.set_list = list(islice(self.expect_set, 1, None, 2)) + + def trace_dispatch(self, frame, event, arg): + # On an 'exception' event, call_exc_trace() in Python/ceval.c discards + # a BdbException raised by the Tracer instance, so we raise it on the + # next trace_dispatch() call that occurs unless the set_quit() or + # set_continue() method has been invoked on the 'exception' event. + if self.cur_except is not None: + raise self.cur_except + + if event == 'exception': + try: + res = super().trace_dispatch(frame, event, arg) + return res + except BdbException as e: + self.cur_except = e + return self.trace_dispatch + else: + return super().trace_dispatch(frame, event, arg) + + def user_call(self, frame, argument_list): + # Adopt the same behavior as pdb and, as a side effect, skip also the + # first 'call' event when the Tracer is started with Tracer.runcall() + # which may be possibly considered as a bug. + if not self.stop_here(frame): + return + self.process_event('call', frame, argument_list) + self.next_set_method() + + def user_line(self, frame): + self.process_event('line', frame) + + if self.dry_run and self.breakpoint_hits: + info = info_breakpoints().strip('\n') + # Indent each line. + for line in info.split('\n'): + print(' ' + line) + self.delete_temporaries() + self.breakpoint_hits = None + + self.next_set_method() + + def user_return(self, frame, return_value): + self.process_event('return', frame, return_value) + self.next_set_method() + + def user_exception(self, frame, exc_info): + self.exc_info = exc_info + self.process_event('exception', frame) + self.next_set_method() + + def do_clear(self, arg): + # The temporary breakpoints are deleted in user_line(). + bp_list = [self.currentbp] + self.breakpoint_hits = (bp_list, bp_list) + + def delete_temporaries(self): + if self.breakpoint_hits: + for n in self.breakpoint_hits[1]: + self.clear_bpbynumber(n) + + def pop_next(self): + self.expect_set_no += 1 + try: + self.expect = self.expected_list.pop(0) + except IndexError: + raise BdbNotExpectedError( + 'expect_set list exhausted, cannot pop item %d' % + self.expect_set_no) + self.set_tuple = self.set_list.pop(0) + + def process_event(self, event, frame, *args): + # Call get_stack() to enable walking the stack with set_up() and + # set_down(). + tb = None + if event == 'exception': + tb = self.exc_info[2] + self.get_stack(frame, tb) + + # A breakpoint has been hit and it is not a temporary. + if self.currentbp is not None and not self.breakpoint_hits: + bp_list = [self.currentbp] + self.breakpoint_hits = (bp_list, []) + + # Pop next event. + self.event= event + self.pop_next() + if self.dry_run: + self.print_state(self.header) + return + + # Validate the expected results. + if self.expect: + self.check_equal(self.expect[0], event, 'Wrong event type') + self.check_lno_name() + + if event in ('call', 'return'): + self.check_expect_max_size(3) + elif len(self.expect) > 3: + if event == 'line': + bps, temporaries = self.expect[3] + bpnums = sorted(bps.keys()) + if not self.breakpoint_hits: + self.raise_not_expected( + 'No breakpoints hit at expect_set item %d' % + self.expect_set_no) + self.check_equal(bpnums, self.breakpoint_hits[0], + 'Breakpoint numbers do not match') + self.check_equal([bps[n] for n in bpnums], + [self.get_bpbynumber(n).hits for + n in self.breakpoint_hits[0]], + 'Wrong breakpoint hit count') + self.check_equal(sorted(temporaries), self.breakpoint_hits[1], + 'Wrong temporary breakpoints') + + elif event == 'exception': + if not isinstance(self.exc_info[1], self.expect[3]): + self.raise_not_expected( + "Wrong exception at expect_set item %d, got '%s'" % + (self.expect_set_no, self.exc_info)) + + def check_equal(self, expected, result, msg): + if expected == result: + return + self.raise_not_expected("%s at expect_set item %d, got '%s'" % + (msg, self.expect_set_no, result)) + + def check_lno_name(self): + """Check the line number and function co_name.""" + s = len(self.expect) + if s > 1: + lineno = self.lno_abs2rel() + self.check_equal(self.expect[1], lineno, 'Wrong line number') + if s > 2: + self.check_equal(self.expect[2], self.frame.f_code.co_name, + 'Wrong function name') + + def check_expect_max_size(self, size): + if len(self.expect) > size: + raise BdbSyntaxError('Invalid size of the %s expect tuple: %s' % + (self.event, self.expect)) + + def lno_abs2rel(self): + fname = self.canonic(self.frame.f_code.co_filename) + lineno = self.frame.f_lineno + return ((lineno - self.frame.f_code.co_firstlineno + 1) + if fname == self.canonic(__file__) else lineno) + + def lno_rel2abs(self, fname, lineno): + return (self.frame.f_code.co_firstlineno + lineno - 1 + if (lineno and self.canonic(fname) == self.canonic(__file__)) + else lineno) + + def get_state(self): + lineno = self.lno_abs2rel() + co_name = self.frame.f_code.co_name + state = "('%s', %d, '%s'" % (self.event, lineno, co_name) + if self.breakpoint_hits: + bps = '{' + for n in self.breakpoint_hits[0]: + if bps != '{': + bps += ', ' + bps += '%s: %s' % (n, self.get_bpbynumber(n).hits) + bps += '}' + bps = '(' + bps + ', ' + str(self.breakpoint_hits[1]) + ')' + state += ', ' + bps + elif self.event == 'exception': + state += ', ' + self.exc_info[0].__name__ + state += '), ' + return state.ljust(32) + str(self.set_tuple) + ',' + + def print_state(self, header=None): + if header is not None and self.expect_set_no == 1: + print() + print(header) + print('%d: %s' % (self.expect_set_no, self.get_state())) + + def raise_not_expected(self, msg): + msg += '\n' + msg += ' Expected: %s\n' % str(self.expect) + msg += ' Got: ' + self.get_state() + raise BdbNotExpectedError(msg) + + def next_set_method(self): + set_type = self.set_tuple[0] + args = self.set_tuple[1] if len(self.set_tuple) == 2 else None + set_method = getattr(self, 'set_' + set_type) + + # The following set methods give back control to the tracer. + if set_type in ('step', 'continue', 'quit'): + set_method() + return + elif set_type in ('next', 'return'): + set_method(self.frame) + return + elif set_type == 'until': + lineno = None + if args: + lineno = self.lno_rel2abs(self.frame.f_code.co_filename, + args[0]) + set_method(self.frame, lineno) + return + + # The following set methods do not give back control to the tracer and + # next_set_method() is called recursively. + if (args and set_type in ('break', 'clear', 'ignore', 'enable', + 'disable')) or set_type in ('up', 'down'): + if set_type in ('break', 'clear'): + fname, lineno, *remain = args + lineno = self.lno_rel2abs(fname, lineno) + args = [fname, lineno] + args.extend(remain) + set_method(*args) + elif set_type in ('ignore', 'enable', 'disable'): + set_method(*args) + elif set_type in ('up', 'down'): + set_method() + + # Process the next expect_set item. + # It is not expected that a test may reach the recursion limit. + self.event= None + self.pop_next() + if self.dry_run: + self.print_state() + else: + if self.expect: + self.check_lno_name() + self.check_expect_max_size(3) + self.next_set_method() + else: + raise BdbSyntaxError('"%s" is an invalid set_tuple' % + self.set_tuple) + +class TracerRun(): + """Provide a context for running a Tracer instance with a test case.""" + + def __init__(self, test_case, skip=None): + self.test_case = test_case + self.dry_run = test_case.dry_run + self.tracer = Tracer(test_case.expect_set, skip=skip, + dry_run=self.dry_run, test_case=test_case.id()) + + def __enter__(self): + # test_pdb does not reset Breakpoint class attributes on exit :-( + reset_Breakpoint() + return self.tracer + + def __exit__(self, type_=None, value=None, traceback=None): + reset_Breakpoint() + sys.settrace(None) + + not_empty = '' + if self.tracer.set_list: + not_empty += 'All paired tuples have not been processed, ' + not_empty += ('the last one was number %d' % + self.tracer.expect_set_no) + + # Make a BdbNotExpectedError a unittest failure. + if type_ is not None and issubclass(BdbNotExpectedError, type_): + if isinstance(value, BaseException) and value.args: + err_msg = value.args[0] + if not_empty: + err_msg += '\n' + not_empty + if self.dry_run: + print(err_msg) + return True + else: + self.test_case.fail(err_msg) + else: + assert False, 'BdbNotExpectedError with empty args' + + if not_empty: + if self.dry_run: + print(not_empty) + else: + self.test_case.fail(not_empty) + +def run_test(modules, set_list, skip=None): + """Run a test and print the dry-run results. + + 'modules': A dictionary mapping module names to their source code as a + string. The dictionary MUST include one module named + 'test_module' with a main() function. + 'set_list': A list of set_type tuples to be run on the module. + + For example, running the following script outputs the following results: + + ***************************** SCRIPT ******************************** + + from test.test_bdb import run_test, break_in_func + + code = ''' + def func(): + lno = 3 + + def main(): + func() + lno = 7 + ''' + + set_list = [ + break_in_func('func', 'test_module.py'), + ('continue', ), + ('step', ), + ('step', ), + ('step', ), + ('quit', ), + ] + + modules = { 'test_module': code } + run_test(modules, set_list) + + **************************** results ******************************** + + 1: ('line', 2, 'tfunc_import'), ('next',), + 2: ('line', 3, 'tfunc_import'), ('step',), + 3: ('call', 5, 'main'), ('break', ('test_module.py', None, False, None, 'func')), + 4: ('None', 5, 'main'), ('continue',), + 5: ('line', 3, 'func', ({1: 1}, [])), ('step',), + BpNum Temp Enb Hits Ignore Where + 1 no yes 1 0 at test_module.py:2 + 6: ('return', 3, 'func'), ('step',), + 7: ('line', 7, 'main'), ('step',), + 8: ('return', 7, 'main'), ('quit',), + + ************************************************************************* + + """ + def gen(a, b): + try: + while 1: + x = next(a) + y = next(b) + yield x + yield y + except StopIteration: + return + + # Step over the import statement in tfunc_import using 'next' and step + # into main() in test_module. + sl = [('next', ), ('step', )] + sl.extend(set_list) + + test = BaseTestCase() + test.dry_run = True + test.id = lambda : None + test.expect_set = list(gen(repeat(()), iter(sl))) + with create_modules(modules): + sys.path.append(os.getcwd()) + with TracerRun(test, skip=skip) as tracer: + tracer.runcall(tfunc_import) + + at contextmanager +def create_modules(modules): + with test.support.temp_cwd(): + try: + for m in modules: + fname = m + '.py' + with open(fname, 'w') as f: + f.write(textwrap.dedent(modules[m])) + linecache.checkcache(fname) + importlib.invalidate_caches() + yield + finally: + for m in modules: + test.support.forget(m) + +def break_in_func(funcname, fname=__file__, temporary=False, cond=None): + return 'break', (fname, None, temporary, cond, funcname) + +TEST_MODULE = 'test_module' +TEST_MODULE_FNAME = TEST_MODULE + '.py' +def tfunc_import(): + import test_module + test_module.main() + +def tfunc_main(): + lno = 2 + tfunc_first() + tfunc_second() + lno = 5 + lno = 6 + lno = 7 + +def tfunc_first(): + lno = 2 + lno = 3 + lno = 4 + +def tfunc_second(): + lno = 2 + +class BaseTestCase(unittest.TestCase): + """Base class for all tests.""" + + dry_run = dry_run + + def fail(self, msg=None): + # Override fail() to use 'raise from None' to avoid repetition of the + # error message and traceback. + raise self.failureException(msg) from None + +class StateTestCase(BaseTestCase): + """Test the step, next, return, until and quit 'set_' methods.""" + + def test_step(self): + self.expect_set = [ + ('line', 2, 'tfunc_main'), ('step', ), + ('line', 3, 'tfunc_main'), ('step', ), + ('call', 1, 'tfunc_first'), ('step', ), + ('line', 2, 'tfunc_first'), ('quit', ), + ] + with TracerRun(self) as tracer: + tracer.runcall(tfunc_main) + + def test_step_next_on_last_statement(self): + for set_type in ('step', 'next'): + with self.subTest(set_type=set_type): + self.expect_set = [ + ('line', 2, 'tfunc_main'), ('step', ), + ('line', 3, 'tfunc_main'), ('step', ), + ('call', 1, 'tfunc_first'), ('break', (__file__, 3)), + ('None', 1, 'tfunc_first'), ('continue', ), + ('line', 3, 'tfunc_first', ({1:1}, [])), (set_type, ), + ('line', 4, 'tfunc_first'), ('quit', ), + ] + with TracerRun(self) as tracer: + tracer.runcall(tfunc_main) + + def test_next(self): + self.expect_set = [ + ('line', 2, 'tfunc_main'), ('step', ), + ('line', 3, 'tfunc_main'), ('next', ), + ('line', 4, 'tfunc_main'), ('step', ), + ('call', 1, 'tfunc_second'), ('step', ), + ('line', 2, 'tfunc_second'), ('quit', ), + ] + with TracerRun(self) as tracer: + tracer.runcall(tfunc_main) + + def test_next_over_import(self): + code = """ + def main(): + lno = 3 + """ + modules = { TEST_MODULE: code } + with create_modules(modules): + self.expect_set = [ + ('line', 2, 'tfunc_import'), ('next', ), + ('line', 3, 'tfunc_import'), ('quit', ), + ] + with TracerRun(self) as tracer: + tracer.runcall(tfunc_import) + + def test_next_on_plain_statement(self): + # Check that set_next() is equivalent to set_step() on a plain + # statement. + self.expect_set = [ + ('line', 2, 'tfunc_main'), ('step', ), + ('line', 3, 'tfunc_main'), ('step', ), + ('call', 1, 'tfunc_first'), ('next', ), + ('line', 2, 'tfunc_first'), ('quit', ), + ] + with TracerRun(self) as tracer: + tracer.runcall(tfunc_main) + + def test_next_in_caller_frame(self): + # Check that set_next() in the caller frame causes the tracer + # to stop next in the caller frame. + self.expect_set = [ + ('line', 2, 'tfunc_main'), ('step', ), + ('line', 3, 'tfunc_main'), ('step', ), + ('call', 1, 'tfunc_first'), ('up', ), + ('None', 3, 'tfunc_main'), ('next', ), + ('line', 4, 'tfunc_main'), ('quit', ), + ] + with TracerRun(self) as tracer: + tracer.runcall(tfunc_main) + + def test_return(self): + self.expect_set = [ + ('line', 2, 'tfunc_main'), ('step', ), + ('line', 3, 'tfunc_main'), ('step', ), + ('call', 1, 'tfunc_first'), ('step', ), + ('line', 2, 'tfunc_first'), ('return', ), + ('return', 4, 'tfunc_first'), ('step', ), + ('line', 4, 'tfunc_main'), ('quit', ), + ] + with TracerRun(self) as tracer: + tracer.runcall(tfunc_main) + + def test_return_in_caller_frame(self): + self.expect_set = [ + ('line', 2, 'tfunc_main'), ('step', ), + ('line', 3, 'tfunc_main'), ('step', ), + ('call', 1, 'tfunc_first'), ('up', ), + ('None', 3, 'tfunc_main'), ('return', ), + ('return', 7, 'tfunc_main'), ('quit', ), + ] + with TracerRun(self) as tracer: + tracer.runcall(tfunc_main) + + def test_until(self): + self.expect_set = [ + ('line', 2, 'tfunc_main'), ('step', ), + ('line', 3, 'tfunc_main'), ('step', ), + ('call', 1, 'tfunc_first'), ('step', ), + ('line', 2, 'tfunc_first'), ('until', (4, )), + ('line', 4, 'tfunc_first'), ('quit', ), + ] + with TracerRun(self) as tracer: + tracer.runcall(tfunc_main) + + def test_until_with_too_large_count(self): + self.expect_set = [ + ('line', 2, 'tfunc_main'), break_in_func('tfunc_first'), + ('None', 2, 'tfunc_main'), ('continue', ), + ('line', 2, 'tfunc_first', ({1:1}, [])), ('until', (9999, )), + ('return', 4, 'tfunc_first'), ('quit', ), + ] + with TracerRun(self) as tracer: + tracer.runcall(tfunc_main) + + def test_until_in_caller_frame(self): + self.expect_set = [ + ('line', 2, 'tfunc_main'), ('step', ), + ('line', 3, 'tfunc_main'), ('step', ), + ('call', 1, 'tfunc_first'), ('up', ), + ('None', 3, 'tfunc_main'), ('until', (6, )), + ('line', 6, 'tfunc_main'), ('quit', ), + ] + with TracerRun(self) as tracer: + tracer.runcall(tfunc_main) + + def test_skip(self): + # Check that tracing is skipped over the import statement in + # 'tfunc_import()'. + code = """ + def main(): + lno = 3 + """ + modules = { TEST_MODULE: code } + with create_modules(modules): + self.expect_set = [ + ('line', 2, 'tfunc_import'), ('step', ), + ('line', 3, 'tfunc_import'), ('quit', ), + ] + skip = ('importlib*', TEST_MODULE) + with TracerRun(self, skip=skip) as tracer: + tracer.runcall(tfunc_import) + + def test_down(self): + # Check that set_down() raises BdbError at the newest frame. + self.expect_set = [ + ('line', 2, 'tfunc_main'), ('down', ), + ] + with TracerRun(self) as tracer: + self.assertRaises(BdbError, tracer.runcall, tfunc_main) + + def test_up(self): + self.expect_set = [ + ('line', 2, 'tfunc_main'), ('step', ), + ('line', 3, 'tfunc_main'), ('step', ), + ('call', 1, 'tfunc_first'), ('up', ), + ('None', 3, 'tfunc_main'), ('quit', ), + ] + with TracerRun(self) as tracer: + tracer.runcall(tfunc_main) + +class BreakpointTestCase(BaseTestCase): + """Test the breakpoint set method.""" + + def test_bp_on_non_existent_module(self): + self.expect_set = [ + ('line', 2, 'tfunc_import'), ('break', ('/non/existent/module.py', 1)) + ] + with TracerRun(self) as tracer: + self.assertRaises(BdbError, tracer.runcall, tfunc_import) + + def test_bp_after_last_statement(self): + code = """ + def main(): + lno = 3 + """ + modules = { TEST_MODULE: code } + with create_modules(modules): + self.expect_set = [ + ('line', 2, 'tfunc_import'), ('break', (TEST_MODULE_FNAME, 4)) + ] + with TracerRun(self) as tracer: + self.assertRaises(BdbError, tracer.runcall, tfunc_import) + + def test_temporary_bp(self): + code = """ + def func(): + lno = 3 + + def main(): + for i in range(2): + func() + """ + modules = { TEST_MODULE: code } + with create_modules(modules): + self.expect_set = [ + ('line', 2, 'tfunc_import'), + break_in_func('func', TEST_MODULE_FNAME, True), + ('None', 2, 'tfunc_import'), + break_in_func('func', TEST_MODULE_FNAME, True), + ('None', 2, 'tfunc_import'), ('continue', ), + ('line', 3, 'func', ({1:1}, [1])), ('continue', ), + ('line', 3, 'func', ({2:1}, [2])), ('quit', ), + ] + with TracerRun(self) as tracer: + tracer.runcall(tfunc_import) + + def test_disabled_temporary_bp(self): + code = """ + def func(): + lno = 3 + + def main(): + for i in range(3): + func() + """ + modules = { TEST_MODULE: code } + with create_modules(modules): + self.expect_set = [ + ('line', 2, 'tfunc_import'), + break_in_func('func', TEST_MODULE_FNAME), + ('None', 2, 'tfunc_import'), + break_in_func('func', TEST_MODULE_FNAME, True), + ('None', 2, 'tfunc_import'), ('disable', (2, )), + ('None', 2, 'tfunc_import'), ('continue', ), + ('line', 3, 'func', ({1:1}, [])), ('enable', (2, )), + ('None', 3, 'func'), ('disable', (1, )), + ('None', 3, 'func'), ('continue', ), + ('line', 3, 'func', ({2:1}, [2])), ('enable', (1, )), + ('None', 3, 'func'), ('continue', ), + ('line', 3, 'func', ({1:2}, [])), ('quit', ), + ] + with TracerRun(self) as tracer: + tracer.runcall(tfunc_import) + + def test_bp_condition(self): + code = """ + def func(a): + lno = 3 + + def main(): + for i in range(3): + func(i) + """ + modules = { TEST_MODULE: code } + with create_modules(modules): + self.expect_set = [ + ('line', 2, 'tfunc_import'), + break_in_func('func', TEST_MODULE_FNAME, False, 'a == 2'), + ('None', 2, 'tfunc_import'), ('continue', ), + ('line', 3, 'func', ({1:3}, [])), ('quit', ), + ] + with TracerRun(self) as tracer: + tracer.runcall(tfunc_import) + + def test_bp_exception_on_condition_evaluation(self): + code = """ + def func(a): + lno = 3 + + def main(): + func(0) + """ + modules = { TEST_MODULE: code } + with create_modules(modules): + self.expect_set = [ + ('line', 2, 'tfunc_import'), + break_in_func('func', TEST_MODULE_FNAME, False, '1 / 0'), + ('None', 2, 'tfunc_import'), ('continue', ), + ('line', 3, 'func', ({1:1}, [])), ('quit', ), + ] + with TracerRun(self) as tracer: + tracer.runcall(tfunc_import) + + def test_bp_ignore_count(self): + code = """ + def func(): + lno = 3 + + def main(): + for i in range(2): + func() + """ + modules = { TEST_MODULE: code } + with create_modules(modules): + self.expect_set = [ + ('line', 2, 'tfunc_import'), + break_in_func('func', TEST_MODULE_FNAME), + ('None', 2, 'tfunc_import'), ('ignore', (1, )), + ('None', 2, 'tfunc_import'), ('continue', ), + ('line', 3, 'func', ({1:2}, [])), ('quit', ), + ] + with TracerRun(self) as tracer: + tracer.runcall(tfunc_import) + + def test_ignore_count_on_disabled_bp(self): + code = """ + def func(): + lno = 3 + + def main(): + for i in range(3): + func() + """ + modules = { TEST_MODULE: code } + with create_modules(modules): + self.expect_set = [ + ('line', 2, 'tfunc_import'), + break_in_func('func', TEST_MODULE_FNAME), + ('None', 2, 'tfunc_import'), + break_in_func('func', TEST_MODULE_FNAME), + ('None', 2, 'tfunc_import'), ('ignore', (1, )), + ('None', 2, 'tfunc_import'), ('disable', (1, )), + ('None', 2, 'tfunc_import'), ('continue', ), + ('line', 3, 'func', ({2:1}, [])), ('enable', (1, )), + ('None', 3, 'func'), ('continue', ), + ('line', 3, 'func', ({2:2}, [])), ('continue', ), + ('line', 3, 'func', ({1:2}, [])), ('quit', ), + ] + with TracerRun(self) as tracer: + tracer.runcall(tfunc_import) + + def test_clear_two_bp_on_same_line(self): + code = """ + def func(): + lno = 3 + lno = 4 + + def main(): + for i in range(3): + func() + """ + modules = { TEST_MODULE: code } + with create_modules(modules): + self.expect_set = [ + ('line', 2, 'tfunc_import'), ('break', (TEST_MODULE_FNAME, 3)), + ('None', 2, 'tfunc_import'), ('break', (TEST_MODULE_FNAME, 3)), + ('None', 2, 'tfunc_import'), ('break', (TEST_MODULE_FNAME, 4)), + ('None', 2, 'tfunc_import'), ('continue', ), + ('line', 3, 'func', ({1:1}, [])), ('continue', ), + ('line', 4, 'func', ({3:1}, [])), ('clear', (TEST_MODULE_FNAME, 3)), + ('None', 4, 'func'), ('continue', ), + ('line', 4, 'func', ({3:2}, [])), ('quit', ), + ] + with TracerRun(self) as tracer: + tracer.runcall(tfunc_import) + + def test_clear_at_no_bp(self): + self.expect_set = [ + ('line', 2, 'tfunc_import'), ('clear', (__file__, 1)) + ] + with TracerRun(self) as tracer: + self.assertRaises(BdbError, tracer.runcall, tfunc_import) + +class RunTestCase(BaseTestCase): + """Test run, runeval and set_trace.""" + + def test_run_step(self): + # Check that the bdb 'run' method stops at the first line event. + code = """ + lno = 2 + """ + self.expect_set = [ + ('line', 2, '<module>'), ('step', ), + ('return', 2, '<module>'), ('quit', ), + ] + with TracerRun(self) as tracer: + tracer.run(compile(textwrap.dedent(code), '<string>', 'exec')) + + def test_runeval_step(self): + # Test bdb 'runeval'. + code = """ + def main(): + lno = 3 + """ + modules = { TEST_MODULE: code } + with create_modules(modules): + self.expect_set = [ + ('line', 1, '<module>'), ('step', ), + ('call', 2, 'main'), ('step', ), + ('line', 3, 'main'), ('step', ), + ('return', 3, 'main'), ('step', ), + ('return', 1, '<module>'), ('quit', ), + ] + import test_module + with TracerRun(self) as tracer: + tracer.runeval('test_module.main()', globals(), locals()) + +class IssuesTestCase(BaseTestCase): + """Test fixed bdb issues.""" + + def test_step_at_return_with_no_trace_in_caller(self): + # Issue #13183. + # Check that the tracer does step into the caller frame when the + # trace function is not set in that frame. + code_1 = """ + from test_module_2 import func + def main(): + func() + lno = 5 + """ + code_2 = """ + def func(): + lno = 3 + """ + modules = { + TEST_MODULE: code_1, + 'test_module_2': code_2, + } + with create_modules(modules): + self.expect_set = [ + ('line', 2, 'tfunc_import'), + break_in_func('func', 'test_module_2.py'), + ('None', 2, 'tfunc_import'), ('continue', ), + ('line', 3, 'func', ({1:1}, [])), ('step', ), + ('return', 3, 'func'), ('step', ), + ('line', 5, 'main'), ('quit', ), + ] + with TracerRun(self) as tracer: + tracer.runcall(tfunc_import) + + def test_next_until_return_in_generator(self): + # Issue #16596. + # Check that set_next(), set_until() and set_return() do not treat the + # `yield` and `yield from` statements as if they were returns and stop + # instead in the current frame. + code = """ + def test_gen(): + yield 0 + lno = 4 + return 123 + + def main(): + it = test_gen() + next(it) + next(it) + lno = 11 + """ + modules = { TEST_MODULE: code } + for set_type in ('next', 'until', 'return'): + with self.subTest(set_type=set_type): + with create_modules(modules): + self.expect_set = [ + ('line', 2, 'tfunc_import'), + break_in_func('test_gen', TEST_MODULE_FNAME), + ('None', 2, 'tfunc_import'), ('continue', ), + ('line', 3, 'test_gen', ({1:1}, [])), (set_type, ), + ] + + if set_type == 'return': + self.expect_set.extend( + [('exception', 10, 'main', StopIteration), ('step',), + ('return', 10, 'main'), ('quit', ), + ] + ) + else: + self.expect_set.extend( + [('line', 4, 'test_gen'), ('quit', ),] + ) + with TracerRun(self) as tracer: + tracer.runcall(tfunc_import) + + def test_next_command_in_generator_for_loop(self): + # Issue #16596. + code = """ + def test_gen(): + yield 0 + lno = 4 + yield 1 + return 123 + + def main(): + for i in test_gen(): + lno = 10 + lno = 11 + """ + modules = { TEST_MODULE: code } + with create_modules(modules): + self.expect_set = [ + ('line', 2, 'tfunc_import'), + break_in_func('test_gen', TEST_MODULE_FNAME), + ('None', 2, 'tfunc_import'), ('continue', ), + ('line', 3, 'test_gen', ({1:1}, [])), ('next', ), + ('line', 4, 'test_gen'), ('next', ), + ('line', 5, 'test_gen'), ('next', ), + ('line', 6, 'test_gen'), ('next', ), + ('exception', 9, 'main', StopIteration), ('step', ), + ('line', 11, 'main'), ('quit', ), + + ] + with TracerRun(self) as tracer: + tracer.runcall(tfunc_import) + + def test_next_command_in_generator_with_subiterator(self): + # Issue #16596. + code = """ + def test_subgen(): + yield 0 + return 123 + + def test_gen(): + x = yield from test_subgen() + return 456 + + def main(): + for i in test_gen(): + lno = 12 + lno = 13 + """ + modules = { TEST_MODULE: code } + with create_modules(modules): + self.expect_set = [ + ('line', 2, 'tfunc_import'), + break_in_func('test_gen', TEST_MODULE_FNAME), + ('None', 2, 'tfunc_import'), ('continue', ), + ('line', 7, 'test_gen', ({1:1}, [])), ('next', ), + ('line', 8, 'test_gen'), ('next', ), + ('exception', 11, 'main', StopIteration), ('step', ), + ('line', 13, 'main'), ('quit', ), + + ] + with TracerRun(self) as tracer: + tracer.runcall(tfunc_import) + + def test_return_command_in_generator_with_subiterator(self): + # Issue #16596. + code = """ + def test_subgen(): + yield 0 + return 123 + + def test_gen(): + x = yield from test_subgen() + return 456 + + def main(): + for i in test_gen(): + lno = 12 + lno = 13 + """ + modules = { TEST_MODULE: code } + with create_modules(modules): + self.expect_set = [ + ('line', 2, 'tfunc_import'), + break_in_func('test_subgen', TEST_MODULE_FNAME), + ('None', 2, 'tfunc_import'), ('continue', ), + ('line', 3, 'test_subgen', ({1:1}, [])), ('return', ), + ('exception', 7, 'test_gen', StopIteration), ('return', ), + ('exception', 11, 'main', StopIteration), ('step', ), + ('line', 13, 'main'), ('quit', ), + + ] + with TracerRun(self) as tracer: + tracer.runcall(tfunc_import) + +def test_main(): + test.support.run_unittest( + StateTestCase, + RunTestCase, + BreakpointTestCase, + IssuesTestCase, + ) + +if __name__ == "__main__": + test_main() diff --git a/Lib/test/test_sundry.py b/Lib/test/test_sundry.py index dddf3265525a..4025c2354a6c 100644 --- a/Lib/test/test_sundry.py +++ b/Lib/test/test_sundry.py @@ -6,7 +6,7 @@ class TestUntestedModules(unittest.TestCase): def test_untested_modules_can_be_imported(self): - untested = ('bdb', 'encodings', 'formatter', 'tabnanny') + untested = ('encodings', 'formatter', 'tabnanny') with support.check_warnings(quiet=True): for name in untested: try: diff --git a/Misc/NEWS.d/next/Tests/2018-01-08-13-33-47.bpo-19417.2asoXy.rst b/Misc/NEWS.d/next/Tests/2018-01-08-13-33-47.bpo-19417.2asoXy.rst new file mode 100644 index 000000000000..739352fcdd67 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2018-01-08-13-33-47.bpo-19417.2asoXy.rst @@ -0,0 +1 @@ +Add test_bdb.py. From webhook-mailer at python.org Sun Mar 18 16:50:44 2018 From: webhook-mailer at python.org (Miss Islington (bot)) Date: Sun, 18 Mar 2018 20:50:44 -0000 Subject: [Python-checkins] bpo-32056: Improve exceptions in aifc, wave and sunau. (GH-5951) Message-ID: <mailman.97.1521406245.1871.python-checkins@python.org> https://github.com/python/cpython/commit/3c0a5a7c7ba8fbbc95dd1fe76cd7a1c0ce167371 commit: 3c0a5a7c7ba8fbbc95dd1fe76cd7a1c0ce167371 branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: GitHub <noreply at github.com> date: 2018-03-18T13:50:41-07:00 summary: bpo-32056: Improve exceptions in aifc, wave and sunau. (GH-5951) (cherry picked from commit 134cb01cda50f02725575808130b05d2d776693f) Co-authored-by: Serhiy Storchaka <storchaka at gmail.com> files: A Misc/NEWS.d/next/Library/2018-03-01-17-49-56.bpo-32056.IlpfgE.rst M Lib/aifc.py M Lib/sunau.py M Lib/test/test_aifc.py M Lib/test/test_sunau.py M Lib/test/test_wave.py M Lib/wave.py diff --git a/Lib/aifc.py b/Lib/aifc.py index 3d2dc56de198..1916e7ef8e7e 100644 --- a/Lib/aifc.py +++ b/Lib/aifc.py @@ -467,6 +467,10 @@ def _read_comm_chunk(self, chunk): self._nframes = _read_long(chunk) self._sampwidth = (_read_short(chunk) + 7) // 8 self._framerate = int(_read_float(chunk)) + if self._sampwidth <= 0: + raise Error('bad sample width') + if self._nchannels <= 0: + raise Error('bad # of channels') self._framesize = self._nchannels * self._sampwidth if self._aifc: #DEBUG: SGI's soundeditor produces a bad size :-( diff --git a/Lib/sunau.py b/Lib/sunau.py index dbad3db8392d..129502b0b417 100644 --- a/Lib/sunau.py +++ b/Lib/sunau.py @@ -208,6 +208,8 @@ def initfp(self, file): raise Error('unknown encoding') self._framerate = int(_read_u32(file)) self._nchannels = int(_read_u32(file)) + if not self._nchannels: + raise Error('bad # of channels') self._framesize = self._framesize * self._nchannels if self._hdr_size > 24: self._info = file.read(self._hdr_size - 24) diff --git a/Lib/test/test_aifc.py b/Lib/test/test_aifc.py index 8fd306a36592..ff52f5b6feb8 100644 --- a/Lib/test/test_aifc.py +++ b/Lib/test/test_aifc.py @@ -268,7 +268,8 @@ def test_read_no_comm_chunk(self): def test_read_no_ssnd_chunk(self): b = b'FORM' + struct.pack('>L', 4) + b'AIFC' - b += b'COMM' + struct.pack('>LhlhhLL', 38, 0, 0, 0, 0, 0, 0) + b += b'COMM' + struct.pack('>LhlhhLL', 38, 1, 0, 8, + 0x4000 | 12, 11025<<18, 0) b += b'NONE' + struct.pack('B', 14) + b'not compressed' + b'\x00' with self.assertRaisesRegex(aifc.Error, 'COMM chunk and/or SSND chunk' ' missing'): @@ -276,13 +277,35 @@ def test_read_no_ssnd_chunk(self): def test_read_wrong_compression_type(self): b = b'FORM' + struct.pack('>L', 4) + b'AIFC' - b += b'COMM' + struct.pack('>LhlhhLL', 23, 0, 0, 0, 0, 0, 0) + b += b'COMM' + struct.pack('>LhlhhLL', 23, 1, 0, 8, + 0x4000 | 12, 11025<<18, 0) b += b'WRNG' + struct.pack('B', 0) self.assertRaises(aifc.Error, aifc.open, io.BytesIO(b)) + def test_read_wrong_number_of_channels(self): + for nchannels in 0, -1: + b = b'FORM' + struct.pack('>L', 4) + b'AIFC' + b += b'COMM' + struct.pack('>LhlhhLL', 38, nchannels, 0, 8, + 0x4000 | 12, 11025<<18, 0) + b += b'NONE' + struct.pack('B', 14) + b'not compressed' + b'\x00' + b += b'SSND' + struct.pack('>L', 8) + b'\x00' * 8 + with self.assertRaisesRegex(aifc.Error, 'bad # of channels'): + aifc.open(io.BytesIO(b)) + + def test_read_wrong_sample_width(self): + for sampwidth in 0, -1: + b = b'FORM' + struct.pack('>L', 4) + b'AIFC' + b += b'COMM' + struct.pack('>LhlhhLL', 38, 1, 0, sampwidth, + 0x4000 | 12, 11025<<18, 0) + b += b'NONE' + struct.pack('B', 14) + b'not compressed' + b'\x00' + b += b'SSND' + struct.pack('>L', 8) + b'\x00' * 8 + with self.assertRaisesRegex(aifc.Error, 'bad sample width'): + aifc.open(io.BytesIO(b)) + def test_read_wrong_marks(self): b = b'FORM' + struct.pack('>L', 4) + b'AIFF' - b += b'COMM' + struct.pack('>LhlhhLL', 18, 0, 0, 0, 0, 0, 0) + b += b'COMM' + struct.pack('>LhlhhLL', 18, 1, 0, 8, + 0x4000 | 12, 11025<<18, 0) b += b'SSND' + struct.pack('>L', 8) + b'\x00' * 8 b += b'MARK' + struct.pack('>LhB', 3, 1, 1) with self.assertWarns(UserWarning) as cm: @@ -293,7 +316,8 @@ def test_read_wrong_marks(self): def test_read_comm_kludge_compname_even(self): b = b'FORM' + struct.pack('>L', 4) + b'AIFC' - b += b'COMM' + struct.pack('>LhlhhLL', 18, 0, 0, 0, 0, 0, 0) + b += b'COMM' + struct.pack('>LhlhhLL', 18, 1, 0, 8, + 0x4000 | 12, 11025<<18, 0) b += b'NONE' + struct.pack('B', 4) + b'even' + b'\x00' b += b'SSND' + struct.pack('>L', 8) + b'\x00' * 8 with self.assertWarns(UserWarning) as cm: @@ -303,7 +327,8 @@ def test_read_comm_kludge_compname_even(self): def test_read_comm_kludge_compname_odd(self): b = b'FORM' + struct.pack('>L', 4) + b'AIFC' - b += b'COMM' + struct.pack('>LhlhhLL', 18, 0, 0, 0, 0, 0, 0) + b += b'COMM' + struct.pack('>LhlhhLL', 18, 1, 0, 8, + 0x4000 | 12, 11025<<18, 0) b += b'NONE' + struct.pack('B', 3) + b'odd' b += b'SSND' + struct.pack('>L', 8) + b'\x00' * 8 with self.assertWarns(UserWarning) as cm: diff --git a/Lib/test/test_sunau.py b/Lib/test/test_sunau.py index 966224b1df5a..470a1007b4d4 100644 --- a/Lib/test/test_sunau.py +++ b/Lib/test/test_sunau.py @@ -1,6 +1,8 @@ import unittest from test import audiotests from audioop import byteswap +import io +import struct import sys import sunau @@ -121,5 +123,40 @@ class SunauMiscTests(audiotests.AudioMiscTests, unittest.TestCase): module = sunau +class SunauLowLevelTest(unittest.TestCase): + + def test_read_bad_magic_number(self): + b = b'SPA' + with self.assertRaises(EOFError): + sunau.open(io.BytesIO(b)) + b = b'SPAM' + with self.assertRaisesRegex(sunau.Error, 'bad magic number'): + sunau.open(io.BytesIO(b)) + + def test_read_too_small_header(self): + b = struct.pack('>LLLLL', sunau.AUDIO_FILE_MAGIC, 20, 0, + sunau.AUDIO_FILE_ENCODING_LINEAR_8, 11025) + with self.assertRaisesRegex(sunau.Error, 'header size too small'): + sunau.open(io.BytesIO(b)) + + def test_read_too_large_header(self): + b = struct.pack('>LLLLLL', sunau.AUDIO_FILE_MAGIC, 124, 0, + sunau.AUDIO_FILE_ENCODING_LINEAR_8, 11025, 1) + b += b'\0' * 100 + with self.assertRaisesRegex(sunau.Error, 'header size ridiculously large'): + sunau.open(io.BytesIO(b)) + + def test_read_wrong_encoding(self): + b = struct.pack('>LLLLLL', sunau.AUDIO_FILE_MAGIC, 24, 0, 0, 11025, 1) + with self.assertRaisesRegex(sunau.Error, r'encoding not \(yet\) supported'): + sunau.open(io.BytesIO(b)) + + def test_read_wrong_number_of_channels(self): + b = struct.pack('>LLLLLL', sunau.AUDIO_FILE_MAGIC, 24, 0, + sunau.AUDIO_FILE_ENCODING_LINEAR_8, 11025, 0) + with self.assertRaisesRegex(sunau.Error, 'bad # of channels'): + sunau.open(io.BytesIO(b)) + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_wave.py b/Lib/test/test_wave.py index c5d2e02450ef..8a42f8e47105 100644 --- a/Lib/test/test_wave.py +++ b/Lib/test/test_wave.py @@ -2,6 +2,8 @@ from test import audiotests from test import support from audioop import byteswap +import io +import struct import sys import wave @@ -111,5 +113,65 @@ def test__all__(self): support.check__all__(self, wave, blacklist=blacklist) +class WaveLowLevelTest(unittest.TestCase): + + def test_read_no_chunks(self): + b = b'SPAM' + with self.assertRaises(EOFError): + wave.open(io.BytesIO(b)) + + def test_read_no_riff_chunk(self): + b = b'SPAM' + struct.pack('<L', 0) + with self.assertRaisesRegex(wave.Error, + 'file does not start with RIFF id'): + wave.open(io.BytesIO(b)) + + def test_read_not_wave(self): + b = b'RIFF' + struct.pack('<L', 4) + b'SPAM' + with self.assertRaisesRegex(wave.Error, + 'not a WAVE file'): + wave.open(io.BytesIO(b)) + + def test_read_no_fmt_no_data_chunk(self): + b = b'RIFF' + struct.pack('<L', 4) + b'WAVE' + with self.assertRaisesRegex(wave.Error, + 'fmt chunk and/or data chunk missing'): + wave.open(io.BytesIO(b)) + + def test_read_no_data_chunk(self): + b = b'RIFF' + struct.pack('<L', 28) + b'WAVE' + b += b'fmt ' + struct.pack('<LHHLLHH', 16, 1, 1, 11025, 11025, 1, 8) + with self.assertRaisesRegex(wave.Error, + 'fmt chunk and/or data chunk missing'): + wave.open(io.BytesIO(b)) + + def test_read_no_fmt_chunk(self): + b = b'RIFF' + struct.pack('<L', 12) + b'WAVE' + b += b'data' + struct.pack('<L', 0) + with self.assertRaisesRegex(wave.Error, 'data chunk before fmt chunk'): + wave.open(io.BytesIO(b)) + + def test_read_wrong_form(self): + b = b'RIFF' + struct.pack('<L', 36) + b'WAVE' + b += b'fmt ' + struct.pack('<LHHLLHH', 16, 2, 1, 11025, 11025, 1, 1) + b += b'data' + struct.pack('<L', 0) + with self.assertRaisesRegex(wave.Error, 'unknown format: 2'): + wave.open(io.BytesIO(b)) + + def test_read_wrong_number_of_channels(self): + b = b'RIFF' + struct.pack('<L', 36) + b'WAVE' + b += b'fmt ' + struct.pack('<LHHLLHH', 16, 1, 0, 11025, 11025, 1, 8) + b += b'data' + struct.pack('<L', 0) + with self.assertRaisesRegex(wave.Error, 'bad # of channels'): + wave.open(io.BytesIO(b)) + + def test_read_wrong_sample_width(self): + b = b'RIFF' + struct.pack('<L', 36) + b'WAVE' + b += b'fmt ' + struct.pack('<LHHLLHH', 16, 1, 1, 11025, 11025, 1, 0) + b += b'data' + struct.pack('<L', 0) + with self.assertRaisesRegex(wave.Error, 'bad sample width'): + wave.open(io.BytesIO(b)) + + if __name__ == '__main__': unittest.main() diff --git a/Lib/wave.py b/Lib/wave.py index cf94d5af72b4..f155879a9a76 100644 --- a/Lib/wave.py +++ b/Lib/wave.py @@ -253,12 +253,22 @@ def readframes(self, nframes): # def _read_fmt_chunk(self, chunk): - wFormatTag, self._nchannels, self._framerate, dwAvgBytesPerSec, wBlockAlign = struct.unpack_from('<HHLLH', chunk.read(14)) + try: + wFormatTag, self._nchannels, self._framerate, dwAvgBytesPerSec, wBlockAlign = struct.unpack_from('<HHLLH', chunk.read(14)) + except struct.error: + raise EOFError from None if wFormatTag == WAVE_FORMAT_PCM: - sampwidth = struct.unpack_from('<H', chunk.read(2))[0] + try: + sampwidth = struct.unpack_from('<H', chunk.read(2))[0] + except struct.error: + raise EOFError from None self._sampwidth = (sampwidth + 7) // 8 + if not self._sampwidth: + raise Error('bad sample width') else: raise Error('unknown format: %r' % (wFormatTag,)) + if not self._nchannels: + raise Error('bad # of channels') self._framesize = self._nchannels * self._sampwidth self._comptype = 'NONE' self._compname = 'not compressed' diff --git a/Misc/NEWS.d/next/Library/2018-03-01-17-49-56.bpo-32056.IlpfgE.rst b/Misc/NEWS.d/next/Library/2018-03-01-17-49-56.bpo-32056.IlpfgE.rst new file mode 100644 index 000000000000..421aa3767794 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-03-01-17-49-56.bpo-32056.IlpfgE.rst @@ -0,0 +1,3 @@ +Improved exceptions raised for invalid number of channels and sample width +when read an audio file in modules :mod:`aifc`, :mod:`wave` and +:mod:`sunau`. From webhook-mailer at python.org Sun Mar 18 20:40:38 2018 From: webhook-mailer at python.org (Eric V. Smith) Date: Mon, 19 Mar 2018 00:40:38 -0000 Subject: [Python-checkins] bpo-32953: Dataclasses: frozen should not be inherited for non-dataclass derived classes (#6147) Message-ID: <mailman.98.1521420039.1871.python-checkins@python.org> https://github.com/python/cpython/commit/f199bc655eb50c28e94010714629b376bbbd077b commit: f199bc655eb50c28e94010714629b376bbbd077b branch: master author: Eric V. Smith <ericvsmith at users.noreply.github.com> committer: GitHub <noreply at github.com> date: 2018-03-18T20:40:34-04:00 summary: bpo-32953: Dataclasses: frozen should not be inherited for non-dataclass derived classes (#6147) If a non-dataclass derives from a frozen dataclass, allow attributes to be set. Require either all of the dataclasses in a class hierarchy to be frozen, or all non-frozen. Store `@dataclass` parameters on the class object under `__dataclass_params__`. This is needed to detect frozen base classes. files: A Misc/NEWS.d/next/Library/2018-03-18-17-38-48.bpo-32953.t8WAWN.rst M Lib/dataclasses.py M Lib/test/test_dataclasses.py diff --git a/Lib/dataclasses.py b/Lib/dataclasses.py index b55a497db302..8ab04dd5b975 100644 --- a/Lib/dataclasses.py +++ b/Lib/dataclasses.py @@ -171,7 +171,11 @@ class _MISSING_TYPE: # The name of an attribute on the class where we store the Field # objects. Also used to check if a class is a Data Class. -_MARKER = '__dataclass_fields__' +_FIELDS = '__dataclass_fields__' + +# The name of an attribute on the class that stores the parameters to +# @dataclass. +_PARAMS = '__dataclass_params__' # The name of the function, that if it exists, is called at the end of # __init__. @@ -192,7 +196,7 @@ class InitVar(metaclass=_InitVarMeta): # name and type are filled in after the fact, not in __init__. They're # not known at the time this class is instantiated, but it's # convenient if they're available later. -# When cls._MARKER is filled in with a list of Field objects, the name +# When cls._FIELDS is filled in with a list of Field objects, the name # and type fields will have been populated. class Field: __slots__ = ('name', @@ -236,6 +240,32 @@ def __repr__(self): ')') +class _DataclassParams: + __slots__ = ('init', + 'repr', + 'eq', + 'order', + 'unsafe_hash', + 'frozen', + ) + def __init__(self, init, repr, eq, order, unsafe_hash, frozen): + self.init = init + self.repr = repr + self.eq = eq + self.order = order + self.unsafe_hash = unsafe_hash + self.frozen = frozen + + def __repr__(self): + return ('_DataclassParams(' + f'init={self.init},' + f'repr={self.repr},' + f'eq={self.eq},' + f'order={self.order},' + f'unsafe_hash={self.unsafe_hash},' + f'frozen={self.frozen}' + ')') + # This function is used instead of exposing Field creation directly, # so that a type checker can be told (via overloads) that this is a # function whose type depends on its parameters. @@ -285,6 +315,7 @@ def _create_fn(name, args, body, *, globals=None, locals=None, args = ','.join(args) body = '\n'.join(f' {b}' for b in body) + # Compute the text of the entire function. txt = f'def {name}({args}){return_annotation}:\n{body}' exec(txt, globals, locals) @@ -432,12 +463,29 @@ def _repr_fn(fields): ')"']) -def _frozen_setattr(self, name, value): - raise FrozenInstanceError(f'cannot assign to field {name!r}') - - -def _frozen_delattr(self, name): - raise FrozenInstanceError(f'cannot delete field {name!r}') +def _frozen_get_del_attr(cls, fields): + # XXX: globals is modified on the first call to _create_fn, then the + # modified version is used in the second call. Is this okay? + globals = {'cls': cls, + 'FrozenInstanceError': FrozenInstanceError} + if fields: + fields_str = '(' + ','.join(repr(f.name) for f in fields) + ',)' + else: + # Special case for the zero-length tuple. + fields_str = '()' + return (_create_fn('__setattr__', + ('self', 'name', 'value'), + (f'if type(self) is cls or name in {fields_str}:', + ' raise FrozenInstanceError(f"cannot assign to field {name!r}")', + f'super(cls, self).__setattr__(name, value)'), + globals=globals), + _create_fn('__delattr__', + ('self', 'name'), + (f'if type(self) is cls or name in {fields_str}:', + ' raise FrozenInstanceError(f"cannot delete field {name!r}")', + f'super(cls, self).__delattr__(name)'), + globals=globals), + ) def _cmp_fn(name, op, self_tuple, other_tuple): @@ -583,23 +631,32 @@ def _set_new_attribute(cls, name, value): # version of this table. -def _process_class(cls, repr, eq, order, unsafe_hash, init, frozen): +def _process_class(cls, init, repr, eq, order, unsafe_hash, frozen): # Now that dicts retain insertion order, there's no reason to use # an ordered dict. I am leveraging that ordering here, because # derived class fields overwrite base class fields, but the order # is defined by the base class, which is found first. fields = {} + setattr(cls, _PARAMS, _DataclassParams(init, repr, eq, order, + unsafe_hash, frozen)) + # Find our base classes in reverse MRO order, and exclude # ourselves. In reversed order so that more derived classes # override earlier field definitions in base classes. + # As long as we're iterating over them, see if any are frozen. + any_frozen_base = False + has_dataclass_bases = False for b in cls.__mro__[-1:0:-1]: # Only process classes that have been processed by our - # decorator. That is, they have a _MARKER attribute. - base_fields = getattr(b, _MARKER, None) + # decorator. That is, they have a _FIELDS attribute. + base_fields = getattr(b, _FIELDS, None) if base_fields: + has_dataclass_bases = True for f in base_fields.values(): fields[f.name] = f + if getattr(b, _PARAMS).frozen: + any_frozen_base = True # Now find fields in our class. While doing so, validate some # things, and set the default values (as class attributes) @@ -623,20 +680,21 @@ def _process_class(cls, repr, eq, order, unsafe_hash, init, frozen): else: setattr(cls, f.name, f.default) - # We're inheriting from a frozen dataclass, but we're not frozen. - if cls.__setattr__ is _frozen_setattr and not frozen: - raise TypeError('cannot inherit non-frozen dataclass from a ' - 'frozen one') + # Check rules that apply if we are derived from any dataclasses. + if has_dataclass_bases: + # Raise an exception if any of our bases are frozen, but we're not. + if any_frozen_base and not frozen: + raise TypeError('cannot inherit non-frozen dataclass from a ' + 'frozen one') - # We're inheriting from a non-frozen dataclass, but we're frozen. - if (hasattr(cls, _MARKER) and cls.__setattr__ is not _frozen_setattr - and frozen): - raise TypeError('cannot inherit frozen dataclass from a ' - 'non-frozen one') + # Raise an exception if we're frozen, but none of our bases are. + if not any_frozen_base and frozen: + raise TypeError('cannot inherit frozen dataclass from a ' + 'non-frozen one') - # Remember all of the fields on our class (including bases). This + # Remember all of the fields on our class (including bases). This also # marks this class as being a dataclass. - setattr(cls, _MARKER, fields) + setattr(cls, _FIELDS, fields) # Was this class defined with an explicit __hash__? Note that if # __eq__ is defined in this class, then python will automatically @@ -704,10 +762,10 @@ def _process_class(cls, repr, eq, order, unsafe_hash, init, frozen): 'functools.total_ordering') if frozen: - for name, fn in [('__setattr__', _frozen_setattr), - ('__delattr__', _frozen_delattr)]: - if _set_new_attribute(cls, name, fn): - raise TypeError(f'Cannot overwrite attribute {name} ' + # XXX: Which fields are frozen? InitVar? ClassVar? hashed-only? + for fn in _frozen_get_del_attr(cls, field_list): + if _set_new_attribute(cls, fn.__name__, fn): + raise TypeError(f'Cannot overwrite attribute {fn.__name__} ' f'in class {cls.__name__}') # Decide if/how we're going to create a hash function. @@ -759,7 +817,7 @@ def dataclass(_cls=None, *, init=True, repr=True, eq=True, order=False, """ def wrap(cls): - return _process_class(cls, repr, eq, order, unsafe_hash, init, frozen) + return _process_class(cls, init, repr, eq, order, unsafe_hash, frozen) # See if we're being called as @dataclass or @dataclass(). if _cls is None: @@ -779,7 +837,7 @@ def fields(class_or_instance): # Might it be worth caching this, per class? try: - fields = getattr(class_or_instance, _MARKER) + fields = getattr(class_or_instance, _FIELDS) except AttributeError: raise TypeError('must be called with a dataclass type or instance') @@ -790,13 +848,13 @@ def fields(class_or_instance): def _is_dataclass_instance(obj): """Returns True if obj is an instance of a dataclass.""" - return not isinstance(obj, type) and hasattr(obj, _MARKER) + return not isinstance(obj, type) and hasattr(obj, _FIELDS) def is_dataclass(obj): """Returns True if obj is a dataclass or an instance of a dataclass.""" - return hasattr(obj, _MARKER) + return hasattr(obj, _FIELDS) def asdict(obj, *, dict_factory=dict): @@ -953,7 +1011,7 @@ class C: # It's an error to have init=False fields in 'changes'. # If a field is not in 'changes', read its value from the provided obj. - for f in getattr(obj, _MARKER).values(): + for f in getattr(obj, _FIELDS).values(): if not f.init: # Error if this field is specified in changes. if f.name in changes: diff --git a/Lib/test/test_dataclasses.py b/Lib/test/test_dataclasses.py index 46d485c0157b..3e6726360940 100755 --- a/Lib/test/test_dataclasses.py +++ b/Lib/test/test_dataclasses.py @@ -2476,41 +2476,92 @@ class D(C): d = D(0, 10) with self.assertRaises(FrozenInstanceError): d.i = 5 + with self.assertRaises(FrozenInstanceError): + d.j = 6 self.assertEqual(d.i, 0) + self.assertEqual(d.j, 10) + + # Test both ways: with an intermediate normal (non-dataclass) + # class and without an intermediate class. + def test_inherit_nonfrozen_from_frozen(self): + for intermediate_class in [True, False]: + with self.subTest(intermediate_class=intermediate_class): + @dataclass(frozen=True) + class C: + i: int - def test_inherit_from_nonfrozen_from_frozen(self): - @dataclass(frozen=True) - class C: - i: int + if intermediate_class: + class I(C): pass + else: + I = C - with self.assertRaisesRegex(TypeError, - 'cannot inherit non-frozen dataclass from a frozen one'): - @dataclass - class D(C): - pass + with self.assertRaisesRegex(TypeError, + 'cannot inherit non-frozen dataclass from a frozen one'): + @dataclass + class D(I): + pass - def test_inherit_from_frozen_from_nonfrozen(self): - @dataclass - class C: - i: int + def test_inherit_frozen_from_nonfrozen(self): + for intermediate_class in [True, False]: + with self.subTest(intermediate_class=intermediate_class): + @dataclass + class C: + i: int - with self.assertRaisesRegex(TypeError, - 'cannot inherit frozen dataclass from a non-frozen one'): - @dataclass(frozen=True) - class D(C): - pass + if intermediate_class: + class I(C): pass + else: + I = C + + with self.assertRaisesRegex(TypeError, + 'cannot inherit frozen dataclass from a non-frozen one'): + @dataclass(frozen=True) + class D(I): + pass def test_inherit_from_normal_class(self): - class C: - pass + for intermediate_class in [True, False]: + with self.subTest(intermediate_class=intermediate_class): + class C: + pass + + if intermediate_class: + class I(C): pass + else: + I = C + + @dataclass(frozen=True) + class D(I): + i: int + + d = D(10) + with self.assertRaises(FrozenInstanceError): + d.i = 5 + + def test_non_frozen_normal_derived(self): + # See bpo-32953. @dataclass(frozen=True) - class D(C): - i: int + class D: + x: int + y: int = 10 - d = D(10) + class S(D): + pass + + s = S(3) + self.assertEqual(s.x, 3) + self.assertEqual(s.y, 10) + s.cached = True + + # But can't change the frozen attributes. with self.assertRaises(FrozenInstanceError): - d.i = 5 + s.x = 5 + with self.assertRaises(FrozenInstanceError): + s.y = 5 + self.assertEqual(s.x, 3) + self.assertEqual(s.y, 10) + self.assertEqual(s.cached, True) if __name__ == '__main__': diff --git a/Misc/NEWS.d/next/Library/2018-03-18-17-38-48.bpo-32953.t8WAWN.rst b/Misc/NEWS.d/next/Library/2018-03-18-17-38-48.bpo-32953.t8WAWN.rst new file mode 100644 index 000000000000..fbea34aa9a2a --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-03-18-17-38-48.bpo-32953.t8WAWN.rst @@ -0,0 +1,4 @@ +If a non-dataclass inherits from a frozen dataclass, allow attributes to be +added to the derived class. Only attributes from from the frozen dataclass +cannot be assigned to. Require all dataclasses in a hierarchy to be either +all frozen or all non-frozen. From webhook-mailer at python.org Sun Mar 18 21:03:39 2018 From: webhook-mailer at python.org (Eric V. Smith) Date: Mon, 19 Mar 2018 01:03:39 -0000 Subject: [Python-checkins] bpo-32953: Dataclasses: frozen should not be inherited for non-dataclass derived classes (GH-6147) (GH-6148) Message-ID: <mailman.99.1521421421.1871.python-checkins@python.org> https://github.com/python/cpython/commit/45648312e540cda3b10109b6a808cbf6955c84eb commit: 45648312e540cda3b10109b6a808cbf6955c84eb branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Eric V. Smith <ericvsmith at users.noreply.github.com> date: 2018-03-18T21:03:36-04:00 summary: bpo-32953: Dataclasses: frozen should not be inherited for non-dataclass derived classes (GH-6147) (GH-6148) If a non-dataclass derives from a frozen dataclass, allow attributes to be set. Require either all of the dataclasses in a class hierarchy to be frozen, or all non-frozen. Store `@dataclass` parameters on the class object under `__dataclass_params__`. This is needed to detect frozen base classes. (cherry picked from commit f199bc655eb50c28e94010714629b376bbbd077b) Co-authored-by: Eric V. Smith <ericvsmith at users.noreply.github.com> files: A Misc/NEWS.d/next/Library/2018-03-18-17-38-48.bpo-32953.t8WAWN.rst M Lib/dataclasses.py M Lib/test/test_dataclasses.py diff --git a/Lib/dataclasses.py b/Lib/dataclasses.py index b55a497db302..8ab04dd5b975 100644 --- a/Lib/dataclasses.py +++ b/Lib/dataclasses.py @@ -171,7 +171,11 @@ class _MISSING_TYPE: # The name of an attribute on the class where we store the Field # objects. Also used to check if a class is a Data Class. -_MARKER = '__dataclass_fields__' +_FIELDS = '__dataclass_fields__' + +# The name of an attribute on the class that stores the parameters to +# @dataclass. +_PARAMS = '__dataclass_params__' # The name of the function, that if it exists, is called at the end of # __init__. @@ -192,7 +196,7 @@ class InitVar(metaclass=_InitVarMeta): # name and type are filled in after the fact, not in __init__. They're # not known at the time this class is instantiated, but it's # convenient if they're available later. -# When cls._MARKER is filled in with a list of Field objects, the name +# When cls._FIELDS is filled in with a list of Field objects, the name # and type fields will have been populated. class Field: __slots__ = ('name', @@ -236,6 +240,32 @@ def __repr__(self): ')') +class _DataclassParams: + __slots__ = ('init', + 'repr', + 'eq', + 'order', + 'unsafe_hash', + 'frozen', + ) + def __init__(self, init, repr, eq, order, unsafe_hash, frozen): + self.init = init + self.repr = repr + self.eq = eq + self.order = order + self.unsafe_hash = unsafe_hash + self.frozen = frozen + + def __repr__(self): + return ('_DataclassParams(' + f'init={self.init},' + f'repr={self.repr},' + f'eq={self.eq},' + f'order={self.order},' + f'unsafe_hash={self.unsafe_hash},' + f'frozen={self.frozen}' + ')') + # This function is used instead of exposing Field creation directly, # so that a type checker can be told (via overloads) that this is a # function whose type depends on its parameters. @@ -285,6 +315,7 @@ def _create_fn(name, args, body, *, globals=None, locals=None, args = ','.join(args) body = '\n'.join(f' {b}' for b in body) + # Compute the text of the entire function. txt = f'def {name}({args}){return_annotation}:\n{body}' exec(txt, globals, locals) @@ -432,12 +463,29 @@ def _repr_fn(fields): ')"']) -def _frozen_setattr(self, name, value): - raise FrozenInstanceError(f'cannot assign to field {name!r}') - - -def _frozen_delattr(self, name): - raise FrozenInstanceError(f'cannot delete field {name!r}') +def _frozen_get_del_attr(cls, fields): + # XXX: globals is modified on the first call to _create_fn, then the + # modified version is used in the second call. Is this okay? + globals = {'cls': cls, + 'FrozenInstanceError': FrozenInstanceError} + if fields: + fields_str = '(' + ','.join(repr(f.name) for f in fields) + ',)' + else: + # Special case for the zero-length tuple. + fields_str = '()' + return (_create_fn('__setattr__', + ('self', 'name', 'value'), + (f'if type(self) is cls or name in {fields_str}:', + ' raise FrozenInstanceError(f"cannot assign to field {name!r}")', + f'super(cls, self).__setattr__(name, value)'), + globals=globals), + _create_fn('__delattr__', + ('self', 'name'), + (f'if type(self) is cls or name in {fields_str}:', + ' raise FrozenInstanceError(f"cannot delete field {name!r}")', + f'super(cls, self).__delattr__(name)'), + globals=globals), + ) def _cmp_fn(name, op, self_tuple, other_tuple): @@ -583,23 +631,32 @@ def _set_new_attribute(cls, name, value): # version of this table. -def _process_class(cls, repr, eq, order, unsafe_hash, init, frozen): +def _process_class(cls, init, repr, eq, order, unsafe_hash, frozen): # Now that dicts retain insertion order, there's no reason to use # an ordered dict. I am leveraging that ordering here, because # derived class fields overwrite base class fields, but the order # is defined by the base class, which is found first. fields = {} + setattr(cls, _PARAMS, _DataclassParams(init, repr, eq, order, + unsafe_hash, frozen)) + # Find our base classes in reverse MRO order, and exclude # ourselves. In reversed order so that more derived classes # override earlier field definitions in base classes. + # As long as we're iterating over them, see if any are frozen. + any_frozen_base = False + has_dataclass_bases = False for b in cls.__mro__[-1:0:-1]: # Only process classes that have been processed by our - # decorator. That is, they have a _MARKER attribute. - base_fields = getattr(b, _MARKER, None) + # decorator. That is, they have a _FIELDS attribute. + base_fields = getattr(b, _FIELDS, None) if base_fields: + has_dataclass_bases = True for f in base_fields.values(): fields[f.name] = f + if getattr(b, _PARAMS).frozen: + any_frozen_base = True # Now find fields in our class. While doing so, validate some # things, and set the default values (as class attributes) @@ -623,20 +680,21 @@ def _process_class(cls, repr, eq, order, unsafe_hash, init, frozen): else: setattr(cls, f.name, f.default) - # We're inheriting from a frozen dataclass, but we're not frozen. - if cls.__setattr__ is _frozen_setattr and not frozen: - raise TypeError('cannot inherit non-frozen dataclass from a ' - 'frozen one') + # Check rules that apply if we are derived from any dataclasses. + if has_dataclass_bases: + # Raise an exception if any of our bases are frozen, but we're not. + if any_frozen_base and not frozen: + raise TypeError('cannot inherit non-frozen dataclass from a ' + 'frozen one') - # We're inheriting from a non-frozen dataclass, but we're frozen. - if (hasattr(cls, _MARKER) and cls.__setattr__ is not _frozen_setattr - and frozen): - raise TypeError('cannot inherit frozen dataclass from a ' - 'non-frozen one') + # Raise an exception if we're frozen, but none of our bases are. + if not any_frozen_base and frozen: + raise TypeError('cannot inherit frozen dataclass from a ' + 'non-frozen one') - # Remember all of the fields on our class (including bases). This + # Remember all of the fields on our class (including bases). This also # marks this class as being a dataclass. - setattr(cls, _MARKER, fields) + setattr(cls, _FIELDS, fields) # Was this class defined with an explicit __hash__? Note that if # __eq__ is defined in this class, then python will automatically @@ -704,10 +762,10 @@ def _process_class(cls, repr, eq, order, unsafe_hash, init, frozen): 'functools.total_ordering') if frozen: - for name, fn in [('__setattr__', _frozen_setattr), - ('__delattr__', _frozen_delattr)]: - if _set_new_attribute(cls, name, fn): - raise TypeError(f'Cannot overwrite attribute {name} ' + # XXX: Which fields are frozen? InitVar? ClassVar? hashed-only? + for fn in _frozen_get_del_attr(cls, field_list): + if _set_new_attribute(cls, fn.__name__, fn): + raise TypeError(f'Cannot overwrite attribute {fn.__name__} ' f'in class {cls.__name__}') # Decide if/how we're going to create a hash function. @@ -759,7 +817,7 @@ def dataclass(_cls=None, *, init=True, repr=True, eq=True, order=False, """ def wrap(cls): - return _process_class(cls, repr, eq, order, unsafe_hash, init, frozen) + return _process_class(cls, init, repr, eq, order, unsafe_hash, frozen) # See if we're being called as @dataclass or @dataclass(). if _cls is None: @@ -779,7 +837,7 @@ def fields(class_or_instance): # Might it be worth caching this, per class? try: - fields = getattr(class_or_instance, _MARKER) + fields = getattr(class_or_instance, _FIELDS) except AttributeError: raise TypeError('must be called with a dataclass type or instance') @@ -790,13 +848,13 @@ def fields(class_or_instance): def _is_dataclass_instance(obj): """Returns True if obj is an instance of a dataclass.""" - return not isinstance(obj, type) and hasattr(obj, _MARKER) + return not isinstance(obj, type) and hasattr(obj, _FIELDS) def is_dataclass(obj): """Returns True if obj is a dataclass or an instance of a dataclass.""" - return hasattr(obj, _MARKER) + return hasattr(obj, _FIELDS) def asdict(obj, *, dict_factory=dict): @@ -953,7 +1011,7 @@ class C: # It's an error to have init=False fields in 'changes'. # If a field is not in 'changes', read its value from the provided obj. - for f in getattr(obj, _MARKER).values(): + for f in getattr(obj, _FIELDS).values(): if not f.init: # Error if this field is specified in changes. if f.name in changes: diff --git a/Lib/test/test_dataclasses.py b/Lib/test/test_dataclasses.py index 46d485c0157b..3e6726360940 100755 --- a/Lib/test/test_dataclasses.py +++ b/Lib/test/test_dataclasses.py @@ -2476,41 +2476,92 @@ class D(C): d = D(0, 10) with self.assertRaises(FrozenInstanceError): d.i = 5 + with self.assertRaises(FrozenInstanceError): + d.j = 6 self.assertEqual(d.i, 0) + self.assertEqual(d.j, 10) + + # Test both ways: with an intermediate normal (non-dataclass) + # class and without an intermediate class. + def test_inherit_nonfrozen_from_frozen(self): + for intermediate_class in [True, False]: + with self.subTest(intermediate_class=intermediate_class): + @dataclass(frozen=True) + class C: + i: int - def test_inherit_from_nonfrozen_from_frozen(self): - @dataclass(frozen=True) - class C: - i: int + if intermediate_class: + class I(C): pass + else: + I = C - with self.assertRaisesRegex(TypeError, - 'cannot inherit non-frozen dataclass from a frozen one'): - @dataclass - class D(C): - pass + with self.assertRaisesRegex(TypeError, + 'cannot inherit non-frozen dataclass from a frozen one'): + @dataclass + class D(I): + pass - def test_inherit_from_frozen_from_nonfrozen(self): - @dataclass - class C: - i: int + def test_inherit_frozen_from_nonfrozen(self): + for intermediate_class in [True, False]: + with self.subTest(intermediate_class=intermediate_class): + @dataclass + class C: + i: int - with self.assertRaisesRegex(TypeError, - 'cannot inherit frozen dataclass from a non-frozen one'): - @dataclass(frozen=True) - class D(C): - pass + if intermediate_class: + class I(C): pass + else: + I = C + + with self.assertRaisesRegex(TypeError, + 'cannot inherit frozen dataclass from a non-frozen one'): + @dataclass(frozen=True) + class D(I): + pass def test_inherit_from_normal_class(self): - class C: - pass + for intermediate_class in [True, False]: + with self.subTest(intermediate_class=intermediate_class): + class C: + pass + + if intermediate_class: + class I(C): pass + else: + I = C + + @dataclass(frozen=True) + class D(I): + i: int + + d = D(10) + with self.assertRaises(FrozenInstanceError): + d.i = 5 + + def test_non_frozen_normal_derived(self): + # See bpo-32953. @dataclass(frozen=True) - class D(C): - i: int + class D: + x: int + y: int = 10 - d = D(10) + class S(D): + pass + + s = S(3) + self.assertEqual(s.x, 3) + self.assertEqual(s.y, 10) + s.cached = True + + # But can't change the frozen attributes. with self.assertRaises(FrozenInstanceError): - d.i = 5 + s.x = 5 + with self.assertRaises(FrozenInstanceError): + s.y = 5 + self.assertEqual(s.x, 3) + self.assertEqual(s.y, 10) + self.assertEqual(s.cached, True) if __name__ == '__main__': diff --git a/Misc/NEWS.d/next/Library/2018-03-18-17-38-48.bpo-32953.t8WAWN.rst b/Misc/NEWS.d/next/Library/2018-03-18-17-38-48.bpo-32953.t8WAWN.rst new file mode 100644 index 000000000000..fbea34aa9a2a --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-03-18-17-38-48.bpo-32953.t8WAWN.rst @@ -0,0 +1,4 @@ +If a non-dataclass inherits from a frozen dataclass, allow attributes to be +added to the derived class. Only attributes from from the frozen dataclass +cannot be assigned to. Require all dataclasses in a hierarchy to be either +all frozen or all non-frozen. From solipsis at pitrou.net Mon Mar 19 05:13:45 2018 From: solipsis at pitrou.net (solipsis at pitrou.net) Date: Mon, 19 Mar 2018 09:13:45 +0000 Subject: [Python-checkins] Daily reference leaks (4243df51fe43): sum=4 Message-ID: <20180319091345.1.5E8703FCBF35C625@psf.io> results for 4243df51fe43 on branch "default" -------------------------------------------- test_collections leaked [0, 7, -7] memory blocks, sum=0 test_functools leaked [0, 3, 1] memory blocks, sum=4 test_multiprocessing_fork leaked [2, -2, 1] memory blocks, sum=1 test_multiprocessing_spawn leaked [-2, 0, 1] memory blocks, sum=-1 Command line was: ['./python', '-m', 'test.regrtest', '-uall', '-R', '3:3:/home/psf-users/antoine/refleaks/reflogkQnloZ', '--timeout', '7200'] From webhook-mailer at python.org Mon Mar 19 13:30:51 2018 From: webhook-mailer at python.org (Mariatta) Date: Mon, 19 Mar 2018 17:30:51 -0000 Subject: [Python-checkins] bpo-19417: Add test_bdb.py (GH-5217) Message-ID: <mailman.100.1521480653.1871.python-checkins@python.org> https://github.com/python/cpython/commit/424f3dafea16fbaee55a30903af2d6717f4d4a6b commit: 424f3dafea16fbaee55a30903af2d6717f4d4a6b branch: 3.6 author: xdegaye <xdegaye at gmail.com> committer: Mariatta <Mariatta at users.noreply.github.com> date: 2018-03-19T10:30:47-07:00 summary: bpo-19417: Add test_bdb.py (GH-5217) (cherry picked from commit 3fe33043ee83d19e15551094fc1e0984617ded3c) files: A Lib/test/test_bdb.py A Misc/NEWS.d/next/Tests/2018-01-08-13-33-47.bpo-19417.2asoXy.rst M Lib/test/test_sundry.py diff --git a/Lib/test/test_bdb.py b/Lib/test/test_bdb.py new file mode 100644 index 000000000000..abefe6c4e57a --- /dev/null +++ b/Lib/test/test_bdb.py @@ -0,0 +1,1151 @@ +""" Test the bdb module. + + A test defines a list of tuples that may be seen as paired tuples, each + pair being defined by 'expect_tuple, set_tuple' as follows: + + ([event, [lineno[, co_name[, eargs]]]]), (set_type, [sargs]) + + * 'expect_tuple' describes the expected current state of the Bdb instance. + It may be the empty tuple and no check is done in that case. + * 'set_tuple' defines the set_*() method to be invoked when the Bdb + instance reaches this state. + + Example of an 'expect_tuple, set_tuple' pair: + + ('line', 2, 'tfunc_main'), ('step', ) + + Definitions of the members of the 'expect_tuple': + event: + Name of the trace event. The set methods that do not give back + control to the tracer [1] do not trigger a tracer event and in + that case the next 'event' may be 'None' by convention, its value + is not checked. + [1] Methods that trigger a trace event are set_step(), set_next(), + set_return(), set_until() and set_continue(). + lineno: + Line number. Line numbers are relative to the start of the + function when tracing a function in the test_bdb module (i.e. this + module). + co_name: + Name of the function being currently traced. + eargs: + A tuple: + * On an 'exception' event the tuple holds a class object, the + current exception must be an instance of this class. + * On a 'line' event, the tuple holds a dictionary and a list. The + dictionary maps each breakpoint number that has been hit on this + line to its hits count. The list holds the list of breakpoint + number temporaries that are being deleted. + + Definitions of the members of the 'set_tuple': + set_type: + The type of the set method to be invoked. This may + be the type of one of the Bdb set methods: 'step', 'next', + 'until', 'return', 'continue', 'break', 'quit', or the type of one + of the set methods added by test_bdb.Bdb: 'ignore', 'enable', + 'disable', 'clear', 'up', 'down'. + sargs: + The arguments of the set method if any, packed in a tuple. +""" + +import bdb as _bdb +import sys +import os +import unittest +import textwrap +import importlib +import linecache +from contextlib import contextmanager +from itertools import islice, repeat +import test.support + +class BdbException(Exception): pass +class BdbError(BdbException): """Error raised by the Bdb instance.""" +class BdbSyntaxError(BdbException): """Syntax error in the test case.""" +class BdbNotExpectedError(BdbException): """Unexpected result.""" + +# When 'dry_run' is set to true, expect tuples are ignored and the actual +# state of the tracer is printed after running each set_*() method of the test +# case. The full list of breakpoints and their attributes is also printed +# after each 'line' event where a breakpoint has been hit. +dry_run = 0 + +def reset_Breakpoint(): + _bdb.Breakpoint.next = 1 + _bdb.Breakpoint.bplist = {} + _bdb.Breakpoint.bpbynumber = [None] + +def info_breakpoints(): + bp_list = [bp for bp in _bdb.Breakpoint.bpbynumber if bp] + if not bp_list: + return '' + + header_added = False + for bp in bp_list: + if not header_added: + info = 'BpNum Temp Enb Hits Ignore Where\n' + header_added = True + + disp = 'yes ' if bp.temporary else 'no ' + enab = 'yes' if bp.enabled else 'no ' + info += ('%-5d %s %s %-4d %-6d at %s:%d' % + (bp.number, disp, enab, bp.hits, bp.ignore, + os.path.basename(bp.file), bp.line)) + if bp.cond: + info += '\n\tstop only if %s' % (bp.cond,) + info += '\n' + return info + +class Bdb(_bdb.Bdb): + """Extend Bdb to enhance test coverage.""" + + def trace_dispatch(self, frame, event, arg): + self.currentbp = None + return super().trace_dispatch(frame, event, arg) + + def set_break(self, filename, lineno, temporary=False, cond=None, + funcname=None): + if isinstance(funcname, str): + if filename == __file__: + globals_ = globals() + else: + module = importlib.import_module(filename[:-3]) + globals_ = module.__dict__ + func = eval(funcname, globals_) + code = func.__code__ + filename = code.co_filename + lineno = code.co_firstlineno + funcname = code.co_name + + res = super().set_break(filename, lineno, temporary=temporary, + cond=cond, funcname=funcname) + if isinstance(res, str): + raise BdbError(res) + return res + + def get_stack(self, f, t): + self.stack, self.index = super().get_stack(f, t) + self.frame = self.stack[self.index][0] + return self.stack, self.index + + def set_ignore(self, bpnum): + """Increment the ignore count of Breakpoint number 'bpnum'.""" + bp = self.get_bpbynumber(bpnum) + bp.ignore += 1 + + def set_enable(self, bpnum): + bp = self.get_bpbynumber(bpnum) + bp.enabled = True + + def set_disable(self, bpnum): + bp = self.get_bpbynumber(bpnum) + bp.enabled = False + + def set_clear(self, fname, lineno): + err = self.clear_break(fname, lineno) + if err: + raise BdbError(err) + + def set_up(self): + """Move up in the frame stack.""" + if not self.index: + raise BdbError('Oldest frame') + self.index -= 1 + self.frame = self.stack[self.index][0] + + def set_down(self): + """Move down in the frame stack.""" + if self.index + 1 == len(self.stack): + raise BdbError('Newest frame') + self.index += 1 + self.frame = self.stack[self.index][0] + +class Tracer(Bdb): + """A tracer for testing the bdb module.""" + + def __init__(self, expect_set, skip=None, dry_run=False, test_case=None): + super().__init__(skip=skip) + self.expect_set = expect_set + self.dry_run = dry_run + self.header = ('Dry-run results for %s:' % test_case if + test_case is not None else None) + self.init_test() + + def init_test(self): + self.cur_except = None + self.expect_set_no = 0 + self.breakpoint_hits = None + self.expected_list = list(islice(self.expect_set, 0, None, 2)) + self.set_list = list(islice(self.expect_set, 1, None, 2)) + + def trace_dispatch(self, frame, event, arg): + # On an 'exception' event, call_exc_trace() in Python/ceval.c discards + # a BdbException raised by the Tracer instance, so we raise it on the + # next trace_dispatch() call that occurs unless the set_quit() or + # set_continue() method has been invoked on the 'exception' event. + if self.cur_except is not None: + raise self.cur_except + + if event == 'exception': + try: + res = super().trace_dispatch(frame, event, arg) + return res + except BdbException as e: + self.cur_except = e + return self.trace_dispatch + else: + return super().trace_dispatch(frame, event, arg) + + def user_call(self, frame, argument_list): + # Adopt the same behavior as pdb and, as a side effect, skip also the + # first 'call' event when the Tracer is started with Tracer.runcall() + # which may be possibly considered as a bug. + if not self.stop_here(frame): + return + self.process_event('call', frame, argument_list) + self.next_set_method() + + def user_line(self, frame): + self.process_event('line', frame) + + if self.dry_run and self.breakpoint_hits: + info = info_breakpoints().strip('\n') + # Indent each line. + for line in info.split('\n'): + print(' ' + line) + self.delete_temporaries() + self.breakpoint_hits = None + + self.next_set_method() + + def user_return(self, frame, return_value): + self.process_event('return', frame, return_value) + self.next_set_method() + + def user_exception(self, frame, exc_info): + self.exc_info = exc_info + self.process_event('exception', frame) + self.next_set_method() + + def do_clear(self, arg): + # The temporary breakpoints are deleted in user_line(). + bp_list = [self.currentbp] + self.breakpoint_hits = (bp_list, bp_list) + + def delete_temporaries(self): + if self.breakpoint_hits: + for n in self.breakpoint_hits[1]: + self.clear_bpbynumber(n) + + def pop_next(self): + self.expect_set_no += 1 + try: + self.expect = self.expected_list.pop(0) + except IndexError: + raise BdbNotExpectedError( + 'expect_set list exhausted, cannot pop item %d' % + self.expect_set_no) + self.set_tuple = self.set_list.pop(0) + + def process_event(self, event, frame, *args): + # Call get_stack() to enable walking the stack with set_up() and + # set_down(). + tb = None + if event == 'exception': + tb = self.exc_info[2] + self.get_stack(frame, tb) + + # A breakpoint has been hit and it is not a temporary. + if self.currentbp is not None and not self.breakpoint_hits: + bp_list = [self.currentbp] + self.breakpoint_hits = (bp_list, []) + + # Pop next event. + self.event= event + self.pop_next() + if self.dry_run: + self.print_state(self.header) + return + + # Validate the expected results. + if self.expect: + self.check_equal(self.expect[0], event, 'Wrong event type') + self.check_lno_name() + + if event in ('call', 'return'): + self.check_expect_max_size(3) + elif len(self.expect) > 3: + if event == 'line': + bps, temporaries = self.expect[3] + bpnums = sorted(bps.keys()) + if not self.breakpoint_hits: + self.raise_not_expected( + 'No breakpoints hit at expect_set item %d' % + self.expect_set_no) + self.check_equal(bpnums, self.breakpoint_hits[0], + 'Breakpoint numbers do not match') + self.check_equal([bps[n] for n in bpnums], + [self.get_bpbynumber(n).hits for + n in self.breakpoint_hits[0]], + 'Wrong breakpoint hit count') + self.check_equal(sorted(temporaries), self.breakpoint_hits[1], + 'Wrong temporary breakpoints') + + elif event == 'exception': + if not isinstance(self.exc_info[1], self.expect[3]): + self.raise_not_expected( + "Wrong exception at expect_set item %d, got '%s'" % + (self.expect_set_no, self.exc_info)) + + def check_equal(self, expected, result, msg): + if expected == result: + return + self.raise_not_expected("%s at expect_set item %d, got '%s'" % + (msg, self.expect_set_no, result)) + + def check_lno_name(self): + """Check the line number and function co_name.""" + s = len(self.expect) + if s > 1: + lineno = self.lno_abs2rel() + self.check_equal(self.expect[1], lineno, 'Wrong line number') + if s > 2: + self.check_equal(self.expect[2], self.frame.f_code.co_name, + 'Wrong function name') + + def check_expect_max_size(self, size): + if len(self.expect) > size: + raise BdbSyntaxError('Invalid size of the %s expect tuple: %s' % + (self.event, self.expect)) + + def lno_abs2rel(self): + fname = self.canonic(self.frame.f_code.co_filename) + lineno = self.frame.f_lineno + return ((lineno - self.frame.f_code.co_firstlineno + 1) + if fname == self.canonic(__file__) else lineno) + + def lno_rel2abs(self, fname, lineno): + return (self.frame.f_code.co_firstlineno + lineno - 1 + if (lineno and self.canonic(fname) == self.canonic(__file__)) + else lineno) + + def get_state(self): + lineno = self.lno_abs2rel() + co_name = self.frame.f_code.co_name + state = "('%s', %d, '%s'" % (self.event, lineno, co_name) + if self.breakpoint_hits: + bps = '{' + for n in self.breakpoint_hits[0]: + if bps != '{': + bps += ', ' + bps += '%s: %s' % (n, self.get_bpbynumber(n).hits) + bps += '}' + bps = '(' + bps + ', ' + str(self.breakpoint_hits[1]) + ')' + state += ', ' + bps + elif self.event == 'exception': + state += ', ' + self.exc_info[0].__name__ + state += '), ' + return state.ljust(32) + str(self.set_tuple) + ',' + + def print_state(self, header=None): + if header is not None and self.expect_set_no == 1: + print() + print(header) + print('%d: %s' % (self.expect_set_no, self.get_state())) + + def raise_not_expected(self, msg): + msg += '\n' + msg += ' Expected: %s\n' % str(self.expect) + msg += ' Got: ' + self.get_state() + raise BdbNotExpectedError(msg) + + def next_set_method(self): + set_type = self.set_tuple[0] + args = self.set_tuple[1] if len(self.set_tuple) == 2 else None + set_method = getattr(self, 'set_' + set_type) + + # The following set methods give back control to the tracer. + if set_type in ('step', 'continue', 'quit'): + set_method() + return + elif set_type in ('next', 'return'): + set_method(self.frame) + return + elif set_type == 'until': + lineno = None + if args: + lineno = self.lno_rel2abs(self.frame.f_code.co_filename, + args[0]) + set_method(self.frame, lineno) + return + + # The following set methods do not give back control to the tracer and + # next_set_method() is called recursively. + if (args and set_type in ('break', 'clear', 'ignore', 'enable', + 'disable')) or set_type in ('up', 'down'): + if set_type in ('break', 'clear'): + fname, lineno, *remain = args + lineno = self.lno_rel2abs(fname, lineno) + args = [fname, lineno] + args.extend(remain) + set_method(*args) + elif set_type in ('ignore', 'enable', 'disable'): + set_method(*args) + elif set_type in ('up', 'down'): + set_method() + + # Process the next expect_set item. + # It is not expected that a test may reach the recursion limit. + self.event= None + self.pop_next() + if self.dry_run: + self.print_state() + else: + if self.expect: + self.check_lno_name() + self.check_expect_max_size(3) + self.next_set_method() + else: + raise BdbSyntaxError('"%s" is an invalid set_tuple' % + self.set_tuple) + +class TracerRun(): + """Provide a context for running a Tracer instance with a test case.""" + + def __init__(self, test_case, skip=None): + self.test_case = test_case + self.dry_run = test_case.dry_run + self.tracer = Tracer(test_case.expect_set, skip=skip, + dry_run=self.dry_run, test_case=test_case.id()) + + def __enter__(self): + # test_pdb does not reset Breakpoint class attributes on exit :-( + reset_Breakpoint() + return self.tracer + + def __exit__(self, type_=None, value=None, traceback=None): + reset_Breakpoint() + sys.settrace(None) + + not_empty = '' + if self.tracer.set_list: + not_empty += 'All paired tuples have not been processed, ' + not_empty += ('the last one was number %d' % + self.tracer.expect_set_no) + + # Make a BdbNotExpectedError a unittest failure. + if type_ is not None and issubclass(BdbNotExpectedError, type_): + if isinstance(value, BaseException) and value.args: + err_msg = value.args[0] + if not_empty: + err_msg += '\n' + not_empty + if self.dry_run: + print(err_msg) + return True + else: + self.test_case.fail(err_msg) + else: + assert False, 'BdbNotExpectedError with empty args' + + if not_empty: + if self.dry_run: + print(not_empty) + else: + self.test_case.fail(not_empty) + +def run_test(modules, set_list, skip=None): + """Run a test and print the dry-run results. + + 'modules': A dictionary mapping module names to their source code as a + string. The dictionary MUST include one module named + 'test_module' with a main() function. + 'set_list': A list of set_type tuples to be run on the module. + + For example, running the following script outputs the following results: + + ***************************** SCRIPT ******************************** + + from test.test_bdb import run_test, break_in_func + + code = ''' + def func(): + lno = 3 + + def main(): + func() + lno = 7 + ''' + + set_list = [ + break_in_func('func', 'test_module.py'), + ('continue', ), + ('step', ), + ('step', ), + ('step', ), + ('quit', ), + ] + + modules = { 'test_module': code } + run_test(modules, set_list) + + **************************** results ******************************** + + 1: ('line', 2, 'tfunc_import'), ('next',), + 2: ('line', 3, 'tfunc_import'), ('step',), + 3: ('call', 5, 'main'), ('break', ('test_module.py', None, False, None, 'func')), + 4: ('None', 5, 'main'), ('continue',), + 5: ('line', 3, 'func', ({1: 1}, [])), ('step',), + BpNum Temp Enb Hits Ignore Where + 1 no yes 1 0 at test_module.py:2 + 6: ('return', 3, 'func'), ('step',), + 7: ('line', 7, 'main'), ('step',), + 8: ('return', 7, 'main'), ('quit',), + + ************************************************************************* + + """ + def gen(a, b): + try: + while 1: + x = next(a) + y = next(b) + yield x + yield y + except StopIteration: + return + + # Step over the import statement in tfunc_import using 'next' and step + # into main() in test_module. + sl = [('next', ), ('step', )] + sl.extend(set_list) + + test = BaseTestCase() + test.dry_run = True + test.id = lambda : None + test.expect_set = list(gen(repeat(()), iter(sl))) + with create_modules(modules): + sys.path.append(os.getcwd()) + with TracerRun(test, skip=skip) as tracer: + tracer.runcall(tfunc_import) + + at contextmanager +def create_modules(modules): + with test.support.temp_cwd(): + try: + for m in modules: + fname = m + '.py' + with open(fname, 'w') as f: + f.write(textwrap.dedent(modules[m])) + linecache.checkcache(fname) + importlib.invalidate_caches() + yield + finally: + for m in modules: + test.support.forget(m) + +def break_in_func(funcname, fname=__file__, temporary=False, cond=None): + return 'break', (fname, None, temporary, cond, funcname) + +TEST_MODULE = 'test_module' +TEST_MODULE_FNAME = TEST_MODULE + '.py' +def tfunc_import(): + import test_module + test_module.main() + +def tfunc_main(): + lno = 2 + tfunc_first() + tfunc_second() + lno = 5 + lno = 6 + lno = 7 + +def tfunc_first(): + lno = 2 + lno = 3 + lno = 4 + +def tfunc_second(): + lno = 2 + +class BaseTestCase(unittest.TestCase): + """Base class for all tests.""" + + dry_run = dry_run + + def fail(self, msg=None): + # Override fail() to use 'raise from None' to avoid repetition of the + # error message and traceback. + raise self.failureException(msg) from None + +class StateTestCase(BaseTestCase): + """Test the step, next, return, until and quit 'set_' methods.""" + + def test_step(self): + self.expect_set = [ + ('line', 2, 'tfunc_main'), ('step', ), + ('line', 3, 'tfunc_main'), ('step', ), + ('call', 1, 'tfunc_first'), ('step', ), + ('line', 2, 'tfunc_first'), ('quit', ), + ] + with TracerRun(self) as tracer: + tracer.runcall(tfunc_main) + + def test_step_next_on_last_statement(self): + for set_type in ('step', 'next'): + with self.subTest(set_type=set_type): + self.expect_set = [ + ('line', 2, 'tfunc_main'), ('step', ), + ('line', 3, 'tfunc_main'), ('step', ), + ('call', 1, 'tfunc_first'), ('break', (__file__, 3)), + ('None', 1, 'tfunc_first'), ('continue', ), + ('line', 3, 'tfunc_first', ({1:1}, [])), (set_type, ), + ('line', 4, 'tfunc_first'), ('quit', ), + ] + with TracerRun(self) as tracer: + tracer.runcall(tfunc_main) + + def test_next(self): + self.expect_set = [ + ('line', 2, 'tfunc_main'), ('step', ), + ('line', 3, 'tfunc_main'), ('next', ), + ('line', 4, 'tfunc_main'), ('step', ), + ('call', 1, 'tfunc_second'), ('step', ), + ('line', 2, 'tfunc_second'), ('quit', ), + ] + with TracerRun(self) as tracer: + tracer.runcall(tfunc_main) + + def test_next_over_import(self): + code = """ + def main(): + lno = 3 + """ + modules = { TEST_MODULE: code } + with create_modules(modules): + self.expect_set = [ + ('line', 2, 'tfunc_import'), ('next', ), + ('line', 3, 'tfunc_import'), ('quit', ), + ] + with TracerRun(self) as tracer: + tracer.runcall(tfunc_import) + + def test_next_on_plain_statement(self): + # Check that set_next() is equivalent to set_step() on a plain + # statement. + self.expect_set = [ + ('line', 2, 'tfunc_main'), ('step', ), + ('line', 3, 'tfunc_main'), ('step', ), + ('call', 1, 'tfunc_first'), ('next', ), + ('line', 2, 'tfunc_first'), ('quit', ), + ] + with TracerRun(self) as tracer: + tracer.runcall(tfunc_main) + + def test_next_in_caller_frame(self): + # Check that set_next() in the caller frame causes the tracer + # to stop next in the caller frame. + self.expect_set = [ + ('line', 2, 'tfunc_main'), ('step', ), + ('line', 3, 'tfunc_main'), ('step', ), + ('call', 1, 'tfunc_first'), ('up', ), + ('None', 3, 'tfunc_main'), ('next', ), + ('line', 4, 'tfunc_main'), ('quit', ), + ] + with TracerRun(self) as tracer: + tracer.runcall(tfunc_main) + + def test_return(self): + self.expect_set = [ + ('line', 2, 'tfunc_main'), ('step', ), + ('line', 3, 'tfunc_main'), ('step', ), + ('call', 1, 'tfunc_first'), ('step', ), + ('line', 2, 'tfunc_first'), ('return', ), + ('return', 4, 'tfunc_first'), ('step', ), + ('line', 4, 'tfunc_main'), ('quit', ), + ] + with TracerRun(self) as tracer: + tracer.runcall(tfunc_main) + + def test_return_in_caller_frame(self): + self.expect_set = [ + ('line', 2, 'tfunc_main'), ('step', ), + ('line', 3, 'tfunc_main'), ('step', ), + ('call', 1, 'tfunc_first'), ('up', ), + ('None', 3, 'tfunc_main'), ('return', ), + ('return', 7, 'tfunc_main'), ('quit', ), + ] + with TracerRun(self) as tracer: + tracer.runcall(tfunc_main) + + def test_until(self): + self.expect_set = [ + ('line', 2, 'tfunc_main'), ('step', ), + ('line', 3, 'tfunc_main'), ('step', ), + ('call', 1, 'tfunc_first'), ('step', ), + ('line', 2, 'tfunc_first'), ('until', (4, )), + ('line', 4, 'tfunc_first'), ('quit', ), + ] + with TracerRun(self) as tracer: + tracer.runcall(tfunc_main) + + def test_until_with_too_large_count(self): + self.expect_set = [ + ('line', 2, 'tfunc_main'), break_in_func('tfunc_first'), + ('None', 2, 'tfunc_main'), ('continue', ), + ('line', 2, 'tfunc_first', ({1:1}, [])), ('until', (9999, )), + ('return', 4, 'tfunc_first'), ('quit', ), + ] + with TracerRun(self) as tracer: + tracer.runcall(tfunc_main) + + def test_until_in_caller_frame(self): + self.expect_set = [ + ('line', 2, 'tfunc_main'), ('step', ), + ('line', 3, 'tfunc_main'), ('step', ), + ('call', 1, 'tfunc_first'), ('up', ), + ('None', 3, 'tfunc_main'), ('until', (6, )), + ('line', 6, 'tfunc_main'), ('quit', ), + ] + with TracerRun(self) as tracer: + tracer.runcall(tfunc_main) + + def test_skip(self): + # Check that tracing is skipped over the import statement in + # 'tfunc_import()'. + code = """ + def main(): + lno = 3 + """ + modules = { TEST_MODULE: code } + with create_modules(modules): + self.expect_set = [ + ('line', 2, 'tfunc_import'), ('step', ), + ('line', 3, 'tfunc_import'), ('quit', ), + ] + skip = ('importlib*', TEST_MODULE) + with TracerRun(self, skip=skip) as tracer: + tracer.runcall(tfunc_import) + + def test_down(self): + # Check that set_down() raises BdbError at the newest frame. + self.expect_set = [ + ('line', 2, 'tfunc_main'), ('down', ), + ] + with TracerRun(self) as tracer: + self.assertRaises(BdbError, tracer.runcall, tfunc_main) + + def test_up(self): + self.expect_set = [ + ('line', 2, 'tfunc_main'), ('step', ), + ('line', 3, 'tfunc_main'), ('step', ), + ('call', 1, 'tfunc_first'), ('up', ), + ('None', 3, 'tfunc_main'), ('quit', ), + ] + with TracerRun(self) as tracer: + tracer.runcall(tfunc_main) + +class BreakpointTestCase(BaseTestCase): + """Test the breakpoint set method.""" + + def test_bp_on_non_existent_module(self): + self.expect_set = [ + ('line', 2, 'tfunc_import'), ('break', ('/non/existent/module.py', 1)) + ] + with TracerRun(self) as tracer: + self.assertRaises(BdbError, tracer.runcall, tfunc_import) + + def test_bp_after_last_statement(self): + code = """ + def main(): + lno = 3 + """ + modules = { TEST_MODULE: code } + with create_modules(modules): + self.expect_set = [ + ('line', 2, 'tfunc_import'), ('break', (TEST_MODULE_FNAME, 4)) + ] + with TracerRun(self) as tracer: + self.assertRaises(BdbError, tracer.runcall, tfunc_import) + + def test_temporary_bp(self): + code = """ + def func(): + lno = 3 + + def main(): + for i in range(2): + func() + """ + modules = { TEST_MODULE: code } + with create_modules(modules): + self.expect_set = [ + ('line', 2, 'tfunc_import'), + break_in_func('func', TEST_MODULE_FNAME, True), + ('None', 2, 'tfunc_import'), + break_in_func('func', TEST_MODULE_FNAME, True), + ('None', 2, 'tfunc_import'), ('continue', ), + ('line', 3, 'func', ({1:1}, [1])), ('continue', ), + ('line', 3, 'func', ({2:1}, [2])), ('quit', ), + ] + with TracerRun(self) as tracer: + tracer.runcall(tfunc_import) + + def test_disabled_temporary_bp(self): + code = """ + def func(): + lno = 3 + + def main(): + for i in range(3): + func() + """ + modules = { TEST_MODULE: code } + with create_modules(modules): + self.expect_set = [ + ('line', 2, 'tfunc_import'), + break_in_func('func', TEST_MODULE_FNAME), + ('None', 2, 'tfunc_import'), + break_in_func('func', TEST_MODULE_FNAME, True), + ('None', 2, 'tfunc_import'), ('disable', (2, )), + ('None', 2, 'tfunc_import'), ('continue', ), + ('line', 3, 'func', ({1:1}, [])), ('enable', (2, )), + ('None', 3, 'func'), ('disable', (1, )), + ('None', 3, 'func'), ('continue', ), + ('line', 3, 'func', ({2:1}, [2])), ('enable', (1, )), + ('None', 3, 'func'), ('continue', ), + ('line', 3, 'func', ({1:2}, [])), ('quit', ), + ] + with TracerRun(self) as tracer: + tracer.runcall(tfunc_import) + + def test_bp_condition(self): + code = """ + def func(a): + lno = 3 + + def main(): + for i in range(3): + func(i) + """ + modules = { TEST_MODULE: code } + with create_modules(modules): + self.expect_set = [ + ('line', 2, 'tfunc_import'), + break_in_func('func', TEST_MODULE_FNAME, False, 'a == 2'), + ('None', 2, 'tfunc_import'), ('continue', ), + ('line', 3, 'func', ({1:3}, [])), ('quit', ), + ] + with TracerRun(self) as tracer: + tracer.runcall(tfunc_import) + + def test_bp_exception_on_condition_evaluation(self): + code = """ + def func(a): + lno = 3 + + def main(): + func(0) + """ + modules = { TEST_MODULE: code } + with create_modules(modules): + self.expect_set = [ + ('line', 2, 'tfunc_import'), + break_in_func('func', TEST_MODULE_FNAME, False, '1 / 0'), + ('None', 2, 'tfunc_import'), ('continue', ), + ('line', 3, 'func', ({1:1}, [])), ('quit', ), + ] + with TracerRun(self) as tracer: + tracer.runcall(tfunc_import) + + def test_bp_ignore_count(self): + code = """ + def func(): + lno = 3 + + def main(): + for i in range(2): + func() + """ + modules = { TEST_MODULE: code } + with create_modules(modules): + self.expect_set = [ + ('line', 2, 'tfunc_import'), + break_in_func('func', TEST_MODULE_FNAME), + ('None', 2, 'tfunc_import'), ('ignore', (1, )), + ('None', 2, 'tfunc_import'), ('continue', ), + ('line', 3, 'func', ({1:2}, [])), ('quit', ), + ] + with TracerRun(self) as tracer: + tracer.runcall(tfunc_import) + + def test_ignore_count_on_disabled_bp(self): + code = """ + def func(): + lno = 3 + + def main(): + for i in range(3): + func() + """ + modules = { TEST_MODULE: code } + with create_modules(modules): + self.expect_set = [ + ('line', 2, 'tfunc_import'), + break_in_func('func', TEST_MODULE_FNAME), + ('None', 2, 'tfunc_import'), + break_in_func('func', TEST_MODULE_FNAME), + ('None', 2, 'tfunc_import'), ('ignore', (1, )), + ('None', 2, 'tfunc_import'), ('disable', (1, )), + ('None', 2, 'tfunc_import'), ('continue', ), + ('line', 3, 'func', ({2:1}, [])), ('enable', (1, )), + ('None', 3, 'func'), ('continue', ), + ('line', 3, 'func', ({2:2}, [])), ('continue', ), + ('line', 3, 'func', ({1:2}, [])), ('quit', ), + ] + with TracerRun(self) as tracer: + tracer.runcall(tfunc_import) + + def test_clear_two_bp_on_same_line(self): + code = """ + def func(): + lno = 3 + lno = 4 + + def main(): + for i in range(3): + func() + """ + modules = { TEST_MODULE: code } + with create_modules(modules): + self.expect_set = [ + ('line', 2, 'tfunc_import'), ('break', (TEST_MODULE_FNAME, 3)), + ('None', 2, 'tfunc_import'), ('break', (TEST_MODULE_FNAME, 3)), + ('None', 2, 'tfunc_import'), ('break', (TEST_MODULE_FNAME, 4)), + ('None', 2, 'tfunc_import'), ('continue', ), + ('line', 3, 'func', ({1:1}, [])), ('continue', ), + ('line', 4, 'func', ({3:1}, [])), ('clear', (TEST_MODULE_FNAME, 3)), + ('None', 4, 'func'), ('continue', ), + ('line', 4, 'func', ({3:2}, [])), ('quit', ), + ] + with TracerRun(self) as tracer: + tracer.runcall(tfunc_import) + + def test_clear_at_no_bp(self): + self.expect_set = [ + ('line', 2, 'tfunc_import'), ('clear', (__file__, 1)) + ] + with TracerRun(self) as tracer: + self.assertRaises(BdbError, tracer.runcall, tfunc_import) + +class RunTestCase(BaseTestCase): + """Test run, runeval and set_trace.""" + + def test_run_step(self): + # Check that the bdb 'run' method stops at the first line event. + code = """ + lno = 2 + """ + self.expect_set = [ + ('line', 2, '<module>'), ('step', ), + ('return', 2, '<module>'), ('quit', ), + ] + with TracerRun(self) as tracer: + tracer.run(compile(textwrap.dedent(code), '<string>', 'exec')) + + def test_runeval_step(self): + # Test bdb 'runeval'. + code = """ + def main(): + lno = 3 + """ + modules = { TEST_MODULE: code } + with create_modules(modules): + self.expect_set = [ + ('line', 1, '<module>'), ('step', ), + ('call', 2, 'main'), ('step', ), + ('line', 3, 'main'), ('step', ), + ('return', 3, 'main'), ('step', ), + ('return', 1, '<module>'), ('quit', ), + ] + import test_module + with TracerRun(self) as tracer: + tracer.runeval('test_module.main()', globals(), locals()) + +class IssuesTestCase(BaseTestCase): + """Test fixed bdb issues.""" + + def test_step_at_return_with_no_trace_in_caller(self): + # Issue #13183. + # Check that the tracer does step into the caller frame when the + # trace function is not set in that frame. + code_1 = """ + from test_module_2 import func + def main(): + func() + lno = 5 + """ + code_2 = """ + def func(): + lno = 3 + """ + modules = { + TEST_MODULE: code_1, + 'test_module_2': code_2, + } + with create_modules(modules): + self.expect_set = [ + ('line', 2, 'tfunc_import'), + break_in_func('func', 'test_module_2.py'), + ('None', 2, 'tfunc_import'), ('continue', ), + ('line', 3, 'func', ({1:1}, [])), ('step', ), + ('return', 3, 'func'), ('step', ), + ('line', 5, 'main'), ('quit', ), + ] + with TracerRun(self) as tracer: + tracer.runcall(tfunc_import) + + def test_next_until_return_in_generator(self): + # Issue #16596. + # Check that set_next(), set_until() and set_return() do not treat the + # `yield` and `yield from` statements as if they were returns and stop + # instead in the current frame. + code = """ + def test_gen(): + yield 0 + lno = 4 + return 123 + + def main(): + it = test_gen() + next(it) + next(it) + lno = 11 + """ + modules = { TEST_MODULE: code } + for set_type in ('next', 'until', 'return'): + with self.subTest(set_type=set_type): + with create_modules(modules): + self.expect_set = [ + ('line', 2, 'tfunc_import'), + break_in_func('test_gen', TEST_MODULE_FNAME), + ('None', 2, 'tfunc_import'), ('continue', ), + ('line', 3, 'test_gen', ({1:1}, [])), (set_type, ), + ] + + if set_type == 'return': + self.expect_set.extend( + [('exception', 10, 'main', StopIteration), ('step',), + ('return', 10, 'main'), ('quit', ), + ] + ) + else: + self.expect_set.extend( + [('line', 4, 'test_gen'), ('quit', ),] + ) + with TracerRun(self) as tracer: + tracer.runcall(tfunc_import) + + def test_next_command_in_generator_for_loop(self): + # Issue #16596. + code = """ + def test_gen(): + yield 0 + lno = 4 + yield 1 + return 123 + + def main(): + for i in test_gen(): + lno = 10 + lno = 11 + """ + modules = { TEST_MODULE: code } + with create_modules(modules): + self.expect_set = [ + ('line', 2, 'tfunc_import'), + break_in_func('test_gen', TEST_MODULE_FNAME), + ('None', 2, 'tfunc_import'), ('continue', ), + ('line', 3, 'test_gen', ({1:1}, [])), ('next', ), + ('line', 4, 'test_gen'), ('next', ), + ('line', 5, 'test_gen'), ('next', ), + ('line', 6, 'test_gen'), ('next', ), + ('exception', 9, 'main', StopIteration), ('step', ), + ('line', 11, 'main'), ('quit', ), + + ] + with TracerRun(self) as tracer: + tracer.runcall(tfunc_import) + + def test_next_command_in_generator_with_subiterator(self): + # Issue #16596. + code = """ + def test_subgen(): + yield 0 + return 123 + + def test_gen(): + x = yield from test_subgen() + return 456 + + def main(): + for i in test_gen(): + lno = 12 + lno = 13 + """ + modules = { TEST_MODULE: code } + with create_modules(modules): + self.expect_set = [ + ('line', 2, 'tfunc_import'), + break_in_func('test_gen', TEST_MODULE_FNAME), + ('None', 2, 'tfunc_import'), ('continue', ), + ('line', 7, 'test_gen', ({1:1}, [])), ('next', ), + ('line', 8, 'test_gen'), ('next', ), + ('exception', 11, 'main', StopIteration), ('step', ), + ('line', 13, 'main'), ('quit', ), + + ] + with TracerRun(self) as tracer: + tracer.runcall(tfunc_import) + + def test_return_command_in_generator_with_subiterator(self): + # Issue #16596. + code = """ + def test_subgen(): + yield 0 + return 123 + + def test_gen(): + x = yield from test_subgen() + return 456 + + def main(): + for i in test_gen(): + lno = 12 + lno = 13 + """ + modules = { TEST_MODULE: code } + with create_modules(modules): + self.expect_set = [ + ('line', 2, 'tfunc_import'), + break_in_func('test_subgen', TEST_MODULE_FNAME), + ('None', 2, 'tfunc_import'), ('continue', ), + ('line', 3, 'test_subgen', ({1:1}, [])), ('return', ), + ('exception', 7, 'test_gen', StopIteration), ('return', ), + ('exception', 11, 'main', StopIteration), ('step', ), + ('line', 13, 'main'), ('quit', ), + + ] + with TracerRun(self) as tracer: + tracer.runcall(tfunc_import) + +def test_main(): + test.support.run_unittest( + StateTestCase, + RunTestCase, + BreakpointTestCase, + IssuesTestCase, + ) + +if __name__ == "__main__": + test_main() diff --git a/Lib/test/test_sundry.py b/Lib/test/test_sundry.py index 1fb996469821..fc5ac0ae78e4 100644 --- a/Lib/test/test_sundry.py +++ b/Lib/test/test_sundry.py @@ -6,8 +6,7 @@ class TestUntestedModules(unittest.TestCase): def test_untested_modules_can_be_imported(self): - untested = ('bdb', 'encodings', 'formatter', - 'nturl2path', 'tabnanny') + untested = ('encodings', 'formatter', 'nturl2path', 'tabnanny') with support.check_warnings(quiet=True): for name in untested: try: diff --git a/Misc/NEWS.d/next/Tests/2018-01-08-13-33-47.bpo-19417.2asoXy.rst b/Misc/NEWS.d/next/Tests/2018-01-08-13-33-47.bpo-19417.2asoXy.rst new file mode 100644 index 000000000000..739352fcdd67 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2018-01-08-13-33-47.bpo-19417.2asoXy.rst @@ -0,0 +1 @@ +Add test_bdb.py. From webhook-mailer at python.org Mon Mar 19 14:41:40 2018 From: webhook-mailer at python.org (Ivan Levkivskyi) Date: Mon, 19 Mar 2018 18:41:40 -0000 Subject: [Python-checkins] bpo-33061: Add missing 'NoReturn' to __all__ in typing.py (GH-6127) Message-ID: <mailman.101.1521484902.1871.python-checkins@python.org> https://github.com/python/cpython/commit/4573820d2a9156346392838d455e89f33067e9dd commit: 4573820d2a9156346392838d455e89f33067e9dd branch: master author: aetracht <allen at duolingo.com> committer: Ivan Levkivskyi <levkivskyi at gmail.com> date: 2018-03-19T18:41:32Z summary: bpo-33061: Add missing 'NoReturn' to __all__ in typing.py (GH-6127) files: A Misc/NEWS.d/next/Library/2018-03-16-16-07-33.bpo-33061.TRTTek.rst M Lib/typing.py diff --git a/Lib/typing.py b/Lib/typing.py index 7ca080402e28..56126cfc0247 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -95,6 +95,7 @@ 'NewType', 'no_type_check', 'no_type_check_decorator', + 'NoReturn', 'overload', 'Text', 'TYPE_CHECKING', diff --git a/Misc/NEWS.d/next/Library/2018-03-16-16-07-33.bpo-33061.TRTTek.rst b/Misc/NEWS.d/next/Library/2018-03-16-16-07-33.bpo-33061.TRTTek.rst new file mode 100644 index 000000000000..b82ffb73a143 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-03-16-16-07-33.bpo-33061.TRTTek.rst @@ -0,0 +1 @@ +Add missing ``NoReturn`` to ``__all__`` in typing.py From webhook-mailer at python.org Mon Mar 19 21:07:54 2018 From: webhook-mailer at python.org (Eric V. Smith) Date: Tue, 20 Mar 2018 01:07:54 -0000 Subject: [Python-checkins] bpo-33100: Dataclasses now handles __slots__ and default values correctly. (GH-6152) Message-ID: <mailman.102.1521508074.1871.python-checkins@python.org> https://github.com/python/cpython/commit/7389fd935c95b4b6f094312294e703ee0de18719 commit: 7389fd935c95b4b6f094312294e703ee0de18719 branch: master author: Eric V. Smith <ericvsmith at users.noreply.github.com> committer: GitHub <noreply at github.com> date: 2018-03-19T21:07:51-04:00 summary: bpo-33100: Dataclasses now handles __slots__ and default values correctly. (GH-6152) If the class has a member that's a MemberDescriptorType, it's not a default value, it's from that member being in __slots__. files: A Misc/NEWS.d/next/Library/2018-03-19-20-47-00.bpo-33100.chyIO4.rst M Lib/dataclasses.py M Lib/test/test_dataclasses.py diff --git a/Lib/dataclasses.py b/Lib/dataclasses.py index 8ab04dd5b975..a4afd50376bd 100644 --- a/Lib/dataclasses.py +++ b/Lib/dataclasses.py @@ -519,6 +519,9 @@ def _get_field(cls, a_name, a_type): if isinstance(default, Field): f = default else: + if isinstance(default, types.MemberDescriptorType): + # This is a field in __slots__, so it has no default value. + default = MISSING f = field(default=default) # Assume it's a normal field until proven otherwise. diff --git a/Lib/test/test_dataclasses.py b/Lib/test/test_dataclasses.py index 3e6726360940..db03ec1925f6 100755 --- a/Lib/test/test_dataclasses.py +++ b/Lib/test/test_dataclasses.py @@ -2564,5 +2564,47 @@ class S(D): self.assertEqual(s.cached, True) +class TestSlots(unittest.TestCase): + def test_simple(self): + @dataclass + class C: + __slots__ = ('x',) + x: Any + + # There was a bug where a variable in a slot was assumed + # to also have a default value (of type types.MemberDescriptorType). + with self.assertRaisesRegex(TypeError, + "__init__\(\) missing 1 required positional argument: 'x'"): + C() + + # We can create an instance, and assign to x. + c = C(10) + self.assertEqual(c.x, 10) + c.x = 5 + self.assertEqual(c.x, 5) + + # We can't assign to anything else. + with self.assertRaisesRegex(AttributeError, "'C' object has no attribute 'y'"): + c.y = 5 + + def test_derived_added_field(self): + # See bpo-33100. + @dataclass + class Base: + __slots__ = ('x',) + x: Any + + @dataclass + class Derived(Base): + x: int + y: int + + d = Derived(1, 2) + self.assertEqual((d.x, d.y), (1, 2)) + + # We can add a new field to the derived instance. + d.z = 10 + + if __name__ == '__main__': unittest.main() diff --git a/Misc/NEWS.d/next/Library/2018-03-19-20-47-00.bpo-33100.chyIO4.rst b/Misc/NEWS.d/next/Library/2018-03-19-20-47-00.bpo-33100.chyIO4.rst new file mode 100644 index 000000000000..080a55c0cfb7 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-03-19-20-47-00.bpo-33100.chyIO4.rst @@ -0,0 +1,2 @@ +Dataclasses: If a field has a default value that's a MemberDescriptorType, +then it's from that field being in __slots__, not an actual default value. From webhook-mailer at python.org Mon Mar 19 21:31:25 2018 From: webhook-mailer at python.org (Eric V. Smith) Date: Tue, 20 Mar 2018 01:31:25 -0000 Subject: [Python-checkins] bpo-33100: Dataclasses now handles __slots__ and default values correctly. (GH-6152) (GH-6153) Message-ID: <mailman.103.1521509486.1871.python-checkins@python.org> https://github.com/python/cpython/commit/3d41f482594b6aab12a316202b3c06757262109a commit: 3d41f482594b6aab12a316202b3c06757262109a branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Eric V. Smith <ericvsmith at users.noreply.github.com> date: 2018-03-19T21:31:22-04:00 summary: bpo-33100: Dataclasses now handles __slots__ and default values correctly. (GH-6152) (GH-6153) If the class has a member that's a MemberDescriptorType, it's not a default value, it's from that member being in __slots__. (cherry picked from commit 7389fd935c95b4b6f094312294e703ee0de18719) Co-authored-by: Eric V. Smith <ericvsmith at users.noreply.github.com> files: A Misc/NEWS.d/next/Library/2018-03-19-20-47-00.bpo-33100.chyIO4.rst M Lib/dataclasses.py M Lib/test/test_dataclasses.py diff --git a/Lib/dataclasses.py b/Lib/dataclasses.py index 8ab04dd5b975..a4afd50376bd 100644 --- a/Lib/dataclasses.py +++ b/Lib/dataclasses.py @@ -519,6 +519,9 @@ def _get_field(cls, a_name, a_type): if isinstance(default, Field): f = default else: + if isinstance(default, types.MemberDescriptorType): + # This is a field in __slots__, so it has no default value. + default = MISSING f = field(default=default) # Assume it's a normal field until proven otherwise. diff --git a/Lib/test/test_dataclasses.py b/Lib/test/test_dataclasses.py index 3e6726360940..db03ec1925f6 100755 --- a/Lib/test/test_dataclasses.py +++ b/Lib/test/test_dataclasses.py @@ -2564,5 +2564,47 @@ class S(D): self.assertEqual(s.cached, True) +class TestSlots(unittest.TestCase): + def test_simple(self): + @dataclass + class C: + __slots__ = ('x',) + x: Any + + # There was a bug where a variable in a slot was assumed + # to also have a default value (of type types.MemberDescriptorType). + with self.assertRaisesRegex(TypeError, + "__init__\(\) missing 1 required positional argument: 'x'"): + C() + + # We can create an instance, and assign to x. + c = C(10) + self.assertEqual(c.x, 10) + c.x = 5 + self.assertEqual(c.x, 5) + + # We can't assign to anything else. + with self.assertRaisesRegex(AttributeError, "'C' object has no attribute 'y'"): + c.y = 5 + + def test_derived_added_field(self): + # See bpo-33100. + @dataclass + class Base: + __slots__ = ('x',) + x: Any + + @dataclass + class Derived(Base): + x: int + y: int + + d = Derived(1, 2) + self.assertEqual((d.x, d.y), (1, 2)) + + # We can add a new field to the derived instance. + d.z = 10 + + if __name__ == '__main__': unittest.main() diff --git a/Misc/NEWS.d/next/Library/2018-03-19-20-47-00.bpo-33100.chyIO4.rst b/Misc/NEWS.d/next/Library/2018-03-19-20-47-00.bpo-33100.chyIO4.rst new file mode 100644 index 000000000000..080a55c0cfb7 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-03-19-20-47-00.bpo-33100.chyIO4.rst @@ -0,0 +1,2 @@ +Dataclasses: If a field has a default value that's a MemberDescriptorType, +then it's from that field being in __slots__, not an actual default value. From webhook-mailer at python.org Tue Mar 20 02:41:39 2018 From: webhook-mailer at python.org (Berker Peksag) Date: Tue, 20 Mar 2018 06:41:39 -0000 Subject: [Python-checkins] bpo-33034: Improve exception message when cast fails for {Parse, Split}Result.port (GH-6078) Message-ID: <mailman.104.1521528100.1871.python-checkins@python.org> https://github.com/python/cpython/commit/2cb4661707818cfd92556e7fdf9068a993577002 commit: 2cb4661707818cfd92556e7fdf9068a993577002 branch: master author: Matt Eaton <agnosticdev at gmail.com> committer: Berker Peksag <berker.peksag at gmail.com> date: 2018-03-20T09:41:37+03:00 summary: bpo-33034: Improve exception message when cast fails for {Parse,Split}Result.port (GH-6078) files: A Misc/NEWS.d/next/Library/2018-03-11-08-44-12.bpo-33034.bpb23d.rst M Lib/test/test_urlparse.py M Lib/urllib/parse.py diff --git a/Lib/test/test_urlparse.py b/Lib/test/test_urlparse.py index ddee1c38d8b4..9d5121792833 100644 --- a/Lib/test/test_urlparse.py +++ b/Lib/test/test_urlparse.py @@ -936,6 +936,16 @@ def test_issue14072(self): self.assertEqual(p2.scheme, 'tel') self.assertEqual(p2.path, '+31641044153') + def test_port_casting_failure_message(self): + message = "Port could not be cast to integer value as 'oracle'" + p1 = urllib.parse.urlparse('http://Server=sde; Service=sde:oracle') + with self.assertRaisesRegex(ValueError, message): + p1.port + + p2 = urllib.parse.urlsplit('http://Server=sde; Service=sde:oracle') + with self.assertRaisesRegex(ValueError, message): + p2.port + def test_telurl_params(self): p1 = urllib.parse.urlparse('tel:123-4;phone-context=+1-650-516') self.assertEqual(p1.scheme, 'tel') diff --git a/Lib/urllib/parse.py b/Lib/urllib/parse.py index 58460f9234fb..3f8cfe5300c0 100644 --- a/Lib/urllib/parse.py +++ b/Lib/urllib/parse.py @@ -166,7 +166,11 @@ def hostname(self): def port(self): port = self._hostinfo[1] if port is not None: - port = int(port, 10) + try: + port = int(port, 10) + except ValueError: + message = f'Port could not be cast to integer value as {port!r}' + raise ValueError(message) from None if not ( 0 <= port <= 65535): raise ValueError("Port out of range 0-65535") return port diff --git a/Misc/NEWS.d/next/Library/2018-03-11-08-44-12.bpo-33034.bpb23d.rst b/Misc/NEWS.d/next/Library/2018-03-11-08-44-12.bpo-33034.bpb23d.rst new file mode 100644 index 000000000000..c81683abf592 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-03-11-08-44-12.bpo-33034.bpb23d.rst @@ -0,0 +1,3 @@ +Providing an explicit error message when casting the port property to anything +that is not an integer value using ``urlparse()`` and ``urlsplit()``. +Patch by Matt Eaton. From solipsis at pitrou.net Tue Mar 20 05:16:09 2018 From: solipsis at pitrou.net (solipsis at pitrou.net) Date: Tue, 20 Mar 2018 09:16:09 +0000 Subject: [Python-checkins] Daily reference leaks (4243df51fe43): sum=4 Message-ID: <20180320091609.1.DE6DDA2F4132C4B8@psf.io> results for 4243df51fe43 on branch "default" -------------------------------------------- test_functools leaked [0, 3, 1] memory blocks, sum=4 Command line was: ['./python', '-m', 'test.regrtest', '-uall', '-R', '3:3:/home/psf-users/antoine/refleaks/reflogpAsooi', '--timeout', '7200'] From webhook-mailer at python.org Tue Mar 20 15:16:33 2018 From: webhook-mailer at python.org (Antoine Pitrou) Date: Tue, 20 Mar 2018 19:16:33 -0000 Subject: [Python-checkins] [3.7] bpo-33021: Release the GIL during fstat() calls (GH-6019) (GH-6159) Message-ID: <mailman.105.1521573394.1871.python-checkins@python.org> https://github.com/python/cpython/commit/56cce1ca84344f519f7ed01024434c083031d354 commit: 56cce1ca84344f519f7ed01024434c083031d354 branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Antoine Pitrou <pitrou at free.fr> date: 2018-03-20T20:16:30+01:00 summary: [3.7] bpo-33021: Release the GIL during fstat() calls (GH-6019) (GH-6159) fstat may block for long time if the file descriptor is on a non-responsive NFS server, hanging all threads. Most fstat() calls are handled by _Py_fstat(), releasing the GIL internally, but but _Py_fstat_noraise() does not release the GIL, and most calls release the GIL explicitly around it. This patch fixes last 2 calls to _Py_fstat_no_raise(), avoiding hangs when calling: - mmap.mmap() - os.urandom() - random.seed() (cherry picked from commit 4484f9dca9149da135bbae035f10a50d20d1cbbb) Co-authored-by: Nir Soffer <nirsof at gmail.com> files: A Misc/NEWS.d/next/Library/2018-03-12-00-27-56.bpo-33021.m19B9T.rst M Modules/mmapmodule.c M Python/bootstrap_hash.c diff --git a/Misc/NEWS.d/next/Library/2018-03-12-00-27-56.bpo-33021.m19B9T.rst b/Misc/NEWS.d/next/Library/2018-03-12-00-27-56.bpo-33021.m19B9T.rst new file mode 100644 index 000000000000..50dfafe600d8 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-03-12-00-27-56.bpo-33021.m19B9T.rst @@ -0,0 +1,2 @@ +Release the GIL during fstat() calls, avoiding hang of all threads when +calling mmap.mmap(), os.urandom(), and random.seed(). Patch by Nir Soffer. diff --git a/Modules/mmapmodule.c b/Modules/mmapmodule.c index 6cf454573e93..95d42d6dc5e5 100644 --- a/Modules/mmapmodule.c +++ b/Modules/mmapmodule.c @@ -1050,6 +1050,7 @@ static PyObject * new_mmap_object(PyTypeObject *type, PyObject *args, PyObject *kwdict) { struct _Py_stat_struct status; + int fstat_result = -1; mmap_object *m_obj; Py_ssize_t map_size; off_t offset = 0; @@ -1115,8 +1116,14 @@ new_mmap_object(PyTypeObject *type, PyObject *args, PyObject *kwdict) if (fd != -1) (void)fcntl(fd, F_FULLFSYNC); #endif - if (fd != -1 && _Py_fstat_noraise(fd, &status) == 0 - && S_ISREG(status.st_mode)) { + + if (fd != -1) { + Py_BEGIN_ALLOW_THREADS + fstat_result = _Py_fstat_noraise(fd, &status); + Py_END_ALLOW_THREADS + } + + if (fd != -1 && fstat_result == 0 && S_ISREG(status.st_mode)) { if (map_size == 0) { if (status.st_size == 0) { PyErr_SetString(PyExc_ValueError, diff --git a/Python/bootstrap_hash.c b/Python/bootstrap_hash.c index 9fd5cfb200ea..073b2ea8dbda 100644 --- a/Python/bootstrap_hash.c +++ b/Python/bootstrap_hash.c @@ -301,10 +301,15 @@ dev_urandom(char *buffer, Py_ssize_t size, int raise) if (raise) { struct _Py_stat_struct st; + int fstat_result; if (urandom_cache.fd >= 0) { + Py_BEGIN_ALLOW_THREADS + fstat_result = _Py_fstat_noraise(urandom_cache.fd, &st); + Py_END_ALLOW_THREADS + /* Does the fd point to the same thing as before? (issue #21207) */ - if (_Py_fstat_noraise(urandom_cache.fd, &st) + if (fstat_result || st.st_dev != urandom_cache.st_dev || st.st_ino != urandom_cache.st_ino) { /* Something changed: forget the cached fd (but don't close it, From webhook-mailer at python.org Tue Mar 20 15:40:21 2018 From: webhook-mailer at python.org (Antoine Pitrou) Date: Tue, 20 Mar 2018 19:40:21 -0000 Subject: [Python-checkins] [3.6] bpo-33021: Release the GIL during fstat() calls (GH-6019) (GH-6160) Message-ID: <mailman.106.1521574824.1871.python-checkins@python.org> https://github.com/python/cpython/commit/f3e6eadbcf4f3e0fe53f4784485b1c8464c7d282 commit: f3e6eadbcf4f3e0fe53f4784485b1c8464c7d282 branch: 3.6 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Antoine Pitrou <pitrou at free.fr> date: 2018-03-20T20:40:18+01:00 summary: [3.6] bpo-33021: Release the GIL during fstat() calls (GH-6019) (GH-6160) fstat may block for long time if the file descriptor is on a non-responsive NFS server, hanging all threads. Most fstat() calls are handled by _Py_fstat(), releasing the GIL internally, but but _Py_fstat_noraise() does not release the GIL, and most calls release the GIL explicitly around it. This patch fixes last 2 calls to _Py_fstat_no_raise(), avoiding hangs when calling: - mmap.mmap() - os.urandom() - random.seed() (cherry picked from commit 4484f9dca9149da135bbae035f10a50d20d1cbbb) Co-authored-by: Nir Soffer <nirsof at gmail.com> files: A Misc/NEWS.d/next/Library/2018-03-12-00-27-56.bpo-33021.m19B9T.rst M Modules/mmapmodule.c M Python/random.c diff --git a/Misc/NEWS.d/next/Library/2018-03-12-00-27-56.bpo-33021.m19B9T.rst b/Misc/NEWS.d/next/Library/2018-03-12-00-27-56.bpo-33021.m19B9T.rst new file mode 100644 index 000000000000..50dfafe600d8 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-03-12-00-27-56.bpo-33021.m19B9T.rst @@ -0,0 +1,2 @@ +Release the GIL during fstat() calls, avoiding hang of all threads when +calling mmap.mmap(), os.urandom(), and random.seed(). Patch by Nir Soffer. diff --git a/Modules/mmapmodule.c b/Modules/mmapmodule.c index 426b7cacebb5..8acb61ab98a9 100644 --- a/Modules/mmapmodule.c +++ b/Modules/mmapmodule.c @@ -1078,6 +1078,7 @@ static PyObject * new_mmap_object(PyTypeObject *type, PyObject *args, PyObject *kwdict) { struct _Py_stat_struct status; + int fstat_result = -1; mmap_object *m_obj; Py_ssize_t map_size; off_t offset = 0; @@ -1143,8 +1144,14 @@ new_mmap_object(PyTypeObject *type, PyObject *args, PyObject *kwdict) if (fd != -1) (void)fcntl(fd, F_FULLFSYNC); #endif - if (fd != -1 && _Py_fstat_noraise(fd, &status) == 0 - && S_ISREG(status.st_mode)) { + + if (fd != -1) { + Py_BEGIN_ALLOW_THREADS + fstat_result = _Py_fstat_noraise(fd, &status); + Py_END_ALLOW_THREADS + } + + if (fd != -1 && fstat_result == 0 && S_ISREG(status.st_mode)) { if (map_size == 0) { if (status.st_size == 0) { PyErr_SetString(PyExc_ValueError, diff --git a/Python/random.c b/Python/random.c index e0ee153ec4a2..c62cbeb4ecb7 100644 --- a/Python/random.c +++ b/Python/random.c @@ -301,10 +301,15 @@ dev_urandom(char *buffer, Py_ssize_t size, int raise) if (raise) { struct _Py_stat_struct st; + int fstat_result; if (urandom_cache.fd >= 0) { + Py_BEGIN_ALLOW_THREADS + fstat_result = _Py_fstat_noraise(urandom_cache.fd, &st); + Py_END_ALLOW_THREADS + /* Does the fd point to the same thing as before? (issue #21207) */ - if (_Py_fstat_noraise(urandom_cache.fd, &st) + if (fstat_result || st.st_dev != urandom_cache.st_dev || st.st_ino != urandom_cache.st_ino) { /* Something changed: forget the cached fd (but don't close it, From webhook-mailer at python.org Tue Mar 20 18:23:22 2018 From: webhook-mailer at python.org (Paul Moore) Date: Tue, 20 Mar 2018 22:23:22 -0000 Subject: [Python-checkins] bpo-28247: Document Windows executable creation in zipapp (GH-6158) Message-ID: <mailman.107.1521584603.1871.python-checkins@python.org> https://github.com/python/cpython/commit/4be79f29463f632cd8b48486feadc2ed308fb520 commit: 4be79f29463f632cd8b48486feadc2ed308fb520 branch: master author: Cheryl Sabella <cheryl.sabella at gmail.com> committer: Paul Moore <p.f.moore at gmail.com> date: 2018-03-20T22:23:19Z summary: bpo-28247: Document Windows executable creation in zipapp (GH-6158) files: A Misc/NEWS.d/next/Documentation/2018-03-20-20-11-05.bpo-28247.-V-WS-.rst M Doc/library/zipapp.rst diff --git a/Doc/library/zipapp.rst b/Doc/library/zipapp.rst index 9076593c4b1c..a9a61c969c6f 100644 --- a/Doc/library/zipapp.rst +++ b/Doc/library/zipapp.rst @@ -229,6 +229,12 @@ fits in memory:: >>> with open('myapp.pyz', 'wb') as f: >>> f.write(temp.getvalue()) + +.. _zipapp-specifying-the-interpreter: + +Specifying the Interpreter +-------------------------- + Note that if you specify an interpreter and then distribute your application archive, you need to ensure that the interpreter used is portable. The Python launcher for Windows supports most common forms of POSIX ``#!`` line, but there @@ -245,6 +251,169 @@ are other issues to consider: exact version like "/usr/bin/env python3.4" as you will need to change your shebang line for users of Python 3.5, for example. +Typically, you should use an "/usr/bin/env python2" or "/usr/bin/env python3", +depending on whether your code is written for Python 2 or 3. + + +Creating Standalone Applications with zipapp +-------------------------------------------- + +Using the :mod:`zipapp` module, it is possible to create self-contained Python +programs, which can be distributed to end users who only need to have a +suitable version of Python installed on their system. The key to doing this +is to bundle all of the application's dependencies into the archive, along +with the application code. + +The steps to create a standalone archive are as follows: + +1. Create your application in a directory as normal, so you have a ``myapp`` + directory containing a ``__main__.py`` file, and any supporting application + code. + +2. Install all of your application's dependencies into the ``myapp`` directory, + using pip: + + .. code-block:: sh + + $ python -m pip install -r requirements.txt --target myapp + + (this assumes you have your project requirements in a ``requirements.txt`` + file - if not, you can just list the dependencies manually on the pip command + line). + +3. Optionally, delete the ``.dist-info`` directories created by pip in the + ``myapp`` directory. These hold metadata for pip to manage the packages, and + as you won't be making any further use of pip they aren't required - + although it won't do any harm if you leave them. + +4. Package the application using: + + .. code-block:: sh + + $ python -m zipapp -p "interpreter" myapp + +This will produce a standalone executable, which can be run on any machine with +the appropriate interpreter available. See :ref:`zipapp-specifying-the-interpreter` +for details. It can be shipped to users as a single file. + +On Unix, the ``myapp.pyz`` file is executable as it stands. You can rename the +file to remove the ``.pyz`` extension if you prefer a "plain" command name. On +Windows, the ``myapp.pyz[w]`` file is executable by virtue of the fact that +the Python interpreter registers the ``.pyz`` and ``.pyzw`` file extensions +when installed. + + +Making a Windows executable +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +On Windows, registration of the ``.pyz`` extension is optional, and +furthermore, there are certain places that don't recognise registered +extensions "transparently" (the simplest example is that +``subprocess.run(['myapp'])`` won't find your application - you need to +explicitly specify the extension). + +On Windows, therefore, it is often preferable to create an executable from the +zipapp. This is relatively easy, although it does require a C compiler. The +basic approach relies on the fact that zipfiles can have arbitrary data +prepended, and Windows exe files can have arbitrary data appended. So by +creating a suitable launcher and tacking the ``.pyz`` file onto the end of it, +you end up with a single-file executable that runs your application. + +A suitable launcher can be as simple as the following:: + + #define Py_LIMITED_API 1 + #include "Python.h" + + #define WIN32_LEAN_AND_MEAN + #include <windows.h> + + #ifdef WINDOWS + int WINAPI wWinMain( + HINSTANCE hInstance, /* handle to current instance */ + HINSTANCE hPrevInstance, /* handle to previous instance */ + LPWSTR lpCmdLine, /* pointer to command line */ + int nCmdShow /* show state of window */ + ) + #else + int wmain() + #endif + { + wchar_t **myargv = _alloca((__argc + 1) * sizeof(wchar_t*)); + myargv[0] = __wargv[0]; + memcpy(myargv + 1, __wargv, __argc * sizeof(wchar_t *)); + return Py_Main(__argc+1, myargv); + } + +If you define the ``WINDOWS`` preprocessor symbol, this will generate a +GUI executable, and without it, a console executable. + +To compile the executable, you can either just use the standard MSVC +command line tools, or you can take advantage of the fact that distutils +knows how to compile Python source:: + + >>> from distutils.ccompiler import new_compiler + >>> import distutils.sysconfig + >>> import sys + >>> import os + >>> from pathlib import Path + + >>> def compile(src): + >>> src = Path(src) + >>> cc = new_compiler() + >>> exe = src.stem + >>> cc.add_include_dir(distutils.sysconfig.get_python_inc()) + >>> cc.add_library_dir(os.path.join(sys.base_exec_prefix, 'libs')) + >>> # First the CLI executable + >>> objs = cc.compile([str(src)]) + >>> cc.link_executable(objs, exe) + >>> # Now the GUI executable + >>> cc.define_macro('WINDOWS') + >>> objs = cc.compile([str(src)]) + >>> cc.link_executable(objs, exe + 'w') + + >>> if __name__ == "__main__": + >>> compile("zastub.c") + +The resulting launcher uses the "Limited ABI", so it will run unchanged with +any version of Python 3.x. All it needs is for Python (``python3.dll``) to be +on the user's ``PATH``. + +For a fully standalone distribution, you can distribute the launcher with your +application appended, bundled with the Python "embedded" distribution. This +will run on any PC with the appropriate architecture (32 bit or 64 bit). + + +Caveats +~~~~~~~ + +There are some limitations to the process of bundling your application into +a single file. In most, if not all, cases they can be addressed without +needing major changes to your application. + +1. If your application depends on a package that includes a C extension, that + package cannot be run from a zip file (this is an OS limitation, as executable + code must be present in the filesystem for the OS loader to load it). In this + case, you can exclude that dependency from the zipfile, and either require + your users to have it installed, or ship it alongside your zipfile and add code + to your ``__main__.py`` to include the directory containing the unzipped + module in