From solipsis at pitrou.net Thu Mar 1 04:09:24 2018 From: solipsis at pitrou.net (solipsis at pitrou.net) Date: Thu, 01 Mar 2018 09:09:24 +0000 Subject: [Python-checkins] Daily reference leaks (4243df51fe43): sum=11 Message-ID: <20180301090924.1.B58AA01F01278E5A@psf.io> results for 4243df51fe43 on branch "default" -------------------------------------------- test_collections leaked [0, 0, 7] memory blocks, sum=7 test_functools leaked [0, 3, 1] memory blocks, sum=4 Command line was: ['./python', '-m', 'test.regrtest', '-uall', '-R', '3:3:/home/psf-users/antoine/refleaks/reflogiRNHxE', '--timeout', '7200'] From webhook-mailer at python.org Thu Mar 1 04:14:05 2018 From: webhook-mailer at python.org (Xiang Zhang) Date: Thu, 01 Mar 2018 09:14:05 -0000 Subject: [Python-checkins] bpo-32903: Fix a memory leak in os.chdir() on Windows (GH-5801) Message-ID: https://github.com/python/cpython/commit/3e197c7a6740d564ad52fb7901c07d5ff49460f5 commit: 3e197c7a6740d564ad52fb7901c07d5ff49460f5 branch: master author: Alexey Izbyshev committer: Xiang Zhang date: 2018-03-01T17:13:56+08:00 summary: bpo-32903: Fix a memory leak in os.chdir() on Windows (GH-5801) files: A Misc/NEWS.d/next/Windows/2018-02-28-11-03-24.bpo-32903.1SXY4t.rst M Modules/posixmodule.c diff --git a/Misc/NEWS.d/next/Windows/2018-02-28-11-03-24.bpo-32903.1SXY4t.rst b/Misc/NEWS.d/next/Windows/2018-02-28-11-03-24.bpo-32903.1SXY4t.rst new file mode 100644 index 000000000000..a20a414790f8 --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2018-02-28-11-03-24.bpo-32903.1SXY4t.rst @@ -0,0 +1,2 @@ +Fix a memory leak in os.chdir() on Windows if the current directory is set +to a UNC path. diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 4b592298834d..6bba8ee26e16 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -1529,15 +1529,15 @@ win32_wchdir(LPCWSTR path) return FALSE; } } - if (wcsncmp(new_path, L"\\\\", 2) == 0 || - wcsncmp(new_path, L"//", 2) == 0) - /* UNC path, nothing to do. */ - return TRUE; - env[1] = new_path[0]; - result = SetEnvironmentVariableW(env, new_path); + int is_unc_like_path = (wcsncmp(new_path, L"\\\\", 2) == 0 || + wcsncmp(new_path, L"//", 2) == 0); + if (!is_unc_like_path) { + env[1] = new_path[0]; + result = SetEnvironmentVariableW(env, new_path); + } if (new_path != path_buf) PyMem_RawFree(new_path); - return result; + return result ? TRUE : FALSE; } #endif From webhook-mailer at python.org Thu Mar 1 05:27:37 2018 From: webhook-mailer at python.org (Xiang Zhang) Date: Thu, 01 Mar 2018 10:27:37 -0000 Subject: [Python-checkins] [2.7] bpo-32903: Fix a memory leak in os.chdir() on Windows (GH-5801). (#5947) Message-ID: https://github.com/python/cpython/commit/aa40f92240adea7067c3add8e09cec09dcf24d7f commit: aa40f92240adea7067c3add8e09cec09dcf24d7f branch: 2.7 author: Alexey Izbyshev committer: Xiang Zhang date: 2018-03-01T18:27:34+08:00 summary: [2.7] bpo-32903: Fix a memory leak in os.chdir() on Windows (GH-5801). (#5947) (cherry picked from commit 3e197c7a6740d564ad52fb7901c07d5ff49460f5) Co-authored-by: Alexey Izbyshev files: A Misc/NEWS.d/next/Windows/2018-02-28-11-03-24.bpo-32903.1SXY4t.rst M Modules/posixmodule.c diff --git a/Misc/NEWS.d/next/Windows/2018-02-28-11-03-24.bpo-32903.1SXY4t.rst b/Misc/NEWS.d/next/Windows/2018-02-28-11-03-24.bpo-32903.1SXY4t.rst new file mode 100644 index 000000000000..a20a414790f8 --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2018-02-28-11-03-24.bpo-32903.1SXY4t.rst @@ -0,0 +1,2 @@ +Fix a memory leak in os.chdir() on Windows if the current directory is set +to a UNC path. diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index f8e081ed7d61..2baf9202c303 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -984,6 +984,7 @@ win32_wchdir(LPCWSTR path) wchar_t _new_path[MAX_PATH+1], *new_path = _new_path; int result; wchar_t env[4] = L"=x:"; + int is_unc_like_path; if(!SetCurrentDirectoryW(path)) return FALSE; @@ -1002,15 +1003,15 @@ win32_wchdir(LPCWSTR path) return FALSE; } } - if (wcsncmp(new_path, L"\\\\", 2) == 0 || - wcsncmp(new_path, L"//", 2) == 0) - /* UNC path, nothing to do. */ - return TRUE; - env[1] = new_path[0]; - result = SetEnvironmentVariableW(env, new_path); + is_unc_like_path = (wcsncmp(new_path, L"\\\\", 2) == 0 || + wcsncmp(new_path, L"//", 2) == 0); + if (!is_unc_like_path) { + env[1] = new_path[0]; + result = SetEnvironmentVariableW(env, new_path); + } if (new_path != _new_path) free(new_path); - return result; + return result ? TRUE : FALSE; } #endif From webhook-mailer at python.org Thu Mar 1 05:28:44 2018 From: webhook-mailer at python.org (Xiang Zhang) Date: Thu, 01 Mar 2018 10:28:44 -0000 Subject: [Python-checkins] bpo-32903: Fix a memory leak in os.chdir() on Windows (GH-5801) (#5945) Message-ID: https://github.com/python/cpython/commit/6ae75d9d1221459ab18c2599e42fcc45f9f65617 commit: 6ae75d9d1221459ab18c2599e42fcc45f9f65617 branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Xiang Zhang date: 2018-03-01T18:28:41+08:00 summary: bpo-32903: Fix a memory leak in os.chdir() on Windows (GH-5801) (#5945) (cherry picked from commit 3e197c7a6740d564ad52fb7901c07d5ff49460f5) Co-authored-by: Alexey Izbyshev files: A Misc/NEWS.d/next/Windows/2018-02-28-11-03-24.bpo-32903.1SXY4t.rst M Modules/posixmodule.c diff --git a/Misc/NEWS.d/next/Windows/2018-02-28-11-03-24.bpo-32903.1SXY4t.rst b/Misc/NEWS.d/next/Windows/2018-02-28-11-03-24.bpo-32903.1SXY4t.rst new file mode 100644 index 000000000000..a20a414790f8 --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2018-02-28-11-03-24.bpo-32903.1SXY4t.rst @@ -0,0 +1,2 @@ +Fix a memory leak in os.chdir() on Windows if the current directory is set +to a UNC path. diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 130e8c1b36f7..f6144a1d0527 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -1529,15 +1529,15 @@ win32_wchdir(LPCWSTR path) return FALSE; } } - if (wcsncmp(new_path, L"\\\\", 2) == 0 || - wcsncmp(new_path, L"//", 2) == 0) - /* UNC path, nothing to do. */ - return TRUE; - env[1] = new_path[0]; - result = SetEnvironmentVariableW(env, new_path); + int is_unc_like_path = (wcsncmp(new_path, L"\\\\", 2) == 0 || + wcsncmp(new_path, L"//", 2) == 0); + if (!is_unc_like_path) { + env[1] = new_path[0]; + result = SetEnvironmentVariableW(env, new_path); + } if (new_path != path_buf) PyMem_RawFree(new_path); - return result; + return result ? TRUE : FALSE; } #endif From webhook-mailer at python.org Thu Mar 1 08:01:49 2018 From: webhook-mailer at python.org (Eric V. Smith) Date: Thu, 01 Mar 2018 13:01:49 -0000 Subject: [Python-checkins] Fixed incorrect default value for dataclass unsafe_hash. (GH-5949) Message-ID: https://github.com/python/cpython/commit/5da8cfb838fa1bf3529c085c6dce1adf3d1eaf62 commit: 5da8cfb838fa1bf3529c085c6dce1adf3d1eaf62 branch: master author: Eric V. Smith committer: GitHub date: 2018-03-01T08:01:41-05:00 summary: Fixed incorrect default value for dataclass unsafe_hash. (GH-5949) files: M Lib/dataclasses.py diff --git a/Lib/dataclasses.py b/Lib/dataclasses.py index 54478fec93df..b55a497db302 100644 --- a/Lib/dataclasses.py +++ b/Lib/dataclasses.py @@ -745,7 +745,7 @@ def _process_class(cls, repr, eq, order, unsafe_hash, init, frozen): # underscore. The presence of _cls is used to detect if this # decorator is being called with parameters or not. def dataclass(_cls=None, *, init=True, repr=True, eq=True, order=False, - unsafe_hash=None, frozen=False): + unsafe_hash=False, frozen=False): """Returns the same class as was passed in, with dunder methods added based on the fields defined in the class. @@ -880,7 +880,7 @@ def _astuple_inner(obj, tuple_factory): def make_dataclass(cls_name, fields, *, bases=(), namespace=None, init=True, - repr=True, eq=True, order=False, unsafe_hash=None, + repr=True, eq=True, order=False, unsafe_hash=False, frozen=False): """Return a new dynamically created dataclass. From webhook-mailer at python.org Thu Mar 1 08:30:18 2018 From: webhook-mailer at python.org (Eric V. Smith) Date: Thu, 01 Mar 2018 13:30:18 -0000 Subject: [Python-checkins] Fixed incorrect default value for dataclass unsafe_hash. (GH-5949) (GH-5950) Message-ID: https://github.com/python/cpython/commit/398242a5b61067e3171b11964229fba87a251ef7 commit: 398242a5b61067e3171b11964229fba87a251ef7 branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Eric V. Smith date: 2018-03-01T08:30:13-05:00 summary: Fixed incorrect default value for dataclass unsafe_hash. (GH-5949) (GH-5950) (cherry picked from commit 5da8cfb838fa1bf3529c085c6dce1adf3d1eaf62) Co-authored-by: Eric V. Smith files: M Lib/dataclasses.py diff --git a/Lib/dataclasses.py b/Lib/dataclasses.py index 54478fec93df..b55a497db302 100644 --- a/Lib/dataclasses.py +++ b/Lib/dataclasses.py @@ -745,7 +745,7 @@ def _process_class(cls, repr, eq, order, unsafe_hash, init, frozen): # underscore. The presence of _cls is used to detect if this # decorator is being called with parameters or not. def dataclass(_cls=None, *, init=True, repr=True, eq=True, order=False, - unsafe_hash=None, frozen=False): + unsafe_hash=False, frozen=False): """Returns the same class as was passed in, with dunder methods added based on the fields defined in the class. @@ -880,7 +880,7 @@ def _astuple_inner(obj, tuple_factory): def make_dataclass(cls_name, fields, *, bases=(), namespace=None, init=True, - repr=True, eq=True, order=False, unsafe_hash=None, + repr=True, eq=True, order=False, unsafe_hash=False, frozen=False): """Return a new dynamically created dataclass. From webhook-mailer at python.org Thu Mar 1 13:48:13 2018 From: webhook-mailer at python.org (Alexander Belopolsky) Date: Thu, 01 Mar 2018 18:48:13 -0000 Subject: [Python-checkins] Add What's new entry for datetime.fromisoformat (GH-5559) (GH-5939) Message-ID: https://github.com/python/cpython/commit/0e06be836ca0d578cf9fc0c68979eb682c00f89c commit: 0e06be836ca0d578cf9fc0c68979eb682c00f89c branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Alexander Belopolsky date: 2018-03-01T13:48:10-05:00 summary: Add What's new entry for datetime.fromisoformat (GH-5559) (GH-5939) Documents bpo-15873 (cherry picked from commit 22864bc8e4a076bbac748ccda6c27f1ec41b53e7) Co-authored-by: Paul Ganssle files: M Doc/whatsnew/3.7.rst diff --git a/Doc/whatsnew/3.7.rst b/Doc/whatsnew/3.7.rst index 8fa39abc6ace..46f4f13aab47 100644 --- a/Doc/whatsnew/3.7.rst +++ b/Doc/whatsnew/3.7.rst @@ -502,6 +502,14 @@ Added support for the Blowfish method. The :func:`~crypt.mksalt` function now allows to specify the number of rounds for hashing. (Contributed by Serhiy Storchaka in :issue:`31702`.) +datetime +-------- + +Added the :func:`datetime.datetime.fromisoformat` method, which constructs a +:class:`datetime.datetime` object from a string in one of the formats output +by :func:`datetime.datetime.isoformat`. (Contributed by Paul Ganssle in +:issue:`15873`.) + dis --- From webhook-mailer at python.org Thu Mar 1 16:02:59 2018 From: webhook-mailer at python.org (Ned Deily) Date: Thu, 01 Mar 2018 21:02:59 -0000 Subject: [Python-checkins] bpo-30607: Use external python-doc-theme (GH-2017) Message-ID: https://github.com/python/cpython/commit/bf63e8d55fd2853df3bb99d66de7f428107aadb3 commit: bf63e8d55fd2853df3bb99d66de7f428107aadb3 branch: master author: Jon Wayne Parrott committer: Ned Deily date: 2018-03-01T16:02:50-05:00 summary: bpo-30607: Use external python-doc-theme (GH-2017) files: A Doc/tools/static/changelog_search.js A Misc/NEWS.d/next/Documentation/2018-01-25-13-58-49.bpo-30607.4dXxiq.rst D Doc/tools/pydoctheme/static/pydoctheme.css D Doc/tools/pydoctheme/theme.conf D Doc/tools/static/copybutton.js D Doc/tools/static/py.png D Doc/tools/static/sidebar.js M .travis.yml M Doc/Makefile M Doc/README.rst M Doc/conf.py M Doc/tools/templates/layout.html diff --git a/.travis.yml b/.travis.yml index 292b9c687638..af3cdf54654d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -49,7 +49,8 @@ matrix: - cd Doc # Sphinx is pinned so that new versions that introduce new warnings won't suddenly cause build failures. # (Updating the version is fine as long as no warnings are raised by doing so.) - - python -m pip install sphinx~=1.6.1 blurb + # The theme used by the docs is stored seperately, so we need to install that as well. + - python -m pip install sphinx~=1.6.1 blurb python-docs-theme script: - make check suspicious html SPHINXOPTS="-q -W -j4" - os: osx diff --git a/Doc/Makefile b/Doc/Makefile index 307d1e0e7de1..042f960b93e7 100644 --- a/Doc/Makefile +++ b/Doc/Makefile @@ -123,7 +123,7 @@ clean: venv: $(PYTHON) -m venv $(VENVDIR) - $(VENVDIR)/bin/python3 -m pip install -U Sphinx blurb + $(VENVDIR)/bin/python3 -m pip install -U Sphinx blurb python-docs-theme @echo "The venv has been created in the $(VENVDIR) directory" dist: diff --git a/Doc/README.rst b/Doc/README.rst index a29d1f3a708a..b34916040eb9 100644 --- a/Doc/README.rst +++ b/Doc/README.rst @@ -20,11 +20,11 @@ tree but are maintained separately and are available from * `Sphinx `_ * `blurb `_ +* `python-docs-theme `_ The easiest way to install these tools is to create a virtual environment and install the tools into there. - Using make ---------- diff --git a/Doc/conf.py b/Doc/conf.py index 19a2f7d67ff8..d8efce035c9c 100644 --- a/Doc/conf.py +++ b/Doc/conf.py @@ -46,9 +46,13 @@ # ----------------------- # Use our custom theme. -html_theme = 'pydoctheme' +html_theme = 'python_docs_theme' html_theme_path = ['tools'] -html_theme_options = {'collapsiblesidebar': True} +html_theme_options = { + 'collapsiblesidebar': True, + 'issues_url': 'https://docs.python.org/3/bugs.html', + 'root_include_title': False # We use the version switcher instead. +} # Short title used e.g. for HTML tags. html_short_title = '%s Documentation' % release diff --git a/Doc/tools/pydoctheme/static/pydoctheme.css b/Doc/tools/pydoctheme/static/pydoctheme.css deleted file mode 100644 index 1d5c18e5f623..000000000000 --- a/Doc/tools/pydoctheme/static/pydoctheme.css +++ /dev/null @@ -1,194 +0,0 @@ - at import url("default.css"); - -body { - background-color: white; - margin-left: 1em; - margin-right: 1em; -} - -div.related { - margin-bottom: 1.2em; - padding: 0.5em 0; - border-top: 1px solid #ccc; - margin-top: 0.5em; -} - -div.related a:hover { - color: #0095C4; -} - -div.related:first-child { - border-top: 0; - border-bottom: 1px solid #ccc; -} - -.inline-search { - display: inline; -} -form.inline-search input { - display: inline; -} -form.inline-search input[type="submit"] { - width: 30px; -} - -div.sphinxsidebar { - background-color: #eeeeee; - border-radius: 5px; - line-height: 130%; - font-size: smaller; -} - -div.sphinxsidebar h3, div.sphinxsidebar h4 { - margin-top: 1.5em; -} - -div.sphinxsidebarwrapper > h3:first-child { - margin-top: 0.2em; -} - -div.sphinxsidebarwrapper > ul > li > ul > li { - margin-bottom: 0.4em; -} - -div.sphinxsidebar a:hover { - color: #0095C4; -} - -form.inline-search input, -div.sphinxsidebar input { - font-family: 'Lucida Grande',Arial,sans-serif; - border: 1px solid #999999; - font-size: smaller; - border-radius: 3px; -} - -div.sphinxsidebar input[type=text] { - max-width: 150px; -} - -div.body { - padding: 0 0 0 1.2em; -} - -div.body p { - line-height: 140%; -} - -div.body h1, div.body h2, div.body h3, div.body h4, div.body h5, div.body h6 { - margin: 0; - border: 0; - padding: 0.3em 0; -} - -div.body hr { - border: 0; - background-color: #ccc; - height: 1px; -} - -div.body pre { - border-radius: 3px; - border: 1px solid #ac9; -} - -div.body div.admonition, div.body div.impl-detail { - border-radius: 3px; -} - -div.body div.impl-detail > p { - margin: 0; -} - -div.body div.seealso { - border: 1px solid #dddd66; -} - -div.body a { - color: #0072aa; -} - -div.body a:visited { - color: #6363bb; -} - -div.body a:hover { - color: #00B0E4; -} - -tt, code, pre { - font-family: monospace, sans-serif; - font-size: 96.5%; -} - -div.body tt, div.body code { - border-radius: 3px; -} - -div.body tt.descname, div.body code.descname { - font-size: 120%; -} - -div.body tt.xref, div.body a tt, div.body code.xref, div.body a code { - font-weight: normal; -} - -.deprecated { - border-radius: 3px; -} - -table.docutils { - border: 1px solid #ddd; - min-width: 20%; - border-radius: 3px; - margin-top: 10px; - margin-bottom: 10px; -} - -table.docutils td, table.docutils th { - border: 1px solid #ddd !important; - border-radius: 3px; -} - -table p, table li { - text-align: left !important; -} - -table.docutils th { - background-color: #eee; - padding: 0.3em 0.5em; -} - -table.docutils td { - background-color: white; - padding: 0.3em 0.5em; -} - -table.footnote, table.footnote td { - border: 0 !important; -} - -div.footer { - line-height: 150%; - margin-top: -2em; - text-align: right; - width: auto; - margin-right: 10px; -} - -div.footer a:hover { - color: #0095C4; -} - -.refcount { - color: #060; -} - -.stableabi { - color: #229; -} - -.highlight { - background: none !important; -} - diff --git a/Doc/tools/pydoctheme/theme.conf b/Doc/tools/pydoctheme/theme.conf deleted file mode 100644 index 0c438816740e..000000000000 --- a/Doc/tools/pydoctheme/theme.conf +++ /dev/null @@ -1,23 +0,0 @@ -[theme] -inherit = default -stylesheet = pydoctheme.css -pygments_style = sphinx - -[options] -bodyfont = 'Lucida Grande', Arial, sans-serif -headfont = 'Lucida Grande', Arial, sans-serif -footerbgcolor = white -footertextcolor = #555555 -relbarbgcolor = white -relbartextcolor = #666666 -relbarlinkcolor = #444444 -sidebarbgcolor = white -sidebartextcolor = #444444 -sidebarlinkcolor = #444444 -bgcolor = white -textcolor = #222222 -linkcolor = #0090c0 -visitedlinkcolor = #00608f -headtextcolor = #1a1a1a -headbgcolor = white -headlinkcolor = #aaaaaa diff --git a/Doc/tools/static/changelog_search.js b/Doc/tools/static/changelog_search.js new file mode 100644 index 000000000000..c881a9bd4c84 --- /dev/null +++ b/Doc/tools/static/changelog_search.js @@ -0,0 +1,53 @@ +$(document).ready(function() { + // add the search form and bind the events + $('h1').after([ + '<p>Filter entries by content:', + '<input type="text" value="" id="searchbox" style="width: 50%">', + '<input type="submit" id="searchbox-submit" value="Filter"></p>' + ].join('\n')); + + function dofilter() { + try { + var query = new RegExp($('#searchbox').val(), 'i'); + } + catch (e) { + return; // not a valid regex (yet) + } + // find headers for the versions (What's new in Python X.Y.Z?) + $('#changelog h2').each(function(index1, h2) { + var h2_parent = $(h2).parent(); + var sections_found = 0; + // find headers for the sections (Core, Library, etc.) + h2_parent.find('h3').each(function(index2, h3) { + var h3_parent = $(h3).parent(); + var entries_found = 0; + // find all the entries + h3_parent.find('li').each(function(index3, li) { + var li = $(li); + // check if the query matches the entry + if (query.test(li.text())) { + li.show(); + entries_found++; + } + else { + li.hide(); + } + }); + // if there are entries, show the section, otherwise hide it + if (entries_found > 0) { + h3_parent.show(); + sections_found++; + } + else { + h3_parent.hide(); + } + }); + if (sections_found > 0) + h2_parent.show(); + else + h2_parent.hide(); + }); + } + $('#searchbox').keyup(dofilter); + $('#searchbox-submit').click(dofilter); +}); diff --git a/Doc/tools/static/copybutton.js b/Doc/tools/static/copybutton.js deleted file mode 100644 index 716c9e472f4b..000000000000 --- a/Doc/tools/static/copybutton.js +++ /dev/null @@ -1,62 +0,0 @@ -$(document).ready(function() { - /* Add a [>>>] button on the top-right corner of code samples to hide - * the >>> and ... prompts and the output and thus make the code - * copyable. */ - var div = $('.highlight-python .highlight,' + - '.highlight-python3 .highlight') - var pre = div.find('pre'); - - // get the styles from the current theme - pre.parent().parent().css('position', 'relative'); - var hide_text = 'Hide the prompts and output'; - var show_text = 'Show the prompts and output'; - var border_width = pre.css('border-top-width'); - var border_style = pre.css('border-top-style'); - var border_color = pre.css('border-top-color'); - var button_styles = { - 'cursor':'pointer', 'position': 'absolute', 'top': '0', 'right': '0', - 'border-color': border_color, 'border-style': border_style, - 'border-width': border_width, 'color': border_color, 'text-size': '75%', - 'font-family': 'monospace', 'padding-left': '0.2em', 'padding-right': '0.2em', - 'border-radius': '0 3px 0 0' - } - - // create and add the button to all the code blocks that contain >>> - div.each(function(index) { - var jthis = $(this); - if (jthis.find('.gp').length > 0) { - var button = $('<span class="copybutton">>>></span>'); - button.css(button_styles) - button.attr('title', hide_text); - button.data('hidden', 'false'); - jthis.prepend(button); - } - // tracebacks (.gt) contain bare text elements that need to be - // wrapped in a span to work with .nextUntil() (see later) - jthis.find('pre:has(.gt)').contents().filter(function() { - return ((this.nodeType == 3) && (this.data.trim().length > 0)); - }).wrap('<span>'); - }); - - // define the behavior of the button when it's clicked - $('.copybutton').click(function(e){ - e.preventDefault(); - var button = $(this); - if (button.data('hidden') === 'false') { - // hide the code output - button.parent().find('.go, .gp, .gt').hide(); - button.next('pre').find('.gt').nextUntil('.gp, .go').css('visibility', 'hidden'); - button.css('text-decoration', 'line-through'); - button.attr('title', show_text); - button.data('hidden', 'true'); - } else { - // show the code output - button.parent().find('.go, .gp, .gt').show(); - button.next('pre').find('.gt').nextUntil('.gp, .go').css('visibility', 'visible'); - button.css('text-decoration', 'none'); - button.attr('title', hide_text); - button.data('hidden', 'false'); - } - }); -}); - diff --git a/Doc/tools/static/py.png b/Doc/tools/static/py.png deleted file mode 100644 index 93e4a02c3d32..000000000000 Binary files a/Doc/tools/static/py.png and /dev/null differ diff --git a/Doc/tools/static/sidebar.js b/Doc/tools/static/sidebar.js deleted file mode 100644 index e8d58f4bfaac..000000000000 --- a/Doc/tools/static/sidebar.js +++ /dev/null @@ -1,193 +0,0 @@ -/* - * sidebar.js - * ~~~~~~~~~~ - * - * This script makes the Sphinx sidebar collapsible and implements intelligent - * scrolling. - * - * .sphinxsidebar contains .sphinxsidebarwrapper. This script adds in - * .sphixsidebar, after .sphinxsidebarwrapper, the #sidebarbutton used to - * collapse and expand the sidebar. - * - * When the sidebar is collapsed the .sphinxsidebarwrapper is hidden and the - * width of the sidebar and the margin-left of the document are decreased. - * When the sidebar is expanded the opposite happens. This script saves a - * per-browser/per-session cookie used to remember the position of the sidebar - * among the pages. Once the browser is closed the cookie is deleted and the - * position reset to the default (expanded). - * - * :copyright: Copyright 2007-2011 by the Sphinx team, see AUTHORS. - * :license: BSD, see LICENSE for details. - * - */ - -$(function() { - // global elements used by the functions. - // the 'sidebarbutton' element is defined as global after its - // creation, in the add_sidebar_button function - var jwindow = $(window); - var jdocument = $(document); - var bodywrapper = $('.bodywrapper'); - var sidebar = $('.sphinxsidebar'); - var sidebarwrapper = $('.sphinxsidebarwrapper'); - - // original margin-left of the bodywrapper and width of the sidebar - // with the sidebar expanded - var bw_margin_expanded = bodywrapper.css('margin-left'); - var ssb_width_expanded = sidebar.width(); - - // margin-left of the bodywrapper and width of the sidebar - // with the sidebar collapsed - var bw_margin_collapsed = '.8em'; - var ssb_width_collapsed = '.8em'; - - // colors used by the current theme - var dark_color = '#AAAAAA'; - var light_color = '#CCCCCC'; - - function get_viewport_height() { - if (window.innerHeight) - return window.innerHeight; - else - return jwindow.height(); - } - - function sidebar_is_collapsed() { - return sidebarwrapper.is(':not(:visible)'); - } - - function toggle_sidebar() { - if (sidebar_is_collapsed()) - expand_sidebar(); - else - collapse_sidebar(); - // adjust the scrolling of the sidebar - scroll_sidebar(); - } - - function collapse_sidebar() { - sidebarwrapper.hide(); - sidebar.css('width', ssb_width_collapsed); - bodywrapper.css('margin-left', bw_margin_collapsed); - sidebarbutton.css({ - 'margin-left': '0', - 'height': bodywrapper.height(), - 'border-radius': '5px' - }); - sidebarbutton.find('span').text('?'); - sidebarbutton.attr('title', _('Expand sidebar')); - document.cookie = 'sidebar=collapsed'; - } - - function expand_sidebar() { - bodywrapper.css('margin-left', bw_margin_expanded); - sidebar.css('width', ssb_width_expanded); - sidebarwrapper.show(); - sidebarbutton.css({ - 'margin-left': ssb_width_expanded-12, - 'height': bodywrapper.height(), - 'border-radius': '0 5px 5px 0' - }); - sidebarbutton.find('span').text('?'); - sidebarbutton.attr('title', _('Collapse sidebar')); - //sidebarwrapper.css({'padding-top': - // Math.max(window.pageYOffset - sidebarwrapper.offset().top, 10)}); - document.cookie = 'sidebar=expanded'; - } - - function add_sidebar_button() { - sidebarwrapper.css({ - 'float': 'left', - 'margin-right': '0', - 'width': ssb_width_expanded - 28 - }); - // create the button - sidebar.append( - '<div id="sidebarbutton"><span>«</span></div>' - ); - var sidebarbutton = $('#sidebarbutton'); - // find the height of the viewport to center the '<<' in the page - var viewport_height = get_viewport_height(); - var sidebar_offset = sidebar.offset().top; - var sidebar_height = Math.max(bodywrapper.height(), sidebar.height()); - sidebarbutton.find('span').css({ - 'display': 'block', - 'position': 'fixed', - 'top': Math.min(viewport_height/2, sidebar_height/2 + sidebar_offset) - 10 - }); - - sidebarbutton.click(toggle_sidebar); - sidebarbutton.attr('title', _('Collapse sidebar')); - sidebarbutton.css({ - 'border-radius': '0 5px 5px 0', - 'color': '#444444', - 'background-color': '#CCCCCC', - 'font-size': '1.2em', - 'cursor': 'pointer', - 'height': sidebar_height, - 'padding-top': '1px', - 'padding-left': '1px', - 'margin-left': ssb_width_expanded - 12 - }); - - sidebarbutton.hover( - function () { - $(this).css('background-color', dark_color); - }, - function () { - $(this).css('background-color', light_color); - } - ); - } - - function set_position_from_cookie() { - if (!document.cookie) - return; - var items = document.cookie.split(';'); - for(var k=0; k<items.length; k++) { - var key_val = items[k].split('='); - var key = key_val[0]; - if (key == 'sidebar') { - var value = key_val[1]; - if ((value == 'collapsed') && (!sidebar_is_collapsed())) - collapse_sidebar(); - else if ((value == 'expanded') && (sidebar_is_collapsed())) - expand_sidebar(); - } - } - } - - add_sidebar_button(); - var sidebarbutton = $('#sidebarbutton'); - set_position_from_cookie(); - - - /* intelligent scrolling */ - function scroll_sidebar() { - var sidebar_height = sidebarwrapper.height(); - var viewport_height = get_viewport_height(); - var offset = sidebar.position()['top']; - var wintop = jwindow.scrollTop(); - var winbot = wintop + viewport_height; - var curtop = sidebarwrapper.position()['top']; - var curbot = curtop + sidebar_height; - // does sidebar fit in window? - if (sidebar_height < viewport_height) { - // yes: easy case -- always keep at the top - sidebarwrapper.css('top', $u.min([$u.max([0, wintop - offset - 10]), - jdocument.height() - sidebar_height - 200])); - } - else { - // no: only scroll if top/bottom edge of sidebar is at - // top/bottom edge of window - if (curtop > wintop && curbot > winbot) { - sidebarwrapper.css('top', $u.max([wintop - offset - 10, 0])); - } - else if (curtop < wintop && curbot < winbot) { - sidebarwrapper.css('top', $u.min([winbot - sidebar_height - offset - 20, - jdocument.height() - sidebar_height - 200])); - } - } - } - jwindow.scroll(scroll_sidebar); -}); diff --git a/Doc/tools/templates/layout.html b/Doc/tools/templates/layout.html index c2106678ac60..37811725d86b 100644 --- a/Doc/tools/templates/layout.html +++ b/Doc/tools/templates/layout.html @@ -1,118 +1,25 @@ {% extends "!layout.html" %} + {% block rootrellink %} - <li><img src="{{ pathto('_static/py.png', 1) }}" alt="" - style="vertical-align: middle; margin-top: -1px"/></li> - <li><a href="https://www.python.org/">Python</a>{{ reldelim1 }}</li> - <li> - {%- if switchers is defined %} - <span class="language_switcher_placeholder">{{ language or 'en' }}</span> - <span class="version_switcher_placeholder">{{ release }}</span> - <a href="{{ pathto('index') }}">{% trans %}Documentation {% endtrans %}</a>{{ reldelim1 }} - {%- else %} - <a href="{{ pathto('index') }}">{{ shorttitle }}</a>{{ reldelim1 }} - {%- endif %} - </li> -{% endblock %} -{%- macro searchbox() %} -{# modified from sphinx/themes/basic/searchbox.html #} - {%- if builder != "htmlhelp" %} - <div class="inline-search" style="display: none" role="search"> - <form class="inline-search" action="{{ pathto('search') }}" method="get"> - <input placeholder="{{ _('Quick search') }}" type="text" name="q" /> - <input type="submit" value="{{ _('Go') }}" /> - <input type="hidden" name="check_keywords" value="yes" /> - <input type="hidden" name="area" value="default" /> - </form> - </div> - <script type="text/javascript">$('.inline-search').show(0);</script> - {%- endif %} -{%- endmacro %} -{% block relbar1 %} {% if builder != 'qthelp' %} {{ relbar() }} {% endif %} {% endblock %} -{% block relbar2 %} {% if builder != 'qthelp' %} {{ relbar() }} {% endif %} {% endblock %} -{% block relbaritems %} - {%- if pagename != "search" and builder != "singlehtml" and builder != "htmlhelp" %} - <li class="right"> - {{ searchbox() }} - {{ reldelim2 }} +{{ super() }} + <li> + {%- if switchers is defined %} + <span class="language_switcher_placeholder">{{ language or 'en' }}</span> + <span class="version_switcher_placeholder">{{ release }}</span> + <a href="{{ pathto('index') }}">{% trans %}Documentation {% endtrans %}</a>{{ reldelim1 }} + {%- else %} + <a href="{{ pathto('index') }}">{{ shorttitle }}</a>{{ reldelim1 }} + {%- endif %} </li> - {%- endif %} {% endblock %} + {% block extrahead %} - <link rel="shortcut icon" type="image/png" href="{{ pathto('_static/py.png', 1) }}" /> <link rel="canonical" href="https://docs.python.org/3/{{pagename}}.html" /> {% if builder != "htmlhelp" %} - {% if not embedded %}<script type="text/javascript" src="{{ pathto('_static/copybutton.js', 1) }}"></script>{% endif %} - {% if switchers is defined and not embedded %}<script type="text/javascript" src="{{ pathto('_static/switchers.js', 1) }}"></script>{% endif %} - {% if pagename == 'whatsnew/changelog' and not embedded %} - <script type="text/javascript"> - $(document).ready(function() { - // add the search form and bind the events - $('h1').after([ - '<p>Filter entries by content:', - '<input type="text" value="" id="searchbox" style="width: 50%">', - '<input type="submit" id="searchbox-submit" value="Filter"></p>' - ].join('\n')); - - function dofilter() { - try { - var query = new RegExp($('#searchbox').val(), 'i'); - } - catch (e) { - return; // not a valid regex (yet) - } - // find headers for the versions (What's new in Python X.Y.Z?) - $('#changelog h2').each(function(index1, h2) { - var h2_parent = $(h2).parent(); - var sections_found = 0; - // find headers for the sections (Core, Library, etc.) - h2_parent.find('h3').each(function(index2, h3) { - var h3_parent = $(h3).parent(); - var entries_found = 0; - // find all the entries - h3_parent.find('li').each(function(index3, li) { - var li = $(li); - // check if the query matches the entry - if (query.test(li.text())) { - li.show(); - entries_found++; - } - else { - li.hide(); - } - }); - // if there are entries, show the section, otherwise hide it - if (entries_found > 0) { - h3_parent.show(); - sections_found++; - } - else { - h3_parent.hide(); - } - }); - if (sections_found > 0) - h2_parent.show(); - else - h2_parent.hide(); - }); - } - $('#searchbox').keyup(dofilter); - $('#searchbox-submit').click(dofilter); - }); - </script> - {% endif %} + {% if switchers is defined and not embedded %} + <script type="text/javascript" src="{{ pathto('_static/switchers.js', 1) }}"></script>{% endif %} + {% if pagename == 'whatsnew/changelog' and not embedded %} + <script type="text/javascript" src="{{ pathto('_static/changelog_search.js', 1) }}"></script>{% endif %} {% endif %} {{ super() }} {% endblock %} -{% block footer %} - <div class="footer"> - © <a href="{{ pathto('copyright') }}">{% trans %}Copyright{% endtrans %}</a> {{ copyright|e }}. - <br /> - {% trans %}The Python Software Foundation is a non-profit corporation.{% endtrans %} - <a href="https://www.python.org/psf/donations/">{% trans %}Please donate.{% endtrans %}</a> - <br /> - {% trans last_updated=last_updated|e %}Last updated on {{ last_updated }}.{% endtrans %} - {% trans pathto_bugs=pathto('bugs') %}<a href="{{ pathto_bugs }}">Found a bug</a>?{% endtrans %} - <br /> - {% trans sphinx_version=sphinx_version|e %}Created using <a href="http://sphinx.pocoo.org/">Sphinx</a> {{ sphinx_version }}.{% endtrans %} - </div> -{% endblock %} diff --git a/Misc/NEWS.d/next/Documentation/2018-01-25-13-58-49.bpo-30607.4dXxiq.rst b/Misc/NEWS.d/next/Documentation/2018-01-25-13-58-49.bpo-30607.4dXxiq.rst new file mode 100644 index 000000000000..8ff3b7887810 --- /dev/null +++ b/Misc/NEWS.d/next/Documentation/2018-01-25-13-58-49.bpo-30607.4dXxiq.rst @@ -0,0 +1,2 @@ +Use the externalized ``python-docs-theme`` package when building the +documenation. From tjreedy at udel.edu Thu Mar 1 16:56:25 2018 From: tjreedy at udel.edu (Terry Reedy) Date: Thu, 1 Mar 2018 16:56:25 -0500 Subject: [Python-checkins] bpo-30607: Use external python-doc-theme (GH-2017) In-Reply-To: <3zslJW70nHzFqs4@mail.python.org> References: <3zslJW70nHzFqs4@mail.python.org> Message-ID: <a0491a8b-47a1-5410-a9d7-872b57e6c1a6@udel.edu> On 3/1/2018 4:03 PM, Ned Deily wrote: > https://github.com/python/cpython/commit/bf63e8d55fd2853df3bb99d66de7f428107aadb3 > commit: bf63e8d55fd2853df3bb99d66de7f428107aadb3 > branch: master > author: Jon Wayne Parrott <jonwayne at google.com> > committer: Ned Deily <nad at python.org> > date: 2018-03-01T16:02:50-05:00 > summary: > diff --git a/.travis.yml b/.travis.yml > index 292b9c687638..af3cdf54654d 100644 > --- a/.travis.yml > +++ b/.travis.yml > @@ -49,7 +49,8 @@ matrix: > - cd Doc > # Sphinx is pinned so that new versions that introduce new warnings won't suddenly cause build failures. > # (Updating the version is fine as long as no warnings are raised by doing so.) > - - python -m pip install sphinx~=1.6.1 blurb > + # The theme used by the docs is stored seperately, so we need to install that as well. /seperately/separately/ From solipsis at pitrou.net Fri Mar 2 04:14:09 2018 From: solipsis at pitrou.net (solipsis at pitrou.net) Date: Fri, 02 Mar 2018 09:14:09 +0000 Subject: [Python-checkins] Daily reference leaks (4243df51fe43): sum=4 Message-ID: <20180302091409.1.8D0AEFF7FC9A1F4C@psf.io> results for 4243df51fe43 on branch "default" -------------------------------------------- test_functools leaked [0, 3, 1] memory blocks, sum=4 test_multiprocessing_fork leaked [0, -2, 2] memory blocks, sum=0 Command line was: ['./python', '-m', 'test.regrtest', '-uall', '-R', '3:3:/home/psf-users/antoine/refleaks/reflogPujVAB', '--timeout', '7200'] From webhook-mailer at python.org Fri Mar 2 04:53:59 2018 From: webhook-mailer at python.org (Serhiy Storchaka) Date: Fri, 02 Mar 2018 09:53:59 -0000 Subject: [Python-checkins] bpo-32964: Reuse a testing implementation of the path protocol in tests. (#5930) Message-ID: <mailman.103.1519984440.2831.python-checkins@python.org> https://github.com/python/cpython/commit/b21d155f57d284aecf9092a9bd24258293965c2f commit: b21d155f57d284aecf9092a9bd24258293965c2f branch: master author: Serhiy Storchaka <storchaka at gmail.com> committer: GitHub <noreply at github.com> date: 2018-03-02T11:53:51+02:00 summary: bpo-32964: Reuse a testing implementation of the path protocol in tests. (#5930) files: M Doc/library/test.rst M Lib/test/support/__init__.py M Lib/test/test_compile.py M Lib/test/test_genericpath.py M Lib/test/test_io.py M Lib/test/test_ntpath.py M Lib/test/test_os.py M Lib/test/test_pathlib.py M Lib/test/test_posixpath.py M Lib/test/test_shutil.py M Lib/test/test_subprocess.py diff --git a/Doc/library/test.rst b/Doc/library/test.rst index a366fe2556b5..6041f529c16f 100644 --- a/Doc/library/test.rst +++ b/Doc/library/test.rst @@ -1293,6 +1293,13 @@ The :mod:`test.support` module defines the following classes: Class for logging support. +.. class:: FakePath(path) + + Simple :term:`path-like object`. It implements the :meth:`__fspath__` + method which just returns the *path* argument. If *path* is an exception, + it will be raised in :meth:`!__fspath__`. + + :mod:`test.support.script_helper` --- Utilities for the Python execution tests ============================================================================== diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index b4269f4a5b70..2c2f888352e8 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -2840,3 +2840,21 @@ def restore(self): def with_pymalloc(): import _testcapi return _testcapi.WITH_PYMALLOC + + +class FakePath: + """Simple implementing of the path protocol. + """ + def __init__(self, path): + self.path = path + + def __repr__(self): + return f'<FakePath {self.path!r}>' + + def __fspath__(self): + if (isinstance(self.path, BaseException) or + isinstance(self.path, type) and + issubclass(self.path, BaseException)): + raise self.path + else: + return self.path diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py index 4617a126abee..acebdbdc4636 100644 --- a/Lib/test/test_compile.py +++ b/Lib/test/test_compile.py @@ -6,7 +6,7 @@ import tempfile import types from test import support -from test.support import script_helper +from test.support import script_helper, FakePath class TestSpecifics(unittest.TestCase): @@ -663,13 +663,7 @@ def check_different_constants(const1, const2): def test_path_like_objects(self): # An implicit test for PyUnicode_FSDecoder(). - class PathLike: - def __init__(self, path): - self._path = path - def __fspath__(self): - return self._path - - compile("42", PathLike("test_compile_pathlike"), "single") + compile("42", FakePath("test_compile_pathlike"), "single") def test_stack_overflow(self): # bpo-31113: Stack overflow when compile a long sequence of diff --git a/Lib/test/test_genericpath.py b/Lib/test/test_genericpath.py index ad5a59c44ed7..9ed53902dc12 100644 --- a/Lib/test/test_genericpath.py +++ b/Lib/test/test_genericpath.py @@ -9,6 +9,7 @@ import warnings from test import support from test.support.script_helper import assert_python_ok +from test.support import FakePath def create_file(filename, data=b'foo'): @@ -493,18 +494,9 @@ def test_import(self): class PathLikeTests(unittest.TestCase): - class PathLike: - def __init__(self, path=''): - self.path = path - def __fspath__(self): - if isinstance(self.path, BaseException): - raise self.path - else: - return self.path - def setUp(self): self.file_name = support.TESTFN.lower() - self.file_path = self.PathLike(support.TESTFN) + self.file_path = FakePath(support.TESTFN) self.addCleanup(support.unlink, self.file_name) create_file(self.file_name, b"test_genericpath.PathLikeTests") diff --git a/Lib/test/test_io.py b/Lib/test/test_io.py index 4be5631ab446..343c5a444072 100644 --- a/Lib/test/test_io.py +++ b/Lib/test/test_io.py @@ -37,6 +37,7 @@ from itertools import cycle, count from test import support from test.support.script_helper import assert_python_ok, run_python_until_end +from test.support import FakePath import codecs import io # C implementation of io @@ -891,13 +892,6 @@ def read(self, size): self.assertEqual(bytes(buffer), b"12345") def test_fspath_support(self): - class PathLike: - def __init__(self, path): - self.path = path - - def __fspath__(self): - return self.path - def check_path_succeeds(path): with self.open(path, "w") as f: f.write("egg\n") @@ -905,16 +899,25 @@ def check_path_succeeds(path): with self.open(path, "r") as f: self.assertEqual(f.read(), "egg\n") - check_path_succeeds(PathLike(support.TESTFN)) - check_path_succeeds(PathLike(support.TESTFN.encode('utf-8'))) + check_path_succeeds(FakePath(support.TESTFN)) + check_path_succeeds(FakePath(support.TESTFN.encode('utf-8'))) + + with self.open(support.TESTFN, "w") as f: + bad_path = FakePath(f.fileno()) + with self.assertRaises(TypeError): + self.open(bad_path, 'w') - bad_path = PathLike(TypeError) + bad_path = FakePath(None) with self.assertRaises(TypeError): self.open(bad_path, 'w') + bad_path = FakePath(FloatingPointError) + with self.assertRaises(FloatingPointError): + self.open(bad_path, 'w') + # ensure that refcounting is correct with some error conditions with self.assertRaisesRegex(ValueError, 'read/write/append mode'): - self.open(PathLike(support.TESTFN), 'rwxa') + self.open(FakePath(support.TESTFN), 'rwxa') def test_RawIOBase_readall(self): # Exercise the default unlimited RawIOBase.read() and readall() diff --git a/Lib/test/test_ntpath.py b/Lib/test/test_ntpath.py index 1eec26b20013..2d48be844242 100644 --- a/Lib/test/test_ntpath.py +++ b/Lib/test/test_ntpath.py @@ -3,7 +3,7 @@ import sys import unittest import warnings -from test.support import TestFailed +from test.support import TestFailed, FakePath from test import support, test_genericpath from tempfile import TemporaryFile @@ -459,18 +459,9 @@ class PathLikeTests(unittest.TestCase): path = ntpath - class PathLike: - def __init__(self, path=''): - self.path = path - def __fspath__(self): - if isinstance(self.path, BaseException): - raise self.path - else: - return self.path - def setUp(self): self.file_name = support.TESTFN.lower() - self.file_path = self.PathLike(support.TESTFN) + self.file_path = FakePath(support.TESTFN) self.addCleanup(support.unlink, self.file_name) with open(self.file_name, 'xb', 0) as file: file.write(b"test_ntpath.PathLikeTests") @@ -485,7 +476,7 @@ def test_path_isabs(self): self.assertPathEqual(self.path.isabs) def test_path_join(self): - self.assertEqual(self.path.join('a', self.PathLike('b'), 'c'), + self.assertEqual(self.path.join('a', FakePath('b'), 'c'), self.path.join('a', 'b', 'c')) def test_path_split(self): diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index 77e4a008ae61..4f8a2a7e19d5 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -61,7 +61,7 @@ INT_MAX = PY_SSIZE_T_MAX = sys.maxsize from test.support.script_helper import assert_python_ok -from test.support import unix_shell +from test.support import unix_shell, FakePath root_in_posix = False @@ -85,21 +85,6 @@ def requires_os_func(name): return unittest.skipUnless(hasattr(os, name), 'requires os.%s' % name) -class _PathLike(os.PathLike): - - def __init__(self, path=""): - self.path = path - - def __str__(self): - return str(self.path) - - def __fspath__(self): - if isinstance(self.path, BaseException): - raise self.path - else: - return self.path - - def create_file(filename, content=b'content'): with open(filename, "xb", 0) as fp: fp.write(content) @@ -942,15 +927,14 @@ def test_walk_prune(self, walk_path=None): dirs.remove('SUB1') self.assertEqual(len(all), 2) - self.assertEqual(all[0], - (str(walk_path), ["SUB2"], ["tmp1"])) + self.assertEqual(all[0], (self.walk_path, ["SUB2"], ["tmp1"])) all[1][-1].sort() all[1][1].sort() self.assertEqual(all[1], self.sub2_tree) def test_file_like_path(self): - self.test_walk_prune(_PathLike(self.walk_path)) + self.test_walk_prune(FakePath(self.walk_path)) def test_walk_bottom_up(self): # Walk bottom-up. @@ -2288,7 +2272,7 @@ def test_getppid(self): def test_waitpid(self): args = [sys.executable, '-c', 'pass'] # Add an implicit test for PyUnicode_FSConverter(). - pid = os.spawnv(os.P_NOWAIT, _PathLike(args[0]), args) + pid = os.spawnv(os.P_NOWAIT, FakePath(args[0]), args) status = os.waitpid(pid, 0) self.assertEqual(status, (pid, 0)) @@ -3129,13 +3113,13 @@ def test_path_t_converter(self): bytes_fspath = bytes_filename = None else: bytes_filename = support.TESTFN.encode('ascii') - bytes_fspath = _PathLike(bytes_filename) - fd = os.open(_PathLike(str_filename), os.O_WRONLY|os.O_CREAT) + bytes_fspath = FakePath(bytes_filename) + fd = os.open(FakePath(str_filename), os.O_WRONLY|os.O_CREAT) self.addCleanup(support.unlink, support.TESTFN) self.addCleanup(os.close, fd) - int_fspath = _PathLike(fd) - str_fspath = _PathLike(str_filename) + int_fspath = FakePath(fd) + str_fspath = FakePath(str_filename) for name, allow_fd, extra_args, cleanup_fn in self.functions: with self.subTest(name=name): @@ -3540,16 +3524,16 @@ def test_return_string(self): def test_fsencode_fsdecode(self): for p in "path/like/object", b"path/like/object": - pathlike = _PathLike(p) + pathlike = FakePath(p) self.assertEqual(p, self.fspath(pathlike)) self.assertEqual(b"path/like/object", os.fsencode(pathlike)) self.assertEqual("path/like/object", os.fsdecode(pathlike)) def test_pathlike(self): - self.assertEqual('#feelthegil', self.fspath(_PathLike('#feelthegil'))) - self.assertTrue(issubclass(_PathLike, os.PathLike)) - self.assertTrue(isinstance(_PathLike(), os.PathLike)) + self.assertEqual('#feelthegil', self.fspath(FakePath('#feelthegil'))) + self.assertTrue(issubclass(FakePath, os.PathLike)) + self.assertTrue(isinstance(FakePath('x'), os.PathLike)) def test_garbage_in_exception_out(self): vapor = type('blah', (), {}) @@ -3561,14 +3545,14 @@ def test_argument_required(self): def test_bad_pathlike(self): # __fspath__ returns a value other than str or bytes. - self.assertRaises(TypeError, self.fspath, _PathLike(42)) + self.assertRaises(TypeError, self.fspath, FakePath(42)) # __fspath__ attribute that is not callable. c = type('foo', (), {}) c.__fspath__ = 1 self.assertRaises(TypeError, self.fspath, c()) # __fspath__ raises an exception. self.assertRaises(ZeroDivisionError, self.fspath, - _PathLike(ZeroDivisionError())) + FakePath(ZeroDivisionError())) class TimesTests(unittest.TestCase): diff --git a/Lib/test/test_pathlib.py b/Lib/test/test_pathlib.py index e56e0d20844e..53215550b435 100644 --- a/Lib/test/test_pathlib.py +++ b/Lib/test/test_pathlib.py @@ -11,7 +11,7 @@ from unittest import mock from test import support -TESTFN = support.TESTFN +from test.support import TESTFN, FakePath try: import grp, pwd @@ -191,18 +191,15 @@ def test_constructor_common(self): P = self.cls p = P('a') self.assertIsInstance(p, P) - class PathLike: - def __fspath__(self): - return "a/b/c" P('a', 'b', 'c') P('/a', 'b', 'c') P('a/b/c') P('/a/b/c') - P(PathLike()) + P(FakePath("a/b/c")) self.assertEqual(P(P('a')), P('a')) self.assertEqual(P(P('a'), 'b'), P('a/b')) self.assertEqual(P(P('a'), P('b')), P('a/b')) - self.assertEqual(P(P('a'), P('b'), P('c')), P(PathLike())) + self.assertEqual(P(P('a'), P('b'), P('c')), P(FakePath("a/b/c"))) def _check_str_subclass(self, *args): # Issue #21127: it should be possible to construct a PurePath object diff --git a/Lib/test/test_posixpath.py b/Lib/test/test_posixpath.py index 8a1e33b0c899..96b267cd45fd 100644 --- a/Lib/test/test_posixpath.py +++ b/Lib/test/test_posixpath.py @@ -4,6 +4,7 @@ import warnings from posixpath import realpath, abspath, dirname, basename from test import support, test_genericpath +from test.support import FakePath try: import posix @@ -600,18 +601,9 @@ class PathLikeTests(unittest.TestCase): path = posixpath - class PathLike: - def __init__(self, path=''): - self.path = path - def __fspath__(self): - if isinstance(self.path, BaseException): - raise self.path - else: - return self.path - def setUp(self): self.file_name = support.TESTFN.lower() - self.file_path = self.PathLike(support.TESTFN) + self.file_path = FakePath(support.TESTFN) self.addCleanup(support.unlink, self.file_name) with open(self.file_name, 'xb', 0) as file: file.write(b"test_posixpath.PathLikeTests") @@ -626,7 +618,7 @@ def test_path_isabs(self): self.assertPathEqual(self.path.isabs) def test_path_join(self): - self.assertEqual(self.path.join('a', self.PathLike('b'), 'c'), + self.assertEqual(self.path.join('a', FakePath('b'), 'c'), self.path.join('a', 'b', 'c')) def test_path_split(self): diff --git a/Lib/test/test_shutil.py b/Lib/test/test_shutil.py index f3cf43e50516..2cb2f14643e1 100644 --- a/Lib/test/test_shutil.py +++ b/Lib/test/test_shutil.py @@ -22,7 +22,7 @@ import zipfile from test import support -from test.support import TESTFN +from test.support import TESTFN, FakePath TESTFN2 = TESTFN + "2" @@ -1232,14 +1232,7 @@ def test_register_archive_format(self): def check_unpack_archive(self, format): self.check_unpack_archive_with_converter(format, lambda path: path) self.check_unpack_archive_with_converter(format, pathlib.Path) - - class MyPath: - def __init__(self, path): - self.path = path - def __fspath__(self): - return self.path - - self.check_unpack_archive_with_converter(format, MyPath) + self.check_unpack_archive_with_converter(format, FakePath) def check_unpack_archive_with_converter(self, format, converter): root_dir, base_dir = self._create_files() diff --git a/Lib/test/test_subprocess.py b/Lib/test/test_subprocess.py index 46cb5f117e84..ddee3b94fed9 100644 --- a/Lib/test/test_subprocess.py +++ b/Lib/test/test_subprocess.py @@ -17,6 +17,7 @@ import threading import gc import textwrap +from test.support import FakePath try: import ctypes @@ -359,12 +360,7 @@ def test_cwd(self): def test_cwd_with_pathlike(self): temp_dir = tempfile.gettempdir() temp_dir = self._normalize_cwd(temp_dir) - - class _PathLikeObj: - def __fspath__(self): - return temp_dir - - self._assert_cwd(temp_dir, sys.executable, cwd=_PathLikeObj()) + self._assert_cwd(temp_dir, sys.executable, cwd=FakePath(temp_dir)) @unittest.skipIf(mswindows, "pending resolution of issue #15533") def test_cwd_with_relative_arg(self): From webhook-mailer at python.org Fri Mar 2 05:17:57 2018 From: webhook-mailer at python.org (Miss Islington (bot)) Date: Fri, 02 Mar 2018 10:17:57 -0000 Subject: [Python-checkins] bpo-32964: Reuse a testing implementation of the path protocol in tests. (GH-5930) Message-ID: <mailman.104.1519985879.2831.python-checkins@python.org> https://github.com/python/cpython/commit/a13b65422a1078104e9f53ad41945ea380a80798 commit: a13b65422a1078104e9f53ad41945ea380a80798 branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: GitHub <noreply at github.com> date: 2018-03-02T02:17:51-08:00 summary: bpo-32964: Reuse a testing implementation of the path protocol in tests. (GH-5930) (cherry picked from commit b21d155f57d284aecf9092a9bd24258293965c2f) Co-authored-by: Serhiy Storchaka <storchaka at gmail.com> files: M Doc/library/test.rst M Lib/test/support/__init__.py M Lib/test/test_compile.py M Lib/test/test_genericpath.py M Lib/test/test_io.py M Lib/test/test_ntpath.py M Lib/test/test_os.py M Lib/test/test_pathlib.py M Lib/test/test_posixpath.py M Lib/test/test_shutil.py M Lib/test/test_subprocess.py diff --git a/Doc/library/test.rst b/Doc/library/test.rst index a366fe2556b5..6041f529c16f 100644 --- a/Doc/library/test.rst +++ b/Doc/library/test.rst @@ -1293,6 +1293,13 @@ The :mod:`test.support` module defines the following classes: Class for logging support. +.. class:: FakePath(path) + + Simple :term:`path-like object`. It implements the :meth:`__fspath__` + method which just returns the *path* argument. If *path* is an exception, + it will be raised in :meth:`!__fspath__`. + + :mod:`test.support.script_helper` --- Utilities for the Python execution tests ============================================================================== diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index 78298ca434a1..16cee96108a3 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -2840,3 +2840,21 @@ def restore(self): def with_pymalloc(): import _testcapi return _testcapi.WITH_PYMALLOC + + +class FakePath: + """Simple implementing of the path protocol. + """ + def __init__(self, path): + self.path = path + + def __repr__(self): + return f'<FakePath {self.path!r}>' + + def __fspath__(self): + if (isinstance(self.path, BaseException) or + isinstance(self.path, type) and + issubclass(self.path, BaseException)): + raise self.path + else: + return self.path diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py index 4617a126abee..acebdbdc4636 100644 --- a/Lib/test/test_compile.py +++ b/Lib/test/test_compile.py @@ -6,7 +6,7 @@ import tempfile import types from test import support -from test.support import script_helper +from test.support import script_helper, FakePath class TestSpecifics(unittest.TestCase): @@ -663,13 +663,7 @@ def check_different_constants(const1, const2): def test_path_like_objects(self): # An implicit test for PyUnicode_FSDecoder(). - class PathLike: - def __init__(self, path): - self._path = path - def __fspath__(self): - return self._path - - compile("42", PathLike("test_compile_pathlike"), "single") + compile("42", FakePath("test_compile_pathlike"), "single") def test_stack_overflow(self): # bpo-31113: Stack overflow when compile a long sequence of diff --git a/Lib/test/test_genericpath.py b/Lib/test/test_genericpath.py index ad5a59c44ed7..9ed53902dc12 100644 --- a/Lib/test/test_genericpath.py +++ b/Lib/test/test_genericpath.py @@ -9,6 +9,7 @@ import warnings from test import support from test.support.script_helper import assert_python_ok +from test.support import FakePath def create_file(filename, data=b'foo'): @@ -493,18 +494,9 @@ def test_import(self): class PathLikeTests(unittest.TestCase): - class PathLike: - def __init__(self, path=''): - self.path = path - def __fspath__(self): - if isinstance(self.path, BaseException): - raise self.path - else: - return self.path - def setUp(self): self.file_name = support.TESTFN.lower() - self.file_path = self.PathLike(support.TESTFN) + self.file_path = FakePath(support.TESTFN) self.addCleanup(support.unlink, self.file_name) create_file(self.file_name, b"test_genericpath.PathLikeTests") diff --git a/Lib/test/test_io.py b/Lib/test/test_io.py index 4be5631ab446..343c5a444072 100644 --- a/Lib/test/test_io.py +++ b/Lib/test/test_io.py @@ -37,6 +37,7 @@ from itertools import cycle, count from test import support from test.support.script_helper import assert_python_ok, run_python_until_end +from test.support import FakePath import codecs import io # C implementation of io @@ -891,13 +892,6 @@ def read(self, size): self.assertEqual(bytes(buffer), b"12345") def test_fspath_support(self): - class PathLike: - def __init__(self, path): - self.path = path - - def __fspath__(self): - return self.path - def check_path_succeeds(path): with self.open(path, "w") as f: f.write("egg\n") @@ -905,16 +899,25 @@ def check_path_succeeds(path): with self.open(path, "r") as f: self.assertEqual(f.read(), "egg\n") - check_path_succeeds(PathLike(support.TESTFN)) - check_path_succeeds(PathLike(support.TESTFN.encode('utf-8'))) + check_path_succeeds(FakePath(support.TESTFN)) + check_path_succeeds(FakePath(support.TESTFN.encode('utf-8'))) + + with self.open(support.TESTFN, "w") as f: + bad_path = FakePath(f.fileno()) + with self.assertRaises(TypeError): + self.open(bad_path, 'w') - bad_path = PathLike(TypeError) + bad_path = FakePath(None) with self.assertRaises(TypeError): self.open(bad_path, 'w') + bad_path = FakePath(FloatingPointError) + with self.assertRaises(FloatingPointError): + self.open(bad_path, 'w') + # ensure that refcounting is correct with some error conditions with self.assertRaisesRegex(ValueError, 'read/write/append mode'): - self.open(PathLike(support.TESTFN), 'rwxa') + self.open(FakePath(support.TESTFN), 'rwxa') def test_RawIOBase_readall(self): # Exercise the default unlimited RawIOBase.read() and readall() diff --git a/Lib/test/test_ntpath.py b/Lib/test/test_ntpath.py index 1eec26b20013..2d48be844242 100644 --- a/Lib/test/test_ntpath.py +++ b/Lib/test/test_ntpath.py @@ -3,7 +3,7 @@ import sys import unittest import warnings -from test.support import TestFailed +from test.support import TestFailed, FakePath from test import support, test_genericpath from tempfile import TemporaryFile @@ -459,18 +459,9 @@ class PathLikeTests(unittest.TestCase): path = ntpath - class PathLike: - def __init__(self, path=''): - self.path = path - def __fspath__(self): - if isinstance(self.path, BaseException): - raise self.path - else: - return self.path - def setUp(self): self.file_name = support.TESTFN.lower() - self.file_path = self.PathLike(support.TESTFN) + self.file_path = FakePath(support.TESTFN) self.addCleanup(support.unlink, self.file_name) with open(self.file_name, 'xb', 0) as file: file.write(b"test_ntpath.PathLikeTests") @@ -485,7 +476,7 @@ def test_path_isabs(self): self.assertPathEqual(self.path.isabs) def test_path_join(self): - self.assertEqual(self.path.join('a', self.PathLike('b'), 'c'), + self.assertEqual(self.path.join('a', FakePath('b'), 'c'), self.path.join('a', 'b', 'c')) def test_path_split(self): diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index 77e4a008ae61..4f8a2a7e19d5 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -61,7 +61,7 @@ INT_MAX = PY_SSIZE_T_MAX = sys.maxsize from test.support.script_helper import assert_python_ok -from test.support import unix_shell +from test.support import unix_shell, FakePath root_in_posix = False @@ -85,21 +85,6 @@ def requires_os_func(name): return unittest.skipUnless(hasattr(os, name), 'requires os.%s' % name) -class _PathLike(os.PathLike): - - def __init__(self, path=""): - self.path = path - - def __str__(self): - return str(self.path) - - def __fspath__(self): - if isinstance(self.path, BaseException): - raise self.path - else: - return self.path - - def create_file(filename, content=b'content'): with open(filename, "xb", 0) as fp: fp.write(content) @@ -942,15 +927,14 @@ def test_walk_prune(self, walk_path=None): dirs.remove('SUB1') self.assertEqual(len(all), 2) - self.assertEqual(all[0], - (str(walk_path), ["SUB2"], ["tmp1"])) + self.assertEqual(all[0], (self.walk_path, ["SUB2"], ["tmp1"])) all[1][-1].sort() all[1][1].sort() self.assertEqual(all[1], self.sub2_tree) def test_file_like_path(self): - self.test_walk_prune(_PathLike(self.walk_path)) + self.test_walk_prune(FakePath(self.walk_path)) def test_walk_bottom_up(self): # Walk bottom-up. @@ -2288,7 +2272,7 @@ def test_getppid(self): def test_waitpid(self): args = [sys.executable, '-c', 'pass'] # Add an implicit test for PyUnicode_FSConverter(). - pid = os.spawnv(os.P_NOWAIT, _PathLike(args[0]), args) + pid = os.spawnv(os.P_NOWAIT, FakePath(args[0]), args) status = os.waitpid(pid, 0) self.assertEqual(status, (pid, 0)) @@ -3129,13 +3113,13 @@ def test_path_t_converter(self): bytes_fspath = bytes_filename = None else: bytes_filename = support.TESTFN.encode('ascii') - bytes_fspath = _PathLike(bytes_filename) - fd = os.open(_PathLike(str_filename), os.O_WRONLY|os.O_CREAT) + bytes_fspath = FakePath(bytes_filename) + fd = os.open(FakePath(str_filename), os.O_WRONLY|os.O_CREAT) self.addCleanup(support.unlink, support.TESTFN) self.addCleanup(os.close, fd) - int_fspath = _PathLike(fd) - str_fspath = _PathLike(str_filename) + int_fspath = FakePath(fd) + str_fspath = FakePath(str_filename) for name, allow_fd, extra_args, cleanup_fn in self.functions: with self.subTest(name=name): @@ -3540,16 +3524,16 @@ def test_return_string(self): def test_fsencode_fsdecode(self): for p in "path/like/object", b"path/like/object": - pathlike = _PathLike(p) + pathlike = FakePath(p) self.assertEqual(p, self.fspath(pathlike)) self.assertEqual(b"path/like/object", os.fsencode(pathlike)) self.assertEqual("path/like/object", os.fsdecode(pathlike)) def test_pathlike(self): - self.assertEqual('#feelthegil', self.fspath(_PathLike('#feelthegil'))) - self.assertTrue(issubclass(_PathLike, os.PathLike)) - self.assertTrue(isinstance(_PathLike(), os.PathLike)) + self.assertEqual('#feelthegil', self.fspath(FakePath('#feelthegil'))) + self.assertTrue(issubclass(FakePath, os.PathLike)) + self.assertTrue(isinstance(FakePath('x'), os.PathLike)) def test_garbage_in_exception_out(self): vapor = type('blah', (), {}) @@ -3561,14 +3545,14 @@ def test_argument_required(self): def test_bad_pathlike(self): # __fspath__ returns a value other than str or bytes. - self.assertRaises(TypeError, self.fspath, _PathLike(42)) + self.assertRaises(TypeError, self.fspath, FakePath(42)) # __fspath__ attribute that is not callable. c = type('foo', (), {}) c.__fspath__ = 1 self.assertRaises(TypeError, self.fspath, c()) # __fspath__ raises an exception. self.assertRaises(ZeroDivisionError, self.fspath, - _PathLike(ZeroDivisionError())) + FakePath(ZeroDivisionError())) class TimesTests(unittest.TestCase): diff --git a/Lib/test/test_pathlib.py b/Lib/test/test_pathlib.py index e56e0d20844e..53215550b435 100644 --- a/Lib/test/test_pathlib.py +++ b/Lib/test/test_pathlib.py @@ -11,7 +11,7 @@ from unittest import mock from test import support -TESTFN = support.TESTFN +from test.support import TESTFN, FakePath try: import grp, pwd @@ -191,18 +191,15 @@ def test_constructor_common(self): P = self.cls p = P('a') self.assertIsInstance(p, P) - class PathLike: - def __fspath__(self): - return "a/b/c" P('a', 'b', 'c') P('/a', 'b', 'c') P('a/b/c') P('/a/b/c') - P(PathLike()) + P(FakePath("a/b/c")) self.assertEqual(P(P('a')), P('a')) self.assertEqual(P(P('a'), 'b'), P('a/b')) self.assertEqual(P(P('a'), P('b')), P('a/b')) - self.assertEqual(P(P('a'), P('b'), P('c')), P(PathLike())) + self.assertEqual(P(P('a'), P('b'), P('c')), P(FakePath("a/b/c"))) def _check_str_subclass(self, *args): # Issue #21127: it should be possible to construct a PurePath object diff --git a/Lib/test/test_posixpath.py b/Lib/test/test_posixpath.py index 8a1e33b0c899..96b267cd45fd 100644 --- a/Lib/test/test_posixpath.py +++ b/Lib/test/test_posixpath.py @@ -4,6 +4,7 @@ import warnings from posixpath import realpath, abspath, dirname, basename from test import support, test_genericpath +from test.support import FakePath try: import posix @@ -600,18 +601,9 @@ class PathLikeTests(unittest.TestCase): path = posixpath - class PathLike: - def __init__(self, path=''): - self.path = path - def __fspath__(self): - if isinstance(self.path, BaseException): - raise self.path - else: - return self.path - def setUp(self): self.file_name = support.TESTFN.lower() - self.file_path = self.PathLike(support.TESTFN) + self.file_path = FakePath(support.TESTFN) self.addCleanup(support.unlink, self.file_name) with open(self.file_name, 'xb', 0) as file: file.write(b"test_posixpath.PathLikeTests") @@ -626,7 +618,7 @@ def test_path_isabs(self): self.assertPathEqual(self.path.isabs) def test_path_join(self): - self.assertEqual(self.path.join('a', self.PathLike('b'), 'c'), + self.assertEqual(self.path.join('a', FakePath('b'), 'c'), self.path.join('a', 'b', 'c')) def test_path_split(self): diff --git a/Lib/test/test_shutil.py b/Lib/test/test_shutil.py index f3cf43e50516..2cb2f14643e1 100644 --- a/Lib/test/test_shutil.py +++ b/Lib/test/test_shutil.py @@ -22,7 +22,7 @@ import zipfile from test import support -from test.support import TESTFN +from test.support import TESTFN, FakePath TESTFN2 = TESTFN + "2" @@ -1232,14 +1232,7 @@ def test_register_archive_format(self): def check_unpack_archive(self, format): self.check_unpack_archive_with_converter(format, lambda path: path) self.check_unpack_archive_with_converter(format, pathlib.Path) - - class MyPath: - def __init__(self, path): - self.path = path - def __fspath__(self): - return self.path - - self.check_unpack_archive_with_converter(format, MyPath) + self.check_unpack_archive_with_converter(format, FakePath) def check_unpack_archive_with_converter(self, format, converter): root_dir, base_dir = self._create_files() diff --git a/Lib/test/test_subprocess.py b/Lib/test/test_subprocess.py index 46cb5f117e84..ddee3b94fed9 100644 --- a/Lib/test/test_subprocess.py +++ b/Lib/test/test_subprocess.py @@ -17,6 +17,7 @@ import threading import gc import textwrap +from test.support import FakePath try: import ctypes @@ -359,12 +360,7 @@ def test_cwd(self): def test_cwd_with_pathlike(self): temp_dir = tempfile.gettempdir() temp_dir = self._normalize_cwd(temp_dir) - - class _PathLikeObj: - def __fspath__(self): - return temp_dir - - self._assert_cwd(temp_dir, sys.executable, cwd=_PathLikeObj()) + self._assert_cwd(temp_dir, sys.executable, cwd=FakePath(temp_dir)) @unittest.skipIf(mswindows, "pending resolution of issue #15533") def test_cwd_with_relative_arg(self): From webhook-mailer at python.org Fri Mar 2 17:10:24 2018 From: webhook-mailer at python.org (Brett Cannon) Date: Fri, 02 Mar 2018 22:10:24 -0000 Subject: [Python-checkins] bpo-32963: Fix the tutorial to state source has a default encoding of ASCII (GH-5961) Message-ID: <mailman.105.1520028626.2831.python-checkins@python.org> https://github.com/python/cpython/commit/20003f9162631c47b79202316036d13d74484e4c commit: 20003f9162631c47b79202316036d13d74484e4c branch: 2.7 author: Brett Cannon <brettcannon at users.noreply.github.com> committer: GitHub <noreply at github.com> date: 2018-03-02T14:10:21-08:00 summary: bpo-32963: Fix the tutorial to state source has a default encoding of ASCII (GH-5961) files: M Doc/tutorial/interpreter.rst diff --git a/Doc/tutorial/interpreter.rst b/Doc/tutorial/interpreter.rst index 3bf9c62ca87c..6c8609fabaeb 100644 --- a/Doc/tutorial/interpreter.rst +++ b/Doc/tutorial/interpreter.rst @@ -126,14 +126,7 @@ The Interpreter and Its Environment Source Code Encoding -------------------- -By default, Python source files are treated as encoded in UTF-8. In that -encoding, characters of most languages in the world can be used simultaneously -in string literals, identifiers and comments --- although the standard library -only uses ASCII characters for identifiers, a convention that any portable code -should follow. To display all these characters properly, your editor must -recognize that the file is UTF-8, and it must use a font that supports all the -characters in the file. - +By default, Python source files are treated as encoded in ASCII. To declare an encoding other than the default one, a special comment line should be added as the *first* line of the file. The syntax is as follows:: From solipsis at pitrou.net Sat Mar 3 04:12:36 2018 From: solipsis at pitrou.net (solipsis at pitrou.net) Date: Sat, 03 Mar 2018 09:12:36 +0000 Subject: [Python-checkins] Daily reference leaks (4243df51fe43): sum=3 Message-ID: <20180303091236.1.66CA96248A8A5664@psf.io> results for 4243df51fe43 on branch "default" -------------------------------------------- test_collections leaked [-7, 8, 0] memory blocks, sum=1 test_functools leaked [0, 3, 1] memory blocks, sum=4 test_multiprocessing_forkserver leaked [-1, 1, -2] memory blocks, sum=-2 Command line was: ['./python', '-m', 'test.regrtest', '-uall', '-R', '3:3:/home/psf-users/antoine/refleaks/reflogeUD0VY', '--timeout', '7200'] From webhook-mailer at python.org Sat Mar 3 07:19:35 2018 From: webhook-mailer at python.org (Serhiy Storchaka) Date: Sat, 03 Mar 2018 12:19:35 -0000 Subject: [Python-checkins] [3.6] bpo-32964: Reuse a testing implementation of the path protocol in tests. (GH-5930). (GH-5958) Message-ID: <mailman.106.1520079579.2831.python-checkins@python.org> https://github.com/python/cpython/commit/fbdd075c64a5229dfa26632cf1b2cf2361dc5003 commit: fbdd075c64a5229dfa26632cf1b2cf2361dc5003 branch: 3.6 author: Serhiy Storchaka <storchaka at gmail.com> committer: GitHub <noreply at github.com> date: 2018-03-03T14:19:29+02:00 summary: [3.6] bpo-32964: Reuse a testing implementation of the path protocol in tests. (GH-5930). (GH-5958) (cherry picked from commit b21d155f57d284aecf9092a9bd24258293965c2f) Co-authored-by: Serhiy Storchaka <storchaka at gmail.com> files: M Doc/library/test.rst M Lib/test/support/__init__.py M Lib/test/test_compile.py M Lib/test/test_genericpath.py M Lib/test/test_io.py M Lib/test/test_ntpath.py M Lib/test/test_os.py M Lib/test/test_pathlib.py M Lib/test/test_posixpath.py M Lib/test/test_shutil.py M Lib/test/test_subprocess.py diff --git a/Doc/library/test.rst b/Doc/library/test.rst index 1a3f8f9c5dce..04d6cd87eac5 100644 --- a/Doc/library/test.rst +++ b/Doc/library/test.rst @@ -677,3 +677,10 @@ The :mod:`test.support` module defines the following classes: Class used to record warnings for unit tests. See documentation of :func:`check_warnings` above for more details. + + +.. class:: FakePath(path) + + Simple :term:`path-like object`. It implements the :meth:`__fspath__` + method which just returns the *path* argument. If *path* is an exception, + it will be raised in :meth:`!__fspath__`. diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index e4840e6a9618..74d435511ae3 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -2708,3 +2708,21 @@ def save(self): def restore(self): for signum, handler in self.handlers.items(): self.signal.signal(signum, handler) + + +class FakePath: + """Simple implementing of the path protocol. + """ + def __init__(self, path): + self.path = path + + def __repr__(self): + return f'<FakePath {self.path!r}>' + + def __fspath__(self): + if (isinstance(self.path, BaseException) or + isinstance(self.path, type) and + issubclass(self.path, BaseException)): + raise self.path + else: + return self.path diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py index 7fe34658e04c..c9812e46b486 100644 --- a/Lib/test/test_compile.py +++ b/Lib/test/test_compile.py @@ -6,7 +6,7 @@ import tempfile import types from test import support -from test.support import script_helper +from test.support import script_helper, FakePath class TestSpecifics(unittest.TestCase): @@ -670,13 +670,7 @@ def check_different_constants(const1, const2): def test_path_like_objects(self): # An implicit test for PyUnicode_FSDecoder(). - class PathLike: - def __init__(self, path): - self._path = path - def __fspath__(self): - return self._path - - compile("42", PathLike("test_compile_pathlike"), "single") + compile("42", FakePath("test_compile_pathlike"), "single") class TestStackSize(unittest.TestCase): diff --git a/Lib/test/test_genericpath.py b/Lib/test/test_genericpath.py index ad5a59c44ed7..9ed53902dc12 100644 --- a/Lib/test/test_genericpath.py +++ b/Lib/test/test_genericpath.py @@ -9,6 +9,7 @@ import warnings from test import support from test.support.script_helper import assert_python_ok +from test.support import FakePath def create_file(filename, data=b'foo'): @@ -493,18 +494,9 @@ def test_import(self): class PathLikeTests(unittest.TestCase): - class PathLike: - def __init__(self, path=''): - self.path = path - def __fspath__(self): - if isinstance(self.path, BaseException): - raise self.path - else: - return self.path - def setUp(self): self.file_name = support.TESTFN.lower() - self.file_path = self.PathLike(support.TESTFN) + self.file_path = FakePath(support.TESTFN) self.addCleanup(support.unlink, self.file_name) create_file(self.file_name, b"test_genericpath.PathLikeTests") diff --git a/Lib/test/test_io.py b/Lib/test/test_io.py index 6c30a44e62a6..3f3b3902ea2d 100644 --- a/Lib/test/test_io.py +++ b/Lib/test/test_io.py @@ -36,6 +36,7 @@ from itertools import cycle, count from test import support from test.support.script_helper import assert_python_ok, run_python_until_end +from test.support import FakePath import codecs import io # C implementation of io @@ -880,13 +881,6 @@ def read(self, size): self.assertEqual(bytes(buffer), b"12345") def test_fspath_support(self): - class PathLike: - def __init__(self, path): - self.path = path - - def __fspath__(self): - return self.path - def check_path_succeeds(path): with self.open(path, "w") as f: f.write("egg\n") @@ -894,16 +888,25 @@ def check_path_succeeds(path): with self.open(path, "r") as f: self.assertEqual(f.read(), "egg\n") - check_path_succeeds(PathLike(support.TESTFN)) - check_path_succeeds(PathLike(support.TESTFN.encode('utf-8'))) + check_path_succeeds(FakePath(support.TESTFN)) + check_path_succeeds(FakePath(support.TESTFN.encode('utf-8'))) + + with self.open(support.TESTFN, "w") as f: + bad_path = FakePath(f.fileno()) + with self.assertRaises(TypeError): + self.open(bad_path, 'w') - bad_path = PathLike(TypeError) + bad_path = FakePath(None) with self.assertRaises(TypeError): self.open(bad_path, 'w') + bad_path = FakePath(FloatingPointError) + with self.assertRaises(FloatingPointError): + self.open(bad_path, 'w') + # ensure that refcounting is correct with some error conditions with self.assertRaisesRegex(ValueError, 'read/write/append mode'): - self.open(PathLike(support.TESTFN), 'rwxa') + self.open(FakePath(support.TESTFN), 'rwxa') def test_RawIOBase_readall(self): # Exercise the default unlimited RawIOBase.read() and readall() diff --git a/Lib/test/test_ntpath.py b/Lib/test/test_ntpath.py index 90edb6d0806d..40761843f34c 100644 --- a/Lib/test/test_ntpath.py +++ b/Lib/test/test_ntpath.py @@ -3,7 +3,7 @@ import sys import unittest import warnings -from test.support import TestFailed +from test.support import TestFailed, FakePath from test import support, test_genericpath from tempfile import TemporaryFile @@ -456,18 +456,9 @@ class PathLikeTests(unittest.TestCase): path = ntpath - class PathLike: - def __init__(self, path=''): - self.path = path - def __fspath__(self): - if isinstance(self.path, BaseException): - raise self.path - else: - return self.path - def setUp(self): self.file_name = support.TESTFN.lower() - self.file_path = self.PathLike(support.TESTFN) + self.file_path = FakePath(support.TESTFN) self.addCleanup(support.unlink, self.file_name) with open(self.file_name, 'xb', 0) as file: file.write(b"test_ntpath.PathLikeTests") @@ -482,7 +473,7 @@ def test_path_isabs(self): self.assertPathEqual(self.path.isabs) def test_path_join(self): - self.assertEqual(self.path.join('a', self.PathLike('b'), 'c'), + self.assertEqual(self.path.join('a', FakePath('b'), 'c'), self.path.join('a', 'b', 'c')) def test_path_split(self): diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index 033e544ffed5..26d544ca9925 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -64,7 +64,7 @@ INT_MAX = PY_SSIZE_T_MAX = sys.maxsize from test.support.script_helper import assert_python_ok -from test.support import unix_shell +from test.support import unix_shell, FakePath root_in_posix = False @@ -94,21 +94,6 @@ def requires_os_func(name): return unittest.skipUnless(hasattr(os, name), 'requires os.%s' % name) -class _PathLike(os.PathLike): - - def __init__(self, path=""): - self.path = path - - def __str__(self): - return str(self.path) - - def __fspath__(self): - if isinstance(self.path, BaseException): - raise self.path - else: - return self.path - - def create_file(filename, content=b'content'): with open(filename, "xb", 0) as fp: fp.write(content) @@ -970,15 +955,14 @@ def test_walk_prune(self, walk_path=None): dirs.remove('SUB1') self.assertEqual(len(all), 2) - self.assertEqual(all[0], - (str(walk_path), ["SUB2"], ["tmp1"])) + self.assertEqual(all[0], (self.walk_path, ["SUB2"], ["tmp1"])) all[1][-1].sort() all[1][1].sort() self.assertEqual(all[1], self.sub2_tree) def test_file_like_path(self): - self.test_walk_prune(_PathLike(self.walk_path)) + self.test_walk_prune(FakePath(self.walk_path)) def test_walk_bottom_up(self): # Walk bottom-up. @@ -2294,7 +2278,7 @@ def test_getppid(self): def test_waitpid(self): args = [sys.executable, '-c', 'pass'] # Add an implicit test for PyUnicode_FSConverter(). - pid = os.spawnv(os.P_NOWAIT, _PathLike(args[0]), args) + pid = os.spawnv(os.P_NOWAIT, FakePath(args[0]), args) status = os.waitpid(pid, 0) self.assertEqual(status, (pid, 0)) @@ -3140,13 +3124,13 @@ def test_path_t_converter(self): bytes_fspath = bytes_filename = None else: bytes_filename = support.TESTFN.encode('ascii') - bytes_fspath = _PathLike(bytes_filename) - fd = os.open(_PathLike(str_filename), os.O_WRONLY|os.O_CREAT) + bytes_fspath = FakePath(bytes_filename) + fd = os.open(FakePath(str_filename), os.O_WRONLY|os.O_CREAT) self.addCleanup(support.unlink, support.TESTFN) self.addCleanup(os.close, fd) - int_fspath = _PathLike(fd) - str_fspath = _PathLike(str_filename) + int_fspath = FakePath(fd) + str_fspath = FakePath(str_filename) for name, allow_fd, extra_args, cleanup_fn in self.functions: with self.subTest(name=name): @@ -3519,16 +3503,16 @@ def test_return_string(self): def test_fsencode_fsdecode(self): for p in "path/like/object", b"path/like/object": - pathlike = _PathLike(p) + pathlike = FakePath(p) self.assertEqual(p, self.fspath(pathlike)) self.assertEqual(b"path/like/object", os.fsencode(pathlike)) self.assertEqual("path/like/object", os.fsdecode(pathlike)) def test_pathlike(self): - self.assertEqual('#feelthegil', self.fspath(_PathLike('#feelthegil'))) - self.assertTrue(issubclass(_PathLike, os.PathLike)) - self.assertTrue(isinstance(_PathLike(), os.PathLike)) + self.assertEqual('#feelthegil', self.fspath(FakePath('#feelthegil'))) + self.assertTrue(issubclass(FakePath, os.PathLike)) + self.assertTrue(isinstance(FakePath('x'), os.PathLike)) def test_garbage_in_exception_out(self): vapor = type('blah', (), {}) @@ -3540,14 +3524,14 @@ def test_argument_required(self): def test_bad_pathlike(self): # __fspath__ returns a value other than str or bytes. - self.assertRaises(TypeError, self.fspath, _PathLike(42)) + self.assertRaises(TypeError, self.fspath, FakePath(42)) # __fspath__ attribute that is not callable. c = type('foo', (), {}) c.__fspath__ = 1 self.assertRaises(TypeError, self.fspath, c()) # __fspath__ raises an exception. self.assertRaises(ZeroDivisionError, self.fspath, - _PathLike(ZeroDivisionError())) + FakePath(ZeroDivisionError())) # Only test if the C version is provided, otherwise TestPEP519 already tested # the pure Python implementation. diff --git a/Lib/test/test_pathlib.py b/Lib/test/test_pathlib.py index 6672c448f4fb..a4d2f8d98333 100644 --- a/Lib/test/test_pathlib.py +++ b/Lib/test/test_pathlib.py @@ -11,7 +11,7 @@ from unittest import mock from test import support -TESTFN = support.TESTFN +from test.support import TESTFN, FakePath try: import grp, pwd @@ -191,18 +191,15 @@ def test_constructor_common(self): P = self.cls p = P('a') self.assertIsInstance(p, P) - class PathLike: - def __fspath__(self): - return "a/b/c" P('a', 'b', 'c') P('/a', 'b', 'c') P('a/b/c') P('/a/b/c') - P(PathLike()) + P(FakePath("a/b/c")) self.assertEqual(P(P('a')), P('a')) self.assertEqual(P(P('a'), 'b'), P('a/b')) self.assertEqual(P(P('a'), P('b')), P('a/b')) - self.assertEqual(P(P('a'), P('b'), P('c')), P(PathLike())) + self.assertEqual(P(P('a'), P('b'), P('c')), P(FakePath("a/b/c"))) def _check_str_subclass(self, *args): # Issue #21127: it should be possible to construct a PurePath object diff --git a/Lib/test/test_posixpath.py b/Lib/test/test_posixpath.py index 8a1e33b0c899..96b267cd45fd 100644 --- a/Lib/test/test_posixpath.py +++ b/Lib/test/test_posixpath.py @@ -4,6 +4,7 @@ import warnings from posixpath import realpath, abspath, dirname, basename from test import support, test_genericpath +from test.support import FakePath try: import posix @@ -600,18 +601,9 @@ class PathLikeTests(unittest.TestCase): path = posixpath - class PathLike: - def __init__(self, path=''): - self.path = path - def __fspath__(self): - if isinstance(self.path, BaseException): - raise self.path - else: - return self.path - def setUp(self): self.file_name = support.TESTFN.lower() - self.file_path = self.PathLike(support.TESTFN) + self.file_path = FakePath(support.TESTFN) self.addCleanup(support.unlink, self.file_name) with open(self.file_name, 'xb', 0) as file: file.write(b"test_posixpath.PathLikeTests") @@ -626,7 +618,7 @@ def test_path_isabs(self): self.assertPathEqual(self.path.isabs) def test_path_join(self): - self.assertEqual(self.path.join('a', self.PathLike('b'), 'c'), + self.assertEqual(self.path.join('a', FakePath('b'), 'c'), self.path.join('a', 'b', 'c')) def test_path_split(self): diff --git a/Lib/test/test_shutil.py b/Lib/test/test_shutil.py index 95717a48ff10..588e7d27941c 100644 --- a/Lib/test/test_shutil.py +++ b/Lib/test/test_shutil.py @@ -10,6 +10,7 @@ import os.path import errno import functools +import pathlib import subprocess from contextlib import ExitStack from shutil import (make_archive, @@ -21,9 +22,10 @@ import tarfile import zipfile import warnings +import pathlib from test import support -from test.support import TESTFN, check_warnings, captured_stdout +from test.support import TESTFN, FakePath TESTFN2 = TESTFN + "2" @@ -1231,6 +1233,11 @@ def test_register_archive_format(self): self.assertNotIn('xxx', formats) def check_unpack_archive(self, format): + self.check_unpack_archive_with_converter(format, lambda path: path) + self.check_unpack_archive_with_converter(format, pathlib.Path) + self.check_unpack_archive_with_converter(format, FakePath) + + def check_unpack_archive_with_converter(self, format, converter): root_dir, base_dir = self._create_files() expected = rlistdir(root_dir) expected.remove('outer') diff --git a/Lib/test/test_subprocess.py b/Lib/test/test_subprocess.py index 5c6429cc0227..aca1411cdfb5 100644 --- a/Lib/test/test_subprocess.py +++ b/Lib/test/test_subprocess.py @@ -16,6 +16,7 @@ import shutil import gc import textwrap +from test.support import FakePath try: import ctypes @@ -363,12 +364,7 @@ def test_cwd(self): def test_cwd_with_pathlike(self): temp_dir = tempfile.gettempdir() temp_dir = self._normalize_cwd(temp_dir) - - class _PathLikeObj: - def __fspath__(self): - return temp_dir - - self._assert_cwd(temp_dir, sys.executable, cwd=_PathLikeObj()) + self._assert_cwd(temp_dir, sys.executable, cwd=FakePath(temp_dir)) @unittest.skipIf(mswindows, "pending resolution of issue #15533") def test_cwd_with_relative_arg(self): From webhook-mailer at python.org Sat Mar 3 11:43:58 2018 From: webhook-mailer at python.org (Andrew Svetlov) Date: Sat, 03 Mar 2018 16:43:58 -0000 Subject: [Python-checkins] Fix missing coroutine declaration in the asyncio documentation. (#5964) Message-ID: <mailman.107.1520095440.2831.python-checkins@python.org> https://github.com/python/cpython/commit/13cfd57dcf58485d6242fd8118c6ea4b10e29aab commit: 13cfd57dcf58485d6242fd8118c6ea4b10e29aab branch: master author: Joongi Kim <me at daybreaker.info> committer: Andrew Svetlov <andrew.svetlov at gmail.com> date: 2018-03-03T18:43:54+02:00 summary: Fix missing coroutine declaration in the asyncio documentation. (#5964) files: M Doc/library/asyncio-task.rst diff --git a/Doc/library/asyncio-task.rst b/Doc/library/asyncio-task.rst index 71dbe06c899f..485e1b843d08 100644 --- a/Doc/library/asyncio-task.rst +++ b/Doc/library/asyncio-task.rst @@ -682,7 +682,7 @@ Task functions This function is a :ref:`coroutine <coroutine>`. -.. function:: shield(arg, \*, loop=None) +.. coroutinefunction:: shield(arg, \*, loop=None) Wait for a future, shielding it from cancellation. From webhook-mailer at python.org Sat Mar 3 16:23:11 2018 From: webhook-mailer at python.org (Miss Islington (bot)) Date: Sat, 03 Mar 2018 21:23:11 -0000 Subject: [Python-checkins] Fix missing coroutine declaration in the asyncio documentation. (GH-5964) Message-ID: <mailman.108.1520112192.2831.python-checkins@python.org> https://github.com/python/cpython/commit/0f7cf7eb8a2c9a1766780ebf225a2f4c0640788e commit: 0f7cf7eb8a2c9a1766780ebf225a2f4c0640788e branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: GitHub <noreply at github.com> date: 2018-03-03T13:22:49-08:00 summary: Fix missing coroutine declaration in the asyncio documentation. (GH-5964) (cherry picked from commit 13cfd57dcf58485d6242fd8118c6ea4b10e29aab) Co-authored-by: Joongi Kim <me at daybreaker.info> files: M Doc/library/asyncio-task.rst diff --git a/Doc/library/asyncio-task.rst b/Doc/library/asyncio-task.rst index 71dbe06c899f..485e1b843d08 100644 --- a/Doc/library/asyncio-task.rst +++ b/Doc/library/asyncio-task.rst @@ -682,7 +682,7 @@ Task functions This function is a :ref:`coroutine <coroutine>`. -.. function:: shield(arg, \*, loop=None) +.. coroutinefunction:: shield(arg, \*, loop=None) Wait for a future, shielding it from cancellation. From webhook-mailer at python.org Sat Mar 3 16:23:36 2018 From: webhook-mailer at python.org (Miss Islington (bot)) Date: Sat, 03 Mar 2018 21:23:36 -0000 Subject: [Python-checkins] Fix missing coroutine declaration in the asyncio documentation. (GH-5964) Message-ID: <mailman.109.1520112218.2831.python-checkins@python.org> https://github.com/python/cpython/commit/bd92cfe632228ea4d6a1c0dc9506baa18d353dc1 commit: bd92cfe632228ea4d6a1c0dc9506baa18d353dc1 branch: 3.6 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: GitHub <noreply at github.com> date: 2018-03-03T13:23:31-08:00 summary: Fix missing coroutine declaration in the asyncio documentation. (GH-5964) (cherry picked from commit 13cfd57dcf58485d6242fd8118c6ea4b10e29aab) Co-authored-by: Joongi Kim <me at daybreaker.info> files: M Doc/library/asyncio-task.rst diff --git a/Doc/library/asyncio-task.rst b/Doc/library/asyncio-task.rst index cc8fffb0659f..5b801aaf8ecc 100644 --- a/Doc/library/asyncio-task.rst +++ b/Doc/library/asyncio-task.rst @@ -630,7 +630,7 @@ Task functions This function is a :ref:`coroutine <coroutine>`. -.. function:: shield(arg, \*, loop=None) +.. coroutinefunction:: shield(arg, \*, loop=None) Wait for a future, shielding it from cancellation. From webhook-mailer at python.org Sun Mar 4 00:33:37 2018 From: webhook-mailer at python.org (Benjamin Peterson) Date: Sun, 04 Mar 2018 05:33:37 -0000 Subject: [Python-checkins] bpo-32981: Fix catastrophic backtracking vulns (#5955) Message-ID: <mailman.110.1520141619.2831.python-checkins@python.org> https://github.com/python/cpython/commit/0e6c8ee2358a2e23117501826c008842acb835ac commit: 0e6c8ee2358a2e23117501826c008842acb835ac branch: master author: Jamie Davis <davisjam at vt.edu> committer: Benjamin Peterson <benjamin at python.org> date: 2018-03-03T21:33:32-08:00 summary: bpo-32981: Fix catastrophic backtracking vulns (#5955) * Prevent low-grade poplib REDOS (CVE-2018-1060) The regex to test a mail server's timestamp is susceptible to catastrophic backtracking on long evil responses from the server. Happily, the maximum length of malicious inputs is 2K thanks to a limit introduced in the fix for CVE-2013-1752. A 2KB evil response from the mail server would result in small slowdowns (milliseconds vs. microseconds) accumulated over many apop calls. This is a potential DOS vector via accumulated slowdowns. Replace it with a similar non-vulnerable regex. The new regex is RFC compliant. The old regex was non-compliant in edge cases. * Prevent difflib REDOS (CVE-2018-1061) The default regex for IS_LINE_JUNK is susceptible to catastrophic backtracking. This is a potential DOS vector. Replace it with an equivalent non-vulnerable regex. Also introduce unit and REDOS tests for difflib. Co-authored-by: Tim Peters <tim.peters at gmail.com> Co-authored-by: Christian Heimes <christian at python.org> files: A Misc/NEWS.d/next/Security/2018-03-02-10-24-52.bpo-32981.O_qDyj.rst M Lib/difflib.py M Lib/poplib.py M Lib/test/test_difflib.py M Lib/test/test_poplib.py M Misc/ACKS diff --git a/Lib/difflib.py b/Lib/difflib.py index 82964719dc83..043a169c28be 100644 --- a/Lib/difflib.py +++ b/Lib/difflib.py @@ -1083,7 +1083,7 @@ def _qformat(self, aline, bline, atags, btags): import re -def IS_LINE_JUNK(line, pat=re.compile(r"\s*#?\s*$").match): +def IS_LINE_JUNK(line, pat=re.compile(r"\s*(?:#\s*)?$").match): r""" Return 1 for ignorable line: iff `line` is blank or contains a single '#'. diff --git a/Lib/poplib.py b/Lib/poplib.py index 6bcfa5cfeba3..d8a62c034327 100644 --- a/Lib/poplib.py +++ b/Lib/poplib.py @@ -308,7 +308,7 @@ def rpop(self, user): return self._shortcmd('RPOP %s' % user) - timestamp = re.compile(br'\+OK.*(<[^>]+>)') + timestamp = re.compile(br'\+OK.[^<]*(<.*>)') def apop(self, user, password): """Authorisation diff --git a/Lib/test/test_difflib.py b/Lib/test/test_difflib.py index 156b523c38c1..aaefe6db0291 100644 --- a/Lib/test/test_difflib.py +++ b/Lib/test/test_difflib.py @@ -466,13 +466,33 @@ def _assert_type_error(self, msg, generator, *args): list(generator(*args)) self.assertEqual(msg, str(ctx.exception)) +class TestJunkAPIs(unittest.TestCase): + def test_is_line_junk_true(self): + for line in ['#', ' ', ' #', '# ', ' # ', '']: + self.assertTrue(difflib.IS_LINE_JUNK(line), repr(line)) + + def test_is_line_junk_false(self): + for line in ['##', ' ##', '## ', 'abc ', 'abc #', 'Mr. Moose is up!']: + self.assertFalse(difflib.IS_LINE_JUNK(line), repr(line)) + + def test_is_line_junk_REDOS(self): + evil_input = ('\t' * 1000000) + '##' + self.assertFalse(difflib.IS_LINE_JUNK(evil_input)) + + def test_is_character_junk_true(self): + for char in [' ', '\t']: + self.assertTrue(difflib.IS_CHARACTER_JUNK(char), repr(char)) + + def test_is_character_junk_false(self): + for char in ['a', '#', '\n', '\f', '\r', '\v']: + self.assertFalse(difflib.IS_CHARACTER_JUNK(char), repr(char)) def test_main(): difflib.HtmlDiff._default_prefix = 0 Doctests = doctest.DocTestSuite(difflib) run_unittest( TestWithAscii, TestAutojunk, TestSFpatches, TestSFbugs, - TestOutputFormat, TestBytes, Doctests) + TestOutputFormat, TestBytes, TestJunkAPIs, Doctests) if __name__ == '__main__': test_main() diff --git a/Lib/test/test_poplib.py b/Lib/test/test_poplib.py index fd0db798ee43..bf568bd77bff 100644 --- a/Lib/test/test_poplib.py +++ b/Lib/test/test_poplib.py @@ -308,9 +308,19 @@ def test_noop(self): def test_rpop(self): self.assertOK(self.client.rpop('foo')) - def test_apop(self): + def test_apop_normal(self): self.assertOK(self.client.apop('foo', 'dummypassword')) + def test_apop_REDOS(self): + # Replace welcome with very long evil welcome. + # NB The upper bound on welcome length is currently 2048. + # At this length, evil input makes each apop call take + # on the order of milliseconds instead of microseconds. + evil_welcome = b'+OK' + (b'<' * 1000000) + with test_support.swap_attr(self.client, 'welcome', evil_welcome): + # The evil welcome is invalid, so apop should throw. + self.assertRaises(poplib.error_proto, self.client.apop, 'a', 'kb') + def test_top(self): expected = (b'+OK 116 bytes', [b'From: postmaster at python.org', b'Content-Type: text/plain', diff --git a/Misc/ACKS b/Misc/ACKS index e2addfc210b7..d8179c8b03ab 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -356,6 +356,7 @@ Jonathan Dasteel Pierre-Yves David A. Jesse Jiryu Davis Jake Davis +Jamie (James C.) Davis Ratnadeep Debnath Merlijn van Deen John DeGood diff --git a/Misc/NEWS.d/next/Security/2018-03-02-10-24-52.bpo-32981.O_qDyj.rst b/Misc/NEWS.d/next/Security/2018-03-02-10-24-52.bpo-32981.O_qDyj.rst new file mode 100644 index 000000000000..9ebabb44f91e --- /dev/null +++ b/Misc/NEWS.d/next/Security/2018-03-02-10-24-52.bpo-32981.O_qDyj.rst @@ -0,0 +1,4 @@ +Regexes in difflib and poplib were vulnerable to catastrophic backtracking. +These regexes formed potential DOS vectors (REDOS). They have been +refactored. This resolves CVE-2018-1060 and CVE-2018-1061. +Patch by Jamie Davis. From webhook-mailer at python.org Sun Mar 4 00:55:10 2018 From: webhook-mailer at python.org (Benjamin Peterson) Date: Sun, 04 Mar 2018 05:55:10 -0000 Subject: [Python-checkins] bpo-32981: Fix catastrophic backtracking vulns (GH-5955) Message-ID: <mailman.111.1520142911.2831.python-checkins@python.org> https://github.com/python/cpython/commit/0902a2d6b2d1d9dbde36aeaaccf1788ceaa97143 commit: 0902a2d6b2d1d9dbde36aeaaccf1788ceaa97143 branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Benjamin Peterson <benjamin at python.org> date: 2018-03-03T21:55:07-08:00 summary: bpo-32981: Fix catastrophic backtracking vulns (GH-5955) * Prevent low-grade poplib REDOS (CVE-2018-1060) The regex to test a mail server's timestamp is susceptible to catastrophic backtracking on long evil responses from the server. Happily, the maximum length of malicious inputs is 2K thanks to a limit introduced in the fix for CVE-2013-1752. A 2KB evil response from the mail server would result in small slowdowns (milliseconds vs. microseconds) accumulated over many apop calls. This is a potential DOS vector via accumulated slowdowns. Replace it with a similar non-vulnerable regex. The new regex is RFC compliant. The old regex was non-compliant in edge cases. * Prevent difflib REDOS (CVE-2018-1061) The default regex for IS_LINE_JUNK is susceptible to catastrophic backtracking. This is a potential DOS vector. Replace it with an equivalent non-vulnerable regex. Also introduce unit and REDOS tests for difflib. Co-authored-by: Tim Peters <tim.peters at gmail.com> Co-authored-by: Christian Heimes <christian at python.org> Co-authored-by: Jamie Davis <davisjam at vt.edu> (cherry picked from commit 0e6c8ee2358a2e23117501826c008842acb835ac) files: A Misc/NEWS.d/next/Security/2018-03-02-10-24-52.bpo-32981.O_qDyj.rst M Lib/difflib.py M Lib/poplib.py M Lib/test/test_difflib.py M Lib/test/test_poplib.py M Misc/ACKS diff --git a/Lib/difflib.py b/Lib/difflib.py index 82964719dc83..043a169c28be 100644 --- a/Lib/difflib.py +++ b/Lib/difflib.py @@ -1083,7 +1083,7 @@ def _qformat(self, aline, bline, atags, btags): import re -def IS_LINE_JUNK(line, pat=re.compile(r"\s*#?\s*$").match): +def IS_LINE_JUNK(line, pat=re.compile(r"\s*(?:#\s*)?$").match): r""" Return 1 for ignorable line: iff `line` is blank or contains a single '#'. diff --git a/Lib/poplib.py b/Lib/poplib.py index 6bcfa5cfeba3..d8a62c034327 100644 --- a/Lib/poplib.py +++ b/Lib/poplib.py @@ -308,7 +308,7 @@ def rpop(self, user): return self._shortcmd('RPOP %s' % user) - timestamp = re.compile(br'\+OK.*(<[^>]+>)') + timestamp = re.compile(br'\+OK.[^<]*(<.*>)') def apop(self, user, password): """Authorisation diff --git a/Lib/test/test_difflib.py b/Lib/test/test_difflib.py index 156b523c38c1..aaefe6db0291 100644 --- a/Lib/test/test_difflib.py +++ b/Lib/test/test_difflib.py @@ -466,13 +466,33 @@ def _assert_type_error(self, msg, generator, *args): list(generator(*args)) self.assertEqual(msg, str(ctx.exception)) +class TestJunkAPIs(unittest.TestCase): + def test_is_line_junk_true(self): + for line in ['#', ' ', ' #', '# ', ' # ', '']: + self.assertTrue(difflib.IS_LINE_JUNK(line), repr(line)) + + def test_is_line_junk_false(self): + for line in ['##', ' ##', '## ', 'abc ', 'abc #', 'Mr. Moose is up!']: + self.assertFalse(difflib.IS_LINE_JUNK(line), repr(line)) + + def test_is_line_junk_REDOS(self): + evil_input = ('\t' * 1000000) + '##' + self.assertFalse(difflib.IS_LINE_JUNK(evil_input)) + + def test_is_character_junk_true(self): + for char in [' ', '\t']: + self.assertTrue(difflib.IS_CHARACTER_JUNK(char), repr(char)) + + def test_is_character_junk_false(self): + for char in ['a', '#', '\n', '\f', '\r', '\v']: + self.assertFalse(difflib.IS_CHARACTER_JUNK(char), repr(char)) def test_main(): difflib.HtmlDiff._default_prefix = 0 Doctests = doctest.DocTestSuite(difflib) run_unittest( TestWithAscii, TestAutojunk, TestSFpatches, TestSFbugs, - TestOutputFormat, TestBytes, Doctests) + TestOutputFormat, TestBytes, TestJunkAPIs, Doctests) if __name__ == '__main__': test_main() diff --git a/Lib/test/test_poplib.py b/Lib/test/test_poplib.py index fd0db798ee43..bf568bd77bff 100644 --- a/Lib/test/test_poplib.py +++ b/Lib/test/test_poplib.py @@ -308,9 +308,19 @@ def test_noop(self): def test_rpop(self): self.assertOK(self.client.rpop('foo')) - def test_apop(self): + def test_apop_normal(self): self.assertOK(self.client.apop('foo', 'dummypassword')) + def test_apop_REDOS(self): + # Replace welcome with very long evil welcome. + # NB The upper bound on welcome length is currently 2048. + # At this length, evil input makes each apop call take + # on the order of milliseconds instead of microseconds. + evil_welcome = b'+OK' + (b'<' * 1000000) + with test_support.swap_attr(self.client, 'welcome', evil_welcome): + # The evil welcome is invalid, so apop should throw. + self.assertRaises(poplib.error_proto, self.client.apop, 'a', 'kb') + def test_top(self): expected = (b'+OK 116 bytes', [b'From: postmaster at python.org', b'Content-Type: text/plain', diff --git a/Misc/ACKS b/Misc/ACKS index 4c0b0e5720e1..d598520e9385 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -356,6 +356,7 @@ Jonathan Dasteel Pierre-Yves David A. Jesse Jiryu Davis Jake Davis +Jamie (James C.) Davis Ratnadeep Debnath Merlijn van Deen John DeGood diff --git a/Misc/NEWS.d/next/Security/2018-03-02-10-24-52.bpo-32981.O_qDyj.rst b/Misc/NEWS.d/next/Security/2018-03-02-10-24-52.bpo-32981.O_qDyj.rst new file mode 100644 index 000000000000..9ebabb44f91e --- /dev/null +++ b/Misc/NEWS.d/next/Security/2018-03-02-10-24-52.bpo-32981.O_qDyj.rst @@ -0,0 +1,4 @@ +Regexes in difflib and poplib were vulnerable to catastrophic backtracking. +These regexes formed potential DOS vectors (REDOS). They have been +refactored. This resolves CVE-2018-1060 and CVE-2018-1061. +Patch by Jamie Davis. From webhook-mailer at python.org Sun Mar 4 01:06:04 2018 From: webhook-mailer at python.org (Benjamin Peterson) Date: Sun, 04 Mar 2018 06:06:04 -0000 Subject: [Python-checkins] closes bpo-32980 Remove _PyFrame_Init (GH-5965) Message-ID: <mailman.112.1520143564.2831.python-checkins@python.org> https://github.com/python/cpython/commit/7023644e0c310a3006c984318c2c111c468735b4 commit: 7023644e0c310a3006c984318c2c111c468735b4 branch: master author: Thomas Nyberg <tomnyberg at gmail.com> committer: Benjamin Peterson <benjamin at python.org> date: 2018-03-03T22:06:01-08:00 summary: closes bpo-32980 Remove _PyFrame_Init (GH-5965) files: M Include/pylifecycle.h M Objects/frameobject.c M Python/pylifecycle.c diff --git a/Include/pylifecycle.h b/Include/pylifecycle.h index 659c6df644e3..95dd55b05173 100644 --- a/Include/pylifecycle.h +++ b/Include/pylifecycle.h @@ -148,7 +148,6 @@ PyAPI_FUNC(int) _PySys_EndInit(PyObject *sysdict, _PyMainInterpreterConfig *conf PyAPI_FUNC(_PyInitError) _PyImport_Init(PyInterpreterState *interp); PyAPI_FUNC(void) _PyExc_Init(PyObject * bltinmod); PyAPI_FUNC(_PyInitError) _PyImportHooks_Init(void); -PyAPI_FUNC(int) _PyFrame_Init(void); PyAPI_FUNC(int) _PyFloat_Init(void); PyAPI_FUNC(int) PyByteArray_Init(void); PyAPI_FUNC(_PyInitError) _Py_HashRandomization_Init(const _PyCoreConfig *); diff --git a/Objects/frameobject.c b/Objects/frameobject.c index d308457b7547..b9f380d7b60d 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -552,14 +552,6 @@ PyTypeObject PyFrame_Type = { _Py_IDENTIFIER(__builtins__); -int _PyFrame_Init() -{ - /* Before, PyId___builtins__ was a string created explicitly in - this function. Now there is nothing to initialize anymore, but - the function is kept for backward compatibility. */ - return 1; -} - PyFrameObject* _Py_HOT_FUNCTION _PyFrame_New_NoTrack(PyThreadState *tstate, PyCodeObject *code, PyObject *globals, PyObject *locals) diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 5db586e15dff..a9b9470c7265 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -690,9 +690,6 @@ _Py_InitializeCore(const _PyCoreConfig *core_config) _Py_ReadyTypes(); - if (!_PyFrame_Init()) - return _Py_INIT_ERR("can't init frames"); - if (!_PyLong_Init()) return _Py_INIT_ERR("can't init longs"); From webhook-mailer at python.org Sun Mar 4 01:18:20 2018 From: webhook-mailer at python.org (Benjamin Peterson) Date: Sun, 04 Mar 2018 06:18:20 -0000 Subject: [Python-checkins] [2.7] bpo-32981: Fix catastrophic backtracking vulns (GH-5955) Message-ID: <mailman.113.1520144301.2831.python-checkins@python.org> https://github.com/python/cpython/commit/e052d40cea15f582b50947f7d906b39744dc62a2 commit: e052d40cea15f582b50947f7d906b39744dc62a2 branch: 2.7 author: Benjamin Peterson <benjamin at python.org> committer: GitHub <noreply at github.com> date: 2018-03-03T22:18:17-08:00 summary: [2.7] bpo-32981: Fix catastrophic backtracking vulns (GH-5955) * Prevent low-grade poplib REDOS (CVE-2018-1060) The regex to test a mail server's timestamp is susceptible to catastrophic backtracking on long evil responses from the server. Happily, the maximum length of malicious inputs is 2K thanks to a limit introduced in the fix for CVE-2013-1752. A 2KB evil response from the mail server would result in small slowdowns (milliseconds vs. microseconds) accumulated over many apop calls. This is a potential DOS vector via accumulated slowdowns. Replace it with a similar non-vulnerable regex. The new regex is RFC compliant. The old regex was non-compliant in edge cases. * Prevent difflib REDOS (CVE-2018-1061) The default regex for IS_LINE_JUNK is susceptible to catastrophic backtracking. This is a potential DOS vector. Replace it with an equivalent non-vulnerable regex. Also introduce unit and REDOS tests for difflib. Co-authored-by: Tim Peters <tim.peters at gmail.com> Co-authored-by: Christian Heimes <christian at python.org>. (cherry picked from commit 0e6c8ee2358a2e23117501826c008842acb835ac) files: A Misc/NEWS.d/next/Security/2018-03-02-10-24-52.bpo-32981.O_qDyj.rst M Lib/difflib.py M Lib/poplib.py M Lib/test/test_difflib.py M Lib/test/test_poplib.py M Misc/ACKS diff --git a/Lib/difflib.py b/Lib/difflib.py index 1c6fbdbedcb7..788a92df3f89 100644 --- a/Lib/difflib.py +++ b/Lib/difflib.py @@ -1103,7 +1103,7 @@ def _qformat(self, aline, bline, atags, btags): import re -def IS_LINE_JUNK(line, pat=re.compile(r"\s*#?\s*$").match): +def IS_LINE_JUNK(line, pat=re.compile(r"\s*(?:#\s*)?$").match): r""" Return 1 for ignorable line: iff `line` is blank or contains a single '#'. diff --git a/Lib/poplib.py b/Lib/poplib.py index b91e5f72d2ca..a238510b38fc 100644 --- a/Lib/poplib.py +++ b/Lib/poplib.py @@ -274,7 +274,7 @@ def rpop(self, user): return self._shortcmd('RPOP %s' % user) - timestamp = re.compile(r'\+OK.*(<[^>]+>)') + timestamp = re.compile(br'\+OK.[^<]*(<.*>)') def apop(self, user, secret): """Authorisation diff --git a/Lib/test/test_difflib.py b/Lib/test/test_difflib.py index 35f2c36ca70a..d8277b79b880 100644 --- a/Lib/test/test_difflib.py +++ b/Lib/test/test_difflib.py @@ -269,13 +269,33 @@ def test_range_format_context(self): self.assertEqual(fmt(3,6), '4,6') self.assertEqual(fmt(0,0), '0') +class TestJunkAPIs(unittest.TestCase): + def test_is_line_junk_true(self): + for line in ['#', ' ', ' #', '# ', ' # ', '']: + self.assertTrue(difflib.IS_LINE_JUNK(line), repr(line)) + + def test_is_line_junk_false(self): + for line in ['##', ' ##', '## ', 'abc ', 'abc #', 'Mr. Moose is up!']: + self.assertFalse(difflib.IS_LINE_JUNK(line), repr(line)) + + def test_is_line_junk_REDOS(self): + evil_input = ('\t' * 1000000) + '##' + self.assertFalse(difflib.IS_LINE_JUNK(evil_input)) + + def test_is_character_junk_true(self): + for char in [' ', '\t']: + self.assertTrue(difflib.IS_CHARACTER_JUNK(char), repr(char)) + + def test_is_character_junk_false(self): + for char in ['a', '#', '\n', '\f', '\r', '\v']: + self.assertFalse(difflib.IS_CHARACTER_JUNK(char), repr(char)) def test_main(): difflib.HtmlDiff._default_prefix = 0 Doctests = doctest.DocTestSuite(difflib) run_unittest( TestWithAscii, TestAutojunk, TestSFpatches, TestSFbugs, - TestOutputFormat, Doctests) + TestOutputFormat, TestJunkAPIs) if __name__ == '__main__': test_main() diff --git a/Lib/test/test_poplib.py b/Lib/test/test_poplib.py index 23d688724b95..d2143759ba66 100644 --- a/Lib/test/test_poplib.py +++ b/Lib/test/test_poplib.py @@ -211,6 +211,16 @@ def test_noop(self): def test_rpop(self): self.assertOK(self.client.rpop('foo')) + def test_apop_REDOS(self): + # Replace welcome with very long evil welcome. + # NB The upper bound on welcome length is currently 2048. + # At this length, evil input makes each apop call take + # on the order of milliseconds instead of microseconds. + evil_welcome = b'+OK' + (b'<' * 1000000) + with test_support.swap_attr(self.client, 'welcome', evil_welcome): + # The evil welcome is invalid, so apop should throw. + self.assertRaises(poplib.error_proto, self.client.apop, 'a', 'kb') + def test_top(self): expected = ('+OK 116 bytes', ['From: postmaster at python.org', 'Content-Type: text/plain', diff --git a/Misc/ACKS b/Misc/ACKS index 28b01e8cc7c5..ee2e4fc19393 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -318,6 +318,8 @@ Jonathan Dasteel Pierre-Yves David A. Jesse Jiryu Davis Jake Davis +Jamie (James C.) Davis +Ratnadeep Debnath Merlijn van Deen John DeGood Ned Deily diff --git a/Misc/NEWS.d/next/Security/2018-03-02-10-24-52.bpo-32981.O_qDyj.rst b/Misc/NEWS.d/next/Security/2018-03-02-10-24-52.bpo-32981.O_qDyj.rst new file mode 100644 index 000000000000..9ebabb44f91e --- /dev/null +++ b/Misc/NEWS.d/next/Security/2018-03-02-10-24-52.bpo-32981.O_qDyj.rst @@ -0,0 +1,4 @@ +Regexes in difflib and poplib were vulnerable to catastrophic backtracking. +These regexes formed potential DOS vectors (REDOS). They have been +refactored. This resolves CVE-2018-1060 and CVE-2018-1061. +Patch by Jamie Davis. From webhook-mailer at python.org Sun Mar 4 01:59:15 2018 From: webhook-mailer at python.org (Benjamin Peterson) Date: Sun, 04 Mar 2018 06:59:15 -0000 Subject: [Python-checkins] [3.6] bpo-32981: Fix catastrophic backtracking vulns (GH-5955) Message-ID: <mailman.114.1520146757.2831.python-checkins@python.org> https://github.com/python/cpython/commit/c9516754067d71fd7429a25ccfcb2141fc583523 commit: c9516754067d71fd7429a25ccfcb2141fc583523 branch: 3.6 author: Benjamin Peterson <benjamin at python.org> committer: GitHub <noreply at github.com> date: 2018-03-03T22:59:12-08:00 summary: [3.6] bpo-32981: Fix catastrophic backtracking vulns (GH-5955) * Prevent low-grade poplib REDOS (CVE-2018-1060) The regex to test a mail server's timestamp is susceptible to catastrophic backtracking on long evil responses from the server. Happily, the maximum length of malicious inputs is 2K thanks to a limit introduced in the fix for CVE-2013-1752. A 2KB evil response from the mail server would result in small slowdowns (milliseconds vs. microseconds) accumulated over many apop calls. This is a potential DOS vector via accumulated slowdowns. Replace it with a similar non-vulnerable regex. The new regex is RFC compliant. The old regex was non-compliant in edge cases. * Prevent difflib REDOS (CVE-2018-1061) The default regex for IS_LINE_JUNK is susceptible to catastrophic backtracking. This is a potential DOS vector. Replace it with an equivalent non-vulnerable regex. Also introduce unit and REDOS tests for difflib. Co-authored-by: Tim Peters <tim.peters at gmail.com> Co-authored-by: Christian Heimes <christian at python.org>. (cherry picked from commit 0e6c8ee2358a2e23117501826c008842acb835ac) files: A Misc/NEWS.d/next/Security/2018-03-02-10-24-52.bpo-32981.O_qDyj.rst M Lib/difflib.py M Lib/poplib.py M Lib/test/test_difflib.py M Lib/test/test_poplib.py M Misc/ACKS diff --git a/Lib/difflib.py b/Lib/difflib.py index 2095a5e517cb..72971d5b87a9 100644 --- a/Lib/difflib.py +++ b/Lib/difflib.py @@ -1083,7 +1083,7 @@ def _qformat(self, aline, bline, atags, btags): import re -def IS_LINE_JUNK(line, pat=re.compile(r"\s*#?\s*$").match): +def IS_LINE_JUNK(line, pat=re.compile(r"\s*(?:#\s*)?$").match): r""" Return 1 for ignorable line: iff `line` is blank or contains a single '#'. diff --git a/Lib/poplib.py b/Lib/poplib.py index 6bcfa5cfeba3..d8a62c034327 100644 --- a/Lib/poplib.py +++ b/Lib/poplib.py @@ -308,7 +308,7 @@ def rpop(self, user): return self._shortcmd('RPOP %s' % user) - timestamp = re.compile(br'\+OK.*(<[^>]+>)') + timestamp = re.compile(br'\+OK.[^<]*(<.*>)') def apop(self, user, password): """Authorisation diff --git a/Lib/test/test_difflib.py b/Lib/test/test_difflib.py index 156b523c38c1..aaefe6db0291 100644 --- a/Lib/test/test_difflib.py +++ b/Lib/test/test_difflib.py @@ -466,13 +466,33 @@ def _assert_type_error(self, msg, generator, *args): list(generator(*args)) self.assertEqual(msg, str(ctx.exception)) +class TestJunkAPIs(unittest.TestCase): + def test_is_line_junk_true(self): + for line in ['#', ' ', ' #', '# ', ' # ', '']: + self.assertTrue(difflib.IS_LINE_JUNK(line), repr(line)) + + def test_is_line_junk_false(self): + for line in ['##', ' ##', '## ', 'abc ', 'abc #', 'Mr. Moose is up!']: + self.assertFalse(difflib.IS_LINE_JUNK(line), repr(line)) + + def test_is_line_junk_REDOS(self): + evil_input = ('\t' * 1000000) + '##' + self.assertFalse(difflib.IS_LINE_JUNK(evil_input)) + + def test_is_character_junk_true(self): + for char in [' ', '\t']: + self.assertTrue(difflib.IS_CHARACTER_JUNK(char), repr(char)) + + def test_is_character_junk_false(self): + for char in ['a', '#', '\n', '\f', '\r', '\v']: + self.assertFalse(difflib.IS_CHARACTER_JUNK(char), repr(char)) def test_main(): difflib.HtmlDiff._default_prefix = 0 Doctests = doctest.DocTestSuite(difflib) run_unittest( TestWithAscii, TestAutojunk, TestSFpatches, TestSFbugs, - TestOutputFormat, TestBytes, Doctests) + TestOutputFormat, TestBytes, TestJunkAPIs, Doctests) if __name__ == '__main__': test_main() diff --git a/Lib/test/test_poplib.py b/Lib/test/test_poplib.py index 608eac57f35c..ca9bc6217509 100644 --- a/Lib/test/test_poplib.py +++ b/Lib/test/test_poplib.py @@ -303,9 +303,19 @@ def test_noop(self): def test_rpop(self): self.assertOK(self.client.rpop('foo')) - def test_apop(self): + def test_apop_normal(self): self.assertOK(self.client.apop('foo', 'dummypassword')) + def test_apop_REDOS(self): + # Replace welcome with very long evil welcome. + # NB The upper bound on welcome length is currently 2048. + # At this length, evil input makes each apop call take + # on the order of milliseconds instead of microseconds. + evil_welcome = b'+OK' + (b'<' * 1000000) + with test_support.swap_attr(self.client, 'welcome', evil_welcome): + # The evil welcome is invalid, so apop should throw. + self.assertRaises(poplib.error_proto, self.client.apop, 'a', 'kb') + def test_top(self): expected = (b'+OK 116 bytes', [b'From: postmaster at python.org', b'Content-Type: text/plain', diff --git a/Misc/ACKS b/Misc/ACKS index 75c9b3f0563d..9403e110675a 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -351,6 +351,7 @@ Jonathan Dasteel Pierre-Yves David A. Jesse Jiryu Davis Jake Davis +Jamie (James C.) Davis Merlijn van Deen John DeGood Ned Deily diff --git a/Misc/NEWS.d/next/Security/2018-03-02-10-24-52.bpo-32981.O_qDyj.rst b/Misc/NEWS.d/next/Security/2018-03-02-10-24-52.bpo-32981.O_qDyj.rst new file mode 100644 index 000000000000..9ebabb44f91e --- /dev/null +++ b/Misc/NEWS.d/next/Security/2018-03-02-10-24-52.bpo-32981.O_qDyj.rst @@ -0,0 +1,4 @@ +Regexes in difflib and poplib were vulnerable to catastrophic backtracking. +These regexes formed potential DOS vectors (REDOS). They have been +refactored. This resolves CVE-2018-1060 and CVE-2018-1061. +Patch by Jamie Davis. From solipsis at pitrou.net Sun Mar 4 04:10:59 2018 From: solipsis at pitrou.net (solipsis at pitrou.net) Date: Sun, 04 Mar 2018 09:10:59 +0000 Subject: [Python-checkins] Daily reference leaks (4243df51fe43): sum=0 Message-ID: <20180304091059.1.7A2CF341C0B27086@psf.io> results for 4243df51fe43 on branch "default" -------------------------------------------- test_collections leaked [-7, 1, 0] memory blocks, sum=-6 test_functools leaked [0, 3, 1] memory blocks, sum=4 test_multiprocessing_fork leaked [2, -1, 1] memory blocks, sum=2 Command line was: ['./python', '-m', 'test.regrtest', '-uall', '-R', '3:3:/home/psf-users/antoine/refleaks/reflogUgVuuC', '--timeout', '7200'] From webhook-mailer at python.org Sun Mar 4 05:41:52 2018 From: webhook-mailer at python.org (Serhiy Storchaka) Date: Sun, 04 Mar 2018 10:41:52 -0000 Subject: [Python-checkins] bpo-32857: Raise error when tkinter after_cancel() is called with None. (GH-5701) Message-ID: <mailman.115.1520160113.2831.python-checkins@python.org> https://github.com/python/cpython/commit/74382a3f175ac285cc924a73fd758e8dc3cc41bb commit: 74382a3f175ac285cc924a73fd758e8dc3cc41bb branch: master author: Cheryl Sabella <cheryl.sabella at gmail.com> committer: Serhiy Storchaka <storchaka at gmail.com> date: 2018-03-04T12:41:47+02:00 summary: bpo-32857: Raise error when tkinter after_cancel() is called with None. (GH-5701) files: A Misc/NEWS.d/next/Library/2018-02-16-14-37-14.bpo-32857.-XljAx.rst M Lib/tkinter/__init__.py M Lib/tkinter/test/test_tkinter/test_misc.py diff --git a/Lib/tkinter/__init__.py b/Lib/tkinter/__init__.py index deea791831ed..53bad3fa95ae 100644 --- a/Lib/tkinter/__init__.py +++ b/Lib/tkinter/__init__.py @@ -739,6 +739,7 @@ def after(self, ms, func=None, *args): if not func: # I'd rather use time.sleep(ms*0.001) self.tk.call('after', ms) + return None else: def callit(): try: @@ -762,11 +763,13 @@ def after_cancel(self, id): """Cancel scheduling of function identified with ID. Identifier returned by after or after_idle must be - given as first parameter.""" + given as first parameter. + """ + if not id: + raise ValueError('id must be a valid identifier returned from ' + 'after or after_idle') try: data = self.tk.call('after', 'info', id) - # In Tk 8.3, splitlist returns: (script, type) - # In Tk 8.4, splitlist may return (script, type) or (script,) script = self.tk.splitlist(data)[0] self.deletecommand(script) except TclError: diff --git a/Lib/tkinter/test/test_tkinter/test_misc.py b/Lib/tkinter/test/test_tkinter/test_misc.py index 9dc1e37547fc..1d1a3c29f6bc 100644 --- a/Lib/tkinter/test/test_tkinter/test_misc.py +++ b/Lib/tkinter/test/test_tkinter/test_misc.py @@ -48,6 +48,114 @@ def test_tk_setPalette(self): '^must specify a background color$', root.tk_setPalette, highlightColor='blue') + def test_after(self): + root = self.root + + def callback(start=0, step=1): + nonlocal count + count = start + step + + # Without function, sleeps for ms. + self.assertIsNone(root.after(1)) + + # Set up with callback with no args. + count = 0 + timer1 = root.after(0, callback) + self.assertIn(timer1, root.tk.call('after', 'info')) + (script, _) = root.tk.splitlist(root.tk.call('after', 'info', timer1)) + root.update() # Process all pending events. + self.assertEqual(count, 1) + with self.assertRaises(tkinter.TclError): + root.tk.call(script) + + # Set up with callback with args. + count = 0 + timer1 = root.after(0, callback, 42, 11) + root.update() # Process all pending events. + self.assertEqual(count, 53) + + # Cancel before called. + timer1 = root.after(1000, callback) + self.assertIn(timer1, root.tk.call('after', 'info')) + (script, _) = root.tk.splitlist(root.tk.call('after', 'info', timer1)) + root.after_cancel(timer1) # Cancel this event. + self.assertEqual(count, 53) + with self.assertRaises(tkinter.TclError): + root.tk.call(script) + + def test_after_idle(self): + root = self.root + + def callback(start=0, step=1): + nonlocal count + count = start + step + + # Set up with callback with no args. + count = 0 + idle1 = root.after_idle(callback) + self.assertIn(idle1, root.tk.call('after', 'info')) + (script, _) = root.tk.splitlist(root.tk.call('after', 'info', idle1)) + root.update_idletasks() # Process all pending events. + self.assertEqual(count, 1) + with self.assertRaises(tkinter.TclError): + root.tk.call(script) + + # Set up with callback with args. + count = 0 + idle1 = root.after_idle(callback, 42, 11) + root.update_idletasks() # Process all pending events. + self.assertEqual(count, 53) + + # Cancel before called. + idle1 = root.after_idle(callback) + self.assertIn(idle1, root.tk.call('after', 'info')) + (script, _) = root.tk.splitlist(root.tk.call('after', 'info', idle1)) + root.after_cancel(idle1) # Cancel this event. + self.assertEqual(count, 53) + with self.assertRaises(tkinter.TclError): + root.tk.call(script) + + def test_after_cancel(self): + root = self.root + + def callback(): + nonlocal count + count += 1 + + timer1 = root.after(5000, callback) + idle1 = root.after_idle(callback) + + # No value for id raises a ValueError. + with self.assertRaises(ValueError): + root.after_cancel(None) + + # Cancel timer event. + count = 0 + (script, _) = root.tk.splitlist(root.tk.call('after', 'info', timer1)) + root.tk.call(script) + self.assertEqual(count, 1) + root.after_cancel(timer1) + with self.assertRaises(tkinter.TclError): + root.tk.call(script) + self.assertEqual(count, 1) + with self.assertRaises(tkinter.TclError): + root.tk.call('after', 'info', timer1) + + # Cancel same event - nothing happens. + root.after_cancel(timer1) + + # Cancel idle event. + count = 0 + (script, _) = root.tk.splitlist(root.tk.call('after', 'info', idle1)) + root.tk.call(script) + self.assertEqual(count, 1) + root.after_cancel(idle1) + with self.assertRaises(tkinter.TclError): + root.tk.call(script) + self.assertEqual(count, 1) + with self.assertRaises(tkinter.TclError): + root.tk.call('after', 'info', idle1) + tests_gui = (MiscTest, ) diff --git a/Misc/NEWS.d/next/Library/2018-02-16-14-37-14.bpo-32857.-XljAx.rst b/Misc/NEWS.d/next/Library/2018-02-16-14-37-14.bpo-32857.-XljAx.rst new file mode 100644 index 000000000000..4ebbde4d1946 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-02-16-14-37-14.bpo-32857.-XljAx.rst @@ -0,0 +1 @@ +In :mod:`tkinter`, ``after_cancel(None)`` now raises a :exc:`ValueError` instead of canceling the first scheduled function. Patch by Cheryl Sabella. From webhook-mailer at python.org Sun Mar 4 06:42:48 2018 From: webhook-mailer at python.org (Miss Islington (bot)) Date: Sun, 04 Mar 2018 11:42:48 -0000 Subject: [Python-checkins] bpo-32857: Raise error when tkinter after_cancel() is called with None. (GH-5701) Message-ID: <mailman.116.1520163768.2831.python-checkins@python.org> https://github.com/python/cpython/commit/73a43960c7be50e136c5482404980175cb99f611 commit: 73a43960c7be50e136c5482404980175cb99f611 branch: 3.6 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: GitHub <noreply at github.com> date: 2018-03-04T03:42:41-08:00 summary: bpo-32857: Raise error when tkinter after_cancel() is called with None. (GH-5701) (cherry picked from commit 74382a3f175ac285cc924a73fd758e8dc3cc41bb) Co-authored-by: Cheryl Sabella <cheryl.sabella at gmail.com> files: A Misc/NEWS.d/next/Library/2018-02-16-14-37-14.bpo-32857.-XljAx.rst M Lib/tkinter/__init__.py M Lib/tkinter/test/test_tkinter/test_misc.py diff --git a/Lib/tkinter/__init__.py b/Lib/tkinter/__init__.py index deea791831ed..53bad3fa95ae 100644 --- a/Lib/tkinter/__init__.py +++ b/Lib/tkinter/__init__.py @@ -739,6 +739,7 @@ def after(self, ms, func=None, *args): if not func: # I'd rather use time.sleep(ms*0.001) self.tk.call('after', ms) + return None else: def callit(): try: @@ -762,11 +763,13 @@ def after_cancel(self, id): """Cancel scheduling of function identified with ID. Identifier returned by after or after_idle must be - given as first parameter.""" + given as first parameter. + """ + if not id: + raise ValueError('id must be a valid identifier returned from ' + 'after or after_idle') try: data = self.tk.call('after', 'info', id) - # In Tk 8.3, splitlist returns: (script, type) - # In Tk 8.4, splitlist may return (script, type) or (script,) script = self.tk.splitlist(data)[0] self.deletecommand(script) except TclError: diff --git a/Lib/tkinter/test/test_tkinter/test_misc.py b/Lib/tkinter/test/test_tkinter/test_misc.py index 9dc1e37547fc..1d1a3c29f6bc 100644 --- a/Lib/tkinter/test/test_tkinter/test_misc.py +++ b/Lib/tkinter/test/test_tkinter/test_misc.py @@ -48,6 +48,114 @@ def test_tk_setPalette(self): '^must specify a background color$', root.tk_setPalette, highlightColor='blue') + def test_after(self): + root = self.root + + def callback(start=0, step=1): + nonlocal count + count = start + step + + # Without function, sleeps for ms. + self.assertIsNone(root.after(1)) + + # Set up with callback with no args. + count = 0 + timer1 = root.after(0, callback) + self.assertIn(timer1, root.tk.call('after', 'info')) + (script, _) = root.tk.splitlist(root.tk.call('after', 'info', timer1)) + root.update() # Process all pending events. + self.assertEqual(count, 1) + with self.assertRaises(tkinter.TclError): + root.tk.call(script) + + # Set up with callback with args. + count = 0 + timer1 = root.after(0, callback, 42, 11) + root.update() # Process all pending events. + self.assertEqual(count, 53) + + # Cancel before called. + timer1 = root.after(1000, callback) + self.assertIn(timer1, root.tk.call('after', 'info')) + (script, _) = root.tk.splitlist(root.tk.call('after', 'info', timer1)) + root.after_cancel(timer1) # Cancel this event. + self.assertEqual(count, 53) + with self.assertRaises(tkinter.TclError): + root.tk.call(script) + + def test_after_idle(self): + root = self.root + + def callback(start=0, step=1): + nonlocal count + count = start + step + + # Set up with callback with no args. + count = 0 + idle1 = root.after_idle(callback) + self.assertIn(idle1, root.tk.call('after', 'info')) + (script, _) = root.tk.splitlist(root.tk.call('after', 'info', idle1)) + root.update_idletasks() # Process all pending events. + self.assertEqual(count, 1) + with self.assertRaises(tkinter.TclError): + root.tk.call(script) + + # Set up with callback with args. + count = 0 + idle1 = root.after_idle(callback, 42, 11) + root.update_idletasks() # Process all pending events. + self.assertEqual(count, 53) + + # Cancel before called. + idle1 = root.after_idle(callback) + self.assertIn(idle1, root.tk.call('after', 'info')) + (script, _) = root.tk.splitlist(root.tk.call('after', 'info', idle1)) + root.after_cancel(idle1) # Cancel this event. + self.assertEqual(count, 53) + with self.assertRaises(tkinter.TclError): + root.tk.call(script) + + def test_after_cancel(self): + root = self.root + + def callback(): + nonlocal count + count += 1 + + timer1 = root.after(5000, callback) + idle1 = root.after_idle(callback) + + # No value for id raises a ValueError. + with self.assertRaises(ValueError): + root.after_cancel(None) + + # Cancel timer event. + count = 0 + (script, _) = root.tk.splitlist(root.tk.call('after', 'info', timer1)) + root.tk.call(script) + self.assertEqual(count, 1) + root.after_cancel(timer1) + with self.assertRaises(tkinter.TclError): + root.tk.call(script) + self.assertEqual(count, 1) + with self.assertRaises(tkinter.TclError): + root.tk.call('after', 'info', timer1) + + # Cancel same event - nothing happens. + root.after_cancel(timer1) + + # Cancel idle event. + count = 0 + (script, _) = root.tk.splitlist(root.tk.call('after', 'info', idle1)) + root.tk.call(script) + self.assertEqual(count, 1) + root.after_cancel(idle1) + with self.assertRaises(tkinter.TclError): + root.tk.call(script) + self.assertEqual(count, 1) + with self.assertRaises(tkinter.TclError): + root.tk.call('after', 'info', idle1) + tests_gui = (MiscTest, ) diff --git a/Misc/NEWS.d/next/Library/2018-02-16-14-37-14.bpo-32857.-XljAx.rst b/Misc/NEWS.d/next/Library/2018-02-16-14-37-14.bpo-32857.-XljAx.rst new file mode 100644 index 000000000000..4ebbde4d1946 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-02-16-14-37-14.bpo-32857.-XljAx.rst @@ -0,0 +1 @@ +In :mod:`tkinter`, ``after_cancel(None)`` now raises a :exc:`ValueError` instead of canceling the first scheduled function. Patch by Cheryl Sabella. From webhook-mailer at python.org Sun Mar 4 07:00:36 2018 From: webhook-mailer at python.org (Miss Islington (bot)) Date: Sun, 04 Mar 2018 12:00:36 -0000 Subject: [Python-checkins] bpo-32857: Raise error when tkinter after_cancel() is called with None. (GH-5701) Message-ID: <mailman.117.1520164837.2831.python-checkins@python.org> https://github.com/python/cpython/commit/a5303dd9c263b337f02dda0038f2f5a10208140c commit: a5303dd9c263b337f02dda0038f2f5a10208140c branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: GitHub <noreply at github.com> date: 2018-03-04T04:00:33-08:00 summary: bpo-32857: Raise error when tkinter after_cancel() is called with None. (GH-5701) (cherry picked from commit 74382a3f175ac285cc924a73fd758e8dc3cc41bb) Co-authored-by: Cheryl Sabella <cheryl.sabella at gmail.com> files: A Misc/NEWS.d/next/Library/2018-02-16-14-37-14.bpo-32857.-XljAx.rst M Lib/tkinter/__init__.py M Lib/tkinter/test/test_tkinter/test_misc.py diff --git a/Lib/tkinter/__init__.py b/Lib/tkinter/__init__.py index deea791831ed..53bad3fa95ae 100644 --- a/Lib/tkinter/__init__.py +++ b/Lib/tkinter/__init__.py @@ -739,6 +739,7 @@ def after(self, ms, func=None, *args): if not func: # I'd rather use time.sleep(ms*0.001) self.tk.call('after', ms) + return None else: def callit(): try: @@ -762,11 +763,13 @@ def after_cancel(self, id): """Cancel scheduling of function identified with ID. Identifier returned by after or after_idle must be - given as first parameter.""" + given as first parameter. + """ + if not id: + raise ValueError('id must be a valid identifier returned from ' + 'after or after_idle') try: data = self.tk.call('after', 'info', id) - # In Tk 8.3, splitlist returns: (script, type) - # In Tk 8.4, splitlist may return (script, type) or (script,) script = self.tk.splitlist(data)[0] self.deletecommand(script) except TclError: diff --git a/Lib/tkinter/test/test_tkinter/test_misc.py b/Lib/tkinter/test/test_tkinter/test_misc.py index 9dc1e37547fc..1d1a3c29f6bc 100644 --- a/Lib/tkinter/test/test_tkinter/test_misc.py +++ b/Lib/tkinter/test/test_tkinter/test_misc.py @@ -48,6 +48,114 @@ def test_tk_setPalette(self): '^must specify a background color$', root.tk_setPalette, highlightColor='blue') + def test_after(self): + root = self.root + + def callback(start=0, step=1): + nonlocal count + count = start + step + + # Without function, sleeps for ms. + self.assertIsNone(root.after(1)) + + # Set up with callback with no args. + count = 0 + timer1 = root.after(0, callback) + self.assertIn(timer1, root.tk.call('after', 'info')) + (script, _) = root.tk.splitlist(root.tk.call('after', 'info', timer1)) + root.update() # Process all pending events. + self.assertEqual(count, 1) + with self.assertRaises(tkinter.TclError): + root.tk.call(script) + + # Set up with callback with args. + count = 0 + timer1 = root.after(0, callback, 42, 11) + root.update() # Process all pending events. + self.assertEqual(count, 53) + + # Cancel before called. + timer1 = root.after(1000, callback) + self.assertIn(timer1, root.tk.call('after', 'info')) + (script, _) = root.tk.splitlist(root.tk.call('after', 'info', timer1)) + root.after_cancel(timer1) # Cancel this event. + self.assertEqual(count, 53) + with self.assertRaises(tkinter.TclError): + root.tk.call(script) + + def test_after_idle(self): + root = self.root + + def callback(start=0, step=1): + nonlocal count + count = start + step + + # Set up with callback with no args. + count = 0 + idle1 = root.after_idle(callback) + self.assertIn(idle1, root.tk.call('after', 'info')) + (script, _) = root.tk.splitlist(root.tk.call('after', 'info', idle1)) + root.update_idletasks() # Process all pending events. + self.assertEqual(count, 1) + with self.assertRaises(tkinter.TclError): + root.tk.call(script) + + # Set up with callback with args. + count = 0 + idle1 = root.after_idle(callback, 42, 11) + root.update_idletasks() # Process all pending events. + self.assertEqual(count, 53) + + # Cancel before called. + idle1 = root.after_idle(callback) + self.assertIn(idle1, root.tk.call('after', 'info')) + (script, _) = root.tk.splitlist(root.tk.call('after', 'info', idle1)) + root.after_cancel(idle1) # Cancel this event. + self.assertEqual(count, 53) + with self.assertRaises(tkinter.TclError): + root.tk.call(script) + + def test_after_cancel(self): + root = self.root + + def callback(): + nonlocal count + count += 1 + + timer1 = root.after(5000, callback) + idle1 = root.after_idle(callback) + + # No value for id raises a ValueError. + with self.assertRaises(ValueError): + root.after_cancel(None) + + # Cancel timer event. + count = 0 + (script, _) = root.tk.splitlist(root.tk.call('after', 'info', timer1)) + root.tk.call(script) + self.assertEqual(count, 1) + root.after_cancel(timer1) + with self.assertRaises(tkinter.TclError): + root.tk.call(script) + self.assertEqual(count, 1) + with self.assertRaises(tkinter.TclError): + root.tk.call('after', 'info', timer1) + + # Cancel same event - nothing happens. + root.after_cancel(timer1) + + # Cancel idle event. + count = 0 + (script, _) = root.tk.splitlist(root.tk.call('after', 'info', idle1)) + root.tk.call(script) + self.assertEqual(count, 1) + root.after_cancel(idle1) + with self.assertRaises(tkinter.TclError): + root.tk.call(script) + self.assertEqual(count, 1) + with self.assertRaises(tkinter.TclError): + root.tk.call('after', 'info', idle1) + tests_gui = (MiscTest, ) diff --git a/Misc/NEWS.d/next/Library/2018-02-16-14-37-14.bpo-32857.-XljAx.rst b/Misc/NEWS.d/next/Library/2018-02-16-14-37-14.bpo-32857.-XljAx.rst new file mode 100644 index 000000000000..4ebbde4d1946 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-02-16-14-37-14.bpo-32857.-XljAx.rst @@ -0,0 +1 @@ +In :mod:`tkinter`, ``after_cancel(None)`` now raises a :exc:`ValueError` instead of canceling the first scheduled function. Patch by Cheryl Sabella. From webhook-mailer at python.org Sun Mar 4 18:07:00 2018 From: webhook-mailer at python.org (Ned Deily) Date: Sun, 04 Mar 2018 23:07:00 -0000 Subject: [Python-checkins] bpo-30147: Add re.escape changes to 3.7 What's New (GH-5978) Message-ID: <mailman.118.1520204821.2831.python-checkins@python.org> https://github.com/python/cpython/commit/18fd89246333bfa1b76c1623df689214f3ce2bf3 commit: 18fd89246333bfa1b76c1623df689214f3ce2bf3 branch: master author: Cheryl Sabella <cheryl.sabella at gmail.com> committer: Ned Deily <nad at python.org> date: 2018-03-04T18:06:57-05:00 summary: bpo-30147: Add re.escape changes to 3.7 What's New (GH-5978) files: M Doc/whatsnew/3.7.rst diff --git a/Doc/whatsnew/3.7.rst b/Doc/whatsnew/3.7.rst index 46f4f13aab47..9e65488e37d6 100644 --- a/Doc/whatsnew/3.7.rst +++ b/Doc/whatsnew/3.7.rst @@ -1214,6 +1214,10 @@ Changes in the Python API (Contributed by Serhiy Storchaka in :issue:`25054` and :issue:`32308`.) +* Change :func:`re.escape` to only escape regex special characters instead + of escaping all characters other than ASCII letters, numbers, and ``'_'``. + (Contributed by Serhiy Storchaka in :issue:`29995`.) + * :class:`tracemalloc.Traceback` frames are now sorted from oldest to most recent to be more consistent with :mod:`traceback`. (Contributed by Jesse Bakker in :issue:`32121`.) From webhook-mailer at python.org Sun Mar 4 18:22:35 2018 From: webhook-mailer at python.org (Ned Deily) Date: Sun, 04 Mar 2018 23:22:35 -0000 Subject: [Python-checkins] [3.7] bpo-30147: Add re.escape changes to 3.7 What's New (GH-5978) (GH-5979) Message-ID: <mailman.119.1520205756.2831.python-checkins@python.org> https://github.com/python/cpython/commit/f92478d57a6b4014dcc0882d43b534fae1e7b929 commit: f92478d57a6b4014dcc0882d43b534fae1e7b929 branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Ned Deily <nad at python.org> date: 2018-03-04T18:22:32-05:00 summary: [3.7] bpo-30147: Add re.escape changes to 3.7 What's New (GH-5978) (GH-5979) (cherry picked from commit 18fd89246333bfa1b76c1623df689214f3ce2bf3) Co-authored-by: Cheryl Sabella <cheryl.sabella at gmail.com> files: M Doc/whatsnew/3.7.rst diff --git a/Doc/whatsnew/3.7.rst b/Doc/whatsnew/3.7.rst index 46f4f13aab47..9e65488e37d6 100644 --- a/Doc/whatsnew/3.7.rst +++ b/Doc/whatsnew/3.7.rst @@ -1214,6 +1214,10 @@ Changes in the Python API (Contributed by Serhiy Storchaka in :issue:`25054` and :issue:`32308`.) +* Change :func:`re.escape` to only escape regex special characters instead + of escaping all characters other than ASCII letters, numbers, and ``'_'``. + (Contributed by Serhiy Storchaka in :issue:`29995`.) + * :class:`tracemalloc.Traceback` frames are now sorted from oldest to most recent to be more consistent with :mod:`traceback`. (Contributed by Jesse Bakker in :issue:`32121`.) From solipsis at pitrou.net Mon Mar 5 04:11:02 2018 From: solipsis at pitrou.net (solipsis at pitrou.net) Date: Mon, 05 Mar 2018 09:11:02 +0000 Subject: [Python-checkins] Daily reference leaks (4243df51fe43): sum=4 Message-ID: <20180305091102.1.C253719D3C3EC870@psf.io> results for 4243df51fe43 on branch "default" -------------------------------------------- test_collections leaked [0, 7, -7] memory blocks, sum=0 test_functools leaked [0, 3, 1] memory blocks, sum=4 Command line was: ['./python', '-m', 'test.regrtest', '-uall', '-R', '3:3:/home/psf-users/antoine/refleaks/reflogy0oxKo', '--timeout', '7200'] From webhook-mailer at python.org Mon Mar 5 11:02:49 2018 From: webhook-mailer at python.org (Terry Jan Reedy) Date: Mon, 05 Mar 2018 16:02:49 -0000 Subject: [Python-checkins] bpo-32984: IDLE - set __file__ for startup files (GH-5981) Message-ID: <mailman.120.1520265771.2831.python-checkins@python.org> https://github.com/python/cpython/commit/22c82be5df70c3d51e3f89b54fe1d4fb84728c1e commit: 22c82be5df70c3d51e3f89b54fe1d4fb84728c1e branch: master author: Terry Jan Reedy <tjreedy at udel.edu> committer: GitHub <noreply at github.com> date: 2018-03-05T11:02:46-05:00 summary: bpo-32984: IDLE - set __file__ for startup files (GH-5981) Like Python, IDLE optionally runs one startup file in the Shell window before presenting the first interactive input prompt. For IDLE, option -s runs a file named in environmental variable IDLESTARTUP or PYTHONSTARTUP; -r file runs file. Python sets __file__ to the startup file name before running the file and unsets it before the first prompt. IDLE now does the same when run normally, without the -n option. files: A Misc/NEWS.d/next/IDLE/2018-03-05-01-29-05.bpo-32984.NGjgT4.rst M Lib/idlelib/pyshell.py diff --git a/Lib/idlelib/pyshell.py b/Lib/idlelib/pyshell.py index 8b07d52cc487..ee1313161da3 100755 --- a/Lib/idlelib/pyshell.py +++ b/Lib/idlelib/pyshell.py @@ -635,6 +635,9 @@ def execfile(self, filename, source=None): if source is None: with tokenize.open(filename) as fp: source = fp.read() + if use_subprocess: + source = (f"__file__ = r'''{os.path.abspath(filename)}'''\n" + + source + "\ndel __file__") try: code = compile(source, filename, "exec") except (OverflowError, SyntaxError): diff --git a/Misc/NEWS.d/next/IDLE/2018-03-05-01-29-05.bpo-32984.NGjgT4.rst b/Misc/NEWS.d/next/IDLE/2018-03-05-01-29-05.bpo-32984.NGjgT4.rst new file mode 100644 index 000000000000..15d40b72caaf --- /dev/null +++ b/Misc/NEWS.d/next/IDLE/2018-03-05-01-29-05.bpo-32984.NGjgT4.rst @@ -0,0 +1,7 @@ +Set ``__file__`` while running a startup file. Like Python, IDLE optionally +runs one startup file in the Shell window before presenting the first interactive +input prompt. For IDLE, ``-s`` runs a file named in environmental variable + :envvar:`IDLESTARTUP` or :envvar:`PYTHONSTARTUP`; ``-r file`` runs +``file``. Python sets ``__file__`` to the startup file name before running the +file and unsets it before the first prompt. IDLE now does the same when run +normally, without the ``-n`` option. From webhook-mailer at python.org Mon Mar 5 11:49:21 2018 From: webhook-mailer at python.org (Miss Islington (bot)) Date: Mon, 05 Mar 2018 16:49:21 -0000 Subject: [Python-checkins] bpo-32984: IDLE - set __file__ for startup files (GH-5981) Message-ID: <mailman.121.1520268562.2831.python-checkins@python.org> https://github.com/python/cpython/commit/fd340bf9e308130736c76257ff9a697edbeb082d commit: fd340bf9e308130736c76257ff9a697edbeb082d branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: GitHub <noreply at github.com> date: 2018-03-05T08:49:15-08:00 summary: bpo-32984: IDLE - set __file__ for startup files (GH-5981) Like Python, IDLE optionally runs one startup file in the Shell window before presenting the first interactive input prompt. For IDLE, option -s runs a file named in environmental variable IDLESTARTUP or PYTHONSTARTUP; -r file runs file. Python sets __file__ to the startup file name before running the file and unsets it before the first prompt. IDLE now does the same when run normally, without the -n option. (cherry picked from commit 22c82be5df70c3d51e3f89b54fe1d4fb84728c1e) Co-authored-by: Terry Jan Reedy <tjreedy at udel.edu> files: A Misc/NEWS.d/next/IDLE/2018-03-05-01-29-05.bpo-32984.NGjgT4.rst M Lib/idlelib/pyshell.py diff --git a/Lib/idlelib/pyshell.py b/Lib/idlelib/pyshell.py index 8b07d52cc487..ee1313161da3 100755 --- a/Lib/idlelib/pyshell.py +++ b/Lib/idlelib/pyshell.py @@ -635,6 +635,9 @@ def execfile(self, filename, source=None): if source is None: with tokenize.open(filename) as fp: source = fp.read() + if use_subprocess: + source = (f"__file__ = r'''{os.path.abspath(filename)}'''\n" + + source + "\ndel __file__") try: code = compile(source, filename, "exec") except (OverflowError, SyntaxError): diff --git a/Misc/NEWS.d/next/IDLE/2018-03-05-01-29-05.bpo-32984.NGjgT4.rst b/Misc/NEWS.d/next/IDLE/2018-03-05-01-29-05.bpo-32984.NGjgT4.rst new file mode 100644 index 000000000000..15d40b72caaf --- /dev/null +++ b/Misc/NEWS.d/next/IDLE/2018-03-05-01-29-05.bpo-32984.NGjgT4.rst @@ -0,0 +1,7 @@ +Set ``__file__`` while running a startup file. Like Python, IDLE optionally +runs one startup file in the Shell window before presenting the first interactive +input prompt. For IDLE, ``-s`` runs a file named in environmental variable + :envvar:`IDLESTARTUP` or :envvar:`PYTHONSTARTUP`; ``-r file`` runs +``file``. Python sets ``__file__`` to the startup file name before running the +file and unsets it before the first prompt. IDLE now does the same when run +normally, without the ``-n`` option. From webhook-mailer at python.org Mon Mar 5 14:23:11 2018 From: webhook-mailer at python.org (Miss Islington (bot)) Date: Mon, 05 Mar 2018 19:23:11 -0000 Subject: [Python-checkins] bpo-32984: IDLE - set __file__ for startup files (GH-5981) Message-ID: <mailman.122.1520277793.2831.python-checkins@python.org> https://github.com/python/cpython/commit/6935a511670797a3aaebdf96aad3dcff66baa76e commit: 6935a511670797a3aaebdf96aad3dcff66baa76e branch: 3.6 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: GitHub <noreply at github.com> date: 2018-03-05T11:23:08-08:00 summary: bpo-32984: IDLE - set __file__ for startup files (GH-5981) Like Python, IDLE optionally runs one startup file in the Shell window before presenting the first interactive input prompt. For IDLE, option -s runs a file named in environmental variable IDLESTARTUP or PYTHONSTARTUP; -r file runs file. Python sets __file__ to the startup file name before running the file and unsets it before the first prompt. IDLE now does the same when run normally, without the -n option. (cherry picked from commit 22c82be5df70c3d51e3f89b54fe1d4fb84728c1e) Co-authored-by: Terry Jan Reedy <tjreedy at udel.edu> files: A Misc/NEWS.d/next/IDLE/2018-03-05-01-29-05.bpo-32984.NGjgT4.rst M Lib/idlelib/pyshell.py diff --git a/Lib/idlelib/pyshell.py b/Lib/idlelib/pyshell.py index 8b07d52cc487..ee1313161da3 100755 --- a/Lib/idlelib/pyshell.py +++ b/Lib/idlelib/pyshell.py @@ -635,6 +635,9 @@ def execfile(self, filename, source=None): if source is None: with tokenize.open(filename) as fp: source = fp.read() + if use_subprocess: + source = (f"__file__ = r'''{os.path.abspath(filename)}'''\n" + + source + "\ndel __file__") try: code = compile(source, filename, "exec") except (OverflowError, SyntaxError): diff --git a/Misc/NEWS.d/next/IDLE/2018-03-05-01-29-05.bpo-32984.NGjgT4.rst b/Misc/NEWS.d/next/IDLE/2018-03-05-01-29-05.bpo-32984.NGjgT4.rst new file mode 100644 index 000000000000..15d40b72caaf --- /dev/null +++ b/Misc/NEWS.d/next/IDLE/2018-03-05-01-29-05.bpo-32984.NGjgT4.rst @@ -0,0 +1,7 @@ +Set ``__file__`` while running a startup file. Like Python, IDLE optionally +runs one startup file in the Shell window before presenting the first interactive +input prompt. For IDLE, ``-s`` runs a file named in environmental variable + :envvar:`IDLESTARTUP` or :envvar:`PYTHONSTARTUP`; ``-r file`` runs +``file``. Python sets ``__file__`` to the startup file name before running the +file and unsets it before the first prompt. IDLE now does the same when run +normally, without the ``-n`` option. From webhook-mailer at python.org Mon Mar 5 15:37:21 2018 From: webhook-mailer at python.org (Barry Warsaw) Date: Mon, 05 Mar 2018 20:37:21 -0000 Subject: [Python-checkins] Add What's New for issues 32303 and 32305 (GH-5994) Message-ID: <mailman.123.1520282242.2831.python-checkins@python.org> https://github.com/python/cpython/commit/4c19b95734faee4c390c1d0569dc876980c33d2c commit: 4c19b95734faee4c390c1d0569dc876980c33d2c branch: master author: Barry Warsaw <barry at python.org> committer: GitHub <noreply at github.com> date: 2018-03-05T12:37:12-08:00 summary: Add What's New for issues 32303 and 32305 (GH-5994) files: M Doc/whatsnew/3.7.rst diff --git a/Doc/whatsnew/3.7.rst b/Doc/whatsnew/3.7.rst index 9e65488e37d6..76e1f7b36b0b 100644 --- a/Doc/whatsnew/3.7.rst +++ b/Doc/whatsnew/3.7.rst @@ -958,6 +958,14 @@ Other CPython Implementation Changes by setting the new ``f_trace_opcodes`` attribute to :const:`True` on the frame being traced. (Contributed by Nick Coghlan in :issue:`31344`.) +* Fixed some consistency problems with namespace package module attributes. + Namespace module objects now have an ``__file__`` that is set to ``None`` + (previously unset), and their ``__spec__.origin`` is also set to ``None`` + (previously the string ``"namespace"``). See :issue:`32305`. Also, the + namespace module object's ``__spec__.loader`` is set to the same value as + ``__loader__`` (previously, the former was set to ``None``). See + :issue:`32303`. + Deprecated ========== From webhook-mailer at python.org Mon Mar 5 15:51:59 2018 From: webhook-mailer at python.org (Barry Warsaw) Date: Mon, 05 Mar 2018 20:51:59 -0000 Subject: [Python-checkins] Add What's New for issues 32303 and 32305 (GH-5994) (GH-5995) Message-ID: <mailman.124.1520283121.2831.python-checkins@python.org> https://github.com/python/cpython/commit/063c637cdbe4528b175f7d6122fa29faf66707ee commit: 063c637cdbe4528b175f7d6122fa29faf66707ee branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Barry Warsaw <barry at python.org> date: 2018-03-05T12:51:56-08:00 summary: Add What's New for issues 32303 and 32305 (GH-5994) (GH-5995) (cherry picked from commit 4c19b95734faee4c390c1d0569dc876980c33d2c) Co-authored-by: Barry Warsaw <barry at python.org> files: M Doc/whatsnew/3.7.rst diff --git a/Doc/whatsnew/3.7.rst b/Doc/whatsnew/3.7.rst index 9e65488e37d6..76e1f7b36b0b 100644 --- a/Doc/whatsnew/3.7.rst +++ b/Doc/whatsnew/3.7.rst @@ -958,6 +958,14 @@ Other CPython Implementation Changes by setting the new ``f_trace_opcodes`` attribute to :const:`True` on the frame being traced. (Contributed by Nick Coghlan in :issue:`31344`.) +* Fixed some consistency problems with namespace package module attributes. + Namespace module objects now have an ``__file__`` that is set to ``None`` + (previously unset), and their ``__spec__.origin`` is also set to ``None`` + (previously the string ``"namespace"``). See :issue:`32305`. Also, the + namespace module object's ``__spec__.loader`` is set to the same value as + ``__loader__`` (previously, the former was set to ``None``). See + :issue:`32303`. + Deprecated ========== From webhook-mailer at python.org Mon Mar 5 17:26:18 2018 From: webhook-mailer at python.org (Steve Dower) Date: Mon, 05 Mar 2018 22:26:18 -0000 Subject: [Python-checkins] bpo-33001: Prevent buffer overrun in os.symlink (GH-5989) Message-ID: <mailman.125.1520288779.2831.python-checkins@python.org> https://github.com/python/cpython/commit/6921e73e33edc3c61bc2d78ed558eaa22a89a564 commit: 6921e73e33edc3c61bc2d78ed558eaa22a89a564 branch: master author: Steve Dower <steve.dower at microsoft.com> committer: GitHub <noreply at github.com> date: 2018-03-05T14:26:08-08:00 summary: bpo-33001: Prevent buffer overrun in os.symlink (GH-5989) files: A Misc/NEWS.d/next/Security/2018-03-05-10-09-51.bpo-33001.elj4Aa.rst M Lib/test/test_os.py M Modules/posixmodule.c diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index 4f8a2a7e19d5..e509188243f6 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -2164,6 +2164,40 @@ def test_29248(self): target = os.readlink(r'C:\Users\All Users') self.assertTrue(os.path.samefile(target, r'C:\ProgramData')) + def test_buffer_overflow(self): + # Older versions would have a buffer overflow when detecting + # whether a link source was a directory. This test ensures we + # no longer crash, but does not otherwise validate the behavior + segment = 'X' * 27 + path = os.path.join(*[segment] * 10) + test_cases = [ + # overflow with absolute src + ('\\' + path, segment), + # overflow dest with relative src + (segment, path), + # overflow when joining src + (path[:180], path[:180]), + ] + for src, dest in test_cases: + try: + os.symlink(src, dest) + except FileNotFoundError: + pass + else: + try: + os.remove(dest) + except OSError: + pass + # Also test with bytes, since that is a separate code path. + try: + os.symlink(os.fsencode(src), os.fsencode(dest)) + except FileNotFoundError: + pass + else: + try: + os.remove(dest) + except OSError: + pass @unittest.skipUnless(sys.platform == "win32", "Win32 specific tests") class Win32JunctionTests(unittest.TestCase): diff --git a/Misc/NEWS.d/next/Security/2018-03-05-10-09-51.bpo-33001.elj4Aa.rst b/Misc/NEWS.d/next/Security/2018-03-05-10-09-51.bpo-33001.elj4Aa.rst new file mode 100644 index 000000000000..2acbac9e1af6 --- /dev/null +++ b/Misc/NEWS.d/next/Security/2018-03-05-10-09-51.bpo-33001.elj4Aa.rst @@ -0,0 +1 @@ +Minimal fix to prevent buffer overrun in os.symlink on Windows diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 6bba8ee26e16..f4c01048cdae 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -7471,7 +7471,7 @@ win_readlink(PyObject *self, PyObject *args, PyObject *kwargs) #if defined(MS_WINDOWS) /* Grab CreateSymbolicLinkW dynamically from kernel32 */ -static DWORD (CALLBACK *Py_CreateSymbolicLinkW)(LPCWSTR, LPCWSTR, DWORD) = NULL; +static BOOLEAN (CALLBACK *Py_CreateSymbolicLinkW)(LPCWSTR, LPCWSTR, DWORD) = NULL; static int check_CreateSymbolicLink(void) @@ -7486,47 +7486,51 @@ check_CreateSymbolicLink(void) return Py_CreateSymbolicLinkW != NULL; } -/* Remove the last portion of the path */ -static void +/* Remove the last portion of the path - return 0 on success */ +static int _dirnameW(WCHAR *path) { WCHAR *ptr; + size_t length = wcsnlen_s(path, MAX_PATH); + if (length == MAX_PATH) { + return -1; + } /* walk the path from the end until a backslash is encountered */ - for(ptr = path + wcslen(path); ptr != path; ptr--) { - if (*ptr == L'\\' || *ptr == L'/') + for(ptr = path + length; ptr != path; ptr--) { + if (*ptr == L'\\' || *ptr == L'/') { break; + } } *ptr = 0; + return 0; } /* Is this path absolute? */ static int _is_absW(const WCHAR *path) { - return path[0] == L'\\' || path[0] == L'/' || path[1] == L':'; - + return path[0] == L'\\' || path[0] == L'/' || + (path[0] && path[1] == L':'); } -/* join root and rest with a backslash */ -static void +/* join root and rest with a backslash - return 0 on success */ +static int _joinW(WCHAR *dest_path, const WCHAR *root, const WCHAR *rest) { - size_t root_len; - if (_is_absW(rest)) { - wcscpy(dest_path, rest); - return; + return wcscpy_s(dest_path, MAX_PATH, rest); } - root_len = wcslen(root); + if (wcscpy_s(dest_path, MAX_PATH, root)) { + return -1; + } - wcscpy(dest_path, root); - if(root_len) { - dest_path[root_len] = L'\\'; - root_len++; + if (dest_path[0] && wcscat_s(dest_path, MAX_PATH, L"\\")) { + return -1; } - wcscpy(dest_path+root_len, rest); + + return wcscat_s(dest_path, MAX_PATH, rest); } /* Return True if the path at src relative to dest is a directory */ @@ -7538,10 +7542,14 @@ _check_dirW(LPCWSTR src, LPCWSTR dest) WCHAR src_resolved[MAX_PATH] = L""; /* dest_parent = os.path.dirname(dest) */ - wcscpy(dest_parent, dest); - _dirnameW(dest_parent); + if (wcscpy_s(dest_parent, MAX_PATH, dest) || + _dirnameW(dest_parent)) { + return 0; + } /* src_resolved = os.path.join(dest_parent, src) */ - _joinW(src_resolved, dest_parent, src); + if (_joinW(src_resolved, dest_parent, src)) { + return 0; + } return ( GetFileAttributesExW(src_resolved, GetFileExInfoStandard, &src_info) && src_info.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY @@ -7597,19 +7605,15 @@ os_symlink_impl(PyObject *module, path_t *src, path_t *dst, } #endif - if ((src->narrow && dst->wide) || (src->wide && dst->narrow)) { - PyErr_SetString(PyExc_ValueError, - "symlink: src and dst must be the same type"); - return NULL; - } - #ifdef MS_WINDOWS Py_BEGIN_ALLOW_THREADS + _Py_BEGIN_SUPPRESS_IPH /* if src is a directory, ensure target_is_directory==1 */ target_is_directory |= _check_dirW(src->wide, dst->wide); result = Py_CreateSymbolicLinkW(dst->wide, src->wide, target_is_directory); + _Py_END_SUPPRESS_IPH Py_END_ALLOW_THREADS if (!result) @@ -7617,6 +7621,12 @@ os_symlink_impl(PyObject *module, path_t *src, path_t *dst, #else + if ((src->narrow && dst->wide) || (src->wide && dst->narrow)) { + PyErr_SetString(PyExc_ValueError, + "symlink: src and dst must be the same type"); + return NULL; + } + Py_BEGIN_ALLOW_THREADS #if HAVE_SYMLINKAT if (dir_fd != DEFAULT_DIR_FD) From webhook-mailer at python.org Mon Mar 5 17:26:31 2018 From: webhook-mailer at python.org (Steve Dower) Date: Mon, 05 Mar 2018 22:26:31 -0000 Subject: [Python-checkins] [3.6] bpo-33001: Prevent buffer overrun in os.symlink (GH-5989) (GH-5990) Message-ID: <mailman.126.1520288792.2831.python-checkins@python.org> https://github.com/python/cpython/commit/baa45079466eda1f5636a6d13f3a60c2c00fdcd3 commit: baa45079466eda1f5636a6d13f3a60c2c00fdcd3 branch: 3.6 author: Steve Dower <steve.dower at microsoft.com> committer: GitHub <noreply at github.com> date: 2018-03-05T14:26:28-08:00 summary: [3.6] bpo-33001: Prevent buffer overrun in os.symlink (GH-5989) (GH-5990) files: A Misc/NEWS.d/next/Security/2018-03-05-10-09-51.bpo-33001.elj4Aa.rst M Lib/test/test_os.py M Modules/posixmodule.c diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index 26d544ca9925..240b7c432ba9 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -2170,6 +2170,41 @@ def test_29248(self): target = os.readlink(r'C:\Users\All Users') self.assertTrue(os.path.samefile(target, r'C:\ProgramData')) + def test_buffer_overflow(self): + # Older versions would have a buffer overflow when detecting + # whether a link source was a directory. This test ensures we + # no longer crash, but does not otherwise validate the behavior + segment = 'X' * 27 + path = os.path.join(*[segment] * 10) + test_cases = [ + # overflow with absolute src + ('\\' + path, segment), + # overflow dest with relative src + (segment, path), + # overflow when joining src + (path[:180], path[:180]), + ] + for src, dest in test_cases: + try: + os.symlink(src, dest) + except FileNotFoundError: + pass + else: + try: + os.remove(dest) + except OSError: + pass + # Also test with bytes, since that is a separate code path. + try: + os.symlink(os.fsencode(src), os.fsencode(dest)) + except FileNotFoundError: + pass + else: + try: + os.remove(dest) + except OSError: + pass + @unittest.skipUnless(sys.platform == "win32", "Win32 specific tests") class Win32JunctionTests(unittest.TestCase): diff --git a/Misc/NEWS.d/next/Security/2018-03-05-10-09-51.bpo-33001.elj4Aa.rst b/Misc/NEWS.d/next/Security/2018-03-05-10-09-51.bpo-33001.elj4Aa.rst new file mode 100644 index 000000000000..2acbac9e1af6 --- /dev/null +++ b/Misc/NEWS.d/next/Security/2018-03-05-10-09-51.bpo-33001.elj4Aa.rst @@ -0,0 +1 @@ +Minimal fix to prevent buffer overrun in os.symlink on Windows diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 0837a4a4991e..39ba030b5191 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -7144,7 +7144,7 @@ win_readlink(PyObject *self, PyObject *args, PyObject *kwargs) #if defined(MS_WINDOWS) /* Grab CreateSymbolicLinkW dynamically from kernel32 */ -static DWORD (CALLBACK *Py_CreateSymbolicLinkW)(LPCWSTR, LPCWSTR, DWORD) = NULL; +static BOOLEAN (CALLBACK *Py_CreateSymbolicLinkW)(LPCWSTR, LPCWSTR, DWORD) = NULL; static int check_CreateSymbolicLink(void) @@ -7159,47 +7159,51 @@ check_CreateSymbolicLink(void) return Py_CreateSymbolicLinkW != NULL; } -/* Remove the last portion of the path */ -static void +/* Remove the last portion of the path - return 0 on success */ +static int _dirnameW(WCHAR *path) { WCHAR *ptr; + size_t length = wcsnlen_s(path, MAX_PATH); + if (length == MAX_PATH) { + return -1; + } /* walk the path from the end until a backslash is encountered */ - for(ptr = path + wcslen(path); ptr != path; ptr--) { - if (*ptr == L'\\' || *ptr == L'/') + for(ptr = path + length; ptr != path; ptr--) { + if (*ptr == L'\\' || *ptr == L'/') { break; + } } *ptr = 0; + return 0; } /* Is this path absolute? */ static int _is_absW(const WCHAR *path) { - return path[0] == L'\\' || path[0] == L'/' || path[1] == L':'; - + return path[0] == L'\\' || path[0] == L'/' || + (path[0] && path[1] == L':'); } -/* join root and rest with a backslash */ -static void +/* join root and rest with a backslash - return 0 on success */ +static int _joinW(WCHAR *dest_path, const WCHAR *root, const WCHAR *rest) { - size_t root_len; - if (_is_absW(rest)) { - wcscpy(dest_path, rest); - return; + return wcscpy_s(dest_path, MAX_PATH, rest); } - root_len = wcslen(root); + if (wcscpy_s(dest_path, MAX_PATH, root)) { + return -1; + } - wcscpy(dest_path, root); - if(root_len) { - dest_path[root_len] = L'\\'; - root_len++; + if (dest_path[0] && wcscat_s(dest_path, MAX_PATH, L"\\")) { + return -1; } - wcscpy(dest_path+root_len, rest); + + return wcscat_s(dest_path, MAX_PATH, rest); } /* Return True if the path at src relative to dest is a directory */ @@ -7211,10 +7215,14 @@ _check_dirW(LPCWSTR src, LPCWSTR dest) WCHAR src_resolved[MAX_PATH] = L""; /* dest_parent = os.path.dirname(dest) */ - wcscpy(dest_parent, dest); - _dirnameW(dest_parent); + if (wcscpy_s(dest_parent, MAX_PATH, dest) || + _dirnameW(dest_parent)) { + return 0; + } /* src_resolved = os.path.join(dest_parent, src) */ - _joinW(src_resolved, dest_parent, src); + if (_joinW(src_resolved, dest_parent, src)) { + return 0; + } return ( GetFileAttributesExW(src_resolved, GetFileExInfoStandard, &src_info) && src_info.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY @@ -7270,19 +7278,15 @@ os_symlink_impl(PyObject *module, path_t *src, path_t *dst, } #endif - if ((src->narrow && dst->wide) || (src->wide && dst->narrow)) { - PyErr_SetString(PyExc_ValueError, - "symlink: src and dst must be the same type"); - return NULL; - } - #ifdef MS_WINDOWS Py_BEGIN_ALLOW_THREADS + _Py_BEGIN_SUPPRESS_IPH /* if src is a directory, ensure target_is_directory==1 */ target_is_directory |= _check_dirW(src->wide, dst->wide); result = Py_CreateSymbolicLinkW(dst->wide, src->wide, target_is_directory); + _Py_END_SUPPRESS_IPH Py_END_ALLOW_THREADS if (!result) @@ -7290,6 +7294,12 @@ os_symlink_impl(PyObject *module, path_t *src, path_t *dst, #else + if ((src->narrow && dst->wide) || (src->wide && dst->narrow)) { + PyErr_SetString(PyExc_ValueError, + "symlink: src and dst must be the same type"); + return NULL; + } + Py_BEGIN_ALLOW_THREADS #if HAVE_SYMLINKAT if (dir_fd != DEFAULT_DIR_FD) From webhook-mailer at python.org Mon Mar 5 18:13:03 2018 From: webhook-mailer at python.org (Miss Islington (bot)) Date: Mon, 05 Mar 2018 23:13:03 -0000 Subject: [Python-checkins] bpo-33001: Prevent buffer overrun in os.symlink (GH-5989) Message-ID: <mailman.127.1520291584.2831.python-checkins@python.org> https://github.com/python/cpython/commit/96fdbacb7797a564249fd59ccf86ec153c4bb095 commit: 96fdbacb7797a564249fd59ccf86ec153c4bb095 branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: GitHub <noreply at github.com> date: 2018-03-05T15:12:56-08:00 summary: bpo-33001: Prevent buffer overrun in os.symlink (GH-5989) (cherry picked from commit 6921e73e33edc3c61bc2d78ed558eaa22a89a564) Co-authored-by: Steve Dower <steve.dower at microsoft.com> files: A Misc/NEWS.d/next/Security/2018-03-05-10-09-51.bpo-33001.elj4Aa.rst M Lib/test/test_os.py M Modules/posixmodule.c diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index 4f8a2a7e19d5..e509188243f6 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -2164,6 +2164,40 @@ def test_29248(self): target = os.readlink(r'C:\Users\All Users') self.assertTrue(os.path.samefile(target, r'C:\ProgramData')) + def test_buffer_overflow(self): + # Older versions would have a buffer overflow when detecting + # whether a link source was a directory. This test ensures we + # no longer crash, but does not otherwise validate the behavior + segment = 'X' * 27 + path = os.path.join(*[segment] * 10) + test_cases = [ + # overflow with absolute src + ('\\' + path, segment), + # overflow dest with relative src + (segment, path), + # overflow when joining src + (path[:180], path[:180]), + ] + for src, dest in test_cases: + try: + os.symlink(src, dest) + except FileNotFoundError: + pass + else: + try: + os.remove(dest) + except OSError: + pass + # Also test with bytes, since that is a separate code path. + try: + os.symlink(os.fsencode(src), os.fsencode(dest)) + except FileNotFoundError: + pass + else: + try: + os.remove(dest) + except OSError: + pass @unittest.skipUnless(sys.platform == "win32", "Win32 specific tests") class Win32JunctionTests(unittest.TestCase): diff --git a/Misc/NEWS.d/next/Security/2018-03-05-10-09-51.bpo-33001.elj4Aa.rst b/Misc/NEWS.d/next/Security/2018-03-05-10-09-51.bpo-33001.elj4Aa.rst new file mode 100644 index 000000000000..2acbac9e1af6 --- /dev/null +++ b/Misc/NEWS.d/next/Security/2018-03-05-10-09-51.bpo-33001.elj4Aa.rst @@ -0,0 +1 @@ +Minimal fix to prevent buffer overrun in os.symlink on Windows diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index f6144a1d0527..6bf8c6ba7b7a 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -7471,7 +7471,7 @@ win_readlink(PyObject *self, PyObject *args, PyObject *kwargs) #if defined(MS_WINDOWS) /* Grab CreateSymbolicLinkW dynamically from kernel32 */ -static DWORD (CALLBACK *Py_CreateSymbolicLinkW)(LPCWSTR, LPCWSTR, DWORD) = NULL; +static BOOLEAN (CALLBACK *Py_CreateSymbolicLinkW)(LPCWSTR, LPCWSTR, DWORD) = NULL; static int check_CreateSymbolicLink(void) @@ -7486,47 +7486,51 @@ check_CreateSymbolicLink(void) return Py_CreateSymbolicLinkW != NULL; } -/* Remove the last portion of the path */ -static void +/* Remove the last portion of the path - return 0 on success */ +static int _dirnameW(WCHAR *path) { WCHAR *ptr; + size_t length = wcsnlen_s(path, MAX_PATH); + if (length == MAX_PATH) { + return -1; + } /* walk the path from the end until a backslash is encountered */ - for(ptr = path + wcslen(path); ptr != path; ptr--) { - if (*ptr == L'\\' || *ptr == L'/') + for(ptr = path + length; ptr != path; ptr--) { + if (*ptr == L'\\' || *ptr == L'/') { break; + } } *ptr = 0; + return 0; } /* Is this path absolute? */ static int _is_absW(const WCHAR *path) { - return path[0] == L'\\' || path[0] == L'/' || path[1] == L':'; - + return path[0] == L'\\' || path[0] == L'/' || + (path[0] && path[1] == L':'); } -/* join root and rest with a backslash */ -static void +/* join root and rest with a backslash - return 0 on success */ +static int _joinW(WCHAR *dest_path, const WCHAR *root, const WCHAR *rest) { - size_t root_len; - if (_is_absW(rest)) { - wcscpy(dest_path, rest); - return; + return wcscpy_s(dest_path, MAX_PATH, rest); } - root_len = wcslen(root); + if (wcscpy_s(dest_path, MAX_PATH, root)) { + return -1; + } - wcscpy(dest_path, root); - if(root_len) { - dest_path[root_len] = L'\\'; - root_len++; + if (dest_path[0] && wcscat_s(dest_path, MAX_PATH, L"\\")) { + return -1; } - wcscpy(dest_path+root_len, rest); + + return wcscat_s(dest_path, MAX_PATH, rest); } /* Return True if the path at src relative to dest is a directory */ @@ -7538,10 +7542,14 @@ _check_dirW(LPCWSTR src, LPCWSTR dest) WCHAR src_resolved[MAX_PATH] = L""; /* dest_parent = os.path.dirname(dest) */ - wcscpy(dest_parent, dest); - _dirnameW(dest_parent); + if (wcscpy_s(dest_parent, MAX_PATH, dest) || + _dirnameW(dest_parent)) { + return 0; + } /* src_resolved = os.path.join(dest_parent, src) */ - _joinW(src_resolved, dest_parent, src); + if (_joinW(src_resolved, dest_parent, src)) { + return 0; + } return ( GetFileAttributesExW(src_resolved, GetFileExInfoStandard, &src_info) && src_info.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY @@ -7597,19 +7605,15 @@ os_symlink_impl(PyObject *module, path_t *src, path_t *dst, } #endif - if ((src->narrow && dst->wide) || (src->wide && dst->narrow)) { - PyErr_SetString(PyExc_ValueError, - "symlink: src and dst must be the same type"); - return NULL; - } - #ifdef MS_WINDOWS Py_BEGIN_ALLOW_THREADS + _Py_BEGIN_SUPPRESS_IPH /* if src is a directory, ensure target_is_directory==1 */ target_is_directory |= _check_dirW(src->wide, dst->wide); result = Py_CreateSymbolicLinkW(dst->wide, src->wide, target_is_directory); + _Py_END_SUPPRESS_IPH Py_END_ALLOW_THREADS if (!result) @@ -7617,6 +7621,12 @@ os_symlink_impl(PyObject *module, path_t *src, path_t *dst, #else + if ((src->narrow && dst->wide) || (src->wide && dst->narrow)) { + PyErr_SetString(PyExc_ValueError, + "symlink: src and dst must be the same type"); + return NULL; + } + Py_BEGIN_ALLOW_THREADS #if HAVE_SYMLINKAT if (dir_fd != DEFAULT_DIR_FD) From webhook-mailer at python.org Mon Mar 5 18:29:11 2018 From: webhook-mailer at python.org (Jason R. Coombs) Date: Mon, 05 Mar 2018 23:29:11 -0000 Subject: [Python-checkins] bpo-32991: Restore expectation that inspect.getfile raises TypeError on namespace package (GH-5980) Message-ID: <mailman.128.1520292553.2831.python-checkins@python.org> https://github.com/python/cpython/commit/b9650a04a81355c8a7dcd0464c28febfb4bfc0a9 commit: b9650a04a81355c8a7dcd0464c28febfb4bfc0a9 branch: master author: Jason R. Coombs <jaraco at jaraco.com> committer: GitHub <noreply at github.com> date: 2018-03-05T18:29:08-05:00 summary: bpo-32991: Restore expectation that inspect.getfile raises TypeError on namespace package (GH-5980) * bpo-32991: Add test capturing expectation. DocTestFinder.find should return an empty list for doctests in a namespace package. * bpo-32991: Restore expectation that inspect.getfile on a namespace package raises TypeError. files: M Lib/inspect.py M Lib/test/test_doctest.py diff --git a/Lib/inspect.py b/Lib/inspect.py index 109efc06b268..57c04877c743 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -642,13 +642,13 @@ def cleandoc(doc): def getfile(object): """Work out which source or compiled file an object was defined in.""" if ismodule(object): - if hasattr(object, '__file__'): + if getattr(object, '__file__', None): return object.__file__ raise TypeError('{!r} is a built-in module'.format(object)) if isclass(object): if hasattr(object, '__module__'): object = sys.modules.get(object.__module__) - if hasattr(object, '__file__'): + if getattr(object, '__file__', None): return object.__file__ raise TypeError('{!r} is a built-in class'.format(object)) if ismethod(object): diff --git a/Lib/test/test_doctest.py b/Lib/test/test_doctest.py index 5ad94aba6492..f0eb52881bcf 100644 --- a/Lib/test/test_doctest.py +++ b/Lib/test/test_doctest.py @@ -7,6 +7,8 @@ import functools import os import sys +import importlib +import unittest # NOTE: There are some additional tests relating to interaction with @@ -435,7 +437,7 @@ def basics(): r""" >>> tests = finder.find(sample_func) >>> print(tests) # doctest: +ELLIPSIS - [<DocTest sample_func from ...:19 (1 example)>] + [<DocTest sample_func from ...:21 (1 example)>] The exact name depends on how test_doctest was invoked, so allow for leading path components. @@ -681,6 +683,17 @@ def non_Python_modules(): r""" and 'int' is a type. """ + +class TestDocTestFinder(unittest.TestCase): + + def test_empty_namespace_package(self): + pkg_name = 'doctest_empty_pkg' + os.mkdir(pkg_name) + mod = importlib.import_module(pkg_name) + assert doctest.DocTestFinder().find(mod) == [] + os.rmdir(pkg_name) + + def test_DocTestParser(): r""" Unit tests for the `DocTestParser` class. @@ -2945,6 +2958,10 @@ def test_main(): from test import test_doctest support.run_doctest(test_doctest, verbosity=True) + # Run unittests + support.run_unittest(__name__) + + def test_coverage(coverdir): trace = support.import_module('trace') tracer = trace.Trace(ignoredirs=[sys.base_prefix, sys.base_exec_prefix,], From webhook-mailer at python.org Tue Mar 6 00:59:05 2018 From: webhook-mailer at python.org (Benjamin Peterson) Date: Tue, 06 Mar 2018 05:59:05 -0000 Subject: [Python-checkins] [2.7] closes bpo-32997: Fix REDOS in fpformat (GH-5984) Message-ID: <mailman.129.1520315946.2831.python-checkins@python.org> https://github.com/python/cpython/commit/55d5bfba9482d39080f7b9ec3e6257ecd23f264f commit: 55d5bfba9482d39080f7b9ec3e6257ecd23f264f branch: 2.7 author: Jamie Davis <davisjam at vt.edu> committer: Benjamin Peterson <benjamin at python.org> date: 2018-03-05T21:59:02-08:00 summary: [2.7] closes bpo-32997: Fix REDOS in fpformat (GH-5984) The regex to decode a number in fpformat is susceptible to catastrophic backtracking. This is a potential DOS vector if a server is using fpformat on untrusted number strings. Replace it with an equivalent non-vulnerable regex. The match behavior of the new regex is slightly different. It captures the whole integer part of the number in one group, Leading zeros are stripped off later. files: A Misc/NEWS.d/next/Security/2018-03-05-10-14-42.bpo-32997.hp2s8n.rst M Lib/fpformat.py M Lib/test/test_fpformat.py diff --git a/Lib/fpformat.py b/Lib/fpformat.py index 71cbb25f3c8b..0537a27b8820 100644 --- a/Lib/fpformat.py +++ b/Lib/fpformat.py @@ -19,7 +19,7 @@ __all__ = ["fix","sci","NotANumber"] # Compiled regular expression to "decode" a number -decoder = re.compile(r'^([-+]?)0*(\d*)((?:\.\d*)?)(([eE][-+]?\d+)?)$') +decoder = re.compile(r'^([-+]?)(\d*)((?:\.\d*)?)(([eE][-+]?\d+)?)$') # \0 the whole thing # \1 leading sign or empty # \2 digits left of decimal point @@ -41,6 +41,7 @@ def extract(s): res = decoder.match(s) if res is None: raise NotANumber, s sign, intpart, fraction, exppart = res.group(1,2,3,4) + intpart = intpart.lstrip('0'); if sign == '+': sign = '' if fraction: fraction = fraction[1:] if exppart: expo = int(exppart[1:]) diff --git a/Lib/test/test_fpformat.py b/Lib/test/test_fpformat.py index e6de3b0c11be..428623ebb35f 100644 --- a/Lib/test/test_fpformat.py +++ b/Lib/test/test_fpformat.py @@ -67,6 +67,16 @@ def test_failing_values(self): else: self.fail("No exception on non-numeric sci") + def test_REDOS(self): + # This attack string will hang on the old decoder pattern. + attack = '+0' + ('0' * 1000000) + '++' + digs = 5 # irrelevant + + # fix returns input if it does not decode + self.assertEqual(fpformat.fix(attack, digs), attack) + # sci raises NotANumber + with self.assertRaises(NotANumber): + fpformat.sci(attack, digs) def test_main(): run_unittest(FpformatTest) diff --git a/Misc/NEWS.d/next/Security/2018-03-05-10-14-42.bpo-32997.hp2s8n.rst b/Misc/NEWS.d/next/Security/2018-03-05-10-14-42.bpo-32997.hp2s8n.rst new file mode 100644 index 000000000000..3c78ba61ae34 --- /dev/null +++ b/Misc/NEWS.d/next/Security/2018-03-05-10-14-42.bpo-32997.hp2s8n.rst @@ -0,0 +1,4 @@ +A regex in fpformat was vulnerable to catastrophic backtracking. This regex +was a potential DOS vector (REDOS). Based on typical uses of fpformat the +risk seems low. The regex has been refactored and is now safe. Patch by +Jamie Davis. From solipsis at pitrou.net Tue Mar 6 04:09:25 2018 From: solipsis at pitrou.net (solipsis at pitrou.net) Date: Tue, 06 Mar 2018 09:09:25 +0000 Subject: [Python-checkins] Daily reference leaks (4243df51fe43): sum=4 Message-ID: <20180306090925.1.EEA84D906B3F2938@psf.io> results for 4243df51fe43 on branch "default" -------------------------------------------- test_functools leaked [0, 3, 1] memory blocks, sum=4 test_multiprocessing_fork leaked [2, -1, -1] memory blocks, sum=0 Command line was: ['./python', '-m', 'test.regrtest', '-uall', '-R', '3:3:/home/psf-users/antoine/refleaks/reflogQLD00B', '--timeout', '7200'] From webhook-mailer at python.org Tue Mar 6 05:34:39 2018 From: webhook-mailer at python.org (Xiang Zhang) Date: Tue, 06 Mar 2018 10:34:39 -0000 Subject: [Python-checkins] Fix strncpy warning with gcc 8 (#5840) Message-ID: <mailman.130.1520332480.2831.python-checkins@python.org> https://github.com/python/cpython/commit/efd2bac1564f8141a4eab1bf8779b412974b8d69 commit: efd2bac1564f8141a4eab1bf8779b412974b8d69 branch: master author: Siddhesh Poyarekar <siddhesh.poyarekar at gmail.com> committer: Xiang Zhang <angwerzx at 126.com> date: 2018-03-06T18:34:35+08:00 summary: Fix strncpy warning with gcc 8 (#5840) The length in strncpy is one char too short and as a result it leads to a build warning with gcc 8. Comment out the strncpy since the interpreter aborts immediately after anyway. files: M Python/pystrtod.c diff --git a/Python/pystrtod.c b/Python/pystrtod.c index 9bf936386210..601f7c691edf 100644 --- a/Python/pystrtod.c +++ b/Python/pystrtod.c @@ -1060,8 +1060,8 @@ format_float_short(double d, char format_code, else { /* shouldn't get here: Gay's code should always return something starting with a digit, an 'I', or 'N' */ - strncpy(p, "ERR", 3); - /* p += 3; */ + /* strncpy(p, "ERR", 3); + p += 3; */ Py_UNREACHABLE(); } goto exit; From webhook-mailer at python.org Tue Mar 6 08:31:41 2018 From: webhook-mailer at python.org (Victor Stinner) Date: Tue, 06 Mar 2018 13:31:41 -0000 Subject: [Python-checkins] bpo-33005: Fix _PyGILState_Reinit() (#6001) Message-ID: <mailman.131.1520343102.2831.python-checkins@python.org> https://github.com/python/cpython/commit/5d92647102fac9e116b98ab8bbc632eeed501c34 commit: 5d92647102fac9e116b98ab8bbc632eeed501c34 branch: master author: Victor Stinner <victor.stinner at gmail.com> committer: GitHub <noreply at github.com> date: 2018-03-06T14:31:37+01:00 summary: bpo-33005: Fix _PyGILState_Reinit() (#6001) Fix a crash on fork when using a custom memory allocator (ex: using PYTHONMALLOC env var). _PyGILState_Reinit() and _PyInterpreterState_Enable() now use the default RAW memory allocator to allocate a new interpreters mutex on fork. files: A Misc/NEWS.d/next/Core and Builtins/2018-03-06-12-19-19.bpo-33005.LP-V2U.rst M Python/pystate.c diff --git a/Misc/NEWS.d/next/Core and Builtins/2018-03-06-12-19-19.bpo-33005.LP-V2U.rst b/Misc/NEWS.d/next/Core and Builtins/2018-03-06-12-19-19.bpo-33005.LP-V2U.rst new file mode 100644 index 000000000000..6c8b99cbb897 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2018-03-06-12-19-19.bpo-33005.LP-V2U.rst @@ -0,0 +1,4 @@ +Fix a crash on fork when using a custom memory allocator (ex: using +PYTHONMALLOC env var). _PyGILState_Reinit() and _PyInterpreterState_Enable() +now use the default RAW memory allocator to allocate a new interpreters mutex +on fork. diff --git a/Python/pystate.c b/Python/pystate.c index a87801f56922..140d2fba8efd 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -103,15 +103,24 @@ _PyInitError _PyInterpreterState_Enable(_PyRuntimeState *runtime) { runtime->interpreters.next_id = 0; - /* Since we only call _PyRuntimeState_Init() once per process - (see _PyRuntime_Initialize()), we make sure the mutex is - initialized here. */ + + /* Py_Finalize() calls _PyRuntimeState_Fini() which clears the mutex. + Create a new mutex if needed. */ if (runtime->interpreters.mutex == NULL) { + /* Force default allocator, since _PyRuntimeState_Fini() must + use the same allocator than this function. */ + PyMemAllocatorEx old_alloc; + _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); + runtime->interpreters.mutex = PyThread_allocate_lock(); + + PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); + if (runtime->interpreters.mutex == NULL) { return _Py_INIT_ERR("Can't initialize threads for interpreter"); } } + return _Py_INIT_OK(); } @@ -933,9 +942,19 @@ _PyGILState_Fini(void) void _PyGILState_Reinit(void) { + /* Force default allocator, since _PyRuntimeState_Fini() must + use the same allocator than this function. */ + PyMemAllocatorEx old_alloc; + _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); + _PyRuntime.interpreters.mutex = PyThread_allocate_lock(); - if (_PyRuntime.interpreters.mutex == NULL) + + PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); + + if (_PyRuntime.interpreters.mutex == NULL) { Py_FatalError("Can't initialize threads for interpreter"); + } + PyThreadState *tstate = PyGILState_GetThisThreadState(); PyThread_tss_delete(&_PyRuntime.gilstate.autoTSSkey); if (PyThread_tss_create(&_PyRuntime.gilstate.autoTSSkey) != 0) { From webhook-mailer at python.org Tue Mar 6 08:52:29 2018 From: webhook-mailer at python.org (Miss Islington (bot)) Date: Tue, 06 Mar 2018 13:52:29 -0000 Subject: [Python-checkins] bpo-33005: Fix _PyGILState_Reinit() (GH-6001) Message-ID: <mailman.132.1520344351.2831.python-checkins@python.org> https://github.com/python/cpython/commit/31e2b76f7bbcb8278748565252767a8b7790ff27 commit: 31e2b76f7bbcb8278748565252767a8b7790ff27 branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: GitHub <noreply at github.com> date: 2018-03-06T05:52:27-08:00 summary: bpo-33005: Fix _PyGILState_Reinit() (GH-6001) Fix a crash on fork when using a custom memory allocator (ex: using PYTHONMALLOC env var). _PyGILState_Reinit() and _PyInterpreterState_Enable() now use the default RAW memory allocator to allocate a new interpreters mutex on fork. (cherry picked from commit 5d92647102fac9e116b98ab8bbc632eeed501c34) Co-authored-by: Victor Stinner <victor.stinner at gmail.com> files: A Misc/NEWS.d/next/Core and Builtins/2018-03-06-12-19-19.bpo-33005.LP-V2U.rst M Python/pystate.c diff --git a/Misc/NEWS.d/next/Core and Builtins/2018-03-06-12-19-19.bpo-33005.LP-V2U.rst b/Misc/NEWS.d/next/Core and Builtins/2018-03-06-12-19-19.bpo-33005.LP-V2U.rst new file mode 100644 index 000000000000..6c8b99cbb897 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2018-03-06-12-19-19.bpo-33005.LP-V2U.rst @@ -0,0 +1,4 @@ +Fix a crash on fork when using a custom memory allocator (ex: using +PYTHONMALLOC env var). _PyGILState_Reinit() and _PyInterpreterState_Enable() +now use the default RAW memory allocator to allocate a new interpreters mutex +on fork. diff --git a/Python/pystate.c b/Python/pystate.c index a87801f56922..140d2fba8efd 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -103,15 +103,24 @@ _PyInitError _PyInterpreterState_Enable(_PyRuntimeState *runtime) { runtime->interpreters.next_id = 0; - /* Since we only call _PyRuntimeState_Init() once per process - (see _PyRuntime_Initialize()), we make sure the mutex is - initialized here. */ + + /* Py_Finalize() calls _PyRuntimeState_Fini() which clears the mutex. + Create a new mutex if needed. */ if (runtime->interpreters.mutex == NULL) { + /* Force default allocator, since _PyRuntimeState_Fini() must + use the same allocator than this function. */ + PyMemAllocatorEx old_alloc; + _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); + runtime->interpreters.mutex = PyThread_allocate_lock(); + + PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); + if (runtime->interpreters.mutex == NULL) { return _Py_INIT_ERR("Can't initialize threads for interpreter"); } } + return _Py_INIT_OK(); } @@ -933,9 +942,19 @@ _PyGILState_Fini(void) void _PyGILState_Reinit(void) { + /* Force default allocator, since _PyRuntimeState_Fini() must + use the same allocator than this function. */ + PyMemAllocatorEx old_alloc; + _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); + _PyRuntime.interpreters.mutex = PyThread_allocate_lock(); - if (_PyRuntime.interpreters.mutex == NULL) + + PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); + + if (_PyRuntime.interpreters.mutex == NULL) { Py_FatalError("Can't initialize threads for interpreter"); + } + PyThreadState *tstate = PyGILState_GetThisThreadState(); PyThread_tss_delete(&_PyRuntime.gilstate.autoTSSkey); if (PyThread_tss_create(&_PyRuntime.gilstate.autoTSSkey) != 0) { From webhook-mailer at python.org Tue Mar 6 10:16:14 2018 From: webhook-mailer at python.org (Jason R. Coombs) Date: Tue, 06 Mar 2018 15:16:14 -0000 Subject: [Python-checkins] bpo-32991: Restore expectation that inspect.getfile raises TypeError on namespace package (GH-5980) (GH-5997) Message-ID: <mailman.133.1520349375.2831.python-checkins@python.org> https://github.com/python/cpython/commit/5a0c3987abd6a71b4fadeb525477eb5f560e8514 commit: 5a0c3987abd6a71b4fadeb525477eb5f560e8514 branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Jason R. Coombs <jaraco at jaraco.com> date: 2018-03-06T10:16:11-05:00 summary: bpo-32991: Restore expectation that inspect.getfile raises TypeError on namespace package (GH-5980) (GH-5997) * bpo-32991: Add test capturing expectation. DocTestFinder.find should return an empty list for doctests in a namespace package. * bpo-32991: Restore expectation that inspect.getfile on a namespace package raises TypeError. (cherry picked from commit b9650a04a81355c8a7dcd0464c28febfb4bfc0a9) Co-authored-by: Jason R. Coombs <jaraco at jaraco.com> files: M Lib/inspect.py M Lib/test/test_doctest.py diff --git a/Lib/inspect.py b/Lib/inspect.py index bc97efe179ca..d62968965462 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -642,13 +642,13 @@ def cleandoc(doc): def getfile(object): """Work out which source or compiled file an object was defined in.""" if ismodule(object): - if hasattr(object, '__file__'): + if getattr(object, '__file__', None): return object.__file__ raise TypeError('{!r} is a built-in module'.format(object)) if isclass(object): if hasattr(object, '__module__'): object = sys.modules.get(object.__module__) - if hasattr(object, '__file__'): + if getattr(object, '__file__', None): return object.__file__ raise TypeError('{!r} is a built-in class'.format(object)) if ismethod(object): diff --git a/Lib/test/test_doctest.py b/Lib/test/test_doctest.py index 5ad94aba6492..f0eb52881bcf 100644 --- a/Lib/test/test_doctest.py +++ b/Lib/test/test_doctest.py @@ -7,6 +7,8 @@ import functools import os import sys +import importlib +import unittest # NOTE: There are some additional tests relating to interaction with @@ -435,7 +437,7 @@ def basics(): r""" >>> tests = finder.find(sample_func) >>> print(tests) # doctest: +ELLIPSIS - [<DocTest sample_func from ...:19 (1 example)>] + [<DocTest sample_func from ...:21 (1 example)>] The exact name depends on how test_doctest was invoked, so allow for leading path components. @@ -681,6 +683,17 @@ def non_Python_modules(): r""" and 'int' is a type. """ + +class TestDocTestFinder(unittest.TestCase): + + def test_empty_namespace_package(self): + pkg_name = 'doctest_empty_pkg' + os.mkdir(pkg_name) + mod = importlib.import_module(pkg_name) + assert doctest.DocTestFinder().find(mod) == [] + os.rmdir(pkg_name) + + def test_DocTestParser(): r""" Unit tests for the `DocTestParser` class. @@ -2945,6 +2958,10 @@ def test_main(): from test import test_doctest support.run_doctest(test_doctest, verbosity=True) + # Run unittests + support.run_unittest(__name__) + + def test_coverage(coverdir): trace = support.import_module('trace') tracer = trace.Trace(ignoredirs=[sys.base_prefix, sys.base_exec_prefix,], From webhook-mailer at python.org Tue Mar 6 12:59:49 2018 From: webhook-mailer at python.org (Yury Selivanov) Date: Tue, 06 Mar 2018 17:59:49 -0000 Subject: [Python-checkins] bpo-33009: Fix inspect.signature() for single-parameter partialmethods. (GH-6004) Message-ID: <mailman.134.1520359190.2831.python-checkins@python.org> https://github.com/python/cpython/commit/8a387219bdfb6ee34928d6168ac42ca559f11c9a commit: 8a387219bdfb6ee34928d6168ac42ca559f11c9a branch: master author: Yury Selivanov <yury at magic.io> committer: GitHub <noreply at github.com> date: 2018-03-06T12:59:45-05:00 summary: bpo-33009: Fix inspect.signature() for single-parameter partialmethods. (GH-6004) files: A Misc/NEWS.d/next/Library/2018-03-06-11-54-59.bpo-33009.-Ekysb.rst M Lib/inspect.py M Lib/test/test_inspect.py diff --git a/Lib/inspect.py b/Lib/inspect.py index 57c04877c743..512785f9237e 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -2254,7 +2254,8 @@ def _signature_from_callable(obj, *, return sig else: sig_params = tuple(sig.parameters.values()) - assert first_wrapped_param is not sig_params[0] + assert (not sig_params or + first_wrapped_param is not sig_params[0]) new_params = (first_wrapped_param,) + sig_params return sig.replace(parameters=new_params) diff --git a/Lib/test/test_inspect.py b/Lib/test/test_inspect.py index 1a856f6387e8..3481a57ec833 100644 --- a/Lib/test/test_inspect.py +++ b/Lib/test/test_inspect.py @@ -2580,6 +2580,16 @@ def test(it, a, *, c) -> 'spam': ('c', 1, ..., 'keyword_only')), 'spam')) + class Spam: + def test(self: 'anno', x): + pass + + g = partialmethod(test, 1) + + self.assertEqual(self.signature(Spam.g), + ((('self', ..., 'anno', 'positional_or_keyword'),), + ...)) + def test_signature_on_fake_partialmethod(self): def foo(a): pass foo._partialmethod = 'spam' diff --git a/Misc/NEWS.d/next/Library/2018-03-06-11-54-59.bpo-33009.-Ekysb.rst b/Misc/NEWS.d/next/Library/2018-03-06-11-54-59.bpo-33009.-Ekysb.rst new file mode 100644 index 000000000000..96bc70a8c944 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-03-06-11-54-59.bpo-33009.-Ekysb.rst @@ -0,0 +1 @@ +Fix inspect.signature() for single-parameter partialmethods. From webhook-mailer at python.org Tue Mar 6 13:23:52 2018 From: webhook-mailer at python.org (Miss Islington (bot)) Date: Tue, 06 Mar 2018 18:23:52 -0000 Subject: [Python-checkins] bpo-33009: Fix inspect.signature() for single-parameter partialmethods. (GH-6004) Message-ID: <mailman.135.1520360633.2831.python-checkins@python.org> https://github.com/python/cpython/commit/112f799666bac1bdbb320840d5fda3132255eb5e commit: 112f799666bac1bdbb320840d5fda3132255eb5e branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: GitHub <noreply at github.com> date: 2018-03-06T10:23:48-08:00 summary: bpo-33009: Fix inspect.signature() for single-parameter partialmethods. (GH-6004) (cherry picked from commit 8a387219bdfb6ee34928d6168ac42ca559f11c9a) Co-authored-by: Yury Selivanov <yury at magic.io> files: A Misc/NEWS.d/next/Library/2018-03-06-11-54-59.bpo-33009.-Ekysb.rst M Lib/inspect.py M Lib/test/test_inspect.py diff --git a/Lib/inspect.py b/Lib/inspect.py index d62968965462..f2719923e1ae 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -2254,7 +2254,8 @@ def _signature_from_callable(obj, *, return sig else: sig_params = tuple(sig.parameters.values()) - assert first_wrapped_param is not sig_params[0] + assert (not sig_params or + first_wrapped_param is not sig_params[0]) new_params = (first_wrapped_param,) + sig_params return sig.replace(parameters=new_params) diff --git a/Lib/test/test_inspect.py b/Lib/test/test_inspect.py index 1a856f6387e8..3481a57ec833 100644 --- a/Lib/test/test_inspect.py +++ b/Lib/test/test_inspect.py @@ -2580,6 +2580,16 @@ def test(it, a, *, c) -> 'spam': ('c', 1, ..., 'keyword_only')), 'spam')) + class Spam: + def test(self: 'anno', x): + pass + + g = partialmethod(test, 1) + + self.assertEqual(self.signature(Spam.g), + ((('self', ..., 'anno', 'positional_or_keyword'),), + ...)) + def test_signature_on_fake_partialmethod(self): def foo(a): pass foo._partialmethod = 'spam' diff --git a/Misc/NEWS.d/next/Library/2018-03-06-11-54-59.bpo-33009.-Ekysb.rst b/Misc/NEWS.d/next/Library/2018-03-06-11-54-59.bpo-33009.-Ekysb.rst new file mode 100644 index 000000000000..96bc70a8c944 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-03-06-11-54-59.bpo-33009.-Ekysb.rst @@ -0,0 +1 @@ +Fix inspect.signature() for single-parameter partialmethods. From webhook-mailer at python.org Tue Mar 6 13:48:11 2018 From: webhook-mailer at python.org (Miss Islington (bot)) Date: Tue, 06 Mar 2018 18:48:11 -0000 Subject: [Python-checkins] bpo-33009: Fix inspect.signature() for single-parameter partialmethods. (GH-6004) Message-ID: <mailman.136.1520362092.2831.python-checkins@python.org> https://github.com/python/cpython/commit/387a055261267f5fafd2c12eafef49759c94704f commit: 387a055261267f5fafd2c12eafef49759c94704f branch: 3.6 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: GitHub <noreply at github.com> date: 2018-03-06T10:48:04-08:00 summary: bpo-33009: Fix inspect.signature() for single-parameter partialmethods. (GH-6004) (cherry picked from commit 8a387219bdfb6ee34928d6168ac42ca559f11c9a) Co-authored-by: Yury Selivanov <yury at magic.io> files: A Misc/NEWS.d/next/Library/2018-03-06-11-54-59.bpo-33009.-Ekysb.rst M Lib/inspect.py M Lib/test/test_inspect.py diff --git a/Lib/inspect.py b/Lib/inspect.py index e9c2dbd5c88d..dc2aab2826af 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -2251,7 +2251,8 @@ def _signature_from_callable(obj, *, return sig else: sig_params = tuple(sig.parameters.values()) - assert first_wrapped_param is not sig_params[0] + assert (not sig_params or + first_wrapped_param is not sig_params[0]) new_params = (first_wrapped_param,) + sig_params return sig.replace(parameters=new_params) diff --git a/Lib/test/test_inspect.py b/Lib/test/test_inspect.py index 55436748d6f5..126b26e49227 100644 --- a/Lib/test/test_inspect.py +++ b/Lib/test/test_inspect.py @@ -2526,6 +2526,16 @@ def test(it, a, *, c) -> 'spam': ('c', 1, ..., 'keyword_only')), 'spam')) + class Spam: + def test(self: 'anno', x): + pass + + g = partialmethod(test, 1) + + self.assertEqual(self.signature(Spam.g), + ((('self', ..., 'anno', 'positional_or_keyword'),), + ...)) + def test_signature_on_fake_partialmethod(self): def foo(a): pass foo._partialmethod = 'spam' diff --git a/Misc/NEWS.d/next/Library/2018-03-06-11-54-59.bpo-33009.-Ekysb.rst b/Misc/NEWS.d/next/Library/2018-03-06-11-54-59.bpo-33009.-Ekysb.rst new file mode 100644 index 000000000000..96bc70a8c944 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-03-06-11-54-59.bpo-33009.-Ekysb.rst @@ -0,0 +1 @@ +Fix inspect.signature() for single-parameter partialmethods. From webhook-mailer at python.org Wed Mar 7 00:05:40 2018 From: webhook-mailer at python.org (Xiang Zhang) Date: Wed, 07 Mar 2018 05:05:40 -0000 Subject: [Python-checkins] bpo-32969: Expose some missing constants in zlib and fix the doc (GH-5988) Message-ID: <mailman.137.1520399141.2831.python-checkins@python.org> https://github.com/python/cpython/commit/bc3f2289b9007396bfb7f986bee477b6176c1822 commit: bc3f2289b9007396bfb7f986bee477b6176c1822 branch: master author: Xiang Zhang <angwerzx at 126.com> committer: GitHub <noreply at github.com> date: 2018-03-07T13:05:37+08:00 summary: bpo-32969: Expose some missing constants in zlib and fix the doc (GH-5988) files: A Misc/NEWS.d/next/Library/2018-03-06-00-19-41.bpo-32969.rGTKa0.rst M Doc/library/zlib.rst M Lib/test/test_zlib.py M Modules/zlibmodule.c diff --git a/Doc/library/zlib.rst b/Doc/library/zlib.rst index 3d742ab35b9c..8a531c92b8ff 100644 --- a/Doc/library/zlib.rst +++ b/Doc/library/zlib.rst @@ -51,9 +51,9 @@ The available exception and functions in this module are: Compresses the bytes in *data*, returning a bytes object containing compressed data. *level* is an integer from ``0`` to ``9`` or ``-1`` controlling the level of compression; - ``1`` is fastest and produces the least compression, ``9`` is slowest and - produces the most. ``0`` is no compression. The default value is ``-1`` - (Z_DEFAULT_COMPRESSION). Z_DEFAULT_COMPRESSION represents a default + ``1`` (Z_BEST_SPEED) is fastest and produces the least compression, ``9`` (Z_BEST_COMPRESSION) + is slowest and produces the most. ``0`` (Z_NO_COMPRESSION) is no compression. + The default value is ``-1`` (Z_DEFAULT_COMPRESSION). Z_DEFAULT_COMPRESSION represents a default compromise between speed and compression (currently equivalent to level 6). Raises the :exc:`error` exception if any error occurs. @@ -61,23 +61,25 @@ The available exception and functions in this module are: *level* can now be used as a keyword parameter. -.. function:: compressobj(level=-1, method=DEFLATED, wbits=15, memLevel=8, strategy=Z_DEFAULT_STRATEGY[, zdict]) +.. function:: compressobj(level=-1, method=DEFLATED, wbits=MAX_WBITS, memLevel=DEF_MEM_LEVEL, strategy=Z_DEFAULT_STRATEGY[, zdict]) Returns a compression object, to be used for compressing data streams that won't fit into memory at once. *level* is the compression level -- an integer from ``0`` to ``9`` or ``-1``. - A value of ``1`` is fastest and produces the least compression, while a value of - ``9`` is slowest and produces the most. ``0`` is no compression. The default - value is ``-1`` (Z_DEFAULT_COMPRESSION). Z_DEFAULT_COMPRESSION represents a default - compromise between speed and compression (currently equivalent to level 6). + A value of ``1`` (Z_BEST_SPEED) is fastest and produces the least compression, + while a value of ``9`` (Z_BEST_COMPRESSION) is slowest and produces the most. + ``0`` (Z_NO_COMPRESSION) is no compression. The default value is ``-1`` (Z_DEFAULT_COMPRESSION). + Z_DEFAULT_COMPRESSION represents a default compromise between speed and compression + (currently equivalent to level 6). *method* is the compression algorithm. Currently, the only supported value is - ``DEFLATED``. + :const:`DEFLATED`. The *wbits* argument controls the size of the history buffer (or the "window size") used when compressing data, and whether a header and - trailer is included in the output. It can take several ranges of values: + trailer is included in the output. It can take several ranges of values, + defaulting to ``15`` (MAX_WBITS): * +9 to +15: The base-two logarithm of the window size, which therefore ranges between 512 and 32768. Larger values produce @@ -97,7 +99,8 @@ The available exception and functions in this module are: Higher values use more memory, but are faster and produce smaller output. *strategy* is used to tune the compression algorithm. Possible values are - ``Z_DEFAULT_STRATEGY``, ``Z_FILTERED``, and ``Z_HUFFMAN_ONLY``. + :const:`Z_DEFAULT_STRATEGY`, :const:`Z_FILTERED`, :const:`Z_HUFFMAN_ONLY`, + :const:`Z_RLE` (zlib 1.2.0.1) and :const:`Z_FIXED` (zlib 1.2.2.2). *zdict* is a predefined compression dictionary. This is a sequence of bytes (such as a :class:`bytes` object) containing subsequences that are expected @@ -175,7 +178,7 @@ The available exception and functions in this module are: .. versionchanged:: 3.6 *wbits* and *bufsize* can be used as keyword arguments. -.. function:: decompressobj(wbits=15[, zdict]) +.. function:: decompressobj(wbits=MAX_WBITS[, zdict]) Returns a decompression object, to be used for decompressing data streams that won't fit into memory at once. @@ -213,13 +216,13 @@ Compression objects support the following methods: All pending input is processed, and a bytes object containing the remaining compressed output is returned. *mode* can be selected from the constants - :const:`Z_SYNC_FLUSH`, :const:`Z_FULL_FLUSH`, or :const:`Z_FINISH`, - defaulting to :const:`Z_FINISH`. :const:`Z_SYNC_FLUSH` and - :const:`Z_FULL_FLUSH` allow compressing further bytestrings of data, while - :const:`Z_FINISH` finishes the compressed stream and prevents compressing any - more data. After calling :meth:`flush` with *mode* set to :const:`Z_FINISH`, - the :meth:`compress` method cannot be called again; the only realistic action is - to delete the object. + :const:`Z_NO_FLUSH`, :const:`Z_PARTIAL_FLUSH`, :const:`Z_SYNC_FLUSH`, + :const:`Z_FULL_FLUSH`, :const:`Z_BLOCK` (zlib 1.2.3.4), or :const:`Z_FINISH`, + defaulting to :const:`Z_FINISH`. Except :const:`Z_FINISH`, all constants + allow compressing further bytestrings of data, while :const:`Z_FINISH` finishes the + compressed stream and prevents compressing any more data. After calling :meth:`flush` + with *mode* set to :const:`Z_FINISH`, the :meth:`compress` method cannot be called again; + the only realistic action is to delete the object. .. method:: Compress.copy() diff --git a/Lib/test/test_zlib.py b/Lib/test/test_zlib.py index db950fc1907a..99aa89bdc8c9 100644 --- a/Lib/test/test_zlib.py +++ b/Lib/test/test_zlib.py @@ -434,7 +434,8 @@ def test_clear_unconsumed_tail(self): def test_flushes(self): # Test flush() with the various options, using all the # different levels in order to provide more variations. - sync_opt = ['Z_NO_FLUSH', 'Z_SYNC_FLUSH', 'Z_FULL_FLUSH'] + sync_opt = ['Z_NO_FLUSH', 'Z_SYNC_FLUSH', 'Z_FULL_FLUSH', + 'Z_PARTIAL_FLUSH', 'Z_BLOCK'] sync_opt = [getattr(zlib, opt) for opt in sync_opt if hasattr(zlib, opt)] data = HAMLET_SCENE * 8 diff --git a/Misc/NEWS.d/next/Library/2018-03-06-00-19-41.bpo-32969.rGTKa0.rst b/Misc/NEWS.d/next/Library/2018-03-06-00-19-41.bpo-32969.rGTKa0.rst new file mode 100644 index 000000000000..a92307e67bfa --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-03-06-00-19-41.bpo-32969.rGTKa0.rst @@ -0,0 +1,2 @@ +Expose several missing constants in zlib and fix corresponding +documentation. diff --git a/Modules/zlibmodule.c b/Modules/zlibmodule.c index cb2aae106944..cd587b4ac9c7 100644 --- a/Modules/zlibmodule.c +++ b/Modules/zlibmodule.c @@ -1361,18 +1361,33 @@ PyInit_zlib(void) PyModule_AddIntMacro(m, DEFLATED); PyModule_AddIntMacro(m, DEF_MEM_LEVEL); PyModule_AddIntMacro(m, DEF_BUF_SIZE); + // compression levels + PyModule_AddIntMacro(m, Z_NO_COMPRESSION); PyModule_AddIntMacro(m, Z_BEST_SPEED); PyModule_AddIntMacro(m, Z_BEST_COMPRESSION); PyModule_AddIntMacro(m, Z_DEFAULT_COMPRESSION); + // compression strategies PyModule_AddIntMacro(m, Z_FILTERED); PyModule_AddIntMacro(m, Z_HUFFMAN_ONLY); +#ifdef Z_RLE // 1.2.0.1 + PyModule_AddIntMacro(m, Z_RLE); +#endif +#ifdef Z_FIXED // 1.2.2.2 + PyModule_AddIntMacro(m, Z_FIXED); +#endif PyModule_AddIntMacro(m, Z_DEFAULT_STRATEGY); - - PyModule_AddIntMacro(m, Z_FINISH); + // allowed flush values PyModule_AddIntMacro(m, Z_NO_FLUSH); + PyModule_AddIntMacro(m, Z_PARTIAL_FLUSH); PyModule_AddIntMacro(m, Z_SYNC_FLUSH); PyModule_AddIntMacro(m, Z_FULL_FLUSH); - + PyModule_AddIntMacro(m, Z_FINISH); +#ifdef Z_BLOCK // 1.2.0.5 for inflate, 1.2.3.4 for deflate + PyModule_AddIntMacro(m, Z_BLOCK); +#endif +#ifdef Z_TREES // 1.2.3.4, only for inflate + PyModule_AddIntMacro(m, Z_TREES); +#endif ver = PyUnicode_FromString(ZLIB_VERSION); if (ver != NULL) PyModule_AddObject(m, "ZLIB_VERSION", ver); From webhook-mailer at python.org Wed Mar 7 00:26:22 2018 From: webhook-mailer at python.org (Miss Islington (bot)) Date: Wed, 07 Mar 2018 05:26:22 -0000 Subject: [Python-checkins] bpo-32969: Expose some missing constants in zlib and fix the doc (GH-5988) Message-ID: <mailman.138.1520400384.2831.python-checkins@python.org> https://github.com/python/cpython/commit/c4d77a661138debbbe584b8b08410afc8719a9b1 commit: c4d77a661138debbbe584b8b08410afc8719a9b1 branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: GitHub <noreply at github.com> date: 2018-03-06T21:26:19-08:00 summary: bpo-32969: Expose some missing constants in zlib and fix the doc (GH-5988) (cherry picked from commit bc3f2289b9007396bfb7f986bee477b6176c1822) Co-authored-by: Xiang Zhang <angwerzx at 126.com> files: A Misc/NEWS.d/next/Library/2018-03-06-00-19-41.bpo-32969.rGTKa0.rst M Doc/library/zlib.rst M Lib/test/test_zlib.py M Modules/zlibmodule.c diff --git a/Doc/library/zlib.rst b/Doc/library/zlib.rst index 3d742ab35b9c..8a531c92b8ff 100644 --- a/Doc/library/zlib.rst +++ b/Doc/library/zlib.rst @@ -51,9 +51,9 @@ The available exception and functions in this module are: Compresses the bytes in *data*, returning a bytes object containing compressed data. *level* is an integer from ``0`` to ``9`` or ``-1`` controlling the level of compression; - ``1`` is fastest and produces the least compression, ``9`` is slowest and - produces the most. ``0`` is no compression. The default value is ``-1`` - (Z_DEFAULT_COMPRESSION). Z_DEFAULT_COMPRESSION represents a default + ``1`` (Z_BEST_SPEED) is fastest and produces the least compression, ``9`` (Z_BEST_COMPRESSION) + is slowest and produces the most. ``0`` (Z_NO_COMPRESSION) is no compression. + The default value is ``-1`` (Z_DEFAULT_COMPRESSION). Z_DEFAULT_COMPRESSION represents a default compromise between speed and compression (currently equivalent to level 6). Raises the :exc:`error` exception if any error occurs. @@ -61,23 +61,25 @@ The available exception and functions in this module are: *level* can now be used as a keyword parameter. -.. function:: compressobj(level=-1, method=DEFLATED, wbits=15, memLevel=8, strategy=Z_DEFAULT_STRATEGY[, zdict]) +.. function:: compressobj(level=-1, method=DEFLATED, wbits=MAX_WBITS, memLevel=DEF_MEM_LEVEL, strategy=Z_DEFAULT_STRATEGY[, zdict]) Returns a compression object, to be used for compressing data streams that won't fit into memory at once. *level* is the compression level -- an integer from ``0`` to ``9`` or ``-1``. - A value of ``1`` is fastest and produces the least compression, while a value of - ``9`` is slowest and produces the most. ``0`` is no compression. The default - value is ``-1`` (Z_DEFAULT_COMPRESSION). Z_DEFAULT_COMPRESSION represents a default - compromise between speed and compression (currently equivalent to level 6). + A value of ``1`` (Z_BEST_SPEED) is fastest and produces the least compression, + while a value of ``9`` (Z_BEST_COMPRESSION) is slowest and produces the most. + ``0`` (Z_NO_COMPRESSION) is no compression. The default value is ``-1`` (Z_DEFAULT_COMPRESSION). + Z_DEFAULT_COMPRESSION represents a default compromise between speed and compression + (currently equivalent to level 6). *method* is the compression algorithm. Currently, the only supported value is - ``DEFLATED``. + :const:`DEFLATED`. The *wbits* argument controls the size of the history buffer (or the "window size") used when compressing data, and whether a header and - trailer is included in the output. It can take several ranges of values: + trailer is included in the output. It can take several ranges of values, + defaulting to ``15`` (MAX_WBITS): * +9 to +15: The base-two logarithm of the window size, which therefore ranges between 512 and 32768. Larger values produce @@ -97,7 +99,8 @@ The available exception and functions in this module are: Higher values use more memory, but are faster and produce smaller output. *strategy* is used to tune the compression algorithm. Possible values are - ``Z_DEFAULT_STRATEGY``, ``Z_FILTERED``, and ``Z_HUFFMAN_ONLY``. + :const:`Z_DEFAULT_STRATEGY`, :const:`Z_FILTERED`, :const:`Z_HUFFMAN_ONLY`, + :const:`Z_RLE` (zlib 1.2.0.1) and :const:`Z_FIXED` (zlib 1.2.2.2). *zdict* is a predefined compression dictionary. This is a sequence of bytes (such as a :class:`bytes` object) containing subsequences that are expected @@ -175,7 +178,7 @@ The available exception and functions in this module are: .. versionchanged:: 3.6 *wbits* and *bufsize* can be used as keyword arguments. -.. function:: decompressobj(wbits=15[, zdict]) +.. function:: decompressobj(wbits=MAX_WBITS[, zdict]) Returns a decompression object, to be used for decompressing data streams that won't fit into memory at once. @@ -213,13 +216,13 @@ Compression objects support the following methods: All pending input is processed, and a bytes object containing the remaining compressed output is returned. *mode* can be selected from the constants - :const:`Z_SYNC_FLUSH`, :const:`Z_FULL_FLUSH`, or :const:`Z_FINISH`, - defaulting to :const:`Z_FINISH`. :const:`Z_SYNC_FLUSH` and - :const:`Z_FULL_FLUSH` allow compressing further bytestrings of data, while - :const:`Z_FINISH` finishes the compressed stream and prevents compressing any - more data. After calling :meth:`flush` with *mode* set to :const:`Z_FINISH`, - the :meth:`compress` method cannot be called again; the only realistic action is - to delete the object. + :const:`Z_NO_FLUSH`, :const:`Z_PARTIAL_FLUSH`, :const:`Z_SYNC_FLUSH`, + :const:`Z_FULL_FLUSH`, :const:`Z_BLOCK` (zlib 1.2.3.4), or :const:`Z_FINISH`, + defaulting to :const:`Z_FINISH`. Except :const:`Z_FINISH`, all constants + allow compressing further bytestrings of data, while :const:`Z_FINISH` finishes the + compressed stream and prevents compressing any more data. After calling :meth:`flush` + with *mode* set to :const:`Z_FINISH`, the :meth:`compress` method cannot be called again; + the only realistic action is to delete the object. .. method:: Compress.copy() diff --git a/Lib/test/test_zlib.py b/Lib/test/test_zlib.py index db950fc1907a..99aa89bdc8c9 100644 --- a/Lib/test/test_zlib.py +++ b/Lib/test/test_zlib.py @@ -434,7 +434,8 @@ def test_clear_unconsumed_tail(self): def test_flushes(self): # Test flush() with the various options, using all the # different levels in order to provide more variations. - sync_opt = ['Z_NO_FLUSH', 'Z_SYNC_FLUSH', 'Z_FULL_FLUSH'] + sync_opt = ['Z_NO_FLUSH', 'Z_SYNC_FLUSH', 'Z_FULL_FLUSH', + 'Z_PARTIAL_FLUSH', 'Z_BLOCK'] sync_opt = [getattr(zlib, opt) for opt in sync_opt if hasattr(zlib, opt)] data = HAMLET_SCENE * 8 diff --git a/Misc/NEWS.d/next/Library/2018-03-06-00-19-41.bpo-32969.rGTKa0.rst b/Misc/NEWS.d/next/Library/2018-03-06-00-19-41.bpo-32969.rGTKa0.rst new file mode 100644 index 000000000000..a92307e67bfa --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-03-06-00-19-41.bpo-32969.rGTKa0.rst @@ -0,0 +1,2 @@ +Expose several missing constants in zlib and fix corresponding +documentation. diff --git a/Modules/zlibmodule.c b/Modules/zlibmodule.c index cb2aae106944..cd587b4ac9c7 100644 --- a/Modules/zlibmodule.c +++ b/Modules/zlibmodule.c @@ -1361,18 +1361,33 @@ PyInit_zlib(void) PyModule_AddIntMacro(m, DEFLATED); PyModule_AddIntMacro(m, DEF_MEM_LEVEL); PyModule_AddIntMacro(m, DEF_BUF_SIZE); + // compression levels + PyModule_AddIntMacro(m, Z_NO_COMPRESSION); PyModule_AddIntMacro(m, Z_BEST_SPEED); PyModule_AddIntMacro(m, Z_BEST_COMPRESSION); PyModule_AddIntMacro(m, Z_DEFAULT_COMPRESSION); + // compression strategies PyModule_AddIntMacro(m, Z_FILTERED); PyModule_AddIntMacro(m, Z_HUFFMAN_ONLY); +#ifdef Z_RLE // 1.2.0.1 + PyModule_AddIntMacro(m, Z_RLE); +#endif +#ifdef Z_FIXED // 1.2.2.2 + PyModule_AddIntMacro(m, Z_FIXED); +#endif PyModule_AddIntMacro(m, Z_DEFAULT_STRATEGY); - - PyModule_AddIntMacro(m, Z_FINISH); + // allowed flush values PyModule_AddIntMacro(m, Z_NO_FLUSH); + PyModule_AddIntMacro(m, Z_PARTIAL_FLUSH); PyModule_AddIntMacro(m, Z_SYNC_FLUSH); PyModule_AddIntMacro(m, Z_FULL_FLUSH); - + PyModule_AddIntMacro(m, Z_FINISH); +#ifdef Z_BLOCK // 1.2.0.5 for inflate, 1.2.3.4 for deflate + PyModule_AddIntMacro(m, Z_BLOCK); +#endif +#ifdef Z_TREES // 1.2.3.4, only for inflate + PyModule_AddIntMacro(m, Z_TREES); +#endif ver = PyUnicode_FromString(ZLIB_VERSION); if (ver != NULL) PyModule_AddObject(m, "ZLIB_VERSION", ver); From webhook-mailer at python.org Wed Mar 7 00:46:13 2018 From: webhook-mailer at python.org (Miss Islington (bot)) Date: Wed, 07 Mar 2018 05:46:13 -0000 Subject: [Python-checkins] bpo-32969: Expose some missing constants in zlib and fix the doc (GH-5988) Message-ID: <mailman.139.1520401575.2831.python-checkins@python.org> https://github.com/python/cpython/commit/7592c0a686a80b9fbe2e6d519a486aca58b3260b commit: 7592c0a686a80b9fbe2e6d519a486aca58b3260b branch: 3.6 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: GitHub <noreply at github.com> date: 2018-03-06T21:46:10-08:00 summary: bpo-32969: Expose some missing constants in zlib and fix the doc (GH-5988) (cherry picked from commit bc3f2289b9007396bfb7f986bee477b6176c1822) Co-authored-by: Xiang Zhang <angwerzx at 126.com> files: A Misc/NEWS.d/next/Library/2018-03-06-00-19-41.bpo-32969.rGTKa0.rst M Doc/library/zlib.rst M Lib/test/test_zlib.py M Modules/zlibmodule.c diff --git a/Doc/library/zlib.rst b/Doc/library/zlib.rst index 3d742ab35b9c..8a531c92b8ff 100644 --- a/Doc/library/zlib.rst +++ b/Doc/library/zlib.rst @@ -51,9 +51,9 @@ The available exception and functions in this module are: Compresses the bytes in *data*, returning a bytes object containing compressed data. *level* is an integer from ``0`` to ``9`` or ``-1`` controlling the level of compression; - ``1`` is fastest and produces the least compression, ``9`` is slowest and - produces the most. ``0`` is no compression. The default value is ``-1`` - (Z_DEFAULT_COMPRESSION). Z_DEFAULT_COMPRESSION represents a default + ``1`` (Z_BEST_SPEED) is fastest and produces the least compression, ``9`` (Z_BEST_COMPRESSION) + is slowest and produces the most. ``0`` (Z_NO_COMPRESSION) is no compression. + The default value is ``-1`` (Z_DEFAULT_COMPRESSION). Z_DEFAULT_COMPRESSION represents a default compromise between speed and compression (currently equivalent to level 6). Raises the :exc:`error` exception if any error occurs. @@ -61,23 +61,25 @@ The available exception and functions in this module are: *level* can now be used as a keyword parameter. -.. function:: compressobj(level=-1, method=DEFLATED, wbits=15, memLevel=8, strategy=Z_DEFAULT_STRATEGY[, zdict]) +.. function:: compressobj(level=-1, method=DEFLATED, wbits=MAX_WBITS, memLevel=DEF_MEM_LEVEL, strategy=Z_DEFAULT_STRATEGY[, zdict]) Returns a compression object, to be used for compressing data streams that won't fit into memory at once. *level* is the compression level -- an integer from ``0`` to ``9`` or ``-1``. - A value of ``1`` is fastest and produces the least compression, while a value of - ``9`` is slowest and produces the most. ``0`` is no compression. The default - value is ``-1`` (Z_DEFAULT_COMPRESSION). Z_DEFAULT_COMPRESSION represents a default - compromise between speed and compression (currently equivalent to level 6). + A value of ``1`` (Z_BEST_SPEED) is fastest and produces the least compression, + while a value of ``9`` (Z_BEST_COMPRESSION) is slowest and produces the most. + ``0`` (Z_NO_COMPRESSION) is no compression. The default value is ``-1`` (Z_DEFAULT_COMPRESSION). + Z_DEFAULT_COMPRESSION represents a default compromise between speed and compression + (currently equivalent to level 6). *method* is the compression algorithm. Currently, the only supported value is - ``DEFLATED``. + :const:`DEFLATED`. The *wbits* argument controls the size of the history buffer (or the "window size") used when compressing data, and whether a header and - trailer is included in the output. It can take several ranges of values: + trailer is included in the output. It can take several ranges of values, + defaulting to ``15`` (MAX_WBITS): * +9 to +15: The base-two logarithm of the window size, which therefore ranges between 512 and 32768. Larger values produce @@ -97,7 +99,8 @@ The available exception and functions in this module are: Higher values use more memory, but are faster and produce smaller output. *strategy* is used to tune the compression algorithm. Possible values are - ``Z_DEFAULT_STRATEGY``, ``Z_FILTERED``, and ``Z_HUFFMAN_ONLY``. + :const:`Z_DEFAULT_STRATEGY`, :const:`Z_FILTERED`, :const:`Z_HUFFMAN_ONLY`, + :const:`Z_RLE` (zlib 1.2.0.1) and :const:`Z_FIXED` (zlib 1.2.2.2). *zdict* is a predefined compression dictionary. This is a sequence of bytes (such as a :class:`bytes` object) containing subsequences that are expected @@ -175,7 +178,7 @@ The available exception and functions in this module are: .. versionchanged:: 3.6 *wbits* and *bufsize* can be used as keyword arguments. -.. function:: decompressobj(wbits=15[, zdict]) +.. function:: decompressobj(wbits=MAX_WBITS[, zdict]) Returns a decompression object, to be used for decompressing data streams that won't fit into memory at once. @@ -213,13 +216,13 @@ Compression objects support the following methods: All pending input is processed, and a bytes object containing the remaining compressed output is returned. *mode* can be selected from the constants - :const:`Z_SYNC_FLUSH`, :const:`Z_FULL_FLUSH`, or :const:`Z_FINISH`, - defaulting to :const:`Z_FINISH`. :const:`Z_SYNC_FLUSH` and - :const:`Z_FULL_FLUSH` allow compressing further bytestrings of data, while - :const:`Z_FINISH` finishes the compressed stream and prevents compressing any - more data. After calling :meth:`flush` with *mode* set to :const:`Z_FINISH`, - the :meth:`compress` method cannot be called again; the only realistic action is - to delete the object. + :const:`Z_NO_FLUSH`, :const:`Z_PARTIAL_FLUSH`, :const:`Z_SYNC_FLUSH`, + :const:`Z_FULL_FLUSH`, :const:`Z_BLOCK` (zlib 1.2.3.4), or :const:`Z_FINISH`, + defaulting to :const:`Z_FINISH`. Except :const:`Z_FINISH`, all constants + allow compressing further bytestrings of data, while :const:`Z_FINISH` finishes the + compressed stream and prevents compressing any more data. After calling :meth:`flush` + with *mode* set to :const:`Z_FINISH`, the :meth:`compress` method cannot be called again; + the only realistic action is to delete the object. .. method:: Compress.copy() diff --git a/Lib/test/test_zlib.py b/Lib/test/test_zlib.py index 8e29b1bcebff..5b58c77630b6 100644 --- a/Lib/test/test_zlib.py +++ b/Lib/test/test_zlib.py @@ -434,7 +434,8 @@ def test_clear_unconsumed_tail(self): def test_flushes(self): # Test flush() with the various options, using all the # different levels in order to provide more variations. - sync_opt = ['Z_NO_FLUSH', 'Z_SYNC_FLUSH', 'Z_FULL_FLUSH'] + sync_opt = ['Z_NO_FLUSH', 'Z_SYNC_FLUSH', 'Z_FULL_FLUSH', + 'Z_PARTIAL_FLUSH', 'Z_BLOCK'] sync_opt = [getattr(zlib, opt) for opt in sync_opt if hasattr(zlib, opt)] data = HAMLET_SCENE * 8 diff --git a/Misc/NEWS.d/next/Library/2018-03-06-00-19-41.bpo-32969.rGTKa0.rst b/Misc/NEWS.d/next/Library/2018-03-06-00-19-41.bpo-32969.rGTKa0.rst new file mode 100644 index 000000000000..a92307e67bfa --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-03-06-00-19-41.bpo-32969.rGTKa0.rst @@ -0,0 +1,2 @@ +Expose several missing constants in zlib and fix corresponding +documentation. diff --git a/Modules/zlibmodule.c b/Modules/zlibmodule.c index 29f55bdb19c6..a20dcd6de51e 100644 --- a/Modules/zlibmodule.c +++ b/Modules/zlibmodule.c @@ -1372,18 +1372,33 @@ PyInit_zlib(void) PyModule_AddIntMacro(m, DEFLATED); PyModule_AddIntMacro(m, DEF_MEM_LEVEL); PyModule_AddIntMacro(m, DEF_BUF_SIZE); + // compression levels + PyModule_AddIntMacro(m, Z_NO_COMPRESSION); PyModule_AddIntMacro(m, Z_BEST_SPEED); PyModule_AddIntMacro(m, Z_BEST_COMPRESSION); PyModule_AddIntMacro(m, Z_DEFAULT_COMPRESSION); + // compression strategies PyModule_AddIntMacro(m, Z_FILTERED); PyModule_AddIntMacro(m, Z_HUFFMAN_ONLY); +#ifdef Z_RLE // 1.2.0.1 + PyModule_AddIntMacro(m, Z_RLE); +#endif +#ifdef Z_FIXED // 1.2.2.2 + PyModule_AddIntMacro(m, Z_FIXED); +#endif PyModule_AddIntMacro(m, Z_DEFAULT_STRATEGY); - - PyModule_AddIntMacro(m, Z_FINISH); + // allowed flush values PyModule_AddIntMacro(m, Z_NO_FLUSH); + PyModule_AddIntMacro(m, Z_PARTIAL_FLUSH); PyModule_AddIntMacro(m, Z_SYNC_FLUSH); PyModule_AddIntMacro(m, Z_FULL_FLUSH); - + PyModule_AddIntMacro(m, Z_FINISH); +#ifdef Z_BLOCK // 1.2.0.5 for inflate, 1.2.3.4 for deflate + PyModule_AddIntMacro(m, Z_BLOCK); +#endif +#ifdef Z_TREES // 1.2.3.4, only for inflate + PyModule_AddIntMacro(m, Z_TREES); +#endif ver = PyUnicode_FromString(ZLIB_VERSION); if (ver != NULL) PyModule_AddObject(m, "ZLIB_VERSION", ver); From webhook-mailer at python.org Wed Mar 7 02:27:07 2018 From: webhook-mailer at python.org (INADA Naoki) Date: Wed, 07 Mar 2018 07:27:07 -0000 Subject: [Python-checkins] bpo-32999: Fix ABC.__subclasscheck__ crash (GH-6002) Message-ID: <mailman.140.1520407628.2831.python-checkins@python.org> https://github.com/python/cpython/commit/fc7df0e664198cb05cafd972f190a18ca422989c commit: fc7df0e664198cb05cafd972f190a18ca422989c branch: master author: INADA Naoki <methane at users.noreply.github.com> committer: GitHub <noreply at github.com> date: 2018-03-07T16:27:01+09:00 summary: bpo-32999: Fix ABC.__subclasscheck__ crash (GH-6002) files: A Misc/NEWS.d/next/Library/2018-03-06-20-30-20.bpo-32999.lgFXWl.rst M Lib/test/test_abc.py M Modules/_abc.c diff --git a/Lib/test/test_abc.py b/Lib/test/test_abc.py index af26c1d6b8c5..6fc3c95e4a64 100644 --- a/Lib/test/test_abc.py +++ b/Lib/test/test_abc.py @@ -392,6 +392,24 @@ class MyInt(int): self.assertIsInstance(42, A) self.assertIsInstance(42, (A,)) + def test_issubclass_bad_arguments(self): + class A(metaclass=abc_ABCMeta): + pass + + with self.assertRaises(TypeError): + issubclass({}, A) # unhashable + + with self.assertRaises(TypeError): + issubclass(42, A) # No __mro__ + + # Python version supports any iterable as __mro__. + # But it's implementation detail and don't emulate it in C version. + class C: + __mro__ = 42 # __mro__ is not tuple + + with self.assertRaises(TypeError): + issubclass(C(), A) + def test_all_new_methods_are_called(self): class A(metaclass=abc_ABCMeta): pass diff --git a/Misc/NEWS.d/next/Library/2018-03-06-20-30-20.bpo-32999.lgFXWl.rst b/Misc/NEWS.d/next/Library/2018-03-06-20-30-20.bpo-32999.lgFXWl.rst new file mode 100644 index 000000000000..45e75f939310 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-03-06-20-30-20.bpo-32999.lgFXWl.rst @@ -0,0 +1,2 @@ +Fix C implemetation of ``ABC.__subclasscheck__(cls, subclass)`` crashed when +``subclass`` is not a type object. diff --git a/Modules/_abc.c b/Modules/_abc.c index 504e23d9a74d..862883987fb7 100644 --- a/Modules/_abc.c +++ b/Modules/_abc.c @@ -16,6 +16,7 @@ _Py_IDENTIFIER(__abstractmethods__); _Py_IDENTIFIER(__class__); _Py_IDENTIFIER(__dict__); _Py_IDENTIFIER(__bases__); +_Py_IDENTIFIER(__mro__); _Py_IDENTIFIER(_abc_impl); _Py_IDENTIFIER(__subclasscheck__); _Py_IDENTIFIER(__subclasshook__); @@ -568,7 +569,7 @@ _abc__abc_subclasscheck_impl(PyObject *module, PyObject *self, PyObject *subclass) /*[clinic end generated code: output=b56c9e4a530e3894 input=1d947243409d10b8]*/ { - PyObject *ok, *mro, *subclasses = NULL, *result = NULL; + PyObject *ok, *mro = NULL, *subclasses = NULL, *result = NULL; Py_ssize_t pos; int incache; _abc_data *impl = _get_impl(self); @@ -637,20 +638,31 @@ _abc__abc_subclasscheck_impl(PyObject *module, PyObject *self, } Py_DECREF(ok); - /* 4. Check if it's a direct subclass. */ - mro = ((PyTypeObject *)subclass)->tp_mro; - assert(PyTuple_Check(mro)); - for (pos = 0; pos < PyTuple_GET_SIZE(mro); pos++) { - PyObject *mro_item = PyTuple_GET_ITEM(mro, pos); - if (mro_item == NULL) { + /* 4. Check if it's a direct subclass. + * + * if cls in getattr(subclass, '__mro__', ()): + * cls._abc_cache.add(subclass) + * return True + */ + if (_PyObject_LookupAttrId(subclass, &PyId___mro__, &mro) < 0) { + goto end; + } + if (mro != NULL) { + if (!PyTuple_Check(mro)) { + // Python version supports non-tuple iterable. Keep it as + // implementation detail. + PyErr_SetString(PyExc_TypeError, "__mro__ is not a tuple"); goto end; } - if ((PyObject *)self == mro_item) { - if (_add_to_weak_set(&impl->_abc_cache, subclass) < 0) { + for (pos = 0; pos < PyTuple_GET_SIZE(mro); pos++) { + PyObject *mro_item = PyTuple_GET_ITEM(mro, pos); + if ((PyObject *)self == mro_item) { + if (_add_to_weak_set(&impl->_abc_cache, subclass) < 0) { + goto end; + } + result = Py_True; goto end; } - result = Py_True; - goto end; } } @@ -690,7 +702,8 @@ _abc__abc_subclasscheck_impl(PyObject *module, PyObject *self, result = Py_False; end: - Py_XDECREF(impl); + Py_DECREF(impl); + Py_XDECREF(mro); Py_XDECREF(subclasses); Py_XINCREF(result); return result; From webhook-mailer at python.org Wed Mar 7 02:47:42 2018 From: webhook-mailer at python.org (Miss Islington (bot)) Date: Wed, 07 Mar 2018 07:47:42 -0000 Subject: [Python-checkins] bpo-32999: Fix ABC.__subclasscheck__ crash (GH-6002) Message-ID: <mailman.141.1520408864.2831.python-checkins@python.org> https://github.com/python/cpython/commit/d824b4e4afbe9cf02310e39b14402fb2aa271f8f commit: d824b4e4afbe9cf02310e39b14402fb2aa271f8f branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: GitHub <noreply at github.com> date: 2018-03-06T23:47:40-08:00 summary: bpo-32999: Fix ABC.__subclasscheck__ crash (GH-6002) (cherry picked from commit fc7df0e664198cb05cafd972f190a18ca422989c) Co-authored-by: INADA Naoki <methane at users.noreply.github.com> files: A Misc/NEWS.d/next/Library/2018-03-06-20-30-20.bpo-32999.lgFXWl.rst M Lib/test/test_abc.py M Modules/_abc.c diff --git a/Lib/test/test_abc.py b/Lib/test/test_abc.py index af26c1d6b8c5..6fc3c95e4a64 100644 --- a/Lib/test/test_abc.py +++ b/Lib/test/test_abc.py @@ -392,6 +392,24 @@ class MyInt(int): self.assertIsInstance(42, A) self.assertIsInstance(42, (A,)) + def test_issubclass_bad_arguments(self): + class A(metaclass=abc_ABCMeta): + pass + + with self.assertRaises(TypeError): + issubclass({}, A) # unhashable + + with self.assertRaises(TypeError): + issubclass(42, A) # No __mro__ + + # Python version supports any iterable as __mro__. + # But it's implementation detail and don't emulate it in C version. + class C: + __mro__ = 42 # __mro__ is not tuple + + with self.assertRaises(TypeError): + issubclass(C(), A) + def test_all_new_methods_are_called(self): class A(metaclass=abc_ABCMeta): pass diff --git a/Misc/NEWS.d/next/Library/2018-03-06-20-30-20.bpo-32999.lgFXWl.rst b/Misc/NEWS.d/next/Library/2018-03-06-20-30-20.bpo-32999.lgFXWl.rst new file mode 100644 index 000000000000..45e75f939310 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-03-06-20-30-20.bpo-32999.lgFXWl.rst @@ -0,0 +1,2 @@ +Fix C implemetation of ``ABC.__subclasscheck__(cls, subclass)`` crashed when +``subclass`` is not a type object. diff --git a/Modules/_abc.c b/Modules/_abc.c index 504e23d9a74d..862883987fb7 100644 --- a/Modules/_abc.c +++ b/Modules/_abc.c @@ -16,6 +16,7 @@ _Py_IDENTIFIER(__abstractmethods__); _Py_IDENTIFIER(__class__); _Py_IDENTIFIER(__dict__); _Py_IDENTIFIER(__bases__); +_Py_IDENTIFIER(__mro__); _Py_IDENTIFIER(_abc_impl); _Py_IDENTIFIER(__subclasscheck__); _Py_IDENTIFIER(__subclasshook__); @@ -568,7 +569,7 @@ _abc__abc_subclasscheck_impl(PyObject *module, PyObject *self, PyObject *subclass) /*[clinic end generated code: output=b56c9e4a530e3894 input=1d947243409d10b8]*/ { - PyObject *ok, *mro, *subclasses = NULL, *result = NULL; + PyObject *ok, *mro = NULL, *subclasses = NULL, *result = NULL; Py_ssize_t pos; int incache; _abc_data *impl = _get_impl(self); @@ -637,20 +638,31 @@ _abc__abc_subclasscheck_impl(PyObject *module, PyObject *self, } Py_DECREF(ok); - /* 4. Check if it's a direct subclass. */ - mro = ((PyTypeObject *)subclass)->tp_mro; - assert(PyTuple_Check(mro)); - for (pos = 0; pos < PyTuple_GET_SIZE(mro); pos++) { - PyObject *mro_item = PyTuple_GET_ITEM(mro, pos); - if (mro_item == NULL) { + /* 4. Check if it's a direct subclass. + * + * if cls in getattr(subclass, '__mro__', ()): + * cls._abc_cache.add(subclass) + * return True + */ + if (_PyObject_LookupAttrId(subclass, &PyId___mro__, &mro) < 0) { + goto end; + } + if (mro != NULL) { + if (!PyTuple_Check(mro)) { + // Python version supports non-tuple iterable. Keep it as + // implementation detail. + PyErr_SetString(PyExc_TypeError, "__mro__ is not a tuple"); goto end; } - if ((PyObject *)self == mro_item) { - if (_add_to_weak_set(&impl->_abc_cache, subclass) < 0) { + for (pos = 0; pos < PyTuple_GET_SIZE(mro); pos++) { + PyObject *mro_item = PyTuple_GET_ITEM(mro, pos); + if ((PyObject *)self == mro_item) { + if (_add_to_weak_set(&impl->_abc_cache, subclass) < 0) { + goto end; + } + result = Py_True; goto end; } - result = Py_True; - goto end; } } @@ -690,7 +702,8 @@ _abc__abc_subclasscheck_impl(PyObject *module, PyObject *self, result = Py_False; end: - Py_XDECREF(impl); + Py_DECREF(impl); + Py_XDECREF(mro); Py_XDECREF(subclasses); Py_XINCREF(result); return result; From solipsis at pitrou.net Wed Mar 7 04:14:00 2018 From: solipsis at pitrou.net (solipsis at pitrou.net) Date: Wed, 07 Mar 2018 09:14:00 +0000 Subject: [Python-checkins] Daily reference leaks (4243df51fe43): sum=6 Message-ID: <20180307091400.1.B121A3426EF090D2@psf.io> results for 4243df51fe43 on branch "default" -------------------------------------------- test_functools leaked [0, 3, 1] memory blocks, sum=4 test_multiprocessing_spawn leaked [2, 0, 0] memory blocks, sum=2 Command line was: ['./python', '-m', 'test.regrtest', '-uall', '-R', '3:3:/home/psf-users/antoine/refleaks/refloggulaMw', '--timeout', '7200'] From webhook-mailer at python.org Wed Mar 7 23:24:40 2018 From: webhook-mailer at python.org (Xiang Zhang) Date: Thu, 08 Mar 2018 04:24:40 -0000 Subject: [Python-checkins] Fix some ipaddress documentation errors (GH-6021) Message-ID: <mailman.1.1520483080.2865.python-checkins@python.org> https://github.com/python/cpython/commit/e405096ea91f516d411095b6fea4eec9668eac88 commit: e405096ea91f516d411095b6fea4eec9668eac88 branch: master author: Xiang Zhang <angwerzx at 126.com> committer: GitHub <noreply at github.com> date: 2018-03-08T12:24:36+08:00 summary: Fix some ipaddress documentation errors (GH-6021) * fix a typo: documention -> documentation * fix the type of IPv?Network.hostmask * add documentation about IPv?Network.netmask * fix IPv6Network constructor doc that extended netmasks are not supported files: M Doc/library/ipaddress.rst diff --git a/Doc/library/ipaddress.rst b/Doc/library/ipaddress.rst index 75c9107bd5eb..76177a0b23eb 100644 --- a/Doc/library/ipaddress.rst +++ b/Doc/library/ipaddress.rst @@ -236,7 +236,7 @@ write code that handles both IP versions correctly. groups consisting entirely of zeroes included. - For the following attributes, see the corresponding documention of the + For the following attributes, see the corresponding documentation of the :class:`IPv4Address` class: .. attribute:: packed @@ -442,7 +442,11 @@ so to avoid duplication they are only documented for :class:`IPv4Network`. .. attribute:: hostmask - The host mask, as a string. + The host mask, as an :class:`IPv4Address` object. + + .. attribute:: netmask + + The net mask, as an :class:`IPv4Address` object. .. attribute:: with_prefixlen .. attribute:: compressed @@ -588,13 +592,12 @@ so to avoid duplication they are only documented for :class:`IPv4Network`. 1. A string consisting of an IP address and an optional mask, separated by a slash (``/``). The IP address is the network address, and the mask - can be either a single number, which means it's a *prefix*, or a string - representation of an IPv6 address. If it's the latter, the mask is - interpreted as a *net mask*. If no mask is provided, it's considered to - be ``/128``. + is a single number, which represents a *prefix*. If no mask is provided, + it's considered to be ``/128``. - For example, the following *address* specifications are equivalent: - ``2001:db00::0/24`` and ``2001:db00::0/ffff:ff00::``. + Note that currently expanded netmasks are not supported. That means + ``2001:db00::0/24`` is a valid argument while ``2001:db00::0/ffff:ff00::`` + not. 2. An integer that fits into 128 bits. This is equivalent to a single-address network, with the network address being *address* and @@ -631,6 +634,7 @@ so to avoid duplication they are only documented for :class:`IPv4Network`. .. attribute:: network_address .. attribute:: broadcast_address .. attribute:: hostmask + .. attribute:: netmask .. attribute:: with_prefixlen .. attribute:: compressed .. attribute:: exploded From webhook-mailer at python.org Wed Mar 7 23:35:55 2018 From: webhook-mailer at python.org (Miss Islington (bot)) Date: Thu, 08 Mar 2018 04:35:55 -0000 Subject: [Python-checkins] Fix some ipaddress documentation errors (GH-6021) Message-ID: <mailman.2.1520483756.2865.python-checkins@python.org> https://github.com/python/cpython/commit/c072e258858bf3c125f1bce3ec27ddb176300d04 commit: c072e258858bf3c125f1bce3ec27ddb176300d04 branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: GitHub <noreply at github.com> date: 2018-03-07T20:35:52-08:00 summary: Fix some ipaddress documentation errors (GH-6021) * fix a typo: documention -> documentation * fix the type of IPv?Network.hostmask * add documentation about IPv?Network.netmask * fix IPv6Network constructor doc that extended netmasks are not supported (cherry picked from commit e405096ea91f516d411095b6fea4eec9668eac88) Co-authored-by: Xiang Zhang <angwerzx at 126.com> files: M Doc/library/ipaddress.rst diff --git a/Doc/library/ipaddress.rst b/Doc/library/ipaddress.rst index 75c9107bd5eb..76177a0b23eb 100644 --- a/Doc/library/ipaddress.rst +++ b/Doc/library/ipaddress.rst @@ -236,7 +236,7 @@ write code that handles both IP versions correctly. groups consisting entirely of zeroes included. - For the following attributes, see the corresponding documention of the + For the following attributes, see the corresponding documentation of the :class:`IPv4Address` class: .. attribute:: packed @@ -442,7 +442,11 @@ so to avoid duplication they are only documented for :class:`IPv4Network`. .. attribute:: hostmask - The host mask, as a string. + The host mask, as an :class:`IPv4Address` object. + + .. attribute:: netmask + + The net mask, as an :class:`IPv4Address` object. .. attribute:: with_prefixlen .. attribute:: compressed @@ -588,13 +592,12 @@ so to avoid duplication they are only documented for :class:`IPv4Network`. 1. A string consisting of an IP address and an optional mask, separated by a slash (``/``). The IP address is the network address, and the mask - can be either a single number, which means it's a *prefix*, or a string - representation of an IPv6 address. If it's the latter, the mask is - interpreted as a *net mask*. If no mask is provided, it's considered to - be ``/128``. + is a single number, which represents a *prefix*. If no mask is provided, + it's considered to be ``/128``. - For example, the following *address* specifications are equivalent: - ``2001:db00::0/24`` and ``2001:db00::0/ffff:ff00::``. + Note that currently expanded netmasks are not supported. That means + ``2001:db00::0/24`` is a valid argument while ``2001:db00::0/ffff:ff00::`` + not. 2. An integer that fits into 128 bits. This is equivalent to a single-address network, with the network address being *address* and @@ -631,6 +634,7 @@ so to avoid duplication they are only documented for :class:`IPv4Network`. .. attribute:: network_address .. attribute:: broadcast_address .. attribute:: hostmask + .. attribute:: netmask .. attribute:: with_prefixlen .. attribute:: compressed .. attribute:: exploded From webhook-mailer at python.org Wed Mar 7 23:41:30 2018 From: webhook-mailer at python.org (Miss Islington (bot)) Date: Thu, 08 Mar 2018 04:41:30 -0000 Subject: [Python-checkins] Fix some ipaddress documentation errors (GH-6021) Message-ID: <mailman.3.1520484091.2865.python-checkins@python.org> https://github.com/python/cpython/commit/15425cf2456d816ad25723471375199277887de6 commit: 15425cf2456d816ad25723471375199277887de6 branch: 3.6 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: GitHub <noreply at github.com> date: 2018-03-07T20:41:27-08:00 summary: Fix some ipaddress documentation errors (GH-6021) * fix a typo: documention -> documentation * fix the type of IPv?Network.hostmask * add documentation about IPv?Network.netmask * fix IPv6Network constructor doc that extended netmasks are not supported (cherry picked from commit e405096ea91f516d411095b6fea4eec9668eac88) Co-authored-by: Xiang Zhang <angwerzx at 126.com> files: M Doc/library/ipaddress.rst diff --git a/Doc/library/ipaddress.rst b/Doc/library/ipaddress.rst index 50fb778dfbad..b3c691e444a1 100644 --- a/Doc/library/ipaddress.rst +++ b/Doc/library/ipaddress.rst @@ -234,7 +234,7 @@ write code that handles both IP versions correctly. groups consisting entirely of zeroes included. - For the following attributes, see the corresponding documention of the + For the following attributes, see the corresponding documentation of the :class:`IPv4Address` class: .. attribute:: packed @@ -440,7 +440,11 @@ so to avoid duplication they are only documented for :class:`IPv4Network`. .. attribute:: hostmask - The host mask, as a string. + The host mask, as an :class:`IPv4Address` object. + + .. attribute:: netmask + + The net mask, as an :class:`IPv4Address` object. .. attribute:: with_prefixlen .. attribute:: compressed @@ -561,13 +565,12 @@ so to avoid duplication they are only documented for :class:`IPv4Network`. 1. A string consisting of an IP address and an optional mask, separated by a slash (``/``). The IP address is the network address, and the mask - can be either a single number, which means it's a *prefix*, or a string - representation of an IPv6 address. If it's the latter, the mask is - interpreted as a *net mask*. If no mask is provided, it's considered to - be ``/128``. + is a single number, which represents a *prefix*. If no mask is provided, + it's considered to be ``/128``. - For example, the following *address* specifications are equivalent: - ``2001:db00::0/24`` and ``2001:db00::0/ffff:ff00::``. + Note that currently expanded netmasks are not supported. That means + ``2001:db00::0/24`` is a valid argument while ``2001:db00::0/ffff:ff00::`` + not. 2. An integer that fits into 128 bits. This is equivalent to a single-address network, with the network address being *address* and @@ -604,6 +607,7 @@ so to avoid duplication they are only documented for :class:`IPv4Network`. .. attribute:: network_address .. attribute:: broadcast_address .. attribute:: hostmask + .. attribute:: netmask .. attribute:: with_prefixlen .. attribute:: compressed .. attribute:: exploded From webhook-mailer at python.org Thu Mar 8 00:59:49 2018 From: webhook-mailer at python.org (Xiang Zhang) Date: Thu, 08 Mar 2018 05:59:49 -0000 Subject: [Python-checkins] Add two missing error checks in hamt.c (GH-5851) Message-ID: <mailman.5.1520488790.2865.python-checkins@python.org> https://github.com/python/cpython/commit/3c7ac7ea2098c672e50402d1d1b5ee9f14586437 commit: 3c7ac7ea2098c672e50402d1d1b5ee9f14586437 branch: master author: Xiang Zhang <angwerzx at 126.com> committer: GitHub <noreply at github.com> date: 2018-03-08T13:59:46+08:00 summary: Add two missing error checks in hamt.c (GH-5851) files: M Python/hamt.c diff --git a/Python/hamt.c b/Python/hamt.c index e54d3a4c55f9..53a85723745e 100644 --- a/Python/hamt.c +++ b/Python/hamt.c @@ -1510,6 +1510,9 @@ hamt_node_collision_without(PyHamtNode_Collision *self, PyHamtNode_Collision *new = (PyHamtNode_Collision *) hamt_node_collision_new( self->c_hash, Py_SIZE(self) - 2); + if (new == NULL) { + return W_ERROR; + } /* Copy all other keys from `self` to `new` */ Py_ssize_t i; @@ -1742,7 +1745,10 @@ hamt_node_array_assoc(PyHamtNode_Array *self, Set the key to it./ */ child_node = hamt_node_assoc( node, shift + 5, hash, key, val, added_leaf); - if (child_node == (PyHamtNode *)self) { + if (child_node == NULL) { + return NULL; + } + else if (child_node == (PyHamtNode *)self) { Py_DECREF(child_node); return (PyHamtNode *)self; } From webhook-mailer at python.org Thu Mar 8 01:21:37 2018 From: webhook-mailer at python.org (Miss Islington (bot)) Date: Thu, 08 Mar 2018 06:21:37 -0000 Subject: [Python-checkins] Add two missing error checks in hamt.c (GH-5851) Message-ID: <mailman.6.1520490098.2865.python-checkins@python.org> https://github.com/python/cpython/commit/e724dd4e5e9bdb888169d7cd2ca415a2fc2c29ab commit: e724dd4e5e9bdb888169d7cd2ca415a2fc2c29ab branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: GitHub <noreply at github.com> date: 2018-03-07T22:21:34-08:00 summary: Add two missing error checks in hamt.c (GH-5851) (cherry picked from commit 3c7ac7ea2098c672e50402d1d1b5ee9f14586437) Co-authored-by: Xiang Zhang <angwerzx at 126.com> files: M Python/hamt.c diff --git a/Python/hamt.c b/Python/hamt.c index e54d3a4c55f9..53a85723745e 100644 --- a/Python/hamt.c +++ b/Python/hamt.c @@ -1510,6 +1510,9 @@ hamt_node_collision_without(PyHamtNode_Collision *self, PyHamtNode_Collision *new = (PyHamtNode_Collision *) hamt_node_collision_new( self->c_hash, Py_SIZE(self) - 2); + if (new == NULL) { + return W_ERROR; + } /* Copy all other keys from `self` to `new` */ Py_ssize_t i; @@ -1742,7 +1745,10 @@ hamt_node_array_assoc(PyHamtNode_Array *self, Set the key to it./ */ child_node = hamt_node_assoc( node, shift + 5, hash, key, val, added_leaf); - if (child_node == (PyHamtNode *)self) { + if (child_node == NULL) { + return NULL; + } + else if (child_node == (PyHamtNode *)self) { Py_DECREF(child_node); return (PyHamtNode *)self; } From solipsis at pitrou.net Thu Mar 8 04:12:16 2018 From: solipsis at pitrou.net (solipsis at pitrou.net) Date: Thu, 08 Mar 2018 09:12:16 +0000 Subject: [Python-checkins] Daily reference leaks (4243df51fe43): sum=7 Message-ID: <20180308091216.1.C09F5A28839DCEB0@psf.io> results for 4243df51fe43 on branch "default" -------------------------------------------- test_collections leaked [7, -7, 1] memory blocks, sum=1 test_functools leaked [0, 3, 1] memory blocks, sum=4 test_multiprocessing_fork leaked [0, -1, 2] memory blocks, sum=1 test_multiprocessing_spawn leaked [2, -1, 0] memory blocks, sum=1 Command line was: ['./python', '-m', 'test.regrtest', '-uall', '-R', '3:3:/home/psf-users/antoine/refleaks/reflogKrlbtM', '--timeout', '7200'] From webhook-mailer at python.org Thu Mar 8 10:28:59 2018 From: webhook-mailer at python.org (Ned Deily) Date: Thu, 08 Mar 2018 15:28:59 -0000 Subject: [Python-checkins] [3.6] bpo-30353: Fix pass by value for structs on 64-bit Cygwin/MinGW (GH-1559) (GH-5954) Message-ID: <mailman.7.1520522942.2865.python-checkins@python.org> https://github.com/python/cpython/commit/2f3ba27185a369bcb6b36b13aa3518ffcc970ffa commit: 2f3ba27185a369bcb6b36b13aa3518ffcc970ffa branch: 3.6 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Ned Deily <nad at python.org> date: 2018-03-08T10:28:53-05:00 summary: [3.6] bpo-30353: Fix pass by value for structs on 64-bit Cygwin/MinGW (GH-1559) (GH-5954) (cherry picked from commit 9ba3aa4d02a110d1a1ea464a8aff3be7dd9c63c3) Co-authored-by: Erik Bray <erik.m.bray at gmail.com> files: A Misc/NEWS.d/next/Library/2018-03-08-09-54-01.bpo-30353.XdE5aM.rst M Lib/ctypes/test/test_as_parameter.py M Lib/ctypes/test/test_structures.py M Modules/_ctypes/_ctypes_test.c M Modules/_ctypes/callproc.c diff --git a/Lib/ctypes/test/test_as_parameter.py b/Lib/ctypes/test/test_as_parameter.py index 2a3484bec01b..a2640575a074 100644 --- a/Lib/ctypes/test/test_as_parameter.py +++ b/Lib/ctypes/test/test_as_parameter.py @@ -169,6 +169,10 @@ class S2H(Structure): s2h = dll.ret_2h_func(self.wrap(inp)) self.assertEqual((s2h.x, s2h.y), (99*2, 88*3)) + # Test also that the original struct was unmodified (i.e. was passed by + # value) + self.assertEqual((inp.x, inp.y), (99, 88)) + def test_struct_return_8H(self): class S8I(Structure): _fields_ = [("a", c_int), diff --git a/Lib/ctypes/test/test_structures.py b/Lib/ctypes/test/test_structures.py index 4ad5915ba27a..d1ea43bc7e3b 100644 --- a/Lib/ctypes/test/test_structures.py +++ b/Lib/ctypes/test/test_structures.py @@ -417,6 +417,28 @@ class X(Structure): self.assertEqual(s.second, 0xcafebabe) self.assertEqual(s.third, 0x0bad1dea) + def test_pass_by_value_in_register(self): + class X(Structure): + _fields_ = [ + ('first', c_uint), + ('second', c_uint) + ] + + s = X() + s.first = 0xdeadbeef + s.second = 0xcafebabe + dll = CDLL(_ctypes_test.__file__) + func = dll._testfunc_reg_struct_update_value + func.argtypes = (X,) + func.restype = None + func(s) + self.assertEqual(s.first, 0xdeadbeef) + self.assertEqual(s.second, 0xcafebabe) + got = X.in_dll(dll, "last_tfrsuv_arg") + self.assertEqual(s.first, got.first) + self.assertEqual(s.second, got.second) + + class PointerMemberTestCase(unittest.TestCase): def test(self): diff --git a/Misc/NEWS.d/next/Library/2018-03-08-09-54-01.bpo-30353.XdE5aM.rst b/Misc/NEWS.d/next/Library/2018-03-08-09-54-01.bpo-30353.XdE5aM.rst new file mode 100644 index 000000000000..ddb625c71652 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-03-08-09-54-01.bpo-30353.XdE5aM.rst @@ -0,0 +1 @@ +Fix ctypes pass-by-value for structs on 64-bit Cygwin/MinGW. diff --git a/Modules/_ctypes/_ctypes_test.c b/Modules/_ctypes/_ctypes_test.c index 6119ecdaf90d..f6af49aea385 100644 --- a/Modules/_ctypes/_ctypes_test.c +++ b/Modules/_ctypes/_ctypes_test.c @@ -57,6 +57,24 @@ _testfunc_large_struct_update_value(Test in) ((volatile Test *)&in)->third = 0x0badf00d; } +typedef struct { + unsigned int first; + unsigned int second; +} TestReg; + + +EXPORT(TestReg) last_tfrsuv_arg; + + +EXPORT(void) +_testfunc_reg_struct_update_value(TestReg in) +{ + last_tfrsuv_arg = in; + ((volatile TestReg *)&in)->first = 0x0badf00d; + ((volatile TestReg *)&in)->second = 0x0badf00d; +} + + EXPORT(void)testfunc_array(int values[4]) { printf("testfunc_array %d %d %d %d\n", diff --git a/Modules/_ctypes/callproc.c b/Modules/_ctypes/callproc.c index ff95dffb82c2..8edffabcdf09 100644 --- a/Modules/_ctypes/callproc.c +++ b/Modules/_ctypes/callproc.c @@ -1040,6 +1040,13 @@ GetComError(HRESULT errcode, GUID *riid, IUnknown *pIunk) } #endif +#if (defined(__x86_64__) && (defined(__MINGW64__) || defined(__CYGWIN__))) || \ + defined(__aarch64__) +#define CTYPES_PASS_BY_REF_HACK +#define POW2(x) (((x & ~(x - 1)) == x) ? x : 0) +#define IS_PASS_BY_REF(x) (x > 8 || !POW2(x)) +#endif + /* * Requirements, must be ensured by the caller: * - argtuple is tuple of arguments @@ -1137,8 +1144,20 @@ PyObject *_ctypes_callproc(PPROC pProc, } for (i = 0; i < argcount; ++i) { atypes[i] = args[i].ffi_type; - if (atypes[i]->type == FFI_TYPE_STRUCT - ) +#ifdef CTYPES_PASS_BY_REF_HACK + size_t size = atypes[i]->size; + if (IS_PASS_BY_REF(size)) { + void *tmp = alloca(size); + if (atypes[i]->type == FFI_TYPE_STRUCT) + memcpy(tmp, args[i].value.p, size); + else + memcpy(tmp, (void*)&args[i].value, size); + + avalues[i] = tmp; + } + else +#endif + if (atypes[i]->type == FFI_TYPE_STRUCT) avalues[i] = (void *)args[i].value.p; else avalues[i] = (void *)&args[i].value; From webhook-mailer at python.org Thu Mar 8 11:03:34 2018 From: webhook-mailer at python.org (Steve Dower) Date: Thu, 08 Mar 2018 16:03:34 -0000 Subject: [Python-checkins] bpo-33016: Fix potential use of uninitialized memory in nt._getfinalpathname (#6010) Message-ID: <mailman.8.1520525015.2865.python-checkins@python.org> https://github.com/python/cpython/commit/3b20d3454e8082e07dba93617793de5dc9237828 commit: 3b20d3454e8082e07dba93617793de5dc9237828 branch: master author: Alexey Izbyshev <izbyshev at users.noreply.github.com> committer: Steve Dower <steve.dower at microsoft.com> date: 2018-03-08T08:03:25-08:00 summary: bpo-33016: Fix potential use of uninitialized memory in nt._getfinalpathname (#6010) files: A Misc/NEWS.d/next/Windows/2018-03-07-01-33-33.bpo-33016.Z_Med0.rst M Modules/posixmodule.c diff --git a/Misc/NEWS.d/next/Windows/2018-03-07-01-33-33.bpo-33016.Z_Med0.rst b/Misc/NEWS.d/next/Windows/2018-03-07-01-33-33.bpo-33016.Z_Med0.rst new file mode 100644 index 000000000000..f4f78d489bf1 --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2018-03-07-01-33-33.bpo-33016.Z_Med0.rst @@ -0,0 +1 @@ +Fix potential use of uninitialized memory in nt._getfinalpathname diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index f4c01048cdae..e6721032f211 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -303,12 +303,6 @@ extern int lstat(const char *, struct stat *); #ifdef HAVE_PROCESS_H #include <process.h> #endif -#ifndef VOLUME_NAME_DOS -#define VOLUME_NAME_DOS 0x0 -#endif -#ifndef VOLUME_NAME_NT -#define VOLUME_NAME_NT 0x2 -#endif #ifndef IO_REPARSE_TAG_SYMLINK #define IO_REPARSE_TAG_SYMLINK (0xA000000CL) #endif @@ -3731,11 +3725,10 @@ os__getfinalpathname_impl(PyObject *module, path_t *path) /*[clinic end generated code: output=621a3c79bc29ebfa input=2b6b6c7cbad5fb84]*/ { HANDLE hFile; - int buf_size; - wchar_t *target_path; + wchar_t buf[MAXPATHLEN], *target_path = buf; + int buf_size = Py_ARRAY_LENGTH(buf); int result_length; PyObject *result; - const char *err = NULL; Py_BEGIN_ALLOW_THREADS hFile = CreateFileW( @@ -3747,55 +3740,52 @@ os__getfinalpathname_impl(PyObject *module, path_t *path) /* FILE_FLAG_BACKUP_SEMANTICS is required to open a directory */ FILE_FLAG_BACKUP_SEMANTICS, NULL); + Py_END_ALLOW_THREADS if (hFile == INVALID_HANDLE_VALUE) { - err = "CreateFileW"; - goto done1; + return win32_error_object("CreateFileW", path->object); } /* We have a good handle to the target, use it to determine the target path name. */ - buf_size = GetFinalPathNameByHandleW(hFile, 0, 0, VOLUME_NAME_NT); + while (1) { + Py_BEGIN_ALLOW_THREADS + result_length = GetFinalPathNameByHandleW(hFile, target_path, + buf_size, VOLUME_NAME_DOS); + Py_END_ALLOW_THREADS - if (!buf_size) { - err = "GetFinalPathNameByHandle"; - goto done1; - } -done1: - Py_END_ALLOW_THREADS - if (err) - return win32_error_object(err, path->object); + if (!result_length) { + result = win32_error_object("GetFinalPathNameByHandleW", + path->object); + goto cleanup; + } - target_path = PyMem_New(wchar_t, buf_size+1); - if(!target_path) - return PyErr_NoMemory(); + if (result_length < buf_size) { + break; + } - Py_BEGIN_ALLOW_THREADS - result_length = GetFinalPathNameByHandleW(hFile, target_path, - buf_size, VOLUME_NAME_DOS); - if (!result_length) { - err = "GetFinalPathNameByHandle"; - goto done2; - } + wchar_t *tmp; + tmp = PyMem_Realloc(target_path != buf ? target_path : NULL, + result_length * sizeof(*tmp)); + if (!tmp) { + result = PyErr_NoMemory(); + goto cleanup; + } - if (!CloseHandle(hFile)) { - err = "CloseHandle"; - goto done2; - } -done2: - Py_END_ALLOW_THREADS - if (err) { - PyMem_Free(target_path); - return win32_error_object(err, path->object); + buf_size = result_length; + target_path = tmp; } - target_path[result_length] = 0; result = PyUnicode_FromWideChar(target_path, result_length); - PyMem_Free(target_path); if (path->narrow) Py_SETREF(result, PyUnicode_EncodeFSDefault(result)); - return result; +cleanup: + if (target_path != buf) { + PyMem_Free(target_path); + } + CloseHandle(hFile); + return result; } /*[clinic input] From webhook-mailer at python.org Thu Mar 8 11:26:46 2018 From: webhook-mailer at python.org (Miss Islington (bot)) Date: Thu, 08 Mar 2018 16:26:46 -0000 Subject: [Python-checkins] bpo-33016: Fix potential use of uninitialized memory in nt._getfinalpathname (GH-6010) Message-ID: <mailman.9.1520526407.2865.python-checkins@python.org> https://github.com/python/cpython/commit/8c163bbf370f6f6cedd2c07f7d54c9b36c97d8f2 commit: 8c163bbf370f6f6cedd2c07f7d54c9b36c97d8f2 branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: GitHub <noreply at github.com> date: 2018-03-08T08:26:43-08:00 summary: bpo-33016: Fix potential use of uninitialized memory in nt._getfinalpathname (GH-6010) (cherry picked from commit 3b20d3454e8082e07dba93617793de5dc9237828) Co-authored-by: Alexey Izbyshev <izbyshev at users.noreply.github.com> files: A Misc/NEWS.d/next/Windows/2018-03-07-01-33-33.bpo-33016.Z_Med0.rst M Modules/posixmodule.c diff --git a/Misc/NEWS.d/next/Windows/2018-03-07-01-33-33.bpo-33016.Z_Med0.rst b/Misc/NEWS.d/next/Windows/2018-03-07-01-33-33.bpo-33016.Z_Med0.rst new file mode 100644 index 000000000000..f4f78d489bf1 --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2018-03-07-01-33-33.bpo-33016.Z_Med0.rst @@ -0,0 +1 @@ +Fix potential use of uninitialized memory in nt._getfinalpathname diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 6bf8c6ba7b7a..e8964ce33901 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -303,12 +303,6 @@ extern int lstat(const char *, struct stat *); #ifdef HAVE_PROCESS_H #include <process.h> #endif -#ifndef VOLUME_NAME_DOS -#define VOLUME_NAME_DOS 0x0 -#endif -#ifndef VOLUME_NAME_NT -#define VOLUME_NAME_NT 0x2 -#endif #ifndef IO_REPARSE_TAG_SYMLINK #define IO_REPARSE_TAG_SYMLINK (0xA000000CL) #endif @@ -3731,11 +3725,10 @@ os__getfinalpathname_impl(PyObject *module, path_t *path) /*[clinic end generated code: output=621a3c79bc29ebfa input=2b6b6c7cbad5fb84]*/ { HANDLE hFile; - int buf_size; - wchar_t *target_path; + wchar_t buf[MAXPATHLEN], *target_path = buf; + int buf_size = Py_ARRAY_LENGTH(buf); int result_length; PyObject *result; - const char *err = NULL; Py_BEGIN_ALLOW_THREADS hFile = CreateFileW( @@ -3747,55 +3740,52 @@ os__getfinalpathname_impl(PyObject *module, path_t *path) /* FILE_FLAG_BACKUP_SEMANTICS is required to open a directory */ FILE_FLAG_BACKUP_SEMANTICS, NULL); + Py_END_ALLOW_THREADS if (hFile == INVALID_HANDLE_VALUE) { - err = "CreateFileW"; - goto done1; + return win32_error_object("CreateFileW", path->object); } /* We have a good handle to the target, use it to determine the target path name. */ - buf_size = GetFinalPathNameByHandleW(hFile, 0, 0, VOLUME_NAME_NT); + while (1) { + Py_BEGIN_ALLOW_THREADS + result_length = GetFinalPathNameByHandleW(hFile, target_path, + buf_size, VOLUME_NAME_DOS); + Py_END_ALLOW_THREADS - if (!buf_size) { - err = "GetFinalPathNameByHandle"; - goto done1; - } -done1: - Py_END_ALLOW_THREADS - if (err) - return win32_error_object(err, path->object); + if (!result_length) { + result = win32_error_object("GetFinalPathNameByHandleW", + path->object); + goto cleanup; + } - target_path = PyMem_New(wchar_t, buf_size+1); - if(!target_path) - return PyErr_NoMemory(); + if (result_length < buf_size) { + break; + } - Py_BEGIN_ALLOW_THREADS - result_length = GetFinalPathNameByHandleW(hFile, target_path, - buf_size, VOLUME_NAME_DOS); - if (!result_length) { - err = "GetFinalPathNameByHandle"; - goto done2; - } + wchar_t *tmp; + tmp = PyMem_Realloc(target_path != buf ? target_path : NULL, + result_length * sizeof(*tmp)); + if (!tmp) { + result = PyErr_NoMemory(); + goto cleanup; + } - if (!CloseHandle(hFile)) { - err = "CloseHandle"; - goto done2; - } -done2: - Py_END_ALLOW_THREADS - if (err) { - PyMem_Free(target_path); - return win32_error_object(err, path->object); + buf_size = result_length; + target_path = tmp; } - target_path[result_length] = 0; result = PyUnicode_FromWideChar(target_path, result_length); - PyMem_Free(target_path); if (path->narrow) Py_SETREF(result, PyUnicode_EncodeFSDefault(result)); - return result; +cleanup: + if (target_path != buf) { + PyMem_Free(target_path); + } + CloseHandle(hFile); + return result; } /*[clinic input] From webhook-mailer at python.org Thu Mar 8 17:50:40 2018 From: webhook-mailer at python.org (Steve Dower) Date: Thu, 08 Mar 2018 22:50:40 -0000 Subject: [Python-checkins] bpo-33016: Fix potential use of uninitialized memory in nt._getfinalpathname (GH-6032) Message-ID: <mailman.10.1520549441.2865.python-checkins@python.org> https://github.com/python/cpython/commit/32efcd13069a89abf007373274ee1bc0909d1996 commit: 32efcd13069a89abf007373274ee1bc0909d1996 branch: 3.6 author: Steve Dower <steve.dower at microsoft.com> committer: GitHub <noreply at github.com> date: 2018-03-08T14:50:30-08:00 summary: bpo-33016: Fix potential use of uninitialized memory in nt._getfinalpathname (GH-6032) files: A Misc/NEWS.d/next/Windows/2018-03-07-01-33-33.bpo-33016.Z_Med0.rst M Modules/posixmodule.c diff --git a/Misc/NEWS.d/next/Windows/2018-03-07-01-33-33.bpo-33016.Z_Med0.rst b/Misc/NEWS.d/next/Windows/2018-03-07-01-33-33.bpo-33016.Z_Med0.rst new file mode 100644 index 000000000000..f4f78d489bf1 --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2018-03-07-01-33-33.bpo-33016.Z_Med0.rst @@ -0,0 +1 @@ +Fix potential use of uninitialized memory in nt._getfinalpathname diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 39ba030b5191..9c05acb24f0c 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -306,12 +306,6 @@ extern int lstat(const char *, struct stat *); #ifdef HAVE_PROCESS_H #include <process.h> #endif -#ifndef VOLUME_NAME_DOS -#define VOLUME_NAME_DOS 0x0 -#endif -#ifndef VOLUME_NAME_NT -#define VOLUME_NAME_NT 0x2 -#endif #ifndef IO_REPARSE_TAG_SYMLINK #define IO_REPARSE_TAG_SYMLINK (0xA000000CL) #endif @@ -3672,8 +3666,8 @@ os__getfinalpathname_impl(PyObject *module, PyObject *path) /*[clinic end generated code: output=9bd78d0e52782e75 input=71d5e89334891bf4]*/ { HANDLE hFile; - int buf_size; - wchar_t *target_path; + wchar_t buf[MAXPATHLEN], *target_path = buf; + int buf_size = Py_ARRAY_LENGTH(buf); int result_length; PyObject *result; const wchar_t *path_wchar; @@ -3682,6 +3676,7 @@ os__getfinalpathname_impl(PyObject *module, PyObject *path) if (path_wchar == NULL) return NULL; + Py_BEGIN_ALLOW_THREADS hFile = CreateFileW( path_wchar, 0, /* desired access */ @@ -3691,32 +3686,47 @@ os__getfinalpathname_impl(PyObject *module, PyObject *path) /* FILE_FLAG_BACKUP_SEMANTICS is required to open a directory */ FILE_FLAG_BACKUP_SEMANTICS, NULL); + Py_END_ALLOW_THREADS if(hFile == INVALID_HANDLE_VALUE) return win32_error_object("CreateFileW", path); /* We have a good handle to the target, use it to determine the target path name. */ - buf_size = GetFinalPathNameByHandleW(hFile, 0, 0, VOLUME_NAME_NT); + while (1) { + Py_BEGIN_ALLOW_THREADS + result_length = GetFinalPathNameByHandleW(hFile, target_path, + buf_size, VOLUME_NAME_DOS); + Py_END_ALLOW_THREADS - if(!buf_size) - return win32_error_object("GetFinalPathNameByHandle", path); + if (!result_length) { + result = win32_error_object("GetFinalPathNameByHandleW", path); + goto cleanup; + } - target_path = PyMem_New(wchar_t, buf_size+1); - if(!target_path) - return PyErr_NoMemory(); + if (result_length < buf_size) { + break; + } - result_length = GetFinalPathNameByHandleW(hFile, target_path, - buf_size, VOLUME_NAME_DOS); - if(!result_length) - return win32_error_object("GetFinalPathNamyByHandle", path); + wchar_t *tmp; + tmp = PyMem_Realloc(target_path != buf ? target_path : NULL, + result_length * sizeof(*tmp)); + if (!tmp) { + result = PyErr_NoMemory(); + goto cleanup; + } - if(!CloseHandle(hFile)) - return win32_error_object("CloseHandle", path); + buf_size = result_length; + target_path = tmp; + } - target_path[result_length] = 0; result = PyUnicode_FromWideChar(target_path, result_length); - PyMem_Free(target_path); + +cleanup: + if (target_path != buf) { + PyMem_Free(target_path); + } + CloseHandle(hFile); return result; } From webhook-mailer at python.org Thu Mar 8 21:22:04 2018 From: webhook-mailer at python.org (Xiang Zhang) Date: Fri, 09 Mar 2018 02:22:04 -0000 Subject: [Python-checkins] Clear possible exception before calling PyTuple_Pack in IMPORT_NAME (GH-6033) Message-ID: <mailman.12.1520562125.2865.python-checkins@python.org> https://github.com/python/cpython/commit/34bb88dc5bc447832db8c7ccdc173311e0685eab commit: 34bb88dc5bc447832db8c7ccdc173311e0685eab branch: 2.7 author: Xiang Zhang <angwerzx at 126.com> committer: GitHub <noreply at github.com> date: 2018-03-09T10:21:58+08:00 summary: Clear possible exception before calling PyTuple_Pack in IMPORT_NAME (GH-6033) files: M Python/ceval.c diff --git a/Python/ceval.c b/Python/ceval.c index bae158dc1402..b55b4d66880c 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -2598,6 +2598,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) TARGET(IMPORT_NAME) { + long res; w = GETITEM(names, oparg); x = PyDict_GetItemString(f->f_builtins, "__import__"); if (x == NULL) { @@ -2608,7 +2609,12 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) Py_INCREF(x); v = POP(); u = TOP(); - if (PyInt_AsLong(u) != -1 || PyErr_Occurred()) + res = PyInt_AsLong(u); + if (res != -1 || PyErr_Occurred()) { + if (res == -1) { + assert(PyErr_Occurred()); + PyErr_Clear(); + } w = PyTuple_Pack(5, w, f->f_globals, @@ -2616,6 +2622,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) Py_None : f->f_locals, v, u); + } else w = PyTuple_Pack(4, w, From solipsis at pitrou.net Fri Mar 9 04:13:10 2018 From: solipsis at pitrou.net (solipsis at pitrou.net) Date: Fri, 09 Mar 2018 09:13:10 +0000 Subject: [Python-checkins] Daily reference leaks (4243df51fe43): sum=3 Message-ID: <20180309091310.1.F0B5818E18759A84@psf.io> results for 4243df51fe43 on branch "default" -------------------------------------------- test_collections leaked [0, 7, -7] memory blocks, sum=0 test_functools leaked [0, 3, 1] memory blocks, sum=4 test_multiprocessing_forkserver leaked [-1, 2, 0] memory blocks, sum=1 test_multiprocessing_spawn leaked [-1, 1, -2] memory blocks, sum=-2 Command line was: ['./python', '-m', 'test.regrtest', '-uall', '-R', '3:3:/home/psf-users/antoine/refleaks/reflog_jjzPq', '--timeout', '7200'] From webhook-mailer at python.org Fri Mar 9 12:43:47 2018 From: webhook-mailer at python.org (Yury Selivanov) Date: Fri, 09 Mar 2018 17:43:47 -0000 Subject: [Python-checkins] bpo-32517: fix test_read_pty_output() hangs on macOS 10.13.2+ (GH-6037) Message-ID: <mailman.0.1520617429.1871.python-checkins@python.org> https://github.com/python/cpython/commit/12f74d8608c15cacd9d5786524e2be9ca36f007e commit: 12f74d8608c15cacd9d5786524e2be9ca36f007e branch: 3.6 author: Nathan Henrie <n8henrie at users.noreply.github.com> committer: Yury Selivanov <yury at magic.io> date: 2018-03-09T12:43:45-05:00 summary: bpo-32517: fix test_read_pty_output() hangs on macOS 10.13.2+ (GH-6037) test_asyncio hangs indefinitely on macOS 10.13.2+ on `read_pty_output()` using the KqueueSelector. Closing `proto.transport` (as is done in `write_pty_output()`) seems to fix it. files: A Misc/NEWS.d/next/Tests/2018-03-09-07-05-12.bpo-32517.ugc1iW.rst M Lib/test/test_asyncio/test_events.py M Misc/ACKS diff --git a/Lib/test/test_asyncio/test_events.py b/Lib/test/test_asyncio/test_events.py index 27781a2d91b3..2c4629ab4924 100644 --- a/Lib/test/test_asyncio/test_events.py +++ b/Lib/test/test_asyncio/test_events.py @@ -1533,6 +1533,7 @@ def connect(): self.assertEqual(5, proto.nbytes) os.close(slave) + proto.transport.close() self.loop.run_until_complete(proto.done) self.assertEqual( ['INITIAL', 'CONNECTED', 'EOF', 'CLOSED'], proto.state) diff --git a/Misc/ACKS b/Misc/ACKS index 9403e110675a..b2033ee9d3f3 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -614,6 +614,7 @@ Thomas Heller Malte Helmert Lance Finn Helsten Jonathan Hendry +Nathan Henrie Michael Henry James Henstridge Kasun Herath diff --git a/Misc/NEWS.d/next/Tests/2018-03-09-07-05-12.bpo-32517.ugc1iW.rst b/Misc/NEWS.d/next/Tests/2018-03-09-07-05-12.bpo-32517.ugc1iW.rst new file mode 100644 index 000000000000..43f148f06ecb --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2018-03-09-07-05-12.bpo-32517.ugc1iW.rst @@ -0,0 +1,2 @@ +Fix failing ``test_asyncio`` on macOS 10.12.2+ due to transport of +``KqueueSelector`` loop was not being closed. From webhook-mailer at python.org Fri Mar 9 13:14:19 2018 From: webhook-mailer at python.org (Miss Islington (bot)) Date: Fri, 09 Mar 2018 18:14:19 -0000 Subject: [Python-checkins] bpo-32517: fix test_read_pty_output() hangs on macOS 10.13.2+ (GH-6037) Message-ID: <mailman.1.1520619260.1871.python-checkins@python.org> https://github.com/python/cpython/commit/3c39beb65d003b209a98a79af4aaf94ba16d0d29 commit: 3c39beb65d003b209a98a79af4aaf94ba16d0d29 branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: GitHub <noreply at github.com> date: 2018-03-09T10:14:14-08:00 summary: bpo-32517: fix test_read_pty_output() hangs on macOS 10.13.2+ (GH-6037) test_asyncio hangs indefinitely on macOS 10.13.2+ on `read_pty_output()` using the KqueueSelector. Closing `proto.transport` (as is done in `write_pty_output()`) seems to fix it. (cherry picked from commit 12f74d8608c15cacd9d5786524e2be9ca36f007e) Co-authored-by: Nathan Henrie <n8henrie at users.noreply.github.com> files: A Misc/NEWS.d/next/Tests/2018-03-09-07-05-12.bpo-32517.ugc1iW.rst M Lib/test/test_asyncio/test_events.py M Misc/ACKS diff --git a/Lib/test/test_asyncio/test_events.py b/Lib/test/test_asyncio/test_events.py index 6accbdae8b3e..f8bb01bd4e72 100644 --- a/Lib/test/test_asyncio/test_events.py +++ b/Lib/test/test_asyncio/test_events.py @@ -1502,6 +1502,7 @@ def test_read_pty_output(self): self.assertEqual(5, proto.nbytes) os.close(slave) + proto.transport.close() self.loop.run_until_complete(proto.done) self.assertEqual( ['INITIAL', 'CONNECTED', 'EOF', 'CLOSED'], proto.state) diff --git a/Misc/ACKS b/Misc/ACKS index d598520e9385..b0a9136170d6 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -628,6 +628,7 @@ Thomas Heller Malte Helmert Lance Finn Helsten Jonathan Hendry +Nathan Henrie Michael Henry James Henstridge Kasun Herath diff --git a/Misc/NEWS.d/next/Tests/2018-03-09-07-05-12.bpo-32517.ugc1iW.rst b/Misc/NEWS.d/next/Tests/2018-03-09-07-05-12.bpo-32517.ugc1iW.rst new file mode 100644 index 000000000000..43f148f06ecb --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2018-03-09-07-05-12.bpo-32517.ugc1iW.rst @@ -0,0 +1,2 @@ +Fix failing ``test_asyncio`` on macOS 10.12.2+ due to transport of +``KqueueSelector`` loop was not being closed. From webhook-mailer at python.org Fri Mar 9 14:57:31 2018 From: webhook-mailer at python.org (Mariatta) Date: Fri, 09 Mar 2018 19:57:31 -0000 Subject: [Python-checkins] controlflow: Use full example for "5 through 9" (GH-5907) Message-ID: <mailman.2.1520625452.1871.python-checkins@python.org> https://github.com/python/cpython/commit/83d7062d2dc5eacfef578e072bca4747c346fdae commit: 83d7062d2dc5eacfef578e072bca4747c346fdae branch: master author: Steven M. Vascellaro <S.Vascellaro at gmail.com> committer: Mariatta <Mariatta at users.noreply.github.com> date: 2018-03-09T11:57:21-08:00 summary: controlflow: Use full example for "5 through 9" (GH-5907) Replace example result of "5 through 9" with complete list: "5, 6, 7, 8, 9". This format is more consistent with the surrounding examples. files: M Doc/tutorial/controlflow.rst diff --git a/Doc/tutorial/controlflow.rst b/Doc/tutorial/controlflow.rst index 36b093950e10..30ef159f8b36 100644 --- a/Doc/tutorial/controlflow.rst +++ b/Doc/tutorial/controlflow.rst @@ -105,7 +105,7 @@ is possible to let the range start at another number, or to specify a different increment (even negative; sometimes this is called the 'step'):: range(5, 10) - 5 through 9 + 5, 6, 7, 8, 9 range(0, 10, 3) 0, 3, 6, 9 From webhook-mailer at python.org Fri Mar 9 15:03:26 2018 From: webhook-mailer at python.org (Brett Cannon) Date: Fri, 09 Mar 2018 20:03:26 -0000 Subject: [Python-checkins] bpo-32758: Warn that ast.parse() and ast.literal_eval() can segfault the interpreter (GH-5960) Message-ID: <mailman.3.1520625807.1871.python-checkins@python.org> https://github.com/python/cpython/commit/7a7f100eb352d08938ee0f5ba59c18f56dc4a7b5 commit: 7a7f100eb352d08938ee0f5ba59c18f56dc4a7b5 branch: master author: Brett Cannon <brettcannon at users.noreply.github.com> committer: GitHub <noreply at github.com> date: 2018-03-09T12:03:22-08:00 summary: bpo-32758: Warn that ast.parse() and ast.literal_eval() can segfault the interpreter (GH-5960) files: M Doc/library/ast.rst diff --git a/Doc/library/ast.rst b/Doc/library/ast.rst index b7f610ba8b2a..a75a6afbf2d7 100644 --- a/Doc/library/ast.rst +++ b/Doc/library/ast.rst @@ -113,6 +113,11 @@ and classes for traversing abstract syntax trees: Parse the source into an AST node. Equivalent to ``compile(source, filename, mode, ast.PyCF_ONLY_AST)``. + .. warning:: + It is possible to crash the Python interpreter with a + sufficiently large/complex string due to stack depth limitations + in Python's AST compiler. + .. function:: literal_eval(node_or_string) @@ -126,6 +131,11 @@ and classes for traversing abstract syntax trees: capable of evaluating arbitrarily complex expressions, for example involving operators or indexing. + .. warning:: + It is possible to crash the Python interpreter with a + sufficiently large/complex string due to stack depth limitations + in Python's AST compiler. + .. versionchanged:: 3.2 Now allows bytes and set literals. From webhook-mailer at python.org Fri Mar 9 15:11:33 2018 From: webhook-mailer at python.org (Miss Islington (bot)) Date: Fri, 09 Mar 2018 20:11:33 -0000 Subject: [Python-checkins] controlflow: Use full example for "5 through 9" (GH-5907) Message-ID: <mailman.4.1520626294.1871.python-checkins@python.org> https://github.com/python/cpython/commit/3f7d0b69f2175ff74157673148b2e45b7a2498b6 commit: 3f7d0b69f2175ff74157673148b2e45b7a2498b6 branch: 3.6 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: GitHub <noreply at github.com> date: 2018-03-09T12:11:30-08:00 summary: controlflow: Use full example for "5 through 9" (GH-5907) Replace example result of "5 through 9" with complete list: "5, 6, 7, 8, 9". This format is more consistent with the surrounding examples. (cherry picked from commit 83d7062d2dc5eacfef578e072bca4747c346fdae) Co-authored-by: Steven M. Vascellaro <S.Vascellaro at gmail.com> files: M Doc/tutorial/controlflow.rst diff --git a/Doc/tutorial/controlflow.rst b/Doc/tutorial/controlflow.rst index 36b093950e10..30ef159f8b36 100644 --- a/Doc/tutorial/controlflow.rst +++ b/Doc/tutorial/controlflow.rst @@ -105,7 +105,7 @@ is possible to let the range start at another number, or to specify a different increment (even negative; sometimes this is called the 'step'):: range(5, 10) - 5 through 9 + 5, 6, 7, 8, 9 range(0, 10, 3) 0, 3, 6, 9 From webhook-mailer at python.org Fri Mar 9 15:35:24 2018 From: webhook-mailer at python.org (Brett Cannon) Date: Fri, 09 Mar 2018 20:35:24 -0000 Subject: [Python-checkins] bpo-32758: Warn that ast.parse() and ast.literal_eval() can segfault the interpreter (GH-5960) (GH-6042) Message-ID: <mailman.5.1520627726.1871.python-checkins@python.org> https://github.com/python/cpython/commit/b316c44b0105d11a80ff971636143735f3655bbf commit: b316c44b0105d11a80ff971636143735f3655bbf branch: 3.6 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Brett Cannon <brettcannon at users.noreply.github.com> date: 2018-03-09T12:35:14-08:00 summary: bpo-32758: Warn that ast.parse() and ast.literal_eval() can segfault the interpreter (GH-5960) (GH-6042) (cherry picked from commit 7a7f100eb352d08938ee0f5ba59c18f56dc4a7b5) Co-authored-by: Brett Cannon <brettcannon at users.noreply.github.com> files: M Doc/library/ast.rst diff --git a/Doc/library/ast.rst b/Doc/library/ast.rst index 8d4ae2cc616f..6376f5fe4410 100644 --- a/Doc/library/ast.rst +++ b/Doc/library/ast.rst @@ -113,6 +113,11 @@ and classes for traversing abstract syntax trees: Parse the source into an AST node. Equivalent to ``compile(source, filename, mode, ast.PyCF_ONLY_AST)``. + .. warning:: + It is possible to crash the Python interpreter with a + sufficiently large/complex string due to stack depth limitations + in Python's AST compiler. + .. function:: literal_eval(node_or_string) @@ -126,6 +131,11 @@ and classes for traversing abstract syntax trees: capable of evaluating arbitrarily complex expressions, for example involving operators or indexing. + .. warning:: + It is possible to crash the Python interpreter with a + sufficiently large/complex string due to stack depth limitations + in Python's AST compiler. + .. versionchanged:: 3.2 Now allows bytes and set literals. From webhook-mailer at python.org Fri Mar 9 15:35:45 2018 From: webhook-mailer at python.org (Brett Cannon) Date: Fri, 09 Mar 2018 20:35:45 -0000 Subject: [Python-checkins] bpo-32758: Warn that ast.parse() and ast.literal_eval() can segfault the interpreter (GH-5960) (GH-6041) Message-ID: <mailman.6.1520627747.1871.python-checkins@python.org> https://github.com/python/cpython/commit/f2fffd41b42d88fe36b483852ae33d5a415b7082 commit: f2fffd41b42d88fe36b483852ae33d5a415b7082 branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Brett Cannon <brettcannon at users.noreply.github.com> date: 2018-03-09T12:35:42-08:00 summary: bpo-32758: Warn that ast.parse() and ast.literal_eval() can segfault the interpreter (GH-5960) (GH-6041) (cherry picked from commit 7a7f100eb352d08938ee0f5ba59c18f56dc4a7b5) Co-authored-by: Brett Cannon <brettcannon at users.noreply.github.com> files: M Doc/library/ast.rst diff --git a/Doc/library/ast.rst b/Doc/library/ast.rst index b7f610ba8b2a..a75a6afbf2d7 100644 --- a/Doc/library/ast.rst +++ b/Doc/library/ast.rst @@ -113,6 +113,11 @@ and classes for traversing abstract syntax trees: Parse the source into an AST node. Equivalent to ``compile(source, filename, mode, ast.PyCF_ONLY_AST)``. + .. warning:: + It is possible to crash the Python interpreter with a + sufficiently large/complex string due to stack depth limitations + in Python's AST compiler. + .. function:: literal_eval(node_or_string) @@ -126,6 +131,11 @@ and classes for traversing abstract syntax trees: capable of evaluating arbitrarily complex expressions, for example involving operators or indexing. + .. warning:: + It is possible to crash the Python interpreter with a + sufficiently large/complex string due to stack depth limitations + in Python's AST compiler. + .. versionchanged:: 3.2 Now allows bytes and set literals. From webhook-mailer at python.org Fri Mar 9 16:13:36 2018 From: webhook-mailer at python.org (Brett Cannon) Date: Fri, 09 Mar 2018 21:13:36 -0000 Subject: [Python-checkins] Warn that compile() can crash when compiling to an AST object (GH-6043) Message-ID: <mailman.7.1520630017.1871.python-checkins@python.org> https://github.com/python/cpython/commit/f7a6ff6fcab32a53f262ba3f8a072c27afc330d7 commit: f7a6ff6fcab32a53f262ba3f8a072c27afc330d7 branch: master author: Brett Cannon <brettcannon at users.noreply.github.com> committer: GitHub <noreply at github.com> date: 2018-03-09T13:13:32-08:00 summary: Warn that compile() can crash when compiling to an AST object (GH-6043) files: M Doc/library/functions.rst diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index bfb813cf3906..3ddd280f77eb 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -274,6 +274,12 @@ are always available. They are listed here in alphabetical order. character. This is to facilitate detection of incomplete and complete statements in the :mod:`code` module. + .. warning:: + + It is possible to crash the Python interpreter with a + sufficiently large/complex string when compiling to an AST + object due to stack depth limitations in Python's AST compiler. + .. versionchanged:: 3.2 Allowed use of Windows and Mac newlines. Also input in ``'exec'`` mode does not have to end in a newline anymore. Added the *optimize* parameter. From webhook-mailer at python.org Fri Mar 9 16:40:29 2018 From: webhook-mailer at python.org (Brett Cannon) Date: Fri, 09 Mar 2018 21:40:29 -0000 Subject: [Python-checkins] Warn that compile() can crash when compiling to an AST object (GH-6043) (GH-6045) Message-ID: <mailman.8.1520631630.1871.python-checkins@python.org> https://github.com/python/cpython/commit/fc5e0956fe492551c658ebe325ba77c0aa4f2adb commit: fc5e0956fe492551c658ebe325ba77c0aa4f2adb branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Brett Cannon <brettcannon at users.noreply.github.com> date: 2018-03-09T13:40:26-08:00 summary: Warn that compile() can crash when compiling to an AST object (GH-6043) (GH-6045) (cherry picked from commit f7a6ff6fcab32a53f262ba3f8a072c27afc330d7) Co-authored-by: Brett Cannon <brettcannon at users.noreply.github.com> files: M Doc/library/functions.rst diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index bfb813cf3906..3ddd280f77eb 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -274,6 +274,12 @@ are always available. They are listed here in alphabetical order. character. This is to facilitate detection of incomplete and complete statements in the :mod:`code` module. + .. warning:: + + It is possible to crash the Python interpreter with a + sufficiently large/complex string when compiling to an AST + object due to stack depth limitations in Python's AST compiler. + .. versionchanged:: 3.2 Allowed use of Windows and Mac newlines. Also input in ``'exec'`` mode does not have to end in a newline anymore. Added the *optimize* parameter. From webhook-mailer at python.org Fri Mar 9 16:40:41 2018 From: webhook-mailer at python.org (Brett Cannon) Date: Fri, 09 Mar 2018 21:40:41 -0000 Subject: [Python-checkins] Warn that compile() can crash when compiling to an AST object (GH-6043) (GH-6046) Message-ID: <mailman.9.1520631642.1871.python-checkins@python.org> https://github.com/python/cpython/commit/19b42fe97cc15ff123f7ccb61a26fed817e850f7 commit: 19b42fe97cc15ff123f7ccb61a26fed817e850f7 branch: 3.6 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Brett Cannon <brettcannon at users.noreply.github.com> date: 2018-03-09T13:40:39-08:00 summary: Warn that compile() can crash when compiling to an AST object (GH-6043) (GH-6046) (cherry picked from commit f7a6ff6fcab32a53f262ba3f8a072c27afc330d7) Co-authored-by: Brett Cannon <brettcannon at users.noreply.github.com> files: M Doc/library/functions.rst diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index 3b479bd64294..bc4203081ed0 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -260,6 +260,12 @@ are always available. They are listed here in alphabetical order. character. This is to facilitate detection of incomplete and complete statements in the :mod:`code` module. + .. warning:: + + It is possible to crash the Python interpreter with a + sufficiently large/complex string when compiling to an AST + object due to stack depth limitations in Python's AST compiler. + .. versionchanged:: 3.2 Allowed use of Windows and Mac newlines. Also input in ``'exec'`` mode does not have to end in a newline anymore. Added the *optimize* parameter. From webhook-mailer at python.org Fri Mar 9 18:58:43 2018 From: webhook-mailer at python.org (Brett Cannon) Date: Fri, 09 Mar 2018 23:58:43 -0000 Subject: [Python-checkins] Warn that dbm.dumb.open() can crash Python (GH-6047) Message-ID: <mailman.10.1520639924.1871.python-checkins@python.org> https://github.com/python/cpython/commit/10485ebd40669d3e17ab4f477c8c898543bcccd1 commit: 10485ebd40669d3e17ab4f477c8c898543bcccd1 branch: master author: Brett Cannon <brettcannon at users.noreply.github.com> committer: GitHub <noreply at github.com> date: 2018-03-09T15:58:40-08:00 summary: Warn that dbm.dumb.open() can crash Python (GH-6047) files: M Doc/library/dbm.rst diff --git a/Doc/library/dbm.rst b/Doc/library/dbm.rst index 1abc36c04a74..ab45cac830e2 100644 --- a/Doc/library/dbm.rst +++ b/Doc/library/dbm.rst @@ -361,6 +361,11 @@ The module defines the following: database has to be created. It defaults to octal ``0o666`` (and will be modified by the prevailing umask). + .. warning:: + It is possible to crash the Python interpreter when loading a database + with a sufficiently large/complex entry due to stack depth limitations in + Python's AST compiler. + .. versionchanged:: 3.5 :func:`.open` always creates a new database when the flag has the value ``'n'``. From webhook-mailer at python.org Fri Mar 9 19:12:57 2018 From: webhook-mailer at python.org (Brett Cannon) Date: Sat, 10 Mar 2018 00:12:57 -0000 Subject: [Python-checkins] Warn that dbm.dumb.open() can crash Python (GH-6047) (GH-6048) Message-ID: <mailman.11.1520640778.1871.python-checkins@python.org> https://github.com/python/cpython/commit/6826589415408e68e7872bfde30abbdff17772b2 commit: 6826589415408e68e7872bfde30abbdff17772b2 branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Brett Cannon <brettcannon at users.noreply.github.com> date: 2018-03-09T16:12:54-08:00 summary: Warn that dbm.dumb.open() can crash Python (GH-6047) (GH-6048) (cherry picked from commit 10485ebd40669d3e17ab4f477c8c898543bcccd1) Co-authored-by: Brett Cannon <brettcannon at users.noreply.github.com> files: M Doc/library/dbm.rst diff --git a/Doc/library/dbm.rst b/Doc/library/dbm.rst index 32e80b2cf6ed..0150f5d5c6e8 100644 --- a/Doc/library/dbm.rst +++ b/Doc/library/dbm.rst @@ -347,6 +347,11 @@ The module defines the following: database has to be created. It defaults to octal ``0o666`` (and will be modified by the prevailing umask). + .. warning:: + It is possible to crash the Python interpreter when loading a database + with a sufficiently large/complex entry due to stack depth limitations in + Python's AST compiler. + .. versionchanged:: 3.5 :func:`.open` always creates a new database when the flag has the value ``'n'``. From webhook-mailer at python.org Sat Mar 10 03:44:20 2018 From: webhook-mailer at python.org (Nick Coghlan) Date: Sat, 10 Mar 2018 08:44:20 -0000 Subject: [Python-checkins] bpo-26701: Add documentation for __trunc__ (GH-6022) Message-ID: <mailman.12.1520671461.1871.python-checkins@python.org> https://github.com/python/cpython/commit/308eab979d153f1ab934383dc08bc4546ced8b6c commit: 308eab979d153f1ab934383dc08bc4546ced8b6c branch: master author: Eric Appelt <eric.appelt at gmail.com> committer: Nick Coghlan <ncoghlan at gmail.com> date: 2018-03-10T18:44:12+10:00 summary: bpo-26701: Add documentation for __trunc__ (GH-6022) `int` fails back to `__trunc__` is `__int__` isn't defined, so cover that in the docs. files: M Doc/library/functions.rst M Doc/library/math.rst M Doc/reference/datamodel.rst diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index 3ddd280f77eb..c3b638572338 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -731,8 +731,11 @@ are always available. They are listed here in alphabetical order. Return an integer object constructed from a number or string *x*, or return ``0`` if no arguments are given. If *x* is a number, return - :meth:`x.__int__() <object.__int__>`. For floating point numbers, this - truncates towards zero. + :meth:`x.__int__() <object.__int__>`. If *x* defines + :meth:`x.__trunc__() <object.__trunc__>` but not + :meth:`x.__int__() <object.__int__>`, then return + if :meth:`x.__trunc__() <object.__trunc__>`. For floating point numbers, + this truncates towards zero. If *x* is not a number or if *base* is given, then *x* must be a string, :class:`bytes`, or :class:`bytearray` instance representing an :ref:`integer diff --git a/Doc/library/math.rst b/Doc/library/math.rst index 55eb41b86f5d..33aec573a1f2 100644 --- a/Doc/library/math.rst +++ b/Doc/library/math.rst @@ -203,7 +203,7 @@ Number-theoretic and representation functions Return the :class:`~numbers.Real` value *x* truncated to an :class:`~numbers.Integral` (usually an integer). Delegates to - ``x.__trunc__()``. + :meth:`x.__trunc__() <object.__trunc__>`. Note that :func:`frexp` and :func:`modf` have a different call/return pattern diff --git a/Doc/reference/datamodel.rst b/Doc/reference/datamodel.rst index 8b127a0b7e81..26ad7b8c050c 100644 --- a/Doc/reference/datamodel.rst +++ b/Doc/reference/datamodel.rst @@ -2377,6 +2377,15 @@ left undefined. of the appropriate type. +.. method:: object.__trunc__(self) + + Called to implement :meth:`math.trunc`. Should return the value of the + object truncated to a :class:`numbers.Integral` (typically an + :class:`int`). If a class defines :meth:`__trunc__` but not + :meth:`__int__`, then :meth:`__trunc__` is called to implement the + built-in function :func:`int`. + + .. method:: object.__index__(self) Called to implement :func:`operator.index`, and whenever Python needs to From solipsis at pitrou.net Sat Mar 10 04:08:34 2018 From: solipsis at pitrou.net (solipsis at pitrou.net) Date: Sat, 10 Mar 2018 09:08:34 +0000 Subject: [Python-checkins] Daily reference leaks (4243df51fe43): sum=11 Message-ID: <20180310090834.1.41C0C46E65B1583E@psf.io> results for 4243df51fe43 on branch "default" -------------------------------------------- test_collections leaked [0, 0, 7] memory blocks, sum=7 test_functools leaked [0, 3, 1] memory blocks, sum=4 Command line was: ['./python', '-m', 'test.regrtest', '-uall', '-R', '3:3:/home/psf-users/antoine/refleaks/reflogohDkjm', '--timeout', '7200'] From webhook-mailer at python.org Sat Mar 10 10:10:40 2018 From: webhook-mailer at python.org (Nick Coghlan) Date: Sat, 10 Mar 2018 15:10:40 -0000 Subject: [Python-checkins] bpo-26701: Add documentation for __trunc__ (GH-6049) Message-ID: <mailman.13.1520694641.1871.python-checkins@python.org> https://github.com/python/cpython/commit/f34e0d60e27acff3f9604ec63e9de36878c3743a commit: f34e0d60e27acff3f9604ec63e9de36878c3743a branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Nick Coghlan <ncoghlan at gmail.com> date: 2018-03-11T01:10:32+10:00 summary: bpo-26701: Add documentation for __trunc__ (GH-6049) `int` fails back to `__trunc__` is `__int__` isn't defined, so cover that in the docs. (cherry picked from commit 308eab979d153f1ab934383dc08bc4546ced8b6c) Co-authored-by: Eric Appelt <eric.appelt at gmail.com> files: M Doc/library/functions.rst M Doc/library/math.rst M Doc/reference/datamodel.rst diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index 3ddd280f77eb..c3b638572338 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -731,8 +731,11 @@ are always available. They are listed here in alphabetical order. Return an integer object constructed from a number or string *x*, or return ``0`` if no arguments are given. If *x* is a number, return - :meth:`x.__int__() <object.__int__>`. For floating point numbers, this - truncates towards zero. + :meth:`x.__int__() <object.__int__>`. If *x* defines + :meth:`x.__trunc__() <object.__trunc__>` but not + :meth:`x.__int__() <object.__int__>`, then return + if :meth:`x.__trunc__() <object.__trunc__>`. For floating point numbers, + this truncates towards zero. If *x* is not a number or if *base* is given, then *x* must be a string, :class:`bytes`, or :class:`bytearray` instance representing an :ref:`integer diff --git a/Doc/library/math.rst b/Doc/library/math.rst index 55eb41b86f5d..33aec573a1f2 100644 --- a/Doc/library/math.rst +++ b/Doc/library/math.rst @@ -203,7 +203,7 @@ Number-theoretic and representation functions Return the :class:`~numbers.Real` value *x* truncated to an :class:`~numbers.Integral` (usually an integer). Delegates to - ``x.__trunc__()``. + :meth:`x.__trunc__() <object.__trunc__>`. Note that :func:`frexp` and :func:`modf` have a different call/return pattern diff --git a/Doc/reference/datamodel.rst b/Doc/reference/datamodel.rst index 8b127a0b7e81..26ad7b8c050c 100644 --- a/Doc/reference/datamodel.rst +++ b/Doc/reference/datamodel.rst @@ -2377,6 +2377,15 @@ left undefined. of the appropriate type. +.. method:: object.__trunc__(self) + + Called to implement :meth:`math.trunc`. Should return the value of the + object truncated to a :class:`numbers.Integral` (typically an + :class:`int`). If a class defines :meth:`__trunc__` but not + :meth:`__int__`, then :meth:`__trunc__` is called to implement the + built-in function :func:`int`. + + .. method:: object.__index__(self) Called to implement :func:`operator.index`, and whenever Python needs to From webhook-mailer at python.org Sat Mar 10 10:10:48 2018 From: webhook-mailer at python.org (Nick Coghlan) Date: Sat, 10 Mar 2018 15:10:48 -0000 Subject: [Python-checkins] bpo-26701: Add documentation for __trunc__ (GH-6050) Message-ID: <mailman.14.1520694650.1871.python-checkins@python.org> https://github.com/python/cpython/commit/1028ca4f04c14cf40a8f3e7951623a96ec0143c2 commit: 1028ca4f04c14cf40a8f3e7951623a96ec0143c2 branch: 3.6 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Nick Coghlan <ncoghlan at gmail.com> date: 2018-03-11T01:10:45+10:00 summary: bpo-26701: Add documentation for __trunc__ (GH-6050) `int` fails back to `__trunc__` is `__int__` isn't defined, so cover that in the docs. (cherry picked from commit 308eab979d153f1ab934383dc08bc4546ced8b6c) Co-authored-by: Eric Appelt <eric.appelt at gmail.com> files: M Doc/library/functions.rst M Doc/library/math.rst M Doc/reference/datamodel.rst diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index bc4203081ed0..9cb6b0e1b5ca 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -717,8 +717,11 @@ are always available. They are listed here in alphabetical order. Return an integer object constructed from a number or string *x*, or return ``0`` if no arguments are given. If *x* is a number, return - :meth:`x.__int__() <object.__int__>`. For floating point numbers, this - truncates towards zero. + :meth:`x.__int__() <object.__int__>`. If *x* defines + :meth:`x.__trunc__() <object.__trunc__>` but not + :meth:`x.__int__() <object.__int__>`, then return + if :meth:`x.__trunc__() <object.__trunc__>`. For floating point numbers, + this truncates towards zero. If *x* is not a number or if *base* is given, then *x* must be a string, :class:`bytes`, or :class:`bytearray` instance representing an :ref:`integer diff --git a/Doc/library/math.rst b/Doc/library/math.rst index da2b8cc58627..0f36a3fa99eb 100644 --- a/Doc/library/math.rst +++ b/Doc/library/math.rst @@ -179,7 +179,7 @@ Number-theoretic and representation functions Return the :class:`~numbers.Real` value *x* truncated to an :class:`~numbers.Integral` (usually an integer). Delegates to - ``x.__trunc__()``. + :meth:`x.__trunc__() <object.__trunc__>`. Note that :func:`frexp` and :func:`modf` have a different call/return pattern diff --git a/Doc/reference/datamodel.rst b/Doc/reference/datamodel.rst index c95f4a970930..90f3a2aa0dcd 100644 --- a/Doc/reference/datamodel.rst +++ b/Doc/reference/datamodel.rst @@ -2312,6 +2312,15 @@ left undefined. of the appropriate type. +.. method:: object.__trunc__(self) + + Called to implement :meth:`math.trunc`. Should return the value of the + object truncated to a :class:`numbers.Integral` (typically an + :class:`int`). If a class defines :meth:`__trunc__` but not + :meth:`__int__`, then :meth:`__trunc__` is called to implement the + built-in function :func:`int`. + + .. method:: object.__index__(self) Called to implement :func:`operator.index`, and whenever Python needs to From webhook-mailer at python.org Sat Mar 10 10:18:35 2018 From: webhook-mailer at python.org (Serhiy Storchaka) Date: Sat, 10 Mar 2018 15:18:35 -0000 Subject: [Python-checkins] bpo-26701: Improve documentation for the rounding special methods. (#6054) Message-ID: <mailman.15.1520695117.1871.python-checkins@python.org> https://github.com/python/cpython/commit/496431ffb6c29719332bf2af773349e8dd85e45a commit: 496431ffb6c29719332bf2af773349e8dd85e45a branch: master author: Serhiy Storchaka <storchaka at gmail.com> committer: GitHub <noreply at github.com> date: 2018-03-10T17:18:32+02:00 summary: bpo-26701: Improve documentation for the rounding special methods. (#6054) files: M Doc/reference/datamodel.rst diff --git a/Doc/reference/datamodel.rst b/Doc/reference/datamodel.rst index 26ad7b8c050c..1e93ef4594a9 100644 --- a/Doc/reference/datamodel.rst +++ b/Doc/reference/datamodel.rst @@ -2364,28 +2364,17 @@ left undefined. .. method:: object.__complex__(self) object.__int__(self) object.__float__(self) - object.__round__(self, [,n]) .. index:: builtin: complex builtin: int builtin: float - builtin: round Called to implement the built-in functions :func:`complex`, - :func:`int`, :func:`float` and :func:`round`. Should return a value + :func:`int` and :func:`float`. Should return a value of the appropriate type. -.. method:: object.__trunc__(self) - - Called to implement :meth:`math.trunc`. Should return the value of the - object truncated to a :class:`numbers.Integral` (typically an - :class:`int`). If a class defines :meth:`__trunc__` but not - :meth:`__int__`, then :meth:`__trunc__` is called to implement the - built-in function :func:`int`. - - .. method:: object.__index__(self) Called to implement :func:`operator.index`, and whenever Python needs to @@ -2401,6 +2390,23 @@ left undefined. the same value. +.. method:: object.__round__(self, [,ndigits]) + object.__trunc__(self) + object.__floor__(self) + object.__ceil__(self) + + .. index:: builtin: round + + Called to implement the built-in function :func:`round` and :mod:`math` + functions :func:`~math.trunc`, :func:`~math.floor` and :func:`~math.ceil`. + Unless *ndigits* is passed to :meth:`!__round__` all these methods should + return the value of the object truncated to an :class:`~numbers.Integral` + (typically an :class:`int`). + + If :meth:`__int__` is not defined then the built-in function :func:`int` + falls back to :meth:`__trunc__`. + + .. _context-managers: With Statement Context Managers From webhook-mailer at python.org Sat Mar 10 10:48:38 2018 From: webhook-mailer at python.org (Andrew Svetlov) Date: Sat, 10 Mar 2018 15:48:38 -0000 Subject: [Python-checkins] bpo-33037: Skip sending/receiving after SSL transport closing (GH-6044) Message-ID: <mailman.16.1520696919.1871.python-checkins@python.org> https://github.com/python/cpython/commit/5e80a71ab67045fecec46573a1892e240b569ace commit: 5e80a71ab67045fecec46573a1892e240b569ace branch: master author: Andrew Svetlov <andrew.svetlov at gmail.com> committer: GitHub <noreply at github.com> date: 2018-03-10T17:48:35+02:00 summary: bpo-33037: Skip sending/receiving after SSL transport closing (GH-6044) * Skip write()/data_received() if sslpipe is destroyed files: A Misc/NEWS.d/next/Library/2018-03-09-23-07-07.bpo-33037.nAJ3at.rst M Lib/asyncio/sslproto.py M Lib/test/test_asyncio/test_sslproto.py diff --git a/Lib/asyncio/sslproto.py b/Lib/asyncio/sslproto.py index 863b54313cc6..2bbf134c0f7e 100644 --- a/Lib/asyncio/sslproto.py +++ b/Lib/asyncio/sslproto.py @@ -504,6 +504,10 @@ def data_received(self, data): The argument is a bytes object. """ + if self._sslpipe is None: + # transport closing, sslpipe is destroyed + return + try: ssldata, appdata = self._sslpipe.feed_ssldata(data) except ssl.SSLError as e: @@ -636,7 +640,7 @@ def _on_handshake_complete(self, handshake_exc): def _process_write_backlog(self): # Try to make progress on the write backlog. - if self._transport is None: + if self._transport is None or self._sslpipe is None: return try: diff --git a/Lib/test/test_asyncio/test_sslproto.py b/Lib/test/test_asyncio/test_sslproto.py index c4c30bedb4ec..7b27f4cfa322 100644 --- a/Lib/test/test_asyncio/test_sslproto.py +++ b/Lib/test/test_asyncio/test_sslproto.py @@ -26,16 +26,17 @@ def setUp(self): self.loop = asyncio.new_event_loop() self.set_event_loop(self.loop) - def ssl_protocol(self, waiter=None): + def ssl_protocol(self, *, waiter=None, proto=None): sslcontext = test_utils.dummy_ssl_context() - app_proto = asyncio.Protocol() - proto = sslproto.SSLProtocol(self.loop, app_proto, sslcontext, waiter, - ssl_handshake_timeout=0.1) - self.assertIs(proto._app_transport.get_protocol(), app_proto) - self.addCleanup(proto._app_transport.close) - return proto - - def connection_made(self, ssl_proto, do_handshake=None): + if proto is None: # app protocol + proto = asyncio.Protocol() + ssl_proto = sslproto.SSLProtocol(self.loop, proto, sslcontext, waiter, + ssl_handshake_timeout=0.1) + self.assertIs(ssl_proto._app_transport.get_protocol(), proto) + self.addCleanup(ssl_proto._app_transport.close) + return ssl_proto + + def connection_made(self, ssl_proto, *, do_handshake=None): transport = mock.Mock() sslpipe = mock.Mock() sslpipe.shutdown.return_value = b'' @@ -53,7 +54,7 @@ def test_cancel_handshake(self): # Python issue #23197: cancelling a handshake must not raise an # exception or log an error, even if the handshake failed waiter = asyncio.Future(loop=self.loop) - ssl_proto = self.ssl_protocol(waiter) + ssl_proto = self.ssl_protocol(waiter=waiter) handshake_fut = asyncio.Future(loop=self.loop) def do_handshake(callback): @@ -63,7 +64,7 @@ def do_handshake(callback): return [] waiter.cancel() - self.connection_made(ssl_proto, do_handshake) + self.connection_made(ssl_proto, do_handshake=do_handshake) with test_utils.disable_logger(): self.loop.run_until_complete(handshake_fut) @@ -96,7 +97,7 @@ def test_handshake_timeout_negative(self): def test_eof_received_waiter(self): waiter = asyncio.Future(loop=self.loop) - ssl_proto = self.ssl_protocol(waiter) + ssl_proto = self.ssl_protocol(waiter=waiter) self.connection_made(ssl_proto) ssl_proto.eof_received() test_utils.run_briefly(self.loop) @@ -107,7 +108,7 @@ def test_fatal_error_no_name_error(self): # _fatal_error() generates a NameError if sslproto.py # does not import base_events. waiter = asyncio.Future(loop=self.loop) - ssl_proto = self.ssl_protocol(waiter) + ssl_proto = self.ssl_protocol(waiter=waiter) # Temporarily turn off error logging so as not to spoil test output. log_level = log.logger.getEffectiveLevel() log.logger.setLevel(logging.FATAL) @@ -121,7 +122,7 @@ def test_connection_lost(self): # From issue #472. # yield from waiter hang if lost_connection was called. waiter = asyncio.Future(loop=self.loop) - ssl_proto = self.ssl_protocol(waiter) + ssl_proto = self.ssl_protocol(waiter=waiter) self.connection_made(ssl_proto) ssl_proto.connection_lost(ConnectionAbortedError) test_utils.run_briefly(self.loop) @@ -130,10 +131,7 @@ def test_connection_lost(self): def test_close_during_handshake(self): # bpo-29743 Closing transport during handshake process leaks socket waiter = asyncio.Future(loop=self.loop) - ssl_proto = self.ssl_protocol(waiter) - - def do_handshake(callback): - return [] + ssl_proto = self.ssl_protocol(waiter=waiter) transport = self.connection_made(ssl_proto) test_utils.run_briefly(self.loop) @@ -143,7 +141,7 @@ def do_handshake(callback): def test_get_extra_info_on_closed_connection(self): waiter = asyncio.Future(loop=self.loop) - ssl_proto = self.ssl_protocol(waiter) + ssl_proto = self.ssl_protocol(waiter=waiter) self.assertIsNone(ssl_proto._get_extra_info('socket')) default = object() self.assertIs(ssl_proto._get_extra_info('socket', default), default) @@ -154,12 +152,31 @@ def test_get_extra_info_on_closed_connection(self): def test_set_new_app_protocol(self): waiter = asyncio.Future(loop=self.loop) - ssl_proto = self.ssl_protocol(waiter) + ssl_proto = self.ssl_protocol(waiter=waiter) new_app_proto = asyncio.Protocol() ssl_proto._app_transport.set_protocol(new_app_proto) self.assertIs(ssl_proto._app_transport.get_protocol(), new_app_proto) self.assertIs(ssl_proto._app_protocol, new_app_proto) + def test_data_received_after_closing(self): + ssl_proto = self.ssl_protocol() + self.connection_made(ssl_proto) + transp = ssl_proto._app_transport + + transp.close() + + # should not raise + self.assertIsNone(ssl_proto.data_received(b'data')) + + def test_write_after_closing(self): + ssl_proto = self.ssl_protocol() + self.connection_made(ssl_proto) + transp = ssl_proto._app_transport + transp.close() + + # should not raise + self.assertIsNone(transp.write(b'data')) + ############################################################################## # Start TLS Tests diff --git a/Misc/NEWS.d/next/Library/2018-03-09-23-07-07.bpo-33037.nAJ3at.rst b/Misc/NEWS.d/next/Library/2018-03-09-23-07-07.bpo-33037.nAJ3at.rst new file mode 100644 index 000000000000..2732eeb4534b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-03-09-23-07-07.bpo-33037.nAJ3at.rst @@ -0,0 +1 @@ +Skip sending/receiving data after SSL transport closing. From webhook-mailer at python.org Sat Mar 10 10:49:27 2018 From: webhook-mailer at python.org (Miss Islington (bot)) Date: Sat, 10 Mar 2018 15:49:27 -0000 Subject: [Python-checkins] bpo-26701: Improve documentation for the rounding special methods. (GH-6054) Message-ID: <mailman.17.1520696968.1871.python-checkins@python.org> https://github.com/python/cpython/commit/89090789debb9d76892af566277cb71740808945 commit: 89090789debb9d76892af566277cb71740808945 branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: GitHub <noreply at github.com> date: 2018-03-10T07:49:24-08:00 summary: bpo-26701: Improve documentation for the rounding special methods. (GH-6054) (cherry picked from commit 496431ffb6c29719332bf2af773349e8dd85e45a) Co-authored-by: Serhiy Storchaka <storchaka at gmail.com> files: M Doc/reference/datamodel.rst diff --git a/Doc/reference/datamodel.rst b/Doc/reference/datamodel.rst index 26ad7b8c050c..1e93ef4594a9 100644 --- a/Doc/reference/datamodel.rst +++ b/Doc/reference/datamodel.rst @@ -2364,28 +2364,17 @@ left undefined. .. method:: object.__complex__(self) object.__int__(self) object.__float__(self) - object.__round__(self, [,n]) .. index:: builtin: complex builtin: int builtin: float - builtin: round Called to implement the built-in functions :func:`complex`, - :func:`int`, :func:`float` and :func:`round`. Should return a value + :func:`int` and :func:`float`. Should return a value of the appropriate type. -.. method:: object.__trunc__(self) - - Called to implement :meth:`math.trunc`. Should return the value of the - object truncated to a :class:`numbers.Integral` (typically an - :class:`int`). If a class defines :meth:`__trunc__` but not - :meth:`__int__`, then :meth:`__trunc__` is called to implement the - built-in function :func:`int`. - - .. method:: object.__index__(self) Called to implement :func:`operator.index`, and whenever Python needs to @@ -2401,6 +2390,23 @@ left undefined. the same value. +.. method:: object.__round__(self, [,ndigits]) + object.__trunc__(self) + object.__floor__(self) + object.__ceil__(self) + + .. index:: builtin: round + + Called to implement the built-in function :func:`round` and :mod:`math` + functions :func:`~math.trunc`, :func:`~math.floor` and :func:`~math.ceil`. + Unless *ndigits* is passed to :meth:`!__round__` all these methods should + return the value of the object truncated to an :class:`~numbers.Integral` + (typically an :class:`int`). + + If :meth:`__int__` is not defined then the built-in function :func:`int` + falls back to :meth:`__trunc__`. + + .. _context-managers: With Statement Context Managers From webhook-mailer at python.org Sat Mar 10 10:55:15 2018 From: webhook-mailer at python.org (Miss Islington (bot)) Date: Sat, 10 Mar 2018 15:55:15 -0000 Subject: [Python-checkins] bpo-26701: Improve documentation for the rounding special methods. (GH-6054) Message-ID: <mailman.18.1520697316.1871.python-checkins@python.org> https://github.com/python/cpython/commit/de8567e38c44b1509f0b906aec54437256848f14 commit: de8567e38c44b1509f0b906aec54437256848f14 branch: 3.6 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: GitHub <noreply at github.com> date: 2018-03-10T07:55:13-08:00 summary: bpo-26701: Improve documentation for the rounding special methods. (GH-6054) (cherry picked from commit 496431ffb6c29719332bf2af773349e8dd85e45a) Co-authored-by: Serhiy Storchaka <storchaka at gmail.com> files: M Doc/reference/datamodel.rst diff --git a/Doc/reference/datamodel.rst b/Doc/reference/datamodel.rst index 90f3a2aa0dcd..c08986f6c0a5 100644 --- a/Doc/reference/datamodel.rst +++ b/Doc/reference/datamodel.rst @@ -2299,28 +2299,17 @@ left undefined. .. method:: object.__complex__(self) object.__int__(self) object.__float__(self) - object.__round__(self, [,n]) .. index:: builtin: complex builtin: int builtin: float - builtin: round Called to implement the built-in functions :func:`complex`, - :func:`int`, :func:`float` and :func:`round`. Should return a value + :func:`int` and :func:`float`. Should return a value of the appropriate type. -.. method:: object.__trunc__(self) - - Called to implement :meth:`math.trunc`. Should return the value of the - object truncated to a :class:`numbers.Integral` (typically an - :class:`int`). If a class defines :meth:`__trunc__` but not - :meth:`__int__`, then :meth:`__trunc__` is called to implement the - built-in function :func:`int`. - - .. method:: object.__index__(self) Called to implement :func:`operator.index`, and whenever Python needs to @@ -2336,6 +2325,23 @@ left undefined. the same value. +.. method:: object.__round__(self, [,ndigits]) + object.__trunc__(self) + object.__floor__(self) + object.__ceil__(self) + + .. index:: builtin: round + + Called to implement the built-in function :func:`round` and :mod:`math` + functions :func:`~math.trunc`, :func:`~math.floor` and :func:`~math.ceil`. + Unless *ndigits* is passed to :meth:`!__round__` all these methods should + return the value of the object truncated to an :class:`~numbers.Integral` + (typically an :class:`int`). + + If :meth:`__int__` is not defined then the built-in function :func:`int` + falls back to :meth:`__trunc__`. + + .. _context-managers: With Statement Context Managers From webhook-mailer at python.org Sat Mar 10 11:22:36 2018 From: webhook-mailer at python.org (Serhiy Storchaka) Date: Sat, 10 Mar 2018 16:22:36 -0000 Subject: [Python-checkins] bpo-33041: Fixed bytecode generation for "async for" with a complex target. (#6052) Message-ID: <mailman.19.1520698958.1871.python-checkins@python.org> https://github.com/python/cpython/commit/24d3201eb7f0b39a7eaf2a5b2a2ceca10ad1f8eb commit: 24d3201eb7f0b39a7eaf2a5b2a2ceca10ad1f8eb branch: master author: Serhiy Storchaka <storchaka at gmail.com> committer: GitHub <noreply at github.com> date: 2018-03-10T18:22:34+02:00 summary: bpo-33041: Fixed bytecode generation for "async for" with a complex target. (#6052) A StopAsyncIteration raised on assigning or unpacking will be now propagated instead of stopping the iteration. files: A Misc/NEWS.d/next/Core and Builtins/2018-03-10-15-16-40.bpo-33041.-ak5Fk.rst M Lib/test/test_coroutines.py M Python/compile.c diff --git a/Lib/test/test_coroutines.py b/Lib/test/test_coroutines.py index b37b61b71959..f4a9d2aeb874 100644 --- a/Lib/test/test_coroutines.py +++ b/Lib/test/test_coroutines.py @@ -1949,6 +1949,71 @@ def test_fatal_coro_warning(self): support.gc_collect() self.assertIn("was never awaited", stderr.getvalue()) + def test_for_assign_raising_stop_async_iteration(self): + class BadTarget: + def __setitem__(self, key, value): + raise StopAsyncIteration(42) + tgt = BadTarget() + async def source(): + yield 10 + + async def run_for(): + with self.assertRaises(StopAsyncIteration) as cm: + async for tgt[0] in source(): + pass + self.assertEqual(cm.exception.args, (42,)) + return 'end' + self.assertEqual(run_async(run_for()), ([], 'end')) + + async def run_list(): + with self.assertRaises(StopAsyncIteration) as cm: + return [0 async for tgt[0] in source()] + self.assertEqual(cm.exception.args, (42,)) + return 'end' + self.assertEqual(run_async(run_list()), ([], 'end')) + + async def run_gen(): + gen = (0 async for tgt[0] in source()) + a = gen.asend(None) + with self.assertRaises(RuntimeError) as cm: + await a + self.assertIsInstance(cm.exception.__cause__, StopAsyncIteration) + self.assertEqual(cm.exception.__cause__.args, (42,)) + return 'end' + self.assertEqual(run_async(run_gen()), ([], 'end')) + + def test_for_assign_raising_stop_async_iteration_2(self): + class BadIterable: + def __iter__(self): + raise StopAsyncIteration(42) + async def badpairs(): + yield BadIterable() + + async def run_for(): + with self.assertRaises(StopAsyncIteration) as cm: + async for i, j in badpairs(): + pass + self.assertEqual(cm.exception.args, (42,)) + return 'end' + self.assertEqual(run_async(run_for()), ([], 'end')) + + async def run_list(): + with self.assertRaises(StopAsyncIteration) as cm: + return [0 async for i, j in badpairs()] + self.assertEqual(cm.exception.args, (42,)) + return 'end' + self.assertEqual(run_async(run_list()), ([], 'end')) + + async def run_gen(): + gen = (0 async for i, j in badpairs()) + a = gen.asend(None) + with self.assertRaises(RuntimeError) as cm: + await a + self.assertIsInstance(cm.exception.__cause__, StopAsyncIteration) + self.assertEqual(cm.exception.__cause__.args, (42,)) + return 'end' + self.assertEqual(run_async(run_gen()), ([], 'end')) + class CoroAsyncIOCompatTest(unittest.TestCase): diff --git a/Misc/NEWS.d/next/Core and Builtins/2018-03-10-15-16-40.bpo-33041.-ak5Fk.rst b/Misc/NEWS.d/next/Core and Builtins/2018-03-10-15-16-40.bpo-33041.-ak5Fk.rst new file mode 100644 index 000000000000..af9ccfd89f5d --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2018-03-10-15-16-40.bpo-33041.-ak5Fk.rst @@ -0,0 +1,3 @@ +Fixed bytecode generation for "async for" with a complex target. A +StopAsyncIteration raised on assigning or unpacking will be now propagated +instead of stopping the iteration. diff --git a/Python/compile.c b/Python/compile.c index f14647b4f641..6c9e7954d688 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -2473,8 +2473,8 @@ compiler_async_for(struct compiler *c, stmt_ty s) ADDOP(c, GET_ANEXT); ADDOP_O(c, LOAD_CONST, Py_None, consts); ADDOP(c, YIELD_FROM); - VISIT(c, expr, s->v.AsyncFor.target); ADDOP(c, POP_BLOCK); /* for SETUP_FINALLY */ + VISIT(c, expr, s->v.AsyncFor.target); compiler_pop_fblock(c, EXCEPT, try); ADDOP_JREL(c, JUMP_FORWARD, after_try); @@ -4060,8 +4060,8 @@ compiler_async_comprehension_generator(struct compiler *c, ADDOP(c, GET_ANEXT); ADDOP_O(c, LOAD_CONST, Py_None, consts); ADDOP(c, YIELD_FROM); - VISIT(c, expr, gen->target); ADDOP(c, POP_BLOCK); + VISIT(c, expr, gen->target); compiler_pop_fblock(c, EXCEPT, try); ADDOP_JREL(c, JUMP_FORWARD, after_try); From webhook-mailer at python.org Sat Mar 10 11:27:04 2018 From: webhook-mailer at python.org (Andrew Svetlov) Date: Sat, 10 Mar 2018 16:27:04 -0000 Subject: [Python-checkins] bpo-33037: Skip sending/receiving after SSL transport closing (GH-6044) (GH-6057) Message-ID: <mailman.20.1520699225.1871.python-checkins@python.org> https://github.com/python/cpython/commit/bf0d1165174e8347b4d3a731c4e47e8288f1d01b commit: bf0d1165174e8347b4d3a731c4e47e8288f1d01b branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Andrew Svetlov <andrew.svetlov at gmail.com> date: 2018-03-10T18:27:01+02:00 summary: bpo-33037: Skip sending/receiving after SSL transport closing (GH-6044) (GH-6057) * Skip write()/data_received() if sslpipe is destroyed (cherry picked from commit 5e80a71ab67045fecec46573a1892e240b569ace) Co-authored-by: Andrew Svetlov <andrew.svetlov at gmail.com> files: A Misc/NEWS.d/next/Library/2018-03-09-23-07-07.bpo-33037.nAJ3at.rst M Lib/asyncio/sslproto.py M Lib/test/test_asyncio/test_sslproto.py diff --git a/Lib/asyncio/sslproto.py b/Lib/asyncio/sslproto.py index 863b54313cc6..2bbf134c0f7e 100644 --- a/Lib/asyncio/sslproto.py +++ b/Lib/asyncio/sslproto.py @@ -504,6 +504,10 @@ def data_received(self, data): The argument is a bytes object. """ + if self._sslpipe is None: + # transport closing, sslpipe is destroyed + return + try: ssldata, appdata = self._sslpipe.feed_ssldata(data) except ssl.SSLError as e: @@ -636,7 +640,7 @@ def _on_handshake_complete(self, handshake_exc): def _process_write_backlog(self): # Try to make progress on the write backlog. - if self._transport is None: + if self._transport is None or self._sslpipe is None: return try: diff --git a/Lib/test/test_asyncio/test_sslproto.py b/Lib/test/test_asyncio/test_sslproto.py index c4c30bedb4ec..7b27f4cfa322 100644 --- a/Lib/test/test_asyncio/test_sslproto.py +++ b/Lib/test/test_asyncio/test_sslproto.py @@ -26,16 +26,17 @@ def setUp(self): self.loop = asyncio.new_event_loop() self.set_event_loop(self.loop) - def ssl_protocol(self, waiter=None): + def ssl_protocol(self, *, waiter=None, proto=None): sslcontext = test_utils.dummy_ssl_context() - app_proto = asyncio.Protocol() - proto = sslproto.SSLProtocol(self.loop, app_proto, sslcontext, waiter, - ssl_handshake_timeout=0.1) - self.assertIs(proto._app_transport.get_protocol(), app_proto) - self.addCleanup(proto._app_transport.close) - return proto - - def connection_made(self, ssl_proto, do_handshake=None): + if proto is None: # app protocol + proto = asyncio.Protocol() + ssl_proto = sslproto.SSLProtocol(self.loop, proto, sslcontext, waiter, + ssl_handshake_timeout=0.1) + self.assertIs(ssl_proto._app_transport.get_protocol(), proto) + self.addCleanup(ssl_proto._app_transport.close) + return ssl_proto + + def connection_made(self, ssl_proto, *, do_handshake=None): transport = mock.Mock() sslpipe = mock.Mock() sslpipe.shutdown.return_value = b'' @@ -53,7 +54,7 @@ def test_cancel_handshake(self): # Python issue #23197: cancelling a handshake must not raise an # exception or log an error, even if the handshake failed waiter = asyncio.Future(loop=self.loop) - ssl_proto = self.ssl_protocol(waiter) + ssl_proto = self.ssl_protocol(waiter=waiter) handshake_fut = asyncio.Future(loop=self.loop) def do_handshake(callback): @@ -63,7 +64,7 @@ def do_handshake(callback): return [] waiter.cancel() - self.connection_made(ssl_proto, do_handshake) + self.connection_made(ssl_proto, do_handshake=do_handshake) with test_utils.disable_logger(): self.loop.run_until_complete(handshake_fut) @@ -96,7 +97,7 @@ def test_handshake_timeout_negative(self): def test_eof_received_waiter(self): waiter = asyncio.Future(loop=self.loop) - ssl_proto = self.ssl_protocol(waiter) + ssl_proto = self.ssl_protocol(waiter=waiter) self.connection_made(ssl_proto) ssl_proto.eof_received() test_utils.run_briefly(self.loop) @@ -107,7 +108,7 @@ def test_fatal_error_no_name_error(self): # _fatal_error() generates a NameError if sslproto.py # does not import base_events. waiter = asyncio.Future(loop=self.loop) - ssl_proto = self.ssl_protocol(waiter) + ssl_proto = self.ssl_protocol(waiter=waiter) # Temporarily turn off error logging so as not to spoil test output. log_level = log.logger.getEffectiveLevel() log.logger.setLevel(logging.FATAL) @@ -121,7 +122,7 @@ def test_connection_lost(self): # From issue #472. # yield from waiter hang if lost_connection was called. waiter = asyncio.Future(loop=self.loop) - ssl_proto = self.ssl_protocol(waiter) + ssl_proto = self.ssl_protocol(waiter=waiter) self.connection_made(ssl_proto) ssl_proto.connection_lost(ConnectionAbortedError) test_utils.run_briefly(self.loop) @@ -130,10 +131,7 @@ def test_connection_lost(self): def test_close_during_handshake(self): # bpo-29743 Closing transport during handshake process leaks socket waiter = asyncio.Future(loop=self.loop) - ssl_proto = self.ssl_protocol(waiter) - - def do_handshake(callback): - return [] + ssl_proto = self.ssl_protocol(waiter=waiter) transport = self.connection_made(ssl_proto) test_utils.run_briefly(self.loop) @@ -143,7 +141,7 @@ def do_handshake(callback): def test_get_extra_info_on_closed_connection(self): waiter = asyncio.Future(loop=self.loop) - ssl_proto = self.ssl_protocol(waiter) + ssl_proto = self.ssl_protocol(waiter=waiter) self.assertIsNone(ssl_proto._get_extra_info('socket')) default = object() self.assertIs(ssl_proto._get_extra_info('socket', default), default) @@ -154,12 +152,31 @@ def test_get_extra_info_on_closed_connection(self): def test_set_new_app_protocol(self): waiter = asyncio.Future(loop=self.loop) - ssl_proto = self.ssl_protocol(waiter) + ssl_proto = self.ssl_protocol(waiter=waiter) new_app_proto = asyncio.Protocol() ssl_proto._app_transport.set_protocol(new_app_proto) self.assertIs(ssl_proto._app_transport.get_protocol(), new_app_proto) self.assertIs(ssl_proto._app_protocol, new_app_proto) + def test_data_received_after_closing(self): + ssl_proto = self.ssl_protocol() + self.connection_made(ssl_proto) + transp = ssl_proto._app_transport + + transp.close() + + # should not raise + self.assertIsNone(ssl_proto.data_received(b'data')) + + def test_write_after_closing(self): + ssl_proto = self.ssl_protocol() + self.connection_made(ssl_proto) + transp = ssl_proto._app_transport + transp.close() + + # should not raise + self.assertIsNone(transp.write(b'data')) + ############################################################################## # Start TLS Tests diff --git a/Misc/NEWS.d/next/Library/2018-03-09-23-07-07.bpo-33037.nAJ3at.rst b/Misc/NEWS.d/next/Library/2018-03-09-23-07-07.bpo-33037.nAJ3at.rst new file mode 100644 index 000000000000..2732eeb4534b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-03-09-23-07-07.bpo-33037.nAJ3at.rst @@ -0,0 +1 @@ +Skip sending/receiving data after SSL transport closing. From webhook-mailer at python.org Sat Mar 10 11:49:29 2018 From: webhook-mailer at python.org (Serhiy Storchaka) Date: Sat, 10 Mar 2018 16:49:29 -0000 Subject: [Python-checkins] bpo-33041: Add missed error checks when compile "async for" (#6053) Message-ID: <mailman.21.1520700571.1871.python-checkins@python.org> https://github.com/python/cpython/commit/67ee07795bcd84b679c000780212d4d81a1490a3 commit: 67ee07795bcd84b679c000780212d4d81a1490a3 branch: master author: Serhiy Storchaka <storchaka at gmail.com> committer: GitHub <noreply at github.com> date: 2018-03-10T18:49:26+02:00 summary: bpo-33041: Add missed error checks when compile "async for" (#6053) and remove redundant code. files: M Python/compile.c diff --git a/Python/compile.c b/Python/compile.c index 6c9e7954d688..fbd1fc960ab8 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -2437,7 +2437,7 @@ compiler_async_for(struct compiler *c, stmt_ty s) _Py_IDENTIFIER(StopAsyncIteration); basicblock *try, *except, *end, *after_try, *try_cleanup, - *after_loop, *after_loop_else; + *after_loop_else; PyObject *stop_aiter_error = _PyUnicode_FromId(&PyId_StopAsyncIteration); if (stop_aiter_error == NULL) { @@ -2449,14 +2449,14 @@ compiler_async_for(struct compiler *c, stmt_ty s) end = compiler_new_block(c); after_try = compiler_new_block(c); try_cleanup = compiler_new_block(c); - after_loop = compiler_new_block(c); after_loop_else = compiler_new_block(c); if (try == NULL || except == NULL || end == NULL - || after_try == NULL || try_cleanup == NULL) + || after_try == NULL || try_cleanup == NULL + || after_loop_else == NULL) return 0; - if (!compiler_push_fblock(c, FOR_LOOP, try, after_loop)) + if (!compiler_push_fblock(c, FOR_LOOP, try, end)) return 0; VISIT(c, expr, s->v.AsyncFor.iter); @@ -2504,10 +2504,6 @@ compiler_async_for(struct compiler *c, stmt_ty s) compiler_pop_fblock(c, FOR_LOOP, try); - /* Block reached after `break`ing from loop */ - compiler_use_next_block(c, after_loop); - ADDOP_JABS(c, JUMP_ABSOLUTE, end); - /* `else` block */ compiler_use_next_block(c, after_loop_else); VISIT_SEQ(c, stmt, s->v.For.orelse); @@ -4014,7 +4010,7 @@ compiler_async_comprehension_generator(struct compiler *c, _Py_IDENTIFIER(StopAsyncIteration); comprehension_ty gen; - basicblock *anchor, *skip, *if_cleanup, *try, + basicblock *anchor, *if_cleanup, *try, *after_try, *except, *try_cleanup; Py_ssize_t i, n; @@ -4027,13 +4023,12 @@ compiler_async_comprehension_generator(struct compiler *c, after_try = compiler_new_block(c); try_cleanup = compiler_new_block(c); except = compiler_new_block(c); - skip = compiler_new_block(c); if_cleanup = compiler_new_block(c); anchor = compiler_new_block(c); - if (skip == NULL || if_cleanup == NULL || anchor == NULL || + if (if_cleanup == NULL || anchor == NULL || try == NULL || after_try == NULL || - except == NULL || after_try == NULL) { + except == NULL || try_cleanup == NULL) { return 0; } @@ -4125,8 +4120,6 @@ compiler_async_comprehension_generator(struct compiler *c, default: return 0; } - - compiler_use_next_block(c, skip); } compiler_use_next_block(c, if_cleanup); ADDOP_JABS(c, JUMP_ABSOLUTE, try); From webhook-mailer at python.org Sat Mar 10 12:09:10 2018 From: webhook-mailer at python.org (Andrew Svetlov) Date: Sat, 10 Mar 2018 17:09:10 -0000 Subject: [Python-checkins] [3.6] bpo-33037: Skip sending/receiving after SSL transport closing (GH-6044) (GH-6058) Message-ID: <mailman.22.1520701752.1871.python-checkins@python.org> https://github.com/python/cpython/commit/017e9fda922a143ac9f1601cbde05e80214852d2 commit: 017e9fda922a143ac9f1601cbde05e80214852d2 branch: 3.6 author: Andrew Svetlov <andrew.svetlov at gmail.com> committer: GitHub <noreply at github.com> date: 2018-03-10T19:09:07+02:00 summary: [3.6] bpo-33037: Skip sending/receiving after SSL transport closing (GH-6044) (GH-6058) * Skip write()/data_received() if sslpipe is destroyed. (cherry picked from commit 5e80a71ab67045fecec46573a1892e240b569ace) files: A Misc/NEWS.d/next/Library/2018-03-09-23-07-07.bpo-33037.nAJ3at.rst M Lib/asyncio/sslproto.py M Lib/test/test_asyncio/test_sslproto.py diff --git a/Lib/asyncio/sslproto.py b/Lib/asyncio/sslproto.py index f4d8a4841823..a82babb6b9e2 100644 --- a/Lib/asyncio/sslproto.py +++ b/Lib/asyncio/sslproto.py @@ -497,6 +497,10 @@ def data_received(self, data): The argument is a bytes object. """ + if self._sslpipe is None: + # transport closing, sslpipe is destroyed + return + try: ssldata, appdata = self._sslpipe.feed_ssldata(data) except ssl.SSLError as e: @@ -626,7 +630,7 @@ def _on_handshake_complete(self, handshake_exc): def _process_write_backlog(self): # Try to make progress on the write backlog. - if self._transport is None: + if self._transport is None or self._sslpipe is None: return try: diff --git a/Lib/test/test_asyncio/test_sslproto.py b/Lib/test/test_asyncio/test_sslproto.py index f573ae8fe779..9b198bfd5307 100644 --- a/Lib/test/test_asyncio/test_sslproto.py +++ b/Lib/test/test_asyncio/test_sslproto.py @@ -22,15 +22,16 @@ def setUp(self): self.loop = asyncio.new_event_loop() self.set_event_loop(self.loop) - def ssl_protocol(self, waiter=None): + def ssl_protocol(self, *, waiter=None, proto=None): sslcontext = test_utils.dummy_ssl_context() - app_proto = asyncio.Protocol() - proto = sslproto.SSLProtocol(self.loop, app_proto, sslcontext, waiter) - self.assertIs(proto._app_transport.get_protocol(), app_proto) - self.addCleanup(proto._app_transport.close) - return proto - - def connection_made(self, ssl_proto, do_handshake=None): + if proto is None: # app protocol + proto = asyncio.Protocol() + ssl_proto = sslproto.SSLProtocol(self.loop, proto, sslcontext, waiter) + self.assertIs(ssl_proto._app_transport.get_protocol(), proto) + self.addCleanup(ssl_proto._app_transport.close) + return ssl_proto + + def connection_made(self, ssl_proto, *, do_handshake=None): transport = mock.Mock() sslpipe = mock.Mock() sslpipe.shutdown.return_value = b'' @@ -48,7 +49,7 @@ def test_cancel_handshake(self): # Python issue #23197: cancelling a handshake must not raise an # exception or log an error, even if the handshake failed waiter = asyncio.Future(loop=self.loop) - ssl_proto = self.ssl_protocol(waiter) + ssl_proto = self.ssl_protocol(waiter=waiter) handshake_fut = asyncio.Future(loop=self.loop) def do_handshake(callback): @@ -58,14 +59,14 @@ def do_handshake(callback): return [] waiter.cancel() - self.connection_made(ssl_proto, do_handshake) + self.connection_made(ssl_proto, do_handshake=do_handshake) with test_utils.disable_logger(): self.loop.run_until_complete(handshake_fut) def test_eof_received_waiter(self): waiter = asyncio.Future(loop=self.loop) - ssl_proto = self.ssl_protocol(waiter) + ssl_proto = self.ssl_protocol(waiter=waiter) self.connection_made(ssl_proto) ssl_proto.eof_received() test_utils.run_briefly(self.loop) @@ -76,7 +77,7 @@ def test_fatal_error_no_name_error(self): # _fatal_error() generates a NameError if sslproto.py # does not import base_events. waiter = asyncio.Future(loop=self.loop) - ssl_proto = self.ssl_protocol(waiter) + ssl_proto = self.ssl_protocol(waiter=waiter) # Temporarily turn off error logging so as not to spoil test output. log_level = log.logger.getEffectiveLevel() log.logger.setLevel(logging.FATAL) @@ -90,7 +91,7 @@ def test_connection_lost(self): # From issue #472. # yield from waiter hang if lost_connection was called. waiter = asyncio.Future(loop=self.loop) - ssl_proto = self.ssl_protocol(waiter) + ssl_proto = self.ssl_protocol(waiter=waiter) self.connection_made(ssl_proto) ssl_proto.connection_lost(ConnectionAbortedError) test_utils.run_briefly(self.loop) @@ -99,10 +100,7 @@ def test_connection_lost(self): def test_close_during_handshake(self): # bpo-29743 Closing transport during handshake process leaks socket waiter = asyncio.Future(loop=self.loop) - ssl_proto = self.ssl_protocol(waiter) - - def do_handshake(callback): - return [] + ssl_proto = self.ssl_protocol(waiter=waiter) transport = self.connection_made(ssl_proto) test_utils.run_briefly(self.loop) @@ -112,7 +110,7 @@ def do_handshake(callback): def test_get_extra_info_on_closed_connection(self): waiter = asyncio.Future(loop=self.loop) - ssl_proto = self.ssl_protocol(waiter) + ssl_proto = self.ssl_protocol(waiter=waiter) self.assertIsNone(ssl_proto._get_extra_info('socket')) default = object() self.assertIs(ssl_proto._get_extra_info('socket', default), default) @@ -123,12 +121,31 @@ def test_get_extra_info_on_closed_connection(self): def test_set_new_app_protocol(self): waiter = asyncio.Future(loop=self.loop) - ssl_proto = self.ssl_protocol(waiter) + ssl_proto = self.ssl_protocol(waiter=waiter) new_app_proto = asyncio.Protocol() ssl_proto._app_transport.set_protocol(new_app_proto) self.assertIs(ssl_proto._app_transport.get_protocol(), new_app_proto) self.assertIs(ssl_proto._app_protocol, new_app_proto) + def test_data_received_after_closing(self): + ssl_proto = self.ssl_protocol() + self.connection_made(ssl_proto) + transp = ssl_proto._app_transport + + transp.close() + + # should not raise + self.assertIsNone(ssl_proto.data_received(b'data')) + + def test_write_after_closing(self): + ssl_proto = self.ssl_protocol() + self.connection_made(ssl_proto) + transp = ssl_proto._app_transport + transp.close() + + # should not raise + self.assertIsNone(transp.write(b'data')) + if __name__ == '__main__': unittest.main() diff --git a/Misc/NEWS.d/next/Library/2018-03-09-23-07-07.bpo-33037.nAJ3at.rst b/Misc/NEWS.d/next/Library/2018-03-09-23-07-07.bpo-33037.nAJ3at.rst new file mode 100644 index 000000000000..2732eeb4534b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-03-09-23-07-07.bpo-33037.nAJ3at.rst @@ -0,0 +1 @@ +Skip sending/receiving data after SSL transport closing. From webhook-mailer at python.org Sat Mar 10 13:45:08 2018 From: webhook-mailer at python.org (Serhiy Storchaka) Date: Sat, 10 Mar 2018 18:45:08 -0000 Subject: [Python-checkins] [3.7] bpo-33041: Add missed error checks when compile "async for" (GH-6053) (GH-6060) Message-ID: <mailman.23.1520707509.1871.python-checkins@python.org> https://github.com/python/cpython/commit/9e94c0d3c78d1bc582c865240ed9353fe9689b2a commit: 9e94c0d3c78d1bc582c865240ed9353fe9689b2a branch: 3.7 author: Serhiy Storchaka <storchaka at gmail.com> committer: GitHub <noreply at github.com> date: 2018-03-10T20:45:05+02:00 summary: [3.7] bpo-33041: Add missed error checks when compile "async for" (GH-6053) (GH-6060) and remove redundant code. (cherry picked from commit 67ee07795bcd84b679c000780212d4d81a1490a3) Co-authored-by: Serhiy Storchaka <storchaka at gmail.com> files: M Python/compile.c diff --git a/Python/compile.c b/Python/compile.c index 0dc35662749b..8ab33613b524 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -2337,7 +2337,7 @@ compiler_async_for(struct compiler *c, stmt_ty s) _Py_IDENTIFIER(StopAsyncIteration); basicblock *try, *except, *end, *after_try, *try_cleanup, - *after_loop, *after_loop_else; + *after_loop_else; PyObject *stop_aiter_error = _PyUnicode_FromId(&PyId_StopAsyncIteration); if (stop_aiter_error == NULL) { @@ -2349,14 +2349,14 @@ compiler_async_for(struct compiler *c, stmt_ty s) end = compiler_new_block(c); after_try = compiler_new_block(c); try_cleanup = compiler_new_block(c); - after_loop = compiler_new_block(c); after_loop_else = compiler_new_block(c); if (try == NULL || except == NULL || end == NULL - || after_try == NULL || try_cleanup == NULL) + || after_try == NULL || try_cleanup == NULL + || after_loop_else == NULL) return 0; - ADDOP_JREL(c, SETUP_LOOP, after_loop); + ADDOP_JREL(c, SETUP_LOOP, end); if (!compiler_push_fblock(c, LOOP, try)) return 0; @@ -2404,9 +2404,6 @@ compiler_async_for(struct compiler *c, stmt_ty s) ADDOP(c, POP_BLOCK); /* for SETUP_LOOP */ compiler_pop_fblock(c, LOOP, try); - compiler_use_next_block(c, after_loop); - ADDOP_JABS(c, JUMP_ABSOLUTE, end); - compiler_use_next_block(c, after_loop_else); VISIT_SEQ(c, stmt, s->v.For.orelse); @@ -3893,7 +3890,7 @@ compiler_async_comprehension_generator(struct compiler *c, _Py_IDENTIFIER(StopAsyncIteration); comprehension_ty gen; - basicblock *anchor, *skip, *if_cleanup, *try, + basicblock *anchor, *if_cleanup, *try, *after_try, *except, *try_cleanup; Py_ssize_t i, n; @@ -3906,13 +3903,12 @@ compiler_async_comprehension_generator(struct compiler *c, after_try = compiler_new_block(c); try_cleanup = compiler_new_block(c); except = compiler_new_block(c); - skip = compiler_new_block(c); if_cleanup = compiler_new_block(c); anchor = compiler_new_block(c); - if (skip == NULL || if_cleanup == NULL || anchor == NULL || + if (if_cleanup == NULL || anchor == NULL || try == NULL || after_try == NULL || - except == NULL || after_try == NULL) { + except == NULL || try_cleanup == NULL) { return 0; } @@ -4004,8 +4000,6 @@ compiler_async_comprehension_generator(struct compiler *c, default: return 0; } - - compiler_use_next_block(c, skip); } compiler_use_next_block(c, if_cleanup); ADDOP_JABS(c, JUMP_ABSOLUTE, try); From webhook-mailer at python.org Sat Mar 10 13:58:55 2018 From: webhook-mailer at python.org (Xiang Zhang) Date: Sat, 10 Mar 2018 18:58:55 -0000 Subject: [Python-checkins] bpo-30249: Improve struct.unpack_from() error messages (GH-6059) Message-ID: <mailman.24.1520708337.1871.python-checkins@python.org> https://github.com/python/cpython/commit/c10b288f345aaef66d2c844924b9a576f9ea4f8b commit: c10b288f345aaef66d2c844924b9a576f9ea4f8b branch: master author: Xiang Zhang <angwerzx at 126.com> committer: GitHub <noreply at github.com> date: 2018-03-11T02:58:52+08:00 summary: bpo-30249: Improve struct.unpack_from() error messages (GH-6059) files: A Misc/NEWS.d/next/Library/2018-03-11-00-20-26.bpo-30249.KSkgLB.rst M Doc/library/struct.rst M Lib/test/test_struct.py M Modules/_struct.c M Modules/clinic/_struct.c.h diff --git a/Doc/library/struct.rst b/Doc/library/struct.rst index 2d0866c7e09e..d6a3cb721e83 100644 --- a/Doc/library/struct.rst +++ b/Doc/library/struct.rst @@ -74,8 +74,8 @@ The module defines the following exception and functions: Unpack from *buffer* starting at position *offset*, according to the format string *format*. The result is a tuple even if it contains exactly one - item. The buffer's size in bytes, minus *offset*, must be at least - the size required by the format, as reflected by :func:`calcsize`. + item. The buffer's size in bytes, starting at position *offset*, must be at + least the size required by the format, as reflected by :func:`calcsize`. .. function:: iter_unpack(format, buffer) @@ -428,7 +428,7 @@ The :mod:`struct` module also defines the following type: .. method:: unpack_from(buffer, offset=0) Identical to the :func:`unpack_from` function, using the compiled format. - The buffer's size in bytes, minus *offset*, must be at least + The buffer's size in bytes, starting at position *offset*, must be at least :attr:`size`. diff --git a/Lib/test/test_struct.py b/Lib/test/test_struct.py index 8fd56c91cb7a..454082e66d3f 100644 --- a/Lib/test/test_struct.py +++ b/Lib/test/test_struct.py @@ -579,14 +579,22 @@ def test__sizeof__(self): self.check_sizeof('0c', 0) def test_boundary_error_message(self): - regex = ( + regex1 = ( r'pack_into requires a buffer of at least 6 ' r'bytes for packing 1 bytes at offset 5 ' r'\(actual buffer size is 1\)' ) - with self.assertRaisesRegex(struct.error, regex): + with self.assertRaisesRegex(struct.error, regex1): struct.pack_into('b', bytearray(1), 5, 1) + regex2 = ( + r'unpack_from requires a buffer of at least 6 ' + r'bytes for unpacking 1 bytes at offset 5 ' + r'\(actual buffer size is 1\)' + ) + with self.assertRaisesRegex(struct.error, regex2): + struct.unpack_from('b', bytearray(1), 5) + def test_boundary_error_message_with_negative_offset(self): byte_list = bytearray(10) with self.assertRaisesRegex( @@ -599,16 +607,34 @@ def test_boundary_error_message_with_negative_offset(self): 'offset -11 out of range for 10-byte buffer'): struct.pack_into('<B', byte_list, -11, 123) + with self.assertRaisesRegex( + struct.error, + r'not enough data to unpack 4 bytes at offset -2'): + struct.unpack_from('<I', byte_list, -2) + + with self.assertRaisesRegex( + struct.error, + "offset -11 out of range for 10-byte buffer"): + struct.unpack_from('<B', byte_list, -11) + def test_boundary_error_message_with_large_offset(self): # Test overflows cause by large offset and value size (issue 30245) - regex = ( + regex1 = ( r'pack_into requires a buffer of at least ' + str(sys.maxsize + 4) + r' bytes for packing 4 bytes at offset ' + str(sys.maxsize) + r' \(actual buffer size is 10\)' ) - with self.assertRaisesRegex(struct.error, regex): + with self.assertRaisesRegex(struct.error, regex1): struct.pack_into('<I', bytearray(10), sys.maxsize, 1) + regex2 = ( + r'unpack_from requires a buffer of at least ' + str(sys.maxsize + 4) + + r' bytes for unpacking 4 bytes at offset ' + str(sys.maxsize) + + r' \(actual buffer size is 10\)' + ) + with self.assertRaisesRegex(struct.error, regex2): + struct.unpack_from('<I', bytearray(10), sys.maxsize) + def test_issue29802(self): # When the second argument of struct.unpack() was of wrong type # the Struct object was decrefed twice and the reference to diff --git a/Misc/NEWS.d/next/Library/2018-03-11-00-20-26.bpo-30249.KSkgLB.rst b/Misc/NEWS.d/next/Library/2018-03-11-00-20-26.bpo-30249.KSkgLB.rst new file mode 100644 index 000000000000..f30dff3124eb --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-03-11-00-20-26.bpo-30249.KSkgLB.rst @@ -0,0 +1,2 @@ +Improve struct.unpack_from() exception messages for problems with the buffer +size and offset. diff --git a/Modules/_struct.c b/Modules/_struct.c index 66f74d63b735..053970671f17 100644 --- a/Modules/_struct.c +++ b/Modules/_struct.c @@ -1553,7 +1553,8 @@ Return a tuple containing unpacked values. Values are unpacked according to the format string Struct.format. -The buffer's size in bytes, minus offset, must be at least Struct.size. +The buffer's size in bytes, starting at position offset, must be +at least Struct.size. See help(struct) for more on format strings. [clinic start generated code]*/ @@ -1561,16 +1562,38 @@ See help(struct) for more on format strings. static PyObject * Struct_unpack_from_impl(PyStructObject *self, Py_buffer *buffer, Py_ssize_t offset) -/*[clinic end generated code: output=57fac875e0977316 input=97ade52422f8962f]*/ +/*[clinic end generated code: output=57fac875e0977316 input=cafd4851d473c894]*/ { assert(self->s_codes != NULL); - if (offset < 0) + if (offset < 0) { + if (offset + self->s_size > 0) { + PyErr_Format(StructError, + "not enough data to unpack %zd bytes at offset %zd", + self->s_size, + offset); + return NULL; + } + + if (offset + buffer->len < 0) { + PyErr_Format(StructError, + "offset %zd out of range for %zd-byte buffer", + offset, + buffer->len); + return NULL; + } offset += buffer->len; - if (offset < 0 || buffer->len - offset < self->s_size) { + } + + if ((buffer->len - offset) < self->s_size) { PyErr_Format(StructError, - "unpack_from requires a buffer of at least %zd bytes", - self->s_size); + "unpack_from requires a buffer of at least %zu bytes for " + "unpacking %zd bytes at offset %zd " + "(actual buffer size is %zd)", + (size_t)self->s_size + (size_t)offset, + self->s_size, + offset, + buffer->len); return NULL; } return s_unpack_internal(self, (char*)buffer->buf + offset); diff --git a/Modules/clinic/_struct.c.h b/Modules/clinic/_struct.c.h index 6e43215b737a..2ecadfb1de97 100644 --- a/Modules/clinic/_struct.c.h +++ b/Modules/clinic/_struct.c.h @@ -79,7 +79,8 @@ PyDoc_STRVAR(Struct_unpack_from__doc__, "\n" "Values are unpacked according to the format string Struct.format.\n" "\n" -"The buffer\'s size in bytes, minus offset, must be at least Struct.size.\n" +"The buffer\'s size in bytes, starting at position offset, must be\n" +"at least Struct.size.\n" "\n" "See help(struct) for more on format strings."); @@ -302,4 +303,4 @@ iter_unpack(PyObject *module, PyObject *const *args, Py_ssize_t nargs) return return_value; } -/*[clinic end generated code: output=9119f213a951e4cc input=a9049054013a1b77]*/ +/*[clinic end generated code: output=d79b009652ae0b89 input=a9049054013a1b77]*/ From webhook-mailer at python.org Sat Mar 10 14:32:53 2018 From: webhook-mailer at python.org (Miss Islington (bot)) Date: Sat, 10 Mar 2018 19:32:53 -0000 Subject: [Python-checkins] [3.7] bpo-33041: Add missed error checks when compile "async for" (GH-6053) (GH-6060) Message-ID: <mailman.25.1520710374.1871.python-checkins@python.org> https://github.com/python/cpython/commit/d0826340d96e0953793b86d0b8475d2f43a280b6 commit: d0826340d96e0953793b86d0b8475d2f43a280b6 branch: 3.6 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: GitHub <noreply at github.com> date: 2018-03-10T11:32:49-08:00 summary: [3.7] bpo-33041: Add missed error checks when compile "async for" (GH-6053) (GH-6060) and remove redundant code. (cherry picked from commit 67ee07795bcd84b679c000780212d4d81a1490a3) Co-authored-by: Serhiy Storchaka <storchaka at gmail.com> (cherry picked from commit 9e94c0d3c78d1bc582c865240ed9353fe9689b2a) Co-authored-by: Serhiy Storchaka <storchaka at gmail.com> files: M Python/compile.c diff --git a/Python/compile.c b/Python/compile.c index 3b8f9bbd4426..aae3300febed 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -2189,7 +2189,7 @@ compiler_async_for(struct compiler *c, stmt_ty s) _Py_IDENTIFIER(StopAsyncIteration); basicblock *try, *except, *end, *after_try, *try_cleanup, - *after_loop, *after_loop_else; + *after_loop_else; PyObject *stop_aiter_error = _PyUnicode_FromId(&PyId_StopAsyncIteration); if (stop_aiter_error == NULL) { @@ -2201,14 +2201,14 @@ compiler_async_for(struct compiler *c, stmt_ty s) end = compiler_new_block(c); after_try = compiler_new_block(c); try_cleanup = compiler_new_block(c); - after_loop = compiler_new_block(c); after_loop_else = compiler_new_block(c); if (try == NULL || except == NULL || end == NULL - || after_try == NULL || try_cleanup == NULL) + || after_try == NULL || try_cleanup == NULL + || after_loop_else == NULL) return 0; - ADDOP_JREL(c, SETUP_LOOP, after_loop); + ADDOP_JREL(c, SETUP_LOOP, end); if (!compiler_push_fblock(c, LOOP, try)) return 0; @@ -2257,9 +2257,6 @@ compiler_async_for(struct compiler *c, stmt_ty s) ADDOP(c, POP_BLOCK); /* for SETUP_LOOP */ compiler_pop_fblock(c, LOOP, try); - compiler_use_next_block(c, after_loop); - ADDOP_JABS(c, JUMP_ABSOLUTE, end); - compiler_use_next_block(c, after_loop_else); VISIT_SEQ(c, stmt, s->v.For.orelse); @@ -3753,7 +3750,7 @@ compiler_async_comprehension_generator(struct compiler *c, _Py_IDENTIFIER(StopAsyncIteration); comprehension_ty gen; - basicblock *anchor, *skip, *if_cleanup, *try, + basicblock *anchor, *if_cleanup, *try, *after_try, *except, *try_cleanup; Py_ssize_t i, n; @@ -3766,13 +3763,12 @@ compiler_async_comprehension_generator(struct compiler *c, after_try = compiler_new_block(c); try_cleanup = compiler_new_block(c); except = compiler_new_block(c); - skip = compiler_new_block(c); if_cleanup = compiler_new_block(c); anchor = compiler_new_block(c); - if (skip == NULL || if_cleanup == NULL || anchor == NULL || + if (if_cleanup == NULL || anchor == NULL || try == NULL || after_try == NULL || - except == NULL || after_try == NULL) { + except == NULL || try_cleanup == NULL) { return 0; } @@ -3866,8 +3862,6 @@ compiler_async_comprehension_generator(struct compiler *c, default: return 0; } - - compiler_use_next_block(c, skip); } compiler_use_next_block(c, if_cleanup); ADDOP_JABS(c, JUMP_ABSOLUTE, try); From webhook-mailer at python.org Sat Mar 10 17:08:35 2018 From: webhook-mailer at python.org (Berker Peksag) Date: Sat, 10 Mar 2018 22:08:35 -0000 Subject: [Python-checkins] bpo-27645: Add support for native backup facility of SQLite (GH-4238) Message-ID: <mailman.26.1520719717.1871.python-checkins@python.org> https://github.com/python/cpython/commit/d7aed4102d2a40c74553240c7f03585624d27aea commit: d7aed4102d2a40c74553240c7f03585624d27aea branch: master author: Emanuele Gaifas <lelegaifax at gmail.com> committer: Berker Peksag <berker.peksag at gmail.com> date: 2018-03-11T01:08:31+03:00 summary: bpo-27645: Add support for native backup facility of SQLite (GH-4238) files: A Lib/sqlite3/test/backup.py A Misc/NEWS.d/next/Library/2017-10-05-20-41-48.bpo-27645.1Y_Wag.rst M Doc/library/sqlite3.rst M Doc/whatsnew/3.7.rst M Lib/test/test_sqlite.py M Modules/_sqlite/connection.c M Modules/_sqlite/module.c diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst index e7676a9f3a50..d7eaea638f82 100644 --- a/Doc/library/sqlite3.rst +++ b/Doc/library/sqlite3.rst @@ -532,6 +532,56 @@ Connection Objects f.write('%s\n' % line) + .. method:: backup(target, *, pages=0, progress=None, name="main", sleep=0.250) + + This method makes a backup of a SQLite database even while it's being accessed + by other clients, or concurrently by the same connection. The copy will be + written into the mandatory argument *target*, that must be another + :class:`Connection` instance. + + By default, or when *pages* is either ``0`` or a negative integer, the entire + database is copied in a single step; otherwise the method performs a loop + copying up to *pages* pages at a time. + + If *progress* is specified, it must either be ``None`` or a callable object that + will be executed at each iteration with three integer arguments, respectively + the *status* of the last iteration, the *remaining* number of pages still to be + copied and the *total* number of pages. + + The *name* argument specifies the database name that will be copied: it must be + a string containing either ``"main"``, the default, to indicate the main + database, ``"temp"`` to indicate the temporary database or the name specified + after the ``AS`` keyword in an ``ATTACH DATABASE`` statement for an attached + database. + + The *sleep* argument specifies the number of seconds to sleep by between + successive attempts to backup remaining pages, can be specified either as an + integer or a floating point value. + + Example 1, copy an existing database into another:: + + import sqlite3 + + def progress(status, remaining, total): + print(f'Copied {total-remaining} of {total} pages...') + + con = sqlite3.connect('existing_db.db') + with sqlite3.connect('backup.db') as bck: + con.backup(bck, pages=1, progress=progress) + + Example 2, copy an existing database into a transient copy:: + + import sqlite3 + + source = sqlite3.connect('existing_db.db') + dest = sqlite3.connect(':memory:') + source.backup(dest) + + Availability: SQLite 3.6.11 or higher + + .. versionadded:: 3.7 + + .. _sqlite3-cursor-objects: Cursor Objects diff --git a/Doc/whatsnew/3.7.rst b/Doc/whatsnew/3.7.rst index 76e1f7b36b0b..fc5f5ab77095 100644 --- a/Doc/whatsnew/3.7.rst +++ b/Doc/whatsnew/3.7.rst @@ -630,6 +630,15 @@ can be set within the scope of a group. ``'^$'`` or ``(?=-)`` that matches an empty string. (Contributed by Serhiy Storchaka in :issue:`25054`.) + +sqlite3 +------- + +:class:`sqlite3.Connection` now exposes a :class:`~sqlite3.Connection.backup` +method, if the underlying SQLite library is at version 3.6.11 or higher. +(Contributed by Lele Gaifax in :issue:`27645`.) + + ssl --- diff --git a/Lib/sqlite3/test/backup.py b/Lib/sqlite3/test/backup.py new file mode 100644 index 000000000000..784702fb46e4 --- /dev/null +++ b/Lib/sqlite3/test/backup.py @@ -0,0 +1,162 @@ +import sqlite3 as sqlite +import unittest + + + at unittest.skipIf(sqlite.sqlite_version_info < (3, 6, 11), "Backup API not supported") +class BackupTests(unittest.TestCase): + def setUp(self): + cx = self.cx = sqlite.connect(":memory:") + cx.execute('CREATE TABLE foo (key INTEGER)') + cx.executemany('INSERT INTO foo (key) VALUES (?)', [(3,), (4,)]) + cx.commit() + + def tearDown(self): + self.cx.close() + + def verify_backup(self, bckcx): + result = bckcx.execute("SELECT key FROM foo ORDER BY key").fetchall() + self.assertEqual(result[0][0], 3) + self.assertEqual(result[1][0], 4) + + def test_bad_target_none(self): + with self.assertRaises(TypeError): + self.cx.backup(None) + + def test_bad_target_filename(self): + with self.assertRaises(TypeError): + self.cx.backup('some_file_name.db') + + def test_bad_target_same_connection(self): + with self.assertRaises(ValueError): + self.cx.backup(self.cx) + + def test_bad_target_closed_connection(self): + bck = sqlite.connect(':memory:') + bck.close() + with self.assertRaises(sqlite.ProgrammingError): + self.cx.backup(bck) + + def test_bad_target_in_transaction(self): + bck = sqlite.connect(':memory:') + bck.execute('CREATE TABLE bar (key INTEGER)') + bck.executemany('INSERT INTO bar (key) VALUES (?)', [(3,), (4,)]) + with self.assertRaises(sqlite.OperationalError) as cm: + self.cx.backup(bck) + if sqlite.sqlite_version_info < (3, 8, 7): + self.assertEqual(str(cm.exception), 'target is in transaction') + + def test_keyword_only_args(self): + with self.assertRaises(TypeError): + with sqlite.connect(':memory:') as bck: + self.cx.backup(bck, 1) + + def test_simple(self): + with sqlite.connect(':memory:') as bck: + self.cx.backup(bck) + self.verify_backup(bck) + + def test_progress(self): + journal = [] + + def progress(status, remaining, total): + journal.append(status) + + with sqlite.connect(':memory:') as bck: + self.cx.backup(bck, pages=1, progress=progress) + self.verify_backup(bck) + + self.assertEqual(len(journal), 2) + self.assertEqual(journal[0], sqlite.SQLITE_OK) + self.assertEqual(journal[1], sqlite.SQLITE_DONE) + + def test_progress_all_pages_at_once_1(self): + journal = [] + + def progress(status, remaining, total): + journal.append(remaining) + + with sqlite.connect(':memory:') as bck: + self.cx.backup(bck, progress=progress) + self.verify_backup(bck) + + self.assertEqual(len(journal), 1) + self.assertEqual(journal[0], 0) + + def test_progress_all_pages_at_once_2(self): + journal = [] + + def progress(status, remaining, total): + journal.append(remaining) + + with sqlite.connect(':memory:') as bck: + self.cx.backup(bck, pages=-1, progress=progress) + self.verify_backup(bck) + + self.assertEqual(len(journal), 1) + self.assertEqual(journal[0], 0) + + def test_non_callable_progress(self): + with self.assertRaises(TypeError) as cm: + with sqlite.connect(':memory:') as bck: + self.cx.backup(bck, pages=1, progress='bar') + self.assertEqual(str(cm.exception), 'progress argument must be a callable') + + def test_modifying_progress(self): + journal = [] + + def progress(status, remaining, total): + if not journal: + self.cx.execute('INSERT INTO foo (key) VALUES (?)', (remaining+1000,)) + self.cx.commit() + journal.append(remaining) + + with sqlite.connect(':memory:') as bck: + self.cx.backup(bck, pages=1, progress=progress) + self.verify_backup(bck) + + result = bck.execute("SELECT key FROM foo" + " WHERE key >= 1000" + " ORDER BY key").fetchall() + self.assertEqual(result[0][0], 1001) + + self.assertEqual(len(journal), 3) + self.assertEqual(journal[0], 1) + self.assertEqual(journal[1], 1) + self.assertEqual(journal[2], 0) + + def test_failing_progress(self): + def progress(status, remaining, total): + raise SystemError('nearly out of space') + + with self.assertRaises(SystemError) as err: + with sqlite.connect(':memory:') as bck: + self.cx.backup(bck, progress=progress) + self.assertEqual(str(err.exception), 'nearly out of space') + + def test_database_source_name(self): + with sqlite.connect(':memory:') as bck: + self.cx.backup(bck, name='main') + with sqlite.connect(':memory:') as bck: + self.cx.backup(bck, name='temp') + with self.assertRaises(sqlite.OperationalError) as cm: + with sqlite.connect(':memory:') as bck: + self.cx.backup(bck, name='non-existing') + self.assertIn( + str(cm.exception), + ['SQL logic error', 'SQL logic error or missing database'] + ) + + self.cx.execute("ATTACH DATABASE ':memory:' AS attached_db") + self.cx.execute('CREATE TABLE attached_db.foo (key INTEGER)') + self.cx.executemany('INSERT INTO attached_db.foo (key) VALUES (?)', [(3,), (4,)]) + self.cx.commit() + with sqlite.connect(':memory:') as bck: + self.cx.backup(bck, name='attached_db') + self.verify_backup(bck) + + +def suite(): + return unittest.makeSuite(BackupTests) + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_sqlite.py b/Lib/test/test_sqlite.py index adfcd9994575..9564da35193f 100644 --- a/Lib/test/test_sqlite.py +++ b/Lib/test/test_sqlite.py @@ -7,7 +7,7 @@ import sqlite3 from sqlite3.test import (dbapi, types, userfunctions, factory, transactions, hooks, regression, - dump) + dump, backup) def load_tests(*args): if test.support.verbose: @@ -18,7 +18,8 @@ def load_tests(*args): userfunctions.suite(), factory.suite(), transactions.suite(), hooks.suite(), regression.suite(), - dump.suite()]) + dump.suite(), + backup.suite()]) if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS.d/next/Library/2017-10-05-20-41-48.bpo-27645.1Y_Wag.rst b/Misc/NEWS.d/next/Library/2017-10-05-20-41-48.bpo-27645.1Y_Wag.rst new file mode 100644 index 000000000000..c4b7185614a5 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2017-10-05-20-41-48.bpo-27645.1Y_Wag.rst @@ -0,0 +1,3 @@ +:class:`sqlite3.Connection` now exposes a :class:`~sqlite3.Connection.backup` +method, if the underlying SQLite library is at version 3.6.11 +or higher. Patch by Lele Gaifax. diff --git a/Modules/_sqlite/connection.c b/Modules/_sqlite/connection.c index 3e83fb662bba..14b6a2774e78 100644 --- a/Modules/_sqlite/connection.c +++ b/Modules/_sqlite/connection.c @@ -41,6 +41,10 @@ #endif #endif +#if SQLITE_VERSION_NUMBER >= 3006011 +#define HAVE_BACKUP_API +#endif + _Py_IDENTIFIER(cursor); static const char * const begin_statements[] = { @@ -1447,6 +1451,137 @@ pysqlite_connection_iterdump(pysqlite_Connection* self, PyObject* args) return retval; } +#ifdef HAVE_BACKUP_API +static PyObject * +pysqlite_connection_backup(pysqlite_Connection *self, PyObject *args, PyObject *kwds) +{ + PyObject *target = NULL; + int pages = -1; + PyObject *progress = Py_None; + const char *name = "main"; + int rc; + int callback_error = 0; + double sleep_secs = 0.250; + sqlite3 *bck_conn; + sqlite3_backup *bck_handle; + static char *keywords[] = {"target", "pages", "progress", "name", "sleep", NULL}; + + if (!PyArg_ParseTupleAndKeywords(args, kwds, "O!|$iOsd:backup", keywords, + &pysqlite_ConnectionType, &target, + &pages, &progress, &name, &sleep_secs)) { + return NULL; + } + + if (!pysqlite_check_connection((pysqlite_Connection *)target)) { + return NULL; + } + + if ((pysqlite_Connection *)target == self) { + PyErr_SetString(PyExc_ValueError, "target cannot be the same connection instance"); + return NULL; + } + +#if SQLITE_VERSION_NUMBER < 3008007 + /* Since 3.8.7 this is already done, per commit + https://www.sqlite.org/src/info/169b5505498c0a7e */ + if (!sqlite3_get_autocommit(((pysqlite_Connection *)target)->db)) { + PyErr_SetString(pysqlite_OperationalError, "target is in transaction"); + return NULL; + } +#endif + + if (progress != Py_None && !PyCallable_Check(progress)) { + PyErr_SetString(PyExc_TypeError, "progress argument must be a callable"); + return NULL; + } + + if (pages == 0) { + pages = -1; + } + + bck_conn = ((pysqlite_Connection *)target)->db; + + Py_BEGIN_ALLOW_THREADS + bck_handle = sqlite3_backup_init(bck_conn, "main", self->db, name); + Py_END_ALLOW_THREADS + + if (bck_handle) { + do { + Py_BEGIN_ALLOW_THREADS + rc = sqlite3_backup_step(bck_handle, pages); + Py_END_ALLOW_THREADS + + if (progress != Py_None) { + PyObject *res; + + res = PyObject_CallFunction(progress, "iii", rc, + sqlite3_backup_remaining(bck_handle), + sqlite3_backup_pagecount(bck_handle)); + if (res == NULL) { + /* User's callback raised an error: interrupt the loop and + propagate it. */ + callback_error = 1; + rc = -1; + } else { + Py_DECREF(res); + } + } + + /* Sleep for a while if there are still further pages to copy and + the engine could not make any progress */ + if (rc == SQLITE_BUSY || rc == SQLITE_LOCKED) { + Py_BEGIN_ALLOW_THREADS + sqlite3_sleep(sleep_secs * 1000.0); + Py_END_ALLOW_THREADS + } + } while (rc == SQLITE_OK || rc == SQLITE_BUSY || rc == SQLITE_LOCKED); + + Py_BEGIN_ALLOW_THREADS + rc = sqlite3_backup_finish(bck_handle); + Py_END_ALLOW_THREADS + } else { + rc = _pysqlite_seterror(bck_conn, NULL); + } + + if (!callback_error && rc != SQLITE_OK) { + /* We cannot use _pysqlite_seterror() here because the backup APIs do + not set the error status on the connection object, but rather on + the backup handle. */ + if (rc == SQLITE_NOMEM) { + (void)PyErr_NoMemory(); + } else { +#if SQLITE_VERSION_NUMBER > 3007015 + PyErr_SetString(pysqlite_OperationalError, sqlite3_errstr(rc)); +#else + switch (rc) { + case SQLITE_READONLY: + PyErr_SetString(pysqlite_OperationalError, + "attempt to write a readonly database"); + break; + case SQLITE_BUSY: + PyErr_SetString(pysqlite_OperationalError, "database is locked"); + break; + case SQLITE_LOCKED: + PyErr_SetString(pysqlite_OperationalError, + "database table is locked"); + break; + default: + PyErr_Format(pysqlite_OperationalError, + "unrecognized error code: %d", rc); + break; + } +#endif + } + } + + if (!callback_error && rc == SQLITE_OK) { + Py_RETURN_NONE; + } else { + return NULL; + } +} +#endif + static PyObject * pysqlite_connection_create_collation(pysqlite_Connection* self, PyObject* args) { @@ -1619,6 +1754,10 @@ static PyMethodDef connection_methods[] = { PyDoc_STR("Abort any pending database operation. Non-standard.")}, {"iterdump", (PyCFunction)pysqlite_connection_iterdump, METH_NOARGS, PyDoc_STR("Returns iterator to the dump of the database in an SQL text format. Non-standard.")}, + #ifdef HAVE_BACKUP_API + {"backup", (PyCFunction)pysqlite_connection_backup, METH_VARARGS | METH_KEYWORDS, + PyDoc_STR("Makes a backup of the database. Non-standard.")}, + #endif {"__enter__", (PyCFunction)pysqlite_connection_enter, METH_NOARGS, PyDoc_STR("For context manager. Non-standard.")}, {"__exit__", (PyCFunction)pysqlite_connection_exit, METH_VARARGS, diff --git a/Modules/_sqlite/module.c b/Modules/_sqlite/module.c index 879c66692bc9..6befa073dcf0 100644 --- a/Modules/_sqlite/module.c +++ b/Modules/_sqlite/module.c @@ -322,6 +322,9 @@ static const IntConstantPair _int_constants[] = { #endif #if SQLITE_VERSION_NUMBER >= 3008003 {"SQLITE_RECURSIVE", SQLITE_RECURSIVE}, +#endif +#if SQLITE_VERSION_NUMBER >= 3006011 + {"SQLITE_DONE", SQLITE_DONE}, #endif {(char*)NULL, 0} }; From webhook-mailer at python.org Sat Mar 10 17:29:22 2018 From: webhook-mailer at python.org (Berker Peksag) Date: Sat, 10 Mar 2018 22:29:22 -0000 Subject: [Python-checkins] bpo-27645: Add support for native backup facility of SQLite (GH-4238) Message-ID: <mailman.27.1520720963.1871.python-checkins@python.org> https://github.com/python/cpython/commit/e8a5a92037b1f27809806bb87c17976d2d48d3e9 commit: e8a5a92037b1f27809806bb87c17976d2d48d3e9 branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Berker Peksag <berker.peksag at gmail.com> date: 2018-03-11T01:29:19+03:00 summary: bpo-27645: Add support for native backup facility of SQLite (GH-4238) (cherry picked from commit d7aed4102d2a40c74553240c7f03585624d27aea) Co-authored-by: Emanuele Gaifas <lelegaifax at gmail.com> files: A Lib/sqlite3/test/backup.py A Misc/NEWS.d/next/Library/2017-10-05-20-41-48.bpo-27645.1Y_Wag.rst M Doc/library/sqlite3.rst M Doc/whatsnew/3.7.rst M Lib/test/test_sqlite.py M Modules/_sqlite/connection.c M Modules/_sqlite/module.c diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst index e7676a9f3a50..d7eaea638f82 100644 --- a/Doc/library/sqlite3.rst +++ b/Doc/library/sqlite3.rst @@ -532,6 +532,56 @@ Connection Objects f.write('%s\n' % line) + .. method:: backup(target, *, pages=0, progress=None, name="main", sleep=0.250) + + This method makes a backup of a SQLite database even while it's being accessed + by other clients, or concurrently by the same connection. The copy will be + written into the mandatory argument *target*, that must be another + :class:`Connection` instance. + + By default, or when *pages* is either ``0`` or a negative integer, the entire + database is copied in a single step; otherwise the method performs a loop + copying up to *pages* pages at a time. + + If *progress* is specified, it must either be ``None`` or a callable object that + will be executed at each iteration with three integer arguments, respectively + the *status* of the last iteration, the *remaining* number of pages still to be + copied and the *total* number of pages. + + The *name* argument specifies the database name that will be copied: it must be + a string containing either ``"main"``, the default, to indicate the main + database, ``"temp"`` to indicate the temporary database or the name specified + after the ``AS`` keyword in an ``ATTACH DATABASE`` statement for an attached + database. + + The *sleep* argument specifies the number of seconds to sleep by between + successive attempts to backup remaining pages, can be specified either as an + integer or a floating point value. + + Example 1, copy an existing database into another:: + + import sqlite3 + + def progress(status, remaining, total): + print(f'Copied {total-remaining} of {total} pages...') + + con = sqlite3.connect('existing_db.db') + with sqlite3.connect('backup.db') as bck: + con.backup(bck, pages=1, progress=progress) + + Example 2, copy an existing database into a transient copy:: + + import sqlite3 + + source = sqlite3.connect('existing_db.db') + dest = sqlite3.connect(':memory:') + source.backup(dest) + + Availability: SQLite 3.6.11 or higher + + .. versionadded:: 3.7 + + .. _sqlite3-cursor-objects: Cursor Objects diff --git a/Doc/whatsnew/3.7.rst b/Doc/whatsnew/3.7.rst index 76e1f7b36b0b..fc5f5ab77095 100644 --- a/Doc/whatsnew/3.7.rst +++ b/Doc/whatsnew/3.7.rst @@ -630,6 +630,15 @@ can be set within the scope of a group. ``'^$'`` or ``(?=-)`` that matches an empty string. (Contributed by Serhiy Storchaka in :issue:`25054`.) + +sqlite3 +------- + +:class:`sqlite3.Connection` now exposes a :class:`~sqlite3.Connection.backup` +method, if the underlying SQLite library is at version 3.6.11 or higher. +(Contributed by Lele Gaifax in :issue:`27645`.) + + ssl --- diff --git a/Lib/sqlite3/test/backup.py b/Lib/sqlite3/test/backup.py new file mode 100644 index 000000000000..784702fb46e4 --- /dev/null +++ b/Lib/sqlite3/test/backup.py @@ -0,0 +1,162 @@ +import sqlite3 as sqlite +import unittest + + + at unittest.skipIf(sqlite.sqlite_version_info < (3, 6, 11), "Backup API not supported") +class BackupTests(unittest.TestCase): + def setUp(self): + cx = self.cx = sqlite.connect(":memory:") + cx.execute('CREATE TABLE foo (key INTEGER)') + cx.executemany('INSERT INTO foo (key) VALUES (?)', [(3,), (4,)]) + cx.commit() + + def tearDown(self): + self.cx.close() + + def verify_backup(self, bckcx): + result = bckcx.execute("SELECT key FROM foo ORDER BY key").fetchall() + self.assertEqual(result[0][0], 3) + self.assertEqual(result[1][0], 4) + + def test_bad_target_none(self): + with self.assertRaises(TypeError): + self.cx.backup(None) + + def test_bad_target_filename(self): + with self.assertRaises(TypeError): + self.cx.backup('some_file_name.db') + + def test_bad_target_same_connection(self): + with self.assertRaises(ValueError): + self.cx.backup(self.cx) + + def test_bad_target_closed_connection(self): + bck = sqlite.connect(':memory:') + bck.close() + with self.assertRaises(sqlite.ProgrammingError): + self.cx.backup(bck) + + def test_bad_target_in_transaction(self): + bck = sqlite.connect(':memory:') + bck.execute('CREATE TABLE bar (key INTEGER)') + bck.executemany('INSERT INTO bar (key) VALUES (?)', [(3,), (4,)]) + with self.assertRaises(sqlite.OperationalError) as cm: + self.cx.backup(bck) + if sqlite.sqlite_version_info < (3, 8, 7): + self.assertEqual(str(cm.exception), 'target is in transaction') + + def test_keyword_only_args(self): + with self.assertRaises(TypeError): + with sqlite.connect(':memory:') as bck: + self.cx.backup(bck, 1) + + def test_simple(self): + with sqlite.connect(':memory:') as bck: + self.cx.backup(bck) + self.verify_backup(bck) + + def test_progress(self): + journal = [] + + def progress(status, remaining, total): + journal.append(status) + + with sqlite.connect(':memory:') as bck: + self.cx.backup(bck, pages=1, progress=progress) + self.verify_backup(bck) + + self.assertEqual(len(journal), 2) + self.assertEqual(journal[0], sqlite.SQLITE_OK) + self.assertEqual(journal[1], sqlite.SQLITE_DONE) + + def test_progress_all_pages_at_once_1(self): + journal = [] + + def progress(status, remaining, total): + journal.append(remaining) + + with sqlite.connect(':memory:') as bck: + self.cx.backup(bck, progress=progress) + self.verify_backup(bck) + + self.assertEqual(len(journal), 1) + self.assertEqual(journal[0], 0) + + def test_progress_all_pages_at_once_2(self): + journal = [] + + def progress(status, remaining, total): + journal.append(remaining) + + with sqlite.connect(':memory:') as bck: + self.cx.backup(bck, pages=-1, progress=progress) + self.verify_backup(bck) + + self.assertEqual(len(journal), 1) + self.assertEqual(journal[0], 0) + + def test_non_callable_progress(self): + with self.assertRaises(TypeError) as cm: + with sqlite.connect(':memory:') as bck: + self.cx.backup(bck, pages=1, progress='bar') + self.assertEqual(str(cm.exception), 'progress argument must be a callable') + + def test_modifying_progress(self): + journal = [] + + def progress(status, remaining, total): + if not journal: + self.cx.execute('INSERT INTO foo (key) VALUES (?)', (remaining+1000,)) + self.cx.commit() + journal.append(remaining) + + with sqlite.connect(':memory:') as bck: + self.cx.backup(bck, pages=1, progress=progress) + self.verify_backup(bck) + + result = bck.execute("SELECT key FROM foo" + " WHERE key >= 1000" + " ORDER BY key").fetchall() + self.assertEqual(result[0][0], 1001) + + self.assertEqual(len(journal), 3) + self.assertEqual(journal[0], 1) + self.assertEqual(journal[1], 1) + self.assertEqual(journal[2], 0) + + def test_failing_progress(self): + def progress(status, remaining, total): + raise SystemError('nearly out of space') + + with self.assertRaises(SystemError) as err: + with sqlite.connect(':memory:') as bck: + self.cx.backup(bck, progress=progress) + self.assertEqual(str(err.exception), 'nearly out of space') + + def test_database_source_name(self): + with sqlite.connect(':memory:') as bck: + self.cx.backup(bck, name='main') + with sqlite.connect(':memory:') as bck: + self.cx.backup(bck, name='temp') + with self.assertRaises(sqlite.OperationalError) as cm: + with sqlite.connect(':memory:') as bck: + self.cx.backup(bck, name='non-existing') + self.assertIn( + str(cm.exception), + ['SQL logic error', 'SQL logic error or missing database'] + ) + + self.cx.execute("ATTACH DATABASE ':memory:' AS attached_db") + self.cx.execute('CREATE TABLE attached_db.foo (key INTEGER)') + self.cx.executemany('INSERT INTO attached_db.foo (key) VALUES (?)', [(3,), (4,)]) + self.cx.commit() + with sqlite.connect(':memory:') as bck: + self.cx.backup(bck, name='attached_db') + self.verify_backup(bck) + + +def suite(): + return unittest.makeSuite(BackupTests) + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_sqlite.py b/Lib/test/test_sqlite.py index adfcd9994575..9564da35193f 100644 --- a/Lib/test/test_sqlite.py +++ b/Lib/test/test_sqlite.py @@ -7,7 +7,7 @@ import sqlite3 from sqlite3.test import (dbapi, types, userfunctions, factory, transactions, hooks, regression, - dump) + dump, backup) def load_tests(*args): if test.support.verbose: @@ -18,7 +18,8 @@ def load_tests(*args): userfunctions.suite(), factory.suite(), transactions.suite(), hooks.suite(), regression.suite(), - dump.suite()]) + dump.suite(), + backup.suite()]) if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS.d/next/Library/2017-10-05-20-41-48.bpo-27645.1Y_Wag.rst b/Misc/NEWS.d/next/Library/2017-10-05-20-41-48.bpo-27645.1Y_Wag.rst new file mode 100644 index 000000000000..c4b7185614a5 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2017-10-05-20-41-48.bpo-27645.1Y_Wag.rst @@ -0,0 +1,3 @@ +:class:`sqlite3.Connection` now exposes a :class:`~sqlite3.Connection.backup` +method, if the underlying SQLite library is at version 3.6.11 +or higher. Patch by Lele Gaifax. diff --git a/Modules/_sqlite/connection.c b/Modules/_sqlite/connection.c index 3e83fb662bba..14b6a2774e78 100644 --- a/Modules/_sqlite/connection.c +++ b/Modules/_sqlite/connection.c @@ -41,6 +41,10 @@ #endif #endif +#if SQLITE_VERSION_NUMBER >= 3006011 +#define HAVE_BACKUP_API +#endif + _Py_IDENTIFIER(cursor); static const char * const begin_statements[] = { @@ -1447,6 +1451,137 @@ pysqlite_connection_iterdump(pysqlite_Connection* self, PyObject* args) return retval; } +#ifdef HAVE_BACKUP_API +static PyObject * +pysqlite_connection_backup(pysqlite_Connection *self, PyObject *args, PyObject *kwds) +{ + PyObject *target = NULL; + int pages = -1; + PyObject *progress = Py_None; + const char *name = "main"; + int rc; + int callback_error = 0; + double sleep_secs = 0.250; + sqlite3 *bck_conn; + sqlite3_backup *bck_handle; + static char *keywords[] = {"target", "pages", "progress", "name", "sleep", NULL}; + + if (!PyArg_ParseTupleAndKeywords(args, kwds, "O!|$iOsd:backup", keywords, + &pysqlite_ConnectionType, &target, + &pages, &progress, &name, &sleep_secs)) { + return NULL; + } + + if (!pysqlite_check_connection((pysqlite_Connection *)target)) { + return NULL; + } + + if ((pysqlite_Connection *)target == self) { + PyErr_SetString(PyExc_ValueError, "target cannot be the same connection instance"); + return NULL; + } + +#if SQLITE_VERSION_NUMBER < 3008007 + /* Since 3.8.7 this is already done, per commit + https://www.sqlite.org/src/info/169b5505498c0a7e */ + if (!sqlite3_get_autocommit(((pysqlite_Connection *)target)->db)) { + PyErr_SetString(pysqlite_OperationalError, "target is in transaction"); + return NULL; + } +#endif + + if (progress != Py_None && !PyCallable_Check(progress)) { + PyErr_SetString(PyExc_TypeError, "progress argument must be a callable"); + return NULL; + } + + if (pages == 0) { + pages = -1; + } + + bck_conn = ((pysqlite_Connection *)target)->db; + + Py_BEGIN_ALLOW_THREADS + bck_handle = sqlite3_backup_init(bck_conn, "main", self->db, name); + Py_END_ALLOW_THREADS + + if (bck_handle) { + do { + Py_BEGIN_ALLOW_THREADS + rc = sqlite3_backup_step(bck_handle, pages); + Py_END_ALLOW_THREADS + + if (progress != Py_None) { + PyObject *res; + + res = PyObject_CallFunction(progress, "iii", rc, + sqlite3_backup_remaining(bck_handle), + sqlite3_backup_pagecount(bck_handle)); + if (res == NULL) { + /* User's callback raised an error: interrupt the loop and + propagate it. */ + callback_error = 1; + rc = -1; + } else { + Py_DECREF(res); + } + } + + /* Sleep for a while if there are still further pages to copy and + the engine could not make any progress */ + if (rc == SQLITE_BUSY || rc == SQLITE_LOCKED) { + Py_BEGIN_ALLOW_THREADS + sqlite3_sleep(sleep_secs * 1000.0); + Py_END_ALLOW_THREADS + } + } while (rc == SQLITE_OK || rc == SQLITE_BUSY || rc == SQLITE_LOCKED); + + Py_BEGIN_ALLOW_THREADS + rc = sqlite3_backup_finish(bck_handle); + Py_END_ALLOW_THREADS + } else { + rc = _pysqlite_seterror(bck_conn, NULL); + } + + if (!callback_error && rc != SQLITE_OK) { + /* We cannot use _pysqlite_seterror() here because the backup APIs do + not set the error status on the connection object, but rather on + the backup handle. */ + if (rc == SQLITE_NOMEM) { + (void)PyErr_NoMemory(); + } else { +#if SQLITE_VERSION_NUMBER > 3007015 + PyErr_SetString(pysqlite_OperationalError, sqlite3_errstr(rc)); +#else + switch (rc) { + case SQLITE_READONLY: + PyErr_SetString(pysqlite_OperationalError, + "attempt to write a readonly database"); + break; + case SQLITE_BUSY: + PyErr_SetString(pysqlite_OperationalError, "database is locked"); + break; + case SQLITE_LOCKED: + PyErr_SetString(pysqlite_OperationalError, + "database table is locked"); + break; + default: + PyErr_Format(pysqlite_OperationalError, + "unrecognized error code: %d", rc); + break; + } +#endif + } + } + + if (!callback_error && rc == SQLITE_OK) { + Py_RETURN_NONE; + } else { + return NULL; + } +} +#endif + static PyObject * pysqlite_connection_create_collation(pysqlite_Connection* self, PyObject* args) { @@ -1619,6 +1754,10 @@ static PyMethodDef connection_methods[] = { PyDoc_STR("Abort any pending database operation. Non-standard.")}, {"iterdump", (PyCFunction)pysqlite_connection_iterdump, METH_NOARGS, PyDoc_STR("Returns iterator to the dump of the database in an SQL text format. Non-standard.")}, + #ifdef HAVE_BACKUP_API + {"backup", (PyCFunction)pysqlite_connection_backup, METH_VARARGS | METH_KEYWORDS, + PyDoc_STR("Makes a backup of the database. Non-standard.")}, + #endif {"__enter__", (PyCFunction)pysqlite_connection_enter, METH_NOARGS, PyDoc_STR("For context manager. Non-standard.")}, {"__exit__", (PyCFunction)pysqlite_connection_exit, METH_VARARGS, diff --git a/Modules/_sqlite/module.c b/Modules/_sqlite/module.c index 879c66692bc9..6befa073dcf0 100644 --- a/Modules/_sqlite/module.c +++ b/Modules/_sqlite/module.c @@ -322,6 +322,9 @@ static const IntConstantPair _int_constants[] = { #endif #if SQLITE_VERSION_NUMBER >= 3008003 {"SQLITE_RECURSIVE", SQLITE_RECURSIVE}, +#endif +#if SQLITE_VERSION_NUMBER >= 3006011 + {"SQLITE_DONE", SQLITE_DONE}, #endif {(char*)NULL, 0} }; From webhook-mailer at python.org Sat Mar 10 18:11:50 2018 From: webhook-mailer at python.org (Miss Islington (bot)) Date: Sat, 10 Mar 2018 23:11:50 -0000 Subject: [Python-checkins] bpo-32836: Remove obsolete code from symtable pass (GH-5680) Message-ID: <mailman.28.1520723511.1871.python-checkins@python.org> https://github.com/python/cpython/commit/5506d603021518eaaa89e7037905f7a698c5e95c commit: 5506d603021518eaaa89e7037905f7a698c5e95c branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: GitHub <noreply at github.com> date: 2018-03-10T15:11:47-08:00 summary: bpo-32836: Remove obsolete code from symtable pass (GH-5680) When comprehensions switched to using a nested scope, the old code for generating a temporary name to hold the accumulation target became redundant, but was never actually removed. Patch by Nitish Chandra. (cherry picked from commit 3a087beddd9f0955eb9080a6fd1499ff89ca74bf) Co-authored-by: Nitish Chandra <nitishchandrachinta at gmail.com> files: A Misc/NEWS.d/next/Core and Builtins/2018-02-14-12-35-47.bpo-32836.bThJnx.rst M Include/symtable.h M Python/symtable.c diff --git a/Include/symtable.h b/Include/symtable.h index 86ae3c28e878..007f88db40e7 100644 --- a/Include/symtable.h +++ b/Include/symtable.h @@ -60,7 +60,6 @@ typedef struct _symtable_entry { int ste_col_offset; /* offset of first line of block */ int ste_opt_lineno; /* lineno of last exec or import * */ int ste_opt_col_offset; /* offset of last exec or import * */ - int ste_tmpname; /* counter for listcomp temp vars */ struct symtable *ste_table; } PySTEntryObject; diff --git a/Misc/NEWS.d/next/Core and Builtins/2018-02-14-12-35-47.bpo-32836.bThJnx.rst b/Misc/NEWS.d/next/Core and Builtins/2018-02-14-12-35-47.bpo-32836.bThJnx.rst new file mode 100644 index 000000000000..4eeb9aa2e52c --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2018-02-14-12-35-47.bpo-32836.bThJnx.rst @@ -0,0 +1 @@ +Don't use temporary variables in cases of list/dict/set comprehensions diff --git a/Python/symtable.c b/Python/symtable.c index bbac25cf3767..1c328a996140 100644 --- a/Python/symtable.c +++ b/Python/symtable.c @@ -69,7 +69,6 @@ ste_new(struct symtable *st, identifier name, _Py_block_ty block, ste->ste_varkeywords = 0; ste->ste_opt_lineno = 0; ste->ste_opt_col_offset = 0; - ste->ste_tmpname = 0; ste->ste_lineno = lineno; ste->ste_col_offset = col_offset; @@ -1082,24 +1081,6 @@ symtable_add_def(struct symtable *st, PyObject *name, int flag) } \ } -static int -symtable_new_tmpname(struct symtable *st) -{ - char tmpname[256]; - identifier tmp; - - PyOS_snprintf(tmpname, sizeof(tmpname), "_[%d]", - ++st->st_cur->ste_tmpname); - tmp = PyUnicode_InternFromString(tmpname); - if (!tmp) - return 0; - if (!symtable_add_def(st, tmp, DEF_LOCAL)) - return 0; - Py_DECREF(tmp); - return 1; -} - - static int symtable_record_directive(struct symtable *st, identifier name, stmt_ty s) { @@ -1723,7 +1704,6 @@ symtable_handle_comprehension(struct symtable *st, expr_ty e, expr_ty elt, expr_ty value) { int is_generator = (e->kind == GeneratorExp_kind); - int needs_tmp = !is_generator; comprehension_ty outermost = ((comprehension_ty) asdl_seq_GET(generators, 0)); /* Outermost iterator is evaluated in current scope */ @@ -1742,11 +1722,6 @@ symtable_handle_comprehension(struct symtable *st, expr_ty e, symtable_exit_block(st, (void *)e); return 0; } - /* Allocate temporary name if needed */ - if (needs_tmp && !symtable_new_tmpname(st)) { - symtable_exit_block(st, (void *)e); - return 0; - } VISIT(st, expr, outermost->target); VISIT_SEQ(st, expr, outermost->ifs); VISIT_SEQ_TAIL(st, comprehension, generators, 1); From webhook-mailer at python.org Sat Mar 10 18:25:17 2018 From: webhook-mailer at python.org (Miss Islington (bot)) Date: Sat, 10 Mar 2018 23:25:17 -0000 Subject: [Python-checkins] controlflow: Use full example for "5 through 9" (GH-5907) Message-ID: <mailman.29.1520724318.1871.python-checkins@python.org> https://github.com/python/cpython/commit/3ca5efcef53cdafa74f7acafd7bed7f878e7e584 commit: 3ca5efcef53cdafa74f7acafd7bed7f878e7e584 branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: GitHub <noreply at github.com> date: 2018-03-10T15:25:14-08:00 summary: controlflow: Use full example for "5 through 9" (GH-5907) Replace example result of "5 through 9" with complete list: "5, 6, 7, 8, 9". This format is more consistent with the surrounding examples. (cherry picked from commit 83d7062d2dc5eacfef578e072bca4747c346fdae) Co-authored-by: Steven M. Vascellaro <S.Vascellaro at gmail.com> files: M Doc/tutorial/controlflow.rst diff --git a/Doc/tutorial/controlflow.rst b/Doc/tutorial/controlflow.rst index 36b093950e10..30ef159f8b36 100644 --- a/Doc/tutorial/controlflow.rst +++ b/Doc/tutorial/controlflow.rst @@ -105,7 +105,7 @@ is possible to let the range start at another number, or to specify a different increment (even negative; sometimes this is called the 'step'):: range(5, 10) - 5 through 9 + 5, 6, 7, 8, 9 range(0, 10, 3) 0, 3, 6, 9 From webhook-mailer at python.org Sat Mar 10 19:38:53 2018 From: webhook-mailer at python.org (Berker Peksag) Date: Sun, 11 Mar 2018 00:38:53 -0000 Subject: [Python-checkins] bpo-27645: Skip test_bad_target_in_transaction if SQLite == 3.8.7.1 (GH-6067) Message-ID: <mailman.30.1520728734.1871.python-checkins@python.org> https://github.com/python/cpython/commit/7280a4eef5fbe17e8ac82afd71fe6e51843240a5 commit: 7280a4eef5fbe17e8ac82afd71fe6e51843240a5 branch: master author: Berker Peksag <berker.peksag at gmail.com> committer: GitHub <noreply at github.com> date: 2018-03-11T03:38:50+03:00 summary: bpo-27645: Skip test_bad_target_in_transaction if SQLite == 3.8.7.1 (GH-6067) files: M Lib/sqlite3/test/backup.py diff --git a/Lib/sqlite3/test/backup.py b/Lib/sqlite3/test/backup.py index 784702fb46e4..2a2148937413 100644 --- a/Lib/sqlite3/test/backup.py +++ b/Lib/sqlite3/test/backup.py @@ -37,6 +37,8 @@ def test_bad_target_closed_connection(self): self.cx.backup(bck) def test_bad_target_in_transaction(self): + if sqlite.sqlite_version_info == (3, 8, 7, 1): + self.skipTest('skip until we debug https://bugs.python.org/issue27645#msg313562') bck = sqlite.connect(':memory:') bck.execute('CREATE TABLE bar (key INTEGER)') bck.executemany('INSERT INTO bar (key) VALUES (?)', [(3,), (4,)]) From webhook-mailer at python.org Sat Mar 10 19:59:14 2018 From: webhook-mailer at python.org (Berker Peksag) Date: Sun, 11 Mar 2018 00:59:14 -0000 Subject: [Python-checkins] bpo-27645: Skip test_bad_target_in_transaction if SQLite == 3.8.7.1 (GH-6067) Message-ID: <mailman.31.1520729955.1871.python-checkins@python.org> https://github.com/python/cpython/commit/c546a62c4d7b37ead10f986554a01d6d593227a1 commit: c546a62c4d7b37ead10f986554a01d6d593227a1 branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Berker Peksag <berker.peksag at gmail.com> date: 2018-03-11T03:59:11+03:00 summary: bpo-27645: Skip test_bad_target_in_transaction if SQLite == 3.8.7.1 (GH-6067) (cherry picked from commit 7280a4eef5fbe17e8ac82afd71fe6e51843240a5) Co-authored-by: Berker Peksag <berker.peksag at gmail.com> files: M Lib/sqlite3/test/backup.py diff --git a/Lib/sqlite3/test/backup.py b/Lib/sqlite3/test/backup.py index 784702fb46e4..2a2148937413 100644 --- a/Lib/sqlite3/test/backup.py +++ b/Lib/sqlite3/test/backup.py @@ -37,6 +37,8 @@ def test_bad_target_closed_connection(self): self.cx.backup(bck) def test_bad_target_in_transaction(self): + if sqlite.sqlite_version_info == (3, 8, 7, 1): + self.skipTest('skip until we debug https://bugs.python.org/issue27645#msg313562') bck = sqlite.connect(':memory:') bck.execute('CREATE TABLE bar (key INTEGER)') bck.executemany('INSERT INTO bar (key) VALUES (?)', [(3,), (4,)]) From webhook-mailer at python.org Sat Mar 10 20:00:07 2018 From: webhook-mailer at python.org (Berker Peksag) Date: Sun, 11 Mar 2018 01:00:07 -0000 Subject: [Python-checkins] bpo-33045: Fix typos in SSL documentation (GH-6065) Message-ID: <mailman.32.1520730007.1871.python-checkins@python.org> https://github.com/python/cpython/commit/9cf8c42f3231d3f066670d087a997bd1278482a0 commit: 9cf8c42f3231d3f066670d087a997bd1278482a0 branch: master author: Matt Eaton <agnosticdev at gmail.com> committer: Berker Peksag <berker.peksag at gmail.com> date: 2018-03-11T04:00:04+03:00 summary: bpo-33045: Fix typos in SSL documentation (GH-6065) files: M Doc/library/ssl.rst diff --git a/Doc/library/ssl.rst b/Doc/library/ssl.rst index 2b4bed413985..8082a383d5ab 100644 --- a/Doc/library/ssl.rst +++ b/Doc/library/ssl.rst @@ -1688,7 +1688,7 @@ to speed up repeated connections from the same clients. .. method:: SSLContext.load_dh_params(dhfile) - Load the key generation parameters for Diffie-Helman (DH) key exchange. + Load the key generation parameters for Diffie-Hellman (DH) key exchange. Using DH key exchange improves forward secrecy at the expense of computational resources (both on the server and on the client). The *dhfile* parameter should be the path to a file containing DH @@ -2603,7 +2603,7 @@ with LibreSSL. Documentation of underlying :mod:`socket` class `SSL/TLS Strong Encryption: An Introduction <https://httpd.apache.org/docs/trunk/en/ssl/ssl_intro.html>`_ - Intro from the Apache webserver documentation + Intro from the Apache HTTP Server documentation `RFC 1422: Privacy Enhancement for Internet Electronic Mail: Part II: Certificate-Based Key Management <https://www.ietf.org/rfc/rfc1422>`_ Steve Kent From webhook-mailer at python.org Sat Mar 10 20:21:29 2018 From: webhook-mailer at python.org (Berker Peksag) Date: Sun, 11 Mar 2018 01:21:29 -0000 Subject: [Python-checkins] bpo-33045: Fix typos in SSL documentation (GH-6065) Message-ID: <mailman.33.1520731290.1871.python-checkins@python.org> https://github.com/python/cpython/commit/17b6c19d39229619de649f5a9e28a46779ef1c51 commit: 17b6c19d39229619de649f5a9e28a46779ef1c51 branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Berker Peksag <berker.peksag at gmail.com> date: 2018-03-11T04:21:27+03:00 summary: bpo-33045: Fix typos in SSL documentation (GH-6065) (cherry picked from commit 9cf8c42f3231d3f066670d087a997bd1278482a0) Co-authored-by: Matt Eaton <agnosticdev at gmail.com> files: M Doc/library/ssl.rst diff --git a/Doc/library/ssl.rst b/Doc/library/ssl.rst index 2b4bed413985..8082a383d5ab 100644 --- a/Doc/library/ssl.rst +++ b/Doc/library/ssl.rst @@ -1688,7 +1688,7 @@ to speed up repeated connections from the same clients. .. method:: SSLContext.load_dh_params(dhfile) - Load the key generation parameters for Diffie-Helman (DH) key exchange. + Load the key generation parameters for Diffie-Hellman (DH) key exchange. Using DH key exchange improves forward secrecy at the expense of computational resources (both on the server and on the client). The *dhfile* parameter should be the path to a file containing DH @@ -2603,7 +2603,7 @@ with LibreSSL. Documentation of underlying :mod:`socket` class `SSL/TLS Strong Encryption: An Introduction <https://httpd.apache.org/docs/trunk/en/ssl/ssl_intro.html>`_ - Intro from the Apache webserver documentation + Intro from the Apache HTTP Server documentation `RFC 1422: Privacy Enhancement for Internet Electronic Mail: Part II: Certificate-Based Key Management <https://www.ietf.org/rfc/rfc1422>`_ Steve Kent From webhook-mailer at python.org Sat Mar 10 20:26:04 2018 From: webhook-mailer at python.org (Berker Peksag) Date: Sun, 11 Mar 2018 01:26:04 -0000 Subject: [Python-checkins] bpo-33045: Fix typos in SSL documentation (GH-6065) Message-ID: <mailman.34.1520731565.1871.python-checkins@python.org> https://github.com/python/cpython/commit/3f439d14ad48f5d4f6017fc814326fb1f6237b53 commit: 3f439d14ad48f5d4f6017fc814326fb1f6237b53 branch: 3.6 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Berker Peksag <berker.peksag at gmail.com> date: 2018-03-11T04:26:01+03:00 summary: bpo-33045: Fix typos in SSL documentation (GH-6065) (cherry picked from commit 9cf8c42f3231d3f066670d087a997bd1278482a0) Co-authored-by: Matt Eaton <agnosticdev at gmail.com> files: M Doc/library/ssl.rst diff --git a/Doc/library/ssl.rst b/Doc/library/ssl.rst index fc68e3071436..677e94534702 100644 --- a/Doc/library/ssl.rst +++ b/Doc/library/ssl.rst @@ -1539,7 +1539,7 @@ to speed up repeated connections from the same clients. .. method:: SSLContext.load_dh_params(dhfile) - Load the key generation parameters for Diffie-Helman (DH) key exchange. + Load the key generation parameters for Diffie-Hellman (DH) key exchange. Using DH key exchange improves forward secrecy at the expense of computational resources (both on the server and on the client). The *dhfile* parameter should be the path to a file containing DH @@ -2343,7 +2343,7 @@ with LibreSSL. Documentation of underlying :mod:`socket` class `SSL/TLS Strong Encryption: An Introduction <https://httpd.apache.org/docs/trunk/en/ssl/ssl_intro.html>`_ - Intro from the Apache webserver documentation + Intro from the Apache HTTP Server documentation `RFC 1422: Privacy Enhancement for Internet Electronic Mail: Part II: Certificate-Based Key Management <https://www.ietf.org/rfc/rfc1422>`_ Steve Kent From webhook-mailer at python.org Sat Mar 10 20:48:17 2018 From: webhook-mailer at python.org (Serhiy Storchaka) Date: Sun, 11 Mar 2018 01:48:17 -0000 Subject: [Python-checkins] bpo-32996: Improve What's New in 3.7. (#5983) Message-ID: <mailman.35.1520732898.1871.python-checkins@python.org> https://github.com/python/cpython/commit/51302a5fcc557e6afc0bf1e3b371f5f37c76dc77 commit: 51302a5fcc557e6afc0bf1e3b371f5f37c76dc77 branch: master author: Serhiy Storchaka <storchaka at gmail.com> committer: GitHub <noreply at github.com> date: 2018-03-11T03:48:14+02:00 summary: bpo-32996: Improve What's New in 3.7. (#5983) files: M Doc/library/cgi.rst M Doc/whatsnew/3.7.rst diff --git a/Doc/library/cgi.rst b/Doc/library/cgi.rst index b60e1cc41e4b..17386b831833 100644 --- a/Doc/library/cgi.rst +++ b/Doc/library/cgi.rst @@ -289,11 +289,13 @@ algorithms implemented in this module in other circumstances. This function is deprecated in this module. Use :func:`urllib.parse.parse_qs` instead. It is maintained here only for backward compatibility. + .. function:: parse_qsl(qs, keep_blank_values=False, strict_parsing=False) This function is deprecated in this module. Use :func:`urllib.parse.parse_qsl` instead. It is maintained here only for backward compatibility. + .. function:: parse_multipart(fp, pdict, encoding="utf-8") Parse input of type :mimetype:`multipart/form-data` (for file uploads). @@ -309,6 +311,10 @@ algorithms implemented in this module in other circumstances. uploaded --- in that case, use the :class:`FieldStorage` class instead which is much more flexible. + .. versionchanged:: 3.7 + Added the *encoding* parameter. For non-file fields, the value is now + a list of strings, not bytes. + .. function:: parse_header(string) diff --git a/Doc/whatsnew/3.7.rst b/Doc/whatsnew/3.7.rst index fc5f5ab77095..a4ed481f67a5 100644 --- a/Doc/whatsnew/3.7.rst +++ b/Doc/whatsnew/3.7.rst @@ -91,17 +91,17 @@ rather than ASCII. The platform support definition in :pep:`11` has also been updated to limit full text handling support to suitably configured non-ASCII based locales. -As part of this change, the default error handler for ``stdin`` and ``stdout`` -is now ``surrogateescape`` (rather than ``strict``) when using any of the -defined coercion target locales (currently ``C.UTF-8``, ``C.utf8``, and -``UTF-8``). The default error handler for ``stderr`` continues to be -``backslashreplace``, regardless of locale. +As part of this change, the default error handler for :data:`~sys.stdin` and +:data:`~sys.stdout` is now ``surrogateescape`` (rather than ``strict``) when +using any of the defined coercion target locales (currently ``C.UTF-8``, +``C.utf8``, and ``UTF-8``). The default error handler for :data:`~sys.stderr` +continues to be ``backslashreplace``, regardless of locale. Locale coercion is silent by default, but to assist in debugging potentially locale related integration problems, explicit warnings (emitted directly on -``stderr`` can be requested by setting ``PYTHONCOERCECLOCALE=warn``. This -setting will also cause the Python runtime to emit a warning if the legacy C -locale remains active when the core interpreter is initialized. +:data:`~sys.stderr` can be requested by setting ``PYTHONCOERCECLOCALE=warn``. +This setting will also cause the Python runtime to emit a warning if the +legacy C locale remains active when the core interpreter is initialized. .. seealso:: @@ -114,9 +114,9 @@ locale remains active when the core interpreter is initialized. PEP 553: Built-in breakpoint() ------------------------------ -:pep:`553` describes a new built-in called ``breakpoint()`` which makes it +:pep:`553` describes a new built-in called :func:`breakpoint` which makes it easy and consistent to enter the Python debugger. Built-in ``breakpoint()`` -calls ``sys.breakpointhook()``. By default, this latter imports ``pdb`` and +calls :func:`sys.breakpointhook`. By default, this latter imports :mod:`pdb` and then calls ``pdb.set_trace()``, but by binding ``sys.breakpointhook()`` to the function of your choosing, ``breakpoint()`` can enter any debugger. Or, the environment variable :envvar:`PYTHONBREAKPOINT` can be set to the callable of @@ -165,13 +165,13 @@ PEP 562: Customization of access to module attributes ----------------------------------------------------- It is sometimes convenient to customize or otherwise have control over access -to module attributes. A typical example is managing deprecation warnings. -Typical workarounds are assigning ``__class__`` of a module object to -a custom subclass of :class:`types.ModuleType` or replacing the ``sys.modules`` -item with a custom wrapper instance. This procedure is now simplified by -recognizing ``__getattr__`` defined directly in a module that would act like -a normal ``__getattr__`` method, except that it will be defined on module -*instances*. +to module attributes. A typical example is managing deprecation warnings. +Typical workarounds are assigning :attr:`~instance.__class__` of a module +object to a custom subclass of :class:`types.ModuleType` or replacing the +:data:`sys.modules` item with a custom wrapper instance. This procedure is +now simplified by recognizing ``__getattr__`` defined directly in a module +that would act like a normal :meth:`__getattr__` method, except that it will +be defined on module *instances*. .. seealso:: @@ -217,7 +217,7 @@ following syntax valid:: ... Since this change breaks compatibility, the new behavior can be enabled -on a per-module basis in Python 3.7 using a ``__future__`` import, like +on a per-module basis in Python 3.7 using a :mod:`__future__` import, like this:: from __future__ import annotations @@ -263,7 +263,7 @@ PEP 565: Show DeprecationWarning in ``__main__`` The default handling of :exc:`DeprecationWarning` has been changed such that these warnings are once more shown by default, but only when the code -triggering them is running directly in the ``__main__`` module. As a result, +triggering them is running directly in the :mod:`__main__` module. As a result, developers of single file scripts and those using Python interactively should once again start seeing deprecation warnings for the APIs they use, but deprecation warnings triggered by imported application, library and framework @@ -275,7 +275,7 @@ between three different deprecation warning behaviours: * :exc:`FutureWarning`: always displayed by default, recommended for warnings intended to be seen by application end users (e.g. for deprecated application configuration settings). -* :exc:`DeprecationWarning`: displayed by default only in ``__main__`` and when +* :exc:`DeprecationWarning`: displayed by default only in :mod:`__main__` and when running tests, recommended for warnings intended to be seen by other Python developers where a version upgrade may result in changed behaviour or an error. @@ -316,11 +316,11 @@ environment variable are added to control the UTF-8 mode. PEP 557: Data Classes --------------------- -Adds a new module ``dataclasses``. It provides a class decorator -``dataclass`` which inspects the class's variable annotations (see +Adds a new module :mod:`dataclasses`. It provides a class decorator +:func:`~dataclasses.dataclass` which inspects the class's variable annotations (see :pep:`526`) and using them, adds methods such as ``__init__``, ``__repr__``, and ``__eq__`` to the class. It is similar to -``typing.NamedTuple``, but also works on classes with mutable +:class:`typing.NamedTuple`, but also works on classes with mutable instances, among other features. For example:: @@ -374,7 +374,7 @@ Hash-based pycs Python has traditionally checked the up-to-dateness of bytecode cache files (i.e., ``.pyc`` files) by comparing the source metadata (last-modified timestamp and size) with source metadata saved in the cache file header when it was -generated. While effective, this invalidation method has its drawbacks. When +generated. While effective, this invalidation method has its drawbacks. When filesystem timestamps are too coarse, Python can miss source updates, leading to user confusion. Additionally, having a timestamp in the cache file is problematic for `build reproduciblity <https://reproducible-builds.org/>`_ and @@ -407,7 +407,8 @@ Other Language Changes whitespace, not only spaces. (Contributed by Robert Xiao in :issue:`28927`.) * :exc:`ImportError` now displays module name and module ``__file__`` path when - ``from ... import ...`` fails. (Contributed by Matthias Bussonnier in :issue:`29546`.) + ``from ... import ...`` fails. (Contributed by Matthias Bussonnier in + :issue:`29546`.) * Circular imports involving absolute imports with binding a submodule to a name are now supported. @@ -453,7 +454,6 @@ the user intermix options and positional arguments on the command line, as is possible in many unix commands. It supports most but not all argparse features. (Contributed by paul.j3 in :issue:`14191`.) - binascii -------- @@ -461,7 +461,6 @@ The :func:`~binascii.b2a_uu` function now accepts an optional *backtick* keyword argument. When it's true, zeros are represented by ``'`'`` instead of spaces. (Contributed by Xiang Zhang in :issue:`30103`.) - calendar -------- @@ -469,14 +468,6 @@ The class :class:`~calendar.HTMLCalendar` has new class attributes which ease the customisation of the CSS classes in the produced HTML calendar. (Contributed by Oz Tiram in :issue:`30095`.) -cgi ---- - -:func:`~cgi.parse_multipart` returns the same results as -:class:`~FieldStorage` : for non-file fields, the value associated to a key -is a list of strings, not bytes. -(Contributed by Pierre Quentel in :issue:`29979`.) - contextlib ---------- @@ -490,8 +481,8 @@ Alexander Mohr and Ilya Kulakov in :issue:`29302`.) cProfile -------- -cProfile command line now accepts `-m module_name` as an alternative to -script path. (Contributed by Sanyam Khurana in :issue:`21862`.) +:mod:`cProfile` command line now accepts ``-m module_name`` as an alternative +to script path. (Contributed by Sanyam Khurana in :issue:`21862`.) crypt ----- @@ -505,10 +496,11 @@ for hashing. (Contributed by Serhiy Storchaka in :issue:`31702`.) datetime -------- -Added the :func:`datetime.datetime.fromisoformat` method, which constructs a -:class:`datetime.datetime` object from a string in one of the formats output -by :func:`datetime.datetime.isoformat`. (Contributed by Paul Ganssle in -:issue:`15873`.) +Added the :meth:`datetime.fromisoformat <datetime.datetime.fromisoformat>` +method, which constructs a :class:`~datetime.datetime` object from a string +in one of the formats output by +:meth:`datetime.isoformat <datetime.datetime.isoformat>`. +(Contributed by Paul Ganssle in :issue:`15873`.) dis --- @@ -521,7 +513,7 @@ classes). (Contributed by Serhiy Storchaka in :issue:`11822`.) distutils --------- -README.rst is now included in the list of distutils standard READMEs and +``README.rst`` is now included in the list of distutils standard READMEs and therefore included in source distributions. (Contributed by Ryan Gonzalez in :issue:`11913`.) @@ -537,28 +529,29 @@ equivalent to CR. http.client ----------- -Add Configurable *blocksize* to ``HTTPConnection`` and -``HTTPSConnection`` for improved upload throughput. +Add configurable *blocksize* to :class:`~http.client.HTTPConnection` and +:class:`~http.client.HTTPSConnection` for improved upload throughput. (Contributed by Nir Soffer in :issue:`31945`.) http.server ----------- :class:`~http.server.SimpleHTTPRequestHandler` supports the HTTP -If-Modified-Since header. The server returns the 304 response status if the +``If-Modified-Since`` header. The server returns the 304 response status if the target file was not modified after the time specified in the header. (Contributed by Pierre Quentel in :issue:`29654`.) -Add the parameter ``directory`` to the :class:`~http.server.SimpleHTTPRequestHandler` -and the ``--directory`` to the command line of the module :mod:`~http.server`. -With this parameter, the server serves the specified directory, by default it uses the current working directory. +Add the parameter *directory* to the :class:`~http.server.SimpleHTTPRequestHandler` +and the ``--directory`` to the command line of the module :mod:`http.server`. +With this parameter, the server serves the specified directory, by default it +uses the current working directory. (Contributed by St?phane Wirtel and Julien Palard in :issue:`28707`.) hmac ---- -The hmac module now has an optimized one-shot :func:`~hmac.digest` function, -which is up to three times faster than :func:`~hmac.HMAC`. +The :mod:`hmac` module now has an optimized one-shot :func:`~hmac.digest` +function, which is up to three times faster than :func:`~hmac.HMAC`. (Contributed by Christian Heimes in :issue:`32433`.) importlib @@ -570,11 +563,11 @@ support the loading of resource from packages. locale ------ -Added another argument *monetary* in :meth:`format_string` of :mod:`locale`. +Added another argument *monetary* in :func:`~locale.format_string` of :mod:`locale`. If *monetary* is true, the conversion uses monetary thousands separator and grouping strings. (Contributed by Garvit in :issue:`10379`.) -The :func:`locale.getpreferredencoding` function now always returns ``'UTF-8'`` +The :func:`~locale.getpreferredencoding` function now always returns ``'UTF-8'`` on Android or in the UTF-8 mode (:option:`-X` ``utf8`` option), the locale and the *do_setlocale* argument are ignored. @@ -593,7 +586,7 @@ Serhiy Storchaka in :issue:`28682`.) Added support for :ref:`file descriptors <path_fd>` in :func:`~os.scandir` on Unix. (Contributed by Serhiy Storchaka in :issue:`25996`.) -New function :func:`os.register_at_fork` allows registering Python callbacks +New function :func:`~os.register_at_fork` allows registering Python callbacks to be executed on a process fork. (Contributed by Antoine Pitrou in :issue:`16500`.) @@ -604,8 +597,8 @@ pdb argument. If given, this is printed to the console just before debugging begins. (Contributed by Barry Warsaw in :issue:`31389`.) -pdb command line now accepts `-m module_name` as an alternative to -script file. (Contributed by Mario Corchero in :issue:`32206`.) +:mod:`pdb` command line now accepts ``-m module_name`` as an alternative to +script file. (Contributed by Mario Corchero in :issue:`32206`.) py_compile ---------- @@ -618,7 +611,6 @@ This allows for guaranteeing files when they are created eagerly. (Contributed by Bernhard M. Wiedemann in :issue:`29708`.) - re -- @@ -642,7 +634,7 @@ method, if the underlying SQLite library is at version 3.6.11 or higher. ssl --- -The ssl module now uses OpenSSL's builtin API instead of +The :mod:`ssl` module now uses OpenSSL's builtin API instead of :func:`~ssl.match_hostname` to check host name or IP address. Values are validated during TLS handshake. Any cert validation error including a failing host name match now raises :exc:`~ssl.SSLCertVerificationError` and @@ -682,10 +674,6 @@ The ssl module has preliminary and experimental support for TLS 1.3 and OpenSSL 1.1.1. (Contributed by Christian Heimes in :issue:`32947`, :issue:`20995`, :issue:`29136`, and :issue:`30622`) -:func:`~ssl.wrap_socket` is deprecated. Documentation has been updated to -recommend :meth:`~ssl.SSLContext.wrap_socket` instead. -(Contributed by Christian Heimes in :issue:`28124`.) - :class:`~ssl.SSLSocket` and :class:`~ssl.SSLObject` no longer have a public constructor. Direct instantiation was never a documented and supported feature. Instances must be created with :class:`~ssl.SSLContext` methods @@ -708,27 +696,24 @@ separately. (Contributed by Barry Warsaw in :issue:`1198569`.) subprocess ---------- -On Windows the default for *close_fds* was changed from :const:`False` to -:const:`True` when redirecting the standard handles. It's now possible to set -*close_fds* to :const:`True` when redirecting the standard handles. See +On Windows the default for *close_fds* was changed from ``False`` to +``True`` when redirecting the standard handles. It's now possible to set +*close_fds* to ``True`` when redirecting the standard handles. See :class:`subprocess.Popen`. -This means that *close_fds* now defaults to :const:`True` on all supported -platforms. +This means that *close_fds* now defaults to ``True`` on all supported +platforms. (Contributed by Segev Finer in :issue:`19764`.) sys --- Added :attr:`sys.flags.dev_mode` flag for the new development mode. -Deprecated :func:`sys.set_coroutine_wrapper` and -:func:`sys.get_coroutine_wrapper`. - - tkinter ------- Added :class:`tkinter.ttk.Spinbox`. +(Contributed by Alan Moore in :issue:`32585`.) time ---- @@ -766,11 +751,12 @@ Peterson.) unittest -------- + Added new command-line option ``-k`` to filter tests to run with a substring or Unix shell-like pattern. For example, ``python -m unittest -k foo`` runs the tests ``foo_tests.SomeTest.test_something``, ``bar_tests.SomeTest.test_foo``, but not ``bar_tests.FooTest.test_something``. - +(Contributed by Jonas Haag in :issue:`32071`.) unittest.mock ------------- @@ -779,7 +765,7 @@ The :const:`~unittest.mock.sentinel` attributes now preserve their identity when they are :mod:`copied <copy>` or :mod:`pickled <pickle>`. (Contributed by Serhiy Storchaka in :issue:`20804`.) -New function :const:`~unittest.mock.seal` will disable the creation of mock +New function :func:`~unittest.mock.seal` will disable the creation of mock children by preventing to get or set any new attribute on the sealed mock. The sealing process is performed recursively. (Contributed by Mario Corchero in :issue:`30541`.) @@ -787,8 +773,8 @@ in :issue:`30541`.) urllib.parse ------------ -:func:`urllib.parse.quote` has been updated from RFC 2396 to RFC 3986, -adding `~` to the set of characters that is never quoted by default. +:func:`urllib.parse.quote` has been updated from :rfc:`2396` to :rfc:`3986`, +adding ``~`` to the set of characters that is never quoted by default. (Contributed by Christian Theune and Ratnadeep Debnath in :issue:`16285`.) uu @@ -832,26 +818,27 @@ better readability. (Contributed by Stefan Behnel in :issue:`31648`.) xmlrpc.server ------------- -:meth:`register_function` of :class:`xmlrpc.server.SimpleXMLRPCDispatcher` and -its subclasses can be used as a decorator. (Contributed by Xiang Zhang in +:meth:`register_function` of :class:`~xmlrpc.server.SimpleXMLRPCDispatcher` and +its subclasses can be used as a decorator. (Contributed by Xiang Zhang in :issue:`7769`.) zipapp ------ -Function :func:`zipapp.create_archive` now accepts an optional *filter* +Function :func:`~zipapp.create_archive` now accepts an optional *filter* argument to allow the user to select which files should be included in the -archive, and an optional *compressed* argument to generate a compressed -archive. +archive. (Contributed by Irmen de Jong in :issue:`31072`.) -A command line option ``--compress`` has also been added to support -compression. +Function :func:`~zipapp.create_archive` now accepts an optional *compressed* +argument to generate a compressed archive. A command line option +``--compress`` has also been added to support compression. +(Contributed by Zhiming Wang in :issue:`31638`.) Optimizations ============= -* Added two new opcodes: ``LOAD_METHOD`` and ``CALL_METHOD`` to avoid +* Added two new opcodes: :opcode:`LOAD_METHOD` and :opcode:`CALL_METHOD` to avoid instantiation of bound method objects for method calls, which results in method calls being faster up to 20%. (Contributed by Yury Selivanov and INADA Naoki in :issue:`26110`.) @@ -882,9 +869,10 @@ Optimizations Python 3.6 by about 10% depending on the pattern. (Contributed by INADA Naoki in :issue:`31671`.) -* :meth:`selectors.EpollSelector.modify`, :meth:`selectors.PollSelector.modify` - and :meth:`selectors.DevpollSelector.modify` may be around 10% faster under - heavy loads. (Contributed by Giampaolo Rodola' in :issue:`30014`) +* :meth:`~selectors.BaseSelector.modify` methods of classes + :class:`selectors.EpollSelector`, :class:`selectors.PollSelector` + and :class:`selectors.DevpollSelector` may be around 10% faster under + heavy loads. (Contributed by Giampaolo Rodola' in :issue:`30014`) * Constant folding is moved from peephole optimizer to new AST optimizer. (Contributed by Eugene Toder and INADA Naoki in :issue:`29469`) @@ -900,6 +888,7 @@ Optimizations constructors when not constructing subclasses. (Contributed by Paul Ganssle in :issue:`32403`) + Build and C API Changes ======================= @@ -955,6 +944,16 @@ Build and C API Changes and access to the UTC singleton with :c:data:`PyDateTime_TimeZone_UTC`. Contributed by Paul Ganssle in :issue:`10381`. +- The type of results of :c:func:`PyThread_start_new_thread` and + :c:func:`PyThread_get_thread_ident`, and the *id* parameter of + :c:func:`PyThreadState_SetAsyncExc` changed from :c:type:`long` to + :c:type:`unsigned long`. + (Contributed by Serhiy Storchaka in :issue:`6532`.) + +- :c:func:`PyUnicode_AsWideCharString` now raises a :exc:`ValueError` if the + second argument is *NULL* and the :c:type:`wchar_t*` string contains null + characters. (Contributed by Serhiy Storchaka in :issue:`30708`.) + Other CPython Implementation Changes ==================================== @@ -1007,13 +1006,13 @@ Deprecated - Methods :meth:`MetaPathFinder.find_module() <importlib.abc.MetaPathFinder.find_module>` (replaced by - :meth:`MetaPathFinder.find_spec() <importlib.abc.MetaPathFinder.find_spec>` - ) and + :meth:`MetaPathFinder.find_spec() <importlib.abc.MetaPathFinder.find_spec>`) + and :meth:`PathEntryFinder.find_loader() <importlib.abc.PathEntryFinder.find_loader>` (replaced by :meth:`PathEntryFinder.find_spec() <importlib.abc.PathEntryFinder.find_spec>`) - both deprecated in Python 3.4 now emit :exc:`DeprecationWarning`. (Contributed - by Matthias Bussonnier in :issue:`29576`) + both deprecated in Python 3.4 now emit :exc:`DeprecationWarning`. + (Contributed by Matthias Bussonnier in :issue:`29576`) - Using non-integer value for selecting a plural form in :mod:`gettext` is now deprecated. It never correctly worked. (Contributed by Serhiy Storchaka @@ -1024,23 +1023,17 @@ Deprecated - The :class:`importlib.abc.ResourceLoader` ABC has been deprecated in favour of :class:`importlib.abc.ResourceReader`. +- Deprecated :func:`sys.set_coroutine_wrapper` and + :func:`sys.get_coroutine_wrapper`. -Changes in the C API --------------------- - -- The type of results of :c:func:`PyThread_start_new_thread` and - :c:func:`PyThread_get_thread_ident`, and the *id* parameter of - :c:func:`PyThreadState_SetAsyncExc` changed from :c:type:`long` to - :c:type:`unsigned long`. - (Contributed by Serhiy Storchaka in :issue:`6532`.) - -- :c:func:`PyUnicode_AsWideCharString` now raises a :exc:`ValueError` if the - second argument is *NULL* and the :c:type:`wchar_t*` string contains null - characters. (Contributed by Serhiy Storchaka in :issue:`30708`.) +- :func:`ssl.wrap_socket` is deprecated. Use + :meth:`ssl.SSLContext.wrap_socket` instead. + (Contributed by Christian Heimes in :issue:`28124`.) Windows Only ------------ + - The python launcher, (py.exe), can accept 32 & 64 bit specifiers **without** having to specify a minor version as well. So ``py -3-32`` and ``py -3-64`` become valid as well as ``py -3.7-32``, also the -*m*-64 and -*m.n*-64 forms @@ -1054,6 +1047,7 @@ Windows Only the *short form* list of available specifiers. (Contributed by Steve Barnes in :issue:`30362`.) + Removed ======= @@ -1096,6 +1090,17 @@ API and Feature Removals :func:`~plistlib.readPlistFromBytes` are now normal dicts. You no longer can use attribute access to access items of these dictionaries. +* The ``asyncio.windows_utils.socketpair()`` function has been + removed: use directly :func:`socket.socketpair` which is available on all + platforms since Python 3.5 (before, it wasn't available on Windows). + ``asyncio.windows_utils.socketpair`` was just an alias to + ``socket.socketpair`` on Python 3.5 and newer. + +* :mod:`asyncio`: The module doesn't export :mod:`selectors` and + :mod:`_overlapped` modules as ``asyncio.selectors`` and + ``asyncio._overlapped``. Replace ``from asyncio import selectors`` with + ``import selectors`` for example. + Porting to Python 3.7 ===================== @@ -1132,29 +1137,20 @@ Changes in the Python API ------------------------- * :meth:`socketserver.ThreadingMixIn.server_close` now waits until all - non-daemon threads complete. Use daemonic threads by setting - :data:`ThreadingMixIn.daemon_threads` to ``True`` to not wait until threads - complete. (Contributed by Victor Stinner in :issue:`31233`.) + non-daemon threads complete. Use daemonic threads by setting + :data:`socketserver.ThreadingMixIn.daemon_threads` to ``True`` to not + wait until threads complete. + (Contributed by Victor Stinner in :issue:`31233`.) * :meth:`socketserver.ForkingMixIn.server_close` now waits until all child processes complete. (Contributed by Victor Stinner in :issue:`31151`.) * The :func:`locale.localeconv` function now sets temporarily the ``LC_CTYPE`` locale to the ``LC_NUMERIC`` locale in some cases. + (Contributed by Victor Stinner in :issue:`31900`.) -* The ``asyncio.windows_utils.socketpair()`` function has been - removed: use directly :func:`socket.socketpair` which is available on all - platforms since Python 3.5 (before, it wasn't available on Windows). - ``asyncio.windows_utils.socketpair()`` was just an alias to - ``socket.socketpair`` on Python 3.5 and newer. - -* :mod:`asyncio`: The module doesn't export :mod:`selectors` and - :mod:`_overlapped` modules as ``asyncio.selectors`` and - ``asyncio._overlapped``. Replace ``from asyncio import selectors`` with - ``import selectors`` for example. - -* :meth:`pkgutil.walk_packages` now raises ValueError if *path* is a string. - Previously an empty list was returned. (Contributed by Sanyam Khurana in +* :meth:`pkgutil.walk_packages` now raises :exc:`ValueError` if *path* is a string. + Previously an empty list was returned. (Contributed by Sanyam Khurana in :issue:`24744`.) * A format string argument for :meth:`string.Formatter.format` @@ -1186,6 +1182,11 @@ Changes in the Python API * The :attr:`struct.Struct.format` type is now :class:`str` instead of :class:`bytes`. (Contributed by Victor Stinner in :issue:`21071`.) +* :func:`~cgi.parse_multipart` returns now the same results as + :class:`~FieldStorage`: for non-file fields, the value associated to a key + is a list of strings, not bytes. + (Contributed by Pierre Quentel in :issue:`29979`.) + * Due to internal changes in :mod:`socket` you won't be able to :func:`socket.fromshare` a socket :func:`~socket.socket.share`-ed in older Python versions. @@ -1207,6 +1208,8 @@ Changes in the Python API avoid a warning escape them with a backslash. (Contributed by Serhiy Storchaka in :issue:`30349`.) + .. _Unicode Technical Standard #18: https://unicode.org/reports/tr18/ + * The result of splitting a string on a :mod:`regular expression <re>` that could match an empty string has been changed. For example splitting on ``r'\s*'`` will now split not only on whitespaces as it @@ -1246,8 +1249,6 @@ Changes in the Python API work as expected on all platforms. (Contributed by Yury Selivanov in :issue:`32331`.) -.. _Unicode Technical Standard #18: https://unicode.org/reports/tr18/ - * On Windows the default for the *close_fds* argument of :class:`subprocess.Popen` was changed from :const:`False` to :const:`True` when redirecting the standard handles. If you previously depended on handles @@ -1276,7 +1277,8 @@ CPython bytecode changes * Added two new opcodes: :opcode:`LOAD_METHOD` and :opcode:`CALL_METHOD`. (Contributed by Yury Selivanov and INADA Naoki in :issue:`26110`.) -* Removed the STORE_ANNOTATION opcode. +* Removed the :opcode:`STORE_ANNOTATION` opcode. + (Contributed by Mark Shannon in :issue:`32550`.) Other CPython implementation changes @@ -1293,9 +1295,9 @@ Other CPython implementation changes (Contributed by Nick Coghlan and Eric Snow as part of :issue:`22257`.) * Due to changes in the way the default warnings filters are configured, - setting ``Py_BytesWarningFlag`` to a value greater than one is no longer - sufficient to both emit ``BytesWarning`` messages and have them converted - to exceptions. Instead, the flag must be set (to cause the warnings to be + setting :c:data:`Py_BytesWarningFlag` to a value greater than one is no longer + sufficient to both emit :exc:`BytesWarning` messages and have them converted + to exceptions. Instead, the flag must be set (to cause the warnings to be emitted in the first place), and an explicit ``error::BytesWarning`` warnings filter added to convert them to exceptions. From webhook-mailer at python.org Sun Mar 11 01:29:36 2018 From: webhook-mailer at python.org (Miss Islington (bot)) Date: Sun, 11 Mar 2018 06:29:36 -0000 Subject: [Python-checkins] bpo-32996: Improve What's New in 3.7. (GH-5983) Message-ID: <mailman.36.1520749778.1871.python-checkins@python.org> https://github.com/python/cpython/commit/720a4b69c6b66ae5d6f447d5814bd204170c94b3 commit: 720a4b69c6b66ae5d6f447d5814bd204170c94b3 branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: GitHub <noreply at github.com> date: 2018-03-10T22:29:30-08:00 summary: bpo-32996: Improve What's New in 3.7. (GH-5983) (cherry picked from commit 51302a5fcc557e6afc0bf1e3b371f5f37c76dc77) Co-authored-by: Serhiy Storchaka <storchaka at gmail.com> files: M Doc/library/cgi.rst M Doc/whatsnew/3.7.rst diff --git a/Doc/library/cgi.rst b/Doc/library/cgi.rst index b60e1cc41e4b..17386b831833 100644 --- a/Doc/library/cgi.rst +++ b/Doc/library/cgi.rst @@ -289,11 +289,13 @@ algorithms implemented in this module in other circumstances. This function is deprecated in this module. Use :func:`urllib.parse.parse_qs` instead. It is maintained here only for backward compatibility. + .. function:: parse_qsl(qs, keep_blank_values=False, strict_parsing=False) This function is deprecated in this module. Use :func:`urllib.parse.parse_qsl` instead. It is maintained here only for backward compatibility. + .. function:: parse_multipart(fp, pdict, encoding="utf-8") Parse input of type :mimetype:`multipart/form-data` (for file uploads). @@ -309,6 +311,10 @@ algorithms implemented in this module in other circumstances. uploaded --- in that case, use the :class:`FieldStorage` class instead which is much more flexible. + .. versionchanged:: 3.7 + Added the *encoding* parameter. For non-file fields, the value is now + a list of strings, not bytes. + .. function:: parse_header(string) diff --git a/Doc/whatsnew/3.7.rst b/Doc/whatsnew/3.7.rst index fc5f5ab77095..a4ed481f67a5 100644 --- a/Doc/whatsnew/3.7.rst +++ b/Doc/whatsnew/3.7.rst @@ -91,17 +91,17 @@ rather than ASCII. The platform support definition in :pep:`11` has also been updated to limit full text handling support to suitably configured non-ASCII based locales. -As part of this change, the default error handler for ``stdin`` and ``stdout`` -is now ``surrogateescape`` (rather than ``strict``) when using any of the -defined coercion target locales (currently ``C.UTF-8``, ``C.utf8``, and -``UTF-8``). The default error handler for ``stderr`` continues to be -``backslashreplace``, regardless of locale. +As part of this change, the default error handler for :data:`~sys.stdin` and +:data:`~sys.stdout` is now ``surrogateescape`` (rather than ``strict``) when +using any of the defined coercion target locales (currently ``C.UTF-8``, +``C.utf8``, and ``UTF-8``). The default error handler for :data:`~sys.stderr` +continues to be ``backslashreplace``, regardless of locale. Locale coercion is silent by default, but to assist in debugging potentially locale related integration problems, explicit warnings (emitted directly on -``stderr`` can be requested by setting ``PYTHONCOERCECLOCALE=warn``. This -setting will also cause the Python runtime to emit a warning if the legacy C -locale remains active when the core interpreter is initialized. +:data:`~sys.stderr` can be requested by setting ``PYTHONCOERCECLOCALE=warn``. +This setting will also cause the Python runtime to emit a warning if the +legacy C locale remains active when the core interpreter is initialized. .. seealso:: @@ -114,9 +114,9 @@ locale remains active when the core interpreter is initialized. PEP 553: Built-in breakpoint() ------------------------------ -:pep:`553` describes a new built-in called ``breakpoint()`` which makes it +:pep:`553` describes a new built-in called :func:`breakpoint` which makes it easy and consistent to enter the Python debugger. Built-in ``breakpoint()`` -calls ``sys.breakpointhook()``. By default, this latter imports ``pdb`` and +calls :func:`sys.breakpointhook`. By default, this latter imports :mod:`pdb` and then calls ``pdb.set_trace()``, but by binding ``sys.breakpointhook()`` to the function of your choosing, ``breakpoint()`` can enter any debugger. Or, the environment variable :envvar:`PYTHONBREAKPOINT` can be set to the callable of @@ -165,13 +165,13 @@ PEP 562: Customization of access to module attributes ----------------------------------------------------- It is sometimes convenient to customize or otherwise have control over access -to module attributes. A typical example is managing deprecation warnings. -Typical workarounds are assigning ``__class__`` of a module object to -a custom subclass of :class:`types.ModuleType` or replacing the ``sys.modules`` -item with a custom wrapper instance. This procedure is now simplified by -recognizing ``__getattr__`` defined directly in a module that would act like -a normal ``__getattr__`` method, except that it will be defined on module -*instances*. +to module attributes. A typical example is managing deprecation warnings. +Typical workarounds are assigning :attr:`~instance.__class__` of a module +object to a custom subclass of :class:`types.ModuleType` or replacing the +:data:`sys.modules` item with a custom wrapper instance. This procedure is +now simplified by recognizing ``__getattr__`` defined directly in a module +that would act like a normal :meth:`__getattr__` method, except that it will +be defined on module *instances*. .. seealso:: @@ -217,7 +217,7 @@ following syntax valid:: ... Since this change breaks compatibility, the new behavior can be enabled -on a per-module basis in Python 3.7 using a ``__future__`` import, like +on a per-module basis in Python 3.7 using a :mod:`__future__` import, like this:: from __future__ import annotations @@ -263,7 +263,7 @@ PEP 565: Show DeprecationWarning in ``__main__`` The default handling of :exc:`DeprecationWarning` has been changed such that these warnings are once more shown by default, but only when the code -triggering them is running directly in the ``__main__`` module. As a result, +triggering them is running directly in the :mod:`__main__` module. As a result, developers of single file scripts and those using Python interactively should once again start seeing deprecation warnings for the APIs they use, but deprecation warnings triggered by imported application, library and framework @@ -275,7 +275,7 @@ between three different deprecation warning behaviours: * :exc:`FutureWarning`: always displayed by default, recommended for warnings intended to be seen by application end users (e.g. for deprecated application configuration settings). -* :exc:`DeprecationWarning`: displayed by default only in ``__main__`` and when +* :exc:`DeprecationWarning`: displayed by default only in :mod:`__main__` and when running tests, recommended for warnings intended to be seen by other Python developers where a version upgrade may result in changed behaviour or an error. @@ -316,11 +316,11 @@ environment variable are added to control the UTF-8 mode. PEP 557: Data Classes --------------------- -Adds a new module ``dataclasses``. It provides a class decorator -``dataclass`` which inspects the class's variable annotations (see +Adds a new module :mod:`dataclasses`. It provides a class decorator +:func:`~dataclasses.dataclass` which inspects the class's variable annotations (see :pep:`526`) and using them, adds methods such as ``__init__``, ``__repr__``, and ``__eq__`` to the class. It is similar to -``typing.NamedTuple``, but also works on classes with mutable +:class:`typing.NamedTuple`, but also works on classes with mutable instances, among other features. For example:: @@ -374,7 +374,7 @@ Hash-based pycs Python has traditionally checked the up-to-dateness of bytecode cache files (i.e., ``.pyc`` files) by comparing the source metadata (last-modified timestamp and size) with source metadata saved in the cache file header when it was -generated. While effective, this invalidation method has its drawbacks. When +generated. While effective, this invalidation method has its drawbacks. When filesystem timestamps are too coarse, Python can miss source updates, leading to user confusion. Additionally, having a timestamp in the cache file is problematic for `build reproduciblity <https://reproducible-builds.org/>`_ and @@ -407,7 +407,8 @@ Other Language Changes whitespace, not only spaces. (Contributed by Robert Xiao in :issue:`28927`.) * :exc:`ImportError` now displays module name and module ``__file__`` path when - ``from ... import ...`` fails. (Contributed by Matthias Bussonnier in :issue:`29546`.) + ``from ... import ...`` fails. (Contributed by Matthias Bussonnier in + :issue:`29546`.) * Circular imports involving absolute imports with binding a submodule to a name are now supported. @@ -453,7 +454,6 @@ the user intermix options and positional arguments on the command line, as is possible in many unix commands. It supports most but not all argparse features. (Contributed by paul.j3 in :issue:`14191`.) - binascii -------- @@ -461,7 +461,6 @@ The :func:`~binascii.b2a_uu` function now accepts an optional *backtick* keyword argument. When it's true, zeros are represented by ``'`'`` instead of spaces. (Contributed by Xiang Zhang in :issue:`30103`.) - calendar -------- @@ -469,14 +468,6 @@ The class :class:`~calendar.HTMLCalendar` has new class attributes which ease the customisation of the CSS classes in the produced HTML calendar. (Contributed by Oz Tiram in :issue:`30095`.) -cgi ---- - -:func:`~cgi.parse_multipart` returns the same results as -:class:`~FieldStorage` : for non-file fields, the value associated to a key -is a list of strings, not bytes. -(Contributed by Pierre Quentel in :issue:`29979`.) - contextlib ---------- @@ -490,8 +481,8 @@ Alexander Mohr and Ilya Kulakov in :issue:`29302`.) cProfile -------- -cProfile command line now accepts `-m module_name` as an alternative to -script path. (Contributed by Sanyam Khurana in :issue:`21862`.) +:mod:`cProfile` command line now accepts ``-m module_name`` as an alternative +to script path. (Contributed by Sanyam Khurana in :issue:`21862`.) crypt ----- @@ -505,10 +496,11 @@ for hashing. (Contributed by Serhiy Storchaka in :issue:`31702`.) datetime -------- -Added the :func:`datetime.datetime.fromisoformat` method, which constructs a -:class:`datetime.datetime` object from a string in one of the formats output -by :func:`datetime.datetime.isoformat`. (Contributed by Paul Ganssle in -:issue:`15873`.) +Added the :meth:`datetime.fromisoformat <datetime.datetime.fromisoformat>` +method, which constructs a :class:`~datetime.datetime` object from a string +in one of the formats output by +:meth:`datetime.isoformat <datetime.datetime.isoformat>`. +(Contributed by Paul Ganssle in :issue:`15873`.) dis --- @@ -521,7 +513,7 @@ classes). (Contributed by Serhiy Storchaka in :issue:`11822`.) distutils --------- -README.rst is now included in the list of distutils standard READMEs and +``README.rst`` is now included in the list of distutils standard READMEs and therefore included in source distributions. (Contributed by Ryan Gonzalez in :issue:`11913`.) @@ -537,28 +529,29 @@ equivalent to CR. http.client ----------- -Add Configurable *blocksize* to ``HTTPConnection`` and -``HTTPSConnection`` for improved upload throughput. +Add configurable *blocksize* to :class:`~http.client.HTTPConnection` and +:class:`~http.client.HTTPSConnection` for improved upload throughput. (Contributed by Nir Soffer in :issue:`31945`.) http.server ----------- :class:`~http.server.SimpleHTTPRequestHandler` supports the HTTP -If-Modified-Since header. The server returns the 304 response status if the +``If-Modified-Since`` header. The server returns the 304 response status if the target file was not modified after the time specified in the header. (Contributed by Pierre Quentel in :issue:`29654`.) -Add the parameter ``directory`` to the :class:`~http.server.SimpleHTTPRequestHandler` -and the ``--directory`` to the command line of the module :mod:`~http.server`. -With this parameter, the server serves the specified directory, by default it uses the current working directory. +Add the parameter *directory* to the :class:`~http.server.SimpleHTTPRequestHandler` +and the ``--directory`` to the command line of the module :mod:`http.server`. +With this parameter, the server serves the specified directory, by default it +uses the current working directory. (Contributed by St?phane Wirtel and Julien Palard in :issue:`28707`.) hmac ---- -The hmac module now has an optimized one-shot :func:`~hmac.digest` function, -which is up to three times faster than :func:`~hmac.HMAC`. +The :mod:`hmac` module now has an optimized one-shot :func:`~hmac.digest` +function, which is up to three times faster than :func:`~hmac.HMAC`. (Contributed by Christian Heimes in :issue:`32433`.) importlib @@ -570,11 +563,11 @@ support the loading of resource from packages. locale ------ -Added another argument *monetary* in :meth:`format_string` of :mod:`locale`. +Added another argument *monetary* in :func:`~locale.format_string` of :mod:`locale`. If *monetary* is true, the conversion uses monetary thousands separator and grouping strings. (Contributed by Garvit in :issue:`10379`.) -The :func:`locale.getpreferredencoding` function now always returns ``'UTF-8'`` +The :func:`~locale.getpreferredencoding` function now always returns ``'UTF-8'`` on Android or in the UTF-8 mode (:option:`-X` ``utf8`` option), the locale and the *do_setlocale* argument are ignored. @@ -593,7 +586,7 @@ Serhiy Storchaka in :issue:`28682`.) Added support for :ref:`file descriptors <path_fd>` in :func:`~os.scandir` on Unix. (Contributed by Serhiy Storchaka in :issue:`25996`.) -New function :func:`os.register_at_fork` allows registering Python callbacks +New function :func:`~os.register_at_fork` allows registering Python callbacks to be executed on a process fork. (Contributed by Antoine Pitrou in :issue:`16500`.) @@ -604,8 +597,8 @@ pdb argument. If given, this is printed to the console just before debugging begins. (Contributed by Barry Warsaw in :issue:`31389`.) -pdb command line now accepts `-m module_name` as an alternative to -script file. (Contributed by Mario Corchero in :issue:`32206`.) +:mod:`pdb` command line now accepts ``-m module_name`` as an alternative to +script file. (Contributed by Mario Corchero in :issue:`32206`.) py_compile ---------- @@ -618,7 +611,6 @@ This allows for guaranteeing files when they are created eagerly. (Contributed by Bernhard M. Wiedemann in :issue:`29708`.) - re -- @@ -642,7 +634,7 @@ method, if the underlying SQLite library is at version 3.6.11 or higher. ssl --- -The ssl module now uses OpenSSL's builtin API instead of +The :mod:`ssl` module now uses OpenSSL's builtin API instead of :func:`~ssl.match_hostname` to check host name or IP address. Values are validated during TLS handshake. Any cert validation error including a failing host name match now raises :exc:`~ssl.SSLCertVerificationError` and @@ -682,10 +674,6 @@ The ssl module has preliminary and experimental support for TLS 1.3 and OpenSSL 1.1.1. (Contributed by Christian Heimes in :issue:`32947`, :issue:`20995`, :issue:`29136`, and :issue:`30622`) -:func:`~ssl.wrap_socket` is deprecated. Documentation has been updated to -recommend :meth:`~ssl.SSLContext.wrap_socket` instead. -(Contributed by Christian Heimes in :issue:`28124`.) - :class:`~ssl.SSLSocket` and :class:`~ssl.SSLObject` no longer have a public constructor. Direct instantiation was never a documented and supported feature. Instances must be created with :class:`~ssl.SSLContext` methods @@ -708,27 +696,24 @@ separately. (Contributed by Barry Warsaw in :issue:`1198569`.) subprocess ---------- -On Windows the default for *close_fds* was changed from :const:`False` to -:const:`True` when redirecting the standard handles. It's now possible to set -*close_fds* to :const:`True` when redirecting the standard handles. See +On Windows the default for *close_fds* was changed from ``False`` to +``True`` when redirecting the standard handles. It's now possible to set +*close_fds* to ``True`` when redirecting the standard handles. See :class:`subprocess.Popen`. -This means that *close_fds* now defaults to :const:`True` on all supported -platforms. +This means that *close_fds* now defaults to ``True`` on all supported +platforms. (Contributed by Segev Finer in :issue:`19764`.) sys --- Added :attr:`sys.flags.dev_mode` flag for the new development mode. -Deprecated :func:`sys.set_coroutine_wrapper` and -:func:`sys.get_coroutine_wrapper`. - - tkinter ------- Added :class:`tkinter.ttk.Spinbox`. +(Contributed by Alan Moore in :issue:`32585`.) time ---- @@ -766,11 +751,12 @@ Peterson.) unittest -------- + Added new command-line option ``-k`` to filter tests to run with a substring or Unix shell-like pattern. For example, ``python -m unittest -k foo`` runs the tests ``foo_tests.SomeTest.test_something``, ``bar_tests.SomeTest.test_foo``, but not ``bar_tests.FooTest.test_something``. - +(Contributed by Jonas Haag in :issue:`32071`.) unittest.mock ------------- @@ -779,7 +765,7 @@ The :const:`~unittest.mock.sentinel` attributes now preserve their identity when they are :mod:`copied <copy>` or :mod:`pickled <pickle>`. (Contributed by Serhiy Storchaka in :issue:`20804`.) -New function :const:`~unittest.mock.seal` will disable the creation of mock +New function :func:`~unittest.mock.seal` will disable the creation of mock children by preventing to get or set any new attribute on the sealed mock. The sealing process is performed recursively. (Contributed by Mario Corchero in :issue:`30541`.) @@ -787,8 +773,8 @@ in :issue:`30541`.) urllib.parse ------------ -:func:`urllib.parse.quote` has been updated from RFC 2396 to RFC 3986, -adding `~` to the set of characters that is never quoted by default. +:func:`urllib.parse.quote` has been updated from :rfc:`2396` to :rfc:`3986`, +adding ``~`` to the set of characters that is never quoted by default. (Contributed by Christian Theune and Ratnadeep Debnath in :issue:`16285`.) uu @@ -832,26 +818,27 @@ better readability. (Contributed by Stefan Behnel in :issue:`31648`.) xmlrpc.server ------------- -:meth:`register_function` of :class:`xmlrpc.server.SimpleXMLRPCDispatcher` and -its subclasses can be used as a decorator. (Contributed by Xiang Zhang in +:meth:`register_function` of :class:`~xmlrpc.server.SimpleXMLRPCDispatcher` and +its subclasses can be used as a decorator. (Contributed by Xiang Zhang in :issue:`7769`.) zipapp ------ -Function :func:`zipapp.create_archive` now accepts an optional *filter* +Function :func:`~zipapp.create_archive` now accepts an optional *filter* argument to allow the user to select which files should be included in the -archive, and an optional *compressed* argument to generate a compressed -archive. +archive. (Contributed by Irmen de Jong in :issue:`31072`.) -A command line option ``--compress`` has also been added to support -compression. +Function :func:`~zipapp.create_archive` now accepts an optional *compressed* +argument to generate a compressed archive. A command line option +``--compress`` has also been added to support compression. +(Contributed by Zhiming Wang in :issue:`31638`.) Optimizations ============= -* Added two new opcodes: ``LOAD_METHOD`` and ``CALL_METHOD`` to avoid +* Added two new opcodes: :opcode:`LOAD_METHOD` and :opcode:`CALL_METHOD` to avoid instantiation of bound method objects for method calls, which results in method calls being faster up to 20%. (Contributed by Yury Selivanov and INADA Naoki in :issue:`26110`.) @@ -882,9 +869,10 @@ Optimizations Python 3.6 by about 10% depending on the pattern. (Contributed by INADA Naoki in :issue:`31671`.) -* :meth:`selectors.EpollSelector.modify`, :meth:`selectors.PollSelector.modify` - and :meth:`selectors.DevpollSelector.modify` may be around 10% faster under - heavy loads. (Contributed by Giampaolo Rodola' in :issue:`30014`) +* :meth:`~selectors.BaseSelector.modify` methods of classes + :class:`selectors.EpollSelector`, :class:`selectors.PollSelector` + and :class:`selectors.DevpollSelector` may be around 10% faster under + heavy loads. (Contributed by Giampaolo Rodola' in :issue:`30014`) * Constant folding is moved from peephole optimizer to new AST optimizer. (Contributed by Eugene Toder and INADA Naoki in :issue:`29469`) @@ -900,6 +888,7 @@ Optimizations constructors when not constructing subclasses. (Contributed by Paul Ganssle in :issue:`32403`) + Build and C API Changes ======================= @@ -955,6 +944,16 @@ Build and C API Changes and access to the UTC singleton with :c:data:`PyDateTime_TimeZone_UTC`. Contributed by Paul Ganssle in :issue:`10381`. +- The type of results of :c:func:`PyThread_start_new_thread` and + :c:func:`PyThread_get_thread_ident`, and the *id* parameter of + :c:func:`PyThreadState_SetAsyncExc` changed from :c:type:`long` to + :c:type:`unsigned long`. + (Contributed by Serhiy Storchaka in :issue:`6532`.) + +- :c:func:`PyUnicode_AsWideCharString` now raises a :exc:`ValueError` if the + second argument is *NULL* and the :c:type:`wchar_t*` string contains null + characters. (Contributed by Serhiy Storchaka in :issue:`30708`.) + Other CPython Implementation Changes ==================================== @@ -1007,13 +1006,13 @@ Deprecated - Methods :meth:`MetaPathFinder.find_module() <importlib.abc.MetaPathFinder.find_module>` (replaced by - :meth:`MetaPathFinder.find_spec() <importlib.abc.MetaPathFinder.find_spec>` - ) and + :meth:`MetaPathFinder.find_spec() <importlib.abc.MetaPathFinder.find_spec>`) + and :meth:`PathEntryFinder.find_loader() <importlib.abc.PathEntryFinder.find_loader>` (replaced by :meth:`PathEntryFinder.find_spec() <importlib.abc.PathEntryFinder.find_spec>`) - both deprecated in Python 3.4 now emit :exc:`DeprecationWarning`. (Contributed - by Matthias Bussonnier in :issue:`29576`) + both deprecated in Python 3.4 now emit :exc:`DeprecationWarning`. + (Contributed by Matthias Bussonnier in :issue:`29576`) - Using non-integer value for selecting a plural form in :mod:`gettext` is now deprecated. It never correctly worked. (Contributed by Serhiy Storchaka @@ -1024,23 +1023,17 @@ Deprecated - The :class:`importlib.abc.ResourceLoader` ABC has been deprecated in favour of :class:`importlib.abc.ResourceReader`. +- Deprecated :func:`sys.set_coroutine_wrapper` and + :func:`sys.get_coroutine_wrapper`. -Changes in the C API --------------------- - -- The type of results of :c:func:`PyThread_start_new_thread` and - :c:func:`PyThread_get_thread_ident`, and the *id* parameter of - :c:func:`PyThreadState_SetAsyncExc` changed from :c:type:`long` to - :c:type:`unsigned long`. - (Contributed by Serhiy Storchaka in :issue:`6532`.) - -- :c:func:`PyUnicode_AsWideCharString` now raises a :exc:`ValueError` if the - second argument is *NULL* and the :c:type:`wchar_t*` string contains null - characters. (Contributed by Serhiy Storchaka in :issue:`30708`.) +- :func:`ssl.wrap_socket` is deprecated. Use + :meth:`ssl.SSLContext.wrap_socket` instead. + (Contributed by Christian Heimes in :issue:`28124`.) Windows Only ------------ + - The python launcher, (py.exe), can accept 32 & 64 bit specifiers **without** having to specify a minor version as well. So ``py -3-32`` and ``py -3-64`` become valid as well as ``py -3.7-32``, also the -*m*-64 and -*m.n*-64 forms @@ -1054,6 +1047,7 @@ Windows Only the *short form* list of available specifiers. (Contributed by Steve Barnes in :issue:`30362`.) + Removed ======= @@ -1096,6 +1090,17 @@ API and Feature Removals :func:`~plistlib.readPlistFromBytes` are now normal dicts. You no longer can use attribute access to access items of these dictionaries. +* The ``asyncio.windows_utils.socketpair()`` function has been + removed: use directly :func:`socket.socketpair` which is available on all + platforms since Python 3.5 (before, it wasn't available on Windows). + ``asyncio.windows_utils.socketpair`` was just an alias to + ``socket.socketpair`` on Python 3.5 and newer. + +* :mod:`asyncio`: The module doesn't export :mod:`selectors` and + :mod:`_overlapped` modules as ``asyncio.selectors`` and + ``asyncio._overlapped``. Replace ``from asyncio import selectors`` with + ``import selectors`` for example. + Porting to Python 3.7 ===================== @@ -1132,29 +1137,20 @@ Changes in the Python API ------------------------- * :meth:`socketserver.ThreadingMixIn.server_close` now waits until all - non-daemon threads complete. Use daemonic threads by setting - :data:`ThreadingMixIn.daemon_threads` to ``True`` to not wait until threads - complete. (Contributed by Victor Stinner in :issue:`31233`.) + non-daemon threads complete. Use daemonic threads by setting + :data:`socketserver.ThreadingMixIn.daemon_threads` to ``True`` to not + wait until threads complete. + (Contributed by Victor Stinner in :issue:`31233`.) * :meth:`socketserver.ForkingMixIn.server_close` now waits until all child processes complete. (Contributed by Victor Stinner in :issue:`31151`.) * The :func:`locale.localeconv` function now sets temporarily the ``LC_CTYPE`` locale to the ``LC_NUMERIC`` locale in some cases. + (Contributed by Victor Stinner in :issue:`31900`.) -* The ``asyncio.windows_utils.socketpair()`` function has been - removed: use directly :func:`socket.socketpair` which is available on all - platforms since Python 3.5 (before, it wasn't available on Windows). - ``asyncio.windows_utils.socketpair()`` was just an alias to - ``socket.socketpair`` on Python 3.5 and newer. - -* :mod:`asyncio`: The module doesn't export :mod:`selectors` and - :mod:`_overlapped` modules as ``asyncio.selectors`` and - ``asyncio._overlapped``. Replace ``from asyncio import selectors`` with - ``import selectors`` for example. - -* :meth:`pkgutil.walk_packages` now raises ValueError if *path* is a string. - Previously an empty list was returned. (Contributed by Sanyam Khurana in +* :meth:`pkgutil.walk_packages` now raises :exc:`ValueError` if *path* is a string. + Previously an empty list was returned. (Contributed by Sanyam Khurana in :issue:`24744`.) * A format string argument for :meth:`string.Formatter.format` @@ -1186,6 +1182,11 @@ Changes in the Python API * The :attr:`struct.Struct.format` type is now :class:`str` instead of :class:`bytes`. (Contributed by Victor Stinner in :issue:`21071`.) +* :func:`~cgi.parse_multipart` returns now the same results as + :class:`~FieldStorage`: for non-file fields, the value associated to a key + is a list of strings, not bytes. + (Contributed by Pierre Quentel in :issue:`29979`.) + * Due to internal changes in :mod:`socket` you won't be able to :func:`socket.fromshare` a socket :func:`~socket.socket.share`-ed in older Python versions. @@ -1207,6 +1208,8 @@ Changes in the Python API avoid a warning escape them with a backslash. (Contributed by Serhiy Storchaka in :issue:`30349`.) + .. _Unicode Technical Standard #18: https://unicode.org/reports/tr18/ + * The result of splitting a string on a :mod:`regular expression <re>` that could match an empty string has been changed. For example splitting on ``r'\s*'`` will now split not only on whitespaces as it @@ -1246,8 +1249,6 @@ Changes in the Python API work as expected on all platforms. (Contributed by Yury Selivanov in :issue:`32331`.) -.. _Unicode Technical Standard #18: https://unicode.org/reports/tr18/ - * On Windows the default for the *close_fds* argument of :class:`subprocess.Popen` was changed from :const:`False` to :const:`True` when redirecting the standard handles. If you previously depended on handles @@ -1276,7 +1277,8 @@ CPython bytecode changes * Added two new opcodes: :opcode:`LOAD_METHOD` and :opcode:`CALL_METHOD`. (Contributed by Yury Selivanov and INADA Naoki in :issue:`26110`.) -* Removed the STORE_ANNOTATION opcode. +* Removed the :opcode:`STORE_ANNOTATION` opcode. + (Contributed by Mark Shannon in :issue:`32550`.) Other CPython implementation changes @@ -1293,9 +1295,9 @@ Other CPython implementation changes (Contributed by Nick Coghlan and Eric Snow as part of :issue:`22257`.) * Due to changes in the way the default warnings filters are configured, - setting ``Py_BytesWarningFlag`` to a value greater than one is no longer - sufficient to both emit ``BytesWarning`` messages and have them converted - to exceptions. Instead, the flag must be set (to cause the warnings to be + setting :c:data:`Py_BytesWarningFlag` to a value greater than one is no longer + sufficient to both emit :exc:`BytesWarning` messages and have them converted + to exceptions. Instead, the flag must be set (to cause the warnings to be emitted in the first place), and an explicit ``error::BytesWarning`` warnings filter added to convert them to exceptions. From webhook-mailer at python.org Sun Mar 11 01:32:50 2018 From: webhook-mailer at python.org (Serhiy Storchaka) Date: Sun, 11 Mar 2018 06:32:50 -0000 Subject: [Python-checkins] bpo-33026: Fix jumping out of "with" block by setting f_lineno. (#6026) Message-ID: <mailman.37.1520749971.1871.python-checkins@python.org> https://github.com/python/cpython/commit/26c9f565d016db21257a60d29ab2c99383dd5ac7 commit: 26c9f565d016db21257a60d29ab2c99383dd5ac7 branch: master author: Serhiy Storchaka <storchaka at gmail.com> committer: GitHub <noreply at github.com> date: 2018-03-11T08:32:47+02:00 summary: bpo-33026: Fix jumping out of "with" block by setting f_lineno. (#6026) files: A Misc/NEWS.d/next/Core and Builtins/2018-03-08-09-48-38.bpo-33026.QZA3Ba.rst M Lib/test/test_sys_settrace.py M Objects/frameobject.c diff --git a/Lib/test/test_sys_settrace.py b/Lib/test/test_sys_settrace.py index 7df10cb5c1f7..72cce33392dc 100644 --- a/Lib/test/test_sys_settrace.py +++ b/Lib/test/test_sys_settrace.py @@ -827,6 +827,34 @@ def test_jump_across_with(output): with tracecontext(output, 4): output.append(5) + @jump_test(4, 5, [1, 3, 5, 6]) + def test_jump_out_of_with_block_within_for_block(output): + output.append(1) + for i in [1]: + with tracecontext(output, 3): + output.append(4) + output.append(5) + output.append(6) + + @jump_test(4, 5, [1, 2, 3, 5, -2, 6]) + def test_jump_out_of_with_block_within_with_block(output): + output.append(1) + with tracecontext(output, 2): + with tracecontext(output, 3): + output.append(4) + output.append(5) + output.append(6) + + @jump_test(5, 6, [2, 4, 6, 7]) + def test_jump_out_of_with_block_within_finally_block(output): + try: + output.append(2) + finally: + with tracecontext(output, 4): + output.append(5) + output.append(6) + output.append(7) + @jump_test(8, 11, [1, 3, 5, 11, 12]) def test_jump_out_of_complex_nested_blocks(output): output.append(1) diff --git a/Misc/NEWS.d/next/Core and Builtins/2018-03-08-09-48-38.bpo-33026.QZA3Ba.rst b/Misc/NEWS.d/next/Core and Builtins/2018-03-08-09-48-38.bpo-33026.QZA3Ba.rst new file mode 100644 index 000000000000..dc166d1e5771 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2018-03-08-09-48-38.bpo-33026.QZA3Ba.rst @@ -0,0 +1 @@ +Fixed jumping out of "with" block by setting f_lineno. diff --git a/Objects/frameobject.c b/Objects/frameobject.c index b9f380d7b60d..31ad8d0be226 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -89,7 +89,6 @@ frame_setlineno(PyFrameObject *f, PyObject* p_new_lineno) long l_new_lineno; int overflow; int new_lasti = 0; /* The new value of f_lasti */ - int new_iblock = 0; /* The new value of f_iblock */ unsigned char *code = NULL; /* The bytecode for the frame... */ Py_ssize_t code_len = 0; /* ...and its length */ unsigned char *lnotab = NULL; /* Iterating over co_lnotab */ @@ -99,6 +98,7 @@ frame_setlineno(PyFrameObject *f, PyObject* p_new_lineno) int addr = 0; /* (ditto) */ int delta_iblock = 0; /* Scanning the SETUPs and POPs */ int for_loop_delta = 0; /* (ditto) */ + int delta; int blockstack[CO_MAXBLOCKS]; /* Walking the 'finally' blocks */ int blockstack_top = 0; /* (ditto) */ @@ -258,19 +258,25 @@ frame_setlineno(PyFrameObject *f, PyObject* p_new_lineno) assert(blockstack_top == 0); /* Pop any blocks that we're jumping out of. */ - new_iblock = f->f_iblock - delta_iblock; - while (f->f_iblock > new_iblock) { - PyTryBlock *b = &f->f_blockstack[--f->f_iblock]; - while ((f->f_stacktop - f->f_valuestack) > b->b_level) { - PyObject *v = (*--f->f_stacktop); - Py_DECREF(v); + delta = 0; + if (delta_iblock > 0) { + f->f_iblock -= delta_iblock; + PyTryBlock *b = &f->f_blockstack[f->f_iblock]; + delta = (f->f_stacktop - f->f_valuestack) - b->b_level; + if (b->b_type == SETUP_FINALLY && + code[b->b_handler] == WITH_CLEANUP_START) + { + /* Pop the exit function. */ + delta++; } } /* Pop the iterators of any 'for' loop we're jumping out of. */ - while (for_loop_delta > 0) { + delta += for_loop_delta; + + while (delta > 0) { PyObject *v = (*--f->f_stacktop); Py_DECREF(v); - for_loop_delta--; + delta--; } /* Finally set the new f_lineno and f_lasti and return OK. */ From webhook-mailer at python.org Sun Mar 11 01:38:16 2018 From: webhook-mailer at python.org (Serhiy Storchaka) Date: Sun, 11 Mar 2018 06:38:16 -0000 Subject: [Python-checkins] bpo-32338: OrderedDict import is no longer needed in re. (#4891) Message-ID: <mailman.38.1520750298.1871.python-checkins@python.org> https://github.com/python/cpython/commit/b931bd0a2fe7e9293339019352baf3317166b769 commit: b931bd0a2fe7e9293339019352baf3317166b769 branch: master author: Serhiy Storchaka <storchaka at gmail.com> committer: GitHub <noreply at github.com> date: 2018-03-11T08:38:13+02:00 summary: bpo-32338: OrderedDict import is no longer needed in re. (#4891) files: M Lib/re.py diff --git a/Lib/re.py b/Lib/re.py index a8b6753d3909..94d486579e08 100644 --- a/Lib/re.py +++ b/Lib/re.py @@ -128,12 +128,6 @@ except ImportError: _locale = None -# try _collections first to reduce startup cost -try: - from _collections import OrderedDict -except ImportError: - from collections import OrderedDict - # public symbols __all__ = [ @@ -271,7 +265,7 @@ def escape(pattern): # -------------------------------------------------------------------- # internals -_cache = OrderedDict() +_cache = {} # ordered! _MAXCACHE = 512 def _compile(pattern, flags): @@ -292,9 +286,10 @@ def _compile(pattern, flags): p = sre_compile.compile(pattern, flags) if not (flags & DEBUG): if len(_cache) >= _MAXCACHE: + # Drop the oldest item try: - _cache.popitem(last=False) - except KeyError: + del _cache[next(iter(_cache))] + except (StopIteration, RuntimeError, KeyError): pass _cache[type(pattern), pattern, flags] = p return p From webhook-mailer at python.org Sun Mar 11 03:02:01 2018 From: webhook-mailer at python.org (Miss Islington (bot)) Date: Sun, 11 Mar 2018 07:02:01 -0000 Subject: [Python-checkins] bpo-32338: OrderedDict import is no longer needed in re. (GH-4891) Message-ID: <mailman.39.1520751722.1871.python-checkins@python.org> https://github.com/python/cpython/commit/39441fce0218a3f51a80cf17aa179a32651a02f6 commit: 39441fce0218a3f51a80cf17aa179a32651a02f6 branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: GitHub <noreply at github.com> date: 2018-03-10T23:01:58-08:00 summary: bpo-32338: OrderedDict import is no longer needed in re. (GH-4891) (cherry picked from commit b931bd0a2fe7e9293339019352baf3317166b769) Co-authored-by: Serhiy Storchaka <storchaka at gmail.com> files: M Lib/re.py diff --git a/Lib/re.py b/Lib/re.py index a8b6753d3909..94d486579e08 100644 --- a/Lib/re.py +++ b/Lib/re.py @@ -128,12 +128,6 @@ except ImportError: _locale = None -# try _collections first to reduce startup cost -try: - from _collections import OrderedDict -except ImportError: - from collections import OrderedDict - # public symbols __all__ = [ @@ -271,7 +265,7 @@ def escape(pattern): # -------------------------------------------------------------------- # internals -_cache = OrderedDict() +_cache = {} # ordered! _MAXCACHE = 512 def _compile(pattern, flags): @@ -292,9 +286,10 @@ def _compile(pattern, flags): p = sre_compile.compile(pattern, flags) if not (flags & DEBUG): if len(_cache) >= _MAXCACHE: + # Drop the oldest item try: - _cache.popitem(last=False) - except KeyError: + del _cache[next(iter(_cache))] + except (StopIteration, RuntimeError, KeyError): pass _cache[type(pattern), pattern, flags] = p return p From webhook-mailer at python.org Sun Mar 11 03:30:16 2018 From: webhook-mailer at python.org (Serhiy Storchaka) Date: Sun, 11 Mar 2018 07:30:16 -0000 Subject: [Python-checkins] [3.7] bpo-33026: Fix jumping out of "with" block by setting f_lineno. (GH-6026). (#6074) Message-ID: <mailman.40.1520753418.1871.python-checkins@python.org> https://github.com/python/cpython/commit/04aadf23eac51fec2e436c5960c1362bbb7d03de commit: 04aadf23eac51fec2e436c5960c1362bbb7d03de branch: 3.7 author: Serhiy Storchaka <storchaka at gmail.com> committer: GitHub <noreply at github.com> date: 2018-03-11T09:30:13+02:00 summary: [3.7] bpo-33026: Fix jumping out of "with" block by setting f_lineno. (GH-6026). (#6074) (cherry picked from commit 26c9f565d016db21257a60d29ab2c99383dd5ac7) Co-authored-by: Serhiy Storchaka <storchaka at gmail.com> files: A Misc/NEWS.d/next/Core and Builtins/2018-03-08-09-48-38.bpo-33026.QZA3Ba.rst M Lib/test/test_sys_settrace.py M Objects/frameobject.c diff --git a/Lib/test/test_sys_settrace.py b/Lib/test/test_sys_settrace.py index e6eb80ad86a8..fa5b61af6172 100644 --- a/Lib/test/test_sys_settrace.py +++ b/Lib/test/test_sys_settrace.py @@ -797,6 +797,34 @@ def test_jump_across_with(output): with tracecontext(output, 4): output.append(5) + @jump_test(4, 5, [1, 3, 5, 6]) + def test_jump_out_of_with_block_within_for_block(output): + output.append(1) + for i in [1]: + with tracecontext(output, 3): + output.append(4) + output.append(5) + output.append(6) + + @jump_test(4, 5, [1, 2, 3, 5, -2, 6]) + def test_jump_out_of_with_block_within_with_block(output): + output.append(1) + with tracecontext(output, 2): + with tracecontext(output, 3): + output.append(4) + output.append(5) + output.append(6) + + @jump_test(5, 6, [2, 4, 6, 7]) + def test_jump_out_of_with_block_within_finally_block(output): + try: + output.append(2) + finally: + with tracecontext(output, 4): + output.append(5) + output.append(6) + output.append(7) + @jump_test(8, 11, [1, 3, 5, 11, 12]) def test_jump_out_of_complex_nested_blocks(output): output.append(1) diff --git a/Misc/NEWS.d/next/Core and Builtins/2018-03-08-09-48-38.bpo-33026.QZA3Ba.rst b/Misc/NEWS.d/next/Core and Builtins/2018-03-08-09-48-38.bpo-33026.QZA3Ba.rst new file mode 100644 index 000000000000..dc166d1e5771 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2018-03-08-09-48-38.bpo-33026.QZA3Ba.rst @@ -0,0 +1 @@ +Fixed jumping out of "with" block by setting f_lineno. diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 3083e5b6445c..2825041e6310 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -320,6 +320,13 @@ frame_setlineno(PyFrameObject *f, PyObject* p_new_lineno) PyObject *v = (*--f->f_stacktop); Py_DECREF(v); } + if (b->b_type == SETUP_FINALLY && + code[b->b_handler] == WITH_CLEANUP_START) + { + /* Pop the exit function. */ + PyObject *v = (*--f->f_stacktop); + Py_DECREF(v); + } } /* Finally set the new f_lineno and f_lasti and return OK. */ From webhook-mailer at python.org Sun Mar 11 04:21:16 2018 From: webhook-mailer at python.org (Serhiy Storchaka) Date: Sun, 11 Mar 2018 08:21:16 -0000 Subject: [Python-checkins] [3.6] bpo-33026: Fix jumping out of "with" block by setting f_lineno. (GH-6026). (GH-6074) (GH-6075) Message-ID: <mailman.41.1520756477.1871.python-checkins@python.org> https://github.com/python/cpython/commit/20ac11a9fb027f183914bbaaaed091b57c882e54 commit: 20ac11a9fb027f183914bbaaaed091b57c882e54 branch: 3.6 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Serhiy Storchaka <storchaka at gmail.com> date: 2018-03-11T10:21:13+02:00 summary: [3.6] bpo-33026: Fix jumping out of "with" block by setting f_lineno. (GH-6026). (GH-6074) (GH-6075) (cherry picked from commit 26c9f565d016db21257a60d29ab2c99383dd5ac7) (cherry picked from commit 04aadf23eac51fec2e436c5960c1362bbb7d03de) Co-authored-by: Serhiy Storchaka <storchaka at gmail.com> files: A Misc/NEWS.d/next/Core and Builtins/2018-03-08-09-48-38.bpo-33026.QZA3Ba.rst M Lib/test/test_sys_settrace.py M Objects/frameobject.c diff --git a/Lib/test/test_sys_settrace.py b/Lib/test/test_sys_settrace.py index 5b3b0660b892..a05fbbd04a84 100644 --- a/Lib/test/test_sys_settrace.py +++ b/Lib/test/test_sys_settrace.py @@ -751,6 +751,34 @@ def test_jump_across_with(output): with tracecontext(output, 4): output.append(5) + @jump_test(4, 5, [1, 3, 5, 6]) + def test_jump_out_of_with_block_within_for_block(output): + output.append(1) + for i in [1]: + with tracecontext(output, 3): + output.append(4) + output.append(5) + output.append(6) + + @jump_test(4, 5, [1, 2, 3, 5, -2, 6]) + def test_jump_out_of_with_block_within_with_block(output): + output.append(1) + with tracecontext(output, 2): + with tracecontext(output, 3): + output.append(4) + output.append(5) + output.append(6) + + @jump_test(5, 6, [2, 4, 6, 7]) + def test_jump_out_of_with_block_within_finally_block(output): + try: + output.append(2) + finally: + with tracecontext(output, 4): + output.append(5) + output.append(6) + output.append(7) + @jump_test(8, 11, [1, 3, 5, 11, 12]) def test_jump_out_of_complex_nested_blocks(output): output.append(1) diff --git a/Misc/NEWS.d/next/Core and Builtins/2018-03-08-09-48-38.bpo-33026.QZA3Ba.rst b/Misc/NEWS.d/next/Core and Builtins/2018-03-08-09-48-38.bpo-33026.QZA3Ba.rst new file mode 100644 index 000000000000..dc166d1e5771 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2018-03-08-09-48-38.bpo-33026.QZA3Ba.rst @@ -0,0 +1 @@ +Fixed jumping out of "with" block by setting f_lineno. diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 62f9f34c8ebf..5c73e8560f86 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -317,6 +317,13 @@ frame_setlineno(PyFrameObject *f, PyObject* p_new_lineno) PyObject *v = (*--f->f_stacktop); Py_DECREF(v); } + if (b->b_type == SETUP_FINALLY && + code[b->b_handler] == WITH_CLEANUP_START) + { + /* Pop the exit function. */ + PyObject *v = (*--f->f_stacktop); + Py_DECREF(v); + } } /* Finally set the new f_lineno and f_lasti and return OK. */ From webhook-mailer at python.org Sun Mar 11 04:52:42 2018 From: webhook-mailer at python.org (Serhiy Storchaka) Date: Sun, 11 Mar 2018 08:52:42 -0000 Subject: [Python-checkins] bpo-32946: Speed up "from ... import ..." from non-packages. (GH-5873) Message-ID: <mailman.42.1520758363.1871.python-checkins@python.org> https://github.com/python/cpython/commit/4e2442505c5e9eec396dcef4d2e6bdd2b6f92fc9 commit: 4e2442505c5e9eec396dcef4d2e6bdd2b6f92fc9 branch: master author: Serhiy Storchaka <storchaka at gmail.com> committer: GitHub <noreply at github.com> date: 2018-03-11T10:52:37+02:00 summary: bpo-32946: Speed up "from ... import ..." from non-packages. (GH-5873) files: A Misc/NEWS.d/next/Core and Builtins/2018-02-25-10-52-40.bpo-32946.Lo09rG.rst M Lib/importlib/_bootstrap.py M Python/import.c M Python/importlib.h diff --git a/Lib/importlib/_bootstrap.py b/Lib/importlib/_bootstrap.py index 2bdd1929b42f..ba5a053d6bfe 100644 --- a/Lib/importlib/_bootstrap.py +++ b/Lib/importlib/_bootstrap.py @@ -1016,31 +1016,30 @@ def _handle_fromlist(module, fromlist, import_, *, recursive=False): """ # The hell that is fromlist ... # If a package was imported, try to import stuff from fromlist. - if hasattr(module, '__path__'): - for x in fromlist: - if not isinstance(x, str): - if recursive: - where = module.__name__ + '.__all__' - else: - where = "``from list''" - raise TypeError(f"Item in {where} must be str, " - f"not {type(x).__name__}") - elif x == '*': - if not recursive and hasattr(module, '__all__'): - _handle_fromlist(module, module.__all__, import_, - recursive=True) - elif not hasattr(module, x): - from_name = '{}.{}'.format(module.__name__, x) - try: - _call_with_frames_removed(import_, from_name) - except ModuleNotFoundError as exc: - # Backwards-compatibility dictates we ignore failed - # imports triggered by fromlist for modules that don't - # exist. - if (exc.name == from_name and - sys.modules.get(from_name, _NEEDS_LOADING) is not None): - continue - raise + for x in fromlist: + if not isinstance(x, str): + if recursive: + where = module.__name__ + '.__all__' + else: + where = "``from list''" + raise TypeError(f"Item in {where} must be str, " + f"not {type(x).__name__}") + elif x == '*': + if not recursive and hasattr(module, '__all__'): + _handle_fromlist(module, module.__all__, import_, + recursive=True) + elif not hasattr(module, x): + from_name = '{}.{}'.format(module.__name__, x) + try: + _call_with_frames_removed(import_, from_name) + except ModuleNotFoundError as exc: + # Backwards-compatibility dictates we ignore failed + # imports triggered by fromlist for modules that don't + # exist. + if (exc.name == from_name and + sys.modules.get(from_name, _NEEDS_LOADING) is not None): + continue + raise return module @@ -1102,8 +1101,10 @@ def __import__(name, globals=None, locals=None, fromlist=(), level=0): # Slice end needs to be positive to alleviate need to special-case # when ``'.' not in name``. return sys.modules[module.__name__[:len(module.__name__)-cut_off]] - else: + elif hasattr(module, '__path__'): return _handle_fromlist(module, fromlist, _gcd_import) + else: + return module def _builtin_from_name(name): diff --git a/Misc/NEWS.d/next/Core and Builtins/2018-02-25-10-52-40.bpo-32946.Lo09rG.rst b/Misc/NEWS.d/next/Core and Builtins/2018-02-25-10-52-40.bpo-32946.Lo09rG.rst new file mode 100644 index 000000000000..cb1d2a7fb13c --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2018-02-25-10-52-40.bpo-32946.Lo09rG.rst @@ -0,0 +1,2 @@ +Importing names from already imported module with "from ... import ..." is +now 30% faster if the module is not a package. diff --git a/Python/import.c b/Python/import.c index eb5aeac55461..9f46da336234 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1800,10 +1800,21 @@ PyImport_ImportModuleLevelObject(PyObject *name, PyObject *globals, } } else { - final_mod = _PyObject_CallMethodIdObjArgs(interp->importlib, - &PyId__handle_fromlist, mod, - fromlist, interp->import_func, - NULL); + _Py_IDENTIFIER(__path__); + PyObject *path; + if (_PyObject_LookupAttrId(mod, &PyId___path__, &path) < 0) { + goto error; + } + if (path) { + Py_DECREF(path); + final_mod = _PyObject_CallMethodIdObjArgs( + interp->importlib, &PyId__handle_fromlist, + mod, fromlist, interp->import_func, NULL); + } + else { + final_mod = mod; + Py_INCREF(mod); + } } error: diff --git a/Python/importlib.h b/Python/importlib.h index 83c5dd5d650e..032ad45f96e9 100644 --- a/Python/importlib.h +++ b/Python/importlib.h @@ -1567,262 +1567,262 @@ const unsigned char _Py_M__importlib[] = { 0,115,8,0,0,0,0,9,12,1,8,1,12,1,114,181, 0,0,0,41,1,218,9,114,101,99,117,114,115,105,118,101, 99,3,0,0,0,1,0,0,0,8,0,0,0,11,0,0, - 0,67,0,0,0,115,236,0,0,0,116,0,124,0,100,1, - 131,2,114,232,124,1,68,0,93,216,125,4,116,1,124,4, - 116,2,131,2,115,76,124,3,114,44,124,0,106,3,100,2, - 23,0,125,5,110,4,100,3,125,5,116,4,100,4,124,5, - 155,0,100,5,116,5,124,4,131,1,106,3,155,0,157,4, - 131,1,130,1,113,14,124,4,100,6,107,2,114,118,124,3, - 115,230,116,0,124,0,100,7,131,2,114,230,116,6,124,0, - 124,0,106,7,124,2,100,8,100,9,141,4,1,0,113,14, - 116,0,124,0,124,4,131,2,115,14,100,10,160,8,124,0, - 106,3,124,4,161,2,125,6,122,14,116,9,124,2,124,6, - 131,2,1,0,87,0,113,14,4,0,116,10,107,10,114,228, - 1,0,125,7,1,0,122,42,124,7,106,11,124,6,107,2, - 114,210,116,12,106,13,160,14,124,6,116,15,161,2,100,11, - 107,9,114,210,87,0,89,0,162,8,113,14,130,0,87,0, - 53,0,100,11,125,7,126,7,88,0,89,0,113,14,88,0, - 113,14,124,0,83,0,41,12,122,238,70,105,103,117,114,101, - 32,111,117,116,32,119,104,97,116,32,95,95,105,109,112,111, - 114,116,95,95,32,115,104,111,117,108,100,32,114,101,116,117, - 114,110,46,10,10,32,32,32,32,84,104,101,32,105,109,112, - 111,114,116,95,32,112,97,114,97,109,101,116,101,114,32,105, - 115,32,97,32,99,97,108,108,97,98,108,101,32,119,104,105, - 99,104,32,116,97,107,101,115,32,116,104,101,32,110,97,109, - 101,32,111,102,32,109,111,100,117,108,101,32,116,111,10,32, - 32,32,32,105,109,112,111,114,116,46,32,73,116,32,105,115, - 32,114,101,113,117,105,114,101,100,32,116,111,32,100,101,99, - 111,117,112,108,101,32,116,104,101,32,102,117,110,99,116,105, - 111,110,32,102,114,111,109,32,97,115,115,117,109,105,110,103, - 32,105,109,112,111,114,116,108,105,98,39,115,10,32,32,32, - 32,105,109,112,111,114,116,32,105,109,112,108,101,109,101,110, - 116,97,116,105,111,110,32,105,115,32,100,101,115,105,114,101, - 100,46,10,10,32,32,32,32,114,127,0,0,0,122,8,46, - 95,95,97,108,108,95,95,122,13,96,96,102,114,111,109,32, - 108,105,115,116,39,39,122,8,73,116,101,109,32,105,110,32, - 122,18,32,109,117,115,116,32,98,101,32,115,116,114,44,32, - 110,111,116,32,250,1,42,218,7,95,95,97,108,108,95,95, - 84,41,1,114,182,0,0,0,122,5,123,125,46,123,125,78, - 41,16,114,4,0,0,0,114,170,0,0,0,114,171,0,0, - 0,114,1,0,0,0,114,172,0,0,0,114,13,0,0,0, - 218,16,95,104,97,110,100,108,101,95,102,114,111,109,108,105, - 115,116,114,184,0,0,0,114,38,0,0,0,114,59,0,0, - 0,114,176,0,0,0,114,15,0,0,0,114,14,0,0,0, - 114,79,0,0,0,114,30,0,0,0,114,179,0,0,0,41, - 8,114,83,0,0,0,218,8,102,114,111,109,108,105,115,116, - 114,177,0,0,0,114,182,0,0,0,218,1,120,90,5,119, - 104,101,114,101,90,9,102,114,111,109,95,110,97,109,101,90, - 3,101,120,99,114,10,0,0,0,114,10,0,0,0,114,11, - 0,0,0,114,185,0,0,0,241,3,0,0,115,42,0,0, - 0,0,10,10,1,8,1,10,1,4,1,12,2,4,1,28, - 2,8,1,14,1,10,1,10,1,10,1,14,1,2,1,14, - 1,16,4,10,1,18,1,8,1,22,1,114,185,0,0,0, - 99,1,0,0,0,0,0,0,0,3,0,0,0,6,0,0, - 0,67,0,0,0,115,146,0,0,0,124,0,160,0,100,1, - 161,1,125,1,124,0,160,0,100,2,161,1,125,2,124,1, - 100,3,107,9,114,82,124,2,100,3,107,9,114,78,124,1, - 124,2,106,1,107,3,114,78,116,2,106,3,100,4,124,1, - 155,2,100,5,124,2,106,1,155,2,100,6,157,5,116,4, - 100,7,100,8,141,3,1,0,124,1,83,0,124,2,100,3, - 107,9,114,96,124,2,106,1,83,0,116,2,106,3,100,9, - 116,4,100,7,100,8,141,3,1,0,124,0,100,10,25,0, - 125,1,100,11,124,0,107,7,114,142,124,1,160,5,100,12, - 161,1,100,13,25,0,125,1,124,1,83,0,41,14,122,167, - 67,97,108,99,117,108,97,116,101,32,119,104,97,116,32,95, - 95,112,97,99,107,97,103,101,95,95,32,115,104,111,117,108, - 100,32,98,101,46,10,10,32,32,32,32,95,95,112,97,99, - 107,97,103,101,95,95,32,105,115,32,110,111,116,32,103,117, - 97,114,97,110,116,101,101,100,32,116,111,32,98,101,32,100, - 101,102,105,110,101,100,32,111,114,32,99,111,117,108,100,32, - 98,101,32,115,101,116,32,116,111,32,78,111,110,101,10,32, - 32,32,32,116,111,32,114,101,112,114,101,115,101,110,116,32, - 116,104,97,116,32,105,116,115,32,112,114,111,112,101,114,32, - 118,97,108,117,101,32,105,115,32,117,110,107,110,111,119,110, - 46,10,10,32,32,32,32,114,130,0,0,0,114,89,0,0, - 0,78,122,32,95,95,112,97,99,107,97,103,101,95,95,32, - 33,61,32,95,95,115,112,101,99,95,95,46,112,97,114,101, - 110,116,32,40,122,4,32,33,61,32,250,1,41,233,3,0, - 0,0,41,1,90,10,115,116,97,99,107,108,101,118,101,108, - 122,89,99,97,110,39,116,32,114,101,115,111,108,118,101,32, - 112,97,99,107,97,103,101,32,102,114,111,109,32,95,95,115, - 112,101,99,95,95,32,111,114,32,95,95,112,97,99,107,97, - 103,101,95,95,44,32,102,97,108,108,105,110,103,32,98,97, - 99,107,32,111,110,32,95,95,110,97,109,101,95,95,32,97, - 110,100,32,95,95,112,97,116,104,95,95,114,1,0,0,0, - 114,127,0,0,0,114,117,0,0,0,114,19,0,0,0,41, - 6,114,30,0,0,0,114,119,0,0,0,114,166,0,0,0, - 114,167,0,0,0,114,168,0,0,0,114,118,0,0,0,41, - 3,218,7,103,108,111,98,97,108,115,114,160,0,0,0,114, - 82,0,0,0,114,10,0,0,0,114,10,0,0,0,114,11, - 0,0,0,218,17,95,99,97,108,99,95,95,95,112,97,99, - 107,97,103,101,95,95,23,4,0,0,115,30,0,0,0,0, - 7,10,1,10,1,8,1,18,1,22,2,10,1,4,1,8, - 1,6,2,6,2,10,1,8,1,8,1,14,1,114,191,0, - 0,0,114,10,0,0,0,99,5,0,0,0,0,0,0,0, - 9,0,0,0,5,0,0,0,67,0,0,0,115,166,0,0, - 0,124,4,100,1,107,2,114,18,116,0,124,0,131,1,125, - 5,110,36,124,1,100,2,107,9,114,30,124,1,110,2,105, - 0,125,6,116,1,124,6,131,1,125,7,116,0,124,0,124, - 7,124,4,131,3,125,5,124,3,115,150,124,4,100,1,107, - 2,114,84,116,0,124,0,160,2,100,3,161,1,100,1,25, - 0,131,1,83,0,124,0,115,92,124,5,83,0,116,3,124, - 0,131,1,116,3,124,0,160,2,100,3,161,1,100,1,25, - 0,131,1,24,0,125,8,116,4,106,5,124,5,106,6,100, - 2,116,3,124,5,106,6,131,1,124,8,24,0,133,2,25, - 0,25,0,83,0,110,12,116,7,124,5,124,3,116,0,131, - 3,83,0,100,2,83,0,41,4,97,215,1,0,0,73,109, - 112,111,114,116,32,97,32,109,111,100,117,108,101,46,10,10, - 32,32,32,32,84,104,101,32,39,103,108,111,98,97,108,115, - 39,32,97,114,103,117,109,101,110,116,32,105,115,32,117,115, - 101,100,32,116,111,32,105,110,102,101,114,32,119,104,101,114, - 101,32,116,104,101,32,105,109,112,111,114,116,32,105,115,32, - 111,99,99,117,114,114,105,110,103,32,102,114,111,109,10,32, - 32,32,32,116,111,32,104,97,110,100,108,101,32,114,101,108, - 97,116,105,118,101,32,105,109,112,111,114,116,115,46,32,84, - 104,101,32,39,108,111,99,97,108,115,39,32,97,114,103,117, - 109,101,110,116,32,105,115,32,105,103,110,111,114,101,100,46, - 32,84,104,101,10,32,32,32,32,39,102,114,111,109,108,105, - 115,116,39,32,97,114,103,117,109,101,110,116,32,115,112,101, - 99,105,102,105,101,115,32,119,104,97,116,32,115,104,111,117, - 108,100,32,101,120,105,115,116,32,97,115,32,97,116,116,114, - 105,98,117,116,101,115,32,111,110,32,116,104,101,32,109,111, - 100,117,108,101,10,32,32,32,32,98,101,105,110,103,32,105, - 109,112,111,114,116,101,100,32,40,101,46,103,46,32,96,96, - 102,114,111,109,32,109,111,100,117,108,101,32,105,109,112,111, - 114,116,32,60,102,114,111,109,108,105,115,116,62,96,96,41, - 46,32,32,84,104,101,32,39,108,101,118,101,108,39,10,32, - 32,32,32,97,114,103,117,109,101,110,116,32,114,101,112,114, - 101,115,101,110,116,115,32,116,104,101,32,112,97,99,107,97, - 103,101,32,108,111,99,97,116,105,111,110,32,116,111,32,105, - 109,112,111,114,116,32,102,114,111,109,32,105,110,32,97,32, - 114,101,108,97,116,105,118,101,10,32,32,32,32,105,109,112, - 111,114,116,32,40,101,46,103,46,32,96,96,102,114,111,109, - 32,46,46,112,107,103,32,105,109,112,111,114,116,32,109,111, - 100,96,96,32,119,111,117,108,100,32,104,97,118,101,32,97, - 32,39,108,101,118,101,108,39,32,111,102,32,50,41,46,10, - 10,32,32,32,32,114,19,0,0,0,78,114,117,0,0,0, - 41,8,114,181,0,0,0,114,191,0,0,0,218,9,112,97, - 114,116,105,116,105,111,110,114,158,0,0,0,114,14,0,0, - 0,114,79,0,0,0,114,1,0,0,0,114,185,0,0,0, - 41,9,114,15,0,0,0,114,190,0,0,0,218,6,108,111, - 99,97,108,115,114,186,0,0,0,114,161,0,0,0,114,83, - 0,0,0,90,8,103,108,111,98,97,108,115,95,114,160,0, - 0,0,90,7,99,117,116,95,111,102,102,114,10,0,0,0, - 114,10,0,0,0,114,11,0,0,0,218,10,95,95,105,109, - 112,111,114,116,95,95,50,4,0,0,115,26,0,0,0,0, - 11,8,1,10,2,16,1,8,1,12,1,4,3,8,1,18, - 1,4,1,4,4,26,3,32,2,114,194,0,0,0,99,1, - 0,0,0,0,0,0,0,2,0,0,0,3,0,0,0,67, - 0,0,0,115,38,0,0,0,116,0,160,1,124,0,161,1, - 125,1,124,1,100,0,107,8,114,30,116,2,100,1,124,0, - 23,0,131,1,130,1,116,3,124,1,131,1,83,0,41,2, - 78,122,25,110,111,32,98,117,105,108,116,45,105,110,32,109, - 111,100,117,108,101,32,110,97,109,101,100,32,41,4,114,141, - 0,0,0,114,145,0,0,0,114,70,0,0,0,114,140,0, - 0,0,41,2,114,15,0,0,0,114,82,0,0,0,114,10, - 0,0,0,114,10,0,0,0,114,11,0,0,0,218,18,95, - 98,117,105,108,116,105,110,95,102,114,111,109,95,110,97,109, - 101,85,4,0,0,115,8,0,0,0,0,1,10,1,8,1, - 12,1,114,195,0,0,0,99,2,0,0,0,0,0,0,0, - 10,0,0,0,5,0,0,0,67,0,0,0,115,166,0,0, - 0,124,1,97,0,124,0,97,1,116,2,116,1,131,1,125, - 2,116,1,106,3,160,4,161,0,68,0,93,72,92,2,125, - 3,125,4,116,5,124,4,124,2,131,2,114,26,124,3,116, - 1,106,6,107,6,114,60,116,7,125,5,110,18,116,0,160, - 8,124,3,161,1,114,26,116,9,125,5,110,2,113,26,116, - 10,124,4,124,5,131,2,125,6,116,11,124,6,124,4,131, - 2,1,0,113,26,116,1,106,3,116,12,25,0,125,7,100, - 1,68,0,93,46,125,8,124,8,116,1,106,3,107,7,114, - 138,116,13,124,8,131,1,125,9,110,10,116,1,106,3,124, - 8,25,0,125,9,116,14,124,7,124,8,124,9,131,3,1, - 0,113,114,100,2,83,0,41,3,122,250,83,101,116,117,112, - 32,105,109,112,111,114,116,108,105,98,32,98,121,32,105,109, - 112,111,114,116,105,110,103,32,110,101,101,100,101,100,32,98, - 117,105,108,116,45,105,110,32,109,111,100,117,108,101,115,32, - 97,110,100,32,105,110,106,101,99,116,105,110,103,32,116,104, - 101,109,10,32,32,32,32,105,110,116,111,32,116,104,101,32, - 103,108,111,98,97,108,32,110,97,109,101,115,112,97,99,101, - 46,10,10,32,32,32,32,65,115,32,115,121,115,32,105,115, - 32,110,101,101,100,101,100,32,102,111,114,32,115,121,115,46, - 109,111,100,117,108,101,115,32,97,99,99,101,115,115,32,97, - 110,100,32,95,105,109,112,32,105,115,32,110,101,101,100,101, - 100,32,116,111,32,108,111,97,100,32,98,117,105,108,116,45, - 105,110,10,32,32,32,32,109,111,100,117,108,101,115,44,32, - 116,104,111,115,101,32,116,119,111,32,109,111,100,117,108,101, - 115,32,109,117,115,116,32,98,101,32,101,120,112,108,105,99, - 105,116,108,121,32,112,97,115,115,101,100,32,105,110,46,10, - 10,32,32,32,32,41,3,114,20,0,0,0,114,166,0,0, - 0,114,56,0,0,0,78,41,15,114,49,0,0,0,114,14, - 0,0,0,114,13,0,0,0,114,79,0,0,0,218,5,105, - 116,101,109,115,114,170,0,0,0,114,69,0,0,0,114,141, - 0,0,0,114,75,0,0,0,114,151,0,0,0,114,128,0, - 0,0,114,133,0,0,0,114,1,0,0,0,114,195,0,0, - 0,114,5,0,0,0,41,10,218,10,115,121,115,95,109,111, - 100,117,108,101,218,11,95,105,109,112,95,109,111,100,117,108, - 101,90,11,109,111,100,117,108,101,95,116,121,112,101,114,15, - 0,0,0,114,83,0,0,0,114,93,0,0,0,114,82,0, - 0,0,90,11,115,101,108,102,95,109,111,100,117,108,101,90, - 12,98,117,105,108,116,105,110,95,110,97,109,101,90,14,98, - 117,105,108,116,105,110,95,109,111,100,117,108,101,114,10,0, - 0,0,114,10,0,0,0,114,11,0,0,0,218,6,95,115, - 101,116,117,112,92,4,0,0,115,36,0,0,0,0,9,4, - 1,4,3,8,1,18,1,10,1,10,1,6,1,10,1,6, - 2,2,1,10,1,12,3,10,1,8,1,10,1,10,2,10, - 1,114,199,0,0,0,99,2,0,0,0,0,0,0,0,2, - 0,0,0,3,0,0,0,67,0,0,0,115,38,0,0,0, - 116,0,124,0,124,1,131,2,1,0,116,1,106,2,160,3, - 116,4,161,1,1,0,116,1,106,2,160,3,116,5,161,1, - 1,0,100,1,83,0,41,2,122,48,73,110,115,116,97,108, - 108,32,105,109,112,111,114,116,101,114,115,32,102,111,114,32, - 98,117,105,108,116,105,110,32,97,110,100,32,102,114,111,122, - 101,110,32,109,111,100,117,108,101,115,78,41,6,114,199,0, - 0,0,114,14,0,0,0,114,165,0,0,0,114,109,0,0, - 0,114,141,0,0,0,114,151,0,0,0,41,2,114,197,0, - 0,0,114,198,0,0,0,114,10,0,0,0,114,10,0,0, - 0,114,11,0,0,0,218,8,95,105,110,115,116,97,108,108, - 127,4,0,0,115,6,0,0,0,0,2,10,2,12,1,114, - 200,0,0,0,99,0,0,0,0,0,0,0,0,1,0,0, - 0,4,0,0,0,67,0,0,0,115,32,0,0,0,100,1, - 100,2,108,0,125,0,124,0,97,1,124,0,160,2,116,3, - 106,4,116,5,25,0,161,1,1,0,100,2,83,0,41,3, - 122,57,73,110,115,116,97,108,108,32,105,109,112,111,114,116, - 101,114,115,32,116,104,97,116,32,114,101,113,117,105,114,101, - 32,101,120,116,101,114,110,97,108,32,102,105,108,101,115,121, - 115,116,101,109,32,97,99,99,101,115,115,114,19,0,0,0, - 78,41,6,218,26,95,102,114,111,122,101,110,95,105,109,112, - 111,114,116,108,105,98,95,101,120,116,101,114,110,97,108,114, - 115,0,0,0,114,200,0,0,0,114,14,0,0,0,114,79, - 0,0,0,114,1,0,0,0,41,1,114,201,0,0,0,114, - 10,0,0,0,114,10,0,0,0,114,11,0,0,0,218,27, - 95,105,110,115,116,97,108,108,95,101,120,116,101,114,110,97, - 108,95,105,109,112,111,114,116,101,114,115,135,4,0,0,115, - 6,0,0,0,0,3,8,1,4,1,114,202,0,0,0,41, - 2,78,78,41,1,78,41,2,78,114,19,0,0,0,41,4, - 78,78,114,10,0,0,0,114,19,0,0,0,41,51,114,3, - 0,0,0,114,115,0,0,0,114,12,0,0,0,114,16,0, - 0,0,114,51,0,0,0,114,29,0,0,0,114,36,0,0, - 0,114,17,0,0,0,114,18,0,0,0,114,41,0,0,0, - 114,42,0,0,0,114,45,0,0,0,114,57,0,0,0,114, - 59,0,0,0,114,68,0,0,0,114,74,0,0,0,114,77, - 0,0,0,114,84,0,0,0,114,95,0,0,0,114,96,0, - 0,0,114,102,0,0,0,114,78,0,0,0,114,128,0,0, - 0,114,133,0,0,0,114,136,0,0,0,114,91,0,0,0, - 114,80,0,0,0,114,139,0,0,0,114,140,0,0,0,114, - 81,0,0,0,114,141,0,0,0,114,151,0,0,0,114,156, - 0,0,0,114,162,0,0,0,114,164,0,0,0,114,169,0, - 0,0,114,173,0,0,0,90,15,95,69,82,82,95,77,83, - 71,95,80,82,69,70,73,88,114,175,0,0,0,114,178,0, - 0,0,218,6,111,98,106,101,99,116,114,179,0,0,0,114, - 180,0,0,0,114,181,0,0,0,114,185,0,0,0,114,191, - 0,0,0,114,194,0,0,0,114,195,0,0,0,114,199,0, - 0,0,114,200,0,0,0,114,202,0,0,0,114,10,0,0, - 0,114,10,0,0,0,114,10,0,0,0,114,11,0,0,0, - 218,8,60,109,111,100,117,108,101,62,25,0,0,0,115,96, - 0,0,0,4,0,4,2,8,8,8,8,4,2,4,3,16, - 4,14,68,14,21,14,16,8,37,8,17,8,11,14,8,8, - 11,8,12,8,16,8,36,14,27,14,101,16,26,10,45,14, - 72,8,17,8,17,8,24,8,29,8,23,8,15,14,73,14, - 77,14,13,8,9,8,9,10,47,8,16,4,1,8,2,8, - 27,6,3,8,16,10,15,14,38,8,27,10,35,8,7,8, - 35,8,8, + 0,67,0,0,0,115,226,0,0,0,124,1,68,0,93,216, + 125,4,116,0,124,4,116,1,131,2,115,66,124,3,114,34, + 124,0,106,2,100,1,23,0,125,5,110,4,100,2,125,5, + 116,3,100,3,124,5,155,0,100,4,116,4,124,4,131,1, + 106,2,155,0,157,4,131,1,130,1,113,4,124,4,100,5, + 107,2,114,108,124,3,115,220,116,5,124,0,100,6,131,2, + 114,220,116,6,124,0,124,0,106,7,124,2,100,7,100,8, + 141,4,1,0,113,4,116,5,124,0,124,4,131,2,115,4, + 100,9,160,8,124,0,106,2,124,4,161,2,125,6,122,14, + 116,9,124,2,124,6,131,2,1,0,87,0,113,4,4,0, + 116,10,107,10,114,218,1,0,125,7,1,0,122,42,124,7, + 106,11,124,6,107,2,114,200,116,12,106,13,160,14,124,6, + 116,15,161,2,100,10,107,9,114,200,87,0,89,0,162,8, + 113,4,130,0,87,0,53,0,100,10,125,7,126,7,88,0, + 89,0,113,4,88,0,113,4,124,0,83,0,41,11,122,238, + 70,105,103,117,114,101,32,111,117,116,32,119,104,97,116,32, + 95,95,105,109,112,111,114,116,95,95,32,115,104,111,117,108, + 100,32,114,101,116,117,114,110,46,10,10,32,32,32,32,84, + 104,101,32,105,109,112,111,114,116,95,32,112,97,114,97,109, + 101,116,101,114,32,105,115,32,97,32,99,97,108,108,97,98, + 108,101,32,119,104,105,99,104,32,116,97,107,101,115,32,116, + 104,101,32,110,97,109,101,32,111,102,32,109,111,100,117,108, + 101,32,116,111,10,32,32,32,32,105,109,112,111,114,116,46, + 32,73,116,32,105,115,32,114,101,113,117,105,114,101,100,32, + 116,111,32,100,101,99,111,117,112,108,101,32,116,104,101,32, + 102,117,110,99,116,105,111,110,32,102,114,111,109,32,97,115, + 115,117,109,105,110,103,32,105,109,112,111,114,116,108,105,98, + 39,115,10,32,32,32,32,105,109,112,111,114,116,32,105,109, + 112,108,101,109,101,110,116,97,116,105,111,110,32,105,115,32, + 100,101,115,105,114,101,100,46,10,10,32,32,32,32,122,8, + 46,95,95,97,108,108,95,95,122,13,96,96,102,114,111,109, + 32,108,105,115,116,39,39,122,8,73,116,101,109,32,105,110, + 32,122,18,32,109,117,115,116,32,98,101,32,115,116,114,44, + 32,110,111,116,32,250,1,42,218,7,95,95,97,108,108,95, + 95,84,41,1,114,182,0,0,0,122,5,123,125,46,123,125, + 78,41,16,114,170,0,0,0,114,171,0,0,0,114,1,0, + 0,0,114,172,0,0,0,114,13,0,0,0,114,4,0,0, + 0,218,16,95,104,97,110,100,108,101,95,102,114,111,109,108, + 105,115,116,114,184,0,0,0,114,38,0,0,0,114,59,0, + 0,0,114,176,0,0,0,114,15,0,0,0,114,14,0,0, + 0,114,79,0,0,0,114,30,0,0,0,114,179,0,0,0, + 41,8,114,83,0,0,0,218,8,102,114,111,109,108,105,115, + 116,114,177,0,0,0,114,182,0,0,0,218,1,120,90,5, + 119,104,101,114,101,90,9,102,114,111,109,95,110,97,109,101, + 90,3,101,120,99,114,10,0,0,0,114,10,0,0,0,114, + 11,0,0,0,114,185,0,0,0,241,3,0,0,115,40,0, + 0,0,0,10,8,1,10,1,4,1,12,2,4,1,28,2, + 8,1,14,1,10,1,10,1,10,1,14,1,2,1,14,1, + 16,4,10,1,18,1,8,1,22,1,114,185,0,0,0,99, + 1,0,0,0,0,0,0,0,3,0,0,0,6,0,0,0, + 67,0,0,0,115,146,0,0,0,124,0,160,0,100,1,161, + 1,125,1,124,0,160,0,100,2,161,1,125,2,124,1,100, + 3,107,9,114,82,124,2,100,3,107,9,114,78,124,1,124, + 2,106,1,107,3,114,78,116,2,106,3,100,4,124,1,155, + 2,100,5,124,2,106,1,155,2,100,6,157,5,116,4,100, + 7,100,8,141,3,1,0,124,1,83,0,124,2,100,3,107, + 9,114,96,124,2,106,1,83,0,116,2,106,3,100,9,116, + 4,100,7,100,8,141,3,1,0,124,0,100,10,25,0,125, + 1,100,11,124,0,107,7,114,142,124,1,160,5,100,12,161, + 1,100,13,25,0,125,1,124,1,83,0,41,14,122,167,67, + 97,108,99,117,108,97,116,101,32,119,104,97,116,32,95,95, + 112,97,99,107,97,103,101,95,95,32,115,104,111,117,108,100, + 32,98,101,46,10,10,32,32,32,32,95,95,112,97,99,107, + 97,103,101,95,95,32,105,115,32,110,111,116,32,103,117,97, + 114,97,110,116,101,101,100,32,116,111,32,98,101,32,100,101, + 102,105,110,101,100,32,111,114,32,99,111,117,108,100,32,98, + 101,32,115,101,116,32,116,111,32,78,111,110,101,10,32,32, + 32,32,116,111,32,114,101,112,114,101,115,101,110,116,32,116, + 104,97,116,32,105,116,115,32,112,114,111,112,101,114,32,118, + 97,108,117,101,32,105,115,32,117,110,107,110,111,119,110,46, + 10,10,32,32,32,32,114,130,0,0,0,114,89,0,0,0, + 78,122,32,95,95,112,97,99,107,97,103,101,95,95,32,33, + 61,32,95,95,115,112,101,99,95,95,46,112,97,114,101,110, + 116,32,40,122,4,32,33,61,32,250,1,41,233,3,0,0, + 0,41,1,90,10,115,116,97,99,107,108,101,118,101,108,122, + 89,99,97,110,39,116,32,114,101,115,111,108,118,101,32,112, + 97,99,107,97,103,101,32,102,114,111,109,32,95,95,115,112, + 101,99,95,95,32,111,114,32,95,95,112,97,99,107,97,103, + 101,95,95,44,32,102,97,108,108,105,110,103,32,98,97,99, + 107,32,111,110,32,95,95,110,97,109,101,95,95,32,97,110, + 100,32,95,95,112,97,116,104,95,95,114,1,0,0,0,114, + 127,0,0,0,114,117,0,0,0,114,19,0,0,0,41,6, + 114,30,0,0,0,114,119,0,0,0,114,166,0,0,0,114, + 167,0,0,0,114,168,0,0,0,114,118,0,0,0,41,3, + 218,7,103,108,111,98,97,108,115,114,160,0,0,0,114,82, + 0,0,0,114,10,0,0,0,114,10,0,0,0,114,11,0, + 0,0,218,17,95,99,97,108,99,95,95,95,112,97,99,107, + 97,103,101,95,95,22,4,0,0,115,30,0,0,0,0,7, + 10,1,10,1,8,1,18,1,22,2,10,1,4,1,8,1, + 6,2,6,2,10,1,8,1,8,1,14,1,114,191,0,0, + 0,114,10,0,0,0,99,5,0,0,0,0,0,0,0,9, + 0,0,0,5,0,0,0,67,0,0,0,115,180,0,0,0, + 124,4,100,1,107,2,114,18,116,0,124,0,131,1,125,5, + 110,36,124,1,100,2,107,9,114,30,124,1,110,2,105,0, + 125,6,116,1,124,6,131,1,125,7,116,0,124,0,124,7, + 124,4,131,3,125,5,124,3,115,150,124,4,100,1,107,2, + 114,84,116,0,124,0,160,2,100,3,161,1,100,1,25,0, + 131,1,83,0,124,0,115,92,124,5,83,0,116,3,124,0, + 131,1,116,3,124,0,160,2,100,3,161,1,100,1,25,0, + 131,1,24,0,125,8,116,4,106,5,124,5,106,6,100,2, + 116,3,124,5,106,6,131,1,124,8,24,0,133,2,25,0, + 25,0,83,0,110,26,116,7,124,5,100,4,131,2,114,172, + 116,8,124,5,124,3,116,0,131,3,83,0,124,5,83,0, + 100,2,83,0,41,5,97,215,1,0,0,73,109,112,111,114, + 116,32,97,32,109,111,100,117,108,101,46,10,10,32,32,32, + 32,84,104,101,32,39,103,108,111,98,97,108,115,39,32,97, + 114,103,117,109,101,110,116,32,105,115,32,117,115,101,100,32, + 116,111,32,105,110,102,101,114,32,119,104,101,114,101,32,116, + 104,101,32,105,109,112,111,114,116,32,105,115,32,111,99,99, + 117,114,114,105,110,103,32,102,114,111,109,10,32,32,32,32, + 116,111,32,104,97,110,100,108,101,32,114,101,108,97,116,105, + 118,101,32,105,109,112,111,114,116,115,46,32,84,104,101,32, + 39,108,111,99,97,108,115,39,32,97,114,103,117,109,101,110, + 116,32,105,115,32,105,103,110,111,114,101,100,46,32,84,104, + 101,10,32,32,32,32,39,102,114,111,109,108,105,115,116,39, + 32,97,114,103,117,109,101,110,116,32,115,112,101,99,105,102, + 105,101,115,32,119,104,97,116,32,115,104,111,117,108,100,32, + 101,120,105,115,116,32,97,115,32,97,116,116,114,105,98,117, + 116,101,115,32,111,110,32,116,104,101,32,109,111,100,117,108, + 101,10,32,32,32,32,98,101,105,110,103,32,105,109,112,111, + 114,116,101,100,32,40,101,46,103,46,32,96,96,102,114,111, + 109,32,109,111,100,117,108,101,32,105,109,112,111,114,116,32, + 60,102,114,111,109,108,105,115,116,62,96,96,41,46,32,32, + 84,104,101,32,39,108,101,118,101,108,39,10,32,32,32,32, + 97,114,103,117,109,101,110,116,32,114,101,112,114,101,115,101, + 110,116,115,32,116,104,101,32,112,97,99,107,97,103,101,32, + 108,111,99,97,116,105,111,110,32,116,111,32,105,109,112,111, + 114,116,32,102,114,111,109,32,105,110,32,97,32,114,101,108, + 97,116,105,118,101,10,32,32,32,32,105,109,112,111,114,116, + 32,40,101,46,103,46,32,96,96,102,114,111,109,32,46,46, + 112,107,103,32,105,109,112,111,114,116,32,109,111,100,96,96, + 32,119,111,117,108,100,32,104,97,118,101,32,97,32,39,108, + 101,118,101,108,39,32,111,102,32,50,41,46,10,10,32,32, + 32,32,114,19,0,0,0,78,114,117,0,0,0,114,127,0, + 0,0,41,9,114,181,0,0,0,114,191,0,0,0,218,9, + 112,97,114,116,105,116,105,111,110,114,158,0,0,0,114,14, + 0,0,0,114,79,0,0,0,114,1,0,0,0,114,4,0, + 0,0,114,185,0,0,0,41,9,114,15,0,0,0,114,190, + 0,0,0,218,6,108,111,99,97,108,115,114,186,0,0,0, + 114,161,0,0,0,114,83,0,0,0,90,8,103,108,111,98, + 97,108,115,95,114,160,0,0,0,90,7,99,117,116,95,111, + 102,102,114,10,0,0,0,114,10,0,0,0,114,11,0,0, + 0,218,10,95,95,105,109,112,111,114,116,95,95,49,4,0, + 0,115,30,0,0,0,0,11,8,1,10,2,16,1,8,1, + 12,1,4,3,8,1,18,1,4,1,4,4,26,3,32,1, + 10,1,12,2,114,194,0,0,0,99,1,0,0,0,0,0, + 0,0,2,0,0,0,3,0,0,0,67,0,0,0,115,38, + 0,0,0,116,0,160,1,124,0,161,1,125,1,124,1,100, + 0,107,8,114,30,116,2,100,1,124,0,23,0,131,1,130, + 1,116,3,124,1,131,1,83,0,41,2,78,122,25,110,111, + 32,98,117,105,108,116,45,105,110,32,109,111,100,117,108,101, + 32,110,97,109,101,100,32,41,4,114,141,0,0,0,114,145, + 0,0,0,114,70,0,0,0,114,140,0,0,0,41,2,114, + 15,0,0,0,114,82,0,0,0,114,10,0,0,0,114,10, + 0,0,0,114,11,0,0,0,218,18,95,98,117,105,108,116, + 105,110,95,102,114,111,109,95,110,97,109,101,86,4,0,0, + 115,8,0,0,0,0,1,10,1,8,1,12,1,114,195,0, + 0,0,99,2,0,0,0,0,0,0,0,10,0,0,0,5, + 0,0,0,67,0,0,0,115,166,0,0,0,124,1,97,0, + 124,0,97,1,116,2,116,1,131,1,125,2,116,1,106,3, + 160,4,161,0,68,0,93,72,92,2,125,3,125,4,116,5, + 124,4,124,2,131,2,114,26,124,3,116,1,106,6,107,6, + 114,60,116,7,125,5,110,18,116,0,160,8,124,3,161,1, + 114,26,116,9,125,5,110,2,113,26,116,10,124,4,124,5, + 131,2,125,6,116,11,124,6,124,4,131,2,1,0,113,26, + 116,1,106,3,116,12,25,0,125,7,100,1,68,0,93,46, + 125,8,124,8,116,1,106,3,107,7,114,138,116,13,124,8, + 131,1,125,9,110,10,116,1,106,3,124,8,25,0,125,9, + 116,14,124,7,124,8,124,9,131,3,1,0,113,114,100,2, + 83,0,41,3,122,250,83,101,116,117,112,32,105,109,112,111, + 114,116,108,105,98,32,98,121,32,105,109,112,111,114,116,105, + 110,103,32,110,101,101,100,101,100,32,98,117,105,108,116,45, + 105,110,32,109,111,100,117,108,101,115,32,97,110,100,32,105, + 110,106,101,99,116,105,110,103,32,116,104,101,109,10,32,32, + 32,32,105,110,116,111,32,116,104,101,32,103,108,111,98,97, + 108,32,110,97,109,101,115,112,97,99,101,46,10,10,32,32, + 32,32,65,115,32,115,121,115,32,105,115,32,110,101,101,100, + 101,100,32,102,111,114,32,115,121,115,46,109,111,100,117,108, + 101,115,32,97,99,99,101,115,115,32,97,110,100,32,95,105, + 109,112,32,105,115,32,110,101,101,100,101,100,32,116,111,32, + 108,111,97,100,32,98,117,105,108,116,45,105,110,10,32,32, + 32,32,109,111,100,117,108,101,115,44,32,116,104,111,115,101, + 32,116,119,111,32,109,111,100,117,108,101,115,32,109,117,115, + 116,32,98,101,32,101,120,112,108,105,99,105,116,108,121,32, + 112,97,115,115,101,100,32,105,110,46,10,10,32,32,32,32, + 41,3,114,20,0,0,0,114,166,0,0,0,114,56,0,0, + 0,78,41,15,114,49,0,0,0,114,14,0,0,0,114,13, + 0,0,0,114,79,0,0,0,218,5,105,116,101,109,115,114, + 170,0,0,0,114,69,0,0,0,114,141,0,0,0,114,75, + 0,0,0,114,151,0,0,0,114,128,0,0,0,114,133,0, + 0,0,114,1,0,0,0,114,195,0,0,0,114,5,0,0, + 0,41,10,218,10,115,121,115,95,109,111,100,117,108,101,218, + 11,95,105,109,112,95,109,111,100,117,108,101,90,11,109,111, + 100,117,108,101,95,116,121,112,101,114,15,0,0,0,114,83, + 0,0,0,114,93,0,0,0,114,82,0,0,0,90,11,115, + 101,108,102,95,109,111,100,117,108,101,90,12,98,117,105,108, + 116,105,110,95,110,97,109,101,90,14,98,117,105,108,116,105, + 110,95,109,111,100,117,108,101,114,10,0,0,0,114,10,0, + 0,0,114,11,0,0,0,218,6,95,115,101,116,117,112,93, + 4,0,0,115,36,0,0,0,0,9,4,1,4,3,8,1, + 18,1,10,1,10,1,6,1,10,1,6,2,2,1,10,1, + 12,3,10,1,8,1,10,1,10,2,10,1,114,199,0,0, + 0,99,2,0,0,0,0,0,0,0,2,0,0,0,3,0, + 0,0,67,0,0,0,115,38,0,0,0,116,0,124,0,124, + 1,131,2,1,0,116,1,106,2,160,3,116,4,161,1,1, + 0,116,1,106,2,160,3,116,5,161,1,1,0,100,1,83, + 0,41,2,122,48,73,110,115,116,97,108,108,32,105,109,112, + 111,114,116,101,114,115,32,102,111,114,32,98,117,105,108,116, + 105,110,32,97,110,100,32,102,114,111,122,101,110,32,109,111, + 100,117,108,101,115,78,41,6,114,199,0,0,0,114,14,0, + 0,0,114,165,0,0,0,114,109,0,0,0,114,141,0,0, + 0,114,151,0,0,0,41,2,114,197,0,0,0,114,198,0, + 0,0,114,10,0,0,0,114,10,0,0,0,114,11,0,0, + 0,218,8,95,105,110,115,116,97,108,108,128,4,0,0,115, + 6,0,0,0,0,2,10,2,12,1,114,200,0,0,0,99, + 0,0,0,0,0,0,0,0,1,0,0,0,4,0,0,0, + 67,0,0,0,115,32,0,0,0,100,1,100,2,108,0,125, + 0,124,0,97,1,124,0,160,2,116,3,106,4,116,5,25, + 0,161,1,1,0,100,2,83,0,41,3,122,57,73,110,115, + 116,97,108,108,32,105,109,112,111,114,116,101,114,115,32,116, + 104,97,116,32,114,101,113,117,105,114,101,32,101,120,116,101, + 114,110,97,108,32,102,105,108,101,115,121,115,116,101,109,32, + 97,99,99,101,115,115,114,19,0,0,0,78,41,6,218,26, + 95,102,114,111,122,101,110,95,105,109,112,111,114,116,108,105, + 98,95,101,120,116,101,114,110,97,108,114,115,0,0,0,114, + 200,0,0,0,114,14,0,0,0,114,79,0,0,0,114,1, + 0,0,0,41,1,114,201,0,0,0,114,10,0,0,0,114, + 10,0,0,0,114,11,0,0,0,218,27,95,105,110,115,116, + 97,108,108,95,101,120,116,101,114,110,97,108,95,105,109,112, + 111,114,116,101,114,115,136,4,0,0,115,6,0,0,0,0, + 3,8,1,4,1,114,202,0,0,0,41,2,78,78,41,1, + 78,41,2,78,114,19,0,0,0,41,4,78,78,114,10,0, + 0,0,114,19,0,0,0,41,51,114,3,0,0,0,114,115, + 0,0,0,114,12,0,0,0,114,16,0,0,0,114,51,0, + 0,0,114,29,0,0,0,114,36,0,0,0,114,17,0,0, + 0,114,18,0,0,0,114,41,0,0,0,114,42,0,0,0, + 114,45,0,0,0,114,57,0,0,0,114,59,0,0,0,114, + 68,0,0,0,114,74,0,0,0,114,77,0,0,0,114,84, + 0,0,0,114,95,0,0,0,114,96,0,0,0,114,102,0, + 0,0,114,78,0,0,0,114,128,0,0,0,114,133,0,0, + 0,114,136,0,0,0,114,91,0,0,0,114,80,0,0,0, + 114,139,0,0,0,114,140,0,0,0,114,81,0,0,0,114, + 141,0,0,0,114,151,0,0,0,114,156,0,0,0,114,162, + 0,0,0,114,164,0,0,0,114,169,0,0,0,114,173,0, + 0,0,90,15,95,69,82,82,95,77,83,71,95,80,82,69, + 70,73,88,114,175,0,0,0,114,178,0,0,0,218,6,111, + 98,106,101,99,116,114,179,0,0,0,114,180,0,0,0,114, + 181,0,0,0,114,185,0,0,0,114,191,0,0,0,114,194, + 0,0,0,114,195,0,0,0,114,199,0,0,0,114,200,0, + 0,0,114,202,0,0,0,114,10,0,0,0,114,10,0,0, + 0,114,10,0,0,0,114,11,0,0,0,218,8,60,109,111, + 100,117,108,101,62,25,0,0,0,115,96,0,0,0,4,0, + 4,2,8,8,8,8,4,2,4,3,16,4,14,68,14,21, + 14,16,8,37,8,17,8,11,14,8,8,11,8,12,8,16, + 8,36,14,27,14,101,16,26,10,45,14,72,8,17,8,17, + 8,24,8,29,8,23,8,15,14,73,14,77,14,13,8,9, + 8,9,10,47,8,16,4,1,8,2,8,27,6,3,8,16, + 10,15,14,37,8,27,10,37,8,7,8,35,8,8, }; From webhook-mailer at python.org Sun Mar 11 04:54:50 2018 From: webhook-mailer at python.org (Serhiy Storchaka) Date: Sun, 11 Mar 2018 08:54:50 -0000 Subject: [Python-checkins] bpo-32925: Optimized iterating and containing test for literal lists (GH-5842) Message-ID: <mailman.43.1520758491.1871.python-checkins@python.org> https://github.com/python/cpython/commit/3f7e9aa2ef215917b9f1521441f67f4ecd33a1bc commit: 3f7e9aa2ef215917b9f1521441f67f4ecd33a1bc branch: master author: Serhiy Storchaka <storchaka at gmail.com> committer: GitHub <noreply at github.com> date: 2018-03-11T10:54:47+02:00 summary: bpo-32925: Optimized iterating and containing test for literal lists (GH-5842) consisting of non-constants: `x in [a, b]` and `for x in [a, b]`. The case of all constant elements already was optimized. files: A Misc/NEWS.d/next/Core and Builtins/2018-02-24-00-07-05.bpo-32925.e-7Ufh.rst M Lib/test/test_peepholer.py M Python/ast_opt.c diff --git a/Lib/test/test_peepholer.py b/Lib/test/test_peepholer.py index c24cf77e7b2d..794d104d5919 100644 --- a/Lib/test/test_peepholer.py +++ b/Lib/test/test_peepholer.py @@ -3,6 +3,19 @@ from test.bytecode_helper import BytecodeTestCase +def count_instr_recursively(f, opname): + count = 0 + for instr in dis.get_instructions(f): + if instr.opname == opname: + count += 1 + if hasattr(f, '__code__'): + f = f.__code__ + for c in f.co_consts: + if hasattr(c, 'co_code'): + count += count_instr_recursively(c, opname) + return count + + class TestTranforms(BytecodeTestCase): def test_unot(self): @@ -311,6 +324,17 @@ def test_constant_folding(self): self.assertFalse(instr.opname.startswith('BINARY_')) self.assertFalse(instr.opname.startswith('BUILD_')) + def test_in_literal_list(self): + def containtest(): + return x in [a, b] + self.assertEqual(count_instr_recursively(containtest, 'BUILD_LIST'), 0) + + def test_iterate_literal_list(self): + def forloop(): + for x in [a, b]: + pass + self.assertEqual(count_instr_recursively(forloop, 'BUILD_LIST'), 0) + class TestBuglets(unittest.TestCase): diff --git a/Misc/NEWS.d/next/Core and Builtins/2018-02-24-00-07-05.bpo-32925.e-7Ufh.rst b/Misc/NEWS.d/next/Core and Builtins/2018-02-24-00-07-05.bpo-32925.e-7Ufh.rst new file mode 100644 index 000000000000..e9443e69e2a2 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2018-02-24-00-07-05.bpo-32925.e-7Ufh.rst @@ -0,0 +1,3 @@ +Optimized iterating and containing test for literal lists consisting of +non-constants: ``x in [a, b]`` and ``for x in [a, b]``. The case of all +constant elements already was optimized. diff --git a/Python/ast_opt.c b/Python/ast_opt.c index 65cf3c126417..a54f98c8c721 100644 --- a/Python/ast_opt.c +++ b/Python/ast_opt.c @@ -369,7 +369,8 @@ fold_subscr(expr_ty node, PyArena *arena, int optimize) } /* Change literal list or set of constants into constant - tuple or frozenset respectively. + tuple or frozenset respectively. Change literal list of + non-constants into tuple. Used for right operand of "in" and "not in" tests and for iterable in "for" loop and comprehensions. */ @@ -378,7 +379,21 @@ fold_iter(expr_ty arg, PyArena *arena, int optimize) { PyObject *newval; if (arg->kind == List_kind) { - newval = make_const_tuple(arg->v.List.elts); + /* First change a list into tuple. */ + asdl_seq *elts = arg->v.List.elts; + Py_ssize_t n = asdl_seq_LEN(elts); + for (Py_ssize_t i = 0; i < n; i++) { + expr_ty e = (expr_ty)asdl_seq_GET(elts, i); + if (e->kind == Starred_kind) { + return 1; + } + } + expr_context_ty ctx = arg->v.List.ctx; + arg->kind = Tuple_kind; + arg->v.Tuple.elts = elts; + arg->v.Tuple.ctx = ctx; + /* Try to create a constant tuple. */ + newval = make_const_tuple(elts); } else if (arg->kind == Set_kind) { newval = make_const_tuple(arg->v.Set.elts); From webhook-mailer at python.org Sun Mar 11 04:56:02 2018 From: webhook-mailer at python.org (Serhiy Storchaka) Date: Sun, 11 Mar 2018 08:56:02 -0000 Subject: [Python-checkins] [2.7] bpo-33026: Fix jumping out of "with" block by setting f_lineno. (GH-6026). (GH-6074) (GH-6076) Message-ID: <mailman.44.1520758564.1871.python-checkins@python.org> https://github.com/python/cpython/commit/3854f5885edc8dc67b1aba82fbb525604fbc625b commit: 3854f5885edc8dc67b1aba82fbb525604fbc625b branch: 2.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Serhiy Storchaka <storchaka at gmail.com> date: 2018-03-11T10:55:59+02:00 summary: [2.7] bpo-33026: Fix jumping out of "with" block by setting f_lineno. (GH-6026). (GH-6074) (GH-6076) (cherry picked from commit 26c9f565d016db21257a60d29ab2c99383dd5ac7) (cherry picked from commit 04aadf23eac51fec2e436c5960c1362bbb7d03de) Co-authored-by: Serhiy Storchaka <storchaka at gmail.com> files: A Misc/NEWS.d/next/Core and Builtins/2018-03-08-09-48-38.bpo-33026.QZA3Ba.rst M Lib/test/test_sys_settrace.py M Objects/frameobject.c diff --git a/Lib/test/test_sys_settrace.py b/Lib/test/test_sys_settrace.py index 7e1d7a131f5a..04b1ac339a49 100644 --- a/Lib/test/test_sys_settrace.py +++ b/Lib/test/test_sys_settrace.py @@ -723,6 +723,34 @@ def test_jump_across_with(output): with tracecontext(output, 4): output.append(5) + @jump_test(4, 5, [1, 3, 5, 6]) + def test_jump_out_of_with_block_within_for_block(output): + output.append(1) + for i in [1]: + with tracecontext(output, 3): + output.append(4) + output.append(5) + output.append(6) + + @jump_test(4, 5, [1, 2, 3, 5, -2, 6]) + def test_jump_out_of_with_block_within_with_block(output): + output.append(1) + with tracecontext(output, 2): + with tracecontext(output, 3): + output.append(4) + output.append(5) + output.append(6) + + @jump_test(5, 6, [2, 4, 6, 7]) + def test_jump_out_of_with_block_within_finally_block(output): + try: + output.append(2) + finally: + with tracecontext(output, 4): + output.append(5) + output.append(6) + output.append(7) + @jump_test(8, 11, [1, 3, 5, 11, 12]) def test_jump_out_of_complex_nested_blocks(output): output.append(1) diff --git a/Misc/NEWS.d/next/Core and Builtins/2018-03-08-09-48-38.bpo-33026.QZA3Ba.rst b/Misc/NEWS.d/next/Core and Builtins/2018-03-08-09-48-38.bpo-33026.QZA3Ba.rst new file mode 100644 index 000000000000..dc166d1e5771 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2018-03-08-09-48-38.bpo-33026.QZA3Ba.rst @@ -0,0 +1 @@ +Fixed jumping out of "with" block by setting f_lineno. diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 2c8fb017492e..bf1c7c52737d 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -340,6 +340,11 @@ frame_setlineno(PyFrameObject *f, PyObject* p_new_lineno) PyObject *v = (*--f->f_stacktop); Py_DECREF(v); } + if (b->b_type == SETUP_WITH) { + /* Pop the exit function. */ + PyObject *v = (*--f->f_stacktop); + Py_DECREF(v); + } } /* Finally set the new f_lineno and f_lasti and return OK. */ From webhook-mailer at python.org Sun Mar 11 05:07:09 2018 From: webhook-mailer at python.org (Serhiy Storchaka) Date: Sun, 11 Mar 2018 09:07:09 -0000 Subject: [Python-checkins] bpo-32970: Improve disassembly of the MAKE_FUNCTION instruction. (GH-5937) Message-ID: <mailman.45.1520759230.1871.python-checkins@python.org> https://github.com/python/cpython/commit/e2732d3e66eba9ec13f9d55c499f2437ead552db commit: e2732d3e66eba9ec13f9d55c499f2437ead552db branch: master author: Serhiy Storchaka <storchaka at gmail.com> committer: GitHub <noreply at github.com> date: 2018-03-11T11:07:06+02:00 summary: bpo-32970: Improve disassembly of the MAKE_FUNCTION instruction. (GH-5937) files: A Misc/NEWS.d/next/Library/2018-02-28-18-39-48.bpo-32970.IPWtbS.rst M Lib/dis.py M Lib/test/test_dis.py diff --git a/Lib/dis.py b/Lib/dis.py index 90ddf4f33603..b2b0003203a4 100644 --- a/Lib/dis.py +++ b/Lib/dis.py @@ -17,6 +17,15 @@ classmethod, staticmethod, type) FORMAT_VALUE = opmap['FORMAT_VALUE'] +FORMAT_VALUE_CONVERTERS = ( + (None, ''), + (str, 'str'), + (repr, 'repr'), + (ascii, 'ascii'), +) +MAKE_FUNCTION = opmap['MAKE_FUNCTION'] +MAKE_FUNCTION_FLAGS = ('defaults', 'kwdefaults', 'annotations', 'closure') + def _try_compile(source, name): """Attempts to compile the given source, first as an expression and @@ -339,12 +348,15 @@ def _get_instructions_bytes(code, varnames=None, names=None, constants=None, elif op in hasfree: argval, argrepr = _get_name_info(arg, cells) elif op == FORMAT_VALUE: - argval = ((None, str, repr, ascii)[arg & 0x3], bool(arg & 0x4)) - argrepr = ('', 'str', 'repr', 'ascii')[arg & 0x3] + argval, argrepr = FORMAT_VALUE_CONVERTERS[arg & 0x3] + argval = (argval, bool(arg & 0x4)) if argval[1]: if argrepr: argrepr += ', ' argrepr += 'with format' + elif op == MAKE_FUNCTION: + argrepr = ', '.join(s for i, s in enumerate(MAKE_FUNCTION_FLAGS) + if arg & (1<<i)) yield Instruction(opname[op], op, arg, argval, argrepr, offset, starts_line, is_jump_target) diff --git a/Lib/test/test_dis.py b/Lib/test/test_dis.py index 08393f734ba5..098367ca55d1 100644 --- a/Lib/test/test_dis.py +++ b/Lib/test/test_dis.py @@ -348,7 +348,7 @@ def foo(x): 2 BUILD_TUPLE 1 4 LOAD_CONST 1 (<code object foo at 0x..., file "%s", line %d>) 6 LOAD_CONST 2 ('_h.<locals>.foo') - 8 MAKE_FUNCTION 8 + 8 MAKE_FUNCTION 8 (closure) 10 STORE_FAST 1 (foo) %3d 12 LOAD_FAST 1 (foo) @@ -365,7 +365,7 @@ def foo(x): 2 BUILD_TUPLE 1 4 LOAD_CONST 1 (<code object <listcomp> at 0x..., file "%s", line %d>) 6 LOAD_CONST 2 ('_h.<locals>.foo.<locals>.<listcomp>') - 8 MAKE_FUNCTION 8 + 8 MAKE_FUNCTION 8 (closure) 10 LOAD_DEREF 1 (y) 12 GET_ITER 14 CALL_FUNCTION 1 @@ -862,7 +862,7 @@ def jumpy(): Instruction(opname='BUILD_TUPLE', opcode=102, arg=2, argval=2, argrepr='', offset=6, starts_line=None, is_jump_target=False), Instruction(opname='LOAD_CONST', opcode=100, arg=3, argval=code_object_f, argrepr=repr(code_object_f), offset=8, starts_line=None, is_jump_target=False), Instruction(opname='LOAD_CONST', opcode=100, arg=4, argval='outer.<locals>.f', argrepr="'outer.<locals>.f'", offset=10, starts_line=None, is_jump_target=False), - Instruction(opname='MAKE_FUNCTION', opcode=132, arg=9, argval=9, argrepr='', offset=12, starts_line=None, is_jump_target=False), + Instruction(opname='MAKE_FUNCTION', opcode=132, arg=9, argval=9, argrepr='defaults, closure', offset=12, starts_line=None, is_jump_target=False), Instruction(opname='STORE_FAST', opcode=125, arg=2, argval='f', argrepr='f', offset=14, starts_line=None, is_jump_target=False), Instruction(opname='LOAD_GLOBAL', opcode=116, arg=0, argval='print', argrepr='print', offset=16, starts_line=7, is_jump_target=False), Instruction(opname='LOAD_DEREF', opcode=136, arg=0, argval='a', argrepr='a', offset=18, starts_line=None, is_jump_target=False), @@ -887,7 +887,7 @@ def jumpy(): Instruction(opname='BUILD_TUPLE', opcode=102, arg=4, argval=4, argrepr='', offset=10, starts_line=None, is_jump_target=False), Instruction(opname='LOAD_CONST', opcode=100, arg=3, argval=code_object_inner, argrepr=repr(code_object_inner), offset=12, starts_line=None, is_jump_target=False), Instruction(opname='LOAD_CONST', opcode=100, arg=4, argval='outer.<locals>.f.<locals>.inner', argrepr="'outer.<locals>.f.<locals>.inner'", offset=14, starts_line=None, is_jump_target=False), - Instruction(opname='MAKE_FUNCTION', opcode=132, arg=9, argval=9, argrepr='', offset=16, starts_line=None, is_jump_target=False), + Instruction(opname='MAKE_FUNCTION', opcode=132, arg=9, argval=9, argrepr='defaults, closure', offset=16, starts_line=None, is_jump_target=False), Instruction(opname='STORE_FAST', opcode=125, arg=2, argval='inner', argrepr='inner', offset=18, starts_line=None, is_jump_target=False), Instruction(opname='LOAD_GLOBAL', opcode=116, arg=0, argval='print', argrepr='print', offset=20, starts_line=5, is_jump_target=False), Instruction(opname='LOAD_DEREF', opcode=136, arg=2, argval='a', argrepr='a', offset=22, starts_line=None, is_jump_target=False), diff --git a/Misc/NEWS.d/next/Library/2018-02-28-18-39-48.bpo-32970.IPWtbS.rst b/Misc/NEWS.d/next/Library/2018-02-28-18-39-48.bpo-32970.IPWtbS.rst new file mode 100644 index 000000000000..e97c16df5a0d --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-02-28-18-39-48.bpo-32970.IPWtbS.rst @@ -0,0 +1 @@ +Improved disassembly of the MAKE_FUNCTION instruction. From solipsis at pitrou.net Sun Mar 11 05:11:37 2018 From: solipsis at pitrou.net (solipsis at pitrou.net) Date: Sun, 11 Mar 2018 09:11:37 +0000 Subject: [Python-checkins] Daily reference leaks (4243df51fe43): sum=10 Message-ID: <20180311091137.1.62B4F46094D11A45@psf.io> results for 4243df51fe43 on branch "default" -------------------------------------------- test_collections leaked [7, -7, 8] memory blocks, sum=8 test_functools leaked [0, 3, 1] memory blocks, sum=4 test_multiprocessing_forkserver leaked [0, 0, -2] memory blocks, sum=-2 Command line was: ['./python', '-m', 'test.regrtest', '-uall', '-R', '3:3:/home/psf-users/antoine/refleaks/reflogCuZMFW', '--timeout', '7200'] From webhook-mailer at python.org Sun Mar 11 05:45:19 2018 From: webhook-mailer at python.org (Xiang Zhang) Date: Sun, 11 Mar 2018 09:45:19 -0000 Subject: [Python-checkins] Drop confusing commented out code in pystrtod.c (GH-6072) Message-ID: <mailman.46.1520761520.1871.python-checkins@python.org> https://github.com/python/cpython/commit/9fb84157595a385f15799e5d0729c1e1b0ba9d38 commit: 9fb84157595a385f15799e5d0729c1e1b0ba9d38 branch: master author: Siddhesh Poyarekar <siddhesh.poyarekar at gmail.com> committer: Xiang Zhang <angwerzx at 126.com> date: 2018-03-11T17:45:10+08:00 summary: Drop confusing commented out code in pystrtod.c (GH-6072) files: M Python/pystrtod.c diff --git a/Python/pystrtod.c b/Python/pystrtod.c index 601f7c691edf..3546d44c8424 100644 --- a/Python/pystrtod.c +++ b/Python/pystrtod.c @@ -1060,8 +1060,6 @@ format_float_short(double d, char format_code, else { /* shouldn't get here: Gay's code should always return something starting with a digit, an 'I', or 'N' */ - /* strncpy(p, "ERR", 3); - p += 3; */ Py_UNREACHABLE(); } goto exit; From webhook-mailer at python.org Sun Mar 11 14:21:41 2018 From: webhook-mailer at python.org (Antoine Pitrou) Date: Sun, 11 Mar 2018 18:21:41 -0000 Subject: [Python-checkins] bpo-31804: Fix multiprocessing.Process with broken standard streams (#6079) Message-ID: <mailman.47.1520792503.1871.python-checkins@python.org> https://github.com/python/cpython/commit/e756f66c83786ee82f5f7d45931ae50a6931dd7f commit: e756f66c83786ee82f5f7d45931ae50a6931dd7f branch: master author: Antoine Pitrou <pitrou at free.fr> committer: GitHub <noreply at github.com> date: 2018-03-11T19:21:38+01:00 summary: bpo-31804: Fix multiprocessing.Process with broken standard streams (#6079) In some conditions the standard streams will be None or closed in the child process (for example if using "pythonw" instead of "python" on Windows). Avoid failing with a non-0 exit code in those conditions. Report and initial patch by poxthegreat. files: A Misc/NEWS.d/next/Library/2018-03-11-19-03-52.bpo-31804.i8KUMp.rst M Lib/multiprocessing/popen_fork.py M Lib/multiprocessing/process.py M Lib/multiprocessing/util.py M Lib/test/_test_multiprocessing.py diff --git a/Lib/multiprocessing/popen_fork.py b/Lib/multiprocessing/popen_fork.py index b0fc01396eb2..008b97b36628 100644 --- a/Lib/multiprocessing/popen_fork.py +++ b/Lib/multiprocessing/popen_fork.py @@ -14,14 +14,7 @@ class Popen(object): method = 'fork' def __init__(self, process_obj): - try: - sys.stdout.flush() - except (AttributeError, ValueError): - pass - try: - sys.stderr.flush() - except (AttributeError, ValueError): - pass + util._flush_std_streams() self.returncode = None self.finalizer = None self._launch(process_obj) diff --git a/Lib/multiprocessing/process.py b/Lib/multiprocessing/process.py index 8fff3e105ead..cd592d0bdf09 100644 --- a/Lib/multiprocessing/process.py +++ b/Lib/multiprocessing/process.py @@ -314,8 +314,7 @@ def _bootstrap(self): finally: threading._shutdown() util.info('process exiting with exitcode %d' % exitcode) - sys.stdout.flush() - sys.stderr.flush() + util._flush_std_streams() return exitcode diff --git a/Lib/multiprocessing/util.py b/Lib/multiprocessing/util.py index f0827f0a9693..0c4eb2473273 100644 --- a/Lib/multiprocessing/util.py +++ b/Lib/multiprocessing/util.py @@ -391,6 +391,20 @@ def _close_stdin(): except (OSError, ValueError): pass +# +# Flush standard streams, if any +# + +def _flush_std_streams(): + try: + sys.stdout.flush() + except (AttributeError, ValueError): + pass + try: + sys.stderr.flush() + except (AttributeError, ValueError): + pass + # # Start a program with only specified fds kept open # diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index 1e497a572a76..940fe584e752 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -584,10 +584,19 @@ def test_wait_for_threads(self): self.assertTrue(evt.is_set()) @classmethod - def _test_error_on_stdio_flush(self, evt): + def _test_error_on_stdio_flush(self, evt, break_std_streams={}): + for stream_name, action in break_std_streams.items(): + if action == 'close': + stream = io.StringIO() + stream.close() + else: + assert action == 'remove' + stream = None + setattr(sys, stream_name, None) evt.set() - def test_error_on_stdio_flush(self): + def test_error_on_stdio_flush_1(self): + # Check that Process works with broken standard streams streams = [io.StringIO(), None] streams[0].close() for stream_name in ('stdout', 'stderr'): @@ -601,6 +610,24 @@ def test_error_on_stdio_flush(self): proc.start() proc.join() self.assertTrue(evt.is_set()) + self.assertEqual(proc.exitcode, 0) + finally: + setattr(sys, stream_name, old_stream) + + def test_error_on_stdio_flush_2(self): + # Same as test_error_on_stdio_flush_1(), but standard streams are + # broken by the child process + for stream_name in ('stdout', 'stderr'): + for action in ('close', 'remove'): + old_stream = getattr(sys, stream_name) + try: + evt = self.Event() + proc = self.Process(target=self._test_error_on_stdio_flush, + args=(evt, {stream_name: action})) + proc.start() + proc.join() + self.assertTrue(evt.is_set()) + self.assertEqual(proc.exitcode, 0) finally: setattr(sys, stream_name, old_stream) diff --git a/Misc/NEWS.d/next/Library/2018-03-11-19-03-52.bpo-31804.i8KUMp.rst b/Misc/NEWS.d/next/Library/2018-03-11-19-03-52.bpo-31804.i8KUMp.rst new file mode 100644 index 000000000000..7fcede297aca --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-03-11-19-03-52.bpo-31804.i8KUMp.rst @@ -0,0 +1,2 @@ +Avoid failing in multiprocessing.Process if the standard streams are closed +or None at exit. From webhook-mailer at python.org Sun Mar 11 14:29:08 2018 From: webhook-mailer at python.org (larryhastings) Date: Sun, 11 Mar 2018 18:29:08 -0000 Subject: [Python-checkins] [3.5] bpo-32981: Fix catastrophic backtracking vulns (GH-5955) (#6034) Message-ID: <mailman.48.1520792951.1871.python-checkins@python.org> https://github.com/python/cpython/commit/937ac1fe069a4dc8471dff205f553d82e724015b commit: 937ac1fe069a4dc8471dff205f553d82e724015b branch: 3.5 author: Ned Deily <nad at python.org> committer: larryhastings <larry at hastings.org> date: 2018-03-11T18:29:05Z summary: [3.5] bpo-32981: Fix catastrophic backtracking vulns (GH-5955) (#6034) * Prevent low-grade poplib REDOS (CVE-2018-1060) The regex to test a mail server's timestamp is susceptible to catastrophic backtracking on long evil responses from the server. Happily, the maximum length of malicious inputs is 2K thanks to a limit introduced in the fix for CVE-2013-1752. A 2KB evil response from the mail server would result in small slowdowns (milliseconds vs. microseconds) accumulated over many apop calls. This is a potential DOS vector via accumulated slowdowns. Replace it with a similar non-vulnerable regex. The new regex is RFC compliant. The old regex was non-compliant in edge cases. * Prevent difflib REDOS (CVE-2018-1061) The default regex for IS_LINE_JUNK is susceptible to catastrophic backtracking. This is a potential DOS vector. Replace it with an equivalent non-vulnerable regex. Also introduce unit and REDOS tests for difflib. Co-authored-by: Tim Peters <tim.peters at gmail.com> Co-authored-by: Christian Heimes <christian at python.org>. (cherry picked from commit 0e6c8ee2358a2e23117501826c008842acb835ac) files: A Misc/NEWS.d/next/Security/2018-03-02-10-24-52.bpo-32981.O_qDyj.rst M Lib/difflib.py M Lib/poplib.py M Lib/test/test_difflib.py M Lib/test/test_poplib.py M Misc/ACKS diff --git a/Lib/difflib.py b/Lib/difflib.py index 076bbac01dee..b4ec33505644 100644 --- a/Lib/difflib.py +++ b/Lib/difflib.py @@ -1083,7 +1083,7 @@ def _qformat(self, aline, bline, atags, btags): import re -def IS_LINE_JUNK(line, pat=re.compile(r"\s*#?\s*$").match): +def IS_LINE_JUNK(line, pat=re.compile(r"\s*(?:#\s*)?$").match): r""" Return 1 for ignorable line: iff `line` is blank or contains a single '#'. diff --git a/Lib/poplib.py b/Lib/poplib.py index 516b6f060d28..2437ea0e2717 100644 --- a/Lib/poplib.py +++ b/Lib/poplib.py @@ -308,7 +308,7 @@ def rpop(self, user): return self._shortcmd('RPOP %s' % user) - timestamp = re.compile(br'\+OK.*(<[^>]+>)') + timestamp = re.compile(br'\+OK.[^<]*(<.*>)') def apop(self, user, password): """Authorisation diff --git a/Lib/test/test_difflib.py b/Lib/test/test_difflib.py index ab9debf8e252..b6c8a7dd5bed 100644 --- a/Lib/test/test_difflib.py +++ b/Lib/test/test_difflib.py @@ -466,13 +466,33 @@ def _assert_type_error(self, msg, generator, *args): list(generator(*args)) self.assertEqual(msg, str(ctx.exception)) +class TestJunkAPIs(unittest.TestCase): + def test_is_line_junk_true(self): + for line in ['#', ' ', ' #', '# ', ' # ', '']: + self.assertTrue(difflib.IS_LINE_JUNK(line), repr(line)) + + def test_is_line_junk_false(self): + for line in ['##', ' ##', '## ', 'abc ', 'abc #', 'Mr. Moose is up!']: + self.assertFalse(difflib.IS_LINE_JUNK(line), repr(line)) + + def test_is_line_junk_REDOS(self): + evil_input = ('\t' * 1000000) + '##' + self.assertFalse(difflib.IS_LINE_JUNK(evil_input)) + + def test_is_character_junk_true(self): + for char in [' ', '\t']: + self.assertTrue(difflib.IS_CHARACTER_JUNK(char), repr(char)) + + def test_is_character_junk_false(self): + for char in ['a', '#', '\n', '\f', '\r', '\v']: + self.assertFalse(difflib.IS_CHARACTER_JUNK(char), repr(char)) def test_main(): difflib.HtmlDiff._default_prefix = 0 Doctests = doctest.DocTestSuite(difflib) run_unittest( TestWithAscii, TestAutojunk, TestSFpatches, TestSFbugs, - TestOutputFormat, TestBytes, Doctests) + TestOutputFormat, TestBytes, TestJunkAPIs, Doctests) if __name__ == '__main__': test_main() diff --git a/Lib/test/test_poplib.py b/Lib/test/test_poplib.py index bceeb93ad14a..799e40365214 100644 --- a/Lib/test/test_poplib.py +++ b/Lib/test/test_poplib.py @@ -300,9 +300,19 @@ def test_noop(self): def test_rpop(self): self.assertOK(self.client.rpop('foo')) - def test_apop(self): + def test_apop_normal(self): self.assertOK(self.client.apop('foo', 'dummypassword')) + def test_apop_REDOS(self): + # Replace welcome with very long evil welcome. + # NB The upper bound on welcome length is currently 2048. + # At this length, evil input makes each apop call take + # on the order of milliseconds instead of microseconds. + evil_welcome = b'+OK' + (b'<' * 1000000) + with test_support.swap_attr(self.client, 'welcome', evil_welcome): + # The evil welcome is invalid, so apop should throw. + self.assertRaises(poplib.error_proto, self.client.apop, 'a', 'kb') + def test_top(self): expected = (b'+OK 116 bytes', [b'From: postmaster at python.org', b'Content-Type: text/plain', diff --git a/Misc/ACKS b/Misc/ACKS index 1a35aad66ce7..72c5d740bdd2 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -341,6 +341,7 @@ Kushal Das Jonathan Dasteel Pierre-Yves David A. Jesse Jiryu Davis +Jamie (James C.) Davis Merlijn van Deen John DeGood Ned Deily diff --git a/Misc/NEWS.d/next/Security/2018-03-02-10-24-52.bpo-32981.O_qDyj.rst b/Misc/NEWS.d/next/Security/2018-03-02-10-24-52.bpo-32981.O_qDyj.rst new file mode 100644 index 000000000000..9ebabb44f91e --- /dev/null +++ b/Misc/NEWS.d/next/Security/2018-03-02-10-24-52.bpo-32981.O_qDyj.rst @@ -0,0 +1,4 @@ +Regexes in difflib and poplib were vulnerable to catastrophic backtracking. +These regexes formed potential DOS vectors (REDOS). They have been +refactored. This resolves CVE-2018-1060 and CVE-2018-1061. +Patch by Jamie Davis. From webhook-mailer at python.org Sun Mar 11 14:42:40 2018 From: webhook-mailer at python.org (Antoine Pitrou) Date: Sun, 11 Mar 2018 18:42:40 -0000 Subject: [Python-checkins] bpo-31804: Fix multiprocessing.Process with broken standard streams (GH-6079) (GH-6080) Message-ID: <mailman.49.1520793762.1871.python-checkins@python.org> https://github.com/python/cpython/commit/ff5d21331ec6cefec6ba5b78d256d8dbcd67a069 commit: ff5d21331ec6cefec6ba5b78d256d8dbcd67a069 branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Antoine Pitrou <pitrou at free.fr> date: 2018-03-11T19:42:37+01:00 summary: bpo-31804: Fix multiprocessing.Process with broken standard streams (GH-6079) (GH-6080) In some conditions the standard streams will be None or closed in the child process (for example if using "pythonw" instead of "python" on Windows). Avoid failing with a non-0 exit code in those conditions. Report and initial patch by poxthegreat. (cherry picked from commit e756f66c83786ee82f5f7d45931ae50a6931dd7f) Co-authored-by: Antoine Pitrou <pitrou at free.fr> files: A Misc/NEWS.d/next/Library/2018-03-11-19-03-52.bpo-31804.i8KUMp.rst M Lib/multiprocessing/popen_fork.py M Lib/multiprocessing/process.py M Lib/multiprocessing/util.py M Lib/test/_test_multiprocessing.py diff --git a/Lib/multiprocessing/popen_fork.py b/Lib/multiprocessing/popen_fork.py index b0fc01396eb2..008b97b36628 100644 --- a/Lib/multiprocessing/popen_fork.py +++ b/Lib/multiprocessing/popen_fork.py @@ -14,14 +14,7 @@ class Popen(object): method = 'fork' def __init__(self, process_obj): - try: - sys.stdout.flush() - except (AttributeError, ValueError): - pass - try: - sys.stderr.flush() - except (AttributeError, ValueError): - pass + util._flush_std_streams() self.returncode = None self.finalizer = None self._launch(process_obj) diff --git a/Lib/multiprocessing/process.py b/Lib/multiprocessing/process.py index 8fff3e105ead..cd592d0bdf09 100644 --- a/Lib/multiprocessing/process.py +++ b/Lib/multiprocessing/process.py @@ -314,8 +314,7 @@ def _bootstrap(self): finally: threading._shutdown() util.info('process exiting with exitcode %d' % exitcode) - sys.stdout.flush() - sys.stderr.flush() + util._flush_std_streams() return exitcode diff --git a/Lib/multiprocessing/util.py b/Lib/multiprocessing/util.py index f0827f0a9693..0c4eb2473273 100644 --- a/Lib/multiprocessing/util.py +++ b/Lib/multiprocessing/util.py @@ -391,6 +391,20 @@ def _close_stdin(): except (OSError, ValueError): pass +# +# Flush standard streams, if any +# + +def _flush_std_streams(): + try: + sys.stdout.flush() + except (AttributeError, ValueError): + pass + try: + sys.stderr.flush() + except (AttributeError, ValueError): + pass + # # Start a program with only specified fds kept open # diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index 1e497a572a76..940fe584e752 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -584,10 +584,19 @@ def test_wait_for_threads(self): self.assertTrue(evt.is_set()) @classmethod - def _test_error_on_stdio_flush(self, evt): + def _test_error_on_stdio_flush(self, evt, break_std_streams={}): + for stream_name, action in break_std_streams.items(): + if action == 'close': + stream = io.StringIO() + stream.close() + else: + assert action == 'remove' + stream = None + setattr(sys, stream_name, None) evt.set() - def test_error_on_stdio_flush(self): + def test_error_on_stdio_flush_1(self): + # Check that Process works with broken standard streams streams = [io.StringIO(), None] streams[0].close() for stream_name in ('stdout', 'stderr'): @@ -601,6 +610,24 @@ def test_error_on_stdio_flush(self): proc.start() proc.join() self.assertTrue(evt.is_set()) + self.assertEqual(proc.exitcode, 0) + finally: + setattr(sys, stream_name, old_stream) + + def test_error_on_stdio_flush_2(self): + # Same as test_error_on_stdio_flush_1(), but standard streams are + # broken by the child process + for stream_name in ('stdout', 'stderr'): + for action in ('close', 'remove'): + old_stream = getattr(sys, stream_name) + try: + evt = self.Event() + proc = self.Process(target=self._test_error_on_stdio_flush, + args=(evt, {stream_name: action})) + proc.start() + proc.join() + self.assertTrue(evt.is_set()) + self.assertEqual(proc.exitcode, 0) finally: setattr(sys, stream_name, old_stream) diff --git a/Misc/NEWS.d/next/Library/2018-03-11-19-03-52.bpo-31804.i8KUMp.rst b/Misc/NEWS.d/next/Library/2018-03-11-19-03-52.bpo-31804.i8KUMp.rst new file mode 100644 index 000000000000..7fcede297aca --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-03-11-19-03-52.bpo-31804.i8KUMp.rst @@ -0,0 +1,2 @@ +Avoid failing in multiprocessing.Process if the standard streams are closed +or None at exit. From webhook-mailer at python.org Sun Mar 11 15:09:23 2018 From: webhook-mailer at python.org (Antoine Pitrou) Date: Sun, 11 Mar 2018 19:09:23 -0000 Subject: [Python-checkins] [3.6] bpo-31804: Fix multiprocessing.Process with broken standard streams (GH-6079) (GH-6081) Message-ID: <mailman.50.1520795364.1871.python-checkins@python.org> https://github.com/python/cpython/commit/069b8d20be8018fbd49ed5aaf64c4caba311e48f commit: 069b8d20be8018fbd49ed5aaf64c4caba311e48f branch: 3.6 author: Antoine Pitrou <pitrou at free.fr> committer: GitHub <noreply at github.com> date: 2018-03-11T20:09:20+01:00 summary: [3.6] bpo-31804: Fix multiprocessing.Process with broken standard streams (GH-6079) (GH-6081) In some conditions the standard streams will be None or closed in the child process (for example if using "pythonw" instead of "python" on Windows). Avoid failing with a non-0 exit code in those conditions. Report and initial patch by poxthegreat.. (cherry picked from commit e756f66c83786ee82f5f7d45931ae50a6931dd7f) files: A Misc/NEWS.d/next/Library/2018-03-11-19-03-52.bpo-31804.i8KUMp.rst M Lib/multiprocessing/popen_fork.py M Lib/multiprocessing/process.py M Lib/multiprocessing/util.py M Lib/test/_test_multiprocessing.py diff --git a/Lib/multiprocessing/popen_fork.py b/Lib/multiprocessing/popen_fork.py index 5d0fa569f12e..6396db45b136 100644 --- a/Lib/multiprocessing/popen_fork.py +++ b/Lib/multiprocessing/popen_fork.py @@ -14,14 +14,7 @@ class Popen(object): method = 'fork' def __init__(self, process_obj): - try: - sys.stdout.flush() - except (AttributeError, ValueError): - pass - try: - sys.stderr.flush() - except (AttributeError, ValueError): - pass + util._flush_std_streams() self.returncode = None self._launch(process_obj) diff --git a/Lib/multiprocessing/process.py b/Lib/multiprocessing/process.py index 1d26b5e521e7..c6157b6046a0 100644 --- a/Lib/multiprocessing/process.py +++ b/Lib/multiprocessing/process.py @@ -274,8 +274,7 @@ def _bootstrap(self): traceback.print_exc() finally: util.info('process exiting with exitcode %d' % exitcode) - sys.stdout.flush() - sys.stderr.flush() + util._flush_std_streams() return exitcode diff --git a/Lib/multiprocessing/util.py b/Lib/multiprocessing/util.py index b490caa7e643..24573764fab1 100644 --- a/Lib/multiprocessing/util.py +++ b/Lib/multiprocessing/util.py @@ -388,6 +388,20 @@ def _close_stdin(): except (OSError, ValueError): pass +# +# Flush standard streams, if any +# + +def _flush_std_streams(): + try: + sys.stdout.flush() + except (AttributeError, ValueError): + pass + try: + sys.stderr.flush() + except (AttributeError, ValueError): + pass + # # Start a program with only specified fds kept open # diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index e4d60f880426..dd0a9d7a862a 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -427,10 +427,19 @@ def test_lose_target_ref(self): close_queue(q) @classmethod - def _test_error_on_stdio_flush(self, evt): + def _test_error_on_stdio_flush(self, evt, break_std_streams={}): + for stream_name, action in break_std_streams.items(): + if action == 'close': + stream = io.StringIO() + stream.close() + else: + assert action == 'remove' + stream = None + setattr(sys, stream_name, None) evt.set() - def test_error_on_stdio_flush(self): + def test_error_on_stdio_flush_1(self): + # Check that Process works with broken standard streams streams = [io.StringIO(), None] streams[0].close() for stream_name in ('stdout', 'stderr'): @@ -444,6 +453,24 @@ def test_error_on_stdio_flush(self): proc.start() proc.join() self.assertTrue(evt.is_set()) + self.assertEqual(proc.exitcode, 0) + finally: + setattr(sys, stream_name, old_stream) + + def test_error_on_stdio_flush_2(self): + # Same as test_error_on_stdio_flush_1(), but standard streams are + # broken by the child process + for stream_name in ('stdout', 'stderr'): + for action in ('close', 'remove'): + old_stream = getattr(sys, stream_name) + try: + evt = self.Event() + proc = self.Process(target=self._test_error_on_stdio_flush, + args=(evt, {stream_name: action})) + proc.start() + proc.join() + self.assertTrue(evt.is_set()) + self.assertEqual(proc.exitcode, 0) finally: setattr(sys, stream_name, old_stream) diff --git a/Misc/NEWS.d/next/Library/2018-03-11-19-03-52.bpo-31804.i8KUMp.rst b/Misc/NEWS.d/next/Library/2018-03-11-19-03-52.bpo-31804.i8KUMp.rst new file mode 100644 index 000000000000..7fcede297aca --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-03-11-19-03-52.bpo-31804.i8KUMp.rst @@ -0,0 +1,2 @@ +Avoid failing in multiprocessing.Process if the standard streams are closed +or None at exit. From webhook-mailer at python.org Sun Mar 11 19:39:25 2018 From: webhook-mailer at python.org (Antoine Pitrou) Date: Sun, 11 Mar 2018 23:39:25 -0000 Subject: [Python-checkins] bpo-33021: Release the GIL during fstat() calls (GH-6019) Message-ID: <mailman.51.1520811567.1871.python-checkins@python.org> https://github.com/python/cpython/commit/4484f9dca9149da135bbae035f10a50d20d1cbbb commit: 4484f9dca9149da135bbae035f10a50d20d1cbbb branch: master author: Nir Soffer <nirsof at gmail.com> committer: Antoine Pitrou <pitrou at free.fr> date: 2018-03-12T00:39:22+01:00 summary: bpo-33021: Release the GIL during fstat() calls (GH-6019) fstat may block for long time if the file descriptor is on a non-responsive NFS server, hanging all threads. Most fstat() calls are handled by _Py_fstat(), releasing the GIL internally, but but _Py_fstat_noraise() does not release the GIL, and most calls release the GIL explicitly around it. This patch fixes last 2 calls to _Py_fstat_no_raise(), avoiding hangs when calling: - mmap.mmap() - os.urandom() - random.seed() files: A Misc/NEWS.d/next/Library/2018-03-12-00-27-56.bpo-33021.m19B9T.rst M Modules/mmapmodule.c M Python/bootstrap_hash.c diff --git a/Misc/NEWS.d/next/Library/2018-03-12-00-27-56.bpo-33021.m19B9T.rst b/Misc/NEWS.d/next/Library/2018-03-12-00-27-56.bpo-33021.m19B9T.rst new file mode 100644 index 000000000000..50dfafe600d8 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-03-12-00-27-56.bpo-33021.m19B9T.rst @@ -0,0 +1,2 @@ +Release the GIL during fstat() calls, avoiding hang of all threads when +calling mmap.mmap(), os.urandom(), and random.seed(). Patch by Nir Soffer. diff --git a/Modules/mmapmodule.c b/Modules/mmapmodule.c index 6cf454573e93..6abdc71bbb82 100644 --- a/Modules/mmapmodule.c +++ b/Modules/mmapmodule.c @@ -1050,6 +1050,7 @@ static PyObject * new_mmap_object(PyTypeObject *type, PyObject *args, PyObject *kwdict) { struct _Py_stat_struct status; + int fstat_result; mmap_object *m_obj; Py_ssize_t map_size; off_t offset = 0; @@ -1115,8 +1116,14 @@ new_mmap_object(PyTypeObject *type, PyObject *args, PyObject *kwdict) if (fd != -1) (void)fcntl(fd, F_FULLFSYNC); #endif - if (fd != -1 && _Py_fstat_noraise(fd, &status) == 0 - && S_ISREG(status.st_mode)) { + + if (fd != -1) { + Py_BEGIN_ALLOW_THREADS + fstat_result = _Py_fstat_noraise(fd, &status); + Py_END_ALLOW_THREADS + } + + if (fd != -1 && fstat_result == 0 && S_ISREG(status.st_mode)) { if (map_size == 0) { if (status.st_size == 0) { PyErr_SetString(PyExc_ValueError, diff --git a/Python/bootstrap_hash.c b/Python/bootstrap_hash.c index 9fd5cfb200ea..073b2ea8dbda 100644 --- a/Python/bootstrap_hash.c +++ b/Python/bootstrap_hash.c @@ -301,10 +301,15 @@ dev_urandom(char *buffer, Py_ssize_t size, int raise) if (raise) { struct _Py_stat_struct st; + int fstat_result; if (urandom_cache.fd >= 0) { + Py_BEGIN_ALLOW_THREADS + fstat_result = _Py_fstat_noraise(urandom_cache.fd, &st); + Py_END_ALLOW_THREADS + /* Does the fd point to the same thing as before? (issue #21207) */ - if (_Py_fstat_noraise(urandom_cache.fd, &st) + if (fstat_result || st.st_dev != urandom_cache.st_dev || st.st_ino != urandom_cache.st_ino) { /* Something changed: forget the cached fd (but don't close it, From solipsis at pitrou.net Mon Mar 12 05:10:43 2018 From: solipsis at pitrou.net (solipsis at pitrou.net) Date: Mon, 12 Mar 2018 09:10:43 +0000 Subject: [Python-checkins] Daily reference leaks (4243df51fe43): sum=1 Message-ID: <20180312091043.1.276073873144E509@psf.io> results for 4243df51fe43 on branch "default" -------------------------------------------- test_collections leaked [-7, 1, 0] memory blocks, sum=-6 test_functools leaked [0, 3, 1] memory blocks, sum=4 test_multiprocessing_fork leaked [2, 0, 0] memory blocks, sum=2 test_multiprocessing_spawn leaked [2, 0, -1] memory blocks, sum=1 Command line was: ['./python', '-m', 'test.regrtest', '-uall', '-R', '3:3:/home/psf-users/antoine/refleaks/reflogMVru7n', '--timeout', '7200'] From webhook-mailer at python.org Mon Mar 12 09:42:44 2018 From: webhook-mailer at python.org (Antoine Pitrou) Date: Mon, 12 Mar 2018 13:42:44 -0000 Subject: [Python-checkins] signal: add strsignal() (#6017) Message-ID: <mailman.52.1520862165.1871.python-checkins@python.org> https://github.com/python/cpython/commit/5d2a27de625caf2fd5b763d8a8463790ccd954be commit: 5d2a27de625caf2fd5b763d8a8463790ccd954be branch: master author: Antoine Pietri <antoine.pietri1 at gmail.com> committer: Antoine Pitrou <pitrou at free.fr> date: 2018-03-12T14:42:34+01:00 summary: signal: add strsignal() (#6017) Co-authored-by: Vajrasky Kok <sky.kok at speaklikeaking.com> files: A Misc/NEWS.d/next/Library/2018-03-07-19-37-00.bpo-22674.2sIMmM.rst M Doc/library/signal.rst M Lib/test/test_signal.py M Modules/clinic/signalmodule.c.h M Modules/signalmodule.c diff --git a/Doc/library/signal.rst b/Doc/library/signal.rst index 67eaa2c63813..3f17e08f0f95 100644 --- a/Doc/library/signal.rst +++ b/Doc/library/signal.rst @@ -207,6 +207,15 @@ The :mod:`signal` module defines the following functions: installed from Python. +.. function:: strsignal(signalnum) + + Return the system description of the signal *signalnum*, such as + "Interrupt", "Segmentation fault", etc. Returns :const:`None` if the signal + is not recognized. + + .. versionadded:: 3.8 + + .. function:: pause() Cause the process to sleep until a signal is received; the appropriate handler diff --git a/Lib/test/test_signal.py b/Lib/test/test_signal.py index 48b7a392e50b..fbb12a5b67bf 100644 --- a/Lib/test/test_signal.py +++ b/Lib/test/test_signal.py @@ -43,6 +43,8 @@ def test_out_of_range_signal_number_raises_error(self): self.assertRaises(ValueError, signal.signal, 4242, self.trivial_signal_handler) + self.assertRaises(ValueError, signal.strsignal, 4242) + def test_setting_signal_handler_to_none_raises_error(self): self.assertRaises(TypeError, signal.signal, signal.SIGUSR1, None) @@ -55,6 +57,10 @@ def test_getsignal(self): signal.signal(signal.SIGHUP, hup) self.assertEqual(signal.getsignal(signal.SIGHUP), hup) + def test_strsignal(self): + self.assertEqual(signal.strsignal(signal.SIGINT), "Interrupt") + self.assertEqual(signal.strsignal(signal.SIGTERM), "Terminated") + # Issue 3864, unknown if this affects earlier versions of freebsd also def test_interprocess_signal(self): dirname = os.path.dirname(__file__) diff --git a/Misc/NEWS.d/next/Library/2018-03-07-19-37-00.bpo-22674.2sIMmM.rst b/Misc/NEWS.d/next/Library/2018-03-07-19-37-00.bpo-22674.2sIMmM.rst new file mode 100644 index 000000000000..a9af5da46ef1 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-03-07-19-37-00.bpo-22674.2sIMmM.rst @@ -0,0 +1,2 @@ +Add the strsignal() function in the signal module that returns the system +description of the given signal, as returned by strsignal(3). diff --git a/Modules/clinic/signalmodule.c.h b/Modules/clinic/signalmodule.c.h index dc3aadf878b2..1c439716c498 100644 --- a/Modules/clinic/signalmodule.c.h +++ b/Modules/clinic/signalmodule.c.h @@ -129,6 +129,36 @@ signal_getsignal(PyObject *module, PyObject *arg) return return_value; } +PyDoc_STRVAR(signal_strsignal__doc__, +"strsignal($module, signalnum, /)\n" +"--\n" +"\n" +"Return the system description of the given signal.\n" +"\n" +"The return values can be such as \"Interrupt\", \"Segmentation fault\", etc.\n" +"Returns None if the signal is not recognized."); + +#define SIGNAL_STRSIGNAL_METHODDEF \ + {"strsignal", (PyCFunction)signal_strsignal, METH_O, signal_strsignal__doc__}, + +static PyObject * +signal_strsignal_impl(PyObject *module, int signalnum); + +static PyObject * +signal_strsignal(PyObject *module, PyObject *arg) +{ + PyObject *return_value = NULL; + int signalnum; + + if (!PyArg_Parse(arg, "i:strsignal", &signalnum)) { + goto exit; + } + return_value = signal_strsignal_impl(module, signalnum); + +exit: + return return_value; +} + #if defined(HAVE_SIGINTERRUPT) PyDoc_STRVAR(signal_siginterrupt__doc__, @@ -440,4 +470,4 @@ signal_pthread_kill(PyObject *module, PyObject *const *args, Py_ssize_t nargs) #ifndef SIGNAL_PTHREAD_KILL_METHODDEF #define SIGNAL_PTHREAD_KILL_METHODDEF #endif /* !defined(SIGNAL_PTHREAD_KILL_METHODDEF) */ -/*[clinic end generated code: output=36132f4189381fe0 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=7b41486acf93aa8e input=a9049054013a1b77]*/ diff --git a/Modules/signalmodule.c b/Modules/signalmodule.c index b553eedc0f2c..791616014384 100644 --- a/Modules/signalmodule.c +++ b/Modules/signalmodule.c @@ -504,6 +504,66 @@ signal_getsignal_impl(PyObject *module, int signalnum) } } + +/*[clinic input] +signal.strsignal + + signalnum: int + / + +Return the system description of the given signal. + +The return values can be such as "Interrupt", "Segmentation fault", etc. +Returns None if the signal is not recognized. +[clinic start generated code]*/ + +static PyObject * +signal_strsignal_impl(PyObject *module, int signalnum) +/*[clinic end generated code: output=44e12e1e3b666261 input=b77914b03f856c74]*/ +{ + char *res; + + if (signalnum < 1 || signalnum >= NSIG) { + PyErr_SetString(PyExc_ValueError, + "signal number out of range"); + return NULL; + } + +#ifdef MS_WINDOWS + /* Custom redefinition of POSIX signals allowed on Windows */ + switch (signalnum) { + case SIGINT: + res = "Interrupt"; + break; + case SIGILL: + res = "Illegal instruction"; + break; + case SIGABRT: + res = "Aborted"; + break; + case SIGFPE: + res = "Floating point exception"; + break; + case SIGSEGV: + res = "Segmentation fault"; + break; + case SIGTERM: + res = "Terminated"; + break; + default: + Py_RETURN_NONE; + } +#else + errno = 0; + res = strsignal(signalnum); + + if (errno || res == NULL || strstr(res, "Unknown signal") != NULL) + Py_RETURN_NONE; +#endif + + return Py_BuildValue("s", res); +} + #ifdef HAVE_SIGINTERRUPT /*[clinic input] @@ -1152,6 +1212,7 @@ static PyMethodDef signal_methods[] = { SIGNAL_SETITIMER_METHODDEF SIGNAL_GETITIMER_METHODDEF SIGNAL_SIGNAL_METHODDEF + SIGNAL_STRSIGNAL_METHODDEF SIGNAL_GETSIGNAL_METHODDEF {"set_wakeup_fd", (PyCFunction)signal_set_wakeup_fd, METH_VARARGS | METH_KEYWORDS, set_wakeup_fd_doc}, SIGNAL_SIGINTERRUPT_METHODDEF From webhook-mailer at python.org Mon Mar 12 13:18:46 2018 From: webhook-mailer at python.org (Andrew Svetlov) Date: Mon, 12 Mar 2018 17:18:46 -0000 Subject: [Python-checkins] bpo-33056 FIX leaking fd in concurrent.futures.ProcessPoolExecutor (#6084) Message-ID: <mailman.53.1520875128.1871.python-checkins@python.org> https://github.com/python/cpython/commit/095ee415cee41bf24c3a1108c23307e5baf168dd commit: 095ee415cee41bf24c3a1108c23307e5baf168dd branch: master author: Thomas Moreau <thomas.moreau.2010 at gmail.com> committer: Andrew Svetlov <andrew.svetlov at gmail.com> date: 2018-03-12T19:18:41+02:00 summary: bpo-33056 FIX leaking fd in concurrent.futures.ProcessPoolExecutor (#6084) files: A Misc/NEWS.d/next/Library/2018-03-12-16-40-00.bpo-33056.lNN9Eh.rst M Lib/concurrent/futures/process.py diff --git a/Lib/concurrent/futures/process.py b/Lib/concurrent/futures/process.py index aaa5151e017c..63f22cfca325 100644 --- a/Lib/concurrent/futures/process.py +++ b/Lib/concurrent/futures/process.py @@ -78,11 +78,13 @@ class _ThreadWakeup: - __slot__ = ["_state"] - def __init__(self): self._reader, self._writer = mp.Pipe(duplex=False) + def close(self): + self._writer.close() + self._reader.close() + def wakeup(self): self._writer.send_bytes(b"") @@ -654,6 +656,11 @@ def shutdown(self, wait=True): self._call_queue = None self._result_queue = None self._processes = None + + if self._queue_management_thread_wakeup: + self._queue_management_thread_wakeup.close() + self._queue_management_thread_wakeup = None + shutdown.__doc__ = _base.Executor.shutdown.__doc__ atexit.register(_python_exit) diff --git a/Misc/NEWS.d/next/Library/2018-03-12-16-40-00.bpo-33056.lNN9Eh.rst b/Misc/NEWS.d/next/Library/2018-03-12-16-40-00.bpo-33056.lNN9Eh.rst new file mode 100644 index 000000000000..6acc19a36dc8 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-03-12-16-40-00.bpo-33056.lNN9Eh.rst @@ -0,0 +1 @@ +FIX properly close leaking fds in concurrent.futures.ProcessPoolExecutor. From webhook-mailer at python.org Mon Mar 12 14:50:53 2018 From: webhook-mailer at python.org (Andrew Svetlov) Date: Mon, 12 Mar 2018 18:50:53 -0000 Subject: [Python-checkins] Fix docs markup for asyncio current_task() and all_tasks() (#6089) Message-ID: <mailman.54.1520880655.1871.python-checkins@python.org> https://github.com/python/cpython/commit/b21505e7109bafe75d87ab63d524b4e749157dbd commit: b21505e7109bafe75d87ab63d524b4e749157dbd branch: master author: Andrew Svetlov <andrew.svetlov at gmail.com> committer: GitHub <noreply at github.com> date: 2018-03-12T20:50:50+02:00 summary: Fix docs markup for asyncio current_task() and all_tasks() (#6089) files: M Doc/library/asyncio-task.rst diff --git a/Doc/library/asyncio-task.rst b/Doc/library/asyncio-task.rst index 485e1b843d08..364323d5ca8b 100644 --- a/Doc/library/asyncio-task.rst +++ b/Doc/library/asyncio-task.rst @@ -535,7 +535,7 @@ Task functions not provided, the default event loop is used. -.. function:: current_task(loop=None): +.. function:: current_task(loop=None) Return the current running :class:`Task` instance or ``None``, if no task is running. @@ -546,7 +546,7 @@ Task functions .. versionadded:: 3.7 -.. function:: all_tasks(loop=None): +.. function:: all_tasks(loop=None) Return a set of :class:`Task` objects created for the loop. From webhook-mailer at python.org Mon Mar 12 15:03:17 2018 From: webhook-mailer at python.org (Antoine Pitrou) Date: Mon, 12 Mar 2018 19:03:17 -0000 Subject: [Python-checkins] bpo-22674: fix test_strsignal on OSX (GH-6085) Message-ID: <mailman.55.1520881400.1871.python-checkins@python.org> https://github.com/python/cpython/commit/019f5b3e9e4c2a1297580483c3d5a5a10bddb93b commit: 019f5b3e9e4c2a1297580483c3d5a5a10bddb93b branch: master author: Antoine Pietri <antoine.pietri1 at gmail.com> committer: Antoine Pitrou <pitrou at free.fr> date: 2018-03-12T20:03:14+01:00 summary: bpo-22674: fix test_strsignal on OSX (GH-6085) files: M Lib/test/test_signal.py diff --git a/Lib/test/test_signal.py b/Lib/test/test_signal.py index fbb12a5b67bf..7635eec148d8 100644 --- a/Lib/test/test_signal.py +++ b/Lib/test/test_signal.py @@ -58,8 +58,8 @@ def test_getsignal(self): self.assertEqual(signal.getsignal(signal.SIGHUP), hup) def test_strsignal(self): - self.assertEqual(signal.strsignal(signal.SIGINT), "Interrupt") - self.assertEqual(signal.strsignal(signal.SIGTERM), "Terminated") + self.assertIn("Interrupt", signal.strsignal(signal.SIGINT)) + self.assertIn("Terminated", signal.strsignal(signal.SIGTERM)) # Issue 3864, unknown if this affects earlier versions of freebsd also def test_interprocess_signal(self): From webhook-mailer at python.org Mon Mar 12 15:12:12 2018 From: webhook-mailer at python.org (Miss Islington (bot)) Date: Mon, 12 Mar 2018 19:12:12 -0000 Subject: [Python-checkins] Fix docs markup for asyncio current_task() and all_tasks() (GH-6089) Message-ID: <mailman.56.1520881933.1871.python-checkins@python.org> https://github.com/python/cpython/commit/ee28cd57df3b8fca66d41528bd4b8341aad26f57 commit: ee28cd57df3b8fca66d41528bd4b8341aad26f57 branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: GitHub <noreply at github.com> date: 2018-03-12T12:12:05-07:00 summary: Fix docs markup for asyncio current_task() and all_tasks() (GH-6089) (cherry picked from commit b21505e7109bafe75d87ab63d524b4e749157dbd) Co-authored-by: Andrew Svetlov <andrew.svetlov at gmail.com> files: M Doc/library/asyncio-task.rst diff --git a/Doc/library/asyncio-task.rst b/Doc/library/asyncio-task.rst index 485e1b843d08..364323d5ca8b 100644 --- a/Doc/library/asyncio-task.rst +++ b/Doc/library/asyncio-task.rst @@ -535,7 +535,7 @@ Task functions not provided, the default event loop is used. -.. function:: current_task(loop=None): +.. function:: current_task(loop=None) Return the current running :class:`Task` instance or ``None``, if no task is running. @@ -546,7 +546,7 @@ Task functions .. versionadded:: 3.7 -.. function:: all_tasks(loop=None): +.. function:: all_tasks(loop=None) Return a set of :class:`Task` objects created for the loop. From webhook-mailer at python.org Mon Mar 12 17:07:10 2018 From: webhook-mailer at python.org (Ned Deily) Date: Mon, 12 Mar 2018 21:07:10 -0000 Subject: [Python-checkins] Add macOS installer Conclusion file Message-ID: <mailman.57.1520888830.1871.python-checkins@python.org> https://github.com/python/cpython/commit/37ed6dfbbc2a18de46cad292f9b0fc5ce32bc01d commit: 37ed6dfbbc2a18de46cad292f9b0fc5ce32bc01d branch: master author: Ned Deily <nad at python.org> committer: Ned Deily <nad at python.org> date: 2018-03-12T16:49:59-04:00 summary: Add macOS installer Conclusion file files: A Mac/BuildScript/resources/Conclusion.rtf diff --git a/Mac/BuildScript/resources/Conclusion.rtf b/Mac/BuildScript/resources/Conclusion.rtf new file mode 100644 index 000000000000..9e0fa9fa6eeb --- /dev/null +++ b/Mac/BuildScript/resources/Conclusion.rtf @@ -0,0 +1,20 @@ +{\rtf1\ansi\ansicpg1252\cocoartf1561\cocoasubrtf200 +{\fonttbl\f0\fswiss\fcharset0 Helvetica;\f1\fnil\fcharset0 LucidaGrande-Bold;\f2\fnil\fcharset0 LucidaGrande; +\f3\fnil\fcharset0 Monaco;} +{\colortbl;\red255\green255\blue255;} +{\*\expandedcolortbl;;} +\margl1440\margr1440\vieww10540\viewh8400\viewkind0 +\pard\tx720\tx1440\tx2160\tx2880\tx3600\tx4320\tx5040\tx5760\tx6480\tx7200\tx7920\tx8640\pardirnatural\partightenfactor0 + +\f0\fs28 \cf0 Congratulations! +\fs24 +\f1\b\fs28 Python $FULL_VERSION for macOS $MACOSX_DEPLOYMENT_TARGET +\f2\b0 was successfully installed. +\fs24 \ +\ +One more thing: to verify the identity of secure network connections, this Python needs a set of SSL root certificates. You can download and install a current curated set from {\field{\*\fldinst{HYPERLINK "https://pypi.org/project/certifi/"}}{\fldrslt the Certifi project}} by double-clicking on the +\f3 Install Certificates +\f2 icon in {\field{\*\fldinst{HYPERLINK "file://localhost/Applications/Python%20$VERSION/"}}{\fldrslt the Finder window}}. See {\field{\*\fldinst{HYPERLINK "file://localhost/Applications/Python%20$VERSION/ReadMe.rtf"}}{\fldrslt the +\f3 ReadMe +\f2 file}} for more information.\ +} \ No newline at end of file From webhook-mailer at python.org Mon Mar 12 17:09:54 2018 From: webhook-mailer at python.org (Ned Deily) Date: Mon, 12 Mar 2018 21:09:54 -0000 Subject: [Python-checkins] Add macOS installer Conclusion file Message-ID: <mailman.58.1520888994.1871.python-checkins@python.org> https://github.com/python/cpython/commit/6f415c48197d2ce86bce2a1cf34d3e7faa4ceb98 commit: 6f415c48197d2ce86bce2a1cf34d3e7faa4ceb98 branch: 3.7 author: Ned Deily <nad at python.org> committer: Ned Deily <nad at python.org> date: 2018-03-12T17:08:32-04:00 summary: Add macOS installer Conclusion file files: A Mac/BuildScript/resources/Conclusion.rtf diff --git a/Mac/BuildScript/resources/Conclusion.rtf b/Mac/BuildScript/resources/Conclusion.rtf new file mode 100644 index 000000000000..9e0fa9fa6eeb --- /dev/null +++ b/Mac/BuildScript/resources/Conclusion.rtf @@ -0,0 +1,20 @@ +{\rtf1\ansi\ansicpg1252\cocoartf1561\cocoasubrtf200 +{\fonttbl\f0\fswiss\fcharset0 Helvetica;\f1\fnil\fcharset0 LucidaGrande-Bold;\f2\fnil\fcharset0 LucidaGrande; +\f3\fnil\fcharset0 Monaco;} +{\colortbl;\red255\green255\blue255;} +{\*\expandedcolortbl;;} +\margl1440\margr1440\vieww10540\viewh8400\viewkind0 +\pard\tx720\tx1440\tx2160\tx2880\tx3600\tx4320\tx5040\tx5760\tx6480\tx7200\tx7920\tx8640\pardirnatural\partightenfactor0 + +\f0\fs28 \cf0 Congratulations! +\fs24 +\f1\b\fs28 Python $FULL_VERSION for macOS $MACOSX_DEPLOYMENT_TARGET +\f2\b0 was successfully installed. +\fs24 \ +\ +One more thing: to verify the identity of secure network connections, this Python needs a set of SSL root certificates. You can download and install a current curated set from {\field{\*\fldinst{HYPERLINK "https://pypi.org/project/certifi/"}}{\fldrslt the Certifi project}} by double-clicking on the +\f3 Install Certificates +\f2 icon in {\field{\*\fldinst{HYPERLINK "file://localhost/Applications/Python%20$VERSION/"}}{\fldrslt the Finder window}}. See {\field{\*\fldinst{HYPERLINK "file://localhost/Applications/Python%20$VERSION/ReadMe.rtf"}}{\fldrslt the +\f3 ReadMe +\f2 file}} for more information.\ +} \ No newline at end of file From webhook-mailer at python.org Mon Mar 12 21:16:11 2018 From: webhook-mailer at python.org (Ned Deily) Date: Tue, 13 Mar 2018 01:16:11 -0000 Subject: [Python-checkins] bpo-29719: Remove Date and Release field in whatsnew/3.7 and 8 (GH-6093) Message-ID: <mailman.59.1520903772.1871.python-checkins@python.org> https://github.com/python/cpython/commit/a34510a4c562b4b23c7f9da6ff6e2318484f5f1a commit: a34510a4c562b4b23c7f9da6ff6e2318484f5f1a branch: master author: Ned Deily <nad at python.org> committer: GitHub <noreply at github.com> date: 2018-03-12T21:16:08-04:00 summary: bpo-29719: Remove Date and Release field in whatsnew/3.7 and 8 (GH-6093) files: M Doc/whatsnew/3.7.rst M Doc/whatsnew/3.8.rst diff --git a/Doc/whatsnew/3.7.rst b/Doc/whatsnew/3.7.rst index a4ed481f67a5..7eb19f0f82a6 100644 --- a/Doc/whatsnew/3.7.rst +++ b/Doc/whatsnew/3.7.rst @@ -2,9 +2,6 @@ What's New In Python 3.7 **************************** -:Release: |release| -:Date: |today| - .. Rules for maintenance: * Anyone can add text to this document. Do not spend very much time diff --git a/Doc/whatsnew/3.8.rst b/Doc/whatsnew/3.8.rst index 904a3584e67a..8569341055e1 100644 --- a/Doc/whatsnew/3.8.rst +++ b/Doc/whatsnew/3.8.rst @@ -2,9 +2,6 @@ What's New In Python 3.8 **************************** -:Release: |release| -:Date: |today| - .. Rules for maintenance: * Anyone can add text to this document. Do not spend very much time From webhook-mailer at python.org Mon Mar 12 21:16:57 2018 From: webhook-mailer at python.org (Ned Deily) Date: Tue, 13 Mar 2018 01:16:57 -0000 Subject: [Python-checkins] bpo-29719: Remove Date and Release field in whatsnew/3.7 (GH-6094) Message-ID: <mailman.60.1520903818.1871.python-checkins@python.org> https://github.com/python/cpython/commit/d461ed84b57aae0bdccd117dc38109c087444d50 commit: d461ed84b57aae0bdccd117dc38109c087444d50 branch: 3.7 author: Ned Deily <nad at python.org> committer: GitHub <noreply at github.com> date: 2018-03-12T21:16:54-04:00 summary: bpo-29719: Remove Date and Release field in whatsnew/3.7 (GH-6094) files: M Doc/whatsnew/3.7.rst diff --git a/Doc/whatsnew/3.7.rst b/Doc/whatsnew/3.7.rst index a4ed481f67a5..7eb19f0f82a6 100644 --- a/Doc/whatsnew/3.7.rst +++ b/Doc/whatsnew/3.7.rst @@ -2,9 +2,6 @@ What's New In Python 3.7 **************************** -:Release: |release| -:Date: |today| - .. Rules for maintenance: * Anyone can add text to this document. Do not spend very much time From webhook-mailer at python.org Tue Mar 13 03:44:54 2018 From: webhook-mailer at python.org (=?utf-8?q?=C5=81ukasz?= Langa) Date: Tue, 13 Mar 2018 07:44:54 -0000 Subject: [Python-checkins] bpo-33064: lib2to3: support trailing comma after *args and **kwargs (#6096) Message-ID: <mailman.61.1520927096.1871.python-checkins@python.org> https://github.com/python/cpython/commit/b51f5de71163f096d2d5229ede5379cdb284f651 commit: b51f5de71163f096d2d5229ede5379cdb284f651 branch: master author: ?ukasz Langa <lukasz at langa.pl> committer: GitHub <noreply at github.com> date: 2018-03-13T00:44:49-07:00 summary: bpo-33064: lib2to3: support trailing comma after *args and **kwargs (#6096) New tests also added. I also made the comments in line with the builtin Grammar/Grammar. PEP 306 was withdrawn, Kees Blom's railroad program has been lost to the sands of time for at least 16 years now (I found a python-dev post from people looking for it). files: A Misc/NEWS.d/next/Library/2018-03-12-19-58-25.bpo-33064.LO2KIY.rst M Lib/lib2to3/Grammar.txt M Lib/lib2to3/tests/test_parser.py diff --git a/Lib/lib2to3/Grammar.txt b/Lib/lib2to3/Grammar.txt index 0bdfcafcf3ca..b19b4a21fadd 100644 --- a/Lib/lib2to3/Grammar.txt +++ b/Lib/lib2to3/Grammar.txt @@ -1,26 +1,7 @@ # Grammar for 2to3. This grammar supports Python 2.x and 3.x. -# Note: Changing the grammar specified in this file will most likely -# require corresponding changes in the parser module -# (../Modules/parsermodule.c). If you can't make the changes to -# that module yourself, please co-ordinate the required changes -# with someone who can; ask around on python-dev for help. Fred -# Drake <fdrake at acm.org> will probably be listening there. - -# NOTE WELL: You should also follow all the steps listed in PEP 306, -# "How to Change Python's Grammar" - -# Commands for Kees Blom's railroad program -#diagram:token NAME -#diagram:token NUMBER -#diagram:token STRING -#diagram:token NEWLINE -#diagram:token ENDMARKER -#diagram:token INDENT -#diagram:output\input python.bla -#diagram:token DEDENT -#diagram:output\textwidth 20.04cm\oddsidemargin 0.0cm\evensidemargin 0.0cm -#diagram:rules +# NOTE WELL: You should also follow all the steps listed at +# https://devguide.python.org/grammar/ # Start symbols for the grammar: # file_input is a module or sequence of commands read from an input file; @@ -38,13 +19,13 @@ async_funcdef: 'async' funcdef funcdef: 'def' NAME parameters ['->' test] ':' suite parameters: '(' [typedargslist] ')' typedargslist: ((tfpdef ['=' test] ',')* - ('*' [tname] (',' tname ['=' test])* [',' '**' tname] | '**' tname) + ('*' [tname] (',' tname ['=' test])* [',' ['**' tname [',']]] | '**' tname [',']) | tfpdef ['=' test] (',' tfpdef ['=' test])* [',']) tname: NAME [':' test] tfpdef: tname | '(' tfplist ')' tfplist: tfpdef (',' tfpdef)* [','] varargslist: ((vfpdef ['=' test] ',')* - ('*' [vname] (',' vname ['=' test])* [',' '**' vname] | '**' vname) + ('*' [vname] (',' vname ['=' test])* [',' ['**' vname [',']]] | '**' vname [',']) | vfpdef ['=' test] (',' vfpdef ['=' test])* [',']) vname: NAME vfpdef: vname | '(' vfplist ')' diff --git a/Lib/lib2to3/tests/test_parser.py b/Lib/lib2to3/tests/test_parser.py index 0cbba26bec04..cec2f98e9665 100644 --- a/Lib/lib2to3/tests/test_parser.py +++ b/Lib/lib2to3/tests/test_parser.py @@ -9,7 +9,6 @@ # Testing imports from . import support from .support import driver, driver_no_print_statement -from test.support import verbose # Python imports import difflib @@ -22,7 +21,6 @@ import sys import tempfile import unittest -import warnings # Local imports from lib2to3.pgen2 import driver as pgen2_driver @@ -305,6 +303,38 @@ def test_8(self): *g:6, h:7, i=8, j:9=10, **k:11) -> 12: pass""" self.validate(s) + def test_9(self): + s = """def f( + a: str, + b: int, + *, + c: bool = False, + **kwargs, + ) -> None: + call(c=c, **kwargs,)""" + self.validate(s) + + def test_10(self): + s = """def f( + a: str, + ) -> None: + call(a,)""" + self.validate(s) + + def test_11(self): + s = """def f( + a: str = '', + ) -> None: + call(a=a,)""" + self.validate(s) + + def test_12(self): + s = """def f( + *args: str, + ) -> None: + call(*args,)""" + self.validate(s) + # Adapted from Python 3's Lib/test/test_grammar.py:GrammarTests.test_var_annot class TestVarAnnotations(GrammarTest): @@ -407,7 +437,7 @@ def test_new_syntax(self): self.validate("class B(t, *args): pass") self.validate("class B(t, **kwargs): pass") self.validate("class B(t, *args, **kwargs): pass") - self.validate("class B(t, y=9, *args, **kwargs): pass") + self.validate("class B(t, y=9, *args, **kwargs,): pass") class TestParserIdempotency(support.TestCase): diff --git a/Misc/NEWS.d/next/Library/2018-03-12-19-58-25.bpo-33064.LO2KIY.rst b/Misc/NEWS.d/next/Library/2018-03-12-19-58-25.bpo-33064.LO2KIY.rst new file mode 100644 index 000000000000..c8e955e335cb --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-03-12-19-58-25.bpo-33064.LO2KIY.rst @@ -0,0 +1,2 @@ +lib2to3 now properly supports trailing commas after ``*args`` and +``**kwargs`` in function signatures. From webhook-mailer at python.org Tue Mar 13 04:08:07 2018 From: webhook-mailer at python.org (=?utf-8?q?=C5=81ukasz?= Langa) Date: Tue, 13 Mar 2018 08:08:07 -0000 Subject: [Python-checkins] bpo-33064: lib2to3: support trailing comma after *args and **kwargs (GH-6096) (#6097) Message-ID: <mailman.62.1520928487.1871.python-checkins@python.org> https://github.com/python/cpython/commit/b4c8871ca43d37be167ef5dbe9fb341922c04a9f commit: b4c8871ca43d37be167ef5dbe9fb341922c04a9f branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: ?ukasz Langa <lukasz at langa.pl> date: 2018-03-13T01:08:02-07:00 summary: bpo-33064: lib2to3: support trailing comma after *args and **kwargs (GH-6096) (#6097) New tests also added. I also made the comments in line with the builtin Grammar/Grammar. PEP 306 was withdrawn, Kees Blom's railroad program has been lost to the sands of time for at least 16 years now (I found a python-dev post from people looking for it). (cherry picked from commit b51f5de71163f096d2d5229ede5379cdb284f651) Co-authored-by: ?ukasz Langa <lukasz at langa.pl> files: A Misc/NEWS.d/next/Library/2018-03-12-19-58-25.bpo-33064.LO2KIY.rst M Lib/lib2to3/Grammar.txt M Lib/lib2to3/tests/test_parser.py diff --git a/Lib/lib2to3/Grammar.txt b/Lib/lib2to3/Grammar.txt index 0bdfcafcf3ca..b19b4a21fadd 100644 --- a/Lib/lib2to3/Grammar.txt +++ b/Lib/lib2to3/Grammar.txt @@ -1,26 +1,7 @@ # Grammar for 2to3. This grammar supports Python 2.x and 3.x. -# Note: Changing the grammar specified in this file will most likely -# require corresponding changes in the parser module -# (../Modules/parsermodule.c). If you can't make the changes to -# that module yourself, please co-ordinate the required changes -# with someone who can; ask around on python-dev for help. Fred -# Drake <fdrake at acm.org> will probably be listening there. - -# NOTE WELL: You should also follow all the steps listed in PEP 306, -# "How to Change Python's Grammar" - -# Commands for Kees Blom's railroad program -#diagram:token NAME -#diagram:token NUMBER -#diagram:token STRING -#diagram:token NEWLINE -#diagram:token ENDMARKER -#diagram:token INDENT -#diagram:output\input python.bla -#diagram:token DEDENT -#diagram:output\textwidth 20.04cm\oddsidemargin 0.0cm\evensidemargin 0.0cm -#diagram:rules +# NOTE WELL: You should also follow all the steps listed at +# https://devguide.python.org/grammar/ # Start symbols for the grammar: # file_input is a module or sequence of commands read from an input file; @@ -38,13 +19,13 @@ async_funcdef: 'async' funcdef funcdef: 'def' NAME parameters ['->' test] ':' suite parameters: '(' [typedargslist] ')' typedargslist: ((tfpdef ['=' test] ',')* - ('*' [tname] (',' tname ['=' test])* [',' '**' tname] | '**' tname) + ('*' [tname] (',' tname ['=' test])* [',' ['**' tname [',']]] | '**' tname [',']) | tfpdef ['=' test] (',' tfpdef ['=' test])* [',']) tname: NAME [':' test] tfpdef: tname | '(' tfplist ')' tfplist: tfpdef (',' tfpdef)* [','] varargslist: ((vfpdef ['=' test] ',')* - ('*' [vname] (',' vname ['=' test])* [',' '**' vname] | '**' vname) + ('*' [vname] (',' vname ['=' test])* [',' ['**' vname [',']]] | '**' vname [',']) | vfpdef ['=' test] (',' vfpdef ['=' test])* [',']) vname: NAME vfpdef: vname | '(' vfplist ')' diff --git a/Lib/lib2to3/tests/test_parser.py b/Lib/lib2to3/tests/test_parser.py index 0cbba26bec04..cec2f98e9665 100644 --- a/Lib/lib2to3/tests/test_parser.py +++ b/Lib/lib2to3/tests/test_parser.py @@ -9,7 +9,6 @@ # Testing imports from . import support from .support import driver, driver_no_print_statement -from test.support import verbose # Python imports import difflib @@ -22,7 +21,6 @@ import sys import tempfile import unittest -import warnings # Local imports from lib2to3.pgen2 import driver as pgen2_driver @@ -305,6 +303,38 @@ def test_8(self): *g:6, h:7, i=8, j:9=10, **k:11) -> 12: pass""" self.validate(s) + def test_9(self): + s = """def f( + a: str, + b: int, + *, + c: bool = False, + **kwargs, + ) -> None: + call(c=c, **kwargs,)""" + self.validate(s) + + def test_10(self): + s = """def f( + a: str, + ) -> None: + call(a,)""" + self.validate(s) + + def test_11(self): + s = """def f( + a: str = '', + ) -> None: + call(a=a,)""" + self.validate(s) + + def test_12(self): + s = """def f( + *args: str, + ) -> None: + call(*args,)""" + self.validate(s) + # Adapted from Python 3's Lib/test/test_grammar.py:GrammarTests.test_var_annot class TestVarAnnotations(GrammarTest): @@ -407,7 +437,7 @@ def test_new_syntax(self): self.validate("class B(t, *args): pass") self.validate("class B(t, **kwargs): pass") self.validate("class B(t, *args, **kwargs): pass") - self.validate("class B(t, y=9, *args, **kwargs): pass") + self.validate("class B(t, y=9, *args, **kwargs,): pass") class TestParserIdempotency(support.TestCase): diff --git a/Misc/NEWS.d/next/Library/2018-03-12-19-58-25.bpo-33064.LO2KIY.rst b/Misc/NEWS.d/next/Library/2018-03-12-19-58-25.bpo-33064.LO2KIY.rst new file mode 100644 index 000000000000..c8e955e335cb --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-03-12-19-58-25.bpo-33064.LO2KIY.rst @@ -0,0 +1,2 @@ +lib2to3 now properly supports trailing commas after ``*args`` and +``**kwargs`` in function signatures. From webhook-mailer at python.org Tue Mar 13 04:32:37 2018 From: webhook-mailer at python.org (=?utf-8?q?=C5=81ukasz?= Langa) Date: Tue, 13 Mar 2018 08:32:37 -0000 Subject: [Python-checkins] bpo-33064: lib2to3: support trailing comma after *args and **kwargs (GH-6096) (#6098) Message-ID: <mailman.63.1520929958.1871.python-checkins@python.org> https://github.com/python/cpython/commit/6a526f673878677032c02f7800ee13d4769f391a commit: 6a526f673878677032c02f7800ee13d4769f391a branch: 3.6 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: ?ukasz Langa <lukasz at langa.pl> date: 2018-03-13T01:32:28-07:00 summary: bpo-33064: lib2to3: support trailing comma after *args and **kwargs (GH-6096) (#6098) New tests also added. I also made the comments in line with the builtin Grammar/Grammar. PEP 306 was withdrawn, Kees Blom's railroad program has been lost to the sands of time for at least 16 years now (I found a python-dev post from people looking for it). (cherry picked from commit b51f5de71163f096d2d5229ede5379cdb284f651) Co-authored-by: ?ukasz Langa <lukasz at langa.pl> files: A Misc/NEWS.d/next/Library/2018-03-12-19-58-25.bpo-33064.LO2KIY.rst M Lib/lib2to3/Grammar.txt M Lib/lib2to3/tests/test_parser.py diff --git a/Lib/lib2to3/Grammar.txt b/Lib/lib2to3/Grammar.txt index 2abd5ee65b5b..a7ddad3cf322 100644 --- a/Lib/lib2to3/Grammar.txt +++ b/Lib/lib2to3/Grammar.txt @@ -1,26 +1,7 @@ # Grammar for 2to3. This grammar supports Python 2.x and 3.x. -# Note: Changing the grammar specified in this file will most likely -# require corresponding changes in the parser module -# (../Modules/parsermodule.c). If you can't make the changes to -# that module yourself, please co-ordinate the required changes -# with someone who can; ask around on python-dev for help. Fred -# Drake <fdrake at acm.org> will probably be listening there. - -# NOTE WELL: You should also follow all the steps listed in PEP 306, -# "How to Change Python's Grammar" - -# Commands for Kees Blom's railroad program -#diagram:token NAME -#diagram:token NUMBER -#diagram:token STRING -#diagram:token NEWLINE -#diagram:token ENDMARKER -#diagram:token INDENT -#diagram:output\input python.bla -#diagram:token DEDENT -#diagram:output\textwidth 20.04cm\oddsidemargin 0.0cm\evensidemargin 0.0cm -#diagram:rules +# NOTE WELL: You should also follow all the steps listed at +# https://devguide.python.org/grammar/ # Start symbols for the grammar: # file_input is a module or sequence of commands read from an input file; @@ -38,13 +19,13 @@ async_funcdef: ASYNC funcdef funcdef: 'def' NAME parameters ['->' test] ':' suite parameters: '(' [typedargslist] ')' typedargslist: ((tfpdef ['=' test] ',')* - ('*' [tname] (',' tname ['=' test])* [',' '**' tname] | '**' tname) + ('*' [tname] (',' tname ['=' test])* [',' ['**' tname [',']]] | '**' tname [',']) | tfpdef ['=' test] (',' tfpdef ['=' test])* [',']) tname: NAME [':' test] tfpdef: tname | '(' tfplist ')' tfplist: tfpdef (',' tfpdef)* [','] varargslist: ((vfpdef ['=' test] ',')* - ('*' [vname] (',' vname ['=' test])* [',' '**' vname] | '**' vname) + ('*' [vname] (',' vname ['=' test])* [',' ['**' vname [',']]] | '**' vname [',']) | vfpdef ['=' test] (',' vfpdef ['=' test])* [',']) vname: NAME vfpdef: vname | '(' vfplist ')' diff --git a/Lib/lib2to3/tests/test_parser.py b/Lib/lib2to3/tests/test_parser.py index 3fc499522454..c4d062ff827e 100644 --- a/Lib/lib2to3/tests/test_parser.py +++ b/Lib/lib2to3/tests/test_parser.py @@ -9,7 +9,6 @@ # Testing imports from . import support from .support import driver, driver_no_print_statement -from test.support import verbose # Python imports import difflib @@ -22,7 +21,6 @@ import sys import tempfile import unittest -import warnings # Local imports from lib2to3.pgen2 import driver as pgen2_driver @@ -305,6 +303,38 @@ def test_8(self): *g:6, h:7, i=8, j:9=10, **k:11) -> 12: pass""" self.validate(s) + def test_9(self): + s = """def f( + a: str, + b: int, + *, + c: bool = False, + **kwargs, + ) -> None: + call(c=c, **kwargs,)""" + self.validate(s) + + def test_10(self): + s = """def f( + a: str, + ) -> None: + call(a,)""" + self.validate(s) + + def test_11(self): + s = """def f( + a: str = '', + ) -> None: + call(a=a,)""" + self.validate(s) + + def test_12(self): + s = """def f( + *args: str, + ) -> None: + call(*args,)""" + self.validate(s) + # Adapted from Python 3's Lib/test/test_grammar.py:GrammarTests.test_var_annot class TestVarAnnotations(GrammarTest): @@ -407,7 +437,7 @@ def test_new_syntax(self): self.validate("class B(t, *args): pass") self.validate("class B(t, **kwargs): pass") self.validate("class B(t, *args, **kwargs): pass") - self.validate("class B(t, y=9, *args, **kwargs): pass") + self.validate("class B(t, y=9, *args, **kwargs,): pass") class TestParserIdempotency(support.TestCase): diff --git a/Misc/NEWS.d/next/Library/2018-03-12-19-58-25.bpo-33064.LO2KIY.rst b/Misc/NEWS.d/next/Library/2018-03-12-19-58-25.bpo-33064.LO2KIY.rst new file mode 100644 index 000000000000..c8e955e335cb --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-03-12-19-58-25.bpo-33064.LO2KIY.rst @@ -0,0 +1,2 @@ +lib2to3 now properly supports trailing commas after ``*args`` and +``**kwargs`` in function signatures. From webhook-mailer at python.org Tue Mar 13 04:52:38 2018 From: webhook-mailer at python.org (Serhiy Storchaka) Date: Tue, 13 Mar 2018 08:52:38 -0000 Subject: [Python-checkins] [3.7] bpo-17288: Prevent jumps from 'return' and 'exception' trace events. (GH-5928) Message-ID: <mailman.64.1520931160.1871.python-checkins@python.org> https://github.com/python/cpython/commit/e32bbaf376a09c149fa7c7f2919d7c9ce4e2a055 commit: e32bbaf376a09c149fa7c7f2919d7c9ce4e2a055 branch: 3.7 author: xdegaye <xdegaye at gmail.com> committer: Serhiy Storchaka <storchaka at gmail.com> date: 2018-03-13T10:52:35+02:00 summary: [3.7] bpo-17288: Prevent jumps from 'return' and 'exception' trace events. (GH-5928) files: A Misc/NEWS.d/next/Core and Builtins/2018-02-27-13-36-21.bpo-17288.Gdj24S.rst M Lib/test/test_sys_settrace.py M Objects/frameobject.c diff --git a/Lib/test/test_sys_settrace.py b/Lib/test/test_sys_settrace.py index fa5b61af6172..44ea7d575b45 100644 --- a/Lib/test/test_sys_settrace.py +++ b/Lib/test/test_sys_settrace.py @@ -556,20 +556,35 @@ def g(frame, event, arg): class JumpTracer: """Defines a trace function that jumps from one place to another.""" - def __init__(self, function, jumpFrom, jumpTo): - self.function = function + def __init__(self, function, jumpFrom, jumpTo, event='line', + decorated=False): + self.code = function.__code__ self.jumpFrom = jumpFrom self.jumpTo = jumpTo + self.event = event + self.firstLine = None if decorated else self.code.co_firstlineno self.done = False def trace(self, frame, event, arg): - if not self.done and frame.f_code == self.function.__code__: - firstLine = frame.f_code.co_firstlineno - if event == 'line' and frame.f_lineno == firstLine + self.jumpFrom: + if self.done: + return + # frame.f_code.co_firstlineno is the first line of the decorator when + # 'function' is decorated and the decorator may be written using + # multiple physical lines when it is too long. Use the first line + # trace event in 'function' to find the first line of 'function'. + if (self.firstLine is None and frame.f_code == self.code and + event == 'line'): + self.firstLine = frame.f_lineno - 1 + if (event == self.event and self.firstLine and + frame.f_lineno == self.firstLine + self.jumpFrom): + f = frame + while f is not None and f.f_code != self.code: + f = f.f_back + if f is not None: # Cope with non-integer self.jumpTo (because of # no_jump_to_non_integers below). try: - frame.f_lineno = firstLine + self.jumpTo + frame.f_lineno = self.firstLine + self.jumpTo except TypeError: frame.f_lineno = self.jumpTo self.done = True @@ -609,8 +624,9 @@ def compare_jump_output(self, expected, received): "Expected: " + repr(expected) + "\n" + "Received: " + repr(received)) - def run_test(self, func, jumpFrom, jumpTo, expected, error=None): - tracer = JumpTracer(func, jumpFrom, jumpTo) + def run_test(self, func, jumpFrom, jumpTo, expected, error=None, + event='line', decorated=False): + tracer = JumpTracer(func, jumpFrom, jumpTo, event, decorated) sys.settrace(tracer.trace) output = [] if error is None: @@ -621,15 +637,15 @@ def run_test(self, func, jumpFrom, jumpTo, expected, error=None): sys.settrace(None) self.compare_jump_output(expected, output) - def jump_test(jumpFrom, jumpTo, expected, error=None): + def jump_test(jumpFrom, jumpTo, expected, error=None, event='line'): """Decorator that creates a test that makes a jump from one place to another in the following code. """ def decorator(func): @wraps(func) def test(self): - # +1 to compensate a decorator line - self.run_test(func, jumpFrom+1, jumpTo+1, expected, error) + self.run_test(func, jumpFrom, jumpTo, expected, + error=error, event=event, decorated=True) return test return decorator @@ -1104,6 +1120,36 @@ class fake_function: sys.settrace(None) self.compare_jump_output([2, 3, 2, 3, 4], namespace["output"]) + @jump_test(2, 3, [1], event='call', error=(ValueError, "can't jump from" + " the 'call' trace event of a new frame")) + def test_no_jump_from_call(output): + output.append(1) + def nested(): + output.append(3) + nested() + output.append(5) + + @jump_test(2, 1, [1], event='return', error=(ValueError, + "can only jump from a 'line' trace event")) + def test_no_jump_from_return_event(output): + output.append(1) + return + + @jump_test(2, 1, [1], event='exception', error=(ValueError, + "can only jump from a 'line' trace event")) + def test_no_jump_from_exception_event(output): + output.append(1) + 1 / 0 + + @jump_test(3, 2, [2], event='return', error=(ValueError, + "can't jump from a yield statement")) + def test_no_jump_from_yield(output): + def gen(): + output.append(2) + yield 3 + next(gen()) + output.append(5) + if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS.d/next/Core and Builtins/2018-02-27-13-36-21.bpo-17288.Gdj24S.rst b/Misc/NEWS.d/next/Core and Builtins/2018-02-27-13-36-21.bpo-17288.Gdj24S.rst new file mode 100644 index 000000000000..ce9e84c40313 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2018-02-27-13-36-21.bpo-17288.Gdj24S.rst @@ -0,0 +1 @@ +Prevent jumps from 'return' and 'exception' trace events. diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 2825041e6310..f518dc485616 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -59,6 +59,9 @@ frame_getlineno(PyFrameObject *f, void *closure) * o 'try'/'for'/'while' blocks can't be jumped into because the blockstack * needs to be set up before their code runs, and for 'for' loops the * iterator needs to be on the stack. + * o Jumps cannot be made from within a trace function invoked with a + * 'return' or 'exception' event since the eval loop has been exited at + * that time. */ static int frame_setlineno(PyFrameObject *f, PyObject* p_new_lineno) @@ -94,13 +97,32 @@ frame_setlineno(PyFrameObject *f, PyObject* p_new_lineno) return -1; } + /* Upon the 'call' trace event of a new frame, f->f_lasti is -1 and + * f->f_trace is NULL, check first on the first condition. + * Forbidding jumps from the 'call' event of a new frame is a side effect + * of allowing to set f_lineno only from trace functions. */ + if (f->f_lasti == -1) { + PyErr_Format(PyExc_ValueError, + "can't jump from the 'call' trace event of a new frame"); + return -1; + } + /* You can only do this from within a trace function, not via * _getframe or similar hackery. */ - if (!f->f_trace) - { + if (!f->f_trace) { PyErr_Format(PyExc_ValueError, - "f_lineno can only be set by a" - " line trace function"); + "f_lineno can only be set by a trace function"); + return -1; + } + + /* Forbid jumps upon a 'return' trace event (except after executing a + * YIELD_VALUE or YIELD_FROM opcode, f_stacktop is not NULL in that case) + * and upon an 'exception' trace event. + * Jumps from 'call' trace events have already been forbidden above for new + * frames, so this check does not change anything for 'call' events. */ + if (f->f_stacktop == NULL) { + PyErr_SetString(PyExc_ValueError, + "can only jump from a 'line' trace event"); return -1; } @@ -159,6 +181,16 @@ frame_setlineno(PyFrameObject *f, PyObject* p_new_lineno) /* We're now ready to look at the bytecode. */ PyBytes_AsStringAndSize(f->f_code->co_code, (char **)&code, &code_len); + + /* The trace function is called with a 'return' trace event after the + * execution of a yield statement. */ + assert(f->f_lasti != -1); + if (code[f->f_lasti] == YIELD_VALUE || code[f->f_lasti] == YIELD_FROM) { + PyErr_SetString(PyExc_ValueError, + "can't jump from a yield statement"); + return -1; + } + min_addr = Py_MIN(new_lasti, f->f_lasti); max_addr = Py_MAX(new_lasti, f->f_lasti); From webhook-mailer at python.org Tue Mar 13 05:11:00 2018 From: webhook-mailer at python.org (Antoine Pitrou) Date: Tue, 13 Mar 2018 09:11:00 -0000 Subject: [Python-checkins] bpo-33056 FIX leaking fd in concurrent.futures.ProcessPoolExecutor (GH-6084) (#6092) Message-ID: <mailman.65.1520932262.1871.python-checkins@python.org> https://github.com/python/cpython/commit/f216cbf9ab704da98146a25d57ff0e85aecb49da commit: f216cbf9ab704da98146a25d57ff0e85aecb49da branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Antoine Pitrou <pitrou at free.fr> date: 2018-03-13T10:10:57+01:00 summary: bpo-33056 FIX leaking fd in concurrent.futures.ProcessPoolExecutor (GH-6084) (#6092) (cherry picked from commit 095ee415cee41bf24c3a1108c23307e5baf168dd) Co-authored-by: Thomas Moreau <thomas.moreau.2010 at gmail.com> files: A Misc/NEWS.d/next/Library/2018-03-12-16-40-00.bpo-33056.lNN9Eh.rst M Lib/concurrent/futures/process.py diff --git a/Lib/concurrent/futures/process.py b/Lib/concurrent/futures/process.py index aaa5151e017c..63f22cfca325 100644 --- a/Lib/concurrent/futures/process.py +++ b/Lib/concurrent/futures/process.py @@ -78,11 +78,13 @@ class _ThreadWakeup: - __slot__ = ["_state"] - def __init__(self): self._reader, self._writer = mp.Pipe(duplex=False) + def close(self): + self._writer.close() + self._reader.close() + def wakeup(self): self._writer.send_bytes(b"") @@ -654,6 +656,11 @@ def shutdown(self, wait=True): self._call_queue = None self._result_queue = None self._processes = None + + if self._queue_management_thread_wakeup: + self._queue_management_thread_wakeup.close() + self._queue_management_thread_wakeup = None + shutdown.__doc__ = _base.Executor.shutdown.__doc__ atexit.register(_python_exit) diff --git a/Misc/NEWS.d/next/Library/2018-03-12-16-40-00.bpo-33056.lNN9Eh.rst b/Misc/NEWS.d/next/Library/2018-03-12-16-40-00.bpo-33056.lNN9Eh.rst new file mode 100644 index 000000000000..6acc19a36dc8 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-03-12-16-40-00.bpo-33056.lNN9Eh.rst @@ -0,0 +1 @@ +FIX properly close leaking fds in concurrent.futures.ProcessPoolExecutor. From solipsis at pitrou.net Tue Mar 13 05:12:50 2018 From: solipsis at pitrou.net (solipsis at pitrou.net) Date: Tue, 13 Mar 2018 09:12:50 +0000 Subject: [Python-checkins] Daily reference leaks (4243df51fe43): sum=1 Message-ID: <20180313091250.1.4F9660C55558EC80@psf.io> results for 4243df51fe43 on branch "default" -------------------------------------------- test_asyncio leaked [3, 0, 0] memory blocks, sum=3 test_collections leaked [-7, 8, -7] memory blocks, sum=-6 test_functools leaked [0, 3, 1] memory blocks, sum=4 test_multiprocessing_fork leaked [1, 0, -2] memory blocks, sum=-1 test_multiprocessing_forkserver leaked [2, 0, -1] memory blocks, sum=1 Command line was: ['./python', '-m', 'test.regrtest', '-uall', '-R', '3:3:/home/psf-users/antoine/refleaks/reflogqmRFmj', '--timeout', '7200'] From webhook-mailer at python.org Tue Mar 13 05:56:46 2018 From: webhook-mailer at python.org (Christian Heimes) Date: Tue, 13 Mar 2018 09:56:46 -0000 Subject: [Python-checkins] bpo-32885: Tools/scripts/pathfix.py: Add -n option for no backup~ (#5772) Message-ID: <mailman.66.1520935008.1871.python-checkins@python.org> https://github.com/python/cpython/commit/5affd5c29eb1493cb31ef3cfdde15538ac134689 commit: 5affd5c29eb1493cb31ef3cfdde15538ac134689 branch: master author: Miro Hron?ok <miro at hroncok.cz> committer: Christian Heimes <christian at python.org> date: 2018-03-13T05:56:43-04:00 summary: bpo-32885: Tools/scripts/pathfix.py: Add -n option for no backup~ (#5772) Creating backup files with ~ suffix can be undesirable in some environment, such as when building RPM packages. Instead of requiring the user to remove those files manually, option -n was added, that simply disables this feature. -n was selected because 2to3 has the same option with this behavior. files: A Misc/NEWS.d/next/Tools-Demos/2018-02-20-12-16-47.bpo-32885.dL5x7C.rst M Misc/ACKS M Tools/scripts/pathfix.py diff --git a/Misc/ACKS b/Misc/ACKS index d8179c8b03ab..d752d8a35434 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -687,6 +687,7 @@ Ken Howard Brad Howes Mike Hoy Ben Hoyt +Miro Hron?ok Chiu-Hsiang Hsu Chih-Hao Huang Christian Hudon diff --git a/Misc/NEWS.d/next/Tools-Demos/2018-02-20-12-16-47.bpo-32885.dL5x7C.rst b/Misc/NEWS.d/next/Tools-Demos/2018-02-20-12-16-47.bpo-32885.dL5x7C.rst new file mode 100644 index 000000000000..e003e1d84fd0 --- /dev/null +++ b/Misc/NEWS.d/next/Tools-Demos/2018-02-20-12-16-47.bpo-32885.dL5x7C.rst @@ -0,0 +1,2 @@ +Add an ``-n`` flag for ``Tools/scripts/pathfix.py`` to disbale automatic +backup creation (files with ``~`` suffix). diff --git a/Tools/scripts/pathfix.py b/Tools/scripts/pathfix.py index 562bbc737812..c5bf984306a3 100755 --- a/Tools/scripts/pathfix.py +++ b/Tools/scripts/pathfix.py @@ -7,8 +7,9 @@ # Directories are searched recursively for files whose name looks # like a python module. # Symbolic links are always ignored (except as explicit directory -# arguments). Of course, the original file is kept as a back-up -# (with a "~" attached to its name). +# arguments). +# The original file is kept as a back-up (with a "~" attached to its name), +# -n flag can be used to disable this. # # Undoubtedly you can do this using find and sed or perl, but this is # a nice example of Python code that recurses down a directory tree @@ -31,14 +32,17 @@ new_interpreter = None preserve_timestamps = False +create_backup = True + def main(): global new_interpreter global preserve_timestamps - usage = ('usage: %s -i /interpreter -p file-or-directory ...\n' % + global create_backup + usage = ('usage: %s -i /interpreter -p -n file-or-directory ...\n' % sys.argv[0]) try: - opts, args = getopt.getopt(sys.argv[1:], 'i:p') + opts, args = getopt.getopt(sys.argv[1:], 'i:pn') except getopt.error as msg: err(str(msg) + '\n') err(usage) @@ -48,6 +52,8 @@ def main(): new_interpreter = a.encode() if o == '-p': preserve_timestamps = True + if o == '-n': + create_backup = False if not new_interpreter or not new_interpreter.startswith(b'/') or \ not args: err('-i option or file-or-directory missing\n') @@ -134,10 +140,16 @@ def fix(filename): except OSError as msg: err('%s: warning: chmod failed (%r)\n' % (tempname, msg)) # Then make a backup of the original file as filename~ - try: - os.rename(filename, filename + '~') - except OSError as msg: - err('%s: warning: backup failed (%r)\n' % (filename, msg)) + if create_backup: + try: + os.rename(filename, filename + '~') + except OSError as msg: + err('%s: warning: backup failed (%r)\n' % (filename, msg)) + else: + try: + os.remove(filename) + except OSError as msg: + err('%s: warning: removing failed (%r)\n' % (filename, msg)) # Now move the temp file to the original file try: os.rename(tempname, filename) From webhook-mailer at python.org Tue Mar 13 06:12:19 2018 From: webhook-mailer at python.org (Miss Islington (bot)) Date: Tue, 13 Mar 2018 10:12:19 -0000 Subject: [Python-checkins] [3.7] bpo-17288: Prevent jumps from 'return' and 'exception' trace events. (GH-5928) Message-ID: <mailman.67.1520935942.1871.python-checkins@python.org> https://github.com/python/cpython/commit/cf61a81f1d600064be6dd43896afcf5f976de9b0 commit: cf61a81f1d600064be6dd43896afcf5f976de9b0 branch: 3.6 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: GitHub <noreply at github.com> date: 2018-03-13T03:12:16-07:00 summary: [3.7] bpo-17288: Prevent jumps from 'return' and 'exception' trace events. (GH-5928) (cherry picked from commit e32bbaf376a09c149fa7c7f2919d7c9ce4e2a055) Co-authored-by: xdegaye <xdegaye at gmail.com> files: A Misc/NEWS.d/next/Core and Builtins/2018-02-27-13-36-21.bpo-17288.Gdj24S.rst M Lib/test/test_sys_settrace.py M Objects/frameobject.c diff --git a/Lib/test/test_sys_settrace.py b/Lib/test/test_sys_settrace.py index a05fbbd04a84..46593cf54ad4 100644 --- a/Lib/test/test_sys_settrace.py +++ b/Lib/test/test_sys_settrace.py @@ -510,20 +510,35 @@ def g(frame, event, arg): class JumpTracer: """Defines a trace function that jumps from one place to another.""" - def __init__(self, function, jumpFrom, jumpTo): - self.function = function + def __init__(self, function, jumpFrom, jumpTo, event='line', + decorated=False): + self.code = function.__code__ self.jumpFrom = jumpFrom self.jumpTo = jumpTo + self.event = event + self.firstLine = None if decorated else self.code.co_firstlineno self.done = False def trace(self, frame, event, arg): - if not self.done and frame.f_code == self.function.__code__: - firstLine = frame.f_code.co_firstlineno - if event == 'line' and frame.f_lineno == firstLine + self.jumpFrom: + if self.done: + return + # frame.f_code.co_firstlineno is the first line of the decorator when + # 'function' is decorated and the decorator may be written using + # multiple physical lines when it is too long. Use the first line + # trace event in 'function' to find the first line of 'function'. + if (self.firstLine is None and frame.f_code == self.code and + event == 'line'): + self.firstLine = frame.f_lineno - 1 + if (event == self.event and self.firstLine and + frame.f_lineno == self.firstLine + self.jumpFrom): + f = frame + while f is not None and f.f_code != self.code: + f = f.f_back + if f is not None: # Cope with non-integer self.jumpTo (because of # no_jump_to_non_integers below). try: - frame.f_lineno = firstLine + self.jumpTo + frame.f_lineno = self.firstLine + self.jumpTo except TypeError: frame.f_lineno = self.jumpTo self.done = True @@ -563,8 +578,9 @@ def compare_jump_output(self, expected, received): "Expected: " + repr(expected) + "\n" + "Received: " + repr(received)) - def run_test(self, func, jumpFrom, jumpTo, expected, error=None): - tracer = JumpTracer(func, jumpFrom, jumpTo) + def run_test(self, func, jumpFrom, jumpTo, expected, error=None, + event='line', decorated=False): + tracer = JumpTracer(func, jumpFrom, jumpTo, event, decorated) sys.settrace(tracer.trace) output = [] if error is None: @@ -575,15 +591,15 @@ def run_test(self, func, jumpFrom, jumpTo, expected, error=None): sys.settrace(None) self.compare_jump_output(expected, output) - def jump_test(jumpFrom, jumpTo, expected, error=None): + def jump_test(jumpFrom, jumpTo, expected, error=None, event='line'): """Decorator that creates a test that makes a jump from one place to another in the following code. """ def decorator(func): @wraps(func) def test(self): - # +1 to compensate a decorator line - self.run_test(func, jumpFrom+1, jumpTo+1, expected, error) + self.run_test(func, jumpFrom, jumpTo, expected, + error=error, event=event, decorated=True) return test return decorator @@ -1058,6 +1074,36 @@ class fake_function: sys.settrace(None) self.compare_jump_output([2, 3, 2, 3, 4], namespace["output"]) + @jump_test(2, 3, [1], event='call', error=(ValueError, "can't jump from" + " the 'call' trace event of a new frame")) + def test_no_jump_from_call(output): + output.append(1) + def nested(): + output.append(3) + nested() + output.append(5) + + @jump_test(2, 1, [1], event='return', error=(ValueError, + "can only jump from a 'line' trace event")) + def test_no_jump_from_return_event(output): + output.append(1) + return + + @jump_test(2, 1, [1], event='exception', error=(ValueError, + "can only jump from a 'line' trace event")) + def test_no_jump_from_exception_event(output): + output.append(1) + 1 / 0 + + @jump_test(3, 2, [2], event='return', error=(ValueError, + "can't jump from a yield statement")) + def test_no_jump_from_yield(output): + def gen(): + output.append(2) + yield 3 + next(gen()) + output.append(5) + if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS.d/next/Core and Builtins/2018-02-27-13-36-21.bpo-17288.Gdj24S.rst b/Misc/NEWS.d/next/Core and Builtins/2018-02-27-13-36-21.bpo-17288.Gdj24S.rst new file mode 100644 index 000000000000..ce9e84c40313 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2018-02-27-13-36-21.bpo-17288.Gdj24S.rst @@ -0,0 +1 @@ +Prevent jumps from 'return' and 'exception' trace events. diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 5c73e8560f86..b7a16ad05966 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -56,6 +56,9 @@ frame_getlineno(PyFrameObject *f, void *closure) * o 'try'/'for'/'while' blocks can't be jumped into because the blockstack * needs to be set up before their code runs, and for 'for' loops the * iterator needs to be on the stack. + * o Jumps cannot be made from within a trace function invoked with a + * 'return' or 'exception' event since the eval loop has been exited at + * that time. */ static int frame_setlineno(PyFrameObject *f, PyObject* p_new_lineno) @@ -91,13 +94,32 @@ frame_setlineno(PyFrameObject *f, PyObject* p_new_lineno) return -1; } + /* Upon the 'call' trace event of a new frame, f->f_lasti is -1 and + * f->f_trace is NULL, check first on the first condition. + * Forbidding jumps from the 'call' event of a new frame is a side effect + * of allowing to set f_lineno only from trace functions. */ + if (f->f_lasti == -1) { + PyErr_Format(PyExc_ValueError, + "can't jump from the 'call' trace event of a new frame"); + return -1; + } + /* You can only do this from within a trace function, not via * _getframe or similar hackery. */ - if (!f->f_trace) - { + if (!f->f_trace) { PyErr_Format(PyExc_ValueError, - "f_lineno can only be set by a" - " line trace function"); + "f_lineno can only be set by a trace function"); + return -1; + } + + /* Forbid jumps upon a 'return' trace event (except after executing a + * YIELD_VALUE or YIELD_FROM opcode, f_stacktop is not NULL in that case) + * and upon an 'exception' trace event. + * Jumps from 'call' trace events have already been forbidden above for new + * frames, so this check does not change anything for 'call' events. */ + if (f->f_stacktop == NULL) { + PyErr_SetString(PyExc_ValueError, + "can only jump from a 'line' trace event"); return -1; } @@ -156,6 +178,16 @@ frame_setlineno(PyFrameObject *f, PyObject* p_new_lineno) /* We're now ready to look at the bytecode. */ PyBytes_AsStringAndSize(f->f_code->co_code, (char **)&code, &code_len); + + /* The trace function is called with a 'return' trace event after the + * execution of a yield statement. */ + assert(f->f_lasti != -1); + if (code[f->f_lasti] == YIELD_VALUE || code[f->f_lasti] == YIELD_FROM) { + PyErr_SetString(PyExc_ValueError, + "can't jump from a yield statement"); + return -1; + } + min_addr = Py_MIN(new_lasti, f->f_lasti); max_addr = Py_MAX(new_lasti, f->f_lasti); From webhook-mailer at python.org Tue Mar 13 13:31:36 2018 From: webhook-mailer at python.org (Serhiy Storchaka) Date: Tue, 13 Mar 2018 17:31:36 -0000 Subject: [Python-checkins] bpo-17288: Prevent jumps from 'return' and 'exception' trace events. (GH-6107) Message-ID: <mailman.68.1520962298.1871.python-checkins@python.org> https://github.com/python/cpython/commit/b8e9d6c5cd44ebc9c462fea9ad1bc5d0b970e28a commit: b8e9d6c5cd44ebc9c462fea9ad1bc5d0b970e28a branch: master author: xdegaye <xdegaye at gmail.com> committer: Serhiy Storchaka <storchaka at gmail.com> date: 2018-03-13T19:31:31+02:00 summary: bpo-17288: Prevent jumps from 'return' and 'exception' trace events. (GH-6107) (cherry picked from commit e32bbaf376a09c149fa7c7f2919d7c9ce4e2a055) files: A Misc/NEWS.d/next/Core and Builtins/2018-02-27-13-36-21.bpo-17288.Gdj24S.rst M Lib/test/test_sys_settrace.py M Objects/frameobject.c diff --git a/Lib/test/test_sys_settrace.py b/Lib/test/test_sys_settrace.py index 72cce33392dc..90d1e37a3916 100644 --- a/Lib/test/test_sys_settrace.py +++ b/Lib/test/test_sys_settrace.py @@ -555,20 +555,35 @@ def g(frame, event, arg): class JumpTracer: """Defines a trace function that jumps from one place to another.""" - def __init__(self, function, jumpFrom, jumpTo): - self.function = function + def __init__(self, function, jumpFrom, jumpTo, event='line', + decorated=False): + self.code = function.__code__ self.jumpFrom = jumpFrom self.jumpTo = jumpTo + self.event = event + self.firstLine = None if decorated else self.code.co_firstlineno self.done = False def trace(self, frame, event, arg): - if not self.done and frame.f_code == self.function.__code__: - firstLine = frame.f_code.co_firstlineno - if event == 'line' and frame.f_lineno == firstLine + self.jumpFrom: + if self.done: + return + # frame.f_code.co_firstlineno is the first line of the decorator when + # 'function' is decorated and the decorator may be written using + # multiple physical lines when it is too long. Use the first line + # trace event in 'function' to find the first line of 'function'. + if (self.firstLine is None and frame.f_code == self.code and + event == 'line'): + self.firstLine = frame.f_lineno - 1 + if (event == self.event and self.firstLine and + frame.f_lineno == self.firstLine + self.jumpFrom): + f = frame + while f is not None and f.f_code != self.code: + f = f.f_back + if f is not None: # Cope with non-integer self.jumpTo (because of # no_jump_to_non_integers below). try: - frame.f_lineno = firstLine + self.jumpTo + frame.f_lineno = self.firstLine + self.jumpTo except TypeError: frame.f_lineno = self.jumpTo self.done = True @@ -608,8 +623,9 @@ def compare_jump_output(self, expected, received): "Expected: " + repr(expected) + "\n" + "Received: " + repr(received)) - def run_test(self, func, jumpFrom, jumpTo, expected, error=None): - tracer = JumpTracer(func, jumpFrom, jumpTo) + def run_test(self, func, jumpFrom, jumpTo, expected, error=None, + event='line', decorated=False): + tracer = JumpTracer(func, jumpFrom, jumpTo, event, decorated) sys.settrace(tracer.trace) output = [] if error is None: @@ -620,15 +636,15 @@ def run_test(self, func, jumpFrom, jumpTo, expected, error=None): sys.settrace(None) self.compare_jump_output(expected, output) - def jump_test(jumpFrom, jumpTo, expected, error=None): + def jump_test(jumpFrom, jumpTo, expected, error=None, event='line'): """Decorator that creates a test that makes a jump from one place to another in the following code. """ def decorator(func): @wraps(func) def test(self): - # +1 to compensate a decorator line - self.run_test(func, jumpFrom+1, jumpTo+1, expected, error) + self.run_test(func, jumpFrom, jumpTo, expected, + error=error, event=event, decorated=True) return test return decorator @@ -1128,6 +1144,36 @@ class fake_function: sys.settrace(None) self.compare_jump_output([2, 3, 2, 3, 4], namespace["output"]) + @jump_test(2, 3, [1], event='call', error=(ValueError, "can't jump from" + " the 'call' trace event of a new frame")) + def test_no_jump_from_call(output): + output.append(1) + def nested(): + output.append(3) + nested() + output.append(5) + + @jump_test(2, 1, [1], event='return', error=(ValueError, + "can only jump from a 'line' trace event")) + def test_no_jump_from_return_event(output): + output.append(1) + return + + @jump_test(2, 1, [1], event='exception', error=(ValueError, + "can only jump from a 'line' trace event")) + def test_no_jump_from_exception_event(output): + output.append(1) + 1 / 0 + + @jump_test(3, 2, [2], event='return', error=(ValueError, + "can't jump from a yield statement")) + def test_no_jump_from_yield(output): + def gen(): + output.append(2) + yield 3 + next(gen()) + output.append(5) + if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS.d/next/Core and Builtins/2018-02-27-13-36-21.bpo-17288.Gdj24S.rst b/Misc/NEWS.d/next/Core and Builtins/2018-02-27-13-36-21.bpo-17288.Gdj24S.rst new file mode 100644 index 000000000000..ce9e84c40313 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2018-02-27-13-36-21.bpo-17288.Gdj24S.rst @@ -0,0 +1 @@ +Prevent jumps from 'return' and 'exception' trace events. diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 31ad8d0be226..643be08fa182 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -81,6 +81,9 @@ get_arg(const _Py_CODEUNIT *codestr, Py_ssize_t i) * the blockstack needs to be set up before their code runs. * o 'for' and 'async for' loops can't be jumped into because the * iterator needs to be on the stack. + * o Jumps cannot be made from within a trace function invoked with a + * 'return' or 'exception' event since the eval loop has been exited at + * that time. */ static int frame_setlineno(PyFrameObject *f, PyObject* p_new_lineno) @@ -109,13 +112,32 @@ frame_setlineno(PyFrameObject *f, PyObject* p_new_lineno) return -1; } + /* Upon the 'call' trace event of a new frame, f->f_lasti is -1 and + * f->f_trace is NULL, check first on the first condition. + * Forbidding jumps from the 'call' event of a new frame is a side effect + * of allowing to set f_lineno only from trace functions. */ + if (f->f_lasti == -1) { + PyErr_Format(PyExc_ValueError, + "can't jump from the 'call' trace event of a new frame"); + return -1; + } + /* You can only do this from within a trace function, not via * _getframe or similar hackery. */ - if (!f->f_trace) - { + if (!f->f_trace) { PyErr_Format(PyExc_ValueError, - "f_lineno can only be set by a" - " line trace function"); + "f_lineno can only be set by a trace function"); + return -1; + } + + /* Forbid jumps upon a 'return' trace event (except after executing a + * YIELD_VALUE or YIELD_FROM opcode, f_stacktop is not NULL in that case) + * and upon an 'exception' trace event. + * Jumps from 'call' trace events have already been forbidden above for new + * frames, so this check does not change anything for 'call' events. */ + if (f->f_stacktop == NULL) { + PyErr_SetString(PyExc_ValueError, + "can only jump from a 'line' trace event"); return -1; } @@ -175,6 +197,15 @@ frame_setlineno(PyFrameObject *f, PyObject* p_new_lineno) /* We're now ready to look at the bytecode. */ PyBytes_AsStringAndSize(f->f_code->co_code, (char **)&code, &code_len); + /* The trace function is called with a 'return' trace event after the + * execution of a yield statement. */ + assert(f->f_lasti != -1); + if (code[f->f_lasti] == YIELD_VALUE || code[f->f_lasti] == YIELD_FROM) { + PyErr_SetString(PyExc_ValueError, + "can't jump from a yield statement"); + return -1; + } + /* You can't jump onto a line with an 'except' statement on it - * they expect to have an exception on the top of the stack, which * won't be true if you jump to them. They always start with code From webhook-mailer at python.org Tue Mar 13 13:53:30 2018 From: webhook-mailer at python.org (=?utf-8?q?=C5=81ukasz?= Langa) Date: Tue, 13 Mar 2018 17:53:30 -0000 Subject: [Python-checkins] lib2to3: Add more tests (#6101) Message-ID: <mailman.69.1520963612.1871.python-checkins@python.org> https://github.com/python/cpython/commit/74f56878cdee18d485e4f8b485d55ce62e38f4c9 commit: 74f56878cdee18d485e4f8b485d55ce62e38f4c9 branch: master author: ?ukasz Langa <lukasz at langa.pl> committer: GitHub <noreply at github.com> date: 2018-03-13T10:53:22-07:00 summary: lib2to3: Add more tests (#6101) files: M Lib/lib2to3/tests/test_parser.py diff --git a/Lib/lib2to3/tests/test_parser.py b/Lib/lib2to3/tests/test_parser.py index cec2f98e9665..6813c65a7953 100644 --- a/Lib/lib2to3/tests/test_parser.py +++ b/Lib/lib2to3/tests/test_parser.py @@ -274,6 +274,78 @@ def test_argument_unpacking_2(self): def test_argument_unpacking_3(self): self.validate("""f(2, *a, *b, **b, **c, **d)""") + def test_trailing_commas_1(self): + self.validate("def f(a, b): call(a, b)") + self.validate("def f(a, b,): call(a, b,)") + + def test_trailing_commas_2(self): + self.validate("def f(a, *b): call(a, *b)") + self.validate("def f(a, *b,): call(a, *b,)") + + def test_trailing_commas_3(self): + self.validate("def f(a, b=1): call(a, b=1)") + self.validate("def f(a, b=1,): call(a, b=1,)") + + def test_trailing_commas_4(self): + self.validate("def f(a, **b): call(a, **b)") + self.validate("def f(a, **b,): call(a, **b,)") + + def test_trailing_commas_5(self): + self.validate("def f(*a, b=1): call(*a, b=1)") + self.validate("def f(*a, b=1,): call(*a, b=1,)") + + def test_trailing_commas_6(self): + self.validate("def f(*a, **b): call(*a, **b)") + self.validate("def f(*a, **b,): call(*a, **b,)") + + def test_trailing_commas_7(self): + self.validate("def f(*, b=1): call(*b)") + self.validate("def f(*, b=1,): call(*b,)") + + def test_trailing_commas_8(self): + self.validate("def f(a=1, b=2): call(a=1, b=2)") + self.validate("def f(a=1, b=2,): call(a=1, b=2,)") + + def test_trailing_commas_9(self): + self.validate("def f(a=1, **b): call(a=1, **b)") + self.validate("def f(a=1, **b,): call(a=1, **b,)") + + def test_trailing_commas_lambda_1(self): + self.validate("f = lambda a, b: call(a, b)") + self.validate("f = lambda a, b,: call(a, b,)") + + def test_trailing_commas_lambda_2(self): + self.validate("f = lambda a, *b: call(a, *b)") + self.validate("f = lambda a, *b,: call(a, *b,)") + + def test_trailing_commas_lambda_3(self): + self.validate("f = lambda a, b=1: call(a, b=1)") + self.validate("f = lambda a, b=1,: call(a, b=1,)") + + def test_trailing_commas_lambda_4(self): + self.validate("f = lambda a, **b: call(a, **b)") + self.validate("f = lambda a, **b,: call(a, **b,)") + + def test_trailing_commas_lambda_5(self): + self.validate("f = lambda *a, b=1: call(*a, b=1)") + self.validate("f = lambda *a, b=1,: call(*a, b=1,)") + + def test_trailing_commas_lambda_6(self): + self.validate("f = lambda *a, **b: call(*a, **b)") + self.validate("f = lambda *a, **b,: call(*a, **b,)") + + def test_trailing_commas_lambda_7(self): + self.validate("f = lambda *, b=1: call(*b)") + self.validate("f = lambda *, b=1,: call(*b,)") + + def test_trailing_commas_lambda_8(self): + self.validate("f = lambda a=1, b=2: call(a=1, b=2)") + self.validate("f = lambda a=1, b=2,: call(a=1, b=2,)") + + def test_trailing_commas_lambda_9(self): + self.validate("f = lambda a=1, **b: call(a=1, **b)") + self.validate("f = lambda a=1, **b,: call(a=1, **b,)") + # Adapted from Python 3's Lib/test/test_grammar.py:GrammarTests.testFuncdef class TestFunctionAnnotations(GrammarTest): @@ -335,6 +407,42 @@ def test_12(self): call(*args,)""" self.validate(s) + def test_13(self): + self.validate("def f(a: str, b: int) -> None: call(a, b)") + self.validate("def f(a: str, b: int,) -> None: call(a, b,)") + + def test_14(self): + self.validate("def f(a: str, *b: int) -> None: call(a, *b)") + self.validate("def f(a: str, *b: int,) -> None: call(a, *b,)") + + def test_15(self): + self.validate("def f(a: str, b: int=1) -> None: call(a, b=1)") + self.validate("def f(a: str, b: int=1,) -> None: call(a, b=1,)") + + def test_16(self): + self.validate("def f(a: str, **b: int) -> None: call(a, **b)") + self.validate("def f(a: str, **b: int,) -> None: call(a, **b,)") + + def test_17(self): + self.validate("def f(*a: str, b: int=1) -> None: call(*a, b=1)") + self.validate("def f(*a: str, b: int=1,) -> None: call(*a, b=1,)") + + def test_18(self): + self.validate("def f(*a: str, **b: int) -> None: call(*a, **b)") + self.validate("def f(*a: str, **b: int,) -> None: call(*a, **b,)") + + def test_19(self): + self.validate("def f(*, b: int=1) -> None: call(*b)") + self.validate("def f(*, b: int=1,) -> None: call(*b,)") + + def test_20(self): + self.validate("def f(a: str='', b: int=2) -> None: call(a=a, b=2)") + self.validate("def f(a: str='', b: int=2,) -> None: call(a=a, b=2,)") + + def test_21(self): + self.validate("def f(a: str='', **b: int) -> None: call(a=a, **b)") + self.validate("def f(a: str='', **b: int,) -> None: call(a=a, **b,)") + # Adapted from Python 3's Lib/test/test_grammar.py:GrammarTests.test_var_annot class TestVarAnnotations(GrammarTest): From webhook-mailer at python.org Tue Mar 13 14:20:41 2018 From: webhook-mailer at python.org (=?utf-8?q?=C5=81ukasz?= Langa) Date: Tue, 13 Mar 2018 18:20:41 -0000 Subject: [Python-checkins] lib2to3: Add more tests (GH-6101) (#6108) Message-ID: <mailman.70.1520965243.1871.python-checkins@python.org> https://github.com/python/cpython/commit/7d0528dd3f054b47d6689c1dd75ef267a72e9bb5 commit: 7d0528dd3f054b47d6689c1dd75ef267a72e9bb5 branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: ?ukasz Langa <lukasz at langa.pl> date: 2018-03-13T11:20:38-07:00 summary: lib2to3: Add more tests (GH-6101) (#6108) (cherry picked from commit 74f56878cdee18d485e4f8b485d55ce62e38f4c9) Co-authored-by: ?ukasz Langa <lukasz at langa.pl> files: M Lib/lib2to3/tests/test_parser.py diff --git a/Lib/lib2to3/tests/test_parser.py b/Lib/lib2to3/tests/test_parser.py index cec2f98e9665..6813c65a7953 100644 --- a/Lib/lib2to3/tests/test_parser.py +++ b/Lib/lib2to3/tests/test_parser.py @@ -274,6 +274,78 @@ def test_argument_unpacking_2(self): def test_argument_unpacking_3(self): self.validate("""f(2, *a, *b, **b, **c, **d)""") + def test_trailing_commas_1(self): + self.validate("def f(a, b): call(a, b)") + self.validate("def f(a, b,): call(a, b,)") + + def test_trailing_commas_2(self): + self.validate("def f(a, *b): call(a, *b)") + self.validate("def f(a, *b,): call(a, *b,)") + + def test_trailing_commas_3(self): + self.validate("def f(a, b=1): call(a, b=1)") + self.validate("def f(a, b=1,): call(a, b=1,)") + + def test_trailing_commas_4(self): + self.validate("def f(a, **b): call(a, **b)") + self.validate("def f(a, **b,): call(a, **b,)") + + def test_trailing_commas_5(self): + self.validate("def f(*a, b=1): call(*a, b=1)") + self.validate("def f(*a, b=1,): call(*a, b=1,)") + + def test_trailing_commas_6(self): + self.validate("def f(*a, **b): call(*a, **b)") + self.validate("def f(*a, **b,): call(*a, **b,)") + + def test_trailing_commas_7(self): + self.validate("def f(*, b=1): call(*b)") + self.validate("def f(*, b=1,): call(*b,)") + + def test_trailing_commas_8(self): + self.validate("def f(a=1, b=2): call(a=1, b=2)") + self.validate("def f(a=1, b=2,): call(a=1, b=2,)") + + def test_trailing_commas_9(self): + self.validate("def f(a=1, **b): call(a=1, **b)") + self.validate("def f(a=1, **b,): call(a=1, **b,)") + + def test_trailing_commas_lambda_1(self): + self.validate("f = lambda a, b: call(a, b)") + self.validate("f = lambda a, b,: call(a, b,)") + + def test_trailing_commas_lambda_2(self): + self.validate("f = lambda a, *b: call(a, *b)") + self.validate("f = lambda a, *b,: call(a, *b,)") + + def test_trailing_commas_lambda_3(self): + self.validate("f = lambda a, b=1: call(a, b=1)") + self.validate("f = lambda a, b=1,: call(a, b=1,)") + + def test_trailing_commas_lambda_4(self): + self.validate("f = lambda a, **b: call(a, **b)") + self.validate("f = lambda a, **b,: call(a, **b,)") + + def test_trailing_commas_lambda_5(self): + self.validate("f = lambda *a, b=1: call(*a, b=1)") + self.validate("f = lambda *a, b=1,: call(*a, b=1,)") + + def test_trailing_commas_lambda_6(self): + self.validate("f = lambda *a, **b: call(*a, **b)") + self.validate("f = lambda *a, **b,: call(*a, **b,)") + + def test_trailing_commas_lambda_7(self): + self.validate("f = lambda *, b=1: call(*b)") + self.validate("f = lambda *, b=1,: call(*b,)") + + def test_trailing_commas_lambda_8(self): + self.validate("f = lambda a=1, b=2: call(a=1, b=2)") + self.validate("f = lambda a=1, b=2,: call(a=1, b=2,)") + + def test_trailing_commas_lambda_9(self): + self.validate("f = lambda a=1, **b: call(a=1, **b)") + self.validate("f = lambda a=1, **b,: call(a=1, **b,)") + # Adapted from Python 3's Lib/test/test_grammar.py:GrammarTests.testFuncdef class TestFunctionAnnotations(GrammarTest): @@ -335,6 +407,42 @@ def test_12(self): call(*args,)""" self.validate(s) + def test_13(self): + self.validate("def f(a: str, b: int) -> None: call(a, b)") + self.validate("def f(a: str, b: int,) -> None: call(a, b,)") + + def test_14(self): + self.validate("def f(a: str, *b: int) -> None: call(a, *b)") + self.validate("def f(a: str, *b: int,) -> None: call(a, *b,)") + + def test_15(self): + self.validate("def f(a: str, b: int=1) -> None: call(a, b=1)") + self.validate("def f(a: str, b: int=1,) -> None: call(a, b=1,)") + + def test_16(self): + self.validate("def f(a: str, **b: int) -> None: call(a, **b)") + self.validate("def f(a: str, **b: int,) -> None: call(a, **b,)") + + def test_17(self): + self.validate("def f(*a: str, b: int=1) -> None: call(*a, b=1)") + self.validate("def f(*a: str, b: int=1,) -> None: call(*a, b=1,)") + + def test_18(self): + self.validate("def f(*a: str, **b: int) -> None: call(*a, **b)") + self.validate("def f(*a: str, **b: int,) -> None: call(*a, **b,)") + + def test_19(self): + self.validate("def f(*, b: int=1) -> None: call(*b)") + self.validate("def f(*, b: int=1,) -> None: call(*b,)") + + def test_20(self): + self.validate("def f(a: str='', b: int=2) -> None: call(a=a, b=2)") + self.validate("def f(a: str='', b: int=2,) -> None: call(a=a, b=2,)") + + def test_21(self): + self.validate("def f(a: str='', **b: int) -> None: call(a=a, **b)") + self.validate("def f(a: str='', **b: int,) -> None: call(a=a, **b,)") + # Adapted from Python 3's Lib/test/test_grammar.py:GrammarTests.test_var_annot class TestVarAnnotations(GrammarTest): From webhook-mailer at python.org Tue Mar 13 14:47:42 2018 From: webhook-mailer at python.org (=?utf-8?q?=C5=81ukasz?= Langa) Date: Tue, 13 Mar 2018 18:47:42 -0000 Subject: [Python-checkins] lib2to3: Add more tests (GH-6101) (#6109) Message-ID: <mailman.71.1520966863.1871.python-checkins@python.org> https://github.com/python/cpython/commit/cad3eb2026e5bf1a5001e1d21b8c4e33d5f89c4f commit: cad3eb2026e5bf1a5001e1d21b8c4e33d5f89c4f branch: 3.6 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: ?ukasz Langa <lukasz at langa.pl> date: 2018-03-13T11:47:32-07:00 summary: lib2to3: Add more tests (GH-6101) (#6109) (cherry picked from commit 74f56878cdee18d485e4f8b485d55ce62e38f4c9) Co-authored-by: ?ukasz Langa <lukasz at langa.pl> files: M Lib/lib2to3/tests/test_parser.py diff --git a/Lib/lib2to3/tests/test_parser.py b/Lib/lib2to3/tests/test_parser.py index c4d062ff827e..1c9a17e43abc 100644 --- a/Lib/lib2to3/tests/test_parser.py +++ b/Lib/lib2to3/tests/test_parser.py @@ -274,6 +274,78 @@ def test_argument_unpacking_2(self): def test_argument_unpacking_3(self): self.validate("""f(2, *a, *b, **b, **c, **d)""") + def test_trailing_commas_1(self): + self.validate("def f(a, b): call(a, b)") + self.validate("def f(a, b,): call(a, b,)") + + def test_trailing_commas_2(self): + self.validate("def f(a, *b): call(a, *b)") + self.validate("def f(a, *b,): call(a, *b,)") + + def test_trailing_commas_3(self): + self.validate("def f(a, b=1): call(a, b=1)") + self.validate("def f(a, b=1,): call(a, b=1,)") + + def test_trailing_commas_4(self): + self.validate("def f(a, **b): call(a, **b)") + self.validate("def f(a, **b,): call(a, **b,)") + + def test_trailing_commas_5(self): + self.validate("def f(*a, b=1): call(*a, b=1)") + self.validate("def f(*a, b=1,): call(*a, b=1,)") + + def test_trailing_commas_6(self): + self.validate("def f(*a, **b): call(*a, **b)") + self.validate("def f(*a, **b,): call(*a, **b,)") + + def test_trailing_commas_7(self): + self.validate("def f(*, b=1): call(*b)") + self.validate("def f(*, b=1,): call(*b,)") + + def test_trailing_commas_8(self): + self.validate("def f(a=1, b=2): call(a=1, b=2)") + self.validate("def f(a=1, b=2,): call(a=1, b=2,)") + + def test_trailing_commas_9(self): + self.validate("def f(a=1, **b): call(a=1, **b)") + self.validate("def f(a=1, **b,): call(a=1, **b,)") + + def test_trailing_commas_lambda_1(self): + self.validate("f = lambda a, b: call(a, b)") + self.validate("f = lambda a, b,: call(a, b,)") + + def test_trailing_commas_lambda_2(self): + self.validate("f = lambda a, *b: call(a, *b)") + self.validate("f = lambda a, *b,: call(a, *b,)") + + def test_trailing_commas_lambda_3(self): + self.validate("f = lambda a, b=1: call(a, b=1)") + self.validate("f = lambda a, b=1,: call(a, b=1,)") + + def test_trailing_commas_lambda_4(self): + self.validate("f = lambda a, **b: call(a, **b)") + self.validate("f = lambda a, **b,: call(a, **b,)") + + def test_trailing_commas_lambda_5(self): + self.validate("f = lambda *a, b=1: call(*a, b=1)") + self.validate("f = lambda *a, b=1,: call(*a, b=1,)") + + def test_trailing_commas_lambda_6(self): + self.validate("f = lambda *a, **b: call(*a, **b)") + self.validate("f = lambda *a, **b,: call(*a, **b,)") + + def test_trailing_commas_lambda_7(self): + self.validate("f = lambda *, b=1: call(*b)") + self.validate("f = lambda *, b=1,: call(*b,)") + + def test_trailing_commas_lambda_8(self): + self.validate("f = lambda a=1, b=2: call(a=1, b=2)") + self.validate("f = lambda a=1, b=2,: call(a=1, b=2,)") + + def test_trailing_commas_lambda_9(self): + self.validate("f = lambda a=1, **b: call(a=1, **b)") + self.validate("f = lambda a=1, **b,: call(a=1, **b,)") + # Adapted from Python 3's Lib/test/test_grammar.py:GrammarTests.testFuncdef class TestFunctionAnnotations(GrammarTest): @@ -335,6 +407,42 @@ def test_12(self): call(*args,)""" self.validate(s) + def test_13(self): + self.validate("def f(a: str, b: int) -> None: call(a, b)") + self.validate("def f(a: str, b: int,) -> None: call(a, b,)") + + def test_14(self): + self.validate("def f(a: str, *b: int) -> None: call(a, *b)") + self.validate("def f(a: str, *b: int,) -> None: call(a, *b,)") + + def test_15(self): + self.validate("def f(a: str, b: int=1) -> None: call(a, b=1)") + self.validate("def f(a: str, b: int=1,) -> None: call(a, b=1,)") + + def test_16(self): + self.validate("def f(a: str, **b: int) -> None: call(a, **b)") + self.validate("def f(a: str, **b: int,) -> None: call(a, **b,)") + + def test_17(self): + self.validate("def f(*a: str, b: int=1) -> None: call(*a, b=1)") + self.validate("def f(*a: str, b: int=1,) -> None: call(*a, b=1,)") + + def test_18(self): + self.validate("def f(*a: str, **b: int) -> None: call(*a, **b)") + self.validate("def f(*a: str, **b: int,) -> None: call(*a, **b,)") + + def test_19(self): + self.validate("def f(*, b: int=1) -> None: call(*b)") + self.validate("def f(*, b: int=1,) -> None: call(*b,)") + + def test_20(self): + self.validate("def f(a: str='', b: int=2) -> None: call(a=a, b=2)") + self.validate("def f(a: str='', b: int=2,) -> None: call(a=a, b=2,)") + + def test_21(self): + self.validate("def f(a: str='', **b: int) -> None: call(a=a, **b)") + self.validate("def f(a: str='', **b: int,) -> None: call(a=a, **b,)") + # Adapted from Python 3's Lib/test/test_grammar.py:GrammarTests.test_var_annot class TestVarAnnotations(GrammarTest): From webhook-mailer at python.org Tue Mar 13 17:06:17 2018 From: webhook-mailer at python.org (Serhiy Storchaka) Date: Tue, 13 Mar 2018 21:06:17 -0000 Subject: [Python-checkins] [2.7] bpo-17288: Prevent jumps from 'return' and 'exception' trace events. (GH-6111) Message-ID: <mailman.72.1520975179.1871.python-checkins@python.org> https://github.com/python/cpython/commit/baca85fcc7cdf70af4a76ea0966d4842c173de1a commit: baca85fcc7cdf70af4a76ea0966d4842c173de1a branch: 2.7 author: xdegaye <xdegaye at gmail.com> committer: Serhiy Storchaka <storchaka at gmail.com> date: 2018-03-13T23:06:14+02:00 summary: [2.7] bpo-17288: Prevent jumps from 'return' and 'exception' trace events. (GH-6111) (cherry picked from commit e32bbaf376a09c149fa7c7f2919d7c9ce4e2a055) files: A Misc/NEWS.d/next/Core and Builtins/2018-02-27-13-36-21.bpo-17288.Gdj24S.rst M Lib/test/test_sys_settrace.py M Objects/frameobject.c diff --git a/Lib/test/test_sys_settrace.py b/Lib/test/test_sys_settrace.py index 04b1ac339a49..d8737af3bc8d 100644 --- a/Lib/test/test_sys_settrace.py +++ b/Lib/test/test_sys_settrace.py @@ -482,20 +482,35 @@ def g(frame, why, extra): class JumpTracer: """Defines a trace function that jumps from one place to another.""" - def __init__(self, function, jumpFrom, jumpTo): - self.function = function + def __init__(self, function, jumpFrom, jumpTo, event='line', + decorated=False): + self.code = function.func_code self.jumpFrom = jumpFrom self.jumpTo = jumpTo + self.event = event + self.firstLine = None if decorated else self.code.co_firstlineno self.done = False def trace(self, frame, event, arg): - if not self.done and frame.f_code == self.function.func_code: - firstLine = frame.f_code.co_firstlineno - if event == 'line' and frame.f_lineno == firstLine + self.jumpFrom: + if self.done: + return + # frame.f_code.co_firstlineno is the first line of the decorator when + # 'function' is decorated and the decorator may be written using + # multiple physical lines when it is too long. Use the first line + # trace event in 'function' to find the first line of 'function'. + if (self.firstLine is None and frame.f_code == self.code and + event == 'line'): + self.firstLine = frame.f_lineno - 1 + if (event == self.event and self.firstLine and + frame.f_lineno == self.firstLine + self.jumpFrom): + f = frame + while f is not None and f.f_code != self.code: + f = f.f_back + if f is not None: # Cope with non-integer self.jumpTo (because of # no_jump_to_non_integers below). try: - frame.f_lineno = firstLine + self.jumpTo + frame.f_lineno = self.firstLine + self.jumpTo except TypeError: frame.f_lineno = self.jumpTo self.done = True @@ -535,8 +550,9 @@ def compare_jump_output(self, expected, received): "Expected: " + repr(expected) + "\n" + "Received: " + repr(received)) - def run_test(self, func, jumpFrom, jumpTo, expected, error=None): - tracer = JumpTracer(func, jumpFrom, jumpTo) + def run_test(self, func, jumpFrom, jumpTo, expected, error=None, + event='line', decorated=False): + tracer = JumpTracer(func, jumpFrom, jumpTo, event, decorated) sys.settrace(tracer.trace) output = [] if error is None: @@ -547,15 +563,15 @@ def run_test(self, func, jumpFrom, jumpTo, expected, error=None): sys.settrace(None) self.compare_jump_output(expected, output) - def jump_test(jumpFrom, jumpTo, expected, error=None): + def jump_test(jumpFrom, jumpTo, expected, error=None, event='line'): """Decorator that creates a test that makes a jump from one place to another in the following code. """ def decorator(func): @wraps(func) def test(self): - # +1 to compensate a decorator line - self.run_test(func, jumpFrom+1, jumpTo+1, expected, error) + self.run_test(func, jumpFrom, jumpTo, expected, + error=error, event=event, decorated=True) return test return decorator @@ -1018,6 +1034,36 @@ class fake_function: sys.settrace(None) self.compare_jump_output([2, 3, 2, 3, 4], namespace["output"]) + @jump_test(2, 3, [1], event='call', error=(ValueError, "can't jump from" + " the 'call' trace event of a new frame")) + def test_no_jump_from_call(output): + output.append(1) + def nested(): + output.append(3) + nested() + output.append(5) + + @jump_test(2, 1, [1], event='return', error=(ValueError, + "can only jump from a 'line' trace event")) + def test_no_jump_from_return_event(output): + output.append(1) + return + + @jump_test(2, 1, [1], event='exception', error=(ValueError, + "can only jump from a 'line' trace event")) + def test_no_jump_from_exception_event(output): + output.append(1) + 1 / 0 + + @jump_test(3, 2, [2], event='return', error=(ValueError, + "can't jump from a yield statement")) + def test_no_jump_from_yield(output): + def gen(): + output.append(2) + yield 3 + next(gen()) + output.append(5) + def test_main(): test_support.run_unittest( diff --git a/Misc/NEWS.d/next/Core and Builtins/2018-02-27-13-36-21.bpo-17288.Gdj24S.rst b/Misc/NEWS.d/next/Core and Builtins/2018-02-27-13-36-21.bpo-17288.Gdj24S.rst new file mode 100644 index 000000000000..ce9e84c40313 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2018-02-27-13-36-21.bpo-17288.Gdj24S.rst @@ -0,0 +1 @@ +Prevent jumps from 'return' and 'exception' trace events. diff --git a/Objects/frameobject.c b/Objects/frameobject.c index bf1c7c52737d..175874503bb2 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -89,6 +89,9 @@ frame_getlineno(PyFrameObject *f, void *closure) * o 'try'/'for'/'while' blocks can't be jumped into because the blockstack * needs to be set up before their code runs, and for 'for' loops the * iterator needs to be on the stack. + * o Jumps cannot be made from within a trace function invoked with a + * 'return' or 'exception' event since the eval loop has been exited at + * that time. */ static int frame_setlineno(PyFrameObject *f, PyObject* p_new_lineno) @@ -122,13 +125,32 @@ frame_setlineno(PyFrameObject *f, PyObject* p_new_lineno) return -1; } + /* Upon the 'call' trace event of a new frame, f->f_lasti is -1 and + * f->f_trace is NULL, check first on the first condition. + * Forbidding jumps from the 'call' event of a new frame is a side effect + * of allowing to set f_lineno only from trace functions. */ + if (f->f_lasti == -1) { + PyErr_Format(PyExc_ValueError, + "can't jump from the 'call' trace event of a new frame"); + return -1; + } + /* You can only do this from within a trace function, not via * _getframe or similar hackery. */ - if (!f->f_trace) - { + if (!f->f_trace) { PyErr_Format(PyExc_ValueError, - "f_lineno can only be set by a" - " line trace function"); + "f_lineno can only be set by a trace function"); + return -1; + } + + /* Forbid jumps upon a 'return' trace event (except after executing a + * YIELD_VALUE opcode, f_stacktop is not NULL in that case) and upon an + * 'exception' trace event. + * Jumps from 'call' trace events have already been forbidden above for new + * frames, so this check does not change anything for 'call' events. */ + if (f->f_stacktop == NULL) { + PyErr_SetString(PyExc_ValueError, + "can only jump from a 'line' trace event"); return -1; } @@ -178,6 +200,15 @@ frame_setlineno(PyFrameObject *f, PyObject* p_new_lineno) min_addr = MIN(new_lasti, f->f_lasti); max_addr = MAX(new_lasti, f->f_lasti); + /* The trace function is called with a 'return' trace event after the + * execution of a yield statement. */ + assert(f->f_lasti != -1); + if (code[f->f_lasti] == YIELD_VALUE) { + PyErr_SetString(PyExc_ValueError, + "can't jump from a yield statement"); + return -1; + } + /* You can't jump onto a line with an 'except' statement on it - * they expect to have an exception on the top of the stack, which * won't be true if you jump to them. They always start with code From webhook-mailer at python.org Tue Mar 13 23:40:28 2018 From: webhook-mailer at python.org (Ned Deily) Date: Wed, 14 Mar 2018 03:40:28 -0000 Subject: [Python-checkins] bpo-32726: macOS installer changes for 3.6.5 Message-ID: <mailman.73.1520998829.1871.python-checkins@python.org> https://github.com/python/cpython/commit/94552448d7bcc1eebc53b608e89d96e235054f2f commit: 94552448d7bcc1eebc53b608e89d96e235054f2f branch: 3.6 author: Ned Deily <nad at python.org> committer: Ned Deily <nad at python.org> date: 2018-03-13T21:01:08-04:00 summary: bpo-32726: macOS installer changes for 3.6.5 Backport the new 10.9+ installer variant from 3.7. This variant features more modern options; like 64-bit only (Apple is deprecating 32-bit support in future macOS releases); a built-in version of Tcl/Tk 8.6.8; built with clang rather than gcc-4.2. For 3.6.5, the 10.9+ variant will be offered as an additional alternative to the traditional 10.6+ variant in earlier 3.6.x releases. Binary extension modules (including wheels) built for earlier versions of 3.6.x with the 10.6 variant should continue to work with either 3.6.5 variant without recompilation. In addition, both installer variants have updated 3rd-party libraries: OpenSSL 1.0.2m -> 1.0.2n XZ 5.2.2 -> 5.2.3 SQLite 3.21.0 -> 3.22.0 Also the 10.6 variant now sets CC=gcc instead of CC=gcc-4.2 and does not search for the outdated 10.6 SDK. The variant is built with the same compiler as before. As before, for extension module builds, the CC can be overridden with the CC env variable and an SDK can be specified with the SDKROOT env variable (see man xcrun). These minor changes should be transparent to nearly all users. files: A Mac/BuildScript/resources/Conclusion.rtf A Mac/BuildScript/tk868_on_10_8_10_9.patch A Misc/NEWS.d/next/macOS/2018-03-13-21-00-20.bpo-32726.Mticyn.rst D Mac/BuildScript/issue19373_tk_8_5_15_source.patch D Mac/BuildScript/openssl_sdk_makedepend.patch M Mac/BuildScript/build-installer.py M Mac/BuildScript/resources/ReadMe.rtf M Mac/BuildScript/resources/Welcome.rtf M Mac/BuildScript/scripts/postflight.documentation M configure M configure.ac diff --git a/Mac/BuildScript/build-installer.py b/Mac/BuildScript/build-installer.py index ff02fb37126b..c33737e9d042 100755 --- a/Mac/BuildScript/build-installer.py +++ b/Mac/BuildScript/build-installer.py @@ -1,30 +1,36 @@ #!/usr/bin/env python """ -This script is used to build "official" universal installers on Mac OS X. -It requires at least Mac OS X 10.5, Xcode 3, and the 10.4u SDK for -32-bit builds. 64-bit or four-way universal builds require at least -OS X 10.5 and the 10.5 SDK. +This script is used to build "official" universal installers on macOS. + +NEW for 3.6.5: +- support Intel 64-bit-only () and 32-bit-only installer builds +- build and link with private Tcl/Tk 8.6 for 10.9+ builds +- deprecate use of explicit SDK (--sdk-path=) since all but the oldest + versions of Xcode support implicit setting of an SDK via environment + variables (SDKROOT and friends, see the xcrun man page for more info). + The SDK stuff was primarily needed for building universal installers + for 10.4; so as of 3.6.5, building installers for 10.4 is no longer + supported with build-installer. +- use generic "gcc" as compiler (CC env var) rather than "gcc-4.2" Please ensure that this script keeps working with Python 2.5, to avoid -bootstrap issues (/usr/bin/python is Python 2.5 on OSX 10.5). Sphinx, -which is used to build the documentation, currently requires at least -Python 2.4. However, as of Python 3.4.1, Doc builds require an external -sphinx-build and the current versions of Sphinx now require at least -Python 2.6. - -In addition to what is supplied with OS X 10.5+ and Xcode 3+, the script -requires an installed third-party version of -Tcl/Tk 8.4 (for OS X 10.4 and 10.5 deployment targets) or Tcl/TK 8.5 +bootstrap issues (/usr/bin/python is Python 2.5 on OSX 10.5). Doc builds +use current versions of Sphinx and require a reasonably current python3. +Sphinx and dependencies are installed into a venv using the python3's pip +so will fetch them from PyPI if necessary. Since python3 is now used for +Sphinx, build-installer.py should also be converted to use python3! + +For 10.9 or greater deployment targets, build-installer builds and links +with its own copy of Tcl/Tk 8.5 and the rest of this paragraph does not +apply. Otherwise, build-installer requires an installed third-party version +of Tcl/Tk 8.4 (for OS X 10.4 and 10.5 deployment targets) or Tcl/TK 8.5 (for 10.6 or later) installed in /Library/Frameworks. When installed, the Python built by this script will attempt to dynamically link first to Tcl and Tk frameworks in /Library/Frameworks if available otherwise fall back to the ones in /System/Library/Framework. For the build, we recommend -installing the most recent ActiveTcl 8.4 or 8.5 version. - -32-bit-only installer builds are still possible on OS X 10.4 with Xcode 2.5 -and the installation of additional components, such as a newer Python -(2.5 is needed for Python parser updates) and for the documentation -build either svn (pre-3.4.1) or sphinx-build (3.4.1 and later). +installing the most recent ActiveTcl 8.5 or 8.4 version, depending +on the deployment target. The actual version linked to depends on the +path of /Library/Frameworks/{Tcl,Tk}.framework/Versions/Current. Usage: see USAGE variable in the script. """ @@ -111,32 +117,19 @@ def getFullVersion(): DEPSRC = os.path.join(WORKDIR, 'third-party') DEPSRC = os.path.expanduser('~/Universal/other-sources') -# Location of the preferred SDK - -### There are some issues with the SDK selection below here, -### The resulting binary doesn't work on all platforms that -### it should. Always default to the 10.4u SDK until that -### issue is resolved. -### -##if int(os.uname()[2].split('.')[0]) == 8: -## # Explicitly use the 10.4u (universal) SDK when -## # building on 10.4, the system headers are not -## # useable for a universal build -## SDKPATH = "/Developer/SDKs/MacOSX10.4u.sdk" -##else: -## SDKPATH = "/" - -SDKPATH = "/Developer/SDKs/MacOSX10.4u.sdk" - universal_opts_map = { '32-bit': ('i386', 'ppc',), '64-bit': ('x86_64', 'ppc64',), 'intel': ('i386', 'x86_64'), + 'intel-32': ('i386',), + 'intel-64': ('x86_64',), '3-way': ('ppc', 'i386', 'x86_64'), 'all': ('i386', 'ppc', 'x86_64', 'ppc64',) } default_target_map = { '64-bit': '10.5', '3-way': '10.5', 'intel': '10.5', + 'intel-32': '10.4', + 'intel-64': '10.5', 'all': '10.5', } @@ -154,19 +147,18 @@ def getFullVersion(): )))) # $MACOSX_DEPLOYMENT_TARGET -> minimum OS X level -DEPTARGET = '10.3' +DEPTARGET = '10.5' def getDeptargetTuple(): return tuple([int(n) for n in DEPTARGET.split('.')[0:2]]) def getTargetCompilers(): target_cc_map = { - '10.3': ('gcc-4.0', 'g++-4.0'), '10.4': ('gcc-4.0', 'g++-4.0'), - '10.5': ('gcc-4.2', 'g++-4.2'), - '10.6': ('gcc-4.2', 'g++-4.2'), + '10.5': ('gcc', 'g++'), + '10.6': ('gcc', 'g++'), } - return target_cc_map.get(DEPTARGET, ('clang', 'clang++') ) + return target_cc_map.get(DEPTARGET, ('gcc', 'g++') ) CC, CXX = getTargetCompilers() @@ -180,9 +172,9 @@ def getTargetCompilers(): -b DIR --build-dir=DIR: Create build here (default: %(WORKDIR)r) --third-party=DIR: Store third-party sources here (default: %(DEPSRC)r) - --sdk-path=DIR: Location of the SDK (default: %(SDKPATH)r) + --sdk-path=DIR: Location of the SDK (deprecated, use SDKROOT env variable) --src-dir=DIR: Location of the Python sources (default: %(SRCDIR)r) - --dep-target=10.n OS X deployment target (default: %(DEPTARGET)r) + --dep-target=10.n macOS deployment target (default: %(DEPTARGET)r) --universal-archs=x universal architectures (options: %(UNIVERSALOPTS)r, default: %(UNIVERSALARCHS)r) """)% globals() @@ -194,6 +186,11 @@ def getTargetCompilers(): # '/Library/Frameworks/Tk.framework/Versions/8.5/Tk'] EXPECTED_SHARED_LIBS = {} +# Are we building and linking with our own copy of Tcl/TK? +# For now, do so if deployment target is 10.9+. +def internalTk(): + return getDeptargetTuple() >= (10, 9) + # List of names of third party software built with this installer. # The names will be inserted into the rtf version of the License. THIRD_PARTY_LIBS = [] @@ -213,25 +210,21 @@ def library_recipes(): result.extend([ dict( - name="OpenSSL 1.0.2m", - url="https://www.openssl.org/source/openssl-1.0.2m.tar.gz", - checksum='10e9e37f492094b9ef296f68f24a7666', - patches=[ - "openssl_sdk_makedepend.patch", - ], + name="OpenSSL 1.0.2n", + url="https://www.openssl.org/source/openssl-1.0.2n.tar.gz", + checksum='13bdc1b1d1ff39b6fd42a255e74676a4', buildrecipe=build_universal_openssl, configure=None, install=None, ), ]) -# Disable for now - if False: # if getDeptargetTuple() > (10, 5): + if internalTk(): result.extend([ dict( - name="Tcl 8.5.15", - url="ftp://ftp.tcl.tk/pub/tcl//tcl8_5/tcl8.5.15-src.tar.gz", - checksum='f3df162f92c69b254079c4d0af7a690f', + name="Tcl 8.6.8", + url="ftp://ftp.tcl.tk/pub/tcl//tcl8_6/tcl8.6.8-src.tar.gz", + checksum='81656d3367af032e0ae6157eff134f89', buildDir="unix", configure_pre=[ '--enable-shared', @@ -241,15 +234,15 @@ def library_recipes(): useLDFlags=False, install='make TCL_LIBRARY=%(TCL_LIBRARY)s && make install TCL_LIBRARY=%(TCL_LIBRARY)s DESTDIR=%(DESTDIR)s'%{ "DESTDIR": shellQuote(os.path.join(WORKDIR, 'libraries')), - "TCL_LIBRARY": shellQuote('/Library/Frameworks/Python.framework/Versions/%s/lib/tcl8.5'%(getVersion())), + "TCL_LIBRARY": shellQuote('/Library/Frameworks/Python.framework/Versions/%s/lib/tcl8.6'%(getVersion())), }, ), dict( - name="Tk 8.5.15", - url="ftp://ftp.tcl.tk/pub/tcl//tcl8_5/tk8.5.15-src.tar.gz", - checksum='55b8e33f903210a4e1c8bce0f820657f', + name="Tk 8.6.8", + url="ftp://ftp.tcl.tk/pub/tcl//tcl8_6/tk8.6.8-src.tar.gz", + checksum='5e0faecba458ee1386078fb228d008ba', patches=[ - "issue19373_tk_8_5_15_source.patch", + "tk868_on_10_8_10_9.patch", ], buildDir="unix", configure_pre=[ @@ -261,8 +254,8 @@ def library_recipes(): useLDFlags=False, install='make TCL_LIBRARY=%(TCL_LIBRARY)s TK_LIBRARY=%(TK_LIBRARY)s && make install TCL_LIBRARY=%(TCL_LIBRARY)s TK_LIBRARY=%(TK_LIBRARY)s DESTDIR=%(DESTDIR)s'%{ "DESTDIR": shellQuote(os.path.join(WORKDIR, 'libraries')), - "TCL_LIBRARY": shellQuote('/Library/Frameworks/Python.framework/Versions/%s/lib/tcl8.5'%(getVersion())), - "TK_LIBRARY": shellQuote('/Library/Frameworks/Python.framework/Versions/%s/lib/tk8.5'%(getVersion())), + "TCL_LIBRARY": shellQuote('/Library/Frameworks/Python.framework/Versions/%s/lib/tcl8.6'%(getVersion())), + "TK_LIBRARY": shellQuote('/Library/Frameworks/Python.framework/Versions/%s/lib/tk8.6'%(getVersion())), }, ), ]) @@ -270,9 +263,9 @@ def library_recipes(): if PYTHON_3: result.extend([ dict( - name="XZ 5.2.2", - url="http://tukaani.org/xz/xz-5.2.2.tar.gz", - checksum='7cf6a8544a7dae8e8106fdf7addfa28c', + name="XZ 5.2.3", + url="http://tukaani.org/xz/xz-5.2.3.tar.gz", + checksum='ef68674fb47a8b8e741b34e429d86e9d', configure_pre=[ '--disable-dependency-tracking', ] @@ -315,13 +308,14 @@ def library_recipes(): ), ), dict( - name="SQLite 3.21.0", - url="https://www.sqlite.org/2017/sqlite-autoconf-3210000.tar.gz", - checksum='7913de4c3126ba3c24689cb7a199ea31', + name="SQLite 3.22.0", + url="https://www.sqlite.org/2018/sqlite-autoconf-3220000.tar.gz", + checksum='96b5648d542e8afa6ab7ffb8db8ddc3d', extra_cflags=('-Os ' '-DSQLITE_ENABLE_FTS5 ' '-DSQLITE_ENABLE_FTS4 ' '-DSQLITE_ENABLE_FTS3_PARENTHESIS ' + '-DSQLITE_ENABLE_JSON1 ' '-DSQLITE_ENABLE_RTREE ' '-DSQLITE_TCL=0 ' '%s' % ('','-DSQLITE_WITHOUT_ZONEMALLOC ')[LT_10_5]), @@ -342,11 +336,10 @@ def library_recipes(): url="http://bzip.org/1.0.6/bzip2-1.0.6.tar.gz", checksum='00b516f4704d4a7cb50a1d97e6e8e15b', configure=None, - install='make install CC=%s CXX=%s, PREFIX=%s/usr/local/ CFLAGS="-arch %s -isysroot %s"'%( + install='make install CC=%s CXX=%s, PREFIX=%s/usr/local/ CFLAGS="-arch %s"'%( CC, CXX, shellQuote(os.path.join(WORKDIR, 'libraries')), ' -arch '.join(ARCHLIST), - SDKPATH, ), ), dict( @@ -354,11 +347,10 @@ def library_recipes(): url="http://www.gzip.org/zlib/zlib-1.2.3.tar.gz", checksum='debc62758716a169df9f62e6ab2bc634', configure=None, - install='make install CC=%s CXX=%s, prefix=%s/usr/local/ CFLAGS="-arch %s -isysroot %s"'%( + install='make install CC=%s CXX=%s, prefix=%s/usr/local/ CFLAGS="-arch %s"'%( CC, CXX, shellQuote(os.path.join(WORKDIR, 'libraries')), ' -arch '.join(ARCHLIST), - SDKPATH, ), ), dict( @@ -405,8 +397,7 @@ def pkg_recipes(): source="/Library/Frameworks/Python.framework", readme="""\ This package installs Python.framework, that is the python - interpreter and the standard library. This also includes Python - wrappers for lots of Mac OS X API's. + interpreter and the standard library. """, postflight="scripts/postflight.framework", selected='selected', @@ -483,24 +474,6 @@ def pkg_recipes(): ), ] - if getDeptargetTuple() < (10, 4) and not PYTHON_3: - result.append( - dict( - name="PythonSystemFixes", - long_name="Fix system Python", - readme="""\ - This package updates the system python installation on - Mac OS X 10.3 to ensure that you can build new python extensions - using that copy of python after installing this version. - """, - postflight="../Tools/fixapplepython23.py", - topdir="/Library/Frameworks/Python.framework", - source="/empty-dir", - required=False, - selected=unselected_for_python3, - ) - ) - return result def fatal(msg): @@ -565,55 +538,54 @@ def checkEnvironment(): Check that we're running on a supported system. """ - if sys.version_info[0:2] < (2, 4): - fatal("This script must be run with Python 2.4 or later") + if sys.version_info[0:2] < (2, 5): + fatal("This script must be run with Python 2.5 (or later)") if platform.system() != 'Darwin': - fatal("This script should be run on a Mac OS X 10.4 (or later) system") + fatal("This script should be run on a macOS 10.5 (or later) system") if int(platform.release().split('.')[0]) < 8: - fatal("This script should be run on a Mac OS X 10.4 (or later) system") - - if not os.path.exists(SDKPATH): - fatal("Please install the latest version of Xcode and the %s SDK"%( - os.path.basename(SDKPATH[:-4]))) + fatal("This script should be run on a macOS 10.5 (or later) system") # Because we only support dynamic load of only one major/minor version of + # Tcl/Tk, if we are not using building and using our own private copy of # Tcl/Tk, ensure: - # 1. there are no user-installed frameworks of Tcl/Tk with version - # higher than the Apple-supplied system version in - # SDKROOT/System/Library/Frameworks - # 2. there is a user-installed framework (usually ActiveTcl) in (or linked - # in) SDKROOT/Library/Frameworks with the same version as the system - # version. This allows users to choose to install a newer patch level. - - frameworks = {} - for framework in ['Tcl', 'Tk']: - fwpth = 'Library/Frameworks/%s.framework/Versions/Current' % framework - sysfw = os.path.join(SDKPATH, 'System', fwpth) - libfw = os.path.join(SDKPATH, fwpth) - usrfw = os.path.join(os.getenv('HOME'), fwpth) - frameworks[framework] = os.readlink(sysfw) - if not os.path.exists(libfw): - fatal("Please install a link to a current %s %s as %s so " - "the user can override the system framework." - % (framework, frameworks[framework], libfw)) - if os.readlink(libfw) != os.readlink(sysfw): - fatal("Version of %s must match %s" % (libfw, sysfw) ) - if os.path.exists(usrfw): - fatal("Please rename %s to avoid possible dynamic load issues." - % usrfw) - - if frameworks['Tcl'] != frameworks['Tk']: - fatal("The Tcl and Tk frameworks are not the same version.") - - # add files to check after build - EXPECTED_SHARED_LIBS['_tkinter.so'] = [ - "/Library/Frameworks/Tcl.framework/Versions/%s/Tcl" - % frameworks['Tcl'], - "/Library/Frameworks/Tk.framework/Versions/%s/Tk" - % frameworks['Tk'], - ] + # 1. there is a user-installed framework (usually ActiveTcl) in (or linked + # in) SDKROOT/Library/Frameworks. As of Python 3.6.5, we no longer + # enforce that the version of the user-installed framework also + # exists in the system-supplied Tcl/Tk frameworks. Time to support + # Tcl/Tk 8.6 even if Apple does not. + if not internalTk(): + frameworks = {} + for framework in ['Tcl', 'Tk']: + fwpth = 'Library/Frameworks/%s.framework/Versions/Current' % framework + libfw = os.path.join('/', fwpth) + usrfw = os.path.join(os.getenv('HOME'), fwpth) + frameworks[framework] = os.readlink(libfw) + if not os.path.exists(libfw): + fatal("Please install a link to a current %s %s as %s so " + "the user can override the system framework." + % (framework, frameworks[framework], libfw)) + if os.path.exists(usrfw): + fatal("Please rename %s to avoid possible dynamic load issues." + % usrfw) + + if frameworks['Tcl'] != frameworks['Tk']: + fatal("The Tcl and Tk frameworks are not the same version.") + + print(" -- Building with external Tcl/Tk %s frameworks" + % frameworks['Tk']) + + # add files to check after build + EXPECTED_SHARED_LIBS['_tkinter.so'] = [ + "/Library/Frameworks/Tcl.framework/Versions/%s/Tcl" + % frameworks['Tcl'], + "/Library/Frameworks/Tk.framework/Versions/%s/Tk" + % frameworks['Tk'], + ] + else: + print(" -- Building private copy of Tcl/Tk") + print("") # Remove inherited environment variables which might influence build environ_var_prefixes = ['CPATH', 'C_INCLUDE_', 'DYLD_', 'LANG', 'LC_', @@ -643,7 +615,7 @@ def parseOptions(args=None): """ Parse arguments and update global settings. """ - global WORKDIR, DEPSRC, SDKPATH, SRCDIR, DEPTARGET + global WORKDIR, DEPSRC, SRCDIR, DEPTARGET global UNIVERSALOPTS, UNIVERSALARCHS, ARCHLIST, CC, CXX global FW_VERSION_PREFIX global FW_SSL_DIRECTORY @@ -676,7 +648,7 @@ def parseOptions(args=None): DEPSRC=v elif k in ('--sdk-path',): - SDKPATH=v + print(" WARNING: --sdk-path is no longer supported") elif k in ('--src-dir',): SRCDIR=v @@ -692,7 +664,7 @@ def parseOptions(args=None): if deptarget is None: # Select alternate default deployment # target - DEPTARGET = default_target_map.get(v, '10.3') + DEPTARGET = default_target_map.get(v, '10.5') else: raise NotImplementedError(v) @@ -701,7 +673,6 @@ def parseOptions(args=None): SRCDIR=os.path.abspath(SRCDIR) WORKDIR=os.path.abspath(WORKDIR) - SDKPATH=os.path.abspath(SDKPATH) DEPSRC=os.path.abspath(DEPSRC) CC, CXX = getTargetCompilers() @@ -712,7 +683,6 @@ def parseOptions(args=None): print("-- Settings:") print(" * Source directory: %s" % SRCDIR) print(" * Build directory: %s" % WORKDIR) - print(" * SDK location: %s" % SDKPATH) print(" * Third-party source: %s" % DEPSRC) print(" * Deployment target: %s" % DEPTARGET) print(" * Universal archs: %s" % str(ARCHLIST)) @@ -854,9 +824,9 @@ def build_openssl_arch(archbase, arch): configure_opts.append("no-asm") runCommand(" ".join(["perl", "Configure"] + arch_opts[arch] + configure_opts)) - runCommand("make depend OSX_SDK=%s" % SDKPATH) - runCommand("make all OSX_SDK=%s" % SDKPATH) - runCommand("make install_sw OSX_SDK=%s" % SDKPATH) + runCommand("make depend") + runCommand("make all") + runCommand("make install_sw") # runCommand("make test") return @@ -1015,27 +985,24 @@ def buildRecipe(recipe, basedir, archList): if recipe.get('useLDFlags', 1): configure_args.extend([ - "CFLAGS=%s-mmacosx-version-min=%s -arch %s -isysroot %s " + "CFLAGS=%s-mmacosx-version-min=%s -arch %s " "-I%s/usr/local/include"%( recipe.get('extra_cflags', ''), DEPTARGET, ' -arch '.join(archList), - shellQuote(SDKPATH)[1:-1], shellQuote(basedir)[1:-1],), - "LDFLAGS=-mmacosx-version-min=%s -isysroot %s -L%s/usr/local/lib -arch %s"%( + "LDFLAGS=-mmacosx-version-min=%s -L%s/usr/local/lib -arch %s"%( DEPTARGET, - shellQuote(SDKPATH)[1:-1], shellQuote(basedir)[1:-1], ' -arch '.join(archList)), ]) else: configure_args.extend([ - "CFLAGS=%s-mmacosx-version-min=%s -arch %s -isysroot %s " + "CFLAGS=%s-mmacosx-version-min=%s -arch %s " "-I%s/usr/local/include"%( recipe.get('extra_cflags', ''), DEPTARGET, ' -arch '.join(archList), - shellQuote(SDKPATH)[1:-1], shellQuote(basedir)[1:-1],), ]) @@ -1113,10 +1080,6 @@ def buildPython(): curdir = os.getcwd() os.chdir(buildDir) - # Not sure if this is still needed, the original build script - # claims that parts of the install assume python.exe exists. - os.symlink('python', os.path.join(buildDir, 'python.exe')) - # Extract the version from the configure file, needed to calculate # several paths. version = getVersion() @@ -1127,16 +1090,22 @@ def buildPython(): os.environ['DYLD_LIBRARY_PATH'] = os.path.join(WORKDIR, 'libraries', 'usr', 'local', 'lib') print("Running configure...") - runCommand("%s -C --enable-framework --enable-universalsdk=%s " + runCommand("%s -C --enable-framework --enable-universalsdk=/ " "--with-universal-archs=%s " "%s " "%s " + "%s " + "%s " "LDFLAGS='-g -L%s/libraries/usr/local/lib' " "CFLAGS='-g -I%s/libraries/usr/local/include' 2>&1"%( - shellQuote(os.path.join(SRCDIR, 'configure')), shellQuote(SDKPATH), + shellQuote(os.path.join(SRCDIR, 'configure')), UNIVERSALARCHS, (' ', '--with-computed-gotos ')[PYTHON_3], (' ', '--without-ensurepip ')[PYTHON_3], + (' ', "--with-tcltk-includes='-I%s/libraries/usr/local/include'"%( + shellQuote(WORKDIR)[1:-1],))[internalTk()], + (' ', "--with-tcltk-libs='-L%s/libraries/usr/local/lib -ltcl8.6 -ltk8.6'"%( + shellQuote(WORKDIR)[1:-1],))[internalTk()], shellQuote(WORKDIR)[1:-1], shellQuote(WORKDIR)[1:-1])) @@ -1171,14 +1140,22 @@ def buildPython(): del os.environ['DYLD_LIBRARY_PATH'] print("Copying required shared libraries") if os.path.exists(os.path.join(WORKDIR, 'libraries', 'Library')): - runCommand("mv %s/* %s"%( - shellQuote(os.path.join( + build_lib_dir = os.path.join( WORKDIR, 'libraries', 'Library', 'Frameworks', - 'Python.framework', 'Versions', getVersion(), - 'lib')), - shellQuote(os.path.join(WORKDIR, '_root', 'Library', 'Frameworks', - 'Python.framework', 'Versions', getVersion(), - 'lib')))) + 'Python.framework', 'Versions', getVersion(), 'lib') + fw_lib_dir = os.path.join( + WORKDIR, '_root', 'Library', 'Frameworks', + 'Python.framework', 'Versions', getVersion(), 'lib') + if internalTk(): + # move Tcl and Tk pkgconfig files + runCommand("mv %s/pkgconfig/* %s/pkgconfig"%( + shellQuote(build_lib_dir), + shellQuote(fw_lib_dir) )) + runCommand("rm -r %s/pkgconfig"%( + shellQuote(build_lib_dir), )) + runCommand("mv %s/* %s"%( + shellQuote(build_lib_dir), + shellQuote(fw_lib_dir) )) frmDir = os.path.join(rootDir, 'Library', 'Frameworks', 'Python.framework') frmDirVersioned = os.path.join(frmDir, 'Versions', version) diff --git a/Mac/BuildScript/issue19373_tk_8_5_15_source.patch b/Mac/BuildScript/issue19373_tk_8_5_15_source.patch deleted file mode 100644 index de5d08e9d6c0..000000000000 --- a/Mac/BuildScript/issue19373_tk_8_5_15_source.patch +++ /dev/null @@ -1,13 +0,0 @@ -Issue #19373: Patch to Tk 8.5.15 to correct refresh problem on OS x 10.9. -From upstream checkin https://core.tcl.tk/tk/info/5a5abf71f9 - ---- tk8.5.15/macosx/tkMacOSXDraw.c 2013-09-16 09:41:21.000000000 -0700 -+++ Tk_Source_Code-5a5abf71f9fdb0da/macosx/tkMacOSXDraw.c 2013-10-27 13:27:00.000000000 -0700 -@@ -1688,6 +1688,7 @@ - { - if (dcPtr->context) { - CGContextSynchronize(dcPtr->context); -+ [[dcPtr->view window] setViewsNeedDisplay:YES]; - [[dcPtr->view window] enableFlushWindow]; - if (dcPtr->focusLocked) { - [dcPtr->view unlockFocus]; diff --git a/Mac/BuildScript/openssl_sdk_makedepend.patch b/Mac/BuildScript/openssl_sdk_makedepend.patch deleted file mode 100644 index 0caac0a64c1e..000000000000 --- a/Mac/BuildScript/openssl_sdk_makedepend.patch +++ /dev/null @@ -1,40 +0,0 @@ -# HG changeset patch -# -# using openssl 1.0.2k -# -# - support building with an OS X SDK - -diff Configure - -diff --git a/Configure b/Configure ---- a/Configure -+++ b/Configure -@@ -642,12 +642,12 @@ - - ##### MacOS X (a.k.a. Rhapsody or Darwin) setup - "rhapsody-ppc-cc","cc:-O3 -DB_ENDIAN::(unknown):MACOSX_RHAPSODY::BN_LLONG RC4_CHAR RC4_CHUNK DES_UNROLL BF_PTR:${no_asm}::", --"darwin-ppc-cc","cc:-arch ppc -O3 -DB_ENDIAN -Wa,-force_cpusubtype_ALL::-D_REENTRANT:MACOSX:-Wl,-search_paths_first%:BN_LLONG RC4_CHAR RC4_CHUNK DES_UNROLL BF_PTR:${ppc32_asm}:osx32:dlfcn:darwin-shared:-fPIC -fno-common:-arch ppc -dynamiclib:.\$(SHLIB_MAJOR).\$(SHLIB_MINOR).dylib", --"darwin64-ppc-cc","cc:-arch ppc64 -O3 -DB_ENDIAN::-D_REENTRANT:MACOSX:-Wl,-search_paths_first%:SIXTY_FOUR_BIT_LONG RC4_CHAR RC4_CHUNK DES_UNROLL BF_PTR:${ppc64_asm}:osx64:dlfcn:darwin-shared:-fPIC -fno-common:-arch ppc64 -dynamiclib:.\$(SHLIB_MAJOR).\$(SHLIB_MINOR).dylib", --"darwin-i386-cc","cc:-arch i386 -O3 -fomit-frame-pointer -DL_ENDIAN::-D_REENTRANT:MACOSX:-Wl,-search_paths_first%:BN_LLONG RC4_INT RC4_CHUNK DES_UNROLL BF_PTR:".eval{my $asm=$x86_asm;$asm=~s/cast\-586\.o//;$asm}.":macosx:dlfcn:darwin-shared:-fPIC -fno-common:-arch i386 -dynamiclib:.\$(SHLIB_MAJOR).\$(SHLIB_MINOR).dylib", --"debug-darwin-i386-cc","cc:-arch i386 -g3 -DL_ENDIAN::-D_REENTRANT:MACOSX:-Wl,-search_paths_first%:BN_LLONG RC4_INT RC4_CHUNK DES_UNROLL BF_PTR:${x86_asm}:macosx:dlfcn:darwin-shared:-fPIC -fno-common:-arch i386 -dynamiclib:.\$(SHLIB_MAJOR).\$(SHLIB_MINOR).dylib", --"darwin64-x86_64-cc","cc:-arch x86_64 -O3 -DL_ENDIAN -Wall::-D_REENTRANT:MACOSX:-Wl,-search_paths_first%:SIXTY_FOUR_BIT_LONG RC4_CHUNK DES_INT DES_UNROLL:".eval{my $asm=$x86_64_asm;$asm=~s/rc4\-[^:]+//;$asm}.":macosx:dlfcn:darwin-shared:-fPIC -fno-common:-arch x86_64 -dynamiclib:.\$(SHLIB_MAJOR).\$(SHLIB_MINOR).dylib", --"debug-darwin64-x86_64-cc","cc:-arch x86_64 -ggdb -g2 -O0 -DL_ENDIAN -Wall::-D_REENTRANT:MACOSX:-Wl,-search_paths_first%:SIXTY_FOUR_BIT_LONG RC4_CHUNK DES_INT DES_UNROLL:".eval{my $asm=$x86_64_asm;$asm=~s/rc4\-[^:]+//;$asm}.":macosx:dlfcn:darwin-shared:-fPIC -fno-common:-arch x86_64 -dynamiclib:.\$(SHLIB_MAJOR).\$(SHLIB_MINOR).dylib", -+"darwin-ppc-cc","cc:-arch ppc -isysroot \$(OSX_SDK) -O3 -DB_ENDIAN -Wa,-force_cpusubtype_ALL::-D_REENTRANT:MACOSX:-Wl,-search_paths_first%:BN_LLONG RC4_CHAR RC4_CHUNK DES_UNROLL BF_PTR:${ppc32_asm}:osx32:dlfcn:darwin-shared:-fPIC -fno-common:-arch ppc -dynamiclib:.\$(SHLIB_MAJOR).\$(SHLIB_MINOR).dylib", -+"darwin64-ppc-cc","cc:-arch ppc64 -isysroot \$(OSX_SDK) -O3 -DB_ENDIAN::-D_REENTRANT:MACOSX:-Wl,-search_paths_first%:SIXTY_FOUR_BIT_LONG RC4_CHAR RC4_CHUNK DES_UNROLL BF_PTR:${ppc64_asm}:osx64:dlfcn:darwin-shared:-fPIC -fno-common:-arch ppc64 -dynamiclib:.\$(SHLIB_MAJOR).\$(SHLIB_MINOR).dylib", -+"darwin-i386-cc","cc:-arch i386 -isysroot \$(OSX_SDK) -O3 -fomit-frame-pointer -DL_ENDIAN::-D_REENTRANT:MACOSX:-Wl,-search_paths_first%:BN_LLONG RC4_INT RC4_CHUNK DES_UNROLL BF_PTR:".eval{my $asm=$x86_asm;$asm=~s/cast\-586\.o//;$asm}.":macosx:dlfcn:darwin-shared:-fPIC -fno-common:-arch i386 -dynamiclib:.\$(SHLIB_MAJOR).\$(SHLIB_MINOR).dylib", -+"debug-darwin-i386-cc","cc:-arch i386 -isysroot \$(OSX_SDK) -g3 -DL_ENDIAN::-D_REENTRANT:MACOSX:-Wl,-search_paths_first%:BN_LLONG RC4_INT RC4_CHUNK DES_UNROLL BF_PTR:${x86_asm}:macosx:dlfcn:darwin-shared:-fPIC -fno-common:-arch i386 -dynamiclib:.\$(SHLIB_MAJOR).\$(SHLIB_MINOR).dylib", -+"darwin64-x86_64-cc","cc:-arch x86_64 -isysroot \$(OSX_SDK) -O3 -DL_ENDIAN -Wall::-D_REENTRANT:MACOSX:-Wl,-search_paths_first%:SIXTY_FOUR_BIT_LONG RC4_CHUNK DES_INT DES_UNROLL:".eval{my $asm=$x86_64_asm;$asm=~s/rc4\-[^:]+//;$asm}.":macosx:dlfcn:darwin-shared:-fPIC -fno-common:-arch x86_64 -dynamiclib:.\$(SHLIB_MAJOR).\$(SHLIB_MINOR).dylib", -+"debug-darwin64-x86_64-cc","cc:-arch x86_64 -isysroot \$(OSX_SDK) -ggdb -g2 -O0 -DL_ENDIAN -Wall::-D_REENTRANT:MACOSX:-Wl,-search_paths_first%:SIXTY_FOUR_BIT_LONG RC4_CHUNK DES_INT DES_UNROLL:".eval{my $asm=$x86_64_asm;$asm=~s/rc4\-[^:]+//;$asm}.":macosx:dlfcn:darwin-shared:-fPIC -fno-common:-arch x86_64 -dynamiclib:.\$(SHLIB_MAJOR).\$(SHLIB_MINOR).dylib", - "debug-darwin-ppc-cc","cc:-DBN_DEBUG -DREF_CHECK -DCONF_DEBUG -DCRYPTO_MDEBUG -DB_ENDIAN -g -Wall -O::-D_REENTRANT:MACOSX::BN_LLONG RC4_CHAR RC4_CHUNK DES_UNROLL BF_PTR:${ppc32_asm}:osx32:dlfcn:darwin-shared:-fPIC:-dynamiclib:.\$(SHLIB_MAJOR).\$(SHLIB_MINOR).dylib", - # iPhoneOS/iOS - "iphoneos-cross","llvm-gcc:-O3 -isysroot \$(CROSS_TOP)/SDKs/\$(CROSS_SDK) -fomit-frame-pointer -fno-common::-D_REENTRANT:iOS:-Wl,-search_paths_first%:BN_LLONG RC4_CHAR RC4_CHUNK DES_UNROLL BF_PTR:${no_asm}:dlfcn:darwin-shared:-fPIC -fno-common:-dynamiclib:.\$(SHLIB_MAJOR).\$(SHLIB_MINOR).dylib", -@@ -1728,8 +1728,7 @@ - s/^AR=\s*ar/AR= $ar/; - s/^RANLIB=.*/RANLIB= $ranlib/; - s/^RC=.*/RC= $windres/; -- s/^MAKEDEPPROG=.*$/MAKEDEPPROG= $cc/ if $cc eq "gcc"; -- s/^MAKEDEPPROG=.*$/MAKEDEPPROG= $cc/ if $ecc eq "gcc" || $ecc eq "clang"; -+ s/^MAKEDEPPROG=.*$/MAKEDEPPROG= $cc/; - } - s/^CFLAG=.*$/CFLAG= $cflags/; - s/^DEPFLAG=.*$/DEPFLAG=$depflags/; diff --git a/Mac/BuildScript/resources/Conclusion.rtf b/Mac/BuildScript/resources/Conclusion.rtf new file mode 100644 index 000000000000..9e0fa9fa6eeb --- /dev/null +++ b/Mac/BuildScript/resources/Conclusion.rtf @@ -0,0 +1,20 @@ +{\rtf1\ansi\ansicpg1252\cocoartf1561\cocoasubrtf200 +{\fonttbl\f0\fswiss\fcharset0 Helvetica;\f1\fnil\fcharset0 LucidaGrande-Bold;\f2\fnil\fcharset0 LucidaGrande; +\f3\fnil\fcharset0 Monaco;} +{\colortbl;\red255\green255\blue255;} +{\*\expandedcolortbl;;} +\margl1440\margr1440\vieww10540\viewh8400\viewkind0 +\pard\tx720\tx1440\tx2160\tx2880\tx3600\tx4320\tx5040\tx5760\tx6480\tx7200\tx7920\tx8640\pardirnatural\partightenfactor0 + +\f0\fs28 \cf0 Congratulations! +\fs24 +\f1\b\fs28 Python $FULL_VERSION for macOS $MACOSX_DEPLOYMENT_TARGET +\f2\b0 was successfully installed. +\fs24 \ +\ +One more thing: to verify the identity of secure network connections, this Python needs a set of SSL root certificates. You can download and install a current curated set from {\field{\*\fldinst{HYPERLINK "https://pypi.org/project/certifi/"}}{\fldrslt the Certifi project}} by double-clicking on the +\f3 Install Certificates +\f2 icon in {\field{\*\fldinst{HYPERLINK "file://localhost/Applications/Python%20$VERSION/"}}{\fldrslt the Finder window}}. See {\field{\*\fldinst{HYPERLINK "file://localhost/Applications/Python%20$VERSION/ReadMe.rtf"}}{\fldrslt the +\f3 ReadMe +\f2 file}} for more information.\ +} \ No newline at end of file diff --git a/Mac/BuildScript/resources/ReadMe.rtf b/Mac/BuildScript/resources/ReadMe.rtf index ac68786999df..e81465915380 100644 --- a/Mac/BuildScript/resources/ReadMe.rtf +++ b/Mac/BuildScript/resources/ReadMe.rtf @@ -1,43 +1,44 @@ -{\rtf1\ansi\ansicpg1252\cocoartf1504\cocoasubrtf750 +{\rtf1\ansi\ansicpg1252\cocoartf1561\cocoasubrtf200 {\fonttbl\f0\fswiss\fcharset0 Helvetica;\f1\fmodern\fcharset0 CourierNewPSMT;} {\colortbl;\red255\green255\blue255;} {\*\expandedcolortbl;;} \margl1440\margr1440\vieww13380\viewh14600\viewkind0 \pard\tx720\tx1440\tx2160\tx2880\tx3600\tx4320\tx5040\tx5760\tx6480\tx7200\tx7920\tx8640\pardirnatural\partightenfactor0 -\f0\fs24 \cf0 This package will install Python $FULL_VERSION for Mac OS X $MACOSX_DEPLOYMENT_TARGET for the following architecture(s): $ARCHITECTURES.\ +\f0\fs24 \cf0 This package will install Python $FULL_VERSION for macOS $MACOSX_DEPLOYMENT_TARGET for the following architecture(s): $ARCHITECTURES.\ \ \pard\tx720\tx1440\tx2160\tx2880\tx3600\tx4320\tx5040\tx5760\tx6480\tx7200\tx7920\tx8640\pardirnatural\partightenfactor0 -\b \cf0 \ul \ulc0 Which installer variant should I use? +\b \cf0 \ul \ulc0 Which installer variant should I use? [CHANGED in 3.6.5] \b0 \ulnone \ \ \pard\tx720\tx1440\tx2160\tx2880\tx3600\tx4320\tx5040\tx5760\tx6480\tx7200\tx7920\tx8640\pardirnatural\partightenfactor0 \b \cf0 **NEW** -\b0 For Python 3.6, the python.org website now provides only one installer variant for download: one that installs a +\b0 With Python 3.6.5, the python.org website now provides two installer variants for download: one that installs a +\i 64-bit-only +\i0 Python capable of running on +\i macOS 10.9 (Mavericks) +\i0 or later; and one that installs a \i 64-bit/32-bit Intel \i0 Python capable of running on -\i Mac OS X 10.6 (Snow Leopard) -\i0 or later. This ReadMe was installed with the +\i macOS 10.6 (Snow Leopard) +\i0 or later. (This ReadMe was installed with the \i $MACOSX_DEPLOYMENT_TARGET -\i0 variant. By default, Python will automatically run in 64-bit mode if your system supports it. The Python installed by this installer is built with private copies of some third-party libraries not included with or newer than those in OS X itself. The list of these libraries is included at the end of the License.rtf file. -\b \ul \ -\ -Certificate verification and OpenSSL\ +\i0 variant.) Previous Python 3.6.x releases only provided the 10.6 or later installer. If you are running on macOS 10.9 or later and if you have no need for compatibility with older systems, use the 10.9 variant. Use the 10.6 variant if you are running on macOS 10.6 through 10.8, if you need to maintain compatibility with previous 3.6.x releases, or if you want to produce standalone applications that can run on systems from 10.6. The Pythons installed by these installers are built with private copies of some third-party libraries not included with or newer than those in macOS itself. The list of these libraries varies by installer variant and is included at the end of the License.rtf file.\ \pard\tx720\tx1440\tx2160\tx2880\tx3600\tx4320\tx5040\tx5760\tx6480\tx7200\tx7920\tx8640\pardirnatural\partightenfactor0 -\b0 \cf0 \ulnone \ -\pard\tx720\tx1440\tx2160\tx2880\tx3600\tx4320\tx5040\tx5760\tx6480\tx7200\tx7920\tx8640\pardirnatural\partightenfactor0 +\b \cf0 \ul \ulc0 \ +Certificate verification and OpenSSL\ -\b \cf0 **NEW** -\b0 This variant of Python 3.6 now includes its own private copy of OpenSSL 1.0.2. Unlike previous releases, the deprecated Apple-supplied OpenSSL libraries are no longer used. This also means that the trust certificates in system and user keychains managed by the +\b0 \ulnone \ +This variant of Python 3.6 now includes its own private copy of OpenSSL 1.0.2. Unlike previous releases, the deprecated Apple-supplied OpenSSL libraries are no longer used. This also means that the trust certificates in system and user keychains managed by the \i Keychain Access \i0 application and the \i security \i0 command line utility are no longer used as defaults by the Python \f1 ssl -\f0 module. For 3.6.0, a sample command script is included in +\f0 module. A sample command script is included in \f1 /Applications/Python 3.6 \f0 to install a curated bundle of default root certificates from the third-party \f1 certifi @@ -49,16 +50,17 @@ The bundled \f1 pip \f0 included with the Python 3.6 installer has its own default certificate store for verifying download connections.\ \ -\pard\tx720\tx1440\tx2160\tx2880\tx3600\tx4320\tx5040\tx5760\tx6480\tx7200\tx7920\tx8640\pardirnatural\partightenfactor0 -\b \cf0 \ul Update your version of Tcl/Tk to use IDLE or other Tk applications +\b \ul Using IDLE or other Tk applications [NEW/CHANGED in 3.6.5] \b0 \ulnone \ \ -To use IDLE or other programs that use the Tkinter graphical user interface toolkit, you need to install a newer third-party version of the +The 10.9+ installer variant comes with its own private version of Tcl/Tk 8.6. It does not use system-supplied or third-party supplied versions of Tcl/Tk.\ +\ +For the 10.6+ variant, you continue to need to install a newer third-party version of the \i Tcl/Tk -\i0 frameworks. Visit {\field{\*\fldinst{HYPERLINK "https://www.python.org/download/mac/tcltk/"}}{\fldrslt https://www.python.org/download/mac/tcltk/}} for current information about supported and recommended versions of +\i0 8.5 (not 8.6) frameworks to use IDLE or other programs that use the Tkinter graphical user interface toolkit. Visit {\field{\*\fldinst{HYPERLINK "https://www.python.org/download/mac/tcltk/"}}{\fldrslt https://www.python.org/download/mac/tcltk/}} for current information about supported and recommended versions of \i Tcl/Tk -\i0 for this version of Python and of Mac OS X. For the initial release of Python 3.6, the installer is still linked with Tcl/Tk 8.5.\ +\i0 for this version of Python and of macOS.\ \b \ul \ Other changes\ diff --git a/Mac/BuildScript/resources/Welcome.rtf b/Mac/BuildScript/resources/Welcome.rtf index 3a9ab04454d8..cac9626693dc 100644 --- a/Mac/BuildScript/resources/Welcome.rtf +++ b/Mac/BuildScript/resources/Welcome.rtf @@ -1,34 +1,24 @@ -{\rtf1\ansi\ansicpg1252\cocoartf1504 +{\rtf1\ansi\ansicpg1252\cocoartf1561\cocoasubrtf200 \cocoascreenfonts1{\fonttbl\f0\fswiss\fcharset0 Helvetica;} {\colortbl;\red255\green255\blue255;} -{\*\expandedcolortbl;\csgray\c100000;} +{\*\expandedcolortbl;;} \paperw11905\paperh16837\margl1440\margr1440\vieww12200\viewh10880\viewkind0 -\pard\tx720\tx1440\tx2160\tx2880\tx3600\tx4320\tx5040\tx5760\tx6480\tx7200\tx7920\tx8640\partightenfactor0 +\pard\tx720\tx1440\tx2160\tx2880\tx3600\tx4320\tx5040\tx5760\tx6480\tx7200\tx7920\tx8640\pardirnatural\partightenfactor0 \f0\fs24 \cf0 This package will install \b Python $FULL_VERSION \b0 for -\b Mac OS X $MACOSX_DEPLOYMENT_TARGET +\b macOS $MACOSX_DEPLOYMENT_TARGET \b0 .\ -\ +\pard\tx720\tx1440\tx2160\tx2880\tx3600\tx4320\tx5040\tx5760\tx6480\tx7200\tx7920\tx8640\partightenfactor0 +\cf0 \ -\b Python for Mac OS X -\b0 consists of the Python programming language interpreter, plus a set of programs to allow easy access to it for Mac OS X users including an integrated development environment +\b Python for macOS +\b0 consists of the Python programming language interpreter, plus a set of programs to allow easy access to it for macOS users including an integrated development environment \b IDLE \b0 .\ \ -\pard\tx720\tx1440\tx2160\tx2880\tx3600\tx4320\tx5040\tx5760\tx6480\tx7200\tx7920\tx8640\partightenfactor0 - -\b \cf0 NEW: -\b0 There are important changes in this release regarding network security and trust certificates. Please see the ReadMe for more details.\ -\ -\pard\tx720\tx1440\tx2160\tx2880\tx3600\tx4320\tx5040\tx5760\tx6480\tx7200\tx7920\tx8640\partightenfactor0 -\b \cf0 IMPORTANT: -\b0 -\b IDLE -\b0 and other programs using the -\b tkinter -\b0 graphical user interface toolkit require specific versions of the -\b Tcl/Tk -\b0 platform independent windowing toolkit. Visit {\field{\*\fldinst{HYPERLINK "https://www.python.org/download/mac/tcltk/"}}{\fldrslt https://www.python.org/download/mac/tcltk/}} for current information on supported and recommended versions of Tcl/Tk for this version of Python and Mac OS X.} \ No newline at end of file +\b NEW in 3.6.5: +\b0 two installer variants (10.9+ 64-bit-only, 10.6+ 64-/32-bit), built-in Tcl/Tk 8.6 support in the 10.9+ variant (no additional third-party downloads!)\ +} \ No newline at end of file diff --git a/Mac/BuildScript/scripts/postflight.documentation b/Mac/BuildScript/scripts/postflight.documentation index b9f28a515215..3cbbc1bf10ca 100755 --- a/Mac/BuildScript/scripts/postflight.documentation +++ b/Mac/BuildScript/scripts/postflight.documentation @@ -12,6 +12,7 @@ SHARE_DOCDIR_TO_FWK="../../.." # make link in /Applications/Python m.n/ for Finder users if [ -d "${APPDIR}" ]; then ln -fhs "${FWK_DOCDIR}/index.html" "${APPDIR}/Python Documentation.html" + open "${APPDIR}" || true # open the applications folder fi # make share/doc link in framework for command line users diff --git a/Mac/BuildScript/tk868_on_10_8_10_9.patch b/Mac/BuildScript/tk868_on_10_8_10_9.patch new file mode 100644 index 000000000000..8fe10604a68c --- /dev/null +++ b/Mac/BuildScript/tk868_on_10_8_10_9.patch @@ -0,0 +1,18 @@ +Fix build failure with +quartz variant on OS X 10.8 and 10.9. +Even though Gestalt was deprecated in OS X 10.8, it should work fine +through OS X 10.9, and its replacement NSOperatingSystemVersion was +not introduced until OS X 10.10. + +Patch from MacPorts project and reported upstream: +https://trac.macports.org/ticket/55649 +--- tk8.6.8/macosx/tkMacOSXXStubs.c.orig 2017-12-06 09:25:08.000000000 -0600 ++++ tk8.6.8-patched/macosx/tkMacOSXXStubs.c 2018-01-06 19:34:17.000000000 -0600 +@@ -175,7 +175,7 @@ + { + int major, minor, patch; + +-#if MAC_OS_X_VERSION_MIN_REQUIRED < 1080 ++#if MAC_OS_X_VERSION_MIN_REQUIRED < 101000 + Gestalt(gestaltSystemVersionMajor, (SInt32*)&major); + Gestalt(gestaltSystemVersionMinor, (SInt32*)&minor); + Gestalt(gestaltSystemVersionBugFix, (SInt32*)&patch); diff --git a/Misc/NEWS.d/next/macOS/2018-03-13-21-00-20.bpo-32726.Mticyn.rst b/Misc/NEWS.d/next/macOS/2018-03-13-21-00-20.bpo-32726.Mticyn.rst new file mode 100644 index 000000000000..f2d096a063b7 --- /dev/null +++ b/Misc/NEWS.d/next/macOS/2018-03-13-21-00-20.bpo-32726.Mticyn.rst @@ -0,0 +1,4 @@ +Provide an additional, more modern macOS installer variant that supports +macOS 10.9+ systems in 64-bit mode only. Upgrade the supplied third-party +libraries to OpenSSL 1.0.2n, XZ 5.2.3, and SQLite 3.22.0. The 10.9+ +installer now links with and supplies its own copy of Tcl/Tk 8.6.8. diff --git a/configure b/configure index 1e66117fffbb..a707743b4966 100755 --- a/configure +++ b/configure @@ -1491,7 +1491,8 @@ Optional Packages: --without-PACKAGE do not use PACKAGE (same as --with-PACKAGE=no) --with-universal-archs=ARCH select architectures for universal build ("32-bit", - "64-bit", "3-way", "intel", "intel-32", or "all") + "64-bit", "3-way", "intel", "intel-32", "intel-64", + or "all") --with-framework-name=FRAMEWORK specify an alternate name of the framework built with --enable-framework @@ -7345,6 +7346,11 @@ $as_echo "$CC" >&6; } LIPO_32BIT_FLAGS="" ARCH_RUN_32BIT="" ;; + intel-64) + UNIVERSAL_ARCH_FLAGS="-arch x86_64" + LIPO_32BIT_FLAGS="" + ARCH_RUN_32BIT="true" + ;; 3-way) UNIVERSAL_ARCH_FLAGS="-arch i386 -arch ppc -arch x86_64" LIPO_32BIT_FLAGS="-extract ppc7400 -extract i386" @@ -7355,11 +7361,14 @@ $as_echo "$CC" >&6; } ;; esac - CFLAGS="${UNIVERSAL_ARCH_FLAGS} -isysroot ${UNIVERSALSDK} ${CFLAGS}" - LDFLAGS="${UNIVERSAL_ARCH_FLAGS} -isysroot ${UNIVERSALSDK} ${LDFLAGS}" if test "${UNIVERSALSDK}" != "/" then + CFLAGS="${UNIVERSAL_ARCH_FLAGS} -isysroot ${UNIVERSALSDK} ${CFLAGS}" + LDFLAGS="${UNIVERSAL_ARCH_FLAGS} -isysroot ${UNIVERSALSDK} ${LDFLAGS}" CPPFLAGS="-isysroot ${UNIVERSALSDK} ${CPPFLAGS}" + else + CFLAGS="${UNIVERSAL_ARCH_FLAGS} ${CFLAGS}" + LDFLAGS="${UNIVERSAL_ARCH_FLAGS} ${LDFLAGS}" fi fi diff --git a/configure.ac b/configure.ac index 2eb511bf7f1f..0ec09369a9a0 100644 --- a/configure.ac +++ b/configure.ac @@ -214,7 +214,7 @@ fi AC_SUBST(LIPO_32BIT_FLAGS) AC_MSG_CHECKING(for --with-universal-archs) AC_ARG_WITH(universal-archs, - AS_HELP_STRING([--with-universal-archs=ARCH], [select architectures for universal build ("32-bit", "64-bit", "3-way", "intel", "intel-32", or "all")]), + AS_HELP_STRING([--with-universal-archs=ARCH], [select architectures for universal build ("32-bit", "64-bit", "3-way", "intel", "intel-32", "intel-64", or "all")]), [ UNIVERSAL_ARCHS="$withval" ], @@ -1782,6 +1782,11 @@ yes) LIPO_32BIT_FLAGS="" ARCH_RUN_32BIT="" ;; + intel-64) + UNIVERSAL_ARCH_FLAGS="-arch x86_64" + LIPO_32BIT_FLAGS="" + ARCH_RUN_32BIT="true" + ;; 3-way) UNIVERSAL_ARCH_FLAGS="-arch i386 -arch ppc -arch x86_64" LIPO_32BIT_FLAGS="-extract ppc7400 -extract i386" @@ -1792,11 +1797,14 @@ yes) ;; esac - CFLAGS="${UNIVERSAL_ARCH_FLAGS} -isysroot ${UNIVERSALSDK} ${CFLAGS}" - LDFLAGS="${UNIVERSAL_ARCH_FLAGS} -isysroot ${UNIVERSALSDK} ${LDFLAGS}" if test "${UNIVERSALSDK}" != "/" then + CFLAGS="${UNIVERSAL_ARCH_FLAGS} -isysroot ${UNIVERSALSDK} ${CFLAGS}" + LDFLAGS="${UNIVERSAL_ARCH_FLAGS} -isysroot ${UNIVERSALSDK} ${LDFLAGS}" CPPFLAGS="-isysroot ${UNIVERSALSDK} ${CPPFLAGS}" + else + CFLAGS="${UNIVERSAL_ARCH_FLAGS} ${CFLAGS}" + LDFLAGS="${UNIVERSAL_ARCH_FLAGS} ${LDFLAGS}" fi fi From webhook-mailer at python.org Wed Mar 14 02:40:29 2018 From: webhook-mailer at python.org (Christian Heimes) Date: Wed, 14 Mar 2018 06:40:29 -0000 Subject: [Python-checkins] bpo-30622: Fix backport of NPN fix (#6102) Message-ID: <mailman.74.1521009630.1871.python-checkins@python.org> https://github.com/python/cpython/commit/0ec0290a075ad3ac7d9990efd701c81da16293c0 commit: 0ec0290a075ad3ac7d9990efd701c81da16293c0 branch: 3.6 author: Christian Heimes <christian at python.org> committer: GitHub <noreply at github.com> date: 2018-03-14T07:40:20+01:00 summary: bpo-30622: Fix backport of NPN fix (#6102) Fix backport a79591cf of bpo-30622 to 3.6 branch. Signed-off-by: Christian Heimes <christian at python.org> files: M Modules/_ssl.c diff --git a/Modules/_ssl.c b/Modules/_ssl.c index 2fe696daa5e1..c54e43c2b48a 100644 --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -2747,7 +2747,7 @@ _ssl__SSLContext_impl(PyTypeObject *type, int proto_version) return NULL; } self->ctx = ctx; -#ifdef HAVE_NPN +#if HAVE_NPN self->npn_protocols = NULL; #endif #if HAVE_ALPN From solipsis at pitrou.net Wed Mar 14 05:15:03 2018 From: solipsis at pitrou.net (solipsis at pitrou.net) Date: Wed, 14 Mar 2018 09:15:03 +0000 Subject: [Python-checkins] Daily reference leaks (4243df51fe43): sum=8 Message-ID: <20180314091503.1.D25BCB64D7430751@psf.io> results for 4243df51fe43 on branch "default" -------------------------------------------- test_asyncio leaked [0, 3, 0] memory blocks, sum=3 test_collections leaked [-7, 1, 7] memory blocks, sum=1 test_functools leaked [0, 3, 1] memory blocks, sum=4 test_multiprocessing_forkserver leaked [2, 0, -2] memory blocks, sum=0 Command line was: ['./python', '-m', 'test.regrtest', '-uall', '-R', '3:3:/home/psf-users/antoine/refleaks/reflogvdsJg2', '--timeout', '7200'] From webhook-mailer at python.org Wed Mar 14 13:52:26 2018 From: webhook-mailer at python.org (Christian Heimes) Date: Wed, 14 Mar 2018 17:52:26 -0000 Subject: [Python-checkins] [3.6] bpo-32885: Tools/scripts/pathfix.py: Add -n option for no backup~ (GH-5772) (#6104) Message-ID: <mailman.75.1521049948.1871.python-checkins@python.org> https://github.com/python/cpython/commit/a954919788f2130076e4f9dd91e9eccf69540f7a commit: a954919788f2130076e4f9dd91e9eccf69540f7a branch: 3.6 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Christian Heimes <christian at python.org> date: 2018-03-14T18:52:22+01:00 summary: [3.6] bpo-32885: Tools/scripts/pathfix.py: Add -n option for no backup~ (GH-5772) (#6104) Creating backup files with ~ suffix can be undesirable in some environment, such as when building RPM packages. Instead of requiring the user to remove those files manually, option -n was added, that simply disables this feature. -n was selected because 2to3 has the same option with this behavior. (cherry picked from commit 5affd5c29eb1493cb31ef3cfdde15538ac134689) Co-authored-by: Miro Hron?ok <miro at hroncok.cz> files: A Misc/NEWS.d/next/Tools-Demos/2018-02-20-12-16-47.bpo-32885.dL5x7C.rst M Misc/ACKS M Tools/scripts/pathfix.py diff --git a/Misc/ACKS b/Misc/ACKS index b2033ee9d3f3..ca8eaefc5b46 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -672,6 +672,7 @@ Ken Howard Brad Howes Mike Hoy Ben Hoyt +Miro Hron?ok Chiu-Hsiang Hsu Chih-Hao Huang Christian Hudon diff --git a/Misc/NEWS.d/next/Tools-Demos/2018-02-20-12-16-47.bpo-32885.dL5x7C.rst b/Misc/NEWS.d/next/Tools-Demos/2018-02-20-12-16-47.bpo-32885.dL5x7C.rst new file mode 100644 index 000000000000..e003e1d84fd0 --- /dev/null +++ b/Misc/NEWS.d/next/Tools-Demos/2018-02-20-12-16-47.bpo-32885.dL5x7C.rst @@ -0,0 +1,2 @@ +Add an ``-n`` flag for ``Tools/scripts/pathfix.py`` to disbale automatic +backup creation (files with ``~`` suffix). diff --git a/Tools/scripts/pathfix.py b/Tools/scripts/pathfix.py index 562bbc737812..c5bf984306a3 100755 --- a/Tools/scripts/pathfix.py +++ b/Tools/scripts/pathfix.py @@ -7,8 +7,9 @@ # Directories are searched recursively for files whose name looks # like a python module. # Symbolic links are always ignored (except as explicit directory -# arguments). Of course, the original file is kept as a back-up -# (with a "~" attached to its name). +# arguments). +# The original file is kept as a back-up (with a "~" attached to its name), +# -n flag can be used to disable this. # # Undoubtedly you can do this using find and sed or perl, but this is # a nice example of Python code that recurses down a directory tree @@ -31,14 +32,17 @@ new_interpreter = None preserve_timestamps = False +create_backup = True + def main(): global new_interpreter global preserve_timestamps - usage = ('usage: %s -i /interpreter -p file-or-directory ...\n' % + global create_backup + usage = ('usage: %s -i /interpreter -p -n file-or-directory ...\n' % sys.argv[0]) try: - opts, args = getopt.getopt(sys.argv[1:], 'i:p') + opts, args = getopt.getopt(sys.argv[1:], 'i:pn') except getopt.error as msg: err(str(msg) + '\n') err(usage) @@ -48,6 +52,8 @@ def main(): new_interpreter = a.encode() if o == '-p': preserve_timestamps = True + if o == '-n': + create_backup = False if not new_interpreter or not new_interpreter.startswith(b'/') or \ not args: err('-i option or file-or-directory missing\n') @@ -134,10 +140,16 @@ def fix(filename): except OSError as msg: err('%s: warning: chmod failed (%r)\n' % (tempname, msg)) # Then make a backup of the original file as filename~ - try: - os.rename(filename, filename + '~') - except OSError as msg: - err('%s: warning: backup failed (%r)\n' % (filename, msg)) + if create_backup: + try: + os.rename(filename, filename + '~') + except OSError as msg: + err('%s: warning: backup failed (%r)\n' % (filename, msg)) + else: + try: + os.remove(filename) + except OSError as msg: + err('%s: warning: removing failed (%r)\n' % (filename, msg)) # Now move the temp file to the original file try: os.rename(tempname, filename) From webhook-mailer at python.org Wed Mar 14 13:52:31 2018 From: webhook-mailer at python.org (Christian Heimes) Date: Wed, 14 Mar 2018 17:52:31 -0000 Subject: [Python-checkins] [3.7] bpo-32885: Tools/scripts/pathfix.py: Add -n option for no backup~ (GH-5772) (#6103) Message-ID: <mailman.76.1521049952.1871.python-checkins@python.org> https://github.com/python/cpython/commit/6e65e4462692cbf4461c307a411af7cf40a1ca4a commit: 6e65e4462692cbf4461c307a411af7cf40a1ca4a branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Christian Heimes <christian at python.org> date: 2018-03-14T18:52:28+01:00 summary: [3.7] bpo-32885: Tools/scripts/pathfix.py: Add -n option for no backup~ (GH-5772) (#6103) Creating backup files with ~ suffix can be undesirable in some environment, such as when building RPM packages. Instead of requiring the user to remove those files manually, option -n was added, that simply disables this feature. -n was selected because 2to3 has the same option with this behavior. (cherry picked from commit 5affd5c29eb1493cb31ef3cfdde15538ac134689) Co-authored-by: Miro Hron?ok <miro at hroncok.cz> files: A Misc/NEWS.d/next/Tools-Demos/2018-02-20-12-16-47.bpo-32885.dL5x7C.rst M Misc/ACKS M Tools/scripts/pathfix.py diff --git a/Misc/ACKS b/Misc/ACKS index b0a9136170d6..9c7cb6f0057d 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -686,6 +686,7 @@ Ken Howard Brad Howes Mike Hoy Ben Hoyt +Miro Hron?ok Chiu-Hsiang Hsu Chih-Hao Huang Christian Hudon diff --git a/Misc/NEWS.d/next/Tools-Demos/2018-02-20-12-16-47.bpo-32885.dL5x7C.rst b/Misc/NEWS.d/next/Tools-Demos/2018-02-20-12-16-47.bpo-32885.dL5x7C.rst new file mode 100644 index 000000000000..e003e1d84fd0 --- /dev/null +++ b/Misc/NEWS.d/next/Tools-Demos/2018-02-20-12-16-47.bpo-32885.dL5x7C.rst @@ -0,0 +1,2 @@ +Add an ``-n`` flag for ``Tools/scripts/pathfix.py`` to disbale automatic +backup creation (files with ``~`` suffix). diff --git a/Tools/scripts/pathfix.py b/Tools/scripts/pathfix.py index 562bbc737812..c5bf984306a3 100755 --- a/Tools/scripts/pathfix.py +++ b/Tools/scripts/pathfix.py @@ -7,8 +7,9 @@ # Directories are searched recursively for files whose name looks # like a python module. # Symbolic links are always ignored (except as explicit directory -# arguments). Of course, the original file is kept as a back-up -# (with a "~" attached to its name). +# arguments). +# The original file is kept as a back-up (with a "~" attached to its name), +# -n flag can be used to disable this. # # Undoubtedly you can do this using find and sed or perl, but this is # a nice example of Python code that recurses down a directory tree @@ -31,14 +32,17 @@ new_interpreter = None preserve_timestamps = False +create_backup = True + def main(): global new_interpreter global preserve_timestamps - usage = ('usage: %s -i /interpreter -p file-or-directory ...\n' % + global create_backup + usage = ('usage: %s -i /interpreter -p -n file-or-directory ...\n' % sys.argv[0]) try: - opts, args = getopt.getopt(sys.argv[1:], 'i:p') + opts, args = getopt.getopt(sys.argv[1:], 'i:pn') except getopt.error as msg: err(str(msg) + '\n') err(usage) @@ -48,6 +52,8 @@ def main(): new_interpreter = a.encode() if o == '-p': preserve_timestamps = True + if o == '-n': + create_backup = False if not new_interpreter or not new_interpreter.startswith(b'/') or \ not args: err('-i option or file-or-directory missing\n') @@ -134,10 +140,16 @@ def fix(filename): except OSError as msg: err('%s: warning: chmod failed (%r)\n' % (tempname, msg)) # Then make a backup of the original file as filename~ - try: - os.rename(filename, filename + '~') - except OSError as msg: - err('%s: warning: backup failed (%r)\n' % (filename, msg)) + if create_backup: + try: + os.rename(filename, filename + '~') + except OSError as msg: + err('%s: warning: backup failed (%r)\n' % (filename, msg)) + else: + try: + os.remove(filename) + except OSError as msg: + err('%s: warning: removing failed (%r)\n' % (filename, msg)) # Now move the temp file to the original file try: os.rename(tempname, filename) From webhook-mailer at python.org Wed Mar 14 16:08:11 2018 From: webhook-mailer at python.org (Antoine Pitrou) Date: Wed, 14 Mar 2018 20:08:11 -0000 Subject: [Python-checkins] bpo-33021: Fix GCC 7 warning (-Wmaybe-uninitialized) in mmapmodule.c (#6117) Message-ID: <mailman.77.1521058093.1871.python-checkins@python.org> https://github.com/python/cpython/commit/d6e140466142018ddbb7541185348be2b833a2ce commit: d6e140466142018ddbb7541185348be2b833a2ce branch: master author: Zackery Spytz <zspytz at gmail.com> committer: Antoine Pitrou <pitrou at free.fr> date: 2018-03-14T21:08:01+01:00 summary: bpo-33021: Fix GCC 7 warning (-Wmaybe-uninitialized) in mmapmodule.c (#6117) files: M Modules/mmapmodule.c diff --git a/Modules/mmapmodule.c b/Modules/mmapmodule.c index 6abdc71bbb82..95d42d6dc5e5 100644 --- a/Modules/mmapmodule.c +++ b/Modules/mmapmodule.c @@ -1050,7 +1050,7 @@ static PyObject * new_mmap_object(PyTypeObject *type, PyObject *args, PyObject *kwdict) { struct _Py_stat_struct status; - int fstat_result; + int fstat_result = -1; mmap_object *m_obj; Py_ssize_t map_size; off_t offset = 0; From solipsis at pitrou.net Thu Mar 15 05:11:28 2018 From: solipsis at pitrou.net (solipsis at pitrou.net) Date: Thu, 15 Mar 2018 09:11:28 +0000 Subject: [Python-checkins] Daily reference leaks (4243df51fe43): sum=11 Message-ID: <20180315091128.1.E40783936E8FBE77@psf.io> results for 4243df51fe43 on branch "default" -------------------------------------------- test_collections leaked [0, 0, 7] memory blocks, sum=7 test_functools leaked [0, 3, 1] memory blocks, sum=4 test_multiprocessing_forkserver leaked [1, 0, -2] memory blocks, sum=-1 test_multiprocessing_spawn leaked [-1, 2, 0] memory blocks, sum=1 Command line was: ['./python', '-m', 'test.regrtest', '-uall', '-R', '3:3:/home/psf-users/antoine/refleaks/reflogwKL1DM', '--timeout', '7200'] From solipsis at pitrou.net Fri Mar 16 05:10:00 2018 From: solipsis at pitrou.net (solipsis at pitrou.net) Date: Fri, 16 Mar 2018 09:10:00 +0000 Subject: [Python-checkins] Daily reference leaks (4243df51fe43): sum=1 Message-ID: <20180316091000.1.E3DDD742719EDF04@psf.io> results for 4243df51fe43 on branch "default" -------------------------------------------- test_asyncio leaked [3, 0, 0] memory blocks, sum=3 test_collections leaked [-7, 1, 0] memory blocks, sum=-6 test_functools leaked [0, 3, 1] memory blocks, sum=4 Command line was: ['./python', '-m', 'test.regrtest', '-uall', '-R', '3:3:/home/psf-users/antoine/refleaks/reflogLUdo0M', '--timeout', '7200'] From webhook-mailer at python.org Sat Mar 17 01:41:30 2018 From: webhook-mailer at python.org (Nick Coghlan) Date: Sat, 17 Mar 2018 05:41:30 -0000 Subject: [Python-checkins] bpo-32374: m_traverse may be called with m_state=NULL (GH-5140) Message-ID: <mailman.78.1521265292.1871.python-checkins@python.org> https://github.com/python/cpython/commit/c2b0b12d1a137ada1023ab7c10b8d9a0249d95f9 commit: c2b0b12d1a137ada1023ab7c10b8d9a0249d95f9 branch: master author: Marcel Plch <gmarcel.plch at gmail.com> committer: Nick Coghlan <ncoghlan at gmail.com> date: 2018-03-17T15:41:20+10:00 summary: bpo-32374: m_traverse may be called with m_state=NULL (GH-5140) Multi-phase initialized modules allow m_traverse to be called while the module is still being initialized, so module authors may need to account for that. files: A Misc/NEWS.d/next/C API/2018-01-09-17-03-54.bpo-32374.SwwLoz.rst M Doc/c-api/module.rst M Lib/test/test_importlib/extension/test_loader.py M Modules/_testmultiphase.c M Objects/moduleobject.c diff --git a/Doc/c-api/module.rst b/Doc/c-api/module.rst index 7efab28af724..017b656854a8 100644 --- a/Doc/c-api/module.rst +++ b/Doc/c-api/module.rst @@ -196,17 +196,23 @@ or request "multi-phase initialization" by returning the definition struct itsel .. c:member:: traverseproc m_traverse A traversal function to call during GC traversal of the module object, or - *NULL* if not needed. + *NULL* if not needed. This function may be called before module state + is allocated (:c:func:`PyModule_GetState()` may return `NULL`), + and before the :c:member:`Py_mod_exec` function is executed. .. c:member:: inquiry m_clear A clear function to call during GC clearing of the module object, or - *NULL* if not needed. + *NULL* if not needed. This function may be called before module state + is allocated (:c:func:`PyModule_GetState()` may return `NULL`), + and before the :c:member:`Py_mod_exec` function is executed. .. c:member:: freefunc m_free A function to call during deallocation of the module object, or *NULL* if - not needed. + not needed. This function may be called before module state + is allocated (:c:func:`PyModule_GetState()` may return `NULL`), + and before the :c:member:`Py_mod_exec` function is executed. Single-phase initialization ........................... diff --git a/Lib/test/test_importlib/extension/test_loader.py b/Lib/test/test_importlib/extension/test_loader.py index 8d20040aab8d..53ac3c71d4b5 100644 --- a/Lib/test/test_importlib/extension/test_loader.py +++ b/Lib/test/test_importlib/extension/test_loader.py @@ -9,7 +9,7 @@ import unittest import importlib.util import importlib - +from test.support.script_helper import assert_python_failure class LoaderTests(abc.LoaderTests): @@ -267,6 +267,20 @@ def test_nonascii(self): self.assertEqual(module.__name__, name) self.assertEqual(module.__doc__, "Module named in %s" % lang) + @unittest.skipIf(not hasattr(sys, 'gettotalrefcount'), + '--with-pydebug has to be enabled for this test') + def test_bad_traverse(self): + ''' Issue #32374: Test that traverse fails when accessing per-module + state before Py_mod_exec was executed. + (Multiphase initialization modules only) + ''' + script = """if True: + import importlib.util as util + spec = util.find_spec('_testmultiphase') + spec.name = '_testmultiphase_with_bad_traverse' + m = spec.loader.create_module(spec)""" + assert_python_failure("-c", script) + (Frozen_MultiPhaseExtensionModuleTests, Source_MultiPhaseExtensionModuleTests diff --git a/Misc/NEWS.d/next/C API/2018-01-09-17-03-54.bpo-32374.SwwLoz.rst b/Misc/NEWS.d/next/C API/2018-01-09-17-03-54.bpo-32374.SwwLoz.rst new file mode 100644 index 000000000000..f9cf6d6b99ce --- /dev/null +++ b/Misc/NEWS.d/next/C API/2018-01-09-17-03-54.bpo-32374.SwwLoz.rst @@ -0,0 +1,2 @@ +Document that m_traverse for multi-phase initialized modules can be called +with m_state=NULL, and add a sanity check diff --git a/Modules/_testmultiphase.c b/Modules/_testmultiphase.c index 9b04ebf15586..8090a485f4a9 100644 --- a/Modules/_testmultiphase.c +++ b/Modules/_testmultiphase.c @@ -10,6 +10,10 @@ typedef struct { PyObject *x_attr; /* Attributes dictionary */ } ExampleObject; +typedef struct { + PyObject *integer; +} testmultiphase_state; + /* Example methods */ static int @@ -218,18 +222,21 @@ static int execfunc(PyObject *m) } /* Helper for module definitions; there'll be a lot of them */ -#define TEST_MODULE_DEF(name, slots, methods) { \ + +#define TEST_MODULE_DEF_EX(name, slots, methods, statesize, traversefunc) { \ PyModuleDef_HEAD_INIT, /* m_base */ \ name, /* m_name */ \ PyDoc_STR("Test module " name), /* m_doc */ \ - 0, /* m_size */ \ + statesize, /* m_size */ \ methods, /* m_methods */ \ slots, /* m_slots */ \ - NULL, /* m_traverse */ \ + traversefunc, /* m_traverse */ \ NULL, /* m_clear */ \ NULL, /* m_free */ \ } +#define TEST_MODULE_DEF(name, slots, methods) TEST_MODULE_DEF_EX(name, slots, methods, 0, NULL) + PyModuleDef_Slot main_slots[] = { {Py_mod_exec, execfunc}, {0, NULL}, @@ -613,6 +620,44 @@ PyInit__testmultiphase_exec_unreported_exception(PyObject *spec) return PyModuleDef_Init(&def_exec_unreported_exception); } +static int +bad_traverse(PyObject *self, visitproc visit, void *arg) { + testmultiphase_state *m_state; + + m_state = PyModule_GetState(self); + Py_VISIT(m_state->integer); + return 0; +} + +static int +execfunc_with_bad_traverse(PyObject *mod) { + testmultiphase_state *m_state; + + m_state = PyModule_GetState(mod); + if (m_state == NULL) { + return -1; + } + + m_state->integer = PyLong_FromLong(0x7fffffff); + Py_INCREF(m_state->integer); + + return 0; +} + +static PyModuleDef_Slot slots_with_bad_traverse[] = { + {Py_mod_exec, execfunc_with_bad_traverse}, + {0, NULL} +}; + +static PyModuleDef def_with_bad_traverse = TEST_MODULE_DEF_EX( + "_testmultiphase_with_bad_traverse", slots_with_bad_traverse, NULL, + sizeof(testmultiphase_state), bad_traverse); + +PyMODINIT_FUNC +PyInit__testmultiphase_with_bad_traverse(PyObject *spec) { + return PyModuleDef_Init(&def_with_bad_traverse); +} + /*** Helper for imp test ***/ static PyModuleDef imp_dummy_def = TEST_MODULE_DEF("imp_dummy", main_slots, testexport_methods); @@ -622,3 +667,4 @@ PyInit_imp_dummy(PyObject *spec) { return PyModuleDef_Init(&imp_dummy_def); } + diff --git a/Objects/moduleobject.c b/Objects/moduleobject.c index d6cde4024336..8fb368e41474 100644 --- a/Objects/moduleobject.c +++ b/Objects/moduleobject.c @@ -21,6 +21,17 @@ static PyMemberDef module_members[] = { {0} }; + +/* Helper for sanity check for traverse not handling m_state == NULL + * Issue #32374 */ +#ifdef Py_DEBUG +static int +bad_traverse_test(PyObject *self, void *arg) { + assert(self != NULL); + return 0; +} +#endif + PyTypeObject PyModuleDef_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) "moduledef", /* tp_name */ @@ -345,6 +356,16 @@ PyModule_FromDefAndSpec2(struct PyModuleDef* def, PyObject *spec, int module_api } } + /* Sanity check for traverse not handling m_state == NULL + * This doesn't catch all possible cases, but in many cases it should + * make many cases of invalid code crash or raise Valgrind issues + * sooner than they would otherwise. + * Issue #32374 */ +#ifdef Py_DEBUG + if (def->m_traverse != NULL) { + def->m_traverse(m, bad_traverse_test, NULL); + } +#endif Py_DECREF(nameobj); return m; From webhook-mailer at python.org Sat Mar 17 02:03:59 2018 From: webhook-mailer at python.org (Miss Islington (bot)) Date: Sat, 17 Mar 2018 06:03:59 -0000 Subject: [Python-checkins] bpo-32374: m_traverse may be called with m_state=NULL (GH-5140) Message-ID: <mailman.79.1521266641.1871.python-checkins@python.org> https://github.com/python/cpython/commit/136905fffd5f77395f80e3409630c11756b5469c commit: 136905fffd5f77395f80e3409630c11756b5469c branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: GitHub <noreply at github.com> date: 2018-03-16T23:03:56-07:00 summary: bpo-32374: m_traverse may be called with m_state=NULL (GH-5140) Multi-phase initialized modules allow m_traverse to be called while the module is still being initialized, so module authors may need to account for that. (cherry picked from commit c2b0b12d1a137ada1023ab7c10b8d9a0249d95f9) Co-authored-by: Marcel Plch <gmarcel.plch at gmail.com> files: A Misc/NEWS.d/next/C API/2018-01-09-17-03-54.bpo-32374.SwwLoz.rst M Doc/c-api/module.rst M Lib/test/test_importlib/extension/test_loader.py M Modules/_testmultiphase.c M Objects/moduleobject.c diff --git a/Doc/c-api/module.rst b/Doc/c-api/module.rst index 7efab28af724..017b656854a8 100644 --- a/Doc/c-api/module.rst +++ b/Doc/c-api/module.rst @@ -196,17 +196,23 @@ or request "multi-phase initialization" by returning the definition struct itsel .. c:member:: traverseproc m_traverse A traversal function to call during GC traversal of the module object, or - *NULL* if not needed. + *NULL* if not needed. This function may be called before module state + is allocated (:c:func:`PyModule_GetState()` may return `NULL`), + and before the :c:member:`Py_mod_exec` function is executed. .. c:member:: inquiry m_clear A clear function to call during GC clearing of the module object, or - *NULL* if not needed. + *NULL* if not needed. This function may be called before module state + is allocated (:c:func:`PyModule_GetState()` may return `NULL`), + and before the :c:member:`Py_mod_exec` function is executed. .. c:member:: freefunc m_free A function to call during deallocation of the module object, or *NULL* if - not needed. + not needed. This function may be called before module state + is allocated (:c:func:`PyModule_GetState()` may return `NULL`), + and before the :c:member:`Py_mod_exec` function is executed. Single-phase initialization ........................... diff --git a/Lib/test/test_importlib/extension/test_loader.py b/Lib/test/test_importlib/extension/test_loader.py index 8d20040aab8d..53ac3c71d4b5 100644 --- a/Lib/test/test_importlib/extension/test_loader.py +++ b/Lib/test/test_importlib/extension/test_loader.py @@ -9,7 +9,7 @@ import unittest import importlib.util import importlib - +from test.support.script_helper import assert_python_failure class LoaderTests(abc.LoaderTests): @@ -267,6 +267,20 @@ def test_nonascii(self): self.assertEqual(module.__name__, name) self.assertEqual(module.__doc__, "Module named in %s" % lang) + @unittest.skipIf(not hasattr(sys, 'gettotalrefcount'), + '--with-pydebug has to be enabled for this test') + def test_bad_traverse(self): + ''' Issue #32374: Test that traverse fails when accessing per-module + state before Py_mod_exec was executed. + (Multiphase initialization modules only) + ''' + script = """if True: + import importlib.util as util + spec = util.find_spec('_testmultiphase') + spec.name = '_testmultiphase_with_bad_traverse' + m = spec.loader.create_module(spec)""" + assert_python_failure("-c", script) + (Frozen_MultiPhaseExtensionModuleTests, Source_MultiPhaseExtensionModuleTests diff --git a/Misc/NEWS.d/next/C API/2018-01-09-17-03-54.bpo-32374.SwwLoz.rst b/Misc/NEWS.d/next/C API/2018-01-09-17-03-54.bpo-32374.SwwLoz.rst new file mode 100644 index 000000000000..f9cf6d6b99ce --- /dev/null +++ b/Misc/NEWS.d/next/C API/2018-01-09-17-03-54.bpo-32374.SwwLoz.rst @@ -0,0 +1,2 @@ +Document that m_traverse for multi-phase initialized modules can be called +with m_state=NULL, and add a sanity check diff --git a/Modules/_testmultiphase.c b/Modules/_testmultiphase.c index 9b04ebf15586..8090a485f4a9 100644 --- a/Modules/_testmultiphase.c +++ b/Modules/_testmultiphase.c @@ -10,6 +10,10 @@ typedef struct { PyObject *x_attr; /* Attributes dictionary */ } ExampleObject; +typedef struct { + PyObject *integer; +} testmultiphase_state; + /* Example methods */ static int @@ -218,18 +222,21 @@ static int execfunc(PyObject *m) } /* Helper for module definitions; there'll be a lot of them */ -#define TEST_MODULE_DEF(name, slots, methods) { \ + +#define TEST_MODULE_DEF_EX(name, slots, methods, statesize, traversefunc) { \ PyModuleDef_HEAD_INIT, /* m_base */ \ name, /* m_name */ \ PyDoc_STR("Test module " name), /* m_doc */ \ - 0, /* m_size */ \ + statesize, /* m_size */ \ methods, /* m_methods */ \ slots, /* m_slots */ \ - NULL, /* m_traverse */ \ + traversefunc, /* m_traverse */ \ NULL, /* m_clear */ \ NULL, /* m_free */ \ } +#define TEST_MODULE_DEF(name, slots, methods) TEST_MODULE_DEF_EX(name, slots, methods, 0, NULL) + PyModuleDef_Slot main_slots[] = { {Py_mod_exec, execfunc}, {0, NULL}, @@ -613,6 +620,44 @@ PyInit__testmultiphase_exec_unreported_exception(PyObject *spec) return PyModuleDef_Init(&def_exec_unreported_exception); } +static int +bad_traverse(PyObject *self, visitproc visit, void *arg) { + testmultiphase_state *m_state; + + m_state = PyModule_GetState(self); + Py_VISIT(m_state->integer); + return 0; +} + +static int +execfunc_with_bad_traverse(PyObject *mod) { + testmultiphase_state *m_state; + + m_state = PyModule_GetState(mod); + if (m_state == NULL) { + return -1; + } + + m_state->integer = PyLong_FromLong(0x7fffffff); + Py_INCREF(m_state->integer); + + return 0; +} + +static PyModuleDef_Slot slots_with_bad_traverse[] = { + {Py_mod_exec, execfunc_with_bad_traverse}, + {0, NULL} +}; + +static PyModuleDef def_with_bad_traverse = TEST_MODULE_DEF_EX( + "_testmultiphase_with_bad_traverse", slots_with_bad_traverse, NULL, + sizeof(testmultiphase_state), bad_traverse); + +PyMODINIT_FUNC +PyInit__testmultiphase_with_bad_traverse(PyObject *spec) { + return PyModuleDef_Init(&def_with_bad_traverse); +} + /*** Helper for imp test ***/ static PyModuleDef imp_dummy_def = TEST_MODULE_DEF("imp_dummy", main_slots, testexport_methods); @@ -622,3 +667,4 @@ PyInit_imp_dummy(PyObject *spec) { return PyModuleDef_Init(&imp_dummy_def); } + diff --git a/Objects/moduleobject.c b/Objects/moduleobject.c index d6cde4024336..8fb368e41474 100644 --- a/Objects/moduleobject.c +++ b/Objects/moduleobject.c @@ -21,6 +21,17 @@ static PyMemberDef module_members[] = { {0} }; + +/* Helper for sanity check for traverse not handling m_state == NULL + * Issue #32374 */ +#ifdef Py_DEBUG +static int +bad_traverse_test(PyObject *self, void *arg) { + assert(self != NULL); + return 0; +} +#endif + PyTypeObject PyModuleDef_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) "moduledef", /* tp_name */ @@ -345,6 +356,16 @@ PyModule_FromDefAndSpec2(struct PyModuleDef* def, PyObject *spec, int module_api } } + /* Sanity check for traverse not handling m_state == NULL + * This doesn't catch all possible cases, but in many cases it should + * make many cases of invalid code crash or raise Valgrind issues + * sooner than they would otherwise. + * Issue #32374 */ +#ifdef Py_DEBUG + if (def->m_traverse != NULL) { + def->m_traverse(m, bad_traverse_test, NULL); + } +#endif Py_DECREF(nameobj); return m; From webhook-mailer at python.org Sat Mar 17 02:29:32 2018 From: webhook-mailer at python.org (Miss Islington (bot)) Date: Sat, 17 Mar 2018 06:29:32 -0000 Subject: [Python-checkins] bpo-32374: m_traverse may be called with m_state=NULL (GH-5140) Message-ID: <mailman.80.1521268175.1871.python-checkins@python.org> https://github.com/python/cpython/commit/1da0479f687613a43620430616f4db87e2ed4423 commit: 1da0479f687613a43620430616f4db87e2ed4423 branch: 3.6 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: GitHub <noreply at github.com> date: 2018-03-16T23:29:30-07:00 summary: bpo-32374: m_traverse may be called with m_state=NULL (GH-5140) Multi-phase initialized modules allow m_traverse to be called while the module is still being initialized, so module authors may need to account for that. (cherry picked from commit c2b0b12d1a137ada1023ab7c10b8d9a0249d95f9) Co-authored-by: Marcel Plch <gmarcel.plch at gmail.com> files: A Misc/NEWS.d/next/C API/2018-01-09-17-03-54.bpo-32374.SwwLoz.rst M Doc/c-api/module.rst M Lib/test/test_importlib/extension/test_loader.py M Modules/_testmultiphase.c M Objects/moduleobject.c diff --git a/Doc/c-api/module.rst b/Doc/c-api/module.rst index d3125b86f072..797a67e49c72 100644 --- a/Doc/c-api/module.rst +++ b/Doc/c-api/module.rst @@ -196,17 +196,23 @@ or request "multi-phase initialization" by returning the definition struct itsel .. c:member:: traverseproc m_traverse A traversal function to call during GC traversal of the module object, or - *NULL* if not needed. + *NULL* if not needed. This function may be called before module state + is allocated (:c:func:`PyModule_GetState()` may return `NULL`), + and before the :c:member:`Py_mod_exec` function is executed. .. c:member:: inquiry m_clear A clear function to call during GC clearing of the module object, or - *NULL* if not needed. + *NULL* if not needed. This function may be called before module state + is allocated (:c:func:`PyModule_GetState()` may return `NULL`), + and before the :c:member:`Py_mod_exec` function is executed. .. c:member:: freefunc m_free A function to call during deallocation of the module object, or *NULL* if - not needed. + not needed. This function may be called before module state + is allocated (:c:func:`PyModule_GetState()` may return `NULL`), + and before the :c:member:`Py_mod_exec` function is executed. Single-phase initialization ........................... diff --git a/Lib/test/test_importlib/extension/test_loader.py b/Lib/test/test_importlib/extension/test_loader.py index 8d20040aab8d..53ac3c71d4b5 100644 --- a/Lib/test/test_importlib/extension/test_loader.py +++ b/Lib/test/test_importlib/extension/test_loader.py @@ -9,7 +9,7 @@ import unittest import importlib.util import importlib - +from test.support.script_helper import assert_python_failure class LoaderTests(abc.LoaderTests): @@ -267,6 +267,20 @@ def test_nonascii(self): self.assertEqual(module.__name__, name) self.assertEqual(module.__doc__, "Module named in %s" % lang) + @unittest.skipIf(not hasattr(sys, 'gettotalrefcount'), + '--with-pydebug has to be enabled for this test') + def test_bad_traverse(self): + ''' Issue #32374: Test that traverse fails when accessing per-module + state before Py_mod_exec was executed. + (Multiphase initialization modules only) + ''' + script = """if True: + import importlib.util as util + spec = util.find_spec('_testmultiphase') + spec.name = '_testmultiphase_with_bad_traverse' + m = spec.loader.create_module(spec)""" + assert_python_failure("-c", script) + (Frozen_MultiPhaseExtensionModuleTests, Source_MultiPhaseExtensionModuleTests diff --git a/Misc/NEWS.d/next/C API/2018-01-09-17-03-54.bpo-32374.SwwLoz.rst b/Misc/NEWS.d/next/C API/2018-01-09-17-03-54.bpo-32374.SwwLoz.rst new file mode 100644 index 000000000000..f9cf6d6b99ce --- /dev/null +++ b/Misc/NEWS.d/next/C API/2018-01-09-17-03-54.bpo-32374.SwwLoz.rst @@ -0,0 +1,2 @@ +Document that m_traverse for multi-phase initialized modules can be called +with m_state=NULL, and add a sanity check diff --git a/Modules/_testmultiphase.c b/Modules/_testmultiphase.c index 4daa34e10614..8891d8d21dfc 100644 --- a/Modules/_testmultiphase.c +++ b/Modules/_testmultiphase.c @@ -10,6 +10,10 @@ typedef struct { PyObject *x_attr; /* Attributes dictionary */ } ExampleObject; +typedef struct { + PyObject *integer; +} testmultiphase_state; + /* Example methods */ static int @@ -219,18 +223,21 @@ static int execfunc(PyObject *m) } /* Helper for module definitions; there'll be a lot of them */ -#define TEST_MODULE_DEF(name, slots, methods) { \ + +#define TEST_MODULE_DEF_EX(name, slots, methods, statesize, traversefunc) { \ PyModuleDef_HEAD_INIT, /* m_base */ \ name, /* m_name */ \ PyDoc_STR("Test module " name), /* m_doc */ \ - 0, /* m_size */ \ + statesize, /* m_size */ \ methods, /* m_methods */ \ slots, /* m_slots */ \ - NULL, /* m_traverse */ \ + traversefunc, /* m_traverse */ \ NULL, /* m_clear */ \ NULL, /* m_free */ \ } +#define TEST_MODULE_DEF(name, slots, methods) TEST_MODULE_DEF_EX(name, slots, methods, 0, NULL) + PyModuleDef_Slot main_slots[] = { {Py_mod_exec, execfunc}, {0, NULL}, @@ -614,6 +621,44 @@ PyInit__testmultiphase_exec_unreported_exception(PyObject *spec) return PyModuleDef_Init(&def_exec_unreported_exception); } +static int +bad_traverse(PyObject *self, visitproc visit, void *arg) { + testmultiphase_state *m_state; + + m_state = PyModule_GetState(self); + Py_VISIT(m_state->integer); + return 0; +} + +static int +execfunc_with_bad_traverse(PyObject *mod) { + testmultiphase_state *m_state; + + m_state = PyModule_GetState(mod); + if (m_state == NULL) { + return -1; + } + + m_state->integer = PyLong_FromLong(0x7fffffff); + Py_INCREF(m_state->integer); + + return 0; +} + +static PyModuleDef_Slot slots_with_bad_traverse[] = { + {Py_mod_exec, execfunc_with_bad_traverse}, + {0, NULL} +}; + +static PyModuleDef def_with_bad_traverse = TEST_MODULE_DEF_EX( + "_testmultiphase_with_bad_traverse", slots_with_bad_traverse, NULL, + sizeof(testmultiphase_state), bad_traverse); + +PyMODINIT_FUNC +PyInit__testmultiphase_with_bad_traverse(PyObject *spec) { + return PyModuleDef_Init(&def_with_bad_traverse); +} + /*** Helper for imp test ***/ static PyModuleDef imp_dummy_def = TEST_MODULE_DEF("imp_dummy", main_slots, testexport_methods); @@ -623,3 +668,4 @@ PyInit_imp_dummy(PyObject *spec) { return PyModuleDef_Init(&imp_dummy_def); } + diff --git a/Objects/moduleobject.c b/Objects/moduleobject.c index f357af2b2118..c5979329647b 100644 --- a/Objects/moduleobject.c +++ b/Objects/moduleobject.c @@ -20,6 +20,17 @@ static PyMemberDef module_members[] = { {0} }; + +/* Helper for sanity check for traverse not handling m_state == NULL + * Issue #32374 */ +#ifdef Py_DEBUG +static int +bad_traverse_test(PyObject *self, void *arg) { + assert(self != NULL); + return 0; +} +#endif + PyTypeObject PyModuleDef_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) "moduledef", /* tp_name */ @@ -338,6 +349,16 @@ PyModule_FromDefAndSpec2(struct PyModuleDef* def, PyObject *spec, int module_api } } + /* Sanity check for traverse not handling m_state == NULL + * This doesn't catch all possible cases, but in many cases it should + * make many cases of invalid code crash or raise Valgrind issues + * sooner than they would otherwise. + * Issue #32374 */ +#ifdef Py_DEBUG + if (def->m_traverse != NULL) { + def->m_traverse(m, bad_traverse_test, NULL); + } +#endif Py_DECREF(nameobj); return m; From solipsis at pitrou.net Sat Mar 17 05:15:24 2018 From: solipsis at pitrou.net (solipsis at pitrou.net) Date: Sat, 17 Mar 2018 09:15:24 +0000 Subject: [Python-checkins] Daily reference leaks (4243df51fe43): sum=12 Message-ID: <20180317091524.1.DFC2FCA6FA8B5732@psf.io> results for 4243df51fe43 on branch "default" -------------------------------------------- test_collections leaked [7, -7, 8] memory blocks, sum=8 test_functools leaked [0, 3, 1] memory blocks, sum=4 Command line was: ['./python', '-m', 'test.regrtest', '-uall', '-R', '3:3:/home/psf-users/antoine/refleaks/reflog5_4zyE', '--timeout', '7200'] From webhook-mailer at python.org Sat Mar 17 11:13:51 2018 From: webhook-mailer at python.org (Donald Stufft) Date: Sat, 17 Mar 2018 15:13:51 -0000 Subject: [Python-checkins] Update pip to 9.0.2 and setuptools to 38.6.1 (#6133) Message-ID: <mailman.81.1521299632.1871.python-checkins@python.org> https://github.com/python/cpython/commit/7f81bb2addbbccfa45a2fc1aa6030f26dcf4bd78 commit: 7f81bb2addbbccfa45a2fc1aa6030f26dcf4bd78 branch: master author: Donald Stufft <donald at stufft.io> committer: GitHub <noreply at github.com> date: 2018-03-17T11:13:48-04:00 summary: Update pip to 9.0.2 and setuptools to 38.6.1 (#6133) files: A Lib/ensurepip/_bundled/pip-9.0.2-py2.py3-none-any.whl A Lib/ensurepip/_bundled/setuptools-38.6.1-py2.py3-none-any.whl D Lib/ensurepip/_bundled/pip-9.0.1-py2.py3-none-any.whl D Lib/ensurepip/_bundled/setuptools-28.8.0-py2.py3-none-any.whl M Lib/ensurepip/__init__.py diff --git a/Lib/ensurepip/__init__.py b/Lib/ensurepip/__init__.py index d69e09fab084..d7a6dcf6bc85 100644 --- a/Lib/ensurepip/__init__.py +++ b/Lib/ensurepip/__init__.py @@ -8,9 +8,9 @@ __all__ = ["version", "bootstrap"] -_SETUPTOOLS_VERSION = "28.8.0" +_SETUPTOOLS_VERSION = "38.6.1" -_PIP_VERSION = "9.0.1" +_PIP_VERSION = "9.0.2" _PROJECTS = [ ("setuptools", _SETUPTOOLS_VERSION), diff --git a/Lib/ensurepip/_bundled/pip-9.0.1-py2.py3-none-any.whl b/Lib/ensurepip/_bundled/pip-9.0.2-py2.py3-none-any.whl similarity index 54% rename from Lib/ensurepip/_bundled/pip-9.0.1-py2.py3-none-any.whl rename to Lib/ensurepip/_bundled/pip-9.0.2-py2.py3-none-any.whl index 4b8ecc69db7e..baf1f57e3d8b 100644 Binary files a/Lib/ensurepip/_bundled/pip-9.0.1-py2.py3-none-any.whl and b/Lib/ensurepip/_bundled/pip-9.0.2-py2.py3-none-any.whl differ diff --git a/Lib/ensurepip/_bundled/setuptools-28.8.0-py2.py3-none-any.whl b/Lib/ensurepip/_bundled/setuptools-38.6.1-py2.py3-none-any.whl similarity index 65% rename from Lib/ensurepip/_bundled/setuptools-28.8.0-py2.py3-none-any.whl rename to Lib/ensurepip/_bundled/setuptools-38.6.1-py2.py3-none-any.whl index 502e3cb418c1..2707d8be4a2d 100644 Binary files a/Lib/ensurepip/_bundled/setuptools-28.8.0-py2.py3-none-any.whl and b/Lib/ensurepip/_bundled/setuptools-38.6.1-py2.py3-none-any.whl differ From webhook-mailer at python.org Sat Mar 17 11:46:08 2018 From: webhook-mailer at python.org (Donald Stufft) Date: Sat, 17 Mar 2018 15:46:08 -0000 Subject: [Python-checkins] Update pip to 9.0.2 and setuptools to 38.6.1 (GH-6133) (#6134) Message-ID: <mailman.82.1521301569.1871.python-checkins@python.org> https://github.com/python/cpython/commit/d7b81037412291fb08d8dbdc5249d670e94c645a commit: d7b81037412291fb08d8dbdc5249d670e94c645a branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Donald Stufft <donald at stufft.io> date: 2018-03-17T11:46:05-04:00 summary: Update pip to 9.0.2 and setuptools to 38.6.1 (GH-6133) (#6134) (cherry picked from commit 7f81bb2addbbccfa45a2fc1aa6030f26dcf4bd78) Co-authored-by: Donald Stufft <donald at stufft.io> files: A Lib/ensurepip/_bundled/pip-9.0.2-py2.py3-none-any.whl A Lib/ensurepip/_bundled/setuptools-38.6.1-py2.py3-none-any.whl D Lib/ensurepip/_bundled/pip-9.0.1-py2.py3-none-any.whl D Lib/ensurepip/_bundled/setuptools-28.8.0-py2.py3-none-any.whl M Lib/ensurepip/__init__.py diff --git a/Lib/ensurepip/__init__.py b/Lib/ensurepip/__init__.py index d69e09fab084..d7a6dcf6bc85 100644 --- a/Lib/ensurepip/__init__.py +++ b/Lib/ensurepip/__init__.py @@ -8,9 +8,9 @@ __all__ = ["version", "bootstrap"] -_SETUPTOOLS_VERSION = "28.8.0" +_SETUPTOOLS_VERSION = "38.6.1" -_PIP_VERSION = "9.0.1" +_PIP_VERSION = "9.0.2" _PROJECTS = [ ("setuptools", _SETUPTOOLS_VERSION), diff --git a/Lib/ensurepip/_bundled/pip-9.0.1-py2.py3-none-any.whl b/Lib/ensurepip/_bundled/pip-9.0.2-py2.py3-none-any.whl similarity index 54% rename from Lib/ensurepip/_bundled/pip-9.0.1-py2.py3-none-any.whl rename to Lib/ensurepip/_bundled/pip-9.0.2-py2.py3-none-any.whl index 4b8ecc69db7e..baf1f57e3d8b 100644 Binary files a/Lib/ensurepip/_bundled/pip-9.0.1-py2.py3-none-any.whl and b/Lib/ensurepip/_bundled/pip-9.0.2-py2.py3-none-any.whl differ diff --git a/Lib/ensurepip/_bundled/setuptools-28.8.0-py2.py3-none-any.whl b/Lib/ensurepip/_bundled/setuptools-38.6.1-py2.py3-none-any.whl similarity index 65% rename from Lib/ensurepip/_bundled/setuptools-28.8.0-py2.py3-none-any.whl rename to Lib/ensurepip/_bundled/setuptools-38.6.1-py2.py3-none-any.whl index 502e3cb418c1..2707d8be4a2d 100644 Binary files a/Lib/ensurepip/_bundled/setuptools-28.8.0-py2.py3-none-any.whl and b/Lib/ensurepip/_bundled/setuptools-38.6.1-py2.py3-none-any.whl differ From webhook-mailer at python.org Sat Mar 17 11:54:49 2018 From: webhook-mailer at python.org (Donald Stufft) Date: Sat, 17 Mar 2018 15:54:49 -0000 Subject: [Python-checkins] Update pip to 9.0.2 and setuptools to 38.6.1 (GH-6133) (GH-6135) Message-ID: <mailman.83.1521302090.1871.python-checkins@python.org> https://github.com/python/cpython/commit/4e907d8fafde6b8df9b8b461d9a6fbbf3b8674bd commit: 4e907d8fafde6b8df9b8b461d9a6fbbf3b8674bd branch: 2.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Donald Stufft <donald at stufft.io> date: 2018-03-17T11:54:45-04:00 summary: Update pip to 9.0.2 and setuptools to 38.6.1 (GH-6133) (GH-6135) (cherry picked from commit 7f81bb2addbbccfa45a2fc1aa6030f26dcf4bd78) Co-authored-by: Donald Stufft <donald at stufft.io> files: A Lib/ensurepip/_bundled/pip-9.0.2-py2.py3-none-any.whl A Lib/ensurepip/_bundled/setuptools-38.6.1-py2.py3-none-any.whl D Lib/ensurepip/_bundled/pip-9.0.1-py2.py3-none-any.whl D Lib/ensurepip/_bundled/setuptools-28.8.0-py2.py3-none-any.whl M Lib/ensurepip/__init__.py diff --git a/Lib/ensurepip/__init__.py b/Lib/ensurepip/__init__.py index ea2a5e634822..167696720267 100644 --- a/Lib/ensurepip/__init__.py +++ b/Lib/ensurepip/__init__.py @@ -12,9 +12,9 @@ __all__ = ["version", "bootstrap"] -_SETUPTOOLS_VERSION = "28.8.0" +_SETUPTOOLS_VERSION = "38.6.1" -_PIP_VERSION = "9.0.1" +_PIP_VERSION = "9.0.2" _PROJECTS = [ ("setuptools", _SETUPTOOLS_VERSION), diff --git a/Lib/ensurepip/_bundled/pip-9.0.1-py2.py3-none-any.whl b/Lib/ensurepip/_bundled/pip-9.0.2-py2.py3-none-any.whl similarity index 54% rename from Lib/ensurepip/_bundled/pip-9.0.1-py2.py3-none-any.whl rename to Lib/ensurepip/_bundled/pip-9.0.2-py2.py3-none-any.whl index 4b8ecc69db7e..baf1f57e3d8b 100644 Binary files a/Lib/ensurepip/_bundled/pip-9.0.1-py2.py3-none-any.whl and b/Lib/ensurepip/_bundled/pip-9.0.2-py2.py3-none-any.whl differ diff --git a/Lib/ensurepip/_bundled/setuptools-28.8.0-py2.py3-none-any.whl b/Lib/ensurepip/_bundled/setuptools-38.6.1-py2.py3-none-any.whl similarity index 65% rename from Lib/ensurepip/_bundled/setuptools-28.8.0-py2.py3-none-any.whl rename to Lib/ensurepip/_bundled/setuptools-38.6.1-py2.py3-none-any.whl index 502e3cb418c1..2707d8be4a2d 100644 Binary files a/Lib/ensurepip/_bundled/setuptools-28.8.0-py2.py3-none-any.whl and b/Lib/ensurepip/_bundled/setuptools-38.6.1-py2.py3-none-any.whl differ From webhook-mailer at python.org Sat Mar 17 12:19:45 2018 From: webhook-mailer at python.org (Donald Stufft) Date: Sat, 17 Mar 2018 16:19:45 -0000 Subject: [Python-checkins] Update pip to 9.0.2 and setuptools to 38.6.1 (GH-6133) (#6136) Message-ID: <mailman.84.1521303586.1871.python-checkins@python.org> https://github.com/python/cpython/commit/5bd0fe6a00be51d76c9ed98bd38ccc0237861250 commit: 5bd0fe6a00be51d76c9ed98bd38ccc0237861250 branch: 3.6 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Donald Stufft <donald at stufft.io> date: 2018-03-17T12:19:41-04:00 summary: Update pip to 9.0.2 and setuptools to 38.6.1 (GH-6133) (#6136) (cherry picked from commit 7f81bb2addbbccfa45a2fc1aa6030f26dcf4bd78) Co-authored-by: Donald Stufft <donald at stufft.io> files: A Lib/ensurepip/_bundled/pip-9.0.2-py2.py3-none-any.whl A Lib/ensurepip/_bundled/setuptools-38.6.1-py2.py3-none-any.whl D Lib/ensurepip/_bundled/pip-9.0.1-py2.py3-none-any.whl D Lib/ensurepip/_bundled/setuptools-28.8.0-py2.py3-none-any.whl M Lib/ensurepip/__init__.py diff --git a/Lib/ensurepip/__init__.py b/Lib/ensurepip/__init__.py index d69e09fab084..d7a6dcf6bc85 100644 --- a/Lib/ensurepip/__init__.py +++ b/Lib/ensurepip/__init__.py @@ -8,9 +8,9 @@ __all__ = ["version", "bootstrap"] -_SETUPTOOLS_VERSION = "28.8.0" +_SETUPTOOLS_VERSION = "38.6.1" -_PIP_VERSION = "9.0.1" +_PIP_VERSION = "9.0.2" _PROJECTS = [ ("setuptools", _SETUPTOOLS_VERSION), diff --git a/Lib/ensurepip/_bundled/pip-9.0.1-py2.py3-none-any.whl b/Lib/ensurepip/_bundled/pip-9.0.2-py2.py3-none-any.whl similarity index 54% rename from Lib/ensurepip/_bundled/pip-9.0.1-py2.py3-none-any.whl rename to Lib/ensurepip/_bundled/pip-9.0.2-py2.py3-none-any.whl index 4b8ecc69db7e..baf1f57e3d8b 100644 Binary files a/Lib/ensurepip/_bundled/pip-9.0.1-py2.py3-none-any.whl and b/Lib/ensurepip/_bundled/pip-9.0.2-py2.py3-none-any.whl differ diff --git a/Lib/ensurepip/_bundled/setuptools-28.8.0-py2.py3-none-any.whl b/Lib/ensurepip/_bundled/setuptools-38.6.1-py2.py3-none-any.whl similarity index 65% rename from Lib/ensurepip/_bundled/setuptools-28.8.0-py2.py3-none-any.whl rename to Lib/ensurepip/_bundled/setuptools-38.6.1-py2.py3-none-any.whl index 502e3cb418c1..2707d8be4a2d 100644 Binary files a/Lib/ensurepip/_bundled/setuptools-28.8.0-py2.py3-none-any.whl and b/Lib/ensurepip/_bundled/setuptools-38.6.1-py2.py3-none-any.whl differ From webhook-mailer at python.org Sat Mar 17 20:49:04 2018 From: webhook-mailer at python.org (Berker Peksag) Date: Sun, 18 Mar 2018 00:49:04 -0000 Subject: [Python-checkins] bpo-27645: Fix version number in 'database in transaction' fallback (GH-6131) Message-ID: <mailman.85.1521334145.1871.python-checkins@python.org> https://github.com/python/cpython/commit/bbf7bb7a636b3112ef6f6b31df385606d52517ce commit: bbf7bb7a636b3112ef6f6b31df385606d52517ce branch: master author: Aviv Palivoda <palaviv at gmail.com> committer: Berker Peksag <berker.peksag at gmail.com> date: 2018-03-18T03:48:55+03:00 summary: bpo-27645: Fix version number in 'database in transaction' fallback (GH-6131) It was actually fixed in SQLite 3.8.8, not 3.8.7. files: M Lib/sqlite3/test/backup.py M Modules/_sqlite/connection.c diff --git a/Lib/sqlite3/test/backup.py b/Lib/sqlite3/test/backup.py index 2a2148937413..903bacf49030 100644 --- a/Lib/sqlite3/test/backup.py +++ b/Lib/sqlite3/test/backup.py @@ -37,14 +37,12 @@ def test_bad_target_closed_connection(self): self.cx.backup(bck) def test_bad_target_in_transaction(self): - if sqlite.sqlite_version_info == (3, 8, 7, 1): - self.skipTest('skip until we debug https://bugs.python.org/issue27645#msg313562') bck = sqlite.connect(':memory:') bck.execute('CREATE TABLE bar (key INTEGER)') bck.executemany('INSERT INTO bar (key) VALUES (?)', [(3,), (4,)]) with self.assertRaises(sqlite.OperationalError) as cm: self.cx.backup(bck) - if sqlite.sqlite_version_info < (3, 8, 7): + if sqlite.sqlite_version_info < (3, 8, 8): self.assertEqual(str(cm.exception), 'target is in transaction') def test_keyword_only_args(self): diff --git a/Modules/_sqlite/connection.c b/Modules/_sqlite/connection.c index 14b6a2774e78..dc5061c1d91d 100644 --- a/Modules/_sqlite/connection.c +++ b/Modules/_sqlite/connection.c @@ -1481,8 +1481,8 @@ pysqlite_connection_backup(pysqlite_Connection *self, PyObject *args, PyObject * return NULL; } -#if SQLITE_VERSION_NUMBER < 3008007 - /* Since 3.8.7 this is already done, per commit +#if SQLITE_VERSION_NUMBER < 3008008 + /* Since 3.8.8 this is already done, per commit https://www.sqlite.org/src/info/169b5505498c0a7e */ if (!sqlite3_get_autocommit(((pysqlite_Connection *)target)->db)) { PyErr_SetString(pysqlite_OperationalError, "target is in transaction"); From webhook-mailer at python.org Sun Mar 18 02:24:42 2018 From: webhook-mailer at python.org (Berker Peksag) Date: Sun, 18 Mar 2018 06:24:42 -0000 Subject: [Python-checkins] bpo-27645: Fix version number in 'database in transaction' fallback (GH-6131) Message-ID: <mailman.86.1521354284.1871.python-checkins@python.org> https://github.com/python/cpython/commit/429ca448d2a36040f229ad9edc67e31fc6d18bf4 commit: 429ca448d2a36040f229ad9edc67e31fc6d18bf4 branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Berker Peksag <berker.peksag at gmail.com> date: 2018-03-18T09:24:33+03:00 summary: bpo-27645: Fix version number in 'database in transaction' fallback (GH-6131) It was actually fixed in SQLite 3.8.8, not 3.8.7. (cherry picked from commit bbf7bb7a636b3112ef6f6b31df385606d52517ce) Co-authored-by: Aviv Palivoda <palaviv at gmail.com> files: M Lib/sqlite3/test/backup.py M Modules/_sqlite/connection.c diff --git a/Lib/sqlite3/test/backup.py b/Lib/sqlite3/test/backup.py index 2a2148937413..903bacf49030 100644 --- a/Lib/sqlite3/test/backup.py +++ b/Lib/sqlite3/test/backup.py @@ -37,14 +37,12 @@ def test_bad_target_closed_connection(self): self.cx.backup(bck) def test_bad_target_in_transaction(self): - if sqlite.sqlite_version_info == (3, 8, 7, 1): - self.skipTest('skip until we debug https://bugs.python.org/issue27645#msg313562') bck = sqlite.connect(':memory:') bck.execute('CREATE TABLE bar (key INTEGER)') bck.executemany('INSERT INTO bar (key) VALUES (?)', [(3,), (4,)]) with self.assertRaises(sqlite.OperationalError) as cm: self.cx.backup(bck) - if sqlite.sqlite_version_info < (3, 8, 7): + if sqlite.sqlite_version_info < (3, 8, 8): self.assertEqual(str(cm.exception), 'target is in transaction') def test_keyword_only_args(self): diff --git a/Modules/_sqlite/connection.c b/Modules/_sqlite/connection.c index 14b6a2774e78..dc5061c1d91d 100644 --- a/Modules/_sqlite/connection.c +++ b/Modules/_sqlite/connection.c @@ -1481,8 +1481,8 @@ pysqlite_connection_backup(pysqlite_Connection *self, PyObject *args, PyObject * return NULL; } -#if SQLITE_VERSION_NUMBER < 3008007 - /* Since 3.8.7 this is already done, per commit +#if SQLITE_VERSION_NUMBER < 3008008 + /* Since 3.8.8 this is already done, per commit https://www.sqlite.org/src/info/169b5505498c0a7e */ if (!sqlite3_get_autocommit(((pysqlite_Connection *)target)->db)) { PyErr_SetString(pysqlite_OperationalError, "target is in transaction"); From webhook-mailer at python.org Sun Mar 18 03:53:17 2018 From: webhook-mailer at python.org (Serhiy Storchaka) Date: Sun, 18 Mar 2018 07:53:17 -0000 Subject: [Python-checkins] bpo-33041: Add tests for jumps in/out of 'async with' blocks. (#6110) Message-ID: <mailman.87.1521359600.1871.python-checkins@python.org> https://github.com/python/cpython/commit/bc300ce205f99acb1ef92c37de06dc76147e073b commit: bc300ce205f99acb1ef92c37de06dc76147e073b branch: master author: Serhiy Storchaka <storchaka at gmail.com> committer: GitHub <noreply at github.com> date: 2018-03-18T09:53:08+02:00 summary: bpo-33041: Add tests for jumps in/out of 'async with' blocks. (#6110) files: M Lib/test/test_sys_settrace.py diff --git a/Lib/test/test_sys_settrace.py b/Lib/test/test_sys_settrace.py index 90d1e37a3916..2587794c69a3 100644 --- a/Lib/test/test_sys_settrace.py +++ b/Lib/test/test_sys_settrace.py @@ -6,6 +6,8 @@ import difflib import gc from functools import wraps +import asyncio + class tracecontext: """Context manager that traces its enter and exit.""" @@ -19,6 +21,20 @@ def __enter__(self): def __exit__(self, *exc_info): self.output.append(-self.value) +class asynctracecontext: + """Asynchronous context manager that traces its aenter and aexit.""" + def __init__(self, output, value): + self.output = output + self.value = value + + async def __aenter__(self): + self.output.append(self.value) + + async def __aexit__(self, *exc_info): + self.output.append(-self.value) + + + # A very basic example. If this fails, we're in deep trouble. def basic(): return 1 @@ -636,6 +652,19 @@ def run_test(self, func, jumpFrom, jumpTo, expected, error=None, sys.settrace(None) self.compare_jump_output(expected, output) + def run_async_test(self, func, jumpFrom, jumpTo, expected, error=None, + event='line', decorated=False): + tracer = JumpTracer(func, jumpFrom, jumpTo, event, decorated) + sys.settrace(tracer.trace) + output = [] + if error is None: + asyncio.run(func(output)) + else: + with self.assertRaisesRegex(*error): + asyncio.run(func(output)) + sys.settrace(None) + self.compare_jump_output(expected, output) + def jump_test(jumpFrom, jumpTo, expected, error=None, event='line'): """Decorator that creates a test that makes a jump from one place to another in the following code. @@ -648,6 +677,18 @@ def test(self): return test return decorator + def async_jump_test(jumpFrom, jumpTo, expected, error=None, event='line'): + """Decorator that creates a test that makes a jump + from one place to another in the following asynchronous code. + """ + def decorator(func): + @wraps(func) + def test(self): + self.run_async_test(func, jumpFrom, jumpTo, expected, + error=error, event=event, decorated=True) + return test + return decorator + ## The first set of 'jump' tests are for things that are allowed: @jump_test(1, 3, [3]) @@ -774,12 +815,24 @@ def test_jump_forwards_out_of_with_block(output): output.append(2) output.append(3) + @async_jump_test(2, 3, [1, 3]) + async def test_jump_forwards_out_of_async_with_block(output): + async with asynctracecontext(output, 1): + output.append(2) + output.append(3) + @jump_test(3, 1, [1, 2, 1, 2, 3, -2]) def test_jump_backwards_out_of_with_block(output): output.append(1) with tracecontext(output, 2): output.append(3) + @async_jump_test(3, 1, [1, 2, 1, 2, 3, -2]) + async def test_jump_backwards_out_of_async_with_block(output): + output.append(1) + async with asynctracecontext(output, 2): + output.append(3) + @jump_test(2, 5, [5]) def test_jump_forwards_out_of_try_finally_block(output): try: @@ -843,6 +896,14 @@ def test_jump_across_with(output): with tracecontext(output, 4): output.append(5) + @async_jump_test(2, 4, [1, 4, 5, -4]) + async def test_jump_across_async_with(output): + output.append(1) + async with asynctracecontext(output, 2): + output.append(3) + async with asynctracecontext(output, 4): + output.append(5) + @jump_test(4, 5, [1, 3, 5, 6]) def test_jump_out_of_with_block_within_for_block(output): output.append(1) @@ -852,6 +913,15 @@ def test_jump_out_of_with_block_within_for_block(output): output.append(5) output.append(6) + @async_jump_test(4, 5, [1, 3, 5, 6]) + async def test_jump_out_of_async_with_block_within_for_block(output): + output.append(1) + for i in [1]: + async with asynctracecontext(output, 3): + output.append(4) + output.append(5) + output.append(6) + @jump_test(4, 5, [1, 2, 3, 5, -2, 6]) def test_jump_out_of_with_block_within_with_block(output): output.append(1) @@ -861,6 +931,15 @@ def test_jump_out_of_with_block_within_with_block(output): output.append(5) output.append(6) + @async_jump_test(4, 5, [1, 2, 3, 5, -2, 6]) + async def test_jump_out_of_async_with_block_within_with_block(output): + output.append(1) + with tracecontext(output, 2): + async with asynctracecontext(output, 3): + output.append(4) + output.append(5) + output.append(6) + @jump_test(5, 6, [2, 4, 6, 7]) def test_jump_out_of_with_block_within_finally_block(output): try: @@ -871,6 +950,16 @@ def test_jump_out_of_with_block_within_finally_block(output): output.append(6) output.append(7) + @async_jump_test(5, 6, [2, 4, 6, 7]) + async def test_jump_out_of_async_with_block_within_finally_block(output): + try: + output.append(2) + finally: + async with asynctracecontext(output, 4): + output.append(5) + output.append(6) + output.append(7) + @jump_test(8, 11, [1, 3, 5, 11, 12]) def test_jump_out_of_complex_nested_blocks(output): output.append(1) @@ -894,6 +983,14 @@ def test_jump_out_of_with_assignment(output): output.append(4) output.append(5) + @async_jump_test(3, 5, [1, 2, 5]) + async def test_jump_out_of_async_with_assignment(output): + output.append(1) + async with asynctracecontext(output, 2) \ + as x: + output.append(4) + output.append(5) + @jump_test(3, 6, [1, 6, 8, 9]) def test_jump_over_return_in_try_finally_block(output): output.append(1) @@ -996,12 +1093,24 @@ def test_no_jump_forwards_into_with_block(output): with tracecontext(output, 2): output.append(3) + @async_jump_test(1, 3, [], (ValueError, 'into')) + async def test_no_jump_forwards_into_async_with_block(output): + output.append(1) + async with asynctracecontext(output, 2): + output.append(3) + @jump_test(3, 2, [1, 2, -1], (ValueError, 'into')) def test_no_jump_backwards_into_with_block(output): with tracecontext(output, 1): output.append(2) output.append(3) + @async_jump_test(3, 2, [1, 2, -1], (ValueError, 'into')) + async def test_no_jump_backwards_into_async_with_block(output): + async with asynctracecontext(output, 1): + output.append(2) + output.append(3) + @jump_test(1, 3, [], (ValueError, 'into')) def test_no_jump_forwards_into_try_finally_block(output): output.append(1) @@ -1082,6 +1191,14 @@ def test_no_jump_between_with_blocks(output): with tracecontext(output, 4): output.append(5) + @async_jump_test(3, 5, [1, 2, -2], (ValueError, 'into')) + async def test_no_jump_between_async_with_blocks(output): + output.append(1) + async with asynctracecontext(output, 2): + output.append(3) + async with asynctracecontext(output, 4): + output.append(5) + @jump_test(5, 7, [2, 4], (ValueError, 'finally')) def test_no_jump_over_return_out_of_finally_block(output): try: From webhook-mailer at python.org Sun Mar 18 03:55:56 2018 From: webhook-mailer at python.org (Serhiy Storchaka) Date: Sun, 18 Mar 2018 07:55:56 -0000 Subject: [Python-checkins] bpo-32056: Improve exceptions in aifc, wave and sunau. (GH-5951) Message-ID: <mailman.88.1521359757.1871.python-checkins@python.org> https://github.com/python/cpython/commit/134cb01cda50f02725575808130b05d2d776693f commit: 134cb01cda50f02725575808130b05d2d776693f branch: master author: Serhiy Storchaka <storchaka at gmail.com> committer: GitHub <noreply at github.com> date: 2018-03-18T09:55:53+02:00 summary: bpo-32056: Improve exceptions in aifc, wave and sunau. (GH-5951) files: A Misc/NEWS.d/next/Library/2018-03-01-17-49-56.bpo-32056.IlpfgE.rst M Lib/aifc.py M Lib/sunau.py M Lib/test/test_aifc.py M Lib/test/test_sunau.py M Lib/test/test_wave.py M Lib/wave.py diff --git a/Lib/aifc.py b/Lib/aifc.py index 3d2dc56de198..1916e7ef8e7e 100644 --- a/Lib/aifc.py +++ b/Lib/aifc.py @@ -467,6 +467,10 @@ def _read_comm_chunk(self, chunk): self._nframes = _read_long(chunk) self._sampwidth = (_read_short(chunk) + 7) // 8 self._framerate = int(_read_float(chunk)) + if self._sampwidth <= 0: + raise Error('bad sample width') + if self._nchannels <= 0: + raise Error('bad # of channels') self._framesize = self._nchannels * self._sampwidth if self._aifc: #DEBUG: SGI's soundeditor produces a bad size :-( diff --git a/Lib/sunau.py b/Lib/sunau.py index dbad3db8392d..129502b0b417 100644 --- a/Lib/sunau.py +++ b/Lib/sunau.py @@ -208,6 +208,8 @@ def initfp(self, file): raise Error('unknown encoding') self._framerate = int(_read_u32(file)) self._nchannels = int(_read_u32(file)) + if not self._nchannels: + raise Error('bad # of channels') self._framesize = self._framesize * self._nchannels if self._hdr_size > 24: self._info = file.read(self._hdr_size - 24) diff --git a/Lib/test/test_aifc.py b/Lib/test/test_aifc.py index 8fd306a36592..ff52f5b6feb8 100644 --- a/Lib/test/test_aifc.py +++ b/Lib/test/test_aifc.py @@ -268,7 +268,8 @@ def test_read_no_comm_chunk(self): def test_read_no_ssnd_chunk(self): b = b'FORM' + struct.pack('>L', 4) + b'AIFC' - b += b'COMM' + struct.pack('>LhlhhLL', 38, 0, 0, 0, 0, 0, 0) + b += b'COMM' + struct.pack('>LhlhhLL', 38, 1, 0, 8, + 0x4000 | 12, 11025<<18, 0) b += b'NONE' + struct.pack('B', 14) + b'not compressed' + b'\x00' with self.assertRaisesRegex(aifc.Error, 'COMM chunk and/or SSND chunk' ' missing'): @@ -276,13 +277,35 @@ def test_read_no_ssnd_chunk(self): def test_read_wrong_compression_type(self): b = b'FORM' + struct.pack('>L', 4) + b'AIFC' - b += b'COMM' + struct.pack('>LhlhhLL', 23, 0, 0, 0, 0, 0, 0) + b += b'COMM' + struct.pack('>LhlhhLL', 23, 1, 0, 8, + 0x4000 | 12, 11025<<18, 0) b += b'WRNG' + struct.pack('B', 0) self.assertRaises(aifc.Error, aifc.open, io.BytesIO(b)) + def test_read_wrong_number_of_channels(self): + for nchannels in 0, -1: + b = b'FORM' + struct.pack('>L', 4) + b'AIFC' + b += b'COMM' + struct.pack('>LhlhhLL', 38, nchannels, 0, 8, + 0x4000 | 12, 11025<<18, 0) + b += b'NONE' + struct.pack('B', 14) + b'not compressed' + b'\x00' + b += b'SSND' + struct.pack('>L', 8) + b'\x00' * 8 + with self.assertRaisesRegex(aifc.Error, 'bad # of channels'): + aifc.open(io.BytesIO(b)) + + def test_read_wrong_sample_width(self): + for sampwidth in 0, -1: + b = b'FORM' + struct.pack('>L', 4) + b'AIFC' + b += b'COMM' + struct.pack('>LhlhhLL', 38, 1, 0, sampwidth, + 0x4000 | 12, 11025<<18, 0) + b += b'NONE' + struct.pack('B', 14) + b'not compressed' + b'\x00' + b += b'SSND' + struct.pack('>L', 8) + b'\x00' * 8 + with self.assertRaisesRegex(aifc.Error, 'bad sample width'): + aifc.open(io.BytesIO(b)) + def test_read_wrong_marks(self): b = b'FORM' + struct.pack('>L', 4) + b'AIFF' - b += b'COMM' + struct.pack('>LhlhhLL', 18, 0, 0, 0, 0, 0, 0) + b += b'COMM' + struct.pack('>LhlhhLL', 18, 1, 0, 8, + 0x4000 | 12, 11025<<18, 0) b += b'SSND' + struct.pack('>L', 8) + b'\x00' * 8 b += b'MARK' + struct.pack('>LhB', 3, 1, 1) with self.assertWarns(UserWarning) as cm: @@ -293,7 +316,8 @@ def test_read_wrong_marks(self): def test_read_comm_kludge_compname_even(self): b = b'FORM' + struct.pack('>L', 4) + b'AIFC' - b += b'COMM' + struct.pack('>LhlhhLL', 18, 0, 0, 0, 0, 0, 0) + b += b'COMM' + struct.pack('>LhlhhLL', 18, 1, 0, 8, + 0x4000 | 12, 11025<<18, 0) b += b'NONE' + struct.pack('B', 4) + b'even' + b'\x00' b += b'SSND' + struct.pack('>L', 8) + b'\x00' * 8 with self.assertWarns(UserWarning) as cm: @@ -303,7 +327,8 @@ def test_read_comm_kludge_compname_even(self): def test_read_comm_kludge_compname_odd(self): b = b'FORM' + struct.pack('>L', 4) + b'AIFC' - b += b'COMM' + struct.pack('>LhlhhLL', 18, 0, 0, 0, 0, 0, 0) + b += b'COMM' + struct.pack('>LhlhhLL', 18, 1, 0, 8, + 0x4000 | 12, 11025<<18, 0) b += b'NONE' + struct.pack('B', 3) + b'odd' b += b'SSND' + struct.pack('>L', 8) + b'\x00' * 8 with self.assertWarns(UserWarning) as cm: diff --git a/Lib/test/test_sunau.py b/Lib/test/test_sunau.py index 966224b1df5a..470a1007b4d4 100644 --- a/Lib/test/test_sunau.py +++ b/Lib/test/test_sunau.py @@ -1,6 +1,8 @@ import unittest from test import audiotests from audioop import byteswap +import io +import struct import sys import sunau @@ -121,5 +123,40 @@ class SunauMiscTests(audiotests.AudioMiscTests, unittest.TestCase): module = sunau +class SunauLowLevelTest(unittest.TestCase): + + def test_read_bad_magic_number(self): + b = b'SPA' + with self.assertRaises(EOFError): + sunau.open(io.BytesIO(b)) + b = b'SPAM' + with self.assertRaisesRegex(sunau.Error, 'bad magic number'): + sunau.open(io.BytesIO(b)) + + def test_read_too_small_header(self): + b = struct.pack('>LLLLL', sunau.AUDIO_FILE_MAGIC, 20, 0, + sunau.AUDIO_FILE_ENCODING_LINEAR_8, 11025) + with self.assertRaisesRegex(sunau.Error, 'header size too small'): + sunau.open(io.BytesIO(b)) + + def test_read_too_large_header(self): + b = struct.pack('>LLLLLL', sunau.AUDIO_FILE_MAGIC, 124, 0, + sunau.AUDIO_FILE_ENCODING_LINEAR_8, 11025, 1) + b += b'\0' * 100 + with self.assertRaisesRegex(sunau.Error, 'header size ridiculously large'): + sunau.open(io.BytesIO(b)) + + def test_read_wrong_encoding(self): + b = struct.pack('>LLLLLL', sunau.AUDIO_FILE_MAGIC, 24, 0, 0, 11025, 1) + with self.assertRaisesRegex(sunau.Error, r'encoding not \(yet\) supported'): + sunau.open(io.BytesIO(b)) + + def test_read_wrong_number_of_channels(self): + b = struct.pack('>LLLLLL', sunau.AUDIO_FILE_MAGIC, 24, 0, + sunau.AUDIO_FILE_ENCODING_LINEAR_8, 11025, 0) + with self.assertRaisesRegex(sunau.Error, 'bad # of channels'): + sunau.open(io.BytesIO(b)) + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_wave.py b/Lib/test/test_wave.py index c5d2e02450ef..8a42f8e47105 100644 --- a/Lib/test/test_wave.py +++ b/Lib/test/test_wave.py @@ -2,6 +2,8 @@ from test import audiotests from test import support from audioop import byteswap +import io +import struct import sys import wave @@ -111,5 +113,65 @@ def test__all__(self): support.check__all__(self, wave, blacklist=blacklist) +class WaveLowLevelTest(unittest.TestCase): + + def test_read_no_chunks(self): + b = b'SPAM' + with self.assertRaises(EOFError): + wave.open(io.BytesIO(b)) + + def test_read_no_riff_chunk(self): + b = b'SPAM' + struct.pack('<L', 0) + with self.assertRaisesRegex(wave.Error, + 'file does not start with RIFF id'): + wave.open(io.BytesIO(b)) + + def test_read_not_wave(self): + b = b'RIFF' + struct.pack('<L', 4) + b'SPAM' + with self.assertRaisesRegex(wave.Error, + 'not a WAVE file'): + wave.open(io.BytesIO(b)) + + def test_read_no_fmt_no_data_chunk(self): + b = b'RIFF' + struct.pack('<L', 4) + b'WAVE' + with self.assertRaisesRegex(wave.Error, + 'fmt chunk and/or data chunk missing'): + wave.open(io.BytesIO(b)) + + def test_read_no_data_chunk(self): + b = b'RIFF' + struct.pack('<L', 28) + b'WAVE' + b += b'fmt ' + struct.pack('<LHHLLHH', 16, 1, 1, 11025, 11025, 1, 8) + with self.assertRaisesRegex(wave.Error, + 'fmt chunk and/or data chunk missing'): + wave.open(io.BytesIO(b)) + + def test_read_no_fmt_chunk(self): + b = b'RIFF' + struct.pack('<L', 12) + b'WAVE' + b += b'data' + struct.pack('<L', 0) + with self.assertRaisesRegex(wave.Error, 'data chunk before fmt chunk'): + wave.open(io.BytesIO(b)) + + def test_read_wrong_form(self): + b = b'RIFF' + struct.pack('<L', 36) + b'WAVE' + b += b'fmt ' + struct.pack('<LHHLLHH', 16, 2, 1, 11025, 11025, 1, 1) + b += b'data' + struct.pack('<L', 0) + with self.assertRaisesRegex(wave.Error, 'unknown format: 2'): + wave.open(io.BytesIO(b)) + + def test_read_wrong_number_of_channels(self): + b = b'RIFF' + struct.pack('<L', 36) + b'WAVE' + b += b'fmt ' + struct.pack('<LHHLLHH', 16, 1, 0, 11025, 11025, 1, 8) + b += b'data' + struct.pack('<L', 0) + with self.assertRaisesRegex(wave.Error, 'bad # of channels'): + wave.open(io.BytesIO(b)) + + def test_read_wrong_sample_width(self): + b = b'RIFF' + struct.pack('<L', 36) + b'WAVE' + b += b'fmt ' + struct.pack('<LHHLLHH', 16, 1, 1, 11025, 11025, 1, 0) + b += b'data' + struct.pack('<L', 0) + with self.assertRaisesRegex(wave.Error, 'bad sample width'): + wave.open(io.BytesIO(b)) + + if __name__ == '__main__': unittest.main() diff --git a/Lib/wave.py b/Lib/wave.py index cf94d5af72b4..f155879a9a76 100644 --- a/Lib/wave.py +++ b/Lib/wave.py @@ -253,12 +253,22 @@ def readframes(self, nframes): # def _read_fmt_chunk(self, chunk): - wFormatTag, self._nchannels, self._framerate, dwAvgBytesPerSec, wBlockAlign = struct.unpack_from('<HHLLH', chunk.read(14)) + try: + wFormatTag, self._nchannels, self._framerate, dwAvgBytesPerSec, wBlockAlign = struct.unpack_from('<HHLLH', chunk.read(14)) + except struct.error: + raise EOFError from None if wFormatTag == WAVE_FORMAT_PCM: - sampwidth = struct.unpack_from('<H', chunk.read(2))[0] + try: + sampwidth = struct.unpack_from('<H', chunk.read(2))[0] + except struct.error: + raise EOFError from None self._sampwidth = (sampwidth + 7) // 8 + if not self._sampwidth: + raise Error('bad sample width') else: raise Error('unknown format: %r' % (wFormatTag,)) + if not self._nchannels: + raise Error('bad # of channels') self._framesize = self._nchannels * self._sampwidth self._comptype = 'NONE' self._compname = 'not compressed' diff --git a/Misc/NEWS.d/next/Library/2018-03-01-17-49-56.bpo-32056.IlpfgE.rst b/Misc/NEWS.d/next/Library/2018-03-01-17-49-56.bpo-32056.IlpfgE.rst new file mode 100644 index 000000000000..421aa3767794 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-03-01-17-49-56.bpo-32056.IlpfgE.rst @@ -0,0 +1,3 @@ +Improved exceptions raised for invalid number of channels and sample width +when read an audio file in modules :mod:`aifc`, :mod:`wave` and +:mod:`sunau`. From webhook-mailer at python.org Sun Mar 18 03:56:55 2018 From: webhook-mailer at python.org (Serhiy Storchaka) Date: Sun, 18 Mar 2018 07:56:55 -0000 Subject: [Python-checkins] bpo-32489: Allow 'continue' in 'finally' clause. (GH-5822) Message-ID: <mailman.89.1521359817.1871.python-checkins@python.org> https://github.com/python/cpython/commit/fe2bbb1869b42222a3f331a3dfb8b304a19a5819 commit: fe2bbb1869b42222a3f331a3dfb8b304a19a5819 branch: master author: Serhiy Storchaka <storchaka at gmail.com> committer: GitHub <noreply at github.com> date: 2018-03-18T09:56:52+02:00 summary: bpo-32489: Allow 'continue' in 'finally' clause. (GH-5822) files: A Misc/NEWS.d/next/Core and Builtins/2018-01-03-23-12-43.bpo-32489.SDEPHB.rst M Doc/library/dis.rst M Doc/reference/compound_stmts.rst M Doc/reference/simple_stmts.rst M Doc/whatsnew/3.8.rst M Lib/test/test_compile.py M Lib/test/test_exceptions.py M Lib/test/test_grammar.py M Lib/test/test_syntax.py M Python/compile.c diff --git a/Doc/library/dis.rst b/Doc/library/dis.rst index aa66128d60dd..47f226bd1625 100644 --- a/Doc/library/dis.rst +++ b/Doc/library/dis.rst @@ -698,8 +698,8 @@ iterations of the loop. removed from the block stack. It is similar to :opcode:`END_FINALLY`, but doesn't change the bytecode - counter nor raise an exception. Used for implementing :keyword:`break` - and :keyword:`return` in the :keyword:`finally` block. + counter nor raise an exception. Used for implementing :keyword:`break`, + :keyword:`continue` and :keyword:`return` in the :keyword:`finally` block. .. versionadded:: 3.8 diff --git a/Doc/reference/compound_stmts.rst b/Doc/reference/compound_stmts.rst index d7792f1eaf82..153e85b39491 100644 --- a/Doc/reference/compound_stmts.rst +++ b/Doc/reference/compound_stmts.rst @@ -321,8 +321,8 @@ not handled, the exception is temporarily saved. The :keyword:`finally` clause is executed. If there is a saved exception it is re-raised at the end of the :keyword:`finally` clause. If the :keyword:`finally` clause raises another exception, the saved exception is set as the context of the new exception. -If the :keyword:`finally` clause executes a :keyword:`return` or :keyword:`break` -statement, the saved exception is discarded:: +If the :keyword:`finally` clause executes a :keyword:`return`, :keyword:`break` +or :keyword:`continue` statement, the saved exception is discarded:: >>> def f(): ... try: @@ -343,10 +343,7 @@ the :keyword:`finally` clause. When a :keyword:`return`, :keyword:`break` or :keyword:`continue` statement is executed in the :keyword:`try` suite of a :keyword:`try`...\ :keyword:`finally` -statement, the :keyword:`finally` clause is also executed 'on the way out.' A -:keyword:`continue` statement is illegal in the :keyword:`finally` clause. (The -reason is a problem with the current implementation --- this restriction may be -lifted in the future). +statement, the :keyword:`finally` clause is also executed 'on the way out.' The return value of a function is determined by the last :keyword:`return` statement executed. Since the :keyword:`finally` clause always executes, a @@ -366,6 +363,10 @@ Additional information on exceptions can be found in section :ref:`exceptions`, and information on using the :keyword:`raise` statement to generate exceptions may be found in section :ref:`raise`. +.. versionchanged:: 3.8 + Prior to Python 3.8, a :keyword:`continue` statement was illegal in the + :keyword:`finally` clause due to a problem with the implementation. + .. _with: .. _as: diff --git a/Doc/reference/simple_stmts.rst b/Doc/reference/simple_stmts.rst index ef9a5f0dc854..9186210d951f 100644 --- a/Doc/reference/simple_stmts.rst +++ b/Doc/reference/simple_stmts.rst @@ -686,9 +686,8 @@ The :keyword:`continue` statement continue_stmt: "continue" :keyword:`continue` may only occur syntactically nested in a :keyword:`for` or -:keyword:`while` loop, but not nested in a function or class definition or -:keyword:`finally` clause within that loop. It continues with the next -cycle of the nearest enclosing loop. +:keyword:`while` loop, but not nested in a function or class definition within +that loop. It continues with the next cycle of the nearest enclosing loop. When :keyword:`continue` passes control out of a :keyword:`try` statement with a :keyword:`finally` clause, that :keyword:`finally` clause is executed before diff --git a/Doc/whatsnew/3.8.rst b/Doc/whatsnew/3.8.rst index 8569341055e1..fcc868b041ad 100644 --- a/Doc/whatsnew/3.8.rst +++ b/Doc/whatsnew/3.8.rst @@ -72,6 +72,11 @@ New Features Other Language Changes ====================== +* A :keyword:`continue` statement was illegal in the :keyword:`finally` clause + due to a problem with the implementation. In Python 3.8 this restriction + was lifted. + (Contributed by Serhiy Storchaka in :issue:`32489`.) + * Added support of ``\N{name}`` escapes in :mod:`regular expressions <re>`. (Contributed by Jonathan Eunice and Serhiy Storchaka in :issue:`30688`.) diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py index acebdbdc4636..6b45a2433438 100644 --- a/Lib/test/test_compile.py +++ b/Lib/test/test_compile.py @@ -856,7 +856,7 @@ def test_for_break_continue_inside_try_finally_block(self): """ self.check_stack_size(snippet) - def test_for_break_inside_finally_block(self): + def test_for_break_continue_inside_finally_block(self): snippet = """ for x in y: try: @@ -864,6 +864,8 @@ def test_for_break_inside_finally_block(self): finally: if z: break + elif u: + continue else: a else: diff --git a/Lib/test/test_exceptions.py b/Lib/test/test_exceptions.py index 9d10df5f9425..2a9ec706467f 100644 --- a/Lib/test/test_exceptions.py +++ b/Lib/test/test_exceptions.py @@ -138,15 +138,6 @@ def ckmsg(src, msg): else: self.fail("failed to get expected SyntaxError") - s = '''while 1: - try: - pass - finally: - continue''' - - if not sys.platform.startswith('java'): - ckmsg(s, "'continue' not supported inside 'finally' clause") - s = '''if 1: try: continue diff --git a/Lib/test/test_grammar.py b/Lib/test/test_grammar.py index d89bfdc06335..ee4136286ba1 100644 --- a/Lib/test/test_grammar.py +++ b/Lib/test/test_grammar.py @@ -859,6 +859,59 @@ def test_break_in_finally(self): break self.assertEqual(count, 0) + def test_continue_in_finally(self): + count = 0 + while count < 2: + count += 1 + try: + pass + finally: + continue + break + self.assertEqual(count, 2) + + count = 0 + while count < 2: + count += 1 + try: + break + finally: + continue + self.assertEqual(count, 2) + + count = 0 + while count < 2: + count += 1 + try: + 1/0 + finally: + continue + break + self.assertEqual(count, 2) + + for count in [0, 1]: + try: + pass + finally: + continue + break + self.assertEqual(count, 1) + + for count in [0, 1]: + try: + break + finally: + continue + self.assertEqual(count, 1) + + for count in [0, 1]: + try: + 1/0 + finally: + continue + break + self.assertEqual(count, 1) + def test_return_in_finally(self): def g1(): try: diff --git a/Lib/test/test_syntax.py b/Lib/test/test_syntax.py index 2b96a94401a8..fa1e7aa5d4f2 100644 --- a/Lib/test/test_syntax.py +++ b/Lib/test/test_syntax.py @@ -298,7 +298,7 @@ >>> test() 9 -Start simple, a continue in a finally should not be allowed. +continue in a finally should be ok. >>> def test(): ... for abc in range(10): @@ -306,11 +306,9 @@ ... pass ... finally: ... continue - Traceback (most recent call last): - ... - SyntaxError: 'continue' not supported inside 'finally' clause - -This is essentially a continue in a finally which should not be allowed. + ... print(abc) + >>> test() + 9 >>> def test(): ... for abc in range(10): @@ -321,9 +319,24 @@ ... continue ... except: ... pass - Traceback (most recent call last): - ... - SyntaxError: 'continue' not supported inside 'finally' clause + ... print(abc) + >>> test() + 9 + + >>> def test(): + ... for abc in range(10): + ... try: + ... pass + ... finally: + ... try: + ... pass + ... except: + ... continue + ... print(abc) + >>> test() + 9 + +A continue outside loop should not be allowed. >>> def foo(): ... try: @@ -332,42 +345,7 @@ ... continue Traceback (most recent call last): ... - SyntaxError: 'continue' not supported inside 'finally' clause - - >>> def foo(): - ... for a in (): - ... try: - ... pass - ... finally: - ... continue - Traceback (most recent call last): - ... - SyntaxError: 'continue' not supported inside 'finally' clause - - >>> def foo(): - ... for a in (): - ... try: - ... pass - ... finally: - ... try: - ... continue - ... finally: - ... pass - Traceback (most recent call last): - ... - SyntaxError: 'continue' not supported inside 'finally' clause - - >>> def foo(): - ... for a in (): - ... try: pass - ... finally: - ... try: - ... pass - ... except: - ... continue - Traceback (most recent call last): - ... - SyntaxError: 'continue' not supported inside 'finally' clause + SyntaxError: 'continue' not properly in loop There is one test for a break that is not in a loop. The compiler uses a single data structure to keep track of try-finally and loops, diff --git a/Misc/NEWS.d/next/Core and Builtins/2018-01-03-23-12-43.bpo-32489.SDEPHB.rst b/Misc/NEWS.d/next/Core and Builtins/2018-01-03-23-12-43.bpo-32489.SDEPHB.rst new file mode 100644 index 000000000000..68babebdad3c --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2018-01-03-23-12-43.bpo-32489.SDEPHB.rst @@ -0,0 +1,2 @@ +A :keyword:`continue` statement is now allowed in the :keyword:`finally` +clause. diff --git a/Python/compile.c b/Python/compile.c index fbd1fc960ab8..c3ffaae8ce20 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -2625,10 +2625,6 @@ compiler_continue(struct compiler *c) ADDOP_JABS(c, JUMP_ABSOLUTE, info->fb_block); return 1; } - if (info->fb_type == FINALLY_END) { - return compiler_error(c, - "'continue' not supported inside 'finally' clause"); - } if (!compiler_unwind_fblock(c, info, 0)) return 0; } From solipsis at pitrou.net Sun Mar 18 05:10:31 2018 From: solipsis at pitrou.net (solipsis at pitrou.net) Date: Sun, 18 Mar 2018 09:10:31 +0000 Subject: [Python-checkins] Daily reference leaks (4243df51fe43): sum=71 Message-ID: <20180318091031.1.CA16D930CD8C285F@psf.io> results for 4243df51fe43 on branch "default" -------------------------------------------- test_collections leaked [0, -7, 8] memory blocks, sum=1 test_functools leaked [0, 3, 1] memory blocks, sum=4 test_multiprocessing_forkserver leaked [-2, 2, 0] memory blocks, sum=0 test_multiprocessing_spawn leaked [0, 44, 0] references, sum=44 test_multiprocessing_spawn leaked [0, 21, -1] memory blocks, sum=20 test_multiprocessing_spawn leaked [0, 2, 0] file descriptors, sum=2 Command line was: ['./python', '-m', 'test.regrtest', '-uall', '-R', '3:3:/home/psf-users/antoine/refleaks/reflogmXjf9O', '--timeout', '7200'] From webhook-mailer at python.org Sun Mar 18 06:31:42 2018 From: webhook-mailer at python.org (Serhiy Storchaka) Date: Sun, 18 Mar 2018 10:31:42 -0000 Subject: [Python-checkins] [3.7] bpo-33041: Add tests for jumps in/out of 'async with' blocks. (GH-6110). (GH-6140) Message-ID: <mailman.90.1521369103.1871.python-checkins@python.org> https://github.com/python/cpython/commit/773573e9ac654d4b5c6682e70360f75864acd80e commit: 773573e9ac654d4b5c6682e70360f75864acd80e branch: 3.7 author: Serhiy Storchaka <storchaka at gmail.com> committer: GitHub <noreply at github.com> date: 2018-03-18T12:31:37+02:00 summary: [3.7] bpo-33041: Add tests for jumps in/out of 'async with' blocks. (GH-6110). (GH-6140) (cherry picked from commit bc300ce205f99acb1ef92c37de06dc76147e073b) Co-authored-by: Serhiy Storchaka <storchaka at gmail.com> files: M Lib/test/test_sys_settrace.py diff --git a/Lib/test/test_sys_settrace.py b/Lib/test/test_sys_settrace.py index 44ea7d575b45..bf8c722d0cf7 100644 --- a/Lib/test/test_sys_settrace.py +++ b/Lib/test/test_sys_settrace.py @@ -6,6 +6,8 @@ import difflib import gc from functools import wraps +import asyncio + class tracecontext: """Context manager that traces its enter and exit.""" @@ -19,6 +21,20 @@ def __enter__(self): def __exit__(self, *exc_info): self.output.append(-self.value) +class asynctracecontext: + """Asynchronous context manager that traces its aenter and aexit.""" + def __init__(self, output, value): + self.output = output + self.value = value + + async def __aenter__(self): + self.output.append(self.value) + + async def __aexit__(self, *exc_info): + self.output.append(-self.value) + + + # A very basic example. If this fails, we're in deep trouble. def basic(): return 1 @@ -637,6 +653,19 @@ def run_test(self, func, jumpFrom, jumpTo, expected, error=None, sys.settrace(None) self.compare_jump_output(expected, output) + def run_async_test(self, func, jumpFrom, jumpTo, expected, error=None, + event='line', decorated=False): + tracer = JumpTracer(func, jumpFrom, jumpTo, event, decorated) + sys.settrace(tracer.trace) + output = [] + if error is None: + asyncio.run(func(output)) + else: + with self.assertRaisesRegex(*error): + asyncio.run(func(output)) + sys.settrace(None) + self.compare_jump_output(expected, output) + def jump_test(jumpFrom, jumpTo, expected, error=None, event='line'): """Decorator that creates a test that makes a jump from one place to another in the following code. @@ -649,6 +678,18 @@ def test(self): return test return decorator + def async_jump_test(jumpFrom, jumpTo, expected, error=None, event='line'): + """Decorator that creates a test that makes a jump + from one place to another in the following asynchronous code. + """ + def decorator(func): + @wraps(func) + def test(self): + self.run_async_test(func, jumpFrom, jumpTo, expected, + error=error, event=event, decorated=True) + return test + return decorator + ## The first set of 'jump' tests are for things that are allowed: @jump_test(1, 3, [3]) @@ -744,12 +785,24 @@ def test_jump_forwards_out_of_with_block(output): output.append(2) output.append(3) + @async_jump_test(2, 3, [1, 3]) + async def test_jump_forwards_out_of_async_with_block(output): + async with asynctracecontext(output, 1): + output.append(2) + output.append(3) + @jump_test(3, 1, [1, 2, 1, 2, 3, -2]) def test_jump_backwards_out_of_with_block(output): output.append(1) with tracecontext(output, 2): output.append(3) + @async_jump_test(3, 1, [1, 2, 1, 2, 3, -2]) + async def test_jump_backwards_out_of_async_with_block(output): + output.append(1) + async with asynctracecontext(output, 2): + output.append(3) + @jump_test(2, 5, [5]) def test_jump_forwards_out_of_try_finally_block(output): try: @@ -813,6 +866,14 @@ def test_jump_across_with(output): with tracecontext(output, 4): output.append(5) + @async_jump_test(2, 4, [1, 4, 5, -4]) + async def test_jump_across_async_with(output): + output.append(1) + async with asynctracecontext(output, 2): + output.append(3) + async with asynctracecontext(output, 4): + output.append(5) + @jump_test(4, 5, [1, 3, 5, 6]) def test_jump_out_of_with_block_within_for_block(output): output.append(1) @@ -822,6 +883,15 @@ def test_jump_out_of_with_block_within_for_block(output): output.append(5) output.append(6) + @async_jump_test(4, 5, [1, 3, 5, 6]) + async def test_jump_out_of_async_with_block_within_for_block(output): + output.append(1) + for i in [1]: + async with asynctracecontext(output, 3): + output.append(4) + output.append(5) + output.append(6) + @jump_test(4, 5, [1, 2, 3, 5, -2, 6]) def test_jump_out_of_with_block_within_with_block(output): output.append(1) @@ -831,6 +901,15 @@ def test_jump_out_of_with_block_within_with_block(output): output.append(5) output.append(6) + @async_jump_test(4, 5, [1, 2, 3, 5, -2, 6]) + async def test_jump_out_of_async_with_block_within_with_block(output): + output.append(1) + with tracecontext(output, 2): + async with asynctracecontext(output, 3): + output.append(4) + output.append(5) + output.append(6) + @jump_test(5, 6, [2, 4, 6, 7]) def test_jump_out_of_with_block_within_finally_block(output): try: @@ -841,6 +920,16 @@ def test_jump_out_of_with_block_within_finally_block(output): output.append(6) output.append(7) + @async_jump_test(5, 6, [2, 4, 6, 7]) + async def test_jump_out_of_async_with_block_within_finally_block(output): + try: + output.append(2) + finally: + async with asynctracecontext(output, 4): + output.append(5) + output.append(6) + output.append(7) + @jump_test(8, 11, [1, 3, 5, 11, 12]) def test_jump_out_of_complex_nested_blocks(output): output.append(1) @@ -864,6 +953,14 @@ def test_jump_out_of_with_assignment(output): output.append(4) output.append(5) + @async_jump_test(3, 5, [1, 2, 5]) + async def test_jump_out_of_async_with_assignment(output): + output.append(1) + async with asynctracecontext(output, 2) \ + as x: + output.append(4) + output.append(5) + @jump_test(3, 6, [1, 6, 8, 9]) def test_jump_over_return_in_try_finally_block(output): output.append(1) @@ -982,12 +1079,24 @@ def test_no_jump_forwards_into_with_block(output): with tracecontext(output, 2): output.append(3) + @async_jump_test(1, 3, [], (ValueError, 'into')) + async def test_no_jump_forwards_into_async_with_block(output): + output.append(1) + async with asynctracecontext(output, 2): + output.append(3) + @jump_test(3, 2, [1, 2, -1], (ValueError, 'into')) def test_no_jump_backwards_into_with_block(output): with tracecontext(output, 1): output.append(2) output.append(3) + @async_jump_test(3, 2, [1, 2, -1], (ValueError, 'into')) + async def test_no_jump_backwards_into_async_with_block(output): + async with asynctracecontext(output, 1): + output.append(2) + output.append(3) + @jump_test(1, 3, [], (ValueError, 'into')) def test_no_jump_forwards_into_try_finally_block(output): output.append(1) @@ -1068,6 +1177,14 @@ def test_no_jump_between_with_blocks(output): with tracecontext(output, 4): output.append(5) + @async_jump_test(3, 5, [1, 2, -2], (ValueError, 'into')) + async def test_no_jump_between_async_with_blocks(output): + output.append(1) + async with asynctracecontext(output, 2): + output.append(3) + async with asynctracecontext(output, 4): + output.append(5) + @jump_test(7, 4, [1, 6], (ValueError, 'into')) def test_no_jump_into_for_block_before_else(output): output.append(1) From webhook-mailer at python.org Sun Mar 18 06:32:35 2018 From: webhook-mailer at python.org (Serhiy Storchaka) Date: Sun, 18 Mar 2018 10:32:35 -0000 Subject: [Python-checkins] [3.6] bpo-33041: Add tests for jumps in/out of 'async with' blocks. (GH-6110). (GH-6141) Message-ID: <mailman.91.1521369157.1871.python-checkins@python.org> https://github.com/python/cpython/commit/193760f00aacb122698674ed15327dba412653a5 commit: 193760f00aacb122698674ed15327dba412653a5 branch: 3.6 author: Serhiy Storchaka <storchaka at gmail.com> committer: GitHub <noreply at github.com> date: 2018-03-18T12:32:32+02:00 summary: [3.6] bpo-33041: Add tests for jumps in/out of 'async with' blocks. (GH-6110). (GH-6141) (cherry picked from commit bc300ce205f99acb1ef92c37de06dc76147e073b) files: M Lib/test/test_sys_settrace.py diff --git a/Lib/test/test_sys_settrace.py b/Lib/test/test_sys_settrace.py index 46593cf54ad4..659790efe7cc 100644 --- a/Lib/test/test_sys_settrace.py +++ b/Lib/test/test_sys_settrace.py @@ -7,8 +7,9 @@ import gc from functools import wraps + class tracecontext: - """Contex manager that traces its enter and exit.""" + """Context manager that traces its enter and exit.""" def __init__(self, output, value): self.output = output self.value = value @@ -19,6 +20,36 @@ def __enter__(self): def __exit__(self, *exc_info): self.output.append(-self.value) +class asynctracecontext: + """Asynchronous context manager that traces its aenter and aexit.""" + def __init__(self, output, value): + self.output = output + self.value = value + + async def __aenter__(self): + self.output.append(self.value) + + async def __aexit__(self, *exc_info): + self.output.append(-self.value) + +def asyncio_run(main): + import asyncio + import asyncio.events + import asyncio.coroutines + assert asyncio.events._get_running_loop() is None + assert asyncio.coroutines.iscoroutine(main) + loop = asyncio.events.new_event_loop() + try: + asyncio.events.set_event_loop(loop) + return loop.run_until_complete(main) + finally: + try: + loop.run_until_complete(loop.shutdown_asyncgens()) + finally: + asyncio.events.set_event_loop(None) + loop.close() + + # A very basic example. If this fails, we're in deep trouble. def basic(): return 1 @@ -591,6 +622,19 @@ def run_test(self, func, jumpFrom, jumpTo, expected, error=None, sys.settrace(None) self.compare_jump_output(expected, output) + def run_async_test(self, func, jumpFrom, jumpTo, expected, error=None, + event='line', decorated=False): + tracer = JumpTracer(func, jumpFrom, jumpTo, event, decorated) + sys.settrace(tracer.trace) + output = [] + if error is None: + asyncio_run(func(output)) + else: + with self.assertRaisesRegex(*error): + asyncio_run(func(output)) + sys.settrace(None) + self.compare_jump_output(expected, output) + def jump_test(jumpFrom, jumpTo, expected, error=None, event='line'): """Decorator that creates a test that makes a jump from one place to another in the following code. @@ -603,6 +647,18 @@ def test(self): return test return decorator + def async_jump_test(jumpFrom, jumpTo, expected, error=None, event='line'): + """Decorator that creates a test that makes a jump + from one place to another in the following asynchronous code. + """ + def decorator(func): + @wraps(func) + def test(self): + self.run_async_test(func, jumpFrom, jumpTo, expected, + error=error, event=event, decorated=True) + return test + return decorator + ## The first set of 'jump' tests are for things that are allowed: @jump_test(1, 3, [3]) @@ -698,12 +754,24 @@ def test_jump_forwards_out_of_with_block(output): output.append(2) output.append(3) + @async_jump_test(2, 3, [1, 3]) + async def test_jump_forwards_out_of_async_with_block(output): + async with asynctracecontext(output, 1): + output.append(2) + output.append(3) + @jump_test(3, 1, [1, 2, 1, 2, 3, -2]) def test_jump_backwards_out_of_with_block(output): output.append(1) with tracecontext(output, 2): output.append(3) + @async_jump_test(3, 1, [1, 2, 1, 2, 3, -2]) + async def test_jump_backwards_out_of_async_with_block(output): + output.append(1) + async with asynctracecontext(output, 2): + output.append(3) + @jump_test(2, 5, [5]) def test_jump_forwards_out_of_try_finally_block(output): try: @@ -767,6 +835,14 @@ def test_jump_across_with(output): with tracecontext(output, 4): output.append(5) + @async_jump_test(2, 4, [1, 4, 5, -4]) + async def test_jump_across_async_with(output): + output.append(1) + async with asynctracecontext(output, 2): + output.append(3) + async with asynctracecontext(output, 4): + output.append(5) + @jump_test(4, 5, [1, 3, 5, 6]) def test_jump_out_of_with_block_within_for_block(output): output.append(1) @@ -776,6 +852,15 @@ def test_jump_out_of_with_block_within_for_block(output): output.append(5) output.append(6) + @async_jump_test(4, 5, [1, 3, 5, 6]) + async def test_jump_out_of_async_with_block_within_for_block(output): + output.append(1) + for i in [1]: + async with asynctracecontext(output, 3): + output.append(4) + output.append(5) + output.append(6) + @jump_test(4, 5, [1, 2, 3, 5, -2, 6]) def test_jump_out_of_with_block_within_with_block(output): output.append(1) @@ -785,6 +870,15 @@ def test_jump_out_of_with_block_within_with_block(output): output.append(5) output.append(6) + @async_jump_test(4, 5, [1, 2, 3, 5, -2, 6]) + async def test_jump_out_of_async_with_block_within_with_block(output): + output.append(1) + with tracecontext(output, 2): + async with asynctracecontext(output, 3): + output.append(4) + output.append(5) + output.append(6) + @jump_test(5, 6, [2, 4, 6, 7]) def test_jump_out_of_with_block_within_finally_block(output): try: @@ -795,6 +889,16 @@ def test_jump_out_of_with_block_within_finally_block(output): output.append(6) output.append(7) + @async_jump_test(5, 6, [2, 4, 6, 7]) + async def test_jump_out_of_async_with_block_within_finally_block(output): + try: + output.append(2) + finally: + async with asynctracecontext(output, 4): + output.append(5) + output.append(6) + output.append(7) + @jump_test(8, 11, [1, 3, 5, 11, 12]) def test_jump_out_of_complex_nested_blocks(output): output.append(1) @@ -818,6 +922,14 @@ def test_jump_out_of_with_assignment(output): output.append(4) output.append(5) + @async_jump_test(3, 5, [1, 2, 5]) + async def test_jump_out_of_async_with_assignment(output): + output.append(1) + async with asynctracecontext(output, 2) \ + as x: + output.append(4) + output.append(5) + @jump_test(3, 6, [1, 6, 8, 9]) def test_jump_over_return_in_try_finally_block(output): output.append(1) @@ -936,12 +1048,24 @@ def test_no_jump_forwards_into_with_block(output): with tracecontext(output, 2): output.append(3) + @async_jump_test(1, 3, [], (ValueError, 'into')) + async def test_no_jump_forwards_into_async_with_block(output): + output.append(1) + async with asynctracecontext(output, 2): + output.append(3) + @jump_test(3, 2, [1, 2, -1], (ValueError, 'into')) def test_no_jump_backwards_into_with_block(output): with tracecontext(output, 1): output.append(2) output.append(3) + @async_jump_test(3, 2, [1, 2, -1], (ValueError, 'into')) + async def test_no_jump_backwards_into_async_with_block(output): + async with asynctracecontext(output, 1): + output.append(2) + output.append(3) + @jump_test(1, 3, [], (ValueError, 'into')) def test_no_jump_forwards_into_try_finally_block(output): output.append(1) @@ -1022,6 +1146,14 @@ def test_no_jump_between_with_blocks(output): with tracecontext(output, 4): output.append(5) + @async_jump_test(3, 5, [1, 2, -2], (ValueError, 'into')) + async def test_no_jump_between_async_with_blocks(output): + output.append(1) + async with asynctracecontext(output, 2): + output.append(3) + async with asynctracecontext(output, 4): + output.append(5) + @jump_test(7, 4, [1, 6], (ValueError, 'into')) def test_no_jump_into_for_block_before_else(output): output.append(1) From webhook-mailer at python.org Sun Mar 18 12:54:37 2018 From: webhook-mailer at python.org (=?utf-8?q?=C5=81ukasz?= Langa) Date: Sun, 18 Mar 2018 16:54:37 -0000 Subject: [Python-checkins] Revert "bpo-30406: Make async and await proper keywords (#1669)" (GH-6143) Message-ID: <mailman.93.1521392078.1871.python-checkins@python.org> https://github.com/python/cpython/commit/f64aae46da292f71f6be750026cd052362e066bc commit: f64aae46da292f71f6be750026cd052362e066bc branch: master author: Jelle Zijlstra <jelle.zijlstra at gmail.com> committer: ?ukasz Langa <lukasz at langa.pl> date: 2018-03-18T09:54:33-07:00 summary: Revert "bpo-30406: Make async and await proper keywords (#1669)" (GH-6143) This reverts commit ac317700ce7439e38a8b420218d9a5035bba92ed. (Reverts only the lib2to3 part.) files: M Lib/lib2to3/Grammar.txt M Lib/lib2to3/pgen2/token.py M Lib/lib2to3/pgen2/tokenize.py M Lib/lib2to3/tests/test_parser.py diff --git a/Lib/lib2to3/Grammar.txt b/Lib/lib2to3/Grammar.txt index b19b4a21fadd..4905c91e635e 100644 --- a/Lib/lib2to3/Grammar.txt +++ b/Lib/lib2to3/Grammar.txt @@ -15,7 +15,7 @@ eval_input: testlist NEWLINE* ENDMARKER decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE decorators: decorator+ decorated: decorators (classdef | funcdef | async_funcdef) -async_funcdef: 'async' funcdef +async_funcdef: ASYNC funcdef funcdef: 'def' NAME parameters ['->' test] ':' suite parameters: '(' [typedargslist] ')' typedargslist: ((tfpdef ['=' test] ',')* @@ -66,7 +66,7 @@ exec_stmt: 'exec' expr ['in' test [',' test]] assert_stmt: 'assert' test [',' test] compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | with_stmt | funcdef | classdef | decorated | async_stmt -async_stmt: 'async' (funcdef | with_stmt | for_stmt) +async_stmt: ASYNC (funcdef | with_stmt | for_stmt) if_stmt: 'if' test ':' suite ('elif' test ':' suite)* ['else' ':' suite] while_stmt: 'while' test ':' suite ['else' ':' suite] for_stmt: 'for' exprlist 'in' testlist ':' suite ['else' ':' suite] @@ -105,7 +105,7 @@ shift_expr: arith_expr (('<<'|'>>') arith_expr)* arith_expr: term (('+'|'-') term)* term: factor (('*'|'@'|'/'|'%'|'//') factor)* factor: ('+'|'-'|'~') factor | power -power: ['await'] atom trailer* ['**' factor] +power: [AWAIT] atom trailer* ['**' factor] atom: ('(' [yield_expr|testlist_gexp] ')' | '[' [listmaker] ']' | '{' [dictsetmaker] '}' | @@ -142,7 +142,7 @@ argument: ( test [comp_for] | star_expr ) comp_iter: comp_for | comp_if -comp_for: ['async'] 'for' exprlist 'in' or_test [comp_iter] +comp_for: [ASYNC] 'for' exprlist 'in' or_test [comp_iter] comp_if: 'if' old_test [comp_iter] # As noted above, testlist_safe extends the syntax allowed in list @@ -161,7 +161,7 @@ comp_if: 'if' old_test [comp_iter] # # See https://bugs.python.org/issue27494 old_comp_iter: old_comp_for | old_comp_if -old_comp_for: ['async'] 'for' exprlist 'in' testlist_safe [old_comp_iter] +old_comp_for: [ASYNC] 'for' exprlist 'in' testlist_safe [old_comp_iter] old_comp_if: 'if' old_test [old_comp_iter] testlist1: test (',' test)* diff --git a/Lib/lib2to3/pgen2/token.py b/Lib/lib2to3/pgen2/token.py index 7599396611b2..1a679554d2db 100755 --- a/Lib/lib2to3/pgen2/token.py +++ b/Lib/lib2to3/pgen2/token.py @@ -62,8 +62,10 @@ COMMENT = 53 NL = 54 RARROW = 55 -ERRORTOKEN = 56 -N_TOKENS = 57 +AWAIT = 56 +ASYNC = 57 +ERRORTOKEN = 58 +N_TOKENS = 59 NT_OFFSET = 256 #--end constants-- diff --git a/Lib/lib2to3/pgen2/tokenize.py b/Lib/lib2to3/pgen2/tokenize.py index 14560e4fddff..45afc5f4e53f 100644 --- a/Lib/lib2to3/pgen2/tokenize.py +++ b/Lib/lib2to3/pgen2/tokenize.py @@ -234,7 +234,7 @@ def compat(self, token, iterable): for tok in iterable: toknum, tokval = tok[:2] - if toknum in (NAME, NUMBER): + if toknum in (NAME, NUMBER, ASYNC, AWAIT): tokval += ' ' if toknum == INDENT: @@ -380,6 +380,12 @@ def generate_tokens(readline): contline = None indents = [0] + # 'stashed' and 'async_*' are used for async/await parsing + stashed = None + async_def = False + async_def_indent = 0 + async_def_nl = False + while 1: # loop over lines in stream try: line = readline() @@ -420,6 +426,10 @@ def generate_tokens(readline): pos = pos + 1 if pos == max: break + if stashed: + yield stashed + stashed = None + if line[pos] in '#\r\n': # skip comments or blank lines if line[pos] == '#': comment_token = line[pos:].rstrip('\r\n') @@ -443,8 +453,18 @@ def generate_tokens(readline): ("<tokenize>", lnum, pos, line)) indents = indents[:-1] + if async_def and async_def_indent >= indents[-1]: + async_def = False + async_def_nl = False + async_def_indent = 0 + yield (DEDENT, '', (lnum, pos), (lnum, pos), line) + if async_def and async_def_nl and async_def_indent >= indents[-1]: + async_def = False + async_def_nl = False + async_def_indent = 0 + else: # continued statement if not line: raise TokenError("EOF in multi-line statement", (lnum, 0)) @@ -464,10 +484,18 @@ def generate_tokens(readline): newline = NEWLINE if parenlev > 0: newline = NL + elif async_def: + async_def_nl = True + if stashed: + yield stashed + stashed = None yield (newline, token, spos, epos, line) elif initial == '#': assert not token.endswith("\n") + if stashed: + yield stashed + stashed = None yield (COMMENT, token, spos, epos, line) elif token in triple_quoted: endprog = endprogs[token] @@ -475,6 +503,9 @@ def generate_tokens(readline): if endmatch: # all on one line pos = endmatch.end(0) token = line[start:pos] + if stashed: + yield stashed + stashed = None yield (STRING, token, spos, (lnum, pos), line) else: strstart = (lnum, start) # multiple lines @@ -492,22 +523,63 @@ def generate_tokens(readline): contline = line break else: # ordinary string + if stashed: + yield stashed + stashed = None yield (STRING, token, spos, epos, line) elif initial in namechars: # ordinary name - yield (NAME, token, spos, epos, line) + if token in ('async', 'await'): + if async_def: + yield (ASYNC if token == 'async' else AWAIT, + token, spos, epos, line) + continue + + tok = (NAME, token, spos, epos, line) + if token == 'async' and not stashed: + stashed = tok + continue + + if token == 'def': + if (stashed + and stashed[0] == NAME + and stashed[1] == 'async'): + + async_def = True + async_def_indent = indents[-1] + + yield (ASYNC, stashed[1], + stashed[2], stashed[3], + stashed[4]) + stashed = None + + if stashed: + yield stashed + stashed = None + + yield tok elif initial == '\\': # continued stmt # This yield is new; needed for better idempotency: + if stashed: + yield stashed + stashed = None yield (NL, token, spos, (lnum, pos), line) continued = 1 else: if initial in '([{': parenlev = parenlev + 1 elif initial in ')]}': parenlev = parenlev - 1 + if stashed: + yield stashed + stashed = None yield (OP, token, spos, epos, line) else: yield (ERRORTOKEN, line[pos], (lnum, pos), (lnum, pos+1), line) pos = pos + 1 + if stashed: + yield stashed + stashed = None + for indent in indents[1:]: # pop remaining indent levels yield (DEDENT, '', (lnum, 0), (lnum, 0), '') yield (ENDMARKER, '', (lnum, 0), (lnum, 0), '') diff --git a/Lib/lib2to3/tests/test_parser.py b/Lib/lib2to3/tests/test_parser.py index 6813c65a7953..74653ea80fef 100644 --- a/Lib/lib2to3/tests/test_parser.py +++ b/Lib/lib2to3/tests/test_parser.py @@ -181,34 +181,34 @@ def foo(): pass async def foo(): await x """) - self.validate("await x") - self.validate("""def foo(): - await x""") + self.invalid_syntax("await x") + self.invalid_syntax("""def foo(): + await x""") - self.validate("""def foo(): + self.invalid_syntax("""def foo(): def foo(): pass async def foo(): pass await x """) def test_async_var(self): - self.invalid_syntax("""async = 1""") - self.invalid_syntax("""await = 1""") - self.invalid_syntax("""def async(): pass""") + self.validate("""async = 1""") + self.validate("""await = 1""") + self.validate("""def async(): pass""") def test_async_with(self): self.validate("""async def foo(): async for a in b: pass""") - self.validate("""def foo(): - async for a in b: pass""") + self.invalid_syntax("""def foo(): + async for a in b: pass""") def test_async_for(self): self.validate("""async def foo(): async with a: pass""") - self.validate("""def foo(): - async with a: pass""") + self.invalid_syntax("""def foo(): + async with a: pass""") class TestRaiseChanges(GrammarTest): From webhook-mailer at python.org Sun Mar 18 15:15:55 2018 From: webhook-mailer at python.org (Miss Islington (bot)) Date: Sun, 18 Mar 2018 19:15:55 -0000 Subject: [Python-checkins] Revert "bpo-30406: Make async and await proper keywords (GH-1669)" (GH-6143) Message-ID: <mailman.94.1521400558.1871.python-checkins@python.org> https://github.com/python/cpython/commit/a90df5085b51e8bb9de1c04c408bbef42ce6cbc3 commit: a90df5085b51e8bb9de1c04c408bbef42ce6cbc3 branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: GitHub <noreply at github.com> date: 2018-03-18T12:15:52-07:00 summary: Revert "bpo-30406: Make async and await proper keywords (GH-1669)" (GH-6143) This reverts commit ac317700ce7439e38a8b420218d9a5035bba92ed. (Reverts only the lib2to3 part.) (cherry picked from commit f64aae46da292f71f6be750026cd052362e066bc) Co-authored-by: Jelle Zijlstra <jelle.zijlstra at gmail.com> files: M Lib/lib2to3/Grammar.txt M Lib/lib2to3/pgen2/token.py M Lib/lib2to3/pgen2/tokenize.py M Lib/lib2to3/tests/test_parser.py diff --git a/Lib/lib2to3/Grammar.txt b/Lib/lib2to3/Grammar.txt index b19b4a21fadd..4905c91e635e 100644 --- a/Lib/lib2to3/Grammar.txt +++ b/Lib/lib2to3/Grammar.txt @@ -15,7 +15,7 @@ eval_input: testlist NEWLINE* ENDMARKER decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE decorators: decorator+ decorated: decorators (classdef | funcdef | async_funcdef) -async_funcdef: 'async' funcdef +async_funcdef: ASYNC funcdef funcdef: 'def' NAME parameters ['->' test] ':' suite parameters: '(' [typedargslist] ')' typedargslist: ((tfpdef ['=' test] ',')* @@ -66,7 +66,7 @@ exec_stmt: 'exec' expr ['in' test [',' test]] assert_stmt: 'assert' test [',' test] compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | with_stmt | funcdef | classdef | decorated | async_stmt -async_stmt: 'async' (funcdef | with_stmt | for_stmt) +async_stmt: ASYNC (funcdef | with_stmt | for_stmt) if_stmt: 'if' test ':' suite ('elif' test ':' suite)* ['else' ':' suite] while_stmt: 'while' test ':' suite ['else' ':' suite] for_stmt: 'for' exprlist 'in' testlist ':' suite ['else' ':' suite] @@ -105,7 +105,7 @@ shift_expr: arith_expr (('<<'|'>>') arith_expr)* arith_expr: term (('+'|'-') term)* term: factor (('*'|'@'|'/'|'%'|'//') factor)* factor: ('+'|'-'|'~') factor | power -power: ['await'] atom trailer* ['**' factor] +power: [AWAIT] atom trailer* ['**' factor] atom: ('(' [yield_expr|testlist_gexp] ')' | '[' [listmaker] ']' | '{' [dictsetmaker] '}' | @@ -142,7 +142,7 @@ argument: ( test [comp_for] | star_expr ) comp_iter: comp_for | comp_if -comp_for: ['async'] 'for' exprlist 'in' or_test [comp_iter] +comp_for: [ASYNC] 'for' exprlist 'in' or_test [comp_iter] comp_if: 'if' old_test [comp_iter] # As noted above, testlist_safe extends the syntax allowed in list @@ -161,7 +161,7 @@ comp_if: 'if' old_test [comp_iter] # # See https://bugs.python.org/issue27494 old_comp_iter: old_comp_for | old_comp_if -old_comp_for: ['async'] 'for' exprlist 'in' testlist_safe [old_comp_iter] +old_comp_for: [ASYNC] 'for' exprlist 'in' testlist_safe [old_comp_iter] old_comp_if: 'if' old_test [old_comp_iter] testlist1: test (',' test)* diff --git a/Lib/lib2to3/pgen2/token.py b/Lib/lib2to3/pgen2/token.py index 7599396611b2..1a679554d2db 100755 --- a/Lib/lib2to3/pgen2/token.py +++ b/Lib/lib2to3/pgen2/token.py @@ -62,8 +62,10 @@ COMMENT = 53 NL = 54 RARROW = 55 -ERRORTOKEN = 56 -N_TOKENS = 57 +AWAIT = 56 +ASYNC = 57 +ERRORTOKEN = 58 +N_TOKENS = 59 NT_OFFSET = 256 #--end constants-- diff --git a/Lib/lib2to3/pgen2/tokenize.py b/Lib/lib2to3/pgen2/tokenize.py index 14560e4fddff..45afc5f4e53f 100644 --- a/Lib/lib2to3/pgen2/tokenize.py +++ b/Lib/lib2to3/pgen2/tokenize.py @@ -234,7 +234,7 @@ def compat(self, token, iterable): for tok in iterable: toknum, tokval = tok[:2] - if toknum in (NAME, NUMBER): + if toknum in (NAME, NUMBER, ASYNC, AWAIT): tokval += ' ' if toknum == INDENT: @@ -380,6 +380,12 @@ def generate_tokens(readline): contline = None indents = [0] + # 'stashed' and 'async_*' are used for async/await parsing + stashed = None + async_def = False + async_def_indent = 0 + async_def_nl = False + while 1: # loop over lines in stream try: line = readline() @@ -420,6 +426,10 @@ def generate_tokens(readline): pos = pos + 1 if pos == max: break + if stashed: + yield stashed + stashed = None + if line[pos] in '#\r\n': # skip comments or blank lines if line[pos] == '#': comment_token = line[pos:].rstrip('\r\n') @@ -443,8 +453,18 @@ def generate_tokens(readline): ("<tokenize>", lnum, pos, line)) indents = indents[:-1] + if async_def and async_def_indent >= indents[-1]: + async_def = False + async_def_nl = False + async_def_indent = 0 + yield (DEDENT, '', (lnum, pos), (lnum, pos), line) + if async_def and async_def_nl and async_def_indent >= indents[-1]: + async_def = False + async_def_nl = False + async_def_indent = 0 + else: # continued statement if not line: raise TokenError("EOF in multi-line statement", (lnum, 0)) @@ -464,10 +484,18 @@ def generate_tokens(readline): newline = NEWLINE if parenlev > 0: newline = NL + elif async_def: + async_def_nl = True + if stashed: + yield stashed + stashed = None yield (newline, token, spos, epos, line) elif initial == '#': assert not token.endswith("\n") + if stashed: + yield stashed + stashed = None yield (COMMENT, token, spos, epos, line) elif token in triple_quoted: endprog = endprogs[token] @@ -475,6 +503,9 @@ def generate_tokens(readline): if endmatch: # all on one line pos = endmatch.end(0) token = line[start:pos] + if stashed: + yield stashed + stashed = None yield (STRING, token, spos, (lnum, pos), line) else: strstart = (lnum, start) # multiple lines @@ -492,22 +523,63 @@ def generate_tokens(readline): contline = line break else: # ordinary string + if stashed: + yield stashed + stashed = None yield (STRING, token, spos, epos, line) elif initial in namechars: # ordinary name - yield (NAME, token, spos, epos, line) + if token in ('async', 'await'): + if async_def: + yield (ASYNC if token == 'async' else AWAIT, + token, spos, epos, line) + continue + + tok = (NAME, token, spos, epos, line) + if token == 'async' and not stashed: + stashed = tok + continue + + if token == 'def': + if (stashed + and stashed[0] == NAME + and stashed[1] == 'async'): + + async_def = True + async_def_indent = indents[-1] + + yield (ASYNC, stashed[1], + stashed[2], stashed[3], + stashed[4]) + stashed = None + + if stashed: + yield stashed + stashed = None + + yield tok elif initial == '\\': # continued stmt # This yield is new; needed for better idempotency: + if stashed: + yield stashed + stashed = None yield (NL, token, spos, (lnum, pos), line) continued = 1 else: if initial in '([{': parenlev = parenlev + 1 elif initial in ')]}': parenlev = parenlev - 1 + if stashed: + yield stashed + stashed = None yield (OP, token, spos, epos, line) else: yield (ERRORTOKEN, line[pos], (lnum, pos), (lnum, pos+1), line) pos = pos + 1 + if stashed: + yield stashed + stashed = None + for indent in indents[1:]: # pop remaining indent levels yield (DEDENT, '', (lnum, 0), (lnum, 0), '') yield (ENDMARKER, '', (lnum, 0), (lnum, 0), '') diff --git a/Lib/lib2to3/tests/test_parser.py b/Lib/lib2to3/tests/test_parser.py index 6813c65a7953..74653ea80fef 100644 --- a/Lib/lib2to3/tests/test_parser.py +++ b/Lib/lib2to3/tests/test_parser.py @@ -181,34 +181,34 @@ def foo(): pass async def foo(): await x """) - self.validate("await x") - self.validate("""def foo(): - await x""") + self.invalid_syntax("await x") + self.invalid_syntax("""def foo(): + await x""") - self.validate("""def foo(): + self.invalid_syntax("""def foo(): def foo(): pass async def foo(): pass await x """) def test_async_var(self): - self.invalid_syntax("""async = 1""") - self.invalid_syntax("""await = 1""") - self.invalid_syntax("""def async(): pass""") + self.validate("""async = 1""") + self.validate("""await = 1""") + self.validate("""def async(): pass""") def test_async_with(self): self.validate("""async def foo(): async for a in b: pass""") - self.validate("""def foo(): - async for a in b: pass""") + self.invalid_syntax("""def foo(): + async for a in b: pass""") def test_async_for(self): self.validate("""async def foo(): async with a: pass""") - self.validate("""def foo(): - async with a: pass""") + self.invalid_syntax("""def foo(): + async with a: pass""") class TestRaiseChanges(GrammarTest): From webhook-mailer at python.org Sun Mar 18 16:02:51 2018 From: webhook-mailer at python.org (Mariatta) Date: Sun, 18 Mar 2018 20:02:51 -0000 Subject: [Python-checkins] bpo-19417: Add test_bdb.py (GH-5217) Message-ID: <mailman.95.1521403373.1871.python-checkins@python.org> https://github.com/python/cpython/commit/3fe33043ee83d19e15551094fc1e0984617ded3c commit: 3fe33043ee83d19e15551094fc1e0984617ded3c branch: master author: xdegaye <xdegaye at gmail.com> committer: Mariatta <Mariatta at users.noreply.github.com> date: 2018-03-18T13:02:47-07:00 summary: bpo-19417: Add test_bdb.py (GH-5217) files: A Lib/test/test_bdb.py A Misc/NEWS.d/next/Tests/2018-01-08-13-33-47.bpo-19417.2asoXy.rst M Lib/test/test_sundry.py diff --git a/Lib/test/test_bdb.py b/Lib/test/test_bdb.py new file mode 100644 index 000000000000..abefe6c4e57a --- /dev/null +++ b/Lib/test/test_bdb.py @@ -0,0 +1,1151 @@ +""" Test the bdb module. + + A test defines a list of tuples that may be seen as paired tuples, each + pair being defined by 'expect_tuple, set_tuple' as follows: + + ([event, [lineno[, co_name[, eargs]]]]), (set_type, [sargs]) + + * 'expect_tuple' describes the expected current state of the Bdb instance. + It may be the empty tuple and no check is done in that case. + * 'set_tuple' defines the set_*() method to be invoked when the Bdb + instance reaches this state. + + Example of an 'expect_tuple, set_tuple' pair: + + ('line', 2, 'tfunc_main'), ('step', ) + + Definitions of the members of the 'expect_tuple': + event: + Name of the trace event. The set methods that do not give back + control to the tracer [1] do not trigger a tracer event and in + that case the next 'event' may be 'None' by convention, its value + is not checked. + [1] Methods that trigger a trace event are set_step(), set_next(), + set_return(), set_until() and set_continue(). + lineno: + Line number. Line numbers are relative to the start of the + function when tracing a function in the test_bdb module (i.e. this + module). + co_name: + Name of the function being currently traced. + eargs: + A tuple: + * On an 'exception' event the tuple holds a class object, the + current exception must be an instance of this class. + * On a 'line' event, the tuple holds a dictionary and a list. The + dictionary maps each breakpoint number that has been hit on this + line to its hits count. The list holds the list of breakpoint + number temporaries that are being deleted. + + Definitions of the members of the 'set_tuple': + set_type: + The type of the set method to be invoked. This may + be the type of one of the Bdb set methods: 'step', 'next', + 'until', 'return', 'continue', 'break', 'quit', or the type of one + of the set methods added by test_bdb.Bdb: 'ignore', 'enable', + 'disable', 'clear', 'up', 'down'. + sargs: + The arguments of the set method if any, packed in a tuple. +""" + +import bdb as _bdb +import sys +import os +import unittest +import textwrap +import importlib +import linecache +from contextlib import contextmanager +from itertools import islice, repeat +import test.support + +class BdbException(Exception): pass +class BdbError(BdbException): """Error raised by the Bdb instance.""" +class BdbSyntaxError(BdbException): """Syntax error in the test case.""" +class BdbNotExpectedError(BdbException): """Unexpected result.""" + +# When 'dry_run' is set to true, expect tuples are ignored and the actual +# state of the tracer is printed after running each set_*() method of the test +# case. The full list of breakpoints and their attributes is also printed +# after each 'line' event where a breakpoint has been hit. +dry_run = 0 + +def reset_Breakpoint(): + _bdb.Breakpoint.next = 1 + _bdb.Breakpoint.bplist = {} + _bdb.Breakpoint.bpbynumber = [None] + +def info_breakpoints(): + bp_list = [bp for bp in _bdb.Breakpoint.bpbynumber if bp] + if not bp_list: + return '' + + header_added = False + for bp in bp_list: + if not header_added: + info = 'BpNum Temp Enb Hits Ignore Where\n' + header_added = True + + disp = 'yes ' if bp.temporary else 'no ' + enab = 'yes' if bp.enabled else 'no ' + info += ('%-5d %s %s %-4d %-6d at %s:%d' % + (bp.number, disp, enab, bp.hits, bp.ignore, + os.path.basename(bp.file), bp.line)) + if bp.cond: + info += '\n\tstop only if %s' % (bp.cond,) + info += '\n' + return info + +class Bdb(_bdb.Bdb): + """Extend Bdb to enhance test coverage.""" + + def trace_dispatch(self, frame, event, arg): + self.currentbp = None + return super().trace_dispatch(frame, event, arg) + + def set_break(self, filename, lineno, temporary=False, cond=None, + funcname=None): + if isinstance(funcname, str): + if filename == __file__: + globals_ = globals() + else: + module = importlib.import_module(filename[:-3]) + globals_ = module.__dict__ + func = eval(funcname, globals_) + code = func.__code__ + filename = code.co_filename + lineno = code.co_firstlineno + funcname = code.co_name + + res = super().set_break(filename, lineno, temporary=temporary, + cond=cond, funcname=funcname) + if isinstance(res, str): + raise BdbError(res) + return res + + def get_stack(self, f, t): + self.stack, self.index = super().get_stack(f, t) + self.frame = self.stack[self.index][0] + return self.stack, self.index + + def set_ignore(self, bpnum): + """Increment the ignore count of Breakpoint number 'bpnum'.""" + bp = self.get_bpbynumber(bpnum) + bp.ignore += 1 + + def set_enable(self, bpnum): + bp = self.get_bpbynumber(bpnum) + bp.enabled = True + + def set_disable(self, bpnum): + bp = self.get_bpbynumber(bpnum) + bp.enabled = False + + def set_clear(self, fname, lineno): + err = self.clear_break(fname, lineno) + if err: + raise BdbError(err) + + def set_up(self): + """Move up in the frame stack.""" + if not self.index: + raise BdbError('Oldest frame') + self.index -= 1 + self.frame = self.stack[self.index][0] + + def set_down(self): + """Move down in the frame stack.""" + if self.index + 1 == len(self.stack): + raise BdbError('Newest frame') + self.index += 1 + self.frame = self.stack[self.index][0] + +class Tracer(Bdb): + """A tracer for testing the bdb module.""" + + def __init__(self, expect_set, skip=None, dry_run=False, test_case=None): + super().__init__(skip=skip) + self.expect_set = expect_set + self.dry_run = dry_run + self.header = ('Dry-run results for %s:' % test_case if + test_case is not None else None) + self.init_test() + + def init_test(self): + self.cur_except = None + self.expect_set_no = 0 + self.breakpoint_hits = None + self.expected_list = list(islice(self.expect_set, 0, None, 2)) + self.set_list = list(islice(self.expect_set, 1, None, 2)) + + def trace_dispatch(self, frame, event, arg): + # On an 'exception' event, call_exc_trace() in Python/ceval.c discards + # a BdbException raised by the Tracer instance, so we raise it on the + # next trace_dispatch() call that occurs unless the set_quit() or + # set_continue() method has been invoked on the 'exception' event. + if self.cur_except is not None: + raise self.cur_except + + if event == 'exception': + try: + res = super().trace_dispatch(frame, event, arg) + return res + except BdbException as e: + self.cur_except = e + return self.trace_dispatch + else: + return super().trace_dispatch(frame, event, arg) + + def user_call(self, frame, argument_list): + # Adopt the same behavior as pdb and, as a side effect, skip also the + # first 'call' event when the Tracer is started with Tracer.runcall() + # which may be possibly considered as a bug. + if not self.stop_here(frame): + return + self.process_event('call', frame, argument_list) + self.next_set_method() + + def user_line(self, frame): + self.process_event('line', frame) + + if self.dry_run and self.breakpoint_hits: + info = info_breakpoints().strip('\n') + # Indent each line. + for line in info.split('\n'): + print(' ' + line) + self.delete_temporaries() + self.breakpoint_hits = None + + self.next_set_method() + + def user_return(self, frame, return_value): + self.process_event('return', frame, return_value) + self.next_set_method() + + def user_exception(self, frame, exc_info): + self.exc_info = exc_info + self.process_event('exception', frame) + self.next_set_method() + + def do_clear(self, arg): + # The temporary breakpoints are deleted in user_line(). + bp_list = [self.currentbp] + self.breakpoint_hits = (bp_list, bp_list) + + def delete_temporaries(self): + if self.breakpoint_hits: + for n in self.breakpoint_hits[1]: + self.clear_bpbynumber(n) + + def pop_next(self): + self.expect_set_no += 1 + try: + self.expect = self.expected_list.pop(0) + except IndexError: + raise BdbNotExpectedError( + 'expect_set list exhausted, cannot pop item %d' % + self.expect_set_no) + self.set_tuple = self.set_list.pop(0) + + def process_event(self, event, frame, *args): + # Call get_stack() to enable walking the stack with set_up() and + # set_down(). + tb = None + if event == 'exception': + tb = self.exc_info[2] + self.get_stack(frame, tb) + + # A breakpoint has been hit and it is not a temporary. + if self.currentbp is not None and not self.breakpoint_hits: + bp_list = [self.currentbp] + self.breakpoint_hits = (bp_list, []) + + # Pop next event. + self.event= event + self.pop_next() + if self.dry_run: + self.print_state(self.header) + return + + # Validate the expected results. + if self.expect: + self.check_equal(self.expect[0], event, 'Wrong event type') + self.check_lno_name() + + if event in ('call', 'return'): + self.check_expect_max_size(3) + elif len(self.expect) > 3: + if event == 'line': + bps, temporaries = self.expect[3] + bpnums = sorted(bps.keys()) + if not self.breakpoint_hits: + self.raise_not_expected( + 'No breakpoints hit at expect_set item %d' % + self.expect_set_no) + self.check_equal(bpnums, self.breakpoint_hits[0], + 'Breakpoint numbers do not match') + self.check_equal([bps[n] for n in bpnums], + [self.get_bpbynumber(n).hits for + n in self.breakpoint_hits[0]], + 'Wrong breakpoint hit count') + self.check_equal(sorted(temporaries), self.breakpoint_hits[1], + 'Wrong temporary breakpoints') + + elif event == 'exception': + if not isinstance(self.exc_info[1], self.expect[3]): + self.raise_not_expected( + "Wrong exception at expect_set item %d, got '%s'" % + (self.expect_set_no, self.exc_info)) + + def check_equal(self, expected, result, msg): + if expected == result: + return + self.raise_not_expected("%s at expect_set item %d, got '%s'" % + (msg, self.expect_set_no, result)) + + def check_lno_name(self): + """Check the line number and function co_name.""" + s = len(self.expect) + if s > 1: + lineno = self.lno_abs2rel() + self.check_equal(self.expect[1], lineno, 'Wrong line number') + if s > 2: + self.check_equal(self.expect[2], self.frame.f_code.co_name, + 'Wrong function name') + + def check_expect_max_size(self, size): + if len(self.expect) > size: + raise BdbSyntaxError('Invalid size of the %s expect tuple: %s' % + (self.event, self.expect)) + + def lno_abs2rel(self): + fname = self.canonic(self.frame.f_code.co_filename) + lineno = self.frame.f_lineno + return ((lineno - self.frame.f_code.co_firstlineno + 1) + if fname == self.canonic(__file__) else lineno) + + def lno_rel2abs(self, fname, lineno): + return (self.frame.f_code.co_firstlineno + lineno - 1 + if (lineno and self.canonic(fname) == self.canonic(__file__)) + else lineno) + + def get_state(self): + lineno = self.lno_abs2rel() + co_name = self.frame.f_code.co_name + state = "('%s', %d, '%s'" % (self.event, lineno, co_name) + if self.breakpoint_hits: + bps = '{' + for n in self.breakpoint_hits[0]: + if bps != '{': + bps += ', ' + bps += '%s: %s' % (n, self.get_bpbynumber(n).hits) + bps += '}' + bps = '(' + bps + ', ' + str(self.breakpoint_hits[1]) + ')' + state += ', ' + bps + elif self.event == 'exception': + state += ', ' + self.exc_info[0].__name__ + state += '), ' + return state.ljust(32) + str(self.set_tuple) + ',' + + def print_state(self, header=None): + if header is not None and self.expect_set_no == 1: + print() + print(header) + print('%d: %s' % (self.expect_set_no, self.get_state())) + + def raise_not_expected(self, msg): + msg += '\n' + msg += ' Expected: %s\n' % str(self.expect) + msg += ' Got: ' + self.get_state() + raise BdbNotExpectedError(msg) + + def next_set_method(self): + set_type = self.set_tuple[0] + args = self.set_tuple[1] if len(self.set_tuple) == 2 else None + set_method = getattr(self, 'set_' + set_type) + + # The following set methods give back control to the tracer. + if set_type in ('step', 'continue', 'quit'): + set_method() + return + elif set_type in ('next', 'return'): + set_method(self.frame) + return + elif set_type == 'until': + lineno = None + if args: + lineno = self.lno_rel2abs(self.frame.f_code.co_filename, + args[0]) + set_method(self.frame, lineno) + return + + # The following set methods do not give back control to the tracer and + # next_set_method() is called recursively. + if (args and set_type in ('break', 'clear', 'ignore', 'enable', + 'disable')) or set_type in ('up', 'down'): + if set_type in ('break', 'clear'): + fname, lineno, *remain = args + lineno = self.lno_rel2abs(fname, lineno) + args = [fname, lineno] + args.extend(remain) + set_method(*args) + elif set_type in ('ignore', 'enable', 'disable'): + set_method(*args) + elif set_type in ('up', 'down'): + set_method() + + # Process the next expect_set item. + # It is not expected that a test may reach the recursion limit. + self.event= None + self.pop_next() + if self.dry_run: + self.print_state() + else: + if self.expect: + self.check_lno_name() + self.check_expect_max_size(3) + self.next_set_method() + else: + raise BdbSyntaxError('"%s" is an invalid set_tuple' % + self.set_tuple) + +class TracerRun(): + """Provide a context for running a Tracer instance with a test case.""" + + def __init__(self, test_case, skip=None): + self.test_case = test_case + self.dry_run = test_case.dry_run + self.tracer = Tracer(test_case.expect_set, skip=skip, + dry_run=self.dry_run, test_case=test_case.id()) + + def __enter__(self): + # test_pdb does not reset Breakpoint class attributes on exit :-( + reset_Breakpoint() + return self.tracer + + def __exit__(self, type_=None, value=None, traceback=None): + reset_Breakpoint() + sys.settrace(None) + + not_empty = '' + if self.tracer.set_list: + not_empty += 'All paired tuples have not been processed, ' + not_empty += ('the last one was number %d' % + self.tracer.expect_set_no) + + # Make a BdbNotExpectedError a unittest failure. + if type_ is not None and issubclass(BdbNotExpectedError, type_): + if isinstance(value, BaseException) and value.args: + err_msg = value.args[0] + if not_empty: + err_msg += '\n' + not_empty + if self.dry_run: + print(err_msg) + return True + else: + self.test_case.fail(err_msg) + else: + assert False, 'BdbNotExpectedError with empty args' + + if not_empty: + if self.dry_run: + print(not_empty) + else: + self.test_case.fail(not_empty) + +def run_test(modules, set_list, skip=None): + """Run a test and print the dry-run results. + + 'modules': A dictionary mapping module names to their source code as a + string. The dictionary MUST include one module named + 'test_module' with a main() function. + 'set_list': A list of set_type tuples to be run on the module. + + For example, running the following script outputs the following results: + + ***************************** SCRIPT ******************************** + + from test.test_bdb import run_test, break_in_func + + code = ''' + def func(): + lno = 3 + + def main(): + func() + lno = 7 + ''' + + set_list = [ + break_in_func('func', 'test_module.py'), + ('continue', ), + ('step', ), + ('step', ), + ('step', ), + ('quit', ), + ] + + modules = { 'test_module': code } + run_test(modules, set_list) + + **************************** results ******************************** + + 1: ('line', 2, 'tfunc_import'), ('next',), + 2: ('line', 3, 'tfunc_import'), ('step',), + 3: ('call', 5, 'main'), ('break', ('test_module.py', None, False, None, 'func')), + 4: ('None', 5, 'main'), ('continue',), + 5: ('line', 3, 'func', ({1: 1}, [])), ('step',), + BpNum Temp Enb Hits Ignore Where + 1 no yes 1 0 at test_module.py:2 + 6: ('return', 3, 'func'), ('step',), + 7: ('line', 7, 'main'), ('step',), + 8: ('return', 7, 'main'), ('quit',), + + ************************************************************************* + + """ + def gen(a, b): + try: + while 1: + x = next(a) + y = next(b) + yield x + yield y + except StopIteration: + return + + # Step over the import statement in tfunc_import using 'next' and step + # into main() in test_module. + sl = [('next', ), ('step', )] + sl.extend(set_list) + + test = BaseTestCase() + test.dry_run = True + test.id = lambda : None + test.expect_set = list(gen(repeat(()), iter(sl))) + with create_modules(modules): + sys.path.append(os.getcwd()) + with TracerRun(test, skip=skip) as tracer: + tracer.runcall(tfunc_import) + + at contextmanager +def create_modules(modules): + with test.support.temp_cwd(): + try: + for m in modules: + fname = m + '.py' + with open(fname, 'w') as f: + f.write(textwrap.dedent(modules[m])) + linecache.checkcache(fname) + importlib.invalidate_caches() + yield + finally: + for m in modules: + test.support.forget(m) + +def break_in_func(funcname, fname=__file__, temporary=False, cond=None): + return 'break', (fname, None, temporary, cond, funcname) + +TEST_MODULE = 'test_module' +TEST_MODULE_FNAME = TEST_MODULE + '.py' +def tfunc_import(): + import test_module + test_module.main() + +def tfunc_main(): + lno = 2 + tfunc_first() + tfunc_second() + lno = 5 + lno = 6 + lno = 7 + +def tfunc_first(): + lno = 2 + lno = 3 + lno = 4 + +def tfunc_second(): + lno = 2 + +class BaseTestCase(unittest.TestCase): + """Base class for all tests.""" + + dry_run = dry_run + + def fail(self, msg=None): + # Override fail() to use 'raise from None' to avoid repetition of the + # error message and traceback. + raise self.failureException(msg) from None + +class StateTestCase(BaseTestCase): + """Test the step, next, return, until and quit 'set_' methods.""" + + def test_step(self): + self.expect_set = [ + ('line', 2, 'tfunc_main'), ('step', ), + ('line', 3, 'tfunc_main'), ('step', ), + ('call', 1, 'tfunc_first'), ('step', ), + ('line', 2, 'tfunc_first'), ('quit', ), + ] + with TracerRun(self) as tracer: + tracer.runcall(tfunc_main) + + def test_step_next_on_last_statement(self): + for set_type in ('step', 'next'): + with self.subTest(set_type=set_type): + self.expect_set = [ + ('line', 2, 'tfunc_main'), ('step', ), + ('line', 3, 'tfunc_main'), ('step', ), + ('call', 1, 'tfunc_first'), ('break', (__file__, 3)), + ('None', 1, 'tfunc_first'), ('continue', ), + ('line', 3, 'tfunc_first', ({1:1}, [])), (set_type, ), + ('line', 4, 'tfunc_first'), ('quit', ), + ] + with TracerRun(self) as tracer: + tracer.runcall(tfunc_main) + + def test_next(self): + self.expect_set = [ + ('line', 2, 'tfunc_main'), ('step', ), + ('line', 3, 'tfunc_main'), ('next', ), + ('line', 4, 'tfunc_main'), ('step', ), + ('call', 1, 'tfunc_second'), ('step', ), + ('line', 2, 'tfunc_second'), ('quit', ), + ] + with TracerRun(self) as tracer: + tracer.runcall(tfunc_main) + + def test_next_over_import(self): + code = """ + def main(): + lno = 3 + """ + modules = { TEST_MODULE: code } + with create_modules(modules): + self.expect_set = [ + ('line', 2, 'tfunc_import'), ('next', ), + ('line', 3, 'tfunc_import'), ('quit', ), + ] + with TracerRun(self) as tracer: + tracer.runcall(tfunc_import) + + def test_next_on_plain_statement(self): + # Check that set_next() is equivalent to set_step() on a plain + # statement. + self.expect_set = [ + ('line', 2, 'tfunc_main'), ('step', ), + ('line', 3, 'tfunc_main'), ('step', ), + ('call', 1, 'tfunc_first'), ('next', ), + ('line', 2, 'tfunc_first'), ('quit', ), + ] + with TracerRun(self) as tracer: + tracer.runcall(tfunc_main) + + def test_next_in_caller_frame(self): + # Check that set_next() in the caller frame causes the tracer + # to stop next in the caller frame. + self.expect_set = [ + ('line', 2, 'tfunc_main'), ('step', ), + ('line', 3, 'tfunc_main'), ('step', ), + ('call', 1, 'tfunc_first'), ('up', ), + ('None', 3, 'tfunc_main'), ('next', ), + ('line', 4, 'tfunc_main'), ('quit', ), + ] + with TracerRun(self) as tracer: + tracer.runcall(tfunc_main) + + def test_return(self): + self.expect_set = [ + ('line', 2, 'tfunc_main'), ('step', ), + ('line', 3, 'tfunc_main'), ('step', ), + ('call', 1, 'tfunc_first'), ('step', ), + ('line', 2, 'tfunc_first'), ('return', ), + ('return', 4, 'tfunc_first'), ('step', ), + ('line', 4, 'tfunc_main'), ('quit', ), + ] + with TracerRun(self) as tracer: + tracer.runcall(tfunc_main) + + def test_return_in_caller_frame(self): + self.expect_set = [ + ('line', 2, 'tfunc_main'), ('step', ), + ('line', 3, 'tfunc_main'), ('step', ), + ('call', 1, 'tfunc_first'), ('up', ), + ('None', 3, 'tfunc_main'), ('return', ), + ('return', 7, 'tfunc_main'), ('quit', ), + ] + with TracerRun(self) as tracer: + tracer.runcall(tfunc_main) + + def test_until(self): + self.expect_set = [ + ('line', 2, 'tfunc_main'), ('step', ), + ('line', 3, 'tfunc_main'), ('step', ), + ('call', 1, 'tfunc_first'), ('step', ), + ('line', 2, 'tfunc_first'), ('until', (4, )), + ('line', 4, 'tfunc_first'), ('quit', ), + ] + with TracerRun(self) as tracer: + tracer.runcall(tfunc_main) + + def test_until_with_too_large_count(self): + self.expect_set = [ + ('line', 2, 'tfunc_main'), break_in_func('tfunc_first'), + ('None', 2, 'tfunc_main'), ('continue', ), + ('line', 2, 'tfunc_first', ({1:1}, [])), ('until', (9999, )), + ('return', 4, 'tfunc_first'), ('quit', ), + ] + with TracerRun(self) as tracer: + tracer.runcall(tfunc_main) + + def test_until_in_caller_frame(self): + self.expect_set = [ + ('line', 2, 'tfunc_main'), ('step', ), + ('line', 3, 'tfunc_main'), ('step', ), + ('call', 1, 'tfunc_first'), ('up', ), + ('None', 3, 'tfunc_main'), ('until', (6, )), + ('line', 6, 'tfunc_main'), ('quit', ), + ] + with TracerRun(self) as tracer: + tracer.runcall(tfunc_main) + + def test_skip(self): + # Check that tracing is skipped over the import statement in + # 'tfunc_import()'. + code = """ + def main(): + lno = 3 + """ + modules = { TEST_MODULE: code } + with create_modules(modules): + self.expect_set = [ + ('line', 2, 'tfunc_import'), ('step', ), + ('line', 3, 'tfunc_import'), ('quit', ), + ] + skip = ('importlib*', TEST_MODULE) + with TracerRun(self, skip=skip) as tracer: + tracer.runcall(tfunc_import) + + def test_down(self): + # Check that set_down() raises BdbError at the newest frame. + self.expect_set = [ + ('line', 2, 'tfunc_main'), ('down', ), + ] + with TracerRun(self) as tracer: + self.assertRaises(BdbError, tracer.runcall, tfunc_main) + + def test_up(self): + self.expect_set = [ + ('line', 2, 'tfunc_main'), ('step', ), + ('line', 3, 'tfunc_main'), ('step', ), + ('call', 1, 'tfunc_first'), ('up', ), + ('None', 3, 'tfunc_main'), ('quit', ), + ] + with TracerRun(self) as tracer: + tracer.runcall(tfunc_main) + +class BreakpointTestCase(BaseTestCase): + """Test the breakpoint set method.""" + + def test_bp_on_non_existent_module(self): + self.expect_set = [ + ('line', 2, 'tfunc_import'), ('break', ('/non/existent/module.py', 1)) + ] + with TracerRun(self) as tracer: + self.assertRaises(BdbError, tracer.runcall, tfunc_import) + + def test_bp_after_last_statement(self): + code = """ + def main(): + lno = 3 + """ + modules = { TEST_MODULE: code } + with create_modules(modules): + self.expect_set = [ + ('line', 2, 'tfunc_import'), ('break', (TEST_MODULE_FNAME, 4)) + ] + with TracerRun(self) as tracer: + self.assertRaises(BdbError, tracer.runcall, tfunc_import) + + def test_temporary_bp(self): + code = """ + def func(): + lno = 3 + + def main(): + for i in range(2): + func() + """ + modules = { TEST_MODULE: code } + with create_modules(modules): + self.expect_set = [ + ('line', 2, 'tfunc_import'), + break_in_func('func', TEST_MODULE_FNAME, True), + ('None', 2, 'tfunc_import'), + break_in_func('func', TEST_MODULE_FNAME, True), + ('None', 2, 'tfunc_import'), ('continue', ), + ('line', 3, 'func', ({1:1}, [1])), ('continue', ), + ('line', 3, 'func', ({2:1}, [2])), ('quit', ), + ] + with TracerRun(self) as tracer: + tracer.runcall(tfunc_import) + + def test_disabled_temporary_bp(self): + code = """ + def func(): + lno = 3 + + def main(): + for i in range(3): + func() + """ + modules = { TEST_MODULE: code } + with create_modules(modules): + self.expect_set = [ + ('line', 2, 'tfunc_import'), + break_in_func('func', TEST_MODULE_FNAME), + ('None', 2, 'tfunc_import'), + break_in_func('func', TEST_MODULE_FNAME, True), + ('None', 2, 'tfunc_import'), ('disable', (2, )), + ('None', 2, 'tfunc_import'), ('continue', ), + ('line', 3, 'func', ({1:1}, [])), ('enable', (2, )), + ('None', 3, 'func'), ('disable', (1, )), + ('None', 3, 'func'), ('continue', ), + ('line', 3, 'func', ({2:1}, [2])), ('enable', (1, )), + ('None', 3, 'func'), ('continue', ), + ('line', 3, 'func', ({1:2}, [])), ('quit', ), + ] + with TracerRun(self) as tracer: + tracer.runcall(tfunc_import) + + def test_bp_condition(self): + code = """ + def func(a): + lno = 3 + + def main(): + for i in range(3): + func(i) + """ + modules = { TEST_MODULE: code } + with create_modules(modules): + self.expect_set = [ + ('line', 2, 'tfunc_import'), + break_in_func('func', TEST_MODULE_FNAME, False, 'a == 2'), + ('None', 2, 'tfunc_import'), ('continue', ), + ('line', 3, 'func', ({1:3}, [])), ('quit', ), + ] + with TracerRun(self) as tracer: + tracer.runcall(tfunc_import) + + def test_bp_exception_on_condition_evaluation(self): + code = """ + def func(a): + lno = 3 + + def main(): + func(0) + """ + modules = { TEST_MODULE: code } + with create_modules(modules): + self.expect_set = [ + ('line', 2, 'tfunc_import'), + break_in_func('func', TEST_MODULE_FNAME, False, '1 / 0'), + ('None', 2, 'tfunc_import'), ('continue', ), + ('line', 3, 'func', ({1:1}, [])), ('quit', ), + ] + with TracerRun(self) as tracer: + tracer.runcall(tfunc_import) + + def test_bp_ignore_count(self): + code = """ + def func(): + lno = 3 + + def main(): + for i in range(2): + func() + """ + modules = { TEST_MODULE: code } + with create_modules(modules): + self.expect_set = [ + ('line', 2, 'tfunc_import'), + break_in_func('func', TEST_MODULE_FNAME), + ('None', 2, 'tfunc_import'), ('ignore', (1, )), + ('None', 2, 'tfunc_import'), ('continue', ), + ('line', 3, 'func', ({1:2}, [])), ('quit', ), + ] + with TracerRun(self) as tracer: + tracer.runcall(tfunc_import) + + def test_ignore_count_on_disabled_bp(self): + code = """ + def func(): + lno = 3 + + def main(): + for i in range(3): + func() + """ + modules = { TEST_MODULE: code } + with create_modules(modules): + self.expect_set = [ + ('line', 2, 'tfunc_import'), + break_in_func('func', TEST_MODULE_FNAME), + ('None', 2, 'tfunc_import'), + break_in_func('func', TEST_MODULE_FNAME), + ('None', 2, 'tfunc_import'), ('ignore', (1, )), + ('None', 2, 'tfunc_import'), ('disable', (1, )), + ('None', 2, 'tfunc_import'), ('continue', ), + ('line', 3, 'func', ({2:1}, [])), ('enable', (1, )), + ('None', 3, 'func'), ('continue', ), + ('line', 3, 'func', ({2:2}, [])), ('continue', ), + ('line', 3, 'func', ({1:2}, [])), ('quit', ), + ] + with TracerRun(self) as tracer: + tracer.runcall(tfunc_import) + + def test_clear_two_bp_on_same_line(self): + code = """ + def func(): + lno = 3 + lno = 4 + + def main(): + for i in range(3): + func() + """ + modules = { TEST_MODULE: code } + with create_modules(modules): + self.expect_set = [ + ('line', 2, 'tfunc_import'), ('break', (TEST_MODULE_FNAME, 3)), + ('None', 2, 'tfunc_import'), ('break', (TEST_MODULE_FNAME, 3)), + ('None', 2, 'tfunc_import'), ('break', (TEST_MODULE_FNAME, 4)), + ('None', 2, 'tfunc_import'), ('continue', ), + ('line', 3, 'func', ({1:1}, [])), ('continue', ), + ('line', 4, 'func', ({3:1}, [])), ('clear', (TEST_MODULE_FNAME, 3)), + ('None', 4, 'func'), ('continue', ), + ('line', 4, 'func', ({3:2}, [])), ('quit', ), + ] + with TracerRun(self) as tracer: + tracer.runcall(tfunc_import) + + def test_clear_at_no_bp(self): + self.expect_set = [ + ('line', 2, 'tfunc_import'), ('clear', (__file__, 1)) + ] + with TracerRun(self) as tracer: + self.assertRaises(BdbError, tracer.runcall, tfunc_import) + +class RunTestCase(BaseTestCase): + """Test run, runeval and set_trace.""" + + def test_run_step(self): + # Check that the bdb 'run' method stops at the first line event. + code = """ + lno = 2 + """ + self.expect_set = [ + ('line', 2, '<module>'), ('step', ), + ('return', 2, '<module>'), ('quit', ), + ] + with TracerRun(self) as tracer: + tracer.run(compile(textwrap.dedent(code), '<string>', 'exec')) + + def test_runeval_step(self): + # Test bdb 'runeval'. + code = """ + def main(): + lno = 3 + """ + modules = { TEST_MODULE: code } + with create_modules(modules): + self.expect_set = [ + ('line', 1, '<module>'), ('step', ), + ('call', 2, 'main'), ('step', ), + ('line', 3, 'main'), ('step', ), + ('return', 3, 'main'), ('step', ), + ('return', 1, '<module>'), ('quit', ), + ] + import test_module + with TracerRun(self) as tracer: + tracer.runeval('test_module.main()', globals(), locals()) + +class IssuesTestCase(BaseTestCase): + """Test fixed bdb issues.""" + + def test_step_at_return_with_no_trace_in_caller(self): + # Issue #13183. + # Check that the tracer does step into the caller frame when the + # trace function is not set in that frame. + code_1 = """ + from test_module_2 import func + def main(): + func() + lno = 5 + """ + code_2 = """ + def func(): + lno = 3 + """ + modules = { + TEST_MODULE: code_1, + 'test_module_2': code_2, + } + with create_modules(modules): + self.expect_set = [ + ('line', 2, 'tfunc_import'), + break_in_func('func', 'test_module_2.py'), + ('None', 2, 'tfunc_import'), ('continue', ), + ('line', 3, 'func', ({1:1}, [])), ('step', ), + ('return', 3, 'func'), ('step', ), + ('line', 5, 'main'), ('quit', ), + ] + with TracerRun(self) as tracer: + tracer.runcall(tfunc_import) + + def test_next_until_return_in_generator(self): + # Issue #16596. + # Check that set_next(), set_until() and set_return() do not treat the + # `yield` and `yield from` statements as if they were returns and stop + # instead in the current frame. + code = """ + def test_gen(): + yield 0 + lno = 4 + return 123 + + def main(): + it = test_gen() + next(it) + next(it) + lno = 11 + """ + modules = { TEST_MODULE: code } + for set_type in ('next', 'until', 'return'): + with self.subTest(set_type=set_type): + with create_modules(modules): + self.expect_set = [ + ('line', 2, 'tfunc_import'), + break_in_func('test_gen', TEST_MODULE_FNAME), + ('None', 2, 'tfunc_import'), ('continue', ), + ('line', 3, 'test_gen', ({1:1}, [])), (set_type, ), + ] + + if set_type == 'return': + self.expect_set.extend( + [('exception', 10, 'main', StopIteration), ('step',), + ('return', 10, 'main'), ('quit', ), + ] + ) + else: + self.expect_set.extend( + [('line', 4, 'test_gen'), ('quit', ),] + ) + with TracerRun(self) as tracer: + tracer.runcall(tfunc_import) + + def test_next_command_in_generator_for_loop(self): + # Issue #16596. + code = """ + def test_gen(): + yield 0 + lno = 4 + yield 1 + return 123 + + def main(): + for i in test_gen(): + lno = 10 + lno = 11 + """ + modules = { TEST_MODULE: code } + with create_modules(modules): + self.expect_set = [ + ('line', 2, 'tfunc_import'), + break_in_func('test_gen', TEST_MODULE_FNAME), + ('None', 2, 'tfunc_import'), ('continue', ), + ('line', 3, 'test_gen', ({1:1}, [])), ('next', ), + ('line', 4, 'test_gen'), ('next', ), + ('line', 5, 'test_gen'), ('next', ), + ('line', 6, 'test_gen'), ('next', ), + ('exception', 9, 'main', StopIteration), ('step', ), + ('line', 11, 'main'), ('quit', ), + + ] + with TracerRun(self) as tracer: + tracer.runcall(tfunc_import) + + def test_next_command_in_generator_with_subiterator(self): + # Issue #16596. + code = """ + def test_subgen(): + yield 0 + return 123 + + def test_gen(): + x = yield from test_subgen() + return 456 + + def main(): + for i in test_gen(): + lno = 12 + lno = 13 + """ + modules = { TEST_MODULE: code } + with create_modules(modules): + self.expect_set = [ + ('line', 2, 'tfunc_import'), + break_in_func('test_gen', TEST_MODULE_FNAME), + ('None', 2, 'tfunc_import'), ('continue', ), + ('line', 7, 'test_gen', ({1:1}, [])), ('next', ), + ('line', 8, 'test_gen'), ('next', ), + ('exception', 11, 'main', StopIteration), ('step', ), + ('line', 13, 'main'), ('quit', ), + + ] + with TracerRun(self) as tracer: + tracer.runcall(tfunc_import) + + def test_return_command_in_generator_with_subiterator(self): + # Issue #16596. + code = """ + def test_subgen(): + yield 0 + return 123 + + def test_gen(): + x = yield from test_subgen() + return 456 + + def main(): + for i in test_gen(): + lno = 12 + lno = 13 + """ + modules = { TEST_MODULE: code } + with create_modules(modules): + self.expect_set = [ + ('line', 2, 'tfunc_import'), + break_in_func('test_subgen', TEST_MODULE_FNAME), + ('None', 2, 'tfunc_import'), ('continue', ), + ('line', 3, 'test_subgen', ({1:1}, [])), ('return', ), + ('exception', 7, 'test_gen', StopIteration), ('return', ), + ('exception', 11, 'main', StopIteration), ('step', ), + ('line', 13, 'main'), ('quit', ), + + ] + with TracerRun(self) as tracer: + tracer.runcall(tfunc_import) + +def test_main(): + test.support.run_unittest( + StateTestCase, + RunTestCase, + BreakpointTestCase, + IssuesTestCase, + ) + +if __name__ == "__main__": + test_main() diff --git a/Lib/test/test_sundry.py b/Lib/test/test_sundry.py index dddf3265525a..4025c2354a6c 100644 --- a/Lib/test/test_sundry.py +++ b/Lib/test/test_sundry.py @@ -6,7 +6,7 @@ class TestUntestedModules(unittest.TestCase): def test_untested_modules_can_be_imported(self): - untested = ('bdb', 'encodings', 'formatter', 'tabnanny') + untested = ('encodings', 'formatter', 'tabnanny') with support.check_warnings(quiet=True): for name in untested: try: diff --git a/Misc/NEWS.d/next/Tests/2018-01-08-13-33-47.bpo-19417.2asoXy.rst b/Misc/NEWS.d/next/Tests/2018-01-08-13-33-47.bpo-19417.2asoXy.rst new file mode 100644 index 000000000000..739352fcdd67 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2018-01-08-13-33-47.bpo-19417.2asoXy.rst @@ -0,0 +1 @@ +Add test_bdb.py. From webhook-mailer at python.org Sun Mar 18 16:25:18 2018 From: webhook-mailer at python.org (Miss Islington (bot)) Date: Sun, 18 Mar 2018 20:25:18 -0000 Subject: [Python-checkins] bpo-19417: Add test_bdb.py (GH-5217) Message-ID: <mailman.96.1521404719.1871.python-checkins@python.org> https://github.com/python/cpython/commit/fdd8e8b4ffb68a4e8749bdc3b130fea7bbbf821e commit: fdd8e8b4ffb68a4e8749bdc3b130fea7bbbf821e branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: GitHub <noreply at github.com> date: 2018-03-18T13:25:15-07:00 summary: bpo-19417: Add test_bdb.py (GH-5217) (cherry picked from commit 3fe33043ee83d19e15551094fc1e0984617ded3c) Co-authored-by: xdegaye <xdegaye at gmail.com> files: A Lib/test/test_bdb.py A Misc/NEWS.d/next/Tests/2018-01-08-13-33-47.bpo-19417.2asoXy.rst M Lib/test/test_sundry.py diff --git a/Lib/test/test_bdb.py b/Lib/test/test_bdb.py new file mode 100644 index 000000000000..abefe6c4e57a --- /dev/null +++ b/Lib/test/test_bdb.py @@ -0,0 +1,1151 @@ +""" Test the bdb module. + + A test defines a list of tuples that may be seen as paired tuples, each + pair being defined by 'expect_tuple, set_tuple' as follows: + + ([event, [lineno[, co_name[, eargs]]]]), (set_type, [sargs]) + + * 'expect_tuple' describes the expected current state of the Bdb instance. + It may be the empty tuple and no check is done in that case. + * 'set_tuple' defines the set_*() method to be invoked when the Bdb + instance reaches this state. + + Example of an 'expect_tuple, set_tuple' pair: + + ('line', 2, 'tfunc_main'), ('step', ) + + Definitions of the members of the 'expect_tuple': + event: + Name of the trace event. The set methods that do not give back + control to the tracer [1] do not trigger a tracer event and in + that case the next 'event' may be 'None' by convention, its value + is not checked. + [1] Methods that trigger a trace event are set_step(), set_next(), + set_return(), set_until() and set_continue(). + lineno: + Line number. Line numbers are relative to the start of the + function when tracing a function in the test_bdb module (i.e. this + module). + co_name: + Name of the function being currently traced. + eargs: + A tuple: + * On an 'exception' event the tuple holds a class object, the + current exception must be an instance of this class. + * On a 'line' event, the tuple holds a dictionary and a list. The + dictionary maps each breakpoint number that has been hit on this + line to its hits count. The list holds the list of breakpoint + number temporaries that are being deleted. + + Definitions of the members of the 'set_tuple': + set_type: + The type of the set method to be invoked. This may + be the type of one of the Bdb set methods: 'step', 'next', + 'until', 'return', 'continue', 'break', 'quit', or the type of one + of the set methods added by test_bdb.Bdb: 'ignore', 'enable', + 'disable', 'clear', 'up', 'down'. + sargs: + The arguments of the set method if any, packed in a tuple. +""" + +import bdb as _bdb +import sys +import os +import unittest +import textwrap +import importlib +import linecache +from contextlib import contextmanager +from itertools import islice, repeat +import test.support + +class BdbException(Exception): pass +class BdbError(BdbException): """Error raised by the Bdb instance.""" +class BdbSyntaxError(BdbException): """Syntax error in the test case.""" +class BdbNotExpectedError(BdbException): """Unexpected result.""" + +# When 'dry_run' is set to true, expect tuples are ignored and the actual +# state of the tracer is printed after running each set_*() method of the test +# case. The full list of breakpoints and their attributes is also printed +# after each 'line' event where a breakpoint has been hit. +dry_run = 0 + +def reset_Breakpoint(): + _bdb.Breakpoint.next = 1 + _bdb.Breakpoint.bplist = {} + _bdb.Breakpoint.bpbynumber = [None] + +def info_breakpoints(): + bp_list = [bp for bp in _bdb.Breakpoint.bpbynumber if bp] + if not bp_list: + return '' + + header_added = False + for bp in bp_list: + if not header_added: + info = 'BpNum Temp Enb Hits Ignore Where\n' + header_added = True + + disp = 'yes ' if bp.temporary else 'no ' + enab = 'yes' if bp.enabled else 'no ' + info += ('%-5d %s %s %-4d %-6d at %s:%d' % + (bp.number, disp, enab, bp.hits, bp.ignore, + os.path.basename(bp.file), bp.line)) + if bp.cond: + info += '\n\tstop only if %s' % (bp.cond,) + info += '\n' + return info + +class Bdb(_bdb.Bdb): + """Extend Bdb to enhance test coverage.""" + + def trace_dispatch(self, frame, event, arg): + self.currentbp = None + return super().trace_dispatch(frame, event, arg) + + def set_break(self, filename, lineno, temporary=False, cond=None, + funcname=None): + if isinstance(funcname, str): + if filename == __file__: + globals_ = globals() + else: + module = importlib.import_module(filename[:-3]) + globals_ = module.__dict__ + func = eval(funcname, globals_) + code = func.__code__ + filename = code.co_filename + lineno = code.co_firstlineno + funcname = code.co_name + + res = super().set_break(filename, lineno, temporary=temporary, + cond=cond, funcname=funcname) + if isinstance(res, str): + raise BdbError(res) + return res + + def get_stack(self, f, t): + self.stack, self.index = super().get_stack(f, t) + self.frame = self.stack[self.index][0] + return self.stack, self.index + + def set_ignore(self, bpnum): + """Increment the ignore count of Breakpoint number 'bpnum'.""" + bp = self.get_bpbynumber(bpnum) + bp.ignore += 1 + + def set_enable(self, bpnum): + bp = self.get_bpbynumber(bpnum) + bp.enabled = True + + def set_disable(self, bpnum): + bp = self.get_bpbynumber(bpnum) + bp.enabled = False + + def set_clear(self, fname, lineno): + err = self.clear_break(fname, lineno) + if err: + raise BdbError(err) + + def set_up(self): + """Move up in the frame stack.""" + if not self.index: + raise BdbError('Oldest frame') + self.index -= 1 + self.frame = self.stack[self.index][0] + + def set_down(self): + """Move down in the frame stack.""" + if self.index + 1 == len(self.stack): + raise BdbError('Newest frame') + self.index += 1 + self.frame = self.stack[self.index][0] + +class Tracer(Bdb): + """A tracer for testing the bdb module.""" + + def __init__(self, expect_set, skip=None, dry_run=False, test_case=None): + super().__init__(skip=skip) + self.expect_set = expect_set + self.dry_run = dry_run + self.header = ('Dry-run results for %s:' % test_case if + test_case is not None else None) + self.init_test() + + def init_test(self): + self.cur_except = None + self.expect_set_no = 0 + self.breakpoint_hits = None + self.expected_list = list(islice(self.expect_set, 0, None, 2)) + self.set_list = list(islice(self.expect_set, 1, None, 2)) + + def trace_dispatch(self, frame, event, arg): + # On an 'exception' event, call_exc_trace() in Python/ceval.c discards + # a BdbException raised by the Tracer instance, so we raise it on the + # next trace_dispatch() call that occurs unless the set_quit() or + # set_continue() method has been invoked on the 'exception' event. + if self.cur_except is not None: + raise self.cur_except + + if event == 'exception': + try: + res = super().trace_dispatch(frame, event, arg) + return res + except BdbException as e: + self.cur_except = e + return self.trace_dispatch + else: + return super().trace_dispatch(frame, event, arg) + + def user_call(self, frame, argument_list): + # Adopt the same behavior as pdb and, as a side effect, skip also the + # first 'call' event when the Tracer is started with Tracer.runcall() + # which may be possibly considered as a bug. + if not self.stop_here(frame): + return + self.process_event('call', frame, argument_list) + self.next_set_method() + + def user_line(self, frame): + self.process_event('line', frame) + + if self.dry_run and self.breakpoint_hits: + info = info_breakpoints().strip('\n') + # Indent each line. + for line in info.split('\n'): + print(' ' + line) + self.delete_temporaries() + self.breakpoint_hits = None + + self.next_set_method() + + def user_return(self, frame, return_value): + self.process_event('return', frame, return_value) + self.next_set_method() + + def user_exception(self, frame, exc_info): + self.exc_info = exc_info + self.process_event('exception', frame) + self.next_set_method() + + def do_clear(self, arg): + # The temporary breakpoints are deleted in user_line(). + bp_list = [self.currentbp] + self.breakpoint_hits = (bp_list, bp_list) + + def delete_temporaries(self): + if self.breakpoint_hits: + for n in self.breakpoint_hits[1]: + self.clear_bpbynumber(n) + + def pop_next(self): + self.expect_set_no += 1 + try: + self.expect = self.expected_list.pop(0) + except IndexError: + raise BdbNotExpectedError( + 'expect_set list exhausted, cannot pop item %d' % + self.expect_set_no) + self.set_tuple = self.set_list.pop(0) + + def process_event(self, event, frame, *args): + # Call get_stack() to enable walking the stack with set_up() and + # set_down(). + tb = None + if event == 'exception': + tb = self.exc_info[2] + self.get_stack(frame, tb) + + # A breakpoint has been hit and it is not a temporary. + if self.currentbp is not None and not self.breakpoint_hits: + bp_list = [self.currentbp] + self.breakpoint_hits = (bp_list, []) + + # Pop next event. + self.event= event + self.pop_next() + if self.dry_run: + self.print_state(self.header) + return + + # Validate the expected results. + if self.expect: + self.check_equal(self.expect[0], event, 'Wrong event type') + self.check_lno_name() + + if event in ('call', 'return'): + self.check_expect_max_size(3) + elif len(self.expect) > 3: + if event == 'line': + bps, temporaries = self.expect[3] + bpnums = sorted(bps.keys()) + if not self.breakpoint_hits: + self.raise_not_expected( + 'No breakpoints hit at expect_set item %d' % + self.expect_set_no) + self.check_equal(bpnums, self.breakpoint_hits[0], + 'Breakpoint numbers do not match') + self.check_equal([bps[n] for n in bpnums], + [self.get_bpbynumber(n).hits for + n in self.breakpoint_hits[0]], + 'Wrong breakpoint hit count') + self.check_equal(sorted(temporaries), self.breakpoint_hits[1], + 'Wrong temporary breakpoints') + + elif event == 'exception': + if not isinstance(self.exc_info[1], self.expect[3]): + self.raise_not_expected( + "Wrong exception at expect_set item %d, got '%s'" % + (self.expect_set_no, self.exc_info)) + + def check_equal(self, expected, result, msg): + if expected == result: + return + self.raise_not_expected("%s at expect_set item %d, got '%s'" % + (msg, self.expect_set_no, result)) + + def check_lno_name(self): + """Check the line number and function co_name.""" + s = len(self.expect) + if s > 1: + lineno = self.lno_abs2rel() + self.check_equal(self.expect[1], lineno, 'Wrong line number') + if s > 2: + self.check_equal(self.expect[2], self.frame.f_code.co_name, + 'Wrong function name') + + def check_expect_max_size(self, size): + if len(self.expect) > size: + raise BdbSyntaxError('Invalid size of the %s expect tuple: %s' % + (self.event, self.expect)) + + def lno_abs2rel(self): + fname = self.canonic(self.frame.f_code.co_filename) + lineno = self.frame.f_lineno + return ((lineno - self.frame.f_code.co_firstlineno + 1) + if fname == self.canonic(__file__) else lineno) + + def lno_rel2abs(self, fname, lineno): + return (self.frame.f_code.co_firstlineno + lineno - 1 + if (lineno and self.canonic(fname) == self.canonic(__file__)) + else lineno) + + def get_state(self): + lineno = self.lno_abs2rel() + co_name = self.frame.f_code.co_name + state = "('%s', %d, '%s'" % (self.event, lineno, co_name) + if self.breakpoint_hits: + bps = '{' + for n in self.breakpoint_hits[0]: + if bps != '{': + bps += ', ' + bps += '%s: %s' % (n, self.get_bpbynumber(n).hits) + bps += '}' + bps = '(' + bps + ', ' + str(self.breakpoint_hits[1]) + ')' + state += ', ' + bps + elif self.event == 'exception': + state += ', ' + self.exc_info[0].__name__ + state += '), ' + return state.ljust(32) + str(self.set_tuple) + ',' + + def print_state(self, header=None): + if header is not None and self.expect_set_no == 1: + print() + print(header) + print('%d: %s' % (self.expect_set_no, self.get_state())) + + def raise_not_expected(self, msg): + msg += '\n' + msg += ' Expected: %s\n' % str(self.expect) + msg += ' Got: ' + self.get_state() + raise BdbNotExpectedError(msg) + + def next_set_method(self): + set_type = self.set_tuple[0] + args = self.set_tuple[1] if len(self.set_tuple) == 2 else None + set_method = getattr(self, 'set_' + set_type) + + # The following set methods give back control to the tracer. + if set_type in ('step', 'continue', 'quit'): + set_method() + return + elif set_type in ('next', 'return'): + set_method(self.frame) + return + elif set_type == 'until': + lineno = None + if args: + lineno = self.lno_rel2abs(self.frame.f_code.co_filename, + args[0]) + set_method(self.frame, lineno) + return + + # The following set methods do not give back control to the tracer and + # next_set_method() is called recursively. + if (args and set_type in ('break', 'clear', 'ignore', 'enable', + 'disable')) or set_type in ('up', 'down'): + if set_type in ('break', 'clear'): + fname, lineno, *remain = args + lineno = self.lno_rel2abs(fname, lineno) + args = [fname, lineno] + args.extend(remain) + set_method(*args) + elif set_type in ('ignore', 'enable', 'disable'): + set_method(*args) + elif set_type in ('up', 'down'): + set_method() + + # Process the next expect_set item. + # It is not expected that a test may reach the recursion limit. + self.event= None + self.pop_next() + if self.dry_run: + self.print_state() + else: + if self.expect: + self.check_lno_name() + self.check_expect_max_size(3) + self.next_set_method() + else: + raise BdbSyntaxError('"%s" is an invalid set_tuple' % + self.set_tuple) + +class TracerRun(): + """Provide a context for running a Tracer instance with a test case.""" + + def __init__(self, test_case, skip=None): + self.test_case = test_case + self.dry_run = test_case.dry_run + self.tracer = Tracer(test_case.expect_set, skip=skip, + dry_run=self.dry_run, test_case=test_case.id()) + + def __enter__(self): + # test_pdb does not reset Breakpoint class attributes on exit :-( + reset_Breakpoint() + return self.tracer + + def __exit__(self, type_=None, value=None, traceback=None): + reset_Breakpoint() + sys.settrace(None) + + not_empty = '' + if self.tracer.set_list: + not_empty += 'All paired tuples have not been processed, ' + not_empty += ('the last one was number %d' % + self.tracer.expect_set_no) + + # Make a BdbNotExpectedError a unittest failure. + if type_ is not None and issubclass(BdbNotExpectedError, type_): + if isinstance(value, BaseException) and value.args: + err_msg = value.args[0] + if not_empty: + err_msg += '\n' + not_empty + if self.dry_run: + print(err_msg) + return True + else: + self.test_case.fail(err_msg) + else: + assert False, 'BdbNotExpectedError with empty args' + + if not_empty: + if self.dry_run: + print(not_empty) + else: + self.test_case.fail(not_empty) + +def run_test(modules, set_list, skip=None): + """Run a test and print the dry-run results. + + 'modules': A dictionary mapping module names to their source code as a + string. The dictionary MUST include one module named + 'test_module' with a main() function. + 'set_list': A list of set_type tuples to be run on the module. + + For example, running the following script outputs the following results: + + ***************************** SCRIPT ******************************** + + from test.test_bdb import run_test, break_in_func + + code = ''' + def func(): + lno = 3 + + def main(): + func() + lno = 7 + ''' + + set_list = [ + break_in_func('func', 'test_module.py'), + ('continue', ), + ('step', ), + ('step', ), + ('step', ), + ('quit', ), + ] + + modules = { 'test_module': code } + run_test(modules, set_list) + + **************************** results ******************************** + + 1: ('line', 2, 'tfunc_import'), ('next',), + 2: ('line', 3, 'tfunc_import'), ('step',), + 3: ('call', 5, 'main'), ('break', ('test_module.py', None, False, None, 'func')), + 4: ('None', 5, 'main'), ('continue',), + 5: ('line', 3, 'func', ({1: 1}, [])), ('step',), + BpNum Temp Enb Hits Ignore Where + 1 no yes 1 0 at test_module.py:2 + 6: ('return', 3, 'func'), ('step',), + 7: ('line', 7, 'main'), ('step',), + 8: ('return', 7, 'main'), ('quit',), + + ************************************************************************* + + """ + def gen(a, b): + try: + while 1: + x = next(a) + y = next(b) + yield x + yield y + except StopIteration: + return + + # Step over the import statement in tfunc_import using 'next' and step + # into main() in test_module. + sl = [('next', ), ('step', )] + sl.extend(set_list) + + test = BaseTestCase() + test.dry_run = True + test.id = lambda : None + test.expect_set = list(gen(repeat(()), iter(sl))) + with create_modules(modules): + sys.path.append(os.getcwd()) + with TracerRun(test, skip=skip) as tracer: + tracer.runcall(tfunc_import) + + at contextmanager +def create_modules(modules): + with test.support.temp_cwd(): + try: + for m in modules: + fname = m + '.py' + with open(fname, 'w') as f: + f.write(textwrap.dedent(modules[m])) + linecache.checkcache(fname) + importlib.invalidate_caches() + yield + finally: + for m in modules: + test.support.forget(m) + +def break_in_func(funcname, fname=__file__, temporary=False, cond=None): + return 'break', (fname, None, temporary, cond, funcname) + +TEST_MODULE = 'test_module' +TEST_MODULE_FNAME = TEST_MODULE + '.py' +def tfunc_import(): + import test_module + test_module.main() + +def tfunc_main(): + lno = 2 + tfunc_first() + tfunc_second() + lno = 5 + lno = 6 + lno = 7 + +def tfunc_first(): + lno = 2 + lno = 3 + lno = 4 + +def tfunc_second(): + lno = 2 + +class BaseTestCase(unittest.TestCase): + """Base class for all tests.""" + + dry_run = dry_run + + def fail(self, msg=None): + # Override fail() to use 'raise from None' to avoid repetition of the + # error message and traceback. + raise self.failureException(msg) from None + +class StateTestCase(BaseTestCase): + """Test the step, next, return, until and quit 'set_' methods.""" + + def test_step(self): + self.expect_set = [ + ('line', 2, 'tfunc_main'), ('step', ), + ('line', 3, 'tfunc_main'), ('step', ), + ('call', 1, 'tfunc_first'), ('step', ), + ('line', 2, 'tfunc_first'), ('quit', ), + ] + with TracerRun(self) as tracer: + tracer.runcall(tfunc_main) + + def test_step_next_on_last_statement(self): + for set_type in ('step', 'next'): + with self.subTest(set_type=set_type): + self.expect_set = [ + ('line', 2, 'tfunc_main'), ('step', ), + ('line', 3, 'tfunc_main'), ('step', ), + ('call', 1, 'tfunc_first'), ('break', (__file__, 3)), + ('None', 1, 'tfunc_first'), ('continue', ), + ('line', 3, 'tfunc_first', ({1:1}, [])), (set_type, ), + ('line', 4, 'tfunc_first'), ('quit', ), + ] + with TracerRun(self) as tracer: + tracer.runcall(tfunc_main) + + def test_next(self): + self.expect_set = [ + ('line', 2, 'tfunc_main'), ('step', ), + ('line', 3, 'tfunc_main'), ('next', ), + ('line', 4, 'tfunc_main'), ('step', ), + ('call', 1, 'tfunc_second'), ('step', ), + ('line', 2, 'tfunc_second'), ('quit', ), + ] + with TracerRun(self) as tracer: + tracer.runcall(tfunc_main) + + def test_next_over_import(self): + code = """ + def main(): + lno = 3 + """ + modules = { TEST_MODULE: code } + with create_modules(modules): + self.expect_set = [ + ('line', 2, 'tfunc_import'), ('next', ), + ('line', 3, 'tfunc_import'), ('quit', ), + ] + with TracerRun(self) as tracer: + tracer.runcall(tfunc_import) + + def test_next_on_plain_statement(self): + # Check that set_next() is equivalent to set_step() on a plain + # statement. + self.expect_set = [ + ('line', 2, 'tfunc_main'), ('step', ), + ('line', 3, 'tfunc_main'), ('step', ), + ('call', 1, 'tfunc_first'), ('next', ), + ('line', 2, 'tfunc_first'), ('quit', ), + ] + with TracerRun(self) as tracer: + tracer.runcall(tfunc_main) + + def test_next_in_caller_frame(self): + # Check that set_next() in the caller frame causes the tracer + # to stop next in the caller frame. + self.expect_set = [ + ('line', 2, 'tfunc_main'), ('step', ), + ('line', 3, 'tfunc_main'), ('step', ), + ('call', 1, 'tfunc_first'), ('up', ), + ('None', 3, 'tfunc_main'), ('next', ), + ('line', 4, 'tfunc_main'), ('quit', ), + ] + with TracerRun(self) as tracer: + tracer.runcall(tfunc_main) + + def test_return(self): + self.expect_set = [ + ('line', 2, 'tfunc_main'), ('step', ), + ('line', 3, 'tfunc_main'), ('step', ), + ('call', 1, 'tfunc_first'), ('step', ), + ('line', 2, 'tfunc_first'), ('return', ), + ('return', 4, 'tfunc_first'), ('step', ), + ('line', 4, 'tfunc_main'), ('quit', ), + ] + with TracerRun(self) as tracer: + tracer.runcall(tfunc_main) + + def test_return_in_caller_frame(self): + self.expect_set = [ + ('line', 2, 'tfunc_main'), ('step', ), + ('line', 3, 'tfunc_main'), ('step', ), + ('call', 1, 'tfunc_first'), ('up', ), + ('None', 3, 'tfunc_main'), ('return', ), + ('return', 7, 'tfunc_main'), ('quit', ), + ] + with TracerRun(self) as tracer: + tracer.runcall(tfunc_main) + + def test_until(self): + self.expect_set = [ + ('line', 2, 'tfunc_main'), ('step', ), + ('line', 3, 'tfunc_main'), ('step', ), + ('call', 1, 'tfunc_first'), ('step', ), + ('line', 2, 'tfunc_first'), ('until', (4, )), + ('line', 4, 'tfunc_first'), ('quit', ), + ] + with TracerRun(self) as tracer: + tracer.runcall(tfunc_main) + + def test_until_with_too_large_count(self): + self.expect_set = [ + ('line', 2, 'tfunc_main'), break_in_func('tfunc_first'), + ('None', 2, 'tfunc_main'), ('continue', ), + ('line', 2, 'tfunc_first', ({1:1}, [])), ('until', (9999, )), + ('return', 4, 'tfunc_first'), ('quit', ), + ] + with TracerRun(self) as tracer: + tracer.runcall(tfunc_main) + + def test_until_in_caller_frame(self): + self.expect_set = [ + ('line', 2, 'tfunc_main'), ('step', ), + ('line', 3, 'tfunc_main'), ('step', ), + ('call', 1, 'tfunc_first'), ('up', ), + ('None', 3, 'tfunc_main'), ('until', (6, )), + ('line', 6, 'tfunc_main'), ('quit', ), + ] + with TracerRun(self) as tracer: + tracer.runcall(tfunc_main) + + def test_skip(self): + # Check that tracing is skipped over the import statement in + # 'tfunc_import()'. + code = """ + def main(): + lno = 3 + """ + modules = { TEST_MODULE: code } + with create_modules(modules): + self.expect_set = [ + ('line', 2, 'tfunc_import'), ('step', ), + ('line', 3, 'tfunc_import'), ('quit', ), + ] + skip = ('importlib*', TEST_MODULE) + with TracerRun(self, skip=skip) as tracer: + tracer.runcall(tfunc_import) + + def test_down(self): + # Check that set_down() raises BdbError at the newest frame. + self.expect_set = [ + ('line', 2, 'tfunc_main'), ('down', ), + ] + with TracerRun(self) as tracer: + self.assertRaises(BdbError, tracer.runcall, tfunc_main) + + def test_up(self): + self.expect_set = [ + ('line', 2, 'tfunc_main'), ('step', ), + ('line', 3, 'tfunc_main'), ('step', ), + ('call', 1, 'tfunc_first'), ('up', ), + ('None', 3, 'tfunc_main'), ('quit', ), + ] + with TracerRun(self) as tracer: + tracer.runcall(tfunc_main) + +class BreakpointTestCase(BaseTestCase): + """Test the breakpoint set method.""" + + def test_bp_on_non_existent_module(self): + self.expect_set = [ + ('line', 2, 'tfunc_import'), ('break', ('/non/existent/module.py', 1)) + ] + with TracerRun(self) as tracer: + self.assertRaises(BdbError, tracer.runcall, tfunc_import) + + def test_bp_after_last_statement(self): + code = """ + def main(): + lno = 3 + """ + modules = { TEST_MODULE: code } + with create_modules(modules): + self.expect_set = [ + ('line', 2, 'tfunc_import'), ('break', (TEST_MODULE_FNAME, 4)) + ] + with TracerRun(self) as tracer: + self.assertRaises(BdbError, tracer.runcall, tfunc_import) + + def test_temporary_bp(self): + code = """ + def func(): + lno = 3 + + def main(): + for i in range(2): + func() + """ + modules = { TEST_MODULE: code } + with create_modules(modules): + self.expect_set = [ + ('line', 2, 'tfunc_import'), + break_in_func('func', TEST_MODULE_FNAME, True), + ('None', 2, 'tfunc_import'), + break_in_func('func', TEST_MODULE_FNAME, True), + ('None', 2, 'tfunc_import'), ('continue', ), + ('line', 3, 'func', ({1:1}, [1])), ('continue', ), + ('line', 3, 'func', ({2:1}, [2])), ('quit', ), + ] + with TracerRun(self) as tracer: + tracer.runcall(tfunc_import) + + def test_disabled_temporary_bp(self): + code = """ + def func(): + lno = 3 + + def main(): + for i in range(3): + func() + """ + modules = { TEST_MODULE: code } + with create_modules(modules): + self.expect_set = [ + ('line', 2, 'tfunc_import'), + break_in_func('func', TEST_MODULE_FNAME), + ('None', 2, 'tfunc_import'), + break_in_func('func', TEST_MODULE_FNAME, True), + ('None', 2, 'tfunc_import'), ('disable', (2, )), + ('None', 2, 'tfunc_import'), ('continue', ), + ('line', 3, 'func', ({1:1}, [])), ('enable', (2, )), + ('None', 3, 'func'), ('disable', (1, )), + ('None', 3, 'func'), ('continue', ), + ('line', 3, 'func', ({2:1}, [2])), ('enable', (1, )), + ('None', 3, 'func'), ('continue', ), + ('line', 3, 'func', ({1:2}, [])), ('quit', ), + ] + with TracerRun(self) as tracer: + tracer.runcall(tfunc_import) + + def test_bp_condition(self): + code = """ + def func(a): + lno = 3 + + def main(): + for i in range(3): + func(i) + """ + modules = { TEST_MODULE: code } + with create_modules(modules): + self.expect_set = [ + ('line', 2, 'tfunc_import'), + break_in_func('func', TEST_MODULE_FNAME, False, 'a == 2'), + ('None', 2, 'tfunc_import'), ('continue', ), + ('line', 3, 'func', ({1:3}, [])), ('quit', ), + ] + with TracerRun(self) as tracer: + tracer.runcall(tfunc_import) + + def test_bp_exception_on_condition_evaluation(self): + code = """ + def func(a): + lno = 3 + + def main(): + func(0) + """ + modules = { TEST_MODULE: code } + with create_modules(modules): + self.expect_set = [ + ('line', 2, 'tfunc_import'), + break_in_func('func', TEST_MODULE_FNAME, False, '1 / 0'), + ('None', 2, 'tfunc_import'), ('continue', ), + ('line', 3, 'func', ({1:1}, [])), ('quit', ), + ] + with TracerRun(self) as tracer: + tracer.runcall(tfunc_import) + + def test_bp_ignore_count(self): + code = """ + def func(): + lno = 3 + + def main(): + for i in range(2): + func() + """ + modules = { TEST_MODULE: code } + with create_modules(modules): + self.expect_set = [ + ('line', 2, 'tfunc_import'), + break_in_func('func', TEST_MODULE_FNAME), + ('None', 2, 'tfunc_import'), ('ignore', (1, )), + ('None', 2, 'tfunc_import'), ('continue', ), + ('line', 3, 'func', ({1:2}, [])), ('quit', ), + ] + with TracerRun(self) as tracer: + tracer.runcall(tfunc_import) + + def test_ignore_count_on_disabled_bp(self): + code = """ + def func(): + lno = 3 + + def main(): + for i in range(3): + func() + """ + modules = { TEST_MODULE: code } + with create_modules(modules): + self.expect_set = [ + ('line', 2, 'tfunc_import'), + break_in_func('func', TEST_MODULE_FNAME), + ('None', 2, 'tfunc_import'), + break_in_func('func', TEST_MODULE_FNAME), + ('None', 2, 'tfunc_import'), ('ignore', (1, )), + ('None', 2, 'tfunc_import'), ('disable', (1, )), + ('None', 2, 'tfunc_import'), ('continue', ), + ('line', 3, 'func', ({2:1}, [])), ('enable', (1, )), + ('None', 3, 'func'), ('continue', ), + ('line', 3, 'func', ({2:2}, [])), ('continue', ), + ('line', 3, 'func', ({1:2}, [])), ('quit', ), + ] + with TracerRun(self) as tracer: + tracer.runcall(tfunc_import) + + def test_clear_two_bp_on_same_line(self): + code = """ + def func(): + lno = 3 + lno = 4 + + def main(): + for i in range(3): + func() + """ + modules = { TEST_MODULE: code } + with create_modules(modules): + self.expect_set = [ + ('line', 2, 'tfunc_import'), ('break', (TEST_MODULE_FNAME, 3)), + ('None', 2, 'tfunc_import'), ('break', (TEST_MODULE_FNAME, 3)), + ('None', 2, 'tfunc_import'), ('break', (TEST_MODULE_FNAME, 4)), + ('None', 2, 'tfunc_import'), ('continue', ), + ('line', 3, 'func', ({1:1}, [])), ('continue', ), + ('line', 4, 'func', ({3:1}, [])), ('clear', (TEST_MODULE_FNAME, 3)), + ('None', 4, 'func'), ('continue', ), + ('line', 4, 'func', ({3:2}, [])), ('quit', ), + ] + with TracerRun(self) as tracer: + tracer.runcall(tfunc_import) + + def test_clear_at_no_bp(self): + self.expect_set = [ + ('line', 2, 'tfunc_import'), ('clear', (__file__, 1)) + ] + with TracerRun(self) as tracer: + self.assertRaises(BdbError, tracer.runcall, tfunc_import) + +class RunTestCase(BaseTestCase): + """Test run, runeval and set_trace.""" + + def test_run_step(self): + # Check that the bdb 'run' method stops at the first line event. + code = """ + lno = 2 + """ + self.expect_set = [ + ('line', 2, '<module>'), ('step', ), + ('return', 2, '<module>'), ('quit', ), + ] + with TracerRun(self) as tracer: + tracer.run(compile(textwrap.dedent(code), '<string>', 'exec')) + + def test_runeval_step(self): + # Test bdb 'runeval'. + code = """ + def main(): + lno = 3 + """ + modules = { TEST_MODULE: code } + with create_modules(modules): + self.expect_set = [ + ('line', 1, '<module>'), ('step', ), + ('call', 2, 'main'), ('step', ), + ('line', 3, 'main'), ('step', ), + ('return', 3, 'main'), ('step', ), + ('return', 1, '<module>'), ('quit', ), + ] + import test_module + with TracerRun(self) as tracer: + tracer.runeval('test_module.main()', globals(), locals()) + +class IssuesTestCase(BaseTestCase): + """Test fixed bdb issues.""" + + def test_step_at_return_with_no_trace_in_caller(self): + # Issue #13183. + # Check that the tracer does step into the caller frame when the + # trace function is not set in that frame. + code_1 = """ + from test_module_2 import func + def main(): + func() + lno = 5 + """ + code_2 = """ + def func(): + lno = 3 + """ + modules = { + TEST_MODULE: code_1, + 'test_module_2': code_2, + } + with create_modules(modules): + self.expect_set = [ + ('line', 2, 'tfunc_import'), + break_in_func('func', 'test_module_2.py'), + ('None', 2, 'tfunc_import'), ('continue', ), + ('line', 3, 'func', ({1:1}, [])), ('step', ), + ('return', 3, 'func'), ('step', ), + ('line', 5, 'main'), ('quit', ), + ] + with TracerRun(self) as tracer: + tracer.runcall(tfunc_import) + + def test_next_until_return_in_generator(self): + # Issue #16596. + # Check that set_next(), set_until() and set_return() do not treat the + # `yield` and `yield from` statements as if they were returns and stop + # instead in the current frame. + code = """ + def test_gen(): + yield 0 + lno = 4 + return 123 + + def main(): + it = test_gen() + next(it) + next(it) + lno = 11 + """ + modules = { TEST_MODULE: code } + for set_type in ('next', 'until', 'return'): + with self.subTest(set_type=set_type): + with create_modules(modules): + self.expect_set = [ + ('line', 2, 'tfunc_import'), + break_in_func('test_gen', TEST_MODULE_FNAME), + ('None', 2, 'tfunc_import'), ('continue', ), + ('line', 3, 'test_gen', ({1:1}, [])), (set_type, ), + ] + + if set_type == 'return': + self.expect_set.extend( + [('exception', 10, 'main', StopIteration), ('step',), + ('return', 10, 'main'), ('quit', ), + ] + ) + else: + self.expect_set.extend( + [('line', 4, 'test_gen'), ('quit', ),] + ) + with TracerRun(self) as tracer: + tracer.runcall(tfunc_import) + + def test_next_command_in_generator_for_loop(self): + # Issue #16596. + code = """ + def test_gen(): + yield 0 + lno = 4 + yield 1 + return 123 + + def main(): + for i in test_gen(): + lno = 10 + lno = 11 + """ + modules = { TEST_MODULE: code } + with create_modules(modules): + self.expect_set = [ + ('line', 2, 'tfunc_import'), + break_in_func('test_gen', TEST_MODULE_FNAME), + ('None', 2, 'tfunc_import'), ('continue', ), + ('line', 3, 'test_gen', ({1:1}, [])), ('next', ), + ('line', 4, 'test_gen'), ('next', ), + ('line', 5, 'test_gen'), ('next', ), + ('line', 6, 'test_gen'), ('next', ), + ('exception', 9, 'main', StopIteration), ('step', ), + ('line', 11, 'main'), ('quit', ), + + ] + with TracerRun(self) as tracer: + tracer.runcall(tfunc_import) + + def test_next_command_in_generator_with_subiterator(self): + # Issue #16596. + code = """ + def test_subgen(): + yield 0 + return 123 + + def test_gen(): + x = yield from test_subgen() + return 456 + + def main(): + for i in test_gen(): + lno = 12 + lno = 13 + """ + modules = { TEST_MODULE: code } + with create_modules(modules): + self.expect_set = [ + ('line', 2, 'tfunc_import'), + break_in_func('test_gen', TEST_MODULE_FNAME), + ('None', 2, 'tfunc_import'), ('continue', ), + ('line', 7, 'test_gen', ({1:1}, [])), ('next', ), + ('line', 8, 'test_gen'), ('next', ), + ('exception', 11, 'main', StopIteration), ('step', ), + ('line', 13, 'main'), ('quit', ), + + ] + with TracerRun(self) as tracer: + tracer.runcall(tfunc_import) + + def test_return_command_in_generator_with_subiterator(self): + # Issue #16596. + code = """ + def test_subgen(): + yield 0 + return 123 + + def test_gen(): + x = yield from test_subgen() + return 456 + + def main(): + for i in test_gen(): + lno = 12 + lno = 13 + """ + modules = { TEST_MODULE: code } + with create_modules(modules): + self.expect_set = [ + ('line', 2, 'tfunc_import'), + break_in_func('test_subgen', TEST_MODULE_FNAME), + ('None', 2, 'tfunc_import'), ('continue', ), + ('line', 3, 'test_subgen', ({1:1}, [])), ('return', ), + ('exception', 7, 'test_gen', StopIteration), ('return', ), + ('exception', 11, 'main', StopIteration), ('step', ), + ('line', 13, 'main'), ('quit', ), + + ] + with TracerRun(self) as tracer: + tracer.runcall(tfunc_import) + +def test_main(): + test.support.run_unittest( + StateTestCase, + RunTestCase, + BreakpointTestCase, + IssuesTestCase, + ) + +if __name__ == "__main__": + test_main() diff --git a/Lib/test/test_sundry.py b/Lib/test/test_sundry.py index dddf3265525a..4025c2354a6c 100644 --- a/Lib/test/test_sundry.py +++ b/Lib/test/test_sundry.py @@ -6,7 +6,7 @@ class TestUntestedModules(unittest.TestCase): def test_untested_modules_can_be_imported(self): - untested = ('bdb', 'encodings', 'formatter', 'tabnanny') + untested = ('encodings', 'formatter', 'tabnanny') with support.check_warnings(quiet=True): for name in untested: try: diff --git a/Misc/NEWS.d/next/Tests/2018-01-08-13-33-47.bpo-19417.2asoXy.rst b/Misc/NEWS.d/next/Tests/2018-01-08-13-33-47.bpo-19417.2asoXy.rst new file mode 100644 index 000000000000..739352fcdd67 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2018-01-08-13-33-47.bpo-19417.2asoXy.rst @@ -0,0 +1 @@ +Add test_bdb.py. From webhook-mailer at python.org Sun Mar 18 16:50:44 2018 From: webhook-mailer at python.org (Miss Islington (bot)) Date: Sun, 18 Mar 2018 20:50:44 -0000 Subject: [Python-checkins] bpo-32056: Improve exceptions in aifc, wave and sunau. (GH-5951) Message-ID: <mailman.97.1521406245.1871.python-checkins@python.org> https://github.com/python/cpython/commit/3c0a5a7c7ba8fbbc95dd1fe76cd7a1c0ce167371 commit: 3c0a5a7c7ba8fbbc95dd1fe76cd7a1c0ce167371 branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: GitHub <noreply at github.com> date: 2018-03-18T13:50:41-07:00 summary: bpo-32056: Improve exceptions in aifc, wave and sunau. (GH-5951) (cherry picked from commit 134cb01cda50f02725575808130b05d2d776693f) Co-authored-by: Serhiy Storchaka <storchaka at gmail.com> files: A Misc/NEWS.d/next/Library/2018-03-01-17-49-56.bpo-32056.IlpfgE.rst M Lib/aifc.py M Lib/sunau.py M Lib/test/test_aifc.py M Lib/test/test_sunau.py M Lib/test/test_wave.py M Lib/wave.py diff --git a/Lib/aifc.py b/Lib/aifc.py index 3d2dc56de198..1916e7ef8e7e 100644 --- a/Lib/aifc.py +++ b/Lib/aifc.py @@ -467,6 +467,10 @@ def _read_comm_chunk(self, chunk): self._nframes = _read_long(chunk) self._sampwidth = (_read_short(chunk) + 7) // 8 self._framerate = int(_read_float(chunk)) + if self._sampwidth <= 0: + raise Error('bad sample width') + if self._nchannels <= 0: + raise Error('bad # of channels') self._framesize = self._nchannels * self._sampwidth if self._aifc: #DEBUG: SGI's soundeditor produces a bad size :-( diff --git a/Lib/sunau.py b/Lib/sunau.py index dbad3db8392d..129502b0b417 100644 --- a/Lib/sunau.py +++ b/Lib/sunau.py @@ -208,6 +208,8 @@ def initfp(self, file): raise Error('unknown encoding') self._framerate = int(_read_u32(file)) self._nchannels = int(_read_u32(file)) + if not self._nchannels: + raise Error('bad # of channels') self._framesize = self._framesize * self._nchannels if self._hdr_size > 24: self._info = file.read(self._hdr_size - 24) diff --git a/Lib/test/test_aifc.py b/Lib/test/test_aifc.py index 8fd306a36592..ff52f5b6feb8 100644 --- a/Lib/test/test_aifc.py +++ b/Lib/test/test_aifc.py @@ -268,7 +268,8 @@ def test_read_no_comm_chunk(self): def test_read_no_ssnd_chunk(self): b = b'FORM' + struct.pack('>L', 4) + b'AIFC' - b += b'COMM' + struct.pack('>LhlhhLL', 38, 0, 0, 0, 0, 0, 0) + b += b'COMM' + struct.pack('>LhlhhLL', 38, 1, 0, 8, + 0x4000 | 12, 11025<<18, 0) b += b'NONE' + struct.pack('B', 14) + b'not compressed' + b'\x00' with self.assertRaisesRegex(aifc.Error, 'COMM chunk and/or SSND chunk' ' missing'): @@ -276,13 +277,35 @@ def test_read_no_ssnd_chunk(self): def test_read_wrong_compression_type(self): b = b'FORM' + struct.pack('>L', 4) + b'AIFC' - b += b'COMM' + struct.pack('>LhlhhLL', 23, 0, 0, 0, 0, 0, 0) + b += b'COMM' + struct.pack('>LhlhhLL', 23, 1, 0, 8, + 0x4000 | 12, 11025<<18, 0) b += b'WRNG' + struct.pack('B', 0) self.assertRaises(aifc.Error, aifc.open, io.BytesIO(b)) + def test_read_wrong_number_of_channels(self): + for nchannels in 0, -1: + b = b'FORM' + struct.pack('>L', 4) + b'AIFC' + b += b'COMM' + struct.pack('>LhlhhLL', 38, nchannels, 0, 8, + 0x4000 | 12, 11025<<18, 0) + b += b'NONE' + struct.pack('B', 14) + b'not compressed' + b'\x00' + b += b'SSND' + struct.pack('>L', 8) + b'\x00' * 8 + with self.assertRaisesRegex(aifc.Error, 'bad # of channels'): + aifc.open(io.BytesIO(b)) + + def test_read_wrong_sample_width(self): + for sampwidth in 0, -1: + b = b'FORM' + struct.pack('>L', 4) + b'AIFC' + b += b'COMM' + struct.pack('>LhlhhLL', 38, 1, 0, sampwidth, + 0x4000 | 12, 11025<<18, 0) + b += b'NONE' + struct.pack('B', 14) + b'not compressed' + b'\x00' + b += b'SSND' + struct.pack('>L', 8) + b'\x00' * 8 + with self.assertRaisesRegex(aifc.Error, 'bad sample width'): + aifc.open(io.BytesIO(b)) + def test_read_wrong_marks(self): b = b'FORM' + struct.pack('>L', 4) + b'AIFF' - b += b'COMM' + struct.pack('>LhlhhLL', 18, 0, 0, 0, 0, 0, 0) + b += b'COMM' + struct.pack('>LhlhhLL', 18, 1, 0, 8, + 0x4000 | 12, 11025<<18, 0) b += b'SSND' + struct.pack('>L', 8) + b'\x00' * 8 b += b'MARK' + struct.pack('>LhB', 3, 1, 1) with self.assertWarns(UserWarning) as cm: @@ -293,7 +316,8 @@ def test_read_wrong_marks(self): def test_read_comm_kludge_compname_even(self): b = b'FORM' + struct.pack('>L', 4) + b'AIFC' - b += b'COMM' + struct.pack('>LhlhhLL', 18, 0, 0, 0, 0, 0, 0) + b += b'COMM' + struct.pack('>LhlhhLL', 18, 1, 0, 8, + 0x4000 | 12, 11025<<18, 0) b += b'NONE' + struct.pack('B', 4) + b'even' + b'\x00' b += b'SSND' + struct.pack('>L', 8) + b'\x00' * 8 with self.assertWarns(UserWarning) as cm: @@ -303,7 +327,8 @@ def test_read_comm_kludge_compname_even(self): def test_read_comm_kludge_compname_odd(self): b = b'FORM' + struct.pack('>L', 4) + b'AIFC' - b += b'COMM' + struct.pack('>LhlhhLL', 18, 0, 0, 0, 0, 0, 0) + b += b'COMM' + struct.pack('>LhlhhLL', 18, 1, 0, 8, + 0x4000 | 12, 11025<<18, 0) b += b'NONE' + struct.pack('B', 3) + b'odd' b += b'SSND' + struct.pack('>L', 8) + b'\x00' * 8 with self.assertWarns(UserWarning) as cm: diff --git a/Lib/test/test_sunau.py b/Lib/test/test_sunau.py index 966224b1df5a..470a1007b4d4 100644 --- a/Lib/test/test_sunau.py +++ b/Lib/test/test_sunau.py @@ -1,6 +1,8 @@ import unittest from test import audiotests from audioop import byteswap +import io +import struct import sys import sunau @@ -121,5 +123,40 @@ class SunauMiscTests(audiotests.AudioMiscTests, unittest.TestCase): module = sunau +class SunauLowLevelTest(unittest.TestCase): + + def test_read_bad_magic_number(self): + b = b'SPA' + with self.assertRaises(EOFError): + sunau.open(io.BytesIO(b)) + b = b'SPAM' + with self.assertRaisesRegex(sunau.Error, 'bad magic number'): + sunau.open(io.BytesIO(b)) + + def test_read_too_small_header(self): + b = struct.pack('>LLLLL', sunau.AUDIO_FILE_MAGIC, 20, 0, + sunau.AUDIO_FILE_ENCODING_LINEAR_8, 11025) + with self.assertRaisesRegex(sunau.Error, 'header size too small'): + sunau.open(io.BytesIO(b)) + + def test_read_too_large_header(self): + b = struct.pack('>LLLLLL', sunau.AUDIO_FILE_MAGIC, 124, 0, + sunau.AUDIO_FILE_ENCODING_LINEAR_8, 11025, 1) + b += b'\0' * 100 + with self.assertRaisesRegex(sunau.Error, 'header size ridiculously large'): + sunau.open(io.BytesIO(b)) + + def test_read_wrong_encoding(self): + b = struct.pack('>LLLLLL', sunau.AUDIO_FILE_MAGIC, 24, 0, 0, 11025, 1) + with self.assertRaisesRegex(sunau.Error, r'encoding not \(yet\) supported'): + sunau.open(io.BytesIO(b)) + + def test_read_wrong_number_of_channels(self): + b = struct.pack('>LLLLLL', sunau.AUDIO_FILE_MAGIC, 24, 0, + sunau.AUDIO_FILE_ENCODING_LINEAR_8, 11025, 0) + with self.assertRaisesRegex(sunau.Error, 'bad # of channels'): + sunau.open(io.BytesIO(b)) + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_wave.py b/Lib/test/test_wave.py index c5d2e02450ef..8a42f8e47105 100644 --- a/Lib/test/test_wave.py +++ b/Lib/test/test_wave.py @@ -2,6 +2,8 @@ from test import audiotests from test import support from audioop import byteswap +import io +import struct import sys import wave @@ -111,5 +113,65 @@ def test__all__(self): support.check__all__(self, wave, blacklist=blacklist) +class WaveLowLevelTest(unittest.TestCase): + + def test_read_no_chunks(self): + b = b'SPAM' + with self.assertRaises(EOFError): + wave.open(io.BytesIO(b)) + + def test_read_no_riff_chunk(self): + b = b'SPAM' + struct.pack('<L', 0) + with self.assertRaisesRegex(wave.Error, + 'file does not start with RIFF id'): + wave.open(io.BytesIO(b)) + + def test_read_not_wave(self): + b = b'RIFF' + struct.pack('<L', 4) + b'SPAM' + with self.assertRaisesRegex(wave.Error, + 'not a WAVE file'): + wave.open(io.BytesIO(b)) + + def test_read_no_fmt_no_data_chunk(self): + b = b'RIFF' + struct.pack('<L', 4) + b'WAVE' + with self.assertRaisesRegex(wave.Error, + 'fmt chunk and/or data chunk missing'): + wave.open(io.BytesIO(b)) + + def test_read_no_data_chunk(self): + b = b'RIFF' + struct.pack('<L', 28) + b'WAVE' + b += b'fmt ' + struct.pack('<LHHLLHH', 16, 1, 1, 11025, 11025, 1, 8) + with self.assertRaisesRegex(wave.Error, + 'fmt chunk and/or data chunk missing'): + wave.open(io.BytesIO(b)) + + def test_read_no_fmt_chunk(self): + b = b'RIFF' + struct.pack('<L', 12) + b'WAVE' + b += b'data' + struct.pack('<L', 0) + with self.assertRaisesRegex(wave.Error, 'data chunk before fmt chunk'): + wave.open(io.BytesIO(b)) + + def test_read_wrong_form(self): + b = b'RIFF' + struct.pack('<L', 36) + b'WAVE' + b += b'fmt ' + struct.pack('<LHHLLHH', 16, 2, 1, 11025, 11025, 1, 1) + b += b'data' + struct.pack('<L', 0) + with self.assertRaisesRegex(wave.Error, 'unknown format: 2'): + wave.open(io.BytesIO(b)) + + def test_read_wrong_number_of_channels(self): + b = b'RIFF' + struct.pack('<L', 36) + b'WAVE' + b += b'fmt ' + struct.pack('<LHHLLHH', 16, 1, 0, 11025, 11025, 1, 8) + b += b'data' + struct.pack('<L', 0) + with self.assertRaisesRegex(wave.Error, 'bad # of channels'): + wave.open(io.BytesIO(b)) + + def test_read_wrong_sample_width(self): + b = b'RIFF' + struct.pack('<L', 36) + b'WAVE' + b += b'fmt ' + struct.pack('<LHHLLHH', 16, 1, 1, 11025, 11025, 1, 0) + b += b'data' + struct.pack('<L', 0) + with self.assertRaisesRegex(wave.Error, 'bad sample width'): + wave.open(io.BytesIO(b)) + + if __name__ == '__main__': unittest.main() diff --git a/Lib/wave.py b/Lib/wave.py index cf94d5af72b4..f155879a9a76 100644 --- a/Lib/wave.py +++ b/Lib/wave.py @@ -253,12 +253,22 @@ def readframes(self, nframes): # def _read_fmt_chunk(self, chunk): - wFormatTag, self._nchannels, self._framerate, dwAvgBytesPerSec, wBlockAlign = struct.unpack_from('<HHLLH', chunk.read(14)) + try: + wFormatTag, self._nchannels, self._framerate, dwAvgBytesPerSec, wBlockAlign = struct.unpack_from('<HHLLH', chunk.read(14)) + except struct.error: + raise EOFError from None if wFormatTag == WAVE_FORMAT_PCM: - sampwidth = struct.unpack_from('<H', chunk.read(2))[0] + try: + sampwidth = struct.unpack_from('<H', chunk.read(2))[0] + except struct.error: + raise EOFError from None self._sampwidth = (sampwidth + 7) // 8 + if not self._sampwidth: + raise Error('bad sample width') else: raise Error('unknown format: %r' % (wFormatTag,)) + if not self._nchannels: + raise Error('bad # of channels') self._framesize = self._nchannels * self._sampwidth self._comptype = 'NONE' self._compname = 'not compressed' diff --git a/Misc/NEWS.d/next/Library/2018-03-01-17-49-56.bpo-32056.IlpfgE.rst b/Misc/NEWS.d/next/Library/2018-03-01-17-49-56.bpo-32056.IlpfgE.rst new file mode 100644 index 000000000000..421aa3767794 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-03-01-17-49-56.bpo-32056.IlpfgE.rst @@ -0,0 +1,3 @@ +Improved exceptions raised for invalid number of channels and sample width +when read an audio file in modules :mod:`aifc`, :mod:`wave` and +:mod:`sunau`. From webhook-mailer at python.org Sun Mar 18 20:40:38 2018 From: webhook-mailer at python.org (Eric V. Smith) Date: Mon, 19 Mar 2018 00:40:38 -0000 Subject: [Python-checkins] bpo-32953: Dataclasses: frozen should not be inherited for non-dataclass derived classes (#6147) Message-ID: <mailman.98.1521420039.1871.python-checkins@python.org> https://github.com/python/cpython/commit/f199bc655eb50c28e94010714629b376bbbd077b commit: f199bc655eb50c28e94010714629b376bbbd077b branch: master author: Eric V. Smith <ericvsmith at users.noreply.github.com> committer: GitHub <noreply at github.com> date: 2018-03-18T20:40:34-04:00 summary: bpo-32953: Dataclasses: frozen should not be inherited for non-dataclass derived classes (#6147) If a non-dataclass derives from a frozen dataclass, allow attributes to be set. Require either all of the dataclasses in a class hierarchy to be frozen, or all non-frozen. Store `@dataclass` parameters on the class object under `__dataclass_params__`. This is needed to detect frozen base classes. files: A Misc/NEWS.d/next/Library/2018-03-18-17-38-48.bpo-32953.t8WAWN.rst M Lib/dataclasses.py M Lib/test/test_dataclasses.py diff --git a/Lib/dataclasses.py b/Lib/dataclasses.py index b55a497db302..8ab04dd5b975 100644 --- a/Lib/dataclasses.py +++ b/Lib/dataclasses.py @@ -171,7 +171,11 @@ class _MISSING_TYPE: # The name of an attribute on the class where we store the Field # objects. Also used to check if a class is a Data Class. -_MARKER = '__dataclass_fields__' +_FIELDS = '__dataclass_fields__' + +# The name of an attribute on the class that stores the parameters to +# @dataclass. +_PARAMS = '__dataclass_params__' # The name of the function, that if it exists, is called at the end of # __init__. @@ -192,7 +196,7 @@ class InitVar(metaclass=_InitVarMeta): # name and type are filled in after the fact, not in __init__. They're # not known at the time this class is instantiated, but it's # convenient if they're available later. -# When cls._MARKER is filled in with a list of Field objects, the name +# When cls._FIELDS is filled in with a list of Field objects, the name # and type fields will have been populated. class Field: __slots__ = ('name', @@ -236,6 +240,32 @@ def __repr__(self): ')') +class _DataclassParams: + __slots__ = ('init', + 'repr', + 'eq', + 'order', + 'unsafe_hash', + 'frozen', + ) + def __init__(self, init, repr, eq, order, unsafe_hash, frozen): + self.init = init + self.repr = repr + self.eq = eq + self.order = order + self.unsafe_hash = unsafe_hash + self.frozen = frozen + + def __repr__(self): + return ('_DataclassParams(' + f'init={self.init},' + f'repr={self.repr},' + f'eq={self.eq},' + f'order={self.order},' + f'unsafe_hash={self.unsafe_hash},' + f'frozen={self.frozen}' + ')') + # This function is used instead of exposing Field creation directly, # so that a type checker can be told (via overloads) that this is a # function whose type depends on its parameters. @@ -285,6 +315,7 @@ def _create_fn(name, args, body, *, globals=None, locals=None, args = ','.join(args) body = '\n'.join(f' {b}' for b in body) + # Compute the text of the entire function. txt = f'def {name}({args}){return_annotation}:\n{body}' exec(txt, globals, locals) @@ -432,12 +463,29 @@ def _repr_fn(fields): ')"']) -def _frozen_setattr(self, name, value): - raise FrozenInstanceError(f'cannot assign to field {name!r}') - - -def _frozen_delattr(self, name): - raise FrozenInstanceError(f'cannot delete field {name!r}') +def _frozen_get_del_attr(cls, fields): + # XXX: globals is modified on the first call to _create_fn, then the + # modified version is used in the second call. Is this okay? + globals = {'cls': cls, + 'FrozenInstanceError': FrozenInstanceError} + if fields: + fields_str = '(' + ','.join(repr(f.name) for f in fields) + ',)' + else: + # Special case for the zero-length tuple. + fields_str = '()' + return (_create_fn('__setattr__', + ('self', 'name', 'value'), + (f'if type(self) is cls or name in {fields_str}:', + ' raise FrozenInstanceError(f"cannot assign to field {name!r}")', + f'super(cls, self).__setattr__(name, value)'), + globals=globals), + _create_fn('__delattr__', + ('self', 'name'), + (f'if type(self) is cls or name in {fields_str}:', + ' raise FrozenInstanceError(f"cannot delete field {name!r}")', + f'super(cls, self).__delattr__(name)'), + globals=globals), + ) def _cmp_fn(name, op, self_tuple, other_tuple): @@ -583,23 +631,32 @@ def _set_new_attribute(cls, name, value): # version of this table. -def _process_class(cls, repr, eq, order, unsafe_hash, init, frozen): +def _process_class(cls, init, repr, eq, order, unsafe_hash, frozen): # Now that dicts retain insertion order, there's no reason to use # an ordered dict. I am leveraging that ordering here, because # derived class fields overwrite base class fields, but the order # is defined by the base class, which is found first. fields = {} + setattr(cls, _PARAMS, _DataclassParams(init, repr, eq, order, + unsafe_hash, frozen)) + # Find our base classes in reverse MRO order, and exclude # ourselves. In reversed order so that more derived classes # override earlier field definitions in base classes. + # As long as we're iterating over them, see if any are frozen. + any_frozen_base = False + has_dataclass_bases = False for b in cls.__mro__[-1:0:-1]: # Only process classes that have been processed by our - # decorator. That is, they have a _MARKER attribute. - base_fields = getattr(b, _MARKER, None) + # decorator. That is, they have a _FIELDS attribute. + base_fields = getattr(b, _FIELDS, None) if base_fields: + has_dataclass_bases = True for f in base_fields.values(): fields[f.name] = f + if getattr(b, _PARAMS).frozen: + any_frozen_base = True # Now find fields in our class. While doing so, validate some # things, and set the default values (as class attributes) @@ -623,20 +680,21 @@ def _process_class(cls, repr, eq, order, unsafe_hash, init, frozen): else: setattr(cls, f.name, f.default) - # We're inheriting from a frozen dataclass, but we're not frozen. - if cls.__setattr__ is _frozen_setattr and not frozen: - raise TypeError('cannot inherit non-frozen dataclass from a ' - 'frozen one') + # Check rules that apply if we are derived from any dataclasses. + if has_dataclass_bases: + # Raise an exception if any of our bases are frozen, but we're not. + if any_frozen_base and not frozen: + raise TypeError('cannot inherit non-frozen dataclass from a ' + 'frozen one') - # We're inheriting from a non-frozen dataclass, but we're frozen. - if (hasattr(cls, _MARKER) and cls.__setattr__ is not _frozen_setattr - and frozen): - raise TypeError('cannot inherit frozen dataclass from a ' - 'non-frozen one') + # Raise an exception if we're frozen, but none of our bases are. + if not any_frozen_base and frozen: + raise TypeError('cannot inherit frozen dataclass from a ' + 'non-frozen one') - # Remember all of the fields on our class (including bases). This + # Remember all of the fields on our class (including bases). This also # marks this class as being a dataclass. - setattr(cls, _MARKER, fields) + setattr(cls, _FIELDS, fields) # Was this class defined with an explicit __hash__? Note that if # __eq__ is defined in this class, then python will automatically @@ -704,10 +762,10 @@ def _process_class(cls, repr, eq, order, unsafe_hash, init, frozen): 'functools.total_ordering') if frozen: - for name, fn in [('__setattr__', _frozen_setattr), - ('__delattr__', _frozen_delattr)]: - if _set_new_attribute(cls, name, fn): - raise TypeError(f'Cannot overwrite attribute {name} ' + # XXX: Which fields are frozen? InitVar? ClassVar? hashed-only? + for fn in _frozen_get_del_attr(cls, field_list): + if _set_new_attribute(cls, fn.__name__, fn): + raise TypeError(f'Cannot overwrite attribute {fn.__name__} ' f'in class {cls.__name__}') # Decide if/how we're going to create a hash function. @@ -759,7 +817,7 @@ def dataclass(_cls=None, *, init=True, repr=True, eq=True, order=False, """ def wrap(cls): - return _process_class(cls, repr, eq, order, unsafe_hash, init, frozen) + return _process_class(cls, init, repr, eq, order, unsafe_hash, frozen) # See if we're being called as @dataclass or @dataclass(). if _cls is None: @@ -779,7 +837,7 @@ def fields(class_or_instance): # Might it be worth caching this, per class? try: - fields = getattr(class_or_instance, _MARKER) + fields = getattr(class_or_instance, _FIELDS) except AttributeError: raise TypeError('must be called with a dataclass type or instance') @@ -790,13 +848,13 @@ def fields(class_or_instance): def _is_dataclass_instance(obj): """Returns True if obj is an instance of a dataclass.""" - return not isinstance(obj, type) and hasattr(obj, _MARKER) + return not isinstance(obj, type) and hasattr(obj, _FIELDS) def is_dataclass(obj): """Returns True if obj is a dataclass or an instance of a dataclass.""" - return hasattr(obj, _MARKER) + return hasattr(obj, _FIELDS) def asdict(obj, *, dict_factory=dict): @@ -953,7 +1011,7 @@ class C: # It's an error to have init=False fields in 'changes'. # If a field is not in 'changes', read its value from the provided obj. - for f in getattr(obj, _MARKER).values(): + for f in getattr(obj, _FIELDS).values(): if not f.init: # Error if this field is specified in changes. if f.name in changes: diff --git a/Lib/test/test_dataclasses.py b/Lib/test/test_dataclasses.py index 46d485c0157b..3e6726360940 100755 --- a/Lib/test/test_dataclasses.py +++ b/Lib/test/test_dataclasses.py @@ -2476,41 +2476,92 @@ class D(C): d = D(0, 10) with self.assertRaises(FrozenInstanceError): d.i = 5 + with self.assertRaises(FrozenInstanceError): + d.j = 6 self.assertEqual(d.i, 0) + self.assertEqual(d.j, 10) + + # Test both ways: with an intermediate normal (non-dataclass) + # class and without an intermediate class. + def test_inherit_nonfrozen_from_frozen(self): + for intermediate_class in [True, False]: + with self.subTest(intermediate_class=intermediate_class): + @dataclass(frozen=True) + class C: + i: int - def test_inherit_from_nonfrozen_from_frozen(self): - @dataclass(frozen=True) - class C: - i: int + if intermediate_class: + class I(C): pass + else: + I = C - with self.assertRaisesRegex(TypeError, - 'cannot inherit non-frozen dataclass from a frozen one'): - @dataclass - class D(C): - pass + with self.assertRaisesRegex(TypeError, + 'cannot inherit non-frozen dataclass from a frozen one'): + @dataclass + class D(I): + pass - def test_inherit_from_frozen_from_nonfrozen(self): - @dataclass - class C: - i: int + def test_inherit_frozen_from_nonfrozen(self): + for intermediate_class in [True, False]: + with self.subTest(intermediate_class=intermediate_class): + @dataclass + class C: + i: int - with self.assertRaisesRegex(TypeError, - 'cannot inherit frozen dataclass from a non-frozen one'): - @dataclass(frozen=True) - class D(C): - pass + if intermediate_class: + class I(C): pass + else: + I = C + + with self.assertRaisesRegex(TypeError, + 'cannot inherit frozen dataclass from a non-frozen one'): + @dataclass(frozen=True) + class D(I): + pass def test_inherit_from_normal_class(self): - class C: - pass + for intermediate_class in [True, False]: + with self.subTest(intermediate_class=intermediate_class): + class C: + pass + + if intermediate_class: + class I(C): pass + else: + I = C + + @dataclass(frozen=True) + class D(I): + i: int + + d = D(10) + with self.assertRaises(FrozenInstanceError): + d.i = 5 + + def test_non_frozen_normal_derived(self): + # See bpo-32953. @dataclass(frozen=True) - class D(C): - i: int + class D: + x: int + y: int = 10 - d = D(10) + class S(D): + pass + + s = S(3) + self.assertEqual(s.x, 3) + self.assertEqual(s.y, 10) + s.cached = True + + # But can't change the frozen attributes. with self.assertRaises(FrozenInstanceError): - d.i = 5 + s.x = 5 + with self.assertRaises(FrozenInstanceError): + s.y = 5 + self.assertEqual(s.x, 3) + self.assertEqual(s.y, 10) + self.assertEqual(s.cached, True) if __name__ == '__main__': diff --git a/Misc/NEWS.d/next/Library/2018-03-18-17-38-48.bpo-32953.t8WAWN.rst b/Misc/NEWS.d/next/Library/2018-03-18-17-38-48.bpo-32953.t8WAWN.rst new file mode 100644 index 000000000000..fbea34aa9a2a --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-03-18-17-38-48.bpo-32953.t8WAWN.rst @@ -0,0 +1,4 @@ +If a non-dataclass inherits from a frozen dataclass, allow attributes to be +added to the derived class. Only attributes from from the frozen dataclass +cannot be assigned to. Require all dataclasses in a hierarchy to be either +all frozen or all non-frozen. From webhook-mailer at python.org Sun Mar 18 21:03:39 2018 From: webhook-mailer at python.org (Eric V. Smith) Date: Mon, 19 Mar 2018 01:03:39 -0000 Subject: [Python-checkins] bpo-32953: Dataclasses: frozen should not be inherited for non-dataclass derived classes (GH-6147) (GH-6148) Message-ID: <mailman.99.1521421421.1871.python-checkins@python.org> https://github.com/python/cpython/commit/45648312e540cda3b10109b6a808cbf6955c84eb commit: 45648312e540cda3b10109b6a808cbf6955c84eb branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Eric V. Smith <ericvsmith at users.noreply.github.com> date: 2018-03-18T21:03:36-04:00 summary: bpo-32953: Dataclasses: frozen should not be inherited for non-dataclass derived classes (GH-6147) (GH-6148) If a non-dataclass derives from a frozen dataclass, allow attributes to be set. Require either all of the dataclasses in a class hierarchy to be frozen, or all non-frozen. Store `@dataclass` parameters on the class object under `__dataclass_params__`. This is needed to detect frozen base classes. (cherry picked from commit f199bc655eb50c28e94010714629b376bbbd077b) Co-authored-by: Eric V. Smith <ericvsmith at users.noreply.github.com> files: A Misc/NEWS.d/next/Library/2018-03-18-17-38-48.bpo-32953.t8WAWN.rst M Lib/dataclasses.py M Lib/test/test_dataclasses.py diff --git a/Lib/dataclasses.py b/Lib/dataclasses.py index b55a497db302..8ab04dd5b975 100644 --- a/Lib/dataclasses.py +++ b/Lib/dataclasses.py @@ -171,7 +171,11 @@ class _MISSING_TYPE: # The name of an attribute on the class where we store the Field # objects. Also used to check if a class is a Data Class. -_MARKER = '__dataclass_fields__' +_FIELDS = '__dataclass_fields__' + +# The name of an attribute on the class that stores the parameters to +# @dataclass. +_PARAMS = '__dataclass_params__' # The name of the function, that if it exists, is called at the end of # __init__. @@ -192,7 +196,7 @@ class InitVar(metaclass=_InitVarMeta): # name and type are filled in after the fact, not in __init__. They're # not known at the time this class is instantiated, but it's # convenient if they're available later. -# When cls._MARKER is filled in with a list of Field objects, the name +# When cls._FIELDS is filled in with a list of Field objects, the name # and type fields will have been populated. class Field: __slots__ = ('name', @@ -236,6 +240,32 @@ def __repr__(self): ')') +class _DataclassParams: + __slots__ = ('init', + 'repr', + 'eq', + 'order', + 'unsafe_hash', + 'frozen', + ) + def __init__(self, init, repr, eq, order, unsafe_hash, frozen): + self.init = init + self.repr = repr + self.eq = eq + self.order = order + self.unsafe_hash = unsafe_hash + self.frozen = frozen + + def __repr__(self): + return ('_DataclassParams(' + f'init={self.init},' + f'repr={self.repr},' + f'eq={self.eq},' + f'order={self.order},' + f'unsafe_hash={self.unsafe_hash},' + f'frozen={self.frozen}' + ')') + # This function is used instead of exposing Field creation directly, # so that a type checker can be told (via overloads) that this is a # function whose type depends on its parameters. @@ -285,6 +315,7 @@ def _create_fn(name, args, body, *, globals=None, locals=None, args = ','.join(args) body = '\n'.join(f' {b}' for b in body) + # Compute the text of the entire function. txt = f'def {name}({args}){return_annotation}:\n{body}' exec(txt, globals, locals) @@ -432,12 +463,29 @@ def _repr_fn(fields): ')"']) -def _frozen_setattr(self, name, value): - raise FrozenInstanceError(f'cannot assign to field {name!r}') - - -def _frozen_delattr(self, name): - raise FrozenInstanceError(f'cannot delete field {name!r}') +def _frozen_get_del_attr(cls, fields): + # XXX: globals is modified on the first call to _create_fn, then the + # modified version is used in the second call. Is this okay? + globals = {'cls': cls, + 'FrozenInstanceError': FrozenInstanceError} + if fields: + fields_str = '(' + ','.join(repr(f.name) for f in fields) + ',)' + else: + # Special case for the zero-length tuple. + fields_str = '()' + return (_create_fn('__setattr__', + ('self', 'name', 'value'), + (f'if type(self) is cls or name in {fields_str}:', + ' raise FrozenInstanceError(f"cannot assign to field {name!r}")', + f'super(cls, self).__setattr__(name, value)'), + globals=globals), + _create_fn('__delattr__', + ('self', 'name'), + (f'if type(self) is cls or name in {fields_str}:', + ' raise FrozenInstanceError(f"cannot delete field {name!r}")', + f'super(cls, self).__delattr__(name)'), + globals=globals), + ) def _cmp_fn(name, op, self_tuple, other_tuple): @@ -583,23 +631,32 @@ def _set_new_attribute(cls, name, value): # version of this table. -def _process_class(cls, repr, eq, order, unsafe_hash, init, frozen): +def _process_class(cls, init, repr, eq, order, unsafe_hash, frozen): # Now that dicts retain insertion order, there's no reason to use # an ordered dict. I am leveraging that ordering here, because # derived class fields overwrite base class fields, but the order # is defined by the base class, which is found first. fields = {} + setattr(cls, _PARAMS, _DataclassParams(init, repr, eq, order, + unsafe_hash, frozen)) + # Find our base classes in reverse MRO order, and exclude # ourselves. In reversed order so that more derived classes # override earlier field definitions in base classes. + # As long as we're iterating over them, see if any are frozen. + any_frozen_base = False + has_dataclass_bases = False for b in cls.__mro__[-1:0:-1]: # Only process classes that have been processed by our - # decorator. That is, they have a _MARKER attribute. - base_fields = getattr(b, _MARKER, None) + # decorator. That is, they have a _FIELDS attribute. + base_fields = getattr(b, _FIELDS, None) if base_fields: + has_dataclass_bases = True for f in base_fields.values(): fields[f.name] = f + if getattr(b, _PARAMS).frozen: + any_frozen_base = True # Now find fields in our class. While doing so, validate some # things, and set the default values (as class attributes) @@ -623,20 +680,21 @@ def _process_class(cls, repr, eq, order, unsafe_hash, init, frozen): else: setattr(cls, f.name, f.default) - # We're inheriting from a frozen dataclass, but we're not frozen. - if cls.__setattr__ is _frozen_setattr and not frozen: - raise TypeError('cannot inherit non-frozen dataclass from a ' - 'frozen one') + # Check rules that apply if we are derived from any dataclasses. + if has_dataclass_bases: + # Raise an exception if any of our bases are frozen, but we're not. + if any_frozen_base and not frozen: + raise TypeError('cannot inherit non-frozen dataclass from a ' + 'frozen one') - # We're inheriting from a non-frozen dataclass, but we're frozen. - if (hasattr(cls, _MARKER) and cls.__setattr__ is not _frozen_setattr - and frozen): - raise TypeError('cannot inherit frozen dataclass from a ' - 'non-frozen one') + # Raise an exception if we're frozen, but none of our bases are. + if not any_frozen_base and frozen: + raise TypeError('cannot inherit frozen dataclass from a ' + 'non-frozen one') - # Remember all of the fields on our class (including bases). This + # Remember all of the fields on our class (including bases). This also # marks this class as being a dataclass. - setattr(cls, _MARKER, fields) + setattr(cls, _FIELDS, fields) # Was this class defined with an explicit __hash__? Note that if # __eq__ is defined in this class, then python will automatically @@ -704,10 +762,10 @@ def _process_class(cls, repr, eq, order, unsafe_hash, init, frozen): 'functools.total_ordering') if frozen: - for name, fn in [('__setattr__', _frozen_setattr), - ('__delattr__', _frozen_delattr)]: - if _set_new_attribute(cls, name, fn): - raise TypeError(f'Cannot overwrite attribute {name} ' + # XXX: Which fields are frozen? InitVar? ClassVar? hashed-only? + for fn in _frozen_get_del_attr(cls, field_list): + if _set_new_attribute(cls, fn.__name__, fn): + raise TypeError(f'Cannot overwrite attribute {fn.__name__} ' f'in class {cls.__name__}') # Decide if/how we're going to create a hash function. @@ -759,7 +817,7 @@ def dataclass(_cls=None, *, init=True, repr=True, eq=True, order=False, """ def wrap(cls): - return _process_class(cls, repr, eq, order, unsafe_hash, init, frozen) + return _process_class(cls, init, repr, eq, order, unsafe_hash, frozen) # See if we're being called as @dataclass or @dataclass(). if _cls is None: @@ -779,7 +837,7 @@ def fields(class_or_instance): # Might it be worth caching this, per class? try: - fields = getattr(class_or_instance, _MARKER) + fields = getattr(class_or_instance, _FIELDS) except AttributeError: raise TypeError('must be called with a dataclass type or instance') @@ -790,13 +848,13 @@ def fields(class_or_instance): def _is_dataclass_instance(obj): """Returns True if obj is an instance of a dataclass.""" - return not isinstance(obj, type) and hasattr(obj, _MARKER) + return not isinstance(obj, type) and hasattr(obj, _FIELDS) def is_dataclass(obj): """Returns True if obj is a dataclass or an instance of a dataclass.""" - return hasattr(obj, _MARKER) + return hasattr(obj, _FIELDS) def asdict(obj, *, dict_factory=dict): @@ -953,7 +1011,7 @@ class C: # It's an error to have init=False fields in 'changes'. # If a field is not in 'changes', read its value from the provided obj. - for f in getattr(obj, _MARKER).values(): + for f in getattr(obj, _FIELDS).values(): if not f.init: # Error if this field is specified in changes. if f.name in changes: diff --git a/Lib/test/test_dataclasses.py b/Lib/test/test_dataclasses.py index 46d485c0157b..3e6726360940 100755 --- a/Lib/test/test_dataclasses.py +++ b/Lib/test/test_dataclasses.py @@ -2476,41 +2476,92 @@ class D(C): d = D(0, 10) with self.assertRaises(FrozenInstanceError): d.i = 5 + with self.assertRaises(FrozenInstanceError): + d.j = 6 self.assertEqual(d.i, 0) + self.assertEqual(d.j, 10) + + # Test both ways: with an intermediate normal (non-dataclass) + # class and without an intermediate class. + def test_inherit_nonfrozen_from_frozen(self): + for intermediate_class in [True, False]: + with self.subTest(intermediate_class=intermediate_class): + @dataclass(frozen=True) + class C: + i: int - def test_inherit_from_nonfrozen_from_frozen(self): - @dataclass(frozen=True) - class C: - i: int + if intermediate_class: + class I(C): pass + else: + I = C - with self.assertRaisesRegex(TypeError, - 'cannot inherit non-frozen dataclass from a frozen one'): - @dataclass - class D(C): - pass + with self.assertRaisesRegex(TypeError, + 'cannot inherit non-frozen dataclass from a frozen one'): + @dataclass + class D(I): + pass - def test_inherit_from_frozen_from_nonfrozen(self): - @dataclass - class C: - i: int + def test_inherit_frozen_from_nonfrozen(self): + for intermediate_class in [True, False]: + with self.subTest(intermediate_class=intermediate_class): + @dataclass + class C: + i: int - with self.assertRaisesRegex(TypeError, - 'cannot inherit frozen dataclass from a non-frozen one'): - @dataclass(frozen=True) - class D(C): - pass + if intermediate_class: + class I(C): pass + else: + I = C + + with self.assertRaisesRegex(TypeError, + 'cannot inherit frozen dataclass from a non-frozen one'): + @dataclass(frozen=True) + class D(I): + pass def test_inherit_from_normal_class(self): - class C: - pass + for intermediate_class in [True, False]: + with self.subTest(intermediate_class=intermediate_class): + class C: + pass + + if intermediate_class: + class I(C): pass + else: + I = C + + @dataclass(frozen=True) + class D(I): + i: int + + d = D(10) + with self.assertRaises(FrozenInstanceError): + d.i = 5 + + def test_non_frozen_normal_derived(self): + # See bpo-32953. @dataclass(frozen=True) - class D(C): - i: int + class D: + x: int + y: int = 10 - d = D(10) + class S(D): + pass + + s = S(3) + self.assertEqual(s.x, 3) + self.assertEqual(s.y, 10) + s.cached = True + + # But can't change the frozen attributes. with self.assertRaises(FrozenInstanceError): - d.i = 5 + s.x = 5 + with self.assertRaises(FrozenInstanceError): + s.y = 5 + self.assertEqual(s.x, 3) + self.assertEqual(s.y, 10) + self.assertEqual(s.cached, True) if __name__ == '__main__': diff --git a/Misc/NEWS.d/next/Library/2018-03-18-17-38-48.bpo-32953.t8WAWN.rst b/Misc/NEWS.d/next/Library/2018-03-18-17-38-48.bpo-32953.t8WAWN.rst new file mode 100644 index 000000000000..fbea34aa9a2a --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-03-18-17-38-48.bpo-32953.t8WAWN.rst @@ -0,0 +1,4 @@ +If a non-dataclass inherits from a frozen dataclass, allow attributes to be +added to the derived class. Only attributes from from the frozen dataclass +cannot be assigned to. Require all dataclasses in a hierarchy to be either +all frozen or all non-frozen. From solipsis at pitrou.net Mon Mar 19 05:13:45 2018 From: solipsis at pitrou.net (solipsis at pitrou.net) Date: Mon, 19 Mar 2018 09:13:45 +0000 Subject: [Python-checkins] Daily reference leaks (4243df51fe43): sum=4 Message-ID: <20180319091345.1.5E8703FCBF35C625@psf.io> results for 4243df51fe43 on branch "default" -------------------------------------------- test_collections leaked [0, 7, -7] memory blocks, sum=0 test_functools leaked [0, 3, 1] memory blocks, sum=4 test_multiprocessing_fork leaked [2, -2, 1] memory blocks, sum=1 test_multiprocessing_spawn leaked [-2, 0, 1] memory blocks, sum=-1 Command line was: ['./python', '-m', 'test.regrtest', '-uall', '-R', '3:3:/home/psf-users/antoine/refleaks/reflogkQnloZ', '--timeout', '7200'] From webhook-mailer at python.org Mon Mar 19 13:30:51 2018 From: webhook-mailer at python.org (Mariatta) Date: Mon, 19 Mar 2018 17:30:51 -0000 Subject: [Python-checkins] bpo-19417: Add test_bdb.py (GH-5217) Message-ID: <mailman.100.1521480653.1871.python-checkins@python.org> https://github.com/python/cpython/commit/424f3dafea16fbaee55a30903af2d6717f4d4a6b commit: 424f3dafea16fbaee55a30903af2d6717f4d4a6b branch: 3.6 author: xdegaye <xdegaye at gmail.com> committer: Mariatta <Mariatta at users.noreply.github.com> date: 2018-03-19T10:30:47-07:00 summary: bpo-19417: Add test_bdb.py (GH-5217) (cherry picked from commit 3fe33043ee83d19e15551094fc1e0984617ded3c) files: A Lib/test/test_bdb.py A Misc/NEWS.d/next/Tests/2018-01-08-13-33-47.bpo-19417.2asoXy.rst M Lib/test/test_sundry.py diff --git a/Lib/test/test_bdb.py b/Lib/test/test_bdb.py new file mode 100644 index 000000000000..abefe6c4e57a --- /dev/null +++ b/Lib/test/test_bdb.py @@ -0,0 +1,1151 @@ +""" Test the bdb module. + + A test defines a list of tuples that may be seen as paired tuples, each + pair being defined by 'expect_tuple, set_tuple' as follows: + + ([event, [lineno[, co_name[, eargs]]]]), (set_type, [sargs]) + + * 'expect_tuple' describes the expected current state of the Bdb instance. + It may be the empty tuple and no check is done in that case. + * 'set_tuple' defines the set_*() method to be invoked when the Bdb + instance reaches this state. + + Example of an 'expect_tuple, set_tuple' pair: + + ('line', 2, 'tfunc_main'), ('step', ) + + Definitions of the members of the 'expect_tuple': + event: + Name of the trace event. The set methods that do not give back + control to the tracer [1] do not trigger a tracer event and in + that case the next 'event' may be 'None' by convention, its value + is not checked. + [1] Methods that trigger a trace event are set_step(), set_next(), + set_return(), set_until() and set_continue(). + lineno: + Line number. Line numbers are relative to the start of the + function when tracing a function in the test_bdb module (i.e. this + module). + co_name: + Name of the function being currently traced. + eargs: + A tuple: + * On an 'exception' event the tuple holds a class object, the + current exception must be an instance of this class. + * On a 'line' event, the tuple holds a dictionary and a list. The + dictionary maps each breakpoint number that has been hit on this + line to its hits count. The list holds the list of breakpoint + number temporaries that are being deleted. + + Definitions of the members of the 'set_tuple': + set_type: + The type of the set method to be invoked. This may + be the type of one of the Bdb set methods: 'step', 'next', + 'until', 'return', 'continue', 'break', 'quit', or the type of one + of the set methods added by test_bdb.Bdb: 'ignore', 'enable', + 'disable', 'clear', 'up', 'down'. + sargs: + The arguments of the set method if any, packed in a tuple. +""" + +import bdb as _bdb +import sys +import os +import unittest +import textwrap +import importlib +import linecache +from contextlib import contextmanager +from itertools import islice, repeat +import test.support + +class BdbException(Exception): pass +class BdbError(BdbException): """Error raised by the Bdb instance.""" +class BdbSyntaxError(BdbException): """Syntax error in the test case.""" +class BdbNotExpectedError(BdbException): """Unexpected result.""" + +# When 'dry_run' is set to true, expect tuples are ignored and the actual +# state of the tracer is printed after running each set_*() method of the test +# case. The full list of breakpoints and their attributes is also printed +# after each 'line' event where a breakpoint has been hit. +dry_run = 0 + +def reset_Breakpoint(): + _bdb.Breakpoint.next = 1 + _bdb.Breakpoint.bplist = {} + _bdb.Breakpoint.bpbynumber = [None] + +def info_breakpoints(): + bp_list = [bp for bp in _bdb.Breakpoint.bpbynumber if bp] + if not bp_list: + return '' + + header_added = False + for bp in bp_list: + if not header_added: + info = 'BpNum Temp Enb Hits Ignore Where\n' + header_added = True + + disp = 'yes ' if bp.temporary else 'no ' + enab = 'yes' if bp.enabled else 'no ' + info += ('%-5d %s %s %-4d %-6d at %s:%d' % + (bp.number, disp, enab, bp.hits, bp.ignore, + os.path.basename(bp.file), bp.line)) + if bp.cond: + info += '\n\tstop only if %s' % (bp.cond,) + info += '\n' + return info + +class Bdb(_bdb.Bdb): + """Extend Bdb to enhance test coverage.""" + + def trace_dispatch(self, frame, event, arg): + self.currentbp = None + return super().trace_dispatch(frame, event, arg) + + def set_break(self, filename, lineno, temporary=False, cond=None, + funcname=None): + if isinstance(funcname, str): + if filename == __file__: + globals_ = globals() + else: + module = importlib.import_module(filename[:-3]) + globals_ = module.__dict__ + func = eval(funcname, globals_) + code = func.__code__ + filename = code.co_filename + lineno = code.co_firstlineno + funcname = code.co_name + + res = super().set_break(filename, lineno, temporary=temporary, + cond=cond, funcname=funcname) + if isinstance(res, str): + raise BdbError(res) + return res + + def get_stack(self, f, t): + self.stack, self.index = super().get_stack(f, t) + self.frame = self.stack[self.index][0] + return self.stack, self.index + + def set_ignore(self, bpnum): + """Increment the ignore count of Breakpoint number 'bpnum'.""" + bp = self.get_bpbynumber(bpnum) + bp.ignore += 1 + + def set_enable(self, bpnum): + bp = self.get_bpbynumber(bpnum) + bp.enabled = True + + def set_disable(self, bpnum): + bp = self.get_bpbynumber(bpnum) + bp.enabled = False + + def set_clear(self, fname, lineno): + err = self.clear_break(fname, lineno) + if err: + raise BdbError(err) + + def set_up(self): + """Move up in the frame stack.""" + if not self.index: + raise BdbError('Oldest frame') + self.index -= 1 + self.frame = self.stack[self.index][0] + + def set_down(self): + """Move down in the frame stack.""" + if self.index + 1 == len(self.stack): + raise BdbError('Newest frame') + self.index += 1 + self.frame = self.stack[self.index][0] + +class Tracer(Bdb): + """A tracer for testing the bdb module.""" + + def __init__(self, expect_set, skip=None, dry_run=False, test_case=None): + super().__init__(skip=skip) + self.expect_set = expect_set + self.dry_run = dry_run + self.header = ('Dry-run results for %s:' % test_case if + test_case is not None else None) + self.init_test() + + def init_test(self): + self.cur_except = None + self.expect_set_no = 0 + self.breakpoint_hits = None + self.expected_list = list(islice(self.expect_set, 0, None, 2)) + self.set_list = list(islice(self.expect_set, 1, None, 2)) + + def trace_dispatch(self, frame, event, arg): + # On an 'exception' event, call_exc_trace() in Python/ceval.c discards + # a BdbException raised by the Tracer instance, so we raise it on the + # next trace_dispatch() call that occurs unless the set_quit() or + # set_continue() method has been invoked on the 'exception' event. + if self.cur_except is not None: + raise self.cur_except + + if event == 'exception': + try: + res = super().trace_dispatch(frame, event, arg) + return res + except BdbException as e: + self.cur_except = e + return self.trace_dispatch + else: + return super().trace_dispatch(frame, event, arg) + + def user_call(self, frame, argument_list): + # Adopt the same behavior as pdb and, as a side effect, skip also the + # first 'call' event when the Tracer is started with Tracer.runcall() + # which may be possibly considered as a bug. + if not self.stop_here(frame): + return + self.process_event('call', frame, argument_list) + self.next_set_method() + + def user_line(self, frame): + self.process_event('line', frame) + + if self.dry_run and self.breakpoint_hits: + info = info_breakpoints().strip('\n') + # Indent each line. + for line in info.split('\n'): + print(' ' + line) + self.delete_temporaries() + self.breakpoint_hits = None + + self.next_set_method() + + def user_return(self, frame, return_value): + self.process_event('return', frame, return_value) + self.next_set_method() + + def user_exception(self, frame, exc_info): + self.exc_info = exc_info + self.process_event('exception', frame) + self.next_set_method() + + def do_clear(self, arg): + # The temporary breakpoints are deleted in user_line(). + bp_list = [self.currentbp] + self.breakpoint_hits = (bp_list, bp_list) + + def delete_temporaries(self): + if self.breakpoint_hits: + for n in self.breakpoint_hits[1]: + self.clear_bpbynumber(n) + + def pop_next(self): + self.expect_set_no += 1 + try: + self.expect = self.expected_list.pop(0) + except IndexError: + raise BdbNotExpectedError( + 'expect_set list exhausted, cannot pop item %d' % + self.expect_set_no) + self.set_tuple = self.set_list.pop(0) + + def process_event(self, event, frame, *args): + # Call get_stack() to enable walking the stack with set_up() and + # set_down(). + tb = None + if event == 'exception': + tb = self.exc_info[2] + self.get_stack(frame, tb) + + # A breakpoint has been hit and it is not a temporary. + if self.currentbp is not None and not self.breakpoint_hits: + bp_list = [self.currentbp] + self.breakpoint_hits = (bp_list, []) + + # Pop next event. + self.event= event + self.pop_next() + if self.dry_run: + self.print_state(self.header) + return + + # Validate the expected results. + if self.expect: + self.check_equal(self.expect[0], event, 'Wrong event type') + self.check_lno_name() + + if event in ('call', 'return'): + self.check_expect_max_size(3) + elif len(self.expect) > 3: + if event == 'line': + bps, temporaries = self.expect[3] + bpnums = sorted(bps.keys()) + if not self.breakpoint_hits: + self.raise_not_expected( + 'No breakpoints hit at expect_set item %d' % + self.expect_set_no) + self.check_equal(bpnums, self.breakpoint_hits[0], + 'Breakpoint numbers do not match') + self.check_equal([bps[n] for n in bpnums], + [self.get_bpbynumber(n).hits for + n in self.breakpoint_hits[0]], + 'Wrong breakpoint hit count') + self.check_equal(sorted(temporaries), self.breakpoint_hits[1], + 'Wrong temporary breakpoints') + + elif event == 'exception': + if not isinstance(self.exc_info[1], self.expect[3]): + self.raise_not_expected( + "Wrong exception at expect_set item %d, got '%s'" % + (self.expect_set_no, self.exc_info)) + + def check_equal(self, expected, result, msg): + if expected == result: + return + self.raise_not_expected("%s at expect_set item %d, got '%s'" % + (msg, self.expect_set_no, result)) + + def check_lno_name(self): + """Check the line number and function co_name.""" + s = len(self.expect) + if s > 1: + lineno = self.lno_abs2rel() + self.check_equal(self.expect[1], lineno, 'Wrong line number') + if s > 2: + self.check_equal(self.expect[2], self.frame.f_code.co_name, + 'Wrong function name') + + def check_expect_max_size(self, size): + if len(self.expect) > size: + raise BdbSyntaxError('Invalid size of the %s expect tuple: %s' % + (self.event, self.expect)) + + def lno_abs2rel(self): + fname = self.canonic(self.frame.f_code.co_filename) + lineno = self.frame.f_lineno + return ((lineno - self.frame.f_code.co_firstlineno + 1) + if fname == self.canonic(__file__) else lineno) + + def lno_rel2abs(self, fname, lineno): + return (self.frame.f_code.co_firstlineno + lineno - 1 + if (lineno and self.canonic(fname) == self.canonic(__file__)) + else lineno) + + def get_state(self): + lineno = self.lno_abs2rel() + co_name = self.frame.f_code.co_name + state = "('%s', %d, '%s'" % (self.event, lineno, co_name) + if self.breakpoint_hits: + bps = '{' + for n in self.breakpoint_hits[0]: + if bps != '{': + bps += ', ' + bps += '%s: %s' % (n, self.get_bpbynumber(n).hits) + bps += '}' + bps = '(' + bps + ', ' + str(self.breakpoint_hits[1]) + ')' + state += ', ' + bps + elif self.event == 'exception': + state += ', ' + self.exc_info[0].__name__ + state += '), ' + return state.ljust(32) + str(self.set_tuple) + ',' + + def print_state(self, header=None): + if header is not None and self.expect_set_no == 1: + print() + print(header) + print('%d: %s' % (self.expect_set_no, self.get_state())) + + def raise_not_expected(self, msg): + msg += '\n' + msg += ' Expected: %s\n' % str(self.expect) + msg += ' Got: ' + self.get_state() + raise BdbNotExpectedError(msg) + + def next_set_method(self): + set_type = self.set_tuple[0] + args = self.set_tuple[1] if len(self.set_tuple) == 2 else None + set_method = getattr(self, 'set_' + set_type) + + # The following set methods give back control to the tracer. + if set_type in ('step', 'continue', 'quit'): + set_method() + return + elif set_type in ('next', 'return'): + set_method(self.frame) + return + elif set_type == 'until': + lineno = None + if args: + lineno = self.lno_rel2abs(self.frame.f_code.co_filename, + args[0]) + set_method(self.frame, lineno) + return + + # The following set methods do not give back control to the tracer and + # next_set_method() is called recursively. + if (args and set_type in ('break', 'clear', 'ignore', 'enable', + 'disable')) or set_type in ('up', 'down'): + if set_type in ('break', 'clear'): + fname, lineno, *remain = args + lineno = self.lno_rel2abs(fname, lineno) + args = [fname, lineno] + args.extend(remain) + set_method(*args) + elif set_type in ('ignore', 'enable', 'disable'): + set_method(*args) + elif set_type in ('up', 'down'): + set_method() + + # Process the next expect_set item. + # It is not expected that a test may reach the recursion limit. + self.event= None + self.pop_next() + if self.dry_run: + self.print_state() + else: + if self.expect: + self.check_lno_name() + self.check_expect_max_size(3) + self.next_set_method() + else: + raise BdbSyntaxError('"%s" is an invalid set_tuple' % + self.set_tuple) + +class TracerRun(): + """Provide a context for running a Tracer instance with a test case.""" + + def __init__(self, test_case, skip=None): + self.test_case = test_case + self.dry_run = test_case.dry_run + self.tracer = Tracer(test_case.expect_set, skip=skip, + dry_run=self.dry_run, test_case=test_case.id()) + + def __enter__(self): + # test_pdb does not reset Breakpoint class attributes on exit :-( + reset_Breakpoint() + return self.tracer + + def __exit__(self, type_=None, value=None, traceback=None): + reset_Breakpoint() + sys.settrace(None) + + not_empty = '' + if self.tracer.set_list: + not_empty += 'All paired tuples have not been processed, ' + not_empty += ('the last one was number %d' % + self.tracer.expect_set_no) + + # Make a BdbNotExpectedError a unittest failure. + if type_ is not None and issubclass(BdbNotExpectedError, type_): + if isinstance(value, BaseException) and value.args: + err_msg = value.args[0] + if not_empty: + err_msg += '\n' + not_empty + if self.dry_run: + print(err_msg) + return True + else: + self.test_case.fail(err_msg) + else: + assert False, 'BdbNotExpectedError with empty args' + + if not_empty: + if self.dry_run: + print(not_empty) + else: + self.test_case.fail(not_empty) + +def run_test(modules, set_list, skip=None): + """Run a test and print the dry-run results. + + 'modules': A dictionary mapping module names to their source code as a + string. The dictionary MUST include one module named + 'test_module' with a main() function. + 'set_list': A list of set_type tuples to be run on the module. + + For example, running the following script outputs the following results: + + ***************************** SCRIPT ******************************** + + from test.test_bdb import run_test, break_in_func + + code = ''' + def func(): + lno = 3 + + def main(): + func() + lno = 7 + ''' + + set_list = [ + break_in_func('func', 'test_module.py'), + ('continue', ), + ('step', ), + ('step', ), + ('step', ), + ('quit', ), + ] + + modules = { 'test_module': code } + run_test(modules, set_list) + + **************************** results ******************************** + + 1: ('line', 2, 'tfunc_import'), ('next',), + 2: ('line', 3, 'tfunc_import'), ('step',), + 3: ('call', 5, 'main'), ('break', ('test_module.py', None, False, None, 'func')), + 4: ('None', 5, 'main'), ('continue',), + 5: ('line', 3, 'func', ({1: 1}, [])), ('step',), + BpNum Temp Enb Hits Ignore Where + 1 no yes 1 0 at test_module.py:2 + 6: ('return', 3, 'func'), ('step',), + 7: ('line', 7, 'main'), ('step',), + 8: ('return', 7, 'main'), ('quit',), + + ************************************************************************* + + """ + def gen(a, b): + try: + while 1: + x = next(a) + y = next(b) + yield x + yield y + except StopIteration: + return + + # Step over the import statement in tfunc_import using 'next' and step + # into main() in test_module. + sl = [('next', ), ('step', )] + sl.extend(set_list) + + test = BaseTestCase() + test.dry_run = True + test.id = lambda : None + test.expect_set = list(gen(repeat(()), iter(sl))) + with create_modules(modules): + sys.path.append(os.getcwd()) + with TracerRun(test, skip=skip) as tracer: + tracer.runcall(tfunc_import) + + at contextmanager +def create_modules(modules): + with test.support.temp_cwd(): + try: + for m in modules: + fname = m + '.py' + with open(fname, 'w') as f: + f.write(textwrap.dedent(modules[m])) + linecache.checkcache(fname) + importlib.invalidate_caches() + yield + finally: + for m in modules: + test.support.forget(m) + +def break_in_func(funcname, fname=__file__, temporary=False, cond=None): + return 'break', (fname, None, temporary, cond, funcname) + +TEST_MODULE = 'test_module' +TEST_MODULE_FNAME = TEST_MODULE + '.py' +def tfunc_import(): + import test_module + test_module.main() + +def tfunc_main(): + lno = 2 + tfunc_first() + tfunc_second() + lno = 5 + lno = 6 + lno = 7 + +def tfunc_first(): + lno = 2 + lno = 3 + lno = 4 + +def tfunc_second(): + lno = 2 + +class BaseTestCase(unittest.TestCase): + """Base class for all tests.""" + + dry_run = dry_run + + def fail(self, msg=None): + # Override fail() to use 'raise from None' to avoid repetition of the + # error message and traceback. + raise self.failureException(msg) from None + +class StateTestCase(BaseTestCase): + """Test the step, next, return, until and quit 'set_' methods.""" + + def test_step(self): + self.expect_set = [ + ('line', 2, 'tfunc_main'), ('step', ), + ('line', 3, 'tfunc_main'), ('step', ), + ('call', 1, 'tfunc_first'), ('step', ), + ('line', 2, 'tfunc_first'), ('quit', ), + ] + with TracerRun(self) as tracer: + tracer.runcall(tfunc_main) + + def test_step_next_on_last_statement(self): + for set_type in ('step', 'next'): + with self.subTest(set_type=set_type): + self.expect_set = [ + ('line', 2, 'tfunc_main'), ('step', ), + ('line', 3, 'tfunc_main'), ('step', ), + ('call', 1, 'tfunc_first'), ('break', (__file__, 3)), + ('None', 1, 'tfunc_first'), ('continue', ), + ('line', 3, 'tfunc_first', ({1:1}, [])), (set_type, ), + ('line', 4, 'tfunc_first'), ('quit', ), + ] + with TracerRun(self) as tracer: + tracer.runcall(tfunc_main) + + def test_next(self): + self.expect_set = [ + ('line', 2, 'tfunc_main'), ('step', ), + ('line', 3, 'tfunc_main'), ('next', ), + ('line', 4, 'tfunc_main'), ('step', ), + ('call', 1, 'tfunc_second'), ('step', ), + ('line', 2, 'tfunc_second'), ('quit', ), + ] + with TracerRun(self) as tracer: + tracer.runcall(tfunc_main) + + def test_next_over_import(self): + code = """ + def main(): + lno = 3 + """ + modules = { TEST_MODULE: code } + with create_modules(modules): + self.expect_set = [ + ('line', 2, 'tfunc_import'), ('next', ), + ('line', 3, 'tfunc_import'), ('quit', ), + ] + with TracerRun(self) as tracer: + tracer.runcall(tfunc_import) + + def test_next_on_plain_statement(self): + # Check that set_next() is equivalent to set_step() on a plain + # statement. + self.expect_set = [ + ('line', 2, 'tfunc_main'), ('step', ), + ('line', 3, 'tfunc_main'), ('step', ), + ('call', 1, 'tfunc_first'), ('next', ), + ('line', 2, 'tfunc_first'), ('quit', ), + ] + with TracerRun(self) as tracer: + tracer.runcall(tfunc_main) + + def test_next_in_caller_frame(self): + # Check that set_next() in the caller frame causes the tracer + # to stop next in the caller frame. + self.expect_set = [ + ('line', 2, 'tfunc_main'), ('step', ), + ('line', 3, 'tfunc_main'), ('step', ), + ('call', 1, 'tfunc_first'), ('up', ), + ('None', 3, 'tfunc_main'), ('next', ), + ('line', 4, 'tfunc_main'), ('quit', ), + ] + with TracerRun(self) as tracer: + tracer.runcall(tfunc_main) + + def test_return(self): + self.expect_set = [ + ('line', 2, 'tfunc_main'), ('step', ), + ('line', 3, 'tfunc_main'), ('step', ), + ('call', 1, 'tfunc_first'), ('step', ), + ('line', 2, 'tfunc_first'), ('return', ), + ('return', 4, 'tfunc_first'), ('step', ), + ('line', 4, 'tfunc_main'), ('quit', ), + ] + with TracerRun(self) as tracer: + tracer.runcall(tfunc_main) + + def test_return_in_caller_frame(self): + self.expect_set = [ + ('line', 2, 'tfunc_main'), ('step', ), + ('line', 3, 'tfunc_main'), ('step', ), + ('call', 1, 'tfunc_first'), ('up', ), + ('None', 3, 'tfunc_main'), ('return', ), + ('return', 7, 'tfunc_main'), ('quit', ), + ] + with TracerRun(self) as tracer: + tracer.runcall(tfunc_main) + + def test_until(self): + self.expect_set = [ + ('line', 2, 'tfunc_main'), ('step', ), + ('line', 3, 'tfunc_main'), ('step', ), + ('call', 1, 'tfunc_first'), ('step', ), + ('line', 2, 'tfunc_first'), ('until', (4, )), + ('line', 4, 'tfunc_first'), ('quit', ), + ] + with TracerRun(self) as tracer: + tracer.runcall(tfunc_main) + + def test_until_with_too_large_count(self): + self.expect_set = [ + ('line', 2, 'tfunc_main'), break_in_func('tfunc_first'), + ('None', 2, 'tfunc_main'), ('continue', ), + ('line', 2, 'tfunc_first', ({1:1}, [])), ('until', (9999, )), + ('return', 4, 'tfunc_first'), ('quit', ), + ] + with TracerRun(self) as tracer: + tracer.runcall(tfunc_main) + + def test_until_in_caller_frame(self): + self.expect_set = [ + ('line', 2, 'tfunc_main'), ('step', ), + ('line', 3, 'tfunc_main'), ('step', ), + ('call', 1, 'tfunc_first'), ('up', ), + ('None', 3, 'tfunc_main'), ('until', (6, )), + ('line', 6, 'tfunc_main'), ('quit', ), + ] + with TracerRun(self) as tracer: + tracer.runcall(tfunc_main) + + def test_skip(self): + # Check that tracing is skipped over the import statement in + # 'tfunc_import()'. + code = """ + def main(): + lno = 3 + """ + modules = { TEST_MODULE: code } + with create_modules(modules): + self.expect_set = [ + ('line', 2, 'tfunc_import'), ('step', ), + ('line', 3, 'tfunc_import'), ('quit', ), + ] + skip = ('importlib*', TEST_MODULE) + with TracerRun(self, skip=skip) as tracer: + tracer.runcall(tfunc_import) + + def test_down(self): + # Check that set_down() raises BdbError at the newest frame. + self.expect_set = [ + ('line', 2, 'tfunc_main'), ('down', ), + ] + with TracerRun(self) as tracer: + self.assertRaises(BdbError, tracer.runcall, tfunc_main) + + def test_up(self): + self.expect_set = [ + ('line', 2, 'tfunc_main'), ('step', ), + ('line', 3, 'tfunc_main'), ('step', ), + ('call', 1, 'tfunc_first'), ('up', ), + ('None', 3, 'tfunc_main'), ('quit', ), + ] + with TracerRun(self) as tracer: + tracer.runcall(tfunc_main) + +class BreakpointTestCase(BaseTestCase): + """Test the breakpoint set method.""" + + def test_bp_on_non_existent_module(self): + self.expect_set = [ + ('line', 2, 'tfunc_import'), ('break', ('/non/existent/module.py', 1)) + ] + with TracerRun(self) as tracer: + self.assertRaises(BdbError, tracer.runcall, tfunc_import) + + def test_bp_after_last_statement(self): + code = """ + def main(): + lno = 3 + """ + modules = { TEST_MODULE: code } + with create_modules(modules): + self.expect_set = [ + ('line', 2, 'tfunc_import'), ('break', (TEST_MODULE_FNAME, 4)) + ] + with TracerRun(self) as tracer: + self.assertRaises(BdbError, tracer.runcall, tfunc_import) + + def test_temporary_bp(self): + code = """ + def func(): + lno = 3 + + def main(): + for i in range(2): + func() + """ + modules = { TEST_MODULE: code } + with create_modules(modules): + self.expect_set = [ + ('line', 2, 'tfunc_import'), + break_in_func('func', TEST_MODULE_FNAME, True), + ('None', 2, 'tfunc_import'), + break_in_func('func', TEST_MODULE_FNAME, True), + ('None', 2, 'tfunc_import'), ('continue', ), + ('line', 3, 'func', ({1:1}, [1])), ('continue', ), + ('line', 3, 'func', ({2:1}, [2])), ('quit', ), + ] + with TracerRun(self) as tracer: + tracer.runcall(tfunc_import) + + def test_disabled_temporary_bp(self): + code = """ + def func(): + lno = 3 + + def main(): + for i in range(3): + func() + """ + modules = { TEST_MODULE: code } + with create_modules(modules): + self.expect_set = [ + ('line', 2, 'tfunc_import'), + break_in_func('func', TEST_MODULE_FNAME), + ('None', 2, 'tfunc_import'), + break_in_func('func', TEST_MODULE_FNAME, True), + ('None', 2, 'tfunc_import'), ('disable', (2, )), + ('None', 2, 'tfunc_import'), ('continue', ), + ('line', 3, 'func', ({1:1}, [])), ('enable', (2, )), + ('None', 3, 'func'), ('disable', (1, )), + ('None', 3, 'func'), ('continue', ), + ('line', 3, 'func', ({2:1}, [2])), ('enable', (1, )), + ('None', 3, 'func'), ('continue', ), + ('line', 3, 'func', ({1:2}, [])), ('quit', ), + ] + with TracerRun(self) as tracer: + tracer.runcall(tfunc_import) + + def test_bp_condition(self): + code = """ + def func(a): + lno = 3 + + def main(): + for i in range(3): + func(i) + """ + modules = { TEST_MODULE: code } + with create_modules(modules): + self.expect_set = [ + ('line', 2, 'tfunc_import'), + break_in_func('func', TEST_MODULE_FNAME, False, 'a == 2'), + ('None', 2, 'tfunc_import'), ('continue', ), + ('line', 3, 'func', ({1:3}, [])), ('quit', ), + ] + with TracerRun(self) as tracer: + tracer.runcall(tfunc_import) + + def test_bp_exception_on_condition_evaluation(self): + code = """ + def func(a): + lno = 3 + + def main(): + func(0) + """ + modules = { TEST_MODULE: code } + with create_modules(modules): + self.expect_set = [ + ('line', 2, 'tfunc_import'), + break_in_func('func', TEST_MODULE_FNAME, False, '1 / 0'), + ('None', 2, 'tfunc_import'), ('continue', ), + ('line', 3, 'func', ({1:1}, [])), ('quit', ), + ] + with TracerRun(self) as tracer: + tracer.runcall(tfunc_import) + + def test_bp_ignore_count(self): + code = """ + def func(): + lno = 3 + + def main(): + for i in range(2): + func() + """ + modules = { TEST_MODULE: code } + with create_modules(modules): + self.expect_set = [ + ('line', 2, 'tfunc_import'), + break_in_func('func', TEST_MODULE_FNAME), + ('None', 2, 'tfunc_import'), ('ignore', (1, )), + ('None', 2, 'tfunc_import'), ('continue', ), + ('line', 3, 'func', ({1:2}, [])), ('quit', ), + ] + with TracerRun(self) as tracer: + tracer.runcall(tfunc_import) + + def test_ignore_count_on_disabled_bp(self): + code = """ + def func(): + lno = 3 + + def main(): + for i in range(3): + func() + """ + modules = { TEST_MODULE: code } + with create_modules(modules): + self.expect_set = [ + ('line', 2, 'tfunc_import'), + break_in_func('func', TEST_MODULE_FNAME), + ('None', 2, 'tfunc_import'), + break_in_func('func', TEST_MODULE_FNAME), + ('None', 2, 'tfunc_import'), ('ignore', (1, )), + ('None', 2, 'tfunc_import'), ('disable', (1, )), + ('None', 2, 'tfunc_import'), ('continue', ), + ('line', 3, 'func', ({2:1}, [])), ('enable', (1, )), + ('None', 3, 'func'), ('continue', ), + ('line', 3, 'func', ({2:2}, [])), ('continue', ), + ('line', 3, 'func', ({1:2}, [])), ('quit', ), + ] + with TracerRun(self) as tracer: + tracer.runcall(tfunc_import) + + def test_clear_two_bp_on_same_line(self): + code = """ + def func(): + lno = 3 + lno = 4 + + def main(): + for i in range(3): + func() + """ + modules = { TEST_MODULE: code } + with create_modules(modules): + self.expect_set = [ + ('line', 2, 'tfunc_import'), ('break', (TEST_MODULE_FNAME, 3)), + ('None', 2, 'tfunc_import'), ('break', (TEST_MODULE_FNAME, 3)), + ('None', 2, 'tfunc_import'), ('break', (TEST_MODULE_FNAME, 4)), + ('None', 2, 'tfunc_import'), ('continue', ), + ('line', 3, 'func', ({1:1}, [])), ('continue', ), + ('line', 4, 'func', ({3:1}, [])), ('clear', (TEST_MODULE_FNAME, 3)), + ('None', 4, 'func'), ('continue', ), + ('line', 4, 'func', ({3:2}, [])), ('quit', ), + ] + with TracerRun(self) as tracer: + tracer.runcall(tfunc_import) + + def test_clear_at_no_bp(self): + self.expect_set = [ + ('line', 2, 'tfunc_import'), ('clear', (__file__, 1)) + ] + with TracerRun(self) as tracer: + self.assertRaises(BdbError, tracer.runcall, tfunc_import) + +class RunTestCase(BaseTestCase): + """Test run, runeval and set_trace.""" + + def test_run_step(self): + # Check that the bdb 'run' method stops at the first line event. + code = """ + lno = 2 + """ + self.expect_set = [ + ('line', 2, '<module>'), ('step', ), + ('return', 2, '<module>'), ('quit', ), + ] + with TracerRun(self) as tracer: + tracer.run(compile(textwrap.dedent(code), '<string>', 'exec')) + + def test_runeval_step(self): + # Test bdb 'runeval'. + code = """ + def main(): + lno = 3 + """ + modules = { TEST_MODULE: code } + with create_modules(modules): + self.expect_set = [ + ('line', 1, '<module>'), ('step', ), + ('call', 2, 'main'), ('step', ), + ('line', 3, 'main'), ('step', ), + ('return', 3, 'main'), ('step', ), + ('return', 1, '<module>'), ('quit', ), + ] + import test_module + with TracerRun(self) as tracer: + tracer.runeval('test_module.main()', globals(), locals()) + +class IssuesTestCase(BaseTestCase): + """Test fixed bdb issues.""" + + def test_step_at_return_with_no_trace_in_caller(self): + # Issue #13183. + # Check that the tracer does step into the caller frame when the + # trace function is not set in that frame. + code_1 = """ + from test_module_2 import func + def main(): + func() + lno = 5 + """ + code_2 = """ + def func(): + lno = 3 + """ + modules = { + TEST_MODULE: code_1, + 'test_module_2': code_2, + } + with create_modules(modules): + self.expect_set = [ + ('line', 2, 'tfunc_import'), + break_in_func('func', 'test_module_2.py'), + ('None', 2, 'tfunc_import'), ('continue', ), + ('line', 3, 'func', ({1:1}, [])), ('step', ), + ('return', 3, 'func'), ('step', ), + ('line', 5, 'main'), ('quit', ), + ] + with TracerRun(self) as tracer: + tracer.runcall(tfunc_import) + + def test_next_until_return_in_generator(self): + # Issue #16596. + # Check that set_next(), set_until() and set_return() do not treat the + # `yield` and `yield from` statements as if they were returns and stop + # instead in the current frame. + code = """ + def test_gen(): + yield 0 + lno = 4 + return 123 + + def main(): + it = test_gen() + next(it) + next(it) + lno = 11 + """ + modules = { TEST_MODULE: code } + for set_type in ('next', 'until', 'return'): + with self.subTest(set_type=set_type): + with create_modules(modules): + self.expect_set = [ + ('line', 2, 'tfunc_import'), + break_in_func('test_gen', TEST_MODULE_FNAME), + ('None', 2, 'tfunc_import'), ('continue', ), + ('line', 3, 'test_gen', ({1:1}, [])), (set_type, ), + ] + + if set_type == 'return': + self.expect_set.extend( + [('exception', 10, 'main', StopIteration), ('step',), + ('return', 10, 'main'), ('quit', ), + ] + ) + else: + self.expect_set.extend( + [('line', 4, 'test_gen'), ('quit', ),] + ) + with TracerRun(self) as tracer: + tracer.runcall(tfunc_import) + + def test_next_command_in_generator_for_loop(self): + # Issue #16596. + code = """ + def test_gen(): + yield 0 + lno = 4 + yield 1 + return 123 + + def main(): + for i in test_gen(): + lno = 10 + lno = 11 + """ + modules = { TEST_MODULE: code } + with create_modules(modules): + self.expect_set = [ + ('line', 2, 'tfunc_import'), + break_in_func('test_gen', TEST_MODULE_FNAME), + ('None', 2, 'tfunc_import'), ('continue', ), + ('line', 3, 'test_gen', ({1:1}, [])), ('next', ), + ('line', 4, 'test_gen'), ('next', ), + ('line', 5, 'test_gen'), ('next', ), + ('line', 6, 'test_gen'), ('next', ), + ('exception', 9, 'main', StopIteration), ('step', ), + ('line', 11, 'main'), ('quit', ), + + ] + with TracerRun(self) as tracer: + tracer.runcall(tfunc_import) + + def test_next_command_in_generator_with_subiterator(self): + # Issue #16596. + code = """ + def test_subgen(): + yield 0 + return 123 + + def test_gen(): + x = yield from test_subgen() + return 456 + + def main(): + for i in test_gen(): + lno = 12 + lno = 13 + """ + modules = { TEST_MODULE: code } + with create_modules(modules): + self.expect_set = [ + ('line', 2, 'tfunc_import'), + break_in_func('test_gen', TEST_MODULE_FNAME), + ('None', 2, 'tfunc_import'), ('continue', ), + ('line', 7, 'test_gen', ({1:1}, [])), ('next', ), + ('line', 8, 'test_gen'), ('next', ), + ('exception', 11, 'main', StopIteration), ('step', ), + ('line', 13, 'main'), ('quit', ), + + ] + with TracerRun(self) as tracer: + tracer.runcall(tfunc_import) + + def test_return_command_in_generator_with_subiterator(self): + # Issue #16596. + code = """ + def test_subgen(): + yield 0 + return 123 + + def test_gen(): + x = yield from test_subgen() + return 456 + + def main(): + for i in test_gen(): + lno = 12 + lno = 13 + """ + modules = { TEST_MODULE: code } + with create_modules(modules): + self.expect_set = [ + ('line', 2, 'tfunc_import'), + break_in_func('test_subgen', TEST_MODULE_FNAME), + ('None', 2, 'tfunc_import'), ('continue', ), + ('line', 3, 'test_subgen', ({1:1}, [])), ('return', ), + ('exception', 7, 'test_gen', StopIteration), ('return', ), + ('exception', 11, 'main', StopIteration), ('step', ), + ('line', 13, 'main'), ('quit', ), + + ] + with TracerRun(self) as tracer: + tracer.runcall(tfunc_import) + +def test_main(): + test.support.run_unittest( + StateTestCase, + RunTestCase, + BreakpointTestCase, + IssuesTestCase, + ) + +if __name__ == "__main__": + test_main() diff --git a/Lib/test/test_sundry.py b/Lib/test/test_sundry.py index 1fb996469821..fc5ac0ae78e4 100644 --- a/Lib/test/test_sundry.py +++ b/Lib/test/test_sundry.py @@ -6,8 +6,7 @@ class TestUntestedModules(unittest.TestCase): def test_untested_modules_can_be_imported(self): - untested = ('bdb', 'encodings', 'formatter', - 'nturl2path', 'tabnanny') + untested = ('encodings', 'formatter', 'nturl2path', 'tabnanny') with support.check_warnings(quiet=True): for name in untested: try: diff --git a/Misc/NEWS.d/next/Tests/2018-01-08-13-33-47.bpo-19417.2asoXy.rst b/Misc/NEWS.d/next/Tests/2018-01-08-13-33-47.bpo-19417.2asoXy.rst new file mode 100644 index 000000000000..739352fcdd67 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2018-01-08-13-33-47.bpo-19417.2asoXy.rst @@ -0,0 +1 @@ +Add test_bdb.py. From webhook-mailer at python.org Mon Mar 19 14:41:40 2018 From: webhook-mailer at python.org (Ivan Levkivskyi) Date: Mon, 19 Mar 2018 18:41:40 -0000 Subject: [Python-checkins] bpo-33061: Add missing 'NoReturn' to __all__ in typing.py (GH-6127) Message-ID: <mailman.101.1521484902.1871.python-checkins@python.org> https://github.com/python/cpython/commit/4573820d2a9156346392838d455e89f33067e9dd commit: 4573820d2a9156346392838d455e89f33067e9dd branch: master author: aetracht <allen at duolingo.com> committer: Ivan Levkivskyi <levkivskyi at gmail.com> date: 2018-03-19T18:41:32Z summary: bpo-33061: Add missing 'NoReturn' to __all__ in typing.py (GH-6127) files: A Misc/NEWS.d/next/Library/2018-03-16-16-07-33.bpo-33061.TRTTek.rst M Lib/typing.py diff --git a/Lib/typing.py b/Lib/typing.py index 7ca080402e28..56126cfc0247 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -95,6 +95,7 @@ 'NewType', 'no_type_check', 'no_type_check_decorator', + 'NoReturn', 'overload', 'Text', 'TYPE_CHECKING', diff --git a/Misc/NEWS.d/next/Library/2018-03-16-16-07-33.bpo-33061.TRTTek.rst b/Misc/NEWS.d/next/Library/2018-03-16-16-07-33.bpo-33061.TRTTek.rst new file mode 100644 index 000000000000..b82ffb73a143 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-03-16-16-07-33.bpo-33061.TRTTek.rst @@ -0,0 +1 @@ +Add missing ``NoReturn`` to ``__all__`` in typing.py From webhook-mailer at python.org Mon Mar 19 21:07:54 2018 From: webhook-mailer at python.org (Eric V. Smith) Date: Tue, 20 Mar 2018 01:07:54 -0000 Subject: [Python-checkins] bpo-33100: Dataclasses now handles __slots__ and default values correctly. (GH-6152) Message-ID: <mailman.102.1521508074.1871.python-checkins@python.org> https://github.com/python/cpython/commit/7389fd935c95b4b6f094312294e703ee0de18719 commit: 7389fd935c95b4b6f094312294e703ee0de18719 branch: master author: Eric V. Smith <ericvsmith at users.noreply.github.com> committer: GitHub <noreply at github.com> date: 2018-03-19T21:07:51-04:00 summary: bpo-33100: Dataclasses now handles __slots__ and default values correctly. (GH-6152) If the class has a member that's a MemberDescriptorType, it's not a default value, it's from that member being in __slots__. files: A Misc/NEWS.d/next/Library/2018-03-19-20-47-00.bpo-33100.chyIO4.rst M Lib/dataclasses.py M Lib/test/test_dataclasses.py diff --git a/Lib/dataclasses.py b/Lib/dataclasses.py index 8ab04dd5b975..a4afd50376bd 100644 --- a/Lib/dataclasses.py +++ b/Lib/dataclasses.py @@ -519,6 +519,9 @@ def _get_field(cls, a_name, a_type): if isinstance(default, Field): f = default else: + if isinstance(default, types.MemberDescriptorType): + # This is a field in __slots__, so it has no default value. + default = MISSING f = field(default=default) # Assume it's a normal field until proven otherwise. diff --git a/Lib/test/test_dataclasses.py b/Lib/test/test_dataclasses.py index 3e6726360940..db03ec1925f6 100755 --- a/Lib/test/test_dataclasses.py +++ b/Lib/test/test_dataclasses.py @@ -2564,5 +2564,47 @@ class S(D): self.assertEqual(s.cached, True) +class TestSlots(unittest.TestCase): + def test_simple(self): + @dataclass + class C: + __slots__ = ('x',) + x: Any + + # There was a bug where a variable in a slot was assumed + # to also have a default value (of type types.MemberDescriptorType). + with self.assertRaisesRegex(TypeError, + "__init__\(\) missing 1 required positional argument: 'x'"): + C() + + # We can create an instance, and assign to x. + c = C(10) + self.assertEqual(c.x, 10) + c.x = 5 + self.assertEqual(c.x, 5) + + # We can't assign to anything else. + with self.assertRaisesRegex(AttributeError, "'C' object has no attribute 'y'"): + c.y = 5 + + def test_derived_added_field(self): + # See bpo-33100. + @dataclass + class Base: + __slots__ = ('x',) + x: Any + + @dataclass + class Derived(Base): + x: int + y: int + + d = Derived(1, 2) + self.assertEqual((d.x, d.y), (1, 2)) + + # We can add a new field to the derived instance. + d.z = 10 + + if __name__ == '__main__': unittest.main() diff --git a/Misc/NEWS.d/next/Library/2018-03-19-20-47-00.bpo-33100.chyIO4.rst b/Misc/NEWS.d/next/Library/2018-03-19-20-47-00.bpo-33100.chyIO4.rst new file mode 100644 index 000000000000..080a55c0cfb7 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-03-19-20-47-00.bpo-33100.chyIO4.rst @@ -0,0 +1,2 @@ +Dataclasses: If a field has a default value that's a MemberDescriptorType, +then it's from that field being in __slots__, not an actual default value. From webhook-mailer at python.org Mon Mar 19 21:31:25 2018 From: webhook-mailer at python.org (Eric V. Smith) Date: Tue, 20 Mar 2018 01:31:25 -0000 Subject: [Python-checkins] bpo-33100: Dataclasses now handles __slots__ and default values correctly. (GH-6152) (GH-6153) Message-ID: <mailman.103.1521509486.1871.python-checkins@python.org> https://github.com/python/cpython/commit/3d41f482594b6aab12a316202b3c06757262109a commit: 3d41f482594b6aab12a316202b3c06757262109a branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Eric V. Smith <ericvsmith at users.noreply.github.com> date: 2018-03-19T21:31:22-04:00 summary: bpo-33100: Dataclasses now handles __slots__ and default values correctly. (GH-6152) (GH-6153) If the class has a member that's a MemberDescriptorType, it's not a default value, it's from that member being in __slots__. (cherry picked from commit 7389fd935c95b4b6f094312294e703ee0de18719) Co-authored-by: Eric V. Smith <ericvsmith at users.noreply.github.com> files: A Misc/NEWS.d/next/Library/2018-03-19-20-47-00.bpo-33100.chyIO4.rst M Lib/dataclasses.py M Lib/test/test_dataclasses.py diff --git a/Lib/dataclasses.py b/Lib/dataclasses.py index 8ab04dd5b975..a4afd50376bd 100644 --- a/Lib/dataclasses.py +++ b/Lib/dataclasses.py @@ -519,6 +519,9 @@ def _get_field(cls, a_name, a_type): if isinstance(default, Field): f = default else: + if isinstance(default, types.MemberDescriptorType): + # This is a field in __slots__, so it has no default value. + default = MISSING f = field(default=default) # Assume it's a normal field until proven otherwise. diff --git a/Lib/test/test_dataclasses.py b/Lib/test/test_dataclasses.py index 3e6726360940..db03ec1925f6 100755 --- a/Lib/test/test_dataclasses.py +++ b/Lib/test/test_dataclasses.py @@ -2564,5 +2564,47 @@ class S(D): self.assertEqual(s.cached, True) +class TestSlots(unittest.TestCase): + def test_simple(self): + @dataclass + class C: + __slots__ = ('x',) + x: Any + + # There was a bug where a variable in a slot was assumed + # to also have a default value (of type types.MemberDescriptorType). + with self.assertRaisesRegex(TypeError, + "__init__\(\) missing 1 required positional argument: 'x'"): + C() + + # We can create an instance, and assign to x. + c = C(10) + self.assertEqual(c.x, 10) + c.x = 5 + self.assertEqual(c.x, 5) + + # We can't assign to anything else. + with self.assertRaisesRegex(AttributeError, "'C' object has no attribute 'y'"): + c.y = 5 + + def test_derived_added_field(self): + # See bpo-33100. + @dataclass + class Base: + __slots__ = ('x',) + x: Any + + @dataclass + class Derived(Base): + x: int + y: int + + d = Derived(1, 2) + self.assertEqual((d.x, d.y), (1, 2)) + + # We can add a new field to the derived instance. + d.z = 10 + + if __name__ == '__main__': unittest.main() diff --git a/Misc/NEWS.d/next/Library/2018-03-19-20-47-00.bpo-33100.chyIO4.rst b/Misc/NEWS.d/next/Library/2018-03-19-20-47-00.bpo-33100.chyIO4.rst new file mode 100644 index 000000000000..080a55c0cfb7 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-03-19-20-47-00.bpo-33100.chyIO4.rst @@ -0,0 +1,2 @@ +Dataclasses: If a field has a default value that's a MemberDescriptorType, +then it's from that field being in __slots__, not an actual default value. From webhook-mailer at python.org Tue Mar 20 02:41:39 2018 From: webhook-mailer at python.org (Berker Peksag) Date: Tue, 20 Mar 2018 06:41:39 -0000 Subject: [Python-checkins] bpo-33034: Improve exception message when cast fails for {Parse, Split}Result.port (GH-6078) Message-ID: <mailman.104.1521528100.1871.python-checkins@python.org> https://github.com/python/cpython/commit/2cb4661707818cfd92556e7fdf9068a993577002 commit: 2cb4661707818cfd92556e7fdf9068a993577002 branch: master author: Matt Eaton <agnosticdev at gmail.com> committer: Berker Peksag <berker.peksag at gmail.com> date: 2018-03-20T09:41:37+03:00 summary: bpo-33034: Improve exception message when cast fails for {Parse,Split}Result.port (GH-6078) files: A Misc/NEWS.d/next/Library/2018-03-11-08-44-12.bpo-33034.bpb23d.rst M Lib/test/test_urlparse.py M Lib/urllib/parse.py diff --git a/Lib/test/test_urlparse.py b/Lib/test/test_urlparse.py index ddee1c38d8b4..9d5121792833 100644 --- a/Lib/test/test_urlparse.py +++ b/Lib/test/test_urlparse.py @@ -936,6 +936,16 @@ def test_issue14072(self): self.assertEqual(p2.scheme, 'tel') self.assertEqual(p2.path, '+31641044153') + def test_port_casting_failure_message(self): + message = "Port could not be cast to integer value as 'oracle'" + p1 = urllib.parse.urlparse('http://Server=sde; Service=sde:oracle') + with self.assertRaisesRegex(ValueError, message): + p1.port + + p2 = urllib.parse.urlsplit('http://Server=sde; Service=sde:oracle') + with self.assertRaisesRegex(ValueError, message): + p2.port + def test_telurl_params(self): p1 = urllib.parse.urlparse('tel:123-4;phone-context=+1-650-516') self.assertEqual(p1.scheme, 'tel') diff --git a/Lib/urllib/parse.py b/Lib/urllib/parse.py index 58460f9234fb..3f8cfe5300c0 100644 --- a/Lib/urllib/parse.py +++ b/Lib/urllib/parse.py @@ -166,7 +166,11 @@ def hostname(self): def port(self): port = self._hostinfo[1] if port is not None: - port = int(port, 10) + try: + port = int(port, 10) + except ValueError: + message = f'Port could not be cast to integer value as {port!r}' + raise ValueError(message) from None if not ( 0 <= port <= 65535): raise ValueError("Port out of range 0-65535") return port diff --git a/Misc/NEWS.d/next/Library/2018-03-11-08-44-12.bpo-33034.bpb23d.rst b/Misc/NEWS.d/next/Library/2018-03-11-08-44-12.bpo-33034.bpb23d.rst new file mode 100644 index 000000000000..c81683abf592 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-03-11-08-44-12.bpo-33034.bpb23d.rst @@ -0,0 +1,3 @@ +Providing an explicit error message when casting the port property to anything +that is not an integer value using ``urlparse()`` and ``urlsplit()``. +Patch by Matt Eaton. From solipsis at pitrou.net Tue Mar 20 05:16:09 2018 From: solipsis at pitrou.net (solipsis at pitrou.net) Date: Tue, 20 Mar 2018 09:16:09 +0000 Subject: [Python-checkins] Daily reference leaks (4243df51fe43): sum=4 Message-ID: <20180320091609.1.DE6DDA2F4132C4B8@psf.io> results for 4243df51fe43 on branch "default" -------------------------------------------- test_functools leaked [0, 3, 1] memory blocks, sum=4 Command line was: ['./python', '-m', 'test.regrtest', '-uall', '-R', '3:3:/home/psf-users/antoine/refleaks/reflogpAsooi', '--timeout', '7200'] From webhook-mailer at python.org Tue Mar 20 15:16:33 2018 From: webhook-mailer at python.org (Antoine Pitrou) Date: Tue, 20 Mar 2018 19:16:33 -0000 Subject: [Python-checkins] [3.7] bpo-33021: Release the GIL during fstat() calls (GH-6019) (GH-6159) Message-ID: <mailman.105.1521573394.1871.python-checkins@python.org> https://github.com/python/cpython/commit/56cce1ca84344f519f7ed01024434c083031d354 commit: 56cce1ca84344f519f7ed01024434c083031d354 branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Antoine Pitrou <pitrou at free.fr> date: 2018-03-20T20:16:30+01:00 summary: [3.7] bpo-33021: Release the GIL during fstat() calls (GH-6019) (GH-6159) fstat may block for long time if the file descriptor is on a non-responsive NFS server, hanging all threads. Most fstat() calls are handled by _Py_fstat(), releasing the GIL internally, but but _Py_fstat_noraise() does not release the GIL, and most calls release the GIL explicitly around it. This patch fixes last 2 calls to _Py_fstat_no_raise(), avoiding hangs when calling: - mmap.mmap() - os.urandom() - random.seed() (cherry picked from commit 4484f9dca9149da135bbae035f10a50d20d1cbbb) Co-authored-by: Nir Soffer <nirsof at gmail.com> files: A Misc/NEWS.d/next/Library/2018-03-12-00-27-56.bpo-33021.m19B9T.rst M Modules/mmapmodule.c M Python/bootstrap_hash.c diff --git a/Misc/NEWS.d/next/Library/2018-03-12-00-27-56.bpo-33021.m19B9T.rst b/Misc/NEWS.d/next/Library/2018-03-12-00-27-56.bpo-33021.m19B9T.rst new file mode 100644 index 000000000000..50dfafe600d8 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-03-12-00-27-56.bpo-33021.m19B9T.rst @@ -0,0 +1,2 @@ +Release the GIL during fstat() calls, avoiding hang of all threads when +calling mmap.mmap(), os.urandom(), and random.seed(). Patch by Nir Soffer. diff --git a/Modules/mmapmodule.c b/Modules/mmapmodule.c index 6cf454573e93..95d42d6dc5e5 100644 --- a/Modules/mmapmodule.c +++ b/Modules/mmapmodule.c @@ -1050,6 +1050,7 @@ static PyObject * new_mmap_object(PyTypeObject *type, PyObject *args, PyObject *kwdict) { struct _Py_stat_struct status; + int fstat_result = -1; mmap_object *m_obj; Py_ssize_t map_size; off_t offset = 0; @@ -1115,8 +1116,14 @@ new_mmap_object(PyTypeObject *type, PyObject *args, PyObject *kwdict) if (fd != -1) (void)fcntl(fd, F_FULLFSYNC); #endif - if (fd != -1 && _Py_fstat_noraise(fd, &status) == 0 - && S_ISREG(status.st_mode)) { + + if (fd != -1) { + Py_BEGIN_ALLOW_THREADS + fstat_result = _Py_fstat_noraise(fd, &status); + Py_END_ALLOW_THREADS + } + + if (fd != -1 && fstat_result == 0 && S_ISREG(status.st_mode)) { if (map_size == 0) { if (status.st_size == 0) { PyErr_SetString(PyExc_ValueError, diff --git a/Python/bootstrap_hash.c b/Python/bootstrap_hash.c index 9fd5cfb200ea..073b2ea8dbda 100644 --- a/Python/bootstrap_hash.c +++ b/Python/bootstrap_hash.c @@ -301,10 +301,15 @@ dev_urandom(char *buffer, Py_ssize_t size, int raise) if (raise) { struct _Py_stat_struct st; + int fstat_result; if (urandom_cache.fd >= 0) { + Py_BEGIN_ALLOW_THREADS + fstat_result = _Py_fstat_noraise(urandom_cache.fd, &st); + Py_END_ALLOW_THREADS + /* Does the fd point to the same thing as before? (issue #21207) */ - if (_Py_fstat_noraise(urandom_cache.fd, &st) + if (fstat_result || st.st_dev != urandom_cache.st_dev || st.st_ino != urandom_cache.st_ino) { /* Something changed: forget the cached fd (but don't close it, From webhook-mailer at python.org Tue Mar 20 15:40:21 2018 From: webhook-mailer at python.org (Antoine Pitrou) Date: Tue, 20 Mar 2018 19:40:21 -0000 Subject: [Python-checkins] [3.6] bpo-33021: Release the GIL during fstat() calls (GH-6019) (GH-6160) Message-ID: <mailman.106.1521574824.1871.python-checkins@python.org> https://github.com/python/cpython/commit/f3e6eadbcf4f3e0fe53f4784485b1c8464c7d282 commit: f3e6eadbcf4f3e0fe53f4784485b1c8464c7d282 branch: 3.6 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Antoine Pitrou <pitrou at free.fr> date: 2018-03-20T20:40:18+01:00 summary: [3.6] bpo-33021: Release the GIL during fstat() calls (GH-6019) (GH-6160) fstat may block for long time if the file descriptor is on a non-responsive NFS server, hanging all threads. Most fstat() calls are handled by _Py_fstat(), releasing the GIL internally, but but _Py_fstat_noraise() does not release the GIL, and most calls release the GIL explicitly around it. This patch fixes last 2 calls to _Py_fstat_no_raise(), avoiding hangs when calling: - mmap.mmap() - os.urandom() - random.seed() (cherry picked from commit 4484f9dca9149da135bbae035f10a50d20d1cbbb) Co-authored-by: Nir Soffer <nirsof at gmail.com> files: A Misc/NEWS.d/next/Library/2018-03-12-00-27-56.bpo-33021.m19B9T.rst M Modules/mmapmodule.c M Python/random.c diff --git a/Misc/NEWS.d/next/Library/2018-03-12-00-27-56.bpo-33021.m19B9T.rst b/Misc/NEWS.d/next/Library/2018-03-12-00-27-56.bpo-33021.m19B9T.rst new file mode 100644 index 000000000000..50dfafe600d8 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-03-12-00-27-56.bpo-33021.m19B9T.rst @@ -0,0 +1,2 @@ +Release the GIL during fstat() calls, avoiding hang of all threads when +calling mmap.mmap(), os.urandom(), and random.seed(). Patch by Nir Soffer. diff --git a/Modules/mmapmodule.c b/Modules/mmapmodule.c index 426b7cacebb5..8acb61ab98a9 100644 --- a/Modules/mmapmodule.c +++ b/Modules/mmapmodule.c @@ -1078,6 +1078,7 @@ static PyObject * new_mmap_object(PyTypeObject *type, PyObject *args, PyObject *kwdict) { struct _Py_stat_struct status; + int fstat_result = -1; mmap_object *m_obj; Py_ssize_t map_size; off_t offset = 0; @@ -1143,8 +1144,14 @@ new_mmap_object(PyTypeObject *type, PyObject *args, PyObject *kwdict) if (fd != -1) (void)fcntl(fd, F_FULLFSYNC); #endif - if (fd != -1 && _Py_fstat_noraise(fd, &status) == 0 - && S_ISREG(status.st_mode)) { + + if (fd != -1) { + Py_BEGIN_ALLOW_THREADS + fstat_result = _Py_fstat_noraise(fd, &status); + Py_END_ALLOW_THREADS + } + + if (fd != -1 && fstat_result == 0 && S_ISREG(status.st_mode)) { if (map_size == 0) { if (status.st_size == 0) { PyErr_SetString(PyExc_ValueError, diff --git a/Python/random.c b/Python/random.c index e0ee153ec4a2..c62cbeb4ecb7 100644 --- a/Python/random.c +++ b/Python/random.c @@ -301,10 +301,15 @@ dev_urandom(char *buffer, Py_ssize_t size, int raise) if (raise) { struct _Py_stat_struct st; + int fstat_result; if (urandom_cache.fd >= 0) { + Py_BEGIN_ALLOW_THREADS + fstat_result = _Py_fstat_noraise(urandom_cache.fd, &st); + Py_END_ALLOW_THREADS + /* Does the fd point to the same thing as before? (issue #21207) */ - if (_Py_fstat_noraise(urandom_cache.fd, &st) + if (fstat_result || st.st_dev != urandom_cache.st_dev || st.st_ino != urandom_cache.st_ino) { /* Something changed: forget the cached fd (but don't close it, From webhook-mailer at python.org Tue Mar 20 18:23:22 2018 From: webhook-mailer at python.org (Paul Moore) Date: Tue, 20 Mar 2018 22:23:22 -0000 Subject: [Python-checkins] bpo-28247: Document Windows executable creation in zipapp (GH-6158) Message-ID: <mailman.107.1521584603.1871.python-checkins@python.org> https://github.com/python/cpython/commit/4be79f29463f632cd8b48486feadc2ed308fb520 commit: 4be79f29463f632cd8b48486feadc2ed308fb520 branch: master author: Cheryl Sabella <cheryl.sabella at gmail.com> committer: Paul Moore <p.f.moore at gmail.com> date: 2018-03-20T22:23:19Z summary: bpo-28247: Document Windows executable creation in zipapp (GH-6158) files: A Misc/NEWS.d/next/Documentation/2018-03-20-20-11-05.bpo-28247.-V-WS-.rst M Doc/library/zipapp.rst diff --git a/Doc/library/zipapp.rst b/Doc/library/zipapp.rst index 9076593c4b1c..a9a61c969c6f 100644 --- a/Doc/library/zipapp.rst +++ b/Doc/library/zipapp.rst @@ -229,6 +229,12 @@ fits in memory:: >>> with open('myapp.pyz', 'wb') as f: >>> f.write(temp.getvalue()) + +.. _zipapp-specifying-the-interpreter: + +Specifying the Interpreter +-------------------------- + Note that if you specify an interpreter and then distribute your application archive, you need to ensure that the interpreter used is portable. The Python launcher for Windows supports most common forms of POSIX ``#!`` line, but there @@ -245,6 +251,169 @@ are other issues to consider: exact version like "/usr/bin/env python3.4" as you will need to change your shebang line for users of Python 3.5, for example. +Typically, you should use an "/usr/bin/env python2" or "/usr/bin/env python3", +depending on whether your code is written for Python 2 or 3. + + +Creating Standalone Applications with zipapp +-------------------------------------------- + +Using the :mod:`zipapp` module, it is possible to create self-contained Python +programs, which can be distributed to end users who only need to have a +suitable version of Python installed on their system. The key to doing this +is to bundle all of the application's dependencies into the archive, along +with the application code. + +The steps to create a standalone archive are as follows: + +1. Create your application in a directory as normal, so you have a ``myapp`` + directory containing a ``__main__.py`` file, and any supporting application + code. + +2. Install all of your application's dependencies into the ``myapp`` directory, + using pip: + + .. code-block:: sh + + $ python -m pip install -r requirements.txt --target myapp + + (this assumes you have your project requirements in a ``requirements.txt`` + file - if not, you can just list the dependencies manually on the pip command + line). + +3. Optionally, delete the ``.dist-info`` directories created by pip in the + ``myapp`` directory. These hold metadata for pip to manage the packages, and + as you won't be making any further use of pip they aren't required - + although it won't do any harm if you leave them. + +4. Package the application using: + + .. code-block:: sh + + $ python -m zipapp -p "interpreter" myapp + +This will produce a standalone executable, which can be run on any machine with +the appropriate interpreter available. See :ref:`zipapp-specifying-the-interpreter` +for details. It can be shipped to users as a single file. + +On Unix, the ``myapp.pyz`` file is executable as it stands. You can rename the +file to remove the ``.pyz`` extension if you prefer a "plain" command name. On +Windows, the ``myapp.pyz[w]`` file is executable by virtue of the fact that +the Python interpreter registers the ``.pyz`` and ``.pyzw`` file extensions +when installed. + + +Making a Windows executable +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +On Windows, registration of the ``.pyz`` extension is optional, and +furthermore, there are certain places that don't recognise registered +extensions "transparently" (the simplest example is that +``subprocess.run(['myapp'])`` won't find your application - you need to +explicitly specify the extension). + +On Windows, therefore, it is often preferable to create an executable from the +zipapp. This is relatively easy, although it does require a C compiler. The +basic approach relies on the fact that zipfiles can have arbitrary data +prepended, and Windows exe files can have arbitrary data appended. So by +creating a suitable launcher and tacking the ``.pyz`` file onto the end of it, +you end up with a single-file executable that runs your application. + +A suitable launcher can be as simple as the following:: + + #define Py_LIMITED_API 1 + #include "Python.h" + + #define WIN32_LEAN_AND_MEAN + #include <windows.h> + + #ifdef WINDOWS + int WINAPI wWinMain( + HINSTANCE hInstance, /* handle to current instance */ + HINSTANCE hPrevInstance, /* handle to previous instance */ + LPWSTR lpCmdLine, /* pointer to command line */ + int nCmdShow /* show state of window */ + ) + #else + int wmain() + #endif + { + wchar_t **myargv = _alloca((__argc + 1) * sizeof(wchar_t*)); + myargv[0] = __wargv[0]; + memcpy(myargv + 1, __wargv, __argc * sizeof(wchar_t *)); + return Py_Main(__argc+1, myargv); + } + +If you define the ``WINDOWS`` preprocessor symbol, this will generate a +GUI executable, and without it, a console executable. + +To compile the executable, you can either just use the standard MSVC +command line tools, or you can take advantage of the fact that distutils +knows how to compile Python source:: + + >>> from distutils.ccompiler import new_compiler + >>> import distutils.sysconfig + >>> import sys + >>> import os + >>> from pathlib import Path + + >>> def compile(src): + >>> src = Path(src) + >>> cc = new_compiler() + >>> exe = src.stem + >>> cc.add_include_dir(distutils.sysconfig.get_python_inc()) + >>> cc.add_library_dir(os.path.join(sys.base_exec_prefix, 'libs')) + >>> # First the CLI executable + >>> objs = cc.compile([str(src)]) + >>> cc.link_executable(objs, exe) + >>> # Now the GUI executable + >>> cc.define_macro('WINDOWS') + >>> objs = cc.compile([str(src)]) + >>> cc.link_executable(objs, exe + 'w') + + >>> if __name__ == "__main__": + >>> compile("zastub.c") + +The resulting launcher uses the "Limited ABI", so it will run unchanged with +any version of Python 3.x. All it needs is for Python (``python3.dll``) to be +on the user's ``PATH``. + +For a fully standalone distribution, you can distribute the launcher with your +application appended, bundled with the Python "embedded" distribution. This +will run on any PC with the appropriate architecture (32 bit or 64 bit). + + +Caveats +~~~~~~~ + +There are some limitations to the process of bundling your application into +a single file. In most, if not all, cases they can be addressed without +needing major changes to your application. + +1. If your application depends on a package that includes a C extension, that + package cannot be run from a zip file (this is an OS limitation, as executable + code must be present in the filesystem for the OS loader to load it). In this + case, you can exclude that dependency from the zipfile, and either require + your users to have it installed, or ship it alongside your zipfile and add code + to your ``__main__.py`` to include the directory containing the unzipped + module in ``sys.path``. In this case, you will need to make sure to ship + appropriate binaries for your target architecture(s) (and potentially pick the + correct version to add to ``sys.path`` at runtime, based on the user's machine). + +2. If you are shipping a Windows executable as described above, you either need to + ensure that your users have ``python3.dll`` on their PATH (which is not the + default behaviour of the installer) or you should bundle your application with + the embedded distribution. + +3. The suggested launcher above uses the Python embedding API. This means that in + your application, ``sys.executable`` will be your application, and *not* a + conventional Python interpreter. Your code and its dependencies need to be + prepared for this possibility. For example, if your application uses the + :mod:`multiprocessing` module, it will need to call + :func:`multiprocessing.set_executable` to let the module know where to find the + standard Python interpreter. + + The Python Zip Application Archive Format ----------------------------------------- diff --git a/Misc/NEWS.d/next/Documentation/2018-03-20-20-11-05.bpo-28247.-V-WS-.rst b/Misc/NEWS.d/next/Documentation/2018-03-20-20-11-05.bpo-28247.-V-WS-.rst new file mode 100644 index 000000000000..28a802136fa7 --- /dev/null +++ b/Misc/NEWS.d/next/Documentation/2018-03-20-20-11-05.bpo-28247.-V-WS-.rst @@ -0,0 +1,2 @@ +Update :mod:`zipapp` documentation to describe how to make standalone +applications. From webhook-mailer at python.org Tue Mar 20 18:44:24 2018 From: webhook-mailer at python.org (Miss Islington (bot)) Date: Tue, 20 Mar 2018 22:44:24 -0000 Subject: [Python-checkins] bpo-28247: Document Windows executable creation in zipapp (GH-6158) Message-ID: <mailman.108.1521585866.1871.python-checkins@python.org> https://github.com/python/cpython/commit/a70b8f593657f563116001f654587620271eb9ae commit: a70b8f593657f563116001f654587620271eb9ae branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: GitHub <noreply at github.com> date: 2018-03-20T15:44:21-07:00 summary: bpo-28247: Document Windows executable creation in zipapp (GH-6158) (cherry picked from commit 4be79f29463f632cd8b48486feadc2ed308fb520) Co-authored-by: Cheryl Sabella <cheryl.sabella at gmail.com> files: A Misc/NEWS.d/next/Documentation/2018-03-20-20-11-05.bpo-28247.-V-WS-.rst M Doc/library/zipapp.rst diff --git a/Doc/library/zipapp.rst b/Doc/library/zipapp.rst index 9076593c4b1c..a9a61c969c6f 100644 --- a/Doc/library/zipapp.rst +++ b/Doc/library/zipapp.rst @@ -229,6 +229,12 @@ fits in memory:: >>> with open('myapp.pyz', 'wb') as f: >>> f.write(temp.getvalue()) + +.. _zipapp-specifying-the-interpreter: + +Specifying the Interpreter +-------------------------- + Note that if you specify an interpreter and then distribute your application archive, you need to ensure that the interpreter used is portable. The Python launcher for Windows supports most common forms of POSIX ``#!`` line, but there @@ -245,6 +251,169 @@ are other issues to consider: exact version like "/usr/bin/env python3.4" as you will need to change your shebang line for users of Python 3.5, for example. +Typically, you should use an "/usr/bin/env python2" or "/usr/bin/env python3", +depending on whether your code is written for Python 2 or 3. + + +Creating Standalone Applications with zipapp +-------------------------------------------- + +Using the :mod:`zipapp` module, it is possible to create self-contained Python +programs, which can be distributed to end users who only need to have a +suitable version of Python installed on their system. The key to doing this +is to bundle all of the application's dependencies into the archive, along +with the application code. + +The steps to create a standalone archive are as follows: + +1. Create your application in a directory as normal, so you have a ``myapp`` + directory containing a ``__main__.py`` file, and any supporting application + code. + +2. Install all of your application's dependencies into the ``myapp`` directory, + using pip: + + .. code-block:: sh + + $ python -m pip install -r requirements.txt --target myapp + + (this assumes you have your project requirements in a ``requirements.txt`` + file - if not, you can just list the dependencies manually on the pip command + line). + +3. Optionally, delete the ``.dist-info`` directories created by pip in the + ``myapp`` directory. These hold metadata for pip to manage the packages, and + as you won't be making any further use of pip they aren't required - + although it won't do any harm if you leave them. + +4. Package the application using: + + .. code-block:: sh + + $ python -m zipapp -p "interpreter" myapp + +This will produce a standalone executable, which can be run on any machine with +the appropriate interpreter available. See :ref:`zipapp-specifying-the-interpreter` +for details. It can be shipped to users as a single file. + +On Unix, the ``myapp.pyz`` file is executable as it stands. You can rename the +file to remove the ``.pyz`` extension if you prefer a "plain" command name. On +Windows, the ``myapp.pyz[w]`` file is executable by virtue of the fact that +the Python interpreter registers the ``.pyz`` and ``.pyzw`` file extensions +when installed. + + +Making a Windows executable +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +On Windows, registration of the ``.pyz`` extension is optional, and +furthermore, there are certain places that don't recognise registered +extensions "transparently" (the simplest example is that +``subprocess.run(['myapp'])`` won't find your application - you need to +explicitly specify the extension). + +On Windows, therefore, it is often preferable to create an executable from the +zipapp. This is relatively easy, although it does require a C compiler. The +basic approach relies on the fact that zipfiles can have arbitrary data +prepended, and Windows exe files can have arbitrary data appended. So by +creating a suitable launcher and tacking the ``.pyz`` file onto the end of it, +you end up with a single-file executable that runs your application. + +A suitable launcher can be as simple as the following:: + + #define Py_LIMITED_API 1 + #include "Python.h" + + #define WIN32_LEAN_AND_MEAN + #include <windows.h> + + #ifdef WINDOWS + int WINAPI wWinMain( + HINSTANCE hInstance, /* handle to current instance */ + HINSTANCE hPrevInstance, /* handle to previous instance */ + LPWSTR lpCmdLine, /* pointer to command line */ + int nCmdShow /* show state of window */ + ) + #else + int wmain() + #endif + { + wchar_t **myargv = _alloca((__argc + 1) * sizeof(wchar_t*)); + myargv[0] = __wargv[0]; + memcpy(myargv + 1, __wargv, __argc * sizeof(wchar_t *)); + return Py_Main(__argc+1, myargv); + } + +If you define the ``WINDOWS`` preprocessor symbol, this will generate a +GUI executable, and without it, a console executable. + +To compile the executable, you can either just use the standard MSVC +command line tools, or you can take advantage of the fact that distutils +knows how to compile Python source:: + + >>> from distutils.ccompiler import new_compiler + >>> import distutils.sysconfig + >>> import sys + >>> import os + >>> from pathlib import Path + + >>> def compile(src): + >>> src = Path(src) + >>> cc = new_compiler() + >>> exe = src.stem + >>> cc.add_include_dir(distutils.sysconfig.get_python_inc()) + >>> cc.add_library_dir(os.path.join(sys.base_exec_prefix, 'libs')) + >>> # First the CLI executable + >>> objs = cc.compile([str(src)]) + >>> cc.link_executable(objs, exe) + >>> # Now the GUI executable + >>> cc.define_macro('WINDOWS') + >>> objs = cc.compile([str(src)]) + >>> cc.link_executable(objs, exe + 'w') + + >>> if __name__ == "__main__": + >>> compile("zastub.c") + +The resulting launcher uses the "Limited ABI", so it will run unchanged with +any version of Python 3.x. All it needs is for Python (``python3.dll``) to be +on the user's ``PATH``. + +For a fully standalone distribution, you can distribute the launcher with your +application appended, bundled with the Python "embedded" distribution. This +will run on any PC with the appropriate architecture (32 bit or 64 bit). + + +Caveats +~~~~~~~ + +There are some limitations to the process of bundling your application into +a single file. In most, if not all, cases they can be addressed without +needing major changes to your application. + +1. If your application depends on a package that includes a C extension, that + package cannot be run from a zip file (this is an OS limitation, as executable + code must be present in the filesystem for the OS loader to load it). In this + case, you can exclude that dependency from the zipfile, and either require + your users to have it installed, or ship it alongside your zipfile and add code + to your ``__main__.py`` to include the directory containing the unzipped + module in ``sys.path``. In this case, you will need to make sure to ship + appropriate binaries for your target architecture(s) (and potentially pick the + correct version to add to ``sys.path`` at runtime, based on the user's machine). + +2. If you are shipping a Windows executable as described above, you either need to + ensure that your users have ``python3.dll`` on their PATH (which is not the + default behaviour of the installer) or you should bundle your application with + the embedded distribution. + +3. The suggested launcher above uses the Python embedding API. This means that in + your application, ``sys.executable`` will be your application, and *not* a + conventional Python interpreter. Your code and its dependencies need to be + prepared for this possibility. For example, if your application uses the + :mod:`multiprocessing` module, it will need to call + :func:`multiprocessing.set_executable` to let the module know where to find the + standard Python interpreter. + + The Python Zip Application Archive Format ----------------------------------------- diff --git a/Misc/NEWS.d/next/Documentation/2018-03-20-20-11-05.bpo-28247.-V-WS-.rst b/Misc/NEWS.d/next/Documentation/2018-03-20-20-11-05.bpo-28247.-V-WS-.rst new file mode 100644 index 000000000000..28a802136fa7 --- /dev/null +++ b/Misc/NEWS.d/next/Documentation/2018-03-20-20-11-05.bpo-28247.-V-WS-.rst @@ -0,0 +1,2 @@ +Update :mod:`zipapp` documentation to describe how to make standalone +applications. From webhook-mailer at python.org Tue Mar 20 18:52:44 2018 From: webhook-mailer at python.org (Miss Islington (bot)) Date: Tue, 20 Mar 2018 22:52:44 -0000 Subject: [Python-checkins] bpo-28247: Document Windows executable creation in zipapp (GH-6158) Message-ID: <mailman.109.1521586366.1871.python-checkins@python.org> https://github.com/python/cpython/commit/47a0e64ccf711e8d6d0c497d565ca7e2e82de7d4 commit: 47a0e64ccf711e8d6d0c497d565ca7e2e82de7d4 branch: 3.6 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: GitHub <noreply at github.com> date: 2018-03-20T15:52:41-07:00 summary: bpo-28247: Document Windows executable creation in zipapp (GH-6158) (cherry picked from commit 4be79f29463f632cd8b48486feadc2ed308fb520) Co-authored-by: Cheryl Sabella <cheryl.sabella at gmail.com> files: A Misc/NEWS.d/next/Documentation/2018-03-20-20-11-05.bpo-28247.-V-WS-.rst M Doc/library/zipapp.rst diff --git a/Doc/library/zipapp.rst b/Doc/library/zipapp.rst index 9dee9a5ac07e..abd1e9ebac94 100644 --- a/Doc/library/zipapp.rst +++ b/Doc/library/zipapp.rst @@ -207,6 +207,12 @@ fits in memory:: >>> with open('myapp.pyz', 'wb') as f: >>> f.write(temp.getvalue()) + +.. _zipapp-specifying-the-interpreter: + +Specifying the Interpreter +-------------------------- + Note that if you specify an interpreter and then distribute your application archive, you need to ensure that the interpreter used is portable. The Python launcher for Windows supports most common forms of POSIX ``#!`` line, but there @@ -223,6 +229,169 @@ are other issues to consider: exact version like "/usr/bin/env python3.4" as you will need to change your shebang line for users of Python 3.5, for example. +Typically, you should use an "/usr/bin/env python2" or "/usr/bin/env python3", +depending on whether your code is written for Python 2 or 3. + + +Creating Standalone Applications with zipapp +-------------------------------------------- + +Using the :mod:`zipapp` module, it is possible to create self-contained Python +programs, which can be distributed to end users who only need to have a +suitable version of Python installed on their system. The key to doing this +is to bundle all of the application's dependencies into the archive, along +with the application code. + +The steps to create a standalone archive are as follows: + +1. Create your application in a directory as normal, so you have a ``myapp`` + directory containing a ``__main__.py`` file, and any supporting application + code. + +2. Install all of your application's dependencies into the ``myapp`` directory, + using pip: + + .. code-block:: sh + + $ python -m pip install -r requirements.txt --target myapp + + (this assumes you have your project requirements in a ``requirements.txt`` + file - if not, you can just list the dependencies manually on the pip command + line). + +3. Optionally, delete the ``.dist-info`` directories created by pip in the + ``myapp`` directory. These hold metadata for pip to manage the packages, and + as you won't be making any further use of pip they aren't required - + although it won't do any harm if you leave them. + +4. Package the application using: + + .. code-block:: sh + + $ python -m zipapp -p "interpreter" myapp + +This will produce a standalone executable, which can be run on any machine with +the appropriate interpreter available. See :ref:`zipapp-specifying-the-interpreter` +for details. It can be shipped to users as a single file. + +On Unix, the ``myapp.pyz`` file is executable as it stands. You can rename the +file to remove the ``.pyz`` extension if you prefer a "plain" command name. On +Windows, the ``myapp.pyz[w]`` file is executable by virtue of the fact that +the Python interpreter registers the ``.pyz`` and ``.pyzw`` file extensions +when installed. + + +Making a Windows executable +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +On Windows, registration of the ``.pyz`` extension is optional, and +furthermore, there are certain places that don't recognise registered +extensions "transparently" (the simplest example is that +``subprocess.run(['myapp'])`` won't find your application - you need to +explicitly specify the extension). + +On Windows, therefore, it is often preferable to create an executable from the +zipapp. This is relatively easy, although it does require a C compiler. The +basic approach relies on the fact that zipfiles can have arbitrary data +prepended, and Windows exe files can have arbitrary data appended. So by +creating a suitable launcher and tacking the ``.pyz`` file onto the end of it, +you end up with a single-file executable that runs your application. + +A suitable launcher can be as simple as the following:: + + #define Py_LIMITED_API 1 + #include "Python.h" + + #define WIN32_LEAN_AND_MEAN + #include <windows.h> + + #ifdef WINDOWS + int WINAPI wWinMain( + HINSTANCE hInstance, /* handle to current instance */ + HINSTANCE hPrevInstance, /* handle to previous instance */ + LPWSTR lpCmdLine, /* pointer to command line */ + int nCmdShow /* show state of window */ + ) + #else + int wmain() + #endif + { + wchar_t **myargv = _alloca((__argc + 1) * sizeof(wchar_t*)); + myargv[0] = __wargv[0]; + memcpy(myargv + 1, __wargv, __argc * sizeof(wchar_t *)); + return Py_Main(__argc+1, myargv); + } + +If you define the ``WINDOWS`` preprocessor symbol, this will generate a +GUI executable, and without it, a console executable. + +To compile the executable, you can either just use the standard MSVC +command line tools, or you can take advantage of the fact that distutils +knows how to compile Python source:: + + >>> from distutils.ccompiler import new_compiler + >>> import distutils.sysconfig + >>> import sys + >>> import os + >>> from pathlib import Path + + >>> def compile(src): + >>> src = Path(src) + >>> cc = new_compiler() + >>> exe = src.stem + >>> cc.add_include_dir(distutils.sysconfig.get_python_inc()) + >>> cc.add_library_dir(os.path.join(sys.base_exec_prefix, 'libs')) + >>> # First the CLI executable + >>> objs = cc.compile([str(src)]) + >>> cc.link_executable(objs, exe) + >>> # Now the GUI executable + >>> cc.define_macro('WINDOWS') + >>> objs = cc.compile([str(src)]) + >>> cc.link_executable(objs, exe + 'w') + + >>> if __name__ == "__main__": + >>> compile("zastub.c") + +The resulting launcher uses the "Limited ABI", so it will run unchanged with +any version of Python 3.x. All it needs is for Python (``python3.dll``) to be +on the user's ``PATH``. + +For a fully standalone distribution, you can distribute the launcher with your +application appended, bundled with the Python "embedded" distribution. This +will run on any PC with the appropriate architecture (32 bit or 64 bit). + + +Caveats +~~~~~~~ + +There are some limitations to the process of bundling your application into +a single file. In most, if not all, cases they can be addressed without +needing major changes to your application. + +1. If your application depends on a package that includes a C extension, that + package cannot be run from a zip file (this is an OS limitation, as executable + code must be present in the filesystem for the OS loader to load it). In this + case, you can exclude that dependency from the zipfile, and either require + your users to have it installed, or ship it alongside your zipfile and add code + to your ``__main__.py`` to include the directory containing the unzipped + module in ``sys.path``. In this case, you will need to make sure to ship + appropriate binaries for your target architecture(s) (and potentially pick the + correct version to add to ``sys.path`` at runtime, based on the user's machine). + +2. If you are shipping a Windows executable as described above, you either need to + ensure that your users have ``python3.dll`` on their PATH (which is not the + default behaviour of the installer) or you should bundle your application with + the embedded distribution. + +3. The suggested launcher above uses the Python embedding API. This means that in + your application, ``sys.executable`` will be your application, and *not* a + conventional Python interpreter. Your code and its dependencies need to be + prepared for this possibility. For example, if your application uses the + :mod:`multiprocessing` module, it will need to call + :func:`multiprocessing.set_executable` to let the module know where to find the + standard Python interpreter. + + The Python Zip Application Archive Format ----------------------------------------- diff --git a/Misc/NEWS.d/next/Documentation/2018-03-20-20-11-05.bpo-28247.-V-WS-.rst b/Misc/NEWS.d/next/Documentation/2018-03-20-20-11-05.bpo-28247.-V-WS-.rst new file mode 100644 index 000000000000..28a802136fa7 --- /dev/null +++ b/Misc/NEWS.d/next/Documentation/2018-03-20-20-11-05.bpo-28247.-V-WS-.rst @@ -0,0 +1,2 @@ +Update :mod:`zipapp` documentation to describe how to make standalone +applications. From webhook-mailer at python.org Tue Mar 20 20:09:18 2018 From: webhook-mailer at python.org (Xiang Zhang) Date: Wed, 21 Mar 2018 00:09:18 -0000 Subject: [Python-checkins] bpo-18802: Add more details to ipaddress documentation (GH-6083) Message-ID: <mailman.110.1521590961.1871.python-checkins@python.org> https://github.com/python/cpython/commit/5609b78392d59c7362ef8aa5c4a4529325f01f27 commit: 5609b78392d59c7362ef8aa5c4a4529325f01f27 branch: master author: Cheryl Sabella <cheryl.sabella at gmail.com> committer: Xiang Zhang <angwerzx at 126.com> date: 2018-03-21T08:09:15+08:00 summary: bpo-18802: Add more details to ipaddress documentation (GH-6083) Original patch by Jon Foster and Berker Peksag. files: A Misc/NEWS.d/next/Documentation/2018-03-11-18-53-47.bpo-18802.JhAqH3.rst M Doc/library/ipaddress.rst M Lib/test/test_ipaddress.py diff --git a/Doc/library/ipaddress.rst b/Doc/library/ipaddress.rst index 76177a0b23eb..935add17e2c8 100644 --- a/Doc/library/ipaddress.rst +++ b/Doc/library/ipaddress.rst @@ -91,7 +91,8 @@ Address objects The :class:`IPv4Address` and :class:`IPv6Address` objects share a lot of common attributes. Some attributes that are only meaningful for IPv6 addresses are also implemented by :class:`IPv4Address` objects, in order to make it easier to -write code that handles both IP versions correctly. +write code that handles both IP versions correctly. Address objects are +:term:`hashable`, so they can be used as keys in dictionaries. .. class:: IPv4Address(address) @@ -368,6 +369,8 @@ All attributes implemented by address objects are implemented by network objects as well. In addition, network objects implement additional attributes. All of these are common between :class:`IPv4Network` and :class:`IPv6Network`, so to avoid duplication they are only documented for :class:`IPv4Network`. +Network objects are :term:`hashable`, so they can be used as keys in +dictionaries. .. class:: IPv4Network(address, strict=True) @@ -377,8 +380,9 @@ so to avoid duplication they are only documented for :class:`IPv4Network`. a slash (``/``). The IP address is the network address, and the mask can be either a single number, which means it's a *prefix*, or a string representation of an IPv4 address. If it's the latter, the mask is - interpreted as a *net mask* if it starts with a non-zero field, or as - a *host mask* if it starts with a zero field. If no mask is provided, + interpreted as a *net mask* if it starts with a non-zero field, or as a + *host mask* if it starts with a zero field, with the single exception of + an all-zero mask which is treated as a *net mask*. If no mask is provided, it's considered to be ``/32``. For example, the following *address* specifications are equivalent: @@ -408,7 +412,7 @@ so to avoid duplication they are only documented for :class:`IPv4Network`. Unless stated otherwise, all network methods accepting other network/address objects will raise :exc:`TypeError` if the argument's IP version is - incompatible to ``self`` + incompatible to ``self``. .. versionchanged:: 3.5 @@ -418,7 +422,7 @@ so to avoid duplication they are only documented for :class:`IPv4Network`. .. attribute:: max_prefixlen Refer to the corresponding attribute documentation in - :class:`IPv4Address` + :class:`IPv4Address`. .. attribute:: is_multicast .. attribute:: is_private @@ -428,7 +432,7 @@ so to avoid duplication they are only documented for :class:`IPv4Network`. .. attribute:: is_link_local These attributes are true for the network as a whole if they are true - for both the network address and the broadcast address + for both the network address and the broadcast address. .. attribute:: network_address @@ -590,10 +594,10 @@ so to avoid duplication they are only documented for :class:`IPv4Network`. Construct an IPv6 network definition. *address* can be one of the following: - 1. A string consisting of an IP address and an optional mask, separated by - a slash (``/``). The IP address is the network address, and the mask - is a single number, which represents a *prefix*. If no mask is provided, - it's considered to be ``/128``. + 1. A string consisting of an IP address and an optional prefix length, + separated by a slash (``/``). The IP address is the network address, + and the prefix length must be a single number, the *prefix*. If no + prefix length is provided, it's considered to be ``/128``. Note that currently expanded netmasks are not supported. That means ``2001:db00::0/24`` is a valid argument while ``2001:db00::0/ffff:ff00::`` @@ -652,12 +656,12 @@ so to avoid duplication they are only documented for :class:`IPv4Network`. .. method:: compare_networks(other) Refer to the corresponding attribute documentation in - :class:`IPv4Network` + :class:`IPv4Network`. .. attribute:: is_site_local These attribute is true for the network as a whole if it is true - for both the network address and the broadcast address + for both the network address and the broadcast address. Operators @@ -671,8 +675,8 @@ IPv6). Logical operators """"""""""""""""" -Network objects can be compared with the usual set of logical operators, -similarly to address objects. +Network objects can be compared with the usual set of logical operators. +Network objects are ordered first by network address, then by net mask. Iteration @@ -722,6 +726,9 @@ Network objects can act as containers of addresses. Some examples:: Interface objects ----------------- +Interface objects are :term:`hashable`, so they can be used as keys in +dictionaries. + .. class:: IPv4Interface(address) Construct an IPv4 interface. The meaning of *address* is as in the @@ -793,6 +800,30 @@ Interface objects :class:`IPv4Interface`. +Operators +^^^^^^^^^ + +Interface objects support some operators. Unless stated otherwise, operators +can only be applied between compatible objects (i.e. IPv4 with IPv4, IPv6 with +IPv6). + + +Logical operators +""""""""""""""""" + +Interface objects can be compared with the usual set of logical operators. + +For equality comparison (``==`` and ``!=``), both the IP address and network +must be the same for the objects to be equal. An interface will not compare +equal to any address or network object. + +For ordering (``<``, ``>``, etc) the rules are different. Interface and +address objects with the same IP version can be compared, and the address +objects will always sort before the interface objects. Two interface objects +are first compared by their networks and, if those are the same, then by their +IP addresses. + + Other Module Level Functions ---------------------------- @@ -858,7 +889,7 @@ The module also provides the following module level functions: doesn't make sense. There are some times however, where you may wish to have :mod:`ipaddress` sort these anyway. If you need to do this, you can use - this function as the ``key`` argument to :func:`sorted()`. + this function as the *key* argument to :func:`sorted()`. *obj* is either a network or address object. @@ -876,4 +907,4 @@ module defines the following exceptions: .. exception:: NetmaskValueError(ValueError) - Any value error related to the netmask. + Any value error related to the net mask. diff --git a/Lib/test/test_ipaddress.py b/Lib/test/test_ipaddress.py index dbf68b3f8f1a..a5aeb790faeb 100644 --- a/Lib/test/test_ipaddress.py +++ b/Lib/test/test_ipaddress.py @@ -404,6 +404,9 @@ def test_weakref(self): class NetmaskTestMixin_v4(CommonTestMixin_v4): """Input validation on interfaces and networks is very similar""" + def test_no_mask(self): + self.assertEqual(str(self.factory('1.2.3.4')), '1.2.3.4/32') + def test_split_netmask(self): addr = "1.2.3.4/32/24" with self.assertAddressError("Only one '/' permitted in %r" % addr): diff --git a/Misc/NEWS.d/next/Documentation/2018-03-11-18-53-47.bpo-18802.JhAqH3.rst b/Misc/NEWS.d/next/Documentation/2018-03-11-18-53-47.bpo-18802.JhAqH3.rst new file mode 100644 index 000000000000..cb9cc2599aca --- /dev/null +++ b/Misc/NEWS.d/next/Documentation/2018-03-11-18-53-47.bpo-18802.JhAqH3.rst @@ -0,0 +1 @@ +Documentation changes for ipaddress. Patch by Jon Foster and Berker Peksag. From webhook-mailer at python.org Tue Mar 20 20:25:16 2018 From: webhook-mailer at python.org (Xiang Zhang) Date: Wed, 21 Mar 2018 00:25:16 -0000 Subject: [Python-checkins] bpo-27683: Fix a regression for host() of ipaddress network objects (GH-6016) Message-ID: <mailman.111.1521591918.1871.python-checkins@python.org> https://github.com/python/cpython/commit/10b134a07c898c2fbc5fd3582503680a54ed80a2 commit: 10b134a07c898c2fbc5fd3582503680a54ed80a2 branch: master author: Xiang Zhang <angwerzx at 126.com> committer: GitHub <noreply at github.com> date: 2018-03-21T08:25:13+08:00 summary: bpo-27683: Fix a regression for host() of ipaddress network objects (GH-6016) The result of host() was not empty when the network is constructed by a tuple containing an integer mask and only 1 bit left for addresses. files: A Misc/NEWS.d/next/Library/2018-03-07-22-28-17.bpo-27683.572Rv4.rst M Doc/library/ipaddress.rst M Lib/ipaddress.py M Lib/test/test_ipaddress.py diff --git a/Doc/library/ipaddress.rst b/Doc/library/ipaddress.rst index 935add17e2c8..b7b502aff15e 100644 --- a/Doc/library/ipaddress.rst +++ b/Doc/library/ipaddress.rst @@ -485,12 +485,16 @@ dictionaries. Returns an iterator over the usable hosts in the network. The usable hosts are all the IP addresses that belong to the network, except the - network address itself and the network broadcast address. + network address itself and the network broadcast address. For networks + with a mask length of 31, the network address and network broadcast + address are also included in the result. >>> list(ip_network('192.0.2.0/29').hosts()) #doctest: +NORMALIZE_WHITESPACE [IPv4Address('192.0.2.1'), IPv4Address('192.0.2.2'), IPv4Address('192.0.2.3'), IPv4Address('192.0.2.4'), IPv4Address('192.0.2.5'), IPv4Address('192.0.2.6')] + >>> list(ip_network('192.0.2.0/31').hosts()) + [IPv4Address('192.0.2.0'), IPv4Address('192.0.2.1')] .. method:: overlaps(other) @@ -647,6 +651,12 @@ dictionaries. .. attribute:: num_addresses .. attribute:: prefixlen .. method:: hosts() + + Returns an iterator over the usable hosts in the network. The usable + hosts are all the IP addresses that belong to the network, except the + Subnet-Router anycast address. For networks with a mask length of 127, + the Subnet-Router anycast address is also included in the result. + .. method:: overlaps(other) .. method:: address_exclude(network) .. method:: subnets(prefixlen_diff=1, new_prefix=None) diff --git a/Lib/ipaddress.py b/Lib/ipaddress.py index 77df0511e78a..15507d61dec8 100644 --- a/Lib/ipaddress.py +++ b/Lib/ipaddress.py @@ -1514,45 +1514,28 @@ def __init__(self, address, strict=True): # Constructing from a packed address or integer if isinstance(address, (int, bytes)): - self.network_address = IPv4Address(address) - self.netmask, self._prefixlen = self._make_netmask(self._max_prefixlen) - #fixme: address/network test here. - return - - if isinstance(address, tuple): - if len(address) > 1: - arg = address[1] - else: - # We weren't given an address[1] - arg = self._max_prefixlen - self.network_address = IPv4Address(address[0]) - self.netmask, self._prefixlen = self._make_netmask(arg) - packed = int(self.network_address) - if packed & int(self.netmask) != packed: - if strict: - raise ValueError('%s has host bits set' % self) - else: - self.network_address = IPv4Address(packed & - int(self.netmask)) - return - + addr = address + mask = self._max_prefixlen + # Constructing from a tuple (addr, [mask]) + elif isinstance(address, tuple): + addr = address[0] + mask = address[1] if len(address) > 1 else self._max_prefixlen # Assume input argument to be string or any object representation # which converts into a formatted IP prefix string. - addr = _split_optional_netmask(address) - self.network_address = IPv4Address(self._ip_int_from_string(addr[0])) - - if len(addr) == 2: - arg = addr[1] else: - arg = self._max_prefixlen - self.netmask, self._prefixlen = self._make_netmask(arg) - - if strict: - if (IPv4Address(int(self.network_address) & int(self.netmask)) != - self.network_address): + args = _split_optional_netmask(address) + addr = self._ip_int_from_string(args[0]) + mask = args[1] if len(args) == 2 else self._max_prefixlen + + self.network_address = IPv4Address(addr) + self.netmask, self._prefixlen = self._make_netmask(mask) + packed = int(self.network_address) + if packed & int(self.netmask) != packed: + if strict: raise ValueError('%s has host bits set' % self) - self.network_address = IPv4Address(int(self.network_address) & - int(self.netmask)) + else: + self.network_address = IPv4Address(packed & + int(self.netmask)) if self._prefixlen == (self._max_prefixlen - 1): self.hosts = self.__iter__ @@ -2207,46 +2190,30 @@ def __init__(self, address, strict=True): """ _BaseNetwork.__init__(self, address) - # Efficient constructor from integer or packed address - if isinstance(address, (bytes, int)): - self.network_address = IPv6Address(address) - self.netmask, self._prefixlen = self._make_netmask(self._max_prefixlen) - return - - if isinstance(address, tuple): - if len(address) > 1: - arg = address[1] - else: - arg = self._max_prefixlen - self.netmask, self._prefixlen = self._make_netmask(arg) - self.network_address = IPv6Address(address[0]) - packed = int(self.network_address) - if packed & int(self.netmask) != packed: - if strict: - raise ValueError('%s has host bits set' % self) - else: - self.network_address = IPv6Address(packed & - int(self.netmask)) - return - + # Constructing from a packed address or integer + if isinstance(address, (int, bytes)): + addr = address + mask = self._max_prefixlen + # Constructing from a tuple (addr, [mask]) + elif isinstance(address, tuple): + addr = address[0] + mask = address[1] if len(address) > 1 else self._max_prefixlen # Assume input argument to be string or any object representation # which converts into a formatted IP prefix string. - addr = _split_optional_netmask(address) - - self.network_address = IPv6Address(self._ip_int_from_string(addr[0])) - - if len(addr) == 2: - arg = addr[1] else: - arg = self._max_prefixlen - self.netmask, self._prefixlen = self._make_netmask(arg) - - if strict: - if (IPv6Address(int(self.network_address) & int(self.netmask)) != - self.network_address): + args = _split_optional_netmask(address) + addr = self._ip_int_from_string(args[0]) + mask = args[1] if len(args) == 2 else self._max_prefixlen + + self.network_address = IPv6Address(addr) + self.netmask, self._prefixlen = self._make_netmask(mask) + packed = int(self.network_address) + if packed & int(self.netmask) != packed: + if strict: raise ValueError('%s has host bits set' % self) - self.network_address = IPv6Address(int(self.network_address) & - int(self.netmask)) + else: + self.network_address = IPv6Address(packed & + int(self.netmask)) if self._prefixlen == (self._max_prefixlen - 1): self.hosts = self.__iter__ diff --git a/Lib/test/test_ipaddress.py b/Lib/test/test_ipaddress.py index a5aeb790faeb..0e0753f34c49 100644 --- a/Lib/test/test_ipaddress.py +++ b/Lib/test/test_ipaddress.py @@ -1127,10 +1127,30 @@ def testHosts(self): self.assertEqual(ipaddress.IPv4Address('1.2.3.1'), hosts[0]) self.assertEqual(ipaddress.IPv4Address('1.2.3.254'), hosts[-1]) + ipv6_network = ipaddress.IPv6Network('2001:658:22a:cafe::/120') + hosts = list(ipv6_network.hosts()) + self.assertEqual(255, len(hosts)) + self.assertEqual(ipaddress.IPv6Address('2001:658:22a:cafe::1'), hosts[0]) + self.assertEqual(ipaddress.IPv6Address('2001:658:22a:cafe::ff'), hosts[-1]) + # special case where only 1 bit is left for address - self.assertEqual([ipaddress.IPv4Address('2.0.0.0'), - ipaddress.IPv4Address('2.0.0.1')], - list(ipaddress.ip_network('2.0.0.0/31').hosts())) + addrs = [ipaddress.IPv4Address('2.0.0.0'), + ipaddress.IPv4Address('2.0.0.1')] + str_args = '2.0.0.0/31' + tpl_args = ('2.0.0.0', 31) + self.assertEqual(addrs, list(ipaddress.ip_network(str_args).hosts())) + self.assertEqual(addrs, list(ipaddress.ip_network(tpl_args).hosts())) + self.assertEqual(list(ipaddress.ip_network(str_args).hosts()), + list(ipaddress.ip_network(tpl_args).hosts())) + + addrs = [ipaddress.IPv6Address('2001:658:22a:cafe::'), + ipaddress.IPv6Address('2001:658:22a:cafe::1')] + str_args = '2001:658:22a:cafe::/127' + tpl_args = ('2001:658:22a:cafe::', 127) + self.assertEqual(addrs, list(ipaddress.ip_network(str_args).hosts())) + self.assertEqual(addrs, list(ipaddress.ip_network(tpl_args).hosts())) + self.assertEqual(list(ipaddress.ip_network(str_args).hosts()), + list(ipaddress.ip_network(tpl_args).hosts())) def testFancySubnetting(self): self.assertEqual(sorted(self.ipv4_network.subnets(prefixlen_diff=3)), diff --git a/Misc/NEWS.d/next/Library/2018-03-07-22-28-17.bpo-27683.572Rv4.rst b/Misc/NEWS.d/next/Library/2018-03-07-22-28-17.bpo-27683.572Rv4.rst new file mode 100644 index 000000000000..4e6dfa8e978c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-03-07-22-28-17.bpo-27683.572Rv4.rst @@ -0,0 +1,3 @@ +Fix a regression in :mod:`ipaddress` that result of :meth:`hosts` +is empty when the network is constructed by a tuple containing an +integer mask and only 1 bit left for addresses. From webhook-mailer at python.org Tue Mar 20 20:30:46 2018 From: webhook-mailer at python.org (Miss Islington (bot)) Date: Wed, 21 Mar 2018 00:30:46 -0000 Subject: [Python-checkins] bpo-18802: Add more details to ipaddress documentation (GH-6083) Message-ID: <mailman.112.1521592249.1871.python-checkins@python.org> https://github.com/python/cpython/commit/a323eee4c481c88f2b4030bbb224d9bc6bc14c9c commit: a323eee4c481c88f2b4030bbb224d9bc6bc14c9c branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: GitHub <noreply at github.com> date: 2018-03-20T17:30:43-07:00 summary: bpo-18802: Add more details to ipaddress documentation (GH-6083) Original patch by Jon Foster and Berker Peksag. (cherry picked from commit 5609b78392d59c7362ef8aa5c4a4529325f01f27) Co-authored-by: Cheryl Sabella <cheryl.sabella at gmail.com> files: A Misc/NEWS.d/next/Documentation/2018-03-11-18-53-47.bpo-18802.JhAqH3.rst M Doc/library/ipaddress.rst M Lib/test/test_ipaddress.py diff --git a/Doc/library/ipaddress.rst b/Doc/library/ipaddress.rst index 76177a0b23eb..935add17e2c8 100644 --- a/Doc/library/ipaddress.rst +++ b/Doc/library/ipaddress.rst @@ -91,7 +91,8 @@ Address objects The :class:`IPv4Address` and :class:`IPv6Address` objects share a lot of common attributes. Some attributes that are only meaningful for IPv6 addresses are also implemented by :class:`IPv4Address` objects, in order to make it easier to -write code that handles both IP versions correctly. +write code that handles both IP versions correctly. Address objects are +:term:`hashable`, so they can be used as keys in dictionaries. .. class:: IPv4Address(address) @@ -368,6 +369,8 @@ All attributes implemented by address objects are implemented by network objects as well. In addition, network objects implement additional attributes. All of these are common between :class:`IPv4Network` and :class:`IPv6Network`, so to avoid duplication they are only documented for :class:`IPv4Network`. +Network objects are :term:`hashable`, so they can be used as keys in +dictionaries. .. class:: IPv4Network(address, strict=True) @@ -377,8 +380,9 @@ so to avoid duplication they are only documented for :class:`IPv4Network`. a slash (``/``). The IP address is the network address, and the mask can be either a single number, which means it's a *prefix*, or a string representation of an IPv4 address. If it's the latter, the mask is - interpreted as a *net mask* if it starts with a non-zero field, or as - a *host mask* if it starts with a zero field. If no mask is provided, + interpreted as a *net mask* if it starts with a non-zero field, or as a + *host mask* if it starts with a zero field, with the single exception of + an all-zero mask which is treated as a *net mask*. If no mask is provided, it's considered to be ``/32``. For example, the following *address* specifications are equivalent: @@ -408,7 +412,7 @@ so to avoid duplication they are only documented for :class:`IPv4Network`. Unless stated otherwise, all network methods accepting other network/address objects will raise :exc:`TypeError` if the argument's IP version is - incompatible to ``self`` + incompatible to ``self``. .. versionchanged:: 3.5 @@ -418,7 +422,7 @@ so to avoid duplication they are only documented for :class:`IPv4Network`. .. attribute:: max_prefixlen Refer to the corresponding attribute documentation in - :class:`IPv4Address` + :class:`IPv4Address`. .. attribute:: is_multicast .. attribute:: is_private @@ -428,7 +432,7 @@ so to avoid duplication they are only documented for :class:`IPv4Network`. .. attribute:: is_link_local These attributes are true for the network as a whole if they are true - for both the network address and the broadcast address + for both the network address and the broadcast address. .. attribute:: network_address @@ -590,10 +594,10 @@ so to avoid duplication they are only documented for :class:`IPv4Network`. Construct an IPv6 network definition. *address* can be one of the following: - 1. A string consisting of an IP address and an optional mask, separated by - a slash (``/``). The IP address is the network address, and the mask - is a single number, which represents a *prefix*. If no mask is provided, - it's considered to be ``/128``. + 1. A string consisting of an IP address and an optional prefix length, + separated by a slash (``/``). The IP address is the network address, + and the prefix length must be a single number, the *prefix*. If no + prefix length is provided, it's considered to be ``/128``. Note that currently expanded netmasks are not supported. That means ``2001:db00::0/24`` is a valid argument while ``2001:db00::0/ffff:ff00::`` @@ -652,12 +656,12 @@ so to avoid duplication they are only documented for :class:`IPv4Network`. .. method:: compare_networks(other) Refer to the corresponding attribute documentation in - :class:`IPv4Network` + :class:`IPv4Network`. .. attribute:: is_site_local These attribute is true for the network as a whole if it is true - for both the network address and the broadcast address + for both the network address and the broadcast address. Operators @@ -671,8 +675,8 @@ IPv6). Logical operators """"""""""""""""" -Network objects can be compared with the usual set of logical operators, -similarly to address objects. +Network objects can be compared with the usual set of logical operators. +Network objects are ordered first by network address, then by net mask. Iteration @@ -722,6 +726,9 @@ Network objects can act as containers of addresses. Some examples:: Interface objects ----------------- +Interface objects are :term:`hashable`, so they can be used as keys in +dictionaries. + .. class:: IPv4Interface(address) Construct an IPv4 interface. The meaning of *address* is as in the @@ -793,6 +800,30 @@ Interface objects :class:`IPv4Interface`. +Operators +^^^^^^^^^ + +Interface objects support some operators. Unless stated otherwise, operators +can only be applied between compatible objects (i.e. IPv4 with IPv4, IPv6 with +IPv6). + + +Logical operators +""""""""""""""""" + +Interface objects can be compared with the usual set of logical operators. + +For equality comparison (``==`` and ``!=``), both the IP address and network +must be the same for the objects to be equal. An interface will not compare +equal to any address or network object. + +For ordering (``<``, ``>``, etc) the rules are different. Interface and +address objects with the same IP version can be compared, and the address +objects will always sort before the interface objects. Two interface objects +are first compared by their networks and, if those are the same, then by their +IP addresses. + + Other Module Level Functions ---------------------------- @@ -858,7 +889,7 @@ The module also provides the following module level functions: doesn't make sense. There are some times however, where you may wish to have :mod:`ipaddress` sort these anyway. If you need to do this, you can use - this function as the ``key`` argument to :func:`sorted()`. + this function as the *key* argument to :func:`sorted()`. *obj* is either a network or address object. @@ -876,4 +907,4 @@ module defines the following exceptions: .. exception:: NetmaskValueError(ValueError) - Any value error related to the netmask. + Any value error related to the net mask. diff --git a/Lib/test/test_ipaddress.py b/Lib/test/test_ipaddress.py index dbf68b3f8f1a..a5aeb790faeb 100644 --- a/Lib/test/test_ipaddress.py +++ b/Lib/test/test_ipaddress.py @@ -404,6 +404,9 @@ def test_weakref(self): class NetmaskTestMixin_v4(CommonTestMixin_v4): """Input validation on interfaces and networks is very similar""" + def test_no_mask(self): + self.assertEqual(str(self.factory('1.2.3.4')), '1.2.3.4/32') + def test_split_netmask(self): addr = "1.2.3.4/32/24" with self.assertAddressError("Only one '/' permitted in %r" % addr): diff --git a/Misc/NEWS.d/next/Documentation/2018-03-11-18-53-47.bpo-18802.JhAqH3.rst b/Misc/NEWS.d/next/Documentation/2018-03-11-18-53-47.bpo-18802.JhAqH3.rst new file mode 100644 index 000000000000..cb9cc2599aca --- /dev/null +++ b/Misc/NEWS.d/next/Documentation/2018-03-11-18-53-47.bpo-18802.JhAqH3.rst @@ -0,0 +1 @@ +Documentation changes for ipaddress. Patch by Jon Foster and Berker Peksag. From webhook-mailer at python.org Tue Mar 20 20:59:03 2018 From: webhook-mailer at python.org (Miss Islington (bot)) Date: Wed, 21 Mar 2018 00:59:03 -0000 Subject: [Python-checkins] bpo-18802: Add more details to ipaddress documentation (GH-6083) Message-ID: <mailman.113.1521593945.1871.python-checkins@python.org> https://github.com/python/cpython/commit/481cbe8d6202658a7908d97f19f7e9e6061e3df3 commit: 481cbe8d6202658a7908d97f19f7e9e6061e3df3 branch: 3.6 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: GitHub <noreply at github.com> date: 2018-03-20T17:59:00-07:00 summary: bpo-18802: Add more details to ipaddress documentation (GH-6083) Original patch by Jon Foster and Berker Peksag. (cherry picked from commit 5609b78392d59c7362ef8aa5c4a4529325f01f27) Co-authored-by: Cheryl Sabella <cheryl.sabella at gmail.com> files: A Misc/NEWS.d/next/Documentation/2018-03-11-18-53-47.bpo-18802.JhAqH3.rst M Doc/library/ipaddress.rst M Lib/test/test_ipaddress.py diff --git a/Doc/library/ipaddress.rst b/Doc/library/ipaddress.rst index b3c691e444a1..66c1aaa6a181 100644 --- a/Doc/library/ipaddress.rst +++ b/Doc/library/ipaddress.rst @@ -89,7 +89,8 @@ Address objects The :class:`IPv4Address` and :class:`IPv6Address` objects share a lot of common attributes. Some attributes that are only meaningful for IPv6 addresses are also implemented by :class:`IPv4Address` objects, in order to make it easier to -write code that handles both IP versions correctly. +write code that handles both IP versions correctly. Address objects are +:term:`hashable`, so they can be used as keys in dictionaries. .. class:: IPv4Address(address) @@ -366,6 +367,8 @@ All attributes implemented by address objects are implemented by network objects as well. In addition, network objects implement additional attributes. All of these are common between :class:`IPv4Network` and :class:`IPv6Network`, so to avoid duplication they are only documented for :class:`IPv4Network`. +Network objects are :term:`hashable`, so they can be used as keys in +dictionaries. .. class:: IPv4Network(address, strict=True) @@ -375,8 +378,9 @@ so to avoid duplication they are only documented for :class:`IPv4Network`. a slash (``/``). The IP address is the network address, and the mask can be either a single number, which means it's a *prefix*, or a string representation of an IPv4 address. If it's the latter, the mask is - interpreted as a *net mask* if it starts with a non-zero field, or as - a *host mask* if it starts with a zero field. If no mask is provided, + interpreted as a *net mask* if it starts with a non-zero field, or as a + *host mask* if it starts with a zero field, with the single exception of + an all-zero mask which is treated as a *net mask*. If no mask is provided, it's considered to be ``/32``. For example, the following *address* specifications are equivalent: @@ -406,7 +410,7 @@ so to avoid duplication they are only documented for :class:`IPv4Network`. Unless stated otherwise, all network methods accepting other network/address objects will raise :exc:`TypeError` if the argument's IP version is - incompatible to ``self`` + incompatible to ``self``. .. versionchanged:: 3.5 @@ -416,7 +420,7 @@ so to avoid duplication they are only documented for :class:`IPv4Network`. .. attribute:: max_prefixlen Refer to the corresponding attribute documentation in - :class:`IPv4Address` + :class:`IPv4Address`. .. attribute:: is_multicast .. attribute:: is_private @@ -426,7 +430,7 @@ so to avoid duplication they are only documented for :class:`IPv4Network`. .. attribute:: is_link_local These attributes are true for the network as a whole if they are true - for both the network address and the broadcast address + for both the network address and the broadcast address. .. attribute:: network_address @@ -563,10 +567,10 @@ so to avoid duplication they are only documented for :class:`IPv4Network`. Construct an IPv6 network definition. *address* can be one of the following: - 1. A string consisting of an IP address and an optional mask, separated by - a slash (``/``). The IP address is the network address, and the mask - is a single number, which represents a *prefix*. If no mask is provided, - it's considered to be ``/128``. + 1. A string consisting of an IP address and an optional prefix length, + separated by a slash (``/``). The IP address is the network address, + and the prefix length must be a single number, the *prefix*. If no + prefix length is provided, it's considered to be ``/128``. Note that currently expanded netmasks are not supported. That means ``2001:db00::0/24`` is a valid argument while ``2001:db00::0/ffff:ff00::`` @@ -623,12 +627,12 @@ so to avoid duplication they are only documented for :class:`IPv4Network`. .. method:: compare_networks(other) Refer to the corresponding attribute documentation in - :class:`IPv4Network` + :class:`IPv4Network`. .. attribute:: is_site_local These attribute is true for the network as a whole if it is true - for both the network address and the broadcast address + for both the network address and the broadcast address. Operators @@ -642,8 +646,8 @@ IPv6). Logical operators """"""""""""""""" -Network objects can be compared with the usual set of logical operators, -similarly to address objects. +Network objects can be compared with the usual set of logical operators. +Network objects are ordered first by network address, then by net mask. Iteration @@ -693,6 +697,9 @@ Network objects can act as containers of addresses. Some examples:: Interface objects ----------------- +Interface objects are :term:`hashable`, so they can be used as keys in +dictionaries. + .. class:: IPv4Interface(address) Construct an IPv4 interface. The meaning of *address* is as in the @@ -764,6 +771,30 @@ Interface objects :class:`IPv4Interface`. +Operators +^^^^^^^^^ + +Interface objects support some operators. Unless stated otherwise, operators +can only be applied between compatible objects (i.e. IPv4 with IPv4, IPv6 with +IPv6). + + +Logical operators +""""""""""""""""" + +Interface objects can be compared with the usual set of logical operators. + +For equality comparison (``==`` and ``!=``), both the IP address and network +must be the same for the objects to be equal. An interface will not compare +equal to any address or network object. + +For ordering (``<``, ``>``, etc) the rules are different. Interface and +address objects with the same IP version can be compared, and the address +objects will always sort before the interface objects. Two interface objects +are first compared by their networks and, if those are the same, then by their +IP addresses. + + Other Module Level Functions ---------------------------- @@ -829,7 +860,7 @@ The module also provides the following module level functions: doesn't make sense. There are some times however, where you may wish to have :mod:`ipaddress` sort these anyway. If you need to do this, you can use - this function as the ``key`` argument to :func:`sorted()`. + this function as the *key* argument to :func:`sorted()`. *obj* is either a network or address object. @@ -847,4 +878,4 @@ module defines the following exceptions: .. exception:: NetmaskValueError(ValueError) - Any value error related to the netmask. + Any value error related to the net mask. diff --git a/Lib/test/test_ipaddress.py b/Lib/test/test_ipaddress.py index 5d9633024f70..2432eb4960e1 100644 --- a/Lib/test/test_ipaddress.py +++ b/Lib/test/test_ipaddress.py @@ -405,6 +405,9 @@ def test_weakref(self): class NetmaskTestMixin_v4(CommonTestMixin_v4): """Input validation on interfaces and networks is very similar""" + def test_no_mask(self): + self.assertEqual(str(self.factory('1.2.3.4')), '1.2.3.4/32') + def test_split_netmask(self): addr = "1.2.3.4/32/24" with self.assertAddressError("Only one '/' permitted in %r" % addr): diff --git a/Misc/NEWS.d/next/Documentation/2018-03-11-18-53-47.bpo-18802.JhAqH3.rst b/Misc/NEWS.d/next/Documentation/2018-03-11-18-53-47.bpo-18802.JhAqH3.rst new file mode 100644 index 000000000000..cb9cc2599aca --- /dev/null +++ b/Misc/NEWS.d/next/Documentation/2018-03-11-18-53-47.bpo-18802.JhAqH3.rst @@ -0,0 +1 @@ +Documentation changes for ipaddress. Patch by Jon Foster and Berker Peksag. From webhook-mailer at python.org Tue Mar 20 21:22:26 2018 From: webhook-mailer at python.org (Miss Islington (bot)) Date: Wed, 21 Mar 2018 01:22:26 -0000 Subject: [Python-checkins] bpo-27683: Fix a regression for host() of ipaddress network objects (GH-6016) Message-ID: <mailman.114.1521595347.1871.python-checkins@python.org> https://github.com/python/cpython/commit/3326c9267f9df15fa77094b8a740be4eaa4b9374 commit: 3326c9267f9df15fa77094b8a740be4eaa4b9374 branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: GitHub <noreply at github.com> date: 2018-03-20T18:22:23-07:00 summary: bpo-27683: Fix a regression for host() of ipaddress network objects (GH-6016) The result of host() was not empty when the network is constructed by a tuple containing an integer mask and only 1 bit left for addresses. (cherry picked from commit 10b134a07c898c2fbc5fd3582503680a54ed80a2) Co-authored-by: Xiang Zhang <angwerzx at 126.com> files: A Misc/NEWS.d/next/Library/2018-03-07-22-28-17.bpo-27683.572Rv4.rst M Doc/library/ipaddress.rst M Lib/ipaddress.py M Lib/test/test_ipaddress.py diff --git a/Doc/library/ipaddress.rst b/Doc/library/ipaddress.rst index 935add17e2c8..b7b502aff15e 100644 --- a/Doc/library/ipaddress.rst +++ b/Doc/library/ipaddress.rst @@ -485,12 +485,16 @@ dictionaries. Returns an iterator over the usable hosts in the network. The usable hosts are all the IP addresses that belong to the network, except the - network address itself and the network broadcast address. + network address itself and the network broadcast address. For networks + with a mask length of 31, the network address and network broadcast + address are also included in the result. >>> list(ip_network('192.0.2.0/29').hosts()) #doctest: +NORMALIZE_WHITESPACE [IPv4Address('192.0.2.1'), IPv4Address('192.0.2.2'), IPv4Address('192.0.2.3'), IPv4Address('192.0.2.4'), IPv4Address('192.0.2.5'), IPv4Address('192.0.2.6')] + >>> list(ip_network('192.0.2.0/31').hosts()) + [IPv4Address('192.0.2.0'), IPv4Address('192.0.2.1')] .. method:: overlaps(other) @@ -647,6 +651,12 @@ dictionaries. .. attribute:: num_addresses .. attribute:: prefixlen .. method:: hosts() + + Returns an iterator over the usable hosts in the network. The usable + hosts are all the IP addresses that belong to the network, except the + Subnet-Router anycast address. For networks with a mask length of 127, + the Subnet-Router anycast address is also included in the result. + .. method:: overlaps(other) .. method:: address_exclude(network) .. method:: subnets(prefixlen_diff=1, new_prefix=None) diff --git a/Lib/ipaddress.py b/Lib/ipaddress.py index e8ce4cef2ded..cc9ae7118d67 100644 --- a/Lib/ipaddress.py +++ b/Lib/ipaddress.py @@ -1515,45 +1515,28 @@ def __init__(self, address, strict=True): # Constructing from a packed address or integer if isinstance(address, (int, bytes)): - self.network_address = IPv4Address(address) - self.netmask, self._prefixlen = self._make_netmask(self._max_prefixlen) - #fixme: address/network test here. - return - - if isinstance(address, tuple): - if len(address) > 1: - arg = address[1] - else: - # We weren't given an address[1] - arg = self._max_prefixlen - self.network_address = IPv4Address(address[0]) - self.netmask, self._prefixlen = self._make_netmask(arg) - packed = int(self.network_address) - if packed & int(self.netmask) != packed: - if strict: - raise ValueError('%s has host bits set' % self) - else: - self.network_address = IPv4Address(packed & - int(self.netmask)) - return - + addr = address + mask = self._max_prefixlen + # Constructing from a tuple (addr, [mask]) + elif isinstance(address, tuple): + addr = address[0] + mask = address[1] if len(address) > 1 else self._max_prefixlen # Assume input argument to be string or any object representation # which converts into a formatted IP prefix string. - addr = _split_optional_netmask(address) - self.network_address = IPv4Address(self._ip_int_from_string(addr[0])) - - if len(addr) == 2: - arg = addr[1] else: - arg = self._max_prefixlen - self.netmask, self._prefixlen = self._make_netmask(arg) - - if strict: - if (IPv4Address(int(self.network_address) & int(self.netmask)) != - self.network_address): + args = _split_optional_netmask(address) + addr = self._ip_int_from_string(args[0]) + mask = args[1] if len(args) == 2 else self._max_prefixlen + + self.network_address = IPv4Address(addr) + self.netmask, self._prefixlen = self._make_netmask(mask) + packed = int(self.network_address) + if packed & int(self.netmask) != packed: + if strict: raise ValueError('%s has host bits set' % self) - self.network_address = IPv4Address(int(self.network_address) & - int(self.netmask)) + else: + self.network_address = IPv4Address(packed & + int(self.netmask)) if self._prefixlen == (self._max_prefixlen - 1): self.hosts = self.__iter__ @@ -2208,46 +2191,30 @@ def __init__(self, address, strict=True): """ _BaseNetwork.__init__(self, address) - # Efficient constructor from integer or packed address - if isinstance(address, (bytes, int)): - self.network_address = IPv6Address(address) - self.netmask, self._prefixlen = self._make_netmask(self._max_prefixlen) - return - - if isinstance(address, tuple): - if len(address) > 1: - arg = address[1] - else: - arg = self._max_prefixlen - self.netmask, self._prefixlen = self._make_netmask(arg) - self.network_address = IPv6Address(address[0]) - packed = int(self.network_address) - if packed & int(self.netmask) != packed: - if strict: - raise ValueError('%s has host bits set' % self) - else: - self.network_address = IPv6Address(packed & - int(self.netmask)) - return - + # Constructing from a packed address or integer + if isinstance(address, (int, bytes)): + addr = address + mask = self._max_prefixlen + # Constructing from a tuple (addr, [mask]) + elif isinstance(address, tuple): + addr = address[0] + mask = address[1] if len(address) > 1 else self._max_prefixlen # Assume input argument to be string or any object representation # which converts into a formatted IP prefix string. - addr = _split_optional_netmask(address) - - self.network_address = IPv6Address(self._ip_int_from_string(addr[0])) - - if len(addr) == 2: - arg = addr[1] else: - arg = self._max_prefixlen - self.netmask, self._prefixlen = self._make_netmask(arg) - - if strict: - if (IPv6Address(int(self.network_address) & int(self.netmask)) != - self.network_address): + args = _split_optional_netmask(address) + addr = self._ip_int_from_string(args[0]) + mask = args[1] if len(args) == 2 else self._max_prefixlen + + self.network_address = IPv6Address(addr) + self.netmask, self._prefixlen = self._make_netmask(mask) + packed = int(self.network_address) + if packed & int(self.netmask) != packed: + if strict: raise ValueError('%s has host bits set' % self) - self.network_address = IPv6Address(int(self.network_address) & - int(self.netmask)) + else: + self.network_address = IPv6Address(packed & + int(self.netmask)) if self._prefixlen == (self._max_prefixlen - 1): self.hosts = self.__iter__ diff --git a/Lib/test/test_ipaddress.py b/Lib/test/test_ipaddress.py index a5aeb790faeb..0e0753f34c49 100644 --- a/Lib/test/test_ipaddress.py +++ b/Lib/test/test_ipaddress.py @@ -1127,10 +1127,30 @@ def testHosts(self): self.assertEqual(ipaddress.IPv4Address('1.2.3.1'), hosts[0]) self.assertEqual(ipaddress.IPv4Address('1.2.3.254'), hosts[-1]) + ipv6_network = ipaddress.IPv6Network('2001:658:22a:cafe::/120') + hosts = list(ipv6_network.hosts()) + self.assertEqual(255, len(hosts)) + self.assertEqual(ipaddress.IPv6Address('2001:658:22a:cafe::1'), hosts[0]) + self.assertEqual(ipaddress.IPv6Address('2001:658:22a:cafe::ff'), hosts[-1]) + # special case where only 1 bit is left for address - self.assertEqual([ipaddress.IPv4Address('2.0.0.0'), - ipaddress.IPv4Address('2.0.0.1')], - list(ipaddress.ip_network('2.0.0.0/31').hosts())) + addrs = [ipaddress.IPv4Address('2.0.0.0'), + ipaddress.IPv4Address('2.0.0.1')] + str_args = '2.0.0.0/31' + tpl_args = ('2.0.0.0', 31) + self.assertEqual(addrs, list(ipaddress.ip_network(str_args).hosts())) + self.assertEqual(addrs, list(ipaddress.ip_network(tpl_args).hosts())) + self.assertEqual(list(ipaddress.ip_network(str_args).hosts()), + list(ipaddress.ip_network(tpl_args).hosts())) + + addrs = [ipaddress.IPv6Address('2001:658:22a:cafe::'), + ipaddress.IPv6Address('2001:658:22a:cafe::1')] + str_args = '2001:658:22a:cafe::/127' + tpl_args = ('2001:658:22a:cafe::', 127) + self.assertEqual(addrs, list(ipaddress.ip_network(str_args).hosts())) + self.assertEqual(addrs, list(ipaddress.ip_network(tpl_args).hosts())) + self.assertEqual(list(ipaddress.ip_network(str_args).hosts()), + list(ipaddress.ip_network(tpl_args).hosts())) def testFancySubnetting(self): self.assertEqual(sorted(self.ipv4_network.subnets(prefixlen_diff=3)), diff --git a/Misc/NEWS.d/next/Library/2018-03-07-22-28-17.bpo-27683.572Rv4.rst b/Misc/NEWS.d/next/Library/2018-03-07-22-28-17.bpo-27683.572Rv4.rst new file mode 100644 index 000000000000..4e6dfa8e978c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-03-07-22-28-17.bpo-27683.572Rv4.rst @@ -0,0 +1,3 @@ +Fix a regression in :mod:`ipaddress` that result of :meth:`hosts` +is empty when the network is constructed by a tuple containing an +integer mask and only 1 bit left for addresses. From webhook-mailer at python.org Tue Mar 20 21:49:44 2018 From: webhook-mailer at python.org (Miss Islington (bot)) Date: Wed, 21 Mar 2018 01:49:44 -0000 Subject: [Python-checkins] bpo-27683: Fix a regression for host() of ipaddress network objects (GH-6016) Message-ID: <mailman.115.1521596985.1871.python-checkins@python.org> https://github.com/python/cpython/commit/ae2feb32e7eff328199ce7d527593b3c2aa1fcab commit: ae2feb32e7eff328199ce7d527593b3c2aa1fcab branch: 3.6 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: GitHub <noreply at github.com> date: 2018-03-20T18:49:41-07:00 summary: bpo-27683: Fix a regression for host() of ipaddress network objects (GH-6016) The result of host() was not empty when the network is constructed by a tuple containing an integer mask and only 1 bit left for addresses. (cherry picked from commit 10b134a07c898c2fbc5fd3582503680a54ed80a2) Co-authored-by: Xiang Zhang <angwerzx at 126.com> files: A Misc/NEWS.d/next/Library/2018-03-07-22-28-17.bpo-27683.572Rv4.rst M Doc/library/ipaddress.rst M Lib/ipaddress.py M Lib/test/test_ipaddress.py diff --git a/Doc/library/ipaddress.rst b/Doc/library/ipaddress.rst index 66c1aaa6a181..4ce1ed1cedb6 100644 --- a/Doc/library/ipaddress.rst +++ b/Doc/library/ipaddress.rst @@ -483,12 +483,16 @@ dictionaries. Returns an iterator over the usable hosts in the network. The usable hosts are all the IP addresses that belong to the network, except the - network address itself and the network broadcast address. + network address itself and the network broadcast address. For networks + with a mask length of 31, the network address and network broadcast + address are also included in the result. >>> list(ip_network('192.0.2.0/29').hosts()) #doctest: +NORMALIZE_WHITESPACE [IPv4Address('192.0.2.1'), IPv4Address('192.0.2.2'), IPv4Address('192.0.2.3'), IPv4Address('192.0.2.4'), IPv4Address('192.0.2.5'), IPv4Address('192.0.2.6')] + >>> list(ip_network('192.0.2.0/31').hosts()) + [IPv4Address('192.0.2.0'), IPv4Address('192.0.2.1')] .. method:: overlaps(other) @@ -620,6 +624,12 @@ dictionaries. .. attribute:: num_addresses .. attribute:: prefixlen .. method:: hosts() + + Returns an iterator over the usable hosts in the network. The usable + hosts are all the IP addresses that belong to the network, except the + Subnet-Router anycast address. For networks with a mask length of 127, + the Subnet-Router anycast address is also included in the result. + .. method:: overlaps(other) .. method:: address_exclude(network) .. method:: subnets(prefixlen_diff=1, new_prefix=None) diff --git a/Lib/ipaddress.py b/Lib/ipaddress.py index 70746f8de85c..583f02ad5427 100644 --- a/Lib/ipaddress.py +++ b/Lib/ipaddress.py @@ -1498,45 +1498,28 @@ def __init__(self, address, strict=True): # Constructing from a packed address or integer if isinstance(address, (int, bytes)): - self.network_address = IPv4Address(address) - self.netmask, self._prefixlen = self._make_netmask(self._max_prefixlen) - #fixme: address/network test here. - return - - if isinstance(address, tuple): - if len(address) > 1: - arg = address[1] - else: - # We weren't given an address[1] - arg = self._max_prefixlen - self.network_address = IPv4Address(address[0]) - self.netmask, self._prefixlen = self._make_netmask(arg) - packed = int(self.network_address) - if packed & int(self.netmask) != packed: - if strict: - raise ValueError('%s has host bits set' % self) - else: - self.network_address = IPv4Address(packed & - int(self.netmask)) - return - + addr = address + mask = self._max_prefixlen + # Constructing from a tuple (addr, [mask]) + elif isinstance(address, tuple): + addr = address[0] + mask = address[1] if len(address) > 1 else self._max_prefixlen # Assume input argument to be string or any object representation # which converts into a formatted IP prefix string. - addr = _split_optional_netmask(address) - self.network_address = IPv4Address(self._ip_int_from_string(addr[0])) - - if len(addr) == 2: - arg = addr[1] else: - arg = self._max_prefixlen - self.netmask, self._prefixlen = self._make_netmask(arg) - - if strict: - if (IPv4Address(int(self.network_address) & int(self.netmask)) != - self.network_address): + args = _split_optional_netmask(address) + addr = self._ip_int_from_string(args[0]) + mask = args[1] if len(args) == 2 else self._max_prefixlen + + self.network_address = IPv4Address(addr) + self.netmask, self._prefixlen = self._make_netmask(mask) + packed = int(self.network_address) + if packed & int(self.netmask) != packed: + if strict: raise ValueError('%s has host bits set' % self) - self.network_address = IPv4Address(int(self.network_address) & - int(self.netmask)) + else: + self.network_address = IPv4Address(packed & + int(self.netmask)) if self._prefixlen == (self._max_prefixlen - 1): self.hosts = self.__iter__ @@ -2191,46 +2174,30 @@ def __init__(self, address, strict=True): """ _BaseNetwork.__init__(self, address) - # Efficient constructor from integer or packed address - if isinstance(address, (bytes, int)): - self.network_address = IPv6Address(address) - self.netmask, self._prefixlen = self._make_netmask(self._max_prefixlen) - return - - if isinstance(address, tuple): - if len(address) > 1: - arg = address[1] - else: - arg = self._max_prefixlen - self.netmask, self._prefixlen = self._make_netmask(arg) - self.network_address = IPv6Address(address[0]) - packed = int(self.network_address) - if packed & int(self.netmask) != packed: - if strict: - raise ValueError('%s has host bits set' % self) - else: - self.network_address = IPv6Address(packed & - int(self.netmask)) - return - + # Constructing from a packed address or integer + if isinstance(address, (int, bytes)): + addr = address + mask = self._max_prefixlen + # Constructing from a tuple (addr, [mask]) + elif isinstance(address, tuple): + addr = address[0] + mask = address[1] if len(address) > 1 else self._max_prefixlen # Assume input argument to be string or any object representation # which converts into a formatted IP prefix string. - addr = _split_optional_netmask(address) - - self.network_address = IPv6Address(self._ip_int_from_string(addr[0])) - - if len(addr) == 2: - arg = addr[1] else: - arg = self._max_prefixlen - self.netmask, self._prefixlen = self._make_netmask(arg) - - if strict: - if (IPv6Address(int(self.network_address) & int(self.netmask)) != - self.network_address): + args = _split_optional_netmask(address) + addr = self._ip_int_from_string(args[0]) + mask = args[1] if len(args) == 2 else self._max_prefixlen + + self.network_address = IPv6Address(addr) + self.netmask, self._prefixlen = self._make_netmask(mask) + packed = int(self.network_address) + if packed & int(self.netmask) != packed: + if strict: raise ValueError('%s has host bits set' % self) - self.network_address = IPv6Address(int(self.network_address) & - int(self.netmask)) + else: + self.network_address = IPv6Address(packed & + int(self.netmask)) if self._prefixlen == (self._max_prefixlen - 1): self.hosts = self.__iter__ diff --git a/Lib/test/test_ipaddress.py b/Lib/test/test_ipaddress.py index 2432eb4960e1..1cef4217bc88 100644 --- a/Lib/test/test_ipaddress.py +++ b/Lib/test/test_ipaddress.py @@ -1042,10 +1042,30 @@ def testHosts(self): self.assertEqual(ipaddress.IPv4Address('1.2.3.1'), hosts[0]) self.assertEqual(ipaddress.IPv4Address('1.2.3.254'), hosts[-1]) + ipv6_network = ipaddress.IPv6Network('2001:658:22a:cafe::/120') + hosts = list(ipv6_network.hosts()) + self.assertEqual(255, len(hosts)) + self.assertEqual(ipaddress.IPv6Address('2001:658:22a:cafe::1'), hosts[0]) + self.assertEqual(ipaddress.IPv6Address('2001:658:22a:cafe::ff'), hosts[-1]) + # special case where only 1 bit is left for address - self.assertEqual([ipaddress.IPv4Address('2.0.0.0'), - ipaddress.IPv4Address('2.0.0.1')], - list(ipaddress.ip_network('2.0.0.0/31').hosts())) + addrs = [ipaddress.IPv4Address('2.0.0.0'), + ipaddress.IPv4Address('2.0.0.1')] + str_args = '2.0.0.0/31' + tpl_args = ('2.0.0.0', 31) + self.assertEqual(addrs, list(ipaddress.ip_network(str_args).hosts())) + self.assertEqual(addrs, list(ipaddress.ip_network(tpl_args).hosts())) + self.assertEqual(list(ipaddress.ip_network(str_args).hosts()), + list(ipaddress.ip_network(tpl_args).hosts())) + + addrs = [ipaddress.IPv6Address('2001:658:22a:cafe::'), + ipaddress.IPv6Address('2001:658:22a:cafe::1')] + str_args = '2001:658:22a:cafe::/127' + tpl_args = ('2001:658:22a:cafe::', 127) + self.assertEqual(addrs, list(ipaddress.ip_network(str_args).hosts())) + self.assertEqual(addrs, list(ipaddress.ip_network(tpl_args).hosts())) + self.assertEqual(list(ipaddress.ip_network(str_args).hosts()), + list(ipaddress.ip_network(tpl_args).hosts())) def testFancySubnetting(self): self.assertEqual(sorted(self.ipv4_network.subnets(prefixlen_diff=3)), diff --git a/Misc/NEWS.d/next/Library/2018-03-07-22-28-17.bpo-27683.572Rv4.rst b/Misc/NEWS.d/next/Library/2018-03-07-22-28-17.bpo-27683.572Rv4.rst new file mode 100644 index 000000000000..4e6dfa8e978c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-03-07-22-28-17.bpo-27683.572Rv4.rst @@ -0,0 +1,3 @@ +Fix a regression in :mod:`ipaddress` that result of :meth:`hosts` +is empty when the network is constructed by a tuple containing an +integer mask and only 1 bit left for addresses. From webhook-mailer at python.org Tue Mar 20 22:00:26 2018 From: webhook-mailer at python.org (Eric V. Smith) Date: Wed, 21 Mar 2018 02:00:26 -0000 Subject: [Python-checkins] bpo-32896: Fix error when subclassing a dataclass with a field that uses a default_factory (GH-6170) Message-ID: <mailman.116.1521597628.1871.python-checkins@python.org> https://github.com/python/cpython/commit/8f6eccdc64cab735c47620fea948e64b19f83684 commit: 8f6eccdc64cab735c47620fea948e64b19f83684 branch: master author: Eric V. Smith <ericvsmith at users.noreply.github.com> committer: GitHub <noreply at github.com> date: 2018-03-20T22:00:23-04:00 summary: bpo-32896: Fix error when subclassing a dataclass with a field that uses a default_factory (GH-6170) Fix the way that new annotations in a class are detected. files: A Misc/NEWS.d/next/Library/2018-03-20-20-53-21.bpo-32896.ewW3Ln.rst M Lib/dataclasses.py M Lib/test/test_dataclasses.py diff --git a/Lib/dataclasses.py b/Lib/dataclasses.py index a4afd50376bd..d61643249148 100644 --- a/Lib/dataclasses.py +++ b/Lib/dataclasses.py @@ -574,17 +574,18 @@ def _get_field(cls, a_name, a_type): def _find_fields(cls): # Return a list of Field objects, in order, for this class (and no - # base classes). Fields are found from __annotations__ (which is - # guaranteed to be ordered). Default values are from class - # attributes, if a field has a default. If the default value is - # a Field(), then it contains additional info beyond (and - # possibly including) the actual default value. Pseudo-fields - # ClassVars and InitVars are included, despite the fact that - # they're not real fields. That's dealt with later. - - annotations = getattr(cls, '__annotations__', {}) - return [_get_field(cls, a_name, a_type) - for a_name, a_type in annotations.items()] + # base classes). Fields are found from the class dict's + # __annotations__ (which is guaranteed to be ordered). Default + # values are from class attributes, if a field has a default. If + # the default value is a Field(), then it contains additional + # info beyond (and possibly including) the actual default value. + # Pseudo-fields ClassVars and InitVars are included, despite the + # fact that they're not real fields. That's dealt with later. + + # If __annotations__ isn't present, then this class adds no new + # annotations. + annotations = cls.__dict__.get('__annotations__', {}) + return [_get_field(cls, name, type) for name, type in annotations.items()] def _set_new_attribute(cls, name, value): diff --git a/Lib/test/test_dataclasses.py b/Lib/test/test_dataclasses.py index db03ec1925f6..9b5aad25745f 100755 --- a/Lib/test/test_dataclasses.py +++ b/Lib/test/test_dataclasses.py @@ -1147,6 +1147,55 @@ class C: C().x self.assertEqual(factory.call_count, 2) + def test_default_factory_derived(self): + # See bpo-32896. + @dataclass + class Foo: + x: dict = field(default_factory=dict) + + @dataclass + class Bar(Foo): + y: int = 1 + + self.assertEqual(Foo().x, {}) + self.assertEqual(Bar().x, {}) + self.assertEqual(Bar().y, 1) + + @dataclass + class Baz(Foo): + pass + self.assertEqual(Baz().x, {}) + + def test_intermediate_non_dataclass(self): + # Test that an intermediate class that defines + # annotations does not define fields. + + @dataclass + class A: + x: int + + class B(A): + y: int + + @dataclass + class C(B): + z: int + + c = C(1, 3) + self.assertEqual((c.x, c.z), (1, 3)) + + # .y was not initialized. + with self.assertRaisesRegex(AttributeError, + 'object has no attribute'): + c.y + + # And if we again derive a non-dataclass, no fields are added. + class D(C): + t: int + d = D(4, 5) + self.assertEqual((d.x, d.z), (4, 5)) + + def x_test_classvar_default_factory(self): # XXX: it's an error for a ClassVar to have a factory function @dataclass diff --git a/Misc/NEWS.d/next/Library/2018-03-20-20-53-21.bpo-32896.ewW3Ln.rst b/Misc/NEWS.d/next/Library/2018-03-20-20-53-21.bpo-32896.ewW3Ln.rst new file mode 100644 index 000000000000..8363da4667a1 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-03-20-20-53-21.bpo-32896.ewW3Ln.rst @@ -0,0 +1,2 @@ +Fix an error where subclassing a dataclass with a field that uses a +default_factory would generate an incorrect class. From webhook-mailer at python.org Wed Mar 21 02:02:46 2018 From: webhook-mailer at python.org (Xiang Zhang) Date: Wed, 21 Mar 2018 06:02:46 -0000 Subject: [Python-checkins] Fix typos in mmap() error messages (GH-6173) Message-ID: <mailman.117.1521612167.1871.python-checkins@python.org> https://github.com/python/cpython/commit/9308dea3e1fd565d50a76a667e4e8ef0568b7053 commit: 9308dea3e1fd565d50a76a667e4e8ef0568b7053 branch: master author: Zackery Spytz <zspytz at gmail.com> committer: Xiang Zhang <angwerzx at 126.com> date: 2018-03-21T14:02:37+08:00 summary: Fix typos in mmap() error messages (GH-6173) files: M Modules/mmapmodule.c diff --git a/Modules/mmapmodule.c b/Modules/mmapmodule.c index 95d42d6dc5e5..9afb79fe2ce0 100644 --- a/Modules/mmapmodule.c +++ b/Modules/mmapmodule.c @@ -1067,7 +1067,7 @@ new_mmap_object(PyTypeObject *type, PyObject *args, PyObject *kwdict) return NULL; if (map_size < 0) { PyErr_SetString(PyExc_OverflowError, - "memory mapped length must be postiive"); + "memory mapped length must be positive"); return NULL; } if (offset < 0) { @@ -1253,7 +1253,7 @@ new_mmap_object(PyTypeObject *type, PyObject *args, PyObject *kwdict) if (map_size < 0) { PyErr_SetString(PyExc_OverflowError, - "memory mapped length must be postiive"); + "memory mapped length must be positive"); return NULL; } if (offset < 0) { From webhook-mailer at python.org Wed Mar 21 02:24:44 2018 From: webhook-mailer at python.org (Miss Islington (bot)) Date: Wed, 21 Mar 2018 06:24:44 -0000 Subject: [Python-checkins] Fix typos in mmap() error messages (GH-6173) Message-ID: <mailman.118.1521613484.1871.python-checkins@python.org> https://github.com/python/cpython/commit/56f530ef8ae8f42bd9c2c8e9d22a13dc03fdff1d commit: 56f530ef8ae8f42bd9c2c8e9d22a13dc03fdff1d branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: GitHub <noreply at github.com> date: 2018-03-20T23:24:41-07:00 summary: Fix typos in mmap() error messages (GH-6173) (cherry picked from commit 9308dea3e1fd565d50a76a667e4e8ef0568b7053) Co-authored-by: Zackery Spytz <zspytz at gmail.com> files: M Modules/mmapmodule.c diff --git a/Modules/mmapmodule.c b/Modules/mmapmodule.c index 95d42d6dc5e5..9afb79fe2ce0 100644 --- a/Modules/mmapmodule.c +++ b/Modules/mmapmodule.c @@ -1067,7 +1067,7 @@ new_mmap_object(PyTypeObject *type, PyObject *args, PyObject *kwdict) return NULL; if (map_size < 0) { PyErr_SetString(PyExc_OverflowError, - "memory mapped length must be postiive"); + "memory mapped length must be positive"); return NULL; } if (offset < 0) { @@ -1253,7 +1253,7 @@ new_mmap_object(PyTypeObject *type, PyObject *args, PyObject *kwdict) if (map_size < 0) { PyErr_SetString(PyExc_OverflowError, - "memory mapped length must be postiive"); + "memory mapped length must be positive"); return NULL; } if (offset < 0) { From webhook-mailer at python.org Wed Mar 21 02:55:47 2018 From: webhook-mailer at python.org (Miss Islington (bot)) Date: Wed, 21 Mar 2018 06:55:47 -0000 Subject: [Python-checkins] Fix typos in mmap() error messages (GH-6173) Message-ID: <mailman.119.1521615349.1871.python-checkins@python.org> https://github.com/python/cpython/commit/7ee093614b7e22fda443666d3ef23805126f4ab1 commit: 7ee093614b7e22fda443666d3ef23805126f4ab1 branch: 2.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: GitHub <noreply at github.com> date: 2018-03-20T23:55:44-07:00 summary: Fix typos in mmap() error messages (GH-6173) (cherry picked from commit 9308dea3e1fd565d50a76a667e4e8ef0568b7053) Co-authored-by: Zackery Spytz <zspytz at gmail.com> files: M Modules/mmapmodule.c diff --git a/Modules/mmapmodule.c b/Modules/mmapmodule.c index 5532c4484db2..2ae52c7b28df 100644 --- a/Modules/mmapmodule.c +++ b/Modules/mmapmodule.c @@ -1114,7 +1114,7 @@ new_mmap_object(PyTypeObject *type, PyObject *args, PyObject *kwdict) return NULL; if (map_size < 0) { PyErr_SetString(PyExc_OverflowError, - "memory mapped length must be postiive"); + "memory mapped length must be positive"); return NULL; } if (offset < 0) { @@ -1300,7 +1300,7 @@ new_mmap_object(PyTypeObject *type, PyObject *args, PyObject *kwdict) if (map_size < 0) { PyErr_SetString(PyExc_OverflowError, - "memory mapped length must be postiive"); + "memory mapped length must be positive"); return NULL; } if (offset < 0) { From webhook-mailer at python.org Wed Mar 21 03:07:26 2018 From: webhook-mailer at python.org (Miss Islington (bot)) Date: Wed, 21 Mar 2018 07:07:26 -0000 Subject: [Python-checkins] Fix typos in mmap() error messages (GH-6173) Message-ID: <mailman.120.1521616047.1871.python-checkins@python.org> https://github.com/python/cpython/commit/7abf34344fd8f1a9ef92d73ee8c5d1591688c4ca commit: 7abf34344fd8f1a9ef92d73ee8c5d1591688c4ca branch: 3.6 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: GitHub <noreply at github.com> date: 2018-03-21T00:07:23-07:00 summary: Fix typos in mmap() error messages (GH-6173) (cherry picked from commit 9308dea3e1fd565d50a76a667e4e8ef0568b7053) Co-authored-by: Zackery Spytz <zspytz at gmail.com> files: M Modules/mmapmodule.c diff --git a/Modules/mmapmodule.c b/Modules/mmapmodule.c index 8acb61ab98a9..4a875cc825a6 100644 --- a/Modules/mmapmodule.c +++ b/Modules/mmapmodule.c @@ -1095,7 +1095,7 @@ new_mmap_object(PyTypeObject *type, PyObject *args, PyObject *kwdict) return NULL; if (map_size < 0) { PyErr_SetString(PyExc_OverflowError, - "memory mapped length must be postiive"); + "memory mapped length must be positive"); return NULL; } if (offset < 0) { @@ -1281,7 +1281,7 @@ new_mmap_object(PyTypeObject *type, PyObject *args, PyObject *kwdict) if (map_size < 0) { PyErr_SetString(PyExc_OverflowError, - "memory mapped length must be postiive"); + "memory mapped length must be positive"); return NULL; } if (offset < 0) { From solipsis at pitrou.net Wed Mar 21 05:15:31 2018 From: solipsis at pitrou.net (solipsis at pitrou.net) Date: Wed, 21 Mar 2018 09:15:31 +0000 Subject: [Python-checkins] Daily reference leaks (4243df51fe43): sum=5 Message-ID: <20180321091531.1.1460EAC711E3626C@psf.io> results for 4243df51fe43 on branch "default" -------------------------------------------- test_collections leaked [7, -7, 1] memory blocks, sum=1 test_functools leaked [0, 3, 1] memory blocks, sum=4 Command line was: ['./python', '-m', 'test.regrtest', '-uall', '-R', '3:3:/home/psf-users/antoine/refleaks/reflogCU1WKl', '--timeout', '7200'] From webhook-mailer at python.org Wed Mar 21 05:22:08 2018 From: webhook-mailer at python.org (Eric V. Smith) Date: Wed, 21 Mar 2018 09:22:08 -0000 Subject: [Python-checkins] bpo-32896: Fix error when subclassing a dataclass with a field that uses a default_factory (GH-6170) (GH-6171) Message-ID: <mailman.121.1521624129.1871.python-checkins@python.org> https://github.com/python/cpython/commit/22136c94b6e43c8c584a54f3a513b83b753b96ee commit: 22136c94b6e43c8c584a54f3a513b83b753b96ee branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Eric V. Smith <ericvsmith at users.noreply.github.com> date: 2018-03-21T05:17:30-04:00 summary: bpo-32896: Fix error when subclassing a dataclass with a field that uses a default_factory (GH-6170) (GH-6171) Fix the way that new annotations in a class are detected. (cherry picked from commit 8f6eccdc64cab735c47620fea948e64b19f83684) Co-authored-by: Eric V. Smith <ericvsmith at users.noreply.github.com> files: A Misc/NEWS.d/next/Library/2018-03-20-20-53-21.bpo-32896.ewW3Ln.rst M Lib/dataclasses.py M Lib/test/test_dataclasses.py diff --git a/Lib/dataclasses.py b/Lib/dataclasses.py index a4afd50376bd..d61643249148 100644 --- a/Lib/dataclasses.py +++ b/Lib/dataclasses.py @@ -574,17 +574,18 @@ def _get_field(cls, a_name, a_type): def _find_fields(cls): # Return a list of Field objects, in order, for this class (and no - # base classes). Fields are found from __annotations__ (which is - # guaranteed to be ordered). Default values are from class - # attributes, if a field has a default. If the default value is - # a Field(), then it contains additional info beyond (and - # possibly including) the actual default value. Pseudo-fields - # ClassVars and InitVars are included, despite the fact that - # they're not real fields. That's dealt with later. - - annotations = getattr(cls, '__annotations__', {}) - return [_get_field(cls, a_name, a_type) - for a_name, a_type in annotations.items()] + # base classes). Fields are found from the class dict's + # __annotations__ (which is guaranteed to be ordered). Default + # values are from class attributes, if a field has a default. If + # the default value is a Field(), then it contains additional + # info beyond (and possibly including) the actual default value. + # Pseudo-fields ClassVars and InitVars are included, despite the + # fact that they're not real fields. That's dealt with later. + + # If __annotations__ isn't present, then this class adds no new + # annotations. + annotations = cls.__dict__.get('__annotations__', {}) + return [_get_field(cls, name, type) for name, type in annotations.items()] def _set_new_attribute(cls, name, value): diff --git a/Lib/test/test_dataclasses.py b/Lib/test/test_dataclasses.py index db03ec1925f6..9b5aad25745f 100755 --- a/Lib/test/test_dataclasses.py +++ b/Lib/test/test_dataclasses.py @@ -1147,6 +1147,55 @@ class C: C().x self.assertEqual(factory.call_count, 2) + def test_default_factory_derived(self): + # See bpo-32896. + @dataclass + class Foo: + x: dict = field(default_factory=dict) + + @dataclass + class Bar(Foo): + y: int = 1 + + self.assertEqual(Foo().x, {}) + self.assertEqual(Bar().x, {}) + self.assertEqual(Bar().y, 1) + + @dataclass + class Baz(Foo): + pass + self.assertEqual(Baz().x, {}) + + def test_intermediate_non_dataclass(self): + # Test that an intermediate class that defines + # annotations does not define fields. + + @dataclass + class A: + x: int + + class B(A): + y: int + + @dataclass + class C(B): + z: int + + c = C(1, 3) + self.assertEqual((c.x, c.z), (1, 3)) + + # .y was not initialized. + with self.assertRaisesRegex(AttributeError, + 'object has no attribute'): + c.y + + # And if we again derive a non-dataclass, no fields are added. + class D(C): + t: int + d = D(4, 5) + self.assertEqual((d.x, d.z), (4, 5)) + + def x_test_classvar_default_factory(self): # XXX: it's an error for a ClassVar to have a factory function @dataclass diff --git a/Misc/NEWS.d/next/Library/2018-03-20-20-53-21.bpo-32896.ewW3Ln.rst b/Misc/NEWS.d/next/Library/2018-03-20-20-53-21.bpo-32896.ewW3Ln.rst new file mode 100644 index 000000000000..8363da4667a1 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-03-20-20-53-21.bpo-32896.ewW3Ln.rst @@ -0,0 +1,2 @@ +Fix an error where subclassing a dataclass with a field that uses a +default_factory would generate an incorrect class. From webhook-mailer at python.org Wed Mar 21 11:50:36 2018 From: webhook-mailer at python.org (Antoine Pitrou) Date: Wed, 21 Mar 2018 15:50:36 -0000 Subject: [Python-checkins] bpo-33078 - Fix queue size on pickling error (GH-6119) Message-ID: <mailman.122.1521647439.1871.python-checkins@python.org> https://github.com/python/cpython/commit/e2f33add635df4fde81be9960bab367e010c19bf commit: e2f33add635df4fde81be9960bab367e010c19bf branch: master author: Thomas Moreau <thomas.moreau.2010 at gmail.com> committer: Antoine Pitrou <pitrou at free.fr> date: 2018-03-21T16:50:28+01:00 summary: bpo-33078 - Fix queue size on pickling error (GH-6119) files: A Misc/NEWS.d/next/Library/2018-03-15-07-38-00.bpo-33078.RmjUF5.rst M Lib/multiprocessing/queues.py M Lib/test/_test_multiprocessing.py diff --git a/Lib/multiprocessing/queues.py b/Lib/multiprocessing/queues.py index d66d37a5c3e2..715a9b0e1290 100644 --- a/Lib/multiprocessing/queues.py +++ b/Lib/multiprocessing/queues.py @@ -161,7 +161,7 @@ def _start_thread(self): target=Queue._feed, args=(self._buffer, self._notempty, self._send_bytes, self._wlock, self._writer.close, self._ignore_epipe, - self._on_queue_feeder_error), + self._on_queue_feeder_error, self._sem), name='QueueFeederThread' ) self._thread.daemon = True @@ -203,7 +203,7 @@ def _finalize_close(buffer, notempty): @staticmethod def _feed(buffer, notempty, send_bytes, writelock, close, ignore_epipe, - onerror): + onerror, queue_sem): debug('starting thread to feed data to pipe') nacquire = notempty.acquire nrelease = notempty.release @@ -255,6 +255,12 @@ def _feed(buffer, notempty, send_bytes, writelock, close, ignore_epipe, info('error in queue thread: %s', e) return else: + # Since the object has not been sent in the queue, we need + # to decrease the size of the queue. The error acts as + # if the object had been silently removed from the queue + # and this step is necessary to have a properly working + # queue. + queue_sem.release() onerror(e, obj) @staticmethod diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index 940fe584e752..c787702f1d4c 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -1056,6 +1056,19 @@ def __reduce__(self): self.assertTrue(q.get(timeout=1.0)) close_queue(q) + with test.support.captured_stderr(): + # bpo-33078: verify that the queue size is correctly handled + # on errors. + q = self.Queue(maxsize=1) + q.put(NotSerializable()) + q.put(True) + self.assertEqual(q.qsize(), 1) + # bpo-30595: use a timeout of 1 second for slow buildbots + self.assertTrue(q.get(timeout=1.0)) + # Check that the size of the queue is correct + self.assertEqual(q.qsize(), 0) + close_queue(q) + def test_queue_feeder_on_queue_feeder_error(self): # bpo-30006: verify feeder handles exceptions using the # _on_queue_feeder_error hook. diff --git a/Misc/NEWS.d/next/Library/2018-03-15-07-38-00.bpo-33078.RmjUF5.rst b/Misc/NEWS.d/next/Library/2018-03-15-07-38-00.bpo-33078.RmjUF5.rst new file mode 100644 index 000000000000..55c2b1de8668 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-03-15-07-38-00.bpo-33078.RmjUF5.rst @@ -0,0 +1,2 @@ +Fix the size handling in multiprocessing.Queue when a pickling error +occurs. From webhook-mailer at python.org Wed Mar 21 12:21:20 2018 From: webhook-mailer at python.org (Antoine Pitrou) Date: Wed, 21 Mar 2018 16:21:20 -0000 Subject: [Python-checkins] bpo-33078 - Fix queue size on pickling error (GH-6119) (GH-6178) Message-ID: <mailman.123.1521649283.1871.python-checkins@python.org> https://github.com/python/cpython/commit/bb5b5291971c104ea773db1a30e46d410b6b3e1e commit: bb5b5291971c104ea773db1a30e46d410b6b3e1e branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Antoine Pitrou <pitrou at free.fr> date: 2018-03-21T17:21:15+01:00 summary: bpo-33078 - Fix queue size on pickling error (GH-6119) (GH-6178) (cherry picked from commit e2f33add635df4fde81be9960bab367e010c19bf) Co-authored-by: Thomas Moreau <thomas.moreau.2010 at gmail.com> files: A Misc/NEWS.d/next/Library/2018-03-15-07-38-00.bpo-33078.RmjUF5.rst M Lib/multiprocessing/queues.py M Lib/test/_test_multiprocessing.py diff --git a/Lib/multiprocessing/queues.py b/Lib/multiprocessing/queues.py index d66d37a5c3e2..715a9b0e1290 100644 --- a/Lib/multiprocessing/queues.py +++ b/Lib/multiprocessing/queues.py @@ -161,7 +161,7 @@ def _start_thread(self): target=Queue._feed, args=(self._buffer, self._notempty, self._send_bytes, self._wlock, self._writer.close, self._ignore_epipe, - self._on_queue_feeder_error), + self._on_queue_feeder_error, self._sem), name='QueueFeederThread' ) self._thread.daemon = True @@ -203,7 +203,7 @@ def _finalize_close(buffer, notempty): @staticmethod def _feed(buffer, notempty, send_bytes, writelock, close, ignore_epipe, - onerror): + onerror, queue_sem): debug('starting thread to feed data to pipe') nacquire = notempty.acquire nrelease = notempty.release @@ -255,6 +255,12 @@ def _feed(buffer, notempty, send_bytes, writelock, close, ignore_epipe, info('error in queue thread: %s', e) return else: + # Since the object has not been sent in the queue, we need + # to decrease the size of the queue. The error acts as + # if the object had been silently removed from the queue + # and this step is necessary to have a properly working + # queue. + queue_sem.release() onerror(e, obj) @staticmethod diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index 940fe584e752..c787702f1d4c 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -1056,6 +1056,19 @@ def __reduce__(self): self.assertTrue(q.get(timeout=1.0)) close_queue(q) + with test.support.captured_stderr(): + # bpo-33078: verify that the queue size is correctly handled + # on errors. + q = self.Queue(maxsize=1) + q.put(NotSerializable()) + q.put(True) + self.assertEqual(q.qsize(), 1) + # bpo-30595: use a timeout of 1 second for slow buildbots + self.assertTrue(q.get(timeout=1.0)) + # Check that the size of the queue is correct + self.assertEqual(q.qsize(), 0) + close_queue(q) + def test_queue_feeder_on_queue_feeder_error(self): # bpo-30006: verify feeder handles exceptions using the # _on_queue_feeder_error hook. diff --git a/Misc/NEWS.d/next/Library/2018-03-15-07-38-00.bpo-33078.RmjUF5.rst b/Misc/NEWS.d/next/Library/2018-03-15-07-38-00.bpo-33078.RmjUF5.rst new file mode 100644 index 000000000000..55c2b1de8668 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-03-15-07-38-00.bpo-33078.RmjUF5.rst @@ -0,0 +1,2 @@ +Fix the size handling in multiprocessing.Queue when a pickling error +occurs. From webhook-mailer at python.org Wed Mar 21 13:56:34 2018 From: webhook-mailer at python.org (Antoine Pitrou) Date: Wed, 21 Mar 2018 17:56:34 -0000 Subject: [Python-checkins] FIX failure on OSX sem_getvalue (#6180) Message-ID: <mailman.124.1521654995.1871.python-checkins@python.org> https://github.com/python/cpython/commit/dec1c7786f642049c2508e909442189dc043b5da commit: dec1c7786f642049c2508e909442189dc043b5da branch: master author: Thomas Moreau <thomas.moreau.2010 at gmail.com> committer: Antoine Pitrou <pitrou at free.fr> date: 2018-03-21T18:56:27+01:00 summary: FIX failure on OSX sem_getvalue (#6180) files: A Misc/NEWS.d/next/Library/2018-03-21-17-59-39.bpo-33078.PQOniT.rst M Lib/test/_test_multiprocessing.py diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index c787702f1d4c..c6a1f5ca9051 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -1062,11 +1062,16 @@ def __reduce__(self): q = self.Queue(maxsize=1) q.put(NotSerializable()) q.put(True) - self.assertEqual(q.qsize(), 1) + try: + self.assertEqual(q.qsize(), 1) + except NotImplementedError: + # qsize is not available on all platform as it + # relies on sem_getvalue + pass # bpo-30595: use a timeout of 1 second for slow buildbots self.assertTrue(q.get(timeout=1.0)) # Check that the size of the queue is correct - self.assertEqual(q.qsize(), 0) + self.assertTrue(q.empty()) close_queue(q) def test_queue_feeder_on_queue_feeder_error(self): diff --git a/Misc/NEWS.d/next/Library/2018-03-21-17-59-39.bpo-33078.PQOniT.rst b/Misc/NEWS.d/next/Library/2018-03-21-17-59-39.bpo-33078.PQOniT.rst new file mode 100644 index 000000000000..8b71bb32e0ec --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-03-21-17-59-39.bpo-33078.PQOniT.rst @@ -0,0 +1 @@ +Fix the failure on OSX caused by the tests relying on sem_getvalue From webhook-mailer at python.org Wed Mar 21 15:01:05 2018 From: webhook-mailer at python.org (Antoine Pitrou) Date: Wed, 21 Mar 2018 19:01:05 -0000 Subject: [Python-checkins] FIX failure on OSX sem_getvalue (GH-6180) (GH-6181) Message-ID: <mailman.125.1521658865.1871.python-checkins@python.org> https://github.com/python/cpython/commit/f5625d58fa3474f654defab19624c62a4915e6c9 commit: f5625d58fa3474f654defab19624c62a4915e6c9 branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Antoine Pitrou <pitrou at free.fr> date: 2018-03-21T20:00:55+01:00 summary: FIX failure on OSX sem_getvalue (GH-6180) (GH-6181) (cherry picked from commit dec1c7786f642049c2508e909442189dc043b5da) Co-authored-by: Thomas Moreau <thomas.moreau.2010 at gmail.com> files: A Misc/NEWS.d/next/Library/2018-03-21-17-59-39.bpo-33078.PQOniT.rst M Lib/test/_test_multiprocessing.py diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index c787702f1d4c..c6a1f5ca9051 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -1062,11 +1062,16 @@ def __reduce__(self): q = self.Queue(maxsize=1) q.put(NotSerializable()) q.put(True) - self.assertEqual(q.qsize(), 1) + try: + self.assertEqual(q.qsize(), 1) + except NotImplementedError: + # qsize is not available on all platform as it + # relies on sem_getvalue + pass # bpo-30595: use a timeout of 1 second for slow buildbots self.assertTrue(q.get(timeout=1.0)) # Check that the size of the queue is correct - self.assertEqual(q.qsize(), 0) + self.assertTrue(q.empty()) close_queue(q) def test_queue_feeder_on_queue_feeder_error(self): diff --git a/Misc/NEWS.d/next/Library/2018-03-21-17-59-39.bpo-33078.PQOniT.rst b/Misc/NEWS.d/next/Library/2018-03-21-17-59-39.bpo-33078.PQOniT.rst new file mode 100644 index 000000000000..8b71bb32e0ec --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-03-21-17-59-39.bpo-33078.PQOniT.rst @@ -0,0 +1 @@ +Fix the failure on OSX caused by the tests relying on sem_getvalue From webhook-mailer at python.org Wed Mar 21 17:10:25 2018 From: webhook-mailer at python.org (Eric V. Smith) Date: Wed, 21 Mar 2018 21:10:25 -0000 Subject: [Python-checkins] Add 'Field' to dataclasses.__all__. (GH-6182) Message-ID: <mailman.126.1521666625.1871.python-checkins@python.org> https://github.com/python/cpython/commit/8e4560a9da6a02aa157dd7df8bd0be0d258c0a73 commit: 8e4560a9da6a02aa157dd7df8bd0be0d258c0a73 branch: master author: Eric V. Smith <ericvsmith at users.noreply.github.com> committer: GitHub <noreply at github.com> date: 2018-03-21T17:10:22-04:00 summary: Add 'Field' to dataclasses.__all__. (GH-6182) - Add missing 'Field' to __all__. - Improve tests to catch this. files: A Misc/NEWS.d/next/Library/2018-03-21-16-52-26.bpo-33116.Tvzerj.rst M Lib/dataclasses.py M Lib/test/test_dataclasses.py diff --git a/Lib/dataclasses.py b/Lib/dataclasses.py index d61643249148..41b5b5da325c 100644 --- a/Lib/dataclasses.py +++ b/Lib/dataclasses.py @@ -5,6 +5,7 @@ __all__ = ['dataclass', 'field', + 'Field', 'FrozenInstanceError', 'InitVar', 'MISSING', @@ -513,7 +514,7 @@ def _get_field(cls, a_name, a_type): # and InitVars are also returned, but marked as such (see # f._field_type). - # If the default value isn't derived from field, then it's + # If the default value isn't derived from Field, then it's # only a normal default value. Convert it to a Field(). default = getattr(cls, a_name, MISSING) if isinstance(default, Field): diff --git a/Lib/test/test_dataclasses.py b/Lib/test/test_dataclasses.py index 9b5aad25745f..69ace36c2c58 100755 --- a/Lib/test/test_dataclasses.py +++ b/Lib/test/test_dataclasses.py @@ -1,7 +1,8 @@ -from dataclasses import ( - dataclass, field, FrozenInstanceError, fields, asdict, astuple, - make_dataclass, replace, InitVar, Field, MISSING, is_dataclass, -) +# Deliberately use "from dataclasses import *". Every name in __all__ +# is tested, so they all must be present. This is a way to catch +# missing ones. + +from dataclasses import * import pickle import inspect diff --git a/Misc/NEWS.d/next/Library/2018-03-21-16-52-26.bpo-33116.Tvzerj.rst b/Misc/NEWS.d/next/Library/2018-03-21-16-52-26.bpo-33116.Tvzerj.rst new file mode 100644 index 000000000000..90072d8e3030 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-03-21-16-52-26.bpo-33116.Tvzerj.rst @@ -0,0 +1 @@ +Add 'Field' to dataclasses.__all__. From webhook-mailer at python.org Wed Mar 21 17:44:26 2018 From: webhook-mailer at python.org (Eric V. Smith) Date: Wed, 21 Mar 2018 21:44:26 -0000 Subject: [Python-checkins] bpo-33116: Add 'Field' to dataclasses.__all__. (GH-6182) (GH-6183) Message-ID: <mailman.127.1521668668.1871.python-checkins@python.org> https://github.com/python/cpython/commit/4ddc99d15963b0374f9dbfd57f14e6194ad65669 commit: 4ddc99d15963b0374f9dbfd57f14e6194ad65669 branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Eric V. Smith <ericvsmith at users.noreply.github.com> date: 2018-03-21T17:44:23-04:00 summary: bpo-33116: Add 'Field' to dataclasses.__all__. (GH-6182) (GH-6183) - Add missing 'Field' to __all__. - Improve tests to catch this. (cherry picked from commit 8e4560a9da6a02aa157dd7df8bd0be0d258c0a73) Co-authored-by: Eric V. Smith <ericvsmith at users.noreply.github.com> files: A Misc/NEWS.d/next/Library/2018-03-21-16-52-26.bpo-33116.Tvzerj.rst M Lib/dataclasses.py M Lib/test/test_dataclasses.py diff --git a/Lib/dataclasses.py b/Lib/dataclasses.py index d61643249148..41b5b5da325c 100644 --- a/Lib/dataclasses.py +++ b/Lib/dataclasses.py @@ -5,6 +5,7 @@ __all__ = ['dataclass', 'field', + 'Field', 'FrozenInstanceError', 'InitVar', 'MISSING', @@ -513,7 +514,7 @@ def _get_field(cls, a_name, a_type): # and InitVars are also returned, but marked as such (see # f._field_type). - # If the default value isn't derived from field, then it's + # If the default value isn't derived from Field, then it's # only a normal default value. Convert it to a Field(). default = getattr(cls, a_name, MISSING) if isinstance(default, Field): diff --git a/Lib/test/test_dataclasses.py b/Lib/test/test_dataclasses.py index 9b5aad25745f..69ace36c2c58 100755 --- a/Lib/test/test_dataclasses.py +++ b/Lib/test/test_dataclasses.py @@ -1,7 +1,8 @@ -from dataclasses import ( - dataclass, field, FrozenInstanceError, fields, asdict, astuple, - make_dataclass, replace, InitVar, Field, MISSING, is_dataclass, -) +# Deliberately use "from dataclasses import *". Every name in __all__ +# is tested, so they all must be present. This is a way to catch +# missing ones. + +from dataclasses import * import pickle import inspect diff --git a/Misc/NEWS.d/next/Library/2018-03-21-16-52-26.bpo-33116.Tvzerj.rst b/Misc/NEWS.d/next/Library/2018-03-21-16-52-26.bpo-33116.Tvzerj.rst new file mode 100644 index 000000000000..90072d8e3030 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-03-21-16-52-26.bpo-33116.Tvzerj.rst @@ -0,0 +1 @@ +Add 'Field' to dataclasses.__all__. From webhook-mailer at python.org Wed Mar 21 22:47:47 2018 From: webhook-mailer at python.org (Donald Stufft) Date: Thu, 22 Mar 2018 02:47:47 -0000 Subject: [Python-checkins] Upgrade pip to v9.0.3 and setuptools to v39.0.1 (#6184) Message-ID: <mailman.128.1521686867.1871.python-checkins@python.org> https://github.com/python/cpython/commit/d93b5161af12291f3f98a260c90cc2975ea9e9cd commit: d93b5161af12291f3f98a260c90cc2975ea9e9cd branch: master author: Donald Stufft <donald at stufft.io> committer: GitHub <noreply at github.com> date: 2018-03-21T22:47:44-04:00 summary: Upgrade pip to v9.0.3 and setuptools to v39.0.1 (#6184) files: A Lib/ensurepip/_bundled/pip-9.0.3-py2.py3-none-any.whl A Lib/ensurepip/_bundled/setuptools-39.0.1-py2.py3-none-any.whl D Lib/ensurepip/_bundled/pip-9.0.2-py2.py3-none-any.whl D Lib/ensurepip/_bundled/setuptools-38.6.1-py2.py3-none-any.whl M Lib/ensurepip/__init__.py diff --git a/Lib/ensurepip/__init__.py b/Lib/ensurepip/__init__.py index d7a6dcf6bc85..4a5b0583f885 100644 --- a/Lib/ensurepip/__init__.py +++ b/Lib/ensurepip/__init__.py @@ -8,9 +8,9 @@ __all__ = ["version", "bootstrap"] -_SETUPTOOLS_VERSION = "38.6.1" +_SETUPTOOLS_VERSION = "39.0.1" -_PIP_VERSION = "9.0.2" +_PIP_VERSION = "9.0.3" _PROJECTS = [ ("setuptools", _SETUPTOOLS_VERSION), diff --git a/Lib/ensurepip/_bundled/pip-9.0.2-py2.py3-none-any.whl b/Lib/ensurepip/_bundled/pip-9.0.3-py2.py3-none-any.whl similarity index 95% rename from Lib/ensurepip/_bundled/pip-9.0.2-py2.py3-none-any.whl rename to Lib/ensurepip/_bundled/pip-9.0.3-py2.py3-none-any.whl index baf1f57e3d8b..5d33bba22868 100644 Binary files a/Lib/ensurepip/_bundled/pip-9.0.2-py2.py3-none-any.whl and b/Lib/ensurepip/_bundled/pip-9.0.3-py2.py3-none-any.whl differ diff --git a/Lib/ensurepip/_bundled/setuptools-38.6.1-py2.py3-none-any.whl b/Lib/ensurepip/_bundled/setuptools-39.0.1-py2.py3-none-any.whl similarity index 70% rename from Lib/ensurepip/_bundled/setuptools-38.6.1-py2.py3-none-any.whl rename to Lib/ensurepip/_bundled/setuptools-39.0.1-py2.py3-none-any.whl index 2707d8be4a2d..edc3ca2d8ec3 100644 Binary files a/Lib/ensurepip/_bundled/setuptools-38.6.1-py2.py3-none-any.whl and b/Lib/ensurepip/_bundled/setuptools-39.0.1-py2.py3-none-any.whl differ From webhook-mailer at python.org Wed Mar 21 23:38:32 2018 From: webhook-mailer at python.org (Donald Stufft) Date: Thu, 22 Mar 2018 03:38:32 -0000 Subject: [Python-checkins] Upgrade pip to v9.0.3 and setuptools to v39.0.1 (GH-6184) Message-ID: <mailman.129.1521689913.1871.python-checkins@python.org> https://github.com/python/cpython/commit/8f46176f0e19d31d8642735e535183a39c5e0bdc commit: 8f46176f0e19d31d8642735e535183a39c5e0bdc branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Donald Stufft <donald at stufft.io> date: 2018-03-21T23:38:24-04:00 summary: Upgrade pip to v9.0.3 and setuptools to v39.0.1 (GH-6184) (cherry picked from commit d93b5161af12291f3f98a260c90cc2975ea9e9cd) Co-authored-by: Donald Stufft <donald at stufft.io> files: A Lib/ensurepip/_bundled/pip-9.0.3-py2.py3-none-any.whl A Lib/ensurepip/_bundled/setuptools-39.0.1-py2.py3-none-any.whl D Lib/ensurepip/_bundled/pip-9.0.2-py2.py3-none-any.whl D Lib/ensurepip/_bundled/setuptools-38.6.1-py2.py3-none-any.whl M Lib/ensurepip/__init__.py diff --git a/Lib/ensurepip/__init__.py b/Lib/ensurepip/__init__.py index d7a6dcf6bc85..4a5b0583f885 100644 --- a/Lib/ensurepip/__init__.py +++ b/Lib/ensurepip/__init__.py @@ -8,9 +8,9 @@ __all__ = ["version", "bootstrap"] -_SETUPTOOLS_VERSION = "38.6.1" +_SETUPTOOLS_VERSION = "39.0.1" -_PIP_VERSION = "9.0.2" +_PIP_VERSION = "9.0.3" _PROJECTS = [ ("setuptools", _SETUPTOOLS_VERSION), diff --git a/Lib/ensurepip/_bundled/pip-9.0.2-py2.py3-none-any.whl b/Lib/ensurepip/_bundled/pip-9.0.3-py2.py3-none-any.whl similarity index 95% rename from Lib/ensurepip/_bundled/pip-9.0.2-py2.py3-none-any.whl rename to Lib/ensurepip/_bundled/pip-9.0.3-py2.py3-none-any.whl index baf1f57e3d8b..5d33bba22868 100644 Binary files a/Lib/ensurepip/_bundled/pip-9.0.2-py2.py3-none-any.whl and b/Lib/ensurepip/_bundled/pip-9.0.3-py2.py3-none-any.whl differ diff --git a/Lib/ensurepip/_bundled/setuptools-38.6.1-py2.py3-none-any.whl b/Lib/ensurepip/_bundled/setuptools-39.0.1-py2.py3-none-any.whl similarity index 70% rename from Lib/ensurepip/_bundled/setuptools-38.6.1-py2.py3-none-any.whl rename to Lib/ensurepip/_bundled/setuptools-39.0.1-py2.py3-none-any.whl index 2707d8be4a2d..edc3ca2d8ec3 100644 Binary files a/Lib/ensurepip/_bundled/setuptools-38.6.1-py2.py3-none-any.whl and b/Lib/ensurepip/_bundled/setuptools-39.0.1-py2.py3-none-any.whl differ From webhook-mailer at python.org Wed Mar 21 23:50:53 2018 From: webhook-mailer at python.org (Donald Stufft) Date: Thu, 22 Mar 2018 03:50:53 -0000 Subject: [Python-checkins] Upgrade pip to v9.0.3 and setuptools to v39.0.1 (GH-6184) Message-ID: <mailman.130.1521690654.1871.python-checkins@python.org> https://github.com/python/cpython/commit/560ea272b01acaa6c531cc7d94331b2ef0854be6 commit: 560ea272b01acaa6c531cc7d94331b2ef0854be6 branch: 3.6 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Donald Stufft <donald at stufft.io> date: 2018-03-21T23:50:50-04:00 summary: Upgrade pip to v9.0.3 and setuptools to v39.0.1 (GH-6184) (cherry picked from commit d93b5161af12291f3f98a260c90cc2975ea9e9cd) Co-authored-by: Donald Stufft <donald at stufft.io> files: A Lib/ensurepip/_bundled/pip-9.0.3-py2.py3-none-any.whl A Lib/ensurepip/_bundled/setuptools-39.0.1-py2.py3-none-any.whl D Lib/ensurepip/_bundled/pip-9.0.2-py2.py3-none-any.whl D Lib/ensurepip/_bundled/setuptools-38.6.1-py2.py3-none-any.whl M Lib/ensurepip/__init__.py diff --git a/Lib/ensurepip/__init__.py b/Lib/ensurepip/__init__.py index d7a6dcf6bc85..4a5b0583f885 100644 --- a/Lib/ensurepip/__init__.py +++ b/Lib/ensurepip/__init__.py @@ -8,9 +8,9 @@ __all__ = ["version", "bootstrap"] -_SETUPTOOLS_VERSION = "38.6.1" +_SETUPTOOLS_VERSION = "39.0.1" -_PIP_VERSION = "9.0.2" +_PIP_VERSION = "9.0.3" _PROJECTS = [ ("setuptools", _SETUPTOOLS_VERSION), diff --git a/Lib/ensurepip/_bundled/pip-9.0.2-py2.py3-none-any.whl b/Lib/ensurepip/_bundled/pip-9.0.3-py2.py3-none-any.whl similarity index 95% rename from Lib/ensurepip/_bundled/pip-9.0.2-py2.py3-none-any.whl rename to Lib/ensurepip/_bundled/pip-9.0.3-py2.py3-none-any.whl index baf1f57e3d8b..5d33bba22868 100644 Binary files a/Lib/ensurepip/_bundled/pip-9.0.2-py2.py3-none-any.whl and b/Lib/ensurepip/_bundled/pip-9.0.3-py2.py3-none-any.whl differ diff --git a/Lib/ensurepip/_bundled/setuptools-38.6.1-py2.py3-none-any.whl b/Lib/ensurepip/_bundled/setuptools-39.0.1-py2.py3-none-any.whl similarity index 70% rename from Lib/ensurepip/_bundled/setuptools-38.6.1-py2.py3-none-any.whl rename to Lib/ensurepip/_bundled/setuptools-39.0.1-py2.py3-none-any.whl index 2707d8be4a2d..edc3ca2d8ec3 100644 Binary files a/Lib/ensurepip/_bundled/setuptools-38.6.1-py2.py3-none-any.whl and b/Lib/ensurepip/_bundled/setuptools-39.0.1-py2.py3-none-any.whl differ From webhook-mailer at python.org Thu Mar 22 00:14:26 2018 From: webhook-mailer at python.org (Donald Stufft) Date: Thu, 22 Mar 2018 04:14:26 -0000 Subject: [Python-checkins] Upgrade pip to v9.0.3 and setuptools to v39.0.1 (GH-6184) Message-ID: <mailman.131.1521692067.1871.python-checkins@python.org> https://github.com/python/cpython/commit/1ce4e5bee6df476836f799456f2caf77cd13dc97 commit: 1ce4e5bee6df476836f799456f2caf77cd13dc97 branch: 2.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Donald Stufft <donald at stufft.io> date: 2018-03-22T00:14:22-04:00 summary: Upgrade pip to v9.0.3 and setuptools to v39.0.1 (GH-6184) (cherry picked from commit d93b5161af12291f3f98a260c90cc2975ea9e9cd) Co-authored-by: Donald Stufft <donald at stufft.io> files: A Lib/ensurepip/_bundled/pip-9.0.3-py2.py3-none-any.whl A Lib/ensurepip/_bundled/setuptools-39.0.1-py2.py3-none-any.whl D Lib/ensurepip/_bundled/pip-9.0.2-py2.py3-none-any.whl D Lib/ensurepip/_bundled/setuptools-38.6.1-py2.py3-none-any.whl M Lib/ensurepip/__init__.py diff --git a/Lib/ensurepip/__init__.py b/Lib/ensurepip/__init__.py index 167696720267..89ed1efdaf2b 100644 --- a/Lib/ensurepip/__init__.py +++ b/Lib/ensurepip/__init__.py @@ -12,9 +12,9 @@ __all__ = ["version", "bootstrap"] -_SETUPTOOLS_VERSION = "38.6.1" +_SETUPTOOLS_VERSION = "39.0.1" -_PIP_VERSION = "9.0.2" +_PIP_VERSION = "9.0.3" _PROJECTS = [ ("setuptools", _SETUPTOOLS_VERSION), diff --git a/Lib/ensurepip/_bundled/pip-9.0.2-py2.py3-none-any.whl b/Lib/ensurepip/_bundled/pip-9.0.3-py2.py3-none-any.whl similarity index 95% rename from Lib/ensurepip/_bundled/pip-9.0.2-py2.py3-none-any.whl rename to Lib/ensurepip/_bundled/pip-9.0.3-py2.py3-none-any.whl index baf1f57e3d8b..5d33bba22868 100644 Binary files a/Lib/ensurepip/_bundled/pip-9.0.2-py2.py3-none-any.whl and b/Lib/ensurepip/_bundled/pip-9.0.3-py2.py3-none-any.whl differ diff --git a/Lib/ensurepip/_bundled/setuptools-38.6.1-py2.py3-none-any.whl b/Lib/ensurepip/_bundled/setuptools-39.0.1-py2.py3-none-any.whl similarity index 70% rename from Lib/ensurepip/_bundled/setuptools-38.6.1-py2.py3-none-any.whl rename to Lib/ensurepip/_bundled/setuptools-39.0.1-py2.py3-none-any.whl index 2707d8be4a2d..edc3ca2d8ec3 100644 Binary files a/Lib/ensurepip/_bundled/setuptools-38.6.1-py2.py3-none-any.whl and b/Lib/ensurepip/_bundled/setuptools-39.0.1-py2.py3-none-any.whl differ From solipsis at pitrou.net Thu Mar 22 05:10:00 2018 From: solipsis at pitrou.net (solipsis at pitrou.net) Date: Thu, 22 Mar 2018 09:10:00 +0000 Subject: [Python-checkins] Daily reference leaks (4243df51fe43): sum=12 Message-ID: <20180322091000.1.74AE09F54712D1FD@psf.io> results for 4243df51fe43 on branch "default" -------------------------------------------- test_collections leaked [0, 7, 0] memory blocks, sum=7 test_functools leaked [0, 3, 1] memory blocks, sum=4 test_multiprocessing_fork leaked [2, 0, -1] memory blocks, sum=1 Command line was: ['./python', '-m', 'test.regrtest', '-uall', '-R', '3:3:/home/psf-users/antoine/refleaks/refloglDecLB', '--timeout', '7200'] From webhook-mailer at python.org Thu Mar 22 07:26:09 2018 From: webhook-mailer at python.org (Ivan Levkivskyi) Date: Thu, 22 Mar 2018 11:26:09 -0000 Subject: [Python-checkins] bpo-33018: Improve issubclass() error checking and message. (GH-5944) Message-ID: <mailman.132.1521717973.1871.python-checkins@python.org> https://github.com/python/cpython/commit/40472dd42de4f7265d456458cd13ad6894d736db commit: 40472dd42de4f7265d456458cd13ad6894d736db branch: master author: jab <jab at users.noreply.github.com> committer: Ivan Levkivskyi <levkivskyi at gmail.com> date: 2018-03-22T11:26:06Z summary: bpo-33018: Improve issubclass() error checking and message. (GH-5944) This improves error message for situations when a non-class is checked w.r.t. an abstract base class. files: A Misc/NEWS.d/next/Core and Builtins/2018-03-22-23-09-06.bpo-33018.0ncEJV.rst M Lib/_py_abc.py M Misc/ACKS M Modules/_abc.c diff --git a/Lib/_py_abc.py b/Lib/_py_abc.py index 6f42ef32fa69..3c3aa8e3d61b 100644 --- a/Lib/_py_abc.py +++ b/Lib/_py_abc.py @@ -107,6 +107,8 @@ def __instancecheck__(cls, instance): def __subclasscheck__(cls, subclass): """Override for issubclass(subclass, cls).""" + if not isinstance(subclass, type): + raise TypeError('issubclass() arg 1 must be a class') # Check cache if subclass in cls._abc_cache: return True diff --git a/Misc/ACKS b/Misc/ACKS index d752d8a35434..05932a8d8991 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -202,6 +202,7 @@ Dillon Brock Richard Brodie Michael Broghton Ammar Brohi +Josh Bronson Daniel Brotsky Jean Brouwers Gary S. Brown diff --git a/Misc/NEWS.d/next/Core and Builtins/2018-03-22-23-09-06.bpo-33018.0ncEJV.rst b/Misc/NEWS.d/next/Core and Builtins/2018-03-22-23-09-06.bpo-33018.0ncEJV.rst new file mode 100644 index 000000000000..e799e9834aa1 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2018-03-22-23-09-06.bpo-33018.0ncEJV.rst @@ -0,0 +1,3 @@ +Improve consistency of errors raised by ``issubclass()`` when called with a +non-class and an abstract base class as the first and second arguments, +respectively. Patch by Josh Bronson. diff --git a/Modules/_abc.c b/Modules/_abc.c index 862883987fb7..7daa18e37e40 100644 --- a/Modules/_abc.c +++ b/Modules/_abc.c @@ -569,6 +569,11 @@ _abc__abc_subclasscheck_impl(PyObject *module, PyObject *self, PyObject *subclass) /*[clinic end generated code: output=b56c9e4a530e3894 input=1d947243409d10b8]*/ { + if (!PyType_Check(subclass)) { + PyErr_SetString(PyExc_TypeError, "issubclass() arg 1 must be a class"); + return NULL; + } + PyObject *ok, *mro = NULL, *subclasses = NULL, *result = NULL; Py_ssize_t pos; int incache; From webhook-mailer at python.org Thu Mar 22 07:49:29 2018 From: webhook-mailer at python.org (Miss Islington (bot)) Date: Thu, 22 Mar 2018 11:49:29 -0000 Subject: [Python-checkins] bpo-33018: Improve issubclass() error checking and message. (GH-5944) Message-ID: <mailman.133.1521719372.1871.python-checkins@python.org> https://github.com/python/cpython/commit/346964ba0586e402610ea886e70bee1294874781 commit: 346964ba0586e402610ea886e70bee1294874781 branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: GitHub <noreply at github.com> date: 2018-03-22T04:49:26-07:00 summary: bpo-33018: Improve issubclass() error checking and message. (GH-5944) This improves error message for situations when a non-class is checked w.r.t. an abstract base class. (cherry picked from commit 40472dd42de4f7265d456458cd13ad6894d736db) Co-authored-by: jab <jab at users.noreply.github.com> files: A Misc/NEWS.d/next/Core and Builtins/2018-03-22-23-09-06.bpo-33018.0ncEJV.rst M Lib/_py_abc.py M Misc/ACKS M Modules/_abc.c diff --git a/Lib/_py_abc.py b/Lib/_py_abc.py index 6f42ef32fa69..3c3aa8e3d61b 100644 --- a/Lib/_py_abc.py +++ b/Lib/_py_abc.py @@ -107,6 +107,8 @@ def __instancecheck__(cls, instance): def __subclasscheck__(cls, subclass): """Override for issubclass(subclass, cls).""" + if not isinstance(subclass, type): + raise TypeError('issubclass() arg 1 must be a class') # Check cache if subclass in cls._abc_cache: return True diff --git a/Misc/ACKS b/Misc/ACKS index 9c7cb6f0057d..729e88e56582 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -202,6 +202,7 @@ Dillon Brock Richard Brodie Michael Broghton Ammar Brohi +Josh Bronson Daniel Brotsky Jean Brouwers Gary S. Brown diff --git a/Misc/NEWS.d/next/Core and Builtins/2018-03-22-23-09-06.bpo-33018.0ncEJV.rst b/Misc/NEWS.d/next/Core and Builtins/2018-03-22-23-09-06.bpo-33018.0ncEJV.rst new file mode 100644 index 000000000000..e799e9834aa1 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2018-03-22-23-09-06.bpo-33018.0ncEJV.rst @@ -0,0 +1,3 @@ +Improve consistency of errors raised by ``issubclass()`` when called with a +non-class and an abstract base class as the first and second arguments, +respectively. Patch by Josh Bronson. diff --git a/Modules/_abc.c b/Modules/_abc.c index 862883987fb7..7daa18e37e40 100644 --- a/Modules/_abc.c +++ b/Modules/_abc.c @@ -569,6 +569,11 @@ _abc__abc_subclasscheck_impl(PyObject *module, PyObject *self, PyObject *subclass) /*[clinic end generated code: output=b56c9e4a530e3894 input=1d947243409d10b8]*/ { + if (!PyType_Check(subclass)) { + PyErr_SetString(PyExc_TypeError, "issubclass() arg 1 must be a class"); + return NULL; + } + PyObject *ok, *mro = NULL, *subclasses = NULL, *result = NULL; Py_ssize_t pos; int incache; From webhook-mailer at python.org Thu Mar 22 08:52:46 2018 From: webhook-mailer at python.org (INADA Naoki) Date: Thu, 22 Mar 2018 12:52:46 -0000 Subject: [Python-checkins] bpo-32999: Revert GH-6002 (fc7df0e6) (GH-6189) Message-ID: <mailman.134.1521723171.1871.python-checkins@python.org> https://github.com/python/cpython/commit/f757b72b2524ce3451d2269f0b8a9f0593a7b27f commit: f757b72b2524ce3451d2269f0b8a9f0593a7b27f branch: master author: INADA Naoki <methane at users.noreply.github.com> committer: GitHub <noreply at github.com> date: 2018-03-22T21:52:42+09:00 summary: bpo-32999: Revert GH-6002 (fc7df0e6) (GH-6189) bpo-33018 (GH-5944) fixed bpo-32999 too. So fc7df0e6 is not required anymore. Revert it except test case. files: M Modules/_abc.c diff --git a/Modules/_abc.c b/Modules/_abc.c index 7daa18e37e40..aa091c717dec 100644 --- a/Modules/_abc.c +++ b/Modules/_abc.c @@ -16,7 +16,6 @@ _Py_IDENTIFIER(__abstractmethods__); _Py_IDENTIFIER(__class__); _Py_IDENTIFIER(__dict__); _Py_IDENTIFIER(__bases__); -_Py_IDENTIFIER(__mro__); _Py_IDENTIFIER(_abc_impl); _Py_IDENTIFIER(__subclasscheck__); _Py_IDENTIFIER(__subclasshook__); @@ -574,7 +573,7 @@ _abc__abc_subclasscheck_impl(PyObject *module, PyObject *self, return NULL; } - PyObject *ok, *mro = NULL, *subclasses = NULL, *result = NULL; + PyObject *ok, *subclasses = NULL, *result = NULL; Py_ssize_t pos; int incache; _abc_data *impl = _get_impl(self); @@ -643,31 +642,20 @@ _abc__abc_subclasscheck_impl(PyObject *module, PyObject *self, } Py_DECREF(ok); - /* 4. Check if it's a direct subclass. - * - * if cls in getattr(subclass, '__mro__', ()): - * cls._abc_cache.add(subclass) - * return True - */ - if (_PyObject_LookupAttrId(subclass, &PyId___mro__, &mro) < 0) { - goto end; - } - if (mro != NULL) { - if (!PyTuple_Check(mro)) { - // Python version supports non-tuple iterable. Keep it as - // implementation detail. - PyErr_SetString(PyExc_TypeError, "__mro__ is not a tuple"); + /* 4. Check if it's a direct subclass. */ + PyObject *mro = ((PyTypeObject *)subclass)->tp_mro; + assert(PyTuple_Check(mro)); + for (pos = 0; pos < PyTuple_GET_SIZE(mro); pos++) { + PyObject *mro_item = PyTuple_GET_ITEM(mro, pos); + if (mro_item == NULL) { goto end; } - for (pos = 0; pos < PyTuple_GET_SIZE(mro); pos++) { - PyObject *mro_item = PyTuple_GET_ITEM(mro, pos); - if ((PyObject *)self == mro_item) { - if (_add_to_weak_set(&impl->_abc_cache, subclass) < 0) { - goto end; - } - result = Py_True; + if ((PyObject *)self == mro_item) { + if (_add_to_weak_set(&impl->_abc_cache, subclass) < 0) { goto end; } + result = Py_True; + goto end; } } @@ -708,7 +696,6 @@ _abc__abc_subclasscheck_impl(PyObject *module, PyObject *self, end: Py_DECREF(impl); - Py_XDECREF(mro); Py_XDECREF(subclasses); Py_XINCREF(result); return result; From webhook-mailer at python.org Thu Mar 22 10:00:15 2018 From: webhook-mailer at python.org (Ivan Levkivskyi) Date: Thu, 22 Mar 2018 14:00:15 -0000 Subject: [Python-checkins] bpo-32999: Revert GH-6002 (fc7df0e6) (GH-6189) (GH-6190) Message-ID: <mailman.135.1521727219.1871.python-checkins@python.org> https://github.com/python/cpython/commit/5d8bb5d07be2a9205e7059090f0ac5360d36b217 commit: 5d8bb5d07be2a9205e7059090f0ac5360d36b217 branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Ivan Levkivskyi <levkivskyi at gmail.com> date: 2018-03-22T14:00:11Z summary: bpo-32999: Revert GH-6002 (fc7df0e6) (GH-6189) (GH-6190) bpo-33018 (GH-5944) fixed bpo-32999 too. So fc7df0e6 is not required anymore. Revert it except test case. (cherry picked from commit f757b72b2524ce3451d2269f0b8a9f0593a7b27f) Co-authored-by: INADA Naoki <methane at users.noreply.github.com> files: M Modules/_abc.c diff --git a/Modules/_abc.c b/Modules/_abc.c index 7daa18e37e40..aa091c717dec 100644 --- a/Modules/_abc.c +++ b/Modules/_abc.c @@ -16,7 +16,6 @@ _Py_IDENTIFIER(__abstractmethods__); _Py_IDENTIFIER(__class__); _Py_IDENTIFIER(__dict__); _Py_IDENTIFIER(__bases__); -_Py_IDENTIFIER(__mro__); _Py_IDENTIFIER(_abc_impl); _Py_IDENTIFIER(__subclasscheck__); _Py_IDENTIFIER(__subclasshook__); @@ -574,7 +573,7 @@ _abc__abc_subclasscheck_impl(PyObject *module, PyObject *self, return NULL; } - PyObject *ok, *mro = NULL, *subclasses = NULL, *result = NULL; + PyObject *ok, *subclasses = NULL, *result = NULL; Py_ssize_t pos; int incache; _abc_data *impl = _get_impl(self); @@ -643,31 +642,20 @@ _abc__abc_subclasscheck_impl(PyObject *module, PyObject *self, } Py_DECREF(ok); - /* 4. Check if it's a direct subclass. - * - * if cls in getattr(subclass, '__mro__', ()): - * cls._abc_cache.add(subclass) - * return True - */ - if (_PyObject_LookupAttrId(subclass, &PyId___mro__, &mro) < 0) { - goto end; - } - if (mro != NULL) { - if (!PyTuple_Check(mro)) { - // Python version supports non-tuple iterable. Keep it as - // implementation detail. - PyErr_SetString(PyExc_TypeError, "__mro__ is not a tuple"); + /* 4. Check if it's a direct subclass. */ + PyObject *mro = ((PyTypeObject *)subclass)->tp_mro; + assert(PyTuple_Check(mro)); + for (pos = 0; pos < PyTuple_GET_SIZE(mro); pos++) { + PyObject *mro_item = PyTuple_GET_ITEM(mro, pos); + if (mro_item == NULL) { goto end; } - for (pos = 0; pos < PyTuple_GET_SIZE(mro); pos++) { - PyObject *mro_item = PyTuple_GET_ITEM(mro, pos); - if ((PyObject *)self == mro_item) { - if (_add_to_weak_set(&impl->_abc_cache, subclass) < 0) { - goto end; - } - result = Py_True; + if ((PyObject *)self == mro_item) { + if (_add_to_weak_set(&impl->_abc_cache, subclass) < 0) { goto end; } + result = Py_True; + goto end; } } @@ -708,7 +696,6 @@ _abc__abc_subclasscheck_impl(PyObject *module, PyObject *self, end: Py_DECREF(impl); - Py_XDECREF(mro); Py_XDECREF(subclasses); Py_XINCREF(result); return result; From webhook-mailer at python.org Thu Mar 22 16:28:52 2018 From: webhook-mailer at python.org (Eric V. Smith) Date: Thu, 22 Mar 2018 20:28:52 -0000 Subject: [Python-checkins] bpo-32505: dataclasses: raise TypeError if a member variable is of type Field, but doesn't have a type annotation. (GH-6192) Message-ID: <mailman.136.1521750533.1871.python-checkins@python.org> https://github.com/python/cpython/commit/56970b8ce9d23269d20a76f13c80e670c856ba7f commit: 56970b8ce9d23269d20a76f13c80e670c856ba7f branch: master author: Eric V. Smith <ericvsmith at users.noreply.github.com> committer: GitHub <noreply at github.com> date: 2018-03-22T16:28:48-04:00 summary: bpo-32505: dataclasses: raise TypeError if a member variable is of type Field, but doesn't have a type annotation. (GH-6192) If a dataclass has a member variable that's of type Field, but it doesn't have a type annotation, raise TypeError. files: A Misc/NEWS.d/next/Library/2018-03-22-16-05-56.bpo-32505.YK1N8v.rst M Lib/dataclasses.py M Lib/test/test_dataclasses.py diff --git a/Lib/dataclasses.py b/Lib/dataclasses.py index 41b5b5da325c..5d4d4a6100ca 100644 --- a/Lib/dataclasses.py +++ b/Lib/dataclasses.py @@ -573,22 +573,6 @@ def _get_field(cls, a_name, a_type): return f -def _find_fields(cls): - # Return a list of Field objects, in order, for this class (and no - # base classes). Fields are found from the class dict's - # __annotations__ (which is guaranteed to be ordered). Default - # values are from class attributes, if a field has a default. If - # the default value is a Field(), then it contains additional - # info beyond (and possibly including) the actual default value. - # Pseudo-fields ClassVars and InitVars are included, despite the - # fact that they're not real fields. That's dealt with later. - - # If __annotations__ isn't present, then this class adds no new - # annotations. - annotations = cls.__dict__.get('__annotations__', {}) - return [_get_field(cls, name, type) for name, type in annotations.items()] - - def _set_new_attribute(cls, name, value): # Never overwrites an existing attribute. Returns True if the # attribute already exists. @@ -663,10 +647,25 @@ def _process_class(cls, init, repr, eq, order, unsafe_hash, frozen): if getattr(b, _PARAMS).frozen: any_frozen_base = True + # Annotations that are defined in this class (not in base + # classes). If __annotations__ isn't present, then this class + # adds no new annotations. We use this to compute fields that + # are added by this class. + # Fields are found from cls_annotations, which is guaranteed to be + # ordered. Default values are from class attributes, if a field + # has a default. If the default value is a Field(), then it + # contains additional info beyond (and possibly including) the + # actual default value. Pseudo-fields ClassVars and InitVars are + # included, despite the fact that they're not real fields. + # That's dealt with later. + cls_annotations = cls.__dict__.get('__annotations__', {}) + # Now find fields in our class. While doing so, validate some # things, and set the default values (as class attributes) # where we can. - for f in _find_fields(cls): + cls_fields = [_get_field(cls, name, type) + for name, type in cls_annotations.items()] + for f in cls_fields: fields[f.name] = f # If the class attribute (which is the default value for @@ -685,6 +684,11 @@ def _process_class(cls, init, repr, eq, order, unsafe_hash, frozen): else: setattr(cls, f.name, f.default) + # Do we have any Field members that don't also have annotations? + for name, value in cls.__dict__.items(): + if isinstance(value, Field) and not name in cls_annotations: + raise TypeError(f'{name!r} is a field but has no type annotation') + # Check rules that apply if we are derived from any dataclasses. if has_dataclass_bases: # Raise an exception if any of our bases are frozen, but we're not. diff --git a/Lib/test/test_dataclasses.py b/Lib/test/test_dataclasses.py index 69ace36c2c58..8aff8ae140a5 100755 --- a/Lib/test/test_dataclasses.py +++ b/Lib/test/test_dataclasses.py @@ -24,6 +24,14 @@ class C: o = C() self.assertEqual(len(fields(C)), 0) + def test_no_fields_but_member_variable(self): + @dataclass + class C: + i = 0 + + o = C() + self.assertEqual(len(fields(C)), 0) + def test_one_field_no_default(self): @dataclass class C: @@ -1906,6 +1914,41 @@ def test_helper_make_dataclass_no_types(self): 'z': 'typing.Any'}) +class TestFieldNoAnnotation(unittest.TestCase): + def test_field_without_annotation(self): + with self.assertRaisesRegex(TypeError, + "'f' is a field but has no type annotation"): + @dataclass + class C: + f = field() + + def test_field_without_annotation_but_annotation_in_base(self): + @dataclass + class B: + f: int + + with self.assertRaisesRegex(TypeError, + "'f' is a field but has no type annotation"): + # This is still an error: make sure we don't pick up the + # type annotation in the base class. + @dataclass + class C(B): + f = field() + + def test_field_without_annotation_but_annotation_in_base_not_dataclass(self): + # Same test, but with the base class not a dataclass. + class B: + f: int + + with self.assertRaisesRegex(TypeError, + "'f' is a field but has no type annotation"): + # This is still an error: make sure we don't pick up the + # type annotation in the base class. + @dataclass + class C(B): + f = field() + + class TestDocString(unittest.TestCase): def assertDocStrEqual(self, a, b): # Because 3.6 and 3.7 differ in how inspect.signature work diff --git a/Misc/NEWS.d/next/Library/2018-03-22-16-05-56.bpo-32505.YK1N8v.rst b/Misc/NEWS.d/next/Library/2018-03-22-16-05-56.bpo-32505.YK1N8v.rst new file mode 100644 index 000000000000..91e97bf53f65 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-03-22-16-05-56.bpo-32505.YK1N8v.rst @@ -0,0 +1,2 @@ +Raise TypeError if a member variable of a dataclass is of type Field, but +doesn't have a type annotation. From webhook-mailer at python.org Thu Mar 22 16:59:09 2018 From: webhook-mailer at python.org (Miss Islington (bot)) Date: Thu, 22 Mar 2018 20:59:09 -0000 Subject: [Python-checkins] bpo-32505: dataclasses: raise TypeError if a member variable is of type Field, but doesn't have a type annotation. (GH-6192) Message-ID: <mailman.137.1521752351.1871.python-checkins@python.org> https://github.com/python/cpython/commit/3b4c6b16c597aa2356f5658dd67da7dcd4434038 commit: 3b4c6b16c597aa2356f5658dd67da7dcd4434038 branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: GitHub <noreply at github.com> date: 2018-03-22T13:58:59-07:00 summary: bpo-32505: dataclasses: raise TypeError if a member variable is of type Field, but doesn't have a type annotation. (GH-6192) If a dataclass has a member variable that's of type Field, but it doesn't have a type annotation, raise TypeError. (cherry picked from commit 56970b8ce9d23269d20a76f13c80e670c856ba7f) Co-authored-by: Eric V. Smith <ericvsmith at users.noreply.github.com> files: A Misc/NEWS.d/next/Library/2018-03-22-16-05-56.bpo-32505.YK1N8v.rst M Lib/dataclasses.py M Lib/test/test_dataclasses.py diff --git a/Lib/dataclasses.py b/Lib/dataclasses.py index 41b5b5da325c..5d4d4a6100ca 100644 --- a/Lib/dataclasses.py +++ b/Lib/dataclasses.py @@ -573,22 +573,6 @@ def _get_field(cls, a_name, a_type): return f -def _find_fields(cls): - # Return a list of Field objects, in order, for this class (and no - # base classes). Fields are found from the class dict's - # __annotations__ (which is guaranteed to be ordered). Default - # values are from class attributes, if a field has a default. If - # the default value is a Field(), then it contains additional - # info beyond (and possibly including) the actual default value. - # Pseudo-fields ClassVars and InitVars are included, despite the - # fact that they're not real fields. That's dealt with later. - - # If __annotations__ isn't present, then this class adds no new - # annotations. - annotations = cls.__dict__.get('__annotations__', {}) - return [_get_field(cls, name, type) for name, type in annotations.items()] - - def _set_new_attribute(cls, name, value): # Never overwrites an existing attribute. Returns True if the # attribute already exists. @@ -663,10 +647,25 @@ def _process_class(cls, init, repr, eq, order, unsafe_hash, frozen): if getattr(b, _PARAMS).frozen: any_frozen_base = True + # Annotations that are defined in this class (not in base + # classes). If __annotations__ isn't present, then this class + # adds no new annotations. We use this to compute fields that + # are added by this class. + # Fields are found from cls_annotations, which is guaranteed to be + # ordered. Default values are from class attributes, if a field + # has a default. If the default value is a Field(), then it + # contains additional info beyond (and possibly including) the + # actual default value. Pseudo-fields ClassVars and InitVars are + # included, despite the fact that they're not real fields. + # That's dealt with later. + cls_annotations = cls.__dict__.get('__annotations__', {}) + # Now find fields in our class. While doing so, validate some # things, and set the default values (as class attributes) # where we can. - for f in _find_fields(cls): + cls_fields = [_get_field(cls, name, type) + for name, type in cls_annotations.items()] + for f in cls_fields: fields[f.name] = f # If the class attribute (which is the default value for @@ -685,6 +684,11 @@ def _process_class(cls, init, repr, eq, order, unsafe_hash, frozen): else: setattr(cls, f.name, f.default) + # Do we have any Field members that don't also have annotations? + for name, value in cls.__dict__.items(): + if isinstance(value, Field) and not name in cls_annotations: + raise TypeError(f'{name!r} is a field but has no type annotation') + # Check rules that apply if we are derived from any dataclasses. if has_dataclass_bases: # Raise an exception if any of our bases are frozen, but we're not. diff --git a/Lib/test/test_dataclasses.py b/Lib/test/test_dataclasses.py index 69ace36c2c58..8aff8ae140a5 100755 --- a/Lib/test/test_dataclasses.py +++ b/Lib/test/test_dataclasses.py @@ -24,6 +24,14 @@ class C: o = C() self.assertEqual(len(fields(C)), 0) + def test_no_fields_but_member_variable(self): + @dataclass + class C: + i = 0 + + o = C() + self.assertEqual(len(fields(C)), 0) + def test_one_field_no_default(self): @dataclass class C: @@ -1906,6 +1914,41 @@ def test_helper_make_dataclass_no_types(self): 'z': 'typing.Any'}) +class TestFieldNoAnnotation(unittest.TestCase): + def test_field_without_annotation(self): + with self.assertRaisesRegex(TypeError, + "'f' is a field but has no type annotation"): + @dataclass + class C: + f = field() + + def test_field_without_annotation_but_annotation_in_base(self): + @dataclass + class B: + f: int + + with self.assertRaisesRegex(TypeError, + "'f' is a field but has no type annotation"): + # This is still an error: make sure we don't pick up the + # type annotation in the base class. + @dataclass + class C(B): + f = field() + + def test_field_without_annotation_but_annotation_in_base_not_dataclass(self): + # Same test, but with the base class not a dataclass. + class B: + f: int + + with self.assertRaisesRegex(TypeError, + "'f' is a field but has no type annotation"): + # This is still an error: make sure we don't pick up the + # type annotation in the base class. + @dataclass + class C(B): + f = field() + + class TestDocString(unittest.TestCase): def assertDocStrEqual(self, a, b): # Because 3.6 and 3.7 differ in how inspect.signature work diff --git a/Misc/NEWS.d/next/Library/2018-03-22-16-05-56.bpo-32505.YK1N8v.rst b/Misc/NEWS.d/next/Library/2018-03-22-16-05-56.bpo-32505.YK1N8v.rst new file mode 100644 index 000000000000..91e97bf53f65 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-03-22-16-05-56.bpo-32505.YK1N8v.rst @@ -0,0 +1,2 @@ +Raise TypeError if a member variable of a dataclass is of type Field, but +doesn't have a type annotation. From solipsis at pitrou.net Fri Mar 23 05:09:21 2018 From: solipsis at pitrou.net (solipsis at pitrou.net) Date: Fri, 23 Mar 2018 09:09:21 +0000 Subject: [Python-checkins] Daily reference leaks (4243df51fe43): sum=1 Message-ID: <20180323090921.1.15BD5FEB8AD6650E@psf.io> results for 4243df51fe43 on branch "default" -------------------------------------------- test_asyncio leaked [0, 3, 0] memory blocks, sum=3 test_collections leaked [-7, 1, 0] memory blocks, sum=-6 test_functools leaked [0, 3, 1] memory blocks, sum=4 Command line was: ['./python', '-m', 'test.regrtest', '-uall', '-R', '3:3:/home/psf-users/antoine/refleaks/reflog597LPB', '--timeout', '7200'] From webhook-mailer at python.org Fri Mar 23 05:19:39 2018 From: webhook-mailer at python.org (INADA Naoki) Date: Fri, 23 Mar 2018 09:19:39 -0000 Subject: [Python-checkins] bpo-32999: ast: Convert useless check to assert (GH-6197) Message-ID: <mailman.138.1521796781.1871.python-checkins@python.org> https://github.com/python/cpython/commit/c65bf3fe4a2bde424b79e350f36b7aaa3f6476f6 commit: c65bf3fe4a2bde424b79e350f36b7aaa3f6476f6 branch: master author: INADA Naoki <methane at users.noreply.github.com> committer: GitHub <noreply at github.com> date: 2018-03-23T18:19:34+09:00 summary: bpo-32999: ast: Convert useless check to assert (GH-6197) files: M Modules/_abc.c diff --git a/Modules/_abc.c b/Modules/_abc.c index aa091c717dec..562a2e6d730d 100644 --- a/Modules/_abc.c +++ b/Modules/_abc.c @@ -647,9 +647,7 @@ _abc__abc_subclasscheck_impl(PyObject *module, PyObject *self, assert(PyTuple_Check(mro)); for (pos = 0; pos < PyTuple_GET_SIZE(mro); pos++) { PyObject *mro_item = PyTuple_GET_ITEM(mro, pos); - if (mro_item == NULL) { - goto end; - } + assert(mro_item != NULL); if ((PyObject *)self == mro_item) { if (_add_to_weak_set(&impl->_abc_cache, subclass) < 0) { goto end; From webhook-mailer at python.org Fri Mar 23 05:43:17 2018 From: webhook-mailer at python.org (Miss Islington (bot)) Date: Fri, 23 Mar 2018 09:43:17 -0000 Subject: [Python-checkins] bpo-32999: ast: Convert useless check to assert (GH-6197) Message-ID: <mailman.139.1521798201.1871.python-checkins@python.org> https://github.com/python/cpython/commit/c71edab15d023360388da8360700d419b5f48c81 commit: c71edab15d023360388da8360700d419b5f48c81 branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: GitHub <noreply at github.com> date: 2018-03-23T02:43:11-07:00 summary: bpo-32999: ast: Convert useless check to assert (GH-6197) (cherry picked from commit c65bf3fe4a2bde424b79e350f36b7aaa3f6476f6) Co-authored-by: INADA Naoki <methane at users.noreply.github.com> files: M Modules/_abc.c diff --git a/Modules/_abc.c b/Modules/_abc.c index aa091c717dec..562a2e6d730d 100644 --- a/Modules/_abc.c +++ b/Modules/_abc.c @@ -647,9 +647,7 @@ _abc__abc_subclasscheck_impl(PyObject *module, PyObject *self, assert(PyTuple_Check(mro)); for (pos = 0; pos < PyTuple_GET_SIZE(mro); pos++) { PyObject *mro_item = PyTuple_GET_ITEM(mro, pos); - if (mro_item == NULL) { - goto end; - } + assert(mro_item != NULL); if ((PyObject *)self == mro_item) { if (_add_to_weak_set(&impl->_abc_cache, subclass) < 0) { goto end; From webhook-mailer at python.org Fri Mar 23 08:34:44 2018 From: webhook-mailer at python.org (Serhiy Storchaka) Date: Fri, 23 Mar 2018 12:34:44 -0000 Subject: [Python-checkins] bpo-33041: Rework compiling an "async for" loop. (#6142) Message-ID: <mailman.140.1521808488.1871.python-checkins@python.org> https://github.com/python/cpython/commit/702f8f3611bc49b73772cce2b9b041bd11ff9b35 commit: 702f8f3611bc49b73772cce2b9b041bd11ff9b35 branch: master author: Serhiy Storchaka <storchaka at gmail.com> committer: GitHub <noreply at github.com> date: 2018-03-23T14:34:35+02:00 summary: bpo-33041: Rework compiling an "async for" loop. (#6142) * Added new opcode END_ASYNC_FOR. * Setting global StopAsyncIteration no longer breaks "async for" loops. * Jumping into an "async for" loop is now disabled. * Jumping out of an "async for" loop no longer corrupts the stack. * Simplify the compiler. files: A Misc/NEWS.d/next/Core and Builtins/2018-03-18-13-56-14.bpo-33041.XwPhI2.rst M Doc/library/dis.rst M Doc/whatsnew/3.8.rst M Include/opcode.h M Lib/importlib/_bootstrap_external.py M Lib/opcode.py M Lib/test/test_coroutines.py M Lib/test/test_dis.py M Lib/test/test_sys_settrace.py M Objects/frameobject.c M Python/ceval.c M Python/compile.c M Python/importlib_external.h M Python/opcode_targets.h diff --git a/Doc/library/dis.rst b/Doc/library/dis.rst index 47f226bd1625..8f505e65bd51 100644 --- a/Doc/library/dis.rst +++ b/Doc/library/dis.rst @@ -588,6 +588,17 @@ the original TOS1. .. versionadded:: 3.5 +.. opcode:: END_ASYNC_FOR + + Terminates an :keyword:`async for` loop. Handles an exception raised + when awaiting a next item. If TOS is :exc:`StopAsyncIteration` pop 7 + values from the stack and restore the exception state using the second + three of them. Otherwise re-raise the exception using the three values + from the stack. An exception handler block is removed from the block stack. + + .. versionadded:: 3.8 + + .. opcode:: BEFORE_ASYNC_WITH Resolves ``__aenter__`` and ``__aexit__`` from the object on top of the diff --git a/Doc/whatsnew/3.8.rst b/Doc/whatsnew/3.8.rst index fcc868b041ad..4e6c85173751 100644 --- a/Doc/whatsnew/3.8.rst +++ b/Doc/whatsnew/3.8.rst @@ -157,3 +157,7 @@ CPython bytecode changes (Contributed by Mark Shannon, Antoine Pitrou and Serhiy Storchaka in :issue:`17611`.) + +* Added new opcode :opcode:`END_ASYNC_FOR` for handling exceptions raised + when awaiting a next item in an :keyword:`async for` loop. + (Contributed by Serhiy Storchaka in :issue:`33041`.) diff --git a/Include/opcode.h b/Include/opcode.h index fba74af44578..e564bb9d598d 100644 --- a/Include/opcode.h +++ b/Include/opcode.h @@ -34,6 +34,7 @@ extern "C" { #define GET_ANEXT 51 #define BEFORE_ASYNC_WITH 52 #define BEGIN_FINALLY 53 +#define END_ASYNC_FOR 54 #define INPLACE_ADD 55 #define INPLACE_SUBTRACT 56 #define INPLACE_MULTIPLY 57 diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py index 27aaf555390f..420ecc83439a 100644 --- a/Lib/importlib/_bootstrap_external.py +++ b/Lib/importlib/_bootstrap_external.py @@ -247,6 +247,7 @@ def _write_atomic(path, data, mode=0o666): # Python 3.7a4 3392 (PEP 552: Deterministic pycs #31650) # Python 3.7b1 3393 (remove STORE_ANNOTATION opcode #32550) # Python 3.8a1 3400 (move frame block handling to compiler #17611) +# Python 3.8a1 3401 (add END_ASYNC_FOR #33041) # # MAGIC must change whenever the bytecode emitted by the compiler may no # longer be understood by older implementations of the eval loop (usually @@ -255,7 +256,7 @@ def _write_atomic(path, data, mode=0o666): # Whenever MAGIC_NUMBER is changed, the ranges in the magic_values array # in PC/launcher.c must also be updated. -MAGIC_NUMBER = (3400).to_bytes(2, 'little') + b'\r\n' +MAGIC_NUMBER = (3401).to_bytes(2, 'little') + b'\r\n' _RAW_MAGIC_NUMBER = int.from_bytes(MAGIC_NUMBER, 'little') # For import.c _PYCACHE = '__pycache__' diff --git a/Lib/opcode.py b/Lib/opcode.py index 6de24c34affe..3fb716b5d96a 100644 --- a/Lib/opcode.py +++ b/Lib/opcode.py @@ -88,7 +88,7 @@ def jabs_op(name, op): def_op('GET_ANEXT', 51) def_op('BEFORE_ASYNC_WITH', 52) def_op('BEGIN_FINALLY', 53) - +def_op('END_ASYNC_FOR', 54) def_op('INPLACE_ADD', 55) def_op('INPLACE_SUBTRACT', 56) def_op('INPLACE_MULTIPLY', 57) diff --git a/Lib/test/test_coroutines.py b/Lib/test/test_coroutines.py index f4a9d2aeb874..fe26199f95af 100644 --- a/Lib/test/test_coroutines.py +++ b/Lib/test/test_coroutines.py @@ -1846,6 +1846,36 @@ def test_comp_4(self): run_async(run_gen()), ([], [121])) + def test_comp_4_2(self): + async def f(it): + for i in it: + yield i + + async def run_list(): + return [i + 10 async for i in f(range(5)) if 0 < i < 4] + self.assertEqual( + run_async(run_list()), + ([], [11, 12, 13])) + + async def run_set(): + return {i + 10 async for i in f(range(5)) if 0 < i < 4} + self.assertEqual( + run_async(run_set()), + ([], {11, 12, 13})) + + async def run_dict(): + return {i + 10: i + 100 async for i in f(range(5)) if 0 < i < 4} + self.assertEqual( + run_async(run_dict()), + ([], {11: 101, 12: 102, 13: 103})) + + async def run_gen(): + gen = (i + 10 async for i in f(range(5)) if 0 < i < 4) + return [g + 100 async for g in gen] + self.assertEqual( + run_async(run_gen()), + ([], [111, 112, 113])) + def test_comp_5(self): async def f(it): for i in it: diff --git a/Lib/test/test_dis.py b/Lib/test/test_dis.py index 098367ca55d1..c86f61f236bd 100644 --- a/Lib/test/test_dis.py +++ b/Lib/test/test_dis.py @@ -747,8 +747,7 @@ def f(c=c): 1: 1 Names: 0: b - 1: StopAsyncIteration - 2: c + 1: c Variable names: 0: a 1: d""" diff --git a/Lib/test/test_sys_settrace.py b/Lib/test/test_sys_settrace.py index 2587794c69a3..1fa43b29ec90 100644 --- a/Lib/test/test_sys_settrace.py +++ b/Lib/test/test_sys_settrace.py @@ -33,6 +33,10 @@ def __init__(self, output, value): async def __aexit__(self, *exc_info): self.output.append(-self.value) +async def asynciter(iterable): + """Convert an iterable to an asynchronous iterator.""" + for x in iterable: + yield x # A very basic example. If this fails, we're in deep trouble. @@ -720,6 +724,23 @@ def test_jump_out_of_block_backwards(output): output.append(6) output.append(7) + @async_jump_test(4, 5, [3, 5]) + async def test_jump_out_of_async_for_block_forwards(output): + for i in [1]: + async for i in asynciter([1, 2]): + output.append(3) + output.append(4) + output.append(5) + + @async_jump_test(5, 2, [2, 4, 2, 4, 5, 6]) + async def test_jump_out_of_async_for_block_backwards(output): + for i in [1]: + output.append(2) + async for i in asynciter([1]): + output.append(4) + output.append(5) + output.append(6) + @jump_test(1, 2, [3]) def test_jump_to_codeless_line(output): output.append(1) @@ -1030,6 +1051,17 @@ def test_jump_over_for_block_before_else(output): output.append(7) output.append(8) + @async_jump_test(1, 7, [7, 8]) + async def test_jump_over_async_for_block_before_else(output): + output.append(1) + if not output: # always false + async for i in asynciter([3]): + output.append(4) + else: + output.append(6) + output.append(7) + output.append(8) + # The second set of 'jump' tests are for things that are not allowed: @jump_test(2, 3, [1], (ValueError, 'after')) @@ -1081,12 +1113,24 @@ def test_no_jump_forwards_into_for_block(output): for i in 1, 2: output.append(3) + @async_jump_test(1, 3, [], (ValueError, 'into')) + async def test_no_jump_forwards_into_async_for_block(output): + output.append(1) + async for i in asynciter([1, 2]): + output.append(3) + @jump_test(3, 2, [2, 2], (ValueError, 'into')) def test_no_jump_backwards_into_for_block(output): for i in 1, 2: output.append(2) output.append(3) + @async_jump_test(3, 2, [2, 2], (ValueError, 'into')) + async def test_no_jump_backwards_into_async_for_block(output): + async for i in asynciter([1, 2]): + output.append(2) + output.append(3) + @jump_test(1, 3, [], (ValueError, 'into')) def test_no_jump_forwards_into_with_block(output): output.append(1) @@ -1220,6 +1264,17 @@ def test_no_jump_into_for_block_before_else(output): output.append(7) output.append(8) + @async_jump_test(7, 4, [1, 6], (ValueError, 'into')) + async def test_no_jump_into_async_for_block_before_else(output): + output.append(1) + if not output: # always false + async for i in asynciter([3]): + output.append(4) + else: + output.append(6) + output.append(7) + output.append(8) + def test_no_jump_to_non_integers(self): self.run_test(no_jump_to_non_integers, 2, "Spam", [True]) diff --git a/Misc/NEWS.d/next/Core and Builtins/2018-03-18-13-56-14.bpo-33041.XwPhI2.rst b/Misc/NEWS.d/next/Core and Builtins/2018-03-18-13-56-14.bpo-33041.XwPhI2.rst new file mode 100644 index 000000000000..34bf6c9b0f1c --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2018-03-18-13-56-14.bpo-33041.XwPhI2.rst @@ -0,0 +1,6 @@ +Added new opcode :opcode:`END_ASYNC_FOR` and fixes the following issues: + +* Setting global :exc:`StopAsyncIteration` no longer breaks ``async for`` + loops. +* Jumping into an ``async for`` loop is now disabled. +* Jumping out of an ``async for`` loop no longer corrupts the stack. diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 643be08fa182..9d37935c2f79 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -100,8 +100,7 @@ frame_setlineno(PyFrameObject *f, PyObject* p_new_lineno) int line = 0; /* (ditto) */ int addr = 0; /* (ditto) */ int delta_iblock = 0; /* Scanning the SETUPs and POPs */ - int for_loop_delta = 0; /* (ditto) */ - int delta; + int delta = 0; int blockstack[CO_MAXBLOCKS]; /* Walking the 'finally' blocks */ int blockstack_top = 0; /* (ditto) */ @@ -256,14 +255,16 @@ frame_setlineno(PyFrameObject *f, PyObject* p_new_lineno) return -1; } if (first_in && !second_in) { - if (op == FOR_ITER && !delta_iblock) { - for_loop_delta++; - } - if (op != FOR_ITER) { + if (op != FOR_ITER && code[target_addr] != END_ASYNC_FOR) { delta_iblock++; } + else if (!delta_iblock) { + /* Pop the iterators of any 'for' and 'async for' loop + * we're jumping out of. */ + delta++; + } } - if (op != FOR_ITER) { + if (op != FOR_ITER && code[target_addr] != END_ASYNC_FOR) { blockstack[blockstack_top++] = target_addr; } break; @@ -289,11 +290,10 @@ frame_setlineno(PyFrameObject *f, PyObject* p_new_lineno) assert(blockstack_top == 0); /* Pop any blocks that we're jumping out of. */ - delta = 0; if (delta_iblock > 0) { f->f_iblock -= delta_iblock; PyTryBlock *b = &f->f_blockstack[f->f_iblock]; - delta = (f->f_stacktop - f->f_valuestack) - b->b_level; + delta += (f->f_stacktop - f->f_valuestack) - b->b_level; if (b->b_type == SETUP_FINALLY && code[b->b_handler] == WITH_CLEANUP_START) { @@ -301,9 +301,6 @@ frame_setlineno(PyFrameObject *f, PyObject* p_new_lineno) delta++; } } - /* Pop the iterators of any 'for' loop we're jumping out of. */ - delta += for_loop_delta; - while (delta > 0) { PyObject *v = (*--f->f_stacktop); Py_DECREF(v); diff --git a/Python/ceval.c b/Python/ceval.c index 1a72413c9e15..14603d330083 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -1944,6 +1944,26 @@ _PyEval_EvalFrameDefault(PyFrameObject *f, int throwflag) } } + TARGET(END_ASYNC_FOR) { + PyObject *exc = POP(); + assert(PyExceptionClass_Check(exc)); + if (PyErr_GivenExceptionMatches(exc, PyExc_StopAsyncIteration)) { + PyTryBlock *b = PyFrame_BlockPop(f); + assert(b->b_type == EXCEPT_HANDLER); + Py_DECREF(exc); + UNWIND_EXCEPT_HANDLER(b); + Py_DECREF(POP()); + JUMPBY(oparg); + FAST_DISPATCH(); + } + else { + PyObject *val = POP(); + PyObject *tb = POP(); + PyErr_Restore(exc, val, tb); + goto exception_unwind; + } + } + TARGET(LOAD_BUILD_CLASS) { _Py_IDENTIFIER(__build_class__); diff --git a/Python/compile.c b/Python/compile.c index c3ffaae8ce20..725bb9b213a2 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -1111,6 +1111,8 @@ stack_effect(int opcode, int oparg, int jump) return 1; case GET_YIELD_FROM_ITER: return 0; + case END_ASYNC_FOR: + return -7; case FORMAT_VALUE: /* If there's a fmt_spec on the stack, we go from 2->1, else 1->1. */ @@ -2434,78 +2436,40 @@ compiler_for(struct compiler *c, stmt_ty s) static int compiler_async_for(struct compiler *c, stmt_ty s) { - _Py_IDENTIFIER(StopAsyncIteration); - - basicblock *try, *except, *end, *after_try, *try_cleanup, - *after_loop_else; - - PyObject *stop_aiter_error = _PyUnicode_FromId(&PyId_StopAsyncIteration); - if (stop_aiter_error == NULL) { - return 0; - } - - try = compiler_new_block(c); + basicblock *start, *except, *end; + start = compiler_new_block(c); except = compiler_new_block(c); end = compiler_new_block(c); - after_try = compiler_new_block(c); - try_cleanup = compiler_new_block(c); - after_loop_else = compiler_new_block(c); - - if (try == NULL || except == NULL || end == NULL - || after_try == NULL || try_cleanup == NULL - || after_loop_else == NULL) - return 0; - if (!compiler_push_fblock(c, FOR_LOOP, try, end)) + if (start == NULL || except == NULL || end == NULL) return 0; VISIT(c, expr, s->v.AsyncFor.iter); ADDOP(c, GET_AITER); - compiler_use_next_block(c, try); - + compiler_use_next_block(c, start); + if (!compiler_push_fblock(c, FOR_LOOP, start, end)) + return 0; /* SETUP_FINALLY to guard the __anext__ call */ ADDOP_JREL(c, SETUP_FINALLY, except); - if (!compiler_push_fblock(c, EXCEPT, try, NULL)) - return 0; - ADDOP(c, GET_ANEXT); ADDOP_O(c, LOAD_CONST, Py_None, consts); ADDOP(c, YIELD_FROM); ADDOP(c, POP_BLOCK); /* for SETUP_FINALLY */ + + /* Success block for __anext__ */ VISIT(c, expr, s->v.AsyncFor.target); - compiler_pop_fblock(c, EXCEPT, try); - ADDOP_JREL(c, JUMP_FORWARD, after_try); + VISIT_SEQ(c, stmt, s->v.AsyncFor.body); + ADDOP_JABS(c, JUMP_ABSOLUTE, start); + compiler_pop_fblock(c, FOR_LOOP, start); /* Except block for __anext__ */ compiler_use_next_block(c, except); - ADDOP(c, DUP_TOP); - ADDOP_O(c, LOAD_GLOBAL, stop_aiter_error, names); - ADDOP_I(c, COMPARE_OP, PyCmp_EXC_MATCH); - ADDOP_JABS(c, POP_JUMP_IF_FALSE, try_cleanup); - - ADDOP(c, POP_TOP); - ADDOP(c, POP_TOP); - ADDOP(c, POP_TOP); - ADDOP(c, POP_EXCEPT); /* for SETUP_FINALLY */ - ADDOP(c, POP_TOP); /* pop iterator from stack */ - ADDOP_JABS(c, JUMP_ABSOLUTE, after_loop_else); - - - compiler_use_next_block(c, try_cleanup); - ADDOP(c, END_FINALLY); - - /* Success block for __anext__ */ - compiler_use_next_block(c, after_try); - VISIT_SEQ(c, stmt, s->v.AsyncFor.body); - ADDOP_JABS(c, JUMP_ABSOLUTE, try); - - compiler_pop_fblock(c, FOR_LOOP, try); + ADDOP(c, END_ASYNC_FOR); /* `else` block */ - compiler_use_next_block(c, after_loop_else); VISIT_SEQ(c, stmt, s->v.For.orelse); compiler_use_next_block(c, end); @@ -4003,28 +3967,14 @@ compiler_async_comprehension_generator(struct compiler *c, asdl_seq *generators, int gen_index, expr_ty elt, expr_ty val, int type) { - _Py_IDENTIFIER(StopAsyncIteration); - comprehension_ty gen; - basicblock *anchor, *if_cleanup, *try, - *after_try, *except, *try_cleanup; + basicblock *start, *if_cleanup, *except; Py_ssize_t i, n; - - PyObject *stop_aiter_error = _PyUnicode_FromId(&PyId_StopAsyncIteration); - if (stop_aiter_error == NULL) { - return 0; - } - - try = compiler_new_block(c); - after_try = compiler_new_block(c); - try_cleanup = compiler_new_block(c); + start = compiler_new_block(c); except = compiler_new_block(c); if_cleanup = compiler_new_block(c); - anchor = compiler_new_block(c); - if (if_cleanup == NULL || anchor == NULL || - try == NULL || after_try == NULL || - except == NULL || try_cleanup == NULL) { + if (start == NULL || if_cleanup == NULL || except == NULL) { return 0; } @@ -4041,39 +3991,14 @@ compiler_async_comprehension_generator(struct compiler *c, ADDOP(c, GET_AITER); } - compiler_use_next_block(c, try); - + compiler_use_next_block(c, start); ADDOP_JREL(c, SETUP_FINALLY, except); - if (!compiler_push_fblock(c, EXCEPT, try, NULL)) - return 0; - ADDOP(c, GET_ANEXT); ADDOP_O(c, LOAD_CONST, Py_None, consts); ADDOP(c, YIELD_FROM); ADDOP(c, POP_BLOCK); VISIT(c, expr, gen->target); - compiler_pop_fblock(c, EXCEPT, try); - ADDOP_JREL(c, JUMP_FORWARD, after_try); - - - compiler_use_next_block(c, except); - ADDOP(c, DUP_TOP); - ADDOP_O(c, LOAD_GLOBAL, stop_aiter_error, names); - ADDOP_I(c, COMPARE_OP, PyCmp_EXC_MATCH); - ADDOP_JABS(c, POP_JUMP_IF_FALSE, try_cleanup); - - ADDOP(c, POP_TOP); - ADDOP(c, POP_TOP); - ADDOP(c, POP_TOP); - ADDOP(c, POP_EXCEPT); /* for SETUP_FINALLY */ - ADDOP_JABS(c, JUMP_ABSOLUTE, anchor); - - - compiler_use_next_block(c, try_cleanup); - ADDOP(c, END_FINALLY); - - compiler_use_next_block(c, after_try); n = asdl_seq_LEN(gen->ifs); for (i = 0; i < n; i++) { @@ -4118,9 +4043,10 @@ compiler_async_comprehension_generator(struct compiler *c, } } compiler_use_next_block(c, if_cleanup); - ADDOP_JABS(c, JUMP_ABSOLUTE, try); - compiler_use_next_block(c, anchor); - ADDOP(c, POP_TOP); + ADDOP_JABS(c, JUMP_ABSOLUTE, start); + + compiler_use_next_block(c, except); + ADDOP(c, END_ASYNC_FOR); return 1; } diff --git a/Python/importlib_external.h b/Python/importlib_external.h index a1f80b5c53c4..220f7147443e 100644 --- a/Python/importlib_external.h +++ b/Python/importlib_external.h @@ -243,7 +243,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,0,114,4,0,0,0,218,13,95,119,114,105,116,101,95, 97,116,111,109,105,99,105,0,0,0,115,26,0,0,0,0, 5,16,1,6,1,26,1,2,3,14,1,20,1,16,1,14, - 1,2,1,14,1,14,1,6,1,114,56,0,0,0,105,72, + 1,2,1,14,1,14,1,6,1,114,56,0,0,0,105,73, 13,0,0,233,2,0,0,0,114,13,0,0,0,115,2,0, 0,0,13,10,90,11,95,95,112,121,99,97,99,104,101,95, 95,122,4,111,112,116,45,122,3,46,112,121,122,4,46,112, @@ -348,7 +348,7 @@ const unsigned char _Py_M__importlib_external[] = { 15,97,108,109,111,115,116,95,102,105,108,101,110,97,109,101, 114,2,0,0,0,114,2,0,0,0,114,4,0,0,0,218, 17,99,97,99,104,101,95,102,114,111,109,95,115,111,117,114, - 99,101,14,1,0,0,115,48,0,0,0,0,18,8,1,6, + 99,101,15,1,0,0,115,48,0,0,0,0,18,8,1,6, 1,6,1,8,1,4,1,8,1,12,1,10,1,12,1,16, 1,8,1,8,1,8,1,24,1,8,1,12,1,6,2,8, 1,8,1,8,1,8,1,14,1,14,1,114,81,0,0,0, @@ -422,7 +422,7 @@ const unsigned char _Py_M__importlib_external[] = { 95,108,101,118,101,108,90,13,98,97,115,101,95,102,105,108, 101,110,97,109,101,114,2,0,0,0,114,2,0,0,0,114, 4,0,0,0,218,17,115,111,117,114,99,101,95,102,114,111, - 109,95,99,97,99,104,101,59,1,0,0,115,46,0,0,0, + 109,95,99,97,99,104,101,60,1,0,0,115,46,0,0,0, 0,9,12,1,8,1,10,1,12,1,12,1,8,1,6,1, 10,1,10,1,8,1,6,1,10,1,8,1,16,1,10,1, 6,1,8,1,16,1,8,1,6,1,8,1,14,1,114,87, @@ -456,7 +456,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,114,36,0,0,0,90,9,101,120,116,101,110,115,105,111, 110,218,11,115,111,117,114,99,101,95,112,97,116,104,114,2, 0,0,0,114,2,0,0,0,114,4,0,0,0,218,15,95, - 103,101,116,95,115,111,117,114,99,101,102,105,108,101,93,1, + 103,101,116,95,115,111,117,114,99,101,102,105,108,101,94,1, 0,0,115,20,0,0,0,0,7,12,1,4,1,16,1,24, 1,4,1,2,1,12,1,18,1,18,1,114,93,0,0,0, 99,1,0,0,0,0,0,0,0,1,0,0,0,8,0,0, @@ -470,7 +470,7 @@ const unsigned char _Py_M__importlib_external[] = { 81,0,0,0,114,68,0,0,0,114,76,0,0,0,41,1, 218,8,102,105,108,101,110,97,109,101,114,2,0,0,0,114, 2,0,0,0,114,4,0,0,0,218,11,95,103,101,116,95, - 99,97,99,104,101,100,112,1,0,0,115,16,0,0,0,0, + 99,97,99,104,101,100,113,1,0,0,115,16,0,0,0,0, 1,14,1,2,1,10,1,14,1,8,1,14,1,4,2,114, 97,0,0,0,99,1,0,0,0,0,0,0,0,2,0,0, 0,8,0,0,0,67,0,0,0,115,52,0,0,0,122,14, @@ -484,7 +484,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,0,0,41,3,114,39,0,0,0,114,41,0,0,0,114, 40,0,0,0,41,2,114,35,0,0,0,114,42,0,0,0, 114,2,0,0,0,114,2,0,0,0,114,4,0,0,0,218, - 10,95,99,97,108,99,95,109,111,100,101,124,1,0,0,115, + 10,95,99,97,108,99,95,109,111,100,101,125,1,0,0,115, 12,0,0,0,0,2,2,1,14,1,14,1,10,3,8,1, 114,99,0,0,0,99,1,0,0,0,0,0,0,0,3,0, 0,0,8,0,0,0,3,0,0,0,115,68,0,0,0,100, @@ -522,7 +522,7 @@ const unsigned char _Py_M__importlib_external[] = { 103,115,90,6,107,119,97,114,103,115,41,1,218,6,109,101, 116,104,111,100,114,2,0,0,0,114,4,0,0,0,218,19, 95,99,104,101,99,107,95,110,97,109,101,95,119,114,97,112, - 112,101,114,144,1,0,0,115,12,0,0,0,0,1,8,1, + 112,101,114,145,1,0,0,115,12,0,0,0,0,1,8,1, 8,1,10,1,4,1,18,1,122,40,95,99,104,101,99,107, 95,110,97,109,101,46,60,108,111,99,97,108,115,62,46,95, 99,104,101,99,107,95,110,97,109,101,95,119,114,97,112,112, @@ -539,7 +539,7 @@ const unsigned char _Py_M__importlib_external[] = { 116,116,114,218,8,95,95,100,105,99,116,95,95,218,6,117, 112,100,97,116,101,41,3,90,3,110,101,119,90,3,111,108, 100,114,53,0,0,0,114,2,0,0,0,114,2,0,0,0, - 114,4,0,0,0,218,5,95,119,114,97,112,155,1,0,0, + 114,4,0,0,0,218,5,95,119,114,97,112,156,1,0,0, 115,8,0,0,0,0,1,8,1,10,1,20,1,122,26,95, 99,104,101,99,107,95,110,97,109,101,46,60,108,111,99,97, 108,115,62,46,95,119,114,97,112,41,1,78,41,3,218,10, @@ -547,7 +547,7 @@ const unsigned char _Py_M__importlib_external[] = { 9,78,97,109,101,69,114,114,111,114,41,3,114,104,0,0, 0,114,105,0,0,0,114,115,0,0,0,114,2,0,0,0, 41,1,114,104,0,0,0,114,4,0,0,0,218,11,95,99, - 104,101,99,107,95,110,97,109,101,136,1,0,0,115,14,0, + 104,101,99,107,95,110,97,109,101,137,1,0,0,115,14,0, 0,0,0,8,14,7,2,1,10,1,14,2,14,5,10,1, 114,118,0,0,0,99,2,0,0,0,0,0,0,0,5,0, 0,0,6,0,0,0,67,0,0,0,115,60,0,0,0,124, @@ -575,7 +575,7 @@ const unsigned char _Py_M__importlib_external[] = { 101,218,6,108,111,97,100,101,114,218,8,112,111,114,116,105, 111,110,115,218,3,109,115,103,114,2,0,0,0,114,2,0, 0,0,114,4,0,0,0,218,17,95,102,105,110,100,95,109, - 111,100,117,108,101,95,115,104,105,109,164,1,0,0,115,10, + 111,100,117,108,101,95,115,104,105,109,165,1,0,0,115,10, 0,0,0,0,10,14,1,16,1,4,1,22,1,114,125,0, 0,0,99,3,0,0,0,0,0,0,0,6,0,0,0,4, 0,0,0,67,0,0,0,115,158,0,0,0,124,0,100,1, @@ -642,7 +642,7 @@ const unsigned char _Py_M__importlib_external[] = { 115,90,5,109,97,103,105,99,114,77,0,0,0,114,69,0, 0,0,114,2,0,0,0,114,2,0,0,0,114,4,0,0, 0,218,13,95,99,108,97,115,115,105,102,121,95,112,121,99, - 181,1,0,0,115,28,0,0,0,0,16,12,1,8,1,16, + 182,1,0,0,115,28,0,0,0,0,16,12,1,8,1,16, 1,12,1,12,1,12,1,10,1,12,1,8,1,16,2,8, 1,16,1,12,1,114,133,0,0,0,99,5,0,0,0,0, 0,0,0,6,0,0,0,4,0,0,0,67,0,0,0,115, @@ -696,7 +696,7 @@ const unsigned char _Py_M__importlib_external[] = { 101,114,100,0,0,0,114,132,0,0,0,114,77,0,0,0, 114,2,0,0,0,114,2,0,0,0,114,4,0,0,0,218, 23,95,118,97,108,105,100,97,116,101,95,116,105,109,101,115, - 116,97,109,112,95,112,121,99,214,1,0,0,115,14,0,0, + 116,97,109,112,95,112,121,99,215,1,0,0,115,14,0,0, 0,0,19,24,1,10,1,12,1,12,1,8,1,24,1,114, 137,0,0,0,99,4,0,0,0,0,0,0,0,4,0,0, 0,3,0,0,0,67,0,0,0,115,38,0,0,0,124,0, @@ -742,7 +742,7 @@ const unsigned char _Py_M__importlib_external[] = { 104,97,115,104,114,100,0,0,0,114,132,0,0,0,114,2, 0,0,0,114,2,0,0,0,114,4,0,0,0,218,18,95, 118,97,108,105,100,97,116,101,95,104,97,115,104,95,112,121, - 99,242,1,0,0,115,8,0,0,0,0,17,16,1,2,1, + 99,243,1,0,0,115,8,0,0,0,0,17,16,1,2,1, 10,1,114,139,0,0,0,99,4,0,0,0,0,0,0,0, 5,0,0,0,5,0,0,0,67,0,0,0,115,80,0,0, 0,116,0,160,1,124,0,161,1,125,4,116,2,124,4,116, @@ -765,7 +765,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,0,114,100,0,0,0,114,91,0,0,0,114,92,0,0, 0,218,4,99,111,100,101,114,2,0,0,0,114,2,0,0, 0,114,4,0,0,0,218,17,95,99,111,109,112,105,108,101, - 95,98,121,116,101,99,111,100,101,10,2,0,0,115,16,0, + 95,98,121,116,101,99,111,100,101,11,2,0,0,115,16,0, 0,0,0,2,10,1,10,1,12,1,8,1,12,1,4,2, 10,1,114,145,0,0,0,114,60,0,0,0,99,3,0,0, 0,0,0,0,0,4,0,0,0,5,0,0,0,67,0,0, @@ -783,7 +783,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,0,218,5,109,116,105,109,101,114,136,0,0,0,114,54, 0,0,0,114,2,0,0,0,114,2,0,0,0,114,4,0, 0,0,218,22,95,99,111,100,101,95,116,111,95,116,105,109, - 101,115,116,97,109,112,95,112,121,99,23,2,0,0,115,12, + 101,115,116,97,109,112,95,112,121,99,24,2,0,0,115,12, 0,0,0,0,2,8,1,14,1,14,1,14,1,16,1,114, 150,0,0,0,84,99,3,0,0,0,0,0,0,0,5,0, 0,0,5,0,0,0,67,0,0,0,115,80,0,0,0,116, @@ -802,7 +802,7 @@ const unsigned char _Py_M__importlib_external[] = { 138,0,0,0,90,7,99,104,101,99,107,101,100,114,54,0, 0,0,114,69,0,0,0,114,2,0,0,0,114,2,0,0, 0,114,4,0,0,0,218,17,95,99,111,100,101,95,116,111, - 95,104,97,115,104,95,112,121,99,33,2,0,0,115,14,0, + 95,104,97,115,104,95,112,121,99,34,2,0,0,115,14,0, 0,0,0,2,8,1,12,1,14,1,16,1,10,1,16,1, 114,152,0,0,0,99,1,0,0,0,0,0,0,0,5,0, 0,0,6,0,0,0,67,0,0,0,115,62,0,0,0,100, @@ -829,7 +829,7 @@ const unsigned char _Py_M__importlib_external[] = { 100,108,105,110,101,218,8,101,110,99,111,100,105,110,103,90, 15,110,101,119,108,105,110,101,95,100,101,99,111,100,101,114, 114,2,0,0,0,114,2,0,0,0,114,4,0,0,0,218, - 13,100,101,99,111,100,101,95,115,111,117,114,99,101,44,2, + 13,100,101,99,111,100,101,95,115,111,117,114,99,101,45,2, 0,0,115,10,0,0,0,0,5,8,1,12,1,10,1,12, 1,114,157,0,0,0,41,2,114,122,0,0,0,218,26,115, 117,98,109,111,100,117,108,101,95,115,101,97,114,99,104,95, @@ -891,7 +891,7 @@ const unsigned char _Py_M__importlib_external[] = { 161,0,0,0,90,7,100,105,114,110,97,109,101,114,2,0, 0,0,114,2,0,0,0,114,4,0,0,0,218,23,115,112, 101,99,95,102,114,111,109,95,102,105,108,101,95,108,111,99, - 97,116,105,111,110,61,2,0,0,115,62,0,0,0,0,12, + 97,116,105,111,110,62,2,0,0,115,62,0,0,0,0,12, 8,4,4,1,10,2,2,1,14,1,14,1,8,2,10,8, 16,1,6,3,8,1,14,1,14,1,10,1,6,1,6,2, 4,3,8,2,10,1,2,1,14,1,14,1,6,2,4,1, @@ -928,7 +928,7 @@ const unsigned char _Py_M__importlib_external[] = { 65,76,95,77,65,67,72,73,78,69,41,2,218,3,99,108, 115,114,3,0,0,0,114,2,0,0,0,114,2,0,0,0, 114,4,0,0,0,218,14,95,111,112,101,110,95,114,101,103, - 105,115,116,114,121,141,2,0,0,115,8,0,0,0,0,2, + 105,115,116,114,121,142,2,0,0,115,8,0,0,0,0,2, 2,1,16,1,14,1,122,36,87,105,110,100,111,119,115,82, 101,103,105,115,116,114,121,70,105,110,100,101,114,46,95,111, 112,101,110,95,114,101,103,105,115,116,114,121,99,2,0,0, @@ -954,7 +954,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,0,0,90,4,104,107,101,121,218,8,102,105,108,101,112, 97,116,104,114,2,0,0,0,114,2,0,0,0,114,4,0, 0,0,218,16,95,115,101,97,114,99,104,95,114,101,103,105, - 115,116,114,121,148,2,0,0,115,22,0,0,0,0,2,6, + 115,116,114,121,149,2,0,0,115,22,0,0,0,0,2,6, 1,8,2,6,1,6,1,22,1,2,1,12,1,26,1,14, 1,8,1,122,38,87,105,110,100,111,119,115,82,101,103,105, 115,116,114,121,70,105,110,100,101,114,46,95,115,101,97,114, @@ -976,7 +976,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,0,218,6,116,97,114,103,101,116,114,178,0,0,0,114, 122,0,0,0,114,168,0,0,0,114,166,0,0,0,114,2, 0,0,0,114,2,0,0,0,114,4,0,0,0,218,9,102, - 105,110,100,95,115,112,101,99,163,2,0,0,115,26,0,0, + 105,110,100,95,115,112,101,99,164,2,0,0,115,26,0,0, 0,0,2,10,1,8,1,4,1,2,1,12,1,14,1,8, 1,14,1,14,1,6,1,8,1,8,1,122,31,87,105,110, 100,111,119,115,82,101,103,105,115,116,114,121,70,105,110,100, @@ -995,7 +995,7 @@ const unsigned char _Py_M__importlib_external[] = { 114,122,0,0,0,41,4,114,172,0,0,0,114,121,0,0, 0,114,35,0,0,0,114,166,0,0,0,114,2,0,0,0, 114,2,0,0,0,114,4,0,0,0,218,11,102,105,110,100, - 95,109,111,100,117,108,101,179,2,0,0,115,8,0,0,0, + 95,109,111,100,117,108,101,180,2,0,0,115,8,0,0,0, 0,7,12,1,8,1,6,2,122,33,87,105,110,100,111,119, 115,82,101,103,105,115,116,114,121,70,105,110,100,101,114,46, 102,105,110,100,95,109,111,100,117,108,101,41,2,78,78,41, @@ -1005,7 +1005,7 @@ const unsigned char _Py_M__importlib_external[] = { 116,104,111,100,114,173,0,0,0,114,179,0,0,0,114,182, 0,0,0,114,183,0,0,0,114,2,0,0,0,114,2,0, 0,0,114,2,0,0,0,114,4,0,0,0,114,170,0,0, - 0,129,2,0,0,115,18,0,0,0,12,5,4,3,4,2, + 0,130,2,0,0,115,18,0,0,0,12,5,4,3,4,2, 4,2,12,7,12,15,2,1,12,15,2,1,114,170,0,0, 0,99,0,0,0,0,0,0,0,0,0,0,0,0,2,0, 0,0,64,0,0,0,115,48,0,0,0,101,0,90,1,100, @@ -1039,7 +1039,7 @@ const unsigned char _Py_M__importlib_external[] = { 41,5,114,102,0,0,0,114,121,0,0,0,114,96,0,0, 0,90,13,102,105,108,101,110,97,109,101,95,98,97,115,101, 90,9,116,97,105,108,95,110,97,109,101,114,2,0,0,0, - 114,2,0,0,0,114,4,0,0,0,114,161,0,0,0,198, + 114,2,0,0,0,114,4,0,0,0,114,161,0,0,0,199, 2,0,0,115,8,0,0,0,0,3,18,1,16,1,14,1, 122,24,95,76,111,97,100,101,114,66,97,115,105,99,115,46, 105,115,95,112,97,99,107,97,103,101,99,2,0,0,0,0, @@ -1050,7 +1050,7 @@ const unsigned char _Py_M__importlib_external[] = { 97,116,105,111,110,46,78,114,2,0,0,0,41,2,114,102, 0,0,0,114,166,0,0,0,114,2,0,0,0,114,2,0, 0,0,114,4,0,0,0,218,13,99,114,101,97,116,101,95, - 109,111,100,117,108,101,206,2,0,0,115,0,0,0,0,122, + 109,111,100,117,108,101,207,2,0,0,115,0,0,0,0,122, 27,95,76,111,97,100,101,114,66,97,115,105,99,115,46,99, 114,101,97,116,101,95,109,111,100,117,108,101,99,2,0,0, 0,0,0,0,0,3,0,0,0,5,0,0,0,67,0,0, @@ -1070,7 +1070,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,0,41,3,114,102,0,0,0,218,6,109,111,100,117,108, 101,114,144,0,0,0,114,2,0,0,0,114,2,0,0,0, 114,4,0,0,0,218,11,101,120,101,99,95,109,111,100,117, - 108,101,209,2,0,0,115,10,0,0,0,0,2,12,1,8, + 108,101,210,2,0,0,115,10,0,0,0,0,2,12,1,8, 1,6,1,10,1,122,25,95,76,111,97,100,101,114,66,97, 115,105,99,115,46,101,120,101,99,95,109,111,100,117,108,101, 99,2,0,0,0,0,0,0,0,2,0,0,0,4,0,0, @@ -1081,14 +1081,14 @@ const unsigned char _Py_M__importlib_external[] = { 97,100,95,109,111,100,117,108,101,95,115,104,105,109,41,2, 114,102,0,0,0,114,121,0,0,0,114,2,0,0,0,114, 2,0,0,0,114,4,0,0,0,218,11,108,111,97,100,95, - 109,111,100,117,108,101,217,2,0,0,115,2,0,0,0,0, + 109,111,100,117,108,101,218,2,0,0,115,2,0,0,0,0, 2,122,25,95,76,111,97,100,101,114,66,97,115,105,99,115, 46,108,111,97,100,95,109,111,100,117,108,101,78,41,8,114, 107,0,0,0,114,106,0,0,0,114,108,0,0,0,114,109, 0,0,0,114,161,0,0,0,114,187,0,0,0,114,192,0, 0,0,114,194,0,0,0,114,2,0,0,0,114,2,0,0, 0,114,2,0,0,0,114,4,0,0,0,114,185,0,0,0, - 193,2,0,0,115,8,0,0,0,12,5,8,8,8,3,8, + 194,2,0,0,115,8,0,0,0,12,5,8,8,8,3,8, 8,114,185,0,0,0,99,0,0,0,0,0,0,0,0,0, 0,0,0,3,0,0,0,64,0,0,0,115,74,0,0,0, 101,0,90,1,100,0,90,2,100,1,100,2,132,0,90,3, @@ -1113,7 +1113,7 @@ const unsigned char _Py_M__importlib_external[] = { 32,32,78,41,1,114,40,0,0,0,41,2,114,102,0,0, 0,114,35,0,0,0,114,2,0,0,0,114,2,0,0,0, 114,4,0,0,0,218,10,112,97,116,104,95,109,116,105,109, - 101,224,2,0,0,115,2,0,0,0,0,6,122,23,83,111, + 101,225,2,0,0,115,2,0,0,0,0,6,122,23,83,111, 117,114,99,101,76,111,97,100,101,114,46,112,97,116,104,95, 109,116,105,109,101,99,2,0,0,0,0,0,0,0,2,0, 0,0,4,0,0,0,67,0,0,0,115,14,0,0,0,100, @@ -1148,7 +1148,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,0,0,41,1,114,196,0,0,0,41,2,114,102,0,0, 0,114,35,0,0,0,114,2,0,0,0,114,2,0,0,0, 114,4,0,0,0,218,10,112,97,116,104,95,115,116,97,116, - 115,232,2,0,0,115,2,0,0,0,0,11,122,23,83,111, + 115,233,2,0,0,115,2,0,0,0,0,11,122,23,83,111, 117,114,99,101,76,111,97,100,101,114,46,112,97,116,104,95, 115,116,97,116,115,99,4,0,0,0,0,0,0,0,4,0, 0,0,4,0,0,0,67,0,0,0,115,12,0,0,0,124, @@ -1171,7 +1171,7 @@ const unsigned char _Py_M__importlib_external[] = { 4,114,102,0,0,0,114,92,0,0,0,90,10,99,97,99, 104,101,95,112,97,116,104,114,54,0,0,0,114,2,0,0, 0,114,2,0,0,0,114,4,0,0,0,218,15,95,99,97, - 99,104,101,95,98,121,116,101,99,111,100,101,245,2,0,0, + 99,104,101,95,98,121,116,101,99,111,100,101,246,2,0,0, 115,2,0,0,0,0,8,122,28,83,111,117,114,99,101,76, 111,97,100,101,114,46,95,99,97,99,104,101,95,98,121,116, 101,99,111,100,101,99,3,0,0,0,0,0,0,0,3,0, @@ -1188,7 +1188,7 @@ const unsigned char _Py_M__importlib_external[] = { 108,101,115,46,10,32,32,32,32,32,32,32,32,78,114,2, 0,0,0,41,3,114,102,0,0,0,114,35,0,0,0,114, 54,0,0,0,114,2,0,0,0,114,2,0,0,0,114,4, - 0,0,0,114,198,0,0,0,255,2,0,0,115,0,0,0, + 0,0,0,114,198,0,0,0,0,3,0,0,115,0,0,0, 0,122,21,83,111,117,114,99,101,76,111,97,100,101,114,46, 115,101,116,95,100,97,116,97,99,2,0,0,0,0,0,0, 0,5,0,0,0,10,0,0,0,67,0,0,0,115,82,0, @@ -1209,7 +1209,7 @@ const unsigned char _Py_M__importlib_external[] = { 102,0,0,0,114,121,0,0,0,114,35,0,0,0,114,155, 0,0,0,218,3,101,120,99,114,2,0,0,0,114,2,0, 0,0,114,4,0,0,0,218,10,103,101,116,95,115,111,117, - 114,99,101,6,3,0,0,115,14,0,0,0,0,2,10,1, + 114,99,101,7,3,0,0,115,14,0,0,0,0,2,10,1, 2,1,14,1,16,1,4,1,28,1,122,23,83,111,117,114, 99,101,76,111,97,100,101,114,46,103,101,116,95,115,111,117, 114,99,101,114,89,0,0,0,41,1,218,9,95,111,112,116, @@ -1231,7 +1231,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,0,114,54,0,0,0,114,35,0,0,0,114,203,0,0, 0,114,2,0,0,0,114,2,0,0,0,114,4,0,0,0, 218,14,115,111,117,114,99,101,95,116,111,95,99,111,100,101, - 16,3,0,0,115,4,0,0,0,0,5,12,1,122,27,83, + 17,3,0,0,115,4,0,0,0,0,5,12,1,122,27,83, 111,117,114,99,101,76,111,97,100,101,114,46,115,111,117,114, 99,101,95,116,111,95,99,111,100,101,99,2,0,0,0,0, 0,0,0,15,0,0,0,9,0,0,0,67,0,0,0,115, @@ -1310,7 +1310,7 @@ const unsigned char _Py_M__importlib_external[] = { 69,0,0,0,90,10,98,121,116,101,115,95,100,97,116,97, 90,11,99,111,100,101,95,111,98,106,101,99,116,114,2,0, 0,0,114,2,0,0,0,114,4,0,0,0,114,188,0,0, - 0,24,3,0,0,115,134,0,0,0,0,7,10,1,4,1, + 0,25,3,0,0,115,134,0,0,0,0,7,10,1,4,1, 4,1,4,1,4,1,4,1,2,1,12,1,14,1,12,2, 2,1,14,1,14,1,8,2,12,1,2,1,14,1,14,1, 6,3,2,1,8,2,2,1,12,1,16,1,12,1,6,1, @@ -1325,7 +1325,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,0,0,114,199,0,0,0,114,198,0,0,0,114,202,0, 0,0,114,206,0,0,0,114,188,0,0,0,114,2,0,0, 0,114,2,0,0,0,114,2,0,0,0,114,4,0,0,0, - 114,195,0,0,0,222,2,0,0,115,14,0,0,0,8,2, + 114,195,0,0,0,223,2,0,0,115,14,0,0,0,8,2, 8,8,8,13,8,10,8,7,8,10,14,8,114,195,0,0, 0,99,0,0,0,0,0,0,0,0,0,0,0,0,4,0, 0,0,0,0,0,0,115,124,0,0,0,101,0,90,1,100, @@ -1354,7 +1354,7 @@ const unsigned char _Py_M__importlib_external[] = { 102,105,110,100,101,114,46,78,41,2,114,100,0,0,0,114, 35,0,0,0,41,3,114,102,0,0,0,114,121,0,0,0, 114,35,0,0,0,114,2,0,0,0,114,2,0,0,0,114, - 4,0,0,0,114,186,0,0,0,115,3,0,0,115,4,0, + 4,0,0,0,114,186,0,0,0,116,3,0,0,115,4,0, 0,0,0,3,6,1,122,19,70,105,108,101,76,111,97,100, 101,114,46,95,95,105,110,105,116,95,95,99,2,0,0,0, 0,0,0,0,2,0,0,0,2,0,0,0,67,0,0,0, @@ -1363,7 +1363,7 @@ const unsigned char _Py_M__importlib_external[] = { 41,2,218,9,95,95,99,108,97,115,115,95,95,114,113,0, 0,0,41,2,114,102,0,0,0,218,5,111,116,104,101,114, 114,2,0,0,0,114,2,0,0,0,114,4,0,0,0,218, - 6,95,95,101,113,95,95,121,3,0,0,115,4,0,0,0, + 6,95,95,101,113,95,95,122,3,0,0,115,4,0,0,0, 0,1,12,1,122,17,70,105,108,101,76,111,97,100,101,114, 46,95,95,101,113,95,95,99,1,0,0,0,0,0,0,0, 1,0,0,0,3,0,0,0,67,0,0,0,115,20,0,0, @@ -1371,7 +1371,7 @@ const unsigned char _Py_M__importlib_external[] = { 1,65,0,83,0,41,1,78,41,3,218,4,104,97,115,104, 114,100,0,0,0,114,35,0,0,0,41,1,114,102,0,0, 0,114,2,0,0,0,114,2,0,0,0,114,4,0,0,0, - 218,8,95,95,104,97,115,104,95,95,125,3,0,0,115,2, + 218,8,95,95,104,97,115,104,95,95,126,3,0,0,115,2, 0,0,0,0,1,122,19,70,105,108,101,76,111,97,100,101, 114,46,95,95,104,97,115,104,95,95,99,2,0,0,0,0, 0,0,0,2,0,0,0,3,0,0,0,3,0,0,0,115, @@ -1386,7 +1386,7 @@ const unsigned char _Py_M__importlib_external[] = { 115,117,112,101,114,114,212,0,0,0,114,194,0,0,0,41, 2,114,102,0,0,0,114,121,0,0,0,41,1,114,213,0, 0,0,114,2,0,0,0,114,4,0,0,0,114,194,0,0, - 0,128,3,0,0,115,2,0,0,0,0,10,122,22,70,105, + 0,129,3,0,0,115,2,0,0,0,0,10,122,22,70,105, 108,101,76,111,97,100,101,114,46,108,111,97,100,95,109,111, 100,117,108,101,99,2,0,0,0,0,0,0,0,2,0,0, 0,1,0,0,0,67,0,0,0,115,6,0,0,0,124,0, @@ -1396,7 +1396,7 @@ const unsigned char _Py_M__importlib_external[] = { 117,110,100,32,98,121,32,116,104,101,32,102,105,110,100,101, 114,46,41,1,114,35,0,0,0,41,2,114,102,0,0,0, 114,121,0,0,0,114,2,0,0,0,114,2,0,0,0,114, - 4,0,0,0,114,159,0,0,0,140,3,0,0,115,2,0, + 4,0,0,0,114,159,0,0,0,141,3,0,0,115,2,0, 0,0,0,3,122,23,70,105,108,101,76,111,97,100,101,114, 46,103,101,116,95,102,105,108,101,110,97,109,101,99,2,0, 0,0,0,0,0,0,3,0,0,0,10,0,0,0,67,0, @@ -1409,7 +1409,7 @@ const unsigned char _Py_M__importlib_external[] = { 114,78,41,3,114,50,0,0,0,114,51,0,0,0,90,4, 114,101,97,100,41,3,114,102,0,0,0,114,35,0,0,0, 114,55,0,0,0,114,2,0,0,0,114,2,0,0,0,114, - 4,0,0,0,114,200,0,0,0,145,3,0,0,115,4,0, + 4,0,0,0,114,200,0,0,0,146,3,0,0,115,4,0, 0,0,0,2,14,1,122,19,70,105,108,101,76,111,97,100, 101,114,46,103,101,116,95,100,97,116,97,99,2,0,0,0, 0,0,0,0,2,0,0,0,3,0,0,0,67,0,0,0, @@ -1418,7 +1418,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,41,2,114,102,0,0,0,114,191,0,0,0,114,2,0, 0,0,114,2,0,0,0,114,4,0,0,0,218,19,103,101, 116,95,114,101,115,111,117,114,99,101,95,114,101,97,100,101, - 114,152,3,0,0,115,6,0,0,0,0,2,10,1,4,1, + 114,153,3,0,0,115,6,0,0,0,0,2,10,1,4,1, 122,30,70,105,108,101,76,111,97,100,101,114,46,103,101,116, 95,114,101,115,111,117,114,99,101,95,114,101,97,100,101,114, 99,2,0,0,0,0,0,0,0,3,0,0,0,4,0,0, @@ -1430,7 +1430,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,0,41,3,114,102,0,0,0,218,8,114,101,115,111,117, 114,99,101,114,35,0,0,0,114,2,0,0,0,114,2,0, 0,0,114,4,0,0,0,218,13,111,112,101,110,95,114,101, - 115,111,117,114,99,101,158,3,0,0,115,4,0,0,0,0, + 115,111,117,114,99,101,159,3,0,0,115,4,0,0,0,0, 1,20,1,122,24,70,105,108,101,76,111,97,100,101,114,46, 111,112,101,110,95,114,101,115,111,117,114,99,101,99,2,0, 0,0,0,0,0,0,3,0,0,0,3,0,0,0,67,0, @@ -1443,7 +1443,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,114,35,0,0,0,41,3,114,102,0,0,0,114,221,0, 0,0,114,35,0,0,0,114,2,0,0,0,114,2,0,0, 0,114,4,0,0,0,218,13,114,101,115,111,117,114,99,101, - 95,112,97,116,104,162,3,0,0,115,8,0,0,0,0,1, + 95,112,97,116,104,163,3,0,0,115,8,0,0,0,0,1, 10,1,4,1,20,1,122,24,70,105,108,101,76,111,97,100, 101,114,46,114,101,115,111,117,114,99,101,95,112,97,116,104, 99,2,0,0,0,0,0,0,0,3,0,0,0,3,0,0, @@ -1454,7 +1454,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,0,114,28,0,0,0,114,38,0,0,0,114,35,0,0, 0,114,44,0,0,0,41,3,114,102,0,0,0,114,100,0, 0,0,114,35,0,0,0,114,2,0,0,0,114,2,0,0, - 0,114,4,0,0,0,114,223,0,0,0,168,3,0,0,115, + 0,114,4,0,0,0,114,223,0,0,0,169,3,0,0,115, 8,0,0,0,0,1,8,1,4,1,20,1,122,22,70,105, 108,101,76,111,97,100,101,114,46,105,115,95,114,101,115,111, 117,114,99,101,99,1,0,0,0,0,0,0,0,1,0,0, @@ -1464,7 +1464,7 @@ const unsigned char _Py_M__importlib_external[] = { 218,4,105,116,101,114,114,1,0,0,0,218,7,108,105,115, 116,100,105,114,114,38,0,0,0,114,35,0,0,0,41,1, 114,102,0,0,0,114,2,0,0,0,114,2,0,0,0,114, - 4,0,0,0,218,8,99,111,110,116,101,110,116,115,174,3, + 4,0,0,0,218,8,99,111,110,116,101,110,116,115,175,3, 0,0,115,2,0,0,0,0,1,122,19,70,105,108,101,76, 111,97,100,101,114,46,99,111,110,116,101,110,116,115,41,17, 114,107,0,0,0,114,106,0,0,0,114,108,0,0,0,114, @@ -1474,7 +1474,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,114,225,0,0,0,114,223,0,0,0,114,228,0,0,0, 90,13,95,95,99,108,97,115,115,99,101,108,108,95,95,114, 2,0,0,0,114,2,0,0,0,41,1,114,213,0,0,0, - 114,4,0,0,0,114,212,0,0,0,110,3,0,0,115,22, + 114,4,0,0,0,114,212,0,0,0,111,3,0,0,115,22, 0,0,0,12,5,8,6,8,4,8,3,16,12,12,5,8, 7,12,6,8,4,8,6,8,6,114,212,0,0,0,99,0, 0,0,0,0,0,0,0,0,0,0,0,3,0,0,0,64, @@ -1496,7 +1496,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,0,218,8,115,116,95,109,116,105,109,101,90,7,115,116, 95,115,105,122,101,41,3,114,102,0,0,0,114,35,0,0, 0,114,211,0,0,0,114,2,0,0,0,114,2,0,0,0, - 114,4,0,0,0,114,197,0,0,0,182,3,0,0,115,4, + 114,4,0,0,0,114,197,0,0,0,183,3,0,0,115,4, 0,0,0,0,2,8,1,122,27,83,111,117,114,99,101,70, 105,108,101,76,111,97,100,101,114,46,112,97,116,104,95,115, 116,97,116,115,99,4,0,0,0,0,0,0,0,5,0,0, @@ -1506,7 +1506,7 @@ const unsigned char _Py_M__importlib_external[] = { 100,101,41,2,114,99,0,0,0,114,198,0,0,0,41,5, 114,102,0,0,0,114,92,0,0,0,114,91,0,0,0,114, 54,0,0,0,114,42,0,0,0,114,2,0,0,0,114,2, - 0,0,0,114,4,0,0,0,114,199,0,0,0,187,3,0, + 0,0,0,114,4,0,0,0,114,199,0,0,0,188,3,0, 0,115,4,0,0,0,0,2,8,1,122,32,83,111,117,114, 99,101,70,105,108,101,76,111,97,100,101,114,46,95,99,97, 99,104,101,95,98,121,116,101,99,111,100,101,105,182,1,0, @@ -1541,7 +1541,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,114,54,0,0,0,114,231,0,0,0,218,6,112,97,114, 101,110,116,114,96,0,0,0,114,27,0,0,0,114,23,0, 0,0,114,201,0,0,0,114,2,0,0,0,114,2,0,0, - 0,114,4,0,0,0,114,198,0,0,0,192,3,0,0,115, + 0,114,4,0,0,0,114,198,0,0,0,193,3,0,0,115, 42,0,0,0,0,2,12,1,4,2,12,1,12,1,12,2, 12,1,10,1,2,1,14,1,14,2,8,1,16,3,6,1, 8,1,28,1,2,1,12,1,16,1,16,2,8,1,122,25, @@ -1550,7 +1550,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,114,106,0,0,0,114,108,0,0,0,114,109,0,0,0, 114,197,0,0,0,114,199,0,0,0,114,198,0,0,0,114, 2,0,0,0,114,2,0,0,0,114,2,0,0,0,114,4, - 0,0,0,114,229,0,0,0,178,3,0,0,115,6,0,0, + 0,0,0,114,229,0,0,0,179,3,0,0,115,6,0,0, 0,12,4,8,5,8,5,114,229,0,0,0,99,0,0,0, 0,0,0,0,0,0,0,0,0,2,0,0,0,64,0,0, 0,115,32,0,0,0,101,0,90,1,100,0,90,2,100,1, @@ -1572,7 +1572,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,114,208,0,0,0,41,5,114,102,0,0,0,114,121,0, 0,0,114,35,0,0,0,114,54,0,0,0,114,132,0,0, 0,114,2,0,0,0,114,2,0,0,0,114,4,0,0,0, - 114,188,0,0,0,227,3,0,0,115,18,0,0,0,0,1, + 114,188,0,0,0,228,3,0,0,115,18,0,0,0,0,1, 10,1,10,4,2,1,8,2,12,1,2,1,14,1,2,1, 122,29,83,111,117,114,99,101,108,101,115,115,70,105,108,101, 76,111,97,100,101,114,46,103,101,116,95,99,111,100,101,99, @@ -1582,14 +1582,14 @@ const unsigned char _Py_M__importlib_external[] = { 116,104,101,114,101,32,105,115,32,110,111,32,115,111,117,114, 99,101,32,99,111,100,101,46,78,114,2,0,0,0,41,2, 114,102,0,0,0,114,121,0,0,0,114,2,0,0,0,114, - 2,0,0,0,114,4,0,0,0,114,202,0,0,0,243,3, + 2,0,0,0,114,4,0,0,0,114,202,0,0,0,244,3, 0,0,115,2,0,0,0,0,2,122,31,83,111,117,114,99, 101,108,101,115,115,70,105,108,101,76,111,97,100,101,114,46, 103,101,116,95,115,111,117,114,99,101,78,41,6,114,107,0, 0,0,114,106,0,0,0,114,108,0,0,0,114,109,0,0, 0,114,188,0,0,0,114,202,0,0,0,114,2,0,0,0, 114,2,0,0,0,114,2,0,0,0,114,4,0,0,0,114, - 234,0,0,0,223,3,0,0,115,4,0,0,0,12,4,8, + 234,0,0,0,224,3,0,0,115,4,0,0,0,12,4,8, 16,114,234,0,0,0,99,0,0,0,0,0,0,0,0,0, 0,0,0,3,0,0,0,64,0,0,0,115,92,0,0,0, 101,0,90,1,100,0,90,2,100,1,90,3,100,2,100,3, @@ -1611,7 +1611,7 @@ const unsigned char _Py_M__importlib_external[] = { 2,114,100,0,0,0,114,35,0,0,0,41,3,114,102,0, 0,0,114,100,0,0,0,114,35,0,0,0,114,2,0,0, 0,114,2,0,0,0,114,4,0,0,0,114,186,0,0,0, - 4,4,0,0,115,4,0,0,0,0,1,6,1,122,28,69, + 5,4,0,0,115,4,0,0,0,0,1,6,1,122,28,69, 120,116,101,110,115,105,111,110,70,105,108,101,76,111,97,100, 101,114,46,95,95,105,110,105,116,95,95,99,2,0,0,0, 0,0,0,0,2,0,0,0,2,0,0,0,67,0,0,0, @@ -1619,7 +1619,7 @@ const unsigned char _Py_M__importlib_external[] = { 22,124,0,106,1,124,1,106,1,107,2,83,0,41,1,78, 41,2,114,213,0,0,0,114,113,0,0,0,41,2,114,102, 0,0,0,114,214,0,0,0,114,2,0,0,0,114,2,0, - 0,0,114,4,0,0,0,114,215,0,0,0,8,4,0,0, + 0,0,114,4,0,0,0,114,215,0,0,0,9,4,0,0, 115,4,0,0,0,0,1,12,1,122,26,69,120,116,101,110, 115,105,111,110,70,105,108,101,76,111,97,100,101,114,46,95, 95,101,113,95,95,99,1,0,0,0,0,0,0,0,1,0, @@ -1628,7 +1628,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,83,0,41,1,78,41,3,114,216,0,0,0,114,100,0, 0,0,114,35,0,0,0,41,1,114,102,0,0,0,114,2, 0,0,0,114,2,0,0,0,114,4,0,0,0,114,217,0, - 0,0,12,4,0,0,115,2,0,0,0,0,1,122,28,69, + 0,0,13,4,0,0,115,2,0,0,0,0,1,122,28,69, 120,116,101,110,115,105,111,110,70,105,108,101,76,111,97,100, 101,114,46,95,95,104,97,115,104,95,95,99,2,0,0,0, 0,0,0,0,3,0,0,0,5,0,0,0,67,0,0,0, @@ -1645,7 +1645,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,114,100,0,0,0,114,35,0,0,0,41,3,114,102,0, 0,0,114,166,0,0,0,114,191,0,0,0,114,2,0,0, 0,114,2,0,0,0,114,4,0,0,0,114,187,0,0,0, - 15,4,0,0,115,10,0,0,0,0,2,4,1,10,1,6, + 16,4,0,0,115,10,0,0,0,0,2,4,1,10,1,6, 1,12,1,122,33,69,120,116,101,110,115,105,111,110,70,105, 108,101,76,111,97,100,101,114,46,99,114,101,97,116,101,95, 109,111,100,117,108,101,99,2,0,0,0,0,0,0,0,2, @@ -1661,7 +1661,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,0,90,12,101,120,101,99,95,100,121,110,97,109,105,99, 114,130,0,0,0,114,100,0,0,0,114,35,0,0,0,41, 2,114,102,0,0,0,114,191,0,0,0,114,2,0,0,0, - 114,2,0,0,0,114,4,0,0,0,114,192,0,0,0,23, + 114,2,0,0,0,114,4,0,0,0,114,192,0,0,0,24, 4,0,0,115,6,0,0,0,0,2,14,1,6,1,122,31, 69,120,116,101,110,115,105,111,110,70,105,108,101,76,111,97, 100,101,114,46,101,120,101,99,95,109,111,100,117,108,101,99, @@ -1679,7 +1679,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,0,78,114,2,0,0,0,41,2,114,22,0,0,0,218, 6,115,117,102,102,105,120,41,1,218,9,102,105,108,101,95, 110,97,109,101,114,2,0,0,0,114,4,0,0,0,250,9, - 60,103,101,110,101,120,112,114,62,32,4,0,0,115,2,0, + 60,103,101,110,101,120,112,114,62,33,4,0,0,115,2,0, 0,0,4,1,122,49,69,120,116,101,110,115,105,111,110,70, 105,108,101,76,111,97,100,101,114,46,105,115,95,112,97,99, 107,97,103,101,46,60,108,111,99,97,108,115,62,46,60,103, @@ -1687,7 +1687,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,0,0,218,3,97,110,121,218,18,69,88,84,69,78,83, 73,79,78,95,83,85,70,70,73,88,69,83,41,2,114,102, 0,0,0,114,121,0,0,0,114,2,0,0,0,41,1,114, - 237,0,0,0,114,4,0,0,0,114,161,0,0,0,29,4, + 237,0,0,0,114,4,0,0,0,114,161,0,0,0,30,4, 0,0,115,6,0,0,0,0,2,14,1,12,1,122,30,69, 120,116,101,110,115,105,111,110,70,105,108,101,76,111,97,100, 101,114,46,105,115,95,112,97,99,107,97,103,101,99,2,0, @@ -1699,7 +1699,7 @@ const unsigned char _Py_M__importlib_external[] = { 97,32,99,111,100,101,32,111,98,106,101,99,116,46,78,114, 2,0,0,0,41,2,114,102,0,0,0,114,121,0,0,0, 114,2,0,0,0,114,2,0,0,0,114,4,0,0,0,114, - 188,0,0,0,35,4,0,0,115,2,0,0,0,0,2,122, + 188,0,0,0,36,4,0,0,115,2,0,0,0,0,2,122, 28,69,120,116,101,110,115,105,111,110,70,105,108,101,76,111, 97,100,101,114,46,103,101,116,95,99,111,100,101,99,2,0, 0,0,0,0,0,0,2,0,0,0,1,0,0,0,67,0, @@ -1709,7 +1709,7 @@ const unsigned char _Py_M__importlib_external[] = { 104,97,118,101,32,110,111,32,115,111,117,114,99,101,32,99, 111,100,101,46,78,114,2,0,0,0,41,2,114,102,0,0, 0,114,121,0,0,0,114,2,0,0,0,114,2,0,0,0, - 114,4,0,0,0,114,202,0,0,0,39,4,0,0,115,2, + 114,4,0,0,0,114,202,0,0,0,40,4,0,0,115,2, 0,0,0,0,2,122,30,69,120,116,101,110,115,105,111,110, 70,105,108,101,76,111,97,100,101,114,46,103,101,116,95,115, 111,117,114,99,101,99,2,0,0,0,0,0,0,0,2,0, @@ -1720,7 +1720,7 @@ const unsigned char _Py_M__importlib_external[] = { 111,117,110,100,32,98,121,32,116,104,101,32,102,105,110,100, 101,114,46,41,1,114,35,0,0,0,41,2,114,102,0,0, 0,114,121,0,0,0,114,2,0,0,0,114,2,0,0,0, - 114,4,0,0,0,114,159,0,0,0,43,4,0,0,115,2, + 114,4,0,0,0,114,159,0,0,0,44,4,0,0,115,2, 0,0,0,0,3,122,32,69,120,116,101,110,115,105,111,110, 70,105,108,101,76,111,97,100,101,114,46,103,101,116,95,102, 105,108,101,110,97,109,101,78,41,14,114,107,0,0,0,114, @@ -1729,7 +1729,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,0,114,192,0,0,0,114,161,0,0,0,114,188,0,0, 0,114,202,0,0,0,114,118,0,0,0,114,159,0,0,0, 114,2,0,0,0,114,2,0,0,0,114,2,0,0,0,114, - 4,0,0,0,114,235,0,0,0,252,3,0,0,115,18,0, + 4,0,0,0,114,235,0,0,0,253,3,0,0,115,18,0, 0,0,12,8,8,4,8,4,8,3,8,8,8,6,8,6, 8,4,8,4,114,235,0,0,0,99,0,0,0,0,0,0, 0,0,0,0,0,0,2,0,0,0,64,0,0,0,115,96, @@ -1770,7 +1770,7 @@ const unsigned char _Py_M__importlib_external[] = { 116,104,95,102,105,110,100,101,114,41,4,114,102,0,0,0, 114,100,0,0,0,114,35,0,0,0,218,11,112,97,116,104, 95,102,105,110,100,101,114,114,2,0,0,0,114,2,0,0, - 0,114,4,0,0,0,114,186,0,0,0,56,4,0,0,115, + 0,114,4,0,0,0,114,186,0,0,0,57,4,0,0,115, 8,0,0,0,0,1,6,1,6,1,14,1,122,23,95,78, 97,109,101,115,112,97,99,101,80,97,116,104,46,95,95,105, 110,105,116,95,95,99,1,0,0,0,0,0,0,0,4,0, @@ -1788,7 +1788,7 @@ const unsigned char _Py_M__importlib_external[] = { 3,100,111,116,90,2,109,101,114,2,0,0,0,114,2,0, 0,0,114,4,0,0,0,218,23,95,102,105,110,100,95,112, 97,114,101,110,116,95,112,97,116,104,95,110,97,109,101,115, - 62,4,0,0,115,8,0,0,0,0,2,18,1,8,2,4, + 63,4,0,0,115,8,0,0,0,0,2,18,1,8,2,4, 3,122,38,95,78,97,109,101,115,112,97,99,101,80,97,116, 104,46,95,102,105,110,100,95,112,97,114,101,110,116,95,112, 97,116,104,95,110,97,109,101,115,99,1,0,0,0,0,0, @@ -1800,7 +1800,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,0,0,90,18,112,97,114,101,110,116,95,109,111,100,117, 108,101,95,110,97,109,101,90,14,112,97,116,104,95,97,116, 116,114,95,110,97,109,101,114,2,0,0,0,114,2,0,0, - 0,114,4,0,0,0,114,244,0,0,0,72,4,0,0,115, + 0,114,4,0,0,0,114,244,0,0,0,73,4,0,0,115, 4,0,0,0,0,1,12,1,122,31,95,78,97,109,101,115, 112,97,99,101,80,97,116,104,46,95,103,101,116,95,112,97, 114,101,110,116,95,112,97,116,104,99,1,0,0,0,0,0, @@ -1816,7 +1816,7 @@ const unsigned char _Py_M__importlib_external[] = { 41,3,114,102,0,0,0,90,11,112,97,114,101,110,116,95, 112,97,116,104,114,166,0,0,0,114,2,0,0,0,114,2, 0,0,0,114,4,0,0,0,218,12,95,114,101,99,97,108, - 99,117,108,97,116,101,76,4,0,0,115,16,0,0,0,0, + 99,117,108,97,116,101,77,4,0,0,115,16,0,0,0,0, 2,12,1,10,1,14,3,18,1,6,1,8,1,6,1,122, 27,95,78,97,109,101,115,112,97,99,101,80,97,116,104,46, 95,114,101,99,97,108,99,117,108,97,116,101,99,1,0,0, @@ -1825,7 +1825,7 @@ const unsigned char _Py_M__importlib_external[] = { 83,0,41,1,78,41,2,114,226,0,0,0,114,251,0,0, 0,41,1,114,102,0,0,0,114,2,0,0,0,114,2,0, 0,0,114,4,0,0,0,218,8,95,95,105,116,101,114,95, - 95,89,4,0,0,115,2,0,0,0,0,1,122,23,95,78, + 95,90,4,0,0,115,2,0,0,0,0,1,122,23,95,78, 97,109,101,115,112,97,99,101,80,97,116,104,46,95,95,105, 116,101,114,95,95,99,3,0,0,0,0,0,0,0,3,0, 0,0,3,0,0,0,67,0,0,0,115,14,0,0,0,124, @@ -1833,7 +1833,7 @@ const unsigned char _Py_M__importlib_external[] = { 41,1,114,243,0,0,0,41,3,114,102,0,0,0,218,5, 105,110,100,101,120,114,35,0,0,0,114,2,0,0,0,114, 2,0,0,0,114,4,0,0,0,218,11,95,95,115,101,116, - 105,116,101,109,95,95,92,4,0,0,115,2,0,0,0,0, + 105,116,101,109,95,95,93,4,0,0,115,2,0,0,0,0, 1,122,26,95,78,97,109,101,115,112,97,99,101,80,97,116, 104,46,95,95,115,101,116,105,116,101,109,95,95,99,1,0, 0,0,0,0,0,0,1,0,0,0,3,0,0,0,67,0, @@ -1841,7 +1841,7 @@ const unsigned char _Py_M__importlib_external[] = { 1,83,0,41,1,78,41,2,114,31,0,0,0,114,251,0, 0,0,41,1,114,102,0,0,0,114,2,0,0,0,114,2, 0,0,0,114,4,0,0,0,218,7,95,95,108,101,110,95, - 95,95,4,0,0,115,2,0,0,0,0,1,122,22,95,78, + 95,96,4,0,0,115,2,0,0,0,0,1,122,22,95,78, 97,109,101,115,112,97,99,101,80,97,116,104,46,95,95,108, 101,110,95,95,99,1,0,0,0,0,0,0,0,1,0,0, 0,3,0,0,0,67,0,0,0,115,12,0,0,0,100,1, @@ -1849,7 +1849,7 @@ const unsigned char _Py_M__importlib_external[] = { 78,97,109,101,115,112,97,99,101,80,97,116,104,40,123,33, 114,125,41,41,2,114,48,0,0,0,114,243,0,0,0,41, 1,114,102,0,0,0,114,2,0,0,0,114,2,0,0,0, - 114,4,0,0,0,218,8,95,95,114,101,112,114,95,95,98, + 114,4,0,0,0,218,8,95,95,114,101,112,114,95,95,99, 4,0,0,115,2,0,0,0,0,1,122,23,95,78,97,109, 101,115,112,97,99,101,80,97,116,104,46,95,95,114,101,112, 114,95,95,99,2,0,0,0,0,0,0,0,2,0,0,0, @@ -1857,7 +1857,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,160,0,161,0,107,6,83,0,41,1,78,41,1,114,251, 0,0,0,41,2,114,102,0,0,0,218,4,105,116,101,109, 114,2,0,0,0,114,2,0,0,0,114,4,0,0,0,218, - 12,95,95,99,111,110,116,97,105,110,115,95,95,101,4,0, + 12,95,95,99,111,110,116,97,105,110,115,95,95,102,4,0, 0,115,2,0,0,0,0,1,122,27,95,78,97,109,101,115, 112,97,99,101,80,97,116,104,46,95,95,99,111,110,116,97, 105,110,115,95,95,99,2,0,0,0,0,0,0,0,2,0, @@ -1865,7 +1865,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,106,0,160,1,124,1,161,1,1,0,100,0,83,0,41, 1,78,41,2,114,243,0,0,0,114,165,0,0,0,41,2, 114,102,0,0,0,114,1,1,0,0,114,2,0,0,0,114, - 2,0,0,0,114,4,0,0,0,114,165,0,0,0,104,4, + 2,0,0,0,114,4,0,0,0,114,165,0,0,0,105,4, 0,0,115,2,0,0,0,0,1,122,21,95,78,97,109,101, 115,112,97,99,101,80,97,116,104,46,97,112,112,101,110,100, 78,41,14,114,107,0,0,0,114,106,0,0,0,114,108,0, @@ -1874,7 +1874,7 @@ const unsigned char _Py_M__importlib_external[] = { 114,254,0,0,0,114,255,0,0,0,114,0,1,0,0,114, 2,1,0,0,114,165,0,0,0,114,2,0,0,0,114,2, 0,0,0,114,2,0,0,0,114,4,0,0,0,114,241,0, - 0,0,49,4,0,0,115,20,0,0,0,12,7,8,6,8, + 0,0,50,4,0,0,115,20,0,0,0,12,7,8,6,8, 10,8,4,8,13,8,3,8,3,8,3,8,3,8,3,114, 241,0,0,0,99,0,0,0,0,0,0,0,0,0,0,0, 0,3,0,0,0,64,0,0,0,115,80,0,0,0,101,0, @@ -1890,7 +1890,7 @@ const unsigned char _Py_M__importlib_external[] = { 78,41,2,114,241,0,0,0,114,243,0,0,0,41,4,114, 102,0,0,0,114,100,0,0,0,114,35,0,0,0,114,247, 0,0,0,114,2,0,0,0,114,2,0,0,0,114,4,0, - 0,0,114,186,0,0,0,110,4,0,0,115,2,0,0,0, + 0,0,114,186,0,0,0,111,4,0,0,115,2,0,0,0, 0,1,122,25,95,78,97,109,101,115,112,97,99,101,76,111, 97,100,101,114,46,95,95,105,110,105,116,95,95,99,2,0, 0,0,0,0,0,0,2,0,0,0,3,0,0,0,67,0, @@ -1907,21 +1907,21 @@ const unsigned char _Py_M__importlib_external[] = { 97,99,101,41,62,41,2,114,48,0,0,0,114,107,0,0, 0,41,2,114,172,0,0,0,114,191,0,0,0,114,2,0, 0,0,114,2,0,0,0,114,4,0,0,0,218,11,109,111, - 100,117,108,101,95,114,101,112,114,113,4,0,0,115,2,0, + 100,117,108,101,95,114,101,112,114,114,4,0,0,115,2,0, 0,0,0,7,122,28,95,78,97,109,101,115,112,97,99,101, 76,111,97,100,101,114,46,109,111,100,117,108,101,95,114,101, 112,114,99,2,0,0,0,0,0,0,0,2,0,0,0,1, 0,0,0,67,0,0,0,115,4,0,0,0,100,1,83,0, 41,2,78,84,114,2,0,0,0,41,2,114,102,0,0,0, 114,121,0,0,0,114,2,0,0,0,114,2,0,0,0,114, - 4,0,0,0,114,161,0,0,0,122,4,0,0,115,2,0, + 4,0,0,0,114,161,0,0,0,123,4,0,0,115,2,0, 0,0,0,1,122,27,95,78,97,109,101,115,112,97,99,101, 76,111,97,100,101,114,46,105,115,95,112,97,99,107,97,103, 101,99,2,0,0,0,0,0,0,0,2,0,0,0,1,0, 0,0,67,0,0,0,115,4,0,0,0,100,1,83,0,41, 2,78,114,30,0,0,0,114,2,0,0,0,41,2,114,102, 0,0,0,114,121,0,0,0,114,2,0,0,0,114,2,0, - 0,0,114,4,0,0,0,114,202,0,0,0,125,4,0,0, + 0,0,114,4,0,0,0,114,202,0,0,0,126,4,0,0, 115,2,0,0,0,0,1,122,27,95,78,97,109,101,115,112, 97,99,101,76,111,97,100,101,114,46,103,101,116,95,115,111, 117,114,99,101,99,2,0,0,0,0,0,0,0,2,0,0, @@ -1931,7 +1931,7 @@ const unsigned char _Py_M__importlib_external[] = { 114,190,0,0,0,84,41,1,114,204,0,0,0,41,1,114, 205,0,0,0,41,2,114,102,0,0,0,114,121,0,0,0, 114,2,0,0,0,114,2,0,0,0,114,4,0,0,0,114, - 188,0,0,0,128,4,0,0,115,2,0,0,0,0,1,122, + 188,0,0,0,129,4,0,0,115,2,0,0,0,0,1,122, 25,95,78,97,109,101,115,112,97,99,101,76,111,97,100,101, 114,46,103,101,116,95,99,111,100,101,99,2,0,0,0,0, 0,0,0,2,0,0,0,1,0,0,0,67,0,0,0,115, @@ -1940,14 +1940,14 @@ const unsigned char _Py_M__importlib_external[] = { 115,32,102,111,114,32,109,111,100,117,108,101,32,99,114,101, 97,116,105,111,110,46,78,114,2,0,0,0,41,2,114,102, 0,0,0,114,166,0,0,0,114,2,0,0,0,114,2,0, - 0,0,114,4,0,0,0,114,187,0,0,0,131,4,0,0, + 0,0,114,4,0,0,0,114,187,0,0,0,132,4,0,0, 115,0,0,0,0,122,30,95,78,97,109,101,115,112,97,99, 101,76,111,97,100,101,114,46,99,114,101,97,116,101,95,109, 111,100,117,108,101,99,2,0,0,0,0,0,0,0,2,0, 0,0,1,0,0,0,67,0,0,0,115,4,0,0,0,100, 0,83,0,41,1,78,114,2,0,0,0,41,2,114,102,0, 0,0,114,191,0,0,0,114,2,0,0,0,114,2,0,0, - 0,114,4,0,0,0,114,192,0,0,0,134,4,0,0,115, + 0,114,4,0,0,0,114,192,0,0,0,135,4,0,0,115, 2,0,0,0,0,1,122,28,95,78,97,109,101,115,112,97, 99,101,76,111,97,100,101,114,46,101,120,101,99,95,109,111, 100,117,108,101,99,2,0,0,0,0,0,0,0,2,0,0, @@ -1965,7 +1965,7 @@ const unsigned char _Py_M__importlib_external[] = { 104,32,123,33,114,125,41,4,114,116,0,0,0,114,130,0, 0,0,114,243,0,0,0,114,193,0,0,0,41,2,114,102, 0,0,0,114,121,0,0,0,114,2,0,0,0,114,2,0, - 0,0,114,4,0,0,0,114,194,0,0,0,137,4,0,0, + 0,0,114,4,0,0,0,114,194,0,0,0,138,4,0,0, 115,6,0,0,0,0,7,6,1,8,1,122,28,95,78,97, 109,101,115,112,97,99,101,76,111,97,100,101,114,46,108,111, 97,100,95,109,111,100,117,108,101,78,41,12,114,107,0,0, @@ -1974,7 +1974,7 @@ const unsigned char _Py_M__importlib_external[] = { 202,0,0,0,114,188,0,0,0,114,187,0,0,0,114,192, 0,0,0,114,194,0,0,0,114,2,0,0,0,114,2,0, 0,0,114,2,0,0,0,114,4,0,0,0,114,3,1,0, - 0,109,4,0,0,115,16,0,0,0,8,1,8,3,12,9, + 0,110,4,0,0,115,16,0,0,0,8,1,8,3,12,9, 8,3,8,3,8,3,8,3,8,3,114,3,1,0,0,99, 0,0,0,0,0,0,0,0,0,0,0,0,4,0,0,0, 64,0,0,0,115,106,0,0,0,101,0,90,1,100,0,90, @@ -2007,7 +2007,7 @@ const unsigned char _Py_M__importlib_external[] = { 104,101,218,6,118,97,108,117,101,115,114,110,0,0,0,114, 6,1,0,0,41,2,114,172,0,0,0,218,6,102,105,110, 100,101,114,114,2,0,0,0,114,2,0,0,0,114,4,0, - 0,0,114,6,1,0,0,155,4,0,0,115,6,0,0,0, + 0,0,114,6,1,0,0,156,4,0,0,115,6,0,0,0, 0,4,14,1,10,1,122,28,80,97,116,104,70,105,110,100, 101,114,46,105,110,118,97,108,105,100,97,116,101,95,99,97, 99,104,101,115,99,2,0,0,0,0,0,0,0,3,0,0, @@ -2027,7 +2027,7 @@ const unsigned char _Py_M__importlib_external[] = { 114,101,0,0,0,41,3,114,172,0,0,0,114,35,0,0, 0,90,4,104,111,111,107,114,2,0,0,0,114,2,0,0, 0,114,4,0,0,0,218,11,95,112,97,116,104,95,104,111, - 111,107,115,163,4,0,0,115,16,0,0,0,0,3,16,1, + 111,107,115,164,4,0,0,115,16,0,0,0,0,3,16,1, 12,1,10,1,2,1,14,1,14,1,12,2,122,22,80,97, 116,104,70,105,110,100,101,114,46,95,112,97,116,104,95,104, 111,111,107,115,99,2,0,0,0,0,0,0,0,3,0,0, @@ -2058,7 +2058,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,0,0,114,35,0,0,0,114,9,1,0,0,114,2,0, 0,0,114,2,0,0,0,114,4,0,0,0,218,20,95,112, 97,116,104,95,105,109,112,111,114,116,101,114,95,99,97,99, - 104,101,176,4,0,0,115,22,0,0,0,0,8,8,1,2, + 104,101,177,4,0,0,115,22,0,0,0,0,8,8,1,2, 1,12,1,14,3,8,1,2,1,14,1,14,1,10,1,16, 1,122,31,80,97,116,104,70,105,110,100,101,114,46,95,112, 97,116,104,95,105,109,112,111,114,116,101,114,95,99,97,99, @@ -2075,7 +2075,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,0,114,121,0,0,0,114,9,1,0,0,114,122,0,0, 0,114,123,0,0,0,114,166,0,0,0,114,2,0,0,0, 114,2,0,0,0,114,4,0,0,0,218,16,95,108,101,103, - 97,99,121,95,103,101,116,95,115,112,101,99,198,4,0,0, + 97,99,121,95,103,101,116,95,115,112,101,99,199,4,0,0, 115,18,0,0,0,0,4,10,1,16,2,10,1,4,1,8, 1,12,1,12,1,6,1,122,27,80,97,116,104,70,105,110, 100,101,114,46,95,108,101,103,97,99,121,95,103,101,116,95, @@ -2106,7 +2106,7 @@ const unsigned char _Py_M__importlib_external[] = { 110,97,109,101,115,112,97,99,101,95,112,97,116,104,90,5, 101,110,116,114,121,114,9,1,0,0,114,166,0,0,0,114, 123,0,0,0,114,2,0,0,0,114,2,0,0,0,114,4, - 0,0,0,218,9,95,103,101,116,95,115,112,101,99,213,4, + 0,0,0,218,9,95,103,101,116,95,115,112,101,99,214,4, 0,0,115,40,0,0,0,0,5,4,1,8,1,14,1,2, 1,10,1,8,1,10,1,14,2,12,1,8,1,2,1,10, 1,8,1,6,1,8,1,8,5,12,2,12,1,6,1,122, @@ -2133,7 +2133,7 @@ const unsigned char _Py_M__importlib_external[] = { 114,160,0,0,0,114,241,0,0,0,41,6,114,172,0,0, 0,114,121,0,0,0,114,35,0,0,0,114,181,0,0,0, 114,166,0,0,0,114,16,1,0,0,114,2,0,0,0,114, - 2,0,0,0,114,4,0,0,0,114,182,0,0,0,245,4, + 2,0,0,0,114,4,0,0,0,114,182,0,0,0,246,4, 0,0,115,26,0,0,0,0,6,8,1,6,1,14,1,8, 1,4,1,10,1,6,1,4,3,6,1,16,1,4,2,6, 2,122,20,80,97,116,104,70,105,110,100,101,114,46,102,105, @@ -2155,7 +2155,7 @@ const unsigned char _Py_M__importlib_external[] = { 114,182,0,0,0,114,122,0,0,0,41,4,114,172,0,0, 0,114,121,0,0,0,114,35,0,0,0,114,166,0,0,0, 114,2,0,0,0,114,2,0,0,0,114,4,0,0,0,114, - 183,0,0,0,13,5,0,0,115,8,0,0,0,0,8,12, + 183,0,0,0,14,5,0,0,115,8,0,0,0,0,8,12, 1,8,1,4,1,122,22,80,97,116,104,70,105,110,100,101, 114,46,102,105,110,100,95,109,111,100,117,108,101,41,1,78, 41,2,78,78,41,1,78,41,12,114,107,0,0,0,114,106, @@ -2163,7 +2163,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,0,114,6,1,0,0,114,11,1,0,0,114,13,1,0, 0,114,14,1,0,0,114,17,1,0,0,114,182,0,0,0, 114,183,0,0,0,114,2,0,0,0,114,2,0,0,0,114, - 2,0,0,0,114,4,0,0,0,114,5,1,0,0,151,4, + 2,0,0,0,114,4,0,0,0,114,5,1,0,0,152,4, 0,0,115,20,0,0,0,12,4,12,8,12,13,12,22,12, 15,2,1,12,31,2,1,12,23,2,1,114,5,1,0,0, 99,0,0,0,0,0,0,0,0,0,0,0,0,3,0,0, @@ -2207,7 +2207,7 @@ const unsigned char _Py_M__importlib_external[] = { 102,2,86,0,1,0,113,2,100,0,83,0,41,1,78,114, 2,0,0,0,41,2,114,22,0,0,0,114,236,0,0,0, 41,1,114,122,0,0,0,114,2,0,0,0,114,4,0,0, - 0,114,238,0,0,0,42,5,0,0,115,2,0,0,0,4, + 0,114,238,0,0,0,43,5,0,0,115,2,0,0,0,4, 0,122,38,70,105,108,101,70,105,110,100,101,114,46,95,95, 105,110,105,116,95,95,46,60,108,111,99,97,108,115,62,46, 60,103,101,110,101,120,112,114,62,114,59,0,0,0,114,89, @@ -2219,7 +2219,7 @@ const unsigned char _Py_M__importlib_external[] = { 114,102,0,0,0,114,35,0,0,0,218,14,108,111,97,100, 101,114,95,100,101,116,97,105,108,115,90,7,108,111,97,100, 101,114,115,114,168,0,0,0,114,2,0,0,0,41,1,114, - 122,0,0,0,114,4,0,0,0,114,186,0,0,0,36,5, + 122,0,0,0,114,4,0,0,0,114,186,0,0,0,37,5, 0,0,115,16,0,0,0,0,4,4,1,12,1,26,1,6, 2,10,1,6,1,8,1,122,19,70,105,108,101,70,105,110, 100,101,114,46,95,95,105,110,105,116,95,95,99,1,0,0, @@ -2229,7 +2229,7 @@ const unsigned char _Py_M__importlib_external[] = { 104,101,32,100,105,114,101,99,116,111,114,121,32,109,116,105, 109,101,46,114,89,0,0,0,78,41,1,114,20,1,0,0, 41,1,114,102,0,0,0,114,2,0,0,0,114,2,0,0, - 0,114,4,0,0,0,114,6,1,0,0,50,5,0,0,115, + 0,114,4,0,0,0,114,6,1,0,0,51,5,0,0,115, 2,0,0,0,0,2,122,28,70,105,108,101,70,105,110,100, 101,114,46,105,110,118,97,108,105,100,97,116,101,95,99,97, 99,104,101,115,99,2,0,0,0,0,0,0,0,3,0,0, @@ -2252,7 +2252,7 @@ const unsigned char _Py_M__importlib_external[] = { 32,78,41,3,114,182,0,0,0,114,122,0,0,0,114,158, 0,0,0,41,3,114,102,0,0,0,114,121,0,0,0,114, 166,0,0,0,114,2,0,0,0,114,2,0,0,0,114,4, - 0,0,0,114,119,0,0,0,56,5,0,0,115,8,0,0, + 0,0,0,114,119,0,0,0,57,5,0,0,115,8,0,0, 0,0,7,10,1,8,1,8,1,122,22,70,105,108,101,70, 105,110,100,101,114,46,102,105,110,100,95,108,111,97,100,101, 114,99,6,0,0,0,0,0,0,0,7,0,0,0,6,0, @@ -2263,7 +2263,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,0,114,167,0,0,0,114,121,0,0,0,114,35,0,0, 0,90,4,115,109,115,108,114,181,0,0,0,114,122,0,0, 0,114,2,0,0,0,114,2,0,0,0,114,4,0,0,0, - 114,17,1,0,0,68,5,0,0,115,6,0,0,0,0,1, + 114,17,1,0,0,69,5,0,0,115,6,0,0,0,0,1, 10,1,8,1,122,20,70,105,108,101,70,105,110,100,101,114, 46,95,103,101,116,95,115,112,101,99,78,99,3,0,0,0, 0,0,0,0,14,0,0,0,8,0,0,0,67,0,0,0, @@ -2317,7 +2317,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,114,167,0,0,0,90,13,105,110,105,116,95,102,105,108, 101,110,97,109,101,90,9,102,117,108,108,95,112,97,116,104, 114,166,0,0,0,114,2,0,0,0,114,2,0,0,0,114, - 4,0,0,0,114,182,0,0,0,73,5,0,0,115,70,0, + 4,0,0,0,114,182,0,0,0,74,5,0,0,115,70,0, 0,0,0,5,4,1,14,1,2,1,24,1,14,1,10,1, 10,1,8,1,6,2,6,1,6,1,10,2,6,1,4,2, 8,1,12,1,14,1,8,1,10,1,8,1,26,4,8,2, @@ -2348,7 +2348,7 @@ const unsigned char _Py_M__importlib_external[] = { 1,124,1,160,0,161,0,146,2,113,4,83,0,114,2,0, 0,0,41,1,114,90,0,0,0,41,2,114,22,0,0,0, 90,2,102,110,114,2,0,0,0,114,2,0,0,0,114,4, - 0,0,0,250,9,60,115,101,116,99,111,109,112,62,150,5, + 0,0,0,250,9,60,115,101,116,99,111,109,112,62,151,5, 0,0,115,2,0,0,0,6,0,122,41,70,105,108,101,70, 105,110,100,101,114,46,95,102,105,108,108,95,99,97,99,104, 101,46,60,108,111,99,97,108,115,62,46,60,115,101,116,99, @@ -2365,7 +2365,7 @@ const unsigned char _Py_M__importlib_external[] = { 101,110,116,115,114,1,1,0,0,114,100,0,0,0,114,248, 0,0,0,114,236,0,0,0,90,8,110,101,119,95,110,97, 109,101,114,2,0,0,0,114,2,0,0,0,114,4,0,0, - 0,114,25,1,0,0,121,5,0,0,115,34,0,0,0,0, + 0,114,25,1,0,0,122,5,0,0,115,34,0,0,0,0, 2,6,1,2,1,22,1,20,3,10,3,12,1,12,7,6, 1,8,1,16,1,4,1,18,2,4,1,12,1,6,1,12, 1,122,22,70,105,108,101,70,105,110,100,101,114,46,95,102, @@ -2403,7 +2403,7 @@ const unsigned char _Py_M__importlib_external[] = { 41,1,114,35,0,0,0,41,2,114,172,0,0,0,114,24, 1,0,0,114,2,0,0,0,114,4,0,0,0,218,24,112, 97,116,104,95,104,111,111,107,95,102,111,114,95,70,105,108, - 101,70,105,110,100,101,114,162,5,0,0,115,6,0,0,0, + 101,70,105,110,100,101,114,163,5,0,0,115,6,0,0,0, 0,2,8,1,12,1,122,54,70,105,108,101,70,105,110,100, 101,114,46,112,97,116,104,95,104,111,111,107,46,60,108,111, 99,97,108,115,62,46,112,97,116,104,95,104,111,111,107,95, @@ -2411,7 +2411,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,0,0,41,3,114,172,0,0,0,114,24,1,0,0,114, 30,1,0,0,114,2,0,0,0,41,2,114,172,0,0,0, 114,24,1,0,0,114,4,0,0,0,218,9,112,97,116,104, - 95,104,111,111,107,152,5,0,0,115,4,0,0,0,0,10, + 95,104,111,111,107,153,5,0,0,115,4,0,0,0,0,10, 14,6,122,20,70,105,108,101,70,105,110,100,101,114,46,112, 97,116,104,95,104,111,111,107,99,1,0,0,0,0,0,0, 0,1,0,0,0,3,0,0,0,67,0,0,0,115,12,0, @@ -2419,7 +2419,7 @@ const unsigned char _Py_M__importlib_external[] = { 78,122,16,70,105,108,101,70,105,110,100,101,114,40,123,33, 114,125,41,41,2,114,48,0,0,0,114,35,0,0,0,41, 1,114,102,0,0,0,114,2,0,0,0,114,2,0,0,0, - 114,4,0,0,0,114,0,1,0,0,170,5,0,0,115,2, + 114,4,0,0,0,114,0,1,0,0,171,5,0,0,115,2, 0,0,0,0,1,122,19,70,105,108,101,70,105,110,100,101, 114,46,95,95,114,101,112,114,95,95,41,1,78,41,15,114, 107,0,0,0,114,106,0,0,0,114,108,0,0,0,114,109, @@ -2428,7 +2428,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,114,182,0,0,0,114,25,1,0,0,114,184,0,0,0, 114,31,1,0,0,114,0,1,0,0,114,2,0,0,0,114, 2,0,0,0,114,2,0,0,0,114,4,0,0,0,114,18, - 1,0,0,27,5,0,0,115,18,0,0,0,12,9,8,14, + 1,0,0,28,5,0,0,115,18,0,0,0,12,9,8,14, 8,4,4,2,8,12,8,5,10,48,8,31,12,18,114,18, 1,0,0,99,4,0,0,0,0,0,0,0,6,0,0,0, 8,0,0,0,67,0,0,0,115,146,0,0,0,124,0,160, @@ -2451,7 +2451,7 @@ const unsigned char _Py_M__importlib_external[] = { 90,9,99,112,97,116,104,110,97,109,101,114,122,0,0,0, 114,166,0,0,0,114,2,0,0,0,114,2,0,0,0,114, 4,0,0,0,218,14,95,102,105,120,95,117,112,95,109,111, - 100,117,108,101,176,5,0,0,115,34,0,0,0,0,2,10, + 100,117,108,101,177,5,0,0,115,34,0,0,0,0,2,10, 1,10,1,4,1,4,1,8,1,8,1,12,2,10,1,4, 1,14,1,2,1,8,1,8,1,8,1,12,1,14,2,114, 36,1,0,0,99,0,0,0,0,0,0,0,0,3,0,0, @@ -2470,7 +2470,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,114,234,0,0,0,114,76,0,0,0,41,3,90,10,101, 120,116,101,110,115,105,111,110,115,90,6,115,111,117,114,99, 101,90,8,98,121,116,101,99,111,100,101,114,2,0,0,0, - 114,2,0,0,0,114,4,0,0,0,114,163,0,0,0,199, + 114,2,0,0,0,114,4,0,0,0,114,163,0,0,0,200, 5,0,0,115,8,0,0,0,0,5,12,1,8,1,8,1, 114,163,0,0,0,99,1,0,0,0,0,0,0,0,12,0, 0,0,9,0,0,0,67,0,0,0,115,156,1,0,0,124, @@ -2521,7 +2521,7 @@ const unsigned char _Py_M__importlib_external[] = { 107,2,86,0,1,0,113,2,100,1,83,0,41,2,114,29, 0,0,0,78,41,1,114,31,0,0,0,41,2,114,22,0, 0,0,114,79,0,0,0,114,2,0,0,0,114,2,0,0, - 0,114,4,0,0,0,114,238,0,0,0,235,5,0,0,115, + 0,114,4,0,0,0,114,238,0,0,0,236,5,0,0,115, 2,0,0,0,4,0,122,25,95,115,101,116,117,112,46,60, 108,111,99,97,108,115,62,46,60,103,101,110,101,120,112,114, 62,114,60,0,0,0,122,30,105,109,112,111,114,116,108,105, @@ -2549,7 +2549,7 @@ const unsigned char _Py_M__importlib_external[] = { 107,114,101,102,95,109,111,100,117,108,101,90,13,119,105,110, 114,101,103,95,109,111,100,117,108,101,114,2,0,0,0,114, 2,0,0,0,114,4,0,0,0,218,6,95,115,101,116,117, - 112,210,5,0,0,115,76,0,0,0,0,8,4,1,6,1, + 112,211,5,0,0,115,76,0,0,0,0,8,4,1,6,1, 6,3,10,1,8,1,10,1,12,2,10,1,14,3,22,1, 12,2,22,1,8,1,10,1,10,1,6,2,2,1,10,1, 10,1,14,1,12,2,8,1,12,1,12,1,18,3,10,1, @@ -2569,7 +2569,7 @@ const unsigned char _Py_M__importlib_external[] = { 2,114,42,1,0,0,90,17,115,117,112,112,111,114,116,101, 100,95,108,111,97,100,101,114,115,114,2,0,0,0,114,2, 0,0,0,114,4,0,0,0,218,8,95,105,110,115,116,97, - 108,108,18,6,0,0,115,8,0,0,0,0,2,8,1,6, + 108,108,19,6,0,0,115,8,0,0,0,0,2,8,1,6, 1,20,1,114,45,1,0,0,41,1,114,47,0,0,0,41, 1,78,41,3,78,78,78,41,2,114,60,0,0,0,114,60, 0,0,0,41,1,84,41,1,78,41,1,78,41,61,114,109, @@ -2601,7 +2601,7 @@ const unsigned char _Py_M__importlib_external[] = { 114,2,0,0,0,114,4,0,0,0,218,8,60,109,111,100, 117,108,101,62,23,0,0,0,115,118,0,0,0,4,0,4, 1,4,1,2,1,6,3,8,17,8,5,8,5,8,6,8, - 12,8,10,8,9,8,5,8,7,10,22,10,127,0,4,16, + 12,8,10,8,9,8,5,8,7,10,22,10,127,0,5,16, 1,12,2,4,1,4,2,6,2,6,2,8,2,16,45,8, 34,8,19,8,12,8,12,8,28,8,17,8,33,8,28,8, 24,10,13,10,10,10,11,8,14,6,3,4,1,14,67,14, diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h index c9c903487fd9..e82959be0864 100644 --- a/Python/opcode_targets.h +++ b/Python/opcode_targets.h @@ -53,7 +53,7 @@ static void *opcode_targets[256] = { &&TARGET_GET_ANEXT, &&TARGET_BEFORE_ASYNC_WITH, &&TARGET_BEGIN_FINALLY, - &&_unknown_opcode, + &&TARGET_END_ASYNC_FOR, &&TARGET_INPLACE_ADD, &&TARGET_INPLACE_SUBTRACT, &&TARGET_INPLACE_MULTIPLY, From webhook-mailer at python.org Fri Mar 23 08:35:36 2018 From: webhook-mailer at python.org (Serhiy Storchaka) Date: Fri, 23 Mar 2018 12:35:36 -0000 Subject: [Python-checkins] bpo-33041: Fixed jumping if the function contains an "async for" loop. (GH-6154) Message-ID: <mailman.141.1521808539.1871.python-checkins@python.org> https://github.com/python/cpython/commit/b9744e924ca07ba7db977e5958b91cd8db565632 commit: b9744e924ca07ba7db977e5958b91cd8db565632 branch: 3.7 author: Serhiy Storchaka <storchaka at gmail.com> committer: GitHub <noreply at github.com> date: 2018-03-23T14:35:33+02:00 summary: bpo-33041: Fixed jumping if the function contains an "async for" loop. (GH-6154) files: A Misc/NEWS.d/next/Core and Builtins/2018-03-18-13-56-14.bpo-33041.XwPhI2.rst M Lib/test/test_coroutines.py M Lib/test/test_sys_settrace.py M Python/compile.c diff --git a/Lib/test/test_coroutines.py b/Lib/test/test_coroutines.py index b37b61b71959..72e404da09be 100644 --- a/Lib/test/test_coroutines.py +++ b/Lib/test/test_coroutines.py @@ -1846,6 +1846,36 @@ def test_comp_4(self): run_async(run_gen()), ([], [121])) + def test_comp_4_2(self): + async def f(it): + for i in it: + yield i + + async def run_list(): + return [i + 10 async for i in f(range(5)) if 0 < i < 4] + self.assertEqual( + run_async(run_list()), + ([], [11, 12, 13])) + + async def run_set(): + return {i + 10 async for i in f(range(5)) if 0 < i < 4} + self.assertEqual( + run_async(run_set()), + ([], {11, 12, 13})) + + async def run_dict(): + return {i + 10: i + 100 async for i in f(range(5)) if 0 < i < 4} + self.assertEqual( + run_async(run_dict()), + ([], {11: 101, 12: 102, 13: 103})) + + async def run_gen(): + gen = (i + 10 async for i in f(range(5)) if 0 < i < 4) + return [g + 100 async for g in gen] + self.assertEqual( + run_async(run_gen()), + ([], [111, 112, 113])) + def test_comp_5(self): async def f(it): for i in it: diff --git a/Lib/test/test_sys_settrace.py b/Lib/test/test_sys_settrace.py index bf8c722d0cf7..2cf55eb97e4b 100644 --- a/Lib/test/test_sys_settrace.py +++ b/Lib/test/test_sys_settrace.py @@ -33,6 +33,10 @@ def __init__(self, output, value): async def __aexit__(self, *exc_info): self.output.append(-self.value) +async def asynciter(iterable): + """Convert an iterable to an asynchronous iterator.""" + for x in iterable: + yield x # A very basic example. If this fails, we're in deep trouble. @@ -721,6 +725,23 @@ def test_jump_out_of_block_backwards(output): output.append(6) output.append(7) + @async_jump_test(4, 5, [3, 5]) + async def test_jump_out_of_async_for_block_forwards(output): + for i in [1]: + async for i in asynciter([1, 2]): + output.append(3) + output.append(4) + output.append(5) + + @async_jump_test(5, 2, [2, 4, 2, 4, 5, 6]) + async def test_jump_out_of_async_for_block_backwards(output): + for i in [1]: + output.append(2) + async for i in asynciter([1]): + output.append(4) + output.append(5) + output.append(6) + @jump_test(1, 2, [3]) def test_jump_to_codeless_line(output): output.append(1) @@ -1000,6 +1021,17 @@ def test_jump_over_for_block_before_else(output): output.append(7) output.append(8) + @async_jump_test(1, 7, [7, 8]) + async def test_jump_over_async_for_block_before_else(output): + output.append(1) + if not output: # always false + async for i in asynciter([3]): + output.append(4) + else: + output.append(6) + output.append(7) + output.append(8) + # The second set of 'jump' tests are for things that are not allowed: @jump_test(2, 3, [1], (ValueError, 'after')) @@ -1051,12 +1083,24 @@ def test_no_jump_forwards_into_for_block(output): for i in 1, 2: output.append(3) + @async_jump_test(1, 3, [], (ValueError, 'into')) + async def test_no_jump_forwards_into_async_for_block(output): + output.append(1) + async for i in asynciter([1, 2]): + output.append(3) + @jump_test(3, 2, [2, 2], (ValueError, 'into')) def test_no_jump_backwards_into_for_block(output): for i in 1, 2: output.append(2) output.append(3) + @async_jump_test(3, 2, [2, 2], (ValueError, 'into')) + async def test_no_jump_backwards_into_async_for_block(output): + async for i in asynciter([1, 2]): + output.append(2) + output.append(3) + @jump_test(2, 4, [], (ValueError, 'into')) def test_no_jump_forwards_into_while_block(output): i = 1 @@ -1196,6 +1240,17 @@ def test_no_jump_into_for_block_before_else(output): output.append(7) output.append(8) + @async_jump_test(7, 4, [1, 6], (ValueError, 'into')) + async def test_no_jump_into_async_for_block_before_else(output): + output.append(1) + if not output: # always false + async for i in asynciter([3]): + output.append(4) + else: + output.append(6) + output.append(7) + output.append(8) + def test_no_jump_to_non_integers(self): self.run_test(no_jump_to_non_integers, 2, "Spam", [True]) diff --git a/Misc/NEWS.d/next/Core and Builtins/2018-03-18-13-56-14.bpo-33041.XwPhI2.rst b/Misc/NEWS.d/next/Core and Builtins/2018-03-18-13-56-14.bpo-33041.XwPhI2.rst new file mode 100644 index 000000000000..97b5e2ef1e54 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2018-03-18-13-56-14.bpo-33041.XwPhI2.rst @@ -0,0 +1 @@ +Fixed jumping when the function contains an ``async for`` loop. diff --git a/Python/compile.c b/Python/compile.c index 8ab33613b524..618f31a47d36 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -2383,24 +2383,19 @@ compiler_async_for(struct compiler *c, stmt_ty s) ADDOP(c, DUP_TOP); ADDOP_O(c, LOAD_GLOBAL, stop_aiter_error, names); ADDOP_I(c, COMPARE_OP, PyCmp_EXC_MATCH); - ADDOP_JABS(c, POP_JUMP_IF_FALSE, try_cleanup); - - ADDOP(c, POP_TOP); - ADDOP(c, POP_TOP); - ADDOP(c, POP_TOP); - ADDOP(c, POP_EXCEPT); /* for SETUP_EXCEPT */ - ADDOP(c, POP_TOP); /* for correct calculation of stack effect */ - ADDOP(c, POP_BLOCK); /* for SETUP_LOOP */ - ADDOP_JABS(c, JUMP_ABSOLUTE, after_loop_else); - - - compiler_use_next_block(c, try_cleanup); + ADDOP_JABS(c, POP_JUMP_IF_TRUE, try_cleanup); ADDOP(c, END_FINALLY); compiler_use_next_block(c, after_try); VISIT_SEQ(c, stmt, s->v.AsyncFor.body); ADDOP_JABS(c, JUMP_ABSOLUTE, try); + compiler_use_next_block(c, try_cleanup); + ADDOP(c, POP_TOP); + ADDOP(c, POP_TOP); + ADDOP(c, POP_TOP); + ADDOP(c, POP_EXCEPT); /* for SETUP_EXCEPT */ + ADDOP(c, POP_TOP); /* for correct calculation of stack effect */ ADDOP(c, POP_BLOCK); /* for SETUP_LOOP */ compiler_pop_fblock(c, LOOP, try); @@ -3890,7 +3885,7 @@ compiler_async_comprehension_generator(struct compiler *c, _Py_IDENTIFIER(StopAsyncIteration); comprehension_ty gen; - basicblock *anchor, *if_cleanup, *try, + basicblock *if_cleanup, *try, *after_try, *except, *try_cleanup; Py_ssize_t i, n; @@ -3901,12 +3896,11 @@ compiler_async_comprehension_generator(struct compiler *c, try = compiler_new_block(c); after_try = compiler_new_block(c); - try_cleanup = compiler_new_block(c); except = compiler_new_block(c); if_cleanup = compiler_new_block(c); - anchor = compiler_new_block(c); + try_cleanup = compiler_new_block(c); - if (if_cleanup == NULL || anchor == NULL || + if (if_cleanup == NULL || try == NULL || after_try == NULL || except == NULL || try_cleanup == NULL) { return 0; @@ -3945,16 +3939,7 @@ compiler_async_comprehension_generator(struct compiler *c, ADDOP(c, DUP_TOP); ADDOP_O(c, LOAD_GLOBAL, stop_aiter_error, names); ADDOP_I(c, COMPARE_OP, PyCmp_EXC_MATCH); - ADDOP_JABS(c, POP_JUMP_IF_FALSE, try_cleanup); - - ADDOP(c, POP_TOP); - ADDOP(c, POP_TOP); - ADDOP(c, POP_TOP); - ADDOP(c, POP_EXCEPT); /* for SETUP_EXCEPT */ - ADDOP_JABS(c, JUMP_ABSOLUTE, anchor); - - - compiler_use_next_block(c, try_cleanup); + ADDOP_JABS(c, POP_JUMP_IF_TRUE, try_cleanup); ADDOP(c, END_FINALLY); compiler_use_next_block(c, after_try); @@ -4003,7 +3988,12 @@ compiler_async_comprehension_generator(struct compiler *c, } compiler_use_next_block(c, if_cleanup); ADDOP_JABS(c, JUMP_ABSOLUTE, try); - compiler_use_next_block(c, anchor); + + compiler_use_next_block(c, try_cleanup); + ADDOP(c, POP_TOP); + ADDOP(c, POP_TOP); + ADDOP(c, POP_TOP); + ADDOP(c, POP_EXCEPT); /* for SETUP_EXCEPT */ ADDOP(c, POP_TOP); return 1; From webhook-mailer at python.org Fri Mar 23 08:46:47 2018 From: webhook-mailer at python.org (Serhiy Storchaka) Date: Fri, 23 Mar 2018 12:46:47 -0000 Subject: [Python-checkins] bpo-30953: Improve error messages and add tests for jumping (GH-6196) Message-ID: <mailman.142.1521809208.1871.python-checkins@python.org> https://github.com/python/cpython/commit/397466dfd905b5132f1c831cd9dff3ecc40b3218 commit: 397466dfd905b5132f1c831cd9dff3ecc40b3218 branch: master author: Serhiy Storchaka <storchaka at gmail.com> committer: GitHub <noreply at github.com> date: 2018-03-23T14:46:45+02:00 summary: bpo-30953: Improve error messages and add tests for jumping (GH-6196) into/out of an except block. files: M Lib/test/test_sys_settrace.py M Objects/frameobject.c diff --git a/Lib/test/test_sys_settrace.py b/Lib/test/test_sys_settrace.py index 1fa43b29ec90..f5125a450511 100644 --- a/Lib/test/test_sys_settrace.py +++ b/Lib/test/test_sys_settrace.py @@ -1201,8 +1201,16 @@ def test_no_jump_between_except_blocks_2(output): output.append(7) output.append(8) - @jump_test(3, 6, [2, 5, 6], (ValueError, 'finally')) + @jump_test(1, 5, [], (ValueError, "into a 'finally'")) def test_no_jump_into_finally_block(output): + output.append(1) + try: + output.append(3) + finally: + output.append(5) + + @jump_test(3, 6, [2, 5, 6], (ValueError, "into a 'finally'")) + def test_no_jump_into_finally_block_from_try_block(output): try: output.append(2) output.append(3) @@ -1211,21 +1219,71 @@ def test_no_jump_into_finally_block(output): output.append(6) output.append(7) - @jump_test(1, 5, [], (ValueError, 'finally')) - def test_no_jump_into_finally_block_2(output): + @jump_test(5, 1, [1, 3], (ValueError, "out of a 'finally'")) + def test_no_jump_out_of_finally_block(output): output.append(1) try: output.append(3) finally: output.append(5) - @jump_test(5, 1, [1, 3], (ValueError, 'finally')) - def test_no_jump_out_of_finally_block(output): + @jump_test(1, 5, [], (ValueError, "into an 'except'")) + def test_no_jump_into_bare_except_block(output): output.append(1) try: output.append(3) - finally: + except: + output.append(5) + + @jump_test(1, 5, [], (ValueError, "into an 'except'")) + def test_no_jump_into_qualified_except_block(output): + output.append(1) + try: + output.append(3) + except Exception: + output.append(5) + + @jump_test(3, 6, [2, 5, 6], (ValueError, "into an 'except'")) + def test_no_jump_into_bare_except_block_from_try_block(output): + try: + output.append(2) + output.append(3) + except: # executed if the jump is failed output.append(5) + output.append(6) + raise + output.append(8) + + @jump_test(3, 6, [2], (ValueError, "into an 'except'")) + def test_no_jump_into_qualified_except_block_from_try_block(output): + try: + output.append(2) + output.append(3) + except ZeroDivisionError: + output.append(5) + output.append(6) + raise + output.append(8) + + @jump_test(7, 1, [1, 3, 6], (ValueError, "out of an 'except'")) + def test_no_jump_out_of_bare_except_block(output): + output.append(1) + try: + output.append(3) + 1/0 + except: + output.append(6) + output.append(7) + + @jump_test(7, 1, [1, 3, 6], (ValueError, "out of an 'except'")) + def test_no_jump_out_of_qualified_except_block(output): + output.append(1) + try: + output.append(3) + 1/0 + except Exception: + output.append(6) + output.append(7) @jump_test(3, 5, [1, 2, -2], (ValueError, 'into')) def test_no_jump_between_with_blocks(output): diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 9d37935c2f79..864a8f977ea3 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -277,8 +277,12 @@ frame_setlineno(PyFrameObject *f, PyObject* p_new_lineno) int first_in = target_addr <= f->f_lasti && f->f_lasti <= addr; int second_in = target_addr <= new_lasti && new_lasti <= addr; if (first_in != second_in) { - PyErr_SetString(PyExc_ValueError, - "can't jump into or out of a 'finally' block"); + op = code[target_addr]; + PyErr_Format(PyExc_ValueError, + "can't jump %s %s block", + second_in ? "into" : "out of", + (op == DUP_TOP || op == POP_TOP) ? + "an 'except'" : "a 'finally'"); return -1; } break; From webhook-mailer at python.org Fri Mar 23 09:45:40 2018 From: webhook-mailer at python.org (Serhiy Storchaka) Date: Fri, 23 Mar 2018 13:45:40 -0000 Subject: [Python-checkins] [3.6] bpo-33041: Fixed jumping if the function contains an "async for" loop. (GH-6154). (GH-6199) Message-ID: <mailman.144.1521812743.1871.python-checkins@python.org> https://github.com/python/cpython/commit/18d7edf32e6832a818621cb8cb3d144928eca232 commit: 18d7edf32e6832a818621cb8cb3d144928eca232 branch: 3.6 author: Serhiy Storchaka <storchaka at gmail.com> committer: GitHub <noreply at github.com> date: 2018-03-23T15:45:37+02:00 summary: [3.6] bpo-33041: Fixed jumping if the function contains an "async for" loop. (GH-6154). (GH-6199) (cherry picked from commit b9744e924ca07ba7db977e5958b91cd8db565632) files: A Misc/NEWS.d/next/Core and Builtins/2018-03-18-13-56-14.bpo-33041.XwPhI2.rst M Lib/test/test_coroutines.py M Lib/test/test_dis.py M Lib/test/test_sys_settrace.py M Python/compile.c diff --git a/Lib/test/test_coroutines.py b/Lib/test/test_coroutines.py index 2b79a17ea703..41126b675b5c 100644 --- a/Lib/test/test_coroutines.py +++ b/Lib/test/test_coroutines.py @@ -1892,6 +1892,36 @@ def test_comp_4(self): run_async(run_gen()), ([], [121])) + def test_comp_4_2(self): + async def f(it): + for i in it: + yield i + + async def run_list(): + return [i + 10 async for i in f(range(5)) if 0 < i < 4] + self.assertEqual( + run_async(run_list()), + ([], [11, 12, 13])) + + async def run_set(): + return {i + 10 async for i in f(range(5)) if 0 < i < 4} + self.assertEqual( + run_async(run_set()), + ([], {11, 12, 13})) + + async def run_dict(): + return {i + 10: i + 100 async for i in f(range(5)) if 0 < i < 4} + self.assertEqual( + run_async(run_dict()), + ([], {11: 101, 12: 102, 13: 103})) + + async def run_gen(): + gen = (i + 10 async for i in f(range(5)) if 0 < i < 4) + return [g + 100 async for g in gen] + self.assertEqual( + run_async(run_gen()), + ([], [111, 112, 113])) + def test_comp_5(self): async def f(it): for i in it: diff --git a/Lib/test/test_dis.py b/Lib/test/test_dis.py index b9b5ac2667ee..1cee3686893f 100644 --- a/Lib/test/test_dis.py +++ b/Lib/test/test_dis.py @@ -597,7 +597,7 @@ def f(c=c): Argument count: 0 Kw-only arguments: 0 Number of locals: 2 -Stack size: 17 +Stack size: 16 Flags: OPTIMIZED, NEWLOCALS, NOFREE, COROUTINE Constants: 0: None diff --git a/Lib/test/test_sys_settrace.py b/Lib/test/test_sys_settrace.py index 659790efe7cc..efee5d6dc35a 100644 --- a/Lib/test/test_sys_settrace.py +++ b/Lib/test/test_sys_settrace.py @@ -32,6 +32,11 @@ def __init__(self, output, value): async def __aexit__(self, *exc_info): self.output.append(-self.value) +async def asynciter(iterable): + """Convert an iterable to an asynchronous iterator.""" + for x in iterable: + yield x + def asyncio_run(main): import asyncio import asyncio.events @@ -690,6 +695,23 @@ def test_jump_out_of_block_backwards(output): output.append(6) output.append(7) + @async_jump_test(4, 5, [3, 5]) + async def test_jump_out_of_async_for_block_forwards(output): + for i in [1]: + async for i in asynciter([1, 2]): + output.append(3) + output.append(4) + output.append(5) + + @async_jump_test(5, 2, [2, 4, 2, 4, 5, 6]) + async def test_jump_out_of_async_for_block_backwards(output): + for i in [1]: + output.append(2) + async for i in asynciter([1]): + output.append(4) + output.append(5) + output.append(6) + @jump_test(1, 2, [3]) def test_jump_to_codeless_line(output): output.append(1) @@ -969,6 +991,17 @@ def test_jump_over_for_block_before_else(output): output.append(7) output.append(8) + @async_jump_test(1, 7, [7, 8]) + async def test_jump_over_async_for_block_before_else(output): + output.append(1) + if not output: # always false + async for i in asynciter([3]): + output.append(4) + else: + output.append(6) + output.append(7) + output.append(8) + # The second set of 'jump' tests are for things that are not allowed: @jump_test(2, 3, [1], (ValueError, 'after')) @@ -1020,12 +1053,24 @@ def test_no_jump_forwards_into_for_block(output): for i in 1, 2: output.append(3) + @async_jump_test(1, 3, [], (ValueError, 'into')) + async def test_no_jump_forwards_into_async_for_block(output): + output.append(1) + async for i in asynciter([1, 2]): + output.append(3) + @jump_test(3, 2, [2, 2], (ValueError, 'into')) def test_no_jump_backwards_into_for_block(output): for i in 1, 2: output.append(2) output.append(3) + @async_jump_test(3, 2, [2, 2], (ValueError, 'into')) + async def test_no_jump_backwards_into_async_for_block(output): + async for i in asynciter([1, 2]): + output.append(2) + output.append(3) + @jump_test(2, 4, [], (ValueError, 'into')) def test_no_jump_forwards_into_while_block(output): i = 1 @@ -1165,6 +1210,17 @@ def test_no_jump_into_for_block_before_else(output): output.append(7) output.append(8) + @async_jump_test(7, 4, [1, 6], (ValueError, 'into')) + async def test_no_jump_into_async_for_block_before_else(output): + output.append(1) + if not output: # always false + async for i in asynciter([3]): + output.append(4) + else: + output.append(6) + output.append(7) + output.append(8) + def test_no_jump_to_non_integers(self): self.run_test(no_jump_to_non_integers, 2, "Spam", [True]) diff --git a/Misc/NEWS.d/next/Core and Builtins/2018-03-18-13-56-14.bpo-33041.XwPhI2.rst b/Misc/NEWS.d/next/Core and Builtins/2018-03-18-13-56-14.bpo-33041.XwPhI2.rst new file mode 100644 index 000000000000..97b5e2ef1e54 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2018-03-18-13-56-14.bpo-33041.XwPhI2.rst @@ -0,0 +1 @@ +Fixed jumping when the function contains an ``async for`` loop. diff --git a/Python/compile.c b/Python/compile.c index aae3300febed..ae0e574276db 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -2237,23 +2237,19 @@ compiler_async_for(struct compiler *c, stmt_ty s) ADDOP(c, DUP_TOP); ADDOP_O(c, LOAD_GLOBAL, stop_aiter_error, names); ADDOP_I(c, COMPARE_OP, PyCmp_EXC_MATCH); - ADDOP_JABS(c, POP_JUMP_IF_FALSE, try_cleanup); - - ADDOP(c, POP_TOP); - ADDOP(c, POP_TOP); - ADDOP(c, POP_TOP); - ADDOP(c, POP_EXCEPT); /* for SETUP_EXCEPT */ - ADDOP(c, POP_BLOCK); /* for SETUP_LOOP */ - ADDOP_JABS(c, JUMP_ABSOLUTE, after_loop_else); - - - compiler_use_next_block(c, try_cleanup); + ADDOP_JABS(c, POP_JUMP_IF_TRUE, try_cleanup); ADDOP(c, END_FINALLY); compiler_use_next_block(c, after_try); VISIT_SEQ(c, stmt, s->v.AsyncFor.body); ADDOP_JABS(c, JUMP_ABSOLUTE, try); + compiler_use_next_block(c, try_cleanup); + ADDOP(c, POP_TOP); + ADDOP(c, POP_TOP); + ADDOP(c, POP_TOP); + ADDOP(c, POP_EXCEPT); /* for SETUP_EXCEPT */ + ADDOP(c, POP_TOP); /* for correct calculation of stack effect */ ADDOP(c, POP_BLOCK); /* for SETUP_LOOP */ compiler_pop_fblock(c, LOOP, try); @@ -3750,7 +3746,7 @@ compiler_async_comprehension_generator(struct compiler *c, _Py_IDENTIFIER(StopAsyncIteration); comprehension_ty gen; - basicblock *anchor, *if_cleanup, *try, + basicblock *if_cleanup, *try, *after_try, *except, *try_cleanup; Py_ssize_t i, n; @@ -3761,12 +3757,11 @@ compiler_async_comprehension_generator(struct compiler *c, try = compiler_new_block(c); after_try = compiler_new_block(c); - try_cleanup = compiler_new_block(c); except = compiler_new_block(c); if_cleanup = compiler_new_block(c); - anchor = compiler_new_block(c); + try_cleanup = compiler_new_block(c); - if (if_cleanup == NULL || anchor == NULL || + if (if_cleanup == NULL || try == NULL || after_try == NULL || except == NULL || try_cleanup == NULL) { return 0; @@ -3807,16 +3802,7 @@ compiler_async_comprehension_generator(struct compiler *c, ADDOP(c, DUP_TOP); ADDOP_O(c, LOAD_GLOBAL, stop_aiter_error, names); ADDOP_I(c, COMPARE_OP, PyCmp_EXC_MATCH); - ADDOP_JABS(c, POP_JUMP_IF_FALSE, try_cleanup); - - ADDOP(c, POP_TOP); - ADDOP(c, POP_TOP); - ADDOP(c, POP_TOP); - ADDOP(c, POP_EXCEPT); /* for SETUP_EXCEPT */ - ADDOP_JABS(c, JUMP_ABSOLUTE, anchor); - - - compiler_use_next_block(c, try_cleanup); + ADDOP_JABS(c, POP_JUMP_IF_TRUE, try_cleanup); ADDOP(c, END_FINALLY); compiler_use_next_block(c, after_try); @@ -3865,7 +3851,12 @@ compiler_async_comprehension_generator(struct compiler *c, } compiler_use_next_block(c, if_cleanup); ADDOP_JABS(c, JUMP_ABSOLUTE, try); - compiler_use_next_block(c, anchor); + + compiler_use_next_block(c, try_cleanup); + ADDOP(c, POP_TOP); + ADDOP(c, POP_TOP); + ADDOP(c, POP_TOP); + ADDOP(c, POP_EXCEPT); /* for SETUP_EXCEPT */ ADDOP(c, POP_TOP); return 1; From webhook-mailer at python.org Fri Mar 23 09:46:55 2018 From: webhook-mailer at python.org (Berker Peksag) Date: Fri, 23 Mar 2018 13:46:55 -0000 Subject: [Python-checkins] Fix a reference to the MRE book in re docs (GH-1113) Message-ID: <mailman.145.1521812817.1871.python-checkins@python.org> https://github.com/python/cpython/commit/a0a42d22d8dff0ec6ea9daa4d9c9e9399f9b4e6c commit: a0a42d22d8dff0ec6ea9daa4d9c9e9399f9b4e6c branch: master author: Berker Peksag <berker.peksag at gmail.com> committer: GitHub <noreply at github.com> date: 2018-03-23T16:46:52+03:00 summary: Fix a reference to the MRE book in re docs (GH-1113) Reported by Maksym Nikulyak on docs.p.o. files: M Doc/library/re.rst diff --git a/Doc/library/re.rst b/Doc/library/re.rst index 475a8d285550..ddb74f8d386c 100644 --- a/Doc/library/re.rst +++ b/Doc/library/re.rst @@ -67,8 +67,8 @@ string *pq* will match AB. This holds unless *A* or *B* contain low precedence operations; boundary conditions between *A* and *B*; or have numbered group references. Thus, complex expressions can easily be constructed from simpler primitive expressions like the ones described here. For details of the theory -and implementation of regular expressions, consult the Friedl book referenced -above, or almost any textbook about compiler construction. +and implementation of regular expressions, consult the Friedl book [Frie09]_, +or almost any textbook about compiler construction. A brief explanation of the format of regular expressions follows. For further information and a gentler presentation, consult the :ref:`regex-howto`. @@ -492,14 +492,6 @@ three digits in length. The ``'\N{name}'`` escape sequence has been added. As in string literals, it expands to the named Unicode character (e.g. ``'\N{EM DASH}'``). -.. seealso:: - - Mastering Regular Expressions - Book on regular expressions by Jeffrey Friedl, published by O'Reilly. The - second edition of the book no longer covers Python at all, but the first - edition covered writing good regular expression patterns in great detail. - - .. _contents-of-module-re: @@ -1585,3 +1577,9 @@ The tokenizer produces the following output:: Token(typ='END', value=';', line=4, column=27) Token(typ='ENDIF', value='ENDIF', line=5, column=4) Token(typ='END', value=';', line=5, column=9) + + +.. [Frie09] Friedl, Jeffrey. Mastering Regular Expressions. 3rd ed., O'Reilly + Media, 2009. The third edition of the book no longer covers Python at all, + but the first edition covered writing good regular expression patterns in + great detail. From webhook-mailer at python.org Fri Mar 23 11:55:29 2018 From: webhook-mailer at python.org (Berker Peksag) Date: Fri, 23 Mar 2018 15:55:29 -0000 Subject: [Python-checkins] Fix a reference to the MRE book in re docs (GH-1113) Message-ID: <mailman.147.1521820530.1871.python-checkins@python.org> https://github.com/python/cpython/commit/67d3f8b85a5bc5e322f3a7d209ed3a800985c122 commit: 67d3f8b85a5bc5e322f3a7d209ed3a800985c122 branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Berker Peksag <berker.peksag at gmail.com> date: 2018-03-23T18:55:26+03:00 summary: Fix a reference to the MRE book in re docs (GH-1113) Reported by Maksym Nikulyak on docs.p.o. (cherry picked from commit a0a42d22d8dff0ec6ea9daa4d9c9e9399f9b4e6c) Co-authored-by: Berker Peksag <berker.peksag at gmail.com> files: M Doc/library/re.rst diff --git a/Doc/library/re.rst b/Doc/library/re.rst index 83ebe7db01ad..d35aaf42f4ab 100644 --- a/Doc/library/re.rst +++ b/Doc/library/re.rst @@ -67,8 +67,8 @@ string *pq* will match AB. This holds unless *A* or *B* contain low precedence operations; boundary conditions between *A* and *B*; or have numbered group references. Thus, complex expressions can easily be constructed from simpler primitive expressions like the ones described here. For details of the theory -and implementation of regular expressions, consult the Friedl book referenced -above, or almost any textbook about compiler construction. +and implementation of regular expressions, consult the Friedl book [Frie09]_, +or almost any textbook about compiler construction. A brief explanation of the format of regular expressions follows. For further information and a gentler presentation, consult the :ref:`regex-howto`. @@ -489,14 +489,6 @@ three digits in length. Unknown escapes consisting of ``'\'`` and an ASCII letter now are errors. -.. seealso:: - - Mastering Regular Expressions - Book on regular expressions by Jeffrey Friedl, published by O'Reilly. The - second edition of the book no longer covers Python at all, but the first - edition covered writing good regular expression patterns in great detail. - - .. _contents-of-module-re: @@ -1582,3 +1574,9 @@ The tokenizer produces the following output:: Token(typ='END', value=';', line=4, column=27) Token(typ='ENDIF', value='ENDIF', line=5, column=4) Token(typ='END', value=';', line=5, column=9) + + +.. [Frie09] Friedl, Jeffrey. Mastering Regular Expressions. 3rd ed., O'Reilly + Media, 2009. The third edition of the book no longer covers Python at all, + but the first edition covered writing good regular expression patterns in + great detail. From webhook-mailer at python.org Fri Mar 23 11:55:49 2018 From: webhook-mailer at python.org (Berker Peksag) Date: Fri, 23 Mar 2018 15:55:49 -0000 Subject: [Python-checkins] Fix a reference to the MRE book in re docs (GH-1113) Message-ID: <mailman.148.1521820550.1871.python-checkins@python.org> https://github.com/python/cpython/commit/fce39ab1c8c0a8916a38290711fd921cb3c8eebe commit: fce39ab1c8c0a8916a38290711fd921cb3c8eebe branch: 3.6 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Berker Peksag <berker.peksag at gmail.com> date: 2018-03-23T18:55:46+03:00 summary: Fix a reference to the MRE book in re docs (GH-1113) Reported by Maksym Nikulyak on docs.p.o. (cherry picked from commit a0a42d22d8dff0ec6ea9daa4d9c9e9399f9b4e6c) Co-authored-by: Berker Peksag <berker.peksag at gmail.com> files: M Doc/library/re.rst diff --git a/Doc/library/re.rst b/Doc/library/re.rst index db92c4808554..64d84e10d160 100644 --- a/Doc/library/re.rst +++ b/Doc/library/re.rst @@ -67,8 +67,8 @@ string *pq* will match AB. This holds unless *A* or *B* contain low precedence operations; boundary conditions between *A* and *B*; or have numbered group references. Thus, complex expressions can easily be constructed from simpler primitive expressions like the ones described here. For details of the theory -and implementation of regular expressions, consult the Friedl book referenced -above, or almost any textbook about compiler construction. +and implementation of regular expressions, consult the Friedl book [Frie09]_, +or almost any textbook about compiler construction. A brief explanation of the format of regular expressions follows. For further information and a gentler presentation, consult the :ref:`regex-howto`. @@ -471,14 +471,6 @@ three digits in length. Unknown escapes consisting of ``'\'`` and an ASCII letter now are errors. -.. seealso:: - - Mastering Regular Expressions - Book on regular expressions by Jeffrey Friedl, published by O'Reilly. The - second edition of the book no longer covers Python at all, but the first - edition covered writing good regular expression patterns in great detail. - - .. _contents-of-module-re: @@ -1561,3 +1553,9 @@ The tokenizer produces the following output:: Token(typ='END', value=';', line=4, column=27) Token(typ='ENDIF', value='ENDIF', line=5, column=4) Token(typ='END', value=';', line=5, column=9) + + +.. [Frie09] Friedl, Jeffrey. Mastering Regular Expressions. 3rd ed., O'Reilly + Media, 2009. The third edition of the book no longer covers Python at all, + but the first edition covered writing good regular expression patterns in + great detail. From webhook-mailer at python.org Fri Mar 23 12:40:41 2018 From: webhook-mailer at python.org (Julien Palard) Date: Fri, 23 Mar 2018 16:40:41 -0000 Subject: [Python-checkins] bpo-31639: Use threads in http.server module. (GH-5018) Message-ID: <mailman.149.1521823243.1871.python-checkins@python.org> https://github.com/python/cpython/commit/8bcfa02e4b1b65634e526e197588bc600674c80b commit: 8bcfa02e4b1b65634e526e197588bc600674c80b branch: master author: Julien Palard <julien at palard.fr> committer: GitHub <noreply at github.com> date: 2018-03-23T17:40:33+01:00 summary: bpo-31639: Use threads in http.server module. (GH-5018) files: A Misc/NEWS.d/next/Library/2017-12-27-21-55-19.bpo-31639.l3avDJ.rst M Doc/library/http.server.rst M Lib/http/server.py diff --git a/Doc/library/http.server.rst b/Doc/library/http.server.rst index c98843de02cb..4fe46cba691f 100644 --- a/Doc/library/http.server.rst +++ b/Doc/library/http.server.rst @@ -33,9 +33,16 @@ handler. Code to create and run the server looks like this:: :attr:`server_port`. The server is accessible by the handler, typically through the handler's :attr:`server` instance variable. +.. class:: ThreadedHTTPServer(server_address, RequestHandlerClass) -The :class:`HTTPServer` must be given a *RequestHandlerClass* on instantiation, -of which this module provides three different variants: + This class is identical to HTTPServer but uses threads to handle + requests by using the :class:`~socketserver.ThreadingMixin`. This + is usefull to handle web browsers pre-opening sockets, on which + :class:`HTTPServer` would wait indefinitly. + +The :class:`HTTPServer` and :class:`ThreadedHTTPServer` must be given +a *RequestHandlerClass* on instantiation, of which this module +provides three different variants: .. class:: BaseHTTPRequestHandler(request, client_address, server) diff --git a/Lib/http/server.py b/Lib/http/server.py index 502bce0c7a40..a2726ab89750 100644 --- a/Lib/http/server.py +++ b/Lib/http/server.py @@ -83,7 +83,7 @@ __version__ = "0.6" __all__ = [ - "HTTPServer", "BaseHTTPRequestHandler", + "HTTPServer", "ThreadedHTTPServer", "BaseHTTPRequestHandler", "SimpleHTTPRequestHandler", "CGIHTTPRequestHandler", ] @@ -140,6 +140,10 @@ def server_bind(self): self.server_port = port +class ThreadedHTTPServer(socketserver.ThreadingMixIn, HTTPServer): + daemon_threads = True + + class BaseHTTPRequestHandler(socketserver.StreamRequestHandler): """HTTP request handler base class. @@ -1213,7 +1217,8 @@ def run_cgi(self): def test(HandlerClass=BaseHTTPRequestHandler, - ServerClass=HTTPServer, protocol="HTTP/1.0", port=8000, bind=""): + ServerClass=ThreadedHTTPServer, + protocol="HTTP/1.0", port=8000, bind=""): """Test the HTTP request handler class. This runs an HTTP server on port 8000 (or the port argument). diff --git a/Misc/NEWS.d/next/Library/2017-12-27-21-55-19.bpo-31639.l3avDJ.rst b/Misc/NEWS.d/next/Library/2017-12-27-21-55-19.bpo-31639.l3avDJ.rst new file mode 100644 index 000000000000..e876f40813de --- /dev/null +++ b/Misc/NEWS.d/next/Library/2017-12-27-21-55-19.bpo-31639.l3avDJ.rst @@ -0,0 +1,2 @@ +http.server now exposes a ThreadedHTTPServer class and uses it when the +module is invoked to cope with web browsers pre-opening sockets. From webhook-mailer at python.org Fri Mar 23 16:31:29 2018 From: webhook-mailer at python.org (Miss Islington (bot)) Date: Fri, 23 Mar 2018 20:31:29 -0000 Subject: [Python-checkins] bpo-31639: Use threads in http.server module. (GH-5018) Message-ID: <mailman.150.1521837091.1871.python-checkins@python.org> https://github.com/python/cpython/commit/f8d2c3cf5f62f0c259b2bf35c631353d22cf1d08 commit: f8d2c3cf5f62f0c259b2bf35c631353d22cf1d08 branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: GitHub <noreply at github.com> date: 2018-03-23T13:31:20-07:00 summary: bpo-31639: Use threads in http.server module. (GH-5018) (cherry picked from commit 8bcfa02e4b1b65634e526e197588bc600674c80b) Co-authored-by: Julien Palard <julien at palard.fr> files: A Misc/NEWS.d/next/Library/2017-12-27-21-55-19.bpo-31639.l3avDJ.rst M Doc/library/http.server.rst M Lib/http/server.py diff --git a/Doc/library/http.server.rst b/Doc/library/http.server.rst index c98843de02cb..4fe46cba691f 100644 --- a/Doc/library/http.server.rst +++ b/Doc/library/http.server.rst @@ -33,9 +33,16 @@ handler. Code to create and run the server looks like this:: :attr:`server_port`. The server is accessible by the handler, typically through the handler's :attr:`server` instance variable. +.. class:: ThreadedHTTPServer(server_address, RequestHandlerClass) -The :class:`HTTPServer` must be given a *RequestHandlerClass* on instantiation, -of which this module provides three different variants: + This class is identical to HTTPServer but uses threads to handle + requests by using the :class:`~socketserver.ThreadingMixin`. This + is usefull to handle web browsers pre-opening sockets, on which + :class:`HTTPServer` would wait indefinitly. + +The :class:`HTTPServer` and :class:`ThreadedHTTPServer` must be given +a *RequestHandlerClass* on instantiation, of which this module +provides three different variants: .. class:: BaseHTTPRequestHandler(request, client_address, server) diff --git a/Lib/http/server.py b/Lib/http/server.py index 502bce0c7a40..a2726ab89750 100644 --- a/Lib/http/server.py +++ b/Lib/http/server.py @@ -83,7 +83,7 @@ __version__ = "0.6" __all__ = [ - "HTTPServer", "BaseHTTPRequestHandler", + "HTTPServer", "ThreadedHTTPServer", "BaseHTTPRequestHandler", "SimpleHTTPRequestHandler", "CGIHTTPRequestHandler", ] @@ -140,6 +140,10 @@ def server_bind(self): self.server_port = port +class ThreadedHTTPServer(socketserver.ThreadingMixIn, HTTPServer): + daemon_threads = True + + class BaseHTTPRequestHandler(socketserver.StreamRequestHandler): """HTTP request handler base class. @@ -1213,7 +1217,8 @@ def run_cgi(self): def test(HandlerClass=BaseHTTPRequestHandler, - ServerClass=HTTPServer, protocol="HTTP/1.0", port=8000, bind=""): + ServerClass=ThreadedHTTPServer, + protocol="HTTP/1.0", port=8000, bind=""): """Test the HTTP request handler class. This runs an HTTP server on port 8000 (or the port argument). diff --git a/Misc/NEWS.d/next/Library/2017-12-27-21-55-19.bpo-31639.l3avDJ.rst b/Misc/NEWS.d/next/Library/2017-12-27-21-55-19.bpo-31639.l3avDJ.rst new file mode 100644 index 000000000000..e876f40813de --- /dev/null +++ b/Misc/NEWS.d/next/Library/2017-12-27-21-55-19.bpo-31639.l3avDJ.rst @@ -0,0 +1,2 @@ +http.server now exposes a ThreadedHTTPServer class and uses it when the +module is invoked to cope with web browsers pre-opening sockets. From webhook-mailer at python.org Fri Mar 23 17:26:38 2018 From: webhook-mailer at python.org (Brett Cannon) Date: Fri, 23 Mar 2018 21:26:38 -0000 Subject: [Python-checkins] bpo-27428: Fix WindowsRegistryFinder documentation to list appropriate ABC (GH-6061) Message-ID: <mailman.151.1521840402.1871.python-checkins@python.org> https://github.com/python/cpython/commit/5cbb84106efefd200933aa31e22abf39267d2557 commit: 5cbb84106efefd200933aa31e22abf39267d2557 branch: master author: Himanshu Lakhara <himanshulakhara1947 at gmail.com> committer: Brett Cannon <brettcannon at users.noreply.github.com> date: 2018-03-23T14:26:35-07:00 summary: bpo-27428: Fix WindowsRegistryFinder documentation to list appropriate ABC (GH-6061) files: A Misc/NEWS.d/next/Documentation/2018-03-11-00-16-56.bpo-27428.B7A8FT.rst M Doc/library/importlib.rst diff --git a/Doc/library/importlib.rst b/Doc/library/importlib.rst index db75f69a0d00..f9387c07e18c 100644 --- a/Doc/library/importlib.rst +++ b/Doc/library/importlib.rst @@ -1030,7 +1030,7 @@ find and load modules. .. class:: WindowsRegistryFinder :term:`Finder` for modules declared in the Windows registry. This class - implements the :class:`importlib.abc.Finder` ABC. + implements the :class:`importlib.abc.MetaPathFinder` ABC. Only class methods are defined by this class to alleviate the need for instantiation. diff --git a/Misc/NEWS.d/next/Documentation/2018-03-11-00-16-56.bpo-27428.B7A8FT.rst b/Misc/NEWS.d/next/Documentation/2018-03-11-00-16-56.bpo-27428.B7A8FT.rst new file mode 100644 index 000000000000..c9ac8e22df08 --- /dev/null +++ b/Misc/NEWS.d/next/Documentation/2018-03-11-00-16-56.bpo-27428.B7A8FT.rst @@ -0,0 +1,2 @@ +Update documentation to clarify that ``WindowsRegistryFinder`` implements +``MetaPathFinder``. (Patch by Himanshu Lakhara) From webhook-mailer at python.org Fri Mar 23 17:41:29 2018 From: webhook-mailer at python.org (Miss Islington (bot)) Date: Fri, 23 Mar 2018 21:41:29 -0000 Subject: [Python-checkins] bpo-27428: Fix WindowsRegistryFinder documentation to list appropriate ABC (GH-6061) Message-ID: <mailman.152.1521841292.1871.python-checkins@python.org> https://github.com/python/cpython/commit/45738ede5ac8507b88b35fb0d6e2806a7b2e2efc commit: 45738ede5ac8507b88b35fb0d6e2806a7b2e2efc branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: GitHub <noreply at github.com> date: 2018-03-23T14:41:26-07:00 summary: bpo-27428: Fix WindowsRegistryFinder documentation to list appropriate ABC (GH-6061) (cherry picked from commit 5cbb84106efefd200933aa31e22abf39267d2557) Co-authored-by: Himanshu Lakhara <himanshulakhara1947 at gmail.com> files: A Misc/NEWS.d/next/Documentation/2018-03-11-00-16-56.bpo-27428.B7A8FT.rst M Doc/library/importlib.rst diff --git a/Doc/library/importlib.rst b/Doc/library/importlib.rst index db75f69a0d00..f9387c07e18c 100644 --- a/Doc/library/importlib.rst +++ b/Doc/library/importlib.rst @@ -1030,7 +1030,7 @@ find and load modules. .. class:: WindowsRegistryFinder :term:`Finder` for modules declared in the Windows registry. This class - implements the :class:`importlib.abc.Finder` ABC. + implements the :class:`importlib.abc.MetaPathFinder` ABC. Only class methods are defined by this class to alleviate the need for instantiation. diff --git a/Misc/NEWS.d/next/Documentation/2018-03-11-00-16-56.bpo-27428.B7A8FT.rst b/Misc/NEWS.d/next/Documentation/2018-03-11-00-16-56.bpo-27428.B7A8FT.rst new file mode 100644 index 000000000000..c9ac8e22df08 --- /dev/null +++ b/Misc/NEWS.d/next/Documentation/2018-03-11-00-16-56.bpo-27428.B7A8FT.rst @@ -0,0 +1,2 @@ +Update documentation to clarify that ``WindowsRegistryFinder`` implements +``MetaPathFinder``. (Patch by Himanshu Lakhara) From webhook-mailer at python.org Fri Mar 23 17:44:57 2018 From: webhook-mailer at python.org (Ivan Levkivskyi) Date: Fri, 23 Mar 2018 21:44:57 -0000 Subject: [Python-checkins] bpo-33061: Add missing 'NoReturn' to __all__ in typing.py (GH-6127) (#6162) Message-ID: <mailman.153.1521841500.1871.python-checkins@python.org> https://github.com/python/cpython/commit/ac5602746ed39ca6591e98e062e587121ac71371 commit: ac5602746ed39ca6591e98e062e587121ac71371 branch: 3.7 author: Ivan Levkivskyi <levkivskyi at gmail.com> committer: GitHub <noreply at github.com> date: 2018-03-23T21:44:54Z summary: bpo-33061: Add missing 'NoReturn' to __all__ in typing.py (GH-6127) (#6162) files: A Misc/NEWS.d/next/Library/2018-03-16-16-07-33.bpo-33061.TRTTek.rst M Lib/typing.py diff --git a/Lib/typing.py b/Lib/typing.py index 7ca080402e28..56126cfc0247 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -95,6 +95,7 @@ 'NewType', 'no_type_check', 'no_type_check_decorator', + 'NoReturn', 'overload', 'Text', 'TYPE_CHECKING', diff --git a/Misc/NEWS.d/next/Library/2018-03-16-16-07-33.bpo-33061.TRTTek.rst b/Misc/NEWS.d/next/Library/2018-03-16-16-07-33.bpo-33061.TRTTek.rst new file mode 100644 index 000000000000..b82ffb73a143 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-03-16-16-07-33.bpo-33061.TRTTek.rst @@ -0,0 +1 @@ +Add missing ``NoReturn`` to ``__all__`` in typing.py From webhook-mailer at python.org Fri Mar 23 17:50:15 2018 From: webhook-mailer at python.org (Miss Islington (bot)) Date: Fri, 23 Mar 2018 21:50:15 -0000 Subject: [Python-checkins] bpo-27428: Fix WindowsRegistryFinder documentation to list appropriate ABC (GH-6061) Message-ID: <mailman.154.1521841818.1871.python-checkins@python.org> https://github.com/python/cpython/commit/643a781188539de038745d23d0e8e5a03b781209 commit: 643a781188539de038745d23d0e8e5a03b781209 branch: 3.6 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: GitHub <noreply at github.com> date: 2018-03-23T14:50:09-07:00 summary: bpo-27428: Fix WindowsRegistryFinder documentation to list appropriate ABC (GH-6061) (cherry picked from commit 5cbb84106efefd200933aa31e22abf39267d2557) Co-authored-by: Himanshu Lakhara <himanshulakhara1947 at gmail.com> files: A Misc/NEWS.d/next/Documentation/2018-03-11-00-16-56.bpo-27428.B7A8FT.rst M Doc/library/importlib.rst diff --git a/Doc/library/importlib.rst b/Doc/library/importlib.rst index d194362fb003..c1cd9485cc78 100644 --- a/Doc/library/importlib.rst +++ b/Doc/library/importlib.rst @@ -799,7 +799,7 @@ find and load modules. .. class:: WindowsRegistryFinder :term:`Finder` for modules declared in the Windows registry. This class - implements the :class:`importlib.abc.Finder` ABC. + implements the :class:`importlib.abc.MetaPathFinder` ABC. Only class methods are defined by this class to alleviate the need for instantiation. diff --git a/Misc/NEWS.d/next/Documentation/2018-03-11-00-16-56.bpo-27428.B7A8FT.rst b/Misc/NEWS.d/next/Documentation/2018-03-11-00-16-56.bpo-27428.B7A8FT.rst new file mode 100644 index 000000000000..c9ac8e22df08 --- /dev/null +++ b/Misc/NEWS.d/next/Documentation/2018-03-11-00-16-56.bpo-27428.B7A8FT.rst @@ -0,0 +1,2 @@ +Update documentation to clarify that ``WindowsRegistryFinder`` implements +``MetaPathFinder``. (Patch by Himanshu Lakhara) From webhook-mailer at python.org Sat Mar 24 01:56:45 2018 From: webhook-mailer at python.org (Serhiy Storchaka) Date: Sat, 24 Mar 2018 05:56:45 -0000 Subject: [Python-checkins] bpo-31544: Avoid calling "PyObject_GetAttrString()" (and potentially executing user code) with a live exception set. (GH-3992) Message-ID: <mailman.155.1521871007.1871.python-checkins@python.org> https://github.com/python/cpython/commit/0694b6a651ba2a53f6323ffb3b23358f43885815 commit: 0694b6a651ba2a53f6323ffb3b23358f43885815 branch: 2.7 author: scoder <stefan_ml at behnel.de> committer: Serhiy Storchaka <storchaka at gmail.com> date: 2018-03-24T07:56:41+02:00 summary: bpo-31544: Avoid calling "PyObject_GetAttrString()" (and potentially executing user code) with a live exception set. (GH-3992) files: A Misc/NEWS.d/next/Library/2017-09-13-19-55-35.bpo-31544.beTh6t.rst M Lib/test/test_xml_etree.py M Modules/_elementtree.c diff --git a/Lib/test/test_xml_etree.py b/Lib/test/test_xml_etree.py index e466867b7cd1..c75d55f05c17 100644 --- a/Lib/test/test_xml_etree.py +++ b/Lib/test/test_xml_etree.py @@ -136,6 +136,13 @@ def wrapper(*args): return test(*args) return wrapper +def cet_only(test): + def wrapper(*args): + if ET is pyET: + raise unittest.SkipTest('only for the C version') + return test(*args) + return wrapper + # -------------------------------------------------------------------- # element tree tests @@ -2229,6 +2236,32 @@ def close(self): ('html', '-//W3C//DTD XHTML 1.0 Transitional//EN', 'http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd')) + @cet_only # PyET does not look up the attributes in XMLParser().__init__() + def test_builder_lookup_errors(self): + class RaisingBuilder(object): + def __init__(self, raise_in=None, what=ValueError): + self.raise_in = raise_in + self.what = what + + def __getattr__(self, name): + if name == self.raise_in: + raise self.what(self.raise_in) + def handle(*args): + pass + return handle + + ET.XMLParser(target=RaisingBuilder()) + # cET also checks for 'close' and 'doctype', PyET does it only at need + for event in ('start', 'data', 'end', 'comment', 'pi'): + with self.assertRaises(ValueError): + ET.XMLParser(target=RaisingBuilder(event)) + + ET.XMLParser(target=RaisingBuilder(what=AttributeError)) + for event in ('start', 'data', 'end', 'comment', 'pi'): + parser = ET.XMLParser(target=RaisingBuilder(event, what=AttributeError)) + parser.feed(self.sample1) + self.assertIsNone(parser.close()) + class XMLParserTest(unittest.TestCase): sample1 = b'<file><line>22</line></file>' diff --git a/Misc/NEWS.d/next/Library/2017-09-13-19-55-35.bpo-31544.beTh6t.rst b/Misc/NEWS.d/next/Library/2017-09-13-19-55-35.bpo-31544.beTh6t.rst new file mode 100644 index 000000000000..9ea3599ee0b0 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2017-09-13-19-55-35.bpo-31544.beTh6t.rst @@ -0,0 +1,2 @@ +The C accelerator module of ElementTree ignored exceptions raised when +looking up TreeBuilder target methods in XMLParser(). diff --git a/Modules/_elementtree.c b/Modules/_elementtree.c index 574559c6313b..7f0e60934003 100644 --- a/Modules/_elementtree.c +++ b/Modules/_elementtree.c @@ -2510,6 +2510,18 @@ expat_unknown_encoding_handler(XMLParserObject *self, const XML_Char *name, /* -------------------------------------------------------------------- */ /* constructor and destructor */ +static int +ignore_attribute_error(PyObject *value) +{ + if (value == NULL) { + if (!PyErr_ExceptionMatches(PyExc_AttributeError)) { + return -1; + } + PyErr_Clear(); + } + return 0; +} + static PyObject* xmlparser(PyObject* self_, PyObject* args, PyObject* kw) { @@ -2578,14 +2590,33 @@ xmlparser(PyObject* self_, PyObject* args, PyObject* kw) self->target = target; self->handle_xml = PyObject_GetAttrString(target, "xml"); + if (ignore_attribute_error(self->handle_xml)) { + return NULL; + } self->handle_start = PyObject_GetAttrString(target, "start"); + if (ignore_attribute_error(self->handle_start)) { + return NULL; + } self->handle_data = PyObject_GetAttrString(target, "data"); + if (ignore_attribute_error(self->handle_data)) { + return NULL; + } self->handle_end = PyObject_GetAttrString(target, "end"); + if (ignore_attribute_error(self->handle_end)) { + return NULL; + } self->handle_comment = PyObject_GetAttrString(target, "comment"); + if (ignore_attribute_error(self->handle_comment)) { + return NULL; + } self->handle_pi = PyObject_GetAttrString(target, "pi"); + if (ignore_attribute_error(self->handle_pi)) { + return NULL; + } self->handle_close = PyObject_GetAttrString(target, "close"); - - PyErr_Clear(); + if (ignore_attribute_error(self->handle_close)) { + return NULL; + } /* configure parser */ EXPAT(SetUserData)(self->parser, self); From solipsis at pitrou.net Sat Mar 24 05:13:21 2018 From: solipsis at pitrou.net (solipsis at pitrou.net) Date: Sat, 24 Mar 2018 09:13:21 +0000 Subject: [Python-checkins] Daily reference leaks (4243df51fe43): sum=7 Message-ID: <20180324091321.1.D2CB2C8B1D10C443@psf.io> results for 4243df51fe43 on branch "default" -------------------------------------------- test_functools leaked [0, 3, 1] memory blocks, sum=4 test_multiprocessing_fork leaked [-2, 2, 0] memory blocks, sum=0 test_multiprocessing_forkserver leaked [0, -1, 2] memory blocks, sum=1 test_multiprocessing_spawn leaked [1, -1, 2] memory blocks, sum=2 Command line was: ['./python', '-m', 'test.regrtest', '-uall', '-R', '3:3:/home/psf-users/antoine/refleaks/reflogDDlYLT', '--timeout', '7200'] From webhook-mailer at python.org Sat Mar 24 06:39:44 2018 From: webhook-mailer at python.org (Xiang Zhang) Date: Sat, 24 Mar 2018 10:39:44 -0000 Subject: [Python-checkins] bpo-32932: More revealing error message when non-str objects in __all__ (GH-5848) Message-ID: <mailman.156.1521887986.1871.python-checkins@python.org> https://github.com/python/cpython/commit/d8b291a74284307610946f1b5801aa95d7f1e052 commit: d8b291a74284307610946f1b5801aa95d7f1e052 branch: master author: Xiang Zhang <angwerzx at 126.com> committer: GitHub <noreply at github.com> date: 2018-03-24T18:39:36+08:00 summary: bpo-32932: More revealing error message when non-str objects in __all__ (GH-5848) files: A Misc/NEWS.d/next/Core and Builtins/2018-02-24-21-51-42.bpo-32932.2cz31L.rst M Lib/test/test_import/__init__.py M Python/ceval.c diff --git a/Lib/test/test_import/__init__.py b/Lib/test/test_import/__init__.py index ceea79f6ad96..606b05784afc 100644 --- a/Lib/test/test_import/__init__.py +++ b/Lib/test/test_import/__init__.py @@ -111,6 +111,27 @@ def test_from_import_missing_attr_path_is_canonical(self): self.assertIn(cm.exception.name, {'posixpath', 'ntpath'}) self.assertIsNotNone(cm.exception) + def test_from_import_star_invalid_type(self): + import re + with _ready_to_import() as (name, path): + with open(path, 'w') as f: + f.write("__all__ = [b'invalid_type']") + globals = {} + with self.assertRaisesRegex( + TypeError, f"{re.escape(name)}\.__all__ must be str" + ): + exec(f"from {name} import *", globals) + self.assertNotIn(b"invalid_type", globals) + with _ready_to_import() as (name, path): + with open(path, 'w') as f: + f.write("globals()[b'invalid_type'] = object()") + globals = {} + with self.assertRaisesRegex( + TypeError, f"{re.escape(name)}\.__dict__ must be str" + ): + exec(f"from {name} import *", globals) + self.assertNotIn(b"invalid_type", globals) + def test_case_sensitivity(self): # Brief digression to test that import is case-sensitive: if we got # this far, we know for sure that "random" exists. diff --git a/Misc/NEWS.d/next/Core and Builtins/2018-02-24-21-51-42.bpo-32932.2cz31L.rst b/Misc/NEWS.d/next/Core and Builtins/2018-02-24-21-51-42.bpo-32932.2cz31L.rst new file mode 100644 index 000000000000..51e3d9b613d5 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2018-02-24-21-51-42.bpo-32932.2cz31L.rst @@ -0,0 +1 @@ +Make error message more revealing when there are non-str objects in ``__all__``. diff --git a/Python/ceval.c b/Python/ceval.c index 14603d330083..d18a284e9f9d 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -4837,6 +4837,7 @@ import_all_from(PyObject *locals, PyObject *v) { _Py_IDENTIFIER(__all__); _Py_IDENTIFIER(__dict__); + _Py_IDENTIFIER(__name__); PyObject *all, *dict, *name, *value; int skip_leading_underscores = 0; int pos, err; @@ -4869,7 +4870,32 @@ import_all_from(PyObject *locals, PyObject *v) PyErr_Clear(); break; } - if (skip_leading_underscores && PyUnicode_Check(name)) { + if (!PyUnicode_Check(name)) { + PyObject *modname = _PyObject_GetAttrId(v, &PyId___name__); + if (modname == NULL) { + Py_DECREF(name); + err = -1; + break; + } + if (!PyUnicode_Check(modname)) { + PyErr_Format(PyExc_TypeError, + "module __name__ must be a string, not %.100s", + Py_TYPE(modname)->tp_name); + } + else { + PyErr_Format(PyExc_TypeError, + "%s in %U.%s must be str, not %.100s", + skip_leading_underscores ? "Key" : "Item", + modname, + skip_leading_underscores ? "__dict__" : "__all__", + Py_TYPE(name)->tp_name); + } + Py_DECREF(modname); + Py_DECREF(name); + err = -1; + break; + } + if (skip_leading_underscores) { if (PyUnicode_READY(name) == -1) { Py_DECREF(name); err = -1; From webhook-mailer at python.org Sat Mar 24 10:36:53 2018 From: webhook-mailer at python.org (Christian Heimes) Date: Sat, 24 Mar 2018 14:36:53 -0000 Subject: [Python-checkins] bpo-24334: Remove inaccurate match_hostname call (#6211) Message-ID: <mailman.157.1521902215.1871.python-checkins@python.org> https://github.com/python/cpython/commit/e42ae915095ebca789cc36f3a336a3331fe35945 commit: e42ae915095ebca789cc36f3a336a3331fe35945 branch: master author: Christian Heimes <christian at python.org> committer: GitHub <noreply at github.com> date: 2018-03-24T15:36:50+01:00 summary: bpo-24334: Remove inaccurate match_hostname call (#6211) Commit 141c5e8c re-added match_hostname() call. The resurrection of the function call was never intended and was solely a merge mistake. Signed-off-by: Christian Heimes <christian at python.org> files: M Lib/ssl.py diff --git a/Lib/ssl.py b/Lib/ssl.py index 2db887354714..fdd161574434 100644 --- a/Lib/ssl.py +++ b/Lib/ssl.py @@ -1106,11 +1106,6 @@ def do_handshake(self, block=False): if timeout == 0.0 and block: self.settimeout(None) self._sslobj.do_handshake() - if self.context.check_hostname: - if not self.server_hostname: - raise ValueError("check_hostname needs server_hostname " - "argument") - match_hostname(self.getpeercert(), self.server_hostname) finally: self.settimeout(timeout) From webhook-mailer at python.org Sat Mar 24 10:41:40 2018 From: webhook-mailer at python.org (Christian Heimes) Date: Sat, 24 Mar 2018 14:41:40 -0000 Subject: [Python-checkins] bpo-33127: Compatibility patch for LibreSSL 2.7.0 (GH-6210) Message-ID: <mailman.158.1521902502.1871.python-checkins@python.org> https://github.com/python/cpython/commit/4ca0739c9d97ac7cd45499e0d31be68dc659d0e1 commit: 4ca0739c9d97ac7cd45499e0d31be68dc659d0e1 branch: master author: Christian Heimes <christian at python.org> committer: GitHub <noreply at github.com> date: 2018-03-24T15:41:37+01:00 summary: bpo-33127: Compatibility patch for LibreSSL 2.7.0 (GH-6210) LibreSSL 2.7 introduced OpenSSL 1.1.0 API. The ssl module now detects LibreSSL 2.7 and only provides API shims for OpenSSL < 1.1.0 and LibreSSL < 2.7. Documentation updates and fixes for failing tests will be provided in another patch set. Signed-off-by: Christian Heimes <christian at python.org> files: A Misc/NEWS.d/next/Library/2018-03-24-15-08-24.bpo-33127.olJmHv.rst M Modules/_ssl.c M Tools/ssl/multissltests.py diff --git a/Misc/NEWS.d/next/Library/2018-03-24-15-08-24.bpo-33127.olJmHv.rst b/Misc/NEWS.d/next/Library/2018-03-24-15-08-24.bpo-33127.olJmHv.rst new file mode 100644 index 000000000000..635aabbde031 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-03-24-15-08-24.bpo-33127.olJmHv.rst @@ -0,0 +1 @@ +The ssl module now compiles with LibreSSL 2.7.1. diff --git a/Modules/_ssl.c b/Modules/_ssl.c index f9e061dfe01d..30c340376a4f 100644 --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -136,6 +136,12 @@ static void _PySSLFixErrno(void) { #if (OPENSSL_VERSION_NUMBER >= 0x10100000L) && !defined(LIBRESSL_VERSION_NUMBER) # define OPENSSL_VERSION_1_1 1 +# define PY_OPENSSL_1_1_API 1 +#endif + +/* LibreSSL 2.7.0 provides necessary OpenSSL 1.1.0 APIs */ +#if defined(LIBRESSL_VERSION_NUMBER) && LIBRESSL_VERSION_NUMBER >= 0x2070000fL +# define PY_OPENSSL_1_1_API 1 #endif /* Openssl comes with TLSv1.1 and TLSv1.2 between 1.0.0h and 1.0.1 @@ -182,13 +188,17 @@ static void _PySSLFixErrno(void) { #define INVALID_SOCKET (-1) #endif -#ifdef OPENSSL_VERSION_1_1 -/* OpenSSL 1.1.0+ */ -#ifndef OPENSSL_NO_SSL2 +/* OpenSSL 1.0.2 and LibreSSL needs extra code for locking */ +#ifndef OPENSSL_VERSION_1_1 +#define HAVE_OPENSSL_CRYPTO_LOCK +#endif + +#if defined(OPENSSL_VERSION_1_1) && !defined(OPENSSL_NO_SSL2) #define OPENSSL_NO_SSL2 #endif -#else /* OpenSSL < 1.1.0 */ -#define HAVE_OPENSSL_CRYPTO_LOCK + +#ifndef PY_OPENSSL_1_1_API +/* OpenSSL 1.1 API shims for OpenSSL < 1.1.0 and LibreSSL < 2.7.0 */ #define TLS_method SSLv23_method #define TLS_client_method SSLv23_client_method @@ -250,7 +260,7 @@ SSL_SESSION_get_ticket_lifetime_hint(const SSL_SESSION *s) return s->tlsext_tick_lifetime_hint; } -#endif /* OpenSSL < 1.1.0 or LibreSSL */ +#endif /* OpenSSL < 1.1.0 or LibreSSL < 2.7.0 */ /* Default cipher suites */ #ifndef PY_SSL_DEFAULT_CIPHERS diff --git a/Tools/ssl/multissltests.py b/Tools/ssl/multissltests.py index 70913c7203b3..a51566c5069d 100755 --- a/Tools/ssl/multissltests.py +++ b/Tools/ssl/multissltests.py @@ -54,7 +54,7 @@ ] LIBRESSL_RECENT_VERSIONS = [ - # "2.6.5", + "2.7.1", ] # store files in ../multissl From webhook-mailer at python.org Sat Mar 24 10:59:19 2018 From: webhook-mailer at python.org (Christian Heimes) Date: Sat, 24 Mar 2018 14:59:19 -0000 Subject: [Python-checkins] [3.7] bpo-24334: Remove inaccurate match_hostname call (GH-6211) (#6212) Message-ID: <mailman.159.1521903560.1871.python-checkins@python.org> https://github.com/python/cpython/commit/1a0bb626f4cfd95f7ec406ea7d3f9433def559fc commit: 1a0bb626f4cfd95f7ec406ea7d3f9433def559fc branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Christian Heimes <christian at python.org> date: 2018-03-24T15:59:16+01:00 summary: [3.7] bpo-24334: Remove inaccurate match_hostname call (GH-6211) (#6212) Commit 141c5e8c re-added match_hostname() call. The resurrection of the function call was never intended and was solely a merge mistake. Signed-off-by: Christian Heimes <christian at python.org> (cherry picked from commit e42ae915095ebca789cc36f3a336a3331fe35945) Co-authored-by: Christian Heimes <christian at python.org> files: M Lib/ssl.py diff --git a/Lib/ssl.py b/Lib/ssl.py index 2db887354714..fdd161574434 100644 --- a/Lib/ssl.py +++ b/Lib/ssl.py @@ -1106,11 +1106,6 @@ def do_handshake(self, block=False): if timeout == 0.0 and block: self.settimeout(None) self._sslobj.do_handshake() - if self.context.check_hostname: - if not self.server_hostname: - raise ValueError("check_hostname needs server_hostname " - "argument") - match_hostname(self.getpeercert(), self.server_hostname) finally: self.settimeout(timeout) From webhook-mailer at python.org Sat Mar 24 13:37:57 2018 From: webhook-mailer at python.org (Christian Heimes) Date: Sat, 24 Mar 2018 17:37:57 -0000 Subject: [Python-checkins] [3.7] bpo-33127: Compatibility patch for LibreSSL 2.7.0 (GH-6210) (GH-6213) Message-ID: <mailman.160.1521913079.1871.python-checkins@python.org> https://github.com/python/cpython/commit/42bd62bc87a52538c0fc2134b76df316f30997da commit: 42bd62bc87a52538c0fc2134b76df316f30997da branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Christian Heimes <christian at python.org> date: 2018-03-24T18:37:54+01:00 summary: [3.7] bpo-33127: Compatibility patch for LibreSSL 2.7.0 (GH-6210) (GH-6213) LibreSSL 2.7 introduced OpenSSL 1.1.0 API. The ssl module now detects LibreSSL 2.7 and only provides API shims for OpenSSL < 1.1.0 and LibreSSL < 2.7. Documentation updates and fixes for failing tests will be provided in another patch set. Signed-off-by: Christian Heimes <christian at python.org> (cherry picked from commit 4ca0739c9d97ac7cd45499e0d31be68dc659d0e1) Co-authored-by: Christian Heimes <christian at python.org> files: A Misc/NEWS.d/next/Library/2018-03-24-15-08-24.bpo-33127.olJmHv.rst M Modules/_ssl.c M Tools/ssl/multissltests.py diff --git a/Misc/NEWS.d/next/Library/2018-03-24-15-08-24.bpo-33127.olJmHv.rst b/Misc/NEWS.d/next/Library/2018-03-24-15-08-24.bpo-33127.olJmHv.rst new file mode 100644 index 000000000000..635aabbde031 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-03-24-15-08-24.bpo-33127.olJmHv.rst @@ -0,0 +1 @@ +The ssl module now compiles with LibreSSL 2.7.1. diff --git a/Modules/_ssl.c b/Modules/_ssl.c index f9e061dfe01d..30c340376a4f 100644 --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -136,6 +136,12 @@ static void _PySSLFixErrno(void) { #if (OPENSSL_VERSION_NUMBER >= 0x10100000L) && !defined(LIBRESSL_VERSION_NUMBER) # define OPENSSL_VERSION_1_1 1 +# define PY_OPENSSL_1_1_API 1 +#endif + +/* LibreSSL 2.7.0 provides necessary OpenSSL 1.1.0 APIs */ +#if defined(LIBRESSL_VERSION_NUMBER) && LIBRESSL_VERSION_NUMBER >= 0x2070000fL +# define PY_OPENSSL_1_1_API 1 #endif /* Openssl comes with TLSv1.1 and TLSv1.2 between 1.0.0h and 1.0.1 @@ -182,13 +188,17 @@ static void _PySSLFixErrno(void) { #define INVALID_SOCKET (-1) #endif -#ifdef OPENSSL_VERSION_1_1 -/* OpenSSL 1.1.0+ */ -#ifndef OPENSSL_NO_SSL2 +/* OpenSSL 1.0.2 and LibreSSL needs extra code for locking */ +#ifndef OPENSSL_VERSION_1_1 +#define HAVE_OPENSSL_CRYPTO_LOCK +#endif + +#if defined(OPENSSL_VERSION_1_1) && !defined(OPENSSL_NO_SSL2) #define OPENSSL_NO_SSL2 #endif -#else /* OpenSSL < 1.1.0 */ -#define HAVE_OPENSSL_CRYPTO_LOCK + +#ifndef PY_OPENSSL_1_1_API +/* OpenSSL 1.1 API shims for OpenSSL < 1.1.0 and LibreSSL < 2.7.0 */ #define TLS_method SSLv23_method #define TLS_client_method SSLv23_client_method @@ -250,7 +260,7 @@ SSL_SESSION_get_ticket_lifetime_hint(const SSL_SESSION *s) return s->tlsext_tick_lifetime_hint; } -#endif /* OpenSSL < 1.1.0 or LibreSSL */ +#endif /* OpenSSL < 1.1.0 or LibreSSL < 2.7.0 */ /* Default cipher suites */ #ifndef PY_SSL_DEFAULT_CIPHERS diff --git a/Tools/ssl/multissltests.py b/Tools/ssl/multissltests.py index 70913c7203b3..a51566c5069d 100755 --- a/Tools/ssl/multissltests.py +++ b/Tools/ssl/multissltests.py @@ -54,7 +54,7 @@ ] LIBRESSL_RECENT_VERSIONS = [ - # "2.6.5", + "2.7.1", ] # store files in ../multissl From webhook-mailer at python.org Sat Mar 24 13:38:17 2018 From: webhook-mailer at python.org (Christian Heimes) Date: Sat, 24 Mar 2018 17:38:17 -0000 Subject: [Python-checkins] [3.6] bpo-33127: Compatibility patch for LibreSSL 2.7.0 (GH-6210) (GH-6214) Message-ID: <mailman.161.1521913100.1871.python-checkins@python.org> https://github.com/python/cpython/commit/f5befbb0d1526f18eb2b24eabb48c3b761c624a2 commit: f5befbb0d1526f18eb2b24eabb48c3b761c624a2 branch: 3.6 author: Christian Heimes <christian at python.org> committer: GitHub <noreply at github.com> date: 2018-03-24T18:38:14+01:00 summary: [3.6] bpo-33127: Compatibility patch for LibreSSL 2.7.0 (GH-6210) (GH-6214) LibreSSL 2.7 introduced OpenSSL 1.1.0 API. The ssl module now detects LibreSSL 2.7 and only provides API shims for OpenSSL < 1.1.0 and LibreSSL < 2.7. Documentation updates and fixes for failing tests will be provided in another patch set. Signed-off-by: Christian Heimes <christian at python.org>. (cherry picked from commit 4ca0739c9d97ac7cd45499e0d31be68dc659d0e1) Co-authored-by: Christian Heimes <christian at python.org> files: A Misc/NEWS.d/next/Library/2018-03-24-15-08-24.bpo-33127.olJmHv.rst M Lib/test/test_ssl.py M Modules/_ssl.c M Tools/ssl/multissltests.py diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py index 8dd3b4145078..9785a59a7e49 100644 --- a/Lib/test/test_ssl.py +++ b/Lib/test/test_ssl.py @@ -1687,6 +1687,7 @@ def test_get_ca_certs_capath(self): self.assertEqual(len(ctx.get_ca_certs()), 1) @needs_sni + @unittest.skipUnless(hasattr(ssl, "PROTOCOL_TLSv1_2"), "needs TLS 1.2") def test_context_setget(self): # Check that the context of a connected socket can be replaced. ctx1 = ssl.SSLContext(ssl.PROTOCOL_TLSv1_2) diff --git a/Misc/NEWS.d/next/Library/2018-03-24-15-08-24.bpo-33127.olJmHv.rst b/Misc/NEWS.d/next/Library/2018-03-24-15-08-24.bpo-33127.olJmHv.rst new file mode 100644 index 000000000000..635aabbde031 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-03-24-15-08-24.bpo-33127.olJmHv.rst @@ -0,0 +1 @@ +The ssl module now compiles with LibreSSL 2.7.1. diff --git a/Modules/_ssl.c b/Modules/_ssl.c index c54e43c2b48a..5e007da858bd 100644 --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -106,6 +106,12 @@ struct py_ssl_library_code { #if (OPENSSL_VERSION_NUMBER >= 0x10100000L) && !defined(LIBRESSL_VERSION_NUMBER) # define OPENSSL_VERSION_1_1 1 +# define PY_OPENSSL_1_1_API 1 +#endif + +/* LibreSSL 2.7.0 provides necessary OpenSSL 1.1.0 APIs */ +#if defined(LIBRESSL_VERSION_NUMBER) && LIBRESSL_VERSION_NUMBER >= 0x2070000fL +# define PY_OPENSSL_1_1_API 1 #endif /* Openssl comes with TLSv1.1 and TLSv1.2 between 1.0.0h and 1.0.1 @@ -152,16 +158,18 @@ struct py_ssl_library_code { #define INVALID_SOCKET (-1) #endif -#ifdef OPENSSL_VERSION_1_1 -/* OpenSSL 1.1.0+ */ -#ifndef OPENSSL_NO_SSL2 -#define OPENSSL_NO_SSL2 -#endif -#else /* OpenSSL < 1.1.0 */ -#if defined(WITH_THREAD) +/* OpenSSL 1.0.2 and LibreSSL needs extra code for locking */ +#if !defined(OPENSSL_VERSION_1_1) && defined(WITH_THREAD) #define HAVE_OPENSSL_CRYPTO_LOCK #endif +#if defined(OPENSSL_VERSION_1_1) && !defined(OPENSSL_NO_SSL2) +#define OPENSSL_NO_SSL2 +#endif + +#ifndef PY_OPENSSL_1_1_API +/* OpenSSL 1.1 API shims for OpenSSL < 1.1.0 and LibreSSL < 2.7.0 */ + #define TLS_method SSLv23_method #define TLS_client_method SSLv23_client_method #define TLS_server_method SSLv23_server_method @@ -227,7 +235,7 @@ SSL_SESSION_get_ticket_lifetime_hint(const SSL_SESSION *s) return s->tlsext_tick_lifetime_hint; } -#endif /* OpenSSL < 1.1.0 or LibreSSL */ +#endif /* OpenSSL < 1.1.0 or LibreSSL < 2.7.0 */ enum py_ssl_error { diff --git a/Tools/ssl/multissltests.py b/Tools/ssl/multissltests.py index ce5bbd85308c..ba4529ae0611 100755 --- a/Tools/ssl/multissltests.py +++ b/Tools/ssl/multissltests.py @@ -57,8 +57,9 @@ ] LIBRESSL_RECENT_VERSIONS = [ - "2.5.3", "2.5.5", + "2.6.4", + "2.7.1", ] # store files in ../multissl From webhook-mailer at python.org Sat Mar 24 14:34:19 2018 From: webhook-mailer at python.org (Christian Heimes) Date: Sat, 24 Mar 2018 18:34:19 -0000 Subject: [Python-checkins] [2.7] bpo-33127: Compatibility patch for LibreSSL 2.7.0 (GH-6210) (GH-6215) Message-ID: <mailman.162.1521916461.1871.python-checkins@python.org> https://github.com/python/cpython/commit/edd541897b9c28ee0d0f0131746aa5f19665a104 commit: edd541897b9c28ee0d0f0131746aa5f19665a104 branch: 2.7 author: Christian Heimes <christian at python.org> committer: GitHub <noreply at github.com> date: 2018-03-24T19:34:15+01:00 summary: [2.7] bpo-33127: Compatibility patch for LibreSSL 2.7.0 (GH-6210) (GH-6215) LibreSSL 2.7 introduced OpenSSL 1.1.0 API. The ssl module now detects LibreSSL 2.7 and only provides API shims for OpenSSL < 1.1.0 and LibreSSL < 2.7. Documentation updates and fixes for failing tests will be provided in another patch set. Signed-off-by: Christian Heimes <christian at python.org>. (cherry picked from commit 4ca0739c9d97ac7cd45499e0d31be68dc659d0e1) Co-authored-by: Christian Heimes <christian at python.org> files: A Misc/NEWS.d/next/Library/2018-03-24-15-08-24.bpo-33127.olJmHv.rst M Modules/_ssl.c M Tools/ssl/multissltests.py diff --git a/Misc/NEWS.d/next/Library/2018-03-24-15-08-24.bpo-33127.olJmHv.rst b/Misc/NEWS.d/next/Library/2018-03-24-15-08-24.bpo-33127.olJmHv.rst new file mode 100644 index 000000000000..635aabbde031 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-03-24-15-08-24.bpo-33127.olJmHv.rst @@ -0,0 +1 @@ +The ssl module now compiles with LibreSSL 2.7.1. diff --git a/Modules/_ssl.c b/Modules/_ssl.c index da8b20f54f35..d0ce913d3d89 100644 --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -102,6 +102,12 @@ struct py_ssl_library_code { #if (OPENSSL_VERSION_NUMBER >= 0x10100000L) && !defined(LIBRESSL_VERSION_NUMBER) # define OPENSSL_VERSION_1_1 1 +# define PY_OPENSSL_1_1_API 1 +#endif + +/* LibreSSL 2.7.0 provides necessary OpenSSL 1.1.0 APIs */ +#if defined(LIBRESSL_VERSION_NUMBER) && LIBRESSL_VERSION_NUMBER >= 0x2070000fL +# define PY_OPENSSL_1_1_API 1 #endif /* Openssl comes with TLSv1.1 and TLSv1.2 between 1.0.0h and 1.0.1 @@ -149,16 +155,18 @@ struct py_ssl_library_code { #define INVALID_SOCKET (-1) #endif -#ifdef OPENSSL_VERSION_1_1 -/* OpenSSL 1.1.0+ */ -#ifndef OPENSSL_NO_SSL2 -#define OPENSSL_NO_SSL2 -#endif -#else /* OpenSSL < 1.1.0 */ -#if defined(WITH_THREAD) +/* OpenSSL 1.0.2 and LibreSSL needs extra code for locking */ +#if !defined(OPENSSL_VERSION_1_1) && defined(WITH_THREAD) #define HAVE_OPENSSL_CRYPTO_LOCK #endif +#if defined(OPENSSL_VERSION_1_1) && !defined(OPENSSL_NO_SSL2) +#define OPENSSL_NO_SSL2 +#endif + +#ifndef PY_OPENSSL_1_1_API +/* OpenSSL 1.1 API shims for OpenSSL < 1.1.0 and LibreSSL < 2.7.0 */ + #define TLS_method SSLv23_method static int X509_NAME_ENTRY_set(const X509_NAME_ENTRY *ne) @@ -201,7 +209,7 @@ static X509_VERIFY_PARAM *X509_STORE_get0_param(X509_STORE *store) { return store->param; } -#endif /* OpenSSL < 1.1.0 or LibreSSL */ +#endif /* OpenSSL < 1.1.0 or LibreSSL < 2.7.0 */ enum py_ssl_error { diff --git a/Tools/ssl/multissltests.py b/Tools/ssl/multissltests.py index 994e420818ed..ffc57f055656 100755 --- a/Tools/ssl/multissltests.py +++ b/Tools/ssl/multissltests.py @@ -58,8 +58,9 @@ ] LIBRESSL_RECENT_VERSIONS = [ - "2.5.3", "2.5.5", + "2.6.4", + "2.7.1", ] # store files in ../multissl From webhook-mailer at python.org Sat Mar 24 16:42:43 2018 From: webhook-mailer at python.org (Serhiy Storchaka) Date: Sat, 24 Mar 2018 20:42:43 -0000 Subject: [Python-checkins] bpo-33132: Fix reference counting issues in the compiler. (GH-6209) Message-ID: <mailman.164.1521924164.1871.python-checkins@python.org> https://github.com/python/cpython/commit/a95d98607efe0c43475b354543e49bf8e240bc6f commit: a95d98607efe0c43475b354543e49bf8e240bc6f branch: master author: Serhiy Storchaka <storchaka at gmail.com> committer: GitHub <noreply at github.com> date: 2018-03-24T22:42:35+02:00 summary: bpo-33132: Fix reference counting issues in the compiler. (GH-6209) files: M Python/compile.c diff --git a/Python/compile.c b/Python/compile.c index 725bb9b213a2..9d2ba7b18f11 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -2909,9 +2909,7 @@ static int compiler_from_import(struct compiler *c, stmt_ty s) { Py_ssize_t i, n = asdl_seq_LEN(s->v.ImportFrom.names); - - PyObject *names = PyTuple_New(n); - PyObject *level; + PyObject *level, *names; static PyObject *empty_string; if (!empty_string) { @@ -2920,14 +2918,15 @@ compiler_from_import(struct compiler *c, stmt_ty s) return 0; } - if (!names) - return 0; - level = PyLong_FromLong(s->v.ImportFrom.level); if (!level) { - Py_DECREF(names); return 0; } + ADDOP_N(c, LOAD_CONST, level, consts); + + names = PyTuple_New(n); + if (!names) + return 0; /* build up the names */ for (i = 0; i < n; i++) { @@ -2938,16 +2937,12 @@ compiler_from_import(struct compiler *c, stmt_ty s) if (s->lineno > c->c_future->ff_lineno && s->v.ImportFrom.module && _PyUnicode_EqualToASCIIString(s->v.ImportFrom.module, "__future__")) { - Py_DECREF(level); Py_DECREF(names); return compiler_error(c, "from __future__ imports must occur " "at the beginning of the file"); } + ADDOP_N(c, LOAD_CONST, names, consts); - ADDOP_O(c, LOAD_CONST, level, consts); - Py_DECREF(level); - ADDOP_O(c, LOAD_CONST, names, consts); - Py_DECREF(names); if (s->v.ImportFrom.module) { ADDOP_NAME(c, IMPORT_NAME, s->v.ImportFrom.module, names); } @@ -2970,7 +2965,6 @@ compiler_from_import(struct compiler *c, stmt_ty s) store_name = alias->asname; if (!compiler_nameop(c, store_name, Store)) { - Py_DECREF(names); return 0; } } @@ -4739,10 +4733,6 @@ compiler_annassign(struct compiler *c, stmt_ty s) if (s->v.AnnAssign.simple && (c->u->u_scope_type == COMPILER_SCOPE_MODULE || c->u->u_scope_type == COMPILER_SCOPE_CLASS)) { - mangled = _Py_Mangle(c->u->u_private, targ->v.Name.id); - if (!mangled) { - return 0; - } if (c->c_future->ff_features & CO_FUTURE_ANNOTATIONS) { VISIT(c, annexpr, s->v.AnnAssign.annotation) } @@ -4750,8 +4740,11 @@ compiler_annassign(struct compiler *c, stmt_ty s) VISIT(c, expr, s->v.AnnAssign.annotation); } ADDOP_NAME(c, LOAD_NAME, __annotations__, names); - ADDOP_O(c, LOAD_CONST, mangled, consts); - Py_DECREF(mangled); + mangled = _Py_Mangle(c->u->u_private, targ->v.Name.id); + if (!mangled) { + return 0; + } + ADDOP_N(c, LOAD_CONST, mangled, consts); ADDOP(c, STORE_SUBSCR); } break; From webhook-mailer at python.org Sat Mar 24 17:20:28 2018 From: webhook-mailer at python.org (Eric V. Smith) Date: Sat, 24 Mar 2018 21:20:28 -0000 Subject: [Python-checkins] Trivial dataclass cleanups: (GH-6218) Message-ID: <mailman.165.1521926429.1871.python-checkins@python.org> https://github.com/python/cpython/commit/f96ddade0094d162cb6c2fd7255c5e8a90b5c37d commit: f96ddade0094d162cb6c2fd7255c5e8a90b5c37d branch: master author: Eric V. Smith <ericvsmith at users.noreply.github.com> committer: GitHub <noreply at github.com> date: 2018-03-24T17:20:26-04:00 summary: Trivial dataclass cleanups: (GH-6218) - When adding a single element to a list, use .append() instead of += and creating a new list. - For consistency, import the copy module, instead of just deepcopy. This leaves only a module at the class level, instead of a function. - Improve some comments. - Improve some whitespace. - Use tuples instead of lists. - Simplify a test. files: M Lib/dataclasses.py diff --git a/Lib/dataclasses.py b/Lib/dataclasses.py index 5d4d4a6100ca..4425408b27d7 100644 --- a/Lib/dataclasses.py +++ b/Lib/dataclasses.py @@ -1,6 +1,6 @@ import sys +import copy import types -from copy import deepcopy import inspect __all__ = ['dataclass', @@ -137,7 +137,7 @@ # For boxes that are blank, __hash__ is untouched and therefore # inherited from the base class. If the base is object, then # id-based hashing is used. -# Note that a class may have already __hash__=None if it specified an +# Note that a class may already have __hash__=None if it specified an # __eq__ method in the class body (not one that was created by # @dataclass). # See _hash_action (below) for a coded version of this table. @@ -147,7 +147,7 @@ class FrozenInstanceError(AttributeError): pass # A sentinel object for default values to signal that a -# default-factory will be used. +# default factory will be used. # This is given a nice repr() which will appear in the function # signature of dataclasses' constructors. class _HAS_DEFAULT_FACTORY_CLASS: @@ -249,6 +249,7 @@ class _DataclassParams: 'unsafe_hash', 'frozen', ) + def __init__(self, init, repr, eq, order, unsafe_hash, frozen): self.init = init self.repr = repr @@ -267,6 +268,7 @@ def __repr__(self): f'frozen={self.frozen}' ')') + # This function is used instead of exposing Field creation directly, # so that a type checker can be told (via overloads) that this is a # function whose type depends on its parameters. @@ -307,6 +309,8 @@ def _tuple_str(obj_name, fields): def _create_fn(name, args, body, *, globals=None, locals=None, return_type=MISSING): # Note that we mutate locals when exec() is called. Caller beware! + # The only callers are internal to this module, so no worries + # about external callers. if locals is None: locals = {} return_annotation = '' @@ -429,18 +433,17 @@ def _init_fn(fields, frozen, has_post_init, self_name): body_lines = [] for f in fields: - # Do not initialize the pseudo-fields, only the real ones. line = _field_init(f, frozen, globals, self_name) - if line is not None: - # line is None means that this field doesn't require - # initialization. Just skip it. + # line is None means that this field doesn't require + # initialization (it's a pseudo-field). Just skip it. + if line: body_lines.append(line) # Does this class have a post-init function? if has_post_init: params_str = ','.join(f.name for f in fields if f._field_type is _FIELD_INITVAR) - body_lines += [f'{self_name}.{_POST_INIT_NAME}({params_str})'] + body_lines.append(f'{self_name}.{_POST_INIT_NAME}({params_str})') # If no body lines, use 'pass'. if not body_lines: @@ -448,7 +451,7 @@ def _init_fn(fields, frozen, has_post_init, self_name): locals = {f'_type_{f.name}': f.type for f in fields} return _create_fn('__init__', - [self_name] +[_init_param(f) for f in fields if f.init], + [self_name] + [_init_param(f) for f in fields if f.init], body_lines, locals=locals, globals=globals, @@ -457,7 +460,7 @@ def _init_fn(fields, frozen, has_post_init, self_name): def _repr_fn(fields): return _create_fn('__repr__', - ['self'], + ('self',), ['return self.__class__.__qualname__ + f"(' + ', '.join([f"{f.name}={{self.{f.name}!r}}" for f in fields]) + @@ -496,7 +499,7 @@ def _cmp_fn(name, op, self_tuple, other_tuple): # '(other.x,other.y)'. return _create_fn(name, - ['self', 'other'], + ('self', 'other'), [ 'if other.__class__ is self.__class__:', f' return {self_tuple}{op}{other_tuple}', 'return NotImplemented']) @@ -505,12 +508,12 @@ def _cmp_fn(name, op, self_tuple, other_tuple): def _hash_fn(fields): self_tuple = _tuple_str('self', fields) return _create_fn('__hash__', - ['self'], + ('self',), [f'return hash({self_tuple})']) def _get_field(cls, a_name, a_type): - # Return a Field object, for this field name and type. ClassVars + # Return a Field object for this field name and type. ClassVars # and InitVars are also returned, but marked as such (see # f._field_type). @@ -560,9 +563,9 @@ def _get_field(cls, a_name, a_type): raise TypeError(f'field {f.name} cannot have a ' 'default factory') # Should I check for other field settings? default_factory - # seems the most serious to check for. Maybe add others. For - # example, how about init=False (or really, - # init=<not-the-default-init-value>)? It makes no sense for + # seems the most serious to check for. Maybe add others. + # For example, how about init=False (or really, + # init=<not-the-default-init-value>)? It makes no sense for # ClassVar and InitVar to specify init=<anything>. # For real fields, disallow mutable defaults for known types. @@ -903,7 +906,7 @@ def _asdict_inner(obj, dict_factory): return type(obj)((_asdict_inner(k, dict_factory), _asdict_inner(v, dict_factory)) for k, v in obj.items()) else: - return deepcopy(obj) + return copy.deepcopy(obj) def astuple(obj, *, tuple_factory=tuple): @@ -943,7 +946,7 @@ def _astuple_inner(obj, tuple_factory): return type(obj)((_astuple_inner(k, tuple_factory), _astuple_inner(v, tuple_factory)) for k, v in obj.items()) else: - return deepcopy(obj) + return copy.deepcopy(obj) def make_dataclass(cls_name, fields, *, bases=(), namespace=None, init=True, @@ -1032,9 +1035,9 @@ class C: if f.name not in changes: changes[f.name] = getattr(obj, f.name) - # Create the new object, which calls __init__() and __post_init__ - # (if defined), using all of the init fields we've added and/or - # left in 'changes'. - # If there are values supplied in changes that aren't fields, this - # will correctly raise a TypeError. + # Create the new object, which calls __init__() and + # __post_init__() (if defined), using all of the init fields + # we've added and/or left in 'changes'. If there are values + # supplied in changes that aren't fields, this will correctly + # raise a TypeError. return obj.__class__(**changes) From webhook-mailer at python.org Sat Mar 24 17:27:09 2018 From: webhook-mailer at python.org (Miss Islington (bot)) Date: Sat, 24 Mar 2018 21:27:09 -0000 Subject: [Python-checkins] bpo-33132: Fix reference counting issues in the compiler. (GH-6209) Message-ID: <mailman.166.1521926831.1871.python-checkins@python.org> https://github.com/python/cpython/commit/471364b4d977fc31bdf3012912954f24e4867d52 commit: 471364b4d977fc31bdf3012912954f24e4867d52 branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: GitHub <noreply at github.com> date: 2018-03-24T14:27:06-07:00 summary: bpo-33132: Fix reference counting issues in the compiler. (GH-6209) (cherry picked from commit a95d98607efe0c43475b354543e49bf8e240bc6f) Co-authored-by: Serhiy Storchaka <storchaka at gmail.com> files: M Python/compile.c diff --git a/Python/compile.c b/Python/compile.c index 618f31a47d36..03b4826b761b 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -2808,9 +2808,7 @@ static int compiler_from_import(struct compiler *c, stmt_ty s) { Py_ssize_t i, n = asdl_seq_LEN(s->v.ImportFrom.names); - - PyObject *names = PyTuple_New(n); - PyObject *level; + PyObject *level, *names; static PyObject *empty_string; if (!empty_string) { @@ -2819,14 +2817,15 @@ compiler_from_import(struct compiler *c, stmt_ty s) return 0; } - if (!names) - return 0; - level = PyLong_FromLong(s->v.ImportFrom.level); if (!level) { - Py_DECREF(names); return 0; } + ADDOP_N(c, LOAD_CONST, level, consts); + + names = PyTuple_New(n); + if (!names) + return 0; /* build up the names */ for (i = 0; i < n; i++) { @@ -2837,16 +2836,12 @@ compiler_from_import(struct compiler *c, stmt_ty s) if (s->lineno > c->c_future->ff_lineno && s->v.ImportFrom.module && _PyUnicode_EqualToASCIIString(s->v.ImportFrom.module, "__future__")) { - Py_DECREF(level); Py_DECREF(names); return compiler_error(c, "from __future__ imports must occur " "at the beginning of the file"); } + ADDOP_N(c, LOAD_CONST, names, consts); - ADDOP_O(c, LOAD_CONST, level, consts); - Py_DECREF(level); - ADDOP_O(c, LOAD_CONST, names, consts); - Py_DECREF(names); if (s->v.ImportFrom.module) { ADDOP_NAME(c, IMPORT_NAME, s->v.ImportFrom.module, names); } @@ -2869,7 +2864,6 @@ compiler_from_import(struct compiler *c, stmt_ty s) store_name = alias->asname; if (!compiler_nameop(c, store_name, Store)) { - Py_DECREF(names); return 0; } } @@ -4687,10 +4681,6 @@ compiler_annassign(struct compiler *c, stmt_ty s) if (s->v.AnnAssign.simple && (c->u->u_scope_type == COMPILER_SCOPE_MODULE || c->u->u_scope_type == COMPILER_SCOPE_CLASS)) { - mangled = _Py_Mangle(c->u->u_private, targ->v.Name.id); - if (!mangled) { - return 0; - } if (c->c_future->ff_features & CO_FUTURE_ANNOTATIONS) { VISIT(c, annexpr, s->v.AnnAssign.annotation) } @@ -4698,8 +4688,11 @@ compiler_annassign(struct compiler *c, stmt_ty s) VISIT(c, expr, s->v.AnnAssign.annotation); } ADDOP_NAME(c, LOAD_NAME, __annotations__, names); - ADDOP_O(c, LOAD_CONST, mangled, consts); - Py_DECREF(mangled); + mangled = _Py_Mangle(c->u->u_private, targ->v.Name.id); + if (!mangled) { + return 0; + } + ADDOP_N(c, LOAD_CONST, mangled, consts); ADDOP(c, STORE_SUBSCR); } break; From webhook-mailer at python.org Sat Mar 24 17:42:30 2018 From: webhook-mailer at python.org (Miss Islington (bot)) Date: Sat, 24 Mar 2018 21:42:30 -0000 Subject: [Python-checkins] Trivial dataclass cleanups: (GH-6218) Message-ID: <mailman.167.1521927751.1871.python-checkins@python.org> https://github.com/python/cpython/commit/02c19a6fc08dd37a3a89cf1b9800c869c338bd3c commit: 02c19a6fc08dd37a3a89cf1b9800c869c338bd3c branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: GitHub <noreply at github.com> date: 2018-03-24T14:42:28-07:00 summary: Trivial dataclass cleanups: (GH-6218) - When adding a single element to a list, use .append() instead of += and creating a new list. - For consistency, import the copy module, instead of just deepcopy. This leaves only a module at the class level, instead of a function. - Improve some comments. - Improve some whitespace. - Use tuples instead of lists. - Simplify a test. (cherry picked from commit f96ddade0094d162cb6c2fd7255c5e8a90b5c37d) Co-authored-by: Eric V. Smith <ericvsmith at users.noreply.github.com> files: M Lib/dataclasses.py diff --git a/Lib/dataclasses.py b/Lib/dataclasses.py index 5d4d4a6100ca..4425408b27d7 100644 --- a/Lib/dataclasses.py +++ b/Lib/dataclasses.py @@ -1,6 +1,6 @@ import sys +import copy import types -from copy import deepcopy import inspect __all__ = ['dataclass', @@ -137,7 +137,7 @@ # For boxes that are blank, __hash__ is untouched and therefore # inherited from the base class. If the base is object, then # id-based hashing is used. -# Note that a class may have already __hash__=None if it specified an +# Note that a class may already have __hash__=None if it specified an # __eq__ method in the class body (not one that was created by # @dataclass). # See _hash_action (below) for a coded version of this table. @@ -147,7 +147,7 @@ class FrozenInstanceError(AttributeError): pass # A sentinel object for default values to signal that a -# default-factory will be used. +# default factory will be used. # This is given a nice repr() which will appear in the function # signature of dataclasses' constructors. class _HAS_DEFAULT_FACTORY_CLASS: @@ -249,6 +249,7 @@ class _DataclassParams: 'unsafe_hash', 'frozen', ) + def __init__(self, init, repr, eq, order, unsafe_hash, frozen): self.init = init self.repr = repr @@ -267,6 +268,7 @@ def __repr__(self): f'frozen={self.frozen}' ')') + # This function is used instead of exposing Field creation directly, # so that a type checker can be told (via overloads) that this is a # function whose type depends on its parameters. @@ -307,6 +309,8 @@ def _tuple_str(obj_name, fields): def _create_fn(name, args, body, *, globals=None, locals=None, return_type=MISSING): # Note that we mutate locals when exec() is called. Caller beware! + # The only callers are internal to this module, so no worries + # about external callers. if locals is None: locals = {} return_annotation = '' @@ -429,18 +433,17 @@ def _init_fn(fields, frozen, has_post_init, self_name): body_lines = [] for f in fields: - # Do not initialize the pseudo-fields, only the real ones. line = _field_init(f, frozen, globals, self_name) - if line is not None: - # line is None means that this field doesn't require - # initialization. Just skip it. + # line is None means that this field doesn't require + # initialization (it's a pseudo-field). Just skip it. + if line: body_lines.append(line) # Does this class have a post-init function? if has_post_init: params_str = ','.join(f.name for f in fields if f._field_type is _FIELD_INITVAR) - body_lines += [f'{self_name}.{_POST_INIT_NAME}({params_str})'] + body_lines.append(f'{self_name}.{_POST_INIT_NAME}({params_str})') # If no body lines, use 'pass'. if not body_lines: @@ -448,7 +451,7 @@ def _init_fn(fields, frozen, has_post_init, self_name): locals = {f'_type_{f.name}': f.type for f in fields} return _create_fn('__init__', - [self_name] +[_init_param(f) for f in fields if f.init], + [self_name] + [_init_param(f) for f in fields if f.init], body_lines, locals=locals, globals=globals, @@ -457,7 +460,7 @@ def _init_fn(fields, frozen, has_post_init, self_name): def _repr_fn(fields): return _create_fn('__repr__', - ['self'], + ('self',), ['return self.__class__.__qualname__ + f"(' + ', '.join([f"{f.name}={{self.{f.name}!r}}" for f in fields]) + @@ -496,7 +499,7 @@ def _cmp_fn(name, op, self_tuple, other_tuple): # '(other.x,other.y)'. return _create_fn(name, - ['self', 'other'], + ('self', 'other'), [ 'if other.__class__ is self.__class__:', f' return {self_tuple}{op}{other_tuple}', 'return NotImplemented']) @@ -505,12 +508,12 @@ def _cmp_fn(name, op, self_tuple, other_tuple): def _hash_fn(fields): self_tuple = _tuple_str('self', fields) return _create_fn('__hash__', - ['self'], + ('self',), [f'return hash({self_tuple})']) def _get_field(cls, a_name, a_type): - # Return a Field object, for this field name and type. ClassVars + # Return a Field object for this field name and type. ClassVars # and InitVars are also returned, but marked as such (see # f._field_type). @@ -560,9 +563,9 @@ def _get_field(cls, a_name, a_type): raise TypeError(f'field {f.name} cannot have a ' 'default factory') # Should I check for other field settings? default_factory - # seems the most serious to check for. Maybe add others. For - # example, how about init=False (or really, - # init=<not-the-default-init-value>)? It makes no sense for + # seems the most serious to check for. Maybe add others. + # For example, how about init=False (or really, + # init=<not-the-default-init-value>)? It makes no sense for # ClassVar and InitVar to specify init=<anything>. # For real fields, disallow mutable defaults for known types. @@ -903,7 +906,7 @@ def _asdict_inner(obj, dict_factory): return type(obj)((_asdict_inner(k, dict_factory), _asdict_inner(v, dict_factory)) for k, v in obj.items()) else: - return deepcopy(obj) + return copy.deepcopy(obj) def astuple(obj, *, tuple_factory=tuple): @@ -943,7 +946,7 @@ def _astuple_inner(obj, tuple_factory): return type(obj)((_astuple_inner(k, tuple_factory), _astuple_inner(v, tuple_factory)) for k, v in obj.items()) else: - return deepcopy(obj) + return copy.deepcopy(obj) def make_dataclass(cls_name, fields, *, bases=(), namespace=None, init=True, @@ -1032,9 +1035,9 @@ class C: if f.name not in changes: changes[f.name] = getattr(obj, f.name) - # Create the new object, which calls __init__() and __post_init__ - # (if defined), using all of the init fields we've added and/or - # left in 'changes'. - # If there are values supplied in changes that aren't fields, this - # will correctly raise a TypeError. + # Create the new object, which calls __init__() and + # __post_init__() (if defined), using all of the init fields + # we've added and/or left in 'changes'. If there are values + # supplied in changes that aren't fields, this will correctly + # raise a TypeError. return obj.__class__(**changes) From webhook-mailer at python.org Sat Mar 24 22:10:17 2018 From: webhook-mailer at python.org (Eric V. Smith) Date: Sun, 25 Mar 2018 02:10:17 -0000 Subject: [Python-checkins] bpo-33134: dataclasses: use function dispatch table for hash, instead of a string lookup which then is tested with if tests. (GH-6222) Message-ID: <mailman.168.1521943818.1871.python-checkins@python.org> https://github.com/python/cpython/commit/01d618c5606a239b03ad1269541eddb6e724775d commit: 01d618c5606a239b03ad1269541eddb6e724775d branch: master author: Eric V. Smith <ericvsmith at users.noreply.github.com> committer: GitHub <noreply at github.com> date: 2018-03-24T22:10:14-04:00 summary: bpo-33134: dataclasses: use function dispatch table for hash, instead of a string lookup which then is tested with if tests. (GH-6222) * Change _hash_action to be a function table lookup, instead of a list of strings which is then tested with if statements. files: A Misc/NEWS.d/next/Library/2018-03-24-19-34-26.bpo-33134.hbVeIX.rst M Lib/dataclasses.py diff --git a/Lib/dataclasses.py b/Lib/dataclasses.py index 4425408b27d7..8ccc4c88aeca 100644 --- a/Lib/dataclasses.py +++ b/Lib/dataclasses.py @@ -585,14 +585,24 @@ def _set_new_attribute(cls, name, value): return False + # Decide if/how we're going to create a hash function. Key is # (unsafe_hash, eq, frozen, does-hash-exist). Value is the action to -# take. -# Actions: -# '': Do nothing. -# 'none': Set __hash__ to None. -# 'add': Always add a generated __hash__function. -# 'exception': Raise an exception. +# take. The common case is to do nothing, so instead of providing a +# function that is a no-op, use None to signify that. + +def _hash_set_none(cls, fields): + return None + +def _hash_add(cls, fields): + flds = [f for f in fields if (f.compare if f.hash is None else f.hash)] + return _hash_fn(flds) + +def _hash_exception(cls, fields): + # Raise an exception. + raise TypeError(f'Cannot overwrite attribute __hash__ ' + f'in class {cls.__name__}') + # # +-------------------------------------- unsafe_hash? # | +------------------------------- eq? @@ -602,22 +612,22 @@ def _set_new_attribute(cls, name, value): # | | | | +------- action # | | | | | # v v v v v -_hash_action = {(False, False, False, False): (''), - (False, False, False, True ): (''), - (False, False, True, False): (''), - (False, False, True, True ): (''), - (False, True, False, False): ('none'), - (False, True, False, True ): (''), - (False, True, True, False): ('add'), - (False, True, True, True ): (''), - (True, False, False, False): ('add'), - (True, False, False, True ): ('exception'), - (True, False, True, False): ('add'), - (True, False, True, True ): ('exception'), - (True, True, False, False): ('add'), - (True, True, False, True ): ('exception'), - (True, True, True, False): ('add'), - (True, True, True, True ): ('exception'), +_hash_action = {(False, False, False, False): None, + (False, False, False, True ): None, + (False, False, True, False): None, + (False, False, True, True ): None, + (False, True, False, False): _hash_set_none, + (False, True, False, True ): None, + (False, True, True, False): _hash_add, + (False, True, True, True ): None, + (True, False, False, False): _hash_add, + (True, False, False, True ): _hash_exception, + (True, False, True, False): _hash_add, + (True, False, True, True ): _hash_exception, + (True, True, False, False): _hash_add, + (True, True, False, True ): _hash_exception, + (True, True, True, False): _hash_add, + (True, True, True, True ): _hash_exception, } # See https://bugs.python.org/issue32929#msg312829 for an if-statement # version of this table. @@ -774,7 +784,6 @@ def _process_class(cls, init, repr, eq, order, unsafe_hash, frozen): 'functools.total_ordering') if frozen: - # XXX: Which fields are frozen? InitVar? ClassVar? hashed-only? for fn in _frozen_get_del_attr(cls, field_list): if _set_new_attribute(cls, fn.__name__, fn): raise TypeError(f'Cannot overwrite attribute {fn.__name__} ' @@ -785,23 +794,10 @@ def _process_class(cls, init, repr, eq, order, unsafe_hash, frozen): bool(eq), bool(frozen), has_explicit_hash] - - # No need to call _set_new_attribute here, since we already know if - # we're overwriting a __hash__ or not. - if hash_action == '': - # Do nothing. - pass - elif hash_action == 'none': - cls.__hash__ = None - elif hash_action == 'add': - flds = [f for f in field_list if (f.compare if f.hash is None else f.hash)] - cls.__hash__ = _hash_fn(flds) - elif hash_action == 'exception': - # Raise an exception. - raise TypeError(f'Cannot overwrite attribute __hash__ ' - f'in class {cls.__name__}') - else: - assert False, f"can't get here: {hash_action}" + if hash_action: + # No need to call _set_new_attribute here, since by the time + # we're here the overwriting is unconditional. + cls.__hash__ = hash_action(cls, field_list) if not getattr(cls, '__doc__'): # Create a class doc-string. diff --git a/Misc/NEWS.d/next/Library/2018-03-24-19-34-26.bpo-33134.hbVeIX.rst b/Misc/NEWS.d/next/Library/2018-03-24-19-34-26.bpo-33134.hbVeIX.rst new file mode 100644 index 000000000000..3f4ce84bef76 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-03-24-19-34-26.bpo-33134.hbVeIX.rst @@ -0,0 +1,3 @@ +When computing dataclass's __hash__, use the lookup table to contain the +function which returns the __hash__ value. This is an improvement over +looking up a string, and then testing that string to see what to do. From webhook-mailer at python.org Sat Mar 24 22:31:32 2018 From: webhook-mailer at python.org (Miss Islington (bot)) Date: Sun, 25 Mar 2018 02:31:32 -0000 Subject: [Python-checkins] bpo-33134: dataclasses: use function dispatch table for hash, instead of a string lookup which then is tested with if tests. (GH-6222) Message-ID: <mailman.169.1521945094.1871.python-checkins@python.org> https://github.com/python/cpython/commit/9989efbb653e8cbd08e51b4d79d094605c8b23b8 commit: 9989efbb653e8cbd08e51b4d79d094605c8b23b8 branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: GitHub <noreply at github.com> date: 2018-03-24T19:31:29-07:00 summary: bpo-33134: dataclasses: use function dispatch table for hash, instead of a string lookup which then is tested with if tests. (GH-6222) * Change _hash_action to be a function table lookup, instead of a list of strings which is then tested with if statements. (cherry picked from commit 01d618c5606a239b03ad1269541eddb6e724775d) Co-authored-by: Eric V. Smith <ericvsmith at users.noreply.github.com> files: A Misc/NEWS.d/next/Library/2018-03-24-19-34-26.bpo-33134.hbVeIX.rst M Lib/dataclasses.py diff --git a/Lib/dataclasses.py b/Lib/dataclasses.py index 4425408b27d7..8ccc4c88aeca 100644 --- a/Lib/dataclasses.py +++ b/Lib/dataclasses.py @@ -585,14 +585,24 @@ def _set_new_attribute(cls, name, value): return False + # Decide if/how we're going to create a hash function. Key is # (unsafe_hash, eq, frozen, does-hash-exist). Value is the action to -# take. -# Actions: -# '': Do nothing. -# 'none': Set __hash__ to None. -# 'add': Always add a generated __hash__function. -# 'exception': Raise an exception. +# take. The common case is to do nothing, so instead of providing a +# function that is a no-op, use None to signify that. + +def _hash_set_none(cls, fields): + return None + +def _hash_add(cls, fields): + flds = [f for f in fields if (f.compare if f.hash is None else f.hash)] + return _hash_fn(flds) + +def _hash_exception(cls, fields): + # Raise an exception. + raise TypeError(f'Cannot overwrite attribute __hash__ ' + f'in class {cls.__name__}') + # # +-------------------------------------- unsafe_hash? # | +------------------------------- eq? @@ -602,22 +612,22 @@ def _set_new_attribute(cls, name, value): # | | | | +------- action # | | | | | # v v v v v -_hash_action = {(False, False, False, False): (''), - (False, False, False, True ): (''), - (False, False, True, False): (''), - (False, False, True, True ): (''), - (False, True, False, False): ('none'), - (False, True, False, True ): (''), - (False, True, True, False): ('add'), - (False, True, True, True ): (''), - (True, False, False, False): ('add'), - (True, False, False, True ): ('exception'), - (True, False, True, False): ('add'), - (True, False, True, True ): ('exception'), - (True, True, False, False): ('add'), - (True, True, False, True ): ('exception'), - (True, True, True, False): ('add'), - (True, True, True, True ): ('exception'), +_hash_action = {(False, False, False, False): None, + (False, False, False, True ): None, + (False, False, True, False): None, + (False, False, True, True ): None, + (False, True, False, False): _hash_set_none, + (False, True, False, True ): None, + (False, True, True, False): _hash_add, + (False, True, True, True ): None, + (True, False, False, False): _hash_add, + (True, False, False, True ): _hash_exception, + (True, False, True, False): _hash_add, + (True, False, True, True ): _hash_exception, + (True, True, False, False): _hash_add, + (True, True, False, True ): _hash_exception, + (True, True, True, False): _hash_add, + (True, True, True, True ): _hash_exception, } # See https://bugs.python.org/issue32929#msg312829 for an if-statement # version of this table. @@ -774,7 +784,6 @@ def _process_class(cls, init, repr, eq, order, unsafe_hash, frozen): 'functools.total_ordering') if frozen: - # XXX: Which fields are frozen? InitVar? ClassVar? hashed-only? for fn in _frozen_get_del_attr(cls, field_list): if _set_new_attribute(cls, fn.__name__, fn): raise TypeError(f'Cannot overwrite attribute {fn.__name__} ' @@ -785,23 +794,10 @@ def _process_class(cls, init, repr, eq, order, unsafe_hash, frozen): bool(eq), bool(frozen), has_explicit_hash] - - # No need to call _set_new_attribute here, since we already know if - # we're overwriting a __hash__ or not. - if hash_action == '': - # Do nothing. - pass - elif hash_action == 'none': - cls.__hash__ = None - elif hash_action == 'add': - flds = [f for f in field_list if (f.compare if f.hash is None else f.hash)] - cls.__hash__ = _hash_fn(flds) - elif hash_action == 'exception': - # Raise an exception. - raise TypeError(f'Cannot overwrite attribute __hash__ ' - f'in class {cls.__name__}') - else: - assert False, f"can't get here: {hash_action}" + if hash_action: + # No need to call _set_new_attribute here, since by the time + # we're here the overwriting is unconditional. + cls.__hash__ = hash_action(cls, field_list) if not getattr(cls, '__doc__'): # Create a class doc-string. diff --git a/Misc/NEWS.d/next/Library/2018-03-24-19-34-26.bpo-33134.hbVeIX.rst b/Misc/NEWS.d/next/Library/2018-03-24-19-34-26.bpo-33134.hbVeIX.rst new file mode 100644 index 000000000000..3f4ce84bef76 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-03-24-19-34-26.bpo-33134.hbVeIX.rst @@ -0,0 +1,3 @@ +When computing dataclass's __hash__, use the lookup table to contain the +function which returns the __hash__ value. This is an improvement over +looking up a string, and then testing that string to see what to do. From webhook-mailer at python.org Sat Mar 24 23:02:25 2018 From: webhook-mailer at python.org (Eric V. Smith) Date: Sun, 25 Mar 2018 03:02:25 -0000 Subject: [Python-checkins] Fix invalid escape sequence: use raw string. (GH-6225) Message-ID: <mailman.170.1521946946.1871.python-checkins@python.org> https://github.com/python/cpython/commit/c42e7aa67ce72a6c4c6cdfe3b0929ca07556d444 commit: c42e7aa67ce72a6c4c6cdfe3b0929ca07556d444 branch: master author: Eric V. Smith <ericvsmith at users.noreply.github.com> committer: GitHub <noreply at github.com> date: 2018-03-24T23:02:21-04:00 summary: Fix invalid escape sequence: use raw string. (GH-6225) files: M Lib/test/test_dataclasses.py diff --git a/Lib/test/test_dataclasses.py b/Lib/test/test_dataclasses.py index 8aff8ae140a5..75e3cffc4a5e 100755 --- a/Lib/test/test_dataclasses.py +++ b/Lib/test/test_dataclasses.py @@ -2667,7 +2667,7 @@ class C: # There was a bug where a variable in a slot was assumed # to also have a default value (of type types.MemberDescriptorType). with self.assertRaisesRegex(TypeError, - "__init__\(\) missing 1 required positional argument: 'x'"): + r"__init__\(\) missing 1 required positional argument: 'x'"): C() # We can create an instance, and assign to x. From webhook-mailer at python.org Sat Mar 24 23:23:06 2018 From: webhook-mailer at python.org (Miss Islington (bot)) Date: Sun, 25 Mar 2018 03:23:06 -0000 Subject: [Python-checkins] Fix invalid escape sequence: use raw string. (GH-6225) Message-ID: <mailman.171.1521948186.1871.python-checkins@python.org> https://github.com/python/cpython/commit/5729b9c0e9833ec8387d1bdaef619a2e66060a28 commit: 5729b9c0e9833ec8387d1bdaef619a2e66060a28 branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: GitHub <noreply at github.com> date: 2018-03-24T20:23:00-07:00 summary: Fix invalid escape sequence: use raw string. (GH-6225) (cherry picked from commit c42e7aa67ce72a6c4c6cdfe3b0929ca07556d444) Co-authored-by: Eric V. Smith <ericvsmith at users.noreply.github.com> files: M Lib/test/test_dataclasses.py diff --git a/Lib/test/test_dataclasses.py b/Lib/test/test_dataclasses.py index 8aff8ae140a5..75e3cffc4a5e 100755 --- a/Lib/test/test_dataclasses.py +++ b/Lib/test/test_dataclasses.py @@ -2667,7 +2667,7 @@ class C: # There was a bug where a variable in a slot was assumed # to also have a default value (of type types.MemberDescriptorType). with self.assertRaisesRegex(TypeError, - "__init__\(\) missing 1 required positional argument: 'x'"): + r"__init__\(\) missing 1 required positional argument: 'x'"): C() # We can create an instance, and assign to x. From webhook-mailer at python.org Sun Mar 25 00:09:28 2018 From: webhook-mailer at python.org (Xiang Zhang) Date: Sun, 25 Mar 2018 04:09:28 -0000 Subject: [Python-checkins] bpo-32943: Fix confusing error message for rot13 codec (GH-5869) Message-ID: <mailman.172.1521950970.1871.python-checkins@python.org> https://github.com/python/cpython/commit/e4ce9fa89cb542dced553710b05de85202bc4715 commit: e4ce9fa89cb542dced553710b05de85202bc4715 branch: master author: Xiang Zhang <angwerzx at 126.com> committer: GitHub <noreply at github.com> date: 2018-03-25T12:09:21+08:00 summary: bpo-32943: Fix confusing error message for rot13 codec (GH-5869) files: M Lib/encodings/rot_13.py diff --git a/Lib/encodings/rot_13.py b/Lib/encodings/rot_13.py index f0b4186dc877..5627bfbc69cb 100755 --- a/Lib/encodings/rot_13.py +++ b/Lib/encodings/rot_13.py @@ -12,18 +12,18 @@ class Codec(codecs.Codec): def encode(self, input, errors='strict'): - return (input.translate(rot13_map), len(input)) + return (str.translate(input, rot13_map), len(input)) def decode(self, input, errors='strict'): - return (input.translate(rot13_map), len(input)) + return (str.translate(input, rot13_map), len(input)) class IncrementalEncoder(codecs.IncrementalEncoder): def encode(self, input, final=False): - return input.translate(rot13_map) + return str.translate(input, rot13_map) class IncrementalDecoder(codecs.IncrementalDecoder): def decode(self, input, final=False): - return input.translate(rot13_map) + return str.translate(input, rot13_map) class StreamWriter(Codec,codecs.StreamWriter): pass From webhook-mailer at python.org Sun Mar 25 00:30:42 2018 From: webhook-mailer at python.org (Miss Islington (bot)) Date: Sun, 25 Mar 2018 04:30:42 -0000 Subject: [Python-checkins] bpo-32943: Fix confusing error message for rot13 codec (GH-5869) Message-ID: <mailman.173.1521952245.1871.python-checkins@python.org> https://github.com/python/cpython/commit/291d5f3e7195659f874fe56b0e3ed717b57597ee commit: 291d5f3e7195659f874fe56b0e3ed717b57597ee branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: GitHub <noreply at github.com> date: 2018-03-24T21:30:39-07:00 summary: bpo-32943: Fix confusing error message for rot13 codec (GH-5869) (cherry picked from commit e4ce9fa89cb542dced553710b05de85202bc4715) Co-authored-by: Xiang Zhang <angwerzx at 126.com> files: M Lib/encodings/rot_13.py diff --git a/Lib/encodings/rot_13.py b/Lib/encodings/rot_13.py index f0b4186dc877..5627bfbc69cb 100755 --- a/Lib/encodings/rot_13.py +++ b/Lib/encodings/rot_13.py @@ -12,18 +12,18 @@ class Codec(codecs.Codec): def encode(self, input, errors='strict'): - return (input.translate(rot13_map), len(input)) + return (str.translate(input, rot13_map), len(input)) def decode(self, input, errors='strict'): - return (input.translate(rot13_map), len(input)) + return (str.translate(input, rot13_map), len(input)) class IncrementalEncoder(codecs.IncrementalEncoder): def encode(self, input, final=False): - return input.translate(rot13_map) + return str.translate(input, rot13_map) class IncrementalDecoder(codecs.IncrementalDecoder): def decode(self, input, final=False): - return input.translate(rot13_map) + return str.translate(input, rot13_map) class StreamWriter(Codec,codecs.StreamWriter): pass From solipsis at pitrou.net Sun Mar 25 05:10:58 2018 From: solipsis at pitrou.net (solipsis at pitrou.net) Date: Sun, 25 Mar 2018 09:10:58 +0000 Subject: [Python-checkins] Daily reference leaks (4243df51fe43): sum=7 Message-ID: <20180325091058.1.46FE5FFD8EA6E4D6@psf.io> results for 4243df51fe43 on branch "default" -------------------------------------------- test_asyncio leaked [3, 0, 0] memory blocks, sum=3 test_collections leaked [0, 7, -7] memory blocks, sum=0 test_functools leaked [0, 3, 1] memory blocks, sum=4 Command line was: ['./python', '-m', 'test.regrtest', '-uall', '-R', '3:3:/home/psf-users/antoine/refleaks/reflogHRPf_L', '--timeout', '7200'] From webhook-mailer at python.org Sun Mar 25 06:36:17 2018 From: webhook-mailer at python.org (Christian Heimes) Date: Sun, 25 Mar 2018 10:36:17 -0000 Subject: [Python-checkins] bpo-33136: Harden ssl module against CVE-2018-8970 (GH-6229) Message-ID: <mailman.174.1521974180.1871.python-checkins@python.org> https://github.com/python/cpython/commit/d02ac25ab0879f1a6de6937573bf00a16b7bd22e commit: d02ac25ab0879f1a6de6937573bf00a16b7bd22e branch: master author: Christian Heimes <christian at python.org> committer: GitHub <noreply at github.com> date: 2018-03-25T12:36:13+02:00 summary: bpo-33136: Harden ssl module against CVE-2018-8970 (GH-6229) Harden ssl module against LibreSSL CVE-2018-8970. X509_VERIFY_PARAM_set1_host() is called with an explicit namelen. A new test ensures that NULL bytes are not allowed. Signed-off-by: Christian Heimes <christian at python.org> files: A Misc/NEWS.d/next/Security/2018-03-25-12-05-43.bpo-33136.TzSN4x.rst M Lib/test/test_ssl.py M Modules/_ssl.c diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py index 8d98b805b49a..36580d55b9e2 100644 --- a/Lib/test/test_ssl.py +++ b/Lib/test/test_ssl.py @@ -1660,6 +1660,9 @@ def test_bad_server_hostname(self): with self.assertRaises(ValueError): ctx.wrap_bio(ssl.MemoryBIO(), ssl.MemoryBIO(), server_hostname=".example.org") + with self.assertRaises(TypeError): + ctx.wrap_bio(ssl.MemoryBIO(), ssl.MemoryBIO(), + server_hostname="example.org\x00evil.com") class MemoryBIOTests(unittest.TestCase): diff --git a/Misc/NEWS.d/next/Security/2018-03-25-12-05-43.bpo-33136.TzSN4x.rst b/Misc/NEWS.d/next/Security/2018-03-25-12-05-43.bpo-33136.TzSN4x.rst new file mode 100644 index 000000000000..c3505167092b --- /dev/null +++ b/Misc/NEWS.d/next/Security/2018-03-25-12-05-43.bpo-33136.TzSN4x.rst @@ -0,0 +1,3 @@ +Harden ssl module against LibreSSL CVE-2018-8970. +X509_VERIFY_PARAM_set1_host() is called with an explicit namelen. A new test +ensures that NULL bytes are not allowed. diff --git a/Modules/_ssl.c b/Modules/_ssl.c index 30c340376a4f..4baabd52bc9f 100644 --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -852,7 +852,8 @@ _ssl_configure_hostname(PySSLSocket *self, const char* server_hostname) if (self->ctx->check_hostname) { X509_VERIFY_PARAM *param = SSL_get0_param(self->ssl); if (ip == NULL) { - if (!X509_VERIFY_PARAM_set1_host(param, server_hostname, 0)) { + if (!X509_VERIFY_PARAM_set1_host(param, server_hostname, + strlen(server_hostname))) { _setSSLError(NULL, 0, __FILE__, __LINE__); goto error; } @@ -4025,7 +4026,7 @@ _ssl__SSLContext__wrap_socket_impl(PySSLContext *self, PyObject *sock, PyObject *res; /* server_hostname is either None (or absent), or to be encoded - as IDN A-label (ASCII str). */ + as IDN A-label (ASCII str) without NULL bytes. */ if (hostname_obj != Py_None) { if (!PyArg_Parse(hostname_obj, "es", "ascii", &hostname)) return NULL; @@ -4063,7 +4064,7 @@ _ssl__SSLContext__wrap_bio_impl(PySSLContext *self, PySSLMemoryBIO *incoming, PyObject *res; /* server_hostname is either None (or absent), or to be encoded - as IDN A-label (ASCII str). */ + as IDN A-label (ASCII str) without NULL bytes. */ if (hostname_obj != Py_None) { if (!PyArg_Parse(hostname_obj, "es", "ascii", &hostname)) return NULL; From webhook-mailer at python.org Sun Mar 25 06:44:33 2018 From: webhook-mailer at python.org (Nick Coghlan) Date: Sun, 25 Mar 2018 10:44:33 -0000 Subject: [Python-checkins] bpo-33042: Fix pre-initialization sys module configuration (GH-6157) Message-ID: <mailman.175.1521974673.1871.python-checkins@python.org> https://github.com/python/cpython/commit/bc77eff8b96be4f035e665ab35c1d06e22f46491 commit: bc77eff8b96be4f035e665ab35c1d06e22f46491 branch: master author: Nick Coghlan <ncoghlan at gmail.com> committer: GitHub <noreply at github.com> date: 2018-03-25T20:44:30+10:00 summary: bpo-33042: Fix pre-initialization sys module configuration (GH-6157) - new test case for pre-initialization of sys.warnoptions and sys._xoptions - restored ability to call these APIs prior to Py_Initialize - updated the docs for the affected APIs to make it clear they can be called before Py_Initialize - also enhanced the existing embedding test cases to check for expected settings in the sys module files: A Misc/NEWS.d/next/C API/2018-03-20-21-43-09.bpo-33042.FPFp64.rst M Doc/c-api/init.rst M Doc/c-api/sys.rst M Doc/whatsnew/3.7.rst M Lib/test/test_embed.py M Programs/_testembed.c M Python/sysmodule.c diff --git a/Doc/c-api/init.rst b/Doc/c-api/init.rst index f2564c441472..694b4669eea8 100644 --- a/Doc/c-api/init.rst +++ b/Doc/c-api/init.rst @@ -31,6 +31,9 @@ The following functions can be safely called before Python is initialized: * :c:func:`Py_SetProgramName` * :c:func:`Py_SetPythonHome` * :c:func:`Py_SetStandardStreamEncoding` + * :c:func:`PySys_AddWarnOption` + * :c:func:`PySys_AddXOption` + * :c:func:`PySys_ResetWarnOptions` * Informative functions: diff --git a/Doc/c-api/sys.rst b/Doc/c-api/sys.rst index e4da96c493cd..994509aa50f2 100644 --- a/Doc/c-api/sys.rst +++ b/Doc/c-api/sys.rst @@ -205,16 +205,24 @@ accessible to C code. They all work with the current interpreter thread's .. c:function:: void PySys_ResetWarnOptions() - Reset :data:`sys.warnoptions` to an empty list. + Reset :data:`sys.warnoptions` to an empty list. This function may be + called prior to :c:func:`Py_Initialize`. .. c:function:: void PySys_AddWarnOption(const wchar_t *s) - Append *s* to :data:`sys.warnoptions`. + Append *s* to :data:`sys.warnoptions`. This function must be called prior + to :c:func:`Py_Initialize` in order to affect the warnings filter list. .. c:function:: void PySys_AddWarnOptionUnicode(PyObject *unicode) Append *unicode* to :data:`sys.warnoptions`. + Note: this function is not currently usable from outside the CPython + implementation, as it must be called prior to the implicit import of + :mod:`warnings` in :c:func:`Py_Initialize` to be effective, but can't be + called until enough of the runtime has been initialized to permit the + creation of Unicode objects. + .. c:function:: void PySys_SetPath(const wchar_t *path) Set :data:`sys.path` to a list object of paths found in *path* which should @@ -260,7 +268,8 @@ accessible to C code. They all work with the current interpreter thread's .. c:function:: void PySys_AddXOption(const wchar_t *s) Parse *s* as a set of :option:`-X` options and add them to the current - options mapping as returned by :c:func:`PySys_GetXOptions`. + options mapping as returned by :c:func:`PySys_GetXOptions`. This function + may be called prior to :c:func:`Py_Initialize`. .. versionadded:: 3.2 diff --git a/Doc/whatsnew/3.7.rst b/Doc/whatsnew/3.7.rst index 7eb19f0f82a6..0b5ad007f30f 100644 --- a/Doc/whatsnew/3.7.rst +++ b/Doc/whatsnew/3.7.rst @@ -951,6 +951,14 @@ Build and C API Changes second argument is *NULL* and the :c:type:`wchar_t*` string contains null characters. (Contributed by Serhiy Storchaka in :issue:`30708`.) +- Changes to the startup sequence and the management of dynamic memory + allocators mean that the long documented requirement to call + :c:func:`Py_Initialize` before calling most C API functions is now + relied on more heavily, and failing to abide by it may lead to segfaults in + embedding applications. See the :ref:`porting-to-python-37` section in this + document and the :ref:`pre-init-safe` section in the C API documentation + for more details. + Other CPython Implementation Changes ==================================== @@ -1098,6 +1106,7 @@ API and Feature Removals ``asyncio._overlapped``. Replace ``from asyncio import selectors`` with ``import selectors`` for example. +.. _porting-to-python-37: Porting to Python 3.7 ===================== @@ -1282,14 +1291,24 @@ Other CPython implementation changes ------------------------------------ * In preparation for potential future changes to the public CPython runtime - initialization API (see :pep:`432` for details), CPython's internal startup + initialization API (see :pep:`432` for an initial, but somewhat outdated, + draft), CPython's internal startup and configuration management logic has been significantly refactored. While these updates are intended to be entirely transparent to both embedding applications and users of the regular CPython CLI, they're being mentioned here as the refactoring changes the internal order of various operations during interpreter startup, and hence may uncover previously latent defects, either in embedding applications, or in CPython itself. - (Contributed by Nick Coghlan and Eric Snow as part of :issue:`22257`.) + (Initially contributed by Nick Coghlan and Eric Snow as part of + :issue:`22257`, and further updated by Nick, Eric, and Victor Stinner in a + number of other issues). Some known details affected: + + * :c:func:`PySys_AddWarnOptionUnicode` is not currently usable by embedding + applications due to the requirement to create a Unicode object prior to + calling `Py_Initialize`. Use :c:func:`PySys_AddWarnOption` instead. + * warnings filters added by an embedding application with + :c:func:`PySys_AddWarnOption` should now more consistently take precedence + over the default filters set by the interpreter * Due to changes in the way the default warnings filters are configured, setting :c:data:`Py_BytesWarningFlag` to a value greater than one is no longer diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index c7f45b59acc8..f926301b8466 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -51,7 +51,7 @@ def run_embedded_interpreter(self, *args, env=None): if p.returncode != 0 and support.verbose: print(f"--- {cmd} failed ---") print(f"stdout:\n{out}") - print(f"stderr:\n{out}") + print(f"stderr:\n{err}") print(f"------") self.assertEqual(p.returncode, 0, @@ -83,7 +83,7 @@ def run_repeated_init_and_subinterpreters(self): for line in out.splitlines(): if line == "--- Pass {} ---".format(numloops): self.assertEqual(len(current_run), 0) - if support.verbose: + if support.verbose > 1: print(line) numloops += 1 continue @@ -96,7 +96,7 @@ def run_repeated_init_and_subinterpreters(self): # Parse the line from the loop. The first line is the main # interpreter and the 3 afterward are subinterpreters. interp = Interp(*match.groups()) - if support.verbose: + if support.verbose > 1: print(interp) self.assertTrue(interp.interp) self.assertTrue(interp.tstate) @@ -190,12 +190,33 @@ def test_forced_io_encoding(self): def test_pre_initialization_api(self): """ - Checks the few parts of the C-API that work before the runtine + Checks some key parts of the C-API that need to work before the runtine is initialized (via Py_Initialize()). """ env = dict(os.environ, PYTHONPATH=os.pathsep.join(sys.path)) out, err = self.run_embedded_interpreter("pre_initialization_api", env=env) - self.assertEqual(out, '') + if sys.platform == "win32": + expected_path = self.test_exe + else: + expected_path = os.path.join(os.getcwd(), "spam") + expected_output = f"sys.executable: {expected_path}\n" + self.assertIn(expected_output, out) + self.assertEqual(err, '') + + def test_pre_initialization_sys_options(self): + """ + Checks that sys.warnoptions and sys._xoptions can be set before the + runtime is initialized (otherwise they won't be effective). + """ + env = dict(PYTHONPATH=os.pathsep.join(sys.path)) + out, err = self.run_embedded_interpreter( + "pre_initialization_sys_options", env=env) + expected_output = ( + "sys.warnoptions: ['once', 'module', 'default']\n" + "sys._xoptions: {'not_an_option': '1', 'also_not_an_option': '2'}\n" + "warnings.filters[:3]: ['default', 'module', 'once']\n" + ) + self.assertIn(expected_output, out) self.assertEqual(err, '') def test_bpo20891(self): diff --git a/Misc/NEWS.d/next/C API/2018-03-20-21-43-09.bpo-33042.FPFp64.rst b/Misc/NEWS.d/next/C API/2018-03-20-21-43-09.bpo-33042.FPFp64.rst new file mode 100644 index 000000000000..f840b55869cc --- /dev/null +++ b/Misc/NEWS.d/next/C API/2018-03-20-21-43-09.bpo-33042.FPFp64.rst @@ -0,0 +1,2 @@ +Embedding applications may once again call PySys_ResetWarnOptions, +PySys_AddWarnOption, and PySys_AddXOption prior to calling Py_Initialize. \ No newline at end of file diff --git a/Programs/_testembed.c b/Programs/_testembed.c index b3d7aa442d1a..09b7bf6d94fc 100644 --- a/Programs/_testembed.c +++ b/Programs/_testembed.c @@ -2,6 +2,7 @@ #include "pythread.h" #include <inttypes.h> #include <stdio.h> +#include <wchar.h> /********************************************************* * Embedded interpreter tests that need a custom exe @@ -130,23 +131,89 @@ static int test_forced_io_encoding(void) * Test parts of the C-API that work before initialization *********************************************************/ +/* The pre-initialization tests tend to break by segfaulting, so explicitly + * flushed progress messages make the broken API easier to find when they fail. + */ +#define _Py_EMBED_PREINIT_CHECK(msg) \ + do {printf(msg); fflush(stdout);} while (0); + static int test_pre_initialization_api(void) { /* Leading "./" ensures getpath.c can still find the standard library */ + _Py_EMBED_PREINIT_CHECK("Checking Py_DecodeLocale\n"); wchar_t *program = Py_DecodeLocale("./spam", NULL); if (program == NULL) { fprintf(stderr, "Fatal error: cannot decode program name\n"); return 1; } + _Py_EMBED_PREINIT_CHECK("Checking Py_SetProgramName\n"); Py_SetProgramName(program); + _Py_EMBED_PREINIT_CHECK("Initializing interpreter\n"); Py_Initialize(); + _Py_EMBED_PREINIT_CHECK("Check sys module contents\n"); + PyRun_SimpleString("import sys; " + "print('sys.executable:', sys.executable)"); + _Py_EMBED_PREINIT_CHECK("Finalizing interpreter\n"); Py_Finalize(); + _Py_EMBED_PREINIT_CHECK("Freeing memory allocated by Py_DecodeLocale\n"); PyMem_RawFree(program); return 0; } + +/* bpo-33042: Ensure embedding apps can predefine sys module options */ +static int test_pre_initialization_sys_options(void) +{ + /* We allocate a couple of the option dynamically, and then delete + * them before calling Py_Initialize. This ensures the interpreter isn't + * relying on the caller to keep the passed in strings alive. + */ + wchar_t *static_warnoption = L"once"; + wchar_t *static_xoption = L"also_not_an_option=2"; + size_t warnoption_len = wcslen(static_warnoption); + size_t xoption_len = wcslen(static_xoption); + wchar_t *dynamic_once_warnoption = calloc(warnoption_len+1, sizeof(wchar_t)); + wchar_t *dynamic_xoption = calloc(xoption_len+1, sizeof(wchar_t)); + wcsncpy(dynamic_once_warnoption, static_warnoption, warnoption_len+1); + wcsncpy(dynamic_xoption, static_xoption, xoption_len+1); + + _Py_EMBED_PREINIT_CHECK("Checking PySys_AddWarnOption\n"); + PySys_AddWarnOption(L"default"); + _Py_EMBED_PREINIT_CHECK("Checking PySys_ResetWarnOptions\n"); + PySys_ResetWarnOptions(); + _Py_EMBED_PREINIT_CHECK("Checking PySys_AddWarnOption linked list\n"); + PySys_AddWarnOption(dynamic_once_warnoption); + PySys_AddWarnOption(L"module"); + PySys_AddWarnOption(L"default"); + _Py_EMBED_PREINIT_CHECK("Checking PySys_AddXOption\n"); + PySys_AddXOption(L"not_an_option=1"); + PySys_AddXOption(dynamic_xoption); + + /* Delete the dynamic options early */ + free(dynamic_once_warnoption); + dynamic_once_warnoption = NULL; + free(dynamic_xoption); + dynamic_xoption = NULL; + + _Py_EMBED_PREINIT_CHECK("Initializing interpreter\n"); + _testembed_Py_Initialize(); + _Py_EMBED_PREINIT_CHECK("Check sys module contents\n"); + PyRun_SimpleString("import sys; " + "print('sys.warnoptions:', sys.warnoptions); " + "print('sys._xoptions:', sys._xoptions); " + "warnings = sys.modules['warnings']; " + "latest_filters = [f[0] for f in warnings.filters[:3]]; " + "print('warnings.filters[:3]:', latest_filters)"); + _Py_EMBED_PREINIT_CHECK("Finalizing interpreter\n"); + Py_Finalize(); + + return 0; +} + + +/* bpo-20891: Avoid race condition when initialising the GIL */ static void bpo20891_thread(void *lockp) { PyThread_type_lock lock = *((PyThread_type_lock*)lockp); @@ -217,6 +284,7 @@ static struct TestCase TestCases[] = { { "forced_io_encoding", test_forced_io_encoding }, { "repeated_init_and_subinterpreters", test_repeated_init_and_subinterpreters }, { "pre_initialization_api", test_pre_initialization_api }, + { "pre_initialization_sys_options", test_pre_initialization_sys_options }, { "bpo20891", test_bpo20891 }, { NULL, NULL } }; @@ -232,13 +300,13 @@ int main(int argc, char *argv[]) /* No match found, or no test name provided, so display usage */ printf("Python " PY_VERSION " _testembed executable for embedded interpreter tests\n" - "Normally executed via 'EmbeddingTests' in Lib/test/test_capi.py\n\n" + "Normally executed via 'EmbeddingTests' in Lib/test/test_embed.py\n\n" "Usage: %s TESTNAME\n\nAll available tests:\n", argv[0]); for (struct TestCase *tc = TestCases; tc && tc->name; tc++) { printf(" %s\n", tc->name); } - /* Non-zero exit code will cause test_capi.py tests to fail. + /* Non-zero exit code will cause test_embed.py tests to fail. This is intentional. */ return -1; } diff --git a/Python/sysmodule.c b/Python/sysmodule.c index fb1dcfa3afe8..7cecff67486a 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -1609,11 +1609,141 @@ list_builtin_module_names(void) return list; } +/* Pre-initialization support for sys.warnoptions and sys._xoptions + * + * Modern internal code paths: + * These APIs get called after _Py_InitializeCore and get to use the + * regular CPython list, dict, and unicode APIs. + * + * Legacy embedding code paths: + * The multi-phase initialization API isn't public yet, so embedding + * apps still need to be able configure sys.warnoptions and sys._xoptions + * before they call Py_Initialize. To support this, we stash copies of + * the supplied wchar * sequences in linked lists, and then migrate the + * contents of those lists to the sys module in _PyInitializeCore. + * + */ + +struct _preinit_entry { + wchar_t *value; + struct _preinit_entry *next; +}; + +typedef struct _preinit_entry *_Py_PreInitEntry; + +static _Py_PreInitEntry _preinit_warnoptions = NULL; +static _Py_PreInitEntry _preinit_xoptions = NULL; + +static _Py_PreInitEntry +_alloc_preinit_entry(const wchar_t *value) +{ + /* To get this to work, we have to initialize the runtime implicitly */ + _PyRuntime_Initialize(); + + /* Force default allocator, so we can ensure that it also gets used to + * destroy the linked list in _clear_preinit_entries. + */ + PyMemAllocatorEx old_alloc; + _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); + + _Py_PreInitEntry node = PyMem_RawCalloc(1, sizeof(*node)); + if (node != NULL) { + node->value = _PyMem_RawWcsdup(value); + if (node->value == NULL) { + PyMem_RawFree(node); + node = NULL; + }; + }; + + PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); + return node; +}; + +static int +_append_preinit_entry(_Py_PreInitEntry *optionlist, const wchar_t *value) +{ + _Py_PreInitEntry new_entry = _alloc_preinit_entry(value); + if (new_entry == NULL) { + return -1; + } + /* We maintain the linked list in this order so it's easy to play back + * the add commands in the same order later on in _Py_InitializeCore + */ + _Py_PreInitEntry last_entry = *optionlist; + if (last_entry == NULL) { + *optionlist = new_entry; + } else { + while (last_entry->next != NULL) { + last_entry = last_entry->next; + } + last_entry->next = new_entry; + } + return 0; +}; + +static void +_clear_preinit_entries(_Py_PreInitEntry *optionlist) +{ + _Py_PreInitEntry current = *optionlist; + *optionlist = NULL; + /* Deallocate the nodes and their contents using the default allocator */ + PyMemAllocatorEx old_alloc; + _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); + while (current != NULL) { + _Py_PreInitEntry next = current->next; + PyMem_RawFree(current->value); + PyMem_RawFree(current); + current = next; + } + PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); +}; + +static void +_clear_all_preinit_options(void) +{ + _clear_preinit_entries(&_preinit_warnoptions); + _clear_preinit_entries(&_preinit_xoptions); +} + +static int +_PySys_ReadPreInitOptions(void) +{ + /* Rerun the add commands with the actual sys module available */ + PyThreadState *tstate = PyThreadState_GET(); + if (tstate == NULL) { + /* Still don't have a thread state, so something is wrong! */ + return -1; + } + _Py_PreInitEntry entry = _preinit_warnoptions; + while (entry != NULL) { + PySys_AddWarnOption(entry->value); + entry = entry->next; + } + entry = _preinit_xoptions; + while (entry != NULL) { + PySys_AddXOption(entry->value); + entry = entry->next; + } + + _clear_all_preinit_options(); + return 0; +}; + static PyObject * get_warnoptions(void) { PyObject *warnoptions = _PySys_GetObjectId(&PyId_warnoptions); if (warnoptions == NULL || !PyList_Check(warnoptions)) { + /* PEP432 TODO: we can reach this if warnoptions is NULL in the main + * interpreter config. When that happens, we need to properly set + * the `warnoptions` reference in the main interpreter config as well. + * + * For Python 3.7, we shouldn't be able to get here due to the + * combination of how _PyMainInterpreter_ReadConfig and _PySys_EndInit + * work, but we expect 3.8+ to make the _PyMainInterpreter_ReadConfig + * call optional for embedding applications, thus making this + * reachable again. + */ Py_XDECREF(warnoptions); warnoptions = PyList_New(0); if (warnoptions == NULL) @@ -1630,6 +1760,12 @@ get_warnoptions(void) void PySys_ResetWarnOptions(void) { + PyThreadState *tstate = PyThreadState_GET(); + if (tstate == NULL) { + _clear_preinit_entries(&_preinit_warnoptions); + return; + } + PyObject *warnoptions = _PySys_GetObjectId(&PyId_warnoptions); if (warnoptions == NULL || !PyList_Check(warnoptions)) return; @@ -1658,6 +1794,11 @@ PySys_AddWarnOptionUnicode(PyObject *option) void PySys_AddWarnOption(const wchar_t *s) { + PyThreadState *tstate = PyThreadState_GET(); + if (tstate == NULL) { + _append_preinit_entry(&_preinit_warnoptions, s); + return; + } PyObject *unicode; unicode = PyUnicode_FromWideChar(s, -1); if (unicode == NULL) @@ -1678,6 +1819,16 @@ get_xoptions(void) { PyObject *xoptions = _PySys_GetObjectId(&PyId__xoptions); if (xoptions == NULL || !PyDict_Check(xoptions)) { + /* PEP432 TODO: we can reach this if xoptions is NULL in the main + * interpreter config. When that happens, we need to properly set + * the `xoptions` reference in the main interpreter config as well. + * + * For Python 3.7, we shouldn't be able to get here due to the + * combination of how _PyMainInterpreter_ReadConfig and _PySys_EndInit + * work, but we expect 3.8+ to make the _PyMainInterpreter_ReadConfig + * call optional for embedding applications, thus making this + * reachable again. + */ Py_XDECREF(xoptions); xoptions = PyDict_New(); if (xoptions == NULL) @@ -1730,6 +1881,11 @@ _PySys_AddXOptionWithError(const wchar_t *s) void PySys_AddXOption(const wchar_t *s) { + PyThreadState *tstate = PyThreadState_GET(); + if (tstate == NULL) { + _append_preinit_entry(&_preinit_xoptions, s); + return; + } if (_PySys_AddXOptionWithError(s) < 0) { /* No return value, therefore clear error state if possible */ if (_PyThreadState_UncheckedGet()) { @@ -2257,6 +2413,7 @@ _PySys_BeginInit(PyObject **sysmod) } *sysmod = m; + return _Py_INIT_OK(); type_init_failed: @@ -2333,6 +2490,11 @@ _PySys_EndInit(PyObject *sysdict, _PyMainInterpreterConfig *config) if (get_xoptions() == NULL) return -1; + /* Transfer any sys.warnoptions and sys._xoptions set directly + * by an embedding application from the linked list to the module. */ + if (_PySys_ReadPreInitOptions() != 0) + return -1; + if (PyErr_Occurred()) return -1; return 0; From webhook-mailer at python.org Sun Mar 25 07:27:59 2018 From: webhook-mailer at python.org (Miss Islington (bot)) Date: Sun, 25 Mar 2018 11:27:59 -0000 Subject: [Python-checkins] bpo-33042: Fix pre-initialization sys module configuration (GH-6157) Message-ID: <mailman.176.1521977283.1871.python-checkins@python.org> https://github.com/python/cpython/commit/c6d94c37f4fd863c73fbfbcc918fd23b458b5301 commit: c6d94c37f4fd863c73fbfbcc918fd23b458b5301 branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: GitHub <noreply at github.com> date: 2018-03-25T04:27:57-07:00 summary: bpo-33042: Fix pre-initialization sys module configuration (GH-6157) - new test case for pre-initialization of sys.warnoptions and sys._xoptions - restored ability to call these APIs prior to Py_Initialize - updated the docs for the affected APIs to make it clear they can be called before Py_Initialize - also enhanced the existing embedding test cases to check for expected settings in the sys module (cherry picked from commit bc77eff8b96be4f035e665ab35c1d06e22f46491) Co-authored-by: Nick Coghlan <ncoghlan at gmail.com> files: A Misc/NEWS.d/next/C API/2018-03-20-21-43-09.bpo-33042.FPFp64.rst M Doc/c-api/init.rst M Doc/c-api/sys.rst M Doc/whatsnew/3.7.rst M Lib/test/test_embed.py M Programs/_testembed.c M Python/sysmodule.c diff --git a/Doc/c-api/init.rst b/Doc/c-api/init.rst index f2564c441472..694b4669eea8 100644 --- a/Doc/c-api/init.rst +++ b/Doc/c-api/init.rst @@ -31,6 +31,9 @@ The following functions can be safely called before Python is initialized: * :c:func:`Py_SetProgramName` * :c:func:`Py_SetPythonHome` * :c:func:`Py_SetStandardStreamEncoding` + * :c:func:`PySys_AddWarnOption` + * :c:func:`PySys_AddXOption` + * :c:func:`PySys_ResetWarnOptions` * Informative functions: diff --git a/Doc/c-api/sys.rst b/Doc/c-api/sys.rst index e4da96c493cd..994509aa50f2 100644 --- a/Doc/c-api/sys.rst +++ b/Doc/c-api/sys.rst @@ -205,16 +205,24 @@ accessible to C code. They all work with the current interpreter thread's .. c:function:: void PySys_ResetWarnOptions() - Reset :data:`sys.warnoptions` to an empty list. + Reset :data:`sys.warnoptions` to an empty list. This function may be + called prior to :c:func:`Py_Initialize`. .. c:function:: void PySys_AddWarnOption(const wchar_t *s) - Append *s* to :data:`sys.warnoptions`. + Append *s* to :data:`sys.warnoptions`. This function must be called prior + to :c:func:`Py_Initialize` in order to affect the warnings filter list. .. c:function:: void PySys_AddWarnOptionUnicode(PyObject *unicode) Append *unicode* to :data:`sys.warnoptions`. + Note: this function is not currently usable from outside the CPython + implementation, as it must be called prior to the implicit import of + :mod:`warnings` in :c:func:`Py_Initialize` to be effective, but can't be + called until enough of the runtime has been initialized to permit the + creation of Unicode objects. + .. c:function:: void PySys_SetPath(const wchar_t *path) Set :data:`sys.path` to a list object of paths found in *path* which should @@ -260,7 +268,8 @@ accessible to C code. They all work with the current interpreter thread's .. c:function:: void PySys_AddXOption(const wchar_t *s) Parse *s* as a set of :option:`-X` options and add them to the current - options mapping as returned by :c:func:`PySys_GetXOptions`. + options mapping as returned by :c:func:`PySys_GetXOptions`. This function + may be called prior to :c:func:`Py_Initialize`. .. versionadded:: 3.2 diff --git a/Doc/whatsnew/3.7.rst b/Doc/whatsnew/3.7.rst index 7eb19f0f82a6..0b5ad007f30f 100644 --- a/Doc/whatsnew/3.7.rst +++ b/Doc/whatsnew/3.7.rst @@ -951,6 +951,14 @@ Build and C API Changes second argument is *NULL* and the :c:type:`wchar_t*` string contains null characters. (Contributed by Serhiy Storchaka in :issue:`30708`.) +- Changes to the startup sequence and the management of dynamic memory + allocators mean that the long documented requirement to call + :c:func:`Py_Initialize` before calling most C API functions is now + relied on more heavily, and failing to abide by it may lead to segfaults in + embedding applications. See the :ref:`porting-to-python-37` section in this + document and the :ref:`pre-init-safe` section in the C API documentation + for more details. + Other CPython Implementation Changes ==================================== @@ -1098,6 +1106,7 @@ API and Feature Removals ``asyncio._overlapped``. Replace ``from asyncio import selectors`` with ``import selectors`` for example. +.. _porting-to-python-37: Porting to Python 3.7 ===================== @@ -1282,14 +1291,24 @@ Other CPython implementation changes ------------------------------------ * In preparation for potential future changes to the public CPython runtime - initialization API (see :pep:`432` for details), CPython's internal startup + initialization API (see :pep:`432` for an initial, but somewhat outdated, + draft), CPython's internal startup and configuration management logic has been significantly refactored. While these updates are intended to be entirely transparent to both embedding applications and users of the regular CPython CLI, they're being mentioned here as the refactoring changes the internal order of various operations during interpreter startup, and hence may uncover previously latent defects, either in embedding applications, or in CPython itself. - (Contributed by Nick Coghlan and Eric Snow as part of :issue:`22257`.) + (Initially contributed by Nick Coghlan and Eric Snow as part of + :issue:`22257`, and further updated by Nick, Eric, and Victor Stinner in a + number of other issues). Some known details affected: + + * :c:func:`PySys_AddWarnOptionUnicode` is not currently usable by embedding + applications due to the requirement to create a Unicode object prior to + calling `Py_Initialize`. Use :c:func:`PySys_AddWarnOption` instead. + * warnings filters added by an embedding application with + :c:func:`PySys_AddWarnOption` should now more consistently take precedence + over the default filters set by the interpreter * Due to changes in the way the default warnings filters are configured, setting :c:data:`Py_BytesWarningFlag` to a value greater than one is no longer diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index c7f45b59acc8..f926301b8466 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -51,7 +51,7 @@ def run_embedded_interpreter(self, *args, env=None): if p.returncode != 0 and support.verbose: print(f"--- {cmd} failed ---") print(f"stdout:\n{out}") - print(f"stderr:\n{out}") + print(f"stderr:\n{err}") print(f"------") self.assertEqual(p.returncode, 0, @@ -83,7 +83,7 @@ def run_repeated_init_and_subinterpreters(self): for line in out.splitlines(): if line == "--- Pass {} ---".format(numloops): self.assertEqual(len(current_run), 0) - if support.verbose: + if support.verbose > 1: print(line) numloops += 1 continue @@ -96,7 +96,7 @@ def run_repeated_init_and_subinterpreters(self): # Parse the line from the loop. The first line is the main # interpreter and the 3 afterward are subinterpreters. interp = Interp(*match.groups()) - if support.verbose: + if support.verbose > 1: print(interp) self.assertTrue(interp.interp) self.assertTrue(interp.tstate) @@ -190,12 +190,33 @@ def test_forced_io_encoding(self): def test_pre_initialization_api(self): """ - Checks the few parts of the C-API that work before the runtine + Checks some key parts of the C-API that need to work before the runtine is initialized (via Py_Initialize()). """ env = dict(os.environ, PYTHONPATH=os.pathsep.join(sys.path)) out, err = self.run_embedded_interpreter("pre_initialization_api", env=env) - self.assertEqual(out, '') + if sys.platform == "win32": + expected_path = self.test_exe + else: + expected_path = os.path.join(os.getcwd(), "spam") + expected_output = f"sys.executable: {expected_path}\n" + self.assertIn(expected_output, out) + self.assertEqual(err, '') + + def test_pre_initialization_sys_options(self): + """ + Checks that sys.warnoptions and sys._xoptions can be set before the + runtime is initialized (otherwise they won't be effective). + """ + env = dict(PYTHONPATH=os.pathsep.join(sys.path)) + out, err = self.run_embedded_interpreter( + "pre_initialization_sys_options", env=env) + expected_output = ( + "sys.warnoptions: ['once', 'module', 'default']\n" + "sys._xoptions: {'not_an_option': '1', 'also_not_an_option': '2'}\n" + "warnings.filters[:3]: ['default', 'module', 'once']\n" + ) + self.assertIn(expected_output, out) self.assertEqual(err, '') def test_bpo20891(self): diff --git a/Misc/NEWS.d/next/C API/2018-03-20-21-43-09.bpo-33042.FPFp64.rst b/Misc/NEWS.d/next/C API/2018-03-20-21-43-09.bpo-33042.FPFp64.rst new file mode 100644 index 000000000000..f840b55869cc --- /dev/null +++ b/Misc/NEWS.d/next/C API/2018-03-20-21-43-09.bpo-33042.FPFp64.rst @@ -0,0 +1,2 @@ +Embedding applications may once again call PySys_ResetWarnOptions, +PySys_AddWarnOption, and PySys_AddXOption prior to calling Py_Initialize. \ No newline at end of file diff --git a/Programs/_testembed.c b/Programs/_testembed.c index b3d7aa442d1a..09b7bf6d94fc 100644 --- a/Programs/_testembed.c +++ b/Programs/_testembed.c @@ -2,6 +2,7 @@ #include "pythread.h" #include <inttypes.h> #include <stdio.h> +#include <wchar.h> /********************************************************* * Embedded interpreter tests that need a custom exe @@ -130,23 +131,89 @@ static int test_forced_io_encoding(void) * Test parts of the C-API that work before initialization *********************************************************/ +/* The pre-initialization tests tend to break by segfaulting, so explicitly + * flushed progress messages make the broken API easier to find when they fail. + */ +#define _Py_EMBED_PREINIT_CHECK(msg) \ + do {printf(msg); fflush(stdout);} while (0); + static int test_pre_initialization_api(void) { /* Leading "./" ensures getpath.c can still find the standard library */ + _Py_EMBED_PREINIT_CHECK("Checking Py_DecodeLocale\n"); wchar_t *program = Py_DecodeLocale("./spam", NULL); if (program == NULL) { fprintf(stderr, "Fatal error: cannot decode program name\n"); return 1; } + _Py_EMBED_PREINIT_CHECK("Checking Py_SetProgramName\n"); Py_SetProgramName(program); + _Py_EMBED_PREINIT_CHECK("Initializing interpreter\n"); Py_Initialize(); + _Py_EMBED_PREINIT_CHECK("Check sys module contents\n"); + PyRun_SimpleString("import sys; " + "print('sys.executable:', sys.executable)"); + _Py_EMBED_PREINIT_CHECK("Finalizing interpreter\n"); Py_Finalize(); + _Py_EMBED_PREINIT_CHECK("Freeing memory allocated by Py_DecodeLocale\n"); PyMem_RawFree(program); return 0; } + +/* bpo-33042: Ensure embedding apps can predefine sys module options */ +static int test_pre_initialization_sys_options(void) +{ + /* We allocate a couple of the option dynamically, and then delete + * them before calling Py_Initialize. This ensures the interpreter isn't + * relying on the caller to keep the passed in strings alive. + */ + wchar_t *static_warnoption = L"once"; + wchar_t *static_xoption = L"also_not_an_option=2"; + size_t warnoption_len = wcslen(static_warnoption); + size_t xoption_len = wcslen(static_xoption); + wchar_t *dynamic_once_warnoption = calloc(warnoption_len+1, sizeof(wchar_t)); + wchar_t *dynamic_xoption = calloc(xoption_len+1, sizeof(wchar_t)); + wcsncpy(dynamic_once_warnoption, static_warnoption, warnoption_len+1); + wcsncpy(dynamic_xoption, static_xoption, xoption_len+1); + + _Py_EMBED_PREINIT_CHECK("Checking PySys_AddWarnOption\n"); + PySys_AddWarnOption(L"default"); + _Py_EMBED_PREINIT_CHECK("Checking PySys_ResetWarnOptions\n"); + PySys_ResetWarnOptions(); + _Py_EMBED_PREINIT_CHECK("Checking PySys_AddWarnOption linked list\n"); + PySys_AddWarnOption(dynamic_once_warnoption); + PySys_AddWarnOption(L"module"); + PySys_AddWarnOption(L"default"); + _Py_EMBED_PREINIT_CHECK("Checking PySys_AddXOption\n"); + PySys_AddXOption(L"not_an_option=1"); + PySys_AddXOption(dynamic_xoption); + + /* Delete the dynamic options early */ + free(dynamic_once_warnoption); + dynamic_once_warnoption = NULL; + free(dynamic_xoption); + dynamic_xoption = NULL; + + _Py_EMBED_PREINIT_CHECK("Initializing interpreter\n"); + _testembed_Py_Initialize(); + _Py_EMBED_PREINIT_CHECK("Check sys module contents\n"); + PyRun_SimpleString("import sys; " + "print('sys.warnoptions:', sys.warnoptions); " + "print('sys._xoptions:', sys._xoptions); " + "warnings = sys.modules['warnings']; " + "latest_filters = [f[0] for f in warnings.filters[:3]]; " + "print('warnings.filters[:3]:', latest_filters)"); + _Py_EMBED_PREINIT_CHECK("Finalizing interpreter\n"); + Py_Finalize(); + + return 0; +} + + +/* bpo-20891: Avoid race condition when initialising the GIL */ static void bpo20891_thread(void *lockp) { PyThread_type_lock lock = *((PyThread_type_lock*)lockp); @@ -217,6 +284,7 @@ static struct TestCase TestCases[] = { { "forced_io_encoding", test_forced_io_encoding }, { "repeated_init_and_subinterpreters", test_repeated_init_and_subinterpreters }, { "pre_initialization_api", test_pre_initialization_api }, + { "pre_initialization_sys_options", test_pre_initialization_sys_options }, { "bpo20891", test_bpo20891 }, { NULL, NULL } }; @@ -232,13 +300,13 @@ int main(int argc, char *argv[]) /* No match found, or no test name provided, so display usage */ printf("Python " PY_VERSION " _testembed executable for embedded interpreter tests\n" - "Normally executed via 'EmbeddingTests' in Lib/test/test_capi.py\n\n" + "Normally executed via 'EmbeddingTests' in Lib/test/test_embed.py\n\n" "Usage: %s TESTNAME\n\nAll available tests:\n", argv[0]); for (struct TestCase *tc = TestCases; tc && tc->name; tc++) { printf(" %s\n", tc->name); } - /* Non-zero exit code will cause test_capi.py tests to fail. + /* Non-zero exit code will cause test_embed.py tests to fail. This is intentional. */ return -1; } diff --git a/Python/sysmodule.c b/Python/sysmodule.c index 873657f4a4f1..d68572d40033 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -1609,11 +1609,141 @@ list_builtin_module_names(void) return list; } +/* Pre-initialization support for sys.warnoptions and sys._xoptions + * + * Modern internal code paths: + * These APIs get called after _Py_InitializeCore and get to use the + * regular CPython list, dict, and unicode APIs. + * + * Legacy embedding code paths: + * The multi-phase initialization API isn't public yet, so embedding + * apps still need to be able configure sys.warnoptions and sys._xoptions + * before they call Py_Initialize. To support this, we stash copies of + * the supplied wchar * sequences in linked lists, and then migrate the + * contents of those lists to the sys module in _PyInitializeCore. + * + */ + +struct _preinit_entry { + wchar_t *value; + struct _preinit_entry *next; +}; + +typedef struct _preinit_entry *_Py_PreInitEntry; + +static _Py_PreInitEntry _preinit_warnoptions = NULL; +static _Py_PreInitEntry _preinit_xoptions = NULL; + +static _Py_PreInitEntry +_alloc_preinit_entry(const wchar_t *value) +{ + /* To get this to work, we have to initialize the runtime implicitly */ + _PyRuntime_Initialize(); + + /* Force default allocator, so we can ensure that it also gets used to + * destroy the linked list in _clear_preinit_entries. + */ + PyMemAllocatorEx old_alloc; + _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); + + _Py_PreInitEntry node = PyMem_RawCalloc(1, sizeof(*node)); + if (node != NULL) { + node->value = _PyMem_RawWcsdup(value); + if (node->value == NULL) { + PyMem_RawFree(node); + node = NULL; + }; + }; + + PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); + return node; +}; + +static int +_append_preinit_entry(_Py_PreInitEntry *optionlist, const wchar_t *value) +{ + _Py_PreInitEntry new_entry = _alloc_preinit_entry(value); + if (new_entry == NULL) { + return -1; + } + /* We maintain the linked list in this order so it's easy to play back + * the add commands in the same order later on in _Py_InitializeCore + */ + _Py_PreInitEntry last_entry = *optionlist; + if (last_entry == NULL) { + *optionlist = new_entry; + } else { + while (last_entry->next != NULL) { + last_entry = last_entry->next; + } + last_entry->next = new_entry; + } + return 0; +}; + +static void +_clear_preinit_entries(_Py_PreInitEntry *optionlist) +{ + _Py_PreInitEntry current = *optionlist; + *optionlist = NULL; + /* Deallocate the nodes and their contents using the default allocator */ + PyMemAllocatorEx old_alloc; + _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); + while (current != NULL) { + _Py_PreInitEntry next = current->next; + PyMem_RawFree(current->value); + PyMem_RawFree(current); + current = next; + } + PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); +}; + +static void +_clear_all_preinit_options(void) +{ + _clear_preinit_entries(&_preinit_warnoptions); + _clear_preinit_entries(&_preinit_xoptions); +} + +static int +_PySys_ReadPreInitOptions(void) +{ + /* Rerun the add commands with the actual sys module available */ + PyThreadState *tstate = PyThreadState_GET(); + if (tstate == NULL) { + /* Still don't have a thread state, so something is wrong! */ + return -1; + } + _Py_PreInitEntry entry = _preinit_warnoptions; + while (entry != NULL) { + PySys_AddWarnOption(entry->value); + entry = entry->next; + } + entry = _preinit_xoptions; + while (entry != NULL) { + PySys_AddXOption(entry->value); + entry = entry->next; + } + + _clear_all_preinit_options(); + return 0; +}; + static PyObject * get_warnoptions(void) { PyObject *warnoptions = _PySys_GetObjectId(&PyId_warnoptions); if (warnoptions == NULL || !PyList_Check(warnoptions)) { + /* PEP432 TODO: we can reach this if warnoptions is NULL in the main + * interpreter config. When that happens, we need to properly set + * the `warnoptions` reference in the main interpreter config as well. + * + * For Python 3.7, we shouldn't be able to get here due to the + * combination of how _PyMainInterpreter_ReadConfig and _PySys_EndInit + * work, but we expect 3.8+ to make the _PyMainInterpreter_ReadConfig + * call optional for embedding applications, thus making this + * reachable again. + */ Py_XDECREF(warnoptions); warnoptions = PyList_New(0); if (warnoptions == NULL) @@ -1630,6 +1760,12 @@ get_warnoptions(void) void PySys_ResetWarnOptions(void) { + PyThreadState *tstate = PyThreadState_GET(); + if (tstate == NULL) { + _clear_preinit_entries(&_preinit_warnoptions); + return; + } + PyObject *warnoptions = _PySys_GetObjectId(&PyId_warnoptions); if (warnoptions == NULL || !PyList_Check(warnoptions)) return; @@ -1658,6 +1794,11 @@ PySys_AddWarnOptionUnicode(PyObject *option) void PySys_AddWarnOption(const wchar_t *s) { + PyThreadState *tstate = PyThreadState_GET(); + if (tstate == NULL) { + _append_preinit_entry(&_preinit_warnoptions, s); + return; + } PyObject *unicode; unicode = PyUnicode_FromWideChar(s, -1); if (unicode == NULL) @@ -1678,6 +1819,16 @@ get_xoptions(void) { PyObject *xoptions = _PySys_GetObjectId(&PyId__xoptions); if (xoptions == NULL || !PyDict_Check(xoptions)) { + /* PEP432 TODO: we can reach this if xoptions is NULL in the main + * interpreter config. When that happens, we need to properly set + * the `xoptions` reference in the main interpreter config as well. + * + * For Python 3.7, we shouldn't be able to get here due to the + * combination of how _PyMainInterpreter_ReadConfig and _PySys_EndInit + * work, but we expect 3.8+ to make the _PyMainInterpreter_ReadConfig + * call optional for embedding applications, thus making this + * reachable again. + */ Py_XDECREF(xoptions); xoptions = PyDict_New(); if (xoptions == NULL) @@ -1730,6 +1881,11 @@ _PySys_AddXOptionWithError(const wchar_t *s) void PySys_AddXOption(const wchar_t *s) { + PyThreadState *tstate = PyThreadState_GET(); + if (tstate == NULL) { + _append_preinit_entry(&_preinit_xoptions, s); + return; + } if (_PySys_AddXOptionWithError(s) < 0) { /* No return value, therefore clear error state if possible */ if (_PyThreadState_UncheckedGet()) { @@ -2257,6 +2413,7 @@ _PySys_BeginInit(PyObject **sysmod) } *sysmod = m; + return _Py_INIT_OK(); type_init_failed: @@ -2333,6 +2490,11 @@ _PySys_EndInit(PyObject *sysdict, _PyMainInterpreterConfig *config) if (get_xoptions() == NULL) return -1; + /* Transfer any sys.warnoptions and sys._xoptions set directly + * by an embedding application from the linked list to the module. */ + if (_PySys_ReadPreInitOptions() != 0) + return -1; + if (PyErr_Occurred()) return -1; return 0; From webhook-mailer at python.org Sun Mar 25 07:28:23 2018 From: webhook-mailer at python.org (Christian Heimes) Date: Sun, 25 Mar 2018 11:28:23 -0000 Subject: [Python-checkins] [3.7] bpo-33136: Harden ssl module against CVE-2018-8970 (GH-6229) (GH-6230) Message-ID: <mailman.177.1521977304.1871.python-checkins@python.org> https://github.com/python/cpython/commit/2dd885eaa0d427e84892673c83d697bca5427c8b commit: 2dd885eaa0d427e84892673c83d697bca5427c8b branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Christian Heimes <christian at python.org> date: 2018-03-25T13:28:20+02:00 summary: [3.7] bpo-33136: Harden ssl module against CVE-2018-8970 (GH-6229) (GH-6230) Harden ssl module against LibreSSL CVE-2018-8970. X509_VERIFY_PARAM_set1_host() is called with an explicit namelen. A new test ensures that NULL bytes are not allowed. Signed-off-by: Christian Heimes <christian at python.org> (cherry picked from commit d02ac25ab0879f1a6de6937573bf00a16b7bd22e) Co-authored-by: Christian Heimes <christian at python.org> files: A Misc/NEWS.d/next/Security/2018-03-25-12-05-43.bpo-33136.TzSN4x.rst M Lib/test/test_ssl.py M Modules/_ssl.c diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py index 8d98b805b49a..36580d55b9e2 100644 --- a/Lib/test/test_ssl.py +++ b/Lib/test/test_ssl.py @@ -1660,6 +1660,9 @@ def test_bad_server_hostname(self): with self.assertRaises(ValueError): ctx.wrap_bio(ssl.MemoryBIO(), ssl.MemoryBIO(), server_hostname=".example.org") + with self.assertRaises(TypeError): + ctx.wrap_bio(ssl.MemoryBIO(), ssl.MemoryBIO(), + server_hostname="example.org\x00evil.com") class MemoryBIOTests(unittest.TestCase): diff --git a/Misc/NEWS.d/next/Security/2018-03-25-12-05-43.bpo-33136.TzSN4x.rst b/Misc/NEWS.d/next/Security/2018-03-25-12-05-43.bpo-33136.TzSN4x.rst new file mode 100644 index 000000000000..c3505167092b --- /dev/null +++ b/Misc/NEWS.d/next/Security/2018-03-25-12-05-43.bpo-33136.TzSN4x.rst @@ -0,0 +1,3 @@ +Harden ssl module against LibreSSL CVE-2018-8970. +X509_VERIFY_PARAM_set1_host() is called with an explicit namelen. A new test +ensures that NULL bytes are not allowed. diff --git a/Modules/_ssl.c b/Modules/_ssl.c index 30c340376a4f..4baabd52bc9f 100644 --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -852,7 +852,8 @@ _ssl_configure_hostname(PySSLSocket *self, const char* server_hostname) if (self->ctx->check_hostname) { X509_VERIFY_PARAM *param = SSL_get0_param(self->ssl); if (ip == NULL) { - if (!X509_VERIFY_PARAM_set1_host(param, server_hostname, 0)) { + if (!X509_VERIFY_PARAM_set1_host(param, server_hostname, + strlen(server_hostname))) { _setSSLError(NULL, 0, __FILE__, __LINE__); goto error; } @@ -4025,7 +4026,7 @@ _ssl__SSLContext__wrap_socket_impl(PySSLContext *self, PyObject *sock, PyObject *res; /* server_hostname is either None (or absent), or to be encoded - as IDN A-label (ASCII str). */ + as IDN A-label (ASCII str) without NULL bytes. */ if (hostname_obj != Py_None) { if (!PyArg_Parse(hostname_obj, "es", "ascii", &hostname)) return NULL; @@ -4063,7 +4064,7 @@ _ssl__SSLContext__wrap_bio_impl(PySSLContext *self, PySSLMemoryBIO *incoming, PyObject *res; /* server_hostname is either None (or absent), or to be encoded - as IDN A-label (ASCII str). */ + as IDN A-label (ASCII str) without NULL bytes. */ if (hostname_obj != Py_None) { if (!PyArg_Parse(hostname_obj, "es", "ascii", &hostname)) return NULL; From webhook-mailer at python.org Sun Mar 25 09:03:13 2018 From: webhook-mailer at python.org (Nick Coghlan) Date: Sun, 25 Mar 2018 13:03:13 -0000 Subject: [Python-checkins] bpo-33053: -m now adds *starting* directory to sys.path (GH-6231) Message-ID: <mailman.178.1521982994.1871.python-checkins@python.org> https://github.com/python/cpython/commit/d5d9e02dd3c6df06a8dd9ce75ee9b52976420a8b commit: d5d9e02dd3c6df06a8dd9ce75ee9b52976420a8b branch: master author: Nick Coghlan <ncoghlan at gmail.com> committer: GitHub <noreply at github.com> date: 2018-03-25T23:03:10+10:00 summary: bpo-33053: -m now adds *starting* directory to sys.path (GH-6231) Historically, -m added the empty string as sys.path zero, meaning it resolved imports against the current working directory, the same way -c and the interactive prompt do. This changes the sys.path initialisation to add the *starting* working directory as sys.path[0] instead, such that changes to the working directory while the program is running will have no effect on imports when using the -m switch. files: A Misc/NEWS.d/next/Core and Builtins/2018-03-25-19-49-06.bpo-33053.V3xlsH.rst M Doc/library/test.rst M Doc/whatsnew/3.7.rst M Lib/test/support/script_helper.py M Lib/test/test_bdb.py M Lib/test/test_cmd_line_script.py M Lib/test/test_doctest.py M Lib/test/test_import/__init__.py M Python/pathconfig.c diff --git a/Doc/library/test.rst b/Doc/library/test.rst index 6041f529c16f..0746fcfde0aa 100644 --- a/Doc/library/test.rst +++ b/Doc/library/test.rst @@ -1332,8 +1332,8 @@ script execution tests. .. function:: run_python_until_end(*args, **env_vars) Set up the environment based on *env_vars* for running the interpreter - in a subprocess. The values can include ``__isolated``, ``__cleavenv``, - and ``TERM``. + in a subprocess. The values can include ``__isolated``, ``__cleanenv``, + ``__cwd``, and ``TERM``. .. function:: assert_python_ok(*args, **env_vars) diff --git a/Doc/whatsnew/3.7.rst b/Doc/whatsnew/3.7.rst index 0b5ad007f30f..e0c19cfa3bd0 100644 --- a/Doc/whatsnew/3.7.rst +++ b/Doc/whatsnew/3.7.rst @@ -421,6 +421,12 @@ Other Language Changes writable. (Contributed by Nathaniel J. Smith in :issue:`30579`.) +* When using the :option:`-m` switch, ``sys.path[0]`` is now eagerly expanded + to the full starting directory path, rather than being left as the empty + directory (which allows imports from the *current* working directory at the + time when an import occurs) + (Contributed by Nick Coghlan in :issue:`33053`.) + New Modules =========== @@ -1138,6 +1144,11 @@ Changes in Python behavior parentheses can be omitted only on calls. (Contributed by Serhiy Storchaka in :issue:`32012` and :issue:`32023`.) +* When using the ``-m`` switch, the starting directory is now added to sys.path, + rather than the current working directory. Any programs that are found to be + relying on the previous behaviour will need to be updated to manipulate + :data:`sys.path` appropriately. + Changes in the Python API ------------------------- diff --git a/Lib/test/support/script_helper.py b/Lib/test/support/script_helper.py index 5a81697708fd..64b25aab8004 100644 --- a/Lib/test/support/script_helper.py +++ b/Lib/test/support/script_helper.py @@ -87,6 +87,7 @@ def fail(self, cmd_line): # Executing the interpreter in a subprocess def run_python_until_end(*args, **env_vars): env_required = interpreter_requires_environment() + cwd = env_vars.pop('__cwd', None) if '__isolated' in env_vars: isolated = env_vars.pop('__isolated') else: @@ -125,7 +126,7 @@ def run_python_until_end(*args, **env_vars): cmd_line.extend(args) proc = subprocess.Popen(cmd_line, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, - env=env) + env=env, cwd=cwd) with proc: try: out, err = proc.communicate() diff --git a/Lib/test/test_bdb.py b/Lib/test/test_bdb.py index abefe6c4e57a..bda74a2be775 100644 --- a/Lib/test/test_bdb.py +++ b/Lib/test/test_bdb.py @@ -524,13 +524,13 @@ def gen(a, b): test.id = lambda : None test.expect_set = list(gen(repeat(()), iter(sl))) with create_modules(modules): - sys.path.append(os.getcwd()) with TracerRun(test, skip=skip) as tracer: tracer.runcall(tfunc_import) @contextmanager def create_modules(modules): with test.support.temp_cwd(): + sys.path.append(os.getcwd()) try: for m in modules: fname = m + '.py' @@ -542,6 +542,7 @@ def create_modules(modules): finally: for m in modules: test.support.forget(m) + sys.path.pop() def break_in_func(funcname, fname=__file__, temporary=False, cond=None): return 'break', (fname, None, temporary, cond, funcname) diff --git a/Lib/test/test_cmd_line_script.py b/Lib/test/test_cmd_line_script.py index 0d0bcd784d26..176200855008 100644 --- a/Lib/test/test_cmd_line_script.py +++ b/Lib/test/test_cmd_line_script.py @@ -87,31 +87,11 @@ def _make_test_zip_pkg(zip_dir, zip_basename, pkg_name, script_basename, importlib.invalidate_caches() return to_return -# There's no easy way to pass the script directory in to get -# -m to work (avoiding that is the whole point of making -# directories and zipfiles executable!) -# So we fake it for testing purposes with a custom launch script -launch_source = """\ -import sys, os.path, runpy -sys.path.insert(0, %s) -runpy._run_module_as_main(%r) -""" - -def _make_launch_script(script_dir, script_basename, module_name, path=None): - if path is None: - path = "os.path.dirname(__file__)" - else: - path = repr(path) - source = launch_source % (path, module_name) - to_return = make_script(script_dir, script_basename, source) - importlib.invalidate_caches() - return to_return - class CmdLineTest(unittest.TestCase): def _check_output(self, script_name, exit_code, data, expected_file, expected_argv0, expected_path0, expected_package, - expected_loader): + expected_loader, expected_cwd=None): if verbose > 1: print("Output from test script %r:" % script_name) print(repr(data)) @@ -121,7 +101,9 @@ def _check_output(self, script_name, exit_code, data, printed_package = '__package__==%r' % expected_package printed_argv0 = 'sys.argv[0]==%a' % expected_argv0 printed_path0 = 'sys.path[0]==%a' % expected_path0 - printed_cwd = 'cwd==%a' % os.getcwd() + if expected_cwd is None: + expected_cwd = os.getcwd() + printed_cwd = 'cwd==%a' % expected_cwd if verbose > 1: print('Expected output:') print(printed_file) @@ -135,23 +117,35 @@ def _check_output(self, script_name, exit_code, data, self.assertIn(printed_path0.encode('utf-8'), data) self.assertIn(printed_cwd.encode('utf-8'), data) - def _check_script(self, script_name, expected_file, + def _check_script(self, script_exec_args, expected_file, expected_argv0, expected_path0, expected_package, expected_loader, - *cmd_line_switches): + *cmd_line_switches, cwd=None, **env_vars): + if isinstance(script_exec_args, str): + script_exec_args = [script_exec_args] run_args = [*support.optim_args_from_interpreter_flags(), - *cmd_line_switches, script_name, *example_args] - rc, out, err = assert_python_ok(*run_args, __isolated=False) - self._check_output(script_name, rc, out + err, expected_file, + *cmd_line_switches, *script_exec_args, *example_args] + if env_vars: + print(env_vars) + rc, out, err = assert_python_ok( + *run_args, __isolated=False, __cwd=cwd, **env_vars + ) + self._check_output(script_exec_args, rc, out + err, expected_file, expected_argv0, expected_path0, - expected_package, expected_loader) + expected_package, expected_loader, cwd) - def _check_import_error(self, script_name, expected_msg, - *cmd_line_switches): - run_args = cmd_line_switches + (script_name,) - rc, out, err = assert_python_failure(*run_args) + def _check_import_error(self, script_exec_args, expected_msg, + *cmd_line_switches, cwd=None, **env_vars): + if isinstance(script_exec_args, str): + script_exec_args = (script_exec_args,) + else: + script_exec_args = tuple(script_exec_args) + run_args = cmd_line_switches + script_exec_args + rc, out, err = assert_python_failure( + *run_args, __isolated=False, __cwd=cwd, **env_vars + ) if verbose > 1: - print('Output from test script %r:' % script_name) + print('Output from test script %r:' % script_exec_args) print(repr(err)) print('Expected output: %r' % expected_msg) self.assertIn(expected_msg.encode('utf-8'), err) @@ -287,35 +281,35 @@ def test_module_in_package(self): pkg_dir = os.path.join(script_dir, 'test_pkg') make_pkg(pkg_dir) script_name = _make_test_script(pkg_dir, 'script') - launch_name = _make_launch_script(script_dir, 'launch', 'test_pkg.script') - self._check_script(launch_name, script_name, script_name, + self._check_script(["-m", "test_pkg.script"], script_name, script_name, script_dir, 'test_pkg', - importlib.machinery.SourceFileLoader) + importlib.machinery.SourceFileLoader, + cwd=script_dir) def test_module_in_package_in_zipfile(self): with support.temp_dir() as script_dir: zip_name, run_name = _make_test_zip_pkg(script_dir, 'test_zip', 'test_pkg', 'script') - launch_name = _make_launch_script(script_dir, 'launch', 'test_pkg.script', zip_name) - self._check_script(launch_name, run_name, run_name, - zip_name, 'test_pkg', zipimport.zipimporter) + self._check_script(["-m", "test_pkg.script"], run_name, run_name, + script_dir, 'test_pkg', zipimport.zipimporter, + PYTHONPATH=zip_name, cwd=script_dir) def test_module_in_subpackage_in_zipfile(self): with support.temp_dir() as script_dir: zip_name, run_name = _make_test_zip_pkg(script_dir, 'test_zip', 'test_pkg', 'script', depth=2) - launch_name = _make_launch_script(script_dir, 'launch', 'test_pkg.test_pkg.script', zip_name) - self._check_script(launch_name, run_name, run_name, - zip_name, 'test_pkg.test_pkg', - zipimport.zipimporter) + self._check_script(["-m", "test_pkg.test_pkg.script"], run_name, run_name, + script_dir, 'test_pkg.test_pkg', + zipimport.zipimporter, + PYTHONPATH=zip_name, cwd=script_dir) def test_package(self): with support.temp_dir() as script_dir: pkg_dir = os.path.join(script_dir, 'test_pkg') make_pkg(pkg_dir) script_name = _make_test_script(pkg_dir, '__main__') - launch_name = _make_launch_script(script_dir, 'launch', 'test_pkg') - self._check_script(launch_name, script_name, + self._check_script(["-m", "test_pkg"], script_name, script_name, script_dir, 'test_pkg', - importlib.machinery.SourceFileLoader) + importlib.machinery.SourceFileLoader, + cwd=script_dir) def test_package_compiled(self): with support.temp_dir() as script_dir: @@ -325,10 +319,10 @@ def test_package_compiled(self): compiled_name = py_compile.compile(script_name, doraise=True) os.remove(script_name) pyc_file = support.make_legacy_pyc(script_name) - launch_name = _make_launch_script(script_dir, 'launch', 'test_pkg') - self._check_script(launch_name, pyc_file, + self._check_script(["-m", "test_pkg"], pyc_file, pyc_file, script_dir, 'test_pkg', - importlib.machinery.SourcelessFileLoader) + importlib.machinery.SourcelessFileLoader, + cwd=script_dir) def test_package_error(self): with support.temp_dir() as script_dir: @@ -336,8 +330,7 @@ def test_package_error(self): make_pkg(pkg_dir) msg = ("'test_pkg' is a package and cannot " "be directly executed") - launch_name = _make_launch_script(script_dir, 'launch', 'test_pkg') - self._check_import_error(launch_name, msg) + self._check_import_error(["-m", "test_pkg"], msg, cwd=script_dir) def test_package_recursion(self): with support.temp_dir() as script_dir: @@ -348,8 +341,7 @@ def test_package_recursion(self): msg = ("Cannot use package as __main__ module; " "'test_pkg' is a package and cannot " "be directly executed") - launch_name = _make_launch_script(script_dir, 'launch', 'test_pkg') - self._check_import_error(launch_name, msg) + self._check_import_error(["-m", "test_pkg"], msg, cwd=script_dir) def test_issue8202(self): # Make sure package __init__ modules see "-m" in sys.argv0 while @@ -365,7 +357,7 @@ def test_issue8202(self): expected = "init_argv0==%r" % '-m' self.assertIn(expected.encode('utf-8'), out) self._check_output(script_name, rc, out, - script_name, script_name, '', 'test_pkg', + script_name, script_name, script_dir, 'test_pkg', importlib.machinery.SourceFileLoader) def test_issue8202_dash_c_file_ignored(self): @@ -394,7 +386,7 @@ def test_issue8202_dash_m_file_ignored(self): rc, out, err = assert_python_ok('-m', 'other', *example_args, __isolated=False) self._check_output(script_name, rc, out, - script_name, script_name, '', '', + script_name, script_name, script_dir, '', importlib.machinery.SourceFileLoader) @contextlib.contextmanager @@ -627,7 +619,7 @@ def test_consistent_sys_path_for_module_execution(self): # direct execution test cases p = spawn_python("-sm", "script_pkg.__main__", cwd=work_dir) out_by_module = kill_python(p).decode().splitlines() - self.assertEqual(out_by_module[0], '') + self.assertEqual(out_by_module[0], work_dir) self.assertNotIn(script_dir, out_by_module) # Package execution should give the same output p = spawn_python("-sm", "script_pkg", cwd=work_dir) diff --git a/Lib/test/test_doctest.py b/Lib/test/test_doctest.py index f0eb52881bcf..83941c129f44 100644 --- a/Lib/test/test_doctest.py +++ b/Lib/test/test_doctest.py @@ -9,7 +9,7 @@ import sys import importlib import unittest - +import tempfile # NOTE: There are some additional tests relating to interaction with # zipimport in the test_zipimport_support test module. @@ -688,10 +688,16 @@ class TestDocTestFinder(unittest.TestCase): def test_empty_namespace_package(self): pkg_name = 'doctest_empty_pkg' - os.mkdir(pkg_name) - mod = importlib.import_module(pkg_name) - assert doctest.DocTestFinder().find(mod) == [] - os.rmdir(pkg_name) + with tempfile.TemporaryDirectory() as parent_dir: + pkg_dir = os.path.join(parent_dir, pkg_name) + os.mkdir(pkg_dir) + sys.path.append(parent_dir) + try: + mod = importlib.import_module(pkg_name) + finally: + support.forget(pkg_name) + sys.path.pop() + assert doctest.DocTestFinder().find(mod) == [] def test_DocTestParser(): r""" diff --git a/Lib/test/test_import/__init__.py b/Lib/test/test_import/__init__.py index 606b05784afc..c23fac1ecc24 100644 --- a/Lib/test/test_import/__init__.py +++ b/Lib/test/test_import/__init__.py @@ -118,7 +118,7 @@ def test_from_import_star_invalid_type(self): f.write("__all__ = [b'invalid_type']") globals = {} with self.assertRaisesRegex( - TypeError, f"{re.escape(name)}\.__all__ must be str" + TypeError, f"{re.escape(name)}\\.__all__ must be str" ): exec(f"from {name} import *", globals) self.assertNotIn(b"invalid_type", globals) @@ -127,7 +127,7 @@ def test_from_import_star_invalid_type(self): f.write("globals()[b'invalid_type'] = object()") globals = {} with self.assertRaisesRegex( - TypeError, f"{re.escape(name)}\.__dict__ must be str" + TypeError, f"{re.escape(name)}\\.__dict__ must be str" ): exec(f"from {name} import *", globals) self.assertNotIn(b"invalid_type", globals) @@ -847,8 +847,11 @@ def test_missing_source_legacy(self): unload(TESTFN) importlib.invalidate_caches() m = __import__(TESTFN) - self.assertEqual(m.__file__, - os.path.join(os.curdir, os.path.relpath(pyc_file))) + try: + self.assertEqual(m.__file__, + os.path.join(os.curdir, os.path.relpath(pyc_file))) + finally: + os.remove(pyc_file) def test___cached__(self): # Modules now also have an __cached__ that points to the pyc file. diff --git a/Misc/NEWS.d/next/Core and Builtins/2018-03-25-19-49-06.bpo-33053.V3xlsH.rst b/Misc/NEWS.d/next/Core and Builtins/2018-03-25-19-49-06.bpo-33053.V3xlsH.rst new file mode 100644 index 000000000000..fd32ac150e4c --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2018-03-25-19-49-06.bpo-33053.V3xlsH.rst @@ -0,0 +1,4 @@ +When using the -m switch, sys.path[0] is now explicitly expanded as the +*starting* working directory, rather than being left as the empty path +(which allows imports from the current working directory at the time of the +import) diff --git a/Python/pathconfig.c b/Python/pathconfig.c index 6de5481bc8da..07aa01c0304f 100644 --- a/Python/pathconfig.c +++ b/Python/pathconfig.c @@ -3,6 +3,7 @@ #include "Python.h" #include "osdefs.h" #include "internal/pystate.h" +#include <wchar.h> #ifdef __cplusplus extern "C" { @@ -255,11 +256,6 @@ Py_GetProgramName(void) return _Py_path_config.program_name; } - -#define _HAVE_SCRIPT_ARGUMENT(argc, argv) \ - (argc > 0 && argv0 != NULL && \ - wcscmp(argv0, L"-c") != 0 && wcscmp(argv0, L"-m") != 0) - /* Compute argv[0] which will be prepended to sys.argv */ PyObject* _PyPathConfig_ComputeArgv0(int argc, wchar_t **argv) @@ -267,6 +263,8 @@ _PyPathConfig_ComputeArgv0(int argc, wchar_t **argv) wchar_t *argv0; wchar_t *p = NULL; Py_ssize_t n = 0; + int have_script_arg = 0; + int have_module_arg = 0; #ifdef HAVE_READLINK wchar_t link[MAXPATHLEN+1]; wchar_t argv0copy[2*MAXPATHLEN+1]; @@ -278,11 +276,25 @@ _PyPathConfig_ComputeArgv0(int argc, wchar_t **argv) wchar_t fullpath[MAX_PATH]; #endif - argv0 = argv[0]; + if (argc > 0 && argv0 != NULL) { + have_module_arg = (wcscmp(argv0, L"-m") == 0); + have_script_arg = !have_module_arg && (wcscmp(argv0, L"-c") != 0); + } + + if (have_module_arg) { + #if defined(HAVE_REALPATH) || defined(MS_WINDOWS) + _Py_wgetcwd(fullpath, Py_ARRAY_LENGTH(fullpath)); + argv0 = fullpath; + n = wcslen(argv0); + #else + argv0 = L"."; + n = 1; + #endif + } #ifdef HAVE_READLINK - if (_HAVE_SCRIPT_ARGUMENT(argc, argv)) + if (have_script_arg) nr = _Py_wreadlink(argv0, link, MAXPATHLEN); if (nr > 0) { /* It's a symlink */ @@ -310,7 +322,7 @@ _PyPathConfig_ComputeArgv0(int argc, wchar_t **argv) #if SEP == '\\' /* Special case for Microsoft filename syntax */ - if (_HAVE_SCRIPT_ARGUMENT(argc, argv)) { + if (have_script_arg) { wchar_t *q; #if defined(MS_WINDOWS) /* Replace the first element in argv with the full path. */ @@ -334,7 +346,7 @@ _PyPathConfig_ComputeArgv0(int argc, wchar_t **argv) } } #else /* All other filename syntaxes */ - if (_HAVE_SCRIPT_ARGUMENT(argc, argv)) { + if (have_script_arg) { #if defined(HAVE_REALPATH) if (_Py_wrealpath(argv0, fullpath, Py_ARRAY_LENGTH(fullpath))) { argv0 = fullpath; From webhook-mailer at python.org Sun Mar 25 09:04:36 2018 From: webhook-mailer at python.org (Eric V. Smith) Date: Sun, 25 Mar 2018 13:04:36 -0000 Subject: [Python-checkins] Trivial improvements to dataclasses tests. (GH-6234) Message-ID: <mailman.179.1521983077.1871.python-checkins@python.org> https://github.com/python/cpython/commit/51c9ab42ab84643081d75c83a586afa45d910383 commit: 51c9ab42ab84643081d75c83a586afa45d910383 branch: master author: Eric V. Smith <ericvsmith at users.noreply.github.com> committer: GitHub <noreply at github.com> date: 2018-03-25T09:04:32-04:00 summary: Trivial improvements to dataclasses tests. (GH-6234) files: M Lib/test/test_dataclasses.py diff --git a/Lib/test/test_dataclasses.py b/Lib/test/test_dataclasses.py index 75e3cffc4a5e..df53b040c0e1 100755 --- a/Lib/test/test_dataclasses.py +++ b/Lib/test/test_dataclasses.py @@ -805,6 +805,7 @@ def bar(self) -> int: self.assertEqual(list(C.__annotations__), ['i']) self.assertEqual(C(10).foo(), 4) self.assertEqual(C(10).bar, 5) + self.assertEqual(C(10).i, 10) def test_post_init(self): # Just make sure it gets called @@ -1488,7 +1489,7 @@ def nt(lst): self.assertIs(type(t), NT) def test_dynamic_class_creation(self): - cls_dict = {'__annotations__': OrderedDict(x=int, y=int), + cls_dict = {'__annotations__': {'x':int, 'y':int}, } # Create the class. @@ -1501,7 +1502,7 @@ def test_dynamic_class_creation(self): self.assertEqual(asdict(cls(1, 2)), {'x': 1, 'y': 2}) def test_dynamic_class_creation_using_field(self): - cls_dict = {'__annotations__': OrderedDict(x=int, y=int), + cls_dict = {'__annotations__': {'x':int, 'y':int}, 'y': field(default=5), } From webhook-mailer at python.org Sun Mar 25 09:27:53 2018 From: webhook-mailer at python.org (Miss Islington (bot)) Date: Sun, 25 Mar 2018 13:27:53 -0000 Subject: [Python-checkins] Trivial improvements to dataclasses tests. (GH-6234) Message-ID: <mailman.180.1521984474.1871.python-checkins@python.org> https://github.com/python/cpython/commit/5666a55da89491bae717e67ad098ac31e310a8e1 commit: 5666a55da89491bae717e67ad098ac31e310a8e1 branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: GitHub <noreply at github.com> date: 2018-03-25T06:27:50-07:00 summary: Trivial improvements to dataclasses tests. (GH-6234) (cherry picked from commit 51c9ab42ab84643081d75c83a586afa45d910383) Co-authored-by: Eric V. Smith <ericvsmith at users.noreply.github.com> files: M Lib/test/test_dataclasses.py diff --git a/Lib/test/test_dataclasses.py b/Lib/test/test_dataclasses.py index 75e3cffc4a5e..df53b040c0e1 100755 --- a/Lib/test/test_dataclasses.py +++ b/Lib/test/test_dataclasses.py @@ -805,6 +805,7 @@ def bar(self) -> int: self.assertEqual(list(C.__annotations__), ['i']) self.assertEqual(C(10).foo(), 4) self.assertEqual(C(10).bar, 5) + self.assertEqual(C(10).i, 10) def test_post_init(self): # Just make sure it gets called @@ -1488,7 +1489,7 @@ def nt(lst): self.assertIs(type(t), NT) def test_dynamic_class_creation(self): - cls_dict = {'__annotations__': OrderedDict(x=int, y=int), + cls_dict = {'__annotations__': {'x':int, 'y':int}, } # Create the class. @@ -1501,7 +1502,7 @@ def test_dynamic_class_creation(self): self.assertEqual(asdict(cls(1, 2)), {'x': 1, 'y': 2}) def test_dynamic_class_creation_using_field(self): - cls_dict = {'__annotations__': OrderedDict(x=int, y=int), + cls_dict = {'__annotations__': {'x':int, 'y':int}, 'y': field(default=5), } From webhook-mailer at python.org Sun Mar 25 09:43:53 2018 From: webhook-mailer at python.org (Nick Coghlan) Date: Sun, 25 Mar 2018 13:43:53 -0000 Subject: [Python-checkins] bpo-33053: -m now adds *starting* directory to sys.path (GH-6231) (#6236) Message-ID: <mailman.181.1521985436.1871.python-checkins@python.org> https://github.com/python/cpython/commit/ee3784594b33c72c3fdca6a71892d22f14045ab6 commit: ee3784594b33c72c3fdca6a71892d22f14045ab6 branch: 3.7 author: Nick Coghlan <ncoghlan at gmail.com> committer: GitHub <noreply at github.com> date: 2018-03-25T23:43:50+10:00 summary: bpo-33053: -m now adds *starting* directory to sys.path (GH-6231) (#6236) Historically, -m added the empty string as sys.path zero, meaning it resolved imports against the current working directory, the same way -c and the interactive prompt do. This changes the sys.path initialisation to add the *starting* working directory as sys.path[0] instead, such that changes to the working directory while the program is running will have no effect on imports when using the -m switch. (cherry picked from commit d5d9e02dd3c6df06a8dd9ce75ee9b52976420a8b) files: A Misc/NEWS.d/next/Core and Builtins/2018-03-25-19-49-06.bpo-33053.V3xlsH.rst M Doc/library/test.rst M Doc/whatsnew/3.7.rst M Lib/test/support/script_helper.py M Lib/test/test_bdb.py M Lib/test/test_cmd_line_script.py M Lib/test/test_doctest.py M Lib/test/test_import/__init__.py M Python/pathconfig.c diff --git a/Doc/library/test.rst b/Doc/library/test.rst index 6041f529c16f..0746fcfde0aa 100644 --- a/Doc/library/test.rst +++ b/Doc/library/test.rst @@ -1332,8 +1332,8 @@ script execution tests. .. function:: run_python_until_end(*args, **env_vars) Set up the environment based on *env_vars* for running the interpreter - in a subprocess. The values can include ``__isolated``, ``__cleavenv``, - and ``TERM``. + in a subprocess. The values can include ``__isolated``, ``__cleanenv``, + ``__cwd``, and ``TERM``. .. function:: assert_python_ok(*args, **env_vars) diff --git a/Doc/whatsnew/3.7.rst b/Doc/whatsnew/3.7.rst index 0b5ad007f30f..e0c19cfa3bd0 100644 --- a/Doc/whatsnew/3.7.rst +++ b/Doc/whatsnew/3.7.rst @@ -421,6 +421,12 @@ Other Language Changes writable. (Contributed by Nathaniel J. Smith in :issue:`30579`.) +* When using the :option:`-m` switch, ``sys.path[0]`` is now eagerly expanded + to the full starting directory path, rather than being left as the empty + directory (which allows imports from the *current* working directory at the + time when an import occurs) + (Contributed by Nick Coghlan in :issue:`33053`.) + New Modules =========== @@ -1138,6 +1144,11 @@ Changes in Python behavior parentheses can be omitted only on calls. (Contributed by Serhiy Storchaka in :issue:`32012` and :issue:`32023`.) +* When using the ``-m`` switch, the starting directory is now added to sys.path, + rather than the current working directory. Any programs that are found to be + relying on the previous behaviour will need to be updated to manipulate + :data:`sys.path` appropriately. + Changes in the Python API ------------------------- diff --git a/Lib/test/support/script_helper.py b/Lib/test/support/script_helper.py index 5a81697708fd..64b25aab8004 100644 --- a/Lib/test/support/script_helper.py +++ b/Lib/test/support/script_helper.py @@ -87,6 +87,7 @@ def fail(self, cmd_line): # Executing the interpreter in a subprocess def run_python_until_end(*args, **env_vars): env_required = interpreter_requires_environment() + cwd = env_vars.pop('__cwd', None) if '__isolated' in env_vars: isolated = env_vars.pop('__isolated') else: @@ -125,7 +126,7 @@ def run_python_until_end(*args, **env_vars): cmd_line.extend(args) proc = subprocess.Popen(cmd_line, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, - env=env) + env=env, cwd=cwd) with proc: try: out, err = proc.communicate() diff --git a/Lib/test/test_bdb.py b/Lib/test/test_bdb.py index abefe6c4e57a..bda74a2be775 100644 --- a/Lib/test/test_bdb.py +++ b/Lib/test/test_bdb.py @@ -524,13 +524,13 @@ def gen(a, b): test.id = lambda : None test.expect_set = list(gen(repeat(()), iter(sl))) with create_modules(modules): - sys.path.append(os.getcwd()) with TracerRun(test, skip=skip) as tracer: tracer.runcall(tfunc_import) @contextmanager def create_modules(modules): with test.support.temp_cwd(): + sys.path.append(os.getcwd()) try: for m in modules: fname = m + '.py' @@ -542,6 +542,7 @@ def create_modules(modules): finally: for m in modules: test.support.forget(m) + sys.path.pop() def break_in_func(funcname, fname=__file__, temporary=False, cond=None): return 'break', (fname, None, temporary, cond, funcname) diff --git a/Lib/test/test_cmd_line_script.py b/Lib/test/test_cmd_line_script.py index 0d0bcd784d26..a9c1309ad6d3 100644 --- a/Lib/test/test_cmd_line_script.py +++ b/Lib/test/test_cmd_line_script.py @@ -87,31 +87,11 @@ def _make_test_zip_pkg(zip_dir, zip_basename, pkg_name, script_basename, importlib.invalidate_caches() return to_return -# There's no easy way to pass the script directory in to get -# -m to work (avoiding that is the whole point of making -# directories and zipfiles executable!) -# So we fake it for testing purposes with a custom launch script -launch_source = """\ -import sys, os.path, runpy -sys.path.insert(0, %s) -runpy._run_module_as_main(%r) -""" - -def _make_launch_script(script_dir, script_basename, module_name, path=None): - if path is None: - path = "os.path.dirname(__file__)" - else: - path = repr(path) - source = launch_source % (path, module_name) - to_return = make_script(script_dir, script_basename, source) - importlib.invalidate_caches() - return to_return - class CmdLineTest(unittest.TestCase): def _check_output(self, script_name, exit_code, data, expected_file, expected_argv0, expected_path0, expected_package, - expected_loader): + expected_loader, expected_cwd=None): if verbose > 1: print("Output from test script %r:" % script_name) print(repr(data)) @@ -121,7 +101,9 @@ def _check_output(self, script_name, exit_code, data, printed_package = '__package__==%r' % expected_package printed_argv0 = 'sys.argv[0]==%a' % expected_argv0 printed_path0 = 'sys.path[0]==%a' % expected_path0 - printed_cwd = 'cwd==%a' % os.getcwd() + if expected_cwd is None: + expected_cwd = os.getcwd() + printed_cwd = 'cwd==%a' % expected_cwd if verbose > 1: print('Expected output:') print(printed_file) @@ -135,23 +117,33 @@ def _check_output(self, script_name, exit_code, data, self.assertIn(printed_path0.encode('utf-8'), data) self.assertIn(printed_cwd.encode('utf-8'), data) - def _check_script(self, script_name, expected_file, + def _check_script(self, script_exec_args, expected_file, expected_argv0, expected_path0, expected_package, expected_loader, - *cmd_line_switches): + *cmd_line_switches, cwd=None, **env_vars): + if isinstance(script_exec_args, str): + script_exec_args = [script_exec_args] run_args = [*support.optim_args_from_interpreter_flags(), - *cmd_line_switches, script_name, *example_args] - rc, out, err = assert_python_ok(*run_args, __isolated=False) - self._check_output(script_name, rc, out + err, expected_file, + *cmd_line_switches, *script_exec_args, *example_args] + rc, out, err = assert_python_ok( + *run_args, __isolated=False, __cwd=cwd, **env_vars + ) + self._check_output(script_exec_args, rc, out + err, expected_file, expected_argv0, expected_path0, - expected_package, expected_loader) + expected_package, expected_loader, cwd) - def _check_import_error(self, script_name, expected_msg, - *cmd_line_switches): - run_args = cmd_line_switches + (script_name,) - rc, out, err = assert_python_failure(*run_args) + def _check_import_error(self, script_exec_args, expected_msg, + *cmd_line_switches, cwd=None, **env_vars): + if isinstance(script_exec_args, str): + script_exec_args = (script_exec_args,) + else: + script_exec_args = tuple(script_exec_args) + run_args = cmd_line_switches + script_exec_args + rc, out, err = assert_python_failure( + *run_args, __isolated=False, __cwd=cwd, **env_vars + ) if verbose > 1: - print('Output from test script %r:' % script_name) + print('Output from test script %r:' % script_exec_args) print(repr(err)) print('Expected output: %r' % expected_msg) self.assertIn(expected_msg.encode('utf-8'), err) @@ -287,35 +279,35 @@ def test_module_in_package(self): pkg_dir = os.path.join(script_dir, 'test_pkg') make_pkg(pkg_dir) script_name = _make_test_script(pkg_dir, 'script') - launch_name = _make_launch_script(script_dir, 'launch', 'test_pkg.script') - self._check_script(launch_name, script_name, script_name, + self._check_script(["-m", "test_pkg.script"], script_name, script_name, script_dir, 'test_pkg', - importlib.machinery.SourceFileLoader) + importlib.machinery.SourceFileLoader, + cwd=script_dir) def test_module_in_package_in_zipfile(self): with support.temp_dir() as script_dir: zip_name, run_name = _make_test_zip_pkg(script_dir, 'test_zip', 'test_pkg', 'script') - launch_name = _make_launch_script(script_dir, 'launch', 'test_pkg.script', zip_name) - self._check_script(launch_name, run_name, run_name, - zip_name, 'test_pkg', zipimport.zipimporter) + self._check_script(["-m", "test_pkg.script"], run_name, run_name, + script_dir, 'test_pkg', zipimport.zipimporter, + PYTHONPATH=zip_name, cwd=script_dir) def test_module_in_subpackage_in_zipfile(self): with support.temp_dir() as script_dir: zip_name, run_name = _make_test_zip_pkg(script_dir, 'test_zip', 'test_pkg', 'script', depth=2) - launch_name = _make_launch_script(script_dir, 'launch', 'test_pkg.test_pkg.script', zip_name) - self._check_script(launch_name, run_name, run_name, - zip_name, 'test_pkg.test_pkg', - zipimport.zipimporter) + self._check_script(["-m", "test_pkg.test_pkg.script"], run_name, run_name, + script_dir, 'test_pkg.test_pkg', + zipimport.zipimporter, + PYTHONPATH=zip_name, cwd=script_dir) def test_package(self): with support.temp_dir() as script_dir: pkg_dir = os.path.join(script_dir, 'test_pkg') make_pkg(pkg_dir) script_name = _make_test_script(pkg_dir, '__main__') - launch_name = _make_launch_script(script_dir, 'launch', 'test_pkg') - self._check_script(launch_name, script_name, + self._check_script(["-m", "test_pkg"], script_name, script_name, script_dir, 'test_pkg', - importlib.machinery.SourceFileLoader) + importlib.machinery.SourceFileLoader, + cwd=script_dir) def test_package_compiled(self): with support.temp_dir() as script_dir: @@ -325,10 +317,10 @@ def test_package_compiled(self): compiled_name = py_compile.compile(script_name, doraise=True) os.remove(script_name) pyc_file = support.make_legacy_pyc(script_name) - launch_name = _make_launch_script(script_dir, 'launch', 'test_pkg') - self._check_script(launch_name, pyc_file, + self._check_script(["-m", "test_pkg"], pyc_file, pyc_file, script_dir, 'test_pkg', - importlib.machinery.SourcelessFileLoader) + importlib.machinery.SourcelessFileLoader, + cwd=script_dir) def test_package_error(self): with support.temp_dir() as script_dir: @@ -336,8 +328,7 @@ def test_package_error(self): make_pkg(pkg_dir) msg = ("'test_pkg' is a package and cannot " "be directly executed") - launch_name = _make_launch_script(script_dir, 'launch', 'test_pkg') - self._check_import_error(launch_name, msg) + self._check_import_error(["-m", "test_pkg"], msg, cwd=script_dir) def test_package_recursion(self): with support.temp_dir() as script_dir: @@ -348,8 +339,7 @@ def test_package_recursion(self): msg = ("Cannot use package as __main__ module; " "'test_pkg' is a package and cannot " "be directly executed") - launch_name = _make_launch_script(script_dir, 'launch', 'test_pkg') - self._check_import_error(launch_name, msg) + self._check_import_error(["-m", "test_pkg"], msg, cwd=script_dir) def test_issue8202(self): # Make sure package __init__ modules see "-m" in sys.argv0 while @@ -365,7 +355,7 @@ def test_issue8202(self): expected = "init_argv0==%r" % '-m' self.assertIn(expected.encode('utf-8'), out) self._check_output(script_name, rc, out, - script_name, script_name, '', 'test_pkg', + script_name, script_name, script_dir, 'test_pkg', importlib.machinery.SourceFileLoader) def test_issue8202_dash_c_file_ignored(self): @@ -394,7 +384,7 @@ def test_issue8202_dash_m_file_ignored(self): rc, out, err = assert_python_ok('-m', 'other', *example_args, __isolated=False) self._check_output(script_name, rc, out, - script_name, script_name, '', '', + script_name, script_name, script_dir, '', importlib.machinery.SourceFileLoader) @contextlib.contextmanager @@ -627,7 +617,7 @@ def test_consistent_sys_path_for_module_execution(self): # direct execution test cases p = spawn_python("-sm", "script_pkg.__main__", cwd=work_dir) out_by_module = kill_python(p).decode().splitlines() - self.assertEqual(out_by_module[0], '') + self.assertEqual(out_by_module[0], work_dir) self.assertNotIn(script_dir, out_by_module) # Package execution should give the same output p = spawn_python("-sm", "script_pkg", cwd=work_dir) diff --git a/Lib/test/test_doctest.py b/Lib/test/test_doctest.py index f0eb52881bcf..83941c129f44 100644 --- a/Lib/test/test_doctest.py +++ b/Lib/test/test_doctest.py @@ -9,7 +9,7 @@ import sys import importlib import unittest - +import tempfile # NOTE: There are some additional tests relating to interaction with # zipimport in the test_zipimport_support test module. @@ -688,10 +688,16 @@ class TestDocTestFinder(unittest.TestCase): def test_empty_namespace_package(self): pkg_name = 'doctest_empty_pkg' - os.mkdir(pkg_name) - mod = importlib.import_module(pkg_name) - assert doctest.DocTestFinder().find(mod) == [] - os.rmdir(pkg_name) + with tempfile.TemporaryDirectory() as parent_dir: + pkg_dir = os.path.join(parent_dir, pkg_name) + os.mkdir(pkg_dir) + sys.path.append(parent_dir) + try: + mod = importlib.import_module(pkg_name) + finally: + support.forget(pkg_name) + sys.path.pop() + assert doctest.DocTestFinder().find(mod) == [] def test_DocTestParser(): r""" diff --git a/Lib/test/test_import/__init__.py b/Lib/test/test_import/__init__.py index ceea79f6ad96..049ee57e58a0 100644 --- a/Lib/test/test_import/__init__.py +++ b/Lib/test/test_import/__init__.py @@ -826,8 +826,11 @@ def test_missing_source_legacy(self): unload(TESTFN) importlib.invalidate_caches() m = __import__(TESTFN) - self.assertEqual(m.__file__, - os.path.join(os.curdir, os.path.relpath(pyc_file))) + try: + self.assertEqual(m.__file__, + os.path.join(os.curdir, os.path.relpath(pyc_file))) + finally: + os.remove(pyc_file) def test___cached__(self): # Modules now also have an __cached__ that points to the pyc file. diff --git a/Misc/NEWS.d/next/Core and Builtins/2018-03-25-19-49-06.bpo-33053.V3xlsH.rst b/Misc/NEWS.d/next/Core and Builtins/2018-03-25-19-49-06.bpo-33053.V3xlsH.rst new file mode 100644 index 000000000000..fd32ac150e4c --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2018-03-25-19-49-06.bpo-33053.V3xlsH.rst @@ -0,0 +1,4 @@ +When using the -m switch, sys.path[0] is now explicitly expanded as the +*starting* working directory, rather than being left as the empty path +(which allows imports from the current working directory at the time of the +import) diff --git a/Python/pathconfig.c b/Python/pathconfig.c index 6de5481bc8da..07aa01c0304f 100644 --- a/Python/pathconfig.c +++ b/Python/pathconfig.c @@ -3,6 +3,7 @@ #include "Python.h" #include "osdefs.h" #include "internal/pystate.h" +#include <wchar.h> #ifdef __cplusplus extern "C" { @@ -255,11 +256,6 @@ Py_GetProgramName(void) return _Py_path_config.program_name; } - -#define _HAVE_SCRIPT_ARGUMENT(argc, argv) \ - (argc > 0 && argv0 != NULL && \ - wcscmp(argv0, L"-c") != 0 && wcscmp(argv0, L"-m") != 0) - /* Compute argv[0] which will be prepended to sys.argv */ PyObject* _PyPathConfig_ComputeArgv0(int argc, wchar_t **argv) @@ -267,6 +263,8 @@ _PyPathConfig_ComputeArgv0(int argc, wchar_t **argv) wchar_t *argv0; wchar_t *p = NULL; Py_ssize_t n = 0; + int have_script_arg = 0; + int have_module_arg = 0; #ifdef HAVE_READLINK wchar_t link[MAXPATHLEN+1]; wchar_t argv0copy[2*MAXPATHLEN+1]; @@ -278,11 +276,25 @@ _PyPathConfig_ComputeArgv0(int argc, wchar_t **argv) wchar_t fullpath[MAX_PATH]; #endif - argv0 = argv[0]; + if (argc > 0 && argv0 != NULL) { + have_module_arg = (wcscmp(argv0, L"-m") == 0); + have_script_arg = !have_module_arg && (wcscmp(argv0, L"-c") != 0); + } + + if (have_module_arg) { + #if defined(HAVE_REALPATH) || defined(MS_WINDOWS) + _Py_wgetcwd(fullpath, Py_ARRAY_LENGTH(fullpath)); + argv0 = fullpath; + n = wcslen(argv0); + #else + argv0 = L"."; + n = 1; + #endif + } #ifdef HAVE_READLINK - if (_HAVE_SCRIPT_ARGUMENT(argc, argv)) + if (have_script_arg) nr = _Py_wreadlink(argv0, link, MAXPATHLEN); if (nr > 0) { /* It's a symlink */ @@ -310,7 +322,7 @@ _PyPathConfig_ComputeArgv0(int argc, wchar_t **argv) #if SEP == '\\' /* Special case for Microsoft filename syntax */ - if (_HAVE_SCRIPT_ARGUMENT(argc, argv)) { + if (have_script_arg) { wchar_t *q; #if defined(MS_WINDOWS) /* Replace the first element in argv with the full path. */ @@ -334,7 +346,7 @@ _PyPathConfig_ComputeArgv0(int argc, wchar_t **argv) } } #else /* All other filename syntaxes */ - if (_HAVE_SCRIPT_ARGUMENT(argc, argv)) { + if (have_script_arg) { #if defined(HAVE_REALPATH) if (_Py_wrealpath(argv0, fullpath, Py_ARRAY_LENGTH(fullpath))) { argv0 = fullpath; From webhook-mailer at python.org Sun Mar 25 09:47:57 2018 From: webhook-mailer at python.org (Nick Coghlan) Date: Sun, 25 Mar 2018 13:47:57 -0000 Subject: [Python-checkins] bpo-33053: Remove test_cmd_line_script debugging print (GH-6237) Message-ID: <mailman.182.1521985680.1871.python-checkins@python.org> https://github.com/python/cpython/commit/a9e5d0e9ef27b14e34631d415e727a07d0f63bef commit: a9e5d0e9ef27b14e34631d415e727a07d0f63bef branch: master author: Nick Coghlan <ncoghlan at gmail.com> committer: GitHub <noreply at github.com> date: 2018-03-25T23:47:54+10:00 summary: bpo-33053: Remove test_cmd_line_script debugging print (GH-6237) I noticed this had slipped into the original commit when resolving a merge conflict for the backport to 3.7. files: M Lib/test/test_cmd_line_script.py diff --git a/Lib/test/test_cmd_line_script.py b/Lib/test/test_cmd_line_script.py index 176200855008..a9c1309ad6d3 100644 --- a/Lib/test/test_cmd_line_script.py +++ b/Lib/test/test_cmd_line_script.py @@ -125,8 +125,6 @@ def _check_script(self, script_exec_args, expected_file, script_exec_args = [script_exec_args] run_args = [*support.optim_args_from_interpreter_flags(), *cmd_line_switches, *script_exec_args, *example_args] - if env_vars: - print(env_vars) rc, out, err = assert_python_ok( *run_args, __isolated=False, __cwd=cwd, **env_vars ) From webhook-mailer at python.org Sun Mar 25 13:28:02 2018 From: webhook-mailer at python.org (Gregory P. Smith) Date: Sun, 25 Mar 2018 17:28:02 -0000 Subject: [Python-checkins] Clarify fd inheritance when close_fds=False. (GH-6240) Message-ID: <mailman.183.1521998884.1871.python-checkins@python.org> https://github.com/python/cpython/commit/dfb6e54dd8dbd735f55109ad8ee9dfcb6178ede9 commit: dfb6e54dd8dbd735f55109ad8ee9dfcb6178ede9 branch: master author: Gregory P. Smith <greg at krypto.org> committer: GitHub <noreply at github.com> date: 2018-03-25T10:27:59-07:00 summary: Clarify fd inheritance when close_fds=False. (GH-6240) Clarify the subprocess documentation. files: M Doc/library/subprocess.rst diff --git a/Doc/library/subprocess.rst b/Doc/library/subprocess.rst index db7a88af6bea..fbf2c3d9fac9 100644 --- a/Doc/library/subprocess.rst +++ b/Doc/library/subprocess.rst @@ -459,7 +459,10 @@ functions. common use of *preexec_fn* to call os.setsid() in the child. If *close_fds* is true, all file descriptors except :const:`0`, :const:`1` and - :const:`2` will be closed before the child process is executed. + :const:`2` will be closed before the child process is executed. Otherwise + when *close_fds* is false, file descriptors obey their inheritable flag + as described in :ref:`fd_inheritance`. + On Windows, if *close_fds* is true then no handles will be inherited by the child process unless explicitly passed in the ``handle_list`` element of :attr:`STARTUPINFO.lpAttributeList`, or by standard handle redirection. From webhook-mailer at python.org Sun Mar 25 13:40:41 2018 From: webhook-mailer at python.org (Miss Islington (bot)) Date: Sun, 25 Mar 2018 17:40:41 -0000 Subject: [Python-checkins] Clarify fd inheritance when close_fds=False. (GH-6240) Message-ID: <mailman.184.1521999642.1871.python-checkins@python.org> https://github.com/python/cpython/commit/bc3e009f652e12f063c781cd1cec25ef71e00a7b commit: bc3e009f652e12f063c781cd1cec25ef71e00a7b branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: GitHub <noreply at github.com> date: 2018-03-25T10:40:38-07:00 summary: Clarify fd inheritance when close_fds=False. (GH-6240) Clarify the subprocess documentation. (cherry picked from commit dfb6e54dd8dbd735f55109ad8ee9dfcb6178ede9) Co-authored-by: Gregory P. Smith <greg at krypto.org> files: M Doc/library/subprocess.rst diff --git a/Doc/library/subprocess.rst b/Doc/library/subprocess.rst index db7a88af6bea..fbf2c3d9fac9 100644 --- a/Doc/library/subprocess.rst +++ b/Doc/library/subprocess.rst @@ -459,7 +459,10 @@ functions. common use of *preexec_fn* to call os.setsid() in the child. If *close_fds* is true, all file descriptors except :const:`0`, :const:`1` and - :const:`2` will be closed before the child process is executed. + :const:`2` will be closed before the child process is executed. Otherwise + when *close_fds* is false, file descriptors obey their inheritable flag + as described in :ref:`fd_inheritance`. + On Windows, if *close_fds* is true then no handles will be inherited by the child process unless explicitly passed in the ``handle_list`` element of :attr:`STARTUPINFO.lpAttributeList`, or by standard handle redirection. From webhook-mailer at python.org Sun Mar 25 20:37:36 2018 From: webhook-mailer at python.org (Eric V. Smith) Date: Mon, 26 Mar 2018 00:37:36 -0000 Subject: [Python-checkins] Minor fixes to dataclass tests. (GH-6243) Message-ID: <mailman.185.1522024657.1871.python-checkins@python.org> https://github.com/python/cpython/commit/2b75fc2bc97702224de0fae8ab026ec0cd0706ab commit: 2b75fc2bc97702224de0fae8ab026ec0cd0706ab branch: master author: Eric V. Smith <ericvsmith at users.noreply.github.com> committer: GitHub <noreply at github.com> date: 2018-03-25T20:37:33-04:00 summary: Minor fixes to dataclass tests. (GH-6243) Also, re-enable a test for ClassVars with default_factory. files: M Lib/test/test_dataclasses.py diff --git a/Lib/test/test_dataclasses.py b/Lib/test/test_dataclasses.py index df53b040c0e1..f7f132ca30d2 100755 --- a/Lib/test/test_dataclasses.py +++ b/Lib/test/test_dataclasses.py @@ -133,8 +133,8 @@ def __eq__(self): self.assertEqual(hash(C(10)), hash((10,))) # Creating this class should generate an exception, because - # __hash__ exists and is not None, which it would be if it had - # been auto-generated do due __eq__ being defined. + # __hash__ exists and is not None, which it would be if it + # had been auto-generated due to __eq__ being defined. with self.assertRaisesRegex(TypeError, 'Cannot overwrite attribute __hash__'): @dataclass(unsafe_hash=True) @@ -145,7 +145,6 @@ def __eq__(self): def __hash__(self): pass - def test_overwrite_fields_in_derived_class(self): # Note that x from C1 replaces x in Base, but the order remains # the same as defined in Base. @@ -624,7 +623,7 @@ class C: self.assertIs(o1.x, o2.x) def test_no_options(self): - # call with dataclass() + # Call with dataclass(). @dataclass() class C: x: int @@ -639,7 +638,7 @@ class Point: y: int self.assertNotEqual(Point(1, 2), (1, 2)) - # And that we can't compare to another unrelated dataclass + # And that we can't compare to another unrelated dataclass. @dataclass class C: x: int @@ -664,7 +663,7 @@ class Date: self.assertNotEqual(Point3D(2017, 6, 3), Date(2017, 6, 3)) self.assertNotEqual(Point3D(1, 2, 3), (1, 2, 3)) - # Make sure we can't unpack + # Make sure we can't unpack. with self.assertRaisesRegex(TypeError, 'unpack'): x, y, z = Point3D(4, 5, 6) @@ -695,7 +694,7 @@ def validate_class(cls): # Verify __init__. signature = inspect.signature(cls.__init__) - # Check the return type, should be None + # Check the return type, should be None. self.assertIs(signature.return_annotation, None) # Check each parameter. @@ -716,12 +715,12 @@ def validate_class(cls): param = next(params) self.assertEqual(param.name, 'k') self.assertIs (param.annotation, F) - # Don't test for the default, since it's set to MISSING + # Don't test for the default, since it's set to MISSING. self.assertEqual(param.kind, inspect.Parameter.POSITIONAL_OR_KEYWORD) param = next(params) self.assertEqual(param.name, 'l') self.assertIs (param.annotation, float) - # Don't test for the default, since it's set to MISSING + # Don't test for the default, since it's set to MISSING. self.assertEqual(param.kind, inspect.Parameter.POSITIONAL_OR_KEYWORD) self.assertRaises(StopIteration, next, params) @@ -867,7 +866,7 @@ def __post_init__(self): self.assertEqual(C().x, 5) - # Now call super(), and it will raise + # Now call super(), and it will raise. @dataclass class C(B): def __post_init__(self): @@ -928,8 +927,8 @@ class C: c = C(5) self.assertEqual(repr(c), 'TestCase.test_class_var.<locals>.C(x=5, y=10)') - self.assertEqual(len(fields(C)), 2) # We have 2 fields - self.assertEqual(len(C.__annotations__), 5) # And 3 ClassVars + self.assertEqual(len(fields(C)), 2) # We have 2 fields. + self.assertEqual(len(C.__annotations__), 5) # And 3 ClassVars. self.assertEqual(c.z, 1000) self.assertEqual(c.w, 2000) self.assertEqual(c.t, 3000) @@ -1205,14 +1204,13 @@ class D(C): d = D(4, 5) self.assertEqual((d.x, d.z), (4, 5)) - - def x_test_classvar_default_factory(self): - # XXX: it's an error for a ClassVar to have a factory function - @dataclass - class C: - x: ClassVar[int] = field(default_factory=int) - - self.assertIs(C().x, int) + def test_classvar_default_factory(self): + # It's an error for a ClassVar to have a factory function. + with self.assertRaisesRegex(TypeError, + 'cannot have a default factory'): + @dataclass + class C: + x: ClassVar[int] = field(default_factory=int) def test_is_dataclass(self): class NotDataClass: @@ -1264,7 +1262,7 @@ class C: pass fields(C()) def test_helper_asdict(self): - # Basic tests for asdict(), it should return a new dictionary + # Basic tests for asdict(), it should return a new dictionary. @dataclass class C: x: int @@ -1279,7 +1277,7 @@ class C: self.assertIs(type(asdict(c)), dict) def test_helper_asdict_raises_on_classes(self): - # asdict() should raise on a class object + # asdict() should raise on a class object. @dataclass class C: x: int @@ -1377,7 +1375,7 @@ class C: self.assertIs(type(d), OrderedDict) def test_helper_astuple(self): - # Basic tests for astuple(), it should return a new tuple + # Basic tests for astuple(), it should return a new tuple. @dataclass class C: x: int @@ -1392,7 +1390,7 @@ class C: self.assertIs(type(astuple(c)), tuple) def test_helper_astuple_raises_on_classes(self): - # astuple() should raise on a class object + # astuple() should raise on a class object. @dataclass class C: x: int @@ -1489,7 +1487,7 @@ def nt(lst): self.assertIs(type(t), NT) def test_dynamic_class_creation(self): - cls_dict = {'__annotations__': {'x':int, 'y':int}, + cls_dict = {'__annotations__': {'x': int, 'y': int}, } # Create the class. @@ -1502,7 +1500,7 @@ def test_dynamic_class_creation(self): self.assertEqual(asdict(cls(1, 2)), {'x': 1, 'y': 2}) def test_dynamic_class_creation_using_field(self): - cls_dict = {'__annotations__': {'x':int, 'y':int}, + cls_dict = {'__annotations__': {'x': int, 'y': int}, 'y': field(default=5), } @@ -1569,8 +1567,8 @@ class C: def test_alternate_classmethod_constructor(self): # Since __post_init__ can't take params, use a classmethod - # alternate constructor. This is mostly an example to show how - # to use this technique. + # alternate constructor. This is mostly an example to show + # how to use this technique. @dataclass class C: x: int @@ -1604,7 +1602,7 @@ def test_field_metadata_mapping(self): class C: i: int = field(metadata=0) - # Make sure an empty dict works + # Make sure an empty dict works. @dataclass class C: i: int = field(metadata={}) @@ -1666,7 +1664,7 @@ class LabeledBox(Generic[T]): self.assertEqual(box.content, 42) self.assertEqual(box.label, '<unknown>') - # subscripting the resulting class should work, etc. + # Subscripting the resulting class should work, etc. Alias = List[LabeledBox[int]] def test_generic_extending(self): @@ -1931,7 +1929,7 @@ class B: with self.assertRaisesRegex(TypeError, "'f' is a field but has no type annotation"): # This is still an error: make sure we don't pick up the - # type annotation in the base class. + # type annotation in the base class. @dataclass class C(B): f = field() @@ -1944,7 +1942,7 @@ class B: with self.assertRaisesRegex(TypeError, "'f' is a field but has no type annotation"): # This is still an error: make sure we don't pick up the - # type annotation in the base class. + # type annotation in the base class. @dataclass class C(B): f = field() @@ -2178,7 +2176,7 @@ def __repr__(self): class TestFrozen(unittest.TestCase): def test_overwriting_frozen(self): - # frozen uses __setattr__ and __delattr__ + # frozen uses __setattr__ and __delattr__. with self.assertRaisesRegex(TypeError, 'Cannot overwrite attribute __setattr__'): @dataclass(frozen=True) @@ -2473,16 +2471,16 @@ class C: def test_hash_no_args(self): # Test dataclasses with no hash= argument. This exists to - # make sure that if the @dataclass parameter name is changed - # or the non-default hashing behavior changes, the default - # hashability keeps working the same way. + # make sure that if the @dataclass parameter name is changed + # or the non-default hashing behavior changes, the default + # hashability keeps working the same way. class Base: def __hash__(self): return 301 # If frozen or eq is None, then use the default value (do not - # specify any value in the decorator). + # specify any value in the decorator). for frozen, eq, base, expected in [ (None, None, object, 'unhashable'), (None, None, Base, 'unhashable'), @@ -2534,9 +2532,9 @@ class C(base): elif expected == 'object': # I'm not sure what test to use here. object's - # hash isn't based on id(), so calling hash() - # won't tell us much. So, just check the function - # used is object's. + # hash isn't based on id(), so calling hash() + # won't tell us much. So, just check the + # function used is object's. self.assertIs(C.__hash__, object.__hash__) elif expected == 'tuple': @@ -2665,8 +2663,9 @@ class C: __slots__ = ('x',) x: Any - # There was a bug where a variable in a slot was assumed - # to also have a default value (of type types.MemberDescriptorType). + # There was a bug where a variable in a slot was assumed to + # also have a default value (of type + # types.MemberDescriptorType). with self.assertRaisesRegex(TypeError, r"__init__\(\) missing 1 required positional argument: 'x'"): C() From webhook-mailer at python.org Sun Mar 25 21:00:46 2018 From: webhook-mailer at python.org (Miss Islington (bot)) Date: Mon, 26 Mar 2018 01:00:46 -0000 Subject: [Python-checkins] Minor fixes to dataclass tests. (GH-6243) Message-ID: <mailman.186.1522026046.1871.python-checkins@python.org> https://github.com/python/cpython/commit/5fc6fc85c42d1a17938db1956c0d0f5454427b3b commit: 5fc6fc85c42d1a17938db1956c0d0f5454427b3b branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: GitHub <noreply at github.com> date: 2018-03-25T18:00:43-07:00 summary: Minor fixes to dataclass tests. (GH-6243) Also, re-enable a test for ClassVars with default_factory. (cherry picked from commit 2b75fc2bc97702224de0fae8ab026ec0cd0706ab) Co-authored-by: Eric V. Smith <ericvsmith at users.noreply.github.com> files: M Lib/test/test_dataclasses.py diff --git a/Lib/test/test_dataclasses.py b/Lib/test/test_dataclasses.py index df53b040c0e1..f7f132ca30d2 100755 --- a/Lib/test/test_dataclasses.py +++ b/Lib/test/test_dataclasses.py @@ -133,8 +133,8 @@ def __eq__(self): self.assertEqual(hash(C(10)), hash((10,))) # Creating this class should generate an exception, because - # __hash__ exists and is not None, which it would be if it had - # been auto-generated do due __eq__ being defined. + # __hash__ exists and is not None, which it would be if it + # had been auto-generated due to __eq__ being defined. with self.assertRaisesRegex(TypeError, 'Cannot overwrite attribute __hash__'): @dataclass(unsafe_hash=True) @@ -145,7 +145,6 @@ def __eq__(self): def __hash__(self): pass - def test_overwrite_fields_in_derived_class(self): # Note that x from C1 replaces x in Base, but the order remains # the same as defined in Base. @@ -624,7 +623,7 @@ class C: self.assertIs(o1.x, o2.x) def test_no_options(self): - # call with dataclass() + # Call with dataclass(). @dataclass() class C: x: int @@ -639,7 +638,7 @@ class Point: y: int self.assertNotEqual(Point(1, 2), (1, 2)) - # And that we can't compare to another unrelated dataclass + # And that we can't compare to another unrelated dataclass. @dataclass class C: x: int @@ -664,7 +663,7 @@ class Date: self.assertNotEqual(Point3D(2017, 6, 3), Date(2017, 6, 3)) self.assertNotEqual(Point3D(1, 2, 3), (1, 2, 3)) - # Make sure we can't unpack + # Make sure we can't unpack. with self.assertRaisesRegex(TypeError, 'unpack'): x, y, z = Point3D(4, 5, 6) @@ -695,7 +694,7 @@ def validate_class(cls): # Verify __init__. signature = inspect.signature(cls.__init__) - # Check the return type, should be None + # Check the return type, should be None. self.assertIs(signature.return_annotation, None) # Check each parameter. @@ -716,12 +715,12 @@ def validate_class(cls): param = next(params) self.assertEqual(param.name, 'k') self.assertIs (param.annotation, F) - # Don't test for the default, since it's set to MISSING + # Don't test for the default, since it's set to MISSING. self.assertEqual(param.kind, inspect.Parameter.POSITIONAL_OR_KEYWORD) param = next(params) self.assertEqual(param.name, 'l') self.assertIs (param.annotation, float) - # Don't test for the default, since it's set to MISSING + # Don't test for the default, since it's set to MISSING. self.assertEqual(param.kind, inspect.Parameter.POSITIONAL_OR_KEYWORD) self.assertRaises(StopIteration, next, params) @@ -867,7 +866,7 @@ def __post_init__(self): self.assertEqual(C().x, 5) - # Now call super(), and it will raise + # Now call super(), and it will raise. @dataclass class C(B): def __post_init__(self): @@ -928,8 +927,8 @@ class C: c = C(5) self.assertEqual(repr(c), 'TestCase.test_class_var.<locals>.C(x=5, y=10)') - self.assertEqual(len(fields(C)), 2) # We have 2 fields - self.assertEqual(len(C.__annotations__), 5) # And 3 ClassVars + self.assertEqual(len(fields(C)), 2) # We have 2 fields. + self.assertEqual(len(C.__annotations__), 5) # And 3 ClassVars. self.assertEqual(c.z, 1000) self.assertEqual(c.w, 2000) self.assertEqual(c.t, 3000) @@ -1205,14 +1204,13 @@ class D(C): d = D(4, 5) self.assertEqual((d.x, d.z), (4, 5)) - - def x_test_classvar_default_factory(self): - # XXX: it's an error for a ClassVar to have a factory function - @dataclass - class C: - x: ClassVar[int] = field(default_factory=int) - - self.assertIs(C().x, int) + def test_classvar_default_factory(self): + # It's an error for a ClassVar to have a factory function. + with self.assertRaisesRegex(TypeError, + 'cannot have a default factory'): + @dataclass + class C: + x: ClassVar[int] = field(default_factory=int) def test_is_dataclass(self): class NotDataClass: @@ -1264,7 +1262,7 @@ class C: pass fields(C()) def test_helper_asdict(self): - # Basic tests for asdict(), it should return a new dictionary + # Basic tests for asdict(), it should return a new dictionary. @dataclass class C: x: int @@ -1279,7 +1277,7 @@ class C: self.assertIs(type(asdict(c)), dict) def test_helper_asdict_raises_on_classes(self): - # asdict() should raise on a class object + # asdict() should raise on a class object. @dataclass class C: x: int @@ -1377,7 +1375,7 @@ class C: self.assertIs(type(d), OrderedDict) def test_helper_astuple(self): - # Basic tests for astuple(), it should return a new tuple + # Basic tests for astuple(), it should return a new tuple. @dataclass class C: x: int @@ -1392,7 +1390,7 @@ class C: self.assertIs(type(astuple(c)), tuple) def test_helper_astuple_raises_on_classes(self): - # astuple() should raise on a class object + # astuple() should raise on a class object. @dataclass class C: x: int @@ -1489,7 +1487,7 @@ def nt(lst): self.assertIs(type(t), NT) def test_dynamic_class_creation(self): - cls_dict = {'__annotations__': {'x':int, 'y':int}, + cls_dict = {'__annotations__': {'x': int, 'y': int}, } # Create the class. @@ -1502,7 +1500,7 @@ def test_dynamic_class_creation(self): self.assertEqual(asdict(cls(1, 2)), {'x': 1, 'y': 2}) def test_dynamic_class_creation_using_field(self): - cls_dict = {'__annotations__': {'x':int, 'y':int}, + cls_dict = {'__annotations__': {'x': int, 'y': int}, 'y': field(default=5), } @@ -1569,8 +1567,8 @@ class C: def test_alternate_classmethod_constructor(self): # Since __post_init__ can't take params, use a classmethod - # alternate constructor. This is mostly an example to show how - # to use this technique. + # alternate constructor. This is mostly an example to show + # how to use this technique. @dataclass class C: x: int @@ -1604,7 +1602,7 @@ def test_field_metadata_mapping(self): class C: i: int = field(metadata=0) - # Make sure an empty dict works + # Make sure an empty dict works. @dataclass class C: i: int = field(metadata={}) @@ -1666,7 +1664,7 @@ class LabeledBox(Generic[T]): self.assertEqual(box.content, 42) self.assertEqual(box.label, '<unknown>') - # subscripting the resulting class should work, etc. + # Subscripting the resulting class should work, etc. Alias = List[LabeledBox[int]] def test_generic_extending(self): @@ -1931,7 +1929,7 @@ class B: with self.assertRaisesRegex(TypeError, "'f' is a field but has no type annotation"): # This is still an error: make sure we don't pick up the - # type annotation in the base class. + # type annotation in the base class. @dataclass class C(B): f = field() @@ -1944,7 +1942,7 @@ class B: with self.assertRaisesRegex(TypeError, "'f' is a field but has no type annotation"): # This is still an error: make sure we don't pick up the - # type annotation in the base class. + # type annotation in the base class. @dataclass class C(B): f = field() @@ -2178,7 +2176,7 @@ def __repr__(self): class TestFrozen(unittest.TestCase): def test_overwriting_frozen(self): - # frozen uses __setattr__ and __delattr__ + # frozen uses __setattr__ and __delattr__. with self.assertRaisesRegex(TypeError, 'Cannot overwrite attribute __setattr__'): @dataclass(frozen=True) @@ -2473,16 +2471,16 @@ class C: def test_hash_no_args(self): # Test dataclasses with no hash= argument. This exists to - # make sure that if the @dataclass parameter name is changed - # or the non-default hashing behavior changes, the default - # hashability keeps working the same way. + # make sure that if the @dataclass parameter name is changed + # or the non-default hashing behavior changes, the default + # hashability keeps working the same way. class Base: def __hash__(self): return 301 # If frozen or eq is None, then use the default value (do not - # specify any value in the decorator). + # specify any value in the decorator). for frozen, eq, base, expected in [ (None, None, object, 'unhashable'), (None, None, Base, 'unhashable'), @@ -2534,9 +2532,9 @@ class C(base): elif expected == 'object': # I'm not sure what test to use here. object's - # hash isn't based on id(), so calling hash() - # won't tell us much. So, just check the function - # used is object's. + # hash isn't based on id(), so calling hash() + # won't tell us much. So, just check the + # function used is object's. self.assertIs(C.__hash__, object.__hash__) elif expected == 'tuple': @@ -2665,8 +2663,9 @@ class C: __slots__ = ('x',) x: Any - # There was a bug where a variable in a slot was assumed - # to also have a default value (of type types.MemberDescriptorType). + # There was a bug where a variable in a slot was assumed to + # also have a default value (of type + # types.MemberDescriptorType). with self.assertRaisesRegex(TypeError, r"__init__\(\) missing 1 required positional argument: 'x'"): C() From webhook-mailer at python.org Mon Mar 26 03:02:09 2018 From: webhook-mailer at python.org (Serhiy Storchaka) Date: Mon, 26 Mar 2018 07:02:09 -0000 Subject: [Python-checkins] bpo-33096: Fix ttk.Treeview.insert. (GH-6228) Message-ID: <mailman.187.1522047732.1871.python-checkins@python.org> https://github.com/python/cpython/commit/3ab44c0783eebdff687014f7d14d5dec59b6bd39 commit: 3ab44c0783eebdff687014f7d14d5dec59b6bd39 branch: master author: Garvit Khatri <garvitdelhi at gmail.com> committer: Serhiy Storchaka <storchaka at gmail.com> date: 2018-03-26T10:02:05+03:00 summary: bpo-33096: Fix ttk.Treeview.insert. (GH-6228) Allow ttk.Treeview.insert to insert iid that has a false boolean value. Note iid=0 and iid=False would be same. files: A Misc/NEWS.d/next/Library/2018-03-25-13-18-16.bpo-33096.ofdbe7.rst M Lib/tkinter/test/test_ttk/test_widgets.py M Lib/tkinter/ttk.py diff --git a/Lib/tkinter/test/test_ttk/test_widgets.py b/Lib/tkinter/test/test_ttk/test_widgets.py index bbc508d35810..5b0e29cdccaf 100644 --- a/Lib/tkinter/test/test_ttk/test_widgets.py +++ b/Lib/tkinter/test/test_ttk/test_widgets.py @@ -1662,6 +1662,15 @@ def test_insert_item(self): self.tv.insert('', 'end', text=value), text=None), value) + # test for values which are not None + itemid = self.tv.insert('', 'end', 0) + self.assertEqual(itemid, '0') + itemid = self.tv.insert('', 'end', 0.0) + self.assertEqual(itemid, '0.0') + # this is because False resolves to 0 and element with 0 iid is already present + self.assertRaises(tkinter.TclError, self.tv.insert, '', 'end', False) + self.assertRaises(tkinter.TclError, self.tv.insert, '', 'end', '') + def test_selection(self): self.assertRaises(TypeError, self.tv.selection, 'spam') diff --git a/Lib/tkinter/ttk.py b/Lib/tkinter/ttk.py index 490b502a65f1..573544dd84a3 100644 --- a/Lib/tkinter/ttk.py +++ b/Lib/tkinter/ttk.py @@ -1361,7 +1361,7 @@ def insert(self, parent, index, iid=None, **kw): already exist in the tree. Otherwise, a new unique identifier is generated.""" opts = _format_optdict(kw) - if iid: + if iid is not None: res = self.tk.call(self._w, "insert", parent, index, "-id", iid, *opts) else: diff --git a/Misc/NEWS.d/next/Library/2018-03-25-13-18-16.bpo-33096.ofdbe7.rst b/Misc/NEWS.d/next/Library/2018-03-25-13-18-16.bpo-33096.ofdbe7.rst new file mode 100644 index 000000000000..c55ea20b337d --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-03-25-13-18-16.bpo-33096.ofdbe7.rst @@ -0,0 +1,4 @@ +Allow ttk.Treeview.insert to insert iid that has a false boolean value. +Note iid=0 and iid=False would be same. +Patch by Garvit Khatri. + From webhook-mailer at python.org Mon Mar 26 04:19:55 2018 From: webhook-mailer at python.org (Miss Islington (bot)) Date: Mon, 26 Mar 2018 08:19:55 -0000 Subject: [Python-checkins] bpo-33096: Fix ttk.Treeview.insert. (GH-6228) Message-ID: <mailman.188.1522052398.1871.python-checkins@python.org> https://github.com/python/cpython/commit/a7b1b847f665aafc22557917cea32ec34c9b4418 commit: a7b1b847f665aafc22557917cea32ec34c9b4418 branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: GitHub <noreply at github.com> date: 2018-03-26T01:19:52-07:00 summary: bpo-33096: Fix ttk.Treeview.insert. (GH-6228) Allow ttk.Treeview.insert to insert iid that has a false boolean value. Note iid=0 and iid=False would be same. (cherry picked from commit 3ab44c0783eebdff687014f7d14d5dec59b6bd39) Co-authored-by: Garvit Khatri <garvitdelhi at gmail.com> files: A Misc/NEWS.d/next/Library/2018-03-25-13-18-16.bpo-33096.ofdbe7.rst M Lib/tkinter/test/test_ttk/test_widgets.py M Lib/tkinter/ttk.py diff --git a/Lib/tkinter/test/test_ttk/test_widgets.py b/Lib/tkinter/test/test_ttk/test_widgets.py index 5325e36b5212..06e3dfe70d5d 100644 --- a/Lib/tkinter/test/test_ttk/test_widgets.py +++ b/Lib/tkinter/test/test_ttk/test_widgets.py @@ -1662,6 +1662,15 @@ def test_insert_item(self): self.tv.insert('', 'end', text=value), text=None), value) + # test for values which are not None + itemid = self.tv.insert('', 'end', 0) + self.assertEqual(itemid, '0') + itemid = self.tv.insert('', 'end', 0.0) + self.assertEqual(itemid, '0.0') + # this is because False resolves to 0 and element with 0 iid is already present + self.assertRaises(tkinter.TclError, self.tv.insert, '', 'end', False) + self.assertRaises(tkinter.TclError, self.tv.insert, '', 'end', '') + def test_selection(self): self.assertRaises(TypeError, self.tv.selection, 'spam') diff --git a/Lib/tkinter/ttk.py b/Lib/tkinter/ttk.py index c1651159b703..12f4ac0bd908 100644 --- a/Lib/tkinter/ttk.py +++ b/Lib/tkinter/ttk.py @@ -1363,7 +1363,7 @@ def insert(self, parent, index, iid=None, **kw): already exist in the tree. Otherwise, a new unique identifier is generated.""" opts = _format_optdict(kw) - if iid: + if iid is not None: res = self.tk.call(self._w, "insert", parent, index, "-id", iid, *opts) else: diff --git a/Misc/NEWS.d/next/Library/2018-03-25-13-18-16.bpo-33096.ofdbe7.rst b/Misc/NEWS.d/next/Library/2018-03-25-13-18-16.bpo-33096.ofdbe7.rst new file mode 100644 index 000000000000..c55ea20b337d --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-03-25-13-18-16.bpo-33096.ofdbe7.rst @@ -0,0 +1,4 @@ +Allow ttk.Treeview.insert to insert iid that has a false boolean value. +Note iid=0 and iid=False would be same. +Patch by Garvit Khatri. + From webhook-mailer at python.org Mon Mar 26 04:20:13 2018 From: webhook-mailer at python.org (Miss Islington (bot)) Date: Mon, 26 Mar 2018 08:20:13 -0000 Subject: [Python-checkins] bpo-33096: Fix ttk.Treeview.insert. (GH-6228) Message-ID: <mailman.189.1522052416.1871.python-checkins@python.org> https://github.com/python/cpython/commit/45116d393f713bbe918f16b33a0bba28b15bc96b commit: 45116d393f713bbe918f16b33a0bba28b15bc96b branch: 3.6 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: GitHub <noreply at github.com> date: 2018-03-26T01:20:10-07:00 summary: bpo-33096: Fix ttk.Treeview.insert. (GH-6228) Allow ttk.Treeview.insert to insert iid that has a false boolean value. Note iid=0 and iid=False would be same. (cherry picked from commit 3ab44c0783eebdff687014f7d14d5dec59b6bd39) Co-authored-by: Garvit Khatri <garvitdelhi at gmail.com> files: A Misc/NEWS.d/next/Library/2018-03-25-13-18-16.bpo-33096.ofdbe7.rst M Lib/tkinter/test/test_ttk/test_widgets.py M Lib/tkinter/ttk.py diff --git a/Lib/tkinter/test/test_ttk/test_widgets.py b/Lib/tkinter/test/test_ttk/test_widgets.py index ab0db2878e13..cb991931853a 100644 --- a/Lib/tkinter/test/test_ttk/test_widgets.py +++ b/Lib/tkinter/test/test_ttk/test_widgets.py @@ -1485,6 +1485,15 @@ def test_insert_item(self): self.tv.insert('', 'end', text=value), text=None), value) + # test for values which are not None + itemid = self.tv.insert('', 'end', 0) + self.assertEqual(itemid, '0') + itemid = self.tv.insert('', 'end', 0.0) + self.assertEqual(itemid, '0.0') + # this is because False resolves to 0 and element with 0 iid is already present + self.assertRaises(tkinter.TclError, self.tv.insert, '', 'end', False) + self.assertRaises(tkinter.TclError, self.tv.insert, '', 'end', '') + def test_selection(self): self.assertRaises(TypeError, self.tv.selection, 'spam') diff --git a/Lib/tkinter/ttk.py b/Lib/tkinter/ttk.py index 42e29bd3e5c8..3cf5faf10d7a 100644 --- a/Lib/tkinter/ttk.py +++ b/Lib/tkinter/ttk.py @@ -1336,7 +1336,7 @@ def insert(self, parent, index, iid=None, **kw): already exist in the tree. Otherwise, a new unique identifier is generated.""" opts = _format_optdict(kw) - if iid: + if iid is not None: res = self.tk.call(self._w, "insert", parent, index, "-id", iid, *opts) else: diff --git a/Misc/NEWS.d/next/Library/2018-03-25-13-18-16.bpo-33096.ofdbe7.rst b/Misc/NEWS.d/next/Library/2018-03-25-13-18-16.bpo-33096.ofdbe7.rst new file mode 100644 index 000000000000..c55ea20b337d --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-03-25-13-18-16.bpo-33096.ofdbe7.rst @@ -0,0 +1,4 @@ +Allow ttk.Treeview.insert to insert iid that has a false boolean value. +Note iid=0 and iid=False would be same. +Patch by Garvit Khatri. + From solipsis at pitrou.net Mon Mar 26 05:13:42 2018 From: solipsis at pitrou.net (solipsis at pitrou.net) Date: Mon, 26 Mar 2018 09:13:42 +0000 Subject: [Python-checkins] Daily reference leaks (4243df51fe43): sum=4 Message-ID: <20180326091342.1.9A958B2373FEF515@psf.io> results for 4243df51fe43 on branch "default" -------------------------------------------- test_functools leaked [0, 3, 1] memory blocks, sum=4 test_multiprocessing_spawn leaked [-2, 2, 0] memory blocks, sum=0 Command line was: ['./python', '-m', 'test.regrtest', '-uall', '-R', '3:3:/home/psf-users/antoine/refleaks/reflogjAk6W7', '--timeout', '7200'] From webhook-mailer at python.org Mon Mar 26 05:41:34 2018 From: webhook-mailer at python.org (Serhiy Storchaka) Date: Mon, 26 Mar 2018 09:41:34 -0000 Subject: [Python-checkins] Fix typo and edit for clarity in the docstrings of sys.float_info. (GH-2251) Message-ID: <mailman.190.1522057295.1871.python-checkins@python.org> https://github.com/python/cpython/commit/0301c9bdd1ebd788d1334cf3fe06c48f35bab0dc commit: 0301c9bdd1ebd788d1334cf3fe06c48f35bab0dc branch: master author: Stefano Taschini <taschini at users.noreply.github.com> committer: Serhiy Storchaka <storchaka at gmail.com> date: 2018-03-26T12:41:30+03:00 summary: Fix typo and edit for clarity in the docstrings of sys.float_info. (GH-2251) files: M Objects/floatobject.c diff --git a/Objects/floatobject.c b/Objects/floatobject.c index 47a174c241e3..67f9e5d5b4ef 100644 --- a/Objects/floatobject.c +++ b/Objects/floatobject.c @@ -1,4 +1,3 @@ - /* Float object implementation */ /* XXX There should be overflow checks here, but it's hard to check @@ -54,7 +53,7 @@ static PyStructSequence_Field floatinfo_fields[] = { "is representable"}, {"max_10_exp", "DBL_MAX_10_EXP -- maximum int e such that 10**e " "is representable"}, - {"min", "DBL_MIN -- Minimum positive normalizer float"}, + {"min", "DBL_MIN -- Minimum positive normalized float"}, {"min_exp", "DBL_MIN_EXP -- minimum int e such that radix**(e-1) " "is a normalized float"}, {"min_10_exp", "DBL_MIN_10_EXP -- minimum int e such that 10**e is " @@ -64,7 +63,7 @@ static PyStructSequence_Field floatinfo_fields[] = { {"epsilon", "DBL_EPSILON -- Difference between 1 and the next " "representable float"}, {"radix", "FLT_RADIX -- radix of exponent"}, - {"rounds", "FLT_ROUNDS -- addition rounds"}, + {"rounds", "FLT_ROUNDS -- rounding mode"}, {0} }; From webhook-mailer at python.org Mon Mar 26 06:11:50 2018 From: webhook-mailer at python.org (Serhiy Storchaka) Date: Mon, 26 Mar 2018 10:11:50 -0000 Subject: [Python-checkins] Gitignore gmon.out (GH-5796) Message-ID: <mailman.191.1522059110.1871.python-checkins@python.org> https://github.com/python/cpython/commit/95ad3822a2b6287772bd752b6ab493c6d4198d4b commit: 95ad3822a2b6287772bd752b6ab493c6d4198d4b branch: master author: Neeraj Badlani <neerajbadlani at gmail.com> committer: Serhiy Storchaka <storchaka at gmail.com> date: 2018-03-26T13:11:47+03:00 summary: Gitignore gmon.out (GH-5796) gmon.out is generated when profiling turned on Full Configuration: ./configure --prefix=$PWD/install --enable-profiling --enable-big-digits=30 --with-pydebug --with-assertions --with-valgrind files: M .gitignore diff --git a/.gitignore b/.gitignore index 05fb6cba0875..58f8bf72f2b9 100644 --- a/.gitignore +++ b/.gitignore @@ -113,3 +113,4 @@ Tools/ssl/amd64 Tools/ssl/win32 .vs/ .vscode/ +gmon.out From webhook-mailer at python.org Mon Mar 26 06:57:43 2018 From: webhook-mailer at python.org (Miss Islington (bot)) Date: Mon, 26 Mar 2018 10:57:43 -0000 Subject: [Python-checkins] Fix typo and edit for clarity in the docstrings of sys.float_info. (GH-2251) Message-ID: <mailman.192.1522061863.1871.python-checkins@python.org> https://github.com/python/cpython/commit/f86b0984d0f2489c83087eab199ac4e877becf84 commit: f86b0984d0f2489c83087eab199ac4e877becf84 branch: 3.6 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: GitHub <noreply at github.com> date: 2018-03-26T03:57:37-07:00 summary: Fix typo and edit for clarity in the docstrings of sys.float_info. (GH-2251) (cherry picked from commit 0301c9bdd1ebd788d1334cf3fe06c48f35bab0dc) Co-authored-by: Stefano Taschini <taschini at users.noreply.github.com> files: M Objects/floatobject.c diff --git a/Objects/floatobject.c b/Objects/floatobject.c index 1f134aa11315..55406790eda4 100644 --- a/Objects/floatobject.c +++ b/Objects/floatobject.c @@ -1,4 +1,3 @@ - /* Float object implementation */ /* XXX There should be overflow checks here, but it's hard to check @@ -48,7 +47,7 @@ static PyStructSequence_Field floatinfo_fields[] = { "is representable"}, {"max_10_exp", "DBL_MAX_10_EXP -- maximum int e such that 10**e " "is representable"}, - {"min", "DBL_MIN -- Minimum positive normalizer float"}, + {"min", "DBL_MIN -- Minimum positive normalized float"}, {"min_exp", "DBL_MIN_EXP -- minimum int e such that radix**(e-1) " "is a normalized float"}, {"min_10_exp", "DBL_MIN_10_EXP -- minimum int e such that 10**e is " @@ -58,7 +57,7 @@ static PyStructSequence_Field floatinfo_fields[] = { {"epsilon", "DBL_EPSILON -- Difference between 1 and the next " "representable float"}, {"radix", "FLT_RADIX -- radix of exponent"}, - {"rounds", "FLT_ROUNDS -- addition rounds"}, + {"rounds", "FLT_ROUNDS -- rounding mode"}, {0} }; From webhook-mailer at python.org Mon Mar 26 06:58:13 2018 From: webhook-mailer at python.org (Miss Islington (bot)) Date: Mon, 26 Mar 2018 10:58:13 -0000 Subject: [Python-checkins] Fix typo and edit for clarity in the docstrings of sys.float_info. (GH-2251) Message-ID: <mailman.193.1522061895.1871.python-checkins@python.org> https://github.com/python/cpython/commit/a6c3299605443bf81e12f08672ba9eb5aac28c90 commit: a6c3299605443bf81e12f08672ba9eb5aac28c90 branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: GitHub <noreply at github.com> date: 2018-03-26T03:58:10-07:00 summary: Fix typo and edit for clarity in the docstrings of sys.float_info. (GH-2251) (cherry picked from commit 0301c9bdd1ebd788d1334cf3fe06c48f35bab0dc) Co-authored-by: Stefano Taschini <taschini at users.noreply.github.com> files: M Objects/floatobject.c diff --git a/Objects/floatobject.c b/Objects/floatobject.c index 47a174c241e3..67f9e5d5b4ef 100644 --- a/Objects/floatobject.c +++ b/Objects/floatobject.c @@ -1,4 +1,3 @@ - /* Float object implementation */ /* XXX There should be overflow checks here, but it's hard to check @@ -54,7 +53,7 @@ static PyStructSequence_Field floatinfo_fields[] = { "is representable"}, {"max_10_exp", "DBL_MAX_10_EXP -- maximum int e such that 10**e " "is representable"}, - {"min", "DBL_MIN -- Minimum positive normalizer float"}, + {"min", "DBL_MIN -- Minimum positive normalized float"}, {"min_exp", "DBL_MIN_EXP -- minimum int e such that radix**(e-1) " "is a normalized float"}, {"min_10_exp", "DBL_MIN_10_EXP -- minimum int e such that 10**e is " @@ -64,7 +63,7 @@ static PyStructSequence_Field floatinfo_fields[] = { {"epsilon", "DBL_EPSILON -- Difference between 1 and the next " "representable float"}, {"radix", "FLT_RADIX -- radix of exponent"}, - {"rounds", "FLT_ROUNDS -- addition rounds"}, + {"rounds", "FLT_ROUNDS -- rounding mode"}, {0} }; From webhook-mailer at python.org Mon Mar 26 06:58:52 2018 From: webhook-mailer at python.org (Miss Islington (bot)) Date: Mon, 26 Mar 2018 10:58:52 -0000 Subject: [Python-checkins] Fix typo and edit for clarity in the docstrings of sys.float_info. (GH-2251) Message-ID: <mailman.194.1522061932.1871.python-checkins@python.org> https://github.com/python/cpython/commit/e3e8bdc5d30c8814b00e88f214cf94bf4d69a766 commit: e3e8bdc5d30c8814b00e88f214cf94bf4d69a766 branch: 2.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: GitHub <noreply at github.com> date: 2018-03-26T03:58:47-07:00 summary: Fix typo and edit for clarity in the docstrings of sys.float_info. (GH-2251) (cherry picked from commit 0301c9bdd1ebd788d1334cf3fe06c48f35bab0dc) Co-authored-by: Stefano Taschini <taschini at users.noreply.github.com> files: M Objects/floatobject.c diff --git a/Objects/floatobject.c b/Objects/floatobject.c index 360ef94d2f7d..37dd67e66938 100644 --- a/Objects/floatobject.c +++ b/Objects/floatobject.c @@ -1,4 +1,3 @@ - /* Float object implementation */ /* XXX There should be overflow checks here, but it's hard to check @@ -80,7 +79,7 @@ static PyStructSequence_Field floatinfo_fields[] = { "is representable"}, {"max_10_exp", "DBL_MAX_10_EXP -- maximum int e such that 10**e " "is representable"}, - {"min", "DBL_MIN -- Minimum positive normalizer float"}, + {"min", "DBL_MIN -- Minimum positive normalized float"}, {"min_exp", "DBL_MIN_EXP -- minimum int e such that radix**(e-1) " "is a normalized float"}, {"min_10_exp", "DBL_MIN_10_EXP -- minimum int e such that 10**e is " @@ -90,7 +89,7 @@ static PyStructSequence_Field floatinfo_fields[] = { {"epsilon", "DBL_EPSILON -- Difference between 1 and the next " "representable float"}, {"radix", "FLT_RADIX -- radix of exponent"}, - {"rounds", "FLT_ROUNDS -- addition rounds"}, + {"rounds", "FLT_ROUNDS -- rounding mode"}, {0} }; From webhook-mailer at python.org Mon Mar 26 07:03:43 2018 From: webhook-mailer at python.org (Serhiy Storchaka) Date: Mon, 26 Mar 2018 11:03:43 -0000 Subject: [Python-checkins] Fix description about SimpleXMLRPCServer constructor parameter bind_and_activate. (GH-776) Message-ID: <mailman.195.1522062225.1871.python-checkins@python.org> https://github.com/python/cpython/commit/e6223579c87b93f3e60d28796f521587d88091d4 commit: e6223579c87b93f3e60d28796f521587d88091d4 branch: master author: cocoatomo <cocoatomo77 at gmail.com> committer: Serhiy Storchaka <storchaka at gmail.com> date: 2018-03-26T14:03:40+03:00 summary: Fix description about SimpleXMLRPCServer constructor parameter bind_and_activate. (GH-776) Passing True as the `bind_and_activate` *do* immediately opening and binding to their socket. files: M Doc/whatsnew/2.6.rst diff --git a/Doc/whatsnew/2.6.rst b/Doc/whatsnew/2.6.rst index 33d9f7ce43b1..ccfdbdce03e9 100644 --- a/Doc/whatsnew/2.6.rst +++ b/Doc/whatsnew/2.6.rst @@ -2611,7 +2611,7 @@ changes, or look through the Subversion logs for all the details. * The XML-RPC :class:`SimpleXMLRPCServer` and :class:`DocXMLRPCServer` classes can now be prevented from immediately opening and binding to - their socket by passing True as the ``bind_and_activate`` + their socket by passing ``False`` as the *bind_and_activate* constructor parameter. This can be used to modify the instance's :attr:`allow_reuse_address` attribute before calling the :meth:`server_bind` and :meth:`server_activate` methods to From webhook-mailer at python.org Mon Mar 26 07:26:37 2018 From: webhook-mailer at python.org (Miss Islington (bot)) Date: Mon, 26 Mar 2018 11:26:37 -0000 Subject: [Python-checkins] Gitignore gmon.out (GH-5796) Message-ID: <mailman.196.1522063598.1871.python-checkins@python.org> https://github.com/python/cpython/commit/daf06e08dafe61f085623a1f3ebe10fb27354cde commit: daf06e08dafe61f085623a1f3ebe10fb27354cde branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: GitHub <noreply at github.com> date: 2018-03-26T04:26:34-07:00 summary: Gitignore gmon.out (GH-5796) gmon.out is generated when profiling turned on Full Configuration: ./configure --prefix=$PWD/install --enable-profiling --enable-big-digits=30 --with-pydebug --with-assertions --with-valgrind (cherry picked from commit 95ad3822a2b6287772bd752b6ab493c6d4198d4b) Co-authored-by: Neeraj Badlani <neerajbadlani at gmail.com> files: M .gitignore diff --git a/.gitignore b/.gitignore index 05fb6cba0875..58f8bf72f2b9 100644 --- a/.gitignore +++ b/.gitignore @@ -113,3 +113,4 @@ Tools/ssl/amd64 Tools/ssl/win32 .vs/ .vscode/ +gmon.out From webhook-mailer at python.org Mon Mar 26 07:40:38 2018 From: webhook-mailer at python.org (Serhiy Storchaka) Date: Mon, 26 Mar 2018 11:40:38 -0000 Subject: [Python-checkins] Corrected link targets in collections.rst (GH-1052) Message-ID: <mailman.197.1522064439.1871.python-checkins@python.org> https://github.com/python/cpython/commit/e105294708ffc78a31566b48468746eb4609abe7 commit: e105294708ffc78a31566b48468746eb4609abe7 branch: master author: Michael Seifert <michaelseifert04 at yahoo.de> committer: Serhiy Storchaka <storchaka at gmail.com> date: 2018-03-26T14:40:35+03:00 summary: Corrected link targets in collections.rst (GH-1052) files: M Doc/library/collections.rst diff --git a/Doc/library/collections.rst b/Doc/library/collections.rst index 96475b9f490c..2a83d3037277 100644 --- a/Doc/library/collections.rst +++ b/Doc/library/collections.rst @@ -114,7 +114,7 @@ The class can be used to simulate nested scopes and is useful in templating. for templating is a read-only chain of mappings. It also features pushing and popping of contexts similar to the :meth:`~collections.ChainMap.new_child` method and the - :meth:`~collections.ChainMap.parents` property. + :attr:`~collections.ChainMap.parents` property. * The `Nested Contexts recipe <https://code.activestate.com/recipes/577434/>`_ has options to control @@ -270,7 +270,7 @@ For example:: Return a list of the *n* most common elements and their counts from the most common to the least. If *n* is omitted or ``None``, - :func:`most_common` returns *all* elements in the counter. + :meth:`most_common` returns *all* elements in the counter. Elements with equal counts are ordered arbitrarily: >>> Counter('abracadabra').most_common(3) # doctest: +SKIP @@ -357,12 +357,12 @@ or subtracting from an empty counter. restrictions on its keys and values. The values are intended to be numbers representing counts, but you *could* store anything in the value field. - * The :meth:`most_common` method requires only that the values be orderable. + * The :meth:`~Counter.most_common` method requires only that the values be orderable. * For in-place operations such as ``c[key] += 1``, the value type need only support addition and subtraction. So fractions, floats, and decimals would work and negative values are supported. The same is also true for - :meth:`update` and :meth:`subtract` which allow negative and zero values + :meth:`~Counter.update` and :meth:`~Counter.subtract` which allow negative and zero values for both inputs and outputs. * The multiset methods are designed only for use cases with positive values. @@ -370,7 +370,7 @@ or subtracting from an empty counter. are created. There are no type restrictions, but the value type needs to support addition, subtraction, and comparison. - * The :meth:`elements` method requires integer counts. It ignores zero and + * The :meth:`~Counter.elements` method requires integer counts. It ignores zero and negative counts. .. seealso:: @@ -388,9 +388,9 @@ or subtracting from an empty counter. Section 4.6.3, Exercise 19*. * To enumerate all distinct multisets of a given size over a given set of - elements, see :func:`itertools.combinations_with_replacement`: + elements, see :func:`itertools.combinations_with_replacement`:: - map(Counter, combinations_with_replacement('ABC', 2)) --> AA AB AC BB BC CC + map(Counter, combinations_with_replacement('ABC', 2)) # --> AA AB AC BB BC CC :class:`deque` objects @@ -640,9 +640,9 @@ the :meth:`~deque.rotate` method:: # Remove an exhausted iterator. iterators.popleft() -The :meth:`rotate` method provides a way to implement :class:`deque` slicing and +The :meth:`~deque.rotate` method provides a way to implement :class:`deque` slicing and deletion. For example, a pure Python implementation of ``del d[n]`` relies on -the :meth:`rotate` method to position elements to be popped:: +the ``rotate()`` method to position elements to be popped:: def delete_nth(d, n): d.rotate(-n) @@ -650,8 +650,8 @@ the :meth:`rotate` method to position elements to be popped:: d.rotate(n) To implement :class:`deque` slicing, use a similar approach applying -:meth:`rotate` to bring a target element to the left side of the deque. Remove -old entries with :meth:`popleft`, add new entries with :meth:`extend`, and then +:meth:`~deque.rotate` to bring a target element to the left side of the deque. Remove +old entries with :meth:`~deque.popleft`, add new entries with :meth:`~deque.extend`, and then reverse the rotation. With minor variations on that approach, it is easy to implement Forth style stack manipulations such as ``dup``, ``drop``, ``swap``, ``over``, ``pick``, @@ -712,7 +712,7 @@ stack manipulations such as ``dup``, ``drop``, ``swap``, ``over``, ``pick``, :class:`defaultdict` Examples ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Using :class:`list` as the :attr:`default_factory`, it is easy to group a +Using :class:`list` as the :attr:`~defaultdict.default_factory`, it is easy to group a sequence of key-value pairs into a dictionary of lists: >>> s = [('yellow', 1), ('blue', 2), ('yellow', 3), ('blue', 4), ('red', 1)] @@ -724,7 +724,7 @@ sequence of key-value pairs into a dictionary of lists: [('blue', [2, 4]), ('red', [1]), ('yellow', [1, 3])] When each key is encountered for the first time, it is not already in the -mapping; so an entry is automatically created using the :attr:`default_factory` +mapping; so an entry is automatically created using the :attr:`~defaultdict.default_factory` function which returns an empty :class:`list`. The :meth:`list.append` operation then attaches the value to the new list. When keys are encountered again, the look-up proceeds normally (returning the list for that key) and the @@ -738,7 +738,7 @@ simpler and faster than an equivalent technique using :meth:`dict.setdefault`: >>> sorted(d.items()) [('blue', [2, 4]), ('red', [1]), ('yellow', [1, 3])] -Setting the :attr:`default_factory` to :class:`int` makes the +Setting the :attr:`~defaultdict.default_factory` to :class:`int` makes the :class:`defaultdict` useful for counting (like a bag or multiset in other languages): @@ -751,7 +751,7 @@ languages): [('i', 4), ('m', 1), ('p', 2), ('s', 4)] When a letter is first encountered, it is missing from the mapping, so the -:attr:`default_factory` function calls :func:`int` to supply a default count of +:attr:`~defaultdict.default_factory` function calls :func:`int` to supply a default count of zero. The increment operation then builds up the count for each letter. The function :func:`int` which always returns zero is just a special case of @@ -766,7 +766,7 @@ zero): >>> '%(name)s %(action)s to %(object)s' % d 'John ran to <missing>' -Setting the :attr:`default_factory` to :class:`set` makes the +Setting the :attr:`~defaultdict.default_factory` to :class:`set` makes the :class:`defaultdict` useful for building a dictionary of sets: >>> s = [('red', 1), ('blue', 2), ('red', 3), ('blue', 4), ('red', 1), ('blue', 4)] @@ -973,7 +973,7 @@ The subclass shown above sets ``__slots__`` to an empty tuple. This helps keep memory requirements low by preventing the creation of instance dictionaries. Subclassing is not useful for adding new, stored fields. Instead, simply -create a new named tuple type from the :attr:`_fields` attribute: +create a new named tuple type from the :attr:`~somenamedtuple._fields` attribute: >>> Point3D = namedtuple('Point3D', Point._fields + ('z',)) @@ -989,7 +989,7 @@ fields: .. versionchanged:: 3.5 Property docstrings became writeable. -Default values can be implemented by using :meth:`_replace` to +Default values can be implemented by using :meth:`~somenamedtuple._replace` to customize a prototype instance: >>> Account = namedtuple('Account', 'owner balance transaction_count') @@ -1200,15 +1200,22 @@ subclass directly from :class:`str`; however, this class can be easier to work with because the underlying string is accessible as an attribute. -.. class:: UserString([sequence]) +.. class:: UserString(seq) - Class that simulates a string or a Unicode string object. The instance's + Class that simulates a string object. The instance's content is kept in a regular string object, which is accessible via the :attr:`data` attribute of :class:`UserString` instances. The instance's - contents are initially set to a copy of *sequence*. The *sequence* can - be an instance of :class:`bytes`, :class:`str`, :class:`UserString` (or a - subclass) or an arbitrary sequence which can be converted into a string using - the built-in :func:`str` function. + contents are initially set to a copy of *seq*. The *seq* argument can + be any object which can be converted into a string using the built-in + :func:`str` function. + + In addition to supporting the methods and operations of strings, + :class:`UserString` instances provide the following attribute: + + .. attribute:: data + + A real :class:`str` object used to store the contents of the + :class:`UserString` class. .. versionchanged:: 3.5 New methods ``__getnewargs__``, ``__rmod__``, ``casefold``, From webhook-mailer at python.org Mon Mar 26 07:53:54 2018 From: webhook-mailer at python.org (Miss Islington (bot)) Date: Mon, 26 Mar 2018 11:53:54 -0000 Subject: [Python-checkins] Gitignore gmon.out (GH-5796) Message-ID: <mailman.198.1522065235.1871.python-checkins@python.org> https://github.com/python/cpython/commit/80af7320e756c42d69971733fe7238e18a1f75fa commit: 80af7320e756c42d69971733fe7238e18a1f75fa branch: 3.6 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: GitHub <noreply at github.com> date: 2018-03-26T04:53:51-07:00 summary: Gitignore gmon.out (GH-5796) gmon.out is generated when profiling turned on Full Configuration: ./configure --prefix=$PWD/install --enable-profiling --enable-big-digits=30 --with-pydebug --with-assertions --with-valgrind (cherry picked from commit 95ad3822a2b6287772bd752b6ab493c6d4198d4b) Co-authored-by: Neeraj Badlani <neerajbadlani at gmail.com> files: M .gitignore diff --git a/.gitignore b/.gitignore index 81f31989c55a..945ee76c609a 100644 --- a/.gitignore +++ b/.gitignore @@ -100,3 +100,4 @@ Tools/ssl/amd64 Tools/ssl/win32 .vs/ .vscode/ +gmon.out From webhook-mailer at python.org Mon Mar 26 08:09:25 2018 From: webhook-mailer at python.org (Miss Islington (bot)) Date: Mon, 26 Mar 2018 12:09:25 -0000 Subject: [Python-checkins] Fix description about SimpleXMLRPCServer constructor parameter bind_and_activate. (GH-776) Message-ID: <mailman.199.1522066166.1871.python-checkins@python.org> https://github.com/python/cpython/commit/7aef29779f7da8a390506494a58f597ba4667d35 commit: 7aef29779f7da8a390506494a58f597ba4667d35 branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: GitHub <noreply at github.com> date: 2018-03-26T05:09:23-07:00 summary: Fix description about SimpleXMLRPCServer constructor parameter bind_and_activate. (GH-776) Passing True as the `bind_and_activate` *do* immediately opening and binding to their socket. (cherry picked from commit e6223579c87b93f3e60d28796f521587d88091d4) Co-authored-by: cocoatomo <cocoatomo77 at gmail.com> files: M Doc/whatsnew/2.6.rst diff --git a/Doc/whatsnew/2.6.rst b/Doc/whatsnew/2.6.rst index 33d9f7ce43b1..ccfdbdce03e9 100644 --- a/Doc/whatsnew/2.6.rst +++ b/Doc/whatsnew/2.6.rst @@ -2611,7 +2611,7 @@ changes, or look through the Subversion logs for all the details. * The XML-RPC :class:`SimpleXMLRPCServer` and :class:`DocXMLRPCServer` classes can now be prevented from immediately opening and binding to - their socket by passing True as the ``bind_and_activate`` + their socket by passing ``False`` as the *bind_and_activate* constructor parameter. This can be used to modify the instance's :attr:`allow_reuse_address` attribute before calling the :meth:`server_bind` and :meth:`server_activate` methods to From webhook-mailer at python.org Mon Mar 26 08:11:18 2018 From: webhook-mailer at python.org (Miss Islington (bot)) Date: Mon, 26 Mar 2018 12:11:18 -0000 Subject: [Python-checkins] Fix description about SimpleXMLRPCServer constructor parameter bind_and_activate. (GH-776) Message-ID: <mailman.200.1522066279.1871.python-checkins@python.org> https://github.com/python/cpython/commit/a2665075cc2fb85129afdfd7a7f04dd70a38e582 commit: a2665075cc2fb85129afdfd7a7f04dd70a38e582 branch: 2.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: GitHub <noreply at github.com> date: 2018-03-26T05:11:15-07:00 summary: Fix description about SimpleXMLRPCServer constructor parameter bind_and_activate. (GH-776) Passing True as the `bind_and_activate` *do* immediately opening and binding to their socket. (cherry picked from commit e6223579c87b93f3e60d28796f521587d88091d4) Co-authored-by: cocoatomo <cocoatomo77 at gmail.com> files: M Doc/whatsnew/2.6.rst diff --git a/Doc/whatsnew/2.6.rst b/Doc/whatsnew/2.6.rst index 7c430094d831..f4c742e1d91d 100644 --- a/Doc/whatsnew/2.6.rst +++ b/Doc/whatsnew/2.6.rst @@ -2607,7 +2607,7 @@ changes, or look through the Subversion logs for all the details. * The XML-RPC :class:`~SimpleXMLRPCServer.SimpleXMLRPCServer` and :class:`~DocXMLRPCServer.DocXMLRPCServer` classes can now be prevented from immediately opening and binding to - their socket by passing True as the ``bind_and_activate`` + their socket by passing ``False`` as the *bind_and_activate* constructor parameter. This can be used to modify the instance's :attr:`allow_reuse_address` attribute before calling the :meth:`server_bind` and :meth:`server_activate` methods to From webhook-mailer at python.org Mon Mar 26 08:20:28 2018 From: webhook-mailer at python.org (Miss Islington (bot)) Date: Mon, 26 Mar 2018 12:20:28 -0000 Subject: [Python-checkins] Fix description about SimpleXMLRPCServer constructor parameter bind_and_activate. (GH-776) Message-ID: <mailman.201.1522066828.1871.python-checkins@python.org> https://github.com/python/cpython/commit/a413a91949e5472aee518a89a61cc21af4b49e3b commit: a413a91949e5472aee518a89a61cc21af4b49e3b branch: 3.6 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: GitHub <noreply at github.com> date: 2018-03-26T05:20:25-07:00 summary: Fix description about SimpleXMLRPCServer constructor parameter bind_and_activate. (GH-776) Passing True as the `bind_and_activate` *do* immediately opening and binding to their socket. (cherry picked from commit e6223579c87b93f3e60d28796f521587d88091d4) Co-authored-by: cocoatomo <cocoatomo77 at gmail.com> files: M Doc/whatsnew/2.6.rst diff --git a/Doc/whatsnew/2.6.rst b/Doc/whatsnew/2.6.rst index 45b0ab9f3453..e57efee85bd9 100644 --- a/Doc/whatsnew/2.6.rst +++ b/Doc/whatsnew/2.6.rst @@ -2611,7 +2611,7 @@ changes, or look through the Subversion logs for all the details. * The XML-RPC :class:`SimpleXMLRPCServer` and :class:`DocXMLRPCServer` classes can now be prevented from immediately opening and binding to - their socket by passing True as the ``bind_and_activate`` + their socket by passing ``False`` as the *bind_and_activate* constructor parameter. This can be used to modify the instance's :attr:`allow_reuse_address` attribute before calling the :meth:`server_bind` and :meth:`server_activate` methods to From webhook-mailer at python.org Mon Mar 26 08:52:40 2018 From: webhook-mailer at python.org (Miss Islington (bot)) Date: Mon, 26 Mar 2018 12:52:40 -0000 Subject: [Python-checkins] Corrected link targets in collections.rst (GH-1052) Message-ID: <mailman.202.1522068762.1871.python-checkins@python.org> https://github.com/python/cpython/commit/d01a805a388ba2e04c0d35ebec911c1f9743dbd7 commit: d01a805a388ba2e04c0d35ebec911c1f9743dbd7 branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: GitHub <noreply at github.com> date: 2018-03-26T05:52:35-07:00 summary: Corrected link targets in collections.rst (GH-1052) (cherry picked from commit e105294708ffc78a31566b48468746eb4609abe7) Co-authored-by: Michael Seifert <michaelseifert04 at yahoo.de> files: M Doc/library/collections.rst diff --git a/Doc/library/collections.rst b/Doc/library/collections.rst index 96475b9f490c..2a83d3037277 100644 --- a/Doc/library/collections.rst +++ b/Doc/library/collections.rst @@ -114,7 +114,7 @@ The class can be used to simulate nested scopes and is useful in templating. for templating is a read-only chain of mappings. It also features pushing and popping of contexts similar to the :meth:`~collections.ChainMap.new_child` method and the - :meth:`~collections.ChainMap.parents` property. + :attr:`~collections.ChainMap.parents` property. * The `Nested Contexts recipe <https://code.activestate.com/recipes/577434/>`_ has options to control @@ -270,7 +270,7 @@ For example:: Return a list of the *n* most common elements and their counts from the most common to the least. If *n* is omitted or ``None``, - :func:`most_common` returns *all* elements in the counter. + :meth:`most_common` returns *all* elements in the counter. Elements with equal counts are ordered arbitrarily: >>> Counter('abracadabra').most_common(3) # doctest: +SKIP @@ -357,12 +357,12 @@ or subtracting from an empty counter. restrictions on its keys and values. The values are intended to be numbers representing counts, but you *could* store anything in the value field. - * The :meth:`most_common` method requires only that the values be orderable. + * The :meth:`~Counter.most_common` method requires only that the values be orderable. * For in-place operations such as ``c[key] += 1``, the value type need only support addition and subtraction. So fractions, floats, and decimals would work and negative values are supported. The same is also true for - :meth:`update` and :meth:`subtract` which allow negative and zero values + :meth:`~Counter.update` and :meth:`~Counter.subtract` which allow negative and zero values for both inputs and outputs. * The multiset methods are designed only for use cases with positive values. @@ -370,7 +370,7 @@ or subtracting from an empty counter. are created. There are no type restrictions, but the value type needs to support addition, subtraction, and comparison. - * The :meth:`elements` method requires integer counts. It ignores zero and + * The :meth:`~Counter.elements` method requires integer counts. It ignores zero and negative counts. .. seealso:: @@ -388,9 +388,9 @@ or subtracting from an empty counter. Section 4.6.3, Exercise 19*. * To enumerate all distinct multisets of a given size over a given set of - elements, see :func:`itertools.combinations_with_replacement`: + elements, see :func:`itertools.combinations_with_replacement`:: - map(Counter, combinations_with_replacement('ABC', 2)) --> AA AB AC BB BC CC + map(Counter, combinations_with_replacement('ABC', 2)) # --> AA AB AC BB BC CC :class:`deque` objects @@ -640,9 +640,9 @@ the :meth:`~deque.rotate` method:: # Remove an exhausted iterator. iterators.popleft() -The :meth:`rotate` method provides a way to implement :class:`deque` slicing and +The :meth:`~deque.rotate` method provides a way to implement :class:`deque` slicing and deletion. For example, a pure Python implementation of ``del d[n]`` relies on -the :meth:`rotate` method to position elements to be popped:: +the ``rotate()`` method to position elements to be popped:: def delete_nth(d, n): d.rotate(-n) @@ -650,8 +650,8 @@ the :meth:`rotate` method to position elements to be popped:: d.rotate(n) To implement :class:`deque` slicing, use a similar approach applying -:meth:`rotate` to bring a target element to the left side of the deque. Remove -old entries with :meth:`popleft`, add new entries with :meth:`extend`, and then +:meth:`~deque.rotate` to bring a target element to the left side of the deque. Remove +old entries with :meth:`~deque.popleft`, add new entries with :meth:`~deque.extend`, and then reverse the rotation. With minor variations on that approach, it is easy to implement Forth style stack manipulations such as ``dup``, ``drop``, ``swap``, ``over``, ``pick``, @@ -712,7 +712,7 @@ stack manipulations such as ``dup``, ``drop``, ``swap``, ``over``, ``pick``, :class:`defaultdict` Examples ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Using :class:`list` as the :attr:`default_factory`, it is easy to group a +Using :class:`list` as the :attr:`~defaultdict.default_factory`, it is easy to group a sequence of key-value pairs into a dictionary of lists: >>> s = [('yellow', 1), ('blue', 2), ('yellow', 3), ('blue', 4), ('red', 1)] @@ -724,7 +724,7 @@ sequence of key-value pairs into a dictionary of lists: [('blue', [2, 4]), ('red', [1]), ('yellow', [1, 3])] When each key is encountered for the first time, it is not already in the -mapping; so an entry is automatically created using the :attr:`default_factory` +mapping; so an entry is automatically created using the :attr:`~defaultdict.default_factory` function which returns an empty :class:`list`. The :meth:`list.append` operation then attaches the value to the new list. When keys are encountered again, the look-up proceeds normally (returning the list for that key) and the @@ -738,7 +738,7 @@ simpler and faster than an equivalent technique using :meth:`dict.setdefault`: >>> sorted(d.items()) [('blue', [2, 4]), ('red', [1]), ('yellow', [1, 3])] -Setting the :attr:`default_factory` to :class:`int` makes the +Setting the :attr:`~defaultdict.default_factory` to :class:`int` makes the :class:`defaultdict` useful for counting (like a bag or multiset in other languages): @@ -751,7 +751,7 @@ languages): [('i', 4), ('m', 1), ('p', 2), ('s', 4)] When a letter is first encountered, it is missing from the mapping, so the -:attr:`default_factory` function calls :func:`int` to supply a default count of +:attr:`~defaultdict.default_factory` function calls :func:`int` to supply a default count of zero. The increment operation then builds up the count for each letter. The function :func:`int` which always returns zero is just a special case of @@ -766,7 +766,7 @@ zero): >>> '%(name)s %(action)s to %(object)s' % d 'John ran to <missing>' -Setting the :attr:`default_factory` to :class:`set` makes the +Setting the :attr:`~defaultdict.default_factory` to :class:`set` makes the :class:`defaultdict` useful for building a dictionary of sets: >>> s = [('red', 1), ('blue', 2), ('red', 3), ('blue', 4), ('red', 1), ('blue', 4)] @@ -973,7 +973,7 @@ The subclass shown above sets ``__slots__`` to an empty tuple. This helps keep memory requirements low by preventing the creation of instance dictionaries. Subclassing is not useful for adding new, stored fields. Instead, simply -create a new named tuple type from the :attr:`_fields` attribute: +create a new named tuple type from the :attr:`~somenamedtuple._fields` attribute: >>> Point3D = namedtuple('Point3D', Point._fields + ('z',)) @@ -989,7 +989,7 @@ fields: .. versionchanged:: 3.5 Property docstrings became writeable. -Default values can be implemented by using :meth:`_replace` to +Default values can be implemented by using :meth:`~somenamedtuple._replace` to customize a prototype instance: >>> Account = namedtuple('Account', 'owner balance transaction_count') @@ -1200,15 +1200,22 @@ subclass directly from :class:`str`; however, this class can be easier to work with because the underlying string is accessible as an attribute. -.. class:: UserString([sequence]) +.. class:: UserString(seq) - Class that simulates a string or a Unicode string object. The instance's + Class that simulates a string object. The instance's content is kept in a regular string object, which is accessible via the :attr:`data` attribute of :class:`UserString` instances. The instance's - contents are initially set to a copy of *sequence*. The *sequence* can - be an instance of :class:`bytes`, :class:`str`, :class:`UserString` (or a - subclass) or an arbitrary sequence which can be converted into a string using - the built-in :func:`str` function. + contents are initially set to a copy of *seq*. The *seq* argument can + be any object which can be converted into a string using the built-in + :func:`str` function. + + In addition to supporting the methods and operations of strings, + :class:`UserString` instances provide the following attribute: + + .. attribute:: data + + A real :class:`str` object used to store the contents of the + :class:`UserString` class. .. versionchanged:: 3.5 New methods ``__getnewargs__``, ``__rmod__``, ``casefold``, From webhook-mailer at python.org Mon Mar 26 11:04:45 2018 From: webhook-mailer at python.org (Serhiy Storchaka) Date: Mon, 26 Mar 2018 15:04:45 -0000 Subject: [Python-checkins] bpo-6986: Add a comment to clarify a test of _json.make_encoder(). (GH-3789) Message-ID: <mailman.203.1522076686.1871.python-checkins@python.org> https://github.com/python/cpython/commit/7c2d97827fd2ccd72ab7a98427f87fcfe3891644 commit: 7c2d97827fd2ccd72ab7a98427f87fcfe3891644 branch: master author: Oren Milman <orenmn at gmail.com> committer: Serhiy Storchaka <storchaka at gmail.com> date: 2018-03-26T18:04:39+03:00 summary: bpo-6986: Add a comment to clarify a test of _json.make_encoder(). (GH-3789) files: M Lib/test/test_json/test_speedups.py diff --git a/Lib/test/test_json/test_speedups.py b/Lib/test/test_json/test_speedups.py index 5dad69208704..fbfee1a58209 100644 --- a/Lib/test/test_json/test_speedups.py +++ b/Lib/test/test_json/test_speedups.py @@ -31,6 +31,8 @@ def test(value): class TestEncode(CTest): def test_make_encoder(self): + # bpo-6986: The interpreter shouldn't crash in case c_make_encoder() + # receives invalid arguments. self.assertRaises(TypeError, self.json.encoder.c_make_encoder, (True, False), b"\xCD\x7D\x3D\x4E\x12\x4C\xF9\x79\xD7\x52\xBA\x82\xF2\x27\x4A\x7D\xA0\xCA\x75", From webhook-mailer at python.org Mon Mar 26 11:14:03 2018 From: webhook-mailer at python.org (Serhiy Storchaka) Date: Mon, 26 Mar 2018 15:14:03 -0000 Subject: [Python-checkins] Fix error message in sqlite connection thread check. (GH-6028) Message-ID: <mailman.204.1522077244.1871.python-checkins@python.org> https://github.com/python/cpython/commit/030345c0bfc2f76684666fe5c61e766ba5debfe6 commit: 030345c0bfc2f76684666fe5c61e766ba5debfe6 branch: master author: Takuya Akiba <469803+iwiwi at users.noreply.github.com> committer: Serhiy Storchaka <storchaka at gmail.com> date: 2018-03-26T18:14:00+03:00 summary: Fix error message in sqlite connection thread check. (GH-6028) files: M Modules/_sqlite/connection.c diff --git a/Modules/_sqlite/connection.c b/Modules/_sqlite/connection.c index dc5061c1d91d..6e0576151ccb 100644 --- a/Modules/_sqlite/connection.c +++ b/Modules/_sqlite/connection.c @@ -1103,8 +1103,8 @@ int pysqlite_check_thread(pysqlite_Connection* self) if (self->check_same_thread) { if (PyThread_get_thread_ident() != self->thread_ident) { PyErr_Format(pysqlite_ProgrammingError, - "SQLite objects created in a thread can only be used in that same thread." - "The object was created in thread id %lu and this is thread id %lu", + "SQLite objects created in a thread can only be used in that same thread. " + "The object was created in thread id %lu and this is thread id %lu.", self->thread_ident, PyThread_get_thread_ident()); return 0; } From webhook-mailer at python.org Mon Mar 26 12:01:26 2018 From: webhook-mailer at python.org (Miss Islington (bot)) Date: Mon, 26 Mar 2018 16:01:26 -0000 Subject: [Python-checkins] Fix error message in sqlite connection thread check. (GH-6028) Message-ID: <mailman.205.1522080086.1871.python-checkins@python.org> https://github.com/python/cpython/commit/00765bb6ae692570c73fe15e0362ce9c8ec59c82 commit: 00765bb6ae692570c73fe15e0362ce9c8ec59c82 branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: GitHub <noreply at github.com> date: 2018-03-26T09:01:22-07:00 summary: Fix error message in sqlite connection thread check. (GH-6028) (cherry picked from commit 030345c0bfc2f76684666fe5c61e766ba5debfe6) Co-authored-by: Takuya Akiba <469803+iwiwi at users.noreply.github.com> files: M Modules/_sqlite/connection.c diff --git a/Modules/_sqlite/connection.c b/Modules/_sqlite/connection.c index dc5061c1d91d..6e0576151ccb 100644 --- a/Modules/_sqlite/connection.c +++ b/Modules/_sqlite/connection.c @@ -1103,8 +1103,8 @@ int pysqlite_check_thread(pysqlite_Connection* self) if (self->check_same_thread) { if (PyThread_get_thread_ident() != self->thread_ident) { PyErr_Format(pysqlite_ProgrammingError, - "SQLite objects created in a thread can only be used in that same thread." - "The object was created in thread id %lu and this is thread id %lu", + "SQLite objects created in a thread can only be used in that same thread. " + "The object was created in thread id %lu and this is thread id %lu.", self->thread_ident, PyThread_get_thread_ident()); return 0; } From webhook-mailer at python.org Mon Mar 26 13:29:19 2018 From: webhook-mailer at python.org (Eric V. Smith) Date: Mon, 26 Mar 2018 17:29:19 -0000 Subject: [Python-checkins] bpo-33141: Have dataclasses.Field pass through __set_name__ to any default argument. (GH-6260) Message-ID: <mailman.206.1522085360.1871.python-checkins@python.org> https://github.com/python/cpython/commit/de7a2f04d6b9427d568fcb43b6f512f9b4c4bd84 commit: de7a2f04d6b9427d568fcb43b6f512f9b4c4bd84 branch: master author: Eric V. Smith <ericvsmith at users.noreply.github.com> committer: GitHub <noreply at github.com> date: 2018-03-26T13:29:16-04:00 summary: bpo-33141: Have dataclasses.Field pass through __set_name__ to any default argument. (GH-6260) This is part of PEP 487 and the descriptor protocol. files: A Misc/NEWS.d/next/Library/2018-03-26-12-33-13.bpo-33141.23wlxf.rst M Lib/dataclasses.py M Lib/test/test_dataclasses.py diff --git a/Lib/dataclasses.py b/Lib/dataclasses.py index 8ccc4c88aeca..8c197fe73904 100644 --- a/Lib/dataclasses.py +++ b/Lib/dataclasses.py @@ -240,6 +240,20 @@ def __repr__(self): f'metadata={self.metadata}' ')') + # This is used to support the PEP 487 __set_name__ protocol in the + # case where we're using a field that contains a descriptor as a + # defaul value. For details on __set_name__, see + # https://www.python.org/dev/peps/pep-0487/#implementation-details. + # Note that in _process_class, this Field object is overwritten with + # the default value, so the end result is a descriptor that had + # __set_name__ called on it at the right time. + def __set_name__(self, owner, name): + func = getattr(self.default, '__set_name__', None) + if func: + # There is a __set_name__ method on the descriptor, + # call it. + func(owner, name) + class _DataclassParams: __slots__ = ('init', diff --git a/Lib/test/test_dataclasses.py b/Lib/test/test_dataclasses.py index f7f132ca30d2..2745eaf6893b 100755 --- a/Lib/test/test_dataclasses.py +++ b/Lib/test/test_dataclasses.py @@ -2698,6 +2698,48 @@ class Derived(Base): # We can add a new field to the derived instance. d.z = 10 +class TestDescriptors(unittest.TestCase): + def test_set_name(self): + # See bpo-33141. + + # Create a descriptor. + class D: + def __set_name__(self, owner, name): + self.name = name + def __get__(self, instance, owner): + if instance is not None: + return 1 + return self + + # This is the case of just normal descriptor behavior, no + # dataclass code is involved in initializing the descriptor. + @dataclass + class C: + c: int=D() + self.assertEqual(C.c.name, 'c') + + # Now test with a default value and init=False, which is the + # only time this is really meaningful. If not using + # init=False, then the descriptor will be overwritten, anyway. + @dataclass + class C: + c: int=field(default=D(), init=False) + self.assertEqual(C.c.name, 'c') + self.assertEqual(C().c, 1) + + def test_non_descriptor(self): + # PEP 487 says __set_name__ should work on non-descriptors. + # Create a descriptor. + + class D: + def __set_name__(self, owner, name): + self.name = name + + @dataclass + class C: + c: int=field(default=D(), init=False) + self.assertEqual(C.c.name, 'c') + if __name__ == '__main__': unittest.main() diff --git a/Misc/NEWS.d/next/Library/2018-03-26-12-33-13.bpo-33141.23wlxf.rst b/Misc/NEWS.d/next/Library/2018-03-26-12-33-13.bpo-33141.23wlxf.rst new file mode 100644 index 000000000000..1d49c08fed54 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-03-26-12-33-13.bpo-33141.23wlxf.rst @@ -0,0 +1,2 @@ +Have Field objects pass through __set_name__ to their default values, if +they have their own __set_name__. From webhook-mailer at python.org Mon Mar 26 13:55:16 2018 From: webhook-mailer at python.org (Miss Islington (bot)) Date: Mon, 26 Mar 2018 17:55:16 -0000 Subject: [Python-checkins] bpo-33141: Have dataclasses.Field pass through __set_name__ to any default argument. (GH-6260) Message-ID: <mailman.207.1522086917.1871.python-checkins@python.org> https://github.com/python/cpython/commit/c6147acd2ce5fa9e344f179b539f3b21b9ae1a6d commit: c6147acd2ce5fa9e344f179b539f3b21b9ae1a6d branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: GitHub <noreply at github.com> date: 2018-03-26T10:55:13-07:00 summary: bpo-33141: Have dataclasses.Field pass through __set_name__ to any default argument. (GH-6260) This is part of PEP 487 and the descriptor protocol. (cherry picked from commit de7a2f04d6b9427d568fcb43b6f512f9b4c4bd84) Co-authored-by: Eric V. Smith <ericvsmith at users.noreply.github.com> files: A Misc/NEWS.d/next/Library/2018-03-26-12-33-13.bpo-33141.23wlxf.rst M Lib/dataclasses.py M Lib/test/test_dataclasses.py diff --git a/Lib/dataclasses.py b/Lib/dataclasses.py index 8ccc4c88aeca..8c197fe73904 100644 --- a/Lib/dataclasses.py +++ b/Lib/dataclasses.py @@ -240,6 +240,20 @@ def __repr__(self): f'metadata={self.metadata}' ')') + # This is used to support the PEP 487 __set_name__ protocol in the + # case where we're using a field that contains a descriptor as a + # defaul value. For details on __set_name__, see + # https://www.python.org/dev/peps/pep-0487/#implementation-details. + # Note that in _process_class, this Field object is overwritten with + # the default value, so the end result is a descriptor that had + # __set_name__ called on it at the right time. + def __set_name__(self, owner, name): + func = getattr(self.default, '__set_name__', None) + if func: + # There is a __set_name__ method on the descriptor, + # call it. + func(owner, name) + class _DataclassParams: __slots__ = ('init', diff --git a/Lib/test/test_dataclasses.py b/Lib/test/test_dataclasses.py index f7f132ca30d2..2745eaf6893b 100755 --- a/Lib/test/test_dataclasses.py +++ b/Lib/test/test_dataclasses.py @@ -2698,6 +2698,48 @@ class Derived(Base): # We can add a new field to the derived instance. d.z = 10 +class TestDescriptors(unittest.TestCase): + def test_set_name(self): + # See bpo-33141. + + # Create a descriptor. + class D: + def __set_name__(self, owner, name): + self.name = name + def __get__(self, instance, owner): + if instance is not None: + return 1 + return self + + # This is the case of just normal descriptor behavior, no + # dataclass code is involved in initializing the descriptor. + @dataclass + class C: + c: int=D() + self.assertEqual(C.c.name, 'c') + + # Now test with a default value and init=False, which is the + # only time this is really meaningful. If not using + # init=False, then the descriptor will be overwritten, anyway. + @dataclass + class C: + c: int=field(default=D(), init=False) + self.assertEqual(C.c.name, 'c') + self.assertEqual(C().c, 1) + + def test_non_descriptor(self): + # PEP 487 says __set_name__ should work on non-descriptors. + # Create a descriptor. + + class D: + def __set_name__(self, owner, name): + self.name = name + + @dataclass + class C: + c: int=field(default=D(), init=False) + self.assertEqual(C.c.name, 'c') + if __name__ == '__main__': unittest.main() diff --git a/Misc/NEWS.d/next/Library/2018-03-26-12-33-13.bpo-33141.23wlxf.rst b/Misc/NEWS.d/next/Library/2018-03-26-12-33-13.bpo-33141.23wlxf.rst new file mode 100644 index 000000000000..1d49c08fed54 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-03-26-12-33-13.bpo-33141.23wlxf.rst @@ -0,0 +1,2 @@ +Have Field objects pass through __set_name__ to their default values, if +they have their own __set_name__. From webhook-mailer at python.org Mon Mar 26 15:49:38 2018 From: webhook-mailer at python.org (Gregory P. Smith) Date: Mon, 26 Mar 2018 19:49:38 -0000 Subject: [Python-checkins] bpo-32844: Fix a subprocess misredirection of a low fd (GH5689) Message-ID: <mailman.208.1522093780.1871.python-checkins@python.org> https://github.com/python/cpython/commit/0e7144b064a19493a146af94175a087b3888c37b commit: 0e7144b064a19493a146af94175a087b3888c37b branch: master author: Alexey Izbyshev <izbyshev at users.noreply.github.com> committer: Gregory P. Smith <greg at krypto.org> date: 2018-03-26T12:49:35-07:00 summary: bpo-32844: Fix a subprocess misredirection of a low fd (GH5689) bpo-32844: subprocess: Fix a potential misredirection of a low fd to stderr. When redirecting, subprocess attempts to achieve the following state: each fd to be redirected to is less than or equal to the fd it is redirected from, which is necessary because redirection occurs in the ascending order of destination descriptors. It fails to do so in a couple of corner cases, for example, if 1 is redirected to 2 and 0 is closed in the parent. files: A Misc/NEWS.d/next/Library/2018-02-28-13-08-00.bpo-32844.u8tnAe.rst M Lib/test/test_subprocess.py M Modules/_posixsubprocess.c diff --git a/Lib/test/test_subprocess.py b/Lib/test/test_subprocess.py index ddee3b94fed9..2a766d7c92ad 100644 --- a/Lib/test/test_subprocess.py +++ b/Lib/test/test_subprocess.py @@ -6,6 +6,7 @@ import platform import signal import io +import itertools import os import errno import tempfile @@ -2134,6 +2135,55 @@ def test_swap_fds(self): self.check_swap_fds(2, 0, 1) self.check_swap_fds(2, 1, 0) + def _check_swap_std_fds_with_one_closed(self, from_fds, to_fds): + saved_fds = self._save_fds(range(3)) + try: + for from_fd in from_fds: + with tempfile.TemporaryFile() as f: + os.dup2(f.fileno(), from_fd) + + fd_to_close = (set(range(3)) - set(from_fds)).pop() + os.close(fd_to_close) + + arg_names = ['stdin', 'stdout', 'stderr'] + kwargs = {} + for from_fd, to_fd in zip(from_fds, to_fds): + kwargs[arg_names[to_fd]] = from_fd + + code = textwrap.dedent(r''' + import os, sys + skipped_fd = int(sys.argv[1]) + for fd in range(3): + if fd != skipped_fd: + os.write(fd, str(fd).encode('ascii')) + ''') + + skipped_fd = (set(range(3)) - set(to_fds)).pop() + + rc = subprocess.call([sys.executable, '-c', code, str(skipped_fd)], + **kwargs) + self.assertEqual(rc, 0) + + for from_fd, to_fd in zip(from_fds, to_fds): + os.lseek(from_fd, 0, os.SEEK_SET) + read_bytes = os.read(from_fd, 1024) + read_fds = list(map(int, read_bytes.decode('ascii'))) + msg = textwrap.dedent(f""" + When testing {from_fds} to {to_fds} redirection, + parent descriptor {from_fd} got redirected + to descriptor(s) {read_fds} instead of descriptor {to_fd}. + """) + self.assertEqual([to_fd], read_fds, msg) + finally: + self._restore_fds(saved_fds) + + # Check that subprocess can remap std fds correctly even + # if one of them is closed (#32844). + def test_swap_std_fds_with_one_closed(self): + for from_fds in itertools.combinations(range(3), 2): + for to_fds in itertools.permutations(range(3), 2): + self._check_swap_std_fds_with_one_closed(from_fds, to_fds) + def test_surrogates_error_message(self): def prepare(): raise ValueError("surrogate:\uDCff") diff --git a/Misc/NEWS.d/next/Library/2018-02-28-13-08-00.bpo-32844.u8tnAe.rst b/Misc/NEWS.d/next/Library/2018-02-28-13-08-00.bpo-32844.u8tnAe.rst new file mode 100644 index 000000000000..67412fe5ba46 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-02-28-13-08-00.bpo-32844.u8tnAe.rst @@ -0,0 +1,2 @@ +Fix wrong redirection of a low descriptor (0 or 1) to stderr in subprocess +if another low descriptor is closed. diff --git a/Modules/_posixsubprocess.c b/Modules/_posixsubprocess.c index dc43ffc2e19d..0150fcb0970c 100644 --- a/Modules/_posixsubprocess.c +++ b/Modules/_posixsubprocess.c @@ -424,7 +424,7 @@ child_exec(char *const exec_array[], either 0, 1 or 2, it is possible that it is overwritten (#12607). */ if (c2pwrite == 0) POSIX_CALL(c2pwrite = dup(c2pwrite)); - if (errwrite == 0 || errwrite == 1) + while (errwrite == 0 || errwrite == 1) POSIX_CALL(errwrite = dup(errwrite)); /* Dup fds for child. From webhook-mailer at python.org Mon Mar 26 16:14:12 2018 From: webhook-mailer at python.org (Miss Islington (bot)) Date: Mon, 26 Mar 2018 20:14:12 -0000 Subject: [Python-checkins] bpo-32844: Fix a subprocess misredirection of a low fd (GH5689) Message-ID: <mailman.209.1522095253.1871.python-checkins@python.org> https://github.com/python/cpython/commit/05455637f3ba9bacd459700f4feab783e5967d69 commit: 05455637f3ba9bacd459700f4feab783e5967d69 branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: GitHub <noreply at github.com> date: 2018-03-26T13:14:09-07:00 summary: bpo-32844: Fix a subprocess misredirection of a low fd (GH5689) bpo-32844: subprocess: Fix a potential misredirection of a low fd to stderr. When redirecting, subprocess attempts to achieve the following state: each fd to be redirected to is less than or equal to the fd it is redirected from, which is necessary because redirection occurs in the ascending order of destination descriptors. It fails to do so in a couple of corner cases, for example, if 1 is redirected to 2 and 0 is closed in the parent. (cherry picked from commit 0e7144b064a19493a146af94175a087b3888c37b) Co-authored-by: Alexey Izbyshev <izbyshev at users.noreply.github.com> files: A Misc/NEWS.d/next/Library/2018-02-28-13-08-00.bpo-32844.u8tnAe.rst M Lib/test/test_subprocess.py M Modules/_posixsubprocess.c diff --git a/Lib/test/test_subprocess.py b/Lib/test/test_subprocess.py index ddee3b94fed9..2a766d7c92ad 100644 --- a/Lib/test/test_subprocess.py +++ b/Lib/test/test_subprocess.py @@ -6,6 +6,7 @@ import platform import signal import io +import itertools import os import errno import tempfile @@ -2134,6 +2135,55 @@ def test_swap_fds(self): self.check_swap_fds(2, 0, 1) self.check_swap_fds(2, 1, 0) + def _check_swap_std_fds_with_one_closed(self, from_fds, to_fds): + saved_fds = self._save_fds(range(3)) + try: + for from_fd in from_fds: + with tempfile.TemporaryFile() as f: + os.dup2(f.fileno(), from_fd) + + fd_to_close = (set(range(3)) - set(from_fds)).pop() + os.close(fd_to_close) + + arg_names = ['stdin', 'stdout', 'stderr'] + kwargs = {} + for from_fd, to_fd in zip(from_fds, to_fds): + kwargs[arg_names[to_fd]] = from_fd + + code = textwrap.dedent(r''' + import os, sys + skipped_fd = int(sys.argv[1]) + for fd in range(3): + if fd != skipped_fd: + os.write(fd, str(fd).encode('ascii')) + ''') + + skipped_fd = (set(range(3)) - set(to_fds)).pop() + + rc = subprocess.call([sys.executable, '-c', code, str(skipped_fd)], + **kwargs) + self.assertEqual(rc, 0) + + for from_fd, to_fd in zip(from_fds, to_fds): + os.lseek(from_fd, 0, os.SEEK_SET) + read_bytes = os.read(from_fd, 1024) + read_fds = list(map(int, read_bytes.decode('ascii'))) + msg = textwrap.dedent(f""" + When testing {from_fds} to {to_fds} redirection, + parent descriptor {from_fd} got redirected + to descriptor(s) {read_fds} instead of descriptor {to_fd}. + """) + self.assertEqual([to_fd], read_fds, msg) + finally: + self._restore_fds(saved_fds) + + # Check that subprocess can remap std fds correctly even + # if one of them is closed (#32844). + def test_swap_std_fds_with_one_closed(self): + for from_fds in itertools.combinations(range(3), 2): + for to_fds in itertools.permutations(range(3), 2): + self._check_swap_std_fds_with_one_closed(from_fds, to_fds) + def test_surrogates_error_message(self): def prepare(): raise ValueError("surrogate:\uDCff") diff --git a/Misc/NEWS.d/next/Library/2018-02-28-13-08-00.bpo-32844.u8tnAe.rst b/Misc/NEWS.d/next/Library/2018-02-28-13-08-00.bpo-32844.u8tnAe.rst new file mode 100644 index 000000000000..67412fe5ba46 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-02-28-13-08-00.bpo-32844.u8tnAe.rst @@ -0,0 +1,2 @@ +Fix wrong redirection of a low descriptor (0 or 1) to stderr in subprocess +if another low descriptor is closed. diff --git a/Modules/_posixsubprocess.c b/Modules/_posixsubprocess.c index dc43ffc2e19d..0150fcb0970c 100644 --- a/Modules/_posixsubprocess.c +++ b/Modules/_posixsubprocess.c @@ -424,7 +424,7 @@ child_exec(char *const exec_array[], either 0, 1 or 2, it is possible that it is overwritten (#12607). */ if (c2pwrite == 0) POSIX_CALL(c2pwrite = dup(c2pwrite)); - if (errwrite == 0 || errwrite == 1) + while (errwrite == 0 || errwrite == 1) POSIX_CALL(errwrite = dup(errwrite)); /* Dup fds for child. From webhook-mailer at python.org Mon Mar 26 16:43:49 2018 From: webhook-mailer at python.org (Miss Islington (bot)) Date: Mon, 26 Mar 2018 20:43:49 -0000 Subject: [Python-checkins] bpo-32844: Fix a subprocess misredirection of a low fd (GH5689) Message-ID: <mailman.210.1522097030.1871.python-checkins@python.org> https://github.com/python/cpython/commit/57db13e582ad269d6e067fe934122207cc992739 commit: 57db13e582ad269d6e067fe934122207cc992739 branch: 3.6 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: GitHub <noreply at github.com> date: 2018-03-26T13:43:46-07:00 summary: bpo-32844: Fix a subprocess misredirection of a low fd (GH5689) bpo-32844: subprocess: Fix a potential misredirection of a low fd to stderr. When redirecting, subprocess attempts to achieve the following state: each fd to be redirected to is less than or equal to the fd it is redirected from, which is necessary because redirection occurs in the ascending order of destination descriptors. It fails to do so in a couple of corner cases, for example, if 1 is redirected to 2 and 0 is closed in the parent. (cherry picked from commit 0e7144b064a19493a146af94175a087b3888c37b) Co-authored-by: Alexey Izbyshev <izbyshev at users.noreply.github.com> files: A Misc/NEWS.d/next/Library/2018-02-28-13-08-00.bpo-32844.u8tnAe.rst M Lib/test/test_subprocess.py M Modules/_posixsubprocess.c diff --git a/Lib/test/test_subprocess.py b/Lib/test/test_subprocess.py index aca1411cdfb5..9ebe3228ac03 100644 --- a/Lib/test/test_subprocess.py +++ b/Lib/test/test_subprocess.py @@ -6,6 +6,7 @@ import platform import signal import io +import itertools import os import errno import tempfile @@ -2108,6 +2109,55 @@ def test_swap_fds(self): self.check_swap_fds(2, 0, 1) self.check_swap_fds(2, 1, 0) + def _check_swap_std_fds_with_one_closed(self, from_fds, to_fds): + saved_fds = self._save_fds(range(3)) + try: + for from_fd in from_fds: + with tempfile.TemporaryFile() as f: + os.dup2(f.fileno(), from_fd) + + fd_to_close = (set(range(3)) - set(from_fds)).pop() + os.close(fd_to_close) + + arg_names = ['stdin', 'stdout', 'stderr'] + kwargs = {} + for from_fd, to_fd in zip(from_fds, to_fds): + kwargs[arg_names[to_fd]] = from_fd + + code = textwrap.dedent(r''' + import os, sys + skipped_fd = int(sys.argv[1]) + for fd in range(3): + if fd != skipped_fd: + os.write(fd, str(fd).encode('ascii')) + ''') + + skipped_fd = (set(range(3)) - set(to_fds)).pop() + + rc = subprocess.call([sys.executable, '-c', code, str(skipped_fd)], + **kwargs) + self.assertEqual(rc, 0) + + for from_fd, to_fd in zip(from_fds, to_fds): + os.lseek(from_fd, 0, os.SEEK_SET) + read_bytes = os.read(from_fd, 1024) + read_fds = list(map(int, read_bytes.decode('ascii'))) + msg = textwrap.dedent(f""" + When testing {from_fds} to {to_fds} redirection, + parent descriptor {from_fd} got redirected + to descriptor(s) {read_fds} instead of descriptor {to_fd}. + """) + self.assertEqual([to_fd], read_fds, msg) + finally: + self._restore_fds(saved_fds) + + # Check that subprocess can remap std fds correctly even + # if one of them is closed (#32844). + def test_swap_std_fds_with_one_closed(self): + for from_fds in itertools.combinations(range(3), 2): + for to_fds in itertools.permutations(range(3), 2): + self._check_swap_std_fds_with_one_closed(from_fds, to_fds) + def test_surrogates_error_message(self): def prepare(): raise ValueError("surrogate:\uDCff") diff --git a/Misc/NEWS.d/next/Library/2018-02-28-13-08-00.bpo-32844.u8tnAe.rst b/Misc/NEWS.d/next/Library/2018-02-28-13-08-00.bpo-32844.u8tnAe.rst new file mode 100644 index 000000000000..67412fe5ba46 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-02-28-13-08-00.bpo-32844.u8tnAe.rst @@ -0,0 +1,2 @@ +Fix wrong redirection of a low descriptor (0 or 1) to stderr in subprocess +if another low descriptor is closed. diff --git a/Modules/_posixsubprocess.c b/Modules/_posixsubprocess.c index 05a08eb87765..ad934dfe7d87 100644 --- a/Modules/_posixsubprocess.c +++ b/Modules/_posixsubprocess.c @@ -424,7 +424,7 @@ child_exec(char *const exec_array[], either 0, 1 or 2, it is possible that it is overwritten (#12607). */ if (c2pwrite == 0) POSIX_CALL(c2pwrite = dup(c2pwrite)); - if (errwrite == 0 || errwrite == 1) + while (errwrite == 0 || errwrite == 1) POSIX_CALL(errwrite = dup(errwrite)); /* Dup fds for child. From webhook-mailer at python.org Mon Mar 26 18:01:21 2018 From: webhook-mailer at python.org (Ivan Levkivskyi) Date: Mon, 26 Mar 2018 22:01:21 -0000 Subject: [Python-checkins] bpo-32873: Treat type variables and special typing forms as immutable by copy and pickle (GH-6216) Message-ID: <mailman.211.1522101683.1871.python-checkins@python.org> https://github.com/python/cpython/commit/834940375ae88bc95794226dd8eff1f25fba1cf9 commit: 834940375ae88bc95794226dd8eff1f25fba1cf9 branch: master author: Ivan Levkivskyi <levkivskyi at gmail.com> committer: GitHub <noreply at github.com> date: 2018-03-26T23:01:12+01:00 summary: bpo-32873: Treat type variables and special typing forms as immutable by copy and pickle (GH-6216) This also fixes python/typing#512 This also fixes python/typing#511 As was discussed in both issues, some typing forms deserve to be treated as immutable by copy and pickle modules, so that: * copy(X) is X * deepcopy(X) is X * loads(dumps(X)) is X # pickled by reference This PR adds such behaviour to: * Type variables * Special forms like Union, Any, ClassVar * Unsubscripted generic aliases to containers like List, Mapping, Iterable This not only resolves inconsistencies mentioned in the issues, but also improves backwards compatibility with previous versions of Python (including 3.6). Note that this requires some dances with __module__ for type variables (similar to NamedTuple) because the class TypeVar itself is define in typing, while type variables should get module where they were defined. https://bugs.python.org/issue32873 files: A Misc/NEWS.d/next/Library/2018-03-24-19-54-48.bpo-32873.cHyoAm.rst M Lib/test/test_typing.py M Lib/typing.py diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index f56caa13a295..09e39fec45e4 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -1057,20 +1057,20 @@ class C(B[int]): self.assertEqual(x.foo, 42) self.assertEqual(x.bar, 'abc') self.assertEqual(x.__dict__, {'foo': 42, 'bar': 'abc'}) - samples = [Any, Union, Tuple, Callable, ClassVar] + samples = [Any, Union, Tuple, Callable, ClassVar, + Union[int, str], ClassVar[List], Tuple[int, ...], Callable[[str], bytes]] for s in samples: for proto in range(pickle.HIGHEST_PROTOCOL + 1): z = pickle.dumps(s, proto) x = pickle.loads(z) self.assertEqual(s, x) - more_samples = [List, typing.Iterable, typing.Type] + more_samples = [List, typing.Iterable, typing.Type, List[int], + typing.Type[typing.Mapping]] for s in more_samples: for proto in range(pickle.HIGHEST_PROTOCOL + 1): z = pickle.dumps(s, proto) x = pickle.loads(z) - self.assertEqual(repr(s), repr(x)) # TODO: fix this - # see also comment in test_copy_and_deepcopy - # the issue is typing/#512 + self.assertEqual(s, x) def test_copy_and_deepcopy(self): T = TypeVar('T') @@ -1082,7 +1082,27 @@ class Node(Generic[T]): ... Union['T', int], List['T'], typing.Mapping['T', int]] for t in things + [Any]: self.assertEqual(t, copy(t)) - self.assertEqual(repr(t), repr(deepcopy(t))) # Use repr() because of TypeVars + self.assertEqual(t, deepcopy(t)) + + def test_immutability_by_copy_and_pickle(self): + # Special forms like Union, Any, etc., generic aliases to containers like List, + # Mapping, etc., and type variabcles are considered immutable by copy and pickle. + global TP, TPB, TPV # for pickle + TP = TypeVar('TP') + TPB = TypeVar('TPB', bound=int) + TPV = TypeVar('TPV', bytes, str) + for X in [TP, TPB, TPV, List, typing.Mapping, ClassVar, typing.Iterable, + Union, Any, Tuple, Callable]: + self.assertIs(copy(X), X) + self.assertIs(deepcopy(X), X) + self.assertIs(pickle.loads(pickle.dumps(X)), X) + # Check that local type variables are copyable. + TL = TypeVar('TL') + TLB = TypeVar('TLB', bound=int) + TLV = TypeVar('TLV', bytes, str) + for X in [TL, TLB, TLV]: + self.assertIs(copy(X), X) + self.assertIs(deepcopy(X), X) def test_copy_generic_instances(self): T = TypeVar('T') diff --git a/Lib/typing.py b/Lib/typing.py index 56126cfc0247..510574c413e3 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -285,8 +285,17 @@ def __init_subclass__(self, *args, **kwds): if '_root' not in kwds: raise TypeError("Cannot subclass special typing classes") +class _Immutable: + """Mixin to indicate that object should not be copied.""" -class _SpecialForm(_Final, _root=True): + def __copy__(self): + return self + + def __deepcopy__(self, memo): + return self + + +class _SpecialForm(_Final, _Immutable, _root=True): """Internal indicator of special typing constructs. See _doc instance attribute for specific docs. """ @@ -328,8 +337,8 @@ def __hash__(self): def __repr__(self): return 'typing.' + self._name - def __copy__(self): - return self # Special forms are immutable. + def __reduce__(self): + return self._name def __call__(self, *args, **kwds): raise TypeError(f"Cannot instantiate {self!r}") @@ -496,7 +505,11 @@ def __repr__(self): return f'ForwardRef({self.__forward_arg__!r})' -class TypeVar(_Final, _root=True): +def _find_name(mod, name): + return getattr(sys.modules[mod], name) + + +class TypeVar(_Final, _Immutable, _root=True): """Type variable. Usage:: @@ -536,10 +549,12 @@ def longest(x: A, y: A) -> A: T.__covariant__ == False T.__contravariant__ = False A.__constraints__ == (str, bytes) + + Note that only type variables defined in global scope can be pickled. """ __slots__ = ('__name__', '__bound__', '__constraints__', - '__covariant__', '__contravariant__') + '__covariant__', '__contravariant__', '_def_mod') def __init__(self, name, *constraints, bound=None, covariant=False, contravariant=False): @@ -558,6 +573,7 @@ def __init__(self, name, *constraints, bound=None, self.__bound__ = _type_check(bound, "Bound must be a type.") else: self.__bound__ = None + self._def_mod = sys._getframe(1).f_globals['__name__'] # for pickling def __getstate__(self): return {'name': self.__name__, @@ -582,6 +598,9 @@ def __repr__(self): prefix = '~' return prefix + self.__name__ + def __reduce__(self): + return (_find_name, (self._def_mod, self.__name__)) + # Special typing constructs Union, Optional, Generic, Callable and Tuple # use three special attributes for internal bookkeeping of generic types: @@ -724,6 +743,11 @@ def __subclasscheck__(self, cls): raise TypeError("Subscripted generics cannot be used with" " class and instance checks") + def __reduce__(self): + if self._special: + return self._name + return super().__reduce__() + class _VariadicGenericAlias(_GenericAlias, _root=True): """Same as _GenericAlias above but for variadic aliases. Currently, diff --git a/Misc/NEWS.d/next/Library/2018-03-24-19-54-48.bpo-32873.cHyoAm.rst b/Misc/NEWS.d/next/Library/2018-03-24-19-54-48.bpo-32873.cHyoAm.rst new file mode 100644 index 000000000000..99f8088cf138 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-03-24-19-54-48.bpo-32873.cHyoAm.rst @@ -0,0 +1,3 @@ +Treat type variables and special typing forms as immutable by copy and +pickle. This fixes several minor issues and inconsistencies, and improves +backwards compatibility with Python 3.6. From webhook-mailer at python.org Mon Mar 26 18:29:15 2018 From: webhook-mailer at python.org (Miss Islington (bot)) Date: Mon, 26 Mar 2018 22:29:15 -0000 Subject: [Python-checkins] bpo-32873: Treat type variables and special typing forms as immutable by copy and pickle (GH-6216) Message-ID: <mailman.212.1522103357.1871.python-checkins@python.org> https://github.com/python/cpython/commit/d0e04c82448c750d4dc27f2bddeddea74bd353ff commit: d0e04c82448c750d4dc27f2bddeddea74bd353ff branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: GitHub <noreply at github.com> date: 2018-03-26T15:29:06-07:00 summary: bpo-32873: Treat type variables and special typing forms as immutable by copy and pickle (GH-6216) This also fixes python/typingGH-512 This also fixes python/typingGH-511 As was discussed in both issues, some typing forms deserve to be treated as immutable by copy and pickle modules, so that: * copy(X) is X * deepcopy(X) is X * loads(dumps(X)) is X GH- pickled by reference This PR adds such behaviour to: * Type variables * Special forms like Union, Any, ClassVar * Unsubscripted generic aliases to containers like List, Mapping, Iterable This not only resolves inconsistencies mentioned in the issues, but also improves backwards compatibility with previous versions of Python (including 3.6). Note that this requires some dances with __module__ for type variables (similar to NamedTuple) because the class TypeVar itself is define in typing, while type variables should get module where they were defined. https://bugs.python.org/issue32873 (cherry picked from commit 834940375ae88bc95794226dd8eff1f25fba1cf9) Co-authored-by: Ivan Levkivskyi <levkivskyi at gmail.com> files: A Misc/NEWS.d/next/Library/2018-03-24-19-54-48.bpo-32873.cHyoAm.rst M Lib/test/test_typing.py M Lib/typing.py diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index f56caa13a295..09e39fec45e4 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -1057,20 +1057,20 @@ class C(B[int]): self.assertEqual(x.foo, 42) self.assertEqual(x.bar, 'abc') self.assertEqual(x.__dict__, {'foo': 42, 'bar': 'abc'}) - samples = [Any, Union, Tuple, Callable, ClassVar] + samples = [Any, Union, Tuple, Callable, ClassVar, + Union[int, str], ClassVar[List], Tuple[int, ...], Callable[[str], bytes]] for s in samples: for proto in range(pickle.HIGHEST_PROTOCOL + 1): z = pickle.dumps(s, proto) x = pickle.loads(z) self.assertEqual(s, x) - more_samples = [List, typing.Iterable, typing.Type] + more_samples = [List, typing.Iterable, typing.Type, List[int], + typing.Type[typing.Mapping]] for s in more_samples: for proto in range(pickle.HIGHEST_PROTOCOL + 1): z = pickle.dumps(s, proto) x = pickle.loads(z) - self.assertEqual(repr(s), repr(x)) # TODO: fix this - # see also comment in test_copy_and_deepcopy - # the issue is typing/#512 + self.assertEqual(s, x) def test_copy_and_deepcopy(self): T = TypeVar('T') @@ -1082,7 +1082,27 @@ class Node(Generic[T]): ... Union['T', int], List['T'], typing.Mapping['T', int]] for t in things + [Any]: self.assertEqual(t, copy(t)) - self.assertEqual(repr(t), repr(deepcopy(t))) # Use repr() because of TypeVars + self.assertEqual(t, deepcopy(t)) + + def test_immutability_by_copy_and_pickle(self): + # Special forms like Union, Any, etc., generic aliases to containers like List, + # Mapping, etc., and type variabcles are considered immutable by copy and pickle. + global TP, TPB, TPV # for pickle + TP = TypeVar('TP') + TPB = TypeVar('TPB', bound=int) + TPV = TypeVar('TPV', bytes, str) + for X in [TP, TPB, TPV, List, typing.Mapping, ClassVar, typing.Iterable, + Union, Any, Tuple, Callable]: + self.assertIs(copy(X), X) + self.assertIs(deepcopy(X), X) + self.assertIs(pickle.loads(pickle.dumps(X)), X) + # Check that local type variables are copyable. + TL = TypeVar('TL') + TLB = TypeVar('TLB', bound=int) + TLV = TypeVar('TLV', bytes, str) + for X in [TL, TLB, TLV]: + self.assertIs(copy(X), X) + self.assertIs(deepcopy(X), X) def test_copy_generic_instances(self): T = TypeVar('T') diff --git a/Lib/typing.py b/Lib/typing.py index 56126cfc0247..510574c413e3 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -285,8 +285,17 @@ def __init_subclass__(self, *args, **kwds): if '_root' not in kwds: raise TypeError("Cannot subclass special typing classes") +class _Immutable: + """Mixin to indicate that object should not be copied.""" -class _SpecialForm(_Final, _root=True): + def __copy__(self): + return self + + def __deepcopy__(self, memo): + return self + + +class _SpecialForm(_Final, _Immutable, _root=True): """Internal indicator of special typing constructs. See _doc instance attribute for specific docs. """ @@ -328,8 +337,8 @@ def __hash__(self): def __repr__(self): return 'typing.' + self._name - def __copy__(self): - return self # Special forms are immutable. + def __reduce__(self): + return self._name def __call__(self, *args, **kwds): raise TypeError(f"Cannot instantiate {self!r}") @@ -496,7 +505,11 @@ def __repr__(self): return f'ForwardRef({self.__forward_arg__!r})' -class TypeVar(_Final, _root=True): +def _find_name(mod, name): + return getattr(sys.modules[mod], name) + + +class TypeVar(_Final, _Immutable, _root=True): """Type variable. Usage:: @@ -536,10 +549,12 @@ def longest(x: A, y: A) -> A: T.__covariant__ == False T.__contravariant__ = False A.__constraints__ == (str, bytes) + + Note that only type variables defined in global scope can be pickled. """ __slots__ = ('__name__', '__bound__', '__constraints__', - '__covariant__', '__contravariant__') + '__covariant__', '__contravariant__', '_def_mod') def __init__(self, name, *constraints, bound=None, covariant=False, contravariant=False): @@ -558,6 +573,7 @@ def __init__(self, name, *constraints, bound=None, self.__bound__ = _type_check(bound, "Bound must be a type.") else: self.__bound__ = None + self._def_mod = sys._getframe(1).f_globals['__name__'] # for pickling def __getstate__(self): return {'name': self.__name__, @@ -582,6 +598,9 @@ def __repr__(self): prefix = '~' return prefix + self.__name__ + def __reduce__(self): + return (_find_name, (self._def_mod, self.__name__)) + # Special typing constructs Union, Optional, Generic, Callable and Tuple # use three special attributes for internal bookkeeping of generic types: @@ -724,6 +743,11 @@ def __subclasscheck__(self, cls): raise TypeError("Subscripted generics cannot be used with" " class and instance checks") + def __reduce__(self): + if self._special: + return self._name + return super().__reduce__() + class _VariadicGenericAlias(_GenericAlias, _root=True): """Same as _GenericAlias above but for variadic aliases. Currently, diff --git a/Misc/NEWS.d/next/Library/2018-03-24-19-54-48.bpo-32873.cHyoAm.rst b/Misc/NEWS.d/next/Library/2018-03-24-19-54-48.bpo-32873.cHyoAm.rst new file mode 100644 index 000000000000..99f8088cf138 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-03-24-19-54-48.bpo-32873.cHyoAm.rst @@ -0,0 +1,3 @@ +Treat type variables and special typing forms as immutable by copy and +pickle. This fixes several minor issues and inconsistencies, and improves +backwards compatibility with Python 3.6. From webhook-mailer at python.org Mon Mar 26 21:29:36 2018 From: webhook-mailer at python.org (Raymond Hettinger) Date: Tue, 27 Mar 2018 01:29:36 -0000 Subject: [Python-checkins] bpo-27212: Modify islice recipe to consume initial values preceding start (GH-6195) Message-ID: <mailman.213.1522114178.1871.python-checkins@python.org> https://github.com/python/cpython/commit/da1734c58d2f97387ccc9676074717d38b044128 commit: da1734c58d2f97387ccc9676074717d38b044128 branch: master author: Cheryl Sabella <cheryl.sabella at gmail.com> committer: Raymond Hettinger <rhettinger at users.noreply.github.com> date: 2018-03-26T18:29:33-07:00 summary: bpo-27212: Modify islice recipe to consume initial values preceding start (GH-6195) files: A Misc/NEWS.d/next/Documentation/2018-03-22-19-23-04.bpo-27212.wrE5KR.rst M Doc/library/itertools.rst M Lib/test/test_itertools.py diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst index 0b3829f19faf..a5a5356a9a1f 100644 --- a/Doc/library/itertools.rst +++ b/Doc/library/itertools.rst @@ -436,15 +436,24 @@ loops that truncate the stream. # islice('ABCDEFG', 2, None) --> C D E F G # islice('ABCDEFG', 0, None, 2) --> A C E G s = slice(*args) - it = iter(range(s.start or 0, s.stop or sys.maxsize, s.step or 1)) + start, stop, step = s.start or 0, s.stop or sys.maxsize, s.step or 1 + it = iter(range(start, stop, step)) try: nexti = next(it) except StopIteration: + # Consume *iterable* up to the *start* position. + for i, element in zip(range(start), iterable): + pass return - for i, element in enumerate(iterable): - if i == nexti: - yield element - nexti = next(it) + try: + for i, element in enumerate(iterable): + if i == nexti: + yield element + nexti = next(it) + except StopIteration: + # Consume to *stop*. + for i, element in zip(range(i + 1, stop), iterable): + pass If *start* is ``None``, then iteration starts at zero. If *step* is ``None``, then the step defaults to one. @@ -688,8 +697,8 @@ which incur interpreter overhead. # tail(3, 'ABCDEFG') --> E F G return iter(collections.deque(iterable, maxlen=n)) - def consume(iterator, n): - "Advance the iterator n-steps ahead. If n is none, consume entirely." + def consume(iterator, n=None): + "Advance the iterator n-steps ahead. If n is None, consume entirely." # Use functions that consume iterators at C speed. if n is None: # feed the entire iterator into a zero-length deque diff --git a/Lib/test/test_itertools.py b/Lib/test/test_itertools.py index 4fcc02acbf44..effd7f0e21be 100644 --- a/Lib/test/test_itertools.py +++ b/Lib/test/test_itertools.py @@ -1192,6 +1192,7 @@ def test_islice(self): (10, 20, 3), (10, 3, 20), (10, 20), + (10, 10), (10, 3), (20,) ]: @@ -1218,6 +1219,10 @@ def test_islice(self): self.assertEqual(list(islice(it, 3)), list(range(3))) self.assertEqual(list(it), list(range(3, 10))) + it = iter(range(10)) + self.assertEqual(list(islice(it, 3, 3)), []) + self.assertEqual(list(it), list(range(3, 10))) + # Test invalid arguments ra = range(10) self.assertRaises(TypeError, islice, ra) @@ -1604,6 +1609,48 @@ def test_takewhile(self): self.assertEqual(list(takewhile(lambda x: x<5, [1,4,6,4,1])), [1,4]) +class TestPurePythonRoughEquivalents(unittest.TestCase): + + @staticmethod + def islice(iterable, *args): + s = slice(*args) + start, stop, step = s.start or 0, s.stop or sys.maxsize, s.step or 1 + it = iter(range(start, stop, step)) + try: + nexti = next(it) + except StopIteration: + # Consume *iterable* up to the *start* position. + for i, element in zip(range(start), iterable): + pass + return + try: + for i, element in enumerate(iterable): + if i == nexti: + yield element + nexti = next(it) + except StopIteration: + # Consume to *stop*. + for i, element in zip(range(i + 1, stop), iterable): + pass + + def test_islice_recipe(self): + self.assertEqual(list(self.islice('ABCDEFG', 2)), list('AB')) + self.assertEqual(list(self.islice('ABCDEFG', 2, 4)), list('CD')) + self.assertEqual(list(self.islice('ABCDEFG', 2, None)), list('CDEFG')) + self.assertEqual(list(self.islice('ABCDEFG', 0, None, 2)), list('ACEG')) + # Test items consumed. + it = iter(range(10)) + self.assertEqual(list(self.islice(it, 3)), list(range(3))) + self.assertEqual(list(it), list(range(3, 10))) + it = iter(range(10)) + self.assertEqual(list(self.islice(it, 3, 3)), []) + self.assertEqual(list(it), list(range(3, 10))) + # Test that slice finishes in predictable state. + c = count() + self.assertEqual(list(self.islice(c, 1, 3, 50)), [1]) + self.assertEqual(next(c), 3) + + class TestGC(unittest.TestCase): def makecycle(self, iterator, container): @@ -2158,6 +2205,17 @@ def test_permutations_sizeof(self): ... "Return function(0), function(1), ..." ... return map(function, count(start)) +>>> import collections +>>> def consume(iterator, n=None): +... "Advance the iterator n-steps ahead. If n is None, consume entirely." +... # Use functions that consume iterators at C speed. +... if n is None: +... # feed the entire iterator into a zero-length deque +... collections.deque(iterator, maxlen=0) +... else: +... # advance to the empty slice starting at position n +... next(islice(iterator, n, n), None) + >>> def nth(iterable, n, default=None): ... "Returns the nth item or a default value" ... return next(islice(iterable, n, None), default) @@ -2298,6 +2356,14 @@ def test_permutations_sizeof(self): >>> list(islice(tabulate(lambda x: 2*x), 4)) [0, 2, 4, 6] +>>> it = iter(range(10)) +>>> consume(it, 3) +>>> next(it) +3 +>>> consume(it) +>>> next(it, 'Done') +'Done' + >>> nth('abcde', 3) 'd' @@ -2386,6 +2452,7 @@ def test_main(verbose=None): test_classes = (TestBasicOps, TestVariousIteratorArgs, TestGC, RegressionTests, LengthTransparency, SubclassWithKwargsTest, TestExamples, + TestPurePythonRoughEquivalents, SizeofTest) support.run_unittest(*test_classes) diff --git a/Misc/NEWS.d/next/Documentation/2018-03-22-19-23-04.bpo-27212.wrE5KR.rst b/Misc/NEWS.d/next/Documentation/2018-03-22-19-23-04.bpo-27212.wrE5KR.rst new file mode 100644 index 000000000000..5910d2c17342 --- /dev/null +++ b/Misc/NEWS.d/next/Documentation/2018-03-22-19-23-04.bpo-27212.wrE5KR.rst @@ -0,0 +1,2 @@ +Modify documentation for the :func:`islice` recipe to consume initial values +up to the start index. From webhook-mailer at python.org Mon Mar 26 22:23:37 2018 From: webhook-mailer at python.org (Raymond Hettinger) Date: Tue, 27 Mar 2018 02:23:37 -0000 Subject: [Python-checkins] bpo-27212: Modify islice recipe to consume initial values preceding start (GH-6195) (#GH-6266) Message-ID: <mailman.214.1522117420.1871.python-checkins@python.org> https://github.com/python/cpython/commit/f328caf4caafd4521c356af8cb8a299f27603c90 commit: f328caf4caafd4521c356af8cb8a299f27603c90 branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Raymond Hettinger <rhettinger at users.noreply.github.com> date: 2018-03-26T19:23:34-07:00 summary: bpo-27212: Modify islice recipe to consume initial values preceding start (GH-6195) (#GH-6266) (cherry picked from commit da1734c58d2f97387ccc9676074717d38b044128) Co-authored-by: Cheryl Sabella <cheryl.sabella at gmail.com> files: A Misc/NEWS.d/next/Documentation/2018-03-22-19-23-04.bpo-27212.wrE5KR.rst M Doc/library/itertools.rst M Lib/test/test_itertools.py diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst index 0b3829f19faf..a5a5356a9a1f 100644 --- a/Doc/library/itertools.rst +++ b/Doc/library/itertools.rst @@ -436,15 +436,24 @@ loops that truncate the stream. # islice('ABCDEFG', 2, None) --> C D E F G # islice('ABCDEFG', 0, None, 2) --> A C E G s = slice(*args) - it = iter(range(s.start or 0, s.stop or sys.maxsize, s.step or 1)) + start, stop, step = s.start or 0, s.stop or sys.maxsize, s.step or 1 + it = iter(range(start, stop, step)) try: nexti = next(it) except StopIteration: + # Consume *iterable* up to the *start* position. + for i, element in zip(range(start), iterable): + pass return - for i, element in enumerate(iterable): - if i == nexti: - yield element - nexti = next(it) + try: + for i, element in enumerate(iterable): + if i == nexti: + yield element + nexti = next(it) + except StopIteration: + # Consume to *stop*. + for i, element in zip(range(i + 1, stop), iterable): + pass If *start* is ``None``, then iteration starts at zero. If *step* is ``None``, then the step defaults to one. @@ -688,8 +697,8 @@ which incur interpreter overhead. # tail(3, 'ABCDEFG') --> E F G return iter(collections.deque(iterable, maxlen=n)) - def consume(iterator, n): - "Advance the iterator n-steps ahead. If n is none, consume entirely." + def consume(iterator, n=None): + "Advance the iterator n-steps ahead. If n is None, consume entirely." # Use functions that consume iterators at C speed. if n is None: # feed the entire iterator into a zero-length deque diff --git a/Lib/test/test_itertools.py b/Lib/test/test_itertools.py index 4fcc02acbf44..effd7f0e21be 100644 --- a/Lib/test/test_itertools.py +++ b/Lib/test/test_itertools.py @@ -1192,6 +1192,7 @@ def test_islice(self): (10, 20, 3), (10, 3, 20), (10, 20), + (10, 10), (10, 3), (20,) ]: @@ -1218,6 +1219,10 @@ def test_islice(self): self.assertEqual(list(islice(it, 3)), list(range(3))) self.assertEqual(list(it), list(range(3, 10))) + it = iter(range(10)) + self.assertEqual(list(islice(it, 3, 3)), []) + self.assertEqual(list(it), list(range(3, 10))) + # Test invalid arguments ra = range(10) self.assertRaises(TypeError, islice, ra) @@ -1604,6 +1609,48 @@ def test_takewhile(self): self.assertEqual(list(takewhile(lambda x: x<5, [1,4,6,4,1])), [1,4]) +class TestPurePythonRoughEquivalents(unittest.TestCase): + + @staticmethod + def islice(iterable, *args): + s = slice(*args) + start, stop, step = s.start or 0, s.stop or sys.maxsize, s.step or 1 + it = iter(range(start, stop, step)) + try: + nexti = next(it) + except StopIteration: + # Consume *iterable* up to the *start* position. + for i, element in zip(range(start), iterable): + pass + return + try: + for i, element in enumerate(iterable): + if i == nexti: + yield element + nexti = next(it) + except StopIteration: + # Consume to *stop*. + for i, element in zip(range(i + 1, stop), iterable): + pass + + def test_islice_recipe(self): + self.assertEqual(list(self.islice('ABCDEFG', 2)), list('AB')) + self.assertEqual(list(self.islice('ABCDEFG', 2, 4)), list('CD')) + self.assertEqual(list(self.islice('ABCDEFG', 2, None)), list('CDEFG')) + self.assertEqual(list(self.islice('ABCDEFG', 0, None, 2)), list('ACEG')) + # Test items consumed. + it = iter(range(10)) + self.assertEqual(list(self.islice(it, 3)), list(range(3))) + self.assertEqual(list(it), list(range(3, 10))) + it = iter(range(10)) + self.assertEqual(list(self.islice(it, 3, 3)), []) + self.assertEqual(list(it), list(range(3, 10))) + # Test that slice finishes in predictable state. + c = count() + self.assertEqual(list(self.islice(c, 1, 3, 50)), [1]) + self.assertEqual(next(c), 3) + + class TestGC(unittest.TestCase): def makecycle(self, iterator, container): @@ -2158,6 +2205,17 @@ def test_permutations_sizeof(self): ... "Return function(0), function(1), ..." ... return map(function, count(start)) +>>> import collections +>>> def consume(iterator, n=None): +... "Advance the iterator n-steps ahead. If n is None, consume entirely." +... # Use functions that consume iterators at C speed. +... if n is None: +... # feed the entire iterator into a zero-length deque +... collections.deque(iterator, maxlen=0) +... else: +... # advance to the empty slice starting at position n +... next(islice(iterator, n, n), None) + >>> def nth(iterable, n, default=None): ... "Returns the nth item or a default value" ... return next(islice(iterable, n, None), default) @@ -2298,6 +2356,14 @@ def test_permutations_sizeof(self): >>> list(islice(tabulate(lambda x: 2*x), 4)) [0, 2, 4, 6] +>>> it = iter(range(10)) +>>> consume(it, 3) +>>> next(it) +3 +>>> consume(it) +>>> next(it, 'Done') +'Done' + >>> nth('abcde', 3) 'd' @@ -2386,6 +2452,7 @@ def test_main(verbose=None): test_classes = (TestBasicOps, TestVariousIteratorArgs, TestGC, RegressionTests, LengthTransparency, SubclassWithKwargsTest, TestExamples, + TestPurePythonRoughEquivalents, SizeofTest) support.run_unittest(*test_classes) diff --git a/Misc/NEWS.d/next/Documentation/2018-03-22-19-23-04.bpo-27212.wrE5KR.rst b/Misc/NEWS.d/next/Documentation/2018-03-22-19-23-04.bpo-27212.wrE5KR.rst new file mode 100644 index 000000000000..5910d2c17342 --- /dev/null +++ b/Misc/NEWS.d/next/Documentation/2018-03-22-19-23-04.bpo-27212.wrE5KR.rst @@ -0,0 +1,2 @@ +Modify documentation for the :func:`islice` recipe to consume initial values +up to the start index. From webhook-mailer at python.org Mon Mar 26 22:24:05 2018 From: webhook-mailer at python.org (Raymond Hettinger) Date: Tue, 27 Mar 2018 02:24:05 -0000 Subject: [Python-checkins] bpo-27212: Modify islice recipe to consume initial values preceding start (GH-6195) (GH-6267) Message-ID: <mailman.215.1522117448.1871.python-checkins@python.org> https://github.com/python/cpython/commit/c8698cff7ccc8dc730c58523c7ef4113ea8d3049 commit: c8698cff7ccc8dc730c58523c7ef4113ea8d3049 branch: 3.6 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Raymond Hettinger <rhettinger at users.noreply.github.com> date: 2018-03-26T19:24:02-07:00 summary: bpo-27212: Modify islice recipe to consume initial values preceding start (GH-6195) (GH-6267) (cherry picked from commit da1734c58d2f97387ccc9676074717d38b044128) Co-authored-by: Cheryl Sabella <cheryl.sabella at gmail.com> files: A Misc/NEWS.d/next/Documentation/2018-03-22-19-23-04.bpo-27212.wrE5KR.rst M Doc/library/itertools.rst M Lib/test/test_itertools.py diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst index 700a13a07feb..3782f40911e2 100644 --- a/Doc/library/itertools.rst +++ b/Doc/library/itertools.rst @@ -435,15 +435,24 @@ loops that truncate the stream. # islice('ABCDEFG', 2, None) --> C D E F G # islice('ABCDEFG', 0, None, 2) --> A C E G s = slice(*args) - it = iter(range(s.start or 0, s.stop or sys.maxsize, s.step or 1)) + start, stop, step = s.start or 0, s.stop or sys.maxsize, s.step or 1 + it = iter(range(start, stop, step)) try: nexti = next(it) except StopIteration: + # Consume *iterable* up to the *start* position. + for i, element in zip(range(start), iterable): + pass return - for i, element in enumerate(iterable): - if i == nexti: - yield element - nexti = next(it) + try: + for i, element in enumerate(iterable): + if i == nexti: + yield element + nexti = next(it) + except StopIteration: + # Consume to *stop*. + for i, element in zip(range(i + 1, stop), iterable): + pass If *start* is ``None``, then iteration starts at zero. If *step* is ``None``, then the step defaults to one. @@ -688,8 +697,8 @@ which incur interpreter overhead. # tail(3, 'ABCDEFG') --> E F G return iter(collections.deque(iterable, maxlen=n)) - def consume(iterator, n): - "Advance the iterator n-steps ahead. If n is none, consume entirely." + def consume(iterator, n=None): + "Advance the iterator n-steps ahead. If n is None, consume entirely." # Use functions that consume iterators at C speed. if n is None: # feed the entire iterator into a zero-length deque diff --git a/Lib/test/test_itertools.py b/Lib/test/test_itertools.py index 1ad37ae35be3..84d11ed6221d 100644 --- a/Lib/test/test_itertools.py +++ b/Lib/test/test_itertools.py @@ -1172,6 +1172,7 @@ def test_islice(self): (10, 20, 3), (10, 3, 20), (10, 20), + (10, 10), (10, 3), (20,) ]: @@ -1198,6 +1199,10 @@ def test_islice(self): self.assertEqual(list(islice(it, 3)), list(range(3))) self.assertEqual(list(it), list(range(3, 10))) + it = iter(range(10)) + self.assertEqual(list(islice(it, 3, 3)), []) + self.assertEqual(list(it), list(range(3, 10))) + # Test invalid arguments ra = range(10) self.assertRaises(TypeError, islice, ra) @@ -1571,6 +1576,48 @@ def test_takewhile(self): self.assertEqual(list(takewhile(lambda x: x<5, [1,4,6,4,1])), [1,4]) +class TestPurePythonRoughEquivalents(unittest.TestCase): + + @staticmethod + def islice(iterable, *args): + s = slice(*args) + start, stop, step = s.start or 0, s.stop or sys.maxsize, s.step or 1 + it = iter(range(start, stop, step)) + try: + nexti = next(it) + except StopIteration: + # Consume *iterable* up to the *start* position. + for i, element in zip(range(start), iterable): + pass + return + try: + for i, element in enumerate(iterable): + if i == nexti: + yield element + nexti = next(it) + except StopIteration: + # Consume to *stop*. + for i, element in zip(range(i + 1, stop), iterable): + pass + + def test_islice_recipe(self): + self.assertEqual(list(self.islice('ABCDEFG', 2)), list('AB')) + self.assertEqual(list(self.islice('ABCDEFG', 2, 4)), list('CD')) + self.assertEqual(list(self.islice('ABCDEFG', 2, None)), list('CDEFG')) + self.assertEqual(list(self.islice('ABCDEFG', 0, None, 2)), list('ACEG')) + # Test items consumed. + it = iter(range(10)) + self.assertEqual(list(self.islice(it, 3)), list(range(3))) + self.assertEqual(list(it), list(range(3, 10))) + it = iter(range(10)) + self.assertEqual(list(self.islice(it, 3, 3)), []) + self.assertEqual(list(it), list(range(3, 10))) + # Test that slice finishes in predictable state. + c = count() + self.assertEqual(list(self.islice(c, 1, 3, 50)), [1]) + self.assertEqual(next(c), 3) + + class TestGC(unittest.TestCase): def makecycle(self, iterator, container): @@ -2125,6 +2172,17 @@ def test_permutations_sizeof(self): ... "Return function(0), function(1), ..." ... return map(function, count(start)) +>>> import collections +>>> def consume(iterator, n=None): +... "Advance the iterator n-steps ahead. If n is None, consume entirely." +... # Use functions that consume iterators at C speed. +... if n is None: +... # feed the entire iterator into a zero-length deque +... collections.deque(iterator, maxlen=0) +... else: +... # advance to the empty slice starting at position n +... next(islice(iterator, n, n), None) + >>> def nth(iterable, n, default=None): ... "Returns the nth item or a default value" ... return next(islice(iterable, n, None), default) @@ -2265,6 +2323,14 @@ def test_permutations_sizeof(self): >>> list(islice(tabulate(lambda x: 2*x), 4)) [0, 2, 4, 6] +>>> it = iter(range(10)) +>>> consume(it, 3) +>>> next(it) +3 +>>> consume(it) +>>> next(it, 'Done') +'Done' + >>> nth('abcde', 3) 'd' @@ -2353,6 +2419,7 @@ def test_main(verbose=None): test_classes = (TestBasicOps, TestVariousIteratorArgs, TestGC, RegressionTests, LengthTransparency, SubclassWithKwargsTest, TestExamples, + TestPurePythonRoughEquivalents, SizeofTest) support.run_unittest(*test_classes) diff --git a/Misc/NEWS.d/next/Documentation/2018-03-22-19-23-04.bpo-27212.wrE5KR.rst b/Misc/NEWS.d/next/Documentation/2018-03-22-19-23-04.bpo-27212.wrE5KR.rst new file mode 100644 index 000000000000..5910d2c17342 --- /dev/null +++ b/Misc/NEWS.d/next/Documentation/2018-03-22-19-23-04.bpo-27212.wrE5KR.rst @@ -0,0 +1,2 @@ +Modify documentation for the :func:`islice` recipe to consume initial values +up to the start index. From solipsis at pitrou.net Tue Mar 27 05:16:43 2018 From: solipsis at pitrou.net (solipsis at pitrou.net) Date: Tue, 27 Mar 2018 09:16:43 +0000 Subject: [Python-checkins] Daily reference leaks (4243df51fe43): sum=5 Message-ID: <20180327091643.1.EA2358BFBD664BE2@psf.io> results for 4243df51fe43 on branch "default" -------------------------------------------- test_asyncio leaked [0, 0, 3] memory blocks, sum=3 test_functools leaked [0, 3, 1] memory blocks, sum=4 test_multiprocessing_forkserver leaked [0, -2, 1] memory blocks, sum=-1 test_multiprocessing_spawn leaked [-2, 2, -1] memory blocks, sum=-1 Command line was: ['./python', '-m', 'test.regrtest', '-uall', '-R', '3:3:/home/psf-users/antoine/refleaks/reflogsjrzt4', '--timeout', '7200'] From webhook-mailer at python.org Tue Mar 27 12:59:44 2018 From: webhook-mailer at python.org (Barry Warsaw) Date: Tue, 27 Mar 2018 16:59:44 -0000 Subject: [Python-checkins] bpo-33151: Handle submodule resources (GH-6268) Message-ID: <mailman.216.1522169986.1871.python-checkins@python.org> https://github.com/python/cpython/commit/30e507dff465a31901d87df791a2bac40dc88530 commit: 30e507dff465a31901d87df791a2bac40dc88530 branch: master author: Barry Warsaw <barry at python.org> committer: GitHub <noreply at github.com> date: 2018-03-27T09:59:38-07:00 summary: bpo-33151: Handle submodule resources (GH-6268) files: M Lib/importlib/resources.py M Lib/test/test_importlib/test_read.py M Lib/test/test_importlib/test_resource.py diff --git a/Lib/importlib/resources.py b/Lib/importlib/resources.py index c4f6bbde45fa..e5654273c06a 100644 --- a/Lib/importlib/resources.py +++ b/Lib/importlib/resources.py @@ -267,11 +267,12 @@ def __init__(self, zipimporter, fullname): self.fullname = fullname def open_resource(self, resource): - path = f'{self.fullname}/{resource}' + fullname_as_path = self.fullname.replace('.', '/') + path = f'{fullname_as_path}/{resource}' try: return BytesIO(self.zipimporter.get_data(path)) except OSError: - raise FileNotFoundError + raise FileNotFoundError(path) def resource_path(self, resource): # All resources are in the zip file, so there is no path to the file. @@ -282,7 +283,8 @@ def resource_path(self, resource): def is_resource(self, name): # Maybe we could do better, but if we can get the data, it's a # resource. Otherwise it isn't. - path = f'{self.fullname}/{name}' + fullname_as_path = self.fullname.replace('.', '/') + path = f'{fullname_as_path}/{name}' try: self.zipimporter.get_data(path) except OSError: diff --git a/Lib/test/test_importlib/test_read.py b/Lib/test/test_importlib/test_read.py index 231f5017b688..ff78d0b2948f 100644 --- a/Lib/test/test_importlib/test_read.py +++ b/Lib/test/test_importlib/test_read.py @@ -1,6 +1,6 @@ import unittest -from importlib import resources +from importlib import import_module, resources from . import data01 from . import util @@ -46,7 +46,16 @@ class ReadDiskTests(ReadTests, unittest.TestCase): class ReadZipTests(ReadTests, util.ZipSetup, unittest.TestCase): - pass + def test_read_submodule_resource(self): + submodule = import_module('ziptestdata.subdirectory') + result = resources.read_binary( + submodule, 'binary.file') + self.assertEqual(result, b'\0\1\2\3') + + def test_read_submodule_resource_by_name(self): + result = resources.read_binary( + 'ziptestdata.subdirectory', 'binary.file') + self.assertEqual(result, b'\0\1\2\3') if __name__ == '__main__': diff --git a/Lib/test/test_importlib/test_resource.py b/Lib/test/test_importlib/test_resource.py index c38ad0358a07..d717e1dd04da 100644 --- a/Lib/test/test_importlib/test_resource.py +++ b/Lib/test/test_importlib/test_resource.py @@ -4,7 +4,7 @@ from . import data01 from . import zipdata02 from . import util -from importlib import resources +from importlib import resources, import_module class ResourceTests: @@ -109,6 +109,26 @@ def test_unrelated_contents(self): set(resources.contents('ziptestdata.two')), {'__init__.py', 'resource2.txt'}) + def test_is_submodule_resource(self): + submodule = import_module('ziptestdata.subdirectory') + self.assertTrue( + resources.is_resource(submodule, 'binary.file')) + + def test_read_submodule_resource_by_name(self): + self.assertTrue( + resources.is_resource('ziptestdata.subdirectory', 'binary.file')) + + def test_submodule_contents(self): + submodule = import_module('ziptestdata.subdirectory') + self.assertEqual( + set(resources.contents(submodule)), + {'__init__.py', 'binary.file'}) + + def test_submodule_contents_by_name(self): + self.assertEqual( + set(resources.contents('ziptestdata.subdirectory')), + {'__init__.py', 'binary.file'}) + class NamespaceTest(unittest.TestCase): def test_namespaces_cant_have_resources(self): From webhook-mailer at python.org Tue Mar 27 13:25:34 2018 From: webhook-mailer at python.org (Barry Warsaw) Date: Tue, 27 Mar 2018 17:25:34 -0000 Subject: [Python-checkins] bpo-33151: Handle submodule resources (GH-6268) (GH-6270) Message-ID: <mailman.217.1522171536.1871.python-checkins@python.org> https://github.com/python/cpython/commit/fd1b8f87b3e63495cadd37c8f1b90191b46ee52a commit: fd1b8f87b3e63495cadd37c8f1b90191b46ee52a branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Barry Warsaw <barry at python.org> date: 2018-03-27T10:25:28-07:00 summary: bpo-33151: Handle submodule resources (GH-6268) (GH-6270) (cherry picked from commit 30e507dff465a31901d87df791a2bac40dc88530) Co-authored-by: Barry Warsaw <barry at python.org> files: M Lib/importlib/resources.py M Lib/test/test_importlib/test_read.py M Lib/test/test_importlib/test_resource.py diff --git a/Lib/importlib/resources.py b/Lib/importlib/resources.py index c4f6bbde45fa..e5654273c06a 100644 --- a/Lib/importlib/resources.py +++ b/Lib/importlib/resources.py @@ -267,11 +267,12 @@ def __init__(self, zipimporter, fullname): self.fullname = fullname def open_resource(self, resource): - path = f'{self.fullname}/{resource}' + fullname_as_path = self.fullname.replace('.', '/') + path = f'{fullname_as_path}/{resource}' try: return BytesIO(self.zipimporter.get_data(path)) except OSError: - raise FileNotFoundError + raise FileNotFoundError(path) def resource_path(self, resource): # All resources are in the zip file, so there is no path to the file. @@ -282,7 +283,8 @@ def resource_path(self, resource): def is_resource(self, name): # Maybe we could do better, but if we can get the data, it's a # resource. Otherwise it isn't. - path = f'{self.fullname}/{name}' + fullname_as_path = self.fullname.replace('.', '/') + path = f'{fullname_as_path}/{name}' try: self.zipimporter.get_data(path) except OSError: diff --git a/Lib/test/test_importlib/test_read.py b/Lib/test/test_importlib/test_read.py index 231f5017b688..ff78d0b2948f 100644 --- a/Lib/test/test_importlib/test_read.py +++ b/Lib/test/test_importlib/test_read.py @@ -1,6 +1,6 @@ import unittest -from importlib import resources +from importlib import import_module, resources from . import data01 from . import util @@ -46,7 +46,16 @@ class ReadDiskTests(ReadTests, unittest.TestCase): class ReadZipTests(ReadTests, util.ZipSetup, unittest.TestCase): - pass + def test_read_submodule_resource(self): + submodule = import_module('ziptestdata.subdirectory') + result = resources.read_binary( + submodule, 'binary.file') + self.assertEqual(result, b'\0\1\2\3') + + def test_read_submodule_resource_by_name(self): + result = resources.read_binary( + 'ziptestdata.subdirectory', 'binary.file') + self.assertEqual(result, b'\0\1\2\3') if __name__ == '__main__': diff --git a/Lib/test/test_importlib/test_resource.py b/Lib/test/test_importlib/test_resource.py index c38ad0358a07..d717e1dd04da 100644 --- a/Lib/test/test_importlib/test_resource.py +++ b/Lib/test/test_importlib/test_resource.py @@ -4,7 +4,7 @@ from . import data01 from . import zipdata02 from . import util -from importlib import resources +from importlib import resources, import_module class ResourceTests: @@ -109,6 +109,26 @@ def test_unrelated_contents(self): set(resources.contents('ziptestdata.two')), {'__init__.py', 'resource2.txt'}) + def test_is_submodule_resource(self): + submodule = import_module('ziptestdata.subdirectory') + self.assertTrue( + resources.is_resource(submodule, 'binary.file')) + + def test_read_submodule_resource_by_name(self): + self.assertTrue( + resources.is_resource('ziptestdata.subdirectory', 'binary.file')) + + def test_submodule_contents(self): + submodule = import_module('ziptestdata.subdirectory') + self.assertEqual( + set(resources.contents(submodule)), + {'__init__.py', 'binary.file'}) + + def test_submodule_contents_by_name(self): + self.assertEqual( + set(resources.contents('ziptestdata.subdirectory')), + {'__init__.py', 'binary.file'}) + class NamespaceTest(unittest.TestCase): def test_namespaces_cant_have_resources(self): From webhook-mailer at python.org Tue Mar 27 17:16:58 2018 From: webhook-mailer at python.org (Ned Deily) Date: Tue, 27 Mar 2018 21:16:58 -0000 Subject: [Python-checkins] bpo-32517: fix test_read_pty_output() hangs on macOS 10.13.2+ (GH-6037) (GH-6113) Message-ID: <mailman.218.1522185421.1871.python-checkins@python.org> https://github.com/python/cpython/commit/8534d53333e4e918be82b041754ecd89af519e5b commit: 8534d53333e4e918be82b041754ecd89af519e5b branch: master author: Ned Deily <nad at python.org> committer: GitHub <noreply at github.com> date: 2018-03-27T17:16:49-04:00 summary: bpo-32517: fix test_read_pty_output() hangs on macOS 10.13.2+ (GH-6037) (GH-6113) test_asyncio hangs indefinitely on macOS 10.13.2+ on `read_pty_output()` using the KqueueSelector. Closing `proto.transport` (as is done in `write_pty_output()`) seems to fix it. (cherry picked from commit 12f74d8608c15cacd9d5786524e2be9ca36f007e) Co-authored-by: Nathan Henrie <n8henrie at users.noreply.github.com> Also, re-enable test_read_pty_output on macOS. files: A Misc/NEWS.d/next/Tests/2018-03-09-07-05-12.bpo-32517.ugc1iW.rst M Lib/test/test_asyncio/test_events.py M Misc/ACKS diff --git a/Lib/test/test_asyncio/test_events.py b/Lib/test/test_asyncio/test_events.py index 6accbdae8b3e..66c77b976dce 100644 --- a/Lib/test/test_asyncio/test_events.py +++ b/Lib/test/test_asyncio/test_events.py @@ -1475,7 +1475,6 @@ def test_unclosed_pipe_transport(self): @unittest.skipUnless(sys.platform != 'win32', "Don't support pipes for Windows") - @unittest.skipIf(sys.platform == 'darwin', 'test hangs on MacOS') def test_read_pty_output(self): proto = MyReadPipeProto(loop=self.loop) @@ -1502,6 +1501,7 @@ def test_read_pty_output(self): self.assertEqual(5, proto.nbytes) os.close(slave) + proto.transport.close() self.loop.run_until_complete(proto.done) self.assertEqual( ['INITIAL', 'CONNECTED', 'EOF', 'CLOSED'], proto.state) diff --git a/Misc/ACKS b/Misc/ACKS index 05932a8d8991..c7bdbd4a2312 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -631,6 +631,7 @@ Thomas Heller Malte Helmert Lance Finn Helsten Jonathan Hendry +Nathan Henrie Michael Henry James Henstridge Kasun Herath diff --git a/Misc/NEWS.d/next/Tests/2018-03-09-07-05-12.bpo-32517.ugc1iW.rst b/Misc/NEWS.d/next/Tests/2018-03-09-07-05-12.bpo-32517.ugc1iW.rst new file mode 100644 index 000000000000..43f148f06ecb --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2018-03-09-07-05-12.bpo-32517.ugc1iW.rst @@ -0,0 +1,2 @@ +Fix failing ``test_asyncio`` on macOS 10.12.2+ due to transport of +``KqueueSelector`` loop was not being closed. From webhook-mailer at python.org Tue Mar 27 17:17:32 2018 From: webhook-mailer at python.org (Ned Deily) Date: Tue, 27 Mar 2018 21:17:32 -0000 Subject: [Python-checkins] bpo-32517: re-enable test_read_pty_output on macOS (GH-6112) Message-ID: <mailman.219.1522185455.1871.python-checkins@python.org> https://github.com/python/cpython/commit/6a2539c43412567a4c693da8e7fdf5e73191fd16 commit: 6a2539c43412567a4c693da8e7fdf5e73191fd16 branch: 3.7 author: Ned Deily <nad at python.org> committer: GitHub <noreply at github.com> date: 2018-03-27T17:17:28-04:00 summary: bpo-32517: re-enable test_read_pty_output on macOS (GH-6112) files: M Lib/test/test_asyncio/test_events.py diff --git a/Lib/test/test_asyncio/test_events.py b/Lib/test/test_asyncio/test_events.py index f8bb01bd4e72..66c77b976dce 100644 --- a/Lib/test/test_asyncio/test_events.py +++ b/Lib/test/test_asyncio/test_events.py @@ -1475,7 +1475,6 @@ def test_unclosed_pipe_transport(self): @unittest.skipUnless(sys.platform != 'win32', "Don't support pipes for Windows") - @unittest.skipIf(sys.platform == 'darwin', 'test hangs on MacOS') def test_read_pty_output(self): proto = MyReadPipeProto(loop=self.loop) From webhook-mailer at python.org Tue Mar 27 20:47:44 2018 From: webhook-mailer at python.org (=?utf-8?q?=C5=81ukasz?= Langa) Date: Wed, 28 Mar 2018 00:47:44 -0000 Subject: [Python-checkins] Fix senfile typo (#6265) Message-ID: <mailman.220.1522198066.1871.python-checkins@python.org> https://github.com/python/cpython/commit/65a34709f60711f7c46031d4c6c420f567bc790a commit: 65a34709f60711f7c46031d4c6c420f567bc790a branch: master author: Sam Dunster <me at sdunster.com> committer: ?ukasz Langa <lukasz at langa.pl> date: 2018-03-27T17:47:38-07:00 summary: Fix senfile typo (#6265) * Also in docs files: M Doc/library/asyncio-eventloop.rst M Lib/asyncio/events.py diff --git a/Doc/library/asyncio-eventloop.rst b/Doc/library/asyncio-eventloop.rst index 3ee9939192c1..ca8055bd162f 100644 --- a/Doc/library/asyncio-eventloop.rst +++ b/Doc/library/asyncio-eventloop.rst @@ -1114,7 +1114,7 @@ SendfileNotAvailableError Sendfile syscall is not available, subclass of :exc:`RuntimeError`. - Raised if the OS does not support senfile syscall for + Raised if the OS does not support sendfile syscall for given socket or file type. diff --git a/Lib/asyncio/events.py b/Lib/asyncio/events.py index fcca5d4cb347..40946bbf6529 100644 --- a/Lib/asyncio/events.py +++ b/Lib/asyncio/events.py @@ -24,7 +24,7 @@ class SendfileNotAvailableError(RuntimeError): """Sendfile syscall is not available. - Raised if OS does not support senfile syscall for given socket or + Raised if OS does not support sendfile syscall for given socket or file type. """ From webhook-mailer at python.org Tue Mar 27 21:34:18 2018 From: webhook-mailer at python.org (=?utf-8?q?=C5=81ukasz?= Langa) Date: Wed, 28 Mar 2018 01:34:18 -0000 Subject: [Python-checkins] Fix senfile typo (GH-6265) (#6274) Message-ID: <mailman.221.1522200859.1871.python-checkins@python.org> https://github.com/python/cpython/commit/211c0dbc6a8b390f914746f05bf13d024933378b commit: 211c0dbc6a8b390f914746f05bf13d024933378b branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: ?ukasz Langa <lukasz at langa.pl> date: 2018-03-27T18:34:15-07:00 summary: Fix senfile typo (GH-6265) (#6274) * Also in docs (cherry picked from commit 65a34709f60711f7c46031d4c6c420f567bc790a) Co-authored-by: Sam Dunster <me at sdunster.com> files: M Doc/library/asyncio-eventloop.rst M Lib/asyncio/events.py diff --git a/Doc/library/asyncio-eventloop.rst b/Doc/library/asyncio-eventloop.rst index 3ee9939192c1..ca8055bd162f 100644 --- a/Doc/library/asyncio-eventloop.rst +++ b/Doc/library/asyncio-eventloop.rst @@ -1114,7 +1114,7 @@ SendfileNotAvailableError Sendfile syscall is not available, subclass of :exc:`RuntimeError`. - Raised if the OS does not support senfile syscall for + Raised if the OS does not support sendfile syscall for given socket or file type. diff --git a/Lib/asyncio/events.py b/Lib/asyncio/events.py index fcca5d4cb347..40946bbf6529 100644 --- a/Lib/asyncio/events.py +++ b/Lib/asyncio/events.py @@ -24,7 +24,7 @@ class SendfileNotAvailableError(RuntimeError): """Sendfile syscall is not available. - Raised if OS does not support senfile syscall for given socket or + Raised if OS does not support sendfile syscall for given socket or file type. """ From webhook-mailer at python.org Wed Mar 28 01:57:16 2018 From: webhook-mailer at python.org (Ned Deily) Date: Wed, 28 Mar 2018 05:57:16 -0000 Subject: [Python-checkins] bpo-32872: Avoid regrtest compatibility issue with namespace packages. (GH-6276) Message-ID: <mailman.222.1522216638.1871.python-checkins@python.org> https://github.com/python/cpython/commit/e52ac045972a4f75d7f52e4ee0d6de128259134d commit: e52ac045972a4f75d7f52e4ee0d6de128259134d branch: master author: Ned Deily <nad at python.org> committer: GitHub <noreply at github.com> date: 2018-03-28T01:57:13-04:00 summary: bpo-32872: Avoid regrtest compatibility issue with namespace packages. (GH-6276) files: A Misc/NEWS.d/next/Tests/2018-03-28-01-35-02.bpo-32872.J5NDUj.rst M Lib/test/libregrtest/setup.py diff --git a/Lib/test/libregrtest/setup.py b/Lib/test/libregrtest/setup.py index bf899a9e4d4a..910aca1b1a6c 100644 --- a/Lib/test/libregrtest/setup.py +++ b/Lib/test/libregrtest/setup.py @@ -57,7 +57,7 @@ def setup_tests(ns): if hasattr(module, '__path__'): for index, path in enumerate(module.__path__): module.__path__[index] = os.path.abspath(path) - if hasattr(module, '__file__'): + if getattr(module, '__file__', None): module.__file__ = os.path.abspath(module.__file__) # MacOSX (a.k.a. Darwin) has a default stack size that is too small diff --git a/Misc/NEWS.d/next/Tests/2018-03-28-01-35-02.bpo-32872.J5NDUj.rst b/Misc/NEWS.d/next/Tests/2018-03-28-01-35-02.bpo-32872.J5NDUj.rst new file mode 100644 index 000000000000..06d656bbfd64 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2018-03-28-01-35-02.bpo-32872.J5NDUj.rst @@ -0,0 +1 @@ +Avoid regrtest compatibility issue with namespace packages. From webhook-mailer at python.org Wed Mar 28 02:39:22 2018 From: webhook-mailer at python.org (Ned Deily) Date: Wed, 28 Mar 2018 06:39:22 -0000 Subject: [Python-checkins] bpo-32872: Avoid regrtest compatibility issue with namespace packages. (GH-6276) (#6277) Message-ID: <mailman.223.1522219164.1871.python-checkins@python.org> https://github.com/python/cpython/commit/d2d5bd8bc4f58b572702d572dc8491f0a50144e6 commit: d2d5bd8bc4f58b572702d572dc8491f0a50144e6 branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Ned Deily <nad at python.org> date: 2018-03-28T02:39:19-04:00 summary: bpo-32872: Avoid regrtest compatibility issue with namespace packages. (GH-6276) (#6277) (cherry picked from commit e52ac045972a4f75d7f52e4ee0d6de128259134d) Co-authored-by: Ned Deily <nad at python.org> files: A Misc/NEWS.d/next/Tests/2018-03-28-01-35-02.bpo-32872.J5NDUj.rst M Lib/test/libregrtest/setup.py diff --git a/Lib/test/libregrtest/setup.py b/Lib/test/libregrtest/setup.py index bf899a9e4d4a..910aca1b1a6c 100644 --- a/Lib/test/libregrtest/setup.py +++ b/Lib/test/libregrtest/setup.py @@ -57,7 +57,7 @@ def setup_tests(ns): if hasattr(module, '__path__'): for index, path in enumerate(module.__path__): module.__path__[index] = os.path.abspath(path) - if hasattr(module, '__file__'): + if getattr(module, '__file__', None): module.__file__ = os.path.abspath(module.__file__) # MacOSX (a.k.a. Darwin) has a default stack size that is too small diff --git a/Misc/NEWS.d/next/Tests/2018-03-28-01-35-02.bpo-32872.J5NDUj.rst b/Misc/NEWS.d/next/Tests/2018-03-28-01-35-02.bpo-32872.J5NDUj.rst new file mode 100644 index 000000000000..06d656bbfd64 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2018-03-28-01-35-02.bpo-32872.J5NDUj.rst @@ -0,0 +1 @@ +Avoid regrtest compatibility issue with namespace packages. From webhook-mailer at python.org Wed Mar 28 02:43:54 2018 From: webhook-mailer at python.org (Ned Deily) Date: Wed, 28 Mar 2018 06:43:54 -0000 Subject: [Python-checkins] bpo-32872: Avoid regrtest compatibility issue with namespace packages. (GH-6276) (GH-6278) Message-ID: <mailman.224.1522219437.1871.python-checkins@python.org> https://github.com/python/cpython/commit/a93662cf8fb98e41f2b7e7c032b680eee834d290 commit: a93662cf8fb98e41f2b7e7c032b680eee834d290 branch: 3.6 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Ned Deily <nad at python.org> date: 2018-03-28T02:43:51-04:00 summary: bpo-32872: Avoid regrtest compatibility issue with namespace packages. (GH-6276) (GH-6278) (cherry picked from commit e52ac045972a4f75d7f52e4ee0d6de128259134d) Co-authored-by: Ned Deily <nad at python.org> files: A Misc/NEWS.d/next/Tests/2018-03-28-01-35-02.bpo-32872.J5NDUj.rst M Lib/test/libregrtest/setup.py diff --git a/Lib/test/libregrtest/setup.py b/Lib/test/libregrtest/setup.py index bf899a9e4d4a..910aca1b1a6c 100644 --- a/Lib/test/libregrtest/setup.py +++ b/Lib/test/libregrtest/setup.py @@ -57,7 +57,7 @@ def setup_tests(ns): if hasattr(module, '__path__'): for index, path in enumerate(module.__path__): module.__path__[index] = os.path.abspath(path) - if hasattr(module, '__file__'): + if getattr(module, '__file__', None): module.__file__ = os.path.abspath(module.__file__) # MacOSX (a.k.a. Darwin) has a default stack size that is too small diff --git a/Misc/NEWS.d/next/Tests/2018-03-28-01-35-02.bpo-32872.J5NDUj.rst b/Misc/NEWS.d/next/Tests/2018-03-28-01-35-02.bpo-32872.J5NDUj.rst new file mode 100644 index 000000000000..06d656bbfd64 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2018-03-28-01-35-02.bpo-32872.J5NDUj.rst @@ -0,0 +1 @@ +Avoid regrtest compatibility issue with namespace packages. From webhook-mailer at python.org Wed Mar 28 03:44:51 2018 From: webhook-mailer at python.org (Ned Deily) Date: Wed, 28 Mar 2018 07:44:51 -0000 Subject: [Python-checkins] bpo-32726: Do not force IDLE.app to launch in 32-bit mode. (GH-6279) Message-ID: <mailman.225.1522223093.1871.python-checkins@python.org> https://github.com/python/cpython/commit/df532ab752680f6e359672c2cd40bec8ac848628 commit: df532ab752680f6e359672c2cd40bec8ac848628 branch: master author: Ned Deily <nad at python.org> committer: GitHub <noreply at github.com> date: 2018-03-28T03:44:48-04:00 summary: bpo-32726: Do not force IDLE.app to launch in 32-bit mode. (GH-6279) Forcing the macOS IDLE.app gui process to launch in 32-mode was a necessary hack for old versions of Tk (Aqua Carbon as in Tk 8.4 and early versions of 8.5); it is not needed for current versions of Tk. Since 32-bit launching will no longer be supported on future releases of macOS, allow IDLE.app to launch in 64-bit mode. files: M Mac/Makefile.in diff --git a/Mac/Makefile.in b/Mac/Makefile.in index 1255b13b4249..95fd4a2722d5 100644 --- a/Mac/Makefile.in +++ b/Mac/Makefile.in @@ -221,10 +221,6 @@ install_IDLE: -test -d "$(DESTDIR)$(PYTHONAPPSDIR)/IDLE.app" && rm -rf "$(DESTDIR)$(PYTHONAPPSDIR)/IDLE.app" /bin/cp -PR "$(srcdir)/IDLE/IDLE.app" "$(DESTDIR)$(PYTHONAPPSDIR)" ln -sf "$(INSTALLED_PYTHONAPP)" "$(DESTDIR)$(PYTHONAPPSDIR)/IDLE.app/Contents/MacOS/Python" -ifneq ($(LIPO_32BIT_FLAGS),) - rm "$(DESTDIR)$(PYTHONAPPSDIR)/IDLE.app/Contents/MacOS/Python" - lipo $(LIPO_32BIT_FLAGS) -output "$(DESTDIR)$(PYTHONAPPSDIR)/IDLE.app/Contents/MacOS/Python" "$(BUILDPYTHON)" -endif sed -e "s!%prefix%!$(prefix)!g" -e 's!%exe%!$(PYTHONFRAMEWORK)!g' < "$(srcdir)/IDLE/IDLE.app/Contents/MacOS/IDLE" > "$(DESTDIR)$(PYTHONAPPSDIR)/IDLE.app/Contents/MacOS/IDLE" sed "s!%version%!`$(RUNSHARED) $(BUILDPYTHON) -c 'import platform; print(platform.python_version())'`!g" < "$(srcdir)/IDLE/IDLE.app/Contents/Info.plist" > "$(DESTDIR)$(PYTHONAPPSDIR)/IDLE.app/Contents/Info.plist" if [ -f "$(DESTDIR)$(LIBDEST)/idlelib/config-main.def" ]; then \ From webhook-mailer at python.org Wed Mar 28 04:16:33 2018 From: webhook-mailer at python.org (Ned Deily) Date: Wed, 28 Mar 2018 08:16:33 -0000 Subject: [Python-checkins] bpo-32726: Do not force IDLE.app to launch in 32-bit mode. (GH-6280) Message-ID: <mailman.226.1522224995.1871.python-checkins@python.org> https://github.com/python/cpython/commit/39c0ef5171e1cdcc2ed59685a81b194e9bfe3809 commit: 39c0ef5171e1cdcc2ed59685a81b194e9bfe3809 branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Ned Deily <nad at python.org> date: 2018-03-28T04:16:23-04:00 summary: bpo-32726: Do not force IDLE.app to launch in 32-bit mode. (GH-6280) Forcing the macOS IDLE.app gui process to launch in 32-mode was a necessary hack for old versions of Tk (Aqua Carbon as in Tk 8.4 and early versions of 8.5); it is not needed for current versions of Tk. Since 32-bit launching will no longer be supported on future releases of macOS, allow IDLE.app to launch in 64-bit mode. (cherry picked from commit df532ab752680f6e359672c2cd40bec8ac848628) Co-authored-by: Ned Deily <nad at python.org> files: M Mac/Makefile.in diff --git a/Mac/Makefile.in b/Mac/Makefile.in index 1255b13b4249..95fd4a2722d5 100644 --- a/Mac/Makefile.in +++ b/Mac/Makefile.in @@ -221,10 +221,6 @@ install_IDLE: -test -d "$(DESTDIR)$(PYTHONAPPSDIR)/IDLE.app" && rm -rf "$(DESTDIR)$(PYTHONAPPSDIR)/IDLE.app" /bin/cp -PR "$(srcdir)/IDLE/IDLE.app" "$(DESTDIR)$(PYTHONAPPSDIR)" ln -sf "$(INSTALLED_PYTHONAPP)" "$(DESTDIR)$(PYTHONAPPSDIR)/IDLE.app/Contents/MacOS/Python" -ifneq ($(LIPO_32BIT_FLAGS),) - rm "$(DESTDIR)$(PYTHONAPPSDIR)/IDLE.app/Contents/MacOS/Python" - lipo $(LIPO_32BIT_FLAGS) -output "$(DESTDIR)$(PYTHONAPPSDIR)/IDLE.app/Contents/MacOS/Python" "$(BUILDPYTHON)" -endif sed -e "s!%prefix%!$(prefix)!g" -e 's!%exe%!$(PYTHONFRAMEWORK)!g' < "$(srcdir)/IDLE/IDLE.app/Contents/MacOS/IDLE" > "$(DESTDIR)$(PYTHONAPPSDIR)/IDLE.app/Contents/MacOS/IDLE" sed "s!%version%!`$(RUNSHARED) $(BUILDPYTHON) -c 'import platform; print(platform.python_version())'`!g" < "$(srcdir)/IDLE/IDLE.app/Contents/Info.plist" > "$(DESTDIR)$(PYTHONAPPSDIR)/IDLE.app/Contents/Info.plist" if [ -f "$(DESTDIR)$(LIBDEST)/idlelib/config-main.def" ]; then \ From webhook-mailer at python.org Wed Mar 28 04:40:36 2018 From: webhook-mailer at python.org (Ned Deily) Date: Wed, 28 Mar 2018 08:40:36 -0000 Subject: [Python-checkins] bpo-32726: Do not force IDLE.app to launch in 32-bit mode. (GH-6279) (#6281) Message-ID: <mailman.227.1522226438.1871.python-checkins@python.org> https://github.com/python/cpython/commit/fb3d3b7a65d8c0521a88c87e17a7554c5ec439d9 commit: fb3d3b7a65d8c0521a88c87e17a7554c5ec439d9 branch: 3.6 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Ned Deily <nad at python.org> date: 2018-03-28T04:40:29-04:00 summary: bpo-32726: Do not force IDLE.app to launch in 32-bit mode. (GH-6279) (#6281) Forcing the macOS IDLE.app gui process to launch in 32-mode was a necessary hack for old versions of Tk (Aqua Carbon as in Tk 8.4 and early versions of 8.5); it is not needed for current versions of Tk. Since 32-bit launching will no longer be supported on future releases of macOS, allow IDLE.app to launch in 64-bit mode. (cherry picked from commit df532ab752680f6e359672c2cd40bec8ac848628) Co-authored-by: Ned Deily <nad at python.org> files: M Mac/Makefile.in diff --git a/Mac/Makefile.in b/Mac/Makefile.in index 1255b13b4249..95fd4a2722d5 100644 --- a/Mac/Makefile.in +++ b/Mac/Makefile.in @@ -221,10 +221,6 @@ install_IDLE: -test -d "$(DESTDIR)$(PYTHONAPPSDIR)/IDLE.app" && rm -rf "$(DESTDIR)$(PYTHONAPPSDIR)/IDLE.app" /bin/cp -PR "$(srcdir)/IDLE/IDLE.app" "$(DESTDIR)$(PYTHONAPPSDIR)" ln -sf "$(INSTALLED_PYTHONAPP)" "$(DESTDIR)$(PYTHONAPPSDIR)/IDLE.app/Contents/MacOS/Python" -ifneq ($(LIPO_32BIT_FLAGS),) - rm "$(DESTDIR)$(PYTHONAPPSDIR)/IDLE.app/Contents/MacOS/Python" - lipo $(LIPO_32BIT_FLAGS) -output "$(DESTDIR)$(PYTHONAPPSDIR)/IDLE.app/Contents/MacOS/Python" "$(BUILDPYTHON)" -endif sed -e "s!%prefix%!$(prefix)!g" -e 's!%exe%!$(PYTHONFRAMEWORK)!g' < "$(srcdir)/IDLE/IDLE.app/Contents/MacOS/IDLE" > "$(DESTDIR)$(PYTHONAPPSDIR)/IDLE.app/Contents/MacOS/IDLE" sed "s!%version%!`$(RUNSHARED) $(BUILDPYTHON) -c 'import platform; print(platform.python_version())'`!g" < "$(srcdir)/IDLE/IDLE.app/Contents/Info.plist" > "$(DESTDIR)$(PYTHONAPPSDIR)/IDLE.app/Contents/Info.plist" if [ -f "$(DESTDIR)$(LIBDEST)/idlelib/config-main.def" ]; then \ From webhook-mailer at python.org Wed Mar 28 04:46:38 2018 From: webhook-mailer at python.org (Ned Deily) Date: Wed, 28 Mar 2018 08:46:38 -0000 Subject: [Python-checkins] bpo-33163: Upgrade pip to 9.0.3 and setuptools to v39.0.1. (GH-6282) Message-ID: <mailman.228.1522226800.1871.python-checkins@python.org> https://github.com/python/cpython/commit/c0518cde7a8404f310cd3495e77e612820ecad4f commit: c0518cde7a8404f310cd3495e77e612820ecad4f branch: master author: Ned Deily <nad at python.org> committer: GitHub <noreply at github.com> date: 2018-03-28T04:46:35-04:00 summary: bpo-33163: Upgrade pip to 9.0.3 and setuptools to v39.0.1. (GH-6282) files: A Misc/NEWS.d/next/Build/2018-03-28-04-15-03.bpo-33163.hfpWuU.rst diff --git a/Misc/NEWS.d/next/Build/2018-03-28-04-15-03.bpo-33163.hfpWuU.rst b/Misc/NEWS.d/next/Build/2018-03-28-04-15-03.bpo-33163.hfpWuU.rst new file mode 100644 index 000000000000..b3f04e3f800c --- /dev/null +++ b/Misc/NEWS.d/next/Build/2018-03-28-04-15-03.bpo-33163.hfpWuU.rst @@ -0,0 +1 @@ +Upgrade pip to 9.0.3 and setuptools to v39.0.1. From webhook-mailer at python.org Wed Mar 28 04:49:27 2018 From: webhook-mailer at python.org (Ned Deily) Date: Wed, 28 Mar 2018 08:49:27 -0000 Subject: [Python-checkins] [3.7] bpo-33163: Upgrade pip to 9.0.3 and setuptools to v39.0.1. (GH-6283) Message-ID: <mailman.229.1522226968.1871.python-checkins@python.org> https://github.com/python/cpython/commit/b9172b9630eeefc6dea1409b5bf4180b27fc359f commit: b9172b9630eeefc6dea1409b5bf4180b27fc359f branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Ned Deily <nad at python.org> date: 2018-03-28T04:49:24-04:00 summary: [3.7] bpo-33163: Upgrade pip to 9.0.3 and setuptools to v39.0.1. (GH-6283) (cherry picked from commit c0518cde7a8404f310cd3495e77e612820ecad4f) Co-authored-by: Ned Deily <nad at python.org> files: A Misc/NEWS.d/next/Build/2018-03-28-04-15-03.bpo-33163.hfpWuU.rst diff --git a/Misc/NEWS.d/next/Build/2018-03-28-04-15-03.bpo-33163.hfpWuU.rst b/Misc/NEWS.d/next/Build/2018-03-28-04-15-03.bpo-33163.hfpWuU.rst new file mode 100644 index 000000000000..b3f04e3f800c --- /dev/null +++ b/Misc/NEWS.d/next/Build/2018-03-28-04-15-03.bpo-33163.hfpWuU.rst @@ -0,0 +1 @@ +Upgrade pip to 9.0.3 and setuptools to v39.0.1. From webhook-mailer at python.org Wed Mar 28 04:52:10 2018 From: webhook-mailer at python.org (Ned Deily) Date: Wed, 28 Mar 2018 08:52:10 -0000 Subject: [Python-checkins] bpo-33163: Upgrade pip to 9.0.3 and setuptools to v39.0.1. (GH-6285) Message-ID: <mailman.230.1522227132.1871.python-checkins@python.org> https://github.com/python/cpython/commit/9dad016784848a53d9f2557292025f7a4104c5a5 commit: 9dad016784848a53d9f2557292025f7a4104c5a5 branch: 3.6 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Ned Deily <nad at python.org> date: 2018-03-28T04:52:07-04:00 summary: bpo-33163: Upgrade pip to 9.0.3 and setuptools to v39.0.1. (GH-6285) (cherry picked from commit c0518cde7a8404f310cd3495e77e612820ecad4f) Co-authored-by: Ned Deily <nad at python.org> files: A Misc/NEWS.d/next/Build/2018-03-28-04-15-03.bpo-33163.hfpWuU.rst diff --git a/Misc/NEWS.d/next/Build/2018-03-28-04-15-03.bpo-33163.hfpWuU.rst b/Misc/NEWS.d/next/Build/2018-03-28-04-15-03.bpo-33163.hfpWuU.rst new file mode 100644 index 000000000000..b3f04e3f800c --- /dev/null +++ b/Misc/NEWS.d/next/Build/2018-03-28-04-15-03.bpo-33163.hfpWuU.rst @@ -0,0 +1 @@ +Upgrade pip to 9.0.3 and setuptools to v39.0.1. From webhook-mailer at python.org Wed Mar 28 04:55:33 2018 From: webhook-mailer at python.org (Ned Deily) Date: Wed, 28 Mar 2018 08:55:33 -0000 Subject: [Python-checkins] bpo-33163: Upgrade pip to 9.0.3 and setuptools to v39.0.1. (GH-6284) Message-ID: <mailman.231.1522227335.1871.python-checkins@python.org> https://github.com/python/cpython/commit/7f48a426fc69d144d4242517ef40eff01c1fd483 commit: 7f48a426fc69d144d4242517ef40eff01c1fd483 branch: 2.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Ned Deily <nad at python.org> date: 2018-03-28T04:55:30-04:00 summary: bpo-33163: Upgrade pip to 9.0.3 and setuptools to v39.0.1. (GH-6284) (cherry picked from commit c0518cde7a8404f310cd3495e77e612820ecad4f) Co-authored-by: Ned Deily <nad at python.org> files: A Misc/NEWS.d/next/Build/2018-03-28-04-15-03.bpo-33163.hfpWuU.rst diff --git a/Misc/NEWS.d/next/Build/2018-03-28-04-15-03.bpo-33163.hfpWuU.rst b/Misc/NEWS.d/next/Build/2018-03-28-04-15-03.bpo-33163.hfpWuU.rst new file mode 100644 index 000000000000..b3f04e3f800c --- /dev/null +++ b/Misc/NEWS.d/next/Build/2018-03-28-04-15-03.bpo-33163.hfpWuU.rst @@ -0,0 +1 @@ +Upgrade pip to 9.0.3 and setuptools to v39.0.1. From solipsis at pitrou.net Wed Mar 28 05:17:08 2018 From: solipsis at pitrou.net (solipsis at pitrou.net) Date: Wed, 28 Mar 2018 09:17:08 +0000 Subject: [Python-checkins] Daily reference leaks (4243df51fe43): sum=-1 Message-ID: <20180328091708.1.9824ACB62B64FAFD@psf.io> results for 4243df51fe43 on branch "default" -------------------------------------------- test_collections leaked [0, 0, -7] memory blocks, sum=-7 test_functools leaked [0, 3, 1] memory blocks, sum=4 test_multiprocessing_forkserver leaked [2, 0, 0] memory blocks, sum=2 Command line was: ['./python', '-m', 'test.regrtest', '-uall', '-R', '3:3:/home/psf-users/antoine/refleaks/refloghOmNoP', '--timeout', '7200'] From webhook-mailer at python.org Wed Mar 28 09:08:05 2018 From: webhook-mailer at python.org (INADA Naoki) Date: Wed, 28 Mar 2018 13:08:05 -0000 Subject: [Python-checkins] s/the the/the/ (GH-6287) Message-ID: <mailman.232.1522242486.1871.python-checkins@python.org> https://github.com/python/cpython/commit/40a536be5337d3723285d597e2906394b26816a5 commit: 40a536be5337d3723285d597e2906394b26816a5 branch: master author: INADA Naoki <methane at users.noreply.github.com> committer: GitHub <noreply at github.com> date: 2018-03-28T22:07:57+09:00 summary: s/the the/the/ (GH-6287) files: M Doc/glossary.rst M Misc/HISTORY diff --git a/Doc/glossary.rst b/Doc/glossary.rst index dcfe086b38b1..2eab00314682 100644 --- a/Doc/glossary.rst +++ b/Doc/glossary.rst @@ -463,7 +463,7 @@ Glossary hash-based pyc - A bytecode cache file that uses the the hash rather than the last-modified + A bytecode cache file that uses the hash rather than the last-modified time of the corresponding source file to determine its validity. See :ref:`pyc-invalidation`. diff --git a/Misc/HISTORY b/Misc/HISTORY index 26c7f2074d7b..d4a1b16a7c04 100644 --- a/Misc/HISTORY +++ b/Misc/HISTORY @@ -1389,7 +1389,7 @@ Release date: 2014-09-22 Core and Builtins ----------------- -- Issue #22258: Fix the the internal function set_inheritable() on Illumos. +- Issue #22258: Fix the internal function set_inheritable() on Illumos. This platform exposes the function ``ioctl(FIOCLEX)``, but calling it fails with errno is ENOTTY: "Inappropriate ioctl for device". set_inheritable() now falls back to the slower ``fcntl()`` (``F_GETFD`` and then ``F_SETFD``). From webhook-mailer at python.org Wed Mar 28 09:23:36 2018 From: webhook-mailer at python.org (Miss Islington (bot)) Date: Wed, 28 Mar 2018 13:23:36 -0000 Subject: [Python-checkins] s/the the/the/ (GH-6287) Message-ID: <mailman.233.1522243417.1871.python-checkins@python.org> https://github.com/python/cpython/commit/6cb556ffa6cbd731ef147eb89fe262a0c738f8b1 commit: 6cb556ffa6cbd731ef147eb89fe262a0c738f8b1 branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: GitHub <noreply at github.com> date: 2018-03-28T06:23:32-07:00 summary: s/the the/the/ (GH-6287) (cherry picked from commit 40a536be5337d3723285d597e2906394b26816a5) Co-authored-by: INADA Naoki <methane at users.noreply.github.com> files: M Doc/glossary.rst M Misc/HISTORY diff --git a/Doc/glossary.rst b/Doc/glossary.rst index dcfe086b38b1..2eab00314682 100644 --- a/Doc/glossary.rst +++ b/Doc/glossary.rst @@ -463,7 +463,7 @@ Glossary hash-based pyc - A bytecode cache file that uses the the hash rather than the last-modified + A bytecode cache file that uses the hash rather than the last-modified time of the corresponding source file to determine its validity. See :ref:`pyc-invalidation`. diff --git a/Misc/HISTORY b/Misc/HISTORY index 26c7f2074d7b..d4a1b16a7c04 100644 --- a/Misc/HISTORY +++ b/Misc/HISTORY @@ -1389,7 +1389,7 @@ Release date: 2014-09-22 Core and Builtins ----------------- -- Issue #22258: Fix the the internal function set_inheritable() on Illumos. +- Issue #22258: Fix the internal function set_inheritable() on Illumos. This platform exposes the function ``ioctl(FIOCLEX)``, but calling it fails with errno is ENOTTY: "Inappropriate ioctl for device". set_inheritable() now falls back to the slower ``fcntl()`` (``F_GETFD`` and then ``F_SETFD``). From webhook-mailer at python.org Wed Mar 28 11:26:38 2018 From: webhook-mailer at python.org (Antoine Pitrou) Date: Wed, 28 Mar 2018 15:26:38 -0000 Subject: [Python-checkins] bpo-33126: Document PyBuffer_ToContiguous() (#6292) Message-ID: <mailman.234.1522250800.1871.python-checkins@python.org> https://github.com/python/cpython/commit/aa50bf08e64f49d57917ab0b1aadf4308a3168a6 commit: aa50bf08e64f49d57917ab0b1aadf4308a3168a6 branch: master author: Antoine Pitrou <pitrou at free.fr> committer: GitHub <noreply at github.com> date: 2018-03-28T17:26:32+02:00 summary: bpo-33126: Document PyBuffer_ToContiguous() (#6292) files: A Misc/NEWS.d/next/Documentation/2018-03-28-17-03-17.bpo-33126.5UGkNv.rst M Doc/c-api/buffer.rst diff --git a/Doc/c-api/buffer.rst b/Doc/c-api/buffer.rst index 8c2de9691f3e..5a9a46fc67e8 100644 --- a/Doc/c-api/buffer.rst +++ b/Doc/c-api/buffer.rst @@ -473,6 +473,15 @@ Buffer-related functions (*order* is ``'A'``). Return ``0`` otherwise. +.. c:function:: int PyBuffer_ToContiguous(void *buf, Py_buffer *src, Py_ssize_t len, char order) + + Copy *len* bytes from *src* to its contiguous representation in *buf*. + *order* can be ``'C'`` or ``'F'`` (for C-style or Fortran-style ordering). + ``0`` is returned on success, ``-1`` on error. + + This function fails if *len* != *src->len*. + + .. c:function:: void PyBuffer_FillContiguousStrides(int ndims, Py_ssize_t *shape, Py_ssize_t *strides, int itemsize, char order) Fill the *strides* array with byte-strides of a :term:`contiguous` (C-style if @@ -497,6 +506,3 @@ Buffer-related functions If this function is used as part of a :ref:`getbufferproc <buffer-structs>`, *exporter* MUST be set to the exporting object and *flags* must be passed unmodified. Otherwise, *exporter* MUST be NULL. - - - diff --git a/Misc/NEWS.d/next/Documentation/2018-03-28-17-03-17.bpo-33126.5UGkNv.rst b/Misc/NEWS.d/next/Documentation/2018-03-28-17-03-17.bpo-33126.5UGkNv.rst new file mode 100644 index 000000000000..1219790e79ec --- /dev/null +++ b/Misc/NEWS.d/next/Documentation/2018-03-28-17-03-17.bpo-33126.5UGkNv.rst @@ -0,0 +1 @@ +Document PyBuffer_ToContiguous(). From webhook-mailer at python.org Wed Mar 28 11:47:37 2018 From: webhook-mailer at python.org (Antoine Pitrou) Date: Wed, 28 Mar 2018 15:47:37 -0000 Subject: [Python-checkins] bpo-33126: Document PyBuffer_ToContiguous() (GH-6292) (GH-6293) Message-ID: <mailman.235.1522252060.1871.python-checkins@python.org> https://github.com/python/cpython/commit/18fdc87207ea65b3906f07cb47c51a609e442f93 commit: 18fdc87207ea65b3906f07cb47c51a609e442f93 branch: 3.6 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Antoine Pitrou <pitrou at free.fr> date: 2018-03-28T17:47:34+02:00 summary: bpo-33126: Document PyBuffer_ToContiguous() (GH-6292) (GH-6293) (cherry picked from commit aa50bf08e64f49d57917ab0b1aadf4308a3168a6) Co-authored-by: Antoine Pitrou <pitrou at free.fr> files: A Misc/NEWS.d/next/Documentation/2018-03-28-17-03-17.bpo-33126.5UGkNv.rst M Doc/c-api/buffer.rst diff --git a/Doc/c-api/buffer.rst b/Doc/c-api/buffer.rst index 8c2de9691f3e..5a9a46fc67e8 100644 --- a/Doc/c-api/buffer.rst +++ b/Doc/c-api/buffer.rst @@ -473,6 +473,15 @@ Buffer-related functions (*order* is ``'A'``). Return ``0`` otherwise. +.. c:function:: int PyBuffer_ToContiguous(void *buf, Py_buffer *src, Py_ssize_t len, char order) + + Copy *len* bytes from *src* to its contiguous representation in *buf*. + *order* can be ``'C'`` or ``'F'`` (for C-style or Fortran-style ordering). + ``0`` is returned on success, ``-1`` on error. + + This function fails if *len* != *src->len*. + + .. c:function:: void PyBuffer_FillContiguousStrides(int ndims, Py_ssize_t *shape, Py_ssize_t *strides, int itemsize, char order) Fill the *strides* array with byte-strides of a :term:`contiguous` (C-style if @@ -497,6 +506,3 @@ Buffer-related functions If this function is used as part of a :ref:`getbufferproc <buffer-structs>`, *exporter* MUST be set to the exporting object and *flags* must be passed unmodified. Otherwise, *exporter* MUST be NULL. - - - diff --git a/Misc/NEWS.d/next/Documentation/2018-03-28-17-03-17.bpo-33126.5UGkNv.rst b/Misc/NEWS.d/next/Documentation/2018-03-28-17-03-17.bpo-33126.5UGkNv.rst new file mode 100644 index 000000000000..1219790e79ec --- /dev/null +++ b/Misc/NEWS.d/next/Documentation/2018-03-28-17-03-17.bpo-33126.5UGkNv.rst @@ -0,0 +1 @@ +Document PyBuffer_ToContiguous(). From webhook-mailer at python.org Wed Mar 28 11:50:27 2018 From: webhook-mailer at python.org (Antoine Pitrou) Date: Wed, 28 Mar 2018 15:50:27 -0000 Subject: [Python-checkins] bpo-33126: Document PyBuffer_ToContiguous() (GH-6292) (GH-6294) Message-ID: <mailman.236.1522252229.1871.python-checkins@python.org> https://github.com/python/cpython/commit/6124d8ec0d1eb8016e5e54a4d341b8f4f995623c commit: 6124d8ec0d1eb8016e5e54a4d341b8f4f995623c branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Antoine Pitrou <pitrou at free.fr> date: 2018-03-28T17:50:24+02:00 summary: bpo-33126: Document PyBuffer_ToContiguous() (GH-6292) (GH-6294) (cherry picked from commit aa50bf08e64f49d57917ab0b1aadf4308a3168a6) Co-authored-by: Antoine Pitrou <pitrou at free.fr> files: A Misc/NEWS.d/next/Documentation/2018-03-28-17-03-17.bpo-33126.5UGkNv.rst M Doc/c-api/buffer.rst diff --git a/Doc/c-api/buffer.rst b/Doc/c-api/buffer.rst index 8c2de9691f3e..5a9a46fc67e8 100644 --- a/Doc/c-api/buffer.rst +++ b/Doc/c-api/buffer.rst @@ -473,6 +473,15 @@ Buffer-related functions (*order* is ``'A'``). Return ``0`` otherwise. +.. c:function:: int PyBuffer_ToContiguous(void *buf, Py_buffer *src, Py_ssize_t len, char order) + + Copy *len* bytes from *src* to its contiguous representation in *buf*. + *order* can be ``'C'`` or ``'F'`` (for C-style or Fortran-style ordering). + ``0`` is returned on success, ``-1`` on error. + + This function fails if *len* != *src->len*. + + .. c:function:: void PyBuffer_FillContiguousStrides(int ndims, Py_ssize_t *shape, Py_ssize_t *strides, int itemsize, char order) Fill the *strides* array with byte-strides of a :term:`contiguous` (C-style if @@ -497,6 +506,3 @@ Buffer-related functions If this function is used as part of a :ref:`getbufferproc <buffer-structs>`, *exporter* MUST be set to the exporting object and *flags* must be passed unmodified. Otherwise, *exporter* MUST be NULL. - - - diff --git a/Misc/NEWS.d/next/Documentation/2018-03-28-17-03-17.bpo-33126.5UGkNv.rst b/Misc/NEWS.d/next/Documentation/2018-03-28-17-03-17.bpo-33126.5UGkNv.rst new file mode 100644 index 000000000000..1219790e79ec --- /dev/null +++ b/Misc/NEWS.d/next/Documentation/2018-03-28-17-03-17.bpo-33126.5UGkNv.rst @@ -0,0 +1 @@ +Document PyBuffer_ToContiguous(). From webhook-mailer at python.org Wed Mar 28 14:27:18 2018 From: webhook-mailer at python.org (Ned Deily) Date: Wed, 28 Mar 2018 18:27:18 -0000 Subject: [Python-checkins] bpo-30622: Fix backport of NPN fix (#6102) Message-ID: <mailman.237.1522261642.1871.python-checkins@python.org> https://github.com/python/cpython/commit/15b6400d6439aad9859faba91ce297dfeae8d31d commit: 15b6400d6439aad9859faba91ce297dfeae8d31d branch: 3.6 author: Christian Heimes <christian at python.org> committer: Ned Deily <nad at python.org> date: 2018-03-28T03:32:58-04:00 summary: bpo-30622: Fix backport of NPN fix (#6102) Fix backport a79591cf of bpo-30622 to 3.6 branch. Signed-off-by: Christian Heimes <christian at python.org> files: M Modules/_ssl.c diff --git a/Modules/_ssl.c b/Modules/_ssl.c index 2fe696daa5e1..c54e43c2b48a 100644 --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -2747,7 +2747,7 @@ _ssl__SSLContext_impl(PyTypeObject *type, int proto_version) return NULL; } self->ctx = ctx; -#ifdef HAVE_NPN +#if HAVE_NPN self->npn_protocols = NULL; #endif #if HAVE_ALPN From webhook-mailer at python.org Wed Mar 28 15:14:35 2018 From: webhook-mailer at python.org (Serhiy Storchaka) Date: Wed, 28 Mar 2018 19:14:35 -0000 Subject: [Python-checkins] Fix duplicating words words. (GH-6296) Message-ID: <mailman.238.1522264477.1871.python-checkins@python.org> https://github.com/python/cpython/commit/bac2d5ba30339298db7d4caa9c8cd31d807cf081 commit: bac2d5ba30339298db7d4caa9c8cd31d807cf081 branch: master author: Serhiy Storchaka <storchaka at gmail.com> committer: GitHub <noreply at github.com> date: 2018-03-28T22:14:26+03:00 summary: Fix duplicating words words. (GH-6296) Most of them have been added in 3.7. files: M Doc/library/asyncio-protocol.rst M Doc/library/contextvars.rst M Doc/library/datetime.rst M Doc/library/test.rst M Doc/whatsnew/3.7.rst M Lib/asyncio/protocols.py M Lib/idlelib/idle_test/test_editmenu.py M Lib/idlelib/idle_test/test_text.py M Misc/NEWS.d/next/Library/2018-01-18-13-09-00.bpo-32585.qpeijr.rst M Misc/NEWS.d/next/Library/2018-03-18-17-38-48.bpo-32953.t8WAWN.rst M Modules/_datetimemodule.c diff --git a/Doc/library/asyncio-protocol.rst b/Doc/library/asyncio-protocol.rst index 004cac80d90c..ef6441605cd7 100644 --- a/Doc/library/asyncio-protocol.rst +++ b/Doc/library/asyncio-protocol.rst @@ -339,7 +339,7 @@ Protocol classes control of the receive buffer. .. versionadded:: 3.7 - **Important:** this has been been added to asyncio in Python 3.7 + **Important:** this has been added to asyncio in Python 3.7 *on a provisional basis*! Treat it as an experimental API that might be changed or removed in Python 3.8. @@ -450,7 +450,7 @@ Streaming protocols with manual receive buffer control ------------------------------------------------------ .. versionadded:: 3.7 - **Important:** :class:`BufferedProtocol` has been been added to + **Important:** :class:`BufferedProtocol` has been added to asyncio in Python 3.7 *on a provisional basis*! Consider it as an experimental API that might be changed or removed in Python 3.8. diff --git a/Doc/library/contextvars.rst b/Doc/library/contextvars.rst index 1e0987ce4d6a..abd0d5fa0fdf 100644 --- a/Doc/library/contextvars.rst +++ b/Doc/library/contextvars.rst @@ -253,7 +253,7 @@ client:: addr = writer.transport.get_extra_info('socket').getpeername() client_addr_var.set(addr) - # In any code that we call is is now possible to get + # In any code that we call is now possible to get # client's address by calling 'client_addr_var.get()'. while True: diff --git a/Doc/library/datetime.rst b/Doc/library/datetime.rst index c1b164ebc1f2..8d91f4ef9346 100644 --- a/Doc/library/datetime.rst +++ b/Doc/library/datetime.rst @@ -2209,8 +2209,8 @@ Notes: :meth:`utcoffset` is transformed into a string of the form ?HHMM[SS[.uuuuuu]], where HH is a 2-digit string giving the number of UTC offset hours, and MM is a 2-digit string giving the number of UTC offset - minutes, SS is a 2-digit string string giving the number of UTC offset - seconds and uuuuuu is a 2-digit string string giving the number of UTC + minutes, SS is a 2-digit string giving the number of UTC offset + seconds and uuuuuu is a 2-digit string giving the number of UTC offset microseconds. The uuuuuu part is omitted when the offset is a whole number of minutes and both the uuuuuu and the SS parts are omitted when the offset is a whole number of minutes. For example, if diff --git a/Doc/library/test.rst b/Doc/library/test.rst index 0746fcfde0aa..7b0971a83bcb 100644 --- a/Doc/library/test.rst +++ b/Doc/library/test.rst @@ -1153,7 +1153,7 @@ The :mod:`test.support` module defines the following functions: *module*. The *name_of_module* argument can specify (as a string or tuple thereof) what - module(s) an API could be defined in in order to be detected as a public + module(s) an API could be defined in order to be detected as a public API. One case for this is when *module* imports part of its public API from other modules, possibly a C backend (like ``csv`` and its ``_csv``). diff --git a/Doc/whatsnew/3.7.rst b/Doc/whatsnew/3.7.rst index e0c19cfa3bd0..1f524884adae 100644 --- a/Doc/whatsnew/3.7.rst +++ b/Doc/whatsnew/3.7.rst @@ -684,7 +684,7 @@ feature. Instances must be created with :class:`~ssl.SSLContext` methods (Contributed by Christian Heimes in :issue:`32951`) OpenSSL 1.1 APIs for setting the minimum and maximum TLS protocol version are -available as as :attr:`~ssl.SSLContext.minimum_version` and +available as :attr:`~ssl.SSLContext.minimum_version` and :attr:`~ssl.SSLContext.maximum_version`. Supported protocols are indicated by new flags like :data:`~ssl.HAS_TLSv1_1`. (Contributed by Christian Heimes in :issue:`32609`.) diff --git a/Lib/asyncio/protocols.py b/Lib/asyncio/protocols.py index 8904478f1ab2..dc298a8d5c95 100644 --- a/Lib/asyncio/protocols.py +++ b/Lib/asyncio/protocols.py @@ -105,7 +105,7 @@ def eof_received(self): class BufferedProtocol(BaseProtocol): """Interface for stream protocol with manual buffer control. - Important: this has been been added to asyncio in Python 3.7 + Important: this has been added to asyncio in Python 3.7 *on a provisional basis*! Consider it as an experimental API that might be changed or removed in Python 3.8. diff --git a/Lib/idlelib/idle_test/test_editmenu.py b/Lib/idlelib/idle_test/test_editmenu.py index 17eb25c4b4c0..17478473a3d1 100644 --- a/Lib/idlelib/idle_test/test_editmenu.py +++ b/Lib/idlelib/idle_test/test_editmenu.py @@ -1,6 +1,6 @@ '''Test (selected) IDLE Edit menu items. -Edit modules have their own test files files +Edit modules have their own test files ''' from test.support import requires requires('gui') diff --git a/Lib/idlelib/idle_test/test_text.py b/Lib/idlelib/idle_test/test_text.py index a5ba7bb21366..0f31179e04b2 100644 --- a/Lib/idlelib/idle_test/test_text.py +++ b/Lib/idlelib/idle_test/test_text.py @@ -9,7 +9,7 @@ class TextTest(object): "Define items common to both sets of tests." - hw = 'hello\nworld' # Several tests insert this after after initialization. + hw = 'hello\nworld' # Several tests insert this after initialization. hwn = hw+'\n' # \n present at initialization, before insert # setUpClass defines cls.Text and maybe cls.root. diff --git a/Misc/NEWS.d/next/Library/2018-01-18-13-09-00.bpo-32585.qpeijr.rst b/Misc/NEWS.d/next/Library/2018-01-18-13-09-00.bpo-32585.qpeijr.rst index c504e8b1e538..0a602045bacd 100644 --- a/Misc/NEWS.d/next/Library/2018-01-18-13-09-00.bpo-32585.qpeijr.rst +++ b/Misc/NEWS.d/next/Library/2018-01-18-13-09-00.bpo-32585.qpeijr.rst @@ -1 +1 @@ -Add Ttk spinbox widget to to tkinter.ttk. Patch by Alan D Moore. +Add Ttk spinbox widget to :mod:`tkinter.ttk`. Patch by Alan D Moore. diff --git a/Misc/NEWS.d/next/Library/2018-03-18-17-38-48.bpo-32953.t8WAWN.rst b/Misc/NEWS.d/next/Library/2018-03-18-17-38-48.bpo-32953.t8WAWN.rst index fbea34aa9a2a..03c1162c7833 100644 --- a/Misc/NEWS.d/next/Library/2018-03-18-17-38-48.bpo-32953.t8WAWN.rst +++ b/Misc/NEWS.d/next/Library/2018-03-18-17-38-48.bpo-32953.t8WAWN.rst @@ -1,4 +1,4 @@ If a non-dataclass inherits from a frozen dataclass, allow attributes to be -added to the derived class. Only attributes from from the frozen dataclass +added to the derived class. Only attributes from the frozen dataclass cannot be assigned to. Require all dataclasses in a hierarchy to be either all frozen or all non-frozen. diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index b69fcdffcc92..6855903bdd57 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -5214,7 +5214,7 @@ get_flip_fold_offset(PyObject *dt) /* PEP 495 exception: Whenever one or both of the operands in * inter-zone comparison is such that its utcoffset() depends - * on the value of its fold fold attribute, the result is False. + * on the value of its fold attribute, the result is False. * * Return 1 if exception applies, 0 if not, and -1 on error. */ From webhook-mailer at python.org Wed Mar 28 16:05:27 2018 From: webhook-mailer at python.org (Serhiy Storchaka) Date: Wed, 28 Mar 2018 20:05:27 -0000 Subject: [Python-checkins] [3.7] Fix duplicating words words. (GH-6296) (GH-6297) Message-ID: <mailman.239.1522267528.1871.python-checkins@python.org> https://github.com/python/cpython/commit/fd93666c8aa100869242ea272f526a4de8db98c1 commit: fd93666c8aa100869242ea272f526a4de8db98c1 branch: 3.7 author: Serhiy Storchaka <storchaka at gmail.com> committer: GitHub <noreply at github.com> date: 2018-03-28T23:05:24+03:00 summary: [3.7] Fix duplicating words words. (GH-6296) (GH-6297) Most of them have been added in 3.7. (cherry picked from commit bac2d5ba30339298db7d4caa9c8cd31d807cf081) files: M Doc/library/asyncio-protocol.rst M Doc/library/contextvars.rst M Doc/library/datetime.rst M Doc/library/test.rst M Doc/whatsnew/3.7.rst M Lib/asyncio/protocols.py M Lib/idlelib/idle_test/test_editmenu.py M Lib/idlelib/idle_test/test_text.py M Misc/NEWS.d/3.7.0b2.rst M Misc/NEWS.d/next/Library/2018-03-18-17-38-48.bpo-32953.t8WAWN.rst M Modules/_datetimemodule.c diff --git a/Doc/library/asyncio-protocol.rst b/Doc/library/asyncio-protocol.rst index 004cac80d90c..ef6441605cd7 100644 --- a/Doc/library/asyncio-protocol.rst +++ b/Doc/library/asyncio-protocol.rst @@ -339,7 +339,7 @@ Protocol classes control of the receive buffer. .. versionadded:: 3.7 - **Important:** this has been been added to asyncio in Python 3.7 + **Important:** this has been added to asyncio in Python 3.7 *on a provisional basis*! Treat it as an experimental API that might be changed or removed in Python 3.8. @@ -450,7 +450,7 @@ Streaming protocols with manual receive buffer control ------------------------------------------------------ .. versionadded:: 3.7 - **Important:** :class:`BufferedProtocol` has been been added to + **Important:** :class:`BufferedProtocol` has been added to asyncio in Python 3.7 *on a provisional basis*! Consider it as an experimental API that might be changed or removed in Python 3.8. diff --git a/Doc/library/contextvars.rst b/Doc/library/contextvars.rst index 1e0987ce4d6a..abd0d5fa0fdf 100644 --- a/Doc/library/contextvars.rst +++ b/Doc/library/contextvars.rst @@ -253,7 +253,7 @@ client:: addr = writer.transport.get_extra_info('socket').getpeername() client_addr_var.set(addr) - # In any code that we call is is now possible to get + # In any code that we call is now possible to get # client's address by calling 'client_addr_var.get()'. while True: diff --git a/Doc/library/datetime.rst b/Doc/library/datetime.rst index c1b164ebc1f2..8d91f4ef9346 100644 --- a/Doc/library/datetime.rst +++ b/Doc/library/datetime.rst @@ -2209,8 +2209,8 @@ Notes: :meth:`utcoffset` is transformed into a string of the form ?HHMM[SS[.uuuuuu]], where HH is a 2-digit string giving the number of UTC offset hours, and MM is a 2-digit string giving the number of UTC offset - minutes, SS is a 2-digit string string giving the number of UTC offset - seconds and uuuuuu is a 2-digit string string giving the number of UTC + minutes, SS is a 2-digit string giving the number of UTC offset + seconds and uuuuuu is a 2-digit string giving the number of UTC offset microseconds. The uuuuuu part is omitted when the offset is a whole number of minutes and both the uuuuuu and the SS parts are omitted when the offset is a whole number of minutes. For example, if diff --git a/Doc/library/test.rst b/Doc/library/test.rst index 0746fcfde0aa..7b0971a83bcb 100644 --- a/Doc/library/test.rst +++ b/Doc/library/test.rst @@ -1153,7 +1153,7 @@ The :mod:`test.support` module defines the following functions: *module*. The *name_of_module* argument can specify (as a string or tuple thereof) what - module(s) an API could be defined in in order to be detected as a public + module(s) an API could be defined in order to be detected as a public API. One case for this is when *module* imports part of its public API from other modules, possibly a C backend (like ``csv`` and its ``_csv``). diff --git a/Doc/whatsnew/3.7.rst b/Doc/whatsnew/3.7.rst index e0c19cfa3bd0..1f524884adae 100644 --- a/Doc/whatsnew/3.7.rst +++ b/Doc/whatsnew/3.7.rst @@ -684,7 +684,7 @@ feature. Instances must be created with :class:`~ssl.SSLContext` methods (Contributed by Christian Heimes in :issue:`32951`) OpenSSL 1.1 APIs for setting the minimum and maximum TLS protocol version are -available as as :attr:`~ssl.SSLContext.minimum_version` and +available as :attr:`~ssl.SSLContext.minimum_version` and :attr:`~ssl.SSLContext.maximum_version`. Supported protocols are indicated by new flags like :data:`~ssl.HAS_TLSv1_1`. (Contributed by Christian Heimes in :issue:`32609`.) diff --git a/Lib/asyncio/protocols.py b/Lib/asyncio/protocols.py index 8904478f1ab2..dc298a8d5c95 100644 --- a/Lib/asyncio/protocols.py +++ b/Lib/asyncio/protocols.py @@ -105,7 +105,7 @@ def eof_received(self): class BufferedProtocol(BaseProtocol): """Interface for stream protocol with manual buffer control. - Important: this has been been added to asyncio in Python 3.7 + Important: this has been added to asyncio in Python 3.7 *on a provisional basis*! Consider it as an experimental API that might be changed or removed in Python 3.8. diff --git a/Lib/idlelib/idle_test/test_editmenu.py b/Lib/idlelib/idle_test/test_editmenu.py index 17eb25c4b4c0..17478473a3d1 100644 --- a/Lib/idlelib/idle_test/test_editmenu.py +++ b/Lib/idlelib/idle_test/test_editmenu.py @@ -1,6 +1,6 @@ '''Test (selected) IDLE Edit menu items. -Edit modules have their own test files files +Edit modules have their own test files ''' from test.support import requires requires('gui') diff --git a/Lib/idlelib/idle_test/test_text.py b/Lib/idlelib/idle_test/test_text.py index a5ba7bb21366..0f31179e04b2 100644 --- a/Lib/idlelib/idle_test/test_text.py +++ b/Lib/idlelib/idle_test/test_text.py @@ -9,7 +9,7 @@ class TextTest(object): "Define items common to both sets of tests." - hw = 'hello\nworld' # Several tests insert this after after initialization. + hw = 'hello\nworld' # Several tests insert this after initialization. hwn = hw+'\n' # \n present at initialization, before insert # setUpClass defines cls.Text and maybe cls.root. diff --git a/Misc/NEWS.d/3.7.0b2.rst b/Misc/NEWS.d/3.7.0b2.rst index fa659d5dccbc..d597cee12309 100644 --- a/Misc/NEWS.d/3.7.0b2.rst +++ b/Misc/NEWS.d/3.7.0b2.rst @@ -390,7 +390,7 @@ found. Patch by Zackery Spytz. .. nonce: qpeijr .. section: Library -Add Ttk spinbox widget to tkinter.ttk. Patch by Alan D Moore. +Add Ttk spinbox widget to :mod:`tkinter.ttk`. Patch by Alan D Moore. .. diff --git a/Misc/NEWS.d/next/Library/2018-03-18-17-38-48.bpo-32953.t8WAWN.rst b/Misc/NEWS.d/next/Library/2018-03-18-17-38-48.bpo-32953.t8WAWN.rst index fbea34aa9a2a..03c1162c7833 100644 --- a/Misc/NEWS.d/next/Library/2018-03-18-17-38-48.bpo-32953.t8WAWN.rst +++ b/Misc/NEWS.d/next/Library/2018-03-18-17-38-48.bpo-32953.t8WAWN.rst @@ -1,4 +1,4 @@ If a non-dataclass inherits from a frozen dataclass, allow attributes to be -added to the derived class. Only attributes from from the frozen dataclass +added to the derived class. Only attributes from the frozen dataclass cannot be assigned to. Require all dataclasses in a hierarchy to be either all frozen or all non-frozen. diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index b69fcdffcc92..6855903bdd57 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -5214,7 +5214,7 @@ get_flip_fold_offset(PyObject *dt) /* PEP 495 exception: Whenever one or both of the operands in * inter-zone comparison is such that its utcoffset() depends - * on the value of its fold fold attribute, the result is False. + * on the value of its fold attribute, the result is False. * * Return 1 if exception applies, 0 if not, and -1 on error. */ From webhook-mailer at python.org Wed Mar 28 16:51:25 2018 From: webhook-mailer at python.org (Terry Jan Reedy) Date: Wed, 28 Mar 2018 20:51:25 -0000 Subject: [Python-checkins] [3.6] Fix duplicating words words. (GH-6296) (GH-6298) Message-ID: <mailman.240.1522270287.1871.python-checkins@python.org> https://github.com/python/cpython/commit/a10a709711df375303429fcfe052d156803ceb4d commit: a10a709711df375303429fcfe052d156803ceb4d branch: 3.6 author: Terry Jan Reedy <tjreedy at udel.edu> committer: GitHub <noreply at github.com> date: 2018-03-28T16:51:22-04:00 summary: [3.6] Fix duplicating words words. (GH-6296) (GH-6298) Backport idlelib duplications. (cherry picked from commit bac2d5b) files: M Lib/idlelib/idle_test/test_editmenu.py M Lib/idlelib/idle_test/test_text.py diff --git a/Lib/idlelib/idle_test/test_editmenu.py b/Lib/idlelib/idle_test/test_editmenu.py index 17eb25c4b4c0..17478473a3d1 100644 --- a/Lib/idlelib/idle_test/test_editmenu.py +++ b/Lib/idlelib/idle_test/test_editmenu.py @@ -1,6 +1,6 @@ '''Test (selected) IDLE Edit menu items. -Edit modules have their own test files files +Edit modules have their own test files ''' from test.support import requires requires('gui') diff --git a/Lib/idlelib/idle_test/test_text.py b/Lib/idlelib/idle_test/test_text.py index a5ba7bb21366..0f31179e04b2 100644 --- a/Lib/idlelib/idle_test/test_text.py +++ b/Lib/idlelib/idle_test/test_text.py @@ -9,7 +9,7 @@ class TextTest(object): "Define items common to both sets of tests." - hw = 'hello\nworld' # Several tests insert this after after initialization. + hw = 'hello\nworld' # Several tests insert this after initialization. hwn = hw+'\n' # \n present at initialization, before insert # setUpClass defines cls.Text and maybe cls.root. From webhook-mailer at python.org Wed Mar 28 17:14:19 2018 From: webhook-mailer at python.org (Julien Palard) Date: Wed, 28 Mar 2018 21:14:19 -0000 Subject: [Python-checkins] Fix typos '.::' should typically just be '::'. (GH-6165) Message-ID: <mailman.241.1522271660.1871.python-checkins@python.org> https://github.com/python/cpython/commit/78553138be3b38d361bded8e641a2a4fd65a9d16 commit: 78553138be3b38d361bded8e641a2a4fd65a9d16 branch: master author: Julien Palard <julien at palard.fr> committer: GitHub <noreply at github.com> date: 2018-03-28T23:14:15+02:00 summary: Fix typos '.::' should typically just be '::'. (GH-6165) files: M Doc/faq/windows.rst M Doc/library/argparse.rst M Doc/library/mmap.rst diff --git a/Doc/faq/windows.rst b/Doc/faq/windows.rst index d703f2862221..f049bc8387b7 100644 --- a/Doc/faq/windows.rst +++ b/Doc/faq/windows.rst @@ -60,7 +60,7 @@ program. So, how do you arrange for the interpreter to handle your Python? First, you need to make sure that your command window recognises the word "python" as an instruction to start the interpreter. If you have opened a command window, you should try entering the command ``python`` and hitting -return.:: +return:: C:\Users\YourName> python diff --git a/Doc/library/argparse.rst b/Doc/library/argparse.rst index 53e670161dd5..cc99cce2b27d 100644 --- a/Doc/library/argparse.rst +++ b/Doc/library/argparse.rst @@ -981,7 +981,7 @@ is used when no command-line argument was present:: Providing ``default=argparse.SUPPRESS`` causes no attribute to be added if the -command-line argument was not present.:: +command-line argument was not present:: >>> parser = argparse.ArgumentParser() >>> parser.add_argument('--foo', default=argparse.SUPPRESS) diff --git a/Doc/library/mmap.rst b/Doc/library/mmap.rst index 184119df5918..ca09a6a3ca99 100644 --- a/Doc/library/mmap.rst +++ b/Doc/library/mmap.rst @@ -127,7 +127,7 @@ To map anonymous memory, -1 should be passed as the fileno along with the length :class:`~mmap.mmap` can also be used as a context manager in a :keyword:`with` - statement.:: + statement:: import mmap From webhook-mailer at python.org Wed Mar 28 17:25:01 2018 From: webhook-mailer at python.org (Julien Palard) Date: Wed, 28 Mar 2018 21:25:01 -0000 Subject: [Python-checkins] FIX documentation and NEWS of ThreadedHTTPServer. (GH-6207) Message-ID: <mailman.242.1522272303.1871.python-checkins@python.org> https://github.com/python/cpython/commit/79c3bab35cce55e6d175aff96a845bc6d932b203 commit: 79c3bab35cce55e6d175aff96a845bc6d932b203 branch: master author: Julien Palard <julien at palard.fr> committer: GitHub <noreply at github.com> date: 2018-03-28T23:24:58+02:00 summary: FIX documentation and NEWS of ThreadedHTTPServer. (GH-6207) files: M Doc/library/http.server.rst M Misc/NEWS.d/next/Library/2017-12-27-21-55-19.bpo-31639.l3avDJ.rst diff --git a/Doc/library/http.server.rst b/Doc/library/http.server.rst index 4fe46cba691f..278ae5537504 100644 --- a/Doc/library/http.server.rst +++ b/Doc/library/http.server.rst @@ -37,8 +37,11 @@ handler. Code to create and run the server looks like this:: This class is identical to HTTPServer but uses threads to handle requests by using the :class:`~socketserver.ThreadingMixin`. This - is usefull to handle web browsers pre-opening sockets, on which - :class:`HTTPServer` would wait indefinitly. + is useful to handle web browsers pre-opening sockets, on which + :class:`HTTPServer` would wait indefinitely. + + .. versionadded:: 3.7 + The :class:`HTTPServer` and :class:`ThreadedHTTPServer` must be given a *RequestHandlerClass* on instantiation, of which this module diff --git a/Misc/NEWS.d/next/Library/2017-12-27-21-55-19.bpo-31639.l3avDJ.rst b/Misc/NEWS.d/next/Library/2017-12-27-21-55-19.bpo-31639.l3avDJ.rst index e876f40813de..581ac8266b16 100644 --- a/Misc/NEWS.d/next/Library/2017-12-27-21-55-19.bpo-31639.l3avDJ.rst +++ b/Misc/NEWS.d/next/Library/2017-12-27-21-55-19.bpo-31639.l3avDJ.rst @@ -1,2 +1,2 @@ http.server now exposes a ThreadedHTTPServer class and uses it when the -module is invoked to cope with web browsers pre-opening sockets. +module is run with ``-m`` to cope with web browsers pre-opening sockets. From webhook-mailer at python.org Thu Mar 29 04:33:56 2018 From: webhook-mailer at python.org (Ned Deily) Date: Thu, 29 Mar 2018 08:33:56 -0000 Subject: [Python-checkins] FIX documentation and NEWS of ThreadedHTTPServer. (GH-6207) (GH-6302) Message-ID: <mailman.0.1522312436.3857.python-checkins@python.org> https://github.com/python/cpython/commit/a48614bee55fea4c46c3b493b56d2ac4d41b3eff commit: a48614bee55fea4c46c3b493b56d2ac4d41b3eff branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Ned Deily <nad at python.org> date: 2018-03-29T04:33:47-04:00 summary: FIX documentation and NEWS of ThreadedHTTPServer. (GH-6207) (GH-6302) (cherry picked from commit 79c3bab35cce55e6d175aff96a845bc6d932b203) Co-authored-by: Julien Palard <julien at palard.fr> files: M Doc/library/http.server.rst M Misc/NEWS.d/next/Library/2017-12-27-21-55-19.bpo-31639.l3avDJ.rst diff --git a/Doc/library/http.server.rst b/Doc/library/http.server.rst index 4fe46cba691f..278ae5537504 100644 --- a/Doc/library/http.server.rst +++ b/Doc/library/http.server.rst @@ -37,8 +37,11 @@ handler. Code to create and run the server looks like this:: This class is identical to HTTPServer but uses threads to handle requests by using the :class:`~socketserver.ThreadingMixin`. This - is usefull to handle web browsers pre-opening sockets, on which - :class:`HTTPServer` would wait indefinitly. + is useful to handle web browsers pre-opening sockets, on which + :class:`HTTPServer` would wait indefinitely. + + .. versionadded:: 3.7 + The :class:`HTTPServer` and :class:`ThreadedHTTPServer` must be given a *RequestHandlerClass* on instantiation, of which this module diff --git a/Misc/NEWS.d/next/Library/2017-12-27-21-55-19.bpo-31639.l3avDJ.rst b/Misc/NEWS.d/next/Library/2017-12-27-21-55-19.bpo-31639.l3avDJ.rst index e876f40813de..581ac8266b16 100644 --- a/Misc/NEWS.d/next/Library/2017-12-27-21-55-19.bpo-31639.l3avDJ.rst +++ b/Misc/NEWS.d/next/Library/2017-12-27-21-55-19.bpo-31639.l3avDJ.rst @@ -1,2 +1,2 @@ http.server now exposes a ThreadedHTTPServer class and uses it when the -module is invoked to cope with web browsers pre-opening sockets. +module is run with ``-m`` to cope with web browsers pre-opening sockets. From webhook-mailer at python.org Thu Mar 29 04:34:47 2018 From: webhook-mailer at python.org (Ned Deily) Date: Thu, 29 Mar 2018 08:34:47 -0000 Subject: [Python-checkins] Fix typos '.::' should typically just be '::'. (GH-6165) (GH-6300) Message-ID: <mailman.1.1522312488.3857.python-checkins@python.org> https://github.com/python/cpython/commit/a470b7462d741fc2606d5369a935ae2a3dc89ae2 commit: a470b7462d741fc2606d5369a935ae2a3dc89ae2 branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Ned Deily <nad at python.org> date: 2018-03-29T04:34:44-04:00 summary: Fix typos '.::' should typically just be '::'. (GH-6165) (GH-6300) (cherry picked from commit 78553138be3b38d361bded8e641a2a4fd65a9d16) Co-authored-by: Julien Palard <julien at palard.fr> files: M Doc/faq/windows.rst M Doc/library/argparse.rst M Doc/library/mmap.rst diff --git a/Doc/faq/windows.rst b/Doc/faq/windows.rst index d703f2862221..f049bc8387b7 100644 --- a/Doc/faq/windows.rst +++ b/Doc/faq/windows.rst @@ -60,7 +60,7 @@ program. So, how do you arrange for the interpreter to handle your Python? First, you need to make sure that your command window recognises the word "python" as an instruction to start the interpreter. If you have opened a command window, you should try entering the command ``python`` and hitting -return.:: +return:: C:\Users\YourName> python diff --git a/Doc/library/argparse.rst b/Doc/library/argparse.rst index 53e670161dd5..cc99cce2b27d 100644 --- a/Doc/library/argparse.rst +++ b/Doc/library/argparse.rst @@ -981,7 +981,7 @@ is used when no command-line argument was present:: Providing ``default=argparse.SUPPRESS`` causes no attribute to be added if the -command-line argument was not present.:: +command-line argument was not present:: >>> parser = argparse.ArgumentParser() >>> parser.add_argument('--foo', default=argparse.SUPPRESS) diff --git a/Doc/library/mmap.rst b/Doc/library/mmap.rst index 184119df5918..ca09a6a3ca99 100644 --- a/Doc/library/mmap.rst +++ b/Doc/library/mmap.rst @@ -127,7 +127,7 @@ To map anonymous memory, -1 should be passed as the fileno along with the length :class:`~mmap.mmap` can also be used as a context manager in a :keyword:`with` - statement.:: + statement:: import mmap From solipsis at pitrou.net Thu Mar 29 05:13:11 2018 From: solipsis at pitrou.net (solipsis at pitrou.net) Date: Thu, 29 Mar 2018 09:13:11 +0000 Subject: [Python-checkins] Daily reference leaks (4243df51fe43): sum=1 Message-ID: <20180329091311.1.BB60C54AF2F75E57@psf.io> results for 4243df51fe43 on branch "default" -------------------------------------------- test_asyncio leaked [3, 0, 0] memory blocks, sum=3 test_collections leaked [-7, 1, 0] memory blocks, sum=-6 test_functools leaked [0, 3, 1] memory blocks, sum=4 test_multiprocessing_spawn leaked [-2, 1, 1] memory blocks, sum=0 Command line was: ['./python', '-m', 'test.regrtest', '-uall', '-R', '3:3:/home/psf-users/antoine/refleaks/reflogLqjPTC', '--timeout', '7200'] From webhook-mailer at python.org Thu Mar 29 07:41:14 2018 From: webhook-mailer at python.org (Ned Deily) Date: Thu, 29 Mar 2018 11:41:14 -0000 Subject: [Python-checkins] bpo-32726: macOS 10.6+ installer now builds Tcl/TK 8.6 (GH-6308) Message-ID: <mailman.2.1522323677.3857.python-checkins@python.org> https://github.com/python/cpython/commit/7a6f59e123db1ff4bdf311711ad179421ee12b83 commit: 7a6f59e123db1ff4bdf311711ad179421ee12b83 branch: 3.7 author: Ned Deily <nad at python.org> committer: GitHub <noreply at github.com> date: 2018-03-29T07:41:11-04:00 summary: bpo-32726: macOS 10.6+ installer now builds Tcl/TK 8.6 (GH-6308) Build and link with private copy of Tcl/Tk 8.6 for the macOS 10.6+ installer. The 10.9+ installer variant already does this. This means that the Python 3.7 provided by the python.org macOS installers no longer need or use any external versions of Tcl/Tk, either system-provided or user- installed, such as ActiveTcl. files: A Misc/NEWS.d/next/macOS/2018-03-29-06-56-12.bpo-32726.urS9uX.rst M Mac/BuildScript/build-installer.py M Mac/BuildScript/resources/ReadMe.rtf M Mac/BuildScript/resources/Welcome.rtf diff --git a/Mac/BuildScript/build-installer.py b/Mac/BuildScript/build-installer.py index 7f6228ba444f..d2e88d896cee 100755 --- a/Mac/BuildScript/build-installer.py +++ b/Mac/BuildScript/build-installer.py @@ -4,7 +4,7 @@ NEW for 3.7.0: - support Intel 64-bit-only () and 32-bit-only installer builds -- use external Tcl/Tk 8.6 for 10.9+ builds +- build and use internal Tcl/Tk 8.6 for 10.6+ builds - deprecate use of explicit SDK (--sdk-path=) since all but the oldest versions of Xcode support implicit setting of an SDK via environment variables (SDKROOT and friends, see the xcrun man page for more info). @@ -24,7 +24,9 @@ so will fetch them from PyPI if necessary. Since python3 is now used for Sphinx, build-installer.py should also be converted to use python3! -build-installer currently requires an installed third-party version of +For 3.7.0, when building for a 10.6 or higher deployment target, +build-installer builds and links with its own copy of Tcl/Tk 8.6. +Otherwise, it requires an installed third-party version of Tcl/Tk 8.4 (for OS X 10.4 and 10.5 deployment targets), Tcl/TK 8.5 (for 10.6 or later), or Tcl/TK 8.6 (for 10.9 or later) installed in /Library/Frameworks. When installed, @@ -190,9 +192,9 @@ def getTargetCompilers(): EXPECTED_SHARED_LIBS = {} # Are we building and linking with our own copy of Tcl/TK? -# For now, do so if deployment target is 10.9+. +# For now, do so if deployment target is 10.6+. def internalTk(): - return getDeptargetTuple() >= (10, 9) + return getDeptargetTuple() >= (10, 6) # List of names of third party software built with this installer. # The names will be inserted into the rtf version of the License. diff --git a/Mac/BuildScript/resources/ReadMe.rtf b/Mac/BuildScript/resources/ReadMe.rtf index aecbaa353cbf..81d4a99475cf 100644 --- a/Mac/BuildScript/resources/ReadMe.rtf +++ b/Mac/BuildScript/resources/ReadMe.rtf @@ -53,16 +53,10 @@ The bundled \f0 included with this installer has its own default certificate store for verifying download connections.\ \ -\b \ul Using IDLE or other Tk applications [NEW/CHANGED in 3.7.0b1] +\b \ul Using IDLE or other Tk applications [NEW/CHANGED in 3.7.0b3] \b0 \ulnone \ \ -The 10.9+ installer variant comes with its own private version of Tcl/Tk 8.6. It does not use system-supplied or third-party supplied versions of Tcl/Tk.\ -\ -For the 10.6+ variant, you continue to need to install a newer third-party version of the -\i Tcl/Tk -\i0 8.5 (not 8.6) frameworks to use IDLE or other programs that use the Tkinter graphical user interface toolkit. Visit {\field{\*\fldinst{HYPERLINK "https://www.python.org/download/mac/tcltk/"}}{\fldrslt https://www.python.org/download/mac/tcltk/}} for current information about supported and recommended versions of -\i Tcl/Tk -\i0 for this version of Python and of macOS.\ +Both installer variants come with their own private version of Tcl/Tk 8.6. They no longer use system-supplied or third-party supplied versions of Tcl/Tk.\ \b \ul \ Other changes\ diff --git a/Mac/BuildScript/resources/Welcome.rtf b/Mac/BuildScript/resources/Welcome.rtf index 92a843077d78..1aa38193cc88 100644 --- a/Mac/BuildScript/resources/Welcome.rtf +++ b/Mac/BuildScript/resources/Welcome.rtf @@ -23,5 +23,5 @@ \ \b NEW in 3.7.0b1: -\b0 two installer variants (10.9+ 64-bit-only, 10.6+ 64-/32-bit), built-in Tcl/Tk 8.6 support in the 10.9+ variant (no additional third-party downloads!), OpenSSL 1.1.0g, and more!\ +\b0 two installer variants (10.9+ 64-bit-only, 10.6+ 64-/32-bit), built-in Tcl/Tk 8.6 support (no additional third-party downloads!), OpenSSL 1.1.0g, and more!\ } \ No newline at end of file diff --git a/Misc/NEWS.d/next/macOS/2018-03-29-06-56-12.bpo-32726.urS9uX.rst b/Misc/NEWS.d/next/macOS/2018-03-29-06-56-12.bpo-32726.urS9uX.rst new file mode 100644 index 000000000000..470dc7f3eb12 --- /dev/null +++ b/Misc/NEWS.d/next/macOS/2018-03-29-06-56-12.bpo-32726.urS9uX.rst @@ -0,0 +1,5 @@ +Build and link with private copy of Tcl/Tk 8.6 for the macOS 10.6+ +installer. The 10.9+ installer variant already does this. This means that +the Python 3.7 provided by the python.org macOS installers no longer need or +use any external versions of Tcl/Tk, either system-provided or user- +installed, such as ActiveTcl. From webhook-mailer at python.org Thu Mar 29 08:47:35 2018 From: webhook-mailer at python.org (Ned Deily) Date: Thu, 29 Mar 2018 12:47:35 -0000 Subject: [Python-checkins] bpo-32726: macOS 10.6+ installer now builds Tcl/TK 8.6 (GH-6307) Message-ID: <mailman.3.1522327657.3857.python-checkins@python.org> https://github.com/python/cpython/commit/b9e7fe38a07a16942cb65cb922c234c95e2823a0 commit: b9e7fe38a07a16942cb65cb922c234c95e2823a0 branch: master author: Ned Deily <nad at python.org> committer: GitHub <noreply at github.com> date: 2018-03-29T08:47:27-04:00 summary: bpo-32726: macOS 10.6+ installer now builds Tcl/TK 8.6 (GH-6307) Build and link with private copy of Tcl/Tk 8.6 for the macOS 10.6+ installer. The 10.9+ installer variant already does this. This means that the Python 3.7 provided by the python.org macOS installers no longer need or use any external versions of Tcl/Tk, either system-provided or user- installed, such as ActiveTcl. files: A Misc/NEWS.d/next/macOS/2018-03-29-06-56-12.bpo-32726.urS9uX.rst M Mac/BuildScript/build-installer.py M Mac/BuildScript/resources/ReadMe.rtf M Mac/BuildScript/resources/Welcome.rtf diff --git a/Mac/BuildScript/build-installer.py b/Mac/BuildScript/build-installer.py index 7f6228ba444f..d2e88d896cee 100755 --- a/Mac/BuildScript/build-installer.py +++ b/Mac/BuildScript/build-installer.py @@ -4,7 +4,7 @@ NEW for 3.7.0: - support Intel 64-bit-only () and 32-bit-only installer builds -- use external Tcl/Tk 8.6 for 10.9+ builds +- build and use internal Tcl/Tk 8.6 for 10.6+ builds - deprecate use of explicit SDK (--sdk-path=) since all but the oldest versions of Xcode support implicit setting of an SDK via environment variables (SDKROOT and friends, see the xcrun man page for more info). @@ -24,7 +24,9 @@ so will fetch them from PyPI if necessary. Since python3 is now used for Sphinx, build-installer.py should also be converted to use python3! -build-installer currently requires an installed third-party version of +For 3.7.0, when building for a 10.6 or higher deployment target, +build-installer builds and links with its own copy of Tcl/Tk 8.6. +Otherwise, it requires an installed third-party version of Tcl/Tk 8.4 (for OS X 10.4 and 10.5 deployment targets), Tcl/TK 8.5 (for 10.6 or later), or Tcl/TK 8.6 (for 10.9 or later) installed in /Library/Frameworks. When installed, @@ -190,9 +192,9 @@ def getTargetCompilers(): EXPECTED_SHARED_LIBS = {} # Are we building and linking with our own copy of Tcl/TK? -# For now, do so if deployment target is 10.9+. +# For now, do so if deployment target is 10.6+. def internalTk(): - return getDeptargetTuple() >= (10, 9) + return getDeptargetTuple() >= (10, 6) # List of names of third party software built with this installer. # The names will be inserted into the rtf version of the License. diff --git a/Mac/BuildScript/resources/ReadMe.rtf b/Mac/BuildScript/resources/ReadMe.rtf index aecbaa353cbf..81d4a99475cf 100644 --- a/Mac/BuildScript/resources/ReadMe.rtf +++ b/Mac/BuildScript/resources/ReadMe.rtf @@ -53,16 +53,10 @@ The bundled \f0 included with this installer has its own default certificate store for verifying download connections.\ \ -\b \ul Using IDLE or other Tk applications [NEW/CHANGED in 3.7.0b1] +\b \ul Using IDLE or other Tk applications [NEW/CHANGED in 3.7.0b3] \b0 \ulnone \ \ -The 10.9+ installer variant comes with its own private version of Tcl/Tk 8.6. It does not use system-supplied or third-party supplied versions of Tcl/Tk.\ -\ -For the 10.6+ variant, you continue to need to install a newer third-party version of the -\i Tcl/Tk -\i0 8.5 (not 8.6) frameworks to use IDLE or other programs that use the Tkinter graphical user interface toolkit. Visit {\field{\*\fldinst{HYPERLINK "https://www.python.org/download/mac/tcltk/"}}{\fldrslt https://www.python.org/download/mac/tcltk/}} for current information about supported and recommended versions of -\i Tcl/Tk -\i0 for this version of Python and of macOS.\ +Both installer variants come with their own private version of Tcl/Tk 8.6. They no longer use system-supplied or third-party supplied versions of Tcl/Tk.\ \b \ul \ Other changes\ diff --git a/Mac/BuildScript/resources/Welcome.rtf b/Mac/BuildScript/resources/Welcome.rtf index 92a843077d78..1aa38193cc88 100644 --- a/Mac/BuildScript/resources/Welcome.rtf +++ b/Mac/BuildScript/resources/Welcome.rtf @@ -23,5 +23,5 @@ \ \b NEW in 3.7.0b1: -\b0 two installer variants (10.9+ 64-bit-only, 10.6+ 64-/32-bit), built-in Tcl/Tk 8.6 support in the 10.9+ variant (no additional third-party downloads!), OpenSSL 1.1.0g, and more!\ +\b0 two installer variants (10.9+ 64-bit-only, 10.6+ 64-/32-bit), built-in Tcl/Tk 8.6 support (no additional third-party downloads!), OpenSSL 1.1.0g, and more!\ } \ No newline at end of file diff --git a/Misc/NEWS.d/next/macOS/2018-03-29-06-56-12.bpo-32726.urS9uX.rst b/Misc/NEWS.d/next/macOS/2018-03-29-06-56-12.bpo-32726.urS9uX.rst new file mode 100644 index 000000000000..470dc7f3eb12 --- /dev/null +++ b/Misc/NEWS.d/next/macOS/2018-03-29-06-56-12.bpo-32726.urS9uX.rst @@ -0,0 +1,5 @@ +Build and link with private copy of Tcl/Tk 8.6 for the macOS 10.6+ +installer. The 10.9+ installer variant already does this. This means that +the Python 3.7 provided by the python.org macOS installers no longer need or +use any external versions of Tcl/Tk, either system-provided or user- +installed, such as ActiveTcl. From webhook-mailer at python.org Thu Mar 29 08:53:32 2018 From: webhook-mailer at python.org (Ned Deily) Date: Thu, 29 Mar 2018 12:53:32 -0000 Subject: [Python-checkins] 3.7.0b3 Message-ID: <mailman.4.1522328014.3857.python-checkins@python.org> https://github.com/python/cpython/commit/4e7efa9c6f3518c4dd4c332cf7929168e7d44e25 commit: 4e7efa9c6f3518c4dd4c332cf7929168e7d44e25 branch: 3.7 author: Ned Deily <nad at python.org> committer: Ned Deily <nad at python.org> date: 2018-03-29T07:57:55-04:00 summary: 3.7.0b3 files: A Misc/NEWS.d/3.7.0b3.rst D Misc/NEWS.d/next/Build/2018-03-28-04-15-03.bpo-33163.hfpWuU.rst D Misc/NEWS.d/next/C API/2018-01-09-17-03-54.bpo-32374.SwwLoz.rst D Misc/NEWS.d/next/C API/2018-03-20-21-43-09.bpo-33042.FPFp64.rst D Misc/NEWS.d/next/Core and Builtins/2018-02-14-12-35-47.bpo-32836.bThJnx.rst D Misc/NEWS.d/next/Core and Builtins/2018-02-27-13-36-21.bpo-17288.Gdj24S.rst D Misc/NEWS.d/next/Core and Builtins/2018-03-06-12-19-19.bpo-33005.LP-V2U.rst D Misc/NEWS.d/next/Core and Builtins/2018-03-08-09-48-38.bpo-33026.QZA3Ba.rst D Misc/NEWS.d/next/Core and Builtins/2018-03-18-13-56-14.bpo-33041.XwPhI2.rst D Misc/NEWS.d/next/Core and Builtins/2018-03-22-23-09-06.bpo-33018.0ncEJV.rst D Misc/NEWS.d/next/Core and Builtins/2018-03-25-19-49-06.bpo-33053.V3xlsH.rst D Misc/NEWS.d/next/Documentation/2018-03-11-00-16-56.bpo-27428.B7A8FT.rst D Misc/NEWS.d/next/Documentation/2018-03-11-18-53-47.bpo-18802.JhAqH3.rst D Misc/NEWS.d/next/Documentation/2018-03-20-20-11-05.bpo-28247.-V-WS-.rst D Misc/NEWS.d/next/Documentation/2018-03-22-19-23-04.bpo-27212.wrE5KR.rst D Misc/NEWS.d/next/Documentation/2018-03-28-17-03-17.bpo-33126.5UGkNv.rst D Misc/NEWS.d/next/IDLE/2018-02-24-18-20-50.bpo-32940.ZaJ1Rf.rst D Misc/NEWS.d/next/IDLE/2018-03-05-01-29-05.bpo-32984.NGjgT4.rst D Misc/NEWS.d/next/Library/2017-10-05-20-41-48.bpo-27645.1Y_Wag.rst D Misc/NEWS.d/next/Library/2017-12-27-21-55-19.bpo-31639.l3avDJ.rst D Misc/NEWS.d/next/Library/2018-02-16-14-37-14.bpo-32857.-XljAx.rst D Misc/NEWS.d/next/Library/2018-02-28-13-08-00.bpo-32844.u8tnAe.rst D Misc/NEWS.d/next/Library/2018-03-01-17-49-56.bpo-32056.IlpfgE.rst D Misc/NEWS.d/next/Library/2018-03-06-00-19-41.bpo-32969.rGTKa0.rst D Misc/NEWS.d/next/Library/2018-03-06-11-54-59.bpo-33009.-Ekysb.rst D Misc/NEWS.d/next/Library/2018-03-06-20-30-20.bpo-32999.lgFXWl.rst D Misc/NEWS.d/next/Library/2018-03-07-22-28-17.bpo-27683.572Rv4.rst D Misc/NEWS.d/next/Library/2018-03-09-23-07-07.bpo-33037.nAJ3at.rst D Misc/NEWS.d/next/Library/2018-03-11-19-03-52.bpo-31804.i8KUMp.rst D Misc/NEWS.d/next/Library/2018-03-12-00-27-56.bpo-33021.m19B9T.rst D Misc/NEWS.d/next/Library/2018-03-12-16-40-00.bpo-33056.lNN9Eh.rst D Misc/NEWS.d/next/Library/2018-03-12-19-58-25.bpo-33064.LO2KIY.rst D Misc/NEWS.d/next/Library/2018-03-15-07-38-00.bpo-33078.RmjUF5.rst D Misc/NEWS.d/next/Library/2018-03-16-16-07-33.bpo-33061.TRTTek.rst D Misc/NEWS.d/next/Library/2018-03-18-17-38-48.bpo-32953.t8WAWN.rst D Misc/NEWS.d/next/Library/2018-03-19-20-47-00.bpo-33100.chyIO4.rst D Misc/NEWS.d/next/Library/2018-03-20-20-53-21.bpo-32896.ewW3Ln.rst D Misc/NEWS.d/next/Library/2018-03-21-16-52-26.bpo-33116.Tvzerj.rst D Misc/NEWS.d/next/Library/2018-03-21-17-59-39.bpo-33078.PQOniT.rst D Misc/NEWS.d/next/Library/2018-03-22-16-05-56.bpo-32505.YK1N8v.rst D Misc/NEWS.d/next/Library/2018-03-24-15-08-24.bpo-33127.olJmHv.rst D Misc/NEWS.d/next/Library/2018-03-24-19-34-26.bpo-33134.hbVeIX.rst D Misc/NEWS.d/next/Library/2018-03-24-19-54-48.bpo-32873.cHyoAm.rst D Misc/NEWS.d/next/Library/2018-03-25-13-18-16.bpo-33096.ofdbe7.rst D Misc/NEWS.d/next/Library/2018-03-26-12-33-13.bpo-33141.23wlxf.rst D Misc/NEWS.d/next/Security/2018-03-02-10-24-52.bpo-32981.O_qDyj.rst D Misc/NEWS.d/next/Security/2018-03-05-10-09-51.bpo-33001.elj4Aa.rst D Misc/NEWS.d/next/Security/2018-03-25-12-05-43.bpo-33136.TzSN4x.rst D Misc/NEWS.d/next/Tests/2018-01-08-13-33-47.bpo-19417.2asoXy.rst D Misc/NEWS.d/next/Tests/2018-03-09-07-05-12.bpo-32517.ugc1iW.rst D Misc/NEWS.d/next/Tests/2018-03-28-01-35-02.bpo-32872.J5NDUj.rst D Misc/NEWS.d/next/Tools-Demos/2018-02-20-12-16-47.bpo-32885.dL5x7C.rst D Misc/NEWS.d/next/Windows/2018-02-28-11-03-24.bpo-32903.1SXY4t.rst D Misc/NEWS.d/next/Windows/2018-03-07-01-33-33.bpo-33016.Z_Med0.rst D Misc/NEWS.d/next/macOS/2018-03-29-06-56-12.bpo-32726.urS9uX.rst M Include/patchlevel.h M Lib/pydoc_data/topics.py M README.rst diff --git a/Include/patchlevel.h b/Include/patchlevel.h index f500dac164e5..e290cd91b599 100644 --- a/Include/patchlevel.h +++ b/Include/patchlevel.h @@ -20,10 +20,10 @@ #define PY_MINOR_VERSION 7 #define PY_MICRO_VERSION 0 #define PY_RELEASE_LEVEL PY_RELEASE_LEVEL_BETA -#define PY_RELEASE_SERIAL 2 +#define PY_RELEASE_SERIAL 3 /* Version as a string */ -#define PY_VERSION "3.7.0b2+" +#define PY_VERSION "3.7.0b3" /*--end constants--*/ /* Version as a single 4-byte hex number, e.g. 0x010502B2 == 1.5.2b2. diff --git a/Lib/pydoc_data/topics.py b/Lib/pydoc_data/topics.py index ede9385886a4..abface57b092 100644 --- a/Lib/pydoc_data/topics.py +++ b/Lib/pydoc_data/topics.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Autogenerated by Sphinx on Tue Feb 27 19:39:14 2018 +# Autogenerated by Sphinx on Thu Mar 29 07:53:04 2018 topics = {'assert': 'The "assert" statement\n' '**********************\n' '\n' @@ -6665,13 +6665,11 @@ 'object.__complex__(self)\n' 'object.__int__(self)\n' 'object.__float__(self)\n' - 'object.__round__(self[, n])\n' '\n' ' Called to implement the built-in functions "complex()", ' - '"int()",\n' - ' "float()" and "round()". Should return a value of the ' - 'appropriate\n' - ' type.\n' + '"int()" and\n' + ' "float()". Should return a value of the appropriate ' + 'type.\n' '\n' 'object.__index__(self)\n' '\n' @@ -6689,7 +6687,25 @@ 'when\n' ' "__index__()" is defined "__int__()" should also be ' 'defined, and\n' - ' both should return the same value.\n', + ' both should return the same value.\n' + '\n' + 'object.__round__(self[, ndigits])\n' + 'object.__trunc__(self)\n' + 'object.__floor__(self)\n' + 'object.__ceil__(self)\n' + '\n' + ' Called to implement the built-in function "round()" and ' + '"math"\n' + ' functions "trunc()", "floor()" and "ceil()". Unless ' + '*ndigits* is\n' + ' passed to "__round__()" all these methods should return ' + 'the value\n' + ' of the object truncated to an "Integral" (typically an ' + '"int").\n' + '\n' + ' If "__int__()" is not defined then the built-in function ' + '"int()"\n' + ' falls back to "__trunc__()".\n', 'objects': 'Objects, values and types\n' '*************************\n' '\n' @@ -9261,13 +9277,11 @@ 'object.__complex__(self)\n' 'object.__int__(self)\n' 'object.__float__(self)\n' - 'object.__round__(self[, n])\n' '\n' ' Called to implement the built-in functions "complex()", ' - '"int()",\n' - ' "float()" and "round()". Should return a value of the ' - 'appropriate\n' - ' type.\n' + '"int()" and\n' + ' "float()". Should return a value of the appropriate ' + 'type.\n' '\n' 'object.__index__(self)\n' '\n' @@ -9287,6 +9301,24 @@ 'defined, and\n' ' both should return the same value.\n' '\n' + 'object.__round__(self[, ndigits])\n' + 'object.__trunc__(self)\n' + 'object.__floor__(self)\n' + 'object.__ceil__(self)\n' + '\n' + ' Called to implement the built-in function "round()" and ' + '"math"\n' + ' functions "trunc()", "floor()" and "ceil()". Unless ' + '*ndigits* is\n' + ' passed to "__round__()" all these methods should return ' + 'the value\n' + ' of the object truncated to an "Integral" (typically an ' + '"int").\n' + '\n' + ' If "__int__()" is not defined then the built-in function ' + '"int()"\n' + ' falls back to "__trunc__()".\n' + '\n' '\n' 'With Statement Context Managers\n' '===============================\n' diff --git a/Misc/NEWS.d/3.7.0b3.rst b/Misc/NEWS.d/3.7.0b3.rst new file mode 100644 index 000000000000..876463d10ec8 --- /dev/null +++ b/Misc/NEWS.d/3.7.0b3.rst @@ -0,0 +1,541 @@ +.. bpo: 33136 +.. date: 2018-03-25-12-05-43 +.. nonce: TzSN4x +.. release date: 2018-03-29 +.. section: Security + +Harden ssl module against LibreSSL CVE-2018-8970. +X509_VERIFY_PARAM_set1_host() is called with an explicit namelen. A new test +ensures that NULL bytes are not allowed. + +.. + +.. bpo: 33001 +.. date: 2018-03-05-10-09-51 +.. nonce: elj4Aa +.. section: Security + +Minimal fix to prevent buffer overrun in os.symlink on Windows + +.. + +.. bpo: 32981 +.. date: 2018-03-02-10-24-52 +.. nonce: O_qDyj +.. section: Security + +Regexes in difflib and poplib were vulnerable to catastrophic backtracking. +These regexes formed potential DOS vectors (REDOS). They have been +refactored. This resolves CVE-2018-1060 and CVE-2018-1061. Patch by Jamie +Davis. + +.. + +.. bpo: 33053 +.. date: 2018-03-25-19-49-06 +.. nonce: V3xlsH +.. section: Core and Builtins + +When using the -m switch, sys.path[0] is now explicitly expanded as the +*starting* working directory, rather than being left as the empty path +(which allows imports from the current working directory at the time of the +import) + +.. + +.. bpo: 33018 +.. date: 2018-03-22-23-09-06 +.. nonce: 0ncEJV +.. section: Core and Builtins + +Improve consistency of errors raised by ``issubclass()`` when called with a +non-class and an abstract base class as the first and second arguments, +respectively. Patch by Josh Bronson. + +.. + +.. bpo: 33041 +.. date: 2018-03-18-13-56-14 +.. nonce: XwPhI2 +.. section: Core and Builtins + +Fixed jumping when the function contains an ``async for`` loop. + +.. + +.. bpo: 33026 +.. date: 2018-03-08-09-48-38 +.. nonce: QZA3Ba +.. section: Core and Builtins + +Fixed jumping out of "with" block by setting f_lineno. + +.. + +.. bpo: 33005 +.. date: 2018-03-06-12-19-19 +.. nonce: LP-V2U +.. section: Core and Builtins + +Fix a crash on fork when using a custom memory allocator (ex: using +PYTHONMALLOC env var). _PyGILState_Reinit() and _PyInterpreterState_Enable() +now use the default RAW memory allocator to allocate a new interpreters +mutex on fork. + +.. + +.. bpo: 17288 +.. date: 2018-02-27-13-36-21 +.. nonce: Gdj24S +.. section: Core and Builtins + +Prevent jumps from 'return' and 'exception' trace events. + +.. + +.. bpo: 32836 +.. date: 2018-02-14-12-35-47 +.. nonce: bThJnx +.. section: Core and Builtins + +Don't use temporary variables in cases of list/dict/set comprehensions + +.. + +.. bpo: 33141 +.. date: 2018-03-26-12-33-13 +.. nonce: 23wlxf +.. section: Library + +Have Field objects pass through __set_name__ to their default values, if +they have their own __set_name__. + +.. + +.. bpo: 33096 +.. date: 2018-03-25-13-18-16 +.. nonce: ofdbe7 +.. section: Library + +Allow ttk.Treeview.insert to insert iid that has a false boolean value. Note +iid=0 and iid=False would be same. Patch by Garvit Khatri. + +.. + +.. bpo: 32873 +.. date: 2018-03-24-19-54-48 +.. nonce: cHyoAm +.. section: Library + +Treat type variables and special typing forms as immutable by copy and +pickle. This fixes several minor issues and inconsistencies, and improves +backwards compatibility with Python 3.6. + +.. + +.. bpo: 33134 +.. date: 2018-03-24-19-34-26 +.. nonce: hbVeIX +.. section: Library + +When computing dataclass's __hash__, use the lookup table to contain the +function which returns the __hash__ value. This is an improvement over +looking up a string, and then testing that string to see what to do. + +.. + +.. bpo: 33127 +.. date: 2018-03-24-15-08-24 +.. nonce: olJmHv +.. section: Library + +The ssl module now compiles with LibreSSL 2.7.1. + +.. + +.. bpo: 32505 +.. date: 2018-03-22-16-05-56 +.. nonce: YK1N8v +.. section: Library + +Raise TypeError if a member variable of a dataclass is of type Field, but +doesn't have a type annotation. + +.. + +.. bpo: 33078 +.. date: 2018-03-21-17-59-39 +.. nonce: PQOniT +.. section: Library + +Fix the failure on OSX caused by the tests relying on sem_getvalue + +.. + +.. bpo: 33116 +.. date: 2018-03-21-16-52-26 +.. nonce: Tvzerj +.. section: Library + +Add 'Field' to dataclasses.__all__. + +.. + +.. bpo: 32896 +.. date: 2018-03-20-20-53-21 +.. nonce: ewW3Ln +.. section: Library + +Fix an error where subclassing a dataclass with a field that uses a +default_factory would generate an incorrect class. + +.. + +.. bpo: 33100 +.. date: 2018-03-19-20-47-00 +.. nonce: chyIO4 +.. section: Library + +Dataclasses: If a field has a default value that's a MemberDescriptorType, +then it's from that field being in __slots__, not an actual default value. + +.. + +.. bpo: 32953 +.. date: 2018-03-18-17-38-48 +.. nonce: t8WAWN +.. section: Library + +If a non-dataclass inherits from a frozen dataclass, allow attributes to be +added to the derived class. Only attributes from the frozen dataclass +cannot be assigned to. Require all dataclasses in a hierarchy to be either +all frozen or all non-frozen. + +.. + +.. bpo: 33061 +.. date: 2018-03-16-16-07-33 +.. nonce: TRTTek +.. section: Library + +Add missing ``NoReturn`` to ``__all__`` in typing.py + +.. + +.. bpo: 33078 +.. date: 2018-03-15-07-38-00 +.. nonce: RmjUF5 +.. section: Library + +Fix the size handling in multiprocessing.Queue when a pickling error occurs. + +.. + +.. bpo: 33064 +.. date: 2018-03-12-19-58-25 +.. nonce: LO2KIY +.. section: Library + +lib2to3 now properly supports trailing commas after ``*args`` and +``**kwargs`` in function signatures. + +.. + +.. bpo: 33056 +.. date: 2018-03-12-16-40-00 +.. nonce: lNN9Eh +.. section: Library + +FIX properly close leaking fds in concurrent.futures.ProcessPoolExecutor. + +.. + +.. bpo: 33021 +.. date: 2018-03-12-00-27-56 +.. nonce: m19B9T +.. section: Library + +Release the GIL during fstat() calls, avoiding hang of all threads when +calling mmap.mmap(), os.urandom(), and random.seed(). Patch by Nir Soffer. + +.. + +.. bpo: 31804 +.. date: 2018-03-11-19-03-52 +.. nonce: i8KUMp +.. section: Library + +Avoid failing in multiprocessing.Process if the standard streams are closed +or None at exit. + +.. + +.. bpo: 33037 +.. date: 2018-03-09-23-07-07 +.. nonce: nAJ3at +.. section: Library + +Skip sending/receiving data after SSL transport closing. + +.. + +.. bpo: 27683 +.. date: 2018-03-07-22-28-17 +.. nonce: 572Rv4 +.. section: Library + +Fix a regression in :mod:`ipaddress` that result of :meth:`hosts` is empty +when the network is constructed by a tuple containing an integer mask and +only 1 bit left for addresses. + +.. + +.. bpo: 32999 +.. date: 2018-03-06-20-30-20 +.. nonce: lgFXWl +.. section: Library + +Fix C implemetation of ``ABC.__subclasscheck__(cls, subclass)`` crashed when +``subclass`` is not a type object. + +.. + +.. bpo: 33009 +.. date: 2018-03-06-11-54-59 +.. nonce: -Ekysb +.. section: Library + +Fix inspect.signature() for single-parameter partialmethods. + +.. + +.. bpo: 32969 +.. date: 2018-03-06-00-19-41 +.. nonce: rGTKa0 +.. section: Library + +Expose several missing constants in zlib and fix corresponding +documentation. + +.. + +.. bpo: 32056 +.. date: 2018-03-01-17-49-56 +.. nonce: IlpfgE +.. section: Library + +Improved exceptions raised for invalid number of channels and sample width +when read an audio file in modules :mod:`aifc`, :mod:`wave` and +:mod:`sunau`. + +.. + +.. bpo: 32844 +.. date: 2018-02-28-13-08-00 +.. nonce: u8tnAe +.. section: Library + +Fix wrong redirection of a low descriptor (0 or 1) to stderr in subprocess +if another low descriptor is closed. + +.. + +.. bpo: 32857 +.. date: 2018-02-16-14-37-14 +.. nonce: -XljAx +.. section: Library + +In :mod:`tkinter`, ``after_cancel(None)`` now raises a :exc:`ValueError` +instead of canceling the first scheduled function. Patch by Cheryl Sabella. + +.. + +.. bpo: 31639 +.. date: 2017-12-27-21-55-19 +.. nonce: l3avDJ +.. section: Library + +http.server now exposes a ThreadedHTTPServer class and uses it when the +module is run with ``-m`` to cope with web browsers pre-opening sockets. + +.. + +.. bpo: 27645 +.. date: 2017-10-05-20-41-48 +.. nonce: 1Y_Wag +.. section: Library + +:class:`sqlite3.Connection` now exposes a +:class:`~sqlite3.Connection.backup` method, if the underlying SQLite library +is at version 3.6.11 or higher. Patch by Lele Gaifax. + +.. + +.. bpo: 33126 +.. date: 2018-03-28-17-03-17 +.. nonce: 5UGkNv +.. section: Documentation + +Document PyBuffer_ToContiguous(). + +.. + +.. bpo: 27212 +.. date: 2018-03-22-19-23-04 +.. nonce: wrE5KR +.. section: Documentation + +Modify documentation for the :func:`islice` recipe to consume initial values +up to the start index. + +.. + +.. bpo: 28247 +.. date: 2018-03-20-20-11-05 +.. nonce: -V-WS- +.. section: Documentation + +Update :mod:`zipapp` documentation to describe how to make standalone +applications. + +.. + +.. bpo: 18802 +.. date: 2018-03-11-18-53-47 +.. nonce: JhAqH3 +.. section: Documentation + +Documentation changes for ipaddress. Patch by Jon Foster and Berker Peksag. + +.. + +.. bpo: 27428 +.. date: 2018-03-11-00-16-56 +.. nonce: B7A8FT +.. section: Documentation + +Update documentation to clarify that ``WindowsRegistryFinder`` implements +``MetaPathFinder``. (Patch by Himanshu Lakhara) + +.. + +.. bpo: 32872 +.. date: 2018-03-28-01-35-02 +.. nonce: J5NDUj +.. section: Tests + +Avoid regrtest compatibility issue with namespace packages. + +.. + +.. bpo: 32517 +.. date: 2018-03-09-07-05-12 +.. nonce: ugc1iW +.. section: Tests + +Fix failing ``test_asyncio`` on macOS 10.12.2+ due to transport of +``KqueueSelector`` loop was not being closed. + +.. + +.. bpo: 19417 +.. date: 2018-01-08-13-33-47 +.. nonce: 2asoXy +.. section: Tests + +Add test_bdb.py. + +.. + +.. bpo: 33163 +.. date: 2018-03-28-04-15-03 +.. nonce: hfpWuU +.. section: Build + +Upgrade pip to 9.0.3 and setuptools to v39.0.1. + +.. + +.. bpo: 33016 +.. date: 2018-03-07-01-33-33 +.. nonce: Z_Med0 +.. section: Windows + +Fix potential use of uninitialized memory in nt._getfinalpathname + +.. + +.. bpo: 32903 +.. date: 2018-02-28-11-03-24 +.. nonce: 1SXY4t +.. section: Windows + +Fix a memory leak in os.chdir() on Windows if the current directory is set +to a UNC path. + +.. + +.. bpo: 32726 +.. date: 2018-03-29-06-56-12 +.. nonce: urS9uX +.. section: macOS + +Build and link with private copy of Tcl/Tk 8.6 for the macOS 10.6+ +installer. The 10.9+ installer variant already does this. This means that +the Python 3.7 provided by the python.org macOS installers no longer need or +use any external versions of Tcl/Tk, either system-provided or user- +installed, such as ActiveTcl. + +.. + +.. bpo: 32984 +.. date: 2018-03-05-01-29-05 +.. nonce: NGjgT4 +.. section: IDLE + +Set ``__file__`` while running a startup file. Like Python, IDLE optionally +runs one startup file in the Shell window before presenting the first +interactive input prompt. For IDLE, ``-s`` runs a file named in +environmental variable :envvar:`IDLESTARTUP` or :envvar:`PYTHONSTARTUP`; +``-r file`` runs ``file``. Python sets ``__file__`` to the startup file +name before running the file and unsets it before the first prompt. IDLE +now does the same when run normally, without the ``-n`` option. + +.. + +.. bpo: 32940 +.. date: 2018-02-24-18-20-50 +.. nonce: ZaJ1Rf +.. section: IDLE + +Simplify and rename StringTranslatePseudoMapping in pyparse. + +.. + +.. bpo: 32885 +.. date: 2018-02-20-12-16-47 +.. nonce: dL5x7C +.. section: Tools/Demos + +Add an ``-n`` flag for ``Tools/scripts/pathfix.py`` to disbale automatic +backup creation (files with ``~`` suffix). + +.. + +.. bpo: 33042 +.. date: 2018-03-20-21-43-09 +.. nonce: FPFp64 +.. section: C API + +Embedding applications may once again call PySys_ResetWarnOptions, +PySys_AddWarnOption, and PySys_AddXOption prior to calling Py_Initialize. + +.. + +.. bpo: 32374 +.. date: 2018-01-09-17-03-54 +.. nonce: SwwLoz +.. section: C API + +Document that m_traverse for multi-phase initialized modules can be called +with m_state=NULL, and add a sanity check diff --git a/Misc/NEWS.d/next/Build/2018-03-28-04-15-03.bpo-33163.hfpWuU.rst b/Misc/NEWS.d/next/Build/2018-03-28-04-15-03.bpo-33163.hfpWuU.rst deleted file mode 100644 index b3f04e3f800c..000000000000 --- a/Misc/NEWS.d/next/Build/2018-03-28-04-15-03.bpo-33163.hfpWuU.rst +++ /dev/null @@ -1 +0,0 @@ -Upgrade pip to 9.0.3 and setuptools to v39.0.1. diff --git a/Misc/NEWS.d/next/C API/2018-01-09-17-03-54.bpo-32374.SwwLoz.rst b/Misc/NEWS.d/next/C API/2018-01-09-17-03-54.bpo-32374.SwwLoz.rst deleted file mode 100644 index f9cf6d6b99ce..000000000000 --- a/Misc/NEWS.d/next/C API/2018-01-09-17-03-54.bpo-32374.SwwLoz.rst +++ /dev/null @@ -1,2 +0,0 @@ -Document that m_traverse for multi-phase initialized modules can be called -with m_state=NULL, and add a sanity check diff --git a/Misc/NEWS.d/next/C API/2018-03-20-21-43-09.bpo-33042.FPFp64.rst b/Misc/NEWS.d/next/C API/2018-03-20-21-43-09.bpo-33042.FPFp64.rst deleted file mode 100644 index f840b55869cc..000000000000 --- a/Misc/NEWS.d/next/C API/2018-03-20-21-43-09.bpo-33042.FPFp64.rst +++ /dev/null @@ -1,2 +0,0 @@ -Embedding applications may once again call PySys_ResetWarnOptions, -PySys_AddWarnOption, and PySys_AddXOption prior to calling Py_Initialize. \ No newline at end of file diff --git a/Misc/NEWS.d/next/Core and Builtins/2018-02-14-12-35-47.bpo-32836.bThJnx.rst b/Misc/NEWS.d/next/Core and Builtins/2018-02-14-12-35-47.bpo-32836.bThJnx.rst deleted file mode 100644 index 4eeb9aa2e52c..000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2018-02-14-12-35-47.bpo-32836.bThJnx.rst +++ /dev/null @@ -1 +0,0 @@ -Don't use temporary variables in cases of list/dict/set comprehensions diff --git a/Misc/NEWS.d/next/Core and Builtins/2018-02-27-13-36-21.bpo-17288.Gdj24S.rst b/Misc/NEWS.d/next/Core and Builtins/2018-02-27-13-36-21.bpo-17288.Gdj24S.rst deleted file mode 100644 index ce9e84c40313..000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2018-02-27-13-36-21.bpo-17288.Gdj24S.rst +++ /dev/null @@ -1 +0,0 @@ -Prevent jumps from 'return' and 'exception' trace events. diff --git a/Misc/NEWS.d/next/Core and Builtins/2018-03-06-12-19-19.bpo-33005.LP-V2U.rst b/Misc/NEWS.d/next/Core and Builtins/2018-03-06-12-19-19.bpo-33005.LP-V2U.rst deleted file mode 100644 index 6c8b99cbb897..000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2018-03-06-12-19-19.bpo-33005.LP-V2U.rst +++ /dev/null @@ -1,4 +0,0 @@ -Fix a crash on fork when using a custom memory allocator (ex: using -PYTHONMALLOC env var). _PyGILState_Reinit() and _PyInterpreterState_Enable() -now use the default RAW memory allocator to allocate a new interpreters mutex -on fork. diff --git a/Misc/NEWS.d/next/Core and Builtins/2018-03-08-09-48-38.bpo-33026.QZA3Ba.rst b/Misc/NEWS.d/next/Core and Builtins/2018-03-08-09-48-38.bpo-33026.QZA3Ba.rst deleted file mode 100644 index dc166d1e5771..000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2018-03-08-09-48-38.bpo-33026.QZA3Ba.rst +++ /dev/null @@ -1 +0,0 @@ -Fixed jumping out of "with" block by setting f_lineno. diff --git a/Misc/NEWS.d/next/Core and Builtins/2018-03-18-13-56-14.bpo-33041.XwPhI2.rst b/Misc/NEWS.d/next/Core and Builtins/2018-03-18-13-56-14.bpo-33041.XwPhI2.rst deleted file mode 100644 index 97b5e2ef1e54..000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2018-03-18-13-56-14.bpo-33041.XwPhI2.rst +++ /dev/null @@ -1 +0,0 @@ -Fixed jumping when the function contains an ``async for`` loop. diff --git a/Misc/NEWS.d/next/Core and Builtins/2018-03-22-23-09-06.bpo-33018.0ncEJV.rst b/Misc/NEWS.d/next/Core and Builtins/2018-03-22-23-09-06.bpo-33018.0ncEJV.rst deleted file mode 100644 index e799e9834aa1..000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2018-03-22-23-09-06.bpo-33018.0ncEJV.rst +++ /dev/null @@ -1,3 +0,0 @@ -Improve consistency of errors raised by ``issubclass()`` when called with a -non-class and an abstract base class as the first and second arguments, -respectively. Patch by Josh Bronson. diff --git a/Misc/NEWS.d/next/Core and Builtins/2018-03-25-19-49-06.bpo-33053.V3xlsH.rst b/Misc/NEWS.d/next/Core and Builtins/2018-03-25-19-49-06.bpo-33053.V3xlsH.rst deleted file mode 100644 index fd32ac150e4c..000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2018-03-25-19-49-06.bpo-33053.V3xlsH.rst +++ /dev/null @@ -1,4 +0,0 @@ -When using the -m switch, sys.path[0] is now explicitly expanded as the -*starting* working directory, rather than being left as the empty path -(which allows imports from the current working directory at the time of the -import) diff --git a/Misc/NEWS.d/next/Documentation/2018-03-11-00-16-56.bpo-27428.B7A8FT.rst b/Misc/NEWS.d/next/Documentation/2018-03-11-00-16-56.bpo-27428.B7A8FT.rst deleted file mode 100644 index c9ac8e22df08..000000000000 --- a/Misc/NEWS.d/next/Documentation/2018-03-11-00-16-56.bpo-27428.B7A8FT.rst +++ /dev/null @@ -1,2 +0,0 @@ -Update documentation to clarify that ``WindowsRegistryFinder`` implements -``MetaPathFinder``. (Patch by Himanshu Lakhara) diff --git a/Misc/NEWS.d/next/Documentation/2018-03-11-18-53-47.bpo-18802.JhAqH3.rst b/Misc/NEWS.d/next/Documentation/2018-03-11-18-53-47.bpo-18802.JhAqH3.rst deleted file mode 100644 index cb9cc2599aca..000000000000 --- a/Misc/NEWS.d/next/Documentation/2018-03-11-18-53-47.bpo-18802.JhAqH3.rst +++ /dev/null @@ -1 +0,0 @@ -Documentation changes for ipaddress. Patch by Jon Foster and Berker Peksag. diff --git a/Misc/NEWS.d/next/Documentation/2018-03-20-20-11-05.bpo-28247.-V-WS-.rst b/Misc/NEWS.d/next/Documentation/2018-03-20-20-11-05.bpo-28247.-V-WS-.rst deleted file mode 100644 index 28a802136fa7..000000000000 --- a/Misc/NEWS.d/next/Documentation/2018-03-20-20-11-05.bpo-28247.-V-WS-.rst +++ /dev/null @@ -1,2 +0,0 @@ -Update :mod:`zipapp` documentation to describe how to make standalone -applications. diff --git a/Misc/NEWS.d/next/Documentation/2018-03-22-19-23-04.bpo-27212.wrE5KR.rst b/Misc/NEWS.d/next/Documentation/2018-03-22-19-23-04.bpo-27212.wrE5KR.rst deleted file mode 100644 index 5910d2c17342..000000000000 --- a/Misc/NEWS.d/next/Documentation/2018-03-22-19-23-04.bpo-27212.wrE5KR.rst +++ /dev/null @@ -1,2 +0,0 @@ -Modify documentation for the :func:`islice` recipe to consume initial values -up to the start index. diff --git a/Misc/NEWS.d/next/Documentation/2018-03-28-17-03-17.bpo-33126.5UGkNv.rst b/Misc/NEWS.d/next/Documentation/2018-03-28-17-03-17.bpo-33126.5UGkNv.rst deleted file mode 100644 index 1219790e79ec..000000000000 --- a/Misc/NEWS.d/next/Documentation/2018-03-28-17-03-17.bpo-33126.5UGkNv.rst +++ /dev/null @@ -1 +0,0 @@ -Document PyBuffer_ToContiguous(). diff --git a/Misc/NEWS.d/next/IDLE/2018-02-24-18-20-50.bpo-32940.ZaJ1Rf.rst b/Misc/NEWS.d/next/IDLE/2018-02-24-18-20-50.bpo-32940.ZaJ1Rf.rst deleted file mode 100644 index 958f9522d4f8..000000000000 --- a/Misc/NEWS.d/next/IDLE/2018-02-24-18-20-50.bpo-32940.ZaJ1Rf.rst +++ /dev/null @@ -1 +0,0 @@ -Simplify and rename StringTranslatePseudoMapping in pyparse. diff --git a/Misc/NEWS.d/next/IDLE/2018-03-05-01-29-05.bpo-32984.NGjgT4.rst b/Misc/NEWS.d/next/IDLE/2018-03-05-01-29-05.bpo-32984.NGjgT4.rst deleted file mode 100644 index 15d40b72caaf..000000000000 --- a/Misc/NEWS.d/next/IDLE/2018-03-05-01-29-05.bpo-32984.NGjgT4.rst +++ /dev/null @@ -1,7 +0,0 @@ -Set ``__file__`` while running a startup file. Like Python, IDLE optionally -runs one startup file in the Shell window before presenting the first interactive -input prompt. For IDLE, ``-s`` runs a file named in environmental variable - :envvar:`IDLESTARTUP` or :envvar:`PYTHONSTARTUP`; ``-r file`` runs -``file``. Python sets ``__file__`` to the startup file name before running the -file and unsets it before the first prompt. IDLE now does the same when run -normally, without the ``-n`` option. diff --git a/Misc/NEWS.d/next/Library/2017-10-05-20-41-48.bpo-27645.1Y_Wag.rst b/Misc/NEWS.d/next/Library/2017-10-05-20-41-48.bpo-27645.1Y_Wag.rst deleted file mode 100644 index c4b7185614a5..000000000000 --- a/Misc/NEWS.d/next/Library/2017-10-05-20-41-48.bpo-27645.1Y_Wag.rst +++ /dev/null @@ -1,3 +0,0 @@ -:class:`sqlite3.Connection` now exposes a :class:`~sqlite3.Connection.backup` -method, if the underlying SQLite library is at version 3.6.11 -or higher. Patch by Lele Gaifax. diff --git a/Misc/NEWS.d/next/Library/2017-12-27-21-55-19.bpo-31639.l3avDJ.rst b/Misc/NEWS.d/next/Library/2017-12-27-21-55-19.bpo-31639.l3avDJ.rst deleted file mode 100644 index 581ac8266b16..000000000000 --- a/Misc/NEWS.d/next/Library/2017-12-27-21-55-19.bpo-31639.l3avDJ.rst +++ /dev/null @@ -1,2 +0,0 @@ -http.server now exposes a ThreadedHTTPServer class and uses it when the -module is run with ``-m`` to cope with web browsers pre-opening sockets. diff --git a/Misc/NEWS.d/next/Library/2018-02-16-14-37-14.bpo-32857.-XljAx.rst b/Misc/NEWS.d/next/Library/2018-02-16-14-37-14.bpo-32857.-XljAx.rst deleted file mode 100644 index 4ebbde4d1946..000000000000 --- a/Misc/NEWS.d/next/Library/2018-02-16-14-37-14.bpo-32857.-XljAx.rst +++ /dev/null @@ -1 +0,0 @@ -In :mod:`tkinter`, ``after_cancel(None)`` now raises a :exc:`ValueError` instead of canceling the first scheduled function. Patch by Cheryl Sabella. diff --git a/Misc/NEWS.d/next/Library/2018-02-28-13-08-00.bpo-32844.u8tnAe.rst b/Misc/NEWS.d/next/Library/2018-02-28-13-08-00.bpo-32844.u8tnAe.rst deleted file mode 100644 index 67412fe5ba46..000000000000 --- a/Misc/NEWS.d/next/Library/2018-02-28-13-08-00.bpo-32844.u8tnAe.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix wrong redirection of a low descriptor (0 or 1) to stderr in subprocess -if another low descriptor is closed. diff --git a/Misc/NEWS.d/next/Library/2018-03-01-17-49-56.bpo-32056.IlpfgE.rst b/Misc/NEWS.d/next/Library/2018-03-01-17-49-56.bpo-32056.IlpfgE.rst deleted file mode 100644 index 421aa3767794..000000000000 --- a/Misc/NEWS.d/next/Library/2018-03-01-17-49-56.bpo-32056.IlpfgE.rst +++ /dev/null @@ -1,3 +0,0 @@ -Improved exceptions raised for invalid number of channels and sample width -when read an audio file in modules :mod:`aifc`, :mod:`wave` and -:mod:`sunau`. diff --git a/Misc/NEWS.d/next/Library/2018-03-06-00-19-41.bpo-32969.rGTKa0.rst b/Misc/NEWS.d/next/Library/2018-03-06-00-19-41.bpo-32969.rGTKa0.rst deleted file mode 100644 index a92307e67bfa..000000000000 --- a/Misc/NEWS.d/next/Library/2018-03-06-00-19-41.bpo-32969.rGTKa0.rst +++ /dev/null @@ -1,2 +0,0 @@ -Expose several missing constants in zlib and fix corresponding -documentation. diff --git a/Misc/NEWS.d/next/Library/2018-03-06-11-54-59.bpo-33009.-Ekysb.rst b/Misc/NEWS.d/next/Library/2018-03-06-11-54-59.bpo-33009.-Ekysb.rst deleted file mode 100644 index 96bc70a8c944..000000000000 --- a/Misc/NEWS.d/next/Library/2018-03-06-11-54-59.bpo-33009.-Ekysb.rst +++ /dev/null @@ -1 +0,0 @@ -Fix inspect.signature() for single-parameter partialmethods. diff --git a/Misc/NEWS.d/next/Library/2018-03-06-20-30-20.bpo-32999.lgFXWl.rst b/Misc/NEWS.d/next/Library/2018-03-06-20-30-20.bpo-32999.lgFXWl.rst deleted file mode 100644 index 45e75f939310..000000000000 --- a/Misc/NEWS.d/next/Library/2018-03-06-20-30-20.bpo-32999.lgFXWl.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix C implemetation of ``ABC.__subclasscheck__(cls, subclass)`` crashed when -``subclass`` is not a type object. diff --git a/Misc/NEWS.d/next/Library/2018-03-07-22-28-17.bpo-27683.572Rv4.rst b/Misc/NEWS.d/next/Library/2018-03-07-22-28-17.bpo-27683.572Rv4.rst deleted file mode 100644 index 4e6dfa8e978c..000000000000 --- a/Misc/NEWS.d/next/Library/2018-03-07-22-28-17.bpo-27683.572Rv4.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix a regression in :mod:`ipaddress` that result of :meth:`hosts` -is empty when the network is constructed by a tuple containing an -integer mask and only 1 bit left for addresses. diff --git a/Misc/NEWS.d/next/Library/2018-03-09-23-07-07.bpo-33037.nAJ3at.rst b/Misc/NEWS.d/next/Library/2018-03-09-23-07-07.bpo-33037.nAJ3at.rst deleted file mode 100644 index 2732eeb4534b..000000000000 --- a/Misc/NEWS.d/next/Library/2018-03-09-23-07-07.bpo-33037.nAJ3at.rst +++ /dev/null @@ -1 +0,0 @@ -Skip sending/receiving data after SSL transport closing. diff --git a/Misc/NEWS.d/next/Library/2018-03-11-19-03-52.bpo-31804.i8KUMp.rst b/Misc/NEWS.d/next/Library/2018-03-11-19-03-52.bpo-31804.i8KUMp.rst deleted file mode 100644 index 7fcede297aca..000000000000 --- a/Misc/NEWS.d/next/Library/2018-03-11-19-03-52.bpo-31804.i8KUMp.rst +++ /dev/null @@ -1,2 +0,0 @@ -Avoid failing in multiprocessing.Process if the standard streams are closed -or None at exit. diff --git a/Misc/NEWS.d/next/Library/2018-03-12-00-27-56.bpo-33021.m19B9T.rst b/Misc/NEWS.d/next/Library/2018-03-12-00-27-56.bpo-33021.m19B9T.rst deleted file mode 100644 index 50dfafe600d8..000000000000 --- a/Misc/NEWS.d/next/Library/2018-03-12-00-27-56.bpo-33021.m19B9T.rst +++ /dev/null @@ -1,2 +0,0 @@ -Release the GIL during fstat() calls, avoiding hang of all threads when -calling mmap.mmap(), os.urandom(), and random.seed(). Patch by Nir Soffer. diff --git a/Misc/NEWS.d/next/Library/2018-03-12-16-40-00.bpo-33056.lNN9Eh.rst b/Misc/NEWS.d/next/Library/2018-03-12-16-40-00.bpo-33056.lNN9Eh.rst deleted file mode 100644 index 6acc19a36dc8..000000000000 --- a/Misc/NEWS.d/next/Library/2018-03-12-16-40-00.bpo-33056.lNN9Eh.rst +++ /dev/null @@ -1 +0,0 @@ -FIX properly close leaking fds in concurrent.futures.ProcessPoolExecutor. diff --git a/Misc/NEWS.d/next/Library/2018-03-12-19-58-25.bpo-33064.LO2KIY.rst b/Misc/NEWS.d/next/Library/2018-03-12-19-58-25.bpo-33064.LO2KIY.rst deleted file mode 100644 index c8e955e335cb..000000000000 --- a/Misc/NEWS.d/next/Library/2018-03-12-19-58-25.bpo-33064.LO2KIY.rst +++ /dev/null @@ -1,2 +0,0 @@ -lib2to3 now properly supports trailing commas after ``*args`` and -``**kwargs`` in function signatures. diff --git a/Misc/NEWS.d/next/Library/2018-03-15-07-38-00.bpo-33078.RmjUF5.rst b/Misc/NEWS.d/next/Library/2018-03-15-07-38-00.bpo-33078.RmjUF5.rst deleted file mode 100644 index 55c2b1de8668..000000000000 --- a/Misc/NEWS.d/next/Library/2018-03-15-07-38-00.bpo-33078.RmjUF5.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix the size handling in multiprocessing.Queue when a pickling error -occurs. diff --git a/Misc/NEWS.d/next/Library/2018-03-16-16-07-33.bpo-33061.TRTTek.rst b/Misc/NEWS.d/next/Library/2018-03-16-16-07-33.bpo-33061.TRTTek.rst deleted file mode 100644 index b82ffb73a143..000000000000 --- a/Misc/NEWS.d/next/Library/2018-03-16-16-07-33.bpo-33061.TRTTek.rst +++ /dev/null @@ -1 +0,0 @@ -Add missing ``NoReturn`` to ``__all__`` in typing.py diff --git a/Misc/NEWS.d/next/Library/2018-03-18-17-38-48.bpo-32953.t8WAWN.rst b/Misc/NEWS.d/next/Library/2018-03-18-17-38-48.bpo-32953.t8WAWN.rst deleted file mode 100644 index 03c1162c7833..000000000000 --- a/Misc/NEWS.d/next/Library/2018-03-18-17-38-48.bpo-32953.t8WAWN.rst +++ /dev/null @@ -1,4 +0,0 @@ -If a non-dataclass inherits from a frozen dataclass, allow attributes to be -added to the derived class. Only attributes from the frozen dataclass -cannot be assigned to. Require all dataclasses in a hierarchy to be either -all frozen or all non-frozen. diff --git a/Misc/NEWS.d/next/Library/2018-03-19-20-47-00.bpo-33100.chyIO4.rst b/Misc/NEWS.d/next/Library/2018-03-19-20-47-00.bpo-33100.chyIO4.rst deleted file mode 100644 index 080a55c0cfb7..000000000000 --- a/Misc/NEWS.d/next/Library/2018-03-19-20-47-00.bpo-33100.chyIO4.rst +++ /dev/null @@ -1,2 +0,0 @@ -Dataclasses: If a field has a default value that's a MemberDescriptorType, -then it's from that field being in __slots__, not an actual default value. diff --git a/Misc/NEWS.d/next/Library/2018-03-20-20-53-21.bpo-32896.ewW3Ln.rst b/Misc/NEWS.d/next/Library/2018-03-20-20-53-21.bpo-32896.ewW3Ln.rst deleted file mode 100644 index 8363da4667a1..000000000000 --- a/Misc/NEWS.d/next/Library/2018-03-20-20-53-21.bpo-32896.ewW3Ln.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix an error where subclassing a dataclass with a field that uses a -default_factory would generate an incorrect class. diff --git a/Misc/NEWS.d/next/Library/2018-03-21-16-52-26.bpo-33116.Tvzerj.rst b/Misc/NEWS.d/next/Library/2018-03-21-16-52-26.bpo-33116.Tvzerj.rst deleted file mode 100644 index 90072d8e3030..000000000000 --- a/Misc/NEWS.d/next/Library/2018-03-21-16-52-26.bpo-33116.Tvzerj.rst +++ /dev/null @@ -1 +0,0 @@ -Add 'Field' to dataclasses.__all__. diff --git a/Misc/NEWS.d/next/Library/2018-03-21-17-59-39.bpo-33078.PQOniT.rst b/Misc/NEWS.d/next/Library/2018-03-21-17-59-39.bpo-33078.PQOniT.rst deleted file mode 100644 index 8b71bb32e0ec..000000000000 --- a/Misc/NEWS.d/next/Library/2018-03-21-17-59-39.bpo-33078.PQOniT.rst +++ /dev/null @@ -1 +0,0 @@ -Fix the failure on OSX caused by the tests relying on sem_getvalue diff --git a/Misc/NEWS.d/next/Library/2018-03-22-16-05-56.bpo-32505.YK1N8v.rst b/Misc/NEWS.d/next/Library/2018-03-22-16-05-56.bpo-32505.YK1N8v.rst deleted file mode 100644 index 91e97bf53f65..000000000000 --- a/Misc/NEWS.d/next/Library/2018-03-22-16-05-56.bpo-32505.YK1N8v.rst +++ /dev/null @@ -1,2 +0,0 @@ -Raise TypeError if a member variable of a dataclass is of type Field, but -doesn't have a type annotation. diff --git a/Misc/NEWS.d/next/Library/2018-03-24-15-08-24.bpo-33127.olJmHv.rst b/Misc/NEWS.d/next/Library/2018-03-24-15-08-24.bpo-33127.olJmHv.rst deleted file mode 100644 index 635aabbde031..000000000000 --- a/Misc/NEWS.d/next/Library/2018-03-24-15-08-24.bpo-33127.olJmHv.rst +++ /dev/null @@ -1 +0,0 @@ -The ssl module now compiles with LibreSSL 2.7.1. diff --git a/Misc/NEWS.d/next/Library/2018-03-24-19-34-26.bpo-33134.hbVeIX.rst b/Misc/NEWS.d/next/Library/2018-03-24-19-34-26.bpo-33134.hbVeIX.rst deleted file mode 100644 index 3f4ce84bef76..000000000000 --- a/Misc/NEWS.d/next/Library/2018-03-24-19-34-26.bpo-33134.hbVeIX.rst +++ /dev/null @@ -1,3 +0,0 @@ -When computing dataclass's __hash__, use the lookup table to contain the -function which returns the __hash__ value. This is an improvement over -looking up a string, and then testing that string to see what to do. diff --git a/Misc/NEWS.d/next/Library/2018-03-24-19-54-48.bpo-32873.cHyoAm.rst b/Misc/NEWS.d/next/Library/2018-03-24-19-54-48.bpo-32873.cHyoAm.rst deleted file mode 100644 index 99f8088cf138..000000000000 --- a/Misc/NEWS.d/next/Library/2018-03-24-19-54-48.bpo-32873.cHyoAm.rst +++ /dev/null @@ -1,3 +0,0 @@ -Treat type variables and special typing forms as immutable by copy and -pickle. This fixes several minor issues and inconsistencies, and improves -backwards compatibility with Python 3.6. diff --git a/Misc/NEWS.d/next/Library/2018-03-25-13-18-16.bpo-33096.ofdbe7.rst b/Misc/NEWS.d/next/Library/2018-03-25-13-18-16.bpo-33096.ofdbe7.rst deleted file mode 100644 index c55ea20b337d..000000000000 --- a/Misc/NEWS.d/next/Library/2018-03-25-13-18-16.bpo-33096.ofdbe7.rst +++ /dev/null @@ -1,4 +0,0 @@ -Allow ttk.Treeview.insert to insert iid that has a false boolean value. -Note iid=0 and iid=False would be same. -Patch by Garvit Khatri. - diff --git a/Misc/NEWS.d/next/Library/2018-03-26-12-33-13.bpo-33141.23wlxf.rst b/Misc/NEWS.d/next/Library/2018-03-26-12-33-13.bpo-33141.23wlxf.rst deleted file mode 100644 index 1d49c08fed54..000000000000 --- a/Misc/NEWS.d/next/Library/2018-03-26-12-33-13.bpo-33141.23wlxf.rst +++ /dev/null @@ -1,2 +0,0 @@ -Have Field objects pass through __set_name__ to their default values, if -they have their own __set_name__. diff --git a/Misc/NEWS.d/next/Security/2018-03-02-10-24-52.bpo-32981.O_qDyj.rst b/Misc/NEWS.d/next/Security/2018-03-02-10-24-52.bpo-32981.O_qDyj.rst deleted file mode 100644 index 9ebabb44f91e..000000000000 --- a/Misc/NEWS.d/next/Security/2018-03-02-10-24-52.bpo-32981.O_qDyj.rst +++ /dev/null @@ -1,4 +0,0 @@ -Regexes in difflib and poplib were vulnerable to catastrophic backtracking. -These regexes formed potential DOS vectors (REDOS). They have been -refactored. This resolves CVE-2018-1060 and CVE-2018-1061. -Patch by Jamie Davis. diff --git a/Misc/NEWS.d/next/Security/2018-03-05-10-09-51.bpo-33001.elj4Aa.rst b/Misc/NEWS.d/next/Security/2018-03-05-10-09-51.bpo-33001.elj4Aa.rst deleted file mode 100644 index 2acbac9e1af6..000000000000 --- a/Misc/NEWS.d/next/Security/2018-03-05-10-09-51.bpo-33001.elj4Aa.rst +++ /dev/null @@ -1 +0,0 @@ -Minimal fix to prevent buffer overrun in os.symlink on Windows diff --git a/Misc/NEWS.d/next/Security/2018-03-25-12-05-43.bpo-33136.TzSN4x.rst b/Misc/NEWS.d/next/Security/2018-03-25-12-05-43.bpo-33136.TzSN4x.rst deleted file mode 100644 index c3505167092b..000000000000 --- a/Misc/NEWS.d/next/Security/2018-03-25-12-05-43.bpo-33136.TzSN4x.rst +++ /dev/null @@ -1,3 +0,0 @@ -Harden ssl module against LibreSSL CVE-2018-8970. -X509_VERIFY_PARAM_set1_host() is called with an explicit namelen. A new test -ensures that NULL bytes are not allowed. diff --git a/Misc/NEWS.d/next/Tests/2018-01-08-13-33-47.bpo-19417.2asoXy.rst b/Misc/NEWS.d/next/Tests/2018-01-08-13-33-47.bpo-19417.2asoXy.rst deleted file mode 100644 index 739352fcdd67..000000000000 --- a/Misc/NEWS.d/next/Tests/2018-01-08-13-33-47.bpo-19417.2asoXy.rst +++ /dev/null @@ -1 +0,0 @@ -Add test_bdb.py. diff --git a/Misc/NEWS.d/next/Tests/2018-03-09-07-05-12.bpo-32517.ugc1iW.rst b/Misc/NEWS.d/next/Tests/2018-03-09-07-05-12.bpo-32517.ugc1iW.rst deleted file mode 100644 index 43f148f06ecb..000000000000 --- a/Misc/NEWS.d/next/Tests/2018-03-09-07-05-12.bpo-32517.ugc1iW.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix failing ``test_asyncio`` on macOS 10.12.2+ due to transport of -``KqueueSelector`` loop was not being closed. diff --git a/Misc/NEWS.d/next/Tests/2018-03-28-01-35-02.bpo-32872.J5NDUj.rst b/Misc/NEWS.d/next/Tests/2018-03-28-01-35-02.bpo-32872.J5NDUj.rst deleted file mode 100644 index 06d656bbfd64..000000000000 --- a/Misc/NEWS.d/next/Tests/2018-03-28-01-35-02.bpo-32872.J5NDUj.rst +++ /dev/null @@ -1 +0,0 @@ -Avoid regrtest compatibility issue with namespace packages. diff --git a/Misc/NEWS.d/next/Tools-Demos/2018-02-20-12-16-47.bpo-32885.dL5x7C.rst b/Misc/NEWS.d/next/Tools-Demos/2018-02-20-12-16-47.bpo-32885.dL5x7C.rst deleted file mode 100644 index e003e1d84fd0..000000000000 --- a/Misc/NEWS.d/next/Tools-Demos/2018-02-20-12-16-47.bpo-32885.dL5x7C.rst +++ /dev/null @@ -1,2 +0,0 @@ -Add an ``-n`` flag for ``Tools/scripts/pathfix.py`` to disbale automatic -backup creation (files with ``~`` suffix). diff --git a/Misc/NEWS.d/next/Windows/2018-02-28-11-03-24.bpo-32903.1SXY4t.rst b/Misc/NEWS.d/next/Windows/2018-02-28-11-03-24.bpo-32903.1SXY4t.rst deleted file mode 100644 index a20a414790f8..000000000000 --- a/Misc/NEWS.d/next/Windows/2018-02-28-11-03-24.bpo-32903.1SXY4t.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix a memory leak in os.chdir() on Windows if the current directory is set -to a UNC path. diff --git a/Misc/NEWS.d/next/Windows/2018-03-07-01-33-33.bpo-33016.Z_Med0.rst b/Misc/NEWS.d/next/Windows/2018-03-07-01-33-33.bpo-33016.Z_Med0.rst deleted file mode 100644 index f4f78d489bf1..000000000000 --- a/Misc/NEWS.d/next/Windows/2018-03-07-01-33-33.bpo-33016.Z_Med0.rst +++ /dev/null @@ -1 +0,0 @@ -Fix potential use of uninitialized memory in nt._getfinalpathname diff --git a/Misc/NEWS.d/next/macOS/2018-03-29-06-56-12.bpo-32726.urS9uX.rst b/Misc/NEWS.d/next/macOS/2018-03-29-06-56-12.bpo-32726.urS9uX.rst deleted file mode 100644 index 470dc7f3eb12..000000000000 --- a/Misc/NEWS.d/next/macOS/2018-03-29-06-56-12.bpo-32726.urS9uX.rst +++ /dev/null @@ -1,5 +0,0 @@ -Build and link with private copy of Tcl/Tk 8.6 for the macOS 10.6+ -installer. The 10.9+ installer variant already does this. This means that -the Python 3.7 provided by the python.org macOS installers no longer need or -use any external versions of Tcl/Tk, either system-provided or user- -installed, such as ActiveTcl. diff --git a/README.rst b/README.rst index d63176088731..a01b069263e0 100644 --- a/README.rst +++ b/README.rst @@ -1,5 +1,5 @@ -This is Python version 3.7.0 beta 2+ -==================================== +This is Python version 3.7.0 beta 3 +=================================== .. image:: https://travis-ci.org/python/cpython.svg?branch=master :alt: CPython build status on Travis CI From webhook-mailer at python.org Thu Mar 29 08:57:41 2018 From: webhook-mailer at python.org (Ned Deily) Date: Thu, 29 Mar 2018 12:57:41 -0000 Subject: [Python-checkins] bump to 3.7.0b3+ Message-ID: <mailman.5.1522328263.3857.python-checkins@python.org> https://github.com/python/cpython/commit/e557b5c479bd7ffca36f0c1b300e8c795c7a8ab4 commit: e557b5c479bd7ffca36f0c1b300e8c795c7a8ab4 branch: 3.7 author: Ned Deily <nad at python.org> committer: Ned Deily <nad at python.org> date: 2018-03-29T08:57:23-04:00 summary: bump to 3.7.0b3+ files: M Include/patchlevel.h M README.rst diff --git a/Include/patchlevel.h b/Include/patchlevel.h index e290cd91b599..70d86c9f8529 100644 --- a/Include/patchlevel.h +++ b/Include/patchlevel.h @@ -23,7 +23,7 @@ #define PY_RELEASE_SERIAL 3 /* Version as a string */ -#define PY_VERSION "3.7.0b3" +#define PY_VERSION "3.7.0b3+" /*--end constants--*/ /* Version as a single 4-byte hex number, e.g. 0x010502B2 == 1.5.2b2. diff --git a/README.rst b/README.rst index a01b069263e0..a795bfece826 100644 --- a/README.rst +++ b/README.rst @@ -1,5 +1,5 @@ -This is Python version 3.7.0 beta 3 -=================================== +This is Python version 3.7.0 beta 3+ +==================================== .. image:: https://travis-ci.org/python/cpython.svg?branch=master :alt: CPython build status on Travis CI From webhook-mailer at python.org Thu Mar 29 11:07:54 2018 From: webhook-mailer at python.org (Eric V. Smith) Date: Thu, 29 Mar 2018 15:07:54 -0000 Subject: [Python-checkins] bpo-33175: dataclasses should look up __set_name__ on class, not instance (GH-6305) Message-ID: <mailman.6.1522336076.3857.python-checkins@python.org> https://github.com/python/cpython/commit/521995205a2cb6b504fe0e39af22a81f785350a3 commit: 521995205a2cb6b504fe0e39af22a81f785350a3 branch: master author: Eric V. Smith <ericvsmith at users.noreply.github.com> committer: GitHub <noreply at github.com> date: 2018-03-29T11:07:48-04:00 summary: bpo-33175: dataclasses should look up __set_name__ on class, not instance (GH-6305) files: A Misc/NEWS.d/next/Library/2018-03-29-04-32-25.bpo-33175._zs1yM.rst M Lib/dataclasses.py M Lib/test/test_dataclasses.py diff --git a/Lib/dataclasses.py b/Lib/dataclasses.py index 8c197fe73904..bd7252c683ca 100644 --- a/Lib/dataclasses.py +++ b/Lib/dataclasses.py @@ -248,11 +248,11 @@ def __repr__(self): # the default value, so the end result is a descriptor that had # __set_name__ called on it at the right time. def __set_name__(self, owner, name): - func = getattr(self.default, '__set_name__', None) + func = getattr(type(self.default), '__set_name__', None) if func: # There is a __set_name__ method on the descriptor, # call it. - func(owner, name) + func(self.default, owner, name) class _DataclassParams: diff --git a/Lib/test/test_dataclasses.py b/Lib/test/test_dataclasses.py index 2745eaf6893b..5cd424cf5760 100755 --- a/Lib/test/test_dataclasses.py +++ b/Lib/test/test_dataclasses.py @@ -2705,7 +2705,7 @@ def test_set_name(self): # Create a descriptor. class D: def __set_name__(self, owner, name): - self.name = name + self.name = name + 'x' def __get__(self, instance, owner): if instance is not None: return 1 @@ -2716,7 +2716,7 @@ def __get__(self, instance, owner): @dataclass class C: c: int=D() - self.assertEqual(C.c.name, 'c') + self.assertEqual(C.c.name, 'cx') # Now test with a default value and init=False, which is the # only time this is really meaningful. If not using @@ -2724,7 +2724,7 @@ class C: @dataclass class C: c: int=field(default=D(), init=False) - self.assertEqual(C.c.name, 'c') + self.assertEqual(C.c.name, 'cx') self.assertEqual(C().c, 1) def test_non_descriptor(self): @@ -2733,12 +2733,41 @@ def test_non_descriptor(self): class D: def __set_name__(self, owner, name): - self.name = name + self.name = name + 'x' @dataclass class C: c: int=field(default=D(), init=False) - self.assertEqual(C.c.name, 'c') + self.assertEqual(C.c.name, 'cx') + + def test_lookup_on_instance(self): + # See bpo-33175. + class D: + pass + + d = D() + # Create an attribute on the instance, not type. + d.__set_name__ = Mock() + + # Make sure d.__set_name__ is not called. + @dataclass + class C: + i: int=field(default=d, init=False) + + self.assertEqual(d.__set_name__.call_count, 0) + + def test_lookup_on_class(self): + # See bpo-33175. + class D: + pass + D.__set_name__ = Mock() + + # Make sure D.__set_name__ is called. + @dataclass + class C: + i: int=field(default=D(), init=False) + + self.assertEqual(D.__set_name__.call_count, 1) if __name__ == '__main__': diff --git a/Misc/NEWS.d/next/Library/2018-03-29-04-32-25.bpo-33175._zs1yM.rst b/Misc/NEWS.d/next/Library/2018-03-29-04-32-25.bpo-33175._zs1yM.rst new file mode 100644 index 000000000000..c872499cd679 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-03-29-04-32-25.bpo-33175._zs1yM.rst @@ -0,0 +1,2 @@ +In dataclasses, Field.__set_name__ now looks up the __set_name__ special +method on the class, not the instance, of the default value. From webhook-mailer at python.org Thu Mar 29 11:32:40 2018 From: webhook-mailer at python.org (Miss Islington (bot)) Date: Thu, 29 Mar 2018 15:32:40 -0000 Subject: [Python-checkins] bpo-33175: dataclasses should look up __set_name__ on class, not instance (GH-6305) Message-ID: <mailman.7.1522337561.3857.python-checkins@python.org> https://github.com/python/cpython/commit/faa6f5c74ca56d4cc0dd9db78d5c8e864b617d0c commit: faa6f5c74ca56d4cc0dd9db78d5c8e864b617d0c branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: GitHub <noreply at github.com> date: 2018-03-29T08:32:36-07:00 summary: bpo-33175: dataclasses should look up __set_name__ on class, not instance (GH-6305) (cherry picked from commit 521995205a2cb6b504fe0e39af22a81f785350a3) Co-authored-by: Eric V. Smith <ericvsmith at users.noreply.github.com> files: A Misc/NEWS.d/next/Library/2018-03-29-04-32-25.bpo-33175._zs1yM.rst M Lib/dataclasses.py M Lib/test/test_dataclasses.py diff --git a/Lib/dataclasses.py b/Lib/dataclasses.py index 8c197fe73904..bd7252c683ca 100644 --- a/Lib/dataclasses.py +++ b/Lib/dataclasses.py @@ -248,11 +248,11 @@ def __repr__(self): # the default value, so the end result is a descriptor that had # __set_name__ called on it at the right time. def __set_name__(self, owner, name): - func = getattr(self.default, '__set_name__', None) + func = getattr(type(self.default), '__set_name__', None) if func: # There is a __set_name__ method on the descriptor, # call it. - func(owner, name) + func(self.default, owner, name) class _DataclassParams: diff --git a/Lib/test/test_dataclasses.py b/Lib/test/test_dataclasses.py index 2745eaf6893b..5cd424cf5760 100755 --- a/Lib/test/test_dataclasses.py +++ b/Lib/test/test_dataclasses.py @@ -2705,7 +2705,7 @@ def test_set_name(self): # Create a descriptor. class D: def __set_name__(self, owner, name): - self.name = name + self.name = name + 'x' def __get__(self, instance, owner): if instance is not None: return 1 @@ -2716,7 +2716,7 @@ def __get__(self, instance, owner): @dataclass class C: c: int=D() - self.assertEqual(C.c.name, 'c') + self.assertEqual(C.c.name, 'cx') # Now test with a default value and init=False, which is the # only time this is really meaningful. If not using @@ -2724,7 +2724,7 @@ class C: @dataclass class C: c: int=field(default=D(), init=False) - self.assertEqual(C.c.name, 'c') + self.assertEqual(C.c.name, 'cx') self.assertEqual(C().c, 1) def test_non_descriptor(self): @@ -2733,12 +2733,41 @@ def test_non_descriptor(self): class D: def __set_name__(self, owner, name): - self.name = name + self.name = name + 'x' @dataclass class C: c: int=field(default=D(), init=False) - self.assertEqual(C.c.name, 'c') + self.assertEqual(C.c.name, 'cx') + + def test_lookup_on_instance(self): + # See bpo-33175. + class D: + pass + + d = D() + # Create an attribute on the instance, not type. + d.__set_name__ = Mock() + + # Make sure d.__set_name__ is not called. + @dataclass + class C: + i: int=field(default=d, init=False) + + self.assertEqual(d.__set_name__.call_count, 0) + + def test_lookup_on_class(self): + # See bpo-33175. + class D: + pass + D.__set_name__ = Mock() + + # Make sure D.__set_name__ is called. + @dataclass + class C: + i: int=field(default=D(), init=False) + + self.assertEqual(D.__set_name__.call_count, 1) if __name__ == '__main__': diff --git a/Misc/NEWS.d/next/Library/2018-03-29-04-32-25.bpo-33175._zs1yM.rst b/Misc/NEWS.d/next/Library/2018-03-29-04-32-25.bpo-33175._zs1yM.rst new file mode 100644 index 000000000000..c872499cd679 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-03-29-04-32-25.bpo-33175._zs1yM.rst @@ -0,0 +1,2 @@ +In dataclasses, Field.__set_name__ now looks up the __set_name__ special +method on the class, not the instance, of the default value. From webhook-mailer at python.org Fri Mar 30 01:36:52 2018 From: webhook-mailer at python.org (Nick Coghlan) Date: Fri, 30 Mar 2018 05:36:52 -0000 Subject: [Python-checkins] bpo-33182: Fix pointer types in _testembed (GH-6310) Message-ID: <mailman.8.1522388214.3857.python-checkins@python.org> https://github.com/python/cpython/commit/69f5c73311a61b05485b19626935bf240ad31c5b commit: 69f5c73311a61b05485b19626935bf240ad31c5b branch: master author: Nick Coghlan <ncoghlan at gmail.com> committer: GitHub <noreply at github.com> date: 2018-03-30T15:36:42+10:00 summary: bpo-33182: Fix pointer types in _testembed (GH-6310) files: A Misc/NEWS.d/next/Build/2018-03-30-14-55-48.bpo-33182.CePczb.rst M Programs/_testembed.c diff --git a/Misc/NEWS.d/next/Build/2018-03-30-14-55-48.bpo-33182.CePczb.rst b/Misc/NEWS.d/next/Build/2018-03-30-14-55-48.bpo-33182.CePczb.rst new file mode 100644 index 000000000000..6310e5d5b557 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2018-03-30-14-55-48.bpo-33182.CePczb.rst @@ -0,0 +1 @@ +The embedding tests can once again be built with clang 6.0 diff --git a/Programs/_testembed.c b/Programs/_testembed.c index 09b7bf6d94fc..7406470ae65c 100644 --- a/Programs/_testembed.c +++ b/Programs/_testembed.c @@ -166,16 +166,18 @@ static int test_pre_initialization_api(void) /* bpo-33042: Ensure embedding apps can predefine sys module options */ static int test_pre_initialization_sys_options(void) { - /* We allocate a couple of the option dynamically, and then delete + /* We allocate a couple of the options dynamically, and then delete * them before calling Py_Initialize. This ensures the interpreter isn't * relying on the caller to keep the passed in strings alive. */ - wchar_t *static_warnoption = L"once"; - wchar_t *static_xoption = L"also_not_an_option=2"; + const wchar_t *static_warnoption = L"once"; + const wchar_t *static_xoption = L"also_not_an_option=2"; size_t warnoption_len = wcslen(static_warnoption); size_t xoption_len = wcslen(static_xoption); - wchar_t *dynamic_once_warnoption = calloc(warnoption_len+1, sizeof(wchar_t)); - wchar_t *dynamic_xoption = calloc(xoption_len+1, sizeof(wchar_t)); + wchar_t *dynamic_once_warnoption = \ + (wchar_t *) calloc(warnoption_len+1, sizeof(wchar_t)); + wchar_t *dynamic_xoption = \ + (wchar_t *) calloc(xoption_len+1, sizeof(wchar_t)); wcsncpy(dynamic_once_warnoption, static_warnoption, warnoption_len+1); wcsncpy(dynamic_xoption, static_xoption, xoption_len+1); From webhook-mailer at python.org Fri Mar 30 02:24:09 2018 From: webhook-mailer at python.org (Nick Coghlan) Date: Fri, 30 Mar 2018 06:24:09 -0000 Subject: [Python-checkins] bpo-33182: Fix pointer types in _testembed (GH-6310) (GH-6311) Message-ID: <mailman.9.1522391051.3857.python-checkins@python.org> https://github.com/python/cpython/commit/2961717986201d639b60de51e5f2e9aa2573856c commit: 2961717986201d639b60de51e5f2e9aa2573856c branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Nick Coghlan <ncoghlan at gmail.com> date: 2018-03-30T16:24:03+10:00 summary: bpo-33182: Fix pointer types in _testembed (GH-6310) (GH-6311) (cherry picked from commit 69f5c73311a61b05485b19626935bf240ad31c5b) Co-authored-by: Nick Coghlan <ncoghlan at gmail.com> files: A Misc/NEWS.d/next/Build/2018-03-30-14-55-48.bpo-33182.CePczb.rst M Programs/_testembed.c diff --git a/Misc/NEWS.d/next/Build/2018-03-30-14-55-48.bpo-33182.CePczb.rst b/Misc/NEWS.d/next/Build/2018-03-30-14-55-48.bpo-33182.CePczb.rst new file mode 100644 index 000000000000..6310e5d5b557 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2018-03-30-14-55-48.bpo-33182.CePczb.rst @@ -0,0 +1 @@ +The embedding tests can once again be built with clang 6.0 diff --git a/Programs/_testembed.c b/Programs/_testembed.c index 09b7bf6d94fc..7406470ae65c 100644 --- a/Programs/_testembed.c +++ b/Programs/_testembed.c @@ -166,16 +166,18 @@ static int test_pre_initialization_api(void) /* bpo-33042: Ensure embedding apps can predefine sys module options */ static int test_pre_initialization_sys_options(void) { - /* We allocate a couple of the option dynamically, and then delete + /* We allocate a couple of the options dynamically, and then delete * them before calling Py_Initialize. This ensures the interpreter isn't * relying on the caller to keep the passed in strings alive. */ - wchar_t *static_warnoption = L"once"; - wchar_t *static_xoption = L"also_not_an_option=2"; + const wchar_t *static_warnoption = L"once"; + const wchar_t *static_xoption = L"also_not_an_option=2"; size_t warnoption_len = wcslen(static_warnoption); size_t xoption_len = wcslen(static_xoption); - wchar_t *dynamic_once_warnoption = calloc(warnoption_len+1, sizeof(wchar_t)); - wchar_t *dynamic_xoption = calloc(xoption_len+1, sizeof(wchar_t)); + wchar_t *dynamic_once_warnoption = \ + (wchar_t *) calloc(warnoption_len+1, sizeof(wchar_t)); + wchar_t *dynamic_xoption = \ + (wchar_t *) calloc(xoption_len+1, sizeof(wchar_t)); wcsncpy(dynamic_once_warnoption, static_warnoption, warnoption_len+1); wcsncpy(dynamic_xoption, static_xoption, xoption_len+1); From webhook-mailer at python.org Fri Mar 30 03:36:15 2018 From: webhook-mailer at python.org (Xiang Zhang) Date: Fri, 30 Mar 2018 07:36:15 -0000 Subject: [Python-checkins] Fix socket type in DatagramHandler documentation: TCP -> UDP (GH-6272) Message-ID: <mailman.10.1522395375.3857.python-checkins@python.org> https://github.com/python/cpython/commit/233de021d915364bd3daee921d1d96d50d46d7fe commit: 233de021d915364bd3daee921d1d96d50d46d7fe branch: master author: Mike DePalatis <mike at depalatis.net> committer: Xiang Zhang <angwerzx at 126.com> date: 2018-03-30T15:36:06+08:00 summary: Fix socket type in DatagramHandler documentation: TCP -> UDP (GH-6272) files: M Doc/library/logging.handlers.rst diff --git a/Doc/library/logging.handlers.rst b/Doc/library/logging.handlers.rst index 0974286e55dc..f7262e60a31b 100644 --- a/Doc/library/logging.handlers.rst +++ b/Doc/library/logging.handlers.rst @@ -524,7 +524,7 @@ over UDP sockets. .. versionchanged:: 3.4 If ``port`` is specified as ``None``, a Unix domain socket is created - using the value in ``host`` - otherwise, a TCP socket is created. + using the value in ``host`` - otherwise, a UDP socket is created. .. method:: emit() From webhook-mailer at python.org Fri Mar 30 03:48:48 2018 From: webhook-mailer at python.org (Miss Islington (bot)) Date: Fri, 30 Mar 2018 07:48:48 -0000 Subject: [Python-checkins] Fix socket type in DatagramHandler documentation: TCP -> UDP (GH-6272) Message-ID: <mailman.11.1522396129.3857.python-checkins@python.org> https://github.com/python/cpython/commit/7dcfd6c66344594844d441e197174366ff9cfe4e commit: 7dcfd6c66344594844d441e197174366ff9cfe4e branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: GitHub <noreply at github.com> date: 2018-03-30T00:48:45-07:00 summary: Fix socket type in DatagramHandler documentation: TCP -> UDP (GH-6272) (cherry picked from commit 233de021d915364bd3daee921d1d96d50d46d7fe) Co-authored-by: Mike DePalatis <mike at depalatis.net> files: M Doc/library/logging.handlers.rst diff --git a/Doc/library/logging.handlers.rst b/Doc/library/logging.handlers.rst index 0974286e55dc..f7262e60a31b 100644 --- a/Doc/library/logging.handlers.rst +++ b/Doc/library/logging.handlers.rst @@ -524,7 +524,7 @@ over UDP sockets. .. versionchanged:: 3.4 If ``port`` is specified as ``None``, a Unix domain socket is created - using the value in ``host`` - otherwise, a TCP socket is created. + using the value in ``host`` - otherwise, a UDP socket is created. .. method:: emit() From webhook-mailer at python.org Fri Mar 30 03:53:38 2018 From: webhook-mailer at python.org (Miss Islington (bot)) Date: Fri, 30 Mar 2018 07:53:38 -0000 Subject: [Python-checkins] Fix socket type in DatagramHandler documentation: TCP -> UDP (GH-6272) Message-ID: <mailman.12.1522396419.3857.python-checkins@python.org> https://github.com/python/cpython/commit/ddc99713d9da6b991933c0ee0e0829fee860d6ec commit: ddc99713d9da6b991933c0ee0e0829fee860d6ec branch: 3.6 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: GitHub <noreply at github.com> date: 2018-03-30T00:53:35-07:00 summary: Fix socket type in DatagramHandler documentation: TCP -> UDP (GH-6272) (cherry picked from commit 233de021d915364bd3daee921d1d96d50d46d7fe) Co-authored-by: Mike DePalatis <mike at depalatis.net> files: M Doc/library/logging.handlers.rst diff --git a/Doc/library/logging.handlers.rst b/Doc/library/logging.handlers.rst index f13f765c0193..6bb7b13af813 100644 --- a/Doc/library/logging.handlers.rst +++ b/Doc/library/logging.handlers.rst @@ -511,7 +511,7 @@ over UDP sockets. .. versionchanged:: 3.4 If ``port`` is specified as ``None``, a Unix domain socket is created - using the value in ``host`` - otherwise, a TCP socket is created. + using the value in ``host`` - otherwise, a UDP socket is created. .. method:: emit() From solipsis at pitrou.net Fri Mar 30 05:12:40 2018 From: solipsis at pitrou.net (solipsis at pitrou.net) Date: Fri, 30 Mar 2018 09:12:40 +0000 Subject: [Python-checkins] Daily reference leaks (4243df51fe43): sum=10 Message-ID: <20180330091240.1.2FECA31E614C01D8@psf.io> results for 4243df51fe43 on branch "default" -------------------------------------------- test_collections leaked [0, 0, 7] memory blocks, sum=7 test_functools leaked [0, 3, 1] memory blocks, sum=4 test_multiprocessing_forkserver leaked [1, 0, -2] memory blocks, sum=-1 Command line was: ['./python', '-m', 'test.regrtest', '-uall', '-R', '3:3:/home/psf-users/antoine/refleaks/reflogjeYRfn', '--timeout', '7200'] From lp_benchmark_robot at intel.com Fri Mar 30 23:25:22 2018 From: lp_benchmark_robot at intel.com (lp_benchmark_robot at intel.com) Date: Fri, 30 Mar 2018 20:25:22 -0700 Subject: [Python-checkins] [2 down, 7 up, 56 flat] Results for Python (master branch) 2018-03-30 Message-ID: <4cc8a947-3bd3-44d8-9e24-a42670d04c2a@orsmsx102.amr.corp.intel.com> Results for project python/master, build date: 2018-03-30 16:50:11-07:00. - commit: 233de02 - previous commit: 2812d3d - revision date: 2018-03-30 15:36:06+08:00 - environment: Broadwell-EP - cpu: Intel(R) Xeon(R) CPU E5-2699 v4 @ 2.20GHz 2x22 cores, stepping 1, LLC 55 MB - mem: 128 GB - os: Ubuntu 16.04.2 LTS - kernel: 4.4.0-62-generic x86_64 GNU/Linux Baseline results were generated using release v3.6.0, with hash 5c4568a from 2016-12-22 23:38:47+00:00. +-----+------------------------+--------+------------+------------+------------+ | | |relative|change since|change since|current rev | | | benchmark|std_dev*| last run | baseline |run with PGO| +-----+------------------------+--------+------------+------------+------------+ | :-| | 2to3| 0.895% | +1.418% | +8.963% | +6.933% | +-----+------------------------+--------+------------+------------+------------+ | :-) | call_method| 2.372% | +8.104% | +25.312% | +12.571% | +-----+------------------------+--------+------------+------------+------------+ | :-| | call_method_slots| 4.123% | +4.160% | +25.212% | +7.523% | +-----+------------------------+--------+------------+------------+------------+ | :-) | call_method_unknown| 2.010% | +7.793% | +24.553% | +10.898% | +-----+------------------------+--------+------------+------------+------------+ | :-| | call_simple| 4.345% | +2.510% | +8.746% | +12.268% | +-----+------------------------+--------+------------+------------+------------+ | :-| | chameleon| 2.206% | +0.151% | +10.781% | +13.300% | +-----+------------------------+--------+------------+------------+------------+ | :-| | chaos| 0.541% | +0.038% | +7.132% | +10.288% | +-----+------------------------+--------+------------+------------+------------+ | :-| | crypto_pyaes| 0.502% | +0.302% | +0.576% | +8.345% | +-----+------------------------+--------+------------+------------+------------+ | :-| | deltablue| 2.136% | +2.561% | +13.549% | +16.218% | +-----+------------------------+--------+------------+------------+------------+ | :-( | django_template| 4.372% | -10.922% | +12.701% | +18.601% | +-----+------------------------+--------+------------+------------+------------+ | :-| | dulwich_log| 1.478% | -1.597% | +4.326% | +8.557% | +-----+------------------------+--------+------------+------------+------------+ | :-| | fannkuch| 0.767% | +2.314% | +7.081% | +4.940% | +-----+------------------------+--------+------------+------------+------------+ | :-| | float| 0.989% | +1.301% | +3.776% | +6.993% | +-----+------------------------+--------+------------+------------+------------+ | :-| | genshi_text| 1.415% | +1.083% | +13.208% | +12.123% | +-----+------------------------+--------+------------+------------+------------+ | :-| | genshi_xml| 1.657% | +0.892% | +9.693% | +13.391% | +-----+------------------------+--------+------------+------------+------------+ | :-| | go| 7.461% | -2.468% | +3.902% | +12.338% | +-----+------------------------+--------+------------+------------+------------+ | :-| | hexiom| 0.918% | +0.874% | +11.685% | +11.810% | +-----+------------------------+--------+------------+------------+------------+ | :-| | html5lib| 2.805% | +1.880% | +12.103% | +9.976% | +-----+------------------------+--------+------------+------------+------------+ | :-| | json_dumps| 3.497% | +0.109% | +1.469% | +12.885% | +-----+------------------------+--------+------------+------------+------------+ | :-| | json_loads| 4.279% | +0.619% | -0.201% | +13.444% | +-----+------------------------+--------+------------+------------+------------+ | :-) | logging_format| 1.798% | +9.540% | +18.760% | +14.248% | +-----+------------------------+--------+------------+------------+------------+ | :-| | logging_silent| 2.919% | +2.101% | +47.961% | +14.059% | +-----+------------------------+--------+------------+------------+------------+ | :-| | logging_simple| 1.379% | +1.473% | +13.389% | +15.509% | +-----+------------------------+--------+------------+------------+------------+ | :-| | mako| 0.465% | -0.221% | +17.121% | +13.562% | +-----+------------------------+--------+------------+------------+------------+ | :-| | mdp| 1.297% | +1.648% | +9.084% | +11.309% | +-----+------------------------+--------+------------+------------+------------+ | :-| | meteor_contest| 0.914% | +2.348% | +4.647% | +6.368% | +-----+------------------------+--------+------------+------------+------------+ | :-| | nbody| 0.754% | -0.335% | -2.787% | +4.714% | +-----+------------------------+--------+------------+------------+------------+ | :-| | nqueens| 0.553% | +0.287% | +5.856% | +8.103% | +-----+------------------------+--------+------------+------------+------------+ | :-| | pathlib| 1.544% | -2.337% | +1.908% | +10.354% | +-----+------------------------+--------+------------+------------+------------+ | :-| | pickle| 1.120% | +2.341% | +1.806% | +17.306% | +-----+------------------------+--------+------------+------------+------------+ | :-| | pickle_dict| 0.282% | +0.551% | +0.933% | +16.079% | +-----+------------------------+--------+------------+------------+------------+ | :-) | pickle_list| 0.908% | +2.838% | +5.182% | +15.886% | +-----+------------------------+--------+------------+------------+------------+ | :-| | pickle_pure_python| 3.829% | +0.234% | +11.782% | +11.675% | +-----+------------------------+--------+------------+------------+------------+ | :-| | pidigits| 0.203% | -0.022% | +0.234% | +9.431% | +-----+------------------------+--------+------------+------------+------------+ | :-) | python_startup| 0.118% | +5.710% | +18.101% | +5.565% | +-----+------------------------+--------+------------+------------+------------+ | :-) | python_startup_no_site| 0.095% | +2.337% | +5.062% | +5.723% | +-----+------------------------+--------+------------+------------+------------+ | :-| | raytrace| 1.066% | +1.289% | +10.738% | +13.847% | +-----+------------------------+--------+------------+------------+------------+ | :-| | regex_compile| 5.437% | +2.377% | +5.392% | +12.469% | +-----+------------------------+--------+------------+------------+------------+ | :-| | regex_dna| 0.603% | -1.718% | -2.537% | +11.725% | +-----+------------------------+--------+------------+------------+------------+ | :-| | regex_effbot| 2.247% | -3.450% | -8.361% | +8.713% | +-----+------------------------+--------+------------+------------+------------+ | :-| | regex_v8| 1.788% | -1.083% | +3.143% | +9.055% | +-----+------------------------+--------+------------+------------+------------+ | :-| | richards| 1.433% | +4.031% | +10.278% | +16.002% | +-----+------------------------+--------+------------+------------+------------+ | :-| | scimark_fft| 0.632% | +1.201% | -0.608% | +3.389% | +-----+------------------------+--------+------------+------------+------------+ | :-| | scimark_lu| 2.475% | +0.623% | +24.392% | +12.106% | +-----+------------------------+--------+------------+------------+------------+ | :-| | scimark_monte_carlo| 2.221% | +0.105% | +4.394% | +6.346% | +-----+------------------------+--------+------------+------------+------------+ | :-| | scimark_sor| 1.209% | +0.717% | +15.091% | +7.922% | +-----+------------------------+--------+------------+------------+------------+ | :-) | scimark_sparse_mat_mult| 0.684% | +4.017% | -3.043% | -4.054% | +-----+------------------------+--------+------------+------------+------------+ | :-| | spectral_norm| 0.377% | +0.824% | +5.344% | +4.463% | +-----+------------------------+--------+------------+------------+------------+ | :-| | sqlalchemy_declarative| 1.909% | -2.423% | +7.224% | +7.840% | +-----+------------------------+--------+------------+------------+------------+ | :-| | sqlalchemy_imperative| 3.127% | -2.696% | +6.639% | +6.488% | +-----+------------------------+--------+------------+------------+------------+ | :-( | sqlite_synth| 3.888% | -20.178% | +0.410% | +12.423% | +-----+------------------------+--------+------------+------------+------------+ | :-| | sympy_expand| 2.992% | +2.072% | +17.058% | +8.943% | +-----+------------------------+--------+------------+------------+------------+ | :-| | sympy_integrate| 1.207% | +2.675% | +12.728% | +7.223% | +-----+------------------------+--------+------------+------------+------------+ | :-| | sympy_str| 3.656% | +1.617% | +17.073% | +10.268% | +-----+------------------------+--------+------------+------------+------------+ | :-| | sympy_sum| 6.178% | +2.791% | +14.852% | +13.292% | +-----+------------------------+--------+------------+------------+------------+ | :-| | telco| 3.298% | +4.781% | +20.929% | +9.389% | +-----+------------------------+--------+------------+------------+------------+ | :-| | tornado_http| 0.945% | +0.130% | +7.116% | +7.171% | +-----+------------------------+--------+------------+------------+------------+ | :-| | unpack_sequence| 1.104% | +0.439% | +2.131% | -0.385% | +-----+------------------------+--------+------------+------------+------------+ | :-| | unpickle| 5.866% | +1.179% | +9.377% | +21.233% | +-----+------------------------+--------+------------+------------+------------+ | :-| | unpickle_list| 0.536% | -0.098% | -1.835% | +16.542% | +-----+------------------------+--------+------------+------------+------------+ | :-| | unpickle_pure_python| 1.669% | +0.215% | +8.352% | +7.406% | +-----+------------------------+--------+------------+------------+------------+ | :-| | xml_etree_generate| 1.249% | -0.267% | +4.436% | +11.838% | +-----+------------------------+--------+------------+------------+------------+ | :-| | xml_etree_iterparse| 2.764% | -0.866% | +1.261% | +10.111% | +-----+------------------------+--------+------------+------------+------------+ | :-| | xml_etree_parse| 4.700% | -3.352% | -7.985% | +11.549% | +-----+------------------------+--------+------------+------------+------------+ | :-| | xml_etree_process| 1.180% | +0.211% | +6.087% | +11.596% | +-----+------------------------+--------+------------+------------+------------+ * Relative Standard Deviation (Standard Deviation/Average) If this is not displayed properly please visit our results page here: http://languagesperformance.intel.com/2-down-7-up-56-flat-results-for-python-master-branch-2018-03-30 Our lab does a nightly source pull and build of the Python project and measures performance changes against the previous stable version and the previous nightly measurement. This is provided as a service to the community so that quality issues with current hardware can be identified quickly. Intel technologies' features and benefits depend on system configuration and may require enabled hardware, software or service activation. Performance varies depending on system configuration. From solipsis at pitrou.net Sat Mar 31 05:11:32 2018 From: solipsis at pitrou.net (solipsis at pitrou.net) Date: Sat, 31 Mar 2018 09:11:32 +0000 Subject: [Python-checkins] Daily reference leaks (4243df51fe43): sum=4 Message-ID: <20180331091132.1.9836C9B5CDBFD0AD@psf.io> results for 4243df51fe43 on branch "default" -------------------------------------------- test_collections leaked [0, 7, -7] memory blocks, sum=0 test_functools leaked [0, 3, 1] memory blocks, sum=4 Command line was: ['./python', '-m', 'test.regrtest', '-uall', '-R', '3:3:/home/psf-users/antoine/refleaks/reflogkvthcn', '--timeout', '7200'] From webhook-mailer at python.org Sat Mar 31 08:23:33 2018 From: webhook-mailer at python.org (Serhiy Storchaka) Date: Sat, 31 Mar 2018 12:23:33 -0000 Subject: [Python-checkins] bpo-31544: Fix a reference leak to 'self' after the previous target error handling fixes. (GH-6318) Message-ID: <mailman.13.1522499016.3857.python-checkins@python.org> https://github.com/python/cpython/commit/c498cd8bf81fc47acf2f1f702e8b3bc9bd4aed65 commit: c498cd8bf81fc47acf2f1f702e8b3bc9bd4aed65 branch: 2.7 author: scoder <stefan_ml at behnel.de> committer: Serhiy Storchaka <storchaka at gmail.com> date: 2018-03-31T15:23:30+03:00 summary: bpo-31544: Fix a reference leak to 'self' after the previous target error handling fixes. (GH-6318) This change generally splits the xmlparser creation code into an unsafe part with "rollback" error handling and a safe "object initialisation done" part with normal decref cleanup. files: A Misc/NEWS.d/next/Library/2017-09-13-19-55-35.bpo-31455.beTh6t.rst M Modules/_elementtree.c diff --git a/Misc/NEWS.d/next/Library/2017-09-13-19-55-35.bpo-31455.beTh6t.rst b/Misc/NEWS.d/next/Library/2017-09-13-19-55-35.bpo-31455.beTh6t.rst new file mode 100644 index 000000000000..9ea3599ee0b0 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2017-09-13-19-55-35.bpo-31455.beTh6t.rst @@ -0,0 +1,2 @@ +The C accelerator module of ElementTree ignored exceptions raised when +looking up TreeBuilder target methods in XMLParser(). diff --git a/Modules/_elementtree.c b/Modules/_elementtree.c index 7f0e60934003..1d316a1c91d2 100644 --- a/Modules/_elementtree.c +++ b/Modules/_elementtree.c @@ -2575,14 +2575,24 @@ xmlparser(PyObject* self_, PyObject* args, PyObject* kw) return NULL; } + ALLOC(sizeof(XMLParserObject), "create expatparser"); + + /* Init to NULL to keep the error handling below manageable. */ + self->target = + self->handle_xml = + self->handle_start = + self->handle_data = + self->handle_end = + self->handle_comment = + self->handle_pi = + self->handle_close = + NULL; + /* setup target handlers */ if (!target) { target = treebuilder_new(); if (!target) { - EXPAT(ParserFree)(self->parser); - PyObject_Del(self->names); - PyObject_Del(self->entity); - PyObject_Del(self); + Py_DECREF(self); return NULL; } } else @@ -2591,30 +2601,37 @@ xmlparser(PyObject* self_, PyObject* args, PyObject* kw) self->handle_xml = PyObject_GetAttrString(target, "xml"); if (ignore_attribute_error(self->handle_xml)) { + Py_DECREF(self); return NULL; } self->handle_start = PyObject_GetAttrString(target, "start"); if (ignore_attribute_error(self->handle_start)) { + Py_DECREF(self); return NULL; } self->handle_data = PyObject_GetAttrString(target, "data"); if (ignore_attribute_error(self->handle_data)) { + Py_DECREF(self); return NULL; } self->handle_end = PyObject_GetAttrString(target, "end"); if (ignore_attribute_error(self->handle_end)) { + Py_DECREF(self); return NULL; } self->handle_comment = PyObject_GetAttrString(target, "comment"); if (ignore_attribute_error(self->handle_comment)) { + Py_DECREF(self); return NULL; } self->handle_pi = PyObject_GetAttrString(target, "pi"); if (ignore_attribute_error(self->handle_pi)) { + Py_DECREF(self); return NULL; } self->handle_close = PyObject_GetAttrString(target, "close"); if (ignore_attribute_error(self->handle_close)) { + Py_DECREF(self); return NULL; } @@ -2650,8 +2667,6 @@ xmlparser(PyObject* self_, PyObject* args, PyObject* kw) ); #endif - ALLOC(sizeof(XMLParserObject), "create expatparser"); - return (PyObject*) self; } From webhook-mailer at python.org Sat Mar 31 08:41:20 2018 From: webhook-mailer at python.org (Ivan Levkivskyi) Date: Sat, 31 Mar 2018 12:41:20 -0000 Subject: [Python-checkins] Allow dynamic creation of generic dataclasses (GH-6319) Message-ID: <mailman.14.1522500081.3857.python-checkins@python.org> https://github.com/python/cpython/commit/5a7092de1226a95a50f0f384eea8ddb288959249 commit: 5a7092de1226a95a50f0f384eea8ddb288959249 branch: master author: Ivan Levkivskyi <levkivskyi at gmail.com> committer: GitHub <noreply at github.com> date: 2018-03-31T13:41:17+01:00 summary: Allow dynamic creation of generic dataclasses (GH-6319) files: M Lib/dataclasses.py M Lib/test/test_dataclasses.py diff --git a/Lib/dataclasses.py b/Lib/dataclasses.py index bd7252c683ca..04e07f8cf8c2 100644 --- a/Lib/dataclasses.py +++ b/Lib/dataclasses.py @@ -1004,7 +1004,9 @@ class C(Base): anns[name] = tp namespace['__annotations__'] = anns - cls = type(cls_name, bases, namespace) + # We use `types.new_class()` instead of simply `type()` to allow dynamic creation + # of generic dataclassses. + cls = types.new_class(cls_name, bases, {}, lambda ns: ns.update(namespace)) return dataclass(cls, init=init, repr=repr, eq=eq, order=order, unsafe_hash=unsafe_hash, frozen=frozen) diff --git a/Lib/test/test_dataclasses.py b/Lib/test/test_dataclasses.py index 5cd424cf5760..26bfc4e75a00 100755 --- a/Lib/test/test_dataclasses.py +++ b/Lib/test/test_dataclasses.py @@ -8,7 +8,7 @@ import inspect import unittest from unittest.mock import Mock -from typing import ClassVar, Any, List, Union, Tuple, Dict, Generic, TypeVar +from typing import ClassVar, Any, List, Union, Tuple, Dict, Generic, TypeVar, Optional from collections import deque, OrderedDict, namedtuple from functools import total_ordering @@ -1690,6 +1690,23 @@ def new_method(self): c = Alias(10, 1.0) self.assertEqual(c.new_method(), 1.0) + def test_generic_dynamic(self): + T = TypeVar('T') + + @dataclass + class Parent(Generic[T]): + x: T + Child = make_dataclass('Child', [('y', T), ('z', Optional[T], None)], + bases=(Parent[int], Generic[T]), namespace={'other': 42}) + self.assertIs(Child[int](1, 2).z, None) + self.assertEqual(Child[int](1, 2, 3).z, 3) + self.assertEqual(Child[int](1, 2, 3).other, 42) + # Check that type aliases work correctly. + Alias = Child[T] + self.assertEqual(Alias[int](1, 2).x, 1) + # Check MRO resolution. + self.assertEqual(Child.__mro__, (Child, Parent, Generic, object)) + def test_helper_replace(self): @dataclass(frozen=True) class C: From webhook-mailer at python.org Sat Mar 31 16:54:16 2018 From: webhook-mailer at python.org (Serhiy Storchaka) Date: Sat, 31 Mar 2018 20:54:16 -0000 Subject: [Python-checkins] [3.6] bpo-33132: Fix reference counting issues in the compiler. (GH-6209). (GH-6321) Message-ID: <mailman.15.1522529656.3857.python-checkins@python.org> https://github.com/python/cpython/commit/25c869edd665cba7187910c5b151a0016fec4754 commit: 25c869edd665cba7187910c5b151a0016fec4754 branch: 3.6 author: Serhiy Storchaka <storchaka at gmail.com> committer: GitHub <noreply at github.com> date: 2018-03-31T23:54:13+03:00 summary: [3.6] bpo-33132: Fix reference counting issues in the compiler. (GH-6209). (GH-6321) (cherry picked from commit a95d98607efe0c43475b354543e49bf8e240bc6f) files: M Python/compile.c diff --git a/Python/compile.c b/Python/compile.c index ae0e574276db..c1df7e8d9824 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -2599,8 +2599,7 @@ compiler_import_as(struct compiler *c, identifier name, identifier asname) PyUnicode_GET_LENGTH(name)); if (!attr) return 0; - ADDOP_O(c, LOAD_ATTR, attr, names); - Py_DECREF(attr); + ADDOP_N(c, LOAD_ATTR, attr, names); pos = dot + 1; } } @@ -2628,8 +2627,7 @@ compiler_import(struct compiler *c, stmt_ty s) if (level == NULL) return 0; - ADDOP_O(c, LOAD_CONST, level, consts); - Py_DECREF(level); + ADDOP_N(c, LOAD_CONST, level, consts); ADDOP_O(c, LOAD_CONST, Py_None, consts); ADDOP_NAME(c, IMPORT_NAME, alias->name, names); @@ -2662,9 +2660,7 @@ static int compiler_from_import(struct compiler *c, stmt_ty s) { Py_ssize_t i, n = asdl_seq_LEN(s->v.ImportFrom.names); - - PyObject *names = PyTuple_New(n); - PyObject *level; + PyObject *level, *names; static PyObject *empty_string; if (!empty_string) { @@ -2673,14 +2669,15 @@ compiler_from_import(struct compiler *c, stmt_ty s) return 0; } - if (!names) - return 0; - level = PyLong_FromLong(s->v.ImportFrom.level); if (!level) { - Py_DECREF(names); return 0; } + ADDOP_N(c, LOAD_CONST, level, consts); + + names = PyTuple_New(n); + if (!names) + return 0; /* build up the names */ for (i = 0; i < n; i++) { @@ -2691,16 +2688,12 @@ compiler_from_import(struct compiler *c, stmt_ty s) if (s->lineno > c->c_future->ff_lineno && s->v.ImportFrom.module && _PyUnicode_EqualToASCIIString(s->v.ImportFrom.module, "__future__")) { - Py_DECREF(level); Py_DECREF(names); return compiler_error(c, "from __future__ imports must occur " "at the beginning of the file"); } + ADDOP_N(c, LOAD_CONST, names, consts); - ADDOP_O(c, LOAD_CONST, level, consts); - Py_DECREF(level); - ADDOP_O(c, LOAD_CONST, names, consts); - Py_DECREF(names); if (s->v.ImportFrom.module) { ADDOP_NAME(c, IMPORT_NAME, s->v.ImportFrom.module, names); } @@ -2723,7 +2716,6 @@ compiler_from_import(struct compiler *c, stmt_ty s) store_name = alias->asname; if (!compiler_nameop(c, store_name, Store)) { - Py_DECREF(names); return 0; } } @@ -3101,8 +3093,7 @@ compiler_nameop(struct compiler *c, identifier name, expr_context_ty ctx) "param invalid for local variable"); return 0; } - ADDOP_O(c, op, mangled, varnames); - Py_DECREF(mangled); + ADDOP_N(c, op, mangled, varnames); return 1; case OP_GLOBAL: switch (ctx) { @@ -4552,11 +4543,11 @@ compiler_annassign(struct compiler *c, stmt_ty s) if (s->v.AnnAssign.simple && (c->u->u_scope_type == COMPILER_SCOPE_MODULE || c->u->u_scope_type == COMPILER_SCOPE_CLASS)) { + VISIT(c, expr, s->v.AnnAssign.annotation); mangled = _Py_Mangle(c->u->u_private, targ->v.Name.id); if (!mangled) { return 0; } - VISIT(c, expr, s->v.AnnAssign.annotation); /* ADDOP_N decrefs its argument */ ADDOP_N(c, STORE_ANNOTATION, mangled, names); } From webhook-mailer at python.org Sat Mar 31 17:29:40 2018 From: webhook-mailer at python.org (Serhiy Storchaka) Date: Sat, 31 Mar 2018 21:29:40 -0000 Subject: [Python-checkins] bpo-33132: Fix more reference counting issues in the compiler. (GH-6323) Message-ID: <mailman.16.1522531782.3857.python-checkins@python.org> https://github.com/python/cpython/commit/aa8e51f5ebb2a71c76059f050de01fc3c985376a commit: aa8e51f5ebb2a71c76059f050de01fc3c985376a branch: master author: Serhiy Storchaka <storchaka at gmail.com> committer: GitHub <noreply at github.com> date: 2018-04-01T00:29:37+03:00 summary: bpo-33132: Fix more reference counting issues in the compiler. (GH-6323) files: M Python/compile.c diff --git a/Python/compile.c b/Python/compile.c index 9d2ba7b18f11..03b703deb041 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -2843,8 +2843,7 @@ compiler_import_as(struct compiler *c, identifier name, identifier asname) attr = PyUnicode_Substring(name, pos, (dot != -1) ? dot : len); if (!attr) return 0; - ADDOP_O(c, IMPORT_FROM, attr, names); - Py_DECREF(attr); + ADDOP_N(c, IMPORT_FROM, attr, names); if (dot == -1) { break; } @@ -3294,8 +3293,7 @@ compiler_nameop(struct compiler *c, identifier name, expr_context_ty ctx) "param invalid for local variable"); return 0; } - ADDOP_O(c, op, mangled, varnames); - Py_DECREF(mangled); + ADDOP_N(c, op, mangled, varnames); return 1; case OP_GLOBAL: switch (ctx) { From webhook-mailer at python.org Sat Mar 31 18:03:53 2018 From: webhook-mailer at python.org (Serhiy Storchaka) Date: Sat, 31 Mar 2018 22:03:53 -0000 Subject: [Python-checkins] [3.6] Fix error message in sqlite connection thread check. (GH-6028). (GH-6324) Message-ID: <mailman.17.1522533833.3857.python-checkins@python.org> https://github.com/python/cpython/commit/d918bbda4bb201c35d1ded3dde686d8b00a91851 commit: d918bbda4bb201c35d1ded3dde686d8b00a91851 branch: 3.6 author: Serhiy Storchaka <storchaka at gmail.com> committer: GitHub <noreply at github.com> date: 2018-04-01T01:03:50+03:00 summary: [3.6] Fix error message in sqlite connection thread check. (GH-6028). (GH-6324) (cherry picked from commit 030345c0bfc2f76684666fe5c61e766ba5debfe6) Co-authored-by: Takuya Akiba <469803+iwiwi at users.noreply.github.com> files: M Modules/_sqlite/connection.c diff --git a/Modules/_sqlite/connection.c b/Modules/_sqlite/connection.c index 1c6aa54c224d..32cd306b3721 100644 --- a/Modules/_sqlite/connection.c +++ b/Modules/_sqlite/connection.c @@ -1125,8 +1125,8 @@ int pysqlite_check_thread(pysqlite_Connection* self) if (self->check_same_thread) { if (PyThread_get_thread_ident() != self->thread_ident) { PyErr_Format(pysqlite_ProgrammingError, - "SQLite objects created in a thread can only be used in that same thread." - "The object was created in thread id %ld and this is thread id %ld", + "SQLite objects created in a thread can only be used in that same thread. " + "The object was created in thread id %ld and this is thread id %ld.", self->thread_ident, PyThread_get_thread_ident()); return 0; } From webhook-mailer at python.org Sat Mar 31 18:04:25 2018 From: webhook-mailer at python.org (Serhiy Storchaka) Date: Sat, 31 Mar 2018 22:04:25 -0000 Subject: [Python-checkins] [2.7] Fix error message in sqlite connection thread check. (GH-6028). (GH-6325) Message-ID: <mailman.18.1522533866.3857.python-checkins@python.org> https://github.com/python/cpython/commit/924035a5e550c9d0bee05c26eadff0e44a0db582 commit: 924035a5e550c9d0bee05c26eadff0e44a0db582 branch: 2.7 author: Serhiy Storchaka <storchaka at gmail.com> committer: GitHub <noreply at github.com> date: 2018-04-01T01:04:22+03:00 summary: [2.7] Fix error message in sqlite connection thread check. (GH-6028). (GH-6325) (cherry picked from commit 030345c0bfc2f76684666fe5c61e766ba5debfe6) Co-authored-by: Takuya Akiba <469803+iwiwi at users.noreply.github.com> files: M Modules/_sqlite/connection.c diff --git a/Modules/_sqlite/connection.c b/Modules/_sqlite/connection.c index 439542e51393..0603b4307b17 100644 --- a/Modules/_sqlite/connection.c +++ b/Modules/_sqlite/connection.c @@ -1106,8 +1106,8 @@ int pysqlite_check_thread(pysqlite_Connection* self) if (self->check_same_thread) { if (PyThread_get_thread_ident() != self->thread_ident) { PyErr_Format(pysqlite_ProgrammingError, - "SQLite objects created in a thread can only be used in that same thread." - "The object was created in thread id %ld and this is thread id %ld", + "SQLite objects created in a thread can only be used in that same thread. " + "The object was created in thread id %ld and this is thread id %ld.", self->thread_ident, PyThread_get_thread_ident()); return 0; } From webhook-mailer at python.org Sat Mar 31 19:41:31 2018 From: webhook-mailer at python.org (Miss Islington (bot)) Date: Sat, 31 Mar 2018 23:41:31 -0000 Subject: [Python-checkins] bpo-33132: Fix more reference counting issues in the compiler. (GH-6323) Message-ID: <mailman.19.1522539693.3857.python-checkins@python.org> https://github.com/python/cpython/commit/9e96e7b24e4d3ff4dce4f24c4e469cd5460712c9 commit: 9e96e7b24e4d3ff4dce4f24c4e469cd5460712c9 branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: GitHub <noreply at github.com> date: 2018-03-31T16:41:28-07:00 summary: bpo-33132: Fix more reference counting issues in the compiler. (GH-6323) (cherry picked from commit aa8e51f5ebb2a71c76059f050de01fc3c985376a) Co-authored-by: Serhiy Storchaka <storchaka at gmail.com> files: M Python/compile.c diff --git a/Python/compile.c b/Python/compile.c index 03b4826b761b..d4245e2f4e50 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -2742,8 +2742,7 @@ compiler_import_as(struct compiler *c, identifier name, identifier asname) attr = PyUnicode_Substring(name, pos, (dot != -1) ? dot : len); if (!attr) return 0; - ADDOP_O(c, IMPORT_FROM, attr, names); - Py_DECREF(attr); + ADDOP_N(c, IMPORT_FROM, attr, names); if (dot == -1) { break; } @@ -3207,8 +3206,7 @@ compiler_nameop(struct compiler *c, identifier name, expr_context_ty ctx) "param invalid for local variable"); return 0; } - ADDOP_O(c, op, mangled, varnames); - Py_DECREF(mangled); + ADDOP_N(c, op, mangled, varnames); return 1; case OP_GLOBAL: switch (ctx) { From webhook-mailer at python.org Sat Mar 31 19:43:01 2018 From: webhook-mailer at python.org (Serhiy Storchaka) Date: Sat, 31 Mar 2018 23:43:01 -0000 Subject: [Python-checkins] [2.7] bpo-33096: Fix ttk.Treeview.insert. (GH-6228) (GH-6326) Message-ID: <mailman.20.1522539783.3857.python-checkins@python.org> https://github.com/python/cpython/commit/e80a232f2cfdab584133d9779c83885c5f9f1ba6 commit: e80a232f2cfdab584133d9779c83885c5f9f1ba6 branch: 2.7 author: Serhiy Storchaka <storchaka at gmail.com> committer: GitHub <noreply at github.com> date: 2018-04-01T02:42:58+03:00 summary: [2.7] bpo-33096: Fix ttk.Treeview.insert. (GH-6228) (GH-6326) Allow ttk.Treeview.insert to insert iid that has a false boolean value. Note iid=0 and iid=False would be same. (cherry picked from commit 3ab44c0783eebdff687014f7d14d5dec59b6bd39) Co-authored-by: Garvit Khatri <garvitdelhi at gmail.com> files: A Lib/tkinter/test/test_ttk/test_widgets.py A Lib/tkinter/ttk.py A Misc/NEWS.d/next/Library/2018-03-25-13-18-16.bpo-33096.ofdbe7.rst M Lib/lib-tk/test/test_ttk/test_widgets.py M Lib/lib-tk/ttk.py diff --git a/Lib/lib-tk/test/test_ttk/test_widgets.py b/Lib/lib-tk/test/test_ttk/test_widgets.py index a84960d241aa..3d5683cc4f5f 100644 --- a/Lib/lib-tk/test/test_ttk/test_widgets.py +++ b/Lib/lib-tk/test/test_ttk/test_widgets.py @@ -1485,6 +1485,15 @@ def test_insert_item(self): self.tv.insert('', 'end', text=value), text=None), value) + # test for values which are not None + itemid = self.tv.insert('', 'end', 0) + self.assertEqual(itemid, '0') + itemid = self.tv.insert('', 'end', 0.0) + self.assertEqual(itemid, '0.0') + # this is because False resolves to 0 and element with 0 iid is already present + self.assertRaises(tkinter.TclError, self.tv.insert, '', 'end', False) + self.assertRaises(tkinter.TclError, self.tv.insert, '', 'end', '') + def test_selection(self): # item 'none' doesn't exist diff --git a/Lib/lib-tk/ttk.py b/Lib/lib-tk/ttk.py index 6da1eb189f12..d4df408e477f 100644 --- a/Lib/lib-tk/ttk.py +++ b/Lib/lib-tk/ttk.py @@ -1332,7 +1332,7 @@ def insert(self, parent, index, iid=None, **kw): already exist in the tree. Otherwise, a new unique identifier is generated.""" opts = _format_optdict(kw) - if iid: + if iid is not None: res = self.tk.call(self._w, "insert", parent, index, "-id", iid, *opts) else: diff --git a/Lib/tkinter/test/test_ttk/test_widgets.py b/Lib/tkinter/test/test_ttk/test_widgets.py new file mode 100644 index 000000000000..5b0e29cdccaf --- /dev/null +++ b/Lib/tkinter/test/test_ttk/test_widgets.py @@ -0,0 +1,1872 @@ +import unittest +import tkinter +from tkinter import ttk, TclError +from test.support import requires +import sys + +from tkinter.test.test_ttk.test_functions import MockTclObj +from tkinter.test.support import (AbstractTkTest, tcl_version, get_tk_patchlevel, + simulate_mouse_click) +from tkinter.test.widget_tests import (add_standard_options, noconv, + AbstractWidgetTest, StandardOptionsTests, IntegerSizeTests, PixelSizeTests, + setUpModule) + +requires('gui') + + +class StandardTtkOptionsTests(StandardOptionsTests): + + def test_class(self): + widget = self.create() + self.assertEqual(widget['class'], '') + errmsg='attempt to change read-only option' + if get_tk_patchlevel() < (8, 6, 0, 'beta', 3): + errmsg='Attempt to change read-only option' + self.checkInvalidParam(widget, 'class', 'Foo', errmsg=errmsg) + widget2 = self.create(class_='Foo') + self.assertEqual(widget2['class'], 'Foo') + + def test_padding(self): + widget = self.create() + self.checkParam(widget, 'padding', 0, expected=('0',)) + self.checkParam(widget, 'padding', 5, expected=('5',)) + self.checkParam(widget, 'padding', (5, 6), expected=('5', '6')) + self.checkParam(widget, 'padding', (5, 6, 7), + expected=('5', '6', '7')) + self.checkParam(widget, 'padding', (5, 6, 7, 8), + expected=('5', '6', '7', '8')) + self.checkParam(widget, 'padding', ('5p', '6p', '7p', '8p')) + self.checkParam(widget, 'padding', (), expected='') + + def test_style(self): + widget = self.create() + self.assertEqual(widget['style'], '') + errmsg = 'Layout Foo not found' + if hasattr(self, 'default_orient'): + errmsg = ('Layout %s.Foo not found' % + getattr(self, 'default_orient').title()) + self.checkInvalidParam(widget, 'style', 'Foo', + errmsg=errmsg) + widget2 = self.create(class_='Foo') + self.assertEqual(widget2['class'], 'Foo') + # XXX + pass + + +class WidgetTest(AbstractTkTest, unittest.TestCase): + """Tests methods available in every ttk widget.""" + + def setUp(self): + super().setUp() + self.widget = ttk.Button(self.root, width=0, text="Text") + self.widget.pack() + self.widget.wait_visibility() + + + def test_identify(self): + self.widget.update_idletasks() + self.assertEqual(self.widget.identify( + int(self.widget.winfo_width() / 2), + int(self.widget.winfo_height() / 2) + ), "label") + self.assertEqual(self.widget.identify(-1, -1), "") + + self.assertRaises(tkinter.TclError, self.widget.identify, None, 5) + self.assertRaises(tkinter.TclError, self.widget.identify, 5, None) + self.assertRaises(tkinter.TclError, self.widget.identify, 5, '') + + + def test_widget_state(self): + # XXX not sure about the portability of all these tests + self.assertEqual(self.widget.state(), ()) + self.assertEqual(self.widget.instate(['!disabled']), True) + + # changing from !disabled to disabled + self.assertEqual(self.widget.state(['disabled']), ('!disabled', )) + # no state change + self.assertEqual(self.widget.state(['disabled']), ()) + # change back to !disable but also active + self.assertEqual(self.widget.state(['!disabled', 'active']), + ('!active', 'disabled')) + # no state changes, again + self.assertEqual(self.widget.state(['!disabled', 'active']), ()) + self.assertEqual(self.widget.state(['active', '!disabled']), ()) + + def test_cb(arg1, **kw): + return arg1, kw + self.assertEqual(self.widget.instate(['!disabled'], + test_cb, "hi", **{"msg": "there"}), + ('hi', {'msg': 'there'})) + + # attempt to set invalid statespec + currstate = self.widget.state() + self.assertRaises(tkinter.TclError, self.widget.instate, + ['badstate']) + self.assertRaises(tkinter.TclError, self.widget.instate, + ['disabled', 'badstate']) + # verify that widget didn't change its state + self.assertEqual(currstate, self.widget.state()) + + # ensuring that passing None as state doesn't modify current state + self.widget.state(['active', '!disabled']) + self.assertEqual(self.widget.state(), ('active', )) + + +class AbstractToplevelTest(AbstractWidgetTest, PixelSizeTests): + _conv_pixels = noconv + + + at add_standard_options(StandardTtkOptionsTests) +class FrameTest(AbstractToplevelTest, unittest.TestCase): + OPTIONS = ( + 'borderwidth', 'class', 'cursor', 'height', + 'padding', 'relief', 'style', 'takefocus', + 'width', + ) + + def create(self, **kwargs): + return ttk.Frame(self.root, **kwargs) + + + at add_standard_options(StandardTtkOptionsTests) +class LabelFrameTest(AbstractToplevelTest, unittest.TestCase): + OPTIONS = ( + 'borderwidth', 'class', 'cursor', 'height', + 'labelanchor', 'labelwidget', + 'padding', 'relief', 'style', 'takefocus', + 'text', 'underline', 'width', + ) + + def create(self, **kwargs): + return ttk.LabelFrame(self.root, **kwargs) + + def test_labelanchor(self): + widget = self.create() + self.checkEnumParam(widget, 'labelanchor', + 'e', 'en', 'es', 'n', 'ne', 'nw', 's', 'se', 'sw', 'w', 'wn', 'ws', + errmsg='Bad label anchor specification {}') + self.checkInvalidParam(widget, 'labelanchor', 'center') + + def test_labelwidget(self): + widget = self.create() + label = ttk.Label(self.root, text='Mupp', name='foo') + self.checkParam(widget, 'labelwidget', label, expected='.foo') + label.destroy() + + +class AbstractLabelTest(AbstractWidgetTest): + + def checkImageParam(self, widget, name): + image = tkinter.PhotoImage(master=self.root, name='image1') + image2 = tkinter.PhotoImage(master=self.root, name='image2') + self.checkParam(widget, name, image, expected=('image1',)) + self.checkParam(widget, name, 'image1', expected=('image1',)) + self.checkParam(widget, name, (image,), expected=('image1',)) + self.checkParam(widget, name, (image, 'active', image2), + expected=('image1', 'active', 'image2')) + self.checkParam(widget, name, 'image1 active image2', + expected=('image1', 'active', 'image2')) + self.checkInvalidParam(widget, name, 'spam', + errmsg='image "spam" doesn\'t exist') + + def test_compound(self): + widget = self.create() + self.checkEnumParam(widget, 'compound', + 'none', 'text', 'image', 'center', + 'top', 'bottom', 'left', 'right') + + def test_state(self): + widget = self.create() + self.checkParams(widget, 'state', 'active', 'disabled', 'normal') + + def test_width(self): + widget = self.create() + self.checkParams(widget, 'width', 402, -402, 0) + + + at add_standard_options(StandardTtkOptionsTests) +class LabelTest(AbstractLabelTest, unittest.TestCase): + OPTIONS = ( + 'anchor', 'background', 'borderwidth', + 'class', 'compound', 'cursor', 'font', 'foreground', + 'image', 'justify', 'padding', 'relief', 'state', 'style', + 'takefocus', 'text', 'textvariable', + 'underline', 'width', 'wraplength', + ) + _conv_pixels = noconv + + def create(self, **kwargs): + return ttk.Label(self.root, **kwargs) + + def test_font(self): + widget = self.create() + self.checkParam(widget, 'font', + '-Adobe-Helvetica-Medium-R-Normal--*-120-*-*-*-*-*-*') + + + at add_standard_options(StandardTtkOptionsTests) +class ButtonTest(AbstractLabelTest, unittest.TestCase): + OPTIONS = ( + 'class', 'command', 'compound', 'cursor', 'default', + 'image', 'padding', 'state', 'style', + 'takefocus', 'text', 'textvariable', + 'underline', 'width', + ) + + def create(self, **kwargs): + return ttk.Button(self.root, **kwargs) + + def test_default(self): + widget = self.create() + self.checkEnumParam(widget, 'default', 'normal', 'active', 'disabled') + + def test_invoke(self): + success = [] + btn = ttk.Button(self.root, command=lambda: success.append(1)) + btn.invoke() + self.assertTrue(success) + + + at add_standard_options(StandardTtkOptionsTests) +class CheckbuttonTest(AbstractLabelTest, unittest.TestCase): + OPTIONS = ( + 'class', 'command', 'compound', 'cursor', + 'image', + 'offvalue', 'onvalue', + 'padding', 'state', 'style', + 'takefocus', 'text', 'textvariable', + 'underline', 'variable', 'width', + ) + + def create(self, **kwargs): + return ttk.Checkbutton(self.root, **kwargs) + + def test_offvalue(self): + widget = self.create() + self.checkParams(widget, 'offvalue', 1, 2.3, '', 'any string') + + def test_onvalue(self): + widget = self.create() + self.checkParams(widget, 'onvalue', 1, 2.3, '', 'any string') + + def test_invoke(self): + success = [] + def cb_test(): + success.append(1) + return "cb test called" + + cbtn = ttk.Checkbutton(self.root, command=cb_test) + # the variable automatically created by ttk.Checkbutton is actually + # undefined till we invoke the Checkbutton + self.assertEqual(cbtn.state(), ('alternate', )) + self.assertRaises(tkinter.TclError, cbtn.tk.globalgetvar, + cbtn['variable']) + + res = cbtn.invoke() + self.assertEqual(res, "cb test called") + self.assertEqual(cbtn['onvalue'], + cbtn.tk.globalgetvar(cbtn['variable'])) + self.assertTrue(success) + + cbtn['command'] = '' + res = cbtn.invoke() + self.assertFalse(str(res)) + self.assertLessEqual(len(success), 1) + self.assertEqual(cbtn['offvalue'], + cbtn.tk.globalgetvar(cbtn['variable'])) + + + at add_standard_options(IntegerSizeTests, StandardTtkOptionsTests) +class EntryTest(AbstractWidgetTest, unittest.TestCase): + OPTIONS = ( + 'background', 'class', 'cursor', + 'exportselection', 'font', 'foreground', + 'invalidcommand', 'justify', + 'show', 'state', 'style', 'takefocus', 'textvariable', + 'validate', 'validatecommand', 'width', 'xscrollcommand', + ) + + def setUp(self): + super().setUp() + self.entry = self.create() + + def create(self, **kwargs): + return ttk.Entry(self.root, **kwargs) + + def test_invalidcommand(self): + widget = self.create() + self.checkCommandParam(widget, 'invalidcommand') + + def test_show(self): + widget = self.create() + self.checkParam(widget, 'show', '*') + self.checkParam(widget, 'show', '') + self.checkParam(widget, 'show', ' ') + + def test_state(self): + widget = self.create() + self.checkParams(widget, 'state', + 'disabled', 'normal', 'readonly') + + def test_validate(self): + widget = self.create() + self.checkEnumParam(widget, 'validate', + 'all', 'key', 'focus', 'focusin', 'focusout', 'none') + + def test_validatecommand(self): + widget = self.create() + self.checkCommandParam(widget, 'validatecommand') + + + def test_bbox(self): + self.assertIsBoundingBox(self.entry.bbox(0)) + self.assertRaises(tkinter.TclError, self.entry.bbox, 'noindex') + self.assertRaises(tkinter.TclError, self.entry.bbox, None) + + + def test_identify(self): + self.entry.pack() + self.entry.wait_visibility() + self.entry.update_idletasks() + + self.assertEqual(self.entry.identify(5, 5), "textarea") + self.assertEqual(self.entry.identify(-1, -1), "") + + self.assertRaises(tkinter.TclError, self.entry.identify, None, 5) + self.assertRaises(tkinter.TclError, self.entry.identify, 5, None) + self.assertRaises(tkinter.TclError, self.entry.identify, 5, '') + + + def test_validation_options(self): + success = [] + test_invalid = lambda: success.append(True) + + self.entry['validate'] = 'none' + self.entry['validatecommand'] = lambda: False + + self.entry['invalidcommand'] = test_invalid + self.entry.validate() + self.assertTrue(success) + + self.entry['invalidcommand'] = '' + self.entry.validate() + self.assertEqual(len(success), 1) + + self.entry['invalidcommand'] = test_invalid + self.entry['validatecommand'] = lambda: True + self.entry.validate() + self.assertEqual(len(success), 1) + + self.entry['validatecommand'] = '' + self.entry.validate() + self.assertEqual(len(success), 1) + + self.entry['validatecommand'] = True + self.assertRaises(tkinter.TclError, self.entry.validate) + + + def test_validation(self): + validation = [] + def validate(to_insert): + if not 'a' <= to_insert.lower() <= 'z': + validation.append(False) + return False + validation.append(True) + return True + + self.entry['validate'] = 'key' + self.entry['validatecommand'] = self.entry.register(validate), '%S' + + self.entry.insert('end', 1) + self.entry.insert('end', 'a') + self.assertEqual(validation, [False, True]) + self.assertEqual(self.entry.get(), 'a') + + + def test_revalidation(self): + def validate(content): + for letter in content: + if not 'a' <= letter.lower() <= 'z': + return False + return True + + self.entry['validatecommand'] = self.entry.register(validate), '%P' + + self.entry.insert('end', 'avocado') + self.assertEqual(self.entry.validate(), True) + self.assertEqual(self.entry.state(), ()) + + self.entry.delete(0, 'end') + self.assertEqual(self.entry.get(), '') + + self.entry.insert('end', 'a1b') + self.assertEqual(self.entry.validate(), False) + self.assertEqual(self.entry.state(), ('invalid', )) + + self.entry.delete(1) + self.assertEqual(self.entry.validate(), True) + self.assertEqual(self.entry.state(), ()) + + + at add_standard_options(IntegerSizeTests, StandardTtkOptionsTests) +class ComboboxTest(EntryTest, unittest.TestCase): + OPTIONS = ( + 'background', 'class', 'cursor', 'exportselection', + 'font', 'foreground', 'height', 'invalidcommand', + 'justify', 'postcommand', 'show', 'state', 'style', + 'takefocus', 'textvariable', + 'validate', 'validatecommand', 'values', + 'width', 'xscrollcommand', + ) + + def setUp(self): + super().setUp() + self.combo = self.create() + + def create(self, **kwargs): + return ttk.Combobox(self.root, **kwargs) + + def test_height(self): + widget = self.create() + self.checkParams(widget, 'height', 100, 101.2, 102.6, -100, 0, '1i') + + def _show_drop_down_listbox(self): + width = self.combo.winfo_width() + self.combo.event_generate('<ButtonPress-1>', x=width - 5, y=5) + self.combo.event_generate('<ButtonRelease-1>', x=width - 5, y=5) + self.combo.update_idletasks() + + + def test_virtual_event(self): + success = [] + + self.combo['values'] = [1] + self.combo.bind('<<ComboboxSelected>>', + lambda evt: success.append(True)) + self.combo.pack() + self.combo.wait_visibility() + + height = self.combo.winfo_height() + self._show_drop_down_listbox() + self.combo.update() + self.combo.event_generate('<Return>') + self.combo.update() + + self.assertTrue(success) + + + def test_postcommand(self): + success = [] + + self.combo['postcommand'] = lambda: success.append(True) + self.combo.pack() + self.combo.wait_visibility() + + self._show_drop_down_listbox() + self.assertTrue(success) + + # testing postcommand removal + self.combo['postcommand'] = '' + self._show_drop_down_listbox() + self.assertEqual(len(success), 1) + + + def test_values(self): + def check_get_current(getval, currval): + self.assertEqual(self.combo.get(), getval) + self.assertEqual(self.combo.current(), currval) + + self.assertEqual(self.combo['values'], + () if tcl_version < (8, 5) else '') + check_get_current('', -1) + + self.checkParam(self.combo, 'values', 'mon tue wed thur', + expected=('mon', 'tue', 'wed', 'thur')) + self.checkParam(self.combo, 'values', ('mon', 'tue', 'wed', 'thur')) + self.checkParam(self.combo, 'values', (42, 3.14, '', 'any string')) + self.checkParam(self.combo, 'values', '', + expected='' if get_tk_patchlevel() < (8, 5, 10) else ()) + + self.combo['values'] = ['a', 1, 'c'] + + self.combo.set('c') + check_get_current('c', 2) + + self.combo.current(0) + check_get_current('a', 0) + + self.combo.set('d') + check_get_current('d', -1) + + # testing values with empty string + self.combo.set('') + self.combo['values'] = (1, 2, '', 3) + check_get_current('', 2) + + # testing values with empty string set through configure + self.combo.configure(values=[1, '', 2]) + self.assertEqual(self.combo['values'], + ('1', '', '2') if self.wantobjects else + '1 {} 2') + + # testing values with spaces + self.combo['values'] = ['a b', 'a\tb', 'a\nb'] + self.assertEqual(self.combo['values'], + ('a b', 'a\tb', 'a\nb') if self.wantobjects else + '{a b} {a\tb} {a\nb}') + + # testing values with special characters + self.combo['values'] = [r'a\tb', '"a"', '} {'] + self.assertEqual(self.combo['values'], + (r'a\tb', '"a"', '} {') if self.wantobjects else + r'a\\tb {"a"} \}\ \{') + + # out of range + self.assertRaises(tkinter.TclError, self.combo.current, + len(self.combo['values'])) + # it expects an integer (or something that can be converted to int) + self.assertRaises(tkinter.TclError, self.combo.current, '') + + # testing creating combobox with empty string in values + combo2 = ttk.Combobox(self.root, values=[1, 2, '']) + self.assertEqual(combo2['values'], + ('1', '2', '') if self.wantobjects else '1 2 {}') + combo2.destroy() + + + at add_standard_options(IntegerSizeTests, StandardTtkOptionsTests) +class PanedWindowTest(AbstractWidgetTest, unittest.TestCase): + OPTIONS = ( + 'class', 'cursor', 'height', + 'orient', 'style', 'takefocus', 'width', + ) + + def setUp(self): + super().setUp() + self.paned = self.create() + + def create(self, **kwargs): + return ttk.PanedWindow(self.root, **kwargs) + + def test_orient(self): + widget = self.create() + self.assertEqual(str(widget['orient']), 'vertical') + errmsg='attempt to change read-only option' + if get_tk_patchlevel() < (8, 6, 0, 'beta', 3): + errmsg='Attempt to change read-only option' + self.checkInvalidParam(widget, 'orient', 'horizontal', + errmsg=errmsg) + widget2 = self.create(orient='horizontal') + self.assertEqual(str(widget2['orient']), 'horizontal') + + def test_add(self): + # attempt to add a child that is not a direct child of the paned window + label = ttk.Label(self.paned) + child = ttk.Label(label) + self.assertRaises(tkinter.TclError, self.paned.add, child) + label.destroy() + child.destroy() + # another attempt + label = ttk.Label(self.root) + child = ttk.Label(label) + self.assertRaises(tkinter.TclError, self.paned.add, child) + child.destroy() + label.destroy() + + good_child = ttk.Label(self.root) + self.paned.add(good_child) + # re-adding a child is not accepted + self.assertRaises(tkinter.TclError, self.paned.add, good_child) + + other_child = ttk.Label(self.paned) + self.paned.add(other_child) + self.assertEqual(self.paned.pane(0), self.paned.pane(1)) + self.assertRaises(tkinter.TclError, self.paned.pane, 2) + good_child.destroy() + other_child.destroy() + self.assertRaises(tkinter.TclError, self.paned.pane, 0) + + + def test_forget(self): + self.assertRaises(tkinter.TclError, self.paned.forget, None) + self.assertRaises(tkinter.TclError, self.paned.forget, 0) + + self.paned.add(ttk.Label(self.root)) + self.paned.forget(0) + self.assertRaises(tkinter.TclError, self.paned.forget, 0) + + + def test_insert(self): + self.assertRaises(tkinter.TclError, self.paned.insert, None, 0) + self.assertRaises(tkinter.TclError, self.paned.insert, 0, None) + self.assertRaises(tkinter.TclError, self.paned.insert, 0, 0) + + child = ttk.Label(self.root) + child2 = ttk.Label(self.root) + child3 = ttk.Label(self.root) + + self.assertRaises(tkinter.TclError, self.paned.insert, 0, child) + + self.paned.insert('end', child2) + self.paned.insert(0, child) + self.assertEqual(self.paned.panes(), (str(child), str(child2))) + + self.paned.insert(0, child2) + self.assertEqual(self.paned.panes(), (str(child2), str(child))) + + self.paned.insert('end', child3) + self.assertEqual(self.paned.panes(), + (str(child2), str(child), str(child3))) + + # reinserting a child should move it to its current position + panes = self.paned.panes() + self.paned.insert('end', child3) + self.assertEqual(panes, self.paned.panes()) + + # moving child3 to child2 position should result in child2 ending up + # in previous child position and child ending up in previous child3 + # position + self.paned.insert(child2, child3) + self.assertEqual(self.paned.panes(), + (str(child3), str(child2), str(child))) + + + def test_pane(self): + self.assertRaises(tkinter.TclError, self.paned.pane, 0) + + child = ttk.Label(self.root) + self.paned.add(child) + self.assertIsInstance(self.paned.pane(0), dict) + self.assertEqual(self.paned.pane(0, weight=None), + 0 if self.wantobjects else '0') + # newer form for querying a single option + self.assertEqual(self.paned.pane(0, 'weight'), + 0 if self.wantobjects else '0') + self.assertEqual(self.paned.pane(0), self.paned.pane(str(child))) + + self.assertRaises(tkinter.TclError, self.paned.pane, 0, + badoption='somevalue') + + + def test_sashpos(self): + self.assertRaises(tkinter.TclError, self.paned.sashpos, None) + self.assertRaises(tkinter.TclError, self.paned.sashpos, '') + self.assertRaises(tkinter.TclError, self.paned.sashpos, 0) + + child = ttk.Label(self.paned, text='a') + self.paned.add(child, weight=1) + self.assertRaises(tkinter.TclError, self.paned.sashpos, 0) + child2 = ttk.Label(self.paned, text='b') + self.paned.add(child2) + self.assertRaises(tkinter.TclError, self.paned.sashpos, 1) + + self.paned.pack(expand=True, fill='both') + self.paned.wait_visibility() + + curr_pos = self.paned.sashpos(0) + self.paned.sashpos(0, 1000) + self.assertNotEqual(curr_pos, self.paned.sashpos(0)) + self.assertIsInstance(self.paned.sashpos(0), int) + + + at add_standard_options(StandardTtkOptionsTests) +class RadiobuttonTest(AbstractLabelTest, unittest.TestCase): + OPTIONS = ( + 'class', 'command', 'compound', 'cursor', + 'image', + 'padding', 'state', 'style', + 'takefocus', 'text', 'textvariable', + 'underline', 'value', 'variable', 'width', + ) + + def create(self, **kwargs): + return ttk.Radiobutton(self.root, **kwargs) + + def test_value(self): + widget = self.create() + self.checkParams(widget, 'value', 1, 2.3, '', 'any string') + + def test_invoke(self): + success = [] + def cb_test(): + success.append(1) + return "cb test called" + + myvar = tkinter.IntVar(self.root) + cbtn = ttk.Radiobutton(self.root, command=cb_test, + variable=myvar, value=0) + cbtn2 = ttk.Radiobutton(self.root, command=cb_test, + variable=myvar, value=1) + + if self.wantobjects: + conv = lambda x: x + else: + conv = int + + res = cbtn.invoke() + self.assertEqual(res, "cb test called") + self.assertEqual(conv(cbtn['value']), myvar.get()) + self.assertEqual(myvar.get(), + conv(cbtn.tk.globalgetvar(cbtn['variable']))) + self.assertTrue(success) + + cbtn2['command'] = '' + res = cbtn2.invoke() + self.assertEqual(str(res), '') + self.assertLessEqual(len(success), 1) + self.assertEqual(conv(cbtn2['value']), myvar.get()) + self.assertEqual(myvar.get(), + conv(cbtn.tk.globalgetvar(cbtn['variable']))) + + self.assertEqual(str(cbtn['variable']), str(cbtn2['variable'])) + + +class MenubuttonTest(AbstractLabelTest, unittest.TestCase): + OPTIONS = ( + 'class', 'compound', 'cursor', 'direction', + 'image', 'menu', 'padding', 'state', 'style', + 'takefocus', 'text', 'textvariable', + 'underline', 'width', + ) + + def create(self, **kwargs): + return ttk.Menubutton(self.root, **kwargs) + + def test_direction(self): + widget = self.create() + self.checkEnumParam(widget, 'direction', + 'above', 'below', 'left', 'right', 'flush') + + def test_menu(self): + widget = self.create() + menu = tkinter.Menu(widget, name='menu') + self.checkParam(widget, 'menu', menu, conv=str) + menu.destroy() + + + at add_standard_options(StandardTtkOptionsTests) +class ScaleTest(AbstractWidgetTest, unittest.TestCase): + OPTIONS = ( + 'class', 'command', 'cursor', 'from', 'length', + 'orient', 'style', 'takefocus', 'to', 'value', 'variable', + ) + _conv_pixels = noconv + default_orient = 'horizontal' + + def setUp(self): + super().setUp() + self.scale = self.create() + self.scale.pack() + self.scale.update() + + def create(self, **kwargs): + return ttk.Scale(self.root, **kwargs) + + def test_from(self): + widget = self.create() + self.checkFloatParam(widget, 'from', 100, 14.9, 15.1, conv=False) + + def test_length(self): + widget = self.create() + self.checkPixelsParam(widget, 'length', 130, 131.2, 135.6, '5i') + + def test_to(self): + widget = self.create() + self.checkFloatParam(widget, 'to', 300, 14.9, 15.1, -10, conv=False) + + def test_value(self): + widget = self.create() + self.checkFloatParam(widget, 'value', 300, 14.9, 15.1, -10, conv=False) + + def test_custom_event(self): + failure = [1, 1, 1] # will need to be empty + + funcid = self.scale.bind('<<RangeChanged>>', lambda evt: failure.pop()) + + self.scale['from'] = 10 + self.scale['from_'] = 10 + self.scale['to'] = 3 + + self.assertFalse(failure) + + failure = [1, 1, 1] + self.scale.configure(from_=2, to=5) + self.scale.configure(from_=0, to=-2) + self.scale.configure(to=10) + + self.assertFalse(failure) + + + def test_get(self): + if self.wantobjects: + conv = lambda x: x + else: + conv = float + + scale_width = self.scale.winfo_width() + self.assertEqual(self.scale.get(scale_width, 0), self.scale['to']) + + self.assertEqual(conv(self.scale.get(0, 0)), conv(self.scale['from'])) + self.assertEqual(self.scale.get(), self.scale['value']) + self.scale['value'] = 30 + self.assertEqual(self.scale.get(), self.scale['value']) + + self.assertRaises(tkinter.TclError, self.scale.get, '', 0) + self.assertRaises(tkinter.TclError, self.scale.get, 0, '') + + + def test_set(self): + if self.wantobjects: + conv = lambda x: x + else: + conv = float + + # set restricts the max/min values according to the current range + max = conv(self.scale['to']) + new_max = max + 10 + self.scale.set(new_max) + self.assertEqual(conv(self.scale.get()), max) + min = conv(self.scale['from']) + self.scale.set(min - 1) + self.assertEqual(conv(self.scale.get()), min) + + # changing directly the variable doesn't impose this limitation tho + var = tkinter.DoubleVar(self.root) + self.scale['variable'] = var + var.set(max + 5) + self.assertEqual(conv(self.scale.get()), var.get()) + self.assertEqual(conv(self.scale.get()), max + 5) + del var + + # the same happens with the value option + self.scale['value'] = max + 10 + self.assertEqual(conv(self.scale.get()), max + 10) + self.assertEqual(conv(self.scale.get()), conv(self.scale['value'])) + + # nevertheless, note that the max/min values we can get specifying + # x, y coords are the ones according to the current range + self.assertEqual(conv(self.scale.get(0, 0)), min) + self.assertEqual(conv(self.scale.get(self.scale.winfo_width(), 0)), max) + + self.assertRaises(tkinter.TclError, self.scale.set, None) + + + at add_standard_options(StandardTtkOptionsTests) +class ProgressbarTest(AbstractWidgetTest, unittest.TestCase): + OPTIONS = ( + 'class', 'cursor', 'orient', 'length', + 'mode', 'maximum', 'phase', + 'style', 'takefocus', 'value', 'variable', + ) + _conv_pixels = noconv + default_orient = 'horizontal' + + def create(self, **kwargs): + return ttk.Progressbar(self.root, **kwargs) + + def test_length(self): + widget = self.create() + self.checkPixelsParam(widget, 'length', 100.1, 56.7, '2i') + + def test_maximum(self): + widget = self.create() + self.checkFloatParam(widget, 'maximum', 150.2, 77.7, 0, -10, conv=False) + + def test_mode(self): + widget = self.create() + self.checkEnumParam(widget, 'mode', 'determinate', 'indeterminate') + + def test_phase(self): + # XXX + pass + + def test_value(self): + widget = self.create() + self.checkFloatParam(widget, 'value', 150.2, 77.7, 0, -10, + conv=False) + + + at unittest.skipIf(sys.platform == 'darwin', + 'ttk.Scrollbar is special on MacOSX') + at add_standard_options(StandardTtkOptionsTests) +class ScrollbarTest(AbstractWidgetTest, unittest.TestCase): + OPTIONS = ( + 'class', 'command', 'cursor', 'orient', 'style', 'takefocus', + ) + default_orient = 'vertical' + + def create(self, **kwargs): + return ttk.Scrollbar(self.root, **kwargs) + + + at add_standard_options(IntegerSizeTests, StandardTtkOptionsTests) +class NotebookTest(AbstractWidgetTest, unittest.TestCase): + OPTIONS = ( + 'class', 'cursor', 'height', 'padding', 'style', 'takefocus', 'width', + ) + + def setUp(self): + super().setUp() + self.nb = self.create(padding=0) + self.child1 = ttk.Label(self.root) + self.child2 = ttk.Label(self.root) + self.nb.add(self.child1, text='a') + self.nb.add(self.child2, text='b') + + def create(self, **kwargs): + return ttk.Notebook(self.root, **kwargs) + + def test_tab_identifiers(self): + self.nb.forget(0) + self.nb.hide(self.child2) + self.assertRaises(tkinter.TclError, self.nb.tab, self.child1) + self.assertEqual(self.nb.index('end'), 1) + self.nb.add(self.child2) + self.assertEqual(self.nb.index('end'), 1) + self.nb.select(self.child2) + + self.assertTrue(self.nb.tab('current')) + self.nb.add(self.child1, text='a') + + self.nb.pack() + self.nb.wait_visibility() + if sys.platform == 'darwin': + tb_idx = "@20,5" + else: + tb_idx = "@5,5" + self.assertEqual(self.nb.tab(tb_idx), self.nb.tab('current')) + + for i in range(5, 100, 5): + try: + if self.nb.tab('@%d, 5' % i, text=None) == 'a': + break + except tkinter.TclError: + pass + + else: + self.fail("Tab with text 'a' not found") + + + def test_add_and_hidden(self): + self.assertRaises(tkinter.TclError, self.nb.hide, -1) + self.assertRaises(tkinter.TclError, self.nb.hide, 'hi') + self.assertRaises(tkinter.TclError, self.nb.hide, None) + self.assertRaises(tkinter.TclError, self.nb.add, None) + self.assertRaises(tkinter.TclError, self.nb.add, ttk.Label(self.root), + unknown='option') + + tabs = self.nb.tabs() + self.nb.hide(self.child1) + self.nb.add(self.child1) + self.assertEqual(self.nb.tabs(), tabs) + + child = ttk.Label(self.root) + self.nb.add(child, text='c') + tabs = self.nb.tabs() + + curr = self.nb.index('current') + # verify that the tab gets readded at its previous position + child2_index = self.nb.index(self.child2) + self.nb.hide(self.child2) + self.nb.add(self.child2) + self.assertEqual(self.nb.tabs(), tabs) + self.assertEqual(self.nb.index(self.child2), child2_index) + self.assertEqual(str(self.child2), self.nb.tabs()[child2_index]) + # but the tab next to it (not hidden) is the one selected now + self.assertEqual(self.nb.index('current'), curr + 1) + + + def test_forget(self): + self.assertRaises(tkinter.TclError, self.nb.forget, -1) + self.assertRaises(tkinter.TclError, self.nb.forget, 'hi') + self.assertRaises(tkinter.TclError, self.nb.forget, None) + + tabs = self.nb.tabs() + child1_index = self.nb.index(self.child1) + self.nb.forget(self.child1) + self.assertNotIn(str(self.child1), self.nb.tabs()) + self.assertEqual(len(tabs) - 1, len(self.nb.tabs())) + + self.nb.add(self.child1) + self.assertEqual(self.nb.index(self.child1), 1) + self.assertNotEqual(child1_index, self.nb.index(self.child1)) + + + def test_index(self): + self.assertRaises(tkinter.TclError, self.nb.index, -1) + self.assertRaises(tkinter.TclError, self.nb.index, None) + + self.assertIsInstance(self.nb.index('end'), int) + self.assertEqual(self.nb.index(self.child1), 0) + self.assertEqual(self.nb.index(self.child2), 1) + self.assertEqual(self.nb.index('end'), 2) + + + def test_insert(self): + # moving tabs + tabs = self.nb.tabs() + self.nb.insert(1, tabs[0]) + self.assertEqual(self.nb.tabs(), (tabs[1], tabs[0])) + self.nb.insert(self.child1, self.child2) + self.assertEqual(self.nb.tabs(), tabs) + self.nb.insert('end', self.child1) + self.assertEqual(self.nb.tabs(), (tabs[1], tabs[0])) + self.nb.insert('end', 0) + self.assertEqual(self.nb.tabs(), tabs) + # bad moves + self.assertRaises(tkinter.TclError, self.nb.insert, 2, tabs[0]) + self.assertRaises(tkinter.TclError, self.nb.insert, -1, tabs[0]) + + # new tab + child3 = ttk.Label(self.root) + self.nb.insert(1, child3) + self.assertEqual(self.nb.tabs(), (tabs[0], str(child3), tabs[1])) + self.nb.forget(child3) + self.assertEqual(self.nb.tabs(), tabs) + self.nb.insert(self.child1, child3) + self.assertEqual(self.nb.tabs(), (str(child3), ) + tabs) + self.nb.forget(child3) + self.assertRaises(tkinter.TclError, self.nb.insert, 2, child3) + self.assertRaises(tkinter.TclError, self.nb.insert, -1, child3) + + # bad inserts + self.assertRaises(tkinter.TclError, self.nb.insert, 'end', None) + self.assertRaises(tkinter.TclError, self.nb.insert, None, 0) + self.assertRaises(tkinter.TclError, self.nb.insert, None, None) + + + def test_select(self): + self.nb.pack() + self.nb.wait_visibility() + + success = [] + tab_changed = [] + + self.child1.bind('<Unmap>', lambda evt: success.append(True)) + self.nb.bind('<<NotebookTabChanged>>', + lambda evt: tab_changed.append(True)) + + self.assertEqual(self.nb.select(), str(self.child1)) + self.nb.select(self.child2) + self.assertTrue(success) + self.assertEqual(self.nb.select(), str(self.child2)) + + self.nb.update() + self.assertTrue(tab_changed) + + + def test_tab(self): + self.assertRaises(tkinter.TclError, self.nb.tab, -1) + self.assertRaises(tkinter.TclError, self.nb.tab, 'notab') + self.assertRaises(tkinter.TclError, self.nb.tab, None) + + self.assertIsInstance(self.nb.tab(self.child1), dict) + self.assertEqual(self.nb.tab(self.child1, text=None), 'a') + # newer form for querying a single option + self.assertEqual(self.nb.tab(self.child1, 'text'), 'a') + self.nb.tab(self.child1, text='abc') + self.assertEqual(self.nb.tab(self.child1, text=None), 'abc') + self.assertEqual(self.nb.tab(self.child1, 'text'), 'abc') + + + def test_tabs(self): + self.assertEqual(len(self.nb.tabs()), 2) + + self.nb.forget(self.child1) + self.nb.forget(self.child2) + + self.assertEqual(self.nb.tabs(), ()) + + + def test_traversal(self): + self.nb.pack() + self.nb.wait_visibility() + + self.nb.select(0) + + simulate_mouse_click(self.nb, 5, 5) + self.nb.focus_force() + self.nb.event_generate('<Control-Tab>') + self.assertEqual(self.nb.select(), str(self.child2)) + self.nb.focus_force() + self.nb.event_generate('<Shift-Control-Tab>') + self.assertEqual(self.nb.select(), str(self.child1)) + self.nb.focus_force() + self.nb.event_generate('<Shift-Control-Tab>') + self.assertEqual(self.nb.select(), str(self.child2)) + + self.nb.tab(self.child1, text='a', underline=0) + self.nb.enable_traversal() + self.nb.focus_force() + simulate_mouse_click(self.nb, 5, 5) + if sys.platform == 'darwin': + self.nb.event_generate('<Option-a>') + else: + self.nb.event_generate('<Alt-a>') + self.assertEqual(self.nb.select(), str(self.child1)) + + at add_standard_options(IntegerSizeTests, StandardTtkOptionsTests) +class SpinboxTest(EntryTest, unittest.TestCase): + OPTIONS = ( + 'background', 'class', 'command', 'cursor', 'exportselection', + 'font', 'foreground', 'format', 'from', 'increment', + 'invalidcommand', 'justify', 'show', 'state', 'style', + 'takefocus', 'textvariable', 'to', 'validate', 'validatecommand', + 'values', 'width', 'wrap', 'xscrollcommand', + ) + + def setUp(self): + super().setUp() + self.spin = self.create() + self.spin.pack() + + def create(self, **kwargs): + return ttk.Spinbox(self.root, **kwargs) + + def _click_increment_arrow(self): + width = self.spin.winfo_width() + height = self.spin.winfo_height() + x = width - 5 + y = height//2 - 5 + self.spin.event_generate('<ButtonPress-1>', x=x, y=y) + self.spin.event_generate('<ButtonRelease-1>', x=x, y=y) + self.spin.update_idletasks() + + def _click_decrement_arrow(self): + width = self.spin.winfo_width() + height = self.spin.winfo_height() + x = width - 5 + y = height//2 + 4 + self.spin.event_generate('<ButtonPress-1>', x=x, y=y) + self.spin.event_generate('<ButtonRelease-1>', x=x, y=y) + self.spin.update_idletasks() + + def test_command(self): + success = [] + + self.spin['command'] = lambda: success.append(True) + self.spin.update() + self._click_increment_arrow() + self.spin.update() + self.assertTrue(success) + + self._click_decrement_arrow() + self.assertEqual(len(success), 2) + + # testing postcommand removal + self.spin['command'] = '' + self.spin.update_idletasks() + self._click_increment_arrow() + self._click_decrement_arrow() + self.spin.update() + self.assertEqual(len(success), 2) + + def test_to(self): + self.spin['from'] = 0 + self.spin['to'] = 5 + self.spin.set(4) + self.spin.update() + self._click_increment_arrow() # 5 + + self.assertEqual(self.spin.get(), '5') + + self._click_increment_arrow() # 5 + self.assertEqual(self.spin.get(), '5') + + def test_from(self): + self.spin['from'] = 1 + self.spin['to'] = 10 + self.spin.set(2) + self.spin.update() + self._click_decrement_arrow() # 1 + self.assertEqual(self.spin.get(), '1') + self._click_decrement_arrow() # 1 + self.assertEqual(self.spin.get(), '1') + + def test_increment(self): + self.spin['from'] = 0 + self.spin['to'] = 10 + self.spin['increment'] = 4 + self.spin.set(1) + self.spin.update() + + self._click_increment_arrow() # 5 + self.assertEqual(self.spin.get(), '5') + self.spin['increment'] = 2 + self.spin.update() + self._click_decrement_arrow() # 3 + self.assertEqual(self.spin.get(), '3') + + def test_format(self): + self.spin.set(1) + self.spin['format'] = '%10.3f' + self.spin.update() + self._click_increment_arrow() + value = self.spin.get() + + self.assertEqual(len(value), 10) + self.assertEqual(value.index('.'), 6) + + self.spin['format'] = '' + self.spin.update() + self._click_increment_arrow() + value = self.spin.get() + self.assertTrue('.' not in value) + self.assertEqual(len(value), 1) + + def test_wrap(self): + self.spin['to'] = 10 + self.spin['from'] = 1 + self.spin.set(1) + self.spin['wrap'] = True + self.spin.update() + + self._click_decrement_arrow() + self.assertEqual(self.spin.get(), '10') + + self._click_increment_arrow() + self.assertEqual(self.spin.get(), '1') + + self.spin['wrap'] = False + self.spin.update() + + self._click_decrement_arrow() + self.assertEqual(self.spin.get(), '1') + + def test_values(self): + self.assertEqual(self.spin['values'], + () if tcl_version < (8, 5) else '') + self.checkParam(self.spin, 'values', 'mon tue wed thur', + expected=('mon', 'tue', 'wed', 'thur')) + self.checkParam(self.spin, 'values', ('mon', 'tue', 'wed', 'thur')) + self.checkParam(self.spin, 'values', (42, 3.14, '', 'any string')) + self.checkParam( + self.spin, + 'values', + '', + expected='' if get_tk_patchlevel() < (8, 5, 10) else () + ) + + self.spin['values'] = ['a', 1, 'c'] + + # test incrementing / decrementing values + self.spin.set('a') + self.spin.update() + self._click_increment_arrow() + self.assertEqual(self.spin.get(), '1') + + self._click_decrement_arrow() + self.assertEqual(self.spin.get(), 'a') + + # testing values with empty string set through configure + self.spin.configure(values=[1, '', 2]) + self.assertEqual(self.spin['values'], + ('1', '', '2') if self.wantobjects else + '1 {} 2') + + # testing values with spaces + self.spin['values'] = ['a b', 'a\tb', 'a\nb'] + self.assertEqual(self.spin['values'], + ('a b', 'a\tb', 'a\nb') if self.wantobjects else + '{a b} {a\tb} {a\nb}') + + # testing values with special characters + self.spin['values'] = [r'a\tb', '"a"', '} {'] + self.assertEqual(self.spin['values'], + (r'a\tb', '"a"', '} {') if self.wantobjects else + r'a\\tb {"a"} \}\ \{') + + # testing creating spinbox with empty string in values + spin2 = ttk.Spinbox(self.root, values=[1, 2, '']) + self.assertEqual(spin2['values'], + ('1', '2', '') if self.wantobjects else '1 2 {}') + spin2.destroy() + + + at add_standard_options(StandardTtkOptionsTests) +class TreeviewTest(AbstractWidgetTest, unittest.TestCase): + OPTIONS = ( + 'class', 'columns', 'cursor', 'displaycolumns', + 'height', 'padding', 'selectmode', 'show', + 'style', 'takefocus', 'xscrollcommand', 'yscrollcommand', + ) + + def setUp(self): + super().setUp() + self.tv = self.create(padding=0) + + def create(self, **kwargs): + return ttk.Treeview(self.root, **kwargs) + + def test_columns(self): + widget = self.create() + self.checkParam(widget, 'columns', 'a b c', + expected=('a', 'b', 'c')) + self.checkParam(widget, 'columns', ('a', 'b', 'c')) + self.checkParam(widget, 'columns', (), + expected='' if get_tk_patchlevel() < (8, 5, 10) else ()) + + def test_displaycolumns(self): + widget = self.create() + widget['columns'] = ('a', 'b', 'c') + self.checkParam(widget, 'displaycolumns', 'b a c', + expected=('b', 'a', 'c')) + self.checkParam(widget, 'displaycolumns', ('b', 'a', 'c')) + self.checkParam(widget, 'displaycolumns', '#all', + expected=('#all',)) + self.checkParam(widget, 'displaycolumns', (2, 1, 0)) + self.checkInvalidParam(widget, 'displaycolumns', ('a', 'b', 'd'), + errmsg='Invalid column index d') + self.checkInvalidParam(widget, 'displaycolumns', (1, 2, 3), + errmsg='Column index 3 out of bounds') + self.checkInvalidParam(widget, 'displaycolumns', (1, -2), + errmsg='Column index -2 out of bounds') + + def test_height(self): + widget = self.create() + self.checkPixelsParam(widget, 'height', 100, -100, 0, '3c', conv=False) + self.checkPixelsParam(widget, 'height', 101.2, 102.6, conv=noconv) + + def test_selectmode(self): + widget = self.create() + self.checkEnumParam(widget, 'selectmode', + 'none', 'browse', 'extended') + + def test_show(self): + widget = self.create() + self.checkParam(widget, 'show', 'tree headings', + expected=('tree', 'headings')) + self.checkParam(widget, 'show', ('tree', 'headings')) + self.checkParam(widget, 'show', ('headings', 'tree')) + self.checkParam(widget, 'show', 'tree', expected=('tree',)) + self.checkParam(widget, 'show', 'headings', expected=('headings',)) + + def test_bbox(self): + self.tv.pack() + self.assertEqual(self.tv.bbox(''), '') + self.tv.wait_visibility() + self.tv.update() + + item_id = self.tv.insert('', 'end') + children = self.tv.get_children() + self.assertTrue(children) + + bbox = self.tv.bbox(children[0]) + self.assertIsBoundingBox(bbox) + + # compare width in bboxes + self.tv['columns'] = ['test'] + self.tv.column('test', width=50) + bbox_column0 = self.tv.bbox(children[0], 0) + root_width = self.tv.column('#0', width=None) + if not self.wantobjects: + root_width = int(root_width) + self.assertEqual(bbox_column0[0], bbox[0] + root_width) + + # verify that bbox of a closed item is the empty string + child1 = self.tv.insert(item_id, 'end') + self.assertEqual(self.tv.bbox(child1), '') + + + def test_children(self): + # no children yet, should get an empty tuple + self.assertEqual(self.tv.get_children(), ()) + + item_id = self.tv.insert('', 'end') + self.assertIsInstance(self.tv.get_children(), tuple) + self.assertEqual(self.tv.get_children()[0], item_id) + + # add item_id and child3 as children of child2 + child2 = self.tv.insert('', 'end') + child3 = self.tv.insert('', 'end') + self.tv.set_children(child2, item_id, child3) + self.assertEqual(self.tv.get_children(child2), (item_id, child3)) + + # child3 has child2 as parent, thus trying to set child2 as a children + # of child3 should result in an error + self.assertRaises(tkinter.TclError, + self.tv.set_children, child3, child2) + + # remove child2 children + self.tv.set_children(child2) + self.assertEqual(self.tv.get_children(child2), ()) + + # remove root's children + self.tv.set_children('') + self.assertEqual(self.tv.get_children(), ()) + + + def test_column(self): + # return a dict with all options/values + self.assertIsInstance(self.tv.column('#0'), dict) + # return a single value of the given option + if self.wantobjects: + self.assertIsInstance(self.tv.column('#0', width=None), int) + # set a new value for an option + self.tv.column('#0', width=10) + # testing new way to get option value + self.assertEqual(self.tv.column('#0', 'width'), + 10 if self.wantobjects else '10') + self.assertEqual(self.tv.column('#0', width=None), + 10 if self.wantobjects else '10') + # check read-only option + self.assertRaises(tkinter.TclError, self.tv.column, '#0', id='X') + + self.assertRaises(tkinter.TclError, self.tv.column, 'invalid') + invalid_kws = [ + {'unknown_option': 'some value'}, {'stretch': 'wrong'}, + {'anchor': 'wrong'}, {'width': 'wrong'}, {'minwidth': 'wrong'} + ] + for kw in invalid_kws: + self.assertRaises(tkinter.TclError, self.tv.column, '#0', + **kw) + + + def test_delete(self): + self.assertRaises(tkinter.TclError, self.tv.delete, '#0') + + item_id = self.tv.insert('', 'end') + item2 = self.tv.insert(item_id, 'end') + self.assertEqual(self.tv.get_children(), (item_id, )) + self.assertEqual(self.tv.get_children(item_id), (item2, )) + + self.tv.delete(item_id) + self.assertFalse(self.tv.get_children()) + + # reattach should fail + self.assertRaises(tkinter.TclError, + self.tv.reattach, item_id, '', 'end') + + # test multiple item delete + item1 = self.tv.insert('', 'end') + item2 = self.tv.insert('', 'end') + self.assertEqual(self.tv.get_children(), (item1, item2)) + + self.tv.delete(item1, item2) + self.assertFalse(self.tv.get_children()) + + + def test_detach_reattach(self): + item_id = self.tv.insert('', 'end') + item2 = self.tv.insert(item_id, 'end') + + # calling detach without items is valid, although it does nothing + prev = self.tv.get_children() + self.tv.detach() # this should do nothing + self.assertEqual(prev, self.tv.get_children()) + + self.assertEqual(self.tv.get_children(), (item_id, )) + self.assertEqual(self.tv.get_children(item_id), (item2, )) + + # detach item with children + self.tv.detach(item_id) + self.assertFalse(self.tv.get_children()) + + # reattach item with children + self.tv.reattach(item_id, '', 'end') + self.assertEqual(self.tv.get_children(), (item_id, )) + self.assertEqual(self.tv.get_children(item_id), (item2, )) + + # move a children to the root + self.tv.move(item2, '', 'end') + self.assertEqual(self.tv.get_children(), (item_id, item2)) + self.assertEqual(self.tv.get_children(item_id), ()) + + # bad values + self.assertRaises(tkinter.TclError, + self.tv.reattach, 'nonexistent', '', 'end') + self.assertRaises(tkinter.TclError, + self.tv.detach, 'nonexistent') + self.assertRaises(tkinter.TclError, + self.tv.reattach, item2, 'otherparent', 'end') + self.assertRaises(tkinter.TclError, + self.tv.reattach, item2, '', 'invalid') + + # multiple detach + self.tv.detach(item_id, item2) + self.assertEqual(self.tv.get_children(), ()) + self.assertEqual(self.tv.get_children(item_id), ()) + + + def test_exists(self): + self.assertEqual(self.tv.exists('something'), False) + self.assertEqual(self.tv.exists(''), True) + self.assertEqual(self.tv.exists({}), False) + + # the following will make a tk.call equivalent to + # tk.call(treeview, "exists") which should result in an error + # in the tcl interpreter since tk requires an item. + self.assertRaises(tkinter.TclError, self.tv.exists, None) + + + def test_focus(self): + # nothing is focused right now + self.assertEqual(self.tv.focus(), '') + + item1 = self.tv.insert('', 'end') + self.tv.focus(item1) + self.assertEqual(self.tv.focus(), item1) + + self.tv.delete(item1) + self.assertEqual(self.tv.focus(), '') + + # try focusing inexistent item + self.assertRaises(tkinter.TclError, self.tv.focus, 'hi') + + + def test_heading(self): + # check a dict is returned + self.assertIsInstance(self.tv.heading('#0'), dict) + + # check a value is returned + self.tv.heading('#0', text='hi') + self.assertEqual(self.tv.heading('#0', 'text'), 'hi') + self.assertEqual(self.tv.heading('#0', text=None), 'hi') + + # invalid option + self.assertRaises(tkinter.TclError, self.tv.heading, '#0', + background=None) + # invalid value + self.assertRaises(tkinter.TclError, self.tv.heading, '#0', + anchor=1) + + def test_heading_callback(self): + def simulate_heading_click(x, y): + simulate_mouse_click(self.tv, x, y) + self.tv.update() + + success = [] # no success for now + + self.tv.pack() + self.tv.wait_visibility() + self.tv.heading('#0', command=lambda: success.append(True)) + self.tv.column('#0', width=100) + self.tv.update() + + # assuming that the coords (5, 5) fall into heading #0 + simulate_heading_click(5, 5) + if not success: + self.fail("The command associated to the treeview heading wasn't " + "invoked.") + + success = [] + commands = self.tv.master._tclCommands + self.tv.heading('#0', command=str(self.tv.heading('#0', command=None))) + self.assertEqual(commands, self.tv.master._tclCommands) + simulate_heading_click(5, 5) + if not success: + self.fail("The command associated to the treeview heading wasn't " + "invoked.") + + # XXX The following raises an error in a tcl interpreter, but not in + # Python + #self.tv.heading('#0', command='I dont exist') + #simulate_heading_click(5, 5) + + + def test_index(self): + # item 'what' doesn't exist + self.assertRaises(tkinter.TclError, self.tv.index, 'what') + + self.assertEqual(self.tv.index(''), 0) + + item1 = self.tv.insert('', 'end') + item2 = self.tv.insert('', 'end') + c1 = self.tv.insert(item1, 'end') + c2 = self.tv.insert(item1, 'end') + self.assertEqual(self.tv.index(item1), 0) + self.assertEqual(self.tv.index(c1), 0) + self.assertEqual(self.tv.index(c2), 1) + self.assertEqual(self.tv.index(item2), 1) + + self.tv.move(item2, '', 0) + self.assertEqual(self.tv.index(item2), 0) + self.assertEqual(self.tv.index(item1), 1) + + # check that index still works even after its parent and siblings + # have been detached + self.tv.detach(item1) + self.assertEqual(self.tv.index(c2), 1) + self.tv.detach(c1) + self.assertEqual(self.tv.index(c2), 0) + + # but it fails after item has been deleted + self.tv.delete(item1) + self.assertRaises(tkinter.TclError, self.tv.index, c2) + + + def test_insert_item(self): + # parent 'none' doesn't exist + self.assertRaises(tkinter.TclError, self.tv.insert, 'none', 'end') + + # open values + self.assertRaises(tkinter.TclError, self.tv.insert, '', 'end', + open='') + self.assertRaises(tkinter.TclError, self.tv.insert, '', 'end', + open='please') + self.assertFalse(self.tv.delete(self.tv.insert('', 'end', open=True))) + self.assertFalse(self.tv.delete(self.tv.insert('', 'end', open=False))) + + # invalid index + self.assertRaises(tkinter.TclError, self.tv.insert, '', 'middle') + + # trying to duplicate item id is invalid + itemid = self.tv.insert('', 'end', 'first-item') + self.assertEqual(itemid, 'first-item') + self.assertRaises(tkinter.TclError, self.tv.insert, '', 'end', + 'first-item') + self.assertRaises(tkinter.TclError, self.tv.insert, '', 'end', + MockTclObj('first-item')) + + # unicode values + value = '\xe1ba' + item = self.tv.insert('', 'end', values=(value, )) + self.assertEqual(self.tv.item(item, 'values'), + (value,) if self.wantobjects else value) + self.assertEqual(self.tv.item(item, values=None), + (value,) if self.wantobjects else value) + + self.tv.item(item, values=self.root.splitlist(self.tv.item(item, values=None))) + self.assertEqual(self.tv.item(item, values=None), + (value,) if self.wantobjects else value) + + self.assertIsInstance(self.tv.item(item), dict) + + # erase item values + self.tv.item(item, values='') + self.assertFalse(self.tv.item(item, values=None)) + + # item tags + item = self.tv.insert('', 'end', tags=[1, 2, value]) + self.assertEqual(self.tv.item(item, tags=None), + ('1', '2', value) if self.wantobjects else + '1 2 %s' % value) + self.tv.item(item, tags=[]) + self.assertFalse(self.tv.item(item, tags=None)) + self.tv.item(item, tags=(1, 2)) + self.assertEqual(self.tv.item(item, tags=None), + ('1', '2') if self.wantobjects else '1 2') + + # values with spaces + item = self.tv.insert('', 'end', values=('a b c', + '%s %s' % (value, value))) + self.assertEqual(self.tv.item(item, values=None), + ('a b c', '%s %s' % (value, value)) if self.wantobjects else + '{a b c} {%s %s}' % (value, value)) + + # text + self.assertEqual(self.tv.item( + self.tv.insert('', 'end', text="Label here"), text=None), + "Label here") + self.assertEqual(self.tv.item( + self.tv.insert('', 'end', text=value), text=None), + value) + + # test for values which are not None + itemid = self.tv.insert('', 'end', 0) + self.assertEqual(itemid, '0') + itemid = self.tv.insert('', 'end', 0.0) + self.assertEqual(itemid, '0.0') + # this is because False resolves to 0 and element with 0 iid is already present + self.assertRaises(tkinter.TclError, self.tv.insert, '', 'end', False) + self.assertRaises(tkinter.TclError, self.tv.insert, '', 'end', '') + + + def test_selection(self): + self.assertRaises(TypeError, self.tv.selection, 'spam') + # item 'none' doesn't exist + self.assertRaises(tkinter.TclError, self.tv.selection_set, 'none') + self.assertRaises(tkinter.TclError, self.tv.selection_add, 'none') + self.assertRaises(tkinter.TclError, self.tv.selection_remove, 'none') + self.assertRaises(tkinter.TclError, self.tv.selection_toggle, 'none') + + item1 = self.tv.insert('', 'end') + item2 = self.tv.insert('', 'end') + c1 = self.tv.insert(item1, 'end') + c2 = self.tv.insert(item1, 'end') + c3 = self.tv.insert(item1, 'end') + self.assertEqual(self.tv.selection(), ()) + + self.tv.selection_set(c1, item2) + self.assertEqual(self.tv.selection(), (c1, item2)) + self.tv.selection_set(c2) + self.assertEqual(self.tv.selection(), (c2,)) + + self.tv.selection_add(c1, item2) + self.assertEqual(self.tv.selection(), (c1, c2, item2)) + self.tv.selection_add(item1) + self.assertEqual(self.tv.selection(), (item1, c1, c2, item2)) + self.tv.selection_add() + self.assertEqual(self.tv.selection(), (item1, c1, c2, item2)) + + self.tv.selection_remove(item1, c3) + self.assertEqual(self.tv.selection(), (c1, c2, item2)) + self.tv.selection_remove(c2) + self.assertEqual(self.tv.selection(), (c1, item2)) + self.tv.selection_remove() + self.assertEqual(self.tv.selection(), (c1, item2)) + + self.tv.selection_toggle(c1, c3) + self.assertEqual(self.tv.selection(), (c3, item2)) + self.tv.selection_toggle(item2) + self.assertEqual(self.tv.selection(), (c3,)) + self.tv.selection_toggle() + self.assertEqual(self.tv.selection(), (c3,)) + + self.tv.insert('', 'end', id='with spaces') + self.tv.selection_set('with spaces') + self.assertEqual(self.tv.selection(), ('with spaces',)) + + self.tv.insert('', 'end', id='{brace') + self.tv.selection_set('{brace') + self.assertEqual(self.tv.selection(), ('{brace',)) + + self.tv.insert('', 'end', id='unicode\u20ac') + self.tv.selection_set('unicode\u20ac') + self.assertEqual(self.tv.selection(), ('unicode\u20ac',)) + + self.tv.insert('', 'end', id=b'bytes\xe2\x82\xac') + self.tv.selection_set(b'bytes\xe2\x82\xac') + self.assertEqual(self.tv.selection(), ('bytes\xe2\x82\xac',)) + + self.tv.selection_set() + self.assertEqual(self.tv.selection(), ()) + + # Old interface + self.tv.selection_set((c1, item2)) + self.assertEqual(self.tv.selection(), (c1, item2)) + self.tv.selection_add((c1, item1)) + self.assertEqual(self.tv.selection(), (item1, c1, item2)) + self.tv.selection_remove((item1, c3)) + self.assertEqual(self.tv.selection(), (c1, item2)) + self.tv.selection_toggle((c1, c3)) + self.assertEqual(self.tv.selection(), (c3, item2)) + + + def test_set(self): + self.tv['columns'] = ['A', 'B'] + item = self.tv.insert('', 'end', values=['a', 'b']) + self.assertEqual(self.tv.set(item), {'A': 'a', 'B': 'b'}) + + self.tv.set(item, 'B', 'a') + self.assertEqual(self.tv.item(item, values=None), + ('a', 'a') if self.wantobjects else 'a a') + + self.tv['columns'] = ['B'] + self.assertEqual(self.tv.set(item), {'B': 'a'}) + + self.tv.set(item, 'B', 'b') + self.assertEqual(self.tv.set(item, column='B'), 'b') + self.assertEqual(self.tv.item(item, values=None), + ('b', 'a') if self.wantobjects else 'b a') + + self.tv.set(item, 'B', 123) + self.assertEqual(self.tv.set(item, 'B'), + 123 if self.wantobjects else '123') + self.assertEqual(self.tv.item(item, values=None), + (123, 'a') if self.wantobjects else '123 a') + self.assertEqual(self.tv.set(item), + {'B': 123} if self.wantobjects else {'B': '123'}) + + # inexistent column + self.assertRaises(tkinter.TclError, self.tv.set, item, 'A') + self.assertRaises(tkinter.TclError, self.tv.set, item, 'A', 'b') + + # inexistent item + self.assertRaises(tkinter.TclError, self.tv.set, 'notme') + + + def test_tag_bind(self): + events = [] + item1 = self.tv.insert('', 'end', tags=['call']) + item2 = self.tv.insert('', 'end', tags=['call']) + self.tv.tag_bind('call', '<ButtonPress-1>', + lambda evt: events.append(1)) + self.tv.tag_bind('call', '<ButtonRelease-1>', + lambda evt: events.append(2)) + + self.tv.pack() + self.tv.wait_visibility() + self.tv.update() + + pos_y = set() + found = set() + for i in range(0, 100, 10): + if len(found) == 2: # item1 and item2 already found + break + item_id = self.tv.identify_row(i) + if item_id and item_id not in found: + pos_y.add(i) + found.add(item_id) + + self.assertEqual(len(pos_y), 2) # item1 and item2 y pos + for y in pos_y: + simulate_mouse_click(self.tv, 0, y) + + # by now there should be 4 things in the events list, since each + # item had a bind for two events that were simulated above + self.assertEqual(len(events), 4) + for evt in zip(events[::2], events[1::2]): + self.assertEqual(evt, (1, 2)) + + + def test_tag_configure(self): + # Just testing parameter passing for now + self.assertRaises(TypeError, self.tv.tag_configure) + self.assertRaises(tkinter.TclError, self.tv.tag_configure, + 'test', sky='blue') + self.tv.tag_configure('test', foreground='blue') + self.assertEqual(str(self.tv.tag_configure('test', 'foreground')), + 'blue') + self.assertEqual(str(self.tv.tag_configure('test', foreground=None)), + 'blue') + self.assertIsInstance(self.tv.tag_configure('test'), dict) + + def test_tag_has(self): + item1 = self.tv.insert('', 'end', text='Item 1', tags=['tag1']) + item2 = self.tv.insert('', 'end', text='Item 2', tags=['tag2']) + self.assertRaises(TypeError, self.tv.tag_has) + self.assertRaises(TclError, self.tv.tag_has, 'tag1', 'non-existing') + self.assertTrue(self.tv.tag_has('tag1', item1)) + self.assertFalse(self.tv.tag_has('tag1', item2)) + self.assertFalse(self.tv.tag_has('tag2', item1)) + self.assertTrue(self.tv.tag_has('tag2', item2)) + self.assertFalse(self.tv.tag_has('tag3', item1)) + self.assertFalse(self.tv.tag_has('tag3', item2)) + self.assertEqual(self.tv.tag_has('tag1'), (item1,)) + self.assertEqual(self.tv.tag_has('tag2'), (item2,)) + self.assertEqual(self.tv.tag_has('tag3'), ()) + + + at add_standard_options(StandardTtkOptionsTests) +class SeparatorTest(AbstractWidgetTest, unittest.TestCase): + OPTIONS = ( + 'class', 'cursor', 'orient', 'style', 'takefocus', + # 'state'? + ) + default_orient = 'horizontal' + + def create(self, **kwargs): + return ttk.Separator(self.root, **kwargs) + + + at add_standard_options(StandardTtkOptionsTests) +class SizegripTest(AbstractWidgetTest, unittest.TestCase): + OPTIONS = ( + 'class', 'cursor', 'style', 'takefocus', + # 'state'? + ) + + def create(self, **kwargs): + return ttk.Sizegrip(self.root, **kwargs) + +tests_gui = ( + ButtonTest, CheckbuttonTest, ComboboxTest, EntryTest, + FrameTest, LabelFrameTest, LabelTest, MenubuttonTest, + NotebookTest, PanedWindowTest, ProgressbarTest, + RadiobuttonTest, ScaleTest, ScrollbarTest, SeparatorTest, + SizegripTest, SpinboxTest, TreeviewTest, WidgetTest, + ) + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/tkinter/ttk.py b/Lib/tkinter/ttk.py new file mode 100644 index 000000000000..573544dd84a3 --- /dev/null +++ b/Lib/tkinter/ttk.py @@ -0,0 +1,1660 @@ +"""Ttk wrapper. + +This module provides classes to allow using Tk themed widget set. + +Ttk is based on a revised and enhanced version of +TIP #48 (http://tip.tcl.tk/48) specified style engine. + +Its basic idea is to separate, to the extent possible, the code +implementing a widget's behavior from the code implementing its +appearance. Widget class bindings are primarily responsible for +maintaining the widget state and invoking callbacks, all aspects +of the widgets appearance lies at Themes. +""" + +__version__ = "0.3.1" + +__author__ = "Guilherme Polo <ggpolo at gmail.com>" + +__all__ = ["Button", "Checkbutton", "Combobox", "Entry", "Frame", "Label", + "Labelframe", "LabelFrame", "Menubutton", "Notebook", "Panedwindow", + "PanedWindow", "Progressbar", "Radiobutton", "Scale", "Scrollbar", + "Separator", "Sizegrip", "Spinbox", "Style", "Treeview", + # Extensions + "LabeledScale", "OptionMenu", + # functions + "tclobjs_to_py", "setup_master"] + +import tkinter +from tkinter import _flatten, _join, _stringify, _splitdict + +# Verify if Tk is new enough to not need the Tile package +_REQUIRE_TILE = True if tkinter.TkVersion < 8.5 else False + +def _load_tile(master): + if _REQUIRE_TILE: + import os + tilelib = os.environ.get('TILE_LIBRARY') + if tilelib: + # append custom tile path to the list of directories that + # Tcl uses when attempting to resolve packages with the package + # command + master.tk.eval( + 'global auto_path; ' + 'lappend auto_path {%s}' % tilelib) + + master.tk.eval('package require tile') # TclError may be raised here + master._tile_loaded = True + +def _format_optvalue(value, script=False): + """Internal function.""" + if script: + # if caller passes a Tcl script to tk.call, all the values need to + # be grouped into words (arguments to a command in Tcl dialect) + value = _stringify(value) + elif isinstance(value, (list, tuple)): + value = _join(value) + return value + +def _format_optdict(optdict, script=False, ignore=None): + """Formats optdict to a tuple to pass it to tk.call. + + E.g. (script=False): + {'foreground': 'blue', 'padding': [1, 2, 3, 4]} returns: + ('-foreground', 'blue', '-padding', '1 2 3 4')""" + + opts = [] + for opt, value in optdict.items(): + if not ignore or opt not in ignore: + opts.append("-%s" % opt) + if value is not None: + opts.append(_format_optvalue(value, script)) + + return _flatten(opts) + +def _mapdict_values(items): + # each value in mapdict is expected to be a sequence, where each item + # is another sequence containing a state (or several) and a value + # E.g. (script=False): + # [('active', 'selected', 'grey'), ('focus', [1, 2, 3, 4])] + # returns: + # ['active selected', 'grey', 'focus', [1, 2, 3, 4]] + opt_val = [] + for *state, val in items: + # hacks for backward compatibility + state[0] # raise IndexError if empty + if len(state) == 1: + # if it is empty (something that evaluates to False), then + # format it to Tcl code to denote the "normal" state + state = state[0] or '' + else: + # group multiple states + state = ' '.join(state) # raise TypeError if not str + opt_val.append(state) + if val is not None: + opt_val.append(val) + return opt_val + +def _format_mapdict(mapdict, script=False): + """Formats mapdict to pass it to tk.call. + + E.g. (script=False): + {'expand': [('active', 'selected', 'grey'), ('focus', [1, 2, 3, 4])]} + + returns: + + ('-expand', '{active selected} grey focus {1, 2, 3, 4}')""" + + opts = [] + for opt, value in mapdict.items(): + opts.extend(("-%s" % opt, + _format_optvalue(_mapdict_values(value), script))) + + return _flatten(opts) + +def _format_elemcreate(etype, script=False, *args, **kw): + """Formats args and kw according to the given element factory etype.""" + spec = None + opts = () + if etype in ("image", "vsapi"): + if etype == "image": # define an element based on an image + # first arg should be the default image name + iname = args[0] + # next args, if any, are statespec/value pairs which is almost + # a mapdict, but we just need the value + imagespec = _join(_mapdict_values(args[1:])) + spec = "%s %s" % (iname, imagespec) + + else: + # define an element whose visual appearance is drawn using the + # Microsoft Visual Styles API which is responsible for the + # themed styles on Windows XP and Vista. + # Availability: Tk 8.6, Windows XP and Vista. + class_name, part_id = args[:2] + statemap = _join(_mapdict_values(args[2:])) + spec = "%s %s %s" % (class_name, part_id, statemap) + + opts = _format_optdict(kw, script) + + elif etype == "from": # clone an element + # it expects a themename and optionally an element to clone from, + # otherwise it will clone {} (empty element) + spec = args[0] # theme name + if len(args) > 1: # elementfrom specified + opts = (_format_optvalue(args[1], script),) + + if script: + spec = '{%s}' % spec + opts = ' '.join(opts) + + return spec, opts + +def _format_layoutlist(layout, indent=0, indent_size=2): + """Formats a layout list so we can pass the result to ttk::style + layout and ttk::style settings. Note that the layout doesn't have to + be a list necessarily. + + E.g.: + [("Menubutton.background", None), + ("Menubutton.button", {"children": + [("Menubutton.focus", {"children": + [("Menubutton.padding", {"children": + [("Menubutton.label", {"side": "left", "expand": 1})] + })] + })] + }), + ("Menubutton.indicator", {"side": "right"}) + ] + + returns: + + Menubutton.background + Menubutton.button -children { + Menubutton.focus -children { + Menubutton.padding -children { + Menubutton.label -side left -expand 1 + } + } + } + Menubutton.indicator -side right""" + script = [] + + for layout_elem in layout: + elem, opts = layout_elem + opts = opts or {} + fopts = ' '.join(_format_optdict(opts, True, ("children",))) + head = "%s%s%s" % (' ' * indent, elem, (" %s" % fopts) if fopts else '') + + if "children" in opts: + script.append(head + " -children {") + indent += indent_size + newscript, indent = _format_layoutlist(opts['children'], indent, + indent_size) + script.append(newscript) + indent -= indent_size + script.append('%s}' % (' ' * indent)) + else: + script.append(head) + + return '\n'.join(script), indent + +def _script_from_settings(settings): + """Returns an appropriate script, based on settings, according to + theme_settings definition to be used by theme_settings and + theme_create.""" + script = [] + # a script will be generated according to settings passed, which + # will then be evaluated by Tcl + for name, opts in settings.items(): + # will format specific keys according to Tcl code + if opts.get('configure'): # format 'configure' + s = ' '.join(_format_optdict(opts['configure'], True)) + script.append("ttk::style configure %s %s;" % (name, s)) + + if opts.get('map'): # format 'map' + s = ' '.join(_format_mapdict(opts['map'], True)) + script.append("ttk::style map %s %s;" % (name, s)) + + if 'layout' in opts: # format 'layout' which may be empty + if not opts['layout']: + s = 'null' # could be any other word, but this one makes sense + else: + s, _ = _format_layoutlist(opts['layout']) + script.append("ttk::style layout %s {\n%s\n}" % (name, s)) + + if opts.get('element create'): # format 'element create' + eopts = opts['element create'] + etype = eopts[0] + + # find where args end, and where kwargs start + argc = 1 # etype was the first one + while argc < len(eopts) and not hasattr(eopts[argc], 'items'): + argc += 1 + + elemargs = eopts[1:argc] + elemkw = eopts[argc] if argc < len(eopts) and eopts[argc] else {} + spec, opts = _format_elemcreate(etype, True, *elemargs, **elemkw) + + script.append("ttk::style element create %s %s %s %s" % ( + name, etype, spec, opts)) + + return '\n'.join(script) + +def _list_from_statespec(stuple): + """Construct a list from the given statespec tuple according to the + accepted statespec accepted by _format_mapdict.""" + nval = [] + for val in stuple: + typename = getattr(val, 'typename', None) + if typename is None: + nval.append(val) + else: # this is a Tcl object + val = str(val) + if typename == 'StateSpec': + val = val.split() + nval.append(val) + + it = iter(nval) + return [_flatten(spec) for spec in zip(it, it)] + +def _list_from_layouttuple(tk, ltuple): + """Construct a list from the tuple returned by ttk::layout, this is + somewhat the reverse of _format_layoutlist.""" + ltuple = tk.splitlist(ltuple) + res = [] + + indx = 0 + while indx < len(ltuple): + name = ltuple[indx] + opts = {} + res.append((name, opts)) + indx += 1 + + while indx < len(ltuple): # grab name's options + opt, val = ltuple[indx:indx + 2] + if not opt.startswith('-'): # found next name + break + + opt = opt[1:] # remove the '-' from the option + indx += 2 + + if opt == 'children': + val = _list_from_layouttuple(tk, val) + + opts[opt] = val + + return res + +def _val_or_dict(tk, options, *args): + """Format options then call Tk command with args and options and return + the appropriate result. + + If no option is specified, a dict is returned. If an option is + specified with the None value, the value for that option is returned. + Otherwise, the function just sets the passed options and the caller + shouldn't be expecting a return value anyway.""" + options = _format_optdict(options) + res = tk.call(*(args + options)) + + if len(options) % 2: # option specified without a value, return its value + return res + + return _splitdict(tk, res, conv=_tclobj_to_py) + +def _convert_stringval(value): + """Converts a value to, hopefully, a more appropriate Python object.""" + value = str(value) + try: + value = int(value) + except (ValueError, TypeError): + pass + + return value + +def _to_number(x): + if isinstance(x, str): + if '.' in x: + x = float(x) + else: + x = int(x) + return x + +def _tclobj_to_py(val): + """Return value converted from Tcl object to Python object.""" + if val and hasattr(val, '__len__') and not isinstance(val, str): + if getattr(val[0], 'typename', None) == 'StateSpec': + val = _list_from_statespec(val) + else: + val = list(map(_convert_stringval, val)) + + elif hasattr(val, 'typename'): # some other (single) Tcl object + val = _convert_stringval(val) + + return val + +def tclobjs_to_py(adict): + """Returns adict with its values converted from Tcl objects to Python + objects.""" + for opt, val in adict.items(): + adict[opt] = _tclobj_to_py(val) + + return adict + +def setup_master(master=None): + """If master is not None, itself is returned. If master is None, + the default master is returned if there is one, otherwise a new + master is created and returned. + + If it is not allowed to use the default root and master is None, + RuntimeError is raised.""" + if master is None: + if tkinter._support_default_root: + master = tkinter._default_root or tkinter.Tk() + else: + raise RuntimeError( + "No master specified and tkinter is " + "configured to not support default root") + return master + + +class Style(object): + """Manipulate style database.""" + + _name = "ttk::style" + + def __init__(self, master=None): + master = setup_master(master) + + if not getattr(master, '_tile_loaded', False): + # Load tile now, if needed + _load_tile(master) + + self.master = master + self.tk = self.master.tk + + + def configure(self, style, query_opt=None, **kw): + """Query or sets the default value of the specified option(s) in + style. + + Each key in kw is an option and each value is either a string or + a sequence identifying the value for that option.""" + if query_opt is not None: + kw[query_opt] = None + result = _val_or_dict(self.tk, kw, self._name, "configure", style) + if result or query_opt: + return result + + + def map(self, style, query_opt=None, **kw): + """Query or sets dynamic values of the specified option(s) in + style. + + Each key in kw is an option and each value should be a list or a + tuple (usually) containing statespecs grouped in tuples, or list, + or something else of your preference. A statespec is compound of + one or more states and then a value.""" + if query_opt is not None: + return _list_from_statespec(self.tk.splitlist( + self.tk.call(self._name, "map", style, '-%s' % query_opt))) + + return _splitdict( + self.tk, + self.tk.call(self._name, "map", style, *_format_mapdict(kw)), + conv=_tclobj_to_py) + + + def lookup(self, style, option, state=None, default=None): + """Returns the value specified for option in style. + + If state is specified it is expected to be a sequence of one + or more states. If the default argument is set, it is used as + a fallback value in case no specification for option is found.""" + state = ' '.join(state) if state else '' + + return self.tk.call(self._name, "lookup", style, '-%s' % option, + state, default) + + + def layout(self, style, layoutspec=None): + """Define the widget layout for given style. If layoutspec is + omitted, return the layout specification for given style. + + layoutspec is expected to be a list or an object different than + None that evaluates to False if you want to "turn off" that style. + If it is a list (or tuple, or something else), each item should be + a tuple where the first item is the layout name and the second item + should have the format described below: + + LAYOUTS + + A layout can contain the value None, if takes no options, or + a dict of options specifying how to arrange the element. + The layout mechanism uses a simplified version of the pack + geometry manager: given an initial cavity, each element is + allocated a parcel. Valid options/values are: + + side: whichside + Specifies which side of the cavity to place the + element; one of top, right, bottom or left. If + omitted, the element occupies the entire cavity. + + sticky: nswe + Specifies where the element is placed inside its + allocated parcel. + + children: [sublayout... ] + Specifies a list of elements to place inside the + element. Each element is a tuple (or other sequence) + where the first item is the layout name, and the other + is a LAYOUT.""" + lspec = None + if layoutspec: + lspec = _format_layoutlist(layoutspec)[0] + elif layoutspec is not None: # will disable the layout ({}, '', etc) + lspec = "null" # could be any other word, but this may make sense + # when calling layout(style) later + + return _list_from_layouttuple(self.tk, + self.tk.call(self._name, "layout", style, lspec)) + + + def element_create(self, elementname, etype, *args, **kw): + """Create a new element in the current theme of given etype.""" + spec, opts = _format_elemcreate(etype, False, *args, **kw) + self.tk.call(self._name, "element", "create", elementname, etype, + spec, *opts) + + + def element_names(self): + """Returns the list of elements defined in the current theme.""" + return tuple(n.lstrip('-') for n in self.tk.splitlist( + self.tk.call(self._name, "element", "names"))) + + + def element_options(self, elementname): + """Return the list of elementname's options.""" + return tuple(o.lstrip('-') for o in self.tk.splitlist( + self.tk.call(self._name, "element", "options", elementname))) + + + def theme_create(self, themename, parent=None, settings=None): + """Creates a new theme. + + It is an error if themename already exists. If parent is + specified, the new theme will inherit styles, elements and + layouts from the specified parent theme. If settings are present, + they are expected to have the same syntax used for theme_settings.""" + script = _script_from_settings(settings) if settings else '' + + if parent: + self.tk.call(self._name, "theme", "create", themename, + "-parent", parent, "-settings", script) + else: + self.tk.call(self._name, "theme", "create", themename, + "-settings", script) + + + def theme_settings(self, themename, settings): + """Temporarily sets the current theme to themename, apply specified + settings and then restore the previous theme. + + Each key in settings is a style and each value may contain the + keys 'configure', 'map', 'layout' and 'element create' and they + are expected to have the same format as specified by the methods + configure, map, layout and element_create respectively.""" + script = _script_from_settings(settings) + self.tk.call(self._name, "theme", "settings", themename, script) + + + def theme_names(self): + """Returns a list of all known themes.""" + return self.tk.splitlist(self.tk.call(self._name, "theme", "names")) + + + def theme_use(self, themename=None): + """If themename is None, returns the theme in use, otherwise, set + the current theme to themename, refreshes all widgets and emits + a <<ThemeChanged>> event.""" + if themename is None: + # Starting on Tk 8.6, checking this global is no longer needed + # since it allows doing self.tk.call(self._name, "theme", "use") + return self.tk.eval("return $ttk::currentTheme") + + # using "ttk::setTheme" instead of "ttk::style theme use" causes + # the variable currentTheme to be updated, also, ttk::setTheme calls + # "ttk::style theme use" in order to change theme. + self.tk.call("ttk::setTheme", themename) + + +class Widget(tkinter.Widget): + """Base class for Tk themed widgets.""" + + def __init__(self, master, widgetname, kw=None): + """Constructs a Ttk Widget with the parent master. + + STANDARD OPTIONS + + class, cursor, takefocus, style + + SCROLLABLE WIDGET OPTIONS + + xscrollcommand, yscrollcommand + + LABEL WIDGET OPTIONS + + text, textvariable, underline, image, compound, width + + WIDGET STATES + + active, disabled, focus, pressed, selected, background, + readonly, alternate, invalid + """ + master = setup_master(master) + if not getattr(master, '_tile_loaded', False): + # Load tile now, if needed + _load_tile(master) + tkinter.Widget.__init__(self, master, widgetname, kw=kw) + + + def identify(self, x, y): + """Returns the name of the element at position x, y, or the empty + string if the point does not lie within any element. + + x and y are pixel coordinates relative to the widget.""" + return self.tk.call(self._w, "identify", x, y) + + + def instate(self, statespec, callback=None, *args, **kw): + """Test the widget's state. + + If callback is not specified, returns True if the widget state + matches statespec and False otherwise. If callback is specified, + then it will be invoked with *args, **kw if the widget state + matches statespec. statespec is expected to be a sequence.""" + ret = self.tk.getboolean( + self.tk.call(self._w, "instate", ' '.join(statespec))) + if ret and callback: + return callback(*args, **kw) + + return ret + + + def state(self, statespec=None): + """Modify or inquire widget state. + + Widget state is returned if statespec is None, otherwise it is + set according to the statespec flags and then a new state spec + is returned indicating which flags were changed. statespec is + expected to be a sequence.""" + if statespec is not None: + statespec = ' '.join(statespec) + + return self.tk.splitlist(str(self.tk.call(self._w, "state", statespec))) + + +class Button(Widget): + """Ttk Button widget, displays a textual label and/or image, and + evaluates a command when pressed.""" + + def __init__(self, master=None, **kw): + """Construct a Ttk Button widget with the parent master. + + STANDARD OPTIONS + + class, compound, cursor, image, state, style, takefocus, + text, textvariable, underline, width + + WIDGET-SPECIFIC OPTIONS + + command, default, width + """ + Widget.__init__(self, master, "ttk::button", kw) + + + def invoke(self): + """Invokes the command associated with the button.""" + return self.tk.call(self._w, "invoke") + + +class Checkbutton(Widget): + """Ttk Checkbutton widget which is either in on- or off-state.""" + + def __init__(self, master=None, **kw): + """Construct a Ttk Checkbutton widget with the parent master. + + STANDARD OPTIONS + + class, compound, cursor, image, state, style, takefocus, + text, textvariable, underline, width + + WIDGET-SPECIFIC OPTIONS + + command, offvalue, onvalue, variable + """ + Widget.__init__(self, master, "ttk::checkbutton", kw) + + + def invoke(self): + """Toggles between the selected and deselected states and + invokes the associated command. If the widget is currently + selected, sets the option variable to the offvalue option + and deselects the widget; otherwise, sets the option variable + to the option onvalue. + + Returns the result of the associated command.""" + return self.tk.call(self._w, "invoke") + + +class Entry(Widget, tkinter.Entry): + """Ttk Entry widget displays a one-line text string and allows that + string to be edited by the user.""" + + def __init__(self, master=None, widget=None, **kw): + """Constructs a Ttk Entry widget with the parent master. + + STANDARD OPTIONS + + class, cursor, style, takefocus, xscrollcommand + + WIDGET-SPECIFIC OPTIONS + + exportselection, invalidcommand, justify, show, state, + textvariable, validate, validatecommand, width + + VALIDATION MODES + + none, key, focus, focusin, focusout, all + """ + Widget.__init__(self, master, widget or "ttk::entry", kw) + + + def bbox(self, index): + """Return a tuple of (x, y, width, height) which describes the + bounding box of the character given by index.""" + return self._getints(self.tk.call(self._w, "bbox", index)) + + + def identify(self, x, y): + """Returns the name of the element at position x, y, or the + empty string if the coordinates are outside the window.""" + return self.tk.call(self._w, "identify", x, y) + + + def validate(self): + """Force revalidation, independent of the conditions specified + by the validate option. Returns False if validation fails, True + if it succeeds. Sets or clears the invalid state accordingly.""" + return self.tk.getboolean(self.tk.call(self._w, "validate")) + + +class Combobox(Entry): + """Ttk Combobox widget combines a text field with a pop-down list of + values.""" + + def __init__(self, master=None, **kw): + """Construct a Ttk Combobox widget with the parent master. + + STANDARD OPTIONS + + class, cursor, style, takefocus + + WIDGET-SPECIFIC OPTIONS + + exportselection, justify, height, postcommand, state, + textvariable, values, width + """ + Entry.__init__(self, master, "ttk::combobox", **kw) + + + def current(self, newindex=None): + """If newindex is supplied, sets the combobox value to the + element at position newindex in the list of values. Otherwise, + returns the index of the current value in the list of values + or -1 if the current value does not appear in the list.""" + if newindex is None: + return self.tk.getint(self.tk.call(self._w, "current")) + return self.tk.call(self._w, "current", newindex) + + + def set(self, value): + """Sets the value of the combobox to value.""" + self.tk.call(self._w, "set", value) + + +class Frame(Widget): + """Ttk Frame widget is a container, used to group other widgets + together.""" + + def __init__(self, master=None, **kw): + """Construct a Ttk Frame with parent master. + + STANDARD OPTIONS + + class, cursor, style, takefocus + + WIDGET-SPECIFIC OPTIONS + + borderwidth, relief, padding, width, height + """ + Widget.__init__(self, master, "ttk::frame", kw) + + +class Label(Widget): + """Ttk Label widget displays a textual label and/or image.""" + + def __init__(self, master=None, **kw): + """Construct a Ttk Label with parent master. + + STANDARD OPTIONS + + class, compound, cursor, image, style, takefocus, text, + textvariable, underline, width + + WIDGET-SPECIFIC OPTIONS + + anchor, background, font, foreground, justify, padding, + relief, text, wraplength + """ + Widget.__init__(self, master, "ttk::label", kw) + + +class Labelframe(Widget): + """Ttk Labelframe widget is a container used to group other widgets + together. It has an optional label, which may be a plain text string + or another widget.""" + + def __init__(self, master=None, **kw): + """Construct a Ttk Labelframe with parent master. + + STANDARD OPTIONS + + class, cursor, style, takefocus + + WIDGET-SPECIFIC OPTIONS + labelanchor, text, underline, padding, labelwidget, width, + height + """ + Widget.__init__(self, master, "ttk::labelframe", kw) + +LabelFrame = Labelframe # tkinter name compatibility + + +class Menubutton(Widget): + """Ttk Menubutton widget displays a textual label and/or image, and + displays a menu when pressed.""" + + def __init__(self, master=None, **kw): + """Construct a Ttk Menubutton with parent master. + + STANDARD OPTIONS + + class, compound, cursor, image, state, style, takefocus, + text, textvariable, underline, width + + WIDGET-SPECIFIC OPTIONS + + direction, menu + """ + Widget.__init__(self, master, "ttk::menubutton", kw) + + +class Notebook(Widget): + """Ttk Notebook widget manages a collection of windows and displays + a single one at a time. Each child window is associated with a tab, + which the user may select to change the currently-displayed window.""" + + def __init__(self, master=None, **kw): + """Construct a Ttk Notebook with parent master. + + STANDARD OPTIONS + + class, cursor, style, takefocus + + WIDGET-SPECIFIC OPTIONS + + height, padding, width + + TAB OPTIONS + + state, sticky, padding, text, image, compound, underline + + TAB IDENTIFIERS (tab_id) + + The tab_id argument found in several methods may take any of + the following forms: + + * An integer between zero and the number of tabs + * The name of a child window + * A positional specification of the form "@x,y", which + defines the tab + * The string "current", which identifies the + currently-selected tab + * The string "end", which returns the number of tabs (only + valid for method index) + """ + Widget.__init__(self, master, "ttk::notebook", kw) + + + def add(self, child, **kw): + """Adds a new tab to the notebook. + + If window is currently managed by the notebook but hidden, it is + restored to its previous position.""" + self.tk.call(self._w, "add", child, *(_format_optdict(kw))) + + + def forget(self, tab_id): + """Removes the tab specified by tab_id, unmaps and unmanages the + associated window.""" + self.tk.call(self._w, "forget", tab_id) + + + def hide(self, tab_id): + """Hides the tab specified by tab_id. + + The tab will not be displayed, but the associated window remains + managed by the notebook and its configuration remembered. Hidden + tabs may be restored with the add command.""" + self.tk.call(self._w, "hide", tab_id) + + + def identify(self, x, y): + """Returns the name of the tab element at position x, y, or the + empty string if none.""" + return self.tk.call(self._w, "identify", x, y) + + + def index(self, tab_id): + """Returns the numeric index of the tab specified by tab_id, or + the total number of tabs if tab_id is the string "end".""" + return self.tk.getint(self.tk.call(self._w, "index", tab_id)) + + + def insert(self, pos, child, **kw): + """Inserts a pane at the specified position. + + pos is either the string end, an integer index, or the name of + a managed child. If child is already managed by the notebook, + moves it to the specified position.""" + self.tk.call(self._w, "insert", pos, child, *(_format_optdict(kw))) + + + def select(self, tab_id=None): + """Selects the specified tab. + + The associated child window will be displayed, and the + previously-selected window (if different) is unmapped. If tab_id + is omitted, returns the widget name of the currently selected + pane.""" + return self.tk.call(self._w, "select", tab_id) + + + def tab(self, tab_id, option=None, **kw): + """Query or modify the options of the specific tab_id. + + If kw is not given, returns a dict of the tab option values. If option + is specified, returns the value of that option. Otherwise, sets the + options to the corresponding values.""" + if option is not None: + kw[option] = None + return _val_or_dict(self.tk, kw, self._w, "tab", tab_id) + + + def tabs(self): + """Returns a list of windows managed by the notebook.""" + return self.tk.splitlist(self.tk.call(self._w, "tabs") or ()) + + + def enable_traversal(self): + """Enable keyboard traversal for a toplevel window containing + this notebook. + + This will extend the bindings for the toplevel window containing + this notebook as follows: + + Control-Tab: selects the tab following the currently selected + one + + Shift-Control-Tab: selects the tab preceding the currently + selected one + + Alt-K: where K is the mnemonic (underlined) character of any + tab, will select that tab. + + Multiple notebooks in a single toplevel may be enabled for + traversal, including nested notebooks. However, notebook traversal + only works properly if all panes are direct children of the + notebook.""" + # The only, and good, difference I see is about mnemonics, which works + # after calling this method. Control-Tab and Shift-Control-Tab always + # works (here at least). + self.tk.call("ttk::notebook::enableTraversal", self._w) + + +class Panedwindow(Widget, tkinter.PanedWindow): + """Ttk Panedwindow widget displays a number of subwindows, stacked + either vertically or horizontally.""" + + def __init__(self, master=None, **kw): + """Construct a Ttk Panedwindow with parent master. + + STANDARD OPTIONS + + class, cursor, style, takefocus + + WIDGET-SPECIFIC OPTIONS + + orient, width, height + + PANE OPTIONS + + weight + """ + Widget.__init__(self, master, "ttk::panedwindow", kw) + + + forget = tkinter.PanedWindow.forget # overrides Pack.forget + + + def insert(self, pos, child, **kw): + """Inserts a pane at the specified positions. + + pos is either the string end, and integer index, or the name + of a child. If child is already managed by the paned window, + moves it to the specified position.""" + self.tk.call(self._w, "insert", pos, child, *(_format_optdict(kw))) + + + def pane(self, pane, option=None, **kw): + """Query or modify the options of the specified pane. + + pane is either an integer index or the name of a managed subwindow. + If kw is not given, returns a dict of the pane option values. If + option is specified then the value for that option is returned. + Otherwise, sets the options to the corresponding values.""" + if option is not None: + kw[option] = None + return _val_or_dict(self.tk, kw, self._w, "pane", pane) + + + def sashpos(self, index, newpos=None): + """If newpos is specified, sets the position of sash number index. + + May adjust the positions of adjacent sashes to ensure that + positions are monotonically increasing. Sash positions are further + constrained to be between 0 and the total size of the widget. + + Returns the new position of sash number index.""" + return self.tk.getint(self.tk.call(self._w, "sashpos", index, newpos)) + +PanedWindow = Panedwindow # tkinter name compatibility + + +class Progressbar(Widget): + """Ttk Progressbar widget shows the status of a long-running + operation. They can operate in two modes: determinate mode shows the + amount completed relative to the total amount of work to be done, and + indeterminate mode provides an animated display to let the user know + that something is happening.""" + + def __init__(self, master=None, **kw): + """Construct a Ttk Progressbar with parent master. + + STANDARD OPTIONS + + class, cursor, style, takefocus + + WIDGET-SPECIFIC OPTIONS + + orient, length, mode, maximum, value, variable, phase + """ + Widget.__init__(self, master, "ttk::progressbar", kw) + + + def start(self, interval=None): + """Begin autoincrement mode: schedules a recurring timer event + that calls method step every interval milliseconds. + + interval defaults to 50 milliseconds (20 steps/second) if omitted.""" + self.tk.call(self._w, "start", interval) + + + def step(self, amount=None): + """Increments the value option by amount. + + amount defaults to 1.0 if omitted.""" + self.tk.call(self._w, "step", amount) + + + def stop(self): + """Stop autoincrement mode: cancels any recurring timer event + initiated by start.""" + self.tk.call(self._w, "stop") + + +class Radiobutton(Widget): + """Ttk Radiobutton widgets are used in groups to show or change a + set of mutually-exclusive options.""" + + def __init__(self, master=None, **kw): + """Construct a Ttk Radiobutton with parent master. + + STANDARD OPTIONS + + class, compound, cursor, image, state, style, takefocus, + text, textvariable, underline, width + + WIDGET-SPECIFIC OPTIONS + + command, value, variable + """ + Widget.__init__(self, master, "ttk::radiobutton", kw) + + + def invoke(self): + """Sets the option variable to the option value, selects the + widget, and invokes the associated command. + + Returns the result of the command, or an empty string if + no command is specified.""" + return self.tk.call(self._w, "invoke") + + +class Scale(Widget, tkinter.Scale): + """Ttk Scale widget is typically used to control the numeric value of + a linked variable that varies uniformly over some range.""" + + def __init__(self, master=None, **kw): + """Construct a Ttk Scale with parent master. + + STANDARD OPTIONS + + class, cursor, style, takefocus + + WIDGET-SPECIFIC OPTIONS + + command, from, length, orient, to, value, variable + """ + Widget.__init__(self, master, "ttk::scale", kw) + + + def configure(self, cnf=None, **kw): + """Modify or query scale options. + + Setting a value for any of the "from", "from_" or "to" options + generates a <<RangeChanged>> event.""" + if cnf: + kw.update(cnf) + Widget.configure(self, **kw) + if any(['from' in kw, 'from_' in kw, 'to' in kw]): + self.event_generate('<<RangeChanged>>') + + + def get(self, x=None, y=None): + """Get the current value of the value option, or the value + corresponding to the coordinates x, y if they are specified. + + x and y are pixel coordinates relative to the scale widget + origin.""" + return self.tk.call(self._w, 'get', x, y) + + +class Scrollbar(Widget, tkinter.Scrollbar): + """Ttk Scrollbar controls the viewport of a scrollable widget.""" + + def __init__(self, master=None, **kw): + """Construct a Ttk Scrollbar with parent master. + + STANDARD OPTIONS + + class, cursor, style, takefocus + + WIDGET-SPECIFIC OPTIONS + + command, orient + """ + Widget.__init__(self, master, "ttk::scrollbar", kw) + + +class Separator(Widget): + """Ttk Separator widget displays a horizontal or vertical separator + bar.""" + + def __init__(self, master=None, **kw): + """Construct a Ttk Separator with parent master. + + STANDARD OPTIONS + + class, cursor, style, takefocus + + WIDGET-SPECIFIC OPTIONS + + orient + """ + Widget.__init__(self, master, "ttk::separator", kw) + + +class Sizegrip(Widget): + """Ttk Sizegrip allows the user to resize the containing toplevel + window by pressing and dragging the grip.""" + + def __init__(self, master=None, **kw): + """Construct a Ttk Sizegrip with parent master. + + STANDARD OPTIONS + + class, cursor, state, style, takefocus + """ + Widget.__init__(self, master, "ttk::sizegrip", kw) + + +class Spinbox(Entry): + """Ttk Spinbox is an Entry with increment and decrement arrows + + It is commonly used for number entry or to select from a list of + string values. + """ + + def __init__(self, master=None, **kw): + """Construct a Ttk Spinbox widget with the parent master. + + STANDARD OPTIONS + + class, cursor, style, takefocus, validate, + validatecommand, xscrollcommand, invalidcommand + + WIDGET-SPECIFIC OPTIONS + + to, from_, increment, values, wrap, format, command + """ + Entry.__init__(self, master, "ttk::spinbox", **kw) + + + def set(self, value): + """Sets the value of the Spinbox to value.""" + self.tk.call(self._w, "set", value) + + +class Treeview(Widget, tkinter.XView, tkinter.YView): + """Ttk Treeview widget displays a hierarchical collection of items. + + Each item has a textual label, an optional image, and an optional list + of data values. The data values are displayed in successive columns + after the tree label.""" + + def __init__(self, master=None, **kw): + """Construct a Ttk Treeview with parent master. + + STANDARD OPTIONS + + class, cursor, style, takefocus, xscrollcommand, + yscrollcommand + + WIDGET-SPECIFIC OPTIONS + + columns, displaycolumns, height, padding, selectmode, show + + ITEM OPTIONS + + text, image, values, open, tags + + TAG OPTIONS + + foreground, background, font, image + """ + Widget.__init__(self, master, "ttk::treeview", kw) + + + def bbox(self, item, column=None): + """Returns the bounding box (relative to the treeview widget's + window) of the specified item in the form x y width height. + + If column is specified, returns the bounding box of that cell. + If the item is not visible (i.e., if it is a descendant of a + closed item or is scrolled offscreen), returns an empty string.""" + return self._getints(self.tk.call(self._w, "bbox", item, column)) or '' + + + def get_children(self, item=None): + """Returns a tuple of children belonging to item. + + If item is not specified, returns root children.""" + return self.tk.splitlist( + self.tk.call(self._w, "children", item or '') or ()) + + + def set_children(self, item, *newchildren): + """Replaces item's child with newchildren. + + Children present in item that are not present in newchildren + are detached from tree. No items in newchildren may be an + ancestor of item.""" + self.tk.call(self._w, "children", item, newchildren) + + + def column(self, column, option=None, **kw): + """Query or modify the options for the specified column. + + If kw is not given, returns a dict of the column option values. If + option is specified then the value for that option is returned. + Otherwise, sets the options to the corresponding values.""" + if option is not None: + kw[option] = None + return _val_or_dict(self.tk, kw, self._w, "column", column) + + + def delete(self, *items): + """Delete all specified items and all their descendants. The root + item may not be deleted.""" + self.tk.call(self._w, "delete", items) + + + def detach(self, *items): + """Unlinks all of the specified items from the tree. + + The items and all of their descendants are still present, and may + be reinserted at another point in the tree, but will not be + displayed. The root item may not be detached.""" + self.tk.call(self._w, "detach", items) + + + def exists(self, item): + """Returns True if the specified item is present in the tree, + False otherwise.""" + return self.tk.getboolean(self.tk.call(self._w, "exists", item)) + + + def focus(self, item=None): + """If item is specified, sets the focus item to item. Otherwise, + returns the current focus item, or '' if there is none.""" + return self.tk.call(self._w, "focus", item) + + + def heading(self, column, option=None, **kw): + """Query or modify the heading options for the specified column. + + If kw is not given, returns a dict of the heading option values. If + option is specified then the value for that option is returned. + Otherwise, sets the options to the corresponding values. + + Valid options/values are: + text: text + The text to display in the column heading + image: image_name + Specifies an image to display to the right of the column + heading + anchor: anchor + Specifies how the heading text should be aligned. One of + the standard Tk anchor values + command: callback + A callback to be invoked when the heading label is + pressed. + + To configure the tree column heading, call this with column = "#0" """ + cmd = kw.get('command') + if cmd and not isinstance(cmd, str): + # callback not registered yet, do it now + kw['command'] = self.master.register(cmd, self._substitute) + + if option is not None: + kw[option] = None + + return _val_or_dict(self.tk, kw, self._w, 'heading', column) + + + def identify(self, component, x, y): + """Returns a description of the specified component under the + point given by x and y, or the empty string if no such component + is present at that position.""" + return self.tk.call(self._w, "identify", component, x, y) + + + def identify_row(self, y): + """Returns the item ID of the item at position y.""" + return self.identify("row", 0, y) + + + def identify_column(self, x): + """Returns the data column identifier of the cell at position x. + + The tree column has ID #0.""" + return self.identify("column", x, 0) + + + def identify_region(self, x, y): + """Returns one of: + + heading: Tree heading area. + separator: Space between two columns headings; + tree: The tree area. + cell: A data cell. + + * Availability: Tk 8.6""" + return self.identify("region", x, y) + + + def identify_element(self, x, y): + """Returns the element at position x, y. + + * Availability: Tk 8.6""" + return self.identify("element", x, y) + + + def index(self, item): + """Returns the integer index of item within its parent's list + of children.""" + return self.tk.getint(self.tk.call(self._w, "index", item)) + + + def insert(self, parent, index, iid=None, **kw): + """Creates a new item and return the item identifier of the newly + created item. + + parent is the item ID of the parent item, or the empty string + to create a new top-level item. index is an integer, or the value + end, specifying where in the list of parent's children to insert + the new item. If index is less than or equal to zero, the new node + is inserted at the beginning, if index is greater than or equal to + the current number of children, it is inserted at the end. If iid + is specified, it is used as the item identifier, iid must not + already exist in the tree. Otherwise, a new unique identifier + is generated.""" + opts = _format_optdict(kw) + if iid is not None: + res = self.tk.call(self._w, "insert", parent, index, + "-id", iid, *opts) + else: + res = self.tk.call(self._w, "insert", parent, index, *opts) + + return res + + + def item(self, item, option=None, **kw): + """Query or modify the options for the specified item. + + If no options are given, a dict with options/values for the item + is returned. If option is specified then the value for that option + is returned. Otherwise, sets the options to the corresponding + values as given by kw.""" + if option is not None: + kw[option] = None + return _val_or_dict(self.tk, kw, self._w, "item", item) + + + def move(self, item, parent, index): + """Moves item to position index in parent's list of children. + + It is illegal to move an item under one of its descendants. If + index is less than or equal to zero, item is moved to the + beginning, if greater than or equal to the number of children, + it is moved to the end. If item was detached it is reattached.""" + self.tk.call(self._w, "move", item, parent, index) + + reattach = move # A sensible method name for reattaching detached items + + + def next(self, item): + """Returns the identifier of item's next sibling, or '' if item + is the last child of its parent.""" + return self.tk.call(self._w, "next", item) + + + def parent(self, item): + """Returns the ID of the parent of item, or '' if item is at the + top level of the hierarchy.""" + return self.tk.call(self._w, "parent", item) + + + def prev(self, item): + """Returns the identifier of item's previous sibling, or '' if + item is the first child of its parent.""" + return self.tk.call(self._w, "prev", item) + + + def see(self, item): + """Ensure that item is visible. + + Sets all of item's ancestors open option to True, and scrolls + the widget if necessary so that item is within the visible + portion of the tree.""" + self.tk.call(self._w, "see", item) + + + def selection(self): + """Returns the tuple of selected items.""" + return self.tk.splitlist(self.tk.call(self._w, "selection")) + + + def _selection(self, selop, items): + if len(items) == 1 and isinstance(items[0], (tuple, list)): + items = items[0] + + self.tk.call(self._w, "selection", selop, items) + + + def selection_set(self, *items): + """The specified items becomes the new selection.""" + self._selection("set", items) + + + def selection_add(self, *items): + """Add all of the specified items to the selection.""" + self._selection("add", items) + + + def selection_remove(self, *items): + """Remove all of the specified items from the selection.""" + self._selection("remove", items) + + + def selection_toggle(self, *items): + """Toggle the selection state of each specified item.""" + self._selection("toggle", items) + + + def set(self, item, column=None, value=None): + """Query or set the value of given item. + + With one argument, return a dictionary of column/value pairs + for the specified item. With two arguments, return the current + value of the specified column. With three arguments, set the + value of given column in given item to the specified value.""" + res = self.tk.call(self._w, "set", item, column, value) + if column is None and value is None: + return _splitdict(self.tk, res, + cut_minus=False, conv=_tclobj_to_py) + else: + return res + + + def tag_bind(self, tagname, sequence=None, callback=None): + """Bind a callback for the given event sequence to the tag tagname. + When an event is delivered to an item, the callbacks for each + of the item's tags option are called.""" + self._bind((self._w, "tag", "bind", tagname), sequence, callback, add=0) + + + def tag_configure(self, tagname, option=None, **kw): + """Query or modify the options for the specified tagname. + + If kw is not given, returns a dict of the option settings for tagname. + If option is specified, returns the value for that option for the + specified tagname. Otherwise, sets the options to the corresponding + values for the given tagname.""" + if option is not None: + kw[option] = None + return _val_or_dict(self.tk, kw, self._w, "tag", "configure", + tagname) + + + def tag_has(self, tagname, item=None): + """If item is specified, returns 1 or 0 depending on whether the + specified item has the given tagname. Otherwise, returns a list of + all items which have the specified tag. + + * Availability: Tk 8.6""" + if item is None: + return self.tk.splitlist( + self.tk.call(self._w, "tag", "has", tagname)) + else: + return self.tk.getboolean( + self.tk.call(self._w, "tag", "has", tagname, item)) + + +# Extensions + +class LabeledScale(Frame): + """A Ttk Scale widget with a Ttk Label widget indicating its + current value. + + The Ttk Scale can be accessed through instance.scale, and Ttk Label + can be accessed through instance.label""" + + def __init__(self, master=None, variable=None, from_=0, to=10, **kw): + """Construct a horizontal LabeledScale with parent master, a + variable to be associated with the Ttk Scale widget and its range. + If variable is not specified, a tkinter.IntVar is created. + + WIDGET-SPECIFIC OPTIONS + + compound: 'top' or 'bottom' + Specifies how to display the label relative to the scale. + Defaults to 'top'. + """ + self._label_top = kw.pop('compound', 'top') == 'top' + + Frame.__init__(self, master, **kw) + self._variable = variable or tkinter.IntVar(master) + self._variable.set(from_) + self._last_valid = from_ + + self.label = Label(self) + self.scale = Scale(self, variable=self._variable, from_=from_, to=to) + self.scale.bind('<<RangeChanged>>', self._adjust) + + # position scale and label according to the compound option + scale_side = 'bottom' if self._label_top else 'top' + label_side = 'top' if scale_side == 'bottom' else 'bottom' + self.scale.pack(side=scale_side, fill='x') + tmp = Label(self).pack(side=label_side) # place holder + self.label.place(anchor='n' if label_side == 'top' else 's') + + # update the label as scale or variable changes + self.__tracecb = self._variable.trace_variable('w', self._adjust) + self.bind('<Configure>', self._adjust) + self.bind('<Map>', self._adjust) + + + def destroy(self): + """Destroy this widget and possibly its associated variable.""" + try: + self._variable.trace_vdelete('w', self.__tracecb) + except AttributeError: + pass + else: + del self._variable + super().destroy() + self.label = None + self.scale = None + + + def _adjust(self, *args): + """Adjust the label position according to the scale.""" + def adjust_label(): + self.update_idletasks() # "force" scale redraw + + x, y = self.scale.coords() + if self._label_top: + y = self.scale.winfo_y() - self.label.winfo_reqheight() + else: + y = self.scale.winfo_reqheight() + self.label.winfo_reqheight() + + self.label.place_configure(x=x, y=y) + + from_ = _to_number(self.scale['from']) + to = _to_number(self.scale['to']) + if to < from_: + from_, to = to, from_ + newval = self._variable.get() + if not from_ <= newval <= to: + # value outside range, set value back to the last valid one + self.value = self._last_valid + return + + self._last_valid = newval + self.label['text'] = newval + self.after_idle(adjust_label) + + @property + def value(self): + """Return current scale value.""" + return self._variable.get() + + @value.setter + def value(self, val): + """Set new scale value.""" + self._variable.set(val) + + +class OptionMenu(Menubutton): + """Themed OptionMenu, based after tkinter's OptionMenu, which allows + the user to select a value from a menu.""" + + def __init__(self, master, variable, default=None, *values, **kwargs): + """Construct a themed OptionMenu widget with master as the parent, + the resource textvariable set to variable, the initially selected + value specified by the default parameter, the menu values given by + *values and additional keywords. + + WIDGET-SPECIFIC OPTIONS + + style: stylename + Menubutton style. + direction: 'above', 'below', 'left', 'right', or 'flush' + Menubutton direction. + command: callback + A callback that will be invoked after selecting an item. + """ + kw = {'textvariable': variable, 'style': kwargs.pop('style', None), + 'direction': kwargs.pop('direction', None)} + Menubutton.__init__(self, master, **kw) + self['menu'] = tkinter.Menu(self, tearoff=False) + + self._variable = variable + self._callback = kwargs.pop('command', None) + if kwargs: + raise tkinter.TclError('unknown option -%s' % ( + next(iter(kwargs.keys())))) + + self.set_menu(default, *values) + + + def __getitem__(self, item): + if item == 'menu': + return self.nametowidget(Menubutton.__getitem__(self, item)) + + return Menubutton.__getitem__(self, item) + + + def set_menu(self, default=None, *values): + """Build a new menu of radiobuttons with *values and optionally + a default value.""" + menu = self['menu'] + menu.delete(0, 'end') + for val in values: + menu.add_radiobutton(label=val, + command=tkinter._setit(self._variable, val, self._callback), + variable=self._variable) + + if default: + self._variable.set(default) + + + def destroy(self): + """Destroy this widget and its associated variable.""" + try: + del self._variable + except AttributeError: + pass + super().destroy() diff --git a/Misc/NEWS.d/next/Library/2018-03-25-13-18-16.bpo-33096.ofdbe7.rst b/Misc/NEWS.d/next/Library/2018-03-25-13-18-16.bpo-33096.ofdbe7.rst new file mode 100644 index 000000000000..c55ea20b337d --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-03-25-13-18-16.bpo-33096.ofdbe7.rst @@ -0,0 +1,4 @@ +Allow ttk.Treeview.insert to insert iid that has a false boolean value. +Note iid=0 and iid=False would be same. +Patch by Garvit Khatri. + From webhook-mailer at python.org Sat Mar 31 19:44:04 2018 From: webhook-mailer at python.org (Serhiy Storchaka) Date: Sat, 31 Mar 2018 23:44:04 -0000 Subject: [Python-checkins] [2.7] Gitignore gmon.out (GH-5796) (GH-6328) Message-ID: <mailman.21.1522539846.3857.python-checkins@python.org> https://github.com/python/cpython/commit/4a3c4babd9bad31724b7c3f20ead9848d7404726 commit: 4a3c4babd9bad31724b7c3f20ead9848d7404726 branch: 2.7 author: Serhiy Storchaka <storchaka at gmail.com> committer: GitHub <noreply at github.com> date: 2018-04-01T02:44:01+03:00 summary: [2.7] Gitignore gmon.out (GH-5796) (GH-6328) gmon.out is generated when profiling turned on Full Configuration: ./configure --prefix=$PWD/install --enable-profiling --enable-big-digits=30 --with-pydebug --with-assertions --with-valgrind. (cherry picked from commit 95ad3822a2b6287772bd752b6ab493c6d4198d4b) Co-authored-by: Neeraj Badlani <neerajbadlani at gmail.com> files: M .gitignore diff --git a/.gitignore b/.gitignore index a44b733b1017..de25253df82e 100644 --- a/.gitignore +++ b/.gitignore @@ -86,3 +86,4 @@ TAGS coverage/ externals/ htmlcov/ +gmon.out