Mailman 3 January 2018 - Python-checkins

bpo-32726: Build macOS 10.9+ installer with private copy of Tcl/Tk 8.6
by Ned Deily Jan. 31, 2018

Jan. 31, 2018

https://github.com/python/cpython/commit/1ca2ffdc532622ef001418585c146e6b0f… commit: 1ca2ffdc532622ef001418585c146e6b0f31027f branch: master author: Ned Deily <nad(a)python.org> committer: Ned Deily <nad(a)python.org> date: 2018-01-31T17:14:12-05:00 summary: bpo-32726: Build macOS 10.9+ installer with private copy of Tcl/Tk 8.6 files: D Mac/BuildScript/issue19373_tk_8_5_15_source.patch M Mac/BuildScript/build-installer.py M Mac/BuildScript/resources/ReadMe.rtf M Mac/BuildScript/resources/Welcome.rtf M Misc/NEWS.d/next/macOS/2018-01-30-07-13-10.bpo-32726.tcARLK.rst diff --git a/Mac/BuildScript/build-installer.py b/Mac/BuildScript/build-installer.py index 8e2e104a0b7e..1a7080c18f1b 100755 --- a/Mac/BuildScript/build-installer.py +++ b/Mac/BuildScript/build-installer.py @@ -189,6 +189,11 @@ def getTargetCompilers(): # '/Library/Frameworks/Tk.framework/Versions/8.5/Tk'] EXPECTED_SHARED_LIBS = {} +# Are we building and linking with our own copy of Tcl/TK? +# For now, do so if deployment target is 10.9+. +def internalTk(): + return getDeptargetTuple() >= (10, 9) + # List of names of third party software built with this installer. # The names will be inserted into the rtf version of the License. THIRD_PARTY_LIBS = [] @@ -217,13 +222,12 @@ def library_recipes(): ), ]) -# Disable for now - if False: # if getDeptargetTuple() > (10, 5): + if internalTk(): result.extend([ dict( - name="Tcl 8.5.15", - url="ftp://ftp.tcl.tk/pub/tcl//tcl8_5/tcl8.5.15-src.tar.gz", - checksum='f3df162f92c69b254079c4d0af7a690f', + name="Tcl 8.6.7", + url="ftp://ftp.tcl.tk/pub/tcl//tcl8_6/tcl8.6.7-src.tar.gz", + checksum='5673aaf45b5de5d8dd80bb3daaeb8838', buildDir="unix", configure_pre=[ '--enable-shared', @@ -233,16 +237,13 @@ def library_recipes(): useLDFlags=False, install='make TCL_LIBRARY=%(TCL_LIBRARY)s && make install TCL_LIBRARY=%(TCL_LIBRARY)s DESTDIR=%(DESTDIR)s'%{ "DESTDIR": shellQuote(os.path.join(WORKDIR, 'libraries')), - "TCL_LIBRARY": shellQuote('/Library/Frameworks/Python.framework/Versions/%s/lib/tcl8.5'%(getVersion())), + "TCL_LIBRARY": shellQuote('/Library/Frameworks/Python.framework/Versions/%s/lib/tcl8.6'%(getVersion())), }, ), dict( - name="Tk 8.5.15", - url="ftp://ftp.tcl.tk/pub/tcl//tcl8_5/tk8.5.15-src.tar.gz", - checksum='55b8e33f903210a4e1c8bce0f820657f', - patches=[ - "issue19373_tk_8_5_15_source.patch", - ], + name="Tk 8.6.7", + url="ftp://ftp.tcl.tk/pub/tcl//tcl8_6/tk8.6.7-src.tar.gz", + checksum='46ea9c0165c515d87393700f4891ab6f', buildDir="unix", configure_pre=[ '--enable-aqua', @@ -253,8 +254,8 @@ def library_recipes(): useLDFlags=False, install='make TCL_LIBRARY=%(TCL_LIBRARY)s TK_LIBRARY=%(TK_LIBRARY)s && make install TCL_LIBRARY=%(TCL_LIBRARY)s TK_LIBRARY=%(TK_LIBRARY)s DESTDIR=%(DESTDIR)s'%{ "DESTDIR": shellQuote(os.path.join(WORKDIR, 'libraries')), - "TCL_LIBRARY": shellQuote('/Library/Frameworks/Python.framework/Versions/%s/lib/tcl8.5'%(getVersion())), - "TK_LIBRARY": shellQuote('/Library/Frameworks/Python.framework/Versions/%s/lib/tk8.5'%(getVersion())), + "TCL_LIBRARY": shellQuote('/Library/Frameworks/Python.framework/Versions/%s/lib/tcl8.6'%(getVersion())), + "TK_LIBRARY": shellQuote('/Library/Frameworks/Python.framework/Versions/%s/lib/tk8.6'%(getVersion())), }, ), ]) @@ -547,42 +548,45 @@ def checkEnvironment(): fatal("This script should be run on a macOS 10.5 (or later) system") # Because we only support dynamic load of only one major/minor version of + # Tcl/Tk, if we are not using building and using our own private copy of # Tcl/Tk, ensure: # 1. there is a user-installed framework (usually ActiveTcl) in (or linked # in) SDKROOT/Library/Frameworks. As of Python 3.7.0, we no longer # enforce that the version of the user-installed framework also # exists in the system-supplied Tcl/Tk frameworks. Time to support # Tcl/Tk 8.6 even if Apple does not. - - frameworks = {} - for framework in ['Tcl', 'Tk']: - fwpth = 'Library/Frameworks/%s.framework/Versions/Current' % framework - libfw = os.path.join('/', fwpth) - usrfw = os.path.join(os.getenv('HOME'), fwpth) - frameworks[framework] = os.readlink(libfw) - if not os.path.exists(libfw): - fatal("Please install a link to a current %s %s as %s so " - "the user can override the system framework." - % (framework, frameworks[framework], libfw)) - if os.path.exists(usrfw): - fatal("Please rename %s to avoid possible dynamic load issues." - % usrfw) - - if frameworks['Tcl'] != frameworks['Tk']: - fatal("The Tcl and Tk frameworks are not the same version.") - - print(" -- Building with Tcl/Tk %s frameworks" - % frameworks['Tk']) + if not internalTk(): + frameworks = {} + for framework in ['Tcl', 'Tk']: + fwpth = 'Library/Frameworks/%s.framework/Versions/Current' % framework + libfw = os.path.join('/', fwpth) + usrfw = os.path.join(os.getenv('HOME'), fwpth) + frameworks[framework] = os.readlink(libfw) + if not os.path.exists(libfw): + fatal("Please install a link to a current %s %s as %s so " + "the user can override the system framework." + % (framework, frameworks[framework], libfw)) + if os.path.exists(usrfw): + fatal("Please rename %s to avoid possible dynamic load issues." + % usrfw) + + if frameworks['Tcl'] != frameworks['Tk']: + fatal("The Tcl and Tk frameworks are not the same version.") + + print(" -- Building with external Tcl/Tk %s frameworks" + % frameworks['Tk']) + + # add files to check after build + EXPECTED_SHARED_LIBS['_tkinter.so'] = [ + "/Library/Frameworks/Tcl.framework/Versions/%s/Tcl" + % frameworks['Tcl'], + "/Library/Frameworks/Tk.framework/Versions/%s/Tk" + % frameworks['Tk'], + ] + else: + print(" -- Building private copy of Tcl/Tk") print("") - # add files to check after build - EXPECTED_SHARED_LIBS['_tkinter.so'] = [ - "/Library/Frameworks/Tcl.framework/Versions/%s/Tcl" - % frameworks['Tcl'], - "/Library/Frameworks/Tk.framework/Versions/%s/Tk" - % frameworks['Tk'], - ] - # Remove inherited environment variables which might influence build environ_var_prefixes = ['CPATH', 'C_INCLUDE_', 'DYLD_', 'LANG', 'LC_', 'LD_', 'LIBRARY_', 'PATH', 'PYTHON'] @@ -1086,12 +1090,18 @@ def buildPython(): "--with-universal-archs=%s " "%s " "%s " + "%s " + "%s " "LDFLAGS='-g -L%s/libraries/usr/local/lib' " "CFLAGS='-g -I%s/libraries/usr/local/include' 2>&1"%( shellQuote(os.path.join(SRCDIR, 'configure')), UNIVERSALARCHS, (' ', '--with-computed-gotos ')[PYTHON_3], (' ', '--without-ensurepip ')[PYTHON_3], + (' ', "--with-tcltk-includes='-I%s/libraries/usr/local/include'"%( + shellQuote(WORKDIR)[1:-1],))[internalTk()], + (' ', "--with-tcltk-libs='-L%s/libraries/usr/local/lib -ltcl8.6 -ltk8.6'"%( + shellQuote(WORKDIR)[1:-1],))[internalTk()], shellQuote(WORKDIR)[1:-1], shellQuote(WORKDIR)[1:-1])) @@ -1126,14 +1136,22 @@ def buildPython(): del os.environ['DYLD_LIBRARY_PATH'] print("Copying required shared libraries") if os.path.exists(os.path.join(WORKDIR, 'libraries', 'Library')): - runCommand("mv %s/* %s"%( - shellQuote(os.path.join( + build_lib_dir = os.path.join( WORKDIR, 'libraries', 'Library', 'Frameworks', - 'Python.framework', 'Versions', getVersion(), - 'lib')), - shellQuote(os.path.join(WORKDIR, '_root', 'Library', 'Frameworks', - 'Python.framework', 'Versions', getVersion(), - 'lib')))) + 'Python.framework', 'Versions', getVersion(), 'lib') + fw_lib_dir = os.path.join( + WORKDIR, '_root', 'Library', 'Frameworks', + 'Python.framework', 'Versions', getVersion(), 'lib') + if internalTk(): + # move Tcl and Tk pkgconfig files + runCommand("mv %s/pkgconfig/* %s/pkgconfig"%( + shellQuote(build_lib_dir), + shellQuote(fw_lib_dir) )) + runCommand("rm -r %s/pkgconfig"%( + shellQuote(build_lib_dir), )) + runCommand("mv %s/* %s"%( + shellQuote(build_lib_dir), + shellQuote(fw_lib_dir) )) frmDir = os.path.join(rootDir, 'Library', 'Frameworks', 'Python.framework') frmDirVersioned = os.path.join(frmDir, 'Versions', version) diff --git a/Mac/BuildScript/issue19373_tk_8_5_15_source.patch b/Mac/BuildScript/issue19373_tk_8_5_15_source.patch deleted file mode 100644 index de5d08e9d6c0..000000000000 --- a/Mac/BuildScript/issue19373_tk_8_5_15_source.patch +++ /dev/null @@ -1,13 +0,0 @@ -Issue #19373: Patch to Tk 8.5.15 to correct refresh problem on OS x 10.9. -From upstream checkin https://core.tcl.tk/tk/info/5a5abf71f9 - ---- tk8.5.15/macosx/tkMacOSXDraw.c 2013-09-16 09:41:21.000000000 -0700 -+++ Tk_Source_Code-5a5abf71f9fdb0da/macosx/tkMacOSXDraw.c 2013-10-27 13:27:00.000000000 -0700 -@@ -1688,6 +1688,7 @@ - { - if (dcPtr->context) { - CGContextSynchronize(dcPtr->context); -+ [[dcPtr->view window] setViewsNeedDisplay:YES]; - [[dcPtr->view window] enableFlushWindow]; - if (dcPtr->focusLocked) { - [dcPtr->view unlockFocus]; diff --git a/Mac/BuildScript/resources/ReadMe.rtf b/Mac/BuildScript/resources/ReadMe.rtf index 1eea1e291bda..cb005af7f051 100644 --- a/Mac/BuildScript/resources/ReadMe.rtf +++ b/Mac/BuildScript/resources/ReadMe.rtf @@ -31,6 +31,20 @@ For Python.3.7, python.org currently provides two installer variants for downloa \i0 variant. Use the 10.6 variant if you are running on macOS 10.6 through 10.8 or if you want to produce standalone applications that can run on systems from 10.6. If you are running on macOS 10.9 or later and if you have no need for compatibility with older systems, use the 10.9 variant. The Pythons installed by these installers are built with private copies of some third-party libraries not included with or newer than those in macOS itself. The list of these libraries varies by installer variant and is included at the end of the License.rtf file. \b \ul \ \ +Using IDLE or other Tk applications [NEW/CHANGED in 3.7.0b1] +\b0 \ulnone \ +\ +The 10.9+ installer variant comes with its own private version of Tcl/Tk 8.6. It does not use system-supplied or third-party supplied versions of Tcl/Tk.\ +\ +\pard\tx720\tx1440\tx2160\tx2880\tx3600\tx4320\tx5040\tx5760\tx6480\tx7200\tx7920\tx8640\pardirnatural\partightenfactor0 +\cf0 For the 10.6+ variant, you continue to need to install a newer third-party version of the +\i Tcl/Tk +\i0 8.5 (not 8.6) frameworks to use IDLE or other programs that use the Tkinter graphical user interface toolkit. Visit {\field{\*\fldinst{HYPERLINK "https://www.python.org/download/mac/tcltk/"}}{\fldrslt https://www.python.org/download/mac/tcltk/}} for current information about supported and recommended versions of +\i Tcl/Tk +\i0 for this version of Python and of macOS.\ +\pard\tx720\tx1440\tx2160\tx2880\tx3600\tx4320\tx5040\tx5760\tx6480\tx7200\tx7920\tx8640\pardirnatural\partightenfactor0 + +\b \cf0 \ul \ulc0 \ Certificate verification and OpenSSL \{UPDATED in 3.7.0b1]\ \b0 \ulnone \ @@ -53,17 +67,7 @@ The bundled \f0 included with this installer has its own default certificate store for verifying download connections.\ \ -\b \ul Update your version of Tcl/Tk to use IDLE or other Tk applications [CHANGED in 3.7.0b1] -\b0 \ulnone \ -\ -To use IDLE or other programs that use the Tkinter graphical user interface toolkit, you need to install a newer third-party version of the -\i Tcl/Tk -\i0 frameworks. Visit {\field{\*\fldinst{HYPERLINK "https://www.python.org/download/mac/tcltk/"}}{\fldrslt https://www.python.org/download/mac/tcltk/}} for current information about supported and recommended versions of -\i Tcl/Tk -\i0 for this version of Python and of Mac OS X. For 3.7.0b1, the 10.9 installer variant is linked with Tcl/Tk 8.6 which you must install separately. The 10.6 installer links /with Tcl/Tk 8.5 which you should also download and install.\ - -\b \ul \ -Other changes\ +\b \ul Other changes\ \b0 \ulnone \ For other changes in this release, see the diff --git a/Mac/BuildScript/resources/Welcome.rtf b/Mac/BuildScript/resources/Welcome.rtf index 18f3290608ec..4908fed0eaa5 100644 --- a/Mac/BuildScript/resources/Welcome.rtf +++ b/Mac/BuildScript/resources/Welcome.rtf @@ -23,7 +23,7 @@ \ \b NEW in 3.7.0b1: -\b0 two installer variants (10.9+ 64-bit-only, 10.6+ 64-/32-bit), Tcl/Tk 8.6 support in the 10.9+ variant, OpenSSL 1.1.0g, and more!\ +\b0 two installer variants (10.9+ 64-bit-only, 10.6+ 64-/32-bit), built-in Tcl/Tk 8.6 support in the 10.9+ variant (no additional third-party downloads!), OpenSSL 1.1.0g, and more!\ \ \b IMPORTANT: diff --git a/Misc/NEWS.d/next/macOS/2018-01-30-07-13-10.bpo-32726.tcARLK.rst b/Misc/NEWS.d/next/macOS/2018-01-30-07-13-10.bpo-32726.tcARLK.rst index bb7ebf1a62d2..6bdf0f12e972 100644 --- a/Misc/NEWS.d/next/macOS/2018-01-30-07-13-10.bpo-32726.tcARLK.rst +++ b/Misc/NEWS.d/next/macOS/2018-01-30-07-13-10.bpo-32726.tcARLK.rst @@ -1,4 +1,4 @@ Provide an additional, more modern macOS installer variant that supports macOS 10.9+ systems in 64-bit mode only. Upgrade the supplied third-party libraries to OpenSSL 1.1.0g and to SQLite 3.22.0. The 10.9+ installer now -links with Tcl/Tk 8.6. +links with and supplies its own copy of Tcl/Tk 8.6.

1 0

bpo-32735: Fix typo in f-strings datetime format specifier example (GH-5464) (GH-5465)
by Mariatta Jan. 31, 2018

Jan. 31, 2018

https://github.com/python/cpython/commit/581ce25e1ffa374e62547ef266b6326bee… commit: 581ce25e1ffa374e62547ef266b6326bee0c54e5 branch: 3.6 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: Mariatta <Mariatta(a)users.noreply.github.com> date: 2018-01-31T17:17:02-05:00 summary: bpo-32735: Fix typo in f-strings datetime format specifier example (GH-5464) (GH-5465) The f-string example for using datetime format specifier does not match the given output. Changed the format from %b to %B so it matches the output of "January". (cherry picked from commit b299393cc372f3ecbef4304f8eaa4c7838e975ca) Co-authored-by: Cheryl Sabella <cheryl.sabella(a)gmail.com> files: M Doc/reference/lexical_analysis.rst diff --git a/Doc/reference/lexical_analysis.rst b/Doc/reference/lexical_analysis.rst index 7ad83947e04a..6d093dcc2a22 100644 --- a/Doc/reference/lexical_analysis.rst +++ b/Doc/reference/lexical_analysis.rst @@ -676,7 +676,7 @@ Some examples of formatted string literals:: >>> f"result: {value:{width}.{precision}}" # nested fields 'result: 12.35' >>> today = datetime(year=2017, month=1, day=27) - >>> f"{today:%b %d, %Y}" # using date format specifier + >>> f"{today:%B %d, %Y}" # using date format specifier 'January 27, 2017' >>> number = 1024 >>> f"{number:#0x}" # using integer format specifier

1 0

bpo-32735: Fix typo in f-strings datetime format specifier example (GH-5464)
by Mariatta Jan. 31, 2018

Jan. 31, 2018

https://github.com/python/cpython/commit/b299393cc372f3ecbef4304f8eaa4c7838… commit: b299393cc372f3ecbef4304f8eaa4c7838e975ca branch: master author: Cheryl Sabella <cheryl.sabella(a)gmail.com> committer: Mariatta <Mariatta(a)users.noreply.github.com> date: 2018-01-31T16:37:51-05:00 summary: bpo-32735: Fix typo in f-strings datetime format specifier example (GH-5464) The f-string example for using datetime format specifier does not match the given output. Changed the format from %b to %B so it matches the output of "January". files: M Doc/reference/lexical_analysis.rst diff --git a/Doc/reference/lexical_analysis.rst b/Doc/reference/lexical_analysis.rst index 95b0f53a83e6..c6d96ca612b8 100644 --- a/Doc/reference/lexical_analysis.rst +++ b/Doc/reference/lexical_analysis.rst @@ -676,7 +676,7 @@ Some examples of formatted string literals:: >>> f"result: {value:{width}.{precision}}" # nested fields 'result: 12.35' >>> today = datetime(year=2017, month=1, day=27) - >>> f"{today:%b %d, %Y}" # using date format specifier + >>> f"{today:%B %d, %Y}" # using date format specifier 'January 27, 2017' >>> number = 1024 >>> f"{number:#0x}" # using integer format specifier

1 0

bpo-32727: smtplib's SMTP.send_message behaves differently with from_addr and to_addrs (GH-5451) (#5455)
by R. David Murray Jan. 31, 2018

Jan. 31, 2018

https://github.com/python/cpython/commit/85a92d00bd278022cc00fda6b124c06f61… commit: 85a92d00bd278022cc00fda6b124c06f614c5ebb branch: 3.6 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: R. David Murray <rdmurray(a)bitdance.com> date: 2018-01-31T15:54:09-05:00 summary: bpo-32727: smtplib's SMTP.send_message behaves differently with from_addr and to_addrs (GH-5451) (#5455) Do not pass the name field in the 'from' address in the SMTP envelope. (cherry picked from commit 8d83e4ba7823827bcbc119db887004d5c3a63dc6) Co-authored-by: Stéphane Wirtel <stephane(a)wirtel.be> files: A Misc/NEWS.d/next/Library/2018-01-30-17-46-18.bpo-32727.aHVsRC.rst M Lib/smtplib.py M Lib/test/test_smtplib.py diff --git a/Lib/smtplib.py b/Lib/smtplib.py index 5e422b704ad4..b679875fd2c5 100755 --- a/Lib/smtplib.py +++ b/Lib/smtplib.py @@ -933,6 +933,7 @@ def send_message(self, msg, from_addr=None, to_addrs=None, from_addr = (msg[header_prefix + 'Sender'] if (header_prefix + 'Sender') in msg else msg[header_prefix + 'From']) + from_addr = email.utils.getaddresses([from_addr])[0][1] if to_addrs is None: addr_fields = [f for f in (msg[header_prefix + 'To'], msg[header_prefix + 'Bcc'], diff --git a/Lib/test/test_smtplib.py b/Lib/test/test_smtplib.py index dd3a6eeca69c..67b3b478f76f 100644 --- a/Lib/test/test_smtplib.py +++ b/Lib/test/test_smtplib.py @@ -818,6 +818,7 @@ class SimSMTPServer(smtpd.SMTPServer): def __init__(self, *args, **kw): self._extra_features = [] + self._addresses = {} smtpd.SMTPServer.__init__(self, *args, **kw) def handle_accepted(self, conn, addr): @@ -826,7 +827,8 @@ def handle_accepted(self, conn, addr): decode_data=self._decode_data) def process_message(self, peer, mailfrom, rcpttos, data): - pass + self._addresses['from'] = mailfrom + self._addresses['tos'] = rcpttos def add_feature(self, feature): self._extra_features.append(feature) @@ -1066,6 +1068,21 @@ def test_send_unicode_without_SMTPUTF8(self): self.assertRaises(UnicodeEncodeError, smtp.sendmail, 'Alice', 'Böb', '') self.assertRaises(UnicodeEncodeError, smtp.mail, 'Älice') + def test_name_field_not_included_in_envelop_addresses(self): + smtp = smtplib.SMTP( + HOST, self.port, local_hostname='localhost', timeout=3 + ) + self.addCleanup(smtp.close) + + message = EmailMessage() + message['From'] = email.utils.formataddr(('Michaël', 'michael(a)example.com')) + message['To'] = email.utils.formataddr(('René', 'rene(a)example.com')) + + self.assertDictEqual(smtp.send_message(message), {}) + + self.assertEqual(self.serv._addresses['from'], 'michael(a)example.com') + self.assertEqual(self.serv._addresses['tos'], ['rene(a)example.com']) + class SimSMTPUTF8Server(SimSMTPServer): diff --git a/Misc/NEWS.d/next/Library/2018-01-30-17-46-18.bpo-32727.aHVsRC.rst b/Misc/NEWS.d/next/Library/2018-01-30-17-46-18.bpo-32727.aHVsRC.rst new file mode 100644 index 000000000000..22c219636de2 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-01-30-17-46-18.bpo-32727.aHVsRC.rst @@ -0,0 +1 @@ +Do not include name field in SMTP envelope from address. Patch by Stéphane Wirtel

1 0

bpo-25988: Emit a warning when use or import ABCs from 'collections'. (#5460)
by Serhiy Storchaka Jan. 31, 2018

Jan. 31, 2018

https://github.com/python/cpython/commit/c66f9f8d3909f588c251957d499599a168… commit: c66f9f8d3909f588c251957d499599a1680e2320 branch: master author: Serhiy Storchaka <storchaka(a)gmail.com> committer: GitHub <noreply(a)github.com> date: 2018-01-31T19:19:33+02:00 summary: bpo-25988: Emit a warning when use or import ABCs from 'collections'. (#5460) files: M Lib/collections/__init__.py M Modules/_decimal/_decimal.c diff --git a/Lib/collections/__init__.py b/Lib/collections/__init__.py index 8aeee359cd62..f0b41fd9d10d 100644 --- a/Lib/collections/__init__.py +++ b/Lib/collections/__init__.py @@ -17,16 +17,7 @@ __all__ = ['deque', 'defaultdict', 'namedtuple', 'UserDict', 'UserList', 'UserString', 'Counter', 'OrderedDict', 'ChainMap'] -# For backwards compatibility, continue to make the collections ABCs -# through Python 3.6 available through the collections module. -# Note, no new collections ABCs were added in Python 3.7 import _collections_abc -from _collections_abc import (AsyncGenerator, AsyncIterable, AsyncIterator, - Awaitable, ByteString, Callable, Collection, Container, Coroutine, - Generator, Hashable, ItemsView, Iterable, Iterator, KeysView, Mapping, - MappingView, MutableMapping, MutableSequence, MutableSet, Reversible, - Sequence, Set, Sized, ValuesView) - from operator import itemgetter as _itemgetter, eq as _eq from keyword import iskeyword as _iskeyword import sys as _sys @@ -40,7 +31,7 @@ except ImportError: pass else: - MutableSequence.register(deque) + _collections_abc.MutableSequence.register(deque) try: from _collections import defaultdict @@ -48,22 +39,37 @@ pass +def __getattr__(name): + # For backwards compatibility, continue to make the collections ABCs + # through Python 3.6 available through the collections module. + # Note, no new collections ABCs were added in Python 3.7 + if name in _collections_abc.__all__: + obj = getattr(_collections_abc, name) + import warnings + warnings.warn("Using or importing the ABCs from 'collections' instead " + "of from 'collections.abc' is deprecated, " + "and in 3.8 it will stop working", + DeprecationWarning, stacklevel=2) + globals()[name] = obj + return obj + raise AttributeError(f'module {__name__!r} has no attribute {name!r}') + ################################################################################ ### OrderedDict ################################################################################ -class _OrderedDictKeysView(KeysView): +class _OrderedDictKeysView(_collections_abc.KeysView): def __reversed__(self): yield from reversed(self._mapping) -class _OrderedDictItemsView(ItemsView): +class _OrderedDictItemsView(_collections_abc.ItemsView): def __reversed__(self): for key in reversed(self._mapping): yield (key, self._mapping[key]) -class _OrderedDictValuesView(ValuesView): +class _OrderedDictValuesView(_collections_abc.ValuesView): def __reversed__(self): for key in reversed(self._mapping): @@ -215,7 +221,7 @@ def __sizeof__(self): size += sizeof(self.__root) * n # proxy objects return size - update = __update = MutableMapping.update + update = __update = _collections_abc.MutableMapping.update def keys(self): "D.keys() -> a set-like object providing a view on D's keys" @@ -229,7 +235,7 @@ def values(self): "D.values() -> an object providing a view on D's values" return _OrderedDictValuesView(self) - __ne__ = MutableMapping.__ne__ + __ne__ = _collections_abc.MutableMapping.__ne__ __marker = object() @@ -636,7 +642,7 @@ def update(*args, **kwds): raise TypeError('expected at most 1 arguments, got %d' % len(args)) iterable = args[0] if args else None if iterable is not None: - if isinstance(iterable, Mapping): + if isinstance(iterable, _collections_abc.Mapping): if self: self_get = self.get for elem, count in iterable.items(): @@ -673,7 +679,7 @@ def subtract(*args, **kwds): iterable = args[0] if args else None if iterable is not None: self_get = self.get - if isinstance(iterable, Mapping): + if isinstance(iterable, _collections_abc.Mapping): for elem, count in iterable.items(): self[elem] = self_get(elem, 0) - count else: @@ -875,7 +881,7 @@ def __iand__(self, other): ### ChainMap ######################################################################## -class ChainMap(MutableMapping): +class ChainMap(_collections_abc.MutableMapping): ''' A ChainMap groups multiple dicts (or other mappings) together to create a single, updateable view. @@ -983,7 +989,7 @@ def clear(self): ### UserDict ################################################################################ -class UserDict(MutableMapping): +class UserDict(_collections_abc.MutableMapping): # Start by filling-out the abstract methods def __init__(*args, **kwargs): @@ -1050,7 +1056,7 @@ def fromkeys(cls, iterable, value=None): ### UserList ################################################################################ -class UserList(MutableSequence): +class UserList(_collections_abc.MutableSequence): """A more or less complete user-defined wrapper around list objects.""" def __init__(self, initlist=None): self.data = [] @@ -1123,7 +1129,7 @@ def extend(self, other): ### UserString ################################################################################ -class UserString(Sequence): +class UserString(_collections_abc.Sequence): def __init__(self, seq): if isinstance(seq, str): self.data = seq diff --git a/Modules/_decimal/_decimal.c b/Modules/_decimal/_decimal.c index fddb39ef652a..5bce780cb7fb 100644 --- a/Modules/_decimal/_decimal.c +++ b/Modules/_decimal/_decimal.c @@ -5521,6 +5521,7 @@ PyInit__decimal(void) PyObject *numbers = NULL; PyObject *Number = NULL; PyObject *collections = NULL; + PyObject *collections_abc = NULL; PyObject *MutableMapping = NULL; PyObject *obj = NULL; DecCondMap *cm; @@ -5595,7 +5596,8 @@ PyInit__decimal(void) Py_CLEAR(obj); /* MutableMapping */ - ASSIGN_PTR(MutableMapping, PyObject_GetAttrString(collections, + ASSIGN_PTR(collections_abc, PyImport_ImportModule("collections.abc")); + ASSIGN_PTR(MutableMapping, PyObject_GetAttrString(collections_abc, "MutableMapping")); /* Create SignalDict type */ ASSIGN_PTR(PyDecSignalDict_Type, @@ -5606,6 +5608,7 @@ PyInit__decimal(void) /* Done with collections, MutableMapping */ Py_CLEAR(collections); + Py_CLEAR(collections_abc); Py_CLEAR(MutableMapping); @@ -5765,6 +5768,7 @@ PyInit__decimal(void) Py_CLEAR(Number); /* GCOV_NOT_REACHED */ Py_CLEAR(Rational); /* GCOV_NOT_REACHED */ Py_CLEAR(collections); /* GCOV_NOT_REACHED */ + Py_CLEAR(collections_abc); /* GCOV_NOT_REACHED */ Py_CLEAR(MutableMapping); /* GCOV_NOT_REACHED */ Py_CLEAR(SignalTuple); /* GCOV_NOT_REACHED */ Py_CLEAR(DecimalTuple); /* GCOV_NOT_REACHED */

1 0

Make site documentation more clear (#5461)
by Xiang Zhang Jan. 31, 2018

Jan. 31, 2018

https://github.com/python/cpython/commit/a29ddf407831c9103e4df4850df6d67b43… commit: a29ddf407831c9103e4df4850df6d67b4352f4f9 branch: master author: Xiang Zhang <angwerzx(a)126.com> committer: GitHub <noreply(a)github.com> date: 2018-01-31T23:50:50+08:00 summary: Make site documentation more clear (#5461) Mention only ImportError caused by importing sitecustomize.py/usercustomize.py themselves will be silently ignored. files: M Doc/library/site.rst diff --git a/Doc/library/site.rst b/Doc/library/site.rst index 43daf790b77c..ae408133a7ce 100644 --- a/Doc/library/site.rst +++ b/Doc/library/site.rst @@ -97,12 +97,12 @@ not mentioned in either path configuration file. After these path manipulations, an attempt is made to import a module named :mod:`sitecustomize`, which can perform arbitrary site-specific customizations. It is typically created by a system administrator in the site-packages -directory. If this import fails with an :exc:`ImportError` exception, it is -silently ignored. If Python is started without output streams available, as +directory. If this import fails with an :exc:`ImportError` or its subclass +exception, and the exception's :attr:`name` attribute equals to ``'sitecustomize'``, +it is silently ignored. If Python is started without output streams available, as with :file:`pythonw.exe` on Windows (which is used by default to start IDLE), -attempted output from :mod:`sitecustomize` is ignored. Any exception other -than :exc:`ImportError` causes a silent and perhaps mysterious failure of the -process. +attempted output from :mod:`sitecustomize` is ignored. Any other exception +causes a silent and perhaps mysterious failure of the process. .. index:: module: usercustomize @@ -110,7 +110,9 @@ After this, an attempt is made to import a module named :mod:`usercustomize`, which can perform arbitrary user-specific customizations, if :data:`ENABLE_USER_SITE` is true. This file is intended to be created in the user site-packages directory (see below), which is part of ``sys.path`` unless -disabled by :option:`-s`. An :exc:`ImportError` will be silently ignored. +disabled by :option:`-s`. If this import fails with an :exc:`ImportError` or +its subclass exception, and the exception's :attr:`name` attribute equals to +``'usercustomize'``, it is silently ignored. Note that for some non-Unix systems, ``sys.prefix`` and ``sys.exec_prefix`` are empty, and the path manipulations are skipped; however the import of

1 0

[3.6] bpo-32583: Fix possible crashing in builtin Unicode decoders (GH-5325) (#5459)
by Xiang Zhang Jan. 31, 2018

Jan. 31, 2018

https://github.com/python/cpython/commit/ea94fce6960d90fffeeda131e310246179… commit: ea94fce6960d90fffeeda131e31024617912d231 branch: 3.6 author: Xiang Zhang <angwerzx(a)126.com> committer: GitHub <noreply(a)github.com> date: 2018-01-31T21:34:17+08:00 summary: [3.6] bpo-32583: Fix possible crashing in builtin Unicode decoders (GH-5325) (#5459) When using customized decode error handlers, it is possible for builtin decoders to write out-of-bounds and then crash.. (cherry picked from commit 2c7fd46e11333ef5e5cce34212f7d087694f3658) files: A Misc/NEWS.d/next/Core and Builtins/2018-01-26-21-20-21.bpo-32583.Fh3fau.rst M Lib/test/test_codeccallbacks.py M Objects/unicodeobject.c diff --git a/Lib/test/test_codeccallbacks.py b/Lib/test/test_codeccallbacks.py index 0c066e633fa8..e2e7463389ef 100644 --- a/Lib/test/test_codeccallbacks.py +++ b/Lib/test/test_codeccallbacks.py @@ -1044,6 +1044,58 @@ def mutating(exc): for (encoding, data) in baddata: self.assertEqual(data.decode(encoding, "test.mutating"), "\u4242") + # issue32583 + def test_crashing_decode_handler(self): + # better generating one more character to fill the extra space slot + # so in debug build it can steadily fail + def forward_shorter_than_end(exc): + if isinstance(exc, UnicodeDecodeError): + # size one character, 0 < forward < exc.end + return ('\ufffd', exc.start+1) + else: + raise TypeError("don't know how to handle %r" % exc) + codecs.register_error( + "test.forward_shorter_than_end", forward_shorter_than_end) + + self.assertEqual( + b'\xd8\xd8\xd8\xd8\xd8\x00\x00\x00'.decode( + 'utf-16-le', 'test.forward_shorter_than_end'), + '\ufffd\ufffd\ufffd\ufffd\xd8\x00' + ) + self.assertEqual( + b'\xd8\xd8\xd8\xd8\x00\xd8\x00\x00'.decode( + 'utf-16-be', 'test.forward_shorter_than_end'), + '\ufffd\ufffd\ufffd\ufffd\xd8\x00' + ) + self.assertEqual( + b'\x11\x11\x11\x11\x11\x00\x00\x00\x00\x00\x00'.decode( + 'utf-32-le', 'test.forward_shorter_than_end'), + '\ufffd\ufffd\ufffd\u1111\x00' + ) + self.assertEqual( + b'\x11\x11\x11\x00\x00\x11\x11\x00\x00\x00\x00'.decode( + 'utf-32-be', 'test.forward_shorter_than_end'), + '\ufffd\ufffd\ufffd\u1111\x00' + ) + + def replace_with_long(exc): + if isinstance(exc, UnicodeDecodeError): + exc.object = b"\x00" * 8 + return ('\ufffd', exc.start) + else: + raise TypeError("don't know how to handle %r" % exc) + codecs.register_error("test.replace_with_long", replace_with_long) + + self.assertEqual( + b'\x00'.decode('utf-16', 'test.replace_with_long'), + '\ufffd\x00\x00\x00\x00' + ) + self.assertEqual( + b'\x00'.decode('utf-32', 'test.replace_with_long'), + '\ufffd\x00\x00' + ) + + def test_fake_error_class(self): handlers = [ codecs.strict_errors, diff --git a/Misc/NEWS.d/next/Core and Builtins/2018-01-26-21-20-21.bpo-32583.Fh3fau.rst b/Misc/NEWS.d/next/Core and Builtins/2018-01-26-21-20-21.bpo-32583.Fh3fau.rst new file mode 100644 index 000000000000..45f1d043f9d6 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2018-01-26-21-20-21.bpo-32583.Fh3fau.rst @@ -0,0 +1,2 @@ +Fix possible crashing in builtin Unicode decoders caused by write +out-of-bound errors when using customized decode error handlers. diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 9ccd06eebb48..a246756d9412 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -4429,7 +4429,10 @@ unicode_decode_call_errorhandler_writer( Py_ssize_t insize; Py_ssize_t newpos; Py_ssize_t replen; + Py_ssize_t remain; PyObject *inputobj = NULL; + int need_to_grow = 0; + const char *new_inptr; if (*errorHandler == NULL) { *errorHandler = PyCodec_LookupError(errors); @@ -4463,6 +4466,7 @@ unicode_decode_call_errorhandler_writer( if (!PyBytes_Check(inputobj)) { PyErr_Format(PyExc_TypeError, "exception attribute object must be bytes"); } + remain = *inend - *input - *endinpos; *input = PyBytes_AS_STRING(inputobj); insize = PyBytes_GET_SIZE(inputobj); *inend = *input + insize; @@ -4482,6 +4486,19 @@ unicode_decode_call_errorhandler_writer( replen = PyUnicode_GET_LENGTH(repunicode); if (replen > 1) { writer->min_length += replen - 1; + need_to_grow = 1; + } + new_inptr = *input + newpos; + if (*inend - new_inptr > remain) { + /* We don't know the decoding algorithm here so we make the worst + assumption that one byte decodes to one unicode character. + If unfortunately one byte could decode to more unicode characters, + the decoder may write out-of-bound then. Is it possible for the + algorithms using this function? */ + writer->min_length += *inend - new_inptr - remain; + need_to_grow = 1; + } + if (need_to_grow) { writer->overallocate = 1; if (_PyUnicodeWriter_Prepare(writer, writer->min_length, PyUnicode_MAX_CHAR_VALUE(repunicode)) == -1) @@ -4491,7 +4508,7 @@ unicode_decode_call_errorhandler_writer( goto onError; *endinpos = newpos; - *inptr = *input + newpos; + *inptr = new_inptr; /* we made it! */ Py_XDECREF(restuple); @@ -5663,7 +5680,8 @@ PyUnicode_DecodeUTF16Stateful(const char *s, #endif /* Note: size will always be longer than the resulting Unicode - character count */ + character count normally. Error handler will take care of + resizing when needed. */ _PyUnicodeWriter_Init(&writer); writer.min_length = (e - q + 1) / 2; if (_PyUnicodeWriter_Prepare(&writer, writer.min_length, 127) == -1)

1 0

bpo-32583: Fix possible crashing in builtin Unicode decoders (#5325)
by Xiang Zhang Jan. 31, 2018

Jan. 31, 2018

https://github.com/python/cpython/commit/2c7fd46e11333ef5e5cce34212f7d08769… commit: 2c7fd46e11333ef5e5cce34212f7d087694f3658 branch: master author: Xiang Zhang <angwerzx(a)126.com> committer: GitHub <noreply(a)github.com> date: 2018-01-31T20:48:05+08:00 summary: bpo-32583: Fix possible crashing in builtin Unicode decoders (#5325) When using customized decode error handlers, it is possible for builtin decoders to write out-of-bounds and then crash. files: A Misc/NEWS.d/next/Core and Builtins/2018-01-26-21-20-21.bpo-32583.Fh3fau.rst M Lib/test/test_codeccallbacks.py M Objects/unicodeobject.c diff --git a/Lib/test/test_codeccallbacks.py b/Lib/test/test_codeccallbacks.py index 0c066e633fa8..e2e7463389ef 100644 --- a/Lib/test/test_codeccallbacks.py +++ b/Lib/test/test_codeccallbacks.py @@ -1044,6 +1044,58 @@ def mutating(exc): for (encoding, data) in baddata: self.assertEqual(data.decode(encoding, "test.mutating"), "\u4242") + # issue32583 + def test_crashing_decode_handler(self): + # better generating one more character to fill the extra space slot + # so in debug build it can steadily fail + def forward_shorter_than_end(exc): + if isinstance(exc, UnicodeDecodeError): + # size one character, 0 < forward < exc.end + return ('\ufffd', exc.start+1) + else: + raise TypeError("don't know how to handle %r" % exc) + codecs.register_error( + "test.forward_shorter_than_end", forward_shorter_than_end) + + self.assertEqual( + b'\xd8\xd8\xd8\xd8\xd8\x00\x00\x00'.decode( + 'utf-16-le', 'test.forward_shorter_than_end'), + '\ufffd\ufffd\ufffd\ufffd\xd8\x00' + ) + self.assertEqual( + b'\xd8\xd8\xd8\xd8\x00\xd8\x00\x00'.decode( + 'utf-16-be', 'test.forward_shorter_than_end'), + '\ufffd\ufffd\ufffd\ufffd\xd8\x00' + ) + self.assertEqual( + b'\x11\x11\x11\x11\x11\x00\x00\x00\x00\x00\x00'.decode( + 'utf-32-le', 'test.forward_shorter_than_end'), + '\ufffd\ufffd\ufffd\u1111\x00' + ) + self.assertEqual( + b'\x11\x11\x11\x00\x00\x11\x11\x00\x00\x00\x00'.decode( + 'utf-32-be', 'test.forward_shorter_than_end'), + '\ufffd\ufffd\ufffd\u1111\x00' + ) + + def replace_with_long(exc): + if isinstance(exc, UnicodeDecodeError): + exc.object = b"\x00" * 8 + return ('\ufffd', exc.start) + else: + raise TypeError("don't know how to handle %r" % exc) + codecs.register_error("test.replace_with_long", replace_with_long) + + self.assertEqual( + b'\x00'.decode('utf-16', 'test.replace_with_long'), + '\ufffd\x00\x00\x00\x00' + ) + self.assertEqual( + b'\x00'.decode('utf-32', 'test.replace_with_long'), + '\ufffd\x00\x00' + ) + + def test_fake_error_class(self): handlers = [ codecs.strict_errors, diff --git a/Misc/NEWS.d/next/Core and Builtins/2018-01-26-21-20-21.bpo-32583.Fh3fau.rst b/Misc/NEWS.d/next/Core and Builtins/2018-01-26-21-20-21.bpo-32583.Fh3fau.rst new file mode 100644 index 000000000000..45f1d043f9d6 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2018-01-26-21-20-21.bpo-32583.Fh3fau.rst @@ -0,0 +1,2 @@ +Fix possible crashing in builtin Unicode decoders caused by write +out-of-bound errors when using customized decode error handlers. diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 775bd15cf6d1..3d9e09d7fab1 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -4190,7 +4190,10 @@ unicode_decode_call_errorhandler_writer( Py_ssize_t insize; Py_ssize_t newpos; Py_ssize_t replen; + Py_ssize_t remain; PyObject *inputobj = NULL; + int need_to_grow = 0; + const char *new_inptr; if (*errorHandler == NULL) { *errorHandler = PyCodec_LookupError(errors); @@ -4221,6 +4224,7 @@ unicode_decode_call_errorhandler_writer( inputobj = PyUnicodeDecodeError_GetObject(*exceptionObject); if (!inputobj) goto onError; + remain = *inend - *input - *endinpos; *input = PyBytes_AS_STRING(inputobj); insize = PyBytes_GET_SIZE(inputobj); *inend = *input + insize; @@ -4238,6 +4242,19 @@ unicode_decode_call_errorhandler_writer( replen = PyUnicode_GET_LENGTH(repunicode); if (replen > 1) { writer->min_length += replen - 1; + need_to_grow = 1; + } + new_inptr = *input + newpos; + if (*inend - new_inptr > remain) { + /* We don't know the decoding algorithm here so we make the worst + assumption that one byte decodes to one unicode character. + If unfortunately one byte could decode to more unicode characters, + the decoder may write out-of-bound then. Is it possible for the + algorithms using this function? */ + writer->min_length += *inend - new_inptr - remain; + need_to_grow = 1; + } + if (need_to_grow) { writer->overallocate = 1; if (_PyUnicodeWriter_Prepare(writer, writer->min_length, PyUnicode_MAX_CHAR_VALUE(repunicode)) == -1) @@ -4247,7 +4264,7 @@ unicode_decode_call_errorhandler_writer( goto onError; *endinpos = newpos; - *inptr = *input + newpos; + *inptr = new_inptr; /* we made it! */ Py_DECREF(restuple); @@ -5572,7 +5589,8 @@ PyUnicode_DecodeUTF16Stateful(const char *s, #endif /* Note: size will always be longer than the resulting Unicode - character count */ + character count normally. Error handler will take care of + resizing when needed. */ _PyUnicodeWriter_Init(&writer); writer.min_length = (e - q + 1) / 2; if (_PyUnicodeWriter_Prepare(&writer, writer.min_length, 127) == -1)

1 0

bpo-30693: zip+tarfile: sort directory listing (#2263)
by Victor Stinner Jan. 31, 2018

Jan. 31, 2018

https://github.com/python/cpython/commit/84521047e413d7d1150aaa1c333580b683… commit: 84521047e413d7d1150aaa1c333580b683b3f4b1 branch: master author: Bernhard M. Wiedemann <githubbmw(a)lsmod.de> committer: Victor Stinner <victor.stinner(a)gmail.com> date: 2018-01-31T11:17:10+01:00 summary: bpo-30693: zip+tarfile: sort directory listing (#2263) tarfile and zipfile now sort directory listing to generate tar and zip archives in a more reproducible way. See also https://reproducible-builds.org/docs/stable-inputs/ on that topic. files: A Misc/NEWS.d/next/Library/2017-11-27-15-09-49.bpo-30693.yC4mJ7.rst A Misc/NEWS.d/next/Library/2017-11-27-15-09-49.bpo-30693.yC4mJ8.rst M Doc/library/tarfile.rst M Doc/library/zipfile.rst M Lib/tarfile.py M Lib/test/test_tarfile.py M Lib/zipfile.py diff --git a/Doc/library/tarfile.rst b/Doc/library/tarfile.rst index 2450716a1d91..9cd07158e7f6 100644 --- a/Doc/library/tarfile.rst +++ b/Doc/library/tarfile.rst @@ -451,7 +451,8 @@ be finalized; only the internally used file object will be closed. See the (directory, fifo, symbolic link, etc.). If given, *arcname* specifies an alternative name for the file in the archive. Directories are added recursively by default. This can be avoided by setting *recursive* to - :const:`False`. If *filter* is given, it + :const:`False`. Recursion adds entries in sorted order. + If *filter* is given, it should be a function that takes a :class:`TarInfo` object argument and returns the changed :class:`TarInfo` object. If it instead returns :const:`None` the :class:`TarInfo` object will be excluded from the @@ -460,6 +461,9 @@ be finalized; only the internally used file object will be closed. See the .. versionchanged:: 3.2 Added the *filter* parameter. + .. versionchanged:: 3.7 + Recursion adds entries in sorted order. + .. method:: TarFile.addfile(tarinfo, fileobj=None) diff --git a/Doc/library/zipfile.rst b/Doc/library/zipfile.rst index 7c9a8c802254..c0f2a89a3a17 100644 --- a/Doc/library/zipfile.rst +++ b/Doc/library/zipfile.rst @@ -491,7 +491,7 @@ The :class:`PyZipFile` constructor takes the same parameters as the :file:`\*.pyc` are added at the top level. If the directory is a package directory, then all :file:`\*.pyc` are added under the package name as a file path, and if any subdirectories are package directories, - all of these are added recursively. + all of these are added recursively in sorted order. *basename* is intended for internal use only. @@ -524,6 +524,9 @@ The :class:`PyZipFile` constructor takes the same parameters as the .. versionchanged:: 3.6.2 The *pathname* parameter accepts a :term:`path-like object`. + .. versionchanged:: 3.7 + Recursion sorts directory entries. + .. _zipinfo-objects: diff --git a/Lib/tarfile.py b/Lib/tarfile.py index 0b8d31f85cf3..a24ee42abf82 100755 --- a/Lib/tarfile.py +++ b/Lib/tarfile.py @@ -1943,7 +1943,7 @@ def add(self, name, arcname=None, recursive=True, *, filter=None): elif tarinfo.isdir(): self.addfile(tarinfo) if recursive: - for f in os.listdir(name): + for f in sorted(os.listdir(name)): self.add(os.path.join(name, f), os.path.join(arcname, f), recursive, filter=filter) diff --git a/Lib/test/test_tarfile.py b/Lib/test/test_tarfile.py index 179cbc6dfffc..8ef4294921b2 100644 --- a/Lib/test/test_tarfile.py +++ b/Lib/test/test_tarfile.py @@ -1129,6 +1129,30 @@ def test_directory_size(self): finally: support.rmdir(path) + # mock the following: + # os.listdir: so we know that files are in the wrong order + @unittest.mock.patch('os.listdir') + def test_ordered_recursion(self, mock_listdir): + path = os.path.join(TEMPDIR, "directory") + os.mkdir(path) + open(os.path.join(path, "1"), "a").close() + open(os.path.join(path, "2"), "a").close() + mock_listdir.return_value = ["2", "1"] + try: + tar = tarfile.open(tmpname, self.mode) + try: + tar.add(path) + paths = [] + for m in tar.getmembers(): + paths.append(os.path.split(m.name)[-1]) + self.assertEqual(paths, ["directory", "1", "2"]); + finally: + tar.close() + finally: + support.unlink(os.path.join(path, "1")) + support.unlink(os.path.join(path, "2")) + support.rmdir(path) + def test_gettarinfo_pathlike_name(self): with tarfile.open(tmpname, self.mode) as tar: path = pathlib.Path(TEMPDIR) / "file" diff --git a/Lib/zipfile.py b/Lib/zipfile.py index 5df7b1bf75b9..b90b60f72e2b 100644 --- a/Lib/zipfile.py +++ b/Lib/zipfile.py @@ -1940,7 +1940,7 @@ def writepy(self, pathname, basename="", filterfunc=None): if self.debug: print("Adding", arcname) self.write(fname, arcname) - dirlist = os.listdir(pathname) + dirlist = sorted(os.listdir(pathname)) dirlist.remove("__init__.py") # Add all *.py files and package subdirectories for filename in dirlist: @@ -1965,7 +1965,7 @@ def writepy(self, pathname, basename="", filterfunc=None): # This is NOT a package directory, add its files at top level if self.debug: print("Adding files from directory", pathname) - for filename in os.listdir(pathname): + for filename in sorted(os.listdir(pathname)): path = os.path.join(pathname, filename) root, ext = os.path.splitext(filename) if ext == ".py": @@ -2116,7 +2116,7 @@ def addToZip(zf, path, zippath): elif os.path.isdir(path): if zippath: zf.write(path, zippath) - for nm in os.listdir(path): + for nm in sorted(os.listdir(path)): addToZip(zf, os.path.join(path, nm), os.path.join(zippath, nm)) # else: ignore diff --git a/Misc/NEWS.d/next/Library/2017-11-27-15-09-49.bpo-30693.yC4mJ7.rst b/Misc/NEWS.d/next/Library/2017-11-27-15-09-49.bpo-30693.yC4mJ7.rst new file mode 100644 index 000000000000..9c895c53de12 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2017-11-27-15-09-49.bpo-30693.yC4mJ7.rst @@ -0,0 +1 @@ +The ZipFile class now recurses directories in a reproducible way. diff --git a/Misc/NEWS.d/next/Library/2017-11-27-15-09-49.bpo-30693.yC4mJ8.rst b/Misc/NEWS.d/next/Library/2017-11-27-15-09-49.bpo-30693.yC4mJ8.rst new file mode 100644 index 000000000000..a622e7ed6e5d --- /dev/null +++ b/Misc/NEWS.d/next/Library/2017-11-27-15-09-49.bpo-30693.yC4mJ8.rst @@ -0,0 +1 @@ +The TarFile class now recurses directories in a reproducible way.

1 0

Daily reference leaks (4243df51fe43): sum=-3
by solipsis＠pitrou.net Jan. 31, 2018

Jan. 31, 2018

results for 4243df51fe43 on branch "default" -------------------------------------------- test_collections leaked [-7, 8, -7] memory blocks, sum=-6 test_functools leaked [0, 3, 1] memory blocks, sum=4 test_multiprocessing_forkserver leaked [-2, 2, -1] memory blocks, sum=-1 test_multiprocessing_spawn leaked [0, -2, 2] memory blocks, sum=0 Command line was: ['./python', '-m', 'test.regrtest', '-uall', '-R', '3:3:/home/psf-users/antoine/refleaks/reflogsse3Qi', '--timeout', '7200']

1 0