Mailman 3 January 2018 - Python-checkins

bpo-32727: smtplib's SMTP.send_message behaves differently with from_addr and to_addrs (#5451)
by R. David Murray Jan. 31, 2018

Jan. 31, 2018

https://github.com/python/cpython/commit/8d83e4ba7823827bcbc119db887004d5c3… commit: 8d83e4ba7823827bcbc119db887004d5c3a63dc6 branch: master author: Stéphane Wirtel <stephane(a)wirtel.be> committer: R. David Murray <rdmurray(a)bitdance.com> date: 2018-01-30T19:02:51-05:00 summary: bpo-32727: smtplib's SMTP.send_message behaves differently with from_addr and to_addrs (#5451) Do not pass the name field in the 'from' address in the SMTP envelope. files: A Misc/NEWS.d/next/Library/… [View More]2018-01-30-17-46-18.bpo-32727.aHVsRC.rst M Lib/smtplib.py M Lib/test/test_smtplib.py diff --git a/Lib/smtplib.py b/Lib/smtplib.py index 5e422b704ad4..b679875fd2c5 100755 --- a/Lib/smtplib.py +++ b/Lib/smtplib.py @@ -933,6 +933,7 @@ def send_message(self, msg, from_addr=None, to_addrs=None, from_addr = (msg[header_prefix + 'Sender'] if (header_prefix + 'Sender') in msg else msg[header_prefix + 'From']) + from_addr = email.utils.getaddresses([from_addr])[0][1] if to_addrs is None: addr_fields = [f for f in (msg[header_prefix + 'To'], msg[header_prefix + 'Bcc'], diff --git a/Lib/test/test_smtplib.py b/Lib/test/test_smtplib.py index 040ad4e05962..7991174fb5d7 100644 --- a/Lib/test/test_smtplib.py +++ b/Lib/test/test_smtplib.py @@ -825,6 +825,7 @@ class SimSMTPServer(smtpd.SMTPServer): def __init__(self, *args, **kw): self._extra_features = [] + self._addresses = {} smtpd.SMTPServer.__init__(self, *args, **kw) def handle_accepted(self, conn, addr): @@ -833,7 +834,8 @@ def handle_accepted(self, conn, addr): decode_data=self._decode_data) def process_message(self, peer, mailfrom, rcpttos, data): - pass + self._addresses['from'] = mailfrom + self._addresses['tos'] = rcpttos def add_feature(self, feature): self._extra_features.append(feature) @@ -1072,6 +1074,21 @@ def test_send_unicode_without_SMTPUTF8(self): self.assertRaises(UnicodeEncodeError, smtp.sendmail, 'Alice', 'Böb', '') self.assertRaises(UnicodeEncodeError, smtp.mail, 'Älice') + def test_name_field_not_included_in_envelop_addresses(self): + smtp = smtplib.SMTP( + HOST, self.port, local_hostname='localhost', timeout=3 + ) + self.addCleanup(smtp.close) + + message = EmailMessage() + message['From'] = email.utils.formataddr(('Michaël', 'michael(a)example.com')) + message['To'] = email.utils.formataddr(('René', 'rene(a)example.com')) + + self.assertDictEqual(smtp.send_message(message), {}) + + self.assertEqual(self.serv._addresses['from'], 'michael(a)example.com') + self.assertEqual(self.serv._addresses['tos'], ['rene(a)example.com']) + class SimSMTPUTF8Server(SimSMTPServer): diff --git a/Misc/NEWS.d/next/Library/2018-01-30-17-46-18.bpo-32727.aHVsRC.rst b/Misc/NEWS.d/next/Library/2018-01-30-17-46-18.bpo-32727.aHVsRC.rst new file mode 100644 index 000000000000..22c219636de2 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-01-30-17-46-18.bpo-32727.aHVsRC.rst @@ -0,0 +1 @@ +Do not include name field in SMTP envelope from address. Patch by Stéphane Wirtel [View Less]

1 0

[3.6] compare with difflib not diff(1) (GH-5450) (GH-5453)
by Benjamin Peterson Jan. 30, 2018

Jan. 30, 2018

https://github.com/python/cpython/commit/eb126eddbd7542ac9d7cd2736116aee2e0… commit: eb126eddbd7542ac9d7cd2736116aee2e0bd03dd branch: 3.6 author: Benjamin Peterson <benjamin(a)python.org> committer: GitHub <noreply(a)github.com> date: 2018-01-30T11:31:10-08:00 summary: [3.6] compare with difflib not diff(1) (GH-5450) (GH-5453) Co-authored-by: Benjamin Peterson <benjamin(a)python.org> files: M Lib/lib2to3/tests/test_parser.py diff --git a/Lib/lib2to3/tests/test_parser.py b/… [View More]

1 0

[3.6] closes bpo-30117: fix lib2to3 ParserIdempotency test (GH-1242) (GH-5443)
by Benjamin Peterson Jan. 30, 2018

Jan. 30, 2018

https://github.com/python/cpython/commit/a23a2c555c4187f349276fe2f2ceffa953… commit: a23a2c555c4187f349276fe2f2ceffa953d0afe9 branch: 3.6 author: Benjamin Peterson <benjamin(a)python.org> committer: GitHub <noreply(a)github.com> date: 2018-01-30T10:23:17-08:00 summary: [3.6] closes bpo-30117: fix lib2to3 ParserIdempotency test (GH-1242) (GH-5443) Fix two (in my opinion) spurious failure conditions in the lib2to3.tests.test_parser.TestParserIdempotency test_parser test. Use … [View More]the same encoding found in the initial file to write a temp file for a diff. This retains the BOM if the encoding was initially utf-8-sig. If the file cannot be parsed using the normal grammar, try again with no print statement which should succeed for valid files using future print_function For case (1), the driver was correctly handling a BOM in a utf-8 file, but then the test was not writing a comparison file using 'utf-8-sig' to diff against, so the BOM got removed. I don't think that is the fault of the parser, and lib2to3 will retain the BOM. For case (2), lib2to3 pre-detects the use of from __future__ import print_function or allows the user to force this interpretation with a -p flag, and then selects a different grammar with the print statement removed. That makes the test cases unfair to this test as the driver itself doesn't know which grammar to use. As a minimal fix, the test will try using a grammar with the print statement, and if that fails fall back on a grammar without it. A more thorough handling of the idempotency test would to be to parse all files using both grammars and ignore if one of the two failed but otherwise check both. I didn't think this was necessary but can change.. (cherry picked from commit 14e976e00e65bf343ba0fca016c3c9132a843daf) files: M Lib/lib2to3/tests/support.py M Lib/lib2to3/tests/test_parser.py diff --git a/Lib/lib2to3/tests/support.py b/Lib/lib2to3/tests/support.py index ae7cfe8ee274..fe084e8903fc 100644 --- a/Lib/lib2to3/tests/support.py +++ b/Lib/lib2to3/tests/support.py @@ -15,7 +15,13 @@ proj_dir = os.path.normpath(os.path.join(test_dir, "..")) grammar_path = os.path.join(test_dir, "..", "Grammar.txt") grammar = pgen2_driver.load_grammar(grammar_path) +grammar_no_print_statement = pgen2_driver.load_grammar(grammar_path) +del grammar_no_print_statement.keywords["print"] driver = pgen2_driver.Driver(grammar, convert=pytree.convert) +driver_no_print_statement = pgen2_driver.Driver( + grammar_no_print_statement, + convert=pytree.convert +) def parse_string(string): return driver.parse_string(reformat(string), debug=True) diff --git a/Lib/lib2to3/tests/test_parser.py b/Lib/lib2to3/tests/test_parser.py index 0a139507b8de..9d444c204aa6 100644 --- a/Lib/lib2to3/tests/test_parser.py +++ b/Lib/lib2to3/tests/test_parser.py @@ -8,7 +8,7 @@ # Testing imports from . import support -from .support import driver +from .support import driver, driver_no_print_statement from test.support import verbose # Python imports @@ -413,8 +413,6 @@ class TestParserIdempotency(support.TestCase): """A cut-down version of pytree_idempotency.py.""" - # Issue 13125 - @unittest.expectedFailure def test_all_project_files(self): for filepath in support.all_project_files(): with open(filepath, "rb") as fp: @@ -425,12 +423,13 @@ def test_all_project_files(self): source = fp.read() try: tree = driver.parse_string(source) - except ParseError as err: - if verbose > 0: - warnings.warn('ParseError on file %s (%s)' % (filepath, err)) - continue + except ParseError: + try: + tree = driver_no_print_statement.parse_string(source) + except ParseError as err: + self.fail('ParseError on file %s (%s)' % (filepath, err)) new = str(tree) - x = diff(filepath, new) + x = diff(filepath, new, encoding=encoding) if x: self.fail("Idempotency failed: %s" % filepath) @@ -474,9 +473,9 @@ def test_multiline_str_literals(self): self.validate(s) -def diff(fn, result): +def diff(fn, result, encoding='utf-8'): try: - with open('@', 'w') as f: + with open('@', 'w', encoding=encoding, newline='\n') as f: f.write(str(result)) fn = fn.replace('"', '\\"') return subprocess.call(['diff', '-u', fn, '@'], stdout=(subprocess.DEVNULL if verbose < 1 else None)) [View Less]

1 0

compare with difflib not diff(1) (GH-5450)
by Benjamin Peterson Jan. 30, 2018

Jan. 30, 2018

https://github.com/python/cpython/commit/1e17d4aaff5c7ca972bab437949d2bb51c… commit: 1e17d4aaff5c7ca972bab437949d2bb51c5b30f7 branch: master author: Benjamin Peterson <benjamin(a)python.org> committer: GitHub <noreply(a)github.com> date: 2018-01-30T09:03:12-08:00 summary: compare with difflib not diff(1) (GH-5450) files: M Lib/lib2to3/tests/test_parser.py diff --git a/Lib/lib2to3/tests/test_parser.py b/Lib/lib2to3/tests/test_parser.py index 15f5b63f1f1e..0cbba26bec04 100644 --- a/… [View More]

1 0

bpo-32726: macOS installer and framework enhancements and changes for 3.7.0 (GH-5448)
by Ned Deily Jan. 30, 2018

Jan. 30, 2018

https://github.com/python/cpython/commit/8c9bb72e8b6474ae722f7d6521cfb3cb2e… commit: 8c9bb72e8b6474ae722f7d6521cfb3cb2e26e3c5 branch: master author: Ned Deily <nad(a)python.org> committer: GitHub <noreply(a)github.com> date: 2018-01-30T07:42:14-05:00 summary: bpo-32726: macOS installer and framework enhancements and changes for 3.7.0 (GH-5448) This issue covers various changes for the macOS installers provided via python.org for 3.7.0. - Provide a provisional new installer … [View More]variant for macOS 10.9 and later systems with 64-bit (x86_64) architecture only. Apple has made it known that future versions of macOS will only fully support 64-bit executables and some other third-party software suppliers have chosen 10.9 as their oldest supported system. - Support **Tcl/Tk 8.6** with the 10.9 installer variant. - Upgrade **OpenSSL** to 1.1.0g and **SQLite** to 3.22.0. - The compiler name used for the interpreter build and for modules built with **Distutils / pip** is now _gcc_ rather than _gcc-4.2_. And extension module builds will no longer try to force use of an old SDK if present. files: A Misc/NEWS.d/next/macOS/2018-01-30-07-13-10.bpo-32726.tcARLK.rst D Mac/BuildScript/openssl_sdk_makedepend.patch M Mac/BuildScript/build-installer.py M Mac/BuildScript/resources/ReadMe.rtf M Mac/BuildScript/resources/Welcome.rtf M configure M configure.ac diff --git a/Mac/BuildScript/build-installer.py b/Mac/BuildScript/build-installer.py index bff4eae31b4b..8e2e104a0b7e 100755 --- a/Mac/BuildScript/build-installer.py +++ b/Mac/BuildScript/build-installer.py @@ -1,30 +1,39 @@ #!/usr/bin/env python """ -This script is used to build "official" universal installers on Mac OS X. -It requires at least Mac OS X 10.5, Xcode 3, and the 10.4u SDK for -32-bit builds. 64-bit or four-way universal builds require at least -OS X 10.5 and the 10.5 SDK. +This script is used to build "official" universal installers on macOS. + +NEW for 3.7.0: +- support Intel 64-bit-only () and 32-bit-only installer builds +- use external Tcl/Tk 8.6 for 10.9+ builds +- deprecate use of explicit SDK (--sdk-path=) since all but the oldest + versions of Xcode support implicit setting of an SDK via environment + variables (SDKROOT and friends, see the xcrun man page for more info). + The SDK stuff was primarily needed for building universal installers + for 10.4; so as of 3.7.0, building installers for 10.4 is no longer + supported with build-installer. +- use generic "gcc" as compiler (CC env var) rather than "gcc-4.2" + +TODO: +- support SDKROOT and DEVELOPER_DIR xcrun env variables +- test with 10.5 and 10.4 and determine support status Please ensure that this script keeps working with Python 2.5, to avoid -bootstrap issues (/usr/bin/python is Python 2.5 on OSX 10.5). Sphinx, -which is used to build the documentation, currently requires at least -Python 2.4. However, as of Python 3.4.1, Doc builds require an external -sphinx-build and the current versions of Sphinx now require at least -Python 2.6. - -In addition to what is supplied with OS X 10.5+ and Xcode 3+, the script -requires an installed third-party version of -Tcl/Tk 8.4 (for OS X 10.4 and 10.5 deployment targets) or Tcl/TK 8.5 -(for 10.6 or later) installed in /Library/Frameworks. When installed, +bootstrap issues (/usr/bin/python is Python 2.5 on OSX 10.5). Doc builds +use current versions of Sphinx and require a reasonably current python3. +Sphinx and dependencies are installed into a venv using the python3's pip +so will fetch them from PyPI if necessary. Since python3 is now used for +Sphinx, build-installer.py should also be converted to use python3! + +build-installer currently requires an installed third-party version of +Tcl/Tk 8.4 (for OS X 10.4 and 10.5 deployment targets), Tcl/TK 8.5 +(for 10.6 or later), or Tcl/TK 8.6 (for 10.9 or later) +installed in /Library/Frameworks. When installed, the Python built by this script will attempt to dynamically link first to Tcl and Tk frameworks in /Library/Frameworks if available otherwise fall back to the ones in /System/Library/Framework. For the build, we recommend -installing the most recent ActiveTcl 8.4 or 8.5 version. - -32-bit-only installer builds are still possible on OS X 10.4 with Xcode 2.5 -and the installation of additional components, such as a newer Python -(2.5 is needed for Python parser updates) and for the documentation -build either svn (pre-3.4.1) or sphinx-build (3.4.1 and later). +installing the most recent ActiveTcl 8.6. 8.5, or 8.4 version, depending +on the deployment target. The actual version linked to depends on the +path of /Library/Frameworks/{Tcl,Tk}.framework/Versions/Current. Usage: see USAGE variable in the script. """ @@ -111,32 +120,19 @@ def getFullVersion(): DEPSRC = os.path.join(WORKDIR, 'third-party') DEPSRC = os.path.expanduser('~/Universal/other-sources') -# Location of the preferred SDK - -### There are some issues with the SDK selection below here, -### The resulting binary doesn't work on all platforms that -### it should. Always default to the 10.4u SDK until that -### issue is resolved. -### -##if int(os.uname()[2].split('.')[0]) == 8: -## # Explicitly use the 10.4u (universal) SDK when -## # building on 10.4, the system headers are not -## # useable for a universal build -## SDKPATH = "/Developer/SDKs/MacOSX10.4u.sdk" -##else: -## SDKPATH = "/" - -SDKPATH = "/Developer/SDKs/MacOSX10.4u.sdk" - universal_opts_map = { '32-bit': ('i386', 'ppc',), '64-bit': ('x86_64', 'ppc64',), 'intel': ('i386', 'x86_64'), + 'intel-32': ('i386',), + 'intel-64': ('x86_64',), '3-way': ('ppc', 'i386', 'x86_64'), 'all': ('i386', 'ppc', 'x86_64', 'ppc64',) } default_target_map = { '64-bit': '10.5', '3-way': '10.5', 'intel': '10.5', + 'intel-32': '10.4', + 'intel-64': '10.5', 'all': '10.5', } @@ -154,19 +150,18 @@ def getFullVersion(): )))) # $MACOSX_DEPLOYMENT_TARGET -> minimum OS X level -DEPTARGET = '10.3' +DEPTARGET = '10.5' def getDeptargetTuple(): return tuple([int(n) for n in DEPTARGET.split('.')[0:2]]) def getTargetCompilers(): target_cc_map = { - '10.3': ('gcc-4.0', 'g++-4.0'), '10.4': ('gcc-4.0', 'g++-4.0'), - '10.5': ('gcc-4.2', 'g++-4.2'), - '10.6': ('gcc-4.2', 'g++-4.2'), + '10.5': ('gcc', 'g++'), + '10.6': ('gcc', 'g++'), } - return target_cc_map.get(DEPTARGET, ('clang', 'clang++') ) + return target_cc_map.get(DEPTARGET, ('gcc', 'gcc++') ) CC, CXX = getTargetCompilers() @@ -180,9 +175,9 @@ def getTargetCompilers(): -b DIR --build-dir=DIR: Create build here (default: %(WORKDIR)r) --third-party=DIR: Store third-party sources here (default: %(DEPSRC)r) - --sdk-path=DIR: Location of the SDK (default: %(SDKPATH)r) + --sdk-path=DIR: Location of the SDK (deprecated, use SDKROOT env variable) --src-dir=DIR: Location of the Python sources (default: %(SRCDIR)r) - --dep-target=10.n OS X deployment target (default: %(DEPTARGET)r) + --dep-target=10.n macOS deployment target (default: %(DEPTARGET)r) --universal-archs=x universal architectures (options: %(UNIVERSALOPTS)r, default: %(UNIVERSALARCHS)r) """)% globals() @@ -213,12 +208,9 @@ def library_recipes(): result.extend([ dict( - name="OpenSSL 1.0.2m", - url="https://www.openssl.org/source/openssl-1.0.2m.tar.gz", - checksum='10e9e37f492094b9ef296f68f24a7666', - patches=[ - "openssl_sdk_makedepend.patch", - ], + name="OpenSSL 1.1.0g", + url="https://www.openssl.org/source/openssl-1.1.0g.tar.gz", + checksum='ba5f1b8b835b88cadbce9b35ed9531a6', buildrecipe=build_universal_openssl, configure=None, install=None, @@ -315,9 +307,9 @@ def library_recipes(): ), ), dict( - name="SQLite 3.21.0", - url="https://www.sqlite.org/2017/sqlite-autoconf-3210000.tar.gz", - checksum='7913de4c3126ba3c24689cb7a199ea31', + name="SQLite 3.22.0", + url="https://www.sqlite.org/2018/sqlite-autoconf-3220000.tar.gz", + checksum='96b5648d542e8afa6ab7ffb8db8ddc3d', extra_cflags=('-Os ' '-DSQLITE_ENABLE_FTS5 ' '-DSQLITE_ENABLE_FTS4 ' @@ -343,11 +335,10 @@ def library_recipes(): url="http://bzip.org/1.0.6/bzip2-1.0.6.tar.gz", checksum='00b516f4704d4a7cb50a1d97e6e8e15b', configure=None, - install='make install CC=%s CXX=%s, PREFIX=%s/usr/local/ CFLAGS="-arch %s -isysroot %s"'%( + install='make install CC=%s CXX=%s, PREFIX=%s/usr/local/ CFLAGS="-arch %s"'%( CC, CXX, shellQuote(os.path.join(WORKDIR, 'libraries')), ' -arch '.join(ARCHLIST), - SDKPATH, ), ), dict( @@ -355,11 +346,10 @@ def library_recipes(): url="http://www.gzip.org/zlib/zlib-1.2.3.tar.gz", checksum='debc62758716a169df9f62e6ab2bc634', configure=None, - install='make install CC=%s CXX=%s, prefix=%s/usr/local/ CFLAGS="-arch %s -isysroot %s"'%( + install='make install CC=%s CXX=%s, prefix=%s/usr/local/ CFLAGS="-arch %s"'%( CC, CXX, shellQuote(os.path.join(WORKDIR, 'libraries')), ' -arch '.join(ARCHLIST), - SDKPATH, ), ), dict( @@ -406,8 +396,7 @@ def pkg_recipes(): source="/Library/Frameworks/Python.framework", readme="""\ This package installs Python.framework, that is the python - interpreter and the standard library. This also includes Python - wrappers for lots of Mac OS X API's. + interpreter and the standard library. """, postflight="scripts/postflight.framework", selected='selected', @@ -484,24 +473,6 @@ def pkg_recipes(): ), ] - if getDeptargetTuple() < (10, 4) and not PYTHON_3: - result.append( - dict( - name="PythonSystemFixes", - long_name="Fix system Python", - readme="""\ - This package updates the system python installation on - Mac OS X 10.3 to ensure that you can build new python extensions - using that copy of python after installing this version. - """, - postflight="../Tools/fixapplepython23.py", - topdir="/Library/Frameworks/Python.framework", - source="/empty-dir", - required=False, - selected=unselected_for_python3, - ) - ) - return result def fatal(msg): @@ -566,41 +537,33 @@ def checkEnvironment(): Check that we're running on a supported system. """ - if sys.version_info[0:2] < (2, 4): - fatal("This script must be run with Python 2.4 or later") + if sys.version_info[0:2] < (2, 5): + fatal("This script must be run with Python 2.5 (or later)") if platform.system() != 'Darwin': - fatal("This script should be run on a Mac OS X 10.4 (or later) system") + fatal("This script should be run on a macOS 10.5 (or later) system") if int(platform.release().split('.')[0]) < 8: - fatal("This script should be run on a Mac OS X 10.4 (or later) system") - - if not os.path.exists(SDKPATH): - fatal("Please install the latest version of Xcode and the %s SDK"%( - os.path.basename(SDKPATH[:-4]))) + fatal("This script should be run on a macOS 10.5 (or later) system") # Because we only support dynamic load of only one major/minor version of # Tcl/Tk, ensure: - # 1. there are no user-installed frameworks of Tcl/Tk with version - # higher than the Apple-supplied system version in - # SDKROOT/System/Library/Frameworks - # 2. there is a user-installed framework (usually ActiveTcl) in (or linked - # in) SDKROOT/Library/Frameworks with the same version as the system - # version. This allows users to choose to install a newer patch level. + # 1. there is a user-installed framework (usually ActiveTcl) in (or linked + # in) SDKROOT/Library/Frameworks. As of Python 3.7.0, we no longer + # enforce that the version of the user-installed framework also + # exists in the system-supplied Tcl/Tk frameworks. Time to support + # Tcl/Tk 8.6 even if Apple does not. frameworks = {} for framework in ['Tcl', 'Tk']: fwpth = 'Library/Frameworks/%s.framework/Versions/Current' % framework - sysfw = os.path.join(SDKPATH, 'System', fwpth) - libfw = os.path.join(SDKPATH, fwpth) + libfw = os.path.join('/', fwpth) usrfw = os.path.join(os.getenv('HOME'), fwpth) - frameworks[framework] = os.readlink(sysfw) + frameworks[framework] = os.readlink(libfw) if not os.path.exists(libfw): fatal("Please install a link to a current %s %s as %s so " "the user can override the system framework." % (framework, frameworks[framework], libfw)) - if os.readlink(libfw) != os.readlink(sysfw): - fatal("Version of %s must match %s" % (libfw, sysfw) ) if os.path.exists(usrfw): fatal("Please rename %s to avoid possible dynamic load issues." % usrfw) @@ -608,6 +571,10 @@ def checkEnvironment(): if frameworks['Tcl'] != frameworks['Tk']: fatal("The Tcl and Tk frameworks are not the same version.") + print(" -- Building with Tcl/Tk %s frameworks" + % frameworks['Tk']) + print("") + # add files to check after build EXPECTED_SHARED_LIBS['_tkinter.so'] = [ "/Library/Frameworks/Tcl.framework/Versions/%s/Tcl" @@ -644,7 +611,7 @@ def parseOptions(args=None): """ Parse arguments and update global settings. """ - global WORKDIR, DEPSRC, SDKPATH, SRCDIR, DEPTARGET + global WORKDIR, DEPSRC, SRCDIR, DEPTARGET global UNIVERSALOPTS, UNIVERSALARCHS, ARCHLIST, CC, CXX global FW_VERSION_PREFIX global FW_SSL_DIRECTORY @@ -677,7 +644,7 @@ def parseOptions(args=None): DEPSRC=v elif k in ('--sdk-path',): - SDKPATH=v + print(" WARNING: --sdk-path is no longer supported") elif k in ('--src-dir',): SRCDIR=v @@ -693,7 +660,7 @@ def parseOptions(args=None): if deptarget is None: # Select alternate default deployment # target - DEPTARGET = default_target_map.get(v, '10.3') + DEPTARGET = default_target_map.get(v, '10.5') else: raise NotImplementedError(v) @@ -702,7 +669,6 @@ def parseOptions(args=None): SRCDIR=os.path.abspath(SRCDIR) WORKDIR=os.path.abspath(WORKDIR) - SDKPATH=os.path.abspath(SDKPATH) DEPSRC=os.path.abspath(DEPSRC) CC, CXX = getTargetCompilers() @@ -713,7 +679,6 @@ def parseOptions(args=None): print("-- Settings:") print(" * Source directory: %s" % SRCDIR) print(" * Build directory: %s" % WORKDIR) - print(" * SDK location: %s" % SDKPATH) print(" * Third-party source: %s" % DEPSRC) print(" * Deployment target: %s" % DEPTARGET) print(" * Universal archs: %s" % str(ARCHLIST)) @@ -837,17 +802,13 @@ def build_openssl_arch(archbase, arch): "ppc64": ["darwin64-ppc-cc"], } configure_opts = [ - "no-krb5", "no-idea", "no-mdc2", "no-rc5", "no-zlib", - "enable-tlsext", - "no-ssl2", "no-ssl3", # "enable-unit-test", "shared", - "--install_prefix=%s"%shellQuote(archbase), "--prefix=%s"%os.path.join("/", *FW_VERSION_PREFIX), "--openssldir=%s"%os.path.join("/", *FW_SSL_DIRECTORY), ] @@ -855,9 +816,9 @@ def build_openssl_arch(archbase, arch): configure_opts.append("no-asm") runCommand(" ".join(["perl", "Configure"] + arch_opts[arch] + configure_opts)) - runCommand("make depend OSX_SDK=%s" % SDKPATH) - runCommand("make all OSX_SDK=%s" % SDKPATH) - runCommand("make install_sw OSX_SDK=%s" % SDKPATH) + runCommand("make depend") + runCommand("make all") + runCommand("make install_sw DESTDIR=%s"%shellQuote(archbase)) # runCommand("make test") return @@ -1016,27 +977,24 @@ def buildRecipe(recipe, basedir, archList): if recipe.get('useLDFlags', 1): configure_args.extend([ - "CFLAGS=%s-mmacosx-version-min=%s -arch %s -isysroot %s " + "CFLAGS=%s-mmacosx-version-min=%s -arch %s " "-I%s/usr/local/include"%( recipe.get('extra_cflags', ''), DEPTARGET, ' -arch '.join(archList), - shellQuote(SDKPATH)[1:-1], shellQuote(basedir)[1:-1],), - "LDFLAGS=-mmacosx-version-min=%s -isysroot %s -L%s/usr/local/lib -arch %s"%( + "LDFLAGS=-mmacosx-version-min=%s -L%s/usr/local/lib -arch %s"%( DEPTARGET, - shellQuote(SDKPATH)[1:-1], shellQuote(basedir)[1:-1], ' -arch '.join(archList)), ]) else: configure_args.extend([ - "CFLAGS=%s-mmacosx-version-min=%s -arch %s -isysroot %s " + "CFLAGS=%s-mmacosx-version-min=%s -arch %s " "-I%s/usr/local/include"%( recipe.get('extra_cflags', ''), DEPTARGET, ' -arch '.join(archList), - shellQuote(SDKPATH)[1:-1], shellQuote(basedir)[1:-1],), ]) @@ -1114,10 +1072,6 @@ def buildPython(): curdir = os.getcwd() os.chdir(buildDir) - # Not sure if this is still needed, the original build script - # claims that parts of the install assume python.exe exists. - os.symlink('python', os.path.join(buildDir, 'python.exe')) - # Extract the version from the configure file, needed to calculate # several paths. version = getVersion() @@ -1128,13 +1082,13 @@ def buildPython(): os.environ['DYLD_LIBRARY_PATH'] = os.path.join(WORKDIR, 'libraries', 'usr', 'local', 'lib') print("Running configure...") - runCommand("%s -C --enable-framework --enable-universalsdk=%s " + runCommand("%s -C --enable-framework --enable-universalsdk=/ " "--with-universal-archs=%s " "%s " "%s " "LDFLAGS='-g -L%s/libraries/usr/local/lib' " "CFLAGS='-g -I%s/libraries/usr/local/include' 2>&1"%( - shellQuote(os.path.join(SRCDIR, 'configure')), shellQuote(SDKPATH), + shellQuote(os.path.join(SRCDIR, 'configure')), UNIVERSALARCHS, (' ', '--with-computed-gotos ')[PYTHON_3], (' ', '--without-ensurepip ')[PYTHON_3], diff --git a/Mac/BuildScript/openssl_sdk_makedepend.patch b/Mac/BuildScript/openssl_sdk_makedepend.patch deleted file mode 100644 index 0caac0a64c1e..000000000000 --- a/Mac/BuildScript/openssl_sdk_makedepend.patch +++ /dev/null @@ -1,40 +0,0 @@ -# HG changeset patch -# -# using openssl 1.0.2k -# -# - support building with an OS X SDK - -diff Configure - -diff --git a/Configure b/Configure ---- a/Configure -+++ b/Configure -@@ -642,12 +642,12 @@ - - ##### MacOS X (a.k.a. Rhapsody or Darwin) setup - "rhapsody-ppc-cc","cc:-O3 -DB_ENDIAN::(unknown):MACOSX_RHAPSODY::BN_LLONG RC4_CHAR RC4_CHUNK DES_UNROLL BF_PTR:${no_asm}::", --"darwin-ppc-cc","cc:-arch ppc -O3 -DB_ENDIAN -Wa,-force_cpusubtype_ALL::-D_REENTRANT:MACOSX:-Wl,-search_paths_first%:BN_LLONG RC4_CHAR RC4_CHUNK DES_UNROLL BF_PTR:${ppc32_asm}:osx32:dlfcn:darwin-shared:-fPIC -fno-common:-arch ppc -dynamiclib:.\$(SHLIB_MAJOR).\$(SHLIB_MINOR).dylib", --"darwin64-ppc-cc","cc:-arch ppc64 -O3 -DB_ENDIAN::-D_REENTRANT:MACOSX:-Wl,-search_paths_first%:SIXTY_FOUR_BIT_LONG RC4_CHAR RC4_CHUNK DES_UNROLL BF_PTR:${ppc64_asm}:osx64:dlfcn:darwin-shared:-fPIC -fno-common:-arch ppc64 -dynamiclib:.\$(SHLIB_MAJOR).\$(SHLIB_MINOR).dylib", --"darwin-i386-cc","cc:-arch i386 -O3 -fomit-frame-pointer -DL_ENDIAN::-D_REENTRANT:MACOSX:-Wl,-search_paths_first%:BN_LLONG RC4_INT RC4_CHUNK DES_UNROLL BF_PTR:".eval{my $asm=$x86_asm;$asm=~s/cast\-586\.o//;$asm}.":macosx:dlfcn:darwin-shared:-fPIC -fno-common:-arch i386 -dynamiclib:.\$(SHLIB_MAJOR).\$(SHLIB_MINOR).dylib", --"debug-darwin-i386-cc","cc:-arch i386 -g3 -DL_ENDIAN::-D_REENTRANT:MACOSX:-Wl,-search_paths_first%:BN_LLONG RC4_INT RC4_CHUNK DES_UNROLL BF_PTR:${x86_asm}:macosx:dlfcn:darwin-shared:-fPIC -fno-common:-arch i386 -dynamiclib:.\$(SHLIB_MAJOR).\$(SHLIB_MINOR).dylib", --"darwin64-x86_64-cc","cc:-arch x86_64 -O3 -DL_ENDIAN -Wall::-D_REENTRANT:MACOSX:-Wl,-search_paths_first%:SIXTY_FOUR_BIT_LONG RC4_CHUNK DES_INT DES_UNROLL:".eval{my $asm=$x86_64_asm;$asm=~s/rc4\-[^:]+//;$asm}.":macosx:dlfcn:darwin-shared:-fPIC -fno-common:-arch x86_64 -dynamiclib:.\$(SHLIB_MAJOR).\$(SHLIB_MINOR).dylib", --"debug-darwin64-x86_64-cc","cc:-arch x86_64 -ggdb -g2 -O0 -DL_ENDIAN -Wall::-D_REENTRANT:MACOSX:-Wl,-search_paths_first%:SIXTY_FOUR_BIT_LONG RC4_CHUNK DES_INT DES_UNROLL:".eval{my $asm=$x86_64_asm;$asm=~s/rc4\-[^:]+//;$asm}.":macosx:dlfcn:darwin-shared:-fPIC -fno-common:-arch x86_64 -dynamiclib:.\$(SHLIB_MAJOR).\$(SHLIB_MINOR).dylib", -+"darwin-ppc-cc","cc:-arch ppc -isysroot \$(OSX_SDK) -O3 -DB_ENDIAN -Wa,-force_cpusubtype_ALL::-D_REENTRANT:MACOSX:-Wl,-search_paths_first%:BN_LLONG RC4_CHAR RC4_CHUNK DES_UNROLL BF_PTR:${ppc32_asm}:osx32:dlfcn:darwin-shared:-fPIC -fno-common:-arch ppc -dynamiclib:.\$(SHLIB_MAJOR).\$(SHLIB_MINOR).dylib", -+"darwin64-ppc-cc","cc:-arch ppc64 -isysroot \$(OSX_SDK) -O3 -DB_ENDIAN::-D_REENTRANT:MACOSX:-Wl,-search_paths_first%:SIXTY_FOUR_BIT_LONG RC4_CHAR RC4_CHUNK DES_UNROLL BF_PTR:${ppc64_asm}:osx64:dlfcn:darwin-shared:-fPIC -fno-common:-arch ppc64 -dynamiclib:.\$(SHLIB_MAJOR).\$(SHLIB_MINOR).dylib", -+"darwin-i386-cc","cc:-arch i386 -isysroot \$(OSX_SDK) -O3 -fomit-frame-pointer -DL_ENDIAN::-D_REENTRANT:MACOSX:-Wl,-search_paths_first%:BN_LLONG RC4_INT RC4_CHUNK DES_UNROLL BF_PTR:".eval{my $asm=$x86_asm;$asm=~s/cast\-586\.o//;$asm}.":macosx:dlfcn:darwin-shared:-fPIC -fno-common:-arch i386 -dynamiclib:.\$(SHLIB_MAJOR).\$(SHLIB_MINOR).dylib", -+"debug-darwin-i386-cc","cc:-arch i386 -isysroot \$(OSX_SDK) -g3 -DL_ENDIAN::-D_REENTRANT:MACOSX:-Wl,-search_paths_first%:BN_LLONG RC4_INT RC4_CHUNK DES_UNROLL BF_PTR:${x86_asm}:macosx:dlfcn:darwin-shared:-fPIC -fno-common:-arch i386 -dynamiclib:.\$(SHLIB_MAJOR).\$(SHLIB_MINOR).dylib", -+"darwin64-x86_64-cc","cc:-arch x86_64 -isysroot \$(OSX_SDK) -O3 -DL_ENDIAN -Wall::-D_REENTRANT:MACOSX:-Wl,-search_paths_first%:SIXTY_FOUR_BIT_LONG RC4_CHUNK DES_INT DES_UNROLL:".eval{my $asm=$x86_64_asm;$asm=~s/rc4\-[^:]+//;$asm}.":macosx:dlfcn:darwin-shared:-fPIC -fno-common:-arch x86_64 -dynamiclib:.\$(SHLIB_MAJOR).\$(SHLIB_MINOR).dylib", -+"debug-darwin64-x86_64-cc","cc:-arch x86_64 -isysroot \$(OSX_SDK) -ggdb -g2 -O0 -DL_ENDIAN -Wall::-D_REENTRANT:MACOSX:-Wl,-search_paths_first%:SIXTY_FOUR_BIT_LONG RC4_CHUNK DES_INT DES_UNROLL:".eval{my $asm=$x86_64_asm;$asm=~s/rc4\-[^:]+//;$asm}.":macosx:dlfcn:darwin-shared:-fPIC -fno-common:-arch x86_64 -dynamiclib:.\$(SHLIB_MAJOR).\$(SHLIB_MINOR).dylib", - "debug-darwin-ppc-cc","cc:-DBN_DEBUG -DREF_CHECK -DCONF_DEBUG -DCRYPTO_MDEBUG -DB_ENDIAN -g -Wall -O::-D_REENTRANT:MACOSX::BN_LLONG RC4_CHAR RC4_CHUNK DES_UNROLL BF_PTR:${ppc32_asm}:osx32:dlfcn:darwin-shared:-fPIC:-dynamiclib:.\$(SHLIB_MAJOR).\$(SHLIB_MINOR).dylib", - # iPhoneOS/iOS - "iphoneos-cross","llvm-gcc:-O3 -isysroot \$(CROSS_TOP)/SDKs/\$(CROSS_SDK) -fomit-frame-pointer -fno-common::-D_REENTRANT:iOS:-Wl,-search_paths_first%:BN_LLONG RC4_CHAR RC4_CHUNK DES_UNROLL BF_PTR:${no_asm}:dlfcn:darwin-shared:-fPIC -fno-common:-dynamiclib:.\$(SHLIB_MAJOR).\$(SHLIB_MINOR).dylib", -@@ -1728,8 +1728,7 @@ - s/^AR=\s*ar/AR= $ar/; - s/^RANLIB=.*/RANLIB= $ranlib/; - s/^RC=.*/RC= $windres/; -- s/^MAKEDEPPROG=.*$/MAKEDEPPROG= $cc/ if $cc eq "gcc"; -- s/^MAKEDEPPROG=.*$/MAKEDEPPROG= $cc/ if $ecc eq "gcc" || $ecc eq "clang"; -+ s/^MAKEDEPPROG=.*$/MAKEDEPPROG= $cc/; - } - s/^CFLAG=.*$/CFLAG= $cflags/; - s/^DEPFLAG=.*$/DEPFLAG=$depflags/; diff --git a/Mac/BuildScript/resources/ReadMe.rtf b/Mac/BuildScript/resources/ReadMe.rtf index dba96b6068c6..1eea1e291bda 100644 --- a/Mac/BuildScript/resources/ReadMe.rtf +++ b/Mac/BuildScript/resources/ReadMe.rtf @@ -1,4 +1,4 @@ -{\rtf1\ansi\ansicpg1252\cocoartf1504\cocoasubrtf830 +{\rtf1\ansi\ansicpg1252\cocoartf1561\cocoasubrtf200 {\fonttbl\f0\fswiss\fcharset0 Helvetica;\f1\fmodern\fcharset0 CourierNewPSMT;} {\colortbl;\red255\green255\blue255;} {\*\expandedcolortbl;;} @@ -10,21 +10,37 @@ \pard\tx720\tx1440\tx2160\tx2880\tx3600\tx4320\tx5040\tx5760\tx6480\tx7200\tx7920\tx8640\partightenfactor0 \b \cf0 NOTE: -\b0 This is an early developer preview of the next feature release of Python 3. Many features are yet to be added and existing features may change substantially or be deleted prior to the feature code freeze.\ +\b0 This is a beta preview of the next feature release of Python 3. Existing features may change or be deleted prior to the release code freeze.\ \pard\tx720\tx1440\tx2160\tx2880\tx3600\tx4320\tx5040\tx5760\tx6480\tx7200\tx7920\tx8640\pardirnatural\partightenfactor0 \cf0 \ \pard\tx720\tx1440\tx2160\tx2880\tx3600\tx4320\tx5040\tx5760\tx6480\tx7200\tx7920\tx8640\pardirnatural\partightenfactor0 -\b \cf0 \ul \ulc0 Certificate verification and OpenSSL\ +\b \cf0 \ul \ulc0 Which installer variant should I use? [CHANGED in 3.7.0b1] +\b0 \ulnone \ +\ +For Python.3.7, python.org currently provides two installer variants for download: one that installs a +\i 64-bit/32-bit Intel +\i0 Python capable of running on +\i macOS 10.6 (Snow Leopard) +\i0 or later; and one that installs a +\i 64-bit-only +\i0 Python capable of running on +\i macOS 10.9 (Mavericks) +\i0 or later. This ReadMe was installed with the +\i $MACOSX_DEPLOYMENT_TARGET +\i0 variant. Use the 10.6 variant if you are running on macOS 10.6 through 10.8 or if you want to produce standalone applications that can run on systems from 10.6. If you are running on macOS 10.9 or later and if you have no need for compatibility with older systems, use the 10.9 variant. The Pythons installed by these installers are built with private copies of some third-party libraries not included with or newer than those in macOS itself. The list of these libraries varies by installer variant and is included at the end of the License.rtf file. +\b \ul \ +\ +Certificate verification and OpenSSL \{UPDATED in 3.7.0b1]\ \b0 \ulnone \ -This variant of Python 3.7 includes its own private copy of OpenSSL 1.0.2. The deprecated Apple-supplied OpenSSL libraries are no longer used. This means that the trust certificates in system and user keychains managed by the +This variant of Python 3.7 includes its own private copy of OpenSSL 1.1.0. The deprecated Apple-supplied OpenSSL libraries are no longer used. This means that the trust certificates in system and user keychains managed by the \i Keychain Access \i0 application and the \i security \i0 command line utility are no longer used as defaults by the Python \f1 ssl -\f0 module. For this preview release, a sample command script is included in +\f0 module. For this beta release, a sample command script is included in \f1 /Applications/Python 3.7 \f0 to install a curated bundle of default root certificates from the third-party \f1 certifi @@ -37,14 +53,14 @@ The bundled \f0 included with this installer has its own default certificate store for verifying download connections.\ \ -\b \ul Update your version of Tcl/Tk to use IDLE or other Tk applications +\b \ul Update your version of Tcl/Tk to use IDLE or other Tk applications [CHANGED in 3.7.0b1] \b0 \ulnone \ \ To use IDLE or other programs that use the Tkinter graphical user interface toolkit, you need to install a newer third-party version of the \i Tcl/Tk \i0 frameworks. Visit {\field{\*\fldinst{HYPERLINK "https://www.python.org/download/mac/tcltk/"}}{\fldrslt https://www.python.org/download/mac/tcltk/}} for current information about supported and recommended versions of \i Tcl/Tk -\i0 for this version of Python and of Mac OS X. For this developer preview, the installer is still linked with Tcl/Tk 8.5.\ +\i0 for this version of Python and of Mac OS X. For 3.7.0b1, the 10.9 installer variant is linked with Tcl/Tk 8.6 which you must install separately. The 10.6 installer links /with Tcl/Tk 8.5 which you should also download and install.\ \b \ul \ Other changes\ diff --git a/Mac/BuildScript/resources/Welcome.rtf b/Mac/BuildScript/resources/Welcome.rtf index 528e2513af77..18f3290608ec 100644 --- a/Mac/BuildScript/resources/Welcome.rtf +++ b/Mac/BuildScript/resources/Welcome.rtf @@ -1,4 +1,4 @@ -{\rtf1\ansi\ansicpg1252\cocoartf1504\cocoasubrtf830 +{\rtf1\ansi\ansicpg1252\cocoartf1561\cocoasubrtf200 \cocoascreenfonts1{\fonttbl\f0\fswiss\fcharset0 Helvetica;} {\colortbl;\red255\green255\blue255;} {\*\expandedcolortbl;;} @@ -19,7 +19,11 @@ \ \b NOTE: -\b0 This is an early developer preview of the next feature release of Python 3. Many features are yet to be added and existing features may change substantially or be deleted prior to the feature code freeze.\ +\b0 This is an beta preview of the next feature release of Python 3. New features may change substantially or be deleted prior to the final code freeze.\ +\ + +\b NEW in 3.7.0b1: +\b0 two installer variants (10.9+ 64-bit-only, 10.6+ 64-/32-bit), Tcl/Tk 8.6 support in the 10.9+ variant, OpenSSL 1.1.0g, and more!\ \ \b IMPORTANT: diff --git a/Misc/NEWS.d/next/macOS/2018-01-30-07-13-10.bpo-32726.tcARLK.rst b/Misc/NEWS.d/next/macOS/2018-01-30-07-13-10.bpo-32726.tcARLK.rst new file mode 100644 index 000000000000..bb7ebf1a62d2 --- /dev/null +++ b/Misc/NEWS.d/next/macOS/2018-01-30-07-13-10.bpo-32726.tcARLK.rst @@ -0,0 +1,4 @@ +Provide an additional, more modern macOS installer variant that supports +macOS 10.9+ systems in 64-bit mode only. Upgrade the supplied third-party +libraries to OpenSSL 1.1.0g and to SQLite 3.22.0. The 10.9+ installer now +links with Tcl/Tk 8.6. diff --git a/configure b/configure index caa86677ea11..f75acf68ce2c 100755 --- a/configure +++ b/configure @@ -1489,7 +1489,8 @@ Optional Packages: --without-PACKAGE do not use PACKAGE (same as --with-PACKAGE=no) --with-universal-archs=ARCH select architectures for universal build ("32-bit", - "64-bit", "3-way", "intel", "intel-32", or "all") + "64-bit", "3-way", "intel", "intel-32", "intel-64", + or "all") --with-framework-name=FRAMEWORK specify an alternate name of the framework built with --enable-framework @@ -7224,6 +7225,11 @@ $as_echo "$CC" >&6; } LIPO_32BIT_FLAGS="" ARCH_RUN_32BIT="" ;; + intel-64) + UNIVERSAL_ARCH_FLAGS="-arch x86_64" + LIPO_32BIT_FLAGS="" + ARCH_RUN_32BIT="true" + ;; 3-way) UNIVERSAL_ARCH_FLAGS="-arch i386 -arch ppc -arch x86_64" LIPO_32BIT_FLAGS="-extract ppc7400 -extract i386" @@ -7234,11 +7240,14 @@ $as_echo "$CC" >&6; } ;; esac - CFLAGS="${UNIVERSAL_ARCH_FLAGS} -isysroot ${UNIVERSALSDK} ${CFLAGS}" - LDFLAGS="${UNIVERSAL_ARCH_FLAGS} -isysroot ${UNIVERSALSDK} ${LDFLAGS}" if test "${UNIVERSALSDK}" != "/" then + CFLAGS="${UNIVERSAL_ARCH_FLAGS} -isysroot ${UNIVERSALSDK} ${CFLAGS}" + LDFLAGS="${UNIVERSAL_ARCH_FLAGS} -isysroot ${UNIVERSALSDK} ${LDFLAGS}" CPPFLAGS="-isysroot ${UNIVERSALSDK} ${CPPFLAGS}" + else + CFLAGS="${UNIVERSAL_ARCH_FLAGS} ${CFLAGS}" + LDFLAGS="${UNIVERSAL_ARCH_FLAGS} ${LDFLAGS}" fi fi diff --git a/configure.ac b/configure.ac index af10a6d3cad5..525e572c1fff 100644 --- a/configure.ac +++ b/configure.ac @@ -212,7 +212,7 @@ fi AC_SUBST(LIPO_32BIT_FLAGS) AC_MSG_CHECKING(for --with-universal-archs) AC_ARG_WITH(universal-archs, - AS_HELP_STRING([--with-universal-archs=ARCH], [select architectures for universal build ("32-bit", "64-bit", "3-way", "intel", "intel-32", or "all")]), + AS_HELP_STRING([--with-universal-archs=ARCH], [select architectures for universal build ("32-bit", "64-bit", "3-way", "intel", "intel-32", "intel-64", or "all")]), [ UNIVERSAL_ARCHS="$withval" ], @@ -1768,6 +1768,11 @@ yes) LIPO_32BIT_FLAGS="" ARCH_RUN_32BIT="" ;; + intel-64) + UNIVERSAL_ARCH_FLAGS="-arch x86_64" + LIPO_32BIT_FLAGS="" + ARCH_RUN_32BIT="true" + ;; 3-way) UNIVERSAL_ARCH_FLAGS="-arch i386 -arch ppc -arch x86_64" LIPO_32BIT_FLAGS="-extract ppc7400 -extract i386" @@ -1778,11 +1783,14 @@ yes) ;; esac - CFLAGS="${UNIVERSAL_ARCH_FLAGS} -isysroot ${UNIVERSALSDK} ${CFLAGS}" - LDFLAGS="${UNIVERSAL_ARCH_FLAGS} -isysroot ${UNIVERSALSDK} ${LDFLAGS}" if test "${UNIVERSALSDK}" != "/" then + CFLAGS="${UNIVERSAL_ARCH_FLAGS} -isysroot ${UNIVERSALSDK} ${CFLAGS}" + LDFLAGS="${UNIVERSAL_ARCH_FLAGS} -isysroot ${UNIVERSALSDK} ${LDFLAGS}" CPPFLAGS="-isysroot ${UNIVERSALSDK} ${CPPFLAGS}" + else + CFLAGS="${UNIVERSAL_ARCH_FLAGS} ${CFLAGS}" + LDFLAGS="${UNIVERSAL_ARCH_FLAGS} ${LDFLAGS}" fi fi [View Less]

1 0

bpo-28914: Fix compilation of select on Android (#5447)
by Victor Stinner Jan. 30, 2018

Jan. 30, 2018

https://github.com/python/cpython/commit/b8d90328ad2abc9d6d2c9b8ce769fb82b1… commit: b8d90328ad2abc9d6d2c9b8ce769fb82b18bbc10 branch: master author: Victor Stinner <victor.stinner(a)gmail.com> committer: GitHub <noreply(a)github.com> date: 2018-01-30T12:18:54+01:00 summary: bpo-28914: Fix compilation of select on Android (#5447) EPOLL_CLOEXEC is not defined on Android. Co-Authored-By: Wataru Matsumoto <sxsns243(a)gmail.com> files: M Modules/selectmodule.c diff --git a/… [View More]

1 0

bpo-28440: Don't add /Library/Python/3.x/site-packages to sys.path (#5445)
by Ned Deily Jan. 30, 2018

Jan. 30, 2018

https://github.com/python/cpython/commit/763f094b1f0ce2a07768828e78afe9687e… commit: 763f094b1f0ce2a07768828e78afe9687e9ad3bb branch: master author: Ned Deily <nad(a)python.org> committer: GitHub <noreply(a)github.com> date: 2018-01-30T05:14:09-05:00 summary: bpo-28440: Don't add /Library/Python/3.x/site-packages to sys.path (#5445) No longer add /Library/Python/3.x/site-packages, the Apple-supplied system Python site-packages directory, to sys.path for macOS framework builds in … [View More]case Apple ships a version of Python 3. A similar change was made earlier to Python 2.7 where it was found that the coupling between the system Python and a user-installed framework Python often caused confusion or pip install failures. files: A Misc/NEWS.d/next/macOS/2018-01-30-04-40-12.bpo-28440.W_BUWU.rst M Lib/site.py M Lib/test/test_site.py diff --git a/Lib/site.py b/Lib/site.py index 7dc1b041c192..950e7038a505 100644 --- a/Lib/site.py +++ b/Lib/site.py @@ -340,11 +340,6 @@ def getsitepackages(prefixes=None): else: sitepackages.append(prefix) sitepackages.append(os.path.join(prefix, "lib", "site-packages")) - # for framework builds *only* we add the standard Apple locations. - if sys.platform == "darwin" and sys._framework: - sitepackages.append( - os.path.join("/Library", sys._framework, - '%d.%d' % sys.version_info[:2], "site-packages")) return sitepackages def addsitepackages(known_paths, prefixes=None): diff --git a/Lib/test/test_site.py b/Lib/test/test_site.py index 266adf01e04d..b6648d67ca68 100644 --- a/Lib/test/test_site.py +++ b/Lib/test/test_site.py @@ -261,20 +261,8 @@ def test_getusersitepackages(self): def test_getsitepackages(self): site.PREFIXES = ['xoxo'] dirs = site.getsitepackages() - - if (sys.platform == "darwin" and - sysconfig.get_config_var("PYTHONFRAMEWORK")): - # OS X framework builds - site.PREFIXES = ['Python.framework'] - dirs = site.getsitepackages() - self.assertEqual(len(dirs), 2) - wanted = os.path.join('/Library', - sysconfig.get_config_var("PYTHONFRAMEWORK"), - '%d.%d' % sys.version_info[:2], - 'site-packages') - self.assertEqual(dirs[1], wanted) - elif os.sep == '/': - # OS X non-framework builds, Linux, FreeBSD, etc + if os.sep == '/': + # OS X, Linux, FreeBSD, etc self.assertEqual(len(dirs), 1) wanted = os.path.join('xoxo', 'lib', 'python%d.%d' % sys.version_info[:2], diff --git a/Misc/NEWS.d/next/macOS/2018-01-30-04-40-12.bpo-28440.W_BUWU.rst b/Misc/NEWS.d/next/macOS/2018-01-30-04-40-12.bpo-28440.W_BUWU.rst new file mode 100644 index 000000000000..e973e0957755 --- /dev/null +++ b/Misc/NEWS.d/next/macOS/2018-01-30-04-40-12.bpo-28440.W_BUWU.rst @@ -0,0 +1,2 @@ +No longer add /Library/Python/3.x/site-packages to sys.path for macOS +framework builds to avoid future conflicts. [View Less]

1 0

bpo-32724: Fix references to commands in Doc/pdb.rst (GH-5444)
by Andrew Svetlov Jan. 30, 2018

Jan. 30, 2018

https://github.com/python/cpython/commit/67adb31a416864f853772c3e74a06caead… commit: 67adb31a416864f853772c3e74a06caeadc9b6f1 branch: master author: Stéphane Wirtel <stephane(a)wirtel.be> committer: Andrew Svetlov <andrew.svetlov(a)gmail.com> date: 2018-01-30T11:34:33+02:00 summary: bpo-32724: Fix references to commands in Doc/pdb.rst (GH-5444) Some commands are specified in the documentation, but there is no direct references. files: A Misc/NEWS.d/next/Documentation/2018-01-30-… [View More]

1 0

Daily reference leaks (4243df51fe43): sum=8
by solipsis＠pitrou.net Jan. 30, 2018

Jan. 30, 2018

results for 4243df51fe43 on branch "default" -------------------------------------------- test_asyncio leaked [0, 0, 3] memory blocks, sum=3 test_collections leaked [7, -7, 1] memory blocks, sum=1 test_functools leaked [0, 3, 1] memory blocks, sum=4 test_multiprocessing_fork leaked [0, -2, 2] memory blocks, sum=0 Command line was: ['./python', '-m', 'test.regrtest', '-uall', '-R', '3:3:/home/psf-users/antoine/refleaks/reflogjpD75O', '--timeout', '7200']

1 0

bpo-22908: Add seek and tell functionality to ZipExtFile (GH-4966)
by Gregory P. Smith Jan. 30, 2018

Jan. 30, 2018

https://github.com/python/cpython/commit/066df4fd454d6ff9be66e80b2a65995b10… commit: 066df4fd454d6ff9be66e80b2a65995b10af174f branch: master author: John Jolly <john.jolly(a)gmail.com> committer: Gregory P. Smith <greg(a)krypto.org> date: 2018-01-30T00:51:35-08:00 summary: bpo-22908: Add seek and tell functionality to ZipExtFile (GH-4966) This allows for nested zip files, tar files within zip files, zip files within tar files, etc. Contributed by: John Jolly files: A Misc/NEWS.d/… [View More]next/Library/2017-12-21-22-00-11.bpo-22908.cVm89I.rst M Doc/library/zipfile.rst M Lib/test/test_zipfile.py M Lib/zipfile.py diff --git a/Doc/library/zipfile.rst b/Doc/library/zipfile.rst index d58efe0b4175..7c9a8c802254 100644 --- a/Doc/library/zipfile.rst +++ b/Doc/library/zipfile.rst @@ -246,9 +246,9 @@ ZipFile Objects With *mode* ``'r'`` the file-like object (``ZipExtFile``) is read-only and provides the following methods: :meth:`~io.BufferedIOBase.read`, :meth:`~io.IOBase.readline`, - :meth:`~io.IOBase.readlines`, :meth:`__iter__`, - :meth:`~iterator.__next__`. These objects can operate independently of - the ZipFile. + :meth:`~io.IOBase.readlines`, :meth:`~io.IOBase.seek`, + :meth:`~io.IOBase.tell`, :meth:`__iter__`, :meth:`~iterator.__next__`. + These objects can operate independently of the ZipFile. With ``mode='w'``, a writable file handle is returned, which supports the :meth:`~io.BufferedIOBase.write` method. While a writable file handle is open, diff --git a/Lib/test/test_zipfile.py b/Lib/test/test_zipfile.py index 94db858a1517..61c3e349a69e 100644 --- a/Lib/test/test_zipfile.py +++ b/Lib/test/test_zipfile.py @@ -1628,6 +1628,40 @@ def test_open_conflicting_handles(self): self.assertEqual(zipf.read('baz'), msg3) self.assertEqual(zipf.namelist(), ['foo', 'bar', 'baz']) + def test_seek_tell(self): + # Test seek functionality + txt = b"Where's Bruce?" + bloc = txt.find(b"Bruce") + # Check seek on a file + with zipfile.ZipFile(TESTFN, "w") as zipf: + zipf.writestr("foo.txt", txt) + with zipfile.ZipFile(TESTFN, "r") as zipf: + with zipf.open("foo.txt", "r") as fp: + fp.seek(bloc, os.SEEK_SET) + self.assertEqual(fp.tell(), bloc) + fp.seek(-bloc, os.SEEK_CUR) + self.assertEqual(fp.tell(), 0) + fp.seek(bloc, os.SEEK_CUR) + self.assertEqual(fp.tell(), bloc) + self.assertEqual(fp.read(5), txt[bloc:bloc+5]) + fp.seek(0, os.SEEK_END) + self.assertEqual(fp.tell(), len(txt)) + # Check seek on memory file + data = io.BytesIO() + with zipfile.ZipFile(data, mode="w") as zipf: + zipf.writestr("foo.txt", txt) + with zipfile.ZipFile(data, mode="r") as zipf: + with zipf.open("foo.txt", "r") as fp: + fp.seek(bloc, os.SEEK_SET) + self.assertEqual(fp.tell(), bloc) + fp.seek(-bloc, os.SEEK_CUR) + self.assertEqual(fp.tell(), 0) + fp.seek(bloc, os.SEEK_CUR) + self.assertEqual(fp.tell(), bloc) + self.assertEqual(fp.read(5), txt[bloc:bloc+5]) + fp.seek(0, os.SEEK_END) + self.assertEqual(fp.tell(), len(txt)) + def tearDown(self): unlink(TESTFN) unlink(TESTFN2) diff --git a/Lib/zipfile.py b/Lib/zipfile.py index f9db45f58a2b..5df7b1bf75b9 100644 --- a/Lib/zipfile.py +++ b/Lib/zipfile.py @@ -696,6 +696,18 @@ def __init__(self, file, pos, close, lock, writing): self._close = close self._lock = lock self._writing = writing + self.seekable = file.seekable + self.tell = file.tell + + def seek(self, offset, whence=0): + with self._lock: + if self.writing(): + raise ValueError("Can't reposition in the ZIP file while " + "there is an open writing handle on it. " + "Close the writing handle before trying to read.") + self._file.seek(self._pos) + self._pos = self._file.tell() + return self._pos def read(self, n=-1): with self._lock: @@ -746,6 +758,9 @@ class ZipExtFile(io.BufferedIOBase): # Read from compressed files in 4k blocks. MIN_READ_SIZE = 4096 + # Chunk size to read during seek + MAX_SEEK_READ = 1 << 24 + def __init__(self, fileobj, mode, zipinfo, decrypter=None, close_fileobj=False): self._fileobj = fileobj @@ -778,6 +793,17 @@ def __init__(self, fileobj, mode, zipinfo, decrypter=None, else: self._expected_crc = None + self._seekable = False + try: + if fileobj.seekable(): + self._orig_compress_start = fileobj.tell() + self._orig_compress_size = zipinfo.compress_size + self._orig_file_size = zipinfo.file_size + self._orig_start_crc = self._running_crc + self._seekable = True + except AttributeError: + pass + def __repr__(self): result = ['<%s.%s' % (self.__class__.__module__, self.__class__.__qualname__)] @@ -963,6 +989,62 @@ def close(self): finally: super().close() + def seekable(self): + return self._seekable + + def seek(self, offset, whence=0): + if not self._seekable: + raise io.UnsupportedOperation("underlying stream is not seekable") + curr_pos = self.tell() + if whence == 0: # Seek from start of file + new_pos = offset + elif whence == 1: # Seek from current position + new_pos = curr_pos + offset + elif whence == 2: # Seek from EOF + new_pos = self._orig_file_size + offset + else: + raise ValueError("whence must be os.SEEK_SET (0), " + "os.SEEK_CUR (1), or os.SEEK_END (2)") + + if new_pos > self._orig_file_size: + new_pos = self._orig_file_size + + if new_pos < 0: + new_pos = 0 + + read_offset = new_pos - curr_pos + buff_offset = read_offset + self._offset + + if buff_offset >= 0 and buff_offset < len(self._readbuffer): + # Just move the _offset index if the new position is in the _readbuffer + self._offset = buff_offset + read_offset = 0 + elif read_offset < 0: + # Position is before the current position. Reset the ZipExtFile + + self._fileobj.seek(self._orig_compress_start) + self._running_crc = self._orig_start_crc + self._compress_left = self._orig_compress_size + self._left = self._orig_file_size + self._readbuffer = b'' + self._offset = 0 + self._decompressor = zipfile._get_decompressor(self._compress_type) + self._eof = False + read_offset = new_pos + + while read_offset > 0: + read_len = min(self.MAX_SEEK_READ, read_offset) + self.read(read_len) + read_offset -= read_len + + return self.tell() + + def tell(self): + if not self._seekable: + raise io.UnsupportedOperation("underlying stream is not seekable") + filepos = self._orig_file_size - self._left - len(self._readbuffer) + self._offset + return filepos + class _ZipWriteFile(io.BufferedIOBase): def __init__(self, zf, zinfo, zip64): diff --git a/Misc/NEWS.d/next/Library/2017-12-21-22-00-11.bpo-22908.cVm89I.rst b/Misc/NEWS.d/next/Library/2017-12-21-22-00-11.bpo-22908.cVm89I.rst new file mode 100644 index 000000000000..4f3cc0166019 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2017-12-21-22-00-11.bpo-22908.cVm89I.rst @@ -0,0 +1,2 @@ +Added seek and tell to the ZipExtFile class. This only works if the file +object used to open the zipfile is seekable. [View Less]

1 0