Mailman 3 August 2011 - Python-checkins

cpython (merge 2.7 -> 2.7): Branch merge
by eric.araujo 01 Aug '11

01 Aug '11

http://hg.python.org/cpython/rev/8065f927b4f6 changeset: 71675:8065f927b4f6 branch: 2.7 parent: 71660:42f40f53fd73 parent: 71674:a425408f1e52 user: Éric Araujo <merwok(a)netwok.org> date: Mon Aug 01 14:48:19 2011 +0200 summary: Branch merge files: Doc/distutils/sourcedist.rst | 29 ++++++---- Lib/distutils/command/sdist.py | 48 ++++++++++------- Lib/distutils/tests/test_sdist.py | 43 ++++++++++++--- Lib/lib2to3/tests/test_refactor.py | 18 ++++-- Misc/ACKS | 2 + Misc/NEWS | 6 ++ Tools/scripts/patchcheck.py | 20 ++++-- 7 files changed, 112 insertions(+), 54 deletions(-) diff --git a/Doc/distutils/sourcedist.rst b/Doc/distutils/sourcedist.rst --- a/Doc/distutils/sourcedist.rst +++ b/Doc/distutils/sourcedist.rst @@ -111,12 +111,22 @@ :file:`MANIFEST`, you must specify everything: the default set of files described above does not apply in this case. -.. versionadded:: 2.7 +.. versionchanged:: 2.7 + An existing generated :file:`MANIFEST` will be regenerated without + :command:`sdist` comparing its modification time to the one of + :file:`MANIFEST.in` or :file:`setup.py`. + +.. versionchanged:: 2.7.1 :file:`MANIFEST` files start with a comment indicating they are generated. Files without this comment are not overwritten or removed. +.. versionchanged:: 2.7.3 + :command:`sdist` will read a :file:`MANIFEST` file if no :file:`MANIFEST.in` + exists, like it did before 2.7. + See :ref:`manifest_template` section for a syntax reference. + .. _manifest-options: Manifest-related options @@ -124,16 +134,16 @@ The normal course of operations for the :command:`sdist` command is as follows: -* if the manifest file, :file:`MANIFEST` doesn't exist, read :file:`MANIFEST.in` - and create the manifest +* if the manifest file (:file:`MANIFEST` by default) exists and the first line + does not have a comment indicating it is generated from :file:`MANIFEST.in`, + then it is used as is, unaltered + +* if the manifest file doesn't exist or has been previously automatically + generated, read :file:`MANIFEST.in` and create the manifest * if neither :file:`MANIFEST` nor :file:`MANIFEST.in` exist, create a manifest with just the default file set -* if either :file:`MANIFEST.in` or the setup script (:file:`setup.py`) are more - recent than :file:`MANIFEST`, recreate :file:`MANIFEST` by reading - :file:`MANIFEST.in` - * use the list of files now in :file:`MANIFEST` (either just generated or read in) to create the source distribution archive(s) @@ -271,8 +281,3 @@ ``a-z``, ``a-zA-Z``, ``a-f0-9_.``). The definition of "regular filename character" is platform-specific: on Unix it is anything except slash; on Windows anything except backslash or colon. - -.. versionchanged:: 2.7 - An existing generated :file:`MANIFEST` will be regenerated without - :command:`sdist` comparing its modification time to the one of - :file:`MANIFEST.in` or :file:`setup.py`. diff --git a/Lib/distutils/command/sdist.py b/Lib/distutils/command/sdist.py --- a/Lib/distutils/command/sdist.py +++ b/Lib/distutils/command/sdist.py @@ -182,14 +182,20 @@ reading the manifest, or just using the default file set -- it all depends on the user's options. """ - # new behavior: + # new behavior when using a template: # the file list is recalculated everytime because # even if MANIFEST.in or setup.py are not changed # the user might have added some files in the tree that # need to be included. # - # This makes --force the default and only behavior. + # This makes --force the default and only behavior with templates. template_exists = os.path.isfile(self.template) + if not template_exists and self._manifest_is_not_generated(): + self.read_manifest() + self.filelist.sort() + self.filelist.remove_duplicates() + return + if not template_exists: self.warn(("manifest template '%s' does not exist " + "(using default file list)") % @@ -352,23 +358,28 @@ by 'add_defaults()' and 'read_template()') to the manifest file named by 'self.manifest'. """ - if os.path.isfile(self.manifest): - fp = open(self.manifest) - try: - first_line = fp.readline() - finally: - fp.close() - - if first_line != '# file GENERATED by distutils, do NOT edit\n': - log.info("not writing to manually maintained " - "manifest file '%s'" % self.manifest) - return + if self._manifest_is_not_generated(): + log.info("not writing to manually maintained " + "manifest file '%s'" % self.manifest) + return content = self.filelist.files[:] content.insert(0, '# file GENERATED by distutils, do NOT edit') self.execute(file_util.write_file, (self.manifest, content), "writing manifest file '%s'" % self.manifest) + def _manifest_is_not_generated(self): + # check for special comment used in 2.7.1 and higher + if not os.path.isfile(self.manifest): + return False + + fp = open(self.manifest, 'rU') + try: + first_line = fp.readline() + finally: + fp.close() + return first_line != '# file GENERATED by distutils, do NOT edit\n' + def read_manifest(self): """Read the manifest file (named by 'self.manifest') and use it to fill in 'self.filelist', the list of files to include in the source @@ -376,12 +387,11 @@ """ log.info("reading manifest file '%s'", self.manifest) manifest = open(self.manifest) - while 1: - line = manifest.readline() - if line == '': # end of file - break - if line[-1] == '\n': - line = line[0:-1] + for line in manifest: + # ignore comments and blank lines + line = line.strip() + if line.startswith('#') or not line: + continue self.filelist.append(line) manifest.close() diff --git a/Lib/distutils/tests/test_sdist.py b/Lib/distutils/tests/test_sdist.py --- a/Lib/distutils/tests/test_sdist.py +++ b/Lib/distutils/tests/test_sdist.py @@ -1,9 +1,11 @@ """Tests for distutils.command.sdist.""" import os +import tarfile import unittest -import shutil +import warnings import zipfile -import tarfile +from os.path import join +from textwrap import dedent # zlib is not used here, but if it's not available # the tests that use zipfile may fail @@ -19,19 +21,13 @@ except ImportError: UID_GID_SUPPORT = False -from os.path import join -import sys -import tempfile -import warnings - from test.test_support import captured_stdout, check_warnings, run_unittest from distutils.command.sdist import sdist, show_formats from distutils.core import Distribution from distutils.tests.test_config import PyPIRCCommandTestCase -from distutils.errors import DistutilsExecError, DistutilsOptionError +from distutils.errors import DistutilsOptionError from distutils.spawn import find_executable -from distutils.tests import support from distutils.log import WARN from distutils.archive_util import ARCHIVE_FORMATS @@ -405,13 +401,33 @@ self.assertEqual(manifest[0], '# file GENERATED by distutils, do NOT edit') + @unittest.skipUnless(zlib, 'requires zlib') + def test_manifest_comments(self): + # make sure comments don't cause exceptions or wrong includes + contents = dedent("""\ + # bad.py + #bad.py + good.py + """) + dist, cmd = self.get_cmd() + cmd.ensure_finalized() + self.write_file((self.tmp_dir, cmd.manifest), contents) + self.write_file((self.tmp_dir, 'good.py'), '# pick me!') + self.write_file((self.tmp_dir, 'bad.py'), "# don't pick me!") + self.write_file((self.tmp_dir, '#bad.py'), "# don't pick me!") + cmd.run() + self.assertEqual(cmd.filelist.files, ['good.py']) + @unittest.skipUnless(zlib, "requires zlib") def test_manual_manifest(self): # check that a MANIFEST without a marker is left alone dist, cmd = self.get_cmd() cmd.ensure_finalized() self.write_file((self.tmp_dir, cmd.manifest), 'README.manual') + self.write_file((self.tmp_dir, 'README.manual'), + 'This project maintains its MANIFEST file itself.') cmd.run() + self.assertEqual(cmd.filelist.files, ['README.manual']) f = open(cmd.manifest) try: @@ -422,6 +438,15 @@ self.assertEqual(manifest, ['README.manual']) + archive_name = join(self.tmp_dir, 'dist', 'fake-1.0.tar.gz') + archive = tarfile.open(archive_name) + try: + filenames = [tarinfo.name for tarinfo in archive] + finally: + archive.close() + self.assertEqual(sorted(filenames), ['fake-1.0', 'fake-1.0/PKG-INFO', + 'fake-1.0/README.manual']) + def test_suite(): return unittest.makeSuite(SDistTestCase) diff --git a/Lib/lib2to3/tests/test_refactor.py b/Lib/lib2to3/tests/test_refactor.py --- a/Lib/lib2to3/tests/test_refactor.py +++ b/Lib/lib2to3/tests/test_refactor.py @@ -177,22 +177,26 @@ self.assertEqual(results, expected) def check_file_refactoring(self, test_file, fixers=_2TO3_FIXERS): + tmpdir = tempfile.mkdtemp(prefix="2to3-test_refactor") + self.addCleanup(shutil.rmtree, tmpdir) + # make a copy of the tested file that we can write to + shutil.copy(test_file, tmpdir) + test_file = os.path.join(tmpdir, os.path.basename(test_file)) + os.chmod(test_file, 0o644) + def read_file(): with open(test_file, "rb") as fp: return fp.read() + old_contents = read_file() rt = self.rt(fixers=fixers) rt.refactor_file(test_file) self.assertEqual(old_contents, read_file()) - try: - rt.refactor_file(test_file, True) - new_contents = read_file() - self.assertNotEqual(old_contents, new_contents) - finally: - with open(test_file, "wb") as fp: - fp.write(old_contents) + rt.refactor_file(test_file, True) + new_contents = read_file() + self.assertNotEqual(old_contents, new_contents) return new_contents def test_refactor_file(self): diff --git a/Misc/ACKS b/Misc/ACKS --- a/Misc/ACKS +++ b/Misc/ACKS @@ -194,6 +194,7 @@ Vincent Delft Arnaud Delobelle Erik Demaine +John Dennis Roger Dev Raghuram Devarakonda Catherine Devlin @@ -813,6 +814,7 @@ Tobias Thelen James Thomas Robin Thomas +Stephen Thorne Eric Tiedemann Tracy Tims Oren Tirosh diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -37,6 +37,9 @@ Library ------- +- Issues #11104, #8688: Fix the behavior of distutils' sdist command with + manually-maintained MANIFEST files. + - Issue #8887: "pydoc somebuiltin.somemethod" (or help('somebuiltin.somemethod') in Python code) now finds the doc of the method. @@ -158,6 +161,9 @@ Tests ----- +- Issue #12331: The test suite for lib2to3 can now run from an installed + Python. + - Issue #12549: Correct test_platform to not fail when OS X returns 'x86_64' as the processor type on some Mac systems. diff --git a/Tools/scripts/patchcheck.py b/Tools/scripts/patchcheck.py --- a/Tools/scripts/patchcheck.py +++ b/Tools/scripts/patchcheck.py @@ -4,11 +4,15 @@ import shutil import os.path import subprocess +import sysconfig import reindent import untabify +SRCDIR = sysconfig.get_config_var('srcdir') + + def n_files_str(count): """Return 'N file(s)' with the proper plurality on 'file'.""" return "{} file{}".format(count, "s" if count != 1 else "") @@ -36,7 +40,7 @@ info=lambda x: n_files_str(len(x))) def changed_files(): """Get the list of changed or added files from the VCS.""" - if os.path.isdir('.hg'): + if os.path.isdir(os.path.join(SRCDIR, '.hg')): vcs = 'hg' cmd = 'hg status --added --modified --no-status' elif os.path.isdir('.svn'): @@ -75,7 +79,7 @@ reindent.makebackup = False # No need to create backups. fixed = [] for path in (x for x in file_paths if x.endswith('.py')): - if reindent.check(path): + if reindent.check(os.path.join(SRCDIR, path)): fixed.append(path) return fixed @@ -85,10 +89,11 @@ """Report if any C files """ fixed = [] for path in file_paths: - with open(path, 'r') as f: + abspath = os.path.join(SRCDIR, path) + with open(abspath, 'r') as f: if '\t' not in f.read(): continue - untabify.process(path, 8, verbose=False) + untabify.process(abspath, 8, verbose=False) fixed.append(path) return fixed @@ -99,13 +104,14 @@ def normalize_docs_whitespace(file_paths): fixed = [] for path in file_paths: + abspath = os.path.join(SRCDIR, path) try: - with open(path, 'rb') as f: + with open(abspath, 'rb') as f: lines = f.readlines() new_lines = [ws_re.sub(br'\1', line) for line in lines] if new_lines != lines: - shutil.copyfile(path, path + '.bak') - with open(path, 'wb') as f: + shutil.copyfile(abspath, abspath + '.bak') + with open(abspath, 'wb') as f: f.writelines(new_lines) fixed.append(path) except Exception as err: -- Repository URL: http://hg.python.org/cpython

1 0

cpython (2.7): Stop trying to write into the stdlib during lib2to3 tests (#12331).
by eric.araujo 01 Aug '11

01 Aug '11

http://hg.python.org/cpython/rev/a425408f1e52 changeset: 71674:a425408f1e52 branch: 2.7 user: Éric Araujo <merwok(a)netwok.org> date: Sun Jul 31 17:59:28 2011 +0200 summary: Stop trying to write into the stdlib during lib2to3 tests (#12331). This prevents tests from failing when run from a Python installed in a read-only directory. files: Lib/lib2to3/tests/test_refactor.py | 18 +++++++++++------- Misc/NEWS | 3 +++ 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/Lib/lib2to3/tests/test_refactor.py b/Lib/lib2to3/tests/test_refactor.py --- a/Lib/lib2to3/tests/test_refactor.py +++ b/Lib/lib2to3/tests/test_refactor.py @@ -177,22 +177,26 @@ self.assertEqual(results, expected) def check_file_refactoring(self, test_file, fixers=_2TO3_FIXERS): + tmpdir = tempfile.mkdtemp(prefix="2to3-test_refactor") + self.addCleanup(shutil.rmtree, tmpdir) + # make a copy of the tested file that we can write to + shutil.copy(test_file, tmpdir) + test_file = os.path.join(tmpdir, os.path.basename(test_file)) + os.chmod(test_file, 0o644) + def read_file(): with open(test_file, "rb") as fp: return fp.read() + old_contents = read_file() rt = self.rt(fixers=fixers) rt.refactor_file(test_file) self.assertEqual(old_contents, read_file()) - try: - rt.refactor_file(test_file, True) - new_contents = read_file() - self.assertNotEqual(old_contents, new_contents) - finally: - with open(test_file, "wb") as fp: - fp.write(old_contents) + rt.refactor_file(test_file, True) + new_contents = read_file() + self.assertNotEqual(old_contents, new_contents) return new_contents def test_refactor_file(self): diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -161,6 +161,9 @@ Tests ----- +- Issue #12331: The test suite for lib2to3 can now run from an installed + Python. + - Issue #12549: Correct test_platform to not fail when OS X returns 'x86_64' as the processor type on some Mac systems. -- Repository URL: http://hg.python.org/cpython

1 0

cpython (2.7): Fix regression with distutils MANIFEST handing (#11104, #8688).
by eric.araujo 01 Aug '11

01 Aug '11

http://hg.python.org/cpython/rev/21feea7f35e5 changeset: 71673:21feea7f35e5 branch: 2.7 user: Éric Araujo <merwok(a)netwok.org> date: Sun Jul 31 02:04:00 2011 +0200 summary: Fix regression with distutils MANIFEST handing (#11104, #8688). The changed behavior of sdist in 2.7 broke packaging for projects that wanted to use a manually-maintained MANIFEST file (instead of having a MANIFEST.in template and letting distutils generate the MANIFEST). The fixes that were committed for #8688 (d29399100973 by Tarek and f7639dcdffc3 by me) did not fix all issues exposed in the bug report, and also added one problem: the MANIFEST file format gained comments, but the read_manifest method was not updated to handle (i.e. ignore) them. This changeset should fix everything; the tests have been expanded and I successfully tested with Mercurial, which suffered from this regression. I have grouped the versionchanged directives for these bugs in one place and added micro version numbers to help users know the quirks of the exact version they’re using. I also removed a stanza in the docs that was forgotten in Tarek’s first changeset. Initial report, thorough diagnosis and patch by John Dennis, further work on the patch by Stephen Thorne, and a few edits and additions by me. files: Doc/distutils/sourcedist.rst | 29 ++++++---- Lib/distutils/command/sdist.py | 48 +++++++++++------- Lib/distutils/tests/test_sdist.py | 43 +++++++++++++--- Misc/ACKS | 2 + Misc/NEWS | 3 + 5 files changed, 85 insertions(+), 40 deletions(-) diff --git a/Doc/distutils/sourcedist.rst b/Doc/distutils/sourcedist.rst --- a/Doc/distutils/sourcedist.rst +++ b/Doc/distutils/sourcedist.rst @@ -111,12 +111,22 @@ :file:`MANIFEST`, you must specify everything: the default set of files described above does not apply in this case. -.. versionadded:: 2.7 +.. versionchanged:: 2.7 + An existing generated :file:`MANIFEST` will be regenerated without + :command:`sdist` comparing its modification time to the one of + :file:`MANIFEST.in` or :file:`setup.py`. + +.. versionchanged:: 2.7.1 :file:`MANIFEST` files start with a comment indicating they are generated. Files without this comment are not overwritten or removed. +.. versionchanged:: 2.7.3 + :command:`sdist` will read a :file:`MANIFEST` file if no :file:`MANIFEST.in` + exists, like it did before 2.7. + See :ref:`manifest_template` section for a syntax reference. + .. _manifest-options: Manifest-related options @@ -124,16 +134,16 @@ The normal course of operations for the :command:`sdist` command is as follows: -* if the manifest file, :file:`MANIFEST` doesn't exist, read :file:`MANIFEST.in` - and create the manifest +* if the manifest file (:file:`MANIFEST` by default) exists and the first line + does not have a comment indicating it is generated from :file:`MANIFEST.in`, + then it is used as is, unaltered + +* if the manifest file doesn't exist or has been previously automatically + generated, read :file:`MANIFEST.in` and create the manifest * if neither :file:`MANIFEST` nor :file:`MANIFEST.in` exist, create a manifest with just the default file set -* if either :file:`MANIFEST.in` or the setup script (:file:`setup.py`) are more - recent than :file:`MANIFEST`, recreate :file:`MANIFEST` by reading - :file:`MANIFEST.in` - * use the list of files now in :file:`MANIFEST` (either just generated or read in) to create the source distribution archive(s) @@ -271,8 +281,3 @@ ``a-z``, ``a-zA-Z``, ``a-f0-9_.``). The definition of "regular filename character" is platform-specific: on Unix it is anything except slash; on Windows anything except backslash or colon. - -.. versionchanged:: 2.7 - An existing generated :file:`MANIFEST` will be regenerated without - :command:`sdist` comparing its modification time to the one of - :file:`MANIFEST.in` or :file:`setup.py`. diff --git a/Lib/distutils/command/sdist.py b/Lib/distutils/command/sdist.py --- a/Lib/distutils/command/sdist.py +++ b/Lib/distutils/command/sdist.py @@ -182,14 +182,20 @@ reading the manifest, or just using the default file set -- it all depends on the user's options. """ - # new behavior: + # new behavior when using a template: # the file list is recalculated everytime because # even if MANIFEST.in or setup.py are not changed # the user might have added some files in the tree that # need to be included. # - # This makes --force the default and only behavior. + # This makes --force the default and only behavior with templates. template_exists = os.path.isfile(self.template) + if not template_exists and self._manifest_is_not_generated(): + self.read_manifest() + self.filelist.sort() + self.filelist.remove_duplicates() + return + if not template_exists: self.warn(("manifest template '%s' does not exist " + "(using default file list)") % @@ -352,23 +358,28 @@ by 'add_defaults()' and 'read_template()') to the manifest file named by 'self.manifest'. """ - if os.path.isfile(self.manifest): - fp = open(self.manifest) - try: - first_line = fp.readline() - finally: - fp.close() - - if first_line != '# file GENERATED by distutils, do NOT edit\n': - log.info("not writing to manually maintained " - "manifest file '%s'" % self.manifest) - return + if self._manifest_is_not_generated(): + log.info("not writing to manually maintained " + "manifest file '%s'" % self.manifest) + return content = self.filelist.files[:] content.insert(0, '# file GENERATED by distutils, do NOT edit') self.execute(file_util.write_file, (self.manifest, content), "writing manifest file '%s'" % self.manifest) + def _manifest_is_not_generated(self): + # check for special comment used in 2.7.1 and higher + if not os.path.isfile(self.manifest): + return False + + fp = open(self.manifest, 'rU') + try: + first_line = fp.readline() + finally: + fp.close() + return first_line != '# file GENERATED by distutils, do NOT edit\n' + def read_manifest(self): """Read the manifest file (named by 'self.manifest') and use it to fill in 'self.filelist', the list of files to include in the source @@ -376,12 +387,11 @@ """ log.info("reading manifest file '%s'", self.manifest) manifest = open(self.manifest) - while 1: - line = manifest.readline() - if line == '': # end of file - break - if line[-1] == '\n': - line = line[0:-1] + for line in manifest: + # ignore comments and blank lines + line = line.strip() + if line.startswith('#') or not line: + continue self.filelist.append(line) manifest.close() diff --git a/Lib/distutils/tests/test_sdist.py b/Lib/distutils/tests/test_sdist.py --- a/Lib/distutils/tests/test_sdist.py +++ b/Lib/distutils/tests/test_sdist.py @@ -1,9 +1,11 @@ """Tests for distutils.command.sdist.""" import os +import tarfile import unittest -import shutil +import warnings import zipfile -import tarfile +from os.path import join +from textwrap import dedent # zlib is not used here, but if it's not available # the tests that use zipfile may fail @@ -19,19 +21,13 @@ except ImportError: UID_GID_SUPPORT = False -from os.path import join -import sys -import tempfile -import warnings - from test.test_support import captured_stdout, check_warnings, run_unittest from distutils.command.sdist import sdist, show_formats from distutils.core import Distribution from distutils.tests.test_config import PyPIRCCommandTestCase -from distutils.errors import DistutilsExecError, DistutilsOptionError +from distutils.errors import DistutilsOptionError from distutils.spawn import find_executable -from distutils.tests import support from distutils.log import WARN from distutils.archive_util import ARCHIVE_FORMATS @@ -405,13 +401,33 @@ self.assertEqual(manifest[0], '# file GENERATED by distutils, do NOT edit') + @unittest.skipUnless(zlib, 'requires zlib') + def test_manifest_comments(self): + # make sure comments don't cause exceptions or wrong includes + contents = dedent("""\ + # bad.py + #bad.py + good.py + """) + dist, cmd = self.get_cmd() + cmd.ensure_finalized() + self.write_file((self.tmp_dir, cmd.manifest), contents) + self.write_file((self.tmp_dir, 'good.py'), '# pick me!') + self.write_file((self.tmp_dir, 'bad.py'), "# don't pick me!") + self.write_file((self.tmp_dir, '#bad.py'), "# don't pick me!") + cmd.run() + self.assertEqual(cmd.filelist.files, ['good.py']) + @unittest.skipUnless(zlib, "requires zlib") def test_manual_manifest(self): # check that a MANIFEST without a marker is left alone dist, cmd = self.get_cmd() cmd.ensure_finalized() self.write_file((self.tmp_dir, cmd.manifest), 'README.manual') + self.write_file((self.tmp_dir, 'README.manual'), + 'This project maintains its MANIFEST file itself.') cmd.run() + self.assertEqual(cmd.filelist.files, ['README.manual']) f = open(cmd.manifest) try: @@ -422,6 +438,15 @@ self.assertEqual(manifest, ['README.manual']) + archive_name = join(self.tmp_dir, 'dist', 'fake-1.0.tar.gz') + archive = tarfile.open(archive_name) + try: + filenames = [tarinfo.name for tarinfo in archive] + finally: + archive.close() + self.assertEqual(sorted(filenames), ['fake-1.0', 'fake-1.0/PKG-INFO', + 'fake-1.0/README.manual']) + def test_suite(): return unittest.makeSuite(SDistTestCase) diff --git a/Misc/ACKS b/Misc/ACKS --- a/Misc/ACKS +++ b/Misc/ACKS @@ -194,6 +194,7 @@ Vincent Delft Arnaud Delobelle Erik Demaine +John Dennis Roger Dev Raghuram Devarakonda Catherine Devlin @@ -813,6 +814,7 @@ Tobias Thelen James Thomas Robin Thomas +Stephen Thorne Eric Tiedemann Tracy Tims Oren Tirosh diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -37,6 +37,9 @@ Library ------- +- Issues #11104, #8688: Fix the behavior of distutils' sdist command with + manually-maintained MANIFEST files. + - Issue #8887: "pydoc somebuiltin.somemethod" (or help('somebuiltin.somemethod') in Python code) now finds the doc of the method. -- Repository URL: http://hg.python.org/cpython

1 0

cpython (2.7): Let “make patchcheck” work for out-of-dir builds (#9860)
by eric.araujo 01 Aug '11

01 Aug '11

http://hg.python.org/cpython/rev/bea11ce24bb0 changeset: 71672:bea11ce24bb0 branch: 2.7 parent: 71628:5a248fcfa112 user: Éric Araujo <merwok(a)netwok.org> date: Sat Jul 30 21:34:04 2011 +0200 summary: Let “make patchcheck” work for out-of-dir builds (#9860) files: Tools/scripts/patchcheck.py | 20 +++++++++++++------- 1 files changed, 13 insertions(+), 7 deletions(-) diff --git a/Tools/scripts/patchcheck.py b/Tools/scripts/patchcheck.py --- a/Tools/scripts/patchcheck.py +++ b/Tools/scripts/patchcheck.py @@ -4,11 +4,15 @@ import shutil import os.path import subprocess +import sysconfig import reindent import untabify +SRCDIR = sysconfig.get_config_var('srcdir') + + def n_files_str(count): """Return 'N file(s)' with the proper plurality on 'file'.""" return "{} file{}".format(count, "s" if count != 1 else "") @@ -36,7 +40,7 @@ info=lambda x: n_files_str(len(x))) def changed_files(): """Get the list of changed or added files from the VCS.""" - if os.path.isdir('.hg'): + if os.path.isdir(os.path.join(SRCDIR, '.hg')): vcs = 'hg' cmd = 'hg status --added --modified --no-status' elif os.path.isdir('.svn'): @@ -75,7 +79,7 @@ reindent.makebackup = False # No need to create backups. fixed = [] for path in (x for x in file_paths if x.endswith('.py')): - if reindent.check(path): + if reindent.check(os.path.join(SRCDIR, path)): fixed.append(path) return fixed @@ -85,10 +89,11 @@ """Report if any C files """ fixed = [] for path in file_paths: - with open(path, 'r') as f: + abspath = os.path.join(SRCDIR, path) + with open(abspath, 'r') as f: if '\t' not in f.read(): continue - untabify.process(path, 8, verbose=False) + untabify.process(abspath, 8, verbose=False) fixed.append(path) return fixed @@ -99,13 +104,14 @@ def normalize_docs_whitespace(file_paths): fixed = [] for path in file_paths: + abspath = os.path.join(SRCDIR, path) try: - with open(path, 'rb') as f: + with open(abspath, 'rb') as f: lines = f.readlines() new_lines = [ws_re.sub(br'\1', line) for line in lines] if new_lines != lines: - shutil.copyfile(path, path + '.bak') - with open(path, 'wb') as f: + shutil.copyfile(abspath, abspath + '.bak') + with open(abspath, 'wb') as f: f.writelines(new_lines) fixed.append(path) except Exception as err: -- Repository URL: http://hg.python.org/cpython

1 0

cpython (merge 3.2 -> default): Merge 3.2
by eric.araujo 01 Aug '11

01 Aug '11

http://hg.python.org/cpython/rev/7e9bfd0395d1 changeset: 71671:7e9bfd0395d1 parent: 71670:abd8380c15e2 parent: 71669:da79c5d0ed17 user: Éric Araujo <merwok(a)netwok.org> date: Mon Aug 01 14:45:01 2011 +0200 summary: Merge 3.2 files: -- Repository URL: http://hg.python.org/cpython

1 0

cpython (merge 3.2 -> 3.2): Branch merge
by eric.araujo 01 Aug '11

01 Aug '11

http://hg.python.org/cpython/rev/da79c5d0ed17 changeset: 71669:da79c5d0ed17 branch: 3.2 parent: 71661:ba5bd8c1ae27 parent: 71665:2b5a0c4e052b user: Éric Araujo <merwok(a)netwok.org> date: Mon Aug 01 14:43:45 2011 +0200 summary: Branch merge files: Doc/distutils/sourcedist.rst | 25 ++++++--- Lib/distutils/command/sdist.py | 48 ++++++++++------- Lib/distutils/tests/test_sdist.py | 39 ++++++++++++-- Lib/lib2to3/tests/test_refactor.py | 18 ++++-- Misc/ACKS | 2 + Misc/NEWS | 6 ++ Tools/scripts/patchcheck.py | 20 ++++-- 7 files changed, 111 insertions(+), 47 deletions(-) diff --git a/Doc/distutils/sourcedist.rst b/Doc/distutils/sourcedist.rst --- a/Doc/distutils/sourcedist.rst +++ b/Doc/distutils/sourcedist.rst @@ -103,10 +103,20 @@ :file:`MANIFEST`, you must specify everything: the default set of files described above does not apply in this case. -.. versionadded:: 3.1 +.. versionchanged:: 3.1 + An existing generated :file:`MANIFEST` will be regenerated without + :command:`sdist` comparing its modification time to the one of + :file:`MANIFEST.in` or :file:`setup.py`. + +.. versionchanged:: 3.1.3 :file:`MANIFEST` files start with a comment indicating they are generated. Files without this comment are not overwritten or removed. +.. versionchanged:: 3.2.2 + :command:`sdist` will read a :file:`MANIFEST` file if no :file:`MANIFEST.in` + exists, like it used to do. + + The manifest template has one command per line, where each command specifies a set of files to include or exclude from the source distribution. For an example, again we turn to the Distutils' own manifest template:: @@ -185,8 +195,12 @@ The normal course of operations for the :command:`sdist` command is as follows: -* if the manifest file, :file:`MANIFEST` doesn't exist, read :file:`MANIFEST.in` - and create the manifest +* if the manifest file (:file:`MANIFEST` by default) exists and the first line + does not have a comment indicating it is generated from :file:`MANIFEST.in`, + then it is used as is, unaltered + +* if the manifest file doesn't exist or has been previously automatically + generated, read :file:`MANIFEST.in` and create the manifest * if neither :file:`MANIFEST` nor :file:`MANIFEST.in` exist, create a manifest with just the default file set @@ -204,8 +218,3 @@ python setup.py sdist --manifest-only :option:`-o` is a shortcut for :option:`--manifest-only`. - -.. versionchanged:: 3.1 - An existing generated :file:`MANIFEST` will be regenerated without - :command:`sdist` comparing its modification time to the one of - :file:`MANIFEST.in` or :file:`setup.py`. diff --git a/Lib/distutils/command/sdist.py b/Lib/distutils/command/sdist.py --- a/Lib/distutils/command/sdist.py +++ b/Lib/distutils/command/sdist.py @@ -174,14 +174,20 @@ reading the manifest, or just using the default file set -- it all depends on the user's options. """ - # new behavior: + # new behavior when using a template: # the file list is recalculated everytime because # even if MANIFEST.in or setup.py are not changed # the user might have added some files in the tree that # need to be included. # - # This makes --force the default and only behavior. + # This makes --force the default and only behavior with templates. template_exists = os.path.isfile(self.template) + if not template_exists and self._manifest_is_not_generated(): + self.read_manifest() + self.filelist.sort() + self.filelist.remove_duplicates() + return + if not template_exists: self.warn(("manifest template '%s' does not exist " + "(using default file list)") % @@ -336,23 +342,28 @@ by 'add_defaults()' and 'read_template()') to the manifest file named by 'self.manifest'. """ - if os.path.isfile(self.manifest): - fp = open(self.manifest) - try: - first_line = fp.readline() - finally: - fp.close() - - if first_line != '# file GENERATED by distutils, do NOT edit\n': - log.info("not writing to manually maintained " - "manifest file '%s'" % self.manifest) - return + if self._manifest_is_not_generated(): + log.info("not writing to manually maintained " + "manifest file '%s'" % self.manifest) + return content = self.filelist.files[:] content.insert(0, '# file GENERATED by distutils, do NOT edit') self.execute(file_util.write_file, (self.manifest, content), "writing manifest file '%s'" % self.manifest) + def _manifest_is_not_generated(self): + # check for special comment used in 3.1.3 and higher + if not os.path.isfile(self.manifest): + return False + + fp = open(self.manifest) + try: + first_line = fp.readline() + finally: + fp.close() + return first_line != '# file GENERATED by distutils, do NOT edit\n' + def read_manifest(self): """Read the manifest file (named by 'self.manifest') and use it to fill in 'self.filelist', the list of files to include in the source @@ -360,12 +371,11 @@ """ log.info("reading manifest file '%s'", self.manifest) manifest = open(self.manifest) - while True: - line = manifest.readline() - if line == '': # end of file - break - if line[-1] == '\n': - line = line[0:-1] + for line in manifest: + # ignore comments and blank lines + line = line.strip() + if line.startswith('#') or not line: + continue self.filelist.append(line) manifest.close() diff --git a/Lib/distutils/tests/test_sdist.py b/Lib/distutils/tests/test_sdist.py --- a/Lib/distutils/tests/test_sdist.py +++ b/Lib/distutils/tests/test_sdist.py @@ -1,21 +1,19 @@ """Tests for distutils.command.sdist.""" import os +import tarfile import unittest -import shutil +import warnings import zipfile from os.path import join -import sys -import tempfile -import warnings +from textwrap import dedent from test.support import captured_stdout, check_warnings, run_unittest from distutils.command.sdist import sdist, show_formats from distutils.core import Distribution from distutils.tests.test_config import PyPIRCCommandTestCase -from distutils.errors import DistutilsExecError, DistutilsOptionError +from distutils.errors import DistutilsOptionError from distutils.spawn import find_executable -from distutils.tests import support from distutils.log import WARN from distutils.archive_util import ARCHIVE_FORMATS @@ -346,13 +344,33 @@ self.assertEqual(manifest[0], '# file GENERATED by distutils, do NOT edit') + @unittest.skipUnless(ZLIB_SUPPORT, "Need zlib support to run") + def test_manifest_comments(self): + # make sure comments don't cause exceptions or wrong includes + contents = dedent("""\ + # bad.py + #bad.py + good.py + """) + dist, cmd = self.get_cmd() + cmd.ensure_finalized() + self.write_file((self.tmp_dir, cmd.manifest), contents) + self.write_file((self.tmp_dir, 'good.py'), '# pick me!') + self.write_file((self.tmp_dir, 'bad.py'), "# don't pick me!") + self.write_file((self.tmp_dir, '#bad.py'), "# don't pick me!") + cmd.run() + self.assertEqual(cmd.filelist.files, ['good.py']) + @unittest.skipUnless(ZLIB_SUPPORT, 'Need zlib support to run') def test_manual_manifest(self): # check that a MANIFEST without a marker is left alone dist, cmd = self.get_cmd() cmd.ensure_finalized() self.write_file((self.tmp_dir, cmd.manifest), 'README.manual') + self.write_file((self.tmp_dir, 'README.manual'), + 'This project maintains its MANIFEST file itself.') cmd.run() + self.assertEqual(cmd.filelist.files, ['README.manual']) f = open(cmd.manifest) try: @@ -363,6 +381,15 @@ self.assertEqual(manifest, ['README.manual']) + archive_name = join(self.tmp_dir, 'dist', 'fake-1.0.tar.gz') + archive = tarfile.open(archive_name) + try: + filenames = [tarinfo.name for tarinfo in archive] + finally: + archive.close() + self.assertEqual(sorted(filenames), ['fake-1.0', 'fake-1.0/PKG-INFO', + 'fake-1.0/README.manual']) + def test_suite(): return unittest.makeSuite(SDistTestCase) diff --git a/Lib/lib2to3/tests/test_refactor.py b/Lib/lib2to3/tests/test_refactor.py --- a/Lib/lib2to3/tests/test_refactor.py +++ b/Lib/lib2to3/tests/test_refactor.py @@ -177,22 +177,26 @@ self.assertEqual(results, expected) def check_file_refactoring(self, test_file, fixers=_2TO3_FIXERS): + tmpdir = tempfile.mkdtemp(prefix="2to3-test_refactor") + self.addCleanup(shutil.rmtree, tmpdir) + # make a copy of the tested file that we can write to + shutil.copy(test_file, tmpdir) + test_file = os.path.join(tmpdir, os.path.basename(test_file)) + os.chmod(test_file, 0o644) + def read_file(): with open(test_file, "rb") as fp: return fp.read() + old_contents = read_file() rt = self.rt(fixers=fixers) rt.refactor_file(test_file) self.assertEqual(old_contents, read_file()) - try: - rt.refactor_file(test_file, True) - new_contents = read_file() - self.assertNotEqual(old_contents, new_contents) - finally: - with open(test_file, "wb") as fp: - fp.write(old_contents) + rt.refactor_file(test_file, True) + new_contents = read_file() + self.assertNotEqual(old_contents, new_contents) return new_contents def test_refactor_file(self): diff --git a/Misc/ACKS b/Misc/ACKS --- a/Misc/ACKS +++ b/Misc/ACKS @@ -215,6 +215,7 @@ Vincent Delft Arnaud Delobelle Erik Demaine +John Dennis Roger Dev Raghuram Devarakonda Caleb Deveraux @@ -875,6 +876,7 @@ Tobias Thelen James Thomas Robin Thomas +Stephen Thorne Jeremy Thurgood Eric Tiedemann July Tikhonov diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -41,6 +41,9 @@ Library ------- +- Issues #11104, #8688: Fix the behavior of distutils' sdist command with + manually-maintained MANIFEST files. + - Issue #12464: tempfile.TemporaryDirectory.cleanup() should not follow symlinks: fix it. Patch by Petri Lehtinen. @@ -137,6 +140,9 @@ Tests ----- +- Issue #12331: The test suite for lib2to3 can now run from an installed + Python. + - Issue #12626: In regrtest, allow to filter tests using a glob filter with the ``-m`` (or ``--match``) option. This works with all test cases using the unittest module. This is useful with long test suites diff --git a/Tools/scripts/patchcheck.py b/Tools/scripts/patchcheck.py --- a/Tools/scripts/patchcheck.py +++ b/Tools/scripts/patchcheck.py @@ -4,11 +4,15 @@ import shutil import os.path import subprocess +import sysconfig import reindent import untabify +SRCDIR = sysconfig.get_config_var('srcdir') + + def n_files_str(count): """Return 'N file(s)' with the proper plurality on 'file'.""" return "{} file{}".format(count, "s" if count != 1 else "") @@ -36,7 +40,7 @@ info=lambda x: n_files_str(len(x))) def changed_files(): """Get the list of changed or added files from the VCS.""" - if os.path.isdir('.hg'): + if os.path.isdir(os.path.join(SRCDIR, '.hg')): vcs = 'hg' cmd = 'hg status --added --modified --no-status' elif os.path.isdir('.svn'): @@ -75,7 +79,7 @@ reindent.makebackup = False # No need to create backups. fixed = [] for path in (x for x in file_paths if x.endswith('.py')): - if reindent.check(path): + if reindent.check(os.path.join(SRCDIR, path)): fixed.append(path) return fixed @@ -85,10 +89,11 @@ """Report if any C files """ fixed = [] for path in file_paths: - with open(path, 'r') as f: + abspath = os.path.join(SRCDIR, path) + with open(abspath, 'r') as f: if '\t' not in f.read(): continue - untabify.process(path, 8, verbose=False) + untabify.process(abspath, 8, verbose=False) fixed.append(path) return fixed @@ -99,13 +104,14 @@ def normalize_docs_whitespace(file_paths): fixed = [] for path in file_paths: + abspath = os.path.join(SRCDIR, path) try: - with open(path, 'rb') as f: + with open(abspath, 'rb') as f: lines = f.readlines() new_lines = [ws_re.sub(br'\1', line) for line in lines] if new_lines != lines: - shutil.copyfile(path, path + '.bak') - with open(path, 'wb') as f: + shutil.copyfile(abspath, abspath + '.bak') + with open(abspath, 'wb') as f: f.writelines(new_lines) fixed.append(path) except Exception as err: -- Repository URL: http://hg.python.org/cpython

1 0

cpython (merge default -> default): Branch merge
by eric.araujo 01 Aug '11

01 Aug '11

http://hg.python.org/cpython/rev/abd8380c15e2 changeset: 71670:abd8380c15e2 parent: 71662:2d69900c0820 parent: 71668:7ee8f413188e user: Éric Araujo <merwok(a)netwok.org> date: Mon Aug 01 14:44:17 2011 +0200 summary: Branch merge files: Doc/distutils/sourcedist.rst | 25 +++++-- Lib/distutils/command/sdist.py | 48 ++++++++----- Lib/distutils/tests/test_sdist.py | 39 +++++++++- Lib/lib2to3/tests/test_refactor.py | 18 +++-- Lib/packaging/tests/test_database.py | 54 +++++++++------ Misc/ACKS | 2 + Misc/NEWS | 9 ++ Tools/scripts/patchcheck.py | 31 ++++---- 8 files changed, 148 insertions(+), 78 deletions(-) diff --git a/Doc/distutils/sourcedist.rst b/Doc/distutils/sourcedist.rst --- a/Doc/distutils/sourcedist.rst +++ b/Doc/distutils/sourcedist.rst @@ -103,10 +103,20 @@ :file:`MANIFEST`, you must specify everything: the default set of files described above does not apply in this case. -.. versionadded:: 3.1 +.. versionchanged:: 3.1 + An existing generated :file:`MANIFEST` will be regenerated without + :command:`sdist` comparing its modification time to the one of + :file:`MANIFEST.in` or :file:`setup.py`. + +.. versionchanged:: 3.1.3 :file:`MANIFEST` files start with a comment indicating they are generated. Files without this comment are not overwritten or removed. +.. versionchanged:: 3.2.2 + :command:`sdist` will read a :file:`MANIFEST` file if no :file:`MANIFEST.in` + exists, like it used to do. + + The manifest template has one command per line, where each command specifies a set of files to include or exclude from the source distribution. For an example, again we turn to the Distutils' own manifest template:: @@ -185,8 +195,12 @@ The normal course of operations for the :command:`sdist` command is as follows: -* if the manifest file, :file:`MANIFEST` doesn't exist, read :file:`MANIFEST.in` - and create the manifest +* if the manifest file (:file:`MANIFEST` by default) exists and the first line + does not have a comment indicating it is generated from :file:`MANIFEST.in`, + then it is used as is, unaltered + +* if the manifest file doesn't exist or has been previously automatically + generated, read :file:`MANIFEST.in` and create the manifest * if neither :file:`MANIFEST` nor :file:`MANIFEST.in` exist, create a manifest with just the default file set @@ -204,8 +218,3 @@ python setup.py sdist --manifest-only :option:`-o` is a shortcut for :option:`--manifest-only`. - -.. versionchanged:: 3.1 - An existing generated :file:`MANIFEST` will be regenerated without - :command:`sdist` comparing its modification time to the one of - :file:`MANIFEST.in` or :file:`setup.py`. diff --git a/Lib/distutils/command/sdist.py b/Lib/distutils/command/sdist.py --- a/Lib/distutils/command/sdist.py +++ b/Lib/distutils/command/sdist.py @@ -174,14 +174,20 @@ reading the manifest, or just using the default file set -- it all depends on the user's options. """ - # new behavior: + # new behavior when using a template: # the file list is recalculated everytime because # even if MANIFEST.in or setup.py are not changed # the user might have added some files in the tree that # need to be included. # - # This makes --force the default and only behavior. + # This makes --force the default and only behavior with templates. template_exists = os.path.isfile(self.template) + if not template_exists and self._manifest_is_not_generated(): + self.read_manifest() + self.filelist.sort() + self.filelist.remove_duplicates() + return + if not template_exists: self.warn(("manifest template '%s' does not exist " + "(using default file list)") % @@ -336,23 +342,28 @@ by 'add_defaults()' and 'read_template()') to the manifest file named by 'self.manifest'. """ - if os.path.isfile(self.manifest): - fp = open(self.manifest) - try: - first_line = fp.readline() - finally: - fp.close() - - if first_line != '# file GENERATED by distutils, do NOT edit\n': - log.info("not writing to manually maintained " - "manifest file '%s'" % self.manifest) - return + if self._manifest_is_not_generated(): + log.info("not writing to manually maintained " + "manifest file '%s'" % self.manifest) + return content = self.filelist.files[:] content.insert(0, '# file GENERATED by distutils, do NOT edit') self.execute(file_util.write_file, (self.manifest, content), "writing manifest file '%s'" % self.manifest) + def _manifest_is_not_generated(self): + # check for special comment used in 3.1.3 and higher + if not os.path.isfile(self.manifest): + return False + + fp = open(self.manifest) + try: + first_line = fp.readline() + finally: + fp.close() + return first_line != '# file GENERATED by distutils, do NOT edit\n' + def read_manifest(self): """Read the manifest file (named by 'self.manifest') and use it to fill in 'self.filelist', the list of files to include in the source @@ -360,12 +371,11 @@ """ log.info("reading manifest file '%s'", self.manifest) manifest = open(self.manifest) - while True: - line = manifest.readline() - if line == '': # end of file - break - if line[-1] == '\n': - line = line[0:-1] + for line in manifest: + # ignore comments and blank lines + line = line.strip() + if line.startswith('#') or not line: + continue self.filelist.append(line) manifest.close() diff --git a/Lib/distutils/tests/test_sdist.py b/Lib/distutils/tests/test_sdist.py --- a/Lib/distutils/tests/test_sdist.py +++ b/Lib/distutils/tests/test_sdist.py @@ -1,21 +1,19 @@ """Tests for distutils.command.sdist.""" import os +import tarfile import unittest -import shutil +import warnings import zipfile from os.path import join -import sys -import tempfile -import warnings +from textwrap import dedent from test.support import captured_stdout, check_warnings, run_unittest from distutils.command.sdist import sdist, show_formats from distutils.core import Distribution from distutils.tests.test_config import PyPIRCCommandTestCase -from distutils.errors import DistutilsExecError, DistutilsOptionError +from distutils.errors import DistutilsOptionError from distutils.spawn import find_executable -from distutils.tests import support from distutils.log import WARN from distutils.archive_util import ARCHIVE_FORMATS @@ -346,13 +344,33 @@ self.assertEqual(manifest[0], '# file GENERATED by distutils, do NOT edit') + @unittest.skipUnless(ZLIB_SUPPORT, "Need zlib support to run") + def test_manifest_comments(self): + # make sure comments don't cause exceptions or wrong includes + contents = dedent("""\ + # bad.py + #bad.py + good.py + """) + dist, cmd = self.get_cmd() + cmd.ensure_finalized() + self.write_file((self.tmp_dir, cmd.manifest), contents) + self.write_file((self.tmp_dir, 'good.py'), '# pick me!') + self.write_file((self.tmp_dir, 'bad.py'), "# don't pick me!") + self.write_file((self.tmp_dir, '#bad.py'), "# don't pick me!") + cmd.run() + self.assertEqual(cmd.filelist.files, ['good.py']) + @unittest.skipUnless(ZLIB_SUPPORT, 'Need zlib support to run') def test_manual_manifest(self): # check that a MANIFEST without a marker is left alone dist, cmd = self.get_cmd() cmd.ensure_finalized() self.write_file((self.tmp_dir, cmd.manifest), 'README.manual') + self.write_file((self.tmp_dir, 'README.manual'), + 'This project maintains its MANIFEST file itself.') cmd.run() + self.assertEqual(cmd.filelist.files, ['README.manual']) f = open(cmd.manifest) try: @@ -363,6 +381,15 @@ self.assertEqual(manifest, ['README.manual']) + archive_name = join(self.tmp_dir, 'dist', 'fake-1.0.tar.gz') + archive = tarfile.open(archive_name) + try: + filenames = [tarinfo.name for tarinfo in archive] + finally: + archive.close() + self.assertEqual(sorted(filenames), ['fake-1.0', 'fake-1.0/PKG-INFO', + 'fake-1.0/README.manual']) + def test_suite(): return unittest.makeSuite(SDistTestCase) diff --git a/Lib/lib2to3/tests/test_refactor.py b/Lib/lib2to3/tests/test_refactor.py --- a/Lib/lib2to3/tests/test_refactor.py +++ b/Lib/lib2to3/tests/test_refactor.py @@ -177,22 +177,26 @@ self.assertEqual(results, expected) def check_file_refactoring(self, test_file, fixers=_2TO3_FIXERS): + tmpdir = tempfile.mkdtemp(prefix="2to3-test_refactor") + self.addCleanup(shutil.rmtree, tmpdir) + # make a copy of the tested file that we can write to + shutil.copy(test_file, tmpdir) + test_file = os.path.join(tmpdir, os.path.basename(test_file)) + os.chmod(test_file, 0o644) + def read_file(): with open(test_file, "rb") as fp: return fp.read() + old_contents = read_file() rt = self.rt(fixers=fixers) rt.refactor_file(test_file) self.assertEqual(old_contents, read_file()) - try: - rt.refactor_file(test_file, True) - new_contents = read_file() - self.assertNotEqual(old_contents, new_contents) - finally: - with open(test_file, "wb") as fp: - fp.write(old_contents) + rt.refactor_file(test_file, True) + new_contents = read_file() + self.assertNotEqual(old_contents, new_contents) return new_contents def test_refactor_file(self): diff --git a/Lib/packaging/tests/test_database.py b/Lib/packaging/tests/test_database.py --- a/Lib/packaging/tests/test_database.py +++ b/Lib/packaging/tests/test_database.py @@ -39,20 +39,40 @@ return [path, digest, size] -class CommonDistributionTests: +class FakeDistsMixin: + + def setUp(self): + super(FakeDistsMixin, self).setUp() + self.addCleanup(enable_cache) + disable_cache() + + # make a copy that we can write into for our fake installed + # distributions + tmpdir = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, tmpdir) + self.fake_dists_path = os.path.join(tmpdir, 'fake_dists') + fake_dists_src = os.path.abspath( + os.path.join(os.path.dirname(__file__), 'fake_dists')) + shutil.copytree(fake_dists_src, self.fake_dists_path) + # XXX ugly workaround: revert copystat calls done by shutil behind our + # back (to avoid getting a read-only copy of a read-only file). we + # could pass a custom copy_function to change the mode of files, but + # shutil gives no control over the mode of directories :( + for root, dirs, files in os.walk(self.fake_dists_path): + os.chmod(root, 0o755) + for f in files: + os.chmod(os.path.join(root, f), 0o644) + for d in dirs: + os.chmod(os.path.join(root, d), 0o755) + + +class CommonDistributionTests(FakeDistsMixin): """Mixin used to test the interface common to both Distribution classes. Derived classes define cls, sample_dist, dirs and records. These attributes are used in test methods. See source code for details. """ - def setUp(self): - super(CommonDistributionTests, self).setUp() - self.addCleanup(enable_cache) - disable_cache() - self.fake_dists_path = os.path.abspath( - os.path.join(os.path.dirname(__file__), 'fake_dists')) - def test_instantiation(self): # check that useful attributes are here name, version, distdir = self.sample_dist @@ -110,6 +130,7 @@ self.records = {} for distinfo_dir in self.dirs: + record_file = os.path.join(distinfo_dir, 'RECORD') with open(record_file, 'w') as file: record_writer = csv.writer( @@ -138,12 +159,6 @@ record_data[path] = md5_, size self.records[distinfo_dir] = record_data - def tearDown(self): - for distinfo_dir in self.dirs: - record_file = os.path.join(distinfo_dir, 'RECORD') - open(record_file, 'wb').close() - super(TestDistribution, self).tearDown() - def test_instantiation(self): super(TestDistribution, self).test_instantiation() self.assertIsInstance(self.dist.requested, bool) @@ -252,20 +267,13 @@ class TestDatabase(support.LoggingCatcher, + FakeDistsMixin, unittest.TestCase): def setUp(self): super(TestDatabase, self).setUp() - disable_cache() - # Setup the path environment with our fake distributions - current_path = os.path.abspath(os.path.dirname(__file__)) - self.fake_dists_path = os.path.join(current_path, 'fake_dists') sys.path.insert(0, self.fake_dists_path) - - def tearDown(self): - sys.path.remove(self.fake_dists_path) - enable_cache() - super(TestDatabase, self).tearDown() + self.addCleanup(sys.path.remove, self.fake_dists_path) def test_distinfo_dirname(self): # Given a name and a version, we expect the distinfo_dirname function diff --git a/Misc/ACKS b/Misc/ACKS --- a/Misc/ACKS +++ b/Misc/ACKS @@ -228,6 +228,7 @@ Arnaud Delobelle Konrad Delong Erik Demaine +John Dennis Roger Dev Raghuram Devarakonda Caleb Deveraux @@ -931,6 +932,7 @@ Tobias Thelen James Thomas Robin Thomas +Stephen Thorne Jeremy Thurgood Eric Tiedemann July Tikhonov diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -249,6 +249,9 @@ Library ------- +- Issues #11104, #8688: Fix the behavior of distutils' sdist command with + manually-maintained MANIFEST files. + - Issue #11281: smtplib.STMP gets source_address parameter, which adds the ability to bind to specific source address on a machine with multiple interfaces. Patch by Paulo Scardine. @@ -1144,6 +1147,12 @@ Tests ----- +- Issue #12331: The test suite for the packaging module can now run from an + installed Python. + +- Issue #12331: The test suite for lib2to3 can now run from an installed + Python. + - Issue #12626: In regrtest, allow to filter tests using a glob filter with the ``-m`` (or ``--match``) option. This works with all test cases using the unittest module. This is useful with long test suites diff --git a/Tools/scripts/patchcheck.py b/Tools/scripts/patchcheck.py --- a/Tools/scripts/patchcheck.py +++ b/Tools/scripts/patchcheck.py @@ -4,11 +4,15 @@ import shutil import os.path import subprocess +import sysconfig import reindent import untabify +SRCDIR = sysconfig.get_config_var('srcdir') + + def n_files_str(count): """Return 'N file(s)' with the proper plurality on 'file'.""" return "{} file{}".format(count, "s" if count != 1 else "") @@ -35,18 +39,13 @@ @status("Getting the list of files that have been added/changed", info=lambda x: n_files_str(len(x))) def changed_files(): - """Get the list of changed or added files from the VCS.""" - if os.path.isdir('.hg'): - cmd = 'hg status --added --modified --no-status' - else: + """Get the list of changed or added files from Mercurial.""" + if not os.path.isdir(os.path.join(SRCDIR, '.hg')): sys.exit('need a checkout to get modified files') - st = subprocess.Popen(cmd.split(), stdout=subprocess.PIPE) - try: - st.wait() + cmd = 'hg status --added --modified --no-status' + with subprocess.Popen(cmd.split(), stdout=subprocess.PIPE) as st: return [x.decode().rstrip() for x in st.stdout] - finally: - st.stdout.close() def report_modified_files(file_paths): @@ -65,7 +64,7 @@ """Make sure that the whitespace for .py files have been normalized.""" reindent.makebackup = False # No need to create backups. fixed = [path for path in file_paths if path.endswith('.py') and - reindent.check(path)] + reindent.check(os.path.join(SRCDIR, path))] return fixed @@ -74,10 +73,11 @@ """Report if any C files """ fixed = [] for path in file_paths: - with open(path, 'r') as f: + abspath = os.path.join(SRCDIR, path) + with open(abspath, 'r') as f: if '\t' not in f.read(): continue - untabify.process(path, 8, verbose=False) + untabify.process(abspath, 8, verbose=False) fixed.append(path) return fixed @@ -88,13 +88,14 @@ def normalize_docs_whitespace(file_paths): fixed = [] for path in file_paths: + abspath = os.path.join(SRCDIR, path) try: - with open(path, 'rb') as f: + with open(abspath, 'rb') as f: lines = f.readlines() new_lines = [ws_re.sub(br'\1', line) for line in lines] if new_lines != lines: - shutil.copyfile(path, path + '.bak') - with open(path, 'wb') as f: + shutil.copyfile(abspath, abspath + '.bak') + with open(abspath, 'wb') as f: f.writelines(new_lines) fixed.append(path) except Exception as err: -- Repository URL: http://hg.python.org/cpython

1 0

cpython: Stop trying to write into the stdlib during packaging tests (#12331).
by eric.araujo 01 Aug '11

01 Aug '11

http://hg.python.org/cpython/rev/7ee8f413188e changeset: 71668:7ee8f413188e user: Éric Araujo <merwok(a)netwok.org> date: Sun Jul 31 20:47:47 2011 +0200 summary: Stop trying to write into the stdlib during packaging tests (#12331). This prevents tests from failing when run from a Python installed in a read-only directory. The code is a bit uglier; shutil.copytree calls copystat on directories behind our back, so I had to add an os.walk with os.chmod (*and* os.path.join!) calls. shutil, I am disappoint. This changeset is dedicated to the hundreds of neurons that were lost while I was debugging this on an otherwise fine afternoon. files: Lib/packaging/tests/test_database.py | 54 +++++++++------ Misc/NEWS | 3 + 2 files changed, 34 insertions(+), 23 deletions(-) diff --git a/Lib/packaging/tests/test_database.py b/Lib/packaging/tests/test_database.py --- a/Lib/packaging/tests/test_database.py +++ b/Lib/packaging/tests/test_database.py @@ -39,20 +39,40 @@ return [path, digest, size] -class CommonDistributionTests: +class FakeDistsMixin: + + def setUp(self): + super(FakeDistsMixin, self).setUp() + self.addCleanup(enable_cache) + disable_cache() + + # make a copy that we can write into for our fake installed + # distributions + tmpdir = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, tmpdir) + self.fake_dists_path = os.path.join(tmpdir, 'fake_dists') + fake_dists_src = os.path.abspath( + os.path.join(os.path.dirname(__file__), 'fake_dists')) + shutil.copytree(fake_dists_src, self.fake_dists_path) + # XXX ugly workaround: revert copystat calls done by shutil behind our + # back (to avoid getting a read-only copy of a read-only file). we + # could pass a custom copy_function to change the mode of files, but + # shutil gives no control over the mode of directories :( + for root, dirs, files in os.walk(self.fake_dists_path): + os.chmod(root, 0o755) + for f in files: + os.chmod(os.path.join(root, f), 0o644) + for d in dirs: + os.chmod(os.path.join(root, d), 0o755) + + +class CommonDistributionTests(FakeDistsMixin): """Mixin used to test the interface common to both Distribution classes. Derived classes define cls, sample_dist, dirs and records. These attributes are used in test methods. See source code for details. """ - def setUp(self): - super(CommonDistributionTests, self).setUp() - self.addCleanup(enable_cache) - disable_cache() - self.fake_dists_path = os.path.abspath( - os.path.join(os.path.dirname(__file__), 'fake_dists')) - def test_instantiation(self): # check that useful attributes are here name, version, distdir = self.sample_dist @@ -110,6 +130,7 @@ self.records = {} for distinfo_dir in self.dirs: + record_file = os.path.join(distinfo_dir, 'RECORD') with open(record_file, 'w') as file: record_writer = csv.writer( @@ -138,12 +159,6 @@ record_data[path] = md5_, size self.records[distinfo_dir] = record_data - def tearDown(self): - for distinfo_dir in self.dirs: - record_file = os.path.join(distinfo_dir, 'RECORD') - open(record_file, 'wb').close() - super(TestDistribution, self).tearDown() - def test_instantiation(self): super(TestDistribution, self).test_instantiation() self.assertIsInstance(self.dist.requested, bool) @@ -252,20 +267,13 @@ class TestDatabase(support.LoggingCatcher, + FakeDistsMixin, unittest.TestCase): def setUp(self): super(TestDatabase, self).setUp() - disable_cache() - # Setup the path environment with our fake distributions - current_path = os.path.abspath(os.path.dirname(__file__)) - self.fake_dists_path = os.path.join(current_path, 'fake_dists') sys.path.insert(0, self.fake_dists_path) - - def tearDown(self): - sys.path.remove(self.fake_dists_path) - enable_cache() - super(TestDatabase, self).tearDown() + self.addCleanup(sys.path.remove, self.fake_dists_path) def test_distinfo_dirname(self): # Given a name and a version, we expect the distinfo_dirname function diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -1147,6 +1147,9 @@ Tests ----- +- Issue #12331: The test suite for the packaging module can now run from an + installed Python. + - Issue #12331: The test suite for lib2to3 can now run from an installed Python. -- Repository URL: http://hg.python.org/cpython

1 0

cpython: Small cleanup
by eric.araujo 01 Aug '11

01 Aug '11

http://hg.python.org/cpython/rev/d72e04ff0f97 changeset: 71667:d72e04ff0f97 user: Éric Araujo <merwok(a)netwok.org> date: Sun Jul 31 18:41:25 2011 +0200 summary: Small cleanup files: Tools/scripts/patchcheck.py | 13 ++++--------- 1 files changed, 4 insertions(+), 9 deletions(-) diff --git a/Tools/scripts/patchcheck.py b/Tools/scripts/patchcheck.py --- a/Tools/scripts/patchcheck.py +++ b/Tools/scripts/patchcheck.py @@ -39,18 +39,13 @@ @status("Getting the list of files that have been added/changed", info=lambda x: n_files_str(len(x))) def changed_files(): - """Get the list of changed or added files from the VCS.""" - if os.path.isdir(os.path.join(SRCDIR, '.hg')): - cmd = 'hg status --added --modified --no-status' - else: + """Get the list of changed or added files from Mercurial.""" + if not os.path.isdir(os.path.join(SRCDIR, '.hg')): sys.exit('need a checkout to get modified files') - st = subprocess.Popen(cmd.split(), stdout=subprocess.PIPE) - try: - st.wait() + cmd = 'hg status --added --modified --no-status' + with subprocess.Popen(cmd.split(), stdout=subprocess.PIPE) as st: return [x.decode().rstrip() for x in st.stdout] - finally: - st.stdout.close() def report_modified_files(file_paths): -- Repository URL: http://hg.python.org/cpython

1 0

cpython (3.2): Stop trying to write into the stdlib during lib2to3 tests (#12331).
by eric.araujo 01 Aug '11

01 Aug '11

http://hg.python.org/cpython/rev/2b5a0c4e052b changeset: 71665:2b5a0c4e052b branch: 3.2 user: Éric Araujo <merwok(a)netwok.org> date: Sun Jul 31 17:58:46 2011 +0200 summary: Stop trying to write into the stdlib during lib2to3 tests (#12331). This prevents tests from failing when run from a Python installed in a read-only directory. files: Lib/lib2to3/tests/test_refactor.py | 18 +++++++++++------- Misc/NEWS | 3 +++ 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/Lib/lib2to3/tests/test_refactor.py b/Lib/lib2to3/tests/test_refactor.py --- a/Lib/lib2to3/tests/test_refactor.py +++ b/Lib/lib2to3/tests/test_refactor.py @@ -177,22 +177,26 @@ self.assertEqual(results, expected) def check_file_refactoring(self, test_file, fixers=_2TO3_FIXERS): + tmpdir = tempfile.mkdtemp(prefix="2to3-test_refactor") + self.addCleanup(shutil.rmtree, tmpdir) + # make a copy of the tested file that we can write to + shutil.copy(test_file, tmpdir) + test_file = os.path.join(tmpdir, os.path.basename(test_file)) + os.chmod(test_file, 0o644) + def read_file(): with open(test_file, "rb") as fp: return fp.read() + old_contents = read_file() rt = self.rt(fixers=fixers) rt.refactor_file(test_file) self.assertEqual(old_contents, read_file()) - try: - rt.refactor_file(test_file, True) - new_contents = read_file() - self.assertNotEqual(old_contents, new_contents) - finally: - with open(test_file, "wb") as fp: - fp.write(old_contents) + rt.refactor_file(test_file, True) + new_contents = read_file() + self.assertNotEqual(old_contents, new_contents) return new_contents def test_refactor_file(self): diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -140,6 +140,9 @@ Tests ----- +- Issue #12331: The test suite for lib2to3 can now run from an installed + Python. + - Issue #12626: In regrtest, allow to filter tests using a glob filter with the ``-m`` (or ``--match``) option. This works with all test cases using the unittest module. This is useful with long test suites -- Repository URL: http://hg.python.org/cpython

1 0