http://hg.python.org/distutils2/rev/4d6a9f287543 changeset: 1289:4d6a9f287543 user: Éric Araujo <merwok@netwok.org> date: Mon Feb 27 11:52:34 2012 +0100 summary: Fix bugs with MANIFEST.in parsing on Windows (#6884). These regex changes fix a number of issues on Windows: - #6884: impossible to include a file starting with 'build' - #9691 and #14004: sdist includes too many files - #13193: test_manifest failures Thanks to Nadeem Vawda for his help. files: CHANGES.txt | 1 + CONTRIBUTORS.txt | 1 + distutils2/manifest.py | 18 +- distutils2/tests/fixer/__init__.py | 1 + distutils2/tests/test_manifest.py | 138 ++++++++++++---- 5 files changed, 119 insertions(+), 40 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt --- a/CHANGES.txt +++ b/CHANGES.txt @@ -167,6 +167,7 @@ - #1326113: build_ext now correctly parses multiple values given to the --libraries option [éric] - #13974: add test for util.set_platform [tshepang] +- #6884: Fix MANIFEST.in parsing bugs on Windows [éric, nadeem] 1.0a3 - 2010-10-08 diff --git a/CONTRIBUTORS.txt b/CONTRIBUTORS.txt --- a/CONTRIBUTORS.txt +++ b/CONTRIBUTORS.txt @@ -69,3 +69,4 @@ - Vinay Sajip - Victor Stinner - Alexandre Vassalotti +- Nadeem Vawda diff --git a/distutils2/manifest.py b/distutils2/manifest.py --- a/distutils2/manifest.py +++ b/distutils2/manifest.py @@ -278,6 +278,7 @@ Return True if files are found. """ + # XXX docstring lying about what the special chars are? files_found = False pattern_re = _translate_pattern(pattern, anchor, prefix, is_regex) @@ -341,10 +342,14 @@ # IMHO is wrong -- '?' and '*' aren't supposed to match slash in Unix, # and by extension they shouldn't match such "special characters" under # any OS. So change all non-escaped dots in the RE to match any - # character except the special characters. - # XXX currently the "special characters" are just slash -- i.e. this is - # Unix-only. - pattern_re = re.sub(r'((?<!\\)(\\\\)*)\.', r'\1[^/]', pattern_re) + # character except the special characters (currently: just os.sep). + sep = os.sep + if os.sep == '\\': + # we're using a regex to manipulate a regex, so we need + # to escape the backslash twice + sep = r'\\\\' + escaped = r'\1[^%s]' % sep + pattern_re = re.sub(r'((?<!\\)(\\\\)*)\.', escaped, pattern_re) if pattern_re.endswith('$'): # make it look like fnmatch in Python 2.6 and newer @@ -376,7 +381,10 @@ # ditch end of pattern character empty_pattern = _glob_to_re('') prefix_re = _glob_to_re(prefix)[:-len(empty_pattern)] - pattern_re = "^" + os.path.join(prefix_re, ".*" + pattern_re) + sep = os.sep + if os.sep == '\\': + sep = r'\\' + pattern_re = "^" + sep.join((prefix_re, ".*" + pattern_re)) else: # no prefix -- respect anchor flag if anchor: pattern_re = "^" + pattern_re diff --git a/distutils2/tests/fixer/__init__.py b/distutils2/tests/fixer/__init__.py --- a/distutils2/tests/fixer/__init__.py +++ b/distutils2/tests/fixer/__init__.py @@ -0,0 +1,1 @@ +"""Example fixers used in tests.""" diff --git a/distutils2/tests/test_manifest.py b/distutils2/tests/test_manifest.py --- a/distutils2/tests/test_manifest.py +++ b/distutils2/tests/test_manifest.py @@ -7,7 +7,22 @@ from distutils2.tests import unittest, support -_MANIFEST = """\ +MANIFEST_IN = """\ +include ok +include xo +exclude xo +include foo.tmp +include buildout.cfg +global-include *.x +global-include *.txt +global-exclude *.tmp +recursive-include f *.oo +recursive-exclude global *.x +graft dir +prune dir3 +""" + +MANIFEST_IN_2 = """\ recursive-include foo *.py # ok # nothing here @@ -17,12 +32,17 @@ *.dat *.txt """ -_MANIFEST2 = """\ +MANIFEST_IN_3 = """\ README file1 """ +def make_local_path(s): + """Converts '/' in a string to os.sep""" + return s.replace('/', os.sep) + + class ManifestTestCase(support.TempdirManager, support.LoggingCatcher, unittest.TestCase): @@ -38,7 +58,7 @@ MANIFEST = os.path.join(tmpdir, 'MANIFEST.in') f = open(MANIFEST, 'w') try: - f.write(_MANIFEST) + f.write(MANIFEST_IN_2) finally: f.close() @@ -69,27 +89,74 @@ os.chdir(tmpdir) self.write_file('README', 'xxx') self.write_file('file1', 'xxx') - content = StringIO(_MANIFEST2) + content = StringIO(MANIFEST_IN_3) manifest = Manifest() manifest.read_template(content) self.assertEqual(['README', 'file1'], manifest.files) def test_glob_to_re(self): - # simple cases - self.assertEqual(_glob_to_re('foo*'), 'foo[^/]*\\Z(?ms)') - self.assertEqual(_glob_to_re('foo?'), 'foo[^/]\\Z(?ms)') - self.assertEqual(_glob_to_re('foo??'), 'foo[^/][^/]\\Z(?ms)') + sep = os.sep + if os.sep == '\\': + sep = r'\\' - # special cases - self.assertEqual(_glob_to_re(r'foo\\*'), r'foo\\\\[^/]*\Z(?ms)') - self.assertEqual(_glob_to_re(r'foo\\\*'), r'foo\\\\\\[^/]*\Z(?ms)') - self.assertEqual(_glob_to_re('foo????'), r'foo[^/][^/][^/][^/]\Z(?ms)') - self.assertEqual(_glob_to_re(r'foo\\??'), r'foo\\\\[^/][^/]\Z(?ms)') + for glob, regex in ( + # simple cases + ('foo*', r'foo[^%(sep)s]*\Z(?ms)'), + ('foo?', r'foo[^%(sep)s]\Z(?ms)'), + ('foo??', r'foo[^%(sep)s][^%(sep)s]\Z(?ms)'), + # special cases + (r'foo\\*', r'foo\\\\[^%(sep)s]*\Z(?ms)'), + (r'foo\\\*', r'foo\\\\\\[^%(sep)s]*\Z(?ms)'), + ('foo????', r'foo[^%(sep)s][^%(sep)s][^%(sep)s][^%(sep)s]\Z(?ms)'), + (r'foo\\??', r'foo\\\\[^%(sep)s][^%(sep)s]\Z(?ms)'), + ): + regex = regex % {'sep': sep} + self.assertEqual(_glob_to_re(glob), regex) + + def test_process_template_line(self): + # testing all MANIFEST.in template patterns + manifest = Manifest() + l = make_local_path + + # simulated file list + manifest.allfiles = ['foo.tmp', 'ok', 'xo', 'four.txt', + 'buildout.cfg', + # filelist does not filter out VCS directories, + # it's sdist that does + l('.hg/last-message.txt'), + l('global/one.txt'), + l('global/two.txt'), + l('global/files.x'), + l('global/here.tmp'), + l('f/o/f.oo'), + l('dir/graft-one'), + l('dir/dir2/graft2'), + l('dir3/ok'), + l('dir3/sub/ok.txt'), + ] + + for line in MANIFEST_IN.split('\n'): + if line.strip() == '': + continue + manifest._process_template_line(line) + + wanted = ['ok', + 'buildout.cfg', + 'four.txt', + l('.hg/last-message.txt'), + l('global/one.txt'), + l('global/two.txt'), + l('f/o/f.oo'), + l('dir/graft-one'), + l('dir/dir2/graft2'), + ] + + self.assertEqual(manifest.files, wanted) def test_remove_duplicates(self): manifest = Manifest() manifest.files = ['a', 'b', 'a', 'g', 'c', 'g'] - # files must be sorted beforehand + # files must be sorted beforehand (like sdist does) manifest.sort() manifest.remove_duplicates() self.assertEqual(manifest.files, ['a', 'b', 'c', 'g']) @@ -149,6 +216,7 @@ self.assertEqual(manifest.allfiles, ['a.py', 'b.txt']) def test_process_template(self): + l = make_local_path # invalid lines manifest = Manifest() for action in ('include', 'exclude', 'global-include', @@ -159,7 +227,7 @@ # implicit include manifest = Manifest() - manifest.allfiles = ['a.py', 'b.txt', 'd/c.py'] + manifest.allfiles = ['a.py', 'b.txt', l('d/c.py')] manifest._process_template_line('*.py') self.assertEqual(manifest.files, ['a.py']) @@ -167,7 +235,7 @@ # include manifest = Manifest() - manifest.allfiles = ['a.py', 'b.txt', 'd/c.py'] + manifest.allfiles = ['a.py', 'b.txt', l('d/c.py')] manifest._process_template_line('include *.py') self.assertEqual(manifest.files, ['a.py']) @@ -179,31 +247,31 @@ # exclude manifest = Manifest() - manifest.files = ['a.py', 'b.txt', 'd/c.py'] + manifest.files = ['a.py', 'b.txt', l('d/c.py')] manifest._process_template_line('exclude *.py') - self.assertEqual(manifest.files, ['b.txt', 'd/c.py']) + self.assertEqual(manifest.files, ['b.txt', l('d/c.py')]) self.assertNoWarnings() manifest._process_template_line('exclude *.rb') - self.assertEqual(manifest.files, ['b.txt', 'd/c.py']) + self.assertEqual(manifest.files, ['b.txt', l('d/c.py')]) self.assertWarnings() # global-include manifest = Manifest() - manifest.allfiles = ['a.py', 'b.txt', 'd/c.py'] + manifest.allfiles = ['a.py', 'b.txt', l('d/c.py')] manifest._process_template_line('global-include *.py') - self.assertEqual(manifest.files, ['a.py', 'd/c.py']) + self.assertEqual(manifest.files, ['a.py', l('d/c.py')]) self.assertNoWarnings() manifest._process_template_line('global-include *.rb') - self.assertEqual(manifest.files, ['a.py', 'd/c.py']) + self.assertEqual(manifest.files, ['a.py', l('d/c.py')]) self.assertWarnings() # global-exclude manifest = Manifest() - manifest.files = ['a.py', 'b.txt', 'd/c.py'] + manifest.files = ['a.py', 'b.txt', l('d/c.py')] manifest._process_template_line('global-exclude *.py') self.assertEqual(manifest.files, ['b.txt']) @@ -215,50 +283,50 @@ # recursive-include manifest = Manifest() - manifest.allfiles = ['a.py', 'd/b.py', 'd/c.txt', 'd/d/e.py'] + manifest.allfiles = ['a.py', l('d/b.py'), l('d/c.txt'), l('d/d/e.py')] manifest._process_template_line('recursive-include d *.py') - self.assertEqual(manifest.files, ['d/b.py', 'd/d/e.py']) + self.assertEqual(manifest.files, [l('d/b.py'), l('d/d/e.py')]) self.assertNoWarnings() manifest._process_template_line('recursive-include e *.py') - self.assertEqual(manifest.files, ['d/b.py', 'd/d/e.py']) + self.assertEqual(manifest.files, [l('d/b.py'), l('d/d/e.py')]) self.assertWarnings() # recursive-exclude manifest = Manifest() - manifest.files = ['a.py', 'd/b.py', 'd/c.txt', 'd/d/e.py'] + manifest.files = ['a.py', l('d/b.py'), l('d/c.txt'), l('d/d/e.py')] manifest._process_template_line('recursive-exclude d *.py') - self.assertEqual(manifest.files, ['a.py', 'd/c.txt']) + self.assertEqual(manifest.files, ['a.py', l('d/c.txt')]) self.assertNoWarnings() manifest._process_template_line('recursive-exclude e *.py') - self.assertEqual(manifest.files, ['a.py', 'd/c.txt']) + self.assertEqual(manifest.files, ['a.py', l('d/c.txt')]) self.assertWarnings() # graft manifest = Manifest() - manifest.allfiles = ['a.py', 'd/b.py', 'd/d/e.py', 'f/f.py'] + manifest.allfiles = ['a.py', l('d/b.py'), l('d/d/e.py'), l('f/f.py')] manifest._process_template_line('graft d') - self.assertEqual(manifest.files, ['d/b.py', 'd/d/e.py']) + self.assertEqual(manifest.files, [l('d/b.py'), l('d/d/e.py')]) self.assertNoWarnings() manifest._process_template_line('graft e') - self.assertEqual(manifest.files, ['d/b.py', 'd/d/e.py']) + self.assertEqual(manifest.files, [l('d/b.py'), l('d/d/e.py')]) self.assertWarnings() # prune manifest = Manifest() - manifest.files = ['a.py', 'd/b.py', 'd/d/e.py', 'f/f.py'] + manifest.files = ['a.py', l('d/b.py'), l('d/d/e.py'), l('f/f.py')] manifest._process_template_line('prune d') - self.assertEqual(manifest.files, ['a.py', 'f/f.py']) + self.assertEqual(manifest.files, ['a.py', l('f/f.py')]) self.assertNoWarnings() manifest._process_template_line('prune e') - self.assertEqual(manifest.files, ['a.py', 'f/f.py']) + self.assertEqual(manifest.files, ['a.py', l('f/f.py')]) self.assertWarnings() -- Repository URL: http://hg.python.org/distutils2