Read the existing MANIFEST.in file for files to ignore.

CHANGES.rst

@@ -5,6 +5,8 @@ Changelog
 0.17 (unreleased)
 -----------------
 
+* Read the existing MANIFEST.in file for files to ignore.
+
 
 0.16 (2013-10-01)
 -----------------

check_manifest.py

@@ -350,6 +350,12 @@ def add_directories(names):
     '*.mo',
 ]
 
+IGNORE_REGEXPS = [
+    # Regular expressions for filename to ignore.  This is useful for
+    # filename patterns where the '*' part must not search in
+    # directories.
+    ]
+
 WARN_ABOUT_FILES_IN_VCS = [
     # generated files should not be committed into the VCS
     'PKG-INFO',
@@ -394,15 +400,77 @@ def read_config():
         IGNORE.extend(p for p in patterns if p)
 
 
+def read_manifest():
+    """Read existing configuration from MANIFEST.in.
+
+    We use that to ignore anything the MANIFEST.in ignores.
+    """
+    # XXX modifies global state, which is kind of evil
+    if not os.path.isfile('MANIFEST.in'):
+        return
+    contents = open('MANIFEST.in').read()
+    ignore, ignore_regexps = _get_ignore_from_manifest(contents)
+    IGNORE.extend(ignore)
+    IGNORE_REGEXPS.extend(ignore_regexps)
+
+
+def _get_ignore_from_manifest(contents):
+    """Gather the various ignore patterns from MANIFEST.in.
+
+    'contents' should be a string, which may contain newlines.
+
+    Returns a list of standard ignore patterns and a list of regular
+    expressions to ignore.
+    """
+    ignore = []
+    ignore_regexps = []
+    for line in contents.splitlines():
+        if line.startswith('exclude '):
+            # An exclude of 'dirname/*css' can match 'dirname/foo.css'
+            # but not 'dirname/subdir/bar.css'.  We need a regular
+            # expression for that.
+            rest = line[len('exclude '):].strip().split()
+            for pat in rest:
+                if '*' in pat:
+                    pat = pat.replace('*', '[^/]*')
+                    ignore_regexps.append(pat)
+                else:
+                    # No need for special handling.
+                    ignore.append(pat)
+        elif line.startswith('global-exclude '):
+            rest = line[len('global-exclude '):].strip().split()
+            ignore.extend(rest)
+        elif line.startswith('recursive-exclude '):
+            rest = line[len('recursive-exclude '):].strip()
+            dirname, patterns = rest.split(' ', 1)
+            for pattern in patterns.strip().split():
+                ignore.append(dirname + os.path.sep + pattern)
+        elif line.startswith('prune '):
+            dirname = line[len('prune '):].strip()
+            ignore.append(dirname)
+            ignore.append(dirname + os.path.sep + '*')
+    return ignore, ignore_regexps
+
+
 def file_matches(filename, patterns):
     """Does this filename match any of the patterns?"""
     return any(fnmatch.fnmatch(filename, pat) for pat in patterns)
 
 
+def file_matches_regexps(filename, patterns):
+    """Does this filename match any of the regular expressions?"""
+    return any(re.match(pat, filename) for pat in patterns)
+
+
 def strip_sdist_extras(filelist):
-    """Strip generated files that are only present in source distributions."""
+    """Strip generated files that are only present in source distributions.
+
+    We also strip files that are ignored for other reasons, like
+    command line arguments, setup.cfg rules or MANIFEST.in rules.
+    """
     return [name for name in filelist
-            if not file_matches(name, IGNORE)]
+            if not file_matches(name, IGNORE)
+            and not file_matches_regexps(name, IGNORE_REGEXPS)]
 
 
 def find_bad_ideas(filelist):
@@ -451,6 +519,7 @@ def check_manifest(source_tree='.', create=False, update=False,
         if not is_package(source_tree):
             raise Failure('This is not a Python project (no setup.py).')
         read_config()
+        read_manifest()
         info_begin("listing source files under version control")
         all_source_files = sorted(get_vcs_files())
         source_files = strip_sdist_extras(all_source_files)

tests.py

@@ -82,6 +82,69 @@ def test_strip_sdist_extras(self):
         ]
         self.assertEqual(strip_sdist_extras(filelist), expected)
 
+    def test_strip_sdist_extras_with_manifest(self):
+        import check_manifest
+        from check_manifest import strip_sdist_extras
+        from check_manifest import _get_ignore_from_manifest as parse
+        orig_ignore = check_manifest.IGNORE
+        orig_ignore_regexps = check_manifest.IGNORE_REGEXPS
+        manifest_in = """
+graft src
+exclude *.cfg
+global-exclude *.mo
+prune src/dump
+recursive-exclude src/zope *.sh
+"""
+        filelist = [
+            '.gitignore',
+            'setup.py',
+            'setup.cfg',
+            'MANIFEST.in',
+            'README.txt',
+            'src',
+            'src/helper.sh',
+            'src/dump',
+            'src/dump/__init__.py',
+            'src/zope',
+            'src/zope/__init__.py',
+            'src/zope/zopehelper.sh',
+            'src/zope/foo',
+            'src/zope/foo/__init__.py',
+            'src/zope/foo/language.po',
+            'src/zope/foo/language.mo',
+            'src/zope/foo/config.cfg',
+            'src/zope/foo/foohelper.sh',
+            'src/zope.foo.egg-info',
+            'src/zope.foo.egg-info/SOURCES.txt',
+        ]
+        expected = [
+            'setup.py',
+            'MANIFEST.in',
+            'README.txt',
+            'src',
+            'src/helper.sh',
+            'src/zope',
+            'src/zope/__init__.py',
+            'src/zope/foo',
+            'src/zope/foo/__init__.py',
+            'src/zope/foo/language.po',
+            'src/zope/foo/config.cfg',
+        ]
+
+        # This will change the definitions.
+        try:
+            # This is normally done in read_manifest:
+            ignore, ignore_regexps = parse(manifest_in)
+            check_manifest.IGNORE.extend(ignore)
+            check_manifest.IGNORE_REGEXPS.extend(ignore_regexps)
+            # Filter the file list.
+            result = strip_sdist_extras(filelist)
+        finally:
+            # Restore the original definitions
+            check_manifest.IGNORE = orig_ignore
+            check_manifest.IGNORE_REGEXPS = orig_ignore_regexps
+        self.assertEqual(result, expected)
+
     def test_find_bad_ideas(self):
         from check_manifest import find_bad_ideas
         filelist = [
@@ -137,6 +200,70 @@ def test_find_suggestions_generic_fallback_rules(self):
         self.assertEqual(find_suggestions(['src/id-lang.map']),
                          (['recursive-include src *.map'], []))
 
+    def test_get_ignore_from_manifest(self):
+        from check_manifest import _get_ignore_from_manifest as parse
+        # The return value is a tuple with two lists:
+        # ([<list of filename ignores>], [<list of regular expressions>])
+        self.assertEqual(parse(''),
+                         ([], []))
+        self.assertEqual(parse('      \n        '),
+                         ([], []))
+        self.assertEqual(parse('exclude *.cfg'),
+                         ([], ['[^/]*.cfg']))
+        self.assertEqual(parse('#exclude *.cfg'),
+                         ([], []))
+        self.assertEqual(parse('exclude          *.cfg'),
+                         ([], ['[^/]*.cfg']))
+        self.assertEqual(parse('exclude *.cfg foo.*   bar.txt'),
+                         (['bar.txt'], ['[^/]*.cfg', 'foo.[^/]*']))
+        self.assertEqual(parse('exclude some/directory/*.cfg'),
+                         ([], ['some/directory/[^/]*.cfg']))
+        self.assertEqual(parse('include *.cfg'),
+                         ([], []))
+        self.assertEqual(parse('global-exclude *.pyc'),
+                         (['*.pyc'], []))
+        self.assertEqual(parse('global-exclude *.pyc *.sh'),
+                         (['*.pyc', '*.sh'], []))
+        self.assertEqual(parse('recursive-exclude dir *.pyc'),
+                         (['dir/*.pyc'], []))
+        self.assertEqual(parse('recursive-exclude dir *.pyc *.sh'),
+                         (['dir/*.pyc', 'dir/*.sh'], []))
+        self.assertEqual(parse('prune dir'),
+                         (['dir', 'dir/*'], []))
+        text = """
+#exclude *.01
+exclude *.02
+exclude *.03 04.*   bar.txt
+exclude          *.05
+exclude some/directory/*.cfg
+global-exclude *.10 *.11
+global-exclude *.12
+include *.20
+prune 30
+recursive-exclude    40      *.41
+recursive-exclude 42 *.43 44.*
+"""
+        self.assertEqual(
+            parse(text),
+            ([
+             'bar.txt',
+             '*.10',
+             '*.11',
+             '*.12',
+             '30',
+             '30/*',
+             '40/*.41',
+             '42/*.43',
+             '42/44.*',
+             ],
+             [
+             '[^/]*.02',
+             '[^/]*.03',
+             '04.[^/]*',
+             '[^/]*.05',
+             'some/directory/[^/]*.cfg',
+             ]))
+
 
 class VCSMixin(object):