Add support for PEP 420 namespace packages to find_packages()

On Python 3.3+, `find_packages()` now considers any subdirectory of the
start directory that's not a regular package (i.e., that doesn't have an
`__init__.py`) to be a namespace package. Because this will often
include non-package directories, a new `include` argument has been added
to `find_packages()`.

`include` can make it easier to narrow which directories are considered
packages instead of having to specify numerous excludes. In
particular, it's an easy way to keep top-level, non-source directories
from being considered packages.

The other way this supports PEP 420 is by making sure `__pycache__`
directories are never added to the list of packages.

Refs issue #97

CHANGES.txt

@@ -23,6 +23,11 @@ CHANGES
   security vulnerabilities presented by use of tar archives in ez_setup.py.
   It also leverages the security features added to ZipFile.extract in Python 2.7.4.
 * Issue #65: Removed deprecated Features functionality.
+* Issue #97: ``find_packages()`` now supports PEP 420 namespace packages. It
+  does so by considering all subdirectories of the start directory that
+  aren't regular packages to be PEP 420 namespace packages (on Python 3.3+
+  only). Also, a list of packages to include can be now be specified; this
+  provides an easy way to narrow the list of candidate directories.
 
 ---
 2.3

setuptools/__init__.py

@@ -27,7 +27,7 @@ run_2to3_on_doctests = True
 # Standard package names for fixer packages
 lib2to3_fixer_packages = ['lib2to3.fixes']
 
-def find_packages(where='.', exclude=()):
+def find_packages(where='.', exclude=(), include=()):
     """Return a list all Python packages found within directory 'where'
 
     'where' should be supplied as a "cross-platform" (i.e. URL-style) path; it
@@ -35,7 +35,19 @@ def find_packages(where='.', exclude=()):
     sequence of package names to exclude; '*' can be used as a wildcard in the
     names, such that 'foo.*' will exclude all subpackages of 'foo' (but not
     'foo' itself).
+
+    'include' is a sequence of package names to include.  If it's specified,
+    only the named packages will be included.  If it's not specified, all found
+    packages will be included.  'include' can contain shell style wildcard
+    patterns just like 'exclude'.
+
+    The list of included packages is built up first and then any explicitly
+    excluded packages are removed from it.
+
     """
+    from fnmatch import fnmatchcase
+    include = list(include)
+    exclude = list(exclude) + ['ez_setup', '*__pycache__']
     out = []
     stack=[(convert_path(where), '')]
     while stack:
@@ -45,13 +57,18 @@ def find_packages(where='.', exclude=()):
             looks_like_package = (
                 '.' not in name
                 and os.path.isdir(fn)
-                and os.path.isfile(os.path.join(fn, '__init__.py'))
+                and (
+                    os.path.isfile(os.path.join(fn, '__init__.py'))
+                    or sys.version_info[:2] >= (3, 3)  # PEP 420
+                )
             )
             if looks_like_package:
-                out.append(prefix+name)
-                stack.append((fn, prefix+name+'.'))
-    for pat in list(exclude)+['ez_setup']:
-        from fnmatch import fnmatchcase
+                pkg_name = prefix + name
+                if (not include or
+                        any(fnmatchcase(pkg_name, pat) for pat in include)):
+                    out.append(pkg_name)
+                    stack.append((fn, pkg_name + '.'))
+    for pat in exclude:
         out = [item for item in out if not fnmatchcase(item,pat)]
     return out
 

setuptools/tests/test_find_packages.py

+"""Tests for setuptools.find_packages()."""
+import os
+import shutil
+import sys
+import tempfile
+import unittest
+
+from setuptools import find_packages
+
+
+PEP420 = sys.version_info[:2] >= (3, 3)
+
+
+class TestFindPackages(unittest.TestCase):
+
+    def setUp(self):
+        self.dist_dir = tempfile.mkdtemp()
+        self._make_pkg_structure()
+
+    def tearDown(self):
+        shutil.rmtree(self.dist_dir)
+
+    def _make_pkg_structure(self):
+        """Make basic package structure.
+
+        dist/
+            docs/
+                conf.py
+            pkg/
+                __pycache__/
+                nspkg/
+                    mod.py
+                subpkg/
+                    assets/
+                        asset
+                    __init__.py
+            setup.py
+
+        """
+        self.docs_dir = self._mkdir('docs', self.dist_dir)
+        self._touch('conf.py', self.docs_dir)
+        self.pkg_dir = self._mkdir('pkg', self.dist_dir)
+        self._mkdir('__pycache__', self.pkg_dir)
+        self.ns_pkg_dir = self._mkdir('nspkg', self.pkg_dir)
+        self._touch('mod.py', self.ns_pkg_dir)
+        self.sub_pkg_dir = self._mkdir('subpkg', self.pkg_dir)
+        self.asset_dir = self._mkdir('assets', self.sub_pkg_dir)
+        self._touch('asset', self.asset_dir)
+        self._touch('__init__.py', self.sub_pkg_dir)
+        self._touch('setup.py', self.dist_dir)
+
+    def _mkdir(self, path, parent_dir=None):
+        if parent_dir:
+            path = os.path.join(parent_dir, path)
+        os.mkdir(path)
+        return path
+
+    def _touch(self, path, dir_=None):
+        if dir_:
+            path = os.path.join(dir_, path)
+        fp = open(path, 'w')
+        fp.close()
+        return path
+
+    @unittest.skipIf(PEP420, 'Not a PEP 420 env')
+    def test_regular_package(self):
+        self._touch('__init__.py', self.pkg_dir)
+        packages = find_packages(self.dist_dir)
+        self.assertEqual(packages, ['pkg', 'pkg.subpkg'])
+
+    def test_dir_with_dot_is_skipped(self):
+        shutil.rmtree(os.path.join(self.dist_dir, 'pkg/subpkg/assets'))
+        data_dir = self._mkdir('some.data', self.pkg_dir)
+        self._touch('__init__.py', data_dir)
+        self._touch('file.dat', data_dir)
+        packages = find_packages(self.dist_dir)
+        self.assertNotIn('pkg.some.data', packages)
+
+    @unittest.skipIf(not PEP420, 'PEP 420 only')
+    def test_pep420_ns_package(self):
+        packages = find_packages(
+            self.dist_dir, include=['pkg*'], exclude=['pkg.subpkg.assets'])
+        self.assertEqual(packages, ['pkg', 'pkg.nspkg', 'pkg.subpkg'])
+
+    @unittest.skipIf(not PEP420, 'PEP 420 only')
+    def test_pep420_ns_package_no_includes(self):
+        packages = find_packages(
+            self.dist_dir, exclude=['pkg.subpkg.assets'])
+        self.assertEqual(packages, ['docs', 'pkg', 'pkg.nspkg', 'pkg.subpkg'])
+
+    @unittest.skipIf(not PEP420, 'PEP 420 only')
+    def test_pep420_ns_package_no_includes_or_excludes(self):
+        packages = find_packages(self.dist_dir)
+        expected = [
+            'docs', 'pkg', 'pkg.nspkg', 'pkg.subpkg', 'pkg.subpkg.assets']
+        self.assertEqual(packages, expected)
+
+    @unittest.skipIf(not PEP420, 'PEP 420 only')
+    def test_regular_package_with_nested_pep420_ns_packages(self):
+        self._touch('__init__.py', self.pkg_dir)
+        packages = find_packages(
+            self.dist_dir, exclude=['docs', 'pkg.subpkg.assets'])
+        self.assertEqual(packages, ['pkg', 'pkg.nspkg', 'pkg.subpkg'])
+
+    @unittest.skipIf(not PEP420, 'PEP 420 only')
+    def test_pep420_ns_package_no_non_package_dirs(self):
+        shutil.rmtree(self.docs_dir)
+        shutil.rmtree(os.path.join(self.dist_dir, 'pkg/subpkg/assets'))
+        packages = find_packages(self.dist_dir)
+        self.assertEqual(packages, ['pkg', 'pkg.nspkg', 'pkg.subpkg'])