Merged in embray/setuptools (pull request #167)

Possible fix for #207

pkg_resources/__init__.py

@@ -755,7 +755,7 @@ class WorkingSet(object):
         will be called.
         """
         if insert:
-            dist.insert_on(self.entries, entry)
+            dist.insert_on(self.entries, entry, replace=replace)
 
         if entry is None:
             entry = dist.location
@@ -2182,9 +2182,17 @@ def _handle_ns(packageName, path_item):
         path = module.__path__
         path.append(subpath)
         loader.load_module(packageName)
-        for path_item in path:
-            if path_item not in module.__path__:
-                module.__path__.append(path_item)
+
+        # Rebuild mod.__path__ ensuring that all entries are ordered
+        # corresponding to their sys.path order
+        sys_path= [(p and _normalize_cached(p) or p) for p in sys.path]
+        def sort_key(p):
+            parts = p.split(os.sep)
+            parts = parts[:-(packageName.count('.') + 1)]
+            return sys_path.index(_normalize_cached(os.sep.join(parts)))
+
+        path.sort(key=sort_key)
+        module.__path__[:] = [_normalize_cached(p) for p in path]
     return subpath
 
 def declare_namespace(packageName):
@@ -2639,7 +2647,7 @@ class Distribution(object):
         """Ensure distribution is importable on `path` (default=sys.path)"""
         if path is None:
             path = sys.path
-        self.insert_on(path)
+        self.insert_on(path, replace=True)
         if path is sys.path:
             fixup_namespace_packages(self.location)
             for pkg in self._get_metadata('namespace_packages.txt'):
@@ -2716,7 +2724,7 @@ class Distribution(object):
         """Return the EntryPoint object for `group`+`name`, or ``None``"""
         return self.get_entry_map(group).get(name)
 
-    def insert_on(self, path, loc = None):
+    def insert_on(self, path, loc=None, replace=False):
         """Insert self.location in path before its nearest parent directory"""
 
         loc = loc or self.location
@@ -2740,7 +2748,10 @@ class Distribution(object):
         else:
             if path is sys.path:
                 self.check_version_conflict()
-            path.append(loc)
+            if replace:
+                path.insert(0, loc)
+            else:
+                path.append(loc)
             return
 
         # p is the spot where we found or inserted loc; now remove duplicates

pkg_resources/tests/test_resources.py

@@ -612,18 +612,32 @@ class TestNamespaces:
 
     def setup_method(self, method):
         self._ns_pkgs = pkg_resources._namespace_packages.copy()
-        self._tmpdir = tempfile.mkdtemp(prefix="tests-setuptools-")
+
+        # Further, test case where the temp dir is a symlink, where applicable
+        # See #231
+        if hasattr(os, 'symlink'):
+            real_tmpdir = tempfile.mkdtemp(prefix="real-tests-setuptools-")
+            tmpdir_base, tmpdir_name = os.path.split(real_tmpdir)
+            tmpdir = os.path.join(tmpdir_base, tmpdir_name[5:])
+            os.symlink(real_tmpdir, tmpdir)
+            self._real_tmpdir = real_tmpdir
+            self._tmpdir = tmpdir
+        else:
+            tmpdir = tempfile.mkdtemp(prefix="tests-setuptools-")
+            self._real_tmpdir = self._tmpdir = tmpdir
+
         os.makedirs(os.path.join(self._tmpdir, "site-pkgs"))
         self._prev_sys_path = sys.path[:]
         sys.path.append(os.path.join(self._tmpdir, "site-pkgs"))
 
     def teardown_method(self, method):
-        shutil.rmtree(self._tmpdir)
+        shutil.rmtree(self._real_tmpdir)
+        if os.path.islink(self._tmpdir):
+            os.unlink(self._tmpdir)
+
         pkg_resources._namespace_packages = self._ns_pkgs.copy()
         sys.path = self._prev_sys_path[:]
 
-    @pytest.mark.skipif(os.path.islink(tempfile.gettempdir()),
-        reason="Test fails when /tmp is a symlink. See #231")
     def test_two_levels_deep(self):
         """
         Test nested namespace packages
@@ -655,7 +669,41 @@ class TestNamespaces:
         assert pkg_resources._namespace_packages["pkg1"] == ["pkg1.pkg2"]
         # check the __path__ attribute contains both paths
         expected = [
-            os.path.join(self._tmpdir, "site-pkgs", "pkg1", "pkg2"),
-            os.path.join(self._tmpdir, "site-pkgs2", "pkg1", "pkg2"),
+            os.path.join(self._real_tmpdir, "site-pkgs", "pkg1", "pkg2"),
+            os.path.join(self._real_tmpdir, "site-pkgs2", "pkg1", "pkg2"),
         ]
         assert pkg1.pkg2.__path__ == expected
+
+    def test_path_order(self):
+        """
+        Test that if multiple versions of the same namespace package subpackage
+        are on different sys.path entries, that only the one earliest on
+        sys.path is imported, and that the namespace package's __path__ is in
+        the correct order.
+
+        Regression test for https://bitbucket.org/pypa/setuptools/issues/207
+        """
+
+        site_pkgs = ["site-pkgs", "site-pkgs2", "site-pkgs3"]
+
+        ns_str = "__import__('pkg_resources').declare_namespace(__name__)\n"
+        vers_str = "__version__ = %r"
+
+        for idx, site in enumerate(site_pkgs):
+            if idx > 0:
+                sys.path.append(os.path.join(self._tmpdir, site))
+            os.makedirs(os.path.join(self._tmpdir, site, "nspkg", "subpkg"))
+            with open(os.path.join(self._tmpdir, site, "nspkg",
+                                   "__init__.py"), "w") as f:
+                f.write(ns_str)
+
+            with open(os.path.join(self._tmpdir, site, "nspkg", "subpkg",
+                                   "__init__.py"), "w") as f:
+                f.write(vers_str % (idx + 1))
+
+        import nspkg.subpkg
+        import nspkg
+        assert nspkg.__path__ == [os.path.join(self._real_tmpdir, site,
+                                               "nspkg")
+                                  for site in site_pkgs]
+        assert nspkg.subpkg.__version__ == 1

setuptools/tests/contexts.py

@@ -6,6 +6,7 @@ import contextlib
 import site
 
 from setuptools.extern import six
+import pkg_resources
 
 
 @contextlib.contextmanager
@@ -77,6 +78,18 @@ def save_user_site_setting():
         site.ENABLE_USER_SITE = saved
 
 
+@contextlib.contextmanager
+def save_pkg_resources_state():
+    pr_state = pkg_resources.__getstate__()
+    # also save sys.path
+    sys_path = sys.path[:]
+    try:
+        yield pr_state, sys_path
+    finally:
+        sys.path[:] = sys_path
+        pkg_resources.__setstate__(pr_state)
+
+
 @contextlib.contextmanager
 def suppress_exceptions(*excs):
     try:

setuptools/tests/test_easy_install.py

@@ -18,6 +18,7 @@ import io
 
 from setuptools.extern import six
 from setuptools.extern.six.moves import urllib
+import time
 
 import pytest
 try:
@@ -310,32 +311,32 @@ class TestSetupRequires:
         """
         with contexts.tempdir() as dir:
             dist_path = os.path.join(dir, 'setuptools-test-fetcher-1.0.tar.gz')
-            script = DALS("""
-                import setuptools
-                setuptools.setup(
-                    name="setuptools-test-fetcher",
-                    version="1.0",
-                    setup_requires = ['does-not-exist'],
-                )
-                """)
-            make_trivial_sdist(dist_path, script)
+            make_sdist(dist_path, [
+                ('setup.py', DALS("""
+                    import setuptools
+                    setuptools.setup(
+                        name="setuptools-test-fetcher",
+                        version="1.0",
+                        setup_requires = ['does-not-exist'],
+                    )
+                """))])
             yield dist_path
 
     def test_setup_requires_overrides_version_conflict(self):
         """
-        Regression test for issue #323.
+        Regression test for distribution issue 323:
+        https://bitbucket.org/tarek/distribute/issues/323
 
         Ensures that a distribution's setup_requires requirements can still be
         installed and used locally even if a conflicting version of that
         requirement is already on the path.
         """
 
-        pr_state = pkg_resources.__getstate__()
         fake_dist = PRDistribution('does-not-matter', project_name='foobar',
                                    version='0.0')
         working_set.add(fake_dist)
 
-        try:
+        with contexts.save_pkg_resources_state():
             with contexts.tempdir() as temp_dir:
                 test_pkg = create_setup_requires_package(temp_dir)
                 test_setup_py = os.path.join(test_pkg, 'setup.py')
@@ -347,19 +348,154 @@ class TestSetupRequires:
                 lines = stdout.readlines()
                 assert len(lines) > 0
                 assert lines[-1].strip(), 'test_pkg'
-        finally:
-            pkg_resources.__setstate__(pr_state)
 
+    def test_setup_requires_override_nspkg(self):
+        """
+        Like ``test_setup_requires_overrides_version_conflict`` but where the
+        ``setup_requires`` package is part of a namespace package that has
+        *already* been imported.
+        """
+
+        with contexts.save_pkg_resources_state():
+            with contexts.tempdir() as temp_dir:
+                foobar_1_archive = os.path.join(temp_dir, 'foo.bar-0.1.tar.gz')
+                make_nspkg_sdist(foobar_1_archive, 'foo.bar', '0.1')
+                # Now actually go ahead an extract to the temp dir and add the
+                # extracted path to sys.path so foo.bar v0.1 is importable
+                foobar_1_dir = os.path.join(temp_dir, 'foo.bar-0.1')
+                os.mkdir(foobar_1_dir)
+                with tarfile.open(foobar_1_archive) as tf:
+                    tf.extractall(foobar_1_dir)
+                sys.path.insert(1, foobar_1_dir)
+
+                dist = PRDistribution(foobar_1_dir, project_name='foo.bar',
+                                      version='0.1')
+                working_set.add(dist)
+
+                template = DALS("""\
+                    import foo  # Even with foo imported first the
+                                # setup_requires package should override
+                    import setuptools
+                    setuptools.setup(**%r)
+
+                    if not (hasattr(foo, '__path__') and
+                            len(foo.__path__) == 2):
+                        print('FAIL')
+
+                    if 'foo.bar-0.2' not in foo.__path__[0]:
+                        print('FAIL')
+                """)
+
+                test_pkg = create_setup_requires_package(
+                    temp_dir, 'foo.bar', '0.2', make_nspkg_sdist, template)
+
+                test_setup_py = os.path.join(test_pkg, 'setup.py')
 
-def create_setup_requires_package(path):
+                with contexts.quiet() as (stdout, stderr):
+                    try:
+                        # Don't even need to install the package, just
+                        # running the setup.py at all is sufficient
+                        run_setup(test_setup_py, ['--name'])
+                    except VersionConflict:
+                        self.fail('Installing setup.py requirements '
+                            'caused a VersionConflict')
+
+                assert 'FAIL' not in stdout.getvalue()
+                lines = stdout.readlines()
+                assert len(lines) > 0
+                assert lines[-1].strip() == 'test_pkg'
+
+
+def make_trivial_sdist(dist_path, distname, version):
+    """
+    Create a simple sdist tarball at dist_path, containing just a simple
+    setup.py.
+    """
+
+    make_sdist(dist_path, [
+        ('setup.py',
+         DALS("""\
+             import setuptools
+             setuptools.setup(
+                 name=%r,
+                 version=%r
+             )
+         """ % (distname, version)))])
+
+
+def make_nspkg_sdist(dist_path, distname, version):
+    """
+    Make an sdist tarball with distname and version which also contains one
+    package with the same name as distname.  The top-level package is
+    designated a namespace package).
+    """
+
+    parts = distname.split('.')
+    nspackage = parts[0]
+
+    packages = ['.'.join(parts[:idx]) for idx in range(1, len(parts) + 1)]
+
+    setup_py = DALS("""\
+        import setuptools
+        setuptools.setup(
+            name=%r,
+            version=%r,
+            packages=%r,
+            namespace_packages=[%r]
+        )
+    """ % (distname, version, packages, nspackage))
+
+    init = "__import__('pkg_resources').declare_namespace(__name__)"
+
+    files = [('setup.py', setup_py),
+             (os.path.join(nspackage, '__init__.py'), init)]
+    for package in packages[1:]:
+        filename = os.path.join(*(package.split('.') + ['__init__.py']))
+        files.append((filename, ''))
+
+    make_sdist(dist_path, files)
+
+
+def make_sdist(dist_path, files):
+    """
+    Create a simple sdist tarball at dist_path, containing the files
+    listed in ``files`` as ``(filename, content)`` tuples.
+    """
+
+    dist = tarfile.open(dist_path, 'w:gz')
+
+    try:
+        # Python 3 (StringIO gets converted to io module)
+        MemFile = BytesIO
+    except AttributeError:
+        MemFile = StringIO
+
+    try:
+        for filename, content in files:
+            file_bytes = MemFile(content.encode('utf-8'))
+            file_info = tarfile.TarInfo(name=filename)
+            file_info.size = len(file_bytes.getvalue())
+            file_info.mtime = int(time.time())
+            dist.addfile(file_info, fileobj=file_bytes)
+    finally:
+        dist.close()
+
+
+def create_setup_requires_package(path, distname='foobar', version='0.1',
+                                  make_package=make_trivial_sdist,
+                                  setup_py_template=None):
     """Creates a source tree under path for a trivial test package that has a
     single requirement in setup_requires--a tarball for that requirement is
     also created and added to the dependency_links argument.
+
+    ``distname`` and ``version`` refer to the name/version of the package that
+    the test package requires via ``setup_requires``.  The name of the test
+    package itself is just 'test_pkg'.
     """
 
     test_setup_attrs = {
         'name': 'test_pkg', 'version': '0.0',
-        'setup_requires': ['foobar==0.1'],
+        'setup_requires': ['%s==%s' % (distname, version)],
         'dependency_links': [os.path.abspath(path)]
     }
 
@@ -367,22 +503,17 @@ def create_setup_requires_package(path):
     test_setup_py = os.path.join(test_pkg, 'setup.py')
     os.mkdir(test_pkg)
 
-    with open(test_setup_py, 'w') as f:
-        f.write(DALS("""
+    if setup_py_template is None:
+        setup_py_template = DALS("""\
             import setuptools
             setuptools.setup(**%r)
-        """ % test_setup_attrs))
+        """)
 
-    foobar_path = os.path.join(path, 'foobar-0.1.tar.gz')
-    make_trivial_sdist(
-        foobar_path,
-        DALS("""
-            import setuptools
-            setuptools.setup(
-                name='foobar',
-                version='0.1'
-            )
-        """))
+    with open(test_setup_py, 'w') as f:
+        f.write(setup_py_template % test_setup_attrs)
+
+    foobar_path = os.path.join(path, '%s-%s.tar.gz' % (distname, version))
+    make_package(foobar_path, distname, version)
 
     return test_pkg