merge

--HG--
branch : distribute
extra : rebase_source : 5fbe57fd3b9bb48afcc78a87a7f7a46d9f4d2567

.hgignore

@@ -10,3 +10,4 @@ lib
 bin
 include
 \.Python
+*.swp

CHANGES.txt

@@ -8,6 +8,8 @@ CHANGES
 
 * Issue 160: 2.7 gives ValueError("Invalid IPv6 URL")
 * Issue 150: Fixed using ~/.local even in a --no-site-packages virtualenv
+* Issue 163: scan index links before external links, and don't use the md5 when
+  comparing two distributions
 
 ------
 0.6.12

CONTRIBUTORS.txt

@@ -5,6 +5,7 @@ Contributors
 * Alex Grönholm
 * Alice Bevan-McGregor
 * Arfrever Frehtes Taifersar Arahesis
+* Christophe Combelles
 * Daniel Stutzbach
 * Hanno Schlichting
 * Jannis Leidel

pkg_resources.py

@@ -14,6 +14,7 @@ method.
 """
 
 import sys, os, zipimport, time, re, imp, types
+from urlparse import urlparse, urlunparse
 
 try:
     frozenset
@@ -2044,8 +2045,13 @@ class EntryPoint(object):
     parse_map = classmethod(parse_map)
 
 
-
-
+def _remove_md5_fragment(location):
+    if not location:
+        return ''
+    parsed = urlparse(location)
+    if parsed[-1].startswith('md5='):
+        return urlunparse(parsed[:-1] + ('',))
+    return location
 
 
 class Distribution(object):
@@ -2079,10 +2085,14 @@ class Distribution(object):
         )
     from_location = classmethod(from_location)
 
+
     hashcmp = property(
         lambda self: (
-            getattr(self,'parsed_version',()), self.precedence, self.key,
-            -len(self.location or ''), self.location, self.py_version,
+            getattr(self,'parsed_version',()),
+            self.precedence,
+            self.key,
+            _remove_md5_fragment(self.location),
+            self.py_version,
             self.platform
         )
     )

setup.py

@@ -41,6 +41,8 @@ VERSION = "0.6.13"
 
 from setuptools import setup, find_packages
 from setuptools.command.build_py import build_py as _build_py
+from setuptools.command.test import test as _test
+
 scripts = []
 
 # specific command that is used to generate windows .exe files
@@ -64,6 +66,36 @@ class build_py(_build_py):
                 if copied and srcfile in self.distribution.convert_2to3_doctests:
                     self.__doctests_2to3.append(outf)
 
+class test(_test):
+    """Specific test class to avoid rewriting the entry_points.txt"""
+    def run(self):
+        entry_points = os.path.join('distribute.egg-info', 'entry_points.txt')
+
+        if not os.path.exists(entry_points):
+            try:
+                _test.run(self)
+            finally:
+                return
+
+        f = open(entry_points)
+
+        # running the test
+        try:
+            ep_content = f.read()
+        finally:
+            f.close()
+
+        try:
+            _test.run(self)
+        finally:
+            # restoring the file
+            f = open(entry_points, 'w')
+            try:
+                f.write(ep_content)
+            finally:
+                f.close()
+
+
 # if we are installing Distribute using "python setup.py install"
 # we need to get setuptools out of the way
 def _easy_install_marker():
@@ -90,6 +122,7 @@ if _being_installed():
     from distribute_setup import _before_install
     _before_install()
 
+
 dist = setup(
     name="distribute",
     version=VERSION,
@@ -110,6 +143,7 @@ dist = setup(
 
     zip_safe = (sys.version>="2.5"),   # <2.5 needs unzipped for -m to work
 
+    cmdclass = {'test': test},
     entry_points = {
 
         "distutils.commands" : [

setuptools/package_index.py

@@ -203,11 +203,11 @@ class PackageIndex(Environment):
             charset = f.headers.get_param('charset') or 'latin-1'
             page = page.decode(charset, "ignore")
         f.close()
-        if url.startswith(self.index_url) and getattr(f,'code',None)!=404:
-            page = self.process_index(url, page)
         for match in HREF.finditer(page):
             link = urlparse.urljoin(base, htmldecode(match.group(1)))
             self.process_url(link)
+        if url.startswith(self.index_url) and getattr(f,'code',None)!=404:
+            page = self.process_index(url, page)
 
     def process_filename(self, fn, nested=False):
         # process filenames or directories

setuptools/tests/indexes/test_links_priority/external.html

+<html><body>
+<a href="/foobar-0.1.tar.gz#md5=1__bad_md5___">bad old link</a>
+</body></html>

setuptools/tests/indexes/test_links_priority/simple/foobar/index.html

+<html><body>
+<a href="/foobar-0.1.tar.gz#md5=0_correct_md5">foobar-0.1.tar.gz</a><br/>
+<a href="../../external.html" rel="homepage">external homepage</a><br/>
+</body></html>

setuptools/tests/server.py

+"""Basic http server for tests to simulate PyPI or custom indexes
+"""
+import urllib2
+from threading import Thread
+from BaseHTTPServer import HTTPServer
+from SimpleHTTPServer import SimpleHTTPRequestHandler
+
+class IndexServer(HTTPServer):
+    """Basic single-threaded http server simulating a package index
+
+    You can use this server in unittest like this::
+        s = IndexServer()
+        s.start()
+        index_url = s.base_url() + 'mytestindex'
+        # do some test requests to the index
+        # The index files should be located in setuptools/tests/indexes
+        s.stop()
+    """
+    def __init__(self):
+        HTTPServer.__init__(self, ('', 0), SimpleHTTPRequestHandler)
+        self._run = True
+
+    def serve(self):
+        while True:
+            self.handle_request()
+            if not self._run: break
+
+    def start(self):
+        self.thread = Thread(target=self.serve)
+        self.thread.start()
+
+    def stop(self):
+        """self.shutdown is not supported on python < 2.6"""
+        self._run = False
+        urllib2.urlopen('http://127.0.0.1:%s/' % self.server_port)
+        self.thread.join()
+
+    def base_url(self):
+        port = self.server_port
+        return 'http://127.0.0.1:%s/setuptools/tests/indexes/' % port

setuptools/tests/test_packageindex.py

@@ -5,6 +5,7 @@ import sys
 import os, shutil, tempfile, unittest, urllib2
 import pkg_resources
 import setuptools.package_index
+from server import IndexServer
 
 class TestPackageIndex(unittest.TestCase):
 
@@ -74,3 +75,38 @@ class TestPackageIndex(unittest.TestCase):
         url = 'file:///tmp/test_package_index'
         self.assert_(index.url_ok(url, True))
 
+    def test_links_priority(self):
+        """
+        Download links from the pypi simple index should be used before
+        external download links.
+        http://bitbucket.org/tarek/distribute/issue/163/md5-validation-error
+
+        Usecase :
+        - someone uploads a package on pypi, a md5 is generated
+        - someone manually copies this link (with the md5 in the url) onto an
+          external page accessible from the package page.
+        - someone reuploads the package (with a different md5)
+        - while easy_installing, an MD5 error occurs because the external link
+          is used
+        -> Distribute should use the link from pypi, not the external one.
+        """
+        # start an index server
+        server = IndexServer()
+        server.start()
+        index_url = server.base_url() + 'test_links_priority/simple/'
+
+        # scan a test index
+        pi = setuptools.package_index.PackageIndex(index_url)
+        requirement = pkg_resources.Requirement.parse('foobar')
+        pi.find_packages(requirement)
+        server.stop()
+
+        # the distribution has been found
+        self.assert_('foobar' in pi)
+        # we have only one link, because links are compared without md5
+        self.assert_(len(pi['foobar'])==1)
+        # the link should be from the index
+        self.assert_('correct_md5' in pi['foobar'][0].location)
+
+
+