Removed all special support for Sourceforge mirrors, as Sourceforge's

mirror system now works well for non-browser downloads.
(backport from trunk)

--HG--
branch : setuptools-0.6
extra : convert_revision : svn%3A6015fed2-1504-0410-9fe1-9d1591cc4771/sandbox/branches/setuptools-0.6%4053178

EasyInstall.txt

@@ -355,40 +355,6 @@ The above example would then allow downloads only from hosts in the
 ``python.org`` and ``myintranet.example.com`` domains, unless overridden on the
 command line.
 
-Note that if you want to allow downloads from Sourceforge, you need to enable
-the ``dl.sourceforge.net`` host.  All Sourceforge mirror downloads are treated
-as if they had this hostname, regardless of which mirror is actually used to
-do the downloading.  If you want to restrict downloading to specific
-Sourceforge hosts, you must use the ``--sf-mirrors`` option to set what hosts
-will be substituted for ``dl.sourceforge.net``.  See the next section for more
-details.
-
-
-Selecting Your Preferred Sourceforge Mirror(s)
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-You can use the ``--sf-mirrors`` option on the command line, or via the
-standard `configuration files`_, to select what Sourceforge mirrors you want
-downloads to take place from.  For example, putting this in your
-configuration::
-
-    [easy_install]
-    sf_mirrors = superb-east, superb-west, easynews
-
-Would cause ``easy_install`` to always perform Sourceforge downloads using
-``superb-east.dl.sourceforge.net``, ``superb-west.dl.sourceforge.net``, or
-``easynews.dl.sourceforge.net``.  You can also specify IP addresses or full
-domain names.
-
-The actual mirror used will be selected at runtime.  If the selected mirror
-fails, another mirror will be selected, after eliminating the failed mirror as
-a possible choice.
-
-If you don't specify any mirrors, ``easy_install`` will randomly select mirrors
-from the list of IP addresses for ``sf-mirrors.telecommunity.com``.  And if
-the DNS lookup fails, it will simply make a single download attempt to
-``dl.sourceforge.net``, without using any mirrors at all.
-
 
 Installing on Un-networked Machines
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -909,32 +875,6 @@ Command-Line Options
     setting for this option in their `configuration files`_, and then manually
     override the setting on the command line as needed.
 
-    Note that if you wish to allow Sourceforge downloads, you must allow access
-    to ``dl.sourceforge.net``.  You do not have to list individual Sourceforge
-    mirror addresses, as mirror selection is controlled by the ``--sf-mirrors``
-    option.
-
-``--sf-mirrors=NAMES`` (New in 0.6c4)
-    Set the list of Sourceforge mirror sites to use for downloads published by
-    Sourceforge.  EasyInstall will randomly select one for each Sourceforge
-    download attempt.
-
-    Mirror sites can be given by name (e.g. ``easynews``, ``superb-east``,
-    etc.) or by full hostname/IP address (e.g. ``easynews.dl.sf.net``).  Use a
-    comma to separate mirrors.
-
-    If you do not provide any names, EasyInstall will use the list of IP
-    addresses provided by the ``sf-mirrors.telecommunity.com`` subdomain, which
-    is automatically updated daily from Sourceforge's UI pages and DNS.
-
-    If, due to firewall protections or server failure, it isn't possible to get
-    the mirror list from ``sf-mirrors.telecommunity.com``, EasyInstall will
-    attempt to perform all downloads directly from ``dl.sourceforge.net``
-    without selecting a mirror.  (Note, however, that this is extremely
-    unreliable due to Sourceforge not keeping the ``dl.sourceforge.net`` IP
-    addresses up to date with their UI!  This is why the backup system at
-    ``sf-mirrors.telecommunity.com`` exists.)
-
 ``--prefix=DIR`` (New in 0.6a10)
     Use the specified directory as a base for computing the default
     installation and script directories.  On Windows, the resulting default
@@ -1255,16 +1195,12 @@ Release Notes/Change History
 ============================
 
 0.6c4
+ * Removed all special support for Sourceforge mirrors, as Sourceforge's
+   mirror system now works well for non-browser downloads.
+
  * Fixed not recogninzing ``win32.exe`` installers that included a custom
    bitmap.
 
- * Fixed SF downloads aborting when a SF mirror returns an HTML page when it
-   should've returned a 404.  Fall back to ``sf-mirrors.telecommunity.com``
-   round-robin address for SF mirrors if ``dl.sourceforge.net`` doesn't work.
-
- * Allow explicit selection of Sourceforge mirror(s) with ``--sf-mirror``, and
-   further refine download/retry algorithm.
-
  * Fixed not allowing ``os.open()`` of paths outside the sandbox, even if they
    are opened read-only (e.g. reading ``/dev/urandom`` for random numbers, as
    is done by ``os.urandom()`` on some platforms).

setuptools/command/easy_install.py

@@ -70,7 +70,6 @@ class easy_install(Command):
         ('editable', 'e', "Install specified packages in editable form"),
         ('no-deps', 'N', "don't install dependencies"),
         ('allow-hosts=', 'H', "pattern(s) that hostnames must match"),
-        ('sf-mirrors=', None, "Sourceforge mirror(s) to use"),
     ]
     boolean_options = [
         'zip-ok', 'multi-version', 'exclude-scripts', 'upgrade', 'always-copy',
@@ -80,6 +79,7 @@ class easy_install(Command):
     negative_opt = {'always-unzip': 'zip-ok'}
     create_index = PackageIndex
 
+
     def initialize_options(self):
         self.zip_ok = None
         self.install_dir = self.script_dir = self.exclude_scripts = None
@@ -90,7 +90,7 @@ class easy_install(Command):
         self.optimize = self.record = None
         self.upgrade = self.always_copy = self.multi_version = None
         self.editable = self.no_deps = self.allow_hosts = None
-        self.root = self.prefix = self.no_report = self.sf_mirrors = None
+        self.root = self.prefix = self.no_report = None
 
         # Options not specifiable via command line
         self.package_index = None
@@ -169,7 +169,6 @@ class easy_install(Command):
         if self.package_index is None:
             self.package_index = self.create_index(
                 self.index_url, search_path = self.shadow_path, hosts=hosts,
-                sf_mirrors = self.sf_mirrors
             )
         self.local_index = Environment(self.shadow_path+sys.path)
 
@@ -203,6 +202,7 @@ class easy_install(Command):
 
         self.outputs = []
 
+
     def run(self):
         if self.verbose<>self.distribution.verbose:
             log.set_verbosity(self.verbose)

setuptools/package_index.py

@@ -14,7 +14,7 @@ PYPI_MD5 = re.compile(
     '<a href="([^"#]+)">([^<]+)</a>\n\s+\\(<a (?:title="MD5 hash"\n\s+)'
     'href="[^?]+\?:action=show_md5&amp;digest=([0-9a-f]{32})">md5</a>\\)'
 )
-SF_DOWNLOAD = 'dl.sourceforge.net'
+
 URL_SCHEME = re.compile('([-+.a-z0-9]{2,}):',re.I).match
 EXTENSIONS = ".tar.gz .tar.bz2 .tar .zip .tgz".split()
 
@@ -146,27 +146,11 @@ user_agent = "Python-urllib/%s setuptools/%s" % (
 )
 
 
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
 class PackageIndex(Environment):
     """A distribution index that scans web pages for download URLs"""
 
     def __init__(self, index_url="http://www.python.org/pypi", hosts=('*',),
-        sf_mirrors=None, *args, **kw
+        *args, **kw
     ):
         Environment.__init__(self,*args,**kw)
         self.index_url = index_url + "/"[:not index_url.endswith('/')]
@@ -175,39 +159,14 @@ class PackageIndex(Environment):
         self.package_pages = {}
         self.allows = re.compile('|'.join(map(translate,hosts))).match
         self.to_scan = []
-        if sf_mirrors:
-            if isinstance(sf_mirrors,str):
-                self.sf_mirrors = map(str.strip, sf_mirrors.split(','))
-            else:
-                self.sf_mirrors = map(str.strip, sf_mirrors)
-        else:
-            self.sf_mirrors = ()
-
 
-    def _get_mirrors(self):
-        mirrors = []
-        for mirror in self.sf_mirrors:
-            if mirror:
-                if '.' not in mirror:
-                    mirror += '.dl.sourceforge.net'
-                mirrors.append(mirror)
 
-        if not mirrors:
-            try:
-                mirrors.extend(
-                    socket.gethostbyname_ex('sf-mirrors.telecommunity.com')[-1]
-                )
-            except socket.error:
-                # DNS-bl0ck1n9 f1r3w4llz sUx0rs!
-                mirrors[:] = [SF_DOWNLOAD]
-
-        return mirrors
 
     def process_url(self, url, retrieve=False):
         """Evaluate a URL as a possible download, and maybe retrieve it"""
-        url = fix_sf_url(url)
         if url in self.scanned_urls and not retrieve:
             return
+
         self.scanned_urls[url] = True
         if not URL_SCHEME(url):
             self.process_filename(url)
@@ -613,47 +572,6 @@ class PackageIndex(Environment):
         pass    # no-op
 
 
-    def _attempt_download(self, url, filename):
-        headers = self._download_to(url, filename)
-        if 'html' in headers['content-type'].lower():
-            return self._download_html(url, headers, filename)
-        else:
-            return filename
-
-    def _retry_sf_download(self, url, filename):
-        self.url_ok(url, True)   # raises error if not allowed
-        scheme, server, path, param, query, frag = urlparse.urlparse(url)
-
-        if server == SF_DOWNLOAD:
-            mirrors = self._get_mirrors()
-            query = ''
-        else:
-            mirrors = [server]
-
-        while mirrors or server != SF_DOWNLOAD:
-            mirror = random.choice(mirrors)
-            url = urlparse.urlunparse((scheme,mirror,path,param,query,frag))
-
-            try:
-                return self._attempt_download(url, filename)
-            except (KeyboardInterrupt,SystemExit):
-                raise
-            except:
-                if server != SF_DOWNLOAD:
-                    raise
-                self.warn("Download failed: %s", sys.exc_info()[1])
-                mirrors.remove(mirror)
-
-        raise   # fail if no mirror works
-
-
-
-
-
-
-
-
-
     def open_url(self, url):
         if url.startswith('file:'):
             return local_open(url)
@@ -689,15 +607,19 @@ class PackageIndex(Environment):
         elif scheme=='file':
             return urllib2.url2pathname(urlparse.urlparse(url)[2])
         else:
-            return self._retry_sf_download(url, filename)
-
-
-
-
+            self.url_ok(url, True)   # raises error if not allowed
+            return self._attempt_download(url, filename)
 
     def scan_url(self, url):
         self.process_url(url, True)
 
+    def _attempt_download(self, url, filename):
+        headers = self._download_to(url, filename)
+        if 'html' in headers['content-type'].lower():
+            return self._download_html(url, headers, filename)
+        else:
+            return filename
+
     def _download_html(self, url, headers, filename):
         file = open(filename)
         for line in file:
@@ -728,13 +650,12 @@ class PackageIndex(Environment):
     def warn(self, msg, *args):
         log.warn(msg, *args)
 
+
+
+
+
 def fix_sf_url(url):
-    scheme, server, path, param, query, frag = urlparse.urlparse(url)
-    if server!='prdownloads.sourceforge.net':
-        return url
-    return urlparse.urlunparse(
-        (scheme, SF_DOWNLOAD, 'sourceforge'+path, param, '', frag)
-    )
+    return url      # backward compatibility
 
 def local_open(url):
     """Read a local path, with special support for directories"""
@@ -773,7 +694,4 @@ def local_open(url):
 
 
 
-
-
-
 # this line is a kludge to keep the trailing blank lines for pje's editor