Commit c51417f8 authored by PJ Eby's avatar PJ Eby

Added automatic retry for Sourceforge mirrors. The new download process is

to first just try dl.sourceforge.net, then randomly select mirror IPs and
remove ones that fail, until something works.  The removed IPs stay removed
for the remainder of the run.

--HG--
branch : setuptools
extra : convert_revision : svn%3A6015fed2-1504-0410-9fe1-9d1591cc4771/sandbox/trunk/setuptools%4045219
parent 14ddd20c
...@@ -1097,9 +1097,11 @@ set, if you haven't already got this set up on your machine. ...@@ -1097,9 +1097,11 @@ set, if you haven't already got this set up on your machine.
Release Notes/Change History Release Notes/Change History
============================ ============================
Known Issues 0.6a12
* There's no automatic retry for borked Sourceforge mirrors, which can easily * Added automatic retry for Sourceforge mirrors. The new download process is
time out or be missing a file. to first just try dl.sourceforge.net, then randomly select mirror IPs and
remove ones that fail, until something works. The removed IPs stay removed
for the remainder of the run.
0.6a11 0.6a11
* Process ``dependency_links.txt`` if found in a distribution, by adding the * Process ``dependency_links.txt`` if found in a distribution, by adding the
......
...@@ -530,6 +530,47 @@ class PackageIndex(Environment): ...@@ -530,6 +530,47 @@ class PackageIndex(Environment):
def reporthook(self, url, filename, blocknum, blksize, size): def reporthook(self, url, filename, blocknum, blksize, size):
pass # no-op pass # no-op
def retry_sf_download(self, url, filename):
try:
return self._download_to(url, filename)
except:
scheme, server, path, param, query, frag = urlparse.urlparse(url)
if server!='dl.sourceforge.net':
raise
mirror = get_sf_ip()
while _sf_mirrors:
self.warn("Download failed: %s", sys.exc_info()[1])
url = urlparse.urlunparse((scheme, mirror, path, param, '', frag))
try:
return self._download_to(url, filename)
except:
_sf_mirrors.remove(mirror) # don't retry the same mirror
mirror = get_sf_ip()
raise # fail if no mirror works
def open_url(self, url): def open_url(self, url):
try: try:
...@@ -562,7 +603,7 @@ class PackageIndex(Environment): ...@@ -562,7 +603,7 @@ class PackageIndex(Environment):
if scheme=='svn' or scheme.startswith('svn+'): if scheme=='svn' or scheme.startswith('svn+'):
return self._download_svn(url, filename) return self._download_svn(url, filename)
else: else:
headers = self._download_to(url, filename) headers = self.retry_sf_download(url, filename)
if 'html' in headers['content-type'].lower(): if 'html' in headers['content-type'].lower():
return self._download_html(url, headers, filename, tmpdir) return self._download_html(url, headers, filename, tmpdir)
else: else:
...@@ -618,19 +659,19 @@ def fix_sf_url(url): ...@@ -618,19 +659,19 @@ def fix_sf_url(url):
if server!='prdownloads.sourceforge.net': if server!='prdownloads.sourceforge.net':
return url return url
return urlparse.urlunparse( return urlparse.urlunparse(
(scheme, get_sf_ip(), 'sourceforge'+path, param, '', frag) (scheme, 'dl.sourceforge.net', 'sourceforge'+path, param, '', frag)
) )
def get_sf_ip(_mirrors=[]): _sf_mirrors = []
if not _mirrors:
def get_sf_ip():
if not _sf_mirrors:
try: try:
_mirrors[:] = socket.gethostbyname_ex('dl.sourceforge.net')[-1] _sf_mirrors[:] = socket.gethostbyname_ex('dl.sourceforge.net')[-1]
except socket.error: except socket.error:
# DNS-bl0ck1n9 f1r3w4llz sUx0rs! # DNS-bl0ck1n9 f1r3w4llz sUx0rs!
_mirrors[:] = ['dl.sourceforge.net'] _sf_mirrors[:] = ['dl.sourceforge.net']
return random.choice(_mirrors) return random.choice(_sf_mirrors)
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment