src/zc/buildout/__init__.py

@@ -20,3 +20,16 @@ class UserError(Exception):
 
     def __str__(self):
         return " ".join(map(str, self.args))
+
+# Used for Python 2-3 compatibility
+if str is bytes:
+    bytes2str = str2bytes = lambda s: s
+    def unicode2str(s):
+        return s.encode('utf-8')
+else:
+    def bytes2str(s):
+        return s.decode()
+    def str2bytes(s):
+        return s.encode()
+    def unicode2str(s):
+        return s

src/zc/buildout/download.py

@@ -20,43 +20,44 @@ except ImportError:
 
 try:
     # Python 3
-    from urllib.request import urlretrieve
-    from urllib.parse import urlparse
+    from urllib.request import Request, splitport, splituser, urlopen
+    from urllib.parse import urlparse, urlunparse
 except ImportError:
     # Python 2
-    import base64
     from urlparse import urlparse
     from urlparse import urlunparse
-    import urllib2
-
-    def urlretrieve(url, tmp_path):
-        """Work around Python issue 24599 includig basic auth support
-        """
-        scheme, netloc, path, params, query, frag = urlparse(url)
-        auth, host = urllib2.splituser(netloc)
-        if auth:
-            url = urlunparse((scheme, host, path, params, query, frag))
-            req = urllib2.Request(url)
-            base64string = base64.encodestring(auth)[:-1]
-            basic = "Basic " + base64string
-            req.add_header("Authorization", basic)
-        else:
-            req = urllib2.Request(url)
-        url_obj = urllib2.urlopen(req)
-        with open(tmp_path, 'wb') as fp:
-            fp.write(url_obj.read())
-        return tmp_path, url_obj.info()
-
+    from urllib2 import Request, splitport, splituser, urlopen
 
 from zc.buildout.easy_install import realpath
+from base64 import b64encode
+from contextlib import closing
+import errno
 import logging
+import netrc
 import os
 import os.path
 import re
 import shutil
 import tempfile
 import zc.buildout
+from . import bytes2str, str2bytes
+
+class netrc(netrc.netrc):
+
+    def __init__(*args):
+        pass
 
+    def authenticators(self, host):
+        self.__class__, = self.__class__.__bases__
+        try:
+            self.__init__()
+        except IOError as e:
+            if e.errno != errno.ENOENT:
+                raise
+            self.__init__(os.devnull)
+        return self.authenticators(host)
+
+netrc = netrc()
 
 class ChecksumError(zc.buildout.UserError):
     pass
@@ -213,13 +214,14 @@ class Download(object):
                 nc.get('signature-certificate-list'), md5sum):
                 # Download from original url if not cached or md5sum doesn't match.
                 try:
-                    tmp_path, headers = urlretrieve(url, tmp_path)
+                    tmp_path, headers = self.urlretrieve(url, tmp_path)
                 except HTTPError:
                     if not alternate_url:
                         raise
                     self.logger.info('using alternate URL: %s', alternate_url)
                     download_url = alternate_url
-                    tmp_path, headers = urlretrieve(alternate_url, tmp_path)
+                    tmp_path, headers = self.urlretrieve(
+                        alternate_url, tmp_path)
                 if not check_md5sum(tmp_path, md5sum):
                     raise ChecksumError(
                         'MD5 checksum mismatch downloading %r' % download_url)
@@ -272,6 +274,27 @@ class Download(object):
             url_host, url_port = parsed[-2:]
             return '%s:%s' % (url_host, url_port)
 
+    def urlretrieve(self, url, tmp_path):
+        scheme, netloc, path, params, query, frag = urlparse(url)
+        req = url
+        while scheme in ('http', 'https'): # not a loop
+            auth, host = splituser(netloc)
+            if auth:
+                url = urlunparse((scheme, host, path, params, query, frag))
+            else:
+                auth = netrc.authenticators(splitport(host)[0])
+                if not auth:
+                    break
+                auth = '{0}:{2}'.format(*auth)
+            req = Request(url)
+            req.add_header("Authorization",
+                           "Basic " + bytes2str(b64encode(str2bytes(auth))))
+            break
+        with closing(urlopen(req)) as url_obj:
+            with open(tmp_path, 'wb') as fp:
+                shutil.copyfileobj(url_obj, fp)
+            return tmp_path, url_obj.info()
+
 
 def check_md5sum(path, md5sum):
     """Tell whether the MD5 checksum of the file at path matches.

src/zc/buildout/download.txt

@@ -152,6 +152,37 @@ This is a foo text.
 
 >>> remove(path)
 
+HTTP basic authentication:
+
+>>> download = Download()
+>>> user_url = server_url.replace('/localhost:', '/%s@localhost:') + 'private/'
+>>> path, is_temp = download(user_url % 'foo:' + 'foo:')
+>>> is_temp; remove(path)
+True
+>>> path, is_temp = download(user_url % 'foo:bar' + 'foo:bar')
+>>> is_temp; remove(path)
+True
+>>> download(user_url % 'bar:' + 'foo:')
+Traceback (most recent call last):
+UserError: Error downloading ...: HTTP Error 403: Forbidden
+
+... with netrc:
+
+>>> url = server_url + 'private/foo:bar'
+>>> download(url)
+Traceback (most recent call last):
+UserError: Error downloading ...: HTTP Error 403: Forbidden
+>>> import os, zc.buildout.download
+>>> old_home = os.environ['HOME']
+>>> home = os.environ['HOME'] = tmpdir('test-netrc')
+>>> netrc = join(home, '.netrc')
+>>> write(netrc, 'machine localhost login foo password bar')
+>>> os.chmod(netrc, 0o600)
+>>> zc.buildout.download.netrc.__init__()
+>>> path, is_temp = download(url)
+>>> is_temp; remove(path)
+True
+>>> os.environ['HOME'] = old_home
 
 Downloading using the download cache
 ------------------------------------

src/zc/buildout/testing.py

@@ -23,6 +23,7 @@ except ImportError:
     from BaseHTTPServer import HTTPServer, BaseHTTPRequestHandler
     from urllib2        import urlopen
 
+import base64
 import errno
 import logging
 import os
@@ -364,6 +365,23 @@ class Handler(BaseHTTPRequestHandler):
             self.__server.__log = False
             return k()
 
+        if self.path.startswith('/private/'):
+            auth = self.headers.get('Authorization')
+            if auth and auth.startswith('Basic ') and \
+                self.path[9:].encode() == base64.b64decode(
+                    self.headers.get('Authorization')[6:]):
+                return k()
+            # But not returning 401+WWW-Authenticate, we check that the client
+            # skips auth challenge, which is not free (in terms of performance)
+            # and useless for what we support.
+            self.send_response(403, 'Forbidden')
+            out = '<html><body>Forbidden</body></html>'.encode()
+            self.send_header('Content-Length', str(len(out)))
+            self.send_header('Content-Type', 'text/html')
+            self.end_headers()
+            self.wfile.write(out)
+            return
+
         path = os.path.abspath(os.path.join(self.tree, *self.path.split('/')))
         if not (
             ((path == self.tree) or path.startswith(self.tree+os.path.sep))