Rewrite 'urlretrieve' helper to fix various download-related issues

- Py3: stop using legacy API of urllib.request and
       fix download of http(s) URLs containing user:passwd@
- Py2: avoid OOM when downloading huge files

Also build opener only once for performance reasons.

src/zc/buildout/__init__.py

@@ -20,3 +20,16 @@ class UserError(Exception):
 
     def __str__(self):
         return " ".join(map(str, self.args))
+
+# Used for Python 2-3 compatibility
+if str is bytes:
+    bytes2str = str2bytes = lambda s: s
+    def unicode2str(s):
+        return s.encode('utf-8')
+else:
+    def bytes2str(s):
+        return s.decode()
+    def str2bytes(s):
+        return s.encode()
+    def unicode2str(s):
+        return s

src/zc/buildout/download.py

@@ -20,35 +20,16 @@ except ImportError:
 
 try:
     # Python 3
-    from urllib.request import urlretrieve
-    from urllib.parse import urlparse
+    from urllib.request import build_opener, splituser, Request
+    from urllib.parse import urlparse, urlunparse
 except ImportError:
     # Python 2
-    import base64
     from urlparse import urlparse
     from urlparse import urlunparse
-    import urllib2
-
-    def urlretrieve(url, tmp_path):
-        """Work around Python issue 24599 includig basic auth support
-        """
-        scheme, netloc, path, params, query, frag = urlparse(url)
-        auth, host = urllib2.splituser(netloc)
-        if auth:
-            url = urlunparse((scheme, host, path, params, query, frag))
-            req = urllib2.Request(url)
-            base64string = base64.encodestring(auth)[:-1]
-            basic = "Basic " + base64string
-            req.add_header("Authorization", basic)
-        else:
-            req = urllib2.Request(url)
-        url_obj = urllib2.urlopen(req)
-        with open(tmp_path, 'wb') as fp:
-            fp.write(url_obj.read())
-        return tmp_path, url_obj.info()
-
+    from urllib2 import build_opener, splituser, Request
 
 from zc.buildout.easy_install import realpath
+from base64 import b64encode
 import logging
 import os
 import os.path
@@ -56,7 +37,26 @@ import re
 import shutil
 import tempfile
 import zc.buildout
-
+from . import bytes2str, str2bytes
+
+_opener = None
+
+def urlretrieve(url, tmp_path):
+    global _opener
+    if _opener is None:
+        _opener = build_opener()
+    scheme, netloc, path, params, query, frag = urlparse(url)
+    auth, host = splituser(netloc)
+    if auth and scheme in ('http', 'https'):
+        req = Request(urlunparse((scheme, host, path, params, query, frag)))
+        req.add_header("Authorization",
+                       "Basic " + bytes2str(b64encode(str2bytes(auth))))
+    else:
+        req = Request(url)
+    url_obj = _opener.open(req)
+    with open(tmp_path, 'wb') as fp:
+        shutil.copyfileobj(url_obj, fp)
+    return tmp_path, url_obj.info()
 
 class ChecksumError(zc.buildout.UserError):
     pass

src/zc/buildout/download.txt

@@ -152,6 +152,19 @@ This is a foo text.
 
 >>> remove(path)
 
+HTTP basic authentication:
+
+>>> download = Download()
+>>> user_url = server_url.replace('/localhost:', '/%s@localhost:') + 'private/'
+>>> path, is_temp = download(user_url % 'foo:' + 'foo:')
+>>> is_temp; remove(path)
+True
+>>> path, is_temp = download(user_url % 'foo:bar' + 'foo:bar')
+>>> is_temp; remove(path)
+True
+>>> download(user_url % 'bar:' + 'foo:')
+Traceback (most recent call last):
+UserError: Error downloading ...: HTTP Error 403: Forbidden
 
 Downloading using the download cache
 ------------------------------------

src/zc/buildout/testing.py

@@ -23,6 +23,7 @@ except ImportError:
     from BaseHTTPServer import HTTPServer, BaseHTTPRequestHandler
     from urllib2        import urlopen
 
+import base64
 import errno
 import logging
 import os
@@ -364,6 +365,20 @@ class Handler(BaseHTTPRequestHandler):
             self.__server.__log = False
             return k()
 
+        if self.path.startswith('/private/'):
+            auth = self.headers.get('Authorization')
+            if auth and auth.startswith('Basic ') and \
+                self.path[9:].encode() == base64.b64decode(
+                    self.headers.get('Authorization')[6:]):
+                return k()
+            self.send_response(403, 'Forbidden')
+            out = '<html><body>Forbidden</body></html>'.encode()
+            self.send_header('Content-Length', str(len(out)))
+            self.send_header('Content-Type', 'text/html')
+            self.end_headers()
+            self.wfile.write(out)
+            return
+
         path = os.path.abspath(os.path.join(self.tree, *self.path.split('/')))
         if not (
             ((path == self.tree) or path.startswith(self.tree+os.path.sep))