Commit 6f470ca9 authored by Jérome Perrin's avatar Jérome Perrin

easy_install: rough prototype of supporting --hash in version

This currently cause distributions to be downloaded twice, even with a
download cache
parent 6a9722bb
......@@ -58,11 +58,11 @@ from setuptools.package_index import HREF, URL_SCHEME, \
try:
# Python 3
from urllib.error import HTTPError
from urllib.parse import urljoin
from urllib.parse import urljoin, urlparse
except ImportError:
# Python 2
from urllib2 import HTTPError
from urlparse import urljoin
from urlparse import urljoin, urlparse
default_index_url = os.environ.get(
'BUILDOUT_TESTING_INDEX_URL',
......@@ -380,13 +380,13 @@ class Installer:
if name.lower() in line.lower()]
return '\n '.join(output)
def _satisfied(self, req, source=None):
def _satisfied(self, req, source=None, hashes=None):
dists = [dist for dist in self._env[req.project_name] if dist in req]
if not dists:
logger.debug('We have no distributions for %s that satisfies %r.',
req.project_name, str(req))
return None, self._obtain(req, source)
return None, self._obtain(req, source, hashes)
# Note that dists are sorted from best to worst, as promised by
# env.__getitem__
......@@ -424,7 +424,7 @@ class Installer:
# any are newer. We only do this if we're willing to install
# something, which is only true if dest is not None:
best_available = self._obtain(req, source)
best_available = self._obtain(req, source, hashes)
if best_available is None:
# That's a bit odd. There aren't any distros available.
......@@ -547,11 +547,12 @@ class Installer:
finally:
shutil.rmtree(tmp)
def _obtain(self, requirement, source=None):
def _obtain(self, requirement, source=None, hashes=None):
# get the non-patched version
req = str(requirement)
if PATCH_MARKER in req:
requirement = pkg_resources.Requirement.parse(re.sub(orig_versions_re, '', req))
# XXX hashes: maybe we can just put the hash in requirement ??
# initialize out index for this project:
index = self._index
......@@ -582,6 +583,42 @@ class Installer:
# There are final dists, so only use those
dists = fdists
# if we are using hashes, filter for dists matching our hashes
if hashes is not None:
def dist_key(dist):
# optimization: sort to try first the distributions with the
# hash in the URL, so that if there's a matching dist we use
# it directly without trying all dists
for hash_ in hashes:
# the --hash format uses alg:hash, but in the URL fragments
# it is alg=hash
if hash_.replace(':', '=') in urlparse(dist.location).fragment:
return 0
return 1
# TODO: can we reuse download cache here ????
#tmp = self._download_cache
#if tmp is None:
# tmp = tempfile.mkdtemp('check_hash')
downloader = zc.buildout.download.Download()
def hash_match(dist):
try:
downloaded_filename, is_temp = downloader(dist.location, hashes=hashes)
if is_temp:
os.remove(downloaded_filename)
except zc.buildout.download.ChecksumError:
return False
return True
dists.sort(key=dist_key)
if hash_match(dists[0]):
dists = [dists[0]]
else:
dists = [dist for dist in dists[1:] if hash_match(dist)]
# Now find the best one:
best = []
bestv = None
......@@ -657,12 +694,12 @@ class Installer:
return dist.clone(location=new_location)
def _get_dist(self, requirement, ws, for_buildout_run=False):
def _get_dist(self, requirement, ws, for_buildout_run=False, hashes=None):
__doing__ = 'Getting distribution for %r.', str(requirement)
# Maybe an existing dist is already the best dist that satisfies the
# requirement
dist, avail = self._satisfied(requirement)
dist, avail = self._satisfied(requirement, hashes=hashes)
if dist is None:
if self._dest is None:
......@@ -720,6 +757,10 @@ class Installer:
if tmp != self._download_cache:
shutil.rmtree(tmp)
# TODO hashes if we:
# 1. install version1 h1
# 2. change to version1 h2 (to get another lower wheel build number)
# ... will it keep using h1 ??? maybe this is not something we can support.
self._env.scan([self._dest])
dist = self._env.best_match(requirement, ws)
logger.info("Got %s.", dist)
......@@ -819,6 +860,22 @@ class Installer:
requirements = [self._constrain(pkg_resources.Requirement.parse(spec))
for spec in specs]
def get_hashes(spec):
version = self._versions.get(spec, '')
if '--hash' in version:
# TODO see pip actual implementation
# TODO for test, there can be multiple --hash like with pip
hashes = []
next_is_hash = False
for part in version.split():
if next_is_hash:
hashes.append(part)
next_is_hash = part == '--hash'
return ' '.join(hashes)
return None
hashes_dict = {spec: get_hashes(spec) for spec in specs}
if working_set is None:
ws = pkg_resources.WorkingSet([])
else:
......@@ -829,7 +886,8 @@ class Installer:
self._env.scan(
self.build(str(requirement), {}, patch_dict=patch_dict))
for dist in self._get_dist(requirement, ws,
for_buildout_run=for_buildout_run):
for_buildout_run=for_buildout_run,
hashes=hashes_dict.get(requirement.project_name)):
ws.add(dist)
self._maybe_add_setuptools(ws, dist)
......@@ -906,6 +964,8 @@ class Installer:
requirement = self._constrain(pkg_resources.Requirement.parse(spec))
print("TODO: Why are we here ???") # XXX hash this branch is not covered
import pdb; pdb.set_trace()
dist, avail = self._satisfied(requirement, 1)
if dist is not None:
return [dist.location]
......@@ -1766,6 +1826,8 @@ def redo_pyc(egg):
call_subprocess(args)
def _constrained_requirement(constraint, requirement):
if "--hash" in constraint:
constraint = constraint.split()[0]
if constraint[0] not in '<>':
if constraint.startswith('='):
assert constraint.startswith('==')
......@@ -1865,3 +1927,9 @@ def _move_to_eggs_dir_and_compile(dist, dest):
# Remember that temporary directories must be removed
shutil.rmtree(tmp_dest)
return newloc
# TODO hash sudy more:
# https://pip.pypa.io/en/stable/cli/pip_install/#requirements-file-format
# https://www.python.org/dev/peps/pep-0440/
# https://pip.pypa.io/en/stable/cli/pip_install/#hash-checking-mode
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment