Commit fb785cb3 authored by Julien Muchembled's avatar Julien Muchembled

easy_install: add slapos.libnetworkcache support

parent 47ab68e0
Pipeline #20461 failed with stage
in 0 seconds
...@@ -38,6 +38,8 @@ import sys ...@@ -38,6 +38,8 @@ import sys
import tempfile import tempfile
import zc.buildout import zc.buildout
import warnings import warnings
from contextlib import closing
from setuptools.package_index import distros_for_location, URL_SCHEME
WHL_DIST = pkg_resources.EGG_DIST + 1 WHL_DIST = pkg_resources.EGG_DIST + 1
...@@ -97,6 +99,9 @@ python_lib = distutils.sysconfig.get_python_lib() ...@@ -97,6 +99,9 @@ python_lib = distutils.sysconfig.get_python_lib()
FILE_SCHEME = re.compile('file://', re.I).match FILE_SCHEME = re.compile('file://', re.I).match
DUNDER_FILE_PATTERN = re.compile(r"__file__ = '(?P<filename>.+)'$") DUNDER_FILE_PATTERN = re.compile(r"__file__ = '(?P<filename>.+)'$")
networkcache_key = 'pypi:{}={}'.format
class _Monkey(object): class _Monkey(object):
def __init__(self, module, **kw): def __init__(self, module, **kw):
mdict = self._mdict = module.__dict__ mdict = self._mdict = module.__dict__
...@@ -441,29 +446,62 @@ class Installer: ...@@ -441,29 +446,62 @@ class Installer:
finally: finally:
shutil.rmtree(tmp) shutil.rmtree(tmp)
def _obtain(self, requirement, source=None): def _obtain(self, requirement, source=None, networkcache_failed=False):
# get the non-patched version # get the non-patched version
req = str(requirement) req = str(requirement)
if PATCH_MARKER in req: if PATCH_MARKER in req:
requirement = pkg_resources.Requirement.parse(re.sub(orig_versions_re, '', req)) requirement = pkg_resources.Requirement.parse(re.sub(orig_versions_re, '', req))
# initialize out index for this project: wheel = getattr(requirement, 'wheel', False)
def filter_precedence(dist):
return (dist.precedence == WHL_DIST) == wheel and (
not source or dist.precedence == pkg_resources.SOURCE_DIST)
index = self._index index = self._index
if not networkcache_failed:
try:
(operator, version,), = requirement.specs
except ValueError:
pass
else:
# Network cache is not expected to contain all versions so it
# couldn't tell whether a found version is the best existing
# one. Therefore, it's only accessed when we have a
# specification for a single version, which is anyway enough
# for our usage (picked versions not allowed).
if operator == '==':
# But first, avoid any network access by checking local
# urls. PackageIndex.add_find_links scans them immediately.
dists = [dist for dist in index[requirement.project_name]
if dist in requirement and filter_precedence(dist) and (
FILE_SCHEME(dist.location) or
not URL_SCHEME(dist.location))]
if dists:
return max(dists)
from .buildout import networkcache_client as nc
if nc:
key = networkcache_key(requirement.key, version)
if nc.tryDownload(key):
with nc:
for entry in nc.select(key):
basename = entry['basename']
dist = next(iter(distros_for_location(
entry['sha512'], basename)))
if filter_precedence(dist):
dist.networkcache = (
basename, requirement, source)
dists.append(dist)
if dists:
return max(dists)
# initialize out index for this project:
if index.obtain(requirement) is None: if index.obtain(requirement) is None:
# Nothing is available. # Nothing is available.
return None return None
# Filter the available dists for the requirement and source flag # Filter the available dists for the requirement and source flag
wheel = getattr(requirement, 'wheel', False)
dists = [dist for dist in index[requirement.project_name] dists = [dist for dist in index[requirement.project_name]
if ((dist in requirement) if dist in requirement and filter_precedence(dist)]
and (dist.precedence == WHL_DIST) == wheel and
((not source) or
(dist.precedence == pkg_resources.SOURCE_DIST)
)
)
]
# If we prefer final dists, filter for final and use the # If we prefer final dists, filter for final and use the
# result if it is non empty. # result if it is non empty.
...@@ -500,16 +538,38 @@ class Installer: ...@@ -500,16 +538,38 @@ class Installer:
): ):
return dist return dist
best.sort() return max(best)
return best[-1]
def _fetch(self, dist, tmp, download_cache): def _fetch(self, dist, tmp, download_cache):
if (download_cache from .buildout import networkcache_client as nc
and (realpath(os.path.dirname(dist.location)) == download_cache) while hasattr(dist, 'networkcache'):
): basename, requirement, source = dist.networkcache
return dist new_location = os.path.join(tmp, basename)
with nc, closing(nc.download(dist.location)) as src, \
open(new_location, 'wb') as dst:
shutil.copyfileobj(src, dst)
break
# Downloading content from network cache failed: let's resume index
# lookup to get a fallback url. This will respect _satisfied()
# decision because the specification is for a single version.
dist = self._obtain(requirement, source, networkcache_failed=True)
if dist is None:
raise zc.buildout.UserError(
"Couldn't find a distribution for %r."
% str(requirement))
else:
if download_cache and (
realpath(os.path.dirname(dist.location)) == download_cache):
return dist
new_location = self._index.download(dist.location, tmp)
if nc:
key = networkcache_key(dist.key, dist.version)
if nc.tryUpload(key):
with nc, open(new_location, 'rb') as f:
nc.upload(f, key,
basename=os.path.basename(new_location))
new_location = self._index.download(dist.location, tmp)
if (download_cache if (download_cache
and (realpath(new_location) == realpath(dist.location)) and (realpath(new_location) == realpath(dist.location))
and os.path.isfile(new_location) and os.path.isfile(new_location)
......
...@@ -1362,9 +1362,8 @@ Now when we install the distributions: ...@@ -1362,9 +1362,8 @@ Now when we install the distributions:
... ['demo==0.2'], dest, ... ['demo==0.2'], dest,
... links=[link_server], index=link_server+'index/') ... links=[link_server], index=link_server+'index/')
GET 200 / GET 200 /
GET 404 /index/demo/
GET 200 /index/
GET 404 /index/demoneeded/ GET 404 /index/demoneeded/
GET 200 /index/
>>> zc.buildout.easy_install.build( >>> zc.buildout.easy_install.build(
... 'extdemo', dest, ... 'extdemo', dest,
...@@ -1386,6 +1385,7 @@ from the link server: ...@@ -1386,6 +1385,7 @@ from the link server:
>>> ws = zc.buildout.easy_install.install( >>> ws = zc.buildout.easy_install.install(
... ['demo'], dest, ... ['demo'], dest,
... links=[link_server], index=link_server+'index/') ... links=[link_server], index=link_server+'index/')
GET 404 /index/demo/
GET 200 /demo-0.3-py2.4.egg GET 200 /demo-0.3-py2.4.egg
Normally, the download cache is the preferred source of downloads, but Normally, the download cache is the preferred source of downloads, but
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment