Commit d8f72f75 authored by Julien Muchembled's avatar Julien Muchembled Committed by Xavier Thompson

[feat] easy_install: add slapos.libnetworkcache support

parent faaab49c
...@@ -44,6 +44,8 @@ import zc.buildout.rmtree ...@@ -44,6 +44,8 @@ import zc.buildout.rmtree
from zc.buildout import WINDOWS from zc.buildout import WINDOWS
from zc.buildout import PY3 from zc.buildout import PY3
import warnings import warnings
from contextlib import closing
from setuptools.package_index import distros_for_location, URL_SCHEME
import csv import csv
try: try:
...@@ -114,6 +116,9 @@ python_lib = distutils.sysconfig.get_python_lib() ...@@ -114,6 +116,9 @@ python_lib = distutils.sysconfig.get_python_lib()
FILE_SCHEME = re.compile('file://', re.I).match FILE_SCHEME = re.compile('file://', re.I).match
DUNDER_FILE_PATTERN = re.compile(r"__file__ = '(?P<filename>.+)'$") DUNDER_FILE_PATTERN = re.compile(r"__file__ = '(?P<filename>.+)'$")
networkcache_key = 'pypi:{}={}'.format
class _Monkey(object): class _Monkey(object):
def __init__(self, module, **kw): def __init__(self, module, **kw):
mdict = self._mdict = module.__dict__ mdict = self._mdict = module.__dict__
...@@ -467,32 +472,71 @@ class Installer(object): ...@@ -467,32 +472,71 @@ class Installer(object):
finally: finally:
zc.buildout.rmtree.rmtree(tmp) zc.buildout.rmtree.rmtree(tmp)
def _obtain(self, requirement, source=None): def _obtain(self, requirement, source=None, networkcache_failed=False):
# get the non-patched version # get the non-patched version
req = str(requirement) req = str(requirement)
if PATCH_MARKER in req: if PATCH_MARKER in req:
requirement = pkg_resources.Requirement.parse(re.sub(orig_versions_re, '', req)) requirement = pkg_resources.Requirement.parse(re.sub(orig_versions_re, '', req))
# initialize out index for this project: wheel = getattr(requirement, 'wheel', False)
def filter_precedence(dist):
return (dist.precedence == WHL_DIST) == wheel and (
dist.precedence == pkg_resources.SOURCE_DIST if source
else not (dist.precedence == pkg_resources.DEVELOP_DIST
and {'setup.py', 'pyproject.toml'}.isdisjoint(
os.listdir(dist.location)))
)
index = self._index index = self._index
if not networkcache_failed:
try:
(operator, version,), = requirement.specs
except ValueError:
pass
else:
# Network cache is not expected to contain all versions so it
# couldn't tell whether a found version is the best existing
# one. Therefore, it's only accessed when we have a
# specification for a single version, which is anyway enough
# for our usage (picked versions not allowed).
if operator == '==':
# But first, avoid any network access by checking local
# urls. PackageIndex.add_find_links scans them immediately.
dists = [dist for dist in index[requirement.project_name]
if dist in requirement and filter_precedence(dist) and (
FILE_SCHEME(dist.location) or
not URL_SCHEME(dist.location))]
if dists:
return max(dists)
from .buildout import networkcache_client as nc
if nc:
key = networkcache_key(requirement.key, version)
if nc.tryDownload(key):
with nc:
for entry in nc.select(key):
basename = entry['basename']
for dist in distros_for_location(
entry['sha512'], basename):
# The version comparison is to keep
# the one that's correctly parsed by
# distros_for_location.
if (dist.version == version and
self._env.can_add(dist) and
filter_precedence(dist)):
dist.networkcache = (
basename, requirement, source)
dists.append(dist)
if dists:
return max(dists)
# initialize out index for this project:
if index.obtain(requirement) is None: if index.obtain(requirement) is None:
# Nothing is available. # Nothing is available.
return None return None
# Filter the available dists for the requirement and source flag # Filter the available dists for the requirement and source flag
wheel = getattr(requirement, 'wheel', False)
dists = [dist for dist in index[requirement.project_name] dists = [dist for dist in index[requirement.project_name]
if ((dist in requirement) if dist in requirement and filter_precedence(dist)]
and (dist.precedence == WHL_DIST) == wheel and
(dist.precedence == pkg_resources.SOURCE_DIST if source
else not (dist.precedence == pkg_resources.DEVELOP_DIST
and {'setup.py', 'pyproject.toml'}.isdisjoint(
os.listdir(dist.location))
)
)
)
]
# If we prefer final dists, filter for final and use the # If we prefer final dists, filter for final and use the
# result if it is non empty. # result if it is non empty.
...@@ -529,10 +573,26 @@ class Installer(object): ...@@ -529,10 +573,26 @@ class Installer(object):
): ):
return dist return dist
best.sort() return max(best)
return best[-1]
def _fetch(self, dist, tmp, download_cache): def _fetch(self, dist, tmp, download_cache):
from .buildout import networkcache_client as nc
while hasattr(dist, 'networkcache'):
basename, requirement, source = dist.networkcache
new_location = os.path.join(tmp, basename)
with nc, closing(nc.download(dist.location)) as src, \
open(new_location, 'wb') as dst:
shutil.copyfileobj(src, dst)
break
# Downloading content from network cache failed: let's resume index
# lookup to get a fallback url. This will respect _satisfied()
# decision because the specification is for a single version.
dist = self._obtain(requirement, source, networkcache_failed=True)
if dist is None:
raise zc.buildout.UserError(
"Couldn't find a distribution for %r."
% str(requirement))
else:
if (download_cache if (download_cache
and (realpath(os.path.dirname(dist.location)) == download_cache) and (realpath(os.path.dirname(dist.location)) == download_cache)
): ):
...@@ -541,6 +601,13 @@ class Installer(object): ...@@ -541,6 +601,13 @@ class Installer(object):
logger.debug("Fetching %s from: %s", dist, dist.location) logger.debug("Fetching %s from: %s", dist, dist.location)
new_location = self._index.download(dist.location, tmp) new_location = self._index.download(dist.location, tmp)
if nc:
key = networkcache_key(dist.key, dist.version)
if nc.tryUpload(key):
with nc, open(new_location, 'rb') as f:
nc.upload(f, key,
basename=os.path.basename(new_location))
if (download_cache if (download_cache
and (realpath(new_location) == realpath(dist.location)) and (realpath(new_location) == realpath(dist.location))
and os.path.isfile(new_location) and os.path.isfile(new_location)
......
...@@ -1451,9 +1451,8 @@ Now when we install the distributions: ...@@ -1451,9 +1451,8 @@ Now when we install the distributions:
... ['demo==0.2'], dest, ... ['demo==0.2'], dest,
... links=[link_server], index=link_server+'index/') ... links=[link_server], index=link_server+'index/')
GET 200 / GET 200 /
GET 404 /index/demo/
GET 200 /index/
GET 404 /index/demoneeded/ GET 404 /index/demoneeded/
GET 200 /index/
>>> zc.buildout.easy_install.build( >>> zc.buildout.easy_install.build(
... 'extdemo', dest, ... 'extdemo', dest,
...@@ -1475,6 +1474,7 @@ from the link server: ...@@ -1475,6 +1474,7 @@ from the link server:
>>> ws = zc.buildout.easy_install.install( >>> ws = zc.buildout.easy_install.install(
... ['demo'], dest, ... ['demo'], dest,
... links=[link_server], index=link_server+'index/') ... links=[link_server], index=link_server+'index/')
GET 404 /index/demo/
GET 200 /demo-0.3-py2.4.egg GET 200 /demo-0.3-py2.4.egg
Normally, the download cache is the preferred source of downloads, but Normally, the download cache is the preferred source of downloads, but
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment