Commit d8f72f75 authored by Julien Muchembled's avatar Julien Muchembled Committed by Xavier Thompson

[feat] easy_install: add slapos.libnetworkcache support

parent faaab49c
......@@ -44,6 +44,8 @@ import zc.buildout.rmtree
from zc.buildout import WINDOWS
from zc.buildout import PY3
import warnings
from contextlib import closing
from setuptools.package_index import distros_for_location, URL_SCHEME
import csv
try:
......@@ -114,6 +116,9 @@ python_lib = distutils.sysconfig.get_python_lib()
FILE_SCHEME = re.compile('file://', re.I).match
DUNDER_FILE_PATTERN = re.compile(r"__file__ = '(?P<filename>.+)'$")
networkcache_key = 'pypi:{}={}'.format
class _Monkey(object):
def __init__(self, module, **kw):
mdict = self._mdict = module.__dict__
......@@ -467,32 +472,71 @@ class Installer(object):
finally:
zc.buildout.rmtree.rmtree(tmp)
def _obtain(self, requirement, source=None):
def _obtain(self, requirement, source=None, networkcache_failed=False):
# get the non-patched version
req = str(requirement)
if PATCH_MARKER in req:
requirement = pkg_resources.Requirement.parse(re.sub(orig_versions_re, '', req))
# initialize out index for this project:
wheel = getattr(requirement, 'wheel', False)
def filter_precedence(dist):
return (dist.precedence == WHL_DIST) == wheel and (
dist.precedence == pkg_resources.SOURCE_DIST if source
else not (dist.precedence == pkg_resources.DEVELOP_DIST
and {'setup.py', 'pyproject.toml'}.isdisjoint(
os.listdir(dist.location)))
)
index = self._index
if not networkcache_failed:
try:
(operator, version,), = requirement.specs
except ValueError:
pass
else:
# Network cache is not expected to contain all versions so it
# couldn't tell whether a found version is the best existing
# one. Therefore, it's only accessed when we have a
# specification for a single version, which is anyway enough
# for our usage (picked versions not allowed).
if operator == '==':
# But first, avoid any network access by checking local
# urls. PackageIndex.add_find_links scans them immediately.
dists = [dist for dist in index[requirement.project_name]
if dist in requirement and filter_precedence(dist) and (
FILE_SCHEME(dist.location) or
not URL_SCHEME(dist.location))]
if dists:
return max(dists)
from .buildout import networkcache_client as nc
if nc:
key = networkcache_key(requirement.key, version)
if nc.tryDownload(key):
with nc:
for entry in nc.select(key):
basename = entry['basename']
for dist in distros_for_location(
entry['sha512'], basename):
# The version comparison is to keep
# the one that's correctly parsed by
# distros_for_location.
if (dist.version == version and
self._env.can_add(dist) and
filter_precedence(dist)):
dist.networkcache = (
basename, requirement, source)
dists.append(dist)
if dists:
return max(dists)
# initialize out index for this project:
if index.obtain(requirement) is None:
# Nothing is available.
return None
# Filter the available dists for the requirement and source flag
wheel = getattr(requirement, 'wheel', False)
dists = [dist for dist in index[requirement.project_name]
if ((dist in requirement)
and (dist.precedence == WHL_DIST) == wheel and
(dist.precedence == pkg_resources.SOURCE_DIST if source
else not (dist.precedence == pkg_resources.DEVELOP_DIST
and {'setup.py', 'pyproject.toml'}.isdisjoint(
os.listdir(dist.location))
)
)
)
]
if dist in requirement and filter_precedence(dist)]
# If we prefer final dists, filter for final and use the
# result if it is non empty.
......@@ -529,10 +573,26 @@ class Installer(object):
):
return dist
best.sort()
return best[-1]
return max(best)
def _fetch(self, dist, tmp, download_cache):
from .buildout import networkcache_client as nc
while hasattr(dist, 'networkcache'):
basename, requirement, source = dist.networkcache
new_location = os.path.join(tmp, basename)
with nc, closing(nc.download(dist.location)) as src, \
open(new_location, 'wb') as dst:
shutil.copyfileobj(src, dst)
break
# Downloading content from network cache failed: let's resume index
# lookup to get a fallback url. This will respect _satisfied()
# decision because the specification is for a single version.
dist = self._obtain(requirement, source, networkcache_failed=True)
if dist is None:
raise zc.buildout.UserError(
"Couldn't find a distribution for %r."
% str(requirement))
else:
if (download_cache
and (realpath(os.path.dirname(dist.location)) == download_cache)
):
......@@ -541,6 +601,13 @@ class Installer(object):
logger.debug("Fetching %s from: %s", dist, dist.location)
new_location = self._index.download(dist.location, tmp)
if nc:
key = networkcache_key(dist.key, dist.version)
if nc.tryUpload(key):
with nc, open(new_location, 'rb') as f:
nc.upload(f, key,
basename=os.path.basename(new_location))
if (download_cache
and (realpath(new_location) == realpath(dist.location))
and os.path.isfile(new_location)
......
......@@ -1451,9 +1451,8 @@ Now when we install the distributions:
... ['demo==0.2'], dest,
... links=[link_server], index=link_server+'index/')
GET 200 /
GET 404 /index/demo/
GET 200 /index/
GET 404 /index/demoneeded/
GET 200 /index/
>>> zc.buildout.easy_install.build(
... 'extdemo', dest,
......@@ -1475,6 +1474,7 @@ from the link server:
>>> ws = zc.buildout.easy_install.install(
... ['demo'], dest,
... links=[link_server], index=link_server+'index/')
GET 404 /index/demo/
GET 200 /demo-0.3-py2.4.egg
Normally, the download cache is the preferred source of downloads, but
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment