Commit fb785cb3 authored by Julien Muchembled's avatar Julien Muchembled

easy_install: add slapos.libnetworkcache support

parent 47ab68e0
Pipeline #20461 failed with stage
in 0 seconds
......@@ -38,6 +38,8 @@ import sys
import tempfile
import zc.buildout
import warnings
from contextlib import closing
from setuptools.package_index import distros_for_location, URL_SCHEME
WHL_DIST = pkg_resources.EGG_DIST + 1
......@@ -97,6 +99,9 @@ python_lib = distutils.sysconfig.get_python_lib()
FILE_SCHEME = re.compile('file://', re.I).match
DUNDER_FILE_PATTERN = re.compile(r"__file__ = '(?P<filename>.+)'$")
networkcache_key = 'pypi:{}={}'.format
class _Monkey(object):
def __init__(self, module, **kw):
mdict = self._mdict = module.__dict__
......@@ -441,29 +446,62 @@ class Installer:
def _obtain(self, requirement, source=None):
def _obtain(self, requirement, source=None, networkcache_failed=False):
# get the non-patched version
req = str(requirement)
if PATCH_MARKER in req:
requirement = pkg_resources.Requirement.parse(re.sub(orig_versions_re, '', req))
# initialize out index for this project:
wheel = getattr(requirement, 'wheel', False)
def filter_precedence(dist):
return (dist.precedence == WHL_DIST) == wheel and (
not source or dist.precedence == pkg_resources.SOURCE_DIST)
index = self._index
if not networkcache_failed:
(operator, version,), = requirement.specs
except ValueError:
# Network cache is not expected to contain all versions so it
# couldn't tell whether a found version is the best existing
# one. Therefore, it's only accessed when we have a
# specification for a single version, which is anyway enough
# for our usage (picked versions not allowed).
if operator == '==':
# But first, avoid any network access by checking local
# urls. PackageIndex.add_find_links scans them immediately.
dists = [dist for dist in index[requirement.project_name]
if dist in requirement and filter_precedence(dist) and (
FILE_SCHEME(dist.location) or
not URL_SCHEME(dist.location))]
if dists:
return max(dists)
from .buildout import networkcache_client as nc
if nc:
key = networkcache_key(requirement.key, version)
if nc.tryDownload(key):
with nc:
for entry in
basename = entry['basename']
dist = next(iter(distros_for_location(
entry['sha512'], basename)))
if filter_precedence(dist):
dist.networkcache = (
basename, requirement, source)
if dists:
return max(dists)
# initialize out index for this project:
if index.obtain(requirement) is None:
# Nothing is available.
return None
# Filter the available dists for the requirement and source flag
wheel = getattr(requirement, 'wheel', False)
dists = [dist for dist in index[requirement.project_name]
if ((dist in requirement)
and (dist.precedence == WHL_DIST) == wheel and
((not source) or
(dist.precedence == pkg_resources.SOURCE_DIST)
if dist in requirement and filter_precedence(dist)]
# If we prefer final dists, filter for final and use the
# result if it is non empty.
......@@ -500,16 +538,38 @@ class Installer:
return dist
return best[-1]
return max(best)
def _fetch(self, dist, tmp, download_cache):
if (download_cache
and (realpath(os.path.dirname(dist.location)) == download_cache)
return dist
from .buildout import networkcache_client as nc
while hasattr(dist, 'networkcache'):
basename, requirement, source = dist.networkcache
new_location = os.path.join(tmp, basename)
with nc, closing( as src, \
open(new_location, 'wb') as dst:
shutil.copyfileobj(src, dst)
# Downloading content from network cache failed: let's resume index
# lookup to get a fallback url. This will respect _satisfied()
# decision because the specification is for a single version.
dist = self._obtain(requirement, source, networkcache_failed=True)
if dist is None:
raise zc.buildout.UserError(
"Couldn't find a distribution for %r."
% str(requirement))
if download_cache and (
realpath(os.path.dirname(dist.location)) == download_cache):
return dist
new_location =, tmp)
if nc:
key = networkcache_key(dist.key, dist.version)
if nc.tryUpload(key):
with nc, open(new_location, 'rb') as f:
nc.upload(f, key,
new_location =, tmp)
if (download_cache
and (realpath(new_location) == realpath(dist.location))
and os.path.isfile(new_location)
......@@ -1362,9 +1362,8 @@ Now when we install the distributions:
... ['demo==0.2'], dest,
... links=[link_server], index=link_server+'index/')
GET 200 /
GET 404 /index/demo/
GET 200 /index/
GET 404 /index/demoneeded/
GET 200 /index/
... 'extdemo', dest,
......@@ -1386,6 +1385,7 @@ from the link server:
>>> ws = zc.buildout.easy_install.install(
... ['demo'], dest,
... links=[link_server], index=link_server+'index/')
GET 404 /index/demo/
GET 200 /demo-0.3-py2.4.egg
Normally, the download cache is the preferred source of downloads, but
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment