Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
S
slapos.buildout
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
0
Issues
0
List
Boards
Labels
Milestones
Merge Requests
0
Merge Requests
0
Analytics
Analytics
Repository
Value Stream
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Commits
Issue Boards
Open sidebar
Rafael Monnerat
slapos.buildout
Commits
84d9c96b
Commit
84d9c96b
authored
Aug 28, 2012
by
Cédric de Saint Martin
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
Cache pypi index, extensions and recipes using buildout internals.
parent
b0bcdf03
Changes
3
Hide whitespace changes
Inline
Side-by-side
Showing
3 changed files
with
303 additions
and
58 deletions
+303
-58
src/zc/buildout/buildout.py
src/zc/buildout/buildout.py
+11
-1
src/zc/buildout/easy_install.py
src/zc/buildout/easy_install.py
+174
-26
src/zc/buildout/networkcache.py
src/zc/buildout/networkcache.py
+118
-31
No files found.
src/zc/buildout/buildout.py
View file @
84d9c96b
...
@@ -1193,7 +1193,17 @@ class Buildout(UserDict.DictMixin):
...
@@ -1193,7 +1193,17 @@ class Buildout(UserDict.DictMixin):
links
=
self
[
'buildout'
].
get
(
'find-links'
,
''
).
split
(),
links
=
self
[
'buildout'
].
get
(
'find-links'
,
''
).
split
(),
index
=
self
[
'buildout'
].
get
(
'index'
),
index
=
self
[
'buildout'
].
get
(
'index'
),
newest
=
self
.
newest
,
allow_hosts
=
self
.
_allow_hosts
,
newest
=
self
.
newest
,
allow_hosts
=
self
.
_allow_hosts
,
prefer_final
=
not
self
.
accept_buildout_test_releases
)
prefer_final
=
not
self
.
accept_buildout_test_releases
,
download_cache_url
=
self
.
download_cache_url
,
download_dir_url
=
self
.
download_dir_url
,
upload_cache_url
=
self
.
upload_cache_url
,
upload_dir_url
=
self
.
upload_dir_url
,
signature_private_key_file
=
self
.
signature_private_key_file
,
signature_certificate_list
=
self
.
signature_certificate_list
,
shacache_cert_file
=
self
.
shacache_cert_file
,
shacache_key_file
=
self
.
shacache_key_file
,
shadir_cert_file
=
self
.
shadir_cert_file
,
shadir_key_file
=
self
.
shadir_key_file
,)
# Clear cache because extensions might now let us read pages we
# Clear cache because extensions might now let us read pages we
# couldn't read before.
# couldn't read before.
...
...
src/zc/buildout/easy_install.py
View file @
84d9c96b
...
@@ -44,7 +44,9 @@ def realpath(path):
...
@@ -44,7 +44,9 @@ def realpath(path):
from
zc.buildout.networkcache
import
get_filename_from_url
,
\
from
zc.buildout.networkcache
import
get_filename_from_url
,
\
upload_network_cached
,
\
upload_network_cached
,
\
download_network_cached
download_network_cached
,
\
download_index_network_cached
,
\
upload_index_network_cached
default_index_url
=
os
.
environ
.
get
(
default_index_url
=
os
.
environ
.
get
(
...
@@ -199,22 +201,137 @@ def _get_version(executable):
...
@@ -199,22 +201,137 @@ def _get_version(executable):
return version
return version
FILE_SCHEME = re.compile('
file
:
//
', re.I).match
FILE_SCHEME = re.compile('
file
:
//
', re.I).match
URL_SCHEME = re.compile('
([
-+
.
a
-
z0
-
9
]{
2
,}):
',re.I).match
HREF = re.compile("""href
\
\
s*=
\
\
s*['"]?([^'"
>
]
+
)
""", re.I)
from setuptools.package_index import distros_for_url, htmldecode
import urlparse
class AllowHostsPackageIndex(setuptools.package_index.PackageIndex):
class AllowHostsPackageIndex(setuptools.package_index.PackageIndex):
"""
Will
allow
urls
that
are
local
to
the
system
.
"""
Will
allow
urls
that
are
local
to
the
system
.
No
matter
what
is
allow_hosts
.
No
matter
what
is
allow_hosts
.
"""
"""
def __init__(self, index_url, hosts, search_path, python,):
setuptools.package_index.PackageIndex.__init__(self,
index_url, hosts=hosts, search_path=search_path,
python=python)
def url_ok(self, url, fatal=False):
def url_ok(self, url, fatal=False):
if FILE_SCHEME(url):
if FILE_SCHEME(url):
return True
return True
return setuptools.package_index.PackageIndex.url_ok(self, url, False)
return setuptools.package_index.PackageIndex.url_ok(self, url, False)
def obtain(self, requirement, installer=None):
# XXX BEGIN HARDCODE
self._current_requirement = requirement.__str__()
# XXX END HARDCODE
self.prescan(); self.find_packages(requirement)
for dist in self[requirement.key]:
if dist in requirement:
return dist
self.debug("%s does not match %s", requirement, dist)
return super(PackageIndex, self).obtain(requirement,installer)
def find_packages(self, requirement):
# XXX BEGIN HARDCODE
self._current_requirement = requirement.__str__()
# XXX END HARDCODE
self.scan_url(self.index_url + requirement.unsafe_name+'/')
if not self.package_pages.get(requirement.key):
# Fall back to safe version of the name
self.scan_url(self.index_url + requirement.project_name+'/')
if not self.package_pages.get(requirement.key):
# We couldn't find the target package, so search the index page too
self.not_found_in_index(requirement)
for url in list(self.package_pages.get(requirement.key,())):
# scan each page that might be related to the desired package
self.scan_url(url)
def process_url(self, url, retrieve=False):
"""
Evaluate
a
URL
as
a
possible
download
,
and
maybe
retrieve
it
"""
if url in self.scanned_urls and not retrieve:
return
self.scanned_urls[url] = True
if not URL_SCHEME(url):
self.process_filename(url)
return
else:
dists = list(distros_for_url(url))
if dists:
if not self.url_ok(url):
return
self.debug("Found link: %s", url)
if dists or not retrieve or url in self.fetched_urls:
map(self.add, dists)
return # don't need the actual page
if not self.url_ok(url):
self.fetched_urls[url] = True
return
self.info("Reading %s", url)
self.fetched_urls[url] = True # prevent multiple fetch attempts
# We download from cache only if we have a specific version for egg requirement
# Reason: If no specific version, we don't want to be blocked with an old index.
# If there is specific version, we want index to freeze forever.
download_result = False
if getattr(self, '_current_requirement', None)
\
and "==" in self._current_requirement
\
and globals().get('_download_dir_url')
\
and globals().get('_download_dir_url')
\
and globals().get('_signature_certificate_list'):
download_result = download_index_network_cached(_download_dir_url,
_download_cache_url, url, self._current_requirement, logger,
_signature_certificate_list)
if download_result:
# Successfully fetched from cache. Hardcode some things...
page, base = download_result
f = None
self.fetched_urls[base] = True
else:
f = self.open_url(url, "Download error: %s -- Some packages may not be found!")
if f is None: return
self.fetched_urls[f.url] = True
if 'html' not in f.headers.get('content-type', '').lower():
f.close() # not html, we can't process it
return
base = f.url # handle redirects
page = f.read()
f.close()
# Check if shacache options exist in global vars, then check if
# it is not null
if globals().get('_upload_cache_url')
\
and globals().get('_upload_dir_url')
\
and getattr(self, '_current_requirement', None)
\
and "==" in self._current_requirement
\
and getattr(f,'code',None)!=404:
upload_index_network_cached(_upload_dir_url,
_upload_cache_url, url, base, self._current_requirement, page, logger,
globals().get('_signature_private_key_file'),
globals().get('_shacache_cert_file'),
globals().get('_shacache_key_file'),
globals().get('_shadir_cert_file'),
globals().get('_shadir_key_file'))
if url.startswith(self.index_url) and getattr(f,'code',None)!=404:
page = self.process_index(url, page)
for match in HREF.finditer(page):
link = urlparse.urljoin(base, htmldecode(match.group(1)))
self.process_url(link)
_indexes = {}
_indexes = {}
def _get_index(executable, index_url, find_links, allow_hosts=('*',),
def _get_index(executable, index_url, find_links, allow_hosts=('*',),
path=None):
path=None
,
):
# If path is None, the index will use sys.path. If you provide an empty
# If path is None, the index will use sys.path. If you provide an empty
# path ([]), it will complain uselessly about missing index pages for
# path ([]), it will complain uselessly about missing index pages for
# packages found in the paths that you expect to use. Therefore, this path
# packages found in the paths that you expect to use. Therefore, this path
...
@@ -228,7 +345,7 @@ def _get_index(executable, index_url, find_links, allow_hosts=('*',),
...
@@ -228,7 +345,7 @@ def _get_index(executable, index_url, find_links, allow_hosts=('*',),
index_url = default_index_url
index_url = default_index_url
index = AllowHostsPackageIndex(
index = AllowHostsPackageIndex(
index_url, hosts=allow_hosts, search_path=path,
index_url, hosts=allow_hosts, search_path=path,
python=_get_version(executable)
python=_get_version(executable)
,
)
)
if find_links:
if find_links:
...
@@ -352,6 +469,39 @@ class Installer:
...
@@ -352,6 +469,39 @@ class Installer:
shadir_cert_file=None,
shadir_cert_file=None,
shadir_key_file=None,
shadir_key_file=None,
):
):
# XXX Setting shacahe options to global variables in order to be able
# to use them from recipes not aware of it. I now hate myself.
if download_dir_url:
global _download_dir_url
_download_dir_url = download_dir_url
if download_cache_url:
global _download_cache_url
_download_cache_url = download_cache_url
if upload_dir_url:
global _upload_dir_url
_upload_dir_url = upload_dir_url
if upload_cache_url:
global _upload_cache_url
_upload_cache_url = upload_cache_url
if signature_certificate_list:
global _signature_certificate_list
_signature_certificate_list = signature_certificate_list
if signature_private_key_file:
global _signature_private_key_file
_signature_private_key_file = signature_private_key_file
if shacache_cert_file:
global _shacache_cert_file
_shacache_cert_file = shacache_cert_file
if shacache_key_file:
global _shacache_key_file
_shacache_key_file = shacache_key_file
if shadir_cert_file:
global _shadir_cert_file
_shadir_cert_file = shadir_cert_file
if shadir_key_file:
global _shadir_key_file
_shadir_key_file = shadir_key_file
self._dest = dest
self._dest = dest
self._allow_hosts = allow_hosts
self._allow_hosts = allow_hosts
...
@@ -414,22 +564,11 @@ class Installer:
...
@@ -414,22 +564,11 @@ class Installer:
self._env = pkg_resources.Environment(path,
self._env = pkg_resources.Environment(path,
python=_get_version(executable))
python=_get_version(executable))
self._index = _get_index(executable, index, links, self._allow_hosts,
self._index = _get_index(executable, index, links, self._allow_hosts,
self
.
_path
)
self._path
,
)
if versions is not None:
if versions is not None:
self._versions = versions
self._versions = versions
self
.
_download_dir_url
=
download_dir_url
self
.
_download_cache_url
=
download_cache_url
self
.
_upload_dir_url
=
upload_dir_url
self
.
_upload_cache_url
=
upload_cache_url
self
.
_signature_certificate_list
=
signature_certificate_list
self
.
_signature_private_key_file
=
signature_private_key_file
self
.
_shacache_cert_file
=
shacache_cert_file
self
.
_shacache_key_file
=
shacache_key_file
self
.
_shadir_cert_file
=
shadir_cert_file
self
.
_shadir_key_file
=
shadir_key_file
_allowed_eggs_from_site_packages_regex = None
_allowed_eggs_from_site_packages_regex = None
def allow_site_package_egg(self, name):
def allow_site_package_egg(self, name):
if (not self._include_site_packages or
if (not self._include_site_packages or
...
@@ -731,16 +870,25 @@ class Installer:
...
@@ -731,16 +870,25 @@ class Installer:
filename = get_filename_from_url(dist.location)
filename = get_filename_from_url(dist.location)
new_location = os.path.join(tmp, filename)
new_location = os.path.join(tmp, filename)
if
not
download_network_cached
(
self
.
_download_dir_url
,
# Try to download from shacache first. If not possible, downloads from
self
.
_download_cache_url
,
new_location
,
dist
.
location
,
logger
,
# Original location and uploads it to shacache.
self
.
_signature_certificate_list
):
downloaded_from_cache = False
if globals().get('_download_dir_url')
\
and globals().get('_download_dir_url')
\
and globals().get('_signature_certificate_list'):
downloaded_from_cache = download_network_cached(_download_dir_url,
_download_cache_url, new_location, dist.location, logger,
_signature_certificate_list)
if not downloaded_from_cache:
new_location = self._index.download(dist.location, tmp)
new_location = self._index.download(dist.location, tmp)
if
self
.
_upload_cache_url
and
self
.
_upload_dir_url
:
# XXX Better check for unbound global variables
upload_network_cached
(
self
.
_upload_dir_url
,
if globals().get('_upload_cache_url')
\
self
.
_upload_cache_url
,
dist
.
location
,
new_location
,
logger
,
and globals().get('_upload_dir_url'):
self
.
_signature_private_key_file
,
upload_network_cached(_upload_dir_url,
self
.
_shacache_cert_file
,
self
.
_shacache_key_file
,
_upload_cache_url, dist.location, new_location, logger,
self
.
_shadir_cert_file
,
self
.
_shadir_key_file
)
_signature_private_key_file,
_shacache_cert_file, _shacache_key_file,
_shadir_cert_file, _shadir_key_file)
if (download_cache
if (download_cache
and (realpath(new_location) == realpath(dist.location))
and (realpath(new_location) == realpath(dist.location))
...
@@ -854,7 +1002,7 @@ class Installer:
...
@@ -854,7 +1002,7 @@ class Installer:
self._links.append(link)
self._links.append(link)
self._index = _get_index(self._executable,
self._index = _get_index(self._executable,
self._index_url, self._links,
self._index_url, self._links,
self
.
_allow_hosts
,
self
.
_path
)
self._allow_hosts, self._path
,
)
for dist in dists:
for dist in dists:
# Check whether we picked a version and, if we did, report it:
# Check whether we picked a version and, if we did, report it:
...
...
src/zc/buildout/networkcache.py
View file @
84d9c96b
...
@@ -12,12 +12,12 @@
...
@@ -12,12 +12,12 @@
#
#
##############################################################################
##############################################################################
#XXX factor with slapos/grid/networkcache.py and use libnetworkcache helpers
import
hashlib
import
hashlib
import
os
import
os
import
posixpath
import
posixpath
import
re
import
re
import
shutil
import
urllib2
import
urllib2
import
urlparse
import
urlparse
import
traceback
import
traceback
...
@@ -25,7 +25,11 @@ import traceback
...
@@ -25,7 +25,11 @@ import traceback
try
:
try
:
try
:
try
:
from
slapos.libnetworkcache
import
NetworkcacheClient
,
UploadError
,
\
from
slapos.libnetworkcache
import
NetworkcacheClient
,
UploadError
,
\
DirectoryNotFound
DirectoryNotFound
from
slapos.networkcachehelper
import
\
helper_download_network_cached
,
\
helper_upload_network_cached_from_file
,
\
helper_download_network_cached_to_file
except
ImportError
:
except
ImportError
:
LIBNETWORKCACHE_ENABLED
=
False
LIBNETWORKCACHE_ENABLED
=
False
else
:
else
:
...
@@ -68,12 +72,19 @@ def get_directory_key(url):
...
@@ -68,12 +72,19 @@ def get_directory_key(url):
Basically check if the url belongs to pypi:
Basically check if the url belongs to pypi:
- if yes, the directory key will be pypi-buildout-urlmd5
- if yes, the directory key will be pypi-buildout-urlmd5
- if not, the directory key will be slapos-buildout-urlmd5
- if not, the directory key will be slapos-buildout-urlmd5
# XXX why is that?
"""
"""
urlmd5
=
hashlib
.
md5
(
url
).
hexdigest
()
urlmd5
=
hashlib
.
md5
(
url
).
hexdigest
()
if
'pypi'
in
url
:
if
'pypi'
in
url
:
return
'pypi-buildout-%s'
%
urlmd5
return
'pypi-buildout-%s'
%
urlmd5
return
'slapos-buildout-%s'
%
urlmd5
return
'slapos-buildout-%s'
%
urlmd5
@
fallback_call
def
get_index_directory_key
(
url
,
requirement
):
"""Returns directory hash based on egg requirement.
"""
return
'pypi-index-%s-%s'
%
(
hashlib
.
md5
(
url
).
hexdigest
(),
requirement
)
@
fallback_call
@
fallback_call
def
download_network_cached
(
dir_url
,
cache_url
,
path
,
url
,
logger
,
def
download_network_cached
(
dir_url
,
cache_url
,
path
,
url
,
logger
,
...
@@ -88,56 +99,74 @@ def download_network_cached(dir_url, cache_url, path, url, logger,
...
@@ -88,56 +99,74 @@ def download_network_cached(dir_url, cache_url, path, url, logger,
if
not
LIBNETWORKCACHE_ENABLED
:
if
not
LIBNETWORKCACHE_ENABLED
:
return
False
return
False
if
not
(
dir_url
and
cache_url
):
return
False
if
md5sum
is
None
:
if
md5sum
is
None
:
md5sum
=
_get_md5_from_url
(
url
)
md5sum
=
_get_md5_from_url
(
url
)
directory_key
=
get_directory_key
(
url
)
directory_key
=
get_directory_key
(
url
)
url
=
os
.
path
.
basename
(
url
)
url
=
os
.
path
.
basename
(
url
)
if
len
(
signature_certificate_list
)
==
0
:
# convert [] into None in order to call nc nicely
signature_certificate_list
=
None
try
:
nc
=
NetworkcacheClient
(
cache_url
,
dir_url
,
signature_certificate_list
=
signature_certificate_list
)
except
TypeError
:
logger
.
warning
(
'Incompatible version of networkcache, not using it.'
)
return
False
logger
.
debug
(
'Trying to download %s from network cache...'
%
url
)
logger
.
debug
(
'Trying to download %s from network cache...'
%
url
)
try
:
file_descriptor
=
nc
.
select
(
directory_key
)
f
=
open
(
path
,
'w+b'
)
if
helper_download_network_cached_to_file
(
try
:
dir_url
=
dir_url
,
shutil
.
copyfileobj
(
file_descriptor
,
f
)
cache_url
=
cache_url
,
finally
:
signature_certificate_list
=
signature_certificate_list
,
f
.
close
()
directory_key
=
directory_key
,
file_descriptor
.
close
()
path
=
path
):
logger
.
info
(
'Downloaded %s from network cache.'
%
url
)
logger
.
info
(
'Downloaded %s from network cache.'
%
url
)
if
not
check_md5sum
(
path
,
md5sum
):
if
not
check_md5sum
(
path
,
md5sum
):
logger
.
info
(
'MD5 checksum mismatch downloading %s'
%
url
)
logger
.
info
(
'MD5 checksum mismatch downloading %s'
%
url
)
return
False
return
False
return
True
return
False
except
(
IOError
,
DirectoryNotFound
),
e
:
@
fallback_call
if
isinstance
(
e
,
urllib2
.
HTTPError
)
and
e
.
code
==
404
:
def
download_index_network_cached
(
dir_url
,
cache_url
,
url
,
requirement
,
logger
,
logger
.
debug
(
'%s does not exist in network cache.'
%
url
)
signature_certificate_list
):
else
:
"""
logger
.
debug
(
'Failed to download from network cache %s: %s'
%
\
XXX description
(
url
,
str
(
e
)))
Downloads pypi index from a network cache provider
If something fail (providor be offline, or hash_string fail), we ignore
network cached index.
return index if succeeded, False otherwise.
"""
if
not
LIBNETWORKCACHE_ENABLED
:
return
False
return
False
return
True
directory_key
=
get_index_directory_key
(
url
,
requirement
)
wanted_metadata_dict
=
{
'urlmd5'
:
hashlib
.
md5
(
url
).
hexdigest
(),
'requirement'
:
requirement
}
required_key_list
=
[
'base'
]
result
=
helper_download_network_cached
(
dir_url
,
cache_url
,
signature_certificate_list
,
directory_key
,
wanted_metadata_dict
,
required_key_list
)
if
result
:
file_descriptor
,
metadata
=
result
try
:
content
=
file_descriptor
.
read
()
logger
.
info
(
'Downloaded %s from network cache.'
%
url
)
return
content
,
metadata
[
'base'
]
except
(
IOError
,
DirectoryNotFound
),
e
:
if
isinstance
(
e
,
urllib2
.
HTTPError
)
and
e
.
code
==
404
:
logger
.
debug
(
'%s does not exist in network cache.'
%
url
)
else
:
logger
.
debug
(
'Failed to download from network cache %s: %s'
%
\
(
url
,
str
(
e
)))
return
False
@
fallback_call
@
fallback_call
def
upload_network_cached
(
dir_url
,
cache_url
,
external_url
,
path
,
logger
,
def
upload_network_cached
(
dir_url
,
cache_url
,
external_url
,
path
,
logger
,
signature_private_key_file
,
shacache_cert_file
,
shacache_key_file
,
signature_private_key_file
,
shacache_cert_file
,
shacache_key_file
,
shadir_cert_file
,
shadir_key_file
):
shadir_cert_file
,
shadir_key_file
):
"""Upload file to a network cache server"""
"""Upload file to a network cache server"""
# XXX use helper and FACTOR code
if
not
LIBNETWORKCACHE_ENABLED
:
if
not
LIBNETWORKCACHE_ENABLED
:
return
False
return
False
...
@@ -187,6 +216,64 @@ def upload_network_cached(dir_url, cache_url, external_url, path, logger,
...
@@ -187,6 +216,64 @@ def upload_network_cached(dir_url, cache_url, external_url, path, logger,
return
True
return
True
@
fallback_call
def
upload_index_network_cached
(
dir_url
,
cache_url
,
external_url
,
base
,
requirement
,
content
,
logger
,
signature_private_key_file
,
shacache_cert_file
,
shacache_key_file
,
shadir_cert_file
,
shadir_key_file
):
# XXX use helper and FACTOR code
"""Upload content of a web page to a network cache server"""
if
not
LIBNETWORKCACHE_ENABLED
:
return
False
if
not
(
dir_url
and
cache_url
):
return
False
logger
.
info
(
'Uploading %s content into network cache.'
%
external_url
)
directory_key
=
get_index_directory_key
(
external_url
,
requirement
)
kw
=
dict
(
file
=
"file"
,
base
=
base
,
urlmd5
=
hashlib
.
md5
(
external_url
).
hexdigest
(),
requirement
=
requirement
)
import
tempfile
f
=
tempfile
.
TemporaryFile
()
f
.
write
(
content
)
# convert '' into None in order to call nc nicely
if
not
signature_private_key_file
:
signature_private_key_file
=
None
if
not
shacache_cert_file
:
shacache_cert_file
=
None
if
not
shacache_key_file
:
shacache_key_file
=
None
if
not
shadir_cert_file
:
shadir_cert_file
=
None
if
not
shadir_key_file
:
shadir_key_file
=
None
try
:
nc
=
NetworkcacheClient
(
cache_url
,
dir_url
,
signature_private_key_file
=
signature_private_key_file
,
shacache_cert_file
=
shacache_cert_file
,
shacache_key_file
=
shacache_key_file
,
shadir_cert_file
=
shadir_cert_file
,
shadir_key_file
=
shadir_key_file
)
except
TypeError
:
logger
.
warning
(
'Incompatible version of networkcache, not using it.'
)
return
False
try
:
return
nc
.
upload_generic
(
f
,
directory_key
,
**
kw
)
except
(
IOError
,
UploadError
),
e
:
logger
.
info
(
'Fail to upload file. %s'
%
\
(
str
(
e
)))
return
False
finally
:
f
.
close
()
return
True
@
fallback_call
@
fallback_call
def
get_filename_from_url
(
url
):
def
get_filename_from_url
(
url
):
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment