Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
S
slapos.buildout
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
0
Issues
0
List
Boards
Labels
Milestones
Merge Requests
0
Merge Requests
0
Analytics
Analytics
Repository
Value Stream
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Commits
Issue Boards
Open sidebar
Xavier Thompson
slapos.buildout
Commits
54a9d788
Commit
54a9d788
authored
May 05, 2011
by
Lucas Carvalho
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
First version of network cache integration with zc.buildout.
parent
f0d0ef63
Changes
5
Hide whitespace changes
Inline
Side-by-side
Showing
5 changed files
with
241 additions
and
15 deletions
+241
-15
src/zc/buildout/buildout.py
src/zc/buildout/buildout.py
+5
-1
src/zc/buildout/download.py
src/zc/buildout/download.py
+25
-11
src/zc/buildout/easy_install.py
src/zc/buildout/easy_install.py
+13
-3
src/zc/buildout/networkcache.py
src/zc/buildout/networkcache.py
+195
-0
zc.recipe.egg_/src/zc/recipe/egg/egg.py
zc.recipe.egg_/src/zc/recipe/egg/egg.py
+3
-0
No files found.
src/zc/buildout/buildout.py
View file @
54a9d788
...
...
@@ -138,6 +138,7 @@ _buildout_default_options = _annotate_section({
'socket-timeout'
:
''
,
'unzip'
:
'false'
,
'use-dependency-links'
:
'true'
,
'network-cache'
:
'http://127.0.0.1:5001/'
,
},
'DEFAULT_VALUE'
)
...
...
@@ -362,6 +363,7 @@ class Buildout(UserDict.DictMixin):
newest
=
self
.
newest
,
allow_hosts
=
self
.
_allow_hosts
,
prefer_final
=
not
self
.
accept_buildout_test_releases
,
network_cache
=
options
[
'network-cache'
],
)
# Now copy buildout and setuptools eggs, and record destination eggs:
...
...
@@ -860,6 +862,7 @@ class Buildout(UserDict.DictMixin):
path
=
[
options
[
'develop-eggs-directory'
]],
allow_hosts
=
self
.
_allow_hosts
,
prefer_final
=
not
self
.
accept_buildout_test_releases
,
network_cache
=
options
.
get
(
'network-cache'
),
)
upgraded
=
[]
...
...
@@ -1087,7 +1090,8 @@ def _install_and_load(spec, group, entry, buildout):
working_set
=
pkg_resources
.
working_set
,
newest
=
buildout
.
newest
,
allow_hosts
=
buildout
.
_allow_hosts
,
prefer_final
=
not
buildout
.
accept_buildout_test_releases
)
prefer_final
=
not
buildout
.
accept_buildout_test_releases
,
network_cache
=
buildout_options
.
get
(
'network-cache'
))
__doing__
=
'Loading %s recipe entry %s:%s.'
,
group
,
spec
,
entry
return
pkg_resources
.
load_entry_point
(
...
...
src/zc/buildout/download.py
View file @
54a9d788
...
...
@@ -40,6 +40,11 @@ class ChecksumError(zc.buildout.UserError):
url_opener
=
URLOpener
()
from
zc.buildout.networkcache
import
get_sha256sum_from_networkcached
,
\
download_network_cached
,
\
upload_network_cached
class
Download
(
object
):
"""Configurable download utility.
...
...
@@ -72,6 +77,7 @@ class Download(object):
self
.
fallback
=
fallback
self
.
hash_name
=
hash_name
self
.
logger
=
logger
or
logging
.
getLogger
(
'zc.buildout'
)
self
.
network_cache
=
options
.
get
(
'network-cache'
)
@
property
def
download_cache
(
self
):
...
...
@@ -174,20 +180,28 @@ class Download(object):
self
.
logger
.
info
(
'Downloading %s'
%
url
)
urllib
.
_urlopener
=
url_opener
handle
,
tmp_path
=
tempfile
.
mkstemp
(
prefix
=
'buildout-'
)
sha256sum
=
get_sha256sum_from_networkcached
(
self
.
network_cache
,
url
,
self
.
logger
)
try
:
tmp_path
,
headers
=
urllib
.
urlretrieve
(
url
,
tmp_path
)
if
not
check_md5sum
(
tmp_path
,
md5sum
):
raise
ChecksumError
(
'MD5 checksum mismatch downloading %r'
%
url
)
except
IOError
,
e
:
os
.
remove
(
tmp_path
)
raise
zc
.
buildout
.
UserError
(
"Error downloading extends for URL "
"%s: %r"
%
(
url
,
e
[
1
:
3
]))
except
Exception
,
e
:
try
:
if
not
download_network_cached
(
self
.
network_cache
,
tmp_path
,
sha256sum
,
self
.
logger
):
# Download from original url
tmp_path
,
headers
=
urllib
.
urlretrieve
(
url
,
tmp_path
)
if
not
check_md5sum
(
tmp_path
,
md5sum
):
raise
ChecksumError
(
'MD5 checksum mismatch downloading %r'
%
url
)
# Upload the file to networkcached.
upload_network_cached
(
self
.
network_cache
,
url
,
tmp_path
,
self
.
logger
)
finally
:
os
.
close
(
handle
)
except
:
os
.
remove
(
tmp_path
)
raise
finally
:
os
.
close
(
handle
)
if
path
:
shutil
.
move
(
tmp_path
,
path
)
...
...
src/zc/buildout/easy_install.py
View file @
54a9d788
...
...
@@ -18,6 +18,8 @@ It doesn't install scripts. It uses setuptools and requires it to be
installed.
"""
from
zc.buildout.networkcache
import
fetch_from_network_cache
,
\
upload_network_cached
import
distutils.errors
import
fnmatch
import
glob
...
...
@@ -335,6 +337,7 @@ class Installer:
include_site_packages
=
None
,
allowed_eggs_from_site_packages
=
None
,
prefer_final
=
None
,
network_cache
=
None
,
):
self
.
_dest
=
dest
self
.
_allow_hosts
=
allow_hosts
...
...
@@ -403,6 +406,8 @@ class Installer:
if
versions
is
not
None
:
self
.
_versions
=
versions
self
.
_network_cache
=
network_cache
_allowed_eggs_from_site_packages_regex
=
None
def
allow_site_package_egg
(
self
,
name
):
if
(
not
self
.
_include_site_packages
or
...
...
@@ -701,8 +706,13 @@ class Installer:
and
(
realpath
(
os
.
path
.
dirname
(
dist
.
location
))
==
download_cache
)
):
return
dist
new_location
=
fetch_from_network_cache
(
self
.
_network_cache
,
dist
.
location
,
tmp
,
logger
)
if
new_location
is
None
:
new_location
=
self
.
_index
.
download
(
dist
.
location
,
tmp
)
upload_network_cached
(
self
.
_network_cache
,
dist
.
location
,
new_location
,
logger
)
new_location
=
self
.
_index
.
download
(
dist
.
location
,
tmp
)
if
(
download_cache
and
(
realpath
(
new_location
)
==
realpath
(
dist
.
location
))
and
os
.
path
.
isfile
(
new_location
)
...
...
@@ -1080,13 +1090,13 @@ def install(specs, dest,
path
=
None
,
working_set
=
None
,
newest
=
True
,
versions
=
None
,
use_dependency_links
=
None
,
allow_hosts
=
(
'*'
,),
include_site_packages
=
None
,
allowed_eggs_from_site_packages
=
None
,
prefer_final
=
None
):
prefer_final
=
None
,
network_cache
=
None
):
installer
=
Installer
(
dest
,
links
,
index
,
executable
,
always_unzip
,
path
,
newest
,
versions
,
use_dependency_links
,
allow_hosts
=
allow_hosts
,
include_site_packages
=
include_site_packages
,
allowed_eggs_from_site_packages
=
allowed_eggs_from_site_packages
,
prefer_final
=
prefer_final
)
prefer_final
=
prefer_final
,
network_cache
=
network_cache
)
return
installer
.
install
(
specs
,
working_set
)
...
...
src/zc/buildout/networkcache.py
0 → 100644
View file @
54a9d788
##############################################################################
#
# Copyright (c) 2010 ViFiB SARL and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
import
os
import
urllib
import
re
import
posixpath
import
base64
import
hashlib
import
json
_md5_re
=
re
.
compile
(
r'md5=([a-f0-9]+)'
)
def
_get_md5_from_url
(
url
):
match
=
_md5_re
.
search
(
url
)
if
match
:
return
match
.
group
(
1
)
return
None
def
_get_hash_from_file
(
path
,
hash_object
):
"""
Calculate the hash from file.
"""
f
=
open
(
path
,
'rb'
)
try
:
chunk
=
f
.
read
(
2
**
16
)
while
chunk
:
hash_object
.
update
(
chunk
)
chunk
=
f
.
read
(
2
**
16
)
return
hash_object
.
hexdigest
()
finally
:
f
.
close
()
def
_get_sha256_from_file
(
path
):
"""
Return the sha256sum from file.
"""
if
not
os
.
path
.
isfile
(
path
):
raise
ValueError
,
'Could not extract sha256sum.'
\
' The path is not a file: %s '
%
path
return
_get_hash_from_file
(
path
,
hashlib
.
sha256
())
def
check_sha256sum
(
path
,
sha256sum
):
"""Tell whether the SHA256sum checksum of the file at path matches
"""
return
sha256sum
==
_get_hash_from_file
(
path
,
hashlib
.
sha256
())
def
_update_network_cached_map
(
network_cache
,
sha256sum
,
url
,
logger
):
"""
If the url is not into the map file, it must be updated.
"""
# Keep the log
print
os
.
environ
.
get
(
'PWD'
)
map_path
=
os
.
path
.
join
(
os
.
environ
.
get
(
'PWD'
),
'.networkcached.log'
)
mode
=
'r+'
if
not
os
.
path
.
exists
(
map_path
):
mode
=
'w+'
f
=
open
(
map_path
,
mode
)
try
:
flag
=
False
for
line
in
f
:
if
url
in
line
:
flag
=
True
break
if
not
flag
:
f
.
seek
(
0
,
2
)
f
.
write
(
'%s %s
\
n
'
%
(
sha256sum
,
url
))
f
.
truncate
(
f
.
tell
())
finally
:
f
.
close
()
# Update the map at networkcached server.
network_cached_url
=
os
.
path
.
join
(
network_cache
,
'map/add'
)
try
:
result
=
urllib
.
urlopen
(
network_cached_url
,
\
urllib
.
urlencode
({
'sha256sum'
:
sha256sum
,
'url'
:
url
}))
if
int
(
result
.
code
)
!=
200
:
logger
.
info
(
'Fail to update the network cache map: %s %s'
%
\
(
sha256sum
,
url
))
except
IOError
,
e
:
logger
.
info
(
'An error occurred to update the map in networkcached.'
\
' %s'
%
str
(
e
))
def
get_sha256sum_from_networkcached
(
network_cache
,
url
,
logger
):
"""
Return the sha256sum if the url exists in networkcached server.
"""
network_cached_url
=
os
.
path
.
join
(
network_cache
,
'map/get'
,
base64
.
encodestring
(
url
))
try
:
result
=
urllib
.
urlopen
(
network_cached_url
)
if
int
(
result
.
code
)
==
200
:
return
json
.
loads
(
result
.
read
().
strip
()).
get
(
'sha256sum'
,
None
)
except
IOError
,
e
:
logger
.
info
(
'An error occurred to get sha256sum from networkcached:'
\
' %s'
%
str
(
e
))
def
download_network_cached
(
network_cache
,
path
,
sha256sum
,
logger
):
"""Download from a network cache provider
If something fail (providor be offline, or hash_string fail), we ignore
network cached files.
return True if download succeeded.
"""
if
sha256sum
is
None
:
# Not able to use network cache
return
False
url
=
os
.
path
.
join
(
network_cache
,
sha256sum
)
logger
.
info
(
'Downloading from network cache %s'
%
url
)
try
:
path
,
headers
=
urllib
.
urlretrieve
(
url
,
path
)
if
not
check_sha256sum
(
path
,
sha256sum
):
logger
.
info
(
'MD5/SHA256 checksum mismatch downloading %r'
%
url
)
return
False
except
IOError
,
e
:
logger
.
info
(
'Fail to download from network cache %s'
%
url
)
return
False
return
True
def
upload_network_cached
(
network_cache
,
external_url
,
path
,
logger
):
"""Upload file to a network cache server"""
if
network_cache
in
[
None
,
''
]:
logger
.
debug
(
'Upload cache ignored, network-cache was not provided'
)
return
False
sha256sum
=
_get_sha256_from_file
(
path
)
try
:
f
=
open
(
path
,
'r'
)
data
=
f
.
read
()
url
=
os
.
path
.
join
(
network_cache
,
sha256sum
)
try
:
result
=
urllib
.
urlopen
(
url
,
urllib
.
urlencode
({
"data"
:
base64
.
encodestring
(
data
)}))
if
result
.
code
==
200
and
\
json
.
loads
(
result
.
read
()).
get
(
'sha'
)
==
sha256sum
:
_update_network_cached_map
(
network_cache
,
sha256sum
,
external_url
,
logger
)
except
(
IOError
,
EOFError
),
e
:
logger
.
info
(
'Fail to upload cache on %s. %s'
%
(
external_url
,
str
(
e
)))
finally
:
f
.
close
()
return
True
def
_get_filename_from_url
(
url
):
"""Inspired how pip get filename from url.
"""
url
=
url
.
split
(
'#'
,
1
)[
0
]
url
=
url
.
split
(
'?'
,
1
)[
0
]
url
=
url
.
rstrip
(
'/'
)
name
=
posixpath
.
basename
(
url
)
assert
name
,
(
'URL %r produced no filename'
%
url
)
return
name
def
fetch_from_network_cache
(
network_cache
,
location
,
tmp
,
logger
=
None
):
""" Try to download from a network cache and preserve
original filename.
"""
# Make a id from URL
md5sum
=
_get_md5_from_url
(
location
)
filename
=
_get_filename_from_url
(
location
)
path
=
os
.
path
.
join
(
tmp
,
filename
)
sha256sum
=
get_sha256sum_from_networkcached
(
network_cache
,
location
,
logger
)
if
sha256sum
is
None
:
return
None
is_downloaded
=
download_network_cached
(
network_cache
,
path
,
sha256sum
,
logger
)
if
is_downloaded
:
return
path
return
None
zc.recipe.egg_/src/zc/recipe/egg/egg.py
View file @
54a9d788
...
...
@@ -88,6 +88,9 @@ class Eggs(object):
kw
=
{}
if
'unzip'
in
options
:
kw
[
'always_unzip'
]
=
options
.
query_bool
(
'unzip'
,
None
)
if
'network-cache'
in
b_options
:
kw
[
'network_cache'
]
=
b_options
.
get
(
'network-cache'
)
ws
=
zc
.
buildout
.
easy_install
.
install
(
distributions
,
options
[
'eggs-directory'
],
links
=
self
.
links
,
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment