Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
S
slapos.core
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
0
Issues
0
List
Boards
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Analytics
Analytics
CI / CD
Repository
Value Stream
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
Léo-Paul Géneau
slapos.core
Commits
0690e743
Commit
0690e743
authored
Jan 26, 2012
by
Yingjie Xu
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
Implement download.
parent
f0c93b97
Changes
2
Hide whitespace changes
Inline
Side-by-side
Showing
2 changed files
with
46 additions
and
40 deletions
+46
-40
slapos/grid/SlapObject.py
slapos/grid/SlapObject.py
+31
-19
slapos/grid/networkcache.py
slapos/grid/networkcache.py
+15
-21
No files found.
slapos/grid/SlapObject.py
View file @
0690e743
...
@@ -82,25 +82,37 @@ class Software(object):
...
@@ -82,25 +82,37 @@ class Software(object):
tarname
=
self
.
software_url_hash
tarname
=
self
.
software_url_hash
cache_dir
=
tempfile
.
mkdtemp
()
cache_dir
=
tempfile
.
mkdtemp
()
tarpath
=
os
.
path
.
join
(
cache_dir
,
tarname
)
tarpath
=
os
.
path
.
join
(
cache_dir
,
tarname
)
self
.
_install_from_buildout
()
if
os
.
path
.
exists
(
self
.
software_path
):
tar
=
tarfile
.
open
(
tarpath
,
"w:gz"
)
self
.
_install_from_buildout
()
try
:
else
:
tar
.
add
(
self
.
software_path
,
arcname
=
self
.
software_url_hash
)
if
download_network_cached
(
finally
:
self
.
download_binary_cache_url
,
tar
.
close
()
self
.
download_binary_dir_url
,
upload_network_cached
(
self
.
url
,
self
.
software_url_hash
,
self
.
software_root
,
tarpath
,
self
.
logger
,
self
.
url
,
self
.
signature_certificate_list
):
self
.
software_url_hash
,
tar
=
tarfile
.
open
(
tarpath
)
self
.
upload_binary_cache_url
,
try
:
self
.
upload_binary_dir_url
,
tar
.
extractall
(
path
=
self
.
software_root
)
tarpath
,
self
.
logger
,
finally
:
self
.
signature_private_key_file
,
tar
.
close
()
self
.
shacache_cert_file
,
else
:
self
.
shacache_key_file
,
tar
=
tarfile
.
open
(
tarpath
,
"w:gz"
)
self
.
shadir_cert_file
,
try
:
self
.
shadir_key_file
tar
.
add
(
self
.
software_path
,
arcname
=
self
.
software_url_hash
)
)
finally
:
tar
.
close
()
upload_network_cached
(
self
.
software_root
,
self
.
url
,
self
.
software_url_hash
,
self
.
upload_binary_cache_url
,
self
.
upload_binary_dir_url
,
tarpath
,
self
.
logger
,
self
.
signature_private_key_file
,
self
.
shacache_cert_file
,
self
.
shacache_key_file
,
self
.
shadir_cert_file
,
self
.
shadir_key_file
)
def
_install_from_buildout
(
self
):
def
_install_from_buildout
(
self
):
""" Fetches buildout configuration from the server, run buildout with
""" Fetches buildout configuration from the server, run buildout with
...
...
slapos/grid/networkcache.py
View file @
0690e743
...
@@ -21,6 +21,7 @@ import shutil
...
@@ -21,6 +21,7 @@ import shutil
import
urlparse
import
urlparse
import
traceback
import
traceback
import
utils
import
utils
import
json
try
:
try
:
try
:
try
:
...
@@ -53,29 +54,19 @@ def fallback_call(function):
...
@@ -53,29 +54,19 @@ def fallback_call(function):
@
fallback_call
@
fallback_call
def
download_network_cached
(
dir_url
,
cache_url
,
path
,
url
,
logger
,
def
download_network_cached
(
cache_url
,
dir_url
,
software_url
,
key
,
path
,
signature_certificate_list
,
md5sum
=
None
):
logger
,
signature_certificate_list
):
"""Downloads from a network cache provider
"""Downloads from a network cache provider
If something fail (providor be offline, or hash_string fail), we ignore
network cached files.
return True if download succeeded.
return True if download succeeded.
"""
"""
if
not
LIBNETWORKCACHE_ENABLED
:
if
not
LIBNETWORKCACHE_ENABLED
:
return
False
return
False
if
not
(
dir_url
and
cache
_url
):
if
not
(
cache_url
and
dir
_url
):
return
False
return
False
if
md5sum
is
None
:
md5sum
=
_get_md5_from_url
(
url
)
directory_key
=
get_directory_key
(
url
)
url
=
os
.
path
.
basename
(
url
)
if
len
(
signature_certificate_list
)
==
0
:
if
len
(
signature_certificate_list
)
==
0
:
# convert [] into None in order to call nc nicely
signature_certificate_list
=
None
signature_certificate_list
=
None
try
:
try
:
nc
=
NetworkcacheClient
(
cache_url
,
dir_url
,
nc
=
NetworkcacheClient
(
cache_url
,
dir_url
,
...
@@ -84,21 +75,24 @@ def download_network_cached(dir_url, cache_url, path, url, logger,
...
@@ -84,21 +75,24 @@ def download_network_cached(dir_url, cache_url, path, url, logger,
logger
.
warning
(
'Incompatible version of networkcache, not using it.'
)
logger
.
warning
(
'Incompatible version of networkcache, not using it.'
)
return
False
return
False
logger
.
info
(
'Downloading %s
from network cache.'
%
url
)
logger
.
info
(
'Downloading %s
binary from network cache.'
%
software_
url
)
try
:
try
:
file_descriptor
=
nc
.
select
(
directory_key
)
json_entry_list
=
nc
.
select_generic
(
key
)
for
entry
in
json_entry_list
:
json_information
,
_
=
entry
try
:
tags
=
json
.
loads
(
json_information
)
sha512
=
tags
.
get
(
'sha512'
)
file_descriptor
=
nc
.
download
(
sha512
)
break
except
Exception
:
continue
f
=
open
(
path
,
'w+b'
)
f
=
open
(
path
,
'w+b'
)
try
:
try
:
shutil
.
copyfileobj
(
file_descriptor
,
f
)
shutil
.
copyfileobj
(
file_descriptor
,
f
)
finally
:
finally
:
f
.
close
()
f
.
close
()
file_descriptor
.
close
()
file_descriptor
.
close
()
if
not
check_md5sum
(
path
,
md5sum
):
logger
.
info
(
'MD5 checksum mismatch downloading %s'
%
url
)
return
False
except
(
IOError
,
DirectoryNotFound
),
e
:
except
(
IOError
,
DirectoryNotFound
),
e
:
logger
.
info
(
'Failed to download from network cache %s: %s'
%
\
logger
.
info
(
'Failed to download from network cache %s: %s'
%
\
(
url
,
str
(
e
)))
(
url
,
str
(
e
)))
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment