Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
S
slapos.core
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
0
Issues
0
List
Boards
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Analytics
Analytics
CI / CD
Repository
Value Stream
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
Eric Zheng
slapos.core
Commits
63b17738
Commit
63b17738
authored
Dec 31, 2019
by
Łukasz Nowak
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
slapgrid: binary cache only option
/reviewed-on
!173
parent
78f01daf
Changes
3
Hide whitespace changes
Inline
Side-by-side
Showing
3 changed files
with
30 additions
and
0 deletions
+30
-0
slapos.cfg.example
slapos.cfg.example
+4
-0
slapos/grid/SlapObject.py
slapos/grid/SlapObject.py
+16
-0
slapos/grid/slapgrid.py
slapos/grid/slapgrid.py
+10
-0
No files found.
slapos.cfg.example
View file @
63b17738
...
@@ -162,3 +162,7 @@ upload-to-binary-cache-url-blacklist =
...
@@ -162,3 +162,7 @@ upload-to-binary-cache-url-blacklist =
http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD
http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD
http://git.erp5.org/gitweb/slapos.git/blob_plain/refs/heads
http://git.erp5.org/gitweb/slapos.git/blob_plain/refs/heads
/
/
# List of URL(s) which are forced to be downloaded from binary cache
#download-from-binary-cache-force-url-list =
# https://lab.node.vifib.com/nexedi/slapos/raw/1.0.
slapos/grid/SlapObject.py
View file @
63b17738
...
@@ -112,6 +112,7 @@ class Software(object):
...
@@ -112,6 +112,7 @@ class Software(object):
download_binary_dir_url
=
None
,
upload_binary_dir_url
=
None
,
download_binary_dir_url
=
None
,
upload_binary_dir_url
=
None
,
download_from_binary_cache_url_blacklist
=
None
,
download_from_binary_cache_url_blacklist
=
None
,
upload_to_binary_cache_url_blacklist
=
None
,
upload_to_binary_cache_url_blacklist
=
None
,
download_from_binary_cache_force_url_list
=
None
,
software_min_free_space
=
None
,
software_min_free_space
=
None
,
buildout_debug
=
False
,
buildout_debug
=
False
,
shared_part_list
=
''
):
shared_part_list
=
''
):
...
@@ -124,6 +125,9 @@ class Software(object):
...
@@ -124,6 +125,9 @@ class Software(object):
if
upload_to_binary_cache_url_blacklist
is
None
:
if
upload_to_binary_cache_url_blacklist
is
None
:
upload_to_binary_cache_url_blacklist
=
[]
upload_to_binary_cache_url_blacklist
=
[]
if
download_from_binary_cache_force_url_list
is
None
:
download_from_binary_cache_force_url_list
=
[]
self
.
url
=
url
self
.
url
=
url
self
.
software_root
=
software_root
self
.
software_root
=
software_root
self
.
software_url_hash
=
md5digest
(
self
.
url
)
self
.
software_url_hash
=
md5digest
(
self
.
url
)
...
@@ -151,6 +155,8 @@ class Software(object):
...
@@ -151,6 +155,8 @@ class Software(object):
download_from_binary_cache_url_blacklist
download_from_binary_cache_url_blacklist
self
.
upload_to_binary_cache_url_blacklist
=
\
self
.
upload_to_binary_cache_url_blacklist
=
\
upload_to_binary_cache_url_blacklist
upload_to_binary_cache_url_blacklist
self
.
download_from_binary_cache_force_url_list
=
\
download_from_binary_cache_force_url_list
self
.
software_min_free_space
=
software_min_free_space
self
.
software_min_free_space
=
software_min_free_space
def
check_free_space
(
self
):
def
check_free_space
(
self
):
...
@@ -175,6 +181,12 @@ class Software(object):
...
@@ -175,6 +181,12 @@ class Software(object):
try
:
try
:
tarpath
=
os
.
path
.
join
(
cache_dir
,
self
.
software_url_hash
)
tarpath
=
os
.
path
.
join
(
cache_dir
,
self
.
software_url_hash
)
# Check if we can download from cache
# Check if we can download from cache
force_binary_cache
=
False
for
url_match
in
self
.
download_from_binary_cache_force_url_list
:
if
self
.
url
.
startswith
(
url_match
):
force_binary_cache
=
True
self
.
logger
.
debug
(
'Binary cache forced for %r because of match %r'
,
self
.
url
,
url_match
)
break
if
(
not
os
.
path
.
exists
(
self
.
software_path
))
\
if
(
not
os
.
path
.
exists
(
self
.
software_path
))
\
and
download_network_cached
(
and
download_network_cached
(
self
.
download_binary_cache_url
,
self
.
download_binary_cache_url
,
...
@@ -190,6 +202,10 @@ class Software(object):
...
@@ -190,6 +202,10 @@ class Software(object):
tar
.
extractall
(
path
=
self
.
software_root
)
tar
.
extractall
(
path
=
self
.
software_root
)
finally
:
finally
:
tar
.
close
()
tar
.
close
()
elif
force_binary_cache
:
message
=
'Binary cache forced for %r, but failed to download, will retry again'
%
(
self
.
url
,)
self
.
logger
.
error
(
message
)
raise
BuildoutFailedError
(
message
)
else
:
else
:
self
.
_install_from_buildout
()
self
.
_install_from_buildout
()
# Upload to binary cache if possible and allowed
# Upload to binary cache if possible and allowed
...
...
slapos/grid/slapgrid.py
View file @
63b17738
...
@@ -188,6 +188,9 @@ def merged_options(args, configp):
...
@@ -188,6 +188,9 @@ def merged_options(args, configp):
options
[
"upload-to-binary-cache-url-blacklist"
]
=
[
options
[
"upload-to-binary-cache-url-blacklist"
]
=
[
url
.
strip
()
for
url
in
options
.
get
(
url
.
strip
()
for
url
in
options
.
get
(
"upload-to-binary-cache-url-blacklist"
,
""
).
split
(
'
\
n
'
)
if
url
]
"upload-to-binary-cache-url-blacklist"
,
""
).
split
(
'
\
n
'
)
if
url
]
options
[
"download-from-binary-cache-force-url-list"
]
=
[
url
.
strip
()
for
url
in
options
.
get
(
"download-from-binary-cache-force-url-list"
,
""
).
split
(
'
\
n
'
)
if
url
]
options
[
'firewall'
]
=
{}
options
[
'firewall'
]
=
{}
if
configp
.
has_section
(
'firewall'
):
if
configp
.
has_section
(
'firewall'
):
...
@@ -259,6 +262,8 @@ def create_slapgrid_object(options, logger):
...
@@ -259,6 +262,8 @@ def create_slapgrid_object(options, logger):
op
.
get
(
'download-from-binary-cache-url-blacklist'
,
[]),
op
.
get
(
'download-from-binary-cache-url-blacklist'
,
[]),
upload_to_binary_cache_url_blacklist
=
upload_to_binary_cache_url_blacklist
=
op
.
get
(
'upload-to-binary-cache-url-blacklist'
,
[]),
op
.
get
(
'upload-to-binary-cache-url-blacklist'
,
[]),
download_from_binary_cache_force_url_list
=
op
.
get
(
'download-from-binary-cache-force-url-list'
,
[]),
upload_cache_url
=
op
.
get
(
'upload-cache-url'
),
upload_cache_url
=
op
.
get
(
'upload-cache-url'
),
download_binary_dir_url
=
op
.
get
(
'download-binary-dir-url'
),
download_binary_dir_url
=
op
.
get
(
'download-binary-dir-url'
),
upload_binary_dir_url
=
op
.
get
(
'upload-binary-dir-url'
),
upload_binary_dir_url
=
op
.
get
(
'upload-binary-dir-url'
),
...
@@ -317,6 +322,7 @@ class Slapgrid(object):
...
@@ -317,6 +322,7 @@ class Slapgrid(object):
upload_binary_cache_url
=
None
,
upload_binary_cache_url
=
None
,
download_from_binary_cache_url_blacklist
=
None
,
download_from_binary_cache_url_blacklist
=
None
,
upload_to_binary_cache_url_blacklist
=
None
,
upload_to_binary_cache_url_blacklist
=
None
,
download_from_binary_cache_force_url_list
=
None
,
upload_cache_url
=
None
,
upload_cache_url
=
None
,
download_binary_dir_url
=
None
,
download_binary_dir_url
=
None
,
upload_binary_dir_url
=
None
,
upload_binary_dir_url
=
None
,
...
@@ -362,6 +368,8 @@ class Slapgrid(object):
...
@@ -362,6 +368,8 @@ class Slapgrid(object):
download_from_binary_cache_url_blacklist
download_from_binary_cache_url_blacklist
self
.
upload_to_binary_cache_url_blacklist
=
\
self
.
upload_to_binary_cache_url_blacklist
=
\
upload_to_binary_cache_url_blacklist
upload_to_binary_cache_url_blacklist
self
.
download_from_binary_cache_force_url_list
=
\
download_from_binary_cache_force_url_list
self
.
upload_cache_url
=
upload_cache_url
self
.
upload_cache_url
=
upload_cache_url
self
.
download_binary_dir_url
=
download_binary_dir_url
self
.
download_binary_dir_url
=
download_binary_dir_url
self
.
upload_binary_dir_url
=
upload_binary_dir_url
self
.
upload_binary_dir_url
=
upload_binary_dir_url
...
@@ -555,6 +563,8 @@ stderr_logfile_backups=1
...
@@ -555,6 +563,8 @@ stderr_logfile_backups=1
self
.
download_from_binary_cache_url_blacklist
,
self
.
download_from_binary_cache_url_blacklist
,
upload_to_binary_cache_url_blacklist
=
upload_to_binary_cache_url_blacklist
=
self
.
upload_to_binary_cache_url_blacklist
,
self
.
upload_to_binary_cache_url_blacklist
,
download_from_binary_cache_force_url_list
=
self
.
download_from_binary_cache_force_url_list
,
upload_cache_url
=
self
.
upload_cache_url
,
upload_cache_url
=
self
.
upload_cache_url
,
download_binary_dir_url
=
self
.
download_binary_dir_url
,
download_binary_dir_url
=
self
.
download_binary_dir_url
,
upload_binary_dir_url
=
self
.
upload_binary_dir_url
,
upload_binary_dir_url
=
self
.
upload_binary_dir_url
,
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment