Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
S
slapos.core
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
0
Issues
0
List
Boards
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Analytics
Analytics
CI / CD
Repository
Value Stream
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
Jérome Perrin
slapos.core
Commits
65622b9a
Commit
65622b9a
authored
11 years ago
by
Marco Mariani
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
make pep8 happy
parent
c1058afd
No related merge requests found
Changes
4
Hide whitespace changes
Inline
Side-by-side
Showing
4 changed files
with
78 additions
and
89 deletions
+78
-89
slapos/grid/SlapObject.py
slapos/grid/SlapObject.py
+27
-32
slapos/grid/distribution.py
slapos/grid/distribution.py
+0
-1
slapos/grid/slapgrid.py
slapos/grid/slapgrid.py
+49
-53
slapos/grid/utils.py
slapos/grid/utils.py
+2
-3
No files found.
slapos/grid/SlapObject.py
View file @
65622b9a
...
...
@@ -43,7 +43,7 @@ from supervisor import xmlrpc
from
slapos.grid.utils
import
(
md5digest
,
getCleanEnvironment
,
SlapPopen
,
dropPrivileges
,
updateFile
)
from
slapos.grid
import
utils
# for mocked methods
from
slapos.grid
import
utils
# for mocked methods
from
slapos.slap.slap
import
NotFoundError
from
slapos.grid.svcbackend
import
getSupervisorRPC
from
slapos.grid.exception
import
(
BuildoutFailedError
,
WrongPermissionError
,
...
...
@@ -166,8 +166,8 @@ class Software(object):
]
if
(
self
.
signature_private_key_file
or
self
.
upload_cache_url
or
self
.
upload_dir_url
):
self
.
upload_cache_url
or
self
.
upload_dir_url
):
buildout_parameter_list
.
append
(
'buildout:networkcache-section=networkcache'
)
for
buildout_option
,
value
in
[
(
'%ssignature-private-key-file=%s'
,
self
.
signature_private_key_file
),
...
...
@@ -306,10 +306,7 @@ class Partition(object):
uid
,
gid
=
self
.
getUserGroupId
()
for
name
,
path
in
[
(
'certificate'
,
self
.
cert_file
),
(
'key'
,
self
.
key_file
),
]:
for
name
,
path
in
[(
'certificate'
,
self
.
cert_file
),
(
'key'
,
self
.
key_file
)]:
new_content
=
partition_certificate
[
name
]
old_content
=
None
if
os
.
path
.
exists
(
path
):
...
...
@@ -325,7 +322,6 @@ class Partition(object):
fout
.
write
(
new_content
)
os
.
chown
(
path
,
uid
,
gid
)
def
getUserGroupId
(
self
):
"""Returns tuple of (uid, gid) of partition"""
stat_info
=
os
.
stat
(
self
.
instance_path
)
...
...
@@ -341,17 +337,17 @@ class Partition(object):
for
runner
in
runner_list
:
self
.
partition_supervisor_configuration
+=
'
\
n
'
+
\
program_partition_template
%
{
'program_id'
:
'_'
.
join
([
partition_id
,
runner
]),
'program_directory'
:
self
.
instance_path
,
'program_command'
:
os
.
path
.
join
(
path
,
runner
),
'program_name'
:
runner
+
extension
,
'instance_path'
:
self
.
instance_path
,
'user_id'
:
uid
,
'group_id'
:
gid
,
# As supervisord has no environment to inherit, setup a minimalistic one
'HOME'
:
pwd
.
getpwuid
(
uid
).
pw_dir
,
'USER'
:
pwd
.
getpwuid
(
uid
).
pw_name
,
}
'program_id'
:
'_'
.
join
([
partition_id
,
runner
]),
'program_directory'
:
self
.
instance_path
,
'program_command'
:
os
.
path
.
join
(
path
,
runner
),
'program_name'
:
runner
+
extension
,
'instance_path'
:
self
.
instance_path
,
'user_id'
:
uid
,
'group_id'
:
gid
,
# As supervisord has no environment to inherit, setup a minimalistic one
'HOME'
:
pwd
.
getpwuid
(
uid
).
pw_dir
,
'USER'
:
pwd
.
getpwuid
(
uid
).
pw_name
,
}
def
updateSymlink
(
self
,
sr_symlink
,
software_path
):
if
os
.
path
.
lexists
(
sr_symlink
):
...
...
@@ -391,7 +387,7 @@ class Partition(object):
if
not
os
.
path
.
exists
(
self
.
software_path
):
# XXX What should it raise?
raise
IOError
(
'Software Release %s is not present on system.
\
n
'
'Cannot deploy instance.'
%
self
.
software_release_url
)
'Cannot deploy instance.'
%
self
.
software_release_url
)
# Generate buildout instance profile from template in Software Release
template_location
=
os
.
path
.
join
(
self
.
software_path
,
'instance.cfg'
)
...
...
@@ -412,13 +408,13 @@ class Partition(object):
buildout_text
=
open
(
config_location
).
read
()
buildout_text
+=
'
\
n
\
n
'
+
pkg_resources
.
resource_string
(
__name__
,
'templates/buildout-tail.cfg.in'
)
%
{
'computer_id'
:
self
.
computer_id
,
'partition_id'
:
self
.
partition_id
,
'server_url'
:
self
.
server_url
,
'software_release_url'
:
self
.
software_release_url
,
'key_file'
:
self
.
key_file
,
'cert_file'
:
self
.
cert_file
,
}
'computer_id'
:
self
.
computer_id
,
'partition_id'
:
self
.
partition_id
,
'server_url'
:
self
.
server_url
,
'software_release_url'
:
self
.
software_release_url
,
'key_file'
:
self
.
key_file
,
'cert_file'
:
self
.
cert_file
,
}
open
(
config_location
,
'w'
).
write
(
buildout_text
)
os
.
chmod
(
config_location
,
0o640
)
# Try to find the best possible buildout:
...
...
@@ -482,7 +478,6 @@ class Partition(object):
logger
=
self
.
logger
)
self
.
generateSupervisorConfigurationFile
()
def
generateSupervisorConfigurationFile
(
self
):
"""
Generates supervisord configuration file from template.
...
...
@@ -510,10 +505,10 @@ class Partition(object):
group_partition_template
=
pkg_resources
.
resource_stream
(
__name__
,
'templates/group_partition_supervisord.conf.in'
).
read
()
self
.
partition_supervisor_configuration
=
group_partition_template
%
{
'instance_id'
:
partition_id
,
'program_list'
:
','
.
join
([
'_'
.
join
([
partition_id
,
runner
])
for
runner
in
runner_list
+
service_list
])
}
'instance_id'
:
partition_id
,
'program_list'
:
','
.
join
([
'_'
.
join
([
partition_id
,
runner
])
for
runner
in
runner_list
+
service_list
])
}
# Same method to add to service and run
self
.
addServiceToGroup
(
partition_id
,
runner_list
,
self
.
run_path
)
self
.
addServiceToGroup
(
partition_id
,
service_list
,
self
.
service_path
,
...
...
This diff is collapsed.
Click to expand it.
slapos/grid/distribution.py
View file @
65622b9a
...
...
@@ -62,4 +62,3 @@ def patched_linux_distribution(distname='', version='', id='',
pass
return
platform
.
linux_distribution
(
distname
,
version
,
id
,
supported_dists
,
full_distribution_name
)
This diff is collapsed.
Click to expand it.
slapos/grid/slapgrid.py
View file @
65622b9a
...
...
@@ -100,23 +100,23 @@ def check_missing_parameters(options):
def
check_missing_files
(
options
):
req_files
=
[
options
.
get
(
'key_file'
),
options
.
get
(
'cert_file'
),
options
.
get
(
'master_ca_file'
),
options
.
get
(
'shacache-cert-file'
),
options
.
get
(
'shacache-key-file'
),
options
.
get
(
'shadir-cert-file'
),
options
.
get
(
'shadir-key-file'
),
options
.
get
(
'signature_private_key_file'
)
]
options
.
get
(
'key_file'
),
options
.
get
(
'cert_file'
),
options
.
get
(
'master_ca_file'
),
options
.
get
(
'shacache-cert-file'
),
options
.
get
(
'shacache-key-file'
),
options
.
get
(
'shadir-cert-file'
),
options
.
get
(
'shadir-key-file'
),
options
.
get
(
'signature_private_key_file'
)
]
req_dirs
=
[
options
.
get
(
'certificate_repository_path'
)
]
options
.
get
(
'certificate_repository_path'
)
]
for
f
in
req_files
:
if
f
and
not
os
.
path
.
exists
(
f
):
raise
RuntimeError
(
'File %r does not exist.'
%
f
)
raise
RuntimeError
(
'File %r does not exist.'
%
f
)
for
d
in
req_dirs
:
if
d
and
not
os
.
path
.
isdir
(
d
):
...
...
@@ -149,8 +149,8 @@ def merged_options(args, configp):
# Parse cache / binary cache options
# Backward compatibility about "binary-cache-url-blacklist" deprecated option
if
options
.
get
(
"binary-cache-url-blacklist"
)
and
not
\
options
.
get
(
"download-from-binary-cache-url-blacklist"
):
if
(
options
.
get
(
"binary-cache-url-blacklist"
)
and
not
options
.
get
(
"download-from-binary-cache-url-blacklist"
)
):
options
[
"download-from-binary-cache-url-blacklist"
]
=
\
options
[
"binary-cache-url-blacklist"
]
options
[
"download-from-binary-cache-url-blacklist"
]
=
[
...
...
@@ -175,8 +175,8 @@ def random_delay(options, logger):
maximal_delay
=
int
(
options
.
get
(
'maximal_delay'
,
'0'
))
if
maximal_delay
:
duration
=
random
.
randint
(
1
,
maximal_delay
)
logger
.
info
(
'Sleeping for %s seconds. To disable this feature, '
\
'check --now parameter in slapgrid help.'
%
duration
)
logger
.
info
(
'Sleeping for %s seconds. To disable this feature, '
'check --now parameter in slapgrid help.'
,
duration
)
time
.
sleep
(
duration
)
...
...
@@ -185,10 +185,10 @@ def create_slapgrid_object(options, logger):
if
'signature-certificate-list'
in
options
:
cert_marker
=
'-----BEGIN CERTIFICATE-----'
signature_certificate_list
=
[
cert_marker
+
'
\
n
'
+
q
.
strip
()
for
q
in
options
[
'signature-certificate-list'
].
split
(
cert_marker
)
if
q
.
strip
()
]
cert_marker
+
'
\
n
'
+
q
.
strip
()
for
q
in
options
[
'signature-certificate-list'
].
split
(
cert_marker
)
if
q
.
strip
()
]
op
=
options
return
Slapgrid
(
software_root
=
op
[
'software_root'
],
...
...
@@ -199,17 +199,17 @@ def create_slapgrid_object(options, logger):
supervisord_configuration_path
=
op
[
'supervisord_configuration_path'
],
buildout
=
op
.
get
(
'buildout'
),
logger
=
logger
,
force_periodicity
=
op
.
get
(
'force_periodicity'
,
False
),
maximum_periodicity
=
op
.
get
(
'maximum_periodicity'
,
86400
),
force_periodicity
=
op
.
get
(
'force_periodicity'
,
False
),
maximum_periodicity
=
op
.
get
(
'maximum_periodicity'
,
86400
),
key_file
=
op
.
get
(
'key_file'
),
cert_file
=
op
.
get
(
'cert_file'
),
signature_private_key_file
=
op
.
get
(
'signature_private_key_file'
),
signature_certificate_list
=
signature_certificate_list
,
download_binary_cache_url
=
op
.
get
(
'download-binary-cache-url'
),
upload_binary_cache_url
=
op
.
get
(
'upload-binary-cache-url'
),
download_from_binary_cache_url_blacklist
=
\
download_from_binary_cache_url_blacklist
=
op
.
get
(
'download-from-binary-cache-url-blacklist'
,
[]),
upload_to_binary_cache_url_blacklist
=
\
upload_to_binary_cache_url_blacklist
=
op
.
get
(
'upload-to-binary-cache-url-blacklist'
,
[]),
upload_cache_url
=
op
.
get
(
'upload-cache-url'
),
download_binary_dir_url
=
op
.
get
(
'download-binary-dir-url'
),
...
...
@@ -232,8 +232,8 @@ def create_slapgrid_object(options, logger):
def
check_required_only_partitions
(
existing
,
required
):
missing
=
set
(
required
)
-
set
(
existing
)
if
missing
:
plural
=
[
's'
,
''
][
len
(
missing
)
==
1
]
raise
ValueError
(
'Unknown partition%s: %s'
%
(
plural
,
', '
.
join
(
sorted
(
missing
))))
plural
=
[
's'
,
''
][
len
(
missing
)
==
1
]
raise
ValueError
(
'Unknown partition%s: %s'
%
(
plural
,
', '
.
join
(
sorted
(
missing
))))
class
Slapgrid
(
object
):
...
...
@@ -334,8 +334,8 @@ class Slapgrid(object):
invocation_list
=
[
WATCHDOG_PATH
]
invocation_list
.
append
(
"--master-url '%s' "
%
self
.
master_url
)
if
self
.
certificate_repository_path
:
invocation_list
.
append
(
"--certificate-repository-path '%s'"
\
%
self
.
certificate_repository_path
)
invocation_list
.
append
(
"--certificate-repository-path '%s'"
%
self
.
certificate_repository_path
)
invocation_list
.
append
(
"--computer-id '%s'"
%
self
.
computer_id
)
return
' '
.
join
(
invocation_list
)
...
...
@@ -371,7 +371,8 @@ class Slapgrid(object):
'supervisord_pidfile'
:
os
.
path
.
abspath
(
os
.
path
.
join
(
self
.
instance_root
,
'var'
,
'run'
,
'supervisord.pid'
)),
'supervisord_logfile_backups'
:
'10'
,
'watchdog_command'
:
self
.
getWatchdogLine
(),
})
}
)
def
getComputerPartitionList
(
self
):
try
:
...
...
@@ -401,9 +402,9 @@ class Slapgrid(object):
signature_certificate_list
=
self
.
signature_certificate_list
,
download_binary_cache_url
=
self
.
download_binary_cache_url
,
upload_binary_cache_url
=
self
.
upload_binary_cache_url
,
download_from_binary_cache_url_blacklist
=
\
download_from_binary_cache_url_blacklist
=
self
.
download_from_binary_cache_url_blacklist
,
upload_to_binary_cache_url_blacklist
=
\
upload_to_binary_cache_url_blacklist
=
self
.
upload_to_binary_cache_url_blacklist
,
upload_cache_url
=
self
.
upload_cache_url
,
download_binary_dir_url
=
self
.
download_binary_dir_url
,
...
...
@@ -415,10 +416,10 @@ class Slapgrid(object):
shadir_key_file
=
self
.
shadir_key_file
)
if
state
==
'available'
:
completed_tag
=
os
.
path
.
join
(
software_path
,
'.completed'
)
if
self
.
develop
or
(
not
os
.
path
.
exists
(
completed_tag
)
and
\
len
(
self
.
software_release_filter_list
)
==
0
)
or
\
url_hash
in
self
.
software_release_filter_list
or
\
url_hash
in
(
md5digest
(
uri
)
for
uri
in
self
.
software_release_filter_list
):
if
(
self
.
develop
or
(
not
os
.
path
.
exists
(
completed_tag
)
and
len
(
self
.
software_release_filter_list
)
==
0
)
or
url_hash
in
self
.
software_release_filter_list
or
url_hash
in
(
md5digest
(
uri
)
for
uri
in
self
.
software_release_filter_list
)
)
:
try
:
software_release
.
building
()
except
NotFoundError
:
...
...
@@ -471,7 +472,6 @@ class Slapgrid(object):
return
SLAPGRID_FAIL
return
SLAPGRID_SUCCESS
def
_launchSupervisord
(
self
):
launchSupervisord
(
self
.
supervisord_socket
,
self
.
supervisord_configuration_path
,
...
...
@@ -592,8 +592,8 @@ class Slapgrid(object):
# Check if timestamp from server is more recent than local one.
# If not: it's not worth processing this partition (nothing has
# changed).
if
computer_partition_id
not
in
self
.
computer_partition_filter_list
and
\
(
not
self
.
develop
)
and
os
.
path
.
exists
(
timestamp_path
):
if
(
computer_partition_id
not
in
self
.
computer_partition_filter_list
and
not
self
.
develop
and
os
.
path
.
exists
(
timestamp_path
)
):
old_timestamp
=
open
(
timestamp_path
).
read
()
last_runtime
=
int
(
os
.
path
.
getmtime
(
timestamp_path
))
if
timestamp
:
...
...
@@ -808,7 +808,6 @@ class Slapgrid(object):
return
SLAPGRID_PROMISE_FAIL
return
SLAPGRID_SUCCESS
def
validateXML
(
self
,
to_be_validated
,
xsd_model
):
"""Validates a given xml file"""
#We retrieve the xsd model
...
...
@@ -819,8 +818,8 @@ class Slapgrid(object):
try
:
document
=
etree
.
fromstring
(
to_be_validated
)
except
(
etree
.
XMLSyntaxError
,
etree
.
DocumentInvalid
)
as
exc
:
self
.
logger
.
info
(
'Failed to parse this XML report : %s
\
n
%s'
%
\
(
to_be_validated
,
_formatXMLError
(
exc
)))
self
.
logger
.
info
(
'Failed to parse this XML report : %s
\
n
%s'
%
(
to_be_validated
,
_formatXMLError
(
exc
)))
self
.
logger
.
error
(
_formatXMLError
(
exc
))
return
False
...
...
@@ -967,9 +966,8 @@ class Slapgrid(object):
computer_partition
.
error
(
'
\
n
'
.
join
(
failed_script_list
),
logger
=
self
.
logger
)
# Whatever happens, don't stop processing other instances
except
Exception
:
self
.
logger
.
info
(
'Cannot run usage script(s) for %r: %s'
%
(
computer_partition
.
getId
(),
traceback
.
format_exc
()))
self
.
logger
.
info
(
'Cannot run usage script(s) for %r: %s'
%
(
computer_partition
.
getId
(),
traceback
.
format_exc
()))
#Now we loop through the different computer partitions to report
report_usage_issue_cp_list
=
[]
...
...
@@ -1002,7 +1000,7 @@ class Slapgrid(object):
# report
else
:
computer_partition_usage
=
self
.
slap
.
registerComputerPartition
(
self
.
computer_id
,
computer_partition_id
)
self
.
computer_id
,
computer_partition_id
)
computer_partition_usage
.
setUsage
(
usage
)
computer_partition_usage_list
.
append
(
computer_partition_usage
)
filename_delete_list
.
append
(
filename
)
...
...
@@ -1015,14 +1013,12 @@ class Slapgrid(object):
# Whatever happens, don't stop processing other instances
except
Exception
:
self
.
logger
.
info
(
'Cannot run usage script(s) for %r: %s'
%
(
computer_partition
.
getId
(),
traceback
.
format_exc
()))
self
.
logger
.
info
(
'Cannot run usage script(s) for %r: %s'
%
(
computer_partition
.
getId
(),
traceback
.
format_exc
()))
for
computer_partition_usage
in
computer_partition_usage_list
:
self
.
logger
.
info
(
'computer_partition_usage_list: %s - %s'
%
(
computer_partition_usage
.
usage
,
computer_partition_usage
.
getId
()))
self
.
logger
.
info
(
'computer_partition_usage_list: %s - %s'
%
(
computer_partition_usage
.
usage
,
computer_partition_usage
.
getId
()))
#If there is, at least, one report
if
computer_partition_usage_list
!=
[]:
...
...
@@ -1041,8 +1037,8 @@ class Slapgrid(object):
raise
ValueError
(
'XML file generated by asXML is not valid !'
)
except
Exception
:
issue
=
"Cannot report usage for %r: %s"
%
(
computer_partition
.
getId
(),
traceback
.
format_exc
())
computer_partition
.
getId
(),
traceback
.
format_exc
())
self
.
logger
.
info
(
issue
)
computer_partition
.
error
(
issue
,
logger
=
self
.
logger
)
report_usage_issue_cp_list
.
append
(
computer_partition_id
)
...
...
This diff is collapsed.
Click to expand it.
slapos/grid/utils.py
View file @
65622b9a
...
...
@@ -124,7 +124,7 @@ def getCleanEnvironment(logger, home_path='/tmp'):
env
=
os
.
environ
.
copy
()
# Clean python related environment variables
for
k
in
PYTHON_ENVIRONMENT_REMOVE_LIST
+
SYSTEM_ENVIRONMENT_REMOVE_LIST
\
+
LOCALE_ENVIRONMENT_REMOVE_LIST
:
+
LOCALE_ENVIRONMENT_REMOVE_LIST
:
old
=
env
.
pop
(
k
,
None
)
if
old
is
not
None
:
removed_env
.
append
(
k
)
...
...
@@ -197,8 +197,7 @@ def dropPrivileges(uid, gid, logger):
# assert that privileges are dropped
message_pre
=
'After dropping to uid = %r and gid = %r '
\
'and group_list = %s'
%
(
uid
,
gid
,
group_list
)
'and group_list = %s'
%
(
uid
,
gid
,
group_list
)
new_uid
,
new_gid
,
new_group_list
=
os
.
getuid
(),
os
.
getgid
(),
os
.
getgroups
()
if
not
(
new_uid
==
uid
and
new_gid
==
gid
and
set
(
new_group_list
)
==
group_list
):
raise
OSError
(
'%s new_uid = %r and new_gid = %r and '
...
...
This diff is collapsed.
Click to expand it.
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment