Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
S
slapos.core
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Labels
Merge Requests
16
Merge Requests
16
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Analytics
Analytics
CI / CD
Repository
Value Stream
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Jobs
Commits
Open sidebar
nexedi
slapos.core
Commits
9de598f2
Commit
9de598f2
authored
Oct 30, 2018
by
Bryton Lacquement
🚪
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
wip
parent
d1e941b3
Changes
14
Expand all
Hide whitespace changes
Inline
Side-by-side
Showing
14 changed files
with
203 additions
and
205 deletions
+203
-205
slapos/cli/configure_local/__init__.py
slapos/cli/configure_local/__init__.py
+4
-4
slapos/client.py
slapos/client.py
+2
-2
slapos/collect/entity.py
slapos/collect/entity.py
+6
-10
slapos/collect/reporter.py
slapos/collect/reporter.py
+4
-2
slapos/collect/snapshot.py
slapos/collect/snapshot.py
+3
-1
slapos/collect/temperature/__init__.py
slapos/collect/temperature/__init__.py
+2
-1
slapos/grid/promise/generic.py
slapos/grid/promise/generic.py
+2
-2
slapos/manager/portredir.py
slapos/manager/portredir.py
+2
-3
slapos/proxy/db_version.py
slapos/proxy/db_version.py
+1
-1
slapos/proxy/views.py
slapos/proxy/views.py
+2
-2
slapos/slap/slap.py
slapos/slap/slap.py
+3
-3
slapos/tests/collect.py
slapos/tests/collect.py
+24
-24
slapos/tests/promise.py
slapos/tests/promise.py
+148
-148
slapos/util.py
slapos/util.py
+0
-2
No files found.
slapos/cli/configure_local/__init__.py
View file @
9de598f2
...
...
@@ -149,8 +149,8 @@ def _generateSlaposNodeConfigurationFile(slapos_node_config_path, args):
slapos_node_configuration_content = re.sub(
'
(
key_file
|
cert_file
|
certificate_repository_path
).
*=
.
*
\
n
',
'', slapos_node_configuration_content)
with open(slapos_node_config_path, '
w
b
') as fout:
fout.write(slapos_node_configuration_content
.encode('
utf8
')
)
with open(slapos_node_config_path, '
w
') as fout:
fout.write(slapos_node_configuration_content)
def _generateSlaposProxyConfigurationFile(conf):
template_arg_list = (__name__, '
..
/
..
/
slapos
-
proxy
.
cfg
.
example
')
...
...
@@ -172,8 +172,8 @@ def _generateSlaposProxyConfigurationFile(conf):
slapos_proxy_configuration_content = _replaceParameterValue(
slapos_proxy_configuration_template, to_replace)
with open(slapos_proxy_configuration_path, '
w
b
') as fout:
fout.write(slapos_proxy_configuration_content
.encode('
utf8
')
)
with open(slapos_proxy_configuration_path, '
w
') as fout:
fout.write(slapos_proxy_configuration_content)
return slapos_proxy_configuration_path
...
...
slapos/client.py
View file @
9de598f2
...
...
@@ -28,7 +28,7 @@
##############################################################################
import
atexit
import
six.moves.
configparser
from
six.moves
import
configparser
import
os
import
sys
...
...
@@ -52,7 +52,7 @@ class ClientConfig(object):
# Merges the arguments and configuration
try
:
configuration_dict
=
dict
(
configp
.
items
(
'slapconsole'
))
except
ConfigP
arser
.
NoSectionError
:
except
configp
arser
.
NoSectionError
:
pass
else
:
for
key
in
configuration_dict
:
...
...
slapos/collect/entity.py
View file @
9de598f2
...
...
@@ -74,19 +74,15 @@ class User(object):
time_cycle
=
self
.
disk_snapshot_params
.
get
(
'time_cycle'
,
0
)
database
.
connect
()
if
time_cycle
:
order
=
'date DESC, time DESC'
limit
=
1
query
=
database
.
select
(
table
=
"folder"
,
columns
=
"date, time"
,
order
=
order
,
limit
=
limit
,
where
=
"partition='%s'"
%
self
.
name
)
query_result
=
zip
(
*
query
)
if
len
(
query_result
):
date
,
time
=
(
query_result
[
0
][
0
],
query_result
[
1
][
0
])
latest_date
=
datetime
.
strptime
(
'%s %s'
%
(
date
,
time
),
"%Y-%m-%d %H:%M:%S"
)
for
date_time
in
database
.
select
(
table
=
"folder"
,
columns
=
"date, time"
,
order
=
'date DESC, time DESC'
,
limit
=
1
,
where
=
"partition='%s'"
%
self
.
name
):
latest_date
=
datetime
.
strptime
(
'%s %s'
%
date_time
,
"%Y-%m-%d %H:%M:%S"
)
if
(
datetime
.
now
()
-
latest_date
).
seconds
<
time_cycle
:
# wait the time cycle
return
break
pid_file
=
self
.
disk_snapshot_params
.
get
(
'pid_folder'
,
None
)
if
pid_file
is
not
None
:
pid_file
=
os
.
path
.
join
(
pid_file
,
'%s_disk_size.pid'
%
self
.
name
)
...
...
slapos/collect/reporter.py
View file @
9de598f2
...
...
@@ -40,6 +40,8 @@ import tarfile
import
time
import
psutil
import
six
log_file
=
False
class
Dumper
(
object
):
...
...
@@ -59,10 +61,10 @@ class SystemReporter(Dumper):
""" Dump data """
_date
=
time
.
strftime
(
"%Y-%m-%d"
)
self
.
db
.
connect
()
for
item
,
collected_item_list
in
s
elf
.
db
.
exportSystemAsDict
(
_date
).
iteritems
(
):
for
item
,
collected_item_list
in
s
ix
.
iteritems
(
self
.
db
.
exportSystemAsDict
(
_date
)
):
self
.
writeFile
(
item
,
folder
,
collected_item_list
)
for
partition
,
collected_item_list
in
s
elf
.
db
.
exportDiskAsDict
(
_date
).
iteritems
(
):
for
partition
,
collected_item_list
in
s
ix
.
iteritems
(
self
.
db
.
exportDiskAsDict
(
_date
)
):
partition_id
=
"_"
.
join
(
partition
.
split
(
"-"
)[:
-
1
]).
replace
(
"/"
,
"_"
)
item
=
"memory_%s"
%
partition
.
split
(
"-"
)[
-
1
]
self
.
writeFile
(
"disk_%s_%s"
%
(
item
,
partition_id
),
folder
,
collected_item_list
)
...
...
slapos/collect/snapshot.py
View file @
9de598f2
...
...
@@ -36,6 +36,8 @@ from .temperature import collectComputerTemperature, \
from
.temperature.heating
import
get_contribution_ratio
import
six
MEASURE_INTERVAL
=
5
class
_Snapshot
(
object
):
...
...
@@ -242,6 +244,6 @@ class ComputerSnapshot(_Snapshot):
DiskPartitionSnapshot
(
partition
.
device
,
partition
.
mountpoint
))
return
[(
k
,
v
)
for
k
,
v
in
partition_dict
.
iteritems
(
)]
return
[(
k
,
v
)
for
k
,
v
in
six
.
iteritems
(
partition_dict
)]
slapos/collect/temperature/__init__.py
View file @
9de598f2
...
...
@@ -18,7 +18,8 @@ def collectComputerTemperature(sensor_bin="sensors"):
cmd
=
[
"%s -u"
%
sensor_bin
]
sp
=
subprocess
.
Popen
(
cmd
,
stdout
=
subprocess
.
PIPE
,
stderr
=
subprocess
.
PIPE
,
shell
=
True
)
stderr
=
subprocess
.
PIPE
,
shell
=
True
,
universal_newlines
=
True
)
stdout
,
stderr
=
sp
.
communicate
()
...
...
slapos/grid/promise/generic.py
View file @
9de598f2
...
...
@@ -161,8 +161,8 @@ class GenericPromise(object):
self
.
logger
.
removeHandler
(
handler
)
if
self
.
__log_folder
is
None
:
# configure logger with StringIO
import
cStringIO
self
.
__logger_buffer
=
cStringIO
.
StringIO
()
from
six.moves
import
cStringIO
self
.
__logger_buffer
=
cStringIO
()
logger_handler
=
logging
.
StreamHandler
(
self
.
__logger_buffer
)
self
.
__log_file
=
None
else
:
...
...
slapos/manager/portredir.py
View file @
9de598f2
...
...
@@ -6,7 +6,7 @@ import os
from
.interface
import
IManager
from
six.moves
import
filter
from
zope
import
interface
from
zope
.interface
import
implementer
logger
=
logging
.
getLogger
(
__name__
)
...
...
@@ -22,9 +22,8 @@ def which(exename):
return
full_path
return
None
@
implementer
(
IManager
)
class
Manager
(
object
):
interface
.
implements
(
IManager
)
port_redirect_filename
=
'.slapos-port-redirect'
def
__init__
(
self
,
config
):
...
...
slapos/proxy/db_version.py
View file @
9de598f2
...
...
@@ -3,5 +3,5 @@
import
pkg_resources
from
slapos.util
import
bytes2str
DB_VERSION
=
bytes2str
(
pkg_resources
.
resource_stream
(
'slapos.proxy'
,
'schema.sql'
).
readline
()
.
strip
().
split
(
b':'
)[
1
])
DB_VERSION
=
bytes2str
(
pkg_resources
.
resource_stream
(
'slapos.proxy'
,
'schema.sql'
).
readline
()
).
strip
().
split
(
':'
)[
1
]
slapos/proxy/views.py
View file @
9de598f2
...
...
@@ -298,7 +298,7 @@ def useComputer():
@
app
.
route
(
'/loadComputerConfigurationFromXML'
,
methods
=
[
'POST'
])
def
loadComputerConfigurationFromXML
():
xml
=
request
.
form
[
'xml'
]
computer_dict
=
xml_marshaller
.
xml_marshaller
.
loads
(
str
(
xml
)
.
encode
(
'utf-8'
))
computer_dict
=
xml_marshaller
.
xml_marshaller
.
loads
(
xml
.
encode
(
'utf-8'
))
execute_db
(
'computer'
,
'INSERT OR REPLACE INTO %s values(:reference, :address, :netmask)'
,
computer_dict
)
for
partition
in
computer_dict
[
'partition_list'
]:
...
...
@@ -622,7 +622,7 @@ def requestNotSlave(software_release, software_type, partition_reference, partit
execute_db
(
'partition'
,
q
,
args
)
args
=
[]
partition
=
execute_db
(
'partition'
,
'SELECT * FROM %s WHERE reference=? and computer_reference=?'
,
[
partition
[
'reference'
]
.
encode
(),
partition
[
'computer_reference'
].
encode
()
],
one
=
True
)
[
partition
[
'reference'
]
,
partition
[
'computer_reference'
]
],
one
=
True
)
address_list
=
[]
for
address
in
execute_db
(
'partition_network'
,
'SELECT * FROM %s WHERE partition_reference=?'
,
[
partition
[
'reference'
]]):
address_list
.
append
((
address
[
'reference'
],
address
[
'address'
]))
...
...
slapos/slap/slap.py
View file @
9de598f2
...
...
@@ -873,12 +873,12 @@ class slap:
# XXX-Cedric: should raise something smarter than NotFound
raise
NotFoundError
xml
=
bytes
(
self
.
_connection_helper
.
GET
(
'registerComputerPartition'
,
xml
=
self
.
_connection_helper
.
GET
(
'registerComputerPartition'
,
params
=
{
'computer_reference'
:
computer_guid
,
'computer_partition_reference'
:
partition_id
,
}
)
)
)
result
=
xml_marshaller
.
loads
(
xml
)
# XXX: dirty hack to make computer partition usable. xml_marshaller is too
# low-level for our needs here.
...
...
@@ -913,7 +913,7 @@ class slap:
'software_release_url parameters are specified.'
)
params
[
'software_release_url'
]
=
software_release_url
xml
=
bytes
(
self
.
_connection_helper
.
GET
(
url
,
params
=
params
)
)
xml
=
self
.
_connection_helper
.
GET
(
url
,
params
=
params
)
result
=
xml_marshaller
.
loads
(
xml
)
assert
(
type
(
result
)
==
list
)
return
result
...
...
slapos/tests/collect.py
View file @
9de598f2
...
...
@@ -364,7 +364,7 @@ class TestCollectReport(unittest.TestCase):
with
tarfile
.
open
(
"%s.tar.gz"
%
dump_folder
)
as
tf
:
self
.
assertEquals
(
tf
.
getmembers
()[
0
].
name
,
"1990-01-01"
)
self
.
assertEquals
(
tf
.
getmembers
()[
1
].
name
,
"1990-01-01/test.txt"
)
self
.
assertEquals
(
tf
.
extractfile
(
tf
.
getmembers
()[
1
]).
read
(),
'hi'
)
self
.
assertEquals
(
tf
.
extractfile
(
tf
.
getmembers
()[
1
]).
read
(),
b
'hi'
)
class
TestCollectSnapshot
(
unittest
.
TestCase
):
...
...
@@ -482,10 +482,10 @@ class TestCollectEntity(unittest.TestCase):
config
.
set
(
'slapos'
,
'instance_root'
,
self
.
instance_root
)
user_dict
=
entity
.
get_user_list
(
config
)
username_
list
=
[
'slapuser0'
,
'slapuser1'
,
'slapuser2'
]
self
.
assertEquals
(
username_
list
,
user_dict
.
keys
(
))
username_
set
=
{
'slapuser0'
,
'slapuser1'
,
'slapuser2'
}
self
.
assertEquals
(
username_
set
,
set
(
user_dict
))
for
name
in
username_
lis
t
:
for
name
in
username_
se
t
:
self
.
assertEquals
(
user_dict
[
name
].
name
,
name
)
self
.
assertEquals
(
user_dict
[
name
].
snapshot_list
,
[])
expected_path
=
"%s/slappart%s"
%
(
self
.
instance_root
,
name
.
strip
(
"slapuser"
))
...
...
@@ -508,11 +508,11 @@ class TestCollectEntity(unittest.TestCase):
self
.
assertEquals
(
database
.
invoked_method_list
[
1
][
0
],
"insertUserSnapshot"
)
self
.
assertEquals
(
database
.
invoked_method_list
[
1
][
1
][
0
],
(
"fakeuser0"
,))
self
.
assertEquals
(
database
.
invoked_method_list
[
1
][
1
][
1
].
keys
(
),
[
'cpu_time'
,
'cpu_percent'
,
'process'
,
self
.
assertEquals
(
set
(
database
.
invoked_method_list
[
1
][
1
][
1
]
),
{
'cpu_time'
,
'cpu_percent'
,
'process'
,
'memory_rss'
,
'pid'
,
'memory_percent'
,
'io_rw_counter'
,
'insertion_date'
,
'insertion_time'
,
'io_cycles_counter'
,
'cpu_num_threads'
]
)
'io_cycles_counter'
,
'cpu_num_threads'
}
)
self
.
assertEquals
(
database
.
invoked_method_list
[
2
],
(
"commit"
,
""
))
self
.
assertEquals
(
database
.
invoked_method_list
[
3
],
(
"close"
,
""
))
...
...
@@ -527,19 +527,19 @@ class TestCollectEntity(unittest.TestCase):
self
.
assertEquals
(
database
.
invoked_method_list
[
1
][
0
],
"insertUserSnapshot"
)
self
.
assertEquals
(
database
.
invoked_method_list
[
1
][
1
][
0
],
(
"fakeuser0"
,))
self
.
assertEquals
(
database
.
invoked_method_list
[
1
][
1
][
1
].
keys
(
),
[
'cpu_time'
,
'cpu_percent'
,
'process'
,
self
.
assertEquals
(
set
(
database
.
invoked_method_list
[
1
][
1
][
1
]
),
{
'cpu_time'
,
'cpu_percent'
,
'process'
,
'memory_rss'
,
'pid'
,
'memory_percent'
,
'io_rw_counter'
,
'insertion_date'
,
'insertion_time'
,
'io_cycles_counter'
,
'cpu_num_threads'
]
)
'io_cycles_counter'
,
'cpu_num_threads'
}
)
self
.
assertEquals
(
database
.
invoked_method_list
[
2
],
(
"commit"
,
""
))
self
.
assertEquals
(
database
.
invoked_method_list
[
3
],
(
"close"
,
""
))
self
.
assertEquals
(
database
.
invoked_method_list
[
4
],
(
"connect"
,
""
))
self
.
assertEquals
(
database
.
invoked_method_list
[
5
][
0
],
"inserFolderSnapshot"
)
self
.
assertEquals
(
database
.
invoked_method_list
[
5
][
1
][
0
],
(
"fakeuser0"
,))
self
.
assertEquals
(
database
.
invoked_method_list
[
5
][
1
][
1
].
keys
(
),
[
'insertion_date'
,
'disk_usage'
,
'insertion_time'
]
)
self
.
assertEquals
(
set
(
database
.
invoked_method_list
[
5
][
1
][
1
]
),
{
'insertion_date'
,
'disk_usage'
,
'insertion_time'
}
)
self
.
assertEquals
(
database
.
invoked_method_list
[
6
],
(
"commit"
,
""
))
self
.
assertEquals
(
database
.
invoked_method_list
[
7
],
(
"close"
,
""
))
...
...
@@ -554,23 +554,23 @@ class TestCollectEntity(unittest.TestCase):
self
.
assertEquals
(
database
.
invoked_method_list
[
1
][
0
],
"insertUserSnapshot"
)
self
.
assertEquals
(
database
.
invoked_method_list
[
1
][
1
][
0
],
(
"fakeuser0"
,))
self
.
assertEquals
(
database
.
invoked_method_list
[
1
][
1
][
1
].
keys
(
),
[
'cpu_time'
,
'cpu_percent'
,
'process'
,
self
.
assertEquals
(
set
(
database
.
invoked_method_list
[
1
][
1
][
1
]
),
{
'cpu_time'
,
'cpu_percent'
,
'process'
,
'memory_rss'
,
'pid'
,
'memory_percent'
,
'io_rw_counter'
,
'insertion_date'
,
'insertion_time'
,
'io_cycles_counter'
,
'cpu_num_threads'
]
)
'io_cycles_counter'
,
'cpu_num_threads'
}
)
self
.
assertEquals
(
database
.
invoked_method_list
[
2
],
(
"commit"
,
""
))
self
.
assertEquals
(
database
.
invoked_method_list
[
3
],
(
"close"
,
""
))
self
.
assertEquals
(
database
.
invoked_method_list
[
4
],
(
"connect"
,
""
))
self
.
assertEquals
(
database
.
invoked_method_list
[
5
][
0
],
"select"
)
self
.
assertEquals
(
database
.
invoked_method_list
[
5
][
1
][
0
],
())
self
.
assertEquals
(
database
.
invoked_method_list
[
5
][
1
][
1
].
keys
(
),
[
'table'
,
'where'
,
'limit'
,
'order'
,
'columns'
]
)
self
.
assertEquals
(
set
(
database
.
invoked_method_list
[
5
][
1
][
1
]
),
{
'table'
,
'where'
,
'limit'
,
'order'
,
'columns'
}
)
self
.
assertEquals
(
database
.
invoked_method_list
[
6
][
0
],
"inserFolderSnapshot"
)
self
.
assertEquals
(
database
.
invoked_method_list
[
6
][
1
][
0
],
(
"fakeuser0"
,))
self
.
assertEquals
(
database
.
invoked_method_list
[
6
][
1
][
1
].
keys
(
),
[
'insertion_date'
,
'disk_usage'
,
'insertion_time'
]
)
self
.
assertEquals
(
set
(
database
.
invoked_method_list
[
6
][
1
][
1
]
),
{
'insertion_date'
,
'disk_usage'
,
'insertion_time'
}
)
self
.
assertEquals
(
database
.
invoked_method_list
[
7
],
(
"commit"
,
""
))
self
.
assertEquals
(
database
.
invoked_method_list
[
8
],
(
"close"
,
""
))
...
...
@@ -583,14 +583,14 @@ class TestCollectEntity(unittest.TestCase):
self
.
assertEquals
(
database
.
invoked_method_list
[
1
][
0
],
"insertComputerSnapshot"
)
self
.
assertEquals
(
database
.
invoked_method_list
[
1
][
1
][
0
],
())
self
.
assertEquals
(
database
.
invoked_method_list
[
1
][
1
][
1
].
keys
(
),
[
'insertion_time'
,
'insertion_date'
,
'cpu_num_core'
,
self
.
assertEquals
(
set
(
database
.
invoked_method_list
[
1
][
1
][
1
]
),
{
'insertion_time'
,
'insertion_date'
,
'cpu_num_core'
,
'partition_list'
,
'cpu_frequency'
,
'memory_size'
,
'cpu_type'
,
'memory_type'
]
)
'cpu_type'
,
'memory_type'
}
)
self
.
assertEquals
(
database
.
invoked_method_list
[
2
][
0
],
"insertSystemSnapshot"
)
self
.
assertEquals
(
database
.
invoked_method_list
[
2
][
1
][
0
],
())
self
.
assertEquals
(
set
(
database
.
invoked_method_list
[
2
][
1
][
1
]
.
keys
()
),
self
.
assertEquals
(
set
(
database
.
invoked_method_list
[
2
][
1
][
1
]),
set
([
'memory_used'
,
'cpu_percent'
,
'insertion_date'
,
'insertion_time'
,
'loadavg'
,
'memory_free'
,
'net_in_bytes'
,
'net_in_dropped'
,
'net_in_errors'
,
'net_out_bytes'
,
'net_out_dropped'
,
...
...
@@ -598,7 +598,7 @@ class TestCollectEntity(unittest.TestCase):
self
.
assertEquals
(
database
.
invoked_method_list
[
3
][
0
],
"insertDiskPartitionSnapshot"
)
self
.
assertEquals
(
database
.
invoked_method_list
[
3
][
1
][
0
],
())
self
.
assertEquals
(
set
(
database
.
invoked_method_list
[
3
][
1
][
1
]
.
keys
()
),
self
.
assertEquals
(
set
(
database
.
invoked_method_list
[
3
][
1
][
1
]),
set
([
'used'
,
'insertion_date'
,
'partition'
,
'free'
,
'mountpoint'
,
'insertion_time'
]))
...
...
slapos/tests/promise.py
View file @
9de598f2
This diff is collapsed.
Click to expand it.
slapos/util.py
View file @
9de598f2
...
...
@@ -84,8 +84,6 @@ def string_to_boolean(string):
The parser is completely arbitrary, see code for actual implementation.
"""
if
isinstance
(
string
,
bytes
):
string
=
string
.
decode
(
'utf-8'
)
try
:
return
(
'false'
,
'true'
).
index
(
string
.
lower
())
except
Exception
:
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment