Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
S
slapos.core
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
0
Issues
0
List
Boards
Labels
Milestones
Merge Requests
1
Merge Requests
1
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Analytics
Analytics
CI / CD
Repository
Value Stream
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
Romain Courteaud
slapos.core
Commits
7467ef4c
Commit
7467ef4c
authored
Nov 29, 2018
by
Bryton Lacquement
🚪
Committed by
Julien Muchembled
Dec 14, 2018
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
Add Python 3 support
parent
ae15060e
Changes
47
Expand all
Hide whitespace changes
Inline
Side-by-side
Showing
47 changed files
with
734 additions
and
777 deletions
+734
-777
slapos/bang.py
slapos/bang.py
+2
-2
slapos/cli/boot.py
slapos/cli/boot.py
+18
-16
slapos/cli/config.py
slapos/cli/config.py
+2
-2
slapos/cli/configure_client.py
slapos/cli/configure_client.py
+2
-1
slapos/cli/configure_local/__init__.py
slapos/cli/configure_local/__init__.py
+7
-6
slapos/cli/console.py
slapos/cli/console.py
+5
-1
slapos/cli/entry.py
slapos/cli/entry.py
+1
-1
slapos/cli/list.py
slapos/cli/list.py
+2
-1
slapos/cli/proxy_show.py
slapos/cli/proxy_show.py
+20
-8
slapos/cli/register.py
slapos/cli/register.py
+3
-2
slapos/client.py
slapos/client.py
+3
-3
slapos/collect/__init__.py
slapos/collect/__init__.py
+3
-2
slapos/collect/entity.py
slapos/collect/entity.py
+6
-10
slapos/collect/reporter.py
slapos/collect/reporter.py
+22
-33
slapos/collect/snapshot.py
slapos/collect/snapshot.py
+16
-15
slapos/collect/temperature/__init__.py
slapos/collect/temperature/__init__.py
+7
-13
slapos/format.py
slapos/format.py
+11
-10
slapos/grid/SlapObject.py
slapos/grid/SlapObject.py
+14
-13
slapos/grid/networkcache.py
slapos/grid/networkcache.py
+8
-7
slapos/grid/promise/__init__.py
slapos/grid/promise/__init__.py
+9
-10
slapos/grid/promise/generic.py
slapos/grid/promise/generic.py
+40
-46
slapos/grid/promise/wrapper.py
slapos/grid/promise/wrapper.py
+8
-11
slapos/grid/slapgrid.py
slapos/grid/slapgrid.py
+12
-11
slapos/grid/svcbackend.py
slapos/grid/svcbackend.py
+4
-3
slapos/grid/utils.py
slapos/grid/utils.py
+26
-24
slapos/grid/watchdog.py
slapos/grid/watchdog.py
+2
-1
slapos/manager/cpuset.py
slapos/manager/cpuset.py
+2
-4
slapos/manager/devperm.py
slapos/manager/devperm.py
+0
-1
slapos/manager/portredir.py
slapos/manager/portredir.py
+4
-5
slapos/manager/prerm.py
slapos/manager/prerm.py
+2
-3
slapos/proxy/__init__.py
slapos/proxy/__init__.py
+3
-1
slapos/proxy/db_version.py
slapos/proxy/db_version.py
+2
-1
slapos/proxy/views.py
slapos/proxy/views.py
+67
-88
slapos/slap/__init__.py
slapos/slap/__init__.py
+1
-1
slapos/slap/slap.py
slapos/slap/slap.py
+75
-85
slapos/tests/cli.py
slapos/tests/cli.py
+37
-58
slapos/tests/collect.py
slapos/tests/collect.py
+27
-27
slapos/tests/configure_local.py
slapos/tests/configure_local.py
+1
-1
slapos/tests/grid_utils.py
slapos/tests/grid_utils.py
+4
-4
slapos/tests/interface.py
slapos/tests/interface.py
+6
-8
slapos/tests/promise.py
slapos/tests/promise.py
+42
-42
slapos/tests/slap.py
slapos/tests/slap.py
+53
-52
slapos/tests/slapformat.py
slapos/tests/slapformat.py
+14
-2
slapos/tests/slapgrid.py
slapos/tests/slapgrid.py
+88
-86
slapos/tests/slapproxy/__init__.py
slapos/tests/slapproxy/__init__.py
+34
-34
slapos/tests/util.py
slapos/tests/util.py
+3
-12
slapos/util.py
slapos/util.py
+16
-10
No files found.
slapos/bang.py
View file @
7467ef4c
...
...
@@ -45,6 +45,6 @@ def do_bang(configp, message):
slap
=
slapos
.
slap
.
slap
()
slap
.
initializeConnection
(
master_url
,
key_file
=
key_file
,
cert_file
=
cert_file
)
computer
=
slap
.
registerComputer
(
computer_id
)
print
'Banging to %r'
%
master_url
print
(
'Banging to %r'
%
master_url
)
computer
.
bang
(
message
)
print
'Bang with message %r'
%
message
print
(
'Bang with message %r'
%
message
)
slapos/cli/boot.py
View file @
7467ef4c
...
...
@@ -27,8 +27,10 @@
#
##############################################################################
from
__future__
import
print_function
import
subprocess
import
urlparse
from
six.moves.urllib.parse
import
urlparse
from
time
import
sleep
import
glob
import
os
...
...
@@ -46,14 +48,14 @@ def _removeTimestamp(instancehome):
"""
timestamp_glob_path
=
"%s/slappart*/.timestamp"
%
instancehome
for
timestamp_path
in
glob
.
glob
(
timestamp_glob_path
):
print
"Removing %s"
%
timestamp_path
print
(
"Removing %s"
%
timestamp_path
)
os
.
remove
(
timestamp_path
)
def
_runBang
(
app
):
"""
Launch slapos node format.
"""
print
"[BOOT] Invoking slapos node bang..."
print
(
"[BOOT] Invoking slapos node bang..."
)
result
=
app
.
run
([
'node'
,
'bang'
,
'-m'
,
'Reboot'
])
if
result
==
1
:
return
0
...
...
@@ -63,7 +65,7 @@ def _runFormat(app):
"""
Launch slapos node format.
"""
print
"[BOOT] Invoking slapos node format..."
print
(
"[BOOT] Invoking slapos node format..."
)
result
=
app
.
run
([
'node'
,
'format'
,
'--now'
,
'--verbose'
])
if
result
==
1
:
return
0
...
...
@@ -73,30 +75,30 @@ def _ping(hostname):
"""
Ping a hostname
"""
print
"[BOOT] Invoking ipv4 ping to %s..."
%
hostname
print
(
"[BOOT] Invoking ipv4 ping to %s..."
%
hostname
)
p
=
subprocess
.
Popen
(
[
"ping"
,
"-c"
,
"2"
,
hostname
],
stdout
=
subprocess
.
PIPE
,
stderr
=
subprocess
.
PIPE
)
stdout
,
stderr
=
p
.
communicate
()
if
p
.
returncode
==
0
:
print
"[BOOT] IPv4 network reachable..."
print
(
"[BOOT] IPv4 network reachable..."
)
return
1
print
"[BOOT] [ERROR] IPv4 network unreachable..."
print
(
"[BOOT] [ERROR] IPv4 network unreachable..."
)
return
0
def
_ping6
(
hostname
):
"""
Ping an ipv6 address
"""
print
"[BOOT] Invoking ipv6 ping to %s..."
%
hostname
print
(
"[BOOT] Invoking ipv6 ping to %s..."
%
hostname
)
p
=
subprocess
.
Popen
(
[
"ping6"
,
"-c"
,
"2"
,
hostname
],
stdout
=
subprocess
.
PIPE
,
stderr
=
subprocess
.
PIPE
)
stdout
,
stderr
=
p
.
communicate
()
if
p
.
returncode
==
0
:
print
"[BOOT] IPv6 network reachable..."
print
(
"[BOOT] IPv6 network reachable..."
)
return
1
print
"[BOOT] [ERROR] IPv6 network unreachable..."
print
(
"[BOOT] [ERROR] IPv6 network unreachable..."
)
return
0
def
_test_ping
(
hostname
):
...
...
@@ -126,7 +128,7 @@ def _waitIpv6Ready(ipv6_interface):
test if ipv6 is ready on ipv6_interface
"""
ipv6_address
=
""
print
"[BOOT] Checking if %r has IPv6..."
%
ipv6_interface
print
(
"[BOOT] Checking if %r has IPv6..."
%
ipv6_interface
)
while
ipv6_address
==
""
:
for
inet_dict
in
netifaces
.
ifaddresses
(
ipv6_interface
)[
socket
.
AF_INET6
]:
ipv6_address
=
inet_dict
[
'addr'
].
split
(
'%'
)[
0
]
...
...
@@ -134,8 +136,8 @@ def _waitIpv6Ready(ipv6_interface):
break
else
:
ipv6_address
=
""
print
"[BOOT] [ERROR] No IPv6 found on interface %r, "
\
"try again in 5 seconds..."
%
ipv6_interface
print
(
"[BOOT] [ERROR] No IPv6 found on interface %r, "
"try again in 5 seconds..."
%
ipv6_interface
))
sleep
(
5
)
class
BootCommand
(
ConfigCommand
):
...
...
@@ -155,7 +157,7 @@ class BootCommand(ConfigCommand):
def
take_action
(
self
,
args
):
configp
=
self
.
fetch_config
(
args
)
instance_root
=
configp
.
get
(
'slapos'
,
'instance_root'
)
master_url
=
urlparse
.
urlparse
(
configp
.
get
(
'slapos'
,
'master_url'
))
master_url
=
urlparse
(
configp
.
get
(
'slapos'
,
'master_url'
))
master_hostname
=
master_url
.
hostname
# Check that we have IPv6 ready
...
...
@@ -177,12 +179,12 @@ class BootCommand(ConfigCommand):
app
=
SlapOSApp
()
# Make sure slapos node format returns ok
while
not
_runFormat
(
app
):
print
"[BOOT] [ERROR] Fail to format, try again in 15 seconds..."
print
(
"[BOOT] [ERROR] Fail to format, try again in 15 seconds..."
)
sleep
(
15
)
# Make sure slapos node bang returns ok
while
not
_runBang
(
app
):
print
"[BOOT] [ERROR] Fail to bang, try again in 15 seconds..."
print
(
"[BOOT] [ERROR] Fail to bang, try again in 15 seconds..."
)
sleep
(
15
)
_removeTimestamp
(
instance_root
)
slapos/cli/config.py
View file @
7467ef4c
...
...
@@ -27,7 +27,7 @@
#
##############################################################################
import
ConfigP
arser
from
six.moves
import
configp
arser
import
os
from
slapos.cli.command
import
Command
...
...
@@ -77,7 +77,7 @@ class ConfigCommand(Command):
if
not
os
.
path
.
exists
(
cfg_path
):
raise
ConfigError
(
'Configuration file does not exist: %s'
%
cfg_path
)
configp
=
ConfigP
arser
.
SafeConfigParser
()
configp
=
configp
arser
.
SafeConfigParser
()
if
configp
.
read
(
cfg_path
)
!=
[
cfg_path
]:
# bad permission, etc.
raise
ConfigError
(
'Cannot parse configuration file: %s'
%
cfg_path
)
...
...
slapos/cli/configure_client.py
View file @
7467ef4c
...
...
@@ -31,6 +31,7 @@ import re
import
os
import
sys
import
json
from
six.moves
import
input
import
requests
...
...
@@ -94,7 +95,7 @@ def fetch_configuration_template():
def
do_configure_client
(
logger
,
master_url_web
,
token
,
config_path
,
master_url
):
while
not
token
:
token
=
raw_
input
(
'Credential security token: '
).
strip
()
token
=
input
(
'Credential security token: '
).
strip
()
# Check for existence of previous configuration, certificate or key files
# where we expect to create them. If so, ask the use to manually remove them.
...
...
slapos/cli/configure_local/__init__.py
View file @
7467ef4c
...
...
@@ -39,6 +39,7 @@ from slapos.cli.config import ConfigCommand
from
slapos.grid.slapgrid
import
create_slapgrid_object
from
slapos.grid.utils
import
updateFile
,
createPrivateDirectory
from
slapos.grid.svcbackend
import
launchSupervisord
from
slapos.util
import
bytes2str
DEFAULT_COMPUTER_ID
=
'local_computer'
...
...
@@ -126,8 +127,8 @@ def _replaceParameterValue(original_content, to_replace):
def _generateSlaposNodeConfigurationFile(slapos_node_config_path, args):
template_arg_list = (__name__, '
..
/
..
/
slapos
.
cfg
.
example
')
with pkg_resources.resource_stream(*template_arg_list) as fout:
slapos_node_configuration_template = fout.read(
)
slapos_node_configuration_template =
\
bytes2str(pkg_resources.resource_string(*template_arg_list)
)
master_url = '
http
:
//%
s
:
%
s
' % (args.daemon_listen_ip, args.daemon_listen_port)
slapos_home = args.slapos_buildout_directory
to_replace = [
...
...
@@ -149,12 +150,12 @@ def _generateSlaposNodeConfigurationFile(slapos_node_config_path, args):
'
(
key_file
|
cert_file
|
certificate_repository_path
).
*=
.
*
\
n
',
'', slapos_node_configuration_content)
with open(slapos_node_config_path, '
w
') as fout:
fout.write(slapos_node_configuration_content
.encode('
utf8
')
)
fout.write(slapos_node_configuration_content)
def _generateSlaposProxyConfigurationFile(conf):
template_arg_list = (__name__, '
..
/
..
/
slapos
-
proxy
.
cfg
.
example
')
with pkg_resources.resource_stream(*template_arg_list) as fout:
slapos_proxy_configuration_template = fout.read(
)
slapos_proxy_configuration_template =
\
bytes2str(pkg_resources.resource_string(*template_arg_list)
)
slapos_proxy_configuration_path = os.path.join(
conf.slapos_configuration_directory, '
slapos
-
proxy
.
cfg
')
listening_ip, listening_port =
\
...
...
@@ -172,7 +173,7 @@ def _generateSlaposProxyConfigurationFile(conf):
slapos_proxy_configuration_template, to_replace)
with open(slapos_proxy_configuration_path, '
w
') as fout:
fout.write(slapos_proxy_configuration_content
.encode('
utf8
')
)
fout.write(slapos_proxy_configuration_content)
return slapos_proxy_configuration_path
...
...
slapos/cli/console.py
View file @
7467ef4c
...
...
@@ -32,6 +32,8 @@ import textwrap
from
slapos.cli.config
import
ClientConfigCommand
from
slapos.client
import
init
,
do_console
,
ClientConfig
from
six
import
exec_
class
ShellNotFound
(
Exception
):
pass
...
...
@@ -90,7 +92,9 @@ class ConsoleCommand(ClientConfigCommand):
local
=
init
(
conf
,
self
.
app
.
log
)
if
args
.
script_file
:
return
execfile
(
args
.
script_file
,
globals
(),
local
)
with
open
(
args
.
script_file
)
as
f
:
code
=
compile
(
f
.
read
(),
args
.
script_file
,
'exec'
)
return
exec_
(
code
,
globals
(),
local
)
if
not
any
([
args
.
python
,
args
.
ipython
,
args
.
bpython
]):
args
.
ipython
=
True
...
...
slapos/cli/entry.py
View file @
7467ef4c
...
...
@@ -247,7 +247,7 @@ class SlapOSApp(App):
return
if
self
.
options
.
log_color
:
import
coloredlogs
from
slapos.cli
import
coloredlogs
console
=
coloredlogs
.
ColoredStreamHandler
(
show_name
=
True
,
# logger name (slapos) and PID
show_severity
=
True
,
show_timestamps
=
self
.
options
.
log_time
,
...
...
slapos/cli/list.py
View file @
7467ef4c
...
...
@@ -29,6 +29,7 @@
import
logging
import
sys
import
six
from
slapos.cli.config
import
ClientConfigCommand
from
slapos.client
import
init
,
ClientConfig
...
...
@@ -64,5 +65,5 @@ def do_list(logger, conf, local):
logger
.
info
(
'No existing service.'
)
return
logger
.
info
(
'List of services:'
)
for
title
,
instance
in
instance_dict
.
iteritems
(
):
for
title
,
instance
in
six
.
iteritems
(
instance_dict
):
logger
.
info
(
'%s %s'
,
title
,
instance
.
_software_release_url
)
slapos/cli/proxy_show.py
View file @
7467ef4c
...
...
@@ -34,7 +34,6 @@ import logging
import
sys
import
os
import
subprocess
import
StringIO
import
lxml.etree
import
prettytable
...
...
@@ -43,8 +42,17 @@ import sqlite3
from
slapos.cli.config
import
ConfigCommand
from
slapos.proxy
import
ProxyConfig
from
slapos.proxy.db_version
import
DB_VERSION
from
slapos.util
import
sqlite_connect
from
slapos.util
import
sqlite_connect
,
str2bytes
if
bytes
is
str
:
from
io
import
BytesIO
class
StringIO
(
BytesIO
):
# Something between strict io.BytesIO and laxist/slow StringIO.StringIO
# (which starts returning unicode once unicode is written) for logging.
def
write
(
self
,
b
):
return
BytesIO
.
write
(
self
,
b
.
encode
(
'utf-8'
))
else
:
from
io
import
StringIO
class
ProxyShowCommand
(
ConfigCommand
):
"""
...
...
@@ -139,7 +147,7 @@ def log_params(logger, conn):
if
not
row
[
'connection_xml'
]:
continue
xml
=
str
(
row
[
'connection_xml'
])
xml
=
str
2bytes
(
row
[
'connection_xml'
])
logger
.
info
(
'%s: %s (type %s)'
,
row
[
'reference'
],
row
[
'partition_reference'
],
row
[
'software_type'
])
instance
=
lxml
.
etree
.
fromstring
(
xml
)
for
parameter
in
list
(
instance
):
...
...
@@ -150,9 +158,12 @@ def log_params(logger, conn):
# _ is usually json encoded - re-format to make it easier to read
if
name
==
'_'
:
try
:
text
=
json
.
dumps
(
json
.
loads
(
text
),
indent
=
2
)
text
=
json
.
dumps
(
json
.
loads
(
text
),
indent
=
2
,
sort_keys
=
True
)
except
ValueError
:
pass
else
:
# to avoid differences between Py2 and Py3 in unit tests
text
=
'
\
n
'
.
join
(
map
(
str
.
rstrip
,
text
.
splitlines
()))
logger
.
info
(
' %s = %s'
,
name
,
text
)
...
...
@@ -208,7 +219,7 @@ def do_show(conf):
# to paginate input, honoring $PAGER.
output
=
sys
.
stdout
if
output
.
isatty
():
output
=
StringIO
.
StringIO
()
output
=
StringIO
()
proxy_show_logger
=
logging
.
getLogger
(
__name__
)
handler
=
logging
.
StreamHandler
(
output
)
handler
.
setLevel
(
logging
.
DEBUG
)
...
...
@@ -221,7 +232,8 @@ def do_show(conf):
conn
=
sqlite_connect
(
conf
.
database_uri
)
conn
.
row_factory
=
sqlite3
.
Row
conn
.
create_function
(
'md5'
,
1
,
lambda
s
:
hashlib
.
md5
(
s
).
hexdigest
())
conn
.
create_function
(
'md5'
,
1
,
lambda
s
:
hashlib
.
md5
(
str2bytes
(
s
)).
hexdigest
())
call_table
=
[
(
conf
.
computers
,
log_computer_table
),
...
...
@@ -248,4 +260,4 @@ def do_show(conf):
close_fds
=
True
,
shell
=
True
,
stdin
=
subprocess
.
PIPE
,)
pager
.
communicate
(
output
.
getvalue
().
encode
(
'utf-8'
))
pager
.
communicate
(
str2bytes
(
output
.
getvalue
()
))
slapos/cli/register.py
View file @
7467ef4c
...
...
@@ -36,6 +36,7 @@ import sys
import
pkg_resources
import
requests
import
json
from
six.moves
import
input
from
slapos.cli.command
import
Command
,
must_be_root
...
...
@@ -327,7 +328,7 @@ def gen_auth(conf):
else
:
yield
conf
.
login
,
getpass
.
getpass
()
while
ask
:
yield
raw_
input
(
'SlapOS Master Login: '
),
getpass
.
getpass
()
yield
input
(
'SlapOS Master Login: '
),
getpass
.
getpass
()
def
do_register
(
conf
):
...
...
@@ -348,7 +349,7 @@ def do_register(conf):
password
=
password
)
else
:
while
not
conf
.
token
:
conf
.
token
=
raw_
input
(
'Computer security token: '
).
strip
()
conf
.
token
=
input
(
'Computer security token: '
).
strip
()
certificate
,
key
=
get_certificate_key_pair
(
conf
.
logger
,
conf
.
master_url_web
,
...
...
slapos/client.py
View file @
7467ef4c
...
...
@@ -28,7 +28,7 @@
##############################################################################
import
atexit
import
ConfigP
arser
from
six.moves
import
configp
arser
import
os
import
sys
...
...
@@ -52,7 +52,7 @@ class ClientConfig(object):
# Merges the arguments and configuration
try
:
configuration_dict
=
dict
(
configp
.
items
(
'slapconsole'
))
except
ConfigP
arser
.
NoSectionError
:
except
configp
arser
.
NoSectionError
:
pass
else
:
for
key
in
configuration_dict
:
...
...
@@ -119,7 +119,7 @@ def _getSoftwareReleaseFromSoftwareString(logger, software_string, product):
try
:
return
product
.
__getattr__
(
software_string
[
len
(
SOFTWARE_PRODUCT_NAMESPACE
):])
except
AttributeError
as
e
:
logger
.
error
(
'Error: %s Exiting now.'
%
e
.
messag
e
)
logger
.
error
(
'Error: %s Exiting now.'
,
e
)
sys
.
exit
(
1
)
def
do_console
(
local
):
...
...
slapos/collect/__init__.py
View file @
7467ef4c
...
...
@@ -27,6 +27,7 @@
#
##############################################################################
from
__future__
import
print_function
from
psutil
import
process_iter
,
NoSuchProcess
,
AccessDenied
from
time
import
strftime
import
shutil
...
...
@@ -42,7 +43,7 @@ from slapos.collect.reporter import RawCSVDumper, \
compressLogFolder
,
\
ConsumptionReport
from
entity
import
get_user_list
,
Computer
from
.
entity
import
get_user_list
,
Computer
def
_get_time
():
return
strftime
(
"%Y-%m-%d -- %H:%M:%S"
).
split
(
" -- "
)
...
...
@@ -157,5 +158,5 @@ def do_collect(conf):
database
.
garbageCollect
()
except
AccessDenied
:
print
"You HAVE TO execute this script with root permission."
print
(
"You HAVE TO execute this script with root permission."
)
slapos/collect/entity.py
View file @
7467ef4c
...
...
@@ -74,19 +74,15 @@ class User(object):
time_cycle
=
self
.
disk_snapshot_params
.
get
(
'time_cycle'
,
0
)
database
.
connect
()
if
time_cycle
:
order
=
'date DESC, time DESC'
limit
=
1
query
=
database
.
select
(
table
=
"folder"
,
columns
=
"date, time"
,
order
=
order
,
limit
=
limit
,
where
=
"partition='%s'"
%
self
.
name
)
query_result
=
zip
(
*
query
)
if
len
(
query_result
):
date
,
time
=
(
query_result
[
0
][
0
],
query_result
[
1
][
0
])
latest_date
=
datetime
.
strptime
(
'%s %s'
%
(
date
,
time
),
"%Y-%m-%d %H:%M:%S"
)
for
date_time
in
database
.
select
(
table
=
"folder"
,
columns
=
"date, time"
,
order
=
'date DESC, time DESC'
,
limit
=
1
,
where
=
"partition='%s'"
%
self
.
name
):
latest_date
=
datetime
.
strptime
(
'%s %s'
%
date_time
,
"%Y-%m-%d %H:%M:%S"
)
if
(
datetime
.
now
()
-
latest_date
).
seconds
<
time_cycle
:
# wait the time cycle
return
break
pid_file
=
self
.
disk_snapshot_params
.
get
(
'pid_folder'
,
None
)
if
pid_file
is
not
None
:
pid_file
=
os
.
path
.
join
(
pid_file
,
'%s_disk_size.pid'
%
self
.
name
)
...
...
slapos/collect/reporter.py
View file @
7467ef4c
...
...
@@ -40,6 +40,8 @@ import tarfile
import
time
import
psutil
import
six
log_file
=
False
class
Dumper
(
object
):
...
...
@@ -59,10 +61,10 @@ class SystemReporter(Dumper):
""" Dump data """
_date
=
time
.
strftime
(
"%Y-%m-%d"
)
self
.
db
.
connect
()
for
item
,
collected_item_list
in
s
elf
.
db
.
exportSystemAsDict
(
_date
).
iteritems
(
):
for
item
,
collected_item_list
in
s
ix
.
iteritems
(
self
.
db
.
exportSystemAsDict
(
_date
)
):
self
.
writeFile
(
item
,
folder
,
collected_item_list
)
for
partition
,
collected_item_list
in
s
elf
.
db
.
exportDiskAsDict
(
_date
).
iteritems
(
):
for
partition
,
collected_item_list
in
s
ix
.
iteritems
(
self
.
db
.
exportDiskAsDict
(
_date
)
):
partition_id
=
"_"
.
join
(
partition
.
split
(
"-"
)[:
-
1
]).
replace
(
"/"
,
"_"
)
item
=
"memory_%s"
%
partition
.
split
(
"-"
)[
-
1
]
self
.
writeFile
(
"disk_%s_%s"
%
(
item
,
partition_id
),
folder
,
collected_item_list
)
...
...
@@ -135,62 +137,53 @@ class ConsumptionReportBase(object):
def
getPartitionCPULoadAverage
(
self
,
partition_id
,
date_scope
):
self
.
db
.
connect
()
query_result_cursor
=
self
.
db
.
select
(
"user"
,
date_scope
,
(
cpu_percent_sum
,),
=
self
.
db
.
select
(
"user"
,
date_scope
,
columns
=
"SUM(cpu_percent)"
,
where
=
"partition = '%s'"
%
partition_id
)
cpu_percent_sum
=
zip
(
*
query_result_cursor
)
if
len
(
cpu_percent_sum
)
and
cpu_percent_sum
[
0
][
0
]
is
None
:
if
cpu_percent_sum
is
None
:
return
query_result_cursor
=
self
.
db
.
select
(
"user"
,
date_scope
,
(
sample_amount
,),
=
self
.
db
.
select
(
"user"
,
date_scope
,
columns
=
"COUNT(DISTINCT time)"
,
where
=
"partition = '%s'"
%
partition_id
)
sample_amount
=
zip
(
*
query_result_cursor
)
self
.
db
.
close
()
if
len
(
sample_amount
)
and
len
(
cpu_percent_sum
):
return
cpu_percent_sum
[
0
][
0
]
/
sample_amount
[
0
][
0
]
return
cpu_percent_sum
/
sample_amount
def
getPartitionUsedMemoryAverage
(
self
,
partition_id
,
date_scope
):
self
.
db
.
connect
()
query_result_cursor
=
self
.
db
.
select
(
"user"
,
date_scope
,
(
memory_sum
,),
=
self
.
db
.
select
(
"user"
,
date_scope
,
columns
=
"SUM(memory_rss)"
,
where
=
"partition = '%s'"
%
partition_id
)
memory_sum
=
zip
(
*
query_result_cursor
)
if
len
(
memory_sum
)
and
memory_sum
[
0
][
0
]
is
None
:
if
memory_sum
is
None
:
return
query_result_cursor
=
self
.
db
.
select
(
"user"
,
date_scope
,
(
sample_amount
,),
=
self
.
db
.
select
(
"user"
,
date_scope
,
columns
=
"COUNT(DISTINCT time)"
,
where
=
"partition = '%s'"
%
partition_id
)
sample_amount
=
zip
(
*
query_result_cursor
)
self
.
db
.
close
()
if
len
(
sample_amount
)
and
len
(
memory_sum
):
return
memory_sum
[
0
][
0
]
/
sample_amount
[
0
][
0
]
return
memory_sum
/
sample_amount
def
getPartitionDiskUsedAverage
(
self
,
partition_id
,
date_scope
):
self
.
db
.
connect
()
query_result_cursor
=
self
.
db
.
select
(
"folder"
,
date_scope
,
(
disk_used_sum
,),
=
self
.
db
.
select
(
"folder"
,
date_scope
,
columns
=
"SUM(disk_used)"
,
where
=
"partition = '%s'"
%
partition_id
)
disk_used_sum
=
zip
(
*
query_result_cursor
)
if
len
(
disk_used_sum
)
and
disk_used_sum
[
0
][
0
]
is
None
:
if
disk_used_sum
is
None
:
return
query_result_cursor
=
self
.
db
.
select
(
"folder"
,
date_scope
,
(
collect_amount
,),
=
self
.
db
.
select
(
"folder"
,
date_scope
,
columns
=
"COUNT(DISTINCT time)"
,
where
=
"partition = '%s'"
%
partition_id
)
collect_amount
=
zip
(
*
query_result_cursor
)
self
.
db
.
close
()
if
len
(
collect_amount
)
and
len
(
disk_used_sum
):
return
disk_used_sum
[
0
][
0
]
/
collect_amount
[
0
][
0
]
return
disk_used_sum
/
collect_amount
class
ConsumptionReport
(
ConsumptionReportBase
):
...
...
@@ -287,7 +280,7 @@ class ConsumptionReport(ConsumptionReportBase):
reference
=
user
,
category
=
""
)
with
open
(
xml_report_path
,
'w'
)
as
f
:
with
open
(
xml_report_path
,
'w
b
'
)
as
f
:
f
.
write
(
journal
.
getXML
())
f
.
close
()
...
...
@@ -298,23 +291,19 @@ class ConsumptionReport(ConsumptionReportBase):
def
_getCpuLoadAverageConsumption
(
self
,
date_scope
):
self
.
db
.
connect
()
query_result_cursor
=
self
.
db
.
select
(
"system"
,
date_scope
,
(
cpu_load_percent_list
,),
=
self
.
db
.
select
(
"system"
,
date_scope
,
columns
=
"SUM(cpu_percent)/COUNT(cpu_percent)"
)
cpu_load_percent_list
=
zip
(
*
query_result_cursor
)
self
.
db
.
close
()
if
len
(
cpu_load_percent_list
):
return
cpu_load_percent_list
[
0
][
0
]
return
cpu_load_percent_list
def
_getMemoryAverageConsumption
(
self
,
date_scope
):
self
.
db
.
connect
()
query_result_cursor
=
self
.
db
.
select
(
"system"
,
date_scope
,
(
memory_used_list
,),
=
self
.
db
.
select
(
"system"
,
date_scope
,
columns
=
"SUM(memory_used)/COUNT(memory_used)"
)
memory_used_list
=
zip
(
*
query_result_cursor
)
self
.
db
.
close
()
if
len
(
memory_used_list
):
return
memory_used_list
[
0
][
0
]
return
memory_used_list
def
_getZeroEmissionContribution
(
self
):
self
.
db
.
connect
()
...
...
@@ -329,7 +318,7 @@ class Journal(object):
def
getXML
(
self
):
report
=
ElementTree
.
tostring
(
self
.
root
)
return
"<?xml version='1.0' encoding='utf-8'?>%s"
%
report
return
b
"<?xml version='1.0' encoding='utf-8'?>%s"
%
report
def
newTransaction
(
self
,
portal_type
=
"Sale Packing List"
):
transaction
=
ElementTree
.
SubElement
(
self
.
root
,
"transaction"
)
...
...
slapos/collect/snapshot.py
View file @
7467ef4c
...
...
@@ -27,13 +27,15 @@
#
##############################################################################
from
__future__
import
print_function
import
psutil
import
os
import
subprocess
from
temperature
import
collectComputerTemperature
,
\
launchTemperatureTest
from
.temperature
import
collectComputerTemperature
,
launchTemperatureTest
from
temperature.heating
import
get_contribution_ratio
from
.temperature.heating
import
get_contribution_ratio
import
six
MEASURE_INTERVAL
=
5
...
...
@@ -159,7 +161,7 @@ class HeatingContributionSnapshot(_Snapshot):
result
=
launchTemperatureTest
(
sensor_id
)
if
result
is
None
:
print
"Impossible to test sensor: %s "
%
sensor_id
print
(
"Impossible to test sensor: %s "
%
sensor_id
)
initial_temperature
,
final_temperature
,
duration
=
result
...
...
@@ -215,8 +217,7 @@ class ComputerSnapshot(_Snapshot):
#
self
.
system_snapshot
=
SystemSnapshot
()
self
.
temperature_snapshot_list
=
self
.
_get_temperature_snapshot_list
()
self
.
disk_snapshot_list
=
[]
self
.
partition_list
=
self
.
_get_physical_disk_info
()
self
.
_get_physical_disk_info
()
if
test_heating
and
model_id
is
not
None
\
and
sensor_id
is
not
None
:
...
...
@@ -231,16 +232,16 @@ class ComputerSnapshot(_Snapshot):
return
temperature_snapshot_list
def
_get_physical_disk_info
(
self
):
partition_dict
=
{}
# XXX: merge the following 2 to avoid calling disk_usage() twice
self
.
disk_snapshot_list
=
[]
self
.
partition_list
=
[]
partition_set
=
set
()
for
partition
in
psutil
.
disk_partitions
():
if
partition
.
device
not
in
partition_dict
:
dev
=
partition
.
device
if
dev
not
in
partition_set
:
# XXX: useful ?
partition_set
.
add
(
dev
)
usage
=
psutil
.
disk_usage
(
partition
.
mountpoint
)
partition_dict
[
partition
.
device
]
=
usage
.
total
self
.
partition_list
.
append
((
dev
,
usage
.
total
))
self
.
disk_snapshot_list
.
append
(
DiskPartitionSnapshot
(
partition
.
device
,
partition
.
mountpoint
))
return
[(
k
,
v
)
for
k
,
v
in
partition_dict
.
iteritems
()]
DiskPartitionSnapshot
(
dev
,
partition
.
mountpoint
))
slapos/collect/temperature/__init__.py
View file @
7467ef4c
from
__future__
import
print_function
from
multiprocessing
import
Process
,
active_children
,
cpu_count
,
Pipe
import
subprocess
...
...
@@ -14,28 +15,21 @@ except NotImplementedError:
DEFAULT_CPU
=
1
def
collectComputerTemperature
(
sensor_bin
=
"sensors"
):
cmd
=
[
"%s -u"
%
sensor_bin
]
sp
=
subprocess
.
Popen
(
cmd
,
stdout
=
subprocess
.
PIPE
,
stderr
=
subprocess
.
PIPE
,
shell
=
True
)
stdout
,
stderr
=
sp
.
communicate
()
stdout
=
subprocess
.
check_output
((
sensor_bin
,
'-u'
),
universal_newlines
=
True
)
sensor_output_list
=
stdout
.
splitlines
()
adapter_name
=
""
sensor_temperature_list
=
[]
for
line_number
in
range
(
len
(
sensor_output_list
)):
found_sensor
=
None
stripped_line
=
sensor_output_list
[
line_number
].
strip
()
for
line_number
,
sensor_output
in
enumerate
(
sensor_output_list
):
stripped_line
=
sensor_output
.
strip
()
if
stripped_line
.
startswith
(
"Adapter:"
):
adapter_name
=
sensor_output_list
[
line_number
-
1
]
elif
stripped_line
.
startswith
(
"temp"
)
and
"_input"
in
stripped_line
:
temperature
=
sensor_output
_list
[
line_number
].
split
()[
-
1
]
temperature
=
sensor_output
.
split
()[
-
1
]
found_sensor
=
[
"%s %s"
%
(
adapter_name
,
sensor_output_list
[
line_number
-
1
]),
float
(
temperature
)]
if
found_sensor
is
not
None
:
critical
=
'1000'
maximal
=
'1000'
for
next_line
in
sensor_output_list
[
line_number
+
1
:
line_number
+
3
]:
...
...
@@ -120,7 +114,7 @@ def launchTemperatureTest(sensor_id, sensor_bin="sensors", timeout=600, interval
for
connection
in
process_connection_list
:
try
:
print
connection
.
recv
(
)
print
(
connection
.
recv
()
)
except
EOFError
:
continue
...
...
slapos/format.py
View file @
7467ef4c
...
...
@@ -28,7 +28,7 @@
#
##############################################################################
import
ConfigP
arser
from
six.moves
import
configp
arser
import
errno
import
fcntl
import
grp
...
...
@@ -51,12 +51,13 @@ import time
import
traceback
import
zipfile
import
platform
from
urllib2
import
urlopen
from
six.moves.urllib.request
import
urlopen
import
six
import
lxml.etree
import
xml_marshaller.xml_marshaller
from
slapos.util
import
mkdir_p
,
ipv6FromBin
,
binFromIpv6
,
lenNetmaskIpv6
from
slapos.util
import
dumps
,
mkdir_p
,
ipv6FromBin
,
binFromIpv6
,
lenNetmaskIpv6
import
slapos.slap
as
slap