Commit 7467ef4c authored by Bryton Lacquement's avatar Bryton Lacquement 🚪 Committed by Julien Muchembled

Add Python 3 support

parent ae15060e
......@@ -45,6 +45,6 @@ def do_bang(configp, message):
slap = slapos.slap.slap()
slap.initializeConnection(master_url, key_file=key_file, cert_file=cert_file)
computer = slap.registerComputer(computer_id)
print 'Banging to %r' % master_url
print('Banging to %r' % master_url)
computer.bang(message)
print 'Bang with message %r' % message
print('Bang with message %r' % message)
......@@ -27,8 +27,10 @@
#
##############################################################################
from __future__ import print_function
import subprocess
import urlparse
from six.moves.urllib.parse import urlparse
from time import sleep
import glob
import os
......@@ -46,14 +48,14 @@ def _removeTimestamp(instancehome):
"""
timestamp_glob_path = "%s/slappart*/.timestamp" % instancehome
for timestamp_path in glob.glob(timestamp_glob_path):
print "Removing %s" % timestamp_path
print("Removing %s" % timestamp_path)
os.remove(timestamp_path)
def _runBang(app):
"""
Launch slapos node format.
"""
print "[BOOT] Invoking slapos node bang..."
print("[BOOT] Invoking slapos node bang...")
result = app.run(['node', 'bang', '-m', 'Reboot'])
if result == 1:
return 0
......@@ -63,7 +65,7 @@ def _runFormat(app):
"""
Launch slapos node format.
"""
print "[BOOT] Invoking slapos node format..."
print("[BOOT] Invoking slapos node format...")
result = app.run(['node', 'format', '--now', '--verbose'])
if result == 1:
return 0
......@@ -73,30 +75,30 @@ def _ping(hostname):
"""
Ping a hostname
"""
print "[BOOT] Invoking ipv4 ping to %s..." % hostname
print("[BOOT] Invoking ipv4 ping to %s..." % hostname)
p = subprocess.Popen(
["ping", "-c", "2", hostname],
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
stdout, stderr = p.communicate()
if p.returncode == 0:
print "[BOOT] IPv4 network reachable..."
print("[BOOT] IPv4 network reachable...")
return 1
print "[BOOT] [ERROR] IPv4 network unreachable..."
print("[BOOT] [ERROR] IPv4 network unreachable...")
return 0
def _ping6(hostname):
"""
Ping an ipv6 address
"""
print "[BOOT] Invoking ipv6 ping to %s..." % hostname
print("[BOOT] Invoking ipv6 ping to %s..." % hostname)
p = subprocess.Popen(
["ping6", "-c", "2", hostname],
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
stdout, stderr = p.communicate()
if p.returncode == 0:
print "[BOOT] IPv6 network reachable..."
print("[BOOT] IPv6 network reachable...")
return 1
print "[BOOT] [ERROR] IPv6 network unreachable..."
print("[BOOT] [ERROR] IPv6 network unreachable...")
return 0
def _test_ping(hostname):
......@@ -126,7 +128,7 @@ def _waitIpv6Ready(ipv6_interface):
test if ipv6 is ready on ipv6_interface
"""
ipv6_address = ""
print "[BOOT] Checking if %r has IPv6..." % ipv6_interface
print("[BOOT] Checking if %r has IPv6..." % ipv6_interface)
while ipv6_address == "":
for inet_dict in netifaces.ifaddresses(ipv6_interface)[socket.AF_INET6]:
ipv6_address = inet_dict['addr'].split('%')[0]
......@@ -134,8 +136,8 @@ def _waitIpv6Ready(ipv6_interface):
break
else:
ipv6_address = ""
print "[BOOT] [ERROR] No IPv6 found on interface %r, " \
"try again in 5 seconds..." % ipv6_interface
print("[BOOT] [ERROR] No IPv6 found on interface %r, "
"try again in 5 seconds..." % ipv6_interface))
sleep(5)
class BootCommand(ConfigCommand):
......@@ -155,7 +157,7 @@ class BootCommand(ConfigCommand):
def take_action(self, args):
configp = self.fetch_config(args)
instance_root = configp.get('slapos','instance_root')
master_url = urlparse.urlparse(configp.get('slapos','master_url'))
master_url = urlparse(configp.get('slapos','master_url'))
master_hostname = master_url.hostname
# Check that we have IPv6 ready
......@@ -177,12 +179,12 @@ class BootCommand(ConfigCommand):
app = SlapOSApp()
# Make sure slapos node format returns ok
while not _runFormat(app):
print "[BOOT] [ERROR] Fail to format, try again in 15 seconds..."
print("[BOOT] [ERROR] Fail to format, try again in 15 seconds...")
sleep(15)
# Make sure slapos node bang returns ok
while not _runBang(app):
print "[BOOT] [ERROR] Fail to bang, try again in 15 seconds..."
print("[BOOT] [ERROR] Fail to bang, try again in 15 seconds...")
sleep(15)
_removeTimestamp(instance_root)
......@@ -27,7 +27,7 @@
#
##############################################################################
import ConfigParser
from six.moves import configparser
import os
from slapos.cli.command import Command
......@@ -77,7 +77,7 @@ class ConfigCommand(Command):
if not os.path.exists(cfg_path):
raise ConfigError('Configuration file does not exist: %s' % cfg_path)
configp = ConfigParser.SafeConfigParser()
configp = configparser.SafeConfigParser()
if configp.read(cfg_path) != [cfg_path]:
# bad permission, etc.
raise ConfigError('Cannot parse configuration file: %s' % cfg_path)
......
......@@ -31,6 +31,7 @@ import re
import os
import sys
import json
from six.moves import input
import requests
......@@ -94,7 +95,7 @@ def fetch_configuration_template():
def do_configure_client(logger, master_url_web, token, config_path, master_url):
while not token:
token = raw_input('Credential security token: ').strip()
token = input('Credential security token: ').strip()
# Check for existence of previous configuration, certificate or key files
# where we expect to create them. If so, ask the use to manually remove them.
......
......@@ -39,6 +39,7 @@ from slapos.cli.config import ConfigCommand
from slapos.grid.slapgrid import create_slapgrid_object
from slapos.grid.utils import updateFile, createPrivateDirectory
from slapos.grid.svcbackend import launchSupervisord
from slapos.util import bytes2str
DEFAULT_COMPUTER_ID = 'local_computer'
......@@ -126,8 +127,8 @@ def _replaceParameterValue(original_content, to_replace):
def _generateSlaposNodeConfigurationFile(slapos_node_config_path, args):
template_arg_list = (__name__, '../../slapos.cfg.example')
with pkg_resources.resource_stream(*template_arg_list) as fout:
slapos_node_configuration_template = fout.read()
slapos_node_configuration_template = \
bytes2str(pkg_resources.resource_string(*template_arg_list))
master_url = 'http://%s:%s' % (args.daemon_listen_ip, args.daemon_listen_port)
slapos_home = args.slapos_buildout_directory
to_replace = [
......@@ -149,12 +150,12 @@ def _generateSlaposNodeConfigurationFile(slapos_node_config_path, args):
'(key_file|cert_file|certificate_repository_path).*=.*\n',
'', slapos_node_configuration_content)
with open(slapos_node_config_path, 'w') as fout:
fout.write(slapos_node_configuration_content.encode('utf8'))
fout.write(slapos_node_configuration_content)
def _generateSlaposProxyConfigurationFile(conf):
template_arg_list = (__name__, '../../slapos-proxy.cfg.example')
with pkg_resources.resource_stream(*template_arg_list) as fout:
slapos_proxy_configuration_template = fout.read()
slapos_proxy_configuration_template = \
bytes2str(pkg_resources.resource_string(*template_arg_list))
slapos_proxy_configuration_path = os.path.join(
conf.slapos_configuration_directory, 'slapos-proxy.cfg')
listening_ip, listening_port = \
......@@ -172,7 +173,7 @@ def _generateSlaposProxyConfigurationFile(conf):
slapos_proxy_configuration_template, to_replace)
with open(slapos_proxy_configuration_path, 'w') as fout:
fout.write(slapos_proxy_configuration_content.encode('utf8'))
fout.write(slapos_proxy_configuration_content)
return slapos_proxy_configuration_path
......
......@@ -32,6 +32,8 @@ import textwrap
from slapos.cli.config import ClientConfigCommand
from slapos.client import init, do_console, ClientConfig
from six import exec_
class ShellNotFound(Exception):
pass
......@@ -90,7 +92,9 @@ class ConsoleCommand(ClientConfigCommand):
local = init(conf, self.app.log)
if args.script_file:
return execfile(args.script_file, globals(), local)
with open(args.script_file) as f:
code = compile(f.read(), args.script_file, 'exec')
return exec_(code, globals(), local)
if not any([args.python, args.ipython, args.bpython]):
args.ipython = True
......
......@@ -247,7 +247,7 @@ class SlapOSApp(App):
return
if self.options.log_color:
import coloredlogs
from slapos.cli import coloredlogs
console = coloredlogs.ColoredStreamHandler(show_name=True, # logger name (slapos) and PID
show_severity=True,
show_timestamps=self.options.log_time,
......
......@@ -29,6 +29,7 @@
import logging
import sys
import six
from slapos.cli.config import ClientConfigCommand
from slapos.client import init, ClientConfig
......@@ -64,5 +65,5 @@ def do_list(logger, conf, local):
logger.info('No existing service.')
return
logger.info('List of services:')
for title, instance in instance_dict.iteritems():
for title, instance in six.iteritems(instance_dict):
logger.info('%s %s', title, instance._software_release_url)
......@@ -34,7 +34,6 @@ import logging
import sys
import os
import subprocess
import StringIO
import lxml.etree
import prettytable
......@@ -43,8 +42,17 @@ import sqlite3
from slapos.cli.config import ConfigCommand
from slapos.proxy import ProxyConfig
from slapos.proxy.db_version import DB_VERSION
from slapos.util import sqlite_connect
from slapos.util import sqlite_connect, str2bytes
if bytes is str:
from io import BytesIO
class StringIO(BytesIO):
# Something between strict io.BytesIO and laxist/slow StringIO.StringIO
# (which starts returning unicode once unicode is written) for logging.
def write(self, b):
return BytesIO.write(self, b.encode('utf-8'))
else:
from io import StringIO
class ProxyShowCommand(ConfigCommand):
"""
......@@ -139,7 +147,7 @@ def log_params(logger, conn):
if not row['connection_xml']:
continue
xml = str(row['connection_xml'])
xml = str2bytes(row['connection_xml'])
logger.info('%s: %s (type %s)', row['reference'], row['partition_reference'], row['software_type'])
instance = lxml.etree.fromstring(xml)
for parameter in list(instance):
......@@ -150,9 +158,12 @@ def log_params(logger, conn):
# _ is usually json encoded - re-format to make it easier to read
if name == '_':
try:
text = json.dumps(json.loads(text), indent=2)
text = json.dumps(json.loads(text),
indent=2, sort_keys=True)
except ValueError:
pass
else: # to avoid differences between Py2 and Py3 in unit tests
text = '\n'.join(map(str.rstrip, text.splitlines()))
logger.info(' %s = %s', name, text)
......@@ -208,7 +219,7 @@ def do_show(conf):
# to paginate input, honoring $PAGER.
output = sys.stdout
if output.isatty():
output = StringIO.StringIO()
output = StringIO()
proxy_show_logger = logging.getLogger(__name__)
handler = logging.StreamHandler(output)
handler.setLevel(logging.DEBUG)
......@@ -221,7 +232,8 @@ def do_show(conf):
conn = sqlite_connect(conf.database_uri)
conn.row_factory = sqlite3.Row
conn.create_function('md5', 1, lambda s: hashlib.md5(s).hexdigest())
conn.create_function('md5', 1,
lambda s: hashlib.md5(str2bytes(s)).hexdigest())
call_table = [
(conf.computers, log_computer_table),
......@@ -248,4 +260,4 @@ def do_show(conf):
close_fds=True,
shell=True,
stdin=subprocess.PIPE,)
pager.communicate(output.getvalue().encode('utf-8'))
pager.communicate(str2bytes(output.getvalue()))
......@@ -36,6 +36,7 @@ import sys
import pkg_resources
import requests
import json
from six.moves import input
from slapos.cli.command import Command, must_be_root
......@@ -327,7 +328,7 @@ def gen_auth(conf):
else:
yield conf.login, getpass.getpass()
while ask:
yield raw_input('SlapOS Master Login: '), getpass.getpass()
yield input('SlapOS Master Login: '), getpass.getpass()
def do_register(conf):
......@@ -348,7 +349,7 @@ def do_register(conf):
password=password)
else:
while not conf.token:
conf.token = raw_input('Computer security token: ').strip()
conf.token = input('Computer security token: ').strip()
certificate, key = get_certificate_key_pair(conf.logger,
conf.master_url_web,
......
......@@ -28,7 +28,7 @@
##############################################################################
import atexit
import ConfigParser
from six.moves import configparser
import os
import sys
......@@ -52,7 +52,7 @@ class ClientConfig(object):
# Merges the arguments and configuration
try:
configuration_dict = dict(configp.items('slapconsole'))
except ConfigParser.NoSectionError:
except configparser.NoSectionError:
pass
else:
for key in configuration_dict:
......@@ -119,7 +119,7 @@ def _getSoftwareReleaseFromSoftwareString(logger, software_string, product):
try:
return product.__getattr__(software_string[len(SOFTWARE_PRODUCT_NAMESPACE):])
except AttributeError as e:
logger.error('Error: %s Exiting now.' % e.message)
logger.error('Error: %s Exiting now.', e)
sys.exit(1)
def do_console(local):
......
......@@ -27,6 +27,7 @@
#
##############################################################################
from __future__ import print_function
from psutil import process_iter, NoSuchProcess, AccessDenied
from time import strftime
import shutil
......@@ -42,7 +43,7 @@ from slapos.collect.reporter import RawCSVDumper, \
compressLogFolder, \
ConsumptionReport
from entity import get_user_list, Computer
from .entity import get_user_list, Computer
def _get_time():
return strftime("%Y-%m-%d -- %H:%M:%S").split(" -- ")
......@@ -157,5 +158,5 @@ def do_collect(conf):
database.garbageCollect()
except AccessDenied:
print "You HAVE TO execute this script with root permission."
print("You HAVE TO execute this script with root permission.")
......@@ -74,19 +74,15 @@ class User(object):
time_cycle = self.disk_snapshot_params.get('time_cycle', 0)
database.connect()
if time_cycle:
order = 'date DESC, time DESC'
limit = 1
query = database.select(table="folder", columns="date, time",
order=order, limit=limit,
where="partition='%s'" % self.name)
query_result = zip(*query)
if len(query_result):
date, time = (query_result[0][0], query_result[1][0])
latest_date = datetime.strptime('%s %s' % (date, time),
"%Y-%m-%d %H:%M:%S")
for date_time in database.select(table="folder", columns="date, time",
order='date DESC, time DESC', limit=1,
where="partition='%s'" % self.name):
latest_date = datetime.strptime('%s %s' % date_time,
"%Y-%m-%d %H:%M:%S")
if (datetime.now() - latest_date).seconds < time_cycle:
# wait the time cycle
return
break
pid_file = self.disk_snapshot_params.get('pid_folder', None)
if pid_file is not None:
pid_file = os.path.join(pid_file, '%s_disk_size.pid' % self.name)
......
......@@ -40,6 +40,8 @@ import tarfile
import time
import psutil
import six
log_file = False
class Dumper(object):
......@@ -59,10 +61,10 @@ class SystemReporter(Dumper):
""" Dump data """
_date = time.strftime("%Y-%m-%d")
self.db.connect()
for item, collected_item_list in self.db.exportSystemAsDict(_date).iteritems():
for item, collected_item_list in six.iteritems(self.db.exportSystemAsDict(_date)):
self.writeFile(item, folder, collected_item_list)
for partition, collected_item_list in self.db.exportDiskAsDict(_date).iteritems():
for partition, collected_item_list in six.iteritems(self.db.exportDiskAsDict(_date)):
partition_id = "_".join(partition.split("-")[:-1]).replace("/", "_")
item = "memory_%s" % partition.split("-")[-1]
self.writeFile("disk_%s_%s" % (item, partition_id), folder, collected_item_list)
......@@ -135,62 +137,53 @@ class ConsumptionReportBase(object):
def getPartitionCPULoadAverage(self, partition_id, date_scope):
self.db.connect()
query_result_cursor = self.db.select("user", date_scope,
(cpu_percent_sum,), = self.db.select("user", date_scope,
columns="SUM(cpu_percent)",
where="partition = '%s'" % partition_id)
cpu_percent_sum = zip(*query_result_cursor)
if len(cpu_percent_sum) and cpu_percent_sum[0][0] is None:
if cpu_percent_sum is None:
return
query_result_cursor = self.db.select("user", date_scope,
(sample_amount,), = self.db.select("user", date_scope,
columns="COUNT(DISTINCT time)",
where="partition = '%s'" % partition_id)
sample_amount = zip(*query_result_cursor)
self.db.close()
if len(sample_amount) and len(cpu_percent_sum):
return cpu_percent_sum[0][0]/sample_amount[0][0]
return cpu_percent_sum/sample_amount
def getPartitionUsedMemoryAverage(self, partition_id, date_scope):
self.db.connect()
query_result_cursor = self.db.select("user", date_scope,
(memory_sum,), = self.db.select("user", date_scope,
columns="SUM(memory_rss)",
where="partition = '%s'" % partition_id)
memory_sum = zip(*query_result_cursor)
if len(memory_sum) and memory_sum[0][0] is None:
if memory_sum is None:
return
query_result_cursor = self.db.select("user", date_scope,
(sample_amount,), = self.db.select("user", date_scope,
columns="COUNT(DISTINCT time)",
where="partition = '%s'" % partition_id)
sample_amount = zip(*query_result_cursor)
self.db.close()
if len(sample_amount) and len(memory_sum):
return memory_sum[0][0]/sample_amount[0][0]
return memory_sum/sample_amount
def getPartitionDiskUsedAverage(self, partition_id, date_scope):
self.db.connect()
query_result_cursor = self.db.select("folder", date_scope,
(disk_used_sum,), = self.db.select("folder", date_scope,
columns="SUM(disk_used)",
where="partition = '%s'" % partition_id)
disk_used_sum = zip(*query_result_cursor)
if len(disk_used_sum) and disk_used_sum[0][0] is None:
if disk_used_sum is None:
return
query_result_cursor = self.db.select("folder", date_scope,
(collect_amount,), = self.db.select("folder", date_scope,
columns="COUNT(DISTINCT time)",
where="partition = '%s'" % partition_id)
collect_amount = zip(*query_result_cursor)
self.db.close()
if len(collect_amount) and len(disk_used_sum):
return disk_used_sum[0][0]/collect_amount[0][0]
return disk_used_sum/collect_amount
class ConsumptionReport(ConsumptionReportBase):
......@@ -287,7 +280,7 @@ class ConsumptionReport(ConsumptionReportBase):
reference=user,
category="")
with open(xml_report_path, 'w') as f:
with open(xml_report_path, 'wb') as f:
f.write(journal.getXML())
f.close()
......@@ -298,23 +291,19 @@ class ConsumptionReport(ConsumptionReportBase):
def _getCpuLoadAverageConsumption(self, date_scope):
self.db.connect()
query_result_cursor = self.db.select("system", date_scope,
(cpu_load_percent_list,), = self.db.select("system", date_scope,
columns="SUM(cpu_percent)/COUNT(cpu_percent)")
cpu_load_percent_list = zip(*query_result_cursor)
self.db.close()
if len(cpu_load_percent_list):
return cpu_load_percent_list[0][0]
return cpu_load_percent_list
def _getMemoryAverageConsumption(self, date_scope):
self.db.connect()
query_result_cursor = self.db.select("system", date_scope,
(memory_used_list,), = self.db.select("system", date_scope,
columns="SUM(memory_used)/COUNT(memory_used)")
memory_used_list = zip(*query_result_cursor)
self.db.close()
if len(memory_used_list):
return memory_used_list[0][0]
return memory_used_list
def _getZeroEmissionContribution(self):
self.db.connect()
......@@ -329,7 +318,7 @@ class Journal(object):
def getXML(self):
report = ElementTree.tostring(self.root)
return "<?xml version='1.0' encoding='utf-8'?>%s" % report
return b"<?xml version='1.0' encoding='utf-8'?>%s" % report
def newTransaction(self, portal_type="Sale Packing List"):
transaction = ElementTree.SubElement(self.root, "transaction")
......
......@@ -27,13 +27,15 @@
#
##############################################################################
from __future__ import print_function
import psutil
import os
import subprocess
from temperature import collectComputerTemperature, \
launchTemperatureTest
from .temperature import collectComputerTemperature, launchTemperatureTest
from temperature.heating import get_contribution_ratio
from .temperature.heating import get_contribution_ratio
import six
MEASURE_INTERVAL = 5
......@@ -159,7 +161,7 @@ class HeatingContributionSnapshot(_Snapshot):
result = launchTemperatureTest(sensor_id)
if result is None:
print "Impossible to test sensor: %s " % sensor_id
print("Impossible to test sensor: %s " % sensor_id)
initial_temperature, final_temperature, duration = result
......@@ -215,8 +217,7 @@ class ComputerSnapshot(_Snapshot):
#
self.system_snapshot = SystemSnapshot()
self.temperature_snapshot_list = self._get_temperature_snapshot_list()
self.disk_snapshot_list = []
self.partition_list = self._get_physical_disk_info()
self._get_physical_disk_info()
if test_heating and model_id is not None \
and sensor_id is not None:
......@@ -231,16 +232,16 @@ class ComputerSnapshot(_Snapshot):
return temperature_snapshot_list
def _get_physical_disk_info(self):
partition_dict = {}
# XXX: merge the following 2 to avoid calling disk_usage() twice
self.disk_snapshot_list = []
self.partition_list = []
partition_set = set()
for partition in psutil.disk_partitions():
if partition.device not in partition_dict:
dev = partition.device
if dev not in partition_set: # XXX: useful ?
partition_set.add(dev)
usage = psutil.disk_usage(partition.mountpoint)
partition_dict[partition.device] = usage.total
self.partition_list.append((dev, usage.total))
self.disk_snapshot_list.append(
DiskPartitionSnapshot(partition.device,
partition.mountpoint))
return [(k, v) for k, v in partition_dict.iteritems()]
DiskPartitionSnapshot(dev, partition.mountpoint))
from __future__ import print_function
from multiprocessing import Process, active_children, cpu_count, Pipe
import subprocess
......@@ -14,28 +15,21 @@ except NotImplementedError:
DEFAULT_CPU = 1
def collectComputerTemperature(sensor_bin="sensors"):
cmd = ["%s -u" % sensor_bin]
sp = subprocess.Popen(cmd, stdout=subprocess.PIPE,
stderr=subprocess.PIPE, shell=True)
stdout, stderr = sp.communicate()
stdout = subprocess.check_output((sensor_bin, '-u'), universal_newlines=True)
sensor_output_list = stdout.splitlines()
adapter_name = ""
sensor_temperature_list = []
for line_number in range(len(sensor_output_list)):
found_sensor = None
stripped_line = sensor_output_list[line_number].strip()
for line_number, sensor_output in enumerate(sensor_output_list):
stripped_line = sensor_output.strip()
if stripped_line.startswith("Adapter:"):
adapter_name = sensor_output_list[line_number-1]
elif stripped_line.startswith("temp") and "_input" in stripped_line:
temperature = sensor_output_list[line_number].split()[-1]
temperature = sensor_output.split()[-1]
found_sensor = ["%s %s" % (adapter_name, sensor_output_list[line_number-1]), float(temperature)]
if found_sensor is not None:
critical = '1000'
maximal = '1000'
for next_line in sensor_output_list[line_number+1:line_number+3]:
......@@ -120,7 +114,7 @@ def launchTemperatureTest(sensor_id, sensor_bin="sensors", timeout=600, interval
for connection in process_connection_list:
try:
print connection.recv()
print(connection.recv())
except EOFError:
continue
......
......@@ -28,7 +28,7 @@
#
##############################################################################
import ConfigParser
from six.moves import configparser
import errno
import fcntl
import grp
......@@ -51,12 +51,13 @@ import time
import traceback
import zipfile
import platform
from urllib2 import urlopen
from six.moves.urllib.request import urlopen
import six
import lxml.etree
import xml_marshaller.xml_marshaller
from slapos.util import mkdir_p, ipv6FromBin, binFromIpv6, lenNetmaskIpv6
from slapos.util import dumps, mkdir_p, ipv6FromBin, binFromIpv6, lenNetmaskIpv6
import slapos.slap as slap