Commit 221ca8a6 authored by Lutra Conseil's avatar Lutra Conseil Committed by Rafael Monnerat

[slapos.collect] Collect temperature and report consumption

 In relation to data collection this commit introduces:

  - data collector for temperature (parsing sensors)
  - Heating testing for determinate the Zero Emission Ratio of the computer.

 In relation to reports this commits introduces:

  - Consumption report for computer usage (CPU Load, Zero Emission Ratio, Memory)
  - Consumption report for partitions (CPU Load and Memory)
  - All consumption reports uses TioSafe XML Format
parent 4ca4b68e
...@@ -29,6 +29,8 @@ ...@@ -29,6 +29,8 @@
from psutil import process_iter, NoSuchProcess, AccessDenied from psutil import process_iter, NoSuchProcess, AccessDenied
from time import strftime from time import strftime
import shutil
import datetime
from slapos.collect.db import Database from slapos.collect.db import Database
from slapos.util import mkdir_p from slapos.util import mkdir_p
import os import os
...@@ -37,7 +39,8 @@ import stat ...@@ -37,7 +39,8 @@ import stat
from slapos.collect.snapshot import ProcessSnapshot, ComputerSnapshot from slapos.collect.snapshot import ProcessSnapshot, ComputerSnapshot
from slapos.collect.reporter import RawCSVDumper, \ from slapos.collect.reporter import RawCSVDumper, \
SystemCSVReporterDumper, \ SystemCSVReporterDumper, \
compressLogFolder compressLogFolder, \
ConsumptionReport
from entity import get_user_list, Computer from entity import get_user_list, Computer
...@@ -50,6 +53,12 @@ def build_snapshot(proc): ...@@ -50,6 +53,12 @@ def build_snapshot(proc):
except NoSuchProcess: except NoSuchProcess:
return None return None
def _get_uptime():
# Linux only
if os.path.exists('/proc/uptime'):
with open('/proc/uptime', 'r') as f:
return datetime.timedelta(seconds=float(f.readline().split()[0]))
def current_state(user_dict): def current_state(user_dict):
""" """
Iterator used to apply build_snapshot(...) on every single relevant process. Iterator used to apply build_snapshot(...) on every single relevant process.
...@@ -81,13 +90,46 @@ def do_collect(conf): ...@@ -81,13 +90,46 @@ def do_collect(conf):
log_directory = "%s/var/data-log" % conf.get("slapos", "instance_root") log_directory = "%s/var/data-log" % conf.get("slapos", "instance_root")
mkdir_p(log_directory, 0o755) mkdir_p(log_directory, 0o755)
consumption_report_directory = "%s/var/consumption-report" % \
conf.get("slapos", "instance_root")
mkdir_p(consumption_report_directory, 0o755)
xml_report_directory = "%s/var/xml_report/%s" % \
(conf.get("slapos", "instance_root"),
conf.get("slapos", "computer_id"))
mkdir_p(xml_report_directory, 0o755)
if stat.S_IMODE(os.stat(log_directory).st_mode) != 0o755: if stat.S_IMODE(os.stat(log_directory).st_mode) != 0o755:
os.chmod(log_directory, 0o755) os.chmod(log_directory, 0o755)
database = Database(log_directory) database = Database(log_directory)
computer = Computer(ComputerSnapshot()) if conf.has_option("slapformat", "computer_model_id"):
computer_model_id = conf.get("slapformat",
"computer_model_id")
else:
computer_model_id = "no_model"
uptime = _get_uptime()
if conf.has_option("slapformat", "heating_sensor_id"):
heating_sensor_id = conf.get("slapformat",
"heating_sensor_id")
database.connect()
test_heating = uptime is not None and \
uptime > datetime.timedelta(seconds=86400) and \
database.getLastHeatingTestTime() > uptime
database.close()
else:
heating_sensor_id = "no_sensor"
test_heating = False
computer = Computer(ComputerSnapshot(model_id=computer_model_id,
sensor_id = heating_sensor_id,
test_heating=test_heating))
computer.save(database, collected_date, collected_time) computer.save(database, collected_date, collected_time)
for user in user_dict.values(): for user in user_dict.values():
...@@ -95,6 +137,19 @@ def do_collect(conf): ...@@ -95,6 +137,19 @@ def do_collect(conf):
SystemCSVReporterDumper(database).dump(log_directory) SystemCSVReporterDumper(database).dump(log_directory)
RawCSVDumper(database).dump(log_directory) RawCSVDumper(database).dump(log_directory)
consumption_report = ConsumptionReport(
computer_id=conf.get("slapos", "computer_id"),
user_list=get_user_list(conf),
database=database,
location=consumption_report_directory)
base = datetime.datetime.today()
for x in range(1, 3):
report_file = consumption_report.buildXMLReport(
(base - datetime.timedelta(days=x)).strftime("%Y-%m-%d"))
if report_file is not None:
shutil.copy(report_file, xml_report_directory)
compressLogFolder(log_directory) compressLogFolder(log_directory)
......
...@@ -35,7 +35,10 @@ import datetime ...@@ -35,7 +35,10 @@ import datetime
class Database: class Database:
database_name = "collector.db" database_name = "collector.db"
table_list = ["user", "computer", "system", "disk"] table_list = ["user", "computer", "system", "disk", \
"temperature", "heating"]
preserve_table_list = ["heating"]
CREATE_USER_TABLE = "create table if not exists user " \ CREATE_USER_TABLE = "create table if not exists user " \
"(partition text, pid real, process text, " \ "(partition text, pid real, process text, " \
" cpu_percent real, cpu_time real, " \ " cpu_percent real, cpu_time real, " \
...@@ -61,6 +64,14 @@ class Database: ...@@ -61,6 +64,14 @@ class Database:
"(partition text, used text, free text, mountpoint text, " \ "(partition text, used text, free text, mountpoint text, " \
" date text, time text, reported integer NULL DEFAULT 0)" " date text, time text, reported integer NULL DEFAULT 0)"
CREATE_TEMPERATURE_TABLE = "create table if not exists temperature " \
"(sensor_id name, temperature real, alarm integer, "\
"date text, time text, reported integer NULL DEFAULT 0)"
CREATE_HEATING_TABLE = "create table if not exists heating " \
"(model_id name, sensor_id name, initial_temperature real, "\
" final_temperature real, delta_time real, zero_emission_ratio real, "\
"date text, time text, reported integer NULL DEFAULT 0)"
INSERT_USER_TEMPLATE = "insert into user(" \ INSERT_USER_TEMPLATE = "insert into user(" \
"partition, pid, process, cpu_percent, cpu_time, " \ "partition, pid, process, cpu_percent, cpu_time, " \
...@@ -87,6 +98,17 @@ class Database: ...@@ -87,6 +98,17 @@ class Database:
" date, time) values "\ " date, time) values "\
"( %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, '%s', '%s' )" "( %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, '%s', '%s' )"
INSERT_TEMPERATURE_TEMPLATE = "insert into temperature("\
" sensor_id, temperature, alarm," \
" date, time) values "\
"( '%s', %s, %s, '%s', '%s' )"
INSERT_HEATING_TEMPLATE = "insert into heating("\
" model_id, sensor_id, initial_temperature, final_temperature, "\
" delta_time, zero_emission_ratio," \
" date, time) values "\
"( '%s', '%s', %s, %s, %s, %s, '%s', '%s' )"
def __init__(self, directory = None): def __init__(self, directory = None):
assert self.database_name is not None assert self.database_name is not None
self.uri = os.path.join(directory, self.database_name) self.uri = os.path.join(directory, self.database_name)
...@@ -118,6 +140,8 @@ class Database: ...@@ -118,6 +140,8 @@ class Database:
self._execute(self.CREATE_COMPUTER_TABLE) self._execute(self.CREATE_COMPUTER_TABLE)
self._execute(self.CREATE_SYSTEM_TABLE) self._execute(self.CREATE_SYSTEM_TABLE)
self._execute(self.CREATE_DISK_PARTITION) self._execute(self.CREATE_DISK_PARTITION)
self._execute(self.CREATE_TEMPERATURE_TABLE)
self._execute(self.CREATE_HEATING_TABLE)
self.commit() self.commit()
self.close() self.close()
...@@ -174,6 +198,28 @@ class Database: ...@@ -174,6 +198,28 @@ class Database:
self._execute(insertion_sql) self._execute(insertion_sql)
return insertion_sql return insertion_sql
def insertTemperatureSnapshot(self, sensor_id, temperature, alarm,
insertion_date, insertion_time):
""" Include Temperature information Snapshot on the database
"""
insertion_sql = self.INSERT_TEMPERATURE_TEMPLATE % \
(sensor_id, temperature, alarm, insertion_date, insertion_time)
self._execute(insertion_sql)
return insertion_sql
def insertHeatingSnapshot(self, model_id, sensor_id, initial_temperature,
final_temperature, delta_time, zero_emission_ratio,
insertion_date, insertion_time):
""" Include Heating information Snapshot on the database
"""
insertion_sql = self.INSERT_HEATING_TEMPLATE % \
(model_id, sensor_id, initial_temperature, final_temperature,
delta_time, zero_emission_ratio, insertion_date, insertion_time)
self._execute(insertion_sql)
return insertion_sql
def getTableList(self): def getTableList(self):
""" Get the list of tables from the database """ Get the list of tables from the database
""" """
...@@ -202,7 +248,8 @@ class Database: ...@@ -202,7 +248,8 @@ class Database:
self.connect() self.connect()
for table in self.table_list: for table in self.table_list:
self._execute(delete_sql % (table, where_clause)) if table not in self.preserve_table_list:
self._execute(delete_sql % (table, where_clause))
self.commit() self.commit()
self.close() self.close()
...@@ -227,12 +274,17 @@ class Database: ...@@ -227,12 +274,17 @@ class Database:
for table in table_list: for table in table_list:
self._execute(update_sql % (table, date_scope)) self._execute(update_sql % (table, date_scope))
def select(self, table, date=None, columns="*"): def select(self, table, date=None, columns="*", where=None):
""" Query database for a full table information """ """ Query database for a full table information """
if date is not None: if date is not None:
where_clause = " WHERE date = '%s' " % date where_clause = " WHERE date = '%s' " % date
else: else:
where_clause = "" where_clause = ""
if where is not None:
if where_clause == "":
where_clause += " WHERE 1 = 1 "
where_clause += " AND %s " % where
select_sql = "SELECT %s FROM %s %s " % (columns, table, where_clause) select_sql = "SELECT %s FROM %s %s " % (columns, table, where_clause)
return self._execute(select_sql) return self._execute(select_sql)
...@@ -334,3 +386,32 @@ class Database: ...@@ -334,3 +386,32 @@ class Database:
return collected_entry_dict return collected_entry_dict
def getLastHeatingTestTime(self):
select_sql = "SELECT date, time FROM heating ORDER BY date, time DESC LIMIT 1"
for __date, __time in self._execute(select_sql):
_date = datetime.datetime.strptime("%s %s" % (__date, __time), "%Y-%m-%d %H:%M:%S")
return datetime.datetime.now() - _date
return datetime.timedelta(weeks=520)
def getLastZeroEmissionRatio(self):
select_sql = "SELECT zero_emission_ratio FROM heating ORDER BY date, time DESC LIMIT 1"
for entry in self._execute(select_sql):
return entry[0]
return -1
def getCollectedTemperatureList(self, sensor_id=None, limit=1):
""" Query database for a full table information """
if limit > 0:
limit_clause = "LIMIT %s" % (limit,)
else:
limit_clause = ""
if sensor_id is not None:
where_clause = "WHERE sensor_id = '%s'" % (sensor_id)
else:
where_clause = ""
select_sql = "SELECT * FROM temperature %s ORDER BY time DESC %s" % (where_clause, limit_clause)
return self._execute(select_sql)
...@@ -85,6 +85,8 @@ class Computer(dict): ...@@ -85,6 +85,8 @@ class Computer(dict):
self._save_computer_snapshot(database, collected_date, collected_time) self._save_computer_snapshot(database, collected_date, collected_time)
self._save_system_snapshot(database, collected_date, collected_time) self._save_system_snapshot(database, collected_date, collected_time)
self._save_disk_partition_snapshot(database, collected_date, collected_time) self._save_disk_partition_snapshot(database, collected_date, collected_time)
self._save_temperature_snapshot(database, collected_date, collected_time)
self._save_heating_snapshot(database, collected_date, collected_time)
database.commit() database.commit()
database.close() database.close()
...@@ -127,3 +129,25 @@ class Computer(dict): ...@@ -127,3 +129,25 @@ class Computer(dict):
insertion_date=collected_date, insertion_date=collected_date,
insertion_time=collected_time) insertion_time=collected_time)
def _save_temperature_snapshot(self, database, collected_date, collected_time):
for temperature_snapshot in self.computer_snapshot.get("temperature_snapshot_list"):
database.insertTemperatureSnapshot(
sensor_id=temperature_snapshot.sensor_id,
temperature=temperature_snapshot.temperature,
alarm=temperature_snapshot.alarm,
insertion_date=collected_date,
insertion_time=collected_time)
def _save_heating_snapshot(self, database, collected_date, collected_time):
heating_snapshot = self.computer_snapshot.get("heating_contribution_snapshot")
if heating_snapshot is not None and \
heating_snapshot.initial_temperature is not None:
database.insertHeatingSnapshot(
initial_temperature=heating_snapshot.initial_temperature,
final_temperature=heating_snapshot.final_temperature,
delta_time=heating_snapshot.delta_time,
model_id=heating_snapshot.model_id,
sensor_id=heating_snapshot.sensor_id,
zero_emission_ratio=heating_snapshot.zero_emission_ratio,
insertion_date=collected_date,
insertion_time=collected_time)
...@@ -27,15 +27,20 @@ ...@@ -27,15 +27,20 @@
# #
############################################################################## ##############################################################################
from lxml import etree as ElementTree
from slapos.util import mkdir_p from slapos.util import mkdir_p
import os.path
import json import csv
import tarfile
import glob import glob
import json
import os
import os.path
import shutil import shutil
import csv import tarfile
from time import strftime import time
import psutil
log_file = False
class Dumper(object): class Dumper(object):
...@@ -52,7 +57,7 @@ class SystemReporter(Dumper): ...@@ -52,7 +57,7 @@ class SystemReporter(Dumper):
def dump(self, folder): def dump(self, folder):
""" Dump data """ """ Dump data """
_date = strftime("%Y-%m-%d") _date = time.strftime("%Y-%m-%d")
self.db.connect() self.db.connect()
for item, collected_item_list in self.db.exportSystemAsDict(_date).iteritems(): for item, collected_item_list in self.db.exportSystemAsDict(_date).iteritems():
self.writeFile(item, folder, collected_item_list) self.writeFile(item, folder, collected_item_list)
...@@ -87,7 +92,7 @@ class RawDumper(Dumper): ...@@ -87,7 +92,7 @@ class RawDumper(Dumper):
""" Dump raw data in a certain format """ Dump raw data in a certain format
""" """
def dump(self, folder): def dump(self, folder):
date = strftime("%Y-%m-%d") date = time.strftime("%Y-%m-%d")
self.db.connect() self.db.connect()
table_list = self.db.getTableList() table_list = self.db.getTableList()
for date_scope, amount in self.db.getDateScopeList(ignore_date=date): for date_scope, amount in self.db.getDateScopeList(ignore_date=date):
...@@ -123,5 +128,199 @@ def compressLogFolder(log_directory): ...@@ -123,5 +128,199 @@ def compressLogFolder(log_directory):
finally: finally:
os.chdir(initial_folder) os.chdir(initial_folder)
class ConsumptionReport(object):
def __init__(self, database, computer_id, location, user_list):
self.computer_id = computer_id
self.db = database
self.user_list = user_list
self.location = location
def buildXMLReport(self, date_scope):
xml_report_path = "%s/%s.xml" % (self.location, date_scope)
if os.path.exists(xml_report_path):
return
if os.path.exists('%s.uploaded' % xml_report_path):
return
journal = Journal()
transaction = journal.newTransaction()
journal.setProperty(transaction, "title", "Eco Information for %s " % self.computer_id)
journal.setProperty(transaction, "start_date", "%s 00:00:00" % date_scope)
journal.setProperty(transaction, "stop_date", "%s 23:59:59" % date_scope)
journal.setProperty(transaction, "reference", "%s-global" % date_scope)
journal.setProperty(transaction, "currency", "")
journal.setProperty(transaction, "payment_mode", "")
journal.setProperty(transaction, "category", "")
arrow = ElementTree.SubElement(transaction, "arrow")
arrow.set("type", "Destination")
cpu_load_percent = self._getCpuLoadAverageConsumption(date_scope)
if cpu_load_percent is not None:
journal.newMovement(transaction,
resource="service_module/cpu_load_percent",
title="CPU Load Percent Average",
quantity=str(cpu_load_percent),
reference=self.computer_id,
category="")
memory_used = self._getMemoryAverageConsumption(date_scope)
if memory_used is not None:
journal.newMovement(transaction,
resource="service_module/memory_used",
title="Used Memory",
quantity=str(memory_used),
reference=self.computer_id,
category="")
if self._getZeroEmissionContribution() is not None:
journal.newMovement(transaction,
resource="service_module/zero_emission_ratio",
title="Zero Emission Ratio",
quantity=str(self._getZeroEmissionContribution()),
reference=self.computer_id,
category="")
core_amount = psutil.NUM_CPUS
for user in self.user_list:
partition_cpu_load_percent = self._getPartitionCPULoadAverage(user, date_scope)
if partition_cpu_load_percent is not None:
journal.newMovement(transaction,
resource="service_module/cpu_load_percent",
title="CPU Load Percent Average for %s" % (user),
quantity=str(partition_cpu_load_percent/core_amount),
reference=user,
category="")
mb = float(2 ** 20)
for user in self.user_list:
partition_memory_used = self._getPartitionUsedMemoryAverage(user, date_scope)
if partition_memory_used is not None:
journal.newMovement(transaction,
resource="service_module/memory_used",
title="Memory Used Average for %s" % (user),
quantity=str(partition_memory_used/mb),
reference=user,
category="")
with open(xml_report_path, 'w') as f:
f.write(journal.getXML())
f.close()
return xml_report_path
def _getAverageFromList(self, data_list):
return sum(data_list)/len(data_list)
def _getCpuLoadAverageConsumption(self, date_scope):
self.db.connect()
query_result_cursor = self.db.select("system", date_scope,
columns="SUM(cpu_percent)/COUNT(cpu_percent)")
cpu_load_percent_list = zip(*query_result_cursor)
self.db.close()
if len(cpu_load_percent_list):
return cpu_load_percent_list[0][0]
def _getMemoryAverageConsumption(self, date_scope):
self.db.connect()
query_result_cursor = self.db.select("system", date_scope,
columns="SUM(memory_used)/COUNT(memory_used)")
memory_used_list = zip(*query_result_cursor)
self.db.close()
if len(memory_used_list):
return memory_used_list[0][0]
def _getZeroEmissionContribution(self):
self.db.connect()
zer = self.db.getLastZeroEmissionRatio()
self.db.close()
return zer
def _getPartitionCPULoadAverage(self, partition_id, date_scope):
self.db.connect()
query_result_cursor = self.db.select("user", date_scope,
columns="SUM(cpu_percent)",
where="partition = '%s'" % partition_id)
cpu_percent_sum = zip(*query_result_cursor)
if len(cpu_percent_sum) and cpu_percent_sum[0][0] is None:
return
query_result_cursor = self.db.select("user", date_scope,
columns="COUNT(DISTINCT time)",
where="partition = '%s'" % partition_id)
sample_amount = zip(*query_result_cursor)
self.db.close()
if len(sample_amount) and len(cpu_percent_sum):
return cpu_percent_sum[0][0]/sample_amount[0][0]
def _getPartitionUsedMemoryAverage(self, partition_id, date_scope):
self.db.connect()
query_result_cursor = self.db.select("user", date_scope,
columns="SUM(memory_rss)",
where="partition = '%s'" % partition_id)
memory_sum = zip(*query_result_cursor)
if len(memory_sum) and memory_sum[0][0] is None:
return
query_result_cursor = self.db.select("user", date_scope,
columns="COUNT(DISTINCT time)",
where="partition = '%s'" % partition_id)
sample_amount = zip(*query_result_cursor)
self.db.close()
if len(sample_amount) and len(memory_sum):
return memory_sum[0][0]/sample_amount[0][0]
class Journal(object):
def __init__(self):
self.root = ElementTree.Element("journal")
def getXML(self):
report = ElementTree.tostring(self.root)
return "<?xml version='1.0' encoding='utf-8'?>%s" % report
def newTransaction(self, portal_type="Sale Packing List"):
transaction = ElementTree.SubElement(self.root, "transaction")
transaction.set("type", portal_type)
return transaction
def setProperty(self, element, name, value):
property_element = ElementTree.SubElement(element, name)
property_element.text = value
def newMovement(self, transaction, resource, title,
quantity, reference, category):
movement = ElementTree.SubElement(transaction, "movement")
self.setProperty(movement, "resource", resource)
self.setProperty(movement, "title", title)
self.setProperty(movement, "reference", reference)
self.setProperty(movement, "quantity", quantity)
self.setProperty(movement, "price", "0.0")
self.setProperty(movement, "VAT", "")
# Provide units
self.setProperty(movement, "category", category)
return movement
...@@ -29,6 +29,12 @@ ...@@ -29,6 +29,12 @@
import psutil import psutil
import os import os
from temperature import collectComputerTemperature, \
launchTemperatureTest
from temperature.heating import get_contribution_ratio
MEASURE_INTERVAL = 5
class _Snapshot(object): class _Snapshot(object):
def get(self, property, default=None): def get(self, property, default=None):
...@@ -61,20 +67,26 @@ class ProcessSnapshot(_Snapshot): ...@@ -61,20 +67,26 @@ class ProcessSnapshot(_Snapshot):
self.io_cycles_counter = ui_counter_list[0] + ui_counter_list[1] self.io_cycles_counter = ui_counter_list[0] + ui_counter_list[1]
def update_cpu_percent(self): def update_cpu_percent(self):
# CPU percentage, we will have to get actual absolute value if self.process_object.is_running():
self.cpu_percent = self.process_object.get_cpu_percent() # CPU percentage, we will have to get actual absolute value
self.cpu_percent = self.process_object.get_cpu_percent()
class SystemSnapshot(_Snapshot): class SystemSnapshot(_Snapshot):
""" Take a snapshot from current system usage """ Take a snapshot from current system usage
""" """
def __init__(self): def __init__(self, interval=MEASURE_INTERVAL):
cpu_idle_percentage = psutil.cpu_times_percent(interval=interval).idle
load_percent = 100 - cpu_idle_percentage
memory = psutil.phymem_usage() memory = psutil.phymem_usage()
net_io = psutil.net_io_counters() net_io = psutil.net_io_counters()
self.memory_used = memory.used self.memory_used = memory.used
self.memory_free = memory.free self.memory_free = memory.free
self.memory_percent = memory.percent self.memory_percent = memory.percent
self.cpu_percent = psutil.cpu_percent() #self.cpu_percent = psutil.cpu_percent()
self.cpu_percent = load_percent
self.load = os.getloadavg()[0] self.load = os.getloadavg()[0]
self.net_in_bytes = net_io.bytes_recv self.net_in_bytes = net_io.bytes_recv
self.net_in_errors = net_io.errin self.net_in_errors = net_io.errin
...@@ -83,6 +95,48 @@ class SystemSnapshot(_Snapshot): ...@@ -83,6 +95,48 @@ class SystemSnapshot(_Snapshot):
self.net_out_errors = net_io.errout self.net_out_errors = net_io.errout
self.net_out_dropped = net_io.dropout self.net_out_dropped = net_io.dropout
class TemperatureSnapshot(_Snapshot):
""" Take a snapshot from the current temperature on
all available sensors
"""
def __init__(self, sensor_id, temperature, alarm):
self.sensor_id = sensor_id
self.temperature = temperature
self.alarm = alarm
class HeatingContributionSnapshot(_Snapshot):
def __init__(self, sensor_id, model_id):
self.initial_temperature = None
result = launchTemperatureTest(sensor_id)
if result is None:
print "Impossible to test sensor: %s " % sensor_id
initial_temperature, final_temperature, duration = result
self.initial_temperature = initial_temperature
self.final_temperature = final_temperature
self.delta_time = duration
self.model_id = model_id
self.sensor_id = sensor_id
self.zero_emission_ratio = self._get_contribution_ratio()
def _get_contribution_ratio(self):
delta_temperature = (self.final_temperature-self.initial_temperature)
contribution_value = delta_temperature/self.delta_time
return get_contribution_ratio(self.model_id, contribution_value)
def _get_uptime(self):
# Linux only
if os.path.exists('/proc/uptime'):
with open('/proc/uptime', 'r') as f:
return float(f.readline().split()[0])
return -1
class DiskPartitionSnapshot(_Snapshot): class DiskPartitionSnapshot(_Snapshot):
""" Take Snapshot from general disk partitions """ Take Snapshot from general disk partitions
usage usage
...@@ -100,7 +154,7 @@ class DiskPartitionSnapshot(_Snapshot): ...@@ -100,7 +154,7 @@ class DiskPartitionSnapshot(_Snapshot):
class ComputerSnapshot(_Snapshot): class ComputerSnapshot(_Snapshot):
""" Take a snapshot from computer informations """ Take a snapshot from computer informations
""" """
def __init__(self): def __init__(self, model_id=None, sensor_id=None, test_heating=False):
self.cpu_num_core = psutil.NUM_CPUS self.cpu_num_core = psutil.NUM_CPUS
self.cpu_frequency = 0 self.cpu_frequency = 0
self.cpu_type = 0 self.cpu_type = 0
...@@ -112,9 +166,22 @@ class ComputerSnapshot(_Snapshot): ...@@ -112,9 +166,22 @@ class ComputerSnapshot(_Snapshot):
# on a Computer Snapshot # on a Computer Snapshot
# #
self.system_snapshot = SystemSnapshot() self.system_snapshot = SystemSnapshot()
self.temperature_snapshot_list = self._get_temperature_snapshot_list()
self.disk_snapshot_list = [] self.disk_snapshot_list = []
self.partition_list = self._get_physical_disk_info() self.partition_list = self._get_physical_disk_info()
if test_heating and model_id is not None \
and sensor_id is not None:
self.heating_contribution_snapshot = HeatingContributionSnapshot(sensor_id, model_id)
def _get_temperature_snapshot_list(self):
temperature_snapshot_list = []
for sensor_entry in collectComputerTemperature():
sensor_id, temperature, maximal, critical, alarm = sensor_entry
temperature_snapshot_list.append(
TemperatureSnapshot(sensor_id, temperature, alarm))
return temperature_snapshot_list
def _get_physical_disk_info(self): def _get_physical_disk_info(self):
partition_dict = {} partition_dict = {}
...@@ -127,4 +194,3 @@ class ComputerSnapshot(_Snapshot): ...@@ -127,4 +194,3 @@ class ComputerSnapshot(_Snapshot):
partition.mountpoint)) partition.mountpoint))
return [(k, v) for k, v in partition_dict.iteritems()] return [(k, v) for k, v in partition_dict.iteritems()]
from multiprocessing import Process, active_children, cpu_count, Pipe
import subprocess
import os
import signal
import sys
import time
FIB_N = 100
DEFAULT_TIME = 60
try:
DEFAULT_CPU = cpu_count()
except NotImplementedError:
DEFAULT_CPU = 1
def collectComputerTemperature(sensor_bin="sensors"):
cmd = ["%s -u" % sensor_bin]
sp = subprocess.Popen(cmd, stdout=subprocess.PIPE,
stderr=subprocess.PIPE, shell=True)
stdout, stderr = sp.communicate()
sensor_output_list = stdout.splitlines()
adapter_name = ""
sensor_temperature_list = []
for line_number in range(len(sensor_output_list)):
found_sensor = None
stripped_line = sensor_output_list[line_number].strip()
if stripped_line.startswith("Adapter:"):
adapter_name = sensor_output_list[line_number-1]
elif stripped_line.startswith("temp") and "_input" in stripped_line:
temperature = sensor_output_list[line_number].split()[-1]
found_sensor = ["%s %s" % (adapter_name, sensor_output_list[line_number-1]), float(temperature)]
if found_sensor is not None:
critical = '1000'
maximal = '1000'
for next_line in sensor_output_list[line_number+1:line_number+3]:
stripped_next_line = next_line.strip()
if stripped_next_line.startswith("temp") and "_max" in stripped_next_line:
maximal = stripped_next_line.split()[-1]
elif stripped_next_line.startswith("temp") and "_crit" in stripped_next_line:
critical = stripped_next_line.split()[-1]
found_sensor.extend([float(maximal), float(critical)])
found_sensor.append(checkAlarm(float(temperature), float(maximal), float(critical)))
sensor_temperature_list.append(found_sensor)
return sensor_temperature_list
def checkAlarm(temperature, maximal, critical):
"""
Returns :
O if the temperature is below the maximal limit.
1 if the temperature is above the maximal limit.
2 if the temperature is above the crical limit.
"""
alarm = 0
if temperature >= maximal:
alarm += 1
if temperature >= critical:
alarm += 1
return alarm
def loop(connection):
connection.send(os.getpid())
connection.close()
while True:
fib(FIB_N)
def fib(n):
if n < 2:
return 1
else:
return fib(n - 1) + fib(n - 2)
def sigint_handler(signum, frame):
procs = active_children()
for p in procs:
p.terminate()
os._exit(1)
def launchTemperatureTest(sensor_id, sensor_bin="sensors", timeout=600, interval=30):
signal.signal(signal.SIGINT, sigint_handler)
def getTemperatureForSensor(s_id):
for collected_temperature in collectComputerTemperature(sensor_bin):
if collected_temperature[0] == sensor_id:
return collected_temperature[1], collected_temperature[4]
return None, None
process_list = []
process_connection_list = []
begin_time = time.time()
initial_temperature, alarm = getTemperatureForSensor(sensor_id)
if initial_temperature is None:
return
if alarm > 0:
# Skip to test if temperature is too high, because we cannot
# measure appropriatetly.
return
candidate_temperature = initial_temperature
for i in range(DEFAULT_CPU):
parent_connection, child_connection = Pipe()
process = Process(target=loop, args=(child_connection,))
process.start()
process_list.append(process)
process_connection_list.append(parent_connection)
for connection in process_connection_list:
try:
print connection.recv()
except EOFError:
continue
time.sleep(interval)
current_temperature = getTemperatureForSensor(sensor_id)
while current_temperature > candidate_temperature:
candidate_temperature = current_temperature
time.sleep(interval)
current_temperature = getTemperatureForSensor(sensor_id)
for process in process_list:
process.terminate()
return initial_temperature, current_temperature, time.time() - begin_time
CONTRIBUTION_MAPPING = {
"shuttle_ds61_i7" : 0.045,
"nuc_i7": 0.055
}
def get_contribution_ratio(model_id, contribution):
zero_emission_ratio_limit = CONTRIBUTION_MAPPING.get(model_id)
if zero_emission_ratio_limit is None:
raise ValueError("Unknown heating contibution")
if contribution < zero_emission_ratio_limit:
# The machine don't contribute for heating
return 0
else:
# The machine contributes for the heating
return 100
...@@ -1059,6 +1059,34 @@ class Slapgrid(object): ...@@ -1059,6 +1059,34 @@ class Slapgrid(object):
self.logger.info('computer_partition_usage_list: %s - %s' % self.logger.info('computer_partition_usage_list: %s - %s' %
(computer_partition_usage.usage, computer_partition_usage.getId())) (computer_partition_usage.usage, computer_partition_usage.getId()))
filename_delete_list = []
computer_report_dir = os.path.join(self.instance_root,
'var', 'xml_report', self.computer_id)
# The directory xml_report contain a number of files equal
# to the number of software instance running inside the same partition
if os.path.isdir(computer_report_dir):
filename_list = os.listdir(computer_report_dir)
else:
filename_list = []
for filename in filename_list:
file_path = os.path.join(computer_report_dir, filename)
if os.path.exists(file_path):
usage = open(file_path, 'r').read()
if self.validateXML(usage, computer_consumption_model):
self.logger.info('XML file generated by asXML is valid')
slap_computer_usage.reportUsage(usage)
filename_delete_list.append(filename)
else:
self.logger.info('XML file is invalid %s' % filename)
# After sending the aggregated file we remove all the valid xml reports
for filename in filename_delete_list:
os.remove(os.path.join(computer_report_dir, filename))
# If there is, at least, one report # If there is, at least, one report
if computer_partition_usage_list != []: if computer_partition_usage_list != []:
try: try:
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment