...
 
Commits (3)
......@@ -89,8 +89,10 @@ def do_collect(conf):
user_dict[snapshot.username].append(snapshot)
except (KeyboardInterrupt, SystemExit, NoSuchProcess):
raise
days_to_preserve = conf.getint("slapos", "collect_cache", 15)
days_to_preserve = 15
if conf.has_option("slapos", "collect_cache"):
days_to_preserve = conf.getint("slapos", "collect_cache")
log_directory = "%s/var/data-log" % conf.get("slapos", "instance_root")
mkdir_p(log_directory, 0o755)
......@@ -149,7 +151,7 @@ def do_collect(conf):
base = datetime.datetime.utcnow().date()
for x in range(1, 3):
report_file = consumption_report.buildXMLReport(
(base - datetime.timedelta(days=x)).strftime("%Y-%m-%d"), gmtime())
(base - datetime.timedelta(days=x)).strftime("%Y-%m-%d"))
if report_file is not None:
shutil.copy(report_file, xml_report_directory)
......@@ -162,7 +164,7 @@ def do_collect(conf):
compressLogFolder(log_directory)
# Drop older entries already reported
database.garbageCollect(int(days_to_preserve))
database.garbageCollect(days_to_preserve)
except AccessDenied:
print("You HAVE TO execute this script with root permission.")
......
......@@ -38,15 +38,18 @@ import importlib
import traceback
import psutil
import inspect
import hashlib
from datetime import datetime
from multiprocessing import Process, Queue as MQueue
from six.moves import queue, reload_module
from slapos.util import mkdir_p, chownDirectory
from slapos.util import str2bytes, mkdir_p, chownDirectory
from slapos.grid.utils import dropPrivileges, killProcessTree
from slapos.grid.promise import interface
from slapos.grid.promise.generic import (GenericPromise, PromiseQueueResult,
AnomalyResult, TestResult,
PROMISE_STATE_FOLDER_NAME,
PROMISE_RESULT_FOLDER_NAME,
PROMISE_HISTORY_RESULT_FOLDER_NAME,
PROMISE_PARAMETER_NAME)
from slapos.grid.promise.wrapper import WrapPromise
from slapos.version import version
......@@ -342,6 +345,14 @@ class PromiseLauncher(object):
mkdir_p(self.promise_output_dir)
self._updateFolderOwner()
self.promise_history_output_dir = os.path.join(
self.partition_folder,
PROMISE_HISTORY_RESULT_FOLDER_NAME
)
if not os.path.exists(self.promise_history_output_dir):
mkdir_p(self.promise_history_output_dir)
self._updateFolderOwner()
def _generatePromiseResult(self, promise_process, promise_name, promise_path,
message, execution_time=0):
if self.check_anomaly:
......@@ -378,6 +389,81 @@ class PromiseLauncher(object):
json.dump(result.serialize(), outputfile)
os.rename(promise_tmp_file, promise_output_file)
def _savePromiseHistoryResult(self, result):
state_dict = result.serialize()
previous_state_dict = {}
promise_status_file = os.path.join(PROMISE_STATE_FOLDER_NAME,
'promise_status.json')
if os.path.exists(promise_status_file):
with open(promise_status_file) as f:
try:
previous_state_dict = json.load(f)
except ValueError:
pass
history_file = os.path.join(
self.promise_history_output_dir,
'%s.history.json' % result.title
)
# Remove useless informations
result_dict = state_dict.pop('result')
result_dict["change-date"] = result_dict["date"]
state_dict.update(result_dict)
state_dict.pop('path', '')
state_dict.pop('type', '')
state_dict["status"] = "ERROR" if result.item.hasFailed() else "OK"
if not os.path.exists(history_file) or not os.stat(history_file).st_size:
with open(history_file, 'w') as f:
data_dict = {
"date": time.time(),
"data": [state_dict]
}
json.dump(data_dict, f)
else:
previous_state_list = previous_state_dict.get(result.name, None)
if previous_state_list is not None:
_, change_date, checksum = previous_state_list
current_sum = hashlib.md5(str2bytes(state_dict.get('message', ''))).hexdigest()
if state_dict['change-date'] == change_date and \
current_sum == checksum:
# Only save the changes and not the same info
return
state_dict.pop('title', '')
state_dict.pop('name', '')
with open (history_file, mode="r+") as f:
f.seek(0,2)
f.seek(f.tell() -2)
f.write('%s}' % ',{}]'.format(json.dumps(state_dict)))
def _saveStatisticsData(self, stat_file_path, date, success, error):
# csv-like document for success/error statictics
if not os.path.exists(stat_file_path) or os.stat(stat_file_path).st_size == 0:
with open(stat_file_path, 'w') as fstat:
data_dict = {
"date": time.time(),
"data": ["Date, Success, Error, Warning"]
}
fstat.write(json.dumps(data_dict))
current_state = '%s, %s, %s, %s' % (
date,
success,
error,
'')
# append to file
# XXX this is bad, it is getting everywhere.
if current_state:
with open (stat_file_path, mode="r+") as fstat:
fstat.seek(0,2)
position = fstat.tell() -2
fstat.seek(position)
fstat.write('%s}' % ',"{}"]'.format(current_state))
def _loadPromiseResult(self, promise_title):
promise_output_file = os.path.join(
self.promise_output_dir,
......@@ -409,6 +495,7 @@ class PromiseLauncher(object):
self.bang_called = True
# Send result
self._savePromiseResult(result_item)
self._savePromiseHistoryResult(result_item)
def _emptyQueue(self):
"""Remove all entries from queue until it's empty"""
......@@ -565,6 +652,29 @@ class PromiseLauncher(object):
promise_list = []
failed_promise_name = ""
failed_promise_output = ""
previous_state_dict = {}
new_state_dict = {}
report_date = datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%S+0000')
promise_status_file = os.path.join(self.partition_folder,
PROMISE_STATE_FOLDER_NAME,
'promise_status.json')
promise_result_file = os.path.join(self.partition_folder,
PROMISE_STATE_FOLDER_NAME,
'promise_result.json')
promise_stats_file = os.path.join(self.partition_folder,
PROMISE_STATE_FOLDER_NAME,
'promise_stats.json')
if os.path.exists(promise_status_file):
with open(promise_status_file) as f:
try:
previous_state_dict = json.load(f)
except ValueError:
pass
new_state_dict = previous_state_dict.copy()
base_config = {
'log-folder': self.log_folder,
'partition-folder': self.partition_folder,
......@@ -578,7 +688,8 @@ class PromiseLauncher(object):
'queue': self.queue_result,
'slapgrid-version': version,
}
error = 0
success = 0
if os.path.exists(self.promise_folder) and os.path.isdir(self.promise_folder):
for promise_name in os.listdir(self.promise_folder):
if promise_name.startswith('__init__') or \
......@@ -597,9 +708,36 @@ class PromiseLauncher(object):
config.update(base_config)
promise_result = self._launchPromise(promise_name, promise_path, config)
if promise_result and promise_result.hasFailed() and not failed_promise_name:
failed_promise_name = promise_name
failed_promise_output = promise_result.message
if promise_result:
change_date = promise_result.date.strftime('%Y-%m-%dT%H:%M:%S+0000')
if promise_result.hasFailed():
promise_status = 'FAILED'
error += 1
else:
promise_status = "OK"
success += 1
if promise_name in previous_state_dict:
status, previous_change_date, _ = previous_state_dict[promise_name]
if promise_status == status:
change_date = previous_change_date
message = promise_result.message if promise_result.message else ""
new_state_dict[promise_name] = [
promise_status,
change_date,
hashlib.md5(str2bytes(message)).hexdigest()]
if promise_result.hasFailed() and not failed_promise_name:
failed_promise_name = promise_name
failed_promise_output = promise_result.message
else:
# The promise was skip, so for statistic point of view we preserve
# the previous result
if promise_name in new_state_dict:
if new_state_dict[promise_name][0] == "FAILED":
error += 1
else:
success += 1
if not self.run_only_promise_list and os.path.exists(self.legacy_promise_folder) \
and os.path.isdir(self.legacy_promise_folder):
......@@ -621,11 +759,46 @@ class PromiseLauncher(object):
promise_path,
config,
wrap_process=True)
if promise_result and promise_result.hasFailed() and not failed_promise_name:
failed_promise_name = promise_name
failed_promise_output = promise_result.message
if promise_result:
change_date = promise_result.date.strftime('%Y-%m-%dT%H:%M:%S+0000')
if promise_result.hasFailed():
promise_status = 'FAILED'
error += 1
else:
promise_status = "OK"
success += 1
if promise_name in previous_state_dict:
status, previous_change_date, _ = previous_state_dict[promise_name]
if promise_status == status:
change_date = previous_change_date
message = promise_result.message if promise_result.message else ""
new_state_dict[promise_name] = [
promise_status,
change_date,
hashlib.md5(str2bytes(message)).hexdigest()]
if promise_result.hasFailed() and not failed_promise_name:
failed_promise_name = promise_name
failed_promise_output = promise_result.message
else:
# The promise was skip, so for statistic point of view we preserve
# the previous result
if promise_name in new_state_dict:
if new_state_dict[promise_name][0] == "FAILED":
error += 1
else:
success += 1
self._updateFolderOwner(self.promise_output_dir)
# Save Global State
with open(promise_status_file, "w") as f:
json.dump(new_state_dict, f)
self._saveStatisticsData(promise_stats_file,
report_date, success, error)
if self._skipped_amount > 0:
self.logger.info("%s promises didn't need to be checked." % \
self._skipped_amount)
......@@ -633,3 +806,4 @@ class PromiseLauncher(object):
raise PromiseError("Promise %r failed with output: %s" % (
failed_promise_name,
failed_promise_output))
......@@ -44,7 +44,7 @@ from datetime import datetime, timedelta
PROMISE_STATE_FOLDER_NAME = '.slapgrid/promise'
PROMISE_RESULT_FOLDER_NAME = '.slapgrid/promise/result'
PROMISE_LOG_FOLDER_NAME = '.slapgrid/promise/log'
PROMISE_HISTORY_RESULT_FOLDER_NAME = '.slapgrid/promise/history'
PROMISE_PARAMETER_NAME = 'extra_config_dict'
LOGLINE_RE = r"(\d{4}-\d{2}-\d{2}\s+\d{2}:\d{2}:\d{2})\s+\-?\s*(\w+)\s+\-?\s+(\d+\-\d{3})\s+\-?\s*(.*)"
......
......@@ -374,6 +374,7 @@ class SlapOSInstanceTestCase(unittest.TestCase):
except BaseException:
cls.logger.exception("Error during setUpClass")
cls._storeSnapshot("{}.setUpClass".format(cls.__name__))
cls._cleanup()
cls.setUp = lambda self: self.fail('Setup Class failed.')
raise
......@@ -384,25 +385,30 @@ class SlapOSInstanceTestCase(unittest.TestCase):
"""
cls._cleanup()
def tearDown(self):
@classmethod
def _storeSnapshot(cls, name):
# copy log files from standalone
for standalone_log in glob.glob(os.path.join(
self._base_directory, 'var', 'log', '*')):
self._snapshot_instance_file(standalone_log)
cls._base_directory, 'var', 'log', '*')):
cls._snapshot_instance_file(standalone_log, name)
# copy config and log files from partitions
for pattern in self._save_instance_file_pattern_list:
for f in glob.glob(os.path.join(self.slap.instance_directory, pattern)):
self._snapshot_instance_file(f)
for pattern in cls._save_instance_file_pattern_list:
for f in glob.glob(os.path.join(cls.slap.instance_directory, pattern)):
cls._snapshot_instance_file(f, name)
def _snapshot_instance_file(self, source_file_name):
def tearDown(self):
self._storeSnapshot(self.id())
@classmethod
def _snapshot_instance_file(cls, source_file_name, name):
"""Save a file for later inspection.
The path are made relative to slapos root directory and
we keep the same directory structure.
"""
# we cannot use os.path.commonpath on python2, so implement something similar
common_path = os.path.commonprefix((source_file_name, self._base_directory))
common_path = os.path.commonprefix((source_file_name, cls._base_directory))
if not os.path.isdir(common_path):
common_path = os.path.dirname(common_path)
......@@ -410,14 +416,14 @@ class SlapOSInstanceTestCase(unittest.TestCase):
if relative_path[0] == os.sep:
relative_path = relative_path[1:]
destination = os.path.join(
self._test_file_snapshot_directory,
self.software_id,
self.id(),
cls._test_file_snapshot_directory,
cls.software_id,
name,
relative_path)
destination_dirname = os.path.dirname(destination)
mkdir_p(destination_dirname)
if os.path.isfile(source_file_name):
self.logger.debug("copy %s as %s", source_file_name, destination)
cls.logger.debug("copy %s as %s", source_file_name, destination)
shutil.copy(source_file_name, destination)
# implementation methods
......