...
 
Commits (3)
......@@ -89,8 +89,10 @@ def do_collect(conf):
user_dict[snapshot.username].append(snapshot)
except (KeyboardInterrupt, SystemExit, NoSuchProcess):
raise
days_to_preserve = conf.getint("slapos", "collect_cache", 15)
days_to_preserve = 15
if conf.has_option("slapos", "collect_cache"):
days_to_preserve = conf.getint("slapos", "collect_cache")
log_directory = "%s/var/data-log" % conf.get("slapos", "instance_root")
mkdir_p(log_directory, 0o755)
......@@ -149,7 +151,7 @@ def do_collect(conf):
base = datetime.datetime.utcnow().date()
for x in range(1, 3):
report_file = consumption_report.buildXMLReport(
(base - datetime.timedelta(days=x)).strftime("%Y-%m-%d"), gmtime())
(base - datetime.timedelta(days=x)).strftime("%Y-%m-%d"))
if report_file is not None:
shutil.copy(report_file, xml_report_directory)
......@@ -162,7 +164,7 @@ def do_collect(conf):
compressLogFolder(log_directory)
# Drop older entries already reported
database.garbageCollect(int(days_to_preserve))
database.garbageCollect(days_to_preserve)
except AccessDenied:
print("You HAVE TO execute this script with root permission.")
......
......@@ -38,15 +38,17 @@ import importlib
import traceback
import psutil
import inspect
import hashlib
from multiprocessing import Process, Queue as MQueue
from six.moves import queue, reload_module
from slapos.util import mkdir_p, chownDirectory
from slapos.util import str2bytes, mkdir_p, chownDirectory
from slapos.grid.utils import dropPrivileges, killProcessTree
from slapos.grid.promise import interface
from slapos.grid.promise.generic import (GenericPromise, PromiseQueueResult,
AnomalyResult, TestResult,
PROMISE_STATE_FOLDER_NAME,
PROMISE_RESULT_FOLDER_NAME,
PROMISE_HISTORY_RESULT_FOLDER_NAME,
PROMISE_PARAMETER_NAME)
from slapos.grid.promise.wrapper import WrapPromise
from slapos.version import version
......@@ -342,6 +344,14 @@ class PromiseLauncher(object):
mkdir_p(self.promise_output_dir)
self._updateFolderOwner()
self.promise_history_output_dir = os.path.join(
self.partition_folder,
PROMISE_HISTORY_RESULT_FOLDER_NAME
)
if not os.path.exists(self.promise_history_output_dir):
mkdir_p(self.promise_history_output_dir)
self._updateFolderOwner()
def _generatePromiseResult(self, promise_process, promise_name, promise_path,
message, execution_time=0):
if self.check_anomaly:
......@@ -378,6 +388,61 @@ class PromiseLauncher(object):
json.dump(result.serialize(), outputfile)
os.rename(promise_tmp_file, promise_output_file)
def _savePromiseHistoryResult(self, result):
#promise_name, state_dict, previous_state_list,
#history_folder):
state_dict = result.serialize()
promise_status_file = os.path.join(self.partition_folder,
'.slapgrid/promise/global.status')
previous_state_dict = {}
if os.path.exists(promise_status_file):
with open(promise_status_file) as f:
try:
previous_state_dict = json.load(f)
except ValueError:
pass
history_file = os.path.join(
self.promise_history_output_dir,
'%s.history.json' % result.name
)
# Remove useless informations
result_dict = state_dict.pop('result')
result_dict["change-date"] = result_dict["date"]
state_dict.update(result_dict)
state_dict.pop('path', '')
state_dict.pop('type', '')
state_dict["status"] = "ERROR" if result.item.hasFailed() else "OK"
if not os.path.exists(history_file) or not os.stat(history_file).st_size:
with open(history_file, 'w') as f:
data_dict = {
"date": time.time(),
"data": [state_dict]
}
json.dump(data_dict, f)
else:
previous_state_list = previous_state_dict.get(result.name, None)
if previous_state_list is not None:
_, change_date, checksum = previous_state_list
current_sum = hashlib.md5(str2bytes(state_dict.get('message', ''))).hexdigest()
if state_dict['change-date'] == change_date and \
current_sum == checksum:
# Only save the changes and not the same info
return
state_dict.pop('title', '')
state_dict.pop('name', '')
with open (history_file, mode="r+") as f:
f.seek(0,2)
f.seek(f.tell() -2)
f.write('%s}' % ',{}]'.format(json.dumps(state_dict)))
def _loadPromiseResult(self, promise_title):
promise_output_file = os.path.join(
self.promise_output_dir,
......@@ -409,6 +474,7 @@ class PromiseLauncher(object):
self.bang_called = True
# Send result
self._savePromiseResult(result_item)
self._savePromiseHistoryResult(result_item)
def _emptyQueue(self):
"""Remove all entries from queue until it's empty"""
......@@ -565,6 +631,19 @@ class PromiseLauncher(object):
promise_list = []
failed_promise_name = ""
failed_promise_output = ""
previous_state_dict = {}
new_state_dict = {}
promise_status_file = os.path.join(self.partition_folder,
'.slapgrid/promise/global.status')
if os.path.exists(promise_status_file):
with open(promise_status_file) as f:
try:
previous_state_dict = json.load(f)
except ValueError:
pass
new_state_dict = previous_state_dict.copy()
base_config = {
'log-folder': self.log_folder,
'partition-folder': self.partition_folder,
......@@ -597,9 +676,23 @@ class PromiseLauncher(object):
config.update(base_config)
promise_result = self._launchPromise(promise_name, promise_path, config)
if promise_result and promise_result.hasFailed() and not failed_promise_name:
failed_promise_name = promise_name
failed_promise_output = promise_result.message
if promise_result:
change_date = promise_result.date.strftime('%Y-%m-%dT%H:%M:%S+0000')
promise_status = 'FAILED' if promise_result.hasFailed() else "OK"
if promise_name in previous_state_dict:
status, previous_change_date, _ = previous_state_dict[promise_name]
if promise_status == status:
change_date = previous_change_date
message = promise_result.message if promise_result.message else ""
new_state_dict[promise_name] = [
promise_status,
change_date,
hashlib.md5(str2bytes(message)).hexdigest()]
if promise_result.hasFailed() and not failed_promise_name:
failed_promise_name = promise_name
failed_promise_output = promise_result.message
if not self.run_only_promise_list and os.path.exists(self.legacy_promise_folder) \
and os.path.isdir(self.legacy_promise_folder):
......@@ -621,9 +714,23 @@ class PromiseLauncher(object):
promise_path,
config,
wrap_process=True)
if promise_result and promise_result.hasFailed() and not failed_promise_name:
failed_promise_name = promise_name
failed_promise_output = promise_result.message
if promise_result:
change_date = promise_result.date.strftime('%Y-%m-%dT%H:%M:%S+0000')
promise_status = 'FAILED' if promise_result.hasFailed() else "OK"
if promise_name in previous_state_dict:
status, previous_change_date, _ = previous_state_dict[promise_name]
if promise_status == status:
change_date = previous_change_date
message = promise_result.message if promise_result.message else ""
new_state_dict[promise_name] = [
promise_status,
change_date,
hashlib.md5(str2bytes(message)).hexdigest()]
if promise_result.hasFailed() and not failed_promise_name:
failed_promise_name = promise_name
failed_promise_output = promise_result.message
self._updateFolderOwner(self.promise_output_dir)
if self._skipped_amount > 0:
......@@ -633,3 +740,7 @@ class PromiseLauncher(object):
raise PromiseError("Promise %r failed with output: %s" % (
failed_promise_name,
failed_promise_output))
with open(promise_status_file, "w") as f:
json.dump(new_state_dict, f)
......@@ -44,7 +44,7 @@ from datetime import datetime, timedelta
PROMISE_STATE_FOLDER_NAME = '.slapgrid/promise'
PROMISE_RESULT_FOLDER_NAME = '.slapgrid/promise/result'
PROMISE_LOG_FOLDER_NAME = '.slapgrid/promise/log'
PROMISE_HISTORY_RESULT_FOLDER_NAME = '.slapgrid/promise/history'
PROMISE_PARAMETER_NAME = 'extra_config_dict'
LOGLINE_RE = r"(\d{4}-\d{2}-\d{2}\s+\d{2}:\d{2}:\d{2})\s+\-?\s*(\w+)\s+\-?\s+(\d+\-\d{3})\s+\-?\s*(.*)"
......
......@@ -244,9 +244,6 @@ def setComputerPartitionConnectionXml():
requested_by = execute_db('partition',
'SELECT requested_by FROM %s WHERE reference=? AND computer_reference=?',
(computer_partition_id, computer_id), one=True)['requested_by']
execute_db('partition',
"UPDATE %s SET timestamp=? WHERE reference=? AND computer_reference=?",
(time.time(), requested_by, computer_id))
else:
query = 'UPDATE %s SET connection_xml=? , hosted_by=? WHERE reference=?'
argument_list = [connection_xml, computer_partition_id, slave_reference]
......