Commit 95ac5087 authored by Alain Takoudjou's avatar Alain Takoudjou

monitor stack: reduce amount of history files

parent ca60ce6f
......@@ -104,6 +104,7 @@ def main(args_list):
'success': success,
'warning': warning,
},
type='global',
date=latest_date,
_links={"rss_url": {"href": "%s/public/feed" % base_url},
"public_url": {"href": "%s/share/jio_public/" % base_url},
......
......@@ -97,7 +97,6 @@ class Monitoring(object):
self.report_script_folder = config.get("monitor", "report-folder")
self.webdav_url = '%s/share' % config.get("monitor", "base-url")
self.public_url = '%s/public' % config.get("monitor", "base-url")
self.status_history_folder = os.path.join(self.public_folder, 'history')
self.python = config.get("monitor", "python") or "python"
self.public_path_list = config.get("monitor", "public-path-list").split()
self.private_path_list = config.get("monitor", "private-path-list").split()
......@@ -107,7 +106,7 @@ class Monitoring(object):
self.parameter_cfg_file = config.get("monitor", "parameter-file-path").strip()
self.config_folder = os.path.join(self.private_folder, 'config')
self.report_folder = os.path.join(self.private_folder, 'report')
self.report_folder = self.private_folder
self.promise_dict = {}
for promise_folder in self.promise_folder_list:
......@@ -237,14 +236,14 @@ class Monitoring(object):
else:
response = urllib2.urlopen(url)
except urllib2.HTTPError:
return 'Unknow Instance'
return 'Unknown Instance'
else:
try:
monitor_dict = json.loads(response.read())
return monitor_dict.get('title', 'Unknow Instance')
return monitor_dict.get('title', 'Unknown Instance')
except ValueError, e:
print "Bad Json file at %s" % url
return 'Unknow Instance'
return 'Unknown Instance'
def configureFolders(self):
# configure public and private folder
......@@ -256,7 +255,6 @@ class Monitoring(object):
jio_private = os.path.join(self.webdav_folder, 'jio_private')
mkdirAll(jio_public)
mkdirAll(jio_private)
mkdirAll(self.status_history_folder)
try:
os.symlink(self.public_folder, os.path.join(jio_public, '.jio_documents'))
except OSError, e:
......@@ -269,11 +267,9 @@ class Monitoring(object):
raise
self.data_folder = os.path.join(self.private_folder, 'data', '.jio_documents')
self.report_folder = os.path.join(self.report_folder, '.jio_documents')
config_folder = os.path.join(self.config_folder, '.jio_documents')
mkdirAll(self.data_folder)
mkdirAll(config_folder)
mkdirAll(self.report_folder)
try:
os.symlink(os.path.join(self.private_folder, 'data'),
os.path.join(jio_private, 'data'))
......@@ -285,11 +281,6 @@ class Monitoring(object):
except OSError, e:
if e.errno != os.errno.EEXIST:
raise
try:
os.symlink(self.report_folder, os.path.join(jio_private, 'report'))
except OSError, e:
if e.errno != os.errno.EEXIST:
raise
def makeConfigurationFiles(self):
config_folder = os.path.join(self.config_folder, '.jio_documents')
......@@ -397,7 +388,7 @@ class Monitoring(object):
def generateReportCronEntries(self):
cron_line_list = []
# We should add the possibility to modify this parameter later from monitor interface
report_frequency = "*/30 * * * *"
report_frequency = "*/20 * * * *"
report_name_list = [name.replace('.report.json', '')
for name in os.listdir(self.report_folder) if name.endswith('.report.json')]
......@@ -451,7 +442,7 @@ class Monitoring(object):
for service_name, promise in self.promise_items:
service_config = promise["configuration"]
service_status_path = "%s/%s.status.json" % (self.public_folder, service_name) # hardcoded
service_status_path = "%s/%s.status.json" % (self.public_folder, service_name)
mkdirAll(os.path.dirname(service_status_path))
promise_cmd_line = [
......@@ -463,7 +454,7 @@ class Monitoring(object):
'--promise_script "%s"' % promise["path"],
'--promise_name "%s"' % service_name,
'--monitor_url "%s/jio_private/"' % self.webdav_url, # XXX hardcoded,
'--history_folder "%s"' % self.status_history_folder,
'--history_folder "%s"' % self.public_folder,
'--instance_name "%s"' % self.title,
'--hosting_name "%s"' % self.root_title]
......@@ -539,13 +530,21 @@ class Monitoring(object):
# Rotate monitor data files
option_list = [
'daily', 'nocreate', 'noolddir', 'rotate 30',
'daily', 'nocreate', 'noolddir', 'rotate 5',
'nocompress', 'extension .json', 'dateext',
'dateformat -%Y-%m-%d', 'notifempty'
]
file_list = ["%s/*.data.json" % self.data_folder]
self.generateLogrotateEntry('monitor.data', file_list, option_list)
# Rotate public history status file, delete data of previous days
option_list = [
'daily', 'nocreate', 'rotate 0',
'nocompress', 'notifempty'
]
file_list = ["%s/*.history.json" % self.public_folder]
self.generateLogrotateEntry('monitor.service.status', file_list, option_list)
# Add cron entry for SlapOS Collect
command = "%s %s --output_folder %s --collector_db %s" % (self.python,
self.collect_script, self.data_folder, self.collector_db)
......
......@@ -65,6 +65,7 @@ def main():
status_json['title'] = parser.promise_name
status_json['instance'] = parser.instance_name
status_json['hosting_subscription'] = parser.hosting_name
status_json['type'] = parser.promise_type
# Save the lastest status change date (needed for rss)
status_json['change-time'] = ps_process.create_time()
......@@ -85,9 +86,7 @@ def main():
os.remove(parser.pid_path)
def updateStatusHistoryFolder(name, status_file, history_folder, promise_type):
old_history_list = []
keep_item_amount = 25
history_path = os.path.join(history_folder, name, '.jio_documents')
history_path = os.path.join(history_folder)
if not os.path.exists(status_file):
return
if not os.path.exists(history_folder):
......@@ -101,14 +100,35 @@ def updateStatusHistoryFolder(name, status_file, history_folder, promise_type):
else: raise
with open(status_file, 'r') as sf:
status_dict = json.loads(sf.read())
filename = '%s.%s.json' % (
status_dict['start-date'].replace(' ', '_').replace(':', ''),
promise_type)
if promise_type == 'status':
filename = '%s.history.json' % name
history_file = os.path.join(history_path, filename)
# Remove links from history (not needed)
status_dict.pop('_links', None)
if not os.path.exists(history_file):
with open(history_file, 'w') as f_history:
data_dict = {
"date": time.time(),
"data": [status_dict]
}
f_history.write(json.dumps(data_dict))
else:
with open (history_file, mode="r+") as f_history:
f_history.seek(0,2)
position = f_history.tell() -2
f_history.seek(position)
#f_history.write(',%s]}' % str(status_dict))
f_history.write('%s}' % ',{}]'.format(json.dumps(status_dict)))
elif promise_type == 'report':
# keep_item_amount = 3
filename = '%s.history.json' % (
name)
copyfile(status_file, os.path.join(history_path, filename))
# Don't let history foler grow too much, keep xx files
"""# Don't let history foler grow too much, keep xx files
file_list = filter(os.path.isfile,
glob.glob("%s/*.%s.json" % (history_path, promise_type))
glob.glob("%s/*.%s.history.json" % (history_path, promise_type))
)
file_count = len(file_list)
if file_count > keep_item_amount:
......@@ -119,7 +139,7 @@ def updateStatusHistoryFolder(name, status_file, history_folder, promise_type):
os.unlink(to_delete)
file_count -= 1
except OSError:
raise
raise"""
def generateStatusJsonFromProcess(process, start_date=None, title=None):
stdout, stderr = process.communicate()
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment