Commit 7b2fc830 authored by Łukasz Nowak's avatar Łukasz Nowak

monitor: Cleanup stale history.json files

Additionally improve python3 support as it is required for proper
cleanup procedure.

/reviewed-on !74
parents c3eeed9d 687beaae
...@@ -100,10 +100,10 @@ def generateMonitoringData(config, public_folder, private_folder, public_url, ...@@ -100,10 +100,10 @@ def generateMonitoringData(config, public_folder, private_folder, public_url,
private_url, feed_url): private_url, feed_url):
feed_output = os.path.join(public_folder, 'feed') feed_output = os.path.join(public_folder, 'feed')
# search for all status files # search for all status files
file_list = filter( file_list = list(filter(
os.path.isfile, os.path.isfile,
glob.glob("%s/promise/*.status.json" % public_folder) glob.glob("%s/promise/*.status.json" % public_folder)
) ))
promises_status_file = os.path.join(private_folder, '_promise_status') promises_status_file = os.path.join(private_folder, '_promise_status')
previous_state_dict = {} previous_state_dict = {}
...@@ -123,6 +123,21 @@ def generateMonitoringData(config, public_folder, private_folder, public_url, ...@@ -123,6 +123,21 @@ def generateMonitoringData(config, public_folder, private_folder, public_url,
except ValueError: except ValueError:
pass pass
# clean up stale history files
expected_history_json_name_list = [
os.path.basename(q).replace('status.json', 'history.json') for q in file_list]
cleanup_history_json_path_list = []
for history_json_name in [q for q in os.listdir(public_folder) if q.endswith('history.json')]:
if history_json_name not in expected_history_json_name_list:
cleanup_history_json_path_list.append(os.path.join(public_folder, history_json_name))
for cleanup_path in cleanup_history_json_path_list:
try:
os.unlink(cleanup_path)
except Exception:
print('ERROR: Failed to remove stale %s' % (cleanup_path,))
else:
print('OK: Removed stale %s' % (cleanup_path,))
for file in file_list: for file in file_list:
try: try:
with open(file, 'r') as temp_file: with open(file, 'r') as temp_file:
......
...@@ -4,6 +4,10 @@ ...@@ -4,6 +4,10 @@
from __future__ import print_function from __future__ import print_function
import sys import sys
try:
import errno
except ImportError:
from os import errno
import os import os
import stat import stat
import json import json
...@@ -53,7 +57,7 @@ def mkdirAll(path): ...@@ -53,7 +57,7 @@ def mkdirAll(path):
try: try:
os.makedirs(path) os.makedirs(path)
except OSError as e: except OSError as e:
if e.errno == os.errno.EEXIST and os.path.isdir(path): if e.errno == errno.EEXIST and os.path.isdir(path):
pass pass
else: raise else: raise
...@@ -67,7 +71,7 @@ def createSymlink(source, destination): ...@@ -67,7 +71,7 @@ def createSymlink(source, destination):
try: try:
os.symlink(source, destination) os.symlink(source, destination)
except OSError as e: except OSError as e:
if e.errno != os.errno.EEXIST: if e.errno != errno.EEXIST:
raise raise
class Monitoring(object): class Monitoring(object):
...@@ -201,7 +205,7 @@ class Monitoring(object): ...@@ -201,7 +205,7 @@ class Monitoring(object):
mkdirAll(dirname) # could also raise OSError mkdirAll(dirname) # could also raise OSError
os.symlink(path, os.path.join(dirname, os.path.basename(path))) os.symlink(path, os.path.join(dirname, os.path.basename(path)))
except OSError as e: except OSError as e:
if e.errno != os.errno.EEXIST: if e.errno != errno.EEXIST:
raise raise
def getMonitorTitleFromUrl(self, monitor_url): def getMonitorTitleFromUrl(self, monitor_url):
......
...@@ -166,8 +166,22 @@ exit %(code)s ...@@ -166,8 +166,22 @@ exit %(code)s
self.assertTrue(os.path.exists(os.path.join(self.output_dir, 'promise_4.status.json'))) self.assertTrue(os.path.exists(os.path.join(self.output_dir, 'promise_4.status.json')))
os.symlink(self.output_dir, '%s/public/promise' % self.base_dir) os.symlink(self.output_dir, '%s/public/promise' % self.base_dir)
# create files for cleanup
must_stay_public = os.path.join(self.public_dir, 'must_stay')
must_unlink_public = os.path.join(self.public_dir, 'must_unlink.history.json')
for f in [must_stay_public, must_unlink_public]:
with open(f, 'w') as fh:
fh.write('data')
# generate instance state files # generate instance state files
document_list_file = os.path.join(self.public_dir, '_document_list')
with open(document_list_file, 'a+') as fh:
fh.write('must_unlink.history')
globalstate.run(self.monitor_config_file) globalstate.run(self.monitor_config_file)
with open(document_list_file) as fh:
self.assertNotIn('must_unlink.history', fh.read())
self.assertTrue(os.path.exists(must_stay_public))
self.assertFalse(os.path.exists(must_unlink_public))
self.assertTrue(os.path.exists(os.path.join(self.public_dir, 'feed'))) self.assertTrue(os.path.exists(os.path.join(self.public_dir, 'feed')))
self.assertTrue(os.path.exists(os.path.join(self.public_dir, 'monitor.global.json'))) self.assertTrue(os.path.exists(os.path.join(self.public_dir, 'monitor.global.json')))
self.assertTrue(os.path.exists(os.path.join(self.private_dir, 'monitor.global.json'))) self.assertTrue(os.path.exists(os.path.join(self.private_dir, 'monitor.global.json')))
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment