Commit 4b861ea0 authored by Alain Takoudjou's avatar Alain Takoudjou

add tests for cache

parent fc092505
......@@ -95,7 +95,6 @@ class PromiseProcess(Process):
self._periodicity = None
self.cache_folder = os.path.join(self.partition_folder,
PROMISE_CACHE_FOLDER_NAME)
mkdir_p(self.cache_folder)
self.cache_file = os.path.join(self.cache_folder, self.getPromiseTitle())
# XXX - remove old files used to store promise timestamp and periodicity
self._cleanupDeprecated()
......@@ -112,12 +111,6 @@ class PromiseProcess(Process):
if os.path.exists(periodicity_file) and os.path.isfile(periodicity_file):
os.unlink(periodicity_file)
def getNextPromiseTime(self, periodicity):
"""
Return the next promise execution timestamp from now
"""
return time.time() + (periodicity * 60.0)
def getPromiseTitle(self):
return os.path.splitext(self.name)[0]
......@@ -171,6 +164,7 @@ class PromiseProcess(Process):
promise_started = False
if self.uid and self.gid:
dropPrivileges(self.uid, self.gid, logger=self.logger)
mkdir_p(self.cache_folder)
if self.wrap_promise:
promise_instance = WrapPromise(self.argument_dict)
else:
......@@ -338,6 +332,7 @@ class PromiseLauncher(object):
self.queue_result = MQueue()
self.bang_called = False
self._skipped_amount = 0
self.promise_output_dir = os.path.join(
self.partition_folder,
......@@ -467,10 +462,12 @@ class PromiseLauncher(object):
or not self.check_anomaly and not promise_cache_dict.get('is_tested'):
# promise is skipped, send empty result
self._writePromiseResult(PromiseQueueResult())
self._skipped_amount += 1
return
if not self.force and (promise_cache_dict is not None and not
self.isPeriodicityMatch(promise_cache_dict.get('next_run_after'))):
# we won't start the promise process, just get the latest result
self._skipped_amount += 1
result = self._loadPromiseResult(promise_process.getPromiseTitle())
if result is not None:
if result.item.hasFailed():
......@@ -627,6 +624,8 @@ class PromiseLauncher(object):
failed_promise_name = promise_name
self._updateFolderOwner(self.promise_output_dir)
if self._skipped_amount > 0:
self.logger.info("%s promises didn't need to be checked." % \
self._skipped_amount)
if failed_promise_name:
raise PromiseError("Promise %r failed." % failed_promise_name)
......@@ -36,7 +36,8 @@ import logging
from datetime import datetime, timedelta
import six
from six.moves import queue
from slapos.grid.promise import interface, PromiseLauncher, PromiseProcess, PromiseError
from slapos.grid.promise import (interface, PromiseLauncher, PromiseProcess,
PromiseError, PROMISE_CACHE_FOLDER_NAME)
from slapos.grid.promise.generic import (GenericPromise, TestResult, AnomalyResult,
PromiseQueueResult, PROMISE_STATE_FOLDER_NAME,
PROMISE_RESULT_FOLDER_NAME,
......@@ -1213,6 +1214,147 @@ exit 1
# no result returned by the promise
self.assertTrue(self.called)
def test_promise_cache(self):
promise_name = 'my_promise.py'
promise_file = os.path.join(self.plugin_dir, promise_name)
self.configureLauncher(timeout=1, enable_anomaly=True)
self.generatePromiseScript(promise_name, success=True, periodicity=0.01,
with_anomaly=True, is_tested=False)
# run promise, no failure
self.launcher.run()
cache_folder = os.path.join(self.partition_dir, PROMISE_CACHE_FOLDER_NAME)
cache_file = os.path.join(cache_folder, 'my_promise')
self.assertTrue(os.path.exists(cache_folder))
self.assertTrue(os.path.exists(cache_file))
file_stat = os.stat(promise_file)
with open(cache_file) as f:
cache_dict = json.load(f)
timestamp = cache_dict.pop('timestamp')
info_dict = {
u'is_tested': False,
u'is_anomaly_detected': True,
u'periodicity': 0.01,
u'next_run_after' : (timestamp + 0.01 * 60.0),
u'module_file': u'%s' % promise_file,
u'module_file_mtime': file_stat.st_mtime,
}
# next run is in future
self.assertTrue(info_dict['next_run_after'] > time.time())
self.assertEqual(info_dict, cache_dict)
def test_promise_cache_expire_with_periodicity(self):
self.called = False
def test_method(result):
self.called = True
promise_name = 'my_promise.py'
promise_file = os.path.join(self.plugin_dir, promise_name)
self.configureLauncher(save_method=test_method, timeout=1, enable_anomaly=True)
self.generatePromiseScript(promise_name, success=True, periodicity=0.01,
with_anomaly=True, is_tested=False)
# run promise, no failure
self.launcher.run()
cache_folder = os.path.join(self.partition_dir, PROMISE_CACHE_FOLDER_NAME)
cache_file = os.path.join(cache_folder, 'my_promise')
self.assertTrue(os.path.exists(cache_folder))
self.assertTrue(os.path.exists(cache_file))
file_stat = os.stat(promise_file)
with open(cache_file) as f:
cache_dict = json.load(f)
timestamp = cache_dict.pop('timestamp')
info_dict = {
u'is_tested': False,
u'is_anomaly_detected': True,
u'periodicity': 0.01,
u'next_run_after' : (timestamp + 0.01 * 60.0),
u'module_file': u'%s' % promise_file,
u'module_file_mtime': file_stat.st_mtime,
}
self.assertEqual(info_dict, cache_dict)
self.assertTrue(self.called)
next_run_after = cache_dict['next_run_after']
# periodicity not match
self.called = False
self.configureLauncher(save_method=test_method, timeout=1, enable_anomaly=True)
self.launcher.run()
self.assertFalse(self.called)
with open(cache_file) as f:
cache_dict = json.load(f)
# no change!
current_timestamp = cache_dict.pop('timestamp')
self.assertEqual(current_timestamp, timestamp)
self.assertEqual(info_dict, cache_dict)
time.sleep(1)
# periodicity match
self.configureLauncher(save_method=test_method, timeout=1, enable_anomaly=True)
self.launcher.run()
# cached was updated
with open(cache_file) as f:
cache_dict = json.load(f)
new_timestamp = cache_dict.pop('timestamp')
info_dict = {
u'is_tested': False,
u'is_anomaly_detected': True,
u'periodicity': 0.01,
u'next_run_after' : (new_timestamp + 0.01 * 60.0),
u'module_file': u'%s' % promise_file,
u'module_file_mtime': file_stat.st_mtime,
}
self.assertTrue(new_timestamp > timestamp)
# next run is in future
self.assertTrue(cache_dict['next_run_after'] > next_run_after)
self.assertEqual(info_dict, cache_dict)
def test_promise_cache_invalidated_if_file_change(self):
promise_name = 'my_promise.py'
promise_file = os.path.join(self.plugin_dir, promise_name)
self.configureLauncher(timeout=1, enable_anomaly=True)
self.generatePromiseScript(promise_name, success=True, periodicity=1,
with_anomaly=False, is_tested=True)
# run promise, no failure
self.launcher.run()
cache_folder = os.path.join(self.partition_dir, PROMISE_CACHE_FOLDER_NAME)
cache_file = os.path.join(cache_folder, 'my_promise')
file_stat = os.stat(promise_file)
with open(cache_file) as f:
cache_dict = json.load(f)
timestamp = cache_dict.pop('timestamp')
info_dict = {
u'is_tested': True,
u'is_anomaly_detected': False,
u'periodicity': 1,
u'next_run_after' : timestamp,
u'module_file': u'%s' % promise_file,
u'module_file_mtime': file_stat.st_mtime,
}
self.assertEqual(info_dict, cache_dict)
# regenerate promise script, we will run anomaly now
self.generatePromiseScript(promise_name, success=True, periodicity=1,
with_anomaly=True, is_tested=True)
# allow reload module in a same process
os.system('rm %s/*.pyc' % self.plugin_dir)
file_stat = os.stat(promise_file)
self.assertNotEqual(file_stat.st_mtime, info_dict['module_file_mtime'])
self.configureLauncher(timeout=1, enable_anomaly=True)
self.launcher.run()
with open(cache_file) as f:
cache_dict = json.load(f)
timestamp = cache_dict.pop('timestamp')
info_dict['module_file_mtime'] = file_stat.st_mtime
info_dict['next_run_after'] = timestamp + 60.0
info_dict['is_anomaly_detected'] = True
# cache has changed
self.assertEqual(info_dict, cache_dict)
class TestSlapOSGenericPromise(TestSlapOSPromiseMixin):
def initialisePromise(self, promise_content="", success=True, timeout=60):
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment