slapgrid.promise: Improve promise check speed

Use cache to  save some promise instance information.
First run of slapgrid fill each promise cached (when there is no cache). The next run will check some usefull info like if promise `is test less` or promise `is anomaly less` directly from cache without need to import the module.

Cache is updated if promise file or module change, or if the promise periodicity expire (when the promise it update cache).

/reviewed-on nexedi/slapos.core!101

slapos/grid/promise/generic.py

@@ -46,7 +46,6 @@ PROMISE_RESULT_FOLDER_NAME = '.slapgrid/promise/result'
 PROMISE_LOG_FOLDER_NAME = '.slapgrid/promise/log'
 
 PROMISE_PARAMETER_NAME = 'extra_config_dict'
-PROMISE_PERIOD_FILE_NAME = '%s.periodicity'
 
 LOGLINE_RE = r"(\d{4}-\d{2}-\d{2}\s+\d{2}:\d{2}:\d{2})\s+\-?\s*(\w{4,7})\s+\-?\s+(\d+\-\d{3})\s+\-?\s*(.*)"
 matchLogStr = re.compile(LOGLINE_RE).match
@@ -159,10 +158,6 @@ class GenericPromise(with_metaclass(ABCMeta, object)):
     self.__promise_path = self.__config.pop('path', None)
     self.__queue = self.__config.pop('queue', None)
     self.__logger_buffer = None
-    self.__periodicity_file = os.path.join(
-      self.__partition_folder,
-      PROMISE_STATE_FOLDER_NAME,
-      PROMISE_PERIOD_FILE_NAME % self.__name)
     self.setPeriodicity(self.__config.pop('periodicity', 2))
 
     self.__transaction_id = '%s-%s' % (int(time.time()), random.randint(100, 999))
@@ -236,8 +231,6 @@ class GenericPromise(with_metaclass(ABCMeta, object)):
     if minute <= 0:
       raise ValueError("Cannot set promise periodicity to a value less than 1")
     self.__periodicity = minute
-    with open(self.__periodicity_file, 'w') as f:
-      f.write('%s' % minute)
 
   def getPeriodicity(self):
     return self.__periodicity

slapos/tests/test_promise.py

@@ -36,7 +36,8 @@ import logging
 from datetime import datetime, timedelta
 import six
 from six.moves import queue
-from slapos.grid.promise import interface, PromiseLauncher, PromiseProcess, PromiseError
+from slapos.grid.promise import (interface, PromiseLauncher, PromiseProcess,
+                                 PromiseError, PROMISE_CACHE_FOLDER_NAME)
 from slapos.grid.promise.generic import (GenericPromise, TestResult, AnomalyResult,
                                          PromiseQueueResult, PROMISE_STATE_FOLDER_NAME,
                                          PROMISE_RESULT_FOLDER_NAME,
@@ -93,7 +94,7 @@ class TestSlapOSPromiseMixin(unittest.TestCase):
 
     self.launcher = PromiseLauncher(
       config=parameter_dict,
-      logger=logging.getLogger('slapos.test.promise'),
+      #logger=logging.getLogger('slapos.test.promise'),
       dry_run=dry_run
     )
     if save_method:
@@ -1213,6 +1214,102 @@ exit 1
     # no result returned by the promise
     self.assertTrue(self.called)
 
+  def test_promise_cache(self):
+    promise_name = 'my_promise.py'
+    promise_file = os.path.join(self.plugin_dir, promise_name)
+    self.configureLauncher(timeout=1, enable_anomaly=True)
+    self.generatePromiseScript(promise_name, success=True, periodicity=0.01,
+      with_anomaly=True, is_tested=False)
+
+    # run promise, no failure
+    self.launcher.run()
+    cache_folder = os.path.join(self.partition_dir, PROMISE_CACHE_FOLDER_NAME)
+    cache_file = os.path.join(cache_folder, 'my_promise')
+    self.assertTrue(os.path.exists(cache_folder))
+    self.assertTrue(os.path.exists(cache_file))
+    file_stat = os.stat(promise_file)
+    with open(cache_file) as f:
+      cache_dict = json.load(f)
+    timestamp = cache_dict.pop('timestamp')
+    info_dict = {
+      u'is_tested': False,
+      u'is_anomaly_detected': True,
+      u'periodicity': 0.01,
+      u'next_run_after' : (timestamp + 0.01 * 60.0),
+      u'module_file': u'%s' % promise_file,
+      u'module_file_mtime': file_stat.st_mtime,
+    }
+    # next run is in future
+    self.assertTrue(info_dict['next_run_after'] > time.time())
+    self.assertEqual(info_dict, cache_dict)
+
+  def test_promise_cache_expire_with_periodicity(self):
+    self.called = False
+    def test_method(result):
+      self.called = True
+
+    promise_name = 'my_promise.py'
+    promise_file = os.path.join(self.plugin_dir, promise_name)
+    self.configureLauncher(save_method=test_method, timeout=1, enable_anomaly=True)
+    self.generatePromiseScript(promise_name, success=True, periodicity=0.01,
+      with_anomaly=True, is_tested=False)
+
+    # run promise, no failure
+    self.launcher.run()
+    cache_folder = os.path.join(self.partition_dir, PROMISE_CACHE_FOLDER_NAME)
+    cache_file = os.path.join(cache_folder, 'my_promise')
+    self.assertTrue(os.path.exists(cache_folder))
+    self.assertTrue(os.path.exists(cache_file))
+    file_stat = os.stat(promise_file)
+    with open(cache_file) as f:
+      cache_dict = json.load(f)
+    timestamp = cache_dict.pop('timestamp')
+    info_dict = {
+      u'is_tested': False,
+      u'is_anomaly_detected': True,
+      u'periodicity': 0.01,
+      u'next_run_after' : (timestamp + 0.01 * 60.0),
+      u'module_file': u'%s' % promise_file,
+      u'module_file_mtime': file_stat.st_mtime,
+    }
+    self.assertEqual(info_dict, cache_dict)
+    self.assertTrue(self.called)
+    next_run_after = cache_dict['next_run_after']
+
+    # periodicity not match
+    self.called = False
+    self.configureLauncher(save_method=test_method, timeout=1, enable_anomaly=True)
+    self.launcher.run()
+    self.assertFalse(self.called)
+    with open(cache_file) as f:
+      cache_dict = json.load(f)
+    # no change!
+    current_timestamp = cache_dict.pop('timestamp')
+    self.assertEqual(current_timestamp, timestamp)
+    self.assertEqual(info_dict, cache_dict)
+
+    time.sleep(1)
+    # periodicity match
+    self.configureLauncher(save_method=test_method, timeout=1, enable_anomaly=True)
+    self.launcher.run()
+
+    # cached was updated
+    with open(cache_file) as f:
+      cache_dict = json.load(f)
+    new_timestamp = cache_dict.pop('timestamp')
+    info_dict = {
+      u'is_tested': False,
+      u'is_anomaly_detected': True,
+      u'periodicity': 0.01,
+      u'next_run_after' : (new_timestamp + 0.01 * 60.0),
+      u'module_file': u'%s' % promise_file,
+      u'module_file_mtime': file_stat.st_mtime,
+    }
+    self.assertTrue(new_timestamp > timestamp)
+    # next run is in future
+    self.assertTrue(cache_dict['next_run_after'] > next_run_after)
+    self.assertEqual(info_dict, cache_dict)
+
 class TestSlapOSGenericPromise(TestSlapOSPromiseMixin):
 
   def initialisePromise(self, promise_content="", success=True, timeout=60):