Feature/promise check surykatka json

/reviewed-on nexedi/slapos.toolbox!72

slapos/promise/plugin/check_surykatka_json.py

+from zope.interface import implementer
+from slapos.grid.promise import interface
+from slapos.grid.promise.generic import GenericPromise
+
+import datetime
+import email.utils
+import json
+import os
+import time
+
+
+@implementer(interface.IPromise)
+class RunPromise(GenericPromise):
+  def __init__(self, config):
+    super(RunPromise, self).__init__(config)
+    # Set frequency compatible to default surykatka interval - 2 minutes
+    self.setPeriodicity(float(self.getConfig('frequency', 2)))
+
+  def senseBotStatus(self):
+    key = 'bot_status'
+
+    def logError(msg, *args):
+      self.logger.error(key + ': ' + msg, *args)
+
+    if key not in self.surykatka_json:
+      logError("%r not in %r", key, self.json_file)
+      return
+    bot_status_list = self.surykatka_json[key]
+    if len(bot_status_list) == 0:
+      logError("%r empty in %r", key, self.json_file)
+      return
+    bot_status = bot_status_list[0]
+    if bot_status.get('text') != 'loop':
+      logError("No type loop detected in %r", self.json_file)
+      return
+    timetuple = email.utils.parsedate(bot_status['date'])
+    last_bot_datetime = datetime.datetime.fromtimestamp(time.mktime(timetuple))
+    delta = self.utcnow - last_bot_datetime
+    # sanity check
+    if delta < datetime.timedelta(minutes=0):
+      logError('Last bot datetime %s is in future, UTC now %s',
+               last_bot_datetime, self.utcnow)
+      return
+    if delta > datetime.timedelta(minutes=15):
+      logError('Last bot datetime %s is more than 15 minutes old, UTC now %s',
+               last_bot_datetime, self.utcnow)
+      return
+
+    self.logger.info(
+      '%s: Last bot status from %s ok, UTC now is %s',
+      key, last_bot_datetime, self.utcnow)
+
+  def senseHttpQuery(self):
+    key = 'http_query'
+
+    def logError(msg, *args):
+      self.logger.error(key + ': ' + msg, *args)
+
+    if key not in self.surykatka_json:
+      logError("%r not in %r", key, self.json_file)
+      return
+
+    url = self.getConfig('url')
+    status_code = self.getConfig('status-code')
+    ip_list = self.getConfig('ip-list', '').split()
+
+    entry_list = [q for q in self.surykatka_json[key] if q['url'] == url]
+    if len(entry_list) == 0:
+      logError('No data for %r', url)
+      return
+    error_list = []
+    for entry in entry_list:
+      if str(entry['status_code']) != str(status_code):
+        error_list.append(
+          'IP %s got status code %s instead of %s' % (
+            entry['ip'], entry['status_code'], status_code))
+    db_ip_list = [q['ip'] for q in entry_list]
+    if len(ip_list):
+      if set(ip_list) != set(db_ip_list):
+        error_list.append(
+          'expected IPs %s differes from got %s' % (
+            ' '.join(ip_list), ' '.join(db_ip_list)))
+    if len(error_list):
+      logError('Problem with %s: ' % (url,) + ', '.join(error_list))
+      return
+    if len(ip_list) > 0:
+      self.logger.info(
+        '%s: %s replied correctly with status code %s on ip list %s',
+        key, url, status_code, ' '.join(ip_list))
+    else:
+      self.logger.info(
+        '%s: %s replied correctly with status code %s',
+        key, url, status_code)
+
+  def sense(self):
+    """
+      Check if frontend URL is available
+    """
+    test_utcnow = self.getConfig('test-utcnow')
+    if test_utcnow:
+      self.utcnow = datetime.datetime.fromtimestamp(
+        time.mktime(email.utils.parsedate(test_utcnow)))
+    else:
+      self.utcnow = datetime.datetime.utcnow()
+
+    self.json_file = self.getConfig('json-file', '')
+    if not os.path.exists(self.json_file):
+      self.logger.error('File %r does not exists', self.json_file)
+      return
+    with open(self.json_file) as fh:
+      try:
+        self.surykatka_json = json.load(fh)
+      except Exception:
+        self.logger.error("Problem loading JSON from %r", self.json_file)
+        return
+
+    report = self.getConfig('report')
+    if report == 'bot_status':
+      return self.senseBotStatus()
+    elif report == 'http_query':
+      return self.senseHttpQuery()
+    else:
+      self.logger.error("Report %r is not supported", report)
+      return
+
+  def anomaly(self):
+    return self._test(result_count=3, failure_amount=3)

slapos/test/promise/plugin/test_check_surykatka_json.py

+from slapos.grid.promise import PromiseError
+from slapos.test.promise.plugin import TestPromisePluginMixin
+
+import os
+import shutil
+import tempfile
+
+
+class CheckSurykatkaJSONMixin(TestPromisePluginMixin):
+  promise_name = 'check-surykatka-json.py'
+
+  def setUp(self):
+    self.working_directory = tempfile.mkdtemp()
+    self.json_file = os.path.join(self.working_directory, 'surykatka.json')
+    self.addCleanup(shutil.rmtree, self.working_directory)
+    TestPromisePluginMixin.setUp(self)
+
+  def writeSurykatkaPromise(self, d=None):
+    if d is None:
+      d = {}
+    content_list = [
+      "from slapos.promise.plugin.check_surykatka_json import RunPromise"]
+    content_list.append('extra_config_dict = {')
+    for k, v in d.items():
+      content_list.append("  '%s': '%s'," % (k, v))
+    content_list.append('}')
+    self.writePromise(self.promise_name, '\n'.join(content_list))
+
+  def writeSurykatkaJson(self, content):
+    with open(self.json_file, 'w') as fh:
+      fh.write(content)
+
+  def assertFailedMessage(self, result, message):
+    self.assertEqual(result['result']['failed'], True)
+    self.assertEqual(
+      result['result']['message'],
+      message)
+
+  def assertPassedMessage(self, result, message):
+    self.assertEqual(result['result']['failed'], False)
+    self.assertEqual(
+      result['result']['message'],
+      message)
+
+
+class TestCheckSurykatkaJSON(CheckSurykatkaJSONMixin):
+  def test_no_config(self):
+    self.writeSurykatkaPromise()
+    self.configureLauncher()
+    with self.assertRaises(PromiseError):
+      self.launcher.run()
+    self.assertFailedMessage(
+      self.getPromiseResult(self.promise_name),
+      "File '' does not exists")
+
+  def test_not_existing_file(self):
+    self.writeSurykatkaPromise({'json-file': self.json_file})
+    self.configureLauncher()
+    with self.assertRaises(PromiseError):
+      self.launcher.run()
+    self.assertFailedMessage(
+      self.getPromiseResult(self.promise_name),
+      "File '%s' does not exists" % (self.json_file,))
+
+  def test_empty_file(self):
+    self.writeSurykatkaPromise({'json-file': self.json_file})
+    self.writeSurykatkaJson('')
+    self.configureLauncher()
+    with self.assertRaises(PromiseError):
+      self.launcher.run()
+    self.assertFailedMessage(
+      self.getPromiseResult(self.promise_name),
+      "Problem loading JSON from '%s'" % (self.json_file,))
+
+
+class TestCheckSurykatkaJSONUnknownReport(CheckSurykatkaJSONMixin):
+  def test(self):
+    self.writeSurykatkaPromise(
+      {
+        'report': 'NOT_EXISTING_ENTRY',
+        'json-file': self.json_file,
+      }
+    )
+    self.writeSurykatkaJson("""{
+}
+""")
+    self.configureLauncher()
+    with self.assertRaises(PromiseError):
+      self.launcher.run()
+    self.assertFailedMessage(
+      self.getPromiseResult(self.promise_name),
+      "Report 'NOT_EXISTING_ENTRY' is not supported")
+
+
+class TestCheckSurykatkaJSONBotStatus(CheckSurykatkaJSONMixin):
+  def test(self):
+    self.writeSurykatkaPromise(
+      {
+        'report': 'bot_status',
+        'json-file': self.json_file,
+        'test-utcnow': 'Wed, 13 Dec 2222 09:11:12 -0000'
+      }
+    )
+    self.writeSurykatkaJson("""{
+    "bot_status": [
+        {
+            "date": "Wed, 13 Dec 2222 09:10:11 -0000",
+            "text": "loop"
+        }
+    ]
+}
+""")
+    self.configureLauncher()
+    self.launcher.run()
+    self.assertPassedMessage(
+      self.getPromiseResult(self.promise_name),
+      "bot_status: Last bot status from 2222-12-13 09:10:11 ok, "
+      "UTC now is 2222-12-13 09:11:12"
+    )
+
+  def test_bot_status_future(self):
+    self.writeSurykatkaPromise(
+      {
+        'report': 'bot_status',
+        'json-file': self.json_file,
+        'test-utcnow': 'Wed, 13 Dec 2222 09:11:12 -0000'
+      }
+    )
+    self.writeSurykatkaJson("""{
+    "bot_status": [
+        {
+            "date": "Wed, 13 Dec 2223 09:10:11 -0000",
+            "text": "loop"
+        }
+    ]
+}
+""")
+    self.configureLauncher()
+    with self.assertRaises(PromiseError):
+      self.launcher.run()
+    self.assertFailedMessage(
+      self.getPromiseResult(self.promise_name),
+      "bot_status: Last bot datetime 2223-12-13 09:10:11 is in "
+      "future, UTC now 2222-12-13 09:11:12"
+    )
+
+  def test_bot_status_old(self):
+    self.writeSurykatkaPromise(
+      {
+        'report': 'bot_status',
+        'json-file': self.json_file,
+        'test-utcnow': 'Wed, 13 Dec 2223 09:26:12 -0000'
+      }
+    )
+    self.writeSurykatkaJson("""{
+    "bot_status": [
+        {
+            "date": "Wed, 13 Dec 2223 09:10:11 -0000",
+            "text": "loop"
+        }
+    ]
+}
+""")
+    self.configureLauncher()
+    with self.assertRaises(PromiseError):
+      self.launcher.run()
+    self.assertFailedMessage(
+      self.getPromiseResult(self.promise_name),
+      "bot_status: Last bot datetime 2223-12-13 09:10:11 is "
+      "more than 15 minutes old, UTC now 2223-12-13 09:26:12"
+    )
+
+  def test_not_bot_status(self):
+    self.writeSurykatkaPromise(
+      {
+        'report': 'bot_status',
+        'json-file': self.json_file,
+      }
+    )
+    self.writeSurykatkaJson("""{
+}
+""")
+    self.configureLauncher()
+    with self.assertRaises(PromiseError):
+      self.launcher.run()
+    self.assertFailedMessage(
+      self.getPromiseResult(self.promise_name),
+      "bot_status: 'bot_status' not in '%s'" % (self.json_file,))
+
+  def test_empty_bot_status(self):
+    self.writeSurykatkaPromise(
+      {
+        'report': 'bot_status',
+        'json-file': self.json_file,
+      }
+    )
+    self.writeSurykatkaJson("""{
+  "bot_status": []
+}
+""")
+    self.configureLauncher()
+    with self.assertRaises(PromiseError):
+      self.launcher.run()
+    self.assertFailedMessage(
+      self.getPromiseResult(self.promise_name),
+      "bot_status: 'bot_status' empty in '%s'" % (self.json_file,))
+
+
+class TestCheckSurykatkaJSONHttpQuery(CheckSurykatkaJSONMixin):
+  def test(self):
+    self.writeSurykatkaPromise(
+      {
+        'report': 'http_query',
+        'json-file': self.json_file,
+        'url': 'https://www.erp5.com/',
+        'status-code': '302',
+        'ip-list': '127.0.0.1 127.0.0.2'
+      }
+    )
+    self.writeSurykatkaJson("""{
+    "http_query": [
+        {
+            "date": "Wed, 11 Dec 2019 09:35:28 -0000",
+            "ip": "127.0.0.1",
+            "status_code": 302,
+            "url": "https://www.erp5.com/"
+        },
+        {
+            "date": "Wed, 11 Dec 2019 09:35:28 -0000",
+            "ip": "127.0.0.2",
+            "status_code": 302,
+            "url": "https://www.erp5.com/"
+        },
+        {
+            "date": "Wed, 11 Dec 2019 09:35:28 -0000",
+            "ip": "176.31.129.213",
+            "status_code": 200,
+            "url": "https://www.erp5.org/"
+        }
+    ]
+}
+""")
+    self.configureLauncher()
+    self.launcher.run()
+    self.assertPassedMessage(
+      self.getPromiseResult(self.promise_name),
+      "http_query: https://www.erp5.com/ replied correctly with "
+      "status code 302 on ip list 127.0.0.1 127.0.0.2"
+    )
+
+  def test_no_ip_list(self):
+    self.writeSurykatkaPromise(
+      {
+        'report': 'http_query',
+        'json-file': self.json_file,
+        'url': 'https://www.erp5.com/',
+        'status-code': '302',
+      }
+    )
+    self.writeSurykatkaJson("""{
+    "http_query": [
+        {
+            "date": "Wed, 11 Dec 2019 09:35:28 -0000",
+            "ip": "127.0.0.1",
+            "status_code": 302,
+            "url": "https://www.erp5.com/"
+        },
+        {
+            "date": "Wed, 11 Dec 2019 09:35:28 -0000",
+            "ip": "127.0.0.2",
+            "status_code": 302,
+            "url": "https://www.erp5.com/"
+        },
+        {
+            "date": "Wed, 11 Dec 2019 09:35:28 -0000",
+            "ip": "176.31.129.213",
+            "status_code": 200,
+            "url": "https://www.erp5.org/"
+        }
+    ]
+}
+""")
+    self.configureLauncher()
+    self.launcher.run()
+    self.assertPassedMessage(
+      self.getPromiseResult(self.promise_name),
+      "http_query: https://www.erp5.com/ replied correctly with "
+      "status code 302"
+    )
+
+  def test_bad_code(self):
+    self.writeSurykatkaPromise(
+      {
+        'report': 'http_query',
+        'json-file': self.json_file,
+        'url': 'https://www.erp5.com/',
+        'status-code': '301',
+      }
+    )
+    self.writeSurykatkaJson("""{
+    "http_query": [
+        {
+            "date": "Wed, 11 Dec 2019 09:35:28 -0000",
+            "ip": "127.0.0.1",
+            "status_code": 302,
+            "url": "https://www.erp5.com/"
+        },
+        {
+            "date": "Wed, 11 Dec 2019 09:35:28 -0000",
+            "ip": "127.0.0.2",
+            "status_code": 301,
+            "url": "https://www.erp5.com/"
+        },
+        {
+            "date": "Wed, 11 Dec 2019 09:35:28 -0000",
+            "ip": "176.31.129.213",
+            "status_code": 200,
+            "url": "https://www.erp5.org/"
+        }
+    ]
+}
+""")
+    self.configureLauncher()
+    with self.assertRaises(PromiseError):
+      self.launcher.run()
+    self.assertFailedMessage(
+      self.getPromiseResult(self.promise_name),
+      "http_query: Problem with https://www.erp5.com/: "
+      "IP 127.0.0.1 got status code 302 instead of 301"
+    )
+
+  def test_bad_ip(self):
+    self.writeSurykatkaPromise(
+      {
+        'report': 'http_query',
+        'json-file': self.json_file,
+        'url': 'https://www.erp5.com/',
+        'status-code': '301',
+        'ip-list': '127.0.0.1 127.0.0.2'
+      }
+    )
+    self.writeSurykatkaJson("""{
+    "http_query": [
+        {
+            "date": "Wed, 11 Dec 2019 09:35:28 -0000",
+            "ip": "127.0.0.1",
+            "status_code": 301,
+            "url": "https://www.erp5.com/"
+        },
+        {
+            "date": "Wed, 11 Dec 2019 09:35:28 -0000",
+            "ip": "127.0.0.4",
+            "status_code": 301,
+            "url": "https://www.erp5.com/"
+        },
+        {
+            "date": "Wed, 11 Dec 2019 09:35:28 -0000",
+            "ip": "176.31.129.213",
+            "status_code": 200,
+            "url": "https://www.erp5.org/"
+        }
+    ]
+}
+""")
+    self.configureLauncher()
+    with self.assertRaises(PromiseError):
+      self.launcher.run()
+    self.assertFailedMessage(
+      self.getPromiseResult(self.promise_name),
+      "http_query: Problem with https://www.erp5.com/: "
+      "expected IPs 127.0.0.1 127.0.0.2 differes from got 127.0.0.1 127.0.0.4"
+    )
+
+  def test_bad_ip_status_code(self):
+    self.writeSurykatkaPromise(
+      {
+        'report': 'http_query',
+        'json-file': self.json_file,
+        'url': 'https://www.erp5.com/',
+        'status-code': '301',
+        'ip-list': '127.0.0.1 127.0.0.2'
+      }
+    )
+    self.writeSurykatkaJson("""{
+    "http_query": [
+        {
+            "date": "Wed, 11 Dec 2019 09:35:28 -0000",
+            "ip": "127.0.0.1",
+            "status_code": 302,
+            "url": "https://www.erp5.com/"
+        },
+        {
+            "date": "Wed, 11 Dec 2019 09:35:28 -0000",
+            "ip": "127.0.0.4",
+            "status_code": 301,
+            "url": "https://www.erp5.com/"
+        },
+        {
+            "date": "Wed, 11 Dec 2019 09:35:28 -0000",
+            "ip": "176.31.129.213",
+            "status_code": 200,
+            "url": "https://www.erp5.org/"
+        }
+    ]
+}
+""")
+    self.configureLauncher()
+    with self.assertRaises(PromiseError):
+      self.launcher.run()
+    self.assertFailedMessage(
+      self.getPromiseResult(self.promise_name),
+      "http_query: Problem with https://www.erp5.com/: "
+      "IP 127.0.0.1 got status code 302 instead of 301, "
+      "expected IPs 127.0.0.1 127.0.0.2 differes from got "
+      "127.0.0.1 127.0.0.4"
+    )