monitor: Implement failure-amount

By default one failure is considered as bad thing, but allows to select sites
which shall be treated with some tolerance.

software/monitor/buildout.hash.cfg

@@ -14,7 +14,7 @@
 # not need these here).
 [template]
 filename = instance.cfg
-md5sum = 1e64cee5a2249d19f6241312f94d06b7
+md5sum = 07ef40555dc76f87ecd345b99ac65c4b
 
 [template-monitor]
 _update_hash_filename_ = instance-monitor.cfg.jinja2
@@ -30,7 +30,7 @@ md5sum = 9e237dbdda59e788202f0da194a57d41
 
 [template-monitor-edgebot]
 _update_hash_filename_ = instance-monitor-edgebot.cfg.jinja2
-md5sum = b8f3c467cfd988e29be2ba70aa7a059e
+md5sum = f28a329e830ed737d468abcb4e89e1a2
 
 [network-bench-cfg]
 filename = network_bench.cfg.in

software/monitor/instance-edgetest-input-schema.json

@@ -30,6 +30,12 @@
       "title": "Default maximum elapsed time for a site to reply (seconds)",
       "description": "Default maximum elapsed time for a site to reply to be considered good (default: 2s).",
       "type": "string"
+    },
+    "failure-amount": {
+      "default": "1",
+      "title": "Default amount of failures to consider URL as in bad state",
+      "description": "Default amount of failures to consider URL as in bad state, can be set to higher value for endpoints with accepted short outages (default: 1).",
+      "type": "string"
     }
   }
 }

software/monitor/instance-edgetest-slave-input-schema.json

@@ -29,6 +29,12 @@
       "title": "Maximum elapsed time for a site to reply (seconds)",
       "description": "Maximum elapsed time for a site to reply to be considered good.(default: comes from master partition).",
       "type": "string"
+    },
+    "failure-amount": {
+      "default": "Master default",
+      "title": "Amount of failures to consider URL as in bad state",
+      "description": "Amount of failures to consider URL as in bad state, can be set to higher value for endpoints with accepted short outages (default: comes from master partition).",
+      "type": "string"
     }
   }
 }

software/monitor/instance-monitor-edgebot.cfg.jinja2

@@ -16,6 +16,9 @@
 {%-   if 'check-certificate-expiration-days' not in slave %}
 {%-     do slave.__setitem__('check-certificate-expiration-days', CONFIGURATION['check-certificate-expiration-days']) %}
 {%-   endif %}
+{%-   if 'failure-amount' not in slave %}
+{%-     do slave.__setitem__('failure-amount', CONFIGURATION['failure-amount']) %}
+{%-   endif %}
 {%-   if 'check-maximum-elapsed-time' not in slave %}
 {%-     do slave.__setitem__('check-maximum-elapsed-time', CONFIGURATION['check-maximum-elapsed-time']) %}
 {%-   endif %}
@@ -46,6 +49,7 @@ config-report = http_query
 config-url = {{ slave['url'] }}
 config-status-code = {{ slave['check-status-code'] }}
 config-certificate-expiration-days = {{ slave['check-certificate-expiration-days'] }}
+config-failure-amount = {{ slave['failure-amount'] }}
 config-maximum-elapsed-time = {{ slave['check-maximum-elapsed-time'] }}
 config-ip-list = {{ slave['check-frontend-ip'] }}
 config-json-file = ${surykatka-config-{{ class }}:json}

software/monitor/instance.cfg

@@ -80,6 +80,7 @@ configuration.nameserver =
 configuration.check-frontend-ip =
 configuration.check-certificate-expiration-days = 15
 configuration.check-maximum-elapsed-time = 2
+configuration.failure-amount = 2
 # use monitor-base-port to have monitor listening on each instance
 # on different port and also on different port than other services
 # it makes it possible to instantiate it correctly on signle IP, for

software/monitor/test/test.py

@@ -303,6 +303,10 @@ URL =
       'http-query-backend-300-promise.py',
       "'json-file': '%s'" % (self.surykatka_dict[2]['json-file'],)
     )
+    self.assertPromiseContent(
+      'http-query-backend-300-promise.py',
+      "'failure-amount': '2'"
+    )
 
     self.assertPromiseContent(
       'http-query-backend-promise.py',
@@ -323,6 +327,10 @@ URL =
       'http-query-backend-promise.py',
       "'json-file': '%s'" % (self.surykatka_dict[2]['json-file'],)
     )
+    self.assertPromiseContent(
+      'http-query-backend-promise.py',
+      "'failure-amount': '2'"
+    )
 
   def requestEdgetestSlaves(self):
     self.requestEdgetestSlave(
@@ -377,6 +385,10 @@ URL =
       'http-query-backend-promise.py',
       "'json-file': '%s'" % (self.surykatka_dict[2]['json-file'],)
     )
+    self.assertPromiseContent(
+      'http-query-backend-promise.py',
+      "'failure-amount': '2'"
+    )
 
   def requestEdgetestSlaves(self):
     self.requestEdgetestSlave(
@@ -422,6 +434,10 @@ URL =
       'http-query-backend-501-promise.py',
       "'json-file': '%s'" % (self.surykatka_dict[2]['json-file'],)
     )
+    self.assertPromiseContent(
+      'http-query-backend-501-promise.py',
+      "'failure-amount': '2'"
+    )
 
     self.assertPromiseContent(
       'http-query-backend-promise.py',
@@ -442,6 +458,10 @@ URL =
       'http-query-backend-promise.py',
       "'json-file': '%s'" % (self.surykatka_dict[2]['json-file'],)
     )
+    self.assertPromiseContent(
+      'http-query-backend-promise.py',
+      "'failure-amount': '2'"
+    )
 
   def requestEdgetestSlaves(self):
     self.requestEdgetestSlave(
@@ -492,6 +512,10 @@ URL =
       'http-query-backend-20-promise.py',
       "'json-file': '%s'" % (self.surykatka_dict[2]['json-file'],)
     )
+    self.assertPromiseContent(
+      'http-query-backend-20-promise.py',
+      "'failure-amount': '2'"
+    )
 
     self.assertPromiseContent(
       'http-query-backend-promise.py',
@@ -512,6 +536,10 @@ URL =
       'http-query-backend-promise.py',
       "'json-file': '%s'" % (self.surykatka_dict[2]['json-file'],)
     )
+    self.assertPromiseContent(
+      'http-query-backend-promise.py',
+      "'failure-amount': '2'"
+    )
 
   def requestEdgetestSlaves(self):
     self.requestEdgetestSlave(
@@ -574,6 +602,10 @@ URL =
       'http-query-backend-20-promise.py',
       "'json-file': '%s'" % (self.surykatka_dict[20]['json-file'],)
     )
+    self.assertPromiseContent(
+      'http-query-backend-20-promise.py',
+      "'failure-amount': '2'"
+    )
 
     self.assertPromiseContent(
       'http-query-backend-default-promise.py',
@@ -594,6 +626,10 @@ URL =
       'http-query-backend-default-promise.py',
       "'json-file': '%s'" % (self.surykatka_dict[5]['json-file'],)
     )
+    self.assertPromiseContent(
+      'http-query-backend-default-promise.py',
+      "'failure-amount': '2'"
+    )
 
     self.assertPromiseContent(
       'http-query-backend-1-promise.py',
@@ -614,6 +650,10 @@ URL =
       'http-query-backend-1-promise.py',
       "'json-file': '%s'" % (self.surykatka_dict[1]['json-file'],)
     )
+    self.assertPromiseContent(
+      'http-query-backend-1-promise.py',
+      "'failure-amount': '2'"
+    )
 
   def requestEdgetestSlaves(self):
     self.requestEdgetestSlave(
@@ -630,3 +670,69 @@ URL =
       {'url': 'https://www.erp5.net/',
        'check-maximum-elapsed-time': '1'},
     )
+
+
+class TestEdgeFailureAmount(
+  EdgeSlaveMixin, SlapOSInstanceTestCase):
+  surykatka_dict = {
+    2: {'expected_ini': """[SURYKATKA]
+INTERVAL = 120
+TIMEOUT = 4
+SQLITE = %(db_file)s
+URL =
+  https://www.erp5.org/
+  https://www.erp5.com/"""}
+  }
+
+  @classmethod
+  def getInstanceParameterDict(cls):
+    return {
+      'failure-amount': '5'
+    }
+
+  def assertSurykatkaPromises(self):
+    self.assertPromiseContent(
+      'http-query-backend-promise.py',
+      "'report': 'http_query'")
+    self.assertPromiseContent(
+      'http-query-backend-promise.py',
+      "'status-code': '200'")
+    self.assertPromiseContent(
+      'http-query-backend-promise.py',
+      "'url': 'https://www.erp5.com/'")
+    self.assertPromiseContent(
+      'http-query-backend-promise.py',
+      "'json-file': '%s'" % (self.surykatka_dict[2]['json-file'],)
+    )
+    self.assertPromiseContent(
+      'http-query-backend-promise.py',
+      "'failure-amount': '5'"
+    )
+
+    self.assertPromiseContent(
+      'http-query-backend-10-promise.py',
+      "'report': 'http_query'")
+    self.assertPromiseContent(
+      'http-query-backend-10-promise.py',
+      "'status-code': '200'")
+    self.assertPromiseContent(
+      'http-query-backend-10-promise.py',
+      "'url': 'https://www.erp5.org/'")
+    self.assertPromiseContent(
+      'http-query-backend-10-promise.py',
+      "'json-file': '%s'" % (self.surykatka_dict[2]['json-file'],)
+    )
+    self.assertPromiseContent(
+      'http-query-backend-10-promise.py',
+      "'failure-amount': '10'"
+    )
+
+  def requestEdgetestSlaves(self):
+    self.requestEdgetestSlave(
+      'backend',
+      {'url': 'https://www.erp5.com/'},
+    )
+    self.requestEdgetestSlave(
+      'backend-10',
+      {'url': 'https://www.erp5.org/', 'failure-amount': '10'},
+    )