Monitor edge testing

# To review

 - [x] [master parameters](https://lab.nexedi.com/luke/slapos/blob/feature/monitor-edge-test/software/monitor/instance-edgetest-input-schema.json)
 - [x] [slave parameters](https://lab.nexedi.com/luke/slapos/blob/feature/monitor-edge-test/software/monitor/instance-edgetest-slave-input-schema.json)

# Tasks

  - [x] assure that slaves are correctly destroyed (using SlapOS Master)


/reviewed-on !660

component/surykatka/buildout.cfg

+[buildout]
+extends =
+  buildout.hash.cfg
+  ../python3/buildout.cfg
+
+parts =
+  surykatka
+
+[surykatka-requirements]
+recipe = slapos.recipe.build:download 
+url = ${:_profile_base_location_}/${:_update_hash_filename_}
+
+[surykatka]
+recipe = plone.recipe.command
+command =
+  bash -c "${python3.7:executable} -m venv ${:location} && \
+    . ${:location}/bin/activate && \
+    pip install -r ${surykatka-requirements:target}"
+location = ${buildout:parts-directory}/${:_buildout_section_name_}
+stop-on-error = true
+executable = ${:location}/bin/surykatka

component/surykatka/buildout.hash.cfg

+# THIS IS NOT A BUILDOUT FILE, despite purposedly using a compatible syntax.
+# The only allowed lines here are (regexes):
+# - "^#" comments, copied verbatim
+# - "^[" section beginings, copied verbatim
+# - lines containing an "=" sign which must fit in the following categorie.
+#   - "^\s*filename\s*=\s*path\s*$" where "path" is relative to this file
+#     Copied verbatim.
+#   - "^\s*hashtype\s*=.*" where "hashtype" is one of the values supported
+#     by the re-generation script.
+#     Re-generated.
+# - other lines are copied verbatim
+# Substitution (${...:...}), extension ([buildout] extends = ...) and
+# section inheritance (< = ...) are NOT supported (but you should really
+# not need these here).
+[surykatka-requirements]
+_update_hash_filename_ = requirements.txt
+md5sum = 897c9a296625b576817f2a1dc1801a90

component/surykatka/requirements.txt

+certifi==2019.11.28
+chardet==3.0.4
+Click==7.0
+dnspython==1.16.0
+forcediphttpsadapter==1.0.1
+idna==2.8
+miniupnpc==2.0.2
+peewee==3.13.1
+requests==2.22.0
+surykatka==0.1.1
+urllib3==1.25.7

software/monitor/README.edgetest.rst

+============
+Edge testing
+============
+
+``edgetest`` is a special software type of monitor software release used for website monitoring by using bots.
+
+It uses `surykatka <https://lab.nexedi.com/nexedi/surykatka>`_ and `check_surykatka_json <https://lab.nexedi.com/nexedi/slapos.toolbox/blob/master/slapos/promise/plugin/check_surykatka_json.py>`_ to monitor websites.
+
+``surykatka`` provides a bot to query list of hosts and a JSON reporting system.
+
+``check_surykatka_json`` is used in promises to provide monitoring information about the websites.
+
+In order to monitor an url one need to:
+
+ * request a monitor software release with ``edgetest`` software type, configured as described in ``instance-edgetest-input-schema.json``,
+ * request a slave to monitor with ``edgetest`` software type, configured as described in ``instance-edgetest-slave-input-schema.json``.

software/monitor/buildout.hash.cfg

@@ -14,16 +14,32 @@
 # not need these here).
 [template]
 filename = instance.cfg
-md5sum = 1b7d2d097f208f6641bf98a17df079c8
+md5sum = cb814297f2f76dc8e08014a7d662439f
 
 [template-monitor]
 _update_hash_filename_ = instance-monitor.cfg.jinja2
-md5sum = 373c79480e6425c20480fc911a56c3fd
-
-[template-monitor-distributor]
-_update_hash_filename_ = instance-monitor-distributor.cfg.jinja2
-md5sum = 61c0bfdfc0a2b51ba15fe4a49baf6091
+md5sum = 165a15672fc85981f68b9af2d6253254
 
 [json-test-template]
 _update_hash_filename_ = json-test-template.json.in.jinja2
 md5sum = 2eb5596544d9c341acf653d4f7ce2680
+
+[template-monitor-edgetest]
+_update_hash_filename_ = instance-monitor-edgetest.cfg.jinja2
+md5sum = 9e237dbdda59e788202f0da194a57d41
+
+[template-monitor-edgebot]
+_update_hash_filename_ = instance-monitor-edgebot.cfg.jinja2
+md5sum = cd4c5d2ecf8285ea8f0905ed1150d8a0
+
+[network-bench-cfg]
+filename = network_bench.cfg.in
+md5sum = cfcbf2002b8eff5153e2bf68ed24b720
+
+[monitor-collect-csv-dump]
+filename = script/collect_csv_dump.py
+md5sum = cad2402bbd21907cfed6bc5af8c5d3ab
+
+[template-surykatka-ini]
+_update_hash_filename_ = surykatka.ini.jinja2
+md5sum = 40870921e05d93b5843ab34abd7e3902

software/monitor/instance-default-input-schema.json

+{
+  "$schema": "http://json-schema.org/draft-04/schema#",
+  "properties": {
+    "dummy": {
+      "title": "dummy",
+      "description": "Dummy",
+      "type": "string"
+    }
+  }
+}

software/monitor/instance-default-output-schema.json

+{
+  "$schema": "http://json-schema.org/draft-04/schema#",
+  "description": "Values returned by Re6st Master instanciation",
+  "properties": {
+    "re6stry-url": {
+      "description": "ipv6 url to access your re6st registry service",
+      "type": "string"
+    }
+  },
+  "type": "object"
+}

software/monitor/instance-edgetest-input-schema.json

+{
+  "$schema": "http://json-schema.org/draft-04/schema#",
+  "properties": {
+    "nameserver": {
+      "default": "",
+      "title": "Nameserver",
+      "description": "Space separated list of name servers to use.",
+      "type": "string"
+    },
+    "check-status-code": {
+      "default": "200",
+      "title": "Default Check HTTP Code",
+      "description": "Default HTTP code to check against (default: 200).",
+      "type": "string"
+    },
+    "check-frontend-ip": {
+      "default": "",
+      "title": "Default space separated list of Frontend IPs to check",
+      "description": "Default list of Frontend IPs to check, if empty no constraint is used.",
+      "type": "string"
+    }
+  }
+}

software/monitor/instance-edgetest-slave-input-schema.json

+{
+  "$schema": "http://json-schema.org/draft-04/schema#",
+  "properties": {
+    "url": {
+      "title": "URL to check",
+      "description": "URL to check, like https://example.com",
+      "type": "string"
+    },
+    "check-status-code": {
+      "default": "Master default",
+      "title": "Default Check HTTP Code.",
+      "description": "HTTP code to check against (default: comes from master partition).",
+      "type": "string"
+    },
+    "check-frontend-ip": {
+      "default": "Master default",
+      "title": "Space separated list of Frontend IPs to check",
+      "description": "List of Frontend IPs to check, if empty no constraint is used (default: comes from master partition).",
+      "type": "string"
+    }
+  }
+}

software/monitor/instance-monitor-distributor.cfg.jinja2

-
-[jinja2-template-base]
-recipe = slapos.recipe.template:jinja2
-rendered = ${buildout:directory}/${:filename}
-extra-context =
-context =
-    import json_module json
-    ${:extra-context}
-
-[slave-test-configuration]
-<=jinja2-template-base
-template = {{ template_json_distributor_test }}
-filename = srv/monitor/private/test.json
-extensions = jinja2.ext.do 
-extra-context = 
-  section slave_information slap-configuration
-
-{% set part_list = [] -%}
-# Publish information for each slave
-
-{% set directory_list = [] -%}
-
-{% for slave_instance in slave_instance_list -%}
-{%   set publish_section_title = 'publish-%s' % slave_instance.get('slave_reference')  -%}
-{%   do part_list.append(publish_section_title) -%}
-[{{ publish_section_title }}]
-recipe = slapos.cookbook:publish
--slave-reference = {{ slave_instance.get('slave_reference') }}
-log-access-url = ${monitor-frontend:connection-site_url}/{{ slave_instance.get('slave_reference') }}
-log-access-url-v6 = ${monitor-httpd-conf-parameter:url}/{{ slave_instance.get('slave_reference') }}
-{% endfor %}
-
-
-{% set data_source_dict = slapparameter_dict.get('data-source', None)  -%}
-{% set cron_min_count = 0 -%}
-{% if data_source_dict %}
-{% for entry in data_source_dict -%}
-{% set cron_min_count = cron_min_count + 1 -%}
-{% set cron_min = cron_min_count%60 -%}
-{% do part_list.append('cron-crawl-' + entry) -%}
-{% do directory_list.append(entry) -%}
-[cron-crawl-{{ entry }}]
-<= cron
-recipe = slapos.cookbook:cron.d
-name = cron-crawler-{{ entry }}
-frequency = * * * * *
-command = cd ${monitor-directory:crawl-log}/{{ entry }} && ${crawler-bin:wrapper-path} {{ data_source_dict.get(entry) }} 
-{% endfor %}
-{% endif %}
-
-[monitor-directory]
-crawl-log = ${:srv}/crawlog
-network-user-logs = ${:private}/network-user-logs/
-{% for slave_instance in slave_instance_list -%}
-user-log-{{ slave_instance.get('slave_reference') }}-folder = ${:private}/network-user-logs/{{ slave_instance.get('slave_reference') }}
-user-log-{{ slave_instance.get('slave_reference') }}-ping-folder = ${:private}/network-user-logs/{{ slave_instance.get('slave_reference') }}/ping
-user-log-{{ slave_instance.get('slave_reference') }}-ping6-folder = ${:private}/network-user-logs/{{ slave_instance.get('slave_reference') }}/ping6
-
-{% endfor -%}
-{% for directory in directory_list %}
-{{ '%s = ${:crawl-log}/%s' % (directory, directory) }}
-{% endfor %}
-
-
-[crawler-bin]
-recipe = slapos.cookbook:wrapper
-command-line = 
-  {{ wget_bin }} --no-check-certificate -l1 -r -nd --timestamp  
-wrapper-path = ${monitor-directory:bin}/log-crawler
-
-[buildout]
-extends = {{ instance_base_monitor }}
-parts +=
-  slave-test-configuration
-{% for part in part_list %}
-{{ '  %s' % part }}
-{% endfor %}
-

software/monitor/instance-monitor-edgebot.cfg.jinja2

+{%- if slap_software_type == software_type %}
+{%- set CONFIGURATION = {} %}
+{%- for k, v in sorted(slap_configuration.items()) %}
+{%-   if k.startswith('configuration.') %}
+{%-     do CONFIGURATION.__setitem__(k[14:], v) %}
+{%-   endif %}
+{%- endfor %}
+{%- set slave_instance_list = [] %}
+{%- set extra_slave_instance_list = slapparameter_dict.get('extra_slave_instance_list') %}
+{%- if extra_slave_instance_list %}
+{#- Create slaves to process with setting up defaults #}
+{%-   for slave in sorted(json_module.loads(extra_slave_instance_list)) %}
+{%-   if 'check-status-code' not in slave %}
+{%-     do slave.__setitem__('check-status-code', CONFIGURATION['check-status-code']) %}
+{%-   endif %}
+{%-   if 'check-frontend-ip' not in slave %}
+{%-     do slave.__setitem__('check-frontend-ip', CONFIGURATION['check-frontend-ip']) %}
+{%-   endif %}
+{%-   if 'url' in slave %}
+{%-     do slave_instance_list.append(slave) %}
+{%-   endif %}
+{%-   endfor %}
+{%- endif %}
+{%- set part_list = [] %}
+{%- for slave in sorted(slave_instance_list) %}
+{%-   set part_id = 'http-query-' ~ slave['slave_reference'] ~ '-promise' %}
+{%-   do part_list.append(part_id) %}
+{%-   set safe_name = part_id.replace('_', '').replace('.', '-').replace(' ', '-') %}
+[{{part_id}}]
+<= monitor-promise-base
+module = check_surykatka_json
+name = {{ safe_name }}.py
+config-report = http_query
+config-url = {{ slave['url'] }}
+config-status-code = {{ slave['check-status-code'] }}
+config-ip-list = {{ slave['check-frontend-ip'] }}
+config-json-file = ${surykatka-config:json}
+{% endfor %}
+
+[surykatka-bot-promise]
+<= monitor-promise-base
+module = check_surykatka_json
+name = surykatka-bot-promise.py
+config-report = bot_status
+config-json-file = ${surykatka-config:json}
+
+[buildout]
+extends = {{ monitor_template_output }}
+
+parts =
+  cron
+  cron-entry-surykatka-status
+  monitor-base
+  publish-connection-information
+  surykatka
+  surykatka-bot-promise
+{% for part_id in sorted(part_list) %}
+  {{ part_id }}
+{% endfor %}
+
+eggs-directory = {{ eggs_directory }}
+develop-eggs-directory = {{ develop_eggs_directory }}
+offline = true
+
+[surykatka-config]
+recipe = slapos.recipe.template:jinja2
+db = ${directory:srv}/surykatka.db
+rendered = ${directory:etc}/surykatka.ini
+template = {{ template_surykatka_ini }}
+slave_instance_list = {{ dumps(slave_instance_list) }}
+nameserver = {{ dumps(CONFIGURATION['nameserver']) }}
+json = ${directory:srv}/surykatka.json
+
+context =
+  import json_module json
+  key db :db
+  key nameserver :nameserver
+  key slave_instance_list :slave_instance_list
+
+[surykatka]
+recipe = slapos.cookbook:wrapper
+config = ${surykatka-config:rendered}
+command-line =
+  {{ surykatka_binary }} --run crawl --reload --configuration ${:config}
+wrapper-path = ${monitor-directory:service}/${:_buildout_section_name_}
+hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
+
+[surykatka-status-json]
+recipe = slapos.recipe.template:jinja2
+template = inline:#!/bin/sh
+  if {{ surykatka_binary }} --run status --configuration ${surykatka:config} --output json > ${surykatka-config:json}.tmp ; then
+    mv -f ${surykatka-config:json}.tmp ${surykatka-config:json}
+  else
+    rm -f ${surykatka-config:json}.tmp
+  fi
+rendered = ${monitor-directory:bin}/${:_buildout_section_name_}
+mode = 0755
+
+[cron-entry-surykatka-status]
+recipe = slapos.cookbook:cron.d
+cron-entries = ${directory:etc}/cron.d
+name = surykatka-status
+frequency = */2 * * * *
+command = ${surykatka-status-json:rendered}
+
+[publish-connection-information]
+recipe = slapos.cookbook:publish.serialised
+monitor-base-url = ${monitor-publish-parameters:monitor-base-url}
+monitor-url = ${monitor-publish-parameters:monitor-url}
+monitor-user = ${monitor-publish-parameters:monitor-user}
+monitor-password = ${monitor-publish-parameters:monitor-password}
+
+[monitor-instance-parameter]
+monitor-httpd-port = {{ slapparameter_dict['monitor-httpd-port'] }}
+cors-domains = {{ slapparameter_dict.get('monitor-cors-domains', 'monitor.app.officejs.com') }}
+{% if slapparameter_dict.get('monitor-username', '') -%}
+username = {{ slapparameter_dict['monitor-username'] }}
+{% endif -%}
+{% if slapparameter_dict.get('monitor-password', '') -%}
+password = {{ slapparameter_dict['monitor-password'] }}
+{% endif -%}
+interface-url = {{ slapparameter_dict.get('monitor-interface-url', 'https://monitor.app.officejs.com') }}
+
+[monitor-directory]
+service = ${buildout:directory}/etc/service
+var = ${buildout:directory}/var
+srv = ${buildout:directory}/srv
+server-log = ${:private}/server-log
+monitor-log = ${:private}/monitor-log
+
+[slap-configuration]
+recipe = slapos.cookbook:slapconfiguration.serialised
+computer = ${slap-connection:computer-id}
+partition = ${slap-connection:partition-id}
+url = ${slap-connection:server-url}
+key = ${slap-connection:key-file}
+cert = ${slap-connection:cert-file}
+
+[slap-parameter]
+{%- endif %}

software/monitor/instance-monitor-edgetest.cfg.jinja2

+{%- if slap_software_type == software_type %}
+
+[jinja2-template-base]
+recipe = slapos.recipe.template:jinja2
+rendered = ${buildout:directory}/${:filename}
+extra-context =
+context =
+    import json_module json
+    ${:extra-context}
+
+[slave-test-configuration]
+<=jinja2-template-base
+template = {{ template_json_edgetest_test }}
+filename = srv/monitor/private/test.json
+extensions = jinja2.ext.do 
+extra-context = 
+  section slave_information slap-configuration
+
+{% set part_list = [] -%}
+# Publish information for each slave
+{%- set edgebot_software_type = 'edgebot' %}
+{%- set edgebot_quantity = slapparameter_dict.pop('edgebot-quantity', '1') | int %}
+{%- set edgebot_list = [] %}
+{%- set edgebot_section_list = [] %}
+{%- set slave_list_name = 'extra_slave_instance_list' %}
+{%- set request_dict = {} %}
+{%- set namebase = "edgebot" %}
+{%- set authorized_slave_list = [] %}
+{%- set monitor_base_url_dict = {} -%}
+{%- for slave in sorted(slave_instance_list) %}
+{%-   do authorized_slave_list.append(slave) %}
+{%- endfor %}
+{%- set monitor_base_port = int(slap_configuration['configuration.monitor-base-port']) %}
+{%- for i in range(1, edgebot_quantity + 1) %}
+{%-   set edgebot_name = "%s-%s" % (namebase, i) %}
+{%-   set request_section_title = 'request-%s' % edgebot_name %}
+{%-   do edgebot_list.append(edgebot_name) %}
+{%-   do edgebot_section_list.append(request_section_title) %}
+{%-   do part_list.append(request_section_title) %}
+{%-   do request_dict.__setitem__(request_section_title,
+                                  {
+                                  'config': {'monitor-httpd-port': monitor_base_port + i},
+                                  'name': edgebot_name,
+                                  'sla': {},
+                                  'state': 'started', 
+                                  }) %}
+{%- endfor %}
+
+[replicate]
+<= slap-connection
+recipe = slapos.cookbook:request.serialised
+config-monitor-cors-domains = {{ slapparameter_dict.get('monitor-cors-domains', 'monitor.app.officejs.com') }}
+config-monitor-username = ${monitor-instance-parameter:username}
+config-monitor-password = ${monitor-htpasswd:passwd}
+software-url = ${slap-connection:software-release-url}
+software-type = {{edgebot_software_type}}
+return = monitor-base-url
+
+{% for section, edgebot_request in request_dict.iteritems() %}
+[{{section}}]
+<= replicate
+name = {{ edgebot_request.get('name') }}
+{%-   if edgebot_request.get('state') %}
+state = {{ edgebot_request.get('state') }}
+{%-   endif%}
+{%-   set slave_configuration_dict = slapparameter_dict %}
+{%-   do slave_configuration_dict.update(edgebot_request.get('config')) %}
+{%-   do slave_configuration_dict.__setitem__(slave_list_name, json_module.dumps(authorized_slave_list)) %}
+{%-   for config_key, config_value in slave_configuration_dict.iteritems() %}
+config-{{ config_key }} = {{ dumps(config_value) }}
+{%   endfor -%}
+{%-   if edgebot_request.get('sla') %}
+{%-     for parameter, value in edgebot_request.get('sla').iteritems() %}
+sla-{{ parameter }} = {{ value }}
+{%-     endfor %}
+{%-   else %}
+# As no SLA was provided, by default it is requested on the same computer
+sla-computer_guid = ${slap-connection:computer-id}
+{%   endif %}
+{%-  do monitor_base_url_dict.__setitem__(section, '${' ~ section ~ ':connection-monitor-base-url}') -%}
+{%- endfor %}
+{%- set directory_list = [] -%}
+{%- for slave_instance in slave_instance_list -%}
+{%-   set publish_section_title = 'publish-%s' % slave_instance.get('slave_reference')  -%}
+{%-   do part_list.append(publish_section_title) %}
+[{{ publish_section_title }}]
+recipe = slapos.cookbook:publish
+-slave-reference = {{ slave_instance.get('slave_reference') }}
+{% endfor %}
+
+[monitor-conf-parameters]
+monitor-title = Monitor
+password = ${monitor-htpasswd:passwd}
+
+[monitor-base-url-dict]
+{% for key, value in monitor_base_url_dict.items() -%}
+{{ key }} = {{ value }}
+{% endfor %}
+
+[buildout]
+extends = {{ instance_base_monitor }}
+parts +=
+  slave-test-configuration
+{% for part in part_list %}
+{{ '  %s' % part }}
+{%- endfor %}
+{%- endif %}

software/monitor/instance-monitor.cfg.jinja2

@@ -57,14 +57,15 @@ recipe = slapos.cookbook:generate.password
 user = admin
 bytes = 16
 
+[monitor-instance-parameter]
+monitor-httpd-port = {{ slap_configuration['configuration.monitor-base-port'] }}
+
 [monitor-directory]
 service = ${buildout:directory}/etc/service
 var = ${buildout:directory}/var
 srv = ${buildout:directory}/srv
 server-log = ${:private}/server-log
 monitor-log = ${:private}/monitor-log
-cache = ${:var}/cache
-mod-ssl = ${:cache}/httpd_mod_ssl
 system-log = ${:private}/system-log
 consumption = ${:log}/consumption
 

software/monitor/instance.cfg

@@ -8,38 +8,65 @@ develop-eggs-directory = ${buildout:develop-eggs-directory}
 [switch_softwaretype]
 recipe = slapos.cookbook:softwaretype
 default = $${instance-base-monitor:rendered}
-distributor = $${instance-base-distributor:rendered}
+edgetest = $${instance-base-edgetest:rendered}
+edgebot = $${instance-base-edgebot:rendered}
 
 [instance-base-monitor]
 recipe = slapos.recipe.template:jinja2
-template = ${template-monitor:destination}
+template = ${template-monitor:target}
 rendered = $${buildout:directory}/template-base-monitor.cfg
 extensions = jinja2.ext.do
 context = key develop_eggs_directory buildout:develop-eggs-directory
           key eggs_directory buildout:eggs-directory
           key slapparameter_dict slap-configuration:configuration
+          section slap_configuration slap-configuration
           raw buildout_bin ${buildout:bin-directory}
           raw monitor_template_output ${monitor-template:output}
           raw network_benck_cfg_output ${network-bench-cfg:output}
           raw monitor_collect_csv_dump ${monitor-collect-csv-dump:output}
 mode = 0644
 
-[instance-base-distributor]
+[instance-base-edgetest]
 recipe = slapos.recipe.template:jinja2
-template = ${template-monitor-distributor:destination}
-rendered = $${buildout:directory}/template-monitor-base-distributor.cfg
+template = ${template-monitor-edgetest:target}
+rendered = $${buildout:directory}/template-monitor-base-edgetest.cfg
 extensions = jinja2.ext.do
 context = import json_module json
           key develop_eggs_directory buildout:develop-eggs-directory
           key eggs_directory buildout:eggs-directory
           key slapparameter_dict slap-configuration:configuration
+          key slap_software_type slap-configuration:slap-software-type
+          section slap_configuration slap-configuration
+          raw software_type edgetest
           key instance_base_monitor instance-base-monitor:rendered
           key slave_instance_list slap-configuration:slave-instance-list 
           raw buildout_bin ${buildout:bin-directory}
-          raw template_json_distributor_test ${json-test-template:destination}
-          raw wget_bin ${wget:location}/bin/wget
+          raw template_json_edgetest_test ${json-test-template:target}
 mode = 0644
 
+[instance-base-edgebot]
+recipe = slapos.recipe.template:jinja2
+template = ${template-monitor-edgebot:target}
+rendered = $${buildout:directory}/template-monitor-edgebot.cfg
+extensions = jinja2.ext.do
+surykatka-binary = ${surykatka:executable}
+template-surykatka-ini = ${template-surykatka-ini:target}
+
+context = import json_module json
+          key develop_eggs_directory buildout:develop-eggs-directory
+          key eggs_directory buildout:eggs-directory
+          section slap_configuration slap-configuration
+          key slapparameter_dict slap-configuration:configuration
+          key slap_software_type slap-configuration:slap-software-type
+          raw software_type edgebot
+          key surykatka_binary :surykatka-binary
+          key template_surykatka_ini :template-surykatka-ini
+          raw buildout_bin ${buildout:bin-directory}
+          raw monitor_template_output ${monitor-template:output}
+          raw monitor_collect_csv_dump ${monitor-collect-csv-dump:output}
+mode = 0644
+
+
 [slap-configuration]
 recipe = slapos.cookbook:slapconfiguration.serialised
 computer = $${slap-connection:computer-id}
@@ -47,3 +74,12 @@ partition = $${slap-connection:partition-id}
 url = $${slap-connection:server-url}
 key = $${slap-connection:key-file}
 cert = $${slap-connection:cert-file}
+# Defaults
+configuration.check-status-code = 200
+configuration.nameserver =
+configuration.check-frontend-ip =
+# use monitor-base-port to have monitor listening on each instance
+# on different port and also on different port than other services
+# it makes it possible to instantiate it correctly on signle IP, for
+# example in case of webrunner
+configuration.monitor-base-port = 9700

software/monitor/software.cfg

@@ -4,17 +4,16 @@ extends =
   buildout.hash.cfg
   ../../component/pycurl/buildout.cfg
   ../../component/python-cryptography/buildout.cfg
-  ../../component/wget/buildout.cfg
+  ../../component/surykatka/buildout.cfg
   ../../stack/monitor/buildout.cfg
   ../../stack/slapos.cfg
 
 parts = 
-  wget
   slapos-cookbook
   network-bench-cfg
   json-test-template
   template
-  template-monitor-distributor
+  template-monitor-edgetest
   template-monitor
   monitor-collect-csv-dump
 
@@ -27,26 +26,31 @@ mode = 0644
 [template-monitor]
 recipe = slapos.recipe.build:download 
 url = ${:_profile_base_location_}/${:_update_hash_filename_}
-destination = ${buildout:directory}/template-base-monitor.cfg
 mode = 0644
 
-[template-monitor-distributor]
+[template-monitor-edgetest]
 recipe = slapos.recipe.build:download
 url = ${:_profile_base_location_}/${:_update_hash_filename_}
-destination = ${buildout:directory}/template-monitor-base-distributor.cfg
 mode = 0644
 
+[template-monitor-edgebot]
+recipe = slapos.recipe.build:download
+url = ${:_profile_base_location_}/${:_update_hash_filename_}
+mode = 0644
+
+[template-surykatka-ini]
+recipe = slapos.recipe.build:download
+url = ${:_profile_base_location_}/${:_update_hash_filename_}
+
 [json-test-template]
 recipe = slapos.recipe.build:download
 url = ${:_profile_base_location_}/${:_update_hash_filename_}
-destination = ${buildout:directory}/json-test-template.json.in.jinja2
 mode = 0644
 
 [network-bench-cfg]
 recipe = slapos.recipe.template
-url = ${:_profile_base_location_}/network_bench.cfg.in
-md5sum  = cfcbf2002b8eff5153e2bf68ed24b720
-output = ${buildout:directory}/template-network-bench-cfg.in
+url = ${:_profile_base_location_}/${:filename}
+output = ${buildout:parts-directory}/${:_buildout_section_name_}
 mode = 0644
 
 [monitor-collect-csv-dump]
@@ -54,7 +58,6 @@ mode = 0644
 url = ${:_profile_base_location_}/script/${:filename}
 filename = collect_csv_dump.py
 output = ${:destination}/${:filename}
-md5sum = cad2402bbd21907cfed6bc5af8c5d3ab
 
 [extra-eggs]
 <= monitor-eggs
@@ -83,6 +86,5 @@ eggs +=
   hexagonit.recipe.download
   plone.recipe.command
 
-
 [versions]
 slapos.recipe.template = 4.3

software/monitor/software.cfg.json

+{
+  "name": "Monitor",
+  "description": "Software release for Monitoring purpose",
+  "serialisation": "xml",
+  "software-type": {
+    "default": {
+      "title": "Default",
+      "description": "Standalone Monitor",
+      "request": "instance-default-input-schema.json",
+      "response": "instance-default-output-schema.json",
+      "index": 0
+    },
+    "edgetest": {
+      "title": "Edge Test",
+      "description": "Cluster of bots to perform a distributed monitoring ",
+      "request": "instance-edgetest-input-schema.json",
+      "response": "instance-default-output-schema.json",
+      "index": 1
+    },
+    "edgetest-slave": {
+      "title": "Edge Test Slave",
+      "software-type": "edgetest",
+      "description": "Cluster of bots to perform a distributed monitoring ",
+      "request": "instance-edgetest-slave-input-schema.json",
+      "response": "instance-default-output-schema.json",
+      "index": 2
+    }
+  }
+}

software/monitor/surykatka.ini.jinja2

+[SURYKATKA]
+INTERVAL = 120
+SQLITE = {{ db }}
+{%- set nameserver_list = nameserver.split() %}
+{%- if len(nameserver_list) > 0 %}
+NAMESERVER =
+{%-   for nameserver_entry in sorted(nameserver_list) %}
+  {{ nameserver_entry }}
+{%-   endfor %}
+{% endif %}
+URL =
+{%- for slave in sorted(slave_instance_list) %}
+{%-   if 'url' in slave %}
+  {{ slave['url'] }}
+{%-   endif -%}
+{% endfor %}
+

software/monitor/test/test.py

@@ -25,10 +25,12 @@
 #
 ##############################################################################
 
+import json
 import os
-
+import re
+import requests
+import xml.etree.ElementTree as ET
 from slapos.recipe.librecipe import generateHashFromFiles
-
 from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
 
 setUpModule, SlapOSInstanceTestCase = makeModuleSetUpAndTestCaseClass(
@@ -59,3 +61,346 @@ class ServicesTestCase(SlapOSInstanceTestCase):
       expected_process_name = name.format(hash=h)
 
       self.assertIn(expected_process_name, process_names)
+
+
+class MonitorTestMixin(object):
+  monitor_setup_url_key = 'monitor-setup-url'
+
+  def test_monitor_setup(self):
+    connection_parameter_dict = self\
+      .computer_partition.getConnectionParameterDict()
+    self.assertTrue(
+      self.monitor_setup_url_key in connection_parameter_dict,
+      '%s not in %s' % (self.monitor_setup_url_key, connection_parameter_dict))
+    monitor_setup_url_value = connection_parameter_dict[
+      self.monitor_setup_url_key]
+    monitor_url_match = re.match(r'.*url=(.*)', monitor_setup_url_value)
+    self.assertNotEqual(
+      None, monitor_url_match, '%s not parsable' % (monitor_setup_url_value,))
+    self.assertEqual(1, len(monitor_url_match.groups()))
+    monitor_url = monitor_url_match.groups()[0]
+    monitor_url_split = monitor_url.split('&')
+    self.assertEqual(
+      3, len(monitor_url_split), '%s not splitabble' % (monitor_url,))
+    self.monitor_url = monitor_url_split[0]
+    monitor_username = monitor_url_split[1].split('=')
+    self.assertEqual(
+      2, len(monitor_username), '%s not splittable' % (monitor_username))
+    monitor_password = monitor_url_split[2].split('=')
+    self.assertEqual(
+      2, len(monitor_password), '%s not splittable' % (monitor_password))
+    self.monitor_username = monitor_username[1]
+    self.monitor_password = monitor_password[1]
+
+    opml_text = requests.get(self.monitor_url, verify=False).text
+    opml = ET.fromstring(opml_text)
+
+    body = opml[1]
+    self.assertEqual('body', body.tag)
+
+    outline_list = body[0].findall('outline')
+
+    self.assertEqual(
+      self.monitor_configuration_list,
+      [q.attrib for q in outline_list]
+    )
+
+    expected_status_code_list = []
+    got_status_code_list = []
+    for monitor_configuration in self.monitor_configuration_list:
+      status_code = requests.get(
+          monitor_configuration['url'],
+          verify=False,
+          auth=(self.monitor_username, self.monitor_password)
+        ).status_code
+      expected_status_code_list.append(
+        {
+          'url': monitor_configuration['url'],
+          'status_code': 200
+        }
+      )
+      got_status_code_list.append(
+        {
+          'url': monitor_configuration['url'],
+          'status_code': status_code
+        }
+      )
+    self.assertEqual(
+      expected_status_code_list,
+      got_status_code_list
+    )
+
+
+class EdgeSlaveMixin(MonitorTestMixin):
+  __partition_reference__ = 'edge'
+  instance_max_retry = 20
+
+  @classmethod
+  def getInstanceSoftwareType(cls):
+    return 'edgetest'
+
+  def requestEdgetestSlave(self, partition_reference, partition_parameter_kw):
+    software_url = self.getSoftwareURL()
+    self.slap.request(
+      software_release=software_url,
+      software_type='edgetest',
+      partition_reference=partition_reference,
+      partition_parameter_kw=partition_parameter_kw,
+      shared=True
+    )
+
+  def setUp(self):
+    self.bot_partition_path = os.path.join(
+      self.slap.instance_directory,
+      self.__partition_reference__ + '1')
+    self.surykatka_json = os.path.join(
+        self.bot_partition_path, 'srv', 'surykatka.json')
+    self.surykatka_status_json = os.path.join(
+        self.bot_partition_path, 'bin', 'surykatka-status-json')
+    self.monitor_configuration_list = [
+      {
+        'xmlUrl': 'https://[%s]:9700/public/feed' % (self._ipv6_address,),
+        'version': 'RSS',
+        'title': 'testing partition 0',
+        'url': 'https://[%s]:9700/share/private/' % (self._ipv6_address,),
+        'text': 'testing partition 0',
+        'type': 'rss',
+        'htmlUrl': 'https://[%s]:9700/public/feed' % (self._ipv6_address,)
+      },
+      {
+        'xmlUrl': 'https://[%s]:9701/public/feed' % (self._ipv6_address,),
+        'version': 'RSS',
+        'title': 'edgebot-1',
+        'url': 'https://[%s]:9701/share/private/' % (self._ipv6_address,),
+        'text': 'edgebot-1',
+        'type': 'rss',
+        'htmlUrl': 'https://[%s]:9701/public/feed' % (self._ipv6_address,)
+      }
+    ]
+
+  def assertSurykatkaIni(self):
+    surykatka_ini = open(
+      os.path.join(
+        self.bot_partition_path, 'etc', 'surykatka.ini')).read().strip()
+
+    expected = self.surykatka_ini % dict(
+        partition_path=self.bot_partition_path)
+    self.assertEqual(
+      expected.strip(),
+      surykatka_ini)
+
+  def assertPromiseContent(self, name, content):
+    promise = open(
+      os.path.join(
+        self.bot_partition_path, 'etc', 'plugin', name
+      )).read().strip()
+
+    self.assertTrue(content in promise)
+
+  def assertSurykatkaBotPromise(self):
+    self.assertPromiseContent(
+      'surykatka-bot-promise.py',
+      "'report': 'bot_status'")
+    self.assertPromiseContent(
+      'surykatka-bot-promise.py',
+      "'json-file': '%s'" % (self.surykatka_json,)
+    )
+
+  def assertSurykatkaCron(self):
+    surykatka_cron = open(
+      os.path.join(
+        self.bot_partition_path, 'etc', 'cron.d', 'surykatka-status')
+        ).read().strip()
+    self.assertEqual(
+      '*/2 * * * * %s' % (self.surykatka_status_json,),
+      surykatka_cron
+    )
+
+  def initiateSurykatkaRun(self):
+    try:
+      self.slap.waitForInstance(max_retry=2)
+    except Exception:
+      pass
+
+  def assertSurykatkaStatusJSON(self):
+    if os.path.exists(self.surykatka_json):
+      os.unlink(self.surykatka_json)
+    self.assertEqual(0, os.system(self.surykatka_status_json))
+    self.assertTrue(os.path.exists(self.surykatka_json))
+    with open(self.surykatka_json) as fh:
+      status_json = json.load(fh)
+    self.assertTrue('bot_status' in status_json)
+
+  def test(self):
+    # Note: Those tests do not run surykatka and do not do real checks, as
+    #       this depends too much on the environment and is really hard to
+    #       mock
+    #       So it is possible that some bugs might slip under the radar
+    #       Nevertheless the surykatka and check_surykatka_json are heavily
+    #       unit tested, and configuration created by the profiles is asserted
+    #       here, so it shall be enough as reasonable status
+    self.requestEdgetestSlaves()
+    self.initiateSurykatkaRun()
+    self.assertSurykatkaStatusJSON()
+    self.assertSurykatkaIni()
+    self.assertSurykatkaBotPromise()
+    self.assertSurykatkaPromises()
+    self.assertSurykatkaCron()
+
+
+class TestEdge(EdgeSlaveMixin, SlapOSInstanceTestCase):
+  surykatka_ini = """[SURYKATKA]
+INTERVAL = 120
+SQLITE = %(partition_path)s/srv/surykatka.db
+URL =
+  https://www.erp5.com/
+  https://www.erp5.org/"""
+
+  def assertSurykatkaPromises(self):
+    self.assertPromiseContent(
+      'http-query-backend-300-promise.py',
+      "'ip-list': ''")
+    self.assertPromiseContent(
+      'http-query-backend-300-promise.py',
+      "'report': 'http_query'")
+    self.assertPromiseContent(
+      'http-query-backend-300-promise.py',
+      "'status-code': '300'")
+    self.assertPromiseContent(
+      'http-query-backend-300-promise.py',
+      "'url': 'https://www.erp5.org/'")
+    self.assertPromiseContent(
+      'http-query-backend-300-promise.py',
+      "'json-file': '%s'" % (self.surykatka_json,)
+    )
+
+    self.assertPromiseContent(
+      'http-query-backend-promise.py',
+      "'ip-list': ''")
+    self.assertPromiseContent(
+      'http-query-backend-promise.py',
+      "'report': 'http_query'")
+    self.assertPromiseContent(
+      'http-query-backend-promise.py',
+      "'status-code': '200'")
+    self.assertPromiseContent(
+      'http-query-backend-promise.py',
+      "'url': 'https://www.erp5.com/'")
+    self.assertPromiseContent(
+      'http-query-backend-promise.py',
+      "'json-file': '%s'" % (self.surykatka_json,)
+    )
+
+  def requestEdgetestSlaves(self):
+    self.requestEdgetestSlave(
+      'backend',
+      {'url': 'https://www.erp5.com/'},
+    )
+    self.requestEdgetestSlave(
+      'backend-300',
+      {'url': 'https://www.erp5.org/', 'check-status-code': '300'},
+    )
+
+
+class TestEdgeNameserverCheckFrontendIp(
+  EdgeSlaveMixin, SlapOSInstanceTestCase):
+  surykatka_ini = """[SURYKATKA]
+INTERVAL = 120
+SQLITE = %(partition_path)s/srv/surykatka.db
+NAMESERVER =
+  127.0.1.1
+  127.0.1.2
+
+URL =
+  https://www.erp5.com/"""
+
+  @classmethod
+  def getInstanceParameterDict(cls):
+    return {
+      'nameserver': '127.0.1.1 127.0.1.2',
+      'check-frontend-ip': '127.0.0.1 127.0.0.2',
+    }
+
+  def assertSurykatkaPromises(self):
+    self.assertPromiseContent(
+      'http-query-backend-promise.py',
+      "'ip-list': '127.0.0.1 127.0.0.2'")
+    self.assertPromiseContent(
+      'http-query-backend-promise.py',
+      "'report': 'http_query'")
+    self.assertPromiseContent(
+      'http-query-backend-promise.py',
+      "'status-code': '200'")
+    self.assertPromiseContent(
+      'http-query-backend-promise.py',
+      "'url': 'https://www.erp5.com/'")
+    self.assertPromiseContent(
+      'http-query-backend-promise.py',
+      "'json-file': '%s'" % (self.surykatka_json,)
+    )
+
+  def requestEdgetestSlaves(self):
+    self.requestEdgetestSlave(
+      'backend',
+      {'url': 'https://www.erp5.com/'},
+    )
+
+
+class TestEdgeCheckStatusCode(EdgeSlaveMixin, SlapOSInstanceTestCase):
+  surykatka_ini = """[SURYKATKA]
+INTERVAL = 120
+SQLITE = %(partition_path)s/srv/surykatka.db
+URL =
+  https://www.erp5.com/
+  https://www.erp5.org/"""
+
+  @classmethod
+  def getInstanceParameterDict(cls):
+    return {
+      'check-status-code': '500',
+    }
+
+  def assertSurykatkaPromises(self):
+    self.assertPromiseContent(
+      'http-query-backend-501-promise.py',
+      "'ip-list': ''")
+    self.assertPromiseContent(
+      'http-query-backend-501-promise.py',
+      "'report': 'http_query'")
+    self.assertPromiseContent(
+      'http-query-backend-501-promise.py',
+      "'status-code': '501'")
+    self.assertPromiseContent(
+      'http-query-backend-501-promise.py',
+      "'url': 'https://www.erp5.org/'")
+    self.assertPromiseContent(
+      'http-query-backend-501-promise.py',
+      "'json-file': '%s'" % (self.surykatka_json,)
+    )
+
+    self.assertPromiseContent(
+      'http-query-backend-promise.py',
+      "'ip-list': ''")
+    self.assertPromiseContent(
+      'http-query-backend-promise.py',
+      "'report': 'http_query'")
+    self.assertPromiseContent(
+      'http-query-backend-promise.py',
+      "'status-code': '500'")
+    self.assertPromiseContent(
+      'http-query-backend-promise.py',
+      "'url': 'https://www.erp5.com/'")
+    self.assertPromiseContent(
+      'http-query-backend-promise.py',
+      "'json-file': '%s'" % (self.surykatka_json,)
+    )
+
+  def requestEdgetestSlaves(self):
+    self.requestEdgetestSlave(
+      'backend',
+      {'url': 'https://www.erp5.com/'},
+    )
+    self.requestEdgetestSlave(
+      'backend-501',
+      {'url': 'https://www.erp5.org/', 'check-status-code': '501'},
+    )

stack/slapos.cfg

@@ -144,7 +144,7 @@ slapos.libnetworkcache = 0.20
 slapos.rebootstrap = 4.2
 slapos.recipe.build = 0.42
 slapos.recipe.cmmi = 0.12
-slapos.toolbox = 0.100
+slapos.toolbox = 0.101
 stevedore = 1.21.0
 subprocess32 = 3.5.3
 unicodecsv = 0.14.1