Edge Test

software/monitor/buildout.hash.cfg

+# THIS IS NOT A BUILDOUT FILE, despite purposedly using a compatible syntax.
+# The only allowed lines here are (regexes):
+# - "^#" comments, copied verbatim
+# - "^[" section beginings, copied verbatim
+# - lines containing an "=" sign which must fit in the following categorie.
+#   - "^\s*filename\s*=\s*path\s*$" where "path" is relative to this file
+#     Copied verbatim.
+#   - "^\s*hashtype\s*=.*" where "hashtype" is one of the values supported
+#     by the re-generation script.
+#     Re-generated.
+# - other lines are copied verbatim
+# Substitution (${...:...}), extension ([buildout] extends = ...) and
+# section inheritance (< = ...) are NOT supported (but you should really
+# not need these here).
+
+[template]
+filename = instance.cfg
+md5sum = 677f827c03309bdcecbc3c6405bd760c
+
+[template-monitor]
+filename = instance-monitor.cfg.jinja2
+md5sum = dffe797be3a66d6838ffc70ee8f5f3ff
+
+[template-monitor-edgetest]
+filename = instance-monitor-edgetest.cfg.jinja2
+md5sum = 4ff61df3f3823274d36c19521dc5a536
+
+[template-monitor-bot]
+filename = instance-monitor-bot.cfg.jinja2
+md5sum = b1ea718c10c9c9b56a86caa2d3320ed7
+
+[json-test-template]
+filename = json-test-template.json.in.jinja2
+md5sum = 2eb5596544d9c341acf653d4f7ce2680
+
+[network-bench-cfg]
+filename = network_bench.cfg.in 
+md5sum = cfcbf2002b8eff5153e2bf68ed24b720
+
+[monitor-collect-csv-dump]
+filename = script/collect_csv_dump.py
+md5sum = cad2402bbd21907cfed6bc5af8c5d3ab

software/monitor/instance-default-input-schema.json

+{
+  "$schema": "http://json-schema.org/draft-04/schema#",
+  "properties": {
+    "dummy": {
+      "title": "dummy",
+      "description": "Dummy",
+      "type": "string"
+    }
+  }
+}

software/monitor/instance-default-output-schema.json

+{
+  "$schema": "http://json-schema.org/draft-04/schema#",
+  "description": "Values returned by Re6st Master instanciation",
+  "properties": {
+    "re6stry-url": {
+      "description": "ipv6 url to access your re6st registry service",
+      "type": "string"
+    }
+  },
+  "type": "object"
+}

software/monitor/instance-edgetest-input-schema.json

+{
+  "$schema": "http://json-schema.org/draft-04/schema#",
+  "properties": {
+    "dummy": {
+      "title": "dummy",
+      "description": "Dummy",
+      "type": "string"
+    }
+  }
+}

software/monitor/instance-edgetest-slave-input-schema.json

+{
+  "$schema": "http://json-schema.org/draft-04/schema#",
+  "properties": {
+    "dummy": {
+      "title": "dummy",
+      "description": "Dummy",
+      "type": "string"
+    }
+  }
+}

software/monitor/instance-monitor-bot.cfg.jinja2

+
+{% set slave_instance_list = [] %}
+{% set extra_slave_instance_list = slapparameter_dict.get('extra_slave_instance_list') %}
+{% if extra_slave_instance_list %}
+  {% set slave_instance_list = json_module.loads(extra_slave_instance_list) %}
+{% endif %}
+[buildout]
+
+extends = {{ monitor_template_output }}
+
+parts =
+  cron
+  monitor-base
+  publish-connection-information
+
+eggs-directory = {{ eggs_directory }}
+develop-eggs-directory = {{ develop_eggs_directory }}
+offline = true
+
+[publish-connection-information]
+recipe = slapos.cookbook:publish.serialised
+monitor-base-url = ${monitor-publish-parameters:monitor-base-url}
+monitor-url = ${monitor-publish-parameters:monitor-url}
+monitor-user = ${monitor-publish-parameters:monitor-user}
+monitor-password = ${monitor-publish-parameters:monitor-password}
+
+[monitor-instance-parameter]
+cors-domains = {{ slapparameter_dict.get('monitor-cors-domains', 'monitor.app.officejs.com') }}
+{% if slapparameter_dict.get('monitor-username', '') -%}
+username = {{ slapparameter_dict['monitor-username'] }}
+{% endif -%}
+{% if slapparameter_dict.get('monitor-password', '') -%}
+password = {{ slapparameter_dict['monitor-password'] }}
+{% endif -%}
+interface-url = {{ slapparameter_dict.get('monitor-interface-url', 'https://monitor.app.officejs.com') }}
+
+
+[monitor-directory]
+service = ${buildout:directory}/etc/service
+var = ${buildout:directory}/var
+srv = ${buildout:directory}/srv
+server-log = ${:private}/server-log
+monitor-log = ${:private}/monitor-log
+cache = ${:var}/cache
+mod-ssl = ${:cache}/httpd_mod_ssl
+
+[python-symlink]
+recipe = plone.recipe.command
+command = ln -sf {{ buildout_bin }}/pythonwitheggs ${monitor-directory:bin}/python
+update-command = ${:command}
+
+[slap-configuration]
+recipe = slapos.cookbook:slapconfiguration.serialised
+computer = ${slap-connection:computer-id}
+partition = ${slap-connection:partition-id}
+url = ${slap-connection:server-url}
+key = ${slap-connection:key-file}
+cert = ${slap-connection:cert-file}
+
+[slap-parameter]

software/monitor/instance-monitor-distributor.cfg.jinja2 → software/monitor/instance-monitor-edgetest.cfg.jinja2

@@ -9,7 +9,7 @@ context =
 
 [slave-test-configuration]
 <=jinja2-template-base
-template = {{ template_json_distributor_test }}
+template = {{ template_json_edgetest_test }}
 filename = srv/monitor/private/test.json
 extensions = jinja2.ext.do 
 extra-context = 
@@ -17,6 +17,66 @@ extra-context =
 
 {% set part_list = [] -%}
 # Publish information for each slave
+{% set bot_software_type = 'bot' %}
+{% set bot_quantity = slapparameter_dict.pop('bot-quantity', '1') | int %}
+{% set bot_list = [] %}
+{% set bot_section_list = [] %}
+{% set slave_list_name = 'extra_slave_instance_list' %}
+{% set request_dict = {} %}
+{% set namebase = "bot" %}
+{% set authorized_slave_list = [] %}
+{% set monitor_base_url_dict = {} -%}
+
+{% for slave in sorted(slave_instance_list) %}
+{%   do authorized_slave_list.append(slave) %}
+{% endfor %}
+
+
+{% for i in range(1, bot_quantity + 1) %}
+{%   set bot_name = "%s-%s" % (namebase, i) %}
+{%   set request_section_title = 'request-%s' % bot_name %}
+{%   do bot_list.append(bot_name) %}
+{%   do bot_section_list.append(request_section_title) %}
+{%   do part_list.append(request_section_title) %}
+{%   do request_dict.__setitem__(request_section_title,
+                                  {
+                                  'config': {},
+                                  'name': bot_name,
+                                  'sla': {},
+                                  'state': 'started', 
+                                  }) %}
+{% endfor %}
+
+[replicate]
+<= slap-connection
+recipe = slapos.cookbook:request.serialised
+config-monitor-cors-domains = {{ slapparameter_dict.get('monitor-cors-domains', 'monitor.app.officejs.com') }}
+config-monitor-username = ${monitor-instance-parameter:username}
+config-monitor-password = ${monitor-htpasswd:passwd}
+software-url = ${slap-connection:software-release-url}
+software-type = {{bot_software_type}}
+return = monitor-base-url
+
+{% for section, bot_request in request_dict.iteritems() %}
+[{{section}}]
+<= replicate
+name = {{ bot_request.get('name') }}
+{%   if bot_request.get('state') %}
+state = {{ bot_request.get('state') }}
+{%   endif%}
+{%   set slave_configuration_dict = slapparameter_dict %}
+{%   do slave_configuration_dict.update(bot_request.get('config')) %}
+{%   do slave_configuration_dict.__setitem__(slave_list_name, json_module.dumps(authorized_slave_list)) %}
+{%-   for config_key, config_value in slave_configuration_dict.iteritems() %}
+config-{{ config_key }} = {{ dumps(config_value) }}
+{%   endfor -%}
+{%   if bot_request.get('sla') %}
+{%     for parameter, value in bot_request.get('sla').iteritems() %}
+sla-{{ parameter }} = {{ value }}
+{%     endfor %}
+{%   endif %}
+{%  do monitor_base_url_dict.__setitem__(section, '${' ~ section ~ ':connection-monitor-base-url}') -%}
+{% endfor %}
 
 {% set directory_list = [] -%}
 
@@ -26,8 +86,6 @@ extra-context =
 [{{ publish_section_title }}]
 recipe = slapos.cookbook:publish
 -slave-reference = {{ slave_instance.get('slave_reference') }}
-log-access-url = ${monitor-frontend:connection-site_url}/{{ slave_instance.get('slave_reference') }}
-log-access-url-v6 = ${monitor-httpd-conf-parameter:url}/{{ slave_instance.get('slave_reference') }}
 {% endfor %}
 
 
@@ -61,6 +119,14 @@ user-log-{{ slave_instance.get('slave_reference') }}-ping6-folder = ${:private}/
 {{ '%s = ${:crawl-log}/%s' % (directory, directory) }}
 {% endfor %}
 
+[monitor-conf-parameters]
+monitor-title = Monitor
+password = ${monitor-htpasswd:passwd}
+
+[monitor-base-url-dict]
+{% for key, value in monitor_base_url_dict.items() -%}
+{{ key }} = {{ value }}
+{% endfor %}
 
 [crawler-bin]
 recipe = slapos.cookbook:wrapper

software/monitor/instance.cfg

@@ -8,11 +8,12 @@ develop-eggs-directory = ${buildout:develop-eggs-directory}
 [switch_softwaretype]
 recipe = slapos.cookbook:softwaretype
 default = $${instance-base-monitor:rendered}
-distributor = $${instance-base-distributor:rendered}
+edgetest = $${instance-base-edgetest:rendered}
+bot = $${instance-base-bot:rendered}
 
 [instance-base-monitor]
 recipe = slapos.recipe.template:jinja2
-template = ${template-monitor:destination}
+template = ${template-monitor:target}
 rendered = $${buildout:directory}/template-base-monitor.cfg
 extensions = jinja2.ext.do
 context = key develop_eggs_directory buildout:develop-eggs-directory
@@ -24,10 +25,10 @@ context = key develop_eggs_directory buildout:develop-eggs-directory
           raw monitor_collect_csv_dump ${monitor-collect-csv-dump:output}
 mode = 0644
 
-[instance-base-distributor]
+[instance-base-edgetest]
 recipe = slapos.recipe.template:jinja2
-template = ${template-monitor-distributor:destination}
-rendered = $${buildout:directory}/template-monitor-base-distributor.cfg
+template = ${template-monitor-edgetest:target}
+rendered = $${buildout:directory}/template-monitor-base-edgetest.cfg
 extensions = jinja2.ext.do
 context = import json_module json
           key develop_eggs_directory buildout:develop-eggs-directory
@@ -36,10 +37,25 @@ context = import json_module json
           key instance_base_monitor instance-base-monitor:rendered
           key slave_instance_list slap-configuration:slave-instance-list 
           raw buildout_bin ${buildout:bin-directory}
-          raw template_json_distributor_test ${json-test-template:destination}
+          raw template_json_edgetest_test ${json-test-template:target}
           raw wget_bin ${wget:location}/bin/wget
 mode = 0644
 
+[instance-base-bot]
+recipe = slapos.recipe.template:jinja2
+template = ${template-monitor-bot:target}
+rendered = $${buildout:directory}/template-monitor-bot.cfg
+extensions = jinja2.ext.do
+context = import json_module json
+          key develop_eggs_directory buildout:develop-eggs-directory
+          key eggs_directory buildout:eggs-directory
+          key slapparameter_dict slap-configuration:configuration
+          raw buildout_bin ${buildout:bin-directory}
+          raw monitor_template_output ${monitor-template:output}
+          raw monitor_collect_csv_dump ${monitor-collect-csv-dump:output}
+mode = 0644
+
+
 [slap-configuration]
 recipe = slapos.cookbook:slapconfiguration.serialised
 computer = $${slap-connection:computer-id}

software/monitor/software.cfg

 [buildout]
 
 extends =
+  buildout.hash.cfg
   ../../component/pycurl/buildout.cfg
   ../../component/python-cryptography/buildout.cfg
   ../../component/wget/buildout.cfg
@@ -13,7 +14,7 @@ parts =
   network-bench-cfg
   json-test-template
   template
-  template-monitor-distributor
+  template-monitor-edgetest
   template-monitor
   monitor-collect-csv-dump
 
@@ -21,35 +22,32 @@ parts =
 recipe = slapos.recipe.template 
 url = ${:_profile_base_location_}/instance.cfg
 output = ${buildout:directory}/template.cfg
-md5sum = 1b7d2d097f208f6641bf98a17df079c8
 mode = 0644
 
 [template-monitor]
 recipe = slapos.recipe.build:download 
-url = ${:_profile_base_location_}/instance-monitor.cfg.jinja2
-destination = ${buildout:directory}/template-base-monitor.cfg
-md5sum = dffe797be3a66d6838ffc70ee8f5f3ff
+url = ${:_profile_base_location_}/${:filename}
 mode = 0644
 
-[template-monitor-distributor]
+[template-monitor-edgetest]
 recipe = slapos.recipe.build:download
-url = ${:_profile_base_location_}/instance-monitor-distributor.cfg.jinja2
-destination = ${buildout:directory}/template-monitor-base-distributor.cfg
-md5sum = 61c0bfdfc0a2b51ba15fe4a49baf6091
+url = ${:_profile_base_location_}/${:filename}
+mode = 0644
+
+[template-monitor-bot]
+recipe = slapos.recipe.build:download
+url = ${:_profile_base_location_}/${:filename}
 mode = 0644
 
 [json-test-template]
 recipe = slapos.recipe.build:download
-url = ${:_profile_base_location_}/json-test-template.json.in.jinja2
-destination = ${buildout:directory}/json-test-template.json.in.jinja2
-md5sum = 2eb5596544d9c341acf653d4f7ce2680
+url = ${:_profile_base_location_}/${:filename}
 mode = 0644
 
 [network-bench-cfg]
 recipe = slapos.recipe.template
-url = ${:_profile_base_location_}/network_bench.cfg.in
-md5sum  = cfcbf2002b8eff5153e2bf68ed24b720
-output = ${buildout:directory}/template-network-bench-cfg.in
+url = ${:_profile_base_location_}/${:filename}
+output = ${buildout:parts-directory}/${:_buildout_section_name_}
 mode = 0644
 
 [monitor-collect-csv-dump]
@@ -57,7 +55,6 @@ mode = 0644
 url = ${:_profile_base_location_}/script/${:filename}
 filename = collect_csv_dump.py
 output = ${:destination}/${:filename}
-md5sum = cad2402bbd21907cfed6bc5af8c5d3ab
 
 [extra-eggs]
 <= monitor-eggs
@@ -86,6 +83,5 @@ eggs +=
   hexagonit.recipe.download
   plone.recipe.command
 
-
 [versions]
 slapos.recipe.template = 4.3

software/monitor/software.cfg.json

+{
+  "name": "Monitor",
+  "description": "Software release for Monitoring purpose",
+  "serialisation": "xml",
+  "software-type": {
+    "default": {
+      "title": "Default",
+      "description": "Standalone Monitor",
+      "request": "instance-default-input-schema.json",
+      "response": "instance-default-output-schema.json",
+      "index": 0
+    },
+    "edgetest": {
+      "title": "Edge Test",
+      "description": "Cluster of bots to perform a distributed monitoring ",
+      "request": "instance-edgetest-input-schema.json",
+      "response": "instance-default-output-schema.json",
+      "index": 1
+    },
+    "edgetest-slave": {
+      "title": "Edge Test Case",
+      "software-type": "edgetest",
+      "description": "Cluster of bots to perform a distributed monitoring ",
+      "request": "instance-edgetest-slave-input-schema.json",
+      "response": "instance-default-output-schema.json",
+      "index": 1
+    }
+  }
+}