monitor: Improve and fix monitor software release

  Drop fluentd as the instalation is not working well
  Stop rsync logs and use symlink
  Clean up installation
  Make network bench use parameters instead of a static test list.

software/monitor/instance.cfg.in → software/monitor/instance-monitor.cfg.jinja2

@@ -18,9 +18,10 @@ parts =
   cgi-httpd-graceful-wrapper
   monitor-promise
   monitor-instance-log-access
-  cron-rsync-logs
   cron-smart-monitor-logs
   cron-network-bench
+  symlink-re6st-logs
+  symlink-collected-logs
   
 extends = ${monitor-template:output}
 
@@ -29,13 +30,6 @@ eggs-directory = ${buildout:eggs-directory}
 develop-eggs-directory = ${buildout:develop-eggs-directory}
 offline = true
 
-[cron-rsync-logs]
-<= cron
-recipe = slapos.cookbook:cron.d
-name = rsync-logs
-frequency = * * * * *
-command = $${rsync-logs:rendered}
-
 [cron-smart-monitor-logs]
 <= cron
 recipe = slapos.cookbook:cron.d
@@ -47,20 +41,19 @@ command = $${smartmon-logs:rendered}
 <= cron
 recipe = slapos.cookbook:cron.d
 name = network-bench-test
-frequency = */3 * * * *
+frequency = */10 * * * *
 command = ${buildout:bin-directory}/networkbench $${network-bench-configuration:rendered} $${monitor-directory:monitor-log}
 
-[rsync-logs]
-recipe = slapos.recipe.template:jinja2
-template = ${rsync-logs-script:output}
-rendered = $${monitor-directory:bin}/rsync-logs.sh
-mode = 0744
-context =
-  key buildout_directory buildout:directory
-  raw rsync_binary ${rsync:location}/bin/rsync
-  key server_log_directory monitor-directory:server-log 
-  raw re6st_log_directory /var/log/re6stnet
-  key system_log_directory monitor-directory:system-log 
+
+[symlink-re6st-logs]
+recipe = cns.recipe.symlink
+symlink = /var/log/re6stnet = $${monitor-directory:system-log}/re6stnet
+autocreate = true
+
+[symlink-collected-logs]
+recipe = cns.recipe.symlink
+symlink = /srv/slapgrid/var/data-log = $${monitor-directory:server-log}/data-log
+autocreate = true
 
 [smartmon-logs]
 recipe = slapos.recipe.template:jinja2
@@ -79,6 +72,7 @@ template = ${network-bench-cfg:output}
 rendered = $${monitor-directory:etc}/network_bench.cfg
 mode = 0744
 context =
+  key slapparameter_dict slap-parameters:configuration
   
 
 [pwgen]
@@ -103,17 +97,21 @@ private-hash = $${pwgen:passwd}$${pwgen32:passwd}
 frontend-domain = 
 
 [monitor-frontend]
-<= slap-connection
 recipe = slapos.cookbook:requestoptional
 name = Monitor Frontend
 # XXX We have hardcoded SR URL here.
 software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg
 slave = true
-config = url domain
 config-url = $${monitor-parameters:url}
 config-domain = $${slap-parameter:frontend-domain}
 return = site_url domain
 
+server-url = $${slap-connection:server-url}
+key-file = $${slap-connection:key-file}
+cert-file = $${slap-connection:cert-file}
+computer-id = $${slap-connection:computer-id}
+partition-id = $${slap-connection:partition-id}
+
 [publish-connection-informations]
 recipe = slapos.cookbook:publish
 monitor_url = $${monitor-parameters:url}

software/monitor/instance.cfg

+[buildout]
+parts =
+  switch_softwaretype
+
+eggs-directory = ${buildout:eggs-directory}
+develop-eggs-directory = ${buildout:develop-eggs-directory}
+
+[switch_softwaretype]
+recipe = slapos.cookbook:softwaretype
+default = $${instance-base-runner:rendered}
+
+[instance-base-runner]
+recipe = slapos.recipe.template:jinja2
+template = ${template-monitor:output}
+rendered = $${buildout:directory}/template-monitor.cfg
+extensions = jinja2.ext.do
+context = key buildout buildout:bin-directory
+          key develop_eggs_directory buildout:develop-eggs-directory
+          key eggs_directory buildout:eggs-directory
+          key slapparameter_dict slap-configuration:configuration
+mode = 0644
+
+[slap-configuration]
+recipe = slapos.cookbook:slapconfiguration
+computer = $${slap-connection:computer-id}
+partition = $${slap-connection:partition-id}
+url = $${slap-connection:server-url}
+key = $${slap-connection:key-file}
+cert = $${slap-connection:cert-file}
\ No newline at end of file

software/monitor/network_bench.cfg

-[network_bench]
-
-url =
-    http://www.tiolive.com/robots.txt
-    http://www.erp5.com/robots.txt
-    http://www.erp5.cn/robots.txt
-    http://www.nexedi.cn/robots.txt
-    http://www.osoe-project.cn/robots.txt
-    http://www.nexedi.cn/robots.txt
-    http://www.osoe-project.org/robots.txt
-    http://www.osoe-project.cn/robots.txt
-    http://www.google.com/robots.txt
-    http://ipv6.google.com/robots.txt
-
-    https://[2001:67c:1254:e:a7::2e20]:9685/monitor-public/rssfeed.html
-    https://[2001:67c:1254:69::115c]:9685/monitor-public/rssfeed.html
-    https://[2001:67c:1254:8a::2621]:9685/monitor-public/rssfeed.html
-    https://[2001:67c:1254:e:c7::89ae]:9685/monitor-public/rssfeed.html
-    https://[2001:67c:1254:e:93::2e31]:9685/monitor-public/rssfeed.html
-    https://[2001:67c:1254:1f::2723]:9685/monitor-public/rssfeed.html
-    https://[2001:41d0:1:a78b::3c2a]:9685/monitor-public/rssfeed.html
-    http://[2001:67c:1254:4::1]/index.html
-
-dns =
-    erp5.com
-    www.erp5.com
-    erp5.cn
-    www.erp5.cn
-    nexedi.cn
-    www.nexedi.com
-    www.nexedi.cn
-    www.osoe-project.org
-    www.osoe-project.cn
-    free.fr
-    google.com
-
-ping =
-    erp5.com
-    erp5.cn
-    google.com
-    free.fr
-    frontend1.nexedi.com
-    frontend2.nexedi.com
-    frontend3.nexedi.cn
-    42.192.4.15
-    195.208.185.22

software/monitor/network_bench.cfg.in

+[network_bench]
+
+url =
+  {{ slapparameter_dict.get('test_http_url_list', '').split('\n') | join('\n  ') }}
+
+dns =
+  {{ slapparameter_dict.get('test_name_list', '').split('\n') | join('\n  ') }}
+
+ping =
+  {{ slapparameter_dict.get('ping_ip_list', '').split('\n') | join('\n  ') }}
+
+ping6 =
+  {{ slapparameter_dict.get('ping6_ip_list', '').split('\n') | join('\n  ') }}
\ No newline at end of file

software/monitor/rsync-logs.sh.in

-#!${dash-output:dash}
-
-RSYNC_BIN={{ rsync_binary }}
-SLAPGRID_LOGS={{ buildout_directory }}/../var/data-log/
-RE6STLOGS={{ re6st_log_directory }}
-
-SYSTEMLOGS={{ system_log_directory }}
-SERVERLOGS={{ server_log_directory }}
-
-$RSYNC_BIN -a --delete $SLAPGRID_LOGS $SERVERLOGS
-
-if [ -d $RE6STLOGS ]; then
-  $RSYNC_BIN -a --delete $RE6STLOGS $SYSTEMLOGS
-fi

software/monitor/software.cfg

@@ -3,7 +3,6 @@
 extends =
   ../../stack/monitor/buildout.cfg
   ../../stack/slapos.cfg
-  ../../component/rsync/buildout.cfg
   ../../component/smartmontools/buildout.cfg
   ../../component/lmsensors/buildout.cfg
   ../../component/fluentd/buildout.cfg
@@ -12,17 +11,25 @@ parts =
   slapos-cookbook
   slapos-toolbox
   template
+  network-bench-cfg
+  template-monitor
   eggs
-  rsync
   smartmontools
   lmsensors
-  fluentd
+
   
 [template]
 recipe = slapos.recipe.template 
-url = ${:_profile_base_location_}/instance.cfg.in
-output = ${buildout:directory}/instance.cfg
-md5sum = 184a27424b48514cfe9374e881e1cc0c 
+url = ${:_profile_base_location_}/instance.cfg
+output = ${buildout:directory}/template.cfg
+md5sum =  daf1125d0d67b6b140a4b6de8c8e35f3
+mode = 0644
+
+[template-monitor]
+recipe = slapos.recipe.template 
+url = ${:_profile_base_location_}/instance-monitor.cfg.jinja2
+output = ${buildout:directory}/template-monitor.cfg
+md5sum =  b5ef3caea873f5ca202792220263aef2
 mode = 0644
 
 # stupify index for now
@@ -34,13 +41,6 @@ md5sum = 876f18b159fbd9325332d0f42e9172ac
 url = ${:_profile_base_location_}/${:filename}
 md5sum = 1f30b17f5ea11fd033984d17c38bcfd3
 
-[rsync-logs-script]
-recipe = slapos.recipe.template
-url = ${:_profile_base_location_}/rsync-logs.sh.in
-md5sum = 8828868268a6f9761eccbcec1660cc9a 
-output = ${buildout:directory}/template-rsync-logs.sh.in
-mode = 0644
-
 [smartmon-logs-script]
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/smartmon.py.in
@@ -50,8 +50,8 @@ mode = 0644
 
 [network-bench-cfg]
 recipe = slapos.recipe.template
-url = ${:_profile_base_location_}/network_bench.cfg
-md5sum  = 44f8db5fca5e16ae87f570623a3ad0b4 
+url = ${:_profile_base_location_}/network_bench.cfg.in
+md5sum  = a3510050a1b237dc11ab46a9d2b78f56 
 output = ${buildout:directory}/template-network-bench-cfg.in
 mode = 0644
 
@@ -80,6 +80,7 @@ cns.recipe.symlink = 0.2.3
 plone.recipe.command = 1.1
 slapos.recipe.template = 2.7
 z3c.recipe.scripts = 1.0.1
+rubygemsrecipe = 0.2.1
 
 apache-libcloud = 0.16.0
 ecdsa = 0.11