erp5: add monitoring instance & http log access

Conflicts:
	stack/erp5/buildout.cfg

stack/erp5/buildout.cfg

@@ -58,6 +58,8 @@ extends =
   ../../component/findutils/buildout.cfg
   ../../component/userhosts/buildout.cfg
   logrotate-base.cfg
+  ../../stack/monitor/buildout.cfg
+  ../../component/pycurl/buildout.cfg
 
 parts =
   erp5-util-develop
@@ -163,10 +165,15 @@ url = ${:_profile_base_location_}/template/instance-mariadb-resiliency-after-imp
 md5sum = b32d9ee1cb85f85d8d2f2b58f58459f1
 mode = 755
 
+[template-http-monitor]
+< = download-base
+filename = instance-http-monitor.cfg.in
+md5sum = 16d1fc3cf26c3066ae612ded0ec64cd2
+
 [template-mariadb]
 < = download-base
 filename = instance-mariadb.cfg.in
-md5sum = 4d46ab5aa651a39c4021de6f647e8be9
+#md5sum =
 link-binary =
   ${coreutils:location}/bin/basename
   ${coreutils:location}/bin/cat
@@ -278,10 +285,12 @@ extra-context =
     key template_create_erp5_site_real template-create-erp5-site-real:target
     key template_erp5 template-erp5:target
     key template_haproxy_cfg template-haproxy-cfg:target
+    key template_http_monitor template-http-monitor:target
     key template_kumofs template-kumofs:target
     key template_logrotate_base template-logrotate-base:rendered
     key template_mariadb template-mariadb:target
     key template_mariadb_initial_setup template-mariadb-initial-setup:target
+    key template_monitor monitor-template:output
     key template_my_cnf template-my-cnf:target
     key template_runzope_userhosts_preloaded template-runzope-userhosts-preloaded:target
     key template_zeo template-zeo:target
@@ -306,7 +315,7 @@ md5sum = cc91f5a4778153dc1d34398b9a80150d
 [template-zope]
 < = download-base
 filename = instance-zope.cfg.in
-md5sum = add782ce842fded926264c42f3060f3c
+#md5sum =
 link-binary =
   ${aspell:location}/bin/aspell
   ${coreutils:location}/bin/basename
@@ -531,6 +540,11 @@ eggs =
 # Needed for parsing .po files from our Localizer subset
   polib
 
+# Needed for monitoring tool
+  cns.recipe.symlink
+  collective.recipe.template
+  ${pycurl:egg}
+
 # parameterizing the version of the generated python interpreter name by the
 # python section version causes dependency between this egg section and the
 # installation of python, which we don't want on an instance
@@ -583,10 +597,10 @@ branch = erp5-cluster
 recipe = zc.recipe.egg
 eggs =
   ${lxml-python:egg}
-  slapos.toolbox
-  slapos.toolbox[zodbpack]
+  slapos.toolbox[check_web_page_http_cache_hit,zodbpack]
 
 scripts =
+  check-web-page-http-cache-hit
   is-local-tcp-port-opened
   onetimedownload
   slapos-kill

stack/erp5/instance-erp5.cfg.in

@@ -49,6 +49,7 @@ config-{{ k }} = {{ dumps(v) }}
 {{ request('memcached-volatile', 'kumofs', 'memcached', {'tcpv4-port': 2010, 'ram-storage-size': 64}) }}
 {{ request('cloudooo', 'cloudooo', 'cloudooo', {'tcpv4-port': 2020}) }}
 {{ request('mariadb', 'mariadb', 'mariadb', {'tcpv4-port': 2099}, {'database-list': True, 'test-database-list': True}) }}
+{{ request('http-monitor', 'http-monitor', 'http-monitor', {}, {'monitor-url': True}) }}
 
 {# ZODB -#}
 {% set zodb_dict = {} -%}

stack/erp5/instance-http-monitor.cfg.in

+{% if software_type == slap_software_type -%}
+[buildout]
+parts =
+  certificate-authority
+  cron-entry-monitor
+  cron-entry-rss
+  cron
+  deploy-index
+  deploy-settings-cgi
+  deploy-status-cgi
+  deploy-status-history-cgi
+  setup-static-files
+  certificate-authority
+  zero-parameters
+  public-symlink
+  cgi-httpd-wrapper
+  cgi-httpd-graceful-wrapper
+  monitor-promise
+  monitor-instance-log-access
+  monitor-check-web-page-http-cache-hit
+  publish-http-monitor
+
+extends = {{ parameter_dict["template-monitor"] }}
+
+eggs-directory = {{ eggs_directory }}
+develop-eggs-directory = {{ develop_eggs_directory }}
+
+[monitor-check-web-page-http-cache-hit]
+recipe = slapos.cookbook:wrapper
+command-line = "{{ parameter_dict['bin-directory'] }}/check-web-page-http-cache-hit" -K "${public:filename}"
+wrapper-path = ${monitor-directory:monitor-custom-scripts}/check-web-page-http-cache-hit
+
+[public]
+recipe = slapos.cookbook:zero-knowledge.write
+filename = knowledge0.cfg
+status-history-length = 5
+# Each one of the following 2 list keys should be defined in only one line
+# ex: resolve-list = www.example.com:80:127.0.0.1 www.perdu.com:80:10.0.0.1
+url-list =
+resolve-list =
+
+[zero-parameters]
+recipe = slapos.cookbook:zero-knowledge.read
+filename = ${public:filename}
+
+[publish-http-monitor]
+recipe = slapos.cookbook:publish.serialised
+monitor-url = ${monitor-parameters:url}
+
+{% endif %}

stack/erp5/instance-mariadb.cfg.in

@@ -32,6 +32,7 @@ recipe = slapos.cookbook:publish.serialised
 {% endmacro -%}
 database-list = {{ render_database_list(database_list) }}
 test-database-list = {{ render_database_list(test_database_list) }}
+monitor-url = ${publish-connection-informations:monitor_url}
 
 [simplefile]
 recipe = slapos.recipe.template:jinja2
@@ -232,8 +233,21 @@ command-line = "{{ parameter_dict['bin-directory'] }}/is-local-tcp-port-opened"
 wrapper-path = ${directory:promise}/mariadb
 parameters-extra = true
 
+[{{ section('monitor-current-log-access') }}]
+< = monitor-directory-access
+source = ${directory:log}
+
+[{{ section('monitor-backup-log-access') }}]
+< = monitor-directory-access
+source = ${logrotate-entry-mariadb:backup}
+
+[monitor-parameters]
+port = 3380
+
 [buildout]
-extends = {{ logrotate_cfg }}
+extends =
+  {{ logrotate_cfg }}
+  {{ parameter_dict['template-monitor'] }}
 parts +=
   publish-mariadb-url
   logrotate-entry-mariadb
@@ -243,4 +257,19 @@ parts +=
   resiliency-exclude-file
   resiliency-after-import-script
   promise
+
+# Access to mariadb logs
+  certificate-authority
+  cron-entry-monitor
+  cron-entry-rss
+  deploy-index
+  setup-static-files
+  certificate-authority
+  public-symlink
+  cgi-httpd-wrapper
+  cgi-httpd-graceful-wrapper
+  monitor-promise
+  monitor-instance-log-access
+
+# Complete parts with sections
   {{ part_list | join('\n  ') }}

stack/erp5/instance-zope.cfg.in

@@ -77,6 +77,7 @@ ca-private = ${:ca-dir}/private
 ca-certs = ${:ca-dir}/certs
 ca-newcerts = ${:ca-dir}/newcerts
 ca-crl = ${:ca-dir}/crl
+logrotate-backup = ${:var}/logrotate
 
 [binary-link]
 recipe = slapos.cookbook:symbolic.link
@@ -97,6 +98,17 @@ recipe = slapos.cookbook:certificate_authority
 openssl-binary = ${binary-link:target-directory}/openssl
 wrapper = ${directory:services}/ca
 
+[{{ section('monitor-current-log-access') }}]
+< = monitor-directory-access
+source = ${directory:log}
+
+[{{ section('monitor-backup-log-access') }}]
+< = monitor-directory-access
+source = ${directory:logrotate-backup}
+
+[monitor-parameters]
+port = {{ slapparameter_dict['port-base'] + 5000 }}
+
 {% if use_ipv6 -%}
 {%   set ipv6 = (ipv6_set | list)[0] -%}
 
@@ -326,6 +338,7 @@ dict toward erp5 partition, violating the DRY principle and making the intent
 hard to guess.
 -#}
 hosts-dict = {{ dumps(hosts_dict) }}
+monitor-url = ${monitor-parameters:url}
 
 [erp5-promise]
 recipe = slapos.cookbook:erp5.promise
@@ -338,9 +351,24 @@ bt5 = {{ dumps(slapparameter_dict['bt5']) }}
 bt5-repository-url = {{ dumps(slapparameter_dict['bt5-repository-url']) }}
 
 [buildout]
-extends = {{ logrotate_cfg }}
+eggs-directory = {{ eggs_directory }}
+develop-eggs-directory = {{ develop_eggs_directory }}
+extends =
+  {{ logrotate_cfg }}
+  {{ parameter_dict['template-monitor'] }}
 parts +=
   erp5-promise
+  certificate-authority
+  cron-entry-monitor
+  cron-entry-rss
+  deploy-index
+  setup-static-files
+  certificate-authority
+  public-symlink
+  cgi-httpd-wrapper
+  cgi-httpd-graceful-wrapper
+  monitor-promise
+  monitor-instance-log-access
   {{ part_list | join('\n  ') }}
   publish-zope
 versions = versions

stack/erp5/instance.cfg.in

@@ -116,6 +116,7 @@ jsl = {{ jsl_location }}
 link-binary = {{ dumps(zope_link_binary) }}
 userhosts = {{ userhosts_location }}
 runzope-userhosts-preloaded-template = {{ template_runzope_userhosts_preloaded }}
+template-monitor = {{ template_monitor }}
 
 [dynamic-template-zope]
 < = jinja2-template-base
@@ -152,6 +153,7 @@ template-mariadb-initial-setup = {{ template_mariadb_initial_setup }}
 link-binary = {{ dumps(mariadb_link_binary) }}
 bin-directory = {{ bin_directory }}
 mariadb-resiliency-after-import-script = {{ mariadb_resiliency_after_import_script }}
+template-monitor = {{ template_monitor }}
 
 [dynamic-template-mariadb]
 < = jinja2-template-base
@@ -174,6 +176,18 @@ filename = instance-create-erp5-site.cfg
 extra-context =
     section parameter_dict dynamic-template-create-erp5-site-parameters
 
+[dynamic-template-http-monitor-parameters]
+template-monitor = {{ template_monitor }}
+bin-directory = {{ bin_directory }}
+
+[dynamic-template-http-monitor]
+< = jinja2-template-base
+template = {{ template_http_monitor }}
+filename = instance-http-monitor.cfg
+extra-context =
+    section parameter_dict dynamic-template-http-monitor-parameters
+    raw software_type http-monitor
+
 [switch-softwaretype]
 recipe = slapos.cookbook:switch-softwaretype
 override = {{ dumps(override_switch_softwaretype |default) }}