Update Release Candidate

software/erp5/instance-erp5-input-schema.json

@@ -798,7 +798,7 @@
         },
         "apachedex-promise-threshold": {
           "type": "number",
-          "title": "Promise fails if the overall Apdex score for the previous day is below than this value.",
+          "title": "Promise fails if the overall Apdex score for the previous day is below than this value. A value of 0 disables the promise.",
           "default": 70,
           "minimum": 0,
           "maximum": 100

software/erp5/test/test/test_balancer.py

 import ipaddress
 import json
 import logging
+import lzma
 import os
 import re
 import socket
@@ -104,7 +105,7 @@ class BalancerTestCase(ERP5InstanceTestCase):
             '--js-embed',
             '--quiet',
           ],
-        'apachedex-promise-threshold': 100,
+        'apachedex-promise-threshold': 0,
         'haproxy-server-check-path': '/',
         'zope-family-dict': {
             'default': ['dummy_http_server'],
@@ -264,7 +265,7 @@ class TestLog(BalancerTestCase, CrontabMixin):
     # make a request so that we have something in the logs
     requests.get(self.default_balancer_zope_url, verify=False)
 
-    # crontab for apachedex is executed
+    # crontab for daily apachedex is executed
     self._executeCrontabAtDate('generate-apachedex-report', '23:59')
     # it creates a report for the day
     apachedex_report, = (
@@ -278,6 +279,39 @@ class TestLog(BalancerTestCase, CrontabMixin):
     # having this table means that apachedex could parse some lines.
     self.assertIn('<h2>Hits per status code</h2>', report_text)
 
+    # weekly apachedex uses the logs after rotation, we'll run log rotation
+    # until we have a xz file for two days ago and a non compressed file for
+    # yesterday
+    # run logrotate a first time so that it create state files
+    self._executeCrontabAtDate('logrotate', '2000-01-01')
+    requests.get(urllib.parse.urljoin(self.default_balancer_zope_url, 'error-two-days-ago'), verify=False)
+    self._executeCrontabAtDate('logrotate', 'yesterday 00:00')
+    requests.get(urllib.parse.urljoin(self.default_balancer_zope_url, 'error-yesterday'), verify=False)
+    self._executeCrontabAtDate('logrotate', '00:00')
+
+    # this apachedex command uses compressed files, verify that our test setup
+    # is correct and that the error from two days ago is in the compressed file.
+    two_days_ago_log, = (
+      self.computer_partition_root_path / 'srv' / 'backup'/ 'logrotate'
+    ).glob("apache-access.log-*.xz")
+    with lzma.open(two_days_ago_log) as f:
+      self.assertIn(b'GET /error-two-days-ago', f.read())
+
+    self._executeCrontabAtDate('generate-weekly-apachedex-report', '23:59')
+    # this creates a report for the week
+    apachedex_weekly_report, = (
+      self.computer_partition_root_path
+        / 'srv'
+        / 'monitor'
+        / 'private'
+        / 'apachedex'
+        / 'weekly').glob('*.html')
+    weekly_report_text = apachedex_weekly_report.read_text()
+    self.assertIn('APacheDEX', weekly_report_text)
+    # because we run apachedex with error details, we can see our error requests
+    self.assertIn('error-two-days-ago', weekly_report_text)
+    self.assertIn('error-yesterday', weekly_report_text)
+
   def test_access_log_rotation(self) -> None:
     # run logrotate a first time so that it create state files
     self._executeCrontabAtDate('logrotate', '2000-01-01')

software/neoppod/buildout.hash.cfg

@@ -18,7 +18,7 @@ md5sum = b4baf7f21f450fa522c2a69f5a4aedf7
 
 [root-common]
 filename = root-common.cfg.in
-md5sum = eefc3358852b7509bfe1b18da19abca0
+md5sum = 1320dbea362ce6909f1490ba349c3e6c
 
 [instance-neo-admin]
 filename = instance-neo-admin.cfg.in
@@ -30,7 +30,7 @@ md5sum = 9f27195d770b2f57461c60a82c851ab9
 
 [instance-neo]
 filename = instance-neo.cfg.in
-md5sum = 53b2c73a0c5c7cb5f6b841ba6892fa46
+md5sum = d8a62f4a2b68fe97146871d07a500443
 
 [template-neo-my-cnf]
 filename = my.cnf.in

software/neoppod/instance-neo-input-schema.json

@@ -28,6 +28,49 @@
           "description": "Master nodes in the cluster to backup.",
           "type": "string"
         },
+        "reflink": {
+          "description": "Track references between OIDs of an external ZODB.",
+          "additionalProperties": false,
+          "required": [
+            "zurl"
+          ],
+          "properties": {
+            "zurl": {
+              "description": "ZODB to track.",
+              "type": "string"
+            },
+            "_ca": {
+              "type": "string"
+            },
+            "_cert": {
+              "type": "string"
+            },
+            "_key": {
+              "type": "string"
+            },
+            "max-txn-size": {
+              "description": "Maximum number of OIDs to delete per transaction.",
+              "default": 2097151,
+              "type": "integer"
+            },
+            "commit-interval": {
+              "description": "Commit every SECONDS of work.",
+              "default": 10,
+              "type": "integer"
+            },
+            "period": {
+              "description": "Age of the historical revision at which a GC is performed (this can be seen as a grace period). See --period option.",
+              "default": 86400,
+              "type": "integer"
+            },
+            "no-gc": {
+              "description": "Only track references.",
+              "default": false,
+              "type": "boolean"
+            },
+          },
+          "type": "object"
+        },
         "monitor": {
           "description": "Parameters for monitoring.",
           "properties": {
@@ -124,7 +167,7 @@
               "type": "array"
             },
             "storage-type": {
-              "description": "Storage type. Required when several types are configured and you select which one to use via 'node!' parameter. Defaults to whatever is configured ('sqlite' or 'mysql'), else MySQL if available, else SQLite.",
+              "description": "Storage type. Required when several types are configured and you select which one to use via 'node!' parameter. Defaults to whatever is configured ('sqlite' or 'mysql'), else MySQL if available, else SQLite. SQLite is recommended for a reflink DB.",
               "enum": [
                 "MySQL",
                 "SQLite"

software/neoppod/instance-neo.cfg.in

@@ -194,6 +194,44 @@ post = {{ bin_directory }}/slapos-kill -s RTMIN+1 -- {{ bin_directory }}/neostor
 
 {% endfor -%}
 
+{% set reflink = slapparameter_dict.get('reflink') -%}
+{% if reflink -%}
+{%   set zurl = 'neo://%s@%s' % (slapparameter_dict['cluster'], slapparameter_dict['masters'].replace(' ', ',')) -%}
+{%   if slapparameter_dict['ssl'] -%}
+{%     set zurl = zurl + '?ca=${ca.crt:output}&cert=${neo.crt:output}&key=${neo.key:output}' -%}
+{%   endif -%}
+[{{ section('reflink') }}]
+recipe = slapos.cookbook:wrapper
+wrapper-path = ${directory:etc_run}/reflink
+{%   set args = [] -%}
+{%   if 'commit-interval' in reflink -%}
+{%     do args.extend(('-i', str(reflink['commit-interval']))) -%}
+{%   endif -%}
+{%   if 'period' in reflink -%}
+{%     do args.extend(('-p', str(reflink['period']))) -%}
+{%   endif -%}
+{%   if 'no-gc' in reflink -%}
+{%     do args.append('--no-gc') -%}
+{%   endif -%}
+command-line = {{ bin_directory }}/reflink -v {{ zurl }} run {{ ' '.join(args) }} {{ reflink['zurl'] }}
+{%   if reflink['zurl'].startswith('neos://') -%}
+environment =
+  NEO_CA=${reflink-ca:output}
+  NEO_CERT=${reflink-cert:output}
+  NEO_KEY=${reflink-key:output}
+
+{%     for x in 'ca', 'cert', 'key' -%}
+[reflink-{{x}}]
+recipe = slapos.recipe.template:jinja2
+output = ${directory:etc}/${:_buildout_section_name_}.pem
+inline =
+  {{'{{'}}pem}}
+context = key pem :pem
+pem = {{dumps(reflink['_'+x])}}
+{%     endfor -%}
+{%   endif -%}
+{% endif -%}
+
 {% if mysql -%}
 [init-script]
 recipe = slapos.recipe.template

software/neoppod/root-common.cfg.in

@@ -133,6 +133,9 @@ config-monitor-port = {{ dumps(port) }}
 {%-     endif %}
 config-monitor = {{ dumps(parameter_dict.get('monitor', {})) }}
 {%-   endif %}
+{%- if 'reflink' in parameter_dict %}
+config-reflink = {{ dumps(parameter_dict.pop('reflink')) }}
+{%- endif %}
 {%-   for k, v in six.iteritems(node) %}
 config-{{ k }} = {{ dumps(v) }}
 {%-   endfor %}

stack/erp5-zope2/buildout.hash.cfg

@@ -74,7 +74,7 @@ md5sum = 6d9a926e07b674ffdaecd381d763c068
 
 [template-erp5]
 filename = instance-erp5.cfg.in
-md5sum = 6b10ab0c54278156caf058ebb7246645
+md5sum = 7ac187ff9fe306297724a730459e1a4d
 
 [template-zeo]
 filename = instance-zeo.cfg.in

stack/erp5-zope2/instance-erp5.cfg.in

@@ -186,6 +186,7 @@ connection-url = smtp://127.0.0.2:0/
 {%     set ((name, server_dict),) = server_dict.items() -%}
 {%     do neo.append(server_dict.get('cluster')) -%}
 {%     do server_dict.update(cluster='${publish-early:neo-cluster}') -%}
+{{     assert('reflink' not in server_dict, 'reflink option is meaningless in ERP5 SR') }}
 {{     root_common.request_neo(server_dict, 'zodb-neo', 'neo-', monitor_base_url_dict) }}
 {%     set client_dict = zodb_dict[name].setdefault('storage-dict', {}) -%}
 {%     for k in 'ssl', '_ca', '_cert', '_key' -%}

stack/erp5/buildout.cfg

@@ -529,6 +529,7 @@ eggs =
   lock_file
   astor
   APacheDEX
+  ${backports.lzma:egg}
   chardet
   collective.recipe.template
   erp5diff

stack/erp5/buildout.hash.cfg

@@ -74,7 +74,7 @@ md5sum = 8e452bd32fc0d4d858b275a2b3ee790b
 
 [template-erp5]
 filename = instance-erp5.cfg.in
-md5sum = 1333d2fc21f64da4010a4eafea59d141
+md5sum = bd8cb060b37d8dc6a1ab4e25a2a7e109
 
 [template-zeo]
 filename = instance-zeo.cfg.in
@@ -94,7 +94,7 @@ md5sum = 9c580be982d8c63ec06fc273ef3cb971
 
 [template-balancer]
 filename = instance-balancer.cfg.in
-md5sum = 409a7505548576ebf0e4d5cc218e0753
+md5sum = e9aa89754085bdc7a6fb9e53c0c97f9d
 
 [template-haproxy-cfg]
 filename = haproxy.cfg.in

stack/erp5/instance-balancer.cfg.in

@@ -400,7 +400,8 @@ command-line = "{{ parameter_dict['socat'] }}/bin/socat" unix-connect:${haproxy-
 
 [rsyslogd-cfg-parameter-dict]
 log-socket = ${directory:run}/log.sock
-access-log-file = ${directory:log}/apache-access.log
+access-log-file-basename = apache-access.log
+access-log-file = ${directory:log}/${:access-log-file-basename}
 error-log-file = ${directory:log}/apache-error.log
 pid-file = ${directory:run}/rsyslogd.pid
 spool-directory = ${directory:rsyslogd-spool}
@@ -483,6 +484,7 @@ backup-caucased-haproxy-certificate = ${:srv}/backup/caucased{{ caucase_haproxy_
 caucase-updater-haproxy-certificate = ${:srv}/caucase-updater-haproxy-certificate
 tmp = ${buildout:directory}/tmp
 apachedex = ${monitor-directory:private}/apachedex
+apachedex-weekly = ${:apachedex}/weekly
 rsyslogd-spool = ${:run}/rsyslogd-spool
 {% if frontend_caucase_url_list -%}
 ca-cert = ${:etc}/ssl.crt
@@ -511,12 +513,38 @@ wrapper-path = ${directory:bin}/${:command}
 command-line = "{{ parameter_dict['run-apachedex-location'] }}" "{{ parameter_dict['apachedex-location'] }}" "${directory:apachedex}" ${monitor-publish-parameters:monitor-base-url}/private/apachedex --apache-log-list "${apachedex-parameters:apache-log-list}" --configuration ${apachedex-parameters:configuration}
 command = generate-apachedex-report
 
+[{{ section('monitor-generate-weekly-apachedex-report') }}]
+recipe = slapos.cookbook:cron.d
+cron-entries = ${cron:cron-entries}
+name = generate-weekly-apachedex-report
+frequency = 1 * * * 1
+command = ${monitor-generate-weekly-apachedex-report-wrapper:output}
+
+[monitor-generate-weekly-apachedex-report-wrapper]
+recipe = slapos.recipe.template
+output = ${buildout:bin-directory}/${:_buildout_section_name_}
+inline =
+  #!/bin/sh
+  # Yesterday's log file is not compressed
+  LOG_FILES=${logrotate-rsyslogd:backup}/${rsyslogd-cfg-parameter-dict:access-log-file-basename}-$(date +"%Y%m%d")
+  # Days before are compressed
+  {% for i in range(1, 6) -%}
+  LOG_FILE=${logrotate-rsyslogd:backup}/${rsyslogd-cfg-parameter-dict:access-log-file-basename}-$(date -d "{{i}} days ago" +"%Y%m%d").xz
+  if [ -f "$${LOG_FILE}" ]; then
+    LOG_FILES="$${LOG_FILES} $${LOG_FILE}"
+  fi
+  {% endfor %}
+  exec {{ parameter_dict['apachedex-location'] }} \
+    -o ${directory:apachedex-weekly}/$(date -d "1 days ago" +"%Y-%m-%d").html \
+    @${apachedex-parameters:configuration} \
+    -- $${LOG_FILES}
+
 [monitor-apachedex-report-config]
 recipe = slapos.recipe.template
 output = ${directory:etc}/${:_buildout_section_name_}
 inline =
   {% for line in slapparameter_dict['apachedex-configuration'] -%}
-    {# apachedex config files use shlex.split, so we need to quote the arguments. #}
+    {# apachedex config files use shlex.split, so we need to quote the arguments. -#}
     {{ six.moves.shlex_quote(line) }}
   {% endfor %}
 
@@ -525,11 +553,13 @@ apache-log-list = ${rsyslogd-cfg-parameter-dict:access-log-file}
 configuration = ${monitor-apachedex-report-config:output}
 promise-threshold = {{ slapparameter_dict['apachedex-promise-threshold'] }}
 
+{%if slapparameter_dict['apachedex-promise-threshold'] %}
 [{{ section('monitor-promise-apachedex-result') }}]
 <= monitor-promise-base
 promise = check_command_execute
 name = check-apachedex-result.py
 config-command = "{{ parameter_dict['promise-check-apachedex-result'] }}" --apachedex_path "${directory:apachedex}" --status_file ${monitor-directory:private}/apachedex.report.json --threshold "${apachedex-parameters:promise-threshold}"
+{% endif %}
 
 [{{ section('promise-check-computer-memory') }}]
 <= monitor-promise-base

stack/erp5/instance-erp5.cfg.in

@@ -183,6 +183,7 @@ connection-url = smtp://127.0.0.2:0/
 {%     set ((name, server_dict),) = server_dict.items() -%}
 {%     do neo.append(server_dict.get('cluster')) -%}
 {%     do server_dict.update(cluster='${publish-early:neo-cluster}') -%}
+{{     assert('reflink' not in server_dict, 'reflink option is meaningless in ERP5 SR') }}
 {{     root_common.request_neo(server_dict, 'zodb-neo', 'neo-', monitor_base_url_dict) }}
 {%     set client_dict = zodb_dict[name].setdefault('storage-dict', {}) -%}
 {%     for k in 'ssl', '_ca', '_cert', '_key' -%}