software/erp5: compress pt-query digest reports

These reports are large, no need to keep them as text

software/erp5/test/setup.py

@@ -48,6 +48,7 @@ setup(name=name,
         'psutil',
         'requests',
         'mysqlclient',
+        'backports.lzma',
         'cryptography',
         'pyOpenSSL',
         ],

software/erp5/test/test/test_mariadb.py

@@ -31,14 +31,19 @@ import json
 import glob
 import urlparse
 import socket
+import sys
 import time
 import contextlib
 import datetime
 import subprocess
 import gzip
 
+from backports import lzma
 import MySQLdb
 
+from slapos.testing.utils import CrontabMixin
+from slapos.testing.utils import getPromisePluginParameterDict
+
 from . import ERP5InstanceTestCase
 from . import setUpModule
 setUpModule  # pyflakes
@@ -58,8 +63,8 @@ class MariaDBTestCase(ERP5InstanceTestCase):
     return {
         'tcpv4-port': 3306,
         'max-connection-count': 5,
-        'max-slowqueries-threshold': 5,
-        'slowest-query-threshold': 10,
+        'max-slowqueries-threshold': 1,
+        'slowest-query-threshold': 0.1,
         # XXX what is this ? should probably not be needed here
         'name': cls.__name__,
         'monitor-passwd': 'secret',
@@ -88,31 +93,7 @@ class MariaDBTestCase(ERP5InstanceTestCase):
     )
 
 
-class TestCrontabs(MariaDBTestCase):
-
-  def _getCrontabCommand(self, crontab_name):
-    # type: (str) -> str
-    """Read a crontab and return the command that is executed.
-    """
-    with open(
-        os.path.join(
-            self.computer_partition_root_path,
-            'etc',
-            'cron.d',
-            crontab_name,
-        )) as f:
-      crontab_spec = f.read()
-    return " ".join(crontab_spec.split()[5:])
-
-  def _executeCrontabAtDate(self, crontab_name, date):
-    # type: (str, str) -> None
-    """Executes a crontab as if the current date was date
-    """
-    crontab_command =  self._getCrontabCommand(crontab_name)
-    subprocess.check_call(
-        "faketime {date} bash -o pipefail -e -c '{crontab_command}'".format(**locals()),
-        shell=True,
-    )
+class TestCrontabs(MariaDBTestCase, CrontabMixin):
 
   def test_full_backup(self):
     self._executeCrontabAtDate('mariadb-backup', '2050-01-01')
@@ -127,6 +108,72 @@ class TestCrontabs(MariaDBTestCase):
         'r') as dump:
       self.assertIn('CREATE TABLE', dump.read())
 
+  def test_logrotate_and_slow_query_digest(self):
+    # slow query digest needs to run after logrotate, since it operates on the rotated
+    # file, so this tests both logrotate and slow query digest.
+
+    # run logrotate a first time so that it create state files
+    self._executeCrontabAtDate('logrotate', '2000-01-01')
+
+    # make two slow queries
+    cnx = self.getDatabaseConnection()
+    with contextlib.closing(cnx):
+      cnx.query("SELECT SLEEP(1.1)")
+      cnx.store_result()
+      cnx.query("SELECT SLEEP(1.2)")
+
+    # slow query crontab depends on crontab for log rotation
+    # to be executed first.
+    self._executeCrontabAtDate('logrotate', '2050-01-01')
+    # this logrotate leaves the log for the day as non compressed
+    rotated_log_file = os.path.join(
+        self.computer_partition_root_path,
+        'srv',
+        'backup',
+        'logrotate',
+        'mariadb_slowquery.log-20500101',
+    )
+    self.assertTrue(os.path.exists(rotated_log_file))
+
+    # then crontab to generate slow query report is executed
+    self._executeCrontabAtDate('generate-mariadb-slow-query-report', '2050-01-01')
+    # and it creates a report for the day
+    slow_query_report = os.path.join(
+        self.computer_partition_root_path,
+        'srv',
+        'monitor',
+        'private',
+        'slowquery_digest',
+        'slowquery_digest.txt-2050-01-01.xz',
+    )
+    with lzma.open(slow_query_report, 'r') as f:
+      # this is the hash for our "select sleep(n)" slow query
+      self.assertIn("ID 0xF9A57DD5A41825CA", f.read())
+
+    # on next day execution of logrotate, log files are compressed
+    self._executeCrontabAtDate('logrotate', '2050-01-02')
+    self.assertTrue(os.path.exists(rotated_log_file + '.xz'))
+    self.assertFalse(os.path.exists(rotated_log_file))
+
+    # there's a promise checking that the threshold is not exceeded
+    # and it reports a problem since we set a threshold of 1 slow query
+    check_slow_query_promise_plugin = getPromisePluginParameterDict(
+        os.path.join(
+            self.computer_partition_root_path,
+            'etc',
+            'plugin',
+            'check-slow-query-pt-digest-result.py',
+        ))
+    with self.assertRaises(subprocess.CalledProcessError) as error_context:
+      subprocess.check_output('faketime 2050-01-01 %s' % check_slow_query_promise_plugin['command'], shell=True)
+    self.assertEqual(
+        error_context.exception.output,
+"""\
+Threshold is lower than expected: 
+Expected total queries : 1.0 and current is: 2
+Expected slowest query : 0.1 and current is: 1
+""")
+
 
 class TestMariaDB(MariaDBTestCase):
   def test_utf8_collation(self):

stack/erp5/buildout.cfg

@@ -9,6 +9,7 @@ extends =
   ../../component/git/buildout.cfg
   ../../component/graphviz/buildout.cfg
   ../../component/gzip/buildout.cfg
+  ../../component/xz-utils/buildout.cfg
   ../../component/haproxy/buildout.cfg
   ../../component/hookbox/buildout.cfg
   ../../component/findutils/buildout.cfg
@@ -178,6 +179,7 @@ context =
     key erp5_location erp5:location
     key findutils_location findutils:location
     key gzip_location gzip:location
+    key xz_utils_location xz-utils:location
     key haproxy_location haproxy:location
     key instance_common_cfg instance-common:rendered
     key jsl_location jsl:location

stack/erp5/buildout.hash.cfg

@@ -18,7 +18,7 @@ md5sum = 85ce1e2f3d251aa435fef8118dca8a63
 
 [mariadb-slow-query-report-script]
 filename = mysql-querydigest.sh.in
-md5sum = 0c0d98a68230cd0ad36046bb25b35f4a
+md5sum = 7b14d2b81f2c864e47682ddb03b1b663
 
 [mariadb-start-clone-from-backup]
 filename = instance-mariadb-start-clone-from-backup.sh.in
@@ -26,7 +26,7 @@ md5sum = d10b8e35b02b5391cf46bf0c7dbb1196
 
 [template-mariadb]
 filename = instance-mariadb.cfg.in
-md5sum = bfed6ac56c3ba0e96be4c9474dac6f20
+md5sum = 00edae50016d4fcc53b30356c5ba2396
 
 [template-kumofs]
 filename = instance-kumofs.cfg.in
@@ -70,7 +70,7 @@ md5sum = cc19560b9400cecbd23064d55c501eec
 
 [template]
 filename = instance.cfg.in
-md5sum = 328ea2bb5f2bff18f8be8c541c01f260
+md5sum = 5c5250112b87a3937f939028f9594b85
 
 [monitor-template-dummy]
 filename = dummy.cfg

stack/erp5/instance-mariadb.cfg.in

@@ -326,6 +326,7 @@ context =
   raw slow_query_path ${directory:srv}/backup/logrotate/mariadb_slowquery.log
   raw pt_query_exec ${binary-wrap-pt-digest:wrapper-path}
   raw dash {{ parameter_dict['dash-location'] }}/bin/dash
+  raw xz {{ parameter_dict['xz-utils-location'] }}/bin/xz
   key output_folder directory:slowquery
 
 [slow-query-digest-parameters]

stack/erp5/instance.cfg.in

@@ -144,6 +144,7 @@ coreutils-location = {{ coreutils_location }}
 dash-location = {{ dash_location }}
 findutils-location = {{ findutils_location }}
 gzip-location = {{ gzip_location }}
+xz-utils-location = {{ xz_utils_location }}
 mariadb-location = {{ mariadb_location }}
 template-my-cnf = {{ template_my_cnf }}
 template-mariadb-initial-setup = {{ template_mariadb_initial_setup }}

stack/erp5/mysql-querydigest.sh.in

 #!{{dash}}
 # BEWARE: This file is operated by slapgrid
 
+set -e
+
 SLOW_QUERY_PATH='{{slow_query_path}}'
 OUTPUT_FOLDER='{{output_folder}}'
 PT_QUERY_EXEC='{{pt_query_exec}}'
@@ -21,6 +23,6 @@ if [ ! -f "$SLOW_LOG" ]; then
   exit 1
 fi
 
-"$PT_QUERY_EXEC" "$SLOW_LOG" > "$OUTPUT_FILE" && \
-echo "Report generated successfully." || \
-echo "Report failed with code $?"
+"$PT_QUERY_EXEC" "$SLOW_LOG" > "$OUTPUT_FILE"
+
+{{ xz }} -9 "$OUTPUT_FILE"