monitor: generate mysql slow query repport using pt-query-digest

stack/monitor/buildout.cfg

@@ -46,7 +46,7 @@ recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/monitor.cfg.in
 output = ${buildout:directory}/monitor.cfg
 filename = monitor.cfg
-md5sum = 87d7c22ed77a0e77b06b6c7869024b74
+md5sum = 8d6f6dc112543d818c9620edbdefcbf3
 mode = 0644
 
 [monitor-bin]
@@ -153,6 +153,14 @@ destination = ${buildout:parts-directory}/monitor-template-rss-bin
 filename = status2rss.py
 mode = 0644
 
+[mysql-querydigest]
+recipe = hexagonit.recipe.download
+url = ${:_profile_base_location_}/${:filename}
+download-only = true
+md5sum = 603101d11bc6e371dcade98aaa27990e
+filename = mysql-querydigest.py.in
+mode = 0644
+
 [run-apachedex]
 recipe = hexagonit.recipe.download
 url = ${:_profile_base_location_}/${:filename}

stack/monitor/monitor.cfg.in

@@ -55,6 +55,7 @@ apachedex-result = $${:srv}/apachedex
 
 private-directory = $${:srv}/monitor-private
 log-rss-directory = $${:srv}/ERRORLogAsRSS
+mysql-slowquery-report = $${:srv}/mysql-slowquery-report
 
 [public-symlink]
 recipe = cns.recipe.symlink
@@ -301,6 +302,28 @@ name = $${apachedex-entries:script-name}
 frequency = 0 3 * * *
 command = $${apachedex-entries:rendered}
 
+# Generate Mysql slowqueries report with pt-querry-digest
+[mysql-slowquery-report]
+recipe = slapos.recipe.template:jinja2
+template = ${mysql-querydigest:location}/${mysql-querydigest:filename}
+rendered = $${cron-mysql-slowquery-report:command}
+mode = 0700
+extensions = jinja2.ext.do
+output-folder = $${monitor-directory:mysql-slowquery-report}
+extra-context =
+context = 
+  raw python_executable ${buildout:executable}
+  raw pt_query_executable ${perl:siteprefix}/bin/pt-query-digest
+  key output_folder :output-folder
+  $${:extra-context}
+
+[cron-mysql-slowquery-report]
+<= cron
+recipe = slapos.cookbook:cron.d
+name = mysql-slowquery-report
+frequency = 0 3 * * *
+command = $${monitor-directory:bin}/mysql_slowquery_report.py
+
 [monitor-instance-log-access]
 recipe = plone.recipe.command
 command = if [ -d $${:source} ]; then ln -s $${:source} $${monitor-directory:private-directory}/instance-logs; fi

stack/monitor/mysql-querydigest.py.in

+#!{{ python_executable }}
+# BEWARE: This file is operated by slapgrid
+# BEWARE: It will be overwritten automatically
+
+import os, errno
+import subprocess
+from datetime import date
+import gzip
+
+slow_query_path = "{{ slow_query_path }}".strip()
+output_folder = "{{ output_folder }}".strip()
+
+if not slow_query_path:
+  exit(0)
+
+#pt-query-digest --processlist h=host,user=user,password=pass --output=slow.log
+today = date.today().strftime("%Y-%m-%d")
+folder_month = os.path.join(output_folder, 'Mysql-SlowQuery-Digest-%s' % 
+                                date.today().strftime("%Y-%m"))
+out_file = os.path.join(folder_month, 'slow_query_report_%s.log' % today)
+slow_log = slow_query_path % {'date': date.today().strftime("%Y%m%d")}
+delete = False
+
+if not os.path.exists(slow_log) or not os.path.isfile(slow_log):
+  print "ERROR: cannot read mysql slow query log file '%s'. Exiting..." % slow_log
+  exit(1)
+  
+try:
+  os.makedirs(folder_month)
+except OSError as exc:
+  if exc.errno == errno.EEXIST and os.path.isdir(folder_month):
+    pass
+  else: raise
+
+if slow_log.endswith('.gz'):
+  slow_query_log = os.path.join(folder_month, 'slow_%s.log' % today)
+  delete = True
+  with open(slow_query_log, 'w') as slowfile:
+    f = gzip.open(slow_log, 'rb')
+    slowfile.write(f.read())
+    f.close()
+else:
+  slow_query_log = slow_log
+
+pt_query_digest = "{{ pt_query_executable }}".strip()
+argument_list = [pt_query_digest, slow_query_log]
+output = open(out_file, 'w')
+
+subprocess.check_call(argument_list, stdout=output)
+output.close()
+if delete:
+  os.unlink(slow_query_log)
+