slapos.collect: Compress historical logs and fix folder permissions.

slapos/collect/__init__.py

@@ -27,16 +27,17 @@
 #
 ##############################################################################
 
-
 from psutil import process_iter, NoSuchProcess, AccessDenied
 from time import strftime
 from slapos.collect.db import Database
 from slapos.util import mkdir_p
-
+import os
+import stat
 
 from slapos.collect.snapshot import ProcessSnapshot, ComputerSnapshot
 from slapos.collect.reporter import RawCSVDumper, \
-                                    SystemCSVReporterDumper
+                                    SystemCSVReporterDumper, \
+                                    compressLogFolder
 
 from entity import get_user_list, Computer
 
@@ -80,6 +81,10 @@ def do_collect(conf):
       
     log_directory = "%s/var/data-log" % conf.get("slapos", "instance_root")
     mkdir_p(log_directory, 0o755)
+
+    if stat.S_IMODE(os.stat(log_directory).st_mode) != 0o755:
+      os.chmod(log_directory, 0o755)    
+
     database = Database(log_directory)
 
     computer = Computer(ComputerSnapshot())
@@ -91,6 +96,8 @@ def do_collect(conf):
     SystemCSVReporterDumper(database).dump(log_directory)
     RawCSVDumper(database).dump(log_directory)
 
+    compressLogFolder(log_directory)
+
     # Drop older entries already reported
     database.garbageCollect()
 

slapos/collect/reporter.py

@@ -31,6 +31,9 @@
 from slapos.util import mkdir_p
 import os.path
 import json
+import tarfile
+import glob
+import shutil
 import csv
 from time import strftime
 
@@ -39,6 +42,12 @@ class Dumper(object):
   def __init__(self, database):
     self.db = database
 
+  def dump(self, folder):
+    raise NotImplemented("Implemented on Subclass")
+
+  def writeFile(self, **kw):
+    raise NotImplemented("Implemented on Subclass")
+
 class SystemReporter(Dumper):
   
   def dump(self, folder):
@@ -94,9 +103,25 @@ class RawDumper(Dumper):
 class RawCSVDumper(RawDumper):
   
   def writeFile(self, name, folder, date_scope, rows):
-    mkdir_p(os.path.join(folder, date_scope))
+    mkdir_p(os.path.join(folder, date_scope), 0o755)
     file_io = open(os.path.join(folder, "%s/dump_%s.csv" % (date_scope, name)), "w")
     csv_output = csv.writer(file_io)
     csv_output.writerows(rows)
     file_io.close()
 
+def compressLogFolder(log_directory):
+  
+    initial_folder = os.getcwd()
+    os.chdir(log_directory)
+    try:
+      for backup_to_archive in glob.glob("*-*-*/"):
+        filename = '%s.tar.gz' % backup_to_archive.strip("/")
+        with tarfile.open(filename, 'w:gz') as tfile:
+          tfile.add(backup_to_archive)
+          tfile.close() 
+        shutil.rmtree(backup_to_archive)
+    finally:
+      os.chdir(initial_folder)
+
+
+

slapos/tests/collect.py

@@ -29,6 +29,7 @@ import os
 import glob
 import unittest
 import shutil
+import tarfile
 import tempfile
 import slapos.slap
 import psutil
@@ -290,6 +291,32 @@ class TestCollectReport(unittest.TestCase):
 
         self.assertEquals(set(glob.glob("%s/*.csv" % self.instance_root)), set(csv_path_list)) 
 
+    def test_compress_log_directory(self):
+        log_directory = "%s/test_compress" % self.instance_root
+        dump_folder = "%s/1990-01-01" % log_directory
+
+        if not os.path.exists(log_directory):
+            os.mkdir(log_directory)
+
+        if os.path.exists(dump_folder):
+            shutil.rmtree(dump_folder)
+
+        os.mkdir("%s/1990-01-01" % log_directory)
+        with open("%s/test.txt" % dump_folder, "w") as dump_file:
+            dump_file.write("hi")
+            dump_file.close()
+
+        reporter.compressLogFolder(log_directory)
+
+        self.assertFalse(os.path.exists(dump_folder))
+
+        self.assertTrue(os.path.exists("%s.tar.gz" % dump_folder))
+
+        with tarfile.open("%s.tar.gz" % dump_folder) as tf:
+            self.assertEquals(tf.getmembers()[0].name, "1990-01-01")
+            self.assertEquals(tf.getmembers()[1].name, "1990-01-01/test.txt")
+            self.assertEquals(tf.extractfile(tf.getmembers()[1]).read(), 'hi')
+
 class TestCollectSnapshot(unittest.TestCase):
 
     def setUp(self):