Commit f9edadf3 authored by Rafael Monnerat's avatar Rafael Monnerat

slapos.collect: Compress historical logs and fix folder permissions.

parent 9414e8e6
......@@ -27,16 +27,17 @@
#
##############################################################################
from psutil import process_iter, NoSuchProcess, AccessDenied
from time import strftime
from slapos.collect.db import Database
from slapos.util import mkdir_p
import os
import stat
from slapos.collect.snapshot import ProcessSnapshot, ComputerSnapshot
from slapos.collect.reporter import RawCSVDumper, \
SystemCSVReporterDumper
SystemCSVReporterDumper, \
compressLogFolder
from entity import get_user_list, Computer
......@@ -80,6 +81,10 @@ def do_collect(conf):
log_directory = "%s/var/data-log" % conf.get("slapos", "instance_root")
mkdir_p(log_directory, 0o755)
if stat.S_IMODE(os.stat(log_directory).st_mode) != 0o755:
os.chmod(log_directory, 0o755)
database = Database(log_directory)
computer = Computer(ComputerSnapshot())
......@@ -91,6 +96,8 @@ def do_collect(conf):
SystemCSVReporterDumper(database).dump(log_directory)
RawCSVDumper(database).dump(log_directory)
compressLogFolder(log_directory)
# Drop older entries already reported
database.garbageCollect()
......
......@@ -31,6 +31,9 @@
from slapos.util import mkdir_p
import os.path
import json
import tarfile
import glob
import shutil
import csv
from time import strftime
......@@ -39,6 +42,12 @@ class Dumper(object):
def __init__(self, database):
self.db = database
def dump(self, folder):
raise NotImplemented("Implemented on Subclass")
def writeFile(self, **kw):
raise NotImplemented("Implemented on Subclass")
class SystemReporter(Dumper):
def dump(self, folder):
......@@ -94,9 +103,25 @@ class RawDumper(Dumper):
class RawCSVDumper(RawDumper):
def writeFile(self, name, folder, date_scope, rows):
mkdir_p(os.path.join(folder, date_scope))
mkdir_p(os.path.join(folder, date_scope), 0o755)
file_io = open(os.path.join(folder, "%s/dump_%s.csv" % (date_scope, name)), "w")
csv_output = csv.writer(file_io)
csv_output.writerows(rows)
file_io.close()
def compressLogFolder(log_directory):
initial_folder = os.getcwd()
os.chdir(log_directory)
try:
for backup_to_archive in glob.glob("*-*-*/"):
filename = '%s.tar.gz' % backup_to_archive.strip("/")
with tarfile.open(filename, 'w:gz') as tfile:
tfile.add(backup_to_archive)
tfile.close()
shutil.rmtree(backup_to_archive)
finally:
os.chdir(initial_folder)
......@@ -29,6 +29,7 @@ import os
import glob
import unittest
import shutil
import tarfile
import tempfile
import slapos.slap
import psutil
......@@ -290,6 +291,32 @@ class TestCollectReport(unittest.TestCase):
self.assertEquals(set(glob.glob("%s/*.csv" % self.instance_root)), set(csv_path_list))
def test_compress_log_directory(self):
log_directory = "%s/test_compress" % self.instance_root
dump_folder = "%s/1990-01-01" % log_directory
if not os.path.exists(log_directory):
os.mkdir(log_directory)
if os.path.exists(dump_folder):
shutil.rmtree(dump_folder)
os.mkdir("%s/1990-01-01" % log_directory)
with open("%s/test.txt" % dump_folder, "w") as dump_file:
dump_file.write("hi")
dump_file.close()
reporter.compressLogFolder(log_directory)
self.assertFalse(os.path.exists(dump_folder))
self.assertTrue(os.path.exists("%s.tar.gz" % dump_folder))
with tarfile.open("%s.tar.gz" % dump_folder) as tf:
self.assertEquals(tf.getmembers()[0].name, "1990-01-01")
self.assertEquals(tf.getmembers()[1].name, "1990-01-01/test.txt")
self.assertEquals(tf.extractfile(tf.getmembers()[1]).read(), 'hi')
class TestCollectSnapshot(unittest.TestCase):
def setUp(self):
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment