Commit 30e7be4e by Rafael Monnerat

[slapos.collect] Implement garbage collection for the sqlite

  Remove older registers, already reported (or exported to csv)
  (older them 3 days)
1 parent 7048eca5
...@@ -61,6 +61,9 @@ def do_collect(conf): ...@@ -61,6 +61,9 @@ def do_collect(conf):
SystemCSVReporterDumper(database).dump(log_directory) SystemCSVReporterDumper(database).dump(log_directory)
RawCSVDumper(database).dump(log_directory) RawCSVDumper(database).dump(log_directory)
# Drop older entries already reported
database.garbageCollect()
except AccessDenied: except AccessDenied:
print "You HAVE TO execute this script with root permission." print "You HAVE TO execute this script with root permission."
import sqlite3 import sqlite3
import os import os
from time import time, strftime from time import time, strftime
import datetime
class Database: class Database:
...@@ -150,6 +151,34 @@ class Database: ...@@ -150,6 +151,34 @@ class Database:
return [i[0] for i in self._execute( return [i[0] for i in self._execute(
"SELECT name FROM sqlite_master WHERE type='table'")] "SELECT name FROM sqlite_master WHERE type='table'")]
def _getGarbageCollectionDateList(self, days_to_preserve=3):
""" Return the list of dates to Preserve when data collect
"""
base = datetime.datetime.today()
date_list = []
for x in range(0, days_to_preserve):
date_list.append((base - datetime.timedelta(days=x)).strftime("%Y-%m-%d"))
return date_list
def garbageCollect(self):
""" Garbase collect the database, by removing older records already
reported.
"""
date_list = self._getGarbageCollectionDateList()
print date_list
where_clause = "reported = 1"
for _date in date_list:
where_clause += " AND date != '%s' " % _date
delete_sql = "DELETE FROM %s WHERE %s"
self.connect()
for table in self.table_list:
self._execute(delete_sql % (table, where_clause))
self.commit()
self.close()
def getDateScopeList(self, ignore_date=None, reported=0): def getDateScopeList(self, ignore_date=None, reported=0):
""" Get from the present unique dates from the system """ Get from the present unique dates from the system
Use a smaller table to sabe time. Use a smaller table to sabe time.
......
...@@ -167,6 +167,40 @@ class TestCollectDatabase(unittest.TestCase): ...@@ -167,6 +167,40 @@ class TestCollectDatabase(unittest.TestCase):
finally: finally:
database.close() database.close()
def test_garbage_collection_date_list(self):
database = db.Database(self.instance_root)
self.assertEquals(len(database._getGarbageCollectionDateList()), 3)
self.assertEquals(len(database._getGarbageCollectionDateList(1)), 1)
self.assertEquals(len(database._getGarbageCollectionDateList(0)), 0)
self.assertEquals(database._getGarbageCollectionDateList(1),
[strftime("%Y-%m-%d")])
def test_garbage(self):
database = db.Database(self.instance_root)
database.connect()
database.insertSystemSnapshot("0.1", '10.0', '100.0', '100.0',
'10.0', '1', '2', '12.0', '1', '1', '1983-01-10', 'TIME')
database.insertDiskPartitionSnapshot(
'/dev/sdx1', '10', '20', '/mnt', '1983-01-10', 'TIME')
database.insertComputerSnapshot(
'1', '0', '0', '100', '0', '/dev/sdx1', '1983-01-10', 'TIME')
database.commit()
database.markDayAsReported(date_scope="1983-01-10",
table_list=database.table_list)
database.commit()
self.assertEquals(len([x for x in database.select('system')]), 1)
self.assertEquals(len([x for x in database.select('computer')]), 1)
self.assertEquals(len([x for x in database.select('disk')]), 1)
database.close()
database.garbageCollect()
database.connect()
self.assertEquals(len([x for x in database.select('system')]), 0)
self.assertEquals(len([x for x in database.select('computer')]), 0)
self.assertEquals(len([x for x in database.select('disk')]), 0)
def test_mark_day_as_reported(self): def test_mark_day_as_reported(self):
......
Styling with Markdown is supported
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!