Commit 30e7be4e by Rafael Monnerat

[slapos.collect] Implement garbage collection for the sqlite

  Remove older registers, already reported (or exported to csv)
  (older them 3 days)
1 parent 7048eca5
......@@ -61,6 +61,9 @@ def do_collect(conf):
SystemCSVReporterDumper(database).dump(log_directory)
RawCSVDumper(database).dump(log_directory)
# Drop older entries already reported
database.garbageCollect()
except AccessDenied:
print "You HAVE TO execute this script with root permission."
import sqlite3
import os
from time import time, strftime
import datetime
class Database:
......@@ -150,6 +151,34 @@ class Database:
return [i[0] for i in self._execute(
"SELECT name FROM sqlite_master WHERE type='table'")]
def _getGarbageCollectionDateList(self, days_to_preserve=3):
""" Return the list of dates to Preserve when data collect
"""
base = datetime.datetime.today()
date_list = []
for x in range(0, days_to_preserve):
date_list.append((base - datetime.timedelta(days=x)).strftime("%Y-%m-%d"))
return date_list
def garbageCollect(self):
""" Garbase collect the database, by removing older records already
reported.
"""
date_list = self._getGarbageCollectionDateList()
print date_list
where_clause = "reported = 1"
for _date in date_list:
where_clause += " AND date != '%s' " % _date
delete_sql = "DELETE FROM %s WHERE %s"
self.connect()
for table in self.table_list:
self._execute(delete_sql % (table, where_clause))
self.commit()
self.close()
def getDateScopeList(self, ignore_date=None, reported=0):
""" Get from the present unique dates from the system
Use a smaller table to sabe time.
......
......@@ -167,6 +167,40 @@ class TestCollectDatabase(unittest.TestCase):
finally:
database.close()
def test_garbage_collection_date_list(self):
database = db.Database(self.instance_root)
self.assertEquals(len(database._getGarbageCollectionDateList()), 3)
self.assertEquals(len(database._getGarbageCollectionDateList(1)), 1)
self.assertEquals(len(database._getGarbageCollectionDateList(0)), 0)
self.assertEquals(database._getGarbageCollectionDateList(1),
[strftime("%Y-%m-%d")])
def test_garbage(self):
database = db.Database(self.instance_root)
database.connect()
database.insertSystemSnapshot("0.1", '10.0', '100.0', '100.0',
'10.0', '1', '2', '12.0', '1', '1', '1983-01-10', 'TIME')
database.insertDiskPartitionSnapshot(
'/dev/sdx1', '10', '20', '/mnt', '1983-01-10', 'TIME')
database.insertComputerSnapshot(
'1', '0', '0', '100', '0', '/dev/sdx1', '1983-01-10', 'TIME')
database.commit()
database.markDayAsReported(date_scope="1983-01-10",
table_list=database.table_list)
database.commit()
self.assertEquals(len([x for x in database.select('system')]), 1)
self.assertEquals(len([x for x in database.select('computer')]), 1)
self.assertEquals(len([x for x in database.select('disk')]), 1)
database.close()
database.garbageCollect()
database.connect()
self.assertEquals(len([x for x in database.select('system')]), 0)
self.assertEquals(len([x for x in database.select('computer')]), 0)
self.assertEquals(len([x for x in database.select('disk')]), 0)
def test_mark_day_as_reported(self):
......
Styling with Markdown is supported
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!