Commit 87b96e1b authored by Łukasz Nowak's avatar Łukasz Nowak

promise: Support threshold-days in check_free_disk_space

The most important information is if the disk won't become full in some time,
by default 30 days. This check uses collector db history to calculate speed
of filling the disk and predicts when it will become full.
parent ba523080
...@@ -22,7 +22,60 @@ class RunPromise(GenericPromise): ...@@ -22,7 +22,60 @@ class RunPromise(GenericPromise):
# check disk space at least every 3 minutes # check disk space at least every 3 minutes
self.setPeriodicity(minute=3) self.setPeriodicity(minute=3)
def getDaysUntilFull(self, disk_partition, database, date, time, day_range):
"""Returns estimation of days until the disk_partition would become full
It uses date and time in order to find current disk free percentage, then rewinds
day_range back in history and calculates average speed of losing free space, which
is assumed constant and used to predict in how many days the disk would become full.
"""
database = Database(database, create=False, timeout=10)
try:
# fetch free disk space
database.connect()
result_max = database.select(
"disk",
date,
columns="free*1.0/(used+free) AS percent, max(datetime(date || ' ' || time))",
where="time between '%s:00' and '%s:30' and partition='%s'" % (time, time, disk_partition),
limit=1,
).fetchone()
if not result_max or not result_max[1]:
return None
result_min = database.select(
"disk",
columns="free*1.0/(used+free) AS percent, min(datetime(date || ' ' || time))",
where="datetime(date || ' ' || time) >= datetime('%s', '-%s days') and partition='%s'" % (result_max[1], day_range, disk_partition,),
limit=1,
).fetchone()
if not result_min or not result_min[1] or result_min == result_max:
return None
change = result_max[0] - result_min[0]
if change > 0.:
return None
timep = '%Y-%m-%d %H:%M:%S'
timespan = datetime.datetime.strptime(
result_max[1], timep) - datetime.datetime.strptime(
result_min[1], timep)
delta_days = timespan.total_seconds() / (3600.*24)
try:
return (-(1. - result_max[0]) / (change / delta_days), result_min[1], result_max[1], delta_days)
  • result_max[0] is already the current free disk space so I don't see why you'd do 1. - result_max[0]

  • You're right. I will apply it and update tests.

Please register or sign in to reply
except ZeroDivisionError as e:
# no data
return None
except sqlite3.OperationalError as e:
# if database is still locked after timeout expiration (another process is using it)
# we print warning message and try the promise at next run until max warn count
locked_message = "database is locked"
if locked_message in str(e) and \
not self.raiseOnDatabaseLocked(locked_message):
return None
raise
finally:
try:
database.close()
except Exception:
pass
def getDiskSize(self, disk_partition, database): def getDiskSize(self, disk_partition, database):
database = Database(database, create=False, timeout=10) database = Database(database, create=False, timeout=10)
...@@ -168,6 +221,16 @@ class RunPromise(GenericPromise): ...@@ -168,6 +221,16 @@ class RunPromise(GenericPromise):
free_space = self.getFreeSpace(disk_partition, db_path, currentdate, free_space = self.getFreeSpace(disk_partition, db_path, currentdate,
currenttime) currenttime)
days_until_full_tuple = self.getDaysUntilFull(disk_partition, db_path, currentdate, currenttime, threshold_days/2)
if days_until_full_tuple is not None:
days_until_full, min_date, max_date, day_span = days_until_full_tuple
message = "Disk will become full in %.2f days (threshold: %.2f days), checked from %s to %s, %.2f days span" % (
days_until_full, threshold_days, min_date, max_date, day_span)
if days_until_full < threshold_days:
self.logger.error(message + ': ERROR')
else:
self.logger.info(message + ': OK')
if free_space == 0: if free_space == 0:
return return
elif free_space > threshold*1024*1024*1024: elif free_space > threshold*1024*1024*1024:
......
BEGIN TRANSACTION; BEGIN TRANSACTION;
CREATE TABLE disk (partition text, used text, free text, mountpoint text, date text, time text, reported integer NULL DEFAULT 0); CREATE TABLE disk (partition text, used text, free text, mountpoint text, date text, time text, reported integer NULL DEFAULT 0);
INSERT INTO "disk" VALUES('/dev/sda1','157210666368','290958396032','/','2017-10-02','01:16:01',1);
INSERT INTO "disk" VALUES('/dev/sda1','159220666368','288948396032','/','2017-10-02','09:17:01',1); INSERT INTO "disk" VALUES('/dev/sda1','159220666368','288948396032','/','2017-10-02','09:17:01',1);
INSERT INTO "disk" VALUES('/dev/sda1','159237537792','288931524608','/','2017-10-02','09:18:01',1); INSERT INTO "disk" VALUES('/dev/sda1','159237537792','288931524608','/','2017-10-02','09:18:01',1);
INSERT INTO "disk" VALUES('/dev/sda1','159238090752','288930971648','/','2017-10-02','09:19:02',1); INSERT INTO "disk" VALUES('/dev/sda1','159238090752','288930971648','/','2017-10-02','09:19:02',1);
......
...@@ -54,6 +54,7 @@ class TestCheckFreeDiskSpace(TestPromisePluginMixin): ...@@ -54,6 +54,7 @@ class TestCheckFreeDiskSpace(TestPromisePluginMixin):
extra_config_dict = { extra_config_dict = {
'collectordb': '%(collectordb)s', 'collectordb': '%(collectordb)s',
'test-check-date': '2017-10-02', 'test-check-date': '2017-10-02',
'threshold-days': '25'
} }
""" % {'collectordb': self.db_file} """ % {'collectordb': self.db_file}
self.writePromise(self.promise_name, content) self.writePromise(self.promise_name, content)
...@@ -69,7 +70,8 @@ extra_config_dict = { ...@@ -69,7 +70,8 @@ extra_config_dict = {
extra_config_dict = { extra_config_dict = {
'collectordb': '%(collectordb)s', 'collectordb': '%(collectordb)s',
'test-check-date': '2017-09-14', 'test-check-date': '2017-09-14',
'test-check-time': '18:00' 'test-check-time': '18:00',
'threshold-days': '25'
} }
""" % {'collectordb': self.db_file} """ % {'collectordb': self.db_file}
self.writePromise(self.promise_name, content) self.writePromise(self.promise_name, content)
...@@ -85,7 +87,26 @@ extra_config_dict = { ...@@ -85,7 +87,26 @@ extra_config_dict = {
self.launcher.run() self.launcher.run()
result = self.getPromiseResult(self.promise_name) result = self.getPromiseResult(self.promise_name)
self.assertEqual(result['result']['failed'], False) self.assertEqual(result['result']['failed'], False)
self.assertEqual(result['result']['message'], "Disk usage: OK") self.assertEqual(result['result']['message'],
"Disk will become full in 26.46 days (threshold: 25.00 days), checked from 2017-10-02 01:16:01 to 2017-10-02 09:17:01, 0.33 days span: OK\nDisk usage: OK")
def test_disk_space_ok_will_full_soon(self):
content = """from slapos.promise.plugin.check_free_disk_space import RunPromise
extra_config_dict = {
'collectordb': '%(collectordb)s',
'test-check-date': '2017-10-02',
}
""" % {'collectordb': self.db_file}
self.writePromise(self.promise_name, content)
self.configureLauncher()
with self.assertRaises(PromiseError):
self.launcher.run()
result = self.getPromiseResult(self.promise_name)
self.assertEqual(result['result']['failed'], True)
self.assertEqual(
result['result']['message'],
"Disk will become full in 26.46 days (threshold: 30.00 days), checked from 2017-10-02 01:16:01 to 2017-10-02 09:17:01, 0.33 days span: ERROR\nDisk usage: OK")
def test_disk_space_nok(self): def test_disk_space_nok(self):
content = """from slapos.promise.plugin.check_free_disk_space import RunPromise content = """from slapos.promise.plugin.check_free_disk_space import RunPromise
...@@ -94,6 +115,7 @@ extra_config_dict = { ...@@ -94,6 +115,7 @@ extra_config_dict = {
'collectordb': '%(collectordb)s', 'collectordb': '%(collectordb)s',
'test-check-date': '2017-10-02', 'test-check-date': '2017-10-02',
'threshold': '278', 'threshold': '278',
'threshold-days': '25',
} }
""" % {'collectordb': self.db_file} """ % {'collectordb': self.db_file}
self.writePromise(self.promise_name, content) self.writePromise(self.promise_name, content)
...@@ -104,14 +126,26 @@ extra_config_dict = { ...@@ -104,14 +126,26 @@ extra_config_dict = {
result = self.getPromiseResult(self.promise_name) result = self.getPromiseResult(self.promise_name)
self.assertEqual(result['result']['failed'], True) self.assertEqual(result['result']['failed'], True)
self.assertEqual(result['result']['message'], self.assertEqual(result['result']['message'],
"Free disk space low: remaining 269.1 G (threshold: 278.0 G)") "Disk will become full in 26.46 days (threshold: 25.00 days), checked from 2017-10-02 01:16:01 to 2017-10-02 09:17:01, 0.33 days span: OK\nFree disk space low: remaining 269.1 G (threshold: 278.0 G)")
def test_disk_space_nok_will_full_soon(self):
content = """from slapos.promise.plugin.check_free_disk_space import RunPromise
extra_config_dict = {
'collectordb': '%(collectordb)s',
'test-check-date': '2017-10-02',
'threshold': '278',
}
""" % {'collectordb': self.db_file}
self.writePromise(self.promise_name, content)
self.configureLauncher() self.configureLauncher()
with self.assertRaises(PromiseError): with self.assertRaises(PromiseError):
self.launcher.run() self.launcher.run()
result = self.getPromiseResult(self.promise_name) result = self.getPromiseResult(self.promise_name)
self.assertEqual(result['result']['failed'], True) self.assertEqual(result['result']['failed'], True)
self.assertEqual(result['result']['message'], "Free disk space low: remaining 269.1 G (threshold: 278.0 G)") self.assertEqual(result['result']['message'],
"Disk will become full in 26.46 days (threshold: 30.00 days), checked from 2017-10-02 01:16:01 to 2017-10-02 09:17:01, 0.33 days span: ERROR\nFree disk space low: remaining 269.1 G (threshold: 278.0 G)")
def test_check_free_disk_with_unicode_string_path(self): def test_check_free_disk_with_unicode_string_path(self):
# set path unicode # set path unicode
...@@ -120,7 +154,7 @@ extra_config_dict = { ...@@ -120,7 +154,7 @@ extra_config_dict = {
self.launcher.run() self.launcher.run()
result = self.getPromiseResult(self.promise_name) result = self.getPromiseResult(self.promise_name)
self.assertEqual(result['result']['failed'], False) self.assertEqual(result['result']['failed'], False)
self.assertEqual(result['result']['message'], "Disk usage: OK") self.assertEqual(result['result']['message'], "Disk will become full in 26.46 days (threshold: 25.00 days), checked from 2017-10-02 01:16:01 to 2017-10-02 09:17:01, 0.33 days span: OK\nDisk usage: OK")
if __name__ == '__main__': if __name__ == '__main__':
unittest.main() unittest.main()
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment