Commit ba523080 authored by Łukasz Nowak's avatar Łukasz Nowak

promise: Support simple threshold in check_free_disk_space

Instead of outdated approach with threshold-file just support threshold,
provided in float in GB, defaulting to 5% of disk size if found in the
database and fallback to 2.0 GB, to raise promise error.
parent 42bd8c1a
...@@ -22,8 +22,36 @@ class RunPromise(GenericPromise): ...@@ -22,8 +22,36 @@ class RunPromise(GenericPromise):
# check disk space at least every 3 minutes # check disk space at least every 3 minutes
self.setPeriodicity(minute=3) self.setPeriodicity(minute=3)
def getFreeSpace(self, disk_partition, database, date, time):
def getDiskSize(self, disk_partition, database):
database = Database(database, create=False, timeout=10)
try:
# fetch disk size
database.connect()
where_query = "partition='%s'" % (disk_partition)
order = "datetime(date || ' ' || time) DESC"
query_result = database.select("disk", columns="free+used", where=where_query, order=order, limit=1)
result = query_result.fetchone()
if not result or not result[0]:
return None
disk_size = result[0]
except sqlite3.OperationalError as e:
# if database is still locked after timeout expiration (another process is using it)
# we print warning message and try the promise at next run until max warn count
locked_message = "database is locked"
if locked_message in str(e) and \
not self.raiseOnDatabaseLocked(locked_message):
return None
raise
finally:
try:
database.close()
except Exception:
pass
return disk_size
def getFreeSpace(self, disk_partition, database, date, time):
database = Database(database, create=False, timeout=10) database = Database(database, create=False, timeout=10)
try: try:
# fetch free disk space # fetch free disk space
...@@ -99,7 +127,6 @@ class RunPromise(GenericPromise): ...@@ -99,7 +127,6 @@ class RunPromise(GenericPromise):
def sense(self): def sense(self):
# find if a disk is mounted on the path # find if a disk is mounted on the path
disk_partition = "" disk_partition = ""
disk_threshold_file = self.getConfig('threshold-file')
db_path = self.getConfig('collectordb') db_path = self.getConfig('collectordb')
check_date = self.getConfig('test-check-date') check_date = self.getConfig('test-check-date')
path = os.path.join(self.getPartitionFolder(), "") + "extrafolder" path = os.path.join(self.getPartitionFolder(), "") + "extrafolder"
...@@ -117,23 +144,8 @@ class RunPromise(GenericPromise): ...@@ -117,23 +144,8 @@ class RunPromise(GenericPromise):
self.logger.error("Couldn't find disk partition") self.logger.error("Couldn't find disk partition")
return return
min_free_size = 1024*1024*1024*2 # 2G by default if db_path.endswith("collector.db"):
if disk_threshold_file is not None: db_path=db_path[:-len("collector.db")]
if os.path.exists(disk_threshold_file):
with open(disk_threshold_file) as f:
min_size_str = f.read().strip()
if min_size_str == '0':
# disable check
self.logger.info("Free disk space check is disabled\n set a number up to 0 to enable!")
return
if min_size_str.isdigit():
value = int(min_size_str)
if value >= 200:
# Minimum value is 200Mb, it's already low
min_free_size = int(min_size_str)*1024*1024
else:
with open(disk_threshold_file, 'w') as f:
f.write(str(min_free_size//(1024*1024)))
if check_date: if check_date:
# testing mode # testing mode
...@@ -147,14 +159,18 @@ class RunPromise(GenericPromise): ...@@ -147,14 +159,18 @@ class RunPromise(GenericPromise):
currenttime = now - datetime.timedelta(minutes=1) currenttime = now - datetime.timedelta(minutes=1)
currenttime = currenttime.time().strftime('%H:%M') currenttime = currenttime.time().strftime('%H:%M')
if db_path.endswith("collector.db"): disk_size = self.getDiskSize(disk_partition, db_path)
db_path=db_path[:-len("collector.db")] default_threshold = None
if disk_size is not None:
default_threshold = round(disk_size/(1024*1024*1024) * 0.05, 2)
threshold = float(self.getConfig('threshold', default_threshold) or 2.0)
threshold_days = float(self.getConfig('threshold-days', '30'))
free_space = self.getFreeSpace(disk_partition, db_path, currentdate, free_space = self.getFreeSpace(disk_partition, db_path, currentdate,
currenttime) currenttime)
if free_space == 0: if free_space == 0:
return return
elif free_space > min_free_size: elif free_space > threshold*1024*1024*1024:
inode_usage = self.getInodeUsage(self.getPartitionFolder()) inode_usage = self.getInodeUsage(self.getPartitionFolder())
if inode_usage: if inode_usage:
self.logger.error(inode_usage) self.logger.error(inode_usage)
...@@ -163,9 +179,8 @@ class RunPromise(GenericPromise): ...@@ -163,9 +179,8 @@ class RunPromise(GenericPromise):
return return
free_space = round(free_space/(1024*1024*1024), 2) free_space = round(free_space/(1024*1024*1024), 2)
min_space = round(min_free_size/(1024*1024*1024), 2)
self.logger.error('Free disk space low: remaining %s G (threshold: %s G)' % ( self.logger.error('Free disk space low: remaining %s G (threshold: %s G)' % (
free_space, min_space)) free_space, threshold))
def test(self): def test(self):
return self._test(result_count=1, failure_amount=1) return self._test(result_count=1, failure_amount=1)
......
...@@ -48,18 +48,14 @@ class TestCheckFreeDiskSpace(TestPromisePluginMixin): ...@@ -48,18 +48,14 @@ class TestCheckFreeDiskSpace(TestPromisePluginMixin):
self.conn.close() self.conn.close()
self.promise_name = "check-free-disk-space.py" self.promise_name = "check-free-disk-space.py"
self.th_file = os.path.join(self.partition_dir, 'min-disk-value')
with open(self.th_file, 'w') as f:
f.write('2048')
content = """from slapos.promise.plugin.check_free_disk_space import RunPromise content = """from slapos.promise.plugin.check_free_disk_space import RunPromise
extra_config_dict = { extra_config_dict = {
'collectordb': '%(collectordb)s', 'collectordb': '%(collectordb)s',
'threshold-file': '%(th_file)s',
'test-check-date': '2017-10-02', 'test-check-date': '2017-10-02',
} }
""" % {'collectordb': self.db_file, 'th_file': self.th_file} """ % {'collectordb': self.db_file}
self.writePromise(self.promise_name, content) self.writePromise(self.promise_name, content)
def tearDown(self): def tearDown(self):
...@@ -72,11 +68,10 @@ extra_config_dict = { ...@@ -72,11 +68,10 @@ extra_config_dict = {
extra_config_dict = { extra_config_dict = {
'collectordb': '%(collectordb)s', 'collectordb': '%(collectordb)s',
'threshold-file': '%(th_file)s',
'test-check-date': '2017-09-14', 'test-check-date': '2017-09-14',
'test-check-time': '18:00' 'test-check-time': '18:00'
} }
""" % {'collectordb': self.db_file, 'th_file': self.th_file} """ % {'collectordb': self.db_file}
self.writePromise(self.promise_name, content) self.writePromise(self.promise_name, content)
self.configureLauncher() self.configureLauncher()
...@@ -93,14 +88,30 @@ extra_config_dict = { ...@@ -93,14 +88,30 @@ extra_config_dict = {
self.assertEqual(result['result']['message'], "Disk usage: OK") self.assertEqual(result['result']['message'], "Disk usage: OK")
def test_disk_space_nok(self): def test_disk_space_nok(self):
with open(self.th_file, 'w') as f: content = """from slapos.promise.plugin.check_free_disk_space import RunPromise
f.write('298927494144')
extra_config_dict = {
'collectordb': '%(collectordb)s',
'test-check-date': '2017-10-02',
'threshold': '278',
}
""" % {'collectordb': self.db_file}
self.writePromise(self.promise_name, content)
self.configureLauncher()
with self.assertRaises(PromiseError):
self.launcher.run()
result = self.getPromiseResult(self.promise_name)
self.assertEqual(result['result']['failed'], True)
self.assertEqual(result['result']['message'],
"Free disk space low: remaining 269.1 G (threshold: 278.0 G)")
self.configureLauncher() self.configureLauncher()
with self.assertRaises(PromiseError): with self.assertRaises(PromiseError):
self.launcher.run() self.launcher.run()
result = self.getPromiseResult(self.promise_name) result = self.getPromiseResult(self.promise_name)
self.assertEqual(result['result']['failed'], True) self.assertEqual(result['result']['failed'], True)
self.assertEqual(result['result']['message'], "Free disk space low: remaining 269.1 G (threshold: 291921381.0 G)") self.assertEqual(result['result']['message'], "Free disk space low: remaining 269.1 G (threshold: 278.0 G)")
def test_check_free_disk_with_unicode_string_path(self): def test_check_free_disk_with_unicode_string_path(self):
# set path unicode # set path unicode
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment