promise: move check_free_disk_space promise to promise.plugin

check_free_disk_space is now in promise.plugin, with some improvement.
Database connection timeout is now 10 seconds
If database is locked, promise will try again 10 times before start to raise error with "database is locked" message.

setup.py

@@ -72,7 +72,6 @@ setup(name=name,
           'agent = slapos.agent.agent:main',
           'apache-mpm-watchdog = slapos.promise.apache_mpm_watchdog:main',
           'check-computer-memory = slapos.promise.check_computer_memory:main',
-          'check-free-disk = slapos.promise.check_free_disk:main',
           'check-web-page-http-cache-hit = slapos.promise.check_web_page_http_cache_hit:main',
           'check-feed-as-promise = slapos.checkfeedaspromise:main',
           'check-error-on-apache-log = slapos.promise.check_error_on_apache_log:main',

slapos/promise/check_free_disk/__init__.py

-#!/usr/bin/env python
-
-"""
-Check if free disk space is less than given threshold.
-"""
-
-import sys
-import os
-
-import sqlite3
-import argparse
-import datetime
-import psutil
-
-from slapos.collect.db import Database
-
-def getFreeSpace(disk_partition, database, date, time):
-
-  database = Database(database, create=False, timeout=5)
-  try:
-    # fetch free disk space
-    database.connect()
-    where_query = "time between '%s:00' and '%s:30' and partition='%s'" % (time, time, disk_partition)
-    query_result = database.select("disk", date, "free", where=where_query)
-    result = zip(*query_result)
-    if not result or not result[0][0]: 
-      print "No result from collector database: disk check skipped"
-      return 0
-    disk_free = result[0][0]
-  finally:
-    database.close()
-  return int(disk_free)
-
-
-def getInodeUsage(path):
-  max_inode_usage = 97.99 # < 98% usage
-  stat = os.statvfs(path)
-  usage_output = ""
-  total_inode = stat.f_files
-  free_inode = stat.f_ffree
-  usage = round((float(total_inode - free_inode) / total_inode), 4) * 100
-  if usage > max_inode_usage:
-    return "Disk Inodes usages is really high: %s%%" % usage
-  elif os.path.exists('/tmp'):
-    # check if /tmp is mounted on another disk than path
-    tmp_stat = os.statvfs('/tmp')
-    if tmp_stat.f_blocks != stat.f_blocks:
-      tmp_usage = round((float(tmp_stat.f_files - tmp_stat.f_ffree) / tmp_stat.f_files), 4) * 100
-      if tmp_usage > max_inode_usage:
-        return "Disk Inodes usage is high: %s%%" % tmp_usage
-  return ""
-
-def main():
-  parser = argparse.ArgumentParser()
-  parser.add_argument("--collectordb", required=True)
-  parser.add_argument("--home_path", required=True)
-  parser.add_argument("--config", required=True)
-  args = parser.parse_args()
-
-  # find if a disk is mounted on the path
-  disk_partition = ""
-  path = os.path.join(args.home_path, "") +"extrafolder"
-  partitions = psutil.disk_partitions()
-  while path is not '/':
-    if not disk_partition:
-      path = os.path.dirname(path)
-    else:
-      break
-    for p in partitions:
-      if p.mountpoint is path:
-        disk_partition = p.device
-        break
-  if not disk_partition:
-    print "Couldn't find disk partition"
-    exit(2)
-
-  min_free_size = 1024*1024*1024*2 # 2G by default
-  if os.path.exists(args.config):
-    with open(args.config) as f:
-      min_size_str = f.read().strip()
-      if min_size_str == '0':
-        # disable check
-        print "Free disk space check is disabled\n set a number up to 0 to enable!"
-        exit(0)
-      if min_size_str.isdigit():
-        value = int(min_size_str)
-        if value >= 200:
-          # Minimum value is 200Mb, it's already low
-          min_free_size = int(min_size_str)*1024*1024
-  else:
-    with open(args.config, 'w') as f:
-      f.write(str(min_free_size/(1024*1024)))
-
-  # get last minute
-  now = datetime.datetime.now()
-  currentdate = now.strftime('%Y-%m-%d')
-  currenttime = now - datetime.timedelta(minutes=1)
-  currenttime = currenttime.time().strftime('%H:%M')
-
-  db_path = args.collectordb
-  if db_path.endswith("collector.db"):
-    db_path=db_path[:-len("collector.db")]
-
-  free_space = getFreeSpace(disk_partition, db_path, currentdate, currenttime)
-  if free_space and free_space > min_free_size:
-    inode_usage = getInodeUsage(args.home_path)
-    if inode_usage:
-      print inode_usage
-      exit(2)
-    print "Disk usage: OK"
-    exit(0)
-  
-  free_space = round(free_space/(1024.0*1024*1024), 2)
-  min_space = round(min_free_size/(1024.0*1024*1024), 2)
-  print 'Free disk space low: remaining %s G (threshold: %s G)' % (
-    free_space, min_space)
-  print 'Please modify minimum value in your monitor interface.'
-  exit(1)
-
-if __name__ == "__main__":
-  sys.exit(main())
-

slapos/promise/plugin/check_free_disk_space.py

+from zope import interface as zope_interface
+from slapos.grid.promise import interface
+from slapos.grid.promise.generic import GenericPromise
+
+import os
+import sys
+
+import sqlite3
+import argparse
+import datetime
+import psutil
+
+from slapos.collect.db import Database
+
+class RunPromise(GenericPromise):
+
+  zope_interface.implements(interface.IPromise)
+
+  def __init__(self, config):
+    GenericPromise.__init__(self, config)
+    # check disk space at least every 3 minutes
+    self.setPeriodicity(minute=3)
+
+  def raiseOnDatabaseLocked(self, locked_message):
+    max_warn = 10
+    latest_result_list = self.getLastPromiseResultList(result_count=max_warn)
+    warning_count = 0
+    if len(latest_result_list) < max_warn:
+      return False
+
+    for result in latest_result_list[0]:
+      if result['status'] == "ERROR" and locked_message in result["message"]:
+        return True
+
+    for result_list in latest_result_list:
+      found = False
+      for result in result_list:
+        if result['status'] == "WARNING" and locked_message in result["message"]:
+          found = True
+          warning_count += 1
+          break
+      if not found:
+        break
+    if warning_count == max_warn:
+      # too many warning on database locked, now fail.
+      return True
+
+    self.logger.warn("collector database is locked by another process: %s %s" % (warning_count, len(latest_result_list)))
+    return False
+
+  def functio(self):
+    raise sqlite3.OperationalError("database is locked")
+
+  def getFreeSpace(self, disk_partition, database, date, time):
+
+    database = Database(database, create=False, timeout=10)
+    try:
+      # fetch free disk space
+      database.connect()
+      where_query = "time between '%s:00' and '%s:30' and partition='%s'" % (time, time, disk_partition)
+      query_result = database.select("disk", date, "free", where=where_query)
+      result = zip(*query_result)
+      if not result or not result[0][0]: 
+        self.logger.info("No result from collector database: disk check skipped")
+        return 0
+      disk_free = result[0][0]
+    except sqlite3.OperationalError, e:
+      # if database is still locked after timeout expiration (another process is using it)
+      # we print warning message and try the promise at next run until max warn msg
+      locked_message = "database is locked"
+      if locked_message in str(e) and \
+          not self.raiseOnDatabaseLocked(locked_message):
+        return 0
+      raise
+    finally:
+      database.close()
+      pass
+    return int(disk_free)
+  
+  
+  def getInodeUsage(self, path):
+    max_inode_usage = 97.99 # < 98% usage
+    stat = os.statvfs(path)
+    usage_output = ""
+    total_inode = stat.f_files
+    free_inode = stat.f_ffree
+    usage = round((float(total_inode - free_inode) / total_inode), 4) * 100
+    if usage > max_inode_usage:
+      return "Disk Inodes usages is really high: %s%%" % usage
+    elif os.path.exists('/tmp'):
+      # check if /tmp is mounted on another disk than path
+      tmp_stat = os.statvfs('/tmp')
+      if tmp_stat.f_blocks != stat.f_blocks:
+        tmp_usage = round((float(tmp_stat.f_files - tmp_stat.f_ffree) / tmp_stat.f_files), 4) * 100
+        if tmp_usage > max_inode_usage:
+          return "Disk Inodes usage is high: %s%%" % tmp_usage
+    return ""
+
+
+  def sense(self):
+    # find if a disk is mounted on the path
+    disk_partition = ""
+    disk_threshold_file = self.getConfig('threshold-file')
+    db_path = self.getConfig('collectordb')
+    check_date = self.getConfig('test-check-date')
+    path = os.path.join(self.getPartitionFolder(), "") + "extrafolder"
+    partitions = psutil.disk_partitions()
+    while path is not '/':
+      if not disk_partition:
+        path = os.path.dirname(path)
+      else:
+        break
+      for p in partitions:
+        if p.mountpoint is path:
+          disk_partition = p.device
+          break
+    if not disk_partition:
+      self.logger.error("Couldn't find disk partition")
+      return
+
+    min_free_size = 1024*1024*1024*2 # 2G by default
+    if disk_threshold_file is not None:
+      if os.path.exists(disk_threshold_file):
+        with open(disk_threshold_file) as f:
+          min_size_str = f.read().strip()
+          if min_size_str == '0':
+            # disable check
+            self.logger.info("Free disk space check is disabled\n set a number up to 0 to enable!")
+            return
+          if min_size_str.isdigit():
+            value = int(min_size_str)
+            if value >= 200:
+              # Minimum value is 200Mb, it's already low
+              min_free_size = int(min_size_str)*1024*1024
+      else:
+        with open(disk_threshold_file, 'w') as f:
+          f.write(str(min_free_size/(1024*1024)))
+
+    if check_date:
+      # testing mode
+      currentdate = check_date
+      currenttime = self.getConfig('test-check-time', '09:17')
+      disk_partition = self.getConfig('test-disk-partition', '/dev/sda1')
+    else:
+      # get last minute
+      now = datetime.datetime.now()
+      currentdate = now.strftime('%Y-%m-%d')
+      currenttime = now - datetime.timedelta(minutes=1)
+      currenttime = currenttime.time().strftime('%H:%M')
+
+    if db_path.endswith("collector.db"):
+      db_path=db_path[:-len("collector.db")]
+
+    free_space = self.getFreeSpace(disk_partition, db_path, currentdate, currenttime)
+    if free_space == 0:
+      return
+    elif free_space > min_free_size:
+      inode_usage = self.getInodeUsage(self.getPartitionFolder())
+      if inode_usage:
+        self.logger.error(inode_usage)
+      else:
+        self.logger.info("Disk usage: OK")
+      return
+
+    free_space = round(free_space/(1024.0*1024*1024), 2)
+    min_space = round(min_free_size/(1024.0*1024*1024), 2)
+    self.logger.error('Free disk space low: remaining %s G (threshold: %s G)' % (
+      free_space, min_space))
+
+  def test(self):
+    return self._test(result_count=1, failure_amount=1)
+
+  def anomaly(self):
+    return self._test(result_count=3, failure_amount=3)

slapos/test/promise/test_check_free_disk.py → slapos/test/promise/plugin/test_check_free_disk_space.py

 ##############################################################################
 #
-# Copyright (c) 2017 Vifib SARL and Contributors. All Rights Reserved.
+# Copyright (c) 2018 Vifib SARL and Contributors. All Rights Reserved.
 #
 # WARNING: This program as such is intended to be used by professional
 # programmers who take the whole responsibility of assessing all potential
@@ -25,26 +25,50 @@
 #
 ##############################################################################
 
-import unittest
+from slapos.test.promise.plugin import TestPromisePluginMixin
+from slapos.grid.promise import PromiseError
 import os
 import sqlite3
-
 from slapos.test.promise import data
 from slapos.promise.check_free_disk import getFreeSpace
+from slapos.grid.promise import PromiseError
 
-class TestFreeDisk(unittest.TestCase):
+class TestCheckFreeDiskSpace(TestPromisePluginMixin):
 
   def setUp(self):
-    self.base_path = "/".join(data.__file__.split("/")[:-1])
-    self.status = "ok"
+    TestPromisePluginMixin.setUp(self)
+    log_folder = os.path.join(self.partition_dir, 'var/log')
+    os.makedirs(log_folder)
+
     self.db_file = '/tmp/collector.db'
+    self.base_path = "/".join(data.__file__.split("/")[:-1])
 
     # populate db
     self.conn = sqlite3.connect(self.db_file)
     f = open(self.base_path+"/disktest.sql")
     sql = f.read()
     self.conn.executescript(sql)
-    self.conn.close() 
+    self.conn.close()
+
+    self.promise_name = "check-free-disk-space.py"
+    self.th_file = os.path.join(self.partition_dir, 'min-disk-value')
+    with open(self.th_file, 'w') as f:
+      f.write('2048')
+
+    content = """from slapos.promise.plugin.check_free_disk_space import RunPromise
+
+extra_config_dict = {
+  'collectordb': '%(collectordb)s',
+  'threshold-file': '%(th_file)s',
+  'test-check-date': '2017-10-02',
+}
+""" % {'collectordb': self.db_file, 'th_file': self.th_file}
+    self.writePromise(self.promise_name, content)
+
+  def tearDown(self):
+    TestPromisePluginMixin.tearDown(self)
+    if os.path.exists(self.db_file):
+      os.remove(self.db_file)
 
   def test_check_disk(self):
     self.assertEquals(288739385344,
@@ -53,10 +77,22 @@ class TestFreeDisk(unittest.TestCase):
   def test_check_free_disk_with_unavailable_dates(self):
     self.assertEquals(0, getFreeSpace('/', '/tmp', '18:00', '2017-09-14'))
 
-  def tearDown(self):
-    if os.path.exists(self.db_file):
-      os.remove(self.db_file)
+  def test_disk_space_ok(self):
+    self.configureLauncher()
+    self.launcher.run()
+    result = self.getPromiseResult(self.promise_name)
+    self.assertEquals(result['result']['failed'], False)
+    self.assertEquals(result['result']['message'], "Disk usage: OK")
+
+  def test_disk_space_nok(self):
+    with open(self.th_file, 'w') as f:
+      f.write('298927494144')
+    self.configureLauncher()
+    with self.assertRaises(PromiseError):
+      self.launcher.run()
+    result = self.getPromiseResult(self.promise_name)
+    self.assertEquals(result['result']['failed'], True)
+    self.assertEquals(result['result']['message'], "Free disk space low: remaining 269.1 G (threshold: 291921381.0 G)")
 
 if __name__ == '__main__':
   unittest.main()
-

[Jérome Perrin]

Isn't this error because slapos/test/promise/plugin/__init__.py is missing ?

[Alain Takoudjou]

@jerome thank you for pointing this out, I'm fixing it.