added options: software_min_free_space (def.200MB) and instance_min_free_space (def.100MB)

slapos/grid/SlapObject.py

@@ -48,8 +48,9 @@ from slapos.grid import utils  # for methods that could be mocked, access them t
 from slapos.slap.slap import NotFoundError
 from slapos.grid.svcbackend import getSupervisorRPC
 from slapos.grid.exception import (BuildoutFailedError, WrongPermissionError,
-                                   PathDoesNotExistError)
+                                   PathDoesNotExistError, DiskSpaceError)
 from slapos.grid.networkcache import download_network_cached, upload_network_cached
+from slapos.human import bytes2human
 
 
 WATCHDOG_MARK = '-on-watch'
@@ -61,6 +62,38 @@ PROGRAM_PARTITION_TEMPLATE = pkg_resources.resource_stream(__name__,
             'templates/program_partition_supervisord.conf.in').read()
 
 
+def free_space(path, fn):
+  while True:
+    try:
+      disk = os.statvfs(path)
+      return fn(disk)
+    except OSError:
+      pass
+    if os.sep not in path:
+      break
+    path = os.path.split(path)[0]
+
+
+def free_space_root(path):
+  """
+  Returns free space available to the root user, in bytes.
+
+  A non-existent path can be provided, and the ancestors
+  will be queried instead.
+  """
+  return free_space(path, lambda d: d.bsize * d.f_bfree)
+
+
+def free_space_nonroot(path):
+  """
+  Returns free space available to non-root users, in bytes.
+
+  A non-existent path can be provided, and the ancestors
+  will be queried instead.
+  """
+  return free_space(path, lambda d: d.f_bsize * d.f_bavail)
+
+
 class Software(object):
   """This class is responsible for installing a software release"""
 
@@ -73,7 +106,8 @@ class Software(object):
                download_binary_cache_url=None, upload_binary_cache_url=None,
                download_binary_dir_url=None, upload_binary_dir_url=None,
                download_from_binary_cache_url_blacklist=None,
-               upload_to_binary_cache_url_blacklist=None):
+               upload_to_binary_cache_url_blacklist=None,
+               software_min_free_space=None):
     """Initialisation of class parameters
     """
 
@@ -106,6 +140,17 @@ class Software(object):
         download_from_binary_cache_url_blacklist
     self.upload_to_binary_cache_url_blacklist = \
         upload_to_binary_cache_url_blacklist
+    self.software_min_free_space = software_min_free_space
+
+  def check_free_space(self):
+    required = self.software_min_free_space
+    available = free_space_nonroot(self.software_path)
+
+    if available < required:
+      msg = "Not enough space for {path}: available {available}, required {required} (option 'software_min_free_space')"
+      raise DiskSpaceError(msg.format(path=self.software_path,
+                                      available=bytes2human(available),
+                                      required=bytes2human(required)))
 
   def install(self):
     """ Fetches binary cache if possible.
@@ -113,6 +158,9 @@ class Software(object):
     """
     self.logger.info("Installing software release %s..." % self.url)
     cache_dir = tempfile.mkdtemp()
+
+    self.check_free_space()
+
     try:
       tarpath = os.path.join(cache_dir, self.software_url_hash)
       # Check if we can download from cache
@@ -293,6 +341,7 @@ class Partition(object):
                logger,
                certificate_repository_path=None,
                retention_delay='0',
+               instance_min_free_space=None
                ):
     """Initialisation of class parameters"""
     self.buildout = buildout
@@ -333,6 +382,19 @@ class Partition(object):
         self.instance_path, self.retention_lock_date_filename
     )
 
+    self.instance_min_free_space = instance_min_free_space
+
+
+  def check_free_space(self):
+    required = self.instance_min_free_space
+    available = free_space_nonroot(self.instance_path)
+
+    if available < required:
+      msg = "Not enough space for {path}: available {available}, required {required} (option 'instance_min_free_space')"
+      raise DiskSpaceError(msg.format(path=self.instance_path,
+                                      available=bytes2human(available),
+                                      required=bytes2human(required)))
+
   def _updateCertificate(self):
     try:
       partition_certificate = self.computer_partition.getCertificate()
@@ -397,6 +459,9 @@ class Partition(object):
     """
     self.logger.info("Installing Computer Partition %s..."
         % self.computer_partition.getId())
+
+    self.check_free_space()
+
     # Checks existence and permissions of Partition directory
     # Note : Partitions have to be created and configured before running slapgrid
     if not os.path.isdir(self.instance_path):
@@ -505,6 +570,7 @@ class Partition(object):
                                 ['buildout:bin-directory=%s' %
                                     os.path.join(self.instance_path, 'sbin')])
       buildout_binary = os.path.join(self.instance_path, 'sbin', 'buildout')
+
     # Launches buildout
     utils.launchBuildout(path=self.instance_path,
                          buildout_binary=buildout_binary,

slapos/grid/exception.py

@@ -38,3 +38,8 @@ class WrongPermissionError(Exception):
 
 class BuildoutFailedError(Exception):
   pass
+
+
+class DiskSpaceError(Exception):
+  pass
+

slapos/grid/slapgrid.py

@@ -56,6 +56,7 @@ from slapos.grid.SlapObject import Software, Partition
 from slapos.grid.svcbackend import launchSupervisord
 from slapos.grid.utils import (md5digest, createPrivateDirectory, dropPrivileges,
                                SlapPopen, updateFile)
+from slapos.human import human2bytes
 import slapos.slap
 
 
@@ -201,6 +202,9 @@ def create_slapgrid_object(options, logger):
     ]
 
   op = options
+  software_min_free_space = human2bytes(op.get('software_min_free_space', '200M'))
+  instance_min_free_space = human2bytes(op.get('instance_min_free_space', '100M'))
+
   return Slapgrid(software_root=op['software_root'],
                   instance_root=op['instance_root'],
                   master_url=op['master_url'],
@@ -235,7 +239,9 @@ def create_slapgrid_object(options, logger):
                   # Try to fetch from deprecated argument
                   software_release_filter_list=op.get('only-sr', op.get('only_sr')),
                   # Try to fetch from deprecated argument
-                  computer_partition_filter_list=op.get('only-cp', op.get('only_cp')))
+                  computer_partition_filter_list=op.get('only-cp', op.get('only_cp')),
+                  software_min_free_space=software_min_free_space,
+                  instance_min_free_space=instance_min_free_space)
 
 
 def check_required_only_partitions(existing, required):
@@ -288,6 +294,8 @@ class Slapgrid(object):
                develop=False,
                software_release_filter_list=None,
                computer_partition_filter_list=None,
+               software_min_free_space=None,
+               instance_min_free_space=None,
                ):
     """Makes easy initialisation of class parameters"""
     # Parses arguments
@@ -339,6 +347,8 @@ class Slapgrid(object):
       self.computer_partition_filter_list = \
           computer_partition_filter_list.split(",")
     self.maximum_periodicity = maximum_periodicity
+    self.software_min_free_space = software_min_free_space
+    self.instance_min_free_space = instance_min_free_space
 
   def getWatchdogLine(self):
     invocation_list = [WATCHDOG_PATH]
@@ -433,7 +443,8 @@ class Slapgrid(object):
             shacache_cert_file=self.shacache_cert_file,
             shacache_key_file=self.shacache_key_file,
             shadir_cert_file=self.shadir_cert_file,
-            shadir_key_file=self.shadir_key_file)
+            shadir_key_file=self.shadir_key_file,
+            software_min_free_space=self.software_min_free_space)
         if state == 'available':
           completed_tag = os.path.join(software_path, '.completed')
           if (self.develop or (not os.path.exists(completed_tag) and
@@ -670,6 +681,7 @@ class Slapgrid(object):
         buildout=self.buildout,
         logger=self.logger,
         retention_delay=retention_delay,
+        instance_min_free_space=self.instance_min_free_space,
       )
       computer_partition_state = computer_partition.getState()
 

slapos/human.py

+#!/usr/bin/env python
+
+import sys
+
+"""
+Bytes-to-human / human-to-bytes converter.
+Based on: http://goo.gl/kTQMs
+Working with Python 2.x and 3.x.
+
+Author: Giampaolo Rodola' <g.rodola [AT] gmail [DOT] com>
+License: MIT
+"""
+
+# see: http://goo.gl/kTQMs
+SYMBOLS = {
+    'customary'     : ('B', 'K', 'M', 'G', 'T', 'P', 'E', 'Z', 'Y'),
+    'slapos'        : ('', 'KB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB'),
+    'customary_ext' : ('byte', 'kilo', 'mega', 'giga', 'tera', 'peta', 'exa',
+                       'zetta', 'iotta'),
+    'iec'           : ('Bi', 'Ki', 'Mi', 'Gi', 'Ti', 'Pi', 'Ei', 'Zi', 'Yi'),
+    'iec_ext'       : ('byte', 'kibi', 'mebi', 'gibi', 'tebi', 'pebi', 'exbi',
+                       'zebi', 'yobi'),
+}
+
+def bytes2human(n, format='%(value).1f %(symbol)s', symbols='slapos'):
+    """
+    Convert n bytes into a human readable string based on format.
+    symbols can be either "customary", "customary_ext", "iec" or "iec_ext",
+    see: http://goo.gl/kTQMs
+
+      >>> bytes2human(0)
+      '0.0 B'
+      >>> bytes2human(0.9)
+      '0.0 B'
+      >>> bytes2human(1)
+      '1.0 B'
+      >>> bytes2human(1.9)
+      '1.0 B'
+      >>> bytes2human(1024)
+      '1.0 K'
+      >>> bytes2human(1048576)
+      '1.0 M'
+      >>> bytes2human(1099511627776127398123789121)
+      '909.5 Y'
+
+      >>> bytes2human(9856, symbols="customary")
+      '9.6 K'
+      >>> bytes2human(9856, symbols="customary_ext")
+      '9.6 kilo'
+      >>> bytes2human(9856, symbols="iec")
+      '9.6 Ki'
+      >>> bytes2human(9856, symbols="iec_ext")
+      '9.6 kibi'
+
+      >>> bytes2human(10000, "%(value).1f %(symbol)s/sec")
+      '9.8 K/sec'
+
+      >>> # precision can be adjusted by playing with %f operator
+      >>> bytes2human(10000, format="%(value).5f %(symbol)s")
+      '9.76562 K'
+    """
+    n = int(n)
+    if n < 0:
+        raise ValueError("n < 0")
+    symbols = SYMBOLS[symbols]
+    prefix = {}
+    for i, s in enumerate(symbols[1:]):
+        prefix[s] = 1 << (i+1)*10
+    for symbol in reversed(symbols[1:]):
+        if n >= prefix[symbol]:
+            value = float(n) / prefix[symbol]
+            return format % locals()
+    return format % dict(symbol=symbols[0], value=n)
+
+def human2bytes(s):
+    """
+    Attempts to guess the string format based on default symbols
+    set and return the corresponding bytes as an integer.
+    When unable to recognize the format ValueError is raised.
+
+      >>> human2bytes('0 B')
+      0
+      >>> human2bytes('1 K')
+      1024
+      >>> human2bytes('1 M')
+      1048576
+      >>> human2bytes('1 Gi')
+      1073741824
+      >>> human2bytes('1 tera')
+      1099511627776
+
+      >>> human2bytes('0.5kilo')
+      512
+      >>> human2bytes('0.1  byte')
+      0
+      >>> human2bytes('1 k')  # k is an alias for K
+      1024
+      >>> human2bytes('12 foo')
+      Traceback (most recent call last):
+          ...
+      ValueError: can't interpret '12 foo'
+    """
+    init = s
+    num = ""
+    while s and s[0:1].isdigit() or s[0:1] == '.':
+        num += s[0]
+        s = s[1:]
+    num = float(num)
+    letter = s.strip()
+    for name, sset in SYMBOLS.items():
+        if letter in sset:
+            break
+    else:
+        if letter == 'k':
+            # treat 'k' as an alias for 'K' as per: http://goo.gl/kTQMs
+            sset = SYMBOLS['customary']
+            letter = letter.upper()
+        else:
+            raise ValueError("can't interpret %r" % init)
+    prefix = {sset[0]:1}
+    for i, s in enumerate(sset[1:]):
+        prefix[s] = 1 << (i+1)*10
+    return int(num * prefix[letter])
+
+
+if __name__ == "__main__":
+    import doctest
+    doctest.testmod()