Commit b1fd424c authored by Marco Mariani's avatar Marco Mariani Committed by Rafael Monnerat

added options: software_min_free_space (def.200MB) and instance_min_free_space (def.100MB)

parent 0bf32663
...@@ -48,8 +48,9 @@ from slapos.grid import utils # for methods that could be mocked, access them t ...@@ -48,8 +48,9 @@ from slapos.grid import utils # for methods that could be mocked, access them t
from slapos.slap.slap import NotFoundError from slapos.slap.slap import NotFoundError
from slapos.grid.svcbackend import getSupervisorRPC from slapos.grid.svcbackend import getSupervisorRPC
from slapos.grid.exception import (BuildoutFailedError, WrongPermissionError, from slapos.grid.exception import (BuildoutFailedError, WrongPermissionError,
PathDoesNotExistError) PathDoesNotExistError, DiskSpaceError)
from slapos.grid.networkcache import download_network_cached, upload_network_cached from slapos.grid.networkcache import download_network_cached, upload_network_cached
from slapos.human import bytes2human
WATCHDOG_MARK = '-on-watch' WATCHDOG_MARK = '-on-watch'
...@@ -61,6 +62,38 @@ PROGRAM_PARTITION_TEMPLATE = pkg_resources.resource_stream(__name__, ...@@ -61,6 +62,38 @@ PROGRAM_PARTITION_TEMPLATE = pkg_resources.resource_stream(__name__,
'templates/program_partition_supervisord.conf.in').read() 'templates/program_partition_supervisord.conf.in').read()
def free_space(path, fn):
while True:
try:
disk = os.statvfs(path)
return fn(disk)
except OSError:
pass
if os.sep not in path:
break
path = os.path.split(path)[0]
def free_space_root(path):
"""
Returns free space available to the root user, in bytes.
A non-existent path can be provided, and the ancestors
will be queried instead.
"""
return free_space(path, lambda d: d.bsize * d.f_bfree)
def free_space_nonroot(path):
"""
Returns free space available to non-root users, in bytes.
A non-existent path can be provided, and the ancestors
will be queried instead.
"""
return free_space(path, lambda d: d.f_bsize * d.f_bavail)
class Software(object): class Software(object):
"""This class is responsible for installing a software release""" """This class is responsible for installing a software release"""
...@@ -73,7 +106,8 @@ class Software(object): ...@@ -73,7 +106,8 @@ class Software(object):
download_binary_cache_url=None, upload_binary_cache_url=None, download_binary_cache_url=None, upload_binary_cache_url=None,
download_binary_dir_url=None, upload_binary_dir_url=None, download_binary_dir_url=None, upload_binary_dir_url=None,
download_from_binary_cache_url_blacklist=None, download_from_binary_cache_url_blacklist=None,
upload_to_binary_cache_url_blacklist=None): upload_to_binary_cache_url_blacklist=None,
software_min_free_space=None):
"""Initialisation of class parameters """Initialisation of class parameters
""" """
...@@ -106,6 +140,17 @@ class Software(object): ...@@ -106,6 +140,17 @@ class Software(object):
download_from_binary_cache_url_blacklist download_from_binary_cache_url_blacklist
self.upload_to_binary_cache_url_blacklist = \ self.upload_to_binary_cache_url_blacklist = \
upload_to_binary_cache_url_blacklist upload_to_binary_cache_url_blacklist
self.software_min_free_space = software_min_free_space
def check_free_space(self):
required = self.software_min_free_space
available = free_space_nonroot(self.software_path)
if available < required:
msg = "Not enough space for {path}: available {available}, required {required} (option 'software_min_free_space')"
raise DiskSpaceError(msg.format(path=self.software_path,
available=bytes2human(available),
required=bytes2human(required)))
def install(self): def install(self):
""" Fetches binary cache if possible. """ Fetches binary cache if possible.
...@@ -113,6 +158,9 @@ class Software(object): ...@@ -113,6 +158,9 @@ class Software(object):
""" """
self.logger.info("Installing software release %s..." % self.url) self.logger.info("Installing software release %s..." % self.url)
cache_dir = tempfile.mkdtemp() cache_dir = tempfile.mkdtemp()
self.check_free_space()
try: try:
tarpath = os.path.join(cache_dir, self.software_url_hash) tarpath = os.path.join(cache_dir, self.software_url_hash)
# Check if we can download from cache # Check if we can download from cache
...@@ -293,6 +341,7 @@ class Partition(object): ...@@ -293,6 +341,7 @@ class Partition(object):
logger, logger,
certificate_repository_path=None, certificate_repository_path=None,
retention_delay='0', retention_delay='0',
instance_min_free_space=None
): ):
"""Initialisation of class parameters""" """Initialisation of class parameters"""
self.buildout = buildout self.buildout = buildout
...@@ -333,6 +382,19 @@ class Partition(object): ...@@ -333,6 +382,19 @@ class Partition(object):
self.instance_path, self.retention_lock_date_filename self.instance_path, self.retention_lock_date_filename
) )
self.instance_min_free_space = instance_min_free_space
def check_free_space(self):
required = self.instance_min_free_space
available = free_space_nonroot(self.instance_path)
if available < required:
msg = "Not enough space for {path}: available {available}, required {required} (option 'instance_min_free_space')"
raise DiskSpaceError(msg.format(path=self.instance_path,
available=bytes2human(available),
required=bytes2human(required)))
def _updateCertificate(self): def _updateCertificate(self):
try: try:
partition_certificate = self.computer_partition.getCertificate() partition_certificate = self.computer_partition.getCertificate()
...@@ -397,6 +459,9 @@ class Partition(object): ...@@ -397,6 +459,9 @@ class Partition(object):
""" """
self.logger.info("Installing Computer Partition %s..." self.logger.info("Installing Computer Partition %s..."
% self.computer_partition.getId()) % self.computer_partition.getId())
self.check_free_space()
# Checks existence and permissions of Partition directory # Checks existence and permissions of Partition directory
# Note : Partitions have to be created and configured before running slapgrid # Note : Partitions have to be created and configured before running slapgrid
if not os.path.isdir(self.instance_path): if not os.path.isdir(self.instance_path):
...@@ -505,6 +570,7 @@ class Partition(object): ...@@ -505,6 +570,7 @@ class Partition(object):
['buildout:bin-directory=%s' % ['buildout:bin-directory=%s' %
os.path.join(self.instance_path, 'sbin')]) os.path.join(self.instance_path, 'sbin')])
buildout_binary = os.path.join(self.instance_path, 'sbin', 'buildout') buildout_binary = os.path.join(self.instance_path, 'sbin', 'buildout')
# Launches buildout # Launches buildout
utils.launchBuildout(path=self.instance_path, utils.launchBuildout(path=self.instance_path,
buildout_binary=buildout_binary, buildout_binary=buildout_binary,
......
...@@ -38,3 +38,8 @@ class WrongPermissionError(Exception): ...@@ -38,3 +38,8 @@ class WrongPermissionError(Exception):
class BuildoutFailedError(Exception): class BuildoutFailedError(Exception):
pass pass
class DiskSpaceError(Exception):
pass
...@@ -56,6 +56,7 @@ from slapos.grid.SlapObject import Software, Partition ...@@ -56,6 +56,7 @@ from slapos.grid.SlapObject import Software, Partition
from slapos.grid.svcbackend import launchSupervisord from slapos.grid.svcbackend import launchSupervisord
from slapos.grid.utils import (md5digest, createPrivateDirectory, dropPrivileges, from slapos.grid.utils import (md5digest, createPrivateDirectory, dropPrivileges,
SlapPopen, updateFile) SlapPopen, updateFile)
from slapos.human import human2bytes
import slapos.slap import slapos.slap
...@@ -201,6 +202,9 @@ def create_slapgrid_object(options, logger): ...@@ -201,6 +202,9 @@ def create_slapgrid_object(options, logger):
] ]
op = options op = options
software_min_free_space = human2bytes(op.get('software_min_free_space', '200M'))
instance_min_free_space = human2bytes(op.get('instance_min_free_space', '100M'))
return Slapgrid(software_root=op['software_root'], return Slapgrid(software_root=op['software_root'],
instance_root=op['instance_root'], instance_root=op['instance_root'],
master_url=op['master_url'], master_url=op['master_url'],
...@@ -235,7 +239,9 @@ def create_slapgrid_object(options, logger): ...@@ -235,7 +239,9 @@ def create_slapgrid_object(options, logger):
# Try to fetch from deprecated argument # Try to fetch from deprecated argument
software_release_filter_list=op.get('only-sr', op.get('only_sr')), software_release_filter_list=op.get('only-sr', op.get('only_sr')),
# Try to fetch from deprecated argument # Try to fetch from deprecated argument
computer_partition_filter_list=op.get('only-cp', op.get('only_cp'))) computer_partition_filter_list=op.get('only-cp', op.get('only_cp')),
software_min_free_space=software_min_free_space,
instance_min_free_space=instance_min_free_space)
def check_required_only_partitions(existing, required): def check_required_only_partitions(existing, required):
...@@ -288,6 +294,8 @@ class Slapgrid(object): ...@@ -288,6 +294,8 @@ class Slapgrid(object):
develop=False, develop=False,
software_release_filter_list=None, software_release_filter_list=None,
computer_partition_filter_list=None, computer_partition_filter_list=None,
software_min_free_space=None,
instance_min_free_space=None,
): ):
"""Makes easy initialisation of class parameters""" """Makes easy initialisation of class parameters"""
# Parses arguments # Parses arguments
...@@ -339,6 +347,8 @@ class Slapgrid(object): ...@@ -339,6 +347,8 @@ class Slapgrid(object):
self.computer_partition_filter_list = \ self.computer_partition_filter_list = \
computer_partition_filter_list.split(",") computer_partition_filter_list.split(",")
self.maximum_periodicity = maximum_periodicity self.maximum_periodicity = maximum_periodicity
self.software_min_free_space = software_min_free_space
self.instance_min_free_space = instance_min_free_space
def getWatchdogLine(self): def getWatchdogLine(self):
invocation_list = [WATCHDOG_PATH] invocation_list = [WATCHDOG_PATH]
...@@ -433,7 +443,8 @@ class Slapgrid(object): ...@@ -433,7 +443,8 @@ class Slapgrid(object):
shacache_cert_file=self.shacache_cert_file, shacache_cert_file=self.shacache_cert_file,
shacache_key_file=self.shacache_key_file, shacache_key_file=self.shacache_key_file,
shadir_cert_file=self.shadir_cert_file, shadir_cert_file=self.shadir_cert_file,
shadir_key_file=self.shadir_key_file) shadir_key_file=self.shadir_key_file,
software_min_free_space=self.software_min_free_space)
if state == 'available': if state == 'available':
completed_tag = os.path.join(software_path, '.completed') completed_tag = os.path.join(software_path, '.completed')
if (self.develop or (not os.path.exists(completed_tag) and if (self.develop or (not os.path.exists(completed_tag) and
...@@ -670,6 +681,7 @@ class Slapgrid(object): ...@@ -670,6 +681,7 @@ class Slapgrid(object):
buildout=self.buildout, buildout=self.buildout,
logger=self.logger, logger=self.logger,
retention_delay=retention_delay, retention_delay=retention_delay,
instance_min_free_space=self.instance_min_free_space,
) )
computer_partition_state = computer_partition.getState() computer_partition_state = computer_partition.getState()
......
#!/usr/bin/env python
import sys
"""
Bytes-to-human / human-to-bytes converter.
Based on: http://goo.gl/kTQMs
Working with Python 2.x and 3.x.
Author: Giampaolo Rodola' <g.rodola [AT] gmail [DOT] com>
License: MIT
"""
# see: http://goo.gl/kTQMs
SYMBOLS = {
'customary' : ('B', 'K', 'M', 'G', 'T', 'P', 'E', 'Z', 'Y'),
'slapos' : ('', 'KB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB'),
'customary_ext' : ('byte', 'kilo', 'mega', 'giga', 'tera', 'peta', 'exa',
'zetta', 'iotta'),
'iec' : ('Bi', 'Ki', 'Mi', 'Gi', 'Ti', 'Pi', 'Ei', 'Zi', 'Yi'),
'iec_ext' : ('byte', 'kibi', 'mebi', 'gibi', 'tebi', 'pebi', 'exbi',
'zebi', 'yobi'),
}
def bytes2human(n, format='%(value).1f %(symbol)s', symbols='slapos'):
"""
Convert n bytes into a human readable string based on format.
symbols can be either "customary", "customary_ext", "iec" or "iec_ext",
see: http://goo.gl/kTQMs
>>> bytes2human(0)
'0.0 B'
>>> bytes2human(0.9)
'0.0 B'
>>> bytes2human(1)
'1.0 B'
>>> bytes2human(1.9)
'1.0 B'
>>> bytes2human(1024)
'1.0 K'
>>> bytes2human(1048576)
'1.0 M'
>>> bytes2human(1099511627776127398123789121)
'909.5 Y'
>>> bytes2human(9856, symbols="customary")
'9.6 K'
>>> bytes2human(9856, symbols="customary_ext")
'9.6 kilo'
>>> bytes2human(9856, symbols="iec")
'9.6 Ki'
>>> bytes2human(9856, symbols="iec_ext")
'9.6 kibi'
>>> bytes2human(10000, "%(value).1f %(symbol)s/sec")
'9.8 K/sec'
>>> # precision can be adjusted by playing with %f operator
>>> bytes2human(10000, format="%(value).5f %(symbol)s")
'9.76562 K'
"""
n = int(n)
if n < 0:
raise ValueError("n < 0")
symbols = SYMBOLS[symbols]
prefix = {}
for i, s in enumerate(symbols[1:]):
prefix[s] = 1 << (i+1)*10
for symbol in reversed(symbols[1:]):
if n >= prefix[symbol]:
value = float(n) / prefix[symbol]
return format % locals()
return format % dict(symbol=symbols[0], value=n)
def human2bytes(s):
"""
Attempts to guess the string format based on default symbols
set and return the corresponding bytes as an integer.
When unable to recognize the format ValueError is raised.
>>> human2bytes('0 B')
0
>>> human2bytes('1 K')
1024
>>> human2bytes('1 M')
1048576
>>> human2bytes('1 Gi')
1073741824
>>> human2bytes('1 tera')
1099511627776
>>> human2bytes('0.5kilo')
512
>>> human2bytes('0.1 byte')
0
>>> human2bytes('1 k') # k is an alias for K
1024
>>> human2bytes('12 foo')
Traceback (most recent call last):
...
ValueError: can't interpret '12 foo'
"""
init = s
num = ""
while s and s[0:1].isdigit() or s[0:1] == '.':
num += s[0]
s = s[1:]
num = float(num)
letter = s.strip()
for name, sset in SYMBOLS.items():
if letter in sset:
break
else:
if letter == 'k':
# treat 'k' as an alias for 'K' as per: http://goo.gl/kTQMs
sset = SYMBOLS['customary']
letter = letter.upper()
else:
raise ValueError("can't interpret %r" % init)
prefix = {sset[0]:1}
for i, s in enumerate(sset[1:]):
prefix[s] = 1 << (i+1)*10
return int(num * prefix[letter])
if __name__ == "__main__":
import doctest
doctest.testmod()
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment