Commit b1fd424c authored by Marco Mariani's avatar Marco Mariani Committed by Rafael Monnerat

added options: software_min_free_space (def.200MB) and instance_min_free_space (def.100MB)

parent 0bf32663
......@@ -48,8 +48,9 @@ from slapos.grid import utils # for methods that could be mocked, access them t
from slapos.slap.slap import NotFoundError
from slapos.grid.svcbackend import getSupervisorRPC
from slapos.grid.exception import (BuildoutFailedError, WrongPermissionError,
PathDoesNotExistError)
PathDoesNotExistError, DiskSpaceError)
from slapos.grid.networkcache import download_network_cached, upload_network_cached
from slapos.human import bytes2human
WATCHDOG_MARK = '-on-watch'
......@@ -61,6 +62,38 @@ PROGRAM_PARTITION_TEMPLATE = pkg_resources.resource_stream(__name__,
'templates/program_partition_supervisord.conf.in').read()
def free_space(path, fn):
while True:
try:
disk = os.statvfs(path)
return fn(disk)
except OSError:
pass
if os.sep not in path:
break
path = os.path.split(path)[0]
def free_space_root(path):
"""
Returns free space available to the root user, in bytes.
A non-existent path can be provided, and the ancestors
will be queried instead.
"""
return free_space(path, lambda d: d.bsize * d.f_bfree)
def free_space_nonroot(path):
"""
Returns free space available to non-root users, in bytes.
A non-existent path can be provided, and the ancestors
will be queried instead.
"""
return free_space(path, lambda d: d.f_bsize * d.f_bavail)
class Software(object):
"""This class is responsible for installing a software release"""
......@@ -73,7 +106,8 @@ class Software(object):
download_binary_cache_url=None, upload_binary_cache_url=None,
download_binary_dir_url=None, upload_binary_dir_url=None,
download_from_binary_cache_url_blacklist=None,
upload_to_binary_cache_url_blacklist=None):
upload_to_binary_cache_url_blacklist=None,
software_min_free_space=None):
"""Initialisation of class parameters
"""
......@@ -106,6 +140,17 @@ class Software(object):
download_from_binary_cache_url_blacklist
self.upload_to_binary_cache_url_blacklist = \
upload_to_binary_cache_url_blacklist
self.software_min_free_space = software_min_free_space
def check_free_space(self):
required = self.software_min_free_space
available = free_space_nonroot(self.software_path)
if available < required:
msg = "Not enough space for {path}: available {available}, required {required} (option 'software_min_free_space')"
raise DiskSpaceError(msg.format(path=self.software_path,
available=bytes2human(available),
required=bytes2human(required)))
def install(self):
""" Fetches binary cache if possible.
......@@ -113,6 +158,9 @@ class Software(object):
"""
self.logger.info("Installing software release %s..." % self.url)
cache_dir = tempfile.mkdtemp()
self.check_free_space()
try:
tarpath = os.path.join(cache_dir, self.software_url_hash)
# Check if we can download from cache
......@@ -293,6 +341,7 @@ class Partition(object):
logger,
certificate_repository_path=None,
retention_delay='0',
instance_min_free_space=None
):
"""Initialisation of class parameters"""
self.buildout = buildout
......@@ -333,6 +382,19 @@ class Partition(object):
self.instance_path, self.retention_lock_date_filename
)
self.instance_min_free_space = instance_min_free_space
def check_free_space(self):
required = self.instance_min_free_space
available = free_space_nonroot(self.instance_path)
if available < required:
msg = "Not enough space for {path}: available {available}, required {required} (option 'instance_min_free_space')"
raise DiskSpaceError(msg.format(path=self.instance_path,
available=bytes2human(available),
required=bytes2human(required)))
def _updateCertificate(self):
try:
partition_certificate = self.computer_partition.getCertificate()
......@@ -397,6 +459,9 @@ class Partition(object):
"""
self.logger.info("Installing Computer Partition %s..."
% self.computer_partition.getId())
self.check_free_space()
# Checks existence and permissions of Partition directory
# Note : Partitions have to be created and configured before running slapgrid
if not os.path.isdir(self.instance_path):
......@@ -505,6 +570,7 @@ class Partition(object):
['buildout:bin-directory=%s' %
os.path.join(self.instance_path, 'sbin')])
buildout_binary = os.path.join(self.instance_path, 'sbin', 'buildout')
# Launches buildout
utils.launchBuildout(path=self.instance_path,
buildout_binary=buildout_binary,
......
......@@ -38,3 +38,8 @@ class WrongPermissionError(Exception):
class BuildoutFailedError(Exception):
pass
class DiskSpaceError(Exception):
pass
......@@ -56,6 +56,7 @@ from slapos.grid.SlapObject import Software, Partition
from slapos.grid.svcbackend import launchSupervisord
from slapos.grid.utils import (md5digest, createPrivateDirectory, dropPrivileges,
SlapPopen, updateFile)
from slapos.human import human2bytes
import slapos.slap
......@@ -201,6 +202,9 @@ def create_slapgrid_object(options, logger):
]
op = options
software_min_free_space = human2bytes(op.get('software_min_free_space', '200M'))
instance_min_free_space = human2bytes(op.get('instance_min_free_space', '100M'))
return Slapgrid(software_root=op['software_root'],
instance_root=op['instance_root'],
master_url=op['master_url'],
......@@ -235,7 +239,9 @@ def create_slapgrid_object(options, logger):
# Try to fetch from deprecated argument
software_release_filter_list=op.get('only-sr', op.get('only_sr')),
# Try to fetch from deprecated argument
computer_partition_filter_list=op.get('only-cp', op.get('only_cp')))
computer_partition_filter_list=op.get('only-cp', op.get('only_cp')),
software_min_free_space=software_min_free_space,
instance_min_free_space=instance_min_free_space)
def check_required_only_partitions(existing, required):
......@@ -288,6 +294,8 @@ class Slapgrid(object):
develop=False,
software_release_filter_list=None,
computer_partition_filter_list=None,
software_min_free_space=None,
instance_min_free_space=None,
):
"""Makes easy initialisation of class parameters"""
# Parses arguments
......@@ -339,6 +347,8 @@ class Slapgrid(object):
self.computer_partition_filter_list = \
computer_partition_filter_list.split(",")
self.maximum_periodicity = maximum_periodicity
self.software_min_free_space = software_min_free_space
self.instance_min_free_space = instance_min_free_space
def getWatchdogLine(self):
invocation_list = [WATCHDOG_PATH]
......@@ -433,7 +443,8 @@ class Slapgrid(object):
shacache_cert_file=self.shacache_cert_file,
shacache_key_file=self.shacache_key_file,
shadir_cert_file=self.shadir_cert_file,
shadir_key_file=self.shadir_key_file)
shadir_key_file=self.shadir_key_file,
software_min_free_space=self.software_min_free_space)
if state == 'available':
completed_tag = os.path.join(software_path, '.completed')
if (self.develop or (not os.path.exists(completed_tag) and
......@@ -670,6 +681,7 @@ class Slapgrid(object):
buildout=self.buildout,
logger=self.logger,
retention_delay=retention_delay,
instance_min_free_space=self.instance_min_free_space,
)
computer_partition_state = computer_partition.getState()
......
#!/usr/bin/env python
import sys
"""
Bytes-to-human / human-to-bytes converter.
Based on: http://goo.gl/kTQMs
Working with Python 2.x and 3.x.
Author: Giampaolo Rodola' <g.rodola [AT] gmail [DOT] com>
License: MIT
"""
# see: http://goo.gl/kTQMs
SYMBOLS = {
'customary' : ('B', 'K', 'M', 'G', 'T', 'P', 'E', 'Z', 'Y'),
'slapos' : ('', 'KB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB'),
'customary_ext' : ('byte', 'kilo', 'mega', 'giga', 'tera', 'peta', 'exa',
'zetta', 'iotta'),
'iec' : ('Bi', 'Ki', 'Mi', 'Gi', 'Ti', 'Pi', 'Ei', 'Zi', 'Yi'),
'iec_ext' : ('byte', 'kibi', 'mebi', 'gibi', 'tebi', 'pebi', 'exbi',
'zebi', 'yobi'),
}
def bytes2human(n, format='%(value).1f %(symbol)s', symbols='slapos'):
"""
Convert n bytes into a human readable string based on format.
symbols can be either "customary", "customary_ext", "iec" or "iec_ext",
see: http://goo.gl/kTQMs
>>> bytes2human(0)
'0.0 B'
>>> bytes2human(0.9)
'0.0 B'
>>> bytes2human(1)
'1.0 B'
>>> bytes2human(1.9)
'1.0 B'
>>> bytes2human(1024)
'1.0 K'
>>> bytes2human(1048576)
'1.0 M'
>>> bytes2human(1099511627776127398123789121)
'909.5 Y'
>>> bytes2human(9856, symbols="customary")
'9.6 K'
>>> bytes2human(9856, symbols="customary_ext")
'9.6 kilo'
>>> bytes2human(9856, symbols="iec")
'9.6 Ki'
>>> bytes2human(9856, symbols="iec_ext")
'9.6 kibi'
>>> bytes2human(10000, "%(value).1f %(symbol)s/sec")
'9.8 K/sec'
>>> # precision can be adjusted by playing with %f operator
>>> bytes2human(10000, format="%(value).5f %(symbol)s")
'9.76562 K'
"""
n = int(n)
if n < 0:
raise ValueError("n < 0")
symbols = SYMBOLS[symbols]
prefix = {}
for i, s in enumerate(symbols[1:]):
prefix[s] = 1 << (i+1)*10
for symbol in reversed(symbols[1:]):
if n >= prefix[symbol]:
value = float(n) / prefix[symbol]
return format % locals()
return format % dict(symbol=symbols[0], value=n)
def human2bytes(s):
"""
Attempts to guess the string format based on default symbols
set and return the corresponding bytes as an integer.
When unable to recognize the format ValueError is raised.
>>> human2bytes('0 B')
0
>>> human2bytes('1 K')
1024
>>> human2bytes('1 M')
1048576
>>> human2bytes('1 Gi')
1073741824
>>> human2bytes('1 tera')
1099511627776
>>> human2bytes('0.5kilo')
512
>>> human2bytes('0.1 byte')
0
>>> human2bytes('1 k') # k is an alias for K
1024
>>> human2bytes('12 foo')
Traceback (most recent call last):
...
ValueError: can't interpret '12 foo'
"""
init = s
num = ""
while s and s[0:1].isdigit() or s[0:1] == '.':
num += s[0]
s = s[1:]
num = float(num)
letter = s.strip()
for name, sset in SYMBOLS.items():
if letter in sset:
break
else:
if letter == 'k':
# treat 'k' as an alias for 'K' as per: http://goo.gl/kTQMs
sset = SYMBOLS['customary']
letter = letter.upper()
else:
raise ValueError("can't interpret %r" % init)
prefix = {sset[0]:1}
for i, s in enumerate(sset[1:]):
prefix[s] = 1 << (i+1)*10
return int(num * prefix[letter])
if __name__ == "__main__":
import doctest
doctest.testmod()
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment