Commit fa8d8cc6 authored by Julien Muchembled's avatar Julien Muchembled Committed by Matevz Golob

Try to reuse existing file to avoid excessive IO on update + other minor optimisations

parent 1cc92abe
......@@ -26,6 +26,7 @@
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
import errno
import io
import logging
import os
......@@ -33,6 +34,7 @@ import sys
import inspect
import re
import shutil
import stat
import urllib
import urlparse
......@@ -92,9 +94,27 @@ class GenericBaseRecipe(object):
"""Create a file with content
The parent directory should exists, else it would raise IOError"""
with open(name, 'w') as fileobject:
fileobject.write(content)
os.chmod(fileobject.name, mode)
if not isinstance(content, bytes):
content = content.encode('utf-8')
# Try to reuse existing file. This is particularly
# important to avoid excessive IO during update.
try:
with open(name, 'rb') as f:
if f.read(len(content)+1) == content:
if None is not mode != stat.S_IMODE(os.fstat(f.fileno()).st_mode):
os.fchmod(f.fileno(), mode)
return os.path.abspath(name)
except (IOError, OSError) as e:
pass
try:
os.unlink(name)
except OSError as e:
if e.errno != errno.ENOENT:
raise
with open(name, 'wb') as f:
if mode is not None:
os.fchmod(f.fileno(), mode)
f.write(content)
return os.path.abspath(name)
def createExecutable(self, name, content, mode=0700):
......
......@@ -24,18 +24,17 @@
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
import json
import re
import logging, os
import zc.buildout.easy_install
from pprint import pformat
from slapos.recipe.librecipe import GenericBaseRecipe
script_template = '''# This script is auto generated by slapgrid, do not edit!
import json
import sys
sys.path[0:0] = %(path)s
extra_config_dict = json.loads("""%(config)s""", strict=False)
extra_config_dict = %(config)s
# We want to cleanup all imported modules from slapos namespace, because
# they will conflict with slapos.core.
......@@ -69,12 +68,9 @@ class Recipe(GenericBaseRecipe):
"""Return a mapping where to store generated working sets.
from https://github.com/buildout/buildout/blob/master/zc.recipe.egg_/src/zc/recipe/egg/egg.py#L170
"""
cache_storage = getattr(
self.buildout,
self._WORKING_SET_CACHE_NAME,
None
)
if cache_storage is None:
try:
return getattr(self.buildout, self._WORKING_SET_CACHE_NAME)
except AttributeError:
cache_storage = {}
try:
setattr(
......@@ -83,29 +79,29 @@ class Recipe(GenericBaseRecipe):
cache_storage
)
except AttributeError:
if type(self.buildout) == type({}):
# failed to set attribute in test mode, cache not used
pass
else:
if not isinstance(self.buildout, dict):
raise
# failed to set attribute in test mode, cache not used
return cache_storage
def install(self):
develop_eggs_dir = self.options['develop-eggs-directory']
eggs_dir = self.options['eggs-directory']
egg_list = [
egg_list = tuple(
egg.strip()
for egg in self.options['eggs'].split('\n')
for egg in self.options['eggs'].splitlines()
if egg.strip()
]
)
cache_storage = self._get_cache_storage()
cache_key = (
tuple(egg_list),
egg_list,
eggs_dir,
develop_eggs_dir,
)
if cache_key not in cache_storage:
try:
working_set = cache_storage[cache_key]
except KeyError:
if develop_eggs_dir and eggs_dir:
working_set = zc.buildout.easy_install.working_set(
egg_list,
......@@ -114,8 +110,6 @@ class Recipe(GenericBaseRecipe):
cache_storage[cache_key] = working_set
else:
working_set = set()
else:
working_set = cache_storage[cache_key]
regex = r"^[\w_\-\.\s]+$"
import_path = self.options.get('import', '').strip()
......@@ -129,24 +123,14 @@ class Recipe(GenericBaseRecipe):
if not re.search(regex, content_string):
raise ValueError("Promise content %r is not valid" % content_string)
output = self.options['output']
mode = self.options.get('mode', '0644')
path_list = []
for dist in working_set:
path_list.append(dist.location)
config_dict = dict()
for key in self.options:
if key.startswith('config-'):
config_dict[key[7:]] = self.options[key]
option_dict = dict(path=json.dumps(path_list, indent=2),
content=content_string,
config=json.dumps(config_dict, indent=2, sort_keys=True))
with open(output, 'w') as f:
f.write(script_template % option_dict)
config_dict = {key[7:]: self.options[key]
for key in self.options
if key.startswith('config-')}
os.chmod(output, int(mode, 8))
return (output,)
return self.createFile(self.options['output'], script_template % {
'path': pformat([dist.location for dist in working_set], indent=2),
'content': content_string,
'config': pformat(config_dict, indent=2),
}, int(self.options.get('mode', '0644'), 8)),
update = install
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment