Skip to content

S
slapos.recipe.template
Commit 0f8a8b27 authored by Julien Muchembled's avatar Julien Muchembled
Browse files
Options

default: bring some of the jinja2 improvements by refactoring the 2 recipes 

- try to not rewrite on update if there's no change
- handle 'mode' the same way as in the jinja2 recipe
parent 86e4e5b8
Hide whitespace changes
Inline Side-by-side
Showing with 113 additions and 115 deletions
slapos/recipe/template/__init__.py
View file @ 0f8a8b27
......@@ -24,40 +24,107 @@
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
import errno
import os
import stat
import tempfile
import zc.buildout
class Recipe(object):
def __init__(self, buildout, name, options):
download = zc.buildout.download.Download(buildout['buildout'],
hash_name=True)
path, is_temp = download(options.pop('url'),
md5sum=options.get('md5sum'))
if str is bytes:
str2bytes = lambda s: s
else:
str2bytes = lambda s: s.encode()
def get_mode(fd):
return stat.S_IMODE(os.fstat(fd).st_mode)
def get_umask():
global get_umask
while 1:
p = tempfile.mktemp()
try:
fd = os.open(p, os.O_CREAT | os.O_EXCL, 0o777)
break
except OSError as e:
if e.errno != errno.EEXIST:
if os.name == 'nt' and e.errno == errno.EACCES:
p = os.path.basename(p)
if os.path.isdir(p) and os.access(p, os.W_OK):
continue
raise
try:
self.mode = None
if 'mode' in options:
# Mode is in octal notation
self.mode = int(options['mode'], 8)
os.unlink(p)
umask = get_mode(fd)
finally:
os.close(fd)
get_umask = lambda: umask
return umask
with open(path) as inputfile:
self.output_content = '$'.join(options._sub(s, None)
for s in inputfile.read().split('$$'))
self.output_filename = options['output']
finally:
if is_temp:
os.remove(path)
class Recipe(object):
def __init__(self, buildout, name, options):
self.buildout = buildout
self.md5sum = options.get('md5sum')
mode = options.get('mode')
self.mode = int(mode, 8) if mode else None
self._init(name, options)
def _init(self, name, options):
self.output = options['output']
options_sub = options._sub
self.rendered = '$'.join(options_sub(s, None)
for s in self._read(options['url']).split('$$'))
def install(self):
with open(self.output_filename, 'w') as outputfile:
outputfile.write(self.output_content)
def _read(self, url, *args):
path, is_temp = zc.buildout.download.Download(
self.buildout, hash_name=True,
)(url, self.md5sum or None)
try:
with open(path, *args) as f:
return f.read()
finally:
if is_temp:
os.unlink(path)
if self.mode is not None:
os.chmod(self.output_filename, self.mode)
def _render(self):
return str2bytes(self.rendered)
return self.output_filename
def install(self):
output = self.output
rendered = self._render()
mode = self.mode
mask = (0o777 if rendered.startswith(b'#!') else 0o666
) if mode is None else 0
# Try to reuse existing file. This is particularly
# important to avoid excessive IO because we may render on update.
try:
with open(output, 'rb') as f:
if f.read(len(rendered)+1) == rendered:
m = get_umask() & mask if mode is None else mode
if get_mode(f.fileno()) != m:
os.fchmod(f.fileno(), m)
return output
except (IOError, OSError) as e:
pass
# Unlink any existing file so that umask applies.
try:
os.unlink(output)
except OSError as e:
if e.errno != errno.ENOENT:
raise
outdir = os.path.dirname(output)
if outdir and not os.path.exists(outdir):
os.makedirs(outdir)
fd = os.open(output, os.O_CREAT | os.O_EXCL | os.O_WRONLY, mask)
try:
os.write(fd, rendered)
if mode is not None:
os.fchmod(fd, mode)
finally:
os.close(fd)
return output
def update(self):
return self.install()
def update(self):
if not self.md5sum:
self.install()
slapos/recipe/template/jinja2_template.py
View file @ 0f8a8b27
......@@ -24,15 +24,13 @@
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
import errno
import os
import json
import stat
import tempfile
import zc.buildout
from jinja2 import Environment, StrictUndefined, \
BaseLoader, TemplateNotFound, PrefixLoader
import six
from . import Recipe
DEFAULT_CONTEXT = {x.__name__: x for x in (
abs, all, any, bin, bool, bytes, callable, chr, complex, dict, divmod,
......@@ -131,36 +129,12 @@ LOADER_TYPE_DICT = {
'folder': (FolderLoader, getKey),
}
def get_mode(fd):
return stat.S_IMODE(os.fstat(fd).st_mode)
def get_umask():
global get_umask
while 1:
p = tempfile.mktemp()
try:
fd = os.open(p, os.O_CREAT | os.O_EXCL, 0o777)
break
except OSError as e:
if e.errno != errno.EEXIST:
if os.name == 'nt' and e.errno == errno.EACCES:
p = os.path.basename(p)
if os.path.isdir(p) and os.access(p, os.W_OK):
continue
raise
try:
os.unlink(p)
umask = get_mode(fd)
finally:
os.close(fd)
get_umask = lambda: umask
return umask
compiled_source_cache = {}
class Recipe(object):
class Recipe(Recipe):
def __init__(self, buildout, name, options):
self.buildout = buildout
def _init(self, name, options):
args = self.buildout, name, options
self.once = options.get('once')
self.encoding = options.get('encoding', 'utf-8')
import_delimiter = options.get('import-delimiter',
DEFAULT_IMPORT_DELIMITER)
......@@ -174,7 +148,7 @@ class Recipe(object):
loader_type, expression_handler = LOADER_TYPE_DICT[
expression_type]
import_dict[alias] = loader_type(
expression_handler(expression, buildout, name, options),
expression_handler(expression, *args),
import_delimiter, self.encoding,
)
if import_dict:
......@@ -182,9 +156,8 @@ class Recipe(object):
delimiter=import_delimiter)
else:
loader = None
self.rendered = options['rendered']
self.output = options['rendered']
self.template = options['template']
self.md5sum = options.get('md5sum')
extension_list = [x for x in (y.strip()
for y in options.get('extensions', '').split()) if x]
self.context = context = DEFAULT_CONTEXT.copy()
......@@ -198,18 +171,13 @@ class Recipe(object):
raise ValueError('Duplicate context entry %r' % (
variable_name, ))
context[variable_name] = EXPRESSION_HANDLER[expression_type](
expression, buildout, name, options)
mode = options.get('mode')
self.mode = int(mode, 8) if mode else None
expression, *args)
self.env = Environment(
extensions=extension_list,
undefined=StrictUndefined,
loader=loader)
self.once = options.get('once')
def install(self):
if self.once and os.path.exists(self.once):
return
def _render(self):
template = self.template
env = self.env
if template.startswith('inline:'):
......@@ -219,60 +187,23 @@ class Recipe(object):
try:
compiled_source = compiled_source_cache[template]
except KeyError:
download_path, is_temp = zc.buildout.download.Download(
self.buildout['buildout'],
hash_name=True,
)(
template,
md5sum=self.md5sum,
)
try:
with open(download_path, 'rb') as f:
source = f.read().decode(self.encoding)
finally:
if is_temp:
os.remove(download_path)
source = self._read(template, 'rb').decode(self.encoding)
compiled_source_cache[template] = compiled_source = \
env.compile(source, filename=template)
template_object = env.template_class.from_code(env,
compiled_source,
env.make_globals(None), None)
rendered = template_object.render(**self.context).encode(self.encoding)
mode = self.mode
mask = (0o777 if rendered.startswith(b'#!') else 0o666
) if mode is None else 0
# Try to reuse existing file. This is particularly
# important to avoid excessive IO because we render on update.
try:
with open(self.rendered, 'rb') as f:
if f.read(len(rendered)+1) == rendered:
m = get_umask() & mask if mode is None else mode
if get_mode(f.fileno()) != m:
os.fchmod(f.fileno(), m)
return self.rendered
except (IOError, OSError) as e:
pass
# Unlink any existing file so that umask applies.
try:
os.unlink(self.rendered)
except OSError as e:
if e.errno != errno.ENOENT:
raise
outdir = os.path.dirname(self.rendered)
if outdir and not os.path.exists(outdir):
os.makedirs(outdir)
fd = os.open(self.rendered, os.O_CREAT | os.O_EXCL | os.O_WRONLY, mask)
try:
os.write(fd, rendered)
if mode is not None:
os.fchmod(fd, mode)
finally:
os.close(fd)
if self.once:
open(self.once, 'ab').close()
return template_object.render(**self.context).encode(self.encoding)
def install(self):
once = self.once
if once and os.path.exists(once):
return
return self.rendered
installed = super(Recipe, self).install()
if once:
open(once, 'ab').close()
return
return installed
update = install
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7