Commit 1d9358a6 by Rafael Monnerat

Update Release Candidate

2 parents a1e5090d 65acdca0
Changes
=======
1.0.118 (2019-08-13)
--------------------
* NEO: new recipe to fix/optimize propagation of the 'masters' parameter
* publish_early: new '-update' option, keep published values out of buildout installed file
* publish: new -publish option to list explicitly options to publish
* re6stnet: Fix typo
* librecipe: Try to reuse existing file to avoid excessive IO on update and other minor optimisations
* certificate_authority: unique_subject = no
* wrapper: handle "=" in environment variables' content
1.0.92 (2019-02-21)
-------------------
......
......@@ -28,7 +28,7 @@ from setuptools import setup, find_packages
import glob
import os
version = '1.0.92'
version = '1.0.118'
name = 'slapos.cookbook'
long_description = open("README.rst").read() + "\n" + \
open("CHANGES.rst").read() + "\n"
......@@ -134,6 +134,7 @@ setup(name=name,
'mydumper = slapos.recipe.mydumper:Recipe',
'mysql = slapos.recipe.mysql:Recipe',
'nbdserver = slapos.recipe.nbdserver:Recipe',
'neoppod.cluster = slapos.recipe.neoppod:Cluster',
'neoppod.admin = slapos.recipe.neoppod:Admin',
'neoppod.master = slapos.recipe.neoppod:Master',
'neoppod.storage = slapos.recipe.neoppod:Storage',
......
......@@ -26,8 +26,38 @@
##############################################################################
import os
import shlex
from slapos.recipe.librecipe import GenericBaseRecipe
from zc.buildout import UserError
from .librecipe import GenericBaseRecipe
class Cluster(object):
def __init__(self, buildout, name, options):
self.buildout = buildout
self.options = options
def publish_early(self, publish_dict):
masters = publish_dict.setdefault('masters', '')
result_dict = {
'connection-admin': [],
'connection-master': [],
}
node_list = []
for node in sorted(self.options['nodes'].split()):
node = self.buildout[node]
node_list.append(node)
for k, v in result_dict.iteritems():
x = node[k]
if x:
v.append(x)
publish_dict['admins'] = ' '.join(result_dict.pop('connection-admin'))
x = ' '.join(result_dict.pop('connection-master'))
if masters != x:
publish_dict['masters'] = x
for node in node_list:
node['config-masters'] = x
node.recipe.__init__(self.buildout, node.name, node)
install = update = lambda self: None
class NeoBaseRecipe(GenericBaseRecipe):
......
......@@ -44,9 +44,14 @@ class Recipe(GenericSlapRecipe):
def _install(self):
publish_dict = {}
for name in self._extend_set:
for k, v in self.buildout[name].iteritems():
if k != 'recipe' and not k.startswith('-'):
publish_dict[k] = v
section = self.buildout[name]
try:
publish = section['-publish'].split()
except KeyError:
publish = (k for k in section
if k != 'recipe' and not k.startswith('-'))
for k in publish:
publish_dict[k] = section[k]
self._setConnectionDict(publish_dict, self.options.get('-slave-reference'))
return []
......
......@@ -25,9 +25,23 @@
#
##############################################################################
import slapos.slap
from slapos.recipe.librecipe import unwrap, wrap
from slapos.recipe.librecipe import GenericSlapRecipe
from collections import defaultdict
from .librecipe import unwrap, wrap, GenericSlapRecipe
def patchOptions(options, override):
def get(option, *args, **kw):
try:
return override[option]
except KeyError:
return options_get(option, *args, **kw)
try:
options_get = options._get
except AttributeError:
options_get = options.get
options.get = get
else:
options._get = get
class Recipe(GenericSlapRecipe):
"""
......@@ -43,6 +57,8 @@ class Recipe(GenericSlapRecipe):
-init =
foo gen-foo:x
bar gen-bar:y
-update =
baz update-baz:z
bar = z
[gen-foo]
......@@ -58,31 +74,75 @@ class Recipe(GenericSlapRecipe):
(and in this case, it is published immediately as a way to save the value).
${publish-early:bar} is forced to 'z' (${gen-bar:y} ignored):
a line like 'bar = z' is usually rendered conditionally with Jinja2.
The '-update' option has the same syntax than '-init'. The recipes of the
specified sections must implement 'publish_early(publish_dict)':
- it is always called, just before early publishing
- publish_dict is a dict with already published values
- 'publish_early' can change published values by modifying publish_dict.
In the above example:
- publish_dict is {'z': ...}
- during the execution of 'publish_early', other sections can access the
value with ${update-baz:z}
- once [publish-early] is initialized, the value should be accessed with
${publish-early:bar} ([update-baz] does not have it if it's accessed
before [publish-early])
"""
def __init__(self, buildout, name, options):
GenericSlapRecipe.__init__(self, buildout, name, options)
published_dict = None
publish = False
publish_dict = {}
for line in options['-init'].splitlines():
if line:
k, v = line.split()
if k not in options:
if published_dict is None:
self.slap.initializeConnection(self.server_url, self.key_file,
self.cert_file)
computer_partition = self.slap.registerComputerPartition(
self.computer_id, self.computer_partition_id)
published_dict = unwrap(
computer_partition.getConnectionParameterDict())
init = defaultdict(dict)
update = defaultdict(dict)
for d, k in (init, '-init'), (update, '-update'):
for line in options.get(k, '').splitlines():
if line:
k, v = line.split()
if k not in options:
section, v = v.split(':')
d[section][k] = v
if init or update:
self.slap.initializeConnection(self.server_url, self.key_file,
self.cert_file)
computer_partition = self.slap.registerComputerPartition(
self.computer_id, self.computer_partition_id)
published_dict = unwrap(computer_partition.getConnectionParameterDict())
publish = False
publish_dict = {}
for section, init in init.iteritems():
for k, v in init.iteritems():
try:
publish_dict[k] = published_dict[k]
except KeyError:
section, key = v.split(":")
publish_dict[k] = self.buildout[section][key]
publish_dict[k] = buildout[section][v]
publish = True
if publish:
computer_partition.setConnectionDict(wrap(publish_dict))
options.update(publish_dict)
for section, update in update.iteritems():
override = {}
for k, v in update.iteritems():
try:
override[v] = published_dict[k]
except KeyError:
pass
section = buildout[section]
patchOptions(section, override)
old = override.copy()
section.recipe.publish_early(override)
if override != old:
publish = True
for k, v in update.iteritems():
try:
publish_dict[k] = override[v]
except KeyError:
pass
if publish:
computer_partition.setConnectionDict(wrap(publish_dict))
publish = [k for k in options
if k != 'recipe' and not k.startswith('-')]
publish += publish_dict
publish_dict['-publish'] = ' '.join(publish)
patchOptions(options, publish_dict)
install = update = lambda self: None
......@@ -44,7 +44,6 @@ eggs =
[versions]
ecdsa = 0.13
erp5.util = 0.4.51
gitdb = 0.6.4
pycrypto = 2.6.1
pycurl = 7.43.0
......
......@@ -84,5 +84,4 @@ packages +=
python-dev
[versions]
erp5.util = 0.4.51
slapos.recipe.template = 4.3
......@@ -59,5 +59,4 @@ template =
raw runTestSuite_py ${buildout:bin-directory}/${runTestSuite_py:interpreter}
[versions]
erp5.util = 0.4.51
slapos.recipe.template = 4.3
......@@ -38,6 +38,5 @@ eggs =
[versions]
cns.recipe.symlink = 0.2.3
collective.recipe.environment = 0.2.0
erp5.util = 0.4.51
plone.recipe.command = 1.1
slapos.recipe.template = 4.3
......@@ -61,7 +61,6 @@ output = ${buildout:directory}/template-default.cfg
mode = 0644
[versions]
erp5.util = 0.4.58
slapos.recipe.template = 4.3
dnspython = 1.15.0
PyXML = 0.8.5
......@@ -93,4 +92,4 @@ zope.event = 4.4
# Required by:
# zope.testbrowser==5.3.2
zope.schema = 4.9.3
\ No newline at end of file
zope.schema = 4.9.3
......@@ -73,6 +73,3 @@ recipe = slapos.recipe.template
url = ${:_profile_base_location_}/${:filename}
output = ${buildout:directory}/template.cfg
mode = 0644
[versions]
erp5.util = 0.4.51
......@@ -115,7 +115,7 @@ output = ${buildout:directory}/template-nginx.cfg.in
output = ${buildout:directory}/runTestSuite.in
[versions]
erp5.util = 0.4.51
slapos.recipe.template = 4.3
selenium = 3.14.1
urllib3 = 1.24
......
......@@ -200,7 +200,6 @@ context =
# XXX - use websockify = 0.5.1 for compatibility with kvm frontend
websockify = 0.5.1
erp5.util = 0.4.51
collective.recipe.environment = 0.2.0
gitdb = 0.6.4
pycurl = 7.43.0
......
......@@ -35,7 +35,6 @@ eggs = erp5.util
interpreter = ${:_buildout_section_name_}
[versions]
erp5.util = 0.4.51
# To match ERP5
ZConfig = 2.9.3
zc.lockfile = 1.0.2
......
......@@ -124,7 +124,6 @@ ZODB3 = 3.11.0
numpy = 1.16.4
zope.testing = 4.6.2
pygolang = 0.0.0.dev4
erp5.util = 0.4.51
# Required by:
# ZEO==4.3.1
......
......@@ -25,4 +25,7 @@
Dav off
</IfModule>
</Directory>
ErrorLog "{{ parameter_dict['log-dir'] }}/nextcloud-error.log"
CustomLog "{{ parameter_dict['log-dir'] }}/nextcloud-access.log" combined
</VirtualHost>
\ No newline at end of file
......@@ -131,7 +131,7 @@ mode = 744
recipe = slapos.cookbook:cron.d
cron-entries = ${cron:cron-entries}
name = nextcloud-backup
frequency = 0 0 * * * *
frequency = 0 0 * * *
command = ${nextcloud-backup:output}
[nextcloud-optimize]
......
......@@ -24,7 +24,7 @@ md5sum = a2281f86f6a26a8ff40a57a495505977
[template-apache-httpd]
<= nc-download-base
filename = apache-httpd.conf.in
md5sum = f3bca64bf991526fd8221035a86aacbf
md5sum = 839258624e273aac71a96516bf34c7e6
[template-nextcloud-config.json]
<= nc-download-base
......@@ -36,7 +36,7 @@ recipe = slapos.recipe.template:jinja2
template = ${:_profile_base_location_}/nextcloud-instance.cfg.in
rendered = ${buildout:directory}/instance-nextcloud.cfg
extensions = jinja2.ext.do
md5sum = 5c721c7eb8a06147b849a65ba92becba
md5sum = 59e9c65e655cf9cf144d97dd36863ede
context =
key gzip_location gzip:location
key python3_location python3.6.6:location
......
......@@ -46,7 +46,6 @@ mode = 0644
[versions]
slapos.recipe.template = 4.3
erp5.util = 0.4.51
inotifyx = 0.2.2
gitdb2 = 2.0.3
smmap2 = 2.0.3
......@@ -67,7 +67,6 @@ cns.recipe.symlink = 0.2.3
plone.recipe.command = 1.1
slapos.recipe.template = 4.3
dnspython = 1.15.0
erp5.util = 0.4.53
passlib = 1.7.1
GitPython = 2.1.11
lockfile = 0.12.2
......
[buildout]
extends = software.cfg
shared-parts = /opt/slapgrid/shared-parts
eggs-directory = /opt/slapgrid/shared-eggs
abi-tag-eggs = true
......@@ -142,9 +142,6 @@ output = ${buildout:directory}/template.cfg
mode = 640
[versions]
# Recent erp5.util is needed
erp5.util = 0.4.58
# Various needed versions
Pillow = 5.3.0
PyNaCl = 1.3.0
......
[buildout]
extends = software.cfg
shared-parts = /opt/slapgrid/shared-parts
eggs-directory = /opt/slapgrid/shared-eggs
abi-tag-eggs = true
......@@ -8,6 +8,7 @@ extends =
../../software/erp5/software.cfg
parts +=
wendelin
erp5-bin
scipy
msgpack-python
msgpack-numpy
......@@ -60,10 +61,14 @@ initialization =
${testrunner:initialization}
[erp5_repository_list]
repository_id_list += wendelin
repository_id_list +=
wendelin
erp5-bin
[local-bt5-repository]
list += ${wendelin:location}/bt5
list +=
${wendelin:location}/bt5
${erp5-bin:location}/bt5
# Jupyter is by default enabled in Wendelin
[erp5-defaults]
......@@ -75,6 +80,12 @@ git-executable = ${git:location}/bin/git
repository = https://lab.nexedi.com/nexedi/wendelin.git
branch = master
[erp5-bin]
recipe = slapos.recipe.build:gitclone
git-executable = ${git:location}/bin/git
repository = https://lab.nexedi.com/nexedi/erp5-bin.git
branch = master
[versions]
msgpack = 0.6.1
msgpack-numpy = 0.4.4.3
......
......@@ -91,7 +91,3 @@ slapos.recipe.template = 4.3
# PasteScript==2.0
# cloudooo==1.2.5.dev0
PasteDeploy = 1.5.2
# Required by:
# cloudooo==1.2.5.dev0
erp5.util = 0.4.51
......@@ -78,7 +78,7 @@ md5sum = d41d8cd98f00b204e9800998ecf8427e
[template-erp5]
filename = instance-erp5.cfg.in
md5sum = 7dd00dedef4cc4320ec6977a7e2dc110
md5sum = 079244ece03662c7ceadb68570b69d2f
[template-zeo]
filename = instance-zeo.cfg.in
......
......@@ -397,11 +397,11 @@ recipe = slapos.cookbook:publish-early
{%- if has_posftix %}
smtpd-sasl-password gen-smtpd-sasl-password:passwd
{%- endif %}
{% if test_runner_enabled -%}
{% for zope_family_name in zope_family_name_list %}
{%- if test_runner_enabled %}
{%- for zope_family_name in zope_family_name_list %}
{{ zope_family_name }}-test-runner-url-list default-balancer-test-runner-url-list:default
{% endfor -%}
{% endif -%}
{%- endfor %}
{%- endif %}
{%- if neo %}
neo-cluster gen-neo-cluster:name
{%- if neo[0] %}
......
......@@ -119,7 +119,7 @@ context =
<= template-download-base
filename = instance-apache-php.cfg.in
output = ${buildout:parts-directory}/${:_buildout_section_name_}/${:filename}
md5sum = 7edcbca22e800f2c90559298c44d5a48
md5sum = 44796786448fd0319dde14923fbd798b
[instance-lamp]
<= template-download-base
......
......@@ -129,6 +129,7 @@ port = ${apache-network-configuration:listening-port}
url = https://[${:ip}]:${:port}/
error-log = ${directory:httpd-log}/error.log
access-log = ${directory:httpd-log}/access.log
log-dir = ${directory:httpd-log}
php-ini-dir = ${directory:php-ini-dir}
cert-file = ${ca-directory:certs}/httpd.crt
key-file = ${ca-directory:certs}/httpd.key
......@@ -153,7 +154,7 @@ wait-for-files =
[apache-graceful]
recipe = collective.recipe.template
output = ${directory:bin}/apache-httpd-graceful
output = ${directory:scripts}/apache-httpd-graceful
mode = 700
input = inline:
#!/bin/sh
......@@ -162,7 +163,7 @@ input = inline:
[logrotate-entry-apache]
<= logrotate-entry-base
name = apache
log = ${apache-php-configuration:error-log} ${apache-php-configuration:access-log}
log = ${apache-php-configuration:log-dir}/*.log
frequency = daily
rotate-num = 30
......
......@@ -136,8 +136,8 @@ pyparsing = 2.2.0
pytz = 2016.10
requests = 2.13.0
six = 1.11.0
slapos.cookbook = 1.0.92
slapos.core = 1.4.25
slapos.cookbook = 1.0.118
slapos.core = 1.4.26
slapos.extension.strip = 0.4
slapos.extension.shared = 1.0
slapos.libnetworkcache = 0.19
......@@ -154,7 +154,7 @@ CacheControl = 0.12.5
msgpack = 0.6.1
# Required by:
# slapos.core==1.4.25
# slapos.core==1.4.26
Flask = 0.12
# Required by:
......@@ -195,7 +195,7 @@ enum34 = 1.1.6
# Required by:
# slapos.toolbox==0.94
erp5.util = 0.4.51
erp5.util = 0.4.59
# Required by:
# slapos.toolbox==0.94
......@@ -218,7 +218,7 @@ pyrsistent = 0.14.5
ipaddress = 1.0.18
# Required by:
# slapos.cookbook==1.0.92
# slapos.cookbook==1.0.118
jsonschema = 3.0.0a3
# Required by:
......@@ -226,7 +226,7 @@ jsonschema = 3.0.0a3
lockfile = 0.12.2
# Required by:
# slapos.core==1.4.25
# slapos.core==1.4.26
# XXX 'slapos node format' raises an exception with netifaces 0.10.5.
netifaces = 0.10.4
......@@ -259,7 +259,7 @@ python-dateutil = 2.7.3
rpdb = 0.1.5
# Required by:
# slapos.core==1.4.25
# slapos.core==1.4.26
supervisor = 3.3.3
# Required by:
......@@ -267,11 +267,11 @@ supervisor = 3.3.3
tzlocal = 1.5.1
# Required by:
# slapos.core==1.4.25
# slapos.core==1.4.26
uritemplate = 3.0.0
# Required by:
# slapos.core==1.4.25
# slapos.core==1.4.26
zope.interface = 4.3.3
[networkcache]
......
Styling with Markdown is supported
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!