Commit f77a9213 authored by Julien Muchembled's avatar Julien Muchembled

publish_early: rework API

Commit ba6c3331 does not handle non-string
values properly. While fixing this bug, I also found that '-init' was bad:
sections were uninstalled on next runs, which is a problem with recipes
whose install() return paths.
parent 66e05850
master 1.0 ZODB3 alain-wrapper-recipe armin arnau arnau-RD-future arnau-RD-py3-master arnau-zope4py2 backup bbu cert-auto check_system_gfortran cloudooo_test_with_client_certificate cmf_upgrade_versions cythonplus debug-resiliency default_software_type_default django-tutorial draft_cloudooo_py3 drupal-lamp e2e e2e-ors erp5-component erp5-zope2-tests extend-erp5-wcfs-integration-tests failover-backend feat/buildout_testing_python3_new feat/debian11 feat/erp5-incremenatal-repozo feat/fix_kvm_wipe feat/gitea feat/inkscape-1.3 feat/ipv6_range feat/kvm-import-nice feat/libreoffice-24.2 feat/libreoffice-7.6 feat/mariadb-10.11 feat/mariadb-10.11-unsafe feat/mariadb-10.11bis feat/mariadb-10.11ter feat/mariadb-10.5 feat/mariadb-10.6 feat/mariadb-11.4 feat/mariadb-11.4-unsafe feat/mgroonga feat/new_vps_software feat/nxdbom feat/obs_test feat/openssl-3.0 feat/openssl-3.0-bis feat/openssl-3.0-py2 feat/telegraf-version-up feat/testcase feat/theia-1.24.0 feat/theia-bash-completions feat/version_up_ldap_graphviz feat/zope-testrunner feat/zope5.11-pygolang-zodbtools firefox-version-up fix-1.0.135.1 fix-1.0.135.2 fix-1.0.190.1 fix-1.0.207.1 fix-1.0.240.1 fix-libdb fix/cloudooo-rpath fix/erp5-haproxy-wait-reload fix/erp5_remove_unused_eggs fix/gcc_min_version fix/golang-20 fix/haproxy-up fix/jinja2_version_up fix/netrc-parse-error fix/re6st-token-manager fix/resilient_test fix/slapos.toolbox-py2 fix/version_up fix_kvm_monitor for_testrunner_1 gitlab-fixup gitlab-upgrade gitlab-v14.10 goodbye-openssl-1.0.x html5as html5as-base html5as-test html5as-tutorial improve-monitor-httpd intentially_broken_cloudooo jm json-api kvm_auto_reboot lle-bout/ppc64le logrotate/truncate master+ZODB4-wc2 matomo-tutorial moodle my2to3 neo-reflink netframe nextcloud nextcloud-fix-tests nirina_tutorial nofile old-mail-server ors ors-amarisoft/aw2s ors-dev ors-dev2 ors-dev3 ors-dev3-bak ors-dnsmasq ors-fake-lteenb ors-handover ors-oran-ru ors-oran-ru-toolbox ors-promise ors-ptt ors-publish ors-ssb-nr-arfcn ors-test ors-tests ors-ue osc patrowl-dev proview-r public-deltachat-core push-to-wendolin python3.12 remove_inkscape remove_tempstorage repman-1.0 repman-test resilience_all_parameters revert-e7b48c0b ru-alarms runner-multi-sr scalability_crash_mariadb simpleran-dev slapos-node_python3.9 slapos_master_load_balancer_test sozu-dev stack/supervisord systemd-python tdd_config_5ms_6ul_3dl telecom-matomo-hotfix test-fix-erp5-resiliency textsynth tomo_openradio_e2e_testing top-stand1 ttrm-novel-fix-testnode ttrn upgrade_erp5_test_updated upgrade_responses vanilla-slapos webdav wipapi wrapper xy/lte-multiru y/wc2-next zope2py2-reorder-versions-priority zope2zope4py2 zope4 zope4py2-faketime zope4py3 zope4py3-bstr zope4py3-kaz 1.0.396 1.0.395 1.0.394 1.0.393 1.0.392 1.0.391 1.0.390 1.0.389 1.0.388 1.0.387 1.0.386 1.0.385 1.0.384 1.0.383 1.0.382 1.0.381 1.0.380 1.0.379 1.0.378 1.0.377 1.0.376 1.0.375 1.0.374 1.0.373 1.0.372 1.0.371 1.0.370 1.0.369 1.0.368 1.0.367 1.0.366 1.0.365 1.0.364.3 1.0.364.2 1.0.364.1 1.0.364 1.0.363 1.0.362 1.0.361 1.0.360 1.0.359 1.0.358 1.0.357 1.0.356 1.0.355 1.0.354 1.0.354-mariadb-replication-efc8bfe3c 1.0.354-mariadb-replication-8b12a75b5 1.0.353 1.0.352 1.0.351 1.0.350 1.0.349 1.0.348 1.0.347 1.0.346 1.0.345 1.0.344.1 1.0.344 1.0.343 1.0.342 1.0.341 1.0.340 1.0.339 1.0.338 1.0.337 1.0.336 1.0.335 1.0.334 1.0.333 1.0.332 1.0.331 1.0.330 1.0.329 1.0.328 1.0.327 1.0.326 1.0.325 1.0.324 1.0.323 1.0.322 1.0.321 1.0.320 1.0.319 1.0.318 1.0.317 1.0.316 1.0.315 1.0.314 1.0.313 1.0.312 1.0.311 1.0.310 1.0.309 1.0.308 1.0.307 1.0.306 1.0.305 1.0.304 1.0.303 1.0.302 1.0.301 1.0.300 1.0.299 1.0.298 1.0.297 1.0.296 1.0.295 1.0.294 1.0.293 1.0.292 1.0.291 1.0.290 1.0.289 1.0.288 1.0.287 1.0.286 1.0.285 1.0.284 1.0.283 1.0.282 1.0.281 1.0.280 1.0.279 1.0.278 1.0.277 1.0.276 1.0.275 1.0.274 1.0.273 1.0.272 1.0.271 1.0.270 1.0.269 1.0.268 1.0.267 1.0.266 1.0.265 1.0.264 1.0.263 1.0.262 1.0.261 1.0.260 1.0.259 1.0.258 1.0.257 1.0.256 1.0.255 1.0.254 1.0.253 1.0.252 1.0.251 1.0.250 1.0.249 1.0.248 1.0.247 1.0.246 1.0.245 1.0.244 1.0.243 1.0.242 1.0.241 1.0.240.1 1.0.240 1.0.239 1.0.238 1.0.237 1.0.236 1.0.235 1.0.234 1.0.233 1.0.232 1.0.231 1.0.230 1.0.229 1.0.228 1.0.227 1.0.226 1.0.225 1.0.224 1.0.223 1.0.222 1.0.221 1.0.220 1.0.219 1.0.218 1.0.217 1.0.216 1.0.215 1.0.214 1.0.213 1.0.212 1.0.211 1.0.210 1.0.209 1.0.208 1.0.207.1 1.0.207 1.0.206 1.0.205 1.0.204 1.0.203 1.0.202 1.0.201 1.0.200 1.0.199 1.0.198 1.0.197 1.0.196 1.0.195 1.0.194 1.0.193 1.0.192 1.0.191 1.0.190.1 1.0.190 1.0.189 1.0.188 1.0.187 1.0.186 1.0.185 1.0.184 1.0.183 1.0.182 1.0.181 1.0.180 1.0.179 1.0.178 1.0.177 1.0.176 1.0.175 1.0.174 1.0.173 1.0.172 1.0.171 1.0.170 1.0.169 1.0.168 1.0.167.10 1.0.167.9 1.0.167.8 1.0.167.7 1.0.167.6 1.0.167.5 1.0.167.4 1.0.167.3 1.0.167.2 1.0.167.1 1.0.167 1.0.166 1.0.165 1.0.164 1.0.163 1.0.162 1.0.161.1 1.0.161 1.0.160 1.0.159 1.0.158 1.0.157 1.0.156 1.0.155 1.0.154 1.0.153 1.0.152 1.0.151 1.0.150 1.0.149 1.0.148 1.0.147 1.0.146 1.0.145 1.0.144 1.0.143 1.0.142 1.0.141 1.0.140 1.0.137 1.0.136 1.0.135.2 1.0.135.1 1.0.135 1.0.134 1.0.133 1.0.132 1.0.131 1.0.130 1.0.129 1.0.128 1.0.127 1.0.126 1.0.125 1.0.124 1.0.123 1.0.122 1.0.121 1.0.120 1.0.119 slapos.cookbook-1.0.124 seleniumrunner-firefox60 re6stnet-0.551+2 re6stnet-0.548+3 re6stnet-0.541+1 re6stnet-0.539+3 re6stnet-0.539+2 re6stnet-0.539+1 re6stnet-0.536+2
No related merge requests found
......@@ -32,30 +32,26 @@ from .librecipe import GenericBaseRecipe
class Cluster(object):
def __init__(self, buildout, name, options):
self.buildout = buildout
self.options = options
def publish_early(self, publish_dict):
masters = publish_dict.setdefault('masters', '')
masters = options.setdefault('masters', '')
result_dict = {
'connection-admin': [],
'connection-master': [],
}
node_list = []
for node in sorted(self.options['nodes'].split()):
node = self.buildout[node]
for node in sorted(options['nodes'].split()):
node = buildout[node]
node_list.append(node)
for k, v in result_dict.iteritems():
x = node[k]
if x:
v.append(x)
publish_dict['admins'] = ' '.join(result_dict.pop('connection-admin'))
options['admins'] = ' '.join(result_dict.pop('connection-admin'))
x = ' '.join(result_dict.pop('connection-master'))
if masters != x:
publish_dict['masters'] = x
options['masters'] = x
for node in node_list:
node['config-masters'] = x
node.recipe.__init__(self.buildout, node.name, node)
node.recipe.__init__(buildout, node.name, node)
install = update = lambda self: None
......
......@@ -28,20 +28,14 @@
from collections import defaultdict
from .librecipe import unwrap, wrap, GenericSlapRecipe
def patchOptions(options, override):
def get(option, *args, **kw):
try:
return override[option]
except KeyError:
return options_get(option, *args, **kw)
try:
options_get = options._get
except AttributeError:
options_get = options.get
options.get = get
else:
options._get = get
def volatileOptions(options, volatile):
def copy():
copy = options_copy()
for key in volatile:
copy.pop(key, None)
return copy
options_copy = options.copy
options.copy = copy
class Recipe(GenericSlapRecipe):
"""
......@@ -57,8 +51,6 @@ class Recipe(GenericSlapRecipe):
-init =
foo gen-foo:x
bar gen-bar:y
-update =
baz update-baz:z
bar = z
[gen-foo]
......@@ -69,72 +61,74 @@ class Recipe(GenericSlapRecipe):
-extends = publish-early
...
${publish-early:foo} is initialized with the value of the published
parameter 'foo', or ${gen-foo:x} if it hasn't been published yet
(and in this case, it is published immediately as a way to save the value).
Just before the recipe of [gen-foo] is instantiated, 'x' is overridden with
the published value 'foo' if it exists. If its __init__ modifies 'x', the new
value is published. To prevent [gen-foo] from being accessed too early, 'x'
is then removed and the value can only be accessed with ${publish-early:foo}.
Generated values don't end up in the buildout installed file, which is good
if they're secret. Note however that buildout won't detect if values change
and it may only call update().
${publish-early:bar} is forced to 'z' (${gen-bar:y} ignored):
a line like 'bar = z' is usually rendered conditionally with Jinja2.
The '-update' option has the same syntax than '-init'. The recipes of the
specified sections must implement 'publish_early(publish_dict)':
- it is always called, just before early publishing
- publish_dict is a dict with already published values
- 'publish_early' can change published values by modifying publish_dict.
In the above example:
- publish_dict is {'z': ...}
- during the execution of 'publish_early', other sections can access the
value with ${update-baz:z}
- once [publish-early] is initialized, the value should be accessed with
${publish-early:bar} ([update-baz] does not have it if it's accessed
before [publish-early])
"""
def __init__(self, buildout, name, options):
GenericSlapRecipe.__init__(self, buildout, name, options)
init = defaultdict(dict)
update = defaultdict(dict)
for d, k in (init, '-init'), (update, '-update'):
for line in options.get(k, '').splitlines():
if line:
k, v = line.split()
if k not in options:
section, v = v.split(':')
d[section][k] = v
if init or update:
for line in options['-init'].splitlines():
if line:
k, v = line.split()
if k not in options:
section, v = v.split(':')
init[section][k] = v
if init:
self.slap.initializeConnection(self.server_url, self.key_file,
self.cert_file)
computer_partition = self.slap.registerComputerPartition(
self.computer_id, self.computer_partition_id)
published_dict = unwrap(computer_partition.getConnectionParameterDict())
Options = buildout.Options
if 'Options' in buildout.__dict__:
def revertOptions():
buildout.Options = Options
else:
def revertOptions():
try:
del buildout.Options
except AttributeError:
pass
def newOptions(buildout, section, data):
assert section == init_section, (section, init_section)
revertOptions()
self = buildout.Options(buildout, section, data)
self.update(override)
return self
publish = False
publish_dict = {}
for section, init in init.iteritems():
for k, v in init.iteritems():
try:
publish_dict[k] = published_dict[k]
except KeyError:
publish_dict[k] = buildout[section][v]
try:
for init_section, init in init.iteritems():
override = {}
for k, v in init.iteritems():
try:
override[v] = published_dict[k]
except KeyError:
pass
buildout.Options = newOptions
init_section = buildout[init_section]
assert buildout.Options is Options
new = {}
for k, v in init.iteritems():
try:
publish_dict[k] = new[v] = init_section.pop(v)
except KeyError:
pass
if new != override:
publish = True
for section, update in update.iteritems():
override = {}
for k, v in update.iteritems():
try:
override[v] = published_dict[k]
except KeyError:
pass
section = buildout[section]
patchOptions(section, override)
old = override.copy()
section.recipe.publish_early(override)
if override != old:
publish = True
for k, v in update.iteritems():
try:
publish_dict[k] = override[v]
except KeyError:
pass
finally:
revertOptions()
if publish:
computer_partition.setConnectionDict(wrap(publish_dict))
......@@ -143,6 +137,7 @@ class Recipe(GenericSlapRecipe):
if k != 'recipe' and not k.startswith('-')]
publish += publish_dict
publish_dict['-publish'] = ' '.join(publish)
patchOptions(options, publish_dict)
volatileOptions(options, list(publish_dict))
options.update(publish_dict)
install = update = lambda self: None
......@@ -36,8 +36,8 @@ import errno
import os
import random
import string
from slapos.recipe.librecipe import GenericBaseRecipe
from .librecipe import GenericBaseRecipe
from .publish_early import volatileOptions
class Integer(object):
"""
......@@ -54,7 +54,9 @@ class Integer(object):
Resulting integer.
"""
def __init__(self, buildout, name, options):
options['value'] = random.randint(int(options['minimum']), int(options['maximum']))
if 'value' not in options:
options['value'] = random.randint(int(options['minimum']),
int(options['maximum']))
def install(self):
pass
......@@ -65,10 +67,9 @@ class Time(object):
"""Generate a random time from a 24h time clock"""
def __init__(self, buildout, name, options):
self.name = name
self.buildout = buildout
self.options = options
self.options['time'] = "%d:%d" % (random.randint(0, 23), random.randint(0, 59))
if 'time' not in options:
options['time'] = "%u:%02u" % (
random.randint(0, 23), random.randint(0, 59))
def install(self):
pass
......@@ -76,26 +77,33 @@ class Time(object):
update = install
class Mac(GenericBaseRecipe):
class Mac(object):
def __init__(self, buildout, name, options):
if os.path.exists(options['storage-path']):
open_file = open(options['storage-path'], 'r')
options['mac-address'] = open_file.read()
open_file.close()
if options.get('mac-address', '') == '':
# First octet has to represent a locally administered address
octet_list = [254] + [random.randint(0x00, 0xff) for x in range(5)]
options['mac-address'] = ':'.join(['%02x' % x for x in octet_list])
return GenericBaseRecipe.__init__(self, buildout, name, options)
self.storage_path = options['storage-path']
mac = options.get('mac-address')
if not mac:
try:
with open(self.storage_path) as f:
mac = f.read()
except IOError as e:
if e.errno != errno.ENOENT:
raise
if not mac:
# First octet has to represent a locally administered address
octet_list = [254] + [random.randint(0x00, 0xff) for x in range(5)]
mac = ':'.join(['%02x' % x for x in octet_list])
self.update = self.install
options['mac-address'] = mac
self.mac = mac
def install(self):
open_file = open(self.options['storage-path'], 'w')
open_file.write(self.options['mac-address'])
open_file.close()
return [self.options['storage-path']]
with open(self.storage_path, 'w') as f:
f.write(self.mac)
return self.storage_path
def update(self):
pass
def generatePassword(length):
return ''.join(random.SystemRandom().sample(string.ascii_lowercase, length))
......@@ -130,30 +138,24 @@ class Password(object):
except KeyError:
self.storage_path = options['storage-path'] = os.path.join(
buildout['buildout']['parts-directory'], name)
passwd = None
if self.storage_path:
try:
with open(self.storage_path) as f:
passwd = f.read().strip('\n')
except IOError as e:
if e.errno != errno.ENOENT:
raise
passwd = options.get('passwd')
if not passwd:
passwd = self.generatePassword(int(options.get('bytes', '8')))
self.update = self.install
self.passwd = passwd
if self.storage_path:
try:
with open(self.storage_path) as f:
passwd = f.read().strip('\n')
except IOError as e:
if e.errno != errno.ENOENT:
raise
if not passwd:
passwd = self.generatePassword(int(options.get('bytes', '8')))
self.update = self.install
options['passwd'] = passwd
# Password must not go into .installed file, for 2 reasons:
# security of course but also to prevent buildout to always reinstall.
def get(option, *args, **kw):
return passwd if option == 'passwd' else options_get(option, *args, **kw)
try:
options_get = options._get
except AttributeError:
options_get = options.get
options.get = get
else:
options._get = get
# publish_early already does it, but this recipe may also be used alone.
volatileOptions(options, ('passwd',))
self.passwd = passwd
generatePassword = staticmethod(generatePassword)
......@@ -179,4 +181,4 @@ class Password(object):
return self.storage_path
def update(self):
return ()
pass
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment