diff --git a/software/erp5/instance-erp5-input-schema.json b/software/erp5/instance-erp5-input-schema.json index 945c5142ce8e45f0536b07ecb8b0629c150473f2..337d7e7c4ec0249f330a7f2c075d07152f468431 100644 --- a/software/erp5/instance-erp5-input-schema.json +++ b/software/erp5/instance-erp5-input-schema.json @@ -39,6 +39,23 @@ "uniqueItems": true, "type": "array" }, + "hosts": { + "description": "Host entries to be used in addition to and/or overriding auto-generated ones", + "default": { + "erp5-catalog-0": "some-ip", + "erp5-catalog-...": "some-ip", + "erp5-cloudooo": "some-ip", + "erp5-memcached-persistent": "some-ip", + "erp5-memcached-volatile": "some-ip" + }, + "patternProperties": { + ".*": { + "description": "An IP to which current entry will resolve", + "type": "string" + } + }, + "type": "object" + }, "frontend": { "description": "Front-end slave instance request parameters", "properties": { diff --git a/stack/erp5/buildout.cfg b/stack/erp5/buildout.cfg index c141a44be82c5f28be5fae8ed8159d494a665241..d547d7b933982a0776e5d461bacd881e4ba58406 100644 --- a/stack/erp5/buildout.cfg +++ b/stack/erp5/buildout.cfg @@ -66,6 +66,7 @@ extends = ../../component/jsl/buildout.cfg ../../component/6tunnel/buildout.cfg ../../component/findutils/buildout.cfg + ../../component/libuserhosts/buildout.cfg parts = rdiff-backup @@ -101,6 +102,7 @@ parts = dcron dash wget + libuserhosts # Buildoutish patched-eggs @@ -205,6 +207,11 @@ md5sum = 6d52007d9bdc25ed0c83a49d63d59a18 filename = zope.conf.in md5sum = 8fe36c41ab784f547b968dc6edd0ca29 +[template-runzope-userhosts-preloaded] +< = download-base +filename = runzope_userhosts_preloaded.in +md5sum = 479fc3a107ff965dfb6e7e1222d81f28 + [template-my-cnf] < = download-base filename = my.cnf.in @@ -218,19 +225,19 @@ md5sum = b8d2d9af0c4cab45c8337aeac28d5fae [template-create-erp5-site] < = download-base filename = instance-create-erp5-site.cfg.in -md5sum = 4504b8e58cf6eb0f17ef30c29c04432d +md5sum = 2ce65ad372433ae8e11e30f3191bca68 [template-create-erp5-site-real] < = download-base filename = instance-create-erp5-site-real.cfg.in -md5sum = ba1b23177e101b5b9f03e1c5009c81fc +md5sum = e106cdd73c0a86a9b2c806062573ca8f [template] < = template-jinja2-base # XXX: "template.cfg" is hardcoded in instanciation recipe filename = template.cfg template = ${:_profile_base_location_}/instance.cfg.in -md5sum = ed053189e234a27f0b2f2b8b53c532bf +md5sum = 3900ab427f44e41668d9b36411e46149 extra-context = key mariadb_link_binary template-mariadb:link-binary key zope_link_binary template-zope:link-binary @@ -266,6 +273,7 @@ extra-context = key libpng12_location libpng12:location key libreoffice_bin_location libreoffice-bin:location key librsvg_location librsvg:location + key libuserhosts_location libuserhosts:location key libxcb_location libxcb:location key local_bt5_repository local-bt5-repository:list key logrotate_location logrotate:location @@ -286,6 +294,7 @@ extra-context = key template_mariadb_initial_setup template-mariadb-initial-setup:target key template_my_cnf template-my-cnf:target key template_neo template-neo:target + key template_runzope_userhosts_preloaded template-runzope-userhosts-preloaded:target key template_zeo template-zeo:target key template_zope template-zope:target key template_zope_conf template-zope-conf:target @@ -295,7 +304,7 @@ extra-context = [template-erp5] < = download-base filename = instance-erp5.cfg.in -md5sum = 62d07372e84d237c6b7c4ef33f98a679 +md5sum = b628da98dbed2f7e46f0de207b4bd228 [template-neo] < = download-base @@ -310,7 +319,7 @@ md5sum = 7bbb690cb2ea38cd2aa84c8a79c50399 [template-zope] < = download-base filename = instance-zope.cfg.in -md5sum = 6a566960a2617a59fe0c8c77622dbac9 +md5sum = 0db31172368f14eeeb2ae8e22af30d9e link-binary = ${aspell:location}/bin/aspell ${coreutils:location}/bin/basename diff --git a/stack/erp5/instance-create-erp5-site-real.cfg.in b/stack/erp5/instance-create-erp5-site-real.cfg.in index e9b58334a7ecbc902e7085f745771e443e49815d..d8919afb05d34daaeda2f6a7534a78f3056124b7 100644 --- a/stack/erp5/instance-create-erp5-site-real.cfg.in +++ b/stack/erp5/instance-create-erp5-site-real.cfg.in @@ -7,8 +7,14 @@ promise = ${:etc}/promise [erp5-bootstrap] recipe = slapos.cookbook:erp5.bootstrap runner-path = ${directory:services}/erp5-bootstrap +{# Note: a random domain name will be picked if several point to the same IP -#} +{% set reverse_hosts = {} -%} +{% for x, y in publish['hosts-dict'].iteritems() -%} +{% do reverse_hosts.__setitem__(y, x) -%} +{% endfor -%} {# XXX: Expect the first database to be the one to use for catalog. -#} -mysql-url = {{ publish['mariadb-database-list'][0] }} +{% set mysql_parsed = urlparse.urlparse(publish['mariadb-database-list'][0]) -%} +mysql-url = {{ urlparse.urlunparse(mysql_parsed[:1] + (reverse_hosts.get(mysql_parsed.hostname, mysql_parsed.hostname) ~ ':' ~ mysql_parsed.port, ) + mysql_parsed[2:]) }} {# Pick the first http[s] family found, they should be all equivalent anyway. -#} {% set family_list = [] -%} {% for key, value in publish.items() -%} diff --git a/stack/erp5/instance-create-erp5-site.cfg.in b/stack/erp5/instance-create-erp5-site.cfg.in index a3552e10ea7096ad61455c1c1e6094535937f90d..196845ae6f2b896483ffbf3f1723e9f08ca8e590 100644 --- a/stack/erp5/instance-create-erp5-site.cfg.in +++ b/stack/erp5/instance-create-erp5-site.cfg.in @@ -16,6 +16,7 @@ template = {{ parameter_dict['template-create-erp5-site-real'] }} rendered = ${buildout:directory}/instance-create-erp5-site-real.cfg extensions = jinja2.ext.do context = + import urlparse urlparse section publish publish section parameter_dict instance-create-erp5-site-real-parameters key eggs_directory buildout:eggs-directory diff --git a/stack/erp5/instance-erp5.cfg.in b/stack/erp5/instance-erp5.cfg.in index f4944660a2f9cbbc6f5ea32ac59c4e88d6420c6a..4cb22fd89893993c0df29f84de3627b52f1e8d65 100644 --- a/stack/erp5/instance-erp5.cfg.in +++ b/stack/erp5/instance-erp5.cfg.in @@ -67,12 +67,14 @@ recipe = slapos.cookbook:generate.password < = request-common return = zope-address-list + hosts-dict extra-config = bt5 bt5-repository-url cloudooo-url deadlock-debugger-password developer-list + hosts-dict inituser-login inituser-password instance-count @@ -99,6 +101,7 @@ config-bt5-repository-url = {{ dumps(slapparameter_dict.get('bt5-repository-url' config-cloudooo-url = ${request-cloudooo:connection-url} config-deadlock-debugger-password = ${deadlock-debugger-password:passwd} config-developer-list = {{ dumps(slapparameter_dict.get('developer-list', [inituser_login])) }} +config-hosts-dict = {{ dumps(slapparameter_dict.get('hosts-dict', {})) }} config-inituser-login = {{ dumps(inituser_login) }} config-inituser-password = ${inituser-password:passwd} config-kumofs-url = ${request-memcached-persistent:connection-url} @@ -208,6 +211,11 @@ return = site_url recipe = slapos.cookbook:publish.serialised deadlock-debugger-password = ${deadlock-debugger-password:passwd} inituser-password = ${inituser-password:passwd} +{# +Pick any published hosts-dict, they are expected to be identical - and there is +no way to check here. +-#} +hosts-dict = {{ '${' ~ zope_address_list_id_dict.keys()[0] ~ ':connection-hosts-dict}' }} {% for name, value in publish_dict.items() -%} {{ name }} = {{ value }} {% endfor -%} diff --git a/stack/erp5/instance-zope.cfg.in b/stack/erp5/instance-zope.cfg.in index 5430dc65d55cf997bec43198be8594a1f46f106f..2864bb877e9c7e178779222e9e5c6c681082679e 100644 --- a/stack/erp5/instance-zope.cfg.in +++ b/stack/erp5/instance-zope.cfg.in @@ -164,6 +164,53 @@ ipv4 = ipv4-port = {% endif -%} +{% set hosts_dict = {} -%} +{% for alias, url in ( + ('erp5-memcached-volatile', slapparameter_dict['memcached-url']), + ('erp5-memcached-persistent', slapparameter_dict['kumofs-url']), + ('erp5-cloudooo', slapparameter_dict['cloudooo-url']), + ) -%} +{% do hosts_dict.__setitem__( + alias, + urlparse.urlparse(url).hostname, + ) -%} +{%- endfor %} +{# jinja2 does not support enumerate... -#} +{% set catalog_counter = 0 %} +{% for url in slapparameter_dict['mysql-url-list'] -%} +{% do hosts_dict.__setitem__( + 'erp5-catalog-' ~ catalog_counter, + urlparse.urlparse(url).hostname, + ) -%} +{% set catalog_counter = catalog_counter + 1 -%} +{%- endfor %} +{% do hosts_dict.update(slapparameter_dict['hosts-dict']) -%} +[hosts-parameter] +host-dict = {{ dumps(hosts_dict) }} + +[hosts] +recipe = slapos.recipe.template:jinja2 +template = inline: {{ ' + {% for alias, aliased in host_dict.items() -%} + {{ aliased }} {{ alias }} + {% endfor %} +' }} +rendered = ${directory:etc}/hosts +context = key host_dict hosts-parameter:host-dict + +[preload-userhosts-runzope-parameter] +runzope-binary = {{ bin_directory }}/runzope +libuserhosts = {{ parameter_dict['libuserhosts'] }} +shell-path = {{ parameter_dict['dash'] }}/bin/dash +hosts = ${hosts:rendered} + +[preload-userhosts-runzope] +recipe = slapos.recipe.template:jinja2 +rendered = ${directory:bin}/runzope_userhosts_preloaded +context = section parameter_dict preload-userhosts-runzope-parameter +template = {{ parameter_dict['runzope-userhosts-preloaded-template'] }} +mode = 755 + [zope-base] recipe = slapos.cookbook:generic.zope.zeo.client inituser = ${directory:instance}/inituser @@ -173,7 +220,7 @@ timezone = {{ dumps(slapparameter_dict['timezone']) }} tmp-path = ${directory:tmp} bin-path = ${directory:bin} site-zcml = ${directory:instance-etc}/site.zcml -runzope-binary = {{ bin_directory }}/runzope +runzope-binary = ${preload-userhosts-runzope:rendered} bt5-repository = [zope-conf-parameter-base] @@ -295,6 +342,14 @@ post = {{ bin_directory }}/killpidfromfile {{ '${' ~ conf_parameter_name ~ ':pid [publish-zope] recipe = slapos.cookbook:publish.serialised zope-address-list = {{ dumps(publish_list) }} +{# +Note: hosts_dist is generated at zope level rather than at erp5 (root partition) +level, as it is easier: we can access urls as python values trivially here. +This has the downside of making each zope partition publish the (hopefuly) same +dict toward erp5 partition, violating the DRY principle and making the intent +hard to guess. +-#} +hosts-dict = {{ dumps(hosts_dict) }} [erp5-promise] recipe = slapos.cookbook:erp5.promise diff --git a/stack/erp5/instance.cfg.in b/stack/erp5/instance.cfg.in index c8581358677e14a7a1c49431fd511632106d9d85..2da8e2193ac6b2d5cc8bb356eba97bb2b6f22dd5 100644 --- a/stack/erp5/instance.cfg.in +++ b/stack/erp5/instance.cfg.in @@ -141,6 +141,8 @@ buildout-bin-directory = {{ buildout_bin_directory }} dash = {{ dash_location }} jsl = {{ jsl_location }} link-binary = {{ dumps(zope_link_binary) }} +libuserhosts = {{ libuserhosts_location }} +runzope-userhosts-preloaded-template = {{ template_runzope_userhosts_preloaded }} [dynamic-template-zope] < = jinja2-template-base @@ -150,6 +152,7 @@ extensions = jinja2.ext.do extra-context = key buildout_directory buildout:directory section parameter_dict dynamic-template-zope-parameters + import urlparse urlparse # Must match the key id in [switch-softwaretype] which uses this section. key software_type :software-type software-type = zope diff --git a/stack/erp5/runzope_userhosts_preloaded.in b/stack/erp5/runzope_userhosts_preloaded.in new file mode 100644 index 0000000000000000000000000000000000000000..cad96f71a027d20fa87f97d3d3b61eb29e5a333b --- /dev/null +++ b/stack/erp5/runzope_userhosts_preloaded.in @@ -0,0 +1,3 @@ +#!{{ parameter_dict['shell-path'] }} +export HOSTS='{{ parameter_dict['hosts'] }}' LD_PRELOAD='{{ parameter_dict['libuserhosts'] }}' +exec {{ parameter_dict['runzope-binary'] }} "$@"