Skip to content

slapos
slapos
Commit d35284d8 authored by Saurabh's avatar Saurabh Committed by Julien Muchembled
1 Browse files
Options

Make it possible to instanciate 1 NEO DB inside an ERP5 instance 

Before it was only possible to make an ERP5 cluster connect to a NEO cluster
that was instanciated separately, by passing "name" and "master_nodes"
connection parameters in "storage-dict".

For an internal NEO DB, "name" and "master_nodes" is filled automatically
and you must instead pass a "server" dict, with same parameters as in NEO SR.
Currently, a NEO cluster name must be given. Later, we hope to generate a good
name automatically.

All this was implemented by refactoring NEO & ERP5 SR, with common files.
For the ERP5 SR, the root partition also serves as "root" partition for NEO
partitions: in other words, there's no second empty partition.
parent 6bbb61a8
Hide whitespace changes
Inline Side-by-side
Showing with 334 additions and 335 deletions
software/erp5/instance-erp5-input-schema.json
View file @ d35284d8
......@@ -212,9 +212,10 @@
"type": "string"
},
"server": {
"description": "Instantiate a server. If missing, 'storage-dict' must contain the necessary properties to mount the ZODB.",
"description": "Instantiate a server. If missing, 'storage-dict' must contain the necessary properties to mount the ZODB. For ZEO, the partition reference is 'zodb'. For NEO, they are 'neo-0', 'neo-1', ...",
"anyOf": [
{"$ref": "./instance-zeo-schema.json"}
{"$ref": "./instance-zeo-schema.json"},
{"$ref": "../neoppod/instance-neo-input-schema.json"}
]
},
"storage-dict": {
......
software/neoppod/cluster.cfg.in
View file @ d35284d8
{% macro section(name) %}{% do part_list.append(name) %}{{ name }}{% endmacro -%}
{% set part_list = [] -%}
{% set section_id_list = [] -%}
{% set sla_dict = {} -%}
{% for sla, ref_list in slapparameter_dict.get('sla-dict', {}).iteritems() -%}
{% do sla_dict.update(dict.fromkeys(ref_list, sla)) -%}
{% endfor -%}
{% import "root_common" as common_macro with context %}
[request-common]
recipe = slapos.cookbook:request.serialised
software-url = ${slap-connection:software-release-url}
server-url = ${slap-connection:server-url}
key-file = ${slap-connection:key-file}
cert-file = ${slap-connection:cert-file}
computer-id = ${slap-connection:computer-id}
partition-id = ${slap-connection:partition-id}
config-cluster = {{ dumps(slapparameter_dict['cluster']) }}
{% set replicas = slapparameter_dict.get('replicas', 0) -%}
config-partitions = {{ dumps(slapparameter_dict.get('partitions', 12)) }}
config-replicas = {{ dumps(replicas) }}
config-upstream-cluster = {{ dumps(slapparameter_dict.get('upstream-cluster', '')) }}
config-upstream-masters = {{ dumps(slapparameter_dict.get('upstream-masters', '')) }}
software-type = neo
{% macro sla(name) -%}
{% if name in sla_dict -%}
{% for k, (v,) in urlparse.parse_qs(sla_dict.pop(name), strict_parsing=1).iteritems() -%}
sla-{{ k }} = {{ v }}
{% endfor -%}
{% else -%}
sla-computer_guid = ${slap-connection:computer-id}
{% endif -%}
{% endmacro -%}
<= request-common-base
{{ common_macro.request_neo(slapparameter_dict, 'neo', 'node-') }}
[publish]
recipe = slapos.cookbook:publish.serialised
masters = ${node-0-final:connection-masters}
admins = ${node-0-final:connection-admins}
{% set node_list = slapparameter_dict.get('node-list', ({},)) -%}
{% do assert(replicas < len(node_list)) -%}
{% for i, node in enumerate(node_list) -%}
{% set section_id = 'node-' ~ i -%}
{% do section_id_list.append(section_id) -%}
[{{ section_id }}-base]
name = {{ section_id }}
{% for k, v in node.iteritems() -%}
config-{{ k }} = {{ dumps(v) }}
{% endfor -%}
{{ sla(section_id) }}
[{{ section_id }}]
<= request-common
{{ section_id }}-base
return =
master
admin
{% endfor -%}
[final-base]
{% for i, section_id in enumerate(section_id_list) -%}
config-master-{{i}} = {{ '${' + section_id + ':connection-master}' }}
config-admin-{{i}} = {{ '${' + section_id + ':connection-admin}' }}
{% endfor -%}
{% for section_id in section_id_list -%}
[{{ section(section_id + '-final') }}]
<= request-common
final-base
{{ section_id }}-base
{% if loop.first -%}
return =
masters
admins
{% endif -%}
{% endfor -%}
[buildout]
parts =
{{ part_list | join('\n\t') }}
publish
neo-masters = ${node-0-final:connection-masters}
neo-admins = ${node-0-final:connection-admins}
{{ common_macro.common_section() }}
software/neoppod/instance-common.cfg.in 0 → 100644
View file @ d35284d8
[buildout]
parts = switch-softwaretype
eggs-directory = {{ eggs_directory }}
develop-eggs-directory = {{ develop_eggs_directory }}
[slap-configuration]
recipe = slapos.cookbook:slapconfiguration.serialised
computer = ${slap-connection:computer-id}
partition = ${slap-connection:partition-id}
url = ${slap-connection:server-url}
key = ${slap-connection:key-file}
cert = ${slap-connection:cert-file}
[jinja2-template-base]
recipe = slapos.recipe.template:jinja2
filename = ${:_buildout_section_name_}.cfg
rendered = ${buildout:parts-directory}/${:_buildout_section_name_}/${:filename}
extensions = jinja2.ext.do
extra-context =
context =
key ipv4_set slap-configuration:ipv4
key ipv6_set slap-configuration:ipv6
key slapparameter_dict slap-configuration:configuration
raw logrotate_cfg {{ template_logrotate_base }}
Please register or sign in to reply
raw bin_directory {{ bin_directory }}
${:extra-context}
[neo-admin]
<= jinja2-template-base
template = {{ neo_admin }}
[neo-master]
<= jinja2-template-base
template = {{ neo_master }}
[neo-storage-mysql]
<= jinja2-template-base
template = {{ neo_storage_mysql }}
extra-context =
key master_cfg neo-master:rendered
key admin_cfg neo-admin:rendered
raw mariadb_location {{ mariadb_location }}
raw template_neo_my_cnf {{ template_neo_my_cnf }}
software/neoppod/instance-neo-input-schema.json
View file @ d35284d8
......@@ -27,7 +27,7 @@
"type": "string"
},
"sla-dict": {
"description": "Where to request instances. Each key is a query string for criterions (e.g. \"computer_guid=foo\"), and each value is a list of partition references.",
"description": "[NEO SR only] Where to request instances. Each key is a query string for criterions (e.g. \"computer_guid=foo\"), and each value is a list of partition references ('node-0', 'node-1', ...).",
"additionalProperties": {
"type": "array",
"items": { "type": "string" },
......
software/neoppod/instance-neo-storage-mysql.cfg.in
View file @ d35284d8
......@@ -24,7 +24,7 @@ init-file = ${init-script:rendered}
recipe = slapos.recipe.template:jinja2
extensions = jinja2.ext.do
rendered = ${directory:etc}/mariadb.cnf
template = {{ template_my_cnf }}
template = {{ template_neo_my_cnf }}
context = section parameter_dict my-cnf-parameters
{% set master_list = [] -%}
......
software/neoppod/instance.cfg.in
View file @ d35284d8
[buildout]
parts = switch-softwaretype
eggs-directory = {{ eggs_directory }}
develop-eggs-directory = {{ develop_eggs_directory }}
[slap-configuration]
recipe = slapos.cookbook:slapconfiguration.serialised
computer = ${slap-connection:computer-id}
partition = ${slap-connection:partition-id}
url = ${slap-connection:server-url}
key = ${slap-connection:key-file}
cert = ${slap-connection:cert-file}
extends = {{ instance_common_cfg }}
[jinja2-template-base]
recipe = slapos.recipe.template:jinja2
rendered = ${buildout:parts-directory}/${:_buildout_section_name_}.cfg
extensions = jinja2.ext.do
extra-context =
context =
key ipv4_set slap-configuration:ipv4
raw bin_directory {{ bin_directory }}
key slapparameter_dict slap-configuration:configuration
raw logrotate_cfg {{ logrotate_base }}
${:extra-context}
[neo-cluster]
<= jinja2-template-base
template = {{ cluster }}
extra-context =
import urlparse urlparse
[neo-admin]
<= jinja2-template-base
template = {{ neo_admin }}
[neo-master]
<= jinja2-template-base
template = {{ neo_master }}
[neo-storage-mysql]
<= jinja2-template-base
template = {{ neo_storage_mysql }}
extra-context =
key master_cfg neo-master:rendered
key admin_cfg neo-admin:rendered
raw mariadb_location {{ mariadb_location }}
raw template_my_cnf {{ template_my_cnf }}
import-list =
rawfile root_common {{ root_common }}
[switch-softwaretype]
recipe = slapos.cookbook:switch-softwaretype
......
software/neoppod/root-common.cfg.in 0 → 100644
View file @ d35284d8
{% macro section(name) %}{% do part_list.append(name) %}{{ name }}{% endmacro -%}
{% set part_list = [] -%}
{% set sla_dict = {} -%}
{% for sla, ref_list in slapparameter_dict.get('sla-dict', {}).iteritems() -%}
{% do sla_dict.update(dict.fromkeys(ref_list, sla)) -%}
{% endfor -%}
{% macro sla(name, required=False) -%}
{% if required or name in sla_dict -%}
{% for k, (v,) in urlparse.parse_qs(sla_dict.pop(name), strict_parsing=1).iteritems() -%}
sla-{{ k }} = {{ v }}
{% endfor -%}
{% else -%}
sla-computer_guid = ${slap-connection:computer-id}
{% endif -%}
{% endmacro -%}
{% macro common_section() -%}
[request-common-base]
recipe = slapos.cookbook:request.serialised
software-url = ${slap-connection:software-release-url}
server-url = ${slap-connection:server-url}
key-file = ${slap-connection:key-file}
cert-file = ${slap-connection:cert-file}
computer-id = ${slap-connection:computer-id}
partition-id = ${slap-connection:partition-id}
[buildout]
parts =
publish
{{ part_list | join('\n\t') }}
{{- assert(not sla_dict, sla_dict) }}
{% endmacro -%}
{% macro request_neo(parameter_dict, software_type, prefix='node-') -%}
{% set section_id_list = [] -%}
[{{ prefix }}request-common]
<= request-common-base
config-cluster = {{ dumps(parameter_dict['cluster']) }}
{% set replicas = parameter_dict.get('replicas', 0) -%}
config-partitions = {{ dumps(parameter_dict.get('partitions', 12)) }}
config-replicas = {{ dumps(replicas) }}
config-upstream-cluster = {{ dumps(parameter_dict.get('upstream-cluster', '')) }}
config-upstream-masters = {{ dumps(parameter_dict.get('upstream-masters', '')) }}
software-type = {{ software_type }}
{% set node_list = parameter_dict.get('node-list', ({},)) -%}
{% do assert(replicas < len(node_list)) -%}
{% for i, node in enumerate(node_list) -%}
{% set section_id = prefix ~ i -%}
{% do section_id_list.append(section_id) -%}
[{{ section_id }}-base]
name = {{ section_id }}
{% for k, v in node.iteritems() -%}
config-{{ k }} = {{ dumps(v) }}
{% endfor -%}
{{ sla(section_id) }}
[{{ section_id }}]
<= {{ prefix }}request-common
{{ section_id }}-base
return =
master
admin
{% endfor -%}
[final-base]
{% for i, section_id in enumerate(section_id_list) -%}
config-master-{{i}} = {{ '${' + section_id + ':connection-master}' }}
config-admin-{{i}} = {{ '${' + section_id + ':connection-admin}' }}
{% endfor -%}
{% for section_id in section_id_list -%}
[{{ section(section_id + '-final') }}]
<= {{ prefix }}request-common
final-base
{{ section_id }}-base
{% if loop.first -%}
return =
masters
admins
{% endif -%}
{% endfor -%}
{% endmacro -%}
software/neoppod/software-common.cfg 0 → 100644
View file @ d35284d8
# Note on LXML/END LXML: they delimit areas where lxml magic is needed. lxml is
# a slapos.cookbook dependency, so it should be fetched automatically. But when
# automatically fetched, it gets built against system headers/libs, which is
# forbidden in slapos. So we need to fetch lxml explicitly so it is properly
# built.
[buildout]
extends =
../../stack/slapos.cfg
../../component/dcron/buildout.cfg
../../component/gzip/buildout.cfg
../../component/logrotate/buildout.cfg
#LXML
../../component/lxml-python/buildout.cfg
#END LXML
../../component/python-2.7/buildout.cfg
../../component/mariadb/buildout.cfg
../../component/mysql-python/buildout.cfg
parts =
slapos-deps-eggs
slapos-cookbook-develop
slapos-cookbook
# NEO & dependencies
python2.7
mariadb
mysql-python
neoppod
[slapos.cookbook-repository]
branch = erp5-cluster
[slapos-deps-eggs]
recipe = zc.recipe.egg
eggs =
${lxml-python:egg}
slapos.toolbox
scripts =
slapos-kill
[template-logrotate-base]
recipe = slapos.recipe.template:jinja2
template = ${:_profile_base_location_}/${:filename}.in
rendered = ${buildout:directory}/${:filename}
filename = instance-logrotate-base.cfg
md5sum = af19ff0c7817df85987c69738fb083f2
context =
key dcron_location dcron:location
key gzip_location gzip:location
key logrotate_location logrotate:location
[download-base-neo]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/${:_buildout_section_name_}.cfg.in
# XXX: following mode should be the default
mode = 644
# XXX: must be rendered, not just dled
[instance-common]
recipe = slapos.recipe.template:jinja2
template = ${:_profile_base_location_}/${:_buildout_section_name_}.cfg.in
rendered = ${buildout:directory}/${:_buildout_section_name_}.cfg
md5sum = e8f5a83580e9791a32c66f5935cb6840
context =
key bin_directory buildout:bin-directory
key develop_eggs_directory buildout:develop-eggs-directory
key eggs_directory buildout:eggs-directory
key mariadb_location mariadb:location
key neo_admin instance-neo-admin:target
key neo_master instance-neo-master:target
key neo_storage_mysql instance-neo-storage-mysql:target
key template_logrotate_base template-logrotate-base:rendered
key template_neo_my_cnf template-neo-my-cnf:target
[root-common]
<= download-base-neo
md5sum = f009499b02cacff311a27aff6550e501
[instance-neo-admin]
<= download-base-neo
md5sum = 16d11f0fe74de06aebbadcff3527db1c
[instance-neo-master]
<= download-base-neo
md5sum = 9563ce56676bf9ae5e77fe12e9020289
[instance-neo-storage-mysql]
<= download-base-neo
md5sum = 51ad624103bb6d1d5a0b7159e7add4fa
[template-neo-my-cnf]
<= download-base-neo
url = ${:_profile_base_location_}/my.cnf.in
md5sum = febd3ed58043ce1367b86cf6e4e69700
[neoppod]
recipe = zc.recipe.egg
eggs = neoppod[admin, ctl, master, storage-importer, storage-mysqldb]
ZODB3
ZODB3-patches = ${:_profile_base_location_}/../../component/egg-patch/ZODB3-3.10.5.patch
ZODB3-patch-options = -p1
[versions]
# patched eggs
ZODB3 = 3.10.5+SlapOSPatched001
software/neoppod/software.cfg
View file @ d35284d8
# Note on LXML/END LXML: they delimit areas where lxml magic is needed. lxml is
# a slapos.cookbook dependency, so it should be fetched automatically. But when
# automatically fetched, it gets built against system headers/libs, which is
# forbidden in slapos. So we need to fetch lxml explicitly so it is properly
# built.
[buildout]
extends =
../../stack/slapos.cfg
../../stack/erp5/logrotate-base.cfg
#LXML
../../component/lxml-python/buildout.cfg
#END LXML
../../component/python-2.7/buildout.cfg
../../component/mariadb/buildout.cfg
../../component/mysql-python/buildout.cfg
software-common.cfg
parts =
slapos-deps-eggs
slapos-cookbook-develop
slapos-cookbook
# NEO & dependencies
python2.7
mariadb
mysql-python
neoppod
parts +=
# NEO instanciation
template
[slapos.cookbook-repository]
branch = erp5-cluster
[slapos-deps-eggs]
recipe = zc.recipe.egg
eggs =
${lxml-python:egg}
slapos.toolbox
scripts =
slapos-kill
[cluster]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/${:_buildout_section_name_}.cfg.in
md5sum = 13f668f6e39eb75b2a2dfe2af0401f9b
# XXX: following mode should be the default
mode = 644
[instance-neo-admin]
< = cluster
md5sum = 16d11f0fe74de06aebbadcff3527db1c
[instance-neo-master]
< = cluster
md5sum = 9563ce56676bf9ae5e77fe12e9020289
[instance-neo-storage-mysql]
< = cluster
md5sum = 6ad04cea50d2de97079cbbf2c73cd5d1
[template-my-cnf]
< = cluster
url = ${:_profile_base_location_}/my.cnf.in
md5sum = febd3ed58043ce1367b86cf6e4e69700
[template]
recipe = slapos.recipe.template:jinja2
template = ${:_profile_base_location_}/instance.cfg.in
md5sum = e0917a4a4aa102d6fa4274530d56959e
md5sum = 777c00a7dbcb145a75f980421a9b20b5
# XXX: "template.cfg" is hardcoded in instanciation recipe
rendered = ${buildout:directory}/template.cfg
context =
key eggs_directory buildout:eggs-directory
key develop_eggs_directory buildout:develop-eggs-directory
key bin_directory buildout:bin-directory
key cluster cluster:target
key neo_admin instance-neo-admin:target
key neo_master instance-neo-master:target
key neo_storage_mysql instance-neo-storage-mysql:target
key mariadb_location mariadb:location
key template_my_cnf template-my-cnf:target
key logrotate_base template-logrotate-base:rendered
key instance_common_cfg instance-common:rendered
key root_common root-common:target
[neoppod]
recipe = zc.recipe.egg
eggs = neoppod[admin, ctl, master, storage-importer, storage-mysqldb]
[cluster]
<= download-base-neo
md5sum = ee8401a4e7d82bf488a57e3399f9ce48
stack/erp5/buildout.cfg
View file @ d35284d8
......@@ -5,7 +5,6 @@ find-links +=
extends =
# Exact version of Zope
https://raw.github.com/zopefoundation/Zope/2.13.22/versions.cfg
../../stack/slapos.cfg
../../component/cups/buildout.cfg
../../component/dbus/buildout.cfg
../../component/file/buildout.cfg
......@@ -25,17 +24,13 @@ extends =
../../component/libffi/buildout.cfg
../../component/libpng/buildout.cfg
../../component/libreoffice-bin/buildout.cfg
../../component/lxml-python/buildout.cfg
../../component/mesa/buildout.cfg
../../component/numpy/buildout.cfg
../../component/percona-toolkit/buildout.cfg
../../component/mariadb/buildout.cfg
../../component/mysql-python/buildout.cfg
../../component/patch/buildout.cfg
../../component/pillow/buildout.cfg
../../component/pycrypto-python/buildout.cfg
../../component/pysvn-python/buildout.cfg
../../component/python-2.7/buildout.cfg
../../component/python-ldap-python/buildout.cfg
../../component/rdiff-backup/buildout.cfg
../../component/stunnel/buildout.cfg
......@@ -57,9 +52,9 @@ extends =
../../component/6tunnel/buildout.cfg
../../component/findutils/buildout.cfg
../../component/userhosts/buildout.cfg
logrotate-base.cfg
../../software/neoppod/software-common.cfg
parts =
parts +=
erp5-util-develop
slapos-cookbook-develop
slapos-cookbook
......@@ -78,7 +73,6 @@ parts =
poppler
libpng
ghostscript
mariadb
mroonga-mariadb
imagemagick
inkscape
......@@ -132,9 +126,8 @@ parts =
template
[download-base]
recipe = slapos.recipe.build:download
<= download-base-neo
url = ${:_profile_base_location_}/${:filename}
mode = 644
[check-recipe]
recipe = plone.recipe.command
......@@ -142,21 +135,6 @@ stop-on-error = true
update-command = ${:command}
command = grep parts ${buildout:develop-eggs-directory}/slapos.cookbook.egg-link
[template-jinja2-base]
recipe = slapos.recipe.template:jinja2
mode = 640
template = ${:_profile_base_location_}/${:filename}.in
rendered = ${buildout:directory}/${:filename}
# XXX: extra-context is needed because we cannot append to a key of an extended
# section.
extra-context =
context =
key bin_directory buildout:bin-directory
key develop_eggs_directory buildout:develop-eggs-directory
key eggs_directory buildout:eggs-directory
key slapos_core_version versions:slapos.core
${:extra-context}
[mariadb-resiliency-after-import-script]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/template/instance-mariadb-resiliency-after-import-script.sh.in
......@@ -164,7 +142,7 @@ md5sum = b32d9ee1cb85f85d8d2f2b58f58459f1
mode = 755
[template-mariadb]
< = download-base
<= download-base
filename = instance-mariadb.cfg.in
md5sum = 4d46ab5aa651a39c4021de6f647e8be9
link-binary =
......@@ -180,56 +158,58 @@ link-binary =
${mariadb:location}/bin/mysqlbinlog
[template-kumofs]
< = download-base
<= download-base
filename = instance-kumofs.cfg.in
md5sum = 7d9760fe65f454700342eeffec14c884
[template-cloudooo]
< = download-base
<= download-base
filename = instance-cloudoo.cfg.in
md5sum = 050fa6ff4eb397c5d4cb41a9f75afb3f
[template-zope-conf]
< = download-base
<= download-base
filename = zope.conf.in
md5sum = 0bf51218ecbf2bd319214192448a3ef7
[template-runzope-userhosts-preloaded]
< = download-base
<= download-base
filename = runzope_userhosts_preloaded.in
md5sum = bc6048b85b410693e60e5a77399dd1b7
[template-my-cnf]
< = download-base
<= download-base
filename = my.cnf.in
md5sum = dd779e54d22105702aa72cadc994d957
[template-mariadb-initial-setup]
< = download-base
<= download-base
filename = mariadb_initial_setup.sql.in
md5sum = b8d2d9af0c4cab45c8337aeac28d5fae
[template-create-erp5-site]
< = download-base
<= download-base
filename = instance-create-erp5-site.cfg.in
md5sum = 408fd955a876a1004ed35f578029a7d2
[template-create-erp5-site-real]
< = download-base
<= download-base
filename = instance-create-erp5-site-real.cfg.in
md5sum = e40e2e39f4941c6372f4357e8589a5cf
[template]
< = template-jinja2-base
recipe = slapos.recipe.template:jinja2
# XXX: "template.cfg" is hardcoded in instanciation recipe
filename = template.cfg
rendered = ${buildout:directory}/template.cfg
template = ${:_profile_base_location_}/instance.cfg.in
md5sum = ae7b1bd8bd36203b6a2f04647f5eb5f4
extra-context =
md5sum = 6a2c99f8df8bb8fe8e2b7fa96f68f62b
mode = 640
context =
key mariadb_link_binary template-mariadb:link-binary
key zope_link_binary template-zope:link-binary
key apache_location apache:location
key aspell_location aspell:location
key bin_directory buildout:bin-directory
key buildout_bin_directory buildout:bin-directory
key coreutils_location coreutils:location
key cups_location cups:location
......@@ -248,6 +228,7 @@ extra-context =
key glu_location glu:location
key gzip_location gzip:location
key haproxy_location haproxy:location
key instance_common_cfg instance-common:rendered
key jsl_location jsl:location
key kumo_location kumo:location
key libICE_location libICE:location
......@@ -269,7 +250,9 @@ extra-context =
key mariadb_resiliency_after_import_script mariadb-resiliency-after-import-script:target
key mesa_location mesa:location
key openssl_location openssl:location
key root_common root-common:target
key sixtunnel_location 6tunnel:location
key slapos_core_version versions:slapos.core
key stunnel_location stunnel:location
key template_apache_conf template-apache-conf:target
key template_balancer template-balancer:target
......@@ -279,7 +262,6 @@ extra-context =
key template_erp5 template-erp5:target
key template_haproxy_cfg template-haproxy-cfg:target
key template_kumofs template-kumofs:target
key template_logrotate_base template-logrotate-base:rendered
key template_mariadb template-mariadb:target
key template_mariadb_initial_setup template-mariadb-initial-setup:target
key template_my_cnf template-my-cnf:target
......@@ -294,19 +276,20 @@ extra-context =
key zlib_location zlib:location
[template-erp5]
< = download-base
<= download-base
filename = instance-erp5.cfg.in
md5sum = 55ccb85dd3334d6c07b49a6f1c11ace9
md5sum = df944bcd8d95d4ce667de91c54ff455b
[template-zeo]
< = download-base
<= download-base
filename = instance-zeo.cfg.in
md5sum = 9670cf63099e2c520017a23defff51a4
[template-zope]
< = download-base
<= download-base
filename = instance-zope.cfg.in
md5sum = ea110d1e180151b8ca6d47e9a98a635f
md5sum = 38551c47a3ee3f4e40d2176301e99482
link-binary =
${aspell:location}/bin/aspell
${dmtx-utils:location}/bin/dmtxwrite
......@@ -328,17 +311,17 @@ link-binary =
${w3m:location}/bin/w3m
[template-balancer]
< = download-base
<= download-base
filename = instance-balancer.cfg.in
md5sum = 3628f0ba358e7ee96a8d0185c4775911
[template-apache-conf]
< = download-base
<= download-base
filename = apache.conf.in
md5sum = dedfe759c3421e0186703fe1257bae8f
[template-haproxy-cfg]
< = download-base
<= download-base
filename = haproxy.cfg.in
md5sum = 7f13123698afe017dfcde9de6beea0f5
......@@ -430,7 +413,7 @@ initialization =
sys.path[0:0] = ['/'.join(['''${buildout:parts-directory}''', x]) for x in repository_id_list]
[eggs]
recipe = zc.recipe.egg
<= neoppod
eggs =
${numpy:egg}
${mysql-python:egg}
......@@ -530,9 +513,6 @@ eggs =
# installation of python, which we don't want on an instance
interpreter = ${buildout:python}
scripts =
neoctl
neolog
neomigrate
repozo
runzope
runzeo
......@@ -549,8 +529,6 @@ Acquisition-patches = ${:_profile_base_location_}/../../component/egg-patch/Acqu
Acquisition-patch-options = -p1
Products.DCWorkflow-patches = ${:_profile_base_location_}/../../component/egg-patch/Products.DCWorkflow/workflow_method.patch
Products.DCWorkflow-patch-options = -p1
ZODB3-patches = ${:_profile_base_location_}/../../component/egg-patch/ZODB3-3.10.5.patch
ZODB3-patch-options = -p1
[zodbanalyze]
recipe = zc.recipe.egg
......@@ -570,9 +548,6 @@ setup = ${erp5:location}
branch =
revision = 3241978a6ec832f6aa71d1df1a62e22a8feae2f1
[slapos.cookbook-repository]
branch = erp5-cluster
[slapos-toolbox]
recipe = zc.recipe.egg
eggs =
......@@ -590,7 +565,6 @@ scripts =
# patched eggs
Acquisition = 2.13.8+SlapOSPatched001
Products.DCWorkflow = 2.2.4+SlapOSPatched001
ZODB3 = 3.10.5+SlapOSPatched001
# specify dev version to be sure that an old released version is not used
cloudooo = 1.2.5-dev
......
stack/erp5/instance-erp5.cfg.in
View file @ d35284d8
{% set sla_dict = {} -%}
{% for sla, ref_list in slapparameter_dict.get('sla-dict', {}).iteritems() -%}
{% do sla_dict.update(dict.fromkeys(ref_list, sla)) -%}
{% endfor -%}
{% import "root_common" as root_common with context %}
{% set frontend_dict = slapparameter_dict.get('frontend', {}) -%}
{% set has_frontend = frontend_dict.get('software-url', '') != '' -%}
{% set site_id = slapparameter_dict.get('site-id', 'erp5') -%}
{% set inituser_login = slapparameter_dict.get('inituser-login', 'zope') -%}
{% set publish_dict = {'site-id': site_id, 'inituser-login': inituser_login} -%}
[request-common]
recipe = slapos.cookbook:request.serialised
software-url = ${slap-connection:software-release-url}
server-url = ${slap-connection:server-url}
key-file = ${slap-connection:key-file}
cert-file = ${slap-connection:cert-file}
computer-id = ${slap-connection:computer-id}
partition-id = ${slap-connection:partition-id}
<= request-common-base
config-use-ipv6 = {{ dumps(slapparameter_dict.get('use-ipv6', False)) }}
{% macro sla(name, required=False) -%}
{% if required or name in sla_dict -%}
{% for k, (v,) in urlparse.parse_qs(sla_dict.pop(name), strict_parsing=1).iteritems() -%}
sla-{{ k }} = {{ v }}
{% endfor -%}
{% else -%}
sla-computer_guid = ${slap-connection:computer-id}
{% endif -%}
{% endmacro -%}
{% macro request(name, software_type, config_key, config, ret={'url': True}) -%}
{% do config.update(slapparameter_dict.get(config_key, {})) -%}
{% set section = 'request-' ~ name -%}
[{{ section }}]
< = request-common
<= request-common
name = {{ name }}
software-type = {{ software_type }}
return = {{ ' '.join(ret) }}
......@@ -40,7 +21,7 @@ return = {{ ' '.join(ret) }}
{% do publish_dict.__setitem__(name ~ '-' ~ ret, '${' ~ section ~ ':connection-' ~ ret ~ '}')%}
{% endif -%}
{% endfor -%}
{{ sla(name) }}
{{ root_common.sla(name) }}
{% for k, v in config.iteritems() -%}
config-{{ k }} = {{ dumps(v) }}
{% endfor -%}
......@@ -66,9 +47,15 @@ config-{{ k }} = {{ dumps(v) }}
{% endfor -%}
{% do assert(len(mountpoints) == len(zodb_dict)) %}
{% for server_type, server_dict in storage_dict.iteritems() -%}
{% do assert(server_type == 'zeo', server_type) -%}
{% if server_type == 'neo' -%}
{% set ((name, server_dict),) = server_dict.items() -%}
{% do zodb_dict[name].setdefault('storage-dict', {}).__setitem__('name', server_dict['cluster']) -%}
{{ root_common.request_neo(server_dict, 'zodb-neo', 'neo-') }}
{% else -%}
{{ assert(server_type == 'zeo', server_type) -}}
{# BBB: for compatibility, keep 'zodb' as partition_reference for ZEO -#}
{{ request('zodb', 'zodb-' ~ server_type, 'zodb-' ~ server_type, {'tcpv4-port': 2100, 'zodb-dict': server_dict}, dict.fromkeys(('storage-dict', 'tidstorage-ip', 'tidstorage-port'))) }}
{{ request('zodb', 'zodb-' ~ server_type, 'zodb-' ~ server_type, {'tcpv4-port': 2100, 'zodb-dict': server_dict}, dict.fromkeys(('storage-dict', 'tidstorage-ip', 'tidstorage-port'))) }}
{% endif -%}
{% endfor -%}
[inituser-password]
......@@ -88,7 +75,7 @@ recipe = slapos.cookbook:generate.password
{% endif -%}
[request-zope-base]
< = request-common
<= request-common
return =
zope-address-list
hosts-dict
......@@ -109,11 +96,16 @@ config-site-id = {{ dumps(site_id) }}
config-smtp-url = {{ dumps(slapparameter_dict.get('smtp-url', 'smtp://localhost:25/')) }}
config-timezone = {{ dumps(slapparameter_dict.get('timezone', 'UTC')) }}
config-zodb-dict = {{ dumps(zodb_dict) }}
{% if storage_dict -%}
{% for server_type, server_dict in storage_dict.iteritems() -%}
{% if server_type == 'neo' -%}
config-neo-name = {{ server_dict.keys()[0] }}
config-neo-masters = ${neo-0-final:connection-masters}
{% else -%}
config-zodb-zeo = ${request-zodb:connection-storage-dict}
config-tidstorage-ip = ${request-zodb:connection-tidstorage-ip}
config-tidstorage-port = ${request-zodb:connection-tidstorage-port}
{% endif -%}
{% endif -%}
{% endfor -%}
software-type = zope
{% set zope_family_dict = {} -%}
......@@ -122,9 +114,9 @@ software-type = zope
{% set section_name = 'request-' ~ partition_name -%}
{% do zope_family_dict.setdefault(zope_parameter_dict.get('family', 'default'), []).append(section_name) -%}
[{{ section_name }}]
< = request-zope-base
<= request-zope-base
name = {{ partition_name }}
{{ sla(partition_name) }}
{{ root_common.sla(partition_name) }}
config-name = {{ dumps(custom_name) }}
config-instance-count = {{ dumps(zope_parameter_dict.get('instance-count', 1)) }}
config-thread-amount = {{ dumps(zope_parameter_dict.get('thread-amount', 4)) }}
......@@ -148,7 +140,7 @@ config-webdav = {{ dumps(zope_parameter_dict.get('webdav', False)) }}
{% set frontend_name = 'frontend-' ~ family_name -%}
{% do publish_dict.__setitem__('family-' ~ family_name, '${' ~ frontend_name ~ ':connection-site_url}' ) -%}
[{{ frontend_name }}]
< = request-frontend-base
<= request-frontend-base
name = {{ frontend_name }}
config-url = ${request-balancer:{{ family_name }}-v6}
{% else -%}
......@@ -159,10 +151,10 @@ config-url = ${request-balancer:{{ family_name }}-v6}
{% set balancer_dict = slapparameter_dict.get('balancer', {}) -%}
[request-balancer]
< = request-common
<= request-common
name = balancer
software-type = balancer
{{ sla('balancer') }}
{{ root_common.sla('balancer') }}
return =
{%- for family in zope_family_dict %}
{{ family }}
......@@ -180,10 +172,10 @@ config-ssl = {{ dumps(balancer_dict.get('ssl', {})) }}
[request-frontend-base]
{% if has_frontend -%}
< = request-common
<= request-common
software-url = {{ dumps(frontend_dict['software-url']) }}
software-type = {{ dumps(frontend_dict.get('software-type', 'RootSoftwareInstance')) }}
{{ sla('frontend', True) }}
{{ root_common.sla('frontend', True) }}
slave = true
{% set config_dict = {
'type': 'zope',
......@@ -199,6 +191,10 @@ return = site_url
[publish]
recipe = slapos.cookbook:publish.serialised
{% if 'neo' in storage_dict -%}
neo-masters = ${neo-0-final:connection-masters}
neo-admins = ${neo-0-final:connection-admins}
{% endif -%}
deadlock-debugger-password = ${deadlock-debugger-password:passwd}
inituser-password = ${inituser-password:passwd}
{#
......@@ -210,7 +206,4 @@ hosts-dict = {{ '${' ~ zope_address_list_id_dict.keys()[0] ~ ':connection-hosts-
{{ name }} = {{ value }}
{% endfor -%}
[buildout]
parts = publish
{{- assert(not sla_dict, sla_dict) }}
{{ root_common.common_section() }}
stack/erp5/instance-zope.cfg.in
View file @ d35284d8
......@@ -250,6 +250,9 @@ node-id = {{ dumps(node_id_base ~ '-' ~ index) }}
{% set log = buildout_directory ~ '/var/log/' ~ name ~ '-neo-' ~ db_name ~ '.log' -%}
{% do log_list.append(log) -%}
{% do zodb['storage-dict'].__setitem__('logfile', log) -%}
{% if db_name == slapparameter_dict.get('neo-name') -%}
{% do zodb['storage-dict'].__setitem__('master_nodes', slapparameter_dict['neo-masters']) -%}
{% endif -%}
{% endif -%}
{% endfor -%}
import-list = {{ dumps(list(import_set)) }}
......
stack/erp5/instance.cfg.in
View file @ d35284d8
[buildout]
parts = switch-softwaretype
eggs-directory = {{ eggs_directory }}
develop-eggs-directory = {{ develop_eggs_directory }}
[slap-configuration]
recipe = slapos.cookbook:slapconfiguration.serialised
computer = ${slap-connection:computer-id}
partition = ${slap-connection:partition-id}
url = ${slap-connection:server-url}
key = ${slap-connection:key-file}
cert = ${slap-connection:cert-file}
extends = {{ instance_common_cfg }}
[jinja2-template-base]
recipe = slapos.recipe.template:jinja2
rendered = ${buildout:parts-directory}/${:_buildout_section_name_}/${:filename}
extra-context =
context =
key ipv4_set slap-configuration:ipv4
key ipv6_set slap-configuration:ipv6
key slapparameter_dict slap-configuration:configuration
context +=
raw slapos_core_version {{ slapos_core_version }}
raw logrotate_cfg {{ template_logrotate_base }}
${:extra-context}
[dynamic-template-cloudooo-parameters]
cups = {{ cups_location }}
......@@ -54,7 +36,7 @@ buildout-bin-directory = {{ buildout_bin_directory }}
dash = {{ dash_location }}
[dynamic-template-cloudooo]
< = jinja2-template-base
<= jinja2-template-base
template = {{ template_cloudooo }}
filename = instance-cloudoo.cfg
extra-context =
......@@ -64,13 +46,14 @@ extra-context =
local-bt5-repository = {{ local_bt5_repository }}
[dynamic-template-erp5]
< = jinja2-template-base
<= jinja2-template-base
template = {{ template_erp5 }}
filename = instance-erp5.cfg
extensions = jinja2.ext.do
extra-context =
key local_bt5_repository dynamic-template-erp5-parameters:local-bt5-repository
import urlparse urlparse
import-list =
rawfile root_common {{ root_common }}
[dynamic-template-balancer-parameters]
apache = {{ apache_location }}
......@@ -83,9 +66,8 @@ template-haproxy-cfg = {{ template_haproxy_cfg }}
template-apache-conf = {{ template_apache_conf }}
[dynamic-template-balancer]
< = jinja2-template-base
<= jinja2-template-base
template = {{ template_balancer }}
extensions = jinja2.ext.do
filename = instance-balancer.cfg
extra-context =
section parameter_dict dynamic-template-balancer-parameters
......@@ -94,10 +76,9 @@ extra-context =
buildout-bin-directory = {{ buildout_bin_directory }}
[dynamic-template-zeo]
< = jinja2-template-base
<= jinja2-template-base
template = {{ template_zeo }}
filename = instance-zeo.cfg
extensions = jinja2.ext.do
extra-context =
key buildout_directory buildout:directory
section parameter_dict dynamic-template-zeo-parameters
......@@ -116,10 +97,9 @@ userhosts = {{ userhosts_location }}
runzope-userhosts-preloaded-template = {{ template_runzope_userhosts_preloaded }}
[dynamic-template-zope]
< = jinja2-template-base
<= jinja2-template-base
template = {{ template_zope }}
filename = instance-zope.cfg
extensions = jinja2.ext.do
extra-context =
key buildout_directory buildout:directory
section parameter_dict dynamic-template-zope-parameters
......@@ -133,7 +113,7 @@ kumo-location = {{ kumo_location }}
logrotate-location = {{ logrotate_location }}
[dynamic-template-kumofs]
< = jinja2-template-base
<= jinja2-template-base
template = {{ template_kumofs }}
filename = instance-kumofs.cfg
extra-context =
......@@ -152,10 +132,9 @@ bin-directory = {{ bin_directory }}
mariadb-resiliency-after-import-script = {{ mariadb_resiliency_after_import_script }}
[dynamic-template-mariadb]
< = jinja2-template-base
<= jinja2-template-base
template = {{ template_mariadb }}
filename = instance-mariadb.cfg
extensions = jinja2.ext.do
extra-context =
section parameter_dict dynamic-template-mariadb-parameters
......@@ -166,7 +145,7 @@ dash-location = {{ dash_location }}
curl-location = {{ curl_location }}
[dynamic-template-create-erp5-site]
< = jinja2-template-base
<= jinja2-template-base
template = {{ template_create_erp5_site }}
filename = instance-create-erp5-site.cfg
extra-context =
......@@ -187,4 +166,5 @@ cloudooo = dynamic-template-cloudooo:rendered
mariadb = dynamic-template-mariadb:rendered
balancer = dynamic-template-balancer:rendered
zodb-zeo = dynamic-template-zeo:rendered
zodb-neo = neo-storage-mysql:rendered
zope = dynamic-template-zope:rendered
stack/erp5/logrotate-base.cfg deleted 100644 → 0
View file @ 6bbb61a8
# also extended by software/neoppod
[buildout]
extends =
../../component/dcron/buildout.cfg
../../component/gzip/buildout.cfg
../../component/logrotate/buildout.cfg
[template-logrotate-base]
recipe = slapos.recipe.template:jinja2
template = ${:_profile_base_location_}/${:filename}.in
rendered = ${buildout:directory}/${:filename}
filename = instance-logrotate-base.cfg
md5sum = af19ff0c7817df85987c69738fb083f2
context =
key dcron_location dcron:location
key gzip_location gzip:location
key logrotate_location logrotate:location
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7