Skip to content

slapos
slapos
Commit a546487f authored by Julien Muchembled's avatar Julien Muchembled
Browse files
Options

Make ERP5 instantiable with an external NEO storage 

In order not to conflict with a future integration of NEO in ERP5:
- the input schema has a new parameter for external storages.
- zodb-software-type & zodb is used only for internal storages
  and only ZEO is supported.

NEO logging is also enabled for clients.
parent 67dbe6d0
Hide whitespace changes
Inline Side-by-side
Showing with 104 additions and 126 deletions
software/erp5/instance-erp5-input-schema.json
View file @ a546487f
......@@ -33,7 +33,7 @@
"description": "List of logins which should get the Developper role (required to modify portal_components' content)",
"default": "inituser-login's value",
"items": {
"pattern": "/^\S+$/",
"pattern": "/^\\S+$/",
"type": "string"
},
"uniqueItems": true,
......@@ -190,16 +190,54 @@
},
"zodb-software-type": {
"description": "Zope Object DataBase implementation",
"default": "zeo",
"enum": ["zeo", "neo"],
"enum": ["zeo"],
"type": "string"
},
"zodb": {
"description": "Zope Object DataBase service",
"oneOf": [
{"$ref": "./instance-neo-input-schema.json#properties"},
{"$ref": "./instance-zeo-input-schema.json#properties"}
]
},
"zodb-extern": {
"description": "External ZODB Mountpoints. See https://github.com/zopefoundation/ZODB/blob/3.10/src/ZODB/component.xml for extra options.",
"minItems": 0,
"items": {
"required": ["storage_dict"],
"properties": {
"name": {
"description": "Database name",
"default": "main",
"type": "string"
},
"mount_point": {
"description": "Mount point",
"default": "/",
"type": "string"
},
"storage-type": {
"description": "Storage type",
"default": "NEOStorage",
"type": "string"
},
"storage-dict": {
"description": "Storage configuration. For NEOStorage, 'logfile' is automatically set (see http://git.erp5.org/gitweb/neoppod.git/blob/HEAD:/neo/client/component.xml for other settings).",
"patternProperties": {
".": {
"type": "string"
}
},
"type": "object"
},
"patternProperties": {
".": {
"type": "string"
}
}
},
"type": "object"
},
"type": "array"
}
}
}
software/erp5/instance-neo-input-schema.json deleted 100644 → 0
View file @ 67dbe6d0
{
"$schema": "http://json-schema.org/draft-04/schema#",
"extends": "./schema-definitions.json#",
"required": ["zodb-dict"],
"properties": {
"zodb-dict": {
"description": "Maps an export identifier to its settings",
"patternProperties": {
".*": {
"properties": {
"mount-point": {
"description": "Traversal path export should be mounted at (opaque to this software type)",
"default": "/",
"type": "string"
},
"cache-size": {
"description": "Client-side cache size in object count (opaque to this software type)",
"default": -1,
"type": "integer"
},
"storage-dict": {
"default": {},
"required": ["name", "master_nodes"],
"properties": {
"name": {
"description": "NEO cluster's name",
"type": "string"
},
"master_nodes": {
"description": "Client-side settings for this mountpoint",
"minItems": 1,
"items": {
"type": "string"
},
"type": "array"
}
},
"type": "object"
}
},
"type": "object"
}
},
"type": "object"
}
}
}
software/erp5/instance-neo-output-schema.json deleted 100644 → 0
View file @ 67dbe6d0
{
"$schema": "http://json-schema.org/draft-04/schema#",
"description": "Values returned by ZEO instanciation",
"properties": {
"zodb-storage-type": {
"description": "zope.cfg-friendly storage type",
"enum": ["NEOStorage"],
"type": "string"
},
"zodb-dict": {
"description": "Maps an export identifier to values received as parameters",
"patternProperties": {
".*": {
"description": "Contains the mount-point, cache-size and client-dict received as parameters for current export identifier (client-dict's 'storage' and 'server' entries are overwritten with actual values)",
"type": "array"
}
},
"type": "object"
},
},
"type": "object"
}
stack/erp5/buildout.cfg
View file @ a546487f
......@@ -183,7 +183,7 @@ md5sum = 838e56f46ba749dfc53b8e69e21be008
[template-zope-conf]
< = download-base
filename = zope.conf.in
md5sum = 8fe36c41ab784f547b968dc6edd0ca29
md5sum = 9e68f6a221c16e654efb05632fed5aee
[template-runzope-userhosts-preloaded]
< = download-base
......@@ -215,7 +215,7 @@ md5sum = e40e2e39f4941c6372f4357e8589a5cf
# XXX: "template.cfg" is hardcoded in instanciation recipe
filename = template.cfg
template = ${:_profile_base_location_}/instance.cfg.in
md5sum = 5e303e59c6674c98944120d6f1d01c28
md5sum = 99159fc1e32a0651b35f824709d561d0
extra-context =
key mariadb_link_binary template-mariadb:link-binary
key zope_link_binary template-zope:link-binary
......@@ -270,7 +270,6 @@ extra-context =
key template_mariadb template-mariadb:target
key template_mariadb_initial_setup template-mariadb-initial-setup:target
key template_my_cnf template-my-cnf:target
key template_neo template-neo:target
key template_runzope_userhosts_preloaded template-runzope-userhosts-preloaded:target
key template_zeo template-zeo:target
key template_zope template-zope:target
......@@ -282,12 +281,7 @@ extra-context =
[template-erp5]
< = download-base
filename = instance-erp5.cfg.in
md5sum = ef382581a0b8f2ad01ffc523a59bfe66
[template-neo]
< = download-base
filename = instance-neo.cfg.in
md5sum = 7dcac2eae332e2def17579888c045320
md5sum = 2ac3bbeb031eb74fbd6f9d033c71b677
[template-zeo]
< = download-base
......@@ -297,7 +291,7 @@ md5sum = 651fa2ec2feb768e339936b7855774ca
[template-zope]
< = download-base
filename = instance-zope.cfg.in
md5sum = e128a47f37ba78dc7154daa80c2dd272
md5sum = 7f185192e0536ecacc403cf6b1736d2d
link-binary =
${aspell:location}/bin/aspell
${coreutils:location}/bin/basename
......@@ -494,6 +488,7 @@ eggs =
# Needed for checking ZODB Components source code
pylint
neoppod[client]
# Zope
ZODB3
Zope2
......@@ -539,6 +534,9 @@ eggs =
# installation of python, which we don't want on an instance
interpreter = ${buildout:python}
scripts =
neoctl
neolog
neomigrate
repozo
runzope
runzeo
......
stack/erp5/instance-erp5.cfg.in
View file @ a546487f
......@@ -43,8 +43,13 @@ config-{{ option }} = {{ dumps(value) }}
{{ request('cloudooo', 'cloudooo', 'cloudooo', {'tcpv4-port': 2020}) }}
{{ request('mariadb', 'mariadb', 'mariadb', {'tcpv4-port': 2099}, {'database-list': True, 'test-database-list': True}) }}
{# Fail early if an unexpected value is provided -#}
{% set possible_software_type_dict = {'zeo': 'zeo', 'neo': 'neo'} -%}
{{ request('zodb', 'zodb-' ~ possible_software_type_dict[slapparameter_dict.get('zodb-software-type', 'zeo')], 'zodb', {'tcpv4-port': 2100, 'zodb-dict': {'root': {}}}, {'zodb-storage-type': False, 'zodb-dict': False, 'tidstorage-ip': False, 'tidstorage-port': False}) }}
{% set zodb_type = slapparameter_dict.get('zodb-software-type') -%}
{% set zodb_extern = slapparameter_dict.get('zodb-extern') -%}
{% if {'zeo': 1, None: 0}[zodb_type] -%}
{{ request('zodb', 'zodb-' ~ zodb_type, 'zodb', {'tcpv4-port': 2100, 'zodb-dict': {'root': {}}}, {'zodb-storage-type': False, 'zodb-dict': False, 'tidstorage-ip': False, 'tidstorage-port': False}) }}
{% else -%}
{% do zodb_extern[0] %}
{% endif -%}
[inituser-password]
{% set inituser_password = slapparameter_dict.get('inituser-password') -%}
......@@ -67,7 +72,7 @@ recipe = slapos.cookbook:generate.password
return =
zope-address-list
hosts-dict
extra-config =
{% set extra_config = '
bt5
bt5-repository-url
cloudooo-url
......@@ -88,13 +93,9 @@ extra-config =
site-id
smtp-url
thread-amount
tidstorage-ip
tidstorage-port
timerserver-interval
timezone
webdav
zodb-dict
zodb-storage-type
webdav' -%}
config-bt5 = {{ dumps(slapparameter_dict.get('bt5', 'erp5_full_text_myisam_catalog erp5_configurator_standard erp5_configurator_maxma_demo erp5_configurator_ung erp5_configurator_run_my_doc')) }}
config-bt5-repository-url = {{ dumps(slapparameter_dict.get('bt5-repository-url', local_bt5_repository)) }}
config-cloudooo-url = ${request-cloudooo:connection-url}
......@@ -109,12 +110,25 @@ config-mysql-test-url-list = ${request-mariadb:connection-test-database-list}
config-mysql-url-list = ${request-mariadb:connection-database-list}
config-site-id = {{ dumps(site_id) }}
config-smtp-url = {{ dumps(slapparameter_dict.get('smtp-url', 'smtp://localhost:25/')) }}
config-timezone = {{ dumps(slapparameter_dict.get('timezone', 'UTC')) }}
{% if zodb_type -%}
config-tidstorage-ip = ${request-zodb:connection-tidstorage-ip}
config-tidstorage-port = ${request-zodb:connection-tidstorage-port}
config-timezone = {{ dumps(slapparameter_dict.get('timezone', 'UTC')) }}
config-zodb-dict = ${request-zodb:connection-zodb-dict}
config-zodb-storage-type = ${request-zodb:connection-zodb-storage-type}
{% set extra_config = extra_config ~ '
tidstorage-ip
tidstorage-port
zodb-dict
zodb-storage-type' -%}
{% endif -%}
{% if zodb_extern -%}
config-zodb-extern = {{ dumps(zodb_extern) }}
{% set extra_config = extra_config ~ '
zodb-extern' -%}
{% endif -%}
software-type = zope
extra-config = {{ extra_config }}
{% set zope_family_dict = {} -%}
{% for custom_name, zope_parameter_dict in slapparameter_dict.get('zope-partition-dict', {'1': {}}).items() -%}
......
stack/erp5/instance-neo.cfg.in deleted 100644 → 0
View file @ 67dbe6d0
{% set zodb_dict = {} -%}
{% for export_id, mountpoint_dict in slapparameter_dict['zodb-dict'].items() -%}
{% do zodb_dict.__setitem__(export_id, [
mountpoint_dict.get('mount-point', '/'),
mountpoint_dict.get('cache-size', -1),
mountpoint_dict.get('storage-dict', {}),
]) -%}
{% endfor -%}
[publish]
recipe = slapos.cookbook:publish.serialised
zodb-storage-type = NEOStorage
zodb-dict = {{ dumps(zodb_dict) }}
[buildout]
parts +=
publish
stack/erp5/instance-zope.cfg.in
View file @ a546487f
......@@ -5,7 +5,7 @@
{% set node_id_base = slapparameter_dict['name'] -%}
{% set part_list = [] -%}
{% set publish_list = [] -%}
{% set zodb_dict = slapparameter_dict['zodb-dict'] -%}
{% set zodb_dict = slapparameter_dict.get('zodb-dict', {}) -%}
{% set longrequest_logger_base_path = buildout_directory ~ '/var/log/longrequest_logger_' -%}
{% macro section(name) %}{% do part_list.append(name) %}{{ name }}{% endmacro -%}
{% set bin_directory = parameter_dict['buildout-bin-directory'] -%}
......@@ -225,11 +225,13 @@ bt5-repository =
[zope-conf-parameter-base]
ip = {{ ipv4 }}
site-id = {{ site_id }}
{% set zodb_list = [] -%}
{% set zodb_list = slapparameter_dict.get('zodb-extern', []) -%}
{% for key, (mount_point, cache_size, storage_dict) in zodb_dict.items() -%}
{% do zodb_list.append([key, mount_point, cache_size, storage_type, storage_dict]) -%}
{% do zodb_list.append({'name': key, 'mount-point': mount_point, 'storage-type': storage_type, 'storage-dict': storage_dict}) -%}
{% if cache_size >= 0 -%}
{% do zodb_list[-1].__setitem__('cache-size', cache_size) -%}
{% endif -%}
{% endfor -%}
zodb-list = {{ dumps(zodb_list) }}
developer-list = {{ dumps(slapparameter_dict['developer-list']) }}
instance = ${directory:instance}
instance-products = ${directory:instance-products}
......@@ -277,6 +279,18 @@ port = {{ port }}
event-log = ${directory:log}/{{ name }}-event.log
z2-log = ${directory:log}/{{ name }}-Z2.log
node-id = {{ dumps(node_id_base ~ '-' ~ index) }}
{% set log_list = [] -%}
{% set import_set = {} -%}
{% for zodb in zodb_list -%}
{% if zodb.setdefault('storage-type', 'NEOStorage') == 'NEOStorage' -%}
{% do import_set.__setitem__('neo.client', None) -%}
{% set log = buildout_directory ~ '/var/log/' ~ name ~ '-neo-' ~ zodb.get('name', 'main') ~ '.log' -%}
{% do log_list.append(log) -%}
{% do zodb.setdefault('storage-dict', {}).__setitem__('logfile', log) -%}
{% endif -%}
{% endfor -%}
import-list = {{ dumps(import_set.keys()) }}
zodb-list = {{ dumps(zodb_list) }}
[{{ conf_name }}]
< = zope-conf-base
......@@ -324,7 +338,7 @@ path = ${directory:promises}/{{ zope_tunnel_base_name }}
[{{ section('logrotate-entry-' ~ name) }}]
< = logrotate-entry-base
name = {{ name }}
log = {{ '${' ~ conf_parameter_name ~ ':event-log}' }} {{ '${' ~ conf_parameter_name ~ ':z2-log}' }} {{ '${' ~ name ~ ':longrequest-logger-file}' }}
log = {{ '${' ~ conf_parameter_name ~ ':event-log}' }} {{ '${' ~ conf_parameter_name ~ ':z2-log}' }} {{ '${' ~ name ~ ':longrequest-logger-file}' }} {{ ' '.join(log_list) }}
post = {{ bin_directory }}/slapos-kill --pidfile {{ '${' ~ conf_parameter_name ~ ':pid-file}' }} -s USR2
{% endmacro -%}
......
stack/erp5/instance.cfg.in
View file @ a546487f
......@@ -85,12 +85,6 @@ filename = instance-balancer.cfg
extra-context =
section parameter_dict dynamic-template-balancer-parameters
[dynamic-template-neo]
< = jinja2-template-base
template = {{ template_neo }}
filename = instance-neo.cfg
extensions = jinja2.ext.do
[dynamic-template-zeo-parameters]
buildout-bin-directory = {{ buildout_bin_directory }}
6tunnel = {{ sixtunnel_location }}
......
stack/erp5/zope.conf.in
View file @ a546487f
......@@ -100,13 +100,18 @@ products {{ parameter_dict['instance-products'] }}
developers {{ developer_list | join(' ') }}
</ERP5Type>
{%- endif %}
{% for db_name, mount_point, cache_size, storage_type, storage_dict in parameter_dict['zodb-list'] -%}
<zodb_db {{ db_name }}>
{%- if cache_size >= 0 %}
cache-size {{ cache_size }}
{%- endif %}
mount-point {{ mount_point }}
{% endif -%}
{% for m in parameter_dict['import-list'] -%}
%import {{ m }}
{% endfor -%}
{% for zodb_dict in parameter_dict['zodb-list'] %}
<zodb_db {{ zodb_dict.pop('name', 'main') }}>
{%- set storage_type = zodb_dict.pop('storage-type', 'NEOStorage') %}
{%- set storage_dict = zodb_dict.pop('storage-dict', {}) %}
mount-point {{ zodb_dict.pop('mount-point', '/') }}
{%- for key, value in zodb_dict.iteritems() %}
{{ key }} {{ value }}
{%- endfor %}
<{{ storage_type }}>
{%- for key, value in storage_dict.iteritems() %}
{{ key }} {{ value }}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7