Skip to content

slapos
slapos
Commit a2ba55e0 authored by Julien Muchembled's avatar Julien Muchembled
Browse files
Options

erp5: review request parameters for SLA & ZODB 

All parameters about SLA, i.e. computer-guid & instance-guid, are removed in
favor of a new "sla-dict" parameter, which is easier to implement and much more
versatile.

All changes in the request parameters are incompatible. The old ones are
ignored without warning/error.

For compatibility, the reference of ZEO partition is still "zodb".

Default settings were also fine. Default name of mount-point and FileStorage
file is reverted to 'root' instead of 'main'.
parent fcaa03d8
Hide whitespace changes
Inline Side-by-side
Showing with 189 additions and 290 deletions
software/erp5/instance-erp5-input-schema.json
View file @ a2ba55e0
......@@ -4,6 +4,17 @@
"description": "Parameters to instantiate ERP5",
"additionalProperties": false,
"properties": {
"sla-dict": {
"description": "Where to request instances. Each key is a query string for criterions (e.g. \"computer_guid=foo\"), and each value is a list of partition references.",
"patternProperties": {
".": {
"type": "array",
"items": { "type": "string" },
"uniqueItems": true
}
},
"type": "object"
},
"site-id": {
"description": "ERP5Site object's id",
"default": "erp5",
......@@ -60,7 +71,7 @@
"description": "Front-end slave instance request parameters",
"properties": {
"software-url": {
"description": "Front-end's software type. If this parameter is the empty string, no front-end instance is requested.",
"description": "Front-end's software type. If this parameter is empty, no front-end instance is requested. Else, sla-dict must specify 'frontend' which is a special value matching all frontends (e.g. {\"instance_guid=bar\": [\"frontend\"]}).",
"default": "",
"type": "string"
},
......@@ -74,10 +85,6 @@
"default": "RootSoftwareInstance",
"type": ""
},
"instance-guid": {
"description": "GUID of computer instance must be requested on. Mandatory if software-url is a non-empty string.",
"type": "string"
}
},
"type": "object"
},
......@@ -130,20 +137,12 @@
"default": 2200
}]
},
"computer-guid": {
"description": "GUID of computer instance must be requested on",
"type": "string"
}
},
"type": "object"
}
},
"type": "object"
},
"kumofs-computer-guid": {
"description": "GUID of computer instance must be requested on",
"type": "string"
},
"kumofs": {
"description": "Persistent memcached service",
"additionalProperties": {
......@@ -151,10 +150,6 @@
},
"type": "object"
},
"memcached-computer-guid": {
"description": "GUID of computer instance must be requested on",
"type": "string"
},
"memcached": {
"description": "Volatile memcached service",
"additionalProperties": {
......@@ -162,10 +157,6 @@
},
"type": "object"
},
"cloudooo-computer-guid": {
"description": "GUID of computer instance must be requested on",
"type": "string"
},
"cloudooo": {
"description": "Format conversion service",
"additionalProperties": {
......@@ -173,10 +164,6 @@
},
"type": "object"
},
"mariadb-computer-guid": {
"description": "GUID of computer instance must be requested on",
"type": "string"
},
"mariadb": {
"description": "Relational database service",
"additionalProperties": {
......@@ -184,26 +171,33 @@
},
"type": "object"
},
"zodb-computer-guid": {
"description": "GUID of computer instance must be requested on",
"type": "string"
},
"zodb-software-type": {
"description": "Zope Object DataBase implementation",
"enum": ["zeo"],
"type": "string"
"zodb-zeo": {
"description": "Common settings ZEO servers",
"properties": {
"tcpv4-port": {
"allOf": [{
"$ref": "#/definitions/tcpv4port"
}, {
"description": "Start allocating ports at this value, going upward"
}]
},
"backup-periodicity": {
"description": "When to backup, specified in the same format as for systemd.time(7) calendar events (years & seconds not supported, DoW & DoM can not be combined). Enter 'never' to disable backups.",
"default": "daily",
"type": "string",
},
"tidstorage-timestamp-path": {
"description": "Path to backup timestamp file, occurrences of '%(backup)s' being replaced with the path to partition's srv/backup/tidstorage directory",
"default": "%(backup)s/repozo_tidstorage_timestamp.log",
"type": "string"
}
},
"type": "object"
},
"zodb": {
"description": "Zope Object DataBase service",
"oneOf": [
{"$ref": "./instance-zeo-schema.json#properties"}
]
},
"zodb-extern": {
"description": "External ZODB Mountpoints. See https://github.com/zopefoundation/ZODB/blob/3.10/src/ZODB/component.xml for extra options.",
"minItems": 0,
"description": "Zope Object DataBase mountpoints. See https://github.com/zopefoundation/ZODB/blob/3.10/src/ZODB/component.xml for extra options.",
"items": {
"required": ["storage_dict"],
"required": ["type"],
"properties": {
"name": {
"description": "Database name",
......@@ -215,13 +209,19 @@
"default": "/",
"type": "string"
},
"storage-type": {
"type": {
"description": "Storage type",
"default": "NEOStorage",
"enum": ["zeo", "neo"],
"type": "string"
},
"server": {
"description": "Instantiate a server. If missing, 'storage-dict' must contain the necessary properties to mount the ZODB.",
"oneOf": [
{"$ref": "./instance-zeo-schema.json#properties"}
]
},
"storage-dict": {
"description": "Storage configuration. For NEOStorage, 'logfile' is automatically set (see http://git.erp5.org/gitweb/neoppod.git/blob/HEAD:/neo/client/component.xml for other settings).",
"description": "Storage configuration. For NEO, 'logfile' is automatically set (see http://git.erp5.org/gitweb/neoppod.git/blob/HEAD:/neo/client/component.xml for other settings).",
"patternProperties": {
".": {
"type": "string"
......@@ -229,10 +229,10 @@
},
"type": "object"
},
"patternProperties": {
".": {
"type": "string"
}
},
"patternProperties": {
".": {
"type": "string"
}
},
"type": "object"
......
software/erp5/instance-zeo-schema.json
View file @ a2ba55e0
{
"$schema": "http://json-schema.org/draft-04/schema#",
"extends": "./schema-definitions.json#",
"required": ["tcpv4-port", "zodb-dict"],
"properties": {
"tcpv4-port": {
"allOf": [{
"$ref": "#/definitions/tcpv4port"
}, {
"description": "Start allocating ports at this value, going upward"
}]
},
"backup": {
"description": "Controls ZODB backup generation and retention (disabled if null)",
"default": {},
"properties": {
"periodicity": {
"description": "When to backup, specified in the same format as for systemd.time(7) calendar events (years & seconds not supported, DoW & DoM can not be combined)",
"default": "daily",
"type": "string",
},
"zodb-dict": {
"description": "Maps an export identifier to the path it must be backed up into",
"default": {},
"patternProperties": {
".*": {
"description": "'%(backup)s' is expanded to partition's ZODB backup path (typically 'srv/backup/zodb'), and %(name)s with the export id",
"default": "%(backup)s/%(name)s",
"type": "string"
}
},
"type": "object"
},
"tidstorage-timestamp-path": {
"description": "Path to backup timestamp file, occurrences of '%(backup)s' being eplaced with the path to partition's srv/backup/tidstorage directory",
"default": "%(backup)s/repozo_tidstorage_timestamp.log",
"type": "string"
}
},
"type": ["object", "null"]
"description": "'%(backup)s' is expanded to partition's ZODB backup path (typically 'srv/backup/zodb'), and %(name)s with the export id",
"default": "%(backup)s/%(name)s",
"type": "string"
},
"family": {
"description": "Opaque name used to regroup/separate mountpoints under different ZEO processes (must be valid as a file name and as a ConfigParser section name)",
"default": "default",
"pattern": "^[^<>:\"/\\|?*\\]\\[ ]*$",
"type": "string"
},
"path": {
"description": "FileStorage file path, '%(zodb)s' occurrences are replaced with the path to partition's srv/zodb directory, and %(name)s with the export id",
"default": "%(zodb)s/%(name)s.fs",
"type": "string"
},
"zodb-dict": {
"description": "Maps an export identifier to its settings",
"patternProperties": {
".*": {
"properties": {
"storage-family": {
"description": "Opaque name used to regroup/separate mountpoints under different ZEO processes (must be valid as a file name and as a ConfigParser section name)",
"default": "default",
"pattern": "^[^<>:\"/\\|?*\\]\\[ ]*$",
"type": "string"
},
"mount-point": {
"description": "Traversal path export should be mounted at (opaque to this software type)",
"default": "/",
"type": "string"
},
"cache-size": {
"description": "Client-side cache size in object count (opaque to this software type)",
"default": -1,
"type": "integer"
},
"storage-dict": {
"default": {},
"properties": {
"path": {
"description": "FileStorage file path, '%(zodb)s' occurrences are replaced with the path to partition's srv/zodb directory, and %(name)s with the export id",
"default": "%(zodb)s/%(name)s.fs",
"type": "string"
},
"client": {
"description": "Client-side settings for this mountpoint",
"default": {},
"patternProperties": {
".*": {
"type": "string"
}
},
"type": "object"
}
},
"type": "object"
}
},
"type": "object"
}
},
"type": "object"
}
}
}
stack/erp5/buildout.cfg
View file @ a2ba55e0
......@@ -186,7 +186,7 @@ md5sum = 48d5e8c3efc5bfd6fc1027b5ebe55e64
[template-zope-conf]
< = download-base
filename = zope.conf.in
md5sum = 9e68f6a221c16e654efb05632fed5aee
md5sum = 0bf51218ecbf2bd319214192448a3ef7
[template-runzope-userhosts-preloaded]
< = download-base
......@@ -218,7 +218,7 @@ md5sum = e40e2e39f4941c6372f4357e8589a5cf
# XXX: "template.cfg" is hardcoded in instanciation recipe
filename = template.cfg
template = ${:_profile_base_location_}/instance.cfg.in
md5sum = 79bc92c8d6ad75e78b901e96c8804314
md5sum = 2738587138c9bd10af60c6552ecac360
extra-context =
key mariadb_link_binary template-mariadb:link-binary
key zope_link_binary template-zope:link-binary
......@@ -289,17 +289,17 @@ extra-context =
[template-erp5]
< = download-base
filename = instance-erp5.cfg.in
md5sum = 386f5f15d71636f855f22201d38367e5
md5sum = 28db329061ae863367a76951b9b1152c
[template-zeo]
< = download-base
filename = instance-zeo.cfg.in
md5sum = 6bf4c5f7f476d420b2429dd0eca8ec0b
md5sum = e810ce0bf324a33d0d52d3ecda67bc4d
[template-zope]
< = download-base
filename = instance-zope.cfg.in
md5sum = 5ffaadfb2d465cba53bcbc4287df041e
md5sum = 129b0468957dec95c0c04c3484df8624
link-binary =
${aspell:location}/bin/aspell
${coreutils:location}/bin/basename
......@@ -687,7 +687,7 @@ python-memcached = 1.53
qrcode = 5.1
restkit = 4.2.2
rtjp-eventlet = 0.3.2
slapos.recipe.template = 2.5
slapos.recipe.template = 2.6
slapos.toolbox = 0.45.1
smmap = 0.8.3
socketpool = 0.5.3
......
stack/erp5/instance-erp5.cfg.in
View file @ a2ba55e0
{% set frontend_dict = slapparameter_dict.get('frontend', {}) %}
{% macro assert(x) %}{{ ("",)[not x] }}{% endmacro -%}
{% set sla_dict = {} -%}
{% for sla, ref_list in slapparameter_dict.get('sla-dict', {}).iteritems() -%}
{% do sla_dict.update(dict.fromkeys(ref_list, sla)) -%}
{% endfor -%}
{% set frontend_dict = slapparameter_dict.get('frontend', {}) -%}
{% set has_frontend = frontend_dict.get('software-url', '') != '' -%}
{% set site_id = slapparameter_dict.get('site-id', 'erp5') -%}
{% set inituser_login = slapparameter_dict.get('inituser-login', 'zope') -%}
......@@ -13,6 +18,14 @@ computer-id = ${slap-connection:computer-id}
partition-id = ${slap-connection:partition-id}
config-use-ipv6 = {{ dumps(slapparameter_dict.get('use-ipv6', False)) }}
{% macro sla(name, required=False) -%}
{% if required or name in sla_dict -%}
{% for k, (v,) in urlparse.parse_qs(slap_dict.pop(name), strict_parsing=1).iteritems() -%}
sla-{{ k }} = {{ dumps(v) }}
{% endfor -%}
{% endif -%}
{% endmacro -%}
{% macro request(name, software_type, config_key, config, ret={'url': True}) -%}
{% do config.update(slapparameter_dict.get(config_key, {})) -%}
{% set section = 'request-' ~ name -%}
......@@ -20,15 +33,15 @@ config-use-ipv6 = {{ dumps(slapparameter_dict.get('use-ipv6', False)) }}
< = request-common
name = {{ name }}
software-type = {{ software_type }}
return = {{ ret.keys() | join(' ') }}
{% for ret, publish in ret.items() -%}
return = {{ ' '.join(ret) }}
{% for ret, publish in ret.iteritems() -%}
{% if publish -%}
{% do publish_dict.__setitem__(name ~ '-' ~ ret, '${' ~ section ~ ':connection-' ~ ret ~ '}')%}
{% endif -%}
{% endfor -%}
sla-computer_guid = {{ dumps(slapparameter_dict.get(config_key + '-computer-guid', computer_id)) }}
{% for option, value in config.items() -%}
config-{{ option }} = {{ dumps(value) }}
{{ sla(name) }}
{% for k, v in config.iteritems() -%}
config-{{ k }} = {{ dumps(v) }}
{% endfor -%}
{% endmacro -%}
......@@ -36,14 +49,26 @@ config-{{ option }} = {{ dumps(value) }}
{{ request('memcached-volatile', 'kumofs', 'memcached', {'tcpv4-port': 2010, 'ram-storage-size': 64}) }}
{{ request('cloudooo', 'cloudooo', 'cloudooo', {'tcpv4-port': 2020}) }}
{{ request('mariadb', 'mariadb', 'mariadb', {'tcpv4-port': 2099}, {'database-list': True, 'test-database-list': True}) }}
{# Fail early if an unexpected value is provided -#}
{% set zodb_type = slapparameter_dict.get('zodb-software-type') -%}
{% set zodb_extern = slapparameter_dict.get('zodb-extern') -%}
{% if {'zeo': 1, None: 0}[zodb_type] -%}
{{ request('zodb', 'zodb-' ~ zodb_type, 'zodb', {'tcpv4-port': 2100, 'zodb-dict': {'root': {}}}, {'zodb-storage-type': False, 'zodb-dict': False, 'tidstorage-ip': False, 'tidstorage-port': False}) }}
{% else -%}
{% do zodb_extern[0] %}
{% endif -%}
{# ZODB -#}
{% set zodb_dict = {} -%}
{% set storage_dict = {} -%}
{% set mountpoints = set() -%}
{% for zodb in slapparameter_dict.get('zodb') or ({'type': 'zeo', 'server': {}},) -%}
{% do mountpoints.add(zodb.setdefault('mount-point', '/')) -%}
{% set name = zodb.pop('name', 'root') -%}
{% do assert(name not in zodb_dict) -%}
{% do zodb_dict.__setitem__(name, zodb) -%}
{% if 'server' in zodb -%}
{% do storage_dict.setdefault(zodb['type'], {}).__setitem__(name, zodb.pop('server')) -%}
{% endif -%}
{% endfor -%}
{% do assert(len(mountpoints) == len(zodb_dict)) %}
{% for server_type, server_dict in storage_dict.iteritems() -%}
{% do assert(server_type == 'zeo') -%}
{# BBB: for compatibility, keep 'zodb' as partition_reference for ZEO -#}
{{ request('zodb', 'zodb-' ~ server_type, 'zodb-' ~ server_type, {'tcpv4-port': 2100, 'zodb-dict': server_dict}, dict.fromkeys(('storage-dict', 'tidstorage-ip', 'tidstorage-port'))) }}
{% endfor -%}
[inituser-password]
{% set inituser_password = slapparameter_dict.get('inituser-password') -%}
......@@ -81,14 +106,11 @@ config-mysql-url-list = ${request-mariadb:connection-database-list}
config-site-id = {{ dumps(site_id) }}
config-smtp-url = {{ dumps(slapparameter_dict.get('smtp-url', 'smtp://localhost:25/')) }}
config-timezone = {{ dumps(slapparameter_dict.get('timezone', 'UTC')) }}
{% if zodb_type -%}
config-zodb-dict = {{ dumps(zodb_dict) }}
{% if storage_dict -%}
config-zodb-zeo = ${request-zodb:connection-storage-dict}
config-tidstorage-ip = ${request-zodb:connection-tidstorage-ip}
config-tidstorage-port = ${request-zodb:connection-tidstorage-port}
config-zodb-dict = ${request-zodb:connection-zodb-dict}
config-zodb-storage-type = ${request-zodb:connection-zodb-storage-type}
{% endif -%}
{% if zodb_extern -%}
config-zodb-extern = {{ dumps(zodb_extern) }}
{% endif -%}
software-type = zope
......@@ -100,6 +122,7 @@ software-type = zope
[{{ section_name }}]
< = request-zope-base
name = {{ partition_name }}
{{ sla(partition_name) }}
config-name = {{ dumps(custom_name) }}
config-instance-count = {{ dumps(zope_parameter_dict.get('instance-count', 1)) }}
config-thread-amount = {{ dumps(zope_parameter_dict.get('thread-amount', 1)) }}
......@@ -108,7 +131,6 @@ config-longrequest-logger-interval = {{ dumps(zope_parameter_dict.get('longreque
config-longrequest-logger-timeout = {{ dumps(zope_parameter_dict.get('longrequest-logger-timeout', 1)) }}
config-port-base = {{ dumps(zope_parameter_dict.get('port-base', 2200)) }}
config-webdav = {{ dumps(zope_parameter_dict.get('webdav', False)) }}
sla-computer_guid = {{ dumps(zope_parameter_dict.get('computer-guid', computer_id)) }}
{% endfor -%}
{# We need to concatenate lists that we cannot read as lists, so this gets hairy. -#}
......@@ -138,7 +160,7 @@ config-url = ${request-balancer:{{ family_name }}-v6}
< = request-common
name = balancer
software-type = balancer
sla-computer_guid = {{ dumps(slapparameter_dict.get('balancer-computer-guid', computer_id)) }}
{{ sla('balancer') }}
return =
{%- for family in zope_family_dict %}
{{ family }}
......@@ -159,7 +181,7 @@ config-ssl = {{ dumps(balancer_dict.get('ssl', {})) }}
< = request-common
software-url = {{ dumps(frontend_dict['software-url']) }}
software-type = {{ dumps(frontend_dict.get('software-type', 'RootSoftwareInstance')) }}
sla-instance_guid = {{ dumps(frontend_dict['instance-guid']) }}
{{ sla('frontend', True) }}
slave = true
{% set config_dict = {
'type': 'zope',
......@@ -188,3 +210,5 @@ hosts-dict = {{ '${' ~ zope_address_list_id_dict.keys()[0] ~ ':connection-hosts-
[buildout]
parts = publish
{{- assert(not sla_dict) }}
stack/erp5/instance-zeo.cfg.in
View file @ a2ba55e0
{% set next_port = slapparameter_dict['tcpv4-port'] -%}
{% set ipv4 = (ipv4_set | list)[0] -%}
{% set backup_periodicity = slapparameter_dict.get('backup-periodicity', 'daily') -%}
{% set default_tidstorage_timestamp_path = slapparameter_dict.get('tidstorage-timestamp-path', buildout_directory ~ '/srv/backup/tidstorage') -%}
{% set part_list = [] -%}
{% macro section(name) %}{% do part_list.append(name) %}{{ name }}{% endmacro -%}
{% set storage_dict = {} -%}
{% set zodb_dict = {} -%}
{% set tidstorage_dict = slapparameter_dict.get('backup', {}) -%}
{% set known_tid_storage_identifier_dict = {} -%}
{% set default_zodb_path = buildout_directory ~ '/srv/zodb' -%}
{% set zodb_backup_path = buildout_directory ~ '/srv/backup/zodb' -%}
{% set default_tidstorage_timestamp_path = buildout_directory ~ '/srv/backup/tidstorage' -%}
{% macro section(name) %}{% do part_list.append(name) %}{{ name }}{% endmacro -%}
{% set ipv4 = (ipv4_set | list)[0] -%}
{% set tidstorage_ip = ipv4 -%}
{% set tidstorage_port = 6001 -%}
{% set default_backup_path = buildout_directory ~ '/srv/backup/zodb' -%}
{% set bin_directory = parameter_dict['buildout-bin-directory'] -%}
[logrotate-base]
......@@ -27,50 +23,33 @@ wrapper-path = ${directory:services}/${:base-name}
binary-path = {{ bin_directory }}/runzeo
ip = {{ ipv4 }}
{% set backup_directory_list = [] -%}
{% for export_id, mountpoint_dict in slapparameter_dict['zodb-dict'].items() -%}
{% do storage_dict.setdefault(
mountpoint_dict.get('storage-family', 'default').lower(), []
).append((
export_id,
mountpoint_dict.get('mount-point', '/'),
mountpoint_dict.get('cache-size', -1),
mountpoint_dict.get('storage-dict', {}),
)) -%}
{% set known_tid_storage_identifier_dict = {} -%}
{% set zodb_dict = {} -%}
{% for name, zodb in slapparameter_dict['zodb-dict'].iteritems() -%}
{% do zodb_dict.setdefault(zodb.get('family', 'default').lower(), []).append((name, zodb)) -%}
{% endfor -%}
{% for storage_family, export_list in storage_dict.items() -%}
{% set known_tid_storage_identifier_host = ((ipv4, next_port), ) -%}
{% for export_id, mount_point, cache_size, export_storage_dict in export_list -%}
{% do export_storage_dict.__setitem__('path', export_storage_dict.get('path', '%(zodb)s/%(name)s.fs') % {'zodb': default_zodb_path, 'name': export_id}) -%}
{% set client_dict = export_storage_dict.get('client', {}).copy() -%}
{% do client_dict.__setitem__('storage', export_id) -%}
{# XXX: I would like to raise if export_id is present in zodb_dict -#}
{% do zodb_dict.__setitem__(export_id, [
mount_point,
cache_size,
client_dict,
]) -%}
{% if tidstorage_dict != None -%}
{% set backup_directory = tidstorage_dict.get('zodb-dict', {}).get(export_id, '%(backup)s/%(name)s') % {'backup': zodb_backup_path, 'name': export_id} -%}
{% do backup_directory_list.append(backup_directory) -%}
{% do known_tid_storage_identifier_dict.__setitem__(
json_module.dumps((known_tid_storage_identifier_host, export_id)), (
export_storage_dict['path'],
backup_directory,
mount_point,
),
) -%}
{% endif -%}
{% set tidstorage_port = next_port + len(zodb_dict) -%}
{% for family, zodb in zodb_dict.iteritems() -%}
{% set storage_list = [] -%}
{% set known_tid_storage_identifier_host = (ipv4, next_port), -%}
{% for name, zodb in zodb -%}
{% do storage_dict.__setitem__(name, {'server': ipv4 ~ ':' ~ next_port, 'storage': name}) %}
{% set path = zodb.get('path', '%(zodb)s/%(name)s.fs') % {'zodb': default_zodb_path, 'name': name} -%}
{% do storage_list.append((name, path)) -%}
{% set backup_directory = zodb.get('backup', '%(backup)s/%(name)s') % {'backup': default_backup_path, 'name': name} -%}
{# BBB: No mount-point specified because they're meaningless for ZEO and
TIDStorage. Pass '' for compatibility, and not None
because this would disable TIDStorage bootstrapping. -#}
{% do known_tid_storage_identifier_dict.__setitem__(json_module.dumps(
(known_tid_storage_identifier_host, name)),
(path, backup_directory, '')) -%}
{% endfor -%}
{% set zeo_section_name = 'zeo-' ~ storage_family %}
{% set zeo_section_name = 'zeo-' ~ family %}
[{{ zeo_section_name }}]
< = zeo-base
base-name = zeo-{{ storage_family }}
base-name = zeo-{{ family }}
port = {{ next_port }}
{% set storage_list = [] -%}
{% for storage_name, _, _, export_storage_dict in export_list -%}
{% do storage_list.append((storage_name, export_storage_dict['path'])) -%}
{% endfor -%}
storage = {{ dumps(storage_list) }}
[{{ section("logrotate-" ~ zeo_section_name) }}]
......@@ -82,38 +61,39 @@ post = {{ bin_directory }}/slapos-kill --pidfile {{ "${" ~ zeo_section_name ~ ":
[{{ section(zeo_section_name ~ "-promise") }}]
recipe = slapos.cookbook:check_port_listening
hostname = {{ "${" ~ zeo_section_name ~ ":ip}" }}
port = {{ next_port }}
path = ${directory:promises}/zeo-{{ storage_family }}
port = {{ "${" ~ zeo_section_name ~ ":port}" }}
path = ${directory:promises}/zeo-{{ family }}
{% set address = ipv4 ~ ':' ~ next_port -%}
{% for export in export_list -%}
{% do zodb_dict[export[0]][2].__setitem__('server', address) -%}
{% endfor -%}
{% set next_port = next_port + 1 -%}
{% endfor -%}
{% if tidstorage_dict != None -%}
{% if backup_periodicity == 'never' -%}
{% set known_tid_storage_identifier_dict = () %}
{% else -%}
[tidstorage]
recipe = slapos.cookbook:tidstorage
known-tid-storage-identifier-dict = {{ dumps(known_tid_storage_identifier_dict) }}
configuration-path = ${directory:etc}/tidstorage.py
ip = {{ tidstorage_ip }}
ip = {{ ipv4 }}
port = {{ tidstorage_port }}
timestamp-file-path = {{ dumps(tidstorage_dict.get('tidstorage-timestamp-path', '%(backup)s/repozo_tidstorage_timestamp.log' % {'backup': default_tidstorage_timestamp_path})) }}
logfile-name = ${directory:log}/tidstorage.log
timestamp-file-path = {{ dumps(slapparameter_dict.get('tidstorage-timestamp-path', '%(backup)s/repozo_tidstorage_timestamp.log') % {'backup': default_tidstorage_timestamp_path}) }}
{# BBB: recipe requires logfile-name for nothing because tidstorage runs in foreground mode -#}
logfile-name =
pidfile-name = ${directory:run}/tidstorage.pid
status-file = ${directory:tidstorage}/tidstorage.tid
tidstorage-repozo-binary = {{ bin_directory }}/tidstorage_repozo
tidstoraged-binary = {{ bin_directory }}/tidstoraged
repozo-binary = {{ bin_directory }}/repozo
tidstorage-wrapper = ${directory:services}/tidstoraged
repozo-wrapper = ${buildout:bin-directory}/tidstorage-repozo
{% if len(known_tid_storage_identifier_dict) > 1 -%}
tidstorage-wrapper = ${directory:services}/tidstoraged
[{{ section("promise-tidstorage") }}]
recipe = slapos.cookbook:check_port_listening
hostname = ${tidstorage:ip}
port = ${tidstorage:port}
path = ${directory:promises}/tidstorage
{% endif -%}
[{{ section("cron-entry-tidstorage-backup") }}]
# TODO:
......@@ -122,40 +102,20 @@ path = ${directory:promises}/tidstorage
recipe = slapos.cookbook:cron.d
cron-entries = ${cron:cron-entries}
name = tidstorage
time = {{ dumps(tidstorage_dict.get('periodicity', 'daily')) }}
time = {{ dumps(backup_periodicity) }}
command = ${tidstorage:repozo-wrapper}
{% if backup_directory_list -%}
[{{ section("cron-entry-purge-repozo-fsindex") }}]
recipe = slapos.cookbook:cron.d
cron-entries = ${cron:cron-entries}
name = purge-repozo-fsindex
time = ${cron-entry-tidstorage-backup:time}
command = ${cron-script-purge-repozo-fsindex:wrapper-path}
[cron-script-purge-repozo-fsindex]
recipe = slapos.cookbook:wrapper
command-line = "{{ parameter_dict['findutils-location'] }}/bin/find" {% for backup_directory in backup_directory_list %}'{{ backup_directory.replace("\\", "\\\\").replace("'", "\\'") }}' {% endfor %}-name "????-??-??-??-??-??.index" -daystart -mtime +2 -delete
wrapper-path = ${directory:bin}/cron-purge-repozo-fsindex
{% endif -%}
[{{ section("logrotate-tidstorage") }}]
< = logrotate-base
name = tidstorage
log = ${tidstorage:logfile-name}
post = {{ bin_directory }}/slapos-kill --pidfile ${tidstorage:pidfile-name} -s HUP
{% else -%}
[tidstorage]
ip =
port =
{% endif -%}
[publish]
recipe = slapos.cookbook:publish.serialised
zodb-storage-type = zeoclient
zodb-dict = {{ dumps(zodb_dict) }}
storage-dict = {{ dumps(storage_dict) }}
{% if len(known_tid_storage_identifier_dict) > 1 -%}
tidstorage-ip = ${tidstorage:ip}
tidstorage-port = ${tidstorage:port}
{% else -%}
tidstorage-ip =
tidstorage-port =
{% endif -%}
[directory]
recipe = slapos.cookbook:mkdirectory
......@@ -166,7 +126,7 @@ promises = ${:etc}/promise
var = ${buildout:directory}/var
log = ${:var}/log
run = ${:var}/run
backup-zodb = {{ zodb_backup_path }}
backup-zodb = {{ default_backup_path }}
zodb = {{ default_zodb_path }}
tidstorage = {{ default_tidstorage_timestamp_path }}
......
stack/erp5/instance-zope.cfg.in
View file @ a2ba55e0
{% set use_ipv6 = slapparameter_dict.get('use-ipv6', False) -%}
{% set next_port = slapparameter_dict['port-base'] -%}
{% set site_id = slapparameter_dict['site-id'] -%}
{% set storage_type = slapparameter_dict['zodb-storage-type'] -%}
{% set zodb_dict = slapparameter_dict['zodb-dict'] -%}
{% set node_id_base = slapparameter_dict['name'] -%}
{% set part_list = [] -%}
{% set publish_list = [] -%}
{% set zodb_dict = slapparameter_dict.get('zodb-dict', {}) -%}
{% set longrequest_logger_base_path = buildout_directory ~ '/var/log/longrequest_logger_' -%}
{% macro section(name) %}{% do part_list.append(name) %}{{ name }}{% endmacro -%}
{% set bin_directory = parameter_dict['buildout-bin-directory'] -%}
......@@ -110,16 +109,6 @@ ipv4 = {{ ipv4 }}
ipv6 = {{ ipv6 }}
{% endif -%}
{% if slapparameter_dict.get('tidstorage-ip') -%}
[tidstorage]
ipv4 = {{ dumps(slapparameter_dict['tidstorage-ip']) }}
ipv4-port = {{ dumps(slapparameter_dict['tidstorage-port']) }}
{% else -%}
[tidstorage]
ipv4 =
ipv4-port =
{% endif -%}
{% set hosts_dict = {} -%}
{% for alias, url in (
('erp5-memcached-volatile', slapparameter_dict['memcached-url']),
......@@ -131,14 +120,11 @@ ipv4-port =
urlparse.urlparse(url).hostname,
) -%}
{%- endfor %}
{# jinja2 does not support enumerate... -#}
{% set catalog_counter = 0 %}
{% for url in slapparameter_dict['mysql-url-list'] -%}
{% for i, url in enumerate(slapparameter_dict['mysql-url-list']) -%}
{% do hosts_dict.__setitem__(
'erp5-catalog-' ~ catalog_counter,
'erp5-catalog-' ~ i,
urlparse.urlparse(url).hostname,
) -%}
{% set catalog_counter = catalog_counter + 1 -%}
{%- endfor %}
{% do hosts_dict.update(slapparameter_dict['hosts-dict']) -%}
[hosts-parameter]
......@@ -182,20 +168,19 @@ bt5-repository =
[zope-conf-parameter-base]
ip = {{ ipv4 }}
site-id = {{ site_id }}
{% set zodb_list = slapparameter_dict.get('zodb-extern', []) -%}
{% for key, (mount_point, cache_size, storage_dict) in zodb_dict.items() -%}
{% do zodb_list.append({'name': key, 'mount-point': mount_point, 'storage-type': storage_type, 'storage-dict': storage_dict}) -%}
{% if cache_size >= 0 -%}
{% do zodb_list[-1].__setitem__('cache-size', cache_size) -%}
{% endif -%}
{% set storage_dict = {'neo': {}, 'zeo': slapparameter_dict.get('zodb-zeo', {})} -%}
{% for name, zodb in zodb_dict.iteritems() -%}
{% do zodb.setdefault('storage-dict', {}).update(storage_dict[zodb['type']].get(name, {})) -%}
{% endfor -%}
developer-list = {{ dumps(slapparameter_dict['developer-list']) }}
instance = ${directory:instance}
instance-products = ${directory:instance-products}
deadlock-path = /manage_debug_threads
deadlock-debugger-password = {{ dumps(slapparameter_dict['deadlock-debugger-password']) }}
tidstorage-ip = ${tidstorage:ipv4}
tidstorage-port = ${tidstorage:ipv4-port}
{% if slapparameter_dict.get('tidstorage-ip') -%}
tidstorage-ip = {{ dumps(slapparameter_dict['tidstorage-ip']) }}
tidstorage-port = {{ dumps(slapparameter_dict['tidstorage-port']) }}
{% endif -%}
promise-path = ${erp5-promise:promise-path}
{% set thread_amount = slapparameter_dict['thread-amount'] -%}
thread-amount = {{ thread_amount }}
......@@ -237,17 +222,17 @@ event-log = ${directory:log}/{{ name }}-event.log
z2-log = ${directory:log}/{{ name }}-Z2.log
node-id = {{ dumps(node_id_base ~ '-' ~ index) }}
{% set log_list = [] -%}
{% set import_set = {} -%}
{% for zodb in zodb_list -%}
{% if zodb.setdefault('storage-type', 'NEOStorage') == 'NEOStorage' -%}
{% do import_set.__setitem__('neo.client', None) -%}
{% set log = buildout_directory ~ '/var/log/' ~ name ~ '-neo-' ~ zodb.get('name', 'main') ~ '.log' -%}
{% set import_set = set() -%}
{% for db_name, zodb in zodb_dict.iteritems() -%}
{% if zodb['type'] == 'neo' -%}
{% do import_set.add('neo.client') -%}
{% set log = buildout_directory ~ '/var/log/' ~ name ~ '-neo-' ~ db_name ~ '.log' -%}
{% do log_list.append(log) -%}
{% do zodb.setdefault('storage-dict', {}).__setitem__('logfile', log) -%}
{% do zodb['storage-dict'].__setitem__('logfile', log) -%}
{% endif -%}
{% endfor -%}
import-list = {{ dumps(import_set.keys()) }}
zodb-list = {{ dumps(zodb_list) }}
import-list = {{ dumps(list(import_set)) }}
zodb-dict = {{ dumps(zodb_dict) }}
[{{ conf_name }}]
< = zope-conf-base
......
stack/erp5/instance.cfg.in
View file @ a2ba55e0
......@@ -71,6 +71,7 @@ filename = instance-erp5.cfg
extensions = jinja2.ext.do
extra-context =
key local_bt5_repository dynamic-template-erp5-parameters:local-bt5-repository
import urlparse urlparse
[dynamic-template-balancer-parameters]
apache = {{ apache_location }}
......@@ -92,7 +93,6 @@ extra-context =
[dynamic-template-zeo-parameters]
buildout-bin-directory = {{ buildout_bin_directory }}
findutils-location = {{ findutils_location }}
[dynamic-template-zeo]
< = jinja2-template-base
......
stack/erp5/zope.conf.in
View file @ a2ba55e0
......@@ -42,10 +42,9 @@ products {{ parameter_dict['instance-products'] }}
secret {{ parameter_dict['deadlock-debugger-password'] }}
</product-config>
{% set tidstorage_ip = parameter_dict['tidstorage-ip'] -%}
{% if tidstorage_ip -%}
{% if 'tidstorage-ip' in parameter_dict -%}
<product-config TIDStorage>
backend-ip {{ tidstorage_ip }}
backend-ip {{ parameter_dict['tidstorage-ip'] }}
backend-port {{ parameter_dict['tidstorage-port'] }}
</product-config>
......@@ -104,11 +103,11 @@ products {{ parameter_dict['instance-products'] }}
{% for m in parameter_dict['import-list'] -%}
%import {{ m }}
{% endfor -%}
{% for zodb_dict in parameter_dict['zodb-list'] %}
<zodb_db {{ zodb_dict.pop('name', 'main') }}>
{%- set storage_type = zodb_dict.pop('storage-type', 'NEOStorage') %}
{%- set storage_dict = zodb_dict.pop('storage-dict', {}) %}
mount-point {{ zodb_dict.pop('mount-point', '/') }}
{% set type_dict = {'neo': 'NEOStorage', 'zeo': 'zeoclient'} %}
{% for name, zodb_dict in parameter_dict['zodb-dict'].iteritems() %}
<zodb_db {{ name }}>
{%- set storage_type = type_dict[zodb_dict.pop('type')] %}
{%- set storage_dict = zodb_dict.pop('storage-dict') %}
{%- for key, value in zodb_dict.iteritems() %}
{{ key }} {{ value }}
{%- endfor %}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7