Commit f0fb3596 authored by Julien Muchembled's avatar Julien Muchembled

NEO: review schema and add support for SQLite configuration

See nexedi/neoppod@71564067

See merge request nexedi/slapos!1566
parent d1823bd1
...@@ -14,7 +14,7 @@ ...@@ -14,7 +14,7 @@
# not need these here). # not need these here).
[instance-common] [instance-common]
filename = instance-common.cfg.in filename = instance-common.cfg.in
md5sum = e000e7134113b9d1c63d40861eaf0489 md5sum = ecc98da90cd446ea224ddeece1374190
[root-common] [root-common]
filename = root-common.cfg.in filename = root-common.cfg.in
...@@ -30,7 +30,7 @@ md5sum = 9f27195d770b2f57461c60a82c851ab9 ...@@ -30,7 +30,7 @@ md5sum = 9f27195d770b2f57461c60a82c851ab9
[instance-neo] [instance-neo]
filename = instance-neo.cfg.in filename = instance-neo.cfg.in
md5sum = fda911d5ef9efee365f1b0ff9843a50b md5sum = 200ae55715cb735b0f97f8c835a3071f
[template-neo-my-cnf] [template-neo-my-cnf]
filename = my.cnf.in filename = my.cnf.in
......
...@@ -41,6 +41,7 @@ url = {{ neo_master }} ...@@ -41,6 +41,7 @@ url = {{ neo_master }}
<= jinja2-template-base <= jinja2-template-base
url = {{ neo }} url = {{ neo }}
extra-context = extra-context =
import urllib urllib
key master_cfg neo-master:output key master_cfg neo-master:output
key admin_cfg neo-admin:output key admin_cfg neo-admin:output
{%- if mariadb_location is defined %} {%- if mariadb_location is defined %}
......
...@@ -110,40 +110,58 @@ ...@@ -110,40 +110,58 @@
"default": false, "default": false,
"type": "boolean" "type": "boolean"
}, },
"private-tmpfs": {
"description": "Size of private tmpfs mount to store the database. See filesystems/tmpfs.txt in Linux documentation. Use only for testing.",
"type": "string"
},
"test-binding-list": {
"description": "List of bindings to test when running the test suite.",
"type": "array"
},
"storage-type": { "storage-type": {
"description": "Storage type. Defaults to MySQL if available, else SQLite.", "description": "Storage type. Required when several types are configured and you select which one to use via 'node!' parameter. Defaults to whatever is configured ('sqlite' or 'mysql'), else MySQL if available, else SQLite.",
"enum": [ "enum": [
"MySQL", "MySQL",
"SQLite" "SQLite"
], ],
"type": "string" "type": "string"
}, },
"private-tmpfs": { "sqlite": {
"description": "Size of private tmpfs mount to store the database. See filesystems/tmpfs.txt in Linux documentation. Use only for testing.", "description": "Storage backend configuration.",
"type": "string" "properties": {
"relaxed-writes": {
"description": "When enabled, sets synchronous = OFF and journal_mode = MEMORY - RTFM, those options are dangerous.",
"default": false,
"type": "boolean"
}
}, },
"test-binding-list": { "additionalProperties": {
"description": "List of bindings to test when running the test suite.", "description": "See NEO documentation for the list of supported settings.",
"type": "array" "type": ["number", "string"]
},
"type": "object"
}, },
"mysql": { "mysql": {
"description": "Dictionary containing parameters for MySQL.", "description": "MariaDB server configuration.",
"default": {},
"properties": { "properties": {
"relaxed-writes": { "relaxed-writes": {
"description": "When enabled, sets innodb_flush_log_at_trx_commit = 0, innodb_flush_method = nosync, innodb_doublewrite = 0 and sync_frm = 0 - RTFM, those options are dangerous", "description": "When enabled, sets innodb_flush_log_at_trx_commit = 0, innodb_flush_method = nosync, innodb_doublewrite = 0 and sync_frm = 0 - RTFM, those options are dangerous.",
"default": false, "default": false,
"type": "boolean" "type": "boolean"
} }
}, },
"additionalProperties": { "additionalProperties": {
"description": "To configure important parameters like innodb_buffer_pool_size, rocksdb_block_cache_size, etc.", "description": "To configure parameters like innodb_buffer_pool_size, rocksdb_block_cache_size, etc.",
"type": "string" "type": ["number", "string"]
}, },
"type": "object" "type": "object"
}, },
"engine": { "engine": {
"description": "Configures storage engine, currently only InnoDB and RocksDB are supported. Defaults to NEO's default.", "description": "[MySQL only] For NEO, this is a creation-time parameter and it defaults to NEO's default. For mysqld, this sets plugins to load and it defaults to load all.",
"enum": [
"InnoDB",
"RocksDB"
],
"type": "string" "type": "string"
} }
}, },
......
{% macro section(name) %}{% do part_list.append(name) %}{{ name }}{% endmacro -%} {% macro section(name) %}{% do part_list.append(name) %}{{ name }}{% endmacro -%}
{% set part_list = [] -%} {% set part_list = [] -%}
{% set init_list = [] -%} {% set init_list = [] -%}
{% set directory_dict = {} -%}
{% set private_tmpfs = slapparameter_dict.get('private-tmpfs') -%} {% set private_tmpfs = slapparameter_dict.get('private-tmpfs') -%}
{% set storage_count = slapparameter_dict.get('storage-count', 1) -%} {% set storage_count = slapparameter_dict.get('storage-count', 1) -%}
{# When mixing different storage types via node specialisation
('node!' parameter), it can be convenient to configure all types in the
common 'node' parameter and then switch between the 2 with 'storage-type'.
So we must be quite tolerant. -#}
{% if storage_count -%}
{% if 'mysql' in slapparameter_dict -%}
{% if 'sqlite' in slapparameter_dict -%}
{% set storage_type = slapparameter_dict['storage-type'] -%}
{% else -%}
{% set storage_type = 'MySQL' -%}
{% endif -%}
{% elif 'sqlite' in slapparameter_dict -%}
{% set storage_type = 'SQLite' -%}
{% else -%}
{% set storage_type = slapparameter_dict.get('storage-type') or ( {% set storage_type = slapparameter_dict.get('storage-type') or (
'MySQL' if mariadb_location is defined else 'SQLite') -%} 'MySQL' if mariadb_location is defined else 'SQLite') -%}
{% set mysql = storage_count and storage_type != 'SQLite' -%} {% endif -%}
{% do assert(slapparameter_dict.get('storage-type', storage_type) == storage_type) -%}
{% else -%}
{% set storage_type = '' -%}
{% endif -%}
{% set mysql = storage_type == 'MySQL' -%}
{% if mysql -%} {% if mysql -%}
{% set extra_dict = slapparameter_dict.get('mysql') or {} -%}
[{{ section('mysqld') }}] [{{ section('mysqld') }}]
{% if private_tmpfs -%} {% if private_tmpfs -%}
...@@ -37,7 +59,7 @@ tmp-directory = ${directory:tmp} ...@@ -37,7 +59,7 @@ tmp-directory = ${directory:tmp}
pid-file = ${directory:var_run}/mariadb.pid pid-file = ${directory:var_run}/mariadb.pid
error-log = ${directory:log}/mariadb_error.log error-log = ${directory:log}/mariadb_error.log
slow-query-log = ${directory:log}/mariadb_slowquery.log slow-query-log = ${directory:log}/mariadb_slowquery.log
extra-dict = {{ dumps(slapparameter_dict.get('mysql', {})) }} extra-dict = {{ dumps(extra_dict) }}
init-file = ${init-script:output} init-file = ${init-script:output}
engine = {{ slapparameter_dict.get('engine', '') }} engine = {{ slapparameter_dict.get('engine', '') }}
...@@ -54,6 +76,18 @@ command-line = '{{ mariadb_location }}/bin/${:command}' --defaults-file="${my-cn ...@@ -54,6 +76,18 @@ command-line = '{{ mariadb_location }}/bin/${:command}' --defaults-file="${my-cn
wrapper-path = ${directory:bin}/${:command} wrapper-path = ${directory:bin}/${:command}
command = mysql command = mysql
{% elif storage_type == 'SQLite' -%}
{% set extra_dict = slapparameter_dict.get('sqlite') or {} -%}
{% if extra_dict.pop('relaxed-writes', False) -%}
{% do extra_dict.setdefault('synchronous', 'OFF') -%}
{% do extra_dict.setdefault('journal_mode', 'MEMORY') -%}
{% endif -%}
{% set query_string = urllib.urlencode(extra_dict) -%}
{% else -%}
{% do assert(not storage_count) -%}
{% endif -%} {% endif -%}
[{{ section('binary-neolog') }}] [{{ section('binary-neolog') }}]
...@@ -123,14 +157,19 @@ logfile = ${directory:log}/{{ 'neostorage-' ~ i }}.log ...@@ -123,14 +157,19 @@ logfile = ${directory:log}/{{ 'neostorage-' ~ i }}.log
{%- if mysql %} {%- if mysql %}
{%- do init_list.append('CREATE DATABASE IF NOT EXISTS neo' ~ i ~ ';') %} {%- do init_list.append('CREATE DATABASE IF NOT EXISTS neo' ~ i ~ ';') %}
database-parameters = root@neo{{ i }}${my-cnf-parameters:socket} database-parameters = root@neo{{ i }}${my-cnf-parameters:socket}
{%- elif private_tmpfs %} {%- else %}
{%- if private_tmpfs %}
private-tmpfs = {{ private_tmpfs }} ${directory:tmp} private-tmpfs = {{ private_tmpfs }} ${directory:tmp}
database-parameters = ${directory:tmp}/db.sqlite {%- set path = '${directory:tmp}/db.sqlite' %}
{%- else %} {%- else %}
database-parameters = ${directory:db-{{i}}}/db.sqlite {%- set path = '${directory:db-' ~ i ~ '}/db.sqlite' %}
{%- do directory_dict.__setitem__('db-' ~ i, '${:srv}/' + storage_id) %}
[directory] {%- endif %}
db-{{i}} = ${:srv}/{{ storage_id }} {%- if query_string %}
database-parameters = file:{{ path }}?{{ query_string }}
{%- else %}
database-parameters = {{ path }}
{%- endif -%}
{%- endif %} {%- endif %}
[{{ section('logrotate-storage-' ~ i) }}] [{{ section('logrotate-storage-' ~ i) }}]
...@@ -141,17 +180,6 @@ post = {{ bin_directory }}/slapos-kill -s RTMIN+1 -- {{ bin_directory }}/neostor ...@@ -141,17 +180,6 @@ post = {{ bin_directory }}/slapos-kill -s RTMIN+1 -- {{ bin_directory }}/neostor
{% endfor -%} {% endfor -%}
[directory]
recipe = slapos.cookbook:mkdirectory
bin = ${buildout:directory}/bin
etc = ${buildout:directory}/etc
var = ${buildout:directory}/var
etc_run = ${:etc}/run
var_run = ${:var}/run
log = ${buildout:directory}/var/log
tmp = ${buildout:directory}/tmp
srv = ${buildout:directory}/srv
{% if mysql -%} {% if mysql -%}
[init-script] [init-script]
recipe = slapos.recipe.template recipe = slapos.recipe.template
...@@ -196,13 +224,26 @@ context = ...@@ -196,13 +224,26 @@ context =
key datadir my-cnf-parameters:data-directory key datadir my-cnf-parameters:data-directory
key results_directory directory:results key results_directory directory:results
[directory] {%- do directory_dict.__setitem__('results', '${directory:srv}/tests') %}
results = ${directory:srv}/tests
{%- endif %} {%- endif %}
{%- endif %} {%- endif %}
{%- endif %} {%- endif %}
[directory]
recipe = slapos.cookbook:mkdirectory
bin = ${buildout:directory}/bin
etc = ${buildout:directory}/etc
var = ${buildout:directory}/var
etc_run = ${:etc}/run
var_run = ${:var}/run
log = ${buildout:directory}/var/log
tmp = ${buildout:directory}/tmp
srv = ${buildout:directory}/srv
{%- for k, v in directory_dict.iteritems() %}
{{ k }} = {{ v }}
{%- endfor %}
[buildout] [buildout]
extends = extends =
{{ logrotate_cfg }} {{ logrotate_cfg }}
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment