instance-mariadb.cfg.in 11.8 KB
Newer Older
1 2 3 4
{% set part_list = [] -%}
{% macro section(name) %}{% do part_list.append(name) %}{{ name }}{% endmacro -%}
{% set use_ipv6 = slapparameter_dict.get('use-ipv6', False) -%}
{% set database_list = slapparameter_dict.get('database-list', [{'name': 'erp5', 'user': 'user', 'password': 'insecure'}]) -%}
5
{% set test_database_list = [] %}
6
{% for database_count in range(slapparameter_dict.get('test-database-amount', 1)) -%}
7
{%   do test_database_list.append({'name': 'erp5_test_' ~ database_count, 'user': 'testuser_' ~ database_count, 'password': 'testpassword' ~ database_count}) -%}
8 9
{% endfor -%}
{% set catalog_backup = slapparameter_dict.get('catalog-backup', {}) -%}
10
{% set backup_periodicity = slapparameter_dict.get('backup-periodicity', 'daily') -%}
11 12
{% set full_backup_retention_days = catalog_backup.get('full-retention-days', 7) -%}
{% set incremental_backup_retention_days = catalog_backup.get('incremental-retention-days', full_backup_retention_days) -%}
13
{% set port = slapparameter_dict['tcpv4-port'] %}
14
{% if use_ipv6 -%}
15
{%   set ip = (ipv6_set | list)[0] -%}
16
{% else -%}
17
{%   set ip = (ipv4_set | list)[0] -%}
18
{% endif -%}
19
{% set dash = parameter_dict['dash-location'] ~ '/bin/dash' %}
20

21
[{{ section('publish') }}]
22
recipe = slapos.cookbook:publish.serialised
23
-extends = publish-early
24 25 26
{% macro render_database_list(database_list) -%}
{% set publish_database_list = [] -%}
{% for database in database_list -%}
27 28
{%   if database.get('user') -%}
{%     do publish_database_list.append("mysql://" ~ database['user'] ~ ":" ~ database['password'] ~ "@" ~ ip ~ ":" ~ port ~ "/" ~ database['name']) -%}
29
{%   else -%}
30
{%     do publish_database_list.append("mysql://" ~ ip ~ ":" ~ port ~ "/" ~ database['name']) -%}
31
{%   endif -%}
32 33 34 35 36
{% endfor -%}
{{ dumps(publish_database_list) }}
{% endmacro -%}
database-list = {{ render_database_list(database_list) }}
test-database-list = {{ render_database_list(test_database_list) }}
37
monitor-base-url = ${monitor-publish-parameters:monitor-base-url}
38

39 40 41 42 43 44 45 46 47 48 49 50 51 52
[publish-early]
recipe = slapos.cookbook:publish-early
-init =
  server-id gen-server-id:value
{%- set server_id = slapparameter_dict.get('server-id') %}
{%- if server_id %}
server-id = {{ dumps(server_id) }}
{%- endif %}

[gen-server-id]
recipe = slapos.cookbook:random.integer
minimum = {{ dumps(1) }}
maximum = {{ dumps(2**32 - 1) }}

53
[jinja2-template-base]
54
recipe = slapos.recipe.template:jinja2
55 56 57 58 59 60 61 62
mode = 644

[jinja2-template-executable]
< = jinja2-template-base
mode = 755

[simplefile]
< = jinja2-template-base
63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92
template = inline:{{ '{{ content }}' }}

{% macro simplefile(section_name, file_path, content, mode='') -%}
{%   set content_section_name = section_name ~ '-content' -%}
[{{  content_section_name }}]
content = {{ dumps(content) }}

[{{  section(section_name) }}]
< = simplefile
rendered = {{ file_path }}
context = key content {{content_section_name}}:content
mode = {{ mode }}
{%- endmacro %}
{% set ssl_dict = {} -%}
{% macro sslfile(key, content, mode='644') -%}
{%   set path = '${directory:mariadb-ssl}/' ~ key ~ '.pem' -%}
{%   do ssl_dict.__setitem__(key, path) -%}
{{   simplefile('ssl-file-' ~ key, path, content, mode) }}
{%- endmacro %}
{% set ssl_parameter_dict = slapparameter_dict.get('ssl') -%}
{% if ssl_parameter_dict -%}
{%   set base_directory = '${directory:mariadb-ssl}/' -%}
{# Note: The key content will be stored in .installed.cfg, and this template's
rendering, so the only point of mode is to avoid risking mariadb complaining
about laxist file mode. -#}
{{   sslfile('key', ssl_parameter_dict['key'], mode='600') }}
{{   sslfile('crt', ssl_parameter_dict['crt']) }}
{%   if 'ca-crt' in ssl_parameter_dict -%}
{{     sslfile('ca-crt', ssl_parameter_dict['ca-crt']) }}
{%   endif -%}
93 94 95
{%   if 'crl' in ssl_parameter_dict -%}
{{     sslfile('crl', ssl_parameter_dict['crl']) }}
{%   endif -%}
96 97
{%- endif %}

98 99
{% if full_backup_retention_days > -1 -%}
[{{ section('cron-entry-mariadb-backup') }}]
100
recipe = slapos.cookbook:cron.d
101
cron-entries = ${cron:cron-entries}
102
name = mariadb-backup
103
time = {{ dumps(backup_periodicity) }}
104 105 106 107 108 109
{# When binlogs are enabled:
# flush-logs: used so no manipulation on binlogs is needed to restore from
#   full + binlogs. The first binlog after a dump starts from dump snapshot and
#   can be fully restored.
# master-data: use value "2" as we are not in a replication case
#}
110
command = "${binary-wrap-mysqldump:wrapper-path}" -u root --all-databases --single-transaction {% if incremental_backup_retention_days > -1 %}--flush-logs --master-data=2 {% endif %}| {{ parameter_dict['gzip-location'] }}/bin/gzip > "${directory:mariadb-backup-full}/$({{ parameter_dict['coreutils-location'] }}/bin/date "+%Y%m%d%H%M%S").sql.gz"
111 112 113 114 115 116 117
{# KEEP GLOB PATTERN IN SYNC with generated filenames above
#           YYYYmmddHHMMSS -#}
file-glob = ??????????????.sql.gz

{% if full_backup_retention_days > 0 -%}
[{{ section("cron-entry-mariadb-backup-expire") }}]
recipe = slapos.cookbook:cron.d
118
cron-entries = ${cron:cron-entries}
119
name = mariadb-backup-expire
120
time = {{ dumps(backup_periodicity) }}
121 122 123 124 125
command = {{ parameter_dict['findutils-location'] }}/bin/find "${directory:mariadb-backup-full}" -maxdepth 1 -name "${cron-entry-mariadb-backup:file-glob}" -daystart -mtime +{{ full_backup_retention_days }} -delete
{%- endif %}
{%- endif %}

[my-cnf-parameters]
126 127
ip = {{ ip }}
port = {{ port }}
128 129
socket = ${directory:run}/mariadb.sock
data-directory = ${directory:mariadb-data}
130
tmp-directory = ${directory:tmp}
131 132 133
pid-file = ${directory:run}/mariadb.pid
error-log = ${directory:log}/mariadb_error.log
slow-query-log = ${directory:log}/mariadb_slowquery.log
134
long-query-time = {{ dumps(slapparameter_dict.get('long-query-time', 1)) }}
135
innodb-buffer-pool-size = {{ dumps(slapparameter_dict.get('innodb-buffer-pool-size', 0)) }}
136
innodb-buffer-pool-instances = {{ dumps(slapparameter_dict.get('innodb-buffer-pool-instances', 0)) }}
137 138
innodb-log-file-size = {{ dumps(slapparameter_dict.get('innodb-log-file-size', 0)) }}
innodb-log-buffer-size = {{ dumps(slapparameter_dict.get('innodb-log-buffer-size', 0)) }}
139
relaxed-writes = {{ dumps(slapparameter_dict.get('relaxed-writes', False)) }}
140 141 142
{% if incremental_backup_retention_days > -1 -%}
binlog-path = ${directory:mariadb-backup-incremental}/binlog
# XXX: binlog rotation happens along with other log's rotation
143
binlog-expire-days = {{ dumps(incremental_backup_retention_days) }}
144
server-id = ${publish-early:server-id}
145 146
{% else %}
binlog-path =
147 148 149 150
{%- endif %}
{%- for key, value in ssl_dict.items() -%}
ssl-{{ key }} = {{ value }}
{% endfor %}
151 152

[my-cnf]
153
< = jinja2-template-base
154 155 156 157 158
rendered = ${directory:etc}/mariadb.cnf
template = {{ parameter_dict['template-my-cnf'] }}
context = section parameter_dict my-cnf-parameters

[init-script-parameters]
159
database-list = {{ dumps(database_list + test_database_list) }}
160 161

[init-script]
162
< = jinja2-template-executable
163 164 165 166 167
# XXX: is there a better location ?
rendered = ${directory:etc}/mariadb_initial_setup.sql
template = {{ parameter_dict['template-mariadb-initial-setup'] }}
context = section parameter_dict init-script-parameters

168
[{{ section('update-mysql') }}]
169 170 171 172 173
recipe = slapos.cookbook:generic.mysql.wrap_update_mysql
output = ${directory:services}/mariadb_update
binary = ${binary-wrap-mysql_upgrade:wrapper-path}
mysql = ${binary-wrap-mysql:wrapper-path}
init-script = ${init-script:rendered}
174
mysql_tzinfo_to_sql = ${binary-wrap-mysql_tzinfo_to_sql:wrapper-path}
175

176 177 178 179 180 181
[{{ section('mysqld') }}]
< = jinja2-template-executable
# Note: all rendering is done when this file is rendered, not when the mysqld
# section is installed - so I only use jinja2 as a fancy way to write an
# executable file with partition-dependent but instance-parameters independent
# content.
182
template = inline:#!{{ dash }}
183 184 185 186 187 188 189 190 191
  '{{ parameter_dict['mariadb-location'] }}/scripts/mysql_install_db' \
    --defaults-file='${my-cnf:rendered}' \
    --skip-name-resolve \
    --datadir='${my-cnf-parameters:data-directory}' \
    --basedir='{{ parameter_dict['mariadb-location'] }}' \
  && exec '{{ parameter_dict['mariadb-location'] }}/bin/mysqld' \
    --defaults-file='${my-cnf:rendered}' \
    "$@"
rendered = ${directory:services}/mariadb
192

193
[{{ section('logrotate-entry-mariadb') }}]
194
< = logrotate-entry-base
195
name = mariadb
196
log = ${my-cnf-parameters:error-log} ${my-cnf-parameters:slow-query-log}
197
post = "${binary-wrap-mysql:wrapper-path}" -B -u root -e "FLUSH LOGS"
198

199
[{{ section('binary-link') }}]
200
recipe = slapos.cookbook:symbolic.link
201
target-directory = ${directory:bin}
202
link-binary = {{ dumps(parameter_dict['link-binary']) }}
203

204 205 206 207
[{{ section("binary-link-mysqlbinlog") }}]
< = binary-link
link-binary = {{ parameter_dict['mariadb-location'] }}/bin/mysqlbinlog

208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223
[binary-wrap-base]
recipe = slapos.cookbook:wrapper
# Note: --defaults-file must be the first argument, otherwise wrapped binary
# will reject it.
command-line = "{{ parameter_dict['mariadb-location'] }}/bin/${:command}" --defaults-file="${my-cnf:rendered}"
wrapper-path = ${directory:bin}/${:command}
parameters-extra = true

[binary-wrap-mysql]
< = binary-wrap-base
command = mysql

[binary-wrap-mysqldump]
< = binary-wrap-base
command = mysqldump

224
[binary-wrap-mysql_upgrade]
225
< = binary-wrap-base
226 227 228 229
command = mysql_upgrade

[binary-wrap-mysql_tzinfo_to_sql]
< = binary-wrap-base
230
command-line = "{{ parameter_dict['mariadb-location'] }}/bin/${:command}"
231
command = mysql_tzinfo_to_sql
232

233 234 235 236 237 238
[binary-wrap-pt-digest]
<= binary-wrap-base
command-line = "{{ parameter_dict['percona-tools-location'] }}/bin/${:command}"
command = pt-query-digest
 

239 240
[directory]
recipe = slapos.cookbook:mkdirectory
241 242 243 244 245
bin = ${buildout:directory}/bin
etc = ${buildout:directory}/etc
services = ${:etc}/run
promise = ${:etc}/promise
srv = ${buildout:directory}/srv
246
tmp = ${buildout:directory}/tmp
247 248 249 250
backup = ${:srv}/backup
mariadb-backup-full = ${:backup}/mariadb-full
mariadb-backup-incremental = ${:backup}/mariadb-incremental
mariadb-data = ${:srv}/mariadb
251
mariadb-ssl = ${:etc}/mariadb-ssl
252 253 254
var = ${buildout:directory}/var
log = ${:var}/log
run = ${:var}/run
255

256
[{{ section('resiliency-exclude-file') }}]
257
# Generate rdiff exclude file in case of resiliency
258
< = jinja2-template-base
259 260
template = {{ 'inline:{{ "${directory:mariadb-data}/**\\n" }}' }}
rendered = ${directory:srv}/exporter.exclude
261

262 263 264
[dash]
dash = {{ dumps(dash) }}

265 266
[resiliency-after-import-script]
# Generate after import script used by importer instance of webrunner
267 268 269 270 271 272 273 274 275 276 277 278 279
< = jinja2-template-executable
template = {{ parameter_dict['mariadb-resiliency-after-import-script'] }}
rendered = ${directory:srv}/runner-import-restore
context =
  key dash dash:dash
  key mysql_executable binary-wrap-mysql:wrapper-path
  key mariadb_data_directory directory:mariadb-data
  key mariadb_backup_directory directory:mariadb-backup-full
  key instance_directory buildout:directory
  key pid_file my-cnf-parameters:pid-file
  key binlog_path my-cnf-parameters:binlog-path
  key server_executable mysqld:rendered
  key update_executable update-mysql:output
280

281 282 283 284 285 286 287 288 289 290 291 292 293
[{{ section('monitor-generate-mariadb-slow-query-report') }}]
recipe = slapos.recipe.template:jinja2
template = {{ parameter_dict['mariadb-slow-query-report-script'] }}
rendered = ${monitor-directory:reports}/${:filename}
output-folder = ${directory:srv}/monitor/
filename = mariadb_slow_query_every_23_hour
mode = 755
context = 
  raw slow_query_path ${directory:srv}/backup/logrotate/mariadb_slowquery.log
  raw pt_query_exec ${binary-wrap-pt-digest:wrapper-path}
  raw dash {{ parameter_dict['dash-location'] }}/bin/dash
  key output_folder :output-folder 

294
[{{ section('promise') }}]
295 296 297 298
recipe = slapos.cookbook:wrapper
command-line = "{{ parameter_dict['bin-directory'] }}/is-local-tcp-port-opened" "${my-cnf-parameters:ip}" "${my-cnf-parameters:port}"
wrapper-path = ${directory:promise}/mariadb
parameters-extra = true
299

300
{% set monitor_passwd = slapparameter_dict['monitor-passwd'] -%}
301 302 303 304
[monitor-instance-parameter]
monitor-httpd-ipv6 = {{ (ipv6_set | list)[0] }}
monitor-httpd-port = {{ port + 1 }}
monitor-title = Mariadb monitor
305
password = {{monitor_passwd}}
306

307
[buildout]
308 309
extends =
  {{ logrotate_cfg }}
310
  {{ parameter_dict['template-monitor'] }}
311
parts +=
312
  {{ part_list | join('\n  ') }}