1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
{% set part_list = [] -%}
{% macro section(name) %}{% do part_list.append(name) %}{{ name }}{% endmacro -%}
{% set use_ipv6 = slapparameter_dict.get('use-ipv6', False) -%}
{% set database_list = slapparameter_dict.get('database-list', [{'name': 'erp5', 'user': 'user', 'password': 'insecure'}]) -%}
{% set test_database_list = [] %}
{% for database_count in range(slapparameter_dict.get('test-database-amount', 1)) -%}
{% do test_database_list.append({'name': 'erp5_test_' ~ database_count, 'user': 'testuser_' ~ database_count, 'password': 'testpassword' ~ database_count}) -%}
{% endfor -%}
{% set catalog_backup = slapparameter_dict.get('catalog-backup', {}) -%}
{% set backup_periodicity = slapparameter_dict.get('backup-periodicity', 'daily') -%}
{% set full_backup_retention_days = catalog_backup.get('full-retention-days', 7) -%}
{% set incremental_backup_retention_days = catalog_backup.get('incremental-retention-days', full_backup_retention_days) -%}
{% set port = slapparameter_dict['tcpv4-port'] %}
{% if use_ipv6 -%}
{% set ip = (ipv6_set | list)[0] -%}
{% else -%}
{% set ip = (ipv4_set | list)[0] -%}
{% endif -%}
{% set dash = parameter_dict['dash-location'] ~ '/bin/dash' %}
[{{ section('publish') }}]
recipe = slapos.cookbook:publish.serialised
-extends = publish-early
{% macro render_database_list(database_list) -%}
{% set publish_database_list = [] -%}
{% for database in database_list -%}
{% if database.get('user') -%}
{% do publish_database_list.append("mysql://" ~ database['user'] ~ ":" ~ database['password'] ~ "@" ~ ip ~ ":" ~ port ~ "/" ~ database['name']) -%}
{% else -%}
{% do publish_database_list.append("mysql://" ~ ip ~ ":" ~ port ~ "/" ~ database['name']) -%}
{% endif -%}
{% endfor -%}
{{ dumps(publish_database_list) }}
{% endmacro -%}
database-list = {{ render_database_list(database_list) }}
test-database-list = {{ render_database_list(test_database_list) }}
monitor-base-url = ${monitor-publish-parameters:monitor-base-url}
[publish-early]
recipe = slapos.cookbook:publish-early
-init =
server-id gen-server-id:value
{%- set server_id = slapparameter_dict.get('server-id') %}
{%- if server_id %}
server-id = {{ dumps(server_id) }}
{%- endif %}
[gen-server-id]
recipe = slapos.cookbook:random.integer
minimum = {{ dumps(1) }}
maximum = {{ dumps(2**32 - 1) }}
[jinja2-template-base]
recipe = slapos.recipe.template:jinja2
mode = 644
[jinja2-template-executable]
< = jinja2-template-base
mode = 755
[simplefile]
< = jinja2-template-base
template = inline:{{ '{{ content }}' }}
{% macro simplefile(section_name, file_path, content, mode='') -%}
{% set content_section_name = section_name ~ '-content' -%}
[{{ content_section_name }}]
content = {{ dumps(content) }}
[{{ section(section_name) }}]
< = simplefile
rendered = {{ file_path }}
context = key content {{content_section_name}}:content
mode = {{ mode }}
{%- endmacro %}
{% set ssl_dict = {} -%}
{% macro sslfile(key, content, mode='644') -%}
{% set path = '${directory:mariadb-ssl}/' ~ key ~ '.pem' -%}
{% do ssl_dict.__setitem__(key, path) -%}
{{ simplefile('ssl-file-' ~ key, path, content, mode) }}
{%- endmacro %}
{% set ssl_parameter_dict = slapparameter_dict.get('ssl') -%}
{% if ssl_parameter_dict -%}
{% set base_directory = '${directory:mariadb-ssl}/' -%}
{# Note: The key content will be stored in .installed.cfg, and this template's
rendering, so the only point of mode is to avoid risking mariadb complaining
about laxist file mode. -#}
{{ sslfile('key', ssl_parameter_dict['key'], mode='600') }}
{{ sslfile('crt', ssl_parameter_dict['crt']) }}
{% if 'ca-crt' in ssl_parameter_dict -%}
{{ sslfile('ca-crt', ssl_parameter_dict['ca-crt']) }}
{% endif -%}
{% if 'crl' in ssl_parameter_dict -%}
{{ sslfile('crl', ssl_parameter_dict['crl']) }}
{% endif -%}
{%- endif %}
{% if full_backup_retention_days > -1 -%}
[{{ section('cron-entry-mariadb-backup') }}]
recipe = slapos.cookbook:cron.d
cron-entries = ${cron:cron-entries}
name = mariadb-backup
time = {{ dumps(backup_periodicity) }}
{# When binlogs are enabled:
# flush-logs: used so no manipulation on binlogs is needed to restore from
# full + binlogs. The first binlog after a dump starts from dump snapshot and
# can be fully restored.
# master-data: use value "2" as we are not in a replication case
#}
command = "${binary-wrap-mysqldump:wrapper-path}" -u root --all-databases --single-transaction {% if incremental_backup_retention_days > -1 %}--flush-logs --master-data=2 {% endif %}| {{ parameter_dict['gzip-location'] }}/bin/gzip > "${directory:mariadb-backup-full}/$({{ parameter_dict['coreutils-location'] }}/bin/date "+%Y%m%d%H%M%S").sql.gz"
{# KEEP GLOB PATTERN IN SYNC with generated filenames above
# YYYYmmddHHMMSS -#}
file-glob = ??????????????.sql.gz
{% if full_backup_retention_days > 0 -%}
[{{ section("cron-entry-mariadb-backup-expire") }}]
recipe = slapos.cookbook:cron.d
cron-entries = ${cron:cron-entries}
name = mariadb-backup-expire
time = {{ dumps(backup_periodicity) }}
command = {{ parameter_dict['findutils-location'] }}/bin/find "${directory:mariadb-backup-full}" -maxdepth 1 -name "${cron-entry-mariadb-backup:file-glob}" -daystart -mtime +{{ full_backup_retention_days }} -delete
{%- endif %}
{%- endif %}
[my-cnf-parameters]
ip = {{ ip }}
port = {{ port }}
socket = ${directory:run}/mariadb.sock
data-directory = ${directory:mariadb-data}
tmp-directory = ${directory:tmp}
pid-file = ${directory:run}/mariadb.pid
error-log = ${directory:log}/mariadb_error.log
slow-query-log = ${directory:log}/mariadb_slowquery.log
long-query-time = {{ dumps(slapparameter_dict.get('long-query-time', 1)) }}
innodb-buffer-pool-size = {{ dumps(slapparameter_dict.get('innodb-buffer-pool-size', 0)) }}
innodb-buffer-pool-instances = {{ dumps(slapparameter_dict.get('innodb-buffer-pool-instances', 0)) }}
innodb-log-file-size = {{ dumps(slapparameter_dict.get('innodb-log-file-size', 0)) }}
innodb-log-buffer-size = {{ dumps(slapparameter_dict.get('innodb-log-buffer-size', 0)) }}
relaxed-writes = {{ dumps(slapparameter_dict.get('relaxed-writes', False)) }}
{% if incremental_backup_retention_days > -1 -%}
binlog-path = ${directory:mariadb-backup-incremental}/binlog
# XXX: binlog rotation happens along with other log's rotation
binlog-expire-days = {{ dumps(incremental_backup_retention_days) }}
server-id = ${publish-early:server-id}
{% else %}
binlog-path =
{%- endif %}
{%- for key, value in ssl_dict.items() -%}
ssl-{{ key }} = {{ value }}
{% endfor %}
[my-cnf]
< = jinja2-template-base
rendered = ${directory:etc}/mariadb.cnf
template = {{ parameter_dict['template-my-cnf'] }}
context = section parameter_dict my-cnf-parameters
[init-script-parameters]
database-list = {{ dumps(database_list + test_database_list) }}
[init-script]
< = jinja2-template-executable
# XXX: is there a better location ?
rendered = ${directory:etc}/mariadb_initial_setup.sql
template = {{ parameter_dict['template-mariadb-initial-setup'] }}
context = section parameter_dict init-script-parameters
[{{ section('update-mysql') }}]
recipe = slapos.cookbook:generic.mysql.wrap_update_mysql
output = ${directory:services}/mariadb_update
binary = ${binary-wrap-mysql_upgrade:wrapper-path}
mysql = ${binary-wrap-mysql:wrapper-path}
init-script = ${init-script:rendered}
mysql_tzinfo_to_sql = ${binary-wrap-mysql_tzinfo_to_sql:wrapper-path}
[{{ section('mysqld') }}]
< = jinja2-template-executable
# Note: all rendering is done when this file is rendered, not when the mysqld
# section is installed - so I only use jinja2 as a fancy way to write an
# executable file with partition-dependent but instance-parameters independent
# content.
template = inline:#!{{ dash }}
'{{ parameter_dict['mariadb-location'] }}/scripts/mysql_install_db' \
--defaults-file='${my-cnf:rendered}' \
--skip-name-resolve \
--datadir='${my-cnf-parameters:data-directory}' \
--basedir='{{ parameter_dict['mariadb-location'] }}' \
&& exec '{{ parameter_dict['mariadb-location'] }}/bin/mysqld' \
--defaults-file='${my-cnf:rendered}' \
"$@"
rendered = ${directory:services}/mariadb
[{{ section('logrotate-entry-mariadb') }}]
< = logrotate-entry-base
name = mariadb
log = ${my-cnf-parameters:error-log} ${my-cnf-parameters:slow-query-log}
post = "${binary-wrap-mysql:wrapper-path}" -B -u root -e "FLUSH LOGS"
[{{ section('binary-link') }}]
recipe = slapos.cookbook:symbolic.link
target-directory = ${directory:bin}
link-binary = {{ dumps(parameter_dict['link-binary']) }}
[{{ section("binary-link-mysqlbinlog") }}]
< = binary-link
link-binary = {{ parameter_dict['mariadb-location'] }}/bin/mysqlbinlog
[binary-wrap-base]
recipe = slapos.cookbook:wrapper
# Note: --defaults-file must be the first argument, otherwise wrapped binary
# will reject it.
command-line = "{{ parameter_dict['mariadb-location'] }}/bin/${:command}" --defaults-file="${my-cnf:rendered}"
wrapper-path = ${directory:bin}/${:command}
parameters-extra = true
[binary-wrap-mysql]
< = binary-wrap-base
command = mysql
[binary-wrap-mysqldump]
< = binary-wrap-base
command = mysqldump
[binary-wrap-mysql_upgrade]
< = binary-wrap-base
command = mysql_upgrade
[binary-wrap-mysql_tzinfo_to_sql]
< = binary-wrap-base
command-line = "{{ parameter_dict['mariadb-location'] }}/bin/${:command}"
command = mysql_tzinfo_to_sql
[binary-wrap-pt-digest]
<= binary-wrap-base
command-line = "{{ parameter_dict['percona-tools-location'] }}/bin/${:command}"
command = pt-query-digest
[directory]
recipe = slapos.cookbook:mkdirectory
bin = ${buildout:directory}/bin
etc = ${buildout:directory}/etc
services = ${:etc}/run
promise = ${:etc}/promise
srv = ${buildout:directory}/srv
tmp = ${buildout:directory}/tmp
backup = ${:srv}/backup
mariadb-backup-full = ${:backup}/mariadb-full
mariadb-backup-incremental = ${:backup}/mariadb-incremental
mariadb-data = ${:srv}/mariadb
mariadb-ssl = ${:etc}/mariadb-ssl
var = ${buildout:directory}/var
log = ${:var}/log
run = ${:var}/run
[{{ section('resiliency-exclude-file') }}]
# Generate rdiff exclude file in case of resiliency
< = jinja2-template-base
template = {{ 'inline:{{ "${directory:mariadb-data}/**\\n" }}' }}
rendered = ${directory:srv}/exporter.exclude
[dash]
dash = {{ dumps(dash) }}
[resiliency-after-import-script]
# Generate after import script used by importer instance of webrunner
< = jinja2-template-executable
template = {{ parameter_dict['mariadb-resiliency-after-import-script'] }}
rendered = ${directory:srv}/runner-import-restore
context =
key dash dash:dash
key mysql_executable binary-wrap-mysql:wrapper-path
key mariadb_data_directory directory:mariadb-data
key mariadb_backup_directory directory:mariadb-backup-full
key instance_directory buildout:directory
key pid_file my-cnf-parameters:pid-file
key binlog_path my-cnf-parameters:binlog-path
key server_executable mysqld:rendered
key update_executable update-mysql:output
[{{ section('monitor-generate-mariadb-slow-query-report') }}]
recipe = slapos.recipe.template:jinja2
template = {{ parameter_dict['mariadb-slow-query-report-script'] }}
rendered = ${monitor-directory:reports}/${:filename}
filename = mariadb_slow_query_every_23_hour
mode = 755
context =
raw slow_query_path ${directory:srv}/backup/logrotate/mariadb_slowquery.log
raw pt_query_exec ${binary-wrap-pt-digest:wrapper-path}
raw dash {{ parameter_dict['dash-location'] }}/bin/dash
key output_folder monitor-directory:private
[slow-query-digest-parameters]
max_queries_threshold = {{ slapparameter_dict['max-slowqueries-threshold'] }}
slowest_queries_threshold = {{ slapparameter_dict['slowest-query-threshold'] }}
[{{ section('monitor-promise-slowquery-result') }}]
recipe = slapos.cookbook:wrapper
recipe =
wrapper-path = ${directory:promise}/check-slow-query-pt-digest-result
command-line = "{{ parameter_dict['promise-check-slow-queries-digest-result'] }}" --ptdigest_file "${monitor-directory:private}/slowquery_digest.txt" --status_file ${monitor-directory:private}/mariadb_slow_query.report.json --max_queries_threshold "${slow-query-digest-parameters:max_queries_threshold}" --slowest_query_threshold "${slow-query-digest-parameters:slowest_queries_threshold}"
[{{ section('promise') }}]
recipe = slapos.cookbook:wrapper
command-line = "{{ parameter_dict['bin-directory'] }}/is-local-tcp-port-opened" "${my-cnf-parameters:ip}" "${my-cnf-parameters:port}"
wrapper-path = ${directory:promise}/mariadb
parameters-extra = true
[monitor-instance-parameter]
monitor-httpd-ipv6 = {{ (ipv6_set | list)[0] }}
monitor-httpd-port = {{ port + 1 }}
monitor-title = {{ slapparameter_dict['name'] }}
password = {{ slapparameter_dict['monitor-passwd'] }}
[buildout]
extends =
{{ logrotate_cfg }}
{{ parameter_dict['template-monitor'] }}
parts +=
{{ part_list | join('\n ') }}