{% set part_list = [] -%} {% macro section(name) %}{% do part_list.append(name) %}{{ name }}{% endmacro -%} {% set use_ipv6 = slapparameter_dict.get('use-ipv6', False) -%} {% set database_list = slapparameter_dict.get('database-list', [{'name': 'erp5', 'user': 'user', 'password': 'insecure'}]) -%} {% set test_database_list = [] %} {% for database_count in range(slapparameter_dict.get('test-database-amount', 1)) -%} {% do test_database_list.append({'name': 'erp5_test_' ~ database_count, 'user': 'testuser_' ~ database_count, 'password': 'testpassword' ~ database_count}) -%} {% endfor -%} {% set catalog_backup = slapparameter_dict.get('catalog-backup', {}) -%} {% set backup_periodicity = slapparameter_dict.get('backup-periodicity', 'daily') -%} {% set full_backup_retention_days = catalog_backup.get('full-retention-days', 7) -%} {% set incremental_backup_retention_days = catalog_backup.get('incremental-retention-days', full_backup_retention_days) -%} {% set port = slapparameter_dict['tcpv4-port'] %} {% if use_ipv6 -%} {% set ip = (ipv6_set | list)[0] -%} {% else -%} {% set ip = (ipv4_set | list)[0] -%} {% endif -%} {% set dash = parameter_dict['dash-location'] ~ '/bin/dash' %} [{{ section('publish') }}] recipe = slapos.cookbook:publish.serialised -extends = publish-early {% macro render_database_list(database_list) -%} {% set publish_database_list = [] -%} {% for database in database_list -%} {% if database.get('user') -%} {% do publish_database_list.append("mysql://" ~ database['user'] ~ ":" ~ database['password'] ~ "@" ~ ip ~ ":" ~ port ~ "/" ~ database['name']) -%} {% else -%} {% do publish_database_list.append("mysql://" ~ ip ~ ":" ~ port ~ "/" ~ database['name']) -%} {% endif -%} {% endfor -%} {{ dumps(publish_database_list) }} {% endmacro -%} database-list = {{ render_database_list(database_list) }} test-database-list = {{ render_database_list(test_database_list) }} monitor-base-url = ${monitor-publish-parameters:monitor-base-url} [publish-early] recipe = slapos.cookbook:publish-early -init = server-id gen-server-id:value {%- set server_id = slapparameter_dict.get('server-id') %} {%- if server_id %} server-id = {{ dumps(server_id) }} {%- endif %} [gen-server-id] recipe = slapos.cookbook:random.integer minimum = {{ dumps(1) }} maximum = {{ dumps(2**32 - 1) }} [jinja2-template-base] recipe = slapos.recipe.template:jinja2 mode = 644 [jinja2-template-executable] < = jinja2-template-base mode = 755 [simplefile] < = jinja2-template-base template = inline:{{ '{{ content }}' }} {% macro simplefile(section_name, file_path, content, mode='') -%} {% set content_section_name = section_name ~ '-content' -%} [{{ content_section_name }}] content = {{ dumps(content) }} [{{ section(section_name) }}] < = simplefile rendered = {{ file_path }} context = key content {{content_section_name}}:content mode = {{ mode }} {%- endmacro %} {% set ssl_dict = {} -%} {% macro sslfile(key, content, mode='644') -%} {% set path = '${directory:mariadb-ssl}/' ~ key ~ '.pem' -%} {% do ssl_dict.__setitem__(key, path) -%} {{ simplefile('ssl-file-' ~ key, path, content, mode) }} {%- endmacro %} {% set ssl_parameter_dict = slapparameter_dict.get('ssl') -%} {% if ssl_parameter_dict -%} {% set base_directory = '${directory:mariadb-ssl}/' -%} {# Note: The key content will be stored in .installed.cfg, and this template's rendering, so the only point of mode is to avoid risking mariadb complaining about laxist file mode. -#} {{ sslfile('key', ssl_parameter_dict['key'], mode='600') }} {{ sslfile('crt', ssl_parameter_dict['crt']) }} {% if 'ca-crt' in ssl_parameter_dict -%} {{ sslfile('ca-crt', ssl_parameter_dict['ca-crt']) }} {% endif -%} {% if 'crl' in ssl_parameter_dict -%} {{ sslfile('crl', ssl_parameter_dict['crl']) }} {% endif -%} {%- endif %} {% if full_backup_retention_days > -1 -%} [{{ section('cron-entry-mariadb-backup') }}] recipe = slapos.cookbook:cron.d cron-entries = ${cron:cron-entries} name = mariadb-backup time = {{ dumps(backup_periodicity) }} {# When binlogs are enabled: # flush-logs: used so no manipulation on binlogs is needed to restore from # full + binlogs. The first binlog after a dump starts from dump snapshot and # can be fully restored. # master-data: use value "2" as we are not in a replication case #} command = "${binary-wrap-mysqldump:wrapper-path}" -u root --all-databases --single-transaction {% if incremental_backup_retention_days > -1 %}--flush-logs --master-data=2 {% endif %}| {{ parameter_dict['gzip-location'] }}/bin/gzip > "${directory:mariadb-backup-full}/$({{ parameter_dict['coreutils-location'] }}/bin/date "+%Y%m%d%H%M%S").sql.gz" {# KEEP GLOB PATTERN IN SYNC with generated filenames above # YYYYmmddHHMMSS -#} file-glob = ??????????????.sql.gz {% if full_backup_retention_days > 0 -%} [{{ section("cron-entry-mariadb-backup-expire") }}] recipe = slapos.cookbook:cron.d cron-entries = ${cron:cron-entries} name = mariadb-backup-expire time = {{ dumps(backup_periodicity) }} command = {{ parameter_dict['findutils-location'] }}/bin/find "${directory:mariadb-backup-full}" -maxdepth 1 -name "${cron-entry-mariadb-backup:file-glob}" -daystart -mtime +{{ full_backup_retention_days }} -delete {%- endif %} {%- endif %} [my-cnf-parameters] ip = {{ ip }} port = {{ port }} socket = ${directory:run}/mariadb.sock data-directory = ${directory:mariadb-data} tmp-directory = ${directory:tmp} pid-file = ${directory:run}/mariadb.pid error-log = ${directory:log}/mariadb_error.log slow-query-log = ${directory:log}/mariadb_slowquery.log long-query-time = {{ dumps(slapparameter_dict.get('long-query-time', 1)) }} innodb-buffer-pool-size = {{ dumps(slapparameter_dict.get('innodb-buffer-pool-size', 0)) }} innodb-buffer-pool-instances = {{ dumps(slapparameter_dict.get('innodb-buffer-pool-instances', 0)) }} innodb-log-file-size = {{ dumps(slapparameter_dict.get('innodb-log-file-size', 0)) }} innodb-log-buffer-size = {{ dumps(slapparameter_dict.get('innodb-log-buffer-size', 0)) }} relaxed-writes = {{ dumps(slapparameter_dict.get('relaxed-writes', False)) }} {% if incremental_backup_retention_days > -1 -%} binlog-path = ${directory:mariadb-backup-incremental}/binlog # XXX: binlog rotation happens along with other log's rotation binlog-expire-days = {{ dumps(incremental_backup_retention_days) }} server-id = ${publish-early:server-id} {% else %} binlog-path = {%- endif %} {%- for key, value in ssl_dict.items() -%} ssl-{{ key }} = {{ value }} {% endfor %} [my-cnf] < = jinja2-template-base rendered = ${directory:etc}/mariadb.cnf template = {{ parameter_dict['template-my-cnf'] }} context = section parameter_dict my-cnf-parameters [init-script-parameters] database-list = {{ dumps(database_list + test_database_list) }} [init-script] < = jinja2-template-executable # XXX: is there a better location ? rendered = ${directory:etc}/mariadb_initial_setup.sql template = {{ parameter_dict['template-mariadb-initial-setup'] }} context = section parameter_dict init-script-parameters [{{ section('update-mysql') }}] recipe = slapos.cookbook:generic.mysql.wrap_update_mysql output = ${directory:services}/mariadb_update binary = ${binary-wrap-mysql_upgrade:wrapper-path} mysql = ${binary-wrap-mysql:wrapper-path} init-script = ${init-script:rendered} mysql_tzinfo_to_sql = ${binary-wrap-mysql_tzinfo_to_sql:wrapper-path} [{{ section('mysqld') }}] < = jinja2-template-executable # Note: all rendering is done when this file is rendered, not when the mysqld # section is installed - so I only use jinja2 as a fancy way to write an # executable file with partition-dependent but instance-parameters independent # content. template = inline:#!{{ dash }} '{{ parameter_dict['mariadb-location'] }}/scripts/mysql_install_db' \ --defaults-file='${my-cnf:rendered}' \ --skip-name-resolve \ --datadir='${my-cnf-parameters:data-directory}' \ --basedir='{{ parameter_dict['mariadb-location'] }}' \ && exec '{{ parameter_dict['mariadb-location'] }}/bin/mysqld' \ --defaults-file='${my-cnf:rendered}' \ "$@" rendered = ${directory:services}/mariadb [{{ section('logrotate-entry-mariadb') }}] < = logrotate-entry-base name = mariadb log = ${my-cnf-parameters:error-log} ${my-cnf-parameters:slow-query-log} post = "${binary-wrap-mysql:wrapper-path}" -B -u root -e "FLUSH LOGS" [{{ section('binary-link') }}] recipe = slapos.cookbook:symbolic.link target-directory = ${directory:bin} link-binary = {{ dumps(parameter_dict['link-binary']) }} [{{ section("binary-link-mysqlbinlog") }}] < = binary-link link-binary = {{ parameter_dict['mariadb-location'] }}/bin/mysqlbinlog [binary-wrap-base] recipe = slapos.cookbook:wrapper # Note: --defaults-file must be the first argument, otherwise wrapped binary # will reject it. command-line = "{{ parameter_dict['mariadb-location'] }}/bin/${:command}" --defaults-file="${my-cnf:rendered}" wrapper-path = ${directory:bin}/${:command} parameters-extra = true [binary-wrap-mysql] < = binary-wrap-base command = mysql [binary-wrap-mysqldump] < = binary-wrap-base command = mysqldump [binary-wrap-mysql_upgrade] < = binary-wrap-base command = mysql_upgrade [binary-wrap-mysql_tzinfo_to_sql] < = binary-wrap-base command-line = "{{ parameter_dict['mariadb-location'] }}/bin/${:command}" command = mysql_tzinfo_to_sql [binary-wrap-pt-digest] <= binary-wrap-base command-line = "{{ parameter_dict['percona-tools-location'] }}/bin/${:command}" command = pt-query-digest [directory] recipe = slapos.cookbook:mkdirectory bin = ${buildout:directory}/bin etc = ${buildout:directory}/etc services = ${:etc}/run promise = ${:etc}/promise srv = ${buildout:directory}/srv tmp = ${buildout:directory}/tmp backup = ${:srv}/backup mariadb-backup-full = ${:backup}/mariadb-full mariadb-backup-incremental = ${:backup}/mariadb-incremental mariadb-data = ${:srv}/mariadb mariadb-ssl = ${:etc}/mariadb-ssl var = ${buildout:directory}/var log = ${:var}/log run = ${:var}/run [{{ section('resiliency-exclude-file') }}] # Generate rdiff exclude file in case of resiliency < = jinja2-template-base template = {{ 'inline:{{ "${directory:mariadb-data}/**\\n" }}' }} rendered = ${directory:srv}/exporter.exclude [dash] dash = {{ dumps(dash) }} [resiliency-after-import-script] # Generate after import script used by importer instance of webrunner < = jinja2-template-executable template = {{ parameter_dict['mariadb-resiliency-after-import-script'] }} rendered = ${directory:srv}/runner-import-restore context = key dash dash:dash key mysql_executable binary-wrap-mysql:wrapper-path key mariadb_data_directory directory:mariadb-data key mariadb_backup_directory directory:mariadb-backup-full key instance_directory buildout:directory key pid_file my-cnf-parameters:pid-file key binlog_path my-cnf-parameters:binlog-path key server_executable mysqld:rendered key update_executable update-mysql:output [{{ section('monitor-generate-mariadb-slow-query-report') }}] recipe = slapos.recipe.template:jinja2 template = {{ parameter_dict['mariadb-slow-query-report-script'] }} rendered = ${monitor-directory:reports}/${:filename} output-folder = ${directory:srv}/monitor/ filename = mariadb_slow_query_every_23_hour mode = 755 context = raw slow_query_path ${directory:srv}/backup/logrotate/mariadb_slowquery.log raw pt_query_exec ${binary-wrap-pt-digest:wrapper-path} raw dash {{ parameter_dict['dash-location'] }}/bin/dash key output_folder :output-folder [{{ section('promise') }}] recipe = slapos.cookbook:wrapper command-line = "{{ parameter_dict['bin-directory'] }}/is-local-tcp-port-opened" "${my-cnf-parameters:ip}" "${my-cnf-parameters:port}" wrapper-path = ${directory:promise}/mariadb parameters-extra = true [monitor-instance-parameter] monitor-httpd-ipv6 = {{ (ipv6_set | list)[0] }} monitor-httpd-port = {{ port + 1 }} monitor-title = {{ slapparameter_dict['name'] }} password = {{ slapparameter_dict['monitor-passwd'] }} [buildout] extends = {{ logrotate_cfg }} {{ parameter_dict['template-monitor'] }} parts += {{ part_list | join('\n ') }}