repman: update configs to use more tags by default, fix publish and parameter json schema

software/repman/buildout.hash.cfg

@@ -14,11 +14,11 @@
 # not need these here).
 [instance.cfg]
 filename = instance.cfg.in
-md5sum = da50540b1c0fc69ffbf8f6e345a3baad
+md5sum = b41f521b5f7980c64260ed0e5c494450
 
 [instance-repman.cfg]
 _update_hash_filename_ = instance-repman.cfg.jinja2.in
-md5sum = 2b5552d587e7735b4ff6daec24fcb8c1
+md5sum = 4ddb88852c1bb89d64ecca2f2a7ada0a
 
 [config-toml.in]
 _update_hash_filename_ = templates/config.toml.in
@@ -34,15 +34,15 @@ md5sum = 0eeb24c6aa0760f0d33c4cc2828ddf30
 
 [template-mariadb.cfg]
 _update_hash_filename_ = instance-mariadb.cfg.jinja2.in
-md5sum = 63af2f4fc184cfc9d09f8b72dee5165a
+md5sum = 44c4b22c37adc6b7add3581aee7bbadf
 
 [template-my-cnf]
 _update_hash_filename_ = templates/my.cnf.in
-md5sum = 04a12d3271857fcf29cb010fd7e38a5c
+md5sum = f3661b788099bb31d71ba6e7d36836d9
 
 [template-mariadb-initial-setup]
 _update_hash_filename_ = templates/mariadb_initial_setup.sql.in
-md5sum = c31618ef115ecd72b30ae59d0440a3ec
+md5sum = 9be53e2e92333b93e92556b8a01d9c42
 
 [mariadb-init-root-sql]
 _update_hash_filename_ = templates/mariadb_init_root.sql.in
@@ -54,11 +54,11 @@ md5sum = 3ba02b99c3fcbc24b3659917980b310f
 
 [repman-manager-sh.in]
 _update_hash_filename_ = templates/repman-manager.sh.in
-md5sum = 99324b56192003254081ef336dcee94c
+md5sum = 96fcd571ebf5546957a9a8677a87c129
 
 [dbjobs-in]
 _update_hash_filename_ = templates/dbjobs.in
-md5sum = 9731eff8918fa3e68198de6b1d45ecfa
+md5sum = f273ed3cb2173f36ff8230b04424d4e3
 
 [mysqld-need-start.sh.in]
 _update_hash_filename_ = templates/mysqld-need-start.sh.in

software/repman/instance-mariadb.cfg.jinja2.in

@@ -10,7 +10,7 @@
 {%   set ip = (ipv4_set | list)[0] -%}
 {% endif -%}
 {% set dash = parameter_dict['dash-location'] ~ '/bin/dash' %}
-{% set database_list = slapparameter_dict.get('database-list', [{'name': 'repdb', 'user': 'user', 'password': 'insecure'}]) -%}
+{% set database_list = slapparameter_dict['database-list'] -%}
 
 # XXX- TODO: add cron with check db need restart then restart
 # API check restard needed: http://repman/api/clusters/{clusterName}/servers/{serverName}/{serverPort}/need-restart
@@ -26,17 +26,8 @@ receiver-port = ${dbjob-parameter:socat-port}
 [publish-early]
 recipe = slapos.cookbook:publish-early
 -init =
-  server-id gen-server-id:value
-{%- set server_id = slapparameter_dict.get('server-id') %}
-{%- if server_id %}
-server-id = {{ dumps(server_id) }}
-{%- endif %}
-database-list = {{ dumps(database_list) }}
 
-[gen-server-id]
-recipe = slapos.cookbook:random.integer
-minimum = {{ dumps(1) }}
-maximum = {{ dumps(2**32 - 1) }}
+database-list = {{ dumps(database_list) }}
 
 [jinja2-template-base]
 recipe = slapos.recipe.template:jinja2
@@ -67,14 +58,14 @@ socket = ${directory:run}/mariadb.sock
 ip = {{ ip }}
 data-directory = ${directory:srv}/mariadb
 pid-file = ${directory:run}/mariadb.pid
-max-connection-count = {{ dumps(slapparameter_dict.get('max-connection-count', 1000)) }}
+plugin-directory = {{ dumps(parameter_dict['mroonga-mariadb-plugin-dir']) }}
+groonga-plugins-path = {{ parameter_dict['groonga-plugins-path'] }}
 innodb-buffer-pool-size = {{ dumps(slapparameter_dict.get('innodb-buffer-pool-size', 0)) }}
 innodb-buffer-pool-instances = {{ dumps(slapparameter_dict.get('innodb-buffer-pool-instances', 0)) }}
 innodb-log-file-size = {{ dumps(slapparameter_dict.get('innodb-log-file-size', 0)) }}
 innodb-file-per-table = {{ dumps(slapparameter_dict.get('innodb-file-per-table', 0)) }}
 innodb-log-buffer-size = {{ dumps(slapparameter_dict.get('innodb-log-buffer-size', 0)) }}
 relaxed-writes = {{ dumps(slapparameter_dict.get('relaxed-writes', False)) }}
-server-id = ${publish-early:server-id}
 ssl-crt = ${directory:mariadb-ssl}/crt.pem
 ssl-key = ${directory:mariadb-ssl}/key.pem
 ssl-ca-crt = ${certificate-authority:ca-dir}/cacert.pem
@@ -95,6 +86,7 @@ mode = 600
 [init-script-parameters]
 password = {{ slapparameter_dict['root-password'] }}
 database-list = {{ dumps(database_list) }}
+mroonga-mariadb-install-sql = {{ dumps(parameter_dict['mroonga-mariadb-install-sql']) }}
 root-user = repman
 
 [init-script]
@@ -136,6 +128,7 @@ context =
   key datadir my-cnf-parameters:data-directory
   key environ :environ
 environ =
+  GRN_PLUGINS_PATH='${my-cnf-parameters:groonga-plugins-path}'
   ODBCSYSINI='${directory:etc}'
   LD_LIBRARY_PATH=$${LD_LIBRARY_PATH:+$LD_LIBRARY_PATH:}'{{ parameter_dict['unixodbc-location'] }}/lib'
   {%- for variable in slapparameter_dict.get('environment-variables', ()) %}
@@ -170,12 +163,11 @@ template = inline:{% raw -%}
   {%- endraw %}
 context = section parameter_dict odbc-ini-text
 
-# XXX - probably not needed anymore, to check
-#[logrotate-entry-mariadb]
-#< = logrotate-entry-base
-#name = mariadb
-#log = ${my-cnf-parameters:error-log} ${my-cnf-parameters:slow-query-log}
-#post = "${binary-wrap-mysql:wrapper-path}" -B -e "FLUSH LOGS"
+[{{ section('logrotate-entry-mariadb') }}]
+< = logrotate-entry-base
+name = mariadb
+log = ${dbjob-parameter:log-dir}/errors.log ${dbjob-parameter:log-dir}/sql-errors
+post = "${binary-wrap-mysql:wrapper-path}" -B -e "FLUSH LOGS"
 
 [{{ section('binary-link') }}]
 recipe = slapos.cookbook:symbolic.link

software/repman/instance-repman-input-schema.json

@@ -87,7 +87,7 @@
 			"title": "Provisioning db tags",
 			"description": "Provisioning db tags",
 			"type": "string",
-			"default": "gtidstrict,bind,pkg,innodb,noquerycache,slow,pfs,linux,readonly,diskmonitor,sqlerror,compressbinlog"
+			"default": "gtidstrict,bind,pkg,innodb,noquerycache,slow,pfs,linux,readonly,diskmonitor,sqlerror,compressbinlog,bm4ci,mroonga,utctime,readcommitted,nohandshake"
 		},
 		"http-session-lifetime": {
 			"title": "Web Session life time in s",
@@ -97,7 +97,7 @@
 		},
 		"http-refresh-interval": {
 			"title": "Web refresh interval in s",
-			"description": "Web interface refresh interval in ms",
+			"description": "Web interface refresh interval in s. Default 4s.",
 			"type": "integer",
 			"default": 4
 		},

software/repman/instance-repman-output-schema.json

@@ -5,21 +5,18 @@
       "title": "Backend URL",
       "description": "URL used to connect directly to backend without frontend. Requires IPv6.",
       "type": "string",
-      "format": "uri",
-      "require": true
+      "format": "uri"
     },
     "url": {
       "title": "URL",
       "description": "URL used to connect to the service.",
       "type": "string",
-      "format": "uri",
-      "require": false
+      "format": "uri"
     },
     "repman-password": {
       "title": "Repman password",
       "description": "Password for Replication Manager service.",
-      "type": "string",
-      "require": false
+      "type": "string"
     }
   }
 }

software/repman/instance-repman.cfg.jinja2.in

@@ -8,7 +8,7 @@
 {% set ip = (ipv6_set | list)[0] -%}
 {% set ipv4 = (ipv4_set | list)[0] -%}
 {% set cluster_list = [] -%}
-{% set tags = "gtidstrict,bind,pkg,innodb,noquerycache,slow,pfs,linux,readonly,diskmonitor,sqlerror,compressbinlog" -%}
+{% set tags = "gtidstrict,bind,pkg,innodb,noquerycache,slow,pfs,linux,readonly,diskmonitor,sqlerror,compressbinlog,bm4ci,mroonga,utctime,readcommitted,nohandshake" -%}
 {% set frontend_parameter_dict = slapparameter_dict.get('slave-frontend', {}) -%}
 
 [directory]
@@ -67,6 +67,7 @@ mode = 755
 {% do mariadb_dict.__setitem__('innodb-file-per-table', parameter_dict.get('innodb-file-per-table', 1)) -%}
 {% do mariadb_dict.__setitem__('use-ipv6', parameter_dict.get('use-ipv6', True)) -%}
 
+{% set database_list = parameter_dict.get('database-list', [{'name': 'repdb', 'user': 'user', 'password': 'insecure'}]) -%}
 # Request mariadb instances
 {% set db_amount = parameter_dict.get('database-amount', 2) -%}
 {% if db_amount < 2 -%}
@@ -94,6 +95,7 @@ config-repman-url = ${nginx-parameter:backend-url}
 config-repman-secure-url = ${nginx-parameter:backend-ssl-url}
 config-cluster = {{ name }}
 config-name = {{ dbname }}
+config-database-list = {{ dumps(database_list) }}
 return = 
   database-host
   receiver-port
@@ -220,13 +222,26 @@ config-port = {{ '${' ~ name ~ '-cluster-parameter:proxy-port}' }}
 
 {% set service_name = "proxysql-" ~ name -%}
 {% set proxysql_dict = {"name": service_name, "command": "${" ~ service_name ~ "-wrapper:wrapper-path}",
-  "stopwaitsecs": 60, "environment": []} %}
+  "stopwaitsecs": 60, "environment": [],
+  "stdout_logfile": "${repman:proxies-log}/" ~ service_name ~ ".log",
+  "stderr_logfile": "${repman:proxies-log}/" ~ service_name ~ ".log" } %}
 
 {{ supervisord_lib.supervisord_program(proxysql_controller, service_name, proxysql_dict) }}
 {% do part_list.append(proxysql_controller ~ "-" ~ service_name) %}
 
 {% do part_list.append('config-' ~ name) -%}
 {% do cluster_list.append("{'name': '" ~ name ~ "', 'host': '" ~ ipv4 ~ "', 'port': '${" ~ name ~ "-cluster-parameter:proxy-admin-port}'}") -%}
+
+{% set publish_database_list = [] -%}
+{% for database in database_list -%}
+{%   if database.get('user') -%}
+{%     do publish_database_list.append("mysql://" ~ database['user'] ~ ":" ~ database['password'] ~ "@" ~ ipv4 ~ ":${" ~ name ~ "-cluster-parameter:proxy-port}/" ~ database['name']) -%}
+{%   else -%}
+{%     do publish_database_list.append("mysql://" ~ ipv4 ~ ":${" ~ name ~ "-cluster-parameter:proxy-port}/" ~ database['name']) -%}
+{%   endif -%}
+{% endfor -%}
+
+{% do publish_dict.__setitem__(name ~ '-database-list', "!py!['" ~ publish_database_list | join("', '") ~ "']") -%}
 {% endfor -%}
 
 [slap-configuration]
@@ -256,6 +271,7 @@ proxies  = ${:etc}/proxy
 proxy-data = ${:data-dir}/proxy
 config-tmp = ${directory:tmp}/config
 bootstrap = ${:etc}/bootstrap
+proxies-log = ${directory:log}/proxy
 
 [nginx-parameter]
 ipv6 = ${instance-parameter:ipv6-random}

software/repman/instance.cfg.in

@@ -91,6 +91,9 @@ unixodbc-location = {{ unixodbc_location }}
 curl-location = {{ curl_location }}
 dbjobs-template = {{ dbjobs_in }}
 socat-location = {{ socat_location }}
+mroonga-mariadb-install-sql = {{ mroonga_mariadb_install_sql }}
+mroonga-mariadb-plugin-dir = {{ mroonga_mariadb_plugin_dir }}
+groonga-plugins-path = {{ groonga_plugin_dir }}:{{ groonga_mysql_normalizer_plugin_dir }}
 
 [template-instance-mariadb.cfg]
 <= jinja2-template-base

software/repman/software.cfg

@@ -2,8 +2,6 @@
 
 extends =
   buildout.hash.cfg
-  ../neoppod/software-common.cfg
-  ../../stack/monitor/buildout.cfg
   ../../component/restic/buildout.cfg
   ../../component/replication-manager/buildout.cfg
   ../../component/mariadb/buildout.cfg
@@ -21,7 +19,7 @@ extends =
   ../../component/rsync/buildout.cfg
   ../../stack/supervisord/buildout.cfg
   ../../stack/monitor/buildout.cfg
-  ../../stack/slapos.cfg
+  ../neoppod/software-common.cfg
 
 parts =
   slapos-cookbook
@@ -31,11 +29,6 @@ parts =
   template-mysqld-wrapper
   gowork
 
-
-[mariadb]
-# Compile dir is for plugins, there's no plugin yet
-keep-compile-dir = false
-
 [instance.cfg]
 recipe = slapos.recipe.template:jinja2
 rendered = ${buildout:directory}/instance.cfg
@@ -63,6 +56,10 @@ context =
     key mariadb_link_binary template-mariadb.cfg:link-binary
     key mariadb_location mariadb:location
     key mysqld_start_template mysqld-need-start.sh.in:target
+    key mroonga_mariadb_install_sql mroonga-mariadb:install-sql
+    key mroonga_mariadb_plugin_dir mroonga-mariadb:plugin-dir
+    key groonga_plugin_dir groonga:groonga-plugin-dir
+    key groonga_mysql_normalizer_plugin_dir groonga-normalizer-mysql:groonga-plugin-dir
     key nginx_conf_in nginx.conf.in:target
     key nginx_location nginx:location
     key percona_toolkit_location percona-toolkit:location
@@ -152,5 +149,5 @@ link-binary =
 
 # Pin versions of eggs used that are not already pinned by stack/slapos.cfg
 [versions]
-slapos.recipe.template = 4.3
+slapos.recipe.template = 4.4
 rubygemsrecipe  = 0.2.2+slapos001

software/repman/software.cfg.json

@@ -5,6 +5,7 @@
   "software-type": {
     "default": {
       "title": "Default",
+      "serialisation": "json-in-xml",
       "description": "Replication Manager",
       "request": "instance-repman-input-schema.json",
       "response": "instance-repman-output-schema.json",

software/repman/templates/dbjobs.in

@@ -58,7 +58,6 @@ partialRestore()
 }
 
 kill -9 $(lsof -t -i:{{ parameter_dict['socat-port'] }} -sTCP:LISTEN)
-kill -9 $(lsof -t -i:{{ parameter_dict['socat-port'] }} -sTCP6:LISTEN)
 
 for job in "${JOBS[@]}"
 do
@@ -78,7 +77,7 @@ do
       reseedmysqldump)
        echo "Waiting backup." >  {{ parameter_dict['log-dir'] }}/dbjob.out
        pauseJob
-       socat -u {{ listen }}:{{ parameter_dict['socat-port'] }},bind={{ parameter_dict['host'] }},reuseaddr STDOUT | gunzip | mysql -h{{ parameter_dict['ip'] }} -P{{ parameter_dict['port'] }} -p$PASSWORD -u$USER --init-command="set sql_log_bin=0" > {{ parameter_dict['log-dir'] }}/dbjob.out 2>&1
+       socat -u {{ listen }}:{{ parameter_dict['socat-port'] }},bind={{ parameter_dict['host'] }},reuseaddr STDOUT | gunzip | mysql -h{{ parameter_dict['ip'] }} -P{{ parameter_dict['port'] }} -p$PASSWORD -u$USER --init-command="reset master;set sql_log_bin=0" > {{ parameter_dict['log-dir'] }}/dbjob.out 2>&1
         mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }}  -e 'start slave;'
       ;;
       flashbackmysqldump)
@@ -112,11 +111,9 @@ do
       ;;
       error)
        cat $ERROLOG| socat -u stdio TCP:$ADDRESS &>{{ parameter_dict['log-dir'] }}/dbjob.out
-       > $ERROLOG
       ;;
       slowquery)
        cat $SLOWLOG| socat -u stdio TCP:$ADDRESS &>{{ parameter_dict['log-dir'] }}/dbjob.out
-       > $SLOWLOG
       ;;
       optimize)
        mysqlcheck --defaults-file={{ parameter_dict['dbjob-cnf'] }} -o --all-databases --skip-write-binlog &>{{ parameter_dict['log-dir'] }}/dbjob.out

software/repman/templates/mariadb_initial_setup.sql.in

+SET @@SESSION.SQL_LOG_BIN=0;
+
 USE mysql;
-DROP FUNCTION IF EXISTS last_insert_grn_id;
-DROP FUNCTION IF EXISTS mroonga_snippet;
-DROP FUNCTION IF EXISTS mroonga_command;
 {% set mroonga = parameter_dict.get('mroonga', 'ha_mroonga.so') -%}
-{% if mroonga %}CREATE FUNCTION last_insert_grn_id RETURNS INTEGER SONAME '{{ mroonga }}';
-CREATE FUNCTION mroonga_snippet RETURNS STRING SONAME '{{ mroonga }}';
-CREATE FUNCTION mroonga_command RETURNS STRING SONAME '{{ mroonga }}';{% endif %}
+{% if mroonga %}
+SOURCE {{ parameter_dict['mroonga-mariadb-install-sql'] }};
+{% endif %}
 DROP FUNCTION IF EXISTS sphinx_snippets;
 #CREATE FUNCTION sphinx_snippets RETURNS STRING SONAME 'ha_sphinx.so';
 

software/repman/templates/my.cnf.in

@@ -11,16 +11,9 @@
 #sql_mode="NO_ENGINE_SUBSTITUTION"
 
 socket = {{ socket }}
-bind_address = {{ parameter_dict['ip'] }}
 datadir = {{ parameter_dict['data-directory'] }}
 pid_file = {{ parameter_dict['pid-file'] }}
 
-default_time_zone = '+00:00'
-
-plugin_load = ha_mroonga
-
-max_connections = {{ parameter_dict['max-connection-count'] }}
-
 {% set innodb_buffer_pool_size = parameter_dict['innodb-buffer-pool-size'] -%}
 {% if innodb_buffer_pool_size %}innodb_buffer_pool_size = {{ innodb_buffer_pool_size }}{% endif %}
 {% set innodb_buffer_pool_instances = parameter_dict['innodb-buffer-pool-instances'] -%}
@@ -33,8 +26,10 @@ max_connections = {{ parameter_dict['max-connection-count'] }}
 # very important to allow parallel indexing
 # Note: this is compatible with binlog-based incremental backups, because ERP5
 # doesn't use "insert ... select" (in any number of queries) pattern.
-innodb_locks_unsafe_for_binlog = 1
+# innodb_locks_unsafe_for_binlog = 1
 
+#plugin_load = ha_mroonga
+plugin-dir = {{ parameter_dict['plugin-directory'] }}
 
 {% if 'ssl-key' in parameter_dict -%}
 ssl
@@ -51,8 +46,6 @@ ssl-cipher = {{ parameter_dict['ssl-cipher'] }}
 {%-  endif %}
 {%- endif %}
 
-#server_id = {{ parameter_dict['server-id'] }}
-
 
 # Some dangerous settings you may want to uncomment temporarily
 # if you only want performance or less disk access.
@@ -62,10 +55,7 @@ ssl-cipher = {{ parameter_dict['ssl-cipher'] }}
 {{x}}innodb_doublewrite = 0
 {{x}}sync_frm = 0
 
-# Force utf8 usage
-character_set_server = utf8mb4
-collation_server = utf8mb4_general_ci
-skip_character_set_client_handshake
+# skip_character_set_client_handshake
 
 
 [client]

software/repman/templates/repman-manager.sh.in

@@ -8,7 +8,7 @@ curl () {
 
 wait_database () {
   NAME=$1
-  for retry in {1..5}; do
+  for retry in {1..10}; do
     echo ">> Wait until $NAME databases are ready...";
     CODE=$(curl -H "Authorization: Bearer ${TOKEN}" -o /dev/null -w "%{http_code}" {{ secure_url }}/api/clusters/$NAME/actions/waitdatabases);
     if [ $CODE -eq 504 ]; then