Commit 0eea2229 authored by Jérome Perrin's avatar Jérome Perrin

Mariadb in ERP5 SR test

Test Mariadb in ERP5 SR test and a few cleanups, mainly around mariadb:
 - We need a more "strict" promise than just checking if something is listening on the port
 - We don't need to wait so long between setup retries
 - We can use mroonga provided setup script instead of writing the setup ourselves.

See merge request !722
parents b7542e3f 2de4c80f
......@@ -104,6 +104,7 @@ configure-options =
--with-mysql-config=${mariadb:location}/bin/mysql_config
--disable-static
--disable-document
install-sql = @@LOCATION@@/share/mroonga/install.sql
patch-options = -p1
patches =
${:_profile_base_location_}/mroonga_boolean.patch#c818568fe35ca6a4298f18e575d962a0
......
......@@ -5,7 +5,7 @@ import sys
import pytz
def updateMysql(mysql_upgrade_binary, mysql_binary, mysql_script_file):
sleep = 30
sleep = 0
with open(mysql_script_file) as script_file:
mysql_script = script_file.read()
mysql_list = mysql_binary, '-B'
......@@ -43,6 +43,7 @@ def updateMysql(mysql_upgrade_binary, mysql_binary, mysql_script_file):
break
print 'SlapOS initialisation script succesfully applied on database.'
return
sleep = max(sleep+1, 30)
print 'Sleeping for %ss and retrying' % sleep
sys.stdout.flush()
sys.stderr.flush()
......
[buildout]
extends = software.cfg
shared-parts = /opt/slapgrid/shared-parts
eggs-directory = /opt/slapgrid/shared-eggs
abi-tag-eggs = true
......@@ -47,6 +47,7 @@ setup(name=name,
'erp5.util',
'psutil',
'requests',
'mysqlclient',
],
zip_safe=True,
test_suite='test',
......
##############################################################################
# coding: utf-8
#
# Copyright (c) 2018 Nexedi SA and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
import os
import json
import glob
import urlparse
import socket
import time
import contextlib
import datetime
import MySQLdb
from . import ERP5InstanceTestCase
from . import setUpModule
setUpModule # pyflakes
class MariaDBTestCase(ERP5InstanceTestCase):
"""Base test case for mariadb tests.
"""
__partition_reference__ = 'm'
# XXX mariadb_update service sleeps for 30 seconds before retrying,
# so this takes a lot of time to instance
instance_max_retry = 60
@classmethod
def getInstanceSoftwareType(cls):
return "mariadb"
@classmethod
def _getInstanceParameterDict(cls):
return {
'tcpv4-port': 3306,
'max-connection-count': 5,
'max-slowqueries-threshold': 5,
'slowest-query-threshold': 10,
# XXX what is this ? should probably not be needed here
'name': cls.__name__,
'monitor-passwd': 'secret',
# XXX should probably not be needed here
'computer-memory-percent-threshold': 100,
}
@classmethod
def getInstanceParameterDict(cls):
return {'_': json.dumps(cls._getInstanceParameterDict())}
def getDatabaseConnection(self):
connection_parameter_dict = json.loads(
self.computer_partition.getConnectionParameterDict()['_'])
db_url = urlparse.urlparse(connection_parameter_dict['database-list'][0])
self.assertEqual('mysql', db_url.scheme)
self.assertTrue(db_url.path.startswith('/'))
database_name = db_url.path[1:]
return MySQLdb.connect(
user=db_url.username,
passwd=db_url.password,
host=db_url.hostname,
port=db_url.port,
db=database_name,
)
class TestMariaDB(MariaDBTestCase):
def test_utf8_collation(self):
cnx = self.getDatabaseConnection()
with contextlib.closing(cnx):
cnx.query(
"""
CREATE TABLE test_utf8_collation (
col1 CHAR(10)
)
""")
cnx.store_result()
cnx.query(
"""
insert into test_utf8_collation values ("à"), ("あ")
""")
cnx.store_result()
cnx.query(
"""
select * from test_utf8_collation where col1 = "a"
""")
self.assertEqual((('à',),), cnx.store_result().fetch_row(maxrows=2))
class TestMroonga(MariaDBTestCase):
def test_mroonga_plugin_loaded(self):
cnx = self.getDatabaseConnection()
with contextlib.closing(cnx):
cnx.query("show plugins")
plugins = cnx.store_result().fetch_row(maxrows=1000)
self.assertIn(
('Mroonga', 'ACTIVE', 'STORAGE ENGINE', 'ha_mroonga.so', 'GPL'),
plugins)
def test_mroonga_normalize_udf(self):
# example from https://mroonga.org/docs/reference/udf/mroonga_normalize.html#usage
cnx = self.getDatabaseConnection()
with contextlib.closing(cnx):
cnx.query(
"""
SELECT mroonga_normalize("ABCDあぃうぇ㍑")
""")
self.assertEqual((('abcdあぃうぇリットル',),),
cnx.store_result().fetch_row(maxrows=2))
if 0:
# this example fail with:
# OperationalError: (1123, "Can't initialize function 'mroonga_normalize'; mroonga_normalize(): nonexistent normalizer NormalizerMySQLUnicodeCIExceptKanaCI")
# same error on mroonga "official" docker images using mysql
# https://hub.docker.com/layers/groonga/mroonga/latest/images/sha256-e5a979801c95544ca3a1228d2c4d819820850e0162649553f2e94850e5e1c988?context=explore
# so it's probably OK to ignore
cnx.query(
"""
SELECT mroonga_normalize("aBcDあぃウェ㍑", "NormalizerMySQLUnicodeCIExceptKanaCIKanaWithVoicedSoundMark")
""")
self.assertEqual((('ABCDあぃうぇ㍑',),),
cnx.store_result().fetch_row(maxrows=2))
def test_mroonga_full_text_normalizer(self):
# example from https://mroonga.org//docs/tutorial/storage.html#how-to-specify-the-normalizer
cnx = self.getDatabaseConnection()
with contextlib.closing(cnx):
cnx.query("SET NAMES utf8")
cnx.store_result()
cnx.query(
"""
CREATE TABLE diaries (
day DATE PRIMARY KEY,
content VARCHAR(64) NOT NULL,
FULLTEXT INDEX (content) COMMENT 'normalizer "NormalizerAuto"'
) Engine=Mroonga DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci
""")
cnx.store_result()
cnx.query(
"""INSERT INTO diaries VALUES ("2013-04-23", "ブラックコーヒーを飲んだ。")""")
cnx.store_result()
cnx.query(
"""
SELECT *
FROM diaries
WHERE MATCH (content) AGAINST ("+ふらつく" IN BOOLEAN MODE)
""")
self.assertEqual((), cnx.store_result().fetch_row(maxrows=2))
cnx.query(
"""
SELECT *
FROM diaries
WHERE MATCH (content) AGAINST ("+ブラック" IN BOOLEAN MODE)
""")
self.assertEqual(
((datetime.date(2013, 4, 23), 'ブラックコーヒーを飲んだ。'),),
cnx.store_result().fetch_row(maxrows=2),
)
def test_mroonga_full_text_stem(self):
# example from https://mroonga.org//docs/tutorial/storage.html#how-to-specify-the-token-filters
cnx = self.getDatabaseConnection()
with contextlib.closing(cnx):
cnx.query("SELECT mroonga_command('register token_filters/stem')")
self.assertEqual((('true',),), cnx.store_result().fetch_row(maxrows=2))
cnx.query(
"""
CREATE TABLE memos (
id INT NOT NULL PRIMARY KEY,
content TEXT NOT NULL,
FULLTEXT INDEX (content) COMMENT 'normalizer "NormalizerAuto", token_filters "TokenFilterStem"'
) Engine=Mroonga DEFAULT CHARSET=utf8
""")
cnx.store_result()
cnx.query(
"""INSERT INTO memos VALUES (1, "I develop Groonga"), (2, "I'm developing Groonga"), (3, "I developed Groonga")"""
)
cnx.store_result()
cnx.query(
"""
SELECT *
FROM memos
WHERE MATCH (content) AGAINST ("+develops" IN BOOLEAN MODE)
""")
self.assertEqual([
(1, "I develop Groonga"),
(2, "I'm developing Groonga"),
(3, "I developed Groonga"),
], list(sorted(cnx.store_result().fetch_row(maxrows=4))))
......@@ -6,6 +6,7 @@ extends =
../../component/git/buildout.cfg
../../component/pillow/buildout.cfg
../../component/python-cryptography/buildout.cfg
../../component/python-mysqlclient/buildout.cfg
../../component/python-pynacl/buildout.cfg
../../component/quic_client-bin/buildout.cfg
../../component/python-backports-lzma/buildout.cfg
......@@ -156,6 +157,7 @@ eggs =
erp5.util
${python-pynacl:egg}
${python-cryptography:egg}
${python-mysqlclient:egg}
${bcrypt:egg}
slapos.libnetworkcache
supervisor
......@@ -282,3 +284,4 @@ mock = 2.0.0
testfixtures = 6.11
funcsigs = 1.0.2
PyPDF2 = 1.26.0
mysqlclient = 1.3.12
\ No newline at end of file
......@@ -203,6 +203,7 @@ context =
key mariadb_resiliency_after_import_script mariadb-resiliency-after-import-script:target
key mariadb_slow_query_report_script mariadb-slow-query-report-script:target
key mariadb_start_clone_from_backup mariadb-start-clone-from-backup:target
key mroonga_mariadb_install_sql mroonga-mariadb:install-sql
key matplotlibrc_location matplotlibrc:location
key parts_directory buildout:parts-directory
key openssl_location openssl:location
......
......@@ -26,7 +26,7 @@ md5sum = d10b8e35b02b5391cf46bf0c7dbb1196
[template-mariadb]
filename = instance-mariadb.cfg.in
md5sum = 1c332e1a3388258572fcc6528aca5725
md5sum = 63bac9fc58537e55a6c8c42d0be54fbe
[template-kumofs]
filename = instance-kumofs.cfg.in
......@@ -46,7 +46,7 @@ md5sum = 771bfd921aa5e59e1b4712ba113fa84b
[template-mariadb-initial-setup]
filename = mariadb_initial_setup.sql.in
md5sum = dec33a617fa1b307c8ddb883efcfe3ce
md5sum = 1102c3e37a5a2e8aa2d8a2607ab633c8
[template-postfix]
filename = instance-postfix.cfg.in
......@@ -70,7 +70,7 @@ md5sum = cc19560b9400cecbd23064d55c501eec
[template]
filename = instance.cfg.in
md5sum = 520b6bf3461dddc9c8b862e50b14465d
md5sum = 1a218a5676208dda092e6045f25bf77f
[monitor-template-dummy]
filename = dummy.cfg
......
......@@ -160,6 +160,7 @@ context = section parameter_dict my-cnf-parameters
[init-script-parameters]
database-list = {{ dumps(database_list + test_database_list) }}
mroonga-mariadb-install-sql = {{ dumps(parameter_dict['mroonga-mariadb-install-sql']) }}
[init-script]
< = jinja2-template-executable
......@@ -344,7 +345,7 @@ config-command = "{{ parameter_dict["check-computer-memory-binary"] }}" -db ${mo
<= monitor-promise-base
module = check_command_execute
name = mariadb.py
config-command = "{{ parameter_dict['bin-directory'] }}/is-local-tcp-port-opened" "${my-cnf-parameters:ip}" "${my-cnf-parameters:port}"
config-command = "${binary-wrap-mysql:wrapper-path}" --execute ';' {% if database_list and database_list[0].get('user') %} --host="${my-cnf-parameters:ip}" --port="${my-cnf-parameters:port}" --user="{{ database_list[0]['user'] }}" --password="{{ database_list[0]['password'] }}" {% endif %}
[monitor-instance-parameter]
monitor-httpd-ipv6 = {{ (ipv6_set | list)[0] }}
......
......@@ -151,6 +151,7 @@ mariadb-start-clone-from-backup = {{ mariadb_start_clone_from_backup }}
promise-check-slow-queries-digest-result = {{ bin_directory }}/check-slow-queries-digest-result
percona-tools-location = {{ percona_toolkit_location }}
unixodbc-location = {{ unixodbc_location }}
mroonga-mariadb-install-sql = {{ mroonga_mariadb_install_sql }}
[dynamic-template-mariadb]
<= jinja2-template-base
......
USE mysql;
DROP FUNCTION IF EXISTS last_insert_grn_id;
DROP FUNCTION IF EXISTS mroonga_snippet;
DROP FUNCTION IF EXISTS mroonga_command;
{% set mroonga = parameter_dict.get('mroonga', 'ha_mroonga.so') -%}
{% if mroonga %}CREATE FUNCTION last_insert_grn_id RETURNS INTEGER SONAME '{{ mroonga }}';
CREATE FUNCTION mroonga_snippet RETURNS STRING SONAME '{{ mroonga }}';
CREATE FUNCTION mroonga_command RETURNS STRING SONAME '{{ mroonga }}';{% endif %}
{% if mroonga %}
SOURCE {{ parameter_dict['mroonga-mariadb-install-sql'] }};
{% endif %}
DROP FUNCTION IF EXISTS sphinx_snippets;
#CREATE FUNCTION sphinx_snippets RETURNS STRING SONAME 'ha_sphinx.so';
......
......@@ -103,6 +103,7 @@ context =
key mariadb_resiliency_after_import_script mariadb-resiliency-after-import-script:target
key mariadb_slow_query_report_script mariadb-slow-query-report-script:target
key mariadb_start_clone_from_backup mariadb-start-clone-from-backup:target
key mroonga_mariadb_install_sql mroonga-mariadb:install-sql
key percona_toolkit_location percona-toolkit:location
key template_php_ini template-php.ini:output
key template_apache_conf template-apache.conf:output
......
......@@ -14,7 +14,7 @@
# not need these here).
[instance]
filename = instance.cfg.in
md5sum = b4fbd7dc8d7fda7dbd1b80f3de273ecf
md5sum = 92a93bb3b9f6663d967b0fb57244a1fa
[instance-apache-php]
filename = instance-apache-php.cfg.in
......
......@@ -89,6 +89,7 @@ mariadb-start-clone-from-backup = {{ mariadb_start_clone_from_backup }}
promise-check-slow-queries-digest-result = {{ bin_directory }}/check-slow-queries-digest-result
percona-tools-location = {{ percona_toolkit_location }}
unixodbc-location = {{ unixodbc_location }}
mroonga-mariadb-install-sql = {{ mroonga_mariadb_install_sql }}
check-computer-memory-binary = {{ bin_directory }}/check-computer-memory
bin-directory = {{ bin_directory }}
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment