...
 
Commits (8)
Changes
=======
1.0.121 (2019-09-12)
--------------------
- generic.mysql.wrap_update_mysql: prepare for MariaDB 10.4
- publish-early: process -init entries in specified order
- Partial support of Python 3
- Remove unused generic.mysql recipe
1.0.119 (2019-08-14)
--------------------
......
......@@ -29,8 +29,8 @@ parts =
[mariadb]
recipe = slapos.recipe.cmmi
url = https://downloads.mariadb.org/f/mariadb-${:version}/source/mariadb-${:version}.tar.gz/from/http%3A//fr.mirror.babylon.network/mariadb/?serve
version = 10.3.17
md5sum = 6bcee042bcc393cfcffeccbddbf6fe0e
version = 10.4.7
md5sum = a9a57d2861b981f67ecde0b177be9d69
location = ${buildout:parts-directory}/${:_buildout_section_name_}
configure-command = ${cmake:location}/bin/cmake
configure-options =
......@@ -72,28 +72,24 @@ environment =
CMAKE_INCLUDE_PATH=${bzip2:location}/include:${libaio:location}/include:${libaio:location}/include:${libxml2:location}/include:${ncurses:location}/include:${openssl:location}/include:${readline5:location}/include:${xz-utils:location}/include:${zlib:location}/include:${unixodbc:location}/include:${lz4:location}/include:${snappy:location}/include:${zstd:location}/include
CMAKE_LIBRARY_PATH=${:CMAKE_LIBRARY_PATH}
LDFLAGS=-L${bzip2:location}/lib -L${jemalloc:location}/lib -L${libaio:location}/lib -L${xz-utils:location}/lib -L${zlib:location}/lib -L${unixodbc:location}/lib -L${lz4:location}/lib -L${snappy:location}/lib -L${zstd:location}/lib
post-install =
mkdir -p ${:location}/include/wsrep &&
cp -p wsrep/wsrep_api.h ${:location}/include/wsrep
post-install = set -e
set -- wsrep-lib/wsrep-API/*/wsrep_api.h
install -DpT $1 ${:location}/$1
cp -a wsrep-lib/include ${:location}/wsrep-lib
[mroonga-mariadb]
# mroonga - a storage engine for MySQL. It provides fast fulltext search feature to all MySQL users.
# http://mroonga.github.com/
recipe = slapos.recipe.cmmi
url = https://packages.groonga.org/source/mroonga/mroonga-9.05.tar.gz
md5sum = d81629fc6042a44ce56cc6990d563164
url = https://packages.groonga.org/source/mroonga/mroonga-9.07.tar.gz
md5sum = dca35d72a44cc0f79ac34da8667668f1
pre-configure = set -e
rm -rf fake_mariadb_source
mkdir -p fake_mariadb_source
test -d ${mariadb:location}/include/mysql/server/private && (
mkdir -p fake_mariadb_source/sql
cp -a ${mariadb:location}/include/mysql/server/private/* fake_mariadb_source/sql
cp -a ${mariadb:location}/include/mysql/server/*.h fake_mariadb_source/sql
cp -a ${mariadb:location}/include/mysql/server/mysql fake_mariadb_source/sql
) || ln -s ${mariadb:location}/include/mysql/private fake_mariadb_source/sql
cp -ar ${mariadb:location}/include fake_mariadb_source/include
! test -f fake_mariadb_source/include/mysql/mariadb_stmt.h ||
sed -i -E -e '/^enum enum_(cursor|indicator)_type$/,/};/d' fake_mariadb_source/include/mysql/mariadb_stmt.h
cd fake_mariadb_source
ln -s ${mariadb:location}/wsrep-lib
cp -a ${mariadb:location}/include/mysql/server include
mv include/private sql
configure-options =
--with-mysql-source=fake_mariadb_source
--with-mysql-config=${mariadb:location}/bin/mysql_config
......@@ -102,8 +98,6 @@ configure-options =
patch-options = -p1
patches =
${:_profile_base_location_}/mroonga_boolean.patch#c818568fe35ca6a4298f18e575d962a0
pre-build =
sed -i -e "s,${mariadb:location}/include,$(pwd)/fake_mariadb_source/include,g" Makefile */Makefile
environment =
PATH=${binutils:location}/bin:${groonga:location}/bin:${patch:location}/bin:${pkgconfig:location}/bin:%(PATH)s
CPPFLAGS=-I${groonga:location}/include/groonga
......
......@@ -17,8 +17,8 @@ parts =
[openssl]
recipe = slapos.recipe.cmmi
shared = true
url = https://www.openssl.org/source/openssl-1.1.0j.tar.gz
md5sum = b4ca5b78ae6ae79da80790b30dbedbdc
url = https://www.openssl.org/source/openssl-1.1.1c.tar.gz
md5sum = 15e21da6efe8aa0e0768ffd8cd37a5f6
location = @@LOCATION@@
# 'prefix' option to override --openssldir/--prefix (which is useful
# when combined with DESTDIR). Used by slapos.package.git/obs
......@@ -57,8 +57,8 @@ openssl = ${openssl:location}/bin/openssl
[openssl-1.0]
recipe = slapos.recipe.cmmi
shared = true
url = https://www.openssl.org/source/openssl-1.0.2p.tar.gz
md5sum = ac5eb30bf5798aa14b1ae6d0e7da58df
url = https://www.openssl.org/source/openssl-1.0.2s.tar.gz
md5sum = 98ec4e085962689b91d25e1dcdfc14a2
location = @@LOCATION@@
# 'prefix' option to override --openssldir/--prefix (which is useful
# when combined with INSTALL_PREFIX). Used by slapos.package.git/obs
......
......@@ -28,7 +28,7 @@ from setuptools import setup, find_packages
import glob
import os
version = '1.0.119'
version = '1.0.121'
name = 'slapos.cookbook'
long_description = open("README.rst").read() + "\n" + \
open("CHANGES.rst").read() + "\n"
......
......@@ -32,10 +32,10 @@ class WrapUpdateMySQL(GenericBaseRecipe):
self.createPythonScript(
self.options['output'],
__name__ + '.mysql.updateMysql',
[{
kw = {
'mysql_upgrade_binary': self.options['binary'],
'mysql_binary': self.options['mysql'],
'mysql_script_file': self.options['init-script'],
}]
}
),
]
......@@ -4,69 +4,45 @@ import time
import sys
import pytz
def updateMysql(conf):
def updateMysql(mysql_upgrade_binary, mysql_binary, mysql_script_file):
sleep = 30
is_succeed = False
try:
script_filename = conf.pop('mysql_script_file')
except KeyError:
pass
else:
assert 'mysql_script' not in conf
with open(script_filename) as script_file:
conf['mysql_script'] = script_file.read()
is_succeeded = False
with open(mysql_script_file) as script_file:
mysql_script = script_file.read()
mysql_list = mysql_binary, '-B'
mysql_tzinfo_to_sql_list = (
os.path.join(os.path.dirname(mysql_binary), 'mysql_tzinfo_to_sql'),
os.path.join(os.path.dirname(pytz.__file__), 'zoneinfo'),
)
while True:
while True:
mysql_upgrade_list = [conf['mysql_upgrade_binary'], '--user=root']
if 'socket' in conf:
mysql_upgrade_list.append('--socket=' + conf['socket'])
mysql_upgrade = subprocess.Popen(mysql_upgrade_list, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
mysql_upgrade = subprocess.Popen(mysql_upgrade_binary,
stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
result = mysql_upgrade.communicate()[0]
if mysql_upgrade.returncode is None:
mysql_upgrade.kill()
if mysql_upgrade.returncode == 0:
print "MySQL database upgraded with result:\n%s" % result
elif 'is already upgraded' in result:
print "No need to upgrade MySQL database"
else:
print "Command %r failed with result:\n%s" % (mysql_upgrade_list, result)
if mysql_upgrade.returncode:
print "Command %r failed with result:\n%s" % (mysql_upgrade_binary, result)
break
mysql_list = [conf['mysql_binary'].strip(), '-B', '--user=root']
if 'socket' in conf:
mysql_list.append('--socket=' + conf['socket'])
print "MySQL database upgraded with result:\n%s" % result
mysql = subprocess.Popen(mysql_list, stdin=subprocess.PIPE,
stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
result = mysql.communicate(conf['mysql_script'])[0]
if mysql.returncode is None:
mysql.kill()
if mysql.returncode != 0:
result = mysql.communicate(mysql_script)[0]
if mysql.returncode:
print 'Command %r failed with:\n%s' % (mysql_list, result)
break
# import timezone database
mysql_tzinfo_to_sql_binary = os.path.join(
os.path.dirname(conf['mysql_binary'].strip()), 'mysql_tzinfo_to_sql')
zoneinfo_directory = '%s/zoneinfo' % os.path.dirname(pytz.__file__)
mysql_tzinfo_to_sql_list = [mysql_tzinfo_to_sql_binary, zoneinfo_directory]
mysql_tzinfo_to_sql = subprocess.Popen(mysql_tzinfo_to_sql_list, stdin=subprocess.PIPE,
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
timezone_sql = mysql_tzinfo_to_sql.communicate()[0]
if mysql_tzinfo_to_sql.returncode != 0:
print 'Command %r failed with:\n%s' % (mysql_tzinfo_to_sql_list, result)
break
mysql = subprocess.Popen(mysql_list + ['mysql',], stdin=subprocess.PIPE,
mysql = subprocess.Popen(mysql_list + ('mysql',), stdin=subprocess.PIPE,
stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
result = mysql.communicate(timezone_sql)[0]
if mysql.returncode is None:
mysql.kill()
if mysql.returncode != 0:
if mysql.returncode:
print 'Command %r failed with:\n%s' % (mysql_list, result)
break
is_succeeded = True
break
if is_succeeded:
print 'SlapOS initialisation script succesfully applied on database.'
break
return
print 'Sleeping for %ss and retrying' % sleep
sys.stdout.flush()
sys.stderr.flush()
......
......@@ -38,6 +38,7 @@ template = {{ neo_master }}
<= jinja2-template-base
template = {{ neo }}
extra-context =
key user slap-configuration:user-name
key master_cfg neo-master:rendered
key admin_cfg neo-admin:rendered
{%- if mariadb_location is defined %}
......
......@@ -42,7 +42,7 @@
"periodicity": {
"description": "Periodicity in minutes at which the cluster health is checked.",
"default": 10,
"type": "float"
"type": "number"
}
},
"type": "object"
......
......@@ -25,6 +25,7 @@ slow-query-log = ${directory:log}/mariadb_slowquery.log
extra-dict = {{ dumps(slapparameter_dict.get('mysql', {})) }}
init-file = ${init-script:rendered}
engine = {{ slapparameter_dict.get('engine', '') }}
user = {{ user }}
[my-cnf]
recipe = slapos.recipe.template:jinja2
......@@ -102,7 +103,7 @@ wrapper = ${directory:etc_run}/{{ 'neostorage-' ~ i }}
logfile = ${directory:log}/{{ 'neostorage-' ~ i }}.log
{%- if mysql %}
{%- do init_list.append('CREATE DATABASE IF NOT EXISTS neo' ~ i ~ ';') %}
database-parameters = root@neo{{ i }}${my-cnf-parameters:socket}
database-parameters = ${my-cnf-parameters:user}@neo{{ i }}${my-cnf-parameters:socket}
{%- else %}
database-parameters = ${directory:db-{{i}}}/db.sqlite
......
......@@ -59,4 +59,4 @@ skip_character_set_client_handshake
[client]
socket = {{ socket }}
user = root
user = {{ parameter_dict['user'] }}
......@@ -94,7 +94,7 @@ mode = 644
recipe = slapos.recipe.template:jinja2
template = ${:_profile_base_location_}/${:_buildout_section_name_}.cfg.in
rendered = ${buildout:directory}/${:_buildout_section_name_}.cfg
md5sum = 0a3a54fcc7be0bbd63cbd64f006ceebc
md5sum = efdb9d6ca0e9d34c5c78583426ab356d
context =
key bin_directory buildout:bin-directory
key develop_eggs_directory buildout:develop-eggs-directory
......@@ -124,12 +124,12 @@ md5sum = 728c407bc526252aec2222752226dde6
[instance-neo]
<= download-base-neo
md5sum = d18e049d580720e733502b774b0d6790
md5sum = 25a7d15df985512047a620d3d0485fc3
[template-neo-my-cnf]
<= download-base-neo
url = ${:_profile_base_location_}/my.cnf.in
md5sum = 9f6f8f2b5f4cb0d97d50ffc1d3837e2f
md5sum = 65288470b8283aaeeb09fbee7a5327ed
[template-mysqld-wrapper]
recipe = slapos.recipe.template:jinja2
......
......@@ -26,7 +26,7 @@ md5sum = d95e8500bdc72d1f40b97cc414656e7e
[template-mariadb]
filename = instance-mariadb.cfg.in
md5sum = 591fe60e2f615c7690fa5078473b1b0a
md5sum = 091ac04b43c8861c6b3fdf4fcadbe6d5
[template-kumofs]
filename = instance-kumofs.cfg.in
......@@ -42,7 +42,7 @@ md5sum = d32417746fcf671d4e86a70379815039
[template-my-cnf]
filename = my.cnf.in
md5sum = 5a6f337117ba8b72d7fe3b7a9f26f5f6
md5sum = a30b651dd66cad827b23401d58c3ff77
[template-mariadb-initial-setup]
filename = mariadb_initial_setup.sql.in
......@@ -70,7 +70,7 @@ md5sum = c64f35f825200fe35328641b2b8e0fdd
[template]
filename = instance.cfg.in
md5sum = f6c6820f9b3653d0d5c29708606fc591
md5sum = 7a33bcfa014a9dab8359815495cd4ede
[monitor-template-dummy]
filename = dummy.cfg
......
......@@ -107,7 +107,7 @@ time = {{ dumps(backup_periodicity) }}
# can be fully restored.
# master-data: use value "2" as we are not in a replication case
#}
command = "${binary-wrap-mysqldump:wrapper-path}" -u root --all-databases --flush-privileges --single-transaction --max-allowed-packet=128M {% if incremental_backup_retention_days > -1 %}--flush-logs --master-data=2 {% endif %}| {{ parameter_dict['gzip-location'] }}/bin/gzip > "${directory:mariadb-backup-full}/$({{ parameter_dict['coreutils-location'] }}/bin/date "+%Y%m%d%H%M%S").sql.gz"
command = "${binary-wrap-mysqldump:wrapper-path}" --all-databases --flush-privileges --single-transaction --max-allowed-packet=128M {% if incremental_backup_retention_days > -1 %}--flush-logs --master-data=2 {% endif %}| {{ parameter_dict['gzip-location'] }}/bin/gzip > "${directory:mariadb-backup-full}/$({{ parameter_dict['coreutils-location'] }}/bin/date "+%Y%m%d%H%M%S").sql.gz"
{# KEEP GLOB PATTERN IN SYNC with generated filenames above
# YYYYmmddHHMMSS -#}
file-glob = ??????????????.sql.gz
......@@ -125,6 +125,7 @@ command = {{ parameter_dict['findutils-location'] }}/bin/find "${directory:maria
[my-cnf-parameters]
ip = {{ ip }}
port = {{ port }}
user = {{ user }}
socket = ${directory:run}/mariadb.sock
data-directory = ${directory:srv}/mariadb
tmp-directory = ${directory:tmp}
......@@ -206,7 +207,7 @@ context = section parameter_dict odbc-ini-text
< = logrotate-entry-base
name = mariadb
log = ${my-cnf-parameters:error-log} ${my-cnf-parameters:slow-query-log}
post = "${binary-wrap-mysql:wrapper-path}" -B -u root -e "FLUSH LOGS"
post = "${binary-wrap-mysql:wrapper-path}" -B -e "FLUSH LOGS"
[{{ section('binary-link') }}]
recipe = slapos.cookbook:symbolic.link
......
......@@ -155,6 +155,7 @@ unixodbc-location = {{ unixodbc_location }}
template = {{ template_mariadb }}
filename = instance-mariadb.cfg
extra-context =
key user slap-configuration:user-name
section parameter_dict dynamic-template-mariadb-parameters
# Keep a section for backward compatibility for removed types
......
......@@ -84,7 +84,7 @@ ssl_cipher = {{ parameter_dict['ssl-cipher'] }}
[client]
socket = {{ socket }}
user = root
user = {{ parameter_dict['user'] }}
[mysql]
no_auto_rehash
......
......@@ -136,7 +136,7 @@ pyparsing = 2.2.0
pytz = 2016.10
requests = 2.13.0
six = 1.12.0
slapos.cookbook = 1.0.119
slapos.cookbook = 1.0.121
slapos.core = 1.4.26
slapos.extension.strip = 0.4
slapos.extension.shared = 1.0
......@@ -218,8 +218,8 @@ pyrsistent = 0.14.5
ipaddress = 1.0.18
# Required by:
# slapos.cookbook==1.0.119
jsonschema = 3.0.0a3
# slapos.cookbook==1.0.121
jsonschema = 3.0.2
# Required by:
# slapos.toolbox==0.94
......