Commit 6171fe46 by Cédric de Saint Martin Committed by Rafael Monnerat

slapproxy: add automatic migration procedure to new schema.

Migration to new database schema is automatically done when an old version is detected.
It keeps the old tables untouched in case of needed rollback to an older version.
1 parent 100a6ba1
--version:11
CREATE TABLE IF NOT EXISTS software%(version)s (
url VARCHAR(255),
computer_reference VARCHAR(255) DEFAULT 'computer',
computer_reference VARCHAR(255) DEFAULT '%(computer)s',
CONSTRAINT uniq PRIMARY KEY (url, computer_reference)
);
CREATE TABLE IF NOT EXISTS computer%(version)s (
reference VARCHAR(255),
reference VARCHAR(255) DEFAULT '%(computer)s',
address VARCHAR(255),
netmask VARCHAR(255),
CONSTRAINT uniq PRIMARY KEY (reference)
......@@ -14,7 +14,7 @@ CREATE TABLE IF NOT EXISTS computer%(version)s (
CREATE TABLE IF NOT EXISTS partition%(version)s (
reference VARCHAR(255),
computer_reference VARCHAR(255) DEFAULT 'computer',
computer_reference VARCHAR(255) DEFAULT '%(computer)s',
slap_state VARCHAR(255) DEFAULT 'free',
software_release VARCHAR(255),
xml TEXT,
......@@ -30,7 +30,7 @@ CREATE TABLE IF NOT EXISTS partition%(version)s (
CREATE TABLE IF NOT EXISTS slave%(version)s (
reference VARCHAR(255),
computer_reference VARCHAR(255),
computer_reference VARCHAR(255) DEFAULT '%(computer)s',
connection_xml TEXT,
hosted_by VARCHAR(255),
asked_by VARCHAR(255) -- only used for debugging,
......@@ -39,8 +39,9 @@ CREATE TABLE IF NOT EXISTS slave%(version)s (
CREATE TABLE IF NOT EXISTS partition_network%(version)s (
partition_reference VARCHAR(255),
computer_reference VARCHAR(255),
computer_reference VARCHAR(255) DEFAULT '%(computer)s',
reference VARCHAR(255),
address VARCHAR(255),
netmask VARCHAR(255)
);
......@@ -108,9 +108,14 @@ def partitiondict2partition(partition):
return slap_partition
def execute_db(table, query, args=(), one=False):
def execute_db(table, query, args=(), one=False, db_version=None, log=False):
if not db_version:
db_version = DB_VERSION
query = query % (table + db_version,)
if log:
print query
try:
cur = g.db.execute(query % (table + DB_VERSION,), args)
cur = g.db.execute(query, args)
except:
app.logger.error('There was some issue during processing query %r on table %r with args %r' % (query, table, args))
raise
......@@ -122,15 +127,66 @@ def execute_db(table, query, args=(), one=False):
def connect_db():
return sqlite3.connect(app.config['DATABASE_URI'])
def _getTableList():
return g.db.execute("SELECT name FROM sqlite_master WHERE type='table' ORDER BY Name").fetchall()
@app.before_request
def before_request():
g.db = connect_db()
def _getCurrentDatabaseSchemaVersion():
"""
Return version of database schema.
As there is no actual definition of version, analyse
name of all tables (containing version) and take the
highest version (as several versions can live in the db).
"""
# XXX: define an actual version and proper migration/repair procedure.
version = -1
for table_name in _getTableList():
try:
table_version = int(table_name[0][-2:])
except ValueError:
table_version = int(table_name[0][-1:])
if table_version > version:
version = table_version
return str(version)
def _upgradeDatabaseIfNeeded():
"""
Analyses current database compared to defined schema,
and adapt tables/data it if needed.
"""
current_schema_version = _getCurrentDatabaseSchemaVersion()
# If version of current database is not old, do nothing
if current_schema_version == DB_VERSION:
return
schema = app.open_resource('schema.sql')
schema = schema.read() % dict(version=DB_VERSION)
schema = schema.read() % dict(version=DB_VERSION, computer=app.config['computer_id'])
g.db.cursor().executescript(schema)
g.db.commit()
if current_schema_version == '-1':
return
# Migrate all data to new tables
app.logger.info('Old schema detected: Migrating old tables...')
app.logger.info('Note that old tables are not alterated.')
for table in ('software', 'computer', 'partition', 'slave', 'partition_network'):
for row in execute_db(table, 'SELECT * from %s', db_version=current_schema_version):
columns = ', '.join(row.keys())
placeholders = ':'+', :'.join(row.keys())
query = 'INSERT INTO %s (%s) VALUES (%s)' % ('%s', columns, placeholders)
execute_db(table, query, row, log=True)
g.db.commit()
is_schema_already_executed = False
@app.before_request
def before_request():
g.db = connect_db()
global is_schema_already_executed
if not is_schema_already_executed:
_upgradeDatabaseIfNeeded()
is_schema_already_executed = True
@app.after_request
def after_request(response):
......
......@@ -40,6 +40,9 @@ import slapos.proxy
import slapos.proxy.views as views
import slapos.slap.slap
import sqlite3
import pkg_resources
class WrongFormat(Exception):
pass
......@@ -137,6 +140,7 @@ database_uri = %(tempdir)s/lib/proxy.db
Remove files generated for test
"""
shutil.rmtree(self._tempdir, True)
views.is_schema_already_executed = False
class TestInformation(BasicMixin, unittest.TestCase):
......@@ -841,3 +845,81 @@ class TestMultiNodeSupport(MasterMixin):
except slapos.slap.NotFoundError:
self.fail('Could not fetch informations for registered computer.')
class TestMigrateVersion10To11(TestInformation, TestRequest, TestSlaveRequest, TestMultiNodeSupport):
"""
Test that old database version are automatically migrated without failure
"""
def setUp(self):
super(TestMigrateVersion10To11, self).setUp()
schema = pkg_resources.resource_stream('slapos.tests.slapproxy', 'database_dump_version_10.sql')
schema = schema.read() % dict(version='11')
self.db = sqlite3.connect(self.proxy_db)
self.db.cursor().executescript(schema)
self.db.commit()
def test_automatic_migration(self):
table_list = ('software11', 'computer11', 'partition11', 'slave11', 'partition_network11')
for table in table_list:
self.assertRaises(sqlite3.OperationalError, self.db.execute, "SELECT name FROM computer11")
# Run a dummy request to cause migration
self.app.get('/getComputerInformation?computer_id=computer')
# Check some partition parameters
self.assertEqual(
loads(self.app.get('/getComputerInformation?computer_id=computer').data)._computer_partition_list[0]._parameter_dict['slap_software_type'],
'production'
)
# Lower level tests
computer_list = self.db.execute("SELECT * FROM computer11").fetchall()
self.assertEqual(
computer_list,
[(u'computer', u'127.0.0.1', u'255.255.255.255')]
)
software_list = self.db.execute("SELECT * FROM software11").fetchall()
self.assertEqual(
software_list,
[(u'/srv/slapgrid//srv//runner/project//slapos/software.cfg', u'computer')]
)
partition_list = self.db.execute("select * from partition11").fetchall()
self.assertEqual(
partition_list,
[(u'slappart0', u'computer', u'busy', u'/srv/slapgrid//srv//runner/project//slapos/software.cfg', u'<?xml version=\'1.0\' encoding=\'utf-8\'?>\n<instance>\n <parameter id="json">{\n "site-id": "erp5"\n }\n}</parameter>\n</instance>\n', None, None, u'production', u'slapos', None, u'started'), (u'slappart1', u'computer', u'busy', u'/srv/slapgrid//srv//runner/project//slapos/software.cfg', u"<?xml version='1.0' encoding='utf-8'?>\n<instance/>\n", u'<?xml version=\'1.0\' encoding=\'utf-8\'?>\n<instance>\n <parameter id="url">mysql://127.0.0.1:45678/erp5</parameter>\n</instance>\n', None, u'mariadb', u'MariaDB DataBase', u'slappart0', u'started'), (u'slappart2', u'computer', u'busy', u'/srv/slapgrid//srv//runner/project//slapos/software.cfg', u'<?xml version=\'1.0\' encoding=\'utf-8\'?>\n<instance>\n <parameter id="cloudooo-json"></parameter>\n</instance>\n', u'<?xml version=\'1.0\' encoding=\'utf-8\'?>\n<instance>\n <parameter id="url">cloudooo://127.0.0.1:23000/</parameter>\n</instance>\n', None, u'cloudooo', u'Cloudooo', u'slappart0', u'started'), (u'slappart3', u'computer', u'busy', u'/srv/slapgrid//srv//runner/project//slapos/software.cfg', u"<?xml version='1.0' encoding='utf-8'?>\n<instance/>\n", u'<?xml version=\'1.0\' encoding=\'utf-8\'?>\n<instance>\n <parameter id="url">memcached://127.0.0.1:11000/</parameter>\n</instance>\n', None, u'memcached', u'Memcached', u'slappart0', u'started'), (u'slappart4', u'computer', u'busy', u'/srv/slapgrid//srv//runner/project//slapos/software.cfg', u"<?xml version='1.0' encoding='utf-8'?>\n<instance/>\n", u'<?xml version=\'1.0\' encoding=\'utf-8\'?>\n<instance>\n <parameter id="url">memcached://127.0.0.1:13301/</parameter>\n</instance>\n', None, u'kumofs', u'KumoFS', u'slappart0', u'started'), (u'slappart5', u'computer', u'busy', u'/srv/slapgrid//srv//runner/project//slapos/software.cfg', u'<?xml version=\'1.0\' encoding=\'utf-8\'?>\n<instance>\n <parameter id="kumofs-url">memcached://127.0.0.1:13301/</parameter>\n <parameter id="memcached-url">memcached://127.0.0.1:11000/</parameter>\n <parameter id="cloudooo-url">cloudooo://127.0.0.1:23000/</parameter>\n</instance>\n', u'<?xml version=\'1.0\' encoding=\'utf-8\'?>\n<instance>\n <parameter id="url">https://[fc00::1]:10001</parameter>\n</instance>\n', None, u'tidstorage', u'TidStorage', u'slappart0', u'started'), (u'slappart6', u'computer', u'free', None, None, None, None, None, None, None, u'started'), (u'slappart7', u'computer', u'free', None, None, None, None, None, None, None, u'started'), (u'slappart8', u'computer', u'free', None, None, None, None, None, None, None, u'started'), (u'slappart9', u'computer', u'free', None, None, None, None, None, None, None, u'started')]
)
slave_list = self.db.execute("select * from slave11").fetchall()
self.assertEqual(
slave_list,
[]
)
partition_network_list = self.db.execute("select * from partition_network11").fetchall()
self.assertEqual(
partition_network_list,
[(u'slappart0', u'computer', u'slappart0', u'127.0.0.1', u'255.255.255.255'), (u'slappart0', u'computer', u'slappart0', u'fc00::1', u'ffff:ffff:ffff::'), (u'slappart1', u'computer', u'slappart1', u'127.0.0.1', u'255.255.255.255'), (u'slappart1', u'computer', u'slappart1', u'fc00::1', u'ffff:ffff:ffff::'), (u'slappart2', u'computer', u'slappart2', u'127.0.0.1', u'255.255.255.255'), (u'slappart2', u'computer', u'slappart2', u'fc00::1', u'ffff:ffff:ffff::'), (u'slappart3', u'computer', u'slappart3', u'127.0.0.1', u'255.255.255.255'), (u'slappart3', u'computer', u'slappart3', u'fc00::1', u'ffff:ffff:ffff::'), (u'slappart4', u'computer', u'slappart4', u'127.0.0.1', u'255.255.255.255'), (u'slappart4', u'computer', u'slappart4', u'fc00::1', u'ffff:ffff:ffff::'), (u'slappart5', u'computer', u'slappart5', u'127.0.0.1', u'255.255.255.255'), (u'slappart5', u'computer', u'slappart5', u'fc00::1', u'ffff:ffff:ffff::'), (u'slappart6', u'computer', u'slappart6', u'127.0.0.1', u'255.255.255.255'), (u'slappart6', u'computer', u'slappart6', u'fc00::1', u'ffff:ffff:ffff::'), (u'slappart7', u'computer', u'slappart7', u'127.0.0.1', u'255.255.255.255'), (u'slappart7', u'computer', u'slappart7', u'fc00::1', u'ffff:ffff:ffff::'), (u'slappart8', u'computer', u'slappart8', u'127.0.0.1', u'255.255.255.255'), (u'slappart8', u'computer', u'slappart8', u'fc00::1', u'ffff:ffff:ffff::'), (u'slappart9', u'computer', u'slappart9', u'127.0.0.1', u'255.255.255.255'), (u'slappart9', u'computer', u'slappart9', u'fc00::1', u'ffff:ffff:ffff::')]
)
# Override several tests that needs an empty database
@unittest.skip("Not implemented")
def test_multi_node_support_different_software_release_list(self):
pass
@unittest.skip("Not implemented")
def test_multi_node_support_instance_default_computer(self):
pass
@unittest.skip("Not implemented")
def test_multi_node_support_instance_guid(self):
pass
@unittest.skip("Not implemented")
def test_partition_are_empty(self):
pass
@unittest.skip("Not implemented")
def test_request_consistent_parameters(self):
pass
-- Real world example of webrunner database running version 10 of proxy db.
PRAGMA foreign_keys=OFF;
BEGIN TRANSACTION;
CREATE TABLE software10 (url VARCHAR(255) UNIQUE);
INSERT INTO "software10" VALUES('/srv/slapgrid//srv//runner/project//slapos/software.cfg');
CREATE TABLE computer10 (
address VARCHAR(255),
netmask VARCHAR(255),
CONSTRAINT uniq PRIMARY KEY (address, netmask));
INSERT INTO "computer10" VALUES('127.0.0.1','255.255.255.255');
CREATE TABLE partition10 (
reference VARCHAR(255) UNIQUE,
slap_state VARCHAR(255) DEFAULT 'free',
software_release VARCHAR(255),
xml TEXT,
connection_xml TEXT,
slave_instance_list TEXT,
software_type VARCHAR(255),
partition_reference VARCHAR(255),
requested_by VARCHAR(255), -- only used for debugging,
-- slapproxy does not support proper scope
requested_state VARCHAR(255) NOT NULL DEFAULT 'started'
);
INSERT INTO "partition10" VALUES('slappart0','busy','/srv/slapgrid//srv//runner/project//slapos/software.cfg','<?xml version=''1.0'' encoding=''utf-8''?>
<instance>
<parameter id="json">{
"site-id": "erp5"
}
}</parameter>
</instance>
',NULL,NULL,'production','slapos',NULL,'started');
INSERT INTO "partition10" VALUES('slappart1','busy','/srv/slapgrid//srv//runner/project//slapos/software.cfg','<?xml version=''1.0'' encoding=''utf-8''?>
<instance/>
','<?xml version=''1.0'' encoding=''utf-8''?>
<instance>
<parameter id="url">mysql://127.0.0.1:45678/erp5</parameter>
</instance>
',NULL,'mariadb','MariaDB DataBase','slappart0','started');
INSERT INTO "partition10" VALUES('slappart2','busy','/srv/slapgrid//srv//runner/project//slapos/software.cfg','<?xml version=''1.0'' encoding=''utf-8''?>
<instance>
<parameter id="cloudooo-json"></parameter>
</instance>
','<?xml version=''1.0'' encoding=''utf-8''?>
<instance>
<parameter id="url">cloudooo://127.0.0.1:23000/</parameter>
</instance>
',NULL,'cloudooo','Cloudooo','slappart0','started');
INSERT INTO "partition10" VALUES('slappart3','busy','/srv/slapgrid//srv//runner/project//slapos/software.cfg','<?xml version=''1.0'' encoding=''utf-8''?>
<instance/>
','<?xml version=''1.0'' encoding=''utf-8''?>
<instance>
<parameter id="url">memcached://127.0.0.1:11000/</parameter>
</instance>
',NULL,'memcached','Memcached','slappart0','started');
INSERT INTO "partition10" VALUES('slappart4','busy','/srv/slapgrid//srv//runner/project//slapos/software.cfg','<?xml version=''1.0'' encoding=''utf-8''?>
<instance/>
','<?xml version=''1.0'' encoding=''utf-8''?>
<instance>
<parameter id="url">memcached://127.0.0.1:13301/</parameter>
</instance>
',NULL,'kumofs','KumoFS','slappart0','started');
INSERT INTO "partition10" VALUES('slappart5','busy','/srv/slapgrid//srv//runner/project//slapos/software.cfg','<?xml version=''1.0'' encoding=''utf-8''?>
<instance>
<parameter id="kumofs-url">memcached://127.0.0.1:13301/</parameter>
<parameter id="memcached-url">memcached://127.0.0.1:11000/</parameter>
<parameter id="cloudooo-url">cloudooo://127.0.0.1:23000/</parameter>
</instance>
','<?xml version=''1.0'' encoding=''utf-8''?>
<instance>
<parameter id="url">https://[fc00::1]:10001</parameter>
</instance>
',NULL,'tidstorage','TidStorage','slappart0','started');
INSERT INTO "partition10" VALUES('slappart6','free',NULL,NULL,NULL,NULL,NULL,NULL,NULL,'started');
INSERT INTO "partition10" VALUES('slappart7','free',NULL,NULL,NULL,NULL,NULL,NULL,NULL,'started');
INSERT INTO "partition10" VALUES('slappart8','free',NULL,NULL,NULL,NULL,NULL,NULL,NULL,'started');
INSERT INTO "partition10" VALUES('slappart9','free',NULL,NULL,NULL,NULL,NULL,NULL,NULL,'started');
CREATE TABLE slave10 (
reference VARCHAR(255) UNIQUE,
connection_xml TEXT,
hosted_by VARCHAR(255),
asked_by VARCHAR(255) -- only used for debugging,
-- slapproxy does not support proper scope
);
CREATE TABLE partition_network10 (
partition_reference VARCHAR(255),
reference VARCHAR(255),
address VARCHAR(255),
netmask VARCHAR(255)
);
INSERT INTO "partition_network10" VALUES('slappart0','slappart0','127.0.0.1','255.255.255.255');
INSERT INTO "partition_network10" VALUES('slappart0','slappart0','fc00::1','ffff:ffff:ffff::');
INSERT INTO "partition_network10" VALUES('slappart1','slappart1','127.0.0.1','255.255.255.255');
INSERT INTO "partition_network10" VALUES('slappart1','slappart1','fc00::1','ffff:ffff:ffff::');
INSERT INTO "partition_network10" VALUES('slappart2','slappart2','127.0.0.1','255.255.255.255');
INSERT INTO "partition_network10" VALUES('slappart2','slappart2','fc00::1','ffff:ffff:ffff::');
INSERT INTO "partition_network10" VALUES('slappart3','slappart3','127.0.0.1','255.255.255.255');
INSERT INTO "partition_network10" VALUES('slappart3','slappart3','fc00::1','ffff:ffff:ffff::');
INSERT INTO "partition_network10" VALUES('slappart4','slappart4','127.0.0.1','255.255.255.255');
INSERT INTO "partition_network10" VALUES('slappart4','slappart4','fc00::1','ffff:ffff:ffff::');
INSERT INTO "partition_network10" VALUES('slappart5','slappart5','127.0.0.1','255.255.255.255');
INSERT INTO "partition_network10" VALUES('slappart5','slappart5','fc00::1','ffff:ffff:ffff::');
INSERT INTO "partition_network10" VALUES('slappart6','slappart6','127.0.0.1','255.255.255.255');
INSERT INTO "partition_network10" VALUES('slappart6','slappart6','fc00::1','ffff:ffff:ffff::');
INSERT INTO "partition_network10" VALUES('slappart7','slappart7','127.0.0.1','255.255.255.255');
INSERT INTO "partition_network10" VALUES('slappart7','slappart7','fc00::1','ffff:ffff:ffff::');
INSERT INTO "partition_network10" VALUES('slappart8','slappart8','127.0.0.1','255.255.255.255');
INSERT INTO "partition_network10" VALUES('slappart8','slappart8','fc00::1','ffff:ffff:ffff::');
INSERT INTO "partition_network10" VALUES('slappart9','slappart9','127.0.0.1','255.255.255.255');
INSERT INTO "partition_network10" VALUES('slappart9','slappart9','fc00::1','ffff:ffff:ffff::');
COMMIT;
Styling with Markdown is supported
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!