Commit 2f83e1e7 by Jérome Perrin

slapproxy: remove old tables on migration

Instead of keeping old tables which cause confusion for humans and tools
directly operating on the database, remove old tables and create a
separate backup file.
1 parent ed6c77d5
......@@ -31,6 +31,7 @@
from lxml import etree
import random
import string
from datetime import datetime
from slapos.slap.slap import Computer, ComputerPartition, \
SoftwareRelease, SoftwareInstance, NotFoundError
from slapos.proxy.db_version import DB_VERSION
......@@ -176,6 +177,22 @@ def _upgradeDatabaseIfNeeded():
if current_schema_version == DB_VERSION:
previous_table_list = _getTableList()
# first, make a backup of current database
if current_schema_version != '-1':
backup_file_name = "{}-backup-{}to{}-{}.sql".format(
'Old schema detected: Creating a backup of current tables at %s',
with open(backup_file_name, 'w') as f:
for line in g.db.iterdump():
f.write('%s\n' % line)
with app.open_resource('schema.sql', 'r') as f:
schema = % dict(version=DB_VERSION, computer=app.config['computer_id'])
......@@ -186,14 +203,15 @@ def _upgradeDatabaseIfNeeded():
# Migrate all data to new tables'Old schema detected: Migrating old tables...')'Note that old tables are not alterated.')
for table in ('software', 'computer', 'partition', 'slave', 'partition_network'):
for row in execute_db(table, 'SELECT * from %s', db_version=current_schema_version):
columns = ', '.join(row.keys())
placeholders = ':'+', :'.join(row.keys())
query = 'INSERT INTO %s (%s) VALUES (%s)' % ('%s', columns, placeholders)
execute_db(table, query, row)
# then drop old tables
for previous_table in previous_table_list:
g.db.execute("DROP table %s" % previous_table)
is_schema_already_executed = False
......@@ -42,6 +42,7 @@ import sys
import tempfile
import time
import unittest
import mock
import slapos.proxy
import slapos.proxy.views as views
......@@ -1387,12 +1388,28 @@ class TestMigrateVersion10To12(TestInformation, TestRequest, TestSlaveRequest, T
def test_automatic_migration(self):
# create an old table, to assert it is properly removed.
self.db.execute("create table software9 (int a)")
# Make sure that in the initial state we only have version 10 of the tables.
table_list = self.db.execute("SELECT name FROM sqlite_master WHERE type='table' ORDER BY name").fetchall()
self.assertEqual(table_list, [('computer10', ), ('partition10', ), ('partition_network10', ), ('slave10', ), ('software10', )])
self.assertEqual(table_list, [('computer10', ), ('partition10', ), ('partition_network10', ), ('slave10', ), ('software10', ), ('software9', )])
# Run a dummy request to cause migration'/getComputerInformation?computer_id=computer')
from slapos.proxy.views import app
with mock.patch.object(app.logger, 'info') as logger:'/getComputerInformation?computer_id=computer')
# This creates a backup dump
logger.assert_has_calls(('Old schema detected: Creating a backup of current tables at %s', mock.ANY),'Old schema detected: Migrating old tables...')))
backup_name = logger.call_args_list[0][0][1]
with open(backup_name, 'rt') as f:
dump =
self.assertIn("CREATE TABLE", dump)
self.assertIn('INSERT INTO', dump)
# Check some partition parameters
......@@ -1431,6 +1448,10 @@ class TestMigrateVersion10To12(TestInformation, TestRequest, TestSlaveRequest, T
[(u'slappart0', u'computer', u'slappart0', u'', u''), (u'slappart0', u'computer', u'slappart0', u'fc00::1', u'ffff:ffff:ffff::'), (u'slappart1', u'computer', u'slappart1', u'', u''), (u'slappart1', u'computer', u'slappart1', u'fc00::1', u'ffff:ffff:ffff::'), (u'slappart2', u'computer', u'slappart2', u'', u''), (u'slappart2', u'computer', u'slappart2', u'fc00::1', u'ffff:ffff:ffff::'), (u'slappart3', u'computer', u'slappart3', u'', u''), (u'slappart3', u'computer', u'slappart3', u'fc00::1', u'ffff:ffff:ffff::'), (u'slappart4', u'computer', u'slappart4', u'', u''), (u'slappart4', u'computer', u'slappart4', u'fc00::1', u'ffff:ffff:ffff::'), (u'slappart5', u'computer', u'slappart5', u'', u''), (u'slappart5', u'computer', u'slappart5', u'fc00::1', u'ffff:ffff:ffff::'), (u'slappart6', u'computer', u'slappart6', u'', u''), (u'slappart6', u'computer', u'slappart6', u'fc00::1', u'ffff:ffff:ffff::'), (u'slappart7', u'computer', u'slappart7', u'', u''), (u'slappart7', u'computer', u'slappart7', u'fc00::1', u'ffff:ffff:ffff::'), (u'slappart8', u'computer', u'slappart8', u'', u''), (u'slappart8', u'computer', u'slappart8', u'fc00::1', u'ffff:ffff:ffff::'), (u'slappart9', u'computer', u'slappart9', u'', u''), (u'slappart9', u'computer', u'slappart9', u'fc00::1', u'ffff:ffff:ffff::')]
# Check that we only have new tables
table_list = self.db.execute("SELECT name FROM sqlite_master WHERE type='table' ORDER BY name").fetchall()
self.assertEqual(table_list, [('computer12', ), ('forwarded_partition_request12',), ('partition12', ), ('partition_network12', ), ('slave12', ), ('software12', )])
# Override several tests that needs an empty database
@unittest.skip("Not implemented")
def test_multi_node_support_different_software_release_list(self):
Styling with Markdown is supported
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!