Skip to content

S
slapos.core
Commit 189fa3a0 authored by Xavier Thompson's avatar Xavier Thompson
Browse files
Options

slapproxy: Add software release root path option 

The 'local_software_release_root' parameter can be set in slapos.cfg:
    [slapproxy]
    local_software_release_root = <path>
or via the command line using:
    slapos proxy start --local-software-release-root <path>

When slapproxy starts, it updates all local URLs that are subpaths of
the previous root path to be subpaths of the new root path instead:
    <previous-root-path>/<subpath>   =>   <new-root-path>/<subpath>

The default root path is '/'. This means that by default slapproxy
behaves the same as before.
parent d8301b86
Expand all Show whitespace changes
Inline Side-by-side
Showing with 251 additions and 76 deletions
slapos/cli/proxy_start.py
View file @ 189fa3a0
...@@ -47,6 +47,8 @@ class ProxyStartCommand(ConfigCommand): ...@@ -47,6 +47,8 @@ class ProxyStartCommand(ConfigCommand):
help='Port to use') help='Port to use')
ap.add_argument('--host', ap.add_argument('--host',
help='Host to use') help='Host to use')
ap.add_argument('--local-software-release-root',
help='Root path for local Software Releases')
return ap return ap
......
slapos/proxy/__init__.py
View file @ 189fa3a0
...@@ -93,6 +93,7 @@ def setupFlaskConfiguration(conf): ...@@ -93,6 +93,7 @@ def setupFlaskConfiguration(conf):
app.config['DATABASE_URI'] = conf.database_uri app.config['DATABASE_URI'] = conf.database_uri
app.config['software_product_list'] = conf.software_product_list app.config['software_product_list'] = conf.software_product_list
app.config['multimaster'] = conf.multimaster app.config['multimaster'] = conf.multimaster
app.config['local_software_release_root'] = getattr(conf,'local_software_release_root', None)
def connectDB(): def connectDB():
# if first connection, create an empty db at DATABASE_URI path # if first connection, create an empty db at DATABASE_URI path
......
slapos/proxy/schema.sql
View file @ 189fa3a0
--version:14 --version:15
CREATE TABLE IF NOT EXISTS local_software_release_root%(version)s (
path VARCHAR(255)
);
INSERT INTO local_software_release_root%(version)s VALUES(NULL);
CREATE TABLE IF NOT EXISTS software%(version)s ( CREATE TABLE IF NOT EXISTS software%(version)s (
url VARCHAR(255), url VARCHAR(255),
computer_reference VARCHAR(255) DEFAULT '%(computer)s', computer_reference VARCHAR(255) DEFAULT '%(computer)s',
......
slapos/proxy/views.py
View file @ 189fa3a0
...@@ -32,6 +32,7 @@ import random ...@@ -32,6 +32,7 @@ import random
import string import string
import time import time
import re import re
import os
from datetime import datetime from datetime import datetime
from slapos.slap.slap import Computer, ComputerPartition, \ from slapos.slap.slap import Computer, ComputerPartition, \
SoftwareRelease, SoftwareInstance, NotFoundError SoftwareRelease, SoftwareInstance, NotFoundError
...@@ -180,6 +181,8 @@ def _upgradeDatabaseIfNeeded(): ...@@ -180,6 +181,8 @@ def _upgradeDatabaseIfNeeded():
table_list = ('software', 'computer', 'partition', 'slave', 'partition_network') table_list = ('software', 'computer', 'partition', 'slave', 'partition_network')
if int(current_schema_version) >= 11: if int(current_schema_version) >= 11:
table_list += ('forwarded_partition_request',) table_list += ('forwarded_partition_request',)
if int(current_schema_version) >= 15:
table_list += ('local_software_release_root',)
for table in table_list: for table in table_list:
for row in execute_db(table, 'SELECT * from %s', db_version=current_schema_version): for row in execute_db(table, 'SELECT * from %s', db_version=current_schema_version):
columns = ', '.join(row.keys()) columns = ', '.join(row.keys())
...@@ -191,6 +194,28 @@ def _upgradeDatabaseIfNeeded(): ...@@ -191,6 +194,28 @@ def _upgradeDatabaseIfNeeded():
g.db.execute("DROP table %s" % previous_table) g.db.execute("DROP table %s" % previous_table)
g.db.commit() g.db.commit()
def _updateLocalSoftwareReleaseRootPathIfNeeded():
"""
Update the local software release root path if it changed,
and rebase all URLs in the database relatively to the new path.
"""
# Retrieve the current root path and replace it with the new one
current_root_path = execute_db('local_software_release_root', 'SELECT * from %s', one=True)['path'] or os.sep
new_root_path = app.config['local_software_release_root'] or os.sep
execute_db('local_software_release_root', 'UPDATE %s SET path=?', [new_root_path])
# Rebase all URLs relative to the new root path
if current_root_path != new_root_path:
def migrate_url(url):
if not url or urlparse(url).scheme:
return url
rel = os.path.relpath(url, current_root_path)
if rel.startswith(os.pardir + os.sep):
return url
return os.path.join(new_root_path, rel)
g.db.create_function('migrate_url', 1, migrate_url)
execute_db('software', 'UPDATE %s SET url=migrate_url(url)')
execute_db('partition', 'UPDATE %s SET software_release=migrate_url(software_release)')
is_schema_already_executed = False is_schema_already_executed = False
@app.before_request @app.before_request
def before_request(): def before_request():
...@@ -198,6 +223,7 @@ def before_request(): ...@@ -198,6 +223,7 @@ def before_request():
global is_schema_already_executed global is_schema_already_executed
if not is_schema_already_executed: if not is_schema_already_executed:
_upgradeDatabaseIfNeeded() _upgradeDatabaseIfNeeded()
_updateLocalSoftwareReleaseRootPathIfNeeded()
is_schema_already_executed = True is_schema_already_executed = True
......
slapos/tests/test_slapproxy.py
View file @ 189fa3a0
...@@ -1682,16 +1682,15 @@ database_uri = %(tempdir)s/lib/external_proxy.db ...@@ -1682,16 +1682,15 @@ database_uri = %(tempdir)s/lib/external_proxy.db
Start external slapproxy Start external slapproxy
""" """
logging.getLogger().info('Starting external proxy, listening to %s:%s' % (self.external_proxy_host, self.external_proxy_port)) logging.getLogger().info('Starting external proxy, listening to %s:%s' % (self.external_proxy_host, self.external_proxy_port))
# XXX This uses a hack to run current code of slapos.core
import slapos
self.external_proxy_process = subprocess.Popen( self.external_proxy_process = subprocess.Popen(
[ [
sys.executable, '%s/../cli/entry.py' % os.path.dirname(slapos.tests.__file__), sys.executable, '-m', 'slapos.cli.entry',
'proxy', 'start', '--cfg', self.external_slapproxy_configuration_file_location 'proxy', 'start', '--cfg', self.external_slapproxy_configuration_file_location
], ],
stdout=subprocess.DEVNULL, stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
env={"PYTHONPATH": ':'.join(sys.path)} env={"PYTHONPATH": ':'.join(sys.path)},
cwd=os.chdir(os.path.join(os.path.dirname(slapos.proxy.__file__), os.pardir, os.pardir)),
) )
# Wait a bit for proxy to be started # Wait a bit for proxy to be started
attempts = 0 attempts = 0
...@@ -2050,7 +2049,7 @@ class _MigrationTestCase(TestInformation, TestRequest, TestSlaveRequest, TestMul ...@@ -2050,7 +2049,7 @@ class _MigrationTestCase(TestInformation, TestRequest, TestSlaveRequest, TestMul
""" """
dump_filename = NotImplemented dump_filename = NotImplemented
initial_table_list = NotImplemented initial_table_list = NotImplemented
current_version = '14' current_version = '15'
def setUp(self): def setUp(self):
TestInformation.setUp(self) TestInformation.setUp(self)
...@@ -2136,6 +2135,7 @@ class _MigrationTestCase(TestInformation, TestRequest, TestSlaveRequest, TestMul ...@@ -2136,6 +2135,7 @@ class _MigrationTestCase(TestInformation, TestRequest, TestSlaveRequest, TestMul
self.assertEqual([x[0] for x in table_list], self.assertEqual([x[0] for x in table_list],
['computer{}'.format(self.current_version), ['computer{}'.format(self.current_version),
'forwarded_partition_request{}'.format(self.current_version), 'forwarded_partition_request{}'.format(self.current_version),
'local_software_release_root{}'.format(self.current_version),
'partition{}'.format(self.current_version), 'partition{}'.format(self.current_version),
'partition_network{}'.format(self.current_version), 'partition_network{}'.format(self.current_version),
'slave{}'.format(self.current_version), 'slave{}'.format(self.current_version),
...@@ -2184,4 +2184,9 @@ class TestMigrateVersion13ToLatest(_MigrationTestCase): ...@@ -2184,4 +2184,9 @@ class TestMigrateVersion13ToLatest(_MigrationTestCase):
initial_table_list = ['computer13', 'forwarded_partition_request13', 'partition13', 'partition_network13', 'slave13', 'software13', ] initial_table_list = ['computer13', 'forwarded_partition_request13', 'partition13', 'partition_network13', 'slave13', 'software13', ]
class TestMigrateVersion14ToLatest(_MigrationTestCase):
dump_filename = 'database_dump_version_14.sql'
initial_table_list = ['computer14', 'forwarded_partition_request14', 'partition14', 'partition_network14', 'slave14', 'software14', ]
del _MigrationTestCase del _MigrationTestCase
slapos/tests/test_slapproxy/database_dump_version_14.sql 0 → 100644
View file @ 189fa3a0
PRAGMA foreign_keys=OFF;
BEGIN TRANSACTION;
CREATE TABLE software14 (
url VARCHAR(255),
computer_reference VARCHAR(255) DEFAULT 'computer',
requested_state VARCHAR(255) DEFAULT 'available',
CONSTRAINT uniq PRIMARY KEY (url, computer_reference)
);
INSERT INTO "software14" VALUES('/srv/slapgrid//srv//runner/project//slapos/software.cfg','computer','available');
CREATE TABLE computer14 (
reference VARCHAR(255) DEFAULT 'computer',
address VARCHAR(255),
netmask VARCHAR(255),
CONSTRAINT uniq PRIMARY KEY (reference)
);
INSERT INTO "computer14" VALUES('computer','127.0.0.1','255.255.255.255');
CREATE TABLE partition14 (
reference VARCHAR(255),
computer_reference VARCHAR(255) DEFAULT 'computer',
slap_state VARCHAR(255) DEFAULT 'free',
software_release VARCHAR(255),
xml TEXT,
connection_xml TEXT,
slave_instance_list TEXT,
software_type VARCHAR(255),
partition_reference VARCHAR(255), -- name of the instance
requested_by VARCHAR(255), -- only used for debugging,
-- slapproxy does not support proper scope
requested_state VARCHAR(255) NOT NULL DEFAULT 'started',
timestamp REAL,
CONSTRAINT uniq PRIMARY KEY (reference, computer_reference)
);
INSERT INTO "partition14" VALUES('slappart0','computer','busy','/srv/slapgrid//srv//runner/project//slapos/software.cfg','<?xml version=''1.0'' encoding=''utf-8''?>
<instance>
<parameter id="json">{
"site-id": "erp5"
}
}</parameter>
</instance>
',NULL,NULL,'production','slapos',NULL,'started',NULL);
INSERT INTO "partition14" VALUES('slappart1','computer','busy','/srv/slapgrid//srv//runner/project//slapos/software.cfg','<?xml version=''1.0'' encoding=''utf-8''?>
<instance/>
','<?xml version=''1.0'' encoding=''utf-8''?>
<instance>
<parameter id="url">mysql://127.0.0.1:45678/erp5</parameter>
</instance>
',NULL,'mariadb','MariaDB DataBase','slappart0','started',NULL);
INSERT INTO "partition14" VALUES('slappart2','computer','busy','/srv/slapgrid//srv//runner/project//slapos/software.cfg','<?xml version=''1.0'' encoding=''utf-8''?>
<instance>
<parameter id="cloudooo-json"></parameter>
</instance>
','<?xml version=''1.0'' encoding=''utf-8''?>
<instance>
<parameter id="url">cloudooo://127.0.0.1:23000/</parameter>
</instance>
',NULL,'cloudooo','Cloudooo','slappart0','started',NULL);
INSERT INTO "partition14" VALUES('slappart3','computer','busy','/srv/slapgrid//srv//runner/project//slapos/software.cfg','<?xml version=''1.0'' encoding=''utf-8''?>
<instance/>
','<?xml version=''1.0'' encoding=''utf-8''?>
<instance>
<parameter id="url">memcached://127.0.0.1:11000/</parameter>
</instance>
',NULL,'memcached','Memcached','slappart0','started',NULL);
INSERT INTO "partition14" VALUES('slappart4','computer','busy','/srv/slapgrid//srv//runner/project//slapos/software.cfg','<?xml version=''1.0'' encoding=''utf-8''?>
<instance/>
','<?xml version=''1.0'' encoding=''utf-8''?>
<instance>
<parameter id="url">memcached://127.0.0.1:13301/</parameter>
</instance>
',NULL,'kumofs','KumoFS','slappart0','started',NULL);
INSERT INTO "partition14" VALUES('slappart5','computer','busy','/srv/slapgrid//srv//runner/project//slapos/software.cfg','<?xml version=''1.0'' encoding=''utf-8''?>
<instance>
<parameter id="kumofs-url">memcached://127.0.0.1:13301/</parameter>
<parameter id="memcached-url">memcached://127.0.0.1:11000/</parameter>
<parameter id="cloudooo-url">cloudooo://127.0.0.1:23000/</parameter>
</instance>
','<?xml version=''1.0'' encoding=''utf-8''?>
<instance>
<parameter id="url">https://[fc00::1]:10001</parameter>
</instance>
',NULL,'tidstorage','TidStorage','slappart0','started',NULL);
INSERT INTO "partition14" VALUES('slappart6','computer','free',NULL,NULL,NULL,NULL,NULL,NULL,NULL,'started',NULL);
INSERT INTO "partition14" VALUES('slappart7','computer','free',NULL,NULL,NULL,NULL,NULL,NULL,NULL,'started',NULL);
INSERT INTO "partition14" VALUES('slappart8','computer','free',NULL,NULL,NULL,NULL,NULL,NULL,NULL,'started',NULL);
INSERT INTO "partition14" VALUES('slappart9','computer','free',NULL,NULL,NULL,NULL,NULL,NULL,NULL,'started',NULL);
CREATE TABLE slave14 (
reference VARCHAR(255), -- unique slave reference
computer_reference VARCHAR(255) DEFAULT 'computer',
connection_xml TEXT,
hosted_by VARCHAR(255),
asked_by VARCHAR(255) -- only used for debugging,
-- slapproxy does not support proper scope
);
CREATE TABLE partition_network14 (
partition_reference VARCHAR(255),
computer_reference VARCHAR(255) DEFAULT 'computer',
reference VARCHAR(255),
address VARCHAR(255),
netmask VARCHAR(255)
);
INSERT INTO "partition_network14" VALUES('slappart0','computer','slappart0','127.0.0.1','255.255.255.255');
INSERT INTO "partition_network14" VALUES('slappart0','computer','slappart0','fc00::1','ffff:ffff:ffff::');
INSERT INTO "partition_network14" VALUES('slappart1','computer','slappart1','127.0.0.1','255.255.255.255');
INSERT INTO "partition_network14" VALUES('slappart1','computer','slappart1','fc00::1','ffff:ffff:ffff::');
INSERT INTO "partition_network14" VALUES('slappart2','computer','slappart2','127.0.0.1','255.255.255.255');
INSERT INTO "partition_network14" VALUES('slappart2','computer','slappart2','fc00::1','ffff:ffff:ffff::');
INSERT INTO "partition_network14" VALUES('slappart3','computer','slappart3','127.0.0.1','255.255.255.255');
INSERT INTO "partition_network14" VALUES('slappart3','computer','slappart3','fc00::1','ffff:ffff:ffff::');
INSERT INTO "partition_network14" VALUES('slappart4','computer','slappart4','127.0.0.1','255.255.255.255');
INSERT INTO "partition_network14" VALUES('slappart4','computer','slappart4','fc00::1','ffff:ffff:ffff::');
INSERT INTO "partition_network14" VALUES('slappart5','computer','slappart5','127.0.0.1','255.255.255.255');
INSERT INTO "partition_network14" VALUES('slappart5','computer','slappart5','fc00::1','ffff:ffff:ffff::');
INSERT INTO "partition_network14" VALUES('slappart6','computer','slappart6','127.0.0.1','255.255.255.255');
INSERT INTO "partition_network14" VALUES('slappart6','computer','slappart6','fc00::1','ffff:ffff:ffff::');
INSERT INTO "partition_network14" VALUES('slappart7','computer','slappart7','127.0.0.1','255.255.255.255');
INSERT INTO "partition_network14" VALUES('slappart7','computer','slappart7','fc00::1','ffff:ffff:ffff::');
INSERT INTO "partition_network14" VALUES('slappart8','computer','slappart8','127.0.0.1','255.255.255.255');
INSERT INTO "partition_network14" VALUES('slappart8','computer','slappart8','fc00::1','ffff:ffff:ffff::');
INSERT INTO "partition_network14" VALUES('slappart9','computer','slappart9','127.0.0.1','255.255.255.255');
INSERT INTO "partition_network14" VALUES('slappart9','computer','slappart9','fc00::1','ffff:ffff:ffff::');
CREATE TABLE forwarded_partition_request14 (
partition_reference VARCHAR(255), -- a.k.a source_instance_id
master_url VARCHAR(255)
);
COMMIT;
slapos/tests/test_slapproxy/generate_dump.sh
View file @ 189fa3a0
...@@ -41,6 +41,13 @@ EOF ...@@ -41,6 +41,13 @@ EOF
sqlite3 ${TMPD}/proxy.db < $DUMP_BEFORE sqlite3 ${TMPD}/proxy.db < $DUMP_BEFORE
slapos proxy start --cfg ${TMPD}/slapos.cfg & slapos proxy start --cfg ${TMPD}/slapos.cfg &
# If you are running tests locally and you want to refer to the slapos being tested,
# you can use the test python executable in this way:
# cd ../../..
# python -m slapos.cli.entry proxy start --cfg ${TMPD}/slapos.cfg &
# cd -
SLAPOS_PROXY_PID=$! SLAPOS_PROXY_PID=$!
curl --silent --retry-connrefused --retry 3 http://127.0.0.1:${PORT}/getComputerInformation?computer_id=$COMPUTER_ID curl --silent --retry-connrefused --retry 3 http://127.0.0.1:${PORT}/getComputerInformation?computer_id=$COMPUTER_ID
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7