slapproxy: Add local software release path

See merge request nexedi/slapos.core!288

slapos/cli/proxy_start.py

@@ -47,6 +47,8 @@ class ProxyStartCommand(ConfigCommand):
                         help='Port to use')
         ap.add_argument('--host',
                         help='Host to use')
+        ap.add_argument('--local-software-release-root',
+                        help='Root path for local Software Releases')
 
         return ap
 

slapos/proxy/__init__.py

@@ -93,6 +93,7 @@ def setupFlaskConfiguration(conf):
   app.config['DATABASE_URI'] = conf.database_uri
   app.config['software_product_list'] = conf.software_product_list
   app.config['multimaster'] = conf.multimaster
+  app.config['local_software_release_root'] = getattr(conf,'local_software_release_root', None)
 
 def connectDB():
   # if first connection, create an empty db at DATABASE_URI path

slapos/proxy/schema.sql

---version:14
+--version:15
+CREATE TABLE IF NOT EXISTS local_software_release_root%(version)s (
+  path VARCHAR(255)
+);
+INSERT INTO local_software_release_root%(version)s VALUES(NULL);
+
 CREATE TABLE IF NOT EXISTS software%(version)s (
   url VARCHAR(255),
   computer_reference VARCHAR(255) DEFAULT '%(computer)s',

slapos/proxy/views.py

@@ -32,6 +32,7 @@ import random
 import string
 import time
 import re
+import os
 from datetime import datetime
 from slapos.slap.slap import Computer, ComputerPartition, \
     SoftwareRelease, SoftwareInstance, NotFoundError
@@ -177,7 +178,12 @@ def _upgradeDatabaseIfNeeded():
 
   # Migrate all data to new tables
   app.logger.info('Old schema detected: Migrating old tables...')
-  for table in ('software', 'computer', 'partition', 'slave', 'partition_network'):
+  table_list = ('software', 'computer', 'partition', 'slave', 'partition_network')
+  if int(current_schema_version) >= 11:
+    table_list += ('forwarded_partition_request',)
+  if int(current_schema_version) >= 15:
+    table_list += ('local_software_release_root',)
+  for table in table_list:
     for row in execute_db(table, 'SELECT * from %s', db_version=current_schema_version):
       columns = ', '.join(row.keys())
       placeholders = ':'+', :'.join(row.keys())
@@ -188,6 +194,33 @@ def _upgradeDatabaseIfNeeded():
     g.db.execute("DROP table %s" % previous_table)
   g.db.commit()
 
+def _updateLocalSoftwareReleaseRootPathIfNeeded():
+  """
+  Update the local software release root path if it changed,
+  and rebase all URLs in the database relatively to the new path.
+  """
+  # Retrieve the current root path and replace it with the new one
+  current_root_path = execute_db('local_software_release_root', 'SELECT * from %s', one=True)['path'] or os.sep
+  new_root_path = app.config['local_software_release_root'] or os.sep
+  execute_db('local_software_release_root', 'UPDATE %s SET path=?', [new_root_path])
+  # Rebase all URLs relative to the new root path
+  if current_root_path != new_root_path:
+    app.logger.info('Updating local software release root path: %s --> %s', current_root_path, new_root_path)
+    def migrate_url(url):
+      if not url or urlparse(url).scheme:
+        app.logger.debug('Migrate URL ? N: %s is not a path', url)
+        return url
+      rel = os.path.relpath(url, current_root_path)
+      if rel.startswith(os.pardir + os.sep):
+        app.logger.debug('Migrate URL ? N: %s is not a subpath', url)
+        return url
+      new = os.path.join(new_root_path, rel)
+      app.logger.debug('Migrate URL ? Y: %s -> %s', url, new)
+      return new
+    g.db.create_function('migrate_url', 1, migrate_url)
+    execute_db('software', 'UPDATE %s SET url=migrate_url(url)')
+    execute_db('partition', 'UPDATE %s SET software_release=migrate_url(software_release)')
+
 is_schema_already_executed = False
 @app.before_request
 def before_request():
@@ -195,6 +228,7 @@ def before_request():
   global is_schema_already_executed
   if not is_schema_already_executed:
     _upgradeDatabaseIfNeeded()
+    _updateLocalSoftwareReleaseRootPathIfNeeded()
     is_schema_already_executed = True
 
 

slapos/slap/standalone.py

@@ -211,6 +211,7 @@ class SlapOSConfigWriter(ConfigWriter):
               host = {standalone_slapos._server_ip}
               port = {standalone_slapos._server_port}
               database_uri = {standalone_slapos._proxy_database}
+              local_software_release_root = {standalone_slapos._local_software_release_root}
 
               {partition_forward_configuration}
               """).format(**locals()))
@@ -318,6 +319,7 @@ class StandaloneSlapOS(object):
       shared_part_root=None,
       partition_forward_configuration=(),
       slapos_bin='slapos',
+      local_software_release_root=os.sep,
     ):
     # type: (str, str, int, str, Iterable[str], Optional[str], Optional[str], Optional[str], Iterable[Union[PartitionForwardConfiguration, PartitionForwardAsPartitionConfiguration]]) -> None
     """Constructor, creates a standalone slapos in `base_directory`.
@@ -332,6 +334,7 @@ class StandaloneSlapOS(object):
       * `shared_part_root` -- directory to hold shared parts software, default to "shared" in `base_directory`.
       * `partition_forward_configuration` -- configuration of partition request forwarding to external SlapOS master.
       * `slapos_bin` -- slapos executable to use, default to "slapos" (thus depending on the runtime PATH).
+      * `local_software_release_root` -- root for local Software Releases paths in the SlapOS proxy, default to `/`.
 
     Error cases:
       * `PathTooDeepError` when `base_directory` is too deep. Because of limitation
@@ -344,6 +347,7 @@ class StandaloneSlapOS(object):
     self._server_ip = server_ip
     self._server_port = server_port
     self._master_url = "http://{server_ip}:{server_port}".format(**locals())
+    self._local_software_release_root = local_software_release_root
 
     self._base_directory = base_directory
     self._shared_part_list = list(shared_part_list)

slapos/tests/test_slapproxy/database_dump_version_14.sql

+PRAGMA foreign_keys=OFF;
+BEGIN TRANSACTION;
+CREATE TABLE software14 (
+  url VARCHAR(255),
+  computer_reference VARCHAR(255) DEFAULT 'computer',
+  requested_state VARCHAR(255) DEFAULT 'available',
+  CONSTRAINT uniq PRIMARY KEY (url, computer_reference)
+);
+INSERT INTO "software14" VALUES('/srv/slapgrid//srv//runner/project//slapos/software.cfg','computer','available');
+CREATE TABLE computer14 (
+  reference VARCHAR(255) DEFAULT 'computer',
+  address VARCHAR(255),
+  netmask VARCHAR(255),
+  CONSTRAINT uniq PRIMARY KEY (reference)
+);
+INSERT INTO "computer14" VALUES('computer','127.0.0.1','255.255.255.255');
+CREATE TABLE partition14 (
+  reference VARCHAR(255),
+  computer_reference VARCHAR(255) DEFAULT 'computer',
+  slap_state VARCHAR(255) DEFAULT 'free',
+  software_release VARCHAR(255),
+  xml TEXT,
+  connection_xml TEXT,
+  slave_instance_list TEXT,
+  software_type VARCHAR(255),
+  partition_reference VARCHAR(255), -- name of the instance
+  requested_by VARCHAR(255), -- only used for debugging,
+                             -- slapproxy does not support proper scope
+  requested_state VARCHAR(255) NOT NULL DEFAULT 'started',
+  timestamp REAL,
+  CONSTRAINT uniq PRIMARY KEY (reference, computer_reference)
+);
+INSERT INTO "partition14" VALUES('slappart0','computer','busy','/srv/slapgrid//srv//runner/project//slapos/software.cfg','<?xml version=''1.0'' encoding=''utf-8''?>
+<instance>
+  <parameter id="json">{
+  "site-id": "erp5"
+  }
+}</parameter>
+</instance>
+',NULL,NULL,'production','slapos',NULL,'started',NULL);
+INSERT INTO "partition14" VALUES('slappart1','computer','busy','/srv/slapgrid//srv//runner/project//slapos/software.cfg','<?xml version=''1.0'' encoding=''utf-8''?>
+<instance/>
+','<?xml version=''1.0'' encoding=''utf-8''?>
+<instance>
+  <parameter id="url">mysql://127.0.0.1:45678/erp5</parameter>
+</instance>
+',NULL,'mariadb','MariaDB DataBase','slappart0','started',NULL);
+INSERT INTO "partition14" VALUES('slappart2','computer','busy','/srv/slapgrid//srv//runner/project//slapos/software.cfg','<?xml version=''1.0'' encoding=''utf-8''?>
+<instance>
+  <parameter id="cloudooo-json"></parameter>
+</instance>
+','<?xml version=''1.0'' encoding=''utf-8''?>
+<instance>
+  <parameter id="url">cloudooo://127.0.0.1:23000/</parameter>
+</instance>
+',NULL,'cloudooo','Cloudooo','slappart0','started',NULL);
+INSERT INTO "partition14" VALUES('slappart3','computer','busy','/srv/slapgrid//srv//runner/project//slapos/software.cfg','<?xml version=''1.0'' encoding=''utf-8''?>
+<instance/>
+','<?xml version=''1.0'' encoding=''utf-8''?>
+<instance>
+  <parameter id="url">memcached://127.0.0.1:11000/</parameter>
+</instance>
+',NULL,'memcached','Memcached','slappart0','started',NULL);
+INSERT INTO "partition14" VALUES('slappart4','computer','busy','/srv/slapgrid//srv//runner/project//slapos/software.cfg','<?xml version=''1.0'' encoding=''utf-8''?>
+<instance/>
+','<?xml version=''1.0'' encoding=''utf-8''?>
+<instance>
+  <parameter id="url">memcached://127.0.0.1:13301/</parameter>
+</instance>
+',NULL,'kumofs','KumoFS','slappart0','started',NULL);
+INSERT INTO "partition14" VALUES('slappart5','computer','busy','/srv/slapgrid//srv//runner/project//slapos/software.cfg','<?xml version=''1.0'' encoding=''utf-8''?>
+<instance>
+  <parameter id="kumofs-url">memcached://127.0.0.1:13301/</parameter>
+  <parameter id="memcached-url">memcached://127.0.0.1:11000/</parameter>
+  <parameter id="cloudooo-url">cloudooo://127.0.0.1:23000/</parameter>
+</instance>
+','<?xml version=''1.0'' encoding=''utf-8''?>
+<instance>
+  <parameter id="url">https://[fc00::1]:10001</parameter>
+</instance>
+',NULL,'tidstorage','TidStorage','slappart0','started',NULL);
+INSERT INTO "partition14" VALUES('slappart6','computer','free',NULL,NULL,NULL,NULL,NULL,NULL,NULL,'started',NULL);
+INSERT INTO "partition14" VALUES('slappart7','computer','free',NULL,NULL,NULL,NULL,NULL,NULL,NULL,'started',NULL);
+INSERT INTO "partition14" VALUES('slappart8','computer','free',NULL,NULL,NULL,NULL,NULL,NULL,NULL,'started',NULL);
+INSERT INTO "partition14" VALUES('slappart9','computer','free',NULL,NULL,NULL,NULL,NULL,NULL,NULL,'started',NULL);
+CREATE TABLE slave14 (
+  reference VARCHAR(255), -- unique slave reference
+  computer_reference VARCHAR(255) DEFAULT 'computer',
+  connection_xml TEXT,
+  hosted_by VARCHAR(255),
+  asked_by VARCHAR(255) -- only used for debugging,
+                        -- slapproxy does not support proper scope
+);
+CREATE TABLE partition_network14 (
+  partition_reference VARCHAR(255),
+  computer_reference VARCHAR(255) DEFAULT 'computer',
+  reference VARCHAR(255),
+  address VARCHAR(255),
+  netmask VARCHAR(255)
+);
+INSERT INTO "partition_network14" VALUES('slappart0','computer','slappart0','127.0.0.1','255.255.255.255');
+INSERT INTO "partition_network14" VALUES('slappart0','computer','slappart0','fc00::1','ffff:ffff:ffff::');
+INSERT INTO "partition_network14" VALUES('slappart1','computer','slappart1','127.0.0.1','255.255.255.255');
+INSERT INTO "partition_network14" VALUES('slappart1','computer','slappart1','fc00::1','ffff:ffff:ffff::');
+INSERT INTO "partition_network14" VALUES('slappart2','computer','slappart2','127.0.0.1','255.255.255.255');
+INSERT INTO "partition_network14" VALUES('slappart2','computer','slappart2','fc00::1','ffff:ffff:ffff::');
+INSERT INTO "partition_network14" VALUES('slappart3','computer','slappart3','127.0.0.1','255.255.255.255');
+INSERT INTO "partition_network14" VALUES('slappart3','computer','slappart3','fc00::1','ffff:ffff:ffff::');
+INSERT INTO "partition_network14" VALUES('slappart4','computer','slappart4','127.0.0.1','255.255.255.255');
+INSERT INTO "partition_network14" VALUES('slappart4','computer','slappart4','fc00::1','ffff:ffff:ffff::');
+INSERT INTO "partition_network14" VALUES('slappart5','computer','slappart5','127.0.0.1','255.255.255.255');
+INSERT INTO "partition_network14" VALUES('slappart5','computer','slappart5','fc00::1','ffff:ffff:ffff::');
+INSERT INTO "partition_network14" VALUES('slappart6','computer','slappart6','127.0.0.1','255.255.255.255');
+INSERT INTO "partition_network14" VALUES('slappart6','computer','slappart6','fc00::1','ffff:ffff:ffff::');
+INSERT INTO "partition_network14" VALUES('slappart7','computer','slappart7','127.0.0.1','255.255.255.255');
+INSERT INTO "partition_network14" VALUES('slappart7','computer','slappart7','fc00::1','ffff:ffff:ffff::');
+INSERT INTO "partition_network14" VALUES('slappart8','computer','slappart8','127.0.0.1','255.255.255.255');
+INSERT INTO "partition_network14" VALUES('slappart8','computer','slappart8','fc00::1','ffff:ffff:ffff::');
+INSERT INTO "partition_network14" VALUES('slappart9','computer','slappart9','127.0.0.1','255.255.255.255');
+INSERT INTO "partition_network14" VALUES('slappart9','computer','slappart9','fc00::1','ffff:ffff:ffff::');
+CREATE TABLE forwarded_partition_request14 (
+  partition_reference VARCHAR(255), -- a.k.a source_instance_id
+  master_url VARCHAR(255)
+);
+COMMIT;

slapos/tests/test_slapproxy/generate_dump.sh

@@ -41,6 +41,13 @@ EOF
 sqlite3 ${TMPD}/proxy.db < $DUMP_BEFORE
 
 slapos proxy start --cfg ${TMPD}/slapos.cfg &
+
+# If you are running tests locally and you want to refer to the slapos being tested,
+# you can use the test python executable in this way:
+# cd ../../..
+# python -m slapos.cli.entry proxy start --cfg ${TMPD}/slapos.cfg &
+# cd -
+
 SLAPOS_PROXY_PID=$!
 
 curl --silent --retry-connrefused --retry 3 http://127.0.0.1:${PORT}/getComputerInformation?computer_id=$COMPUTER_ID

slapos/tests/test_slapproxy/slapos_multimaster.cfg.in

 [slapos]
-software_root = %(tempdir)s/opt/slapgrid
-instance_root = %(tempdir)s/srv/slapgrid
+software_root = %(rootdir)s/opt/slapgrid
+instance_root = %(rootdir)s/srv/slapgrid
 master_url = %(proxyaddr)s
 computer_id = computer
 
 [slapproxy]
 host = 127.0.0.1
 port = 8080
-database_uri = %(tempdir)s/lib/proxy.db
+database_uri = %(rootdir)s/lib/proxy.db
 
 # Here goes the list of slapos masters that slapproxy can contact
 # Each section beginning by multimaster is a different SlapOS Master, represented by arbitrary name.