From c755cba8e0e9ee2c1d4b08a4673c8ed859fbc7e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9rome=20Perrin?= <jerome@nexedi.com> Date: Mon, 11 May 2020 09:40:36 +0900 Subject: [PATCH] slapos/proxy: support forwarding requests as a partition In SlapOS, both users or partitions can requests partitions. With the multimaster support of slapos proxy, all requests where made as a user, but in the case of recursive slapos - where a partition from the "outer" slapos includes an "inner" slapos, it makes sense to forward partitions requests as the partition in the "outer" slapos - this way when this partition is destroyed all partitions that might have been requested are also destroyed. To request as partition, the multi-master entry must define two extra keys: - computer, that can be optained by $${slap-configuration:computer} from instance buildout - partition, that can be optained by $${slap-configuration:partition} When these are not set, the request will be made as a user, like it was the case before. We also change the test to unset SLAPGRID_INSTANCE_ROOT because this implementation has side effect - the environment variable is set and never unset. Without this, test fail when running the full test suite because a previous test was leaking SLAPGRID_INSTANCE_ROOT. This is definitely something we'll have to improve later. --- slapos/proxy/views.py | 45 ++++++++++---- slapos/tests/test_slapproxy.py | 59 +++++++++++++++++++ .../test_slapproxy/slapos_multimaster.cfg.in | 16 +++++ 3 files changed, 107 insertions(+), 13 deletions(-) diff --git a/slapos/proxy/views.py b/slapos/proxy/views.py index f6da744fd..57a2cfb56 100644 --- a/slapos/proxy/views.py +++ b/slapos/proxy/views.py @@ -563,23 +563,42 @@ def forwardRequestToExternalMaster(master_url, request_form): slap.initializeConnection(master_url) partition_reference = unicode2str(request_form['partition_reference']) + filter_kw = loads(request_form['filter_xml'].encode('utf-8')) + partition_parameter_kw = loads(request_form['partition_parameter_xml'].encode('utf-8')) + + app.logger.info("Forwarding request of %s to %s", partition_reference, master_url) + app.logger.debug("request_form: %s", request_form) + # Store in database execute_db('forwarded_partition_request', 'INSERT OR REPLACE INTO %s values(:partition_reference, :master_url)', {'partition_reference':partition_reference, 'master_url': master_url}) - new_request_form = request_form.copy() - filter_kw = loads(new_request_form['filter_xml'].encode('utf-8')) - filter_kw['source_instance_id'] = partition_reference - - partition = slap.registerOpenOrder().request( - software_release=request_form['software_release'], - partition_reference=request_form['partition_reference'], - partition_parameter_kw=loads(request_form['partition_parameter_xml'].encode('utf-8')), - software_type=request_form.get('software_type', ''), - filter_kw=filter_kw, - state=loads(request_form['state'].encode('utf-8')), - shared=loads(request_form['shared_xml'].encode('utf-8')), - ) + if master_entry.get('computer') and master_entry.get('partition'): + app.logger.debug("requesting from partition %s", master_entry) + # XXX ComputerPartition.request and OpenOrder.request have different signatures + partition = slap.registerComputerPartition( + master_entry['computer'], + master_entry['partition'], + ).request( + software_release=request_form['software_release'], + software_type=request_form.get('software_type', ''), + partition_reference=partition_reference, + shared=loads(request_form['shared_xml'].encode('utf-8')), + partition_parameter_kw=partition_parameter_kw, + filter_kw=filter_kw, + state=loads(request_form['state'].encode('utf-8')), + ) + else: + filter_kw['source_instance_id'] = partition_reference + partition = slap.registerOpenOrder().request( + software_release=request_form['software_release'], + partition_reference=partition_reference, + partition_parameter_kw=partition_parameter_kw, + software_type=request_form.get('software_type', ''), + filter_kw=filter_kw, + state=loads(request_form['state'].encode('utf-8')), + shared=loads(request_form['shared_xml'].encode('utf-8')), + ) # XXX move to other end partition._master_url = master_url # type: ignore diff --git a/slapos/tests/test_slapproxy.py b/slapos/tests/test_slapproxy.py index 832ccd1c6..110161fcd 100644 --- a/slapos/tests/test_slapproxy.py +++ b/slapos/tests/test_slapproxy.py @@ -78,6 +78,7 @@ class BasicMixin(object): logging.basicConfig(level=logging.DEBUG) self.setFiles() self.startProxy() + os.environ.pop('SLAPGRID_INSTANCE_ROOT', None) def createSlapOSConfigurationFile(self): with open(self.slapos_cfg, 'w') as f: @@ -1455,6 +1456,25 @@ database_uri = %(tempdir)s/lib/external_proxy.db self.external_proxy_slap._connection_helper.POST('/loadComputerConfigurationFromXML', data=request_dict) + def external_proxy_create_requested_partition(self): + # type: () -> None + """Create an already requested partition as slappart0, so that we can + request from this partition. + """ + external_slap = slapos.slap.slap() + external_slap.initializeConnection(self.external_master_url) + external_slap.registerSupply().supply( + 'https://example.com/dummy/software.cfg', + computer_guid=self.external_computer_id, + ) + partition = external_slap.registerOpenOrder().request( + 'https://example.com/dummy/software.cfg', + 'instance', + ) + # XXX this has to match what is set in slapos_multimaster.cfg.in + self.assertEqual('external_computer', partition.slap_computer_id) + self.assertEqual('slappart0', partition.slap_computer_partition_id) + def _checkInstanceIsFowarded(self, name, partition_parameter_kw, software_release): """ Test there is no instance on local proxy. @@ -1580,6 +1600,45 @@ database_uri = %(tempdir)s/lib/external_proxy.db self.assertEqual(self.external_software_release, partition.getSoftwareRelease()) self.assertEqual({}, partition.getConnectionParameterDict()) + def testForwardRequestFromPartition(self): + """ + Test that instance request is forwarded and requested from computer partition. + """ + dummy_parameter_dict = {'foo': 'bar'} + instance_reference = 'MyFirstInstance' + self.format_for_number_of_partitions(1) + self.external_proxy_format_for_number_of_partitions(2) + self.external_proxy_create_requested_partition() + + partition = self.request( + 'https://example.com/request/from/partition/software.cfg', + None, + instance_reference, + 'slappart0', + partition_parameter_kw=dummy_parameter_dict, + ) + + instance_parameter_dict = partition.getInstanceParameterDict() + instance_parameter_dict.pop('timestamp') + self.assertEqual(dummy_parameter_dict, instance_parameter_dict) + self.assertEqual('https://example.com/request/from/partition/software.cfg', partition.getSoftwareRelease()) + self.assertEqual({}, partition.getConnectionParameterDict()) + + with sqlite3.connect(os.path.join( + self._tempdir, + 'lib', + 'external_proxy.db', + )) as db: + requested_by = slapos.proxy.views.execute_db( + "partition", "select reference, requested_by from %s", db=db) + self.assertEqual([{ + 'reference': 'slappart0', + 'requested_by': None + }, { + 'reference': 'slappart1', + 'requested_by': 'slappart0' + }], requested_by) + def testRequestToCurrentMaster(self): """ Explicitely ask deployment of an instance to current master diff --git a/slapos/tests/test_slapproxy/slapos_multimaster.cfg.in b/slapos/tests/test_slapproxy/slapos_multimaster.cfg.in index 25a5ecfb6..d46cf314b 100644 --- a/slapos/tests/test_slapproxy/slapos_multimaster.cfg.in +++ b/slapos/tests/test_slapproxy/slapos_multimaster.cfg.in @@ -27,3 +27,19 @@ software_release_list = software_release_list = http://mywebsite.me/exteral_software_release.cfg +# Request as a computer partition, so that requested partitions are linked +# to the partition requesting them. +[multimaster/https://slap.example.com] +key = /path/to/cert.key +cert = /path/to/cert.cert +computer = COMP-12345 +partition = slappart1 +software_release_list = + https://example.com/software.cfg + +[multimaster/http://%(external_proxy_host)s:%(external_proxy_port)s/] +# No certificate here: it is http. +computer = external_computer +partition = slappart0 +software_release_list = + https://example.com/request/from/partition/software.cfg -- 2.30.9