Add multimaster support to slapproxy.

parent 955bc036
......@@ -30,6 +30,7 @@
import logging
from slapos.proxy.views import app
def _generateSoftwareProductListFromString(software_product_list_string):
"""
......@@ -52,33 +53,48 @@ def _generateSoftwareProductListFromString(software_product_list_string):
class ProxyConfig(object):
def __init__(self, logger):
self.logger = logger
self.multimaster = {}
self.software_product_list = []
def mergeConfig(self, args, configp):
# Set options parameters
# Set arguments parameters (from CLI) as members of self
for option, value in args.__dict__.items():
setattr(self, option, value)
# Merge the arguments and configuration
for section in ("slapproxy", "slapos"):
for section in configp.sections():
configuration_dict = dict(configp.items(section))
for key in configuration_dict:
if not getattr(self, key, None):
setattr(self, key, configuration_dict[key])
if section in ("slapproxy", "slapos"):
# Merge the arguments and configuration as member of self
for key in configuration_dict:
if not getattr(self, key, None):
setattr(self, key, configuration_dict[key])
elif section.startswith('multimaster/'):
# Merge multimaster configuration if any
# XXX: check for duplicate SR entries
for key, value in configuration_dict.iteritems():
if key == 'software_release_list':
# Split multi-lines values
configuration_dict[key] = [line.strip() for line in value.strip().split('\n')]
self.multimaster[section.split('multimaster/')[1]] = configuration_dict
def setConfig(self):
if not self.database_uri:
raise ValueError('database-uri is required.')
# XXX: check for duplicate SR entries.
self.software_product_list = _generateSoftwareProductListFromString(
getattr(self, 'software_product_list', ''))
def setupFlaskConfiguration(conf):
app.config['computer_id'] = conf.computer_id
app.config['DATABASE_URI'] = conf.database_uri
app.config['software_product_list'] = conf.software_product_list
app.config['multimaster'] = conf.multimaster
def do_proxy(conf):
from slapos.proxy.views import app
for handler in conf.logger.handlers:
app.logger.addHandler(handler)
app.logger.setLevel(logging.INFO)
app.config['computer_id'] = conf.computer_id
app.config['DATABASE_URI'] = conf.database_uri
app.config['software_product_list'] = \
_generateSoftwareProductListFromString(
getattr(conf, 'software_product_list', ""))
app.run(host=conf.host, port=int(conf.port))
setupFlaskConfiguration(conf)
app.run(host=conf.host, port=int(conf.port), threaded=True)
......@@ -45,3 +45,7 @@ CREATE TABLE IF NOT EXISTS partition_network%(version)s (
netmask VARCHAR(255)
);
CREATE TABLE IF NOT EXISTS forwarded_partition_request%(version)s (
partition_reference VARCHAR(255), -- a.k.a source_instance_id
master_url VARCHAR(255)
);
......@@ -29,10 +29,13 @@
##############################################################################
from lxml import etree
import random
import sqlite3
import string
from slapos.slap.slap import Computer, ComputerPartition, \
SoftwareRelease, SoftwareInstance, NotFoundError
from slapos.proxy.db_version import DB_VERSION
import slapos.slap
from flask import g, Flask, request, abort
import xml_marshaller
......@@ -112,14 +115,16 @@ def partitiondict2partition(partition):
return slap_partition
def execute_db(table, query, args=(), one=False, db_version=None, log=False):
def execute_db(table, query, args=(), one=False, db_version=None, log=False, db=None):
if not db:
db = g.db
if not db_version:
db_version = DB_VERSION
query = query % (table + db_version,)
if log:
print query
try:
cur = g.db.execute(query, args)
cur = db.execute(query, args)
except:
app.logger.error('There was some issue during processing query %r on table %r with args %r' % (query, table, args))
raise
......@@ -161,7 +166,7 @@ def _upgradeDatabaseIfNeeded():
# If version of current database is not old, do nothing
if current_schema_version == DB_VERSION:
return
schema = app.open_resource('schema.sql')
schema = schema.read() % dict(version=DB_VERSION, computer=app.config['computer_id'])
g.db.cursor().executescript(schema)
......@@ -330,9 +335,7 @@ def supplySupply():
@app.route('/requestComputerPartition', methods=['POST'])
def requestComputerPartition():
parsed_form_dict = parseRequestComputerPartitionForm(request.form)
# By default, ALWAYS request instance on default computer
parsed_form_dict['filter_kw'].setdefault('computer_guid', app.config['computer_id'])
parsed_request_dict = parseRequestComputerPartitionForm(request.form)
# Is it a slave instance?
slave = loads(request.form.get('shared_xml', EMPTY_DICT_XML).encode())
......@@ -341,29 +344,36 @@ def requestComputerPartition():
if slave:
# XXX: change schema to include a simple "partition_reference" which
# is name of the instance. Then, no need to do complex search here.
slave_reference = parsed_form_dict['partition_id'] + '_' + parsed_form_dict['partition_reference']
requested_computer_id = parsed_form_dict['filter_kw']['computer_guid']
slave_reference = parsed_request_dict['partition_id'] + '_' + parsed_request_dict['partition_reference']
requested_computer_id = parsed_request_dict['filter_kw'].get('computer_guid', app.config['computer_id'])
matching_partition = getAllocatedSlaveInstance(slave_reference, requested_computer_id)
else:
matching_partition = getAllocatedInstance(parsed_form_dict['partition_reference'])
matching_partition = getAllocatedInstance(parsed_request_dict['partition_reference'])
if matching_partition:
# Then the instance is already allocated, just update it
# XXX: split request and request slave into different update/allocate functions and simplify.
# By default, ALWAYS request instance on default computer
parsed_request_dict['filter_kw'].setdefault('computer_guid', app.config['computer_id'])
if slave:
software_instance = requestSlave(**parsed_form_dict)
software_instance = requestSlave(**parsed_request_dict)
else:
software_instance = requestNotSlave(**parsed_form_dict)
software_instance = requestNotSlave(**parsed_request_dict)
else:
# Instance is not yet allocated: try to do it.
# XXX Insert here multimaster allocation
external_master_url = isRequestToBeForwardedToExternalMaster(parsed_request_dict)
if external_master_url:
return forwardRequestToExternalMaster(external_master_url, request.form)
# XXX add support for automatic deployment on specific node depending on available SR and partitions on each Node.
# Note: only deploy on default node if SLA not specified
# Note: It only deploys on default node if SLA not specified
# XXX: split request and request slave into different update/allocate functions and simplify.
# By default, ALWAYS request instance on default computer
parsed_request_dict['filter_kw'].setdefault('computer_guid', app.config['computer_id'])
if slave:
software_instance = requestSlave(**parsed_form_dict)
software_instance = requestSlave(**parsed_request_dict)
else:
software_instance = requestNotSlave(**parsed_form_dict)
software_instance = requestNotSlave(**parsed_request_dict)
return dumps(software_instance)
......@@ -378,15 +388,123 @@ def parseRequestComputerPartitionForm(form):
parsed_dict['partition_id'] = form.get('computer_partition_id', '').encode()
parsed_dict['partition_parameter_kw'] = loads(form.get('partition_parameter_xml', EMPTY_DICT_XML).encode())
parsed_dict['filter_kw'] = loads(form.get('filter_xml', EMPTY_DICT_XML).encode())
# Note: currently ignored on for slave instance (slave instances
# Note: currently ignored for slave instance (slave instances
# are always started).
parsed_dict['requested_state'] = loads(form.get('state').encode())
return parsed_dict
run_id = ''.join([random.choice(string.ascii_letters + string.digits) for n in xrange(32)])
def checkIfMasterIsCurrentMaster(master_url):
"""
Because there are several ways to contact this server, we can't easily check
in a request() if master_url is ourself or not. So we contact master_url,
and if it returns an ID we know: it is ourself
"""
# Dumb way: compare with listening host/port
host = request.host
port = request.environ['SERVER_PORT']
if master_url == 'http://%s:%s/' % (host, port):
return True
# Hack way: call ourself
slap = slapos.slap.slap()
slap.initializeConnection(master_url)
try:
master_run_id = slap._connection_helper.GET('/getRunId')
except:
return False
if master_run_id == run_id:
return True
return False
@app.route('/getRunId', methods=['GET'])
def getRunId():
return run_id
def checkMasterUrl(master_url):
"""
Check if master_url doesn't represent ourself, and check if it is whitelisted
in multimaster configuration.
"""
if not master_url:
return False
if checkIfMasterIsCurrentMaster(master_url):
# master_url is current server: don't forward
return False
master_entry = app.config.get('multimaster').get(master_url, None)
# Check if this master is known
if not master_entry:
# Check if it is ourself
if not master_url.startswith('https') and checkIfMasterIsCurrentMaster(master_url):
return False
app.logger.warning('External SlapOS Master URL %s is not listed in multimaster list.' % master_url)
abort(404)
return True
def isRequestToBeForwardedToExternalMaster(parsed_request_dict):
"""
Check if we HAVE TO forward the request.
Several cases:
* The request specifies a master_url in filter_kw
* The software_release of the request is in a automatic forward list
"""
master_url = parsed_request_dict['filter_kw'].get('master_url')
if checkMasterUrl(master_url):
# Don't allocate the instance locally, but forward to specified master
return master_url
software_release = parsed_request_dict['software_release']
for mutimaster_url, mutimaster_entry in app.config.get('multimaster').iteritems():
if software_release in mutimaster_entry['software_release_list']:
# Don't allocate the instance locally, but forward to specified master
return mutimaster_url
return None
def forwardRequestToExternalMaster(master_url, request_form):
"""
Forward instance request to external SlapOS Master.
"""
master_entry = app.config.get('multimaster').get(master_url, {})
key_file = master_entry.get('key')
cert_file = master_entry.get('cert')
if master_url.startswith('https') and (not key_file or not cert_file):
app.logger.warning('External master %s configuration did not specify key or certificate.' % master_url)
abort(404)
if master_url.startswith('https') and not master_url.startswith('https') and (key_file or cert_file):
app.logger.warning('External master %s configurqtion specifies key or certificate but is using plain http.' % master_url)
abort(404)
slap = slapos.slap.slap()
if key_file:
slap.initializeConnection(master_url, key_file=key_file, cert_file=cert_file)
else:
slap.initializeConnection(master_url)
partition_reference = request_form['partition_reference'].encode()
# Store in database
execute_db('forwarded_partition_request', 'INSERT OR REPLACE INTO %s values(:partition_reference, :master_url)',
{'partition_reference':partition_reference, 'master_url': master_url})
new_request_form = request_form.copy()
filter_kw = loads(new_request_form['filter_xml'].encode())
filter_kw['source_instance_id'] = partition_reference
new_request_form['filter_xml'] = dumps(filter_kw)
partition = loads(slap._connection_helper.POST('/requestComputerPartition', new_request_form))
# XXX move to other end
partition._master_url = master_url
return dumps(partition)
def getAllocatedInstance(partition_reference):
"""
Look for existence of instance, if so return the
Look for existence of instance, if so return the
corresponding partition dict, else return None
"""
args = []
......@@ -398,7 +516,7 @@ def getAllocatedInstance(partition_reference):
def getAllocatedSlaveInstance(slave_reference, requested_computer_id):
"""
Look for existence of instance, if so return the
Look for existence of instance, if so return the
corresponding partition dict, else return None
"""
args = []
......
# -*- coding: utf-8 -*-
# vim: set et sts=2:
##############################################################################
#
# Copyright (c) 2012 Vifib SARL and Contributors.
# Copyright (c) 2012, 2013, 2014 Vifib SARL and Contributors.
# All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
......@@ -31,7 +32,9 @@ import ConfigParser
import os
import logging
import shutil
import subprocess
import tempfile
import time
import unittest
import xml_marshaller
from xml_marshaller.xml_marshaller import loads, dumps
......@@ -69,14 +72,7 @@ class BasicMixin(object):
self.setFiles()
self.startProxy()
def setFiles(self):
"""
Set environment to run slapproxy
"""
self.slapos_cfg = os.path.join(self._tempdir, 'slapos.cfg')
self.proxy_db = os.path.join(self._tempdir, 'lib', 'proxy.db')
self.proxyaddr = 'http://127.0.0.1:8080/'
self.computer_id = 'computer'
def createSlapOSConfigurationFile(self):
open(self.slapos_cfg, 'w').write("""[slapos]
software_root = %(tempdir)s/opt/slapgrid
instance_root = %(tempdir)s/srv/slapgrid
......@@ -87,6 +83,16 @@ host = 127.0.0.1
port = 8080
database_uri = %(tempdir)s/lib/proxy.db
""" % {'tempdir': self._tempdir, 'proxyaddr': self.proxyaddr})
def setFiles(self):
"""
Set environment to run slapproxy
"""
self.slapos_cfg = os.path.join(self._tempdir, 'slapos.cfg')
self.proxy_db = os.path.join(self._tempdir, 'lib', 'proxy.db')
self.proxyaddr = 'http://localhost:80/'
self.computer_id = 'computer'
self.createSlapOSConfigurationFile()
for directory in ['opt', 'srv', 'lib']:
path = os.path.join(self._tempdir, directory)
os.mkdir(path)
......@@ -101,10 +107,8 @@ database_uri = %(tempdir)s/lib/proxy.db
conf.mergeConfig(ProxyOption(self.proxy_db), configp)
conf.setConfig()
views.app.config['TESTING'] = True
views.app.config['computer_id'] = self.computer_id
views.app.config['DATABASE_URI'] = self.proxy_db
views.app.config['HOST'] = conf.host
views.app.config['port'] = conf.port
slapos.proxy.setupFlaskConfiguration(conf)
self.app_config = views.app.config
self.app = views.app.test_client()
......@@ -850,7 +854,280 @@ class TestMultiNodeSupport(MasterMixin):
except slapos.slap.NotFoundError:
self.fail('Could not fetch informations for registered computer.')
class TestMultiMasterSupport(MasterMixin):
"""
Test multimaster support in slapproxy.
"""
external_software_release = 'http://mywebsite.me/exteral_software_release.cfg'
software_release_not_in_list = 'http://mywebsite.me/exteral_software_release_not_listed.cfg'
def setUp(self):
# XXX don't use lo
self.external_proxy_host = '127.0.0.1'
self.external_proxy_port = 8281
self.external_master_url = 'http://%s:%s' % (self.external_proxy_host, self.external_proxy_port)
self.external_computer_id = 'external_computer'
super(TestMultiMasterSupport, self).setUp()
self.db = sqlite3.connect(self.proxy_db)
self.external_slapproxy_configuration_file_location = os.path.join(
self._tempdir, 'external_slapos.cfg')
self.createExternalProxyConfigurationFile()
self.startExternalProxy()
def tearDown(self):
self.external_proxy_process.kill()
super(TestMultiMasterSupport, self).tearDown()
def createExternalProxyConfigurationFile(self):
open(self.external_slapproxy_configuration_file_location, 'w').write("""[slapos]
computer_id = %(external_computer_id)s
[slapproxy]
host = 127.0.0.1
port = %(port)s
database_uri = %(tempdir)s/lib/external_proxy.db
""" % {
'tempdir': self._tempdir,
'port': self.external_proxy_port,
'external_computer_id': self.external_computer_id
})
def startExternalProxy(self):
"""
Start external slapproxy
"""
# XXX use current dev version, not standard one installed through package
self.external_proxy_process = subprocess.Popen(['slapos', 'proxy', 'start', '--cfg', self.external_slapproxy_configuration_file_location ])
# Wait a bit for proxy to be started
time.sleep(0.5)
self.external_proxy_slap = slapos.slap.slap()
self.external_proxy_slap.initializeConnection(self.external_master_url)
def createSlapOSConfigurationFile(self):
"""
Overwrite default slapos configuration file to enable specific multimaster
behaviours.
"""
configuration = pkg_resources.resource_stream(
'slapos.tests.slapproxy', 'slapos_multimaster.cfg.in'
).read() % {
'tempdir': self._tempdir, 'proxyaddr': self.proxyaddr,
'external_proxy_host': self.external_proxy_host,
'external_proxy_port': self.external_proxy_port
}
open(self.slapos_cfg, 'w').write(configuration)
def external_proxy_supply(self, url, computer_id=None):
if computer_id is None:
computer_id = self.external_computer_id
self.external_proxy_slap.registerSupply().supply(url, computer_id)
def external_proxy_add_free_partition(self, partition_amount, computer_id=None):
"""
Will simulate a slapformat first run
and create "partition_amount" partitions
"""
if not computer_id:
computer_id = self.external_computer_id
computer_dict = {
'reference': computer_id,
'address': '123.456.789',
'netmask': 'fffffffff',
'partition_list': [],
}
for i in range(partition_amount):
partition_example = {
'reference': 'slappart%s' % i,
'address_list': [
{'addr': '1.2.3.4', 'netmask': '255.255.255.255'},
{'addr': '4.3.2.1', 'netmask': '255.255.255.255'}
],
'tap': {'name': 'tap0'},
}
computer_dict['partition_list'].append(partition_example)
request_dict = {
'computer_id': self.computer_id,
'xml': xml_marshaller.xml_marshaller.dumps(computer_dict),
}
self.external_proxy_slap._connection_helper.POST('/loadComputerConfigurationFromXML',
parameter_dict=request_dict)
def _checkInstanceIsFowarded(self, name, partition_parameter_kw, software_release):
"""
Test there is no instance on local proxy.
Test there is instance on external proxy.
Test there is instance reference in external table of databse of local proxy.
"""
# Test it has been correctly added to local database
forwarded_instance_list = slapos.proxy.views.execute_db('forwarded_partition_request', 'SELECT * from %s', db=self.db)
self.assertEqual(len(forwarded_instance_list), 1)
forwarded_instance = forwarded_instance_list[0]
self.assertEqual(forwarded_instance['partition_reference'], name)
self.assertEqual(forwarded_instance['master_url'], self.external_master_url)
# Test there is nothing allocated locally
computer = loads(self.app.get(
'/getFullComputerInformation?computer_id=%s' % self.computer_id
).data)
self.assertEqual(
computer._computer_partition_list[0]._software_release_document,
None
)
# Test there is an instance allocated in external master
external_slap = slapos.slap.slap()
external_slap.initializeConnection(self.external_master_url)
external_computer = external_slap.registerComputer(self.external_computer_id)
external_partition = external_computer.getComputerPartitionList()[0]
for k, v in partition_parameter_kw.iteritems():
self.assertEqual(
external_partition.getInstanceParameter(k),
v
)
self.assertEqual(
external_partition._software_release_document._software_release,
software_release
)
def _checkInstanceIsAllocatedLocally(self, name, partition_parameter_kw, software_release):
"""
Test there is one instance on local proxy.
Test there NO is instance reference in external table of databse of local proxy.
Test there is not instance on external proxy.
"""
# Test it has NOT been added to local database
forwarded_instance_list = slapos.proxy.views.execute_db('forwarded_partition_request', 'SELECT * from %s', db=self.db)
self.assertEqual(len(forwarded_instance_list), 0)
# Test there is an instance allocated locally
computer = loads(self.app.get(
'/getFullComputerInformation?computer_id=%s' % self.computer_id
).data)
partition = computer._computer_partition_list[0]
for k, v in partition_parameter_kw.iteritems():
self.assertEqual(
partition.getInstanceParameter(k),
v
)
self.assertEqual(
partition._software_release_document._software_release,
software_release
)
# Test there is NOT instance allocated in external master
external_slap = slapos.slap.slap()
external_slap.initializeConnection(self.external_master_url)
external_computer = external_slap.registerComputer(self.external_computer_id)
external_partition = external_computer.getComputerPartitionList()[0]
self.assertEqual(
external_partition._software_release_document,
None
)
def testForwardToMasterInList(self):
"""
Test that explicitely asking a master_url in SLA causes
proxy to forward request to this master.
"""
dummy_parameter_dict = {'foo': 'bar'}
instance_reference = 'MyFirstInstance'
self.add_free_partition(1)
self.external_proxy_supply(self.external_software_release)
self.external_proxy_add_free_partition(1)
filter_kw = {'master_url': self.external_master_url}
partition = self.request(self.software_release_not_in_list, None, instance_reference, 'slappart0',
filter_kw=filter_kw, partition_parameter_kw=dummy_parameter_dict)
self._checkInstanceIsFowarded(instance_reference, dummy_parameter_dict, self.software_release_not_in_list)
self.assertEqual(
partition._master_url,
self.external_master_url
)
def testForwardToMasterNotInList(self):
"""
Test that explicitely asking a master_url in SLA causes
proxy to refuse to forward if this master_url is not whitelisted
"""
self.add_free_partition(1)
self.external_proxy_supply(self.external_software_release)
self.external_proxy_add_free_partition(1)
filter_kw = {'master_url': self.external_master_url + 'bad'}
rv = self._requestComputerPartition(self.software_release_not_in_list, None, 'MyFirstInstance', 'slappart0', filter_kw=filter_kw)
self.assertEqual(rv._status_code, 404)
def testForwardRequest_SoftwareReleaseList(self):
"""
Test that instance request is automatically forwarded
if its Software Release matches list.
"""
dummy_parameter_dict = {'foo': 'bar'}
instance_reference = 'MyFirstInstance'
self.add_free_partition(1)
self.external_proxy_supply(self.external_software_release)
self.external_proxy_add_free_partition(1)
partition = self.request(self.external_software_release, None, instance_reference, 'slappart0',
partition_parameter_kw=dummy_parameter_dict)
self._checkInstanceIsFowarded(instance_reference, dummy_parameter_dict, self.external_software_release)
def testRequestToCurrentMaster(self):
"""
Explicitely ask deployment of an instance to current master
"""
self.add_free_partition(1)
self.external_proxy_add_free_partition(1)
instance_reference = 'MyFirstInstance'
dummy_parameter_dict = {'foo': 'bar'}
filter_kw = {'master_url': self.proxyaddr}
self.request(self.software_release_not_in_list, None, instance_reference, 'slappart0',
filter_kw=filter_kw, partition_parameter_kw=dummy_parameter_dict)
self._checkInstanceIsAllocatedLocally(instance_reference, dummy_parameter_dict, self.software_release_not_in_list)
def testRequestExplicitelyOnExternalMasterThenRequestAgain(self):
"""
Request an instance that will get forwarded to another an instance.
Test that subsequent request without SLA doesn't forward
"""
dummy_parameter_dict = {'foo': 'bar'}
self.testForwardToMasterInList()
partition = self.request(self.software_release_not_in_list, None, 'MyFirstInstance', 'slappart0', partition_parameter_kw=dummy_parameter_dict)
self.assertEqual(
getattr(partition, '_master_url', None),
None
)
# Test it has not been removed from local database (we keep track)
forwarded_instance_list = slapos.proxy.views.execute_db('forwarded_partition_request', 'SELECT * from %s', db=self.db)
self.assertEqual(len(forwarded_instance_list), 1)
# Test there is an instance allocated locally
computer = loads(self.app.get(
'/getFullComputerInformation?computer_id=%s' % self.computer_id
).data)
partition = computer._computer_partition_list[0]
for k, v in dummy_parameter_dict.iteritems():
self.assertEqual(
partition.getInstanceParameter(k),
v
)
self.assertEqual(
partition._software_release_document._software_release,
self.software_release_not_in_list
)
# XXX: when testing new schema version,
# rename to "TestMigrateVersion10ToLatest" and test accordingly.
# Of course, also test version 11 to latest (should be 12).
class TestMigrateVersion10To11(TestInformation, TestRequest, TestSlaveRequest, TestMultiNodeSupport):
"""
Test that old database version are automatically migrated without failure
......
[slapos]
software_root = %(tempdir)s/opt/slapgrid
instance_root = %(tempdir)s/srv/slapgrid
master_url = %(proxyaddr)s
computer_id = computer
[slapproxy]
host = 127.0.0.1
port = 8080
database_uri = %(tempdir)s/lib/proxy.db
# Here goes the list of slapos masters that slapproxy can contact
# Each section beginning by multimaster is a different SlapOS Master, represented by arbitrary name.
# For each section, you need to specify the URL of the SlapOS Master.
# For each section, you can specify if needed the location of key/certificate used to authenticate to this slapOS Master.
# For each section, you can specify a list of Software Releases. Any instance request matching this Softwrare Release will be automatically forwarded to this SlapOS Master and will not be allocated locally.
[multimaster/https://slap.vifib.com]
key = /path/to/cert.key
cert = /path/to/cert.cert
# XXX add wildcard support for SR list.
software_release_list =
http://something.io/software.cfg
/some/arbitrary/local/unix/path
[multimaster/http://%(external_proxy_host)s:%(external_proxy_port)s]
# No certificate here: it is http.
software_release_list =
http://mywebsite.me/exteral_software_release.cfg
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment