diff --git a/slapos/proxy/views.py b/slapos/proxy/views.py index 46d243737bb03cf1a8ea5566c41fbc70e08cf0c0..12924d84d87ce878073887d24dc0bcf843229e28 100644 --- a/slapos/proxy/views.py +++ b/slapos/proxy/views.py @@ -44,6 +44,7 @@ from slapos.util import loads, dumps import six from six.moves import range +from six.moves.urllib.parse import urlparse app = Flask(__name__) @@ -349,11 +350,14 @@ def loadComputerConfigurationFromXML(): computer_dict = loads(xml.encode('utf-8')) execute_db('computer', 'INSERT OR REPLACE INTO %s values(:reference, :address, :netmask)', computer_dict) + + # remove references to old partitions. + execute_db('partition', 'DELETE FROM %s WHERE computer_reference = :reference', computer_dict) + execute_db('partition_network', 'DELETE FROM %s WHERE computer_reference = :reference', computer_dict) + for partition in computer_dict['partition_list']: partition['computer_reference'] = computer_dict['reference'] execute_db('partition', 'INSERT OR IGNORE INTO %s (reference, computer_reference) values(:reference, :computer_reference)', partition) - execute_db('partition_network', 'DELETE FROM %s WHERE partition_reference = ? AND computer_reference = ?', - [partition['reference'], partition['computer_reference']]) for address in partition['address_list']: address['reference'] = partition['tap']['name'] address['partition_reference'] = partition['reference'] @@ -391,12 +395,61 @@ def supplySupply(): @app.route('/requestComputerPartition', methods=['POST']) def requestComputerPartition(): parsed_request_dict = parseRequestComputerPartitionForm(request.form) - # Is it a slave instance? slave = loads(request.form.get('shared_xml', EMPTY_DICT_XML).encode('utf-8')) # Check first if instance is already allocated if slave: + # slapproxy cannot request frontends, but we can workaround common cases, + # so that during tests promises are succesful. + if not isRequestToBeForwardedToExternalMaster(parsed_request_dict): + # if client request a "simple" frontend for an URL, we can tell this + # client to use the URL directly. + apache_frontend_sr_url_list = ( + 'http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg', + ) + if parsed_request_dict['software_release'] in apache_frontend_sr_url_list \ + and parsed_request_dict.get('software_type', '') in ('', 'RootSoftwareInstance', 'default'): + url = parsed_request_dict['partition_parameter_kw'].get('url') + if url: + app.logger.warning("Bypassing frontend for %s => %s", parsed_request_dict, url) + partition = ComputerPartition('', 'Fake frontend for {}'.format(url)) + partition.slap_computer_id = '' + partition.slap_computer_partition_id = '' + partition._parameter_dict = {} + partition._connection_dict = { + 'secure_access': url, + 'domain': urlparse(url).netloc, + } + return dumps(partition) + # another similar case is for KVM frontends. This is used in + # request-slave-frontend from software/kvm/instance-kvm.cfg.jinja2 + # requested values by 'return' recipe are: url resource port domainname + kvm_frontend_sr_url_list = ( + 'http://git.erp5.org/gitweb/slapos.git/blob_plain/refs/tags/slapos-0.92:/software/kvm/software.cfg', + ) + if parsed_request_dict['software_release'] in kvm_frontend_sr_url_list \ + and parsed_request_dict.get('software_type') in ('frontend', ): + host = parsed_request_dict['partition_parameter_kw'].get('host') + port = parsed_request_dict['partition_parameter_kw'].get('port') + if host and port: + # host is supposed to be ipv6 without brackets. + if ':' in host and host[0] != '[': + host = '[%s]' % host + url = 'https://%s:%s/' % (host, port) + app.logger.warning("Bypassing KVM VNC frontend for %s => %s", parsed_request_dict, url) + partition = ComputerPartition('', 'Fake KVM VNC frontend for {}'.format(url)) + partition.slap_computer_id = '' + partition.slap_computer_partition_id = '' + partition._parameter_dict = {} + partition._connection_dict = { + 'url': url, + 'domainname': host, + 'port': port, + 'path': '/' + } + return dumps(partition) + # XXX: change schema to include a simple "partition_reference" which # is name of the instance. Then, no need to do complex search here. slave_reference = parsed_request_dict['partition_id'] + '_' + parsed_request_dict['partition_reference'] diff --git a/slapos/slap/standalone.py b/slapos/slap/standalone.py new file mode 100644 index 0000000000000000000000000000000000000000..dfc542e86cfe5dda02b00ca0261c55e3aa53db9a --- /dev/null +++ b/slapos/slap/standalone.py @@ -0,0 +1,722 @@ +# -*- coding: utf-8 -*- +############################################################################## +# +# Copyright (c) 2018 Vifib SARL and Contributors. All Rights Reserved. +# +# WARNING: This program as such is intended to be used by professional +# programmers who take the whole responsibility of assessing all potential +# consequences resulting from its eventual inadequacies and bugs +# End users who are looking for a ready-to-use solution with commercial +# guarantees and support are strongly adviced to contract a Free Software +# Service Company +# +# This program is Free Software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 3 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# +############################################################################## + +import glob +import os +import textwrap +import logging +import time +import errno +import socket +import shutil + +from six.moves import urllib +from six.moves import http_client + +try: + import subprocess32 as subprocess +except ImportError: + import subprocess + +import xml_marshaller +import zope.interface +import psutil + +from .interface.slap import IException +from .interface.slap import ISupply +from .interface.slap import IRequester + +from .slap import slap +from ..grid.svcbackend import getSupervisorRPC + + +@zope.interface.implementer(IException) +class SlapOSNodeCommandError(Exception): + """Exception raised when running a SlapOS Node command failed. + """ + def __str__(self): + return "{} exitstatus: {} output:\n{}".format( + self.__class__.__name__, + self.args[0]['exitstatus'], + self.args[0]['output'], + ) + + +@zope.interface.implementer(IException) +class PathTooDeepError(Exception): + """Exception raised when path is too deep to create an unix socket. + """ + + +class ConfigWriter(object): + """Base class for an object writing a config file or wrapper script. + """ + def __init__(self, standalone_slapos): + self._standalone_slapos = standalone_slapos + + def writeConfig(self, path): + NotImplemented + + +class SupervisorConfigWriter(ConfigWriter): + """Write supervisor configuration at etc/supervisor.conf + """ + def _getProgramConfig(self, program_name, command, stdout_logfile): + """Format a supervisor program block. + """ + return textwrap.dedent( + """\ + [program:{program_name}] + command = {command} + autostart = false + autorestart = false + startretries = 0 + startsecs = 0 + redirect_stderr = true + stdout_logfile = {stdout_logfile} + stdout_logfile_maxbytes = 5MB + stdout_logfile_backups = 10 + + """.format(**locals())) + + def _getSupervisorConfigParts(self): + """Iterator on parts of formatted config. + """ + standalone_slapos = self._standalone_slapos + yield textwrap.dedent( + """ + [unix_http_server] + file = {standalone_slapos._supervisor_socket} + + [supervisorctl] + serverurl = unix://{standalone_slapos._supervisor_socket} + + [supervisord] + logfile = {standalone_slapos._supervisor_log} + pidfile = {standalone_slapos._supervisor_pid} + childlogdir = {standalone_slapos._log_directory} + + [rpcinterface:supervisor] + supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface + + [program:slapos-proxy] + command = slapos proxy start --cfg {standalone_slapos._slapos_config} --verbose + startretries = 0 + startsecs = 0 + redirect_stderr = true + """.format(**locals())) + + for program, program_config in standalone_slapos._slapos_commands.items(): + yield self._getProgramConfig( + program, + program_config['command'].format( + self=standalone_slapos, debug_args=''), + stdout_logfile=program_config.get( + 'stdout_logfile', 'AUTO').format(self=standalone_slapos)) + + def writeConfig(self, path): + with open(path, 'w') as f: + for part in self._getSupervisorConfigParts(): + f.write(part) + + +class SlapOSConfigWriter(ConfigWriter): + """Write slapos configuration at etc/slapos.cfg + """ + def writeConfig(self, path): + standalone_slapos = self._standalone_slapos # type: StandaloneSlapOS + read_only_shared_part_list = '\n '.join( # pylint: disable=unused-variable; used in format() + standalone_slapos._shared_part_list) + with open(path, 'w') as f: + f.write( + textwrap.dedent( + """ + [slapos] + software_root = {standalone_slapos._software_root} + instance_root = {standalone_slapos._instance_root} + shared_part_list = + {read_only_shared_part_list} + {standalone_slapos._shared_part_root} + master_url = {standalone_slapos._master_url} + computer_id = {standalone_slapos._computer_id} + root_check = False + pidfile_software = {standalone_slapos._instance_pid} + pidfile_instance = {standalone_slapos._software_pid} + pidfile_report = {standalone_slapos._report_pid} + + [slapproxy] + host = {standalone_slapos._server_ip} + port = {standalone_slapos._server_port} + database_uri = {standalone_slapos._proxy_database} + """.format(**locals()))) + + +class SlapOSCommandWriter(ConfigWriter): + """Write a bin/slapos wrapper. + """ + def writeConfig(self, path): + with open(path, 'w') as f: + f.write( + textwrap.dedent( + """\ + #!/bin/sh + SLAPOS_CONFIGURATION={self._standalone_slapos._slapos_config} \\ + SLAPOS_CLIENT_CONFIGURATION=$SLAPOS_CONFIGURATION \\ + exec slapos "$@" + """.format(**locals()))) + os.chmod(path, 0o755) + + +@zope.interface.implementer(ISupply, IRequester) +class StandaloneSlapOS(object): + """A SlapOS that can be embedded in other applications, also useful for testing. + + This plays the role of an `IComputer` where users of classes implementing this + interface can install software, create partitions and access parameters of the + running partitions. + + Extends the existing `IRequester` and `ISupply`, with the special behavior that + `IRequester.request` and `ISupply.supply` will only use the embedded computer. + """ + def __init__( + self, + base_directory, + server_ip, + server_port, + computer_id='local', + shared_part_list=(), + software_root=None, + instance_root=None, + shared_part_root=None): + """Constructor, creates a standalone slapos in `base_directory`. + + Arguments: + * `base_directory` -- the directory which will contain softwares and instances. + * `server_ip`, `server_port` -- the address this SlapOS proxy will listen to. + * `computer_id` -- the id of this computer. + * `shared_part_list` -- list of extra paths to use as read-only ${buildout:shared-part-list}. + * `software_root` -- directory to install software, default to "soft" in `base_directory` + * `instance_root` -- directory to create instances, default to "inst" in `base_directory` + * `shared_part_root` -- directory to hold shared parts software, default to "shared" in `base_directory`. + + Error cases: + * `PathTooDeepError` when `base_directory` is too deep. Because of limitation + with the length of paths of UNIX sockets, too deep paths cannot be used. + Note that once slapns work is integrated, this should not be an issue anymore. + """ + self._logger = logging.getLogger(__name__) + + # slapos proxy address + self._server_ip = server_ip + self._server_port = server_port + self._master_url = "http://{server_ip}:{server_port}".format(**locals()) + + self._base_directory = base_directory + self._shared_part_list = list(shared_part_list) + + self._slapos_commands = { + 'slapos-node-software': { + 'command': + 'slapos node software --cfg {self._slapos_config} --all {debug_args}', + 'debug_args': + '--buildout-debug', + 'stdout_logfile': + '{self._log_directory}/slapos-node-software.log', + }, + 'slapos-node-instance': { + 'command': + 'slapos node instance --cfg {self._slapos_config} --all {debug_args}', + 'debug_args': + '--buildout-debug', + 'stdout_logfile': + '{self._log_directory}/slapos-node-instance.log', + }, + 'slapos-node-report': { + 'command': + 'slapos node report --cfg {self._slapos_config} {debug_args}', + 'log_file': + '{self._log_directory}/slapos-node-report.log', + } + } + self._computer_id = computer_id + self._slap = slap() + self._slap.initializeConnection(self._master_url) + + self._initBaseDirectory(software_root, instance_root, shared_part_root) + + def _initBaseDirectory(self, software_root, instance_root, shared_part_root): + """Create the directory after checking it's not too deep. + """ + base_directory = self._base_directory + # To prevent error: Cannot open an HTTP server: socket.error reported + # AF_UNIX path too long This `base_directory` should not be too deep. + # Socket path is 108 char max on linux + # https://github.com/torvalds/linux/blob/3848ec5/net/unix/af_unix.c#L234-L238 + # Supervisord socket name contains the pid number, which is why we add + # .xxxxxxx in this check. + if len(os.path.join(base_directory, 'supervisord.socket.xxxxxxx')) > 108: + raise PathTooDeepError( + 'working directory ( {base_directory} ) is too deep'.format( + **locals())) + + def ensureDirectoryExists(d): + if not os.path.exists(d): + os.mkdir(d) + + self._software_root = software_root if software_root else os.path.join( + base_directory, 'soft') + self._instance_root = instance_root if instance_root else os.path.join( + base_directory, 'inst') + self._shared_part_root = shared_part_root if shared_part_root else os.path.join( + base_directory, 'shared') + for d in (self._software_root, self._instance_root, self._shared_part_root): + ensureDirectoryExists(d) + os.chmod(d, 0o750) + + etc_directory = os.path.join(base_directory, 'etc') + ensureDirectoryExists(etc_directory) + self._supervisor_config = os.path.join(etc_directory, 'supervisord.conf') + self._slapos_config = os.path.join(etc_directory, 'slapos.cfg') + + var_directory = os.path.join(base_directory, 'var') + ensureDirectoryExists(var_directory) + self._proxy_database = os.path.join(var_directory, 'proxy.db') + + # for convenience, make a slapos command for this instance + bin_directory = os.path.join(base_directory, 'bin') + ensureDirectoryExists(bin_directory) + + self._slapos_bin = os.path.join(bin_directory, 'slapos') + + self._log_directory = os.path.join(var_directory, 'log') + ensureDirectoryExists(self._log_directory) + self._supervisor_log = os.path.join(self._log_directory, 'supervisord.log') + + run_directory = os.path.join(var_directory, 'run') + ensureDirectoryExists(run_directory) + self._supervisor_pid = os.path.join(run_directory, 'supervisord.pid') + self._software_pid = os.path.join(run_directory, 'slapos-node-software.pid') + self._instance_pid = os.path.join(run_directory, 'slapos-node-instance.pid') + self._report_pid = os.path.join(run_directory, 'slapos-node-report.pid') + + self._supervisor_socket = os.path.join(run_directory, 'supervisord.sock') + + SupervisorConfigWriter(self).writeConfig(self._supervisor_config) + SlapOSConfigWriter(self).writeConfig(self._slapos_config) + SlapOSCommandWriter(self).writeConfig(self._slapos_bin) + + self.start() + + @property + def computer(self): + """Access the computer. + """ + return self._slap.registerComputer(self._computer_id) + + @property + def software_directory(self): + # type: () -> str + """Path to software directory + """ + return self._software_root + + @property + def shared_directory(self): + # type: () -> str + """Path to shared parts directory + """ + return self._shared_part_root + + @property + def instance_directory(self): + # type: () -> str + """Path to instance directory + """ + return self._instance_root + + @property + def system_supervisor_rpc(self): + """A xmlrpc connection to control the "System" supervisor. + + The system supervisor is used internally by StandaloneSlapOS to start + slap proxy and run slapos node commands. + + This should be used as a context manager. + """ + return getSupervisorRPC(self._supervisor_socket) + + @property + def instance_supervisor_rpc(self): + """A xmlrpc connection to control the "Instance" supervisor. + + The instance supervisor is the one started implictly by slapos node instance. + + This should be used as a context manager. + """ + return getSupervisorRPC( + # this socket path is not configurable. + os.path.join(self._instance_root, "supervisord.socket")) + + def format( + self, + partition_count, + ipv4_address, + ipv6_address, + partition_base_name="slappart"): + """Creates `partition_count` partitions. + + All partitions have the same `ipv4_address` and `ipv6_address` and + use the current system user. + + When calling this a second time with a lower `partition_count` or with + different `partition_base_name` will delete existing partitions. + + Error cases: + * ValueError when re-formatting should delete partitions that are busy. + """ + for path in ( + self._software_root, + self._shared_part_root, + self._instance_root, + ): + if not os.path.exists(path): + os.mkdir(path) + + # check for partitions to remove + unknown_partition_set = set([]) + for path in glob.glob(os.path.join(self._instance_root, '*')): + # var and etc are some slapos "system" directories, not partitions + if os.path.isdir(path) and os.path.basename(path) not in ('var', 'etc'): + unknown_partition_set.add(path) + + # create partitions and configure computer + partition_list = [] + for i in range(partition_count): + partition_reference = '%s%s' % (partition_base_name, i) + + partition_path = os.path.join(self._instance_root, partition_reference) + unknown_partition_set.discard(partition_path) + if not (os.path.exists(partition_path)): + os.mkdir(partition_path) + os.chmod(partition_path, 0o750) + partition_list.append({ + 'address_list': [ + { + 'addr': ipv4_address, + 'netmask': '255.255.255.255' + }, + { + 'addr': ipv6_address, + 'netmask': 'ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff' + }, + ], + 'path': partition_path, + 'reference': partition_reference, + 'tap': { + 'name': partition_reference + }, + }) + + if unknown_partition_set: + # sanity check that we are not removing partitions in use + computer_partition_dict = { + computer_part.getId(): computer_part + for computer_part in self.computer.getComputerPartitionList() + } + for part in unknown_partition_set: + # used in format(**locals()) below + part_id = os.path.basename(part) # pylint: disable=unused-variable + computer_partition = computer_partition_dict.get(os.path.basename(part)) + if computer_partition is not None \ + and computer_partition.getState() != "destroyed": + raise ValueError( + "Cannot reformat to remove busy partition at {part_id}".format( + **locals())) + + self.computer.updateConfiguration( + xml_marshaller.xml_marshaller.dumps({ + 'address': ipv4_address, + 'netmask': '255.255.255.255', + 'partition_list': partition_list, + 'reference': self._computer_id, + 'instance_root': self._instance_root, + 'software_root': self._software_root + })) + + for part in unknown_partition_set: + self._logger.debug( + "removing partition no longer part of format spec %s", part) + shutil.rmtree(part) + + def supply(self, software_url, computer_guid=None, state="available"): + """Supply a software, see ISupply.supply + + Software can only be supplied on this embedded computer. + """ + if computer_guid not in (None, self._computer_id): + raise ValueError("Can only supply on embedded computer") + self._slap.registerSupply().supply( + software_url, + self._computer_id, + state=state, + ) + + def request( + self, + software_release, + partition_reference, + software_type=None, + shared=False, + partition_parameter_kw=None, + filter_kw=None, + state=None): + """Request an instance, see IRequester.request + + Instance can only be requested on this embedded computer. + """ + if filter_kw is not None: + raise ValueError("Can only request on embedded computer") + return self._slap.registerOpenOrder().request( + software_release, + software_type=software_type, + partition_reference=partition_reference, + shared=shared, + partition_parameter_kw=partition_parameter_kw, + filter_kw=filter_kw, + state=state) + + def start(self): + """Start the system. + + If system was stopped, it will start partitions. + If system was already running, this does not restart partitions. + """ + self._logger.debug("Starting StandaloneSlapOS in %s", self._base_directory) + self._ensureSupervisordStarted() + self._ensureSlapOSAvailable() + + def stop(self): + """Stops all services. + + This methods blocks until services are stopped or a timeout is reached. + + Error cases: + * `Exception` when unexpected error occurs trying to stop supervisors. + """ + self._logger.info("shutting down") + + # shutdown slapos node instance supervisor, if it has been created. + instance_process_alive = [] + if os.path.exists(os.path.join(self._instance_root, 'etc', + 'supervisord.conf')): + try: + with self.instance_supervisor_rpc as instance_supervisor: + instance_supervisor_process = psutil.Process( + instance_supervisor.getPID()) + instance_supervisor.stopAllProcesses() + instance_supervisor.shutdown() + # shutdown returns before process is completly stopped, + # so wait for process. + _, instance_process_alive = psutil.wait_procs( + [instance_supervisor_process], timeout=10) + except BaseException as e: + self._logger.info("Ignoring exception while stopping instances: %s", e) + + with self.system_supervisor_rpc as system_supervisor: + system_supervisor_process = psutil.Process(system_supervisor.getPID()) + system_supervisor.stopAllProcesses() + system_supervisor.shutdown() + _, alive = psutil.wait_procs([system_supervisor_process], timeout=10) + if alive + instance_process_alive: + raise RuntimeError( + "Could not terminate some processes: {}".format( + alive + instance_process_alive)) + + def waitForSoftware(self, max_retry=0, debug=False, error_lines=30): + """Synchronously install or uninstall all softwares previously supplied/removed. + + This method retries on errors. If after `max_retry` times there's + still an error, the error is raised, containing `error_lines` of output + from the buildout command. + + If `debug` is true, buildout is executed in the foreground, with flags to + drop in a debugger session if error occurs. + + Error cases: + * `SlapOSNodeCommandError` when buildout error while installing software. + * Unexpected `Exception` if unable to connect to embedded slap server. + """ + return self._runSlapOSCommand( + 'slapos-node-software', + max_retry=max_retry, + debug=debug, + error_lines=error_lines, + ) + + def waitForInstance(self, max_retry=0, debug=False, error_lines=500): + """Instantiate all partitions previously requested for start. + + This method retries on errors. If after `max_retry` times there's + still an error, the error is raised, containing `error_lines` of output + from the buildout command. + + With instance with multiple partition, the failing partition is not + always the last processed one, so by default we include more lines of + output. + + If `debug` is true, buildout is executed in the foreground, with flags to + drop in a debugger session if error occurs. + + Error cases: + * `SlapOSNodeCommandError` when buildout error while creating instances. + * Unexpected `Exception` if unable to connect to embedded slap server. + """ + return self._runSlapOSCommand( + 'slapos-node-instance', + max_retry=max_retry, + debug=debug, + error_lines=error_lines, + ) + + def waitForReport(self, max_retry=0, debug=False, error_lines=30): + """Destroy all partitions previously requested for destruction. + + This method retries on errors. If after `max_retry` times there's + still an error, the error is raised, containing `error_lines` of output + from the buildout command. + + If `debug` is true, buildout is executed in the foreground, with flags to + drop in a debugger session if error occurs. + + Error cases: + * `SlapOSNodeCommandError` when buildout error while destroying instances. + * Unexpected `Exception` if unable to connect to embedded slap server. + """ + return self._runSlapOSCommand( + 'slapos-node-report', + max_retry=max_retry, + debug=debug, + error_lines=error_lines, + ) + + def _runSlapOSCommand( + self, command, max_retry=0, debug=False, error_lines=30): + if debug: + prog = self._slapos_commands[command] + # used in format(**locals()) below + debug_args = prog.get('debug_args', '') # pylint: disable=unused-variable + return subprocess.check_call( + prog['command'].format(**locals()), shell=True) + + with self.system_supervisor_rpc as supervisor: + retry = 0 + while True: + self._logger.info("starting command %s (retry:%s)", command, retry) + supervisor.startProcess(command, False) + + delay = 0.1 + while True: + self._logger.debug("retry %s: sleeping %s", retry, delay) + # we start waiting a short delay and increase the delay each loop, + # because when software is already built, this should return fast, + # but when we build a full software we don't need to poll the + # supervisor too often. + time.sleep(delay) + delay = min(delay * 1.2, 300) + process_info = supervisor.getProcessInfo(command) + if process_info['statename'] in ('EXITED', 'FATAL'): + self._logger.debug("SlapOS command finished %s" % process_info) + if process_info['exitstatus'] == 0: + return + if retry >= max_retry: + # get the last lines of output, at most `error_lines`. If + # these lines are long, the output may be truncated. + _, log_offset, _ = supervisor.tailProcessStdoutLog(command, 0, 0) + output, _, _ = supervisor.tailProcessStdoutLog( + command, log_offset - (2 << 13), 2 << 13) + raise SlapOSNodeCommandError({ + 'output': '\n'.join(output.splitlines()[-error_lines:]), + 'exitstatus': process_info['exitstatus'], + }) + break + retry += 1 + + def _ensureSupervisordStarted(self): + if os.path.exists(self._supervisor_pid): + with open(self._supervisor_pid, 'r') as f: + try: + pid = int(f.read()) + except (ValueError, TypeError): + self._logger.debug( + "Error reading supervisor pid from file, assuming it's not running" + ) + else: + try: + process = psutil.Process(pid) + except psutil.NoSuchProcess: + pass + else: + if process.name() == 'supervisord': + # OK looks already running + self._logger.debug("Supervisor running with pid %s", pid) + return + self._logger.debug("Supervisor pid file seem stale") + # start new supervisord + output = subprocess.check_output( + ['supervisord'], + cwd=self._base_directory, + ) + self._logger.debug("Started new supervisor: %s", output) + + def _isSlapOSAvailable(self): + try: + urllib.request.urlopen(self._master_url).close() + except urllib.error.HTTPError as e: + # root URL (/) of slapproxy is 404 + if e.code == http_client.NOT_FOUND: + return True + raise + except urllib.error.URLError as e: + if e.reason.errno == errno.ECONNREFUSED: + return False + raise + except socket.error as e: + if e.errno == errno.ECONNRESET: + return False + raise + except http_client.HTTPException: + return False + return True # (if / becomes 200 OK) + + def _ensureSlapOSAvailable(self): + # Wait for proxy to accept connections + for i in range(2**8): + if self._isSlapOSAvailable(): + return + time.sleep(i * .01) + raise RuntimeError("SlapOS not started") diff --git a/slapos/testing/__init__.py b/slapos/testing/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..4f12c9de251efedb563dd43c973bb22e624d28cd --- /dev/null +++ b/slapos/testing/__init__.py @@ -0,0 +1,27 @@ +# -*- coding: utf-8 -*- +############################################################################## +# +# Copyright (c) 2018 Vifib SARL and Contributors. All Rights Reserved. +# +# WARNING: This program as such is intended to be used by professional +# programmers who take the whole responsibility of assessing all potential +# consequences resulting from its eventual inadequacies and bugs +# End users who are looking for a ready-to-use solution with commercial +# guarantees and support are strongly adviced to contract a Free Software +# Service Company +# +# This program is Free Software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 3 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# +############################################################################## diff --git a/slapos/testing/testcase.py b/slapos/testing/testcase.py new file mode 100644 index 0000000000000000000000000000000000000000..b2ef57bda91d6baace0dc92e02bb6c15df03bd18 --- /dev/null +++ b/slapos/testing/testcase.py @@ -0,0 +1,419 @@ +# -*- coding: utf-8 -*- +############################################################################## +# +# Copyright (c) 2018 Vifib SARL and Contributors. All Rights Reserved. +# +# WARNING: This program as such is intended to be used by professional +# programmers who take the whole responsibility of assessing all potential +# consequences resulting from its eventual inadequacies and bugs +# End users who are looking for a ready-to-use solution with commercial +# guarantees and support are strongly adviced to contract a Free Software +# Service Company +# +# This program is Free Software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 3 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# +############################################################################## + +import unittest +import os +import glob +import logging + +try: + import subprocess32 as subprocess +except ImportError: + import subprocess + subprocess # pyflakes + +from .utils import getPortFromPath + +from ..slap.standalone import StandaloneSlapOS +from ..slap.standalone import SlapOSNodeCommandError +from ..slap.standalone import PathTooDeepError +from ..grid.utils import md5digest + +try: + from typing import Iterable, Tuple, Callable, Type +except ImportError: + pass + + +def makeModuleSetUpAndTestCaseClass( + software_url, + base_directory=None, + ipv4_address=os.environ['SLAPOS_TEST_IPV4'], + ipv6_address=os.environ['SLAPOS_TEST_IPV6'], + debug=bool(int(os.environ.get('SLAPOS_TEST_DEBUG', 0))), + verbose=bool(int(os.environ.get('SLAPOS_TEST_VERBOSE', 0))), + shared_part_list=os.environ.get('SLAPOS_TEST_SHARED_PART_LIST', + '').split(os.pathsep), +): + # type: (str, str, str, str, bool, bool, List[str]) -> Tuple[Callable[[], None], Type[SlapOSInstanceTestCase]] + """Create a setup module function and a testcase for testing `software_url`. + + This function returns a tuple of two arguments: + * a function to install the software, to be used as `unittest`'s + `setUpModule` + * a base class for test cases. + + The SlapOS instance will be using ip addresses defined by + environment variables `SLAPOS_TEST_IPV4` and `SLAPOS_TEST_IPV6`, or by the + explicits `ipv4_address` and `ipv6_address` arguments. + + To ease development and troubleshooting, two switches are available: + * `verbose` (also controlled by `SLAPOS_TEST_VERBOSE` environment variable) + to tell the test framework to log information describing the actions taken. + * `debug` (also controlled by `SLAPOS_TEST_DEBUG` environment variable) to + enable debugging mode which will drop in a debugger session when errors + occurs. + + The base_directory directory is by default .slapos in current directory, + or a path from `SLAPOS_TEST_WORKING_DIR` environment variable. + + This test class will use its own directory for shared parts and can also + paths from `shared_part_list` argument to lookup existing parts. + This is controlled by SLAPOS_TEST_SHARED_PART_LIST environment variable, + which should be a : separated list of path. + + A note about paths: + SlapOS itself and some services running in SlapOS uses unix sockets and + (sometimes very) deep paths, which does not play very well together. + To workaround this, users can set `SLAPOS_TEST_WORKING_DIR` environment + variable to the path of a short enough directory and local slapos will + use this directory. + The partitions references will be named after the unittest class name, + which can also lead to long paths. For this, unit test classes can define + a `__partition_reference__` attribute which will be used as partition + reference. If the class names are long, the trick is then to use a shorter + `__partition_reference__`. + See https://lab.nexedi.com/kirr/slapns for a solution to this problem. + """ + if base_directory is None: + base_directory = os.path.realpath( + os.environ.get( + 'SLAPOS_TEST_WORKING_DIR', os.path.join(os.getcwd(), '.slapos'))) + # TODO: fail if already running ? + try: + slap = StandaloneSlapOS( + base_directory=base_directory, + server_ip=ipv4_address, + server_port=getPortFromPath(base_directory), + shared_part_list=shared_part_list) + except PathTooDeepError: + raise RuntimeError( + 'base directory ( {} ) is too deep, try setting ' + 'SLAPOS_TEST_WORKING_DIR to a shallow enough directory'.format( + base_directory)) + + cls = type( + 'SlapOSInstanceTestCase for {}'.format(software_url), + (SlapOSInstanceTestCase,), { + 'slap': slap, + 'getSoftwareURL': classmethod(lambda _cls: software_url), + '_debug': debug, + '_verbose': verbose, + '_ipv4_address': ipv4_address, + '_ipv6_address': ipv6_address + }) + + class SlapOSInstanceTestCase_(cls, SlapOSInstanceTestCase): + # useless intermediate class so that editors provide completion anyway. + pass + + def setUpModule(): + # type: () -> None + if debug: + unittest.installHandler() + if verbose or debug: + logging.basicConfig(level=logging.DEBUG) + installSoftwareUrlList(cls, [software_url], debug=debug) + + return setUpModule, SlapOSInstanceTestCase_ + + +def checkSoftware(slap, software_url): + # type: (StandaloneSlapOS, str) -> None + """Check software installation. + + This perform a few basic static checks for common problems + with software installations. + """ + software_hash = md5digest(software_url) + + error_list = [] + # Check that all components set rpath correctly and we don't have miss linking any libraries. + for path in (os.path.join(slap.software_directory, + software_hash), slap.shared_directory): + if not glob.glob(os.path.join(path, '*')): + # shared might be empty (when using a slapos command that does not support shared yet). + continue + out = '' + try: + out = subprocess.check_output( + "find . -type f -executable " + + # We ignore parts that are binary distributions. + "| egrep -v /parts/java-re.*/ " + "| egrep -v /parts/firefox-.*/ " + "| egrep -v /parts/chromium-.*/ " + "| egrep -v /parts/chromedriver-.*/ " + + # nss has no valid rpath. It does not seem to be a problem in our case. + "| egrep -v /parts/nss/ " + "| xargs ldd " + r"| egrep '(^\S|not found)' " + "| grep -B1 'not found'", + shell=True, + stderr=subprocess.STDOUT, + cwd=path, + ) + except subprocess.CalledProcessError as e: + # The "good case" is when grep does not match anything, but in + # that case, it exists with exit code 1, so we accept this case. + if e.returncode != 1 or e.output: + error_list.append(e.output) + if out: + error_list.append(out) + + # check this software is not referenced in any shared parts. + for signature_file in glob.glob(os.path.join(slap.shared_directory, '*', '*', + '.*slapos.*.signature')): + with open(signature_file) as f: + signature_content = f.read() + if software_hash in signature_content: + error_list.append( + "Software hash present in signature {}\n{}\n".format( + signature_file, signature_content)) + + if error_list: + raise RuntimeError('\n'.join(error_list)) + + +def installSoftwareUrlList(cls, software_url_list, max_retry=2, debug=False): + # type: (Type[SlapOSInstanceTestCase], Iterable[str], int, bool) -> None + """Install softwares on the current testing slapos, for use in `setUpModule`. + + This also check softwares with `checkSoftware` + """ + try: + for software_url in software_url_list: + cls.logger.debug("Supplying %s", software_url) + cls.slap.supply(software_url) + cls.logger.debug("Waiting for slapos node software to build") + cls.slap.waitForSoftware(max_retry=max_retry, debug=debug) + for software_url in software_url_list: + checkSoftware(cls.slap, software_url) + except BaseException as e: + if not debug: + cls.logger.exception("Error building software, removing") + try: + for software_url in software_url_list: + cls.logger.debug("Removing %s", software_url) + cls.slap.supply(software_url, state="destroyed") + cls.logger.debug("Waiting for slapos node software to remove") + cls.slap.waitForSoftware(max_retry=max_retry, debug=debug) + except BaseException: + cls.logger.exception("Error removing software") + pass + cls._cleanup() + raise e + + +class SlapOSInstanceTestCase(unittest.TestCase): + """Install one slapos instance. + + This test case install software(s) and request one instance + during `setUpClass` and destroy that instance during `tearDownClass`. + + Software Release URL, Instance Software Type and Instance Parameters + can be defined on the class. + + All tests from the test class will run with the same instance. + + The following class attributes are available: + + * `computer_partition`: the `slapos.slap.slap.ComputerPartition` + computer partition instance. + + * `computer_partition_root_path`: the path of the instance root + directory. + + This class is not supposed to be imported directly, but needs to be setup by + calling makeModuleSetUpAndTestCaseClass. + """ + # can set this to true to enable debugging utilities + _debug = False + # can set this to true to enable more verbose output + _verbose = False + # maximum retries for `slapos node instance` + instance_max_retry = 10 + # maximum retries for `slapos node report` + report_max_retry = 0 + # number of partitions needed for this instance + partition_count = 10 + # reference of the default requested partition + default_partition_reference = 'testing partition 0' + + # a logger for messages of the testing framework + logger = logging.getLogger(__name__) + + # Dynamic members + slap = None # type: StandaloneSlapOS + _ipv4_address = "" + _ipv6_address = "" + + # Methods to be defined by subclasses. + @classmethod + def getSoftwareURL(cls): + """Return URL of software release to request instance. + + This method will be defined when initialising the class + with makeModuleSetUpAndTestCaseClass. + """ + raise NotImplementedError() + + @classmethod + def getInstanceParameterDict(cls): + """Return instance parameters. + + To be defined by subclasses if they need to request instance + with specific parameters. + """ + return {} + + @classmethod + def getInstanceSoftwareType(cls): + """Return software type for instance, default "". + + To be defined by subclasses if they need to request instance with specific + software type. + """ + return "" + + # Unittest methods + @classmethod + def setUpClass(cls): + """Request an instance. + """ + cls._instance_parameter_dict = cls.getInstanceParameterDict() + + try: + cls.logger.debug("Starting") + cls.slap.start() + cls.logger.debug( + "Formatting to remove old partitions XXX should not be needed because we delete ..." + ) + cls.slap.format(0, cls._ipv4_address, cls._ipv6_address) + cls.logger.debug("Formatting with %s partitions", cls.partition_count) + cls.slap.format( + cls.partition_count, cls._ipv4_address, cls._ipv6_address, + getattr(cls, '__partition_reference__', '{}-'.format(cls.__name__))) + + # request + cls.requestDefaultInstance() + + # slapos node instance + cls.logger.debug("Waiting for instance") + # waitForInstance does not tolerate any error but with instances, + # promises sometimes fail on first run, because services did not + # have time to start. + # To make debug usable, we tolerate instance_max_retry-1 errors and + # only debug the last. + if cls._debug and cls.instance_max_retry: + try: + cls.slap.waitForInstance(max_retry=cls.instance_max_retry - 1) + except SlapOSNodeCommandError: + cls.slap.waitForInstance(debug=True) + else: + cls.slap.waitForInstance( + max_retry=cls.instance_max_retry, debug=cls._debug) + + # expose some class attributes so that tests can use them: + # the main ComputerPartition instance, to use getInstanceParameterDict + cls.computer_partition = cls.requestDefaultInstance() + + # the path of the instance on the filesystem, for low level inspection + cls.computer_partition_root_path = os.path.join( + cls.slap._instance_root, cls.computer_partition.getId()) + cls.logger.debug("setUpClass done") + + except BaseException: + cls.logger.exception("Error during setUpClass") + cls._cleanup() + cls.setUp = lambda self: self.fail('Setup Class failed.') + raise + + @classmethod + def tearDownClass(cls): + """Tear down class, stop the processes and destroy instance. + """ + cls._cleanup() + + # implementation methods + @classmethod + def _cleanup(cls): + """Destroy all instances and stop subsystem. + Catches and log all exceptions. + """ + try: + cls.requestDefaultInstance(state='destroyed') + except: + cls.logger.exception("Error during request destruction") + try: + cls.slap.waitForReport(max_retry=cls.report_max_retry, debug=cls._debug) + except: + cls.logger.exception("Error during actual destruction") + leaked_partitions = [ + cp for cp in cls.slap.computer.getComputerPartitionList() + if cp.getState() != 'destroyed' + ] + if leaked_partitions: + cls.logger.critical( + "The following partitions were not cleaned up: %s", + [cp.getId() for cp in leaked_partitions]) + for cp in leaked_partitions: + try: + cls.slap.request( + software_release=cp.getSoftwareRelease().getURI(), + # software_type=cp.getType(), # TODO + # XXX is this really the reference ? + partition_reference=cp.getInstanceParameterDict()['instance_title'], + state="destroyed") + except: + cls.logger.exception( + "Error during request destruction of leaked partition") + try: + cls.slap.waitForReport(max_retry=cls.report_max_retry, debug=cls._debug) + except: + cls.logger.exception("Error during leaked partitions actual destruction") + try: + cls.slap.stop() + except: + cls.logger.exception("Error during stop") + + @classmethod + def requestDefaultInstance(cls, state='started'): + software_url = cls.getSoftwareURL() + software_type = cls.getInstanceSoftwareType() + cls.logger.debug( + 'requesting "%s" software:%s type:%s state:%s parameters:%s', + cls.default_partition_reference, software_url, software_type, state, + cls._instance_parameter_dict) + return cls.slap.request( + software_release=software_url, + software_type=software_type, + partition_reference=cls.default_partition_reference, + partition_parameter_kw=cls._instance_parameter_dict, + state=state) diff --git a/slapos/testing/utils.py b/slapos/testing/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..2523c0bcee3fd6a35c4bcf253d66187a2c1ba526 --- /dev/null +++ b/slapos/testing/utils.py @@ -0,0 +1,52 @@ +# -*- coding: utf-8 -*- +############################################################################## +# +# Copyright (c) 2018 Vifib SARL and Contributors. All Rights Reserved. +# +# WARNING: This program as such is intended to be used by professional +# programmers who take the whole responsibility of assessing all potential +# consequences resulting from its eventual inadequacies and bugs +# End users who are looking for a ready-to-use solution with commercial +# guarantees and support are strongly adviced to contract a Free Software +# Service Company +# +# This program is Free Software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 3 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# +############################################################################## + +import socket +import hashlib +from contextlib import closing + + +# Utility functions +def findFreeTCPPort(ip=''): + # type: (str) -> int + """Find a free TCP port to listen to. + """ + s = socket.socket( + socket.AF_INET6 if ':' in ip else socket.AF_INET, socket.SOCK_STREAM) + with closing(s): + s.bind((ip, 0)) + return s.getsockname()[1] + + +def getPortFromPath(path): + # type: (str) -> int + """A stable port using a hash from path. + """ + return 1024 + int( + hashlib.md5(path.encode('utf-8', 'backslashreplace')).hexdigest(), + 16) % (65535 - 1024) diff --git a/slapos/tests/test_slapproxy.py b/slapos/tests/test_slapproxy.py index 9d035920561588e3043bda893cb6e6cc66bf6b69..be1c13db6420c0c53a495ef17c47b69cd2dda371 100644 --- a/slapos/tests/test_slapproxy.py +++ b/slapos/tests/test_slapproxy.py @@ -129,8 +129,8 @@ database_uri = %(tempdir)s/lib/proxy.db computer_id = self.computer_id computer_dict = { 'reference': computer_id, - 'address': '123.456.789', - 'netmask': 'fffffffff', + 'address': '12.34.56.78', + 'netmask': '255.255.255.255', 'partition_list': [], } for i in range(partition_amount): @@ -160,6 +160,47 @@ database_uri = %(tempdir)s/lib/proxy.db views.is_schema_already_executed = False +class TestLoadComputerConfiguration(BasicMixin, unittest.TestCase): + """tests /loadComputerConfigurationFromXML the endpoint for format + """ + def test_loadComputerConfigurationFromXML_remove_partitions(self): + computer_dict = { + 'reference': self.computer_id, + 'address': '12.34.56.78', + 'netmask': '255.255.255.255', + 'partition_list': [ + { + 'reference': 'slappart1', + 'address_list': [ + { + 'addr': '1.2.3.4', + 'netmask': '255.255.255.255' + }, + ], + 'tap': {'name': 'tap0'}, + } + ], + } + rv = self.app.post('/loadComputerConfigurationFromXML', data={ + 'computer_id': self.computer_id, + 'xml': dumps(computer_dict), + }) + self.assertEqual(rv._status_code, 200) + # call again with different partition reference, old partition will be removed + # and a new partition will be used. + computer_dict['partition_list'][0]['reference'] = 'something else' + rv = self.app.post('/loadComputerConfigurationFromXML', data={ + 'computer_id': self.computer_id, + 'xml': dumps(computer_dict), + }) + self.assertEqual(rv._status_code, 200) + computer = loads( + self.app.get('/getFullComputerInformation', query_string={'computer_id': self.computer_id}).data) + self.assertEqual( + ['something else'], + [p.getId() for p in computer._computer_partition_list]) + + class TestInformation(BasicMixin, unittest.TestCase): """ Test Basic response of slapproxy @@ -682,6 +723,44 @@ class TestRequest(MasterMixin): partition_new = self.request('http://sr//', None, 'myinstance', 'slappart0') self.assertEqual(partition_new.getConnectionParameter('foo'), '1') + def test_request_frontend(self): + # slapproxy tells client to bypass "simple" frontends by just using the URL. + request = self.request( + 'http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg', + None, + self.id(), + 'slappart0', + shared=True, + partition_parameter_kw={'url': 'https://[::1]:123/', }) + self.assertEqual( + 'https://[::1]:123/', + request.getConnectionParameterDict()['secure_access']) + self.assertEqual( + '[::1]:123', + request.getConnectionParameterDict()['domain']) + + def test_request_kvm_frontend(self): + # slapproxy tells client to bypass kvm vnc frontends by building an URL using the backend. + request = self.request( + 'http://git.erp5.org/gitweb/slapos.git/blob_plain/refs/tags/slapos-0.92:/software/kvm/software.cfg', + 'frontend', + self.id(), + 'slappart0', + shared=True, + partition_parameter_kw={'host': '::1', 'port': '123'}) + self.assertEqual( + 'https://[::1]:123/', + request.getConnectionParameterDict()['url']) + self.assertEqual( + '[::1]', + request.getConnectionParameterDict()['domainname']) + self.assertEqual( + '123', + request.getConnectionParameterDict()['port']) + self.assertEqual( + '/', + request.getConnectionParameterDict()['path']) + class TestSlaveRequest(MasterMixin): """ @@ -1221,8 +1300,8 @@ database_uri = %(tempdir)s/lib/external_proxy.db computer_id = self.external_computer_id computer_dict = { 'reference': computer_id, - 'address': '123.456.789', - 'netmask': 'fffffffff', + 'address': '12.34.56.78', + 'netmask': '255.255.255.255', 'partition_list': [], } for i in range(partition_amount): diff --git a/slapos/tests/test_standalone.py b/slapos/tests/test_standalone.py new file mode 100644 index 0000000000000000000000000000000000000000..58b313ec23017025b5455843ac83cf46c33bf33f --- /dev/null +++ b/slapos/tests/test_standalone.py @@ -0,0 +1,370 @@ +############################################################################## +# +# Copyright (c) 2018 Vifib SARL and Contributors. All Rights Reserved. +# +# WARNING: This program as such is intended to be used by professional +# programmers who take the whole responsibility of assessing all potential +# consequences resulting from its eventual inadequacies and bugs +# End users who are looking for a ready-to-use solution with commercial +# guarantees and support are strongly adviced to contract a Free Software +# Service Company +# +# This program is Free Software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 3 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# +############################################################################## + +import unittest +import mock +import os +import tempfile +import textwrap +import shutil +import hashlib +import socket +import errno +import time +import multiprocessing +from contextlib import closing + +import psutil + +from slapos.slap.standalone import StandaloneSlapOS +from slapos.slap.standalone import SlapOSNodeCommandError + +SLAPOS_TEST_IPV4 = os.environ['SLAPOS_TEST_IPV4'] +SLAPOS_TEST_IPV6 = os.environ['SLAPOS_TEST_IPV6'] +SLAPOS_TEST_PORT = int(os.environ.get('SLAPOS_TEST_PORT', 33333)) + + +def checkPortIsFree(): + """Sanity check that we did not leak a process listening on this port. + """ + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + with closing(s): + try: + s.connect((SLAPOS_TEST_IPV4, SLAPOS_TEST_PORT)) + raise RuntimeError( + "Port needed for tests ({SLAPOS_TEST_IPV4}:{SLAPOS_TEST_PORT}) is already in use" + .format(**globals())) + except socket.error as e: + if e.errno == errno.ECONNREFUSED: + return + raise + + +class TestSlapOSStandaloneSetup(unittest.TestCase): + # BBB python2 + assertRaisesRegex = getattr( + unittest.TestCase, 'assertRaisesRegex', + unittest.TestCase.assertRaisesRegexp) + + def setUp(self): + checkPortIsFree() + + def test_format(self): + working_dir = tempfile.mkdtemp(prefix=__name__) + self.addCleanup(shutil.rmtree, working_dir) + standalone = StandaloneSlapOS( + working_dir, SLAPOS_TEST_IPV4, SLAPOS_TEST_PORT) + self.addCleanup(standalone.stop) + standalone.format(3, SLAPOS_TEST_IPV4, SLAPOS_TEST_IPV6) + + self.assertTrue(os.path.exists(standalone.software_directory)) + self.assertTrue(os.path.exists(standalone.instance_directory)) + self.assertTrue( + os.path.exists( + os.path.join(standalone.instance_directory, 'slappart0'))) + self.assertTrue( + os.path.exists( + os.path.join(standalone.instance_directory, 'slappart1'))) + self.assertTrue( + os.path.exists( + os.path.join(standalone.instance_directory, 'slappart2'))) + + def test_reformat_less_partitions(self): + working_dir = tempfile.mkdtemp(prefix=__name__) + self.addCleanup(shutil.rmtree, working_dir) + standalone = StandaloneSlapOS( + working_dir, SLAPOS_TEST_IPV4, SLAPOS_TEST_PORT) + self.addCleanup(standalone.stop) + standalone.format(2, SLAPOS_TEST_IPV4, SLAPOS_TEST_IPV6) + standalone.format(1, SLAPOS_TEST_IPV4, SLAPOS_TEST_IPV6) + self.assertFalse( + os.path.exists( + os.path.join(standalone.instance_directory, 'slappart1'))) + self.assertEqual( + ['slappart0'], + [cp.getId() for cp in standalone.computer.getComputerPartitionList()]) + + def test_reformat_different_base_name(self): + working_dir = tempfile.mkdtemp(prefix=__name__) + self.addCleanup(shutil.rmtree, working_dir) + standalone = StandaloneSlapOS( + working_dir, SLAPOS_TEST_IPV4, SLAPOS_TEST_PORT) + self.addCleanup(standalone.stop) + standalone.format( + 1, SLAPOS_TEST_IPV4, SLAPOS_TEST_IPV6, partition_base_name="a") + self.assertTrue( + os.path.exists(os.path.join(standalone.instance_directory, 'a0'))) + standalone.format( + 1, SLAPOS_TEST_IPV4, SLAPOS_TEST_IPV6, partition_base_name="b") + self.assertFalse( + os.path.exists(os.path.join(standalone.instance_directory, 'a0'))) + self.assertTrue( + os.path.exists(os.path.join(standalone.instance_directory, 'b0'))) + self.assertEqual( + ['b0'], + [cp.getId() for cp in standalone.computer.getComputerPartitionList()]) + + def test_reformat_refuse_deleting_running_partition(self): + working_dir = tempfile.mkdtemp(prefix=__name__) + self.addCleanup(shutil.rmtree, working_dir) + standalone = StandaloneSlapOS( + working_dir, SLAPOS_TEST_IPV4, SLAPOS_TEST_PORT) + self.addCleanup(standalone.stop) + standalone.format(1, SLAPOS_TEST_IPV4, SLAPOS_TEST_IPV6) + with mock.patch("slapos.slap.ComputerPartition.getState", return_value="busy"),\ + self.assertRaisesRegex(ValueError, "Cannot reformat to remove busy partition at .*slappart0"): + standalone.format(0, SLAPOS_TEST_IPV4, SLAPOS_TEST_IPV6) + + def test_two_instance_from_same_directory(self): + working_dir = tempfile.mkdtemp(prefix=__name__) + self.addCleanup(shutil.rmtree, working_dir) + standalone1 = StandaloneSlapOS( + working_dir, SLAPOS_TEST_IPV4, SLAPOS_TEST_PORT) + + def maybestop(): + # try to stop anyway, not to leak processes if test fail + try: + standalone1.stop() + except: + pass + + self.addCleanup(maybestop) + + # create another class instance, will control the same standanlone slapos. + standalone2 = StandaloneSlapOS( + working_dir, SLAPOS_TEST_IPV4, SLAPOS_TEST_PORT) + standalone2.stop() + + # stopping standalone2 stops everything + with self.assertRaises(BaseException): + standalone1.supply("https://example.com/software.cfg") + with self.assertRaises(BaseException): + standalone1.stop() + + +class SlapOSStandaloneTestCase(unittest.TestCase): + # This test case takes care of stopping the standalone instance + # in a cleanup, but subclasses who needs to control shutdown + # can set this class attribute to False to prevent this behavior. + _auto_stop_standalone = True + + def setUp(self): + checkPortIsFree() + working_dir = tempfile.mkdtemp(prefix=__name__) + self.addCleanup(shutil.rmtree, working_dir) + self.standalone = StandaloneSlapOS( + working_dir, SLAPOS_TEST_IPV4, SLAPOS_TEST_PORT) + if self._auto_stop_standalone: + self.addCleanup(self.standalone.stop) + self.standalone.format(1, SLAPOS_TEST_IPV4, SLAPOS_TEST_IPV6) + + +class TestSlapOSStandaloneSoftware(SlapOSStandaloneTestCase): + def test_install_software(self): + with tempfile.NamedTemporaryFile(suffix="-%s.cfg" % self.id()) as f: + f.write( + textwrap.dedent( + ''' + [buildout] + parts = instance + [instance] + recipe = plone.recipe.command==1.1 + command = touch ${buildout:directory}/instance.cfg + ''').encode()) + f.flush() + self.standalone.supply(f.name) + self.standalone.waitForSoftware() + + software_hash = hashlib.md5(f.name.encode()).hexdigest() + software_installation_path = os.path.join( + self.standalone.software_directory, software_hash) + self.assertTrue(os.path.exists(software_installation_path)) + self.assertTrue( + os.path.exists(os.path.join(software_installation_path, 'bin'))) + self.assertTrue( + os.path.exists(os.path.join(software_installation_path, 'parts'))) + self.assertTrue( + os.path.exists( + os.path.join(software_installation_path, '.completed'))) + self.assertTrue( + os.path.exists( + os.path.join(software_installation_path, 'instance.cfg'))) + + # destroy + self.standalone.supply(f.name, state='destroyed') + self.standalone.waitForSoftware() + self.assertFalse(os.path.exists(software_installation_path)) + + def test_install_software_failure(self): + with tempfile.NamedTemporaryFile(suffix="-%s.cfg" % self.id()) as f: + f.write( + textwrap.dedent( + ''' + [buildout] + parts = error + [error] + recipe = plone.recipe.command==1.1 + command = bash -c "exit 123" + stop-on-error = true + ''').encode()) + f.flush() + self.standalone.supply(f.name) + + with self.assertRaises(SlapOSNodeCommandError) as e: + self.standalone.waitForSoftware() + + self.assertEqual(1, e.exception.args[0]['exitstatus']) + self.assertIn( + "Error: Non zero exit code (123) while running command.", + e.exception.args[0]['output']) + # SlapOSNodeCommandError.__str__ also displays output nicely + self.assertIn( + "SlapOSNodeCommandError exitstatus: 1 output:", str(e.exception)) + self.assertIn( + "Error: Non zero exit code (123) while running command.", + str(e.exception)) + self.assertNotIn(r"\n", str(e.exception)) + + +class TestSlapOSStandaloneInstance(SlapOSStandaloneTestCase): + _auto_stop_standalone = False # we stop explicitly + + def test_request_instance(self): + with tempfile.NamedTemporaryFile(suffix="-%s.cfg" % self.id()) as f: + # This is a minimal / super fast buildout that's compatible with slapos. + # We don't want to install slapos.cookbook because installation takes too + # much time, so we use simple plone.recipe.command and shell. + # This buildout create an instance with two parts: + # check_parameter: that checks that the requested parameter is set + # publish: that publish some parameters so that we can assert it's published. + software_url = f.name + f.write( + textwrap.dedent( + ''' + [buildout] + parts = instance + + [instance] + recipe = plone.recipe.command==1.1 + stop-on-error = true + # we use @@DOLLAR@@{section:option} for what will become instance substitutions + command = sed -e s/@@DOLLAR@@/$/g < ${buildout:directory}/instance.cfg + [buildout] + parts = check_parameter publish + eggs-directory = ${buildout:eggs-directory} + + [check_parameter] + # check we were requested with request=parameter ( as a way to test + # request parameters are sent ) + recipe = plone.recipe.command==1.1 + stop-on-error = true + command = \\ + curl '@@DOLLAR@@{slap-connection:server-url}/registerComputerPartition?computer_reference=@@DOLLAR@@{slap-connection:computer-id}&computer_partition_reference=@@DOLLAR@@{slap-connection:partition-id}' \\ + | grep 'requestparameter' + + [publish] + # touch a file to check instance exists and publish a hardcoded parameter + recipe = plone.recipe.command==1.1 + stop-on-error = true + command = \\ + touch instance.check \\ + && curl -X POST @@DOLLAR@@{slap-connection:server-url}/setComputerPartitionConnectionXml \\ + -d computer_id=@@DOLLAR@@{slap-connection:computer-id} \\ + -d computer_partition_id=@@DOLLAR@@{slap-connection:partition-id} \\ + -d connection_xml='publishedparameter' + EOF + ''').encode()) + f.flush() + + self.standalone.supply(software_url) + self.standalone.waitForSoftware() + + self.standalone.request( + software_url, + 'default', + 'instance', + partition_parameter_kw={'request': 'parameter'}) + self.standalone.waitForInstance() + + # check published parameters + partition = self.standalone.request( + software_url, + 'default', + 'instance', + partition_parameter_kw={'request': 'parameter'}) + self.assertEqual({'published': 'parameter'}, + partition.getConnectionParameterDict()) + + # check instance files + parition_directory = os.path.join( + self.standalone.instance_directory, 'slappart0') + self.assertTrue( + os.path.exists(os.path.join(parition_directory, '.installed.cfg'))) + self.assertTrue( + os.path.exists(os.path.join(parition_directory, 'instance.check'))) + + # check instance supervisor, there should be only watchdog now + with self.standalone.instance_supervisor_rpc as instance_supervisor_rpc: + self.assertEqual( + ['watchdog'], + [p['name'] for p in instance_supervisor_rpc.getAllProcessInfo()]) + + # delete instance + self.standalone.request( + software_url, + 'default', + 'instance', + partition_parameter_kw={'partition': 'parameter'}, + state='destroyed', + ) + self.standalone.waitForInstance() + # instanciate does nothing, it will be deleted with `report` + self.assertTrue( + os.path.exists(os.path.join(parition_directory, 'instance.check'))) + self.standalone.waitForReport() + self.assertFalse( + os.path.exists(os.path.join(parition_directory, 'instance.check'))) + + # check that stopping leaves no process + process_list = [] + with self.standalone.instance_supervisor_rpc as instance_supervisor_rpc: + process_list.append(psutil.Process(instance_supervisor_rpc.getPID())) + process_list.extend([ + psutil.Process(p['pid']) + for p in instance_supervisor_rpc.getAllProcessInfo() + if p['statename'] == 'RUNNING' + ]) + with self.standalone.system_supervisor_rpc as system_supervisor_rpc: + process_list.append(psutil.Process(system_supervisor_rpc.getPID())) + process_list.extend([ + psutil.Process(p['pid']) + for p in system_supervisor_rpc.getAllProcessInfo() + if p['statename'] == 'RUNNING' + ]) + self.assertEqual(set([True]), set([p.is_running() for p in process_list])) + self.standalone.stop() + self.assertEqual(set([False]), set([p.is_running() for p in process_list]))