diff --git a/slapos/recipe/erp5.recipe.testnode/CHANGES.txt b/slapos/recipe/erp5.recipe.testnode/CHANGES.txt new file mode 100644 index 0000000000000000000000000000000000000000..2e56a7559e4c724e633ec1ba5bdf790e064bee8d --- /dev/null +++ b/slapos/recipe/erp5.recipe.testnode/CHANGES.txt @@ -0,0 +1,6 @@ +Changelog +========= + +1.0 (unreleased) +---------------- + diff --git a/slapos/recipe/erp5.recipe.testnode/MANIFEST.in b/slapos/recipe/erp5.recipe.testnode/MANIFEST.in new file mode 100644 index 0000000000000000000000000000000000000000..7036e1a25a87aa3097762346b9c614ad2c45702b --- /dev/null +++ b/slapos/recipe/erp5.recipe.testnode/MANIFEST.in @@ -0,0 +1,2 @@ +include CHANGES.txt +recursive-include src/erp5/recipe/testnode *.in diff --git a/slapos/recipe/erp5.recipe.testnode/README.txt b/slapos/recipe/erp5.recipe.testnode/README.txt new file mode 100644 index 0000000000000000000000000000000000000000..f08a047f4fa198b88e660f2ae52dfb9c3dcce590 --- /dev/null +++ b/slapos/recipe/erp5.recipe.testnode/README.txt @@ -0,0 +1 @@ +The erp5.recipe.tesnode aims to install generic erp5 testnode. diff --git a/slapos/recipe/erp5.recipe.testnode/setup.cfg b/slapos/recipe/erp5.recipe.testnode/setup.cfg new file mode 100644 index 0000000000000000000000000000000000000000..0c3455bc479bfc7e6771d88e803f423cb3d9e59a --- /dev/null +++ b/slapos/recipe/erp5.recipe.testnode/setup.cfg @@ -0,0 +1,3 @@ +[egg_info] +tag_build = .dev +tag_svn_revision = 1 diff --git a/slapos/recipe/erp5.recipe.testnode/setup.py b/slapos/recipe/erp5.recipe.testnode/setup.py new file mode 100644 index 0000000000000000000000000000000000000000..ae9ba344e17b941e736e9bb27b097ed24c11624e --- /dev/null +++ b/slapos/recipe/erp5.recipe.testnode/setup.py @@ -0,0 +1,40 @@ +from setuptools import setup, find_packages + +name = "erp5.recipe.testnode" +version = '1.0' + +def read(name): + return open(name).read() + +long_description=( read('README.txt') + + '\n' + + read('CHANGES.txt') + ) + +setup( + name = name, + version = version, + description = "ZC Buildout recipe for create an testnode instance", + long_description=long_description, + license = "GPLv3", + keywords = "buildout erp5 test", + classifiers=[ + "Framework :: Buildout :: Recipe", + "Programming Language :: Python", + ], + packages = find_packages('src'), + package_dir = {'': 'src'}, + include_package_data=True, + install_requires = [ + 'setuptools', + 'slapos.lib.recipe', + 'xml_marshaller', + 'zc.buildout', + 'zc.recipe.egg', + # below are requirements to provide full blown python interpreter + 'lxml', + 'PyXML', + ], + namespace_packages = ['erp5', 'erp5.recipe'], + entry_points = {'zc.buildout': ['default = %s:Recipe' % name]}, + ) diff --git a/slapos/recipe/erp5.recipe.testnode/src/erp5/__init__.py b/slapos/recipe/erp5.recipe.testnode/src/erp5/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..f48ad10528712b2b8960f1863d156b88ed1ce311 --- /dev/null +++ b/slapos/recipe/erp5.recipe.testnode/src/erp5/__init__.py @@ -0,0 +1,6 @@ +# See http://peak.telecommunity.com/DevCenter/setuptools#namespace-packages +try: + __import__('pkg_resources').declare_namespace(__name__) +except ImportError: + from pkgutil import extend_path + __path__ = extend_path(__path__, __name__) diff --git a/slapos/recipe/erp5.recipe.testnode/src/erp5/recipe/__init__.py b/slapos/recipe/erp5.recipe.testnode/src/erp5/recipe/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..f48ad10528712b2b8960f1863d156b88ed1ce311 --- /dev/null +++ b/slapos/recipe/erp5.recipe.testnode/src/erp5/recipe/__init__.py @@ -0,0 +1,6 @@ +# See http://peak.telecommunity.com/DevCenter/setuptools#namespace-packages +try: + __import__('pkg_resources').declare_namespace(__name__) +except ImportError: + from pkgutil import extend_path + __path__ = extend_path(__path__, __name__) diff --git a/slapos/recipe/erp5.recipe.testnode/src/erp5/recipe/testnode/SlapOSControler.py b/slapos/recipe/erp5.recipe.testnode/src/erp5/recipe/testnode/SlapOSControler.py new file mode 100644 index 0000000000000000000000000000000000000000..b3a47262a48b60886a1d572656abb1e40720d4ae --- /dev/null +++ b/slapos/recipe/erp5.recipe.testnode/src/erp5/recipe/testnode/SlapOSControler.py @@ -0,0 +1,80 @@ +import slapos.slap, subprocess, os, time +from xml_marshaller import xml_marshaller + +class SlapOSControler(object): + + def __init__(self, config, process_group_pid_list=None): + self.config = config + self.process_group_pid_list = [] + # By erasing everything, we make sure that we are able to "update" + # existing profiles. This is quite dirty way to do updates... + if os.path.exists(config['proxy_database']): + os.unlink(config['proxy_database']) + proxy = subprocess.Popen([config['slapproxy_binary'], + config['slapos_config']], close_fds=True, preexec_fn=os.setsid) + process_group_pid_list.append(proxy.pid) + # XXX: dirty, giving some time for proxy to being able to accept + # connections + time.sleep(2) + slap = slapos.slap.slap() + slap.initializeConnection(config['master_url']) + # register software profile + self.software_profile = config['custom_profile_path'] + slap.registerSupply().supply( + self.software_profile, + computer_guid=config['computer_id']) + computer = slap.registerComputer(config['computer_id']) + # create partition and configure computer + partition_reference = config['partition_reference'] + partition_path = os.path.join(config['instance_root'], partition_reference) + if not os.path.exists(partition_path): + os.mkdir(partition_path) + os.chmod(partition_path, 0750) + computer.updateConfiguration(xml_marshaller.dumps({ + 'address': config['ipv4_address'], + 'instance_root': config['instance_root'], + 'netmask': '255.255.255.255', + 'partition_list': [{'address_list': [{'addr': config['ipv4_address'], + 'netmask': '255.255.255.255'}, + {'addr': config['ipv6_address'], + 'netmask': 'ffff:ffff:ffff::'}, + ], + 'path': partition_path, + 'reference': partition_reference, + 'tap': {'name': partition_reference}, + } + ], + 'reference': config['computer_id'], + 'software_root': config['software_root']})) + + def runSoftwareRelease(self, config, process_group_pid_list=None): + print "SlapOSControler.runSoftwareRelease" + while True: + cpu_count = os.sysconf("SC_NPROCESSORS_ONLN") + os.putenv('MAKEFLAGS', '-j%s' % cpu_count) + slapgrid = subprocess.Popen([config['slapgrid_software_binary'], '-v', '-c', + #'--buildout-parameter',"'-U -N' -o", + config['slapos_config']], + close_fds=True, preexec_fn=os.setsid) + process_group_pid_list.append(slapgrid.pid) + slapgrid.wait() + if slapgrid.returncode == 0: + print 'Software installed properly' + break + else: + raise ValueError("Slapgrid software failed") + print 'Problem with software installation, trying again' + time.sleep(600) + + def runComputerPartition(self, config, process_group_pid_list=None): + print "SlapOSControler.runSoftwareRelease" + slap = slapos.slap.slap() + slap.registerOpenOrder().request(self.software_profile, + partition_reference='testing partition', + partition_parameter_kw=config['instance_dict']) + slapgrid = subprocess.Popen([config['slapgrid_partition_binary'], + config['slapos_config'], '-c', '-v'], close_fds=True, preexec_fn=os.setsid) + process_group_pid_list.append(slapgrid.pid) + slapgrid.wait() + if slapgrid.returncode != 0: + raise ValueError('Slapgrid instance failed') diff --git a/slapos/recipe/erp5.recipe.testnode/src/erp5/recipe/testnode/Updater.py b/slapos/recipe/erp5.recipe.testnode/src/erp5/recipe/testnode/Updater.py new file mode 100644 index 0000000000000000000000000000000000000000..b8c1b7d714531d0e0c88b8cce396ac26a266b501 --- /dev/null +++ b/slapos/recipe/erp5.recipe.testnode/src/erp5/recipe/testnode/Updater.py @@ -0,0 +1,189 @@ +import os, sys, subprocess, re, threading +from testnode import SubprocessError + +_format_command_search = re.compile("[[\\s $({?*\\`#~';<>&|]").search +_format_command_escape = lambda s: "'%s'" % r"'\''".join(s.split("'")) +def format_command(*args, **kw): + cmdline = [] + for k, v in sorted(kw.items()): + if _format_command_search(v): + v = _format_command_escape(v) + cmdline.append('%s=%s' % (k, v)) + for v in args: + if _format_command_search(v): + v = _format_command_escape(v) + cmdline.append(v) + return ' '.join(cmdline) + +def subprocess_capture(p, quiet=False): + def readerthread(input, output, buffer): + while True: + data = input.readline() + if not data: + break + output(data) + buffer.append(data) + if p.stdout: + stdout = [] + output = quiet and (lambda data: None) or sys.stdout.write + stdout_thread = threading.Thread(target=readerthread, + args=(p.stdout, output, stdout)) + stdout_thread.setDaemon(True) + stdout_thread.start() + if p.stderr: + stderr = [] + stderr_thread = threading.Thread(target=readerthread, + args=(p.stderr, sys.stderr.write, stderr)) + stderr_thread.setDaemon(True) + stderr_thread.start() + if p.stdout: + stdout_thread.join() + if p.stderr: + stderr_thread.join() + p.wait() + return (p.stdout and ''.join(stdout), + p.stderr and ''.join(stderr)) + +GIT_TYPE = 'git' +SVN_TYPE = 'svn' + +class Updater(object): + + _git_cache = {} + realtime_output = True + stdin = file(os.devnull) + + def __init__(self, repository_path, revision=None, git_binary=None): + self.revision = revision + self._path_list = [] + self.repository_path = repository_path + self.git_binary = git_binary + + def getRepositoryPath(self): + return self.repository_path + + def getRepositoryType(self): + try: + return self.repository_type + except AttributeError: + # guess the type of repository we have + if os.path.isdir(os.path.join( + self.getRepositoryPath(), '.git')): + repository_type = GIT_TYPE + elif os.path.isdir(os.path.join( + self.getRepositoryPath(), '.svn')): + repository_type = SVN_TYPE + else: + raise NotImplementedError + self.repository_type = repository_type + return repository_type + + def deletePycFiles(self, path): + """Delete *.pyc files so that deleted/moved files can not be imported""" + for path, dir_list, file_list in os.walk(path): + for file in file_list: + if file[-4:] in ('.pyc', '.pyo'): + # allow several processes clean the same folder at the same time + try: + os.remove(os.path.join(path, file)) + except OSError, e: + if e.errno != errno.ENOENT: + raise + + def spawn(self, *args, **kw): + quiet = kw.pop('quiet', False) + env = kw and dict(os.environ, **kw) or None + command = format_command(*args, **kw) + print '\n$ ' + command + sys.stdout.flush() + p = subprocess.Popen(args, stdin=self.stdin, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, env=env, + cwd=self.getRepositoryPath()) + if self.realtime_output: + stdout, stderr = subprocess_capture(p, quiet) + else: + stdout, stderr = p.communicate() + if not quiet: + sys.stdout.write(stdout) + sys.stderr.write(stderr) + result = dict(status_code=p.returncode, command=command, + stdout=stdout, stderr=stderr) + if p.returncode: + raise SubprocessError(result) + return result + + def _git(self, *args, **kw): + return self.spawn(self.git_binary, *args, **kw)['stdout'].strip() + + def _git_find_rev(self, ref): + try: + return self._git_cache[ref] + except KeyError: + if os.path.exists('.git/svn'): + r = self._git('svn', 'find-rev', ref) + assert r + self._git_cache[ref[0] != 'r' and 'r%u' % int(r) or r] = ref + else: + r = self._git('rev-list', '--topo-order', '--count', ref), ref + self._git_cache[ref] = r + return r + + def getRevision(self, *path_list): + if not path_list: + path_list = self._path_list + if self.getRepositoryType() == GIT_TYPE: + h = self._git('log', '-1', '--format=%H', '--', *path_list) + return self._git_find_rev(h) + elif self.getRepositoryType() == SVN_TYPE: + stdout = self.spawn('svn', 'info', *path_list)['stdout'] + return str(max(map(int, SVN_CHANGED_REV.findall(stdout)))) + raise NotImplementedError + + def checkout(self, *path_list): + if not path_list: + path_list = '.', + revision = self.revision + if self.getRepositoryType() == GIT_TYPE: + # edit .git/info/sparse-checkout if you want sparse checkout + if revision: + if type(revision) is str: + h = self._git_find_rev('r' + revision) + else: + h = revision[1] + if h != self._git('rev-parse', 'HEAD'): + self.deletePycFiles('.') + self._git('reset', '--merge', h) + else: + self.deletePycFiles('.') + if os.path.exists('.git/svn'): + self._git('svn', 'rebase') + else: + self._git('pull', '--ff-only') + self.revision = self._git_find_rev(self._git('rev-parse', 'HEAD')) + elif self.getRepositoryType() == SVN_TYPE: + # following code allows sparse checkout + def svn_mkdirs(path): + path = os.path.dirname(path) + if path and not os.path.isdir(path): + svn_mkdirs(path) + self.spawn(*(args + ['--depth=empty', path])) + for path in path_list: + args = ['svn', 'up', '--force', '--non-interactive'] + if revision: + args.append('-r%s' % revision) + svn_mkdirs(path) + args += '--set-depth=infinity', path + self.deletePycFiles(path) + try: + status_dict = self.spawn(*args) + except SubprocessError, e: + if 'cleanup' not in e.stderr: + raise + self.spawn('svn', 'cleanup', path) + status_dict = self.spawn(*args) + if not revision: + self.revision = revision = SVN_UP_REV.findall( + status_dict['stdout'].splitlines()[-1])[0] + else: + raise NotImplementedError + self._path_list += path_list diff --git a/slapos/recipe/erp5.recipe.testnode/src/erp5/recipe/testnode/__init__.py b/slapos/recipe/erp5.recipe.testnode/src/erp5/recipe/testnode/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..fa8eaf925d4d941180efbdd0740150c420a734d6 --- /dev/null +++ b/slapos/recipe/erp5.recipe.testnode/src/erp5/recipe/testnode/__init__.py @@ -0,0 +1,168 @@ +############################################################################## +# +# Copyright (c) 2010 Vifib SARL and Contributors. All Rights Reserved. +# +# WARNING: This program as such is intended to be used by professional +# programmers who take the whole responsibility of assessing all potential +# consequences resulting from its eventual inadequacies and bugs +# End users who are looking for a ready-to-use solution with commercial +# guarantees and support are strongly adviced to contract a Free Software +# Service Company +# +# This program is Free Software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 3 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# +############################################################################## +from slapos.lib.recipe.BaseSlapRecipe import BaseSlapRecipe +import os +import pkg_resources +import zc.buildout +import zc.recipe.egg +import sys + +CONFIG = dict( + proxy_port='5000', + computer_id='COMPUTER', + partition_reference='test0', +) + +class Recipe(BaseSlapRecipe): + def __init__(self, buildout, name, options): + self.egg = zc.recipe.egg.Egg(buildout, options['recipe'], options) + BaseSlapRecipe.__init__(self, buildout, name, options) + + def installSlapOs(self): + CONFIG['slapos_directory'] = self.createDataDirectory('slapos') + CONFIG['working_directory'] = self.createDataDirectory('testnode') + CONFIG['software_root'] = os.path.join(CONFIG['slapos_directory'], + 'software') + CONFIG['instance_root'] = os.path.join(CONFIG['slapos_directory'], + 'instance') + CONFIG['proxy_database'] = os.path.join(CONFIG['slapos_directory'], + 'proxy.db') + CONFIG['proxy_host'] = self.getLocalIPv4Address() + CONFIG['master_url'] = 'http://%s:%s' % (CONFIG['proxy_host'], + CONFIG['proxy_port']) + self._createDirectory(CONFIG['software_root']) + self._createDirectory(CONFIG['instance_root']) + CONFIG['slapos_config'] = self.createConfigurationFile('slapos.cfg', + self.substituteTemplate(pkg_resources.resource_filename(__name__, + 'template/slapos.cfg.in'), CONFIG)) + self.path_list.append(CONFIG['slapos_config']) + + def setupRunningWrapper(self): + self.path_list.extend(zc.buildout.easy_install.scripts([( + 'testnode', + __name__+'.testnode', 'run')], self.ws, + sys.executable, self.wrapper_directory, arguments=[ + dict( + computer_id=CONFIG['computer_id'], + instance_dict=eval(self.parameter_dict.get('instance_dict', '{}')), + instance_root=CONFIG['instance_root'], + ipv4_address=self.getLocalIPv4Address(), + ipv6_address=self.getGlobalIPv6Address(), + master_url=CONFIG['master_url'], + profile_url=self.parameter_dict['profile_url'], + proxy_database=CONFIG['proxy_database'], + proxy_port=CONFIG['proxy_port'], + slapgrid_partition_binary=self.options['slapgrid_partition_binary'], + slapgrid_software_binary=self.options['slapgrid_software_binary'], + slapos_config=CONFIG['slapos_config'], + slapproxy_binary=self.options['slapproxy_binary'], + git_binary=self.options['git_binary'], + software_root=CONFIG['software_root'], + working_directory=CONFIG['working_directory'], + vcs_repository=self.parameter_dict.get('vcs_repository'), + node_quantity=self.parameter_dict.get('node_quantity', '1'), + test_suite_master_url=self.parameter_dict.get( + 'test_suite_master_url', None), + test_suite_name=self.parameter_dict.get('test_suite_name'), + #slave_name=self.parameter_dict['slave_name'], + #slave_password=self.parameter_dict['slave_password'], + bin_directory=self.bin_directory, + foo='bar', + # botenvironemnt is splittable string of key=value to substitute + # environment of running bot + bot_environment=self.parameter_dict.get('bot_environment', ''), + partition_reference=CONFIG['partition_reference'], + ) + ])) + + def installLocalSvn(self): + svn_dict = dict(svn_binary = self.options['svn_binary']) + svn_dict.update(self.parameter_dict) + self._writeExecutable(os.path.join(self.bin_directory, 'svn'), """\ +#!/bin/sh +%(svn_binary)s --username %(svn_username)s --password %(svn_password)s \ +--non-interactive --trust-server-cert --no-auth-cache "$@" """% svn_dict) + + svnversion = os.path.join(self.bin_directory, 'svnversion') + if os.path.lexists(svnversion): + os.unlink(svnversion) + os.symlink(self.options['svnversion_binary'], svnversion) + + def installLocalGit(self): + git_dict = dict(git_binary = self.options['git_binary']) + git_dict.update(self.parameter_dict) + double_slash_end_position = 1 + # XXX, this should be provided by slapos + print "bin_directory : %r" % self.bin_directory + home_directory = os.path.join(*os.path.split(self.bin_directory)[0:-1]) + print "home_directory : %r" % home_directory + git_dict.setdefault("git_server_name", "git.erp5.org") + netrc_file = open(os.path.join(home_directory, '.netrc'), 'w') + netrc_file.write(""" +machine %(git_server_name)s +login %(vcs_username)s +password %(vcs_password)s""" % git_dict) + netrc_file.close() + + def installLocalRepository(self): + if self.parameter_dict.get('vcs_repository').endswith('git'): + self.installLocalGit() + else: + self.installLocalSvn() + + def installLocalZip(self): + zip = os.path.join(self.bin_directory, 'zip') + if os.path.lexists(zip): + os.unlink(zip) + os.symlink(self.options['zip_binary'], zip) + + def installLocalPython(self): + """Installs local python fully featured with eggs""" + self.path_list.extend(zc.buildout.easy_install.scripts([], self.ws, + sys.executable, self.bin_directory, scripts=None, + interpreter='python')) + + def installLocalRunUnitTest(self): + link = os.path.join(self.bin_directory, 'runUnitTest') + destination = os.path.join(CONFIG['instance_root'], + CONFIG['partition_reference'], 'bin', 'runUnitTest') + if os.path.lexists(link): + if not os.readlink(link) != destination: + os.unlink(link) + if not os.path.lexists(link): + os.symlink(destination, link) + + def _install(self): + self.requirements, self.ws = self.egg.working_set([__name__]) + self.path_list = [] + self.installSlapOs() + self.setupRunningWrapper() + self.installLocalRepository() + self.installLocalZip() + self.installLocalPython() + self.installLocalRunUnitTest() + return self.path_list diff --git a/slapos/recipe/erp5.recipe.testnode/src/erp5/recipe/testnode/template/slapos.cfg.in b/slapos/recipe/erp5.recipe.testnode/src/erp5/recipe/testnode/template/slapos.cfg.in new file mode 100644 index 0000000000000000000000000000000000000000..713f719a322502bca230db83a0c2aa4c6678607c --- /dev/null +++ b/slapos/recipe/erp5.recipe.testnode/src/erp5/recipe/testnode/template/slapos.cfg.in @@ -0,0 +1,10 @@ +[slapos] +software_root = %(software_root)s +instance_root = %(instance_root)s +master_url = %(master_url)s +computer_id = %(computer_id)s + +[slapproxy] +host = %(proxy_host)s +port = %(proxy_port)s +database_uri = %(proxy_database)s diff --git a/slapos/recipe/erp5.recipe.testnode/src/erp5/recipe/testnode/testnode.py b/slapos/recipe/erp5.recipe.testnode/src/erp5/recipe/testnode/testnode.py new file mode 100644 index 0000000000000000000000000000000000000000..9e5583df54ea2f4e936b8198febeb263d0fcb0c1 --- /dev/null +++ b/slapos/recipe/erp5.recipe.testnode/src/erp5/recipe/testnode/testnode.py @@ -0,0 +1,202 @@ +from xml_marshaller import xml_marshaller +import os, xmlrpclib, time, imp +from glob import glob +import signal +import slapos.slap +import subprocess +import sys +from SlapOSControler import SlapOSControler + + +class SubprocessError(EnvironmentError): + def __init__(self, status_dict): + self.status_dict = status_dict + def __getattr__(self, name): + return self.status_dict[name] + def __str__(self): + return 'Error %i' % self.status_code + + +from Updater import Updater + +process_group_pid_list = [] +process_pid_file_list = [] +process_command_list = [] +def sigterm_handler(signal, frame): + for pgpid in process_group_pid_list: + try: + os.killpg(pgpid, signal.SIGTERM) + except: + pass + for pid_file in process_pid_file_list: + try: + os.kill(int(open(pid_file).read().strip()), signal.SIGTERM) + except: + pass + for p in process_command_list: + try: + subprocess.call(p) + except: + pass + sys.exit(1) + +signal.signal(signal.SIGTERM, sigterm_handler) + +def safeRpcCall(function, *args): + retry = 64 + while True: + try: + return function(*args) + except (socket.error, xmlrpclib.ProtocolError), e: + print >>sys.stderr, e + pprint.pprint(args, file(function._Method__name, 'w')) + time.sleep(retry) + retry += retry >> 1 + +slapos_controler = None + +def run(args): + config = args[0] + slapgrid = None + supervisord_pid_file = os.path.join(config['instance_root'], 'var', 'run', + 'supervisord.pid') + subprocess.check_call([config['git_binary'], + "config", "--global", "http.sslVerify", "false"]) + previous_revision = None + run_software = True + # find what will be the path of the repository + repository_name = config['vcs_repository'].split('/')[-1].split('.')[0] + repository_path = os.path.join(config['working_directory'],repository_name) + config['repository_path'] = repository_path + sys.path.append(repository_path) + + # Write our own software.cfg to use the local repository + custom_profile_path = os.path.join(config['working_directory'], 'software.cfg') + config['custom_profile_path'] = custom_profile_path + if not os.path.exists(custom_profile_path): + # create a profile in order to use the repository we already have + custom_profile = open(custom_profile_path, 'w') + profile_content = """ +[buildout] +extends = %(software_config_path)s + +[%(repository_name)s_repository] +repository = %(repository_path)s +""" % {'software_config_path': os.path.join(repository_path, + config['profile_url']), + 'repository_name': repository_name, + 'repository_path' : repository_path} + custom_profile.write(profile_content) + custom_profile.close() + try: + while True: + # Make sure we have local repository + if not os.path.exists(repository_path): + subprocess.check_call([config['git_binary'], + 'clone', config['vcs_repository'], repository_path]) + # XXX this looks like to not wait the end of the command + # Make sure we have local repository + updater = Updater(repository_path, git_binary=config['git_binary']) + updater.checkout() + revision = updater.getRevision() + if previous_revision == revision: + time.sleep(120) + continue + previous_revision = revision + + + print config + portal_url = config['test_suite_master_url'] + test_result_path = None + test_result = (test_result_path, revision) + if portal_url: + if portal_url[-1] != '/': + portal_url += '/' + portal = xmlrpclib.ServerProxy("%s%s" % + (portal_url, 'portal_task_distribution'), + allow_none=1) + master = portal.portal_task_distribution + assert master.getProtocolRevision() == 1 + test_result = safeRpcCall(master.createTestResult, + config['test_suite_name'], revision, [], + False) + print "testnode, test_result : %r" % (test_result,) + if test_result: + test_result_path, test_revision = test_result + if revision != test_revision: + # other testnodes on other boxes are already ready to test another + # revision + updater = Updater(repository_path, git_binary=config['git_binary'], + revision=test_revision) + updater.checkout() + + # Now prepare the installation of SlapOS + slapos_controler = SlapOSControler(config, + process_group_pid_list=process_group_pid_list) + if run_software: + # this should be always true later, but it is too slow for now + slapos_controler.runSoftwareRelease(config, + process_group_pid_list=process_group_pid_list, + ) + run_software = False + + # create instances, it should take some seconds only + slapos_controler.runComputerPartition(config, + process_group_pid_list=process_group_pid_list) + + # update repositories downloaded by buildout. Later we should get + # from master a list of repositories + repository_path_list = glob(os.path.join(config['software_root'], + '*', 'parts', 'git_repository', '*')) + assert len(repository_path_list) >= 0 + for repository_path in repository_path_list: + updater = Updater(repository_path, git_binary=config['git_binary']) + updater.checkout() + if os.path.split(repository_path)[-1] == repository_name: + # redo checkout with good revision, the previous one is used + # to pull last code + updater = Updater(repository_path, git_binary=config['git_binary'], + revision=revision) + updater.checkout() + # calling dist/externals is only there for backward compatibility, + # the code will be removed soon + if os.path.exists(os.path.join(repository_path, 'dist/externals.py')): + process = subprocess.Popen(['dist/externals.py'], + cwd=repository_path) + process.wait() + + partition_path = os.path.join(config['instance_root'], + config['partition_reference']) + run_test_suite_path = os.path.join(partition_path, 'bin', + 'runTestSuite') + if not os.path.exists(run_test_suite_path): + raise ValueError('No %r provided' % run_test_suite_path) + + run_test_suite_revision = revision + if isinstance(revision, tuple): + revision = ','.join(revision) + run_test_suite = subprocess.Popen([run_test_suite_path, + '--test_suite', config['test_suite_name'], + '--revision', revision, + '--node_quantity', config['node_quantity'], + '--master_url', config['test_suite_master_url'], + ], ) + process_group_pid_list.append(run_test_suite.pid) + run_test_suite.wait() + + finally: + # Nice way to kill *everything* generated by run process -- process + # groups working only in POSIX compilant systems + # Exceptions are swallowed during cleanup phase + print "going to kill %r" % (process_group_pid_list,) + for pgpid in process_group_pid_list: + try: + os.killpg(pgpid, signal.SIGTERM) + except: + pass + try: + if os.path.exists(supervisord_pid_file): + os.kill(int(open(supervisord_pid_file).read().strip()), signal.SIGTERM) + except: + pass +