diff --git a/slapos/recipe/erp5testnode/SlapOSControler.py b/slapos/recipe/erp5testnode/SlapOSControler.py
deleted file mode 100644
index e9ed42f711874efb143e05787cf30f4cc6787eac..0000000000000000000000000000000000000000
--- a/slapos/recipe/erp5testnode/SlapOSControler.py
+++ /dev/null
@@ -1,94 +0,0 @@
-import slapos.slap, subprocess, os, time
-from xml_marshaller import xml_marshaller
-
-class SlapOSControler(object):
-
- def __init__(self, config, process_group_pid_set=None):
- self.config = config
- # By erasing everything, we make sure that we are able to "update"
- # existing profiles. This is quite dirty way to do updates...
- if os.path.exists(config['proxy_database']):
- os.unlink(config['proxy_database'])
- proxy = subprocess.Popen([config['slapproxy_binary'],
- config['slapos_config']], close_fds=True, preexec_fn=os.setsid)
- process_group_pid_set.add(proxy.pid)
- # XXX: dirty, giving some time for proxy to being able to accept
- # connections
- time.sleep(10)
- slap = slapos.slap.slap()
- slap.initializeConnection(config['master_url'])
- # register software profile
- self.software_profile = config['custom_profile_path']
- slap.registerSupply().supply(
- self.software_profile,
- computer_guid=config['computer_id'])
- computer = slap.registerComputer(config['computer_id'])
- # create partition and configure computer
- partition_reference = config['partition_reference']
- partition_path = os.path.join(config['instance_root'], partition_reference)
- if not os.path.exists(partition_path):
- os.mkdir(partition_path)
- os.chmod(partition_path, 0750)
- computer.updateConfiguration(xml_marshaller.dumps({
- 'address': config['ipv4_address'],
- 'instance_root': config['instance_root'],
- 'netmask': '255.255.255.255',
- 'partition_list': [{'address_list': [{'addr': config['ipv4_address'],
- 'netmask': '255.255.255.255'},
- {'addr': config['ipv6_address'],
- 'netmask': 'ffff:ffff:ffff::'},
- ],
- 'path': partition_path,
- 'reference': partition_reference,
- 'tap': {'name': partition_reference},
- }
- ],
- 'reference': config['computer_id'],
- 'software_root': config['software_root']}))
-
- def runSoftwareRelease(self, config, environment, process_group_pid_set=None,
- stdout=None, stderr=None):
- print "SlapOSControler.runSoftwareRelease"
- cpu_count = os.sysconf("SC_NPROCESSORS_ONLN")
- os.putenv('MAKEFLAGS', '-j%s' % cpu_count)
- os.environ['PATH'] = environment['PATH']
- slapgrid = subprocess.Popen([config['slapgrid_software_binary'], '-v', '-c',
- #'--buildout-parameter',"'-U -N' -o",
- config['slapos_config']],
- stdout=stdout, stderr=stderr,
- close_fds=True, preexec_fn=os.setsid)
- process_group_pid_set.add(slapgrid.pid)
- slapgrid.wait()
- stdout.seek(0)
- stderr.seek(0)
- process_group_pid_set.remove(slapgrid.pid)
- status_dict = {'status_code':slapgrid.returncode,
- 'stdout':stdout.read(),
- 'stderr':stderr.read()}
- stdout.close()
- stderr.close()
- return status_dict
-
- def runComputerPartition(self, config, environment,
- process_group_pid_set=None,
- stdout=None, stderr=None):
- print "SlapOSControler.runSoftwareRelease"
- slap = slapos.slap.slap()
- slap.registerOpenOrder().request(self.software_profile,
- partition_reference='testing partition',
- partition_parameter_kw=config['instance_dict'])
- slapgrid = subprocess.Popen([config['slapgrid_partition_binary'],
- config['slapos_config'], '-c', '-v'],
- stdout=stdout, stderr=stderr,
- close_fds=True, preexec_fn=os.setsid)
- process_group_pid_set.add(slapgrid.pid)
- slapgrid.wait()
- stdout.seek(0)
- stderr.seek(0)
- process_group_pid_set.remove(slapgrid.pid)
- status_dict = {'status_code':slapgrid.returncode,
- 'stdout':stdout.read(),
- 'stderr':stderr.read()}
- stdout.close()
- stderr.close()
- return status_dict
diff --git a/slapos/recipe/erp5testnode/Updater.py b/slapos/recipe/erp5testnode/Updater.py
deleted file mode 100644
index 83ea60a1e7738bdd55316ce4bb7c5937f838203c..0000000000000000000000000000000000000000
--- a/slapos/recipe/erp5testnode/Updater.py
+++ /dev/null
@@ -1,189 +0,0 @@
-import os, sys, subprocess, re, threading
-from testnode import SubprocessError
-
-_format_command_search = re.compile("[[\\s $({?*\\`#~';<>&|]").search
-_format_command_escape = lambda s: "'%s'" % r"'\''".join(s.split("'"))
-def format_command(*args, **kw):
- cmdline = []
- for k, v in sorted(kw.items()):
- if _format_command_search(v):
- v = _format_command_escape(v)
- cmdline.append('%s=%s' % (k, v))
- for v in args:
- if _format_command_search(v):
- v = _format_command_escape(v)
- cmdline.append(v)
- return ' '.join(cmdline)
-
-def subprocess_capture(p, quiet=False):
- def readerthread(input, output, buffer):
- while True:
- data = input.readline()
- if not data:
- break
- output(data)
- buffer.append(data)
- if p.stdout:
- stdout = []
- output = quiet and (lambda data: None) or sys.stdout.write
- stdout_thread = threading.Thread(target=readerthread,
- args=(p.stdout, output, stdout))
- stdout_thread.setDaemon(True)
- stdout_thread.start()
- if p.stderr:
- stderr = []
- stderr_thread = threading.Thread(target=readerthread,
- args=(p.stderr, sys.stderr.write, stderr))
- stderr_thread.setDaemon(True)
- stderr_thread.start()
- if p.stdout:
- stdout_thread.join()
- if p.stderr:
- stderr_thread.join()
- p.wait()
- return (p.stdout and ''.join(stdout),
- p.stderr and ''.join(stderr))
-
-GIT_TYPE = 'git'
-SVN_TYPE = 'svn'
-
-class Updater(object):
-
- _git_cache = {}
- realtime_output = True
- stdin = file(os.devnull)
-
- def __init__(self, repository_path, revision=None, git_binary=None):
- self.revision = revision
- self._path_list = []
- self.repository_path = repository_path
- self.git_binary = git_binary
-
- def getRepositoryPath(self):
- return self.repository_path
-
- def getRepositoryType(self):
- try:
- return self.repository_type
- except AttributeError:
- # guess the type of repository we have
- if os.path.isdir(os.path.join(
- self.getRepositoryPath(), '.git')):
- repository_type = GIT_TYPE
- elif os.path.isdir(os.path.join(
- self.getRepositoryPath(), '.svn')):
- repository_type = SVN_TYPE
- else:
- raise NotImplementedError
- self.repository_type = repository_type
- return repository_type
-
- def deletePycFiles(self, path):
- """Delete *.pyc files so that deleted/moved files can not be imported"""
- for path, dir_list, file_list in os.walk(path):
- for file in file_list:
- if file[-4:] in ('.pyc', '.pyo'):
- # allow several processes clean the same folder at the same time
- try:
- os.remove(os.path.join(path, file))
- except OSError, e:
- if e.errno != errno.ENOENT:
- raise
-
- def spawn(self, *args, **kw):
- quiet = kw.pop('quiet', False)
- env = kw and dict(os.environ, **kw) or None
- command = format_command(*args, **kw)
- print '\n$ ' + command
- sys.stdout.flush()
- p = subprocess.Popen(args, stdin=self.stdin, stdout=subprocess.PIPE,
- stderr=subprocess.PIPE, env=env,
- cwd=self.getRepositoryPath())
- if self.realtime_output:
- stdout, stderr = subprocess_capture(p, quiet)
- else:
- stdout, stderr = p.communicate()
- if not quiet:
- sys.stdout.write(stdout)
- sys.stderr.write(stderr)
- result = dict(status_code=p.returncode, command=command,
- stdout=stdout, stderr=stderr)
- if p.returncode:
- raise SubprocessError(result)
- return result
-
- def _git(self, *args, **kw):
- return self.spawn(self.git_binary, *args, **kw)['stdout'].strip()
-
- def _git_find_rev(self, ref):
- try:
- return self._git_cache[ref]
- except KeyError:
- if os.path.exists('.git/svn'):
- r = self._git('svn', 'find-rev', ref)
- assert r
- self._git_cache[ref[0] != 'r' and 'r%u' % int(r) or r] = ref
- else:
- r = self._git('rev-list', '--topo-order', '--count', ref), ref
- self._git_cache[ref] = r
- return r
-
- def getRevision(self, *path_list):
- if not path_list:
- path_list = self._path_list
- if self.getRepositoryType() == GIT_TYPE:
- h = self._git('log', '-1', '--format=%H', '--', *path_list)
- return self._git_find_rev(h)
- elif self.getRepositoryType() == SVN_TYPE:
- stdout = self.spawn('svn', 'info', *path_list)['stdout']
- return str(max(map(int, SVN_CHANGED_REV.findall(stdout))))
- raise NotImplementedError
-
- def checkout(self, *path_list):
- if not path_list:
- path_list = '.',
- revision = self.revision
- if self.getRepositoryType() == GIT_TYPE:
- # edit .git/info/sparse-checkout if you want sparse checkout
- if revision:
- if type(revision) is str:
- h = revision
- else:
- h = revision[1]
- if h != self._git('rev-parse', 'HEAD'):
- self.deletePycFiles('.')
- self._git('reset', '--merge', h)
- else:
- self.deletePycFiles('.')
- if os.path.exists('.git/svn'):
- self._git('svn', 'rebase')
- else:
- self._git('pull', '--ff-only')
- self.revision = self._git_find_rev(self._git('rev-parse', 'HEAD'))
- elif self.getRepositoryType() == SVN_TYPE:
- # following code allows sparse checkout
- def svn_mkdirs(path):
- path = os.path.dirname(path)
- if path and not os.path.isdir(path):
- svn_mkdirs(path)
- self.spawn(*(args + ['--depth=empty', path]))
- for path in path_list:
- args = ['svn', 'up', '--force', '--non-interactive']
- if revision:
- args.append('-r%s' % revision)
- svn_mkdirs(path)
- args += '--set-depth=infinity', path
- self.deletePycFiles(path)
- try:
- status_dict = self.spawn(*args)
- except SubprocessError, e:
- if 'cleanup' not in e.stderr:
- raise
- self.spawn('svn', 'cleanup', path)
- status_dict = self.spawn(*args)
- if not revision:
- self.revision = revision = SVN_UP_REV.findall(
- status_dict['stdout'].splitlines()[-1])[0]
- else:
- raise NotImplementedError
- self._path_list += path_list
diff --git a/slapos/recipe/erp5testnode/__init__.py b/slapos/recipe/erp5testnode/__init__.py
index cd8f0f53b69913c533cfb7b276a8fe0c50a923aa..af7177dc2b0ef95e2c56373f4c37a9aa10182f97 100644
--- a/slapos/recipe/erp5testnode/__init__.py
+++ b/slapos/recipe/erp5testnode/__init__.py
@@ -31,129 +31,79 @@ import zc.buildout
import zc.recipe.egg
import sys
-CONFIG = dict(
- proxy_port='5000',
- computer_id='COMPUTER',
- partition_reference='test0',
-)
-
class Recipe(BaseSlapRecipe):
def __init__(self, buildout, name, options):
self.egg = zc.recipe.egg.Egg(buildout, options['recipe'], options)
BaseSlapRecipe.__init__(self, buildout, name, options)
- def installSlapOs(self):
+ def _install(self):
+ self.requirements, self.ws = self.egg.working_set()
+ path_list = []
+ CONFIG = dict()
CONFIG['slapos_directory'] = self.createDataDirectory('slapos')
CONFIG['working_directory'] = self.createDataDirectory('testnode')
- CONFIG['software_root'] = os.path.join(CONFIG['slapos_directory'],
- 'software')
- CONFIG['instance_root'] = os.path.join(CONFIG['slapos_directory'],
- 'instance')
- CONFIG['proxy_database'] = os.path.join(CONFIG['slapos_directory'],
- 'proxy.db')
+ CONFIG['test_suite_directory'] = self.createDataDirectory('test_suite')
CONFIG['proxy_host'] = self.getLocalIPv4Address()
- CONFIG['master_url'] = 'http://%s:%s' % (CONFIG['proxy_host'],
- CONFIG['proxy_port'])
- self._createDirectory(CONFIG['software_root'])
- self._createDirectory(CONFIG['instance_root'])
- CONFIG['slapos_config'] = self.createConfigurationFile('slapos.cfg',
- self.substituteTemplate(pkg_resources.resource_filename(__name__,
- 'template/slapos.cfg.in'), CONFIG))
- self.path_list.append(CONFIG['slapos_config'])
-
- def setupRunningWrapper(self):
- self.path_list.extend(zc.buildout.easy_install.scripts([(
- 'testnode',
- __name__+'.testnode', 'run')], self.ws,
- sys.executable, self.wrapper_directory, arguments=[
- dict(
- computer_id=CONFIG['computer_id'],
- instance_dict=eval(self.parameter_dict.get('instance_dict', '{}')),
- instance_root=CONFIG['instance_root'],
- ipv4_address=self.getLocalIPv4Address(),
- ipv6_address=self.getGlobalIPv6Address(),
- master_url=CONFIG['master_url'],
- profile_path=self.parameter_dict['profile_path'],
- proxy_database=CONFIG['proxy_database'],
- proxy_port=CONFIG['proxy_port'],
- slapgrid_partition_binary=self.options['slapgrid_partition_binary'],
- slapgrid_software_binary=self.options['slapgrid_software_binary'],
- slapos_config=CONFIG['slapos_config'],
- slapproxy_binary=self.options['slapproxy_binary'],
- git_binary=self.options['git_binary'],
- software_root=CONFIG['software_root'],
- working_directory=CONFIG['working_directory'],
- vcs_repository_list=eval(self.parameter_dict.get('vcs_repository_list'),),
- node_quantity=self.parameter_dict.get('node_quantity', '1'),
- test_suite_master_url=self.parameter_dict.get(
- 'test_suite_master_url', None),
- test_suite=self.parameter_dict.get('test_suite'),
- test_suite_title=self.parameter_dict.get('test_suite_title'),
- test_node_title=self.parameter_dict.get('test_node_title'),
- project_title=self.parameter_dict.get('project_title'),
- bin_directory=self.bin_directory,
- # botenvironemnt is splittable string of key=value to substitute
- # environment of running bot
- bot_environment=self.parameter_dict.get('bot_environment', ''),
- partition_reference=CONFIG['partition_reference'],
- environment=dict(PATH=os.environ['PATH']),
- vcs_authentication_list=eval(self.parameter_dict.get(
- 'vcs_authentication_list', 'None')),
- )
- ]))
-
- def installLocalGit(self):
- git_dict = dict(git_binary = self.options['git_binary'])
- git_dict.update(self.parameter_dict)
- double_slash_end_position = 1
- # XXX, this should be provided by slapos
- print "bin_directory : %r" % self.bin_directory
- home_directory = os.path.join(*os.path.split(self.bin_directory)[0:-1])
- print "home_directory : %r" % home_directory
- git_dict.setdefault("git_server_name", "git.erp5.org")
- if git_dict.get('vcs_authentication_list', None) is not None:
- vcs_authentication_list = eval(git_dict['vcs_authentication_list'])
- netrc_file = open(os.path.join(home_directory, '.netrc'), 'w')
- for vcs_authentication_dict in vcs_authentication_list:
- netrc_file.write("""
-machine %(host)s
-login %(user_name)s
-password %(password)s
-""" % vcs_authentication_dict)
- netrc_file.close()
+ CONFIG['proxy_port'] = '5000'
+ CONFIG['log_directory'] = self.createDataDirectory('testnodelog')
+ CONFIG['run_directory'] = self.createDataDirectory('testnoderun')
+ CONFIG['test_suite_title'] = self.parameter_dict.get('test_suite_title')
+ CONFIG['test_node_title'] = self.parameter_dict.get('test_node_title')
+ CONFIG['test_suite'] = self.parameter_dict.get('test_suite')
+ CONFIG['node_quantity'] = self.parameter_dict.get('node_quantity', '1')
+ CONFIG['project_title'] = self.parameter_dict.get('project_title')
+ CONFIG['ipv4_address'] = self.getLocalIPv4Address()
+ CONFIG['ipv6_address'] = self.getGlobalIPv6Address()
+ CONFIG['test_suite_master_url'] = self.parameter_dict.get(
+ 'test_suite_master_url', None)
+ CONFIG['git_binary'] = self.options['git_binary']
+ CONFIG['slapgrid_partition_binary'] = self.options[
+ 'slapgrid_partition_binary']
+ CONFIG['slapgrid_software_binary'] = self.options[
+ 'slapgrid_software_binary']
+ CONFIG['slapproxy_binary'] = self.options['slapproxy_binary']
+ CONFIG['zip_binary'] = self.options['zip_binary']
+ CONFIG['PATH'] = os.environ['PATH']
+ additional_bt5_repository_id = \
+ self.parameter_dict.get('additional_bt5_repository_id')
- def installLocalRepository(self):
- self.installLocalGit()
+ CONFIG['bt5_path'] = None
+ if additional_bt5_repository_id is not None:
+ CONFIG['bt5_path'] = ""
+ additional_bt5_repository_id_list = additional_bt5_repository_id.split(",")
+ for id in additional_bt5_repository_id_list:
+ id_path = os.path.join(CONFIG['slapos_directory'], id)
+ bt_path = os.path.join(id_path, "bt5")
+ CONFIG['bt5_path'] += "%s,%s," % (id_path, bt_path)
+ CONFIG['instance_dict'] = ''
+ if 'instance_dict' in self.parameter_dict:
+ CONFIG['instance_dict'] = '[instance_dict]\n'
+ for k,v in eval(self.parameter_dict['instance_dict']).iteritems():
+ CONFIG['instance_dict'] += '%s = %s\n' % (k,v)
- def installLocalZip(self):
- zip = os.path.join(self.bin_directory, 'zip')
- if os.path.lexists(zip):
- os.unlink(zip)
- os.symlink(self.options['zip_binary'], zip)
-
- def installLocalPython(self):
- """Installs local python fully featured with eggs"""
- self.path_list.extend(zc.buildout.easy_install.scripts([], self.ws,
- sys.executable, self.bin_directory, scripts=None,
- interpreter='python'))
-
- def installLocalRunUnitTest(self):
- link = os.path.join(self.bin_directory, 'runUnitTest')
- destination = os.path.join(CONFIG['instance_root'],
- CONFIG['partition_reference'], 'bin', 'runUnitTest')
- if os.path.lexists(link):
- if not os.readlink(link) != destination:
- os.unlink(link)
- if not os.path.lexists(link):
- os.symlink(destination, link)
-
- def _install(self):
- self.requirements, self.ws = self.egg.working_set()
- self.path_list = []
- self.installSlapOs()
- self.setupRunningWrapper()
- self.installLocalRepository()
- self.installLocalZip()
- self.installLocalPython()
- self.installLocalRunUnitTest()
- return self.path_list
+ CONFIG['repository_list'] = ''
+ i = 0
+ for repository in eval(self.parameter_dict['vcs_repository_list']):
+ CONFIG['repository_list'] += '[vcs_repository_%s]\n' % i
+ CONFIG['repository_list'] += 'url = %s\n' % repository['url']
+ if 'branch' in repository:
+ CONFIG['repository_list'] += 'branch = %s\n' % repository['branch']
+ if 'profile_path' in repository:
+ CONFIG['repository_list'] += 'profile_path = %s\n' % repository[
+ 'profile_path']
+ if 'buildout_section_id' in repository:
+ CONFIG['repository_list'] += 'buildout_section_id = %s\n' % repository[
+ 'buildout_section_id']
+ CONFIG['repository_list'] += '\n'
+ i += 1
+ testnode_config = self.createConfigurationFile('erp5testnode.cfg',
+ self.substituteTemplate(pkg_resources.resource_filename(__name__,
+ 'template/erp5testnode.cfg.in'), CONFIG))
+ testnode_log = os.path.join(self.log_directory, 'erp5testnode.log')
+ wrapper = zc.buildout.easy_install.scripts([('erp5testnode',
+ 'slapos.recipe.librecipe.execute', 'executee')], self.ws, sys.executable,
+ self.wrapper_directory, arguments=[[self.options['testnode'], '-l',
+ testnode_log, testnode_config], {'GIT_SSL_NO_VERIFY': '1'}])[0]
+ path_list.append(testnode_config)
+ path_list.append(wrapper)
+ return path_list
diff --git a/slapos/recipe/erp5testnode/template/erp5testnode.cfg.in b/slapos/recipe/erp5testnode/template/erp5testnode.cfg.in
new file mode 100644
index 0000000000000000000000000000000000000000..08c4619a211392c6e48094b9b0f333e4ceaa9bb6
--- /dev/null
+++ b/slapos/recipe/erp5testnode/template/erp5testnode.cfg.in
@@ -0,0 +1,31 @@
+[testnode]
+slapos_directory = %(slapos_directory)s
+working_directory = %(slapos_directory)s
+test_suite_directory = %(test_suite_directory)s
+log_directory = %(log_directory)s
+run_directory = %(run_directory)s
+proxy_host = %(proxy_host)s
+proxy_port = %(proxy_port)s
+test_suite_title = %(test_suite_title)s
+test_suite = %(test_suite)s
+node_quantity = %(node_quantity)s
+test_node_title = %(test_node_title)s
+project_title= %(project_title)s
+ipv4_address = %(ipv4_address)s
+ipv6_address = %(ipv6_address)s
+test_suite_master_url = %(test_suite_master_url)s
+bt5_path = %(bt5_path)s
+
+# Binaries
+git_binary = %(git_binary)s
+slapgrid_partition_binary = %(slapgrid_partition_binary)s
+slapgrid_software_binary = %(slapgrid_software_binary)s
+slapproxy_binary = %(slapproxy_binary)s
+zip_binary = %(zip_binary)s
+
+[environment]
+PATH = %(PATH)s
+
+%(instance_dict)s
+
+%(repository_list)s
diff --git a/slapos/recipe/erp5testnode/template/slapos.cfg.in b/slapos/recipe/erp5testnode/template/slapos.cfg.in
deleted file mode 100644
index 713f719a322502bca230db83a0c2aa4c6678607c..0000000000000000000000000000000000000000
--- a/slapos/recipe/erp5testnode/template/slapos.cfg.in
+++ /dev/null
@@ -1,10 +0,0 @@
-[slapos]
-software_root = %(software_root)s
-instance_root = %(instance_root)s
-master_url = %(master_url)s
-computer_id = %(computer_id)s
-
-[slapproxy]
-host = %(proxy_host)s
-port = %(proxy_port)s
-database_uri = %(proxy_database)s
diff --git a/slapos/recipe/erp5testnode/testnode.py b/slapos/recipe/erp5testnode/testnode.py
deleted file mode 100644
index f090a3a8668fcc91f4a1012dcac5593156a0334b..0000000000000000000000000000000000000000
--- a/slapos/recipe/erp5testnode/testnode.py
+++ /dev/null
@@ -1,245 +0,0 @@
-from xml_marshaller import xml_marshaller
-import os, xmlrpclib, time, imp
-from glob import glob
-import signal
-import slapos.slap
-import subprocess
-import sys
-import socket
-import pprint
-from SlapOSControler import SlapOSControler
-
-
-class SubprocessError(EnvironmentError):
- def __init__(self, status_dict):
- self.status_dict = status_dict
- def __getattr__(self, name):
- return self.status_dict[name]
- def __str__(self):
- return 'Error %i' % self.status_code
-
-
-from Updater import Updater
-
-process_group_pid_set = set()
-process_pid_file_list = []
-process_command_list = []
-def sigterm_handler(signal, frame):
- for pgpid in process_group_pid_set:
- try:
- os.killpg(pgpid, signal.SIGTERM)
- except:
- pass
- for pid_file in process_pid_file_list:
- try:
- os.kill(int(open(pid_file).read().strip()), signal.SIGTERM)
- except:
- pass
- for p in process_command_list:
- try:
- subprocess.call(p)
- except:
- pass
- sys.exit(1)
-
-signal.signal(signal.SIGTERM, sigterm_handler)
-
-def safeRpcCall(function, *args):
- retry = 64
- while True:
- try:
- return function(*args)
- except (socket.error, xmlrpclib.ProtocolError), e:
- print >>sys.stderr, e
- pprint.pprint(args, file(function._Method__name, 'w'))
- time.sleep(retry)
- retry += retry >> 1
-
-def getInputOutputFileList(config, command_name):
- stdout = open(os.path.join(
- config['instance_root'],'.%s_out' % command_name),
- 'w+')
- stdout.write("%s\n" % command_name)
- stderr = open(os.path.join(
- config['instance_root'],'.%s_err' % command_name),
- 'w+')
- return (stdout, stderr)
-
-slapos_controler = None
-
-def run(args):
- config = args[0]
- slapgrid = None
- supervisord_pid_file = os.path.join(config['instance_root'], 'var', 'run',
- 'supervisord.pid')
- subprocess.check_call([config['git_binary'],
- "config", "--global", "http.sslVerify", "false"])
- previous_revision = None
-
- run_software = True
- # Write our own software.cfg to use the local repository
- custom_profile_path = os.path.join(config['working_directory'], 'software.cfg')
- config['custom_profile_path'] = custom_profile_path
- vcs_repository_list = config['vcs_repository_list']
- profile_content = None
- assert len(vcs_repository_list), "we must have at least one repository"
- for vcs_repository in vcs_repository_list:
- url = vcs_repository['url']
- buildout_section_id = vcs_repository.get('buildout_section_id', None)
- repository_id = buildout_section_id or \
- url.split('/')[-1].split('.')[0]
- repository_path = os.path.join(config['working_directory'],repository_id)
- vcs_repository['repository_id'] = repository_id
- vcs_repository['repository_path'] = repository_path
- if profile_content is None:
- profile_content = """
-[buildout]
-extends = %(software_config_path)s
-""" % {'software_config_path': os.path.join(repository_path,
- config['profile_path'])}
- if not(buildout_section_id is None):
- profile_content += """
-[%(buildout_section_id)s]
-repository = %(repository_path)s
-branch = %(branch)s
-""" % {'buildout_section_id': buildout_section_id,
- 'repository_path' : repository_path,
- 'branch' : vcs_repository.get('branch','master')}
-
- custom_profile = open(custom_profile_path, 'w')
- custom_profile.write(profile_content)
- custom_profile.close()
- config['repository_path'] = repository_path
- sys.path.append(repository_path)
- test_suite_title = config['test_suite_title'] or config['test_suite']
-
- retry_software = False
- try:
- while True:
- # kill processes from previous loop if any
- try:
- for pgpid in process_group_pid_set:
- try:
- os.killpg(pgpid, signal.SIGTERM)
- except:
- pass
- process_group_pid_set.clear()
- full_revision_list = []
- # Make sure we have local repository
- for vcs_repository in vcs_repository_list:
- repository_path = vcs_repository['repository_path']
- repository_id = vcs_repository['repository_id']
- if not os.path.exists(repository_path):
- parameter_list = [config['git_binary'], 'clone',
- vcs_repository['url']]
- if vcs_repository.get('branch') is not None:
- parameter_list.extend(['-b',vcs_repository.get('branch')])
- parameter_list.append(repository_path)
- subprocess.check_call(parameter_list)
- # Make sure we have local repository
- updater = Updater(repository_path, git_binary=config['git_binary'])
- updater.checkout()
- revision = "-".join(updater.getRevision())
- full_revision_list.append('%s=%s' % (repository_id, revision))
- revision = ','.join(full_revision_list)
- if previous_revision == revision:
- time.sleep(120)
- if not(retry_software):
- continue
- retry_software = False
- previous_revision = revision
-
- print config
- portal_url = config['test_suite_master_url']
- test_result_path = None
- test_result = (test_result_path, revision)
- if portal_url:
- if portal_url[-1] != '/':
- portal_url += '/'
- portal = xmlrpclib.ServerProxy("%s%s" %
- (portal_url, 'portal_task_distribution'),
- allow_none=1)
- master = portal.portal_task_distribution
- assert master.getProtocolRevision() == 1
- test_result = safeRpcCall(master.createTestResult,
- config['test_suite'], revision, [],
- False, test_suite_title,
- config['test_node_title'], config['project_title'])
- print "testnode, test_result : %r" % (test_result,)
- if test_result:
- test_result_path, test_revision = test_result
- if revision != test_revision:
- for i, repository_revision in enumerate(test_revision.split(',')):
- vcs_repository = vcs_repository_list[i]
- repository_path = vcs_repository['repository_path']
- # other testnodes on other boxes are already ready to test another
- # revision
- updater = Updater(repository_path, git_binary=config['git_binary'],
- revision=repository_revision.split('-')[1])
- updater.checkout()
-
- # Now prepare the installation of SlapOS and create instance
- slapos_controler = SlapOSControler(config,
- process_group_pid_set=process_group_pid_set)
- for method_name in ("runSoftwareRelease", "runComputerPartition"):
- stdout, stderr = getInputOutputFileList(config, method_name)
- slapos_method = getattr(slapos_controler, method_name)
- status_dict = slapos_method(config,
- environment=config['environment'],
- process_group_pid_set=process_group_pid_set,
- stdout=stdout, stderr=stderr
- )
- if status_dict['status_code'] != 0:
- break
- if status_dict['status_code'] != 0:
- safeRpcCall(master.reportTaskFailure,
- test_result_path, status_dict, config['test_node_title'])
- retry_software = True
- continue
-
- partition_path = os.path.join(config['instance_root'],
- config['partition_reference'])
- run_test_suite_path = os.path.join(partition_path, 'bin',
- 'runTestSuite')
- if not os.path.exists(run_test_suite_path):
- raise ValueError('No %r provided' % run_test_suite_path)
-
- run_test_suite_revision = revision
- if isinstance(revision, tuple):
- revision = ','.join(revision)
- # Deal with Shebang size limitation
- file_object = open(run_test_suite_path, 'r')
- line = file_object.readline()
- file_object.close()
- invocation_list = []
- if line[:2] == '#!':
- invocation_list = line[2:].split()
- invocation_list.extend([run_test_suite_path,
- '--test_suite', config['test_suite'],
- '--revision', revision,
- '--test_suite_title', test_suite_title,
- '--node_quantity', config['node_quantity'],
- '--master_url', config['test_suite_master_url']])
- run_test_suite = subprocess.Popen(invocation_list)
- process_group_pid_set.add(run_test_suite.pid)
- run_test_suite.wait()
- process_group_pid_set.remove(run_test_suite.pid)
- except SubprocessError:
- time.sleep(120)
- continue
-
- finally:
- # Nice way to kill *everything* generated by run process -- process
- # groups working only in POSIX compilant systems
- # Exceptions are swallowed during cleanup phase
- print "going to kill %r" % (process_group_pid_set,)
- for pgpid in process_group_pid_set:
- try:
- os.killpg(pgpid, signal.SIGTERM)
- except:
- pass
- try:
- if os.path.exists(supervisord_pid_file):
- os.kill(int(open(supervisord_pid_file).read().strip()), signal.SIGTERM)
- except:
- pass
\ No newline at end of file
diff --git a/software/erp5testnode/instance.cfg b/software/erp5testnode/instance.cfg
index 2b0c318416e2e56226958a4c1174bfadc828c8db..5c75e01d537f21a362dc5e16e83070017998f5af 100644
--- a/software/erp5testnode/instance.cfg
+++ b/software/erp5testnode/instance.cfg
@@ -8,11 +8,11 @@ develop-eggs-directory = ${buildout:develop-eggs-directory}
[testnode]
recipe = slapos.cookbook:erp5testnode
-buildbot_binary = ${buildout:bin-directory}/buildbot
+git_binary = ${git:location}/bin/git
slapgrid_partition_binary = ${buildout:bin-directory}/slapgrid-cp
slapgrid_software_binary = ${buildout:bin-directory}/slapgrid-sr
slapproxy_binary = ${buildout:bin-directory}/slapproxy
svn_binary = ${subversion:location}/bin/svn
-git_binary = ${git:location}/bin/git
svnversion_binary = ${subversion:location}/bin/svnversion
+testnode = ${buildout:bin-directory}/testnode
zip_binary = ${zip:location}/bin/zip
diff --git a/software/erp5testnode/software.cfg b/software/erp5testnode/software.cfg
index 3fdec1ac67996817ffdbf8143f38dd0b00b8e6f1..7060022cf5486fabdeb783a6093016207d7b7a17 100644
--- a/software/erp5testnode/software.cfg
+++ b/software/erp5testnode/software.cfg
@@ -1,8 +1,4 @@
[buildout]
-extensions =
- slapos.rebootstrap
- slapos.zcbworkarounds
- mr.developer
find-links = http://www.nexedi.org/static/packages/source/slapos.buildout/
http://dist.repoze.org
@@ -14,10 +10,10 @@ include-site-packages = false
exec-sitecustomize = false
versions = versions
-rebootstrap-section = python2.6
extends =
- ../../component/python-2.6/buildout.cfg
+ ../../stack/shacache-client.cfg
+ ../../component/python-2.7/buildout.cfg
../../component/subversion/buildout.cfg
../../component/git/buildout.cfg
../../component/lxml-python/buildout.cfg
@@ -25,37 +21,125 @@ extends =
parts =
template
- bootstrap
+ lxml-python
eggs
subversion
zip
git
+ checkrecipe
-[bootstrap]
-recipe = zc.recipe.egg
-eggs = zc.buildout
-suffix =
-scripts =
- buildout=bootstrap2.6
-arguments = sys.argv[1:] + ["bootstrap"]
+# Separate from site eggs
+allowed-eggs-from-site-packages =
+include-site-packages = false
+exec-sitecustomize = false
+
+# Use only quite well working sites.
+allow-hosts =
+ *.nexedi.org
+ *.python.org
+ *.sourceforge.net
+ dist.repoze.org
+ effbot.org
+ github.com
+ peak.telecommunity.com
+ psutil.googlecode.com
+ www.dabeaz.com
+
+develop =
+ ${:parts-directory}/slapos.cookbook-repository
+ ${:parts-directory}/erp5.util-repository
-[rebootstrap]
-section = python2.6
-version = 1
+[checkrecipe]
+recipe = plone.recipe.command
+stop-on-error = true
+update-command = ${:command}
+command = grep parts ${buildout:develop-eggs-directory}/slapos.cookbook.egg-link && grep parts ${buildout:develop-eggs-directory}/erp5.util.egg-link
+
+[slapos.cookbook-repository]
+recipe = plone.recipe.command
+location = ${buildout:parts-directory}/${:_buildout_section_name_}
+stop-on-error = true
+repository = http://git.erp5.org/repos/slapos.git
+branch = erp5testnode
+revision =
+command = ${git:location}/bin/git clone --quiet -b ${:branch} ${:repository} ${:location} && if [ -n ${:revision} ]; then cd ${:location} && ${git:location}/bin/git reset --quiet --hard ${:revision} ; fi
+update-command = cd ${:location} && ${git:location}/bin/git pull --quiet && if [ -n ${:revision} ]; then cd ${:location} && ${git:location}/bin/git reset --quiet --hard ${:revision} ; fi
+
+[erp5.util-repository]
+recipe = plone.recipe.command
+location = ${buildout:parts-directory}/${:_buildout_section_name_}
+stop-on-error = true
+repository = http://git.erp5.org/repos/erp5.git
+branch = master
+revision =
+command = ${git:location}/bin/git clone --quiet -b ${:branch} ${:repository} ${:location} && if [ -n ${:revision} ]; then cd ${:location} && ${git:location}/bin/git reset --quiet --hard ${:revision} ; fi
+update-command = cd ${:location} && ${git:location}/bin/git pull --quiet && if [ -n ${:revision} ]; then cd ${:location} && ${git:location}/bin/git reset --quiet --hard ${:revision} ; fi
[versions]
# Use SlapOS patched zc.buildout
-zc.buildout = 1.5.3-dev-SlapOS-001
+zc.buildout = 1.5.3-dev-SlapOS-009
[eggs]
recipe = zc.recipe.egg
+# Just so buildout executes [slapos.cookbook-repository] before [eggs], as
+# - [eggs] references [slapos.cookbook-repository]
+# - [instance-recipe] needs [slapos.cookbook-repository] to be finished
+# - we cannot rely on anything else being executed before [eggs]
+dummy =
+ ${slapos.cookbook-repository:location}
+ ${erp5.util-repository:location}
eggs =
${lxml-python:egg}
+ zc.buildout
+ slapos.libnetworkcache
slapos.core
slapos.cookbook
+ erp5.util[testnode]
+
+scripts =
+ testnode = erp5.util.testnode:main
+ slapgrid-cp = slapos.grid.slapgrid:runComputerPartition
+ slapgrid-sr = slapos.grid.slapgrid:runSoftwareRelease
+ slapproxy = slapos.proxy:main
+
+python = python2.7
+
+[lxml-python]
+python = python2.7
[template]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance.cfg
output = ${buildout:directory}/template.cfg
mode = 0644
+md5sum = 08e3f92bce41efc5bfe044bb9d354786
+
+[networkcache]
+# Romain Courteaud + Sebastien Robin signature certificate
+signature-certificate-list =
+ -----BEGIN CERTIFICATE-----
+ MIIB4DCCAUkCADANBgkqhkiG9w0BAQsFADA5MQswCQYDVQQGEwJGUjEZMBcGA1UE
+ CBMQRGVmYXVsdCBQcm92aW5jZTEPMA0GA1UEChMGTmV4ZWRpMB4XDTExMDkxNTA5
+ MDAwMloXDTEyMDkxNTA5MDAwMlowOTELMAkGA1UEBhMCRlIxGTAXBgNVBAgTEERl
+ ZmF1bHQgUHJvdmluY2UxDzANBgNVBAoTBk5leGVkaTCBnzANBgkqhkiG9w0BAQEF
+ AAOBjQAwgYkCgYEApYZv6OstoqNzxG1KI6iE5U4Ts2Xx9lgLeUGAMyfJLyMmRLhw
+ boKOyJ9Xke4dncoBAyNPokUR6iWOcnPHtMvNOsBFZ2f7VA28em3+E1JRYdeNUEtX
+ Z0s3HjcouaNAnPfjFTXHYj4um1wOw2cURSPuU5dpzKBbV+/QCb5DLheynisCAwEA
+ ATANBgkqhkiG9w0BAQsFAAOBgQBCZLbTVdrw3RZlVVMFezSHrhBYKAukTwZrNmJX
+ mHqi2tN8tNo6FX+wmxUUAf3e8R2Ymbdbn2bfbPpcKQ2fG7PuKGvhwMG3BlF9paEC
+ q7jdfWO18Zp/BG7tagz0jmmC4y/8akzHsVlruo2+2du2freE8dK746uoMlXlP93g
+ QUUGLQ==
+ -----END CERTIFICATE-----
+ -----BEGIN CERTIFICATE-----
+ MIIB8jCCAVugAwIBAgIJAPu2zchZ2BxoMA0GCSqGSIb3DQEBBQUAMBIxEDAOBgNV
+ BAMMB3RzeGRldjMwHhcNMTExMDE0MTIxNjIzWhcNMTIxMDEzMTIxNjIzWjASMRAw
+ DgYDVQQDDAd0c3hkZXYzMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQCrPbh+
+ YGmo6mWmhVb1vTqX0BbeU0jCTB8TK3i6ep3tzSw2rkUGSx3niXn9LNTFNcIn3MZN
+ XHqbb4AS2Zxyk/2tr3939qqOrS4YRCtXBwTCuFY6r+a7pZsjiTNddPsEhuj4lEnR
+ L8Ax5mmzoi9nE+hiPSwqjRwWRU1+182rzXmN4QIDAQABo1AwTjAdBgNVHQ4EFgQU
+ /4XXREzqBbBNJvX5gU8tLWxZaeQwHwYDVR0jBBgwFoAU/4XXREzqBbBNJvX5gU8t
+ LWxZaeQwDAYDVR0TBAUwAwEB/zANBgkqhkiG9w0BAQUFAAOBgQA07q/rKoE7fAda
+ FED57/SR00OvY9wLlFEF2QJ5OLu+O33YUXDDbGpfUSF9R8l0g9dix1JbWK9nQ6Yd
+ R/KCo6D0sw0ZgeQv1aUXbl/xJ9k4jlTxmWbPeiiPZEqU1W9wN5lkGuLxV4CEGTKU
+ hJA/yXa1wbwIPGvX3tVKdOEWPRXZLg==
+ -----END CERTIFICATE-----