Commit bc1024a6 authored by Łukasz Nowak's avatar Łukasz Nowak

Testnode instantiation recipe.

parent 7f469d08
Changelog
=========
1.0 (unreleased)
----------------
include CHANGES.txt
recursive-include src/slapos/recipe/testnode *.in
The slapos.recipe.tesnode aims to install generic testnode.
from setuptools import setup, find_packages
name = "slapos.recipe.testnode"
version = '1.3-dev-7'
def read(name):
return open(name).read()
long_description=( read('README.txt')
+ '\n' +
read('CHANGES.txt')
)
setup(
name = name,
version = version,
description = "ZC Buildout recipe for create an testnode instance",
long_description=long_description,
license = "GPLv3",
keywords = "buildout slapos test",
classifiers=[
"Framework :: Buildout :: Recipe",
"Programming Language :: Python",
],
packages = find_packages('src'),
package_dir = {'': 'src'},
include_package_data=True,
install_requires = [
'setuptools',
'slapos.lib.recipe',
'xml_marshaller',
'zc.buildout',
'zc.recipe.egg',
# below are requirements to provide full blown python interpreter
'lxml',
'PyXML',
],
namespace_packages = ['slapos', 'slapos.recipe'],
entry_points = {'zc.buildout': ['default = %s:Recipe' % name]},
)
# See http://peak.telecommunity.com/DevCenter/setuptools#namespace-packages
try:
__import__('pkg_resources').declare_namespace(__name__)
except ImportError:
from pkgutil import extend_path
__path__ = extend_path(__path__, __name__)
# See http://peak.telecommunity.com/DevCenter/setuptools#namespace-packages
try:
__import__('pkg_resources').declare_namespace(__name__)
except ImportError:
from pkgutil import extend_path
__path__ = extend_path(__path__, __name__)
##############################################################################
#
# Copyright (c) 2010 Vifib SARL and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
from slapos.lib.recipe.BaseSlapRecipe import BaseSlapRecipe
import os
import pkg_resources
import zc.buildout
import zc.recipe.egg
import sys
CONFIG = dict(
proxy_port='5000',
computer_id='COMPUTER',
partition_reference='test0',
)
class Recipe(BaseSlapRecipe):
def __init__(self, buildout, name, options):
self.egg = zc.recipe.egg.Egg(buildout, options['recipe'], options)
BaseSlapRecipe.__init__(self, buildout, name, options)
def installSlapOs(self):
CONFIG['slapos_directory'] = self.createDataDirectory('slapos')
CONFIG['working_directory'] = self.createDataDirectory('testnode')
CONFIG['software_root'] = os.path.join(CONFIG['slapos_directory'],
'software')
CONFIG['instance_root'] = os.path.join(CONFIG['slapos_directory'],
'instance')
CONFIG['proxy_database'] = os.path.join(CONFIG['slapos_directory'],
'proxy.db')
CONFIG['proxy_host'] = self.getLocalIPv4Address()
CONFIG['master_url'] = 'http://%s:%s' % (CONFIG['proxy_host'],
CONFIG['proxy_port'])
self._createDirectory(CONFIG['software_root'])
self._createDirectory(CONFIG['instance_root'])
CONFIG['slapos_config'] = self.createConfigurationFile('slapos.cfg',
self.substituteTemplate(pkg_resources.resource_filename(__name__,
'template/slapos.cfg.in'), CONFIG))
self.path_list.append(CONFIG['slapos_config'])
def setupRunningWrapper(self):
self.path_list.extend(zc.buildout.easy_install.scripts([(
'testnode',
__name__+'.testnode', 'run')], self.ws,
sys.executable, self.wrapper_directory, arguments=[
dict(
environment=self.getRuntimeEnvironment(),
computer_id=CONFIG['computer_id'],
instance_dict=eval(self.parameter_dict.get('instance_dict', '{}')),
instance_root=CONFIG['instance_root'],
ipv4_address=self.getLocalIPv4Address(),
ipv6_address=self.getGlobalIPv6Address(),
master_url=CONFIG['master_url'],
profile_url=self.parameter_dict['profile_url'],
proxy_database=CONFIG['proxy_database'],
slapgrid_partition_binary=self.options['slapgrid_partition_binary'],
slapgrid_software_binary=self.options['slapgrid_software_binary'],
slapos_config=CONFIG['slapos_config'],
slapproxy_binary=self.options['slapproxy_binary'],
software_root=CONFIG['software_root'],
buildbot_binary=self.options['buildbot_binary'],
working_directory=CONFIG['working_directory'],
buildbot_host=self.parameter_dict['buildbot_host'],
slave_name=self.parameter_dict['slave_name'],
slave_password=self.parameter_dict['slave_password'],
bin_directory=self.bin_directory,
# botenvironemnt is splittable string of key=value to substitute
# environment of running bot
bot_environment=self.parameter_dict.get('bot_environment', ''),
partition_reference=CONFIG['partition_reference'],
)
]))
def installLocalSvn(self):
svn_dict = dict(svn_binary = self.options['svn_binary'])
svn_dict.update(self.parameter_dict)
svn_path = os.path.join(self.bin_directory, 'svn')
self._writeExecutable(svn_path, """\
#!/bin/sh
%(svn_binary)s --username %(svn_username)s --password %(svn_password)s \
--non-interactive --trust-server-cert --no-auth-cache "$@" """% svn_dict)
self.path_list.append(svn_path)
svnversion = os.path.join(self.bin_directory, 'svnversion')
if os.path.lexists(svnversion):
os.unlink(svnversion)
os.symlink(self.options['svnversion_binary'], svnversion)
self.path_list.append(svnversion)
def installLocalGit(self):
git = os.path.join(self.bin_directory, 'git')
if os.path.lexists(git):
os.unlink(git)
os.symlink(self.options['git_binary'], git)
self.path_list.append(git)
def installLocalZip(self):
zip = os.path.join(self.bin_directory, 'zip')
if os.path.lexists(zip):
os.unlink(zip)
os.symlink(self.options['zip_binary'], zip)
self.path_list.append(zip)
def installLocalPython(self):
"""Installs local python fully featured with eggs"""
self.path_list.extend(zc.buildout.easy_install.scripts([], self.ws,
sys.executable, self.bin_directory, scripts=None,
interpreter='python'))
def installLocalRunUnitTest(self):
link = os.path.join(self.bin_directory, 'runUnitTest')
destination = os.path.join(CONFIG['instance_root'],
CONFIG['partition_reference'], 'bin', 'runUnitTest')
if os.path.lexists(link):
if not os.readlink(link) != destination:
os.unlink(link)
if not os.path.lexists(link):
os.symlink(destination, link)
self.path_list.append(link)
def _installBuildbot(self):
self.setupRunningWrapper()
self.installLocalPython()
self.installLocalGit()
self.installLocalSvn()
self.installLocalRunUnitTest()
return self.path_list
def getRuntimeEnvironment(self):
env = {}
env['PATH'] = ':'.join([self.bin_directory] +
os.environ['PATH'].split(':'))
return env
def _installProfileTesting(self):
self.path_list.extend(zc.buildout.easy_install.scripts([(
'testnode',
__name__+'.profile_testnode', 'run')], self.ws,
sys.executable, self.wrapper_directory, arguments=[
dict(
environment=self.getRuntimeEnvironment(),
slapgrid_environment=eval(self.parameter_dict.get(
'slapgrid_environment', '{}')),
profile_path=self.parameter_dict.get('profile_path',
'slapos/software.cfg'),
repository=self.parameter_dict['repository'],
# Optional URL of test aggreagation system
test_suite_master_url=self.parameter_dict['test_suite_master_url'],
suite_name=self.parameter_dict['suite_name'],
branch=self.parameter_dict.get('branch', 'master'),
# internal parameters
software_root=CONFIG['software_root'],
computer_id=CONFIG['computer_id'],
git_binary=self.options['git_binary'],
master_url=CONFIG['master_url'],
proxy_database=CONFIG['proxy_database'],
slapgrid_software_binary=self.options['slapgrid_software_binary'],
slapos_config=CONFIG['slapos_config'],
slapproxy_binary=self.options['slapproxy_binary'],
working_directory=CONFIG['working_directory'],
bin_directory=self.bin_directory,
partition_reference=CONFIG['partition_reference'],
)
]))
return self.path_list
def _install(self):
self.requirements, self.ws = self.egg.working_set([__name__])
self.path_list = []
self.installSlapOs()
self.installLocalZip()
flavour = self.parameter_dict.get('flavour', 'buildbot')
if flavour == 'buildbot':
return self._installBuildbot()
elif flavour == 'profile-testing':
return self._installProfileTesting()
raise NotImplementedError('Falvour %r is unknown'% flavour)
import urlparse
import urllib
import httplib
import mimetools
from random import randint
import tempfile
import os
import stat
import zipfile
import mimetypes
import datetime
TB_SEP = "============================================================="\
"========="
def get_content_type(f):
return mimetypes.guess_type(f.name)[0] or 'application/octet-stream'
class ConnectionHelper:
def __init__(self, url):
self.conn = urlparse.urlparse(url)
if self.conn.scheme == 'http':
connection_type = httplib.HTTPConnection
if self.conn.port is None:
self.port = 80
else:
connection_type = httplib.HTTPSConnection
if self.conn.port is None:
self.port = 443
self.connection_type = connection_type
def _connect(self):
self.connection = self.connection_type(self.conn.hostname + ':' +
str(self.conn.port or self.port))
def POST(self, path, parameter_dict, file_list=None):
self._connect()
parameter_dict.update(__ac_name=self.conn.username,
__ac_password=self.conn.password)
header_dict = {'Content-type': "application/x-www-form-urlencoded"}
if file_list is None:
body = urllib.urlencode(parameter_dict)
else:
boundary = mimetools.choose_boundary()
header_dict['Content-type'] = 'multipart/form-data; boundary=%s' % (
boundary,)
body = ''
for k, v in parameter_dict.iteritems():
body += '--%s\r\n' % boundary
body += 'Content-Disposition: form-data; name="%s"\r\n' % k
body += '\r\n'
body += '%s\r\n' % v
for name, filename in file_list:
f = open(filename, 'r')
body += '--%s\r\n' % boundary
body += 'Content-Disposition: form-data; name="%s"; filename="%s"\r\n'\
% (name, name)
body += 'Content-Type: %s\r\n' % get_content_type(f)
body += 'Content-Length: %d\r\n' % os.fstat(f.fileno())[stat.ST_SIZE]
body += '\r\n'
body += f.read()
f.close()
body += '\r\n'
self.connection.request("POST", self.conn.path + '/' + path,
body, header_dict)
self.response = self.connection.getresponse()
class ERP5TestReportHandler:
def __init__(self, url, suite_name):
# random test id
self.test_id = "%s-%X" % (
("%s" % datetime.date.today()).replace("-", ""),
randint(1, 10000000000000000),
)
self.connection_helper = ConnectionHelper(url)
self.suite_name = suite_name
def reportStart(self):
# report that test is running
print 'Starting test with id %s' % self.test_id
self.connection_helper.POST('TestResultModule_reportRunning', dict(
test_suite=self.suite_name,
test_report_id=self.test_id,
))
def reportFinished(self, out_file, revision, success, duration, text):
# make file parsable by erp5_test_results
tempcmd = tempfile.mkstemp()[1]
tempcmd2 = tempfile.mkstemp()[1]
tempout = tempfile.mkstemp()[1]
templog = tempfile.mkstemp()[1]
log_lines = open(out_file, 'r').readlines()
tl = open(templog, 'w')
tl.write(TB_SEP + '\n')
if len(log_lines) > 900:
tl.write('...[truncated]... \n\n')
for log_line in log_lines[-900:]:
starts = log_line.startswith
if starts('Ran') or starts('FAILED') or starts('OK') or starts(TB_SEP):
continue
if starts('ERROR: ') or starts('FAIL: '):
tl.write('internal-test: ' + log_line)
continue
tl.write(log_line)
tl.write("----------------------------------------------------------------------\n")
tl.write('Ran 1 test in %.2fs\n' % duration)
if success:
tl.write('OK\n')
else:
tl.write('FAILED (failures=1)\n')
tl.write(TB_SEP + '\n')
tl.close()
open(tempcmd, 'w').write("""svn info dummy""")
open(tempcmd2, 'w').write(self.suite_name)
open(tempout, 'w').write("Revision: %s\n%s" % (revision, text))
# create nice zip archive
tempzip = tempfile.mkstemp()[1]
zip = zipfile.ZipFile(tempzip, 'w')
zip.write(tempcmd, 'dummy/001/cmdline')
zip.write(tempout, 'dummy/001/stdout')
zip.write(templog, 'dummy/001/stderr')
zip.write(tempout, '%s/002/stdout' % self.suite_name)
zip.write(templog, '%s/002/stderr' % self.suite_name)
zip.write(tempcmd2, '%s/002/cmdline' % self.suite_name)
zip.close()
os.unlink(templog)
os.unlink(tempcmd)
os.unlink(tempout)
os.unlink(tempcmd2)
# post it to ERP5
self.connection_helper.POST('TestResultModule_reportCompleted', dict(
test_report_id=self.test_id),
file_list=[('filepath', tempzip)]
)
os.unlink(tempzip)
import os
import socket
import signal
import shutil
import slapos.slap
import subprocess
import time
import atexit
from erp5testreporthandler import ERP5TestReportHandler
process_group_pid_list = []
def clean():
for pgpid in process_group_pid_list:
try:
os.killpg(pgpid, signal.SIGTERM)
except:
pass
def sigterm_handler(signal, frame):
clean()
def sigint_handler(signal, frame):
clean()
raise KeyboardInterrupt
signal.signal(signal.SIGINT, sigint_handler)
signal.signal(signal.SIGTERM, sigterm_handler)
atexit.register(clean)
def getCurrentBranchName(config, p):
r = subprocess.Popen([config['git_binary'], 'branch'], stdout=subprocess.PIPE, cwd=p).communicate()[0]
for f in r.splitlines():
if f.startswith('*'):
return f.split()[1]
return ''
def getRevision(config, p):
return subprocess.Popen([config['git_binary'], 'rev-parse', 'HEAD'], stdout=subprocess.PIPE, cwd=p).communicate()[0].strip()
def getCurrentFetchRemote(config, p):
r = subprocess.Popen([config['git_binary'], 'remote', '-v'], stdout=subprocess.PIPE, cwd=p).communicate()[0]
remote = ''
for f in r.splitlines():
if f.startswith('origin') and f.endswith('(fetch)'):
if remote != '':
raise ValueError('Too many remotes: %s' % r)
remote = r.split()[1]
return remote
def getMachineIdString():
"""Returns machine identification string"""
kw = dict(stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
idstr = subprocess.Popen(["uname", "-m"], **kw).communicate()[0].strip()
# try to detect gcc version
try:
gcc_list = subprocess.Popen(["gcc", "-v"], **kw).communicate()[0].split(
'\n')
for gcc in gcc_list:
if gcc.startswith('gcc version'):
idstr += ' gcc:' + gcc.split()[2]
break
except IndexError:
pass
# try to detect libc version
try:
libdir = os.path.sep + 'lib'
for libso in os.listdir(libdir):
if libso.startswith('libc.') and os.path.islink(os.path.join(libdir,
libso)):
libc = os.readlink(os.path.join(libdir, libso))
if libc.endswith('.so'):
idstr += ' libc:' + libc.split('-')[1][:-3]
else:
idstr += ' ' + libc
break
except IndexError:
pass
return idstr
def run(args):
config = args[0]
for k,v in config['environment'].iteritems():
os.environ[k] = v
proxy = None
slapgrid = None
last_revision_file = os.path.join(config['working_directory'],
'revision.txt')
if os.path.exists(last_revision_file):
os.unlink(last_revision_file)
# fetch repository from git
repository_clone = os.path.join(config['working_directory'], 'repository')
profile_path = os.path.join(repository_clone, config['profile_path'])
if os.path.exists(config['proxy_database']):
os.unlink(config['proxy_database'])
proxy = subprocess.Popen([config['slapproxy_binary'],
config['slapos_config']], close_fds=True, preexec_fn=os.setsid)
process_group_pid_list.append(proxy.pid)
slap = slapos.slap.slap()
slap.initializeConnection(config['master_url'])
while True:
try:
slap.registerSupply().supply(profile_path,
computer_guid=config['computer_id'])
except socket.error:
time.sleep(1)
pass
else:
break
while True:
info_list = []
a = info_list.append
while True:
try:
if os.path.exists(repository_clone):
if getCurrentFetchRemote(config, repository_clone) != config['repository']:
shutil.rmtree(repository_clone)
if not os.path.exists(repository_clone):
subprocess.check_call([config['git_binary'], 'clone',
config['repository'], repository_clone])
# switch to branch
branch = getCurrentBranchName(config, repository_clone)
if branch != config['branch']:
subprocess.check_call([config['git_binary'], 'checkout', '--force',
'--track', '-b', config['branch'], 'origin/'+config['branch']],
cwd=repository_clone)
subprocess.check_call([config['git_binary'], 'pull', '--rebase'],
cwd=repository_clone)
except Exception:
print 'Retrying git in 60s'
time.sleep(60)
else:
break
a('Tested repository: %s' % config['repository'])
a('Machine identification: %s' % getMachineIdString())
erp5_report = ERP5TestReportHandler(config['test_suite_master_url'],
'@'.join([config['suite_name'], branch]))
last_revision = ''
if os.path.exists(last_revision_file):
last_revision = open(last_revision_file).read().strip()
revision = getRevision(config, repository_clone)
open(last_revision_file, 'w').write(revision)
if revision != last_revision:
print 'Running for revision %r' % revision
while True:
try:
erp5_report.reportStart()
except Exception:
print 'Retrying in 5s'
time.sleep(5)
else:
break
if os.path.exists(config['software_root']):
shutil.rmtree(config['software_root'])
os.mkdir(config['software_root'])
out_file = os.path.join(config['working_directory'], 'slapgrid.out')
if os.path.exists(out_file):
os.unlink(out_file)
out = open(out_file, 'w')
begin = time.time()
slapgrid_environment = os.environ.copy()
for k, v in config['slapgrid_environment'].iteritems():
slapgrid_environment[k] = v
a('Slapgrid environment: %r'% config['slapgrid_environment'])
slapgrid = subprocess.Popen([config['slapgrid_software_binary'], '-vc',
config['slapos_config']], close_fds=True, preexec_fn=os.setsid,
stdout=out, stderr=subprocess.STDOUT, env=slapgrid_environment)
process_group_pid_list.append(slapgrid.pid)
slapgrid.communicate()
out.close()
while True:
try:
erp5_report.reportFinished(out_file,revision,
slapgrid.returncode == 0, time.time() - begin,
'\n'.join(info_list))
except Exception:
print 'Retrying in 5s'
time.sleep(5)
else:
break
print 'Sleeping for 600s'
time.sleep(600)
[slapos]
software_root = %(software_root)s
instance_root = %(instance_root)s
master_url = %(master_url)s
computer_id = %(computer_id)s
[slapproxy]
host = %(proxy_host)s
port = %(proxy_port)s
database_uri = %(proxy_database)s
from xml_marshaller import xml_marshaller
import os
import signal
import slapos.slap
import socket
import subprocess
import sys
import time
process_group_pid_list = []
process_pid_file_list = []
process_command_list = []
def sigterm_handler(signal, frame):
for pgpid in process_group_pid_list:
try:
os.killpg(pgpid, signal.SIGTERM)
except:
pass
for pid_file in process_pid_file_list:
try:
os.kill(int(open(pid_file).read().strip()), signal.SIGTERM)
except:
pass
for p in process_command_list:
try:
subprocess.call(p)
except:
pass
sys.exit(1)
signal.signal(signal.SIGTERM, sigterm_handler)
def run(args):
config = args[0]
for k,v in config['environment'].iteritems():
os.environ[k] = v
proxy = None
slapgrid = None
supervisord_pid_file = os.path.join(config['instance_root'], 'var', 'run',
'supervisord.pid')
if os.path.exists(config['proxy_database']):
os.unlink(config['proxy_database'])
try:
proxy = subprocess.Popen([config['slapproxy_binary'],
config['slapos_config']], close_fds=True, preexec_fn=os.setsid)
process_group_pid_list.append(proxy.pid)
slap = slapos.slap.slap()
slap.initializeConnection(config['master_url'])
while True:
try:
slap.registerSupply().supply(config['profile_url'],
computer_guid=config['computer_id'])
except socket.error:
time.sleep(1)
pass
else:
break
while True:
slapgrid = subprocess.Popen([config['slapgrid_software_binary'], '-vc',
config['slapos_config']], close_fds=True, preexec_fn=os.setsid)
process_group_pid_list.append(slapgrid.pid)
slapgrid.wait()
if slapgrid.returncode == 0:
print 'Software installed properly'