Commit 81e95395 authored by Julien Muchembled's avatar Julien Muchembled

rina: new SR to build packages and test 'rina-tools' component

This is implement as a test suite so that it can be automatically on any commit
to rina/slapos repositories.

Currently, the Debian packaging of the kernel only supports Debian Jessie.
parent 1e75ac96
......@@ -44,14 +44,30 @@ environment =
PKG_CONFIG_PATH=${glib:location}/lib/pkgconfig:${gnutls:location}/lib/pkgconfig:${pcre:location}/lib/pkgconfig:${pixman:location}/lib/pkgconfig
LDFLAGS=-L${pixman:location}/lib -Wl,-rpath=${pixman:location}/lib
[debian-amd64-netinst.iso]
# Download the installer of Debian 8 (Jessie)
[debian-netinst-base]
recipe = hexagonit.recipe.download
ignore-existing = true
filename = ${:_buildout_section_name_}
url = http://cdimage.debian.org/debian-cd/${:version}/amd64/iso-cd/debian-${:version}-amd64-netinst.iso
version = 8.6.0
md5sum = e9f61bf327db6d8f7cee05a99f2353cc
url = http://cdimage.debian.org/cdimage/${:release}/${:arch}/iso-cd/debian-${:version}-${:arch}-netinst.iso
release = archive/${:version}
download-only = true
mode = 0644
location = ${buildout:parts-directory}/${:_buildout_section_name_}
[debian-amd64-netinst-base]
<= debian-netinst-base
arch = amd64
[debian-amd64-netinst.iso]
# Download the installer of Debian 8 (Jessie)
<= debian-amd64-netinst-base
release = release/current
version = 8.6.0
md5sum = e9f61bf327db6d8f7cee05a99f2353cc
[debian-amd64-testing-netinst.iso]
# Download the installer of Debian Stretch
<= debian-amd64-netinst-base
release = stretch_di_alpha7
version = stretch-DI-alpha7
md5sum = 3fe53635b904553b26588491e1473e99
......@@ -10,7 +10,7 @@ parts = rina-tools
[rina-tools-repository]
recipe = slapos.recipe.build:gitclone
repository = https://github.com/jmuchemb/irati-stack.git
revision = 47e2b6a5f32d1090ec75206ccdb797b78a1a3330
revision = af3cd8350ec43ab2d309c785daf0dd16d3835005
git-executable = ${git:location}/bin/git
[rina-tools]
......
[buildout]
extends =
../file/buildout.cfg
../openssh/buildout.cfg
../p7zip/buildout.cfg
../qemu-kvm/buildout.cfg
parts = vm-debian
[vm-run-environment]
PATH = ${openssh:location}/bin:${qemu:location}/bin:%(PATH)s
[vm-install-environment]
PATH = ${file:location}/bin:${p7zip:location}/bin:${vm-run-environment:PATH}
[vm-run-base]
recipe = slapos.recipe.build:vm.run
environment = vm-run-environment
vm = ${vm-debian:location}
dist = ${vm-debian:dists}
[vm-debian]
recipe = slapos.recipe.build:vm.install-debian
environment = vm-install-environment
dists = debian-jessie
size = 1Gi
late-command =
# rdnssd causes too much trouble with recent QEMU, because the latter acts as
# a DNS proxy on both IPv4 and IPv6 without translating queries to what the
# host supports.
dpkg -P rdnssd
debconf.debconf =
debconf/frontend noninteractive
debconf/priority critical
# minimal size
preseed.recommends = false
preseed.tasks =
packages = localepurge ssh
[debian-jessie]
x86_64.iso = debian-amd64-netinst.iso
x86_64.kernel = install.amd/vmlinuz
x86_64.initrd = install.amd/initrd.gz
[debian-stretch]
<= debian-jessie
x86_64.iso = debian-amd64-testing-netinst.iso
[buildout]
extends =
../../component/vm-img/debian.cfg
parts =
debuild-rina-tools
test-rina
[vm-run-base]
smp = 4
# estimate the of RAM needed depending on the number CPUs
mem = 2048
apt-sources =
apt-update =
for x in ${:apt-sources}
do echo deb [trusted=yes] file:`map $x` ./
done >>/etc/apt/sources.list
apt-get update
[debuild-rina-base]
<= vm-run-base
mount.rina = ${rina-tools-repository:location}
cd-source =
git clone -snc core.sparseCheckout=true /mnt/rina /rina
cd /rina
echo /${:component} > .git/info/sparse-checkout
git reset --hard
cd ${:component}
# following lines are for debchange
export NAME=`git show -s --format=%cn`
export EMAIL=`git show -s --format=%ce`
dist=${:suite}
[ "$dist" ] && dist=`lsb_release -sc`/$dist || dist=UNRELEASED
prepare =
build-deps =
mk-build-deps -irt 'apt-get -y'
build =
dpkg-buildpackage -uc -b -jauto
finalize =
df
cd ..
mv *.changes *.deb $PARTDIR
cd $PARTDIR
dpkg-scanpackages -m . /dev/null |gzip --best >Packages.gz
r=`apt-ftparchive release .`
echo "$r" > Release
command =
${:apt-update}
${:cd-source}
( ${:prepare}
${:build-deps}
${:build}
) 2>&1 |tee $PARTDIR/build.log
${:finalize}
[debuild-rina-kernel]
<= debuild-rina-base
component = linux
prepare =
debian/dch-snapshot --force-distribution -D $dist
apt-get -y install kernel-wedge quilt python-six
QUILT_PATCHES=$PWD/debian/patches QUILT_PC=.pc quilt push --quiltrc - -a -q --fuzz=0
debian/rules debian/rules.gen || :
build =
arch=`dpkg-architecture -qDEB_HOST_ARCH`
make -j${:smp} -f debian/rules.gen binary-arch_$${arch}_none binary-libc-dev_$${arch}
dpkg-genchanges -b -UBinary -UDescription \
>../linux_`dpkg-parsechangelog -S Version`_$${arch}.changes
[debuild-librina]
<= debuild-rina-base
component = librina
apt-sources = ${debuild-rina-kernel:location}
prepare =
debian/rules DIST=$dist
[debuild-rinad]
<= debuild-librina
component = rinad
apt-sources = ${debuild-librina:location}
[rina-tools-deps]
apt-sources = ${debuild-librina:location} ${debuild-rinad:location}
[debuild-rina-tools]
<= debuild-librina
rina-tools-deps
component = rina-tools
# check that the built packages install
finalize =
dpkg -i ../*.deb
${debuild-librina:finalize}
[test-rina]
<= vm-run-base
rina-tools-deps
mount.rina = ${rina-tools-repository:location}
mount.slapos = ${:_profile_base_location_}/../..
mount.slapos.package = ${slapos.package-repository:location}
commands = install test
install =
${:apt-update}
apt-get -y install librinad-dev rinad
: || {
mkdir rina-tools
cd rina-tools
cat <<EOF >buildout.cfg
[buildout]
extends = /mnt/slapos/software/hellorina/software.cfg
develop = /mnt/slapos
[slapos-cookbook-develop]
recipe =
setup =
[rina-tools-repository]
repository = /mnt/rina
[versions]
slapos.cookbook =
EOF
sed s,${buildout:directory}/,/mnt/buildout/, /mnt/buildout/bin/buildout |
python - bootstrap
MAKEFLAGS=-j${:smp} bin/buildout
}
/mnt/slapos.package/playbook/roles/rina/gen-ipcm-conf
systemctl enable ipcm-re6st
arch=`dpkg-architecture -qDEB_HOST_ARCH`
dpkg -i `map ${debuild-rina-kernel:location}`/linux-image-*-$${arch}_*.deb
reboot
test = ( set -x
# TODO: more tests
grep re6st/1// /sys/rina/ipcps/1/name
) 2>&1 |tee $PARTDIR/result
{
"$schema": "http://json-schema.org/draft-04/schema#",
"description": "Parameters to build/publish RINA packages and test hellorina SR.",
"type": "object",
"additionalProperties": false,
"properties": {
"publish": {
"description": "Upload built packages automatically to a Debian repository when successful.",
"type": "object",
"required": ["suite", "host", "key"],
"properties": {
"suite": {
"type": "string"
},
"host": {
"type": "string"
},
"port": {
"type": "integer",
"default": 22
},
"key": {
"type": "string"
}
}
}
}
}
#!{{parameter_dict['runTestSuite_py']}}
from __future__ import print_function
import argparse, errno, os, shutil, subprocess, sys, traceback
from time import gmtime, strftime, time
# These are the 2 modules to reuse when using ERP5 for managing test bots.
# What we do here is currently too new to reuse more from testsuite.
from erp5.util import taskdistribution
from erp5.util.testsuite import format_command
{% set vm = parameter_dict['vm'] -%}
dist_list = {{vm['dists'].split()}}
publish = {{slapparameter_dict.get('publish')}}
BUILDDIR = 'build'
BUILDOUT = os.path.realpath(os.path.join('software_release', 'bin', 'buildout'))
STAT_MAP = dict(
TOTAL = 'test_count',
PASS = None,
SKIP = 'skip_count',
# Same as for NEO. ERP5 must be changed to only distinguish
# - expected (-> 'failure' for now) and
# - unexpected (-> 'error') failures.
XFAIL = 'failure_count',
FAIL = 'error_count',
XPASS = 'error_count',
ERROR = 'error_count',
)
class DummyTestResult:
class DummyTestResultLine:
def stop(self, duration, stdout='', **kw):
print('\n' + stdout)
print('Ran in %.3fs' % duration)
done = 0
def __init__(self, test_name_list):
self.test_name_list = test_name_list
def start(self):
test_result_line = self.DummyTestResultLine()
try:
test_result_line.name = self.test_name_list[self.done]
except IndexError:
return
self.done += 1
return test_result_line
def main():
os.environ.update({k: v.strip() % os.environ
for k, v in {{parameter_dict['environment'].items()}}})
parser = argparse.ArgumentParser(description='Run a test suite.')
parser.add_argument('--test_suite', help='The test suite name')
parser.add_argument('--test_suite_title', help='The test suite title')
parser.add_argument('--test_node_title', help='The test node title')
parser.add_argument('--project_title', help='The project title')
parser.add_argument('--revision', help='The revision to test',
default='dummy_revision')
parser.add_argument('--node_quantity', type=int,
help='Number of CPUs to use for the VM')
parser.add_argument('--master_url',
help='The Url of Master controling many suites')
args = parser.parse_args()
test_title = args.test_suite_title or args.test_suite
if args.master_url:
tool = taskdistribution.TaskDistributionTool(args.master_url)
test_result = tool.createTestResult(args.revision,
dist_list,
args.test_node_title,
test_title=test_title,
project_title=args.project_title)
if test_result is None:
return
else:
test_result = DummyTestResult(dist_list)
if os.path.exists(BUILDDIR):
shutil.rmtree(BUILDDIR)
os.mkdir(BUILDDIR)
oldpwd = os.getcwd()
os.chdir(BUILDDIR)
with open('buildout.cfg', 'w') as f:
f.write("""\
[buildout]
extends = {{parameter_dict['profile_base_location']}}/build.cfg
offline = true
develop-eggs-directory = {{buildout['develop-eggs-directory']}}
eggs-directory = {{buildout['eggs-directory']}}
[vm-run-base]
environment =
vm = {{vm['location']}}
[rina-tools-repository]
location = {{parameter_dict['rina_tools']}}
[slapos.package-repository]
location = {{parameter_dict['slapos_package']}}
""")
librina_log = os.path.join('parts', 'debuild-librina', 'build.log')
stderr_write = sys.stderr.write
while 1:
test_result_line = test_result.start()
if not test_result_line:
break
dist = test_result_line.name
cmd = [BUILDOUT,
'vm-run-base:dist=' + dist,
'debuild-rina-base:suite=' + (publish['suite'] if publish else ''),
]
if args.node_quantity:
cmd.append('vm-run-base:smp=%s' % args.node_quantity)
status_dict = {'command': format_command(*cmd)}
print('$', status_dict['command'])
# Wanted on test result lines:
# status: UNKNOWN in case of buildout failure
# (even if the test suite could be run)
# output: test suite summary if any
# error: buildout traceback or test suite log
start = time()
try:
p = subprocess.Popen(cmd, stderr=subprocess.PIPE)
stderr = []
while 1:
line = p.stderr.readline()
if not line:
break
stderr_write(line)
stderr.append(line)
returncode = p.wait()
end = time()
del p
if returncode:
iter_err = enumerate(reversed(stderr), 1)
for i, line in iter_err:
if line == "Traceback (most recent call last):\n":
for i, line in iter_err:
if line == '\n':
break
for i, line in iter_err:
if line[0] != ' ':
break
break
if line == "While:\n":
del stderr[:-i]
status_dict['stderr'] = ''.join(stderr)
with open(librina_log) as f:
log = f.readlines()
del log[:log.index('make check-TESTS\n')]
for i, line in enumerate(log):
if line.startswith('Testsuite summary'):
del log[log.index(log[i+1], i+2):]
status_dict['stdout'] = ''.join(log[i:])
stat = {}
for line in log[i+2:]:
k, v = line[2:].split(':')
k = STAT_MAP[k]
if k:
stat[k] = stat.get(k, 0) + int(v.strip())
if not returncode:
status_dict.update(stat)
status_dict.setdefault('stderr', ''.join(log[:i-1]))
break
except Exception:
status_dict.setdefault('stderr', traceback.format_exc())
test_result_line.stop(
date = strftime("%Y/%m/%d %H:%M:%S", gmtime(end)),
duration = end - start,
**status_dict)
# TODO: upload packages if 'publish' parameter is given
if args.master_url:
os.chdir(oldpwd)
shutil.rmtree(BUILDDIR)
if __name__ == "__main__":
main()
[buildout]
extends =
../../stack/slapos.cfg
../../component/vm-img/debian.cfg
../../component/rina-tools/buildout.cfg
parts =
slapos-cookbook
template
[template]
recipe = slapos.recipe.template:jinja2
# XXX: "template.cfg" is hardcoded in instanciation recipe
rendered = ${buildout:directory}/template.cfg
template = inline:
[buildout]
eggs-directory = ${buildout:eggs-directory}
develop-eggs-directory = ${buildout:develop-eggs-directory}
parts = runTestSuite
[slap-configuration]
recipe = slapos.cookbook:slapconfiguration.serialised
computer = $${slap-connection:computer-id}
partition = $${slap-connection:partition-id}
url = $${slap-connection:server-url}
key = $${slap-connection:key-file}
cert = $${slap-connection:cert-file}
[directory]
recipe = slapos.cookbook:mkdirectory
bin = $${buildout:directory}/bin
[runTestSuite]
recipe = slapos.recipe.template:jinja2
rendered = $${directory:bin}/$${:_buildout_section_name_}
template = ${:_profile_base_location_}/$${:_buildout_section_name_}.in
mode = 0755
context =
{##} section buildout buildout
{##} section parameter_dict runTestSuite-parameters
{##} key slapparameter_dict slap-configuration:configuration
[runTestSuite-parameters]
environment = {{dumps(environment)}}
vm = {{dumps(vm)}}
runTestSuite_py = ${buildout:bin-directory}/${runTestSuite_py:interpreter}
profile_base_location = ${:_profile_base_location_}
rina_tools = ${rina-tools-repository:location}
slapos_package = ${slapos.package-repository:location}
context =
section environment vm-run-environment
section vm vm-debian
[runTestSuite_py]
recipe = zc.recipe.egg
eggs = erp5.util
interpreter = ${:_buildout_section_name_}
[rina-tools-repository]
sparse-checkout = /.gitignore
[slapos.package-repository]
recipe = slapos.recipe.build:gitclone
repository = https://lab.nexedi.com/nexedi/slapos.package.git
git-executable = ${git:location}/bin/git
sparse-checkout = /playbook/roles/rina
[vm-debian]
# building a generic Debian kernel uses a lot of space
size = 16Gi
packages +=
# generic (another SR that build packages automatically would use the same list)
apt-utils build-essential devscripts equivs lsb-release
# specific
git ca-certificates python
# biggest and common build-deps for RINA
dh-autoreconf pkg-config doxygen maven xmlto
......@@ -58,7 +58,7 @@ eggs = collective.recipe.template
[versions]
collective.recipe.template = 1.11
plone.recipe.command = 1.1
slapos.recipe.build = 0.24
slapos.recipe.build = 0.25
slapos.recipe.template = 2.7
# Replicate slapos stack, but without shacache to not have to compile the entire world for a simple test.
......
......@@ -137,7 +137,7 @@ slapos.cookbook = 1.0.35
slapos.core = 1.3.16
slapos.extension.strip = 0.1
slapos.libnetworkcache = 0.14.5
slapos.recipe.build = 0.24
slapos.recipe.build = 0.25
slapos.recipe.cmmi = 0.2
stevedore = 1.17.1
unicodecsv = 0.14.1
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment