Commit 43e365ee authored by Thomas Gambier's avatar Thomas Gambier 🚴🏼

Update Release Candidate

parents e14ca562 4d27608d
# SlapOS component for ZEO.
# https://zeo.readthedocs.io/
[buildout]
extends =
../git/buildout.cfg
parts = ZEO/scripts
# ZEO allows to use either ZEO4 or ZEO5
# To select which version to use users should do:
#
# [ZEO]
# egg = ${ZEO<version>:egg}
#
# By default ZEO4 is used.
[ZEO]
recipe = zc.recipe.egg:eggs
egg = ${ZEO4:egg}
eggs = ${:egg}
# ZEO/scripts installs scripts from ZEO
[ZEO/scripts]
recipe = zc.recipe.egg:scripts
eggs = ${ZEO:egg}
# ZEO4: we maintain our own 4-nxd branch with patches
[ZEO4]
recipe = zc.recipe.egg:develop
setup = ${ZEO4-repository:location}
egg = ZEO
[ZEO4-repository]
recipe = slapos.recipe.build:gitclone
repository = https://lab.nexedi.com/nexedi/ZEO.git
branch = 4-nxd
revision= 5114f909e5a5
location = ${buildout:parts-directory}/ZEO4
git-executable = ${git:location}/bin/git
# ZEO5 is plain upstream egg
[ZEO5]
recipe = zc.recipe.egg:eggs
egg = ZEO
eggs = ${:egg}
......@@ -11,7 +11,7 @@ parts =
command = bash -c ". ${gowork:env.sh} && cd ${go_github.com_caddyserver_caddy:location} && GO111MODULE=on go install -v $(echo -n '${gowork:install}' |tr '\n' ' ')"
[gowork]
golang = ${golang1.12:location}
golang = ${golang1.14:location}
install =
github.com/caddyserver/caddy/...
......
From 8b31ccec54584a287cc61501948283d7d6ee7073 Mon Sep 17 00:00:00 2001
From: Julien Muchembled <jm@nexedi.com>
Date: Mon, 26 Mar 2018 20:39:07 +0200
Subject: [PATCH] Enable TCP_NODELAY for inet(6) sockets
See commit 3d886d426243655b9f5a2528636e42b5c7662c19.
---
src/ZEO/zrpc/client.py | 2 ++
src/ZEO/zrpc/server.py | 1 +
2 files changed, 3 insertions(+)
diff --git a/src/ZEO/zrpc/client.py b/src/ZEO/zrpc/client.py
index 32a7a877..669f5962 100644
--- a/src/ZEO/zrpc/client.py
+++ b/src/ZEO/zrpc/client.py
@@ -568,6 +568,8 @@ def __init__(self, domain, addr, mgr, client):
self.close()
return
self.sock.setblocking(0)
+ if domain in (socket.AF_INET, socket.AF_INET6):
+ self.sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
self.state = "opened"
def connect_procedure(self):
diff --git a/src/ZEO/zrpc/server.py b/src/ZEO/zrpc/server.py
index b83cc004..af91e3e4 100644
--- a/src/ZEO/zrpc/server.py
+++ b/src/ZEO/zrpc/server.py
@@ -66,6 +66,7 @@ def _open_socket(self):
socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, True)
else:
self.create_socket(socket.AF_INET, socket.SOCK_STREAM)
+ self.socket.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
else:
self.create_socket(socket.AF_UNIX, socket.SOCK_STREAM)
self.set_reuse_addr()
--
2.14.1
[buildout]
extends =
../cmake/buildout.cfg
parts =
fmtlib
[fmtlib]
recipe = slapos.recipe.cmmi
shared = true
url = https://github.com/fmtlib/fmt/archive/7.0.3.tar.gz
md5sum = 57392b7ea09592a2b237a02950f35bb0
configure-command = ${cmake:location}/bin/cmake
configure-options =
-DCMAKE_INSTALL_PREFIX=@@LOCATION@@
-DBUILD_SHARED_LIBS=ON
-DFMT_TEST=OFF
# SlapOS component for GDB.
# https://www.gnu.org/software/gdb
[buildout]
extends =
../xz-utils/buildout.cfg
../binutils/buildout.cfg
../libexpat/buildout.cfg
../texinfo/buildout.cfg
parts = gdb
[gdb]
recipe = slapos.recipe.cmmi
shared = true
url = http://ftp.gnu.org/gnu/gdb/gdb-9.2.tar.xz
md5sum = db95524e554870209ab7d9f8fd8dc557
location = @@LOCATION@@
# gdb refuses to build in-tree -> build it inside build/
pre-configure =
mkdir -p build
configure-command =
cd build && ../configure
configure-options =
--prefix=${:location}
--disable-bootstrap
--with-mpc=${mpc:location}
--with-mpfr=${mpfr:location}
--with-gmp=${gmp:location}
--with-isl=${isl:location}
--with-expat=${libexpat:location}
make-binary =
make-targets = cd build && make && make install
environment =
PATH=${texinfo:location}/bin:${xz-utils:location}/bin:%(PATH)s
LDFLAGS=-Wl,-rpath=${libexpat:location}/lib
......@@ -6,7 +6,7 @@ extends =
../git/buildout.cfg
../pkgconfig/buildout.cfg
parts = gowork
parts = gowork go
# ---- Go builds itself ----
......@@ -84,6 +84,18 @@ environment-extra =
#
# [gowork]
# buildflags = -race
#
# ${go:exe} is standalone executable that runs go in activated gowork environment.
[go]
recipe = slapos.recipe.template:jinja2
exe = ${buildout:bin-directory}/go
rendered= ${:exe}
mode = 755
template= inline:
#!/bin/sh -e
. ${gowork:env.sh}
exec go "$@"
[gowork]
directory = ${buildout:directory}/go.work
src = ${:directory}/src
......
......@@ -4,3 +4,7 @@ parts = numpy
[numpy]
recipe = zc.recipe.egg:custom
egg = numpy
[versions]
numpy = 1.16.4
# SlapOS component for pygolang development.
[buildout]
extends =
buildout.cfg
../git/buildout.cfg
# override pygolang to install it from latest git version
[pygolang]
recipe = zc.recipe.egg:develop
setup = ${pygolang-repository:location}
[pygolang-repository]
recipe = slapos.recipe.build:gitclone
repository = https://lab.nexedi.com/nexedi/pygolang.git
branch = master
location = ${buildout:parts-directory}/pygolang-dev
git-executable = ${git:location}/bin/git
# unpin pygolang from versions, so that buildout does not fallback to
# installing non-dev egg if dev one has changed its version.
[versions]
pygolang =
# SlapOS component for pygolang.
# pygolang.nexedi.com
[buildout]
extends =
../../component/cython/buildout.cfg
parts =
pygolang
gpython
# pygolang installed from released egg from pypi
[pygolang]
recipe = zc.recipe.egg:custom
egg = pygolang
setup-eggs =
${cython:egg}
setuptools-dso
gevent
# gpython program
[gpython]
recipe = zc.recipe.egg:scripts
eggs = ${pygolang:egg}
scripts = gpython
# convenience for gpython users
exe = ${buildout:bin-directory}/gpython
# python-interpreter provides python interpreter with all specified eggs.
# eggs default to pygolang, but can be overwritten or changed in inherited section.
# if eggs are changes, they must still have pygolang.
[python-interpreter]
recipe = zc.recipe.egg:scripts
eggs = ${pygolang:egg}
interpreter = python
# interpreter code that buildout generates cannot process `-m pytest --<pytest-option>`
# -> use pymain from gpython to workaround that.
initialization =
# set sys.executable to self, so that subprocess and friends go through us
# and this way spawn children with correct sys.path where all eggs that
# parent have are present. TODO consider migrating this into pymain
sys.executable = sys.argv[0]
# tail to pymain
from gpython import pymain
pymain(sys.argv[1:])
sys.exit(0)
# don't install scripts from listed eggs (avoid conflict with other sections
# that use zc.recipe.egg with eggs overlapping with ${:eggs} - ex neoppod)
# (we cannot use zc.recipe.egg:eggs because interpreter does not work there)
# NOTE with scripts=ø interpreter is not handled, so we use `scripts=python` as
# a workaround.
scripts = ${:interpreter}
[versions]
pygolang = 0.0.7.post1
# SlapOS software release to test pygolang on Nexedi testing infrastructure.
[buildout]
extends =
../../stack/nxdtest.cfg
../pytest/buildout.cfg
../numpy/buildout.cfg
buildout-dev.cfg
parts =
pygolang
gpython
pygolang-python
# for instance
slapos-cookbook
instance.cfg
# tune pygolang to install with all and for-tests extras.
[pygolang]
egg = pygolang[all_test]
# bin/python is preinstalled with sys.path to pygolang & friends.
[pygolang-python]
<= python-interpreter
eggs = ${pygolang:egg}
# env.sh for pygolang's python/gpython to be on $PATH.
[pygolang-env.sh]
recipe = slapos.recipe.template:jinja2
rendered = ${buildout:directory}/${:_buildout_section_name_}
template = inline:
export PS1="(pygolang-env) $PS1"
export PATH=${buildout:bin-directory}:$PATH
# instance to run nxdtest.
[instance.cfg]
<= jinja2-template
template = inline:
[buildout]
extends = ${nxdtest-instance.cfg:rendered}
[runTestSuite]
env.sh = ${pygolang-env.sh:rendered}
workdir = ${pygolang-repository:location}
[versions]
ipython = 5.10.0
ipython-genutils = 0.2.0
traitlets = 4.3.3
simplegeneric = 0.8.1
Pygments = 2.5.2
prompt-toolkit = 1.0.18
pickleshare = 0.7.5
pexpect = 4.8.0
backports.shutil-get-terminal-size = 1.0.0
ptyprocess = 0.6.0
# SlapOS component for pytest.
# https://pytest.org
[buildout]
parts = pytest/scripts
[pytest]
recipe = zc.recipe.egg:eggs
eggs = pytest
[pytest/scripts]
recipe = zc.recipe.egg:scripts
eggs = ${pytest:eggs}
[versions]
pytest = 4.6.11
# SlapOS component for wendelin.core
[buildout]
extends = ../git/buildout.cfg
extends =
../pygolang/buildout.cfg
../git/buildout.cfg
parts =
wendelin.core
......@@ -7,18 +12,23 @@ parts =
# wendelin.core installed from released egg from pypi
[wendelin.core]
recipe = zc.recipe.egg:custom
egg = wendelin.core
<= wendelin.core-eggcommon
# wendelin.core installed from latest git version
[wendelin.core-dev]
recipe = zc.recipe.egg:develop
egg = wendelin.core
setup = ${wendelin.core-repository:location}
environment = wendelin.core-dev-env
<= wendelin.core-eggcommon
# common parts in between wendelin.core and wendelin.core-dev
[wendelin.core-eggcommon]
egg = wendelin.core
setup-eggs =
${pygolang:egg}[pyx.build]
environment = wendelin.core-env
[wendelin.core-dev-env]
# wendelin.core-dev needs git to build
[wendelin.core-env]
# wendelin.core needs git(dev) to build
PATH = ${git:location}/bin:%(PATH)s
......
# SlapOS software release to test wendelin.core on Nexedi testing infrastructure.
# Common parts.
[buildout]
extends =
# test*.cfg first extend from neoppod/software<ZODB-flavour>.cfg to use
# appropriate ZODB and versions of other components.
../pytest/buildout.cfg
../gdb/buildout.cfg
../../stack/nxdtest.cfg
buildout.cfg
parts =
# keep neoppod first and in parts so that ZODB is built correctly
neoppod
wendelin.core-dev
# for instance
wendelin.core-python
slapos-cookbook
instance.cfg
# bin/python s python interpreter with wendelin.core and all other eggs.
[wendelin.core-python]
<= python-interpreter
eggs =
wendelin.core[test]
pygolang[pyx.build]
neoppod[tests]
ZEO[test]
# env.sh for that python to be on $PATH
[wendelin.core-env.sh]
recipe = slapos.recipe.template:jinja2
rendered = ${buildout:directory}/${:_buildout_section_name_}
template = inline:
export PS1="(wendelin.core-env) $PS1"
export PATH=${buildout:bin-directory}:${gdb:location}/bin:$PATH
# instance to run nxdtest.
[instance.cfg]
<= jinja2-template
template = inline:
[buildout]
extends = ${nxdtest-instance.cfg:rendered}
[runTestSuite]
env.sh = ${wendelin.core-env.sh:rendered}
workdir = ${wendelin.core-repository:location}
[versions]
neoppod = 1.12.0
zope.testrunner = 5.2
random2 = 1.0.1
manuel = 1.10.1
# SlapOS software release to test wendelin.core/ZODB5 on Nexedi testing infrastructure.
[buildout]
extends =
../../stack/erp5/buildout.cfg
../../software/neoppod/software-zodb5.cfg
test-common.cfg
# SlapOS software release to test wendelin.core on Nexedi testing infrastructure.
[buildout]
extends =
../../stack/erp5/buildout.cfg
../../software/neoppod/software.cfg
test-common.cfg
# SlapOS component for zodbtools development.
[buildout]
extends =
buildout.cfg
../git/buildout.cfg
# override zodbtools to install it from latest git version
[zodbtools]
recipe = zc.recipe.egg:develop
setup = ${zodbtools-repository:location}
[zodbtools-repository]
recipe = slapos.recipe.build:gitclone
repository = https://lab.nexedi.com/nexedi/zodbtools.git
branch = master
location = ${buildout:parts-directory}/zodbtools-dev
git-executable = ${git:location}/bin/git
# unpin zodbtools from versions, so that buildout does not fallback to
# installing non-dev egg if dev one has changed its version.
[versions]
zodbtools =
# SlapOS component for zodbtools.
# https://lab.nexedi.com/nexedi/zodbtools
[buildout]
extends =
../pygolang/buildout.cfg
../ZEO/buildout.cfg
parts =
zodbtools/scripts
[zodbtools]
recipe = zc.recipe.egg:eggs
egg = zodbtools
eggs =
${:egg}
# dependent eggs that must come through in-tree recipes
depends =
${pygolang:egg}
# ZEO comes through zodbtools -> zodburi -> ZEO
${ZEO:egg}
[zodbtools/scripts]
recipe = zc.recipe.egg:scripts
eggs = ${zodbtools:eggs}
[versions]
zodbtools = 0.0.0.dev8
# SlapOS software release to test zodbtools on Nexedi testing infrastructure.
# Common parts.
[buildout]
extends =
# test*.cfg first extend from neoppod/software<ZODB-flabour>.cfg to use
# appropriate ZODB and versions of other components.
../../stack/nxdtest.cfg
../pytest/buildout.cfg
buildout-dev.cfg
parts =
zodbtools
# for instance
zodbtools-python
slapos-cookbook
instance.cfg
# bin/python is preinstalled with sys.path to zodbtools & friends.
[zodbtools-python]
<= python-interpreter
eggs = zodbtools[test]
# env.sh for zodbtools's python to be on $PATH.
[zodbtools-env.sh]
recipe = slapos.recipe.template:jinja2
rendered = ${buildout:directory}/${:_buildout_section_name_}
template = inline:
export PS1="(zodbtools-env) $PS1"
export PATH=${buildout:bin-directory}:$PATH
# instance to run nxdtest.
[instance.cfg]
<= jinja2-template
template = inline:
[buildout]
extends = ${nxdtest-instance.cfg:rendered}
[runTestSuite]
env.sh = ${zodbtools-env.sh:rendered}
workdir = ${zodbtools-repository:location}
[versions]
freezegun = 1.0.0
# SlapOS software release to test zodbtools/ZODB5 on Nexedi testing infrastructure.
[buildout]
extends =
../../software/neoppod/software-zodb5.cfg
test-common.cfg
# SlapOS software release to test zodbtools on Nexedi testing infrastructure.
[buildout]
extends =
../../software/neoppod/software.cfg
test-common.cfg
......@@ -11,6 +11,7 @@ extends =
../../component/6tunnel/buildout.cfg
../../component/xz-utils/buildout.cfg
../../component/rsyslogd/buildout.cfg
../../component/numpy/buildout.cfg
../../component/haproxy/buildout.cfg
../../component/nginx/buildout.cfg
......@@ -251,7 +252,6 @@ pycrypto = 2.6.1
rdiff-backup = 1.0.5+SlapOSPatched001
slapos.recipe.template = 4.4
smmap = 0.9.0
numpy = 1.16.4
websockify = 0.8.0
furl = 2.1.0
......
......@@ -102,5 +102,4 @@ context =
[versions]
slapos.recipe.template = 4.4
coverage = 4.5.1
numpy = 1.16.4
pycodestyle = 2.5.0
......@@ -40,7 +40,6 @@ rpy2 = 2.4.0
pydot = 1.0.28
xlrd = 0.9.3
xlwt = 0.7.5
numpy = 1.16.4
scipy = 0.13.3
simpy = 3.0.5
zope.dottedname = 4.1.0
......
......@@ -33,7 +33,7 @@ with open("README.md") as f:
setup(name=name,
version=version,
description="Test for SlapOS' ERP5 software releae",
description="Test for SlapOS' ERP5 software release",
long_description=long_description,
long_description_content_type='text/markdown',
maintainer="Nexedi",
......@@ -50,8 +50,9 @@ setup(name=name,
'mysqlclient',
'backports.lzma',
'cryptography',
'pexpect',
'pyOpenSSL',
],
zip_safe=True,
'typing; python_version<"3"',
],
test_suite='test',
)
from . import ERP5InstanceTestCase
from . import setUpModule
from slapos.testing.utils import findFreeTCPPort
from BaseHTTPServer import HTTPServer
from BaseHTTPServer import BaseHTTPRequestHandler
import OpenSSL.SSL
from cryptography.hazmat.backends import default_backend
from cryptography.hazmat.primitives import serialization, hashes
from cryptography.hazmat.primitives.asymmetric import rsa
from cryptography import x509
from cryptography.x509.oid import NameOID
import glob
import hashlib
import json
import multiprocessing
import logging
import os
import requests
import re
import shutil
import socket
import subprocess
import tempfile
import time
import urlparse
from BaseHTTPServer import BaseHTTPRequestHandler
from typing import Any, Dict, Optional
import idna
import mock
import OpenSSL.SSL
import pexpect
import psutil
import requests
from cryptography import x509
from cryptography.hazmat.backends import default_backend
from cryptography.hazmat.primitives import hashes, serialization
from cryptography.hazmat.primitives.asymmetric import rsa
from cryptography.x509.oid import NameOID
from slapos.testing.testcase import ManagedResource
from slapos.testing.utils import (CrontabMixin, ManagedHTTPServer,
findFreeTCPPort)
from . import ERP5InstanceTestCase, setUpModule
setUpModule # pyflakes
class TestHandler(BaseHTTPRequestHandler):
def do_GET(self):
self.send_response(200)
self.send_header("Content-Type", "application/json")
response = {
'Path': self.path,
'Incoming Headers': self.headers.dict
}
response = json.dumps(response, indent=2)
self.end_headers()
self.wfile.write(response)
class TestFrontendXForwardedFor(ERP5InstanceTestCase):
__partition_reference__ = 'xff'
http_server_process = None
frontend_caucase_dir = None
frontend_caucased_process = None
backend_caucase_dir = None
backend_caucased_process = None
class EchoHTTPServer(ManagedHTTPServer):
"""An HTTP Server responding with the request path and incoming headers,
encoded in json.
"""
class RequestHandler(BaseHTTPRequestHandler):
def do_GET(self):
# type: () -> None
self.send_response(200)
self.send_header("Content-Type", "application/json")
response = json.dumps(
{
'Path': self.path,
'Incoming Headers': self.headers.dict
},
indent=2,
)
self.end_headers()
self.wfile.write(response)
@classmethod
def getInstanceSoftwareType(cls):
return 'balancer'
log_message = logging.getLogger(__name__ + '.HeaderEchoHandler').info
@classmethod
def setUpClass(cls):
# start a dummy web server echoing headers.
http_server_port = findFreeTCPPort(cls._ipv4_address)
server = HTTPServer(
(cls._ipv4_address, http_server_port),
TestHandler)
cls.http_server_process = multiprocessing.Process(
target=server.serve_forever, name='HTTPServer')
cls.http_server_process.start()
cls.http_server_netloc = '%s:%s' % (cls._ipv4_address, http_server_port)
# start a caucased and generate a valid client certificate.
cls.computer_partition_root_path = os.path.abspath(os.curdir)
cls.frontend_caucase_dir = tempfile.mkdtemp()
frontend_caucased_dir = os.path.join(cls.frontend_caucase_dir, 'caucased')
os.mkdir(frontend_caucased_dir)
frontend_user_dir = os.path.join(cls.frontend_caucase_dir, 'user')
os.mkdir(frontend_user_dir)
frontend_service_dir = os.path.join(cls.frontend_caucase_dir, 'service')
os.mkdir(frontend_service_dir)
frontend_caucased_netloc = '%s:%s' % (cls._ipv4_address, findFreeTCPPort(cls._ipv4_address))
cls.frontend_caucased_url = 'http://' + frontend_caucased_netloc
cls.user_certificate = frontend_user_key = os.path.join(frontend_user_dir, 'client.key.pem')
frontend_user_csr = os.path.join(frontend_user_dir, 'client.csr.pem')
key = rsa.generate_private_key(
public_exponent=65537,
key_size=2048,
backend=default_backend()
)
with open(frontend_user_key, 'wb') as f:
f.write(key.private_bytes(
encoding=serialization.Encoding.PEM,
format=serialization.PrivateFormat.TraditionalOpenSSL,
encryption_algorithm=serialization.NoEncryption(),
))
csr = x509.CertificateSigningRequestBuilder().subject_name(x509.Name([
x509.NameAttribute(NameOID.COMMON_NAME, u'user'),
])).sign(key, hashes.SHA256(), default_backend())
with open(frontend_user_csr, 'wb') as f:
f.write(csr.public_bytes(serialization.Encoding.PEM))
class CaucaseService(ManagedResource):
"""A caucase service.
"""
url = None # type: str
directory = None # type: str
_caucased_process = None # type: subprocess.Popen
cls.software_release_root_path = os.path.join(
cls.slap._software_root,
hashlib.md5(cls.getSoftwareURL()).hexdigest(),
def open(self):
# type: () -> None
# start a caucased and server certificate.
software_release_root_path = os.path.join(
self._cls.slap._software_root,
hashlib.md5(self._cls.getSoftwareURL().encode()).hexdigest(),
)
caucased_path = os.path.join(cls.software_release_root_path, 'bin', 'caucased')
caucase_path = os.path.join(cls.software_release_root_path, 'bin', 'caucase')
cls.frontend_caucased_process = subprocess.Popen(
[
caucased_path,
'--db', os.path.join(frontend_caucased_dir, 'caucase.sqlite'),
'--server-key', os.path.join(frontend_caucased_dir, 'server.key.pem'),
'--netloc', frontend_caucased_netloc,
'--service-auto-approve-count', '1',
],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
caucased_path = os.path.join(software_release_root_path, 'bin', 'caucased')
self.directory = tempfile.mkdtemp()
caucased_dir = os.path.join(self.directory, 'caucased')
os.mkdir(caucased_dir)
os.mkdir(os.path.join(caucased_dir, 'user'))
os.mkdir(os.path.join(caucased_dir, 'service'))
backend_caucased_netloc = '%s:%s' % (self._cls._ipv4_address, findFreeTCPPort(self._cls._ipv4_address))
self.url = 'http://' + backend_caucased_netloc
self._caucased_process = subprocess.Popen(
[
caucased_path,
'--db', os.path.join(caucased_dir, 'caucase.sqlite'),
'--server-key', os.path.join(caucased_dir, 'server.key.pem'),
'--netloc', backend_caucased_netloc,
'--service-auto-approve-count', '1',
],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
)
for _ in range(10):
for _ in range(30):
try:
if requests.get(cls.frontend_caucased_url).status_code == 200:
if requests.get(self.url).status_code == 200:
break
except Exception:
pass
......@@ -118,173 +100,591 @@ class TestFrontendXForwardedFor(ERP5InstanceTestCase):
else:
raise RuntimeError('caucased failed to start.')
cau_args = [
caucase_path,
'--ca-url', cls.frontend_caucased_url,
'--ca-crt', os.path.join(frontend_user_dir, 'service-ca-crt.pem'),
'--crl', os.path.join(frontend_user_dir, 'service.crl'),
'--user-ca-crt', os.path.join(frontend_user_dir, 'user-ca-crt.pem'),
'--user-crl', os.path.join(frontend_user_dir, 'user.crl'),
]
def close(self):
# type: () -> None
self._caucased_process.terminate()
self._caucased_process.wait()
shutil.rmtree(self.directory)
cas_args = [
caucase_path,
'--ca-url', cls.frontend_caucased_url,
'--ca-crt', os.path.join(frontend_service_dir, 'service-ca-crt.pem'),
'--crl', os.path.join(frontend_service_dir, 'service.crl'),
'--user-ca-crt', os.path.join(frontend_service_dir, 'user-ca-crt.pem'),
'--user-crl', os.path.join(frontend_service_dir, 'user.crl'),
@property
def ca_crt_path(self):
# type: () -> str
"""Path of the CA certificate from this caucase.
"""
ca_crt_path = os.path.join(self.directory, 'ca.crt.pem')
if not os.path.exists(ca_crt_path):
with open(ca_crt_path, 'w') as f:
f.write(
requests.get(urlparse.urljoin(
self.url,
'/cas/crt/ca.crt.pem',
)).text)
return ca_crt_path
class BalancerTestCase(ERP5InstanceTestCase):
@classmethod
def getInstanceSoftwareType(cls):
return 'balancer'
@classmethod
def _getInstanceParameterDict(cls):
# type: () -> Dict
return {
'tcpv4-port': 8000,
'computer-memory-percent-threshold': 100,
# XXX what is this ? should probably not be needed here
'name': cls.__name__,
'monitor-passwd': 'secret',
'apachedex-configuration': '--erp5-base +erp5 .*/VirtualHostRoot/erp5(/|\\?|$) --base +other / --skip-user-agent Zabbix --error-detail --js-embed --quiet',
'apachedex-promise-threshold': 100,
'haproxy-server-check-path': '/',
'zope-family-dict': {
'default': ['dummy_http_server'],
},
'dummy_http_server': [[cls.getManagedResource("backend_web_server", EchoHTTPServer).netloc, 1, False]],
'backend-path-dict': {
'default': '',
},
'ssl-authentication-dict': {},
'ssl': {
'caucase-url': cls.getManagedResource("caucase", CaucaseService).url,
}
}
@classmethod
def getInstanceParameterDict(cls):
# type: () -> Dict
return {'_': json.dumps(cls._getInstanceParameterDict())}
def setUp(self):
self.default_balancer_url = json.loads(
self.computer_partition.getConnectionParameterDict()['_'])['default']
class SlowHTTPServer(ManagedHTTPServer):
"""An HTTP Server which reply after 3 seconds.
"""
class RequestHandler(BaseHTTPRequestHandler):
def do_GET(self):
# type: () -> None
self.send_response(200)
self.send_header("Content-Type", "text/plain")
time.sleep(3)
self.end_headers()
self.wfile.write("OK\n")
log_message = logging.getLogger(__name__ + '.SlowHandler').info
class TestAccessLog(BalancerTestCase, CrontabMixin):
"""Check access logs emitted by balancer
"""
__partition_reference__ = 'l'
@classmethod
def _getInstanceParameterDict(cls):
# type: () -> Dict
parameter_dict = super(TestAccessLog, cls)._getInstanceParameterDict()
# use a slow server instead
parameter_dict['dummy_http_server'] = [[cls.getManagedResource("slow_web_server", SlowHTTPServer).netloc, 1, False]]
return parameter_dict
def test_access_log_format(self):
# type: () -> None
requests.get(
urlparse.urljoin(self.default_balancer_url, '/url_path'),
verify=False,
)
with open(os.path.join(self.computer_partition_root_path, 'var', 'log', 'apache-access.log')) as access_log_file:
access_line = access_log_file.read()
self.assertIn('/url_path', access_line)
# last \d is the request time in micro seconds, since this SlowHTTPServer
# sleeps for 3 seconds, it should take between 3 and 4 seconds to process
# the request - but our test machines can be slow sometimes, so we tolerate
# it can take up to 20 seconds.
match = re.match(
r'([(\d\.)]+) - - \[(.*?)\] "(.*?)" (\d+) (\d+) "(.*?)" "(.*?)" (\d+)',
access_line
)
self.assertTrue(match)
assert match
request_time = int(match.groups()[-1])
self.assertGreater(request_time, 3 * 1000 * 1000)
self.assertLess(request_time, 20 * 1000 * 1000)
def test_access_log_apachedex_report(self):
# type: () -> None
# make a request so that we have something in the logs
requests.get(self.default_balancer_url, verify=False)
# crontab for apachedex is executed
self._executeCrontabAtDate('generate-apachedex-report', '23:59')
# it creates a report for the day
apachedex_report, = glob.glob(
os.path.join(
self.computer_partition_root_path,
'srv',
'monitor',
'private',
'apachedex',
'ApacheDex-*.html',
))
with open(apachedex_report, 'r') as f:
report_text = f.read()
self.assertIn('APacheDEX', report_text)
# having this table means that apachedex could parse some lines.
self.assertIn('<h2>Hits per status code</h2>', report_text)
def test_access_log_rotation(self):
# type: () -> None
# run logrotate a first time so that it create state files
self._executeCrontabAtDate('logrotate', '2000-01-01')
# make a request so that we have something in the logs
requests.get(self.default_balancer_url, verify=False).raise_for_status()
# slow query crontab depends on crontab for log rotation
# to be executed first.
self._executeCrontabAtDate('logrotate', '2050-01-01')
# this logrotate leaves the log for the day as non compressed
rotated_log_file = os.path.join(
self.computer_partition_root_path,
'srv',
'backup',
'logrotate',
'apache-access.log-20500101',
)
self.assertTrue(os.path.exists(rotated_log_file))
requests.get(self.default_balancer_url, verify=False).raise_for_status()
# on next day execution of logrotate, log files are compressed
self._executeCrontabAtDate('logrotate', '2050-01-02')
self.assertTrue(os.path.exists(rotated_log_file + '.xz'))
self.assertFalse(os.path.exists(rotated_log_file))
class BalancerCookieHTTPServer(ManagedHTTPServer):
"""An HTTP Server which can set balancer cookie.
This server set cookie when requested /set-cookie path.
The reply body is the name used when registering this resource
using getManagedResource. This way we can assert which
backend replied.
"""
@property
def RequestHandler(self):
server = self
class RequestHandler(BaseHTTPRequestHandler):
def do_GET(self):
# type: () -> None
self.send_response(200)
self.send_header("Content-Type", "text/plain")
if self.path == '/set_cookie':
# the balancer tells the backend what's the name of the balancer cookie with
# the X-Balancer-Current-Cookie header.
self.send_header('Set-Cookie', '%s=anything' % self.headers['X-Balancer-Current-Cookie'])
# The name of this cookie is SERVERID
assert self.headers['X-Balancer-Current-Cookie'] == 'SERVERID'
self.end_headers()
self.wfile.write(server._name)
log_message = logging.getLogger(__name__ + '.BalancerCookieHTTPServer').info
return RequestHandler
class TestBalancer(BalancerTestCase):
"""Check balancing capabilities
"""
__partition_reference__ = 'b'
@classmethod
def _getInstanceParameterDict(cls):
# type: () -> Dict
parameter_dict = super(TestBalancer, cls)._getInstanceParameterDict()
# use two backend servers
parameter_dict['dummy_http_server'] = [
[cls.getManagedResource("backend_web_server1", BalancerCookieHTTPServer).netloc, 1, False],
[cls.getManagedResource("backend_web_server2", BalancerCookieHTTPServer).netloc, 1, False],
]
return parameter_dict
caucase_process = subprocess.Popen(
cau_args + [
'--mode', 'user',
'--send-csr', frontend_user_csr,
],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
def test_balancer_round_robin(self):
# requests are by default balanced to both servers
self.assertEqual(
{requests.get(self.default_balancer_url, verify=False).text for _ in range(10)},
{'backend_web_server1', 'backend_web_server2'}
)
result = caucase_process.communicate()
csr_id = result[0].split()[0]
subprocess.check_call(
cau_args + [
'--mode', 'user',
'--get-crt', csr_id, frontend_user_key,
],
def test_balancer_server_down(self):
# if one backend is down, it is excluded from balancer
self.getManagedResource("backend_web_server2", BalancerCookieHTTPServer).close()
self.addCleanup(self.getManagedResource("backend_web_server2", BalancerCookieHTTPServer).open)
self.assertEqual(
{requests.get(self.default_balancer_url, verify=False).text for _ in range(10)},
{'backend_web_server1',}
)
def test_balancer_set_cookie(self):
# if backend provides a "SERVERID" cookie, balancer will overwrite it with the
# backend selected by balancing algorithm
self.assertIn(
requests.get(urlparse.urljoin(self.default_balancer_url, '/set_cookie'), verify=False).cookies['SERVERID'],
('default-0', 'default-1'),
)
def test_balancer_respects_sticky_cookie(self):
# if request is made with the sticky cookie, the client stick on one balancer
cookies = dict(SERVERID='default-1')
self.assertEqual(
{requests.get(self.default_balancer_url, verify=False, cookies=cookies).text for _ in range(10)},
{'backend_web_server2',}
)
# if that backend becomes down, requests are balanced to another server
self.getManagedResource("backend_web_server2", BalancerCookieHTTPServer).close()
self.addCleanup(self.getManagedResource("backend_web_server2", BalancerCookieHTTPServer).open)
self.assertEqual(
requests.get(self.default_balancer_url, verify=False, cookies=cookies).text,
'backend_web_server1')
class TestHTTP(BalancerTestCase):
"""Check HTTP protocol
"""
__partition_reference__ = 'h'
def test_http_version(self):
# type: () -> None
# https://stackoverflow.com/questions/37012486/python-3-x-how-to-get-http-version-using-requests-library/37012810
self.assertEqual(
requests.get(self.default_balancer_url, verify=False).raw.version, 11)
def test_keep_alive(self):
# type: () -> None
# when doing two requests, connection is established only once
session = requests.Session()
session.verify = False
# do a first request, which establish a first connection
session.get(self.default_balancer_url).raise_for_status()
# "break" new connection method and check we can make another request
with mock.patch(
"requests.packages.urllib3.connectionpool.HTTPSConnectionPool._new_conn",
) as new_conn:
session.get(self.default_balancer_url).raise_for_status()
new_conn.assert_not_called()
parsed_url = urlparse.urlparse(self.default_balancer_url)
# check that we have an open file for the ip connection
self.assertTrue([
c for c in psutil.Process(os.getpid()).connections()
if c.status == 'ESTABLISHED' and c.raddr.ip == parsed_url.hostname
and c.raddr.port == parsed_url.port
])
class TestTLS(BalancerTestCase):
"""Check TLS
"""
__partition_reference__ = 's'
def _getServerCertificate(self, hostname, port):
# type: (Optional[str], Optional[int]) -> Any
hostname_idna = idna.encode(hostname)
sock = socket.socket()
sock.connect((hostname, port))
ctx = OpenSSL.SSL.Context(OpenSSL.SSL.SSLv23_METHOD)
ctx.check_hostname = False
ctx.verify_mode = OpenSSL.SSL.VERIFY_NONE
sock_ssl = OpenSSL.SSL.Connection(ctx, sock)
sock_ssl.set_connect_state()
sock_ssl.set_tlsext_host_name(hostname_idna)
sock_ssl.do_handshake()
cert = sock_ssl.get_peer_certificate()
crypto_cert = cert.to_cryptography()
sock_ssl.close()
sock.close()
return crypto_cert
def test_certificate_validates_with_caucase_ca(self):
# type: () -> None
caucase = self.getManagedResource("caucase", CaucaseService)
requests.get(self.default_balancer_url, verify=caucase.ca_crt_path)
def test_certificate_renewal(self):
# type: () -> None
caucase = self.getManagedResource("caucase", CaucaseService)
balancer_parsed_url = urlparse.urlparse(self.default_balancer_url)
certificate_before_renewal = self._getServerCertificate(
balancer_parsed_url.hostname,
balancer_parsed_url.port)
# run caucase updater 90 days in the future, so that certificate is
# renewed.
caucase_updater = os.path.join(
self.computer_partition_root_path,
'etc',
'service',
'caucase-updater',
)
process = pexpect.spawnu(
"faketime +90days %s" % caucase_updater,
env=dict(os.environ, PYTHONPATH=''),
)
logger = self.logger
class DebugLogFile:
def write(self, msg):
logger.info("output from caucase_updater: %s", msg)
def flush(self):
pass
process.logfile = DebugLogFile()
process.expect(u"Renewing .*\nNext wake-up.*")
process.terminate()
process.wait()
cls.client_certificate = frontend_service_key = os.path.join(frontend_service_dir, 'crt.pem')
frontend_service_csr = os.path.join(frontend_service_dir, 'csr.pem')
# wait for server to use new certificate
for _ in range(30):
certificate_after_renewal = self._getServerCertificate(
balancer_parsed_url.hostname,
balancer_parsed_url.port)
if certificate_after_renewal.not_valid_before > certificate_before_renewal.not_valid_before:
break
time.sleep(.5)
self.assertGreater(
certificate_after_renewal.not_valid_before,
certificate_before_renewal.not_valid_before,
)
# requests are served properly after cert renewal
requests.get(self.default_balancer_url, verify=caucase.ca_crt_path).raise_for_status()
class ContentTypeHTTPServer(ManagedHTTPServer):
"""An HTTP Server which reply with content type from path.
For example when requested http://host/text/plain it will reply
with Content-Type: text/plain header.
The body is always "OK"
"""
class RequestHandler(BaseHTTPRequestHandler):
def do_GET(self):
# type: () -> None
self.send_response(200)
if self.path == '/':
return self.end_headers()
content_type = self.path[1:]
self.send_header("Content-Type", content_type)
self.end_headers()
self.wfile.write("OK")
log_message = logging.getLogger(__name__ + '.ContentTypeHTTPServer').info
class TestContentEncoding(BalancerTestCase):
"""Test how responses are gzip encoded or not depending on content type header.
"""
__partition_reference__ = 'ce'
@classmethod
def _getInstanceParameterDict(cls):
# type: () -> Dict
parameter_dict = super(TestContentEncoding, cls)._getInstanceParameterDict()
parameter_dict['dummy_http_server'] = [
[cls.getManagedResource("content_type_server", ContentTypeHTTPServer).netloc, 1, False],
]
return parameter_dict
def test_gzip_encoding(self):
# type: () -> None
for content_type in (
'text/cache-manifest',
'text/html',
'text/plain',
'text/css',
'application/hal+json',
'application/json',
'application/x-javascript',
'text/xml',
'application/xml',
'application/rss+xml',
'text/javascript',
'application/javascript',
'image/svg+xml',
'application/x-font-ttf',
'application/font-woff',
'application/font-woff2',
'application/x-font-opentype',
'application/wasm',):
resp = requests.get(urlparse.urljoin(self.default_balancer_url, content_type), verify=False)
self.assertEqual(resp.headers['Content-Type'], content_type)
self.assertEqual(
resp.headers['Content-Encoding'],
'gzip',
'%s uses wrong encoding: %s' % (content_type, resp.headers['Content-Encoding']))
self.assertEqual(resp.text, 'OK')
def test_no_gzip_encoding(self):
# type: () -> None
resp = requests.get(urlparse.urljoin(self.default_balancer_url, '/image/png'), verify=False)
self.assertNotIn('Content-Encoding', resp.headers)
self.assertEqual(resp.text, 'OK')
class CaucaseClientCertificate(ManagedResource):
"""A client certificate issued by a caucase services.
"""
ca_crt_file = None # type: str
crl_file = None # type: str
csr_file = None # type: str
cert_file = None # type: str
key_file = None # type: str
def open(self):
# type: () -> None
self.tmpdir = tempfile.mkdtemp()
self.ca_crt_file = os.path.join(self.tmpdir, 'ca-crt.pem')
self.crl_file = os.path.join(self.tmpdir, 'ca-crl.pem')
self.csr_file = os.path.join(self.tmpdir, 'csr.pem')
self.cert_file = os.path.join(self.tmpdir, 'crt.pem')
self.key_file = os.path.join(self.tmpdir, 'key.pem')
def close(self):
# type: () -> None
shutil.rmtree(self.tmpdir)
@property
def _caucase_path(self):
# type: () -> str
"""path of caucase executable.
"""
software_release_root_path = os.path.join(
self._cls.slap._software_root,
hashlib.md5(self._cls.getSoftwareURL().encode()).hexdigest(),
)
return os.path.join(software_release_root_path, 'bin', 'caucase')
def request(self, common_name, caucase):
# type: (str, CaucaseService) -> None
"""Generate certificate and request signature to the caucase service.
This overwrite any previously requested certificate for this instance.
"""
cas_args = [
self._caucase_path,
'--ca-url', caucase.url,
'--ca-crt', self.ca_crt_file,
'--crl', self.crl_file,
]
key = rsa.generate_private_key(
public_exponent=65537,
key_size=2048,
backend=default_backend()
public_exponent=65537,
key_size=2048,
backend=default_backend()
)
with open(frontend_service_key, 'wb') as f:
f.write(key.private_bytes(
encoding=serialization.Encoding.PEM,
format=serialization.PrivateFormat.TraditionalOpenSSL,
encryption_algorithm=serialization.NoEncryption(),
))
csr = x509.CertificateSigningRequestBuilder().subject_name(x509.Name([
x509.NameAttribute(NameOID.COMMON_NAME, u'service'),
])).sign(key, hashes.SHA256(), default_backend())
with open(frontend_service_csr, 'wb') as f:
with open(self.key_file, 'wb') as f:
f.write(
key.private_bytes(
encoding=serialization.Encoding.PEM,
format=serialization.PrivateFormat.TraditionalOpenSSL,
encryption_algorithm=serialization.NoEncryption(),
))
csr = x509.CertificateSigningRequestBuilder().subject_name(
x509.Name([
x509.NameAttribute(
NameOID.COMMON_NAME,
common_name,
),
])).sign(
key,
hashes.SHA256(),
default_backend(),
)
with open(self.csr_file, 'wb') as f:
f.write(csr.public_bytes(serialization.Encoding.PEM))
caucase_process = subprocess.Popen(
csr_id = subprocess.check_output(
cas_args + [
'--send-csr', frontend_service_csr,
'--send-csr', self.csr_file,
],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
)
result = caucase_process.communicate()
csr_id = result[0].split()[0]
).split()[0]
assert csr_id
for _ in range(10):
for _ in range(30):
if not subprocess.call(
cas_args + [
'--get-crt', csr_id, frontend_service_key,
'--get-crt', csr_id, self.cert_file,
],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
) == 0:
break
else:
time.sleep(1)
else:
raise RuntimeError('getting service certificate failed.')
with open(self.cert_file) as f:
assert 'BEGIN CERTIFICATE' in f.read()
# start a caucased and server certificate.
cls.backend_caucase_dir = tempfile.mkdtemp()
backend_caucased_dir = os.path.join(cls.backend_caucase_dir, 'caucased')
os.mkdir(backend_caucased_dir)
backend_caucased_netloc = '%s:%s' % (cls._ipv4_address, findFreeTCPPort(cls._ipv4_address))
cls.backend_caucased_url = 'http://' + backend_caucased_netloc
cls.backend_caucased_process = subprocess.Popen(
[
caucased_path,
'--db', os.path.join(backend_caucased_dir, 'caucase.sqlite'),
'--server-key', os.path.join(backend_caucased_dir, 'server.key.pem'),
'--netloc', backend_caucased_netloc,
'--service-auto-approve-count', '1',
],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
)
for _ in range(10):
try:
if requests.get(cls.backend_caucased_url).status_code == 200:
break
except Exception:
pass
time.sleep(1)
else:
raise RuntimeError('caucased failed to start.')
def revoke(self, caucase):
# type: (str, CaucaseService) -> None
"""Revoke the client certificate on this caucase instance.
"""
subprocess.check_call([
self._caucase_path,
'--ca-url', caucase.url,
'--ca-crt', self.ca_crt_file,
'--crl', self.crl_file,
'--revoke-crt', self.cert_file, self.key_file,
])
super(TestFrontendXForwardedFor, cls).setUpClass()
@classmethod
def getInstanceParameterDict(cls):
return {
'_': json.dumps({
'tcpv4-port': 3306,
'computer-memory-percent-threshold': 100,
# XXX what is this ? should probably not be needed here
'name': cls.__name__,
'monitor-passwd': 'secret',
'apachedex-configuration': '',
'apachedex-promise-threshold': 100,
'haproxy-server-check-path': '/',
'zope-family-dict': {
'default': ['dummy_http_server'],
'default-auth': ['dummy_http_server'],
},
'dummy_http_server': [[cls.http_server_netloc, 1, False]],
'backend-path-dict': {
'default': '/',
'default-auth': '/',
},
'ssl-authentication-dict': {
'default': False,
'default-auth': True,
},
'ssl': {
'caucase-url': cls.backend_caucased_url,
'frontend-caucase-url-list': [cls.frontend_caucased_url],
},
})
}
class TestFrontendXForwardedFor(BalancerTestCase):
__partition_reference__ = 'xff'
@classmethod
def _cleanup(cls, snapshot_name):
if cls.http_server_process:
cls.http_server_process.terminate()
if cls.frontend_caucased_process:
cls.frontend_caucased_process.terminate()
if cls.frontend_caucase_dir:
shutil.rmtree(cls.frontend_caucase_dir)
if cls.backend_caucased_process:
cls.backend_caucased_process.terminate()
if cls.backend_caucase_dir:
shutil.rmtree(cls.backend_caucase_dir)
super(TestFrontendXForwardedFor, cls)._cleanup(snapshot_name)
def _getInstanceParameterDict(cls):
# type: () -> Dict
frontend_caucase = cls.getManagedResource('frontend_caucase', CaucaseService)
certificate = cls.getManagedResource('client_certificate', CaucaseClientCertificate)
certificate.request(u'shared frontend', frontend_caucase)
parameter_dict = super(TestFrontendXForwardedFor, cls)._getInstanceParameterDict()
# add another "-auth" backend, that will have ssl-authentication enabled
parameter_dict['zope-family-dict']['default-auth'] = ['dummy_http_server']
parameter_dict['backend-path-dict']['default-auth'] = '/'
parameter_dict['ssl-authentication-dict'] = {
'default': False,
'default-auth': True,
}
parameter_dict['ssl']['frontend-caucase-url-list'] = [frontend_caucase.url]
return parameter_dict
def test_x_forwarded_for_added_when_verified_connection(self):
# type: () -> None
client_certificate = self.getManagedResource('client_certificate', CaucaseClientCertificate)
for backend in ('default', 'default-auth'):
balancer_url = json.loads(self.computer_partition.getConnectionParameterDict()['_'])[backend]
result = requests.get(
balancer_url,
headers={'X-Forwarded-For': '1.2.3.4'},
cert=self.client_certificate,
cert=(client_certificate.cert_file, client_certificate.key_file),
verify=False,
).json()
self.assertEqual(result['Incoming Headers'].get('x-forwarded-for').split(', ')[0], '1.2.3.4')
def test_x_forwarded_for_stripped_when_not_verified_connection(self):
# type: () -> None
balancer_url = json.loads(self.computer_partition.getConnectionParameterDict()['_'])['default']
result = requests.get(
balancer_url,
......@@ -299,3 +699,103 @@ class TestFrontendXForwardedFor(ERP5InstanceTestCase):
headers={'X-Forwarded-For': '1.2.3.4'},
verify=False,
)
class TestClientTLS(BalancerTestCase):
__partition_reference__ = 'c'
@classmethod
def _getInstanceParameterDict(cls):
# type: () -> Dict
frontend_caucase1 = cls.getManagedResource('frontend_caucase1', CaucaseService)
certificate1 = cls.getManagedResource('client_certificate1', CaucaseClientCertificate)
certificate1.request(u'client_certificate1', frontend_caucase1)
frontend_caucase2 = cls.getManagedResource('frontend_caucase2', CaucaseService)
certificate2 = cls.getManagedResource('client_certificate2', CaucaseClientCertificate)
certificate2.request(u'client_certificate2', frontend_caucase2)
parameter_dict = super(TestClientTLS, cls)._getInstanceParameterDict()
parameter_dict['ssl-authentication-dict'] = {
'default': True,
}
parameter_dict['ssl']['frontend-caucase-url-list'] = [
frontend_caucase1.url,
frontend_caucase2.url,
]
return parameter_dict
def test_refresh_crl(self):
# type: () -> None
logger = self.logger
class DebugLogFile:
def write(self, msg):
logger.info("output from caucase_updater: %s", msg)
def flush(self):
pass
for client_certificate_name, caucase_name in (
('client_certificate1', 'frontend_caucase1'),
('client_certificate2', 'frontend_caucase2'),
):
client_certificate = self.getManagedResource(client_certificate_name,
CaucaseClientCertificate)
# when client certificate can be authenticated, backend receive the CN of
# the client certificate in "remote-user" header
def _make_request():
return requests.get(
self.default_balancer_url,
cert=(client_certificate.cert_file, client_certificate.key_file),
verify=False,
).json()
self.assertEqual(_make_request()['Incoming Headers'].get('remote-user'),
client_certificate_name)
# when certificate is revoked, updater service should update the CRL
# used by balancer from the caucase service used for client certificates
# (ie. the one used by frontend).
caucase = self.getManagedResource(caucase_name, CaucaseService)
client_certificate.revoke(caucase)
# until the CRL is updated, the client certificate is still accepted.
self.assertEqual(_make_request()['Incoming Headers'].get('remote-user'),
client_certificate_name)
# We have two services, in charge of updating CRL and CA certificates for
# each frontend CA
caucase_updater_list = glob.glob(
os.path.join(
self.computer_partition_root_path,
'etc',
'service',
'caucase-updater-*',
))
self.assertEqual(len(caucase_updater_list), 2)
# find the one corresponding to this caucase
for caucase_updater_candidate in caucase_updater_list:
with open(caucase_updater_candidate) as f:
if caucase.url in f.read():
caucase_updater = caucase_updater_candidate
break
else:
self.fail("Could not find caucase updater script for %s" % caucase.url)
# simulate running updater service in the future, to confirm that it fetches
# the new CRL and make sure balancer uses that new CRL.
process = pexpect.spawnu(
"faketime +1day %s" % caucase_updater,
env=dict(os.environ, PYTHONPATH=''),
)
process.logfile = DebugLogFile()
process.expect(u"Got new CRL.*Next wake-up at.*")
process.terminate()
process.wait()
with self.assertRaisesRegexp(Exception, 'certificate revoked'):
_make_request()
......@@ -31,6 +31,7 @@ import glob
import urlparse
import socket
import time
import tempfile
import psutil
import requests
......@@ -43,7 +44,7 @@ setUpModule # pyflakes
class TestPublishedURLIsReachableMixin(object):
"""Mixin that checks that default page of ERP5 is reachable.
"""
def _checkERP5IsReachable(self, url):
def _checkERP5IsReachable(self, url, verify):
# What happens is that instanciation just create the services, but does not
# wait for ERP5 to be initialized. When this test run ERP5 instance is
# instanciated, but zope is still busy creating the site and haproxy replies
......@@ -51,7 +52,7 @@ class TestPublishedURLIsReachableMixin(object):
# erp5 site is not created, with 500 when mysql is not yet reachable, so we
# retry in a loop until we get a succesful response.
for i in range(1, 60):
r = requests.get(url, verify=False) # XXX can we get CA from caucase already ?
r = requests.get(url, verify=verify)
if r.status_code != requests.codes.ok:
delay = i * 2
self.logger.warn("ERP5 was not available, sleeping for %ds and retrying", delay)
......@@ -62,19 +63,36 @@ class TestPublishedURLIsReachableMixin(object):
self.assertIn("ERP5", r.text)
def _getCaucaseServiceCACertificate(self):
ca_cert = tempfile.NamedTemporaryFile(
prefix="ca.crt.pem",
mode="w",
delete=False,
)
ca_cert.write(
requests.get(
urlparse.urljoin(
self.getRootPartitionConnectionParameterDict()['caucase-http-url'],
'/cas/crt/ca.crt.pem',
)).text)
self.addCleanup(os.unlink, ca_cert.name)
return ca_cert.name
def test_published_family_default_v6_is_reachable(self):
"""Tests the IPv6 URL published by the root partition is reachable.
"""
param_dict = self.getRootPartitionConnectionParameterDict()
self._checkERP5IsReachable(
urlparse.urljoin(param_dict['family-default-v6'], param_dict['site-id']))
urlparse.urljoin(param_dict['family-default-v6'], param_dict['site-id']),
self._getCaucaseServiceCACertificate())
def test_published_family_default_v4_is_reachable(self):
"""Tests the IPv4 URL published by the root partition is reachable.
"""
param_dict = self.getRootPartitionConnectionParameterDict()
self._checkERP5IsReachable(
urlparse.urljoin(param_dict['family-default'], param_dict['site-id']))
urlparse.urljoin(param_dict['family-default'], param_dict['site-id']),
self._getCaucaseServiceCACertificate())
class TestDefaultParameters(ERP5InstanceTestCase, TestPublishedURLIsReachableMixin):
......
......@@ -24,4 +24,3 @@ gems +=
[versions]
slapos.recipe.template = 4.4
rubygemsrecipe = 0.2.2+slapos001
......@@ -405,6 +405,5 @@ strip-top-level-dir = true
cns.recipe.symlink = 0.2.3
docutils = 0.12
plone.recipe.command = 1.1
rubygemsrecipe = 0.2.2+slapos002
slapos.recipe.template = 4.3
z3c.recipe.scripts = 1.0.1
......@@ -29,6 +29,9 @@ parts =
helloweb-go
[python]
part = python3
# Macro for jinja templates. The filename is set in buildout.hash.cfg
# in the section using this template
[jinja-template]
......@@ -54,4 +57,3 @@ context =
# Pin versions of eggs used that are not already pinned by stack/slapos.cfg
[versions]
slapos.recipe.template = 4.4
rubygemsrecipe = 0.2.2+slapos001
......@@ -55,7 +55,7 @@ md5sum = b7e87479a289f472b634a046b44b5257
[template-kvm-run]
filename = template/template-kvm-run.in
md5sum = 4a6f149177a453a13436f320f6841518
md5sum = a97b8462955dd422a30fbe02d6906172
[template-kvm-controller]
filename = template/kvm-controller-run.in
......
......@@ -505,9 +505,9 @@
},
"boot-image-url-list": {
"title": "Boot image list",
"description": "The list shall be list of direct URLs to images, followed by hash (#), then by image MD5SUM. Each image shall appear on newline, like: \"https://example.com/image.iso#06226c7fac5bacfa385872a19bb99684<newline>https://example.com/another-image.iso#31b40d58b18e038498ddb46caea1361c\". They will be provided in KVM image list according to the order on the list. After updating the list, the instance has to be restarted to refresh it. Amount of images is limited to 4, and one image can be maximum 5G. Image will be downloaded and checked against its MD5SUM 4 times, then it will be considered as impossible to download with given MD5SUM. Each image has to be downloaded in time shorter than 4 hours, so in case of very slow images to access, it can take up to 16 hours to download all of them. Note: The instance has to be restarted in order to update the list of available images in the VM.",
"description": "The list shall be list of direct URLs to images, followed by hash (#), then by image MD5SUM. Each image shall appear on newline, like: \"https://example.com/image.iso#06226c7fac5bacfa385872a19bb99684<newline>https://example.com/another-image.iso#31b40d58b18e038498ddb46caea1361c\". They will be provided in KVM image list according to the order on the list. After updating the list, the instance has to be restarted to refresh it. Amount of images is limited to 4, and one image can be maximum 5G. Image will be downloaded and checked against its MD5SUM 4 times, then it will be considered as impossible to download with given MD5SUM. Each image has to be downloaded in time shorter than 4 hours, so in case of very slow images to access, it can take up to 16 hours to download all of them. Note: The instance has to be restarted in order to update the list of available images in the VM. Note: Maximum 3 ISOs are supported.",
"type": "string",
"textarea": "true"
"textarea": true
},
"boot-image-url-select": {
"title": "Boot image",
......
......@@ -368,9 +368,9 @@
},
"boot-image-url-list": {
"title": "Boot image list",
"description": "The list shall be list of direct URLs to images, followed by hash (#), then by image MD5SUM. Each image shall appear on newline, like: \"https://example.com/image.iso#06226c7fac5bacfa385872a19bb99684<newline>https://example.com/another-image.iso#31b40d58b18e038498ddb46caea1361c\". They will be provided in KVM image list according to the order on the list. After updating the list, the instance has to be restarted to refresh it. Amount of images is limited to 4, and one image can be maximum 5G. Image will be downloaded and checked against its MD5SUM 4 times, then it will be considered as impossible to download with given MD5SUM. Each image has to be downloaded in time shorter than 4 hours, so in case of very slow images to access, it can take up to 16 hours to download all of them. Note: The instance has to be restarted in order to update the list of available images in the VM.",
"description": "The list shall be list of direct URLs to images, followed by hash (#), then by image MD5SUM. Each image shall appear on newline, like: \"https://example.com/image.iso#06226c7fac5bacfa385872a19bb99684<newline>https://example.com/another-image.iso#31b40d58b18e038498ddb46caea1361c\". They will be provided in KVM image list according to the order on the list. After updating the list, the instance has to be restarted to refresh it. Amount of images is limited to 4, and one image can be maximum 5G. Image will be downloaded and checked against its MD5SUM 4 times, then it will be considered as impossible to download with given MD5SUM. Each image has to be downloaded in time shorter than 4 hours, so in case of very slow images to access, it can take up to 16 hours to download all of them. Note: The instance has to be restarted in order to update the list of available images in the VM. Note: Maximum 3 ISOs are supported.",
"type": "string",
"textarea": "true"
"textarea": true
},
"boot-image-url-select": {
"title": "Boot image",
......
......@@ -10,6 +10,7 @@ extends =
../../component/netcat/buildout.cfg
../../component/lxml-python/buildout.cfg
../../component/pycurl/buildout.cfg
../../component/numpy/buildout.cfg
../../component/gzip/buildout.cfg
../../stack/slapos.cfg
../../component/nodejs/buildout.cfg
......@@ -206,4 +207,3 @@ gitdb = 0.6.4
pycurl = 7.43.0
slapos.recipe.template = 4.4
smmap = 0.9.0
numpy = 1.16.4
......@@ -371,7 +371,8 @@ for nbd_ip, nbd_port in nbd_list:
'-drive',
'file=nbd:[%s]:%s,media=cdrom' % (nbd_ip, nbd_port)])
else:
index = 0
# Note: Do not get tempted to use virtio-scsi-pci, as it does not work with
# Debian installation CDs, rendering it uninstallable
if boot_image_url_select_json_config:
# Support boot-image-url-select
with open(boot_image_url_select_json_config) as fh:
......@@ -381,11 +382,9 @@ else:
link = os.path.join(image_config['destination-directory'], image['link'])
if os.path.exists(link) and os.path.islink(link):
kvm_argument_list.extend([
'-drive', 'file=%s,media=cdrom,if=none,id=cdrom%s' % (link, index),
'-device', 'virtio-scsi-pci,id=scsi%s' % (index,),
'-device', 'scsi-cd,bus=scsi%s.0,drive=cdrom%s' % (index, index)
'-drive',
'file=%s,media=cdrom' % (link,)
])
index += 1
if boot_image_url_list_json_config:
# Support boot-image-url-list
with open(boot_image_url_list_json_config) as fh:
......@@ -395,16 +394,12 @@ else:
link = os.path.join(image_config['destination-directory'], image['link'])
if os.path.exists(link) and os.path.islink(link):
kvm_argument_list.extend([
'-drive', 'file=%s,media=cdrom,if=none,id=cdrom%s' % (link, index),
'-device', 'virtio-scsi-pci,id=scsi%s' % (index,),
'-device', 'scsi-cd,bus=scsi%s.0,drive=cdrom%s' % (index, index)
'-drive',
'file=%s,media=cdrom' % (link,)
])
index += 1
# Always add by default the default image
kvm_argument_list.extend([
'-drive', 'file=%s,media=cdrom,if=none,id=cdrom%s' % (default_cdrom_iso, index),
'-device', 'virtio-scsi-pci,id=scsi%s' % (index,),
'-device', 'scsi-cd,bus=scsi%s.0,drive=cdrom%s' % (index, index)
'-drive', 'file=%s,media=cdrom' % default_cdrom_iso
])
......
......@@ -604,7 +604,7 @@ class TestBootImageUrlList(InstanceTestCase):
# check that the image is NOT YET available in kvm
self.assertEqual(
['file=/parts/debian-amd64-netinst.iso/debian-amd64-netinst.iso,'
'media=cdrom,if=none,id=cdrom0'],
'media=cdrom'],
getRunningImageList()
)
......@@ -618,12 +618,10 @@ class TestBootImageUrlList(InstanceTestCase):
# now the image is available in the kvm, and its above default image
self.assertEqual(
[
'file=/srv/%s/image_001,media=cdrom,if=none,id=cdrom0' % (
self.image_directory,),
'file=/srv/%s/image_002,media=cdrom,if=none,id=cdrom1' % (
self.image_directory,),
'file=/srv/%s/image_001,media=cdrom' % (self.image_directory,),
'file=/srv/%s/image_002,media=cdrom' % (self.image_directory,),
'file=/parts/debian-amd64-netinst.iso/debian-amd64-netinst.iso,'
'media=cdrom,if=none,id=cdrom2'
'media=cdrom'
],
getRunningImageList()
)
......@@ -647,7 +645,7 @@ class TestBootImageUrlList(InstanceTestCase):
# again only default image is available in the running process
self.assertEqual(
['file=/parts/debian-amd64-netinst.iso/debian-amd64-netinst.iso,'
'media=cdrom,if=none,id=cdrom0'],
'media=cdrom'],
getRunningImageList()
)
......@@ -786,7 +784,7 @@ class TestBootImageUrlSelect(TestBootImageUrlList):
# check that the image is NOT YET available in kvm
self.assertEqual(
['file=/parts/debian-amd64-netinst.iso/debian-amd64-netinst.iso,'
'media=cdrom,if=none,id=cdrom0'],
'media=cdrom'],
getRunningImageList()
)
......@@ -800,12 +798,10 @@ class TestBootImageUrlSelect(TestBootImageUrlList):
# now the image is available in the kvm, and its above default image
self.assertEqual(
[
'file=/srv/boot-image-url-select-repository/image_001,media=cdrom,'
'if=none,id=cdrom0',
'file=/srv/boot-image-url-list-repository/image_001,media=cdrom,'
'if=none,id=cdrom1',
'file=/srv/boot-image-url-select-repository/image_001,media=cdrom',
'file=/srv/boot-image-url-list-repository/image_001,media=cdrom',
'file=/parts/debian-amd64-netinst.iso/debian-amd64-netinst.iso,'
'media=cdrom,if=none,id=cdrom2'
'media=cdrom'
],
getRunningImageList()
)
......@@ -834,7 +830,7 @@ class TestBootImageUrlSelect(TestBootImageUrlList):
# again only default image is available in the running process
self.assertEqual(
['file=/parts/debian-amd64-netinst.iso/debian-amd64-netinst.iso,'
'media=cdrom,if=none,id=cdrom0'],
'media=cdrom'],
getRunningImageList()
)
......
......@@ -15,10 +15,11 @@ extends =
../../component/lxml-python/buildout.cfg
#END LXML
../../component/msgpack-python/buildout.cfg
../../component/patch/buildout.cfg
../../component/python-mysqlclient/buildout.cfg
../../component/python-cryptography/buildout.cfg
../../component/pycurl/buildout.cfg
../../component/ZEO/buildout.cfg
../../component/zodbtools/buildout.cfg
parts =
# keep neoppod first so that ZODB is built correctly,
......@@ -50,14 +51,10 @@ eggs = neoppod[admin, ctl, master]
psutil
ZODB
zope.testing
zodbtools
${zodbtools:egg}
coverage
setproctitle
adapter-egg = ${python-mysqlclient:egg}
patch-binary = ${patch:location}/bin/patch
ZEO-patch-options = -p1
ZEO-patches =
${:_profile_base_location_}/../../component/egg-patch/ZEO4/TCP_NODELAY.patch#b07288522d5c6857738240d948321df6
# XXX: buildout fails to install properly eggs with setup_requires
[BTrees]
......@@ -152,23 +149,9 @@ setproctitle = 1.1.10
slapos.recipe.template = 4.4
transaction = 1.7.0
zodbpickle = 1.0.4
zodbtools = 0.0.0.dev4
cython-zstd = 0.2
python-dateutil = 2.7.3
# Required by:
# zodbtools==0.0.0dev4
zodburi = 2.3.0
# Required by:
# zodburi==2.0
# ZEO 5 requires transaction >= 2
ZEO = 4.3.1+SlapOSPatched001
# Required by:
# ZEO==4.3.1
zdaemon = 4.2.0
# Required by:
# mock = 3.0.5
funcsigs = 1.0.2
......
......@@ -3,7 +3,9 @@ extends = software.cfg
[neoppod]
eggs += mock
ZEO-patches =
[ZEO]
egg = ${ZEO5:egg}
[versions]
ZODB = 5.6.0
......
......@@ -19,11 +19,3 @@ md5sum = bd46e95f1cea62c3b0082fe8c0c9c90b
[neotest]
filename = neotest.in
md5sum = fb3b4109128c1db1739ef5bb6abd1d94
[neotest-runTestSuite]
filename = neotest-runTestSuite.in
md5sum = 6a4281730b68cdba5c873817a6754428
[instance.cfg]
filename = instance.cfg.in
md5sum = 8a40d365c85d70f057ce997405ac3e88
......@@ -5,25 +5,34 @@
depends_gitfetch =
${go_github.com_DataDog_czlib:recipe}
${go_github.com_cznic_strutil:recipe}
${go_github.com_davecgh_go-spew:recipe}
${go_github.com_fsnotify_fsnotify:recipe}
${go_github.com_golang_glog:recipe}
${go_github.com_gwenn_gosqlite:recipe}
${go_github.com_gwenn_yacr:recipe}
${go_github.com_kisielk_og-rek:recipe}
${go_github.com_kylelemons_godebug:recipe}
${go_github.com_philhofer_fwd:recipe}
${go_github.com_pkg_errors:recipe}
${go_github.com_pkg_profile:recipe}
${go_github.com_pmezard_go-difflib:recipe}
${go_github.com_shamaton_msgpack:recipe}
${go_github.com_soheilhy_cmux:recipe}
${go_github.com_someonegg_gocontainer:recipe}
${go_github.com_someonegg_gox:recipe}
${go_github.com_stretchr_testify:recipe}
${go_github.com_tinylib_msgp:recipe}
${go_github.com_ttacon_chalk:recipe}
${go_golang.org_x_crypto:recipe}
${go_golang.org_x_mod:recipe}
${go_golang.org_x_net:recipe}
${go_golang.org_x_perf:recipe}
${go_golang.org_x_sync:recipe}
${go_golang.org_x_sys:recipe}
${go_golang.org_x_text:recipe}
${go_golang.org_x_tools:recipe}
${go_golang.org_x_xerrors:recipe}
${go_gopkg.in_yaml.v3:recipe}
${go_lab.nexedi.com_kirr_go123:recipe}
${go_lab.nexedi.com_kirr_neo:recipe}
......@@ -40,11 +49,17 @@ go.importpath = github.com/cznic/strutil
repository = https://github.com/cznic/strutil
revision = 529a34b1c1
[go_github.com_davecgh_go-spew]
<= go-git-package
go.importpath = github.com/davecgh/go-spew
repository = https://github.com/davecgh/go-spew
revision = v1.1.1-1-gd8f796af33
[go_github.com_fsnotify_fsnotify]
<= go-git-package
go.importpath = github.com/fsnotify/fsnotify
repository = https://github.com/fsnotify/fsnotify
revision = ccc981bf80
revision = 7f4cf4dd2b
[go_github.com_golang_glog]
<= go-git-package
......@@ -56,112 +71,160 @@ revision = 23def4e6c1
<= go-git-package
go.importpath = github.com/gwenn/gosqlite
repository = https://github.com/gwenn/gosqlite
revision = 29cd841087
revision = 24878be1a2
[go_github.com_gwenn_yacr]
<= go-git-package
go.importpath = github.com/gwenn/yacr
repository = https://github.com/gwenn/yacr
revision = 77093bdc7e
revision = bbe82c1f4d
[go_github.com_kisielk_og-rek]
<= go-git-package
go.importpath = github.com/kisielk/og-rek
repository = https://github.com/kisielk/og-rek
revision = 8b25c4cefd
revision = 24bb08c22e
[go_github.com_kylelemons_godebug]
<= go-git-package
go.importpath = github.com/kylelemons/godebug
repository = https://github.com/kylelemons/godebug
revision = d65d576e93
revision = fa7b53cdfc
[go_github.com_philhofer_fwd]
<= go-git-package
go.importpath = github.com/philhofer/fwd
repository = https://github.com/philhofer/fwd
revision = 414ae1bb9e
[go_github.com_pkg_errors]
<= go-git-package
go.importpath = github.com/pkg/errors
repository = https://github.com/pkg/errors
revision = v0.8.0-12-g816c908556
revision = v0.8.1-31-g614d223910
[go_github.com_pkg_profile]
<= go-git-package
go.importpath = github.com/pkg/profile
repository = https://github.com/pkg/profile
revision = v1.2.1-0-g5b67d42886
revision = v1.5.0-1-g3704c8d233
[go_github.com_pmezard_go-difflib]
<= go-git-package
go.importpath = github.com/pmezard/go-difflib
repository = https://github.com/pmezard/go-difflib
revision = v1.0.0-4-g5d4384ee4f
[go_github.com_shamaton_msgpack]
<= go-git-package
go.importpath = github.com/shamaton/msgpack
repository = https://github.com/shamaton/msgpack
revision = 29a4ba0bb9
[go_github.com_soheilhy_cmux]
<= go-git-package
go.importpath = github.com/soheilhy/cmux
repository = https://github.com/soheilhy/cmux
revision = e09e9389d8
revision = 8a8ea3c539
[go_github.com_someonegg_gocontainer]
<= go-git-package
go.importpath = github.com/someonegg/gocontainer
repository = https://github.com/someonegg/gocontainer
revision = fc2c7e84b5
revision = 24fb283ab9
[go_github.com_someonegg_gox]
<= go-git-package
go.importpath = github.com/someonegg/gox
repository = https://github.com/someonegg/gox
revision = 4915b7fd7c
revision = c9bfdf8eb4
[go_github.com_stretchr_testify]
<= go-git-package
go.importpath = github.com/stretchr/testify
repository = https://github.com/stretchr/testify
revision = 363ebb24d0
revision = v1.2.2-210-g54d05a4e18
[go_github.com_tinylib_msgp]
<= go-git-package
go.importpath = github.com/tinylib/msgp
repository = https://github.com/tinylib/msgp
revision = 87c1ec45d5
[go_github.com_ttacon_chalk]
<= go-git-package
go.importpath = github.com/ttacon/chalk
repository = https://github.com/ttacon/chalk
revision = v0.1-3-g22c06c80ed
[go_golang.org_x_crypto]
<= go-git-package
go.importpath = golang.org/x/crypto
repository = https://go.googlesource.com/crypto
revision = 88942b9c40
revision = 5c72a88397
[go_golang.org_x_mod]
<= go-git-package
go.importpath = golang.org/x/mod
repository = https://go.googlesource.com/mod
revision = ce943fd024
[go_golang.org_x_net]
<= go-git-package
go.importpath = golang.org/x/net
repository = https://go.googlesource.com/net
revision = 6078986fec
revision = 62affa334b
[go_golang.org_x_perf]
<= go-git-package
go.importpath = golang.org/x/perf
repository = https://go.googlesource.com/perf
revision = 8c788eb673
revision = d949658356
[go_golang.org_x_sync]
<= go-git-package
go.importpath = golang.org/x/sync
repository = https://go.googlesource.com/sync
revision = 1d60e4601c
revision = 6e8e738ad2
[go_golang.org_x_sys]
<= go-git-package
go.importpath = golang.org/x/sys
repository = https://go.googlesource.com/sys
revision = 91ee8cde43
revision = aee5d888a8
[go_golang.org_x_text]
<= go-git-package
go.importpath = golang.org/x/text
repository = https://go.googlesource.com/text
revision = v0.3.0-42-gab48842968
revision = v0.3.3-2-ga8b4671254
[go_golang.org_x_tools]
<= go-git-package
go.importpath = golang.org/x/tools
repository = https://go.googlesource.com/tools
revision = 77106db15f
revision = ba800b16d8
[go_golang.org_x_xerrors]
<= go-git-package
go.importpath = golang.org/x/xerrors
repository = https://go.googlesource.com/xerrors
revision = 5ec99f83af
[go_gopkg.in_yaml.v3]
<= go-git-package
go.importpath = gopkg.in/yaml.v3
repository = https://gopkg.in/yaml.v3
revision = v2.1.1-97-geeeca48fe7
[go_lab.nexedi.com_kirr_go123]
<= go-git-package
go.importpath = lab.nexedi.com/kirr/go123
repository = https://lab.nexedi.com/kirr/go123.git
revision = 76f667ba6c
revision = 316617668e
[go_lab.nexedi.com_kirr_neo]
<= go-git-package
go.importpath = lab.nexedi.com/kirr/neo
repository = https://lab.nexedi.com/kirr/neo.git
revision = v1.8.1-1634-g4000df14e4
revision = v1.9-2480-g5ee3c077b3
#!/bin/bash -e
# neotest's runTestSuite wraper so it could be run without any environment preset
. ${buildout:directory}/neotest-env.sh
exec ${gowork:src}/lab.nexedi.com/kirr/neo/go/neo/t/nxd/runTestSuite "$@"
......@@ -17,6 +17,10 @@ extends =
../../component/lmbench/buildout.cfg
../../component/coreutils/buildout.cfg
../../component/util-linux/buildout.cfg
../../component/pygolang/buildout.cfg
../../component/numpy/buildout.cfg
../../stack/nxdtest.cfg
parts =
gowork
......@@ -36,7 +40,6 @@ parts =
neotest-env.sh
neotest
neotest-runTestSuite
# for instance
slapos-cookbook
......@@ -73,23 +76,28 @@ output = ${buildout:directory}/${:_buildout_section_name_}
output = ${buildout:bin-directory}/${:_buildout_section_name_}
mode = 0755
[neotest-runTestSuite]
<= buildout-template
output = ${buildout:bin-directory}/${:_buildout_section_name_}
mode = 0755
# instance
[jinja2-template]
recipe = slapos.recipe.template:jinja2
template= ${:_profile_base_location_}/${:filename}
rendered= ${buildout:directory}/${:_buildout_section_name_}
mode = 0644
context =
section buildout buildout
# instance to run nxdtest.
[instance.cfg]
<= jinja2-template
template = inline:
[buildout]
extends = ${nxdtest-instance.cfg:rendered}
parts += tnxdtest
[runTestSuite]
env.sh = ${neotest-env.sh:output}
workdir = $${directory:t}
[directory]
t = $${:home}/t
# instance/t/.nxdtest -> .nxdtest inside go/neo
[tnxdtest]
recipe = plone.recipe.command
stop-on-error = yes
command = ln -s -t $${directory:t} ${gowork:src}/lab.nexedi.com/kirr/neo/go/neo/t/.nxdtest
# eggs:
......@@ -99,15 +107,15 @@ eggs =
# wendelin.core still requires ZODB3 but having ZODB4 or 5 installed satisfies latest ZODB3
ZODB3
# also for wc
numpy
${numpy:egg}
# to install not only wendelin.core modules but also scripts
wendelin.core
# for ZEO scripts (runzeo)
ZEO
# for nxd/runTestSuite
erp5.util
${ZEO:egg}
# for e.g. tcpu.py
pygolang
${pygolang:egg}
# for instance
plone.recipe.command
# wendelin.core: latest not yet released
......@@ -117,9 +125,7 @@ revision= v0.11-15-gf785ac079b
# ping eggs versions
[versions]
ZODB3 = 3.11.0
numpy = 1.16.4
zope.testing = 4.6.2
pygolang = 0.0.0.dev4
# Required by:
# ZEO==4.3.1
......
......@@ -153,4 +153,3 @@ link-binary =
# Pin versions of eggs used that are not already pinned by stack/slapos.cfg
[versions]
slapos.recipe.template = 4.4
rubygemsrecipe = 0.2.2+slapos001
......@@ -34,8 +34,10 @@ import unittest
import urlparse
import base64
import hashlib
import logging
import contextlib
from BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer
from BaseHTTPServer import BaseHTTPRequestHandler
from io import BytesIO
import paramiko
......@@ -48,81 +50,69 @@ from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.ui import WebDriverWait
from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
from slapos.testing.utils import findFreeTCPPort, ImageComparisonTestCase
from slapos.testing.utils import findFreeTCPPort, ImageComparisonTestCase, ManagedHTTPServer
setUpModule, SeleniumServerTestCase = makeModuleSetUpAndTestCaseClass(
os.path.abspath(
os.path.join(os.path.dirname(__file__), '..', 'software.cfg')))
class WebServer(ManagedHTTPServer):
class RequestHandler(BaseHTTPRequestHandler):
"""Request handler for our test server.
The implemented server is:
- submit q and you'll get a page with q as title
- upload a file and the file content will be displayed in div.uploadedfile
"""
def do_GET(self):
self.send_response(200)
self.end_headers()
self.wfile.write(
'''
<html>
<title>Test page</title>
<body>
<style> p { font-family: Arial; } </style>
<form action="/" method="POST" enctype="multipart/form-data">
<input name="q" type="text"></input>
<input name="f" type="file" ></input>
<input type="submit" value="I'm feeling lucky"></input>
</form>
<p>the quick brown fox jumps over the lazy dog</p>
</body>
</html>''')
def do_POST(self):
form = cgi.FieldStorage(
fp=self.rfile,
headers=self.headers,
environ={
'REQUEST_METHOD': 'POST',
'CONTENT_TYPE': self.headers['Content-Type'],
})
self.send_response(200)
self.end_headers()
file_data = 'no file'
if form.has_key('f'):
file_data = form['f'].file.read()
self.wfile.write(
'''
<html>
<title>%s</title>
<div>%s</div>
</html>
''' % (form['q'].value, file_data))
log_message = logging.getLogger(__name__ + '.WebServer').info
class WebServerMixin(object):
"""Mixin class which provides a simple web server reachable at self.server_url
"""
def setUp(self):
"""Start a minimal web server.
"""
class TestHandler(BaseHTTPRequestHandler):
"""Request handler for our test server.
The implemented server is:
- submit q and you'll get a page with q as title
- upload a file and the file content will be displayed in div.uploadedfile
"""
def log_message(self, *args, **kw):
if SeleniumServerTestCase._debug:
BaseHTTPRequestHandler.log_message(self, *args, **kw)
def do_GET(self):
self.send_response(200)
self.end_headers()
self.wfile.write(
'''
<html>
<title>Test page</title>
<body>
<style> p { font-family: Arial; } </style>
<form action="/" method="POST" enctype="multipart/form-data">
<input name="q" type="text"></input>
<input name="f" type="file" ></input>
<input type="submit" value="I'm feeling lucky"></input>
</form>
<p>the quick brown fox jumps over the lazy dog</p>
</body>
</html>''')
def do_POST(self):
form = cgi.FieldStorage(
fp=self.rfile,
headers=self.headers,
environ={
'REQUEST_METHOD': 'POST',
'CONTENT_TYPE': self.headers['Content-Type'],
})
self.send_response(200)
self.end_headers()
file_data = 'no file'
if form.has_key('f'):
file_data = form['f'].file.read()
self.wfile.write(
'''
<html>
<title>%s</title>
<div>%s</div>
</html>
''' % (form['q'].value, file_data))
super(WebServerMixin, self).setUp()
ip = os.environ.get('SLAPOS_TEST_IPV4', '127.0.1.1')
port = findFreeTCPPort(ip)
server = HTTPServer((ip, port), TestHandler)
self.server_process = multiprocessing.Process(target=server.serve_forever)
self.server_process.start()
self.server_url = 'http://%s:%s/' % (ip, port)
def tearDown(self):
self.server_process.terminate()
self.server_process.join()
super(WebServerMixin, self).tearDown()
self.server_url = self.getManagedResource('web_server', WebServer).url
class BrowserCompatibilityMixin(WebServerMixin):
......
......@@ -12,6 +12,7 @@ eggs -=
[template]
extra =
${slapos.test.helloworld-setup:setup}
${slapos.test.monitor-setup:setup}
${slapos.test.powerdns-setup:setup}
${slapos.test.proftpd-setup:setup}
......
......@@ -259,7 +259,6 @@ extra =
${slapos.test.plantuml-setup:setup}
${slapos.test.re6stnet-setup:setup}
${slapos.test.seleniumserver-setup:setup}
${slapos.test.helloworld-setup:setup}
${slapos.test.jstestnode-setup:setup}
${slapos.test.jupyter-setup:setup}
${slapos.test.nextcloud-setup:setup}
......@@ -317,3 +316,4 @@ funcsigs = 1.0.2
mysqlclient = 1.3.12
pexpect = 4.8.0
ptyprocess = 0.6.0
typing = 3.7.4.3
......@@ -19,7 +19,7 @@ md5sum = 397fcb3279029af3055b23525d147445
[yarn.lock]
filename = yarn.lock
md5sum = d058e73c3d90ac3da44734c2d47eac95
md5sum = c7aa84922a1b80fd8a4c3d96f6ac7e25
[python-language-server-requirements.txt]
filename = python-language-server-requirements.txt
......
This source diff could not be displayed because it is too large. You can view the blob instead.
......@@ -455,7 +455,7 @@ eggs = ${neoppod:eggs}
# Zope acquisition patch
Acquisition
# for runzeo
ZEO
${ZEO:egg}
# Other Zope 2 packages
Products.PluggableAuthService
......
......@@ -78,7 +78,7 @@ md5sum = 68b329da9893e34099c7d8ad5cb9c940
[template-erp5]
filename = instance-erp5.cfg.in
md5sum = 82dc695e212be124d60ceb1143e56b0d
md5sum = 0920a53b10d3811a5f49930adffb62d8
[template-zeo]
filename = instance-zeo.cfg.in
......@@ -90,7 +90,7 @@ md5sum = 2f3ddd328ac1c375e483ecb2ef5ffb57
[template-balancer]
filename = instance-balancer.cfg.in
md5sum = bb9a953ce22f7d5188385f0171b6198e
md5sum = ecf119142e6b5cd85a2ba397552d2142
[template-haproxy-cfg]
filename = haproxy.cfg.in
......
......@@ -18,19 +18,52 @@ per partition. No more (undefined result), no less (IndexError).
recipe = slapos.recipe.template:jinja2
mode = 644
[balancer-csr-request-config]
< = jinja2-template-base
template = inline:
[req]
prompt = no
req_extensions = req_ext
distinguished_name = dn
[ dn ]
CN = example.com
[ req_ext ]
subjectAltName = @alt_names
[ alt_names ]
IP.1 = {{ ipv4 }}
{% if ipv6_set -%}
IP.2 = {{ ipv6 }}
{% endif %}
rendered = ${buildout:parts-directory}/${:_buildout_section_name_}/${:_buildout_section_name_}.txt
[balancer-csr-request]
recipe = plone.recipe.command
command = {{ parameter_dict["openssl"] }}/bin/openssl req \
-newkey rsa:2048 \
-batch \
-new \
-nodes \
-keyout '${apache-conf-ssl:key}' \
-config '${balancer-csr-request-config:rendered}' \
-out '${:csr}'
stop-on-error = true
csr = ${directory:etc}/${:_buildout_section_name_}.csr.pem
{{ caucase.updater(
prefix='caucase-updater',
buildout_bin_directory=parameter_dict['bin-directory'],
updater_path='${directory:services-on-watch}/caucase-updater',
url=ssl_parameter_dict['caucase-url'],
data_dir='${directory:srv}/caucase-updater',
crt_path='${apache-conf-ssl:caucase-cert}',
crt_path='${apache-conf-ssl:cert}',
ca_path='${directory:srv}/caucase-updater/ca.crt',
crl_path='${directory:srv}/caucase-updater/crl.pem',
key_path='${apache-conf-ssl:caucase-key}',
key_path='${apache-conf-ssl:key}',
on_renew='${apache-graceful:output}',
max_sleep=ssl_parameter_dict.get('max-crl-update-delay', 1.0),
template_csr_pem=ssl_parameter_dict.get('csr'),
template_csr=None if ssl_parameter_dict.get('csr') else '${balancer-csr-request:csr}',
openssl=parameter_dict['openssl'] ~ '/bin/openssl',
)}}
{% do section('caucase-updater') -%}
......@@ -176,9 +209,6 @@ hash-files = ${haproxy-cfg:rendered}
[apache-conf-ssl]
cert = ${directory:apache-conf}/apache.crt
key = ${directory:apache-conf}/apache.pem
# XXX caucase certificate is not supported by caddy for now
caucase-cert = ${directory:apache-conf}/apache-caucase.crt
caucase-key = ${directory:apache-conf}/apache-caucase.pem
{% if frontend_caucase_url_list -%}
depends = ${caucase-updater-housekeeper-run:recipe}
ca-cert-dir = ${directory:apache-ca-cert-dir}
......@@ -201,19 +231,6 @@ context = key content {{content_section_name}}:content
mode = {{ mode }}
{%- endmacro %}
[apache-ssl]
{% if ssl_parameter_dict.get('key') -%}
key = ${apache-ssl-key:rendered}
cert = ${apache-ssl-cert:rendered}
{{ simplefile('apache-ssl-key', '${apache-conf-ssl:key}', ssl_parameter_dict['key']) }}
{{ simplefile('apache-ssl-cert', '${apache-conf-ssl:cert}', ssl_parameter_dict['cert']) }}
{% else %}
recipe = plone.recipe.command
command = "{{ parameter_dict['openssl'] }}/bin/openssl" req -newkey rsa -batch -new -x509 -days 3650 -nodes -keyout "${:key}" -out "${:cert}"
key = ${apache-conf-ssl:key}
cert = ${apache-conf-ssl:cert}
{%- endif %}
[apache-conf-parameter-dict]
backend-list = {{ dumps(apache_dict.values()) }}
zope-virtualhost-monster-backend-dict = {{ dumps(zope_virtualhost_monster_backend_dict) }}
......@@ -225,8 +242,8 @@ access-log = ${directory:log}/apache-access.log
# Apache 2.4's default value (60 seconds) can be a bit too short
timeout = 300
# Basic SSL server configuration
cert = ${apache-ssl:cert}
key = ${apache-ssl:key}
cert = ${apache-conf-ssl:cert}
key = ${apache-conf-ssl:key}
cipher =
ssl-session-cache = ${directory:log}/apache-ssl-session-cache
{% if frontend_caucase_url_list -%}
......
......@@ -176,11 +176,16 @@ return =
{%- if test_runner_enabled %}
test-runner-address-list
{% endif %}
{% set bt5_default_list = 'erp5_full_text_mroonga_catalog erp5_configurator_standard erp5_configurator_maxma_demo erp5_configurator_run_my_doc' -%}
{% set bt5_default_list = [
'erp5_full_text_mroonga_catalog',
'erp5_configurator_standard',
'erp5_configurator_maxma_demo',
'erp5_configurator_run_my_doc',
] -%}
{% if has_jupyter -%}
{% set bt5_default_list = bt5_default_list + ' erp5_data_notebook' -%}
{% do bt5_default_list.append('erp5_data_notebook') -%}
{% endif -%}
config-bt5 = {{ dumps(slapparameter_dict.get('bt5', bt5_default_list)) }}
config-bt5 = {{ dumps(slapparameter_dict.get('bt5', ' '.join(bt5_default_list))) }}
config-bt5-repository-url = {{ dumps(slapparameter_dict.get('bt5-repository-url', local_bt5_repository)) }}
config-cloudooo-url = {{ dumps(slapparameter_dict.get('cloudooo-url', default_cloudooo_url)) }}
config-caucase-url = {{ dumps(caucase_url) }}
......
# Stack for creating testnode instances to be run via nxdtest on Nexedi testing
# infrastructure.
#
# Usage:
#
# ---- 8< ---- (<software>/test.cfg)
# [buildout]
# extends = .../stack/nxdtest.cfg
#
# parts =
# ...
#
# # for instance
# slapos-cookbook
# instance.cfg
#
# ...
#
# [instance.cfg]
# <= jinja2-template
# template = inline:
# [buildout]
# extends = ${nxdtest-instance.cfg:rendered}
#
# [runTestSuite]
# env.sh = ...
# workdir = ...
#
# Created instance will have bin/runTestSuite that sources env.sh and runs
# nxdtest in workdir.
[buildout]
extends =
slapos.cfg
../component/git/buildout.cfg
nxdtest/buildout.hash.cfg
[jinja2-template]
recipe = slapos.recipe.template:jinja2
template = ${:_profile_base_location_}/${:filename}
rendered = ${buildout:directory}/${:_buildout_section_name_}
mode = 0644
context =
section buildout buildout
[nxdtest-instance.cfg]
<= jinja2-template
template = ${:_profile_base_location_}/nxdtest/${:filename}
# NOTE += does not work
context =
section buildout buildout
section nxdtest nxdtest
[nxdtest]
recipe = zc.recipe.egg:scripts
eggs = ${nxdtest-egg:egg}
scripts = nxdtest
# convenience for nxdtest users
exe = ${buildout:bin-directory}/nxdtest
[nxdtest-egg]
recipe = zc.recipe.egg:develop
setup = ${nxdtest-repository:location}
egg = nxdtest
[nxdtest-repository]
recipe = slapos.recipe.build:gitclone
repository = https://lab.nexedi.com/nexedi/nxdtest.git
revision = bd91f6f1579a
location = ${buildout:parts-directory}/nxdtest
git-executable = ${git:location}/bin/git
[versions]
slapos.recipe.template = 4.4
# THIS IS NOT A BUILDOUT FILE ... -> use update-hash to update it.
[nxdtest-instance.cfg]
filename = instance.cfg.in
md5sum = ea54ba85655f52ce491c84450400e5af
# NEO test instance: run neotest under Nexedi testing infrastructure
# instance to test a software via nxdtest under Nexedi testing infrastructure.
[buildout]
parts = runTestSuite
......@@ -17,12 +17,17 @@ bin = {{ buildout['bin-directory'] }}
recipe = slapos.cookbook:mkdirectory
home = ${buildout:directory}
bin = ${:home}/bin
neotest = ${:home}/neotest
# script to run the testsuite from inside <instance>/neotest/
# script to run the testsuite via nxdtest
# located @ <instance>/bin/runTestSuite so testnode can see this as run tests entrypoint.
#
# instance that uses stack/nxdtest.cfg must provide
#
# [runTestSuite]
# env.sh = ...
# workdir = ...
[runTestSuite]
recipe = slapos.cookbook:wrapper
wrapper-path = ${directory:bin}/${:_buildout_section_name_}
command-line = /bin/bash -c 'cd ${directory:neotest} && ${software:bin}/neotest-runTestSuite "$@"' runTestSuite
command-line = /bin/bash -c '. ${:env.sh} && cd ${:workdir} && {{ nxdtest['exe'] }} "$@"' nxdtest
# "$@" is implicitly added to argv
......@@ -143,10 +143,13 @@ zc.recipe.egg = 2.0.3+slapos003
hexagonit.recipe.download = 1.7.post4
Jinja2 = 2.9.5
Importing = 1.10
MarkupSafe = 1.0
PyYAML = 3.13
Werkzeug = 0.12
asn1crypto = 1.3.0
atomicwrites = 1.4.0
backports.functools-lru-cache = 1.6.1
backports.lzma = 0.0.14
cffi = 1.14.0
click = 6.7
......@@ -154,23 +157,38 @@ cliff = 2.4.0
cmd2 = 0.7.0
collective.recipe.shelloutput = 0.1
collective.recipe.template = 2.0
configparser = 4.0.2
contextlib2 = 0.6.0.post1
cryptography = 2.9.2
dateparser = 0.7.6
decorator = 4.3.0
funcsigs = 1.0.2
gevent = 20.9.0
greenlet = 0.4.17
idna = 2.9
importlib-metadata = 1.7.0
inotify-simple = 1.1.1
itsdangerous = 0.24
lock-file = 2.0
lxml = 4.4.3
meld3 = 1.0.2
more-itertools = 5.0.0
netaddr = 0.7.19
pathlib2 = 2.3.5
pbr = 2.0.0
plone.recipe.command = 1.1
prettytable = 0.7.2
psutil = 5.6.3
pluggy = 0.13.1
py = 1.9.0
pyOpenSSL = 19.1.0
pyparsing = 2.2.0
pytz = 2016.10
regex = 2020.9.27
requests = 2.24.0
scandir = 1.10.0
setuptools-dso = 1.7
rubygemsrecipe = 0.3.0
six = 1.12.0
slapos.cookbook = 1.0.167
slapos.core = 1.6.2
......@@ -184,7 +202,13 @@ slapos.toolbox = 0.112
stevedore = 1.21.0
subprocess32 = 3.5.3
unicodecsv = 0.14.1
wcwidth = 0.2.5
wheel = 0.35.1
xml-marshaller = 1.0.2
zdaemon = 4.2.0
zipp = 1.2.0
zodburi = 2.4.0
zope.event = 3.5.2
paramiko = 2.1.3
CacheControl = 0.12.5
msgpack = 0.6.2
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment