Commit 0d06f59a authored by Alain Takoudjou's avatar Alain Takoudjou

Update Release Candidate

parents 20bf35fc 3c56c310
Changes
=======
1.0.75-dev (unreleased)
-----------------------
* Drop ``slapos.recipe:xvfb``, use simple ``slapos.recipe:wrapper`` instead.
* Drop ``slapos.recipe:seleniumrunner`` and ``slapos.recipe:firefox``, they
were not used.
1.0.75 (2018-09-04)
-------------------
......
......@@ -46,17 +46,42 @@
#
# # The list of backends which apache should redirect to.
# "backend-list": [
# # (port, unused, internal_scheme)
# (8000, _, "http://10.0.0.10:8001"),
# (8002, _, "http://10.0.0.10:8003"),
# # (port, unused, internal_scheme, enable_authentication)
# (8000, _, "http://10.0.0.10:8001", True),
# (8002, _, "http://10.0.0.10:8003", False),
# ],
#
# # The mapping of zope paths this apache should redirect to.
# # This is a Zope specific feature.
# # `enable_authentication` has same meaning as for `backend-list`.
# "zope-virtualhost-monster-backend-dict": {
# # {(ip, port): ( enable_authentication, {frontend_path: ( internal_scheme ) }, ) }
# ('[::1]', 8004): (
# True, {
# 'zope-1': 'http://10.0.0.10:8001',
# 'zope-2': 'http://10.0.0.10:8002',
# },
# ),
# },
# }
#
# This sample of `parameter_dict` will make apache listening to :
# - 0.0.0.0:8000 redirecting internaly to http://10.0.0.10:8001
# - [::1]:8000 redirecting internaly to http://10.0.0.10:8001
# - 0.0.0.0:8002 redirecting internaly to http://10.0.0.10:8003
# - [::1]:8002 redirecting internaly to http://10.0.0.10:8003
# This sample of `parameter_dict` will make apache listening to :
# From to `backend-list`:
# - 0.0.0.0:8000 redirecting internaly to http://10.0.0.10:8001 and
# - [::1]:8000 redirecting internaly to http://10.0.0.10:8001
# only accepting requests from clients who provide a valid SSL certificate trusted in `ca-cert`.
# - 0.0.0.0:8002 redirecting internaly to http://10.0.0.10:8003
# - [::1]:8002 redirecting internaly to http://10.0.0.10:8003
# accepting requests from any client.
#
# From zope-virtualhost-monster-backend-dict`:
# - [::1]:8004 with some path based rewrite-rules redirecting to:
# * http://10.0.0.10/8001 when path matches /zope-1(.*)
# * http://10.0.0.10/8002 when path matches /zope-2(.*)
# with some VirtualHostMonster rewrite rules so zope writes URLs with
# [::1]:8004 as server name.
# For more details, refer to
# https://docs.zope.org/zope2/zope2book/VirtualHosting.html#using-virtualhostroot-and-virtualhostbase-together
-#}
LoadModule unixd_module modules/mod_unixd.so
LoadModule access_compat_module modules/mod_access_compat.so
......@@ -153,3 +178,28 @@ Listen {{ ip }}:{{ port }}
RewriteRule ^/(.*) {{ backend }}/$1 [L,P]
</VirtualHost>
{% endfor -%}
{% for (ip, port), (enable_authentication, path_mapping) in parameter_dict.get('zope-virtualhost-monster-backend-dict', {}).items() -%}
Listen {{ ip }}:{{ port }}
<VirtualHost {{ ip }}:{{ port }}>
SSLEngine on
{% if enable_authentication and parameter_dict['ca-cert'] and parameter_dict['crl'] -%}
SSLVerifyClient require
SSLCACertificateFile {{ parameter_dict['ca-cert'] }}
SSLCARevocationCheck chain
SSLCARevocationFile {{ parameter_dict['crl'] }}
LogFormat "%h %l %{REMOTE_USER}i %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\" %D" combined
# We would like to separate the the authentificated logs.
# XXX filename ? is it log-rotated ?
ErrorLog "{{ parameter_dict['log-dir'] }}/apache-service-virtual-host-error.log"
CustomLog "{{ parameter_dict['log-dir'] }}/apache-service-virtual-host-access.log" combined
{% endif -%}
{% for path, backend in path_mapping.items() %}
RewriteRule ^/{{path}}(.*) {{ backend }}/VirtualHostBase/https/{{ ip }}:{{ port }}/VirtualHostRoot/_vh_{{ path }}$1 [L,P]
{% endfor -%}
</VirtualHost>
{% endfor -%}
\ No newline at end of file
......@@ -16,6 +16,7 @@ extends =
../pkgconfig/buildout.cfg
../sqlite3/buildout.cfg
../zlib/buildout.cfg
./buildout.hash.cfg
[apr]
recipe = slapos.recipe.build:download-unpacked
......@@ -196,6 +197,5 @@ make-targets =
[template-apache-backend-conf]
recipe = slapos.recipe.build:download
shared = false
url = ${:_profile_base_location_}/apache-backend.conf.in
md5sum = 8f1f92ad308ab6bad973c790ee583428
url = ${:_profile_base_location_}/${:filename}
mode = 640
# THIS IS NOT A BUILDOUT FILE, despite purposedly using a compatible syntax.
# The only allowed lines here are (regexes):
# - "^#" comments, copied verbatim
# - "^[" section beginings, copied verbatim
# - lines containing an "=" sign which must fit in the following categorie.
# - "^\s*filename\s*=\s*path\s*$" where "path" is relative to this file
# Copied verbatim.
# - "^\s*hashtype\s*=.*" where "hashtype" is one of the values supported
# by the re-generation script.
# Re-generated.
# - other lines are copied verbatim
# Substitution (${...:...}), extension ([buildout] extends = ...) and
# section inheritance (< = ...) are NOT supported (but you should really
# not need these here).
[template-apache-backend-conf]
filename = apache-backend.conf.in
md5sum = 3b430ca726a2707e1b6a2ae41a6c8e21
......@@ -17,5 +17,5 @@ rpath = ${:library-dirs}
[geolite2-country]
recipe = slapos.recipe.build:download-unpacked
url = http://geolite.maxmind.com/download/geoip/database/GeoLite2-Country.tar.gz#${:md5sum}
md5sum = a06d162867715573a919f10f6bf36e48
md5sum = e1389fb31cf83665418b0a83ba8b3147
strip-top-level-dir = true
......@@ -15,8 +15,8 @@ extends =
[groonga]
recipe = slapos.recipe.cmmi
shared = false
url = https://packages.groonga.org/source/groonga/groonga-8.0.5.tar.gz
md5sum = 72ce790646ce561c246ef4e2ea961f79
url = https://packages.groonga.org/source/groonga/groonga-8.0.6.tar.gz
md5sum = 532dab4d5d625942852d021283e974ee
# temporary patch to respect more tokens in natural language mode.
patches =
${:_profile_base_location_}/groonga.patch#9ed02fbe8400402d3eab47eee149978b
......@@ -46,8 +46,8 @@ environment =
[groonga-normalizer-mysql]
recipe = slapos.recipe.cmmi
shared = false
url = http://packages.groonga.org/source/groonga-normalizer-mysql/groonga-normalizer-mysql-1.1.1.tar.gz
md5sum = 5a93c0238da114e5f91db659ffd499b0
url = https://packages.groonga.org/source/groonga-normalizer-mysql/groonga-normalizer-mysql-1.1.3.tar.gz
md5sum = ad30404cb9999a842e98f3902057b152
location = ${groonga:location}
configure-options =
--disable-static
......
......@@ -26,8 +26,8 @@ parts =
[mariadb]
recipe = slapos.recipe.cmmi
url = https://downloads.mariadb.org/f/mariadb-${:version}/source/mariadb-${:version}.tar.gz/from/http%3A//fr.mirror.babylon.network/mariadb/?serve
version = 10.2.16
md5sum = 234156a94f8ea6cff92c4751880b8316
version = 10.2.17
md5sum = 97dac7c5c288dbbbdd97768972daeb2e
patch-options = -p0
patches =
${:_profile_base_location_}/mariadb_10.2.16_create_system_tables__no_test.patch#3fd5f9febabdb42d4b6653969a0194f9
......@@ -79,8 +79,8 @@ post-install =
# mroonga - a storage engine for MySQL. It provides fast fulltext search feature to all MySQL users.
# http://mroonga.github.com/
recipe = slapos.recipe.cmmi
url = https://packages.groonga.org/source/mroonga/mroonga-8.03.tar.gz
md5sum = b394f3dcbff5668ced5d7a82dc648a7e
url = https://packages.groonga.org/source/mroonga/mroonga-8.06.tar.gz
md5sum = 5933363420cb66fcc89e25485072f1b8
pre-configure = set -e
rm -rf fake_mariadb_source
mkdir -p fake_mariadb_source
......
......@@ -17,8 +17,8 @@ parts =
[openssl]
recipe = slapos.recipe.cmmi
shared = true
url = https://www.openssl.org/source/openssl-1.0.2o.tar.gz
md5sum = 44279b8557c3247cbe324e2322ecd114
url = https://www.openssl.org/source/openssl-1.0.2p.tar.gz
md5sum = ac5eb30bf5798aa14b1ae6d0e7da58df
location = @@LOCATION@@
# 'prefix' option to override --openssldir/--prefix (which is useful
# when combined with INSTALL_PREFIX). Used by slapos.package.git/obs
......
......@@ -13,7 +13,7 @@ md5sum = 1bb39745ac23da449019f9f2cb4b0d01
configure-options =
--prefix=${buildout:parts-directory}/${:_buildout_section_name_}
environment =
PATH=${make:location}/bin:%(PATH)s
PATH=${make:location}/bin:${gcc:location}/bin:%(PATH)s
make-target =
install
......@@ -15,8 +15,9 @@ parts =
[w3m]
recipe = slapos.recipe.cmmi
shared = true
md5sum = 1b845a983a50b8dec0169ac48479eacc
url = http://downloads.sourceforge.net/project/w3m/w3m/w3m-0.5.3/w3m-0.5.3.tar.gz
version = v0.5.3+git20180125
md5sum = 199b6ea6f2390bd6675278bcc9f677a9
url = https://salsa.debian.org/debian/w3m/-/archive/${:version}/w3m-${:version}.tar.bz2
configure-options =
--with-gc=${garbage-collector:location}
--with-ssl=${openssl:location}
......@@ -31,15 +32,6 @@ configure-options =
--disable-external-uri-loader
--disable-w3mmailer
patch-options =
-p1
# http://lists.opensuse.org/opensuse-commit/2011-09/msg00222.html
# http://sources.gentoo.org/cgi-bin/viewvc.cgi/gentoo-x86/www-client/w3m/files/w3m-0.5.2-gc72.patch
patches =
${:_profile_base_location_}/w3m.gcc.forward.compat.patch#75422a6f7f671b3a6d9add6724cc0945
${:_profile_base_location_}/w3m-0.5.2-gc72.patch#5a74b6379240c8dd1372efa7926c8bdc
environment =
PATH=${gettext:location}/bin:${patch:location}/bin:${perl:location}/bin:${pkgconfig:location}/bin:%(PATH)s
PKG_CONFIG_PATH=${openssl:location}/lib/pkgconfig:${garbage-collector:location}/lib/pkgconfig
......
diff -Naurb w3m-0.5.2.orig/main.c w3m-0.5.2/main.c
--- w3m-0.5.2.orig/main.c 2007-05-31 10:19:50.000000000 +0900
+++ w3m-0.5.2/main.c 2009-11-09 08:20:02.184953443 +0900
@@ -312,7 +312,11 @@
lock = 0;
}
}
+#if GC_VERSION_MAJOR >= 7 && GC_VERSION_MINOR >= 2
+ else if (orig_GC_warn_proc = GC_get_warn_proc())
+#else
else if (orig_GC_warn_proc)
+#endif
orig_GC_warn_proc(msg, arg);
else
fprintf(stderr, msg, (unsigned long)arg);
@@ -842,7 +846,11 @@
mySignal(SIGPIPE, SigPipe);
#endif
+#if GC_VERSION_MAJOR >= 7 && GC_VERSION_MINOR >= 2
+ GC_set_warn_proc(wrap_GC_warn_proc);
+#else
orig_GC_warn_proc = GC_set_warn_proc(wrap_GC_warn_proc);
+#endif
err_msg = Strnew();
if (load_argc == 0) {
/* no URL specified */
diff --git a/istream.c b/istream.c
index 8967280..d4c788a 100644
--- a/istream.c
+++ b/istream.c
@@ -22,8 +22,8 @@
static void basic_close(int *handle);
static int basic_read(int *handle, char *buf, int len);
-static void file_close(struct file_handle *handle);
-static int file_read(struct file_handle *handle, char *buf, int len);
+static void file_close(struct w3m_file_handle *handle);
+static int file_read(struct w3m_file_handle *handle, char *buf, int len);
static int str_read(Str handle, char *buf, int len);
@@ -114,7 +114,7 @@ newFileStream(FILE * f, void (*closep) ())
stream = New(union input_stream);
init_base_stream(&stream->base, STREAM_BUF_SIZE);
stream->file.type = IST_FILE;
- stream->file.handle = New(struct file_handle);
+ stream->file.handle = New(struct w3m_file_handle);
stream->file.handle->f = f;
if (closep)
stream->file.handle->close = closep;
@@ -658,13 +658,13 @@ basic_read(int *handle, char *buf, int len)
}
static void
-file_close(struct file_handle *handle)
+file_close(struct w3m_file_handle *handle)
{
handle->close(handle->f);
}
static int
-file_read(struct file_handle *handle, char *buf, int len)
+file_read(struct w3m_file_handle *handle, char *buf, int len)
{
return fread(buf, 1, len, handle->f);
}
diff --git a/istream.h b/istream.h
index a220d8b..6d9736d 100644
--- a/istream.h
+++ b/istream.h
@@ -20,7 +20,7 @@ struct stream_buffer {
typedef struct stream_buffer *StreamBuffer;
-struct file_handle {
+struct w3m_file_handle {
FILE *f;
void (*close) ();
};
@@ -53,7 +53,7 @@ struct base_stream {
struct file_stream {
struct stream_buffer stream;
- struct file_handle *handle;
+ struct w3m_file_handle *handle;
char type;
char iseos;
int (*read) ();
......@@ -17,6 +17,7 @@ extends =
../perl-XML-Parser/buildout.cfg
../pkgconfig/buildout.cfg
../zlib/buildout.cfg
./buildout.hash.cfg
parts =
libXdmcp
......@@ -67,7 +68,7 @@ shared = true
url = http://www.x.org/releases/X11R7.7/src/everything/xtrans-1.2.7.tar.bz2
md5sum = 84c66908cf003ad8c272b0eecbdbaee3
patches =
${:_profile_base_location_}/xtrans_tmp_env.patch#37d82a3b6009113023599632117a6855
${:_profile_base_location_}/xtrans_tmp_env.patch#${xtrans_tmp_env.patch:md5sum}
patch-options = -p1
configure-options =
--disable-docs
......@@ -109,7 +110,7 @@ shared = true
url = http://xcb.freedesktop.org/dist/libxcb-1.9.1.tar.bz2
md5sum = ed632cb0dc31b6fbd7ea5c0f931cf5a4
patches =
${:_profile_base_location_}/libxcb_tmp_env.patch#61f39878120ba434a169e24cae2af862
${:_profile_base_location_}/libxcb_tmp_env.patch#${libxcb_tmp_env.patch:md5sum}
patch-options = -p1
configure-options =
--disable-static
......@@ -554,7 +555,7 @@ shared = true
url = http://www.x.org/releases/X11R7.7/src/everything/xorg-server-1.12.2.tar.bz2
md5sum = 791f0323b886abb7954de7f042bb7dc6
patches =
${:_profile_base_location_}/xorg-server_tmp_env.patch#8b60ab8121f0564a681fc00d03101696
${:_profile_base_location_}/xorg-server_tmp_env.patch#${xorg-server_tmp_env.patch:md5sum}
patch-options = -p1
configure-options =
--enable-xvfb
......
# THIS IS NOT A BUILDOUT FILE, despite purposedly using a compatible syntax.
# The only allowed lines here are (regexes):
# - "^#" comments, copied verbatim
# - "^[" section beginings, copied verbatim
# - lines containing an "=" sign which must fit in the following categorie.
# - "^\s*filename\s*=\s*path\s*$" where "path" is relative to this file
# Copied verbatim.
# - "^\s*hashtype\s*=.*" where "hashtype" is one of the values supported
# by the re-generation script.
# Re-generated.
# - other lines are copied verbatim
# Substitution (${...:...}), extension ([buildout] extends = ...) and
# section inheritance (< = ...) are NOT supported (but you should really
# not need these here).
[libxcb_tmp_env.patch]
filename = libxcb_tmp_env.patch
md5sum = 03bc7279f2a37bac7bf426af213123b2
[xorg-server_tmp_env.patch]
filename = xorg-server_tmp_env.patch
md5sum = 8ab5c57af8348787c119c1bafc3b1409
[xtrans_tmp_env.patch]
filename = xtrans_tmp_env.patch
md5sum = 3258214edaba0a52dc0c800f940a0b9f
Store Xorg lock files and sockets in different directories
see xorg-server_tmp_env.patch
--- libxcb-1.7/src/xcb_util.c.back 2012-04-03 13:30:36.000000000 +0200
+++ libxcb-1.7/src/xcb_util.c 2012-04-03 14:47:00.000000000 +0200
@@ -147,11 +147,31 @@
......@@ -30,7 +34,7 @@
- static const char unix_base[] = "/tmp/.X11-unix/X";
- const char *base = unix_base;
+ char *base;
+ base = _xcb_getandappendcompatibleenv("TMPDIR", "/tmp", "/.X11-unix/X");
+ base = _xcb_getandappendcompatibleenv("XORG_LOCK_DIR", "/tmp", "/.X11-unix/X");
size_t filelen;
char *file = NULL;
int actual_filelen;
Store Xorg lock files and sockets in different directories
This patch allow easy enable isolation of multiple Xorg servers and clients
running on same machine, instead of using the convention to store sockets and
lock files in /tmp/ store them in a directory defined in XORG_LOCK_DIR
environment variable.
When this environment variable is not set, fallbacks to the traditional
behavior of using /tmp/ .
--- xorg-server-1.9.3/os/utils.c.old 2012-03-29 18:20:02.000000000 +0200
+++ xorg-server-1.9.3/os/utils.c 2012-03-29 18:39:33.000000000 +0200
@@ -226,7 +226,26 @@ OsSignal(int sig, OsSigHandlerPtr handle
......@@ -24,7 +33,7 @@
+
+}
+
+#define LOCK_DIR getcompatibleenv("TMPDIR", "/tmp")
+#define LOCK_DIR getcompatibleenv("XORG_LOCK_DIR", "/tmp")
#define LOCK_TMP_PREFIX "/.tX"
#define LOCK_PREFIX "/.X"
#define LOCK_SUFFIX "-lock"
Store Xorg lock files and sockets in different directories
see xorg-server_tmp_env.patch
--- xtrans-1.2.6/Xtranssock.c.old 2012-03-29 14:25:45.000000000 +0200
+++ xtrans-1.2.6/Xtranssock.c 2012-03-29 18:40:54.000000000 +0200
@@ -207,29 +207,49 @@ static int TRANS(SocketINETClose) (Xtran
......@@ -27,38 +31,38 @@
#if defined(X11_t)
-#define UNIX_PATH "/tmp/.X11-unix/X"
-#define UNIX_DIR "/tmp/.X11-unix"
+#define UNIX_PATH getandappendcompatibleenv(0, "TMPDIR", "/tmp", "/.X11-unix/X")
+#define UNIX_DIR getandappendcompatibleenv(1, "TMPDIR", "/tmp", "/.X11-unix")
+#define UNIX_PATH getandappendcompatibleenv(0, "XORG_LOCK_DIR", "/tmp", "/.X11-unix/X")
+#define UNIX_DIR getandappendcompatibleenv(1, "XORG_LOCK_DIR", "/tmp", "/.X11-unix")
#endif /* X11_t */
#if defined(XIM_t)
-#define UNIX_PATH "/tmp/.XIM-unix/XIM"
-#define UNIX_DIR "/tmp/.XIM-unix"
+#define UNIX_PATH getandappendcompatibleenv(0, "TMPDIR", "/tmp", "/.XIM-unix/XIM")
+#define UNIX_DIR getandappendcompatibleenv(1, "TMPDIR", "/tmp", "/.XIM-unix")
+#define UNIX_PATH getandappendcompatibleenv(0, "XORG_LOCK_DIR", "/tmp", "/.XIM-unix/XIM")
+#define UNIX_DIR getandappendcompatibleenv(1, "XORG_LOCK_DIR", "/tmp", "/.XIM-unix")
#endif /* XIM_t */
#if defined(FS_t) || defined(FONT_t)
-#define UNIX_PATH "/tmp/.font-unix/fs"
-#define UNIX_DIR "/tmp/.font-unix"
+#define UNIX_PATH getandappendcompatibleenv(0, "TMPDIR", "/tmp", "/.font-unix/fs")
+#define UNIX_DIR getandappendcompatibleenv(1, "TMPDIR", "/tmp", "/.font-unix")
+#define UNIX_PATH getandappendcompatibleenv(0, "XORG_LOCK_DIR", "/tmp", "/.font-unix/fs")
+#define UNIX_DIR getandappendcompatibleenv(1, "XORG_LOCK_DIR", "/tmp", "/.font-unix")
#endif /* FS_t || FONT_t */
#if defined(ICE_t)
-#define UNIX_PATH "/tmp/.ICE-unix/"
-#define UNIX_DIR "/tmp/.ICE-unix"
+#define UNIX_PATH getandappendcompatibleenv(0, "TMPDIR", "/tmp", "/.ICE-unix/")
+#define UNIX_DIR getandappendcompatibleenv(1, "TMPDIR", "/tmp", "/.ICE-unix")
+#define UNIX_PATH getandappendcompatibleenv(0, "XORG_LOCK_DIR", "/tmp", "/.ICE-unix/")
+#define UNIX_DIR getandappendcompatibleenv(1, "XORG_LOCK_DIR", "/tmp", "/.ICE-unix")
#endif /* ICE_t */
#if defined(TEST_t)
-#define UNIX_PATH "/tmp/.Test-unix/test"
-#define UNIX_DIR "/tmp/.Test-unix"
+#define UNIX_PATH getandappendcompatibleenv(0, "TMPDIR", "/tmp", "/.Test-unix/test")
+#define UNIX_DIR getandappendcompatibleenv(1, "TMPDIR", "/tmp", "/.Test-unix")
+#define UNIX_PATH getandappendcompatibleenv(0, "XORG_LOCK_DIR", "/tmp", "/.Test-unix/test")
+#define UNIX_DIR getandappendcompatibleenv(1, "XORG_LOCK_DIR", "/tmp", "/.Test-unix")
#endif
#if defined(LBXPROXY_t)
-#define UNIX_PATH "/tmp/.X11-unix/X"
-#define UNIX_DIR "/tmp/.X11-unix"
+#define UNIX_PATH getandappendcompatibleenv(0, "TMPDIR", "/tmp", "/.X11-unix/X")
+#define UNIX_DIR getandappendcompatibleenv(1, "TMPDIR", "/tmp", "/.X11-unix")
+#define UNIX_PATH getandappendcompatibleenv(0, "XORG_LOCK_DIR", "/tmp", "/.X11-unix/X")
+#define UNIX_DIR getandappendcompatibleenv(1, "XORG_LOCK_DIR", "/tmp", "/.X11-unix")
#endif
......@@ -103,7 +103,6 @@ setup(name=name,
'erp5.promise = slapos.recipe.erp5_promise:Recipe',
'erp5scalabilitytestbed = slapos.recipe.erp5scalabilitytestbed:Recipe',
'erp5testnode = slapos.recipe.erp5testnode:Recipe',
'firefox = slapos.recipe.firefox:Recipe',
'fontconfig = slapos.recipe.fontconfig:Recipe',
'free_port = slapos.recipe.free_port:Recipe',
'generate.mac = slapos.recipe.random:Mac',
......@@ -167,7 +166,6 @@ setup(name=name,
'slapos.recipe.request:RequestOptionalJSONEncoded',
're6stnet.registry = slapos.recipe.re6stnet:Recipe',
'reverseproxy.nginx = slapos.recipe.reverse_proxy_nginx:Recipe',
'seleniumrunner = slapos.recipe.seleniumrunner:Recipe',
'sheepdogtestbed = slapos.recipe.sheepdogtestbed:SheepDogTestBed',
'shell = slapos.recipe.shell:Recipe',
'shellinabox = slapos.recipe.shellinabox:Recipe',
......@@ -194,7 +192,6 @@ setup(name=name,
'userinfo = slapos.recipe.userinfo:Recipe',
'webchecker = slapos.recipe.web_checker:Recipe',
'wrapper = slapos.recipe.wrapper:Recipe',
'xvfb = slapos.recipe.xvfb:Recipe',
'xwiki = slapos.recipe.xwiki:Recipe',
'zabbixagent = slapos.recipe.zabbixagent:Recipe',
'zimbra.kvm = slapos.recipe.zimbra_kvm:Recipe',
......
seleniumrunner
==============
Allows to run selenium tests through browser and xvfb. Posts the results on
Nexedi ERP5.
Parameters
==========
* project : name of the project inside of ERP5 test result instance
* user : username to use in ERP5 instance to test
* password : password to use in ERP5 instance to test
* suite_name : name of test suite to launch
* url : url to portal_test of ERP5 isntance to test
* test_report_instance_url : url of test_result_module to put results
* Example::
<?xml version="1.0" encoding="utf-8"?>
<instance>
<parameter id="project">Vifib</parameter>
<parameter id="user">myuser</parameter>
<parameter id="password">mypassword</parameter>
<parameter id="suite_name">my_zuite</parameter>
<parameter id="url">https://myerp5totest/erp5/portal_tests</parameter>
<parameter id="test_report_instance_url">https://user:passwordwww.myerp5withtestresults.com/test_result_module/</parameter>
</instance>
##############################################################################
#
# Copyright (c) 2012 Vifib SARL and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
#############################################################################
from slapos.recipe.librecipe import GenericBaseRecipe
import sys
class Recipe(GenericBaseRecipe):
def install(self):
prefjs = self.createFile(
self.options['prefsjs-path'],
self.substituteTemplate(self.getTemplateFilename('prefs.js'), {}))
config = {
'firefox_binary': self.options['firefox-path'],
'python_path': sys.executable,
'tmp_path': self.options['tmp-path'],
'pref_path': prefjs,
}
runner = self.createExecutable(
self.options['runner-path'],
self.substituteTemplate(self.getTemplateFilename('firefox_run.in'),
config))
return [runner, prefjs]
#!%(python_path)s
# BEWARE: This file is operated by slapgrid
# BEWARE: It will be overwritten automatically
#
import os
from subprocess import Popen, PIPE
import tempfile
import shutil
import sys
import signal
import subprocess
import time
os.environ['MOZ_NO_REMOTE'] = '1'
os.environ['LC_ALL'] = 'C'
os.environ['MOZ_CRASHREPORTER_DISABLE'] = '1'
os.environ['NO_EM_RESTART'] = '1'
os.environ['TMPDIR'] = "%(tmp_path)s"
os.environ['DISPLAY'] = ':0'
# XXX .Xauthority
profile_path = tempfile.mkdtemp()
shutil.copyfile('%(pref_path)s', os.path.join(profile_path, 'pref.js'))
# os.environ['HOME'] = profile_path
child_pg = None
def sig_handler(signal, frame):
if child_pg is not None:
os.killpg(child_pg, signal.SIGHUP)
os.killpg(child_pg, signal.SIGTERM)
sys.exit(0)
signal.signal(signal.SIGINT, sig_handler)
signal.signal(signal.SIGQUIT, sig_handler)
signal.signal(signal.SIGTERM, sig_handler)
def execute_with_signal_translation(args):
"""Run process as children and translate from SIGTERM to another signal"""
child = subprocess.Popen(args, close_fds=True, preexec_fn=os.setsid)
child_pg = child.pid
try:
while True:
time.sleep(2)
finally:
os.killpg(child_pg, signal.SIGHUP)
os.killpg(child_pg, signal.SIGTERM)
try:
execute_with_signal_translation(
["%(firefox_binary)s", "-no-remote", "-profile", profile_path] + sys.argv[1:])
finally:
shutil.rmtree(profile_path)
// Don't ask if we want to switch default browsers
user_pref("browser.shell.checkDefaultBrowser", false);
user_pref("browser.startup.homepage_override.mstone", "ignore");
// disable application updates
user_pref("app.update.enabled", false)
// disables the 'know your rights' button from displaying on first run
user_pref("browser.rights.3.shown", true);
// Disable pop-up blocking
user_pref("browser.allowpopups", true);
user_pref("dom.disable_open_during_load", false);
user_pref("browser.tabs.warnOnClose", false);
// Configure us as the local proxy
//user_pref("network.proxy.type", 2);
// Disable security warnings
user_pref("security.warn_submit_insecure", false);
user_pref("security.warn_submit_insecure.show_once", false);
user_pref("security.warn_entering_secure", false);
user_pref("security.warn_entering_secure.show_once", false);
user_pref("security.warn_entering_weak", false);
user_pref("security.warn_entering_weak.show_once", false);
user_pref("security.warn_leaving_secure", false);
user_pref("security.warn_leaving_secure.show_once", false);
user_pref("security.warn_viewing_mixed", false);
user_pref("security.warn_viewing_mixed.show_once", false);
// Disable "do you want to remember this password?"
user_pref("signon.rememberSignons", false);
// increase the timeout before warning of unresponsive script
user_pref("dom.max_script_run_time", 120);
// this is required to upload files
// user_pref("capability.principal.codebase.p1.granted", "UniversalFileRead");
// user_pref("signed.applets.codebase_principal_support", true);
// user_pref("capability.principal.codebase.p1.id", "http://");
// user_pref("capability.principal.codebase.p1.subjectName", "");
user_pref("browser.link.open_external", 3);
user_pref("browser.link.open_newwindow", 3);
// disables the request to send performance data from displaying
user_pref("toolkit.telemetry.prompted", 2);
user_pref("toolkit.telemetry.rejected", true);
user_pref("browser.migration.version", 5);
user_pref("extensions.SelectionUI", true);
user_pref("network.cookie.prefsMigrated", true);
user_pref("browser.bookmarks.restore_default_bookmarks", false);
user_pref("browser.places.smartBookmarksVersion", 2);
user_pref("privacy.sanitize.migrateFx3Prefs", true);
##############################################################################
#
# Copyright (c) 2011 Nexedi SARL and Contributors. All Rights Reserved.
# Kazuhiko <kazuhiko@nexedi.com>
# Rafael Monnerat <rafael@nexedi.com>
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsability of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# garantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
# XXX-Cedric : This is an import of
# https://lab.nexedi.com/nexedi/erp5/blob/HEAD/product/ERP5Type/tests/ERP5TypeFunctionalTestCase.py
# Modification of the present file should be ported back to this original file.
import os
import time
import signal
import re
from subprocess import Popen, PIPE
import shutil
#from Products.ERP5Type.tests.ERP5TypeTestCase import ERP5TypeTestCase, \
# _getConversionServerDict
# REGEX FOR ZELENIUM TESTS
TEST_PASS_RE = re.compile('<th[^>]*>Tests passed</th>\n\s*<td[^>]*>([^<]*)')
TEST_FAILURE_RE = re.compile('<th[^>]*>Tests failed</th>\n\s*<td[^>]*>([^<]*)')
IMAGE_RE = re.compile('<img[^>]*?>')
TEST_ERROR_TITLE_RE = re.compile('(?:error.gif.*?>|title status_failed"><td[^>]*>)([^>]*?)</td></tr>', re.S)
TEST_RESULT_RE = re.compile('<div style="padding-top: 10px;">\s*<p>\s*'
'<img.*?</div>\s.*?</div>\s*', re.S)
TEST_ERROR_RESULT_RE = re.compile('.*(?:error.gif|title status_failed).*', re.S)
ZELENIUM_BASE_URL = "%s/portal_tests/%s/core/TestRunner.html?test=../test_suite_html&auto=on&resultsUrl=%s/portal_tests/postResults&__ac_name=%s&__ac_password=%s"
tests_framework_home = os.path.dirname(os.path.abspath(__file__))
# handle 'system global' instance
if tests_framework_home.startswith('/usr/lib'):
real_instance_home = '/var/lib/erp5'
else:
real_instance_home = os.path.sep.join(
tests_framework_home.split(os.path.sep)[:-3])
instance_home = os.path.join(real_instance_home, 'unit_test')
bt5_dir_list = ','.join([
os.path.join(instance_home, 'Products/ERP5/bootstrap'),
os.path.join(instance_home, 'bt5')])
class TimeoutError(Exception):
pass
class Xvfb:
def __init__(self, fbdir, xvfb_location="Xvfb"):
self.display_list = [":%s" % i for i in range(123, 144)]
self.display = None
self.fbdir = fbdir
self.pid = None
self.xvfb_location = xvfb_location
def _runCommand(self, display):
command = [self.xvfb_location, '-fbdir' , self.fbdir, display]
self.process = Popen(" ".join(command),
stdout=PIPE,
stderr=PIPE,
shell=True,
close_fds=True)
def run(self):
for display_try in self.display_list:
lock_filepath = '/tmp/.X%s-lock' % display_try.replace(":", "")
if not os.path.exists(lock_filepath):
self._runCommand(display_try)
self.display = display_try
break
#display = os.environ.get('DISPLAY')
#if display:
# auth = Popen(['xauth', 'list', display], stdout=PIPE).communicate()[0]
# if auth:
# (displayname, protocolname, hexkey) = auth.split()
# Popen(['xauth', 'add', 'localhost/unix:%s' % display, protocolname, hexkey])
print 'Xvfb : %d' % self.process.pid
print 'Take screenshots using xwud -in %s/Xvfb_screen0' % self.fbdir
def quit(self):
if hasattr(self, 'process'):
process_pid = self.process.pid
try:
self.process.terminate()
finally:
if process_pid:
print "Stopping Xvfb on pid: %s" % self.pid
os.kill(process_pid, signal.SIGTERM)
class Browser:
use_xvfb = 1
def __init__(self, profile_dir, host, bin_location=None):
self.bin_location = bin_location
self.profile_dir = profile_dir
self.host = host
self.pid = None
def quit(self):
if self.pid:
os.kill(self.pid, signal.SIGTERM)
def _run(self, url, display):
""" This method should be implemented on a subclass """
raise NotImplementedError
def _setEnviron(self):
pass
def run(self, url, display):
self.clean()
self._setEnviron()
self._setDisplay(display)
self._run(url)
print "Browser %s running on pid: %s" % (self.__class__.__name__, self.pid)
def clean(self):
""" Clean up removing profile dir and recreating it"""
os.system("rm -rf %s" % self.profile_dir)
os.mkdir(self.profile_dir)
def _createFile(self, filename, content):
file_path = os.path.join(self.profile_dir, filename)
with open(file_path, 'w') as f:
f.write(content)
return file_path
def _setDisplay(self, display):
if display is None:
try:
shutil.copy2(os.path.expanduser('~/.Xauthority'), '%s/.Xauthority' % self.profile_dir)
except IOError:
pass
else:
os.environ["DISPLAY"] = display
def _runCommand(self, command_tuple):
print " ".join(list(command_tuple))
self.pid = os.spawnlp(os.P_NOWAIT, *command_tuple)
class Firefox(Browser):
""" Use firefox to open run all the tests"""
def _setEnviron(self):
os.environ['MOZ_NO_REMOTE'] = '1'
os.environ['HOME'] = self.profile_dir
os.environ['LC_ALL'] = 'C'
os.environ["MOZ_CRASHREPORTER_DISABLE"] = "1"
os.environ["NO_EM_RESTART"] = "1"
def _run(self, url):
# Prepare to run
if not self.bin_location:
self.bin_location = "firefox"
self._createFile('prefs.js', self.getPrefJs())
self._runCommand((self.bin_location, "firefox", "-no-remote",
"-profile", self.profile_dir, url))
os.environ['MOZ_NO_REMOTE'] = '0'
def getPrefJs(self):
return """
// Don't ask if we want to switch default browsers
user_pref("browser.shell.checkDefaultBrowser", false);
// Disable pop-up blocking
user_pref("browser.allowpopups", true);
user_pref("dom.disable_open_during_load", false);
// Configure us as the local proxy
//user_pref("network.proxy.type", 2);
// Disable security warnings
user_pref("security.warn_submit_insecure", false);
user_pref("security.warn_submit_insecure.show_once", false);
user_pref("security.warn_entering_secure", false);
user_pref("security.warn_entering_secure.show_once", false);
user_pref("security.warn_entering_weak", false);
user_pref("security.warn_entering_weak.show_once", false);
user_pref("security.warn_leaving_secure", false);
user_pref("security.warn_leaving_secure.show_once", false);
user_pref("security.warn_viewing_mixed", false);
user_pref("security.warn_viewing_mixed.show_once", false);
// Disable "do you want to remember this password?"
user_pref("signon.rememberSignons", false);
// increase the timeout before warning of unresponsive script
user_pref("dom.max_script_run_time", 120);
// this is required to upload files
user_pref("capability.principal.codebase.p1.granted", "UniversalFileRead");
user_pref("signed.applets.codebase_principal_support", true);
user_pref("capability.principal.codebase.p1.id", "http://%s");
user_pref("capability.principal.codebase.p1.subjectName", "");""" % \
(self.host)
class PhantomJS(Browser):
def _createRunJS(self):
run_js = """
var page = new WebPage(),
address;
address = phantom.args[0];
page.open(address, function (status) {
if (status !== 'success') {
console.log('FAIL to load the address');
} else {
console.log('SUCCESS load the address');
}
phantom.exit();
});
"""
return self._createFile('run.js', run_js)
def _run(self, url):
self._runCommand(("phantomjs", "phantomjs", self._createRunJS(), url))
class FunctionalTestRunner:
# There is no test that can take more them 24 hours
timeout = 2.0 * 60 * 60
def __init__(self, host, port, portal, run_only='', use_phanthom=False):
self.instance_home = os.environ['INSTANCE_HOME']
# Such informations should be automatically loaded
self.user = 'ERP5TypeTestCase'
self.password = ''
self.run_only = run_only
profile_dir = os.path.join(self.instance_home, 'profile')
self.portal = portal
if use_phanthom:
self.browser = PhantomJS(profile_dir, host, int(port))
else:
self.browser = Firefox(profile_dir, host, int(port))
def getStatus(self):
transaction.commit()
return self.portal.portal_tests.TestTool_getResults()
def _getTestURL(self):
return ZELENIUM_BASE_URL % (self.portal.portal_url(), self.run_only,
self.portal.portal_url(), self.user, self.password)
def test(self, debug=0):
xvfb = Xvfb(self.instance_home)
try:
start = time.time()
if not debug and self.browser.use_xvfb:
xvfb.run()
self.browser.run(self._getTestURL() , xvfb.display)
while self.getStatus() is None:
time.sleep(10)
if (time.time() - start) > float(self.timeout):
raise TimeoutError("Test took more them %s seconds" % self.timeout)
finally:
self.browser.quit()
xvfb.quit()
def processResult(self):
file_content = self.getStatus().encode("utf-8", "replace")
sucess_amount = TEST_PASS_RE.search(file_content).group(1)
failure_amount = TEST_FAILURE_RE.search(file_content).group(1)
error_title_list = [re.compile('\s+').sub(' ', x).strip()
for x in TEST_ERROR_TITLE_RE.findall(file_content)]
detail = ''
for test_result in TEST_RESULT_RE.findall(file_content):
if TEST_ERROR_RESULT_RE.match(test_result):
detail += test_result
detail = IMAGE_RE.sub('', detail)
if detail:
detail = IMAGE_RE.sub('', detail)
detail = '''<html>
<head>
<style type="text/css">tr.status_failed { background-color:red };</style>
</head>
<body>%s</body>
</html>''' % detail
# When return fix output for handle unicode issues.
return detail, int(sucess_amount), int(failure_amount), error_title_list
class ERP5TypeFunctionalTestCase:
run_only = ""
foreground = 0
use_phanthom = False
def getTitle(self):
return "Zelenium"
def afterSetUp(self):
# create browser_id_manager
if not "browser_id_manager" in self.portal.objectIds():
self.portal.manage_addProduct['Sessions'].constructBrowserIdManager()
transaction.commit()
self.setSystemPreference()
self.portal.portal_tests.TestTool_cleanUpTestResults()
self.stepTic()
def setSystemPreference(self):
conversion_dict = _getConversionServerDict()
self.portal.Zuite_setPreference(
working_copy_list=bt5_dir_list,
conversion_server_hostname=conversion_dict['hostname'],
conversion_server_port=conversion_dict['port']
)
# XXX Memcached is missing
# XXX Persistent cache setup is missing
def testFunctionalTestRunner(self):
# first of all, abort to get rid of the mysql participation inn this
# transaction
self.portal._p_jar.sync()
self.runner = FunctionalTestRunner(self.serverhost, self.serverport,
self.portal, self.run_only, self.use_phanthom)
self.runner.test(debug=self.foreground)
detail, success, failure, error_title_list = self.runner.processResult()
self.logMessage("-" * 79)
total = success + failure
self.logMessage("%s Functional Tests %s Tests, %s Failures" % \
(self.getTitle(), total, failure))
self.logMessage("-" * 79)
self.logMessage(detail)
self.logMessage("-" * 79)
self.assertEquals([], error_title_list)
##############################################################################
#
# Copyright (c) 2010 Vifib SARL and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
#############################################################################
from slapos.recipe.librecipe import GenericBaseRecipe
class Recipe(GenericBaseRecipe):
def install(self):
return self.createPythonScript(
self.options['runner-path'],
__name__+'.testrunner.run',
(self.options['suite-url'],
self.options['report-url'],
self.options['report-project'],
self.options['browser']))
##############################################################################
#
# Copyright (c) 2010 Vifib SARL and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
#############################################################################
from __future__ import absolute_import
import re
import urlparse
import urllib
import httplib
import mimetools
from random import randint
import tempfile
import os
import stat
import zipfile
import mimetypes
import datetime
TB_SEP = "============================================================="\
"========="
# REGEX FOR ZELENIUM TESTS
TEST_PASS_RE = re.compile('<th[^>]*>Tests passed</th>\n\s*<td[^>]*>([^<]*)')
TEST_FAILURE_RE = re.compile('<th[^>]*>Tests failed</th>\n\s*<td[^>]*>([^<]*)')
IMAGE_RE = re.compile('<img[^>]*?>')
TEST_ERROR_TITLE_RE = re.compile('(?:error.gif.*?>|title status_failed"><td[^>]*>)([^>]*?)</td></tr>', re.S)
TEST_RESULT_RE = re.compile('<div style="padding-top: 10px;">\s*<p>\s*'
'<img.*?</div>\s.*?</div>\s*', re.S)
DURATION_RE = re.compile('<th[^>]*>Elapsed time \(sec\)</th>\n\s*<td[^>]*>([^<]*)')
TEST_ERROR_RESULT_RE = re.compile('.*(?:error.gif|title status_failed).*', re.S)
def get_content_type(f):
return mimetypes.guess_type(f.name)[0] or 'application/octet-stream'
class ConnectionHelper:
def __init__(self, url):
self.conn = urlparse.urlparse(url)
if self.conn.scheme == 'http':
connection_type = httplib.HTTPConnection
if self.conn.port is None:
self.port = 80
else:
connection_type = httplib.HTTPSConnection
if self.conn.port is None:
self.port = 443
self.connection_type = connection_type
def _connect(self):
self.connection = self.connection_type(self.conn.hostname + ':' +
str(self.conn.port or self.port))
def POST(self, path, parameter_dict, file_list=None):
self._connect()
parameter_dict.update(__ac_name=self.conn.username,
__ac_password=self.conn.password)
header_dict = {'Content-type': "application/x-www-form-urlencoded"}
if file_list is None:
body = urllib.urlencode(parameter_dict)
else:
boundary = mimetools.choose_boundary()
header_dict['Content-type'] = 'multipart/form-data; boundary=%s' % (
boundary,)
body = ''
for k, v in parameter_dict.iteritems():
body += '--%s\r\n' % boundary
body += 'Content-Disposition: form-data; name="%s"\r\n' % k
body += '\r\n'
body += '%s\r\n' % v
for name, filename in file_list:
f = open(filename, 'r')
body += '--%s\r\n' % boundary
body += 'Content-Disposition: form-data; name="%s"; filename="%s"\r\n'\
% (name, name)
body += 'Content-Type: %s\r\n' % get_content_type(f)
body += 'Content-Length: %d\r\n' % os.fstat(f.fileno())[stat.ST_SIZE]
body += '\r\n'
body += f.read()
f.close()
body += '\r\n'
self.connection.request("POST", self.conn.path + '/' + path,
body, header_dict)
self.response = self.connection.getresponse()
class ERP5TestReportHandler:
def __init__(self, url, suite_name):
# random test id
self.test_id = "%s-%X" % (
("%s" % datetime.date.today()).replace("-", ""),
randint(1, 10000000000000000),
)
self.connection_helper = ConnectionHelper(url)
self.suite_name = suite_name
def reportStart(self):
# report that test is running
print 'Starting test with id %s' % self.test_id
self.connection_helper.POST('TestResultModule_reportRunning', dict(
test_suite=self.suite_name,
test_report_id=self.test_id,
))
def reportFinished(self, out_file):
# make file parsable by erp5_test_results
out_file, success, failure, duration = self.processResult(out_file)
# XXX-Cedric : make correct display in test_result_module
tempcmd = tempfile.mkstemp()[1]
tempcmd2 = tempfile.mkstemp()[1]
tempout = tempfile.mkstemp()[1]
templog = tempfile.mkstemp()[1]
tl = open(templog, 'w')
tl.write(TB_SEP + '\n')
tl.write(out_file)
tl.write("----------------------------------------------------------------------\n")
tl.write('Ran 1 test in %.2fs\n' % duration)
if success:
tl.write('OK\n')
else:
tl.write('FAILED (failures=1)\n')
tl.write(TB_SEP + '\n')
tl.close()
open(tempcmd, 'w').write(""" %s""" % self.suite_name)
# create nice zip archive
tempzip = tempfile.mkstemp()[1]
# XXX-Cedric : support multiple tests
zip = zipfile.ZipFile(tempzip, 'w')
zip.write(tempout, '%s/001/stdout' % self.suite_name)
zip.write(templog, '%s/001/stderr' % self.suite_name)
zip.write(tempcmd, '%s/001/cmdline' % self.suite_name)
zip.close()
os.unlink(templog)
os.unlink(tempcmd)
os.unlink(tempout)
os.unlink(tempcmd2)
# post it to ERP5
self.connection_helper.POST('TestResultModule_reportCompleted',
dict(test_report_id=self.test_id), file_list=[('filepath', tempzip)])
os.unlink(tempzip)
def processResult(self, out_file):
file_content = out_file
sucess_amount = TEST_PASS_RE.search(file_content).group(1)
failure_amount = TEST_FAILURE_RE.search(file_content).group(1)
error_title_list = [re.compile('\s+').sub(' ', x).strip()
for x in TEST_ERROR_TITLE_RE.findall(file_content)]
duration = DURATION_RE.search(file_content).group(1)
detail = ''
for test_result in TEST_RESULT_RE.findall(file_content):
if TEST_ERROR_RESULT_RE.match(test_result):
detail += test_result
detail = IMAGE_RE.sub('', detail)
if detail:
detail = IMAGE_RE.sub('', detail)
detail = '''<html>
<head>
<style type="text/css">tr.status_failed { background-color:red };</style>
</head>
<body>%s</body>
</html>''' % detail
return detail, int(sucess_amount), int(failure_amount), float(duration)
#!/bin/sh
# BEWARE: This file is operated by slapgrid
# BEWARE: It will be overwritten automatically
%(xvfb_binary) :%(display) -screen %(display) 1024x768x24
\ No newline at end of file
##############################################################################
#
# Copyright (c) 2010 Vifib SARL and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
#############################################################################
from datetime import datetime
from erp5functionaltestreporthandler import ERP5TestReportHandler
from ERP5TypeFunctionalTestCase import TimeoutError
from time import sleep
import time
import os
import urllib2
import urlparse
from subprocess import Popen, PIPE
import signal
def run(suite_url, report_url, project, browser_binary):
suite_parsed = urlparse.urlparse(suite_url)
config = {
'suite_name': suite_parsed.path.split('/')[-1],
'base_url': "%s://%s%s" % (suite_parsed.scheme, suite_parsed.hostname,
'/'.join(suite_parsed.path.split('/')[:-1])),
'user': suite_parsed.username,
'password': suite_parsed.password,
}
test_url = assembleTestUrl(config['base_url'], config['suite_name'],
config['user'], config['password'])
# There is no test that can take more them 24 hours
timeout = 2.0 * 60 * 60
while True:
erp5_report = ERP5TestReportHandler(report_url,
project + '@' + config['suite_name'])
try:
try:
start = time.time()
print("Running test on: %s" % test_url)
process = Popen('%s "%s"' % (browser_binary, test_url),
stdout=PIPE,
stderr=PIPE,
shell=True,
close_fds=True)
erp5_report.reportStart()
while not isTestFinished(config['base_url']):
time.sleep(10)
print("Test not finished yet.")
if (time.time() - start) > float(timeout):
raise TimeoutError("Test took more than %s seconds" % timeout)
except TimeoutError:
continue
finally:
if process.pid:
os.kill(process.pid, signal.SIGTERM)
print("Test has finished and Firefox has been killed.")
erp5_report.reportFinished(getStatus(config['base_url']).encode("utf-8",
"replace"))
print("Test finished and report sent, sleeping.")
except urllib2.URLError, urlError:
print "Error: %s" % urlError.msg
sleep(3600)
def openUrl(url):
# Send Accept-Charset headers to activate the UnicodeConflictResolver
# (imitating firefox 3.5.9 here)
headers = { 'Accept-Charset' : 'ISO-8859-1,utf-8;q=0.7,*;q=0.7' }
request = urllib2.Request(url, headers=headers)
# Try to use long timeout, this is needed when there is many
# activities runing
try:
f = urllib2.urlopen(request, timeout=3600*4)
except TypeError:
f = urllib2.urlopen(request)
file_content = f.read()
f.close()
return file_content
def isTestFinished(url):
"""Fetch latest report. If report has been created less than 60 seconds ago,
it must be the current one.
Return true if test is finished, else return false.
"""
latest_report = openUrl('%s/portal_tests/TestTool_getLatestReportId/' % url)
if latest_report is '':
return False
latest_report_date = latest_report[7:]
time_delta = datetime.now() - \
datetime.strptime(latest_report_date, '%Y%m%d_%H%M%S' )
if time_delta.days is not 0:
return False
if time_delta.seconds < 120:
return True
return False
def getStatus(url):
try:
# Try 5 times.
for i in range(5):
try:
status = openUrl('%s/portal_tests/TestTool_getResults/' % (url))
break
except urllib2.URLError, urlError:
if i is 4: raise
print("Warning : %s while getting status" % urlError.msg)
except urllib2.HTTPError, e:
if e.msg == "No Content":
status = ""
else:
raise
return status
def assembleTestUrl(base_url, suite_name, user, password):
"""
Create the full url to the testrunner
"""
test_url = "%s/%s/core/TestRunner.html?test=../test_suite_html&"\
"resultsUrl=%s/postResults&auto=on&__ac_name=%s&__ac_password=%s" % (
base_url, suite_name, base_url, user, password)
return test_url
##############################################################################
#
# Copyright (c) 2012 Vifib SARL and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
#############################################################################
from slapos.recipe.librecipe import GenericBaseRecipe
class Recipe(GenericBaseRecipe):
def install(self):
config = {
'xvfb_binary': self.options['xvfb-path'],
'shell_path': self.options['shell-path'],
'fbdir_path': self.options['fbdir-path'],
'tmp_path': self.options['tmp-path'],
}
xvfb_path = self.createExecutable(
self.options['runner-path'],
self.substituteTemplate(self.getTemplateFilename('xvfb_run.in'),
config))
result = [xvfb_path]
# Allow to take screenshot if needed
if ('xwd-path' in self.options) and ('xwd-hook-path' in self.options):
config['xwd_binary'] = self.options['xwd-path']
result.append(self.createExecutable(
self.options['xwd-hook-path'],
self.substituteTemplate(self.getTemplateFilename('xwd_run.in'),
config)))
return result
#!%(shell_path)s
# BEWARE: This file is operated by slapgrid
# BEWARE: It will be overwritten automatically
export TMPDIR=%(tmp_path)s
exec %(xvfb_binary)s -screen 0 1024x768x24 -fbdir %(fbdir_path)s
#!%(shell_path)s
# BEWARE: This file is operated by slapgrid
# BEWARE: It will be overwritten automatically
export DISPLAY=:0
export TMPDIR=%(tmp_path)s
exec %(xwd_binary)s -root -out $1
......@@ -161,10 +161,14 @@ caddy_custom_https
~~~~~~~~~~~~~~~~~~
Raw Caddy configuration in python template format (i.e. write "%%" for one "%") for the slave listening to the https port. Its content will be templatified in order to access functionalities such as cache access, ssl certificates... The list is available above.
*Note*: The system will reject slaves which does not pass validation of caddy configuration, despite them being in ``-frontend-authorized-slave-string``, as otherwise this will lead to the whole frontend to fail.
caddy_custom_http
~~~~~~~~~~~~~~~~~
Raw Caddy configuration in python template format (i.e. write "%%" for one "%") for the slave listening to the http port. Its content will be templatified in order to access functionalities such as cache access, ssl certificates... The list is available above
*Note*: The system will reject slaves which does not pass validation of caddy configuration, despite them being in ``-frontend-authorized-slave-string``, as otherwise this will lead to the whole frontend to fail.
url
~~~
Necessary to activate cache. ``url`` of backend to use.
......
Generally things to be done with ``caddy-frontend``:
* tests: drop APACHE switches, as it is not taken care anymore
* tests: add assertion with results of promises in etc/promise for each partition
* check the whole frontend slave snippet with ``caddy -validate`` during buildout run, and reject if does not pass validation
* check that all options from ``instance-slave-caddy-input-schema.json`` are safe to be used
* ``apache-ca-certificate`` shall be merged with ``apache-certificate``
* ``apache-ca-certificate`` shall be appended to ``apache-certificate`` if not already there
......@@ -58,6 +58,7 @@ Generally things to be done with ``caddy-frontend``:
* reduce the time of configuration validation (in ``instance-apache-frontend.cfg.in`` sections ``[configtest]``, ``[caddy-configuration]``, ``[nginx-configuration]``), as it is not scalable on frontend with 2000+ slaves (takes few minutes instead of few, < 5, seconds), issue posted `upstream <https://github.com/mholt/caddy/issues/2220>`_
* drop ``6tunnel`` and use ``bind`` in Caddy configuration, as soon as multiple binds will be possible, tracked in upstream `bind: support multiple values <https://github.com/mholt/caddy/pull/2128>`_ and `ipv6: does not bind on ipv4 and ipv6 for sites that resolve to both <https://github.com/mholt/caddy/issues/864>`_
* use caddy-frontend in `standalone style playbooks <https://lab.nexedi.com/nexedi/slapos.package/tree/master/playbook/roles/standalone-shared>`_
* in ``templates/apache-custom-slave-list.cfg.in`` avoid repetetive ``part_list.append`` and use macro like in ERP5 SR (cf `Vincent's comment <https://lab.nexedi.com/nexedi/slapos/merge_requests/373#note_64362>`_)
Things which can't be implemented:
......
......@@ -14,7 +14,7 @@
# not need these here).
[template]
filename = instance.cfg.in
md5sum = 2c2d051825c76384732faad9c407c538
md5sum = bce721468b4c16294404cac8b88356c0
[template-common]
filename = instance-common.cfg.in
......@@ -22,15 +22,15 @@ md5sum = c801b7f9f11f0965677c22e6bbe9281b
[template-apache-frontend]
filename = instance-apache-frontend.cfg.in
md5sum = 5cfc4dd69c2800906d6648c626023cf0
md5sum = 750e2b1c922bf14511a3bc8a42468b1b
[template-apache-replicate]
filename = instance-apache-replicate.cfg.in
md5sum = 8d34141a9cd1e51462aba845c7bea85b
md5sum = e5e537052c533c6d6f1c2197428f77fd
[template-slave-list]
filename = templates/apache-custom-slave-list.cfg.in
md5sum = 8f29aaf247a6b8354292c78abe7a5ad6
md5sum = ed1c086f0548a908661b294e845dc008
[template-slave-configuration]
filename = templates/custom-virtualhost.conf.in
......@@ -46,23 +46,27 @@ md5sum = 7c987ad75fcce6f5b925c7696ff41971
[template-custom-slave-list]
filename = templates/apache-custom-slave-list.cfg.in
md5sum = 8f29aaf247a6b8354292c78abe7a5ad6
md5sum = ed1c086f0548a908661b294e845dc008
[caddy-backend-url-validator]
filename = templates/caddy-backend-url-validator.in
md5sum = 0979a03476e86bf038516c9565dadc17
[caddy-custom-http-validator]
filename = templates/caddy-custom-http-validator.in
md5sum = a264208e960cdcd25ef27ed8cf730240
[template-not-found-html]
filename = templates/notfound.html
md5sum = f20d6c3d2d94fb685f8d26dfca1e822b
[template-default-slave-virtualhost]
filename = templates/default-virtualhost.conf.in
md5sum = d9269cc085752e09f4acce37a18e160c
md5sum = 9e00b6d981b9f93a486ef06a47345ebd
[template-cached-slave-virtualhost]
filename = templates/cached-virtualhost.conf.in
md5sum = f149ab15334d7d15d8c525f02fc4d968
md5sum = 7cbcadc295860821ac9d3aaa3cca72c5
[template-log-access]
filename = templates/template-log-access.conf.in
......@@ -103,3 +107,11 @@ md5sum = ebe5d3d19923eb812a40019cb11276d8
[template-caddy-graceful-script]
filename = templates/caddy-graceful-script.sh.in
md5sum = 455f8765a3afd39fb78562fb9e326c42
[caddyprofiledeps-setup]
filename = setup.py
md5sum = d9b6476bb0b36cf463fddb00d41dfbaa
[caddyprofiledeps-dummy]
filename = caddyprofiledummy.py
md5sum = 38792c2dceae38ab411592ec36fff6a8
class Recipe(object):
def __init__(self, *args, **kwargs):
pass
def install(self):
return []
def update(self):
return self.install()
......@@ -27,6 +27,36 @@ parts +=
npm-modules
proxy-by-url
http-proxy
caddyprofiledeps
[caddyprofiledeps-setup]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/setup.py
[caddyprofiledeps-dummy]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/caddyprofiledummy.py
[caddyprofiledeps-prepare]
recipe = plone.recipe.command
stop-on-error = True
location = ${buildout:parts-directory}/${:_buildout_section_name_}
update-command = ${:command}
command =
rm -fr ${:location} &&
mkdir -p ${:location} &&
cp ${caddyprofiledeps-setup:target} ${:location}/ &&
cp ${caddyprofiledeps-dummy:target} ${:location}/
[caddyprofiledeps-develop]
recipe = zc.recipe.egg:develop
setup = ${caddyprofiledeps-prepare:location}
[caddyprofiledeps]
depends = ${caddyprofiledeps-develop:recipe}
recipe = zc.recipe.egg
eggs =
caddyprofiledeps
# Extent extra-eggs.
[extra-eggs]
......@@ -93,6 +123,7 @@ context =
key template_caddy_replicate template-caddy-replicate:target
key template_replicate_publish_slave_information template-replicate-publish-slave-information:target
key caddy_backend_url_validator caddy-backend-url-validator:output
key caddy_custom_http_validator caddy-custom-http-validator:output
section template_frontend_parameter_dict template-frontend-parameter-section
......@@ -108,6 +139,13 @@ filename = caddy-backend-url-validator.in
output = ${buildout:directory}/caddy-backend-url-validator
mode = 0750
[caddy-custom-http-validator]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/templates/${:filename}
filename = caddy-custom-http-validator.in
output = ${buildout:directory}/caddy-custom-http-validator
mode = 0750
[template-caddy-replicate]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/instance-apache-replicate.cfg.in
......
......@@ -124,7 +124,7 @@ template = {{ parameter_dict['template_slave_list'] }}
filename = custom-personal-instance-slave-list.cfg
extensions = jinja2.ext.do
slave_instance_list = {{ dumps(instance_parameter['slave-instance-list']) }}
extra_slave_instance_list = ${configuration:extra_slave_instance_list}
extra_slave_instance_list = {{ dumps(instance_parameter.get('configuration.extra_slave_instance_list')) }}
local_ipv4 = {{ dumps(instance_parameter['ipv4-random']) }}
local_ipv6 = {{ dumps(instance_parameter['ipv6-random']) }}
software_type = single-custom-personal
......
......@@ -9,6 +9,7 @@ context =
raw common_profile {{ common_profile }}
${:extra-context}
{% set popen = functools_module.partial(subprocess_module.Popen, stdout=subprocess_module.PIPE, stderr=subprocess_module.STDOUT, stdin=subprocess_module.PIPE) %}
{% set part_list = [] %}
{% set single_type_key = 'single-' %}
{% if slap_software_type == "replicate" %}
......@@ -66,25 +67,64 @@ context =
{% set authorized_slave_string = slapparameter_dict.pop('-frontend-authorized-slave-string', '') %}
{% set authorized_slave_list = [] %}
{% set rejected_slave_list = [] %}
{% set used_host_list = [] %}
{% for slave in slave_instance_list %}
{# BBB: apache_custom_https AND apache_custom_http #}
{% if not ((slave.has_key('caddy_custom_http') or slave.has_key('apache_custom_http') or slave.has_key('caddy_custom_https') or slave.has_key('apache_custom_https')) and not slave.get('slave_reference') in authorized_slave_string) %}
{% set slave_ok = True %}
{% if slave.get('url') %}
{% if subprocess_module.call([caddy_backend_url_validator, slave['url']]) == 1 %}
{% set slave_ok = False %}
{% set slave_dict = {'state': True} %}
{# BBB: apache_custom_https AND apache_custom_http #}
{% set custom_domain = slave.get('custom_domain') %}
{% if custom_domain and custom_domain in used_host_list %}
{% do slave_dict.__setitem__('state', False) %}
{% else %}
{% do used_host_list.append(custom_domain) %}
{% endif %}
{% if slave.get('server-alias') %}
{% for slave_alias in slave['server-alias'].split() %}
{% if not validators.domain(slave_alias) %}
{% do slave_dict.__setitem__('state', False) %}
{% else %}
{% if slave_alias in used_host_list %}
{% do slave_dict.__setitem__('state', False) %}
{% else %}
{% do used_host_list.append(slave_alias) %}
{% endif %}
{% endif %}
{% endif %}
{% if slave.get('https-url') %}
{% if subprocess_module.call([caddy_backend_url_validator, slave['https-url']]) == 1 %}
{% set slave_ok = False %}
{% endfor %}
{% endif %}
{% for key in ['caddy_custom_http', 'caddy_custom_https', 'apache_custom_http', 'apache_custom_https'] %}
{% if slave.get(key) %}
{% if not slave.get('slave_reference') in authorized_slave_string %}
{% do slave_dict.__setitem__('state', False) %}
{% elif subprocess_module.call([caddy_custom_http_validator, slave[key]]) == 1 %}
{% do slave_dict.__setitem__('state', False) %}
{% endif %}
{% endif %}
{% if slave_ok %}
{% do authorized_slave_list.append(slave) %}
{% else %}
{% do rejected_slave_list.append(slave.get('slave_reference')) %}
{% endfor %} {# for key in ['caddy_custom_http', 'caddy_custom_https', 'apache_custom_http', 'apache_custom_https'] #}
{% if slave.get('url') %}
{% if subprocess_module.call([caddy_backend_url_validator, slave['url']]) == 1 %}
{% do slave_dict.__setitem__('state', False) %}
{% endif %}
{% endif %}
{% if slave.get('https-url') %}
{% if subprocess_module.call([caddy_backend_url_validator, slave['https-url']]) == 1 %}
{% do slave_dict.__setitem__('state', False) %}
{% endif %}
{% endif %}
{% if slave.get('ssl_key') and slave.get('ssl_crt') %}
{% set key_popen = popen([openssl, 'rsa', '-noout', '-modulus']) %}
{% set crt_popen = popen([openssl, 'x509', '-noout', '-modulus']) %}
{% set key_modulus = key_popen.communicate(slave['ssl_key'])[0] | trim %}
{% set crt_modulus = crt_popen.communicate(slave['ssl_crt'])[0] | trim %}
{% if not key_modulus or key_modulus != crt_modulus %}
{% do slave_dict.__setitem__('state', False) %}
{% endif %}
{% endif %}
{% if slave.get('custom_domain') %}
{% if not validators.domain(slave['custom_domain']) %}
{% do slave_dict.__setitem__('state', False) %}
{% endif %}
{% endif %}
{% if slave_dict['state'] %}
{% do authorized_slave_list.append(slave) %}
{% else %}
{% do rejected_slave_list.append(slave.get('slave_reference')) %}
{% endif %}
......@@ -95,11 +135,10 @@ monitor-httpd-port = {{ slapparameter_dict.get('monitor-httpd-port', '8196') }}
[replicate]
<= slap-connection
recipe = slapos.cookbook:requestoptional
recipe = slapos.cookbook:requestoptional.serialised
config-monitor-cors-domains = {{ slapparameter_dict.get('monitor-cors-domains', 'monitor.app.officejs.com') }}
config-monitor-username = ${monitor-instance-parameter:username}
config-monitor-password = ${monitor-htpasswd:passwd}
config-monitor-httpd-port = {{ slapparameter_dict.get('monitor-httpd-port', '8196') }}
{% set frontend_software_url_key = "-frontend-software-release-url" %}
{% if slapparameter_dict.has_key(frontend_software_url_key) %}
......@@ -121,7 +160,9 @@ state = {{ frontend_request.get('state') }}
{% do slave_configuration_dict.update(frontend_request.get('config')) %}
{% do slave_configuration_dict.__setitem__(slave_list_name, json_module.dumps(authorized_slave_list)) %}
{% do slave_configuration_dict.__setitem__("frontend-name", frontend_request.get('name')) %}
config-_ = {{ json_module.dumps(slave_configuration_dict) }}
{%- for config_key, config_value in slave_configuration_dict.iteritems() %}
config-{{ config_key }} = {{ dumps(config_value) }}
{% endfor -%}
{% if frontend_request.get('sla') %}
{% for parameter, value in frontend_request.get('sla').iteritems() %}
sla-{{ parameter }} = {{ value }}
......
......@@ -5,6 +5,9 @@ parts =
dynamic-template-caddy-replicate
switch-softwaretype
[caddyprofiledeps]
recipe = caddyprofiledeps
[jinja2-template-base]
recipe = slapos.recipe.template:jinja2
rendered = ${buildout:directory}/${:filename}
......@@ -41,12 +44,18 @@ extra-context =
[dynamic-template-caddy-replicate]
< = jinja2-template-base
depends = ${caddyprofiledeps:recipe}
template = {{ template_caddy_replicate }}
filename = instance-caddy-replicate.cfg
extensions = jinja2.ext.do
openssl = {{ template_frontend_parameter_dict['openssl'] ~ '/bin/openssl' }}
extra-context =
import subprocess_module subprocess
import functools_module functools
import validators validators
key openssl :openssl
raw caddy_backend_url_validator {{ caddy_backend_url_validator }}
raw caddy_custom_http_validator {{ caddy_custom_http_validator }}
raw template_publish_slave_information {{ template_replicate_publish_slave_information }}
# Must match the key id in [switch-softwaretype] which uses this section.
raw software_type RootSoftwareInstance-default-custom-personal-replicate
......@@ -81,7 +90,6 @@ configuration.apache-key =
configuration.apache-certificate =
configuration.apache-ca-certificate =
configuration.open-port = 80 443
configuration.extra_slave_instance_list =
configuration.disk-cache-size = 8G
configuration.ram-cache-size = 1G
configuration.trafficserver-autoconf-port = 8083
......@@ -91,4 +99,4 @@ configuration.enable-http2-by-default = true
configuration.enable-quic = false
configuration.mpm-graceful-shutdown-timeout = 5
configuration.monitor-httpd-port = 8072
configuration.frontend-name =
\ No newline at end of file
configuration.frontend-name =
# The caddyprofiledeps egg allows to set dependecies of the Caddy profiles
# which are enabled during the instance run, thanks to using caddyprofiledeps
# recipe
from setuptools import setup
setup(
name='caddyprofiledeps',
install_requires=[
'validators',
],
entry_points={
'zc.buildout': [
'default = caddyprofiledummy:Recipe',
]
}
)
......@@ -2,6 +2,7 @@
extends = common.cfg
[versions]
validators = 0.12.2
PyRSS2Gen = 1.1
apache-libcloud = 0.19.0
cns.recipe.symlink = 0.2.3
......
......@@ -169,25 +169,27 @@ value = {{ dumps(slave_instance.get(cert_name)) }}
{% set cert_file = '/'.join([custom_ssl_directory, cert_title.replace('-','.')]) %}
{% set key_file = '/'.join([custom_ssl_directory, key_title.replace('-','.')]) %}
{% do part_list.append(cert_title) %}
{% do part_list.append(key_title) %}
{% do slave_parameter_dict.__setitem__("ssl_crt", cert_file) %}
{% do slave_parameter_dict.__setitem__("ssl_key", key_file) %}
{% do slave_instance.__setitem__('path_to_ssl_crt', cert_file) %}
{% do slave_instance.__setitem__('path_to_ssl_key', key_file) %}
[{{cert_title}}]
recipe = slapos.cookbook:certificate_authority.request
#openssl-binary = ${openssl:location}/bin/openssl
requests-directory = ${cadirectory:requests}
ca-private = ${cadirectory:private}
ca-certs = ${cadirectory:certs}
ca-newcerts = ${cadirectory:newcerts}
ca-crl = ${cadirectory:crl}
key-file = {{ key_file }}
cert-file = {{ cert_file }}
[{{key_title}}]
< = jinja2-template-base
template = {{ empty_template }}
rendered = {{ key_file }}
key-content = {{ dumps(slave_instance.get('ssl_key')) }}
cert-content = {{ dumps(slave_instance.get('ssl_crt')) }}
extra-context =
key content :key-content
[{{cert_title}}]
< = jinja2-template-base
template = {{ empty_template }}
rendered = {{ cert_file }}
cert-content = {{ dumps(slave_instance.get('ssl_crt')) }}
extra-context =
key content :cert-content
{% endif %}
{# ########################################## #}
......@@ -257,7 +259,7 @@ wrapper-path = {{ promise_directory }}/${:filename}
{% do part_list.append(monitor_ipv6_section_title) %}
[{{ monitor_ipv6_section_title }}]
recipe = slapos.cookbook:wrapper
command-line = {{ bin_directory }}/is-icmp-packet-lost -a {{monitor_ipv6_test}}
command-line = {{ dumps(bin_directory ~ '/is-icmp-packet-lost -a ' ~ monitor_ipv6_test) }}
filename = {{ monitor_ipv6_section_title }}
wrapper-path = {{ promise_directory }}/${:filename}
{% endif %}
......@@ -268,19 +270,24 @@ wrapper-path = {{ promise_directory }}/${:filename}
{% do part_list.append(monitor_ipv4_section_title) %}
[{{ monitor_ipv4_section_title }}]
recipe = slapos.cookbook:wrapper
command-line = {{ bin_directory }}/is-icmp-packet-lost -4 -a {{monitor_ipv4_test}}
command-line = {{ dumps(bin_directory ~ '/is-icmp-packet-lost -4 -a ' ~ monitor_ipv4_test) }}
filename = {{ monitor_ipv4_section_title }}
wrapper-path = {{ promise_directory }}/${:filename}
{% endif %}
{% set re6st_optimal_test = slave_instance.get('re6st-optimal-test', '') %}
{% if re6st_optimal_test %}
{% set re6st_ipv6 = None %}
{% set re6st_ipv4 = None %}
{% if ',' in re6st_optimal_test %}
{% set re6st_ipv6, re6st_ipv4 = re6st_optimal_test.split(",") %}
{% endif %}
{% if re6st_ipv6 and re6st_ipv4 %}
{% set re6st_optimal_test_section_title = 'check-%s-re6st-optimal-test' % slave_instance.get('slave_reference') %}
{% do part_list.append(re6st_optimal_test_section_title) %}
[{{ re6st_optimal_test_section_title }}]
recipe = slapos.cookbook:wrapper
command-line = {{ bin_directory }}/check-re6st-optimal-status -4 {{re6st_ipv4}} -6 {{re6st_ipv6}}
command-line = {{ dumps(bin_directory ~ '/check-re6st-optimal-status -4 ' ~ re6st_ipv4 ~ ' -6 ' ~ re6st_ipv6) }}
filename = {{ re6st_optimal_test_section_title }}
wrapper-path = {{ promise_directory }}/${:filename}
{% endif %}
......@@ -387,7 +394,7 @@ extra-context =
{# Publish information for the instance #}
[publish-caddy-information]
recipe = slapos.cookbook:publish
recipe = slapos.cookbook:publish.serialised
public-ipv4 = {{ public_ipv4 }}
private-ipv4 = {{ local_ipv4 }}
{% if extra_slave_instance_list %}
......
{%- set TRUE_VALUES = ['y', 'yes', '1', 'true'] %}
{%- set server_alias_list = slave_parameter.get('server-alias', '').split() %}
{%- set ssl_proxy_verify = ('' ~ slave_parameter.get('ssl-proxy-verify', '')).lower() in TRUE_VALUES %}
{%- set host_list = [slave_parameter.get('custom_domain')] + server_alias_list %}
{%- set host_list = [] %}
{%- for host in [slave_parameter.get('custom_domain')] + server_alias_list %}
{%- if host not in host_list %}
{%- do host_list.append(host) %}
{%- endif %}
{%- endfor %}
{%- set http_backend_host_list = [] %}
{%- set https_backend_host_list = [] %}
{%- for host in host_list %}
......
#!${dash:location}/bin/dash
config="$1"
echo -e $config | ${caddy:output} -conf stdin -validate > /dev/null 2>&1
......@@ -12,7 +12,12 @@
{%- set disabled_cookie_list = slave_parameter.get('disabled-cookie-list', '').split() %}
{%- set https_only = ('' ~ slave_parameter.get('https-only', '')).lower() in TRUE_VALUES %}
{%- set slave_type = slave_parameter.get('type', '') %}
{%- set host_list = [slave_parameter.get('custom_domain')] + server_alias_list %}
{%- set host_list = [] %}
{%- for host in [slave_parameter.get('custom_domain')] + server_alias_list %}
{%- if host not in host_list %}
{%- do host_list.append(host) %}
{%- endif %}
{%- endfor %}
{%- set backend_url = slave_parameter.get('https-url', slave_parameter.get('url', '')) %}
{%- set http_host_list = [] %}
{%- set https_host_list = [] %}
......@@ -20,6 +25,7 @@
{%- do http_host_list.append('http://%s:%s' % (host, slave_parameter['http_port'] )) %}
{%- do https_host_list.append('https://%s:%s' % (host, slave_parameter['https_port'] )) %}
{%- endfor %} {#- for host in host_list #}
{%- set default_path = slave_parameter.get('default-path', '') | urlencode %}
# SSL enabled hosts
{{ https_host_list|join(', ') }} {
......@@ -86,15 +92,15 @@
{%- endif %} {#- if ssl_proxy_verify #}
} {# proxy #}
{%- endfor %} {#- for (proxy_name, proxy_comment) in proxy_append_list #}
{%- if 'default-path' in slave_parameter %}
{%- if default_path %}
redir 301 {
if {path} is /
/ {scheme}://{host}/{{ slave_parameter.get('default-path') }}
/ {scheme}://{host}/{{ default_path }}
} {# redir #}
{%- endif %} {#- if 'default-path' in slave_parameter #}
{%- endif %} {#- if default_path #}
rewrite {
regexp (.*)
to /VirtualHostBase/{scheme}%2F%2F{hostonly}:{{ slave_parameter.get('virtualhostroot-https-port', '443') }}%2F{{ slave_parameter.get('path', '') }}%2FVirtualHostRoot/{1}
to /VirtualHostBase/{scheme}%2F%2F{hostonly}:{{ slave_parameter.get('virtualhostroot-https-port', '443') | int }}%2F{{ slave_parameter.get('path', '') }}%2FVirtualHostRoot/{1}
} {# rewrite #}
{%- elif slave_type == 'redirect' and backend_url %} {#- if slave_type == 'zope' and backend_url #}
# Redirect configuration
......@@ -103,12 +109,12 @@
} {# redir #}
{%- else %} {#- if slave_type == 'zope' and backend_url #}
# Default configuration
{%- if 'default-path' in slave_parameter %}
{%- if default_path %}
redir 301 {
if {path} is /
/ {scheme}://{host}/{{ slave_parameter.get('default-path') }}
/ {scheme}://{host}/{{ default_path }}
} {# redir #}
{%- endif %} {#- if 'default-path' in slave_parameter #}
{%- endif %} {#- if default_path #}
{%- if backend_url %}
{%- for (proxy_name, proxy_comment) in proxy_append_list %}
......@@ -208,24 +214,24 @@
{%- endif %} {#- if ssl_proxy_verify #}
} {# proxy #}
{%- endfor %} {#- for (proxy_name, proxy_comment) in proxy_append_list #}
{%- if 'default-path' in slave_parameter %}
{%- if default_path %}
redir 301 {
if {path} is /
/ {scheme}://{host}/{{ slave_parameter.get('default-path') }}
/ {scheme}://{host}/{{ default_path }}
} {# redir #}
{%- endif %} {#- if 'default-path' in slave_parameter #}
{%- endif %} {#- if default_path #}
rewrite {
regexp (.*)
to /VirtualHostBase/{scheme}%2F%2F{hostonly}:{{ slave_parameter.get('virtualhostroot-http-port', '80') }}%2F{{ slave_parameter.get('path', '') }}%2FVirtualHostRoot/{1}
to /VirtualHostBase/{scheme}%2F%2F{hostonly}:{{ slave_parameter.get('virtualhostroot-http-port', '80') | int }}%2F{{ slave_parameter.get('path', '') }}%2FVirtualHostRoot/{1}
} {# rewrite #}
{%- else %} {#- if https_only #}
# Default configuration
{%- if 'default-path' in slave_parameter %}
{%- if default_path %}
redir 301 {
if {path} is /
/ {scheme}://{host}/{{ slave_parameter.get('default-path') }}
/ {scheme}://{host}/{{ default_path }}
} {# redir #}
{%- endif %} {#- if 'default-path' in slave_parameter #}
{%- endif %} {#- if default_path #}
{%- if slave_parameter.get('url', '') %}
{%- for (proxy_name, proxy_comment) in proxy_append_list %}
# {{ proxy_comment }}
......
......@@ -3004,3 +3004,482 @@ class TestQuicEnabled(SlaveHttpFrontendTestCase, TestDataMixin):
result_http.headers['Set-Cookie'],
'secured=value;secure, nonsecured=value'
)
class TestSlaveBadParameters(SlaveHttpFrontendTestCase, TestDataMixin):
@classmethod
def getInstanceParameterDict(cls):
return {
'domain': 'example.com',
'nginx-domain': 'nginx.example.com',
'public-ipv4': LOCAL_IPV4,
'apache-certificate': open('wildcard.example.com.crt').read(),
'apache-key': open('wildcard.example.com.key').read(),
'-frontend-authorized-slave-string': '_caddy_custom_http_s-reject',
'port': HTTPS_PORT,
'plain_http_port': HTTP_PORT,
'nginx_port': NGINX_HTTPS_PORT,
'plain_nginx_port': NGINX_HTTP_PORT,
'monitor-httpd-port': MONITOR_HTTPD_PORT,
'-frontend-config-1-monitor-httpd-port': MONITOR_F1_HTTPD_PORT,
'mpm-graceful-shutdown-timeout': 2,
}
@classmethod
def getSlaveParameterDictDict(cls):
return {
'caddy_custom_http_s-reject': {
'caddy_custom_https': """DestroyCaddyHttps
For sure
This shall not be valid
https://www.google.com {}""",
'caddy_custom_http': """DestroyCaddyHttp
For sure
This shall not be valid
https://www.google.com {}""",
},
're6st-optimal-test-nocomma': {
're6st-optimal-test': 'nocomma',
},
're6st-optimal-test-unsafe': {
're6st-optimal-test':
'new\nline;rm -fr ~;,new\line\n[s${esection:eoption}',
},
'custom_domain-unsafe': {
'custom_domain': '${section:option} afterspace\nafternewline',
},
'server-alias-unsafe': {
'server-alias': '${section:option} afterspace',
},
'server-alias-same': {
'url': cls.backend_url,
'server-alias': 'serveraliassame.example.com',
},
'virtualhostroot-http-port-unsafe': {
'type': 'zope',
'url': cls.backend_url,
'virtualhostroot-http-port': '${section:option}',
},
'virtualhostroot-https-port-unsafe': {
'type': 'zope',
'url': cls.backend_url,
'virtualhostroot-https-port': '${section:option}',
},
'default-path-unsafe': {
'type': 'zope',
'url': cls.backend_url,
'default-path': '${section:option}\nn"\newline\n}\n}proxy\n/slashed',
},
'monitor-ipv4-test-unsafe': {
'monitor-ipv4-test': '${section:option}\nafternewline ipv4',
},
'monitor-ipv6-test-unsafe': {
'monitor-ipv6-test': '${section:option}\nafternewline ipv6',
},
'ssl_key-ssl_crt-unsafe': {
'ssl_key': '${section:option}ssl_keyunsafe\nunsafe',
'ssl_crt': '${section:option}ssl_crtunsafe\nunsafe',
},
}
def test_master_partition_state(self):
parameter_dict = self.computer_partition.getConnectionParameterDict()
self.assertKeyWithPop('monitor-setup-url', parameter_dict)
expected_parameter_dict = {
'monitor-base-url': None,
'domain': 'example.com',
'accepted-slave-amount': '8',
'rejected-slave-amount': '4',
'slave-amount': '12',
'rejected-slave-list':
'["_caddy_custom_http_s-reject", "_ssl_key-ssl_crt-unsafe", '
'"_custom_domain-unsafe", "_server-alias-unsafe"]'}
self.assertEqual(
expected_parameter_dict,
parameter_dict
)
def test_server_alias_same(self):
parameter_dict = self.slave_connection_parameter_dict_dict[
'server-alias-same']
self.assertLogAccessUrlWithPop(parameter_dict, 'server-alias-same')
self.assertEqual(
parameter_dict,
{
'domain': 'serveraliassame.example.com',
'replication_number': '1',
'url': 'http://serveraliassame.example.com',
'site_url': 'http://serveraliassame.example.com',
'secure_access': 'https://serveraliassame.example.com',
'public-ipv4': LOCAL_IPV4,
}
)
result = self.fakeHTTPSResult(
parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path')
self.assertEqual(
der2pem(result.peercert),
open('wildcard.example.com.crt').read())
self.assertEqualResultJson(result, 'Path', '/test-path')
def test_re6st_optimal_test_unsafe(self):
parameter_dict = self.slave_connection_parameter_dict_dict[
're6st-optimal-test-unsafe']
self.assertLogAccessUrlWithPop(parameter_dict, 're6st-optimal-test-unsafe')
self.assertEqual(
parameter_dict,
{
'domain': 're6stoptimaltestunsafe.example.com',
'replication_number': '1',
'url': 'http://re6stoptimaltestunsafe.example.com',
'site_url': 'http://re6stoptimaltestunsafe.example.com',
'secure_access': 'https://re6stoptimaltestunsafe.example.com',
'public-ipv4': LOCAL_IPV4,
}
)
result = self.fakeHTTPSResult(
parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path')
self.assertEqual(
der2pem(result.peercert),
open('wildcard.example.com.crt').read())
self.assertEqual(result.status_code, no_backend_response_code)
# rewrite SR/bin/is-icmp-packet-lost
open(
os.path.join(
self.software_path, 'bin', 'check-re6st-optimal-status'), 'w'
).write('echo "$@"')
# call the monitor for this partition
monitor_file = glob.glob(
os.path.join(
self.instance_path, '*', 'etc', 'monitor-promise',
'check-_re6st-optimal-test-unsafe-re6st-optimal-test'))[0]
# Note: The result is a bit differnt from the request (newlines stripped),
# but good enough to prove, that ${esection:eoption} has been
# correctly passed to the script.
self.assertEqual(
'-4 newline [s${esection:eoption} -6 new line;rm -fr ~;',
subprocess.check_output(monitor_file).strip()
)
def test_re6st_optimal_test_nocomma(self):
parameter_dict = self.slave_connection_parameter_dict_dict[
're6st-optimal-test-nocomma']
self.assertLogAccessUrlWithPop(
parameter_dict, 're6st-optimal-test-nocomma')
self.assertEqual(
parameter_dict,
{
'domain': 're6stoptimaltestnocomma.example.com',
'replication_number': '1',
'url': 'http://re6stoptimaltestnocomma.example.com',
'site_url': 'http://re6stoptimaltestnocomma.example.com',
'secure_access': 'https://re6stoptimaltestnocomma.example.com',
'public-ipv4': LOCAL_IPV4,
}
)
result = self.fakeHTTPSResult(
parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path')
self.assertEqual(
der2pem(result.peercert),
open('wildcard.example.com.crt').read())
self.assertEqual(result.status_code, no_backend_response_code)
# assert that there is no nocomma file
monitor_file_list = glob.glob(
os.path.join(
self.instance_path, '*', 'etc', 'monitor-promise',
'check-_re6st-optimal-test-nocomma-re6st-optimal-test'))
self.assertEqual(
[],
monitor_file_list
)
def test_custom_domain_unsafe(self):
parameter_dict = self.slave_connection_parameter_dict_dict[
'custom_domain-unsafe']
self.assertEqual(
parameter_dict,
{}
)
def test_server_alias_unsafe(self):
parameter_dict = self.slave_connection_parameter_dict_dict[
'server-alias-unsafe']
self.assertEqual(
parameter_dict,
{}
)
def test_virtualhostroot_http_port_unsafe(self):
parameter_dict = self.slave_connection_parameter_dict_dict[
'virtualhostroot-http-port-unsafe']
self.assertLogAccessUrlWithPop(
parameter_dict, 'virtualhostroot-http-port-unsafe')
self.assertEqual(
parameter_dict,
{
'domain': 'virtualhostroothttpportunsafe.example.com',
'replication_number': '1',
'url': 'http://virtualhostroothttpportunsafe.example.com',
'site_url': 'http://virtualhostroothttpportunsafe.example.com',
'secure_access':
'https://virtualhostroothttpportunsafe.example.com',
'public-ipv4': LOCAL_IPV4,
}
)
result = self.fakeHTTPResult(
parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path')
self.assertEqualResultJson(
result,
'Path',
'/VirtualHostBase/http//virtualhostroothttpportunsafe'
'.example.com:0//VirtualHostRoot/test-path'
)
def test_virtualhostroot_https_port_unsafe(self):
parameter_dict = self.slave_connection_parameter_dict_dict[
'virtualhostroot-https-port-unsafe']
self.assertLogAccessUrlWithPop(
parameter_dict, 'virtualhostroot-https-port-unsafe')
self.assertEqual(
parameter_dict,
{
'domain': 'virtualhostroothttpsportunsafe.example.com',
'replication_number': '1',
'url': 'http://virtualhostroothttpsportunsafe.example.com',
'site_url': 'http://virtualhostroothttpsportunsafe.example.com',
'secure_access':
'https://virtualhostroothttpsportunsafe.example.com',
'public-ipv4': LOCAL_IPV4,
}
)
result = self.fakeHTTPSResult(
parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path')
self.assertEqual(
der2pem(result.peercert),
open('wildcard.example.com.crt').read())
self.assertEqualResultJson(
result,
'Path',
'/VirtualHostBase/https//virtualhostroothttpsportunsafe'
'.example.com:0//VirtualHostRoot/test-path'
)
def default_path_unsafe(self):
parameter_dict = self.slave_connection_parameter_dict_dict[
'default-path-unsafe']
self.assertLogAccessUrlWithPop(parameter_dict, 'default-path-unsafe')
self.assertEqual(
parameter_dict,
{
'domain': 'defaultpathunsafe.example.com',
'replication_number': '1',
'url': 'http://defaultpathunsafe.example.com',
'site_url': 'http://defaultpathunsafe.example.com',
'secure_access': 'https://defaultpathunsafe.example.com',
'public-ipv4': LOCAL_IPV4,
}
)
result = self.fakeHTTPSResult(
parameter_dict['domain'], parameter_dict['public-ipv4'], '')
self.assertEqual(
der2pem(result.peercert),
open('wildcard.example.com.crt').read())
self.assertEqual(
result.headers['Location'],
'https://defaultpathunsafe.example.com:%s/%%24%%7Bsection%%3Aoption%%7D'
'%%0An%%22%%0Aewline%%0A%%7D%%0A%%7Dproxy%%0A/slashed' % (HTTPS_PORT,)
)
def test_monitor_ipv4_test_unsafe(self):
parameter_dict = self.slave_connection_parameter_dict_dict[
'monitor-ipv4-test-unsafe']
self.assertLogAccessUrlWithPop(parameter_dict, 'monitor-ipv4-test-unsafe')
self.assertEqual(
parameter_dict,
{
'domain': 'monitoripv4testunsafe.example.com',
'replication_number': '1',
'url': 'http://monitoripv4testunsafe.example.com',
'site_url': 'http://monitoripv4testunsafe.example.com',
'secure_access': 'https://monitoripv4testunsafe.example.com',
'public-ipv4': LOCAL_IPV4,
}
)
result = self.fakeHTTPSResult(
parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path')
self.assertEqual(
der2pem(result.peercert),
open('wildcard.example.com.crt').read())
self.assertEqual(result.status_code, no_backend_response_code)
result_http = self.fakeHTTPResult(
parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path')
self.assertEqual(result_http.status_code, no_backend_response_code)
# rewrite SR/bin/is-icmp-packet-lost
open(
os.path.join(self.software_path, 'bin', 'is-icmp-packet-lost'), 'w'
).write('echo "$@"')
# call the monitor for this partition
monitor_file = glob.glob(
os.path.join(
self.instance_path, '*', 'etc', 'monitor-promise',
'check-_monitor-ipv4-test-unsafe-ipv4-packet-list-test'))[0]
self.assertEqual(
'-4 -a ${section:option} afternewline ipv4',
subprocess.check_output(monitor_file).strip()
)
def test_monitor_ipv6_test_unsafe(self):
parameter_dict = self.slave_connection_parameter_dict_dict[
'monitor-ipv6-test-unsafe']
self.assertLogAccessUrlWithPop(parameter_dict, 'monitor-ipv6-test-unsafe')
self.assertEqual(
parameter_dict,
{
'domain': 'monitoripv6testunsafe.example.com',
'replication_number': '1',
'url': 'http://monitoripv6testunsafe.example.com',
'site_url': 'http://monitoripv6testunsafe.example.com',
'secure_access': 'https://monitoripv6testunsafe.example.com',
'public-ipv4': LOCAL_IPV4,
}
)
result = self.fakeHTTPSResult(
parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path')
self.assertEqual(
der2pem(result.peercert),
open('wildcard.example.com.crt').read())
self.assertEqual(result.status_code, no_backend_response_code)
result_http = self.fakeHTTPResult(
parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path')
self.assertEqual(result_http.status_code, no_backend_response_code)
# rewrite SR/bin/is-icmp-packet-lost
open(
os.path.join(self.software_path, 'bin', 'is-icmp-packet-lost'), 'w'
).write('echo "$@"')
# call the monitor for this partition
monitor_file = glob.glob(
os.path.join(
self.instance_path, '*', 'etc', 'monitor-promise',
'check-_monitor-ipv6-test-unsafe-ipv6-packet-list-test'))[0]
self.assertEqual(
'-a ${section:option} afternewline ipv6',
subprocess.check_output(monitor_file).strip()
)
def test_ssl_key_ssl_crt_unsafe(self):
parameter_dict = self.slave_connection_parameter_dict_dict[
'ssl_key-ssl_crt-unsafe']
self.assertEqual(
parameter_dict,
{}
)
def test_caddy_custom_http_s_reject(self):
parameter_dict = self.slave_connection_parameter_dict_dict[
'caddy_custom_http_s-reject']
self.assertEqual(
parameter_dict,
{}
)
class TestDuplicateSiteKeyProtection(SlaveHttpFrontendTestCase, TestDataMixin):
@classmethod
def getInstanceParameterDict(cls):
return {
'domain': 'example.com',
'nginx-domain': 'nginx.example.com',
'public-ipv4': LOCAL_IPV4,
'apache-certificate': open('wildcard.example.com.crt').read(),
'apache-key': open('wildcard.example.com.key').read(),
'-frontend-authorized-slave-string': '_caddy_custom_http_s-reject',
'port': HTTPS_PORT,
'plain_http_port': HTTP_PORT,
'nginx_port': NGINX_HTTPS_PORT,
'plain_nginx_port': NGINX_HTTP_PORT,
'monitor-httpd-port': MONITOR_HTTPD_PORT,
'-frontend-config-1-monitor-httpd-port': MONITOR_F1_HTTPD_PORT,
'mpm-graceful-shutdown-timeout': 2,
}
@classmethod
def getSlaveParameterDictDict(cls):
return {
'site_1': {
'custom_domain': 'duplicate.example.com',
},
'site_2': {
'custom_domain': 'duplicate.example.com',
},
'site_3': {
'server-alias': 'duplicate.example.com',
},
'site_4': {
'custom_domain': 'duplicate.example.com',
'server-alias': 'duplicate.example.com',
},
}
def test_master_partition_state(self):
parameter_dict = self.computer_partition.getConnectionParameterDict()
self.assertKeyWithPop('monitor-setup-url', parameter_dict)
expected_parameter_dict = {
'monitor-base-url': None,
'domain': 'example.com',
'accepted-slave-amount': '1',
'rejected-slave-amount': '3',
'slave-amount': '4',
'rejected-slave-list': '["_site_3", "_site_1", "_site_4"]'}
self.assertEqual(
expected_parameter_dict,
parameter_dict
)
def test_site_2(self):
parameter_dict = self.slave_connection_parameter_dict_dict[
'site_2']
self.assertLogAccessUrlWithPop(parameter_dict, 'site_2')
self.assertEqual(
parameter_dict,
{
'domain': 'duplicate.example.com',
'replication_number': '1',
'url': 'http://duplicate.example.com',
'site_url': 'http://duplicate.example.com',
'secure_access': 'https://duplicate.example.com',
'public-ipv4': LOCAL_IPV4,
}
)
TestDuplicateSiteKeyProtection-0/var/log/monitor-httpd-error.log
TestDuplicateSiteKeyProtection-1/var/log/frontend-access.log
TestDuplicateSiteKeyProtection-1/var/log/frontend-error.log
TestDuplicateSiteKeyProtection-1/var/log/httpd/_site_2_access_log
TestDuplicateSiteKeyProtection-1/var/log/httpd/_site_2_error_log
TestDuplicateSiteKeyProtection-1/var/log/monitor-httpd-error.log
TestDuplicateSiteKeyProtection-1/var/log/nginx-access.log
TestDuplicateSiteKeyProtection-1/var/log/nginx-error.log
TestDuplicateSiteKeyProtection-1/var/log/trafficserver/manager.log
TestDuplicateSiteKeyProtection-1/var/log/trafficserver/traffic.out
\ No newline at end of file
TestDuplicateSiteKeyProtection-0/var/run/monitor-httpd.pid
TestDuplicateSiteKeyProtection-0/var/run/monitor/monitor-bootstrap.pid
TestDuplicateSiteKeyProtection-1/var/run/caddy_configuration.signature
TestDuplicateSiteKeyProtection-1/var/run/httpd.pid
TestDuplicateSiteKeyProtection-1/var/run/monitor-httpd.pid
TestDuplicateSiteKeyProtection-1/var/run/monitor/monitor-bootstrap.pid
TestDuplicateSiteKeyProtection-1/var/run/ncaddy_configuration.signature
TestDuplicateSiteKeyProtection-1/var/run/nginx.pid
\ No newline at end of file
TestDuplicateSiteKeyProtection-1/etc/monitor-promise/check-_site_2-error-log-last-day
TestDuplicateSiteKeyProtection-1/etc/monitor-promise/check-_site_2-error-log-last-hour
\ No newline at end of file
TestDuplicateSiteKeyProtection-0/etc/plugin/__init__.py
TestDuplicateSiteKeyProtection-0/etc/plugin/buildout-TestDuplicateSiteKeyProtection-0-status.py
TestDuplicateSiteKeyProtection-0/etc/plugin/check-free-disk-space.py
TestDuplicateSiteKeyProtection-0/etc/plugin/monitor-bootstrap-status.py
TestDuplicateSiteKeyProtection-1/etc/plugin/__init__.py
TestDuplicateSiteKeyProtection-1/etc/plugin/buildout-TestDuplicateSiteKeyProtection-1-status.py
TestDuplicateSiteKeyProtection-1/etc/plugin/check-free-disk-space.py
TestDuplicateSiteKeyProtection-1/etc/plugin/monitor-bootstrap-status.py
\ No newline at end of file
TestDuplicateSiteKeyProtection-0/etc/promise/monitor-http-frontend
TestDuplicateSiteKeyProtection-0/etc/promise/monitor-httpd-listening-on-tcp
TestDuplicateSiteKeyProtection-0/etc/promise/promise-monitor-httpd-is-process-older-than-dependency-set
TestDuplicateSiteKeyProtection-1/etc/promise/caddy-frontend-is-running-actual-software-release
TestDuplicateSiteKeyProtection-1/etc/promise/caddy_cached
TestDuplicateSiteKeyProtection-1/etc/promise/caddy_frontend_ipv4_http
TestDuplicateSiteKeyProtection-1/etc/promise/caddy_frontend_ipv4_https
TestDuplicateSiteKeyProtection-1/etc/promise/caddy_frontend_ipv6_http
TestDuplicateSiteKeyProtection-1/etc/promise/caddy_frontend_ipv6_https
TestDuplicateSiteKeyProtection-1/etc/promise/caddy_ssl_cached
TestDuplicateSiteKeyProtection-1/etc/promise/frontend-caddy-configuration-promise
TestDuplicateSiteKeyProtection-1/etc/promise/monitor-http-frontend
TestDuplicateSiteKeyProtection-1/etc/promise/monitor-httpd-listening-on-tcp
TestDuplicateSiteKeyProtection-1/etc/promise/nginx-configuration-promise
TestDuplicateSiteKeyProtection-1/etc/promise/nginx_frontend_ipv4_http
TestDuplicateSiteKeyProtection-1/etc/promise/nginx_frontend_ipv4_https
TestDuplicateSiteKeyProtection-1/etc/promise/nginx_frontend_ipv6_http
TestDuplicateSiteKeyProtection-1/etc/promise/nginx_frontend_ipv6_https
TestDuplicateSiteKeyProtection-1/etc/promise/promise-monitor-httpd-is-process-older-than-dependency-set
TestDuplicateSiteKeyProtection-1/etc/promise/promise-nginx-is-process-older-than-dependency-set
TestDuplicateSiteKeyProtection-1/etc/promise/re6st-connectivity
TestDuplicateSiteKeyProtection-1/etc/promise/trafficserver-cache-availability
TestDuplicateSiteKeyProtection-1/etc/promise/trafficserver-port-listening
\ No newline at end of file
TestDuplicateSiteKeyProtection-0:bootstrap-monitor EXITED
TestDuplicateSiteKeyProtection-0:certificate_authority-on-watch RUNNING
TestDuplicateSiteKeyProtection-0:crond RUNNING
TestDuplicateSiteKeyProtection-0:monitor-httpd-graceful EXITED
TestDuplicateSiteKeyProtection-0:monitor-httpd-on-watch RUNNING
TestDuplicateSiteKeyProtection-1:6tunnel-11080-on-watch RUNNING
TestDuplicateSiteKeyProtection-1:6tunnel-11443-on-watch RUNNING
TestDuplicateSiteKeyProtection-1:6tunnel-12080-on-watch RUNNING
TestDuplicateSiteKeyProtection-1:6tunnel-12443-on-watch RUNNING
TestDuplicateSiteKeyProtection-1:6tunnel-26011-on-watch RUNNING
TestDuplicateSiteKeyProtection-1:6tunnel-26012-on-watch RUNNING
TestDuplicateSiteKeyProtection-1:bootstrap-monitor EXITED
TestDuplicateSiteKeyProtection-1:certificate_authority-on-watch RUNNING
TestDuplicateSiteKeyProtection-1:crond-on-watch RUNNING
TestDuplicateSiteKeyProtection-1:frontend-caddy-safe-graceful EXITED
TestDuplicateSiteKeyProtection-1:frontend-nginx-safe-graceful EXITED
TestDuplicateSiteKeyProtection-1:frontend_caddy-on-watch RUNNING
TestDuplicateSiteKeyProtection-1:frontend_nginx-on-watch RUNNING
TestDuplicateSiteKeyProtection-1:monitor-httpd-graceful EXITED
TestDuplicateSiteKeyProtection-1:monitor-httpd-on-watch RUNNING
TestDuplicateSiteKeyProtection-1:trafficserver-on-watch RUNNING
TestDuplicateSiteKeyProtection-1:trafficserver-reload EXITED
watchdog:watchdog RUNNING
\ No newline at end of file
TestSlaveBadParameters-0/var/log/monitor-httpd-error.log
TestSlaveBadParameters-1/var/log/frontend-access.log
TestSlaveBadParameters-1/var/log/frontend-error.log
TestSlaveBadParameters-1/var/log/httpd/_default-path-unsafe_access_log
TestSlaveBadParameters-1/var/log/httpd/_default-path-unsafe_error_log
TestSlaveBadParameters-1/var/log/httpd/_monitor-ipv4-test-unsafe_access_log
TestSlaveBadParameters-1/var/log/httpd/_monitor-ipv4-test-unsafe_error_log
TestSlaveBadParameters-1/var/log/httpd/_monitor-ipv6-test-unsafe_access_log
TestSlaveBadParameters-1/var/log/httpd/_monitor-ipv6-test-unsafe_error_log
TestSlaveBadParameters-1/var/log/httpd/_re6st-optimal-test-nocomma_access_log
TestSlaveBadParameters-1/var/log/httpd/_re6st-optimal-test-nocomma_error_log
TestSlaveBadParameters-1/var/log/httpd/_re6st-optimal-test-unsafe_access_log
TestSlaveBadParameters-1/var/log/httpd/_re6st-optimal-test-unsafe_error_log
TestSlaveBadParameters-1/var/log/httpd/_server-alias-same_access_log
TestSlaveBadParameters-1/var/log/httpd/_server-alias-same_error_log
TestSlaveBadParameters-1/var/log/httpd/_virtualhostroot-http-port-unsafe_access_log
TestSlaveBadParameters-1/var/log/httpd/_virtualhostroot-http-port-unsafe_error_log
TestSlaveBadParameters-1/var/log/httpd/_virtualhostroot-https-port-unsafe_access_log
TestSlaveBadParameters-1/var/log/httpd/_virtualhostroot-https-port-unsafe_error_log
TestSlaveBadParameters-1/var/log/monitor-httpd-error.log
TestSlaveBadParameters-1/var/log/nginx-access.log
TestSlaveBadParameters-1/var/log/nginx-error.log
TestSlaveBadParameters-1/var/log/trafficserver/manager.log
TestSlaveBadParameters-1/var/log/trafficserver/traffic.out
\ No newline at end of file
TestSlaveBadParameters-0/var/run/monitor-httpd.pid
TestSlaveBadParameters-0/var/run/monitor/monitor-bootstrap.pid
TestSlaveBadParameters-1/var/run/caddy_configuration.signature
TestSlaveBadParameters-1/var/run/httpd.pid
TestSlaveBadParameters-1/var/run/monitor-httpd.pid
TestSlaveBadParameters-1/var/run/monitor/monitor-bootstrap.pid
TestSlaveBadParameters-1/var/run/ncaddy_configuration.signature
TestSlaveBadParameters-1/var/run/nginx.pid
\ No newline at end of file
TestSlaveBadParameters-1/etc/monitor-promise/check-_default-path-unsafe-error-log-last-day
TestSlaveBadParameters-1/etc/monitor-promise/check-_default-path-unsafe-error-log-last-hour
TestSlaveBadParameters-1/etc/monitor-promise/check-_monitor-ipv4-test-unsafe-error-log-last-day
TestSlaveBadParameters-1/etc/monitor-promise/check-_monitor-ipv4-test-unsafe-error-log-last-hour
TestSlaveBadParameters-1/etc/monitor-promise/check-_monitor-ipv4-test-unsafe-ipv4-packet-list-test
TestSlaveBadParameters-1/etc/monitor-promise/check-_monitor-ipv6-test-unsafe-error-log-last-day
TestSlaveBadParameters-1/etc/monitor-promise/check-_monitor-ipv6-test-unsafe-error-log-last-hour
TestSlaveBadParameters-1/etc/monitor-promise/check-_monitor-ipv6-test-unsafe-ipv6-packet-list-test
TestSlaveBadParameters-1/etc/monitor-promise/check-_re6st-optimal-test-nocomma-error-log-last-day
TestSlaveBadParameters-1/etc/monitor-promise/check-_re6st-optimal-test-nocomma-error-log-last-hour
TestSlaveBadParameters-1/etc/monitor-promise/check-_re6st-optimal-test-unsafe-error-log-last-day
TestSlaveBadParameters-1/etc/monitor-promise/check-_re6st-optimal-test-unsafe-error-log-last-hour
TestSlaveBadParameters-1/etc/monitor-promise/check-_re6st-optimal-test-unsafe-re6st-optimal-test
TestSlaveBadParameters-1/etc/monitor-promise/check-_server-alias-same-error-log-last-day
TestSlaveBadParameters-1/etc/monitor-promise/check-_server-alias-same-error-log-last-hour
TestSlaveBadParameters-1/etc/monitor-promise/check-_virtualhostroot-http-port-unsafe-error-log-last-day
TestSlaveBadParameters-1/etc/monitor-promise/check-_virtualhostroot-http-port-unsafe-error-log-last-hour
TestSlaveBadParameters-1/etc/monitor-promise/check-_virtualhostroot-https-port-unsafe-error-log-last-day
TestSlaveBadParameters-1/etc/monitor-promise/check-_virtualhostroot-https-port-unsafe-error-log-last-hour
\ No newline at end of file
TestSlaveBadParameters-0/etc/plugin/__init__.py
TestSlaveBadParameters-0/etc/plugin/buildout-TestSlaveBadParameters-0-status.py
TestSlaveBadParameters-0/etc/plugin/check-free-disk-space.py
TestSlaveBadParameters-0/etc/plugin/monitor-bootstrap-status.py
TestSlaveBadParameters-1/etc/plugin/__init__.py
TestSlaveBadParameters-1/etc/plugin/buildout-TestSlaveBadParameters-1-status.py
TestSlaveBadParameters-1/etc/plugin/check-free-disk-space.py
TestSlaveBadParameters-1/etc/plugin/monitor-bootstrap-status.py
\ No newline at end of file
TestSlaveBadParameters-0/etc/promise/monitor-http-frontend
TestSlaveBadParameters-0/etc/promise/monitor-httpd-listening-on-tcp
TestSlaveBadParameters-0/etc/promise/promise-monitor-httpd-is-process-older-than-dependency-set
TestSlaveBadParameters-1/etc/promise/caddy-frontend-is-running-actual-software-release
TestSlaveBadParameters-1/etc/promise/caddy_cached
TestSlaveBadParameters-1/etc/promise/caddy_frontend_ipv4_http
TestSlaveBadParameters-1/etc/promise/caddy_frontend_ipv4_https
TestSlaveBadParameters-1/etc/promise/caddy_frontend_ipv6_http
TestSlaveBadParameters-1/etc/promise/caddy_frontend_ipv6_https
TestSlaveBadParameters-1/etc/promise/caddy_ssl_cached
TestSlaveBadParameters-1/etc/promise/frontend-caddy-configuration-promise
TestSlaveBadParameters-1/etc/promise/monitor-http-frontend
TestSlaveBadParameters-1/etc/promise/monitor-httpd-listening-on-tcp
TestSlaveBadParameters-1/etc/promise/nginx-configuration-promise
TestSlaveBadParameters-1/etc/promise/nginx_frontend_ipv4_http
TestSlaveBadParameters-1/etc/promise/nginx_frontend_ipv4_https
TestSlaveBadParameters-1/etc/promise/nginx_frontend_ipv6_http
TestSlaveBadParameters-1/etc/promise/nginx_frontend_ipv6_https
TestSlaveBadParameters-1/etc/promise/promise-monitor-httpd-is-process-older-than-dependency-set
TestSlaveBadParameters-1/etc/promise/promise-nginx-is-process-older-than-dependency-set
TestSlaveBadParameters-1/etc/promise/re6st-connectivity
TestSlaveBadParameters-1/etc/promise/trafficserver-cache-availability
TestSlaveBadParameters-1/etc/promise/trafficserver-port-listening
\ No newline at end of file
TestSlaveBadParameters-0:bootstrap-monitor EXITED
TestSlaveBadParameters-0:certificate_authority-on-watch RUNNING
TestSlaveBadParameters-0:crond RUNNING
TestSlaveBadParameters-0:monitor-httpd-graceful EXITED
TestSlaveBadParameters-0:monitor-httpd-on-watch RUNNING
TestSlaveBadParameters-1:6tunnel-11080-on-watch RUNNING
TestSlaveBadParameters-1:6tunnel-11443-on-watch RUNNING
TestSlaveBadParameters-1:6tunnel-12080-on-watch RUNNING
TestSlaveBadParameters-1:6tunnel-12443-on-watch RUNNING
TestSlaveBadParameters-1:6tunnel-26011-on-watch RUNNING
TestSlaveBadParameters-1:6tunnel-26012-on-watch RUNNING
TestSlaveBadParameters-1:bootstrap-monitor EXITED
TestSlaveBadParameters-1:certificate_authority-on-watch RUNNING
TestSlaveBadParameters-1:crond-on-watch RUNNING
TestSlaveBadParameters-1:frontend-caddy-safe-graceful EXITED
TestSlaveBadParameters-1:frontend-nginx-safe-graceful EXITED
TestSlaveBadParameters-1:frontend_caddy-on-watch RUNNING
TestSlaveBadParameters-1:frontend_nginx-on-watch RUNNING
TestSlaveBadParameters-1:monitor-httpd-graceful EXITED
TestSlaveBadParameters-1:monitor-httpd-on-watch RUNNING
TestSlaveBadParameters-1:trafficserver-on-watch RUNNING
TestSlaveBadParameters-1:trafficserver-reload EXITED
watchdog:watchdog RUNNING
\ No newline at end of file
......@@ -18,4 +18,4 @@ md5sum = 6e4431cf4b0a0d034402604b1e2844c0
[template-cloudooo-instance]
filename = instance-cloudooo.cfg.in
md5sum = 6ec2461e884761c56ef4ba928a8eede6
md5sum = 31ed5c856ce2dff0305d7029caefc3f6
......@@ -271,17 +271,23 @@ var = ${buildout:directory}/var
framebuffer = ${:srv}/framebuffer
[xvfb-instance]
recipe = slapos.cookbook:xvfb
runner-path = ${directory:services}/xvfb
xvfb-path = {{ parameter_dict["xserver"] }}/bin/Xvfb
fbdir-path = ${directory:framebuffer}
tmp-path = ${directory:run}
shell-path = {{ parameter_dict["dash"] }}/bin/dash
recipe = slapos.cookbook:wrapper
wrapper-path = ${directory:services}/${:_buildout_section_name_}
command-line =
{{ parameter_dict["xserver"] }}/bin/Xvfb
${:display}
-screen 0 1024x768x24
-fbdir ${directory:framebuffer}
environment =
XORG_LOCK_DIR=${:lock-dir}
display = :0
lock-dir = ${directory:run}
[wkhtmltopdf-on-xvfb]
recipe = slapos.cookbook:wrapper
wrapper-path = ${directory:bin}/wkhtmltopdf
environment =
DISPLAY=:0
TMPDIR=${directory:run}
DISPLAY=${xvfb-instance:display}
XORG_LOCK_DIR=${xvfb-instance:lock-dir}
command-line = {{ parameter_dict['wkhtmltopdf'] }}/wkhtmltopdf --use-xserver
......@@ -3,7 +3,7 @@ Available ``software-type`` values
- ``default``
Recommended for developemnt and production use. Automatic creation of
Recommended for development and production use. Automatic creation of
erp5-site.
Notes
......@@ -100,7 +100,7 @@ Zope partitions should be assigned port ranges starting at 2200, incrementing
by some value which depends on how many zope process you want per partition
(see the ``port-base`` parameter in ``zope-partition-dict``).
Notes to the Software Release developper: These ranges are not strictly
Notes to the Software Release developer: These ranges are not strictly
defined. Not each port is actually used so one may reduce alread-assigned
ranges if needed (ex: memcached partitions use actually fewer ports). There
should be enough room for evolution (as between smtp and mariadb types). It is
......
......@@ -345,5 +345,26 @@
},
"type": "object"
}
},
"test-runner": {
"description": "Test runner parameters.",
"properties": {
"enabled": {
"description": "Generate helper scripts to run test suite.",
"default": true,
"type": "boolean"
},
"node-count": {
"description": "Number of tests this instance can execute in parrallel. This must be at least equal to the number of nodes configured on testnode running the test",
"default": 3,
"type": "integer"
},
"extra-database-count": {
"description": "Number of extra databases this instance tests will need.",
"default": 3,
"type": "integer"
}
},
"type": "object"
}
}
......@@ -48,12 +48,6 @@
},
"type": "array"
},
"test-database-amount": {
"description": "The number of test databases to create, adding auto-generated entries to database-list",
"default": 1,
"minimum": 0,
"type": "integer"
},
"catalog-backup": {
"description": "Backup control knobs",
"properties": {
......
[buildout]
extends = software.cfg
shared-parts = /opt/slapgrid/shared-parts
eggs-directory = /opt/slapgrid/shared-eggs
abi-tag-eggs = true
......@@ -58,48 +58,56 @@ output = ${buildout:directory}/template-default.cfg
mode = 0644
[versions]
PyXML = 0.8.5
erp5.util = 0.4.52
erp5.util = 0.4.53
slapos.recipe.template = 4.3
ipython = 5.3.0
apache-libcloud = 2.1.0
gitdb2 = 2.0.2
pyasn1 = 0.3.2
smmap2 = 2.0.3
dnspython = 1.15.0
PyXML = 0.8.5
WebOb = 1.8.2
WebTest = 2.0.30
apache-libcloud = 2.3.0
gitdb2 = 2.0.4
smmap2 = 2.0.4
waitress = 1.1.0
z3c.etestbrowser = 2.0.1
zope.testbrowser = 5.2.4
# Required by:
# slapos.toolbox==0.70
GitPython = 2.1.5
# slapos.toolbox==0.81
GitPython = 2.1.11
# Required by:
# slapos.toolbox==0.70
# zope.testbrowser==5.2.4
WSGIProxy2 = 0.4.4
# Required by:
# slapos.toolbox==0.81
atomize = 0.2.0
# Required by:
# slapos.toolbox==0.70
dnspython = 1.15.0
# WebTest==2.0.30
beautifulsoup4 = 4.6.3
# Required by:
# slapos.toolbox==0.70
# slapos.toolbox==0.81
feedparser = 5.2.1
# Required by:
# slapos.toolbox==0.70
# slapos.toolbox==0.81
lockfile = 0.12.2
# Required by:
# slapos.toolbox==0.70
# slapos.toolbox==0.81
passlib = 1.7.1
# Required by:
# z3c.etestbrowser==2.0.1
zope.testbrowser = 5.2.4
WSGIProxy2 = 0.4.4
WebTest = 2.0.30
# zope.testbrowser==5.2.4
zope.cachedescriptors = 4.3.1
zope.schema = 4.5.0
WebOb = 1.8.2
beautifulsoup4 = 4.6.3
waitress = 1.1.0
# Required by:
# zope.schema==4.5.0
zope.event = 4.3.0
# Required by:
# zope.testbrowser==5.2.4
zope.schema = 4.5.0
......@@ -15,7 +15,7 @@
[instance]
filename = instance.cfg.in
md5sum = 7c907db5f803b03a218b49888a3a3799
md5sum = 74d4c0a5105fa3d74af894fa1afa9ba2
[template-nginx-service]
filename = template-nginx-service.sh.in
......@@ -27,4 +27,4 @@ md5sum = 9f22db89a2679534aa8fd37dbca86782
[template-runTestSuite]
filename = runTestSuite.in
md5sum = af6985e2192b43b5b1dfd37bb538df72
md5sum = d26572727ef1679a8a9e51b021ece524
......@@ -39,15 +39,19 @@ buildout-directory = $${buildout:directory}
mode = 0700
[xvfb-instance]
recipe = slapos.cookbook:xvfb
runner-path = $${directory:services}/xvfb
xvfb-path = ${xserver:location}/bin/Xvfb
fbdir-path = $${directory:framebuffer}
tmp-path = $${directory:run}
shell-path = ${dash:location}/bin/dash
recipe = slapos.cookbook:wrapper
wrapper-path = $${directory:services}/$${:_buildout_section_name_}
command-line =
${xserver:location}/bin/Xvfb
$${:display}
-screen 0 1024x768x24
-fbdir $${directory:framebuffer}
environment =
XORG_LOCK_DIR=$${:lock-dir}
display = :0
lock-dir = $${directory:run}
xwd-path = ${xwd:location}/bin/xwd
xwd-hook-path = $${directory:bin}/xwd
#################################
# Nginx service
......
......@@ -15,8 +15,8 @@ from selenium.webdriver.support import expected_conditions as EC
from subprocess import check_output
import json
os.environ['TMPDIR'] = '$${xvfb-instance:tmp-path}'
os.environ['DISPLAY'] = ':0'
os.environ['XORG_LOCK_DIR'] = '$${xvfb-instance:lock-dir}'
os.environ['DISPLAY'] = '$${xvfb-instance:display}'
BASE_URL = 'http://[$${nginx-configuration:ip}]:$${nginx-configuration:port}/'
......
......@@ -63,7 +63,7 @@ develop = true
[jio-install]
recipe = plone.recipe.command
stop-on-error = true
command = cd ${jio-repository.git:location} && PATH=${git:location}/bin/:${nodejs:location}/bin/:$PATH ${nodejs:location}/bin/npm install jslint@0.9.2 jison@0.4.16 && PATH=${curl:location}/bin/:${nodejs:location}/bin/:$PATH make
command = cd ${jio-repository.git:location} && PATH=${git:location}/bin/:${nodejs:location}/bin/:$PATH ${nodejs:location}/bin/npm install jslint@0.9.2 jison@0.4.16 git://github.com/qunitjs/node-qunit.git#v0.9.3 sinon@1.7.3 && PATH=${curl:location}/bin/:${nodejs:location}/bin/:$PATH make
update-command = ${:command}
[rsvp-repository.git]
......
......@@ -186,7 +186,7 @@ ignore-existing = true
url = ${:_profile_base_location_}/template/template-kvm-run.in
mode = 644
filename = template-kvm-run.in
md5sum = 0a076a9338ea0c25fa4e7c9369473d8a
md5sum = 8d681430f03ae4d35fc64b1625653a22
download-only = true
on-update = true
......
......@@ -238,6 +238,11 @@ if disk_storage_dict:
print('Data folder %s was not used to create external disk %r' % (index +1))
index += 1
additional_disk_options = ''
if disk_aio == 'native':
additional_disk_options += ',cache.direct=on'
# Generate network parameters
# XXX: use_tap should be a boolean
......@@ -293,7 +298,8 @@ else:
kvm_argument_list = [qemu_path,
'-enable-kvm', '-smp', smp, '-name', vm_name, '-m', ram, '-vga', 'std',
'-drive', 'file=%s,if=%s,cache=%s,aio=%s' % (disk_path, disk_type, disk_cache, disk_aio),
'-drive', 'file=%s,if=%s,cache=%s,aio=%s%s' % (disk_path, disk_type, disk_cache,
disk_aio, additional_disk_options),
'-vnc', '%s:1,ipv4,password' % listen_ip,
'-boot', 'order=cd,menu=on',
'-qmp', 'unix:%s,server' % socket_path,
......
......@@ -66,3 +66,14 @@ PyRSS2Gen = 1.1
cns.recipe.symlink = 0.2.3
plone.recipe.command = 1.1
slapos.recipe.template = 4.3
dnspython = 1.15.0
erp5.util = 0.4.53
passlib = 1.7.1
GitPython = 2.1.11
lockfile = 0.12.2
apache-libcloud = 2.3.0
feedparser = 5.2.1
atomize = 0.2.0
gitdb2 = 2.0.4
smmap2 = 2.0.4
......@@ -19,4 +19,4 @@ md5sum = c4ac5de141ae6a64848309af03e51d88
[template-selenium]
filename = instance-selenium.cfg.in
md5sum = 9b6648b8f37baa3f7a6bfb68d4426049
md5sum = 8f06bef6ed0737afa7916bc5ac7db537
......@@ -6,33 +6,6 @@ eggs-directory = ${buildout:eggs-directory}
develop-eggs-directory = ${buildout:develop-eggs-directory}
offline = true
[selenium-instance]
recipe = slapos.cookbook:seleniumrunner
runner-path = $${basedirectory:services}/selenium-runner
browser = $${firefox-instance:runner-path}
suite-url = $${slap-parameter:suite-url}
report-url = $${slap-parameter:report-url}
report-project = $${slap-parameter:report-project}
[firefox-instance]
recipe = slapos.cookbook:firefox
runner-path = $${rootdirectory:bin}/firefox-sandboxed
firefox-path = ${firefox-wrapper:location}
prefsjs-path = $${rootdirectory:etc}/prefs.js
shell-path = ${dash:location}/bin/dash
tmp-path = $${xvfb-instance:tmp-path}
[xvfb-instance]
recipe = slapos.cookbook:xvfb
runner-path = $${basedirectory:services}/xvfb
xvfb-path = ${xserver:location}/bin/Xvfb
fbdir-path = $${basedirectory:framebuffer}
tmp-path = $${basedirectory:run}
shell-path = ${dash:location}/bin/dash
xwd-path = ${xwd:location}/bin/xwd
xwd-hook-path = $${rootdirectory:bin}/xwd
[rootdirectory]
recipe = slapos.cookbook:mkdirectory
etc = $${buildout:directory}/etc
......
......@@ -26,7 +26,7 @@ md5sum = 1af531c51f575a1d1362f2ca2d61620d
[template-mariadb]
filename = instance-mariadb.cfg.in
md5sum = 93f4f4c27d9302977752240acefd4530
md5sum = 14ed6a85e086d52d4f4a553dda570f64
[template-kumofs]
filename = instance-kumofs.cfg.in
......@@ -42,7 +42,7 @@ md5sum = d32417746fcf671d4e86a70379815039
[template-my-cnf]
filename = my.cnf.in
md5sum = d814b984abf2dc444af2a0bc6287e7f5
md5sum = 2e381fb10a7c024c99caa434036173c6
[template-mariadb-initial-setup]
filename = mariadb_initial_setup.sql.in
......@@ -74,7 +74,7 @@ md5sum = d41d8cd98f00b204e9800998ecf8427e
[template-erp5]
filename = instance-erp5.cfg.in
md5sum = 83e0f627633ec3e394dc27d06c627b3b
md5sum = 64fd49395de02e6c2107afd1dd9cbdc9
[template-zeo]
filename = instance-zeo.cfg.in
......@@ -82,11 +82,11 @@ md5sum = 3e650915959ff31c9c13c84069bbcd35
[template-zope]
filename = instance-zope.cfg.in
md5sum = e73678921067506e710ae11e41f0a9a8
md5sum = 408c8f358935d3167448ea3732adc994
[template-balancer]
filename = instance-balancer.cfg.in
md5sum = 89872075cd2f31d824369447aaf0532f
md5sum = dccdf2a8cbfa57e49fa04be9ba4ff595
[template-haproxy-cfg]
filename = haproxy.cfg.in
......
......@@ -97,10 +97,13 @@ ipv4 = {{ ipv4 }}
{% endif -%}
{% set haproxy_dict = {} -%}
{% set apache_dict = {} -%}
{% set zope_virtualhost_monster_backend_dict = {} %}
{% set test_runner_url_dict = {} %} {# family_name => list of apache URLs #}
{% set next_port = itertools.count(slapparameter_dict['tcpv4-port']).next -%}
{% for family_name, parameter_id_list in sorted(
slapparameter_dict['zope-family-dict'].iteritems()) -%}
{% set zope_family_address_list = [] -%}
{% set ssl_authentication = slapparameter_dict['ssl-authentication-dict'].get(family_name, False) -%}
{% set has_webdav = [] -%}
{% for parameter_id in parameter_id_list -%}
{% set zope_address_list = slapparameter_dict[parameter_id] -%}
......@@ -121,8 +124,27 @@ ipv6 = {{ zope_address.split(']:')[0][1:] }}
{% set zope_effective_address = zope_address -%}
{% endif -%}
{% do zope_family_address_list.append((zope_effective_address, maxconn, webdav)) -%}
{# # Generate entries with rewrite rule for test runnners #}
{% set test_runner_backend_mapping = {} %}
{% set test_runner_apache_url_list = [] %}
{% set test_runner_external_port = next_port() %}
{% for i, (test_runner_internal_ip, test_runner_internal_port) in
enumerate(slapparameter_dict[parameter_id ~ '-test-runner-address-list']) %}
{% do test_runner_backend_mapping.__setitem__(
'unit_test_' ~ i,
'http://' ~ test_runner_internal_ip ~ ':' ~ test_runner_internal_port ) %}
{% do test_runner_apache_url_list.append(
'https://' ~ ipv4 ~ ':' ~ test_runner_external_port ~ '/unit_test_' ~ i ~ '/' ) %}
{% endfor %}
{% do zope_virtualhost_monster_backend_dict.__setitem__(
(ipv4, test_runner_external_port),
( ssl_authentication, test_runner_backend_mapping ) ) -%}
{% do test_runner_url_dict.__setitem__(family_name, test_runner_apache_url_list) -%}
{% endfor -%}
{% endfor -%}
{# Make rendering fail artificially if any family has no known backend.
# This is useful as haproxy's hot-reconfiguration mechanism is
# supervisord-incompatible.
......@@ -140,7 +162,6 @@ ipv6 = {{ zope_address.split(']:')[0][1:] }}
{% set internal_scheme = 'http' -%}
{% set external_scheme = 'https' -%}
{% endif -%}
{% set ssl_authentication = slapparameter_dict['ssl-authentication-dict'].get(family_name, False) -%}
{% do apache_dict.__setitem__(family_name, (next_port(), external_scheme, internal_scheme ~ '://' ~ ipv4 ~ ':' ~ haproxy_port ~ backend_path, ssl_authentication)) -%}
{% endfor -%}
......@@ -173,6 +194,7 @@ crl = ${directory:apache-conf}/crl.pem
[apache-conf-parameter-dict]
backend-list = {{ dumps(apache_dict.values()) }}
zope-virtualhost-monster-backend-dict = {{ dumps(zope_virtualhost_monster_backend_dict) }}
ip-list = {{ dumps(apache_ip_list) }}
pid-file = ${directory:run}/apache.pid
log-dir = ${directory:log}
......@@ -225,6 +247,10 @@ recipe = slapos.cookbook:publish.serialised
{{ family_name ~ '-v6' }} = {% if ipv6_set %}{{ scheme ~ '://[' ~ ipv6 ~ ']:' ~ apache_port }}{% endif %}
{{ family_name }} = {{ scheme ~ '://' ~ ipv4 ~ ':' ~ apache_port }}
{% endfor -%}
{% for family_name, test_runner_url_list in test_runner_url_dict.items() -%}
{{ family_name ~ '-test-runner-url-list' }} = {{ dumps(test_runner_url_list) }}
{% endfor -%}
monitor-base-url = ${monitor-publish-parameters:monitor-base-url}
[apache-ssl]
......
......@@ -8,6 +8,16 @@
{% set jupyter_dict = slapparameter_dict.get('jupyter', {}) -%}
{% set has_jupyter = jupyter_dict.get('enable', jupyter_enable_default.lower() in ('true', 'yes')) -%}
{% set jupyter_zope_family = jupyter_dict.get('zope-family', '') -%}
{% set test_runner_enabled = slapparameter_dict.get('test-runner', {}).get('enabled', True) -%}
{% set test_runner_node_count = slapparameter_dict.get('test-runner', {}).get('node-count', 3) -%}
{% set test_runner_extra_database_count = slapparameter_dict.get('test-runner', {}).get('extra-database-count', 3) -%}
{% set test_runner_total_database_count = test_runner_node_count * (1 + test_runner_extra_database_count) -%}
{# Backward compatibility for mariadb.test-database-amount #}
{% set mariadb_test_database_amount = slapparameter_dict.get('mariadb', {}).get('test-database-amount') -%}
{% if mariadb_test_database_amount is not none -%}
{% set test_runner_total_database_count = mariadb_test_database_amount %}
{% set test_runner_enabled = mariadb_test_database_amount > 0 %}
{% endif -%}
{% set monitor_base_url_dict = {} -%}
{% set caucase_url = slapparameter_dict.get('caucase', {}).pop('url', '') -%}
{% set monitor_dict = slapparameter_dict.get('monitor', {}) %}
......@@ -44,7 +54,7 @@ config-name = {{ name }}
{{ request('memcached-persistent', 'kumofs', 'kumofs', {'tcpv4-port': 2000}, {'url': True, 'monitor-base-url': False}, key_config={'monitor-passwd': 'monitor-htpasswd:passwd'}) }}
{{ request('memcached-volatile', 'kumofs', 'memcached', {'tcpv4-port': 2010, 'ram-storage-size': 64}, {'url': True, 'monitor-base-url': False}, key_config={'monitor-passwd': 'monitor-htpasswd:passwd'}) }}
{{ request('mariadb', 'mariadb', 'mariadb', {'tcpv4-port': 2099, 'max-slowqueries-threshold': monitor_dict.get('max-slowqueries-threshold', 1000), 'slowest-query-threshold': monitor_dict.get('slowest-query-threshold', '') }, {'database-list': True, 'test-database-list': True, 'monitor-base-url': False}, key_config={'monitor-passwd': 'monitor-htpasswd:passwd'}) }}
{{ request('mariadb', 'mariadb', 'mariadb', {'tcpv4-port': 2099, 'max-slowqueries-threshold': monitor_dict.get('max-slowqueries-threshold', 1000), 'slowest-query-threshold': monitor_dict.get('slowest-query-threshold', ''), 'test-database-amount': test_runner_total_database_count}, {'database-list': True, 'test-database-list': True, 'monitor-base-url': False}, key_config={'monitor-passwd': 'monitor-htpasswd:passwd'}) }}
{% if has_posftix -%}
{{ request('smtp', 'postfix', 'smtp', {'tcpv4-port': 2025, 'smtpd-sasl-user': 'erp5@nowhere'}, key_config={'smtpd-sasl-password': 'publish-early:smtpd-sasl-password'}) }}
{%- else %}
......@@ -93,44 +103,6 @@ connection-http-url = {{ caucase_url }}
{% endif -%}
{% endfor -%}
[publish-early]
recipe = slapos.cookbook:publish-early
-init =
inituser-password gen-password:passwd
deadlock-debugger-password gen-deadlock-debugger-password:passwd
{%- if has_posftix %}
smtpd-sasl-password gen-smtpd-sasl-password:passwd
{%- endif %}
{%- if neo %}
neo-cluster gen-neo-cluster:name
{%- if neo[0] %}
neo-cluster = {{ dumps(neo[0]) }}
{%- endif %}
{%- endif %}
{%- set inituser_password = slapparameter_dict.get('inituser-password') %}
{%- if inituser_password %}
inituser-password = {{ dumps(inituser_password) }}
{%- endif %}
{%- set deadlock_debugger_password = slapparameter_dict.get('deadlock-debugger-password') -%}
{%- if deadlock_debugger_password %}
deadlock-debugger-password = {{ dumps(deadlock_debugger_password) }}
{%- endif %}
[gen-password]
recipe = slapos.cookbook:generate.password
storage-path =
[gen-deadlock-debugger-password]
<= gen-password
[gen-neo-cluster-base]
<= gen-password
[gen-neo-cluster]
name = neo-${gen-neo-cluster-base:passwd}
[gen-smtpd-sasl-password]
< = gen-password
{% set zope_partition_dict = slapparameter_dict.get('zope-partition-dict', {'1': {}}) -%}
{% set zope_address_list_id_dict = {} -%}
......@@ -142,6 +114,9 @@ return =
zope-address-list
hosts-dict
monitor-base-url
{%- if test_runner_enabled %}
test-runner-address-list
{% endif %}
{% set bt5_default_list = 'erp5_full_text_myisam_catalog erp5_configurator_standard erp5_configurator_maxma_demo erp5_configurator_run_my_doc' -%}
{% if has_jupyter -%}
{% set bt5_default_list = bt5_default_list + ' erp5_data_notebook' -%}
......@@ -169,6 +144,8 @@ config-timezone = {{ dumps(slapparameter_dict.get('timezone', 'UTC')) }}
config-cloudooo-retry-count = {{ slapparameter_dict.get('cloudooo-retry-count', 2) }}
config-wendelin-core-zblk-fmt = {{ dumps(slapparameter_dict.get('wendelin-core-zblk-fmt', '')) }}
config-zodb-dict = {{ dumps(zodb_dict) }}
config-test-runner-enabled = {{ dumps(test_runner_enabled) }}
config-test-runner-node-count = {{ dumps(test_runner_node_count) }}
{% for server_type, server_dict in storage_dict.iteritems() -%}
{% if server_type == 'neo' -%}
config-neo-cluster = ${publish-early:neo-cluster}
......@@ -183,6 +160,7 @@ config-tidstorage-port = ${request-zodb:connection-tidstorage-port}
software-type = zope
{% set zope_family_dict = {} -%}
{% set zope_family_name_list = [] -%}
{% set zope_backend_path_dict = {} -%}
{% set ssl_authentication_dict = {} -%}
{% set jupyter_zope_family_default = [] -%}
......@@ -190,6 +168,7 @@ software-type = zope
{% set partition_name = 'zope-' ~ custom_name -%}
{% set section_name = 'request-' ~ partition_name -%}
{% set zope_family = zope_parameter_dict.get('family', 'default') -%}
{% do zope_family_name_list.append(zope_family) %}
{% set backend_path = zope_parameter_dict.get('backend-path', '/') % {'site-id': site_id} %}
{# # default jupyter zope family is first zope family. -#}
{# # use list.append() to update it, because in jinja2 set changes only local scope. -#}
......@@ -214,6 +193,7 @@ config-longrequest-logger-timeout = {{ dumps(zope_parameter_dict.get('longreques
config-large-file-threshold = {{ dumps(zope_parameter_dict.get('large-file-threshold', "10MB")) }}
config-port-base = {{ dumps(zope_parameter_dict.get('port-base', 2200)) }}
config-webdav = {{ dumps(zope_parameter_dict.get('webdav', False)) }}
config-test-runner-apache-url-list = ${publish-early:{{ zope_family }}-test-runner-url-list}
{% endfor -%}
{# if not explicitly configured, connect jupyter to first zope family, which -#}
......@@ -268,6 +248,9 @@ return =
{%- for family in zope_family_dict %}
{{ family }}
{{ family }}-v6
{% if test_runner_enabled %}
{{ family }}-test-runner-url-list
{% endif %}
{% endfor -%}
{% do monitor_base_url_dict.__setitem__('request-balancer', '${' ~ 'request-balancer' ~ ':connection-monitor-base-url}') -%}
......@@ -275,6 +258,9 @@ config-zope-family-dict = {{ dumps(zope_family_parameter_dict) }}
config-tcpv4-port = {{ dumps(balancer_dict.get('tcpv4-port', 2150)) }}
{% for zope_section_id, name in zope_address_list_id_dict.items() -%}
config-{{ name }} = {{ ' ${' ~ zope_section_id ~ ':connection-zope-address-list}' }}
{% if test_runner_enabled -%}
config-{{ name }}-test-runner-address-list = {{ ' ${' ~ zope_section_id ~ ':connection-test-runner-address-list}' }}
{% endif -%}
{% endfor -%}
# XXX: should those really be same for all families ?
config-haproxy-server-check-path = {{ dumps(balancer_dict.get('haproxy-server-check-path', '/') % {'site-id': site_id}) }}
......@@ -335,6 +321,57 @@ hosts-dict = {{ '${' ~ zope_address_list_id_dict.keys()[0] ~ ':connection-hosts-
{% for name, value in publish_dict.items() -%}
{{ name }} = {{ value }}
{% endfor -%}
{% for zope_family_name in zope_family_name_list -%}
{{ zope_family_name }}-test-runner-url-list = ${request-balancer:connection-{{ zope_family_name }}-test-runner-url-list}
{% endfor -%}
[publish-early]
recipe = slapos.cookbook:publish-early
-init =
inituser-password gen-password:passwd
deadlock-debugger-password gen-deadlock-debugger-password:passwd
{%- if has_posftix %}
smtpd-sasl-password gen-smtpd-sasl-password:passwd
{%- endif %}
{% for zope_family_name in zope_family_name_list %}
{{ zope_family_name }}-test-runner-url-list default-balancer-test-runner-url-list:default
{% endfor -%}
{%- if neo %}
neo-cluster gen-neo-cluster:name
{%- if neo[0] %}
neo-cluster = {{ dumps(neo[0]) }}
{%- endif %}
{%- endif %}
{%- set inituser_password = slapparameter_dict.get('inituser-password') %}
{%- if inituser_password %}
inituser-password = {{ dumps(inituser_password) }}
{%- endif %}
{%- set deadlock_debugger_password = slapparameter_dict.get('deadlock-debugger-password') -%}
{%- if deadlock_debugger_password %}
deadlock-debugger-password = {{ dumps(deadlock_debugger_password) }}
{%- endif %}
[default-balancer-test-runner-url-list]
recipe =
default = not-ready
[gen-password]
recipe = slapos.cookbook:generate.password
storage-path =
[gen-deadlock-debugger-password]
<= gen-password
[gen-neo-cluster-base]
<= gen-password
[gen-neo-cluster]
name = neo-${gen-neo-cluster-base:passwd}
[gen-smtpd-sasl-password]
< = gen-password
[monitor-instance-parameter]
monitor-httpd-port = 8386
......
......@@ -107,7 +107,7 @@ time = {{ dumps(backup_periodicity) }}
# can be fully restored.
# master-data: use value "2" as we are not in a replication case
#}
command = "${binary-wrap-mysqldump:wrapper-path}" -u root --all-databases --flush-privileges --single-transaction {% if incremental_backup_retention_days > -1 %}--flush-logs --master-data=2 {% endif %}| {{ parameter_dict['gzip-location'] }}/bin/gzip > "${directory:mariadb-backup-full}/$({{ parameter_dict['coreutils-location'] }}/bin/date "+%Y%m%d%H%M%S").sql.gz"
command = "${binary-wrap-mysqldump:wrapper-path}" -u root --all-databases --flush-privileges --single-transaction --max-allowed-packet=128M {% if incremental_backup_retention_days > -1 %}--flush-logs --master-data=2 {% endif %}| {{ parameter_dict['gzip-location'] }}/bin/gzip > "${directory:mariadb-backup-full}/$({{ parameter_dict['coreutils-location'] }}/bin/date "+%Y%m%d%H%M%S").sql.gz"
{# KEEP GLOB PATTERN IN SYNC with generated filenames above
# YYYYmmddHHMMSS -#}
file-glob = ??????????????.sql.gz
......
......@@ -8,6 +8,9 @@
{% set node_id_index_format = '-%%0%ii' % (len(str(instance_index_list[-1])), ) -%}
{% set part_list = [] -%}
{% set publish_list = [] -%}
{% set test_runner_address_list = [] -%}
{% set test_runner_enabled = slapparameter_dict['test-runner-enabled'] -%}
{% set test_runner_node_count = slapparameter_dict['test-runner-node-count'] -%}
{% set longrequest_logger_base_path = buildout_directory ~ '/var/log/longrequest_logger_' -%}
{% macro section(name) %}{% do part_list.append(name) %}{{ name }}{% endmacro -%}
{% set bin_directory = parameter_dict['buildout-bin-directory'] -%}
......@@ -68,84 +71,6 @@ ca-certs = ${directory:test-ca-certs}
ca-newcerts = ${directory:test-ca-newcerts}
ca-crl = ${directory:test-ca-crl}
{% if saucelabs_dict -%}
[test-zelenium-runner-parameter]
configuration = {{ dumps(saucelabs_dict) }}
user = {{ dumps(slapparameter_dict['inituser-login']) }}
password = {{ dumps(slapparameter_dict['inituser-password']) }}
bin-path = {{ bin_directory }}/{{ parameter_dict['egg-interpreter'] }}
[{{ section('test-zelenium-runner') }}]
<= jinja2-template-base
template = {{ parameter_dict['run-zelenium-template'] }}
rendered = ${directory:bin}/runTestSuite
mode = 755
context =
import json_module json
key configuration test-zelenium-runner-parameter:configuration
key user test-zelenium-runner-parameter:user
key password test-zelenium-runner-parameter:password
key bin_path test-zelenium-runner-parameter:bin-path
{% else -%}
{% if slapparameter_dict['mysql-test-url-list'] -%}
[{{ section('run-unit-test-userhosts-wrapper') }}]
<= userhosts-wrapper-base
wrapped-command-line = ${runUnitTest:wrapper-path}
wrapper-path = ${buildout:bin-directory}/runUnitTest
[{{ section('run-test-suite-userhosts-wrapper') }}]
<= userhosts-wrapper-base
wrapped-command-line = ${runTestSuite:wrapper-path}
wrapper-path = ${buildout:bin-directory}/runTestSuite
{% set connection_string_list = [] -%}
{% for url in slapparameter_dict['mysql-test-url-list'] -%}
{% set parsed_url = urlparse.urlparse(url) -%}
{% do connection_string_list.append(
'%s@%s:%s %s %s' % (
parsed_url.path.lstrip('/'),
parsed_url.hostname,
parsed_url.port,
parsed_url.username,
parsed_url.password,
),
) -%}
{% endfor -%}
[run-test-common]
< = run-common
environment-extra =
REAL_INSTANCE_HOME=${:instance-home}
OPENSSL_BINARY=${test-certificate-authority:openssl-binary}
TEST_CA_PATH=${test-certificate-authority:ca-dir}
instance-home = ${directory:unit-test-path}
wrapper-path = ${directory:bin}/${:command-name}.real
command-line =
'{{ parameter_dict['bin-directory'] }}/${:command-name}'
${:command-line-extra}
--conversion_server_url={{ slapparameter_dict['cloudooo-url'] }}
--conversion_server_retry_count={{ slapparameter_dict.get('cloudooo-retry-count', 2) }}
{#- BBB: We still have test suites that only accept the following 2 options. #}
--conversion_server_hostname=erp5-cloudooo
--conversion_server_port={{ port_dict['erp5-cloudooo'] }}
--volatile_memcached_server_hostname=erp5-memcached-volatile
--volatile_memcached_server_port={{ port_dict['erp5-memcached-volatile'] }}
--persistent_memcached_server_hostname=erp5-memcached-persistent
--persistent_memcached_server_port={{ port_dict['erp5-memcached-persistent'] }}
[{{ section('runUnitTest') }}]
< = run-test-common
command-name = runUnitTest
command-line-extra =
--erp5_sql_connection_string '{{ connection_string_list[0] }}'
--extra_sql_connection_string_list '{{ ','.join(connection_string_list[1:]) }}'
[{{ section('runTestSuite') }}]
< = run-test-common
command-name = runTestSuite
command-line-extra =
--db_list '{{ ','.join(connection_string_list) }}'
{%- endif %}
{%- endif %}
[directory]
recipe = slapos.cookbook:mkdirectory
......@@ -472,6 +397,111 @@ wrapper-path = ${buildout:bin-directory}/${:_buildout_section_name_}
[{{ section("wait_activities") }}]
<= watch_activities
{% if saucelabs_dict -%}
[test-zelenium-runner-parameter]
configuration = {{ dumps(saucelabs_dict) }}
user = {{ dumps(slapparameter_dict['inituser-login']) }}
password = {{ dumps(slapparameter_dict['inituser-password']) }}
bin-path = {{ bin_directory }}/{{ parameter_dict['egg-interpreter'] }}
[{{ section('test-zelenium-runner') }}]
<= jinja2-template-base
template = {{ parameter_dict['run-zelenium-template'] }}
rendered = ${directory:bin}/runTestSuite
mode = 755
context =
import json_module json
key configuration test-zelenium-runner-parameter:configuration
key user test-zelenium-runner-parameter:user
key password test-zelenium-runner-parameter:password
key bin_path test-zelenium-runner-parameter:bin-path
{% else -%}
{% if test_runner_enabled and test_runner_node_count -%}
{% for _ in range(test_runner_node_count) %}
{% do test_runner_address_list.append((ipv4, next_port())) %}
{% endfor %}
[{{ section('run-unit-test-userhosts-wrapper') }}]
<= userhosts-wrapper-base
wrapped-command-line = ${runUnitTest:wrapper-path}
wrapper-path = ${buildout:bin-directory}/runUnitTest
[{{ section('run-test-suite-userhosts-wrapper') }}]
<= userhosts-wrapper-base
wrapped-command-line = ${runTestSuite:wrapper-path}
wrapper-path = ${buildout:bin-directory}/runTestSuite
{% set connection_string_list = [] -%}
{% for url in slapparameter_dict['mysql-test-url-list'] -%}
{% set parsed_url = urlparse.urlparse(url) -%}
{% do connection_string_list.append(
'%s@%s:%s %s %s' % (
parsed_url.path.lstrip('/'),
parsed_url.hostname,
parsed_url.port,
parsed_url.username,
parsed_url.password,
),
) -%}
{% endfor -%}
[run-test-common]
< = run-common
environment-extra =
REAL_INSTANCE_HOME=${:instance-home}
OPENSSL_BINARY=${test-certificate-authority:openssl-binary}
TEST_CA_PATH=${test-certificate-authority:ca-dir}
instance-home = ${directory:unit-test-path}
wrapper-path = ${directory:bin}/${:command-name}.real
command-line =
'{{ parameter_dict['bin-directory'] }}/${:command-name}'
${:command-line-extra}
--conversion_server_url={{ slapparameter_dict['cloudooo-url'] }}
--conversion_server_retry_count={{ slapparameter_dict.get('cloudooo-retry-count', 2) }}
{#- BBB: We still have test suites that only accept the following 2 options. #}
--conversion_server_hostname=erp5-cloudooo
--conversion_server_port={{ port_dict['erp5-cloudooo'] }}
--volatile_memcached_server_hostname=erp5-memcached-volatile
--volatile_memcached_server_port={{ port_dict['erp5-memcached-volatile'] }}
--persistent_memcached_server_hostname=erp5-memcached-persistent
--persistent_memcached_server_port={{ port_dict['erp5-memcached-persistent'] }}
[{{ section('runUnitTest') }}]
< = run-test-common
command-name = runUnitTest
command-line-extra =
--erp5_sql_connection_string '{{ connection_string_list[0] }}'
--extra_sql_connection_string_list '{{ ','.join(connection_string_list[1:]) }}'
--zserver {{ test_runner_address_list[0][0] ~ ':' ~ test_runner_address_list[0][1] }}
--zserver_frontend_url {{ slapparameter_dict['test-runner-apache-url-list'][0] }}
[{{ section('runTestSuite') }}]
< = run-test-common
command-name = runTestSuite
command-line-extra =
--db_list '{{ ','.join(connection_string_list) }}'
environment-extra +=
{#- turn a list of (ip, port) in a list of 'ip:port' #}
{% set zserver_address_list = [] -%}
{% for ip, port in test_runner_address_list %}
{% do zserver_address_list.append(ip ~ ':' ~ port) %}
{% endfor -%}
zserver_address_list={{ ','.join(zserver_address_list) }}
zserver_frontend_url_list={{ ','.join(slapparameter_dict['test-runner-apache-url-list']) }}
[{{ section("promise-test-runner-apache-url") }}]
# promise to wait for apache partition to have returned the parameter
recipe = slapos.cookbook:check_parameter
value = {{ slapparameter_dict['test-runner-apache-url-list'] }}
expected-not-value = not-ready
path = ${directory:promises}/${:_buildout_section_name_}
expected-value =
{%- endif %}
{%- endif %}
[publish]
recipe = slapos.cookbook:publish.serialised
zope-address-list = {{ dumps(publish_list) }}
......@@ -484,6 +514,7 @@ hard to guess.
-#}
hosts-dict = {{ dumps(hosts_dict) }}
monitor-base-url = ${monitor-publish-parameters:monitor-base-url}
test-runner-address-list = {{ dumps(test_runner_address_list) }}
[monitor-instance-parameter]
monitor-httpd-ipv6 = {{ (ipv6_set | list)[0] }}
......
......@@ -93,3 +93,6 @@ no_auto_rehash
[mysqlhotcopy]
interactive_timeout
[mysqldump]
max_allowed_packet = 128M
......@@ -53,6 +53,11 @@ def main():
help='zuite to run',
default = parser_configuration.get('run-only')
)
parser.add_argument('--max_duration',
type=int,
help='max duration for running test en second',
default = parser_configuration.get('max-duration', 1800)
)
args = parser.parse_args()
test_line_dict = {}
test_suite_title = args.test_suite_title or args.test_suite
......@@ -68,13 +73,17 @@ def main():
'platformName': args.target,
'platformVersion': args.target_version,
'deviceName': args.target_device,
'browserName': args.target_browser
'browserName': args.target_browser,
'maxDuration': args.max_duration,
'name': test_suite_title
}
elif 'Windows' in args.target or 'OS X' in args.target:
capabilities = {
'browserName': args.target_browser,
'platform': args.target,
'version': args.target_version
'version': args.target_version,
'maxDuration': args.max_duration,
'name': test_suite_title
}
if not args.appium_server_auth or not args.remote_access_url:
......@@ -115,7 +124,7 @@ def main():
except Exception:
traceback.print_exc()
time.sleep(10)
tool = taskdistribution.TaskDistributor(portal_url=args.master_url)
browser = webdriver.Remote(appium_url, capabilities)
......@@ -137,10 +146,28 @@ def main():
) - 1 # First child is the file name
# Wait for test to be executed
erp5_zelenium_test_timeout = 90
WebDriverWait(browser, erp5_zelenium_test_timeout * (test_count + 1)).until(EC.presence_of_element_located((
By.XPATH, '//td[@id="testRuns" and contains(text(), "%i")]' % test_count
)))
while 1:
try:
WebDriverWait(browser, args.max_duration).until(EC.presence_of_element_located((
By.XPATH, '//td[@id="testRuns" and contains(text(), "%i")]' % test_count
)))
break
except TimeoutException as error:
raise error
except Exception:
traceback.print_exc()
except:
test_line_dict['UnexpectedException'] = {
'test_count': 1,
'error_count': 0,
'failure_count': 1,
'skip_count': 0,
'duration': 1,
'command': url,
'stdout': agent,
'stderr': traceback.format_exc()
}
finally:
execution_duration = round(time.time() - start_time, 2)
if test_count:
test_execution_duration = execution_duration / test_count
......@@ -167,46 +194,45 @@ def main():
for tr in tbody[1:]:
# First td is the main title
test_name = tr[0][0].text
skip_count = success_count = error_count = 0
test_table = tr[1].xpath('.//table')[0]
test_tbody = tr[1].xpath('.//tbody')[0]
tr_count = len(test_tbody)
for tr in test_tbody:
# print etree.tostring(tr).split('\n')[0]
status = tr.attrib.get('class')
if status is None or 'status_done' in status:
skip_count += 1
elif 'status_passed' in status:
success_count += 1
elif 'status_failed' in status:
error_count += 1
test_line_dict[test_name] = {
'test_count': tr_count,
'error_count': error_count,
'failure_count': tr_count - (skip_count + success_count + error_count),
'skip_count': skip_count,
'duration': test_execution_duration,
if len(tr) == 1:
test_line_dict[test_name] = {
'test_count': 1,
'error_count': 0,
'failure_count': 1,
'skip_count': 0,
'duration': 0,
'command': url,
'stdout': agent,
'stderr': '',
'html_test_result': etree.tostring(test_table)
'html_test_result': 'Not Executed due to saucelabs related error'
}
except:
test_line_dict['UnexpectedException'] = {
'test_count': 1,
'error_count': 0,
'failure_count': 1,
'skip_count': 0,
'duration': 1,
'command': url,
'stdout': agent,
'stderr': traceback.format_exc()
}
finally:
else:
skip_count = success_count = error_count = 0
test_table = tr[1].xpath('.//table')[0]
test_tbody = tr[1].xpath('.//tbody')[0]
tr_count = len(test_tbody)
for tr in test_tbody:
# print etree.tostring(tr).split('\n')[0]
status = tr.attrib.get('class')
if status is None or 'status_done' in status:
skip_count += 1
elif 'status_passed' in status:
success_count += 1
elif 'status_failed' in status:
error_count += 1
test_line_dict[test_name] = {
'test_count': tr_count,
'error_count': error_count,
'failure_count': tr_count - (skip_count + success_count + error_count),
'skip_count': skip_count,
'duration': test_execution_duration,
'command': url,
'stdout': agent,
'stderr': '',
'html_test_result': etree.tostring(test_table)
}
browser.quit()
try:
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment