Commit 121aff64 authored by Alain Takoudjou's avatar Alain Takoudjou

Update Release Candidate

parents 79dd2ee9 8d39cd73
[buildout]
extends =
../numpy/openblas.cfg
../cython/buildout.cfg
../scipy/buildout.cfg
parts =
PyWavelets
[PyWavelets-env]
<= numpy-env
[PyWavelets]
recipe = zc.recipe.egg:custom
egg = PyWavelets
environment = PyWavelets-env
setup-eggs =
${numpy:egg}
${scipy:egg}
${cython:egg}
rpath =
${gcc:location}/lib
${gcc:location}/lib64
${openblas:location}/lib
[PyWavelets-repository]
recipe = slapos.recipe.build:gitclone
git-executable = ${git:location}/bin/git
repository = https://github.com/PyWavelets/pywt.git
location = ${buildout:parts-directory}/PyWavelets
[PyWavelets-develop]
# This only work if we use zc.recipe.egg from zc.buildout 2 patched
# by nexedi (zc.recipe.egg = 2.0.3+slapos001 and zc.buildout = 2.5.2+slapos003)
recipe = zc.recipe.egg:develop
egg = PyWavelets
setup = ${PyWavelets-repository:location}
environment = PyWavelets-env
setup-eggs =
${cython:egg}
${numpy:egg}
${scipy:egg}
rpath =
${gcc-fortran:location}/lib
${gcc-fortran:location}/lib64
${openblas:location}/lib
......@@ -11,13 +11,11 @@ extends =
[leptonica]
recipe = slapos.recipe.cmmi
url = http://leptonica.googlecode.com/files/leptonica-1.68.tar.gz
md5sum = 5cd7092f9ff2ca7e3f3e73bfcd556403
url = http://www.leptonica.com/source/leptonica-1.76.0.tar.gz
md5sum = a263a5e4f7e8f8a661fb121a265d2d20
configure-options =
--disable-static
patch-options = -p1
patches =
${:_profile_base_location_}/leptonica-1.69-zlib-include.patch#cff3dc942075190939b407c38e0d3201
environment =
CPPFLAGS=-I${zlib:location}/include -I${libjpeg:location}/include -I${libpng:location}/include -I${libtiff:location}/include -I${webp:location}/include -I${giflib:location}/include
LDFLAGS=-L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib -L${libjpeg:location}/lib -Wl,-rpath=${libjpeg:location}/lib -L${libpng:location}/lib -Wl,-rpath=${libpng:location}/lib -L${libtiff:location}/lib -Wl,-rpath=${libtiff:location}/lib -L${webp:location}/lib -Wl,-rpath=${webp:location}/lib -L${giflib:location}/lib -Wl,-rpath=${giflib:location}/lib
......
[buildout]
extends =
../numpy/openblas.cfg
../cython/buildout.cfg
../scipy/buildout.cfg
../PyWavelets/buildout.cfg
../pillow/buildout.cfg
parts =
scikit-image
[scikit-image-env]
<= numpy-env
[scikit-image]
recipe = zc.recipe.egg:custom
egg = scikit-image
environment = scikit-image-env
setup-eggs =
${numpy:egg}
${scipy:egg}
${cython:egg}
${PyWavelets:egg}
${pillow-python:egg}
networkx
rpath =
${gcc:location}/lib
${gcc:location}/lib64
${openblas:location}/lib
[scikit-image-repository]
recipe = slapos.recipe.build:gitclone
git-executable = ${git:location}/bin/git
repository = https://github.com/scikit-image/scikit-image.git
location = ${buildout:parts-directory}/scikit-image
[scikit-image-develop]
# This only work if we use zc.recipe.egg from zc.buildout 2 patched
# by nexedi (zc.recipe.egg = 2.0.3+slapos001 and zc.buildout = 2.5.2+slapos003)
recipe = zc.recipe.egg:develop
egg = scikit-image
setup = ${scikit-image-repository:location}
environment = scikit-image-env
setup-eggs =
${cython:egg}
${numpy:egg}
${scipy:egg}
${PyWavelets:egg}
${pillow-python:egg}
networkx
rpath =
${gcc-fortran:location}/lib
${gcc-fortran:location}/lib64
${openblas:location}/lib
[buildout]
extends =
../pkgconfig/buildout.cfg
../autoconf/buildout.cfg
../automake/buildout.cfg
../jbigkit/buildout.cfg
......@@ -11,42 +13,53 @@ extends =
parts =
tesseract
tesseract-traineddata
tesseract-eng-traineddata
tesseract-osd-traineddata
[tesseract]
recipe = slapos.recipe.cmmi
url = http://tesseract-ocr.googlecode.com/files/tesseract-3.01.tar.gz
md5sum = 1ba496e51a42358fb9d3ffe781b2d20a
patch-options =
-p1
patches =
${:_profile_base_location_}/tesseract-3.00-gcc-4.7-build.patch#ca80db3ec489c547b03f3ee48879c1b1
${:_profile_base_location_}/tesseract-3.01-remove-bom.patch#2e691858cb492b7c17d23bf0912b3d24
${:_profile_base_location_}/tesseract-3.01-gcc6-ftbs.patch#f7a6140c0fe390b96fe753a70e9d59fd
url = https://github.com/tesseract-ocr/tesseract/archive/6b250b58121a9858d3e3019a78a6f7d421bd0fc7.tar.gz
md5sum = fdc38148ad8eb1bd0485a217503dd6d5
pkg_config_depends = ${leptonica:location}/lib/pkgconfig:${fontconfig:location}/lib/pkgconfig:${fontconfig:pkg_config_depends}:${lcms2:location}/lib/pkgconfig:${xz-utils:location}/lib/pkgconfig
pre-configure =
libtoolize -f -c
aclocal -I ${libtool:location}/share/aclocal -I config
autoheader -f
automake -c -a -f
aclocal -I${pkgconfig:location}/share/aclocal -I${libtool:location}/share/aclocal
./autogen.sh
autoreconf -ivf
autoconf -Wno-portability
configure-options =
--disable-static
--datarootdir=${tesseract-eng-traineddata:location}
--datarootdir=${tesseract-traineddata:location}
# XXX: tesseract seems not easily configurable at runtime about where to find
# its trained data, so we set its datarootdir above to a controlled location
# tesseract has a non-standard way of testing for leptonica, hence the
# LIBLEPT_HEADERSDIR entry below:
environment =
PATH=${autoconf:location}/bin:${automake:location}/bin:${libtool:location}/bin:${m4:location}/bin:${patch:location}/bin:%(PATH)s
PATH=${pkgconfig:location}/bin:${autoconf:location}/bin:${automake:location}/bin:${libtool:location}/bin:${m4:location}/bin:${patch:location}/bin:%(PATH)s
PKG_CONFIG_PATH=${:pkg_config_depends}
ACLOCAL_ARGS=-I${libtool:location}/share/aclocal
ACLOCAL_PATH=${pkgconfig:location}/share/aclocal:${libtool:location}/share/aclocal
LIBLEPT_HEADERSDIR=${leptonica:location}/include
CPPFLAGS=-I${leptonica:location}/include
LDFLAGS =-L${leptonica:location}/lib -Wl,-rpath=${leptonica:location}/lib -L${jbigkit:location}/lib -Wl,-rpath=${jbigkit:location}/lib -L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib
make-options =
LIBTOOL=${libtool:location}/bin/libtool
[tesseract-traineddata]
location = ${buildout:parts-directory}/${:_buildout_section_name_}
[tesseract-eng-traineddata]
recipe = slapos.recipe.build:download-unpacked
strip-top-level-dir = true
url = http://tesseract-ocr.googlecode.com/files/tesseract-ocr-3.01.eng.tar.gz
md5sum = 89c139a73e0e7b1225809fc7b226b6c9
recipe = slapos.recipe.build:download
destination = ${tesseract-traineddata:location}/tessdata/eng.traineddata
url = https://github.com/tesseract-ocr/tessdata/raw/590567f20dc044f6948a8e2c61afc714c360ad0e/eng.traineddata
md5sum = 57e0df3d84fed9fbf8c7a8e589f8f012
[tesseract-osd-traineddata]
recipe = slapos.recipe.build:download
destination = ${tesseract-traineddata:location}/tessdata/osd.traineddata
url = https://github.com/tesseract-ocr/tessdata/raw/590567f20dc044f6948a8e2c61afc714c360ad0e/osd.traineddata
md5sum = 7611737524efd1ce2dde67eff629bbcf
--- tesseract-3.00/viewer/svutil.cpp.old 2012-04-11 09:34:26.168608940 +0200
+++ tesseract-3.00/viewer/svutil.cpp 2012-04-11 09:34:46.108565692 +0200
@@ -21,6 +21,7 @@
// thread/process creation & synchronization and network connection.
#include <stdio.h>
+#include <unistd.h>
#ifdef WIN32
#include <windows.h>
#include <winsock.h>
commit 58e79a222e12280984ed19ab4d3bcac654e121fa
Author: Arnaud Fontaine <arnaud.fontaine@nexedi.com>
Date: Fri Nov 4 19:10:13 2016 +0900
Fix compilation errors.
diff --git a/cube/char_set.cpp b/cube/char_set.cpp
index 3cf4798..6b29883 100644
--- a/cube/char_set.cpp
+++ b/cube/char_set.cpp
@@ -65,13 +65,13 @@ CharSet *CharSet::Create(TessdataManager *tessdata_manager,
!tessdata_manager->SeekToStart(TESSDATA_UNICHARSET)) {
fprintf(stderr, "Cube ERROR (CharSet::Create): could not find "
"either cube or tesseract unicharset\n");
- return false;
+ return NULL;
}
FILE *charset_fp = tessdata_manager->GetDataFilePtr();
if (!charset_fp) {
fprintf(stderr, "Cube ERROR (CharSet::Create): could not load "
"a unicharset\n");
- return false;
+ return NULL;
}
// If we found a cube unicharset separate from tesseract's, load it and
@@ -84,7 +84,7 @@ CharSet *CharSet::Create(TessdataManager *tessdata_manager,
loaded = char_set->LoadSupportedCharList(charset_fp, NULL);
if (!loaded) {
delete char_set;
- return false;
+ return NULL;
}
char_set->init_ = true;
diff --git a/cube/cube_line_segmenter.cpp b/cube/cube_line_segmenter.cpp
index deee573..3f0b762 100644
--- a/cube/cube_line_segmenter.cpp
+++ b/cube/cube_line_segmenter.cpp
@@ -124,7 +124,7 @@ Pixa *CubeLineSegmenter::CrackLine(Pix *cracked_line_pix,
if (line_con_comps == NULL) {
delete []lines_pixa;
- return false;
+ return NULL;
}
// assign each conn comp to the a line based on its centroid
@@ -142,7 +142,7 @@ Pixa *CubeLineSegmenter::CrackLine(Pix *cracked_line_pix,
delete []lines_pixa;
boxaDestroy(&line_con_comps);
pixaDestroy(&line_con_comps_pix);
- return false;
+ return NULL;
}
}
@@ -413,14 +413,14 @@ Pix *CubeLineSegmenter::Pixa2Pix(Pixa *pixa, Box **dest_box,
(*dest_box) = boxCreate(min_x, min_y, max_x - min_x, max_y - min_y);
if ((*dest_box) == NULL) {
- return false;
+ return NULL;
}
// create the union pix
Pix *union_pix = pixCreate((*dest_box)->w, (*dest_box)->h, img_->d);
if (union_pix == NULL) {
boxDestroy(dest_box);
- return false;
+ return NULL;
}
// create a pix corresponding to the union of all pixs
diff --git a/cube/cube_object.cpp b/cube/cube_object.cpp
index 48bce64..b9a7113 100644
--- a/cube/cube_object.cpp
+++ b/cube/cube_object.cpp
@@ -165,7 +165,7 @@ WordAltList *CubeObject::Recognize(LangModel *lang_mod, bool word_mode) {
if (deslanted_beam_obj_ == NULL) {
fprintf(stderr, "Cube ERROR (CubeObject::Recognize): could not "
"construct deslanted BeamSearch\n");
- return false;
+ return NULL;
}
}
diff --git a/cube/word_list_lang_model.cpp b/cube/word_list_lang_model.cpp
index 18f85c1..0f7f562 100644
--- a/cube/word_list_lang_model.cpp
+++ b/cube/word_list_lang_model.cpp
@@ -74,7 +74,7 @@ LangModEdge **WordListLangModel::GetEdges(CharAltList *alt_list,
// initialize if necessary
if (init_ == false) {
if (Init() == false) {
- return false;
+ return NULL;
}
}
The patch below removes a utf-8 BOM mark.
Avoid touching it as the BOM is invisible, and copy/pasting might not work.
It is needed because old compilers treat the BOM as garbage instead of
whitespace.
--- tesseract-3.01/ccutil/strngs.h.orig 2012-05-24 15:13:22.743808379 +0200
+++ tesseract-3.01/ccutil/strngs.h 2012-05-24 15:16:54.468858282 +0200
@@ -1,4 +1,4 @@
-/**********************************************************************
+/**********************************************************************
* File: strngs.h (Formerly strings.h)
* Description: STRING class definition.
* Author: Ray Smith
......@@ -134,14 +134,14 @@ class Password(object):
if self.storage_path:
try:
with open(self.storage_path) as f:
passwd = f.read()
passwd = f.read().strip('\n')
except IOError as e:
if e.errno != errno.ENOENT:
raise
if not passwd:
passwd = self.generatePassword(int(options.get('bytes', '8')))
self.update = self.install
self.passwd = passwd.strip('\n')
self.passwd = passwd
# Password must not go into .installed file, for 2 reasons:
# security of course but also to prevent buildout to always reinstall.
def get(option, *args, **kw):
......
......@@ -51,7 +51,7 @@ pycrypto = 2.6.1
pycurl = 7.43.0
slapos.recipe.download = 1.0
slapos.recipe.template = 4.3
slapos.toolbox = 0.76
slapos.toolbox = 0.77
smmap = 0.9.0
# Required by:
......
......@@ -11,7 +11,7 @@ plone.recipe.command = 1.1
pycrypto = 2.6.1
rdiff-backup = 1.0.5+SlapOSPatched001
slapos.recipe.template = 4.3
slapos.toolbox = 0.76
slapos.toolbox = 0.77
smmap = 0.9.0
numpy = 1.11.2
pyasn1 = 0.2.3
......
......@@ -82,3 +82,7 @@ packages +=
dh-autoreconf pkg-config doxygen maven xmlto
# hellorina (shouldn't parts like lxml-python depend on the python of the SR?)
python-dev
[versions]
erp5.util = 0.4.51
slapos.recipe.template = 4.3
Generally things to be done with ``caddy-frontend``:
* tests: add assertion with results of promises in etc/promise for each partition
* generated files: ``| trim`` values (like ``slave_password[slave]`` in ``templates/template-log-access.conf.in``) in generated configuration files to have them renfered correctly
* check the whole frontend slave snippet with ``caddy -validate`` during buildout run, and reject if does not pass validation
* ``apache-ca-certificate`` shall be merged with ``apache-certificate``
* ``apache-ca-certificate`` shall be appended to ``apache-certificate`` if not already there
......
......@@ -14,7 +14,7 @@
# not need these here).
[template]
filename = instance.cfg
md5sum = 906e5bd66b1265b8109a86b6ab46e91f
md5sum = b73505ae80d6325a244f5094f8edc0ae
[template-apache-frontend]
filename = instance-apache-frontend.cfg
......@@ -22,7 +22,7 @@ md5sum = b170d0987563b481eb71cf705c3658ab
[template-apache-replicate]
filename = instance-apache-replicate.cfg.in
md5sum = 7f15b5745eda8e1f02d4bf7d886dcdad
md5sum = 27e98547061bd81e5f84cb7dd21b683b
[template-slave-list]
filename = templates/apache-custom-slave-list.cfg.in
......@@ -44,6 +44,10 @@ md5sum = 6689d96fc18d9aad78d77fe87770d4da
filename = templates/apache-custom-slave-list.cfg.in
md5sum = fb6c93f42f232e381174a5951c3fc222
[caddy-backend-url-validator]
filename = templates/caddy-backend-url-validator.in
md5sum = 0979a03476e86bf038516c9565dadc17
[template-not-found-html]
filename = templates/notfound.html
md5sum = f20d6c3d2d94fb685f8d26dfca1e822b
......
......@@ -46,6 +46,13 @@ url = ${:_profile_base_location_}/instance-apache-frontend.cfg
output = ${buildout:directory}/template-caddy-frontend.cfg
mode = 0644
[caddy-backend-url-validator]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/templates/${:filename}
filename = caddy-backend-url-validator.in
output = ${buildout:directory}/caddy-backend-url-validator
mode = 0750
[template-caddy-replicate]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/instance-apache-replicate.cfg.in
......
......@@ -72,10 +72,25 @@ context =
{% for slave in slave_instance_list %}
{# BBB: apache_custom_https AND apache_custom_http #}
{% if not ((slave.has_key('caddy_custom_http') or slave.has_key('apache_custom_http') or slave.has_key('caddy_custom_https') or slave.has_key('apache_custom_https')) and not slave.get('slave_reference') in authorized_slave_string) %}
{% set slave_ok = True %}
{% if slave.get('url') %}
{% if subprocess_module.call([caddy_backend_url_validator, slave['url']]) == 1 %}
{% set slave_ok = False %}
{% endif %}
{% endif %}
{% if slave.get('https-url') %}
{% if subprocess_module.call([caddy_backend_url_validator, slave['https-url']]) == 1 %}
{% set slave_ok = False %}
{% endif %}
{% endif %}
{% if slave_ok %}
{% do authorized_slave_list.append(slave) %}
{% else %}
{% do rejected_slave_list.append(slave.get('slave_reference')) %}
{% endif %}
{% else %}
{% do rejected_slave_list.append(slave.get('slave_reference')) %}
{% endif %}
{% endfor %}
[monitor-instance-parameter]
......
......@@ -43,6 +43,8 @@ template = ${template-caddy-replicate:target}
filename = instance-caddy-replicate.cfg
extensions = jinja2.ext.do
extra-context =
import subprocess_module subprocess
raw caddy_backend_url_validator ${caddy-backend-url-validator:output}
raw template_publish_slave_information ${template-replicate-publish-slave-information:target}
# Must match the key id in [switch-softwaretype] which uses this section.
raw software_type RootSoftwareInstance-default-custom-personal-replicate
......
......@@ -11,7 +11,7 @@ plone.recipe.command = 1.1
pycrypto = 2.6.1
rdiff-backup = 1.0.5+SlapOSPatched001
slapos.recipe.template = 4.3
slapos.toolbox = 0.76
slapos.toolbox = 0.77
smmap = 0.9.0
numpy = 1.11.2
pyasn1 = 0.2.3
......
#!${dash:location}/bin/dash
config="https://example.com {\n proxy / $1 {\n }\n}"
echo -e $config | ${caddy:output} -conf stdin -validate > /dev/null 2>&1
......@@ -2692,3 +2692,91 @@ class TestRe6stVerificationUrlSlave(SlaveHttpFrontendTestCase,
'URL="some-re6st-verification-url"' in
open(re6st_connectivity_promise_list[0]).read()
)
class TestMalformedBackenUrlSlave(SlaveHttpFrontendTestCase,
TestDataMixin):
@classmethod
def getInstanceParameterDict(cls):
return {
'domain': 'example.com',
'nginx-domain': 'nginx.example.com',
'public-ipv4': utils.LOCAL_IPV4,
'apache-certificate': open('wildcard.example.com.crt').read(),
'apache-key': open('wildcard.example.com.key').read(),
'port': HTTPS_PORT,
'plain_http_port': HTTP_PORT,
'nginx_port': NGINX_HTTPS_PORT,
'plain_nginx_port': NGINX_HTTP_PORT,
'monitor-httpd-port': MONITOR_HTTPD_PORT,
'-frontend-config-1-monitor-httpd-port': MONITOR_F1_HTTPD_PORT,
}
@classmethod
def getSlaveParameterDictDict(cls):
return {
'empty': {
},
'url': {
'url': "https://[fd46::c2ae]:!py!u'123123'",
},
'https-url': {
'https-url': "https://[fd46::c2ae]:!py!u'123123'",
}
}
def test_master_partition_state(self):
parameter_dict = self.computer_partition.getConnectionParameterDict()
self.assertKeyWithPop('monitor-setup-url', parameter_dict)
expected_parameter_dict = {
'monitor-base-url': None,
'domain': 'example.com',
'accepted-slave-amount': '1',
'rejected-slave-amount': '2',
'slave-amount': '3',
'rejected-slave-list': '["_url", "_https-url"]'}
self.assertEqual(
expected_parameter_dict,
parameter_dict
)
def test_empty(self):
parameter_dict = self.slave_connection_parameter_dict_dict[
'empty']
self.assertLogAccessUrlWithPop(parameter_dict, 'empty')
self.assertEqual(
parameter_dict,
{
'domain': 'empty.example.com',
'replication_number': '1',
'url': 'http://empty.example.com',
'site_url': 'http://empty.example.com',
'secure_access': 'https://empty.example.com',
'public-ipv4': utils.LOCAL_IPV4,
}
)
result = self.fakeHTTPSResult(
parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path')
self.assertEqual(
utils.der2pem(result.peercert),
open('wildcard.example.com.crt').read())
self.assertEqual(result.status_code, no_backend_response_code)
def test_url(self):
parameter_dict = self.slave_connection_parameter_dict_dict[
'url'].copy()
self.assertEqual(
parameter_dict, {}
)
def test_https_url(self):
parameter_dict = self.slave_connection_parameter_dict_dict[
'https-url'].copy()
self.assertEqual(
parameter_dict, {}
)
TestMalformedBackenUrlSlave-0/var/log/monitor-httpd-error.log
TestMalformedBackenUrlSlave-1/var/log/frontend-access.log
TestMalformedBackenUrlSlave-1/var/log/frontend-error.log
TestMalformedBackenUrlSlave-1/var/log/httpd/_empty_access_log
TestMalformedBackenUrlSlave-1/var/log/httpd/_empty_error_log
TestMalformedBackenUrlSlave-1/var/log/monitor-httpd-error.log
TestMalformedBackenUrlSlave-1/var/log/nginx-access.log
TestMalformedBackenUrlSlave-1/var/log/nginx-error.log
TestMalformedBackenUrlSlave-1/var/log/trafficserver/manager.log
TestMalformedBackenUrlSlave-1/var/log/trafficserver/traffic.out
\ No newline at end of file
TestMalformedBackenUrlSlave-0/var/run/monitor-httpd.pid
TestMalformedBackenUrlSlave-0/var/run/monitor/monitor-bootstrap.pid
TestMalformedBackenUrlSlave-1/var/run/caddy_configuration.signature
TestMalformedBackenUrlSlave-1/var/run/httpd.pid
TestMalformedBackenUrlSlave-1/var/run/monitor-httpd.pid
TestMalformedBackenUrlSlave-1/var/run/monitor/monitor-bootstrap.pid
TestMalformedBackenUrlSlave-1/var/run/ncaddy_configuration.signature
TestMalformedBackenUrlSlave-1/var/run/nginx.pid
TestMalformedBackenUrlSlave-1/etc/monitor-promise/check-_empty-error-log-last-day
TestMalformedBackenUrlSlave-1/etc/monitor-promise/check-_empty-error-log-last-hour
\ No newline at end of file
TestMalformedBackenUrlSlave-0/etc/promise/check-free-disk-space
TestMalformedBackenUrlSlave-0/etc/promise/monitor-http-frontend
TestMalformedBackenUrlSlave-0/etc/promise/monitor-httpd-listening-on-tcp
TestMalformedBackenUrlSlave-0/etc/promise/promise-monitor-httpd-is-process-older-than-dependency-set
TestMalformedBackenUrlSlave-1/etc/promise/caddy-frontend-is-running-actual-software-release
TestMalformedBackenUrlSlave-1/etc/promise/caddy_cached
TestMalformedBackenUrlSlave-1/etc/promise/caddy_frontend_ipv4_http
TestMalformedBackenUrlSlave-1/etc/promise/caddy_frontend_ipv4_https
TestMalformedBackenUrlSlave-1/etc/promise/caddy_frontend_ipv6_http
TestMalformedBackenUrlSlave-1/etc/promise/caddy_frontend_ipv6_https
TestMalformedBackenUrlSlave-1/etc/promise/caddy_ssl_cached
TestMalformedBackenUrlSlave-1/etc/promise/check-free-disk-space
TestMalformedBackenUrlSlave-1/etc/promise/frontend-caddy-configuration-promise
TestMalformedBackenUrlSlave-1/etc/promise/monitor-http-frontend
TestMalformedBackenUrlSlave-1/etc/promise/monitor-httpd-listening-on-tcp
TestMalformedBackenUrlSlave-1/etc/promise/nginx-configuration-promise
TestMalformedBackenUrlSlave-1/etc/promise/nginx_frontend_ipv4_http
TestMalformedBackenUrlSlave-1/etc/promise/nginx_frontend_ipv4_https
TestMalformedBackenUrlSlave-1/etc/promise/nginx_frontend_ipv6_http
TestMalformedBackenUrlSlave-1/etc/promise/nginx_frontend_ipv6_https
TestMalformedBackenUrlSlave-1/etc/promise/promise-monitor-httpd-is-process-older-than-dependency-set
TestMalformedBackenUrlSlave-1/etc/promise/promise-nginx-is-process-older-than-dependency-set
TestMalformedBackenUrlSlave-1/etc/promise/re6st-connectivity
TestMalformedBackenUrlSlave-1/etc/promise/trafficserver-cache-availability
TestMalformedBackenUrlSlave-1/etc/promise/trafficserver-port-listening
\ No newline at end of file
TestMalformedBackenUrlSlave-0:bootstrap-monitor EXITED
TestMalformedBackenUrlSlave-0:certificate_authority-on-watch RUNNING
TestMalformedBackenUrlSlave-0:crond RUNNING
TestMalformedBackenUrlSlave-0:monitor-httpd-graceful EXITED
TestMalformedBackenUrlSlave-0:monitor-httpd-on-watch RUNNING
TestMalformedBackenUrlSlave-1:6tunnel-11080-on-watch RUNNING
TestMalformedBackenUrlSlave-1:6tunnel-11443-on-watch RUNNING
TestMalformedBackenUrlSlave-1:6tunnel-12080-on-watch RUNNING
TestMalformedBackenUrlSlave-1:6tunnel-12443-on-watch RUNNING
TestMalformedBackenUrlSlave-1:6tunnel-26011-on-watch RUNNING
TestMalformedBackenUrlSlave-1:6tunnel-26012-on-watch RUNNING
TestMalformedBackenUrlSlave-1:bootstrap-monitor EXITED
TestMalformedBackenUrlSlave-1:certificate_authority-on-watch RUNNING
TestMalformedBackenUrlSlave-1:crond-on-watch RUNNING
TestMalformedBackenUrlSlave-1:frontend-caddy-safe-graceful EXITED
TestMalformedBackenUrlSlave-1:frontend-nginx-safe-graceful EXITED
TestMalformedBackenUrlSlave-1:frontend_caddy-on-watch RUNNING
TestMalformedBackenUrlSlave-1:frontend_nginx-on-watch RUNNING
TestMalformedBackenUrlSlave-1:monitor-httpd-graceful EXITED
TestMalformedBackenUrlSlave-1:monitor-httpd-on-watch RUNNING
TestMalformedBackenUrlSlave-1:trafficserver-on-watch RUNNING
TestMalformedBackenUrlSlave-1:trafficserver-reload EXITED
watchdog:watchdog RUNNING
......@@ -40,3 +40,6 @@ recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/${:_buildout_section_name_}
mode = 755
md5sum = 78b77a6bda9958f547f7d89b747731e3
[versions]
slapos.recipe.template = 4.3
......@@ -273,6 +273,7 @@ parts =
httpd-promise
publish-connection-information
directory-doc
monitor-htpasswd
# Complete parts with sections
{{ part_list | join('\n ') }}
......
......@@ -21,6 +21,7 @@ offline = true
# += because we need to take up parts (like instance-custom, slapmonitor etc) from the profile we extended
parts +=
monitor-htpasswd
{{ parts.replicate("kvm", backup_amount) }}
publish-connection-information
kvm-frontend-url-promise
......
......@@ -108,7 +108,7 @@ recipe = hexagonit.recipe.download
ignore-existing = true
url = ${:_profile_base_location_}/instance-kvm-cluster.cfg.jinja2.in
mode = 644
md5sum = ba3337b3678ed9d3578cc88749c5cd13
md5sum = d9fe920d31f1ef0e377aa768ccd24f4c
download-only = true
on-update = true
......@@ -117,7 +117,7 @@ recipe = hexagonit.recipe.download
ignore-existing = true
url = ${:_profile_base_location_}/instance-kvm-resilient.cfg.jinja2
mode = 644
md5sum = 93e7143b46c6136b7cafe888fac90aba
md5sum = 1095968487282784a735735aa1b37d35
download-only = true
on-update = true
......@@ -255,7 +255,7 @@ context =
# XXX - use websockify = 0.5.1 for compatibility with kvm frontend
websockify = 0.5.1
slapos.toolbox = 0.76
slapos.toolbox = 0.77
erp5.util = 0.4.51
apache-libcloud = 1.1.0
collective.recipe.environment = 0.2.0
......
......@@ -89,11 +89,6 @@ command-line =
${monitor-directory:bin}/python {{ monitor_collect_csv_dump }} --output_folder ${monitor-directory:consumption}
wrapper-path = ${monitor-directory:reports}/monitor-collect-csv-dump
[monitor-check-cpu-usage]
recipe = slapos.cookbook:wrapper
command-line = ${monitor-directory:bin}/python {{ monitor_check_system_health }} cpu ${init-monitor-parameters:cpu-load-file}
wrapper-path = ${directory:promises}/system-CPU-load-check
[monitor-check-memory-usage]
recipe = slapos.cookbook:wrapper
command-line = {{ buildout_bin}}/check-computer-memory
......@@ -102,20 +97,22 @@ command-line = {{ buildout_bin}}/check-computer-memory
--unit percent
wrapper-path = ${directory:promises}/check-computer-memory-usage
[monitor-check-cpu-usage]
recipe = slapos.cookbook:promise.plugin
eggs =
slapos.toolbox
file = ${monitor-conf-parameters:promise-output-file}
content =
from slapos.promise.plugin.check_server_cpu_load import RunPromise
output = ${directory:plugins}/system-CPU-load-check.py
mode = 600
config-cpu-load-threshold = ${slap-parameter:cpu-load-threshold}
[publish-connection-information]
recipe = slapos.cookbook:publish
monitor-setup-url = https://monitor.app.officejs.com/#page=settings_configurator&url=${monitor-publish-parameters:monitor-url}&username=${monitor-publish-parameters:monitor-user}&password=${monitor-publish-parameters:monitor-password}
server_log_url = ${monitor-publish-parameters:monitor-base-url}/${slap-configuration:private-hash}/
[init-monitor-parameters]
recipe = plone.recipe.command
cpu-load-file = ${directory:monitor}/cpu-load-tolerance
mem-free-file = ${directory:monitor}/mem-free-limit
command =
if [ ! -s "${:cpu-load-file}" ]; then
echo ${slap-parameter:cpu-load-threshold} > ${:cpu-load-file}
fi
[slap-configuration]
recipe = slapos.cookbook:slapconfiguration.serialised
computer = ${slap-connection:computer-id}
......@@ -127,5 +124,6 @@ private-hash = ${pwgen:passwd}${pwgen32:passwd}
frontend-domain =
[slap-parameter]
cpu-load-threshold = 2.0
# Max cpu load for one core on server
cpu-load-threshold = 3.0
memory-percent-threshold = 96
......@@ -22,7 +22,6 @@ context = key develop_eggs_directory buildout:develop-eggs-directory
raw monitor_template_output ${monitor-template:output}
raw network_benck_cfg_output ${network-bench-cfg:output}
raw monitor_collect_csv_dump ${monitor-collect-csv-dump:output}
raw monitor_check_system_health ${monitor-system-health:output}
mode = 0644
[instance-base-distributor]
......
#!/usr/bin/env python
import subprocess
import os
import re
import json
cpu_command_list = ['top', '-n', '1', '-b']
mem_command_list = ['free', '-m']
head_command_list = ['head', '-n', '5']
cpu_core_cmd_list = ['nproc']
def cpu_usage(tolerance=1.5):
# tolerance=1.5 => accept up to 1.5 =150% CPU load
uptime_result = subprocess.check_output(['uptime'])
line = uptime_result.strip().split(' ')
load, load5, long_load = line[-3:]
core_count = int(subprocess.check_output(cpu_core_cmd_list).strip())
threshold = core_count * tolerance
if float(long_load) > threshold:
# display top statistics
top = subprocess.Popen(cpu_command_list, stdout=subprocess.PIPE)
result = subprocess.check_output(head_command_list, stdin=top.stdout)
message = "CPU load is high: %s %s %s\n\n" % (load, load5, long_load)
message += result
return message
def check_last_result(file, last_value, threshold=7.0, elt_count=5):
mem_average = 0.0
value_list = []
if os.path.exists(file):
with open(file) as f:
values = f.read()
value_list = values.split(' ')
size = len(value_list)
value_list.append(str(last_value))
if size >= elt_count:
while len(value_list) > elt_count:
value_list.pop(0)
# calculate average
average = sum([float(l) for l in value_list])/(size * 1.0)
if average < threshold:
mem_average = round(average, 2)
else:
value_list.append(str(last_value))
with open(file, 'w') as f:
f.write(' '.join(value_list))
return mem_average
def memory_usage(storage_file, threshold=7.0, elt_count=5):
mem_stats = subprocess.check_output(mem_command_list)
result_list = mem_stats.split('\n')
usage = re.sub('\s+', ' ', result_list[1])
usage_real = re.sub('\s+', ' ', result_list[2])
usage_list = usage.split(' ')
mem_total = float(usage_list[1])
mem_free = float(usage_real.split(' ')[-1])
if mem_free == 0.0:
mem_available = 0.0
else:
mem_available = round(mem_free * 100 / (mem_total * 1.0), 2)
average = check_last_result(
storage_file,
mem_available,
threshold=threshold,
elt_count=elt_count)
if average != 0.0 and average < threshold:
# mem used at (threshold)% at least
message = "Memory usage is high. %s%% is available (%s%% for last %s minutes).\n\n" % (
mem_available, average, elt_count)
message += mem_stats
return message
swap_usage = re.sub('\s+', ' ', result_list[3])
swap_usage_list = swap_usage.split(' ')
swap_total = float(swap_usage_list[1])
swap_free = float(swap_usage_list[3])
if swap_total > 1:
if swap_free == 0.0:
swap_available = 0.0
else:
swap_available = round(swap_free * 100 / (swap_total * 1.0), 2) * 100
if swap_available < threshold*1.7:
message = "Memory SWAP usage is high. %s%% is available.\n\n" % swap_available
message += mem_stats
return message
if __name__ == '__main__':
if len(sys.argv) < 2:
print "Usage: %s [cpu | mem] CONFIG_FILE [BASE_DIR]" % os.path.basename(sys.argv[0])
exit(2)
check_type = sys.argv[1]
threshold = None
if len(sys.argv) >= 3:
config_file = sys.argv[2]
if os.path.exists(config_file):
with open(config_file) as f:
try:
threshold = float(f.read())
if not threshold > 0:
threshold = None
except ValueError:
pass
if check_type == "cpu":
result = cpu_usage(threshold or 1.5)
if result:
print result
exit(1)
elif check_type == "mem":
directory = ""
if len(sys.argv) >= 4:
directory = sys.argv[3]
if not os.path.exists(directory) or not os.path.isdir(directory):
directory = os.getcwd()
storage_file = os.path.join(directory, 'mem-usage.mo')
result = memory_usage(storage_file, threshold=(threshold or 4.0), elt_count=10)
if result:
print result
exit(1)
else:
exit(3)
exit(0)
\ No newline at end of file
......@@ -21,14 +21,14 @@ parts =
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance.cfg
output = ${buildout:directory}/template.cfg
md5sum = 641c5916739f78171c616af00fe974a2
md5sum = 1b7d2d097f208f6641bf98a17df079c8
mode = 0644
[template-monitor]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/instance-monitor.cfg.jinja2
destination = ${buildout:directory}/template-base-monitor.cfg
md5sum = 79125819f20f4f18a301b806daed2ceb
md5sum = ef3297619e1fc2a5a8d1b0546c1a0db2
mode = 0644
[template-monitor-distributor]
......@@ -59,13 +59,6 @@ filename = collect_csv_dump.py
output = ${:destination}/${:filename}
md5sum = cad2402bbd21907cfed6bc5af8c5d3ab
[monitor-system-health]
<= monitor-template-script
url = ${:_profile_base_location_}/script/${:filename}
filename = check_system_health.py
output = ${:destination}/${:filename}
md5sum = 7eb74a0be4995c6a1015a9a1eb6874c6
[extra-eggs]
<= monitor-eggs
interpreter = pythonwitheggs
......
......@@ -55,6 +55,8 @@ eggs = neoppod[admin, ctl, master, storage-mysqldb]
ZODB
zope.testing
zodbtools
coverage
setproctitle
patch-binary = ${patch:location}/bin/patch
ZEO-patch-options = -p1
ZEO-patches =
......@@ -121,15 +123,18 @@ md5sum = 87d18c7021e4d43756813a83c9da5e97
BTrees = 4.4.1
ZODB = 4.4.5
apache-libcloud = 1.5.0
coverage = 4.5.1
ecdsa = 0.13
gitdb2 = 2.0.0
msgpack = 0.5.6
msgpack-python = 0.5.6
mysqlclient = 1.3.12
persistent = 4.2.3
pycrypto = 2.6.1
pycurl = 7.43.0
setproctitle = 1.1.10
slapos.recipe.template = 4.3
slapos.toolbox = 0.76
slapos.toolbox = 0.77
smmap2 = 2.0.1
transaction = 1.7.0
zodbpickle = 0.6.0
......
......@@ -46,7 +46,7 @@ mode = 0644
[versions]
slapos.recipe.template = 4.3
slapos.toolbox = 0.76
slapos.toolbox = 0.77
dnspython = 1.15.0
PyRSS2Gen = 1.1
erp5.util = 0.4.51
......
......@@ -110,7 +110,7 @@ filename = registry-run.in
md5sum = 0bf4f2c03e06b55c6c6cc55fa33e65d6
[versions]
re6stnet = 0.494
re6stnet = 0.501
apache-libcloud = 0.17.0
ecdsa = 0.13
gitdb = 0.6.4
......@@ -118,7 +118,7 @@ plone.recipe.command = 1.1
pycrypto = 2.6.1
pycurl = 7.43.0
slapos.recipe.template = 4.3
slapos.toolbox = 0.76
slapos.toolbox = 0.77
smmap = 0.9.0
dnspython = 1.15.0
erp5.util = 0.4.51
......
......@@ -175,7 +175,7 @@ Pygments = 2.1.3
slapos.recipe.build = 0.36
slapos.recipe.cmmi = 0.7
slapos.recipe.template = 4.3
slapos.toolbox = 0.76
slapos.toolbox = 0.77
# All depencies should be pinned.
apache-libcloud = 2.3.0
......
......@@ -26,7 +26,7 @@ md5sum = e033845c9c24e4bb20caeedf19f9628a
[instance-runner-import]
filename = instance-runner-import.cfg.in
md5sum = 5cfa49bcf20612844e1c50a85740d0b3
md5sum = 7a879739afe55320ee96409bcc8a52ab
[template-runner-export-script]
filename = template/runner-export.sh.jinja2
......@@ -34,11 +34,11 @@ md5sum = 98ce179badc6af5979a64a7c3d0a2ceb
[instance-runner-export]
filename = instance-runner-export.cfg.in
md5sum = e0e62859fc00207e40af6acb402665fb
md5sum = 1a812a06cc02bb11636009f4ec043d54
[template-resilient]
filename = instance-resilient.cfg.jinja2
md5sum = a902b84ac7d1e29a7fdb06cbc7dec150
md5sum = 8ed180de711d207a540d0acb539b2536
[template_nginx_conf]
filename = nginx_conf.in
......
......@@ -28,6 +28,7 @@ offline = true
# += because we need to take up parts (like instance-custom, slapmonitor etc) from the profile we extended
parts +=
monitor-htpasswd
{{ parts.replicate("runner", number_of_instances + 1) }}
publish-connection-information
......
......@@ -41,6 +41,11 @@ parts +=
monitor-check-resilient-feed-file
monitor-check-webrunner-internal-instance
[directory]
recipe = slapos.cookbook:mkdirectory
# XXX - keep srv path with slash at the end.
srv = ${:home}/srv/
[proxy-free-port]
recipe = slapos.cookbook:free_port
minimum = 49980
......
......@@ -34,6 +34,12 @@ parts +=
monitor-base
[directory]
recipe = slapos.cookbook:mkdirectory
# XXX - keep srv path with slash at the end.
srv = ${:home}/srv/
# For the needs of importer, we run the full slaprunner
# In case both exporter and importer (aka main instance and clone instance)
# run with the same IP (usually for testing purposes),
......
......@@ -171,7 +171,7 @@ prettytable = 0.7.2
pycurl = 7.43.0
slapos.recipe.template = 4.3
collective.recipe.environment = 0.2.0
slapos.toolbox = 0.76
slapos.toolbox = 0.77
smmap = 0.9.0
lockfile = 0.12.2
......
......@@ -95,7 +95,7 @@ futures = 3.1.1
gitdb2 = 2.0.2
gunicorn = 19.7.1
slapos.recipe.template = 4.3
slapos.toolbox = 0.76
slapos.toolbox = 0.77
smmap2 = 2.0.3
# Required by:
......
......@@ -39,6 +39,8 @@ extends =
../../component/python-ldap-python/buildout.cfg
../../component/rdiff-backup/buildout.cfg
../../component/scikit-learn/buildout.cfg
../../component/scikit-image/buildout.cfg
../../component/PyWavelets/buildout.cfg
../../component/stunnel/buildout.cfg
../../component/subversion/buildout.cfg
../../component/tesseract/buildout.cfg
......@@ -92,6 +94,8 @@ parts +=
kumo
w3-validator
tesseract
tesseract-eng-traineddata
tesseract-osd-traineddata
scipy
ocropy
hookbox
......@@ -448,6 +452,7 @@ eggs = ${neoppod:eggs}
${pycrypto-python:egg}
${scipy:egg}
${scikit-learn:egg}
${scikit-image:egg}
sympy
${h5py:egg}
openpyxl
......@@ -497,6 +502,9 @@ eggs = ${neoppod:eggs}
xfw
jsonschema
selenium
pytesseract
decorator
networkx
# Needed for checking ZODB Components source code
pylint
pytracemalloc
......@@ -627,7 +635,6 @@ python-magic = 0.4.12+SlapOSPatched001
# use newer version than specified in ZTK
PasteDeploy = 1.5.2
argparse = 1.4.0
coverage = 4.3.4
zope.dottedname = 4.1.0
# test_UserManagerInterfaces in testERP5Security fails with 1.10.0.
......@@ -657,7 +664,7 @@ zope.app.testing = 3.8.1
# Pinned versions
APacheDEX = 1.6.2
Pillow = 4.0.0
Pillow = 5.2.0
Products.CMFActionIcons = 2.1.3
Products.DCWorkflowGraph = 0.4.1
# Products.ExternalEditor 2.0.0's dtml is not based on Zope2 OFS's one.
......@@ -725,7 +732,14 @@ validictory = 1.1.0
xfw = 0.10
xupdate-processor = 0.4
selenium = 3.8.0
scikit-image = 0.14.0
PyWavelets = 0.5.2
networkx = 2.1
pytesseract = 0.2.2
zbarlight = 2.0
cloudpickle = 0.5.3
dask = 0.18.1
toolz = 0.9.0
# Required by:
# Products.CMFCore==2.2.10
......@@ -782,3 +796,4 @@ unidiff = 0.5.5
# Required by:
# deepdiff = 3.3.0
jsonpickle = 0.9.6
decorator = 4.3.0
......@@ -121,6 +121,6 @@ depends =
PyRSS2Gen = 1.1
cns.recipe.symlink = 0.2.3
pycurl = 7.43.0
slapos.toolbox = 0.76
slapos.toolbox = 0.77
pyasn1 = 0.3.7
......@@ -26,7 +26,7 @@ md5sum = c6c11db5372150019debb1ce519b907d
[template-pull-backup]
filename = instance-pull-backup.cfg.in
md5sum = 85e777bd349a5c881d36e747882aee8a
md5sum = cda4bbedb3ec014ba0311629dd003b3a
[template-replicated]
filename = template-replicated.cfg.in
......
......@@ -277,11 +277,23 @@ monitor-username = admin
recipe = slapos.recipe.template:jinja2
template = inline:
#!${dash:location}/bin/dash
# only check integrity if pull is not running
latest_item=$(ls -t $${pbs:status-item-directory} | head -n1)
if [ ! -z "$latest_item" ]; then
pbs_result=$(cat "$${pbs:status-item-directory}/$latest_item" | python -c "import sys, json; print json.load(sys.stdin)['title']" 2>/dev/null)
if [ "$?" -eq 0 ]; then
echo $pbs_result | egrep "pull_raw\s*:\s*STARTED" > /dev/null
if [ "$?" -eq 0 ]; then
echo "Skipped, PBS pull is running.";
exit 0;
fi
fi
fi
# Raise an error if signatures are different
# Error cannot be deduced if files do not exist
cd $${directory:pbs-backup}
if [ ! -f "proof.signature" ]; then exit 0; fi
backup_signature=$(find . -maxdepth 2 -name backup.signature)
backup_signature=$(find . -maxdepth 2 -name backup.signature -not -path "./tmp/*")
if [ -z "$backup_signature" ]; then
exit 0;
else
......
......@@ -134,7 +134,7 @@ pytz = 2016.10
requests = 2.13.0
six = 1.10.0
slapos.cookbook = 1.0.66
slapos.core = 1.4.8
slapos.core = 1.4.9
slapos.extension.strip = 0.4
slapos.libnetworkcache = 0.15
slapos.rebootstrap = 4.1
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment