Commit 7291d0fc authored by Rafael Monnerat's avatar Rafael Monnerat

Update Release Candidate

parents 7427629c c53d0f43
...@@ -7,14 +7,15 @@ extends = ...@@ -7,14 +7,15 @@ extends =
../bzip2/buildout.cfg ../bzip2/buildout.cfg
../zlib/buildout.cfg ../zlib/buildout.cfg
../bzip2/buildout.cfg ../bzip2/buildout.cfg
../gperf/buildout.cfg
parts = parts =
fontconfig fontconfig
[fontconfig] [fontconfig]
recipe = slapos.recipe.cmmi recipe = slapos.recipe.cmmi
url = http://fontconfig.org/release/fontconfig-2.12.1.tar.bz2 url = http://fontconfig.org/release/fontconfig-2.12.6.tar.bz2
md5sum = b5af5a423ee3b5cfc34846838963c058 md5sum = 733f5e2371ca77b69707bd7b30cc2163
pkg_config_depends = ${freetype:pkg_config_depends}:${freetype:location}/lib/pkgconfig:${libxml2:location}/lib/pkgconfig pkg_config_depends = ${freetype:pkg_config_depends}:${freetype:location}/lib/pkgconfig:${libxml2:location}/lib/pkgconfig
# XXX-Cedric : should we use --with-add-fonts={somefont:location}/share,{someotherfont:location}/share? # XXX-Cedric : should we use --with-add-fonts={somefont:location}/share,{someotherfont:location}/share?
configure-options = configure-options =
...@@ -23,7 +24,7 @@ configure-options = ...@@ -23,7 +24,7 @@ configure-options =
--enable-libxml2 --enable-libxml2
--with-default-fonts=${fonts:location} --with-default-fonts=${fonts:location}
environment = environment =
PATH=${pkgconfig:location}/bin:%(PATH)s PATH=${pkgconfig:location}/bin:${gperf:location}/bin:%(PATH)s
PKG_CONFIG_PATH=${:pkg_config_depends} PKG_CONFIG_PATH=${:pkg_config_depends}
CPPFLAGS=-I${zlib:location}/include -I${bzip2:location}/include CPPFLAGS=-I${zlib:location}/include -I${bzip2:location}/include
LDFLAGS=-L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib -L${bzip2:location}/lib -Wl,-rpath=${bzip2:location}/lib LDFLAGS=-L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib -L${bzip2:location}/lib -Wl,-rpath=${bzip2:location}/lib
...@@ -12,8 +12,8 @@ parts = haproxy ...@@ -12,8 +12,8 @@ parts = haproxy
[haproxy] [haproxy]
recipe = slapos.recipe.cmmi recipe = slapos.recipe.cmmi
url = http://www.haproxy.org/download/1.7/src/haproxy-1.7.9.tar.gz url = http://www.haproxy.org/download/1.8/src/haproxy-1.8.1.tar.gz
md5sum = a2bbbdd45ffe18d99cdcf26aa992f92d md5sum = e42892d4b6ee33200fccaa1d81837e49
configure-command = true configure-command = true
# If the system is running on Linux 2.6, we use "linux26" as the TARGET, # If the system is running on Linux 2.6, we use "linux26" as the TARGET,
# otherwise use "generic". # otherwise use "generic".
......
...@@ -5,7 +5,8 @@ parts = ...@@ -5,7 +5,8 @@ parts =
[jemalloc] [jemalloc]
recipe = slapos.recipe.cmmi recipe = slapos.recipe.cmmi
url = http://www.canonware.com/download/jemalloc/jemalloc-3.6.0.tar.bz2 url = https://github.com/jemalloc/jemalloc/releases/download/${:version}/jemalloc-${:version}.tar.bz2
version = 3.6.0
md5sum = e76665b63a8fddf4c9f26d2fa67afdf2 md5sum = e76665b63a8fddf4c9f26d2fa67afdf2
configure-options = configure-options =
--disable-static --disable-static
......
...@@ -25,7 +25,7 @@ interpreter = keras-python ...@@ -25,7 +25,7 @@ interpreter = keras-python
scripts = keras-python scripts = keras-python
[versions] [versions]
Keras = 2.0.8 Keras = 2.1.0
tensorflow = 1.4.0rc1 tensorflow = 1.4.0
h5py = 2.7.0rc2 h5py = 2.7.0rc2
Cython = 0.25.2 Cython = 0.25.2
...@@ -12,7 +12,6 @@ extends = ...@@ -12,7 +12,6 @@ extends =
../ncurses/buildout.cfg ../ncurses/buildout.cfg
../openssl/buildout.cfg ../openssl/buildout.cfg
../patch/buildout.cfg ../patch/buildout.cfg
../pcre/buildout.cfg
../pkgconfig/buildout.cfg ../pkgconfig/buildout.cfg
../readline/buildout.cfg ../readline/buildout.cfg
../xz-utils/buildout.cfg ../xz-utils/buildout.cfg
...@@ -27,8 +26,9 @@ parts = ...@@ -27,8 +26,9 @@ parts =
[mariadb] [mariadb]
recipe = slapos.recipe.cmmi recipe = slapos.recipe.cmmi
url = https://downloads.mariadb.org/f/mariadb-10.1.28/source/mariadb-10.1.28.tar.gz/from/http%3A//fr.mirror.babylon.network/mariadb/?serve url = https://downloads.mariadb.org/f/mariadb-${:version}/source/mariadb-${:version}.tar.gz/from/http%3A//fr.mirror.babylon.network/mariadb/?serve
md5sum = 38acd5b44c56791701d80fddf088ef38 version = 10.1.30
md5sum = c424fd12bdff388e3da1bdecf42626c9
patch-options = -p0 patch-options = -p0
patches = patches =
${:_profile_base_location_}/mariadb_10.1.21_create_system_tables__no_test.patch#3c76aa9564a162f13aced7c0a3f783b3 ${:_profile_base_location_}/mariadb_10.1.21_create_system_tables__no_test.patch#3c76aa9564a162f13aced7c0a3f783b3
...@@ -45,7 +45,7 @@ configure-options = ...@@ -45,7 +45,7 @@ configure-options =
-DWITH_ZLIB=system -DWITH_ZLIB=system
-DWITH_READLINE=0 -DWITH_READLINE=0
-DWITH_PIC=1 -DWITH_PIC=1
-DWITH_PCRE=system -DWITH_PCRE=bundled
-DENABLE_DTRACE=0 -DENABLE_DTRACE=0
-DWITH_EXTRA_CHARSETS=complex -DWITH_EXTRA_CHARSETS=complex
-DWITH_EMBEDDED_SERVER=0 -DWITH_EMBEDDED_SERVER=0
...@@ -56,22 +56,22 @@ configure-options = ...@@ -56,22 +56,22 @@ configure-options =
-DWITH_INNODB_LZO=OFF -DWITH_INNODB_LZO=OFF
-DWITH_INNODB_SNAPPY=OFF -DWITH_INNODB_SNAPPY=OFF
-DWITH_SAFEMALLOC=OFF -DWITH_SAFEMALLOC=OFF
-DWITHOUT_EXAMPLE_STORAGE_ENGINE=1 -DPLUGIN_DAEMON_EXAMPLE=NO
-DWITHOUT_MROONGA_STORAGE_ENGINE=1 -DPLUGIN_EXAMPLE=NO
-DWITHOUT_DAEMON_EXAMPLE=1 -DPLUGIN_MROONGA=NO
-DCMAKE_C_FLAGS="${:CMAKE_CFLAGS}" -DCMAKE_C_FLAGS="${:CMAKE_CFLAGS}"
-DCMAKE_CXX_FLAGS="${:CMAKE_CFLAGS}" -DCMAKE_CXX_FLAGS="${:CMAKE_CFLAGS}"
-DCMAKE_INSTALL_RPATH=${:CMAKE_LIBRARY_PATH} -DCMAKE_INSTALL_RPATH=${:CMAKE_LIBRARY_PATH}
CMAKE_CFLAGS = -I${bzip2:location}/include -I${jemalloc:location}/include -I${libaio:location}/include -I${libxml2:location}/include -I${ncurses:location}/include -I${openssl:location}/include -I${pcre:location}/include -I${readline5:location}/include -I${xz-utils:location}/include -I${zlib:location}/include ${:extra_cflags} CMAKE_CFLAGS = -I${bzip2:location}/include -I${jemalloc:location}/include -I${libaio:location}/include -I${libxml2:location}/include -I${ncurses:location}/include -I${openssl:location}/include -I${readline5:location}/include -I${xz-utils:location}/include -I${zlib:location}/include ${:extra_cflags}
CMAKE_LIBRARY_PATH = ${bzip2:location}/lib:${jemalloc:location}/lib:${libaio:location}/lib:${libxml2:location}/lib:${ncurses:location}/lib:${openssl:location}/lib:${pcre:location}/lib:${readline5:location}/lib:${xz-utils:location}/lib:${zlib:location}/lib${:extra_library_path} CMAKE_LIBRARY_PATH = ${bzip2:location}/lib:${jemalloc:location}/lib:${libaio:location}/lib:${libxml2:location}/lib:${ncurses:location}/lib:${openssl:location}/lib:${readline5:location}/lib:${xz-utils:location}/lib:${zlib:location}/lib${:extra_library_path}
extra_cflags = extra_cflags =
extra_include_path = extra_include_path =
extra_library_path = extra_library_path =
environment = environment =
CMAKE_PROGRAM_PATH=${cmake:location}/bin CMAKE_PROGRAM_PATH=${cmake:location}/bin
CMAKE_INCLUDE_PATH=${bzip2:location}/include:${libaio:location}/include:${libaio:location}/include:${libxml2:location}/include:${ncurses:location}/include:${openssl:location}/include:${pcre:location}/include:${readline5:location}/include:${xz-utils:location}/include:${zlib:location}/include${:extra_include_path} CMAKE_INCLUDE_PATH=${bzip2:location}/include:${libaio:location}/include:${libaio:location}/include:${libxml2:location}/include:${ncurses:location}/include:${openssl:location}/include:${readline5:location}/include:${xz-utils:location}/include:${zlib:location}/include${:extra_include_path}
CMAKE_LIBRARY_PATH=${:CMAKE_LIBRARY_PATH} CMAKE_LIBRARY_PATH=${:CMAKE_LIBRARY_PATH}
LDFLAGS=-L${bzip2:location}/lib -L${jemalloc:location}/lib -L${libaio:location}/lib -L${pcre:location}/lib -L${xz-utils:location}/lib -L${zlib:location}/lib LDFLAGS=-L${bzip2:location}/lib -L${jemalloc:location}/lib -L${libaio:location}/lib -L${xz-utils:location}/lib -L${zlib:location}/lib
PATH=${patch:location}/bin:%(PATH)s PATH=${patch:location}/bin:%(PATH)s
post-install = post-install =
mkdir -p ${:location}/include/wsrep && mkdir -p ${:location}/include/wsrep &&
...@@ -108,6 +108,6 @@ pre-build = ...@@ -108,6 +108,6 @@ pre-build =
sed -i -e "s,${mariadb:location}/include,$(pwd)/fake_mariadb_source/include,g" Makefile */Makefile sed -i -e "s,${mariadb:location}/include,$(pwd)/fake_mariadb_source/include,g" Makefile */Makefile
environment = environment =
PATH=${groonga:location}/bin:${patch:location}/bin:${pkgconfig:location}/bin:%(PATH)s PATH=${groonga:location}/bin:${patch:location}/bin:${pkgconfig:location}/bin:%(PATH)s
CPPFLAGS=-I${groonga:location}/include/groonga -I${pcre:location}/include CPPFLAGS=-I${groonga:location}/include/groonga
LDFLAGS=-L${groonga:location}/lib LDFLAGS=-L${groonga:location}/lib
PKG_CONFIG_PATH=${groonga:location}/lib/pkgconfig:${groonga-normalizer-mysql:location}/lib/pkgconfig PKG_CONFIG_PATH=${groonga:location}/lib/pkgconfig:${groonga-normalizer-mysql:location}/lib/pkgconfig
# Do not extend any file that touch buildout:parts. # Do not extend any file that touch buildout:parts.
[mariadb] [mariadb]
url = https://downloads.mariadb.org/f/mariadb-10.2.9/source/mariadb-10.2.9.tar.gz/from/http%3A//fr.mirror.babylon.network/mariadb/?serve version = 10.2.12
md5sum = c59999bd182ddeb3db3d55250aecd8f8 md5sum = c5bce588e3c53ebc417e37ecb0967aee
stable-patches = stable-patches =
configure-options += configure-options +=
-DPLUGIN_DAEMON_EXAMPLE=NO
-DPLUGIN_EXAMPLE=NO
-DPLUGIN_MROONGA=NO
-DCMAKE_C_COMPILER=${gcc:location}/bin/gcc -DCMAKE_C_COMPILER=${gcc:location}/bin/gcc
-DCMAKE_CXX_COMPILER=${gcc:location}/bin/g++ -DCMAKE_CXX_COMPILER=${gcc:location}/bin/g++
extra_cflags = -I${zstd:location}/include extra_cflags = -I${zstd:location}/include
......
...@@ -18,6 +18,7 @@ patch-options = -p0 ...@@ -18,6 +18,7 @@ patch-options = -p0
patch-binary = ${patch:location}/bin/patch patch-binary = ${patch:location}/bin/patch
patches = patches =
http://nginx.org/download/patch.2017.ranges.txt#40bf9f37c881cb3b10cfefd84ca92f6a http://nginx.org/download/patch.2017.ranges.txt#40bf9f37c881cb3b10cfefd84ca92f6a
${:_profile_base_location_}/fix-gcc7-implicit-fallthrough-errors.patch
[nginx] [nginx]
<= nginx-common <= nginx-common
......
commit 8449f750e62cd229026e9df3bd023ec7e073a7d4
Author: Maxim Dounin <mdounin@mdounin.ru>
Date: Thu Apr 27 16:57:18 2017 +0300
Added missing "fall through" comments (ticket #1259).
Found by gcc7 (-Wimplicit-fallthrough).
diff --git src/core/ngx_murmurhash.c src/core/ngx_murmurhash.c
index c31e0e03..5ade658d 100644
--- src/core/ngx_murmurhash.c
+++ src/core/ngx_murmurhash.c
@@ -35,8 +35,10 @@ ngx_murmur_hash2(u_char *data, size_t len)
switch (len) {
case 3:
h ^= data[2] << 16;
+ /* fall through */
case 2:
h ^= data[1] << 8;
+ /* fall through */
case 1:
h ^= data[0];
h *= 0x5bd1e995;
diff --git src/http/ngx_http_parse.c src/http/ngx_http_parse.c
index 36220fdc..e8e51563 100644
--- src/http/ngx_http_parse.c
+++ src/http/ngx_http_parse.c
@@ -1396,6 +1396,7 @@ ngx_http_parse_complex_uri(ngx_http_request_t *r, ngx_uint_t merge_slashes)
goto done;
case '+':
r->plus_in_uri = 1;
+ /* fall through */
default:
state = sw_usual;
*u++ = ch;
@@ -1437,6 +1438,7 @@ ngx_http_parse_complex_uri(ngx_http_request_t *r, ngx_uint_t merge_slashes)
goto done;
case '+':
r->plus_in_uri = 1;
+ /* fall through */
default:
state = sw_usual;
*u++ = ch;
@@ -1484,6 +1486,7 @@ ngx_http_parse_complex_uri(ngx_http_request_t *r, ngx_uint_t merge_slashes)
goto done;
case '+':
r->plus_in_uri = 1;
+ /* fall through */
default:
state = sw_usual;
*u++ = ch;
diff --git src/os/unix/ngx_process.c src/os/unix/ngx_process.c
index dd50b5ca..993c032a 100644
--- src/os/unix/ngx_process.c
+++ src/os/unix/ngx_process.c
@@ -413,6 +413,7 @@ ngx_signal_handler(int signo, siginfo_t *siginfo, void *ucontext)
break;
}
ngx_debug_quit = 1;
+ /* fall through */
case ngx_signal_value(NGX_SHUTDOWN_SIGNAL):
ngx_quit = 1;
action = ", shutting down";
...@@ -32,8 +32,8 @@ setup-eggs = ...@@ -32,8 +32,8 @@ setup-eggs =
${matplotlib:egg} ${matplotlib:egg}
${pillow-python:egg} ${pillow-python:egg}
patches = patches =
${:_profile_base_location_}/ocropy.patch ${:_profile_base_location_}/ocropy.patch#dd7a02e1e63ed9df68e3a539b3e919eb
patch-options = -p0 patch-options = -p1
patch-binary = ${patch:location}/bin/patch patch-binary = ${patch:location}/bin/patch
environment = ocropy-env environment = ocropy-env
find-links = https://github.com/tmbdev/ocropy/tarball/4efbddca22bb2f0c639af0694e7a1386f2f097b5/ocropy-1.0.tar.gz find-links = https://github.com/tmbdev/ocropy/tarball/4efbddca22bb2f0c639af0694e7a1386f2f097b5/ocropy-1.0.tar.gz
......
diff --git ocrolib/__init__.py ocrolib/__init__.py From 1bb1546b12b0c08b1b32b293207de2d58d43ff1c Mon Sep 17 00:00:00 2001
From: Francois Le Corre <francois.lecorre@nexedi.com>
Date: Thu, 6 Apr 2017 11:32:27 +0200
Subject: [PATCH] WIP
- New OCROPY_MODEL_PATH environment variable to specify the position of models
- Import for the first time a module that compiles C code from python,
allowing us to have the static library built before runtime and
preventing us from using gcc through zope.
---
ocrolib/__init__.py | 3 ++-
ocrolib/common.py | 7 ++++---
ocrolib/native.py | 1 +
setup.py | 15 ++++++++++++---
4 files changed, 19 insertions(+), 7 deletions(-)
diff --git a/ocrolib/__init__.py b/ocrolib/__init__.py
index 1e0d627..81e85fb 100644 index 1e0d627..81e85fb 100644
--- ocrolib/__init__.py --- a/ocrolib/__init__.py
+++ ocrolib/__init__.py +++ b/ocrolib/__init__.py
@@ -1,7 +1,7 @@ @@ -1,7 +1,7 @@
__all__ = [ __all__ = [
"binnednn","cairoextras","common","components","dbtables", "binnednn","cairoextras","common","components","dbtables",
...@@ -11,18 +27,18 @@ index 1e0d627..81e85fb 100644 ...@@ -11,18 +27,18 @@ index 1e0d627..81e85fb 100644
] ]
################################################################ ################################################################
@@ -9,5 +9,6 @@ __all__ = [ @@ -9,5 +9,6 @@
################################################################ ################################################################
import default import default
+from psegutils import * +from psegutils import *
from common import * from common import *
from default import traceback as trace from default import traceback as trace
diff --git ocrolib/common.py ocrolib/common.py diff --git a/ocrolib/common.py b/ocrolib/common.py
index 27c0f26..14f088f 100644 index 27c0f26..14f088f 100644
--- ocrolib/common.py --- a/ocrolib/common.py
+++ ocrolib/common.py +++ b/ocrolib/common.py
@@ -14,6 +14,7 @@ import unicodedata @@ -14,6 +14,7 @@
import inspect import inspect
import glob import glob
import cPickle import cPickle
...@@ -38,7 +54,7 @@ index 27c0f26..14f088f 100644 ...@@ -38,7 +54,7 @@ index 27c0f26..14f088f 100644
def load_object(fname,zip=0,nofind=0,verbose=0): def load_object(fname,zip=0,nofind=0,verbose=0):
"""Loads an object from disk. By default, this handles zipped files """Loads an object from disk. By default, this handles zipped files
and searches in the usual places for OCRopus. It also handles some and searches in the usual places for OCRopus. It also handles some
@@ -439,8 +441,7 @@ def load_object(fname,zip=0,nofind=0,verbose=0): @@ -439,8 +441,7 @@ class names that have changed."""
if zip==0 and fname.endswith(".gz"): if zip==0 and fname.endswith(".gz"):
zip = 1 zip = 1
if zip>0: if zip>0:
...@@ -57,10 +73,10 @@ index 27c0f26..14f088f 100644 ...@@ -57,10 +73,10 @@ index 27c0f26..14f088f 100644
possible_prefixes.append("/usr/local/share/ocropus") possible_prefixes.append("/usr/local/share/ocropus")
diff --git ocrolib/native.py ocrolib/native.py diff --git a/ocrolib/native.py b/ocrolib/native.py
index b7a207f..240450b 100644 index b7a207f..240450b 100644
--- ocrolib/native.py --- a/ocrolib/native.py
+++ ocrolib/native.py +++ b/ocrolib/native.py
@@ -44,6 +44,7 @@ class CompileError(Exception): @@ -44,6 +44,7 @@ class CompileError(Exception):
def compile_and_find(c_string,prefix=".pynative",opt="-g -O4",libs="-lm", def compile_and_find(c_string,prefix=".pynative",opt="-g -O4",libs="-lm",
...@@ -69,22 +85,22 @@ index b7a207f..240450b 100644 ...@@ -69,22 +85,22 @@ index b7a207f..240450b 100644
if not os.path.exists(prefix): if not os.path.exists(prefix):
os.mkdir(prefix) os.mkdir(prefix)
m = hashlib.md5() m = hashlib.md5()
diff --git setup.py setup.py diff --git a/setup.py b/setup.py
index 2ec5832..6697b12 100644 index 2ec5832..0ad4d85 100644
--- setup.py --- a/setup.py
+++ setup.py +++ b/setup.py
@@ -10,7 +10,9 @@ assert sys.version_info[0]==2 and sys.version_info[1]>=7,\ @@ -10,7 +10,9 @@
from distutils.core import setup #, Extension, Command from distutils.core import setup #, Extension, Command
#from distutils.command.install_data import install_data #from distutils.command.install_data import install_data
-if not os.path.exists("models/en-default.pyrnn.gz"): -if not os.path.exists("models/en-default.pyrnn.gz"):
+models = os.environ.get('OCROPY_MODEL_PATH', '').split(':') or \ +models = os.environ.get('OCROPY_MODEL_PATH')
+ [c for c in glob.glob("models/*pyrnn.gz")] +models = models.split(':') if models else glob.glob("models/*pyrnn.gz")
+if not models: +if not models:
print() print()
print("You should download the default model 'en-default.pyrnn.gz'") print("You should download the default model 'en-default.pyrnn.gz'")
print("and put it into ./models.") print("and put it into ./models.")
@@ -18,16 +20,23 @@ if not os.path.exists("models/en-default.pyrnn.gz"): @@ -18,16 +20,23 @@
print("Check https://github.com/tmbdev/ocropy for the location") print("Check https://github.com/tmbdev/ocropy for the location")
print("of model files.") print("of model files.")
print() print()
...@@ -110,3 +126,6 @@ index 2ec5832..6697b12 100644 ...@@ -110,3 +126,6 @@ index 2ec5832..6697b12 100644
+ data_files= [('share/ocropus', models), ("", ["LICENSE"])], + data_files= [('share/ocropus', models), ("", ["LICENSE"])],
scripts = scripts, scripts = scripts,
) )
--
2.14.1
...@@ -28,3 +28,12 @@ configure-options = ...@@ -28,3 +28,12 @@ configure-options =
--prefix=${buildout:parts-directory}/${:_buildout_section_name_} --prefix=${buildout:parts-directory}/${:_buildout_section_name_}
--exec-prefix=${buildout:parts-directory}/${:_buildout_section_name_} --exec-prefix=${buildout:parts-directory}/${:_buildout_section_name_}
--with-privsep-path=${buildout:parts-directory}/${:_buildout_section_name_}/var/empty --with-privsep-path=${buildout:parts-directory}/${:_buildout_section_name_}/var/empty
[openssh-output]
# Shared binary location to ease migration
recipe = plone.recipe.command
stop-on-error = true
update-command = ${:command}
command = ${coreutils-output:test} -x ${:ssh} -a -x ${:keygen}
ssh = ${openssh:location}/bin/ssh
keygen = ${openssh:location}/bin/ssh-keygen
\ No newline at end of file
...@@ -16,8 +16,8 @@ parts = ...@@ -16,8 +16,8 @@ parts =
[openssl] [openssl]
recipe = slapos.recipe.cmmi recipe = slapos.recipe.cmmi
url = https://www.openssl.org/source/openssl-1.0.2m.tar.gz url = https://www.openssl.org/source/openssl-1.0.2n.tar.gz
md5sum = 10e9e37f492094b9ef296f68f24a7666 md5sum = 13bdc1b1d1ff39b6fd42a255e74676a4
location = ${buildout:parts-directory}/${:_buildout_section_name_} location = ${buildout:parts-directory}/${:_buildout_section_name_}
# 'prefix' option to override --openssldir/--prefix (which is useful # 'prefix' option to override --openssldir/--prefix (which is useful
# when combined with INSTALL_PREFIX). Used by slapos.package.git/obs # when combined with INSTALL_PREFIX). Used by slapos.package.git/obs
......
...@@ -22,6 +22,13 @@ rpath = ...@@ -22,6 +22,13 @@ rpath =
# and pin the egg in the [versions] section of the stack or SR. # and pin the egg in the [versions] section of the stack or SR.
find-links = http://pkgs.fedoraproject.org/repo/pkgs/rdiff-backup/rdiff-backup-1.0.5.tar.gz/fa2a165fa07a94be52c52e3545bc7758/rdiff-backup-1.0.5.tar.gz find-links = http://pkgs.fedoraproject.org/repo/pkgs/rdiff-backup/rdiff-backup-1.0.5.tar.gz/fa2a165fa07a94be52c52e3545bc7758/rdiff-backup-1.0.5.tar.gz
[rdiff-backup-build-1.3.4]
<= rdiff-backup-build
# use our own version
find-links = http://www.nexedi.org/static/packages/source/rdiff-backup-1.3.4nxd2.tar.gz
patches =
${:_profile_base_location_}/rdiff-backup-1.3.4-librsync-1.0.0.patch#31fafc8bc4a00f002f52008a9f3b671f
[rdiff-backup] [rdiff-backup]
# Scripts only generation part for rdiff-backup # Scripts only generation part for rdiff-backup
recipe = zc.recipe.egg recipe = zc.recipe.egg
......
Patch by Roman Tereshonkov and Kari Hautio for rdiff-backup <= 1.2.8 to avoid a build failure with
librsync >= 1.0.0 (which is a security bugfix release). The discussion and solution finding can be
found at https://bugs.launchpad.net/duplicity/+bug/1416344 (for duplicity).
--- rdiff-backup-1.3.4/rdiff_backup/_librsyncmodule.c 2009-03-16 15:36:21.000000000 +0100
+++ rdiff-backup-1.3.4/rdiff_backup/_librsyncmodule.c.librsync-1.0.0 2015-03-02 00:54:24.000000000 +0100
@@ -59,8 +59,13 @@
if (sm == NULL) return NULL;
sm->x_attr = NULL;
+#ifdef RS_DEFAULT_STRONG_LEN
sm->sig_job = rs_sig_begin((size_t)blocklen,
(size_t)RS_DEFAULT_STRONG_LEN);
+#else
+ sm->sig_job = rs_sig_begin((size_t)blocklen,
+ (size_t)8, RS_MD4_SIG_MAGIC);
+#endif
return (PyObject*)sm;
}
...@@ -56,14 +56,16 @@ location = ${buildout:parts-directory}/${:_buildout_section_name_} ...@@ -56,14 +56,16 @@ location = ${buildout:parts-directory}/${:_buildout_section_name_}
need-tensorboard-build = ${tensorboard-build:location} need-tensorboard-build = ${tensorboard-build:location}
egg = tensorflow-tensorboard egg = tensorflow-tensorboard
bazel-bin = ${bazel:location}/bin bazel-bin = ${bazel:location}/bin
java_home_bin = ${bazel:java_home}/bin
numpy-python-command = ${buildout:bin-directory}/${numpy-egg:interpreter} numpy-python-command = ${buildout:bin-directory}/${numpy-egg:interpreter}
script = script =
os.makedirs(location) os.makedirs(location)
workdir = self.options['tensorboard-repository-path'] workdir = self.options['tensorboard-repository-path']
egg_name = 'tensorflow_tensorboard-0.4.0rc1-py2.7.egg' egg_name = 'tensorflow_tensorboard-0.4.0rc3-py2.7.egg'
dist_dir = os.path.join(workdir, 'dist') dist_dir = os.path.join(workdir, 'dist')
dest_dir = os.path.join(self.buildout['buildout']['eggs-directory'], egg_name) dest_dir = os.path.join(self.buildout['buildout']['eggs-directory'], egg_name)
env = {'PATH':':'.join([self.options['bazel-bin'], env = {'PATH':':'.join([self.options['bazel-bin'],
self.options['java_home_bin'],
os.environ['PATH']]), os.environ['PATH']]),
'PYTHON_BIN_PATH':self.options['numpy-python-command'], 'PYTHON_BIN_PATH':self.options['numpy-python-command'],
} }
......
...@@ -27,7 +27,7 @@ location = ${buildout:parts-directory}/${:_buildout_section_name_} ...@@ -27,7 +27,7 @@ location = ${buildout:parts-directory}/${:_buildout_section_name_}
recipe = plone.recipe.command recipe = plone.recipe.command
stop-on-error = true stop-on-error = true
repository = https://github.com/tensorflow/tensorflow repository = https://github.com/tensorflow/tensorflow
tag = r1.4 tag = v1.4.0
git-binary = ${git:location}/bin/git git-binary = ${git:location}/bin/git
patch-binary = ${patch:location}/bin/patch patch-binary = ${patch:location}/bin/patch
location = ${buildout:parts-directory}/${:_buildout_section_name_} location = ${buildout:parts-directory}/${:_buildout_section_name_}
...@@ -110,7 +110,7 @@ egg = tensorflow ...@@ -110,7 +110,7 @@ egg = tensorflow
script = script =
os.makedirs(location) os.makedirs(location)
workdir = self.options['tensorflow-repository-path'] workdir = self.options['tensorflow-repository-path']
egg_name = 'tensorflow-1.4.0rc1-py2.7-linux-x86_64.egg' egg_name = 'tensorflow-1.4.0-py2.7-linux-x86_64.egg'
dist_dir = os.path.join(workdir, 'dist') dist_dir = os.path.join(workdir, 'dist')
dest_dir = os.path.join(self.buildout['buildout']['eggs-directory'], egg_name) dest_dir = os.path.join(self.buildout['buildout']['eggs-directory'], egg_name)
call(['bazel-bin/tensorflow/tools/pip_package/build_pip_package', dist_dir], cwd=workdir) call(['bazel-bin/tensorflow/tools/pip_package/build_pip_package', dist_dir], cwd=workdir)
......
...@@ -55,11 +55,12 @@ setup(name=name, ...@@ -55,11 +55,12 @@ setup(name=name,
packages=find_packages(), packages=find_packages(),
include_package_data=True, include_package_data=True,
install_requires=[ install_requires=[
'enum34', # for inotify-simple
'jsonschema', 'jsonschema',
'hexagonit.recipe.download', 'hexagonit.recipe.download',
'netaddr', # to manipulate on IP addresses 'netaddr', # to manipulate on IP addresses
'setuptools', # namespaces 'setuptools', # namespaces
'inotifyx', # XXX use pyinotify instead 'inotify_simple',
'lock_file', #another lockfile implementation for multiprocess 'lock_file', #another lockfile implementation for multiprocess
'slapos.core', # uses internally 'slapos.core', # uses internally
'zc.buildout', # plays with buildout 'zc.buildout', # plays with buildout
......
...@@ -85,7 +85,7 @@ class Recipe(GenericBaseRecipe): ...@@ -85,7 +85,7 @@ class Recipe(GenericBaseRecipe):
mysql_connection_string_list), mysql_connection_string_list),
] + common_list, **common_dict)])) ] + common_list, **common_dict)]))
path_list.append(self.createPythonScript(self.options['run-test-suite'], path_list.append(self.createPythonScript(self.options['run-test-suite'],
__name__ + '.test.runUnitTest', [dict( __name__ + '.test.runTestSuite', [dict(
call_list=[self.options['run-test-suite-binary'], call_list=[self.options['run-test-suite-binary'],
'--db_list', ','.join(mysql_connection_string_list), '--db_list', ','.join(mysql_connection_string_list),
] + common_list, **common_dict)])) ] + common_list, **common_dict)]))
......
...@@ -34,7 +34,10 @@ def runTestSuite(args): ...@@ -34,7 +34,10 @@ def runTestSuite(args):
if 'test_ca_path' in d: if 'test_ca_path' in d:
env['TEST_CA_PATH'] = d['test_ca_path'] env['TEST_CA_PATH'] = d['test_ca_path']
if 'prepend_path' in d: if 'prepend_path' in d:
env['PATH'] = ':'.join([d['prepend_path']] + os.environ.get('PATH', '').split(':')) try:
env['PATH'] = d['prepend_path'] + ':' + env['PATH']
except KeyError:
env['PATH'] = d['prepend_path']
if 'instance_home' in d: if 'instance_home' in d:
env['INSTANCE_HOME'] = d['instance_home'] env['INSTANCE_HOME'] = d['instance_home']
env['REAL_INSTANCE_HOME'] = d['instance_home'] env['REAL_INSTANCE_HOME'] = d['instance_home']
...@@ -57,29 +60,4 @@ def runTestSuite(args): ...@@ -57,29 +60,4 @@ def runTestSuite(args):
argument_list.append(env) argument_list.append(env)
os.execle(executable_filepath, *argument_list) os.execle(executable_filepath, *argument_list)
def runUnitTest(args): runUnitTest = runTestSuite
env = os.environ.copy()
d = args[0]
if 'openssl_binary' in d:
env['OPENSSL_BINARY'] = d['openssl_binary']
if 'test_ca_path' in d:
env['TEST_CA_PATH'] = d['test_ca_path']
if 'prepend_path' in d:
env['PATH'] = ':'.join([d['prepend_path']] + os.environ.get('PATH', '').split(':'))
if 'instance_home' in d:
env['INSTANCE_HOME'] = d['instance_home']
env['REAL_INSTANCE_HOME'] = d['instance_home']
# Deal with Shebang size limitation
executable_filepath = d['call_list'][0]
file_object = open(executable_filepath, 'r')
line = file_object.readline()
file_object.close()
argument_list = []
if line[:2] == '#!':
executable_filepath = line[2:].strip()
argument_list.append(executable_filepath)
argument_list.extend(d['call_list'])
argument_list.extend(sys.argv[1:])
argument_list.append(env)
os.execle(executable_filepath, *argument_list)
...@@ -3,47 +3,31 @@ import os ...@@ -3,47 +3,31 @@ import os
import signal import signal
import subprocess import subprocess
import time import time
from collections import defaultdict
import inotifyx from inotify_simple import INotify, flags
def _wait_files_creation(file_list): def _wait_files_creation(file_list):
# Etablish a list of directory and subfiles # Establish a list of directory and subfiles.
directories = dict() # and test existence before watching, so that we don't miss an event.
for dirname, filename in [os.path.split(f) for f in file_list]: directories = defaultdict(dict)
directories.setdefault(dirname, dict()) for f in file_list:
directories[dirname][filename] = False dirname, filename = os.path.split(f)
directories[dirname][filename] = os.path.lexists(f)
def all_files_exists(): def all_files_exists():
return all([all(files.values()) for files in directories.values()]) return all(all(files.itervalues()) for files in directories.itervalues())
fd = inotifyx.init() with INotify() as inotify:
try: watchdescriptors = {inotify.add_watch(dirname,
# Watch every directories where the file are flags.CREATE | flags.DELETE | flags.MOVED_TO | flags.MOVED_FROM
watchdescriptors = dict() ): dirname
for dirname in directories.keys(): for dirname in directories}
wd = inotifyx.add_watch(fd,
dirname,
inotifyx.IN_CREATE | inotifyx.IN_DELETE | inotifyx.IN_MOVE)
watchdescriptors[wd] = dirname
# Set to True the file wich exists
for dirname, filename in [os.path.split(f) for f in file_list]:
directories[dirname][filename] = os.path.exists(os.path.join(dirname,
filename))
# Let's wait for every file creation
while not all_files_exists():
events_list = inotifyx.get_events(fd)
for event in events_list:
dirname = watchdescriptors[event.wd]
if event.name in directories[dirname]:
# One of watched file was created or deleted
if event.mask & inotifyx.IN_DELETE:
directories[dirname][event.name] = False
else:
directories[dirname][event.name] = True
finally: while not all_files_exists():
os.close(fd) for event in inotify.read():
directory = directories[watchdescriptors[event.wd]]
if event.name in directory:
directory[event.name] = event.mask & (flags.CREATE | flags.MOVED_TO)
def execute(args): def execute(args):
"""Portable execution with process replacement""" """Portable execution with process replacement"""
...@@ -83,8 +67,8 @@ def generic_exec(args): ...@@ -83,8 +67,8 @@ def generic_exec(args):
os.execve(exec_list[0], exec_list + sys.argv[1:], exec_env) os.execve(exec_list[0], exec_list + sys.argv[1:], exec_env)
def sig_handler(signal, frame): def sig_handler(sig, frame):
print 'Received signal %r, killing children and exiting' % signal print 'Received signal %r, killing children and exiting' % sig
if child_pg is not None: if child_pg is not None:
os.killpg(child_pg, signal.SIGHUP) os.killpg(child_pg, signal.SIGHUP)
os.killpg(child_pg, signal.SIGTERM) os.killpg(child_pg, signal.SIGTERM)
...@@ -97,6 +81,7 @@ signal.signal(signal.SIGTERM, sig_handler) ...@@ -97,6 +81,7 @@ signal.signal(signal.SIGTERM, sig_handler)
def execute_with_signal_translation(args): def execute_with_signal_translation(args):
"""Run process as children and translate from SIGTERM to another signal""" """Run process as children and translate from SIGTERM to another signal"""
global child_pg
child = subprocess.Popen(args, close_fds=True, preexec_fn=os.setsid) child = subprocess.Popen(args, close_fds=True, preexec_fn=os.setsid)
child_pg = child.pid child_pg = child.pid
try: try:
......
...@@ -25,8 +25,7 @@ ...@@ -25,8 +25,7 @@
# #
############################################################################## ##############################################################################
import os import os
from inotify_simple import INotify, flags
import inotifyx
def subfiles(directory): def subfiles(directory):
"""Return the list of subfiles of a directory, and wait for the newly created """Return the list of subfiles of a directory, and wait for the newly created
...@@ -34,18 +33,12 @@ def subfiles(directory): ...@@ -34,18 +33,12 @@ def subfiles(directory):
CAUTION : *DONT TRY TO CONVERT THE RESULT OF THIS FUNCTION INTO A LIST ! CAUTION : *DONT TRY TO CONVERT THE RESULT OF THIS FUNCTION INTO A LIST !
ALWAYS ITERATE OVER IT !!!*""" ALWAYS ITERATE OVER IT !!!*"""
watchfd = inotifyx.init()
inotifyx.add_watch(watchfd, directory, inotifyx.IN_CREATE)
try:
subfiles = set(os.listdir(directory)) with INotify() as inotify:
subfiles |= set([file_.name for file_ in inotifyx.get_events(watchfd, 0)]) inotify.add_watch(directory, flags.CLOSE_WRITE | flags.MOVED_TO)
names = os.listdir(directory)
while True: while True:
for file_ in subfiles: for name in names:
yield os.path.join(directory, file_) yield os.path.join(directory, name)
names = (event.name for event in inotify.read())
subfiles = [file_.name for file_ in inotifyx.get_events(watchfd)]
finally:
os.close(watchfd)
...@@ -92,6 +92,8 @@ class Storage(NeoBaseRecipe): ...@@ -92,6 +92,8 @@ class Storage(NeoBaseRecipe):
engine = self.options.get('engine') engine = self.options.get('engine')
if engine: # old versions of NEO don't support -e if engine: # old versions of NEO don't support -e
r += '-e', engine r += '-e', engine
if self.options.get('dedup'):
r.append('--dedup')
if self.options.get('disable-drop-partitions'): if self.options.get('disable-drop-partitions'):
r.append('--disable-drop-partitions') r.append('--disable-drop-partitions')
return r return r
......
import os, shutil, tempfile, threading, unittest
from slapos.recipe.librecipe import execute, inotify
class TestInotify(unittest.TestCase):
def setUp(self):
self.tmp = tempfile.mkdtemp()
def tearDown(self):
shutil.rmtree(self.tmp)
def test_subfiles(self):
p = lambda x: os.path.join(self.tmp, x)
def create(name, text):
a = open(p(name), 'w')
a.write(text)
a.flush()
return a
def check(name, text):
path = next(notified)
self.assertEqual(path, p(name))
with open(path) as f:
self.assertEqual(f.read(), text)
a = create('first', 'blah')
a.write('...')
notified = inotify.subfiles(self.tmp)
check('first', 'blah')
os.link(p(a.name), p('a hard link')) # ignored
b = create('other', 'hello')
b.close()
check('other', 'hello')
c = create('last', '!!!')
a.close()
check('first', 'blah...')
os.rename(p(a.name), p(b.name))
check('other', 'blah...')
c.close()
check('last', '!!!')
def test_wait_files_creation(self):
file_list = (
'foo',
'bar',
'hello/world',
'hello/world!',
'a/b/c',
)
create = lambda x: open(x, 'w').close()
p = lambda x: os.path.join(self.tmp, x)
P = lambda x: p(file_list[x])
create(P(1))
os.mkdir(p('hello'))
os.makedirs(p('a/b'))
t = threading.Thread(target=execute._wait_files_creation,
args=(map(p, file_list),))
t.daemon = True
t.start()
def check():
t.join(.2)
self.assertTrue(t.is_alive())
check()
for x in P(3), p('a/b/d'), P(0):
create(x)
check()
os.rename(P(3), P(2))
os.rename(p('a/b/d'), P(4))
check()
os.remove(P(1))
for x in P(3), P(1):
create(x)
t.join(10)
self.assertFalse(t.is_alive())
...@@ -50,7 +50,7 @@ gitdb = 0.6.4 ...@@ -50,7 +50,7 @@ gitdb = 0.6.4
pycrypto = 2.6.1 pycrypto = 2.6.1
pycurl = 7.43.0 pycurl = 7.43.0
slapos.recipe.download = 1.0 slapos.recipe.download = 1.0
slapos.recipe.template = 4.1 slapos.recipe.template = 4.2
slapos.toolbox = 0.73 slapos.toolbox = 0.73
smmap = 0.9.0 smmap = 0.9.0
......
...@@ -10,7 +10,7 @@ gitdb = 0.6.4 ...@@ -10,7 +10,7 @@ gitdb = 0.6.4
plone.recipe.command = 1.1 plone.recipe.command = 1.1
pycrypto = 2.6.1 pycrypto = 2.6.1
rdiff-backup = 1.0.5+SlapOSPatched001 rdiff-backup = 1.0.5+SlapOSPatched001
slapos.recipe.template = 4.1 slapos.recipe.template = 4.2
slapos.toolbox = 0.73 slapos.toolbox = 0.73
smmap = 0.9.0 smmap = 0.9.0
numpy = 1.11.2 numpy = 1.11.2
......
...@@ -27,6 +27,7 @@ statistic = $${:srv}/statistic ...@@ -27,6 +27,7 @@ statistic = $${:srv}/statistic
backupscript = $${:etc}/backup backupscript = $${:etc}/backup
www = $${:srv}/www www = $${:srv}/www
home = $${:etc}/home home = $${:etc}/home
promises = $${:etc}/promise
ssl = $${:etc}/ssl ssl = $${:etc}/ssl
ssh = $${:home}/.ssh ssh = $${:home}/.ssh
...@@ -49,12 +50,13 @@ logfile = $${directory:log}/crond.log ...@@ -49,12 +50,13 @@ logfile = $${directory:log}/crond.log
{% set frequency = slave_instance.get('frequency', '') -%} {% set frequency = slave_instance.get('frequency', '') -%}
{% set hostname = slave_instance.get('hostname', '') -%} {% set hostname = slave_instance.get('hostname', '') -%}
{% set connection = slave_instance.get('connection', '') -%} {% set connection = slave_instance.get('connection', '') -%}
{% set connection_port = slave_instance.get('connection_port', '22') -%}
{% set include = slave_instance.get('include', '') -%} {% set include = slave_instance.get('include', '') -%}
{% set include_string = "' --include='".join(include.split(' ')) -%} {% set include_string = "' --include='".join(include.split(' ')) -%}
{% set exclude = slave_instance.get('exclude', '') -%} {% set exclude = slave_instance.get('exclude', '') -%}
{% set exclude_string = '' -%} {% set exclude_string = '' -%}
{% set sudo = slave_instance.get('sudo', 'False') -%} {% set sudo = slave_instance.get('sudo', 'False') -%}
{% set remote_schema = 'rdiff-backup --server --restrict-read-only / -- "$@"' -%} {% set remote_schema = slave_instance.get('remote_rdiff_path', 'rdiff-backup') + ' --server --restrict-read-only / -- "$@"' -%}
{% if (exclude != '') -%} {% if (exclude != '') -%}
{% set exclude_string = "' --exclude='".join(exclude.split(' ')) -%} {% set exclude_string = "' --exclude='".join(exclude.split(' ')) -%}
...@@ -73,23 +75,19 @@ directory = $${directory:backup}/$${:_buildout_section_name_} ...@@ -73,23 +75,19 @@ directory = $${directory:backup}/$${:_buildout_section_name_}
[{{ slave_reference }}-backup-private_key] [{{ slave_reference }}-backup-private_key]
recipe = plone.recipe.command recipe = plone.recipe.command
stop-on-error = false stop-on-error = true
command = ${dropbear-output:keygen} -t $${:type} -s 2048 -f $${:key} command = ${coreutils-output:rm} -f $${:key} $${:public_key} && ${openssh-output:keygen} -t $${:type} -b 2048 -f $${:key} -q -N ""
key = $${directory:ssh}/$${:_buildout_section_name_} key = $${directory:ssh}/$${:_buildout_section_name_}
public_key = $${:key}.pub
location = $${:public_key}
type = rsa type = rsa
[{{ slave_reference }}-backup-public_key]
recipe = plone.recipe.command
stop-on-error = true
command = ${coreutils-output:rm} -f $${:key} && ${dropbear-output:keygen} -y -f {{ '$${' ~ slave_reference }}-backup-private_key:key} | ${grep-output:grep} {{ '$${' ~ slave_reference }}-backup-private_key:type} > $${:key}
key = {{ '$${' ~ slave_reference }}-backup-private_key:key}.pub
location = $${:key}
# Insert as a beginning part, to ensure that all public keys are generated before trying to publish. This will reduce the number of slapgrid-cp run. # Insert as a beginning part, to ensure that all public keys are generated before trying to publish. This will reduce the number of slapgrid-cp run.
{% do part_list.insert(0, "%s-backup-public_key" % slave_reference) -%} {% do part_list.insert(0, "%s-backup-private_key" % slave_reference) -%}
[{{ slave_reference }}-backup-read-public_key] [{{ slave_reference }}-backup-read-public_key]
recipe = slapos.cookbook:readline recipe = slapos.cookbook:readline
storage-path = {{ '$${' ~ slave_reference }}-backup-public_key:key} storage-path = {{ '$${' ~ slave_reference }}-backup-private_key:public_key}
# Publish slave {{ slave_reference }} information # Publish slave {{ slave_reference }} information
[{{ slave_reference }}-backup-publish] [{{ slave_reference }}-backup-publish]
...@@ -107,6 +105,7 @@ mode = 0700 ...@@ -107,6 +105,7 @@ mode = 0700
datadirectory = {{ '$${' ~ slave_reference }}-backup-directory:directory} datadirectory = {{ '$${' ~ slave_reference }}-backup-directory:directory}
sshkey = {{ '$${' ~ slave_reference }}-backup-private_key:key} sshkey = {{ '$${' ~ slave_reference }}-backup-private_key:key}
connection = {{ connection }} connection = {{ connection }}
connection_port = {{ connection_port }}
hostname = {{ hostname }} hostname = {{ hostname }}
include = {{ include_string }} include = {{ include_string }}
exclude_string = {{ exclude_string }} exclude_string = {{ exclude_string }}
...@@ -169,6 +168,12 @@ mode = 0700 ...@@ -169,6 +168,12 @@ mode = 0700
virtual-depends = virtual-depends =
$${nginx-configuration:ip} $${nginx-configuration:ip}
[nginx-listen-promise]
recipe = slapos.cookbook:check_port_listening
hostname = $${nginx-configuration:ip}
port = $${nginx-configuration:port}
path = $${directory:promises}/nginx_listen
[nginx-configuration] [nginx-configuration]
recipe = slapos.recipe.template recipe = slapos.recipe.template
url = ${template-nginx-configuration:output} url = ${template-nginx-configuration:output}
...@@ -187,6 +192,7 @@ ssl_crt = $${directory:ssl}/nginx.crt ...@@ -187,6 +192,7 @@ ssl_crt = $${directory:ssl}/nginx.crt
parts = parts =
dcron-service dcron-service
nginx-service nginx-service
nginx-listen-promise
activate-crontab-file activate-crontab-file
publish-global-rss publish-global-rss
{% for part in part_list -%} {% for part in part_list -%}
......
...@@ -10,7 +10,7 @@ extends = ...@@ -10,7 +10,7 @@ extends =
# ../../component/git/buildout.cfg # ../../component/git/buildout.cfg
# ../../component/subversion/buildout.cfg # ../../component/subversion/buildout.cfg
../../component/rsync/buildout.cfg ../../component/rsync/buildout.cfg
../../component/dropbear/buildout.cfg ../../component/openssh/buildout.cfg
../../component/grep/buildout.cfg ../../component/grep/buildout.cfg
../../component/findutils/buildout.cfg ../../component/findutils/buildout.cfg
# ../../stack/flask.cfg # ../../stack/flask.cfg
...@@ -67,7 +67,7 @@ mode = 0644 ...@@ -67,7 +67,7 @@ mode = 0644
[template-backup-script] [template-backup-script]
recipe = slapos.recipe.template recipe = slapos.recipe.template
url = ${:_profile_base_location_}/template-backup-script.sh.in url = ${:_profile_base_location_}/template-backup-script.sh.in
md5sum = 47b20031db3b575651d8515d5add23e6 md5sum = fa79e0307e12e2f5b1f2adbd261995fc
output = ${buildout:directory}/template-backup-script.sh.in output = ${buildout:directory}/template-backup-script.sh.in
mode = 0644 mode = 0644
...@@ -88,7 +88,7 @@ mode = 0644 ...@@ -88,7 +88,7 @@ mode = 0644
[status2rss] [status2rss]
recipe = slapos.recipe.template recipe = slapos.recipe.template
url = ${:_profile_base_location_}/status2rss.py url = ${:_profile_base_location_}/status2rss.py
md5sum = 0cd1cf97b199dd18fc0168c7281890ea md5sum = a023694817975e73998fb9187a6015d6
output = ${buildout:directory}/status2rss.py output = ${buildout:directory}/status2rss.py
mode = 0644 mode = 0644
...@@ -105,7 +105,7 @@ mode = 0644 ...@@ -105,7 +105,7 @@ mode = 0644
[template-pullrdiffbackup] [template-pullrdiffbackup]
recipe = slapos.recipe.template recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance-pullrdiffbackup.cfg.in url = ${:_profile_base_location_}/instance-pullrdiffbackup.cfg.in
md5sum = 061b98d001b501c9e1beb424e8802d3d md5sum = a2fb7b0cdd944be99da4122eb6f07749
output = ${buildout:directory}/template-pullrdiffbackup.cfg output = ${buildout:directory}/template-pullrdiffbackup.cfg
mode = 0644 mode = 0644
...@@ -116,8 +116,14 @@ md5sum = 42021b325159dff29e4bd4e33b8ff2f3 ...@@ -116,8 +116,14 @@ md5sum = 42021b325159dff29e4bd4e33b8ff2f3
output = ${buildout:directory}/template.cfg output = ${buildout:directory}/template.cfg
mode = 0644 mode = 0644
[rdiff-backup]
eggs =
${rdiff-backup-build-1.3.4:egg}
[versions] [versions]
rdiff-backup = 1.0.5+SlapOSPatched001 # 1.3.4nxd2 is invalid version string, thus pached version string is not '1.3.4nxd2+SlapOSPatched001'
# but '1.3.4nxd2-SlapOSPatched001'.
rdiff-backup = 1.3.4nxd2-SlapOSPatched001
gunicorn = 19.1.1 gunicorn = 19.1.1
plone.recipe.command = 1.1 plone.recipe.command = 1.1
slapos.recipe.template = 2.4.2 slapos.recipe.template = 2.4.2
......
...@@ -29,9 +29,8 @@ while 1: ...@@ -29,9 +29,8 @@ while 1:
title = desc, title = desc,
description = "<p>%s</p>" % "<br/>".join(("%s, %s\n<a href='http://www.nongnu.org/rdiff-backup/FAQ.html#statistics'>Lastest statistic</a>\n%s" % (time, desc, description = "<p>%s</p>" % "<br/>".join(("%s, %s\n<a href='http://www.nongnu.org/rdiff-backup/FAQ.html#statistics'>Lastest statistic</a>\n%s" % (time, desc,
open(statistic).read())).split("\n")), open(statistic).read())).split("\n")),
link = LINK,
pubDate = datetime.datetime.fromtimestamp(mktime_tz(parsedate_tz(time))), pubDate = datetime.datetime.fromtimestamp(mktime_tz(parsedate_tz(time))),
guid = PyRSS2Gen.Guid(base64.b64encode("%s, %s" % (time, desc))) guid = PyRSS2Gen.Guid(base64.b64encode("%s, %s" % (time, desc)), isPermaLink=0)
) )
items.append(rss_item) items.append(rss_item)
......
...@@ -18,7 +18,7 @@ ${rdiff-backup-output:rdiff-backup} \ ...@@ -18,7 +18,7 @@ ${rdiff-backup-output:rdiff-backup} \
$${:exclude_string} \ $${:exclude_string} \
--include='$${:include}' \ --include='$${:include}' \
--exclude='**' \ --exclude='**' \
--remote-schema '${dropbear-output:ssh} -T -y -i $${:sshkey} %s $${:remote_schema}' \ --remote-schema '${openssh-output:ssh} -6 -q -T -y -o "StrictHostKeyChecking no" -i $${:sshkey} -p $${:connection_port} %s $${:remote_schema}' \
$${:connection}::/ ./ $${:connection}::/ ./
RESULT=$? RESULT=$?
......
...@@ -77,7 +77,7 @@ def main(): ...@@ -77,7 +77,7 @@ def main():
test_title = args.test_suite_title or args.test_suite test_title = args.test_suite_title or args.test_suite
if args.master_url: if args.master_url:
tool = taskdistribution.TaskDistributionTool(args.master_url) tool = taskdistribution.TaskDistributor(args.master_url)
test_result = tool.createTestResult(args.revision, test_result = tool.createTestResult(args.revision,
list(test_dict), list(test_dict),
args.test_node_title, args.test_node_title,
......
...@@ -40,4 +40,4 @@ cns.recipe.symlink = 0.2.3 ...@@ -40,4 +40,4 @@ cns.recipe.symlink = 0.2.3
collective.recipe.environment = 0.2.0 collective.recipe.environment = 0.2.0
erp5.util = 0.4.49 erp5.util = 0.4.49
plone.recipe.command = 1.1 plone.recipe.command = 1.1
slapos.recipe.template = 4.1 slapos.recipe.template = 4.2
...@@ -53,7 +53,7 @@ mysqlclient = 1.3.12 ...@@ -53,7 +53,7 @@ mysqlclient = 1.3.12
# indirect dependancies # indirect dependancies
cp.recipe.cmd = 0.5 cp.recipe.cmd = 0.5
plone.recipe.command = 1.1 plone.recipe.command = 1.1
slapos.recipe.template = 4.1 slapos.recipe.template = 4.2
zope.exceptions = 4.0.7 zope.exceptions = 4.0.7
zope.testing = 4.1.3 zope.testing = 4.1.3
zc.recipe.testrunner = 2.0.0 zc.recipe.testrunner = 2.0.0
......
...@@ -3,12 +3,8 @@ Available ``software-type`` values ...@@ -3,12 +3,8 @@ Available ``software-type`` values
- ``default`` - ``default``
Recommended for production use. Recommended for developemnt and production use. Automatic creation of
erp5-site.
- ``create-erp5-site``
Automated creation of ERP5Site instance, for easy deployment.
Usage in production discouraged due to the increased risk of data loss.
Notes Notes
===== =====
......
...@@ -9,13 +9,6 @@ ...@@ -9,13 +9,6 @@
"request": "instance-erp5-input-schema.json", "request": "instance-erp5-input-schema.json",
"response": "instance-erp5-output-schema.json", "response": "instance-erp5-output-schema.json",
"index": 0 "index": 0
},
"create-erp5-site": {
"title": "Create ERP5 Site",
"description": "Automated ERP5Site creation on instanciation when ZODB is found empty.",
"request": "instance-erp5-input-schema.json",
"response": "instance-erp5-output-schema.json",
"index": 1
} }
} }
} }
...@@ -58,8 +58,8 @@ mode = 0644 ...@@ -58,8 +58,8 @@ mode = 0644
[versions] [versions]
PyXML = 0.8.5 PyXML = 0.8.5
erp5.util = 0.4.49 erp5.util = 0.4.50
slapos.recipe.template = 4.1 slapos.recipe.template = 4.2
ipython = 5.3.0 ipython = 5.3.0
apache-libcloud = 2.1.0 apache-libcloud = 2.1.0
gitdb2 = 2.0.2 gitdb2 = 2.0.2
......
...@@ -158,10 +158,10 @@ def main(): ...@@ -158,10 +158,10 @@ def main():
access_url_https = 'https://%s:10443' % (args.partition_ipv4,) access_url_https = 'https://%s:10443' % (args.partition_ipv4,)
os.environ['TEST_ACCESS_URL_HTTP'] = access_url_http os.environ['TEST_ACCESS_URL_HTTP'] = access_url_http
os.environ['TEST_ACCESS_URL_HTTPS'] = access_url_https os.environ['TEST_ACCESS_URL_HTTPS'] = access_url_https
tool = taskdistribution.TaskDistributionTool( distributor = taskdistribution.TaskDistributor(
args.master_url, args.master_url,
logger=logger) logger=logger)
test_result = tool.createTestResult( test_result = distributor.createTestResult(
revision, suite.getTestList(), args.test_node_title, revision, suite.getTestList(), args.test_node_title,
suite.allow_restart, test_suite_title, args.project_title) suite.allow_restart, test_suite_title, args.project_title)
if test_result is None: if test_result is None:
......
...@@ -71,4 +71,4 @@ output = ${buildout:directory}/template.cfg ...@@ -71,4 +71,4 @@ output = ${buildout:directory}/template.cfg
mode = 0644 mode = 0644
[versions] [versions]
erp5.util = 0.4.49 erp5.util = 0.4.50
...@@ -5,6 +5,7 @@ extends = ...@@ -5,6 +5,7 @@ extends =
parts = parts =
instance-template instance-template
slapos-cookbook
[instance-template] [instance-template]
recipe = slapos.recipe.template recipe = slapos.recipe.template
...@@ -14,4 +15,4 @@ md5sum = efd3b712a2294207f265a9c45648d5cf ...@@ -14,4 +15,4 @@ md5sum = efd3b712a2294207f265a9c45648d5cf
mode = 0644 mode = 0644
[versions] [versions]
slapos.recipe.template = 4.1 slapos.recipe.template = 4.2
...@@ -381,5 +381,5 @@ cns.recipe.symlink = 0.2.3 ...@@ -381,5 +381,5 @@ cns.recipe.symlink = 0.2.3
docutils = 0.12 docutils = 0.12
plone.recipe.command = 1.1 plone.recipe.command = 1.1
rubygemsrecipe = 0.2.2+slapos001 rubygemsrecipe = 0.2.2+slapos001
slapos.recipe.template = 4.1 slapos.recipe.template = 4.2
z3c.recipe.scripts = 1.0.1 z3c.recipe.scripts = 1.0.1
...@@ -72,7 +72,7 @@ async = 0.6.1 ...@@ -72,7 +72,7 @@ async = 0.6.1
gitdb = 0.5.4 gitdb = 0.5.4
pycrypto = 2.6 pycrypto = 2.6
rdiff-backup = 1.0.5+SlapOSPatched001 rdiff-backup = 1.0.5+SlapOSPatched001
slapos.recipe.template = 4.1 slapos.recipe.template = 4.2
slapos.toolbox = 0.40.4 slapos.toolbox = 0.40.4
smmap = 0.8.2 smmap = 0.8.2
plone.recipe.command = 1.1 plone.recipe.command = 1.1
......
...@@ -48,4 +48,4 @@ md5sum = 8cde04bfd0c0e9bd56744b988275cfd8 ...@@ -48,4 +48,4 @@ md5sum = 8cde04bfd0c0e9bd56744b988275cfd8
PyRSS2Gen = 1.1 PyRSS2Gen = 1.1
cns.recipe.symlink = 0.2.3 cns.recipe.symlink = 0.2.3
plone.recipe.command = 1.1 plone.recipe.command = 1.1
slapos.recipe.template = 4.1 slapos.recipe.template = 4.2
...@@ -91,8 +91,8 @@ def main(): ...@@ -91,8 +91,8 @@ def main():
if args.target == 'firefox': if args.target == 'firefox':
firefox_capabilities = webdriver.common.desired_capabilities.DesiredCapabilities.FIREFOX firefox_capabilities = webdriver.common.desired_capabilities.DesiredCapabilities.FIREFOX
firefox_capabilities['marionette'] = True firefox_capabilities['marionette'] = True
firefox_capabilities['binary'] = '${firefox:location}/firefox-slapos'
browser = webdriver.Firefox(capabilities=firefox_capabilities, browser = webdriver.Firefox(capabilities=firefox_capabilities,
firefox_binary='${firefox:location}/firefox-slapos',
executable_path='${firefox:location}/geckodriver') executable_path='${firefox:location}/geckodriver')
elif args.target in ['iOS', 'Android']: elif args.target in ['iOS', 'Android']:
# parameters for mobile emulators have different names then parameters for # parameters for mobile emulators have different names then parameters for
...@@ -169,7 +169,7 @@ def main(): ...@@ -169,7 +169,7 @@ def main():
browser.quit() browser.quit()
is_browser_running = False is_browser_running = False
tool = taskdistribution.TaskDistributionTool(portal_url=args.master_url) tool = taskdistribution.TaskDistributor(portal_url=args.master_url)
test_result = tool.createTestResult(revision = revision, test_result = tool.createTestResult(revision = revision,
test_name_list = test_line_dict.keys(), test_name_list = test_line_dict.keys(),
node_title = args.test_node_title, node_title = args.test_node_title,
......
...@@ -107,11 +107,11 @@ mode = 0644 ...@@ -107,11 +107,11 @@ mode = 0644
[template-runTestSuite] [template-runTestSuite]
recipe = slapos.recipe.template recipe = slapos.recipe.template
url = ${:_profile_base_location_}/runTestSuite.in url = ${:_profile_base_location_}/runTestSuite.in
md5sum = fcf15b2a90340e0afe8f8b9921a4ffae md5sum = ff66d13f73982e8257eb5535cdb541c7
output = ${buildout:directory}/runTestSuite.in output = ${buildout:directory}/runTestSuite.in
mode = 0644 mode = 0644
[versions] [versions]
erp5.util = 0.4.49 erp5.util = 0.4.50
slapos.recipe.template = 4.1 slapos.recipe.template = 4.2
selenium = 2.53.1 selenium = 3.8.0
...@@ -85,7 +85,7 @@ pyzmq = 16.0.2 ...@@ -85,7 +85,7 @@ pyzmq = 16.0.2
scikit-learn = 0.18.1 scikit-learn = 0.18.1
seaborn = 0.7.1 seaborn = 0.7.1
simplegeneric = 0.8.1 simplegeneric = 0.8.1
slapos.recipe.template = 4.1 slapos.recipe.template = 4.2
statsmodels = 0.8.0 statsmodels = 0.8.0
terminado = 0.6 terminado = 0.6
tornado = 4.4.2 tornado = 4.4.2
......
...@@ -11,7 +11,7 @@ apache-libcloud = 1.1.0 ...@@ -11,7 +11,7 @@ apache-libcloud = 1.1.0
collective.recipe.environment = 0.2.0 collective.recipe.environment = 0.2.0
gitdb = 0.6.4 gitdb = 0.6.4
pycurl = 7.43.0 pycurl = 7.43.0
slapos.recipe.template = 4.1 slapos.recipe.template = 4.2
smmap = 0.9.0 smmap = 0.9.0
# websockify = 0.8.0 # websockify = 0.8.0
......
...@@ -17,7 +17,7 @@ parts += ...@@ -17,7 +17,7 @@ parts +=
versions = versions versions = versions
[versions] [versions]
slapos.recipe.template = 4.1 slapos.recipe.template = 4.2
[template-instance] [template-instance]
......
...@@ -70,6 +70,11 @@ ...@@ -70,6 +70,11 @@
"default": 1, "default": 1,
"type": "integer" "type": "integer"
}, },
"data-deduplication": {
"description": "Set the --dedup option for storage nodes.",
"default": false,
"type": "boolean"
},
"disable-drop-partitions": { "disable-drop-partitions": {
"description": "Set the --disable-drop-partitions option for storage nodes.", "description": "Set the --disable-drop-partitions option for storage nodes.",
"default": false, "default": false,
......
...@@ -87,6 +87,7 @@ masters = ${publish:masters} ...@@ -87,6 +87,7 @@ masters = ${publish:masters}
database-adapter = MySQL database-adapter = MySQL
wait-database = -1 wait-database = -1
engine = {{ slapparameter_dict.get('engine', '') }} engine = {{ slapparameter_dict.get('engine', '') }}
dedup = {{ dumps(bool(slapparameter_dict.get('data-deduplication'))) }}
disable-drop-partitions = {{ dumps(bool(slapparameter_dict.get('disable-drop-partitions'))) }} disable-drop-partitions = {{ dumps(bool(slapparameter_dict.get('disable-drop-partitions'))) }}
{% for i in range(slapparameter_dict.get('storage-count', 1)) -%} {% for i in range(slapparameter_dict.get('storage-count', 1)) -%}
......
...@@ -98,7 +98,7 @@ md5sum = 1fee10f02c2fa2a581e21878ca0fd704 ...@@ -98,7 +98,7 @@ md5sum = 1fee10f02c2fa2a581e21878ca0fd704
[instance-neo-storage-mysql] [instance-neo-storage-mysql]
<= download-base-neo <= download-base-neo
md5sum = 67d623d631c2f99e33bcabc79fc9cccf md5sum = 366e51c0dbd85e511a31e403b8704735
[template-neo-my-cnf] [template-neo-my-cnf]
<= download-base-neo <= download-base-neo
...@@ -116,7 +116,7 @@ mysqlclient = 1.3.12 ...@@ -116,7 +116,7 @@ mysqlclient = 1.3.12
persistent = 4.2.3 persistent = 4.2.3
pycrypto = 2.6.1 pycrypto = 2.6.1
pycurl = 7.43.0 pycurl = 7.43.0
slapos.recipe.template = 4.1 slapos.recipe.template = 4.2
slapos.toolbox = 0.73 slapos.toolbox = 0.73
smmap2 = 2.0.1 smmap2 = 2.0.1
transaction = 1.7.0 transaction = 1.7.0
......
...@@ -45,5 +45,5 @@ output = ${buildout:directory}/instance-nginx.cfg.in ...@@ -45,5 +45,5 @@ output = ${buildout:directory}/instance-nginx.cfg.in
mode = 0644 mode = 0644
[versions] [versions]
slapos.recipe.template = 4.1 slapos.recipe.template = 4.2
slapos.toolbox = 0.73 slapos.toolbox = 0.73
...@@ -65,4 +65,4 @@ mode = 0644 ...@@ -65,4 +65,4 @@ mode = 0644
PyRSS2Gen = 1.1 PyRSS2Gen = 1.1
cns.recipe.symlink = 0.2.3 cns.recipe.symlink = 0.2.3
plone.recipe.command = 1.1 plone.recipe.command = 1.1
slapos.recipe.template = 4.1 slapos.recipe.template = 4.2
...@@ -110,7 +110,7 @@ gitdb = 0.6.4 ...@@ -110,7 +110,7 @@ gitdb = 0.6.4
plone.recipe.command = 1.1 plone.recipe.command = 1.1
pycrypto = 2.6.1 pycrypto = 2.6.1
pycurl = 7.43.0 pycurl = 7.43.0
slapos.recipe.template = 4.1 slapos.recipe.template = 4.2
slapos.toolbox = 0.73 slapos.toolbox = 0.73
smmap = 0.9.0 smmap = 0.9.0
......
...@@ -47,4 +47,4 @@ mode = 0644 ...@@ -47,4 +47,4 @@ mode = 0644
[versions] [versions]
plone.recipe.command = 1.1 plone.recipe.command = 1.1
slapos.recipe.template = 4.1 slapos.recipe.template = 4.2
...@@ -11,4 +11,4 @@ extends = common.cfg ...@@ -11,4 +11,4 @@ extends = common.cfg
Pygments = 1.6 Pygments = 1.6
collective.recipe.environment = 0.2.0 collective.recipe.environment = 0.2.0
collective.recipe.template = 1.10 collective.recipe.template = 1.10
slapos.recipe.template = 4.1 slapos.recipe.template = 4.2
...@@ -15,7 +15,7 @@ ...@@ -15,7 +15,7 @@
# not need these here). # not need these here).
[template-erp5] [template-erp5]
filename = instance-erp5.cfg.in filename = instance-erp5.cfg.in
md5sum = 251c1fcec1817fe70ec0cf7da69ccf3a md5sum = 4a77ee4a6367fee27552f8bfe9d87aab
[template-balancer] [template-balancer]
filename = instance-balancer.cfg.in filename = instance-balancer.cfg.in
......
...@@ -143,7 +143,7 @@ return = ...@@ -143,7 +143,7 @@ return =
zope-address-list zope-address-list
hosts-dict hosts-dict
monitor-base-url monitor-base-url
{% set bt5_default_list = 'erp5_full_text_myisam_catalog erp5_configurator_standard erp5_configurator_maxma_demo erp5_configurator_run_my_doc slapos_configurator' -%} {% set bt5_default_list = 'erp5_full_text_myisam_catalog slapos_configurator' -%}
{% if has_jupyter -%} {% if has_jupyter -%}
{% set bt5_default_list = bt5_default_list + ' erp5_data_notebook' -%} {% set bt5_default_list = bt5_default_list + ' erp5_data_notebook' -%}
{% endif -%} {% endif -%}
......
...@@ -97,4 +97,4 @@ mode = 640 ...@@ -97,4 +97,4 @@ mode = 640
Pygments = 2.1.3 Pygments = 2.1.3
collective.recipe.template = 1.10 collective.recipe.template = 1.10
plone.recipe.command = 1.1 plone.recipe.command = 1.1
slapos.recipe.template = 4.1 slapos.recipe.template = 4.2
...@@ -152,6 +152,7 @@ eggs = ...@@ -152,6 +152,7 @@ eggs =
gunicorn==19.7.1 gunicorn==19.7.1
futures futures
${slapos-cookbook:eggs} ${slapos-cookbook:eggs}
slapos.core # listed explicitly for scripts generation
[extra-eggs] [extra-eggs]
recipe = zc.recipe.egg recipe = zc.recipe.egg
......
...@@ -15,7 +15,7 @@ gitdb = 0.6.4 ...@@ -15,7 +15,7 @@ gitdb = 0.6.4
gunicorn = 19.7.1 gunicorn = 19.7.1
prettytable = 0.7.2 prettytable = 0.7.2
pycurl = 7.43.0 pycurl = 7.43.0
slapos.recipe.template = 4.1 slapos.recipe.template = 4.2
slapos.toolbox = 0.73 slapos.toolbox = 0.73
smmap = 0.9.0 smmap = 0.9.0
......
...@@ -59,7 +59,7 @@ eggs = collective.recipe.template ...@@ -59,7 +59,7 @@ eggs = collective.recipe.template
collective.recipe.template = 1.11 collective.recipe.template = 1.11
plone.recipe.command = 1.1 plone.recipe.command = 1.1
slapos.recipe.build = 0.28 slapos.recipe.build = 0.28
slapos.recipe.template = 4.1 slapos.recipe.template = 4.2
# Replicate slapos stack, but without shacache to not have to compile the entire world for a simple test. # Replicate slapos stack, but without shacache to not have to compile the entire world for a simple test.
[buildout] [buildout]
......
...@@ -51,7 +51,6 @@ MarkupSafe = 0.18 ...@@ -51,7 +51,6 @@ MarkupSafe = 0.18
Werkzeug = 0.8.3 Werkzeug = 0.8.3
buildout-versions = 1.7 buildout-versions = 1.7
hexagonit.recipe.cmmi = 2.0 hexagonit.recipe.cmmi = 2.0
inotifyx = 0.2.0-1
lxml = 3.2.1 lxml = 3.2.1
meld3 = 0.6.10 meld3 = 0.6.10
netaddr = 0.7.10 netaddr = 0.7.10
......
...@@ -44,7 +44,6 @@ hexagonit.recipe.download = 1.6nxd002 ...@@ -44,7 +44,6 @@ hexagonit.recipe.download = 1.6nxd002
# Required by: # Required by:
# slapos.cookbook==0.73.1 # slapos.cookbook==0.73.1
inotifyx = 0.2.0
# Required by: # Required by:
# slapos.cookbook==0.73.1 # slapos.cookbook==0.73.1
......
...@@ -62,4 +62,4 @@ md5sum = 0ea12a4ad2d2e3d406476e35b8d3e3fb ...@@ -62,4 +62,4 @@ md5sum = 0ea12a4ad2d2e3d406476e35b8d3e3fb
mode = 640 mode = 640
[versions] [versions]
slapos.recipe.template = 4.1 slapos.recipe.template = 4.2
[buildout]
extends =
../wendelin/software.cfg
../../component/mariadb/mariarocks.cfg
[local-bt5-repository]
list += ${slapos.cookbook-repository:location}/software/wendelin-scalability
[patch-template]
recipe = slapos.recipe.build
location = ${buildout:directory}/${:_buildout_section_name_}.cfg.in
script =
with open(self.options['location'], 'w') as dst, \
open(self.options['base']) as src:
src = src.read()
i = src.index('[buildout]')
dst.write(src[:i] + self.options['extra'] + '\n' + src[i:])
# ERP5: id-store-interval must be big enough to avoid conflicts (e.g. 1000)
[buildout] [buildout]
extends = test-fluentd-common.cfg extends = test-fluentd-common.cfg
...@@ -21,7 +23,6 @@ recipe = slapos.recipe.build:download ...@@ -21,7 +23,6 @@ recipe = slapos.recipe.build:download
[template-zope-patched] [template-zope-patched]
<= template-fluentd <= template-fluentd
base = ${template-zope-base:target} base = ${template-zope-base:target}
method = unpack
tags = tags =
{%- if slapparameter_dict['family'] == 'fluentd' %} {%- if slapparameter_dict['family'] == 'fluentd' %}
{%- for i, zope in enumerate(publish_list) %} {%- for i, zope in enumerate(publish_list) %}
...@@ -32,7 +33,7 @@ tags = ...@@ -32,7 +33,7 @@ tags =
[feeder] [feeder]
feeder = feeder =
# # Same algorithm as ERP5Site_simulateFluentdIngestion
import os, struct import os, struct
from random import lognormvariate from random import lognormvariate
pack = struct.Struct('!d').pack pack = struct.Struct('!d').pack
......
[buildout] [buildout]
extends = extends =
../../component/gnupg/buildout.cfg
../fluentd/software.cfg ../fluentd/software.cfg
../wendelin/software.cfg test-common.cfg
../../component/mariadb/mariarocks.cfg
[local-bt5-repository]
list += ${slapos.cookbook-repository:location}/software/wendelin-scalability
[patch-template]
recipe = slapos.recipe.build
location = ${buildout:directory}/${:_buildout_section_name_}.cfg.in
script =
with open(self.options['location'], 'w') as dst, \
open(self.options['base']) as src:
src = src.read()
i = src.index('[buildout]')
dst.write(src[:i] + self.options['extra'] + '\n' + src[i:])
[template-erp5] [template-erp5]
recipe = recipe =
...@@ -32,6 +17,7 @@ base = ${template-erp5-base:target} ...@@ -32,6 +17,7 @@ base = ${template-erp5-base:target}
[template-fluentd] [template-fluentd]
<= patch-template <= patch-template
method = unpack
extra = extra =
{%- set tags = {} %} {%- set tags = {} %}
${:tags} ${:tags}
......
# ERP5: id-store-interval must be big enough to avoid conflicts (e.g. 1000)
# XXX: Because supervisord can't handle too many processes
# ("too many open files to spawn" errors), these SR
# should instantiate the source processes differently.
[buildout]
extends = test-fluentd-common.cfg
[template-erp5-patched]
extra =
[request-balancer]
config-inituser-login = {{ dumps(inituser_login) }}
config-inituser-password = $${publish-early:inituser-password}
[template-balancer]
recipe =
target = ${template-balancer-patched:location}
[template-balancer-base]
<= template-balancer
recipe = slapos.recipe.build:download
[template-balancer-patched]
<= template-fluentd
base = ${template-balancer-base:target}
tags =
{%- set port, backend_list = haproxy_dict['fluentd'] %}
{%- for i in range(100*len(backend_list)) %}
{%- do tags.__setitem__('wendelin_tag_' ~ i, ipv4 ~ ':' ~ port) %}
{%- endfor %}
[feeder]
feeder =
#
import collections, random, struct
pack = struct.Struct('!d').pack
data = collections.deque(
''.join(chr(int(random.gauss(0, .68)) % 256) for _ in xrange(2500))
# With a period greater than 64kiB (zlib dictionary size),
# we avoid extra compression due to repetition.
for _ in xrange(30))
interval = 60 # XXX: same as fluentd flush internal
time.sleep(interval * random.random())
while True:
emit('', pack(time.time()) + data[0])
data.rotate()
time.sleep(interval)
# NEO: data deduplication must be enabled
[buildout] [buildout]
extends = test-fluentd-common.cfg extends =
../../component/gnupg/buildout.cfg
test-fluentd-common.cfg
[template-erp5-patched] [template-erp5-patched]
extra = extra =
......
# The 'start_ingest' command causes a zope to fill NEO as fast as possible,
# as if fluentd pushed data.
#
# Use sigma > 0 to have oids of variable size inside NEO.
# To know average compression ratio:
# x=test_scalability_fluentd/ExtensionTemplateItem/portal_components/extension.erp5.ScalabilityFluentd.py
# $x 10 1
# 0.434851958247
# 2155 - 65536 (99th percentile)
# $x 8.787 0
# 0.100036621094
# 6556
[buildout]
extends = test-common.cfg
parts += start_ingest
[start_ingest]
recipe = slapos.recipe.template:jinja2
rendered = ${buildout:bin-directory}/${:_buildout_section_name_}
mode = 0755
template =
inline:#!${buildout:executable}
import argparse, base64, httplib, sys
parser = argparse.ArgumentParser()
_ = parser.add_argument
_('--site-id', default='erp5')
_('hostport', metavar='host[:port]', help='Zope address')
_('password', help="'zope' user password")
_('reference', help='Data Stream reference')
_('mu', type=float)
_('sigma', type=float)
_('chunks_per_transaction', nargs='?', type=int, help='default: 128 (8 MiB)')
args = parser.parse_args()
qs = []
for k in 'reference', 'mu', 'sigma', 'chunks_per_transaction':
v = getattr(args, k)
if v is not None:
t = type(v)
qs.append('%s=%s' % (k if t is str else k + ':' + t.__name__, v))
c = httplib.HTTPConnection(args.hostport)
c.putrequest('GET', '/%s/ERP5Site_simulateFluentdIngestion?%s'
% (args.site_id, '&'.join(qs)))
c.putheader('Authorization',
'Basic ' + base64.b64encode('zope:'+args.password))
c.endheaders()
#!/usr/bin/python
from __future__ import division, print_function
import os, struct
from random import lognormvariate
bigfile_chunk_size = 65536
def simulateFluentdIngestion(self, reference, mu, sigma,
chunks_per_transaction=128):
from time import time
import transaction
note = (self['portal_ingestion_policies']['scalability_test_unpack'].getPath()
+ '/ingest')
module = self['data_stream_module']
try:
data_stream = module[reference]
except KeyError:
data_stream = module.newContent(reference, 'Data Stream')
transaction.commit()
pack = struct.Struct('!d').pack
data = os.urandom(bigfile_chunk_size - 8)
while 1:
txn = transaction.begin()
data_stream.appendData(''.join(
(pack(time()) + data[:int(lognormvariate(mu, sigma))]
).ljust(bigfile_chunk_size, '\0')
for _ in xrange(chunks_per_transaction)))
txn.note(note)
txn.commit()
if __name__ == '__main__':
import sys
mu, sigma = map(float, sys.argv[1:3])
if sigma:
try:
n = int(sys.argv[3])
except IndexError:
n = 1000000
else:
n = 1
x = sorted(min(int(lognormvariate(mu, sigma)), bigfile_chunk_size - 8)
for _ in xrange(n))
print((8 * n + sum(x)) / (bigfile_chunk_size * n))
if n == 1:
print(x[0] + 8)
else:
n //= 100
if n:
print(8 + x[n], '-', 8 + x[-n-1], '(99th percentile)')
else:
print(8 + x[0], '-', 8 + x[-1])
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="Extension Component" module="erp5.portal_type"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>_recorded_property_dict</string> </key>
<value>
<persistent> <string encoding="base64">AAAAAAAAAAI=</string> </persistent>
</value>
</item>
<item>
<key> <string>default_reference</string> </key>
<value> <string>ScalabilityFluentd</string> </value>
</item>
<item>
<key> <string>description</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>extension.erp5.ScalabilityFluentd</string> </value>
</item>
<item>
<key> <string>portal_type</string> </key>
<value> <string>Extension Component</string> </value>
</item>
<item>
<key> <string>sid</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>text_content_error_message</string> </key>
<value>
<tuple/>
</value>
</item>
<item>
<key> <string>text_content_warning_message</string> </key>
<value>
<tuple>
<string>W: 8, 46: Redefining name \'mu\' from outer scope (line 35) (redefined-outer-name)</string>
<string>W: 8, 50: Redefining name \'sigma\' from outer scope (line 35) (redefined-outer-name)</string>
</tuple>
</value>
</item>
<item>
<key> <string>version</string> </key>
<value> <string>erp5</string> </value>
</item>
<item>
<key> <string>workflow_history</string> </key>
<value>
<persistent> <string encoding="base64">AAAAAAAAAAM=</string> </persistent>
</value>
</item>
</dictionary>
</pickle>
</record>
<record id="2" aka="AAAAAAAAAAI=">
<pickle>
<global name="PersistentMapping" module="Persistence.mapping"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>data</string> </key>
<value>
<dictionary/>
</value>
</item>
</dictionary>
</pickle>
</record>
<record id="3" aka="AAAAAAAAAAM=">
<pickle>
<global name="PersistentMapping" module="Persistence.mapping"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>data</string> </key>
<value>
<dictionary>
<item>
<key> <string>component_validation_workflow</string> </key>
<value>
<persistent> <string encoding="base64">AAAAAAAAAAQ=</string> </persistent>
</value>
</item>
</dictionary>
</value>
</item>
</dictionary>
</pickle>
</record>
<record id="4" aka="AAAAAAAAAAQ=">
<pickle>
<global name="WorkflowHistoryList" module="Products.ERP5Type.patches.WorkflowTool"/>
</pickle>
<pickle>
<tuple>
<none/>
<list>
<dictionary>
<item>
<key> <string>action</string> </key>
<value> <string>validate</string> </value>
</item>
<item>
<key> <string>validation_state</string> </key>
<value> <string>validated</string> </value>
</item>
</dictionary>
</list>
</tuple>
</pickle>
</record>
</ZopeData>
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="ExternalMethod" module="Products.ExternalMethod.ExternalMethod"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>_function</string> </key>
<value> <string>simulateFluentdIngestion</string> </value>
</item>
<item>
<key> <string>_module</string> </key>
<value> <string>ScalabilityFluentd</string> </value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>ERP5Site_simulateFluentdIngestion</string> </value>
</item>
<item>
<key> <string>title</string> </key>
<value> <string></string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
extension.erp5.ScalabilityFluentd
\ No newline at end of file
portal_ingestion_policies/scalability_test_* portal_ingestion_policies/scalability_test_*
portal_skins/custom/DataStreamModule_getTotalSize portal_skins/custom/DataStreamModule_getTotalSize
portal_skins/custom/ERP5Site_handleRawDataFluentdIngestion portal_skins/custom/ERP5Site_handleRawDataFluentdIngestion
portal_skins/custom/ERP5Site_simulateFluentdIngestion
\ No newline at end of file
...@@ -10,6 +10,8 @@ tls {{ custom_cert_dict['cert-file'] }} {{ custom_cert_dict['key-file'] }} { ...@@ -10,6 +10,8 @@ tls {{ custom_cert_dict['cert-file'] }} {{ custom_cert_dict['key-file'] }} {
log {{caddy_configuration_dict['access_log']}} log {{caddy_configuration_dict['access_log']}}
errors {{caddy_configuration_dict['error_log']}}
root {{ directory_dict['public_html'] }} root {{ directory_dict['public_html'] }}
{% if parameter_dict['enable-basic-auth'] == 'true' -%} {% if parameter_dict['enable-basic-auth'] == 'true' -%}
...@@ -18,6 +20,8 @@ basicauth / {{ parameter_dict['username'] }} {{parameter_dict['password']}} ...@@ -18,6 +20,8 @@ basicauth / {{ parameter_dict['username'] }} {{parameter_dict['password']}}
} }
bind {{caddy_configuration_dict['ipv6']}}
{%- endif %} {%- endif %}
[{{caddy_configuration_dict['ipv6']}}]:{{parameter_dict['port-ipv6']}} { [{{caddy_configuration_dict['ipv6']}}]:{{parameter_dict['port-ipv6']}} {
...@@ -30,22 +34,30 @@ basicauth / {{ parameter_dict['username'] }} {{parameter_dict['password']}} ...@@ -30,22 +34,30 @@ basicauth / {{ parameter_dict['username'] }} {{parameter_dict['password']}}
log {{caddy_configuration_dict['access_log']}} log {{caddy_configuration_dict['access_log']}}
errors {{caddy_configuration_dict['error_log']}}
root {{ directory_dict['public_html'] }} root {{ directory_dict['public_html'] }}
{% if parameter_dict['enable-basic-auth'] == 'true' -%} {% if parameter_dict['enable-basic-auth'] == 'true' -%}
basicauth / {{ parameter_dict['username'] }} {{parameter_dict['password']}} basicauth / {{ parameter_dict['username'] }} {{parameter_dict['password']}}
{%- endif %} {%- endif %}
bind {{caddy_configuration_dict['ipv6']}}
} }
[{{caddy_configuration_dict['local_ip']}}]:{{parameter_dict['port-ipv4']}} { [{{caddy_configuration_dict['local_ip']}}]:{{parameter_dict['port-ipv4']}} {
log {{caddy_configuration_dict['access_log']}} log {{caddy_configuration_dict['access_log']}}
errors {{caddy_configuration_dict['error_log']}}
root {{ directory_dict['public_html'] }} root {{ directory_dict['public_html'] }}
{% if parameter_dict['enable-basic-auth'] == 'true' -%} {% if parameter_dict['enable-basic-auth'] == 'true' -%}
basicauth / {{ parameter_dict['username'] }} {{parameter_dict['password']}} basicauth / {{ parameter_dict['username'] }} {{parameter_dict['password']}}
{%- endif %} {%- endif %}
bind {{caddy_configuration_dict['local_ip']}}
} }
\ No newline at end of file
...@@ -27,7 +27,7 @@ mode = 0644 ...@@ -27,7 +27,7 @@ mode = 0644
[template-caddyfile] [template-caddyfile]
recipe = slapos.recipe.build:download recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/Caddyfile.in url = ${:_profile_base_location_}/Caddyfile.in
#md5sum = bb3b314ebeb58e9875d547a053a9f268 md5sum = 88c4c33e374ea3f61cdd36b2816d24ba
filename = Caddyfile.in filename = Caddyfile.in
location = ${buildout:parts-directory}/${:_buildout_section_name_} location = ${buildout:parts-directory}/${:_buildout_section_name_}
mode = 0644 mode = 0644
......
...@@ -94,7 +94,7 @@ caucase = 0.1.4 ...@@ -94,7 +94,7 @@ caucase = 0.1.4
futures = 3.1.1 futures = 3.1.1
gitdb2 = 2.0.2 gitdb2 = 2.0.2
gunicorn = 19.7.1 gunicorn = 19.7.1
slapos.recipe.template = 4.1 slapos.recipe.template = 4.2
slapos.toolbox = 0.73 slapos.toolbox = 0.73
smmap2 = 2.0.3 smmap2 = 2.0.3
......
...@@ -89,7 +89,7 @@ PasteScript = 2.0.2 ...@@ -89,7 +89,7 @@ PasteScript = 2.0.2
WSGIUtils = 0.7 WSGIUtils = 0.7
python-magic = 0.4.6 python-magic = 0.4.6
rdiff-backup = 1.0.5+SlapOSPatched001 rdiff-backup = 1.0.5+SlapOSPatched001
slapos.recipe.template = 4.1 slapos.recipe.template = 4.2
# Required by: # Required by:
# PasteScript==2.0 # PasteScript==2.0
......
...@@ -736,7 +736,7 @@ uuid = 1.30 ...@@ -736,7 +736,7 @@ uuid = 1.30
validictory = 1.1.0 validictory = 1.1.0
xfw = 0.10 xfw = 0.10
xupdate-processor = 0.4 xupdate-processor = 0.4
selenium = 2.53.1 selenium = 3.8.0
# Required by: # Required by:
# Products.CMFCore==2.2.10 # Products.CMFCore==2.2.10
......
...@@ -19,7 +19,7 @@ md5sum = c1f1083bf6c911a0e65dcb841fba327d ...@@ -19,7 +19,7 @@ md5sum = c1f1083bf6c911a0e65dcb841fba327d
[mariadb-slow-query-report-script] [mariadb-slow-query-report-script]
filename = mysql-querydigest.sh.in filename = mysql-querydigest.sh.in
md5sum = cfe6ab8ae54a521ecb269e9d9762cbeb md5sum = 0c0d98a68230cd0ad36046bb25b35f4a
[mariadb-start-clone-from-backup] [mariadb-start-clone-from-backup]
filename = instance-mariadb-start-clone-from-backup.sh.in filename = instance-mariadb-start-clone-from-backup.sh.in
...@@ -27,7 +27,7 @@ md5sum = 1af531c51f575a1d1362f2ca2d61620d ...@@ -27,7 +27,7 @@ md5sum = 1af531c51f575a1d1362f2ca2d61620d
[template-mariadb] [template-mariadb]
filename = instance-mariadb.cfg.in filename = instance-mariadb.cfg.in
md5sum = 8ea5033142f450a2e90431817771cb44 md5sum = da7c36ecb490b67360d2afda94b41bff
[template-kumofs] [template-kumofs]
filename = instance-kumofs.cfg.in filename = instance-kumofs.cfg.in
...@@ -79,7 +79,7 @@ md5sum = d41d8cd98f00b204e9800998ecf8427e ...@@ -79,7 +79,7 @@ md5sum = d41d8cd98f00b204e9800998ecf8427e
[template-erp5] [template-erp5]
filename = instance-erp5.cfg.in filename = instance-erp5.cfg.in
md5sum = 2465af81147af322056cee9f6c7de14f md5sum = 02ed5d9b74c70789004d01dd2ecde7b1
[template-zeo] [template-zeo]
filename = instance-zeo.cfg.in filename = instance-zeo.cfg.in
...@@ -91,7 +91,7 @@ md5sum = fd7e8c507cef1950e6c0347ce2a01021 ...@@ -91,7 +91,7 @@ md5sum = fd7e8c507cef1950e6c0347ce2a01021
[template-balancer] [template-balancer]
filename = instance-balancer.cfg.in filename = instance-balancer.cfg.in
md5sum = f64c568f1365eb1164f12f48fede9a99 md5sum = a71ad387eab681b9020e271cba2c7a79
[template-haproxy-cfg] [template-haproxy-cfg]
filename = haproxy.cfg.in filename = haproxy.cfg.in
......
...@@ -287,7 +287,7 @@ apachedex = ${monitor-directory:private}/apachedex ...@@ -287,7 +287,7 @@ apachedex = ${monitor-directory:private}/apachedex
[{{ section('monitor-generate-apachedex-report') }}] [{{ section('monitor-generate-apachedex-report') }}]
recipe = slapos.cookbook:wrapper recipe = slapos.cookbook:wrapper
wrapper-path = ${monitor-directory:reports}/${:command} wrapper-path = ${monitor-directory:reports}/${:command}
command-line = "{{ parameter_dict['run-apachedex-location'] }}" "{{ parameter_dict['apachedex-location'] }}" "${directory:apachedex}" ${monitor-publish-parameters:monitor-base-url}/private/apachedex --apache-log-list "${apachedex-parameters:apache-log-list}" --config "${apachedex-parameters:configuration}" command-line = "{{ parameter_dict['run-apachedex-location'] }}" "{{ parameter_dict['apachedex-location'] }}" "${directory:apachedex}" ${monitor-publish-parameters:monitor-base-url}/private/apachedex --apache-log-list "${apachedex-parameters:apache-log-list}" --configuration "${apachedex-parameters:configuration}"
command = apachedex_every_23_hour command = apachedex_every_23_hour
[apachedex-parameters] [apachedex-parameters]
......
...@@ -284,7 +284,7 @@ config-backend-path-dict = {{ dumps(zope_backend_path_dict) }} ...@@ -284,7 +284,7 @@ config-backend-path-dict = {{ dumps(zope_backend_path_dict) }}
config-ssl-authentication-dict = {{ dumps(ssl_authentication_dict) }} config-ssl-authentication-dict = {{ dumps(ssl_authentication_dict) }}
config-apachedex-promise-threshold = {{ dumps(monitor_dict.get('apachedex-promise-threshold', 70)) }} config-apachedex-promise-threshold = {{ dumps(monitor_dict.get('apachedex-promise-threshold', 70)) }}
config-apachedex-configuration = {{ dumps(monitor_dict.get('apachedex-configuration', config-apachedex-configuration = {{ dumps(monitor_dict.get('apachedex-configuration',
'--erp5-base "/erp5(/|$|/\?)" --skip-user-agent Zabbix --error-detail --js-embed --quiet')) }} '--erp5-base +erp5 .*/VirtualHostRoot/erp5(/|\\?|$) --base +other / --skip-user-agent Zabbix --error-detail --js-embed --quiet')) }}
[request-frontend-base] [request-frontend-base]
{% if has_frontend -%} {% if has_frontend -%}
......
...@@ -107,7 +107,7 @@ time = {{ dumps(backup_periodicity) }} ...@@ -107,7 +107,7 @@ time = {{ dumps(backup_periodicity) }}
# can be fully restored. # can be fully restored.
# master-data: use value "2" as we are not in a replication case # master-data: use value "2" as we are not in a replication case
#} #}
command = "${binary-wrap-mysqldump:wrapper-path}" -u root --all-databases --single-transaction {% if incremental_backup_retention_days > -1 %}--flush-logs --master-data=2 {% endif %}| {{ parameter_dict['gzip-location'] }}/bin/gzip > "${directory:mariadb-backup-full}/$({{ parameter_dict['coreutils-location'] }}/bin/date "+%Y%m%d%H%M%S").sql.gz" command = "${binary-wrap-mysqldump:wrapper-path}" -u root --all-databases --flush-privileges --single-transaction {% if incremental_backup_retention_days > -1 %}--flush-logs --master-data=2 {% endif %}| {{ parameter_dict['gzip-location'] }}/bin/gzip > "${directory:mariadb-backup-full}/$({{ parameter_dict['coreutils-location'] }}/bin/date "+%Y%m%d%H%M%S").sql.gz"
{# KEEP GLOB PATTERN IN SYNC with generated filenames above {# KEEP GLOB PATTERN IN SYNC with generated filenames above
# YYYYmmddHHMMSS -#} # YYYYmmddHHMMSS -#}
file-glob = ??????????????.sql.gz file-glob = ??????????????.sql.gz
...@@ -252,7 +252,7 @@ mariadb-ssl = ${:etc}/mariadb-ssl ...@@ -252,7 +252,7 @@ mariadb-ssl = ${:etc}/mariadb-ssl
var = ${buildout:directory}/var var = ${buildout:directory}/var
log = ${:var}/log log = ${:var}/log
run = ${:var}/run run = ${:var}/run
slowquery = ${monitor-directory:private}/slowquerydex slowquery = ${monitor-directory:private}/slowquery_digest
[{{ section('resiliency-exclude-file') }}] [{{ section('resiliency-exclude-file') }}]
# Generate rdiff exclude file in case of resiliency # Generate rdiff exclude file in case of resiliency
......
...@@ -7,19 +7,20 @@ PT_QUERY_EXEC='{{pt_query_exec}}' ...@@ -7,19 +7,20 @@ PT_QUERY_EXEC='{{pt_query_exec}}'
if [ ! -d "$OUTPUT_FOLDER" ]; then if [ ! -d "$OUTPUT_FOLDER" ]; then
echo "ERROR: output_folder don't exists" echo "ERROR: output_folder don't exists"
exit 0 exit 1
fi fi
OUTPUT_FILE=${OUTPUT_FOLDER}/slowquery_digest.txt dashed_today=$(date +%Y-%m-%d)
today=$(date -d "$dashed_today" +%Y%m%d)
TODAY=`date +%Y%m%d`
SLOW_LOG=$SLOW_QUERY_PATH-$TODAY SLOW_LOG="$SLOW_QUERY_PATH-$today"
OUTPUT_FILE="$OUTPUT_FOLDER/slowquery_digest.txt-$dashed_today"
if [ ! -f "$SLOW_LOG" ]; then if [ ! -f "$SLOW_LOG" ]; then
echo "ERROR: cannot read mysql slow query log file $SLOW_LOG. Exiting." echo "ERROR: cannot read mysql slow query log file $SLOW_LOG. Exiting."
exit 1 exit 1
fi fi
$PT_QUERY_EXEC $SLOW_LOG > $OUTPUT_FILE "$PT_QUERY_EXEC" "$SLOW_LOG" > "$OUTPUT_FILE" && \
echo "ok" echo "Report generated successfully." || \
echo "Report failed with code $?"
...@@ -132,4 +132,5 @@ PyRSS2Gen = 1.1 ...@@ -132,4 +132,5 @@ PyRSS2Gen = 1.1
cns.recipe.symlink = 0.2.3 cns.recipe.symlink = 0.2.3
pycurl = 7.43.0 pycurl = 7.43.0
slapos.toolbox = 0.73 slapos.toolbox = 0.73
pyasn1 = 0.3.7
...@@ -53,6 +53,9 @@ allow-hosts += ...@@ -53,6 +53,9 @@ allow-hosts +=
www.dabeaz.com www.dabeaz.com
www.owlfish.com www.owlfish.com
# Use an https index
index = https://pypi.python.org/simple/
# XXX: Workaround of SlapOS limitation # XXX: Workaround of SlapOS limitation
# Unzippig of eggs is required, as SlapOS do not yet provide nicely working # Unzippig of eggs is required, as SlapOS do not yet provide nicely working
# development / fast switching environment for whole software # development / fast switching environment for whole software
...@@ -88,6 +91,7 @@ eggs = ...@@ -88,6 +91,7 @@ eggs =
slapos.libnetworkcache slapos.libnetworkcache
[versions] [versions]
setuptools = 33.1.1
# Use SlapOS patched zc.buildout # Use SlapOS patched zc.buildout
zc.buildout = 2.5.2+slapos011 zc.buildout = 2.5.2+slapos011
# Use SlapOS patched zc.recipe.egg (zc.recipe.egg 2.x is for Buildout 2) # Use SlapOS patched zc.recipe.egg (zc.recipe.egg 2.x is for Buildout 2)
...@@ -108,7 +112,7 @@ collective.recipe.template = 2.0 ...@@ -108,7 +112,7 @@ collective.recipe.template = 2.0
cryptography = 2.1.1 cryptography = 2.1.1
decorator = 4.0.11 decorator = 4.0.11
idna = 2.2 idna = 2.2
inotifyx = 0.2.2 inotify-simple = 1.1.1
itsdangerous = 0.24 itsdangerous = 0.24
lock-file = 2.0 lock-file = 2.0
lxml = 3.7.3 lxml = 3.7.3
...@@ -122,7 +126,6 @@ pyOpenSSL = 17.2.0 ...@@ -122,7 +126,6 @@ pyOpenSSL = 17.2.0
pyparsing = 2.2.0 pyparsing = 2.2.0
pytz = 2016.10 pytz = 2016.10
requests = 2.13.0 requests = 2.13.0
setuptools = 33.1.1
six = 1.10.0 six = 1.10.0
slapos.cookbook = 1.0.53 slapos.cookbook = 1.0.53
slapos.core = 1.4.3 slapos.core = 1.4.3
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment