Commit 342b5393 authored by Aurélien Vermylen's avatar Aurélien Vermylen

Merge branch 'master' into clearroad

parents c44e2f89 503a278b
...@@ -2,6 +2,7 @@ ...@@ -2,6 +2,7 @@
extends = extends =
../coreutils/buildout.cfg ../coreutils/buildout.cfg
../patch/buildout.cfg ../patch/buildout.cfg
../randomsleep/buildout.cfg
parts = dcron-output parts = dcron-output
...@@ -18,6 +19,7 @@ make-options = ...@@ -18,6 +19,7 @@ make-options =
post-make-hook = ${:_profile_base_location_}/dcron-hooks.py#860e914dff4108b47565965fe5ebc7b5:post_make_hook post-make-hook = ${:_profile_base_location_}/dcron-hooks.py#860e914dff4108b47565965fe5ebc7b5:post_make_hook
environment = environment =
PATH=${patch:location}/bin:%(PATH)s PATH=${patch:location}/bin:%(PATH)s
randomsleep_install = ${randomsleep:location}
[dcron-output] [dcron-output]
# Shared binary location to ease migration # Shared binary location to ease migration
......
...@@ -7,14 +7,15 @@ extends = ...@@ -7,14 +7,15 @@ extends =
../bzip2/buildout.cfg ../bzip2/buildout.cfg
../zlib/buildout.cfg ../zlib/buildout.cfg
../bzip2/buildout.cfg ../bzip2/buildout.cfg
../gperf/buildout.cfg
parts = parts =
fontconfig fontconfig
[fontconfig] [fontconfig]
recipe = slapos.recipe.cmmi recipe = slapos.recipe.cmmi
url = http://fontconfig.org/release/fontconfig-2.12.1.tar.bz2 url = http://fontconfig.org/release/fontconfig-2.12.6.tar.bz2
md5sum = b5af5a423ee3b5cfc34846838963c058 md5sum = 733f5e2371ca77b69707bd7b30cc2163
pkg_config_depends = ${freetype:pkg_config_depends}:${freetype:location}/lib/pkgconfig:${libxml2:location}/lib/pkgconfig pkg_config_depends = ${freetype:pkg_config_depends}:${freetype:location}/lib/pkgconfig:${libxml2:location}/lib/pkgconfig
# XXX-Cedric : should we use --with-add-fonts={somefont:location}/share,{someotherfont:location}/share? # XXX-Cedric : should we use --with-add-fonts={somefont:location}/share,{someotherfont:location}/share?
configure-options = configure-options =
...@@ -23,7 +24,7 @@ configure-options = ...@@ -23,7 +24,7 @@ configure-options =
--enable-libxml2 --enable-libxml2
--with-default-fonts=${fonts:location} --with-default-fonts=${fonts:location}
environment = environment =
PATH=${pkgconfig:location}/bin:%(PATH)s PATH=${pkgconfig:location}/bin:${gperf:location}/bin:%(PATH)s
PKG_CONFIG_PATH=${:pkg_config_depends} PKG_CONFIG_PATH=${:pkg_config_depends}
CPPFLAGS=-I${zlib:location}/include -I${bzip2:location}/include CPPFLAGS=-I${zlib:location}/include -I${bzip2:location}/include
LDFLAGS=-L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib -L${bzip2:location}/lib -Wl,-rpath=${bzip2:location}/lib LDFLAGS=-L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib -L${bzip2:location}/lib -Wl,-rpath=${bzip2:location}/lib
...@@ -2,6 +2,7 @@ ...@@ -2,6 +2,7 @@
# Mostly required to support languages different than C or C++ # Mostly required to support languages different than C or C++
[buildout] [buildout]
extends = extends =
../gettext/buildout.cfg
../gmp/buildout.cfg ../gmp/buildout.cfg
../perl/buildout.cfg ../perl/buildout.cfg
../tar/buildout.cfg ../tar/buildout.cfg
...@@ -32,7 +33,7 @@ configure-options = ...@@ -32,7 +33,7 @@ configure-options =
--with-as=${binutils:location}/bin/as --with-as=${binutils:location}/bin/as
environment = environment =
LDFLAGS=-Wl,-rpath=${gmp:location}/lib -Wl,-rpath=${isl:location}/lib -Wl,-rpath=${mpc:location}/lib -Wl,-rpath=${mpfr:location}/lib LDFLAGS=-Wl,-rpath=${gmp:location}/lib -Wl,-rpath=${isl:location}/lib -Wl,-rpath=${mpc:location}/lib -Wl,-rpath=${mpfr:location}/lib
PATH=${binutils:location}/bin:${perl:location}/bin:${tar:location}/bin:%(PATH)s PATH=${binutils:location}/bin:${gettext:location}/bin:${perl:location}/bin:${tar:location}/bin:%(PATH)s
[gcc-minimal] [gcc-minimal]
<= gcc-common <= gcc-common
......
...@@ -34,8 +34,8 @@ make-targets= cd src && ./make.bash && cp -alf .. ${:location} ...@@ -34,8 +34,8 @@ make-targets= cd src && ./make.bash && cp -alf .. ${:location}
[golang18] [golang18]
<= golang-common <= golang-common
url = https://storage.googleapis.com/golang/go1.8.3.src.tar.gz url = https://dl.google.com/go/go1.8.7.src.tar.gz
md5sum = 64e9380e07bba907e26a00cf5fcbe77e md5sum = c61cfe9c85e7d42f903d3fe146d7cde6
# go1.8 needs go1.4 to bootstrap # go1.8 needs go1.4 to bootstrap
environment-extra = environment-extra =
...@@ -43,8 +43,8 @@ environment-extra = ...@@ -43,8 +43,8 @@ environment-extra =
[golang19] [golang19]
<= golang-common <= golang-common
url = https://storage.googleapis.com/golang/go1.9.2.src.tar.gz url = https://dl.google.com/go/go1.9.4.src.tar.gz
md5sum = 44105c865a1a810464df79233a05a568 md5sum = 6816441fd6680c63865cdd5cb8bc1960
# go1.9 needs go1.4 to bootstrap # go1.9 needs go1.4 to bootstrap
environment-extra = environment-extra =
...@@ -63,6 +63,11 @@ environment-extra = ...@@ -63,6 +63,11 @@ environment-extra =
# lab.nexedi.com/kirr/neo/go/... \ # lab.nexedi.com/kirr/neo/go/... \
# github.com/pkg/profile \ # github.com/pkg/profile \
# golang.org/x/perf/cmd/benchstat # golang.org/x/perf/cmd/benchstat
#
# it is possible to specify Go build flags used for compilation e.g. this way:
#
# [gowork]
# buildflags = -race
[gowork] [gowork]
directory = ${buildout:directory}/go.work directory = ${buildout:directory}/go.work
src = ${:directory}/src src = ${:directory}/src
...@@ -72,6 +77,9 @@ depends = ${gowork.goinstall:recipe} ...@@ -72,6 +77,9 @@ depends = ${gowork.goinstall:recipe}
# go version used for the workspace (possible to override in applications) # go version used for the workspace (possible to override in applications)
golang = ${golang19:location} golang = ${golang19:location}
# no special build flags by default
buildflags =
# everything is done by dependent parts # everything is done by dependent parts
recipe = plone.recipe.command recipe = plone.recipe.command
command = : command = :
...@@ -100,7 +108,7 @@ stop-on-error = true ...@@ -100,7 +108,7 @@ stop-on-error = true
# clients should put package list to install to gowork:install ("..." requests installing everything) # clients should put package list to install to gowork:install ("..." requests installing everything)
[gowork.goinstall] [gowork.goinstall]
recipe = plone.recipe.command recipe = plone.recipe.command
command = bash -c ". ${gowork:env.sh} && go install -v ${gowork:install}" command = bash -c ". ${gowork:env.sh} && go install ${gowork:buildflags} -v ${gowork:install}"
update-command = ${:command} update-command = ${:command}
stop-on-error = true stop-on-error = true
...@@ -115,7 +123,7 @@ git-executable = ${git:location}/bin/git ...@@ -115,7 +123,7 @@ git-executable = ${git:location}/bin/git
# 3) provide repository (which is not the same as importpath in general case) # 3) provide repository (which is not the same as importpath in general case)
# #
# the list of go packages for a go workspace state can be automatically # the list of go packages for a go workspace state can be automatically
# generated with the help of go-pkg-snapshot tool. # generated with the help of gowork-snapshot tool.
[go-git-package] [go-git-package]
<= git-repository <= git-repository
location = ${gowork:src}/${:go.importpath} location = ${gowork:src}/${:go.importpath}
...@@ -25,9 +25,16 @@ gogit_list() { ...@@ -25,9 +25,16 @@ gogit_list() {
# git_upstream_url <repo> - show current branch upstream URL # git_upstream_url <repo> - show current branch upstream URL
git_upstream_url() { git_upstream_url() {
repo=$1 repo=$1
head="`git -C $repo symbolic-ref --short HEAD`" # current branch - e.g. "t" head="`git -C $repo symbolic-ref -q --short HEAD`" # current branch - e.g. "master"
remote="`git -C $repo config --get branch.$head.remote`" # upstream name, e.g. "kirr" if [ -z $head ] ; then
remote="origin" # If we are in detached HEAD, assume the upstream name is origin
else
remote="`git -C $repo config --get branch.$head.remote`" # upstream name, e.g. "origin"
fi
url="`git -C $repo config --get remote.$remote.url`" # upstream URL url="`git -C $repo config --get remote.$remote.url`" # upstream URL
if [ -z $url ] ; then
echo "ERROR: Failed to guess upstream url from $repo" 1>&2
fi
echo "$url" echo "$url"
} }
......
...@@ -2,6 +2,7 @@ ...@@ -2,6 +2,7 @@
extends = extends =
../hdf5/buildout.cfg ../hdf5/buildout.cfg
../cython/buildout.cfg ../cython/buildout.cfg
../numpy/buildout.cfg
parts = parts =
h5py h5py
...@@ -12,7 +13,9 @@ CPPFLAGS = -I${hdf5:location}/include ...@@ -12,7 +13,9 @@ CPPFLAGS = -I${hdf5:location}/include
[h5py] [h5py]
recipe = zc.recipe.egg:custom recipe = zc.recipe.egg:custom
egg = h5py egg = h5py
setup-eggs = ${cython:egg} setup-eggs =
${cython:egg}
${numpy:egg}
include-dirs = include-dirs =
${hdf5:location}/include ${hdf5:location}/include
library-dirs = library-dirs =
......
...@@ -11,8 +11,8 @@ parts = ...@@ -11,8 +11,8 @@ parts =
[libxml2] [libxml2]
recipe = slapos.recipe.cmmi recipe = slapos.recipe.cmmi
url = http://xmlsoft.org/sources/libxml2-2.9.5.tar.gz url = http://xmlsoft.org/sources/libxml2-2.9.7.tar.gz
md5sum = 5ce0da9bdaa267b40c4ca36d35363b8b md5sum = 896608641a08b465098a40ddf51cefba
configure-options = configure-options =
--disable-static --disable-static
--without-python --without-python
......
...@@ -7,8 +7,8 @@ parts = ...@@ -7,8 +7,8 @@ parts =
libxslt libxslt
[libxslt] [libxslt]
url = ftp://xmlsoft.org/libxslt/libxslt-1.1.30.tar.gz url = ftp://xmlsoft.org/libxslt/libxslt-1.1.32.tar.gz
md5sum = 70becbbcb1dad55f14de0b84171b91d5 md5sum = 1fc72f98e98bf4443f1651165f3aa146
recipe = slapos.recipe.cmmi recipe = slapos.recipe.cmmi
configure-options = configure-options =
--disable-static --disable-static
...@@ -20,5 +20,3 @@ configure-options = ...@@ -20,5 +20,3 @@ configure-options =
environment = environment =
CPPFLAGS=-I${zlib:location}/include CPPFLAGS=-I${zlib:location}/include
LDFLAGS=-L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib LDFLAGS=-L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib
LIBXML_CFLAGS=-I${libxml2:location}/include
LIBXML_LIBS=-L${libxml2:location}/lib -Wl,-rpath=${libxml2:location}/lib
...@@ -27,8 +27,8 @@ parts = ...@@ -27,8 +27,8 @@ parts =
[mariadb] [mariadb]
recipe = slapos.recipe.cmmi recipe = slapos.recipe.cmmi
url = https://downloads.mariadb.org/f/mariadb-${:version}/source/mariadb-${:version}.tar.gz/from/http%3A//fr.mirror.babylon.network/mariadb/?serve url = https://downloads.mariadb.org/f/mariadb-${:version}/source/mariadb-${:version}.tar.gz/from/http%3A//fr.mirror.babylon.network/mariadb/?serve
version = 10.1.29 version = 10.1.31
md5sum = 6d2cc6b0f8c5c7525e08aaddad8dca96 md5sum = 14ab0398c019eb531bc29f2c437ccb51
patch-options = -p0 patch-options = -p0
patches = patches =
${:_profile_base_location_}/mariadb_10.1.21_create_system_tables__no_test.patch#3c76aa9564a162f13aced7c0a3f783b3 ${:_profile_base_location_}/mariadb_10.1.21_create_system_tables__no_test.patch#3c76aa9564a162f13aced7c0a3f783b3
......
# Do not extend any file that touch buildout:parts. # Do not extend any file that touch buildout:parts.
[mariadb] [mariadb]
version = 10.2.11 version = 10.2.13
md5sum = 954088299fe5f11b4fda3b540558adbd md5sum = 20c61bd4059ba287e54cfb2862bae81d
stable-patches = stable-patches =
configure-options += configure-options +=
# force build of TokuDB due to a regression in 10.2.11
-DTOKUDB_OK=1
-DCMAKE_C_COMPILER=${gcc:location}/bin/gcc -DCMAKE_C_COMPILER=${gcc:location}/bin/gcc
-DCMAKE_CXX_COMPILER=${gcc:location}/bin/g++ -DCMAKE_CXX_COMPILER=${gcc:location}/bin/g++
extra_cflags = -I${zstd:location}/include extra_cflags = -I${zstd:location}/include
......
...@@ -18,6 +18,7 @@ patch-options = -p0 ...@@ -18,6 +18,7 @@ patch-options = -p0
patch-binary = ${patch:location}/bin/patch patch-binary = ${patch:location}/bin/patch
patches = patches =
http://nginx.org/download/patch.2017.ranges.txt#40bf9f37c881cb3b10cfefd84ca92f6a http://nginx.org/download/patch.2017.ranges.txt#40bf9f37c881cb3b10cfefd84ca92f6a
${:_profile_base_location_}/fix-gcc7-implicit-fallthrough-errors.patch
[nginx] [nginx]
<= nginx-common <= nginx-common
......
commit 8449f750e62cd229026e9df3bd023ec7e073a7d4
Author: Maxim Dounin <mdounin@mdounin.ru>
Date: Thu Apr 27 16:57:18 2017 +0300
Added missing "fall through" comments (ticket #1259).
Found by gcc7 (-Wimplicit-fallthrough).
diff --git src/core/ngx_murmurhash.c src/core/ngx_murmurhash.c
index c31e0e03..5ade658d 100644
--- src/core/ngx_murmurhash.c
+++ src/core/ngx_murmurhash.c
@@ -35,8 +35,10 @@ ngx_murmur_hash2(u_char *data, size_t len)
switch (len) {
case 3:
h ^= data[2] << 16;
+ /* fall through */
case 2:
h ^= data[1] << 8;
+ /* fall through */
case 1:
h ^= data[0];
h *= 0x5bd1e995;
diff --git src/http/ngx_http_parse.c src/http/ngx_http_parse.c
index 36220fdc..e8e51563 100644
--- src/http/ngx_http_parse.c
+++ src/http/ngx_http_parse.c
@@ -1396,6 +1396,7 @@ ngx_http_parse_complex_uri(ngx_http_request_t *r, ngx_uint_t merge_slashes)
goto done;
case '+':
r->plus_in_uri = 1;
+ /* fall through */
default:
state = sw_usual;
*u++ = ch;
@@ -1437,6 +1438,7 @@ ngx_http_parse_complex_uri(ngx_http_request_t *r, ngx_uint_t merge_slashes)
goto done;
case '+':
r->plus_in_uri = 1;
+ /* fall through */
default:
state = sw_usual;
*u++ = ch;
@@ -1484,6 +1486,7 @@ ngx_http_parse_complex_uri(ngx_http_request_t *r, ngx_uint_t merge_slashes)
goto done;
case '+':
r->plus_in_uri = 1;
+ /* fall through */
default:
state = sw_usual;
*u++ = ch;
diff --git src/os/unix/ngx_process.c src/os/unix/ngx_process.c
index dd50b5ca..993c032a 100644
--- src/os/unix/ngx_process.c
+++ src/os/unix/ngx_process.c
@@ -413,6 +413,7 @@ ngx_signal_handler(int signo, siginfo_t *siginfo, void *ucontext)
break;
}
ngx_debug_quit = 1;
+ /* fall through */
case ngx_signal_value(NGX_SHUTDOWN_SIGNAL):
ngx_quit = 1;
action = ", shutting down";
...@@ -32,8 +32,8 @@ setup-eggs = ...@@ -32,8 +32,8 @@ setup-eggs =
${matplotlib:egg} ${matplotlib:egg}
${pillow-python:egg} ${pillow-python:egg}
patches = patches =
${:_profile_base_location_}/ocropy.patch ${:_profile_base_location_}/ocropy.patch#dd7a02e1e63ed9df68e3a539b3e919eb
patch-options = -p0 patch-options = -p1
patch-binary = ${patch:location}/bin/patch patch-binary = ${patch:location}/bin/patch
environment = ocropy-env environment = ocropy-env
find-links = https://github.com/tmbdev/ocropy/tarball/4efbddca22bb2f0c639af0694e7a1386f2f097b5/ocropy-1.0.tar.gz find-links = https://github.com/tmbdev/ocropy/tarball/4efbddca22bb2f0c639af0694e7a1386f2f097b5/ocropy-1.0.tar.gz
......
diff --git ocrolib/__init__.py ocrolib/__init__.py From 1bb1546b12b0c08b1b32b293207de2d58d43ff1c Mon Sep 17 00:00:00 2001
From: Francois Le Corre <francois.lecorre@nexedi.com>
Date: Thu, 6 Apr 2017 11:32:27 +0200
Subject: [PATCH] WIP
- New OCROPY_MODEL_PATH environment variable to specify the position of models
- Import for the first time a module that compiles C code from python,
allowing us to have the static library built before runtime and
preventing us from using gcc through zope.
---
ocrolib/__init__.py | 3 ++-
ocrolib/common.py | 7 ++++---
ocrolib/native.py | 1 +
setup.py | 15 ++++++++++++---
4 files changed, 19 insertions(+), 7 deletions(-)
diff --git a/ocrolib/__init__.py b/ocrolib/__init__.py
index 1e0d627..81e85fb 100644 index 1e0d627..81e85fb 100644
--- ocrolib/__init__.py --- a/ocrolib/__init__.py
+++ ocrolib/__init__.py +++ b/ocrolib/__init__.py
@@ -1,7 +1,7 @@ @@ -1,7 +1,7 @@
__all__ = [ __all__ = [
"binnednn","cairoextras","common","components","dbtables", "binnednn","cairoextras","common","components","dbtables",
...@@ -11,18 +27,18 @@ index 1e0d627..81e85fb 100644 ...@@ -11,18 +27,18 @@ index 1e0d627..81e85fb 100644
] ]
################################################################ ################################################################
@@ -9,5 +9,6 @@ __all__ = [ @@ -9,5 +9,6 @@
################################################################ ################################################################
import default import default
+from psegutils import * +from psegutils import *
from common import * from common import *
from default import traceback as trace from default import traceback as trace
diff --git ocrolib/common.py ocrolib/common.py diff --git a/ocrolib/common.py b/ocrolib/common.py
index 27c0f26..14f088f 100644 index 27c0f26..14f088f 100644
--- ocrolib/common.py --- a/ocrolib/common.py
+++ ocrolib/common.py +++ b/ocrolib/common.py
@@ -14,6 +14,7 @@ import unicodedata @@ -14,6 +14,7 @@
import inspect import inspect
import glob import glob
import cPickle import cPickle
...@@ -38,7 +54,7 @@ index 27c0f26..14f088f 100644 ...@@ -38,7 +54,7 @@ index 27c0f26..14f088f 100644
def load_object(fname,zip=0,nofind=0,verbose=0): def load_object(fname,zip=0,nofind=0,verbose=0):
"""Loads an object from disk. By default, this handles zipped files """Loads an object from disk. By default, this handles zipped files
and searches in the usual places for OCRopus. It also handles some and searches in the usual places for OCRopus. It also handles some
@@ -439,8 +441,7 @@ def load_object(fname,zip=0,nofind=0,verbose=0): @@ -439,8 +441,7 @@ class names that have changed."""
if zip==0 and fname.endswith(".gz"): if zip==0 and fname.endswith(".gz"):
zip = 1 zip = 1
if zip>0: if zip>0:
...@@ -57,10 +73,10 @@ index 27c0f26..14f088f 100644 ...@@ -57,10 +73,10 @@ index 27c0f26..14f088f 100644
possible_prefixes.append("/usr/local/share/ocropus") possible_prefixes.append("/usr/local/share/ocropus")
diff --git ocrolib/native.py ocrolib/native.py diff --git a/ocrolib/native.py b/ocrolib/native.py
index b7a207f..240450b 100644 index b7a207f..240450b 100644
--- ocrolib/native.py --- a/ocrolib/native.py
+++ ocrolib/native.py +++ b/ocrolib/native.py
@@ -44,6 +44,7 @@ class CompileError(Exception): @@ -44,6 +44,7 @@ class CompileError(Exception):
def compile_and_find(c_string,prefix=".pynative",opt="-g -O4",libs="-lm", def compile_and_find(c_string,prefix=".pynative",opt="-g -O4",libs="-lm",
...@@ -69,22 +85,22 @@ index b7a207f..240450b 100644 ...@@ -69,22 +85,22 @@ index b7a207f..240450b 100644
if not os.path.exists(prefix): if not os.path.exists(prefix):
os.mkdir(prefix) os.mkdir(prefix)
m = hashlib.md5() m = hashlib.md5()
diff --git setup.py setup.py diff --git a/setup.py b/setup.py
index 2ec5832..6697b12 100644 index 2ec5832..0ad4d85 100644
--- setup.py --- a/setup.py
+++ setup.py +++ b/setup.py
@@ -10,7 +10,9 @@ assert sys.version_info[0]==2 and sys.version_info[1]>=7,\ @@ -10,7 +10,9 @@
from distutils.core import setup #, Extension, Command from distutils.core import setup #, Extension, Command
#from distutils.command.install_data import install_data #from distutils.command.install_data import install_data
-if not os.path.exists("models/en-default.pyrnn.gz"): -if not os.path.exists("models/en-default.pyrnn.gz"):
+models = os.environ.get('OCROPY_MODEL_PATH', '').split(':') or \ +models = os.environ.get('OCROPY_MODEL_PATH')
+ [c for c in glob.glob("models/*pyrnn.gz")] +models = models.split(':') if models else glob.glob("models/*pyrnn.gz")
+if not models: +if not models:
print() print()
print("You should download the default model 'en-default.pyrnn.gz'") print("You should download the default model 'en-default.pyrnn.gz'")
print("and put it into ./models.") print("and put it into ./models.")
@@ -18,16 +20,23 @@ if not os.path.exists("models/en-default.pyrnn.gz"): @@ -18,16 +20,23 @@
print("Check https://github.com/tmbdev/ocropy for the location") print("Check https://github.com/tmbdev/ocropy for the location")
print("of model files.") print("of model files.")
print() print()
...@@ -110,3 +126,6 @@ index 2ec5832..6697b12 100644 ...@@ -110,3 +126,6 @@ index 2ec5832..6697b12 100644
+ data_files= [('share/ocropus', models), ("", ["LICENSE"])], + data_files= [('share/ocropus', models), ("", ["LICENSE"])],
scripts = scripts, scripts = scripts,
) )
--
2.14.1
From efa86aa5ad5b89ec329b3cde288cb119efb21c89 Mon Sep 17 00:00:00 2001
From: Kirill Smelkov <kirr@nexedi.com>
Date: Thu, 06 Jul 2017 21:15:31 +0200
Subject: [PATCH] asyncore: switch default to use poll instead of select
Recently we started to get the following errors on a Wendelin production
instance:
File ".../bin/runzope", line 211, in <module>
sys.exit(Zope2.Startup.run.run())
File ".../eggs/Zope2-2.13.24-py2.7.egg/Zope2/Startup/run.py", line 26, in run
starter.run()
File ".../eggs/Zope2-2.13.24-py2.7.egg/Zope2/Startup/__init__.py", line 111, in run
Lifetime.loop()
File ".../eggs/Zope2-2.13.24-py2.7.egg/Lifetime/__init__.py", line 43, in loop
lifetime_loop()
File ".../eggs/Zope2-2.13.24-py2.7.egg/Lifetime/__init__.py", line 53, in lifetime_loop
asyncore.poll(timeout, map)
File ".../parts/python2.7/lib/python2.7/asyncore.py", line 145, in poll
r, w, e = select.select(r, w, e, timeout)
ValueError: filedescriptor out of range in select()
Initially we thought the reason here is that number of file descriptors this
process uses goes beyond allowed limit (65K open files on that instance)
because wendelin.core uses 1 fd per an array view (opening
/dev/shm/ramh.XXXXX) but that turned out to be not strictly that case:
The reason here is that select() has limit for how many #fd it can
monitor at all. The limit is system-specific but on Linux it is usually
1024 - http://man7.org/linux/man-pages/man2/select.2.html#BUGS :
$ cat s.c
#include <sys/select.h>
#include <stdio.h>
int main() {
printf("%d\n", FD_SETSIZE);
return 0;
}
$ tcc -run s.c
1024
Also select() can monitor only file descriptors which are by itself
"< FD_SETSIZE", e.g. it cannot monitor fd=1025 even if there are only 10
opened file descriptors because 1025 is not < 1024.
As was said above in wendelin.core every array view uses 1 file
descriptor, so if we are using not so small amount of arrays, even
though #fd does not go beyond process-specific ulimit, because of
select() usage the total number of allowed opened file descriptors
becomes essentially 1024.
So let's switch from select() to poll() which does not have this 1024
#fd limit. Asyncore already support using poll() out of the box -
either via passing use_pull=True to asyncore.loop()
https://docs.python.org/2/library/asyncore.html#asyncore.loop
or by using asyncore.poll2() instead of asyncore.poll()
https://github.com/python/cpython/blob/2.7/Lib/asyncore.py#L170
https://github.com/python/cpython/blob/2.7/Lib/asyncore.py#L125
--------
One option would be to switch asyncore.poll() -> asyncore.poll2() for only 1
place in Zope - inside lifetime_loop(). There are however many such similar
places in Zope (e.g. in medusa) and in other software.
For this reason, for me, what makes sense to do is not to patch all such
places, but instead change via runtime-patching asyncore.poll to be
asyncore.poll2 - this way all software will automatically benefit from
poll() usage instead of select.
P.S.
What might also make sense to do in the future is to actually let
asyncore.poll to use epoll(), as both select() and poll() are doing all
fd registration on _every_ call, so when #fd grows this preparation
time grows too. For epoll() file descriptors are registered only once.
For this to work asyncore.socket_map has to be patched also, since there
are places in code which modify this fd registry directly (e.g. remove
fd from there etc)
Original patch & discussion: https://lab.nexedi.com/kirr/Zope/commit/c6addb05
---
Lib/asyncore.py | 9 +++++++++
1 file changed, 9 insertions(+)
diff --git a/Lib/asyncore.py b/Lib/asyncore.py
index 105982f..a3025a4 100644
--- a/Lib/asyncore.py
+++ b/Lib/asyncore.py
@@ -661,3 +661,12 @@ if os.name == 'posix':
self.socket = file_wrapper(fd)
self._fileno = self.socket.fileno()
self.add_channel()
+
+# monkeypatch to switch everyone's default to use poll instead of select
+# (select has 1024 fd limit)
+# TODO it is better to use epoll
+_poll_select = poll
+_poll_poll = poll2
+poll = _poll_poll
+
+#print >>sys.stderr, 'I(nxd): asyncore: using poll instead of select'
--
2.16.1.101.gde0f0111ea
...@@ -43,6 +43,7 @@ patches = ...@@ -43,6 +43,7 @@ patches =
${:_profile_base_location_}/fix_compiler_module_issue_20613.patch#94443a77f903e9de880a029967fa6aa7 ${:_profile_base_location_}/fix_compiler_module_issue_20613.patch#94443a77f903e9de880a029967fa6aa7
${:_profile_base_location_}/pytracemalloc_pep445.patch#46662cf0ccc7cb7cfb8289bbfd68b21a ${:_profile_base_location_}/pytracemalloc_pep445.patch#46662cf0ccc7cb7cfb8289bbfd68b21a
${:_profile_base_location_}/disabled_module_list.patch#71ad30d32bcdbc50c19cf48675b1246e ${:_profile_base_location_}/disabled_module_list.patch#71ad30d32bcdbc50c19cf48675b1246e
${:_profile_base_location_}/asyncore_poll_insteadof_select.patch#ab6991c0ee6e25aeb8951e71f280a2f1
url = url =
http://www.python.org/ftp/python/${:package_version}/Python-${:package_version}${:package_version_suffix}.tar.xz http://www.python.org/ftp/python/${:package_version}/Python-${:package_version}${:package_version_suffix}.tar.xz
configure-options = configure-options =
......
...@@ -19,12 +19,11 @@ extends = ...@@ -19,12 +19,11 @@ extends =
[kvm] [kvm]
recipe = slapos.recipe.cmmi recipe = slapos.recipe.cmmi
# qemu-kvm and qemu are now the same since 1.3. # qemu-kvm and qemu are now the same since 1.3.
url = http://wiki.qemu-project.org/download/qemu-2.8.0.tar.bz2 url = http://wiki.qemu-project.org/download/qemu-2.11.0.tar.bz2
md5sum = 17940dce063b6ce450a12e719a6c9c43 md5sum = 335994a755bc655e88a87aeb36bfc0b9
configure-options = configure-options =
--target-list="$(uname -m 2>/dev/null|sed 's,^i[456]86$,i386,')-softmmu" --target-list="$(uname -m 2>/dev/null|sed 's,^i[456]86$,i386,')-softmmu"
--enable-system --enable-system
--with-system-pixman
--disable-sdl --disable-sdl
--disable-xen --disable-xen
--disable-vnc-sasl --disable-vnc-sasl
...@@ -58,25 +57,26 @@ location = ${buildout:parts-directory}/${:_buildout_section_name_} ...@@ -58,25 +57,26 @@ location = ${buildout:parts-directory}/${:_buildout_section_name_}
<= debian-netinst-base <= debian-netinst-base
arch = amd64 arch = amd64
[debian-amd64-squeeze-netinst.iso]
<= debian-amd64-netinst-base
version = 6.0.10
md5sum = 7f82d341561035f65933da43f94d5b52
[debian-amd64-wheezy-netinst.iso] [debian-amd64-wheezy-netinst.iso]
<= debian-amd64-netinst-base <= debian-amd64-netinst-base
version = 7.11.0 version = 7.11.0
md5sum = 096c1c18b44c269808bd815d58c53c8f md5sum = 096c1c18b44c269808bd815d58c53c8f
[debian-amd64-jessie-netinst.iso]
<= debian-amd64-netinst-base
release = archive
version = 8.10.0
md5sum = 19dcfc381bd3e609c6056216d203f5bc
[debian-amd64-netinst.iso] [debian-amd64-netinst.iso]
# Download the installer of Debian 8 (Jessie) # Download the installer of Debian 9 (Stretch)
<= debian-amd64-netinst-base <= debian-amd64-netinst-base
release = release/current release = release/current
version = 8.6.0 version = 9.3.0
md5sum = e9f61bf327db6d8f7cee05a99f2353cc md5sum = db8ab7871bc2b7d456c4746e706fb5d3
[debian-amd64-testing-netinst.iso] [debian-amd64-testing-netinst.iso]
# Download the installer of Debian Stretch # Download the installer of Debian Buster
<= debian-amd64-netinst-base <= debian-amd64-netinst-base
release = stretch_di_alpha7 release = stretch_di_alpha7
version = stretch-DI-alpha7 version = stretch-DI-alpha7
......
[buildout]
extends =
../coreutils/buildout.cfg
../bash/buildout.cfg
[randomsleep]
recipe = slapos.recipe.build
location = ${buildout:parts-directory}/${:_buildout_section_name_}
bin_dir = ${buildout:bin-directory}
bash_script_code =
if [ "$#" -ne 1 ]; then
echo "usage: randomsleep maxseconds"
exit
fi
${coreutils:location}/bin/sleep $((RANDOM % $1)).$((RANDOM % 100))
wrapper_script_code =
#!${bash:location}/bin/bash
${bash:location}/bin/bash ${:location}/randomsleep.bash "$@"
script =
os.makedirs(self.options['location'])
bash_script_path = os.path.join(self.options['location'], 'randomsleep.bash')
with open(bash_script_path, 'w') as f:
f.write(self.options['bash_script_code'])
wrapper_script_path = os.path.join(self.options['bin_dir'], 'randomsleep')
with open(wrapper_script_path, 'w') as f:
f.write(self.options['wrapper_script_code'])
os.chmod(wrapper_script_path, 0750)
...@@ -7,8 +7,8 @@ parts = ...@@ -7,8 +7,8 @@ parts =
[rsync] [rsync]
recipe = slapos.recipe.cmmi recipe = slapos.recipe.cmmi
url = http://rsync.samba.org/ftp/rsync/src/rsync-3.1.1.tar.gz url = https://download.samba.org/pub/rsync/rsync-3.1.2.tar.gz
md5sum = 43bd6676f0b404326eee2d63be3cdcfe md5sum = 0f758d7e000c0f7f7d3792610fad70cb
make-options = make-options =
PREFIX=${buildout:parts-directory}/${:_buildout_section_name_} PREFIX=${buildout:parts-directory}/${:_buildout_section_name_}
environment = environment =
......
diff --git a/tensorboard/pip_package/build_pip_package.sh b/tensorboard/pip_package/build_pip_package.sh diff --git a/tensorboard/pip_package/build_pip_package.sh b/tensorboard/pip_package/build_pip_package.sh
index b386d59..f03b056 100755 index 88fb5a2..782dbc5 100755
--- a/tensorboard/pip_package/build_pip_package.sh --- a/tensorboard/pip_package/build_pip_package.sh
+++ b/tensorboard/pip_package/build_pip_package.sh +++ b/tensorboard/pip_package/build_pip_package.sh
@@ -26,6 +26,7 @@ function main() { @@ -26,6 +26,7 @@ function main() {
...@@ -10,14 +10,14 @@ index b386d59..f03b056 100755 ...@@ -10,14 +10,14 @@ index b386d59..f03b056 100755
echo $(date) : "=== Using tmpdir: ${TMPDIR}" echo $(date) : "=== Using tmpdir: ${TMPDIR}"
@@ -45,8 +46,8 @@ function main() { @@ -44,9 +45,7 @@ function main() {
pushd ${TMPDIR} >/dev/null
rm -f MANIFEST rm -f MANIFEST
echo $(date) : "=== Building wheel" echo $(date) : "=== Building python2 wheel in $PWD"
echo $(pwd) - python setup.py bdist_wheel --python-tag py2 >/dev/null
- python setup.py bdist_wheel >/dev/null - echo $(date) : "=== Building python3 wheel in $PWD"
- python3 setup.py bdist_wheel >/dev/null - python setup.py bdist_wheel --python-tag py3 >/dev/null
+ PYTHONPATH=${WORKDIR}/${RUNFILES} $PYTHON_BIN_PATH setup.py bdist_egg >/dev/null + PYTHONPATH=${WORKDIR}/${RUNFILES} $PYTHON_BIN_PATH setup.py bdist_egg >/dev/null
+ #python3 setup.py bdist_wheel >/dev/null
mkdir -p ${DEST} mkdir -p ${DEST}
cp dist/* ${DEST} cp dist/* ${DEST}
popd popd >/dev/null
...@@ -11,11 +11,11 @@ parts = ...@@ -11,11 +11,11 @@ parts =
recipe = plone.recipe.command recipe = plone.recipe.command
stop-on-error = true stop-on-error = true
repository = https://github.com/tensorflow/tensorboard repository = https://github.com/tensorflow/tensorboard
tag = 0.4 tag = 0.4.0
git-binary = ${git:location}/bin/git git-binary = ${git:location}/bin/git
patch-binary = ${patch:location}/bin/patch patch-binary = ${patch:location}/bin/patch
location = ${buildout:parts-directory}/${:_buildout_section_name_} location = ${buildout:parts-directory}/${:_buildout_section_name_}
command = export HOME=${:location}; (${:git-binary} clone --quiet -b ${:tag} ${:repository} ${:location}; cd ${buildout:parts-directory} ; ${:patch-binary} -p1 -d ${:_buildout_section_name_} < ${:_profile_base_location_}/0.4.patch ) || (rm -fr ${:location}; exit 1) command = export HOME=${:location}; (${:git-binary} clone --quiet -b ${:tag} ${:repository} ${:location}; cd ${buildout:parts-directory} ; ${:patch-binary} -p1 -d ${:_buildout_section_name_} < ${:_profile_base_location_}/0.4.0.patch ) || (rm -fr ${:location}; exit 1)
[tensorboard-build] [tensorboard-build]
recipe = slapos.recipe.build recipe = slapos.recipe.build
...@@ -61,7 +61,7 @@ numpy-python-command = ${buildout:bin-directory}/${numpy-egg:interpreter} ...@@ -61,7 +61,7 @@ numpy-python-command = ${buildout:bin-directory}/${numpy-egg:interpreter}
script = script =
os.makedirs(location) os.makedirs(location)
workdir = self.options['tensorboard-repository-path'] workdir = self.options['tensorboard-repository-path']
egg_name = 'tensorflow_tensorboard-0.4.0rc3-py2.7.egg' egg_name = 'tensorflow_tensorboard-0.4.0-py2.7.egg'
dist_dir = os.path.join(workdir, 'dist') dist_dir = os.path.join(workdir, 'dist')
dest_dir = os.path.join(self.buildout['buildout']['eggs-directory'], egg_name) dest_dir = os.path.join(self.buildout['buildout']['eggs-directory'], egg_name)
env = {'PATH':':'.join([self.options['bazel-bin'], env = {'PATH':':'.join([self.options['bazel-bin'],
......
...@@ -5,8 +5,8 @@ extends = ...@@ -5,8 +5,8 @@ extends =
[util-linux] [util-linux]
recipe = slapos.recipe.cmmi recipe = slapos.recipe.cmmi
url = http://www.kernel.org/pub/linux/utils/util-linux/v2.29/util-linux-2.29.tar.xz url = https://www.kernel.org/pub/linux/utils/util-linux/v2.31/util-linux-2.31.1.tar.xz
md5sum = 07b6845f48a421ad5844aa9d58edb837 md5sum = 7733b583dcb51518944d42aa62ef19ea
configure-options = configure-options =
--disable-static --disable-static
--enable-libuuid --enable-libuuid
...@@ -29,7 +29,7 @@ configure-options = ...@@ -29,7 +29,7 @@ configure-options =
--disable-rename --disable-rename
--disable-schedutils --disable-schedutils
--disable-setterm --disable-setterm
--disable-libsmartcols --enable-libsmartcols
--disable-switch_root --disable-switch_root
--disable-tinfo --disable-tinfo
--disable-tls --disable-tls
......
[buildout]
parts=
zbar
[zbar]
recipe = slapos.recipe.cmmi
url = https://jaist.dl.sourceforge.net/project/zbar/zbar/0.10/zbar-0.10.tar.bz2
md5sum = 0fd61eb590ac1bab62a77913c8b086a5
configure-options =
--disable-video
--without-imagemagick
--without-gtk
--without-xv
--without-qt
--without-python
--without-x
--without-jpg
environment =
CFLAGS=
[buildout]
extends =
../pillow/buildout.cfg
../zbar/buildout.cfg
parts =
zbarlight
[zbarlight]
recipe = zc.recipe.egg:custom
egg = zbarlight
include-dirs =
${zbar:location}/include
library-dirs =
${zbar:location}/lib
rpath =
${zbar:location}/lib
depend-eggs = ${pillow-python:egg}
...@@ -5,7 +5,7 @@ parts = ...@@ -5,7 +5,7 @@ parts =
[zstd] [zstd]
recipe = slapos.recipe.cmmi recipe = slapos.recipe.cmmi
location = ${buildout:parts-directory}/${:_buildout_section_name_} location = ${buildout:parts-directory}/${:_buildout_section_name_}
url = https://github.com/facebook/zstd/archive/v1.3.1.tar.gz url = https://github.com/facebook/zstd/archive/v1.3.3.tar.gz
md5sum = e849ceef2f090240f690c13fba6ca70b md5sum = 187f8df17a75a74f78a23ea4806ac65f
configure-command = : configure-command = :
make-options = PREFIX=${:location} make-options = PREFIX=${:location}
...@@ -72,7 +72,6 @@ setup(name=name, ...@@ -72,7 +72,6 @@ setup(name=name,
'zc.buildout': [ 'zc.buildout': [
'addresiliency = slapos.recipe.addresiliency:Recipe', 'addresiliency = slapos.recipe.addresiliency:Recipe',
'accords = slapos.recipe.accords:Recipe', 'accords = slapos.recipe.accords:Recipe',
'apache.zope.backend = slapos.recipe.apache_zope_backend:Recipe',
'apacheperl = slapos.recipe.apacheperl:Recipe', 'apacheperl = slapos.recipe.apacheperl:Recipe',
'apachephp = slapos.recipe.apachephp:Recipe', 'apachephp = slapos.recipe.apachephp:Recipe',
'apachephpconfigure = slapos.recipe.apachephpconfigure:Recipe', 'apachephpconfigure = slapos.recipe.apachephpconfigure:Recipe',
...@@ -118,14 +117,11 @@ setup(name=name, ...@@ -118,14 +117,11 @@ setup(name=name,
'generic.mysql.wrap_update_mysql = slapos.recipe.generic_mysql:WrapUpdateMySQL', 'generic.mysql.wrap_update_mysql = slapos.recipe.generic_mysql:WrapUpdateMySQL',
'generic.mysql.wrap_mysqld = slapos.recipe.generic_mysql:WrapMySQLd', 'generic.mysql.wrap_mysqld = slapos.recipe.generic_mysql:WrapMySQLd',
'generic.varnish = slapos.recipe.generic_varnish:Recipe', 'generic.varnish = slapos.recipe.generic_varnish:Recipe',
'generic.zope = slapos.recipe.generic_zope:Recipe',
'generic.zope.zeo.client = slapos.recipe.generic_zope_zeo_client:Recipe',
'gitinit = slapos.recipe.gitinit:Recipe', 'gitinit = slapos.recipe.gitinit:Recipe',
'haproxy = slapos.recipe.haproxy:Recipe', 'haproxy = slapos.recipe.haproxy:Recipe',
'ipv4toipv6 = slapos.recipe.6tunnel:FourToSix', 'ipv4toipv6 = slapos.recipe.6tunnel:FourToSix',
'ipv6toipv4 = slapos.recipe.6tunnel:SixToFour', 'ipv6toipv4 = slapos.recipe.6tunnel:SixToFour',
'jsondump = slapos.recipe.jsondump:Recipe', 'jsondump = slapos.recipe.jsondump:Recipe',
'kumofs = slapos.recipe.kumofs:Recipe',
'kvm.frontend = slapos.recipe.kvm_frontend:Recipe', 'kvm.frontend = slapos.recipe.kvm_frontend:Recipe',
'lamp = slapos.recipe.lamp:Request', 'lamp = slapos.recipe.lamp:Request',
'lamp.generic = slapos.recipe.lampgeneric:Recipe', 'lamp.generic = slapos.recipe.lampgeneric:Recipe',
...@@ -136,7 +132,6 @@ setup(name=name, ...@@ -136,7 +132,6 @@ setup(name=name,
'libcloudrequest = slapos.recipe.libcloudrequest:Recipe', 'libcloudrequest = slapos.recipe.libcloudrequest:Recipe',
'logrotate = slapos.recipe.logrotate:Recipe', 'logrotate = slapos.recipe.logrotate:Recipe',
'logrotate.d = slapos.recipe.logrotate:Part', 'logrotate.d = slapos.recipe.logrotate:Part',
'memcached = slapos.recipe.memcached:Recipe',
'mkdirectory = slapos.recipe.mkdirectory:Recipe', 'mkdirectory = slapos.recipe.mkdirectory:Recipe',
'mioga.instantiate = slapos.recipe.mioga.instantiate:Recipe', 'mioga.instantiate = slapos.recipe.mioga.instantiate:Recipe',
'mydumper = slapos.recipe.mydumper:Recipe', 'mydumper = slapos.recipe.mydumper:Recipe',
...@@ -196,7 +191,6 @@ setup(name=name, ...@@ -196,7 +191,6 @@ setup(name=name,
'urlparse = slapos.recipe._urlparse:Recipe', 'urlparse = slapos.recipe._urlparse:Recipe',
'uuid = slapos.recipe._uuid:Recipe', 'uuid = slapos.recipe._uuid:Recipe',
'userinfo = slapos.recipe.userinfo:Recipe', 'userinfo = slapos.recipe.userinfo:Recipe',
'waitfor = slapos.recipe.waitfor:Recipe',
'webchecker = slapos.recipe.web_checker:Recipe', 'webchecker = slapos.recipe.web_checker:Recipe',
'wrapper = slapos.recipe.wrapper:Recipe', 'wrapper = slapos.recipe.wrapper:Recipe',
'xvfb = slapos.recipe.xvfb:Recipe', 'xvfb = slapos.recipe.xvfb:Recipe',
......
...@@ -91,8 +91,8 @@ class Recipe(GenericSlapRecipe): ...@@ -91,8 +91,8 @@ class Recipe(GenericSlapRecipe):
# Generate wrapper # Generate wrapper
wrapper_location = self.createPythonScript(self.options['accords-wrapper'], wrapper_location = self.createPythonScript(self.options['accords-wrapper'],
'%s.accords.runAccords' % __name__, __name__ + '.accords.runAccords',
parameter_dict) (parameter_dict,))
path_list.append(wrapper_location) path_list.append(wrapper_location)
# Generate helper for debug # Generate helper for debug
......
...@@ -38,26 +38,18 @@ class Recipe(GenericSlapRecipe): ...@@ -38,26 +38,18 @@ class Recipe(GenericSlapRecipe):
""" """
def _install(self): def _install(self):
path_list = []
slap_connection = self.buildout['slap-connection'] slap_connection = self.buildout['slap-connection']
takeover_wrapper = self.createPythonScript( return self.createPythonScript(
name=self.options['wrapper-takeover'], self.options['wrapper-takeover'],
absolute_function='slapos.recipe.addresiliency.takeover.run', __name__ + '.takeover.takeover',
arguments={ kw={
'server_url': slap_connection['server-url'], 'server_url': slap_connection['server-url'],
'key_file': slap_connection.get('key-file'), 'key_file': slap_connection.get('key-file'),
'cert_file': slap_connection.get('cert-file'), 'cert_file': slap_connection.get('cert-file'),
'computer_id': slap_connection['computer-id'], 'computer_guid': slap_connection['computer-id'],
'partition_id': slap_connection['partition-id'], 'partition_id': slap_connection['partition-id'],
'software': slap_connection['software-release-url'], 'software_release': slap_connection['software-release-url'],
'namebase': self.parameter_dict['namebase'], 'namebase': self.parameter_dict['namebase'],
'takeover_triggered_file_path': self.options['takeover-triggered-file-path'], 'takeover_triggered_file_path': self.options['takeover-triggered-file-path'],
}) })
path_list.append(takeover_wrapper)
return path_list
...@@ -78,14 +78,3 @@ def takeover(server_url, key_file, cert_file, computer_guid, ...@@ -78,14 +78,3 @@ def takeover(server_url, key_file, cert_file, computer_guid,
# Create "lock" file preventing equeue to run import scripts # Create "lock" file preventing equeue to run import scripts
# XXX hardcoded # XXX hardcoded
open(takeover_triggered_file_path, 'w').write('') open(takeover_triggered_file_path, 'w').write('')
def run(args):
slapos.recipe.addresiliency.takeover.takeover(server_url = args.pop('server_url'),
key_file = args.pop('key_file'),
cert_file = args.pop('cert_file'),
computer_guid = args.pop('computer_id'),
partition_id = args.pop('partition_id'),
software_release = args.pop('software'),
namebase = args.pop('namebase'),
takeover_triggered_file_path = args.pop('takeover_triggered_file_path'))
##############################################################################
#
# Copyright (c) 2011 Vifib SARL and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
from slapos.recipe.librecipe import GenericBaseRecipe
import pkg_resources
class Recipe(GenericBaseRecipe):
def install(self):
try:
backend_list = self.options['backend-list']
except KeyError:
backend_list = [(self.options['port'], self.options['backend'])]
scheme = self.options['scheme']
if scheme == 'http':
required_path_list = []
ssl_enable = ssl_snippet = ''
elif scheme == 'https':
key = self.options['key-file']
certificate = self.options['cert-file']
required_path_list = [key, certificate]
ssl_snippet = self.substituteTemplate(self.getTemplateFilename('snippet.ssl.in'), {
'key': key,
'certificate': certificate,
'ssl_session_cache': self.options['ssl-session-cache'],
})
if 'ssl-authentication' in self.options and self.optionIsTrue(
'ssl-authentication'):
ssl_snippet += self.substituteTemplate(self.getTemplateFilename('snippet.ssl.ca.in'), {
'ca_certificate': self.options['ssl-authentication-certificate'],
'ca_crl': self.options['ssl-authentication-crl'],
})
ssl_enable = 'SSLEngine on'
else:
raise ValueError('Unsupported scheme %s' % scheme)
ip_list = self.options['ip']
if isinstance(ip_list, basestring):
ip_list = [ip_list]
backend_path = self.options.get('backend-path', '/')
vhost_template_name = self.getTemplateFilename('vhost.in')
apache_config_file = self.createFile(
self.options['configuration-file'],
self.substituteTemplate(
self.getTemplateFilename('apache.zope.conf.in'),
{
'path': '/',
'server_admin': 'admin@',
'pid_file': self.options['pid-file'],
'lock_file': self.options['lock-file'],
'error_log': self.options['error-log'],
'access_log': self.options['access-log'],
'access_control_string': self.options['access-control-string'],
'ssl_snippet': ssl_snippet,
'vhosts': ''.join(self.substituteTemplate(vhost_template_name, {
'ip': ip,
'port': port,
'backend': ('%s/%s' % (backend.rstrip('/'), backend_path.strip('/'))).rstrip('/'),
'ssl_enable': ssl_enable,
}) for (port, backend) in backend_list for ip in ip_list),
},
)
)
return [
apache_config_file,
self.createPythonScript(
self.options['wrapper'],
__name__ + '.apache.runApache',
[
{
'required_path_list': required_path_list,
'binary': self.options['apache-binary'],
'config': apache_config_file,
},
],
),
]
import os
import sys
import time
def runApache(args):
sleep = 60
conf = args[0]
while True:
ready = True
for f in conf.get('required_path_list', []):
if not os.path.exists(f):
print 'File %r does not exists, sleeping for %s' % (f, sleep)
ready = False
if ready:
break
time.sleep(sleep)
apache_wrapper_list = [conf['binary'], '-f', conf['config'], '-DFOREGROUND']
apache_wrapper_list.extend(sys.argv[1:])
sys.stdout.flush()
sys.stderr.flush()
os.execl(apache_wrapper_list[0], *apache_wrapper_list)
# Apache configuration file for Zope
# Automatically generated
# List of modules
LoadModule unixd_module modules/mod_unixd.so
LoadModule access_compat_module modules/mod_access_compat.so
LoadModule authz_core_module modules/mod_authz_core.so
LoadModule authz_host_module modules/mod_authz_host.so
LoadModule log_config_module modules/mod_log_config.so
LoadModule setenvif_module modules/mod_setenvif.so
LoadModule version_module modules/mod_version.so
LoadModule proxy_module modules/mod_proxy.so
LoadModule proxy_http_module modules/mod_proxy_http.so
LoadModule socache_shmcb_module modules/mod_socache_shmcb.so
LoadModule ssl_module modules/mod_ssl.so
LoadModule mime_module modules/mod_mime.so
LoadModule dav_module modules/mod_dav.so
LoadModule dav_fs_module modules/mod_dav_fs.so
LoadModule negotiation_module modules/mod_negotiation.so
LoadModule rewrite_module modules/mod_rewrite.so
LoadModule headers_module modules/mod_headers.so
# Basic server configuration
PidFile "%(pid_file)s"
ServerAdmin %(server_admin)s
TypesConfig conf/mime.types
AddType application/x-compress .Z
AddType application/x-gzip .gz .tgz
ServerTokens Prod
ServerSignature Off
TraceEnable Off
# Apache 2.4's default value (60 seconds) can be a bit too short
TimeOut 300
# As backend is trusting REMOTE_USER header unset it always
RequestHeader unset REMOTE_USER
%(ssl_snippet)s
# Log configuration
ErrorLog "%(error_log)s"
# Default apache log format with request time in microsecond at the end
LogFormat "%%h %%l %%u %%t \"%%r\" %%>s %%b \"%%{Referer}i\" \"%%{User-Agent}i\" %%D" combined
CustomLog "%(access_log)s" combined
# Directory protection
<Directory />
Options FollowSymLinks
AllowOverride None
Order deny,allow
Deny from all
</Directory>
# Path protected
<Location %(path)s>
Order Deny,Allow
Deny from all
Allow from %(access_control_string)s
</Location>
# Magic of Zope related rewrite
RewriteEngine On
%(vhosts)s
SSLVerifyClient require
RequestHeader set REMOTE_USER %%{SSL_CLIENT_S_DN_CN}s
SSLCACertificateFile %(ca_certificate)s
SSLCARevocationPath %(ca_crl)s
SSLCertificateFile %(certificate)s
SSLCertificateKeyFile %(key)s
SSLRandomSeed startup builtin
SSLRandomSeed connect builtin
SSLProtocol all -SSLv2 -SSLv3
SSLCipherSuite ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:ECDH+3DES:DH+3DES:RSA+AESGCM:RSA+AES:RSA+3DES:HIGH:!aNULL:!MD5
SSLHonorCipherOrder on
SSLSessionCache shmcb:%(ssl_session_cache)s(512000)
SSLProxyEngine On
Listen %(ip)s:%(port)s
<VirtualHost *:%(port)s>
%(ssl_enable)s
RewriteRule ^/(.*) %(backend)s/$1 [L,P]
</VirtualHost>
...@@ -57,10 +57,9 @@ class Recipe(GenericBaseRecipe): ...@@ -57,10 +57,9 @@ class Recipe(GenericBaseRecipe):
) )
path_list.append(httpd_conf) path_list.append(httpd_conf)
wrapper = self.createPythonScript(self.options['wrapper'], wrapper = self.createWrapper(self.options['wrapper'],
'slapos.recipe.librecipe.execute.execute', (self.options['httpd-binary'], '-f', self.options['httpd-conf'],
[self.options['httpd-binary'], '-f', self.options['httpd-conf'], '-DFOREGROUND'),
'-DFOREGROUND']
) )
path_list.append(wrapper) path_list.append(wrapper)
......
...@@ -92,14 +92,13 @@ class Recipe(GenericBaseRecipe): ...@@ -92,14 +92,13 @@ class Recipe(GenericBaseRecipe):
) )
path_list.append(httpd_conf) path_list.append(httpd_conf)
wrapper = self.createWrapper(name=self.options['wrapper'], wrapper = self.createWrapper(self.options['wrapper'],
command=self.options['httpd-binary'], (self.options['httpd-binary'],
parameters=[
'-f', '-f',
self.options['httpd-conf'], self.options['httpd-conf'],
'-DFOREGROUND' '-DFOREGROUND'
], ),
environment=self.environ) self.environ)
path_list.append(wrapper) path_list.append(wrapper)
secret_key_filename = os.path.join(self.buildout['buildout']['directory'], secret_key_filename = os.path.join(self.buildout['buildout']['directory'],
......
...@@ -118,7 +118,7 @@ class Recipe(GenericBaseRecipe): ...@@ -118,7 +118,7 @@ class Recipe(GenericBaseRecipe):
configureinstall_wrapper_path = self.createPythonScript( configureinstall_wrapper_path = self.createPythonScript(
self.options['configureinstall-location'], self.options['configureinstall-location'],
__name__ + '.runner.executeRunner', __name__ + '.runner.executeRunner',
[argument, delete, rename, chmod, data] (argument, delete, rename, chmod, data)
) )
#TODO finish to port this and remove upper one #TODO finish to port this and remove upper one
......
import subprocess import subprocess
def executeRunner(args): def executeRunner(arguments, delete, rename, chmod, data):
"""Start the instance configure. this may run a python script, move or/and rename """Start the instance configure. this may run a python script, move or/and rename
file or directory when dondition is filled. the condition may be when file exist or when an entry file or directory when dondition is filled. the condition may be when file exist or when an entry
exist into database. exist into database.
""" """
arguments, delete, rename, chmod, data = args if delete:
if delete != []:
print "Calling lampconfigure with 'delete' arguments" print "Calling lampconfigure with 'delete' arguments"
result = subprocess.Popen(arguments + delete) subprocess.call(arguments + delete)
result.wait() if rename:
if rename != []:
for parameters in rename: for parameters in rename:
print "Calling lampconfigure with 'rename' arguments" print "Calling lampconfigure with 'rename' arguments"
result = subprocess.Popen(arguments + parameters) subprocess.call(arguments + parameters)
result.wait() if chmod:
if chmod != []:
print "Calling lampconfigure with 'chmod' arguments" print "Calling lampconfigure with 'chmod' arguments"
result = subprocess.Popen(arguments + chmod) subprocess.call(arguments + chmod)
result.wait() if data:
if data != []:
print "Calling lampconfigure with 'run' arguments" print "Calling lampconfigure with 'run' arguments"
print arguments + data print arguments + data
result = subprocess.Popen(arguments + data) subprocess.call(arguments + data)
result.wait()
return
...@@ -49,13 +49,12 @@ class Recipe(GenericBaseRecipe): ...@@ -49,13 +49,12 @@ class Recipe(GenericBaseRecipe):
) )
path_list.append(httpd_conf) path_list.append(httpd_conf)
wrapper = self.createWrapper(name=self.options['wrapper'], wrapper = self.createWrapper(self.options['wrapper'],
command=self.options['httpd-binary'], (self.options['httpd-binary'],
parameters=[
'-f', '-f',
self.options['httpd-conf'], self.options['httpd-conf'],
'-DFOREGROUND', '-DFOREGROUND',
]) ))
path_list.append(wrapper) path_list.append(wrapper)
......
...@@ -124,12 +124,15 @@ class Recipe(GenericBaseRecipe): ...@@ -124,12 +124,15 @@ class Recipe(GenericBaseRecipe):
#Generate wrapper for php #Generate wrapper for php
wrapperphp = os.path.join(self.home, 'bin/php') wrapperphp = os.path.join(self.home, 'bin/php')
php_wrapper = self.createPythonScript(wrapperphp, php_wrapper = self.createWrapper(wrapperphp,
'slapos.recipe.librecipe.execute.executee', (self.phpbin, '-c', self.phpini),
([self.phpbin, '-c', self.phpini], os.environ)
) )
path_list.append(php_wrapper) path_list.append(php_wrapper)
mysql_dict = dict(db=self.database,
host=self.mysqlhost, port=self.mysqlport,
user=self.username, passwd=self.password)
#Generate python script for MySQL database test (starting) #Generate python script for MySQL database test (starting)
file_status = os.path.join(self.home, '.boinc_config') file_status = os.path.join(self.home, '.boinc_config')
if os.path.exists(file_status): if os.path.exists(file_status):
...@@ -137,11 +140,7 @@ class Recipe(GenericBaseRecipe): ...@@ -137,11 +140,7 @@ class Recipe(GenericBaseRecipe):
mysql_wrapper = self.createPythonScript( mysql_wrapper = self.createPythonScript(
os.path.join(self.wrapperdir, 'start_config'), os.path.join(self.wrapperdir, 'start_config'),
'%s.configure.checkMysql' % __name__, '%s.configure.checkMysql' % __name__,
dict(mysql_port=self.mysqlport, mysql_host=self.mysqlhost, (environment, mysql_dict, file_status)
mysql_user=self.username, mysql_password=self.password,
database=self.database,
file_status=file_status, environment=environment
)
) )
# Generate make project wrapper file # Generate make project wrapper file
...@@ -164,8 +163,7 @@ class Recipe(GenericBaseRecipe): ...@@ -164,8 +163,7 @@ class Recipe(GenericBaseRecipe):
install_wrapper = self.createPythonScript( install_wrapper = self.createPythonScript(
os.path.join(self.wrapperdir, 'make_project'), os.path.join(self.wrapperdir, 'make_project'),
'%s.configure.makeProject' % __name__, '%s.configure.makeProject' % __name__,
dict(launch_args=launch_args, request_file=request_make_boinc, (file_status, launch_args, request_make_boinc, environment)
make_sig=file_status, env=environment)
) )
path_list.append(install_wrapper) path_list.append(install_wrapper)
...@@ -197,7 +195,7 @@ class Recipe(GenericBaseRecipe): ...@@ -197,7 +195,7 @@ class Recipe(GenericBaseRecipe):
) )
start_service = self.createPythonScript( start_service = self.createPythonScript(
os.path.join(self.wrapperdir, 'config_project'), os.path.join(self.wrapperdir, 'config_project'),
'%s.configure.services' % __name__, parameter '%s.configure.services' % __name__, (parameter,)
) )
path_list.append(start_service) path_list.append(start_service)
...@@ -208,14 +206,12 @@ class Recipe(GenericBaseRecipe): ...@@ -208,14 +206,12 @@ class Recipe(GenericBaseRecipe):
os.unlink(start_boinc) os.unlink(start_boinc)
boinc_parameter = dict(service_status=service_status, boinc_parameter = dict(service_status=service_status,
installroot=self.installroot, drop_install=drop_install, installroot=self.installroot, drop_install=drop_install,
mysql_port=self.mysqlport, mysql_host=self.mysqlhost, mysql_dict=mysql_dict, environment=environment,
mysql_user=self.username, mysql_password=self.password,
database=self.database, environment=environment,
start_boinc=start_boinc) start_boinc=start_boinc)
start_wrapper = self.createPythonScript(os.path.join(self.wrapperdir, start_wrapper = self.createPythonScript(os.path.join(self.wrapperdir,
'start_boinc'), 'start_boinc'),
'%s.configure.restart_boinc' % __name__, '%s.configure.restart_boinc' % __name__,
boinc_parameter (boinc_parameter,)
) )
path_list.append(start_wrapper) path_list.append(start_wrapper)
...@@ -362,7 +358,7 @@ class App(GenericBaseRecipe): ...@@ -362,7 +358,7 @@ class App(GenericBaseRecipe):
) )
deploy_app = self.createPythonScript( deploy_app = self.createPythonScript(
os.path.join(wrapperdir, 'boinc_%s' % appname), os.path.join(wrapperdir, 'boinc_%s' % appname),
'%s.configure.deployApp' % __name__, parameter '%s.configure.deployApp' % __name__, (parameter,)
) )
path_list.append(deploy_app) path_list.append(deploy_app)
...@@ -404,17 +400,15 @@ class Client(GenericBaseRecipe): ...@@ -404,17 +400,15 @@ class Client(GenericBaseRecipe):
cc_cmd = '--read_cc_config' cc_cmd = '--read_cc_config'
cmd = self.createPythonScript(cmd_wrapper, cmd = self.createPythonScript(cmd_wrapper,
'%s.configure.runCmd' % __name__, '%s.configure.runCmd' % __name__,
dict(base_cmd=base_cmd, cc_cmd=cc_cmd, installdir=installdir, (base_cmd, cc_cmd, installdir, url, key)
project_url=url, key=key)
) )
path_list.append(cmd) path_list.append(cmd)
#Generate BOINC client wrapper #Generate BOINC client wrapper
boinc = self.createPythonScript(boinc_wrapper, boinc = self.createWrapper(boinc_wrapper,
'slapos.recipe.librecipe.execute.execute', (boincbin, '--allow_multiple_clients', '--gui_rpc_port',
[boincbin, '--allow_multiple_clients', '--gui_rpc_port',
str(self.options['rpc-port']), '--allow_remote_gui_rpc', str(self.options['rpc-port']), '--allow_remote_gui_rpc',
'--dir', installdir, '--redirectio', '--check_all_logins'] '--dir', installdir, '--redirectio', '--check_all_logins'),
) )
path_list.append(boinc) path_list.append(boinc)
......
...@@ -35,27 +35,21 @@ import filecmp ...@@ -35,27 +35,21 @@ import filecmp
from lock_file import LockFile from lock_file import LockFile
def checkMysql(args): def checkMysql(environment, connect_kw, file_status=None):
sys.path += args['environment']['PYTHONPATH'].split(':') sys.path += environment['PYTHONPATH'].split(':')
import MySQLdb import MySQLdb
#Sleep until mysql server becomes available #Sleep until mysql server becomes available
while True: while True:
try: try:
conn = MySQLdb.connect(host = args['mysql_host'], MySQLdb.connect(**connect_kw).close()
user = args['mysql_user'],
port = int(args['mysql_port']),
passwd = args['mysql_password'],
db = args['database'])
conn.close()
print "Successfully connect to MySQL database... "
if args.has_key('file_status'):
writeFile(args['file_status'], "starting")
break break
except Exception, ex: except Exception, ex:
print "The result is: \n" + ex.message print "The result is: \n" + ex.message
print "Could not connect to MySQL database... sleep for 2 secondes" print "Could not connect to MySQL database... sleep for 2 secondes"
time.sleep(2) time.sleep(2)
print "Successfully connect to MySQL database... "
if file_status:
writeFile(file_status, "starting")
def checkFile(file, stime): def checkFile(file, stime):
"""Loop until 'file' is created (exist)""" """Loop until 'file' is created (exist)"""
...@@ -70,18 +64,16 @@ def checkFile(file, stime): ...@@ -70,18 +64,16 @@ def checkFile(file, stime):
def restart_boinc(args): def restart_boinc(args):
"""Stop (if currently is running state) and start all Boinc service""" """Stop (if currently is running state) and start all Boinc service"""
environment = args['environment']
if args['drop_install']: if args['drop_install']:
checkFile(args['service_status'], 3) checkFile(args['service_status'], 3)
else: else:
checkMysql(args) checkMysql(environment, args['mysql_dict'], args.get('file_status'))
print "Restart Boinc..." print "Restart Boinc..."
env = os.environ env = os.environ.copy()
env['PATH'] = args['environment']['PATH'] env.update(environment)
env['PYTHONPATH'] = args['environment']['PYTHONPATH'] subprocess.call((os.path.join(args['installroot'], 'bin', 'stop'),), env=env)
binstart = os.path.join(args['installroot'], 'bin/start') subprocess.call((os.path.join(args['installroot'], 'bin', 'start'),), env=env)
binstop = os.path.join(args['installroot'], 'bin/stop')
os.system(binstop)
os.system(binstart)
writeFile(args['start_boinc'], "started") writeFile(args['start_boinc'], "started")
print "Done." print "Done."
...@@ -122,17 +114,16 @@ def startProcess(launch_args, env=None, cwd=None, stdout=subprocess.PIPE): ...@@ -122,17 +114,16 @@ def startProcess(launch_args, env=None, cwd=None, stdout=subprocess.PIPE):
return False return False
return True return True
def makeProject(args): def makeProject(make_sig, launch_args, request_file, extra_environ):
"""Run BOINC make_project script but once only""" """Run BOINC make_project script but once only"""
#Wait for DateBase initialization... #Wait for DateBase initialization...
checkFile(args['make_sig'], 3) checkFile(make_sig, 3)
print "Cheking if needed to run BOINC make_project..." print "Cheking if needed to run BOINC make_project..."
if os.path.exists(args['request_file']): if os.path.exists(request_file):
env = os.environ env = os.environ.copy()
env['PATH'] = args['env']['PATH'] env.update(extra_environ)
env['PYTHONPATH'] = args['env']['PYTHONPATH'] if startProcess(launch_args, env=env):
if startProcess(args['launch_args'], env=env): os.unlink(request_file)
os.unlink(args['request_file'])
print "Finished running BOINC make_projet...Ending" print "Finished running BOINC make_projet...Ending"
else: else:
print "No new request for make_project. Exiting..." print "No new request for make_project. Exiting..."
...@@ -155,9 +146,8 @@ def services(args): ...@@ -155,9 +146,8 @@ def services(args):
return return
print "execute script xadd..." print "execute script xadd..."
env = os.environ env = os.environ.copy()
env['PATH'] = args['environment']['PATH'] env.update(args['environment'])
env['PYTHONPATH'] = args['environment']['PYTHONPATH']
if not startProcess([os.path.join(args['installroot'], 'bin/xadd')], env): if not startProcess([os.path.join(args['installroot'], 'bin/xadd')], env):
return return
print "Update files and directories permissions..." print "Update files and directories permissions..."
...@@ -212,9 +202,8 @@ def deployManagement(args): ...@@ -212,9 +202,8 @@ def deployManagement(args):
newInstall = True newInstall = True
#Sleep until file .start_boinc exist (File indicate that BOINC has been started) #Sleep until file .start_boinc exist (File indicate that BOINC has been started)
checkFile(args['start_boinc'], 3) checkFile(args['start_boinc'], 3)
env = os.environ env = os.environ.copy()
env['PATH'] = args['environment']['PATH'] env.update(args['environment'])
env['PYTHONPATH'] = args['environment']['PYTHONPATH']
print "setup directories..." print "setup directories..."
numversion = args['version'].replace('.', '') numversion = args['version'].replace('.', '')
...@@ -263,7 +252,7 @@ def deployManagement(args): ...@@ -263,7 +252,7 @@ def deployManagement(args):
privateKeyFile = os.path.join(args['installroot'], 'keys/code_sign_private') privateKeyFile = os.path.join(args['installroot'], 'keys/code_sign_private')
output = open(binary + '.sig', 'w') output = open(binary + '.sig', 'w')
p_sign = subprocess.Popen([sign, binary, privateKeyFile], stdout=output, p_sign = subprocess.Popen([sign, binary, privateKeyFile], stdout=output,
stderr=subprocess.STDOUT) stderr=subprocess.STDOUT, env=env)
result = p_sign.communicate()[0] result = p_sign.communicate()[0]
if p_sign.returncode is None or p_sign.returncode != 0: if p_sign.returncode is None or p_sign.returncode != 0:
print "Failed to execute bin/sign_executable.\nThe error was: %s" % result print "Failed to execute bin/sign_executable.\nThe error was: %s" % result
...@@ -290,10 +279,8 @@ def deployManagement(args): ...@@ -290,10 +279,8 @@ def deployManagement(args):
create_wu(args, env) create_wu(args, env)
print "Restart Boinc..." print "Restart Boinc..."
binstart = os.path.join(args['installroot'], 'bin/start') subprocess.call((os.path.join(args['installroot'], 'bin', 'stop'),), env=env)
binstop = os.path.join(args['installroot'], 'bin/stop') subprocess.call((os.path.join(args['installroot'], 'bin', 'start'),), env=env)
os.system(binstop)
os.system(binstart)
print "Boinc Application deployment is done... writing end signal file..." print "Boinc Application deployment is done... writing end signal file..."
writeFile(token, str(args['wu_number'])) writeFile(token, str(args['wu_number']))
...@@ -315,22 +302,21 @@ def create_wu(args, env): ...@@ -315,22 +302,21 @@ def create_wu(args, env):
startProcess(launch_args, env, args['installroot']) startProcess(launch_args, env, args['installroot'])
def runCmd(args): def runCmd(base_cmd, cc_cmd, installdir, url, key):
"""Wait for Boinc Client started and run boinc cmd""" """Wait for Boinc Client started and run boinc cmd"""
client_config = os.path.join(args['installdir'], 'client_state.xml') client_config = os.path.join(installdir, 'client_state.xml')
checkFile(client_config, 5) checkFile(client_config, 5)
time.sleep(10) time.sleep(10)
#Scan client state xml to find client ipv4 adress #Scan client state xml to find client ipv4 adress
host = re.search("<ip_addr>([\w\d\.:]+)</ip_addr>", host = re.search("<ip_addr>([\w\d\.:]+)</ip_addr>",
open(client_config, 'r').read()).group(1) open(client_config, 'r').read()).group(1)
args['base_cmd'][2] = host + ':' + args['base_cmd'][2] base_cmd[2] = host + ':' + base_cmd[2]
print "Run boinccmd with host at %s " % args['base_cmd'][2] print "Run boinccmd with host at %s " % base_cmd[2]
project_args = args['base_cmd'] + ['--project_attach', args['project_url'], project_args = base_cmd + ['--project_attach', url, key]
args['key']] startProcess(project_args, cwd=installdir)
startProcess(project_args, cwd=args['installdir']) if cc_cmd:
if args['cc_cmd'] != '':
#Load or reload cc_config file #Load or reload cc_config file
startProcess(args['base_cmd'] + [args['cc_cmd']], cwd=args['installdir']) startProcess(base_cmd + [cc_cmd], cwd=installdir)
def writeFile(file, content): def writeFile(file, content):
......
...@@ -62,8 +62,8 @@ class Recipe(GenericBaseRecipe): ...@@ -62,8 +62,8 @@ class Recipe(GenericBaseRecipe):
condor_wrapper_list=condor_wrapper_list, condor_wrapper_list=condor_wrapper_list,
boinc_wrapper_list=boinc_wrapper_list) boinc_wrapper_list=boinc_wrapper_list)
bonjourGrid_wrapper = self.createPythonScript(grid_wrapper, bonjourGrid_wrapper = self.createPythonScript(grid_wrapper,
'%s.configure.launchScript' % __name__, __name__ + '.configure.launchScript',
parameters (parameters,)
) )
path_list.append(bonjourGrid_wrapper) path_list.append(bonjourGrid_wrapper)
...@@ -73,16 +73,15 @@ class Recipe(GenericBaseRecipe): ...@@ -73,16 +73,15 @@ class Recipe(GenericBaseRecipe):
bg_wrapper = self.options['wrapper'].strip() bg_wrapper = self.options['wrapper'].strip()
log = self.options['log_file'].strip() log = self.options['log_file'].strip()
pid_file = self.options['pid_file'].strip() pid_file = self.options['pid_file'].strip()
wrapper = self.createPythonScript(bg_wrapper, wrapper = self.createWrapper(bg_wrapper,
'slapos.recipe.librecipe.execute.execute', (python, bonjourgrid_master, '--log_file', log,
([python, bonjourgrid_master, '--log_file', log,
'--pid_file', pid_file, '--pid_file', pid_file,
'--master_wrapper', grid_wrapper, '--master_wrapper', grid_wrapper,
'--directory', self.options['work_dir'].strip(), '--directory', self.options['work_dir'].strip(),
'--server', self.options['redis-url'].strip(), '--server', self.options['redis-url'].strip(),
'--port', self.options['redis-port'].strip(), '--port', self.options['redis-port'].strip(),
'--num_workers', self.options['nworkers'].strip(), '--num_workers', self.options['nworkers'].strip(),
]) ),
) )
path_list.append(wrapper) path_list.append(wrapper)
...@@ -113,9 +112,8 @@ class Client(GenericBaseRecipe): ...@@ -113,9 +112,8 @@ class Client(GenericBaseRecipe):
bg_wrapper = self.options['wrapper'].strip() bg_wrapper = self.options['wrapper'].strip()
log = self.options['log_file'].strip() log = self.options['log_file'].strip()
pid_file = self.options['pid_file'].strip() pid_file = self.options['pid_file'].strip()
wrapper = self.createPythonScript(bg_wrapper, wrapper = self.createWrapper(bg_wrapper,
'slapos.recipe.librecipe.execute.execute', (python, bonjourgrid_client, '--log_file', log,
([python, bonjourgrid_client, '--log_file', log,
'--pid_file', pid_file, '--pid_file', pid_file,
'--boinc_wrapper', boinc_script, '--boinc_wrapper', boinc_script,
'--condor_wrapper', condor_script, '--condor_wrapper', condor_script,
...@@ -123,7 +121,7 @@ class Client(GenericBaseRecipe): ...@@ -123,7 +121,7 @@ class Client(GenericBaseRecipe):
'--install_directory', self.options['install_dir'].strip(), '--install_directory', self.options['install_dir'].strip(),
'--server', self.options['redis-url'].strip(), '--server', self.options['redis-url'].strip(),
'--port', self.options['redis-port'].strip(), '--port', self.options['redis-port'].strip(),
]) ),
) )
path_list.append(wrapper) path_list.append(wrapper)
......
...@@ -40,13 +40,10 @@ class Recipe(GenericBaseRecipe): ...@@ -40,13 +40,10 @@ class Recipe(GenericBaseRecipe):
self.ca_private = self.options['ca-private'] self.ca_private = self.options['ca-private']
self.ca_certs = self.options['ca-certs'] self.ca_certs = self.options['ca-certs']
self.ca_newcerts = self.options['ca-newcerts'] self.ca_newcerts = self.options['ca-newcerts']
self.ca_crl = self.options['ca-crl']
self.ca_key_ext = '.key' self.ca_key_ext = '.key'
self.ca_crt_ext = '.crt' self.ca_crt_ext = '.crt'
def install(self): def install(self):
path_list = []
ca_country_code = self.options.get('country-code', 'XX') ca_country_code = self.options.get('country-code', 'XX')
ca_email = self.options.get('email', 'xx@example.com') ca_email = self.options.get('email', 'xx@example.com')
# XXX-BBB: State by mistake has been configured as string "('State',)" # XXX-BBB: State by mistake has been configured as string "('State',)"
...@@ -77,21 +74,15 @@ class Recipe(GenericBaseRecipe): ...@@ -77,21 +74,15 @@ class Recipe(GenericBaseRecipe):
self.createFile(openssl_configuration, self.substituteTemplate( self.createFile(openssl_configuration, self.substituteTemplate(
self.getTemplateFilename('openssl.cnf.ca.in'), config)) self.getTemplateFilename('openssl.cnf.ca.in'), config))
ca_wrapper = self.createPythonScript( return self.createPythonScript(
self.options['wrapper'], self.options['wrapper'],
'%s.certificate_authority.runCertificateAuthority' % __name__, __name__ + '.certificate_authority.runCertificateAuthority',
dict( (os.path.join(self.ca_private, 'cakey.pem'),
openssl_configuration=openssl_configuration, os.path.join(self.ca_dir, 'cacert.pem'),
openssl_binary=self.options['openssl-binary'], self.options['openssl-binary'],
certificate=os.path.join(self.ca_dir, 'cacert.pem'), openssl_configuration,
key=os.path.join(self.ca_private, 'cakey.pem'), self.request_directory)
crl=self.ca_crl,
request_dir=self.request_directory
)
) )
path_list.append(ca_wrapper)
return path_list
class Request(Recipe): class Request(Recipe):
...@@ -146,11 +137,10 @@ class Request(Recipe): ...@@ -146,11 +137,10 @@ class Request(Recipe):
path_list = [key_file, cert_file] path_list = [key_file, cert_file]
if request_needed: if request_needed:
wrapper = self.createPythonScript( wrapper = self.createWrapper(
self.options['wrapper'], self.options['wrapper'],
'slapos.recipe.librecipe.execute.execute_wait', (self.options['executable'],),
[ [self.options['executable']], wait_list=(certificate, key),
[certificate, key] ],
) )
path_list.append(wrapper) path_list.append(wrapper)
......
...@@ -102,10 +102,8 @@ class CertificateAuthority: ...@@ -102,10 +102,8 @@ class CertificateAuthority:
'certificate_file')): 'certificate_file')):
print 'Created certificate %r' % parser.get('certificate', 'name') print 'Created certificate %r' % parser.get('certificate', 'name')
def runCertificateAuthority(ca_conf): def runCertificateAuthority(*args):
ca = CertificateAuthority(ca_conf['key'], ca_conf['certificate'], ca = CertificateAuthority(*args)
ca_conf['openssl_binary'], ca_conf['openssl_configuration'],
ca_conf['request_dir'])
while True: while True:
ca.checkAuthority() ca.checkAuthority()
ca.checkRequestDir() ca.checkRequestDir()
......
...@@ -42,8 +42,6 @@ class Recipe(GenericBaseRecipe): ...@@ -42,8 +42,6 @@ class Recipe(GenericBaseRecipe):
options['access-url'] = 'http://[%s]:%s' % (self.ip, self.port) options['access-url'] = 'http://[%s]:%s' % (self.ip, self.port)
def install(self): def install(self):
path_list = []
environment = { environment = {
'PATH': os.path.dirname(self.git) + ':' + os.environ['PATH'], 'PATH': os.path.dirname(self.git) + ':' + os.environ['PATH'],
} }
...@@ -51,10 +49,4 @@ class Recipe(GenericBaseRecipe): ...@@ -51,10 +49,4 @@ class Recipe(GenericBaseRecipe):
cloud9_args = [self.node_executable, self.cloud9, '-l', self.ip, '-p', cloud9_args = [self.node_executable, self.cloud9, '-l', self.ip, '-p',
self.port, '-w', self.workdir] self.port, '-w', self.workdir]
wrapper = self.createPythonScript(self.wrapper, return self.createWrapper(self.wrapper, cloud9_args, environment)
'slapos.recipe.librecipe.execute.executee',
(cloud9_args, environment)
)
path_list.append(wrapper)
return path_list
...@@ -178,12 +178,11 @@ class Recipe(GenericBaseRecipe): ...@@ -178,12 +178,11 @@ class Recipe(GenericBaseRecipe):
os.chmod(wrapper_location, 0744) os.chmod(wrapper_location, 0744)
#generate script for start condor #generate script for start condor
start_condor = os.path.join(self.wrapperdir, 'start_condor') wrapper = self.createPythonScript(
start_bin = os.path.join(self.wrapper_sbin, 'condor_master') os.path.join(self.wrapperdir, 'start_condor'),
condor_reconfig = os.path.join(self.wrapper_sbin, 'condor_reconfig') __name__ + '.configure.condorStart',
wrapper = self.createPythonScript(start_condor, (os.path.join(self.wrapper_sbin, 'condor_reconfig'),
'%s.configure.condorStart' % __name__, os.path.join(self.wrapper_sbin, 'condor_master'))
dict(start_bin=start_bin, condor_reconfig=condor_reconfig)
) )
path_list.append(wrapper) path_list.append(wrapper)
return path_list return path_list
...@@ -276,13 +275,11 @@ class AppSubmit(GenericBaseRecipe): ...@@ -276,13 +275,11 @@ class AppSubmit(GenericBaseRecipe):
os.unlink(destination) os.unlink(destination)
os.symlink(app_list[appname]['files'][file], destination) os.symlink(app_list[appname]['files'][file], destination)
#generate wrapper for submitting job #generate wrapper for submitting job
condor_submit = os.path.join(self.options['bin'].strip(), 'condor_submit')
parameter = dict(submit=condor_submit, sig_install=sig_install,
submit_file='submit',
appname=appname, appdir=appdir)
submit_job = self.createPythonScript( submit_job = self.createPythonScript(
os.path.join(self.options['wrapper-dir'].strip(), appname), os.path.join(self.options['wrapper-dir'].strip(), appname),
'%s.configure.submitJob' % __name__, parameter __name__ + '.configure.submitJob',
(os.path.join(self.options['bin'].strip(), 'condor_submit'),
'submit', appdir, appname, sig_install)
) )
path_list.append(submit_job) path_list.append(submit_job)
return path_list return path_list
\ No newline at end of file
...@@ -29,27 +29,25 @@ import os ...@@ -29,27 +29,25 @@ import os
import subprocess import subprocess
import time import time
def submitJob(args): def submitJob(submit, submit_file, appdir, appname, sig_install):
"""Run condor_submit (if needed) for job deployment""" """Run condor_submit (if needed) for job deployment"""
time.sleep(10) time.sleep(10)
print "Check if needed to submit %s job's" % args['appname'] print "Check if needed to submit %s job's" % appname
if not os.path.exists(args['sig_install']): if not os.path.exists(sig_install):
print "Nothing for install or update...Exited" print "Nothing for install or update...Exited"
return return
# '-a', "log = out.log", '-a', "error = error.log", # '-a', "log = out.log", '-a', "error = error.log",
launch_args = [args['submit'], '-verbose', args['submit_file']] launch_args = submit, '-verbose', submit_file
process = subprocess.Popen(launch_args, stdout=subprocess.PIPE, process = subprocess.Popen(launch_args, stdout=subprocess.PIPE,
stderr=subprocess.STDOUT, cwd=args['appdir']) stderr=subprocess.STDOUT, cwd=appdir)
result = process.communicate()[0] result = process.communicate()[0]
if process.returncode is None or process.returncode != 0: if process.returncode is None or process.returncode != 0:
print "Failed to execute condor_submit.\nThe error was: %s" % result print "Failed to execute condor_submit.\nThe error was: %s" % result
else: else:
os.unlink(args['sig_install']) os.unlink(sig_install)
def condorStart(args): def condorStart(condor_reconfig, start_bin):
"""Start Condor if deamons is currently stopped""" """Start Condor if deamons is currently stopped"""
result = os.system(args['condor_reconfig']) if subprocess.call(condor_reconfig):
if result != 0:
#process failled to reconfig condor that mean that condor deamons is not curently started #process failled to reconfig condor that mean that condor deamons is not curently started
os.system(args['start_bin']) subprocess.call(start_bin)
\ No newline at end of file
...@@ -98,26 +98,19 @@ class Recipe(GenericBaseRecipe): ...@@ -98,26 +98,19 @@ class Recipe(GenericBaseRecipe):
) )
path_list.append(config_file) path_list.append(config_file)
wrapper = self.createPythonScript(self.options['wrapper'], wrapper = self.createWrapper(self.options['wrapper'],
'slapos.recipe.librecipe.execute.execute', (self.options['apache-binary'], '-f', config_file, '-DFOREGROUND'))
[self.options['apache-binary'], '-f', config_file, '-DFOREGROUND'])
path_list.append(wrapper) path_list.append(wrapper)
promise = self.createPythonScript(self.options['promise'], promise = self.createPythonScript(self.options['promise'],
__name__ + '.promise', __name__ + '.promise',
dict(host=self.options['ip'], port=int(self.options['port_webdav']), (self.options['ip'], int(self.options['port_webdav']),
user=self.options['user'], password=self.options['password']) self.options['user'], self.options['password']))
)
path_list.append(promise) path_list.append(promise)
return path_list return path_list
def promise(args): def promise(host, port, user, password):
host = args['host']
port = args['port']
user = args['user']
password = args['password']
connection = httplib.HTTPSConnection(host, port) connection = httplib.HTTPSConnection(host, port)
auth = base64.b64encode('%s:%s' % (user, password)) auth = base64.b64encode('%s:%s' % (user, password))
connection.request('OPTIONS', '/', connection.request('OPTIONS', '/',
......
...@@ -35,15 +35,14 @@ class Recipe(GenericBaseRecipe): ...@@ -35,15 +35,14 @@ class Recipe(GenericBaseRecipe):
self.logger.info("Installing dcron...") self.logger.info("Installing dcron...")
options = self.options options = self.options
script = self.createWrapper(name=options['binary'], script = self.createWrapper(options['binary'],
command=options['dcrond-binary'].strip(), (options['dcrond-binary'].strip(),
parameters=[
'-s', options['cron-entries'], '-s', options['cron-entries'],
'-c', options['crontabs'], '-c', options['crontabs'],
'-t', options['cronstamps'], '-t', options['cronstamps'],
'-f', '-l', '5', '-f', '-l', '5',
'-M', options['catcher'] '-M', options['catcher']
]) ))
self.logger.debug('Main cron executable created at : %r', script) self.logger.debug('Main cron executable created at : %r', script)
......
...@@ -57,8 +57,6 @@ class KnownHostsFile(dict): ...@@ -57,8 +57,6 @@ class KnownHostsFile(dict):
class Recipe(GenericBaseRecipe): class Recipe(GenericBaseRecipe):
def install(self): def install(self):
path_list = []
dropbear_cmd = [self.options['dropbear-binary']] dropbear_cmd = [self.options['dropbear-binary']]
# Don't fork into background # Don't fork into background
dropbear_cmd.append('-F') dropbear_cmd.append('-F')
...@@ -95,19 +93,12 @@ class Recipe(GenericBaseRecipe): ...@@ -95,19 +93,12 @@ class Recipe(GenericBaseRecipe):
if 'shell' in self.options: if 'shell' in self.options:
env['DROPBEAR_OVERRIDE_SHELL'] = self.options['shell'] env['DROPBEAR_OVERRIDE_SHELL'] = self.options['shell']
wrapper = self.createPythonScript( return self.createWrapper(self.options['wrapper'], dropbear_cmd, env)
self.options['wrapper'],
'slapos.recipe.librecipe.execute.executee',
(dropbear_cmd, env, )
)
path_list.append(wrapper)
return path_list
class Client(GenericBaseRecipe): class Client(GenericBaseRecipe):
def install(self): def install(self):
env = dict() env = {}
if 'home' in self.options: if 'home' in self.options:
env['HOME'] = self.options['home'] env['HOME'] = self.options['home']
...@@ -120,13 +111,7 @@ class Client(GenericBaseRecipe): ...@@ -120,13 +111,7 @@ class Client(GenericBaseRecipe):
if 'identity-file' in self.options: if 'identity-file' in self.options:
dropbear_cmd.extend(['-i', self.options['identity-file']]) dropbear_cmd.extend(['-i', self.options['identity-file']])
wrapper = self.createPythonScript( return self.createWrapper(self.options['wrapper'], dropbear_cmd, env)
self.options['wrapper'],
'slapos.recipe.librecipe.execute.executee',
(dropbear_cmd, env, )
)
return [wrapper]
class AddAuthorizedKey(GenericBaseRecipe): class AddAuthorizedKey(GenericBaseRecipe):
......
...@@ -46,7 +46,4 @@ class Recipe(GenericBaseRecipe): ...@@ -46,7 +46,4 @@ class Recipe(GenericBaseRecipe):
cmd.extend(options) cmd.extend(options)
cmd.extend([backup_directory, remote_url]) cmd.extend([backup_directory, remote_url])
wrapper = self.createPythonScript(self.options['wrapper'], return self.createWrapper(self.options['wrapper'], cmd)
'slapos.recipe.librecipe.execute.execute', cmd)
return [wrapper]
...@@ -30,23 +30,20 @@ class Recipe(GenericBaseRecipe): ...@@ -30,23 +30,20 @@ class Recipe(GenericBaseRecipe):
def install(self): def install(self):
parameters = [ args = [
self.options['equeue-binary'],
'--database', self.options['database'], '--database', self.options['database'],
'--logfile', self.options['log'], '--logfile', self.options['log'],
'--lockfile', self.options['lockfile'] '--lockfile', self.options['lockfile']
] ]
if 'takeover-triggered-file-path' in self.options: if 'takeover-triggered-file-path' in self.options:
parameters.extend(['--takeover-triggered-file-path', self.options['takeover-triggered-file-path']]) args += ('--takeover-triggered-file-path',
self.options['takeover-triggered-file-path'])
if 'loglevel' in self.options: if 'loglevel' in self.options:
parameters.extend(['--loglevel', self.options['loglevel']]) args += '--loglevel', self.options['loglevel']
parameters.append(self.options['socket']) args.append(self.options['socket'])
wrapper = self.createWrapper(name=self.options['wrapper'],
command=self.options['equeue-binary'],
parameters=parameters)
return [wrapper]
return self.createWrapper(self.options['wrapper'], args)
...@@ -67,7 +67,7 @@ class Recipe(GenericBaseRecipe): ...@@ -67,7 +67,7 @@ class Recipe(GenericBaseRecipe):
openssl_binary=self.options['openssl-binary'], openssl_binary=self.options['openssl-binary'],
test_ca_path=self.options['certificate-authority-path'], test_ca_path=self.options['certificate-authority-path'],
) )
common_list = [ common_list = (
'--conversion_server_url=' + cloudooo_url, '--conversion_server_url=' + cloudooo_url,
# BBB: We still have test suites that only accept the following 2 options. # BBB: We still have test suites that only accept the following 2 options.
'--conversion_server_hostname=%s' % cloudooo_parsed.hostname, '--conversion_server_hostname=%s' % cloudooo_parsed.hostname,
...@@ -76,19 +76,19 @@ class Recipe(GenericBaseRecipe): ...@@ -76,19 +76,19 @@ class Recipe(GenericBaseRecipe):
'--volatile_memcached_server_port=%s' % memcached_parsed.port, '--volatile_memcached_server_port=%s' % memcached_parsed.port,
'--persistent_memcached_server_hostname=%s' % kumofs_parsed.hostname, '--persistent_memcached_server_hostname=%s' % kumofs_parsed.hostname,
'--persistent_memcached_server_port=%s' % kumofs_parsed.port, '--persistent_memcached_server_port=%s' % kumofs_parsed.port,
] )
path_list.append(self.createPythonScript(self.options['run-unit-test'], path_list.append(self.createPythonScript(self.options['run-unit-test'],
__name__ + '.test.runUnitTest', [dict( __name__ + '.test.runUnitTest',
call_list=[self.options['run-unit-test-binary'], ((self.options['run-unit-test-binary'],
'--erp5_sql_connection_string', mysql_connection_string, '--erp5_sql_connection_string', mysql_connection_string,
'--extra_sql_connection_string_list', ','.join( '--extra_sql_connection_string_list', ','.join(
mysql_connection_string_list), mysql_connection_string_list),
] + common_list, **common_dict)])) ) + common_list, common_dict)))
path_list.append(self.createPythonScript(self.options['run-test-suite'], path_list.append(self.createPythonScript(self.options['run-test-suite'],
__name__ + '.test.runUnitTest', [dict( __name__ + '.test.runTestSuite',
call_list=[self.options['run-test-suite-binary'], ((self.options['run-test-suite-binary'],
'--db_list', ','.join(mysql_connection_string_list), '--db_list', ','.join(mysql_connection_string_list),
] + common_list, **common_dict)])) ) + common_list, common_dict)))
return path_list return path_list
...@@ -98,20 +98,18 @@ class CloudoooRecipe(GenericBaseRecipe): ...@@ -98,20 +98,18 @@ class CloudoooRecipe(GenericBaseRecipe):
common_dict = dict( common_dict = dict(
prepend_path=self.options['prepend-path'], prepend_path=self.options['prepend-path'],
) )
common_list = [ common_list = (
"--paster_path", self.options['ooo-paster'], "--paster_path", self.options['ooo-paster'],
self.options['configuration-file'] self.options['configuration-file']
] )
run_unit_test_path = self.createPythonScript(self.options['run-unit-test'], path_list.append(self.createPythonScript(self.options['run-unit-test'],
__name__ + '.test.runUnitTest', [dict( __name__ + '.test.runUnitTest',
call_list=[self.options['run-unit-test-binary'], ((self.options['run-unit-test-binary'],
] + common_list, **common_dict)]) ) + common_list, common_dict)))
path_list.append(run_unit_test_path)
path_list.append(self.createPythonScript(self.options['run-test-suite'], path_list.append(self.createPythonScript(self.options['run-test-suite'],
__name__ + '.test.runTestSuite', [dict( __name__ + '.test.runTestSuite',
call_list=[self.options['run-test-suite-binary'], ((self.options['run-test-suite-binary'],
], **common_dict)])) ), common_dict)))
return path_list return path_list
...@@ -121,32 +119,20 @@ class EggTestRecipe(GenericBaseRecipe): ...@@ -121,32 +119,20 @@ class EggTestRecipe(GenericBaseRecipe):
off a list of Python eggs. off a list of Python eggs.
""" """
def install(self): def install(self):
path_list = []
test_list = self.options['test-list'].strip().replace('\n', ',') test_list = self.options['test-list'].strip().replace('\n', ',')
common_dict = {}
environment_dict = {} common_dict = {}
if self.options.get('environment'): if self.options.get('environment'):
environment_part = self.buildout.get(self.options['environment']) environment_part = self.buildout.get(self.options['environment'])
if environment_part: if environment_part:
for key, value in environment_part.iteritems(): common_dict['environment'] = dict(environment_part)
environment_dict[key] = value
common_list = [ "--source_code_path_list", test_list]
argument_dict = dict(
call_list=[self.options['run-test-suite-binary'],] + common_list,
environment=environment_dict,
**common_dict
)
if 'prepend-path' in self.options: if 'prepend-path' in self.options:
argument_dict['prepend_path'] = self.options['prepend-path'] common_dict['prepend_path'] = self.options['prepend-path']
run_test_suite_script = self.createPythonScript( return self.createPythonScript(
self.options['run-test-suite'], __name__ + '.test.runTestSuite', self.options['run-test-suite'], __name__ + '.test.runTestSuite',
[argument_dict] ((self.options['run-test-suite-binary'],
"--source_code_path_list", test_list),
common_dict)
) )
path_list.append(run_test_suite_script)
return path_list
...@@ -26,15 +26,18 @@ ...@@ -26,15 +26,18 @@
############################################################################## ##############################################################################
import os import os
import sys import sys
def runTestSuite(args):
def runTestSuite(args, d):
env = os.environ.copy() env = os.environ.copy()
d = args[0]
if 'openssl_binary' in d: if 'openssl_binary' in d:
env['OPENSSL_BINARY'] = d['openssl_binary'] env['OPENSSL_BINARY'] = d['openssl_binary']
if 'test_ca_path' in d: if 'test_ca_path' in d:
env['TEST_CA_PATH'] = d['test_ca_path'] env['TEST_CA_PATH'] = d['test_ca_path']
if 'prepend_path' in d: if 'prepend_path' in d:
env['PATH'] = ':'.join([d['prepend_path']] + os.environ.get('PATH', '').split(':')) try:
env['PATH'] = d['prepend_path'] + ':' + env['PATH']
except KeyError:
env['PATH'] = d['prepend_path']
if 'instance_home' in d: if 'instance_home' in d:
env['INSTANCE_HOME'] = d['instance_home'] env['INSTANCE_HOME'] = d['instance_home']
env['REAL_INSTANCE_HOME'] = d['instance_home'] env['REAL_INSTANCE_HOME'] = d['instance_home']
...@@ -44,42 +47,15 @@ def runTestSuite(args): ...@@ -44,42 +47,15 @@ def runTestSuite(args):
env.update(d['environment']) env.update(d['environment'])
# Deal with Shebang size limitation # Deal with Shebang size limitation
executable_filepath = d['call_list'][0] executable_filepath = args[0]
file_object = open(executable_filepath, 'r') with open(executable_filepath, 'r') as f:
line = file_object.readline() line = f.readline()
file_object.close()
argument_list = []
if line[:2] == '#!':
executable_filepath = line[2:].strip()
argument_list.append(executable_filepath)
argument_list.extend(d['call_list'])
argument_list.extend(sys.argv[1:])
argument_list.append(env)
os.execle(executable_filepath, *argument_list)
def runUnitTest(args):
env = os.environ.copy()
d = args[0]
if 'openssl_binary' in d:
env['OPENSSL_BINARY'] = d['openssl_binary']
if 'test_ca_path' in d:
env['TEST_CA_PATH'] = d['test_ca_path']
if 'prepend_path' in d:
env['PATH'] = ':'.join([d['prepend_path']] + os.environ.get('PATH', '').split(':'))
if 'instance_home' in d:
env['INSTANCE_HOME'] = d['instance_home']
env['REAL_INSTANCE_HOME'] = d['instance_home']
# Deal with Shebang size limitation
executable_filepath = d['call_list'][0]
file_object = open(executable_filepath, 'r')
line = file_object.readline()
file_object.close()
argument_list = [] argument_list = []
if line[:2] == '#!': if line[:2] == '#!':
executable_filepath = line[2:].strip() executable_filepath = line[2:].strip()
argument_list.append(executable_filepath) argument_list.append(executable_filepath)
argument_list.extend(d['call_list']) argument_list += args
argument_list.extend(sys.argv[1:]) argument_list += sys.argv[1:]
argument_list.append(env) os.execve(executable_filepath, argument_list, env)
os.execle(executable_filepath, *argument_list)
runUnitTest = runTestSuite
...@@ -68,17 +68,14 @@ class Recipe(GenericBaseRecipe): ...@@ -68,17 +68,14 @@ class Recipe(GenericBaseRecipe):
) )
self.path_list.append(configuration_file) self.path_list.append(configuration_file)
self.path_list.append( self.path_list.append(
self.createPythonScript( self.createWrapper(self.options['wrapper'],
self.options['wrapper'], # Command
'slapos.recipe.librecipe.execute.executee', ( self.options['testnode'], '-l', self.options['log-file'],
[ # Executable configuration_file),
[ self.options['testnode'], '-l', self.options['log-file'],
configuration_file],
# Environment # Environment
{ {
'GIT_SSL_NO_VERIFY': '1', 'GIT_SSL_NO_VERIFY': '1',
} }
],
) )
) )
self.installApache() self.installApache()
...@@ -106,9 +103,8 @@ class Recipe(GenericBaseRecipe): ...@@ -106,9 +103,8 @@ class Recipe(GenericBaseRecipe):
apache_config) apache_config)
) )
self.path_list.append(config_file) self.path_list.append(config_file)
wrapper = self.createPythonScript(self.options['httpd-wrapper'], wrapper = self.createWrapper(self.options['httpd-wrapper'],
'slapos.recipe.librecipe.execute.execute', (self.options['apache-binary'], '-f', config_file, '-DFOREGROUND'))
[self.options['apache-binary'], '-f', config_file, '-DFOREGROUND'])
self.path_list.append(wrapper) self.path_list.append(wrapper)
# create empty html page to not allow listing of / # create empty html page to not allow listing of /
page = open(os.path.join(self.options['log-directory'], "index.html"), "w") page = open(os.path.join(self.options['log-directory'], "index.html"), "w")
......
...@@ -118,5 +118,5 @@ class Recipe(GenericBaseRecipe): ...@@ -118,5 +118,5 @@ class Recipe(GenericBaseRecipe):
path_list.append(config_file) path_list.append(config_file)
path_list.append(self.createPythonScript(self.options['wrapper'], path_list.append(self.createPythonScript(self.options['wrapper'],
'slapos.recipe.librecipe.execute.execute_with_signal_translation', 'slapos.recipe.librecipe.execute.execute_with_signal_translation',
[self.options['ooo-paster'].strip(), 'serve', config_file])) ((self.options['ooo-paster'].strip(), 'serve', config_file),)))
return path_list return path_list
...@@ -154,7 +154,7 @@ class Recipe(GenericBaseRecipe): ...@@ -154,7 +154,7 @@ class Recipe(GenericBaseRecipe):
)] )]
) )
path_list.append(mysqld) path_list.append(mysqld)
environment = dict(PATH='%s' % self.options['bin-directory']) environment = {'PATH': self.options['bin-directory']}
# TODO: move to a separate recipe (ack'ed by Cedric) # TODO: move to a separate recipe (ack'ed by Cedric)
if 'backup-script' in self.options: if 'backup-script' in self.options:
# backup configuration # backup configuration
...@@ -165,9 +165,13 @@ class Recipe(GenericBaseRecipe): ...@@ -165,9 +165,13 @@ class Recipe(GenericBaseRecipe):
'--defaults-file=%s' % mysql_conf_file, '--defaults-file=%s' % mysql_conf_file,
'--socket=%s' % socket.strip(), '--user=root', '--socket=%s' % socket.strip(), '--user=root',
'--ibbackup=%s'% self.options['xtrabackup-binary']] '--ibbackup=%s'% self.options['xtrabackup-binary']]
innobackupex_incremental = self.createPythonScript(self.options['innobackupex-incremental'], 'slapos.recipe.librecipe.execute.executee', [innobackupex_argument_list + ['--incremental'], environment]) innobackupex_incremental = self.createWrapper(
self.options['innobackupex-incremental'],
innobackupex_argument_list + ['--incremental'], environment)
path_list.append(innobackupex_incremental) path_list.append(innobackupex_incremental)
innobackupex_full = self.createPythonScript(self.options['innobackupex-full'], 'slapos.recipe.librecipe.execute.executee', [innobackupex_argument_list, environment]) innobackupex_full = self.createWrapper(
self.options['innobackupex-full'],
innobackupex_argument_list, environment)
path_list.append(innobackupex_full) path_list.append(innobackupex_full)
backup_controller = self.createPythonScript(self.options['backup-script'], __name__ + '.innobackupex.controller', [innobackupex_incremental, innobackupex_full, full_backup, incremental_backup]) backup_controller = self.createPythonScript(self.options['backup-script'], __name__ + '.innobackupex.controller', [innobackupex_incremental, innobackupex_full, full_backup, incremental_backup])
path_list.append(backup_controller) path_list.append(backup_controller)
...@@ -215,7 +219,9 @@ class Recipe(GenericBaseRecipe): ...@@ -215,7 +219,9 @@ class Recipe(GenericBaseRecipe):
'--defaults-file=%s' % mysql_conf_file, '--defaults-file=%s' % mysql_conf_file,
'--socket=%s' % socket.strip(), '--user=root', '--socket=%s' % socket.strip(), '--user=root',
] ]
pt_exe = self.createPythonScript(os.path.join(self.options['bin-directory'], pt_script_name), 'slapos.recipe.librecipe.execute.executee', [pt_argument_list, environment]) pt_exe = self.createWrapper(
os.path.join(self.options['bin-directory'], pt_script_name),
pt_argument_list, environment)
path_list.append(pt_exe) path_list.append(pt_exe)
return path_list return path_list
......
import os import os
import glob import glob
def controller(args): def controller(innobackupex_incremental, innobackupex_full,
full_backup, incremental_backup):
"""Creates full or incremental backup """Creates full or incremental backup
If no full backup is done, it is created If no full backup is done, it is created
...@@ -9,8 +10,6 @@ def controller(args): ...@@ -9,8 +10,6 @@ def controller(args):
base is the newest (according to date) full or incremental backup base is the newest (according to date) full or incremental backup
""" """
innobackupex_incremental, innobackupex_full, full_backup, incremental_backup \
= args
if len(os.listdir(full_backup)) == 0: if len(os.listdir(full_backup)) == 0:
print 'Doing full backup in %r' % full_backup print 'Doing full backup in %r' % full_backup
os.execv(innobackupex_full, [innobackupex_full, full_backup]) os.execv(innobackupex_full, [innobackupex_full, full_backup])
......
...@@ -5,9 +5,8 @@ import sys ...@@ -5,9 +5,8 @@ import sys
import pytz import pytz
def runMysql(args): def runMysql(conf):
sleep = 60 sleep = 60
conf = args[0]
mysqld_wrapper_list = [conf['mysqld_binary'], '--defaults-file=%s' % mysqld_wrapper_list = [conf['mysqld_binary'], '--defaults-file=%s' %
conf['configuration_file']] conf['configuration_file']]
# we trust mysql_install that if mysql directory is available mysql was # we trust mysql_install that if mysql directory is available mysql was
...@@ -54,8 +53,7 @@ def runMysql(args): ...@@ -54,8 +53,7 @@ def runMysql(args):
os.execl(mysqld_wrapper_list[0], *mysqld_wrapper_list) os.execl(mysqld_wrapper_list[0], *mysqld_wrapper_list)
def updateMysql(args): def updateMysql(conf):
conf = args[0]
sleep = 30 sleep = 30
is_succeed = False is_succeed = False
try: try:
......
##############################################################################
#
# Copyright (c) 2011 Vifib SARL and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
from slapos.recipe.librecipe import GenericBaseRecipe
import binascii
import hashlib
import os
import re
import zc.buildout
_isurl = re.compile('([a-zA-Z0-9+.-]+)://').match
# based on Zope2.utilities.mkzopeinstance.write_inituser
def Zope2InitUser(path, username, password):
# Set password only once
# Currently, rely on existence of a simple file:
# Create it the first time, then next time, detect this file and do no-op.
inituser_done_path = '%s_done' % path
if os.path.exists(inituser_done_path):
return
if os.path.exists(path):
return
open(path, 'w').write('')
os.chmod(path, 0600)
open(path, 'w').write('%s:{SHA}%s\n' % (
username,binascii.b2a_base64(hashlib.sha1(password).digest())[:-1]))
open(inituser_done_path, 'w').write('"inituser" file already created once.')
class Recipe(GenericBaseRecipe):
def _options(self, options):
options['password'] = self.generatePassword()
options['deadlock-password'] = self.generatePassword()
def install(self):
"""
All zope have to share file created by portal_classes
(until everything is integrated into the ZODB).
So, do not request zope instance and create multiple in the same partition.
"""
path_list = []
Zope2InitUser(self.options['inituser'], self.options['user'],
self.options['password'])
# Symlink to BT5 repositories defined in instance config.
# Those paths will eventually end up in the ZODB, and having symlinks
# inside the XXX makes it possible to reuse such ZODB with another software
# release[ version].
# Note: this path cannot be used for development, it's really just a
# read-only repository.
repository_path = self.options['bt5-repository']
self.bt5_repository_list = []
append = self.bt5_repository_list.append
for repository in self.options.get('bt5-repository-list', '').split():
repository = repository.strip()
if not repository:
continue
if _isurl(repository) and not repository.startswith("file://"):
# XXX: assume it's a valid URL
append(repository)
continue
if repository.startswith('file://'):
repository = repository.replace('file://', '', '')
if os.path.isabs(repository):
repo_id = hashlib.sha1(repository).hexdigest()
link = os.path.join(repository_path, repo_id)
if os.path.lexists(link):
if not os.path.islink(link):
raise zc.buildout.UserError(
'Target link already %r exists but it is not link' % link)
os.unlink(link)
os.symlink(repository, link)
self.logger.debug('Created link %r -> %r' % (link, repository_path))
# Always provide a URL-Type
append("file://" + link)
# Create zope configuration file
zope_config = dict(
thread_amount=self.options['thread-amount'],
zodb_root_path=self.options['zodb-path'],
zodb_cache_size=int(self.options['zodb-cache-size']),
)
zope_environment = dict(
TMP=self.options['tmp-path'],
TMPDIR=self.options['tmp-path'],
HOME=self.options['tmp-path'],
PATH=self.options['bin-path']
)
# configure default Zope2 zcml
open(self.options['site-zcml'], 'w').write(open(self.getTemplateFilename(
'site.zcml')).read())
zope_config['instance'] = self.options['instance-path']
zope_config['event_log'] = self.options['event-log']
zope_config['z2_log'] = self.options['z2-log']
zope_config['pid-filename'] = self.options['pid-file']
zope_config['lock-filename'] = self.options['lock-file']
zope_config['products'] = 'products %s' % self.options['instance-products']
zope_config['address'] = '%s:%s' % (self.options['ip'], self.options['port'])
zope_config.update(dump_url=self.options['deadlock-path'],
secret=self.options['deadlock-password'])
zope_wrapper_template_location = self.getTemplateFilename('zope.conf.in')
zope_conf_content = self.substituteTemplate(zope_wrapper_template_location,
zope_config)
if ('promise-path' in self.options) and ('site-id' in self.options):
zope_conf_content += self.substituteTemplate(self.getTemplateFilename(
'zope.conf.promise.in'), {
'site-id': self.options['site-id'],
'promise-path': self.options['promise-path'],
})
zope_conf_path = self.createFile(self.options['configuration-file'], zope_conf_content)
path_list.append(zope_conf_path)
# Create init script
path_list.append(self.createPythonScript(self.options['wrapper'], 'slapos.recipe.librecipe.execute.executee', [[self.options['runzope-binary'].strip(), '-C', zope_conf_path], zope_environment]))
return path_list
<configure
xmlns="http://namespaces.zope.org/zope"
xmlns:meta="http://namespaces.zope.org/meta"
xmlns:five="http://namespaces.zope.org/five">
<include package="Products.Five" />
<meta:redefinePermission from="zope2.Public" to="zope.Public" />
<!-- Load the meta -->
<include files="package-includes/*-meta.zcml" />
<five:loadProducts file="meta.zcml"/>
<!-- Load the configuration -->
<include files="package-includes/*-configure.zcml" />
<five:loadProducts />
<!-- Load the configuration overrides-->
<includeOverrides files="package-includes/*-overrides.zcml" />
<five:loadProductsOverrides />
<securityPolicy
component="Products.Five.security.FiveSecurityPolicy" />
</configure>
## Zope 2 configuration file generated by SlapOS
# Some defines
%%define INSTANCE %(instance)s
instancehome $INSTANCE
# Used products
%(products)s
# Environment is setup in running wrapper script
# Reason: zope.conf is read too late for some componets
# No need to debug
debug-mode off
# One thread is safe enough
zserver-threads %(thread_amount)s
# File location
pid-filename %(pid-filename)s
lock-filename %(lock-filename)s
# Encoding
rest-input-encoding utf-8
rest-output-encoding utf-8
default-zpublisher-encoding utf-8
# Disable ownership checking to execute codes generated by alarm
skip-ownership-checking on
# Temporary storage database (for sessions)
<zodb_db temporary>
<temporarystorage>
name temporary storage for sessioning
</temporarystorage>
mount-point /temp_folder
container-class Products.TemporaryFolder.TemporaryContainer
</zodb_db>
# Logging configuration
<eventlog>
<logfile>
dateformat
path %(event_log)s
</logfile>
</eventlog>
<logger access>
<logfile>
dateformat
path %(z2_log)s
</logfile>
</logger>
# Serving configuration
<http-server>
address %(address)s
</http-server>
# ZODB configuration
<zodb_db root>
cache-size %(zodb_cache_size)d
<filestorage>
path %(zodb_root_path)s
</filestorage>
mount-point /
</zodb_db>
<zoperunner>
program $INSTANCE/bin/runzope
</zoperunner>
# DeadlockDebugger configuration
<product-config DeadlockDebugger>
dump_url %(dump_url)s
secret %(secret)s
</product-config>
# ERP5 Timer Service
%%import timerserver
<timer-server>
interval 5
</timer-server>
# ERP5 promise
<product-config /%(site-id)s>
promise_path %(promise-path)s
</product-config>
##############################################################################
#
# Copyright (c) 2011 Vifib SARL and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
from slapos.recipe.librecipe import GenericBaseRecipe
import binascii
import hashlib
import os
import re
import zc.buildout
_isurl = re.compile('([a-zA-Z0-9+.-]+)://').match
# based on Zope2.utilities.mkzopeinstance.write_inituser
def Zope2InitUser(path, username, password):
# Set password only once
# Currently, rely on existence of a simple file:
# Create it the first time, then next time, detect this file and do no-op.
inituser_done_path = '%s_done' % path
if os.path.exists(inituser_done_path):
return
if os.path.exists(path):
return
open(path, 'w').write('')
os.chmod(path, 0600)
open(path, 'w').write('%s:{SHA}%s\n' % (
username,binascii.b2a_base64(hashlib.sha1(password).digest())[:-1]))
open(inituser_done_path, 'w').write('"inituser" file already created once.')
class Recipe(GenericBaseRecipe):
def _options(self, options):
if 'password' not in options:
options['password'] = self.generatePassword()
def install(self):
"""
All zope have to share file created by portal_classes
(until everything is integrated into the ZODB).
So, do not request zope instance and create multiple in the same partition.
"""
path_list = []
Zope2InitUser(self.options['inituser'], self.options['user'],
self.options['password'])
# Symlink to BT5 repositories defined in instance config.
# Those paths will eventually end up in the ZODB, and having symlinks
# inside the XXX makes it possible to reuse such ZODB with another software
# release[ version].
# Note: this path cannot be used for development, it's really just a
# read-only repository.
repository_path = self.options['bt5-repository']
self.bt5_repository_list = []
append = self.bt5_repository_list.append
for repository in self.options.get('bt5-repository-list', '').split():
repository = repository.strip()
if not repository:
continue
if _isurl(repository) and not repository.startswith("file://"):
# XXX: assume it's a valid URL
append(repository)
continue
if repository.startswith('file://'):
repository = repository.replace('file://', '', '')
if os.path.isabs(repository):
repo_id = hashlib.sha1(repository).hexdigest()
link = os.path.join(repository_path, repo_id)
if os.path.lexists(link):
if not os.path.islink(link):
raise zc.buildout.UserError(
'Target link already %r exists but it is not link' % link)
os.unlink(link)
os.symlink(repository, link)
self.logger.debug('Created link %r -> %r' % (link, repository_path))
# Always provide a URL-Type
append("file://" + link)
zope_environment = {
'TMP': self.options['tmp-path'],
'TMPDIR': self.options['tmp-path'],
'HOME': self.options.get('home-path', self.options.get('tmp-path')),
'PATH': self.options['bin-path'],
'TZ': self.options['timezone'],
}
instance_home = self.options.get("instancehome-path", None)
if instance_home:
zope_environment["INSTANCE_HOME"] = instance_home
# configure default Zope2 zcml
open(self.options['site-zcml'], 'w').write(open(self.getTemplateFilename(
'site.zcml')).read())
# Create init script
path_list.append(self.createPythonScript(self.options['wrapper'], 'slapos.recipe.librecipe.execute.executee', [[self.options['runzope-binary'].strip(), '-C', self.options['configuration-file']], zope_environment]))
return path_list
<configure
xmlns="http://namespaces.zope.org/zope"
xmlns:meta="http://namespaces.zope.org/meta"
xmlns:five="http://namespaces.zope.org/five">
<include package="Products.Five" />
<meta:redefinePermission from="zope2.Public" to="zope.Public" />
<!-- Load the meta -->
<include files="package-includes/*-meta.zcml" />
<five:loadProducts file="meta.zcml"/>
<!-- Load the configuration -->
<include files="package-includes/*-configure.zcml" />
<five:loadProducts />
<!-- Load the configuration overrides-->
<includeOverrides files="package-includes/*-overrides.zcml" />
<five:loadProductsOverrides />
<securityPolicy
component="Products.Five.security.FiveSecurityPolicy" />
</configure>
...@@ -120,12 +120,11 @@ class Recipe(GenericBaseRecipe): ...@@ -120,12 +120,11 @@ class Recipe(GenericBaseRecipe):
'server_text': server_snippet}, 'server_text': server_snippet},
) )
) )
wrapper_path = self.createPythonScript( wrapper_path = self.createWrapper(
self.options['wrapper-path'], self.options['wrapper-path'],
'slapos.recipe.librecipe.execute.execute', (self.options['binary-path'].strip(), '-f', configuration_path))
arguments=[self.options['binary-path'].strip(), '-f', configuration_path],)
ctl_path = self.createPythonScript( ctl_path = self.createPythonScript(
self.options['ctl-path'], self.options['ctl-path'],
'%s.haproxy.haproxyctl' % __name__, __name__ + '.haproxy.haproxyctl',
{'socket_path':self.options['socket-path']}) (self.options['socket-path'],))
return [configuration_path, wrapper_path, ctl_path] return [configuration_path, wrapper_path, ctl_path]
...@@ -4,7 +4,7 @@ try: ...@@ -4,7 +4,7 @@ try:
except ImportError: except ImportError:
pass pass
def haproxyctl(conf): def haproxyctl(socket_path):
while True: while True:
try: try:
l = raw_input('> ') l = raw_input('> ')
...@@ -14,7 +14,7 @@ def haproxyctl(conf): ...@@ -14,7 +14,7 @@ def haproxyctl(conf):
if l == 'quit': if l == 'quit':
break break
s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
s.connect(conf['socket_path']) s.connect(socket_path)
s.send('%s\n' % l) s.send('%s\n' % l)
while True: while True:
r = s.recv(1024) r = s.recv(1024)
......
##############################################################################
#
# Copyright (c) 2010 Vifib SARL and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
from slapos.recipe.librecipe import BaseSlapRecipe
import hashlib
import os
import pkg_resources
import sys
import zc.buildout
import ConfigParser
class Recipe(BaseSlapRecipe):
def getTemplateFilename(self, template_name):
return pkg_resources.resource_filename(__name__,
'template/%s' % template_name)
def _install(self):
self.path_list = []
self.requirements, self.ws = self.egg.working_set()
# XXX-Cedric : add logrotate?
self.cron_d = self.installCrond()
kumo_conf = self.installKumo(self.getLocalIPv4Address())
ca_conf = self.installCertificateAuthority()
key, certificate = self.requestCertificate('Login Based Access')
stunnel_conf = self.installStunnel(self.getGlobalIPv6Address(),
self.getLocalIPv4Address(), 12345, kumo_conf['kumo_gateway_port'],
certificate, key, ca_conf['ca_crl'],
ca_conf['certificate_authority_path'])
self.linkBinary()
self.setConnectionDict(dict(
stunnel_ip = stunnel_conf['public_ip'],
stunnel_port = stunnel_conf['public_port'],
))
return self.path_list
def linkBinary(self):
"""Links binaries to instance's bin directory for easier exposal"""
for linkline in self.options.get('link_binary_list', '').splitlines():
if not linkline:
continue
target = linkline.split()
if len(target) == 1:
target = target[0]
path, linkname = os.path.split(target)
else:
linkname = target[1]
target = target[0]
link = os.path.join(self.bin_directory, linkname)
if os.path.lexists(link):
if not os.path.islink(link):
raise zc.buildout.UserError(
'Target link already %r exists but it is not link' % link)
os.unlink(link)
os.symlink(target, link)
self.logger.debug('Created link %r -> %r' % (link, target))
self.path_list.append(link)
def installCrond(self):
timestamps = self.createDataDirectory('cronstamps')
cron_output = os.path.join(self.log_directory, 'cron-output')
self._createDirectory(cron_output)
catcher = zc.buildout.easy_install.scripts([('catchcron',
__name__ + '.catdatefile', 'catdatefile')], self.ws, sys.executable,
self.bin_directory, arguments=[cron_output])[0]
self.path_list.append(catcher)
cron_d = os.path.join(self.etc_directory, 'cron.d')
crontabs = os.path.join(self.etc_directory, 'crontabs')
self._createDirectory(cron_d)
self._createDirectory(crontabs)
# Use execute from erp5.
wrapper = zc.buildout.easy_install.scripts([('crond',
'slapos.recipe.librecipe.execute', 'execute')], self.ws, sys.executable,
self.wrapper_directory, arguments=[
self.options['dcrond_binary'].strip(), '-s', cron_d, '-c', crontabs,
'-t', timestamps, '-f', '-l', '5', '-M', catcher]
)[0]
self.path_list.append(wrapper)
return cron_d
def installLogrotate(self):
"""Installs logortate main configuration file and registers its to cron"""
logrotate_d = os.path.abspath(os.path.join(self.etc_directory,
'logrotate.d'))
self._createDirectory(logrotate_d)
logrotate_backup = self.createBackupDirectory('logrotate')
logrotate_conf = self.createConfigurationFile("logrotate.conf",
"include %s" % logrotate_d)
logrotate_cron = os.path.join(self.cron_d, 'logrotate')
state_file = os.path.join(self.data_root_directory, 'logrotate.status')
open(logrotate_cron, 'w').write('0 0 * * * %s -s %s %s' %
(self.options['logrotate_binary'], state_file, logrotate_conf))
self.path_list.extend([logrotate_d, logrotate_conf, logrotate_cron])
return logrotate_d, logrotate_backup
def registerLogRotation(self, name, log_file_list, postrotate_script):
"""Register new log rotation requirement"""
open(os.path.join(self.logrotate_d, name), 'w').write(
self.substituteTemplate(self.getTemplateFilename(
'logrotate_entry.in'),
dict(file_list=' '.join(['"'+q+'"' for q in log_file_list]),
postrotate=postrotate_script, olddir=self.logrotate_backup)))
def installCertificateAuthority(self, ca_country_code='XX',
ca_email='xx@example.com', ca_state='State', ca_city='City',
ca_company='Company'):
backup_path = self.createBackupDirectory('ca')
self.ca_dir = os.path.join(self.data_root_directory, 'ca')
self._createDirectory(self.ca_dir)
self.ca_request_dir = os.path.join(self.ca_dir, 'requests')
self._createDirectory(self.ca_request_dir)
config = dict(ca_dir=self.ca_dir, request_dir=self.ca_request_dir)
self.ca_private = os.path.join(self.ca_dir, 'private')
self.ca_certs = os.path.join(self.ca_dir, 'certs')
self.ca_crl = os.path.join(self.ca_dir, 'crl')
self.ca_newcerts = os.path.join(self.ca_dir, 'newcerts')
self.ca_key_ext = '.key'
self.ca_crt_ext = '.crt'
for d in [self.ca_private, self.ca_crl, self.ca_newcerts, self.ca_certs]:
self._createDirectory(d)
for f in ['crlnumber', 'serial']:
if not os.path.exists(os.path.join(self.ca_dir, f)):
open(os.path.join(self.ca_dir, f), 'w').write('01')
if not os.path.exists(os.path.join(self.ca_dir, 'index.txt')):
open(os.path.join(self.ca_dir, 'index.txt'), 'w').write('')
openssl_configuration = os.path.join(self.ca_dir, 'openssl.cnf')
config.update(
working_directory=self.ca_dir,
country_code=ca_country_code,
state=ca_state,
city=ca_city,
company=ca_company,
email_address=ca_email,
)
self._writeFile(openssl_configuration, pkg_resources.resource_string(
__name__, 'template/openssl.cnf.ca.in') % config)
self.path_list.extend(zc.buildout.easy_install.scripts([
('certificate_authority',
__name__ + '.certificate_authority', 'runCertificateAuthority')],
self.ws, sys.executable, self.wrapper_directory, arguments=[dict(
openssl_configuration=openssl_configuration,
openssl_binary=self.options['openssl_binary'],
certificate=os.path.join(self.ca_dir, 'cacert.pem'),
key=os.path.join(self.ca_private, 'cakey.pem'),
crl=os.path.join(self.ca_crl),
request_dir=self.ca_request_dir
)]))
# configure backup
backup_cron = os.path.join(self.cron_d, 'ca_rdiff_backup')
open(backup_cron, 'w').write(
'''0 0 * * * %(rdiff_backup)s %(source)s %(destination)s'''%dict(
rdiff_backup=self.options['rdiff_backup_binary'],
source=self.ca_dir,
destination=backup_path))
self.path_list.append(backup_cron)
return dict(
ca_certificate=os.path.join(config['ca_dir'], 'cacert.pem'),
ca_crl=os.path.join(config['ca_dir'], 'crl'),
certificate_authority_path=config['ca_dir']
)
def requestCertificate(self, name):
hash = hashlib.sha512(name).hexdigest()
key = os.path.join(self.ca_private, hash + self.ca_key_ext)
certificate = os.path.join(self.ca_certs, hash + self.ca_crt_ext)
parser = ConfigParser.RawConfigParser()
parser.add_section('certificate')
parser.set('certificate', 'name', name)
parser.set('certificate', 'key_file', key)
parser.set('certificate', 'certificate_file', certificate)
parser.write(open(os.path.join(self.ca_request_dir, hash), 'w'))
return key, certificate
def installStunnel(self, public_ip, private_ip, public_port, private_port,
ca_certificate, key, ca_crl, ca_path):
"""Installs stunnel"""
template_filename = self.getTemplateFilename('stunnel.conf.in')
log = os.path.join(self.log_directory, 'stunnel.log')
pid_file = os.path.join(self.run_directory, 'stunnel.pid')
stunnel_conf = dict(
public_ip=public_ip,
private_ip=private_ip,
public_port=public_port,
pid_file=pid_file,
log=log,
cert = ca_certificate,
key = key,
ca_crl = ca_crl,
ca_path = ca_path,
private_port = private_port,
)
stunnel_conf_path = self.createConfigurationFile("stunnel.conf",
self.substituteTemplate(template_filename,
stunnel_conf))
wrapper = zc.buildout.easy_install.scripts([('stunnel',
'slapos.recipe.librecipe.execute', 'execute_wait')], self.ws,
sys.executable, self.wrapper_directory, arguments=[
[self.options['stunnel_binary'].strip(), stunnel_conf_path],
[ca_certificate, key]]
)[0]
self.path_list.append(wrapper)
return stunnel_conf
def installKumo(self, ip, kumo_manager_port=13101, kumo_server_port=13201,
kumo_server_listen_port=13202, kumo_gateway_port=13301):
# XXX: kumo is not storing pid in file, unless it is not running as daemon
# but running daemons is incompatible with SlapOS, so there is currently
# no way to have Kumo's pid files to rotate logs and send signals to them
config = dict(
kumo_gateway_binary=self.options['kumo_gateway_binary'],
kumo_gateway_ip=ip,
kumo_gateway_log=os.path.join(self.log_directory, "kumo-gateway.log"),
kumo_manager_binary=self.options['kumo_manager_binary'],
kumo_manager_ip=ip,
kumo_manager_log=os.path.join(self.log_directory, "kumo-manager.log"),
kumo_server_binary=self.options['kumo_server_binary'],
kumo_server_ip=ip,
kumo_server_log=os.path.join(self.log_directory, "kumo-server.log"),
kumo_server_storage=os.path.join(self.data_root_directory, "kumodb.tch"),
kumo_manager_port=kumo_manager_port,
kumo_server_port=kumo_server_port,
kumo_server_listen_port=kumo_server_listen_port,
kumo_gateway_port=kumo_gateway_port
)
self.path_list.append(self.createRunningWrapper('kumo_gateway',
self.substituteTemplate(self.getTemplateFilename('kumo_gateway.in'),
config)))
self.path_list.append(self.createRunningWrapper('kumo_manager',
self.substituteTemplate(self.getTemplateFilename('kumo_manager.in'),
config)))
self.path_list.append(self.createRunningWrapper('kumo_server',
self.substituteTemplate(self.getTemplateFilename('kumo_server.in'),
config)))
return dict(
kumo_address = '%s:%s' % (config['kumo_gateway_ip'],
config['kumo_gateway_port']),
kumo_gateway_ip=config['kumo_gateway_ip'],
kumo_gateway_port=config['kumo_gateway_port'],
)
import os
import subprocess
import time
import ConfigParser
def popenCommunicate(command_list, input=None):
subprocess_kw = dict(stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
if input is not None:
subprocess_kw.update(stdin=subprocess.PIPE)
popen = subprocess.Popen(command_list, **subprocess_kw)
result = popen.communicate(input)[0]
if popen.returncode is None:
popen.kill()
if popen.returncode != 0:
raise ValueError('Issue during calling %r, result was:\n%s' % (
command_list, result))
return result
class CertificateAuthority:
def __init__(self, key, certificate, openssl_binary,
openssl_configuration, request_dir):
self.key = key
self.certificate = certificate
self.openssl_binary = openssl_binary
self.openssl_configuration = openssl_configuration
self.request_dir = request_dir
def checkAuthority(self):
file_list = [ self.key, self.certificate ]
ca_ready = True
for f in file_list:
if not os.path.exists(f):
ca_ready = False
break
if ca_ready:
return
for f in file_list:
if os.path.exists(f):
os.unlink(f)
try:
# no CA, let us create new one
popenCommunicate([self.openssl_binary, 'req', '-nodes', '-config',
self.openssl_configuration, '-new', '-x509', '-extensions',
'v3_ca', '-keyout', self.key, '-out', self.certificate,
'-days', '10950'], 'Automatic Certificate Authority\n')
except:
try:
for f in file_list:
if os.path.exists(f):
os.unlink(f)
except:
# do not raise during cleanup
pass
raise
def _checkCertificate(self, common_name, key, certificate):
file_list = [key, certificate]
ready = True
for f in file_list:
if not os.path.exists(f):
ready = False
break
if ready:
return False
for f in file_list:
if os.path.exists(f):
os.unlink(f)
csr = certificate + '.csr'
try:
popenCommunicate([self.openssl_binary, 'req', '-config',
self.openssl_configuration, '-nodes', '-new', '-keyout',
key, '-out', csr, '-days', '3650'],
common_name + '\n')
try:
popenCommunicate([self.openssl_binary, 'ca', '-batch', '-config',
self.openssl_configuration, '-out', certificate,
'-infiles', csr])
finally:
if os.path.exists(csr):
os.unlink(csr)
except:
try:
for f in file_list:
if os.path.exists(f):
os.unlink(f)
except:
# do not raise during cleanup
pass
raise
else:
return True
def checkRequestDir(self):
for request_file in os.listdir(self.request_dir):
parser = ConfigParser.RawConfigParser()
parser.readfp(open(os.path.join(self.request_dir, request_file), 'r'))
if self._checkCertificate(parser.get('certificate', 'name'),
parser.get('certificate', 'key_file'), parser.get('certificate',
'certificate_file')):
print 'Created certificate %r' % parser.get('certificate', 'name')
def runCertificateAuthority(args):
ca_conf = args[0]
ca = CertificateAuthority(ca_conf['key'], ca_conf['certificate'],
ca_conf['openssl_binary'], ca_conf['openssl_configuration'],
ca_conf['request_dir'])
while True:
ca.checkAuthority()
ca.checkRequestDir()
time.sleep(60)
#!/bin/sh
exec %(kumo_gateway_binary)s -F -E -m %(kumo_manager_ip)s:%(kumo_manager_port)s -t %(kumo_gateway_ip)s:%(kumo_gateway_port)s -o %(kumo_gateway_log)s
#!/bin/sh
exec %(kumo_manager_binary)s -a -l %(kumo_manager_ip)s:%(kumo_manager_port)s -o %(kumo_manager_log)s
#!/bin/sh
exec %(kumo_server_binary)s -l %(kumo_server_ip)s:%(kumo_server_port)s -L %(kumo_server_listen_port)s -m %(kumo_manager_ip)s:%(kumo_manager_port)s -s %(kumo_server_storage)s -o %(kumo_server_log)s
#
# OpenSSL example configuration file.
# This is mostly being used for generation of certificate requests.
#
# This definition stops the following lines choking if HOME isn't
# defined.
HOME = .
RANDFILE = $ENV::HOME/.rnd
# Extra OBJECT IDENTIFIER info:
#oid_file = $ENV::HOME/.oid
oid_section = new_oids
# To use this configuration file with the "-extfile" option of the
# "openssl x509" utility, name here the section containing the
# X.509v3 extensions to use:
# extensions =
# (Alternatively, use a configuration file that has only
# X.509v3 extensions in its main [= default] section.)
[ new_oids ]
# We can add new OIDs in here for use by 'ca', 'req' and 'ts'.
# Add a simple OID like this:
# testoid1=1.2.3.4
# Or use config file substitution like this:
# testoid2=${testoid1}.5.6
# Policies used by the TSA examples.
tsa_policy1 = 1.2.3.4.1
tsa_policy2 = 1.2.3.4.5.6
tsa_policy3 = 1.2.3.4.5.7
####################################################################
[ ca ]
default_ca = CA_default # The default ca section
####################################################################
[ CA_default ]
dir = %(working_directory)s # Where everything is kept
certs = $dir/certs # Where the issued certs are kept
crl_dir = $dir/crl # Where the issued crl are kept
database = $dir/index.txt # database index file.
#unique_subject = no # Set to 'no' to allow creation of
# several ctificates with same subject.
new_certs_dir = $dir/newcerts # default place for new certs.
certificate = $dir/cacert.pem # The CA certificate
serial = $dir/serial # The current serial number
crlnumber = $dir/crlnumber # the current crl number
# must be commented out to leave a V1 CRL
crl = $dir/crl.pem # The current CRL
private_key = $dir/private/cakey.pem # The private key
RANDFILE = $dir/private/.rand # private random number file
x509_extensions = usr_cert # The extentions to add to the cert
# Comment out the following two lines for the "traditional"
# (and highly broken) format.
name_opt = ca_default # Subject Name options
cert_opt = ca_default # Certificate field options
# Extension copying option: use with caution.
# copy_extensions = copy
# Extensions to add to a CRL. Note: Netscape communicator chokes on V2 CRLs
# so this is commented out by default to leave a V1 CRL.
# crlnumber must also be commented out to leave a V1 CRL.
# crl_extensions = crl_ext
default_days = 3650 # how long to certify for
default_crl_days= 30 # how long before next CRL
default_md = default # use public key default MD
preserve = no # keep passed DN ordering
# A few difference way of specifying how similar the request should look
# For type CA, the listed attributes must be the same, and the optional
# and supplied fields are just that :-)
policy = policy_match
# For the CA policy
[ policy_match ]
countryName = match
stateOrProvinceName = match
organizationName = match
organizationalUnitName = optional
commonName = supplied
emailAddress = optional
# For the 'anything' policy
# At this point in time, you must list all acceptable 'object'
# types.
[ policy_anything ]
countryName = optional
stateOrProvinceName = optional
localityName = optional
organizationName = optional
organizationalUnitName = optional
commonName = supplied
emailAddress = optional
####################################################################
[ req ]
default_bits = 2048
default_md = sha1
default_keyfile = privkey.pem
distinguished_name = req_distinguished_name
#attributes = req_attributes
x509_extensions = v3_ca # The extentions to add to the self signed cert
# Passwords for private keys if not present they will be prompted for
# input_password = secret
# output_password = secret
# This sets a mask for permitted string types. There are several options.
# default: PrintableString, T61String, BMPString.
# pkix : PrintableString, BMPString (PKIX recommendation before 2004)
# utf8only: only UTF8Strings (PKIX recommendation after 2004).
# nombstr : PrintableString, T61String (no BMPStrings or UTF8Strings).
# MASK:XXXX a literal mask value.
# WARNING: ancient versions of Netscape crash on BMPStrings or UTF8Strings.
string_mask = utf8only
# req_extensions = v3_req # The extensions to add to a certificate request
[ req_distinguished_name ]
countryName = Country Name (2 letter code)
countryName_value = %(country_code)s
countryName_min = 2
countryName_max = 2
stateOrProvinceName = State or Province Name (full name)
stateOrProvinceName_value = %(state)s
localityName = Locality Name (eg, city)
localityName_value = %(city)s
0.organizationName = Organization Name (eg, company)
0.organizationName_value = %(company)s
# we can do this but it is not needed normally :-)
#1.organizationName = Second Organization Name (eg, company)
#1.organizationName_default = World Wide Web Pty Ltd
commonName = Common Name (eg, your name or your server\'s hostname)
commonName_max = 64
emailAddress = Email Address
emailAddress_value = %(email_address)s
emailAddress_max = 64
# SET-ex3 = SET extension number 3
#[ req_attributes ]
#challengePassword = A challenge password
#challengePassword_min = 4
#challengePassword_max = 20
#
#unstructuredName = An optional company name
[ usr_cert ]
# These extensions are added when 'ca' signs a request.
# This goes against PKIX guidelines but some CAs do it and some software
# requires this to avoid interpreting an end user certificate as a CA.
basicConstraints=CA:FALSE
# Here are some examples of the usage of nsCertType. If it is omitted
# the certificate can be used for anything *except* object signing.
# This is OK for an SSL server.
# nsCertType = server
# For an object signing certificate this would be used.
# nsCertType = objsign
# For normal client use this is typical
# nsCertType = client, email
# and for everything including object signing:
# nsCertType = client, email, objsign
# This is typical in keyUsage for a client certificate.
# keyUsage = nonRepudiation, digitalSignature, keyEncipherment
# This will be displayed in Netscape's comment listbox.
nsComment = "OpenSSL Generated Certificate"
# PKIX recommendations harmless if included in all certificates.
subjectKeyIdentifier=hash
authorityKeyIdentifier=keyid,issuer
# This stuff is for subjectAltName and issuerAltname.
# Import the email address.
# subjectAltName=email:copy
# An alternative to produce certificates that aren't
# deprecated according to PKIX.
# subjectAltName=email:move
# Copy subject details
# issuerAltName=issuer:copy
#nsCaRevocationUrl = http://www.domain.dom/ca-crl.pem
#nsBaseUrl
#nsRevocationUrl
#nsRenewalUrl
#nsCaPolicyUrl
#nsSslServerName
# This is required for TSA certificates.
# extendedKeyUsage = critical,timeStamping
[ v3_req ]
# Extensions to add to a certificate request
basicConstraints = CA:FALSE
keyUsage = nonRepudiation, digitalSignature, keyEncipherment
[ v3_ca ]
# Extensions for a typical CA
# PKIX recommendation.
subjectKeyIdentifier=hash
authorityKeyIdentifier=keyid:always,issuer
# This is what PKIX recommends but some broken software chokes on critical
# extensions.
#basicConstraints = critical,CA:true
# So we do this instead.
basicConstraints = CA:true
# Key usage: this is typical for a CA certificate. However since it will
# prevent it being used as an test self-signed certificate it is best
# left out by default.
# keyUsage = cRLSign, keyCertSign
# Some might want this also
# nsCertType = sslCA, emailCA
# Include email address in subject alt name: another PKIX recommendation
# subjectAltName=email:copy
# Copy issuer details
# issuerAltName=issuer:copy
# DER hex encoding of an extension: beware experts only!
# obj=DER:02:03
# Where 'obj' is a standard or added object
# You can even override a supported extension:
# basicConstraints= critical, DER:30:03:01:01:FF
[ crl_ext ]
# CRL extensions.
# Only issuerAltName and authorityKeyIdentifier make any sense in a CRL.
# issuerAltName=issuer:copy
authorityKeyIdentifier=keyid:always
[ proxy_cert_ext ]
# These extensions should be added when creating a proxy certificate
# This goes against PKIX guidelines but some CAs do it and some software
# requires this to avoid interpreting an end user certificate as a CA.
basicConstraints=CA:FALSE
# Here are some examples of the usage of nsCertType. If it is omitted
# the certificate can be used for anything *except* object signing.
# This is OK for an SSL server.
# nsCertType = server
# For an object signing certificate this would be used.
# nsCertType = objsign
# For normal client use this is typical
# nsCertType = client, email
# and for everything including object signing:
# nsCertType = client, email, objsign
# This is typical in keyUsage for a client certificate.
# keyUsage = nonRepudiation, digitalSignature, keyEncipherment
# This will be displayed in Netscape's comment listbox.
nsComment = "OpenSSL Generated Certificate"
# PKIX recommendations harmless if included in all certificates.
subjectKeyIdentifier=hash
authorityKeyIdentifier=keyid,issuer
# This stuff is for subjectAltName and issuerAltname.
# Import the email address.
# subjectAltName=email:copy
# An alternative to produce certificates that aren't
# deprecated according to PKIX.
# subjectAltName=email:move
# Copy subject details
# issuerAltName=issuer:copy
#nsCaRevocationUrl = http://www.domain.dom/ca-crl.pem
#nsBaseUrl
#nsRevocationUrl
#nsRenewalUrl
#nsCaPolicyUrl
#nsSslServerName
# This really needs to be in place for it to be a proxy certificate.
proxyCertInfo=critical,language:id-ppl-anyLanguage,pathlen:3,policy:foo
####################################################################
[ tsa ]
default_tsa = tsa_config1 # the default TSA section
[ tsa_config1 ]
# These are used by the TSA reply generation only.
dir = /etc/pki/tls # TSA root directory
serial = $dir/tsaserial # The current serial number (mandatory)
crypto_device = builtin # OpenSSL engine to use for signing
signer_cert = $dir/tsacert.pem # The TSA signing certificate
# (optional)
certs = $dir/cacert.pem # Certificate chain to include in reply
# (optional)
signer_key = $dir/private/tsakey.pem # The TSA private key (optional)
default_policy = tsa_policy1 # Policy if request did not specify it
# (optional)
other_policies = tsa_policy2, tsa_policy3 # acceptable policies (optional)
digests = md5, sha1 # Acceptable message digests (mandatory)
accuracy = secs:1, millisecs:500, microsecs:100 # (optional)
clock_precision_digits = 0 # number of digits after dot. (optional)
ordering = yes # Is ordering defined for timestamps?
# (optional, default: no)
tsa_name = yes # Must the TSA name be included in the reply?
# (optional, default: no)
ess_cert_id_chain = no # Must the ESS cert id chain be included?
# (optional, default: no)
foreground = yes
output = %(log)s
pid = %(pid_file)s
syslog = no
CApath = %(ca_path)s
key = %(key)s
CRLpath = %(ca_crl)s
cert = %(cert)s
[service]
accept = %(public_ip)s:%(public_port)s
connect = %(private_ip)s:%(private_port)s
...@@ -298,9 +298,9 @@ class Request(BaseRecipe): ...@@ -298,9 +298,9 @@ class Request(BaseRecipe):
'local_host': local_host, 'local_port': local_port, 'local_host': local_host, 'local_port': local_port,
})) }))
wrapper = zc.buildout.easy_install.scripts([('stunnel', wrapper = zc.buildout.easy_install.scripts([('stunnel',
'slapos.recipe.librecipe.execute', 'execute')], self.ws, 'slapos.recipe.librecipe.execute', 'generic_exec')], self.ws,
sys.executable, self.wrapper_directory, arguments=[ sys.executable, self.wrapper_directory, arguments='%r, %r'
self.options['stunnel_binary'].strip(), stunnel_conf_path] % (self.options['stunnel_binary'].strip(), stunnel_conf_path)
)[0] )[0]
self.path_list.append(wrapper) self.path_list.append(wrapper)
return (local_host, local_port,) return (local_host, local_port,)
...@@ -71,13 +71,12 @@ class Recipe(GenericBaseRecipe): ...@@ -71,13 +71,12 @@ class Recipe(GenericBaseRecipe):
) )
path_list.append(httpd_conf) path_list.append(httpd_conf)
wrapper = self.createWrapper(name=self.options['wrapper'], wrapper = self.createWrapper(self.options['wrapper'],
command=self.options['httpd-binary'], (self.options['httpd-binary'],
parameters=[
'-f', '-f',
self.options['httpd-conf'], self.options['httpd-conf'],
'-DFOREGROUND' '-DFOREGROUND'
]) ))
path_list.append(wrapper) path_list.append(wrapper)
......
...@@ -2,7 +2,6 @@ import sys ...@@ -2,7 +2,6 @@ import sys
import os import os
import signal import signal
import subprocess import subprocess
import time
from collections import defaultdict from collections import defaultdict
from inotify_simple import INotify, flags from inotify_simple import INotify, flags
...@@ -29,65 +28,96 @@ def _wait_files_creation(file_list): ...@@ -29,65 +28,96 @@ def _wait_files_creation(file_list):
if event.name in directory: if event.name in directory:
directory[event.name] = event.mask & (flags.CREATE | flags.MOVED_TO) directory[event.name] = event.mask & (flags.CREATE | flags.MOVED_TO)
def execute(args): def _libc():
"""Portable execution with process replacement""" from ctypes import CDLL, get_errno, c_char_p, c_int, c_ulong, util
# XXX: Kept for backward compatibility libc = CDLL(util.find_library('c'), use_errno=True)
generic_exec([args, None, None]) libc_mount = libc.mount
libc_mount.argtypes = c_char_p, c_char_p, c_char_p, c_ulong, c_char_p
def execute_wait(args): def mount(source, target, filesystemtype, mountflags, data):
"""Execution but after all files in args[1] exists""" if libc_mount(source, target, filesystemtype, mountflags, data):
# XXX: Kept for backward compatibility e = get_errno()
generic_exec([args[0], args[1], None]) raise OSError(e, os.strerror(e))
libc_unshare = libc.unshare
libc_unshare.argtypes = c_int,
def unshare(flags):
if libc_unshare(flags):
e = get_errno()
raise OSError(e, os.strerror(e))
return mount, unshare
def generic_exec(args, extra_environ=None, wait_list=None,
pidfile=None, reserve_cpu=False, private_dev_shm=None,
#shebang_workaround=False, # XXX: still needed ?
):
args = list(args)
if pidfile:
import psutil
try:
with open(pidfile) as f:
pid = int(f.read())
running = psutil.Process(pid).cmdline()
except Exception:
pass
else:
# With chained shebangs, several paths may be inserted at the beginning.
n = len(args)
for i in xrange(1+len(running)-n):
if args == running[i:n+i]:
sys.exit("Already running with pid %s." % pid)
with open(pidfile, 'w') as f:
f.write(str(os.getpid()))
args += sys.argv[1:]
if reserve_cpu:
# If the CGROUPS cpuset is available (and prepared by slap format),
# request an exclusive CPU core for this process.
with open(os.path.expanduser('~/.slapos-cpu-exclusive'), 'a') as f:
f.write('%s\n' % os.getpid())
if wait_list:
_wait_files_creation(wait_list)
if private_dev_shm:
mount, unshare = _libc()
CLONE_NEWNS = 0x00020000
CLONE_NEWUSER = 0x10000000
uid = os.getuid()
gid = os.getgid()
unshare(CLONE_NEWUSER |CLONE_NEWNS)
with open('/proc/self/setgroups', 'wb') as f: f.write('deny')
with open('/proc/self/uid_map', 'wb') as f: f.write('%s %s 1' % (uid, uid))
with open('/proc/self/gid_map', 'wb') as f: f.write('%s %s 1' % (gid, gid))
mount('tmpfs', '/dev/shm', 'tmpfs', 0, 'size=' + private_dev_shm)
if extra_environ:
env = os.environ.copy()
env.update(extra_environ)
os.execve(args[0], args, env)
else:
os.execv(args[0], args)
child_pg = None child_pg = None
def executee(args):
"""Portable execution with process replacement and environment manipulation"""
# XXX: Kept for backward compatibility
generic_exec([args[0], None, args[1]])
def executee_wait(args):
"""Portable execution with process replacement and environment manipulation"""
# XXX: Kept for backward compatibility
generic_exec(args)
def generic_exec(args):
exec_list = list(args[0])
file_list = args[1]
environment_overriding = args[2]
exec_env = os.environ.copy()
if environment_overriding is not None:
exec_env.update(environment_overriding)
if file_list is not None:
_wait_files_creation(file_list)
os.execve(exec_list[0], exec_list + sys.argv[1:], exec_env)
def sig_handler(sig, frame): def sig_handler(sig, frame):
print 'Received signal %r, killing children and exiting' % sig print 'Received signal %r, killing children and exiting' % sig
if child_pg is not None: if child_pg is not None:
os.killpg(child_pg, signal.SIGHUP) os.killpg(child_pg, signal.SIGHUP)
os.killpg(child_pg, signal.SIGTERM) os.killpg(child_pg, signal.SIGTERM)
sys.exit(0) sys.exit()
signal.signal(signal.SIGINT, sig_handler)
signal.signal(signal.SIGQUIT, sig_handler)
signal.signal(signal.SIGTERM, sig_handler)
def execute_with_signal_translation(args): def execute_with_signal_translation(args):
"""Run process as children and translate from SIGTERM to another signal""" """Run process as children and translate from SIGTERM to another signal"""
global child_pg global child_pg
signal.signal(signal.SIGINT, sig_handler)
signal.signal(signal.SIGQUIT, sig_handler)
signal.signal(signal.SIGTERM, sig_handler)
child = subprocess.Popen(args, close_fds=True, preexec_fn=os.setsid) child = subprocess.Popen(args, close_fds=True, preexec_fn=os.setsid)
child_pg = child.pid child_pg = child.pid
try: try:
print 'Process %r started' % args print 'Process %r started' % args
while True: signal.pause()
time.sleep(10)
finally: finally:
os.killpg(child_pg, signal.SIGHUP) os.killpg(child_pg, signal.SIGHUP)
os.killpg(child_pg, signal.SIGTERM) os.killpg(child_pg, signal.SIGTERM)
...@@ -33,7 +33,6 @@ import sys ...@@ -33,7 +33,6 @@ import sys
import inspect import inspect
import re import re
import shutil import shutil
from textwrap import dedent
import urllib import urllib
import urlparse import urlparse
...@@ -61,7 +60,13 @@ class GenericBaseRecipe(object): ...@@ -61,7 +60,13 @@ class GenericBaseRecipe(object):
self._options(options) # Options Hook self._options(options) # Options Hook
self.options = options.copy() # Updated options dict self.options = options.copy() # Updated options dict
self._ws = self.getWorkingSet() @property
def _ws(self):
# getWorkingSet() is slow and it is not always needed.
# So _ws should be a lazy attribute.
if getattr(self, '_ws_internal', None) is None:
self._ws_internal = self.getWorkingSet()
return self._ws_internal
def update(self): def update(self):
"""By default update method does the same thing than install""" """By default update method does the same thing than install"""
...@@ -110,92 +115,60 @@ class GenericBaseRecipe(object): ...@@ -110,92 +115,60 @@ class GenericBaseRecipe(object):
with io.open(filepath, 'w+', encoding=encoding) as f: with io.open(filepath, 'w+', encoding=encoding) as f:
f.write(u'\n'.join(lines)) f.write(u'\n'.join(lines))
def createPythonScript(self, name, absolute_function, arguments=''): def createPythonScript(self, name, absolute_function, args=(), kw={}):
"""Create a python script using zc.buildout.easy_install.scripts """Create a python script using zc.buildout.easy_install.scripts
* function should look like 'module.function', or only 'function' * function should look like 'module.function', or only 'function'
if it is a builtin function.""" if it is a builtin function."""
absolute_function = tuple(absolute_function.rsplit('.', 1)) function = absolute_function.rsplit('.', 1)
if len(absolute_function) == 1: if len(function) == 1:
absolute_function = ('__builtin__',) + absolute_function module = '__builtin__'
if len(absolute_function) != 2: function, = function
raise ValueError("A non valid function was given") else:
module, function = function
module, function = absolute_function
path, filename = os.path.split(os.path.abspath(name)) path, filename = os.path.split(os.path.abspath(name))
script = zc.buildout.easy_install.scripts( assert not isinstance(args, (basestring, dict)), args
args = map(repr, args)
args += map('%s=%r'.__mod__, kw.iteritems())
return zc.buildout.easy_install.scripts(
[(filename, module, function)], self._ws, sys.executable, [(filename, module, function)], self._ws, sys.executable,
path, arguments=arguments)[0] path, arguments=', '.join(args))[0]
return script
def createWrapper(self, name, command, parameters, comments=[], def createWrapper(self, path, args, env=None, **kw):
parameters_extra=False, environment=None, """Create a wrapper script for process replacement"""
pidfile=None, reserve_cpu=False assert args
): if kw:
""" return self.createPythonScript(path,
Creates a shell script for process replacement. 'slapos.recipe.librecipe.execute.generic_exec',
Takes care of quoting. (args, env) if env else (args,), kw)
Takes care of #! line limitation when the wrapped command is a script.
if pidfile parameter is specified, then it will make the wrapper a singleton,
accepting to run only if no other instance is running.
:param reserve_cpu: bool, try to reserve one core for the `command` # Simple case: creates a basic shell script for process replacement.
""" # This must be kept minimal to avoid code duplication with generic_exec.
# In particular, do not implement workaround for shebang size limitation
# here (note that this can't be done correctly with a POSIX shell, because
# the process can't be given a name).
lines = ['#!/bin/sh']
if env:
for k, v in sorted(env.iteritems()):
lines.append('export %s=%s' % (k, shlex.quote(v)))
lines.append('exec')
lines = [ '#!/bin/sh' ] args = map(shlex.quote, args)
args.append('"$@"')
if comments: for arg in args:
lines += '# ', '\n# '.join(comments), '\n'
lines.append('COMMAND=' + shlex.quote(command))
for key in environment or ():
lines.append('export %s=%s' % (key, environment[key]))
if pidfile:
lines.append(dedent("""
# Check for other instances
pidfile=%s
if [ -s $pidfile ]; then
if pid=`pgrep -F $pidfile -f "$COMMAND" 2>/dev/null`; then
echo "Already running with pid $pid."
exit 1
fi
fi
echo $$ > $pidfile""" % shlex.quote(pidfile)))
if reserve_cpu:
# if the CGROUPS cpuset is available (and prepared by slap format)
# request an exclusive CPU core for this process
lines.append(dedent("""
# put own PID into waiting list for exclusive CPU-core access
echo $$ >> ~/.slapos-cpu-exclusive
"""))
lines.append(dedent('''
# If the wrapped command uses a shebang, execute the referenced
# executable passing the script path as first argument.
# This is to workaround the limitation of 127 characters in #!
[ ! -f "$COMMAND" ] || {
[ "`head -c2`" != "#!" ] || read -r EXE ARG
} < "$COMMAND"
exec $EXE ${ARG:+"$ARG"} "$COMMAND"'''))
parameters = map(shlex.quote, parameters)
if parameters_extra:
# pass-through further parameters
parameters.append('"$@"')
for param in parameters:
if len(lines[-1]) < 40: if len(lines[-1]) < 40:
lines[-1] += ' ' + param lines[-1] += ' ' + arg
else: else:
lines[-1] += ' \\' lines[-1] += ' \\'
lines.append('\t' + param) lines.append('\t' + arg)
lines.append('') lines.append('')
return self.createFile(name, '\n'.join(lines), 0700) return self.createFile(path, '\n'.join(lines), 0700)
def createDirectory(self, parent, name, mode=0700): def createDirectory(self, parent, name, mode=0700):
path = os.path.join(parent, name) path = os.path.join(parent, name)
......
...@@ -46,10 +46,10 @@ class Recipe(GenericBaseRecipe): ...@@ -46,10 +46,10 @@ class Recipe(GenericBaseRecipe):
state_file = self.options['state-file'] state_file = self.options['state-file']
logrotate = self.createPythonScript( logrotate = self.createWrapper(
self.options['wrapper'], self.options['wrapper'],
'slapos.recipe.librecipe.execute.execute', (self.options['logrotate-binary'],
[self.options['logrotate-binary'], '-s', state_file, logrotate_conf_file, ] '-s', state_file, logrotate_conf_file),
) )
return [logrotate, logrotate_conf_file] return [logrotate, logrotate_conf_file]
......
##############################################################################
#
# Copyright (c) 2010 Vifib SARL and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
from slapos.recipe.librecipe import BaseSlapRecipe
import hashlib
import os
import pkg_resources
import sys
import zc.buildout
import ConfigParser
class Recipe(BaseSlapRecipe):
def getTemplateFilename(self, template_name):
return pkg_resources.resource_filename(__name__,
'template/%s' % template_name)
def _install(self):
self.path_list = []
self.requirements, self.ws = self.egg.working_set()
# XXX-Cedric : add logrotate?
self.cron_d = self.installCrond()
memcached_conf = self.installMemcached(ip=self.getLocalIPv4Address(),
port=11000)
ca_conf = self.installCertificateAuthority()
key, certificate = self.requestCertificate('Memcached')
stunnel_conf = self.installStunnel(self.getGlobalIPv6Address(),
self.getLocalIPv4Address(), 12345, memcached_conf['memcached_port'],
certificate, key, ca_conf['ca_crl'],
ca_conf['certificate_authority_path'])
self.linkBinary()
self.setConnectionDict(dict(
stunnel_ip = stunnel_conf['public_ip'],
stunnel_port = stunnel_conf['public_port'],
))
return self.path_list
def linkBinary(self):
"""Links binaries to instance's bin directory for easier exposal"""
for linkline in self.options.get('link_binary_list', '').splitlines():
if not linkline:
continue
target = linkline.split()
if len(target) == 1:
target = target[0]
path, linkname = os.path.split(target)
else:
linkname = target[1]
target = target[0]
link = os.path.join(self.bin_directory, linkname)
if os.path.lexists(link):
if not os.path.islink(link):
raise zc.buildout.UserError(
'Target link already %r exists but it is not link' % link)
os.unlink(link)
os.symlink(target, link)
self.logger.debug('Created link %r -> %r' % (link, target))
self.path_list.append(link)
def installCrond(self):
timestamps = self.createDataDirectory('cronstamps')
cron_output = os.path.join(self.log_directory, 'cron-output')
self._createDirectory(cron_output)
catcher = zc.buildout.easy_install.scripts([('catchcron',
__name__ + '.catdatefile', 'catdatefile')], self.ws, sys.executable,
self.bin_directory, arguments=[cron_output])[0]
self.path_list.append(catcher)
cron_d = os.path.join(self.etc_directory, 'cron.d')
crontabs = os.path.join(self.etc_directory, 'crontabs')
self._createDirectory(cron_d)
self._createDirectory(crontabs)
# Use execute from erp5.
wrapper = zc.buildout.easy_install.scripts([('crond',
'slapos.recipe.librecipe.execute', 'execute')], self.ws, sys.executable,
self.wrapper_directory, arguments=[
self.options['dcrond_binary'].strip(), '-s', cron_d, '-c', crontabs,
'-t', timestamps, '-f', '-l', '5', '-M', catcher]
)[0]
self.path_list.append(wrapper)
return cron_d
def installLogrotate(self):
"""Installs logortate main configuration file and registers its to cron"""
logrotate_d = os.path.abspath(os.path.join(self.etc_directory,
'logrotate.d'))
self._createDirectory(logrotate_d)
logrotate_backup = self.createBackupDirectory('logrotate')
logrotate_conf = self.createConfigurationFile("logrotate.conf",
"include %s" % logrotate_d)
logrotate_cron = os.path.join(self.cron_d, 'logrotate')
state_file = os.path.join(self.data_root_directory, 'logrotate.status')
open(logrotate_cron, 'w').write('0 0 * * * %s -s %s %s' %
(self.options['logrotate_binary'], state_file, logrotate_conf))
self.path_list.extend([logrotate_d, logrotate_conf, logrotate_cron])
return logrotate_d, logrotate_backup
def registerLogRotation(self, name, log_file_list, postrotate_script):
"""Register new log rotation requirement"""
open(os.path.join(self.logrotate_d, name), 'w').write(
self.substituteTemplate(self.getTemplateFilename(
'logrotate_entry.in'),
dict(file_list=' '.join(['"'+q+'"' for q in log_file_list]),
postrotate=postrotate_script, olddir=self.logrotate_backup)))
def installCertificateAuthority(self, ca_country_code='XX',
ca_email='xx@example.com', ca_state='State', ca_city='City',
ca_company='Company'):
backup_path = self.createBackupDirectory('ca')
self.ca_dir = os.path.join(self.data_root_directory, 'ca')
self._createDirectory(self.ca_dir)
self.ca_request_dir = os.path.join(self.ca_dir, 'requests')
self._createDirectory(self.ca_request_dir)
config = dict(ca_dir=self.ca_dir, request_dir=self.ca_request_dir)
self.ca_private = os.path.join(self.ca_dir, 'private')
self.ca_certs = os.path.join(self.ca_dir, 'certs')
self.ca_crl = os.path.join(self.ca_dir, 'crl')
self.ca_newcerts = os.path.join(self.ca_dir, 'newcerts')
self.ca_key_ext = '.key'
self.ca_crt_ext = '.crt'
for d in [self.ca_private, self.ca_crl, self.ca_newcerts, self.ca_certs]:
self._createDirectory(d)
for f in ['crlnumber', 'serial']:
if not os.path.exists(os.path.join(self.ca_dir, f)):
open(os.path.join(self.ca_dir, f), 'w').write('01')
if not os.path.exists(os.path.join(self.ca_dir, 'index.txt')):
open(os.path.join(self.ca_dir, 'index.txt'), 'w').write('')
openssl_configuration = os.path.join(self.ca_dir, 'openssl.cnf')
config.update(
working_directory=self.ca_dir,
country_code=ca_country_code,
state=ca_state,
city=ca_city,
company=ca_company,
email_address=ca_email,
)
self._writeFile(openssl_configuration, pkg_resources.resource_string(
__name__, 'template/openssl.cnf.ca.in') % config)
self.path_list.extend(zc.buildout.easy_install.scripts([
('certificate_authority',
__name__ + '.certificate_authority', 'runCertificateAuthority')],
self.ws, sys.executable, self.wrapper_directory, arguments=[dict(
openssl_configuration=openssl_configuration,
openssl_binary=self.options['openssl_binary'],
certificate=os.path.join(self.ca_dir, 'cacert.pem'),
key=os.path.join(self.ca_private, 'cakey.pem'),
crl=os.path.join(self.ca_crl),
request_dir=self.ca_request_dir
)]))
# configure backup
backup_cron = os.path.join(self.cron_d, 'ca_rdiff_backup')
open(backup_cron, 'w').write(
'''0 0 * * * %(rdiff_backup)s %(source)s %(destination)s'''%dict(
rdiff_backup=self.options['rdiff_backup_binary'],
source=self.ca_dir,
destination=backup_path))
self.path_list.append(backup_cron)
return dict(
ca_certificate=os.path.join(config['ca_dir'], 'cacert.pem'),
ca_crl=os.path.join(config['ca_dir'], 'crl'),
certificate_authority_path=config['ca_dir']
)
def requestCertificate(self, name):
hash = hashlib.sha512(name).hexdigest()
key = os.path.join(self.ca_private, hash + self.ca_key_ext)
certificate = os.path.join(self.ca_certs, hash + self.ca_crt_ext)
parser = ConfigParser.RawConfigParser()
parser.add_section('certificate')
parser.set('certificate', 'name', name)
parser.set('certificate', 'key_file', key)
parser.set('certificate', 'certificate_file', certificate)
parser.write(open(os.path.join(self.ca_request_dir, hash), 'w'))
return key, certificate
def installStunnel(self, public_ip, private_ip, public_port, private_port,
ca_certificate, key, ca_crl, ca_path):
"""Installs stunnel"""
template_filename = self.getTemplateFilename('stunnel.conf.in')
log = os.path.join(self.log_directory, 'stunnel.log')
pid_file = os.path.join(self.run_directory, 'stunnel.pid')
stunnel_conf = dict(
public_ip=public_ip,
private_ip=private_ip,
public_port=public_port,
pid_file=pid_file,
log=log,
cert = ca_certificate,
key = key,
ca_crl = ca_crl,
ca_path = ca_path,
private_port = private_port,
)
stunnel_conf_path = self.createConfigurationFile("stunnel.conf",
self.substituteTemplate(template_filename,
stunnel_conf))
wrapper = zc.buildout.easy_install.scripts([('stunnel',
'slapos.recipe.librecipe.execute', 'execute_wait')], self.ws,
sys.executable, self.wrapper_directory, arguments=[
[self.options['stunnel_binary'].strip(), stunnel_conf_path],
[ca_certificate, key]]
)[0]
self.path_list.append(wrapper)
return stunnel_conf
def installMemcached(self, ip, port):
config = dict(
memcached_binary=self.options['memcached_binary'],
memcached_ip=ip,
memcached_port=port,
)
self.path_list.append(self.createRunningWrapper('memcached',
self.substituteTemplate(self.getTemplateFilename('memcached.in'),
config)))
return dict(memcached_url='%s:%s' %
(config['memcached_ip'], config['memcached_port']),
memcached_ip=config['memcached_ip'],
memcached_port=config['memcached_port'])
import os
import subprocess
import time
import ConfigParser
def popenCommunicate(command_list, input=None):
subprocess_kw = dict(stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
if input is not None:
subprocess_kw.update(stdin=subprocess.PIPE)
popen = subprocess.Popen(command_list, **subprocess_kw)
result = popen.communicate(input)[0]
if popen.returncode is None:
popen.kill()
if popen.returncode != 0:
raise ValueError('Issue during calling %r, result was:\n%s' % (
command_list, result))
return result
class CertificateAuthority:
def __init__(self, key, certificate, openssl_binary,
openssl_configuration, request_dir):
self.key = key
self.certificate = certificate
self.openssl_binary = openssl_binary
self.openssl_configuration = openssl_configuration
self.request_dir = request_dir
def checkAuthority(self):
file_list = [ self.key, self.certificate ]
ca_ready = True
for f in file_list:
if not os.path.exists(f):
ca_ready = False
break
if ca_ready:
return
for f in file_list:
if os.path.exists(f):
os.unlink(f)
try:
# no CA, let us create new one
popenCommunicate([self.openssl_binary, 'req', '-nodes', '-config',
self.openssl_configuration, '-new', '-x509', '-extensions',
'v3_ca', '-keyout', self.key, '-out', self.certificate,
'-days', '10950'], 'Automatic Certificate Authority\n')
except:
try:
for f in file_list:
if os.path.exists(f):
os.unlink(f)
except:
# do not raise during cleanup
pass
raise
def _checkCertificate(self, common_name, key, certificate):
file_list = [key, certificate]
ready = True
for f in file_list:
if not os.path.exists(f):
ready = False
break
if ready:
return False
for f in file_list:
if os.path.exists(f):
os.unlink(f)
csr = certificate + '.csr'
try:
popenCommunicate([self.openssl_binary, 'req', '-config',
self.openssl_configuration, '-nodes', '-new', '-keyout',
key, '-out', csr, '-days', '3650'],
common_name + '\n')
try:
popenCommunicate([self.openssl_binary, 'ca', '-batch', '-config',
self.openssl_configuration, '-out', certificate,
'-infiles', csr])
finally:
if os.path.exists(csr):
os.unlink(csr)
except:
try:
for f in file_list:
if os.path.exists(f):
os.unlink(f)
except:
# do not raise during cleanup
pass
raise
else:
return True
def checkRequestDir(self):
for request_file in os.listdir(self.request_dir):
parser = ConfigParser.RawConfigParser()
parser.readfp(open(os.path.join(self.request_dir, request_file), 'r'))
if self._checkCertificate(parser.get('certificate', 'name'),
parser.get('certificate', 'key_file'), parser.get('certificate',
'certificate_file')):
print 'Created certificate %r' % parser.get('certificate', 'name')
def runCertificateAuthority(args):
ca_conf = args[0]
ca = CertificateAuthority(ca_conf['key'], ca_conf['certificate'],
ca_conf['openssl_binary'], ca_conf['openssl_configuration'],
ca_conf['request_dir'])
while True:
ca.checkAuthority()
ca.checkRequestDir()
time.sleep(60)
#!/bin/sh
exec %(memcached_binary)s -p %(memcached_port)s -U %(memcached_port)s -l %(memcached_ip)s
#
# OpenSSL example configuration file.
# This is mostly being used for generation of certificate requests.
#
# This definition stops the following lines choking if HOME isn't
# defined.
HOME = .
RANDFILE = $ENV::HOME/.rnd
# Extra OBJECT IDENTIFIER info:
#oid_file = $ENV::HOME/.oid
oid_section = new_oids
# To use this configuration file with the "-extfile" option of the
# "openssl x509" utility, name here the section containing the
# X.509v3 extensions to use:
# extensions =
# (Alternatively, use a configuration file that has only
# X.509v3 extensions in its main [= default] section.)
[ new_oids ]
# We can add new OIDs in here for use by 'ca', 'req' and 'ts'.
# Add a simple OID like this:
# testoid1=1.2.3.4
# Or use config file substitution like this:
# testoid2=${testoid1}.5.6
# Policies used by the TSA examples.
tsa_policy1 = 1.2.3.4.1
tsa_policy2 = 1.2.3.4.5.6
tsa_policy3 = 1.2.3.4.5.7
####################################################################
[ ca ]
default_ca = CA_default # The default ca section
####################################################################
[ CA_default ]
dir = %(working_directory)s # Where everything is kept
certs = $dir/certs # Where the issued certs are kept
crl_dir = $dir/crl # Where the issued crl are kept
database = $dir/index.txt # database index file.
#unique_subject = no # Set to 'no' to allow creation of
# several ctificates with same subject.
new_certs_dir = $dir/newcerts # default place for new certs.
certificate = $dir/cacert.pem # The CA certificate
serial = $dir/serial # The current serial number
crlnumber = $dir/crlnumber # the current crl number
# must be commented out to leave a V1 CRL
crl = $dir/crl.pem # The current CRL
private_key = $dir/private/cakey.pem # The private key
RANDFILE = $dir/private/.rand # private random number file
x509_extensions = usr_cert # The extentions to add to the cert
# Comment out the following two lines for the "traditional"
# (and highly broken) format.
name_opt = ca_default # Subject Name options
cert_opt = ca_default # Certificate field options
# Extension copying option: use with caution.
# copy_extensions = copy
# Extensions to add to a CRL. Note: Netscape communicator chokes on V2 CRLs
# so this is commented out by default to leave a V1 CRL.
# crlnumber must also be commented out to leave a V1 CRL.
# crl_extensions = crl_ext
default_days = 3650 # how long to certify for
default_crl_days= 30 # how long before next CRL
default_md = default # use public key default MD
preserve = no # keep passed DN ordering
# A few difference way of specifying how similar the request should look
# For type CA, the listed attributes must be the same, and the optional
# and supplied fields are just that :-)
policy = policy_match
# For the CA policy
[ policy_match ]
countryName = match
stateOrProvinceName = match
organizationName = match
organizationalUnitName = optional
commonName = supplied
emailAddress = optional
# For the 'anything' policy
# At this point in time, you must list all acceptable 'object'
# types.
[ policy_anything ]
countryName = optional
stateOrProvinceName = optional
localityName = optional
organizationName = optional
organizationalUnitName = optional
commonName = supplied
emailAddress = optional
####################################################################
[ req ]
default_bits = 2048
default_md = sha1
default_keyfile = privkey.pem
distinguished_name = req_distinguished_name
#attributes = req_attributes
x509_extensions = v3_ca # The extentions to add to the self signed cert
# Passwords for private keys if not present they will be prompted for
# input_password = secret
# output_password = secret
# This sets a mask for permitted string types. There are several options.
# default: PrintableString, T61String, BMPString.
# pkix : PrintableString, BMPString (PKIX recommendation before 2004)
# utf8only: only UTF8Strings (PKIX recommendation after 2004).
# nombstr : PrintableString, T61String (no BMPStrings or UTF8Strings).
# MASK:XXXX a literal mask value.
# WARNING: ancient versions of Netscape crash on BMPStrings or UTF8Strings.
string_mask = utf8only
# req_extensions = v3_req # The extensions to add to a certificate request
[ req_distinguished_name ]
countryName = Country Name (2 letter code)
countryName_value = %(country_code)s
countryName_min = 2
countryName_max = 2
stateOrProvinceName = State or Province Name (full name)
stateOrProvinceName_value = %(state)s
localityName = Locality Name (eg, city)
localityName_value = %(city)s
0.organizationName = Organization Name (eg, company)
0.organizationName_value = %(company)s
# we can do this but it is not needed normally :-)
#1.organizationName = Second Organization Name (eg, company)
#1.organizationName_default = World Wide Web Pty Ltd
commonName = Common Name (eg, your name or your server\'s hostname)
commonName_max = 64
emailAddress = Email Address
emailAddress_value = %(email_address)s
emailAddress_max = 64
# SET-ex3 = SET extension number 3
#[ req_attributes ]
#challengePassword = A challenge password
#challengePassword_min = 4
#challengePassword_max = 20
#
#unstructuredName = An optional company name
[ usr_cert ]
# These extensions are added when 'ca' signs a request.
# This goes against PKIX guidelines but some CAs do it and some software
# requires this to avoid interpreting an end user certificate as a CA.
basicConstraints=CA:FALSE
# Here are some examples of the usage of nsCertType. If it is omitted
# the certificate can be used for anything *except* object signing.
# This is OK for an SSL server.
# nsCertType = server
# For an object signing certificate this would be used.
# nsCertType = objsign
# For normal client use this is typical
# nsCertType = client, email
# and for everything including object signing:
# nsCertType = client, email, objsign
# This is typical in keyUsage for a client certificate.
# keyUsage = nonRepudiation, digitalSignature, keyEncipherment
# This will be displayed in Netscape's comment listbox.
nsComment = "OpenSSL Generated Certificate"
# PKIX recommendations harmless if included in all certificates.
subjectKeyIdentifier=hash
authorityKeyIdentifier=keyid,issuer
# This stuff is for subjectAltName and issuerAltname.
# Import the email address.
# subjectAltName=email:copy
# An alternative to produce certificates that aren't
# deprecated according to PKIX.
# subjectAltName=email:move
# Copy subject details
# issuerAltName=issuer:copy
#nsCaRevocationUrl = http://www.domain.dom/ca-crl.pem
#nsBaseUrl
#nsRevocationUrl
#nsRenewalUrl
#nsCaPolicyUrl
#nsSslServerName
# This is required for TSA certificates.
# extendedKeyUsage = critical,timeStamping
[ v3_req ]
# Extensions to add to a certificate request
basicConstraints = CA:FALSE
keyUsage = nonRepudiation, digitalSignature, keyEncipherment
[ v3_ca ]
# Extensions for a typical CA
# PKIX recommendation.
subjectKeyIdentifier=hash
authorityKeyIdentifier=keyid:always,issuer
# This is what PKIX recommends but some broken software chokes on critical
# extensions.
#basicConstraints = critical,CA:true
# So we do this instead.
basicConstraints = CA:true
# Key usage: this is typical for a CA certificate. However since it will
# prevent it being used as an test self-signed certificate it is best
# left out by default.
# keyUsage = cRLSign, keyCertSign
# Some might want this also
# nsCertType = sslCA, emailCA
# Include email address in subject alt name: another PKIX recommendation
# subjectAltName=email:copy
# Copy issuer details
# issuerAltName=issuer:copy
# DER hex encoding of an extension: beware experts only!
# obj=DER:02:03
# Where 'obj' is a standard or added object
# You can even override a supported extension:
# basicConstraints= critical, DER:30:03:01:01:FF
[ crl_ext ]
# CRL extensions.
# Only issuerAltName and authorityKeyIdentifier make any sense in a CRL.
# issuerAltName=issuer:copy
authorityKeyIdentifier=keyid:always
[ proxy_cert_ext ]
# These extensions should be added when creating a proxy certificate
# This goes against PKIX guidelines but some CAs do it and some software
# requires this to avoid interpreting an end user certificate as a CA.
basicConstraints=CA:FALSE
# Here are some examples of the usage of nsCertType. If it is omitted
# the certificate can be used for anything *except* object signing.
# This is OK for an SSL server.
# nsCertType = server
# For an object signing certificate this would be used.
# nsCertType = objsign
# For normal client use this is typical
# nsCertType = client, email
# and for everything including object signing:
# nsCertType = client, email, objsign
# This is typical in keyUsage for a client certificate.
# keyUsage = nonRepudiation, digitalSignature, keyEncipherment
# This will be displayed in Netscape's comment listbox.
nsComment = "OpenSSL Generated Certificate"
# PKIX recommendations harmless if included in all certificates.
subjectKeyIdentifier=hash
authorityKeyIdentifier=keyid,issuer
# This stuff is for subjectAltName and issuerAltname.
# Import the email address.
# subjectAltName=email:copy
# An alternative to produce certificates that aren't
# deprecated according to PKIX.
# subjectAltName=email:move
# Copy subject details
# issuerAltName=issuer:copy
#nsCaRevocationUrl = http://www.domain.dom/ca-crl.pem
#nsBaseUrl
#nsRevocationUrl
#nsRenewalUrl
#nsCaPolicyUrl
#nsSslServerName
# This really needs to be in place for it to be a proxy certificate.
proxyCertInfo=critical,language:id-ppl-anyLanguage,pathlen:3,policy:foo
####################################################################
[ tsa ]
default_tsa = tsa_config1 # the default TSA section
[ tsa_config1 ]
# These are used by the TSA reply generation only.
dir = /etc/pki/tls # TSA root directory
serial = $dir/tsaserial # The current serial number (mandatory)
crypto_device = builtin # OpenSSL engine to use for signing
signer_cert = $dir/tsacert.pem # The TSA signing certificate
# (optional)
certs = $dir/cacert.pem # Certificate chain to include in reply
# (optional)
signer_key = $dir/private/tsakey.pem # The TSA private key (optional)
default_policy = tsa_policy1 # Policy if request did not specify it
# (optional)
other_policies = tsa_policy2, tsa_policy3 # acceptable policies (optional)
digests = md5, sha1 # Acceptable message digests (mandatory)
accuracy = secs:1, millisecs:500, microsecs:100 # (optional)
clock_precision_digits = 0 # number of digits after dot. (optional)
ordering = yes # Is ordering defined for timestamps?
# (optional, default: no)
tsa_name = yes # Must the TSA name be included in the reply?
# (optional, default: no)
ess_cert_id_chain = no # Must the ESS cert id chain be included?
# (optional, default: no)
foreground = yes
output = %(log)s
pid = %(pid_file)s
syslog = no
CApath = %(ca_path)s
key = %(key)s
CRLpath = %(ca_crl)s
cert = %(cert)s
[service]
accept = %(public_ip)s:%(public_port)s
connect = %(private_ip)s:%(private_port)s
...@@ -203,11 +203,10 @@ Include conf/extra/httpd-autoindex.conf ...@@ -203,11 +203,10 @@ Include conf/extra/httpd-autoindex.conf
services_dir = self.options['services_dir'] services_dir = self.options['services_dir']
httpd_wrapper = self.createPythonScript( httpd_wrapper = self.createWrapper(
os.path.join(services_dir, 'httpd_wrapper'), os.path.join(services_dir, 'httpd_wrapper'),
'slapos.recipe.librecipe.execute.execute', (self.options['httpd_binary'],
[self.options['httpd_binary'], '-f', self.options['httpd_conf'], '-f', self.options['httpd_conf'], '-DFOREGROUND'),
'-DFOREGROUND']
) )
path_list.append(httpd_wrapper) path_list.append(httpd_wrapper)
...@@ -220,19 +219,17 @@ Include conf/extra/httpd-autoindex.conf ...@@ -220,19 +219,17 @@ Include conf/extra/httpd-autoindex.conf
site_perl_bin = os.path.join(self.options['site_perl'], 'bin') site_perl_bin = os.path.join(self.options['site_perl'], 'bin')
mioga_conf_path = os.path.join(mioga_base, 'conf', 'Mioga.conf') mioga_conf_path = os.path.join(mioga_base, 'conf', 'Mioga.conf')
notifier_wrapper = self.createPythonScript( notifier_wrapper = self.createWrapper(
os.path.join(services_dir, 'notifier'), os.path.join(services_dir, 'notifier'),
'slapos.recipe.librecipe.execute.execute', (os.path.join(site_perl_bin, 'notifier.pl'),
[ os.path.join(site_perl_bin, 'notifier.pl'), mioga_conf_path),
mioga_conf_path ]
) )
path_list.append(notifier_wrapper) path_list.append(notifier_wrapper)
searchengine_wrapper = self.createPythonScript( searchengine_wrapper = self.createWrapper(
os.path.join(services_dir, 'searchengine'), os.path.join(services_dir, 'searchengine'),
'slapos.recipe.librecipe.execute.execute', (os.path.join(site_perl_bin, 'searchengine.pl'),
[ os.path.join(site_perl_bin, 'searchengine.pl'), mioga_conf_path),
mioga_conf_path ]
) )
path_list.append(searchengine_wrapper) path_list.append(searchengine_wrapper)
......
...@@ -24,7 +24,7 @@ ...@@ -24,7 +24,7 @@
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
# #
############################################################################## ##############################################################################
import subprocess import os
from slapos.recipe.librecipe import GenericBaseRecipe from slapos.recipe.librecipe import GenericBaseRecipe
...@@ -58,14 +58,14 @@ def do_export(args): ...@@ -58,14 +58,14 @@ def do_export(args):
cmd.extend(['-o', args['directory']]) cmd.extend(['-o', args['directory']])
subprocess.check_call(cmd) os.execv(cmd[0], cmd)
def do_import(args): def do_import(args):
cmd = _mydumper_base_cmd(**args) cmd = _mydumper_base_cmd(**args)
cmd.append('--overwrite-tables') cmd.append('--overwrite-tables')
cmd.extend(['-d', args['directory']]) cmd.extend(['-d', args['directory']])
subprocess.check_call(cmd) os.execv(cmd[0], cmd)
class Recipe(GenericBaseRecipe): class Recipe(GenericBaseRecipe):
...@@ -95,9 +95,5 @@ class Recipe(GenericBaseRecipe): ...@@ -95,9 +95,5 @@ class Recipe(GenericBaseRecipe):
config['compression'] = self.optionIsTrue('compression', default=False) config['compression'] = self.optionIsTrue('compression', default=False)
config['rows'] = self.options.get('rows') config['rows'] = self.options.get('rows')
wrapper = self.createPythonScript(name=self.options['wrapper'], return self.createPythonScript(self.options['wrapper'],
absolute_function = '%s.%s' % (__name__, function.func_name), '%s.%s' % (function.__module__, function.__name__), (config,))
arguments=config)
return [wrapper]
...@@ -53,11 +53,10 @@ class Recipe(GenericBaseRecipe): ...@@ -53,11 +53,10 @@ class Recipe(GenericBaseRecipe):
mysql_binary = self.options['mysql-binary'] mysql_binary = self.options['mysql-binary']
socket = self.options['socket'], socket = self.options['socket'],
post_rotate = self.createPythonScript( post_rotate = self.createWrapper(
self.options['logrotate-post'], self.options['logrotate-post'],
'slapos.recipe.librecipe.execute.execute', (mysql_binary, '--no-defaults', '-B', '-u', 'root',
[mysql_binary, '--no-defaults', '-B', '-u', 'root', '--socket=%s' % socket, '-e', '--socket=%s' % socket, '-e', 'FLUSH LOGS'),
'FLUSH LOGS']
) )
path_list.append(post_rotate) path_list.append(post_rotate)
...@@ -85,12 +84,12 @@ class Recipe(GenericBaseRecipe): ...@@ -85,12 +84,12 @@ class Recipe(GenericBaseRecipe):
mysql_update = self.createPythonScript( mysql_update = self.createPythonScript(
self.options['update-wrapper'], self.options['update-wrapper'],
'%s.mysql.updateMysql' % __name__, '%s.mysql.updateMysql' % __name__,
dict( (dict(
mysql_script=mysql_script, mysql_script=mysql_script,
mysql_binary=mysql_binary, mysql_binary=mysql_binary,
mysql_upgrade_binary=mysql_upgrade_binary, mysql_upgrade_binary=mysql_upgrade_binary,
socket=socket, socket=socket,
) ),)
) )
path_list.append(mysql_update) path_list.append(mysql_update)
...@@ -98,7 +97,7 @@ class Recipe(GenericBaseRecipe): ...@@ -98,7 +97,7 @@ class Recipe(GenericBaseRecipe):
mysqld = self.createPythonScript( mysqld = self.createPythonScript(
self.options['wrapper'], self.options['wrapper'],
'%s.mysql.runMysql' % __name__, '%s.mysql.runMysql' % __name__,
dict( (dict(
mysql_install_binary=self.options['mysql-install-binary'], mysql_install_binary=self.options['mysql-install-binary'],
mysqld_binary=mysqld_binary, mysqld_binary=mysqld_binary,
data_directory=mysql_conf['data_directory'], data_directory=mysql_conf['data_directory'],
...@@ -106,7 +105,7 @@ class Recipe(GenericBaseRecipe): ...@@ -106,7 +105,7 @@ class Recipe(GenericBaseRecipe):
socket=socket, socket=socket,
configuration_file=mysql_conf_file, configuration_file=mysql_conf_file,
cwd=self.options['mysql-base-directory'], cwd=self.options['mysql-base-directory'],
) ),)
) )
path_list.append(mysqld) path_list.append(mysqld)
...@@ -115,12 +114,12 @@ class Recipe(GenericBaseRecipe): ...@@ -115,12 +114,12 @@ class Recipe(GenericBaseRecipe):
backup_script = self.createPythonScript( backup_script = self.createPythonScript(
self.options['backup-script'], self.options['backup-script'],
'%s.do_backup' % __name__, '%s.do_backup' % __name__,
dict( (dict(
mydumper_binary=self.options['mydumper-binary'], mydumper_binary=self.options['mydumper-binary'],
database=mysql_conf['mysql_database'], database=mysql_conf['mysql_database'],
socket=mysql_conf['socket'], socket=mysql_conf['socket'],
backup_directory=self.options['backup-directory'] backup_directory=self.options['backup-directory']
), ),)
) )
path_list.append(backup_script) path_list.append(backup_script)
...@@ -129,7 +128,7 @@ class Recipe(GenericBaseRecipe): ...@@ -129,7 +128,7 @@ class Recipe(GenericBaseRecipe):
recovering_script = self.createPythonScript( recovering_script = self.createPythonScript(
self.options['recovering-wrapper'], self.options['recovering-wrapper'],
'%s.import_dump' % __name__, '%s.import_dump' % __name__,
{ ({
'lock_file': os.path.join(self.work_directory, 'lock_file': os.path.join(self.work_directory,
'import_done'), 'import_done'),
'database': mysql_conf['mysql_database'], 'database': mysql_conf['mysql_database'],
...@@ -140,7 +139,7 @@ class Recipe(GenericBaseRecipe): ...@@ -140,7 +139,7 @@ class Recipe(GenericBaseRecipe):
'local_directory': self.mysql_backup_directory, 'local_directory': self.mysql_backup_directory,
'dump_name': dump_filename, 'dump_name': dump_filename,
'zcat_binary': self.options['zcat-binary'], 'zcat_binary': self.options['zcat-binary'],
} },)
) )
path_list.append(recovering_script) path_list.append(recovering_script)
......
...@@ -42,7 +42,8 @@ class NeoBaseRecipe(GenericBaseRecipe): ...@@ -42,7 +42,8 @@ class NeoBaseRecipe(GenericBaseRecipe):
# Only then can this recipe start succeeding and actually doing anything # Only then can this recipe start succeeding and actually doing anything
# useful, as per NEO deploying constraints. # useful, as per NEO deploying constraints.
raise UserError('"masters" parameter is mandatory') raise UserError('"masters" parameter is mandatory')
option_list = [ args = [
options['binary'],
# Keep the -l option first, as expected by logrotate snippets. # Keep the -l option first, as expected by logrotate snippets.
'-l', options['logfile'], '-l', options['logfile'],
'-m', options['masters'], '-m', options['masters'],
...@@ -53,17 +54,13 @@ class NeoBaseRecipe(GenericBaseRecipe): ...@@ -53,17 +54,13 @@ class NeoBaseRecipe(GenericBaseRecipe):
] ]
if options['ssl']: if options['ssl']:
etc = os.path.join(self.buildout['buildout']['directory'], 'etc', '') etc = os.path.join(self.buildout['buildout']['directory'], 'etc', '')
option_list += ( args += (
'--ca', etc + 'ca.crt', '--ca', etc + 'ca.crt',
'--cert', etc + 'neo.crt', '--cert', etc + 'neo.crt',
'--key', etc + 'neo.key', '--key', etc + 'neo.key',
) )
option_list.extend(self._getOptionList()) args += self._getOptionList()
return [self.createWrapper( return self.createWrapper(options['wrapper'], args)
options['wrapper'],
options['binary'],
option_list
)]
def _getBindingAddress(self): def _getBindingAddress(self):
options = self.options options = self.options
......
...@@ -32,19 +32,15 @@ class Recipe(GenericBaseRecipe): ...@@ -32,19 +32,15 @@ class Recipe(GenericBaseRecipe):
def install(self): def install(self):
options = self.options options = self.options
script = self.createWrapper(name=options['wrapper'], # Script that execute the callback(s) upon receiving a notification.
command=options['server-binary'], return self.createWrapper(options['wrapper'],
parameters=[ (options['server-binary'],
'--callbacks', options['callbacks'], '--callbacks', options['callbacks'],
'--feeds', options['feeds'], '--feeds', options['feeds'],
'--equeue-socket', options['equeue-socket'], '--equeue-socket', options['equeue-socket'],
options['host'], options['port'] options['host'], options['port']
], ),
comments=[ )
'',
'Upon receiving a notification, execute the callback(s).',
''])
return [script]
class Callback(GenericBaseRecipe): class Callback(GenericBaseRecipe):
...@@ -80,35 +76,32 @@ class Notify(GenericBaseRecipe): ...@@ -80,35 +76,32 @@ class Notify(GenericBaseRecipe):
# Just a touch # Just a touch
open(log, 'w').close() open(log, 'w').close()
parameters = [ cmd = [notifier_binary,
'-l', log, '-l', log,
'--title', title, '--title', title,
'--feed', feed_url, '--feed', feed_url,
'--max-run', str(max_run), '--max-run', str(max_run),
'--notification-url', '--notification-url',
] ]
parameters.extend(notification_url.split(' ')) cmd += notification_url.split(' ')
parameters.extend(['--executable', executable]) cmd += '--executable', executable
# For a more verbose mode, writing feed items for any action # For a more verbose mode, writing feed items for any action
instance_root_name = instance_root_name or self.options.get('instance-root-name', None) instance_root_name = instance_root_name or self.options.get('instance-root-name', None)
log_url = log_url or self.options.get('log-url', None) log_url = log_url or self.options.get('log-url', None)
status_item_directory = status_item_directory or self.options.get('status-item-directory', None) status_item_directory = status_item_directory or self.options.get('status-item-directory', None)
if instance_root_name and log_url and status_item_directory: if instance_root_name and log_url and status_item_directory:
parameters.extend([ cmd += (
'--instance-root-name', instance_root_name, '--instance-root-name', instance_root_name,
'--log-url', log_url, '--log-url', log_url,
'--status-item-directory', status_item_directory, '--status-item-directory', status_item_directory,
]) )
return self.createWrapper(name=wrapper, kw = {}
command=notifier_binary, if pidfile:
parameters=parameters, kw['pidfile'] = pidfile
pidfile=pidfile,
parameters_extra=True, # Script that call an executable and send notification(s).
comments=[ return self.createWrapper(wrapper, cmd, **kw)
'',
'Call an executable and send notification(s).',
''])
def install(self): def install(self):
......
...@@ -35,10 +35,9 @@ class Recipe(GenericBaseRecipe): ...@@ -35,10 +35,9 @@ class Recipe(GenericBaseRecipe):
""" """
def install(self): def install(self):
runner_path = self.createPythonScript( return self.createWrapper(
self.options['path'], self.options['path'],
'slapos.recipe.librecipe.execute.execute_wait', (
[[
self.options['websockify-path'], self.options['websockify-path'],
'--web', '--web',
self.options['novnc-location'], self.options['novnc-location'],
...@@ -47,8 +46,7 @@ class Recipe(GenericBaseRecipe): ...@@ -47,8 +46,7 @@ class Recipe(GenericBaseRecipe):
'--ssl-only', '--ssl-only',
'%s:%s' % (self.options['ip'], self.options['port']), '%s:%s' % (self.options['ip'], self.options['port']),
'%s:%s' % (self.options['vnc-ip'], self.options['vnc-port']), '%s:%s' % (self.options['vnc-ip'], self.options['vnc-port']),
], ),
[self.options['ssl-key-path'], self.options['ssl-cert-path']]], wait_list=(self.options['ssl-key-path'],
self.options['ssl-cert-path']),
) )
return [runner_path]
...@@ -39,30 +39,18 @@ from slapos.recipe.notifier import Callback ...@@ -39,30 +39,18 @@ from slapos.recipe.notifier import Callback
from slapos.recipe.librecipe import shlex from slapos.recipe.librecipe import shlex
def promise(args): def promise(ssh_client, user, host, port):
# Redirect output to /dev/null # Redirect output to /dev/null
with open("/dev/null") as _dev_null: with open(os.devnull) as _dev_null:
ssh = subprocess.Popen( ssh = subprocess.Popen(
[args['ssh_client'], '%(user)s@%(host)s' % args, '-p', '%(port)s' % args], (ssh_client, '%s@%s' % (user, host), '-p', str(port)),
stdin=subprocess.PIPE, stdout=_dev_null, stderr=None stdin=subprocess.PIPE, stdout=_dev_null)
) ssh.communicate('q' + chr(255) + chr(0) * 7)
if ssh.returncode:
# Rdiff Backup protocol quit command
quitcommand = 'q' + chr(255) + chr(0) * 7
ssh.stdin.write(quitcommand)
ssh.stdin.flush()
ssh.stdin.close()
ssh.wait()
if ssh.poll() is None:
return 1
if ssh.returncode != 0:
sys.stderr.write("SSH Connection failed\n") sys.stderr.write("SSH Connection failed\n")
return ssh.returncode return ssh.returncode
class Recipe(GenericSlapRecipe, Notify, Callback): class Recipe(GenericSlapRecipe, Notify, Callback):
def _options(self, options): def _options(self, options):
options['rdiff-backup-data-folder'] = "" options['rdiff-backup-data-folder'] = ""
...@@ -244,15 +232,11 @@ class Recipe(GenericSlapRecipe, Notify, Callback): ...@@ -244,15 +232,11 @@ class Recipe(GenericSlapRecipe, Notify, Callback):
print 'Processing PBS slave %s with type %s' % (slave_id, slave_type) print 'Processing PBS slave %s with type %s' % (slave_id, slave_type)
promise_path = os.path.join(self.options['promises-directory'], "ssh-to-%s" % slave_id) path_list.append(self.createPythonScript(
promise_dict = dict(ssh_client=self.options['sshclient-binary'], os.path.join(self.options['promises-directory'], "ssh-to-%s" % slave_id),
user=parsed_url.username,
host=parsed_url.hostname,
port=parsed_url.port)
promise = self.createPythonScript(promise_path,
__name__ + '.promise', __name__ + '.promise',
promise_dict) (self.options['sshclient-binary'],
path_list.append(promise) parsed_url.username, parsed_url.hostname, parsed_url.port)))
# Create known_hosts file by default. # Create known_hosts file by default.
# In some case, we don't want to create it (case where we share IP mong partitions) # In some case, we don't want to create it (case where we share IP mong partitions)
...@@ -336,12 +320,11 @@ class Recipe(GenericSlapRecipe, Notify, Callback): ...@@ -336,12 +320,11 @@ class Recipe(GenericSlapRecipe, Notify, Callback):
else: else:
self.logger.info("Server mode") self.logger.info("Server mode")
wrapper = self.createWrapper(name=self.options['wrapper'], wrapper = self.createWrapper(self.options['wrapper'],
command=self.options['rdiffbackup-binary'], (self.options['rdiffbackup-binary'],
parameters=[
'--restrict', self.options['path'], '--restrict', self.options['path'],
'--server' '--server'
]) ))
path_list.append(wrapper) path_list.append(wrapper)
return path_list return path_list
...@@ -162,7 +162,6 @@ class Recipe(GenericBaseRecipe): ...@@ -162,7 +162,6 @@ class Recipe(GenericBaseRecipe):
return hash_string return hash_string
def install(self): def install(self):
path_list = []
token_save_path = os.path.join(self.options['conf-dir'], 'token.json') token_save_path = os.path.join(self.options['conf-dir'], 'token.json')
token_list_path = self.options['token-dir'] token_list_path = self.options['token-dir']
...@@ -190,20 +189,14 @@ class Recipe(GenericBaseRecipe): ...@@ -190,20 +189,14 @@ class Recipe(GenericBaseRecipe):
self.createFile(token_save_path, json.dumps(token_dict)) self.createFile(token_save_path, json.dumps(token_dict))
service_dict = dict(token_base_path=token_list_path, computer_dict = dict(partition_id=self.computer_partition_id,
token_json=token_save_path, computer_guid=self.computer_id,
partition_id=self.computer_partition_id, master_url=self.server_url,
computer_id=self.computer_id,
registry_url=registry_url,
server_url=self.server_url,
cert_file=self.cert_file, cert_file=self.cert_file,
key_file=self.key_file) key_file=self.key_file)
request_add = self.createPythonScript( return self.createPythonScript(
self.options['manager-wrapper'].strip(), self.options['manager-wrapper'].strip(),
'%s.re6stnet.manage' % __name__, service_dict __name__ + '.re6stnet.manage',
(registry_url, token_list_path, token_save_path, computer_dict)
) )
path_list.append(request_add)
return path_list
...@@ -17,9 +17,7 @@ logging.trace = logging.debug ...@@ -17,9 +17,7 @@ logging.trace = logging.debug
def loadJsonFile(path): def loadJsonFile(path):
if os.path.exists(path): if os.path.exists(path):
with open(path, 'r') as f: with open(path, 'r') as f:
content = f.read() return json.load(f)
return json.loads(content)
else:
return {} return {}
def writeFile(path, data): def writeFile(path, data):
...@@ -39,29 +37,25 @@ def updateFile(file_path, value): ...@@ -39,29 +37,25 @@ def updateFile(file_path, value):
return True return True
return False return False
def getComputerPartition(server_url, key_file, cert_file, computer_guid, partition_id): def getComputerPartition(master_url, key_file, cert_file,
computer_guid, partition_id):
slap = slapos.slap.slap() slap = slapos.slap.slap()
# Redeploy instance to update published information # Redeploy instance to update published information
slap.initializeConnection(server_url, slap.initializeConnection(master_url, key_file, cert_file)
key_file, return slap.registerComputerPartition(computer_guid, partition_id)
cert_file)
return slap.registerComputerPartition(computer_guid=computer_guid, def requestAddToken(client, token_base_path):
partition_id=partition_id)
def requestAddToken(client, base_token_path):
time.sleep(3) time.sleep(3)
path_list = [x for x in os.listdir(base_token_path) if x.endswith('.add')] path_list = [x for x in os.listdir(token_base_path) if x.endswith('.add')]
log.info("Searching tokens to add at %s and found %s." % (base_token_path, path_list)) log.info("Searching tokens to add at %s and found %s." % (token_base_path, path_list))
if not path_list: if not path_list:
log.info("No new token to add. Exiting...") log.info("No new token to add. Exiting...")
return return
for reference_key in path_list: for reference_key in path_list:
request_file = os.path.join(base_token_path, reference_key) request_file = os.path.join(token_base_path, reference_key)
token = readFile(request_file) token = readFile(request_file)
log.info("Including token %s for %s" % (token, reference_key)) log.info("Including token %s for %s" % (token, reference_key))
if token : if token :
...@@ -79,21 +73,21 @@ def requestAddToken(client, base_token_path): ...@@ -79,21 +73,21 @@ def requestAddToken(client, base_token_path):
# update information # update information
log.info("New token added for slave instance %s. Updating file status..." % log.info("New token added for slave instance %s. Updating file status..." %
reference) reference)
status_file = os.path.join(base_token_path, '%s.status' % reference) status_file = os.path.join(token_base_path, '%s.status' % reference)
updateFile(status_file, 'TOKEN_ADDED') updateFile(status_file, 'TOKEN_ADDED')
os.unlink(request_file) os.unlink(request_file)
else: else:
log.debug('Bad token. Request add token fail for %s...' % request_file) log.debug('Bad token. Request add token fail for %s...' % request_file)
def requestRemoveToken(client, base_token_path): def requestRemoveToken(client, token_base_path):
path_list = [x for x in os.listdir(base_token_path) if x.endswith('.remove')] path_list = [x for x in os.listdir(token_base_path) if x.endswith('.remove')]
if not path_list: if not path_list:
log.info("No token to delete. Exiting...") log.info("No token to delete. Exiting...")
return return
for reference_key in path_list: for reference_key in path_list:
request_file = os.path.join(base_token_path, reference_key) request_file = os.path.join(token_base_path, reference_key)
token = readFile(request_file) token = readFile(request_file)
if token : if token :
reference = reference_key.split('.')[0] reference = reference_key.split('.')[0]
...@@ -108,7 +102,7 @@ def requestRemoveToken(client, base_token_path): ...@@ -108,7 +102,7 @@ def requestRemoveToken(client, base_token_path):
continue continue
else: else:
# certificate is invalidated, it will be revoked # certificate is invalidated, it will be revoked
writeFile(os.path.join(base_token_path, '%s.revoke' % reference), '') writeFile(os.path.join(token_base_path, '%s.revoke' % reference), '')
if result in (True, 'True'): if result in (True, 'True'):
# update information # update information
...@@ -117,33 +111,17 @@ def requestRemoveToken(client, base_token_path): ...@@ -117,33 +111,17 @@ def requestRemoveToken(client, base_token_path):
if result in ['True', 'False']: if result in ['True', 'False']:
os.unlink(request_file) os.unlink(request_file)
status_file = os.path.join(base_token_path, '%s.status' % reference) status_file = os.path.join(token_base_path, '%s.status' % reference)
if os.path.exists(status_file): if os.path.exists(status_file):
os.unlink(status_file) os.unlink(status_file)
ipv6_file = os.path.join(base_token_path, '%s.ipv6' % reference) ipv6_file = os.path.join(token_base_path, '%s.ipv6' % reference)
if os.path.exists(ipv6_file): if os.path.exists(ipv6_file):
os.unlink(ipv6_file) os.unlink(ipv6_file)
else: else:
log.debug('Bad token. Request add token fail for %s...' % request_file) log.debug('Bad token. Request add token fail for %s...' % request_file)
def requestRevoqueCertificate(args): def checkService(client, token_base_path, token_json, computer_partition):
base_token_path = args['token_base_path']
path_list = [x for x in os.listdir(base_token_path) if x.endswith('.revoke')]
for reference_key in path_list:
reference = reference_key.split('.')[0]
if revokeByMail(args['registry_url'],
'%s@slapos' % reference.lower(),
args['db']):
os.unlink(os.path.join(base_token_path, reference_key))
log.info("Certificate revoked for slave instance %s." % reference)
return
log.info("Failed to revoke email for %s" % reference)
def checkService(client, base_token_path, token_json, computer_partition):
token_dict = loadJsonFile(token_json) token_dict = loadJsonFile(token_json)
updated = False updated = False
if not token_dict: if not token_dict:
...@@ -152,7 +130,7 @@ def checkService(client, base_token_path, token_json, computer_partition): ...@@ -152,7 +130,7 @@ def checkService(client, base_token_path, token_json, computer_partition):
# Check token status # Check token status
for slave_reference, token in token_dict.iteritems(): for slave_reference, token in token_dict.iteritems():
log.info("%s %s" % (slave_reference, token)) log.info("%s %s" % (slave_reference, token))
status_file = os.path.join(base_token_path, '%s.status' % slave_reference) status_file = os.path.join(token_base_path, '%s.status' % slave_reference)
if not os.path.exists(status_file): if not os.path.exists(status_file):
# This token is not added yet! # This token is not added yet!
log.info("Token %s dont exist yet." % status_file) log.info("Token %s dont exist yet." % status_file)
...@@ -206,31 +184,22 @@ def checkService(client, base_token_path, token_json, computer_partition): ...@@ -206,31 +184,22 @@ def checkService(client, base_token_path, token_json, computer_partition):
slave_reference, traceback.format_exc()) slave_reference, traceback.format_exc())
def manage(args, can_bang=True): def manage(registry_url, token_base_path, token_json,
computer_dict, can_bang=True):
computer_guid = args['computer_id']
partition_id = args['partition_id']
server_url = args['server_url']
key_file = args['key_file']
cert_file = args['cert_file']
client = registry.RegistryClient(args['registry_url']) client = registry.RegistryClient(registry_url)
base_token_path = args['token_base_path']
token_json = args['token_json']
log.info("ADD TOKEN") log.info("ADD TOKEN")
# Request Add new tokens # Request Add new tokens
requestAddToken(client, base_token_path) requestAddToken(client, token_base_path)
log.info("Remove TOKEN") log.info("Remove TOKEN")
# Request delete removed token # Request delete removed token
requestRemoveToken(client, base_token_path) requestRemoveToken(client, token_base_path)
computer_partition = getComputerPartition(server_url, key_file, computer_partition = getComputerPartition(**computer_dict)
cert_file, computer_guid, partition_id)
log.info("Update Services") log.info("Update Services")
# check status of all token # check status of all token
checkService(client, base_token_path, checkService(client, token_base_path, token_json, computer_partition)
token_json, computer_partition)
...@@ -25,7 +25,7 @@ ...@@ -25,7 +25,7 @@
# #
############################################################################## ##############################################################################
import os import os
import sys
from slapos.recipe.librecipe import GenericBaseRecipe from slapos.recipe.librecipe import GenericBaseRecipe
class Recipe(GenericBaseRecipe): class Recipe(GenericBaseRecipe):
...@@ -56,10 +56,9 @@ class Recipe(GenericBaseRecipe): ...@@ -56,10 +56,9 @@ class Recipe(GenericBaseRecipe):
configuration)) configuration))
path_list.append(config) path_list.append(config)
redis = self.createPythonScript( redis = self.createWrapper(
self.options['wrapper'], self.options['wrapper'],
'slapos.recipe.librecipe.execute.execute', (self.options['server_bin'], config_file),
[self.options['server_bin'], config_file]
) )
path_list.append(redis) path_list.append(redis)
...@@ -67,11 +66,20 @@ class Recipe(GenericBaseRecipe): ...@@ -67,11 +66,20 @@ class Recipe(GenericBaseRecipe):
if promise_script: if promise_script:
promise = self.createPythonScript( promise = self.createPythonScript(
promise_script, promise_script,
'%s.promise.main' % __name__, __name__ + '.promise',
dict(host=self.options['ipv6'], port=self.options['port'], (self.options['ipv6'], int(self.options['port']),
unixsocket = self.options.get('unixsocket') ) self.options.get('unixsocket'))
) )
path_list.append(promise) path_list.append(promise)
return path_list return path_list
def promise(host, port, unixsocket):
from .MyRedis2410 import Redis
try:
r = Redis(host=host, port=port, unix_socket_path=unixsocket, db=0)
r.publish("Promise-Service","SlapOS Promise")
r.connection_pool.disconnect()
except Exception, e:
sys.exit(e)
#! /usr/bin/env python
# -*- coding: utf-8 -*-
import slapos.recipe.redis.MyRedis2410 as redis
import sys
def main(args):
host = args['host']
port = int(args['port'])
unixsocket = args['unixsocket']
try:
r = redis.Redis(host=host, port=port, unix_socket_path=unixsocket, db=0)
r.publish("Promise-Service","SlapOS Promise")
r.connection_pool.disconnect()
sys.exit(0)
except Exception, e:
print str(e)
sys.exit(1)
\ No newline at end of file
...@@ -70,14 +70,12 @@ class Recipe(GenericSlapRecipe): ...@@ -70,14 +70,12 @@ class Recipe(GenericSlapRecipe):
path_list.append(nginx_configuration_file) path_list.append(nginx_configuration_file)
# Generate Nginx wrapper # Generate Nginx wrapper
wrapper = self.createWrapper( path_list.append(self.createWrapper(
name=self.options['wrapper'], self.options['wrapper'],
command=self.options['nginx-executable'], (self.options['nginx-executable'],
parameters=[
'-c', self.options['configuration-file'], '-c', self.options['configuration-file'],
'-p', self.options['home-directory'] '-p', self.options['home-directory']
] )))
)
# TODO: reload configuration or have feature like apache_map # TODO: reload configuration or have feature like apache_map
......
...@@ -25,23 +25,15 @@ ...@@ -25,23 +25,15 @@
# #
############################################################################# #############################################################################
import os
import sys
import zc.buildout
from slapos.recipe.librecipe import BaseSlapRecipe
from slapos.recipe.librecipe import GenericBaseRecipe from slapos.recipe.librecipe import GenericBaseRecipe
class Recipe(GenericBaseRecipe): class Recipe(GenericBaseRecipe):
def install(self):
runner = self.createPythonScript( def install(self):
return self.createPythonScript(
self.options['runner-path'], self.options['runner-path'],
__name__+'.testrunner.run', __name__+'.testrunner.run',
arguments=[self.options['suite-url'], (self.options['suite-url'],
self.options['report-url'], self.options['report-url'],
self.options['report-project'], self.options['report-project'],
self.options['browser'], self.options['browser']))
])
return [runner]
...@@ -36,12 +36,7 @@ import urlparse ...@@ -36,12 +36,7 @@ import urlparse
from subprocess import Popen, PIPE from subprocess import Popen, PIPE
import signal import signal
def run(args): def run(suite_url, report_url, project, browser_binary):
suite_url = args[0]
report_url = args[1]
project = args[2]
browser_binary = args[3]
suite_parsed = urlparse.urlparse(suite_url) suite_parsed = urlparse.urlparse(suite_url)
config = { config = {
......
...@@ -31,27 +31,12 @@ from slapos.recipe.librecipe import GenericBaseRecipe ...@@ -31,27 +31,12 @@ from slapos.recipe.librecipe import GenericBaseRecipe
class Recipe(GenericBaseRecipe): class Recipe(GenericBaseRecipe):
def install(self): def install(self):
env = os.environ.copy()
path_list = self.options['path'].split('\n')
env.update(PATH=':'.join(path_list))
env.update(SHELL=self.options['shell'])
env.update(HOME=self.options['home'])
ps1 = self.options.get('ps1') ps1 = self.options.get('ps1')
if ps1 is not None: shell = self.options['shell']
env.update(PS1=str(json.loads(ps1))) env = {
else: 'HOME': self.options['home'],
env.update(PS1=env.get('PS1', '> ')) 'PATH': ':'.join(self.options['path'].split('\n')),
'PS1': str(json.loads(ps1)) if ps1 else os.getenv('PS1', '> '),
wrapper = self.createPythonScript( 'SHELL': shell,
self.options['wrapper'], }
'slapos.recipe.librecipe.execute.executee', return self.createWrapper(self.options['wrapper'], (shell,), env)
[ # Executable
[self.options['shell']],
# Environment
env
]
)
return [wrapper]
...@@ -33,24 +33,14 @@ import shlex ...@@ -33,24 +33,14 @@ import shlex
from slapos.recipe.librecipe import GenericBaseRecipe from slapos.recipe.librecipe import GenericBaseRecipe
def login_shell(args): def login_shell(password_file, shell):
password_file = args['password-file']
if password_file: if password_file:
with open(password_file, 'r') as password_file: with open(password_file, 'r') as password_file:
password = password_file.read() password = password_file.read()
if (password != ''): if not password or hmac.compare_digest(getpass(), password):
entered_password = getpass() commandline = shlex.split(shell)
else: os.execv(commandline[0], commandline)
entered_password = ''
if not hmac.compare_digest(entered_password, password):
return 1
else:
commandline = shlex.split(args['shell'])
path = commandline[0]
os.execv(path, commandline)
else:
return 1 return 1
def shellinabox(args): def shellinabox(args):
...@@ -95,22 +85,16 @@ def shellinabox(args): ...@@ -95,22 +85,16 @@ def shellinabox(args):
class Recipe(GenericBaseRecipe): class Recipe(GenericBaseRecipe):
def install(self): def install(self):
path_list = [] login_shell_wrapper = self.createPythonScript(
login_shell = self.createPythonScript(
self.options['login-shell'], self.options['login-shell'],
'%s.login_shell' % __name__, __name__ + '.login_shell',
{ (self.options['password-file'], self.options['shell'])
'password-file': self.options['password-file'],
'shell': self.options['shell']
}
) )
path_list.append(login_shell)
wrapper = self.createPythonScript( shellinabox_wrapper = self.createPythonScript(
self.options['wrapper'], self.options['wrapper'],
'%s.shellinabox' % __name__, __name__ + '.shellinabox',
dict( (dict(
certificate_dir=self.options['certificate-directory'], certificate_dir=self.options['certificate-directory'],
ssl_key=self.options['key-file'], ssl_key=self.options['key-file'],
ssl_certificate=self.options['cert-file'], ssl_certificate=self.options['cert-file'],
...@@ -118,9 +102,8 @@ class Recipe(GenericBaseRecipe): ...@@ -118,9 +102,8 @@ class Recipe(GenericBaseRecipe):
directory=self.options['directory'], directory=self.options['directory'],
ipv6=self.options['ipv6'], ipv6=self.options['ipv6'],
port=self.options['port'], port=self.options['port'],
login_shell=login_shell, login_shell=login_shell_wrapper,
) ),)
) )
path_list.append(wrapper)
return [wrapper] return login_shell_wrapper, shellinabox_wrapper
...@@ -32,6 +32,6 @@ class Recipe(GenericBaseRecipe): ...@@ -32,6 +32,6 @@ class Recipe(GenericBaseRecipe):
self.createPythonScript( self.createPythonScript(
self.options['wrapper-path'], self.options['wrapper-path'],
'slapos.recipe.librecipe.execute.execute_with_signal_translation', 'slapos.recipe.librecipe.execute.execute_with_signal_translation',
[self.options['wrapped-path']] ((self.options['wrapped-path'],),)
) )
] ]
...@@ -68,9 +68,8 @@ class Recipe(GenericBaseRecipe): ...@@ -68,9 +68,8 @@ class Recipe(GenericBaseRecipe):
'root-dir': self.options['root-dir'] 'root-dir': self.options['root-dir']
} }
server = self.createPythonScript( return self.createPythonScript(
self.options['wrapper'].strip(), self.options['wrapper'].strip(),
'%s.simplehttpserver.run' % __name__, parameters __name__ + '.simplehttpserver.run',
(parameters,)
) )
return [server]
...@@ -29,9 +29,9 @@ import time ...@@ -29,9 +29,9 @@ import time
from slapos.recipe.librecipe import GenericBaseRecipe from slapos.recipe.librecipe import GenericBaseRecipe
def log(args): def log(filename):
prefix = time.strftime('%Y-%m-%d.%H:%M.%s:') prefix = time.strftime('%Y-%m-%d.%H:%M.%s:')
with open(args['filename'], 'aw') as logfile: with open(filename, 'a') as logfile:
for line in sys.stdin: for line in sys.stdin:
print >> logfile, prefix, line, print >> logfile, prefix, line,
print >> logfile, prefix, '------------------------' print >> logfile, prefix, '------------------------'
...@@ -39,10 +39,7 @@ def log(args): ...@@ -39,10 +39,7 @@ def log(args):
class Recipe(GenericBaseRecipe): class Recipe(GenericBaseRecipe):
def install(self): def install(self):
wrapper = self.options['wrapper'] return self.createPythonScript(
log = self.options['log'] self.options['wrapper'],
script = self.createPythonScript(wrapper,
__name__ + '.log', __name__ + '.log',
arguments=dict(filename=log)) (self.options['log'],))
return [script]
...@@ -48,11 +48,10 @@ class Recipe(GenericBaseRecipe): ...@@ -48,11 +48,10 @@ class Recipe(GenericBaseRecipe):
) )
# Create init script # Create init script
wrapper = self.createPythonScript( wrapper = self.createWrapper(
self.options['wrapper'], self.options['wrapper'],
'slapos.recipe.librecipe.execute.execute', (self.options['sphinx-searchd-binary'].strip(), '-c',
[self.options['sphinx-searchd-binary'].strip(), '-c', sphinx_conf_path, '--nodetach'),
sphinx_conf_path, '--nodetach'],
) )
return [wrapper, sphinx_conf_path] return [wrapper, sphinx_conf_path]
...@@ -78,21 +78,15 @@ class Recipe(GenericBaseRecipe): ...@@ -78,21 +78,15 @@ class Recipe(GenericBaseRecipe):
self.substituteTemplate(template_filename, config)) self.substituteTemplate(template_filename, config))
# Prepare directories # Prepare directories
prepare_path = self.createPythonScript( prepare_path = self.createWrapper(
self.options['prepare-path'], self.options['prepare-path'],
'slapos.recipe.librecipe.execute.execute', (self.options['binary-path'].strip(),
arguments=[self.options['binary-path'].strip(), '-z', '-f', configuration_path))
'-z',
'-f', configuration_path,
],)
# Create running wrapper # Create running wrapper
wrapper_path = self.createPythonScript( wrapper_path = self.createWrapper(
self.options['wrapper-path'], self.options['wrapper-path'],
'slapos.recipe.librecipe.execute.execute', (self.options['binary-path'].strip(),
arguments=[self.options['binary-path'].strip(), '-N', '-f', configuration_path))
'-N',
'-f', configuration_path,
],)
return [configuration_path, wrapper_path, prepare_path] return [configuration_path, wrapper_path, prepare_path]
...@@ -34,17 +34,14 @@ from slapos.recipe.librecipe import GenericBaseRecipe ...@@ -34,17 +34,14 @@ from slapos.recipe.librecipe import GenericBaseRecipe
from slapos.recipe.librecipe.inotify import subfiles from slapos.recipe.librecipe.inotify import subfiles
# This authority only works with dropbear or openssh sshkey generators # This authority only works with dropbear or openssh sshkey generators
def sshkeys_authority(args): def sshkeys_authority(request_directory, keygen_binary):
requests_directory = args['requests']
keygen_binary = args['sshkeygen']
if 'openssh' in keygen_binary: if 'openssh' in keygen_binary:
authority_type = 'openssh' authority_type = 'openssh'
else: else:
# Keep dropbear for compatibility # Keep dropbear for compatibility
authority_type = 'dropbear' authority_type = 'dropbear'
for request_filename in subfiles(requests_directory): for request_filename in subfiles(request_directory):
with open(request_filename) as request_file: with open(request_filename) as request_file:
request = json.load(request_file) request = json.load(request_file)
...@@ -98,18 +95,13 @@ def sshkeys_authority(args): ...@@ -98,18 +95,13 @@ def sshkeys_authority(args):
public_key_file.write(public_key_value) public_key_file.write(public_key_value)
class Recipe(GenericBaseRecipe): class Recipe(GenericBaseRecipe):
def install(self): def install(self):
args = dict( return self.createPythonScript(self.options['wrapper'],
requests=self.options['request-directory'], __name__ + '.sshkeys_authority',
sshkeygen=self.options['keygen-binary'], (self.options['request-directory'],
) self.options['keygen-binary']))
wrapper = self.createPythonScript(self.options['wrapper'],
__name__ + '.sshkeys_authority', args)
return [wrapper]
class Request(GenericBaseRecipe): class Request(GenericBaseRecipe):
...@@ -160,11 +152,9 @@ class Request(GenericBaseRecipe): ...@@ -160,11 +152,9 @@ class Request(GenericBaseRecipe):
os.symlink(self.private_key, private_key_link) os.symlink(self.private_key, private_key_link)
# end-XXX # end-XXX
wrapper = self.createPythonScript( wrapper = self.createWrapper(
self.options['wrapper'], self.options['wrapper'],
'slapos.recipe.librecipe.execute.execute_wait', (self.options['executable'],),
[ [self.options['executable']], wait_list=(self.private_key, self.public_key))
[self.private_key, self.public_key] ])
return [request_file, wrapper, public_key_link, private_key_link] return [request_file, wrapper, public_key_link, private_key_link]
...@@ -30,13 +30,16 @@ import errno ...@@ -30,13 +30,16 @@ import errno
from slapos.recipe.librecipe import GenericBaseRecipe from slapos.recipe.librecipe import GenericBaseRecipe
def post_rotate(args): def kill(pid_file, sig=signal.SIGUSR1):
pid_file = args['pid_file'] if os.path.exists(pid_file):
with open(pid_file) as f:
if os.path.exist(pid_file): pid = int(f.read().strip())
with open(pid_file, 'r') as file_: try:
pid = file_.read().strip() os.kill(pid, sig)
os.kill(pid, signal.SIGUSR1) except OSError, e:
if e.errno != errno.ESRCH: # No such process
raise e
os.unlink(pid_file)
class Recipe(GenericBaseRecipe): class Recipe(GenericBaseRecipe):
...@@ -76,28 +79,18 @@ class Recipe(GenericBaseRecipe): ...@@ -76,28 +79,18 @@ class Recipe(GenericBaseRecipe):
self.substituteTemplate(template, conf)) self.substituteTemplate(template, conf))
path_list.append(conf_file) path_list.append(conf_file)
wrapper = self.createPythonScript( wrapper = self.createWrapper(
self.options['wrapper'], self.options['wrapper'],
'slapos.recipe.librecipe.execute.execute', (self.options['stunnel-binary'], conf_file),
[self.options['stunnel-binary'], conf_file]
) )
path_list.append(wrapper) path_list.append(wrapper)
if os.path.exists(pid_file):
with open(pid_file, 'r') as file_:
pid = file_.read().strip()
# Reload configuration # Reload configuration
try: kill(pid_file, signal.SIGHUP)
os.kill(int(pid, 10), signal.SIGHUP)
except OSError, e:
if e.errno == errno.ESRCH: # No such process
os.unlink(pid_file)
else:
raise e
if 'post-rotate-script' in self.options: if 'post-rotate-script' in self.options:
self.createPythonScript(self.options['post-rotate-script'], path_list.append(self.createPythonScript(
__name__ + 'post_rotate', self.options['post-rotate-script'],
dict(pid_file=pid_file)) __name__ + '.kill', (pid_file,)))
return path_list return path_list
...@@ -36,19 +36,17 @@ class Recipe(GenericBaseRecipe): ...@@ -36,19 +36,17 @@ class Recipe(GenericBaseRecipe):
r = [configuration_file] r = [configuration_file]
wrapper = self.options.get('tidstorage-wrapper') wrapper = self.options.get('tidstorage-wrapper')
wrapper and r.append(self.createPythonScript(wrapper, wrapper and r.append(self.createWrapper(wrapper,
'slapos.recipe.librecipe.execute.execute', (self.options['tidstoraged-binary'],
[self.options['tidstoraged-binary'], '--nofork', '--config', '--nofork', '--config', configuration_file)))
configuration_file]))
r.append(self.createPythonScript( r.append(self.createWrapper(
self.options['repozo-wrapper'], self.options['repozo-wrapper'],
'slapos.recipe.librecipe.execute.execute', (self.options['tidstorage-repozo-binary'],
[self.options['tidstorage-repozo-binary'],
'--config', configuration_file, '--config', configuration_file,
'--repozo', self.options['repozo-binary'], '--repozo', self.options['repozo-binary'],
'--gzip', '--gzip',
'--quick', '--quick',
])) )))
return r return r
##############################################################################
#
# Copyright (c) 2010 Vifib SARL and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
import shlex
from slapos.recipe.librecipe import GenericBaseRecipe
class Recipe(GenericBaseRecipe):
def install(self):
files = [f for f in self.options['files'].split('\n') if f]
command_line = shlex.split(self.options['command-line'])
wrapper = self.createPythonScript(
self.options['wrapper'],
'slapos.recipe.librecipe.execute.execute_wait',
[ command_line,
files ],
)
return [wrapper]
...@@ -26,7 +26,6 @@ ...@@ -26,7 +26,6 @@
############################################################################## ##############################################################################
import shlex import shlex
import os
from slapos.recipe.librecipe import GenericBaseRecipe from slapos.recipe.librecipe import GenericBaseRecipe
...@@ -38,56 +37,31 @@ class Recipe(GenericBaseRecipe): ...@@ -38,56 +37,31 @@ class Recipe(GenericBaseRecipe):
:param lines wait-for-files: list of files to wait for :param lines wait-for-files: list of files to wait for
:param str pidfile: path to pidfile ensure exclusivity for the process :param str pidfile: path to pidfile ensure exclusivity for the process
:param bool parameters-extra: whether wrapper parameters are passed onto command :param str private-dev-shm: size of private /dev/shm, using user namespaces
:param bool reserve-cpu: command will ask for an exclusive CPU core :param bool reserve-cpu: command will ask for an exclusive CPU core
""" """
def install(self): def install(self):
command_line = shlex.split(self.options['command-line']) args = shlex.split(self.options['command-line'])
wrapper_path = self.options['wrapper-path'] wrapper_path = self.options['wrapper-path']
wait_files = self.options.get('wait-for-files') wait_files = self.options.get('wait-for-files')
environment = self.options.get('environment')
parameters_extra = self.options.get('parameters-extra')
pidfile = self.options.get('pidfile') pidfile = self.options.get('pidfile')
reserve_cpu = self.options.get('reserve-cpu', False) private_dev_shm = self.options.get('private-dev-shm')
if not wait_files and not environment: environment = {}
# Create a simple wrapper as shell script for line in (self.options.get('environment') or '').splitlines():
return [self.createWrapper( line = line.strip()
name=wrapper_path, if line:
command=command_line[0], k, v = line.split('=')
parameters=command_line[1:], environment[k.rstrip()] = v.lstrip()
parameters_extra=parameters_extra,
pidfile=pidfile,
reserve_cpu=reserve_cpu
)]
# More complex needs: create a Python script as wrapper kw = {}
if wait_files:
if wait_files is not None: kw['wait_list'] = wait_files.split()
wait_files = [filename.strip() for filename in wait_files.split() if pidfile:
if filename.strip()] kw['pidfile'] = pidfile
if environment is not None: if private_dev_shm:
environment = dict((k.strip(), v.strip()) for k, v in [ kw['private_dev_shm'] = private_dev_shm
line.split('=') for line in environment.splitlines() if line.strip() ]) if self.isTrueValue(self.options.get('reserve-cpu')):
kw['reserve_cpu'] = True
# We create a python script and a wrapper around the python
# script because the python script might have a too long #! line
if os.path.exists(os.path.join(self.buildout['buildout']['directory'], "bin")):
base_script_path = os.path.join(
self.buildout['buildout']['directory'], "bin/" + wrapper_path.split("/")[-1])
else:
base_script_path = os.path.join(
self.buildout['buildout']['directory'], wrapper_path.split("/")[-1])
python_script = self.createPythonScript(
base_script_path +'.py',
'slapos.recipe.librecipe.execute.generic_exec',
(command_line, wait_files, environment,), )
return [python_script, self.createWrapper(
name=wrapper_path,
command=python_script,
parameters=[],
parameters_extra=parameters_extra,
pidfile=pidfile,
reserve_cpu=reserve_cpu
)]
return self.createWrapper(wrapper_path, args, environment, **kw)
...@@ -71,10 +71,10 @@ class Recipe(BaseSlapRecipe): ...@@ -71,10 +71,10 @@ class Recipe(BaseSlapRecipe):
self._createDirectory(cron_d) self._createDirectory(cron_d)
self._createDirectory(crontabs) self._createDirectory(crontabs)
wrapper = zc.buildout.easy_install.scripts([('crond', wrapper = zc.buildout.easy_install.scripts([('crond',
'slapos.recipe.librecipe.execute', 'execute')], self.ws, sys.executable, 'slapos.recipe.librecipe.execute', 'generic_exec')], self.ws,
self.wrapper_directory, arguments=[ sys.executable, self.wrapper_directory, arguments=repr((
self.options['dcrond_binary'].strip(), '-s', cron_d, '-c', crontabs, self.options['dcrond_binary'].strip(), '-s', cron_d, '-c', crontabs,
'-t', timestamps, '-f', '-l', '5', '-M', catcher] '-t', timestamps, '-f', '-l', '5', '-M', catcher))[1:-1]
)[0] )[0]
self.path_list.append(wrapper) self.path_list.append(wrapper)
return cron_d return cron_d
...@@ -101,10 +101,10 @@ class Recipe(BaseSlapRecipe): ...@@ -101,10 +101,10 @@ class Recipe(BaseSlapRecipe):
self.path_list.append(zabbix_agentd_path) self.path_list.append(zabbix_agentd_path)
wrapper = zc.buildout.easy_install.scripts([('zabbixagentd', wrapper = zc.buildout.easy_install.scripts([('zabbixagentd',
'slapos.recipe.librecipe.execute', 'execute')], self.ws, sys.executable, 'slapos.recipe.librecipe.execute', 'generic_exec')], self.ws,
self.bin_directory, arguments=[ sys.executable, self.bin_directory, arguments=repr((
self.options['zabbix_agentd_binary'].strip(), '-c', self.options['zabbix_agentd_binary'].strip(), '-c',
zabbix_agentd_path])[0] zabbix_agentd_path))[1:-1])[0]
self.path_list.extend(zc.buildout.easy_install.scripts([ self.path_list.extend(zc.buildout.easy_install.scripts([
('zabbixagentd', __name__ + '.svcdaemon', 'svcdaemon')], ('zabbixagentd', __name__ + '.svcdaemon', 'svcdaemon')],
......
...@@ -89,10 +89,9 @@ class Recipe(GenericBaseRecipe): ...@@ -89,10 +89,9 @@ class Recipe(GenericBaseRecipe):
self.substituteTemplate(template_filename, config)) self.substituteTemplate(template_filename, config))
# Create running wrapper # Create running wrapper
wrapper_path = self.createPythonScript( wrapper_path = self.createWrapper(
self.options['wrapper-path'], self.options['wrapper-path'],
'slapos.recipe.librecipe.execute.execute', (self.options['binary-path'].strip(),
arguments=[self.options['binary-path'].strip(), '-C', '-C', self.options['conf-path']))
self.options['conf-path']],)
return [configuration_path, wrapper_path] return [configuration_path, wrapper_path]
...@@ -79,15 +79,14 @@ class Re6stnetTest(unittest.TestCase): ...@@ -79,15 +79,14 @@ class Re6stnetTest(unittest.TestCase):
token_file = os.path.join(self.options['conf-dir'], 'token.json') token_file = os.path.join(self.options['conf-dir'], 'token.json')
with open(path, 'r') as f: with open(path, 'r') as f:
content = f.read() content = f.read()
self.assertIn("'token_json': '%s'" % token_file, content) self.assertIn("('http://%s:%s/', %r, %r," % (
self.options['ipv4'], self.options['port'], self.token_dir, token_file),
content)
self.assertIn("'partition_id': 'slappart0'", content) self.assertIn("'partition_id': 'slappart0'", content)
self.assertIn("'computer_id': 'comp-test'", content) self.assertIn("'computer_guid': 'comp-test'", content)
self.assertIn("'key_file': '/path/to/key'", content) self.assertIn("'key_file': '/path/to/key'", content)
self.assertIn("'cert_file': '/path/to/cert'", content) self.assertIn("'cert_file': '/path/to/cert'", content)
self.assertIn("'server_url': 'http://server.com'", content) self.assertIn("'master_url': 'http://server.com'", content)
self.assertIn("'token_base_path': '%s'" % self.token_dir, content)
self.assertIn("'registry_url': 'http://%s:%s/'" % (self.options['ipv4'],
self.options['port']), content)
def fake_generateCertificates(self): def fake_generateCertificates(self):
return return
......
...@@ -50,8 +50,8 @@ gitdb = 0.6.4 ...@@ -50,8 +50,8 @@ gitdb = 0.6.4
pycrypto = 2.6.1 pycrypto = 2.6.1
pycurl = 7.43.0 pycurl = 7.43.0
slapos.recipe.download = 1.0 slapos.recipe.download = 1.0
slapos.recipe.template = 4.2 slapos.recipe.template = 4.3
slapos.toolbox = 0.73 slapos.toolbox = 0.74
smmap = 0.9.0 smmap = 0.9.0
# Required by: # Required by:
......
...@@ -10,8 +10,8 @@ gitdb = 0.6.4 ...@@ -10,8 +10,8 @@ gitdb = 0.6.4
plone.recipe.command = 1.1 plone.recipe.command = 1.1
pycrypto = 2.6.1 pycrypto = 2.6.1
rdiff-backup = 1.0.5+SlapOSPatched001 rdiff-backup = 1.0.5+SlapOSPatched001
slapos.recipe.template = 4.2 slapos.recipe.template = 4.3
slapos.toolbox = 0.73 slapos.toolbox = 0.74
smmap = 0.9.0 smmap = 0.9.0
numpy = 1.11.2 numpy = 1.11.2
pyasn1 = 0.2.3 pyasn1 = 0.2.3
......
...@@ -40,4 +40,4 @@ cns.recipe.symlink = 0.2.3 ...@@ -40,4 +40,4 @@ cns.recipe.symlink = 0.2.3
collective.recipe.environment = 0.2.0 collective.recipe.environment = 0.2.0
erp5.util = 0.4.49 erp5.util = 0.4.49
plone.recipe.command = 1.1 plone.recipe.command = 1.1
slapos.recipe.template = 4.2 slapos.recipe.template = 4.3
# THIS IS NOT A BUILDOUT FILE, despite purposedly using a compatible syntax.
# The only allowed lines here are (regexes):
# - "^#" comments, copied verbatim
# - "^[" section beginings, copied verbatim
# - lines containing an "=" sign which must fit in the following categorie.
# - "^\s*filename\s*=\s*path\s*$" where "path" is relative to this file
# But avoid directories, they are not portable.
# Copied verbatim.
# - "^\s*hashtype\s*=.*" where "hashtype" is one of the values supported
# by the re-generation script.
# Re-generated.
# - other lines are copied verbatim
# Substitution (${...:...}), extension ([buildout] extends = ...) and
# section inheritance (< = ...) are NOT supported (but you should really
# not need these here).
[template-cloudooo]
filename = instance.cfg.in
md5sum = 6e4431cf4b0a0d034402604b1e2844c0
[template-cloudooo-instance]
filename = instance-cloudooo.cfg.in
md5sum = afbfed1d762e5cdf7c6fd1292e7b28e7
...@@ -10,6 +10,7 @@ ...@@ -10,6 +10,7 @@
{% set next_port = slapparameter_dict.get(instance_parameter_dict['port-parameter-name'], next_port) | int -%} {% set next_port = slapparameter_dict.get(instance_parameter_dict['port-parameter-name'], next_port) | int -%}
{% endif -%} {% endif -%}
{% do assert(next_port > 0) -%} {% do assert(next_port > 0) -%}
{% set next_port = itertools.count(next_port).next -%}
{% set backend_count = instance_parameter_dict['backend-count'] | int -%} {% set backend_count = instance_parameter_dict['backend-count'] | int -%}
{% if instance_parameter_dict.get('backend-count-parameter-name') -%} {% if instance_parameter_dict.get('backend-count-parameter-name') -%}
...@@ -33,9 +34,8 @@ ...@@ -33,9 +34,8 @@
{% set mimetype_entry_addition = mimetype_entry_addition ~ "\n" ~ slapparameter_dict.get(instance_parameter_dict['mimetype-entry-addition-parameter-name'], '') -%} {% set mimetype_entry_addition = mimetype_entry_addition ~ "\n" ~ slapparameter_dict.get(instance_parameter_dict['mimetype-entry-addition-parameter-name'], '') -%}
{% endif -%} {% endif -%}
{% set apache_port = next_port -%} {% set apache_port = next_port() -%}
{% set haproxy_port = next_port + 1 -%} {% set haproxy_port = next_port() -%}
{% set next_port = next_port + 2 -%}
{% set apache_ip_list = [ipv4] -%} {% set apache_ip_list = [ipv4] -%}
{% if ipv6_set -%} {% if ipv6_set -%}
...@@ -184,13 +184,12 @@ ooo-uno-path = {{ parameter_dict['libreoffice-bin'] }}/basis-link/program ...@@ -184,13 +184,12 @@ ooo-uno-path = {{ parameter_dict['libreoffice-bin'] }}/basis-link/program
{% set name = 'cloudooo-' ~ index -%} {% set name = 'cloudooo-' ~ index -%}
[{{ cloudooo(name) }}] [{{ cloudooo(name) }}]
< = cloudooo-base < = cloudooo-base
port = {{ next_port }} port = {{ next_port() }}
openoffice-port = {{ next_port + 1 }} openoffice-port = {{ next_port() }}
configuration-file = ${directory:etc}/{{ name }}.cfg configuration-file = ${directory:etc}/{{ name }}.cfg
data-directory = ${directory:srv}/{{ name }} data-directory = ${directory:srv}/{{ name }}
wrapper = ${directory:services}/{{ name }} wrapper = ${directory:services}/{{ name }}
{% set next_port = next_port + 2 -%}
{% endfor -%} {% endfor -%}
[haproxy] [haproxy]
......
...@@ -39,6 +39,7 @@ extensions = jinja2.ext.do ...@@ -39,6 +39,7 @@ extensions = jinja2.ext.do
extra-context = extra-context =
section parameter_dict cloudooo-dynamic-template-parameter-dict section parameter_dict cloudooo-dynamic-template-parameter-dict
key ipv4_set slap-parameters:ipv4 key ipv4_set slap-parameters:ipv4
import itertools itertools
[switch-softwaretype] [switch-softwaretype]
recipe = slapos.cookbook:softwaretype recipe = slapos.cookbook:softwaretype
......
[buildout] [buildout]
extends = extends =
buildout.hash.cfg
../../stack/cloudooo.cfg ../../stack/cloudooo.cfg
../../stack/logrotate/buildout.cfg ../../stack/logrotate/buildout.cfg
parts = parts =
...@@ -25,8 +26,7 @@ cert = ${slap-connection:cert-file} ...@@ -25,8 +26,7 @@ cert = ${slap-connection:cert-file}
recipe = slapos.recipe.template:jinja2 recipe = slapos.recipe.template:jinja2
# XXX: "template.cfg" is hardcoded in instanciation recipe # XXX: "template.cfg" is hardcoded in instanciation recipe
rendered = ${buildout:directory}/template.cfg rendered = ${buildout:directory}/template.cfg
template = ${:_profile_base_location_}/instance.cfg.in template = ${:_profile_base_location_}/${:filename}
md5sum = 4543e62c41726ef074454252c4951ac8
mode = 640 mode = 640
[template-cloudooo-base] [template-cloudooo-base]
...@@ -81,6 +81,5 @@ template-logrotate-base = ${template-logrotate-base:rendered} ...@@ -81,6 +81,5 @@ template-logrotate-base = ${template-logrotate-base:rendered}
[template-cloudooo-instance] [template-cloudooo-instance]
recipe = slapos.recipe.build:download recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/instance-cloudooo.cfg.in url = ${:_profile_base_location_}/${:filename}
md5sum = 045830db85e1200f765788aa8b62289e
mode = 640 mode = 640
...@@ -53,7 +53,7 @@ mysqlclient = 1.3.12 ...@@ -53,7 +53,7 @@ mysqlclient = 1.3.12
# indirect dependancies # indirect dependancies
cp.recipe.cmd = 0.5 cp.recipe.cmd = 0.5
plone.recipe.command = 1.1 plone.recipe.command = 1.1
slapos.recipe.template = 4.2 slapos.recipe.template = 4.3
zope.exceptions = 4.0.7 zope.exceptions = 4.0.7
zope.testing = 4.1.3 zope.testing = 4.1.3
zc.recipe.testrunner = 2.0.0 zc.recipe.testrunner = 2.0.0
......
...@@ -3,12 +3,8 @@ Available ``software-type`` values ...@@ -3,12 +3,8 @@ Available ``software-type`` values
- ``default`` - ``default``
Recommended for production use. Recommended for developemnt and production use. Automatic creation of
erp5-site.
- ``create-erp5-site``
Automated creation of ERP5Site instance, for easy deployment.
Usage in production discouraged due to the increased risk of data loss.
Notes Notes
===== =====
......
...@@ -117,7 +117,11 @@ ...@@ -117,7 +117,11 @@
"timerserver-interval": { "timerserver-interval": {
"description": "Timerserver tick perdiod, in seconds, or 0 to disable", "description": "Timerserver tick perdiod, in seconds, or 0 to disable",
"default": 5, "default": 5,
"type": "integer" "type": "number"
},
"private-dev-shm": {
"description": "Size of private /dev/shm for wendelin.core. If sysctl kernel.unprivileged_userns_clone exists, it must be set to 1.",
"type": "string"
}, },
"ssl-authentication": { "ssl-authentication": {
"title": "Enable SSL Client authentication on this zope instance.", "title": "Enable SSL Client authentication on this zope instance.",
......
...@@ -9,13 +9,6 @@ ...@@ -9,13 +9,6 @@
"request": "instance-erp5-input-schema.json", "request": "instance-erp5-input-schema.json",
"response": "instance-erp5-output-schema.json", "response": "instance-erp5-output-schema.json",
"index": 0 "index": 0
},
"create-erp5-site": {
"title": "Create ERP5 Site",
"description": "Automated ERP5Site creation on instanciation when ZODB is found empty.",
"request": "instance-erp5-input-schema.json",
"response": "instance-erp5-output-schema.json",
"index": 1
} }
} }
} }
...@@ -59,7 +59,7 @@ mode = 0644 ...@@ -59,7 +59,7 @@ mode = 0644
[versions] [versions]
PyXML = 0.8.5 PyXML = 0.8.5
erp5.util = 0.4.50 erp5.util = 0.4.50
slapos.recipe.template = 4.2 slapos.recipe.template = 4.3
ipython = 5.3.0 ipython = 5.3.0
apache-libcloud = 2.1.0 apache-libcloud = 2.1.0
gitdb2 = 2.0.2 gitdb2 = 2.0.2
......
...@@ -106,6 +106,7 @@ def waitForSite(partition_path): ...@@ -106,6 +106,7 @@ def waitForSite(partition_path):
date=strftime("%Y/%m/%d %H:%M:%S", gmtime(end)), date=strftime("%Y/%m/%d %H:%M:%S", gmtime(end)),
duration=end - start, duration=end - start,
) )
print(try_info + 'status_dict %r' % (status_dict,))
return status_dict return status_dict
......
...@@ -15,4 +15,4 @@ md5sum = efd3b712a2294207f265a9c45648d5cf ...@@ -15,4 +15,4 @@ md5sum = efd3b712a2294207f265a9c45648d5cf
mode = 0644 mode = 0644
[versions] [versions]
slapos.recipe.template = 4.2 slapos.recipe.template = 4.3
...@@ -381,5 +381,5 @@ cns.recipe.symlink = 0.2.3 ...@@ -381,5 +381,5 @@ cns.recipe.symlink = 0.2.3
docutils = 0.12 docutils = 0.12
plone.recipe.command = 1.1 plone.recipe.command = 1.1
rubygemsrecipe = 0.2.2+slapos001 rubygemsrecipe = 0.2.2+slapos001
slapos.recipe.template = 4.2 slapos.recipe.template = 4.3
z3c.recipe.scripts = 1.0.1 z3c.recipe.scripts = 1.0.1
...@@ -72,7 +72,7 @@ async = 0.6.1 ...@@ -72,7 +72,7 @@ async = 0.6.1
gitdb = 0.5.4 gitdb = 0.5.4
pycrypto = 2.6 pycrypto = 2.6
rdiff-backup = 1.0.5+SlapOSPatched001 rdiff-backup = 1.0.5+SlapOSPatched001
slapos.recipe.template = 4.2 slapos.recipe.template = 4.3
slapos.toolbox = 0.40.4 slapos.toolbox = 0.40.4
smmap = 0.8.2 smmap = 0.8.2
plone.recipe.command = 1.1 plone.recipe.command = 1.1
......
...@@ -48,4 +48,4 @@ md5sum = 8cde04bfd0c0e9bd56744b988275cfd8 ...@@ -48,4 +48,4 @@ md5sum = 8cde04bfd0c0e9bd56744b988275cfd8
PyRSS2Gen = 1.1 PyRSS2Gen = 1.1
cns.recipe.symlink = 0.2.3 cns.recipe.symlink = 0.2.3
plone.recipe.command = 1.1 plone.recipe.command = 1.1
slapos.recipe.template = 4.2 slapos.recipe.template = 4.3
...@@ -113,5 +113,5 @@ mode = 0644 ...@@ -113,5 +113,5 @@ mode = 0644
[versions] [versions]
erp5.util = 0.4.50 erp5.util = 0.4.50
slapos.recipe.template = 4.2 slapos.recipe.template = 4.3
selenium = 3.8.0 selenium = 3.8.0
...@@ -85,7 +85,7 @@ pyzmq = 16.0.2 ...@@ -85,7 +85,7 @@ pyzmq = 16.0.2
scikit-learn = 0.18.1 scikit-learn = 0.18.1
seaborn = 0.7.1 seaborn = 0.7.1
simplegeneric = 0.8.1 simplegeneric = 0.8.1
slapos.recipe.template = 4.2 slapos.recipe.template = 4.3
statsmodels = 0.8.0 statsmodels = 0.8.0
terminado = 0.6 terminado = 0.6
tornado = 4.4.2 tornado = 4.4.2
......
[buildout]
parts =
instance
eggs-directory = ${buildout:eggs-directory}
develop-eggs-directory = ${buildout:develop-eggs-directory}
[instance]
recipe = ${instance-recipe:egg}:${instance-recipe:module}
kumo_gateway_binary = ${kumo:location}/bin/kumo-gateway
kumo_manager_binary = ${kumo:location}/bin/kumo-manager
kumo_server_binary = ${kumo:location}/bin/kumo-server
dcrond_binary = ${dcron:location}/sbin/crond
openssl_binary = ${openssl:location}/bin/openssl
rdiff_backup_binary = ${buildout:bin-directory}/rdiff-backup
stunnel_binary = ${stunnel:location}/bin/stunnel
[buildout]
extends =
../../component/kumo/buildout.cfg
../../component/dcron/buildout.cfg
../../component/stunnel/buildout.cfg
../../component/rdiff-backup/buildout.cfg
../../component/lxml-python/buildout.cfg
../../stack/slapos.cfg
parts =
# Create instance template
template
rdiff-backup
dcron
kumo
stunnel
eggs
instance-recipe-egg
[instance-recipe]
egg = slapos.cookbook
module = kumofs
[instance-recipe-egg]
recipe = zc.recipe.egg
eggs = ${instance-recipe:egg}
[eggs]
recipe = zc.recipe.egg
eggs =
${lxml-python:egg}
[template]
# Default template for the instance.
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance.cfg
md5sum = 056a4af7128fd9e31da42c85cc039420
output = ${buildout:directory}/template.cfg
mode = 0644
[versions]
rdiff-backup = 1.0.5+SlapOSPatched001
slapos.recipe.template = 2.4.2
...@@ -41,6 +41,7 @@ parts = ${:common-parts} ...@@ -41,6 +41,7 @@ parts = ${:common-parts}
[eggs] [eggs]
recipe = zc.recipe.egg recipe = zc.recipe.egg
interpreter = python.eggs
eggs = eggs =
${python-cffi:egg} ${python-cffi:egg}
${python-cryptography:egg} ${python-cryptography:egg}
...@@ -89,7 +90,7 @@ command = ...@@ -89,7 +90,7 @@ command =
[template] [template]
recipe = slapos.recipe.template recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance.cfg.in url = ${:_profile_base_location_}/instance.cfg.in
md5sum = bf5ef731c0d8da0267a4939882b4eeee md5sum = 5a17fc127190bbc19361c5ffb10711b3
output = ${buildout:directory}/template.cfg output = ${buildout:directory}/template.cfg
mode = 0644 mode = 0644
...@@ -98,7 +99,7 @@ recipe = hexagonit.recipe.download ...@@ -98,7 +99,7 @@ recipe = hexagonit.recipe.download
ignore-existing = true ignore-existing = true
url = ${:_profile_base_location_}/instance-kvm.cfg.jinja2 url = ${:_profile_base_location_}/instance-kvm.cfg.jinja2
mode = 644 mode = 644
md5sum = e2b8f86bdc12c86e7d959b55c6d54f6d md5sum = 68b66fb3e9020642e57f4a4ee266f2b3
download-only = true download-only = true
on-update = true on-update = true
...@@ -107,7 +108,7 @@ recipe = hexagonit.recipe.download ...@@ -107,7 +108,7 @@ recipe = hexagonit.recipe.download
ignore-existing = true ignore-existing = true
url = ${:_profile_base_location_}/instance-kvm-cluster.cfg.jinja2.in url = ${:_profile_base_location_}/instance-kvm-cluster.cfg.jinja2.in
mode = 644 mode = 644
md5sum = 05b6004e8c7a94de14f247affcef4971 md5sum = ba3337b3678ed9d3578cc88749c5cd13
download-only = true download-only = true
on-update = true on-update = true
...@@ -185,7 +186,7 @@ ignore-existing = true ...@@ -185,7 +186,7 @@ ignore-existing = true
url = ${:_profile_base_location_}/template/template-kvm-run.in url = ${:_profile_base_location_}/template/template-kvm-run.in
mode = 644 mode = 644
filename = template-kvm-run.in filename = template-kvm-run.in
md5sum = 178a24cdad77cb6c2e519ac629dd0e74 md5sum = bd238397af6236b6b24b693012feeece
download-only = true download-only = true
on-update = true on-update = true
...@@ -195,7 +196,7 @@ ignore-existing = true ...@@ -195,7 +196,7 @@ ignore-existing = true
url = ${:_profile_base_location_}/template/kvm-controller-run.in url = ${:_profile_base_location_}/template/kvm-controller-run.in
mode = 644 mode = 644
filename = kvm-controller-run.in filename = kvm-controller-run.in
md5sum = 71afd2d13f6e56993ae413a168e012d7 md5sum = c86cd67bbdd26b7b14b7449a1bbd959b
download-only = true download-only = true
on-update = true on-update = true
...@@ -225,7 +226,7 @@ ignore-existing = true ...@@ -225,7 +226,7 @@ ignore-existing = true
url = ${:_profile_base_location_}/template/qemu-is-ready.in url = ${:_profile_base_location_}/template/qemu-is-ready.in
mode = 644 mode = 644
filename = qemu-is-ready.in filename = qemu-is-ready.in
md5sum = 0066fa0f5f3dd47bded5e5924df2550d md5sum = b304eec8e2cb71f10ea83cac22f6db12
download-only = true download-only = true
on-update = true on-update = true
......
...@@ -189,13 +189,35 @@ ...@@ -189,13 +189,35 @@
"default": "started", "default": "started",
"enum": ["started", "stopped"] "enum": ["started", "stopped"]
}, },
"enable-device-hotplug": {
"title": "Enable device hotplug mode",
"description": "If yes, this will allow to Create devices like CPU and Memory in hotplug mode without restart the VM. Operatin System should be configured to Online new created devices.",
"type": "boolean",
"default": false
},
"ram-size": { "ram-size": {
"title": "RAM size", "title": "RAM size",
"description": "RAM size, in MB.", "description": "RAM size, in MB.",
"type": "integer", "type": "integer",
"default": 1024, "default": 1024,
"minimum": 128, "minimum": 1024,
"multipleOf": 128 "multipleOf": 512
},
"ram-max-size": {
"title": "Maximum RAM size, in MB",
"description": "Define the maximum size of the memory. The size is in MB and should be a multiple of 512.",
"type": "integer",
"default": 51200,
"minimum": 1024,
"multipleOf": 512
},
"ram-hotplug-slot-size": {
"title": "Size of Hotpluggable RAM slot, in MB",
"description": "Define the RAM size to plug on one hotpluggable slot in MB, understand the size of one RAM bar. The RAM hotplugged on each slot will always have the same RAM size.",
"type": "integer",
"default": 512,
"minimum": 512,
"multipleOf": 512
}, },
"auto-ballooning": { "auto-ballooning": {
"title": "Enable qemu auto ballooning.", "title": "Enable qemu auto ballooning.",
...@@ -245,10 +267,13 @@ ...@@ -245,10 +267,13 @@
"type": "integer", "type": "integer",
"minimum": 1 "minimum": 1
}, },
"cpu-options": { "cpu-max-count": {
"title": "CPU Additional options: cores, threads, sockets, maxcpus.", "title": "Maximum CPU amount",
"description": "Additional options to use with cpu-count. Options are separated by coma: [cores=cores][,threads=threads][,sockets=sockets][,maxcpus=maxcpus]. Set this option if you know what you're doing.", "description": "Specifies the maximum number of CPUs.",
"type": "string" "type": "integer",
"default": 24,
"minimum": 1,
"maximum": 64
}, },
"numa": { "numa": {
"title": "Simulate a multi node NUMA system.", "title": "Simulate a multi node NUMA system.",
......
...@@ -8,7 +8,7 @@ ...@@ -8,7 +8,7 @@
{% set slave_frontend_iguid = slave_frontend_dict.get('instance-guid', '') -%} {% set slave_frontend_iguid = slave_frontend_dict.get('instance-guid', '') -%}
{% set kvm_instance_dict = {} -%} {% set kvm_instance_dict = {} -%}
{% set kvm_hostname_list = [] -%} {% set kvm_hostname_list = [] -%}
{% set monitor_url_list = [] -%} {% set monitor_base_url_dict = {} -%}
{% macro setconfig(name, value) -%} {% macro setconfig(name, value) -%}
{# will set a config-name = value if value is not empty -#} {# will set a config-name = value if value is not empty -#}
...@@ -51,12 +51,15 @@ config-authorized-key = {{ dumps(slapparameter_dict.get('authorized-keys') | joi ...@@ -51,12 +51,15 @@ config-authorized-key = {{ dumps(slapparameter_dict.get('authorized-keys') | joi
config-nbd-port = {{ dumps(kvm_parameter_dict.get('nbd-port', 1024)) }} config-nbd-port = {{ dumps(kvm_parameter_dict.get('nbd-port', 1024)) }}
config-nbd2-port = {{ dumps(kvm_parameter_dict.get('nbd-port2', 1024)) }} config-nbd2-port = {{ dumps(kvm_parameter_dict.get('nbd-port2', 1024)) }}
config-ram-size = {{ dumps(kvm_parameter_dict.get('ram-size', 1024)) }} config-ram-size = {{ dumps(kvm_parameter_dict.get('ram-size', 1024)) }}
config-ram-max-size = {{ dumps(kvm_parameter_dict.get('ram-max-size', '51200')) }}
config-enable-device-hotplug = {{ dumps(kvm_parameter_dict.get('enable-device-hotplug', False)) }}
config-ram-hotplug-slot-size = {{ dumps(kvm_parameter_dict.get('ram-hotplug-slot-size', 512)) }}
config-disk-size = {{ dumps(kvm_parameter_dict.get('disk-size', 10)) }} config-disk-size = {{ dumps(kvm_parameter_dict.get('disk-size', 10)) }}
config-disk-type = {{ dumps(kvm_parameter_dict.get('disk-type', 'virtio')) }} config-disk-type = {{ dumps(kvm_parameter_dict.get('disk-type', 'virtio')) }}
config-cpu-count = {{ dumps(kvm_parameter_dict.get('cpu-count', 1)) }} config-cpu-count = {{ dumps(kvm_parameter_dict.get('cpu-count', 1)) }}
config-cpu-max-count = {{ dumps(kvm_parameter_dict.get('cpu-max-count', 24)) }}
{{ setconfig('numa', kvm_parameter_dict.get('numa', '')) }} {{ setconfig('numa', kvm_parameter_dict.get('numa', '')) }}
{{ setconfig('machine-options', kvm_parameter_dict.get('machine-options', '')) }} {{ setconfig('machine-options', kvm_parameter_dict.get('machine-options', '')) }}
{{ setconfig('cpu-options', kvm_parameter_dict.get('cpu-options', '')) }}
{{ setconfig('nbd-host', kvm_parameter_dict.get('nbd-host', '')) }} {{ setconfig('nbd-host', kvm_parameter_dict.get('nbd-host', '')) }}
{{ setconfig('host2', kvm_parameter_dict.get('host2', '')) }} {{ setconfig('host2', kvm_parameter_dict.get('host2', '')) }}
...@@ -67,7 +70,7 @@ config-auto-ballooning = {{ dumps(kvm_parameter_dict.get('auto-ballooning', True ...@@ -67,7 +70,7 @@ config-auto-ballooning = {{ dumps(kvm_parameter_dict.get('auto-ballooning', True
{{ setconfig('disk-cache', kvm_parameter_dict.get('disk-cache', '')) }} {{ setconfig('disk-cache', kvm_parameter_dict.get('disk-cache', '')) }}
{% set nat_rules_list = kvm_parameter_dict.get('nat-rules', []) -%} {% set nat_rules_list = kvm_parameter_dict.get('nat-rules', []) -%}
{{ setconfig('nat-rules', ' '.join(nat_rules_list)) }} {{ setconfig('nat-rules', nat_rules_list | join(' ')) }}
config-publish-nat-url = True config-publish-nat-url = True
config-use-nat = {{ use_nat }} config-use-nat = {{ use_nat }}
config-use-tap = {{ dumps(kvm_parameter_dict.get('use-tap', True)) }} config-use-tap = {{ dumps(kvm_parameter_dict.get('use-tap', True)) }}
...@@ -124,10 +127,10 @@ return = ...@@ -124,10 +127,10 @@ return =
{% if str(kvm_parameter_dict.get('use-tap', 'True')).lower() == 'true' -%} {% if str(kvm_parameter_dict.get('use-tap', 'True')).lower() == 'true' -%}
{{ ' ' }}tap-ipv4 {{ ' ' }}tap-ipv4
{% do monitor_url_list.append('${' ~ section ~ ':connection-monitor-base-url}') -%}
{% do publish_dict.__setitem__('lan-' ~ instance_name, '${' ~ section ~ ':connection-tap-ipv4}') -%} {% do publish_dict.__setitem__('lan-' ~ instance_name, '${' ~ section ~ ':connection-tap-ipv4}') -%}
{% do kvm_hostname_list.append(instance_name ~ ' ' ~ '${' ~ section ~ ':connection-tap-ipv4}') -%} {% do kvm_hostname_list.append(instance_name ~ ' ' ~ '${' ~ section ~ ':connection-tap-ipv4}') -%}
{% endif -%} {% endif -%}
{% do monitor_base_url_dict.__setitem__(instance_name, '${' ~ section ~ ':connection-monitor-base-url}') -%}
{% do publish_dict.__setitem__(instance_name ~ '-backend-url', '${' ~ section ~ ':connection-backend-url}') -%} {% do publish_dict.__setitem__(instance_name ~ '-backend-url', '${' ~ section ~ ':connection-backend-url}') -%}
{% do publish_dict.__setitem__(instance_name ~ '-url', '${' ~ section ~ ':connection-url}') -%} {% do publish_dict.__setitem__(instance_name ~ '-url', '${' ~ section ~ ':connection-url}') -%}
{% do kvm_instance_dict.__setitem__(instance_name, (use_nat, nat_rules_list)) -%} {% do kvm_instance_dict.__setitem__(instance_name, (use_nat, nat_rules_list)) -%}
...@@ -241,11 +244,12 @@ cors-domains = {{ slapparameter_dict.get('monitor-cors-domains', 'monitor.app.of ...@@ -241,11 +244,12 @@ cors-domains = {{ slapparameter_dict.get('monitor-cors-domains', 'monitor.app.of
username = admin username = admin
password = ${monitor-htpasswd:passwd} password = ${monitor-htpasswd:passwd}
[monitor-conf-parameters] [monitor-base-url-dict]
monitor-url-list += {% for key, value in monitor_base_url_dict.items() -%}
{% for url in monitor_url_list -%} {{ key }} = {{ value }}
{{ ' ' ~ url }}
{% endfor %} {% endfor %}
[monitor-conf-parameters]
private-path-list += private-path-list +=
${directory:webroot}/ ${directory:webroot}/
......
...@@ -4,13 +4,35 @@ ...@@ -4,13 +4,35 @@
"title": "Input Parameters", "title": "Input Parameters",
"properties": { "properties": {
"enable-device-hotplug": {
"title": "Enable device hotplug mode",
"description": "If yes, this will allow to Create devices like CPU and Memory in hotplug mode without restart the VM. Operatin System should be configured to Online new created devices.",
"type": "boolean",
"default": false
},
"ram-size": { "ram-size": {
"title": "RAM size", "title": "RAM size",
"description": "RAM size, in MB.", "description": "RAM size, in MB.",
"type": "integer", "type": "integer",
"default": 1024, "default": 1024,
"minimum": 128, "minimum": 1024,
"multipleOf": 128 "multipleOf": 512
},
"ram-max-size": {
"title": "Maximum RAM size, in MB",
"description": "Define the maximum size of the memory. The size is in MB and should be a multiple of 512.",
"type": "integer",
"default": 51200,
"minimum": 1024,
"multipleOf": 512
},
"ram-hotplug-slot-size": {
"title": "Size of Hotpluggable RAM slot, in MB",
"description": "Define the RAM size to plug on one hotpluggable slot in MB, understand the size of one RAM bar. The RAM hotplugged on each slot will always have the same RAM size.",
"type": "integer",
"default": 512,
"minimum": 512,
"multipleOf": 512
}, },
"auto-ballooning": { "auto-ballooning": {
"title": "Enable qemu auto ballooning.", "title": "Enable qemu auto ballooning.",
...@@ -54,10 +76,25 @@ ...@@ -54,10 +76,25 @@
"type": "integer", "type": "integer",
"minimum": 1 "minimum": 1
}, },
"cpu-options": { "cpu-max-count": {
"title": "CPU Additional options: cores, threads, sockets, maxcpus.", "title": "Maximum CPU amount",
"description": "Additional options to use with cpu-count. Options are separated by coma: [cores=cores][,threads=threads][,sockets=sockets][,maxcpus=maxcpus]. Set this option if you know what you're doing.", "description": "Specifies the maximum number of CPUs.",
"type": "string" "type": "integer",
"default": 24,
"minimum": 1,
"maximum": 64
},
"max-cpu-hotplug-count": {
"title": "Maximum hotpluggable CPU amount",
"description": "Specifies the maximum number of hotpluggable CPUs.",
"type": "integer",
"default": 1
},
"cpu-hotplug-amount": {
"title": "CPU hotplug amount",
"description": "Specifies the number of CPUs to hotplug",
"type": "integer",
"default": 0
}, },
"numa": { "numa": {
"title": "Simulate a multi node NUMA system.", "title": "Simulate a multi node NUMA system.",
......
...@@ -5,6 +5,7 @@ ...@@ -5,6 +5,7 @@
{% set nat_restrict = slapparameter_dict.get('nat-restrict-mode', 'False').lower() -%} {% set nat_restrict = slapparameter_dict.get('nat-restrict-mode', 'False').lower() -%}
{% set name = slapparameter_dict.get('name', 'localhost') -%} {% set name = slapparameter_dict.get('name', 'localhost') -%}
{% set disable_ansible_promise = slapparameter_dict.get('disable-ansible-promise', 'True').lower() -%} {% set disable_ansible_promise = slapparameter_dict.get('disable-ansible-promise', 'True').lower() -%}
{% set enable_device_hotplug = slapparameter_dict.get('enable-device-hotplug', 'false').lower() -%}
{% set instance_type = slapparameter_dict.get('type', 'standalone') -%} {% set instance_type = slapparameter_dict.get('type', 'standalone') -%}
{% set nat_rule_list = slapparameter_dict.get('nat-rules', '22 80 443') -%} {% set nat_rule_list = slapparameter_dict.get('nat-rules', '22 80 443') -%}
{% set frontend_software_type = 'default' -%} {% set frontend_software_type = 'default' -%}
...@@ -62,10 +63,15 @@ storage-path = ${directory:srv}/passwd ...@@ -62,10 +63,15 @@ storage-path = ${directory:srv}/passwd
bytes = 8 bytes = 8
[kvm-controller-parameter-dict] [kvm-controller-parameter-dict]
python-path = {{ python_executable }} python-path = {{ python_eggs_executable }}
vnc-passwd = ${gen-passwd:passwd} vnc-passwd = ${gen-passwd:passwd}
socket-path = ${directory:var}/qmp_socket socket-path = ${directory:var}/qmp_socket
pid-file = ${directory:run}/pid_file
kvm-status-path = ${directory:var}/qemu-vm-is-ready kvm-status-path = ${directory:var}/qemu-vm-is-ready
cpu-count = ${slap-parameter:cpu-count}
ram-hotplug-slot-size = ${slap-parameter:ram-hotplug-slot-size}
ram-size = ${slap-parameter:ram-size}
enable-device-hotplug = ${slap-parameter:enable-device-hotplug}
[kvm-parameter-dict] [kvm-parameter-dict]
python-path = {{ python_executable }} python-path = {{ python_executable }}
...@@ -87,13 +93,16 @@ disk-size = ${slap-parameter:disk-size} ...@@ -87,13 +93,16 @@ disk-size = ${slap-parameter:disk-size}
disk-type = ${slap-parameter:disk-type} disk-type = ${slap-parameter:disk-type}
pid-file-path = ${directory:run}/pid_file pid-file-path = ${kvm-controller-parameter-dict:pid-file}
socket-path = ${kvm-controller-parameter-dict:socket-path} socket-path = ${kvm-controller-parameter-dict:socket-path}
smp-count = ${slap-parameter:cpu-count} enable-device-hotplug = ${kvm-controller-parameter-dict:enable-device-hotplug}
smp-options = ${slap-parameter:cpu-options} smp-count = ${kvm-controller-parameter-dict:cpu-count}
ram-size = ${slap-parameter:ram-size} smp-max-count = ${slap-parameter:cpu-max-count}
numa = ${slap-parameter:numa}
ram-size = ${kvm-controller-parameter-dict:ram-size}
ram-max-size = ${slap-parameter:ram-max-size}
init-ram-size = 1024
mac-address = ${create-mac:mac-address} mac-address = ${create-mac:mac-address}
tap-mac-address = ${create-tap-mac:mac-address} tap-mac-address = ${create-tap-mac:mac-address}
...@@ -143,7 +152,7 @@ disk-cache = ${slap-parameter:disk-cache} ...@@ -143,7 +152,7 @@ disk-cache = ${slap-parameter:disk-cache}
disk-aio = ${slap-parameter:disk-aio} disk-aio = ${slap-parameter:disk-aio}
auto-ballooning = ${slap-parameter:auto-ballooning} auto-ballooning = ${slap-parameter:auto-ballooning}
machine-options = ${slap-parameter:machine-options} machine-options = ${slap-parameter:machine-options}
cpu-options = ${slap-parameter:cpu-model} cpu-hotplug-slot-size = ${slap-parameter:cpu-model}
log-file = ${directory:log}/qemu.log log-file = ${directory:log}/qemu.log
...@@ -550,15 +559,15 @@ nbd-host = ...@@ -550,15 +559,15 @@ nbd-host =
nbd2-port = 1024 nbd2-port = 1024
nbd2-host = nbd2-host =
enable-device-hotplug = False
ram-size = 1024 ram-size = 1024
ram-max-size = 51200
ram-hotplug-slot-size = 512
disk-size = 10 disk-size = 10
disk-type = virtio disk-type = virtio
cpu-count = 1 cpu-count = 1
# cpu-option is a string: [cores=cores][,threads=threads][,sockets=sockets][,maxcpus=maxcpus] cpu-max-count = 24
cpu-options =
# list of numa options separate by space: node,nodeid=1,cpus=9-15 node,nodeid=2,cpus=1,3,7
numa =
disk-cache = writeback disk-cache = writeback
disk-aio = native disk-aio = native
auto-ballooning = True auto-ballooning = True
......
...@@ -93,6 +93,7 @@ context = ...@@ -93,6 +93,7 @@ context =
raw netcat_bin ${netcat:location}/bin/netcat raw netcat_bin ${netcat:location}/bin/netcat
raw openssl_executable_location ${openssl:location}/bin/openssl raw openssl_executable_location ${openssl:location}/bin/openssl
raw python_executable ${buildout:executable} raw python_executable ${buildout:executable}
raw python_eggs_executable ${buildout:bin-directory}/${eggs:interpreter}
raw qemu_executable_location ${kvm:location}/bin/qemu-system-x86_64 raw qemu_executable_location ${kvm:location}/bin/qemu-system-x86_64
raw qemu_img_executable_location ${kvm:location}/bin/qemu-img raw qemu_img_executable_location ${kvm:location}/bin/qemu-img
raw qemu_start_promise_tpl ${template-qemu-ready:location}/${template-qemu-ready:filename} raw qemu_start_promise_tpl ${template-qemu-ready:location}/${template-qemu-ready:filename}
......
...@@ -5,13 +5,13 @@ extends = common.cfg ...@@ -5,13 +5,13 @@ extends = common.cfg
# XXX - use websockify = 0.5.1 for compatibility with kvm frontend # XXX - use websockify = 0.5.1 for compatibility with kvm frontend
websockify = 0.5.1 websockify = 0.5.1
slapos.toolbox = 0.73 slapos.toolbox = 0.74
erp5.util = 0.4.49 erp5.util = 0.4.49
apache-libcloud = 1.1.0 apache-libcloud = 1.1.0
collective.recipe.environment = 0.2.0 collective.recipe.environment = 0.2.0
gitdb = 0.6.4 gitdb = 0.6.4
pycurl = 7.43.0 pycurl = 7.43.0
slapos.recipe.template = 4.2 slapos.recipe.template = 4.3
smmap = 0.9.0 smmap = 0.9.0
# websockify = 0.8.0 # websockify = 0.8.0
......
...@@ -6,40 +6,62 @@ ...@@ -6,40 +6,62 @@
import socket import socket
import time import time
import os import os
from slapos.qemuqmpclient import QemuQMPWrapper, getInitialQemuResourceDict
# XXX: to be factored with slapos.toolbox qemu qmp wrapper. # XXX: to be factored with slapos.toolbox qemu qmp wrapper.
socket_path = '{{ parameter_dict.get("socket-path") }}' socket_path = '{{ parameter_dict.get("socket-path") }}'
pid_file = '{{ parameter_dict.get("pid-file") }}'
vnc_password = '{{ parameter_dict.get("vnc-passwd") }}' vnc_password = '{{ parameter_dict.get("vnc-passwd") }}'
status_path = '{{ parameter_dict.get("kvm-status-path") }}' status_path = '{{ parameter_dict.get("kvm-status-path") }}'
cpu_amount = {{ parameter_dict.get("cpu-count") }}
cpu_model = '{{ parameter_dict.get("cpu-model", "qemu64-x86_64-cpu") }}'
slot_hotplug_size = {{ parameter_dict.get("ram-hotplug-slot-size", 512) }}
ram_size = {{ parameter_dict.get("ram-size") }}
enable_device_hotplug = '{{ parameter_dict.get("enable-device-hotplug") }}'.lower()
if os.path.exists(status_path): def write(message):
os.unlink(status_path) with open(status_path, 'w') as status_file:
status_file.write(message)
def update():
# Connect to KVM qmp socket
so = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
connected = False
while not connected:
try: try:
so.connect(socket_path) init_dict = getInitialQemuResourceDict(pid_file)
except socket.error: if os.path.exists(status_path):
time.sleep(1) os.unlink(status_path)
else:
connected = True if init_dict is None:
data = so.recv(1024) # qemu process is not OK
return
# Enable qmp init_ram_size = int(init_dict['ram'].split('M')[0])
so.send('{ "execute": "qmp_capabilities" }') if cpu_amount < 1:
data = so.recv(1024) raise ValueError("CPU should be at least equal to 1.")
hotplug_ram = ram_size - init_ram_size
# Set VNC password if hotplug_ram < 0:
so.send('{ "execute": "change", ' \ raise ValueError("RAM size cannot be less than the initial value %s MB" % init_ram_size)
'"arguments": { "device": "vnc", "target": "password", ' \
' "arg": "' + vnc_password + '" } }') qemu_wrapper = QemuQMPWrapper(socket_path)
data = so.recv(1024) qemu_wrapper.setVNCPassword(vnc_password)
if enable_device_hotplug == 'true':
# Finish write("Qemu Controller is still running...")
so.close() qemu_wrapper.updateDevice({
'device': 'cpu',
with open(status_path, 'w') as status_file: 'amount': cpu_amount,
status_file.write("OK") 'model': cpu_model
})
qemu_wrapper.updateDevice({
'device': 'memory',
'mem': hotplug_ram,
'slot': slot_hotplug_size,
'nslot': 128,
'canreboot': 1
})
except Exception, e:
write(str(e))
raise
write("")
if __name__ == "__main__":
update()
#!{{ dash }} #!{{ dash }}
if [ -f "{{ qemu_ready_path }}" ]; then FILE="{{ qemu_ready_path }}"
# don't start checks too fast
sleep 2
if [ -f "$FILE" ]; then
if [ "$(cat $FILE)" = "" ]; then
echo "VM correctly started." echo "VM correctly started."
else
>&2 echo "Qemu Controller failed"
>&2 cat $FILE
exit 1
fi
else else
log_file="{{ qemu_service_log_file }}" log_file="{{ qemu_service_log_file }}"
>&2 echo "Qemu process is not correctly started." >&2 echo "Qemu process is not correctly started."
......
...@@ -39,13 +39,13 @@ tap_interface = '{{ parameter_dict.get("tap-interface") }}' ...@@ -39,13 +39,13 @@ tap_interface = '{{ parameter_dict.get("tap-interface") }}'
listen_ip = '{{ parameter_dict.get("ipv4") }}' listen_ip = '{{ parameter_dict.get("ipv4") }}'
mac_address = '{{ parameter_dict.get("mac-address") }}' mac_address = '{{ parameter_dict.get("mac-address") }}'
tap_mac_address = '{{ parameter_dict.get("tap-mac-address") }}' tap_mac_address = '{{ parameter_dict.get("tap-mac-address") }}'
smp_count = '{{ parameter_dict.get("smp-count") }}' numa_list = '{{ parameter_dict.get("numa", "") }}'.split()
smp_options = '{{ parameter_dict.get("smp-options") }}'.strip() ram_size = {{ parameter_dict.get("ram-size") }}
numa_list = '{{ parameter_dict.get("numa") }}'.split() ram_max_size = '{{ parameter_dict.get("ram-max-size") }}'
ram_size = '{{ parameter_dict.get("ram-size") }}' init_ram_size = {{ parameter_dict.get("init-ram-size") }}
pid_file_path = '{{ parameter_dict.get("pid-file-path") }}' pid_file_path = '{{ parameter_dict.get("pid-file-path") }}'
external_disk_number = {{ parameter_dict.get("external-disk-number") }} external_disk_number = {{ parameter_dict.get("external-disk-number") }}
external_disk_size = '{{ parameter_dict.get("external-disk-size") }}' external_disk_size = {{ parameter_dict.get("external-disk-size") }}
external_disk_format = '{{ parameter_dict.get("external-disk-format") }}' external_disk_format = '{{ parameter_dict.get("external-disk-format") }}'
disk_storage_dict = {} disk_storage_dict = {}
disk_storage_list = """{{ parameter_dict.get("disk-storage-list") }}""".split('\n') disk_storage_list = """{{ parameter_dict.get("disk-storage-list") }}""".split('\n')
...@@ -69,9 +69,13 @@ disk_cache = disk_cache if disk_cache in ["none", "writeback", "unsafe", ...@@ -69,9 +69,13 @@ disk_cache = disk_cache if disk_cache in ["none", "writeback", "unsafe",
disk_aio = '{{ parameter_dict.get("disk-aio") }}'.strip() disk_aio = '{{ parameter_dict.get("disk-aio") }}'.strip()
disk_aio = disk_aio if disk_aio in ["threads", "native"] and \ disk_aio = disk_aio if disk_aio in ["threads", "native"] and \
disk_cache == "directsync" else "threads" disk_cache == "directsync" else "threads"
smp_count = {{ parameter_dict.get("smp-count") }}
smp_max_count = {{ parameter_dict.get("smp-max-count") }}
machine_options = '{{ parameter_dict.get("machine-options", "") }}'.strip() machine_options = '{{ parameter_dict.get("machine-options", "") }}'.strip()
cpu_model = '{{ parameter_dict.get("cpu-model", "") }}'.strip() cpu_model = '{{ parameter_dict.get("cpu-model", "") }}'.strip()
enable_device_hotplug = '{{ parameter_dict.get("enable-device-hotplug") }}'.lower()
logfile = '{{ parameter_dict.get("log-file") }}' logfile = '{{ parameter_dict.get("log-file") }}'
if hasattr(ssl, '_create_unverified_context') and url_check_certificate == 'false': if hasattr(ssl, '_create_unverified_context') and url_check_certificate == 'false':
...@@ -255,15 +259,14 @@ if use_tap == 'true': ...@@ -255,15 +259,14 @@ if use_tap == 'true':
tap_interface, vhost), tap_interface, vhost),
'-device', 'virtio-net-pci,netdev=lan%s,mac=%s' % (number, tap_mac_address)] '-device', 'virtio-net-pci,netdev=lan%s,mac=%s' % (number, tap_mac_address)]
smp = smp_count if enable_device_hotplug != 'true':
if smp_options: smp = '%s,maxcpus=%s' % (smp_count, smp_max_count)
for option in smp_options.split(','): ram = '%sM,slots=128,maxmem=%sM' % (ram_size, ram_max_size)
key, val = option.split('=') else:
if key in ('cores', 'threads', 'sockets', 'maxcpus') and val.isdigit(): smp = '1,maxcpus=%s' % smp_max_count
smp += ',%s=%s' % (key, val) ram = '%sM,slots=128,maxmem=%s' % (init_ram_size, ram_max_size)
kvm_argument_list = [qemu_path, kvm_argument_list = [qemu_path,
'-enable-kvm', '-smp', smp, '-name', vm_name, '-enable-kvm', '-smp', smp, '-name', vm_name, '-m', ram, '-vga', 'std',
'-m', ram_size, '-vga', 'std',
'-drive', 'file=%s,if=%s,cache=%s,aio=%s' % (disk_path, disk_type, disk_cache, disk_aio), '-drive', 'file=%s,if=%s,cache=%s,aio=%s' % (disk_path, disk_type, disk_cache, disk_aio),
'-vnc', '%s:1,ipv4,password' % listen_ip, '-vnc', '%s:1,ipv4,password' % listen_ip,
'-boot', 'order=cd,menu=on', '-boot', 'order=cd,menu=on',
......
[buildout]
parts =
instance
eggs-directory = ${buildout:eggs-directory}
develop-eggs-directory = ${buildout:develop-eggs-directory}
[instance]
recipe = ${instance-recipe:egg}:${instance-recipe:module}
dcrond_binary = ${dcron:location}/sbin/crond
memcached_binary = ${memcached:location}/bin/memcached
openssl_binary = ${openssl:location}/bin/openssl
rdiff_backup_binary = ${buildout:bin-directory}/rdiff-backup
stunnel_binary = ${stunnel:location}/bin/stunnel
[buildout]
extends =
../../component/memcached/buildout.cfg
../../component/dcron/buildout.cfg
../../component/stunnel/buildout.cfg
../../component/rdiff-backup/buildout.cfg
../../component/lxml-python/buildout.cfg
../../stack/slapos.cfg
parts =
# Create instance template
template
eggs
instance-recipe-egg
[instance-recipe]
egg = slapos.cookbook
module = memcached
[instance-recipe-egg]
recipe = zc.recipe.egg
eggs = ${instance-recipe:egg}
[eggs]
recipe = zc.recipe.egg
eggs =
${lxml-python:egg}
[template]
# Default template for the instance.
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance.cfg
md5sum = 837caf9897332a5f70c72438f1dc5bae
output = ${buildout:directory}/template.cfg
mode = 0644
[versions]
plone.recipe.command = 1.1
slapos.recipe.template = 2.3
...@@ -17,7 +17,7 @@ parts += ...@@ -17,7 +17,7 @@ parts +=
versions = versions versions = versions
[versions] [versions]
slapos.recipe.template = 4.2 slapos.recipe.template = 4.3
[template-instance] [template-instance]
......
...@@ -116,8 +116,8 @@ mysqlclient = 1.3.12 ...@@ -116,8 +116,8 @@ mysqlclient = 1.3.12
persistent = 4.2.3 persistent = 4.2.3
pycrypto = 2.6.1 pycrypto = 2.6.1
pycurl = 7.43.0 pycurl = 7.43.0
slapos.recipe.template = 4.2 slapos.recipe.template = 4.3
slapos.toolbox = 0.73 slapos.toolbox = 0.74
smmap2 = 2.0.1 smmap2 = 2.0.1
transaction = 1.7.0 transaction = 1.7.0
zodbpickle = 0.6.0 zodbpickle = 0.6.0
......
# TODO instance which runs this test periodically automatically and ingests results to ERP5 # NEO test instance: run neotest under Nexedi testing infrastructure
[buildout]
parts = runTestSuite
# std stuff for slapos instance
eggs-directory = {{ buildout['eggs-directory'] }}
develop-eggs-directory = {{ buildout['develop-eggs-directory'] }}
offline = true
# software release we instantiate was supplied here
[software]
dir = {{ buildout['directory'] }}
bin = {{ buildout['bin-directory'] }}
# instance directories
[directory]
recipe = slapos.cookbook:mkdirectory
home = ${buildout:directory}
bin = ${:home}/bin
neotest = ${:home}/neotest
# script to run the testsuite from inside <instance>/neotest/
# located @ <instance>/bin/runTestSuite so testnode can see this as run tests entrypoint.
[runTestSuite]
recipe = slapos.cookbook:wrapper
wrapper-path = ${directory:bin}/${:_buildout_section_name_}
command-line = /bin/bash -c 'cd ${directory:neotest} && ${software:bin}/neotest-runTestSuite "$@"' runTestSuite
# vvv appends "$@" to argv ^^^ without shell-escaping
parameters-extra = true
...@@ -2,6 +2,8 @@ ...@@ -2,6 +2,8 @@
. ${gowork:env.sh} . ${gowork:env.sh}
PATH="${coreutils:location}/bin:$PATH"
PATH="${util-linux:location}/bin:$PATH"
PATH="${ethtool:location}/sbin:$PATH" PATH="${ethtool:location}/sbin:$PATH"
PATH="${ioping:location}/bin:$PATH" PATH="${ioping:location}/bin:$PATH"
PATH="${lmbench:location}/bin:$PATH" PATH="${lmbench:location}/bin:$PATH"
......
#!/bin/bash -e
# neotest's runTestSuite wraper so it could be run without any environment preset
. ${buildout:directory}/neotest-env.sh
exec ${gowork:src}/lab.nexedi.com/kirr/neo/go/neo/t/nxd/runTestSuite "$@"
...@@ -12,6 +12,8 @@ extends = ...@@ -12,6 +12,8 @@ extends =
../../component/ethtool/buildout.cfg ../../component/ethtool/buildout.cfg
../../component/ioping/buildout.cfg ../../component/ioping/buildout.cfg
../../component/lmbench/buildout.cfg ../../component/lmbench/buildout.cfg
../../component/coreutils/buildout.cfg
../../component/util-linux/buildout.cfg
parts = parts =
gowork gowork
...@@ -19,6 +21,11 @@ parts = ...@@ -19,6 +21,11 @@ parts =
ioping ioping
ethtool ethtool
# neotest uses realpath & friends
coreutils
# ----//---- lsblk
util-linux
neoppod-develop neoppod-develop
neoppod neoppod
wendelin.core-dev wendelin.core-dev
...@@ -26,11 +33,12 @@ parts = ...@@ -26,11 +33,12 @@ parts =
neotest-env.sh neotest-env.sh
neotest neotest
neotest-runTestSuite
# for instance # for instance
slapos-deps-eggs slapos-deps-eggs
slapos-cookbook slapos-cookbook
# instance.cfg instance.cfg
# go packages to install (+ automatically their dependencies) # go packages to install (+ automatically their dependencies)
...@@ -53,7 +61,7 @@ output = ${buildout:directory}/${:_buildout_section_name_} ...@@ -53,7 +61,7 @@ output = ${buildout:directory}/${:_buildout_section_name_}
[neotest-env.sh] [neotest-env.sh]
<= buildout-template <= buildout-template
md5sum = 6e2203c7a86e8a3e8e56b8086115f5d4 md5sum = 595eebbbcb56b6a8464d48833cfae57b
[neotest] [neotest]
<= buildout-template <= buildout-template
...@@ -61,11 +69,25 @@ output = ${buildout:bin-directory}/${:_buildout_section_name_} ...@@ -61,11 +69,25 @@ output = ${buildout:bin-directory}/${:_buildout_section_name_}
mode = 0755 mode = 0755
md5sum = fb3b4109128c1db1739ef5bb6abd1d94 md5sum = fb3b4109128c1db1739ef5bb6abd1d94
[neotest-runTestSuite]
<= buildout-template
output = ${buildout:bin-directory}/${:_buildout_section_name_}
mode = 0755
md5sum = 6a4281730b68cdba5c873817a6754428
# instance
[jinja2-template]
recipe = slapos.recipe.template:jinja2
template= ${:_profile_base_location_}/${:_buildout_section_name_}.in
rendered= ${buildout:directory}/${:_buildout_section_name_}
mode = 0644
context =
section buildout buildout
# instance (TODO)
[instance.cfg] [instance.cfg]
<= buildout-template <= jinja2-template
md5sum = 410e1b2d72829824b28cc0299adb472e md5sum = 7bd68ed8842cd25301bf04bcdcef88f6
# eggs: # eggs:
...@@ -80,6 +102,8 @@ eggs = ...@@ -80,6 +102,8 @@ eggs =
wendelin.core wendelin.core
# for ZEO scripts (runzeo) # for ZEO scripts (runzeo)
ZEO ZEO
# for nxd/runTestSuite
erp5.util
# wendelin.core: latest not yet released # wendelin.core: latest not yet released
...@@ -92,6 +116,7 @@ pyasn1 = 0.3.7 ...@@ -92,6 +116,7 @@ pyasn1 = 0.3.7
ZODB3 = 3.11.0 ZODB3 = 3.11.0
numpy = 1.13.3 numpy = 1.13.3
zope.testing = 4.6.2 zope.testing = 4.6.2
erp5.util = 0.4.50
# Required by: # Required by:
# ZEO==4.3.1 # ZEO==4.3.1
...@@ -103,7 +128,3 @@ ZConfig = 3.2.0 ...@@ -103,7 +128,3 @@ ZConfig = 3.2.0
# ZEO==4.3.1 # ZEO==4.3.1
# ZODB==4.4.5 # ZODB==4.4.5
zc.lockfile = 1.2.1 zc.lockfile = 1.2.1
# Required by:
# slapos.toolbox==0.73
erp5.util = 0.4.49
...@@ -45,5 +45,18 @@ output = ${buildout:directory}/instance-nginx.cfg.in ...@@ -45,5 +45,18 @@ output = ${buildout:directory}/instance-nginx.cfg.in
mode = 0644 mode = 0644
[versions] [versions]
slapos.recipe.template = 4.2 slapos.recipe.template = 4.3
slapos.toolbox = 0.73 slapos.toolbox = 0.74
dnspython = 1.15.0
PyRSS2Gen = 1.1
erp5.util = 0.4.50
passlib = 1.7.1
GitPython = 2.1.8
lockfile = 0.12.2
apache-libcloud = 2.2.1
feedparser = 5.2.1
atomize = 0.2.0
inotifyx = 0.2.2
gitdb2 = 2.0.3
pyasn1 = 0.4.2
smmap2 = 2.0.3
...@@ -81,7 +81,7 @@ username = admin ...@@ -81,7 +81,7 @@ username = admin
[monitor-instance-parameter] [monitor-instance-parameter]
monitor-httpd-port = 8099 monitor-httpd-port = 8099
cors-domains = {{ slapparameter_dict.get('monitor-cors-domains', 'monitor.app.officejs.com') }} cors-domains = {{ slapparameter_dict.get('monitor-cors-domains', 'monitor.app.officejs.com') }}
username = ${monitor-instance-parameter:username} username = admin
password = ${monitor-htpasswd:passwd} password = ${monitor-htpasswd:passwd}
[monitor-conf-parameters] [monitor-conf-parameters]
...@@ -109,7 +109,7 @@ config-{{parameter}} = {{ value }} ...@@ -109,7 +109,7 @@ config-{{parameter}} = {{ value }}
{% endfor -%} {% endfor -%}
config-{{ slave_list_name }} = {{ json_module.dumps(slave_instance_list) }} config-{{ slave_list_name }} = {{ json_module.dumps(slave_instance_list) }}
config-monitor-cors-domains = {{ slapparameter_dict.get('monitor-cors-domains', 'monitor.node.vifib.com') }} config-monitor-cors-domains = {{ slapparameter_dict.get('monitor-cors-domains', 'monitor.node.vifib.com') }}
config-monitor-username = ${monitor-instance-parameter:username} config-monitor-username = admin
config-monitor-password = ${monitor-htpasswd:passwd} config-monitor-password = ${monitor-htpasswd:passwd}
[publish-information] [publish-information]
......
...@@ -38,7 +38,7 @@ mode = 640 ...@@ -38,7 +38,7 @@ mode = 640
[template-dns-replicate] [template-dns-replicate]
recipe = slapos.recipe.build:download recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/instance-powerdns-replicate.cfg.jinja2 url = ${:_profile_base_location_}/instance-powerdns-replicate.cfg.jinja2
md5sum = 3699d0751632e0c11b466342789e27a3 md5sum = 46acd4ed071df8d7139dcd0434be42eb
mode = 0644 mode = 0644
[iso-list] [iso-list]
...@@ -65,4 +65,4 @@ mode = 0644 ...@@ -65,4 +65,4 @@ mode = 0644
PyRSS2Gen = 1.1 PyRSS2Gen = 1.1
cns.recipe.symlink = 0.2.3 cns.recipe.symlink = 0.2.3
plone.recipe.command = 1.1 plone.recipe.command = 1.1
slapos.recipe.template = 4.2 slapos.recipe.template = 4.3
...@@ -110,8 +110,8 @@ gitdb = 0.6.4 ...@@ -110,8 +110,8 @@ gitdb = 0.6.4
plone.recipe.command = 1.1 plone.recipe.command = 1.1
pycrypto = 2.6.1 pycrypto = 2.6.1
pycurl = 7.43.0 pycurl = 7.43.0
slapos.recipe.template = 4.2 slapos.recipe.template = 4.3
slapos.toolbox = 0.73 slapos.toolbox = 0.74
smmap = 0.9.0 smmap = 0.9.0
# Required by: # Required by:
......
...@@ -47,4 +47,4 @@ mode = 0644 ...@@ -47,4 +47,4 @@ mode = 0644
[versions] [versions]
plone.recipe.command = 1.1 plone.recipe.command = 1.1
slapos.recipe.template = 4.2 slapos.recipe.template = 4.3
...@@ -11,4 +11,4 @@ extends = common.cfg ...@@ -11,4 +11,4 @@ extends = common.cfg
Pygments = 1.6 Pygments = 1.6
collective.recipe.environment = 0.2.0 collective.recipe.environment = 0.2.0
collective.recipe.template = 1.10 collective.recipe.template = 1.10
slapos.recipe.template = 4.2 slapos.recipe.template = 4.3
...@@ -15,7 +15,7 @@ ...@@ -15,7 +15,7 @@
# not need these here). # not need these here).
[template-erp5] [template-erp5]
filename = instance-erp5.cfg.in filename = instance-erp5.cfg.in
md5sum = 4a77ee4a6367fee27552f8bfe9d87aab md5sum = 7a1d2fe310764f5c06f636e8850c7d63
[template-balancer] [template-balancer]
filename = instance-balancer.cfg.in filename = instance-balancer.cfg.in
......
...@@ -206,6 +206,7 @@ name = {{ partition_name }} ...@@ -206,6 +206,7 @@ name = {{ partition_name }}
{{ root_common.sla(partition_name) }} {{ root_common.sla(partition_name) }}
config-name = {{ dumps(custom_name) }} config-name = {{ dumps(custom_name) }}
config-instance-count = {{ dumps(zope_parameter_dict.get('instance-count', 1)) }} config-instance-count = {{ dumps(zope_parameter_dict.get('instance-count', 1)) }}
config-private-dev-shm = {{ zope_parameter_dict.get('private-dev-shm', '') }}
config-thread-amount = {{ dumps(zope_parameter_dict.get('thread-amount', 4)) }} config-thread-amount = {{ dumps(zope_parameter_dict.get('thread-amount', 4)) }}
config-timerserver-interval = {{ dumps(zope_parameter_dict.get('timerserver-interval', 5)) }} config-timerserver-interval = {{ dumps(zope_parameter_dict.get('timerserver-interval', 5)) }}
config-longrequest-logger-interval = {{ dumps(zope_parameter_dict.get('longrequest-logger-interval', -1)) }} config-longrequest-logger-interval = {{ dumps(zope_parameter_dict.get('longrequest-logger-interval', -1)) }}
...@@ -327,7 +328,7 @@ config-backend-path-dict = {{ dumps(zope_backend_path_dict) }} ...@@ -327,7 +328,7 @@ config-backend-path-dict = {{ dumps(zope_backend_path_dict) }}
config-ssl-authentication-dict = {{ dumps(ssl_authentication_dict) }} config-ssl-authentication-dict = {{ dumps(ssl_authentication_dict) }}
config-apachedex-promise-threshold = {{ dumps(monitor_dict.get('apachedex-promise-threshold', 70)) }} config-apachedex-promise-threshold = {{ dumps(monitor_dict.get('apachedex-promise-threshold', 70)) }}
config-apachedex-configuration = {{ dumps(monitor_dict.get('apachedex-configuration', config-apachedex-configuration = {{ dumps(monitor_dict.get('apachedex-configuration',
'--erp5-base "/erp5(/|$|/\?)" --skip-user-agent Zabbix --error-detail --js-embed --quiet')) }} '--erp5-base +erp5 .*/VirtualHostRoot/erp5(/|\\?|$) --base +other / --skip-user-agent Zabbix --error-detail --js-embed --quiet')) }}
[request-frontend-base] [request-frontend-base]
{% if has_frontend -%} {% if has_frontend -%}
......
...@@ -97,4 +97,4 @@ mode = 640 ...@@ -97,4 +97,4 @@ mode = 640
Pygments = 2.1.3 Pygments = 2.1.3
collective.recipe.template = 1.10 collective.recipe.template = 1.10
plone.recipe.command = 1.1 plone.recipe.command = 1.1
slapos.recipe.template = 4.2 slapos.recipe.template = 4.3
...@@ -19,7 +19,7 @@ md5sum = 713db528880282d568278f09458d2aab ...@@ -19,7 +19,7 @@ md5sum = 713db528880282d568278f09458d2aab
[template-runner] [template-runner]
filename = instance-runner.cfg filename = instance-runner.cfg
md5sum = 7df86928aeef0fbada832b0283b9a454 md5sum = 8b1caca52ab1307343ebada59f5a6c66
[template-runner-import-script] [template-runner-import-script]
filename = template/runner-import.sh.jinja2 filename = template/runner-import.sh.jinja2
...@@ -35,7 +35,7 @@ md5sum = 98ce179badc6af5979a64a7c3d0a2ceb ...@@ -35,7 +35,7 @@ md5sum = 98ce179badc6af5979a64a7c3d0a2ceb
[instance-runner-export] [instance-runner-export]
filename = instance-runner-export.cfg.in filename = instance-runner-export.cfg.in
md5sum = 00ddcaa6e571505d1930612d12de8fde md5sum = d1122988652127af5e252f1503eeacab
[template-resilient] [template-resilient]
filename = instance-resilient.cfg.jinja2 filename = instance-resilient.cfg.jinja2
......
...@@ -7,11 +7,13 @@ parts += ...@@ -7,11 +7,13 @@ parts +=
nginx-launcher nginx-launcher
certificate-authority certificate-authority
ca-nginx ca-nginx
logrotate-entry-nginx
gunicorn-launcher gunicorn-launcher
gunicorn-graceful gunicorn-graceful
publish-connection-information publish-connection-information
slaprunner-promise slaprunner-promise
apache-httpd-promise apache-httpd-promise
logrotate-entry-apache-httpd
slaprunner-supervisord-wrapper slaprunner-supervisord-wrapper
runner-sshd-add-authorized-key runner-sshd-add-authorized-key
runner-sshd-graceful runner-sshd-graceful
...@@ -30,6 +32,7 @@ parts += ...@@ -30,6 +32,7 @@ parts +=
bash-profile bash-profile
supervisord-wrapper supervisord-wrapper
supervisord-promise supervisord-promise
logrotate-entry-supervisord
httpd-graceful-wrapper httpd-graceful-wrapper
## Monitoring part ## Monitoring part
## Monitor for runner ## Monitor for runner
......
...@@ -4,10 +4,12 @@ parts = ...@@ -4,10 +4,12 @@ parts =
nginx-launcher nginx-launcher
certificate-authority certificate-authority
ca-nginx ca-nginx
logrotate-entry-nginx
gunicorn-launcher gunicorn-launcher
gunicorn-graceful gunicorn-graceful
publish-connection-information publish-connection-information
slaprunner-promise slaprunner-promise
logrotate-entry-apache-httpd
apache-httpd-promise apache-httpd-promise
slaprunner-supervisord-wrapper slaprunner-supervisord-wrapper
runner-sshd-add-authorized-key runner-sshd-add-authorized-key
...@@ -27,6 +29,7 @@ parts = ...@@ -27,6 +29,7 @@ parts =
bash-profile bash-profile
supervisord-wrapper supervisord-wrapper
supervisord-promise supervisord-promise
logrotate-entry-supervisord
httpd-graceful-wrapper httpd-graceful-wrapper
{% if slapparameter_dict.get('no-ipv4-frontend', 'false') == 'false' %} {% if slapparameter_dict.get('no-ipv4-frontend', 'false') == 'false' %}
slaprunner-frontend-promise slaprunner-frontend-promise
...@@ -347,6 +350,12 @@ mode = 700 ...@@ -347,6 +350,12 @@ mode = 700
context = context =
section param_nginx_frontend nginx-frontend section param_nginx_frontend nginx-frontend
[logrotate-entry-nginx]
<= logrotate-entry-base
name = nginx
log = $${directory:log}/nginx.access.log $${directory:log}/nginx.error.log
post = kill -SIGUSR1 $(cat $${buildout:directory}/var/run/nginx.pid)
[httpd-parameters] [httpd-parameters]
path_pid = $${directory:run}/httpd.pid path_pid = $${directory:run}/httpd.pid
path_error_log = $${directory:log}/httpd-error.log path_error_log = $${directory:log}/httpd-error.log
...@@ -387,6 +396,12 @@ wait-for-files = ...@@ -387,6 +396,12 @@ wait-for-files =
$${ca-nginx:cert-file} $${ca-nginx:cert-file}
$${ca-nginx:key-file} $${ca-nginx:key-file}
[logrotate-entry-apache-httpd]
<= logrotate-entry-base
name = apache
log = $${directory:log}/httpd-access.log $${directory:log}/httpd-error.log
post = test ! -s $${buildout:directory}/var/run/httpd.pid || $${buildout:directory}/bin/slapos-kill --pidfile $${buildout:directory}/var/run/httpd.pid -s USR1
[httpd-graceful-wrapper] [httpd-graceful-wrapper]
recipe = collective.recipe.template recipe = collective.recipe.template
input = inline: input = inline:
...@@ -777,6 +792,12 @@ recipe = slapos.cookbook:wrapper ...@@ -777,6 +792,12 @@ recipe = slapos.cookbook:wrapper
command-line = $${buildout:directory}/bin/supervisord -c $${supervisord-conf:rendered} --nodaemon command-line = $${buildout:directory}/bin/supervisord -c $${supervisord-conf:rendered} --nodaemon
wrapper-path = $${directory:services}/supervisord wrapper-path = $${directory:services}/supervisord
[logrotate-entry-supervisord]
<= logrotate-entry-base
name = supervisord
log = $${directory:log}/slapproxy.log $${directory:log}/supervisord.log $${directory:log}/supervisord-errors.log
post = kill -SIGUSR2 $(cat $${buildout:directory}/var/run/supervisord.pid)
[supervisord-promise] [supervisord-promise]
recipe = slapos.cookbook:check_port_listening recipe = slapos.cookbook:check_port_listening
path = $${directory:promises}/supervisord path = $${directory:promises}/supervisord
......
...@@ -15,8 +15,8 @@ gitdb = 0.6.4 ...@@ -15,8 +15,8 @@ gitdb = 0.6.4
gunicorn = 19.7.1 gunicorn = 19.7.1
prettytable = 0.7.2 prettytable = 0.7.2
pycurl = 7.43.0 pycurl = 7.43.0
slapos.recipe.template = 4.2 slapos.recipe.template = 4.3
slapos.toolbox = 0.73 slapos.toolbox = 0.74
smmap = 0.9.0 smmap = 0.9.0
# Required by: # Required by:
......
...@@ -59,7 +59,7 @@ eggs = collective.recipe.template ...@@ -59,7 +59,7 @@ eggs = collective.recipe.template
collective.recipe.template = 1.11 collective.recipe.template = 1.11
plone.recipe.command = 1.1 plone.recipe.command = 1.1
slapos.recipe.build = 0.28 slapos.recipe.build = 0.28
slapos.recipe.template = 4.2 slapos.recipe.template = 4.3
# Replicate slapos stack, but without shacache to not have to compile the entire world for a simple test. # Replicate slapos stack, but without shacache to not have to compile the entire world for a simple test.
[buildout] [buildout]
......
...@@ -62,4 +62,4 @@ md5sum = 0ea12a4ad2d2e3d406476e35b8d3e3fb ...@@ -62,4 +62,4 @@ md5sum = 0ea12a4ad2d2e3d406476e35b8d3e3fb
mode = 640 mode = 640
[versions] [versions]
slapos.recipe.template = 4.2 slapos.recipe.template = 4.3
...@@ -10,12 +10,15 @@ ...@@ -10,12 +10,15 @@
# $x 8.787 0 # $x 8.787 0
# 0.100036621094 # 0.100036621094
# 6556 # 6556
#
# The 'start_process' command is similar but by growing a ZBigArray object.
# The random data has a compression ratio of 10%.
[buildout] [buildout]
extends = test-common.cfg extends = test-common.cfg
parts += start_ingest parts += start_ingest start_process
[start_ingest] [start-script-common]
recipe = slapos.recipe.template:jinja2 recipe = slapos.recipe.template:jinja2
rendered = ${buildout:bin-directory}/${:_buildout_section_name_} rendered = ${buildout:bin-directory}/${:_buildout_section_name_}
mode = 0755 mode = 0755
...@@ -27,20 +30,35 @@ template = ...@@ -27,20 +30,35 @@ template =
_('--site-id', default='erp5') _('--site-id', default='erp5')
_('hostport', metavar='host[:port]', help='Zope address') _('hostport', metavar='host[:port]', help='Zope address')
_('password', help="'zope' user password") _('password', help="'zope' user password")
_('reference', help='Data Stream reference') options = []
_('mu', type=float) def option(name, **kw):
_('sigma', type=float) _(name, **kw)
_('chunks_per_transaction', nargs='?', type=int, help='default: 128 (8 MiB)') options.append(name)
${:options}
args = parser.parse_args() args = parser.parse_args()
qs = [] qs = []
for k in 'reference', 'mu', 'sigma', 'chunks_per_transaction': for k in options:
v = getattr(args, k) v = getattr(args, k)
if v is not None: if v is not None:
t = type(v) t = type(v)
qs.append('%s=%s' % (k if t is str else k + ':' + t.__name__, v)) qs.append('%s=%s' % (k if t is str else k + ':' + t.__name__, v))
c = httplib.HTTPConnection(args.hostport) c = httplib.HTTPConnection(args.hostport)
c.putrequest('GET', '/%s/ERP5Site_simulateFluentdIngestion?%s' c.putrequest('GET', '/%s/${:script}?%s' % (args.site_id, '&'.join(qs)))
% (args.site_id, '&'.join(qs)))
c.putheader('Authorization', c.putheader('Authorization',
'Basic ' + base64.b64encode('zope:'+args.password)) 'Basic ' + base64.b64encode('zope:'+args.password))
c.endheaders() c.endheaders()
[start_ingest]
<= start-script-common
options =
option('id', help='Data Stream id')
option('mu', type=float)
option('sigma', type=float)
option('chunks_per_transaction', nargs='?', type=int, help='default: 128 (8 MiB)')
script = ERP5Site_simulateFluentdIngestion
[start_process]
<= start-script-common
options =
option('id', help='Data Array id')
script = ERP5Site_dummyZBigArrayProcessing
...@@ -5,7 +5,7 @@ from random import lognormvariate ...@@ -5,7 +5,7 @@ from random import lognormvariate
bigfile_chunk_size = 65536 bigfile_chunk_size = 65536
def simulateFluentdIngestion(self, reference, mu, sigma, def simulateFluentdIngestion(self, id, mu, sigma,
chunks_per_transaction=128): chunks_per_transaction=128):
from time import time from time import time
import transaction import transaction
...@@ -13,9 +13,9 @@ def simulateFluentdIngestion(self, reference, mu, sigma, ...@@ -13,9 +13,9 @@ def simulateFluentdIngestion(self, reference, mu, sigma,
+ '/ingest') + '/ingest')
module = self['data_stream_module'] module = self['data_stream_module']
try: try:
data_stream = module[reference] data_stream = module[id]
except KeyError: except KeyError:
data_stream = module.newContent(reference, 'Data Stream') data_stream = module.newContent(id, 'Data Stream')
transaction.commit() transaction.commit()
pack = struct.Struct('!d').pack pack = struct.Struct('!d').pack
......
def dummyZBigArrayProcessing(self, id):
import numpy as np
from random import randrange, sample
import transaction
module = self['data_array_module']
try:
array = module[id]
except KeyError:
array = module.newContent(id, 'Data Array')
array.initArray(shape=(0, 64), dtype=np.int32)
transaction.commit()
note = array.getPath() + '/new_data'
array = array.getArray()
rows, cols = array.shape
y = xrange(cols)
n = 10 * (2<<20) // (cols*4)
z = np.ndarray(shape=(n, cols), dtype=array.dtype)
for row in z:
for i in sample(y, 8):
row[i] = randrange(0, 1000)
while 1:
txn = transaction.begin()
np.random.shuffle(z)
rows += n
array.resize((rows, cols))
array[-n:] = z
txn.note(note)
txn.commit()
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="Extension Component" module="erp5.portal_type"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>_recorded_property_dict</string> </key>
<value>
<persistent> <string encoding="base64">AAAAAAAAAAI=</string> </persistent>
</value>
</item>
<item>
<key> <string>default_reference</string> </key>
<value> <string>ScalabilityZBigArray</string> </value>
</item>
<item>
<key> <string>description</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>extension.erp5.ScalabilityZBigArray</string> </value>
</item>
<item>
<key> <string>portal_type</string> </key>
<value> <string>Extension Component</string> </value>
</item>
<item>
<key> <string>sid</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>text_content_error_message</string> </key>
<value>
<tuple/>
</value>
</item>
<item>
<key> <string>text_content_warning_message</string> </key>
<value>
<tuple/>
</value>
</item>
<item>
<key> <string>version</string> </key>
<value> <string>erp5</string> </value>
</item>
<item>
<key> <string>workflow_history</string> </key>
<value>
<persistent> <string encoding="base64">AAAAAAAAAAM=</string> </persistent>
</value>
</item>
</dictionary>
</pickle>
</record>
<record id="2" aka="AAAAAAAAAAI=">
<pickle>
<global name="PersistentMapping" module="Persistence.mapping"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>data</string> </key>
<value>
<dictionary/>
</value>
</item>
</dictionary>
</pickle>
</record>
<record id="3" aka="AAAAAAAAAAM=">
<pickle>
<global name="PersistentMapping" module="Persistence.mapping"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>data</string> </key>
<value>
<dictionary>
<item>
<key> <string>component_validation_workflow</string> </key>
<value>
<persistent> <string encoding="base64">AAAAAAAAAAQ=</string> </persistent>
</value>
</item>
</dictionary>
</value>
</item>
</dictionary>
</pickle>
</record>
<record id="4" aka="AAAAAAAAAAQ=">
<pickle>
<global name="WorkflowHistoryList" module="Products.ERP5Type.patches.WorkflowTool"/>
</pickle>
<pickle>
<tuple>
<none/>
<list>
<dictionary>
<item>
<key> <string>action</string> </key>
<value> <string>validate</string> </value>
</item>
<item>
<key> <string>validation_state</string> </key>
<value> <string>validated</string> </value>
</item>
</dictionary>
</list>
</tuple>
</pickle>
</record>
</ZopeData>
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="ExternalMethod" module="Products.ExternalMethod.ExternalMethod"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>_function</string> </key>
<value> <string>dummyZBigArrayProcessing</string> </value>
</item>
<item>
<key> <string>_module</string> </key>
<value> <string>ScalabilityZBigArray</string> </value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>ERP5Site_dummyZBigArrayProcessing</string> </value>
</item>
<item>
<key> <string>title</string> </key>
<value> <string></string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
...@@ -54,7 +54,7 @@ ...@@ -54,7 +54,7 @@
</item> </item>
<item> <item>
<key> <string>id</string> </key> <key> <string>id</string> </key>
<value> <string>DataStreamModule_getTotalSize</string> </value> <value> <string>Module_getTotalSize</string> </value>
</item> </item>
</dictionary> </dictionary>
</pickle> </pickle>
......
extension.erp5.ScalabilityFluentd extension.erp5.ScalabilityFluentd
extension.erp5.ScalabilityZBigArray
\ No newline at end of file
portal_ingestion_policies/scalability_test_* portal_ingestion_policies/scalability_test_*
portal_skins/custom/DataStreamModule_getTotalSize portal_skins/custom/ERP5Site_dummyZBigArrayProcessing
portal_skins/custom/ERP5Site_handleRawDataFluentdIngestion portal_skins/custom/ERP5Site_handleRawDataFluentdIngestion
portal_skins/custom/ERP5Site_simulateFluentdIngestion portal_skins/custom/ERP5Site_simulateFluentdIngestion
portal_skins/custom/Module_getTotalSize
\ No newline at end of file
...@@ -94,8 +94,8 @@ caucase = 0.1.4 ...@@ -94,8 +94,8 @@ caucase = 0.1.4
futures = 3.1.1 futures = 3.1.1
gitdb2 = 2.0.2 gitdb2 = 2.0.2
gunicorn = 19.7.1 gunicorn = 19.7.1
slapos.recipe.template = 4.2 slapos.recipe.template = 4.3
slapos.toolbox = 0.73 slapos.toolbox = 0.74
smmap2 = 2.0.3 smmap2 = 2.0.3
# Required by: # Required by:
......
...@@ -89,7 +89,7 @@ PasteScript = 2.0.2 ...@@ -89,7 +89,7 @@ PasteScript = 2.0.2
WSGIUtils = 0.7 WSGIUtils = 0.7
python-magic = 0.4.6 python-magic = 0.4.6
rdiff-backup = 1.0.5+SlapOSPatched001 rdiff-backup = 1.0.5+SlapOSPatched001
slapos.recipe.template = 4.2 slapos.recipe.template = 4.3
# Required by: # Required by:
# PasteScript==2.0 # PasteScript==2.0
......
...@@ -63,6 +63,7 @@ extends = ...@@ -63,6 +63,7 @@ extends =
../../component/findutils/buildout.cfg ../../component/findutils/buildout.cfg
../../component/userhosts/buildout.cfg ../../component/userhosts/buildout.cfg
../../component/postfix/buildout.cfg ../../component/postfix/buildout.cfg
../../component/zbarlight/buildout.cfg
../monitor/buildout.cfg ../monitor/buildout.cfg
../../software/caucase/software.cfg ../../software/caucase/software.cfg
../../software/jupyter/software.cfg ../../software/jupyter/software.cfg
...@@ -465,6 +466,7 @@ eggs = ${neoppod:eggs} ...@@ -465,6 +466,7 @@ eggs = ${neoppod:eggs}
${h5py:egg} ${h5py:egg}
openpyxl openpyxl
${statsmodels:egg} ${statsmodels:egg}
${zbarlight:egg}
lock_file lock_file
astor astor
APacheDEX APacheDEX
...@@ -730,7 +732,7 @@ spyne = 2.12.14 ...@@ -730,7 +732,7 @@ spyne = 2.12.14
suds = 0.4 suds = 0.4
facebook-sdk = 2.0.0 facebook-sdk = 2.0.0
threadframe = 0.2 threadframe = 0.2
timerserver = 2.0.2 timerserver = 2.0.4
urlnorm = 1.1.4 urlnorm = 1.1.4
uuid = 1.30 uuid = 1.30
validictory = 1.1.0 validictory = 1.1.0
......
...@@ -19,7 +19,7 @@ md5sum = c1f1083bf6c911a0e65dcb841fba327d ...@@ -19,7 +19,7 @@ md5sum = c1f1083bf6c911a0e65dcb841fba327d
[mariadb-slow-query-report-script] [mariadb-slow-query-report-script]
filename = mysql-querydigest.sh.in filename = mysql-querydigest.sh.in
md5sum = cfe6ab8ae54a521ecb269e9d9762cbeb md5sum = 0c0d98a68230cd0ad36046bb25b35f4a
[mariadb-start-clone-from-backup] [mariadb-start-clone-from-backup]
filename = instance-mariadb-start-clone-from-backup.sh.in filename = instance-mariadb-start-clone-from-backup.sh.in
...@@ -27,7 +27,7 @@ md5sum = 1af531c51f575a1d1362f2ca2d61620d ...@@ -27,7 +27,7 @@ md5sum = 1af531c51f575a1d1362f2ca2d61620d
[template-mariadb] [template-mariadb]
filename = instance-mariadb.cfg.in filename = instance-mariadb.cfg.in
md5sum = 244e185ed26831fe0b5abf49c67753d6 md5sum = da7c36ecb490b67360d2afda94b41bff
[template-kumofs] [template-kumofs]
filename = instance-kumofs.cfg.in filename = instance-kumofs.cfg.in
...@@ -79,7 +79,7 @@ md5sum = d41d8cd98f00b204e9800998ecf8427e ...@@ -79,7 +79,7 @@ md5sum = d41d8cd98f00b204e9800998ecf8427e
[template-erp5] [template-erp5]
filename = instance-erp5.cfg.in filename = instance-erp5.cfg.in
md5sum = 2465af81147af322056cee9f6c7de14f md5sum = 1d6735a803c9d28930bf2ad00706c06b
[template-zeo] [template-zeo]
filename = instance-zeo.cfg.in filename = instance-zeo.cfg.in
...@@ -87,11 +87,11 @@ md5sum = d1f33d406d528ae27d973e2dd0efb1ba ...@@ -87,11 +87,11 @@ md5sum = d1f33d406d528ae27d973e2dd0efb1ba
[template-zope] [template-zope]
filename = instance-zope.cfg.in filename = instance-zope.cfg.in
md5sum = fd7e8c507cef1950e6c0347ce2a01021 md5sum = e08c00d5973916d796bf08aa78dba34a
[template-balancer] [template-balancer]
filename = instance-balancer.cfg.in filename = instance-balancer.cfg.in
md5sum = f64c568f1365eb1164f12f48fede9a99 md5sum = a71ad387eab681b9020e271cba2c7a79
[template-haproxy-cfg] [template-haproxy-cfg]
filename = haproxy.cfg.in filename = haproxy.cfg.in
......
...@@ -287,7 +287,7 @@ apachedex = ${monitor-directory:private}/apachedex ...@@ -287,7 +287,7 @@ apachedex = ${monitor-directory:private}/apachedex
[{{ section('monitor-generate-apachedex-report') }}] [{{ section('monitor-generate-apachedex-report') }}]
recipe = slapos.cookbook:wrapper recipe = slapos.cookbook:wrapper
wrapper-path = ${monitor-directory:reports}/${:command} wrapper-path = ${monitor-directory:reports}/${:command}
command-line = "{{ parameter_dict['run-apachedex-location'] }}" "{{ parameter_dict['apachedex-location'] }}" "${directory:apachedex}" ${monitor-publish-parameters:monitor-base-url}/private/apachedex --apache-log-list "${apachedex-parameters:apache-log-list}" --config "${apachedex-parameters:configuration}" command-line = "{{ parameter_dict['run-apachedex-location'] }}" "{{ parameter_dict['apachedex-location'] }}" "${directory:apachedex}" ${monitor-publish-parameters:monitor-base-url}/private/apachedex --apache-log-list "${apachedex-parameters:apache-log-list}" --configuration "${apachedex-parameters:configuration}"
command = apachedex_every_23_hour command = apachedex_every_23_hour
[apachedex-parameters] [apachedex-parameters]
......
...@@ -205,6 +205,7 @@ name = {{ partition_name }} ...@@ -205,6 +205,7 @@ name = {{ partition_name }}
{{ root_common.sla(partition_name) }} {{ root_common.sla(partition_name) }}
config-name = {{ dumps(custom_name) }} config-name = {{ dumps(custom_name) }}
config-instance-count = {{ dumps(zope_parameter_dict.get('instance-count', 1)) }} config-instance-count = {{ dumps(zope_parameter_dict.get('instance-count', 1)) }}
config-private-dev-shm = {{ zope_parameter_dict.get('private-dev-shm', '') }}
config-thread-amount = {{ dumps(zope_parameter_dict.get('thread-amount', 4)) }} config-thread-amount = {{ dumps(zope_parameter_dict.get('thread-amount', 4)) }}
config-timerserver-interval = {{ dumps(zope_parameter_dict.get('timerserver-interval', 5)) }} config-timerserver-interval = {{ dumps(zope_parameter_dict.get('timerserver-interval', 5)) }}
config-longrequest-logger-interval = {{ dumps(zope_parameter_dict.get('longrequest-logger-interval', -1)) }} config-longrequest-logger-interval = {{ dumps(zope_parameter_dict.get('longrequest-logger-interval', -1)) }}
...@@ -284,7 +285,7 @@ config-backend-path-dict = {{ dumps(zope_backend_path_dict) }} ...@@ -284,7 +285,7 @@ config-backend-path-dict = {{ dumps(zope_backend_path_dict) }}
config-ssl-authentication-dict = {{ dumps(ssl_authentication_dict) }} config-ssl-authentication-dict = {{ dumps(ssl_authentication_dict) }}
config-apachedex-promise-threshold = {{ dumps(monitor_dict.get('apachedex-promise-threshold', 70)) }} config-apachedex-promise-threshold = {{ dumps(monitor_dict.get('apachedex-promise-threshold', 70)) }}
config-apachedex-configuration = {{ dumps(monitor_dict.get('apachedex-configuration', config-apachedex-configuration = {{ dumps(monitor_dict.get('apachedex-configuration',
'--erp5-base "/erp5(/|$|/\?)" --skip-user-agent Zabbix --error-detail --js-embed --quiet')) }} '--erp5-base +erp5 .*/VirtualHostRoot/erp5(/|\\?|$) --base +other / --skip-user-agent Zabbix --error-detail --js-embed --quiet')) }}
[request-frontend-base] [request-frontend-base]
{% if has_frontend -%} {% if has_frontend -%}
......
...@@ -252,7 +252,7 @@ mariadb-ssl = ${:etc}/mariadb-ssl ...@@ -252,7 +252,7 @@ mariadb-ssl = ${:etc}/mariadb-ssl
var = ${buildout:directory}/var var = ${buildout:directory}/var
log = ${:var}/log log = ${:var}/log
run = ${:var}/run run = ${:var}/run
slowquery = ${monitor-directory:private}/slowquerydex slowquery = ${monitor-directory:private}/slowquery_digest
[{{ section('resiliency-exclude-file') }}] [{{ section('resiliency-exclude-file') }}]
# Generate rdiff exclude file in case of resiliency # Generate rdiff exclude file in case of resiliency
......
...@@ -206,6 +206,7 @@ environment = ...@@ -206,6 +206,7 @@ environment =
{% endif %} {% endif %}
parameters-extra = true parameters-extra = true
command-line = '{{ parameter_dict['userhosts'] }}' '{{ bin_directory }}/runzope' -C '${:configuration-file}' command-line = '{{ parameter_dict['userhosts'] }}' '{{ bin_directory }}/runzope' -C '${:configuration-file}'
private-dev-shm = {{ slapparameter_dict['private-dev-shm'] }}
[{{ section('zcml') }}] [{{ section('zcml') }}]
recipe = slapos.cookbook:copyfilelist recipe = slapos.cookbook:copyfilelist
......
...@@ -7,19 +7,20 @@ PT_QUERY_EXEC='{{pt_query_exec}}' ...@@ -7,19 +7,20 @@ PT_QUERY_EXEC='{{pt_query_exec}}'
if [ ! -d "$OUTPUT_FOLDER" ]; then if [ ! -d "$OUTPUT_FOLDER" ]; then
echo "ERROR: output_folder don't exists" echo "ERROR: output_folder don't exists"
exit 0 exit 1
fi fi
OUTPUT_FILE=${OUTPUT_FOLDER}/slowquery_digest.txt dashed_today=$(date +%Y-%m-%d)
today=$(date -d "$dashed_today" +%Y%m%d)
TODAY=`date +%Y%m%d`
SLOW_LOG=$SLOW_QUERY_PATH-$TODAY SLOW_LOG="$SLOW_QUERY_PATH-$today"
OUTPUT_FILE="$OUTPUT_FOLDER/slowquery_digest.txt-$dashed_today"
if [ ! -f "$SLOW_LOG" ]; then if [ ! -f "$SLOW_LOG" ]; then
echo "ERROR: cannot read mysql slow query log file $SLOW_LOG. Exiting." echo "ERROR: cannot read mysql slow query log file $SLOW_LOG. Exiting."
exit 1 exit 1
fi fi
$PT_QUERY_EXEC $SLOW_LOG > $OUTPUT_FILE "$PT_QUERY_EXEC" "$SLOW_LOG" > "$OUTPUT_FILE" && \
echo "ok" echo "Report generated successfully." || \
echo "Report failed with code $?"
...@@ -72,7 +72,7 @@ md5sum = 1695c9a06a2b11ccfe893d7a224e489d ...@@ -72,7 +72,7 @@ md5sum = 1695c9a06a2b11ccfe893d7a224e489d
[monitor-conf] [monitor-conf]
<= monitor-template-base <= monitor-template-base
filename = monitor.conf.in filename = monitor.conf.in
md5sum = fc65084dd1c2fe2487b58a003b576f61 md5sum = 888e2845d09bfaa59c25f56f5bcf76b1
[monitor-instance-info] [monitor-instance-info]
<= monitor-template-base <= monitor-template-base
...@@ -131,6 +131,6 @@ depends = ...@@ -131,6 +131,6 @@ depends =
PyRSS2Gen = 1.1 PyRSS2Gen = 1.1
cns.recipe.symlink = 0.2.3 cns.recipe.symlink = 0.2.3
pycurl = 7.43.0 pycurl = 7.43.0
slapos.toolbox = 0.73 slapos.toolbox = 0.74
pyasn1 = 0.3.7 pyasn1 = 0.3.7
...@@ -15,4 +15,4 @@ ...@@ -15,4 +15,4 @@
# not need these here). # not need these here).
[monitor2-template] [monitor2-template]
filename = instance-monitor.cfg.jinja2.in filename = instance-monitor.cfg.jinja2.in
md5sum = 5027f0b1abcc28ce3817cd70fb667a3b md5sum = 03254b14a2ff242f7588a307d8c27f23
...@@ -42,6 +42,7 @@ services = ${directory:services} ...@@ -42,6 +42,7 @@ services = ${directory:services}
services-conf = ${directory:etc}/monitor.conf.d services-conf = ${directory:etc}/monitor.conf.d
log = ${directory:log}/monitor log = ${directory:log}/monitor
monitor-var = ${directory:var}/monitor monitor-var = ${directory:var}/monitor
monitor-log = ${directory:monitor}/private/monitor-log
[ca-directory] [ca-directory]
recipe = slapos.cookbook:mkdirectory recipe = slapos.cookbook:mkdirectory
...@@ -88,14 +89,11 @@ promise-folder = ${directory:promises} ...@@ -88,14 +89,11 @@ promise-folder = ${directory:promises}
monitor-promise-folder = ${monitor-directory:promises} monitor-promise-folder = ${monitor-directory:promises}
promises-timeout-file = ${monitor-promise-timeout-file:file} promises-timeout-file = ${monitor-promise-timeout-file:file}
pid-file = ${monitor-directory:pids}/monitor-bootstrap.pid pid-file = ${monitor-directory:pids}/monitor-bootstrap.pid
randomsleep = {{ bin_directory }}/randomsleep
public-path-list = public-path-list =
private-path-list = private-path-list = ${directory:log}
${directory:log} monitor-url-list = ${monitor-instance-parameter:monitor-url-list}
#
monitor-url-list =
${monitor-instance-parameter:monitor-url-list}
parameter-file-path = ${monitor-instance-parameter:configuration-file-path} parameter-file-path = ${monitor-instance-parameter:configuration-file-path}
parameter-list = parameter-list =
...@@ -234,6 +232,12 @@ context = ...@@ -234,6 +232,12 @@ context =
raw dash_binary {{ dash_executable_location }} raw dash_binary {{ dash_executable_location }}
command = kill -USR1 $(cat ${monitor-httpd-conf-parameter:pid-file}) command = kill -USR1 $(cat ${monitor-httpd-conf-parameter:pid-file})
[logrotate-entry-monitor-httpd]
<= logrotate-entry-base
name = monitor-apache
log = $${basedirectory:log}/monitor/monitor-httpd-error.log
post = test ! -s $${buildout:directory}/var/run/monitor-httpd.pid || $${buildout:directory}/bin/slapos-kill --pidfile $${buildout:directory}/var/run/monitor-httpd.pid -s USR1
[xnice-bin] [xnice-bin]
recipe = collective.recipe.template recipe = collective.recipe.template
input = inline:#!/bin/sh input = inline:#!/bin/sh
...@@ -264,14 +268,14 @@ recipe = slapos.cookbook:cron.d ...@@ -264,14 +268,14 @@ recipe = slapos.cookbook:cron.d
cron-entries = ${cron:cron-entries} cron-entries = ${cron:cron-entries}
name = monitor-globalstate name = monitor-globalstate
frequency = * * * * * frequency = * * * * *
command = ${monitor-globalstate-wrapper:wrapper-path} command = {{ bin_directory }}/randomsleep 60 && ${monitor-globalstate-wrapper:wrapper-path}
[monitor-configurator-cron-entry] [monitor-configurator-cron-entry]
recipe = slapos.cookbook:cron.d recipe = slapos.cookbook:cron.d
cron-entries = ${cron:cron-entries} cron-entries = ${cron:cron-entries}
name = monitor-configurator name = monitor-configurator
frequency = * * * * * frequency = * * * * *
command = ${monitor-configurator-wrapper:wrapper-path} command = {{ bin_directory }}/randomsleep 60 && ${monitor-configurator-wrapper:wrapper-path}
[monitor-httpd-promise] [monitor-httpd-promise]
recipe = slapos.cookbook:check_url_available recipe = slapos.cookbook:check_url_available
...@@ -407,6 +411,7 @@ depends = ...@@ -407,6 +411,7 @@ depends =
${monitor-bootstrap-promise:file} ${monitor-bootstrap-promise:file}
${promise-check-slapgrid:output} ${promise-check-slapgrid:output}
${promise-monitor-httpd-is-process-older-than-dependency-set:wrapper-path} ${promise-monitor-httpd-is-process-older-than-dependency-set:wrapper-path}
${logrotate-entry-monitor-httpd:name}
[monitor-publish] [monitor-publish]
monitor-base-url = ${monitor-publish-parameters:monitor-base-url} monitor-base-url = ${monitor-publish-parameters:monitor-base-url}
......
[monitor] [monitor]
{% set monitor_url_list = parameter_dict.pop("monitor-url-list", "").strip() -%}
{% for key, value in parameter_dict.items() -%} {% for key, value in parameter_dict.items() -%}
{% if key == "monitor-url-list" and monitor_base_urls is defined -%} {{ key }} = {{ value.strip().replace("\n", "\n ") }}
{{ key }} = {% endfor -%}
monitor-url-list =
{% if monitor_url_list -%}
{{ ' ' ~ monitor_url_list.replace("\n", "\n ") }}
{% else -%}
{% for key, value in monitor_base_urls.items() -%} {% for key, value in monitor_base_urls.items() -%}
{{ ' ' ~ value }} {{ ' ' ~ value }}
{% endfor -%} {% endfor -%}
{% else -%}
{{ key }} = {{ value.strip().replace("\n", "\n ") }}
{% endif -%} {% endif -%}
{% endfor -%}
...@@ -93,7 +93,7 @@ eggs = ...@@ -93,7 +93,7 @@ eggs =
[versions] [versions]
setuptools = 33.1.1 setuptools = 33.1.1
# Use SlapOS patched zc.buildout # Use SlapOS patched zc.buildout
zc.buildout = 2.5.2+slapos011 zc.buildout = 2.5.2+slapos012
# Use SlapOS patched zc.recipe.egg (zc.recipe.egg 2.x is for Buildout 2) # Use SlapOS patched zc.recipe.egg (zc.recipe.egg 2.x is for Buildout 2)
zc.recipe.egg = 2.0.3+slapos003 zc.recipe.egg = 2.0.3+slapos003
# Use own version of h.r.download to be able to open .xz and .lz archives # Use own version of h.r.download to be able to open .xz and .lz archives
...@@ -121,14 +121,14 @@ netaddr = 0.7.19 ...@@ -121,14 +121,14 @@ netaddr = 0.7.19
pbr = 2.0.0 pbr = 2.0.0
plone.recipe.command = 1.1 plone.recipe.command = 1.1
prettytable = 0.7.2 prettytable = 0.7.2
psutil = 5.2.0 psutil = 5.4.3
pyOpenSSL = 17.2.0 pyOpenSSL = 17.2.0
pyparsing = 2.2.0 pyparsing = 2.2.0
pytz = 2016.10 pytz = 2016.10
requests = 2.13.0 requests = 2.13.0
six = 1.10.0 six = 1.10.0
slapos.cookbook = 1.0.53 slapos.cookbook = 1.0.53
slapos.core = 1.4.3 slapos.core = 1.4.4
slapos.extension.strip = 0.4 slapos.extension.strip = 0.4
slapos.libnetworkcache = 0.15 slapos.libnetworkcache = 0.15
slapos.rebootstrap = 4.1 slapos.rebootstrap = 4.1
...@@ -140,7 +140,7 @@ xml-marshaller = 0.9.7 ...@@ -140,7 +140,7 @@ xml-marshaller = 0.9.7
paramiko = 2.1.3 paramiko = 2.1.3
# Required by: # Required by:
# slapos.core==1.4.3 # slapos.core==1.4.4
Flask = 0.12 Flask = 0.12
# Required by: # Required by:
...@@ -160,7 +160,7 @@ ipaddress = 1.0.18 ...@@ -160,7 +160,7 @@ ipaddress = 1.0.18
jsonschema = 2.6.0 jsonschema = 2.6.0
# Required by: # Required by:
# slapos.core==1.4.3 # slapos.core==1.4.4
# XXX 'slapos node format' raises an exception with netifaces 0.10.5. # XXX 'slapos node format' raises an exception with netifaces 0.10.5.
netifaces = 0.10.4 netifaces = 0.10.4
...@@ -173,15 +173,15 @@ packaging = 16.8 ...@@ -173,15 +173,15 @@ packaging = 16.8
pycparser = 2.17 pycparser = 2.17
# Required by: # Required by:
# slapos.core==1.4.3 # slapos.core==1.4.4
supervisor = 3.3.3 supervisor = 3.3.3
# Required by: # Required by:
# slapos.core==1.4.3 # slapos.core==1.4.4
uritemplate = 3.0.0 uritemplate = 3.0.0
# Required by: # Required by:
# slapos.core==1.4.3 # slapos.core==1.4.4
zope.interface = 4.3.3 zope.interface = 4.3.3
[networkcache] [networkcache]
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment