Commit fd802074 authored by Gabriel Monnerat's avatar Gabriel Monnerat

merge master into erp5 branch

parent fd82b4c6
Changes
=======
0.39 (unreleased)
0.40.1 (2012-03-01)
-----------------
* No changes yet.
* Fix manifest to include files needed for apache. [Cedric de Saint Martin]
0.40 (2012-03-01)
-----------------
* apache_frontend initial release. [Cedric de Saint Martin]
0.39 (2012-02-20)
-----------------
* seleniumrunner initial release. [Cedric de Saint Martin]
0.38 (2011-12-05)
-----------------
......@@ -190,9 +200,10 @@ Changes
-----------------
* Add PHPMyAdmin software release used in SlapOS tutorials
[Cedric de Saint Martin]
[Cedric de Saint Martin]
* Add slaprunner software release [Cedric de Saint Martin]
0.9 (2011-06-24)
----------------
......
include CHANGES.txt
include slapos/recipe/generic_zope/template/site.zcml
include slapos/recipe/apache_frontend/template/notfound.html
recursive-include slapos/recipe *.in
recursive-include slapos/recipe *.bin
recursive-include slapos/recipe README.*.txt
[buildout]
parts =
alsa
[alsa]
# Contains libasound
recipe = hexagonit.recipe.cmmi
url = ftp://ftp.alsa-project.org/pub/lib/alsa-lib-1.0.24.1.tar.bz2
#md5sum = d55a9d7d2a79d738a1b7a511cffda4b6
configure-options =
--disable-static
--disable-aload
--disable-mixer
--disable-pcm
--disable-rawmidi
--disable-hwdep
--disable-seq
--disable-ucm
--disable-alisp
--disable-old-symbols
--disable-python
\ No newline at end of file
[buildout]
parts =
binutils
[binutils]
recipe = hexagonit.recipe.cmmi
url = http://ftp.gnu.org/gnu/binutils/binutils-2.21.1.tar.bz2
md5sum = bde820eac53fa3a8d8696667418557ad
[buildout]
parts = boa
[boa-patch-ENOSYS]
recipe = hexagonit.recipe.download
url = http://sources.gentoo.org/cgi-bin/viewvc.cgi/gentoo-x86/www-servers/boa/files/boa-0.94.14_rc21-ENOSYS.patch?revision=1.1
filename = boa-0.94.14_rc21-ENOSYS.patch
md5sum = 7206b342195961501ed1eae38486e5db
download-only = true
[boa]
recipe = slapos.recipe.build
url = http://www.boa.org/boa-0.94.14rc21.tar.gz
md5sum = e24b570bd767a124fcfb40a34d148ba9
patches =
${boa-patch-ENOSYS:location}/${boa-patch-ENOSYS:filename}
slapos_promise =
directory:bin/
file:bin/boa
file:bin/boa_indexer
script =
import shutil
import os
url = self.download(self.options['url'], self.options['md5sum'])
extract_dir = self.extract(url)
workdir = guessworkdir(extract_dir)
self.applyPatchList(self.options.get('patches'), '-p1', cwd=workdir)
call(['./configure'], cwd=workdir)
call(['make'], cwd=workdir)
# Installation of boa. Manually, no make install
os.makedirs('%(location)s/bin/')
shutil.copyfile(workdir + '/src/boa', '%(location)s/bin/boa')
os.chmod('%(location)s/bin/boa', 0755)
shutil.copyfile(workdir + '/src/boa_indexer', '%(location)s/bin/boa_indexer')
os.chmod('%(location)s/bin/boa_indexer', 0755)
[buildout]
parts = busybox
[busybox]
recipe = slapos.recipe.build
url = http://busybox.net/downloads/busybox-1.19.3.tar.bz2
md5sum = c3938e1ac59602387009bbf1dd1af7f6
script =
extract_dir = self.extract(self.download(%(url)r, %(md5sum)r))
workdir = guessworkdir(extract_dir)
self.logger.info("Creating default configuration")
call(['make', 'defconfig'], cwd=workdir, env=env)
self.logger.info("Building")
call(['make'], cwd=workdir, env=env)
self.logger.info("Installing")
call(['make', 'CONFIG_PREFIX=%(location)s', 'install'], cwd=workdir, env=env)
self.logger.info("Installation finished")
[buildout]
extends =
../dbus/buildout.cfg
../fontconfig/buildout.cfg
../glib/buildout.cfg
../gtk-2/buildout.cfg
../libpng/buildout.cfg
../xorg/buildout.cfg
parts =
chromium
[chromium]
recipe = slapos.recipe.build
slapos_promise =
file:chrome
file:chrome-wrapper
file:chrome-slapos
#chromium zip files for linux seem to be corrupted : rights are not correctly
#set (+x) when unzipping using python, but it works when doing "unzip chromium.zip"
#AND it works when unzipping any other archive with python.
#Conclusion : Google, please, learn how to make zip files.
linux_x86 = http://commondatastorage.googleapis.com/chromium-browser-continuous/Linux/109696/chrome-linux.zip 8ba6c022849b2a882b6e65163c147eb9
linux_x86-64 = http://commondatastorage.googleapis.com/chromium-browser-snapshots/Linux_x64/109696/chrome-linux.zip a3ed3feb285ecfe7c722576db80d5099
mac_x86-64 = http://commondatastorage.googleapis.com/chromium-browser-continuous/Mac/100142/chrome-mac.zip cb3a76b8a1a93be94df2f500fb621131
script =
#If part directory already exist, will just throw an error.
import sys
platform = '%%s_%%s' %% (guessOperatingSystem(), guessPlatform())
if not self.options.get('url'): self.options['url'], self.options['md5sum'] = self.options[platform].split(' ')
extract_dir = self.extract(self.download(self.options['url'], self.options.get('md5sum')))
workdir = guessworkdir(extract_dir)
self.copyTree(workdir, "%(location)s")
wrapper_location = os.path.join("%(location)s", "chrome-slapos")
wrapper = open(wrapper_location, 'w')
wrapper.write("""#!/bin/sh
export LD_LIBRARY_PATH=${libXrender:location}/lib/:${fontconfig:location}/lib/:${dbus:location}/lib/:${dbus-glib:location}/lib/:${pango:location}/lib:${cairo:location}/lib:${glib:location}/lib:${gtk-2:location}/lib:${atk:location}/lib:${gdk-pixbuf:location}/lib:${libXt:location}/lib:${gtk-2:location}/lib:${libpng:location}/lib:%(location)s
%(location)s/chrome""")
wrapper.flush()
wrapper.close()
os.chmod(wrapper_location, 0766)
os.chmod(os.path.join("%(location)s", 'chrome'), 0766)
os.chmod(os.path.join("%(location)s", 'chrome-wrapper'), 0766)
# requirements : libXrender1 libxss1 x11-common
\ No newline at end of file
[buildout]
software_home = ${:directory}
extends =
../profiles/versions-common.cfg
../profiles/software-definition.cfg
../profiles/common.cfg
../xpdf/buildout.cfg
../imagemagick/buildout.cfg
../file/buildout.cfg
../pdftk/buildout.cfg
../ffmpeg/buildout.cfg
../python-2.6/buildout.cfg
../libreoffice-bin/buildout.cfg
../lxml-python/buildout.cfg
python = software_definition
versions = versions
parts =
instance_template
cloudooo
imagemagick
instance-egg
libreoffice-bin
file
xpdf
pdftk
ffmpeg
python2.6
bootstrap2.6
cloudooo-packages =
cloudooo.handler.ooo
cloudooo.handler.pdf
cloudooo.handler.ffmpeg
cloudooo.handler.imagemagick
cloudooo
sources = sources
[sources]
# Include information related to official cloudooo source code.
cloudooo = git http://git.erp5.org/repos/cloudooo.git
[instance-egg]
recipe = zc.recipe.egg
[lxml-python]
python = python2.6
eggs =
erp5.recipe.cloudoooinstance
z3c.recipe.mkdir
${buildout:cloudooo-packages}
[cloudooo]
recipe = zc.recipe.egg
python = python2.6
interpreter = pycloudoo
scripts =
eggs =
${lxml-python:egg}
collective.recipe.supervisor
plone.recipe.command
erp5.extension.sectionextender
supervisor
${buildout:cloudooo-packages}
[software_definition]
software_home = ${buildout:directory}
executable = ${python2.6:executable}
cloudooo
PasteScript
scripts =
paster=cloudooo_paster
[buildout]
extends =
../libxml2/buildout.cfg
../pkgconfig/buildout.cfg
../libexpat/buildout.cfg
parts =
dbus
[dbus]
recipe = hexagonit.recipe.cmmi
url = http://dbus.freedesktop.org/releases/dbus/dbus-1.4.10.tar.gz
md5sum = 402a2f8102bedbe236e2891b2b0f31c2
configure-options =
--disable-static
environment =
PATH=${pkgconfig:location}/bin:%(PATH)s
PKG_CONFIG_PATH=${libxml2:location}/lib/pkgconfig
CPPFLAGS=-I${libexpat:location}/include
LDFLAGS=-L${libexpat:location}/lib
[dbus-glib]
recipe = hexagonit.recipe.cmmi
url = http://dbus.freedesktop.org/releases/dbus-glib/dbus-glib-0.94.tar.gz
md5sum = e1f1506a6f4941e67bffd614b1ad5af6
environment =
PATH=${pkgconfig:location}/bin:${glib:location}/bin:%(PATH)s
PKG_CONFIG_PATH=${dbus:location}/lib/pkgconfig:${glib:location}/lib/pkgconfig
CPPFLAGS=-I${libexpat:location}/include
LDFLAGS=-L${libexpat:location}/lib -L${gettext:location}/lib
......@@ -13,7 +13,7 @@ parts =
[dropbear-userspace-patch]
recipe = hexagonit.recipe.download
md5sum = 89f575b9a9586b04ef9073c9c3af13ae
md5sum = 3d934c2c90e8c57536a4fa2cf8ad216d
url = ${:_profile_base_location_}/${:filename}
filename = userspace.patch
download-only = true
......@@ -25,6 +25,20 @@ url = ${:_profile_base_location_}/${:filename}
filename = ipv6-support.patch
download-only = true
[dropbear-no-shell-check-patch]
recipe = hexagonit.recipe.download
md5sum = bb2ac410bd4cb2b07c23bfcc712e45f7
url = ${:_profile_base_location_}/${:filename}
filename = no-shell-checking.patch
download-only = true
[dropbear-static_build_fix.patch]
recipe = hexagonit.recipe.download
md5sum = 6e0ccc1949dfa31b4bc3951b9bb6472e
url = ${:_profile_base_location_}/${:filename}
filename = dropbear-0.53.1-static_build_fix.patch
download-only = true
[dropbear]
recipe = hexagonit.recipe.cmmi
md5sum = 0284ea239083f04c8b874e08e1aca243
......@@ -35,7 +49,7 @@ url = http://matt.ucc.asn.au/dropbear/releases/dropbear-0.53.1.tar.bz2
configure-options =
--prefix=${buildout:parts-directory}/${:_buildout_section_name_}
--with-zlib=${zlib:location}
CFLAGS="-DENABLE_SINGLEUSER "
CFLAGS="-DENABLE_SINGLEUSER -D__DIRTY_NO_SHELL_CHECKING"
environment =
CPPFLAGS =-I${zlib:location}/include
......@@ -44,6 +58,8 @@ environment =
patches=
${dropbear-userspace-patch:location}/${dropbear-userspace-patch:filename}
${dropbear-ipv6-patch:location}/${dropbear-ipv6-patch:filename}
${dropbear-no-shell-check-patch:location}/${dropbear-no-shell-check-patch:filename}
${dropbear-static_build_fix.patch:location}/${dropbear-static_build_fix.patch:filename}
patch-options=
-p1
--- a/Makefile.in 2012-02-13 18:01:23.505279581 +0100
+++ b/Makefile.in 2012-02-13 18:04:17.070140233 +0100
@@ -77,7 +77,7 @@
INSTALL=@INSTALL@
CPPFLAGS=@CPPFLAGS@
CFLAGS+=-I. -I$(srcdir) $(CPPFLAGS) @CFLAGS@
-LIBS+=@LIBS@
+LIBS+=@CRYPTLIB@ @LIBS@
LDFLAGS=@LDFLAGS@
EXEEXT=@EXEEXT@
diff --git a/svr-auth.c b/svr-auth.c
index 87e3c5e..8b93a22 100644
--- a/svr-auth.c
+++ b/svr-auth.c
@@ -267,6 +267,7 @@ static int checkusername(unsigned char *username, unsigned int userlen) {
usershell = "/bin/sh";
}
+#ifndef __DIRTY_NO_SHELL_CHECKING
/* check the shell is valid. If /etc/shells doesn't exist, getusershell()
* should return some standard shells like "/bin/sh" and "/bin/csh" (this
* is platform-specific) */
@@ -288,6 +289,7 @@ static int checkusername(unsigned char *username, unsigned int userlen) {
goodshell:
endusershell();
+#endif
TRACE(("matching shell"))
TRACE(("uid = %d", ses.authstate.pw_uid))
......@@ -48,7 +48,7 @@ index 87e3c5e..adb2e8b 100644
+ if (svr_opts.singleuser) {
+ m_free(username);
+ /* Get the current login of the user running dropbear */
+ username = m_strdup(getlogin());
+ username = m_strdup(getpwuid(getuid())->pw_name);
+ }
+#endif /* ifdef ENABLE_SINGLEUSER */
servicename = buf_getstring(ses.payload, &servicelen);
......
[buildout]
extends =
../alsa/buildout.cfg
../xorg/buildout.cfg
../fontconfig/buildout.cfg
../dbus/buildout.cfg
../gtk-2/buildout.cfg
../libpng/buildout.cfg
parts =
firefox
[firefox]
recipe = slapos.recipe.build
slapos_promise =
file:firefox
file:firefox-bin
depends =
${liberation-fonts:location}
${ipaex-fonts:location}
x86 = http://releases.mozilla.org/pub/mozilla.org/firefox/releases/7.0.1/linux-i686/fr/firefox-7.0.1.tar.bz2 42c2559892f26ed2a0563faaf693a00f
x86-64 = http://releases.mozilla.org/pub/mozilla.org/firefox/releases/7.0.1/linux-x86_64/en-US/firefox-7.0.1.tar.bz2 20d6c8e3dfc97d08d1dec7d0479f924f
script =
if not self.options.get('url'): self.options['url'], self.options['md5sum'] = self.options[guessPlatform()].split(' ')
extract_dir = self.extract(self.download(self.options['url'], self.options.get('md5sum')))
workdir = guessworkdir(extract_dir)
self.copyTree(workdir, "%(location)s")
wrapper_location = os.path.join("%(location)s", "firefox-slapos")
wrapper = open(wrapper_location, 'w')
wrapper.write("""#!/bin/sh
cd %(location)s
export LD_LIBRARY_PATH=${libXrender:location}/lib/:${fontconfig:location}/lib/:${dbus:location}/lib/:${dbus-glib:location}/lib/:${pango:location}/lib:${cairo:location}/lib:${glib:location}/lib:${gtk-2:location}/lib:${atk:location}/lib:${gdk-pixbuf:location}/lib:${libXt:location}/lib:${libpng:location}/lib:${libSM:location}/lib:${libICE:location}/lib:${alsa:location}/lib:%(location)s
export PATH=${fontconfig:location}/bin:$PATH
%(location)s/firefox $*""")
wrapper.close()
os.chmod(wrapper_location, 0777)
......@@ -14,6 +14,7 @@ parts =
recipe = hexagonit.recipe.cmmi
url = http://fontconfig.org/release/fontconfig-2.8.0.tar.gz
md5sum = 77e15a92006ddc2adbb06f840d591c0e
# XXX-Cedric : should we use --with-add-fonts={somefont:location}/share,{someotherfont:location}/share?
configure-options =
--disable-static
--disable-docs
......
[buildout]
extends =
../glib/buildout.cfg
../pkgconfig/buildout.cfg
../libpng/buildout.cfg
../xorg/buildout.cfg
../zlib/buildout.cfg
../freetype/buildout.cfg
../fontconfig/buildout.cfg
../gettext/buildout.cfg
../libtiff/buildout.cfg
../libjpeg/buildout.cfg
parts =
gtk-2
[cairo]
recipe = hexagonit.recipe.cmmi
url = http://cairographics.org/releases/cairo-1.10.2.tar.gz
md5sum = f101a9e88b783337b20b2e26dfd26d5f
configure-options =
--enable-tee=yes
environment =
PATH=${freetype:location}/bin:${pkgconfig:location}/bin:%(PATH)s
PKG_CONFIG_PATH=${fontconfig:location}/lib/pkgconfig:${freetype:location}/lib/pkgconfig:${zlib:location}/lib/pkgconfig:${libpng:location}/lib/pkgconfig:${pixman:location}/lib/pkgconfig:${libX11:location}/lib/pkgconfig:${libXrender:location}/lib/pkgconfig
CPPFLAGS=-I${libpng:location}/include/ -I${zlib:location}/include -I${libX11:location}/include/ -I${xproto:location}/include -I${kbproto:location}/include -I${libXrender:location}/include -I${render:location}/include
LDFLAGS=-L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib -L${libXrender:location}/lib -Wl,-rpath=${libXrender:location}/lib
LD_LIBRARY_PATH=${render:location}/lib:${libX11:location}/lib:${libXrender:location}/lib
[pango]
recipe = hexagonit.recipe.cmmi
url = http://ftp.gnome.org/pub/gnome/sources/pango/1.29/pango-1.29.3.tar.bz2
md5sum = be4e3891353fae6b62a6f8d7689c4266
configure-options =
--disable-static
# XXX-Cedric : Why, god, why do I have to supply the -lXrender option to linker?
environment =
PATH=${glib:location}/bin:${freetype:location}/bin:${pkgconfig:location}/bin:%(PATH)s
PKG_CONFIG_PATH=${glib:location}/lib/pkgconfig:${fontconfig:location}/lib/pkgconfig:${freetype:location}/lib/pkgconfig:${cairo:location}/lib/pkgconfig::${libXrender:location}/lib/pkgconfig
CPPFLAGS=-I${cairo:location}/include/cairo
LDFLAGS=-L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib -L${libXrender:location}/lib -Wl,-rpath=${libXrender:location}/lib -lXrender
LD_LIBRARY_PATH=${render:location}/lib:${libX11:location}/lib:${libXrender:location}/lib
[gdk-pixbuf]
recipe = hexagonit.recipe.cmmi
url = http://ftp.gnome.org/pub/gnome/sources/gdk-pixbuf/2.24/gdk-pixbuf-2.24.0.tar.bz2
md5sum = d8ece3a4ade4a91c768328620e473ab8
configure-options =
--disable-static
--without-libintl-prefix
environment =
PATH=${glib:location}/bin:${pkgconfig:location}/bin:%(PATH)s
PKG_CONFIG_PATH=${glib:location}/lib/pkgconfig:${gettext:location}/lib/pkgconfig:${fontconfig:location}/lib/pkgconfig:${freetype:location}/lib/pkgconfig:${cairo:location}/lib/pkgconfig
CPPFLAGS=-I${libtiff:location}/include -I${libjpeg:location}/include -I${libpng:location}/include
LDFLAGS=-L${gettext:location}/lib -Wl,-rpath=${gettext:location}/lib -L${glib:location}/lib -Wl,-rpath=${glib:location}/lib -L${libtiff:location}/lib -Wl,-rpath=${libtiff:location}/lib -L${libjpeg:location}/lib -Wl,-rpath=${libjpeg:location}/lib -L${libpng:location}/lib -Wl,-rpath=${libpng:location}/lib -L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib
LD_LIBRARY_PATH=${glib:location}/lib:${gettext:location}/lib
[atk]
recipe = hexagonit.recipe.cmmi
url = http://ftp.gnome.org/pub/gnome/sources/atk/2.0/atk-2.0.1.tar.bz2
md5sum = 87f20b78deaedef858ac54358c0786b1
environment =
PATH=${glib:location}/bin:${pkgconfig:location}/bin:%(PATH)s
PKG_CONFIG_PATH=${glib:location}/lib/pkgconfig
LDFLAGS=-L${gettext:location}/lib -Wl,-rpath=${gettext:location}/lib -L${glib:location}/lib -Wl,-rpath=${gettext:location}/lib
LD_LIBRARY_PATH=${glib:location}/lib:${gettext:location}/lib
[gtk-2]
recipe = hexagonit.recipe.cmmi
url = http://ftp.gnome.org/pub/gnome/sources/gtk+/2.24/gtk+-2.24.6.tar.bz2
md5sum = 421100f6597e613234f8dead6091a9fe
configure-options =
--disable-static
--disable-cups
--disable-papi
--enable-explicit-deps
environment =
PATH=${gdk-pixbuf:location}/bin:${glib:location}/bin:${pkgconfig:location}/bin:%(PATH)s
PKG_CONFIG_PATH=${glib:location}/lib/pkgconfig:${gettext:location}/lib/pkgconfig:${fontconfig:location}/lib/pkgconfig:${freetype:location}/lib/pkgconfig:${libXau:location}/lib/pkgconfig:${xproto:location}/lib/pkgconfig:${cairo:location}/lib/pkgconfig:${atk:location}/lib/pkgconfig:${gdk-pixbuf:location}/lib/pkgconfig:${pango:location}/lib/pkgconfig:${libX11:location}/lib/pkgconfig:${libXext:location}/lib/pkgconfig:${pixman:location}/lib/pkgconfig:${libpng:location}/lib/pkgconfig:${kbproto:location}/lib/pkgconfig:${libXrender:location}/lib/pkgconfig:${xextproto:location}/lib/pkgconfig:${libxcb:location}/lib/pkgconfig:${xcbproto:location}/lib/pkgconfig:${xorg-libpthread-stubs:location}/lib/pkgconfig
CPPFLAGS=-I${libtiff:location}/include -I${libjpeg:location}/include -I${libpng:location}/include -I${libX11:location}/include/ -I${xproto:location}/include -I${kbproto:location}/include -I${libXrender:location}/include -I${render:location}/include -I${libXext:location}/include -I${cairo:location}/include/cairo
LDFLAGS=-L${gettext:location}/lib -L${glib:location}/lib -L${libtiff:location}/lib -L${libjpeg:location}/lib -L${libpng:location}/lib -L${zlib:location}/lib -L${libXrender:location}/lib -L${libX11:location}/lib -L${libXext:location}/lib -L${pango:location}/lib
LD_LIBRARY_PATH=${glib:location}/lib:${gettext:location}/lib:${libX11:location}/lib
[buildout]
extends =
../gettext/buildout.cfg
../perl/buildout.cfg
../perl-XML-Parser/buildout.cfg
parts =
intltool
[intltool]
recipe = hexagonit.recipe.cmmi
url = http://edge.launchpad.net/intltool/trunk/0.41.1/+download/intltool-0.41.1.tar.gz
md5sum = d6c91bf06681919ccfdf3624035b75dc
depends =
${perl:version}
${perl-XML-Parser:location}
environment =
PATH=${perl:location}/bin:${gettext:location}/bin:${pkgconfig:location}/bin:%(PATH)s
PKG_CONFIG_PATH=${perl:location}/lib/pkgconfig
[buildout]
extends =
../xorg/buildout.cfg
parts =
mesa
[libdrm]
#does not seem to compile with -jX
recipe = hexagonit.recipe.cmmi
url = http://dri.freedesktop.org/libdrm/libdrm-2.4.26.tar.bz2
md5sum = 062569426773f69b11a47a7712bba770
configure-options =
--enable-nouveau-experimental-api
environment =
PATH=${pkgconfig:location}/bin:%(PATH)s
PKG_CONFIG_PATH=${xorg-libpthread-stubs:location}/lib/pkgconfig:${pciaccess:location}/lib/pkgconfig
[mesa]
recipe = hexagonit.recipe.cmmi
url = ftp://ftp.freedesktop.org//pub/mesa/7.11/MesaLib-7.11.tar.bz2
md5sum = ff03aca82d0560009a076a87c888cf13
configure-options =
--disable-static
--disable-driglx-direct
--disable-glu
--disable-glw
--disable-glut
--disable-egl
--disable-gallium-llvm
--with-gallium-drivers=
--with-dri-drivers=
--with-expat=${libexpat:location}
--with-driver=xlib
environment =
PATH=${pkgconfig:location}/bin:${makedepend:location}/bin:${flex:location}/bin:${bison:location}/bin:%(PATH)s
PKG_CONFIG_PATH=${libdrm:location}/lib/pkgconfig:${dri2proto:location}/lib/pkgconfig:${xorg-libpthread-stubs:location}/lib/pkgconfig:${pciaccess:location}/lib/pkgconfig:${libX11:location}/lib/pkgconfig:${glproto:location}/lib/pkgconfig:${libX11:location}/lib/pkgconfig:${libXext:location}/lib/pkgconfig:${xfixes:location}/lib/pkgconfig:${xdamage:location}/lib/pkgconfig:${xproto:location}/lib/pkgconfig:${xextproto:location}/lib/pkgconfig
CPPFLAGS=-I${libexpat:location}/include -I${libdrm:location}/include/libdrm -I${libdrm:location}/include -I${xproto:location}/include -I${libX11:location}/include -I${libXext:location}/include/ -I${xextproto:location}/include
LDFLAGS=-L${libexpat:location}/lib
PYTHON=${buildout:executable}
[buildout]
extends =
../cmake/buildout.cfg
../glib/buildout.cfg
../pcre/buildout.cfg
../mariadb/buildout.cfg
parts = mydumper
# XXX-Antoine:
# This is really dirty, but it's the only way to install
# mydumper that works
[mydumper]
recipe = slapos.recipe.build
url = http://launchpad.net/mydumper/0.2/0.2.3/+download/mydumper-0.2.3.tar.gz
md5sum = 36e6a1c97a9634a6882ddaac5e2697d5
buildout-bin-dir = ${buildout:bin-directory}
cmake-command = ${cmake:location}/bin/cmake
mysql-config = ${mariadb:location}/bin/mysql_config
doc-dependency = ${mydumper-doc:eggs}
mysqllib = ${mariadb:location}/lib
pkg-config-path = ${glib:location}/lib/pkgconfig/:${pcre:location}/lib/pkgconfig/
libraries = ${zlib:location}/lib/:${glib:location}/lib/:${pcre:location}/lib/:${mariadb:location}/lib/mysql/
includes = ${zlib:location}/include/:${glib:location}/include/:${pcre:location}/include/:${mariadb:location}/include/mysql/
cflags = -I${zlib:location}/include/ -I${glib:location}/include/ -I${pcre:location}/include/ -I${mariadb:location}/include/mysql/
mydumper-patches =
${:_profile_base_location_}/mydumper-remove-warnings-errors.patch 917fea16b5ddea195cfa33fbd9827f57 -p1
slapos_promise =
directory:bin
file:bin/mydumper
file:bin/myloader
script =
import os
url = self.download(self.options['url'], self.options.get('md5sum'))
extract_dir = self.extract(url)
workdir = guessworkdir(extract_dir)
self.applyPatchList(self.options['mydumper-patches'], cwd=workdir)
env['PATH'] = self.options['buildout-bin-dir'] + ':' + env.get('PATH', '')
env['PKG_CONFIG_PATH'] = self.options['pkg-config-path'] + ':' + \
env.get('PKG_CONFIG_PATH', '')
env['CMAKE_INCLUDE_PATH'] = self.options['includes']
env['CMAKE_LIBRARY_PATH'] = self.options['libraries']
env['CFLAGS'] = self.options['cflags']
command_line = [self.options['cmake-command'],
'-DCMAKE_INSTALL_PREFIX=%%s' %% self.options['location'],
'-DMYSQL_CONFIG=%%s' %% self.options['mysql-config'],
'-DCMAKE_C_FLAGS=%%s' %% self.options['cflags'],
'-DCMAKE_INSTALL_RPATH=%%s' %% self.options['libraries'],
'.']
call(command_line, cwd=workdir, env=env)
call(['make'], cwd=workdir, env=env)
call(['make', 'install'], cwd=workdir, env=env)
[mydumper-doc]
recipe = zc.recipe.egg
eggs =
Sphinx
dependent-scripts = true
# XXX-Antoine: here's what I did using hexagonit.recipe.cmmi.
# and it wasn't working !
#[mydumper]
#recipe = hexagonit.recipe.cmmi
#url = http://launchpad.net/mydumper/0.2/0.2.3/+download/mydumper-0.2.3.tar.gz
#md5sum = 36e6a1c97a9634a6882ddaac5e2697d5
#strip-top-level-dir = true
#location = ${buildout:parts-directory}/${:_buildout_section_name_}
#configure-command =
# ${cmake:location}/bin/cmake \
# -DCMAKE_INSTALL_PREFIX=${:location} \
# -DMYSQL_CONFIG=${mariadb:location}/bin/mysql_config \
# -DCMAKE_INCLUDE_PATH=${zlib:location}/include \
# -DCMAKE_LIBRARY_PATH=${zlib:location}/lib \
# .
#environment=
# PATH=$PATH:${buildout:bin-directory}
# PKG_CONFIG_PATH=${glib:location}/lib/pkgconfig/:${pcre:location}/lib/pkgconfig/
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -12,7 +12,7 @@
add_subdirectory(docs)
-set(CMAKE_C_FLAGS "-Wall -Wunused -Wwrite-strings -Wno-strict-aliasing -Wextra -Wshadow -Werror -O3 -g ${MYSQL_CFLAGS}")
+set(CMAKE_C_FLAGS "-Wall -Wunused -Wwrite-strings -Wno-strict-aliasing -Wextra -Wshadow -O3 -g ${MYSQL_CFLAGS}")
include_directories(${MYDUMPER_SOURCE_DIR} ${MYSQL_INCLUDE_DIR} ${GLIB2_INCLUDE_DIR} ${PCRE_INCLUDE_DIR} ${ZLIB_INCLUDE_DIRS})
[buildout]
# Work in progress
extends =
../xorg/buildout.cfg
../fontconfig/buildout.cfg
../dbus/buildout.cfg
../gtk-2/buildout.cfg
../libpng/buildout.cfg
parts =
opera
[opera]
recipe = slapos.recipe.build
slapos_promise =
file:opera
depends =
${liberation-fonts:location}
${ipaex-fonts:location}
x86 = http://arc.opera.com/pub/opera/linux/1151/opera-11.51-1087.i386.linux.tar.bz2 91db21fc001e736a6432627fbf93062f
x86-64 = todohttp://arc.opera.com/pub/opera/linux/1151/opera-11.51-1087.i386.linux.tar.bz2 91db21fc001e736a6432627fbf93062f
script =
if not self.options.get('url'): self.options['url'], self.options['md5sum'] = self.options[guessPlatform()].split(' ')
extract_dir = self.extract(self.download(self.options['url'], self.options.get('md5sum')))
workdir = guessworkdir(extract_dir)
self.copyTree(workdir, "%(location)s")
wrapper_location = os.path.join("%(location)s", "opera-slapos")
wrapper = open(wrapper_location, 'w')
wrapper.write("""#!/bin/sh
cd %(location)s
export LD_LIBRARY_PATH=${libXrender:location}/lib/:${fontconfig:location}/lib/:${dbus:location}/lib/:${dbus-glib:location}/lib/:${pango:location}/lib:${cairo:location}/lib:${glib:location}/lib:${gtk-2:location}/lib:${atk:location}/lib:${gdk-pixbuf:location}/lib:${libXt:location}/lib:${libpng:location}/lib:${libSM:location}/lib:${libICE:location}/lib:%(location)s
export PATH=${fontconfig:location}/bin:$PATH
%(location)s/opera $*""")
wrapper.close()
profile_directory = os.path.join("%(location)s", "profile")
os.mkdir(profile_directory)
opera_config = open(os.path.join(profile_directory, "opera6.ini"), "w")
opera_config.write("""[State]
Reading Plugins=0
Accept License=1
Run=0""")
opera_config.close()
os.chmod(wrapper_location, 0766)
[buildout]
extends =
../libexpat/buildout.cfg
../libxml2/buildout.cfg
../perl/buildout.cfg
../perl-libwww-perl/buildout.cfg
../zlib/buildout.cfg
parts =
perl-XML-Parser
[perl-XML-Parser]
recipe = hexagonit.recipe.cmmi
depends =
${perl:version}
${perl-libwww-perl:location}
url = http://search.cpan.org/CPAN/authors/id/M/MS/MSERGEANT/XML-Parser-2.36.tar.gz
md5sum = 1b868962b658bd87e1563ecd56498ded
configure-command =
${perl:location}/bin/perl Makefile.PL \
EXPATINCPATH=${libexpat:location}/include \
EXPATLIBPATH=${libexpat:location}/lib \
INC="-I${libxml2:location}/include/libxml2" \
LIBS="-L${libxml2:location}/lib -L${zlib:location}/lib"
# Parallel make does not work for this package on fast machines
# with many cores
make-options =
OTHERLDFLAGS=" -Wl,-rpath=${libxml2:location}/lib -Wl,-rpath=${zlib:location}/lib" -j1
environment =
LD_LIBRARY_PATH=${libxml2:location}/lib:${zlib:location}/lib
PERLLIB=blib/lib
[buildout]
parts =
pwgen
[pwgen]
recipe = hexagonit.recipe.cmmi
url = http://downloads.sourceforge.net/project/pwgen/pwgen/2.06/pwgen-2.06.tar.gz
md5sum = 935aebcbe610fbc9de8125e7b7d71297
[buildout]
parts =
qt
[qt]
recipe = slapos.recipe.build
slapos_promisee =
file:plop
# Online installer
x86 = http://get.qt.nokia.com/qtsdk/Qt_SDK_Lin32_online_v1_1_3_en.run eae2e2a1396fec1369b66c71d7df6eab
x86-64 = http://get.qt.nokia.com/qtsdk/Qt_SDK_Lin64_online_v1_1_3_en.run a4d929bc4d6511290c07c3745477b77b
# Offline installer
#x86 = http://get.qt.nokia.com/qtsdk/Qt_SDK_Lin32_offline_v1_1_3_en.run 106fdae4ec8947c491ab0a827a02da12
#x86-64 = http://get.qt.nokia.com/qtsdk/Qt_SDK_Lin64_offline_v1_1_3_en.run 8c280beb11ee763840464572ed80e8b8
# Needs many dependencies.
script =
if not self.options.get('url'): self.options['url'], self.options['md5sum'] = self.options[guessPlatform()].split(' ')
download_file = self.download(self.options['url'], self.options.get('md5sum'))
extract_dir = tempfile.mkdtemp(self.name)
os.chdir(extract_dir)
(download_dir, filename) = os.path.split(download_file)
auto_extract_bin = os.path.join(extract_dir, filename)
shutil.move(download_file, auto_extract_bin)
os.chmod(auto_extract_bin, 0777)
subprocess.call([auto_extract_bin])
self.cleanup_dir_list.append(extract_dir)
workdir = guessworkdir(extract_dir)
import pdb; pdb.set_trace()
self.copyTree(os.path.join(workdir, "jre1.6.0_27"), "%(location)s")
#ldd qt.run
# linux-gate.so.1 => (0xb7827000)
# libutil.so.1 => /lib/i686/cmov/libutil.so.1 (0xb781c000)
# libgobject-2.0.so.0 => not found
# libSM.so.6 => not found
# libICE.so.6 => not found
# libXrender.so.1 => not found
# libfontconfig.so.1 => not found
# libfreetype.so.6 => /usr/lib/libfreetype.so.6 (0xb77a4000)
# libz.so.1 => /usr/lib/libz.so.1 (0xb778f000)
# libXext.so.6 => /usr/lib/libXext.so.6 (0xb7780000)
# libX11.so.6 => /usr/lib/libX11.so.6 (0xb7663000)
# libdl.so.2 => /lib/i686/cmov/libdl.so.2 (0xb765f000)
# libgthread-2.0.so.0 => not found
# librt.so.1 => /lib/i686/cmov/librt.so.1 (0xb7655000)
# libglib-2.0.so.0 => not found
# libpthread.so.0 => /lib/i686/cmov/libpthread.so.0 (0xb763c000)
# libstdc++.so.6 => /usr/lib/libstdc++.so.6 (0xb754d000)
# libm.so.6 => /lib/i686/cmov/libm.so.6 (0xb7527000)
# libgcc_s.so.1 => /lib/libgcc_s.so.1 (0xb7508000)
# libc.so.6 => /lib/i686/cmov/libc.so.6 (0xb73c2000)
# libxcb.so.1 => /usr/lib/libxcb.so.1 (0xb73a9000)
# /lib/ld-linux.so.2 (0xb7828000)
# libXau.so.6 => /usr/lib/libXau.so.6 (0xb73a6000)
# libXdmcp.so.6 => /usr/lib/libXdmcp.so.6 (0xb73a1000)
\ No newline at end of file
From 50ec7439e80bd6a77346dc6482895e481d8cd43a Mon Sep 17 00:00:00 2001
From: Antoine Catton <acatton@tiolive.com>
Date: Tue, 10 Jan 2012 18:30:20 +0100
Subject: [PATCH] Switch to IPv6
---
libhttp/http.h | 4 ++--
libhttp/httpconnection.c | 11 ++++++++++-
libhttp/server.c | 33 +++++++++++++++++++--------------
libhttp/server.h | 6 +++---
shellinabox/shellinaboxd.c | 14 +++++++-------
5 files changed, 41 insertions(+), 27 deletions(-)
diff --git a/libhttp/http.h b/libhttp/http.h
index e7840fa..5cd61e3 100644
--- a/libhttp/http.h
+++ b/libhttp/http.h
@@ -66,8 +66,8 @@ typedef struct ServerConnection ServerConnection;
typedef struct Server Server;
typedef struct URL URL;
-Server *newCGIServer(int localhostOnly, int portMin, int portMax, int timeout);
-Server *newServer(int localhostOnly, int port);
+Server *newCGIServer(char *ipv6, int portMin, int portMax, int timeout);
+Server *newServer(char *ipv6, int port);
void deleteServer(Server *server);
int serverGetListeningPort(Server *server);
int serverGetFd(Server *server);
diff --git a/libhttp/httpconnection.c b/libhttp/httpconnection.c
index c8e69f6..cae467f 100644
--- a/libhttp/httpconnection.c
+++ b/libhttp/httpconnection.c
@@ -823,8 +823,17 @@ static int httpHandleCommand(struct HttpConnection *http,
const char *host = getFromHashMap(&http->header,
"host");
if (host) {
+ int brackets = 0; // For IPv6 hosts
for (char ch; (ch = *host) != '\000'; host++) {
- if (ch == ':') {
+ if (ch == '[') {
+ brackets = 1;
+ break;
+ }
+ if (ch == ']') {
+ brackets = 0;
+ break;
+ }
+ if (!brackets && ch == ':') {
*(char *)host = '\000';
break;
}
diff --git a/libhttp/server.c b/libhttp/server.c
index f52a269..2c30bd8 100644
--- a/libhttp/server.c
+++ b/libhttp/server.c
@@ -170,19 +170,19 @@ static int serverQuitHandler(struct HttpConnection *http, void *arg) {
return HTTP_DONE;
}
-struct Server *newCGIServer(int localhostOnly, int portMin, int portMax,
+struct Server *newCGIServer(char *ipv6, int portMin, int portMax,
int timeout) {
struct Server *server;
check(server = malloc(sizeof(struct Server)));
- initServer(server, localhostOnly, portMin, portMax, timeout);
+ initServer(server, ipv6, portMin, portMax, timeout);
return server;
}
-struct Server *newServer(int localhostOnly, int port) {
- return newCGIServer(localhostOnly, port, port, -1);
+struct Server *newServer(char *ipv6, int port) {
+ return newCGIServer(ipv6, port, port, -1);
}
-void initServer(struct Server *server, int localhostOnly, int portMin,
+void initServer(struct Server *server, char *ipv6, int portMin,
int portMax, int timeout) {
server->looping = 0;
server->exitAll = 0;
@@ -192,14 +192,19 @@ void initServer(struct Server *server, int localhostOnly, int portMin,
server->numConnections = 0;
int true = 1;
- server->serverFd = socket(PF_INET, SOCK_STREAM, 0);
+ server->serverFd = socket(PF_INET6, SOCK_STREAM, 0);
check(server->serverFd >= 0);
check(!setsockopt(server->serverFd, SOL_SOCKET, SO_REUSEADDR,
&true, sizeof(true)));
- struct sockaddr_in serverAddr = { 0 };
- serverAddr.sin_family = AF_INET;
- serverAddr.sin_addr.s_addr = htonl(localhostOnly
- ? INADDR_LOOPBACK : INADDR_ANY);
+ struct sockaddr_in6 serverAddr = { 0 };
+ serverAddr.sin6_family = AF_INET6;
+ if (ipv6 != NULL) {
+ if (!inet_pton(AF_INET6, ipv6, serverAddr.sin6_addr.s6_addr)) {
+ fatal("Bad ipv6 address");
+ }
+ } else {
+ serverAddr.sin6_addr = in6addr_any;
+ }
// Linux unlike BSD does not have support for picking a local port range.
// So, we have to randomly pick a port from our allowed port range, and then
@@ -214,14 +219,14 @@ void initServer(struct Server *server, int localhostOnly, int portMin,
int portStart = rand() % (portMax - portMin + 1) + portMin;
for (int p = 0; p <= portMax-portMin; p++) {
int port = (p+portStart)%(portMax-portMin+1)+ portMin;
- serverAddr.sin_port = htons(port);
+ serverAddr.sin6_port = htons(port);
if (!bind(server->serverFd, (struct sockaddr *)&serverAddr,
sizeof(serverAddr))) {
break;
}
- serverAddr.sin_port = 0;
+ serverAddr.sin6_port = 0;
}
- if (!serverAddr.sin_port) {
+ if (!serverAddr.sin6_port) {
fatal("Failed to find any available port");
}
}
@@ -231,7 +236,7 @@ void initServer(struct Server *server, int localhostOnly, int portMin,
check(!getsockname(server->serverFd, (struct sockaddr *)&serverAddr,
&socklen));
check(socklen == sizeof(serverAddr));
- server->port = ntohs(serverAddr.sin_port);
+ server->port = ntohs(serverAddr.sin6_port);
info("Listening on port %d", server->port);
check(server->pollFds = malloc(sizeof(struct pollfd)));
diff --git a/libhttp/server.h b/libhttp/server.h
index bb879fb..5ffb698 100644
--- a/libhttp/server.h
+++ b/libhttp/server.h
@@ -78,10 +78,10 @@ struct Server {
struct SSLSupport ssl;
};
-struct Server *newCGIServer(int localhostOnly, int portMin, int portMax,
+struct Server *newCGIServer(char *ipv6, int portMin, int portMax,
int timeout);
-struct Server *newServer(int localhostOnly, int port);
-void initServer(struct Server *server, int localhostOnly, int portMin,
+struct Server *newServer(char *ipv6, int port);
+void initServer(struct Server *server, char *ipv6, int portMin,
int portMax, int timeout);
void destroyServer(struct Server *server);
void deleteServer(struct Server *server);
diff --git a/shellinabox/shellinaboxd.c b/shellinabox/shellinaboxd.c
index dcf05ff..2d1d758 100644
--- a/shellinabox/shellinaboxd.c
+++ b/shellinabox/shellinaboxd.c
@@ -80,7 +80,7 @@
static int port;
static int portMin;
static int portMax;
-static int localhostOnly = 0;
+static char *ipv6 = NULL;
static int noBeep = 0;
static int numericHosts = 0;
static int enableSSL = 1;
@@ -747,7 +747,7 @@ static void usage(void) {
" -g, --group=GID switch to this group (default: %s)\n"
" -h, --help print this message\n"
" --linkify=[none|normal|agressive] default is \"normal\"\n"
- " --localhost-only only listen on 127.0.0.1\n"
+ " --ipv6 listen on a specific ipv6\n"
" --no-beep suppress all audio output\n"
" -n, --numeric do not resolve hostnames\n"
" -p, --port=PORT select a port (default: %d)\n"
@@ -839,7 +839,7 @@ static void parseArgs(int argc, char * const argv[]) {
{ "static-file", 1, 0, 'f' },
{ "group", 1, 0, 'g' },
{ "linkify", 1, 0, 0 },
- { "localhost-only", 0, 0, 0 },
+ { "ipv6", 1, 0, 0 },
{ "no-beep", 0, 0, 0 },
{ "numeric", 0, 0, 'n' },
{ "port", 1, 0, 'p' },
@@ -1001,8 +1001,8 @@ static void parseArgs(int argc, char * const argv[]) {
"\"none\", \"normal\", or \"aggressive\".");
}
} else if (!idx--) {
- // Localhost Only
- localhostOnly = 1;
+ // IPv6
+ ipv6 = optarg;
} else if (!idx--) {
// No Beep
noBeep = 1;
@@ -1197,7 +1197,7 @@ int main(int argc, char * const argv[]) {
// Create a new web server
Server *server;
if (port) {
- check(server = newServer(localhostOnly, port));
+ check(server = newServer(ipv6, port));
dropPrivileges();
setUpSSL(server);
} else {
@@ -1217,7 +1217,7 @@ int main(int argc, char * const argv[]) {
_exit(0);
}
check(!NOINTR(close(fds[0])));
- check(server = newCGIServer(localhostOnly, portMin, portMax,
+ check(server = newCGIServer(ipv6, portMin, portMax,
AJAX_TIMEOUT));
cgiServer = server;
setUpSSL(server);
--
1.7.6.5
From eee6f7180dc5dd4523264e7ce0721945ab2b78a1 Mon Sep 17 00:00:00 2001
From: Antoine Catton <acatton@tiolive.com>
Date: Wed, 11 Jan 2012 17:32:15 +0100
Subject: [PATCH 2/2] Allow to run entire command path.
---
shellinabox/launcher.c | 3 +--
1 files changed, 1 insertions(+), 2 deletions(-)
diff --git a/shellinabox/launcher.c b/shellinabox/launcher.c
index fb8a133..e116a75 100644
--- a/shellinabox/launcher.c
+++ b/shellinabox/launcher.c
@@ -1226,8 +1226,7 @@ static void execService(int width, int height, struct Service *service,
extern char **environ;
environ = environment;
- char *cmd = strrchr(argv[0], '/');
- execvp(cmd ? cmd + 1: argv[0], argv);
+ execvp(argv[0], argv);
}
void setWindowSize(int pty, int width, int height) {
--
1.7.6.5
[buildout]
extends =
../zlib/buildout.cfg
../openssl/buildout.cfg
../patch/buildout.cfg
parts = shellinabox
[shellinabox-full-path-patch]
recipe = hexagonit.recipe.download
filename = 0002-Allow-to-run-entire-command-path.patch
url = ${:_profile_base_location_}/${:filename}
download-only = true
[shellinabox-ipv6-patch]
recipe = hexagonit.recipe.download
filename = 0001-Switch-to-IPv6.patch
url = ${:_profile_base_location_}/${:filename}
download-only = true
[shellinabox]
recipe = hexagonit.recipe.cmmi
url = http://shellinabox.googlecode.com/files/shellinabox-2.10.tar.gz
md5sum = 0e144910d85d92edc54702ab9c46f032
environment =
CFLAGS = -I${zlib:location}/include -I${openssl:location}/include
LDFLAGS = -L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib -L${openssl:location}/lib -Wl,-rpath=${openssl:location}/lib
PKG_CONFIG_PATH = ${openssl:location}/lib/pkgconfig/
patch-binary = ${patch:location}/bin/patch
patch-options = -p1
patches =
${shellinabox-ipv6-patch:location}/${shellinabox-ipv6-patch:filename}
${shellinabox-full-path-patch:location}/${shellinabox-full-path-patch:filename}
......@@ -112,25 +112,25 @@ scripts =
slapproxy = slapos.proxy:main
[versions]
zc.buildout = 1.6.0-dev-SlapOS-003
zc.buildout = 1.6.0-dev-SlapOS-004
# Generated by buildout-versions
Jinja2 = 2.6
Werkzeug = 0.8.2
Werkzeug = 0.8.3
buildout-versions = 1.7
collective.recipe.template = 1.9
hexagonit.recipe.cmmi = 1.5.0
lxml = 2.3.3
meld3 = 0.6.8
netaddr = 0.7.6
slapos.core = 0.21
slapos.libnetworkcache = 0.11
slapos.core = 0.23
slapos.libnetworkcache = 0.12
xml-marshaller = 0.9.7
z3c.recipe.scripts = 1.0.1
z3c.recipe.scripts = 1.0.1
zc.recipe.egg = 1.3.2
# Required by:
# slapos.core==0.21
# slapos.core==0.23
Flask = 0.8
# Required by:
......@@ -138,21 +138,21 @@ Flask = 0.8
hexagonit.recipe.download = 1.5.0
# Required by:
# slapos.core==0.21
netifaces = 0.6
# slapos.core==0.23
netifaces = 0.8
# Required by:
# slapos.core==0.21
# slapos.libnetworkcache==0.11
# slapos.core==0.23
# slapos.libnetworkcache==0.12
# supervisor==3.0a12
# zc.buildout==1.6.0-dev-SlapOS-003
# zc.buildout==1.6.0-dev-SlapOS-004
# zope.interface==3.8.0
setuptools = 0.6c12dev-r88846
# Required by:
# slapos.core==0.21
# slapos.core==0.23
supervisor = 3.0a12
# Required by:
# slapos.core==0.21
zope.interface = 3.8.0
\ No newline at end of file
# slapos.core==0.23
zope.interface = 3.8.0
This diff is collapsed.
......@@ -2,7 +2,7 @@ from setuptools import setup, find_packages
import glob
import os
version = '0.39-dev'
version = '0.40.1'
name = 'slapos.cookbook'
long_description = open("README.txt").read() + "\n" + \
open("CHANGES.txt").read() + "\n"
......@@ -30,6 +30,7 @@ setup(name=name,
'lxml', # for full blown python interpreter
'netaddr', # to manipulate on IP addresses
'setuptools', # namespaces
'inotifyx', # to watch filesystem changes (used in lockfile)
'slapos.core', # uses internally
# 'slapos.toolbox', # needed for libcloud, cloudmgr, disabled for now
'xml_marshaller', # need to communication with slapgrid
......@@ -39,14 +40,21 @@ setup(name=name,
zip_safe=True,
entry_points={
'zc.buildout': [
'apache.frontend = slapos.recipe.apache_frontend:Recipe',
'apachephp = slapos.recipe.apachephp:Recipe',
'apacheproxy = slapos.recipe.apacheproxy:Recipe',
'apache.zope.backend = slapos.recipe.apache_zope_backend:Recipe',
'certificate_authority = slapos.recipe.certificate_authority:Recipe',
'certificate_authority.request = slapos.recipe.certificate_authority:Request',
'cron = slapos.recipe.dcron:Recipe',
'cron.d = slapos.recipe.dcron:Part',
'davstorage = slapos.recipe.davstorage:Recipe',
'dropbear = slapos.recipe.dropbear:Recipe',
'dropbear.add_authorized_key = slapos.recipe.dropbear:AddAuthorizedKey',
'dropbear.client = slapos.recipe.dropbear:Client',
'duplicity = slapos.recipe.duplicity:Recipe',
'erp5scalabilitytestbed = slapos.recipe.erp5scalabilitytestbed:Recipe',
'equeue = slapos.recipe.equeue:Recipe',
'erp5testnode = slapos.recipe.erp5testnode:Recipe',
'helloworld = slapos.recipe.helloworld:Recipe',
'generic.cloudooo = slapos.recipe.generic_cloudooo:Recipe',
......@@ -58,33 +66,47 @@ setup(name=name,
'kvm = slapos.recipe.kvm:Recipe',
'libcloud = slapos.recipe.libcloud:Recipe',
'libcloudrequest = slapos.recipe.libcloudrequest:Recipe',
'lockfile = slapos.recipe.lockfile:Recipe',
'memcached = slapos.recipe.memcached:Recipe',
'generic.memcached = slapos.recipe.generic_memcached:Recipe',
'mysql = slapos.recipe.mysql:Recipe',
'mydumper = slapos.recipe.mydumper:Recipe',
'generic.mysql = slapos.recipe.generic_mysql:Recipe',
'mkdirectory = slapos.recipe.mkdirectory:Recipe',
'nbdserver = slapos.recipe.nbdserver:Recipe',
'nosqltestbed = slapos.recipe.nosqltestbed:NoSQLTestBed',
'notifier = slapos.recipe.notifier:Recipe',
'notifier.callback = slapos.recipe.notifier:Callback',
'notifier.notify = slapos.recipe.notifier:Notify',
'lamp = slapos.recipe.lamp:Request',
'lamp.request = slapos.recipe.lamp:Request',
'lamp.static = slapos.recipe.lamp:Static',
'lamp.simple = slapos.recipe.lamp:Simple',
'logrotate = slapos.recipe.logrotate:Recipe',
'logrotate.d = slapos.recipe.logrotate:Part',
'pbs = slapos.recipe.pbs:Recipe',
'publish = slapos.recipe.publish:Recipe',
'publishurl = slapos.recipe.publishurl:Recipe',
'pwgen = slapos.recipe.pwgen:Recipe',
'proactive = slapos.recipe.proactive:Recipe',
'request = slapos.recipe.request:Recipe',
'seleniumrunner = slapos.recipe.seleniumrunner:Recipe',
'sheepdogtestbed = slapos.recipe.sheepdogtestbed:SheepDogTestBed',
'shell = slapos.recipe.shell:Recipe',
'shellinabox = slapos.recipe.shellinabox:Recipe',
'symbolic.link = slapos.recipe.symbolic_link:Recipe',
'softwaretype = slapos.recipe.softwaretype:Recipe',
'siptester = slapos.recipe.siptester:SipTesterRecipe',
'simplelogger = slapos.recipe.simplelogger:Recipe',
'slaprunner = slapos.recipe.slaprunner:Recipe',
'sshkeys_authority = slapos.recipe.sshkeys_authority:Recipe',
'sshkeys_authority.request = slapos.recipe.sshkeys_authority:Request',
'sphinx= slapos.recipe.sphinx:Recipe',
'stunnel = slapos.recipe.stunnel:Recipe',
'testnode = slapos.recipe.testnode:Recipe',
'urlparse = slapos.recipe._urlparse:Recipe',
'vifib = slapos.recipe.vifib:Recipe',
'waitfor = slapos.recipe.waitfor:Recipe',
'xwiki = slapos.recipe.xwiki:Recipe',
'zabbixagent = slapos.recipe.zabbixagent:Recipe',
'generic.zope = slapos.recipe.generic_zope:Recipe',
......
apache_frontend
==========
Frontend using Apache, allowing to rewrite and proxy URLs like myinstance.myfrontenddomainname.com to real IP/URL of myinstance.
\ No newline at end of file
helloworld
==========
It only installs a dummy wrapper saying "Hello World!" and sends to the master an "HelloWorld" parameter.
seleniumrunner
==========
Allows to run selenium tests through browser and xvfb. Posts the results on
Nexedi ERP5.
Parameters
=====
* project : name of the project inside of ERP5 test result instance
* user : username to use in ERP5 instance to test
* password : password to use in ERP5 instance to test
* suite_name : name of test suite to launch
* url : url to portal_test of ERP5 isntance to test
* test_report_instance_url : url of test_result_module to put results
* Example :
<?xml version="1.0" encoding="utf-8"?>
<instance>
<parameter id="project">Vifib</parameter>
<parameter id="user">myuser</parameter>
<parameter id="password">mypassword</parameter>
<parameter id="suite_name">my_zuite</parameter>
<parameter id="url">https://myerp5totest/erp5/portal_tests</parameter>
<parameter id="test_report_instance_url">https://user:passwordwww.myerp5withtestresults.com/test_result_module/</parameter>
</instance>
##############################################################################
#
# Copyright (c) 2010 Vifib SARL and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
from urlparse import urlparse
from slapos.recipe.librecipe import GenericBaseRecipe
class Recipe(GenericBaseRecipe):
def _options(self, options):
url = urlparse(options['url'])
def to_str(data):
if data is None:
return ''
return str(data)
options.update(scheme=url.scheme,
username=to_str(url.username),
password=to_str(url.password),
host=to_str(url.hostname),
port=to_str(url.port),
path=url.path.strip('/'),
params=url.params,
query=url.query,
fragment=url.fragment)
def install(self):
return []
This diff is collapsed.
import os
import subprocess
import time
import ConfigParser
import uuid
def popenCommunicate(command_list, input=None):
subprocess_kw = dict(stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
if input is not None:
subprocess_kw.update(stdin=subprocess.PIPE)
popen = subprocess.Popen(command_list, **subprocess_kw)
result = popen.communicate(input)[0]
if popen.returncode is None:
popen.kill()
if popen.returncode != 0:
raise ValueError('Issue during calling %r, result was:\n%s' % (
command_list, result))
return result
class CertificateAuthority:
def __init__(self, key, certificate, openssl_binary,
openssl_configuration, request_dir):
self.key = key
self.certificate = certificate
self.openssl_binary = openssl_binary
self.openssl_configuration = openssl_configuration
self.request_dir = request_dir
def checkAuthority(self):
file_list = [ self.key, self.certificate ]
ca_ready = True
for f in file_list:
if not os.path.exists(f):
ca_ready = False
break
if ca_ready:
return
for f in file_list:
if os.path.exists(f):
os.unlink(f)
try:
# no CA, let us create new one
popenCommunicate([self.openssl_binary, 'req', '-nodes', '-config',
self.openssl_configuration, '-new', '-x509', '-extensions', 'v3_ca',
'-keyout', self.key, '-out', self.certificate, '-days', '10950'],
# Authority name will be random, so no instance has the same issuer
'Certificate Authority %s\n' % uuid.uuid1())
except:
try:
for f in file_list:
if os.path.exists(f):
os.unlink(f)
except:
# do not raise during cleanup
pass
raise
def _checkCertificate(self, common_name, key, certificate):
file_list = [key, certificate]
ready = True
for f in file_list:
if not os.path.exists(f):
ready = False
break
if ready:
return False
for f in file_list:
if os.path.exists(f):
os.unlink(f)
csr = certificate + '.csr'
try:
popenCommunicate([self.openssl_binary, 'req', '-config',
self.openssl_configuration, '-nodes', '-new', '-keyout',
key, '-out', csr, '-days', '3650'],
common_name + '\n')
try:
popenCommunicate([self.openssl_binary, 'ca', '-batch', '-config',
self.openssl_configuration, '-out', certificate,
'-infiles', csr])
finally:
if os.path.exists(csr):
os.unlink(csr)
except:
try:
for f in file_list:
if os.path.exists(f):
os.unlink(f)
except:
# do not raise during cleanup
pass
raise
else:
return True
def checkRequestDir(self):
for request_file in os.listdir(self.request_dir):
parser = ConfigParser.RawConfigParser()
parser.readfp(open(os.path.join(self.request_dir, request_file), 'r'))
if self._checkCertificate(parser.get('certificate', 'name'),
parser.get('certificate', 'key_file'), parser.get('certificate',
'certificate_file')):
print 'Created certificate %r' % parser.get('certificate', 'name')
def runCertificateAuthority(args):
ca_conf = args[0]
ca = CertificateAuthority(ca_conf['key'], ca_conf['certificate'],
ca_conf['openssl_binary'], ca_conf['openssl_configuration'],
ca_conf['request_dir'])
while True:
ca.checkAuthority()
ca.checkRequestDir()
time.sleep(60)
# Apache configuration file for Zope
# Automatically generated
# Basic server configuration
PidFile "%(pid_file)s"
LockFile "%(lock_file)s"
ServerName %(server_name)s
DocumentRoot %(document_root)s
%(listen)s
ServerAdmin %(server_admin)s
DefaultType text/plain
TypesConfig conf/mime.types
AddType application/x-compress .Z
AddType application/x-gzip .gz .tgz
# As backend is trusting REMOTE_USER header unset it always
RequestHeader unset REMOTE_USER
# SSL Configuration
%(ssl_snippet)s
# Log configuration
ErrorLog "%(error_log)s"
LogLevel warn
LogFormat "%%h %%{REMOTE_USER}i %%l %%u %%t \"%%r\" %%>s %%b \"%%{Referer}i\" \"%%{User-Agent}i\"" combined
LogFormat "%%h %%{REMOTE_USER}i %%l %%u %%t \"%%r\" %%>s %%b" common
CustomLog "%(access_log)s" common
# Directory protection
<Directory />
Options FollowSymLinks
AllowOverride None
Order deny,allow
Deny from all
</Directory>
%(path_enable)s
# Rewrite part
RewriteEngine On
# Define the two rewritemaps : one for zope, one generic
RewriteMap apachemapzope txt:%(apachemapzope_path)s
RewriteMap apachemapgeneric txt:%(apachemap_path)s
# First, we check if we have a zope backend server
# If so, let's use Virtual Host Daemon rewrite
#RewriteCond ${apachemapzope:%%{SERVER_NAME}} >""
#RewriteRule ^/(\w+)($|/.*) ${apachemapzope:$1}/VirtualHostBase/https/%(apache_domain)s:%(port)s/VirtualHostRoot/_vh_$1$2 [L,P]
# If we have generic backend server, let's rewrite without virtual host daemon
RewriteCond ${apachemapgeneric:%%{SERVER_NAME}} >""
RewriteRule ^/(.*)$ ${apachemapgeneric:%%{SERVER_NAME}}/$1 [L,P]
# If nothing exist : put a nice error
ErrorDocument 404 /notfound.html
# List of modules
LoadModule authz_host_module modules/mod_authz_host.so
LoadModule log_config_module modules/mod_log_config.so
LoadModule setenvif_module modules/mod_setenvif.so
LoadModule version_module modules/mod_version.so
LoadModule proxy_module modules/mod_proxy.so
LoadModule proxy_http_module modules/mod_proxy_http.so
LoadModule ssl_module modules/mod_ssl.so
LoadModule mime_module modules/mod_mime.so
LoadModule dav_module modules/mod_dav.so
LoadModule dav_fs_module modules/mod_dav_fs.so
LoadModule negotiation_module modules/mod_negotiation.so
LoadModule rewrite_module modules/mod_rewrite.so
LoadModule headers_module modules/mod_headers.so
LoadModule antiloris_module modules/mod_antiloris.so
# Path protected
<Location %(path)s>
Order Deny,Allow
Allow from %(access_control_string)s
</Location>
<Location %(location)s>
Order Deny,Allow
Deny from all
Allow from %(allow_string)s
</Location>
SSLEngine on
SSLProxyEngine on
SSLCertificateFile %(login_certificate)s
SSLCertificateKeyFile %(login_key)s
SSLRandomSeed startup builtin
SSLRandomSeed connect builtin
%(file_list)s {
daily
dateext
rotate 30
compress
notifempty
sharedscripts
create
postrotate
%(postrotate)s
endscript
olddir %(olddir)s
}
<html>
<head>
<title>Instance not found</title>
</head>
<body>
<h1>This instance has not been found.</h1>
<p>If this error persists, please check your instance URL and status on SlapOS Master.</p>
</body>
</html>
This diff is collapsed.
[%(name)s]
accept = %(public_ip)s:%(public_port)s
connect = %(private_ip)s:%(private_port)s
foreground = yes
output = %(log)s
pid = %(pid_file)s
syslog = no
client = yes
CApath = %(ca_path)s
key = %(key)s
CRLpath = %(ca_crl)s
cert = %(cert)s
sslVersion = SSLv3
socket = l:TCP_NODELAY=1
socket = r:TCP_NODELAY=1
%(entry_str)s
# This is a basic VCL configuration file for varnish. See the vcl(7)
# man page for details on VCL syntax and semantics.
#
# Default backend definition. Set this to point to your content
# server.
#
backend default {
.host = "%(backend_host)s";
.port = "%(backend_port)s";
.probe = {
.url = "/";
.timeout = 10s;
.interval = 10s;
.window = 4;
.threshold = 3;
}
}
#
# Below is a commented-out copy of the default VCL logic. If you
# redefine any of these subroutines, the built-in logic will be
# appended to your code.
#
# sub vcl_recv {
# if (req.http.x-forwarded-for) {
# set req.http.X-Forwarded-For =
# req.http.X-Forwarded-For ", " client.ip;
# } else {
# set req.http.X-Forwarded-For = client.ip;
# }
# if (req.request != "GET" &&
# req.request != "HEAD" &&
# req.request != "PUT" &&
# req.request != "POST" &&
# req.request != "TRACE" &&
# req.request != "OPTIONS" &&
# req.request != "DELETE") {
# /* Non-RFC2616 or CONNECT which is weird. */
# return (pipe);
# }
# if (req.request != "GET" && req.request != "HEAD") {
# /* We only deal with GET and HEAD by default */
# return (pass);
# }
# if (req.http.Authorization || req.http.Cookie) {
# /* Not cacheable by default */
# return (pass);
# }
# return (lookup);
# }
sub vcl_recv {
if (req.http.cache-control ~ "no-cache") {
purge_url(req.url);
}
if (req.url ~ "\.(css|js|ico)$") {
unset req.http.cookie;
}
# remove bogus cookies
if (req.http.Cookie) {
set req.http.Cookie = regsuball(req.http.Cookie, "(^|; ) *__utm.=[^;]+;? *", "\1");
set req.http.Cookie = regsuball(req.http.Cookie, "(^|; ) *__ac_name=\x22\x22;? *", "\1");
set req.http.Cookie = regsuball(req.http.Cookie, "(^|; ) *__ac=\x22Og.3D.3D\x22;? *", "\1");
}
if (req.http.Cookie == "") {
remove req.http.Cookie;
}
if (req.http.x-forwarded-for) {
set req.http.X-Forwarded-For =
req.http.X-Forwarded-For ", " client.ip;
} else {
set req.http.X-Forwarded-For = client.ip;
}
if (req.request != "GET" &&
req.request != "HEAD" &&
req.request != "PUT" &&
req.request != "POST" &&
req.request != "TRACE" &&
req.request != "OPTIONS" &&
req.request != "DELETE") {
/* Non-RFC2616 or CONNECT which is weird. */
return (pipe);
}
if (req.request != "GET" && req.request != "HEAD") {
/* We only deal with GET and HEAD by default */
return (pass);
}
if (req.http.Authorization) {
/* Not cacheable by default */
return (pass);
}
if (req.http.Cookie && req.http.Cookie ~ "(^|; ) *__ac=") {
/* Not cacheable for authorised users,
but KM images are cacheable */
if (!(req.url ~ "/km_img/.*\.(png|gif)$")) {
return (pass);
}
}
# XXX login form can defer based on __ac_name cookie value
if (req.url ~ "/(login_form|WebSite_viewLoginDialog)($|\?)") {
return (pass);
}
if (req.backend.healthy) {
set req.grace = 1h;
} else {
set req.grace = 1w;
}
return (lookup);
}
#
# sub vcl_pipe {
# # Note that only the first request to the backend will have
# # X-Forwarded-For set. If you use X-Forwarded-For and want to
# # have it set for all requests, make sure to have:
# # set req.http.connection = "close";
# # here. It is not set by default as it might break some broken web
# # applications, like IIS with NTLM authentication.
# return (pipe);
# }
#
# sub vcl_pass {
# return (pass);
# }
#
# sub vcl_hash {
# set req.hash += req.url;
# if (req.http.host) {
# set req.hash += req.http.host;
# } else {
# set req.hash += server.ip;
# }
# return (hash);
# }
#
# sub vcl_hit {
# if (!obj.cacheable) {
# return (pass);
# }
# return (deliver);
# }
#
# sub vcl_miss {
# return (fetch);
# }
#
# sub vcl_fetch {
# if (!beresp.cacheable) {
# return (pass);
# }
# if (beresp.http.Set-Cookie) {
# return (pass);
# }
# return (deliver);
# }
sub vcl_fetch {
# we only cache 200 (OK) and 304 (Not Modified) responses.
if (beresp.status != 200 && beresp.status != 304) {
set beresp.cacheable = false;
}
if (beresp.http.cache-control ~ "no-cache") {
set beresp.cacheable = false;
}
if (!beresp.cacheable) {
unset beresp.http.expires;
set beresp.http.cache-control = "no-cache";
return (pass);
}
# we don't care haproxy's cookie.
if (beresp.http.Set-Cookie && beresp.http.Set-Cookie !~ "^SERVERID=[^;]+; path=/$") {
return (pass);
}
if (req.url ~ "\.(css|js|ico)$") {
unset beresp.http.set-cookie;
set beresp.http.cache-control = regsub(beresp.http.cache-control, "^", "public,");
set beresp.http.cache-control = regsub(beresp.http.cache-control, ",$", "");
}
# remove some headers added by caching policy manager to avoid
# '304 Not Modified' in case of login <-> logout switching.
if (beresp.http.content-type ~ "^text/html") {
unset beresp.http.last-modified;
}
if (beresp.cacheable) {
/* Remove Expires from backend, it's not long enough */
unset beresp.http.expires;
/* Set the clients TTL on this object */
set beresp.http.cache-control = "max-age = 900";
/* Set how long Varnish will keep it */
set beresp.ttl = 1w;
/* marker for vcl_deliver to reset Age: */
set beresp.http.magicmarker = "1";
}
set beresp.grace = 1w;
return (deliver);
}
#
# sub vcl_deliver {
# return (deliver);
# }
sub vcl_deliver {
if (resp.http.magicmarker) {
/* Remove the magic marker */
unset resp.http.magicmarker;
/* By definition we have a fresh object */
set resp.http.age = "0";
}
if (obj.hits > 0) {
set resp.http.X-Cache = obj.hits;
} else {
set resp.http.X-Cache = "MISS";
}
return (deliver);
}
#
# sub vcl_error {
# set obj.http.Content-Type = "text/html; charset=utf-8";
# synthetic {"
# <?xml version="1.0" encoding="utf-8"?>
# <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
# "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
# <html>
# <head>
# <title>"} obj.status " " obj.response {"</title>
# </head>
# <body>
# <h1>Error "} obj.status " " obj.response {"</h1>
# <p>"} obj.response {"</p>
# <h3>Guru Meditation:</h3>
# <p>XID: "} req.xid {"</p>
# <hr>
# <p>Varnish cache server</p>
# </body>
# </html>
# "};
# return (deliver);
# }
##############################################################################
#
# Copyright (c) 2010 Vifib SARL and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
import shutil
import os
import signal
from binascii import b2a_uu as uuencode
from slapos.recipe.librecipe import GenericBaseRecipe
class Recipe(GenericBaseRecipe):
def install(self):
path_list = []
# Copy application
shutil.rmtree(self.options['htdocs'])
shutil.copytree(self.options['source'],
self.options['htdocs'])
# Install php.ini
php_ini = self.createFile(os.path.join(self.options['php-ini-dir'],
'php.ini'),
self.substituteTemplate(self.getTemplateFilename('php.ini.in'),
dict(tmp_directory=self.options['tmp-dir']))
)
path_list.append(php_ini)
# Install apache
apache_config = dict(
pid_file=self.options['pid-file'],
lock_file=self.options['lock-file'],
ip=self.options['ip'],
port=self.options['port'],
error_log=self.options['error-log'],
access_log=self.options['access-log'],
document_root=self.options['htdocs'],
php_ini_dir=self.options['php-ini-dir'],
)
httpd_conf = self.createFile(self.options['httpd-conf'],
self.substituteTemplate(self.getTemplateFilename('apache.in'),
apache_config)
)
path_list.append(httpd_conf)
wrapper = self.createPythonScript(self.options['wrapper'],
'slapos.recipe.librecipe.execute.execute',
[self.options['httpd-binary'], '-f', self.options['httpd-conf'],
'-DFOREGROUND']
)
path_list.append(wrapper)
secret_key_filename = os.path.join(self.buildout['buildout']['directory'],
'.php_secret_key')
if not os.path.exists(secret_key_filename):
secret_key = uuencode(os.urandom(45)).strip()
# Remove unsafe characters
secret_key = secret_key.translate(None, '"\'')
with open(secret_key_filename, 'w') as secret_key_file:
secret_key_file.write(secret_key)
else:
with open(secret_key_filename, 'r') as secret_key_file:
secret_key = secret_key_file.read()
application_conf = dict(mysql_database=self.options['mysql-database'],
mysql_user=self.options['mysql-username'],
mysql_password=self.options['mysql-password'],
mysql_host='%s:%s' % (self.options['mysql-host'],
self.options['mysql-port']),
secret_key=secret_key,
)
directory, file_ = os.path.split(self.options['configuration'])
path = self.options['htdocs']
if directory:
path = os.path.join(path, directory)
if not os.path.exists(path):
os.makedirs(path)
if not os.path.isdir(path):
raise OSError("Cannot create %r." % path)
destination = os.path.join(path, file_)
config = self.createFile(destination,
self.substituteTemplate(self.options['template'], application_conf))
path_list.append(config)
if os.path.exists(self.options['pid-file']):
# Reload apache configuration
with open(self.options['pid-file']) as pid_file:
pid = int(pid_file.read().strip(), 10)
os.kill(pid, signal.SIGUSR1) # Graceful restart
return path_list
# Apache static configuration
# Automatically generated
# Basic server configuration
PidFile "%(pid_file)s"
LockFile "%(lock_file)s"
Listen %(ip)s:%(port)s
PHPINIDir %(php_ini_dir)s
ServerAdmin someone@email
DefaultType text/plain
TypesConfig conf/mime.types
AddType application/x-compress .Z
AddType application/x-gzip .gz .tgz
AddType application/x-httpd-php .php .phtml .php5 .php4
AddType application/x-httpd-php-source .phps
# Log configuration
ErrorLog "%(error_log)s"
LogLevel warn
LogFormat "%%h %%{REMOTE_USER}i %%l %%u %%t \"%%r\" %%>s %%b \"%%{Referer}i\" \"%%{User-Agent}i\"" combined
LogFormat "%%h %%{REMOTE_USER}i %%l %%u %%t \"%%r\" %%>s %%b" common
CustomLog "%(access_log)s" common
# Directory protection
<Directory />
Options FollowSymLinks
AllowOverride None
Order deny,allow
Deny from all
</Directory>
<Directory %(document_root)s>
Options FollowSymLinks
AllowOverride All
Order allow,deny
Allow from all
</Directory>
DocumentRoot %(document_root)s
DirectoryIndex index.html index.php
# List of modules
LoadModule authz_host_module modules/mod_authz_host.so
LoadModule log_config_module modules/mod_log_config.so
LoadModule setenvif_module modules/mod_setenvif.so
LoadModule version_module modules/mod_version.so
LoadModule proxy_module modules/mod_proxy.so
LoadModule proxy_http_module modules/mod_proxy_http.so
LoadModule mime_module modules/mod_mime.so
LoadModule dav_module modules/mod_dav.so
LoadModule dav_fs_module modules/mod_dav_fs.so
LoadModule negotiation_module modules/mod_negotiation.so
LoadModule rewrite_module modules/mod_rewrite.so
LoadModule headers_module modules/mod_headers.so
LoadModule dir_module modules/mod_dir.so
LoadModule php5_module modules/libphp5.so
LoadModule alias_module modules/mod_alias.so
[PHP]
engine = On
safe_mode = Off
expose_php = Off
error_reporting = E_ALL & ~(E_DEPRECATED|E_NOTICE|E_WARNING)
display_errors = On
display_startup_errors = Off
log_errors = On
log_errors_max_len = 1024
ignore_repeated_errors = Off
ignore_repeated_source = Off
session.save_path = "%(tmp_directory)s"
session.auto_start = 0
date.timezone = Europe/Paris
file_uploads = On
upload_max_filesize = 8M
post_max_size = 8M
magic_quotes_gpc=0ff
##############################################################################
#
# Copyright (c) 2010 Vifib SARL and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
import os
import signal
from slapos.recipe.librecipe import GenericBaseRecipe
class Recipe(GenericBaseRecipe):
def install(self):
path_list = []
# Install apache
apache_config = dict(
pid_file=self.options['pid-file'],
lock_file=self.options['lock-file'],
ip=self.options['ip'],
port=self.options['port'],
error_log=self.options['error-log'],
access_log=self.options['access-log'],
backend_url=self.options['url'],
)
httpd_conf = self.createFile(self.options['httpd-conf'],
self.substituteTemplate(self.getTemplateFilename('apache.in'),
apache_config)
)
path_list.append(httpd_conf)
wrapper = self.createPythonScript(self.options['wrapper'],
'slapos.recipe.librecipe.execute.execute',
[self.options['httpd-binary'], '-f', self.options['httpd-conf'],
'-DFOREGROUND']
)
path_list.append(wrapper)
if os.path.exists(self.options['pid-file']):
# Reload apache configuration
with open(self.options['pid-file']) as pid_file:
pid = int(pid_file.read().strip(), 10)
os.kill(pid, signal.SIGUSR1) # Graceful restart
return path_list
# Apache static configuration
# Automatically generated
# Basic server configuration
PidFile "%(pid_file)s"
LockFile "%(lock_file)s"
Listen %(ip)s:%(port)s
ServerAdmin someone@email
DefaultType text/plain
TypesConfig conf/mime.types
AddType application/x-compress .Z
AddType application/x-gzip .gz .tgz
# Log configuration
ErrorLog "%(error_log)s"
LogLevel warn
LogFormat "%%h %%{REMOTE_USER}i %%l %%u %%t \"%%r\" %%>s %%b \"%%{Referer}i\" \"%%{User-Agent}i\"" combined
LogFormat "%%h %%{REMOTE_USER}i %%l %%u %%t \"%%r\" %%>s %%b" common
CustomLog "%(access_log)s" common
# Directory protection
<Directory />
Options FollowSymLinks
AllowOverride None
Order deny,allow
Deny from all
</Directory>
ProxyPass / %(backend_url)s
# List of modules
LoadModule authz_host_module modules/mod_authz_host.so
LoadModule log_config_module modules/mod_log_config.so
LoadModule setenvif_module modules/mod_setenvif.so
LoadModule version_module modules/mod_version.so
LoadModule proxy_module modules/mod_proxy.so
LoadModule proxy_http_module modules/mod_proxy_http.so
LoadModule mime_module modules/mod_mime.so
LoadModule negotiation_module modules/mod_negotiation.so
LoadModule rewrite_module modules/mod_rewrite.so
LoadModule headers_module modules/mod_headers.so
LoadModule dir_module modules/mod_dir.so
LoadModule alias_module modules/mod_alias.so
......@@ -118,7 +118,7 @@ class Request(Recipe):
if os.path.islink(link):
os.unlink(link)
elif os.path.exists(link):
raise OSError("%r file should be a symbolic link.")
raise OSError("%r file should be a symbolic link." % link)
os.symlink(key, key_file)
os.symlink(certificate, cert_file)
......
......@@ -2,6 +2,7 @@ import os
import subprocess
import time
import ConfigParser
import uuid
def popenCommunicate(command_list, input=None):
......@@ -44,7 +45,7 @@ class CertificateAuthority:
popenCommunicate([self.openssl_binary, 'req', '-nodes', '-config',
self.openssl_configuration, '-new', '-x509', '-extensions',
'v3_ca', '-keyout', self.key, '-out', self.certificate,
'-days', '10950'], 'Automatic Certificate Authority\n')
'-days', '10950'], 'Certificate Authority %s\n' % uuid.uuid1())
except:
try:
for f in file_list:
......
......@@ -24,106 +24,88 @@
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
from slapos.recipe.librecipe import BaseSlapRecipe
import os
import subprocess
import pkg_resources
import zc.buildout
import zc.recipe.egg
import sys
class Recipe(BaseSlapRecipe):
def getTemplateFilename(self, template_name):
return pkg_resources.resource_filename(__name__,
'template/%s' % template_name)
def _install(self):
self.path_list = []
self.requirements, self.ws = self.egg.working_set()
document_root = self.createDataDirectory('www')
apache_config = self.installApache(document_root)
self.setConnectionUrl(scheme='webdavs',
host=apache_config['ip'],
port=apache_config['port'],
auth=(apache_config['user'],
apache_config['password']))
return self.path_list
def installApache(self, document_root, ip=None, port=None):
if ip is None:
ip=self.getGlobalIPv6Address()
if port is None:
port = '9080'
htpasswd_config = self.createHtpasswd()
ssl_config = self.createCertificate(size=2048)
import httplib
import base64
apache_config = dict(
pid_file=os.path.join(self.run_directory, 'httpd.pid'),
lock_file=os.path.join(self.run_directory, 'httpd.lock'),
davlock_db=os.path.join(self.run_directory, 'davdb.lock'),
ip=ip,
port=port,
error_log=os.path.join(self.log_directory, 'httpd-error.log'),
access_log=os.path.join(self.log_directory, 'httpd-access.log'),
document_root=document_root,
modules_dir=self.options['apache_modules_dir'],
mime_types=self.options['apache_mime_file'],
server_root=self.work_directory,
email_address='admin@vifib.net',
htpasswd_file=htpasswd_config['htpasswd_file'],
ssl_certificate=ssl_config['certificate'],
ssl_key=ssl_config['key'],
)
httpd_config_file = self.createConfigurationFile('httpd.conf',
self.substituteTemplate(self.getTemplateFilename('httpd.conf.in'),
apache_config))
self.path_list.append(httpd_config_file)
apache_runner = zc.buildout.easy_install.scripts(
[('httpd', 'slapos.recipe.librecipe.execute', 'execute')],
self.ws, sys.executable, self.wrapper_directory,
arguments=[self.options['apache_binary'],
'-f', httpd_config_file,
'-DFOREGROUND',
]
)[0]
self.path_list.append(apache_runner)
return dict(ip=apache_config['ip'],
port=apache_config['port'],
user=htpasswd_config['user'],
password=htpasswd_config['password']
)
def createHtpasswd(self):
htpasswd = self.createConfigurationFile('htpasswd', '')
self.path_list.append(htpasswd)
password = self.generatePassword()
user = 'user'
subprocess.check_call([self.options['apache_htpasswd'],
'-bc', htpasswd,
from slapos.recipe.librecipe import GenericBaseRecipe
class Recipe(GenericBaseRecipe):
def _options(self, options):
options['password'] = self.generatePassword()
def install(self):
path_list = []
htpasswd_file = self.options['htpasswd-file']
# Create or empty the file
open(htpasswd_file, 'w').close()
path_list.append(htpasswd_file)
user = self.options['user']
password = self.options['password']
subprocess.check_call([self.options['apache-htpasswd'],
'-bc', htpasswd_file,
user, password
])
return dict(htpasswd_file=htpasswd,
user=user,
password=password)
def createCertificate(self, size=1024, subject='/C=FR/L=Marcq-en-Baroeul/O=Nexedi'):
key_file = os.path.join(self.etc_directory, 'httpd.key')
certificate_file = os.path.join(self.etc_directory, 'httpd.crt')
files = [key_file, certificate_file, ]
if not all([os.path.exists(f) for f in files]):
for f in files:
if os.path.exists(f):
os.unlink(f)
subprocess.check_call([self.options['openssl_binary'],
'req', '-x509', '-nodes',
'-newkey', 'rsa:%s' % size,
'-subj', str(subject),
'-out', certificate_file,
'-keyout', key_file
])
return dict(key=key_file,
certificate=certificate_file)
apache_config = dict(
pid_file=self.options['pid-file'],
lock_file=self.options['lock-file'],
davlock_db=self.options['davdb-lock'],
ip=self.options['ip'],
port=self.options['port'],
error_log=self.options['error-log'],
access_log=self.options['access-log'],
document_root=self.options['htdocs'],
modules_dir=self.options['apache-modules-dir'],
mime_types=self.options['apache-mime-file'],
server_root=self.options['root'],
email_address=self.options['email-address'],
htpasswd_file=htpasswd_file,
ssl_certificate=self.options['cert-file'],
ssl_key=self.options['key-file'],
)
# Create logfiles
for log in [self.options['error-log'], self.options['access-log']]:
open(log, 'a').close()
config_file = self.createFile(self.options['conf-file'],
self.substituteTemplate(self.getTemplateFilename('httpd.conf.in'),
apache_config)
)
path_list.append(config_file)
wrapper = self.createPythonScript(self.options['wrapper'],
'slapos.recipe.librecipe.execute.execute',
[self.options['apache-binary'], '-f', config_file, '-DFOREGROUND'])
path_list.append(wrapper)
promise = self.createPythonScript(self.options['promise'],
__name__ + '.promise',
dict(host=self.options['ip'], port=int(self.options['port']),
user=self.options['user'], password=self.options['password'])
)
path_list.append(promise)
return path_list
def promise(args):
host = args['host']
port = args['port']
user = args['user']
password = args['password']
connection = httplib.HTTPSConnection(host, port)
auth = base64.b64encode('%s:%s' % (user, password))
connection.request('OPTIONS', '/',
headers=dict(
Authorization='Basic %s' % auth,
)
)
connection.getresponse()
return 0
##############################################################################
#
# Copyright (c) 2010 Vifib SARL and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
import os
import itertools
from slapos.recipe.librecipe import GenericBaseRecipe
class KnownHostsFile(dict):
def __init__(self, filename):
self._filename = filename
def _load(self):
if os.path.exists(self._filename):
with open(self._filename, 'r') as keyfile:
for line in keyfile:
host, key = [column.strip() for column in line.split(' ', 1)]
self[host] = key
def _dump(self):
with open(self._filename, 'w') as keyfile:
for key, value in self.items():
if key is not None and value is not None:
keyfile.write('%(host)s %(key)s\n' % {'host': key,
'key': value})
def __enter__(self):
self._load()
def __exit__(self, exc_type, exc_value, traceback):
self._dump()
class AuthorizedKeysFile(object):
def __init__(self, filename):
self.filename = filename
def append(self, key):
"""Append the key to the file if the key's not in the file
"""
# Create the file it it does not exist
try:
file_ = os.open(self.filename, os.O_CREAT | os.O_EXCL)
os.close(file_)
except:
pass
with open(self.filename, 'r') as keyfile:
# itertools.imap avoid loading all the authorized_keys file in
# memory which would be counterproductive.
present = (key.strip() in itertools.imap(lambda k: k.strip(),
keyfile))
try:
keyfile.seek(-1, os.SEEK_END)
ended_by_newline = (keyfile.read() == '\n')
except IOError:
ended_by_newline = True
if not present:
with open(self.filename, 'a') as keyfile:
if not ended_by_newline:
keyfile.write('\n')
keyfile.write(key.strip())
class Recipe(GenericBaseRecipe):
def install(self):
path_list = []
dropbear_cmd = [self.options['dropbear-binary']]
# Don't fork into background
dropbear_cmd.append('-F')
# Log on stderr
dropbear_cmd.append('-E')
# Don't display motd
dropbear_cmd.append('-m')
# Disable password login
dropbear_cmd.extend(['-s', '-g'])
# Disable port forwarding
dropbear_cmd.extend(['-j', '-k'])
host = self.options['host']
if ':' in host:
host = '[%s]' % host
port = self.options['port']
binding_address = '%s:%s' % (host, port)
dropbear_cmd.extend(['-p', binding_address])
# Single user mode
dropbear_cmd.append('-n')
if 'dss-keyfile' in self.options:
dropbear_cmd.extend(['-d', self.options['dss-keyfile']])
else:
dropbear_cmd.extend(['-r', self.options['rsa-keyfile']])
env = {}
if 'home' in self.options:
env['DROPBEAR_OVERRIDE_HOME'] = self.options['home']
if 'shell' in self.options:
env['DROPBEAR_OVERRIDE_SHELL'] = self.options['shell']
wrapper = self.createPythonScript(
self.options['wrapper'],
'slapos.recipe.librecipe.execute.executee',
(dropbear_cmd, env, )
)
path_list.append(wrapper)
return path_list
class Client(GenericBaseRecipe):
def install(self):
env = dict()
if 'home' in self.options:
env['HOME'] = self.options['home']
self.createDirectory(self.options['home'], '.ssh')
dropbear_cmd = [self.options['dbclient-binary'], '-T']
if self.optionIsTrue('force-host-key', default=False):
dropbear_cmd.extend(['-y'])
if 'identity-file' in self.options:
dropbear_cmd.extend(['-i', self.options['identity-file']])
wrapper = self.createPythonScript(
self.options['wrapper'],
'slapos.recipe.librecipe.execute.executee',
(dropbear_cmd, env, )
)
return [wrapper]
class AddAuthorizedKey(GenericBaseRecipe):
def install(self):
path_list = []
ssh = self.createDirectory(self.options['home'], '.ssh')
path_list.append(ssh)
authorized_keys = AuthorizedKeysFile(os.path.join(ssh, 'authorized_keys'))
authorized_keys.append(self.options['key'])
return path_list
......@@ -30,15 +30,23 @@ class Recipe(GenericBaseRecipe):
def install(self):
remote_url = self.options['remote_backup']
backup_directory = self.options['directory']
remote_url = self.options['remote-backup']
backup_directory = self.options['local-directory']
wrapper = self.createPythonScript(
self.options['wrapper'],
'slapos.recipe.librecipe.execute.execute',
[self.options['duplicity_binary'], '--no-encryption',
backup_directory, remote_url]
)
return [wrapper]
cmd = [self.options['duplicity-binary'],]
options = ['--no-encryption', '--archive-dir', self.options['cache']]
if self.optionIsTrue('recover', False):
options.append('--force')
# duplicity [options] remote backup
cmd.extend(options)
cmd.extend([remote_url, backup_directory])
else:
# duplicity [options] local remote
cmd.extend(options)
cmd.extend([backup_directory, remote_url])
wrapper = self.createPythonScript(self.options['wrapper'],
'slapos.recipe.librecipe.execute.execute', cmd)
return [wrapper]
##############################################################################
#
# Copyright (c) 2010 Vifib SARL and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
from slapos.recipe.librecipe import GenericBaseRecipe
class Recipe(GenericBaseRecipe):
def install(self):
commandline = [self.options['equeue-binary']]
commandline.extend(['--database', self.options['database']])
commandline.extend(['-l', self.options['log']])
if 'loglevel' in self.options:
commandline.extend(['--loglevel', self.options['loglevel']])
commandline.append(self.options['socket'])
return [self.createPythonScript(
self.options['wrapper'],
'slapos.recipe.librecipe.execute.execute',
commandline,
)]
......@@ -24,86 +24,80 @@
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
from slapos.recipe.librecipe import BaseSlapRecipe
import ConfigParser
import json
import os
import pkg_resources
import zc.buildout
import zc.recipe.egg
import sys
import StringIO
class Recipe(BaseSlapRecipe):
def __init__(self, buildout, name, options):
self.egg = zc.recipe.egg.Egg(buildout, options['recipe'], options)
BaseSlapRecipe.__init__(self, buildout, name, options)
from slapos.recipe.librecipe import GenericBaseRecipe
def _install(self):
self.requirements, self.ws = self.egg.working_set()
class Recipe(GenericBaseRecipe):
def install(self):
path_list = []
CONFIG = dict()
CONFIG['slapos_directory'] = self.createDataDirectory('slapos')
CONFIG['working_directory'] = self.createDataDirectory('testnode')
CONFIG['test_suite_directory'] = self.createDataDirectory('test_suite')
CONFIG['proxy_host'] = self.getLocalIPv4Address()
CONFIG['proxy_port'] = '5000'
CONFIG['log_directory'] = self.createDataDirectory('testnodelog')
CONFIG['run_directory'] = self.createDataDirectory('testnoderun')
CONFIG['test_suite_title'] = self.parameter_dict.get('test_suite_title')
CONFIG['test_node_title'] = self.parameter_dict.get('test_node_title')
CONFIG['test_suite'] = self.parameter_dict.get('test_suite')
CONFIG['node_quantity'] = self.parameter_dict.get('node_quantity', '1')
CONFIG['project_title'] = self.parameter_dict.get('project_title')
CONFIG['ipv4_address'] = self.getLocalIPv4Address()
CONFIG['ipv6_address'] = self.getGlobalIPv6Address()
CONFIG['test_suite_master_url'] = self.parameter_dict.get(
'test_suite_master_url', None)
CONFIG['git_binary'] = self.options['git_binary']
CONFIG['slapgrid_partition_binary'] = self.options[
'slapgrid_partition_binary']
CONFIG['slapgrid_software_binary'] = self.options[
'slapgrid_software_binary']
CONFIG['slapproxy_binary'] = self.options['slapproxy_binary']
CONFIG['zip_binary'] = self.options['zip_binary']
options = self.options.copy()
del options['recipe']
CONFIG = {k.replace('-', '_'): v for k, v in options.iteritems()}
CONFIG['PATH'] = os.environ['PATH']
additional_bt5_repository_id = \
self.parameter_dict.get('additional_bt5_repository_id')
CONFIG['bt5_path'] = None
if additional_bt5_repository_id is not None:
CONFIG['bt5_path'] = ""
additional_bt5_repository_id_list = additional_bt5_repository_id.split(",")
for id in additional_bt5_repository_id_list:
id_path = os.path.join(CONFIG['slapos_directory'], id)
if CONFIG['bt5_path']:
additional_bt5_repository_id_list = CONFIG['bt5_path'].split(",")
CONFIG['bt5_path'] = ''
for bt5_repository_id in additional_bt5_repository_id_list:
id_path = os.path.join(CONFIG['slapos_directory'], bt5_repository_id)
bt_path = os.path.join(id_path, "bt5")
CONFIG['bt5_path'] += "%s,%s," % (id_path, bt_path)
CONFIG['instance_dict'] = ''
if 'instance_dict' in self.parameter_dict:
CONFIG['instance_dict'] = '[instance_dict]\n'
for k,v in eval(self.parameter_dict['instance_dict']).iteritems():
CONFIG['instance_dict'] += '%s = %s\n' % (k,v)
CONFIG['repository_list'] = ''
if self.options['instance-dict']:
config_instance_dict = ConfigParser.ConfigParser()
config_instance_dict.add_section('instance_dict')
instance_dict = json.loads(self.options['instance-dict'])
for k ,v in instance_dict.iteritems():
config_instance_dict.set('instance_dict', k, v)
value = StringIO.StringIO()
config_instance_dict.write(value)
CONFIG['instance_dict'] = value.getvalue()
vcs_repository_list = json.loads(self.options['repository-list'])
config_repository_list = ConfigParser.ConfigParser()
i = 0
for repository in eval(self.parameter_dict['vcs_repository_list']):
CONFIG['repository_list'] += '[vcs_repository_%s]\n' % i
CONFIG['repository_list'] += 'url = %s\n' % repository['url']
for repository in vcs_repository_list:
section_name = 'vcs_repository_%d' % i
config_repository_list.add_section(section_name)
config_repository_list.set(section_name, 'url', repository['url'])
if 'branch' in repository:
CONFIG['repository_list'] += 'branch = %s\n' % repository['branch']
config_repository_list.set(section_name, 'branch', repository['branch'])
if 'profile_path' in repository:
CONFIG['repository_list'] += 'profile_path = %s\n' % repository[
'profile_path']
config_repository_list.set(section_name, 'profile_path',
repository['profile_path'])
if 'buildout_section_id' in repository:
CONFIG['repository_list'] += 'buildout_section_id = %s\n' % repository[
'buildout_section_id']
CONFIG['repository_list'] += '\n'
config_repository_list.set(section_name, 'buildout_section_id',
repository['buildout_section_id'])
i += 1
testnode_config = self.createConfigurationFile('erp5testnode.cfg',
self.substituteTemplate(pkg_resources.resource_filename(__name__,
'template/erp5testnode.cfg.in'), CONFIG))
testnode_log = os.path.join(self.log_directory, 'erp5testnode.log')
wrapper = zc.buildout.easy_install.scripts([('erp5testnode',
'slapos.recipe.librecipe.execute', 'executee')], self.ws, sys.executable,
self.wrapper_directory, arguments=[[self.options['testnode'], '-l',
testnode_log, testnode_config], {'GIT_SSL_NO_VERIFY': '1'}])[0]
path_list.append(testnode_config)
path_list.append(wrapper)
value = StringIO.StringIO()
config_repository_list.write(value)
CONFIG['repository_list'] = value.getvalue()
configuration_file = self.createFile(
self.options['configuration-file'],
self.substituteTemplate(
self.getTemplateFilename('erp5testnode.cfg.in'),
CONFIG
),
)
path_list.append(configuration_file)
path_list.append(
self.createPythonScript(
self.options['wrapper'],
'slapos.recipe.librecipe.execute.executee',
[ # Executable
[ self.options['testnode'], '-l', self.options['log-file'],
configuration_file],
# Environment
{
'GIT_SSL_NO_VERIFY': '1',
}
],
)
)
return path_list
......@@ -39,6 +39,7 @@ import urlparse
# Use to do from slapos.recipe.librecipe import GenericBaseRecipe
from generic import GenericBaseRecipe
from genericslap import GenericSlapRecipe
from filehash import filehash
class BaseSlapRecipe:
"""Base class for all slap.recipe.*"""
......
......@@ -4,6 +4,46 @@ import signal
import subprocess
import time
import inotifyx
def _wait_files_creation(file_list):
# Etablish a list of directory and subfiles
directories = dict()
for dirname, filename in [os.path.split(f) for f in file_list]:
directories.setdefault(dirname, dict())
directories[dirname][filename] = False
def all_files_exists():
return all([all(files.values()) for files in directories.values()])
fd = inotifyx.init()
try:
# Watch every directories where the file are
watchdescriptors = dict()
for dirname in directories.keys():
wd = inotifyx.add_watch(fd,
dirname,
inotifyx.IN_CREATE | inotifyx.IN_DELETE)
watchdescriptors[wd] = dirname
# Set to True the file wich exists
for dirname, filename in [os.path.split(f) for f in file_list]:
directories[dirname][filename] = os.path.exists(os.path.join(dirname,
filename))
# Let's wait for every file creation
while not all_files_exists():
events_list = inotifyx.get_events(fd)
for event in events_list:
dirname = watchdescriptors[event.wd]
if event.name in directories[dirname]:
# One of watched file was created or deleted
if event.mask & inotifyx.IN_DELETE:
directories[dirname][event.name] = False
else:
directories[dirname][event.name] = True
finally:
os.close(fd)
def execute(args):
"""Portable execution with process replacement"""
......@@ -14,18 +54,7 @@ def execute_wait(args):
"""Execution but after all files in args[1] exists"""
exec_list = list(args[0])
file_list = list(args[1])
sleep = 60
while True:
ready = True
for f in file_list:
if not os.path.exists(f):
print 'File %r does not exists, sleeping for %s' % (f, sleep)
ready = False
if ready:
break
# XXX: It's the same as ../ca/certificate_authoritiy.py
# We should use pyinotify as well. Or select() on socket.
time.sleep(sleep)
_wait_files_creation(file_list)
os.execv(exec_list[0], exec_list + sys.argv[1:])
......@@ -49,16 +78,7 @@ def executee_wait(args):
env = os.environ.copy()
for k,v in environment.iteritems():
env[k] = v
sleep = 60
while True:
ready = True
for f in file_list:
if not os.path.exists(f):
print 'File %r does not exists, sleeping for %s' % (f, sleep)
ready = False
if ready:
break
time.sleep(sleep)
_wait_files_creation(file_list)
os.execve(exec_list[0], exec_list + sys.argv[1:], env)
def sig_handler(signal, frame):
......
##############################################################################
#
# Copyright (c) 2010 Vifib SARL and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
import hashlib
import shutil
import os
DEFAULT_HASH = 'sha512'
class Hash(object):
def __init__(self, name):
self._hash = hashlib.new(name)
def write(self, data):
self._hash.update(data)
def read(self):
return self._hash.hexdigest()
def filehash(filename, type_=DEFAULT_HASH):
if not os.path.isfile(filename):
raise ValueError("%r isn't a file" % filename)
digest = Hash(type_)
with open(filename, 'r') as file_:
shutil.copyfileobj(file_, digest)
return digest.read()
# Home made hashdeep <http://md5deep.sourceforge.net/>
def dirhash(dirname, type_=DEFAULT_HASH):
"""Walk into a directory an return a unique hash for
the directory structure and its files content."""
if not os.path.isdir(dirname):
raise ValueError("%r isn't a directory" % dirname)
digest = Hash(type_)
# List the directory structure
path_list = []
for dirname, dirlist, filelist in os.walk(dirname, followlinks=False):
for filename in filelist:
path_list.append(os.path.join(dirname, filename))
path_list.sort()
for path in path_list:
# Change the hash even if the file or the directory is empty
digest.write(path)
# Update the hash with file content
if os.path.isfile(path):
with open(path, 'r') as file_:
shutil.copyfileobj(file_, digest)
return digest.read()
def pathhash(path, type_=DEFAULT_HASH):
if os.path.isdir(path):
return dirhash(path, type_)
elif os.path.isfile(path):
return filehash(path, type_)
raise ValueError("%r isn't a directory nor a file" % path)
# you can use python -m slapos.recipe.librecipe.filehash [hash] path
if __name__ == '__main__':
import sys
if len(sys.argv) == 1:
raise ValueError("Not enough command line arguments")
if len(sys.argv) == 2:
print sys.argv[1], '-', pathhash(sys.argv[1])
else:
print sys.argv[2], '-', pathhash(sys.argv[2], sys.argv[1])
......@@ -28,6 +28,9 @@ import logging
import os
import sys
import inspect
import re
import urllib
import urlparse
import pkg_resources
import zc.buildout
......@@ -39,11 +42,12 @@ class GenericBaseRecipe(object):
def __init__(self, buildout, name, options):
"""Recipe initialisation"""
self.name = name
self.options = options
self.buildout = buildout
self.logger = logging.getLogger(name)
self.options = options.copy() # If _options use self.optionIsTrue
self._options(options) # Options Hook
self.options = options.copy() # Updated options dict
self._ws = self.getWorkingSet()
......@@ -98,9 +102,17 @@ class GenericBaseRecipe(object):
path, arguments=arguments)[0]
return script
def createDirectory(self, parent, name, mode=0700):
path = os.path.join(parent, name)
if not os.path.exists(path):
os.mkdir(path, mode)
elif not os.path.isdir(path):
raise OSError("%r exists but is not a directory." % name)
return path
def substituteTemplate(self, template_location, mapping_dict):
"""Read from file template_location an substitute content with
mapping_dict douing a dummy python format."""
mapping_dict doing a dummy python format."""
with open(template_location, 'r') as template:
return template.read() % mapping_dict
......@@ -123,3 +135,35 @@ class GenericBaseRecipe(object):
if default is not None and optionname not in self.options:
return default
return self.isTrueValue(self.options[optionname])
def unparseUrl(self, scheme, host, path='', params='', query='',
fragment='', port=None, auth=None):
"""Join a url with auth, host, and port.
* auth can be either a login string or a tuple (login, password).
* if the host is an ipv6 address, brackets will be added to surround it.
"""
# XXX-Antoine: I didn't find any standard module to join an url with
# login, password, ipv6 host and port.
# So instead of copy and past in every recipe I factorized it right here.
netloc = ''
if auth is not None:
auth = tuple(auth)
netloc = urllib.quote(str(auth[0])) # Login
if len(auth) > 1:
netloc += ':%s' % urllib.quote(auth[1]) # Password
netloc += '@'
# host is an ipv6 address whithout brackets
if ':' in host and not re.match(r'^\[.*\]$', host):
netloc += '[%s]' % host
else:
netloc += str(host)
if port is not None:
netloc += ':%s' % port
url = urlparse.urlunparse((scheme, netloc, path, params, query, fragment))
return url
......@@ -26,8 +26,6 @@
##############################################################################
from slapos import slap
import time
import re
import urlparse
from generic import GenericBaseRecipe
......@@ -36,7 +34,6 @@ class GenericSlapRecipe(GenericBaseRecipe):
def __init__(self, buildout, name, options):
"""Default initialisation"""
options['eggs'] = 'slapos.cookbook'
GenericBaseRecipe.__init__(self, buildout, name, options)
self.slap = slap.slap()
......@@ -49,15 +46,6 @@ class GenericSlapRecipe(GenericBaseRecipe):
self.key_file = slap_connection.get('key-file')
self.cert_file = slap_connection.get('cert-file')
# setup auto uninstall/install
self._setupAutoInstallUninstall()
def _setupAutoInstallUninstall(self):
"""By default SlapOS recipes are reinstalled each time"""
# Note: It is possible to create in future subclass which will do no-op in
# this method
self.options['slapos-timestamp'] = str(time.time())
def install(self):
self.slap.initializeConnection(self.server_url, self.key_file,
self.cert_file)
......@@ -80,37 +68,5 @@ class GenericSlapRecipe(GenericBaseRecipe):
raise NotImplementedError('Shall be implemented by subclass')
def setConnectionUrl(self, *args, **kwargs):
url = self._unparseUrl(*args, **kwargs)
url = self.unparseUrl(*args, **kwargs)
self.setConnectionDict(dict(url=url))
def _unparseUrl(self, scheme, host, path='', params='', query='',
fragment='', port=None, auth=None):
"""Join a url with auth, host, and port.
* auth can be either a login string or a tuple (login, password).
* if the host is an ipv6 address, brackets will be added to surround it.
"""
# XXX-Antoine: I didn't find any standard module to join an url with
# login, password, ipv6 host and port.
# So instead of copy and past in every recipe I factorized it right here.
netloc = ''
if auth is not None:
auth = tuple(auth)
netloc = str(auth[0]) # Login
if len(auth) > 1:
netloc += ':%s' % auth[1] # Password
netloc += '@'
# host is an ipv6 address whithout brackets
if ':' in host and not re.match(r'^\[.*\]$', host):
netloc += '[%s]' % host
else:
netloc += str(host)
if port is not None:
netloc += ':%s' % port
url = urlparse.urlunparse((scheme, netloc, path, params, query, fragment))
return url
##############################################################################
#
# Copyright (c) 2010 Vifib SARL and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
import os
import inotifyx
def subfiles(directory):
"""Return the list of subfiles of a directory, and wait for the newly created
ones.
CAUTION : *DONT TRY TO CONVERT THE RESULT OF THIS FUNCTION INTO A LIST !
ALWAYS ITERATE OVER IT !!!*"""
watchfd = inotifyx.init()
inotifyx.add_watch(watchfd, directory, inotifyx.IN_CREATE)
try:
subfiles = set(os.listdir(directory))
subfiles |= set([file_.name for file_ in inotifyx.get_events(watchfd, 0)])
while True:
for file_ in subfiles:
yield os.path.join(directory, file_)
subfiles = [file_.name for file_ in inotifyx.get_events(watchfd)]
finally:
os.close(watchfd)
##############################################################################
#
# Copyright (c) 2010 Vifib SARL and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
import os
import sys
import subprocess
import inotifyx
from slapos.recipe.librecipe import GenericBaseRecipe
class LockFile(object):
class LockException(Exception):
pass
def __init__(self, filename, wait=True, exit=False):
self.filename = filename
if wait:
self.callback = lambda: self.waitDeletion()
elif not exit:
self.callback = lambda: self.raiseException()
else:
self.callback = lambda: sys.exit(1)
def raiseException(self):
raise LockFile.LockException("Not able to lock the file")
def waitDeletion(self):
inotify_fd = inotifyx.init()
try:
inotifyx.add_watch(inotify_fd, self.filename, inotifyx.IN_DELETE)
inotifyx.get_events(inotify_fd)
except IOError: # add_watch failed
pass
finally:
os.close(inotify_fd)
self.__enter__()
def __enter__(self):
try:
# Atomic file acquisition
self._fd = os.open(self.filename, os.O_CREAT | os.O_EXCL)
except OSError:
self.callback()
def __exit__(self, exc_type, exc_value, traceback):
os.close(self._fd)
os.unlink(self.filename)
def locked_run(args):
with LockFile(args['filename'], wait=args['wait'], exit=True):
subprocess.check_call([args['binary']])
class Recipe(GenericBaseRecipe):
def install(self):
wrapper = self.createPythonScript(self.options['wrapper'],
__name__ + '.locked_run',
dict(
filename=self.options['lock-file'],
wait=self.optionIsTrue('wait', False),
binary=self.options['binary'],
)
)
return [wrapper]
##############################################################################
#
# Copyright (c) 2010 Vifib SARL and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
import subprocess
from slapos.recipe.librecipe import GenericBaseRecipe
def dump(args):
mydumper_cmd = [args['mydumper']]
mydumper_cmd.extend(['-B', args['database']])
if args['socket'] is not None:
mydumper_cmd.extend(['-S', args['socket']])
else:
mydumper_cmd.extend(['-h', args['host']])
mydumper_cmd.etxned(['-P', args['port']])
mydumper_cmd.extend(['-u', args['user']])
if args['password'] is not None:
mydumper_cmd.extend(['-p', args['password']])
if args['compression']:
mydumper_cmd.append('--compress')
if args['rows'] is not None:
mydumper_cmd.extend(['-r', args['rows']])
mydumper_cmd.extend(['-o', args['directory']])
subprocess.check_call(mydumper_cmd)
def do_import(args):
mydumper_cmd = [args['mydumper']]
mydumper_cmd.extend(['-B', args['database']])
if args['socket'] is not None:
mydumper_cmd.extend(['-S', args['socket']])
else:
mydumper_cmd.extend(['-h', args['host']])
mydumper_cmd.etxned(['-P', args['port']])
mydumper_cmd.extend(['-u', args['user']])
if args['password'] is not None:
mydumper_cmd.extend(['-p', args['password']])
mydumper_cmd.append('--overwrite-tables')
mydumper_cmd.extend(['-d', args['directory']])
subprocess.check_call(mydumper_cmd)
class Recipe(GenericBaseRecipe):
def install(self):
# Host or socket should be defined
try:
self.options['host']
except:
self.options['socket']
config = dict(database=self.options['database'],
socket=self.options.get('socket'),
host=self.options.get('host'),
port=self.options.get('port', 3306),
directory=self.options['backup-directory'],
user=self.options['user'],
password=self.options.get('password'),
)
name = __name__
if self.optionIsTrue('import', False):
config.update(mydumper=self.options['myloader-binary'])
name += '.do_import'
else:
config.update(mydumper=self.options['mydumper-binary'],
compression=self.optionIsTrue('compression', default=False),
rows=self.options.get('rows'),
)
name += '.dump'
wrapper = self.createPythonScript(self.options['wrapper'],
name,
config)
return [wrapper]
......@@ -24,8 +24,12 @@
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
from slapos.recipe.librecipe import GenericBaseRecipe
import os
import sys
import subprocess
from slapos.recipe.librecipe import GenericBaseRecipe
from slapos.recipe.librecipe import filehash
class Recipe(GenericBaseRecipe):
......@@ -44,7 +48,6 @@ class Recipe(GenericBaseRecipe):
pid_file=self.options['pid-file'],
socket=self.options['socket'],
error_log=self.options['error-log'],
slow_query_log=self.options['slow-query-log'],
mysql_database=self.options['database'],
mysql_user=self.options['user'],
mysql_password=self.options['password'],
......@@ -109,36 +112,24 @@ class Recipe(GenericBaseRecipe):
path_list.append(mysqld)
# backup configuration
mysqldump_binary = self.options['mysqldump-binary']
backup_directory = self.options['backup-directory']
pending_backup_dir = self.options['backup-pending-directory']
dump_filename = self.options['dumpname']
mysqldump_cmd = [mysqldump_binary,
mysql_conf['mysql_database'],
'-u', 'root',
'-S', mysql_conf['socket'].strip(),
'--single-transaction', '--opt',
]
dump_file = os.path.join(backup_directory, dump_filename)
tmpdump_file = os.path.join(pending_backup_dir, dump_filename)
backup_script = self.createPythonScript(
self.options['backup-script'],
'%s.backup.do_backup' % __name__,
{
'mysqldump': mysqldump_cmd,
'gzip': self.options['gzip-binary'],
'tmpdump': tmpdump_file,
'dumpfile': dump_file,
},
)
path_list.append(backup_script)
if self.optionIsTrue('backup', default=False):
backup_script = self.createPythonScript(
self.options['backup-script'],
'%s.do_backup' % __name__,
dict(
mydumper_binary=self.options['mydumper-binary'],
database=mysql_conf['mysql_database'],
socket=mysql_conf['socket'],
backup_directory=self.options['backup-directory']
),
)
path_list.append(backup_script)
# Recovering backup
if self.optionIsTrue('recovering', default=False):
recovering_script = self.createPythonScript(
self.options['recovering-wrapper'],
'%s.recover.import_remote_dump' % __name__,
'%s.import_dump' % __name__,
{
'lock_file': os.path.join(self.work_directory,
'import_done'),
......@@ -156,3 +147,47 @@ class Recipe(GenericBaseRecipe):
return path_list
# Replace zcat dump.sql.gz | mysql
def import_dump(args):
# Get data from kwargs
cache_file = args['cache_file']
database = args['database']
mysql_binary = args['mysql_binary']
mysql_socket = args['mysql_socket']
dump_file = args['dump_file']
zcat_binary = args['zcat_binary']
sha512sum = filehash(dump_file)
with open(cache_file, 'r') as cache_fileobj:
last_sha512sum = cache_fileobj.read().strip()
if sha512sum != last_sha512sum:
zcat = subprocess.Popen([zcat_binary, dump_file], stdout=subprocess.PIPE)
mysql = subprocess.Popen([mysql_binary, '--socket=%s' % mysql_socket, '-D',
database, '-u', 'root'], stdin=zcat.stdout)
zcat.stdout.close()
returncode = mysql.wait()
if returncode == 0:
with open(cache_file, 'w') as cache_fileobj:
cache_fileobj.write(sha512sum)
sys.exit(returncode)
def promise(args):
# This is not a dependency of slapos.cookbook, because it shall be runned
# in a python environment having mysql-python
import MySQLdb
user = args['user']
password = args['password']
db = args['db']
host = args['host']
port = args['port']
db = MySQLdb.connect(host=host, port=port, user=user, passwd=password,
db=db)
cursor = db.cursor()
cursor.close()
import subprocess
import os
# Replace mysqldump | gzip > tmpdump && mv -f tmpdump dumpfile
def do_backup(kwargs):
mysqldump_cmd = kwargs['mysqldump']
gzip_bin = kwargs['gzip']
tmpdump = kwargs['tmpdump']
dumpfile = kwargs['dumpfile']
# mysqldump | gzip > tmpdump
with open(tmpdump, 'w') as output:
mysqldump = subprocess.Popen(mysqldump_cmd,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
gzip = subprocess.Popen([gzip_bin],
stdin=mysqldump.stdout,
stdout=output,
stderr=subprocess.STDOUT)
mysqldump.stdout.close()
if gzip.wait() != 0:
raise ValueError("Gzip return a non zero value.")
os.rename(tmpdump, dumpfile)
import os
import sys
import time
def catdatefile(args):
directory = args[0]
try:
suffix = args[1]
except IndexError:
suffix = '.log'
f = open(os.path.join(directory,
time.strftime('%Y-%m-%d.%H:%M.%s') + suffix), 'aw')
for line in sys.stdin.read():
f.write(line)
f.close()
import sys
import os
import time
import subprocess
def import_remote_dump(kwargs):
# Get data from kwargs
lock_file = kwargs['lock_file']
database = kwargs['database']
mysql_binary = kwargs['mysql_binary']
mysql_socket = kwargs['mysql_socket']
duplicity_binary = kwargs['duplicity_binary']
remote_backup = kwargs['remote_backup']
local_directory = kwargs['local_directory']
dump_name = kwargs['dump_name']
zcat_binary = kwargs['zcat_binary']
# The script start really here
if os.path.exists(lock_file):
sys.exit(127)
while subprocess.call([mysql_binary, '--socket=%s' % mysql_socket,
'-u', 'root', '-e', 'use %s;' % database]) != 0:
time.sleep(10)
subprocess.check_call([duplicity_binary, 'restore', '--no-encryption',
remote_backup, local_directory])
zcat = subprocess.Popen([zcat_binary, os.path.join(local_directory,
dump_name)],
stdout=subprocess.PIPE)
mysql = subprocess.Popen([mysql_binary, '--socket=%s' % mysql_socket,
'-D', database, '-u', 'root'],
stdin=zcat.stdout)
zcat.stdout.close()
returncode = mysql.poll()
if returncode == 0:
open(lock_file, 'w').close() # Just a touch
sys.exit(returncode)
......@@ -15,8 +15,6 @@ socket = %(socket)s
datadir = %(data_directory)s
pid-file = %(pid_file)s
log-error = %(error_log)s
slow_query_log
slow_query_log_file = %(slow_query_log)s
long_query_time = 5
max_allowed_packet = 128M
query_cache_size = 32M
......
##############################################################################
#
# Copyright (c) 2010 Vifib SARL and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
import os
from hashlib import sha512
from slapos.recipe.librecipe import GenericBaseRecipe
class Recipe(GenericBaseRecipe):
def install(self):
commandline = [self.options['server-binary']]
commandline.extend(['--callbacks', self.options['callbacks']])
commandline.extend(['--feeds', self.options['feeds']])
commandline.extend(['--equeue-socket', self.options['equeue-socket']])
commandline.append(self.options['host'])
commandline.append(self.options['port'])
return [self.createPythonScript(self.options['wrapper'],
'slapos.recipe.librecipe.execute.execute',
commandline)]
class Callback(GenericBaseRecipe):
def createCallback(self, notification_id, callback):
callback_id = sha512(notification_id).hexdigest()
callback = self.createFile(os.path.join(self.options['callbacks'],
callback_id),
callback)
return callback
def install(self):
return [self.createCallback(self.options['on-notification-id'],
self.options['callback'])]
class Notify(GenericBaseRecipe):
def createNotifier(self, notifier_binary, executable, wrapper, **kwargs):
if not os.path.exists(kwargs['log']):
# Just a touch
open(kwargs['log'], 'w').close()
commandline = [notifier_binary,
'-l', kwargs['log'],
'--title', kwargs['title'],
'--feed', kwargs['feed_url'],
'--notification-url', kwargs['notification_url'],
executable]
return self.createPythonScript(wrapper,
'slapos.recipe.librecipe.execute.execute',
[str(i) for i in commandline])
def install(self):
feedurl = self.unparseUrl(scheme='http', host=self.options['host'],
port=self.options['port'],
path='/get/%s' % self.options['name'])
script = self.createNotifier(
self.options['notifier-binary'],
wrapper=self.options['wrapper'],
executable=self.options['executable'],
log=os.path.join(self.options['feeds'], self.options['name']),
title=self.options['title'],
notification_url=self.options['notify'],
feed_url=feedurl,
)
return [script]
##############################################################################
#
# Copyright (c) 2010 Vifib SARL and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
from json import loads as unjson
from hashlib import sha512
from urlparse import urlparse
import os
import subprocess
import sys
import signal
import inspect
from slapos.recipe.librecipe import GenericSlapRecipe
from slapos.recipe.dropbear import KnownHostsFile
from slapos.recipe.notifier import Notify
from slapos.recipe.notifier import Callback
from slapos import slap as slapmodule
def promise(args):
def failed_ssh(partition, ssh):
# Bad python 2 syntax, looking forward python 3 to have print(file=)
print >> sys.stderr, "SSH Connection failed"
try:
ssh.terminate()
except:
pass
partition.bang("SSH Connection failed. rdiff-backup is unusable.")
def sigterm_handler(signum, frame):
# Walk up in the stack to get promise local
# variables
ssh = None
for upper_frame in inspect.getouterframes(frame):
# Use promise.func_name insteand of 'promise' in order to be
# detected by editor if promise func name change.
# Else, it's hard to debug this kind of error.
if upper_frame[3] == promise.func_name:
try:
partition = upper_frame[0].f_locals['partition']
ssh = upper_frame[0].f_locals['ssh']
except KeyError:
raise SystemExit("SIGTERM Send too soon.")
break
# If ever promise function wasn't found in the stack.
if ssh is None:
raise SystemExit
failed_ssh(partition, ssh)
signal.signal(signal.SIGTERM, sigterm_handler)
slap = slapmodule.slap()
slap.initializeConnection(args['server_url'],
key_file=args.get('key_file'), cert_file=args.get('cert_file'))
partition = slap.registerComputerPartition(args['computer_id'],
args['partition_id'])
# Rdiff Backup protocol quit command
quitcommand = 'q' + chr(255) + chr(0) * 7
ssh_cmdline = [args['ssh_client'], '%(user)s@%(host)s/%(port)s' % args]
ssh = subprocess.Popen(ssh_cmdline, stdin=subprocess.PIPE,
stdout=open(os.devnull), stderr=open(os.devnull))
ssh.stdin.write(quitcommand)
ssh.stdin.flush()
ssh.stdin.close()
ssh.wait()
if ssh.poll() is None:
return 1
if ssh.returncode != 0:
failed_ssh(partition, ssh)
return ssh.returncode
class Recipe(GenericSlapRecipe, Notify, Callback):
def add_slave(self, entry, known_hosts_file):
path_list = []
url = entry.get('url')
if url is None:
url = ''
# We assume that thanks to sha512 there's no collisions
url_hash = sha512(url).hexdigest()
name_hash = sha512(entry['name']).hexdigest()
promise_path = os.path.join(self.options['promises-directory'],
url_hash)
parsed_url = urlparse(url)
promise_dict = self.promise_base_dict.copy()
promise_dict.update(user=parsed_url.username,
host=parsed_url.hostname,
port=parsed_url.port)
promise = self.createPythonScript(promise_path,
__name__ + '.promise',
promise_dict)
path_list.append(promise)
host = parsed_url.hostname
known_hosts_file[host] = entry['server-key']
remote_schema = '%(ssh)s -p %%s %(user)s@%(host)s' % \
{
'ssh': self.options['sshclient-binary'],
'user': parsed_url.username,
'host': parsed_url.hostname,
}
command = [self.options['rdiffbackup-binary']]
command.extend(['--remote-schema', remote_schema])
remote_directory = '%(port)s::%(path)s' % {'port': parsed_url.port,
'path': parsed_url.path}
local_directory = self.createDirectory(self.options['directory'],
name_hash)
if entry['type'] == 'push':
command.extend(['--restore-as-of', 'now'])
command.append('--force')
command.extend([local_directory, remote_directory])
else:
command.extend([remote_directory, local_directory])
wrapper_basepath = os.path.join(self.options['wrappers-directory'],
url_hash)
wrapper_path = wrapper_basepath
if 'notify' in entry:
wrapper_path = '%s_raw' % wrapper_basepath
wrapper = self.createPythonScript(
wrapper_path,
'slapos.recipe.librecipe.execute.execute',
[str(i) for i in command]
)
path_list.append(wrapper)
if 'notify' in entry:
feed_url = '%s/get/%s' % (self.options['notifier-url'],
entry['notification-id'])
wrapper = self.createNotifier(
self.options['notifier-binary'],
wrapper=wrapper_basepath,
executable=wrapper_path,
log=os.path.join(self.options['feeds'], entry['notification-id']),
title=entry.get('title', 'Untitled'),
notification_url=entry['notify'],
feed_url=feed_url,
)
path_list.append(wrapper)
#self.setConnectionDict(dict(feed_url=feed_url), entry['slave_reference'])
if 'on-notification' in entry:
path_list.append(self.createCallback(str(entry['on-notification']),
wrapper))
else:
cron_entry = os.path.join(self.options['cron-entries'], url_hash)
with open(cron_entry, 'w') as cron_entry_file:
cron_entry_file.write('%s %s' % (entry['frequency'], wrapper))
path_list.append(cron_entry)
return path_list
def _install(self):
path_list = []
if self.optionIsTrue('client', True):
self.logger.info("Client mode")
slap_connection = self.buildout['slap-connection']
self.promise_base_dict = dict(
server_url=slap_connection['server-url'],
computer_id=slap_connection['computer-id'],
cert_file=slap_connection.get('cert-file'),
key_file=slap_connection.get('key-file'),
partition_id=slap_connection['partition-id'],
ssh_client=self.options['sshclient-binary'],
)
slaves = unjson(self.options['slave-instance-list'])
known_hosts = KnownHostsFile(self.options['known-hosts'])
with known_hosts:
for slave in slaves:
path_list.extend(self.add_slave(slave, known_hosts))
else:
command = [self.options['rdiffbackup-binary']]
self.logger.info("Server mode")
command.extend(['--restrict', self.options['path']])
command.append('--server')
wrapper = self.createPythonScript(
self.options['wrapper'],
'slapos.recipe.librecipe.execute.execute',
command)
path_list.append(wrapper)
return path_list
......@@ -31,8 +31,10 @@ from slapos.recipe.librecipe import GenericSlapRecipe
class Recipe(GenericSlapRecipe):
def _install(self):
publish_dict = dict()
for k, v in self.options.iteritems():
if k.startswith('url'):
publish_dict[k] = v
options = self.options.copy()
del options['recipe']
for k, v in options.iteritems():
publish_dict[k] = v
self.setConnectionDict(publish_dict)
return []
......@@ -56,10 +56,10 @@ class Recipe(GenericSlapRecipe):
if option in self.options:
self.urlparts[option] = self.options[option]
if 'user' in self.options:
self.urlparts.update(auth=(self.options['user'],))
if 'username' in self.options:
self.urlparts.update(auth=(self.options['username'],))
if 'password' in self.options:
self.urlparts.update(auth=(self.options['user'],
self.urlparts.update(auth=(self.options['username'],
self.options['password']))
self.setConnectionUrl(**self.urlparts)
......
......@@ -267,7 +267,7 @@ class Recipe(BaseSlapRecipe):
memcached_ip=config['memcached_ip'],
memcached_port=config['memcached_port'])
def createHtdocs(self, source, document_root):
def createHtdocs(self, source, document_root):
source = self.options['source'].strip()
document_root = self.createDataDirectory('htdocs')
for p in os.listdir(document_root):
......
##############################################################################
#
# Copyright (c) 2010 Vifib SARL and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
import subprocess
import os
from slapos.recipe.librecipe import GenericBaseRecipe
class Recipe(GenericBaseRecipe):
def _options(self, options):
if not os.path.exists(self.options['file']):
password = subprocess.check_output([self.options['pwgen-binary'], '-1']).strip()
with open(self.options['file'], 'w') as password_file:
password_file.write(password)
else:
with open(self.options['file'], 'r') as password_file:
password = password_file.read()
options['password'] = password
def install(self):
return []
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
#!/bin/sh
# BEWARE: This file is operated by slapgrid
# BEWARE: It will be overwritten automatically
%(xvfb_binary) :%(display) -screen %(display) 1024x768x24
\ No newline at end of file
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
......@@ -6,5 +6,5 @@ key = %(key)s
cert = %(cert)s
[service]
accept = %(remote_address)s:%(remote_port)s
connect = %(local_address)s:%(local_port)s
accept = %(remote_host)s:%(remote_port)s
connect = %(local_host)s:%(local_port)s
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment