Update Release Candidate

component/6tunnel/buildout.cfg

@@ -10,12 +10,10 @@ extends =
 [6tunnel]
 recipe = slapos.recipe.cmmi
 shared = true
-url = http://toxygen.net/6tunnel/6tunnel-0.11rc2.tar.gz
+url = https://github.com/wojtekka/6tunnel/releases/download/0.11rc2/6tunnel-0.11rc2.tar.gz
 md5sum = 74e02d4f0704b3083a01feda66033449
 pre-configure =
   aclocal
-  autoheader
-  automake --foreign
   autoconf
 environment =
   PATH=${autoconf:location}/bin:${automake:location}/bin:%(PATH)s

component/attr/buildout.cfg

@@ -4,6 +4,7 @@ parts = attr
 
 [attr]
 recipe = slapos.recipe.cmmi
+shared = true
 url = http://download.savannah.gnu.org/releases/attr/attr-2.4.47.src.tar.gz
 md5sum = 84f58dec00b60f2dc8fd1c9709291cc7
 configure-options =

component/boost-lib/buildout.cfg

@@ -5,6 +5,7 @@ extends =
   ../bzip2/buildout.cfg
   ../gcc/buildout.cfg
   ../zlib/buildout.cfg
+  ../xz-utils/buildout.cfg
 
 [boost-lib]
 recipe = slapos.recipe.cmmi
@@ -12,14 +13,16 @@ shared = true
 url = http://downloads.sourceforge.net/sourceforge/boost/boost_1_67_0.tar.bz2
 md5sum = ced776cb19428ab8488774e1415535ab
 location = @@LOCATION@@
-configure-command = ./bootstrap.sh --prefix=${:location} --with-python=${python2.7:location}/bin/python2.7
+configure-command = ./bootstrap.sh --prefix=${:location} --with-python=${python2.7:location}/bin/python2.7 --without-icu
 make-binary =
 make-options =
 make-targets = for a in $MAKEFLAGS; do case $a in -j*) j=$a; break;; esac; done;
-  ./b2 $j link=shared dll-path=${:location}/lib:${bzip2:location}/lib:${gcc:location}/lib:${gcc:location}/lib64:${zlib:location}/lib install
+  ./b2 $j link=shared dll-path=${:location}/lib:${bzip2:location}/lib:${gcc:location}/lib:${gcc:location}/lib64:${zlib:location}/lib:${xz-utils:location}/lib install
 environment =
   PATH=${gcc:location}/bin:%(PATH)s
   BZIP2_INCLUDE=${bzip2:location}/include
   BZIP2_LIBPATH=${bzip2:location}/lib
   ZLIB_INCLUDE=${zlib:location}/include
   ZLIB_LIBPATH=${zlib:location}/lib
+  LZMA_INCLUDE=${xz-utils:location}/include
+  LZMA_LIBRARY_PATH=${xz-utils:location}/lib
\ No newline at end of file

component/coturn/buildout.cfg

@@ -13,4 +13,4 @@ environment =
   PATH=${sqlite3:location}/bin:%(PATH)s
   CPPFLAGS=-I${openssl:location}/include -I${sqlite3:location}/include -I${libevent2:location}/include
   CFLAGS=-I${libevent2:location}/include
-  LDFLAGS=-L${openssl:location}/lib -Wl,-rpath -Wl,${openssl:location}/lib -L${sqlite3:location}/lib -L${libevent2:location}/lib -Wl,-rpath -Wl,${libevent2:location}/lib
+  LDFLAGS=-L${openssl:location}/lib -Wl,-rpath -Wl,${openssl:location}/lib -L${sqlite3:location}/lib -Wl,-rpath -Wl,${sqlite3:location}/lib -L${libevent2:location}/lib -Wl,-rpath -Wl,${libevent2:location}/lib

component/cython/buildout.cfg

@@ -4,3 +4,6 @@ parts = cython
 [cython]
 recipe = zc.recipe.egg:custom
 egg = cython
+
+[versions]
+Cython = 0.28.2

component/gcc/buildout.cfg

@@ -7,8 +7,8 @@ extends =
   ../patch/buildout.cfg
   ../perl/buildout.cfg
   ../tar/buildout.cfg
-  ../xz-utils/buildout.cfg
   ../binutils/buildout.cfg
+  ../zlib/buildout.cfg
 
 parts =
   gcc
@@ -16,16 +16,7 @@ parts =
 [gcc-common]
 recipe = slapos.recipe.cmmi
 shared = true
-url = http://ftp.gnu.org/gnu/gcc/gcc-5.5.0/gcc-5.5.0.tar.gz
-md5sum = 781bc0195edeb0ceaace8428f63ae63d
-# make install does not work when several core are used
-make-targets = install -j1
-patch-options = -p1
-patches =
-  ${:_profile_base_location_}/libsanitizer_Use_pre-computed_size_of_struct_ustat_for_Linux.patch#1e5f33e89f9fe1ca3e406eabcc621762
-
-[gcc]
-<= gcc-common
+url = http://ftp.gnu.org/gnu/gcc/gcc-${:version}/gcc-${:version}.tar.xz
 configure-options =
   --disable-bootstrap
   --disable-multilib
@@ -36,12 +27,32 @@ configure-options =
   --with-isl=${isl:location}
   --with-ld=${binutils:location}/bin/ld
   --with-as=${binutils:location}/bin/as
+post-install =
+  cd '@@LOCATION@@/bin'
+  ln -s gcc cc
 environment =
-  LDFLAGS=-Wl,-rpath=${gmp:location}/lib -Wl,-rpath=${isl:location}/lib -Wl,-rpath=${mpc:location}/lib -Wl,-rpath=${mpfr:location}/lib
+  CPPFLAGS=-I${zlib:location}/include
+  LDFLAGS=-Wl,-rpath=${gmp:location}/lib -Wl,-rpath=${isl:location}/lib -Wl,-rpath=${mpc:location}/lib -Wl,-rpath=${mpfr:location}/lib -L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib
   PATH=${binutils:location}/bin:${gettext:location}/bin:${perl:location}/bin:${tar:location}/bin:%(PATH)s
 
-[gcc-minimal]
+[gcc]
 <= gcc-common
+version = 5.5.0
+md5sum = 0f70424213b4a1113c04ba66ddda0c1f
+# make install does not work when several core are used
+make-targets = install -j1
+patch-binary = ${patch:location}/bin/patch
+patch-options = -p1
+patches =
+  ${:_profile_base_location_}/libsanitizer_Use_pre-computed_size_of_struct_ustat_for_Linux.patch#1e5f33e89f9fe1ca3e406eabcc621762
+
+[gcc-8.2]
+<= gcc-common
+version = 8.2.0
+md5sum = 4ab282f414676496483b3e1793d07862
+
+[gcc-minimal]
+<= gcc
 configure-options =
   --disable-bootstrap
   --disable-multilib
@@ -54,23 +65,3 @@ configure-options =
 environment =
   LDFLAGS=-Wl,-rpath=${gmp:location}/lib -Wl,-rpath=${mpc:location}/lib -Wl,-rpath=${mpfr:location}/lib
   PATH=${perl:location}/bin:${tar:location}/bin:%(PATH)s
-
-
-[gcc-8.2]
-recipe = slapos.recipe.cmmi
-shared = true
-url = https://ftp.gnu.org/gnu/gcc/gcc-8.2.0/gcc-8.2.0.tar.gz
-md5sum = ee04f0c22a941f5f17d93809387f2729
-configure-options =
-  --disable-bootstrap
-  --disable-multilib
-  --with-gmp=${gmp:location}
-  --with-mpfr=${mpfr:location}
-  --with-mpc=${mpc:location}
-  --enable-languages="c,c++,fortran"
-  --with-isl=${isl:location}
-  --with-ld=${binutils:location}/bin/ld
-  --with-as=${binutils:location}/bin/as
-environment =
-  LDFLAGS=-Wl,-rpath=${gmp:location}/lib -Wl,-rpath=${isl:location}/lib -Wl,-rpath=${mpc:location}/lib -Wl,-rpath=${mpfr:location}/lib
-  PATH=${binutils:location}/bin:${gettext:location}/bin:${perl:location}/bin:${tar:location}/bin:%(PATH)s

component/inkscape/buildout.cfg

@@ -46,8 +46,6 @@ pkg_config_depends = ${freetype:location}/lib/pkgconfig:${gtkmm:location}/lib/pk
 configure-command = ${cmake:location}/bin/cmake
 configure-options =
   -DCMAKE_INSTALL_PREFIX=${:location}
-  -DCMAKE_C_COMPILER=${gcc:location}/bin/gcc
-  -DCMAKE_CXX_COMPILER=${gcc:location}/bin/g++
   -DENABLE_POPPLER=OFF
   -DWITH_GNOME_VFS=OFF
   -DWITH_IMAGE_MAGICK=OFF

component/keras/buildout.cfg

@@ -28,4 +28,3 @@ scripts = keras-python
 Keras = 2.1.0
 tensorflow = 1.4.0
 h5py = 2.7.0rc2
-Cython = 0.25.2

component/libcap/buildout.cfg

@@ -6,32 +6,23 @@ extends =
 parts = libcap
 
 [libcap2]
-recipe = slapos.recipe.build
+recipe = slapos.recipe.cmmi
+shared = true
 url = http://pkgs.fedoraproject.org/lookaside/pkgs/libcap/libcap-2.22.tar.bz2/ce64058bdb3f086ddbfca8ce6c919845/libcap-2.22.tar.bz2
 md5sum = ce64058bdb3f086ddbfca8ce6c919845
-attr-include = ${attr:location}/include/
-attr-lib = ${attr:location}/lib/
-slapos_promise =
-  directory:sbin
-  directory:include
-  statlib:lib/libcap.a
-  file:lib/libcap.so
-  file:sbin/getcap
-  file:sbin/setcap
-install =
-  import os
-  url = self.download(options['url'], options['md5sum'])
-  extract_dir = self.extract(url)
-  workdir = guessworkdir(extract_dir)
-  cflags = '-I%(attr-include)s' % options
-  ldflags = '-L%(attr-lib)s -Wl,-rpath=%(attr-lib)s' % options
-  call(['make', 'CFLAGS=' + cflags, 'LDFLAGS=' + ldflags, 'DESTDIR=' + location, 'RAISE_SETFCAP=no', 'prefix=', 'install'],
-       cwd=workdir, env=self.environ)
-  lib64 = os.path.join(location, 'lib64')
-  lib = os.path.join(location, 'lib')
-  # XXX: Dirty if case
-  #      if lib64 exists, then create a symlink from lib to lib64
-  os.path.exists(lib64) and os.symlink(lib64, lib)
+location = @@LOCATION@@
+configure-command = :
+make-options =
+  RAISE_SETFCAP=no
+  prefix=${:location}
+  CFLAGS=-I${attr:location}/include/
+  LDFLAGS="-L${attr:location}/lib/ -Wl,-rpath=${attr:location}/lib/ -Wl,-rpath=${:location}/lib/"
+  install
+
+# if lib64 exists, then create a symlink from lib to lib64
+post-install =
+  cd ${:location}
+  [ -d lib64 ] && ln -s lib64 lib
 
 [libcap]
-<= libcap2
+<= libcap2
\ No newline at end of file

component/libevent/buildout.cfg

@@ -7,6 +7,7 @@ extends =
 recipe = slapos.recipe.cmmi
 url = http://www.monkey.org/~provos/libevent-1.4.13-stable.tar.gz
 md5sum = 0b3ea18c634072d12b3c1ee734263664
+shared = true
 configure-options =
   --disable-static
 

component/libzip/buildout.cfg

@@ -26,4 +26,4 @@ environment =
   PATH=${cmake:location}/bin:%(PATH)s
   CMAKE_INCLUDE_PATH=${zlib:location}/include:${bzip2:location}/include
   CMAKE_LIBRARY_PATH=${zlib:location}/lib:${bzip2:location}/lib
-  LDFLAGS=-L${:location}/lib -Wl,-rpath=${:location}/lib -L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib
+  LDFLAGS=-L${:location}/lib -Wl,-rpath=${:location}/lib -L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib -L${bzip2:location}/lib -Wl,-rpath=${bzip2:location}/lib

component/lz4/buildout.cfg

@@ -5,6 +5,7 @@ parts =
 
 [lz4]
 recipe = slapos.recipe.cmmi
+shared = true
 url = https://github.com/lz4/lz4/archive/v1.9.2.tar.gz
 md5sum = 3898c56c82fb3d9455aefd48db48eaad
 configure-command = true

component/mariadb/buildout.cfg

@@ -34,7 +34,7 @@ version = 10.4.12
 md5sum = 97d7c0f508c04a31c138fdb24e95dbc4
 location = ${buildout:parts-directory}/${:_buildout_section_name_}
 pre-configure =
-  set -e '\bSET(PLUGIN_AUTH_PAM YES)' cmake/build_configurations/mysql_release.cmake
+  set '\bSET(PLUGIN_AUTH_PAM YES)' cmake/build_configurations/mysql_release.cmake
   grep -q "$@"
   sed -i "/$1/d" "$2"
 configure-command = ${cmake:location}/bin/cmake
@@ -68,8 +68,6 @@ configure-options =
   -DCMAKE_INSTALL_RPATH=${:CMAKE_LIBRARY_PATH}
   -DCMAKE_INCLUDE_PATH=${unixodbc:location}/include
   -DCMAKE_LIBRARY_PATH=${unixodbc:location}/lib
-  -DCMAKE_C_COMPILER=${gcc:location}/bin/gcc
-  -DCMAKE_CXX_COMPILER=${gcc:location}/bin/g++
 CMAKE_CFLAGS = -I${bzip2:location}/include -I${jemalloc:location}/include -I${libaio:location}/include -I${libxml2:location}/include -I${ncurses:location}/include -I${openssl:location}/include -I${pcre:location}/include -I${readline5:location}/include -I${xz-utils:location}/include -I${zlib:location}/include -I${unixodbc:location}/include -I${lz4:location}/include -I${snappy:location}/include -I${zstd:location}/include
 CMAKE_LIBRARY_PATH = ${bzip2:location}/lib:${jemalloc:location}/lib:${libaio:location}/lib:${libxml2:location}/lib:${ncurses:location}/lib:${openssl:location}/lib:${pcre:location}/lib:${readline5:location}/lib:${xz-utils:location}/lib:${zlib:location}/lib:${unixodbc:location}/lib:${lz4:location}/lib:${snappy:location}/lib:${zstd:location}/lib:${gcc:location}/lib:${gcc:location}/lib64
 environment =
@@ -77,11 +75,11 @@ environment =
   CMAKE_INCLUDE_PATH=${bzip2:location}/include:${libaio:location}/include:${libaio:location}/include:${libxml2:location}/include:${ncurses:location}/include:${openssl:location}/include:${pcre:location}/include:${readline5:location}/include:${xz-utils:location}/include:${zlib:location}/include:${unixodbc:location}/include:${lz4:location}/include:${snappy:location}/include:${zstd:location}/include
   CMAKE_LIBRARY_PATH=${:CMAKE_LIBRARY_PATH}
   LDFLAGS=-L${bzip2:location}/lib -L${jemalloc:location}/lib -L${libaio:location}/lib -L${pcre:location}/lib -L${xz-utils:location}/lib -L${zlib:location}/lib -L${unixodbc:location}/lib -L${lz4:location}/lib -L${snappy:location}/lib -L${zstd:location}/lib
-  PATH=${patch:location}/bin:%(PATH)s
+  PATH=${gcc:location}/bin:${patch:location}/bin:%(PATH)s
 patch-options = -p1
 patches =
   https://sources.debian.org/data/main/m/mariadb-10.3/1:10.3.22-0+deb10u1/debian/patches/0024-Revert-to-using-system-pcre-library.patch#1c6a0f2634f5a56122299674b77b1131
-post-install = set -e
+post-install = 
   set -- wsrep-lib/wsrep-API/*/wsrep_api.h
   install -DpT $1 ${:location}/$1
   cp -a wsrep-lib/include ${:location}/wsrep-lib
@@ -92,7 +90,7 @@ post-install = set -e
 recipe = slapos.recipe.cmmi
 url = https://packages.groonga.org/source/mroonga/mroonga-9.12.tar.gz
 md5sum = d0af673f1bad3b9ccf33870bb2344a25
-pre-configure = set -e
+pre-configure = 
   rm -rf fake_mariadb_source
   mkdir -p fake_mariadb_source
   cd fake_mariadb_source

component/nano/buildout.cfg

@@ -2,8 +2,9 @@
 parts =
   nano
 
-extends = 
+extends =
   ../file/buildout.cfg
+  ../zlib/buildout.cfg
   ../ncurses/buildout.cfg
 
 [nano]
@@ -17,8 +18,8 @@ location = @@LOCATION@@
 environment=
   PKG_CONFIG=false
   NCURSESW_CONFIG=${ncurses:location}/bin/ncursesw6-config
-  CPPFLAGS=-I${file:location}/include
-  LDFLAGS=-L${file:location}/lib/ -Wl,-rpath=${file:location}/lib/
+  CPPFLAGS=-I${file:location}/include -I${zlib:location}/include
+  LDFLAGS=-L${file:location}/lib/ -Wl,-rpath=${file:location}/lib/ -L${zlib:location}/lib/ -Wl,-rpath=${zlib:location}/lib/
 post-install =
   cd ${:location} && mkdir etc &&
   echo include "${:location}/share/nano/*.nanorc" > etc/nanorc

component/nodejs/buildout.cfg

@@ -15,6 +15,12 @@ parts =
 [nodejs]
 <= nodejs-8.9.4
 
+[nodejs-10.19.0]
+<= nodejs-base
+openssl_location = ${openssl:location}
+version = v10.19.0
+md5sum = 9e433c753d839d2d2a6341861501674f
+
 [nodejs-10.6.0]
 <= nodejs-base
 openssl_location = ${openssl:location}

component/onlyoffice-core/buildout.cfg

@@ -25,7 +25,7 @@ environment =
   CXXFLAGS=-I${libxml2:location}/include -I${zlib:location}/include -I${icu4c-slaposgcc:location}/include -I${boost-lib:location}/include -Wno-comment -Wno-deprecated-declarations -Wno-endif-labels -Wno-parentheses -Wno-reorder -Wno-sign-compare -Wno-switch -Wno-unknown-pragmas -Wno-unused
   LDFLAGS=-L${gcc:location}/lib -Wl,-rpath=${gcc:location}/lib -L${gcc:location}/lib64 -Wl,-rpath=${gcc:location}/lib64 -L${libxml2:location}/lib -Wl,-rpath=${libxml2:location}/lib -L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib -L${icu4c-slaposgcc:location}/lib -Wl,-rpath=${icu4c-slaposgcc:location}/lib -L${boost-lib:location}/lib -Wl,-rpath=${boost-lib:location}/lib -Wl,-rpath=${:location}/lib
 post-install =
-  set -e -x
+  set -x
   mkdir -p ${:location}/bin ${:location}/lib
   mv -t ${:location}/lib build/lib/*/*.so
   mv -t ${:location}/bin build/bin/*/*

component/pole/buildout.cfg

@@ -12,7 +12,7 @@ make-binary =
 make-options =
 make-targets = true
 post-install =
-  set -e -x
+  set -x
   mkdir bin
   cd pole
   g++ -o ../bin/poledump pole.cpp poledump.cpp

component/protobuf/buildout.cfg

 [buildout]
+extends =
+  ../zlib/buildout.cfg
+
 parts = protobuf
 
 [protobuf]
@@ -6,9 +9,13 @@ recipe = slapos.recipe.cmmi
 shared = true
 url = https://github.com/google/protobuf/releases/download/v3.4.0/protobuf-python-3.4.0.tar.gz
 md5sum = 0820cc2e56d71aef8e99794fcbd184cd
+environment =
+  LDFLAGS=-Wl,-rpath=${zlib:location}/lib
 
 [protobuf-cpp]
 recipe = slapos.recipe.cmmi
 shared = true
 url = https://github.com/google/protobuf/releases/download/v3.4.0/protobuf-cpp-3.4.0.tar.gz
 md5sum = 6d59dad503bea5ad420fd09ddad84481
+environment =
+  LDFLAGS=-Wl,-rpath=${zlib:location}/lib
\ No newline at end of file

component/qt/buildout.cfg

@@ -47,7 +47,7 @@ url = http://download.qt.io/archive/qt/4.8/4.8.7/qt-everywhere-opensource-src-4.
 md5sum = d990ee66bf7ab0c785589776f35ba6ad
 # see https://github.com/NixOS/nixpkgs/blob/3e387c3e005c87566b5403d24c86f71f4945a79b/pkgs/development/libraries/qt-4.x/4.8/default.nix#L101
 pre-configure =
-  set -e -x
+  set -x
   sed 's,/usr/X11R6/lib64,${libX11:location}/lib64 ${xproto:location}/lib64 ${libXext:location}/lib64,g' -i mkspecs/*/*.conf
   sed 's,/usr/X11R6/lib,${libX11:location}/lib ${xproto:location}/lib ${libXext:location}/lib,g' -i mkspecs/*/*.conf
   sed 's,/usr/X11R6/include,${libX11:location}/include ${xproto:location}/include ${libXext:location}/include,g' -i mkspecs/*/*.conf

component/rpy2/buildout.cfg

@@ -27,6 +27,7 @@ rpath =
   ${readline:location}/lib
   ${xz-utils:location}/lib
   ${r-language:location}/lib/R/lib
+  ${r-language:location}/lib64/R/lib
 library-dirs =
   ${pcre:location}/lib
   ${readline:location}/lib

component/screen/buildout.cfg

@@ -3,6 +3,7 @@ parts =
   screen
 extends =
   ../ncurses/buildout.cfg
+  ../patch/buildout.cfg
   ../texinfo/buildout.cfg
 
 [screen]
@@ -11,7 +12,10 @@ shared = true
 url = http://ftp.gnu.org/gnu/screen/screen-${:version}.tar.gz
 version = 4.8.0
 md5sum = d276213d3acd10339cd37848b8c4ab1e
+patches =
+  ${:_profile_base_location_}/dependencies.diff
+patch-options = -p2
 environment =
   CFLAGS=-I${ncurses:location}/include
   LDFLAGS=-L${ncurses:location}/lib -Wl,-rpath=${ncurses:location}/lib
-  PATH=${texinfo:location}/bin:%(PATH)s
+  PATH=${patch:location}/bin:${texinfo:location}/bin:%(PATH)s

component/screen/dependencies.diff

+Regenerate the dependencies for screen as they are broken in the screen-v4 branch.
+
+This patch should be removed when we start using screen v5.0.0 because dependencies are fixed in master branch.
+
+This patch has been generated with the following command:
+grep '\.o:' Makefile.in | grep -v '\.c\.o' | sed 's/\.o.*/.c/' | xargs -l gcc -E -MM >> Makefile.in
+
+diff --git a/src/Makefile.in b/src/Makefile.in
+index 08b44d3..80e7da4 100644
+--- a/src/Makefile.in
++++ b/src/Makefile.in
+@@ -278,79 +278,107 @@ depend.in: $(CFILES) term.h
+ ###############################################################################
+ 
+ ### Dependencies:
+-screen.o: layout.h viewport.h canvas.h screen.c config.h screen.h os.h osdef.h ansi.h acls.h \
+- comm.h layer.h term.h image.h display.h window.h braille.h \
+- patchlevel.h logfile.h extern.h
+-ansi.o: layout.h viewport.h canvas.h ansi.c config.h screen.h os.h osdef.h ansi.h acls.h \
+- comm.h layer.h term.h image.h display.h window.h braille.h extern.h \
+- logfile.h
+-fileio.o: layout.h viewport.h canvas.h fileio.c config.h screen.h os.h osdef.h ansi.h acls.h \
+- comm.h layer.h term.h image.h display.h window.h extern.h
+-mark.o: layout.h viewport.h canvas.h mark.c config.h screen.h os.h osdef.h ansi.h acls.h \
+- comm.h layer.h term.h image.h display.h window.h mark.h extern.h
+-misc.o: layout.h viewport.h canvas.h misc.c config.h screen.h os.h osdef.h ansi.h acls.h \
+- comm.h layer.h term.h image.h display.h window.h extern.h
+-resize.o: layout.h viewport.h canvas.h resize.c config.h screen.h os.h osdef.h ansi.h acls.h \
+- comm.h layer.h term.h image.h display.h window.h extern.h
+-socket.o: layout.h viewport.h canvas.h socket.c config.h screen.h os.h osdef.h ansi.h acls.h \
+- comm.h layer.h term.h image.h display.h window.h extern.h
+-search.o: layout.h viewport.h canvas.h search.c config.h screen.h os.h osdef.h ansi.h acls.h \
+- comm.h layer.h term.h image.h display.h window.h mark.h extern.h
+-tty.o: layout.h viewport.h canvas.h tty.c config.h screen.h os.h osdef.h ansi.h acls.h comm.h \
+- layer.h term.h image.h display.h window.h extern.h
+-term.o: layout.h viewport.h canvas.h term.c term.h
+-window.o: layout.h viewport.h canvas.h window.c config.h screen.h os.h osdef.h ansi.h acls.h \
+- comm.h layer.h term.h image.h display.h window.h extern.h logfile.h
+-utmp.o: layout.h viewport.h canvas.h utmp.c config.h screen.h os.h osdef.h ansi.h acls.h \
+- comm.h layer.h term.h image.h display.h window.h extern.h
+-loadav.o: layout.h viewport.h canvas.h loadav.c config.h screen.h os.h osdef.h ansi.h acls.h \
+- comm.h layer.h term.h image.h display.h window.h extern.h
+-putenv.o: layout.h viewport.h canvas.h putenv.c config.h
+-help.o: layout.h viewport.h canvas.h help.c config.h screen.h os.h osdef.h ansi.h acls.h \
+- comm.h layer.h term.h image.h display.h window.h extern.h list_generic.h
+-termcap.o: layout.h viewport.h canvas.h termcap.c config.h screen.h os.h osdef.h ansi.h acls.h \
+- comm.h layer.h term.h image.h display.h window.h extern.h
+-input.o: layout.h viewport.h canvas.h input.c config.h screen.h os.h osdef.h ansi.h acls.h \
+- comm.h layer.h term.h image.h display.h window.h extern.h
+-attacher.o: layout.h viewport.h canvas.h attacher.c config.h screen.h os.h osdef.h ansi.h \
+- acls.h comm.h layer.h term.h image.h display.h window.h extern.h
+-pty.o: layout.h viewport.h canvas.h pty.c config.h screen.h os.h osdef.h ansi.h acls.h comm.h \
+- layer.h term.h image.h display.h window.h extern.h
+-process.o: layout.h viewport.h canvas.h process.c config.h screen.h os.h osdef.h ansi.h acls.h \
+- comm.h layer.h term.h image.h display.h window.h extern.h logfile.h
+-display.o: layout.h viewport.h canvas.h display.c config.h screen.h os.h osdef.h ansi.h acls.h \
+- comm.h layer.h term.h image.h display.h window.h extern.h braille.h
+-canvas.o: layout.h viewport.h canvas.h canvas.c config.h screen.h os.h osdef.h ansi.h acls.h \
+- comm.h layer.h term.h image.h display.h window.h extern.h \
+- braille.h
+-comm.o: layout.h viewport.h canvas.h comm.c config.h acls.h comm.h
+-kmapdef.o: layout.h viewport.h canvas.h kmapdef.c config.h
+-acls.o: layout.h viewport.h canvas.h acls.c config.h screen.h os.h osdef.h ansi.h acls.h comm.h \
+- layer.h term.h image.h display.h window.h extern.h
+-braille.o: layout.h viewport.h canvas.h braille.c config.h screen.h os.h osdef.h ansi.h acls.h \
+- comm.h layer.h term.h image.h display.h window.h extern.h braille.h
+-braille_tsi.o: layout.h viewport.h canvas.h braille_tsi.c config.h screen.h os.h osdef.h ansi.h \
+- acls.h comm.h layer.h term.h image.h display.h window.h extern.h \
+- braille.h
+-logfile.o: layout.h viewport.h canvas.h logfile.c config.h screen.h os.h osdef.h ansi.h acls.h \
+- comm.h layer.h term.h image.h display.h window.h extern.h logfile.h
+-layer.o: layout.h viewport.h canvas.h layer.c config.h screen.h os.h osdef.h ansi.h acls.h \
+- comm.h layer.h term.h image.h display.h window.h extern.h
+-sched.o: layout.h viewport.h canvas.h sched.c config.h screen.h os.h osdef.h ansi.h acls.h \
+- comm.h layer.h term.h image.h display.h window.h extern.h logfile.h
+-teln.o: layout.h viewport.h canvas.h teln.c config.h screen.h os.h osdef.h ansi.h acls.h \
+- comm.h layer.h term.h image.h display.h window.h extern.h
+-nethack.o: layout.h viewport.h canvas.h nethack.c config.h screen.h os.h osdef.h ansi.h acls.h \
+- comm.h layer.h term.h image.h display.h window.h extern.h
+-encoding.o: layout.h viewport.h canvas.h encoding.c config.h screen.h os.h osdef.h ansi.h acls.h \
+- comm.h layer.h term.h image.h display.h window.h extern.h
+-layout.o: layout.h viewport.h canvas.h layout.c config.h screen.h os.h osdef.h ansi.h acls.h \
+- comm.h layer.h term.h image.h display.h window.h extern.h \
+- braille.h
+-viewport.o: layout.h viewport.h canvas.h viewport.c config.h screen.h os.h osdef.h ansi.h acls.h \
+- comm.h layer.h term.h image.h display.h window.h extern.h \
+- braille.h
+-list_generic.o: list_generic.h list_generic.c layer.h screen.h osdef.h
+-list_display.o: list_generic.h list_display.c layer.h screen.h osdef.h
+-list_window.o: list_generic.h list_window.c window.h layer.h screen.h osdef.h comm.h
+-
++screen.o: screen.c config.h screen.h os.h osdef.h ansi.h sched.h acls.h \
++ comm.h layer.h term.h image.h canvas.h display.h layout.h viewport.h \
++ window.h patchlevel.h logfile.h extern.h
++ansi.o: ansi.c config.h screen.h os.h osdef.h ansi.h sched.h acls.h \
++ comm.h layer.h term.h image.h canvas.h display.h layout.h viewport.h \
++ window.h braille.h extern.h logfile.h
++fileio.o: fileio.c config.h screen.h os.h osdef.h ansi.h sched.h acls.h \
++ comm.h layer.h term.h image.h canvas.h display.h layout.h viewport.h \
++ window.h extern.h
++mark.o: mark.c config.h screen.h os.h osdef.h ansi.h sched.h acls.h \
++ comm.h layer.h term.h image.h canvas.h display.h layout.h viewport.h \
++ window.h mark.h extern.h
++misc.o: misc.c config.h screen.h os.h osdef.h ansi.h sched.h acls.h \
++ comm.h layer.h term.h image.h canvas.h display.h layout.h viewport.h \
++ window.h extern.h
++resize.o: resize.c config.h screen.h os.h osdef.h ansi.h sched.h acls.h \
++ comm.h layer.h term.h image.h canvas.h display.h layout.h viewport.h \
++ window.h extern.h
++socket.o: socket.c config.h screen.h os.h osdef.h ansi.h sched.h acls.h \
++ comm.h layer.h term.h image.h canvas.h display.h layout.h viewport.h \
++ window.h extern.h list_generic.h
++search.o: search.c config.h screen.h os.h osdef.h ansi.h sched.h acls.h \
++ comm.h layer.h term.h image.h canvas.h display.h layout.h viewport.h \
++ window.h mark.h extern.h
++tty.o: tty.c config.h screen.h os.h osdef.h ansi.h sched.h acls.h comm.h \
++ layer.h term.h image.h canvas.h display.h layout.h viewport.h window.h \
++ extern.h
++term.o: term.c term.h
++window.o: window.c config.h screen.h os.h osdef.h ansi.h sched.h acls.h \
++ comm.h layer.h term.h image.h canvas.h display.h layout.h viewport.h \
++ window.h extern.h logfile.h
++utmp.o: utmp.c config.h screen.h os.h osdef.h ansi.h sched.h acls.h \
++ comm.h layer.h term.h image.h canvas.h display.h layout.h viewport.h \
++ window.h extern.h
++loadav.o: loadav.c config.h screen.h os.h osdef.h ansi.h sched.h acls.h \
++ comm.h layer.h term.h image.h canvas.h display.h layout.h viewport.h \
++ window.h extern.h
++putenv.o: putenv.c config.h
++help.o: help.c config.h screen.h os.h osdef.h ansi.h sched.h acls.h \
++ comm.h layer.h term.h image.h canvas.h display.h layout.h viewport.h \
++ window.h extern.h list_generic.h
++termcap.o: termcap.c config.h screen.h os.h osdef.h ansi.h sched.h acls.h \
++ comm.h layer.h term.h image.h canvas.h display.h layout.h viewport.h \
++ window.h extern.h
++input.o: input.c config.h screen.h os.h osdef.h ansi.h sched.h acls.h \
++ comm.h layer.h term.h image.h canvas.h display.h layout.h viewport.h \
++ window.h extern.h
++attacher.o: attacher.c config.h screen.h os.h osdef.h ansi.h sched.h \
++ acls.h comm.h layer.h term.h image.h canvas.h display.h layout.h \
++ viewport.h window.h extern.h
++pty.o: pty.c config.h screen.h os.h osdef.h ansi.h sched.h acls.h comm.h \
++ layer.h term.h image.h canvas.h display.h layout.h viewport.h window.h \
++ extern.h
++process.o: process.c config.h screen.h os.h osdef.h ansi.h sched.h acls.h \
++ comm.h layer.h term.h image.h canvas.h display.h layout.h viewport.h \
++ window.h extern.h logfile.h list_generic.h
++display.o: display.c config.h screen.h os.h osdef.h ansi.h sched.h acls.h \
++ comm.h layer.h term.h image.h canvas.h display.h layout.h viewport.h \
++ window.h extern.h braille.h
++canvas.o: canvas.c config.h screen.h os.h osdef.h ansi.h sched.h acls.h \
++ comm.h layer.h term.h image.h canvas.h display.h layout.h viewport.h \
++ window.h extern.h list_generic.h
++comm.o: comm.c config.h os.h acls.h comm.h
++kmapdef.o: kmapdef.c config.h
++acls.o: acls.c config.h screen.h os.h osdef.h ansi.h sched.h acls.h \
++ comm.h layer.h term.h image.h canvas.h display.h layout.h viewport.h \
++ window.h extern.h
++braille.o: braille.c config.h screen.h os.h osdef.h ansi.h sched.h acls.h \
++ comm.h layer.h term.h image.h canvas.h display.h layout.h viewport.h \
++ window.h extern.h braille.h
++braille_tsi.o: braille_tsi.c config.h screen.h os.h osdef.h ansi.h \
++ sched.h acls.h comm.h layer.h term.h image.h canvas.h display.h layout.h \
++ viewport.h window.h extern.h braille.h
++logfile.o: logfile.c config.h screen.h os.h osdef.h ansi.h sched.h acls.h \
++ comm.h layer.h term.h image.h canvas.h display.h layout.h viewport.h \
++ window.h extern.h logfile.h
++layer.o: layer.c config.h screen.h os.h osdef.h ansi.h sched.h acls.h \
++ comm.h layer.h term.h image.h canvas.h display.h layout.h viewport.h \
++ window.h mark.h extern.h braille.h
++sched.o: sched.c config.h screen.h os.h osdef.h ansi.h sched.h acls.h \
++ comm.h layer.h term.h image.h canvas.h display.h layout.h viewport.h \
++ window.h extern.h
++teln.o: teln.c config.h
++nethack.o: nethack.c config.h screen.h os.h osdef.h ansi.h sched.h acls.h \
++ comm.h layer.h term.h image.h canvas.h display.h layout.h viewport.h \
++ window.h
++encoding.o: encoding.c config.h screen.h os.h osdef.h ansi.h sched.h \
++ acls.h comm.h layer.h term.h image.h canvas.h display.h layout.h \
++ viewport.h window.h extern.h
++layout.o: layout.c config.h screen.h os.h osdef.h ansi.h sched.h acls.h \
++ comm.h layer.h term.h image.h canvas.h display.h layout.h viewport.h \
++ window.h extern.h
++viewport.o: viewport.c config.h screen.h os.h osdef.h ansi.h sched.h \
++ acls.h comm.h layer.h term.h image.h canvas.h display.h layout.h \
++ viewport.h window.h extern.h
++list_generic.o: list_generic.c config.h screen.h os.h osdef.h ansi.h \
++ sched.h acls.h comm.h layer.h term.h image.h canvas.h display.h layout.h \
++ viewport.h window.h list_generic.h extern.h
++list_display.o: list_display.c config.h screen.h os.h osdef.h ansi.h \
++ sched.h acls.h comm.h layer.h term.h image.h canvas.h display.h layout.h \
++ viewport.h window.h extern.h list_generic.h
++list_window.o: list_window.c config.h screen.h os.h osdef.h ansi.h \
++ sched.h acls.h comm.h layer.h term.h image.h canvas.h display.h layout.h \
++ viewport.h window.h extern.h list_generic.h
+

component/sqlite3/buildout.cfg

 [buildout]
-extends = ../readline/buildout.cfg
+extends =
+  ../readline/buildout.cfg
+  ../zlib/buildout.cfg
 parts =
   sqlite3
 
@@ -14,5 +16,5 @@ configure-options =
 # Increase MAX_VARIABLE_NUMBER like many os. For example:
 # https://git.archlinux.org/svntogit/packages.git/tree/trunk/PKGBUILD?h=packages/sqlite
 environment =
-  CPPFLAGS=-I${readline:location}/include -I${ncurses:location}/include -DSQLITE_MAX_VARIABLE_NUMBER=250000
-  LDFLAGS=-L@@LOCATION@@ -Wl,-rpath=${readline:location}/lib -Wl,-rpath=${ncurses:location}/lib -L${readline:location}/lib -L${ncurses:location}/lib
+  CPPFLAGS=-I${readline:location}/include -I${ncurses:location}/include -I${zlib:location}/include -DSQLITE_MAX_VARIABLE_NUMBER=250000
+  LDFLAGS=-L@@LOCATION@@ -Wl,-rpath=${readline:location}/lib -Wl,-rpath=${ncurses:location}/lib -L${readline:location}/lib -L${ncurses:location}/lib -L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib

component/tensorflow/buildout.cfg

-# [buildout]
+[buildout]
 extends =
   ../../stack/slapos.cfg
   ../scipy/buildout.cfg

component/tmux/buildout.cfg

@@ -10,8 +10,9 @@ parts +=
 recipe = slapos.recipe.cmmi
 url = https://github.com/tmux/tmux/releases/download/2.9a/tmux-2.9a.tar.gz
 md5sum = f0564dbf4cf6b301c4845219d27ed3ad
+shared = true
 environment =
   CFLAGS=-I${ncurses:location}/include -I${libevent2:location}/include/
   LDFLAGS=-L${ncurses:location}/lib/ -L${libevent2:location}/lib/ -Wl,-rpath=${ncurses:location}/lib/ -Wl,-rpath=${libevent2:location}/lib/
 configure-options =
-  --prefix=${buildout:parts-directory}/${:_buildout_section_name_}
+  --prefix=@@LOCATION@@

component/wendelin.core/buildout.cfg

@@ -14,7 +14,7 @@ egg = wendelin.core
 [wendelin.core-dev]
 recipe = zc.recipe.egg:develop
 egg = wendelin.core
-setup = ${wendelin.core-repository-submoduleinit:location}
+setup = ${wendelin.core-repository:location}
 environment = wendelin.core-dev-env
 
 [wendelin.core-dev-env]
@@ -28,13 +28,3 @@ repository = https://lab.nexedi.com/nexedi/wendelin.core.git
 # dir is pretty name as top-level -dev recipe
 location = ${buildout:parts-directory}/wendelin.core-dev
 git-executable = ${git:location}/bin/git
-
-
-# TODO add `git clone --recursive` to slapos.recipe.build:gitclone
-#      and this way merge this -submoduleinit into -repository part
-[wendelin.core-repository-submoduleinit]
-recipe = plone.recipe.command
-command = cd "${wendelin.core-repository:location}" && ${git:location}/bin/git submodule update --init
-stop-on-error = true
-# propagate location of main repo
-location= ${wendelin.core-repository:location}

component/zstd/buildout.cfg

 [buildout]
-extends = ../zlib/buildout.cfg
+extends =
+  ../zlib/buildout.cfg
+  ../xz-utils/buildout.cfg
+  ../lz4/buildout.cfg
 parts =
   zstd
 
 [zstd]
 recipe = slapos.recipe.cmmi
+shared = true
 url = https://github.com/facebook/zstd/releases/download/v${:version}/zstd-${:version}.tar.gz
 version = 1.4.4
 md5sum = 487f7ee1562dee7c1c8adf85e2a63df9
-shared = true
 location = @@LOCATION@@
 configure-command = :
 environment =
   PREFIX=${:location}
-  LD_FLAGS=-L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib
-  CFLAGS=-I${zlib:location}/include
+  LDFLAGS=-L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib -L${xz-utils:location}/lib -Wl,-rpath=${xz-utils:location}/lib -L${lz4:location}/lib -Wl,-rpath=${lz4:location}/lib
+  CFLAGS=-I${zlib:location}/include -I${xz-utils:location}/include -I${lz4:location}/include

release-cookbook

+#!/bin/bash
+
+git fetch
+
+git checkout master
+git reset origin/master --hard
+git clean -f
+
+# Remove any local tag
+git fetch --prune origin "+refs/tags/*:refs/tags/*"
+rm -rf dist/slapos.cookbook*
+
+# for now this script is hardcoded to release on 1.0.x versions intentionally
+# update to version 2 or 1.1 would require a major reorganisation on the release
+# process
+CURRENT_VERSION=`git tag | grep "^1\+\.0\+\.[0-9]\+$" | sort -t. -k 1,1n -k 2,2n -k 3,3n | tail -1`
+NEXT_MINOR_VERSION=$((`echo $CURRENT_VERSION | cut -f3 -d.`+1))
+NEXT_VERSION=1.0.$NEXT_MINOR_VERSION
+
+CURRENT_EGG_VERSION=`cat setup.py | grep ^version | cut -d\' -f2`
+
+sed -i "s/$CURRENT_EGG_VERSION/$NEXT_VERSION/g" setup.py
+git commit -m "Release slapos.cookbook ($NEXT_VERSION)" setup.py
+
+if [ ! $EGG_ONLY == "Y" ];
+then
+  sed -i "s/slapos.cookbook = $CURRENT_EGG_VERSION/slapos.cookbook = $NEXT_VERSION/g" stack/slapos.cfg
+  sed -i "s/slapos.cookbook==$CURRENT_EGG_VERSION/slapos.cookbook==$NEXT_VERSION/g" stack/slapos.cfg
+  git commit -m "stack/slapos: slapos.cookbook version up ($NEXT_VERSION)" stack/slapos.cfg
+fi
+
+echo "############### Merging into 1.0 and generating the tag ################"
+git checkout 1.0
+git reset --hard origin/1.0 
+git clean -f
+git merge master -m "Update Release Candidate"
+
+# Revert changes from master
+git diff HEAD..master | git apply 
+
+# Download patch to update revisions based on tests
+wget -O update-release.sh https://nexedi.erp5.net/portal_skins/custom/TestResultModule_getReleaseCandidateRevision 
+bash update-release.sh
+git commit -m 'Update git revisions' -a  
+git tag $NEXT_VERSION -m "Release $NEXT_VERSION"
+
+echo "############### Building sdist  ################"
+python setup.py sdist
+
+
+echo "###################################################################"
+echo "You are about to release a new tag, a new version of slapos.cookbook"
+echo " and a new version of SlapOS Software Release"
+echo ""
+echo "Lastest release: $CURRENT_VERSION"
+echo "Next Release to be Tagged: $NEXT_VERSION"
+echo "Current slapos.cookbook version: $CURRENT_EGG_VERSION"
+echo "Next slapos.cookbook to be released: $NEXT_VERSION"
+echo ""
+echo "###################################################################"
+echo ""
+echo ""
+echo "########################################################################"
+echo " Please review the current local changes before continue. Ensure that"
+echo " ALL changes and commits are correct before continue."
+echo ""
+echo " To review master use :: git log master"
+echo " To review 1.0 use    :: git log 1.0"
+echo " To review $NEXT_VERSION use :: git log $NEXT_VERSION"
+echo " To review individual commits :: git show HASH"
+echo ""
+echo ""
+echo "Once everything is ok, please upload slapos.cookbook egg /!\\ BEFORE /!\\" 
+echo "push your changes into the repository. Please use twine for it."
+echo ""
+echo "To upload :: python -m twine upload dist/slapos.cookbook-$NEXT_VERSION*"
+echo "To verify if upload succeed access: https://pypi.org/project/slapos.cookbook/$NEXT_VERSION/"
+echo ""
+echo " More info on twine: https://packaging.python.org/tutorials/packaging-projects/#uploading-the-distribution-archives"
+echo ""
+echo "Once egg is updated please push your changes into main repository:"
+echo ""
+echo " git push origin master 1.0 $NEXT_VERSION"
+echo ""
+echo " !! DO NOT FORGET TO PUSH YOUR CHANGES IN THE END !!"
+echo ""
+echo "###################################################################"
+echo ""

release-sr

+#!/bin/bash
+
+git fetch
+
+git checkout 1.0
+git reset origin/1.0 --hard
+
+# for now this script is hardcoded to release on 1.0.x versions intentionally
+# update to version 2 or 1.1 would require a major reorganisation on the release
+# process
+CURRENT_VERSION=`git tag | grep "^1\+\.0\+\.[0-9]\+$" | sort -t. -k 1,1n -k 2,2n -k 3,3n | tail -1`
+
+NEXT_MINOR_VERSION=$((`echo $CURRENT_VERSION | cut -f3 -d.`+1))
+
+echo "###################################################################"
+echo "You are about to release a new version of SlapOS Software Release"
+echo "Lastest release: $CURRENT_VERSION"
+echo "Next Release to be Tagged: 1.0.$NEXT_MINOR_VERSION"
+echo "###################################################################"
+
+git tag 1.0.$NEXT_MINOR_VERSION -m "Release 1.0.$NEXT_MINOR_VERSION"
+
+
+echo " Please review current tag and them push it:"
+echo " To review use git log 1.0.$NEXT_MINOR_VERSION"
+
+echo ""
+echo " To push into main repository:"
+echo " git push origin 1.0.$NEXT_MINOR_VERSION"

setup.py

@@ -141,8 +141,6 @@ setup(name=name,
           'onetimeupload = slapos.recipe.onetimeupload:Recipe',
           'pbs = slapos.recipe.pbs:Recipe',
           'postgres = slapos.recipe.postgres:Recipe',
-          'postgres.export = slapos.recipe.postgres.backup:ExportRecipe',
-          'postgres.import = slapos.recipe.postgres.backup:ImportRecipe',
           'proactive = slapos.recipe.proactive:Recipe',
           'promise.plugin= slapos.recipe.promise_plugin:Recipe',
           'publish = slapos.recipe.publish:Recipe',

slapos/recipe/postgres/__init__.py → slapos/recipe/postgres.py

@@ -42,7 +42,8 @@ class Recipe(GenericBaseRecipe):
 
         - a Postgres cluster
         - configuration to allow connections from IPv4, IPv6 or unix socket.
-        - a superuser with provided name and generated password
+          IPv4 and IPv6 can be disabled, unix socket will always be available.
+        - a superuser with provided name and password
         - a database with provided name
         - a start script in the services directory
 
@@ -52,50 +53,59 @@ class Recipe(GenericBaseRecipe):
         dbname
             name of the database to be used by the application.
         ipv4
-            set of ipv4 to listen on.
+            ipv4 to listen on, can be multiple ips or can be empty.
         ipv6
-            set of ipv6 to listen on.
+            ipv6 to listen on, can be multiple ips or can be empty.
+        port
+            port to listen on, same for both IPv4 and IPv6.
         pgdata-directory
             path to postgres configuration and data.
         services
             must be ${buildout:directory}/etc/service.
         superuser
             name of the superuser to create.
+        password
+            password for the superuser.
 
     Exposed options:
-        password
-            generated password for the superuser.
         url
-            generated DBAPI connection string.
+            generated DBAPI connection string, on IPv6.
             it can be used as-is (ie. in sqlalchemy) or by the _urlparse.py recipe.
+            this is only available if at least one IPv6 was provided.
     """
 
     def _options(self, options):
-        options['url'] = 'postgresql://%(superuser)s:%(password)s@[%(ipv6-random)s]:%(port)s/%(dbname)s' % options
-
+        if options.get('ipv6'):
+            options['url'] = "postgresql://{superuser}:{password}@[{ipv6}]:{port}/{dbname}".format(
+                superuser=options['superuser'],
+                password=options['password'],
+                ipv6=options['ipv6'].splitlines()[0],
+                port=options['port'],
+                dbname=options['dbname'],
+            )
 
     def install(self):
         pgdata = self.options['pgdata-directory']
 
-        # if the pgdata already exists, skip all steps, we don't need to do anything.
-
+        paths = []
+        # if the pgdata already exists, we don't need to recreate databases.
         if not os.path.exists(pgdata):
             try:
                 self.createCluster()
-                self.createConfig()
+                paths.extend(self.createConfig())
                 self.createDatabase()
                 self.updateSuperuser()
-                self.createRunScript()
+                paths.extend(self.createRunScript())
             except:
                 # do not leave half-installed postgresql - else next time we
                 # run we won't update it.
                 shutil.rmtree(pgdata)
                 raise
         else:
-            self.createConfig()
-            self.createRunScript()
+            paths.extend(self.createConfig())
+            paths.extend(self.createRunScript())
 
-        return []
+        return paths
 
     update = install
 
@@ -129,10 +139,11 @@ class Recipe(GenericBaseRecipe):
 
     def createConfig(self):
         pgdata = self.options['pgdata-directory']
-        ipv4 = self.options['ipv4']
-        ipv6 = self.options['ipv6']
+        ipv4 = self.options['ipv4'].splitlines()
+        ipv6 = self.options['ipv6'].splitlines()
 
-        with open(os.path.join(pgdata, 'postgresql.conf'), 'wb') as cfg:
+        postgres_conf = os.path.join(pgdata, 'postgresql.conf')
+        with open(postgres_conf, 'w') as cfg:
             cfg.write(textwrap.dedent("""\
                     listen_addresses = '%s'
                     logging_collector = on
@@ -149,11 +160,12 @@ class Recipe(GenericBaseRecipe):
                     unix_socket_directories = '%s'
                     unix_socket_permissions = 0700
                     """ % (
-                        ','.join(ipv4.union(ipv6)),
+                        ','.join(set(ipv4).union(ipv6)),
                         pgdata,
                         )))
 
-        with open(os.path.join(pgdata, 'pg_hba.conf'), 'wb') as cfg:
+        pg_hba_conf = os.path.join(pgdata, 'pg_hba.conf')
+        with open(pg_hba_conf, 'w') as cfg:
             # see http://www.postgresql.org/docs/9.2/static/auth-pg-hba-conf.html
 
             cfg_lines = [
@@ -174,7 +186,7 @@ class Recipe(GenericBaseRecipe):
                 cfg_lines.append('host    all             all             %s/%s                   md5' % (ip, ipv6_netmask_bits))
 
             cfg.write('\n'.join(cfg_lines))
-
+        return postgres_conf, pg_hba_conf
 
     def createDatabase(self):
         self.runPostgresCommand(cmd='CREATE DATABASE "%s"' % self.options['dbname'])
@@ -232,6 +244,6 @@ class Recipe(GenericBaseRecipe):
                     -D %(pgdata-directory)s
                 """ % self.options)
         name = os.path.join(self.options['services'], 'postgres-start')
-        self.createExecutable(name, content=content)
+        return [self.createExecutable(name, content=content)]
 
 

slapos/test/recipe/test_postgres.py

+import unittest
+import tempfile
+import shutil
+import os.path
+import zc.buildout.testing
+
+
+class PostgresTest(unittest.TestCase):
+  def setUp(self):
+    self.buildout = buildout = zc.buildout.testing.Buildout()
+    self.pgdata_directory = tempfile.mkdtemp()
+    self.addCleanup(shutil.rmtree, self.pgdata_directory)
+    self.services_directory = tempfile.mkdtemp()
+    self.addCleanup(shutil.rmtree, self.services_directory)
+    buildout['postgres'] = {
+      'bin': 'software/parts/postgres/bin/',
+      'dbname': 'dbname',
+      'ipv4': '127.0.0.1',
+      'ipv6': '::1',
+      'port': '5443',
+      'pgdata-directory': self.pgdata_directory,
+      'services': self.services_directory,
+      'superuser': 'superuser',
+      'password': 'secret',
+    }
+    from slapos.recipe import postgres
+    self.recipe = postgres.Recipe(
+        buildout,
+        'postgres',
+        buildout['postgres'])
+
+  def test_options(self):
+    self.assertEqual(
+        'postgresql://superuser:secret@[::1]:5443/dbname',
+        self.buildout['postgres']['url'])
+
+  def test_install(self):
+    installed = self.recipe.install()
+
+    self.assertIn('postgresql.conf', os.listdir(self.pgdata_directory))
+    self.assertIn('pg_hba.conf', os.listdir(self.pgdata_directory))
+    self.assertIn('postgres-start', os.listdir(self.services_directory))
+
+    self.assertEqual(
+        sorted(installed),
+        sorted([
+            os.path.join(self.pgdata_directory, 'postgresql.conf'),
+            os.path.join(self.pgdata_directory, 'pg_hba.conf'),
+            os.path.join(self.services_directory, 'postgres-start')]))

software/apache-frontend/buildout.hash.cfg

@@ -18,7 +18,7 @@ md5sum = f686f765e55d1dce2e55a400f0714b3e
 
 [template-apache-frontend]
 filename = instance-apache-frontend.cfg
-md5sum = eb2143422d824e96abe99619f11d3523
+md5sum = ec825cba3ea775452716f1da7cd2e549
 
 [template-apache-replicate]
 filename = instance-apache-replicate.cfg.in

software/apache-frontend/common.cfg

@@ -38,9 +38,9 @@ parts +=
   proxy-by-url
   http-proxy
 
-# Extent extra-eggs.
 [extra-eggs]
-eggs +=
+recipe = zc.recipe.egg
+eggs =
   websockify
   erp5.util
 

software/apache-frontend/instance-apache-frontend.cfg

@@ -248,10 +248,7 @@ extra-context =
 [apache-frontend]
 recipe = slapos.cookbook:wrapper
 command-line = ${apache:location}/bin/httpd -f $${dynamic-apache-frontend-template:rendered} -DFOREGROUND
-wrapper-path = $${directory:service}/frontend_apache
-wait-for-files =
-	       $${ca-frontend:cert-file}
-	       $${ca-frontend:key-file}
+wrapper-path = $${directory:bin}/frontend_apache
 
 [not-found-html]
 recipe = slapos.cookbook:symbolic.link
@@ -315,7 +312,7 @@ crl = $${directory:ca-dir}/crl/
 recipe = slapos.cookbook:certificate_authority.request
 key-file = $${cadirectory:certs}/apache_frontend.key
 cert-file = $${cadirectory:certs}/apache_frontend.crt
-executable = $${directory:service}/frontend_apache
+executable = $${apache-frontend:wrapper-path}
 wrapper = $${directory:service}/frontend_apache
 key-content = $${instance-parameter:configuration.apache-key}
 cert-content = $${instance-parameter:configuration.apache-certificate}

software/caddy-frontend/buildout.hash.cfg

@@ -30,7 +30,7 @@ md5sum = 087bd9404cd120bd7602a9fbfcddc064
 
 [template-slave-list]
 filename = templates/apache-custom-slave-list.cfg.in
-md5sum = 46e0ccb54c2406b8a44cdada73c061a6
+md5sum = 29a61267959cc9ba7cdcd96fef41641a
 
 [template-slave-configuration]
 filename = templates/custom-virtualhost.conf.in

software/caddy-frontend/common.cfg

@@ -16,6 +16,7 @@ extends =
   ../../stack/monitor/buildout.cfg
 
 parts +=
+  caucase-eggs
   template
   template-caddy-frontend
   template-caddy-replicate
@@ -72,13 +73,7 @@ depends = ${caddyprofiledeps-develop:recipe}
 recipe = zc.recipe.egg
 eggs =
   caddyprofiledeps
-
-# Extent extra-eggs.
-[extra-eggs]
-eggs +=
   websockify
-  erp5.util
-  ${caucase-eggs:eggs}
   collective.recipe.shelloutput
 
 [template-common]

software/caddy-frontend/templates/apache-custom-slave-list.cfg.in

@@ -154,7 +154,7 @@ create = true
 <= logrotate-entry-base
 name = ${:_buildout_section_name_}
 log = {{slave_parameter_dict.get('access_log')}} {{slave_parameter_dict.get('error_log')}}
-backup = {{ slave_log_folder }} 
+backup = {{ slave_log_folder }}
 {%    if enable_cache %}
 [{{slave_logrotate_cache_direct_section}}]
 <= logrotate-entry-base
@@ -352,15 +352,6 @@ recipe = slapos.cookbook:mkdirectory
 
 {% do part_list.append('slave-log-cache-direct-directories') %}
 
-[caddy-log-access]
-< = jinja2-template-base
-template = {{frontend_configuration.get('template-log-access')}}
-rendered = {{frontend_configuration.get('log-access-configuration')}}
-extra-context =
-    section slave_log_directory slave-log-directory-dict
-    section slave_password slave-password
-    section parameter_dict caddy-log-access-parameters
-
 {% do part_list.append('caddy-log-access') %}
 
 ###############################################
@@ -474,26 +465,28 @@ template = inline:
 
 rendered = ${:file}
 
-[caddy-log-access-header]
+[caddy-log-access-empty]
 # Caddy refuse to start if an `import`ed file is empty, so we prepend a header
 # so that the file is never empty.
 < = jinja2-template-base
-template = inline: # This file contain directives to serve directories with log files
+template = inline: # This file contain directives to serve directories with log files for shared instances, but no shared instances are defined yet.
 rendered = {{frontend_configuration.get('log-access-configuration')}}
 
 [buildout]
 extends =
   {{ common_profile }}
-  {{ logrotate_base_instance }} 
+  {{ logrotate_base_instance }}
   {{ monitor_template }}
 
 parts +=
     kedifa-updater
     kedifa-updater-run
-    caddy-log-access-header
 {% for part in part_list %}
 {{ '    %s' % part }}
 {% endfor %}
+{% if 'caddy-log-access' not in part_list %}
+    caddy-log-access-empty
+{% endif %}
     publish-caddy-information
     tunnel-6to4-base-http_port
     tunnel-6to4-base-https_port

software/dream/buildout.hash.cfg

+[instance]
+filename = instance.cfg
+md5sum = 94e1cd21bbdf7cabb5834223859407e0
\ No newline at end of file

software/dream/instance.cfg

@@ -6,22 +6,26 @@ parts =
   dream_interpreter
   grunt_watch
   publish-connection-parameter
+  dream-platform-url-available
 
 eggs-directory = ${buildout:eggs-directory}
 develop-eggs-directory = ${buildout:develop-eggs-directory}
 offline = true
+extends = ${monitor-template:rendered}
 
 # parameters
 [instance-parameter]
 recipe = slapos.cookbook:slapconfiguration
-computer = $${slap_connection:computer_id}
-partition = $${slap_connection:partition_id}
-url = $${slap_connection:server_url}
-key = $${slap_connection:key_file}
-cert = $${slap_connection:cert_file}
+computer = $${slap-connection:computer-id}
+partition = $${slap-connection:partition-id}
+url = $${slap-connection:server-url}
+key = $${slap-connection:key-file}
+cert = $${slap-connection:cert-file}
+
 [dream_platform_parameter]
-port = 8080
+port = 18080
 host = $${instance-parameter:ipv6-random}
+url = http://[$${:host}]:$${:port}
 
 # interpreter
 [dream_interpreter]
@@ -35,6 +39,13 @@ recipe = slapos.cookbook:wrapper
 command-line = ${buildout:bin-directory}/dream_platform --debug --host $${dream_platform_parameter:host} --port $${dream_platform_parameter:port} --log $${directory:log}/dream_platform.log
 wrapper-path = $${directory:service}/dream_platform
 
+[dream-platform-url-available]
+<= monitor-promise-base
+module = check_url_available
+name = $${:_buildout_section_name_}.py
+config-url=  $${dream_platform_parameter:url}
+
+
 [grunt_watch]
 recipe = slapos.cookbook:wrapper
 command-line = bash -c 'cd ${dream-repository.git:location}; PATH=${nodejs:location}/bin/:$PATH ${dream-repository.git:location}/node_modules/grunt-cli/bin/grunt watch -f > $${directory:log}/grunt.log'
@@ -63,5 +74,5 @@ log = $${:var}/log
 
 [publish-connection-parameter]
 recipe = slapos.cookbook:publishurl
-url = http://[$${dream_platform_parameter:host}]:$${dream_platform_parameter:port}
+url = $${dream_platform_parameter:url}
 

software/dream/software.cfg

@@ -4,17 +4,18 @@ extends =
   ../../stack/slapos.cfg
   ../../stack/nodejs.cfg
   ../../component/manpy/buildout.cfg
+  ../../stack/monitor/buildout.cfg
+  ./buildout.hash.cfg
 parts =
   slapos-cookbook
   manpy
   dream_testrunner
-  dream_interpreter
   npm_install
   instance
 
 [instance]
 recipe = slapos.recipe.template
-url = ${:_profile_base_location_}/instance.cfg
+url = ${:_profile_base_location_}/${:filename}
 output = ${buildout:directory}/instance.cfg
 
 [dream_testrunner]
@@ -24,17 +25,10 @@ script = dream_testrunner
 initialization =
   ${manpy:initialization}
 
-[dream_interpreter]
-recipe = zc.recipe.egg
-eggs = ${manpy:eggs}
-interpreter = dream_interpreter
-initialization =
-  ${manpy:initialization}
-
 [npm_install]
 recipe = plone.recipe.command
 stop-on-error = true
-command = cd ${dream-repository.git:location} && PATH=${git:location}/bin/:$PATH ${nodejs:location}/bin/npm install .
+command = cd ${dream-repository.git:location} && PATH=${git:location}/bin/:${nodejs:location}/bin/:$PATH ${nodejs:location}/bin/npm install .
 update_command = ${:command}
 
 [versions]
@@ -42,7 +36,6 @@ rpy2 = 2.4.0
 pydot = 1.0.28
 xlrd = 0.9.3
 xlwt = 0.7.5
-pyparsing = 2.0.3
 numpy = 1.16.4
 scipy = 0.13.3
 simpy = 3.0.5

software/dream/test/README.md

+Tests for DREAM software release

software/dream/test/setup.py

+##############################################################################
+#
+# Copyright (c) 2018 Nexedi SA and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+from setuptools import setup, find_packages
+
+version = '0.0.1.dev0'
+name = 'slapos.test.dream'
+with open("README.md") as f:
+  long_description = f.read()
+
+setup(name=name,
+      version=version,
+      description="Test for SlapOS' DREAM",
+      long_description=long_description,
+      long_description_content_type='text/markdown',
+      maintainer="Nexedi",
+      maintainer_email="info@nexedi.com",
+      url="https://lab.nexedi.com/nexedi/slapos",
+      packages=find_packages(),
+      install_requires=[
+        'slapos.core',
+        'slapos.cookbook',
+        'slapos.libnetworkcache',
+        'requests'
+        ],
+      zip_safe=True,
+      test_suite='test',
+    )

slapos/recipe/postgres/backup.py → software/dream/test/test.py

 ##############################################################################
+# coding: utf-8
 #
-# Copyright (c) 2010 Vifib SARL and Contributors. All Rights Reserved.
+# Copyright (c) 2018 Nexedi SA and Contributors. All Rights Reserved.
 #
 # WARNING: This program as such is intended to be used by professional
 # programmers who take the whole responsibility of assessing all potential
@@ -25,91 +26,24 @@
 #
 ##############################################################################
 
-import textwrap
+import os
 
-from slapos.recipe.librecipe import GenericBaseRecipe
+import requests
 
+from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
 
 
-class ExportRecipe(GenericBaseRecipe):
-    """\
-    This recipe creates an exporter script for using with the resilient stack.
+setUpModule, DREAMTestCase = makeModuleSetUpAndTestCaseClass(
+    os.path.abspath(
+        os.path.join(os.path.dirname(__file__), '..', 'software.cfg')))
 
-    Required options:
-        backup-directory
-            folder that will contain the dump file.
-        bin
-            path to the 'pg_dump' binary.
-        dbname
-            name of the database to dump.
-        pgdata-directory
-            path to postgres configuration and data.
-        wrapper
-            full path of the exporter script to create.
-    """
-
-    def install(self):
-        wrapper = self.options['wrapper']
-        self.createBackupScript(wrapper)
-        return [wrapper]
-
-
-    def createBackupScript(self, wrapper):
-        """\
-        Create a script to backup the database in 'custom' format.
-        """
-        content = textwrap.dedent("""\
-                #!/bin/sh
-                umask 077
-                %(bin)s/pg_dump \\
-                        --host=%(pgdata-directory)s \\
-                        --username postgres \\
-                        --format=custom \\
-                        --file=%(backup-directory)s/database.dump \\
-                        %(dbname)s
-                """ % self.options)
-        self.createExecutable(wrapper, content=content)
-
-
-
-class ImportRecipe(GenericBaseRecipe):
-    """\
-    This recipe creates an importer script for using with the resilient stack.
-
-    Required options:
-        backup-directory
-            folder that contains the dump file.
-        bin
-            path to the 'pg_restore' binary.
-        dbname
-            name of the database to restore.
-        pgdata-directory
-            path to postgres configuration and data.
-        wrapper
-            full path of the importer script to create.
-    """
-
-    def install(self):
-        wrapper = self.options['wrapper']
-        self.createRestoreScript(wrapper)
-        return [wrapper]
-
-
-    def createRestoreScript(self, wrapper):
-        """\
-        Create a script to restore the database from 'custom' format.
-        """
-        content = textwrap.dedent("""\
-                #!/bin/sh
-                %(bin)s/pg_restore \\
-                        --host=%(pgdata-directory)s \\
-                        --username postgres \\
-                        --dbname=%(dbname)s \\
-                        --clean \\
-                        --no-owner \\
-                        --no-acl \\
-                        %(backup-directory)s/database.dump
-                """ % self.options)
-        self.createExecutable(wrapper, content=content)
 
+class TestHTTPAccess(DREAMTestCase):
+  def setUp(self):
+    self.url = self.computer_partition.getConnectionParameterDict()["url"]
 
+  def test(self):
+    self.assertEqual(
+        requests.codes.ok,
+        requests.get(self.url, verify=False).status_code
+    )

software/erp5testnode/software.cfg

@@ -14,9 +14,9 @@ extends =
   ../../stack/monitor/buildout.cfg
 
 parts =
+  slapos-command
   slapos-cookbook
   template
-  lxml-python
   eggs
   zip
   git
@@ -25,27 +25,8 @@ parts =
 [eggs]
 recipe = zc.recipe.egg
 eggs =
+  erp5.util[testnode]
   ${lxml-python:egg}
-  zc.buildout
-  slapos.libnetworkcache
-  slapos.core
-  slapos.recipe.template
-  supervisor
-  jsonschema
-  hexagonit.recipe.download
-  lock_file
-  pytz
-  erp5.util
-  PyXML
-  z3c.etestbrowser
-
-[testnode]
-
-scripts =
-  testnode = erp5.util.testnode:main
-  slapgrid-cp = slapos.grid.slapgrid:runComputerPartition
-  slapgrid-sr = slapos.grid.slapgrid:runSoftwareRelease
-  slapproxy = slapos.proxy:main
 
 
 [template]

software/gateone/software.cfg

@@ -15,7 +15,7 @@ extends =
   ../../component/nginx/buildout.cfg
   ../../stack/slapos.cfg
 
-parts = 
+parts =
   slapos-cookbook
   kerberos
   eggs
@@ -24,7 +24,7 @@ parts =
   dash
   check-recipe
   template
-  
+
 
 [eggs]
 recipe = zc.recipe.egg
@@ -91,7 +91,6 @@ extra-context =
     key template_logrotate_base template-logrotate-base:rendered
     key template_gateone template-gateone:target
     raw gateone_bin ${buildout:bin-directory}/gateone
-    raw python_with_eggs ${buildout:directory}/bin/${extra-eggs:interpreter}
 
 [template-gateone]
 < = download-base

software/grafana/buildout.hash.cfg

@@ -15,7 +15,7 @@
 
 [instance-profile]
 filename = instance.cfg.in
-md5sum = 214a756a6a9f73455411760b2daa25b7
+md5sum = 0edf9e67bc637d73a7415124c8082a8c
 
 [influxdb-config-file]
 filename = influxdb-config-file.cfg.in

software/grafana/instance.cfg.in

@@ -126,6 +126,18 @@ command-line =
   {{ influx_bin }} -username ${influxdb:auth-username} -password ${influxdb:auth-password} -socket ${influxdb:unix-socket} -execute "CREATE USER ${influxdb:auth-username} WITH PASSWORD '${influxdb:auth-password}' WITH ALL PRIVILEGES"
 wrapper-path = ${directory:promise}/${:_buildout_section_name_}
 
+[influxdb-database-ready-promise]
+recipe = slapos.cookbook:wrapper
+command-line =
+  bash -c "{{ influx_bin }} \
+     -username ${influxdb:auth-username} \
+     -password ${influxdb:auth-password} \
+     -host ${influxdb:host} \
+     -port ${influxdb:http-port} \
+     -unsafeSsl \
+     -ssl \
+     -execute 'show databases' | grep '${influxdb:database}'"
+wrapper-path = ${directory:promise}/${:_buildout_section_name_}
 
 
 [grafana]
@@ -261,6 +273,7 @@ recipe =
 instance-promises =
   ${influxdb-listen-promise:path}
   ${influxdb-password-promise:wrapper-path}
+  ${influxdb-database-ready-promise:wrapper-path}
   ${grafana-listen-promise:path}
   ${loki-listen-promise:path}
   ${promtail-listen-promise:path}

software/jstestnode/buildout.hash.cfg

@@ -15,7 +15,7 @@
 
 [instance]
 filename = instance.cfg.in
-md5sum = e6203cba0084289edec2176b185d3427
+md5sum = 42e7d1825f5582fbe2c55977e5abc574
 
 [template-nginx-service]
 filename = template-nginx-service.sh.in

software/jstestnode/instance.cfg.in

@@ -2,6 +2,7 @@
 parts =
   nginx-service
   runTestSuite-instance
+  promises
   publish
 eggs-directory = ${buildout:eggs-directory}
 develop-eggs-directory = ${buildout:develop-eggs-directory}
@@ -22,6 +23,7 @@ log = $${:var}/log
 varnginx = $${:var}/nginx
 scripts = $${:etc}/run
 services = $${:etc}/service
+promise = $${:etc}/promise
 www = $${:srv}/www
 home = $${:etc}/home
 ssl = $${:etc}/ssl
@@ -51,15 +53,20 @@ recipe = slapos.cookbook:wrapper
 wrapper-path = $${directory:services}/$${:_buildout_section_name_}
 command-line =
   ${xserver:location}/bin/Xvfb
-    $${:display}
+    :$${:display}
     -screen 0 1024x768x24
     -fbdir $${directory:framebuffer}
 environment =
   XORG_LOCK_DIR=$${:lock-dir}
 
-display = :0
+display = 0
 lock-dir = $${directory:run}
 
+[xvfb-promise]
+recipe = slapos.cookbook:wrapper
+wrapper-path = $${directory:promise}/$${:_buildout_section_name_}
+command-line = bash -c "[ -S $${xvfb-instance:lock-dir}/.X11-unix/X$${xvfb-instance:display} ]"
+
 
 #################################
 # Nginx service
@@ -85,14 +92,27 @@ ssl_key = $${directory:ssl}/nginx.key
 ssl_csr = $${directory:ssl}/nginx.csr
 ssl_crt = $${directory:ssl}/nginx.crt
 
+[nginx-listen-promise]
+recipe = slapos.cookbook:check_port_listening
+path = $${directory:promise}/$${:_buildout_section_name_}
+hostname = $${nginx-configuration:ip}
+port = $${nginx-configuration:port}
+
+
 #################################
 # SlapOS service
 #################################
 [instance-parameter]
 recipe = slapos.cookbook:slapconfiguration
-computer = $${slap_connection:computer_id}
-partition = $${slap_connection:partition_id}
-url = $${slap_connection:server_url}
-key = $${slap_connection:key_file}
-cert = $${slap_connection:cert_file}
-configuration._ = {}
+computer = $${slap-connection:computer-id}
+partition = $${slap-connection:partition-id}
+url = $${slap-connection:server-url}
+key = $${slap-connection:key-file}
+cert = $${slap-connection:cert-file}
+configuration._ = {}
\ No newline at end of file
+
+[promises]
+recipe =
+depends =
+  $${nginx-listen-promise:recipe}
+  $${xvfb-promise:recipe}
\ No newline at end of file

software/jupyter/software.cfg

@@ -63,6 +63,7 @@ context =
 Pygments = 2.2.0
 astor = 0.5
 backports-abc = 0.5
+backports.functools-lru-cache = 1.6.1
 backports.shutil-get-terminal-size = 1.0.0
 cycler = 0.10.0
 ipykernel = 4.5.2
@@ -73,7 +74,7 @@ jupyter-client = 5.0.0
 jupyter-core = 4.3.0
 jupyterlab = 0.26.3
 jupyterlab-launcher = 0.3.1
-matplotlib = 2.0.1
+matplotlib = 2.1.2
 mistune = 0.7.3
 nbformat = 4.3.0
 notebook = 4.4.1
@@ -121,7 +122,7 @@ pexpect = 4.2.1
 pickleshare = 0.7.4
 
 # Required by:
-# matplotlib==2.0.0
+# matplotlib==2.1.2
 # pandas==0.19.2
 python-dateutil = 2.6.0
 
@@ -138,7 +139,7 @@ scipy = 0.19.0
 singledispatch = 3.4.0.3
 
 # Required by:
-# matplotlib==2.0.0
+# matplotlib==2.1.2
 subprocess32 = 3.2.7
 
 # Required by:
@@ -152,7 +153,6 @@ jupyter-console = 5.1.0
 # jupyter==1.0.0
 qtconsole = 4.3.0
 
-Cython = 0.28.2
 et-xmlfile = 1.0.1
 h5py = 2.7.1
 mpmath = 1.0.0

software/kvm/buildout.hash.cfg

@@ -15,7 +15,7 @@
 
 [template]
 filename = instance.cfg.in
-md5sum = cad7d490b84a484b089e34b27a5675ff
+md5sum = 763f66b915839a9b073fb93f43a44493
 
 [template-kvm]
 filename = instance-kvm.cfg.jinja2

software/kvm/instance.cfg.in

@@ -26,10 +26,10 @@ RootSoftwareInstance = $${:default}
 
 # XXX - If this configuration is not generated by slapgrid, use empty values
 [storage-configuration]
-storage-home = 
-  
+storage-home =
+
 [network-information]
-global-ipv4-network = 
+global-ipv4-network =
 
 [slap-configuration]
 recipe = slapos.cookbook:slapconfiguration.serialised
@@ -87,7 +87,7 @@ extra-context =
   raw novnc_location ${noVNC:location}
   raw netcat_bin ${netcat:location}/bin/netcat
   raw python_executable ${buildout:executable}
-  raw python_eggs_executable ${buildout:bin-directory}/${extra-eggs:interpreter}
+  raw python_eggs_executable ${buildout:bin-directory}/${python-with-eggs:interpreter}
   raw qemu_executable_location ${kvm:location}/bin/qemu-system-x86_64
   raw qemu_img_executable_location ${kvm:location}/bin/qemu-img
   raw qemu_start_promise_tpl ${template-qemu-ready:location}/${template-qemu-ready:filename}

software/kvm/software.cfg

@@ -8,6 +8,7 @@ extends =
   ../../component/noVNC/buildout.cfg
   ../../component/openssl/buildout.cfg
   ../../component/netcat/buildout.cfg
+  ../../component/lxml-python/buildout.cfg
   ../../component/pycurl/buildout.cfg
   ../../component/gzip/buildout.cfg
   ../../stack/slapos.cfg
@@ -40,9 +41,12 @@ parts = ${:common-parts}
 
 #XXX-Cedric : add list of keyboard layouts (azerty/us querty/...) parameter to qemu
 
-[extra-eggs]
-eggs +=
+[python-with-eggs]
+recipe = zc.recipe.egg
+interpreter = ${:_buildout_section_name_}
+eggs =
   ${python-cffi:egg}
+  ${lxml-python:egg}
   websockify
   slapos.cookbook
   erp5.util

software/neoppod/stress-testing/software.cfg

@@ -49,6 +49,8 @@ context =
 eggs +=
   zc.buildout
   slapos.libnetworkcache
+[slapos-cookbook]
+scripts =
 
 # Just in case one wants to inspect logs on the testnode itself.
 [neolog]

software/nginx-push-stream/software.cfg

 [buildout]
-extends = 
+extends =
   ../../stack/slapos.cfg
   ../../component/dash/buildout.cfg
   ../../component/nginx/buildout.cfg
 
 parts =
   slapos-cookbook
+  slapos-toolbox
   nginx-push-stream-module
   nginx-push-stream
   template
   template-nginx-service
   template-nginx
 
-[slapos-cookbook]
-eggs +=
-  slapos.toolbox
-
 [template]
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance.cfg.in

software/slapos-master/test/test/__init__.py

@@ -46,3 +46,13 @@ class ERP5InstanceTestCase(SlapOSInstanceTestCase):
     """Return the output paramters from the root partition"""
     return json.loads(
         self.computer_partition.getConnectionParameterDict()['_'])
+
+  def getComputerPartition(self, partition_reference):
+    for computer_partition in self.slap.computer.getComputerPartitionList():
+      if partition_reference == computer_partition.getInstanceParameter(
+          'instance_title'):
+        return computer_partition
+
+  def getComputerPartitionPath(self, partition_reference):
+    partition_id = self.getComputerPartition(partition_reference).getId()
+    return os.path.join(self.slap._instance_root, partition_id)

software/slapos-sr-testing/software.cfg

@@ -138,6 +138,11 @@ setup = ${slapos-repository:location}/software/gitlab/test/
 egg = slapos.test.cloudooo
 setup = ${slapos-repository:location}/software/cloudooo/test/
 
+[slapos.test.dream-setup]
+<= setup-develop-egg
+egg = slapos.test.dream
+setup = ${slapos-repository:location}/software/dream/test/
+
 [slapos.core-repository]
 <= git-clone-repository
 repository = https://lab.nexedi.com/nexedi/slapos.core.git
@@ -180,6 +185,7 @@ eggs =
   ${slapos.test.nextcloud-setup:egg}
   ${slapos.test.turnserver-setup:egg}
   ${slapos.test.cloudooo-setup:egg}
+  ${slapos.test.dream-setup:egg}
   ${backports.lzma:egg}
 entry-points =
   runTestSuite=erp5.util.testsuite:runTestSuite
@@ -239,6 +245,7 @@ extra =
   ${slapos.test.grafana-setup:setup}
   ${slapos.test.gitlab-setup:setup}
   ${slapos.test.cloudooo-setup:setup}
+  ${slapos.test.dream-setup:setup}
 
 [versions]
 # slapos.core is used from the clone always

software/slaprunner/buildout.hash.cfg

@@ -18,7 +18,7 @@ md5sum = ec70348dd71b319590a5c5837f3d2e45
 
 [template-runner]
 filename = instance-runner.cfg
-md5sum = dc3a7734b08977f438b35d441a12f628
+md5sum = f342daca2e87f8fa44fd7b6a2a1c22f8
 
 [template-runner-import-script]
 filename = template/runner-import.sh.jinja2

software/slaprunner/instance-runner.cfg

@@ -811,7 +811,7 @@ mode = 0744
 context =
     section supervisord supervisord
     section slaprunner slaprunner
-    raw python_executable ${buildout:bin-directory}/${extra-eggs:interpreter}
+    raw python_executable ${buildout:bin-directory}/${python-with-eggs:interpreter}
 
 [supervisord-wrapper]
 recipe = slapos.cookbook:wrapper

software/slaprunner/software-py3.cfg

@@ -9,7 +9,7 @@ python = python3
 common-parts -=
   rdiff-backup
 
-[extra-eggs]
+[python-with-eggs]
 eggs -=
 # futures is a backport of Py3's concurrent.futures module
   futures

software/slaprunner/software.cfg

@@ -37,6 +37,7 @@ shared-part-list =
 # to avoid versioning issues
 
 common-parts =
+  slapos-command
   slapos-cookbook
   template
   instance-runner-import
@@ -50,6 +51,7 @@ common-parts =
 parts =
   ${:common-parts}
 
+
 [template-base]
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/${:filename}
@@ -141,8 +143,11 @@ filename = slapos-slapuser-script.in
 < = template-base
 output = ${buildout:directory}/buildout-shared-part-list
 
-[extra-eggs]
-eggs +=
+[python-with-eggs]
+recipe = zc.recipe.egg
+interpreter = ${:_buildout_section_name_}
+eggs =
+  ${slapos-toolbox:eggs}
   erp5.util
   lock-file
   slapos.recipe.build
@@ -151,6 +156,14 @@ eggs +=
 # for gunicorn[gthread]
   futures
   supervisor
+# BBB: eggs used as recipe should be kept otherwise sections depending
+# on it can't be uninstalled
+  collective.recipe.shelloutput
+scripts =
+  gunicorn
+  supervisord
+  supervisorctl
+
 
 [versions]
 Flask-Auth = 0.85

software/theia/README.md

@@ -5,15 +5,6 @@ Theia is a cloud (and desktop) IDE https://www.theia-ide.org
 This version comes pre-configured with a few plugins, but does not come with python plugin, to let
 you choose between theia and vscode one.
 
-## Installing vscode extensions
-
-To install vscode's python plugin ( https://code.visualstudio.com/docs/languages/python ):
-
-<key>F1</key> `Deploy plugin by id` and enter `vscode:extension/ms-python.python` and wait for a while.
-Check theia's log files in case of issues.
-
-see https://github.com/theia-ide/theia/wiki/Testing-VS-Code-extensions for more details.
-
 ## jedi
 
 [jedi](https://github.com/davidhalter/jedi) which is used by both thiea and vscode python plugins has
@@ -23,4 +14,4 @@ in bin directory, with maybe 30 eggs in each scripts, so this makes jedi so slow
 an error occurs parsing these scripts, jedi won't be usable. This issue is tracked in
 https://github.com/davidhalter/jedi/issues/1325
 
-A simple workaround is to create and empty `buildout.cfg` file at the root of project folder.
+A simple workaround is to create and empty `buildout.cfg` file at the root of project folder.
\ No newline at end of file

software/theia/buildout.hash.cfg

@@ -15,12 +15,12 @@
 
 [instance]
 filename = instance.cfg.in
-md5sum = 7c9444fbe8dc8faea67ede2b77e188ed
+md5sum = 21735765808aac82fb91d53341a3c0d6
 
 [yarn.lock]
 filename = yarn.lock
-md5sum = b63a993a13e7c3b16b66c25fac5ac8b9
+md5sum = c2523a5c832f617c374ee621d50d9e52
 
 [python-language-server-requirements.txt]
 filename = python-language-server-requirements.txt
-md5sum = 6c940b7015f45de6e679fc44807e5220
+md5sum = 6db2a484cac19787fecd87fffefa4aa9

software/theia/instance.cfg.in

@@ -17,13 +17,31 @@ recipe = slapos.cookbook:generate.password
 username = node
 bytes = 12
 
+[frontend-instance-certificate]
+recipe = plone.recipe.command
+command =
+  if [ ! -e $${:key-file} ]
+  then
+    ${openssl-output:openssl} req -x509 -nodes -days 3650 \
+      -subj "/C=AA/ST=X/L=X/O=Dis/CN=$${:common-name}" \
+      -newkey rsa:1024 -keyout $${:key-file} \
+      -out $${:cert-file}
+  fi
+update-command = $${:command}
+key-file = $${directory:etc}/$${:_buildout_section_name_}.key
+cert-file = $${directory:etc}/$${:_buildout_section_name_}.crt
+common-name = $${frontend-instance-config:ip}
+location =
+  $${:key-file}
+  $${:cert-file}
+
 [frontend-instance-config]
 recipe = slapos.recipe.template:jinja2
 rendered = $${directory:etc}/$${:_buildout_section_name_}
 template = inline:
-  https://$${:hostname}:$${:port} {
+  :$${:port} {
     bind $${:ip}
-    tls self_signed # TODO
+    tls $${frontend-instance-certificate:cert-file} $${frontend-instance-certificate:key-file}
     log stdout
     errors stderr
     gzip
@@ -57,7 +75,7 @@ ip = $${frontend-instance-config:ip}
 hostname = $${frontend-instance-config:hostname}
 port = $${frontend-instance-config:port}
 pidfile = $${directory:pidfiles}/$${:_buildout_section_name_}.pid
-url = https://$${frontend-instance-password:username}:$${frontend-instance-password:passwd}@$${:hostname}:$${:port}/
+url = https://$${:hostname}:$${:port}/
 
 [frontend-reload]
 recipe = slapos.cookbook:wrapper
@@ -108,10 +126,24 @@ name = $${:_buildout_section_name_}.py
 config-hostname = $${frontend-instance:ip}
 config-port = $${frontend-instance:port}
 
+[apache-frontend]
+<= slap-connection
+recipe = slapos.cookbook:requestoptional
+name = Theia Frontend
+# XXX We have hardcoded SR URL here.
+software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg
+slave = true
+config-url = $${frontend-instance:url}
+config-https-only = true
+config-type = websocket
+config-websocket-path-list = /services
+return = domain secure_access
+
 [publish-connection-parameter]
 recipe = slapos.cookbook:publish
-url = $${frontend-instance:url}
-
+url = $${apache-frontend:connection-secure_access}
+username = $${frontend-instance-password:username}
+password = $${frontend-instance-password:passwd}
 
 [instance-parameter]
 recipe = slapos.cookbook:slapconfiguration

software/theia/python-language-server-requirements.txt

 appdirs==1.4.3
+astroid==2.3.3
 attrs==18.2.0
 black==18.9b0
 Click==7.0
 future==0.17.1
-isort==4.3.4
+isort==4.3.21
 jedi==0.13.2
+lazy-object-proxy==1.4.3
 mccabe==0.6.1
-mypy==0.660
-mypy-extensions==0.4.1
+mypy==0.770
+mypy-extensions==0.4.3
 parso==0.3.2
 pluggy==0.8.1
 pydocstyle==3.0.0
 pyflakes==2.1.0
+pygls==0.8.1
+pylint==2.4.4
 python-jsonrpc-server==0.1.2
 -e git+https://github.com/palantir/python-language-server@50d03d5931d564e9908292ccfa21dd629ee817ba#egg=python_language_server
 rope==0.11.0
 six==1.12.0
 snowballstemmer==1.2.1
 toml==0.10.0
-typed-ast==1.2.0
-yapf==0.28.0
+typed-ast==1.4.1
+typing-extensions==3.7.4.2
+wrapt==1.11.2
+yapf==0.29.0
+zc.buildout.languageserver==0.2.0

software/theia/software.cfg

@@ -24,7 +24,7 @@ parts =
 
 
 [nodejs]
-<= nodejs-10.6.0
+<= nodejs-10.19.0
 
 [yarn]
 # this could become a component, but it needs to be invoked from nodejs explicitly,
@@ -59,14 +59,17 @@ command = ${bash:location}/bin/bash -c "
   export TMPDIR=${:location}/tmp PATH=${nodejs:location}/bin/:${python2.7:location}/bin/:$PATH &&
   mkdir -p ${:location} && \
   mkdir -p \$TMPDIR && \
+  mkdir -p ${:THEIA_DEFAULT_PLUGINS} && \
   cd ${:location} && \
   cp ${package.json:rendered} . &&
   cp ${yarn.lock:output} . &&
   ${yarn:location}/bin/yarn && \
-  ${yarn:location}/bin/yarn theia build"
+  ${yarn:location}/bin/yarn theia build && \
+  ${yarn:location}/bin/yarn theia download:plugins"
 location = ${buildout:parts-directory}/${:_buildout_section_name_}
 stop-on-error = true
 uses = ${yarn.lock:recipe}
+THEIA_DEFAULT_PLUGINS = ${:location}/plugins/
 
 [yarn.lock]
 recipe = slapos.recipe.template
@@ -76,11 +79,11 @@ mode = 0644
 
 [package.json]
 recipe = slapos.recipe.template:jinja2
-# this comes from https://github.com/theia-ide/theia-apps/blob/def6d29a5fb3739aebc17c6a5b67f1b5c16caacd/theia-full-docker/next.package.json
-# but "@theia/php": "next" was removed, because we don't have php/composer component so installation failed
+# this comes from https://github.com/theia-ide/theia-apps/blob/0cc6a3dc2a6dec1b2b4f9572e9d878a67dc81ff5/theia-full-docker/latest.package.json
+# but we pin vscode-json-languageserver to 1.2.2, otherwise 1.2.3 is picked and it seems to be incompatible.*
+# See https://github.com/theia-ide/theia-apps/issues/333
 template =
-  inline:
-    {
+  inline:{
       "private": true,
       "theia": {
           "frontend": {
@@ -110,58 +113,126 @@ template =
           }
       },
       "dependencies": {
-          "@theia/callhierarchy": "next",
-          "@theia/core": "next",
-          "@theia/cpp": "next",
-          "@theia/debug": "next",
-          "@theia/debug-nodejs": "next",
-          "@theia/docker": "next",
-          "@theia/editor": "next",
-          "@theia/editorconfig": "next",
-          "@theia/editor-preview": "next",
-          "@theia/file-search": "next",
-          "@theia/filesystem": "next",
-          "@theia/git": "next",
-          "@theia/go": "next",
-          "@theia/getting-started": "next",
-          "@theia/java": "next",
-          "@theia/json": "next",
-          "@theia/keymaps": "next",
-          "@theia/languages": "next",
-          "@theia/markers": "next",
-          "@theia/merge-conflicts": "next",
-          "@theia/messages": "next",
-          "@theia/metrics": "next",
-          "@theia/mini-browser": "next",
-          "@theia/monaco": "next",
-          "@theia/navigator": "next",
-          "@theia/outline-view": "next",
-          "@theia/output": "next",
-          "@theia/plantuml": "next",
-          "@theia/plugin": "next",
-          "@theia/plugin-ext": "next",
-          "@theia/plugin-ext-vscode": "next",
-          "@theia/preferences": "next",
-          "@theia/preview": "next",
-          "@theia/process": "next",
-          "@theia/ruby": "next",
-          "@theia/rust": "next",
-          "@theia/search-in-workspace": "next",
-          "@theia/task": "next",
-          "@theia/terminal": "next",
-          "@theia/textmate-grammars": "next",
-          "@theia/tslint": "next",
-          "@theia/typescript": "next",
-          "@theia/userstorage": "next",
-          "@theia/variable-resolver": "next",
-          "@theia/workspace": "next",
-          "theia-yang-extension": "next",
-          "typescript": "latest"
+          "@theia/callhierarchy": "latest",
+          "@theia/core": "latest",
+          "@theia/cpp": "latest",
+          "@theia/debug": "latest",
+          "@theia/editor": "latest",
+          "@theia/editor-preview": "latest",
+          "@theia/file-search": "latest",
+          "@theia/filesystem": "latest",
+          "@theia/getting-started": "latest",
+          "@theia/git": "latest",
+          "@theia/json": "latest",
+          "@theia/keymaps": "latest",
+          "@theia/languages": "latest",
+          "@theia/markers": "latest",
+          "@theia/messages": "latest",
+          "@theia/metrics": "latest",
+          "@theia/mini-browser": "latest",
+          "@theia/monaco": "latest",
+          "@theia/navigator": "latest",
+          "@theia/outline-view": "latest",
+          "@theia/output": "latest",
+          "@theia/plantuml": "latest",
+          "@theia/plugin": "latest",
+          "@theia/plugin-ext": "latest",
+          "@theia/plugin-ext-vscode": "latest",
+          "@theia/preferences": "latest",
+          "@theia/preview": "latest",
+          "@theia/process": "latest",
+          "@theia/rust": "latest",
+          "@theia/search-in-workspace": "latest",
+          "@theia/task": "latest",
+          "@theia/terminal": "latest",
+          "@theia/userstorage": "latest",
+          "@theia/variable-resolver": "latest",
+          "@theia/workspace": "latest"
+      },
+      "resolutions": {
+          "vscode-json-languageserver": "1.2.2",
+          "vscode-languageserver-protocol": "3.15.0-next.9",
+          "vscode-languageserver-types": "3.15.0-next.4",
+          "**/vscode-json-languageserver/**/vscode-languageserver": "6.0.0-next.1"
       },
       "devDependencies": {
-          "@theia/cli": "next"
+          "@theia/cli": "latest"
+      },
+      "theiaPluginsDir": "plugins",
+      "theiaPlugins": {
+          "vscode-builtin-bat": "https://github.com/theia-ide/vscode-builtin-extensions/releases/download/v1.39.1-prel/bat-1.39.1-prel.vsix",
+          "vscode-builtin-clojure": "https://github.com/theia-ide/vscode-builtin-extensions/releases/download/v1.39.1-prel/clojure-1.39.1-prel.vsix",
+          "vscode-builtin-coffeescript": "https://github.com/theia-ide/vscode-builtin-extensions/releases/download/v1.39.1-prel/coffeescript-1.39.1-prel.vsix",
+          "vscode-builtin-configuration-editing": "https://github.com/theia-ide/vscode-builtin-extensions/releases/download/v1.39.1-prel/configuration-editing-1.39.1-prel.vsix",
+          "vscode-builtin-cpp": "https://github.com/theia-ide/vscode-builtin-extensions/releases/download/v1.39.1-prel/cpp-1.39.1-prel.vsix",
+          "vscode-builtin-csharp": "https://github.com/theia-ide/vscode-builtin-extensions/releases/download/v1.39.1-prel/csharp-1.39.1-prel.vsix",
+          "vscode-builtin-css": "https://github.com/theia-ide/vscode-builtin-extensions/releases/download/v1.39.1-prel/css-1.39.1-prel.vsix",
+          "vscode-builtin-debug-auto-launch": "https://github.com/theia-ide/vscode-builtin-extensions/releases/download/v1.39.1-prel/debug-auto-launch-1.39.1-prel.vsix",
+          "vscode-builtin-docker": "https://github.com/theia-ide/vscode-builtin-extensions/releases/download/v1.39.1-prel/docker-1.39.1-prel.vsix",
+          "vscode-builtin-emmet": "https://github.com/theia-ide/vscode-builtin-extensions/releases/download/v1.39.1-prel/emmet-1.39.1-prel.vsix",
+          "vscode-builtin-fsharp": "https://github.com/theia-ide/vscode-builtin-extensions/releases/download/v1.39.1-prel/fsharp-1.39.1-prel.vsix",
+          "vscode-builtin-go": "https://github.com/theia-ide/vscode-builtin-extensions/releases/download/v1.39.1-prel/go-1.39.1-prel.vsix",
+          "vscode-builtin-groovy": "https://github.com/theia-ide/vscode-builtin-extensions/releases/download/v1.39.1-prel/groovy-1.39.1-prel.vsix",
+          "vscode-builtin-grunt": "https://github.com/theia-ide/vscode-builtin-extensions/releases/download/v1.39.1-prel/grunt-1.39.1-prel.vsix",
+          "vscode-builtin-gulp": "https://github.com/theia-ide/vscode-builtin-extensions/releases/download/v1.39.1-prel/gulp-1.39.1-prel.vsix",
+          "vscode-builtin-handlebars": "https://github.com/theia-ide/vscode-builtin-extensions/releases/download/v1.39.1-prel/handlebars-1.39.1-prel.vsix",
+          "vscode-builtin-hlsl": "https://github.com/theia-ide/vscode-builtin-extensions/releases/download/v1.39.1-prel/hlsl-1.39.1-prel.vsix",
+          "vscode-builtin-html": "https://github.com/theia-ide/vscode-builtin-extensions/releases/download/v1.39.1-prel/html-1.39.1-prel.vsix",
+          "vscode-builtin-ini": "https://github.com/theia-ide/vscode-builtin-extensions/releases/download/v1.39.1-prel/ini-1.39.1-prel.vsix",
+          "vscode-builtin-jake": "https://github.com/theia-ide/vscode-builtin-extensions/releases/download/v1.39.1-prel/jake-1.39.1-prel.vsix",
+          "vscode-builtin-java": "https://github.com/theia-ide/vscode-builtin-extensions/releases/download/v1.39.1-prel/java-1.39.1-prel.vsix",
+          "vscode-builtin-javascript": "https://github.com/theia-ide/vscode-builtin-extensions/releases/download/v1.39.1-prel/javascript-1.39.1-prel.vsix",
+          "vscode-builtin-json": "https://github.com/theia-ide/vscode-builtin-extensions/releases/download/v1.39.1-prel/json-1.39.1-prel.vsix",
+          "vscode-builtin-less": "https://github.com/theia-ide/vscode-builtin-extensions/releases/download/v1.39.1-prel/less-1.39.1-prel.vsix",
+          "vscode-builtin-log": "https://github.com/theia-ide/vscode-builtin-extensions/releases/download/v1.39.1-prel/log-1.39.1-prel.vsix",
+          "vscode-builtin-lua": "https://github.com/theia-ide/vscode-builtin-extensions/releases/download/v1.39.1-prel/lua-1.39.1-prel.vsix",
+          "vscode-builtin-make": "https://github.com/theia-ide/vscode-builtin-extensions/releases/download/v1.39.1-prel/make-1.39.1-prel.vsix",
+          "vscode-builtin-markdown": "https://github.com/theia-ide/vscode-builtin-extensions/releases/download/v1.39.1-prel/markdown-1.39.1-prel.vsix",
+          "vscode-builtin-merge-conflicts": "https://github.com/theia-ide/vscode-builtin-extensions/releases/download/v1.39.1-prel/merge-conflict-1.39.1-prel.vsix",
+          "vscode-builtin-npm": "https://github.com/theia-ide/vscode-builtin-extensions/releases/download/v1.39.1-prel/npm-1.39.1-prel.vsix",
+          "vscode-builtin-node-debug": "https://github.com/theia-ide/vscode-node-debug/releases/download/v1.35.3/node-debug-1.35.3.vsix",
+          "vscode-builtin-node-debug2": "https://github.com/theia-ide/vscode-node-debug2/releases/download/v1.33.0/node-debug2-1.33.0.vsix",
+          "vscode-builtin-objective-c": "https://github.com/theia-ide/vscode-builtin-extensions/releases/download/v1.39.1-prel/objective-c-1.39.1-prel.vsix",
+          "vscode-builtin-perl": "https://github.com/theia-ide/vscode-builtin-extensions/releases/download/v1.39.1-prel/perl-1.39.1-prel.vsix",
+          "vscode-builtin-powershell": "https://github.com/theia-ide/vscode-builtin-extensions/releases/download/v1.39.1-prel/powershell-1.39.1-prel.vsix",
+          "vscode-builtin-pug": "https://github.com/theia-ide/vscode-builtin-extensions/releases/download/v1.39.1-prel/pug-1.39.1-prel.vsix",
+          "vscode-builtin-python": "https://github.com/theia-ide/vscode-builtin-extensions/releases/download/v1.39.1-prel/python-1.39.1-prel.vsix",
+          "vscode-builtin-r": "https://github.com/theia-ide/vscode-builtin-extensions/releases/download/v1.39.1-prel/r-1.39.1-prel.vsix",
+          "vscode-builtin-razor": "https://github.com/theia-ide/vscode-builtin-extensions/releases/download/v1.39.1-prel/razor-1.39.1-prel.vsix",
+          "vscode-builtin-ruby": "https://github.com/theia-ide/vscode-builtin-extensions/releases/download/v1.39.1-prel/ruby-1.39.1-prel.vsix",
+          "vscode-builtin-rust": "https://github.com/theia-ide/vscode-builtin-extensions/releases/download/v1.39.1-prel/rust-1.39.1-prel.vsix",
+          "vscode-builtin-scss": "https://github.com/theia-ide/vscode-builtin-extensions/releases/download/v1.39.1-prel/scss-1.39.1-prel.vsix",
+          "vscode-builtin-shaderlab": "https://github.com/theia-ide/vscode-builtin-extensions/releases/download/v1.39.1-prel/shaderlab-1.39.1-prel.vsix",
+          "vscode-builtin-shellscript": "https://github.com/theia-ide/vscode-builtin-extensions/releases/download/v1.39.1-prel/shellscript-1.39.1-prel.vsix",
+          "vscode-builtin-sql": "https://github.com/theia-ide/vscode-builtin-extensions/releases/download/v1.39.1-prel/sql-1.39.1-prel.vsix",
+          "vscode-builtin-swift": "https://github.com/theia-ide/vscode-builtin-extensions/releases/download/v1.39.1-prel/swift-1.39.1-prel.vsix",
+          "vscode-builtin-theme-abyss": "https://github.com/theia-ide/vscode-builtin-extensions/releases/download/v1.39.1-prel/theme-abyss-1.39.1-prel.vsix",
+          "vscode-builtin-theme-defaults": "https://github.com/theia-ide/vscode-builtin-extensions/releases/download/v1.39.1-prel/theme-defaults-1.39.1-prel.vsix",
+          "vscode-builtin-theme-kimbie-dark": "https://github.com/theia-ide/vscode-builtin-extensions/releases/download/v1.39.1-prel/theme-kimbie-dark-1.39.1-prel.vsix",
+          "vscode-builtin-theme-monokai": "https://github.com/theia-ide/vscode-builtin-extensions/releases/download/v1.39.1-prel/theme-monokai-1.39.1-prel.vsix",
+          "vscode-builtin-theme-dimmed": "https://github.com/theia-ide/vscode-builtin-extensions/releases/download/v1.39.1-prel/theme-monokai-dimmed-1.39.1-prel.vsix",
+          "vscode-builtin-theme-quietlight": "https://github.com/theia-ide/vscode-builtin-extensions/releases/download/v1.39.1-prel/theme-quietlight-1.39.1-prel.vsix",
+          "vscode-builtin-theme-red": "https://github.com/theia-ide/vscode-builtin-extensions/releases/download/v1.39.1-prel/theme-red-1.39.1-prel.vsix",
+          "vscode-builtin-theme-solarized-dark": "https://github.com/theia-ide/vscode-builtin-extensions/releases/download/v1.39.1-prel/theme-solarized-dark-1.39.1-prel.vsix",
+          "vscode-builtin-theme-tomorrow-night-blue": "https://github.com/theia-ide/vscode-builtin-extensions/releases/download/v1.39.1-prel/theme-tomorrow-night-blue-1.39.1-prel.vsix",
+          "vscode-builtin-typescript": "https://github.com/theia-ide/vscode-builtin-extensions/releases/download/v1.39.1-prel/typescript-1.39.1-prel.vsix",
+          "vscode-builtin-typescript-language-features": "https://github.com/theia-ide/vscode-builtin-extensions/releases/download/v1.39.1-prel/typescript-language-features-1.39.1-prel.vsix",
+          "vscode-builtin-vb": "https://github.com/theia-ide/vscode-builtin-extensions/releases/download/v1.39.1-prel/vb-1.39.1-prel.vsix",
+          "vscode-builtin-icon-theme-seti": "https://github.com/theia-ide/vscode-builtin-extensions/releases/download/v1.39.1-prel/vscode-theme-seti-1.39.1-prel.vsix",
+          "vscode-builtin-xml": "https://github.com/theia-ide/vscode-builtin-extensions/releases/download/v1.39.1-prel/xml-1.39.1-prel.vsix",
+          "vscode-builtin-yaml": "https://github.com/theia-ide/vscode-builtin-extensions/releases/download/v1.39.1-prel/yaml-1.39.1-prel.vsix",
+          "vscode-editorconfig": "https://github.com/theia-ide/editorconfig-vscode/releases/download/v0.14.4/EditorConfig-0.14.4.vsix",
+          "vscode-eslint": "https://github.com/theia-ide/vscode-eslint/releases/download/release%2F2.0.15/vscode-eslint-2.0.15.vsix",
+          "vscode-go": "https://github.com/microsoft/vscode-go/releases/download/0.12.0/Go-0.12.0.vsix",
+          "vscode-java-debug": "https://github.com/microsoft/vscode-java-debug/releases/download/0.24.0/vscjava.vscode-java-debug-0.24.0.vsix",
+          "vscode-java-dependency-viewer": "https://github.com/microsoft/vscode-java-dependency/releases/download/0.6.0/vscode-java-dependency-0.6.0.vsix",
+          "vscode-java-redhat": "https://github.com/redhat-developer/vscode-java/releases/download/v0.54.2/redhat.java-0.54.2.vsix",
+          "vscode-java-test": "https://github.com/microsoft/vscode-java-test/releases/download/0.22.0/vscjava.vscode-java-test-0.22.0.vsix",
+          "vscode-python": "https://github.com/microsoft/vscode-python/releases/download/2020.1.58038/ms-python-release.vsix",
+          "vscode-ruby": "https://github.com/rubyide/vscode-ruby/releases/download/v0.25.0/ruby-0.25.0.vsix",
+          "vscode-zc-buildout": "https://github.com/perrinjerome/vscode-zc-buildout/releases/download/v0.2.0/vscode-zc-buildout-0.2.0.vsix"
       }
-  }
+    }
 rendered = ${buildout:directory}/${:_buildout_section_name_}
 mode = 0644
 
@@ -198,6 +269,7 @@ template =
   #!/bin/bash
   export PATH=${nodejs:location}/bin/:${python-language-server:location}/bin/:${bash:location}/bin/:${git:location}/bin/:$PATH
   . ${gowork:env.sh}
+  export THEIA_DEFAULT_PLUGINS="local-dir:${theia:THEIA_DEFAULT_PLUGINS}"
   export SHELL=bash
   # reset PS1 from gowork
   export PS1='$ '

software/theia/test/test.py

@@ -44,19 +44,20 @@ setUpModule, SlapOSInstanceTestCase = makeModuleSetUpAndTestCaseClass(
 
 class TestTheia(SlapOSInstanceTestCase):
   def setUp(self):
-    self.theia_url = self.computer_partition.getConnectionParameterDict(
-    )['url']
+    self.connection_parameters = self.computer_partition.getConnectionParameterDict()
 
   def test_http_get(self):
-    resp = requests.get(self.theia_url, verify=False)
-    self.assertEqual(requests.codes.ok, resp.status_code)
+    resp = requests.get(self.connection_parameters['url'], verify=False)
+    self.assertEqual(requests.codes.unauthorized, resp.status_code)
 
-    # without login/password, this is unauthorized
-    parsed_url = urlparse(self.theia_url)
+    # with login/password, this is allowed
+    parsed_url = urlparse(self.connection_parameters['url'])
     resp = requests.get(
         parsed_url._replace(
-            netloc='[{}]:{}'.format(
+            netloc='{}:{}@[{}]:{}'.format(
+                self.connection_parameters['username'],
+                self.connection_parameters['password'],
                 parsed_url.hostname,
                 parsed_url.port)).geturl(),
         verify=False)
-    self.assertEqual(requests.codes.unauthorized, resp.status_code)
+    self.assertEqual(requests.codes.ok, resp.status_code)

stack/erp5/buildout.cfg

@@ -732,7 +732,6 @@ zope.app.dependable = 3.5.1
 # five.formlib==1.0.4
 zope.app.form = 4.0.2
 
-Cython = 0.26.1
 et-xmlfile = 1.0.1
 more-itertools = 5.0.0
 h5py = 2.7.1
@@ -781,7 +780,3 @@ beautifulsoup4 = 4.8.2
 # WSGIProxy2==0.4.6
 WebOb = 1.8.5
 soupsieve = 1.9.5
-
-# Required by:
-# soupsieve==1.9.5
-backports.functools-lru-cache = 1.6.1

stack/monitor/buildout.cfg

@@ -7,13 +7,13 @@ extends =
   ../../component/dash/buildout.cfg
   ../../component/openssl/buildout.cfg
   ../../component/lxml-python/buildout.cfg
-  ../../component/pycurl/buildout.cfg
   ../../component/python-cryptography/buildout.cfg
   ../../stack/logrotate/buildout.cfg
+  ../../stack/slapos.cfg
 
 parts =
   slapos-cookbook
-  extra-eggs
+  monitor-eggs
   monitor2-template
 
 [monitor-download-base]
@@ -30,25 +30,17 @@ destination = ${buildout:parts-directory}/monitor-scripts
 on-update = true
 
 [monitor-eggs]
-# XXX: merge with extra-eggs?
+recipe = zc.recipe.egg
+interpreter = monitor-pythonwitheggs
 eggs =
-  ${lxml-python:egg}
-  ${pycurl:egg}
-  ${python-cryptography:egg}
   plone.recipe.command
   collective.recipe.template
   cns.recipe.symlink
-  slapos.toolbox
-  slapos.core
-
-[extra-eggs]
-recipe = zc.recipe.egg
-interpreter = pythonwitheggs
-eggs =
-  ${monitor-eggs:eggs}
   psutil
   PyRSS2Gen
   Jinja2
+depends =
+  ${slapos-toolbox:recipe}
 
 
 # Monitor templates files
@@ -82,12 +74,12 @@ rendered = ${buildout:directory}/template-monitor.cfg
 context =
     key apache_location apache:location
     key template_logrotate_base template-logrotate-base:rendered
-    raw monitor_bin ${buildout:directory}/bin/monitor.bootstrap
-    raw monitor_collect ${buildout:directory}/bin/monitor.collect
-    raw monitor_statistic ${buildout:directory}/bin/monitor.statistic
-    raw monitor_runpromise ${buildout:directory}/bin/monitor.runpromise
-    raw monitor_genstatus ${buildout:directory}/bin/monitor.genstatus
-    raw monitor_configwrite ${buildout:directory}/bin/monitor.configwrite
+    raw monitor_bin ${buildout:bin-directory}/monitor.bootstrap
+    raw monitor_collect ${buildout:bin-directory}/monitor.collect
+    raw monitor_statistic ${buildout:bin-directory}/monitor.statistic
+    raw monitor_runpromise ${buildout:bin-directory}/monitor.runpromise
+    raw monitor_genstatus ${buildout:bin-directory}/monitor.genstatus
+    raw monitor_configwrite ${buildout:bin-directory}/monitor.configwrite
     raw monitor_conf_template ${monitor-conf:location}/${monitor-conf:filename}
     raw monitor_https_cors ${monitor-httpd-cors:location}/${monitor-httpd-cors:filename}
     raw curl_executable_location ${curl:location}/bin/curl
@@ -97,9 +89,9 @@ context =
     raw monitor_httpd_template ${monitor-httpd-conf:location}/${monitor-httpd-conf:filename}
     raw openssl_executable_location ${openssl:location}/bin/openssl
     raw python_executable ${buildout:executable}
-    raw python_with_eggs ${buildout:directory}/bin/${extra-eggs:interpreter}
+    raw python_with_eggs ${buildout:bin-directory}/${monitor-eggs:interpreter}
     raw template_wrapper ${monitor-template-wrapper:location}/${monitor-template-wrapper:filename}
-    raw check_disk_space ${buildout:directory}/bin/check-free-disk
+    raw check_disk_space ${buildout:bin-directory}/check-free-disk
     raw bin_directory ${buildout:directory}/bin
 
 [versions]

stack/monitor/buildout.hash.cfg

@@ -14,7 +14,7 @@
 # not need these here).
 [monitor2-template]
 filename = instance-monitor.cfg.jinja2.in
-md5sum = e1c7d1c21a942064e2f3f40463b10177
+md5sum = 84bc2cf29e34b48c51116d93e2be7636
 
 [monitor-httpd-conf]
 _update_hash_filename_ = templates/monitor-httpd.conf.in

stack/monitor/instance-monitor.cfg.jinja2.in

@@ -165,8 +165,8 @@ port = ${monitor-instance-parameter:monitor-httpd-port}
 pid-file = ${directory:run}/monitor-httpd.pid
 access-log = ${directory:log}/monitor-httpd-access.log
 error-log = ${directory:log}/monitor-httpd-error.log
-cert-file = ${ca-directory:certs}/httpd.crt
-key-file = ${ca-directory:certs}/httpd.key
+cert-file = ${ca-directory:certs}/monitor-httpd.crt
+key-file = ${ca-directory:certs}/monitor-httpd.key
 htpasswd-file = ${httpd-monitor-htpasswd:htpasswd-path}
 url = https://[${monitor-instance-parameter:monitor-httpd-ipv6}]:${:port}
 httpd-cors-config-file = ${monitor-httpd-cors:rendered}
@@ -194,8 +194,8 @@ recipe = slapos.cookbook:wrapper
 command-line = {{ apache_location }}/bin/httpd -f ${monitor-httpd-conf:rendered} -DFOREGROUND
 wrapper-path = ${directory:bin}/monitor-httpd
 wait-for-files =
-  ${ca-directory:certs}/httpd.key
-  ${ca-directory:certs}/httpd.crt
+  ${monitor-httpd-conf-parameter:key-file}
+  ${monitor-httpd-conf-parameter:cert-file}
   ${monitor-httpd-graceful-wrapper:rendered}
 
 [monitor-httpd-graceful-wrapper]

stack/resilient/README.rst

@@ -33,8 +33,6 @@ The backup data is automatically used to build an historical, incremental archiv
 export
 ------
 
-example:
-    https://lab.nexedi.com/nexedi/slapos/blob/HEAD/stack/lapp/postgres/instance-postgres-export.cfg.in
 
 This is the *active* instance - the one providing live data to the application.
 
@@ -45,7 +43,6 @@ A backup is run via the bin/exporter script: it will
 The pull-backup, upon receiving the notification, will make a copy of the data and transmit it to the 'import' instances.
 
 You should provide the bin/{mysoftware}-exporter script, see for instance
-  https://lab.nexedi.com/nexedi/slapos/blob/HEAD/slapos/recipe/postgres/__init__.py#L207
   https://lab.nexedi.com/nexedi/slapos/blob/1.0.142/slapos/recipe/mydumper.py#L71
 
 By default, as defined in
@@ -57,9 +54,6 @@ the bin/exporter script is run every 60 minutes.
 import
 ------
 
-example:
-    https://lab.nexedi.com/nexedi/slapos/blob/HEAD/stack/lapp/postgres/instance-postgres-import.cfg.in
-
 This is the *fallback* instance - the one that can be activated and thus become active.
 Any number of import instances can be used. Deciding which one should take over can be done manually
 or through a monitoring + election script.
@@ -67,7 +61,6 @@ or through a monitoring + election script.
 
 You should provide the bin/{mysoftware}-importer script, see for instance
 
-  https://lab.nexedi.com/nexedi/slapos/blob/HEAD/slapos/recipe/postgres/__init__.py#L233
   https://lab.nexedi.com/nexedi/slapos/blob/1.0.142/slapos/recipe/mydumper.py#L71
 
 

stack/resilient/buildout.hash.cfg

@@ -46,7 +46,7 @@ md5sum = d21472f0e58f928fb827f2cbf22c4d4a
 
 [resilient-web-takeover-cgi-script-download]
 filename = resilient-web-takeover-cgi-script.py.in
-md5sum = 675ac9e1cf49ccc8f8eddb541a62d899
+md5sum = a7d6ca463e4b6438e1d826f531533a39
 
 [template-wrapper]
 filename = templates/wrapper.in

stack/resilient/resilient-web-takeover-cgi-script.py.in

@@ -41,17 +41,17 @@ def getLatestBackupDate():
   equeue_database_copy = os.path.join(temporary_directory, 'equeue.db')
   shutil.copyfile(equeue_database, equeue_database_copy)
   db = gdbm.open(equeue_database_copy)
-  # Usually, there is only one callback (so only one key
-  # in the db), but if there are several:
-  # Take the "oldest" one (oldest value).
+  # Usually, there is only one callback (so only one key in the db), but if
+  # there are several we take the "newest" one.  Indeed, sometimes the importer
+  # script change name those introducing a new key inside the db.
   db_keys = db.keys()
   if not db_keys:
     result = False
   else:
-    last_backup = db[db_keys[-1]]
+    last_backup = db[db_keys[0]]
     for callback in db_keys:
       timestamp = float(db[callback])
-      if timestamp < last_backup:
+      if timestamp > last_backup:
         last_backup = timestamp
     result = datetime.datetime.fromtimestamp(last_backup)
   db.close()

stack/slapos.cfg

@@ -30,6 +30,8 @@ extends =
   ../component/python-cachecontrol/buildout.cfg
   ../component/python-cryptography/buildout.cfg
   ../component/python-PyYAML/buildout.cfg
+  ../component/pycurl/buildout.cfg
+
 
 # Separate from site eggs
 allowed-eggs-from-site-packages =
@@ -97,6 +99,22 @@ eggs =
   ${python-cryptography:egg}
   pyOpenSSL
   slapos.cookbook
+
+# slapos.toolbox containing utilities
+[slapos-toolbox]
+recipe = zc.recipe.egg
+eggs =
+  ${lxml-python:egg}
+  ${pycurl:egg}
+  ${python-cryptography:egg}
+  slapos.toolbox
+
+# Install a slapos command with networkcache enabled in ${buildout:bin-directory}
+[slapos-command]
+recipe = zc.recipe.egg
+eggs =
+  ${lxml-python:egg}
+  slapos.core
   slapos.libnetworkcache
 
 [versions]
@@ -144,7 +162,7 @@ slapos.extension.shared = 1.0
 slapos.libnetworkcache = 0.20
 slapos.rebootstrap = 4.4
 slapos.recipe.build = 0.44
-slapos.recipe.cmmi = 0.12
+slapos.recipe.cmmi = 0.13
 slapos.toolbox = 0.109
 stevedore = 1.21.0
 subprocess32 = 3.5.3
@@ -219,7 +237,7 @@ pyrsistent = 0.14.5
 ipaddress = 1.0.18
 
 # Required by:
-# slapos.cookbook==1.0.121
+# slapos.cookbook==1.0.143
 jsonschema = 3.0.2
 
 # Required by: