Update Release Candidate

Conflicts:
	software/slapos-master/software.cfg

component/apache/buildout.cfg

@@ -34,9 +34,9 @@ md5sum = 2202b18f269ad606d70e1864857ed93c
 [apache]
 # inspired on http://old.aclark.net/team/aclark/blog/a-lamp-buildout-for-wordpress-and-other-php-apps/
 recipe = slapos.recipe.cmmi
-version = 2.4.27
+version = 2.4.29
 url = https://archive.apache.org/dist/httpd/httpd-${:version}.tar.bz2
-md5sum = 97b6bbfa83c866dbe20ef317e3afd108
+md5sum = 0c599404ef6b69eee95bcd9fcd094407
 pre-configure =
   cp -ar ${apr:location}/apr-${apr:version} srclib/apr/ &&
   cp -ar ${apr-util:location}/apr-util-${apr-util:version} srclib/apr-util

component/aspell/Compile-Fixes-for-GCC-7.patch

+From 8089fa02122fed0a6394eba14bbedcb1d18e2384 Mon Sep 17 00:00:00 2001
+From: Kevin Atkinson <kevina@gnu.org>
+Date: Thu, 29 Dec 2016 00:50:31 -0500
+Subject: Compile Fixes for GCC 7.
+
+Closes #519.
+---
+ modules/filter/tex.cpp | 2 +-
+ prog/check_funs.cpp    | 2 +-
+ 2 files changed, 2 insertions(+), 2 deletions(-)
+
+diff --git a/modules/filter/tex.cpp b/modules/filter/tex.cpp
+index a979539..19ab63c 100644
+--- a/modules/filter/tex.cpp
++++ b/modules/filter/tex.cpp
+@@ -174,7 +174,7 @@ namespace {
+ 
+     if (c == '{') {
+ 
+-      if (top.in_what == Parm || top.in_what == Opt || top.do_check == '\0')
++      if (top.in_what == Parm || top.in_what == Opt || *top.do_check == '\0')
+ 	push_command(Parm);
+ 
+       top.in_what = Parm;
+diff --git a/prog/check_funs.cpp b/prog/check_funs.cpp
+index db54f3d..89ee09d 100644
+--- a/prog/check_funs.cpp
++++ b/prog/check_funs.cpp
+@@ -647,7 +647,7 @@ static void print_truncate(FILE * out, const char * word, int width) {
+     }
+   }
+   if (i == width-1) {
+-    if (word == '\0')
++    if (*word == '\0')
+       put(out,' ');
+     else if (word[len] == '\0')
+       put(out, word, len);
+-- 
+2.15.0.rc0
+

component/aspell/buildout.cfg

@@ -5,6 +5,7 @@ parts =
 
 extends =
   ../ncurses/buildout.cfg
+  ../patch/buildout.cfg
   ../perl/buildout.cfg
 
 [aspell-dictionary-common]
@@ -15,8 +16,11 @@ configure-command = ./configure --vars ASPELL=${aspell:location}/bin/aspell PREZ
 recipe = slapos.recipe.cmmi
 url = http://ftp.gnu.org/gnu/aspell/aspell-0.60.6.1.tar.gz
 md5sum = e66a9c9af6a60dc46134fdacf6ce97d7
+patch-options = -p1
+patches =
+  ${:_profile_base_location_}/Compile-Fixes-for-GCC-7.patch#0159a8841e1241153d2105d157fc8225
 environment =
-  PATH=${perl:location}/bin:%(PATH)s
+  PATH=${patch:location}/bin:${perl:location}/bin:%(PATH)s
   CPPFLAGS=-I${ncurses:location}/include
   LDFLAGS=-L${ncurses:location}/lib -Wl,-rpath=${ncurses:location}/lib
 

component/bazel/bazel-0.4.3.patch

-diff -N -u -r bazel.orig/src/main/java/com/google/devtools/build/lib/rules/SkylarkRuleImplementationFunctions.java bazel/src/main/java/com/google/devtools/build/lib/rules/SkylarkRuleImplementationFunctions.java
---- bazel.orig/src/main/java/com/google/devtools/build/lib/rules/SkylarkRuleImplementationFunctions.java	1980-01-01 00:00:00.000000000 +0900
-+++ bazel/src/main/java/com/google/devtools/build/lib/rules/SkylarkRuleImplementationFunctions.java	2017-01-02 22:19:09.326924214 +0900
-@@ -172,7 +172,7 @@
-       @Param(
-         name = "use_default_shell_env",
-         type = Boolean.class,
--        defaultValue = "False",
-+        defaultValue = "True",
-         named = true,
-         positional = false,
-         doc = "whether the action should use the built in shell environment or not"

component/bazel/bazel-0.6.1.patch

+diff -u -r a/compile.sh b/compile.sh
+--- a/compile.sh	1980-01-01 00:00:00.000000000 +0900
++++ b/compile.sh	2017-10-24 10:57:21.469702614 +0900
+@@ -49,6 +49,8 @@
+ # a chance of overriding this in case they want to do so.
+ : ${VERBOSE:=no}
+ 
++export LDFLAGS=$LDFLAGS
++
+ source scripts/bootstrap/buildenv.sh
+ 
+ mkdir -p output
+diff -u -r a/src/main/java/com/google/devtools/build/lib/analysis/skylark/SkylarkRuleImplementationFunctions.java b/src/main/java/com/google/devtools/build/lib/analysis/skylark/SkylarkRuleImplementationFunctions.java
+--- a/src/main/java/com/google/devtools/build/lib/analysis/skylark/SkylarkRuleImplementationFunctions.java	1980-01-01 00:00:00.000000000 +0900
++++ b/src/main/java/com/google/devtools/build/lib/analysis/skylark/SkylarkRuleImplementationFunctions.java	2017-10-18 17:44:51.975049512 +0900
+@@ -165,7 +165,7 @@
+       @Param(
+         name = "use_default_shell_env",
+         type = Boolean.class,
+-        defaultValue = "False",
++        defaultValue = "True",
+         named = true,
+         positional = false,
+         doc = "whether the action should use the built in shell environment or not."

component/bazel/bazel_src_main_cpp_BUILD.in

+# Description:
+#   The Bazel launcher.
+package(
+    default_visibility = ["//visibility:public"],
+)
+
+WIN_LINK_OPTS = [
+    "-DEFAULTLIB:advapi32.lib",  # GetUserNameW
+    "-DEFAULTLIB:ole32.lib",  # CoTaskMemFree
+    "-DEFAULTLIB:shell32.lib",  # SHGetKnownFolderPath
+    "-DEFAULTLIB:ws2_32.lib",  # grpc
+]
+
+cc_library(
+    name = "blaze_util",
+    srcs = [
+        "blaze_util.cc",
+        "global_variables.h",
+        "startup_options.h",
+    ] + select({
+        "//src:darwin": [
+            "blaze_util_darwin.cc",
+            "blaze_util_posix.cc",
+        ],
+        "//src:darwin_x86_64": [
+            "blaze_util_darwin.cc",
+            "blaze_util_posix.cc",
+        ],
+        "//src:freebsd": [
+            "blaze_util_freebsd.cc",
+            "blaze_util_posix.cc",
+        ],
+        "//src:windows": [
+            "blaze_util_windows.cc",
+        ],
+        "//src:windows_msys": [
+            "blaze_util_windows.cc",
+        ],
+        "//src:windows_msvc": [
+            "blaze_util_windows.cc",
+        ],
+        "//conditions:default": [
+            "blaze_util_linux.cc",
+            "blaze_util_posix.cc",
+        ],
+    }),
+    hdrs = [
+        "blaze_util.h",
+        "blaze_util_platform.h",
+    ],
+    linkopts = select({
+        "//src:darwin": [
+            "-framework CoreFoundation",
+        ],
+        "//src:darwin_x86_64": [
+            "-framework CoreFoundation",
+        ],
+        "//src:freebsd": [
+        ],
+        "//src:windows": WIN_LINK_OPTS,
+        "//src:windows_msvc": WIN_LINK_OPTS,
+        "//conditions:default": [
+            "-lrt",
+            "{{ linkopts }}",
+        ],
+    }),
+    deps = [
+        "//src/main/cpp/util",
+        "//src/main/cpp/util:blaze_exit_code",
+    ] + select({
+        "//src:windows": ["//src/main/native/windows:lib-file"],
+        "//src:windows_msys": ["//src/main/native/windows:lib-file"],
+        "//src:windows_msvc": ["//src/main/native/windows:lib-file"],
+        "//conditions:default": [],
+    }),
+)
+
+cc_binary(
+    name = "client",
+    srcs = [
+        "blaze.cc",
+        "blaze.h",
+        "global_variables.cc",
+        "global_variables.h",
+        "main.cc",
+    ] + select({
+        "//src:windows": ["//src/main/native/windows:resources.o"],
+        "//src:windows_msvc": ["//src/main/native/windows:resources.o"],
+        "//conditions:default": [],
+    }),
+    copts = select({
+        "//src:windows": ["/wd4018"],
+        "//src:windows_msvc": ["/wd4018"],
+        "//conditions:default": ["-Wno-sign-compare"],
+    }),
+    linkopts = select({
+        "//src:darwin": [
+        ],
+        "//src:darwin_x86_64": [
+        ],
+        "//src:freebsd": [
+            "-lprocstat",
+            "-lm",
+        ],
+        "//src:windows_msvc": [
+        ],
+        "//conditions:default": [
+            "-lrt",
+            "-ldl",
+        ],
+    }),
+    visibility = ["//src:__pkg__"],
+    deps = [
+        ":blaze_util",
+        ":option_processor",
+        ":startup_options",
+        ":workspace_layout",
+        "//src/main/cpp/util",
+        "//src/main/cpp/util:errors",
+        "//src/main/cpp/util:logging",
+        "//src/main/cpp/util:strings",
+        "//src/main/protobuf:command_server_cc_proto",
+        "//third_party/ijar:zip",
+    ],
+)
+
+cc_library(
+    name = "option_processor",
+    srcs = ["option_processor.cc"],
+    hdrs = [
+        "option_processor.h",
+        "option_processor-internal.h",
+    ],
+    visibility = [
+        "//src:__pkg__",
+        "//src/test/cpp:__pkg__",
+    ],
+    deps = [
+        ":blaze_util",
+        ":startup_options",
+        ":workspace_layout",
+        "//src/main/cpp/util",
+        "//src/main/cpp/util:blaze_exit_code",
+        "//src/main/cpp/util:logging",
+    ],
+)
+
+cc_library(
+    name = "startup_options",
+    srcs = ["startup_options.cc"],
+    hdrs = ["startup_options.h"],
+    visibility = [
+        "//src:__pkg__",
+        "//src/test/cpp:__pkg__",
+    ],
+    deps = [
+        ":blaze_util",
+        ":workspace_layout",
+        "//src/main/cpp/util",
+        "//src/main/cpp/util:blaze_exit_code",
+        "//src/main/cpp/util:errors",
+    ],
+)
+
+cc_library(
+    name = "workspace_layout",
+    srcs = ["workspace_layout.cc"],
+    hdrs = ["workspace_layout.h"],
+    visibility = [
+        "//src:__pkg__",
+        "//src/test/cpp:__pkg__",
+    ],
+    deps = [
+        ":blaze_util",
+        "//src/main/cpp/util",
+    ],
+)
+
+filegroup(
+    name = "srcs",
+    srcs = glob(["**"]) + ["//src/main/cpp/util:srcs"],
+    visibility = ["//src:__pkg__"],
+)

component/bazel/bazel_src_main_tools_BUILD.in

+package(default_visibility = ["//src:__subpackages__"])
+
+cc_library(
+    name = "logging",
+    srcs = ["logging.cc"],
+    hdrs = ["logging.h"],
+)
+
+cc_library(
+    name = "process-tools",
+    srcs = ["process-tools.cc"],
+    hdrs = ["process-tools.h"],
+    deps = [":logging"],
+)
+
+cc_binary(
+    name = "process-wrapper",
+    srcs = select({
+        "//src:windows": ["process-wrapper-windows.cc"],
+        "//src:windows_msvc": ["process-wrapper-windows.cc"],
+        "//conditions:default": [
+            "process-wrapper.cc",
+            "process-wrapper.h",
+            "process-wrapper-legacy.cc",
+            "process-wrapper-legacy.h",
+            "process-wrapper-options.cc",
+            "process-wrapper-options.h",
+        ],
+    }),
+    linkopts = ["-lm", "{{ linkopts }}"],
+    deps = select({
+        "//src:windows": [],
+        "//src:windows_msvc": [],
+        "//conditions:default": [
+            ":process-tools",
+            ":logging",
+        ],
+    }),
+)
+
+cc_binary(
+    name = "build-runfiles",
+    srcs = select({
+        "//src:windows": ["build-runfiles-windows.cc"],
+        "//src:windows_msvc": ["build-runfiles-windows.cc"],
+        "//conditions:default": ["build-runfiles.cc"],
+    }),
+    linkopts = ["{{ linkopts }}"],
+)
+
+cc_binary(
+    name = "linux-sandbox",
+    srcs = select({
+        "//src:darwin": ["dummy-sandbox.c"],
+        "//src:darwin_x86_64": ["dummy-sandbox.c"],
+        "//src:freebsd": ["dummy-sandbox.c"],
+        "//src:windows": ["dummy-sandbox.c"],
+        "//src:windows_msys": ["dummy-sandbox.c"],
+        "//src:windows_msvc": ["dummy-sandbox.c"],
+        "//conditions:default": [
+            "linux-sandbox.cc",
+            "linux-sandbox.h",
+            "linux-sandbox-options.cc",
+            "linux-sandbox-options.h",
+            "linux-sandbox-pid1.cc",
+            "linux-sandbox-pid1.h",
+        ],
+    }),
+    linkopts = ["-lm", "{{ linkopts }}"],
+    deps = select({
+        "//src:darwin": [],
+        "//src:darwin_x86_64": [],
+        "//src:freebsd": [],
+        "//src:windows": [],
+        "//src:windows_msys": [],
+        "//src:windows_msvc": [],
+        "//conditions:default": [
+            ":logging",
+            ":process-tools",
+        ],
+    }),
+)
+
+filegroup(
+    name = "jdk-support",
+    srcs = [
+        "jdk.BUILD",
+    ],
+)
+
+exports_files([
+    "build_interface_so",
+])
+
+filegroup(
+    name = "srcs",
+    srcs = glob(["**"]),
+    visibility = ["//src:__pkg__"],
+)

component/bazel/bazel_src_tools_singlejar_BUILD.in

+# Description:
+#   singlejar C++ implementation.
+package(default_visibility = ["//src:__subpackages__"])
+
+JAR_TOOL_PATH_COPT_TPL = "-DJAR_TOOL_PATH=\\\"external/local_jdk/bin/jar%s\\\""
+
+JAR_TOOL_PATH_COPTS = select({
+    "//src:windows": [JAR_TOOL_PATH_COPT_TPL % ".exe"],
+    "//src:windows_msvc": [JAR_TOOL_PATH_COPT_TPL % ".exe"],
+    "//src:windows_msys": [JAR_TOOL_PATH_COPT_TPL % ".exe"],
+    "//conditions:default": [JAR_TOOL_PATH_COPT_TPL % ""],
+})
+
+filegroup(
+    name = "srcs",
+    srcs = glob(["**"]),
+    visibility = ["//src:__pkg__"],
+)
+
+filegroup(
+    name = "embedded_tools",
+    srcs = [
+        "BUILD",
+        "combiners.cc",
+        "combiners.h",
+        "diag.h",
+        "input_jar.cc",
+        "input_jar.h",
+        "mapped_file.h",
+        "options.cc",
+        "options.h",
+        "output_jar.cc",
+        "output_jar.h",
+        "singlejar_main.cc",
+        "token_stream.h",
+        "transient_bytes.h",
+        "zip_headers.h",
+        "zlib_interface.h",
+    ],
+    visibility = ["//visibility:public"],
+)
+
+cc_binary(
+    name = "singlejar",
+    srcs = [
+        "singlejar_main.cc",
+    ],
+    linkopts = select({
+        "//src:freebsd": ["-lm"],
+        "//conditions:default": ["{{ linkopts }}"],
+    }),
+    linkstatic = 1,
+    visibility = ["//visibility:public"],
+    deps = [
+        "options",
+        "output_jar",
+        "//third_party/zlib",
+    ],
+)
+
+cc_test(
+    name = "combiners_test",
+    size = "large",
+    srcs = [
+        "combiners_test.cc",
+        ":zip_headers",
+        ":zlib_interface",
+    ],
+    deps = [
+        ":combiners",
+        ":input_jar",
+        "//third_party:gtest",
+        "//third_party/zlib",
+    ],
+)
+
+cc_test(
+    name = "input_jar_empty_jar_test",
+    srcs = [
+        "input_jar_empty_jar_test.cc",
+    ],
+    data = [
+        "data/empty.zip",
+    ],
+    deps = [
+        ":input_jar",
+        ":test_util",
+        "//src/main/cpp/util",
+        "//third_party:gtest",
+    ],
+)
+
+cc_test(
+    name = "input_jar_preambled_test",
+    srcs = [
+        "input_jar_preambled_test.cc",
+    ],
+    data = [
+        ":test1",
+    ],
+    deps = [
+        ":input_jar",
+        ":test_util",
+        "//src/main/cpp/util",
+        "//third_party:gtest",
+    ],
+)
+
+cc_test(
+    name = "input_jar_scan_jartool_test",
+    size = "large",
+    srcs = [
+        "input_jar_scan_entries_test.h",
+        "input_jar_scan_jartool_test.cc",
+    ],
+    copts = JAR_TOOL_PATH_COPTS,
+    data = [
+        "@local_jdk//:jar",
+        "@local_jdk//:jdk",
+    ],
+    # Timing out, see https://github.com/bazelbuild/bazel/issues/1555
+    tags = ["manual"],
+    deps = [
+        ":input_jar",
+        ":test_util",
+        "//third_party:gtest",
+    ],
+)
+
+cc_test(
+    name = "input_jar_scan_ziptool_test",
+    size = "large",
+    srcs = [
+        "input_jar_scan_entries_test.h",
+        "input_jar_scan_ziptool_test.cc",
+    ],
+    # Timing out, see https://github.com/bazelbuild/bazel/issues/1555
+    tags = ["manual"],
+    deps = [
+        ":input_jar",
+        ":test_util",
+        "//third_party:gtest",
+    ],
+)
+
+cc_test(
+    name = "input_jar_bad_jar_test",
+    srcs = [
+        "input_jar_bad_jar_test.cc",
+    ],
+    deps = [
+        ":input_jar",
+        ":test_util",
+        "//third_party:gtest",
+    ],
+)
+
+cc_test(
+    name = "options_test",
+    srcs = [
+        "options.h",
+        "options_test.cc",
+    ],
+    deps = [
+        ":options",
+        ":test_util",
+        ":token_stream",
+        "//src/main/cpp/util",
+        "//third_party:gtest",
+    ],
+)
+
+sh_test(
+    name = "output_jar_bash_test",
+    srcs = ["output_jar_shell_test.sh"],
+    args = [
+        "src/test/shell",
+        "$(location :singlejar)",
+    ],
+    data = [
+        ":singlejar",
+    ],
+    deps = ["//src/test/shell:bashunit"],
+)
+
+cc_test(
+    name = "output_jar_simple_test",
+    srcs = [
+        "output_jar_simple_test.cc",
+    ],
+    copts = JAR_TOOL_PATH_COPTS,
+    data = [
+        ":data1",
+        ":data2",
+        ":stored_jar",
+        ":test1",
+        ":test2",
+        "@local_jdk//:jar",
+        "@local_jdk//:jdk-default",
+    ],
+    deps = [
+        ":input_jar",
+        ":options",
+        ":output_jar",
+        ":test_util",
+        "//src/main/cpp/util",
+        "//third_party:gtest",
+    ],
+)
+
+cc_test(
+    name = "token_stream_test",
+    srcs = [
+        "token_stream_test.cc",
+    ],
+    deps = [
+        ":test_util",
+        ":token_stream",
+        "//third_party:gtest",
+    ],
+)
+
+cc_test(
+    name = "transient_bytes_test",
+    size = "large",
+    srcs = [
+        "transient_bytes_test.cc",
+        ":transient_bytes",
+        ":zlib_interface",
+    ],
+    # Timing out, see https://github.com/bazelbuild/bazel/issues/1555
+    tags = ["manual"],
+    deps = [
+        ":input_jar",
+        ":test_util",
+        "//third_party:gtest",
+        "//third_party/zlib",
+    ],
+)
+
+cc_test(
+    name = "zip_headers_test",
+    size = "small",
+    srcs = [
+        "zip_headers_test.cc",
+        ":zip_headers",
+    ],
+    deps = ["//third_party:gtest"],
+)
+
+cc_test(
+    name = "zlib_interface_test",
+    srcs = [
+        "zlib_interface_test.cc",
+        ":zlib_interface",
+    ],
+    deps = [
+        ":test_util",
+        "//third_party:gtest",
+        "//third_party/zlib",
+    ],
+)
+
+sh_test(
+    name = "zip64_test",
+    srcs = ["zip64_test.sh"],
+    args = [
+        "src/test/shell",
+        "$(location :singlejar)",
+        "$(location @local_jdk//:jar)",
+    ],
+    data = [
+        ":singlejar",
+        "//tools/defaults:jdk",
+        "@local_jdk//:jar",
+    ],
+    deps = ["//src/test/shell:bashunit"],
+)
+
+cc_library(
+    name = "combiners",
+    srcs = [
+        "combiners.cc",
+        ":transient_bytes",
+        ":zip_headers",
+    ],
+    hdrs = ["combiners.h"],
+    deps = ["//third_party/zlib"],
+)
+
+cc_library(
+    name = "input_jar",
+    srcs = [
+        "diag.h",
+        "input_jar.cc",
+        "mapped_file.h",
+    ],
+    hdrs = [
+        "input_jar.h",
+        "zip_headers.h",
+    ],
+)
+
+cc_library(
+    name = "options",
+    srcs = [
+        "diag.h",
+        "options.cc",
+        "options.h",
+    ],
+    hdrs = ["options.h"],
+    deps = [
+        ":token_stream",
+    ],
+)
+
+cc_library(
+    name = "output_jar",
+    srcs = [
+        "diag.h",
+        "mapped_file.h",
+        "output_jar.cc",
+        "output_jar.h",
+        ":zip_headers",
+    ],
+    hdrs = ["output_jar.h"],
+    deps = [
+        ":combiners",
+        ":input_jar",
+        ":options",
+        "//src/main/cpp/util",
+        "//third_party/zlib",
+    ],
+)
+
+cc_library(
+    name = "test_util",
+    srcs = ["test_util.cc"],
+    hdrs = ["test_util.h"],
+    deps = [
+        "//src/main/cpp/util",
+        "//third_party:gtest",
+    ],
+)
+
+cc_library(
+    name = "token_stream",
+    srcs = ["diag.h"],
+    hdrs = ["token_stream.h"],
+)
+
+filegroup(
+    name = "transient_bytes",
+    srcs = [
+        "diag.h",
+        "transient_bytes.h",
+        "zlib_interface.h",
+        ":zip_headers",
+    ],
+)
+
+filegroup(
+    name = "zip_headers",
+    srcs = ["zip_headers.h"],
+)
+
+filegroup(
+    name = "zlib_interface",
+    srcs = [
+        "diag.h",
+        "zlib_interface.h",
+    ],
+)
+
+java_library(
+    name = "test1",
+    resources = [
+        "options.cc",
+        "zip_headers.h",
+        "zlib_interface.h",
+    ],
+)
+
+java_library(
+    name = "test2",
+    resources = [
+        "token_stream.h",
+        "transient_bytes.h",
+    ],
+)
+
+java_library(
+    name = "data1",
+    resources = [
+        "data/extra_file1",
+        "data/extra_file2",
+    ],
+)
+
+java_library(
+    name = "data2",
+    resources = [
+        "data/extra_file1",
+        "data/extra_file3",
+    ],
+)
+
+genrule(
+    name = "stored_jar",
+    srcs = [
+        "output_jar.cc",
+        "//tools/defaults:jdk",
+    ],
+    outs = ["stored.jar"],
+    cmd = "$(location @local_jdk//:jar) -0cf \"$@\" $(location :output_jar.cc)",
+    tools = ["@local_jdk//:jar"],
+)

component/bazel/bazel_tools_cpp_CROSSTOOL.in

@@ -39,6 +39,21 @@ default_toolchain {
   toolchain_identifier: "local_linux"
 }
 
+default_toolchain {
+  cpu: "ppc"
+  toolchain_identifier: "local_linux"
+}
+
+default_toolchain {
+  cpu: "ppc64"
+  toolchain_identifier: "local_linux"
+}
+
+default_toolchain {
+  cpu: "ios_x86_64"
+  toolchain_identifier: "ios_x86_64"
+}
+
 toolchain {
   abi_version: "armeabi-v7a"
   abi_libc_version: "armeabi-v7a"
@@ -90,8 +105,8 @@ toolchain {
   target_system_name: "local"
   toolchain_identifier: "local_linux"
 
-  tool_path { name: "ar" path: "/usr/bin/ar" }
-  tool_path { name: "compat-ld" path: "/usr/bin/ld" }
+  tool_path { name: "ar" path: "{{ binutils_path }}/ar" }
+  tool_path { name: "compat-ld" path: "{{ binutils_path }}/ld" }
   tool_path { name: "cpp" path: "{{ cpp_path }}" }
   tool_path { name: "dwp" path: "/usr/bin/dwp" }
   tool_path { name: "gcc" path: "{{ gcc_path }}" }
@@ -112,14 +127,14 @@ toolchain {
 
   # C(++) compiles invoke the compiler (as that is the one knowing where
   # to find libraries), but we provide LD so other rules can invoke the linker.
-  tool_path { name: "ld" path: "/usr/bin/ld" }
+  tool_path { name: "ld" path: "{{ binutils_path }}/ld" }
 
-  tool_path { name: "nm" path: "/usr/bin/nm" }
-  tool_path { name: "objcopy" path: "/usr/bin/objcopy" }
+  tool_path { name: "nm" path: "{{ binutils_path }}/nm" }
+  tool_path { name: "objcopy" path: "{{ binutils_path }}/objcopy" }
   objcopy_embed_flag: "-I"
   objcopy_embed_flag: "binary"
-  tool_path { name: "objdump" path: "/usr/bin/objdump" }
-  tool_path { name: "strip" path: "/usr/bin/strip" }
+  tool_path { name: "objdump" path: "{{ binutils_path }}/objdump" }
+  tool_path { name: "strip" path: "{{ binutils_path }}/strip" }
 
   # Anticipated future default.
   unfiltered_cxx_flag: "-no-canonical-prefixes"
@@ -160,9 +175,6 @@ toolchain {
   linker_flag: "-no-canonical-prefixes"
   # Have gcc return the exit code from ld.
   linker_flag: "-pass-exit-codes"
-  # Stamp the binary with a unique identifier.
-  linker_flag: "-Wl,--build-id=md5"
-  linker_flag: "-Wl,--hash-style=gnu"
   # Gold linker only? Can we enable this by default?
   # linker_flag: "-Wl,--warn-execstack"
   # linker_flag: "-Wl,--detect-odr-violations"
@@ -215,9 +227,6 @@ toolchain {
   tool_path { name: "dwp" path: "/usr/bin/dwp" }
   tool_path { name: "gcc" path: "osx_cc_wrapper.sh" }
   cxx_flag: "-std=c++0x"
-  ar_flag: "-static"
-  ar_flag: "-s"
-  ar_flag: "-o"
   linker_flag: "-lstdc++"
   linker_flag: "-undefined"
   linker_flag: "dynamic_lookup"
@@ -378,8 +387,6 @@ toolchain {
   # Have gcc return the exit code from ld.
   #linker_flag: "-pass-exit-codes"
   # Stamp the binary with a unique identifier.
-  #linker_flag: "-Wl,--build-id=md5"
-  linker_flag: "-Wl,--hash-style=gnu"
   # Gold linker only? Can we enable this by default?
   # linker_flag: "-Wl,--warn-execstack"
   # linker_flag: "-Wl,--detect-odr-violations"
@@ -618,7 +625,6 @@ toolchain {
   compiler_flag: "-m64"
   compiler_flag: "/D__inline__=__inline"
   # TODO(pcloudy): Review those flags below, they should be defined by cl.exe
-  compiler_flag: "/DOS_WINDOWS=OS_WINDOWS"
   compiler_flag: "/DCOMPILER_MSVC"
 
   # Don't pollute with GDI macros in windows.h.
@@ -664,46 +670,6 @@ toolchain {
   compiler_flag: "/wd4996"
 
   linker_flag: "-m64"
-
-  feature {
-    name: 'include_paths'
-    flag_set {
-      action: 'preprocess-assemble'
-      action: 'c-compile'
-      action: 'c++-compile'
-      action: 'c++-header-parsing'
-      action: 'c++-header-preprocessing'
-      action: 'c++-module-compile'
-      flag_group {
-        flag: '/I%{quote_include_paths}'
-      }
-      flag_group {
-        flag: '/I%{include_paths}'
-      }
-      flag_group {
-        flag: '/I%{system_include_paths}'
-      }
-    }
-  }
-
-  feature {
-    name: 'dependency_file'
-    flag_set {
-      action: 'assemble'
-      action: 'preprocess-assemble'
-      action: 'c-compile'
-      action: 'c++-compile'
-      action: 'c++-module-compile'
-      action: 'c++-header-preprocessing'
-      action: 'c++-header-parsing'
-      expand_if_all_available: 'dependency_file'
-      flag_group {
-        flag: '/DEPENDENCY_FILE'
-        flag: '%{dependency_file}'
-      }
-    }
-  }
-
   # Stop passing -frandom-seed option
   feature {
     name: 'random_seed'
@@ -745,6 +711,10 @@ toolchain {
         flag: '/Fi%{output_preprocess_file}'
       }
     }
+    implies: 'legacy_compile_flags'
+    implies: 'user_compile_flags'
+    implies: 'sysroot'
+    implies: 'unfiltered_compile_flags'
   }
 
   action_config {
@@ -778,6 +748,132 @@ toolchain {
         flag: '/Fi%{output_preprocess_file}'
       }
     }
+    implies: 'legacy_compile_flags'
+    implies: 'user_compile_flags'
+    implies: 'sysroot'
+    implies: 'unfiltered_compile_flags'
+  }
+
+  # TODO(b/65151735): Remove legacy_compile_flags feature when legacy fields are
+  # not used in this crosstool
+  feature {
+    name: 'legacy_compile_flags'
+    flag_set {
+      expand_if_all_available: 'legacy_compile_flags'
+      action: 'assemble'
+      action: 'preprocess-assemble'
+      action: 'c-compile'
+      action: 'c++-compile'
+      action: 'c++-header-parsing'
+      action: 'c++-header-preprocessing'
+      action: 'c++-module-compile'
+      action: 'c++-module-codegen'
+      flag_group {
+        iterate_over: 'legacy_compile_flags'
+        flag: '%{legacy_compile_flags}'
+      }
+    }
+  }
+
+  feature {
+    name: 'include_paths'
+    flag_set {
+      action: 'preprocess-assemble'
+      action: 'c-compile'
+      action: 'c++-compile'
+      action: 'c++-header-parsing'
+      action: 'c++-header-preprocessing'
+      action: 'c++-module-compile'
+      flag_group {
+        iterate_over: 'quote_include_paths'
+        flag: '/I%{quote_include_paths}'
+      }
+      flag_group {
+        iterate_over: 'include_paths'
+        flag: '/I%{include_paths}'
+      }
+      flag_group {
+        iterate_over: 'system_include_paths'
+        flag: '/I%{system_include_paths}'
+      }
+    }
+  }
+
+  feature {
+    name: 'dependency_file'
+    flag_set {
+      action: 'assemble'
+      action: 'preprocess-assemble'
+      action: 'c-compile'
+      action: 'c++-compile'
+      action: 'c++-module-compile'
+      action: 'c++-header-preprocessing'
+      action: 'c++-header-parsing'
+      expand_if_all_available: 'dependency_file'
+      flag_group {
+        flag: '/DEPENDENCY_FILE'
+        flag: '%{dependency_file}'
+      }
+    }
+  }
+
+  feature {
+    name: 'user_compile_flags'
+    flag_set {
+      expand_if_all_available: 'user_compile_flags'
+      action: 'assemble'
+      action: 'preprocess-assemble'
+      action: 'c-compile'
+      action: 'c++-compile'
+      action: 'c++-header-parsing'
+      action: 'c++-header-preprocessing'
+      action: 'c++-module-compile'
+      action: 'c++-module-codegen'
+      flag_group {
+        iterate_over: 'user_compile_flags'
+        flag: '%{user_compile_flags}'
+      }
+    }
+  }
+
+  feature {
+    name: 'sysroot'
+    flag_set {
+      expand_if_all_available: 'sysroot'
+      action: 'assemble'
+      action: 'preprocess-assemble'
+      action: 'c-compile'
+      action: 'c++-compile'
+      action: 'c++-header-parsing'
+      action: 'c++-header-preprocessing'
+      action: 'c++-module-compile'
+      action: 'c++-module-codegen'
+      action: 'c++-link-executable'
+      action: 'c++-link-dynamic-library'
+      flag_group {
+        iterate_over: 'sysroot'
+        flag: '--sysroot=%{sysroot}'
+      }
+    }
+  }
+
+  feature {
+    name: 'unfiltered_compile_flags'
+    flag_set {
+      expand_if_all_available: 'unfiltered_compile_flags'
+      action: 'assemble'
+      action: 'preprocess-assemble'
+      action: 'c-compile'
+      action: 'c++-compile'
+      action: 'c++-header-parsing'
+      action: 'c++-header-preprocessing'
+      action: 'c++-module-compile'
+      action: 'c++-module-codegen'
+      flag_group {
+        iterate_over: 'unfiltered_compile_flags'
+        flag: '%{unfiltered_compile_flags}'
+      }
+    }
   }
 
   compilation_mode_flags {
@@ -805,3 +901,37 @@ toolchain {
     compiler_flag: "-Xcompilation-mode=opt"
   }
 }
+
+# Stub crosstool to allow construction of apple crosstool configuration in a
+# static configuration set.
+# TODO(b/33804121): Remove this once dynamic configurations are in place.
+toolchain {
+  toolchain_identifier: "ios_x86_64"
+  host_system_name: "x86_64-apple-macosx"
+  target_system_name: "x86_64-apple-ios"
+  target_cpu: "ios_x86_64"
+  target_libc: "ios"
+  compiler: "compiler"
+  abi_version: "local"
+  abi_libc_version: "local"
+  supports_gold_linker: false
+  supports_incremental_linker: false
+  supports_fission: false
+  supports_interface_shared_objects: false
+  supports_normalizing_ar: false
+  supports_start_end_lib: false
+
+  tool_path { name: "ar" path: "/bin/false" }
+  tool_path { name: "compat-ld" path: "/bin/false" }
+  tool_path { name: "cpp" path: "/bin/false" }
+  tool_path { name: "dwp" path: "/bin/false" }
+  tool_path { name: "gcc" path: "/bin/false" }
+  tool_path { name: "gcov" path: "/bin/false" }
+  tool_path { name: "ld" path: "/bin/false" }
+
+  tool_path { name: "nm" path: "/bin/false" }
+  tool_path { name: "objcopy" path: "/bin/false" }
+  tool_path { name: "objdump" path: "/bin/false" }
+  tool_path { name: "strip" path: "/bin/false" }
+  linking_mode_flags { mode: DYNAMIC }
+}

component/bazel/bazelrc

+build --verbose_failures

component/bazel/buildout.cfg

@@ -24,36 +24,74 @@ template = ${:_profile_base_location_}/${:filename}.in
 rendered = ${:location}/${:filename}
 cpp_path = ${gcc:location}/bin/cpp
 gcc_path = ${gcc:location}/bin/gcc
+binutils_path = ${binutils:location}/bin
 gconv_path = ${gcc:location}/bin/gconv
 include_path = ${gcc:location}/include
 gcc_lib64_path = ${gcc:location}/lib64
 context =
     key cpp_path template-bazel-crosstool:cpp_path
     key gcc_path template-bazel-crosstool:gcc_path
+    key binutils_path template-bazel-crosstool:binutils_path
     key gconv_path template-bazel-crosstool:gconv_path
     key include_path template-bazel-crosstool:include_path
     key gcc_lib64_path template-bazel-crosstool:gcc_lib64_path
 
+[template-bazel-src-main-cpp-build]
+recipe = slapos.recipe.template:jinja2
+location = ${buildout:parts-directory}/${:_buildout_section_name_}
+mode = 640
+filename = bazel_src_main_cpp_BUILD
+template = ${:_profile_base_location_}/${:filename}.in
+rendered = ${:location}/${:filename}
+linkopts = -Wl,-rpath,${gcc:location}/lib64
+context =
+    key linkopts template-bazel-src-main-cpp-build:linkopts
+
+[template-bazel-src-main-tools-build]
+<= template-bazel-src-main-cpp-build
+filename = bazel_src_main_tools_BUILD
+
+[template-bazel-src-tools-singlejar-build]
+<= template-bazel-src-main-cpp-build
+filename = bazel_src_tools_singlejar_BUILD
+
 [bazel]
 recipe = slapos.recipe.build
-url = https://github.com/bazelbuild/bazel/releases/download/0.4.3/bazel-0.4.3-dist.zip
-md5sum = cbd53f6f59915506da8998dab2098921
+url = https://github.com/bazelbuild/bazel/releases/download/0.6.1/bazel-0.6.1-dist.zip
+md5sum = 8c5c827e33d3ff74c263c1299810b485
 patch-binary = ${patch:location}/bin/patch
-patch-file-path = ${:_profile_base_location_}/bazel-0.4.3.patch
+patch-file-path = ${:_profile_base_location_}/bazel-0.6.1.patch
 bazel-crosstool-modified-file-path = ${template-bazel-crosstool:rendered}
+bazel-src-main-cpp-build-path = ${template-bazel-src-main-cpp-build:rendered}
+bazel-src-main-tools-build-path = ${template-bazel-src-main-tools-build:rendered}
+bazel-src-tools-singlejar-build-path = ${template-bazel-src-tools-singlejar-build:rendered}
 unzip-bin = ${unzip:location}/bin
 zip-bin = ${zip:location}/bin
 gcc-bin = ${gcc:location}/bin
 gcc-lib = ${gcc:location}/lib
 gcc-lib64 = ${gcc:location}/lib64
 java_home = ${zulu:location}
+bazelrc = ${:_profile_base_location_}/bazelrc
 script =
   extract_dir = self.extract(self.download(self.options['url'], self.options['md5sum']))
   crosstool_path = os.path.join(extract_dir, 'tools', 'cpp', 'CROSSTOOL')
   os.chmod(crosstool_path, 0644)
   shutil.copy(self.options['bazel-crosstool-modified-file-path'],
               crosstool_path)
-  target_path = extract_dir+'/src/main/java/com/google/devtools/build/lib/rules/SkylarkRuleImplementationFunctions.java'
+  src_main_cpp_build_path = os.path.join(extract_dir, 'src', 'main', 'cpp', 'BUILD')
+  os.chmod(src_main_cpp_build_path, 0644)
+  shutil.copy(self.options['bazel-src-main-cpp-build-path'],
+              src_main_cpp_build_path)
+  src_main_tools_build_path = os.path.join(extract_dir, 'src', 'main', 'tools', 'BUILD')
+  os.chmod(src_main_tools_build_path, 0644)
+  shutil.copy(self.options['bazel-src-main-tools-build-path'],
+              src_main_tools_build_path)
+  src_tools_singlejar_build_path = os.path.join(extract_dir, 'src', 'tools', 'singlejar', 'BUILD')
+  os.chmod(src_tools_singlejar_build_path, 0644)
+  shutil.copy(self.options['bazel-src-tools-singlejar-build-path'],
+              src_tools_singlejar_build_path)
+
+  target_path = extract_dir+'/src/main/java/com/google/devtools/build/lib/analysis/skylark/SkylarkRuleImplementationFunctions.java'
   os.chmod(target_path, 0644)
   call([self.buildout['bazel']['patch-binary'], '-p1', '-d', extract_dir, '-i', self.buildout['bazel']['patch-file-path']])
   path = ':'.join((
@@ -72,6 +110,8 @@ script =
          'LDFLAGS':'-Wl,-rpath='+self.options['gcc-lib64'],
          'CC':self.options['gcc-bin']+'/gcc',
          'CXX':self.options['gcc-bin']+'/g++',
+         'BAZELRC':self.options['bazelrc'],
+         'VERBOSE':'yes',
         }
   bin_dir = os.path.join(self.options['location'], 'bin')
   os.makedirs(bin_dir)

component/ca-certificates/buildout.cfg

@@ -11,9 +11,9 @@ parts =
 
 [ca-certificates]
 recipe = slapos.recipe.cmmi
-url = http://http.debian.net/debian/pool/main/c/ca-certificates/ca-certificates_20161130+nmu1.tar.xz
+url = http://http.debian.net/debian/pool/main/c/ca-certificates/ca-certificates_20170717.tar.xz
 patch-binary = ${patch:location}/bin/patch
-md5sum = a09e8b63126188fd0ed77f6fbaf5d35f
+md5sum = 55a6bb6b98afb16b3cde8e3ad1e262eb
 patches =
   ${:_profile_base_location_}/ca-certificates-any-python.patch#087b5e860c7a4b8ff6656c95c5835ee2
   ${:_profile_base_location_}/ca-certificates-sbin-dir.patch#0b4e7d82ce768823c01954ee41ef177b

component/chainer/buildout.cfg

@@ -40,7 +40,7 @@ interpreter = chainer-python
 scripts = chainer-python
 
 [versions]
-chainer = 1.22.0
-filelock = 2.0.7
+chainer = 3.0.0
+filelock = 2.0.12
 nose = 1.3.7
-six = 1.10.0
+six = 1.11.0

component/curl/buildout.cfg

@@ -6,14 +6,15 @@ extends =
   ../openssl/buildout.cfg
   ../perl/buildout.cfg
   ../pkgconfig/buildout.cfg
+  ../xz-utils/buildout.cfg
   ../zlib/buildout.cfg
 parts =
   curl
 
 [curl]
 recipe = slapos.recipe.cmmi
-url = http://curl.haxx.se/download/curl-7.54.1.tar.bz2
-md5sum = 6b6eb722f512e7a24855ff084f54fe55
+url = http://curl.haxx.se/download/curl-7.56.0.tar.xz
+md5sum = 18ebc36c5dc9317d4a0b5db94a4e12ad
 configure-options =
   --disable-static
   --disable-ldap
@@ -44,6 +45,6 @@ configure-options =
   --without-nghttp2
 
 environment =
-  PATH=${perl:location}/bin:${pkgconfig:location}/bin:%(PATH)s
+  PATH=${perl:location}/bin:${pkgconfig:location}/bin:${xz-utils:location}/bin:%(PATH)s
   PKG_CONFIG_PATH=${openssl:location}/lib/pkgconfig
   LDFLAGS=-Wl,-rpath=${zlib:location}/lib -Wl,-rpath=${openssl:location}/lib

component/ethtool/buildout.cfg

+# ethtool - query or control network driver and hardware settings
+# https://www.kernel.org/pub/software/network/ethtool
+[buildout]
+parts   = ethtool
+
+[ethtool]
+recipe  = slapos.recipe.cmmi
+url     = https://www.kernel.org/pub/software/network/ethtool/ethtool-4.11.tar.xz
+md5sum  = 16d38f4ebe23e44f96f7d8b38ed3652c

component/ghostscript/buildout.cfg

@@ -19,7 +19,6 @@ configure-options =
   --without-libidn
   --without-x
   --with-drivers=FILES
-  --with-fontpath=${ghostscript-fonts:location}
 # it seems that parallel build sometimes fails for ghostscript.
 make-options = -j1
 environment =
@@ -35,9 +34,3 @@ environment =
 <= ghostscript-common
 url = https://github.com/ArtifexSoftware/ghostpdl-downloads/releases/download/gs920/ghostscript-9.20.tar.xz
 md5sum = 8f3d383d48da22345937b66b01ab2960
-
-[ghostscript-fonts]
-recipe = slapos.recipe.build:download-unpacked
-strip-top-level-dir = false
-url = http://downloads.ghostscript.com/public/fonts/urw-base35-v1.10.zip
-md5sum = 66e8bbd8228519d5dba82b9433a61bb0

component/golang/buildout.cfg

 # Go language - https://golang.org/
 [buildout]
-parts = golang
+extends =
+  ../findutils/buildout.cfg
+  ../git/buildout.cfg
 
-[golang]
-<= golang18
+parts = gowork
 
+# ---- Go builds itself ----
 
 [golang-common]
 recipe = slapos.recipe.cmmi
@@ -12,42 +14,107 @@ configure-command = :
 location = ${buildout:parts-directory}/${:_buildout_section_name_}
 make-binary =
 make-targets= cd src && ./all.bash && cp -alf .. ${:location}
+# some testdata files have an issue with slapos.extension.strip.
+post-install = ${findutils:location}/bin/find ${:location}/src -type d -name testdata -exec rm -rf {} \; || true
 environment =
   GOROOT_FINAL=${:location}
   ${:environment-extra}
 
+[golang14-repository]
+<= git-repository
+repository = https://github.com/golang/go.git
+revision = go1.4.3-16-g4d5426a
 
 [golang14]
 <= golang-common
-url = https://storage.googleapis.com/golang/go1.4-bootstrap-20161024.tar.gz
-md5sum = 76e42c8152e8560ded880a6d1d1f53cb
+path = ${golang14-repository:location}
 environment-extra =
 
 
-[golang15]
+[golang18]
 <= golang-common
-url = https://storage.googleapis.com/golang/go1.5.4.src.tar.gz
-md5sum = a04d570515c46e4935c63605cbd3a04e
+url = https://storage.googleapis.com/golang/go1.8.3.src.tar.gz
+md5sum = 64e9380e07bba907e26a00cf5fcbe77e
 
-# go1.5 needs go1.4 to bootstrap
+# go1.8 needs go1.4 to bootstrap
 environment-extra =
   GOROOT_BOOTSTRAP=${golang14:location}
 
-
-[golang16]
+[golang19]
 <= golang-common
-url = https://storage.googleapis.com/golang/go1.6.3.src.tar.gz
-md5sum = bf3fce6ccaadd310159c9e874220e2a2
+url = https://storage.googleapis.com/golang/go1.9.2.src.tar.gz
+md5sum = 44105c865a1a810464df79233a05a568
 
-# go1.6 needs go1.4 to bootstrap
+# go1.9 needs go1.4 to bootstrap
 environment-extra =
   GOROOT_BOOTSTRAP=${golang14:location}
 
-[golang18]
-<= golang-common
-url = https://storage.googleapis.com/golang/go1.8.src.tar.gz
-md5sum = 7743960c968760437b6e39093cfe6f67
 
-# go1.8 needs go1.4 to bootstrap
-environment-extra =
-  GOROOT_BOOTSTRAP=${golang14:location}
+# ---- infrastructure to build Go workspaces / projects ----
+
+# gowork is a top-level section representing workspace
+#
+# users should add `install` field to [gowork] to describe packages they want to
+# be installed (+ automatically their dependencies are installed too). e.g.
+#
+#   [gowork]
+#   install =
+#       lab.nexedi.com/kirr/neo/go/...  \
+#       github.com/pkg/profile          \
+#       golang.org/x/perf/cmd/benchstat
+[gowork]
+directory = ${buildout:directory}/go.work
+src	= ${:directory}/src
+bin	= ${:directory}/bin
+depends = ${gowork.goinstall:recipe}
+
+# go version used for the workspace (possible to override in applications)
+golang  = ${golang19:location}
+
+# everything is done by dependent parts
+recipe  = plone.recipe.command
+command = :
+
+# env.sh for compiling and running go programs
+[gowork]
+env.sh  = ${gowork-env.sh:output}
+
+[gowork-env.sh]
+recipe	= slapos.recipe.template
+url     = ${:_profile_base_location_}/goenv.sh.in
+output	= ${gowork:directory}/env.sh
+depends = ${gowork.mkdir:recipe}
+md5sum	= a9a265135931b3da53f4392870748264
+
+[gowork.mkdir]
+# NOTE do not use slapos.cookbook:mkdirectory here - if anything in software (not instance)
+# uses slapos.cookbook:* in recipe - slapos.cookbook will get compiled against system
+# libxml/libxslt or fail to bootstrap at all if those are not present.
+recipe  = plone.recipe.command
+command = mkdir -p ${gowork:directory}
+update-command = ${:command}
+stop-on-error = true
+
+# install go packages
+# clients should put package list to install to gowork:install ("..." requests installing everything)
+[gowork.goinstall]
+recipe  = plone.recipe.command
+command = bash -c ". ${gowork:env.sh} && go install -v ${gowork:install}"
+update-command = ${:command}
+stop-on-error = true
+
+
+[git-repository]
+recipe  = slapos.recipe.build:gitclone
+git-executable = ${git:location}/bin/git
+
+# a go package should:
+# 1) <= go-git-package
+# 2) provide go.importpath
+# 3) provide repository (which is not the same as importpath in general case)
+#
+# the list of go packages for a go workspace state can be automatically
+# generated with the help of go-pkg-snapshot tool.
+[go-git-package]
+<= git-repository
+location = ${gowork:src}/${:go.importpath}

component/golang/goenv.sh.in

+# env.sh for a Go workspace
+# Usage: env.sh [/path/to/env.sh]
+
+# ---- 8< ---- (buildout substitution here)
+# PATH so that go & friends work out of the box
+export PATH=${gowork:golang}/bin:${git:location}/bin:${buildout:bin-directory}:$PATH
+X=${gowork:directory}
+# ---- 8< ----
+
+export GOPATH=$X:$GOPATH
+export PATH=$X/bin:$PATH
+export PS1="(`basename $X`) $PS1"
+
+# strip trailing : from $GOPATH
+GOPATH=$${GOPATH%:}

component/golang/gowork-snapshot

+#!/bin/bash
+# gowork-snapshot - find out installed go packages and produce buildout code to install them pinned
+
+# FIXME currently works only with cwd=gowork/src
+
+echo "# Code generated by gowork-snapshot; DO NOT EDIT."
+
+# list installed go git repositories.
+#
+# this gives something like:
+# github.com/cznic/strutil  https://github.com/cznic/strutil    529a34b1c1
+# golang.org/x/net          https://go.googlesource.com/net     1087133bc4
+# ...
+gogit_list() {
+    find . -name .git | sort | \
+    while read repo; do
+        importpath=${repo#./}
+        importpath=${importpath%/.git}
+        echo -ne "${importpath}\t"
+        echo -ne "`git_upstream_url $repo`\t"
+        git -C $repo describe --long --always --abbrev=10
+    done
+}
+
+# git_upstream_url <repo>   - show current branch upstream URL
+git_upstream_url() {
+    repo=$1
+    head="`git -C $repo symbolic-ref --short HEAD`"             # current branch - e.g. "t"
+    remote="`git -C $repo config --get branch.$head.remote`"    # upstream name, e.g. "kirr"
+    url="`git -C $repo config --get remote.$remote.url`"        # upstream URL
+    echo "$url"
+}
+
+# buildout_safe <name>  - canonicalize name to be allowed to be used in buildout section name
+# e.g. aaa/bbb -> aaa_bbb
+#
+# XXX can't use e.g. "go!lab.nexedi.com/kirr/neo" because buildout disallows
+# "!" or "/" in section name references.
+buildout_safe() {
+    # see _simple regex in slapos.buildout
+    echo -n "$1" | tr --complement -- "-a-zA-Z0-9 ._" _
+}
+
+echo
+echo "# list of go git repositories to fetch"
+echo "[gowork.goinstall]"
+echo "depends_gitfetch  ="
+
+gogit_list | \
+while read pkg _ _; do
+        echo "    \${go_`buildout_safe $pkg`:recipe}"
+done
+
+echo
+gogit_list | \
+while read pkg url rev; do
+    echo
+    echo "[go_`buildout_safe $pkg`]"
+    echo "<= go-git-package"
+    echo "go.importpath = $pkg"
+    echo "repository    = $url"
+    echo "revision  = $rev"
+done

component/helloweb/buildout.cfg

@@ -89,7 +89,7 @@ input   = inline:
 recipe  = slapos.recipe.cmmi
 path    = ${helloweb-repository:location}/go/
 
-go      = ${golang18:location}/bin/go
+go      = ${golang19:location}/bin/go
 configure-command = :
 make-binary =
 make-targets= cd ${:path} &&

component/imagemagick/buildout.cfg

@@ -54,7 +54,6 @@ configure-options =
   --without-wmf
   --with-bzlib=${bzip2:location}
   --with-zlib=${zlib:location}
-  --with-gs-font-dir=${ghostscript-fonts:location}
   --with-frozenpaths
 patch-options = -p1
 patches =

component/ioping/buildout.cfg

+# ioping - simple disk I/O latency measuring tool
+# https://github.com/koct9i/ioping
+[buildout]
+extends =
+    ../git/buildout.cfg
+
+parts   =
+    ioping
+
+[ioping-repository]
+recipe  = slapos.recipe.build:gitclone
+git-executable = ${git:location}/bin/git
+# NOTE we use a bit patched ioping version which shows not only avg latency but also its distribution
+# repository = https://github.com/koct9i/ioping.git
+repository = https://lab.nexedi.com/kirr/ioping.git
+revision = v1.0-9-g34c97f7636
+
+[ioping]
+recipe  = slapos.recipe.cmmi
+path    = ${ioping-repository:location}
+configure-command = :
+make-binary =
+# XXX without vvv PREFIX=${:location} does not work
+location= ${buildout:parts-directory}/${:_buildout_section_name_}
+make-targets= make install PREFIX=${:location}

component/keras/buildout.cfg

@@ -25,7 +25,7 @@ interpreter = keras-python
 scripts = keras-python
 
 [versions]
-Keras = 2.0.1
-tensorflow = 1.0.1
+Keras = 2.0.8
+tensorflow = 1.4.0rc1
 h5py = 2.7.0rc2
 Cython = 0.25.2

component/lmbench/buildout.cfg

+# LMbench - Tools for Performance Analysis
+# http://www.bitmover.com/lmbench/
+[buildout]
+extends =
+    ../git/buildout.cfg
+    ../golang/buildout.cfg
+
+parts   =
+    lmbench
+
+
+[lmbench-repository]
+<= git-repository
+# NOTE we use a bit patched lmbench version with fixes to lat_tcp for errors not to go unnoticed and other addons
+# repository = https://svn.code.sf.net/p/lmbench/code
+repository = https://lab.nexedi.com/kirr/lmbench.git
+revision = 9b108b6ff3
+
+[lmbench]
+recipe  = slapos.recipe.cmmi
+path    = ${lmbench-repository:location}
+configure-command = :
+make-binary =
+# XXX without vvv BASE=${:location} does not work
+location= ${buildout:parts-directory}/${:_buildout_section_name_}
+make-targets= cd ${lmbench-repository:location}/lmbench3/src
+              && make install BASE=${:location}
+              && bash -c ". ${gowork:env.sh} && go build -o ${:location}/bin/lat_tcp_go lat_tcp.go"

component/postgresql/buildout.cfg

@@ -35,5 +35,5 @@ environment =
 
 [postgresql92]
 <= postgresql-common
-url = http://ftp.postgresql.org/pub/source/v9.2.18/postgresql-9.2.22.tar.bz2
-md5sum = c5d3fb5229baf9e94ee2287980c55321
+url = http://ftp.postgresql.org/pub/source/v9.2.23/postgresql-9.2.23.tar.bz2
+md5sum = c972e32b7f17dbc652d2462b7690d116

component/protobuf/buildout.cfg

@@ -3,14 +3,14 @@ parts = protobuf
 
 [protobuf]
 recipe = slapos.recipe.cmmi
-url = https://github.com/google/protobuf/releases/download/v3.1.0/protobuf-python-3.1.0.tar.gz
-md5sum = 7a227a21379a2ea08cc5d7ba1fb1ba5b
+url = https://github.com/google/protobuf/releases/download/v3.4.0/protobuf-python-3.4.0.tar.gz
+md5sum = 0820cc2e56d71aef8e99794fcbd184cd
 location = ${buildout:parts-directory}/${:_buildout_section_name_}
 
 [protobuf-cpp]
 recipe = slapos.recipe.cmmi
-url = https://github.com/google/protobuf/releases/download/v3.1.0/protobuf-cpp-3.1.0.tar.gz
-md5sum = bd5e3eed635a8d32e2b99658633815ef
+url = https://github.com/google/protobuf/releases/download/v3.4.0/protobuf-cpp-3.4.0.tar.gz
+md5sum = 6d59dad503bea5ad420fd09ddad84481
 configure-command =
   ./autogen.sh
   ./configure --prefix=${buildout:parts-directory}/${:_buildout_section_name_}

component/python-cocoapi/buildout.cfg

+[buildout]
+extends =
+  ../../stack/slapos.cfg
+  ../gcc/buildout.cfg
+  ../cython/buildout.cfg
+  ../scipy/buildout.cfg
+parts =
+  python-cocoapi-build-install-egg
+
+[python-cocoapi-repository]
+recipe = plone.recipe.command
+stop-on-error = true
+repository = https://github.com/cocodataset/cocoapi.git
+tag = master
+git-binary = ${git:location}/bin/git
+patch-binary = ${patch:location}/bin/patch
+location = ${buildout:parts-directory}/${:_buildout_section_name_}
+command = export HOME=${:location}; (${:git-binary} clone --quiet -b ${:tag} ${:repository} ${:location}; cd ${:location}; ${:patch-binary} -p1 -d . < ${:_profile_base_location_}/setup.py.patch ) || (rm -fr ${:location}; exit 1)
+
+[python-cocoapi-build-interpreter]
+recipe = zc.recipe.egg
+initialization =
+  import scipy.spatial.ckdtree # load our own libstdc++ explicitly at the very beginning
+eggs =
+  setuptools
+  ${cython:egg}
+  ${scipy:egg}
+  ${numpy:egg}
+interpreter = python-cocoapi-build-interpreter
+scripts = python-cocoapi-build-interpreter
+location = ${buildout:parts-directory}/${:_buildout_section_name_}
+
+[python-cocoapi-build]
+recipe = slapos.recipe.build
+location = ${buildout:parts-directory}/${:_buildout_section_name_}
+workdir = ${python-cocoapi-repository:location}/PythonAPI
+python-bin = ${buildout:bin-directory}/${python-cocoapi-build-interpreter:interpreter}
+gcc-location = ${gcc:location}
+script =
+  os.makedirs(location)
+  workdir = self.options['workdir']
+  python_bin = self.options['python-bin']
+  gcc_location = self.options['gcc-location']
+  env = {'PATH':':'.join([gcc_location+'/bin',
+                          os.environ['PATH']]),
+         'CC':gcc_location+'/bin/gcc',
+         'CXX':gcc_location+'/bin/g++',
+         }
+  call([python_bin, 'setup.py', 'build_ext'], cwd=workdir, env=env)
+  call([python_bin, 'setup.py', 'bdist_egg'], cwd=workdir, env=env)
+
+[python-cocoapi-build-install-egg]
+recipe = slapos.recipe.build
+unzip-binary = ${unzip:location}/bin/unzip
+python-cocoapi-repository-path = ${python-cocoapi-repository:location}
+location = ${buildout:parts-directory}/${:_buildout_section_name_}
+need-python-cocoapi-build = ${python-cocoapi-build:location}
+egg = pycocotools
+script =
+  os.makedirs(location)
+  workdir = self.options['python-cocoapi-repository-path']+'/PythonAPI'
+  egg_name = 'pycocotools-2.0-py2.7-linux-x86_64.egg'
+  dist_dir = os.path.join(workdir, 'dist')
+  dest_dir = os.path.join(self.buildout['buildout']['eggs-directory'], egg_name)
+  call([self.options['unzip-binary'], '-o', os.path.join(dist_dir, egg_name), '-d', dest_dir])

component/python-cocoapi/setup.py.patch

+diff --git a/PythonAPI/setup.py b/PythonAPI/setup.py
+index eb3d508..d619b3c 100644
+--- a/PythonAPI/setup.py
++++ b/PythonAPI/setup.py
+@@ -1,4 +1,5 @@
+-from distutils.core import setup
++#from distutils.core import setup
++from setuptools import setup
+ from Cython.Build import cythonize
+ from distutils.extension import Extension
+ import numpy as np
+@@ -21,4 +22,4 @@ setup(name='pycocotools',
+       version='2.0',
+       ext_modules=
+           cythonize(ext_modules)
+-      )
+\ No newline at end of file
++      )

component/pytorch/buildout.cfg

+[buildout]
+extends =
+  ../../stack/slapos.cfg
+  ../gcc/buildout.cfg
+  ../openblas/buildout.cfg
+  ../cmake/buildout.cfg
+  ../python-cffi/buildout.cfg
+  ../python-PyYAML/buildout.cfg
+  ../python-cocoapi/buildout.cfg
+  ../pillow/buildout.cfg
+  ../scipy/buildout.cfg
+  ../matplotlib/buildout.cfg
+  ../unzip/buildout.cfg
+parts =
+  pytorch-egg
+
+[pytorch-repository]
+recipe = plone.recipe.command
+stop-on-error = true
+repository = https://github.com/pytorch/pytorch
+tag = master
+commit = 4af66c43045a317b477918c503d105f565b4a66b
+git-binary = ${git:location}/bin/git
+patch-binary = ${patch:location}/bin/patch
+location = ${buildout:parts-directory}/${:_buildout_section_name_}
+command = export HOME=${:location}; (${:git-binary} clone --recursive --quiet -b ${:tag} ${:repository} ${:location}; cd ${:location}; ${:git-binary} checkout ${:commit}; ${:patch-binary} -p1 -d . < ${:_profile_base_location_}/pytorch.4af66c4.patch ) || (rm -fr ${:location}; exit 1)
+
+[pytorch-build-interpreter]
+recipe = zc.recipe.egg
+initialization =
+  import scipy.spatial.ckdtree # load our own libstdc++ explicitly at the very beginning
+  import sys
+  sys.path.append('.')
+  sys.path.append('${pytorch-repository:location}/torch/lib/ATen')
+eggs =
+  setuptools
+  ${scipy:egg}
+  ${numpy:egg}
+  ${python-cffi:egg}
+  ${python-PyYAML:egg}
+interpreter = pytorch-build-interpreter
+scripts = pytorch-build-interpreter
+location = ${buildout:parts-directory}/${:_buildout_section_name_}
+
+[pytorch-build]
+recipe = slapos.recipe.build
+location = ${buildout:parts-directory}/${:_buildout_section_name_}
+workdir = ${pytorch-repository:location}
+python-bin = ${buildout:bin-directory}/${pytorch-build-interpreter:interpreter}
+no-cuda = 0
+cmake-bin = ${cmake:location}/bin
+git-bin = ${git:location}/bin
+binutils-location = ${binutils:location}
+gcc-location = ${gcc:location}
+openblas-location = ${openblas:location}
+script =
+  os.makedirs(location)
+  workdir = self.options['workdir']
+  python_bin = self.options['python-bin']
+  binutils_location = self.options['binutils-location']
+  gcc_location = self.options['gcc-location']
+  openblas_location = self.options['openblas-location']
+  env = {'PYTHONPATH':workdir,
+         'PATH':':'.join([self.options['cmake-bin'],
+                          self.options['git-bin'],
+                          binutils_location+'/bin',
+                          gcc_location+'/bin',
+                          os.environ['PATH']]),
+         'CMAKE_INCLUDE_PATH':':'.join([gcc_location+'/include',
+                                        binutils_location+'/include',
+                                        openblas_location+'/include',
+                                        ]),
+         'CMAKE_LIBRARY_PATH':':'.join([gcc_location+'/lib',
+                                        gcc_location+'/lib64',
+                                        binutils_location+'/lib',
+                                        openblas_location+'/lib',
+                                        ]),
+         'CC':gcc_location+'/bin/gcc',
+         'CXX':gcc_location+'/bin/g++',
+         'PYTORCH_PYTHON':python_bin,
+         'SLAPOS_COMPILE_ARGS':' '.join(['-Wl,-rpath,'+gcc_location+'/lib',
+                                         '-Wl,-rpath,'+gcc_location+'/lib64',
+                                         '-Wl,-rpath,'+binutils_location+'/lib',
+                                         '-Wl,-rpath,'+openblas_location+'/lib',
+                                         ])
+         }
+  if self.options.get('no-cuda') == '1':
+    env['NO_CUDA'] = '1'
+  import os.path
+  call([python_bin, 'setup.py', 'build'], cwd=workdir, env=env)
+  call([python_bin, 'setup.py', 'bdist_egg'], cwd=workdir, env=env)
+
+[pytorch-build-install-egg]
+recipe = slapos.recipe.build
+unzip-binary = ${unzip:location}/bin/unzip
+pytorch-repository-path = ${pytorch-repository:location}
+location = ${buildout:parts-directory}/${:_buildout_section_name_}
+need-pytorch-build = ${pytorch-build:location}
+egg = torch
+script =
+  os.makedirs(location)
+  workdir = self.options['pytorch-repository-path']
+  egg_name = 'torch-0.2.0+4af66c4-py2.7-linux-x86_64.egg'
+  dist_dir = os.path.join(workdir, 'dist')
+  dest_dir = os.path.join(self.buildout['buildout']['eggs-directory'], egg_name)
+  call([self.options['unzip-binary'], '-o', os.path.join(dist_dir, egg_name), '-d', dest_dir])
+
+[pytorch-egg]
+recipe = zc.recipe.egg
+initialization =
+  import scipy.spatial.ckdtree # load our own libstdc++ explicitly at the very beginning
+eggs = 
+  ${scipy:egg}
+  ${numpy:egg}
+  ${python-PyYAML:egg}
+  ${pytorch-build-install-egg:egg}
+  ${pillow-python:egg}
+  ${python-cocoapi-build-install-egg:egg}
+  ${matplotlib:egg}
+  six
+  torchvision
+interpreter = pytorch-python
+scripts = pytorch-python
+
+[versions]
+torchvision = 0.1.6

component/pytorch/pytorch.4af66c4.patch

+diff --git a/setup.py b/setup.py
+index 1d9a765..a50e9cb 100644
+--- a/setup.py
++++ b/setup.py
+@@ -90,7 +90,7 @@ def build_libs(libs):
+         assert lib in dep_libs, 'invalid lib: {}'.format(lib)
+     build_libs_cmd = ['bash', 'torch/lib/build_libs.sh']
+     my_env = os.environ.copy()
+-    my_env["PYTORCH_PYTHON"] = sys.executable
++    #my_env["PYTORCH_PYTHON"] = sys.executable
+     if WITH_SYSTEM_NCCL:
+         my_env["NCCL_ROOT_DIR"] = NCCL_ROOT_DIR
+     if WITH_CUDA:
+diff --git a/torch/lib/build_libs.sh b/torch/lib/build_libs.sh
+index af7020e..a91bd05 100755
+--- a/torch/lib/build_libs.sh
++++ b/torch/lib/build_libs.sh
+@@ -24,7 +24,7 @@ C_FLAGS=" -DTH_INDEX_BASE=0 -I$INSTALL_DIR/include \
+   -I$INSTALL_DIR/include/THS -I$INSTALL_DIR/include/THCS \
+   -I$INSTALL_DIR/include/THPP -I$INSTALL_DIR/include/THNN \
+   -I$INSTALL_DIR/include/THCUNN"
+-LDFLAGS="-L$INSTALL_DIR/lib "
++LDFLAGS="-L$INSTALL_DIR/lib $SLAPOS_COMPILE_ARGS"
+ LD_POSTFIX=".so.1"
+ LD_POSTFIX_UNVERSIONED=".so"
+ if [[ $(uname) == 'Darwin' ]]; then

component/slapos/build.sh

@@ -4,6 +4,12 @@
 # /opt/slapos folder, adapt this script as you please.  
 #
 # Be carefull to not run this script were the script is already installed.
+# 
+# Before run this script, ensure dependencies are installed, on debian, you can 
+# use the command bellow:
+#
+# apt-get install python gcc g++ make uml-utilities bridge-utils patch wget
+#
 
 # Use sudo or superuser and create slapos directory (you can pick a different directory)
 mkdir -p /opt/slapos/log/

component/tensorboard/0.4.patch

+diff --git a/tensorboard/pip_package/build_pip_package.sh b/tensorboard/pip_package/build_pip_package.sh
+index b386d59..f03b056 100755
+--- a/tensorboard/pip_package/build_pip_package.sh
++++ b/tensorboard/pip_package/build_pip_package.sh
+@@ -26,6 +26,7 @@ function main() {
+   DEST=$1
+   TMPDIR=$(mktemp -d -t tmp.XXXXXXXXXX)
+   RUNFILES="bazel-bin/tensorboard/pip_package/build_pip_package.runfiles/org_tensorflow_tensorboard"
++  WORKDIR=$(pwd)
+ 
+   echo $(date) : "=== Using tmpdir: ${TMPDIR}"
+ 
+@@ -45,8 +46,8 @@ function main() {
+   rm -f MANIFEST
+   echo $(date) : "=== Building wheel"
+   echo $(pwd)
+-  python setup.py bdist_wheel >/dev/null
+-  python3 setup.py bdist_wheel >/dev/null
++  PYTHONPATH=${WORKDIR}/${RUNFILES} $PYTHON_BIN_PATH setup.py bdist_egg >/dev/null
++  #python3 setup.py bdist_wheel >/dev/null
+   mkdir -p ${DEST}
+   cp dist/* ${DEST}
+   popd

component/tensorboard/buildout.cfg

+[buildout]
+extends =
+  ../zip/buildout.cfg
+  ../bazel/buildout.cfg
+parts =
+  slapos-cookbook-develop
+  slapos-cookbook
+  tensorboard-build-install-egg
+
+[tensorboard-repository]
+recipe = plone.recipe.command
+stop-on-error = true
+repository = https://github.com/tensorflow/tensorboard
+tag = 0.4
+git-binary = ${git:location}/bin/git
+patch-binary = ${patch:location}/bin/patch
+location = ${buildout:parts-directory}/${:_buildout_section_name_}
+command = export HOME=${:location}; (${:git-binary} clone --quiet -b ${:tag} ${:repository} ${:location}; cd ${buildout:parts-directory} ; ${:patch-binary} -p1 -d ${:_buildout_section_name_} < ${:_profile_base_location_}/0.4.patch ) || (rm -fr ${:location}; exit 1)
+
+[tensorboard-build]
+recipe = slapos.recipe.build
+location = ${buildout:parts-directory}/${:_buildout_section_name_}
+workdir = ${tensorboard-repository:location}
+gcc-bin = ${gcc:location}/bin
+gcc-lib = ${gcc:location}/lib
+gcc-lib64 = ${gcc:location}/lib64
+numpy-python-command = ${buildout:bin-directory}/${numpy-egg:interpreter}
+python27-lib = ${python2.7:location}/lib
+java_home_bin = ${bazel:java_home}/bin
+bazel-bin = ${bazel:location}/bin
+script =
+  os.makedirs(location)
+  workdir = self.options['workdir']
+  env = {'PATH':':'.join([self.options['gcc-bin'],
+                          self.options['java_home_bin'],
+                          self.options['bazel-bin'],
+                          os.environ['PATH']]),
+         'COMPILER_PATH':':'.join([self.options['gcc-bin'],
+                                   os.environ.get('COMPILER_PATH') or '']),
+         'LIBRARY_PATH':':'.join([self.options['gcc-lib'],
+                                  self.options['gcc-lib64'],
+                                  os.environ.get('LIBRARY_PATH') or '']),
+         'PYTHON_BIN_PATH':self.options['numpy-python-command'],
+         'PYTHON_LIB_PATH':self.options['python27-lib'],
+         }
+  import os.path
+  env['LD_LIBRARY_PATH'] = env['LIBRARY_PATH']
+  bazel_command = ['bazel', 'build', '--spawn_strategy=standalone', '--verbose_failures', '--sandbox_debug', '//tensorboard/pip_package:build_pip_package']
+  call(bazel_command, cwd=workdir, env=env)
+
+[tensorboard-build-install-egg]
+recipe = slapos.recipe.build
+unzip-binary = ${unzip:location}/bin/unzip
+tensorboard-repository-path = ${tensorboard-repository:location}
+location = ${buildout:parts-directory}/${:_buildout_section_name_}
+need-tensorboard-build = ${tensorboard-build:location}
+egg = tensorflow-tensorboard
+bazel-bin = ${bazel:location}/bin
+numpy-python-command = ${buildout:bin-directory}/${numpy-egg:interpreter}
+script =
+  os.makedirs(location)
+  workdir = self.options['tensorboard-repository-path']
+  egg_name = 'tensorflow_tensorboard-0.4.0rc1-py2.7.egg'
+  dist_dir = os.path.join(workdir, 'dist')
+  dest_dir = os.path.join(self.buildout['buildout']['eggs-directory'], egg_name)
+  env = {'PATH':':'.join([self.options['bazel-bin'],
+                          os.environ['PATH']]),
+         'PYTHON_BIN_PATH':self.options['numpy-python-command'],
+         }
+  call(['tensorboard/pip_package/build_pip_package.sh', dist_dir], cwd=workdir, env=env)
+  call([self.options['unzip-binary'], '-o', os.path.join(dist_dir, egg_name), '-d', dest_dir])
+
+[versions]

component/tensorflow/buildout.cfg

@@ -5,6 +5,7 @@ extends =
   ../zip/buildout.cfg
   ../bazel/buildout.cfg
   ../protobuf-python/buildout.cfg
+  ../tensorboard/buildout.cfg
 parts =
   slapos-cookbook-develop
   slapos-cookbook
@@ -26,11 +27,11 @@ location = ${buildout:parts-directory}/${:_buildout_section_name_}
 recipe = plone.recipe.command
 stop-on-error = true
 repository = https://github.com/tensorflow/tensorflow
-tag = v1.0.1
+tag = r1.4
 git-binary = ${git:location}/bin/git
 patch-binary = ${patch:location}/bin/patch
 location = ${buildout:parts-directory}/${:_buildout_section_name_}
-command = export HOME=${:location}; (${:git-binary} clone --recurse-submodules --quiet -b ${:tag} ${:repository} ${:location}; cd ${buildout:parts-directory} ; ${:patch-binary} -p1 -d ${:_buildout_section_name_} < ${:_profile_base_location_}/tensorflow-v1.0.1.patch ) || (rm -fr ${:location}; exit 1)
+command = export HOME=${:location}; (${:git-binary} clone --recurse-submodules --quiet -b ${:tag} ${:repository} ${:location}; cd ${buildout:parts-directory} ; ${:patch-binary} -p1 -d ${:_buildout_section_name_} < ${:_profile_base_location_}/tensorflow-r1.4.patch ) || (rm -fr ${:location}; exit 1)
 
 [cuda]
 tf_need_cuda = 1
@@ -72,6 +73,10 @@ script =
          'TF_NEED_GCP':'0',
          'TF_NEED_HDFS':'0',
          'TF_NEED_OPENCL':'0',
+         'TF_NEED_MKL':'0',
+         'TF_NEED_VERBS':'0',
+         'TF_CUDA_CLANG':'0',
+         'TF_NEED_MPI':'0',
          'TF_NEED_CUDA':self.buildout['cuda']['tf_need_cuda'],
          ##### FOR CUDA #####
          'GCC_HOST_COMPILER_PATH':os.path.join(self.options['gcc-bin'], 'gcc'),
@@ -98,18 +103,17 @@ recipe = slapos.recipe.build
 unzip-binary = ${unzip:location}/bin/unzip
 tensorflow-repository-path = ${tensorflow-repository:location}
 location = ${buildout:parts-directory}/${:_buildout_section_name_}
+need-tensorboard-build = ${tensorboard-build:location}
 need-tensorflow-build = ${tensorflow-build:location}
 need-protobuf-python = ${protobuf-python:egg}
 egg = tensorflow
 script =
   os.makedirs(location)
   workdir = self.options['tensorflow-repository-path']
-  egg_name = 'tensorflow-1.0.1-py2.7-linux-x86_64.egg'
+  egg_name = 'tensorflow-1.4.0rc1-py2.7-linux-x86_64.egg'
   dist_dir = os.path.join(workdir, 'dist')
   dest_dir = os.path.join(self.buildout['buildout']['eggs-directory'], egg_name)
   call(['bazel-bin/tensorflow/tools/pip_package/build_pip_package', dist_dir], cwd=workdir)
   call([self.options['unzip-binary'], '-o', os.path.join(dist_dir, egg_name), '-d', dest_dir])
 
 [versions]
-protobuf = 3.1.0.post1
-wheel = 0.30.0a0

component/tensorflow/tensorflow-v1.0.1.patch → component/tensorflow/tensorflow-r1.4.patch

 diff --git a/tensorflow/tools/pip_package/build_pip_package.sh b/tensorflow/tools/pip_package/build_pip_package.sh
-index e2bee21..1df90c1 100755
+index cbf06a9..226a423 100755
 --- a/tensorflow/tools/pip_package/build_pip_package.sh
 +++ b/tensorflow/tools/pip_package/build_pip_package.sh
-@@ -136,7 +136,7 @@ function main() {
+@@ -162,7 +162,7 @@ function main() {
    pushd ${TMPDIR}
    rm -f MANIFEST
    echo $(date) : "=== Building wheel"
--  "${PYTHON_BIN_PATH:-python}" setup.py bdist_wheel ${GPU_FLAG} >/dev/null
-+  "${PYTHON_BIN_PATH:-python}" setup.py bdist_egg ${GPU_FLAG} >/dev/null
+-  "${PYTHON_BIN_PATH:-python}" setup.py bdist_wheel ${PKG_NAME_FLAG} >/dev/null
++  "${PYTHON_BIN_PATH:-python}" setup.py bdist_egg ${PKG_NAME_FLAG} >/dev/null
    mkdir -p ${DEST}
    cp dist/* ${DEST}
    popd

component/tmux/buildout.cfg

+[buildout]
+extends =
+  ../libevent/buildout.cfg
+  ../ncurses/buildout.cfg
+
+parts +=
+  tmux
+
+[tmux]
+recipe = slapos.recipe.cmmi
+url = https://github.com/tmux/tmux/releases/download/2.0/tmux-2.0.tar.gz
+md5sum = 9fb6b443392c3978da5d599f1e814eaa
+environment =
+  CFLAGS=-I${ncurses:location}/include -I${libevent:location}/include/
+  LDFLAGS=-L${ncurses:location}/lib/ -L${libevent:location}/lib/ -Wl,-rpath=${ncurses:location}/lib/ -Wl,-rpath=${libevent:location}/lib/
+configure-options =
+  --prefix=${buildout:parts-directory}/${:_buildout_section_name_}

software/agent/software.cfg

@@ -50,8 +50,8 @@ gitdb = 0.6.4
 pycrypto = 2.6.1
 pycurl = 7.43.0
 slapos.recipe.download = 1.0
-slapos.recipe.template = 3.0
-slapos.toolbox = 0.72
+slapos.recipe.template = 4.1
+slapos.toolbox = 0.73
 smmap = 0.9.0
 
 # Required by:

software/apache-frontend/software.cfg

@@ -10,8 +10,8 @@ gitdb = 0.6.4
 plone.recipe.command = 1.1
 pycrypto = 2.6.1
 rdiff-backup = 1.0.5+SlapOSPatched001
-slapos.recipe.template = 3.0
-slapos.toolbox = 0.72
+slapos.recipe.template = 4.1
+slapos.toolbox = 0.73
 smmap = 0.9.0
 numpy = 1.11.2
 pyasn1 = 0.2.3

software/caddy/software.cfg

+[buildout]
+extends =
+  ../../stack/caddy/buildout.cfg
\ No newline at end of file

software/cdn-me/software.cfg

@@ -40,4 +40,4 @@ cns.recipe.symlink = 0.2.3
 collective.recipe.environment = 0.2.0
 erp5.util = 0.4.49
 plone.recipe.command = 1.1
-slapos.recipe.template = 3.0
+slapos.recipe.template = 4.1

software/dream/software.cfg

@@ -53,7 +53,7 @@ mysqlclient = 1.3.12
 # indirect dependancies
 cp.recipe.cmd = 0.5
 plone.recipe.command = 1.1
-slapos.recipe.template = 3.0
+slapos.recipe.template = 4.1
 zope.exceptions = 4.0.7
 zope.testing = 4.1.3
 zc.recipe.testrunner = 2.0.0

software/erp5/instance-erp5-input-schema.json

@@ -17,6 +17,10 @@
       "default": "erp5",
       "type": "string"
     },
+    "bt5": {
+      "description": "Business Template to install at automatic site creation. By default, all configurators are installed.",
+      "type": "string"
+    },
     "timezone": {
       "description": "Zope's timezone. Possible values are determined by host's libc, and typically come from a separate package (tzdata, ...)",
       "default": "UTC",

software/erp5testnode/software.cfg

@@ -60,7 +60,7 @@ mode = 0644
 [versions]
 PyXML = 0.8.5
 erp5.util = 0.4.49
-slapos.recipe.template = 3.0
+slapos.recipe.template = 4.1
 ipython = 5.3.0
 apache-libcloud = 2.1.0
 gitdb2 = 2.0.2

software/erp5testnode/testsuite/dummy/software.cfg

@@ -14,4 +14,4 @@ md5sum = efd3b712a2294207f265a9c45648d5cf
 mode = 0644
 
 [versions]
-slapos.recipe.template = 3.0
+slapos.recipe.template = 4.1

software/fluentd/software.cfg

@@ -19,5 +19,5 @@ context =
 
 [fluentd]
 gems +=
-  fluent-plugin-wendelin==0.1
+  fluent-plugin-wendelin==0.3
   fluent-plugin-bin

software/gitlab/instance-gitlab-test.cfg.in

@@ -14,8 +14,19 @@ passwd = root1234
 environment = 
   GITLAB_ROOT_PASSWORD=${root-password:passwd}
 
-[backend-info]
-# host    = ${instance-parameter:ipv4-random}
+[service-postgresql]
+pgdata-directory = ${directory:srv}/pg
+
+[gitlab-workhorse-dir]
+recipe  = slapos.cookbook:mkdirectory
+srv     = ${directory:srv}/glab-wh
+
+[gitlab-workhorse]
+srv     = ${gitlab-workhorse-dir:srv}
+socket  = ${gitlab-workhorse:srv}/wh.socket
+
+[unicorn]
+socket  = ${:srv}/unc.socket
 
 [publish-instance-info]
 password = ${root-password:passwd}

software/gitlab/software.cfg

@@ -27,7 +27,7 @@ extends =
 
 parts =
     ruby2.1
-    golang16
+    golang19
     git
     postgresql92
     redis28
@@ -211,7 +211,7 @@ make-targets= cd ${git2go-repository:location}
               && cp -a ${git-backup-repository:location}/contrib/gitlab-backup ${gopath:bin}
 environment =
   PKG_CONFIG_PATH=${openssl:location}/lib/pkgconfig:${zlib:location}/lib/pkgconfig
-  PATH=${cmake:location}/bin:${pkgconfig:location}/bin:${git:location}/bin:${golang16:location}/bin:${buildout:bin-directory}:%(PATH)s
+  PATH=${cmake:location}/bin:${pkgconfig:location}/bin:${git:location}/bin:${golang19:location}/bin:${buildout:bin-directory}:%(PATH)s
   GOPATH=${gopath:directory}
 
 [xnice-repository]
@@ -245,7 +245,7 @@ configure-command = :
 make-targets= ${:_buildout_section_name_}
 
 environment =
-  PATH=${golang16:location}/bin:%(PATH)s
+  PATH=${golang19:location}/bin:%(PATH)s
 
 
 ###############################
@@ -336,7 +336,7 @@ md5sum  = 319d7dbe3ad9b260c1e292cfc0d13b11
 
 [instance-gitlab-test.cfg.in]
 <= download-file
-md5sum  = cc8065104458af311c2ffa9ae20235a6
+md5sum  = a4ad76856db98e508af7e773d9ff78f9
 
 [macrolib.cfg.in]
 <= download-file
@@ -381,5 +381,5 @@ cns.recipe.symlink = 0.2.3
 docutils = 0.12
 plone.recipe.command = 1.1
 rubygemsrecipe  = 0.2.2+slapos001
-slapos.recipe.template = 3.0
+slapos.recipe.template = 4.1
 z3c.recipe.scripts = 1.0.1

software/html5as/software.cfg

@@ -72,7 +72,7 @@ async = 0.6.1
 gitdb = 0.5.4
 pycrypto = 2.6
 rdiff-backup = 1.0.5+SlapOSPatched001
-slapos.recipe.template = 2.4.2
+slapos.recipe.template = 4.1
 slapos.toolbox = 0.40.4
 smmap = 0.8.2
 plone.recipe.command = 1.1

software/html5ide/software.cfg

@@ -48,4 +48,4 @@ md5sum = 8cde04bfd0c0e9bd56744b988275cfd8
 PyRSS2Gen = 1.1
 cns.recipe.symlink = 0.2.3
 plone.recipe.command = 1.1
-slapos.recipe.template = 3.0
+slapos.recipe.template = 4.1

software/jstestnode/software.cfg

@@ -113,5 +113,5 @@ mode = 0644
 
 [versions]
 erp5.util = 0.4.49
-slapos.recipe.template = 3.0
+slapos.recipe.template = 4.1
 selenium = 2.53.1

software/jupyter/software.cfg

@@ -85,7 +85,7 @@ pyzmq = 16.0.2
 scikit-learn = 0.18.1
 seaborn = 0.7.1
 simplegeneric = 0.8.1
-slapos.recipe.template = 3.0
+slapos.recipe.template = 4.1
 statsmodels = 0.8.0
 terminado = 0.6
 tornado = 4.4.2

software/kvm/software.cfg

@@ -5,13 +5,13 @@ extends = common.cfg
 # XXX - use websockify = 0.5.1 for compatibility with kvm frontend
 websockify = 0.5.1
 
-slapos.toolbox = 0.72
+slapos.toolbox = 0.73
 erp5.util = 0.4.49
 apache-libcloud = 1.1.0
 collective.recipe.environment = 0.2.0
 gitdb = 0.6.4
 pycurl = 7.43.0
-slapos.recipe.template = 3.0
+slapos.recipe.template = 4.1
 smmap = 0.9.0
 # websockify = 0.8.0
 

software/nayuos/software.cfg

@@ -17,7 +17,7 @@ parts +=
 versions = versions
 
 [versions]
-slapos.recipe.template = 3.0
+slapos.recipe.template = 4.1
 
 
 [template-instance]

software/neoppod/software-common.cfg

@@ -116,12 +116,12 @@ mysqlclient = 1.3.12
 persistent = 4.2.3
 pycrypto = 2.6.1
 pycurl = 7.43.0
-slapos.recipe.template = 3.0
-slapos.toolbox = 0.72
+slapos.recipe.template = 4.1
+slapos.toolbox = 0.73
 smmap2 = 2.0.1
 transaction = 1.7.0
 zodbpickle = 0.6.0
-zodbtools = 0.0.0.dev3
+zodbtools = 0.0.0.dev4
 
 # Required by:
 # slapos.toolbox==0.71
@@ -153,3 +153,20 @@ paramiko = 2.1.2
 # Required by:
 # slapos.toolbox==0.71
 passlib = 1.7.1
+
+# Required by:
+# zodbtools==0.0.0dev4
+zodburi = 2.3.0
+
+# Required by:
+# zodburi==2.0
+# ZEO 5 requires transaction >= 2
+ZEO = 4.3.1
+
+# Required by:
+# zodburi==2.0
+mock = 2.0.0
+
+# Required by:
+# ZEO==4.3.1
+zdaemon = 4.2.0

software/neoppod/software-zodb5.cfg

@@ -3,4 +3,14 @@ extends = software.cfg
 
 [versions]
 ZODB = 5.3.0
+ZEO = 5.1.0
 transaction = 2.1.2
+
+# Required by:
+# ZEO==5.1.0
+# trollius==2.1
+futures = 3.1.1
+
+# Required by:
+# ZEO==5.1.0
+trollius = 2.1

software/neotest/gowork.cfg

+# Code generated by gowork-snapshot; DO NOT EDIT.
+
+# list of go git repositories to fetch
+[gowork.goinstall]
+depends_gitfetch  =
+    ${go_github.com_cznic_strutil:recipe}
+    ${go_github.com_golang_glog:recipe}
+    ${go_github.com_kisielk_og-rek:recipe}
+    ${go_github.com_kylelemons_godebug:recipe}
+    ${go_github.com_pkg_errors:recipe}
+    ${go_github.com_pkg_profile:recipe}
+    ${go_github.com_someonegg_gocontainer:recipe}
+    ${go_golang.org_x_net:recipe}
+    ${go_golang.org_x_perf:recipe}
+    ${go_golang.org_x_sync:recipe}
+    ${go_golang.org_x_tools:recipe}
+    ${go_lab.nexedi.com_kirr_go123:recipe}
+    ${go_lab.nexedi.com_kirr_neo:recipe}
+
+
+[go_github.com_cznic_strutil]
+<= go-git-package
+go.importpath = github.com/cznic/strutil
+repository    = https://github.com/cznic/strutil
+revision  = 529a34b1c1
+
+[go_github.com_golang_glog]
+<= go-git-package
+go.importpath = github.com/golang/glog
+repository    = https://github.com/golang/glog
+revision  = 23def4e6c1
+
+[go_github.com_kisielk_og-rek]
+<= go-git-package
+go.importpath = github.com/kisielk/og-rek
+repository    = https://github.com/kisielk/og-rek
+revision  = dd41cde712
+
+[go_github.com_kylelemons_godebug]
+<= go-git-package
+go.importpath = github.com/kylelemons/godebug
+repository    = https://github.com/kylelemons/godebug
+revision  = d65d576e93
+
+[go_github.com_pkg_errors]
+<= go-git-package
+go.importpath = github.com/pkg/errors
+repository    = https://github.com/pkg/errors
+revision  = v0.8.0-6-g2b3a18b5f0
+
+[go_github.com_pkg_profile]
+<= go-git-package
+go.importpath = github.com/pkg/profile
+repository    = https://github.com/pkg/profile
+revision  = v1.2.1-0-g5b67d42886
+
+[go_github.com_someonegg_gocontainer]
+<= go-git-package
+go.importpath = github.com/someonegg/gocontainer
+repository    = https://github.com/someonegg/gocontainer
+revision  = fc2c7e84b5
+
+[go_golang.org_x_net]
+<= go-git-package
+go.importpath = golang.org/x/net
+repository    = https://go.googlesource.com/net
+revision  = 1087133bc4
+
+[go_golang.org_x_perf]
+<= go-git-package
+go.importpath = golang.org/x/perf
+repository    = https://go.googlesource.com/perf
+revision  = 4469e6ce8c
+
+[go_golang.org_x_sync]
+<= go-git-package
+go.importpath = golang.org/x/sync
+repository    = https://go.googlesource.com/sync
+revision  = 8e0aa688b6
+
+[go_golang.org_x_tools]
+<= go-git-package
+go.importpath = golang.org/x/tools
+repository    = https://go.googlesource.com/tools
+revision  = 9bd2f44268
+
+[go_lab.nexedi.com_kirr_go123]
+<= go-git-package
+go.importpath = lab.nexedi.com/kirr/go123
+repository    = https://lab.nexedi.com/kirr/go123.git
+revision  = 2578d58311
+
+[go_lab.nexedi.com_kirr_neo]
+<= go-git-package
+go.importpath = lab.nexedi.com/kirr/neo
+repository    = https://lab.nexedi.com/kirr/neo.git
+revision  = v1.8-1326-g4d0cd89484

software/neotest/instance.cfg.in

+# TODO instance which runs this test periodically automatically and ingests results to ERP5

software/neotest/neotest-env.sh.in

+# env.sh for putting everything needed to run neotest on path
+
+. ${gowork:env.sh}
+
+PATH="${ethtool:location}/sbin:$PATH"
+PATH="${ioping:location}/bin:$PATH"
+PATH="${lmbench:location}/bin:$PATH"
+PATH="${mariadb:location}/bin:$PATH"
+PATH="${mariadb:location}/scripts:$PATH"
+PATH="${python2.7:location}/bin:$PATH"
+
+# add all eggs and develop-eggs to py path
+#
+# XXX better leverage something like zc.recipe.egg to generate eggs path buildout way
+# XXX see e.g. https://lab.nexedi.com/nexedi/slapos/merge_requests/242#note_49644 for why
+# TODO(kirr) try doing so
+PYTHONPATH=${buildout:develop-eggs-directory}:$PYTHONPATH       # for .egg-link to be found by setuptools
+for egglink in ${buildout:develop-eggs-directory}/*.egg-link; do
+    PYTHONPATH=`cat $egglink |head -1`:$PYTHONPATH
+done
+
+for egg in ${buildout:develop-eggs-directory}/*.egg ${buildout:eggs-directory}/*.egg; do
+        export PYTHONPATH=$egg:$PYTHONPATH
+done
+
+
+export PATH
+export PYTHONPATH
+
+export PS1="(neotest-`basename ${buildout:directory}`) $PS1"

software/neotest/neotest.in

+#!/bin/bash -e
+# neotest wraper so it could be run without any environment preset
+
+. ${buildout:directory}/neotest-env.sh
+exec ${gowork:src}/lab.nexedi.com/kirr/neo/go/neo/t/neotest "$@"

software/neotest/software-dev.cfg

+# neotest SR with main components taken from latest git versions
+[buildout]
+extends = software.cfg
+
+
+[go_lab.nexedi.com_kirr_go123]
+branch	= master
+revision=
+
+[go_lab.nexedi.com_kirr_neo]
+branch	= t
+revision=
+
+[lmbench-repository]
+branch  = x/kirr
+revision=
+
+[ioping-repository]
+branch  = x/hist
+revision=

software/neotest/software.cfg

+# NEO test software-release
+#
+# This software-release prepares environment so that neotest NEO
+# testing/benchmarking can be run in a SlapOS WebRunner.
+[buildout]
+extends =
+    ../../stack/slapos.cfg
+    ../neoppod/software-common.cfg
+    ../../component/golang/buildout.cfg
+    gowork.cfg
+    ../../component/wendelin.core/buildout.cfg
+    ../../component/ethtool/buildout.cfg
+    ../../component/ioping/buildout.cfg
+    ../../component/lmbench/buildout.cfg
+
+parts =
+    gowork
+    lmbench
+    ioping
+    ethtool
+
+    neoppod-develop
+    neoppod
+    wendelin.core-dev
+    eggs
+
+    neotest-env.sh
+    neotest
+
+#   for instance
+    slapos-deps-eggs
+    slapos-cookbook
+#   instance.cfg
+
+
+# go packages to install (+ automatically their dependencies)
+[gowork]
+install =
+    lab.nexedi.com/kirr/neo/go/...  \
+    github.com/pkg/profile          \
+    golang.org/x/perf/cmd/benchstat
+
+# dev-install neo from go checkout
+[neoppod-develop]
+setup   = ${go_lab.nexedi.com_kirr_neo:location}
+
+
+# neotest-env.sh + neotest wrapper
+[buildout-template]
+recipe  = slapos.recipe.template
+url     = ${:_profile_base_location_}/${:_buildout_section_name_}.in
+output  = ${buildout:directory}/${:_buildout_section_name_}
+
+[neotest-env.sh]
+<= buildout-template
+md5sum  = 6e2203c7a86e8a3e8e56b8086115f5d4
+
+[neotest]
+<= buildout-template
+output  = ${buildout:bin-directory}/${:_buildout_section_name_}
+mode    = 0755
+md5sum  = fb3b4109128c1db1739ef5bb6abd1d94
+
+
+# instance (TODO)
+[instance.cfg]
+<= buildout-template
+md5sum  = 410e1b2d72829824b28cc0299adb472e
+
+
+# eggs:
+[eggs]
+recipe  = zc.recipe.egg
+eggs    =
+# wendelin.core still requires ZODB3 but having ZODB4 or 5 installed satisfies latest ZODB3
+    ZODB3
+# also for wc
+    numpy
+# to install not only wendelin.core modules but also scripts
+    wendelin.core
+# for ZEO scripts (runzeo)
+    ZEO
+
+
+# wendelin.core: latest not yet released
+[wendelin.core-repository]
+revision= v0.11-4-g38fbc83ceb
+
+# ping eggs versions
+[versions]
+pyasn1 = 0.3.7
+ZODB3 = 3.11.0
+numpy = 1.13.3
+zope.testing = 4.6.2
+
+# Required by:
+# ZEO==4.3.1
+# ZODB==4.4.5
+# zodburi==2.3.0
+ZConfig = 3.2.0
+
+# Required by:
+# ZEO==4.3.1
+# ZODB==4.4.5
+zc.lockfile = 1.2.1
+
+# Required by:
+# slapos.toolbox==0.73
+erp5.util = 0.4.49

software/nginx-push-stream/software.cfg

@@ -45,5 +45,5 @@ output = ${buildout:directory}/instance-nginx.cfg.in
 mode = 0644
 
 [versions]
-slapos.recipe.template = 3.0
-slapos.toolbox = 0.72
+slapos.recipe.template = 4.1
+slapos.toolbox = 0.73

software/powerdns/software.cfg

@@ -65,4 +65,4 @@ mode = 0644
 PyRSS2Gen = 1.1
 cns.recipe.symlink = 0.2.3
 plone.recipe.command = 1.1
-slapos.recipe.template = 3.0
+slapos.recipe.template = 4.1

software/re6stnet/software.cfg

@@ -110,8 +110,8 @@ gitdb = 0.6.4
 plone.recipe.command = 1.1
 pycrypto = 2.6.1
 pycurl = 7.43.0
-slapos.recipe.template = 3.0
-slapos.toolbox = 0.72
+slapos.recipe.template = 4.1
+slapos.toolbox = 0.73
 smmap = 0.9.0
 
 # Required by:

software/seleniumrunner/software.cfg

@@ -47,4 +47,4 @@ mode = 0644
 
 [versions]
 plone.recipe.command = 1.1
-slapos.recipe.template = 3.0
+slapos.recipe.template = 4.1

software/slapos-in-partition/software.cfg

@@ -11,4 +11,4 @@ extends = common.cfg
 Pygments = 1.6
 collective.recipe.environment = 0.2.0
 collective.recipe.template = 1.10
-slapos.recipe.template = 3.0
+slapos.recipe.template = 4.1

software/slapos-master/apache-backend.conf.in

@@ -108,13 +108,17 @@ SSLProxyEngine On
 
 # As backend is trusting REMOTE_USER header unset it always
 RequestHeader unset REMOTE_USER
+RequestHeader unset SSL_CLIENT_SERIAL
 {% if parameter_dict['ca-cert'] -%}
-SSLVerifyClient require
+SSLVerifyClient optional
 RequestHeader set REMOTE_USER %{SSL_CLIENT_S_DN_CN}s
+RequestHeader set SSL_CLIENT_SERIAL "%{SSL_CLIENT_M_SERIAL}s"
 SSLCACertificateFile {{ parameter_dict['ca-cert'] }}
+{%   if parameter_dict['crl'] -%}
 SSLCARevocationCheck chain
 SSLCARevocationFile {{ parameter_dict['crl'] }}
 {%-   endif %}
+{%- endif %}
 
 ErrorLog "{{ parameter_dict['error-log'] }}"
 # Default apache log format with request time in microsecond at the end
@@ -133,8 +137,10 @@ RewriteEngine On
 Listen {{ ip }}:{{ port }}
 {%   endfor -%}
 <VirtualHost *:{{ port }}>
-{% if enable_authentication -%}
+  SSLEngine on
+{% if enable_authentication and parameter_dict['shared-ca-cert'] and parameter_dict['shared-crl'] -%}
   SSLVerifyClient require
+  # Custom block we use for now different parameters.
   RequestHeader set REMOTE_USER %{SSL_CLIENT_S_DN_CN}s
   SSLCACertificateFile {{ parameter_dict['shared-ca-cert'] }}
   SSLCARevocationPath {{ parameter_dict['shared-crl'] }}
@@ -145,8 +151,6 @@ Listen {{ ip }}:{{ port }}
   ErrorLog "{{ parameter_dict['log-dir'] }}/apache-service-error.log"
   CustomLog "{{ parameter_dict['log-dir'] }}/apache-service-access.log" combined
 {% endif -%}
-  SSLEngine on
   RewriteRule ^/(.*) {{ backend }}/$1 [L,P]
 </VirtualHost>
 {% endfor -%}
-

software/slapos-master/buildout.hash.cfg

@@ -15,16 +15,12 @@
 # not need these here).
 [template-erp5]
 filename = instance-erp5.cfg.in
-md5sum = aa10af616493b97d2b630c46d869a9d7
+md5sum = 95f7341ff490cb70df61fad35d08bc30
 
 [template-balancer]
 filename = instance-balancer.cfg.in
-md5sum = c5ce18fa4d4be9b9a2d789f3bbd37840
+md5sum = c76c4b36c351b7383dad7bed04e3bc90
 
 [template-apache-backend-conf]
 filename = apache-backend.conf.in
-md5sum = ea77222f440bb72fee4939fe1b72976e
-
-[template-create-erp5-site-real]
-filename = instance-create-erp5-site-real.cfg.in
-md5sum = 86a2b244341218cd0c4b6d398c61ee20
+md5sum = 991bfcc4b6abae2ec59b69d71013e8ad

software/slapos-master/instance-balancer.cfg.in

 {% set part_list = [] -%}
 {% set ssl_parameter_dict = slapparameter_dict.get('ssl', {}) %}
+{% set caucase_url = slapparameter_dict.get('caucase-url', '') -%}
 {% macro section(name) %}{% do part_list.append(name) %}{{ name }}{% endmacro -%}
 {% set use_ipv6 = slapparameter_dict.get('use-ipv6', False) -%}
 {% set shared_ca_path = slapparameter_dict['shared-certificate-authority-path'] -%}
@@ -37,6 +38,56 @@ context = key content {{content_section_name}}:content
 mode = {{ mode }}
 {%- endmacro %}
 
+[certificate-request-base]
+recipe = slapos.cookbook:wrapper
+wrapper-path = ${directory:bin}/request-instance-certificate
+parameters-extra = true
+command-line = {{ parameter_dict['bin-directory'] }}/caucase-cliweb
+  --crt-file ${apache-conf-ssl:cert}
+  --key-file ${apache-conf-ssl:key}
+  --crl-file ${apache-conf-ssl:crl}
+  --ca-url {{ caucase_url }}
+  --ca-crt-file ${apache-conf-ssl:ca-cert}
+
+{% macro request_cert(name, common_name) -%}
+{% set get_crl_periodicity = slapparameter_dict.get('crl-update-periodicity', 'daily') -%}
+
+[{{ section(name ~ '-certificate-request') }}]
+recipe = slapos.cookbook:wrapper
+wrapper-path = ${directory:services}/request-{{ name }}-certificate
+command-line =
+  ${certificate-request-base:wrapper-path}
+  --cn {{ common_name }}
+  --request
+
+[{{ section(name ~ '-renew-cron-entry') }}]
+recipe = slapos.cookbook:cron.d
+cron-entries = ${cron:cron-entries}
+name = {{ name }}-certificate-auto-renew
+time = weekly
+# 2592000 = 30*24*60*60  equivalent to one month in seconds
+command = ${certificate-request-base:wrapper-path} --renew --threshold 2592000 --on-renew="${apache-graceful:output}"
+
+[{{ section(name ~ '-download-crl') }}]
+# download the crl for the first time
+recipe = plone.recipe.command
+command = 
+  if [ ! -s "${apache-conf-ssl:crl}" ]; then
+    ${certificate-request-base:wrapper-path} --update-crl
+  fi
+update-command = ${:command}
+stop-on-error = true
+
+[{{ section(name ~ '-update-crl-cron-entry') }}]
+recipe = slapos.cookbook:cron.d
+cron-entries = ${cron:cron-entries}
+name = {{ name }}-update-crl
+time = {{ get_crl_periodicity }}
+# XXX - Update crl call apache graceful restart, it's not recommended to check crl too often, Apache
+# has an issue with reload and can be frozen and stop responding. Default periodicity time = daily
+command = ${certificate-request-base:wrapper-path} --update-crl --on-crl-update="${apache-graceful:output}"
+{%- endmacro %}
+
 {% if use_ipv6 -%}
 [zope-tunnel-base]
 recipe = slapos.cookbook:ipv4toipv6
@@ -82,6 +133,7 @@ ipv6 = {{ zope_address.split(']:')[0][1:] }}
 -#}
 {%   do zope_family_address_list[0][0] -%}
 {%   set haproxy_port = next_port() -%}
+{%   set backend_path = slapparameter_dict['backend-path-dict'][family_name] -%}
 {%   do haproxy_dict.__setitem__(family_name, (haproxy_port, zope_family_address_list)) -%}
 {%   if has_webdav -%}
 {%     set internal_scheme = 'http' -%}{# mod_rewrite does not recognise webdav scheme -#}
@@ -90,8 +142,7 @@ ipv6 = {{ zope_address.split(']:')[0][1:] }}
 {%     set internal_scheme = 'http' -%}
 {%     set external_scheme = 'https' -%}
 {%   endif -%}
-{%   set backend_path = slapparameter_dict['backend-path-dict'][family_name] -%}
-{%   set ssl_authentication = slapparameter_dict['ssl-authentication-dict'][family_name] -%}
+{%   set ssl_authentication = slapparameter_dict['ssl-authentication-dict'].get(family_name, False) -%}
 {%   do apache_dict.__setitem__(family_name, (next_port(), external_scheme, internal_scheme ~ '://' ~ ipv4 ~ ':' ~ haproxy_port ~ backend_path, ssl_authentication)) -%}
 {% endfor -%}
 
@@ -125,9 +176,9 @@ crl = ${directory:apache-conf}/crl.pem
 backend-list = {{ dumps(apache_dict.values()) }}
 ip-list = {{ dumps(apache_ip_list) }}
 pid-file = ${directory:run}/apache.pid
+log-dir = ${directory:log}
 error-log = ${directory:log}/apache-error.log
 access-log = ${directory:log}/apache-access.log
-log-dir = ${directory:log} 
 # Apache 2.4's default value (60 seconds) can be a bit too short
 timeout = 300
 # Basic SSL server configuration
@@ -144,8 +195,6 @@ shared-ca-cert = {{ shared_ca_path }}/cacert.pem
 shared-crl = {{ shared_ca_path }}/crl
 {%- endif %}
 
-
-
 [apache-conf]
 < = jinja2-template-base
 template = {{ parameter_dict['template-apache-conf'] }}
@@ -156,6 +205,18 @@ context = section parameter_dict apache-conf-parameter-dict
 recipe = slapos.cookbook:wrapper
 wrapper-path = ${directory:services}/apache
 command-line = "{{ parameter_dict['apache'] }}/bin/httpd" -f "${apache-conf:rendered}" -DFOREGROUND
+wait-for-files =
+  ${apache-conf-ssl:cert}
+  ${apache-conf-ssl:key}
+
+[apache-graceful]
+recipe = collective.recipe.template
+input = inline:
+  #!/bin/sh
+  kill -USR1 "$(cat '${apache-conf-parameter-dict:pid-file}')"
+
+output = ${directory:bin}/apache-httpd-graceful
+mode = 700
 
 [{{ section('apache-promise') }}]
 # Check any apache port in ipv4, expect other ports and ipv6 to behave consistently
@@ -164,7 +225,7 @@ path = ${directory:promise}/apache
 hostname = {{ ipv4 }}
 port = {{ apache_dict.values()[0][0] }}
 
-[publish]
+[{{ section('publish') }}]
 recipe = slapos.cookbook:publish.serialised
 {% for family_name, (apache_port, scheme, _, _) in apache_dict.items() -%}
 {{   family_name ~ '-v6' }} = {% if ipv6_set %}{{ scheme ~ '://[' ~ ipv6 ~ ']:' ~ apache_port }}{% endif %}
@@ -178,6 +239,11 @@ key = ${apache-ssl-key:rendered}
 cert = ${apache-ssl-cert:rendered}
 {{ simplefile('apache-ssl-key', '${apache-conf-ssl:key}', ssl_parameter_dict['key']) }}
 {{ simplefile('apache-ssl-cert', '${apache-conf-ssl:cert}', ssl_parameter_dict['cert']) }}
+{% elif caucase_url -%}
+key = ${apache-conf-ssl:key}
+cert = ${apache-conf-ssl:cert}
+
+{{ request_cert('erp5', 'instance.apache@erp5') }}
 {% else %}
 recipe = plone.recipe.command
 command = "{{ parameter_dict['openssl'] }}/bin/openssl" req -newkey rsa -batch -new -x509 -days 3650 -nodes -keyout "${:key}" -out "${:cert}"
@@ -191,12 +257,15 @@ cert = ${apache-ssl-ca:rendered}
 crl = ${apache-ssl-crl:rendered}
 {{ simplefile('apache-ssl-ca', '${apache-conf-ssl:ca-cert}', ssl_parameter_dict['ca-cert']) }}
 {{ simplefile('apache-ssl-crl', '${apache-conf-ssl:crl}', ssl_parameter_dict['crl']) }}
+{% elif caucase_url -%}
+cert = ${apache-conf-ssl:ca-cert}
+crl = ${apache-conf-ssl:crl}
+
 {% else %}
 cert =
 crl =
 {%- endif %}
 
-
 {% set apache_service_log_list = {} -%}
 {% for family_name, (_, _, _, authentication) in apache_dict.items() -%}
 {%   if authentication -%}
@@ -211,7 +280,7 @@ post = test ! -s ${apache-conf-parameter-dict:pid-file} || {{ parameter_dict['bi
 {%   endif -%}
 {% endfor -%}
 
-[logrotate-apache]
+[{{ section('logrotate-apache') }}]
 < = logrotate-entry-base
 name = apache
 log = ${apache-conf-parameter-dict:error-log} ${apache-conf-parameter-dict:access-log}
@@ -222,7 +291,7 @@ recipe = slapos.cookbook:mkdirectory
 apache-conf = ${:etc}/apache
 bin = ${buildout:directory}/bin
 etc = ${buildout:directory}/etc
-promise = ${directory:etc}/promise
+promise = ${:etc}/promise
 services = ${:etc}/run
 var = ${buildout:directory}/var
 run = ${:var}/run
@@ -235,39 +304,33 @@ newcerts = ${:ca-dir}/newcerts
 crl = ${:ca-dir}/crl
 apachedex = ${monitor-directory:private}/apachedex
 
-[monitor-generate-apachedex-report]
+[{{ section('monitor-generate-apachedex-report') }}]
 recipe = slapos.cookbook:wrapper
 wrapper-path = ${monitor-directory:reports}/${:command}
-command-line = "{{ parameter_dict['run-apachedex-location'] }}" "{{ parameter_dict['apachedex-location'] }}" "${directory:apachedex}" --default "${apachedex-parameters:default}" --apache-log-list "${apachedex-parameters:apache-log-list}" --base-list "${apachedex-parameters:base-list}" --skip-base-list "${apachedex-parameters:skip-base-list}" --erp5-base-list "${apachedex-parameters:erp5-base-list}"
-command = apachedex_every_3_hour
+command-line = "{{ parameter_dict['run-apachedex-location'] }}" "{{ parameter_dict['apachedex-location'] }}" "${directory:apachedex}" ${monitor-publish-parameters:monitor-base-url}/private/apachedex --apache-log-list "${apachedex-parameters:apache-log-list}" --config "${apachedex-parameters:configuration}"
+command = apachedex_every_23_hour
 
 [apachedex-parameters]
-default_parameter = 
 # XXX - Sample log file with curent date: apache_access.log-%(date)s.gz
 # which will be equivalent to apache_access.log-20150112.gz if the date is 2015-01-12
 apache-log-list = ${apache-conf-parameter-dict:access-log}
-default = ${monitor-directory:etc}/apdex_default
-base-list = ${monitor-directory:etc}/apdex_base_list
-skip-base-list = ${monitor-directory:etc}/apdex_skip_base_list
-erp5-base-list = ${monitor-directory:etc}/apdex_erp5_base_list
+configuration = {{ slapparameter_dict['apachedex-configuration'] }}
+promise-threshold = {{ slapparameter_dict['apachedex-promise-threshold'] }}
+
+[{{ section('monitor-promise-apachedex-result') }}]
+recipe = slapos.cookbook:wrapper
+wrapper-path = ${directory:promise}/check-apachedex-result
+command-line = "{{ parameter_dict['promise-check-apachedex-result'] }}" --apachedex_path "${directory:apachedex}" --status_file ${monitor-directory:private}/apachedex.report.json --threshold "${apachedex-parameters:promise-threshold}"
 
 [monitor-instance-parameter]
 monitor-httpd-ipv6 = {{ (ipv6_set | list)[0] }}
 monitor-httpd-port = {{ next_port() }}
-monitor-title = Balancer monitor
+monitor-title = {{ slapparameter_dict['name'] }}
 password = {{ slapparameter_dict['monitor-passwd'] }}
-instance-configuration = 
-  file apachedex-default ${apachedex-parameters:default}
-  file apachedex-base-list ${apachedex-parameters:base-list}
-  file apachedex-skip-base-list ${apachedex-parameters:skip-base-list}
-  file apachedex-erp5-base-list ${apachedex-parameters:erp5-base-list}
 
 [buildout]
 extends =
   {{ logrotate_cfg }}
   {{ parameter_dict['template-monitor'] }}
 parts +=
-  publish
-  logrotate-apache
-  monitor-generate-apachedex-report
   {{ part_list | join('\n  ') }}

software/slapos-master/instance-create-erp5-site-real.cfg.in

-[directory]
-recipe = slapos.cookbook:mkdirectory
-etc = ${buildout:directory}/etc
-services = ${:etc}/run
-promise = ${:etc}/promise
-
-[erp5-bootstrap]
-recipe = slapos.cookbook:erp5.bootstrap
-runner-path = ${directory:services}/erp5-bootstrap
-{# Note: a random domain name will be picked if several point to the same IP -#}
-{% set reverse_hosts = {} -%}
-{% for x, y in publish['hosts-dict'].iteritems() -%}
-{%   do reverse_hosts.__setitem__(y, x) -%}
-{% endfor -%}
-{# XXX: Expect the first database to be the one to use for catalog. -#}
-{% set mysql_parsed = urlparse.urlparse(publish['mariadb-database-list'][0]) -%}
-mysql-url = {{ dumps(urlparse.urlunparse(mysql_parsed[:1] + (mysql_parsed.username + ":" + mysql_parsed.password + "@" + reverse_hosts.get(mysql_parsed.hostname, mysql_parsed.hostname) + ':' ~ mysql_parsed.port, ) + mysql_parsed[2:])) }}
-{# Pick the first http[s] family found, they should be all equivalent anyway. -#}
-{# Don't pick the https[s] configurated with ssl-authenticat=true. By convention, this family name contain 'service'. -#}
-{% set family_list = [] -%}
-{% for key, value in publish.items() -%}
-{%   if key.startswith('family-') and value.startswith('http') and not 'service' in key -%}
-{%     do family_list.append(value.split('://', 1)) -%}
-{%   endif -%}
-{%  endfor -%}
-zope-url = {{ dumps(family_list[0][0] + '://' + publish['inituser-login'] + ':' + publish_early['inituser-password'] + '@' + family_list[0][1] + '/' + publish['site-id']) }}
-
-[promise-erp5-site]
-recipe = slapos.cookbook:check_url_available
-url = ${erp5-bootstrap:zope-url}
-path = ${directory:promise}/erp5-site
-dash_path = {{ parameter_dict['dash-location'] }}/bin/dash
-curl_path = {{ parameter_dict['curl-location'] }}/bin/curl
-
-[buildout]
-parts = promise-erp5-site
-eggs-directory = {{ eggs_directory }}
-develop-eggs-directory = {{ develop_eggs_directory }}

software/slapos-master/instance-erp5.cfg.in

-{% import "root_common" as root_common with context %}
+{% import "root_common" as root_common with context -%}
 {% set frontend_dict = slapparameter_dict.get('frontend', {}) -%}
 {% set has_frontend = frontend_dict.get('software-url', '') != '' -%}
 {% set site_id = slapparameter_dict.get('site-id', 'erp5') -%}
@@ -9,7 +9,9 @@
 {% set has_jupyter = jupyter_dict.get('enable', jupyter_enable_default.lower() in ('true', 'yes')) -%}
 {% set jupyter_zope_family = jupyter_dict.get('zope-family', '') -%}
 {% set monitor_base_url_dict = {} -%}
+{% set caucase_url = slapparameter_dict.get('caucase', {}).pop('url', '') -%}
 {% set monitor_dict = slapparameter_dict.get('monitor', {}) %}
+{% set crl_update_period = slapparameter_dict.get('caucase', {}).pop('crl-update-periodicity', 'daily') -%}
 [request-common]
 <= request-common-base
 config-use-ipv6 = {{ dumps(slapparameter_dict.get('use-ipv6', False)) }}
@@ -52,6 +54,14 @@ config-name = {{ name }}
 connection-url = smtp://127.0.0.2:0/
 {%- endif %}
 
+{% if caucase_url -%}
+{%   do publish_dict.__setitem__('caucase-http-url', caucase_url) -%}
+[request-caucase]
+connection-http-url = {{ caucase_url }}
+{%- else %}
+{{ request('caucase', 'caucase', 'caucase', {'server-port': 8890, 'server-https-port': 8891, 'auto-sign-csr-amount': 2}, {'http-url': True, 'https-url': False}) }}
+{% endif -%}
+
 {# ZODB -#}
 {% set zodb_dict = {} -%}
 {% set storage_dict = {} -%}
@@ -95,16 +105,16 @@ recipe = slapos.cookbook:publish-early
 {%- if neo %}
   neo-cluster gen-neo-cluster:name
 {%-  if neo[0] %}
-neo-cluster = {{ neo[0] }}
+neo-cluster = {{ dumps(neo[0]) }}
 {%-  endif %}
 {%- endif %}
 {%- set inituser_password = slapparameter_dict.get('inituser-password') %}
 {%- if inituser_password %}
-inituser-password = {{ inituser_password }}
+inituser-password = {{ dumps(inituser_password) }}
 {%- endif %}
 {%- set deadlock_debugger_password = slapparameter_dict.get('deadlock-debugger-password') -%}
 {%- if deadlock_debugger_password %}
-deadlock-debugger-password = {{ deadlock_debugger_password }}
+deadlock-debugger-password = {{ dumps(deadlock_debugger_password) }}
 {%- endif %}
 
 [gen-password]
@@ -133,12 +143,13 @@ return =
   zope-address-list
   hosts-dict
   monitor-base-url
-{% set bt5_default_list = 'erp5_full_text_myisam_catalog erp5_configurator_standard erp5_configurator_maxma_demo erp5_configurator_ung erp5_configurator_run_my_doc slapos_configurator' -%}
+{% set bt5_default_list = 'erp5_full_text_myisam_catalog erp5_configurator_standard erp5_configurator_maxma_demo erp5_configurator_run_my_doc slapos_configurator' -%}
 {% if has_jupyter -%}
 {%   set bt5_default_list = bt5_default_list + ' erp5_data_notebook' -%}
 {% endif -%}
 config-bt5 = {{ dumps(slapparameter_dict.get('bt5', bt5_default_list)) }}
 config-bt5-repository-url = {{ dumps(slapparameter_dict.get('bt5-repository-url', local_bt5_repository)) }}
+config-caucase-url = ${request-caucase:connection-http-url}
 config-cloudooo-url = ${request-cloudooo:connection-url}
 config-deadlock-debugger-password = ${publish-early:deadlock-debugger-password}
 config-developer-list = {{ dumps(slapparameter_dict.get('developer-list', [inituser_login])) }}
@@ -171,23 +182,22 @@ config-tidstorage-port = ${request-zodb:connection-tidstorage-port}
 software-type = zope
 
 {% set zope_family_dict = {} -%}
-{% set jupyter_zope_family_default = [] -%}
 {% set zope_backend_path_dict = {} -%}
 {% set ssl_authentication_dict = {} -%}
-
+{% set jupyter_zope_family_default = [] -%}
 {% for custom_name, zope_parameter_dict in zope_partition_dict.items() -%}
 {%   set partition_name = 'zope-' ~ custom_name -%}
 {%   set section_name = 'request-' ~ partition_name -%}
-{%   set backend_path = zope_parameter_dict.get('backend-path', '/') % {'site-id': site_id} %}
 {%   set zope_family = zope_parameter_dict.get('family', 'default') -%}
+{%   set backend_path = zope_parameter_dict.get('backend-path', '/') % {'site-id': site_id} %}
 {#   # default jupyter zope family is first zope family. -#}
 {#   # use list.append() to update it, because in jinja2 set changes only local scope. -#}
 {%   if not jupyter_zope_family_default -%}
 {%     do jupyter_zope_family_default.append(zope_family) -%}
 {%   endif -%}
 {%   do zope_family_dict.setdefault(zope_family, []).append(section_name) -%}
-{%   do zope_backend_path_dict.setdefault(zope_parameter_dict.get('family', 'default'), backend_path) -%}
-{%   do ssl_authentication_dict.setdefault(zope_parameter_dict.get('family', 'default'), zope_parameter_dict.get('ssl-authentication', False)) -%}
+{%   do zope_backend_path_dict.__setitem__(zope_family, backend_path) -%}
+{%   do ssl_authentication_dict.__setitem__(zope_family, zope_parameter_dict.get('ssl-authentication', False)) -%}
 [{{ section_name }}]
 <= request-zope-base
 name = {{ partition_name }}
@@ -201,7 +211,6 @@ config-longrequest-logger-interval = {{ dumps(zope_parameter_dict.get('longreque
 config-longrequest-logger-timeout = {{ dumps(zope_parameter_dict.get('longrequest-logger-timeout', 1)) }}
 config-port-base = {{ dumps(zope_parameter_dict.get('port-base', 2200)) }}
 config-webdav = {{ dumps(zope_parameter_dict.get('webdav', False)) }}
-config-name = {{ partition_name }}
 {% endfor -%}
 
 {# if not explicitly configured, connect jupyter to first zope family, which  -#}
@@ -211,7 +220,6 @@ config-name = {{ partition_name }}
 {% endif -%}
 
 {# We need to concatenate lists that we cannot read as lists, so this gets hairy. -#}
-{% set zope_address_list_id_dict = {} -%}
 {% set zope_family_parameter_dict = {} -%}
 {% for family_name, zope_section_id_list in zope_family_dict.items() -%}
 {%   for zope_section_id in zope_section_id_list -%}
@@ -300,6 +308,7 @@ return =
   {{ family }}-v6
 {% endfor -%}
 {% do monitor_base_url_dict.__setitem__('request-balancer', '${' ~ 'request-balancer' ~ ':connection-monitor-base-url}') -%}
+
 config-zope-family-dict = {{ dumps(zope_family_parameter_dict) }}
 config-tcpv4-port = {{ dumps(balancer_dict.get('tcpv4-port', 2150)) }}
 {% for zope_section_id, name in zope_address_list_id_dict.items() -%}
@@ -307,16 +316,17 @@ config-{{ name }} = {{ ' ${' ~ zope_section_id ~ ':connection-zope-address-list}
 {% endfor -%}
 # XXX: should those really be same for all families ?
 config-haproxy-server-check-path = {{ dumps(balancer_dict.get('haproxy-server-check-path', '/') % {'site-id': site_id}) }}
-config-backend-path = {{ dumps(balancer_dict.get('apache-backend-path', '/') % {'site-id': site_id}) }}
 config-ssl = {{ dumps(balancer_dict.get('ssl', {})) }}
-config-backend-path-dict = {{ dumps(zope_backend_path_dict) }}
-config-ssl-authentication-dict = {{ dumps(ssl_authentication_dict) }}
-config-shared-certificate-authority-path = ${directory:ca-dir} 
 config-monitor-passwd = ${monitor-htpasswd:passwd}
 config-name = ${:name}
+config-caucase-url = ${request-caucase:connection-http-url}
+config-crl-update-periodicity = {{ crl_update_period }}
+config-shared-certificate-authority-path = ${directory:ca-dir} 
+config-backend-path-dict = {{ dumps(zope_backend_path_dict) }}
+config-ssl-authentication-dict = {{ dumps(ssl_authentication_dict) }}
 config-apachedex-promise-threshold = {{ dumps(monitor_dict.get('apachedex-promise-threshold', 70)) }}
 config-apachedex-configuration = {{ dumps(monitor_dict.get('apachedex-configuration',
- '--erp5-base "/erp5(/|$|/\?)" --skip-user-agent Zabbix  --error-detail --js-embed --quiet')) }}
+'--erp5-base "/erp5(/|$|/\?)" --skip-user-agent Zabbix  --error-detail --js-embed --quiet')) }}
 
 [request-frontend-base]
 {% if has_frontend -%}
@@ -365,10 +375,10 @@ monitor-httpd-port = 8386
 
 [buildout]
 extends = {{ template_monitor }}
+
 parts +=
   apache-certificate-authority
   fix-ca-folder
-  publish
   monitor-base
 
 [monitor-conf-parameters]

software/slapos-master/software.cfg

@@ -58,11 +58,6 @@ filename = instance-balancer.cfg.in
 url = ${:_profile_base_location_}/${:filename}
 filename = apache-backend.conf.in 
 
-[template-create-erp5-site-real]
-< = download-base-part
-filename = instance-create-erp5-site-real.cfg.in
-
-
 # Test Suite: SLAPOS-MASTER-MASTER ran at 2017/07/04 08:41:0.615347 UTC
 #             6 failures, 1 errors, 1316 total, status: FAIL
  

software/slapos-testing/software.cfg

@@ -97,4 +97,4 @@ mode = 640
 Pygments = 2.1.3
 collective.recipe.template = 1.10
 plone.recipe.command = 1.1
-slapos.recipe.template = 3.0
+slapos.recipe.template = 4.1

software/slaprunner/buildout.hash.cfg

@@ -19,7 +19,7 @@ md5sum = 713db528880282d568278f09458d2aab
 
 [template-runner]
 filename = instance-runner.cfg
-md5sum = 8f49df215a1596efcec94e2cca009711
+md5sum = baf70a9f2f433ea0f2f5139700e53b1f
 
 [template-runner-import-script]
 filename = template/runner-import.sh.jinja2
@@ -31,7 +31,7 @@ md5sum = 5cfa49bcf20612844e1c50a85740d0b3
 
 [template-runner-export-script]
 filename = template/runner-export.sh.jinja2
-md5sum = 78b2454951e6dbe124612eafebcf5690
+md5sum = 98ce179badc6af5979a64a7c3d0a2ceb
 
 [instance-runner-export]
 filename = instance-runner-export.cfg.in
@@ -63,11 +63,7 @@ md5sum = f8446fcf254b4929eb828a9a1d7e5f62
 
 [template-bash-profile]
 filename = template/bash_profile.in
-md5sum = 1c88cbca0c1e705eeb4b544ef4616097
-
-[template-bashrc]
-filename = template/bashrc.in
-md5sum = d0b9060ec96d4d5b188e36c84240ea7d
+md5sum = 712ca70488051f97e7a7b11a02a06bb1
 
 [template-supervisord]
 filename = template/supervisord.conf.in

software/slaprunner/common.cfg

@@ -13,6 +13,7 @@ extends =
   ../../component/nano/buildout.cfg
   ../../component/nginx/buildout.cfg
   ../../component/openssh/buildout.cfg
+  ../../component/mosh/buildout.cfg
   ../../component/rsync/buildout.cfg
   ../../component/pycurl/buildout.cfg
   ../../component/python-2.7/buildout.cfg
@@ -137,15 +138,10 @@ filename = resilient_software_release_information.py.in
 < = template-download-base
 filename = slapos-slapuser-script.in
 
-[template-bashrc]
-< = template-download-base
-filename = bashrc.in
-
 [eggs]
 recipe = zc.recipe.egg
 eggs =
   ${pycurl:egg}
-  collective.recipe.environment
   collective.recipe.template
   cns.recipe.symlink
   erp5.util

software/slaprunner/instance-runner.cfg

@@ -167,7 +167,7 @@ minishell_cwd_file = $${directory:etc}/.minishell-cwd
 minishell_history_file = $${directory:etc}/.minishell_history
 software_info_json = $${runnerdirectory:home}/software_info.json
 instance_info_json = $${runnerdirectory:home}/instance_info.json
-path = $${shell:path}
+path = $${shell-environment:path}
 instance_name = $${slap-parameter:instance-name}
 
 default_repository = $${slap-parameter:slapos-repository}
@@ -201,7 +201,8 @@ recipe = slapos.cookbook:wrapper
 arguments = --server_url=$${slap-connection:server-url} --key_file=$${slap-connection:key-file} --cert_file=$${slap-connection:cert-file} --computer_id=$${slap-connection:computer-id} --partition_id=$${slap-connection:partition-id}
 command-line = ${buildout:directory}/bin/slaprunnertest $${:arguments}
 wrapper-path = $${directory:bin}/runTestSuite
-environment = RUNNER_CONFIG=$${slapos-cfg:rendered}
+environment = PATH=$${shell-environment:path}
+  RUNNER_CONFIG=$${slapos-cfg:rendered} 
 parameters-extra = true
 
 # Deploy openssh-server
@@ -226,7 +227,7 @@ template = inline:
   PasswordAuthentication no
   PubkeyAuthentication yes
   AuthorizedKeysFile $${buildout:directory}/.ssh/authorized_keys
-  ForceCommand if [ -z "$SSH_ORIGINAL_COMMAND" ]; then ${bash:location}/bin/bash -l; else eval "$SSH_ORIGINAL_COMMAND"; fi
+  ForceCommand if [ -z "$SSH_ORIGINAL_COMMAND" ]; then $${shell-environment:shell} -l; else SHELL=$${shell-environment:shell} PATH=$${shell-environment:path} eval "$SSH_ORIGINAL_COMMAND"; fi
   Subsystem sftp ${openssh:location}/libexec/sftp-server
 
 [runner-sshd-raw-server]
@@ -424,7 +425,7 @@ recipe = slapos.cookbook:wrapper
 command-line = $${gunicorn:bin_gunicorn} slapos.runner.run:app -p $${gunicorn:path_pid} -b unix:$${gunicorn:socket} -e RUNNER_CONFIG=$${slaprunner:slapos.cfg} --error-logfile $${directory:log}/$${:error-log-file} --timeout 200 --threads 3 --log-level error --preload
 error-log-file = gunicorn-error.log
 wrapper-path = $${gunicorn:bin_launcher}
-environment = PATH=$${environ:PATH}:${git:location}/bin/
+environment = PATH=$${shell-environment:path}
   RUNNER_CONFIG=$${slaprunner:slapos.cfg}
   LANG=en_GB.UTF-8
 
@@ -640,18 +641,11 @@ template = inline:
     --disable-ssl \
     --disable-ssl-menu \
     --unixdomain-only=$${:socket}:$(id -u):$(id -g):0600 \
-    --service "/:$(id -u):$(id -g):HOME:$${shell:wrapper} -l"
+    --service "/:$(id -u):$(id -g):HOME:$${shell-environment:shell} -l"
 
-[shell]
-recipe = slapos.cookbook:shell
-wrapper = $${directory:bin}/bash
+[shell-environment]
 shell = ${bash:location}/bin/bash
-home = $${buildout:directory}
-path = $${environ:PATH}:/usr/bin:/bin/:${nano:location}/bin:${vim:location}/bin:${screen:location}/bin:${git:location}/bin:${tig:location}/bin
-ps1 = "\\w> "
-
-[environ]
-recipe = collective.recipe.environment
+path = ${nano:location}/bin:${vim:location}/bin:${screen:location}/bin:${git:location}/bin:${curl:location}/bin:${python2.7:location}/bin:${tig:location}/bin:${zip:location}/bin:${mosh:location}/bin:${bash:location}/bin:$${buildout:directory}/bin/:/usr/bin:/bin/
 
 [prepare-software]
 recipe = slapos.cookbook:wrapper
@@ -712,16 +706,10 @@ recipe = slapos.recipe.template:jinja2
 template = ${template-bash-profile:location}/${template-bash-profile:filename}
 rendered = $${buildout:directory}/.bash_profile
 context =
-    raw path $PATH:${nano:location}/bin:${vim:location}/bin:${screen:location}/bin:${git:location}/bin:${curl:location}/bin:${python2.7:location}/bin:${tig:location}/bin:${zip:location}/bin
+    raw path $${shell-environment:path}
+    raw shell $${shell-environment:shell}
     key workdir runnerdirectory:home
     
-[bashrc]
-recipe = slapos.recipe.template:jinja2
-template = ${template-bashrc:location}/${template-bashrc:filename}
-rendered = $${buildout:directory}/.bashrc
-context =
-    raw bin_folder ${buildout:bin-directory}
-
 #---------------------------
 #--
 #-- supervisord managing slaprunner automation features
@@ -734,7 +722,7 @@ exitcodes = 0
 logfile = $${directory:log}/supervisord.log
 no_logfile = NONE
 numprocs = 1
-path = $${shell:path}
+path = $${shell-environment:path}
 pidfile = $${directory:run}/supervisord.pid
 ip = $${slaprunner:ipv4}
 server = $${:ip}:$${:port}

software/slaprunner/software.cfg

@@ -10,14 +10,13 @@ extends = common.cfg
 Flask-Auth = 0.85
 apache-libcloud = 1.2.1
 cns.recipe.symlink = 0.2.3
-collective.recipe.environment = 0.2.0
 futures = 3.0.5
 gitdb = 0.6.4
-gunicorn = 19.4.5
+gunicorn = 19.7.1
 prettytable = 0.7.2
 pycurl = 7.43.0
-slapos.recipe.template = 3.0
-slapos.toolbox = 0.72
+slapos.recipe.template = 4.1
+slapos.toolbox = 0.73
 smmap = 0.9.0
 
 # Required by:

software/slaprunner/template/bash_profile.in

@@ -4,6 +4,7 @@
 
 cd {{ workdir }}
 export PATH={{- path }}
+export SHELL={{- shell }}
 export PS1="$ "
 
 if [ -f "$HOME/.bashrc" ] ; then
@@ -29,13 +30,4 @@ alias l='ls -CF'
 
 echo "Welcome to SlapOS slaprunner shell"
 echo 
-# TODO: how to run slapos ?
-#echo "You can use the following slapos CLI commands:"
-#echo "slapos node software"
-#echo "slapos node instance"
-#echo "slapos request"
-#echo ""
-# XXX for now we can use:
-#echo ~/bin/supervisorctl -c ~/etc/supervisorctl.conf
-#echo ~/bin/supervisorctl -c ~/srv/runner/instance/etc/supervisorctl.conf
 

software/slaprunner/template/bashrc.in

-export PATH={{ bin_folder }}:$PATH

software/slaprunner/template/runner-export.sh.jinja2

@@ -6,6 +6,7 @@ umask 077
 # Exit on any error, to prevent inconsistent backup
 # Error on unset variable expansion
 set -eu
+set -o pipefail
 
 # Redirect output to log
 exec > >(tee -ai '{{ output_log_file }}')
@@ -22,18 +23,10 @@ rsync () {
   # Workaround for bug https://bugzilla.samba.org/show_bug.cgi?id=3653
   IGNOREEXIT=24
   IGNOREOUT='^(file has vanished: |rsync warning: some files vanished before they could be transferred)'
-  set -o pipefail
 
   set -x
-  '{{ rsync_binary }}' -rlptgov --stats --safe-links --ignore-missing-args --delete --delete-excluded "$@" 2>1 | (egrep -v "$IGNOREOUT" || true)
+  '{{ rsync_binary }}' -rlptgov --stats --safe-links --ignore-missing-args --delete --delete-excluded "$@" 2>&1 | (egrep -v "$IGNOREOUT" || true) || [ $? = "$IGNOREEXIT" ]
   set +x
-
-  ret=$?
-  if [[ $ret == $IGNOREEXIT ]]; then
-    ret=0
-  fi
-
-  exit $ret
 }
 
 (

software/test-slave-instance-deployment/software.cfg

@@ -59,7 +59,7 @@ eggs = collective.recipe.template
 collective.recipe.template = 1.11
 plone.recipe.command = 1.1
 slapos.recipe.build = 0.28
-slapos.recipe.template = 3.0
+slapos.recipe.template = 4.1
 
 # Replicate slapos stack, but without shacache to not have to compile the entire world for a simple test.
 [buildout]

software/varnish/software.cfg

@@ -62,4 +62,4 @@ md5sum = 0ea12a4ad2d2e3d406476e35b8d3e3fb
 mode = 640
 
 [versions]
-slapos.recipe.template = 3.0
+slapos.recipe.template = 4.1

software/wendelin/software-kerastensorflow.cfg

@@ -2,13 +2,16 @@
 extends =
   ../../component/keras/buildout.cfg
   ../../component/chainer/buildout.cfg
+  ../../component/pytorch/buildout.cfg
   software.cfg
 parts +=
   keras-egg
   chainer-egg
+  pytorch-egg
 
 [eggs]
 eggs +=
+  ${tensorboard-build-install-egg:egg}
   ${tensorflow-build-install-egg:egg}
   ${protobuf-python:egg}
   ${h5py:egg}
@@ -19,6 +22,9 @@ eggs +=
   filelock
   nose
   ${chainer:egg}
+  ${pytorch-build-install-egg:egg}
+  ${python-cocoapi-build-install-egg:egg}
+  torchvision
 
 [cuda]
 # If you use cuda, please adjust paramters for your environment

stack/caddy/Caddyfile.in

+[$${caddy-configuration:ipv6}]:$${caddy-configuration:port}
+
+log $${caddy-configuration:access_log}
+
+root  $${directory:public_html}
\ No newline at end of file

stack/caddy/buildout.cfg

+[buildout]
+extends =
+  ../../component/golang/buildout.cfg
+  ../../stack/slapos.cfg
+  gowork.cfg
+
+parts =
+  gowork
+  slapos-cookbook
+  instance-profile
+  template-caddy
+  caddy
+
+[gowork]
+install =
+  github.com/mholt/caddy
+  
+[instance-profile]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/instance.cfg.in
+md5sum = ae83c35995ce14432ded78ca30cab61b
+output =${buildout:directory}/instance.cfg
+filename = instance.cfg
+mode = 0644
+
+[template-caddyfile]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/Caddyfile.in
+md5sum = 616f9c7cb788e1f1d7cee2093a7dc6ef
+output = ${buildout:directory}/Caddyfile.in
+mode = 0644
+
+[template-caddy]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/instance-caddy.cfg.in
+md5sum = 045022e5698badca5dbdd6b95518f8e1
+output = ${buildout:directory}/instance-caddy.cfg.in
+mode = 0644
+
+[template-public-html]
+recipe = slapos.recipe.template
+url =  ${:_profile_base_location_}/index.html
+md5sum = b5794ac8b10ed90173ad566e6e324b35
+output =  ${buildout:directory}/index.html
+mode = 0644
+
+[caddy]
+recipe  = slapos.recipe.cmmi
+path    = ${go_github.com_mholt_caddy:location}
+go      = ${gowork:golang}/bin/go
+configure-command = :
+make-targets =
+make-binary = cd ${:path}/caddy && ${:go} install -v
+environment =
+  PATH=${pkgconfig:location}/bin:${gowork:golang}/bin:${buildout:bin-directory}:%(PATH)s
+  GOPATH=${gowork:directory}
+output =  ${gowork:bin}/caddy
\ No newline at end of file

stack/caddy/gowork.cfg

+# Code generated by gowork-snapshot; DO NOT EDIT.
+
+# list of go git repositories to fetch
+[gowork.goinstall]
+depends_gitfetch  =
+    ${go_github.com_mholt_caddy:recipe}
+
+
+[go_github.com_mholt_caddy]
+<= go-git-package
+go.importpath = github.com/mholt/caddy
+repository    = https://github.com/mholt/caddy.git
+revision  = v0.10.10-0-gc4dfbb9956

stack/caddy/index.html

+<h1>Welcome</h1>
\ No newline at end of file

stack/caddy/instance-caddy.cfg.in

+[buildout]
+parts =
+  caddy-service
+  caddy-configuration
+  public-html
+  publish-connection-information
+
+
+eggs-directory = ${buildout:eggs-directory}
+develop-eggs-directory = ${buildout:develop-eggs-directory}
+offline = true
+
+[directory]
+recipe = slapos.cookbook:mkdirectory
+etc = $${buildout:directory}/etc
+bin = $${buildout:directory}/bin
+srv = $${buildout:directory}/srv
+var = $${buildout:directory}/var
+service = $${:etc}/service
+public_html =  $${buildout:directory}/public_html
+run = $${:var}/run
+log = $${:var}/log
+
+#www = $${:srv}/www
+#ssl = $${:etc}/ssl
+
+#################################
+# caddy service
+#################################
+[caddy-service]
+recipe = slapos.cookbook:wrapper 
+command-line = ${caddy:output}  -conf $${caddy-configuration:output}
+wrapper-path = $${directory:service}/caddy
+output = $${:wrapper-path}
+
+[caddy-configuration]
+recipe = slapos.recipe.template
+url = ${template-caddyfile:output}
+output = $${directory:etc}/Caddyfile
+mode = 0600
+access_log = $${directory:log}/caddy-access.log
+error_log = $${directory:log}/caddy-error.log
+ipv6 = $${slap-network-information:global-ipv6}
+local_ip = $${slap-network-information:local-ipv4}
+port = 9443
+
+[public-html]
+recipe = slapos.recipe.template
+url = ${template-public-html:output}
+output = $${directory:public_html}/index.html
+mode = 0600
+
+[publish-connection-information]
+recipe = slapos.cookbook:publish
+url = http://[$${caddy-configuration:ipv6}]:$${caddy-configuration:port}
\ No newline at end of file

stack/caddy/instance.cfg.in

+#############################
+#
+# Deploy caddy instance
+#
+#############################
+[buildout]
+parts =
+  switch-softwaretype
+  
+#  publish-connection-parameter
+
+# Define egg directories to be the one from Software Release
+# (/opt/slapgrid/...)
+# Always the same.
+eggs-directory = ${buildout:eggs-directory}
+develop-eggs-directory = ${buildout:develop-eggs-directory}
+offline = true
+
+[switch-softwaretype]
+recipe = slapos.cookbook:softwaretype
+default = $${:caddy}
+caddy = $${dynamic-template-caddy:rendered}
+
+[dynamic-template-caddy]
+recipe = slapos.recipe.template:jinja2
+template = ${template-caddy:output}
+rendered = $${buildout:parts-directory}/${:_buildout_section_name_}/${:filename}
+filename = instance-caddy.cfg
+
+[slap-connection]
+computer-id = {slap_connection:computer_id}
+partition-id = $${slap_connection:partition_id}
+server-url = $${slap_connection:server_url}
+software-release-url = $${slap_connection:software_release_url}
+key-file = $${slap_connection:key_file}
+cert-file = $${slap_connection:cert_file}
+
+[instance-parameter]
+# Fetch arbitrary parameters defined by the user in SlapOS Master for his instance.
+# We use the slapconfiguration recipe with a few parameters (partition id,
+# computer id, certificate, etc).
+# It will then authenticate to SlapOS Master and fetch the instance parameters.
+# The parameters are accessible from {instance-parameter:configuration.name-of-parameter}
+# Always the same. Just copy/paste.
+# See docstring of slapos.cookbook:slapconfiguration for more information.
+recipe = slapos.cookbook:slapconfiguration
+computer = $${slap_connection:computer_id}
+partition = $${slap_connection:partition_id}
+url = $${slap_connection:server_url}
+key = $${slap_connection:key_file}
+cert = $${slap_connection:cert_file}

stack/caucase/buildout.cfg

@@ -94,8 +94,8 @@ caucase = 0.1.4
 futures = 3.1.1
 gitdb2 = 2.0.2
 gunicorn = 19.7.1
-slapos.recipe.template = 3.0
-slapos.toolbox = 0.72
+slapos.recipe.template = 4.1
+slapos.toolbox = 0.73
 smmap2 = 2.0.3
 
 # Required by:

stack/caucase/buildout.hash.cfg

@@ -28,7 +28,7 @@ md5sum = a317d2f948cd3d16c860d05cc07ecf42
 
 [template-caucase]
 filename = instance-caucase.cfg.jinja2.in
-md5sum = 7db9e8bf23cf4689e7986c381b94d2cb
+md5sum = d31d4c9855d653d3d74c6133a7d85530
 
 [instance-caucase]
 filename = instance.cfg.in

stack/caucase/instance-caucase.cfg.jinja2.in

@@ -157,7 +157,7 @@ mode = 700
 
 [caucase-gunicorn]
 recipe = slapos.cookbook:wrapper
-socket = ${directory:ca-dir}/ca.flaskserver.sock
+socket = ${directory:ca-dir}/ng.sock
 command-line =  {{ gunicorn_bin }} caucase.wsgi:app -b unix:${:socket} -e CA_CONFIGURATION_FILE=${caucase-conf:output} --error-logfile ${:log-file} --pid ${:pid-file} --capture-output --timeout 60 --threads 2 --log-level error --preload
 log-file = ${directory:log}/ca-gunicorn-error.log
 pid-file = ${directory:run}/ca-gunicorn.pid

stack/cloudooo.cfg

@@ -89,7 +89,7 @@ PasteScript = 2.0.2
 WSGIUtils = 0.7
 python-magic = 0.4.6
 rdiff-backup = 1.0.5+SlapOSPatched001
-slapos.recipe.template = 3.0
+slapos.recipe.template = 4.1
 
 # Required by:
 # PasteScript==2.0

stack/erp5/buildout.cfg

@@ -167,6 +167,10 @@ stop-on-error = true
 update-command = ${:command}
 command = grep parts ${buildout:develop-eggs-directory}/slapos.cookbook.egg-link
 
+[mariadb-start-clone-from-backup]
+<= download-base
+mode = 755
+
 [mariadb-resiliency-after-import-script]
 <= download-base
 mode = 755
@@ -276,6 +280,7 @@ context =
     key mariadb_location mariadb:location
     key mariadb_resiliency_after_import_script mariadb-resiliency-after-import-script:target
     key mariadb_slow_query_report_script mariadb-slow-query-report-script:target
+    key mariadb_start_clone_from_backup mariadb-start-clone-from-backup:target
     key matplotlibrc_location matplotlibrc:location
     key mesa_location mesa:location
     key onlyoffice_x2t_location onlyoffice-x2t:location
@@ -683,8 +688,6 @@ PyXML = 0.8.5
 Pympler = 0.4.3
 StructuredText = 2.11.1
 WSGIUtils = 0.7
-# ZEO 5 requires transaction >= 2
-ZEO = 4.3.1
 ZODB3 = 3.11.0
 # astroid 1.4.1 breaks testDynamicClassGeneration
 astroid = 1.3.8

stack/erp5/buildout.hash.cfg

@@ -15,15 +15,19 @@
 # not need these here).
 [mariadb-resiliency-after-import-script]
 filename = instance-mariadb-resiliency-after-import-script.sh.in
-md5sum = 844d62cd6f9d6e3d1d78d52de2b72a49
+md5sum = c1f1083bf6c911a0e65dcb841fba327d
 
 [mariadb-slow-query-report-script]
 filename = mysql-querydigest.sh.in
 md5sum = cfe6ab8ae54a521ecb269e9d9762cbeb
 
+[mariadb-start-clone-from-backup]
+filename = instance-mariadb-start-clone-from-backup.sh.in
+md5sum = 1af531c51f575a1d1362f2ca2d61620d
+
 [template-mariadb]
 filename = instance-mariadb.cfg.in
-md5sum = 99450bb6426770bd0c27f9dbec9ca542
+md5sum = 8ea5033142f450a2e90431817771cb44
 
 [template-kumofs]
 filename = instance-kumofs.cfg.in
@@ -67,7 +71,7 @@ md5sum = 0969fbb25b05c02ef3c2d437b2f4e1a0
 
 [template]
 filename = instance.cfg.in
-md5sum = f95174eee2de45d1e3ccb1d5f88355f1
+md5sum = 47d09a83d44f38d3ea62743f004e866b
 
 [monitor-template-dummy]
 filename = dummy.cfg
@@ -75,7 +79,7 @@ md5sum = d41d8cd98f00b204e9800998ecf8427e
 
 [template-erp5]
 filename = instance-erp5.cfg.in
-md5sum = 909c8eb4f1f2f2d58ad982cec67228bc
+md5sum = 33c8f97afe58058d428108fb933523ac
 
 [template-zeo]
 filename = instance-zeo.cfg.in
@@ -87,7 +91,7 @@ md5sum = 27d26c6380883cf3bd7b2f003f7888d8
 
 [template-balancer]
 filename = instance-balancer.cfg.in
-md5sum = c80648d5918ae865bc02cb8fccfc56e3
+md5sum = f64c568f1365eb1164f12f48fede9a99
 
 [template-haproxy-cfg]
 filename = haproxy.cfg.in

stack/erp5/instance-balancer.cfg.in

@@ -288,7 +288,7 @@ apachedex = ${monitor-directory:private}/apachedex
 recipe = slapos.cookbook:wrapper
 wrapper-path = ${monitor-directory:reports}/${:command}
 command-line = "{{ parameter_dict['run-apachedex-location'] }}" "{{ parameter_dict['apachedex-location'] }}" "${directory:apachedex}" ${monitor-publish-parameters:monitor-base-url}/private/apachedex --apache-log-list "${apachedex-parameters:apache-log-list}" --config "${apachedex-parameters:configuration}"
-command = apachedex_every_3_hour
+command = apachedex_every_23_hour
 
 [apachedex-parameters]
 # XXX - Sample log file with curent date: apache_access.log-%(date)s.gz
@@ -299,9 +299,8 @@ promise-threshold = {{ slapparameter_dict['apachedex-promise-threshold'] }}
 
 [{{ section('monitor-promise-apachedex-result') }}]
 recipe = slapos.cookbook:wrapper
-recipe =
 wrapper-path = ${directory:promise}/check-apachedex-result
-command-line = "{{ parameter_dict['promise-check-apachedex-result'] }}" --apachedex_file "${directory:apachedex}" --status_file ${monitor-directory:private}/apachedex.report.json --threshold "${apachedex-parameters:promise-threshold}"
+command-line = "{{ parameter_dict['promise-check-apachedex-result'] }}" --apachedex_path "${directory:apachedex}" --status_file ${monitor-directory:private}/apachedex.report.json --threshold "${apachedex-parameters:promise-threshold}"
 
 [monitor-instance-parameter]
 monitor-httpd-ipv6 = {{ (ipv6_set | list)[0] }}

stack/erp5/instance-erp5.cfg.in

@@ -143,7 +143,7 @@ return =
   zope-address-list
   hosts-dict
   monitor-base-url
-{% set bt5_default_list = 'erp5_full_text_myisam_catalog erp5_configurator_standard erp5_configurator_maxma_demo erp5_configurator_ung erp5_configurator_run_my_doc' -%}
+{% set bt5_default_list = 'erp5_full_text_myisam_catalog erp5_configurator_standard erp5_configurator_maxma_demo erp5_configurator_run_my_doc' -%}
 {% if has_jupyter -%}
 {%   set bt5_default_list = bt5_default_list + ' erp5_data_notebook' -%}
 {% endif -%}

stack/erp5/instance-mariadb-resiliency-after-import-script.sh.in

@@ -15,7 +15,6 @@ set -e
 mysql_executable='{{ mysql_executable }}'
 mariadb_data_directory='{{ mariadb_data_directory }}'
 mariadb_backup_directory='{{ mariadb_backup_directory }}'
-instance_directory='{{ instance_directory }}'
 pid_file='{{ pid_file }}'
 binlog_path='{{ binlog_path }}'
 server_executable='{{ server_executable }}'
@@ -71,7 +70,7 @@ fi
 echo "Importing data..."
 # Use latest dump XXX can contain funny characters
 dump=$(ls -r "$mariadb_backup_directory" | head -1)
-zcat "$mariadb_backup_directory/$dump" | $mysql_executable -u root --socket="$instance_directory/var/run/mariadb.sock"
+zcat "$mariadb_backup_directory/$dump" | $mysql_executable -u root
 RESTORE_EXIT_CODE=$?
 
 if [ $RESTORE_EXIT_CODE -eq 0 ]; then

stack/erp5/instance-mariadb-start-clone-from-backup.sh.in

+#!{{ dash }}
+set -eu
+
+if [ $# -ne 7 ]; then
+  echo "Bootstrap a mariadb instance from available backup data so it replicates from given master."
+  echo "  $0 <BACKUP FILE> <MASTER HOST> <MASTER PORT> <MASTER USER> <MASTER SSL CA FILE> <MASTER SSL CERT FILE> <MASTER SSL KEY FILE>"
+  exit 1
+fi
+
+BACKUP=$1
+MASTER_HOST=$2
+MASTER_PORT=$3
+MASTER_USER=$4
+MASTER_SSL_CA=$5
+MASTER_SSL_CERT=$6
+MASTER_SSL_KEY=$7
+
+CLIENT='{{ client }}'
+DATA_DIRECTORY='{{ data_directory }}'
+PID_FILE='{{ pid_file }}'
+SERVER='{{ server }}'
+UPDATE='{{ update }}'
+SOCKET='{{ socket }}'
+
+# Make sure mariadb is not already running
+if [ -e "$PID_FILE" ]; then
+  PID=$(cat "$PID_FILE")
+  if [ $? -ne 0 ]; then
+    echo "Cannot read Mariadb pidfile, assuming running. Aborting."
+    exit 1
+  fi
+  if kill -0 "$PID"; then
+    echo "Mariadb is already running with pid $PID. Aborting."
+    exit 1
+  fi
+fi
+
+BACKUP_HEAD="$(zcat "$BACKUP" | head -n 100)"
+
+SQL_CHANGE_MASTER=$(echo "$BACKUP_HEAD" | grep "^--\s*CHANGE MASTER TO " | sed "s/^--\s*//")
+if [ -z "$SQL_CHANGE_MASTER" ]; then
+  echo "'CHANGE MASTER TO' statement not found in given backup file."
+  echo "Is replication enabled on future master ?"
+  exit 1
+fi
+
+SQL_SET_GTID="$(echo "$BACKUP_HEAD" | grep "^--\s*SET GLOBAL gtid_slave_pos=" | sed "s/^--\s*//")"
+if [ -z "$SQL_SET_GTID" ]; then
+  echo "Info: GTID not found in backup, it will not be enabled."
+  MASTER_USE_GTID=0
+else
+  echo "Info: GTID found in backup, it will be enabled."
+  MASTER_USE_GTID=1
+fi
+
+echo "EXISTING DATABASE CONTENT WILL BE DESTROYED"
+echo "You have 5 seconds to interrupt this script..."
+if sleep 5; then
+  echo "Expired, proceeding"
+else
+  echo "Interrupted, aborting"
+  exit 1
+fi
+
+echo "Emptying data directory..."
+find "$DATA_DIRECTORY" -mindepth 1 -delete
+
+echo -n "Starting mariadb for backup restoration"
+"$SERVER" --innodb-flush-method=nosync --skip-innodb-doublewrite --innodb-flush-log-at-trx-commit=0 --sync-frm=0 --slow-query-log=0 --skip-log-bin &
+PID=$!
+trap "kill $PID; wait; exit 1" EXIT
+while true; do
+  if [ ! -e "/proc/$PID" ]; then
+    trap EXIT
+    echo "Service exited, check logs"
+    wait
+    exit 1
+  fi
+  test -e "$SOCKET" && break
+  echo -n .
+  sleep 0.5
+done
+
+"$UPDATE"
+
+echo "Importing $BACKUP ..."
+zcat "$BACKUP" | "$CLIENT" -u root
+
+echo "Configuring server as slave..."
+if [ "$MASTER_USE_GTID" -eq 1 ]; then
+  MASTER_USE_GTID_SQL="current_pos"
+else
+  MASTER_USE_GTID_SQL="NO"
+fi
+"$CLIENT" -u root -e "
+  CHANGE MASTER TO
+  MASTER_HOST='$MASTER_HOST',
+  MASTER_USER='$MASTER_USER',
+  MASTER_PORT=$MASTER_PORT,
+  MASTER_SSL=1,
+  MASTER_SSL_CA='$MASTER_SSL_CA',
+  MASTER_SSL_CERT='$MASTER_SSL_CERT',
+  MASTER_SSL_KEY='$MASTER_SSL_KEY',
+  MASTER_SSL_VERIFY_SERVER_CERT=1,
+  MASTER_USE_GTID=$MASTER_USE_GTID_SQL;
+"
+"$CLIENT" -u root -e "$SQL_CHANGE_MASTER"
+test "$MASTER_USE_GTID" -eq 1 && "$CLIENT" -u root -e "$SQL_SET_GTID"
+"$CLIENT" -u root -e "START SLAVE;"
+
+echo "Stopping mariadb..."
+trap EXIT
+kill $PID
+wait
+
+echo "Done. Start mariadb normally. You may use 'show slave status' SQL command to monitor progress."

stack/erp5/instance-mariadb.cfg.in

@@ -252,6 +252,7 @@ mariadb-ssl = ${:etc}/mariadb-ssl
 var = ${buildout:directory}/var
 log = ${:var}/log
 run = ${:var}/run
+slowquery =  ${monitor-directory:private}/slowquerydex
 
 [{{ section('resiliency-exclude-file') }}]
 # Generate rdiff exclude file in case of resiliency
@@ -262,7 +263,20 @@ rendered = ${directory:srv}/exporter.exclude
 [dash]
 dash = {{ dumps(dash) }}
 
-[resiliency-after-import-script]
+[{{ section('start-clone-from-backup') }}]
+< = jinja2-template-executable
+template = {{ parameter_dict['mariadb-start-clone-from-backup'] }}
+rendered = ${directory:bin}/start-clone-from-backup
+context =
+  key dash dash:dash
+  key client binary-wrap-mysql:wrapper-path
+  key data_directory directory:mariadb-data
+  key pid_file my-cnf-parameters:pid-file
+  key server mysqld:rendered
+  key update update-mysql:output
+  key socket my-cnf-parameters:socket
+
+[{{ section('resiliency-after-import-script') }}]
 # Generate after import script used by importer instance of webrunner
 < = jinja2-template-executable
 template = {{ parameter_dict['mariadb-resiliency-after-import-script'] }}
@@ -272,7 +286,6 @@ context =
   key mysql_executable binary-wrap-mysql:wrapper-path
   key mariadb_data_directory directory:mariadb-data
   key mariadb_backup_directory directory:mariadb-backup-full
-  key instance_directory buildout:directory
   key pid_file my-cnf-parameters:pid-file
   key binlog_path my-cnf-parameters:binlog-path
   key server_executable mysqld:rendered
@@ -288,7 +301,7 @@ context =
   raw slow_query_path ${directory:srv}/backup/logrotate/mariadb_slowquery.log
   raw pt_query_exec ${binary-wrap-pt-digest:wrapper-path}
   raw dash {{ parameter_dict['dash-location'] }}/bin/dash
-  key output_folder monitor-directory:private
+  key output_folder directory:slowquery
 
 [slow-query-digest-parameters]
 max_queries_threshold = {{ slapparameter_dict['max-slowqueries-threshold'] }}
@@ -296,9 +309,8 @@ slowest_queries_threshold = {{ slapparameter_dict['slowest-query-threshold'] }}
 
 [{{ section('monitor-promise-slowquery-result') }}]
 recipe = slapos.cookbook:wrapper
-recipe =
 wrapper-path = ${directory:promise}/check-slow-query-pt-digest-result
-command-line = "{{ parameter_dict['promise-check-slow-queries-digest-result'] }}" --ptdigest_file "${monitor-directory:private}/slowquery_digest.txt" --status_file ${monitor-directory:private}/mariadb_slow_query.report.json --max_queries_threshold "${slow-query-digest-parameters:max_queries_threshold}" --slowest_query_threshold "${slow-query-digest-parameters:slowest_queries_threshold}"
+command-line = "{{ parameter_dict['promise-check-slow-queries-digest-result'] }}" --ptdigest_path "${directory:slowquery}" --status_file ${monitor-directory:private}/mariadb_slow_query.report.json --max_queries_threshold "${slow-query-digest-parameters:max_queries_threshold}" --slowest_query_threshold "${slow-query-digest-parameters:slowest_queries_threshold}"
 
 
 [{{ section('promise') }}]