Prepare release 1.0.20

component/babeld/buildout.cfg

@@ -3,8 +3,8 @@ parts = babeld
 
 [babeld]
 recipe = slapos.recipe.cmmi
-url = http://git.erp5.org/gitweb/babeld.git/snapshot/v1.6.2-nxd1.tar.xz
-md5sum = 336d25fd7630052ccb3a61d3603029b9
+url = https://lab.nexedi.com/nexedi/babeld/repository/archive.tar.gz?ref=v1.6.2-nxd1
+md5sum = b7137d7772fa670f4cec39838c4d7b1e
 configure-command =
   echo "No configure.."
 

component/curl/buildout.cfg

@@ -12,8 +12,8 @@ parts =
 
 [curl]
 recipe = slapos.recipe.cmmi
-url = http://curl.haxx.se/download/curl-7.44.0.tar.bz2
-md5sum = 6b952ca00e5473b16a11f05f06aa8dae
+url = http://curl.haxx.se/download/curl-7.47.0.tar.bz2
+md5sum = 85c58a00412476993050cb242a3f365d
 configure-options =
   --disable-static
   --disable-ldap
@@ -28,11 +28,15 @@ configure-options =
   --disable-gopher
   --enable-ipv6
   --disable-sspi
-  --without-gnutls
-  --without-spnego
-  --with-ssl=${openssl:location}
   --with-zlib=${zlib:location}
+  --with-ssl=${openssl:location}
+  --without-gnutls
+  --without-polarssl
+  --without-mbedtls
+  --without-cyassl
   --without-nss
+  --without-axtls
+  --without-libpsl
   --without-libmetalink
   --without-libssh2
   --without-librtmp

component/egg-patch/python_magic/magic.patch

+--- python-magic-0.4.10/magic.py.orig	2016-01-14 23:41:31.867145173 +0100
++++ python-magic-0.4.10/magic.py	2016-01-14 23:41:54.563401069 +0100
+@@ -143,7 +143,7 @@
+ 
+ libmagic = None
+ # Let's try to find magic or magic1
+-dll = ctypes.util.find_library('magic') or ctypes.util.find_library('magic1') or ctypes.util.find_library('cygmagic-1')
++dll = ctypes.util.find_library('magic') or ctypes.util.find_library('magic1') or ctypes.util.find_library('cygmagic-1') or 'libmagic.so'
+ 
+ # This is necessary because find_library returns None if it doesn't find the library
+ if dll:

component/golang/buildout.cfg

@@ -26,8 +26,8 @@ environment-extra =
 
 [golang15]
 <= golang-common
-url = https://storage.googleapis.com/golang/go1.5.2.src.tar.gz
-md5sum = 38fed22e7b80672291e7cba7fb9c3475
+url = https://storage.googleapis.com/golang/go1.5.3.src.tar.gz
+md5sum = 80a0eac7ab750b01b3f7096a1d4667b8
 
 # go1.5 needs go1.4 to bootstrap
 environment-extra =

component/groonga/buildout.cfg

@@ -13,8 +13,8 @@ extends =
 
 [groonga]
 recipe = slapos.recipe.cmmi
-url = http://packages.groonga.org/source/groonga/groonga-5.1.1.tar.gz
-md5sum = 50a869f710c005c0bb46ba7b790621fc
+url = http://packages.groonga.org/source/groonga/groonga-5.1.2.tar.gz
+md5sum = 04066b547f1f4dd869eb35ac52a2e88b
 # temporary patch to respect more tokens in natural language mode.
 patches =
   ${:_profile_base_location_}/groonga.patch#9ed02fbe8400402d3eab47eee149978b

component/helloweb/buildout.cfg

@@ -74,12 +74,14 @@ make-targets= cd ${:path} && ${bundler:bundle} install
 
 
 [helloweb-ruby]
-recipe  = slapos.cookbook:wrapper
-wrapper-path = ${buildout:bin-directory}/${:_buildout_section_name_}
-environment  =
-    BUNDLE_GEMFILE = ${helloweb-ruby-bundle:path}/Gemfile
-command-line =
-    ${bundler:bundle} exec sh -c 'helloweb.rb "$@"' ${:_buildout_section_name_}
+# NOTE slapos.cookbook:wrapper also works, but currently _only_ in instance
+recipe  = collective.recipe.template
+output  = ${buildout:bin-directory}/${:_buildout_section_name_}
+mode    = 0755
+input   = inline:
+    #!/bin/sh
+    export BUNDLE_GEMFILE=${helloweb-ruby-bundle:path}/Gemfile
+    exec ${bundler:bundle} exec sh -c 'helloweb.rb "$@"' ${:_buildout_section_name_} "$@"
 
 
 # -*- go -*-

component/libpng/buildout.cfg

@@ -28,5 +28,5 @@ md5sum = 3414d556788e14b4a154369e67eacaa3
 
 [libpng]
 <= libpng-common
-url = http://download.sourceforge.net/libpng/libpng-1.6.20.tar.xz
-md5sum = 3968acb7c66ef81a9dab867f35d0eb4b
+url = http://download.sourceforge.net/libpng/libpng-1.6.21.tar.xz
+md5sum = 3bacb4728f6694a64ad9052769d6a4ce

component/mariadb/buildout.cfg

@@ -21,9 +21,9 @@ parts =
 
 [mariadb]
 recipe = slapos.recipe.cmmi
-version = 10.1.10
+version = 10.1.11
 url = https://downloads.mariadb.org/f/mariadb-${:version}/source/mariadb-${:version}.tar.gz/from/http:/ftp.osuosl.org/pub/mariadb
-md5sum = 9aa4d68b24c1ddd8cb56923a920684b3
+md5sum = c199608ae4282f16007dc8c270ff2439
 location = ${buildout:parts-directory}/${:_buildout_section_name_}
 patch-options = -p0
 patches =
@@ -68,8 +68,8 @@ post-install =
 # mroonga - a storage engine for MySQL. It provides fast fulltext search feature to all MySQL users.
 # http://mroonga.github.com/
 recipe = slapos.recipe.cmmi
-url = http://packages.groonga.org/source/mroonga/mroonga-5.11.tar.gz
-md5sum = 0ca8525da3594685ec039d22a6ceec8d
+url = http://packages.groonga.org/source/mroonga/mroonga-5.12.tar.gz
+md5sum = b42816754058114de708e1da028399bd
 pre-configure =
   mkdir fake_mariadb_source &&
   ln -s ${mariadb:location}/include/mysql/private fake_mariadb_source/sql

component/openblas/buildout.cfg

@@ -2,24 +2,33 @@
 parts =
   openblas
 
+extends =
+  ../gcc/buildout.cfg
+
 [openblas]
 recipe = slapos.recipe.cmmi
-# OpenBLAS 0.2.13 and 0.2.14 does not build on Broadwell
-# CPU (Detecting CPU failed). But version 0.2.15 (which
-# does not exist yet) will fix the issue. Until then you
-# can add in options :
-# TARGET=HASWELL
-url = http://github.com/xianyi/OpenBLAS/archive/v0.2.14.tar.gz
-md5sum = 53cda7f420e1ba0ea55de536b24c9701
-configure-command = true
+url = http://github.com/xianyi/OpenBLAS/archive/v0.2.15.tar.gz
+md5sum = b1190f3d3471685f17cfd1ec1d252ac9
+
+build-common-options =
+  BINARY="$(uname -m | grep -q x86_64 && echo 64 || echo 32)" NO_STATIC=1 USE_OPENMP=1 USE_THREAD=1
+
+# You can specify more options with openblas:build-ext-options parameter.
+# Example :
+# * to build generic binary that supports multiple architecture in one binary
+#   DYNAMIC_ARCH=1
+# * to specify target explicitly
+#   (see https://github.com/xianyi/OpenBLAS/blob/v0.2.15/TargetList.txt )
+#   TARGET=HASWELL
+build-ext-options =
+
+# First try with auto-detected target and if it fails try TARGET=GENERIC.
+configure-command =
+  make ${:build-common-options} ${:build-ext-options} || make ${:build-common-options} TARGET=GENERIC
 make-options =
-  PREFIX="${buildout:parts-directory}/${:_buildout_section_name_}"
-  BINARY="$(uname -m | grep -q x86_64 && echo 64 || echo 32)"
-  NO_STATIC=1
-  USE_OPENMP=1
-  USE_THREAD=1
-# to build generic binary that supports multiple architecture in one binary
-# DYNAMIC_ARCH=1
+  dummy
+make-targets =
+  PREFIX="${buildout:parts-directory}/${:_buildout_section_name_}" install
 environment =
   PATH=${gcc-fortran:location}/bin:%(PATH)s
   LD_LIBRARY_PATH=${gcc-fortran:location}/lib:${gcc-fortran:location}/lib64

component/openssl/buildout.cfg

@@ -16,8 +16,12 @@ parts =
 
 [openssl]
 recipe = slapos.recipe.cmmi
-url = https://www.openssl.org/source/openssl-1.0.2e.tar.gz
-md5sum = 5262bfa25b60ed9de9f28d5d52d77fc5
+url = https://www.openssl.org/source/openssl-1.0.2f.tar.gz
+md5sum = b3bf73f507172be9292ea2a8c28b659d
+location = ${buildout:parts-directory}/${:_buildout_section_name_}
+# 'prefix' option to override --openssldir/--prefix (which is useful
+# when combined with INSTALL_PREFIX). Used by slapos.package.git/obs
+prefix = ${:location}
 patch-binary = ${patch:location}/bin/patch
 patches =
   ${:_profile_base_location_}/openssl-nodoc.patch#a78c14908fe9ec624b1fb9fa97e01bb9
@@ -28,14 +32,17 @@ configure-command = ./config
 configure-options =
   -I${zlib:location}/include
   -L${zlib:location}/lib
-  --openssldir=${buildout:parts-directory}/${:_buildout_section_name_}/etc/ssl
-  --prefix=${buildout:parts-directory}/${:_buildout_section_name_}
+  --openssldir=${:prefix}/etc/ssl
+  --prefix=${:prefix}
   --libdir=lib
   shared no-idea no-mdc2 no-rc5 zlib
-  -Wl,-rpath=${zlib:location}/lib -Wl,-rpath=${buildout:parts-directory}/${:_buildout_section_name_}/lib
+  -Wl,-rpath=${zlib:location}/lib -Wl,-rpath=${:location}/lib
   && make depend
 make-targets =
-  all install_sw && rm -f ${buildout:parts-directory}/${:_buildout_section_name_}/etc/ssl/certs/* && for i in ${ca-certificates:location}/certs/*/*.crt; do ln -sv $i ${buildout:parts-directory}/${:_buildout_section_name_}/etc/ssl/certs/`${buildout:parts-directory}/${:_buildout_section_name_}/bin/openssl x509 -hash -noout -in $i`.0; done; true
+  install_sw && x=${:location}/etc/ssl/certs && rm -f $x/* &&
+  for i in ${ca-certificates:location}/certs/*/*.crt; do
+    ln -sv $i $x/`${:location}/bin/openssl x509 -hash -noout -in $i`.0
+  ; done
 environment =
   PERL=${perl:location}/bin/perl
 

component/python-cliff/buildout.cfg

@@ -3,10 +3,5 @@
 [python-cliff]
 recipe = zc.recipe.egg:custom
 egg = cliff
-setup-eggs = ${python-cliff-prep:eggs}
-
-[python-cliff-prep]
-recipe = zc.recipe.egg
-eggs =
+setup-eggs =
   pbr
-

component/python-cryptography/buildout.cfg

@@ -11,18 +11,10 @@ parts =
 recipe = zc.recipe.egg:custom
 egg = cryptography
 environment = python-cryptography-env
-setup-eggs = ${python-cryptography-prep:eggs}
+setup-eggs = ${python-cffi:egg}
 
 [python-cryptography-env]
 PATH = ${pkgconfig:location}/bin:%(PATH)s
 PKG_CONFIG_PATH = ${openssl:location}/lib/pkgconfig
 LD_LIBRARY_PATH = ${openssl:location}/lib
 CPATH = ${openssl:location}/include
-
-[python-cryptography-prep]
-recipe = zc.recipe.egg
-eggs =
-  ${python-cffi:egg}
-  enum34
-  pycparser
-  six

component/slapos/buildout.cfg

@@ -18,7 +18,13 @@ parts =
   py
   firewalld-patch
 
+extensions -=
+  buildout-versions
 
+extensions +=
+  slapos.rebootstrap
+
+show-picked-versions = true
 
 # separate from system python
 include-site-packages = false
@@ -118,3 +124,9 @@ eggs =
     ${slapos:eggs}
 interpreter = py
 scripts = py
+
+[versions]
+setuptools = 19.6.2
+slapos.rebootstrap = 3.3
+zc.buildout = 2.5.0+slapos001
+zc.recipe.egg = 2.0.3+slapos001

software/erp5/README.rst

@@ -39,6 +39,7 @@ This software release assigns the following port ranges by default:
   zeo                   2100-2149
   balancer              2150-2199
   zope                  2200-*
+  jupyter               8888
   ====================  ==========
 
 Non-zope partitions are unique in an ERP5 cluster, so you shouldn't have to

software/erp5/instance-erp5-input-schema.json

@@ -219,7 +219,7 @@
             ]
           },
           "storage-dict": {
-            "description": "Storage configuration. For NEO, 'logfile' is automatically set (see http://git.erp5.org/gitweb/neoppod.git/blob/HEAD:/neo/client/component.xml for other settings).",
+            "description": "Storage configuration. For NEO, 'logfile' is automatically set (see https://lab.nexedi.com/nexedi/neoppod/blob/master/neo/client/component.xml for other settings).",
             "properties": {
               "ssl": {
                 "description": "For external NEO. Pass false if you want to disable SSL or pass custom values for ca/cert/key.",
@@ -235,6 +235,22 @@
         "type": "object"
       },
       "type": "array"
+    },
+    "jupyter": {
+      "description": "Jupyter slave instance parameters",
+      "properties": {
+        "enable": {
+          "description": "Whether to enable creation of associated slave Jupyter instance",
+          "default": false,
+          "type": "boolean"
+        },
+        "zope-family": {
+          "description": "Zope family to connect Jupyter to by default",
+          "default": "<first instantiated Zope family>",
+          "type": "string"
+        }
+      },
+      "type": "object"
     }
   }
 }

software/erp5/instance-erp5-output-schema.json

@@ -40,6 +40,11 @@
       "description": "Relational database access information",
       "type": "string"
     }
+    "jupyter-url": {
+      "description": "Jupyter notebook web UI access information",
+      "type": "string",
+      "optional": true
+    }
   },
   "patternProperties": {
     "family-.*": {

software/gitlab/gitlab-parameters.cfg

+# Upstream parameters for a GitLab instance
+#
+# Selected parameters - main ones - names and advanced defaults taken from omnibus-gitlab
+#   https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-config-template/gitlab.rb.template
+#   https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-cookbooks/gitlab/attributes/default.rb
+#
+# TODO better autogenerate from ^^^ (?)
+#
+# (last updated for omnibus-gitlab 8.2.3+ce.0-0-g8eda093)
+
+[gitlab-parameters]
+configuration.external_url              = http://lab.example.com
+
+# db advanced
+configuration.db_pool                   = 10
+
+# rack-attack
+configuration.rate_limit_requests_per_period    = 10
+configuration.rate_limit_period                 = 60
+
+configuration.time_zone                 = UTC
+
+configuration.email_enabled             = true
+configuration.email_from                = lab@example.com
+configuration.email_display_name        = GitLab
+configuration.email_reply_to            = noreply@example.com
+
+configuration.smtp_enable               = true
+configuration.smtp_address              = smtp.server
+configuration.smtp_port                 = 465
+configuration.smtp_user_name            = smtp user
+configuration.smtp_password             = smtp password
+configuration.smtp_domain               = lab.example.com
+configuration.smtp_authentication       = login
+configuration.smtp_enable_starttls_auto = true
+
+# none | peer | client_once | fail_if_no_peer_cert -> see gitlab-omnibus links at top
+configuration.smtp_openssl_verify_mode  = peer
+
+configuration.default_can_create_group  = true
+configuration.username_changing_enabled = true
+configuration.default_theme             = 2
+
+configuration.default_projects_features.issues          = true
+configuration.default_projects_features.merge_requests  = true
+configuration.default_projects_features.wiki            = true
+configuration.default_projects_features.snippets        = true
+# NOTE can be public|private|internal
+configuration.default_projects_features.visibility_level= public
+#configuration.default_projects_features.builds          = false
+
+configuration.webhook_timeout           = 10
+
+# 0 means forever (seconds)
+configuration.backup_keep_time          = 0
+
+# NOTE empty = default gitlab limits
+configuration.git_max_size              =
+configuration.git_timeout               =
+
+
+# sidekiq
+configuration.sidekiq_shutdown_timeout  = 4
+configuration.sidekiq_concurrency       = 25
+configuration.sidekiq_memory_killer_max_rss = 1000000
+
+
+# unicorn
+configuration.unicorn_worker_timeout    = 60
+configuration.unicorn_worker_processes  = 2
+
+# unicorn advanced
+configuration.unicorn_backlog_socket    = 1024
+
+configuration.unicorn_worker_memory_limit_min   = 200*(1024**2)
+configuration.unicorn_worker_memory_limit_max   = 250*(1024**2)
+
+
+# nginx
+configuration.nginx_client_max_body_size    = 250m
+
+# NOTE: we don't really need old ciphers - usually we talk directly to frontend only
+configuration.nginx_ssl_ciphers             = ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA:ECDHE-RSA-AES128-SHA:ECDHE-RSA-DES-CBC3-SHA:AES256-GCM-SHA384:AES128-GCM-SHA256:AES256-SHA256:AES128-SHA256:AES256-SHA:AES128-SHA:DES-CBC3-SHA:!aNULL:!eNULL:!EXPORT:!DES:!MD5:!PSK:!RC4
+configuration.nginx_ssl_prefer_server_ciphers = on
+configuration.nginx_ssl_protocols           = TLSv1 TLSv1.1 TLSv1.2
+# the following is gitlab-omnibus default but not nginx's default
+configuration.nginx_ssl_session_cache       = builtin:1000  shared:SSL:10m
+configuration.nginx_ssl_session_timeout     = 5m
+
+configuration.nginx_proxy_read_timeout      = 300
+configuration.nginx_proxy_connect_timeout   = 300
+
+# nginx advanced
+configuration.nginx_worker_processes    = 4
+configuration.nginx_worker_connections  = 10240
+configuration.nginx_log_format          = $remote_addr - $remote_user [$time_local] "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent"
+configuration.nginx_sendfile            = on
+configuration.nginx_tcp_nopush          = on
+configuration.nginx_tcp_nodelay         = on
+configuration.nginx_gzip                = on
+configuration.nginx_gzip_http_version   = 1.0
+configuration.nginx_gzip_comp_level     = 2
+configuration.nginx_gzip_proxied        = any
+configuration.nginx_gzip_types          = text/plain text/css application/x-javascript text/xml application/xml application/xml+rss text/javascript application/json
+configuration.nginx_keepalive_timeout   = 65

software/gitlab/gitlab-unicorn-startup.in

+#!{{ bash_bin }}
+# start up gitlab's unicorn with first making sure db is properly setup and all
+# migrations are up as pre-condition.
+
+RAKE={{ gitlab_rake }}
+
+die() {
+    echo "$*" 1>&2
+    exit 1
+}
+
+# 1. what to do when instance is initially setup
+# see
+#   https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-cookbooks/gitlab/recipes/database_migrations.rb
+#   https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-cookbooks/gitlab/recipes/postgresql.rb
+
+# initial db setup
+pgtables="$({{ psql_bin }}   \
+    -h {{ pgsql['pgdata-directory'] }}  \
+    -U {{ pgsql.superuser }}            \
+    -d {{ pgsql.dbname }}               \
+    -c '\d')" || die "pg query problem"
+
+if echo "$pgtables" | grep -q '^No relations found' ; then
+    $RAKE db:schema:load db:seed_fu  || die "initial db setup failed"
+fi
+
+# re-build ssh keys
+# (we do not use them - just for cleannes)
+force=yes $RAKE gitlab:shell:setup   || die "gitlab:shell:setup failed"
+
+
+# 2. what to do when instance is upgraded
+# see
+#   https://gitlab.com/gitlab-org/gitlab-ce/blob/master/lib/support/deploy/deploy.sh
+#   https://gitlab.com/gitlab-org/gitlab-ce/blob/master/lib/gitlab/upgrader.rb
+#   https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-cookbooks/gitlab/recipes/gitlab-rails.rb
+#   https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-ctl-commands/upgrade.rb
+#
+# Assets compilation is handled at instance deployment time. We do everything else here.
+
+# make sure all migrations are up
+migrate_log="{{ log_dir }}/db-migrate-`date +%s`.log"
+$RAKE db:migrate >$migrate_log 2>&1  || die "db:migrate failed"
+# if it was a no-op "migration" - we don't need info about that - only keep
+# logs of actual migration run.
+test -s $migrate_log || rm $migrate_log
+
+
+# clear cache
+$RAKE cache:clear   || die "cache:clear failed"
+
+
+
+# 3. finally exec to unicorn
+exec {{ gitlab_unicorn }}   \
+    -E production   \
+    -c {{ unicorn_rb.rendered }}    \
+    {{ gitlab_work.location }}/config.ru

software/gitlab/instance.cfg.in

+# GitLab "switch-softwaretype" instance
+[buildout]
+parts = switch-softwaretype
+
+# std stuff for slapos instance
+eggs-directory = ${buildout:eggs-directory}
+develop-eggs-directory = ${buildout:develop-eggs-directory}
+offline = true
+
+
+[switch-softwaretype]
+recipe = slapos.cookbook:softwaretype
+default = $${instance-gitlab.cfg:rendered}
+# TODO -export, -import, -pull-backup
+
+
+[instance-gitlab.cfg]
+recipe  = slapos.recipe.template:jinja2
+mode    = 0644
+template= ${instance-gitlab.cfg.in:target}
+rendered= $${buildout:directory}/instance-gitlab.cfg
+context =
+    import os os
+    import pwd pwd
+    import multiprocessing multiprocessing
+
+    key eggs_directory          buildout:eggs-directory
+    key develop_eggs_directory  buildout:develop-eggs-directory
+    raw gitlab_repository_location          ${gitlab-repository:location}
+    raw gitlab_shell_repository_location    ${gitlab-shell-repository:location}
+
+# program binaries
+    raw bash_bin                    ${bash:location}/bin/bash
+    raw bundler_4gitlab             ${bundler-4gitlab:bundle}
+    raw curl_bin                    ${curl:location}/bin/curl
+    raw dcron_bin                   ${dcron-output:crond}
+    raw git                         ${git:location}/bin/git
+    raw git_location                ${git:location}
+    raw gitlab_workhorse            ${gitlab-workhorse:location}/gitlab-workhorse
+    raw gunzip_bin                  ${gzip:location}/bin/gunzip
+    raw gzip_bin                    ${gzip:location}/bin/gzip
+    raw logrotate_bin               ${logrotate:location}/usr/sbin/logrotate
+    raw nginx_bin                   ${nginx-output:nginx}
+    raw nginx_mime_types            ${nginx-output:mime}
+    raw openssl_bin                 ${openssl-output:openssl}
+    raw postgresql_location         ${postgresql92:location}
+    raw redis_binprefix             ${redis28:location}/bin
+    raw ruby_location               ${bundler-4gitlab:ruby-location}
+    raw watcher_sigkill             ${watcher-sigkill:rendered}
+
+# config files
+    raw config_ru_in                ${config.ru.in:target}
+    raw database_yml_in             ${database.yml.in:target}
+    raw gitconfig_in                ${gitconfig.in:target}
+    raw gitlab_parameters_cfg       ${gitlab-parameters.cfg:target}
+    raw gitlab_shell_config_yml_in  ${gitlab-shell-config.yml.in:target}
+    raw gitlab_unicorn_startup_in   ${gitlab-unicorn-startup.in:target}
+    raw gitlab_yml_in               ${gitlab.yml.in:target}
+    raw macrolib_cfg_in             ${macrolib.cfg.in:target}
+    raw nginx_conf_in               ${nginx.conf.in:target}
+    raw nginx_gitlab_http_conf_in   ${nginx-gitlab-http.conf.in:target}
+    raw rack_attack_rb_in           ${rack_attack.rb.in:target}
+    raw resque_yml_in               ${resque.yml.in:target}
+    raw smtp_settings_rb_in         ${smtp_settings.rb.in:target}
+    raw unicorn_rb_in               ${unicorn.rb.in:target}

software/gitlab/macrolib.cfg.in

+{# common macros for gitlab instance #}
+
+{# cfg(name) -> instance_parameter:configuration.<name> #}
+{% macro cfg(name) %}{{ instance_parameter[str("configuration." + name)] }}{% endmacro %}
+
+{# cfg_bool(name) - like cfg(name), but returns 'true'/''
+   NOTE macros can return only strings - that's why '' is used for false #}
+{% macro cfg_bool(name) %}{{ 'true' if (cfg(name).lower() in ('true', 'yes')) else '' }}{% endmacro %}
+
+
+{# deduce whether to use https from external url
+   ( here - becasue we cannot use jinja2 logic in instance-gitlab.cfg.in to
+     process instance parameters ) #}
+{% set external_url = urlparse.urlparse(cfg('external_url')) %}
+{% set cfg_https = (true if external_url.scheme == 'https' else false) %}
+
+{# for convenience #}
+{% set fqdn = external_url.hostname %}

software/gitlab/software.cfg

+# GitLab software-release
+[buildout]
+extends =
+    ../../stack/slapos.cfg
+    ../../component/ruby/buildout.cfg
+    ../../component/golang/buildout.cfg
+    ../../component/postgresql/buildout.cfg
+    ../../component/redis/buildout.cfg
+    ../../component/cmake/buildout.cfg
+    ../../component/icu/buildout.cfg
+    ../../component/pkgconfig/buildout.cfg
+    ../../component/nodejs/buildout.cfg
+    ../../component/openssl/buildout.cfg
+    ../../component/nginx/buildout.cfg
+
+#   for instance
+    ../../component/bash/buildout.cfg
+    ../../component/curl/buildout.cfg
+    ../../component/gzip/buildout.cfg
+    ../../component/dcron/buildout.cfg
+    ../../component/logrotate/buildout.cfg
+
+parts =
+    ruby2.1
+    golang15
+    git
+    postgresql92
+    redis28
+    cmake
+    icu
+    pkgconfig
+    nginx-output
+
+    python-4gitlab
+    gitlab-shell/vendor
+    gitlab/vendor/bundle
+    gitlab-workhorse
+
+#   for instance
+    instance.cfg
+
+    slapos-cookbook
+    eggs
+
+    bash
+    curl
+    watcher-sigkill
+    gzip
+    dcron-output
+    logrotate
+
+
+############################
+#   Software compilation   #
+############################
+
+# python with eggs, that will be used in gitlab
+[python-4gitlab]
+# NOTE cannot use zc.recipe.egg - github-markup invoks `python2 -S` and
+# interpreter generated by zc.recipe.egg cannot handle that.
+recipe  = z3c.recipe.scripts
+# NOTE github-markup invokes it as `python2`, that's why we are naming it this way
+# https://github.com/github/markup/blob/5393ae93/lib/github/markups.rb#L36
+interpreter = python2
+eggs    =
+    docutils
+
+# rubygemsrecipe with fixed url and this way pinned rubygems version
+[rubygemsrecipe]
+recipe  = rubygemsrecipe
+url     = https://rubygems.org/rubygems/rubygems-2.5.1.zip
+
+
+# bundler, that we'll use to
+# - install gems for gitlab
+# - run gitlab services / jobs  (via `bundle exec ...`)
+[bundler-4gitlab]
+<= rubygemsrecipe
+ruby-location = ${ruby2.1:location}
+ruby-executable = ${:ruby-location}/bin/ruby
+gems    = bundler==1.11.2
+
+# bin installed here
+bundle  = ${buildout:bin-directory}/bundle
+
+# install together with dependencies of gitlab, which we cannot specify using
+#   --with-... gem option
+# ( reason: rubygemsrecipe hardcodes PATH inside generated bin/* and it is
+#   impossible to adjust it later )
+#
+# bundle exec <smth>                ; <smth> starts with `#!/usr/bin/env ruby` as rubygems
+# Rugged needs: cmake, pkgconfig
+# execjs needs: nodejs
+# rails needs db client program on path: psql
+# gitlab wants to check redis version via running: redis-cli
+# gitlab (via github-markup) wants to convert rst -> html via running: python2 (with docutils egg)
+# (python-4gitlab puts interpreter into ${buildout:bin-directory})
+environment =
+  PATH    = ${:ruby-location}/bin:${cmake:location}/bin:${pkgconfig:location}/bin:${nodejs:location}/bin:${postgresql92:location}/bin:${redis28:location}/bin:${buildout:bin-directory}:%(PATH)s
+
+
+# gitlab, gitlab-shell & gitlab-workhorse checked out as git repositories
+# pinned to exact commit
+[git-repository]
+recipe  = slapos.recipe.build:gitclone
+git-executable = ${git:location}/bin/git
+
+[gitlab-repository]
+<= git-repository
+#repository = https://gitlab.com/gitlab-org/gitlab-ce.git
+repository = https://lab.nexedi.com/kirr/gitlab-ce.git
+# 8.2.X + NXD patches:
+revision = v8.2.3-9-g79c127e6e068a619c53a8c22f1db8c1e28ec87d2
+location = ${buildout:parts-directory}/gitlab
+
+[gitlab-shell-repository]
+<= git-repository
+repository = https://gitlab.com/gitlab-org/gitlab-shell.git
+# gitlab 8.2 wants gitlab-shell 2.6.8
+# 2.6.8 + NXD patches
+revision = v2.6.8-2-g216d7e15fe06917198891a895f762ba84fdcc4d4
+location = ${buildout:parts-directory}/gitlab-shell
+
+[gitlab-workhorse-repository]
+<= git-repository
+#repository = https://gitlab.com/gitlab-org/gitlab-workhorse.git
+repository = https://lab.nexedi.com/kirr/gitlab-workhorse.git
+# 0.4.X + NXD patches for blob download speedup
+# (https://gitlab.com/gitlab-org/gitlab-workhorse/merge_requests/17)
+revision = 0.4.1-23-g2beb8c9539433f072e3db540f91f75894ca6b1b0
+location = ${buildout:parts-directory}/gitlab-workhorse
+
+
+
+# build needed-by-gitlab gems via bundler
+[gitlab/vendor/bundle]
+recipe  = slapos.recipe.cmmi
+path    = ${gitlab-repository:location}
+bundle  = ${bundler-4gitlab:bundle}
+
+configure-command = cd ${:path} &&
+    ${:bundle} config --local build.charlock_holmes --with-icu-dir=${icu:location}  &&
+    ${:bundle} config --local build.pg  --with-pg-config=${postgresql92:location}/bin/pg_config
+
+make-binary =
+make-targets= cd ${:path} &&
+    ${:bundle} install --deployment  --without development test  mysql kerberos
+
+
+# build needed-by-gitlab-shell gems via bundler
+# ( there is not vendor/ dir in gitlab-shell, so to avoid having buildout error
+#   on mkdir vendor/bundle, this part name is just /vendor )
+[gitlab-shell/vendor]
+recipe  = slapos.recipe.cmmi
+path    = ${gitlab-shell-repository:location}
+bundle  = ${bundler-4gitlab:bundle}
+
+configure-command = true
+make-binary =
+make-targets= cd ${:path} &&
+    ${:bundle} install --deployment  --without development test
+
+
+# build gitlab-workhorse
+[gitlab-workhorse]
+recipe  = slapos.recipe.cmmi
+path    = ${gitlab-workhorse-repository:location}
+
+configure-command = :
+make-targets= ${:_buildout_section_name_}
+
+environment =
+  PATH=${golang15:location}/bin:%(PATH)s
+
+
+###############################
+#   Trampoline for instance   #
+###############################
+
+# eggs for instance.cfg
+[eggs]
+recipe  = zc.recipe.egg
+eggs    =
+    plone.recipe.command
+    cns.recipe.symlink
+
+
+[instance.cfg]
+recipe  = slapos.recipe.template
+url     = ${:_profile_base_location_}/instance.cfg.in
+output  = ${buildout:directory}/instance.cfg
+md5sum  = b40cd8824b978da867404d8955b06c18
+
+[watcher-sigkill]
+recipe  = slapos.recipe.template:jinja2
+template= ${:_profile_base_location_}/${:_buildout_section_name_}.in
+rendered= ${buildout:bin-directory}/${:_buildout_section_name_}
+mode    = 0755
+md5sum  = 2986dcb006dc9e8508ff81f646656131
+context =
+    section bash    bash
+
+
+# macro: download a file named as section name
+#
+#   [filename]
+#   <= download-file
+#   md5sum = ...
+[download-file]
+recipe  = slapos.recipe.build:download
+url     = ${:_profile_base_location_}/${:_buildout_section_name_}
+destination = ${buildout:directory}/${:_buildout_section_name_}
+
+# like download-file, but download from template/<filename>
+[download-template]
+<= download-file
+url     = ${:_profile_base_location_}/template/${:_buildout_section_name_}
+
+
+[config.ru.in]
+<= download-template
+md5sum  = bb12852c28079f40a0751f7f3559e2a6
+
+[database.yml.in]
+<= download-template
+md5sum  = ee656cfd96e1c82df167f68bb5773291
+
+[gitconfig.in]
+<= download-template
+md5sum  = f4cb11e8bca379e016b062d0db859b74
+
+[gitlab-parameters.cfg]
+<= download-file
+md5sum  = bc98ec10209bc53f6a49888b1a2b9382
+
+[gitlab-shell-config.yml.in]
+<= download-template
+md5sum  = ea351e16b47f0008f61211eb2d7685e2
+
+[gitlab-unicorn-startup.in]
+<= download-file
+md5sum  = 2716afaa9445c0c429c6b211356ebe8f
+
+[gitlab.yml.in]
+<= download-template
+md5sum  = cc32f5053dd2a2461aa5952a5b925310
+
+[instance-gitlab.cfg.in]
+<= download-file
+md5sum  = dfd2b14f846eda999fe9d12108d513b4
+
+[macrolib.cfg.in]
+<= download-file
+md5sum  = a56a44e96f65f5ed20211bb6a54279f4
+
+[nginx-gitlab-http.conf.in]
+<= download-template
+md5sum  = 590da2b00cd198c7bc261c3d893bc199
+
+[nginx.conf.in]
+<= download-template
+md5sum  = f1a6e2bce3f28a2243fed49d1e1601df
+
+[rack_attack.rb.in]
+<= download-template
+md5sum  = 16503c029159ea6db7d0fb5ab67093a3
+
+[resque.yml.in]
+<= download-template
+md5sum  = 7d9cba658f9315cd058dfc74db943a66
+
+[smtp_settings.rb.in]
+<= download-template
+md5sum  = c7c09c241b5fa8163e4995260be52604
+
+[unicorn.rb.in]
+<= download-template
+md5sum  = 9bdca16362fe19c727bca38383e57068
+
+
+[versions]
+cns.recipe.symlink = 0.2.3
+docutils = 0.12
+plone.recipe.command = 1.1
+rubygemsrecipe  = 0.2.2
+slapos.recipe.template = 2.9
+z3c.recipe.scripts = 1.0.1

software/gitlab/template/config.ru.in

+{{ autogenerated }}
+# see:
+# https://gitlab.com/gitlab-org/gitlab-ce/blob/master/config.ru
+# https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-cookbooks/gitlab/templates/default/gitlab-rails-config.ru.erb
+# (last updated for omnibus-gitlab 8.2.3+ce.0-0-g8eda093)
+
+# This file is used by Rack-based servers to start the application.
+
+{% from 'macrolib.cfg.in' import cfg  with context %}
+
+if defined?(Unicorn)
+  require 'unicorn'
+
+  if ENV['RAILS_ENV'] == 'production' || ENV['RAILS_ENV'] == 'staging'
+    # Unicorn self-process killer
+    require 'unicorn/worker_killer'
+
+    # Max memory size (RSS) per worker
+    use Unicorn::WorkerKiller::Oom, ({{ cfg('unicorn_worker_memory_limit_min') }}), ({{ cfg('unicorn_worker_memory_limit_max') }})
+  end
+end
+
+require ::File.expand_path('../config/environment',  __FILE__)
+
+map ENV['RAILS_RELATIVE_URL_ROOT'] || "/" do
+  run Gitlab::Application
+end

software/gitlab/template/database.yml.in

+{{ autogenerated }}
+# see:
+# https://gitlab.com/gitlab-org/gitlab-ce/blob/master/config/database.yml.postgresql
+# https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-cookbooks/gitlab/templates/default/database.yml.erb
+# (last updated for 8.2.3+ce.0-0-g8eda093)
+
+{% from 'macrolib.cfg.in' import cfg  with context %}
+
+production:
+  adapter: postgresql
+  encoding: unicode
+  {# collation is mainly for mysql
+  collation: <%= @db_collation %>
+  #}
+  database: {{ pgsql.dbname }}
+  pool: {{ cfg('db_pool') }}
+  {# XXX is it ok to use superuser, even if the whole database is only for gitlab? #}
+  username: '{{ pgsql.superuser }}'
+  {# we have no password - access is via unix socket #}
+  password:
+  host:     '{{ pgsql["pgdata-directory"] }}'
+  port:
+  socket:
+  {# not needed for unix socket
+  sslmode: <%= single_quote(@db_sslmode) %>
+  sslrootcert: <%= single_quote(@db_sslrootcert) %>
+  #}

software/gitlab/template/gitconfig.in

+{{ autogenerated }}
+# global git configuration for GitLab
+# see:
+# https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-cookbooks/gitlab/attributes/default.rb
+# https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-cookbooks/gitlab/templates/default/gitconfig.erb
+# (last updated for omnibus-gitlab 8.2.3+ce.0-0-g8eda093)
+#
+{% from 'macrolib.cfg.in' import cfg  with context %}
+
+# don't waste memory when packing (each thread uses own work memory)
+# besides it packs better with 1 thread
+[pack]
+        threads = 1
+
+# don't allow corrupt/broken objects to go in
+[receive]
+        fsckObjects = true
+
+
+[user]
+        name = {{ cfg('email_display_name') }}
+        email = {{ cfg('email_from') }}
+[core]
+        autocrlf = input

software/gitlab/template/gitlab-shell-config.yml.in

+{{ autogenerated }}
+# see:
+# https://gitlab.com/gitlab-org/gitlab-shell/blob/master/config.yml.example
+# https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-cookbooks/gitlab/templates/default/gitlab-shell-config.yml.erb
+# (last updated for omnibus-gitlab 8.2.3+ce.0-0-g8eda093)
+
+# GitLab user. git by default
+user: {{ backend_info.user }}
+
+# Url to gitlab instance. Used for api calls. Should end with a slash.
+gitlab_url: "http+unix://{{ urllib.quote_plus(unicorn.socket) }}/"
+
+http_settings:
+{# we don't need any
+  <%= @http_settings.to_json if @http_settings %>
+#}
+#  user: someone
+#  password: somepass
+#  ca_file: /etc/ssl/cert.pem
+#  ca_path: /etc/pki/tls/certs
+#  self_signed_cert: false
+
+# Repositories path
+# Give the canonicalized absolute pathname,
+# REPOS_PATH MUST NOT CONTAIN ANY SYMLINK!!!
+# Check twice that none of the components is a symlink, including "/home".
+repos_path: "{{ gitlab.repositories }}"
+
+# File used as authorized_keys for gitlab user
+# NOTE not used in slapos version (all access via https only)
+auth_file: "{{ gitlab.var }}/sshkeys-notused"
+
+# File that contains the secret key for verifying access to GitLab.
+# Default is .gitlab_shell_secret in the root directory.
+secret_file: "{{ gitlab_shell.secret }}"
+
+
+# Redis settings used for pushing commit notices to gitlab
+redis:
+  bin: {{ redis_binprefix }}/redis-cli
+  host: {# <%= @redis_host %> #}
+  port: {# <%= @redis_port %> #}
+  socket: {{ service_redis.unixsocket }}
+{# we don't use password for redis
+<% if @redis_password %>
+  pass: <%= @redis_password %>
+<% end %>
+#}
+  database: {# <%= @redis_database %> #}
+  namespace: resque:gitlab
+
+# Log file.
+# Default is gitlab-shell.log in the root directory.
+log_file: "{{ gitlab_shell.log }}/gitlab-shell.log"
+
+# Log level. INFO by default
+log_level:
+
+# Audit usernames.
+# Set to true to see real usernames in the logs instead of key ids, which is easier to follow, but
+# incurs an extra API call on every gitlab-shell command.
+audit_usernames:
+
+# Enable git-annex support
+# git-annex allows managing files with git, without checking the file contents into git
+# See https://git-annex.branchable.com/ for documentation
+# If enabled, git-annex needs to be installed on the server where gitlab-shell is setup
+# For Debian and Ubuntu systems this can be done with: sudo apt-get install git-annex
+# For CentOS: sudo yum install epel-release && sudo yum install git-annex
+git_annex_enabled:

software/gitlab/template/nginx-gitlab-http.conf.in

+{{ autogenerated }}
+# see:
+# https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-cookbooks/gitlab/templates/default/nginx-gitlab-http.conf.erb
+# (last updated for omnibus-gitlab 8.2.3+ce.0-0-g8eda093)
+
+{% from 'macrolib.cfg.in' import cfg, cfg_bool, cfg_https, fqdn  with context %}
+
+## GitLab
+## Modified from https://gitlab.com/gitlab-org/gitlab-ce/blob/master/lib/support/nginx/gitlab-ssl & https://gitlab.com/gitlab-org/gitlab-ce/blob/master/lib/support/nginx/gitlab
+##
+## Lines starting with two hashes (##) are comments with information.
+## Lines starting with one hash (#) are configuration parameters that can be uncommented.
+##
+##################################
+##        CHUNKED TRANSFER      ##
+##################################
+##
+## It is a known issue that Git-over-HTTP requires chunked transfer encoding [0]
+## which is not supported by Nginx < 1.3.9 [1]. As a result, pushing a large object
+## with Git (i.e. a single large file) can lead to a 411 error. In theory you can get
+## around this by tweaking this configuration file and either:
+## - installing an old version of Nginx with the chunkin module [2] compiled in, or
+## - using a newer version of Nginx.
+##
+## At the time of writing we do not know if either of these theoretical solutions works.
+## As a workaround users can use Git over SSH to push large files.
+##
+## [0] https://git.kernel.org/cgit/git/git.git/tree/Documentation/technical/http-protocol.txt#n99
+## [1] https://github.com/agentzh/chunkin-nginx-module#status
+## [2] https://github.com/agentzh/chunkin-nginx-module
+##
+###################################
+##         configuration         ##
+###################################
+
+upstream gitlab {
+  server unix:{{ unicorn.socket }} fail_timeout=0;
+}
+
+upstream gitlab-workhorse {
+  server unix:{{ gitlab_workhorse.socket }};
+}
+
+{# not needed for us - the frontend can do the redirection and also
+   gitlab/nginx speaks HSTS on https port so when we access https port via http
+   protocol, it gets redirected to https
+<% if @https && @redirect_http_to_https %>
+## Redirects all HTTP traffic to the HTTPS host
+server {
+<% @listen_addresses.each do |listen_address| %>
+  listen <%= listen_address %>:<%= @redirect_http_to_https_port %>;
+<% end %>
+  server_name <%= @fqdn %>;
+  server_tokens off; ## Don't show the nginx version number, a security best practice
+  return 301 https://<%= @fqdn %>:<%= @port %>$request_uri;
+  access_log  <%= @log_directory %>/gitlab_access.log gitlab_access;
+  error_log   <%= @log_directory %>/gitlab_error.log;
+}
+<% end %>
+#}
+
+server {
+  listen [{{ backend_info.host }}]:{{ backend_info.port }}{% if cfg_https %} ssl spdy{% endif %};
+
+  {# we don't use: kerbeeros
+  <% if @kerberos_enabled && @kerberos_use_dedicated_port %>
+  listen <%= listen_address %>:<%= @kerberos_port %><% if @kerberos_https %> ssl<% end %>;
+  <% end %>
+  #}
+
+  server_name {{ fqdn }};
+  server_tokens off; ## Don't show the nginx version number, a security best practice
+  root {{ gitlab_work.location }}/public;
+
+  ## Increase this if you want to upload large attachments
+  ## Or if you want to accept large git objects over http
+  client_max_body_size {{ cfg('nginx_client_max_body_size') }};
+
+  {% if cfg_https %}
+  ## Strong SSL Security
+  ## https://raymii.org/s/tutorials/Strong_SSL_Security_On_nginx.html & https://cipherli.st/
+  ssl on;
+  ssl_certificate {{ nginx.cert_file }};
+  ssl_certificate_key {{ nginx.key_file }};
+  {# we don't need - most root CA will be included by default
+  <% if @ssl_client_certificate %>
+  ssl_client_certificate <%= @ssl_client_certificate%>;
+ 	<% end %>
+  #}
+
+  # GitLab needs backwards compatible ciphers to retain compatibility with Java IDEs
+  # NOTE(slapos) ^^^ is not relevant for us - we are behind frontend and clients
+  #     directly connects to frontend
+  ssl_ciphers '{{ cfg("nginx_ssl_ciphers") }}';
+  ssl_protocols  {{ cfg('nginx_ssl_protocols') }};
+  ssl_prefer_server_ciphers {{ cfg('nginx_ssl_prefer_server_ciphers') }};
+  ssl_session_cache  {{ cfg('nginx_ssl_session_cache') }};
+  ssl_session_timeout  {{ cfg('nginx_ssl_session_timeout') }};
+
+  {# we do not use: ssl_dhparam
+  <% if @ssl_dhparam %>
+  ssl_dhparam <%= @ssl_dhparam %>;
+  <% end %>
+  #}
+  {% endif %}
+
+  ## Individual nginx logs for this GitLab vhost
+  access_log  {{ nginx.log }}/gitlab_access.log gitlab_access;
+  error_log   {{ nginx.log }}/gitlab_error.log;
+
+  location / {
+    ## Serve static files from defined root folder.
+    ## @gitlab is a named location for the upstream fallback, see below.
+    try_files $uri /index.html $uri.html @gitlab;
+  }
+
+  location /uploads/ {
+    ## If you use HTTPS make sure you disable gzip compression
+    ## to be safe against BREACH attack.
+    {{ 'gzip off;' if cfg_https else ''}}
+
+    ## https://github.com/gitlabhq/gitlabhq/issues/694
+    ## Some requests take more than 30 seconds.
+    proxy_read_timeout      {{ cfg('nginx_proxy_read_timeout') }};
+    proxy_connect_timeout   {{ cfg('nginx_proxy_connect_timeout') }};
+    proxy_redirect          off;
+
+    proxy_set_header    Host                $http_host;
+    proxy_set_header    X-Real-IP           $remote_addr;
+    {% if cfg_https %}
+    proxy_set_header    X-Forwarded-Ssl     on;
+    {% endif %}
+    proxy_set_header    X-Forwarded-For     $proxy_add_x_forwarded_for;
+    proxy_set_header    X-Forwarded-Proto   {{ "https" if cfg_https else "http" }};
+    proxy_set_header    X-Frame-Options     SAMEORIGIN;
+
+    proxy_pass http://gitlab;
+  }
+
+  ## If a file, which is not found in the root folder is requested,
+  ## then the proxy passes the request to the upsteam (gitlab unicorn).
+  location @gitlab {
+    ## If you use HTTPS make sure you disable gzip compression
+    ## to be safe against BREACH attack.
+    {{ 'gzip off;' if cfg_https else ''}}
+
+    ## https://github.com/gitlabhq/gitlabhq/issues/694
+    ## Some requests take more than 30 seconds.
+    proxy_read_timeout      {{ cfg('nginx_proxy_read_timeout') }};
+    proxy_connect_timeout   {{ cfg('nginx_proxy_connect_timeout') }};
+    proxy_redirect          off;
+
+    proxy_set_header    Host                $http_host;
+    proxy_set_header    X-Real-IP           $remote_addr;
+    {% if cfg_https %}
+    proxy_set_header    X-Forwarded-Ssl     on;
+    {% endif %}
+    proxy_set_header    X-Forwarded-For     $proxy_add_x_forwarded_for;
+    proxy_set_header    X-Forwarded-Proto   {{ "https" if cfg_https else "http" }};
+    proxy_set_header    X-Frame-Options     SAMEORIGIN;
+
+    proxy_pass http://gitlab;
+  }
+
+  location ~ ^/[\w\.-]+/[\w\.-]+/gitlab-lfs/objects {
+    client_max_body_size 0;
+    # 'Error' 418 is a hack to re-use the @gitlab-workhorse block
+    error_page 418 = @gitlab-workhorse;
+    return 418;
+  }
+
+  location ~ ^/[\w\.-]+/[\w\.-]+/(info/refs|git-upload-pack|git-receive-pack)$ {
+    client_max_body_size 0;
+    # 'Error' 418 is a hack to re-use the @gitlab-workhorse block
+    error_page 418 = @gitlab-workhorse;
+    return 418;
+  }
+
+  location ~ ^/[\w\.-]+/[\w\.-]+/repository/archive {
+    client_max_body_size 0;
+    # 'Error' 418 is a hack to re-use the @gitlab-workhorse block
+    error_page 418 = @gitlab-workhorse;
+    return 418;
+  }
+
+  location ~ ^/api/v3/projects/.*/repository/archive {
+    client_max_body_size 0;
+    # 'Error' 418 is a hack to re-use the @gitlab-workhorse block
+    error_page 418 = @gitlab-workhorse;
+    return 418;
+  }
+
+  # Build artifacts should be submitted to this location
+  location ~ ^/[\w\.-]+/[\w\.-]+/builds/download {
+    client_max_body_size 0;
+    # 'Error' 418 is a hack to re-use the @gitlab-workhorse block
+    error_page 418 = @gitlab-workhorse;
+    return 418;
+  }
+
+  # Build artifacts should be submitted to this location
+  location ~ /ci/api/v1/builds/[0-9]+/artifacts {
+    client_max_body_size 0;
+    # 'Error' 418 is a hack to re-use the @gitlab-workhorse block
+    error_page 418 = @gitlab-workhorse;
+    return 418;
+  }
+
+  # access to raw blobs -> @gitlab-workhorse
+  location ~ ^/[\w\.-]+/[\w\.-]+/raw/ {
+    client_max_body_size 0;
+    error_page 418 = @gitlab-workhorse;
+    return 418;
+  }
+
+  location @gitlab-workhorse {
+    client_max_body_size 0;
+    ## If you use HTTPS make sure you disable gzip compression
+    ## to be safe against BREACH attack.
+    {{ 'gzip off;' if cfg_https else ''}}
+
+    ## https://github.com/gitlabhq/gitlabhq/issues/694
+    ## Some requests take more than 30 seconds.
+    proxy_read_timeout      {{ cfg('nginx_proxy_read_timeout') }};
+    proxy_connect_timeout   {{ cfg('nginx_proxy_connect_timeout') }};
+    proxy_redirect          off;
+
+    proxy_set_header    Host                $http_host;
+    proxy_set_header    X-Real-IP           $remote_addr;
+    {% if cfg_https %}
+    proxy_set_header    X-Forwarded-Ssl     on;
+    {% endif %}
+    proxy_set_header    X-Forwarded-For     $proxy_add_x_forwarded_for;
+    proxy_set_header    X-Forwarded-Proto   {{ "https" if cfg_https else "http" }};
+
+    proxy_pass http://gitlab-workhorse;
+  }
+
+  ## Enable gzip compression as per rails guide:
+  ## http://guides.rubyonrails.org/asset_pipeline.html#gzip-compression
+  ## WARNING: If you are using relative urls remove the block below
+  ## See config/application.rb under "Relative url support" for the list of
+  ## other files that need to be changed for relative url support
+  location ~ ^/(assets)/ {
+    root {{ gitlab_work.location }}/public;
+    gzip_static on; # to serve pre-gzipped version
+    expires max;
+    add_header Cache-Control public;
+  }
+
+
+  error_page 502 /502.html;
+
+  {# we don't support custom nginx configs
+  <%= @custom_gitlab_server_config %>
+  #}
+}

software/gitlab/template/nginx.conf.in

+{{ autogenerated }}
+# see:
+# https://gitlab.com/gitlab-org/gitlab-ce/blob/master/lib/support/nginx/gitlab-ssl
+# https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-cookbooks/gitlab/templates/default/nginx.conf.erb
+# (last updated for omnibus-gitlab 8.2.3+ce.0-0-g8eda093)
+
+{% from 'macrolib.cfg.in' import cfg  with context %}
+
+# user directive makes sense only when running initially as root
+# (and nginx will complain if not and directive given)
+# user {{ backend_info.user }};
+
+worker_processes {{ cfg('nginx_worker_processes') }};
+error_log stderr;
+pid {{ directory.run }}/nginx.pid;
+
+daemon off;
+
+events {
+  worker_connections {{ cfg('nginx_worker_connections') }};
+}
+
+http {
+  log_format gitlab_access '{{ cfg("nginx_log_format") }}';
+  {# we do not use: ci, mattermost
+  log_format gitlab_ci_access '<%= @gitlab_ci_access_log_format %>';
+  log_format gitlab_mattermost_access '<%= @gitlab_mattermost_access_log_format %>';
+  #}
+
+  sendfile    {{ cfg('nginx_sendfile') }};
+  tcp_nopush  {{ cfg('nginx_tcp_nopush') }};
+  tcp_nodelay {{ cfg('nginx_tcp_nodelay') }};
+
+  keepalive_timeout {{ cfg('nginx_keepalive_timeout') }};
+
+  gzip              {{ cfg('nginx_gzip') }};
+  gzip_http_version {{ cfg('nginx_gzip_http_version') }};
+  gzip_comp_level   {{ cfg('nginx_gzip_comp_level') }};
+  gzip_proxied      {{ cfg('nginx_gzip_proxied') }};
+  gzip_types        {{ cfg('nginx_gzip_types') }};
+
+  include {{ nginx_mime_types }};
+
+  include {{ nginx_gitlab_http_conf }};
+
+  {# we don't need: ci, mattermost
+  include <%= @gitlab_ci_http_config %>
+  include <%= @gitlab_mattermost_http_config %>
+  #}
+}

software/gitlab/template/rack_attack.rb.in

+{{ autogenerated }}
+# see:
+# https://gitlab.com/gitlab-org/gitlab-ce/blob/master/config/initializers/rack_attack.rb.example
+# https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-cookbooks/gitlab/templates/default/rack_attack.rb.erb
+# (last updated for omnibus-gitlab 8.2.3+ce.0-0-g8eda093)
+
+{% from 'macrolib.cfg.in' import cfg  with context %}
+
+# 1. Rename this file to rack_attack.rb
+# 2. Review the paths_to_be_protected and add any other path you need protecting
+#
+
+paths_to_be_protected = [
+  "#{Rails.application.config.relative_url_root}/users/password",
+  "#{Rails.application.config.relative_url_root}/users/sign_in",
+  "#{Rails.application.config.relative_url_root}/api/#{API::API.version}/session.json",
+  "#{Rails.application.config.relative_url_root}/api/#{API::API.version}/session",
+  "#{Rails.application.config.relative_url_root}/users",
+  "#{Rails.application.config.relative_url_root}/users/confirmation",
+  "#{Rails.application.config.relative_url_root}/unsubscribes/"
+
+]
+
+# Create one big regular expression that matches strings starting with any of
+# the paths_to_be_protected.
+paths_regex = Regexp.union(paths_to_be_protected.map { |path| /\A#{Regexp.escape(path)}/ })
+
+unless Rails.env.test?
+  Rack::Attack.throttle('protected paths', limit: {{ cfg('rate_limit_requests_per_period') }}, period: {{ cfg('rate_limit_period') }}.seconds) do |req|
+    if req.post? && req.path =~ paths_regex
+      req.ip
+    end
+  end
+end

software/gitlab/template/resque.yml.in

+{{ autogenerated }}
+# see:
+# https://gitlab.com/gitlab-org/gitlab-ce/blob/master/config/resque.yml.example
+# https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-cookbooks/gitlab/templates/default/resque.yml.erb
+# (last udpdated for omnibus-gitlab 8.2.3+ce.0-0-g8eda093)
+
+production: unix://{{ redis.unixsocket }}

software/gitlab/template/smtp_settings.rb.in

+{{ autogenerated }}
+# see:
+# https://gitlab.com/gitlab-org/gitlab-ce/blob/master/config/initializers/smtp_settings.rb.sample
+# https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-cookbooks/gitlab/templates/default/smtp_settings.rb.erb
+# (last updated for omnibus-gitlab 8.2.3+ce.0-0-g8eda093)
+
+{% from 'macrolib.cfg.in' import cfg, cfg_bool  with context %}
+
+{% if cfg_bool('smtp_enable') %}
+if Rails.env.production?
+  Gitlab::Application.config.action_mailer.delivery_method = :smtp
+
+  ActionMailer::Base.smtp_settings = {
+    address:                "{{ cfg('smtp_address') }}",
+    port:                   {{ cfg('smtp_port') }},
+    user_name:              "{{ cfg('smtp_user_name') }}",
+    password:               "{{ cfg('smtp_password') }}",
+    domain:                 "{{ cfg('smtp_domain') }}",
+    authentication:         :{{ cfg('smtp_authentication') }},
+    enable_starttls_auto:   {{ cfg('smtp_enable_starttls_auto') }},
+    openssl_verify_mode:    '{{ cfg("smtp_openssl_verify_mode") }}'
+    # ca_path:
+    # ca_file:
+  }
+end
+{% else %}
+# SMTP disabled in instance configuration (see `smtp_enable` parameter).
+# Mail sending, if enabled (see `email_enabled`), will be done via sendmail.
+{% endif %}

software/gitlab/template/unicorn.rb.in

+{{ autogenerated }}
+# see:
+# https://gitlab.com/gitlab-org/gitlab-ce/blob/master/config/unicorn.rb.example
+# https://gitlab.com/gitlab-org/gitlab-ce/blob/master/config/unicorn.rb.example.development
+# https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-cookbooks/gitlab/templates/default/unicorn.rb.erb
+# (last updated for omnibus-gitlab 8.2.3+ce.0-0-g8eda093)
+
+{% from 'macrolib.cfg.in' import cfg  with context %}
+
+# What ports/sockets to listen on, and what options for them.
+# we listen only on unix socket
+listen "{{ unicorn.socket }}", :backlog => {{ cfg('unicorn_backlog_socket') }}
+#listen "127.0.0.1:8888", :tcp_nopush => true
+
+working_directory '{{ gitlab_work.location }}'
+
+# What the timeout for killing busy workers is, in seconds
+timeout {{ cfg('unicorn_worker_timeout') }}
+
+# Whether the app should be pre-loaded
+preload_app true
+
+# How many worker processes
+worker_processes {{ cfg('unicorn_worker_processes') }}
+
+# about before_fork / after_fork - see:
+#   https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-cookbooks/gitlab/definitions/unicorn_service.rb
+#   http://bogomips.org/unicorn.git/tree/examples/unicorn.conf.rb?id=3312aca8#n75
+
+# What to do before we fork a worker
+before_fork do |server, worker|
+  # XXX why gitlab does not enable this?
+  # # the following is highly recomended for Rails + "preload_app true"
+  # # as there's no need for the master process to hold a connection
+  # defined?(ActiveRecord::Base) and
+  #   ActiveRecord::Base.connection.disconnect!
+
+  # This allows a new master process to incrementally
+  # phase out the old master process with SIGTTOU to avoid a
+  # thundering herd (especially in the "preload_app false" case)
+  # when doing a transparent upgrade.  The last worker spawned
+  # will then kill off the old master process with a SIGQUIT.
+  old_pid = "#{server.config[:pid]}.oldbin"
+  if old_pid != server.pid
+    begin
+      sig = (worker.nr + 1) >= server.worker_processes ? :QUIT : :TTOU
+      Process.kill(sig, File.read(old_pid).to_i)
+    rescue Errno::ENOENT, Errno::ESRCH
+    end
+  end
+end
+
+# What to do after we fork a worker
+after_fork do |server, worker|
+  # per-process listener ports for debugging/admin/migrations
+  # addr = "127.0.0.1:#{9293 + worker.nr}"
+  # server.listen(addr, :tries => -1, :delay => 5, :tcp_nopush => true)
+
+  # XXX why gitlab does not enable this?
+  # # the following is *required* for Rails + "preload_app true",
+  # defined?(ActiveRecord::Base) and
+  #   ActiveRecord::Base.establish_connection
+end
+
+
+# Where to drop a pidfile
+pid '{{ directory.run }}/unicorn.pid'
+
+# Where stderr gets logged
+stderr_path '{{ unicorn.log }}/unicorn_stderr.log'
+
+# Where stdout gets logged
+stdout_path '{{ unicorn.log }}/unicorn_stdout.log'

software/gitlab/watcher-sigkill.in

+#!{{ bash.location }}/bin/bash
+# run program under SIGKILL watchdog
+# watcher-sigkill <prog> [<progargs> ...]
+#
+# if the program terminates with SIGKILL - it is restarted after grace period.
+# if the program terminates otherwise - whole process terminates.
+
+if [ "$#" -lt 1 ]; then
+    echo "Usage: watcher-sigkill <prog> [<progargs> ...]" 1>&2
+    exit 1
+fi
+prog="$@"
+
+progpid=""
+killexit="137"  # = 128 + 9  (exit code of process terminated by SIGKILL)
+
+# make sure to terminate children, when we exit.
+# needed for e.g. when `slapos node stop ...` kills us.
+trap 'atexit' EXIT
+atexit() {
+    jobs="$(jobs -p)"
+    test -n "$jobs" && kill $jobs
+}
+
+# run prog under monitoring
+while true; do
+    echo "run $prog"
+    $prog &
+    progpid=$!
+    echo "wait $progpid"
+    wait $progpid
+    status=$?
+    echo "-> $status"
+
+    # if program terminated not by SIGKILL - exit
+    if [ "$status" != "$killexit" ] ; then
+        echo "exit $status"
+        exit "$status"
+    fi
+
+    # otherwise sleep a bit and restart
+    sleep 1
+done

software/ipython_notebook/instance.cfg.in

@@ -34,13 +34,17 @@ develop-eggs-directory = {{ develop_eggs_directory }}
 offline = true
 
 [slapconfiguration]
-recipe = slapos.cookbook:slapconfiguration
+recipe = slapos.cookbook:slapconfiguration.serialised
 computer = ${slap-connection:computer-id}
 partition = ${slap-connection:partition-id}
 url = ${slap-connection:server-url}
 key = ${slap-connection:key-file}
 cert = ${slap-connection:cert-file}
 
+# ERP5 URL to use in Jupyter by default
+# default value is empty - which means no default ERP5 URL
+configuration.erp5-url =
+
 [instance-parameter]
 port = 8888
 host = ${slapconfiguration:ipv6-random}
@@ -141,6 +145,7 @@ rendered = ${directory:erp5_kernel_dir}/ERP5kernel.py
 # Use ipython as executable python as we'll be needing requests library in kernel
 context =
   raw python_executable {{ bin_directory }}/ipython
+  key erp5_url slapconfiguration:configuration.erp5-url
 
 [kernel-json]
 <= dynamic-jinja2-template-base

software/ipython_notebook/software.cfg

@@ -43,7 +43,7 @@ md5sum = d7d4a7e19d55bf14007819258bf42100
 [erp5-kernel]
 <= download-file-base
 filename = ERP5kernel.py.jinja
-md5sum = da2f592075c414d4bb26cf7a7dfd147b
+md5sum = 3dfc6a7c16828bff55dec4cf96b730d3
 
 [kernel-json]
 <= download-file-base
@@ -60,7 +60,7 @@ recipe = slapos.recipe.template:jinja2
 template = ${:_profile_base_location_}/instance.cfg.in
 rendered = ${buildout:directory}/template.cfg
 mode = 0644
-md5sum = 1a993b1f8fa3f001c45075fe95a48332
+md5sum = c6b82a386a72ed72301302c3132ffb71
 context =
   key bin_directory buildout:bin-directory
   key develop_eggs_directory buildout:develop-eggs-directory

software/ipython_notebook/template/ERP5kernel.py.jinja

@@ -9,9 +9,11 @@ import requests
 import json
 
 # erp5_url from buildout
-# TODO: Uncomment after adding automated installation of erp5-data-notebook bt5
-# url = ""
-# url  = "%s/erp5/Base_executeJupyter"%url
+erp5_url = "{{ erp5_url }}"
+if not erp5_url:
+    erp5_url = None
+else:
+    erp5_url = "%s/erp5/Base_executeJupyter" % erp5_url
 
 class MagicInfo:
   """
@@ -69,9 +71,12 @@ class ERP5Kernel(Kernel):
     super(ERP5Kernel, self).__init__(*args, **kwargs)
     self.user = user
     self.password = password
-    # Use URL provided by buildout during initiation
+    # By default use URL provided by buildout during initiation
     # It can later be overridden
-    self.url = url
+    if url is None:
+        self.url = erp5_url
+    else:
+        self.url = url
     self.status_code = status_code
     self.reference = None
     self.title = None
@@ -167,11 +172,11 @@ class ERP5Kernel(Kernel):
     """
 
     try:
-      erp5_request = requests.get(
+      erp5_request = requests.post(
         self.url,
         verify=False,
         auth=(self.user, self.password),
-        params={
+        data={
           'python_expression': code,
           'reference': self.reference,
           'title': self.title,

software/neoppod/instance-neo-input-schema.json

 {
   "$schema": "http://json-schema.org/draft-04/schema#",
-  "description": "Parameters to instantiate a NEO cluster. See https://git.erp5.org/gitweb/neoppod.git/blob/HEAD:/neo.conf?js=1 for more information.",
+  "description": "Parameters to instantiate a NEO cluster. See https://lab.nexedi.com/nexedi/neoppod/blob/master/neo.conf for more information.",
   "additionalProperties": false,
   "required": ["cluster"],
   "properties": {

software/neoppod/software-common.cfg

@@ -27,7 +27,7 @@ parts =
 
 [neoppod-repository]
 recipe = slapos.recipe.build:gitclone
-repository = http://git.erp5.org/repos/neoppod.git
+repository = https://lab.nexedi.com/nexedi/neoppod.git
 git-executable = ${git:location}/bin/git
 
 [neoppod-develop]
@@ -40,7 +40,7 @@ eggs = neoppod[admin, ctl, master, storage-importer, storage-mysqldb]
   ${mysql-python:egg}
   ZODB3
 ZODB3-patches =
-  ${:_profile_base_location_}/../../component/egg-patch/ZODB3-3.10.5.patch
+  ${:_profile_base_location_}/../../component/egg-patch/ZODB3-3.10.5.patch#c5fe331b1e3a930446f93ab4f6e97c6e
 ZODB3-patch-options = -p1
 
 [slapos-deps-eggs]
@@ -116,7 +116,7 @@ slapos.recipe.template = 2.8
 ZODB3 = 3.10.5+SlapOSPatched001
 # Required by slapos.toolbox==0.52
 slapos.toolbox = 0.52
-apache-libcloud = 0.20.0
+apache-libcloud = 0.20.1
 atomize = 0.2.0
 ecdsa = 0.13
 feedparser = 5.2.1

software/slaprunner/software.cfg

@@ -9,7 +9,7 @@ extends = common.cfg
 [versions]
 Flask-Auth = 0.85
 PyRSS2Gen = 1.1
-apache-libcloud = 0.19.0
+apache-libcloud = 0.20.1
 cns.recipe.symlink = 0.2.3
 collective.recipe.environment = 0.2.0
 ecdsa = 0.13

software/wendelin/software.cfg

 [buildout]
 versions = versions
 extends =
-  ../../software/ipython_notebook/software.cfg
   ../../component/fluentd/buildout.cfg
-  ../../component/matplotlib/buildout.cfg
-  ../../component/ipython/buildout.cfg
-  ../../component/pandas/buildout.cfg
   ../../component/wendelin.core/buildout.cfg
   ../../component/msgpack-python/buildout.cfg
   ../../component/scipy/buildout.cfg
-  ../../component/scikit-learn/buildout.cfg
   ../../software/erp5/software.cfg
 parts +=
   wendelin
   scipy
-  scikit-learn
-  pandas
   msgpack-python
   ipython
   wendelin.core
-  matplotlib
   fluentd
   ipython-notebook
 
@@ -29,13 +21,10 @@ initialization =
 extra-paths +=
   ${wendelin:location}
 eggs +=
-  ${scikit-learn:egg}
   ${scipy:egg}
-  ${pandas:egg}
   ${msgpack-python:egg}
   ${wendelin.core:egg}
   ${ipython:egg}
-  ${matplotlib:egg}
 
 [erp5_repository_list]
 repository_id_list += wendelin
@@ -43,7 +32,10 @@ repository_id_list += wendelin
 [local-bt5-repository]
 # we need to override it
 list = ${erp5:location}/bt5 ${erp5:location}/product/ERP5/bootstrap ${wendelin:location}/bt5/
-bt5_list = erp5_full_text_myisam_catalog erp5_configurator_standard erp5_configurator_maxma_demo erp5_configurator_ung erp5_configurator_run_my_doc erp5_configurator_ebusiness_lotse erp5_wendelin_configurator
+
+# Jupyter is by default enabled in Wendelin
+[erp5-defaults]
+jupyter-enable-default = true
 
 [wendelin]
 <= erp5
@@ -58,11 +50,6 @@ revision = 8f58ae080576bd3f70b01892e127b5a0552ad17b
 revision = 498ccf943f1dac3dbf549e9d6e5d7bbed111af12 
 
 [versions]
-scikit-learn = 0.16.1
 scipy = 0.15.1
-pandas = 0.16.1
 msgpack-python = 0.4.6
-numpy = 1.9.2
 wendelin.core = 0.5
-ipython = 3.2.0
-matplotlib = 1.4.3

stack/erp5/buildout.cfg

@@ -21,14 +21,17 @@ extends =
   ../../component/libffi/buildout.cfg
   ../../component/libpng/buildout.cfg
   ../../component/libreoffice-bin/buildout.cfg
+  ../../component/matplotlib/buildout.cfg
   ../../component/mesa/buildout.cfg
   ../../component/numpy/buildout.cfg
+  ../../component/pandas/buildout.cfg
   ../../component/percona-toolkit/buildout.cfg
   ../../component/patch/buildout.cfg
   ../../component/pillow/buildout.cfg
   ../../component/pysvn-python/buildout.cfg
   ../../component/python-ldap-python/buildout.cfg
   ../../component/rdiff-backup/buildout.cfg
+  ../../component/scikit-learn/buildout.cfg
   ../../component/stunnel/buildout.cfg
   ../../component/subversion/buildout.cfg
   ../../component/tesseract/buildout.cfg
@@ -49,6 +52,7 @@ extends =
   ../../component/findutils/buildout.cfg
   ../../component/userhosts/buildout.cfg
   ../../component/postfix/buildout.cfg
+  ../../software/ipython_notebook/software.cfg
   ../../software/neoppod/software-common.cfg
 # keep neoppod extends last
 
@@ -123,6 +127,15 @@ parts +=
 # Create instance template
   template
 
+# jupyter
+  ipython-notebook
+  instance-jupyter
+  monitor-eggs
+
+# override instance-jupyter not to render into default template.cfg
+[instance-jupyter]
+rendered = ${buildout:directory}/template-jupyter.cfg
+
 [download-base]
 <= download-base-neo
 url = ${:_profile_base_location_}/${:filename}
@@ -220,7 +233,7 @@ recipe = slapos.recipe.template:jinja2
 # XXX: "template.cfg" is hardcoded in instanciation recipe
 rendered = ${buildout:directory}/template.cfg
 template = ${:_profile_base_location_}/instance.cfg.in
-md5sum = 540956c635acc9707045510c11f80016
+md5sum = 98a4edfb18cfd810ea570f56d502a2cc
 mode = 640
 context =
     key mariadb_link_binary template-mariadb:link-binary
@@ -250,6 +263,7 @@ context =
     key haproxy_location haproxy:location
     key instance_common_cfg instance-common:rendered
     key jsl_location jsl:location
+    key jupyter_enable_default erp5-defaults:jupyter-enable-default
     key kumo_location kumo:location
     key libICE_location libICE:location
     key libSM_location libSM:location
@@ -283,6 +297,7 @@ context =
     key template_create_erp5_site_real template-create-erp5-site-real:target
     key template_erp5 template-erp5:target
     key template_haproxy_cfg template-haproxy-cfg:target
+    key template_jupyter_cfg instance-jupyter:rendered
     key template_kumofs template-kumofs:target
     key template_mariadb template-mariadb:target
     key template_mariadb_initial_setup template-mariadb-initial-setup:target
@@ -314,7 +329,7 @@ rendered = ${monitor-template-dummy:target}
 [template-erp5]
 <= download-base
 filename = instance-erp5.cfg.in
-md5sum = 977119d0b876df827c97bb64e6e98273
+md5sum = 66edf64eeaecded8977459acb26f4424
 
 [template-zeo]
 <= download-base
@@ -384,6 +399,11 @@ update-command = ${:command}
 [erp5_repository_list]
 repository_id_list = erp5
 
+# ERP5 defaults, which can be overridden in inheriting recipes (e.g. wendelin)
+[erp5-defaults]
+# Jupyter is by default disabled in ERP5
+jupyter-enable-default = false
+
 [erp5]
 recipe = slapos.recipe.build:gitclone
 repository = http://git.erp5.org/repos/erp5.git
@@ -451,12 +471,15 @@ initialization =
 <= neoppod
 eggs =
   ${numpy:egg}
+  ${matplotlib:egg}
   ${mysql-python:egg}
   ${lxml-python:egg}
+  ${pandas:egg}
   ${pillow-python:egg}
   ${python-ldap-python:egg}
   ${pysvn-python:egg}
   ${pycrypto-python:egg}
+  ${scikit-learn:egg}
   lock_file
   PyStemmer
   PyXML
@@ -465,7 +488,6 @@ eggs =
   chardet
   collective.recipe.template
   coverage
-  elementtree
   erp5diff
   inotifyx
   interval
@@ -489,7 +511,6 @@ eggs =
   xml_marshaller
   xupdate_processor
   feedparser
-  argparse
   validictory
   erp5.util
   huBarcode
@@ -563,10 +584,12 @@ extra-paths =
 
 # patches for eggs
 patch-binary = ${patch:location}/bin/patch
-Acquisition-patches = ${:_profile_base_location_}/../../component/egg-patch/Acquisition/aq_dynamic.patch
+Acquisition-patches = ${:_profile_base_location_}/../../component/egg-patch/Acquisition/aq_dynamic.patch#e8029103350dad364d25747514a20327
 Acquisition-patch-options = -p1
-Products.DCWorkflow-patches = ${:_profile_base_location_}/../../component/egg-patch/Products.DCWorkflow/workflow_method.patch
+Products.DCWorkflow-patches = ${:_profile_base_location_}/../../component/egg-patch/Products.DCWorkflow/workflow_method.patch#975b49e96bae33ac8563454fe5fa9899
 Products.DCWorkflow-patch-options = -p1
+python-magic-patches = ${:_profile_base_location_}/../../component/egg-patch/python_magic/magic.patch#de0839bffac17801e39b60873a6c2068
+python-magic-patch-options = -p1
 
 [zodbanalyze]
 recipe = zc.recipe.egg
@@ -602,6 +625,7 @@ Acquisition = 2.13.8+SlapOSPatched001
 Products.DCWorkflow = 2.2.4+SlapOSPatched001
 pysvn = 1.7.10+SlapOSPatched002
 python-ldap = 2.4.22+SlapOSPatched001
+python-magic = 0.4.10+SlapOSPatched001
 
 # specify dev version to be sure that an old released version is not used
 cloudooo = 1.2.5-dev
@@ -660,9 +684,8 @@ WSGIUtils = 0.7
 astroid = 1.3.8
 chardet = 2.3.0
 csp-eventlet = 0.7.0
-elementtree = 1.2.6.post20050316
 erp5diff = 0.8.1.7
-eventlet = 0.17.4
+eventlet = 0.18.1
 five.formlib = 1.0.4
 five.localsitemanager = 2.0.5
 greenlet = 0.4.9
@@ -673,8 +696,10 @@ interval = 1.0.0
 ipdb = 0.8.1
 ipython = 4.0.0
 logilab-common = 1.1.0
+matplotlib = 1.4.3
 numpy = 1.10.4
 objgraph = 2.0.1
+pandas = 0.16.1
 ply = 3.8
 polib = 1.0.7
 pprofile = 1.7.3
@@ -683,12 +708,12 @@ pycountry = 1.19
 pyflakes = 1.0.0
 # pylint 1.5.1 breaks testDynamicClassGeneration
 pylint = 1.4.4
-python-magic = 0.4.10
 python-memcached = 1.57
 pytracemalloc = 1.2
-qrcode = 5.1
+qrcode = 5.2.2
 restkit = 4.2.2
 rtjp-eventlet = 0.3.2
+scikit-learn = 0.16.1
 simplegeneric = 0.8.1
 socketpool = 0.5.3
 spyne = 2.12.11
@@ -705,6 +730,10 @@ xupdate-processor = 0.4
 # Products.CMFCore==2.2.9
 Products.ZSQLMethods = 2.13.4
 
+# Required by:
+# qrcode==5.2.2
+colorama = 0.3.6
+
 # Required by:
 # SOAPpy===0.12.0nxd001
 fpconst = 0.7.2

stack/erp5/instance-erp5.cfg.in

@@ -5,6 +5,9 @@
 {% set inituser_login = slapparameter_dict.get('inituser-login', 'zope') -%}
 {% set publish_dict = {'site-id': site_id, 'inituser-login': inituser_login} -%}
 {% set has_posftix = slapparameter_dict.get('smtp', {}).get('postmaster') -%}
+{% set jupyter_dict = slapparameter_dict.get('jupyter', {}) -%}
+{% set has_jupyter = jupyter_dict.get('enable', jupyter_enable_default).lower() in ('true', 'yes') -%}
+{% set jupyter_zope_family = jupyter_dict.get('zope-family', '') -%}
 [request-common]
 <= request-common-base
 config-use-ipv6 = {{ dumps(slapparameter_dict.get('use-ipv6', False)) }}
@@ -119,7 +122,11 @@ name = neo-${gen-neo-cluster-base:passwd}
 return =
   zope-address-list
   hosts-dict
-config-bt5 = {{ dumps(slapparameter_dict.get('bt5', 'erp5_full_text_myisam_catalog erp5_configurator_standard erp5_configurator_maxma_demo erp5_configurator_ung erp5_configurator_run_my_doc')) }}
+{% set bt5_default_list = 'erp5_full_text_myisam_catalog erp5_configurator_standard erp5_configurator_maxma_demo erp5_configurator_ung erp5_configurator_run_my_doc' -%}
+{% if has_jupyter -%}
+{%   set bt5_default_list = bt5_default_list + ' erp5_data_notebook' -%}
+{% endif -%}
+config-bt5 = {{ dumps(slapparameter_dict.get('bt5', bt5_default_list)) }}
 config-bt5-repository-url = {{ dumps(slapparameter_dict.get('bt5-repository-url', local_bt5_repository)) }}
 config-cloudooo-url = ${request-cloudooo:connection-url}
 config-deadlock-debugger-password = ${publish-early:deadlock-debugger-password}
@@ -150,10 +157,17 @@ config-tidstorage-port = ${request-zodb:connection-tidstorage-port}
 software-type = zope
 
 {% set zope_family_dict = {} -%}
+{% set jupyter_zope_family_default = [] -%}
 {% for custom_name, zope_parameter_dict in slapparameter_dict.get('zope-partition-dict', {'1': {}}).items() -%}
 {%   set partition_name = 'zope-' ~ custom_name -%}
 {%   set section_name = 'request-' ~ partition_name -%}
-{%   do zope_family_dict.setdefault(zope_parameter_dict.get('family', 'default'), []).append(section_name) -%}
+{%   set zope_family = zope_parameter_dict.get('family', 'default') -%}
+{#   # default jupyter zope family is first zope family. -#}
+{#   # use list.append() to update it, because in jinja2 set changes only local scope. -#}
+{%   if not jupyter_zope_family_default -%}
+{%     do jupyter_zope_family_default.append(zope_family) -%}
+{%   endif -%}
+{%   do zope_family_dict.setdefault(zope_family, []).append(section_name) -%}
 [{{ section_name }}]
 <= request-zope-base
 name = {{ partition_name }}
@@ -168,6 +182,12 @@ config-port-base = {{ dumps(zope_parameter_dict.get('port-base', 2200)) }}
 config-webdav = {{ dumps(zope_parameter_dict.get('webdav', False)) }}
 {% endfor -%}
 
+{# if not explicitly configured, connect jupyter to first zope family, which  -#}
+{# will be 'default' if zope families are not configured also -#}
+{% if not jupyter_zope_family and jupyter_zope_family_default -%}
+{%   set jupyter_zope_family = jupyter_zope_family_default[0] -%}
+{% endif -%}
+
 {# We need to concatenate lists that we cannot read as lists, so this gets hairy. -#}
 {% set zope_address_list_id_dict = {} -%}
 {% set zope_family_parameter_dict = {} -%}
@@ -190,6 +210,20 @@ config-url = ${request-balancer:connection-{{ family_name }}-v6}
 {%   endif -%}
 {% endfor -%}
 
+{% if has_jupyter -%}
+{# request jupyter connected to balancer of proper zope family -#}
+{{   request('jupyter', 'jupyter', 'jupyter', {}, key_config={'erp5-url': 'request-balancer:connection-' ~ jupyter_zope_family}) }}
+
+{%   if has_frontend -%}
+[frontend-jupyter]
+<= request-frontend-base
+name = frontend-jupyter
+config-url = ${request-jupyter:connection-url}
+{#     # override jupyter-url in publish_dict with frontend address -#}
+{%     do publish_dict.__setitem__('jupyter-url', '${frontend-jupyter:connection-site_url}') -%}
+{%   endif -%}
+{%- endif %}
+
 {% set balancer_dict = slapparameter_dict.get('balancer', {}) -%}
 [request-balancer]
 <= request-common

stack/erp5/instance.cfg.in

@@ -64,6 +64,7 @@ extra-context =
     import urllib urllib
 
 [dynamic-template-erp5-parameters]
+jupyter-enable-default = {{ jupyter_enable_default }}
 local-bt5-repository = {{ local_bt5_repository }}
 
 [dynamic-template-erp5]
@@ -71,6 +72,7 @@ local-bt5-repository = {{ local_bt5_repository }}
 template = {{ template_erp5 }}
 filename = instance-erp5.cfg
 extra-context =
+    key jupyter_enable_default dynamic-template-erp5-parameters:jupyter-enable-default
     key local_bt5_repository dynamic-template-erp5-parameters:local-bt5-repository
     import urlparse urlparse
 import-list =
@@ -177,6 +179,11 @@ filename = instance-create-erp5-site.cfg
 extra-context =
     section parameter_dict dynamic-template-create-erp5-site-parameters
 
+# we need this value to be present in a section,
+# for slapos.cookbook:switch-softwaretype to work
+[dynamic-template-jupyter]
+rendered = {{ template_jupyter_cfg }}
+
 [switch-softwaretype]
 recipe = slapos.cookbook:switch-softwaretype
 override = {{ dumps(override_switch_softwaretype |default) }}
@@ -195,3 +202,4 @@ postfix = dynamic-template-postfix:rendered
 zodb-zeo = dynamic-template-zeo:rendered
 zodb-neo = neo-storage-mysql:rendered
 zope = dynamic-template-zope:rendered
+jupyter = dynamic-template-jupyter:rendered

stack/slapos.cfg

@@ -111,11 +111,11 @@ Jinja2 = 2.8
 PyYAML = 3.11
 Werkzeug = 0.11.3
 buildout-versions = 1.7
-cffi = 1.4.2
+cffi = 1.5.0
 cliff = 1.15.0
 cmd2 = 0.6.8
 collective.recipe.template = 1.13
-cryptography = 1.2.1
+cryptography = 1.2.2
 decorator = 4.0.6
 idna = 2.0
 inotifyx = 0.2.2
@@ -126,7 +126,7 @@ netaddr = 0.7.18
 pbr = 1.8.1
 plone.recipe.command = 1.1
 prettytable = 0.7.2
-psutil = 3.3.0
+psutil = 3.4.2
 pyOpenSSL = 0.15.1
 pyasn1 = 0.1.9
 pyparsing = 2.0.7
@@ -183,7 +183,7 @@ lock-file = 2.0
 netifaces = 0.10.4
 
 # Required by:
-# cffi==1.4.2
+# cffi==1.5.0
 pycparser = 2.14
 
 # Required by: