Merge remote-tracking branch 'origin/master' into erp5-component

component/coreutils/buildout.cfg

@@ -19,7 +19,7 @@ environment =
 recipe = plone.recipe.command
 stop-on-error = true
 update-command = ${:command}
-command = ${:test} -x ${:test} -a -x ${:cat} -a -x ${:rm} -a -x ${:echo} -a -x ${:date} -a -x ${:md5sum} -a -x ${:basename}
+command = ${:test} -x ${:test} -a -x ${:cat} -a -x ${:rm} -a -x ${:echo} -a -x ${:date} -a -x ${:md5sum} -a -x ${:basename} -a -x ${:sort} -a -x ${:tail}
 test = ${coreutils:location}/bin/test
 cat = ${coreutils:location}/bin/cat
 rm = ${coreutils:location}/bin/rm
@@ -27,3 +27,5 @@ echo = ${coreutils:location}/bin/echo
 date = ${coreutils:location}/bin/date
 md5sum = ${coreutils:location}/bin/md5sum
 basename = ${coreutils:location}/bin/basename
+sort = ${coreutils:location}/bin/sort
+tail = ${coreutils:location}/bin/tail

component/findutils/buildout.cfg

+[buildout]
+extends =
+  ../coreutils/buildout.cfg
+parts =
+  findutils-output
+
+[findutils]
+recipe = slapos.recipe.cmmi
+url = http://ftp.gnu.org/gnu/findutils/findutils-4.4.2.tar.gz
+md5sum = 351cc4adb07d54877fa15f75fb77d39f
+
+[findutils-output]
+# Shared binary location to ease migration
+recipe = plone.recipe.command
+stop-on-error = true
+update-command = ${:command}
+command = ${coreutils-output:test} -x ${:find} -a -x ${:xargs}
+find = ${findutils:location}/bin/find
+xargs = ${findutils:location}/bin/xargs

component/nginx/buildout.cfg

@@ -19,6 +19,27 @@ configure-options=
   --with-ld-opt="-L ${zlib:location}/lib -L ${openssl:location}/lib -L ${pcre:location}/lib  -Wl,-rpath=${pcre:location}/lib -Wl,-rpath=${zlib:location}/lib -Wl,-rpath=${openssl:location}/lib"
   --with-cc-opt="-I ${pcre:location}/include -I ${openssl:location}/include -I ${zlib:location}/include"
 
+[webdav-module]
+recipe = hexagonit.recipe.download
+url = https://github.com/arut/nginx-dav-ext-module/archive/master.zip
+strip-top-level-dir = true
+#md5sum =
+mode = 0644
+
+[nginx-webdav]
+recipe = slapos.recipe.cmmi
+url = http://nginx.org/download/nginx-1.5.3.tar.gz
+md5sum = 1e735dd6a6ade2b5c20e924b67c3d355
+configure-options =
+  --with-ipv6
+  --with-http_ssl_module
+  --with-mail
+  --with-mail_ssl_module
+  --with-ld-opt="-L ${zlib:location}/lib -L ${openssl:location}/lib -L ${pcre:location}/lib  -Wl,-rpath=${pcre:location}/lib -Wl,-rpath=${zlib:location}/lib -Wl,-rpath=${openssl:location}/lib -L ${libexpat:location}/lib"
+  --with-cc-opt="-I ${pcre:location}/include -I ${openssl:location}/include -I ${zlib:location}/include -I ${libexpat:location}/include"
+  --with-http_dav_module 
+  --add-module='${webdav-module:location}'
+
 [nginx-output]
 # Shared binary location to ease migration
 recipe = plone.recipe.command

component/python-cffi/buildout.cfg

+[buildout]
+extends =
+  ../libffi/buildout.cfg
+  ../pkgconfig/buildout.cfg
+
+parts =
+  python-cffi
+
+[python-cffi]
+recipe = zc.recipe.egg:custom
+egg = cffi
+environment = python-cffi-env
+library-dirs =
+  ${libffi:location}/lib/
+rpath =
+  ${libffi:location}/lib/
+
+[python-cffi-env]
+PATH = ${pkgconfig:location}/bin:%(PATH)s
+PKG_CONFIG_PATH = ${libffi:location}/lib/pkgconfig

component/python-cryptography/buildout.cfg

+[buildout]
+extends =
+  ../libffi/buildout.cfg
+  ../openssl/buildout.cfg
+  ../pkgconfig/buildout.cfg
+
+parts =
+  python-cryptography
+
+[python-cryptography]
+recipe = zc.recipe.egg:custom
+egg = cryptography
+environment = python-cryptography-env
+
+[python-cryptography-env]
+PATH = ${pkgconfig:location}/bin:%(PATH)s
+PKG_CONFIG_PATH = ${libffi:location}/lib/pkgconfig
+LD_LIBRARY_PATH = ${libffi:location}/lib
+CPATH = ${openssl:location}/include

component/python-openssl/buildout.cfg

-[buildout]
-extends =
-  ../openssl/buildout.cfg
-
-parts =
-  python-openssl
-
-[python-openssl]
-recipe = zc.recipe.egg:custom
-egg = pyOpenSSL
-include-dirs =
-  ${openssl:location}/include/
-library-dirs =
-  ${openssl:location}/lib/
-rpath =
-  ${openssl:location}/lib/

component/slapos/buildout.cfg

@@ -17,7 +17,8 @@ extends =
   ../pkgconfig/buildout.cfg
   ../popt/buildout.cfg
   ../python-2.7/buildout.cfg
-  ../python-openssl/buildout.cfg
+  ../python-cffi/buildout.cfg
+  ../python-cryptography/buildout.cfg
   ../readline/buildout.cfg
   ../sqlite3/buildout.cfg
   ../swig/buildout.cfg
@@ -91,14 +92,19 @@ output = ${buildout:directory}/environment.sh
 [lxml-python]
 python = python2.7
 
-[python-openssl]
+[python-cffi]
+python = python2.7
+
+[python-cryptography]
 python = python2.7
 
 [slapos]
 recipe = z3c.recipe.scripts
 python = python2.7
 eggs =
-  ${python-openssl:egg}
+  ${python-cffi:egg}
+  ${python-cryptography:egg}
+  pyOpenSSL
   slapos.libnetworkcache
   zc.buildout
   ${lxml-python:egg}
@@ -150,14 +156,16 @@ MarkupSafe = 0.18
 Pygments = 1.6
 Werkzeug = 0.9.4
 buildout-versions = 1.7
+cffi = 0.8.1
 cmd2 = 0.6.7
 collective.recipe.template = 1.10
+cryptography = 0.2.1
 itsdangerous = 0.23
 lxml = 3.2.3
 meld3 = 0.6.10
 netaddr = 0.7.10
 prettytable = 0.7.2
-pyOpenSSL = 0.13.1
+pyOpenSSL = 0.14
 pyparsing = 2.0.1
 setuptools = 1.1.6
 slapos.libnetworkcache = 0.13.4

setup.py

@@ -207,6 +207,8 @@ setup(name=name,
           'zabbixagent = slapos.recipe.zabbixagent:Recipe',
           'zimbra.kvm = slapos.recipe.zimbra_kvm:Recipe',
           'zeo = slapos.recipe.zeo:Recipe',
+          'zeroknown.read = slapos.recipe.zeroknown:ReadRecipe',
+          'zeroknown.write = slapos.recipe.zeroknown:WriteRecipe'
         ],
         'slapos.recipe.nosqltestbed.plugin': [
           'kumo = slapos.recipe.nosqltestbed.kumo:KumoTestBed',

slapos/recipe/README.zeroknown.txt

+These recipes provide the ability to save some buildout parameters and their value in a custom file, inside the instance folder.
+
+In both recipes, you HAVE TO give a filename, which will be stored at the root of the instance folder
+
+
+WriteRecipe : 
+-------------
+
+* Is used to create a section (named according to the buildout section_name).
+* You can give then as much parameters you wish, with their default values.
+* Whenever you run buildout, if the parameter has yet been saved in the config file, it will do nothing.
+* If the parameter's value has changed in the config file, it won't be overwritten
+* /!\ If you decide to change the default value of one parameter, ALL other parameters will be reseted in the config file, even if you changed it manually. Explanation : The default values aren't expected to change, except while development purposes.
+
+ReadRecipe :
+
+* It fills its own section with all the options in all the sections of the config file.

slapos/recipe/slaprunner/backup.py

@@ -71,7 +71,7 @@ class ExportRecipe(GenericBaseRecipe):
                   done
                 }
                 sync_element %(srv-directory)s/runner  %(backup-directory)s/runner/ instance project  proxy.db
-                sync_element %(etc-directory)s  %(backup-directory)s/etc/ .rcode .project .users .htpasswd ssh
+                sync_element %(etc-directory)s  %(backup-directory)s/etc/ .rcode .project .users .htpasswd config.json ssh
                 if [ -d %(backup-directory)s/runner/software ]; then
                   rm %(backup-directory)s/runner/software/*
                 fi
@@ -120,7 +120,7 @@ class ImportRecipe(GenericBaseRecipe):
                   done
                 }
                 restore_element %(backup-directory)s/runner/ %(srv-directory)s/runner  instance project  proxy.db
-                restore_element  %(backup-directory)s/etc/ %(etc-directory)s .rcode .project .users .htpasswd ssh
+                restore_element  %(backup-directory)s/etc/ %(etc-directory)s .rcode .project .users .htpasswd config.json ssh
                 ifs=$IFS IFS=';'
                 read user pass remaining < %(etc-directory)s/.users
                 IFS=$ifs

slapos/recipe/zeroknown.py

+##############################################################################
+#
+# Copyright (c) 2013 Vifib SARL and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+
+import ConfigParser
+import os
+import zc.buildout
+
+from slapos.recipe.librecipe import GenericBaseRecipe
+
+
+class WriteRecipe(GenericBaseRecipe):
+  """
+  """
+  def __init__(self, buildout, name, options):
+    if not "filename" in options:
+      raise zc.buildout.UserError("You have to provide the parameter \"filename\"")
+
+    self.filename = options['filename'].strip()
+    self.path = os.path.join(buildout['buildout']['directory'], self.filename)
+    self.name = name
+    self.options = options.copy()
+    del self.options['filename']
+    del self.options['recipe']
+
+    # Set up the parser, and write config file if needed
+    self.parser = ConfigParser.ConfigParser()
+    try:
+      self.parser.read(self.path)
+      #clean_options(options)
+      for key in self.options:
+        if key not in self.parser.options(self.name):
+          self.parser.set(self.name, key, self.options[key])
+      with open(self.path, 'w') as file:
+        self.parser.write(file)
+    # If the file or section do not exist
+    except (ConfigParser.NoSectionError, IOError) as e:
+      self.full_install()
+
+  install = update = lambda self: []
+
+  def full_install(self):
+    """XXX-Nicolas : when some parameter's value is changed in
+    buildout profile, this will override custom user defined values"""
+    self.parser.read(self.path)
+    if self.parser.has_section(self.name):
+      self.parser.remove_section(self.name)
+    self.parser.add_section(self.name)
+    for key in self.options:
+      self.parser.set(self.name, key, self.options[key])
+    with open(self.path, 'w') as file:
+      self.parser.write(file)
+
+
+class ReadRecipe(GenericBaseRecipe):
+  """
+  """
+  def __init__(self, buildout, name, options):
+    if not "filename" in options:
+      raise zc.buildout.UserError("You have to provide the parameter \"filename\"")
+
+    self.filename = options['filename'].strip()
+    self.path = os.path.join(buildout['buildout']['directory'], self.filename)
+
+    # Set up the parser, and write config file if needed
+    self.parser = ConfigParser.ConfigParser()
+    if os.path.exists(self.path):
+      self.parser.read(self.path)
+      for section in self.parser.sections():
+        for key ,value in self.parser.items(section):
+            options[key] = value
+
+  install = update = lambda self: []

software/backupserver/instance-pullrdiffbackup.cfg.in

@@ -23,6 +23,7 @@ crontabs = $${:etc}/crontabs
 cronstamps = $${:etc}/cronstamps
 backup = $${:srv}/backup
 status = $${:srv}/status
+statistic = $${:srv}/statistic
 backupscript = $${:etc}/backup
 www = $${:srv}/www
 home = $${:etc}/home
@@ -111,7 +112,9 @@ include = {{ include_string }}
 exclude_string = {{ exclude_string }}
 remote_schema = {{ remote_schema }}
 status_name = {{ slave_reference }}_status.txt
+statistic_name = {{ slave_reference }}_statistic.txt
 status_log = $${directory:status}/$${:status_name}
+statistic_log = $${directory:statistic}/$${:statistic_name}
 
 [{{ slave_reference }}-backup-crontab-line]
 recipe = slapos.recipe.template

software/backupserver/software.cfg

@@ -12,6 +12,7 @@ extends =
   ../../component/rsync/buildout.cfg
   ../../component/dropbear/buildout.cfg
   ../../component/grep/buildout.cfg
+  ../../component/findutils/buildout.cfg
 #   ../../stack/flask.cfg
   ../../stack/slapos.cfg
 
@@ -196,7 +197,7 @@ mode = 0644
 [template-backup-script]
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/template-backup-script.sh.in
-md5sum = 8a076962fc4df7f154572543899328e3
+md5sum = 3c4ad4b9a1ecef88f35a84ceb84a3ec6
 output = ${buildout:directory}/template-backup-script.sh.in
 mode = 0644
 
@@ -217,7 +218,7 @@ mode = 0644
 [status2rss]
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/status2rss.py
-md5sum = 138c96e0836f2b06414b98ba2643f21c
+md5sum = 0cd1cf97b199dd18fc0168c7281890ea
 output = ${buildout:directory}/status2rss.py
 mode = 0644
 
@@ -234,7 +235,7 @@ mode = 0644
 [template-pullrdiffbackup]
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance-pullrdiffbackup.cfg.in
-md5sum = 9bf3a34fa41ae6fe57b183293b3ff377
+md5sum = 061b98d001b501c9e1beb424e8802d3d
 output = ${buildout:directory}/template-pullrdiffbackup.cfg
 mode = 0644
 
@@ -327,3 +328,12 @@ cliff = 1.4
 cmd2 = 0.6.5.1
 prettytable = 0.7.2
 requests = 1.2.3
+
+cffi = 0.8.1
+cryptography = 0.2.1
+pyOpenSSL = 0.14
+six = 1.5.2
+
+# Required by:
+# cffi==0.8.1
+pycparser = 2.10

software/backupserver/status2rss.py

@@ -23,11 +23,12 @@ while 1:
   if not line:
     break
 
-  time, desc = line.split(',', 1)
+  time, statistic, desc = line.split(',', 2)
 
   rss_item = PyRSS2Gen.RSSItem(
     title = desc,
-    description = "%s, %s" % (time, desc),
+    description = "<p>%s</p>" % "<br/>".join(("%s, %s\n<a href='http://www.nongnu.org/rdiff-backup/FAQ.html#statistics'>Lastest statistic</a>\n%s" % (time, desc,
+      open(statistic).read())).split("\n")),
     link = LINK,
     pubDate = datetime.datetime.fromtimestamp(mktime_tz(parsedate_tz(time))),
     guid = PyRSS2Gen.Guid(base64.b64encode("%s, %s" % (time, desc)))

software/backupserver/template-backup-script.sh.in

@@ -6,9 +6,11 @@ export HOME=$${directory:home}
 
 # Clean status file (no history needed)
 ${coreutils-output:rm} -f $${:status_log}
+${coreutils-output:rm} -f $${:statistic_log}
 
 # Inform about beginning of backup
 ${coreutils-output:echo} "`${coreutils-output:date} -u`, $${:hostname} backup running" >> $${:status_log}
+${coreutils-output:echo} "Available only if backup succeed." >> $${:statistic_log}
 
 # set -e
 cd $${:datadirectory}
@@ -25,9 +27,11 @@ RESULT=$?
 ${coreutils-output:rm} -f $${:status_log}
 if [ $RESULT -eq 0 ]
 then
-  ${coreutils-output:echo} "`${coreutils-output:date} -u`, $${:hostname} backup success" >> $${:status_log}
+  ${coreutils-output:echo} "`${coreutils-output:date} -u`,$${:statistic_log},$${:hostname} backup success" >> $${:status_log}
+  ${coreutils-output:rm} -f $${:statistic_log}
+  ${findutils-output:find} rdiff-backup-data/ -name "session_statistic*" | ${coreutils-output:sort} | ${coreutils-output:tail} -n 1 | ${findutils-output:xargs} ${rdiff-backup-output:rdiff-backup} --calculate-average >> $${:statistic_log}
 else
-  ${coreutils-output:echo} "`${coreutils-output:date} -u`, $${:hostname} backup failed" >> $${:status_log}
+  ${coreutils-output:echo} "`${coreutils-output:date} -u`,$${:statistic_log},$${:hostname} backup failed" >> $${:status_log}
 fi
 
 # python scripts/verify_with_sudo.py ./ $${:connection}:/

software/erp5testnode/software.cfg

@@ -6,6 +6,8 @@ find-links += http://dist.repoze.org
 extends =
   ../../stack/slapos.cfg
   ../../component/python-2.7/buildout.cfg
+  ../../component/python-cffi/buildout.cfg
+  ../../component/python-cryptography/buildout.cfg
   ../../component/git/buildout.cfg
   ../../component/lxml-python/buildout.cfg
   ../../component/zip/buildout.cfg
@@ -67,6 +69,12 @@ python = python2.7
 [lxml-python]
 python = python2.7
 
+[python-cffi]
+python = python2.7
+
+[python-cryptography]
+python = python2.7
+
 [template]
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance.cfg
@@ -196,7 +204,9 @@ PyXML = 0.8.5
 Pygments = 1.6
 Werkzeug = 0.9.4
 buildout-versions = 1.7
+cffi = 0.8.1
 cmd2 = 0.6.7
+cryptography = 0.2.1
 erp5.util = 0.4.36
 inotifyx = 0.2.0-1
 itsdangerous = 0.23
@@ -205,7 +215,7 @@ meld3 = 0.6.10
 netaddr = 0.7.10
 plone.recipe.command = 1.1
 psutil = 1.2.1
-pyOpenSSL = 0.13.1
+pyOpenSSL = 0.14
 pyparsing = 2.0.1
 pytz = 2013.9
 slapos.core = 1.0.2.1

software/officejs/instance.cfg.in

+#############################
+#
+# Deploy officejs instance
+#
+#############################
+[buildout]
+parts =
+  test-runner
+  phantomjs-wrapper
+
+eggs-directory = ${buildout:eggs-directory}
+develop-eggs-directory = ${buildout:develop-eggs-directory}
+offline = true
+
+# Create all needed directories, depending on your needs
+[directory]
+recipe = slapos.cookbook:mkdirectory
+etc = $${buildout:directory}/etc
+bin = $${buildout:directory}/bin/
+
+[download-source]
+recipe = slapos.recipe.build:gitclone
+git-executable = ${git:location}/bin/git
+
+# Local development
+[officejs]
+<= download-source
+repository = ${officejs-repository:location}
+
+[test-runner]
+recipe = slapos.cookbook:egg_test
+run-test-suite = $${directory:bin}/runTestSuite
+run-test-suite-binary = ${buildout:bin-directory}/runTestSuite
+test-list =
+  $${officejs:location}
+prepend-path = ${git:location}/bin:${libxslt:location}/bin:${python2.7:location}/bin:${buildout:bin-directory}
+environment = environment
+
+[environment]
+CPPFLAGS = -I${python2.7:location}/include/python2.7 -I${libxml2:location}/include -I${libxslt:location}/include
+LDFLAGS = -L${python2.7:location}/lib -L${libxml2:location}/lib -L${libxslt:location}/lib -L${zlib:location}/lib -L${fontconfig:location}/lib -L${libexpat:location}/lib -L${freetype:location}/lib
+PYTHONPATH = ${python-setuptools:location}
+LD_LIBRARY_PATH = ${libxslt:location}/lib:${libxml2:location}/lib:${zlib:location}/lib:${fontconfig:location}/lib:${libexpat:location}/lib:${freetype:location}/lib
+
+[phantomjs-wrapper]
+recipe = slapos.cookbook:wrapper
+command-line = ${phantomjs:location}/phantomjs-slapos
+wrapper-path = $${directory:bin}/phantomjs
+# XXX we must use "parameters-extra" (- instead of _)
+# when new slapos.cookbook will be used
+parameters_extra = true

software/officejs/software.cfg

+[buildout]
+
+extends = 
+  ../../stack/slapos.cfg
+  ../../component/git/buildout.cfg
+  ../../component/phantomjs/buildout.cfg
+  ../../component/git/buildout.cfg
+  ../../component/python-2.7/buildout.cfg
+  ../../component/python-setuptools/buildout.cfg
+
+parts =
+  template
+  phantomjs
+  eggs
+
+[eggs]
+recipe = zc.recipe.egg
+eggs =
+  ${lxml-python:egg}
+  slapos.cookbook
+  erp5.util
+  collective.recipe.template
+entry-points =
+  runTestSuite=erp5.util.testsuite:runTestSuite
+scripts =
+  runTestSuite
+
+[template]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/instance.cfg.in
+output = ${buildout:directory}/instance.cfg
+# MD5 checksum can be skipped for development, but must be filled for production
+md5sum = bad9eca44660b819a69816e53c7d5428
+mode = 0644
+
+[officejs-repository]
+recipe = slapos.recipe.build:gitclone
+git-executable = ${git:location}/bin/git
+forbid-download-cache = true
+repository = http://git.erp5.org/repos/officejs.git
+branch = jqs
+
+[versions]
+Jinja2 = 2.6
+Werkzeug = 0.8.3
+buildout-versions = 1.7
+collective.recipe.template = 1.10
+erp5.util = 0.4.34
+hexagonit.recipe.cmmi = 2.0
+lxml = 3.1.1
+meld3 = 0.6.10
+plone.recipe.command = 1.1
+psutil = 0.6.1
+slapos.cookbook = 0.78.1
+slapos.recipe.build = 0.12
+slapos.recipe.template = 2.4.2
+
+# Required by:
+# slapos.core==0.35.1
+Flask = 0.9
+
+# Required by:
+# slapos.cookbook==0.76.0
+inotifyx = 0.2.0
+
+# Required by:
+# slapos.cookbook==0.76.0
+netaddr = 0.7.10
+
+# Required by:
+# slapos.core==0.35.1
+netifaces = 0.8
+
+# Required by:
+# slapos.core==0.35.1
+pyflakes = 0.6.1
+
+# Required by:
+# slapos.cookbook==0.76.0
+pytz = 2013b
+
+# Required by:
+# collective.recipe.template==1.10
+# erp5.util==0.4.33
+# hexagonit.recipe.download==1.6nxd002
+# slapos.cookbook==0.76.0
+# slapos.core==0.35.1
+# supervisor==3.0b1
+# zc.buildout==1.6.0-dev-SlapOS-010
+# zope.interface==4.0.5
+setuptools = 0.6c12dev-r88846
+
+# Required by:
+# slapos.cookbook==0.76.0
+slapos.core = 0.35.1
+
+# Required by:
+# slapos.core==0.35.1
+supervisor = 3.0b1
+
+# Required by:
+# slapos.core==0.35.1
+unittest2 = 0.5.1
+
+# Required by:
+# slapos.cookbook==0.76.0
+xml-marshaller = 0.9.7
+
+# Required by:
+# slapos.core==0.35.1
+zope.interface = 4.0.5

software/seleniumrunner/software.cfg

@@ -5,6 +5,8 @@ extends =
    ../../component/xorg/buildout.cfg
    ../../component/lxml-python/buildout.cfg
    ../../component/python-2.7/buildout.cfg
+   ../../component/python-cffi/buildout.cfg
+   ../../component/python-cryptography/buildout.cfg
    ../../component/firefox/buildout.cfg
    ../../component/dash/buildout.cfg
    ../../stack/shacache-client.cfg
@@ -23,6 +25,15 @@ parts =
 
 unzip = true
 
+[lxml-python]
+python = python2.7
+
+[python-cffi]
+python = python2.7
+
+[python-cryptography]
+python = python2.7
+
 [instance-recipe]
 egg = slapos.cookbook
 module = seleniumrunner
@@ -54,70 +65,90 @@ output = ${buildout:directory}/template-selenium.cfg
 mode = 0644
 
 [versions]
-# Use SlapOS patched zc.buildout
-zc.buildout = 1.6.0-dev-SlapOS-004
+# pin version of setuptools
+setuptools = 2.2
 
-Jinja2 = 2.6
-Werkzeug = 0.8.3
+Jinja2 = 2.7.2
+MarkupSafe = 0.18
+Werkzeug = 0.9.4
 buildout-versions = 1.7
-hexagonit.recipe.cmmi = 1.5.0
-meld3 = 0.6.8
-slapos.cookbook = 0.46
-slapos.recipe.build = 0.7
-slapos.recipe.template = 2.2
+cffi = 0.8.1
+cmd2 = 0.6.7
+cryptography = 0.2.1
+itsdangerous = 0.23
+meld3 = 0.6.10
+pyOpenSSL = 0.14
+pyparsing = 2.0.1
+slapos.cookbook = 0.85
+slapos.recipe.build = 0.12
+slapos.recipe.cmmi = 0.2
+slapos.recipe.template = 2.5
+
+# Required by:
+# slapos.core==1.0.3
+Flask = 0.10.1
 
 # Required by:
-# slapos.core==0.23
-Flask = 0.8
+# slapos.core==1.0.3
+cliff = 1.5.2
 
 # Required by:
-# slapos.cookbook==0.42
-PyXML = 0.8.4
+# slapos.cookbook==0.85
+inotifyx = 0.2.0-1
 
 # Required by:
-# slapos.cookbook==0.42
-inotifyx = 0.2.0
+# slapos.cookbook==0.85
+lock-file = 2.0
 
 # Required by:
-# slapos.cookbook==0.42
-# slapos.core==0.23
+# slapos.cookbook==0.85
+# slapos.core==1.0.3
 # xml-marshaller==0.9.7
-lxml = 2.3.4
+lxml = 3.3.1
 
 # Required by:
-# slapos.cookbook==0.42
-netaddr = 0.7.6
+# slapos.cookbook==0.85
+netaddr = 0.7.10
 
 # Required by:
-# slapos.core==0.23
-netifaces = 0.8
+# slapos.core==1.0.3
+netifaces = 0.8-1
 
 # Required by:
-# slapos.cookbook==0.42
-# slapos.core==0.23
-# zc.buildout==1.5.3-dev-SlapOS-010
-# zc.recipe.egg==1.3.2
-setuptools = 0.6c12dev-r88846
+# cffi==0.8.1
+pycparser = 2.10
 
 # Required by:
-# slapos.cookbook==0.42
-slapos.core = 0.23
+# slapos.cookbook==0.85
+pytz = 2013.9
 
 # Required by:
-# slapos.core==0.23
-supervisor = 3.0a12
+# slapos.core==1.0.3
+requests = 2.2.1
 
 # Required by:
-# slapos.cookbook==0.42
-xml-marshaller = 0.9.7
+# cliff==1.5.2
+six = 1.5.2
+
+# Required by:
+# slapos.cookbook==0.85
+slapos.core = 1.0.3
+
+# Required by:
+# cliff==1.5.2
+stevedore = 0.14.1
+
+# Required by:
+# slapos.core==1.0.3
+supervisor = 3.0
 
 # Required by:
-# slapos.cookbook==0.42
-zc.recipe.egg = 1.3.2
+# slapos.cookbook==0.85
+xml-marshaller = 0.9.7
 
 # Required by:
-# slapos.core==0.23
-zope.interface = 3.8.0
+# slapos.core==1.0.3
+zope.interface = 4.1.0
 
 [networkcache]
 # signature certificates of the following uploaders.

software/slapos-testing/instance.cfg

 [buildout]
 parts =
   slapos.core-setup
+  erp5.util-setup
+  phantomjs-wrapper
   slapos-test-runner
   sh-environment
 
@@ -30,6 +32,9 @@ recipe = plone.recipe.command
 command = echo "Updating setup...";cd $${slapos.core:location}; export PATH="$${slapos-test-runner:prepend-path}:$PATH"; export CPPFLAGS="$${environment:CPPFLAGS}"; export LDFLAGS="$${environment:LDFLAGS}"; export PYTHONPATH="$${environment:PYTHONPATH}"; python setup.py test -n; python setup.py test -n; python setup.py test -n; python setup.py test -n 
 update-command = $${:command}
 
+[slapos.package]
+<= download-source
+repository = ${slapos.package-repository:location}
 
 [slapos.recipe.template]
 <= download-source
@@ -39,6 +44,23 @@ repository = ${slapos.recipe.template-repository:location}
 <= download-source
 repository = ${slapos.recipe.build-repository:location}
 
+[erp5-util]
+<= download-source
+repository = ${erp5-util-repository:location}
+
+[erp5.util-setup]
+recipe = plone.recipe.command
+command = echo "Updating setup...";cd $${erp5-util:location}; export PATH="$${slapos-test-runner:prepend-path}:$PATH"; export CPPFLAGS="$${environment:CPPFLAGS}"; export LDFLAGS="$${environment:LDFLAGS}"; export PYTHONPATH="$${environment:PYTHONPATH}"; python setup.py test -n; python setup.py test -n; python setup.py test -n; python setup.py test -n; python setup.py test -n 
+update-command = $${:command}
+
+[officejs]
+<= download-source
+repository = ${officejs-repository:location}
+
+[jio]
+<= download-source
+repository = ${jio-repository:location}
+
 [slapos-test-runner]
 recipe = slapos.cookbook:egg_test
 run-test-suite = $${create-directory:bin}/runTestSuite
@@ -48,8 +70,12 @@ run-test-suite-binary = ${buildout:bin-directory}/runTestSuite
 #python-list = $${}
 test-list =
   $${slapos.core:location}
+  $${slapos.package:location}
   $${slapos.recipe.template:location}
   $${slapos.recipe.build:location}
+  $${erp5-util:location}
+  $${jio:location}
+  $${officejs:location}
 prepend-path = ${git:location}/bin:${libxslt:location}/bin:${python2.7:location}/bin
 environment = environment
 
@@ -71,3 +97,9 @@ input = inline:
   export PS1="[slapos-testing env Active] $PS1"
 output = $${create-directory:bin}/environment.sh
 mode = 755
+
+[phantomjs-wrapper]
+recipe = slapos.cookbook:wrapper
+command-line = ${phantomjs:location}/phantomjs-slapos
+wrapper-path = $${create-directory:bin}/phantomjs
+parameters-extra = true

software/slapos-testing/software.cfg

@@ -8,14 +8,20 @@ extends =
   ../../component/python-2.7/buildout.cfg
   ../../component/python-setuptools/buildout.cfg
   ../../component/zlib/buildout.cfg
+  ../../component/phantomjs/buildout.cfg
   ../../stack/slapos.cfg
 
 parts =
   slapos.cookbook-repository
   slapos.core-repository
+  slapos.package-repository
   slapos.recipe.template-repository
   slapos.recipe.build-repository
+  erp5-util-repository
+  officejs-repository
+  jio-repository
   eggs
+  phantomjs
   template
 
 [eggs]
@@ -45,6 +51,13 @@ forbid-download-cache = true
 repository = http://git.erp5.org/repos/slapos.core.git
 branch = master
 
+[slapos.package-repository]
+recipe = slapos.recipe.build:gitclone
+git-executable = ${git:location}/bin/git
+forbid-download-cache = true
+repository = http://git.erp5.org/repos/slapos.package.git
+branch = master
+
 [slapos.recipe.template-repository]
 recipe = slapos.recipe.build:gitclone
 git-executable = ${git:location}/bin/git
@@ -59,10 +72,31 @@ forbid-download-cache = true
 repository = http://git.erp5.org/repos/slapos.recipe.build.git
 branch = master
 
+[erp5-util-repository]
+recipe = slapos.recipe.build:gitclone
+git-executable = ${git:location}/bin/git
+forbid-download-cache = true
+repository = http://git.erp5.org/repos/erp5.git
+branch = master
+
+[officejs-repository]
+recipe = slapos.recipe.build:gitclone
+git-executable = ${git:location}/bin/git
+forbid-download-cache = true
+repository = http://git.erp5.org/repos/officejs.git
+branch = jqs
+
+[jio-repository]
+recipe = slapos.recipe.build:gitclone
+git-executable = ${git:location}/bin/git
+forbid-download-cache = true
+repository = http://git.erp5.org/repos/jio.git
+branch = master
+
 [template]
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance.cfg
-md5sum = 1330c8ccd1dab485aadba5e688d2b2e1
+md5sum = b065539ca58a3400b0ed30093ab020ac 
 output =  ${buildout:directory}/template.cfg
 mode = 640
 

software/slaprunner/README.txt

@@ -29,9 +29,18 @@ Optional parameter.
 
 Default: 10
 
+
+Public-directory
+----------------
+You can serve static files with the webrunner. For that, just put your data in "srv/runner/public". All these files will be served throught the url of the webrunner + "/public/". Useful for developping your own static website.
+
+
 AUTO-DEPLOYMENT
 ---------------
 
+for software
+~~~~~~~~~~~~
+
 You can automatically deploy a software release while deploying the webrunner itself, using the paramater XML.
 
 To do this, you only need to pass as a parameter named "slapos-software" : "AAA/BBB", where AAA is the folder of slapos.git where is located your BBB software.
@@ -43,6 +52,23 @@ It is also possible to download you own git repository, by providing the url in
 
 Last but not least, it is also possible to switch the branch with the parameter "slapos-reference" (by default pointing on master)
 
+for instance
+~~~~~~~~~~~~
+
+The parameter "auto-deploy-instance" can be explicitly set to allow or prevent the runner to deploy the instance at START TIME (if you manually restart the runner, or if the server reboots). Values : "true" or "false". Default value is "true", except for the instances of import (while type is resilient or test) which is "false"
+
+There also exists the parameter "autorun", which will build&run your software if set to true. For this, you need "auto_deploy" to true, and set the parameter "slapos-software" to the software you want to deploy. Do not hesitate to clone a different repo than "slapos", or to change the tag/branch to use your custom Software Release. (see "slapos-repository" and "slapos-reference" in previous section).
+
+To deploy the instance with some parameters, just give to the runner parameters starting with "parameter-", they will be correctly forwarded to the instance, which will use them for its configuration. For example, if you want to send to the sofware helloworld the parameter "name" with the value "nicolas", here is how to configure the parameter.xml of the webrunner for auto-depolyment :
+
+<?xml version='1.0' encoding='utf-8'?>
+<instance>
+<parameter id="slapos-software">software/helloworld</parameter>
+<parameter id="auto_deploy">true</parameter>
+<parameter id="autorun">true</parameter>
+<parameter id="parameter-name">nicolas</parameter>
+</instance>
+
 Resilience :
 ------------
 
@@ -95,7 +121,32 @@ Example :
 	<parameter id="custom-frontend-basic-auth">true</parameter>
 	</instance>
 
+Git repositories :
+------------------
+
+It is easy to give access to your git repository/ies to everyone, or to clone it on your own computer. For this, there are 2 urls to remember:
+  - For read only, you can clone : https://[IPV6]:PORT/git-public/YourRepo.git/
+  - For read and write access, using your runner account : https://[IPV6]:PORT/git/YourRepo.git/
+
+To create the repo, go in the folder srv/runner/project and initiate a new git repo (git init/clone --bare XXX).
+
+For the moment, the PORT is the port of monitoring, which is 9685.
+
 Things to notice for the nex developer :
 ----------------------------------------
 
 As you can see in instance-runner-*.cfg, the buildout section extends a hard-coded template file. If one day you need to modify the filename, do not forget to modify it in instance.cfg, but also in these files ! (the problem is that the content of instance.cfg is not known by buildout while the deployment of the software release)
+
+
+List of ports used by the webrunner:
+------------------------------------
+8602 : slapproxy, while running tests
+8080 : shellinabox
+9684 : apache (monitoring of slaprunner, git access)
+22222 : dropbear
+50000 : slapproxy
+50005 : webrunner (flask app), webdav access
+
+Tips:
+-----
+You can use shellinabox in fullscreen, by accessing : https://[IPV6]:8080

software/slaprunner/TODO.txt

 - resilient sr: Cloned instances should not launch slapgrid-sr if it was not launched on export instance
 - shellinabox password should be the same in all the resilient instances
+- add test for parameter auto-deploy-instance

software/slaprunner/common.cfg

@@ -19,53 +19,52 @@ extends =
   ../../stack/flask.cfg
   ../../stack/shacache-client.cfg
   ../../stack/resilient/buildout.cfg
+  ../../stack/monitor/buildout.cfg
   ../../stack/slapos.cfg
 
 parts =
   template
   eggs
-  nginx
   instance-runner-import
   instance-runner-export
   slapos-cookbook
-  slapos-cfg-template 
+  slapos-cfg-template
 # XXX: we have to manually add this for resilience
   rdiff-backup
   collective.recipe.template-egg
 
-
 [template]
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance.cfg
 output = ${buildout:directory}/template.cfg
-md5sum = 8a47421ac6158b4ee476acab212c67d9
+#md5sum = 8a47421ac6158b4ee476acab212c67d9
 mode = 0644
 
 [template-runner]
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance-runner.cfg
 output = ${buildout:directory}/template-runner.cfg.in
-md5sum = b3288126400c4ca8469c255ef130dec9
+md5sum = c66f18b74b958805bb2196fb1c54683e
 mode = 0644
 
 [instance-runner-import]
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance-runner-import.cfg.in
 output = ${buildout:directory}/instance-runner-import.cfg
-md5sum = 82f2d0be111617eac9849cb7b8baac5d
+md5sum = e1769596ef62f6db6bbfca4391e740c0
 mode = 0644
 
 [instance-runner-export]
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance-runner-export.cfg.in
 output = ${buildout:directory}/instance-runner-export.cfg
-md5sum = 9a8cdf65bca8a562accb710a7c3b6595
+md5sum = b7664995a6dd954f4f214252ffeaaa61
 mode = 0644
 
 [template-resilient]
 recipe = slapos.recipe.download
 url = ${:_profile_base_location_}/instance-resilient.cfg.jinja2
-md5sum = f8758a5a2f886d649ad02157d76550ba
+md5sum = a9647ae3c99308b85dd72fae76ef071f
 filename = instance-resilient.cfg.jinja2
 mode = 0644
 
@@ -73,7 +72,7 @@ mode = 0644
 recipe = hexagonit.recipe.download
 url = ${:_profile_base_location_}/instance-resilient-test.cfg.jinja2
 download-only = true
-md5sum = b231383bb63edc0009fb6e0ff5c54d8e
+#md5sum = b231383bb63edc0009fb6e0ff5c54d8e
 filename = instance-resilient-test.cfg.jinja2
 mode = 0644
 
@@ -81,14 +80,22 @@ mode = 0644
 recipe = hexagonit.recipe.download
 url = ${:_profile_base_location_}/nginx_conf.in
 download-only = true
-md5sum = 67d2ce92b4ad6dca6eaf69eeb2c2734c
+md5sum = fa66988e96bb712c0580b5fb90f95700
 filename = nginx_conf.in
 mode = 0644
 
+[template_httpd_conf]
+recipe = hexagonit.recipe.download
+url = ${:_profile_base_location_}/httpd_conf.in
+download-only = true
+md5sum = 0b63a652d1d51e0894a780896fce9893
+filename = httpd_conf.in
+mode = 0644
+
 [template_launcher]
 recipe = slapos.recipe.download
 url = ${:_profile_base_location_}/launcher.in
-md5sum = c7f8b6e9ae84aa94686a9cbaaa3dd693
+#md5sum = c7f8b6e9ae84aa94686a9cbaaa3dd693
 filename = launcher.in
 mode = 0644
 location = ${buildout:parts-directory}/${:_buildout_section_name_}
@@ -96,9 +103,18 @@ location = ${buildout:parts-directory}/${:_buildout_section_name_}
 [slapos-cfg-template]
 recipe = hexagonit.recipe.download
 url = ${:_profile_base_location_}/template/${:filename}
-md5sum = 8207e74c7b97a4a3f3be390ea4f97ae1
+md5sum = d31d1c51a51bc4b2b8dda197777a4bcb
+location = ${buildout:parts-directory}/${:_buildout_section_name_}
+filename = slapos.cfg.in
+download-only = true
+mode = 0644
+
+[parameters-template]
+recipe = hexagonit.recipe.download
+url = ${:_profile_base_location_}/${:filename}
+md5sum = f8446fcf254b4929eb828a9a1d7e5f62
 location = ${buildout:parts-directory}/${:_buildout_section_name_}
-filename = slapos.cfg.in 
+filename = parameters.xml.in
 download-only = true
 mode = 0644
 
@@ -112,6 +128,7 @@ eggs =
   hexagonit.recipe.download
   inotifyx
   lock-file
+  lxml
   netaddr
   plone.recipe.command
   pytz

software/slaprunner/development.cfg

@@ -10,8 +10,8 @@ extends = common.cfg
 
 parts +=
   slapos.cookbook-repository
-  slapos.toolbox-repository
-  erp5.util-repository
+#  slapos.toolbox-repository
+#  erp5.util-repository
   check-recipe
 
 #  slapos.core-repository
@@ -19,20 +19,21 @@ parts +=
 develop =
   ${:parts-directory}/slapos.toolbox-repository
   ${:parts-directory}/slapos.cookbook-repository
-  ${:parts-directory}/erp5.util-repository
+#  ${:parts-directory}/erp5.util-repository
 #  ${:parts-directory}/slapos.core-repository
 
 
 [slapos.toolbox-repository]
 recipe = slapos.recipe.build:gitclone
 repository = http://git.erp5.org/repos/slapos.toolbox.git
-branch = slaprunner-paas
+#branch = slaprunner-paas
+revision = 399dc9f60967593bad0a16dcccf3bec487fc0a8f
 git-executable = ${git:location}/bin/git
 
 [slapos.cookbook-repository]
 recipe = slapos.recipe.build:gitclone
 repository = http://git.erp5.org/repos/slapos.git
-branch = slaprunner-paas-test
+branch = master
 git-executable = ${git:location}/bin/git
 
 # Used for resiliency tests only
@@ -55,7 +56,7 @@ stop-on-error = true
 update-command = ${:command}
 command =
   grep parts ${buildout:develop-eggs-directory}/slapos.cookbook.egg-link &&
-  grep parts ${buildout:develop-eggs-directory}/slapos.toolbox.egg-link &&
-  grep parts ${buildout:develop-eggs-directory}/erp5.util.egg-link
+  grep parts ${buildout:develop-eggs-directory}/slapos.toolbox.egg-link
+#  grep parts ${buildout:develop-eggs-directory}/erp5.util.egg-link
 
 #  grep parts ${buildout:develop-eggs-directory}/slapos.core.egg-link &&

software/slaprunner/httpd_conf.in

+PidFile "{{ parameters.path_pid }}"
+ServerName example.com
+ServerAdmin someone@email
+
+<IfDefine !MonitorPort>
+  Listen [{{ parameters.global_ip }}]:{{ parameters.monitor_port }}
+  Define MonitorPort
+</IfDefine>
+
+LoadModule unixd_module modules/mod_unixd.so
+LoadModule access_compat_module modules/mod_access_compat.so
+LoadModule auth_basic_module modules/mod_auth_basic.so
+LoadModule authz_core_module modules/mod_authz_core.so
+LoadModule authz_user_module modules/mod_authz_user.so
+LoadModule authz_host_module modules/mod_authz_host.so
+LoadModule authn_core_module modules/mod_authn_core.so
+LoadModule authn_file_module modules/mod_authn_file.so
+LoadModule mime_module modules/mod_mime.so
+LoadModule cgid_module modules/mod_cgid.so
+LoadModule ssl_module modules/mod_ssl.so
+LoadModule alias_module modules/mod_alias.so
+LoadModule env_module modules/mod_env.so
+LoadModule rewrite_module modules/mod_rewrite.so
+LoadModule log_config_module modules/mod_log_config.so
+
+ErrorLog "{{ parameters.path_error_log }}"
+LogFormat "%h %l %u %t \"%r\" %>s %b" common
+CustomLog "{{ parameters.path_access_log }}" common
+
+# SSL Configuration
+Define SSLConfigured
+SSLCertificateFile {{ parameters.cert_file }}
+SSLCertificateKeyFile {{ parameters.key_file }}
+SSLRandomSeed startup builtin
+SSLRandomSeed connect builtin
+SSLRandomSeed startup /dev/urandom 256
+SSLRandomSeed connect builtin
+SSLProtocol -ALL +SSLv3 +TLSv1
+SSLHonorCipherOrder On
+SSLCipherSuite RC4-SHA:HIGH:!ADH
+SSLEngine   On
+
+ScriptSock {{ parameters.path_pid }}
+
+SetEnv GIT_PROJECT_ROOT {{ parameters.project_folder }}
+SetEnv GIT_HTTP_EXPORT_ALL
+ScriptAlias /git/ {{ parameters.git_http_backend }}/
+ScriptAlias /git-public/ {{ parameters.git_http_backend }}/
+
+RewriteCond %{QUERY_STRING} service=git-receive-pack [OR]
+RewriteCond %{REQUEST_URI} /git-receive-pack$
+RewriteRule ^/git/ - [E=AUTHREQUIRED:yes]
+RewriteRule ^/git-public/ - [E=AUTHREQUIRED:yes]
+
+<LocationMatch "^/git-public/">
+        Order Deny,Allow
+        Deny from env=AUTHREQUIRED
+
+        AuthType Basic
+        AuthName "Git Access"
+        AuthUserFile "{{ parameters.etc_dir }}/.htpasswd"
+        Require valid-user
+        Satisfy any
+</LocationMatch>
+
+<LocationMatch "^/git/">
+        Order Deny,Allow
+        Deny from env=AUTHREQUIRED
+
+        AuthType Basic
+        AuthName "Git Access"
+        AuthUserFile "{{ parameters.etc_dir }}/.htpasswd"
+        Require valid-user
+</LocationMatch>
+
+include {{ parameters.cgi_httpd_conf }}

software/slaprunner/instance-resilient.cfg.jinja2

@@ -17,17 +17,20 @@ parts +=
 
 # Bubble up the parameters
 [request-runner]
-return = url ssh-public-key ssh-url notification-id ip backend_url url ssh_command password_recovery_code shell_password access_url 1_info
+return = url ssh-public-key ssh-url notification-id ip backend_url url ssh_command access_url 1_info 2_info monitor_url IMPORTANT_monitor_info webdav_url public_url
 
 [publish-connection-informations]
 recipe = slapos.cookbook:publish
 1_info = ${request-runner:connection-1_info}
+2_info = ${request-runner:connection-2_info}
 backend_url = ${request-runner:connection-backend_url}
 access_url = ${request-runner:connection-access_url}
 url = ${request-runner:connection-url}
 ssh_command = ${request-runner:connection-ssh_command}
-password_recovery_code = ${request-runner:connection-password_recovery_code}
-shell_password = ${request-runner:connection-shell_password}
+monitor_url = ${request-runner:connection-monitor_url}
+IMPORTANT_monitor_info = ${request-runner:connection-IMPORTANT_monitor_info}
+webdav_url = ${request-runner:connection-webdav_url}
+public_url = ${request-runner:connection-public_url}
 
 [slap-parameter]
 # Default parameters for distributed deployment

software/slaprunner/instance-runner-export.cfg.in

@@ -18,13 +18,36 @@ parts +=
   slaprunner-promise
   slaprunner-frontend-promise
   dropbear-promise
+  private
   shellinabox-promise
   symlinks
   shellinabox
   slapos-cfg
   slapos-repo-config
-  prepare-software
   cron-entry-backup
+  cron-entry-prepare-software
+  deploy-instance-parameters
+###Parts to add for monitoring
+  slap-parameters
+  certificate-authority
+  cron
+  cron-entry-monitor
+  cron-entry-rss
+  deploy-index
+  deploy-index-template
+  deploy-monitor-script
+  deploy-rss-script
+  deploy-settings-cgi
+  deploy-status-cgi
+  make-rss
+  monitor-frontend
+  monitor-promise
+  setup-static-files
+  certificate-authority
+  public
+  zero-parameters
+  cgi-httpd-wrapper
+  public-symlink
 
 [exporter]
 recipe = slapos.cookbook:slaprunner.export
@@ -35,6 +58,8 @@ backup-directory = $${directory:backup}
 shell-binary = ${dash:location}/bin/dash
 rsync-binary = ${rsync:location}/bin/rsync
 
+[monitor-promise]
+url = $${monitor-frontend:config-url}/$${deploy-index:filename}
 
 # Extends publish section with resilient parameters
 [publish-connection-informations]

software/slaprunner/instance-runner-import.cfg.in

@@ -16,14 +16,36 @@ parts +=
   sshkeys-authority
   slaprunner-promise
   dropbear-promise
+  private
   shellinabox-promise
   shellinabox
   symlinks
   slapos-cfg
   slapos-repo-config
-  prepare-software
+  cron-entry-prepare-software
+  deploy-instance-parameters
 # have to repeat the next one, as it's not inherited from pbsready-import
   import-on-notification
+###Parts to add for monitoring
+  slap-parameters
+  certificate-authority
+  cron
+  cron-entry-monitor
+  cron-entry-rss
+  deploy-index
+  deploy-index-template
+  deploy-monitor-script
+  deploy-rss-script
+  deploy-settings-cgi
+  deploy-status-cgi
+  make-rss
+  monitor-promise
+  setup-static-files
+  certificate-authority
+  public
+  zero-parameters
+  cgi-httpd-wrapper
+  public-symlink
 
 [importer]
 recipe = slapos.cookbook:slaprunner.import
@@ -35,3 +57,6 @@ shell-binary = ${dash:location}/bin/dash
 rsync-binary = ${rsync:location}/bin/rsync
 curl-binary = ${curl:location}/bin/curl
 backend-url = $${slaprunner:access-url}
+
+[slap-parameter]
+auto-deploy-instance = false

software/slaprunner/nginx_conf.in

@@ -64,5 +64,22 @@ http {
             proxy_set_header        X-Forwarded-For   $proxy_add_x_forwarded_for;
 	    proxy_set_header        X-Forwarded-Host  $http_host;
 	}
+        location /public/ {
+            root {{ param_nginx_frontend['work_dir'] }};
+            index index.html index.htm;
+        }
+        location /share {
+            rewrite ^/share / break;
+
+            auth_basic "Restricted";
+            auth_basic_user_file {{ param_nginx_frontend['etc_dir'] }}/.htpasswd;
+
+            dav_methods PUT DELETE MKCOL COPY MOVE;
+            dav_ext_methods PROPFIND OPTIONS;
+
+            root {{ param_nginx_frontend['work_dir'] }};
+
+            create_full_put_path   on;
+        }
     }
 }

software/slaprunner/parameters.xml.in

+{% set inst_parameter_dict = {} -%}
+{% if slapparameter_dict is defined -%}
+{%   for key in slapparameter_dict.keys() -%}
+{%     if key.startswith('parameter-') -%}
+{%       do inst_parameter_dict.__setitem__(key[10:], slapparameter_dict.pop(key)) -%}
+{%     endif -%}
+{%   endfor -%}
+{% endif -%}
+
+<?xml version='1.0' encoding='utf-8'?>
+<instance>
+{% if slapparameter_dict is defined %}
+  {% for parameter_name in inst_parameter_dict.keys() %}
+  <parameter id="{{ parameter_name }}">{{ inst_parameter_dict[parameter_name] }}</parameter>
+  {% endfor %}
+{% endif %}
+</instance>
+

software/slaprunner/schema.json

+{
+        "$schema": "http://json-schema.org/draft-04/schema#",
+                "properties": {
+                        "authorized-key": {
+                                "description" : "SSH public key in order to connect to the SSH server of this runner instance",
+                                "type": "string"
+                        },
+                        "instance-amount": {
+                                "description": "number of slappart to deploy inside the runner",
+                                "type": "integer",
+                                "default": 10
+                        },
+                        "slapos-software": {
+                                "description": "a relative path from the slapos git repo to a folder containing a software release, which will be automaticaly deployed while the runner instanciation, and only if the parameter auto-deploy is set to 'true'. For example: 'software/helloworld",
+                                "type": "string"
+                        }, "auto-deploy": { "description": "authorizes the software declared with 'slapos-software' to be automatically deployed, or not",
+                                "type": "boolean",
+                                "default": "true"
+                        },
+                        "slapos-repository": {
+                                "description": "url of the default git repository that will be download by the runner while its instanciation. Will be cloned in a directory named 'slapos'",
+                                "type": "string",
+                                "default": "http://git.erp5.org/repos/slapos.git"
+                        },
+                        "slapos-reference": {
+                                "description": "commit reference on which the default repository will checkout",
+                                "type": "string",
+                                "default": "master"
+                        },
+                        "auto-deploy-instance": {
+                                "description": "prevent the runner from deploying and starting instances",
+                                "type": "boolean",
+                                "default": "true, but is set to false for instances of type 'import' in resiliency"
+                        },
+                        "autorun": {
+                                "description": "let automaticaly build and run a declared software with 'slapos-software'. Only works if 'slapos-software' is set, and 'auto-deploy' is true",
+                                "type": "boolean",
+                                "default": "false"
+                        },
+                        "parameter-*": {
+                                "description": "'*' is a parameter which will be used to configure the instance inside the runner.",
+                                "type": "string"
+                        },
+                        "custom-frontend-backend-url": {
+                                "description": "return an ipv4 frontend of the given ipv6(+optional port)",
+                                "type": "string"
+                        },
+                        "custom-frontend-basic-auth": {
+                                "description": "if the ip given with 'custom-frontend-backend-url' is secure, set it to true for the promise do not fail",
+                                "type": "boolean",
+                                "default": "false"
+                        }
+                        "monitor-port": {
+                                "description": "allow to manually change the port on wich the apache server running monitoring interface is listening. The default value for the webrunner is different from the default value of the standalone stack-monitor server",
+                                "type": "integer",
+                                "default": 9684
+                        }
+                }
+}

software/slaprunner/software.cfg

@@ -5,7 +5,7 @@
 # 2/ Define list of trusted certificates for the cache.
 
 [buildout]
-extends = common.cfg
+extends = development.cfg
 
 [networkcache]
 # signature certificates of the following uploaders.
@@ -165,19 +165,19 @@ signature-certificate-list =
   +uZ16u1DbO9rYoKgWqjLk1GfiLw5v86pd5+wZd5I9QJ0/Sbz2vZk5S4ciMIGwArc
   m711+GzlW5xe6GyH9SZaGOPAdUbI6JTDwLzEgA==
   -----END CERTIFICATE-----
-  -----BEGIN CERTIFICATE-----
-  MIIB+DCCAWGgAwIBAgIJAJAOvB8P2YYMMA0GCSqGSIb3DQEBBQUAMBQxEjAQBgNV
-  BAMMCUNPTVAtMTg1MjAgFw0xMzEyMDMxMjU5NDJaGA8yMTEzMTEwOTEyNTk0Mlow
-  FDESMBAGA1UEAwwJQ09NUC0xODUyMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKB
-  gQDgR2qKVURmOUmzc7KVL2Gk2VezWlHlJy3V3k1WVYahhFsSjf3O4VyTtroHbIGu
-  YMJaHLSvGvvvkVJIz/DKqOvl1/6435t+coYECimvzqzTBDWFlEKJbEixz/3nPnXe
-  CFqVzuCQ7e3nRIrIGLml6jZnXwPlzx8ANn1LO8pwKHj+xwIDAQABo1AwTjAdBgNV
-  HQ4EFgQUvo7qkSq82Ax5/vnxHoJOhySQSGowHwYDVR0jBBgwFoAUvo7qkSq82Ax5
-  /vnxHoJOhySQSGowDAYDVR0TBAUwAwEB/zANBgkqhkiG9w0BAQUFAAOBgQCbNxtq
-  BckDSWb92gy/p+JbkMXLjRSSWgeKafvWQXMXpPHW847CPzUDB7zT0rT8a8SRjVuL
-  Vt/oo2VlJ9xbKEz/qppX8w1TwEZhEJk6h8ky51s2EyyITUy8LHESJes9l46DRbr8
-  Rm6WS5sk5oiCd146YkWhfZuOsQHXSG/+WC/GPQ==
-  -----END CERTIFICATE-----
+#  -----BEGIN CERTIFICATE-----
+#  MIIB+DCCAWGgAwIBAgIJAJAOvB8P2YYMMA0GCSqGSIb3DQEBBQUAMBQxEjAQBgNV
+#  BAMMCUNPTVAtMTg1MjAgFw0xMzEyMDMxMjU5NDJaGA8yMTEzMTEwOTEyNTk0Mlow
+#  FDESMBAGA1UEAwwJQ09NUC0xODUyMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKB
+#  gQDgR2qKVURmOUmzc7KVL2Gk2VezWlHlJy3V3k1WVYahhFsSjf3O4VyTtroHbIGu
+#  YMJaHLSvGvvvkVJIz/DKqOvl1/6435t+coYECimvzqzTBDWFlEKJbEixz/3nPnXe
+#  CFqVzuCQ7e3nRIrIGLml6jZnXwPlzx8ANn1LO8pwKHj+xwIDAQABo1AwTjAdBgNV
+#  HQ4EFgQUvo7qkSq82Ax5/vnxHoJOhySQSGowHwYDVR0jBBgwFoAUvo7qkSq82Ax5
+#  /vnxHoJOhySQSGowDAYDVR0TBAUwAwEB/zANBgkqhkiG9w0BAQUFAAOBgQCbNxtq
+#  BckDSWb92gy/p+JbkMXLjRSSWgeKafvWQXMXpPHW847CPzUDB7zT0rT8a8SRjVuL
+#  Vt/oo2VlJ9xbKEz/qppX8w1TwEZhEJk6h8ky51s2EyyITUy8LHESJes9l46DRbr8
+#  Rm6WS5sk5oiCd146YkWhfZuOsQHXSG/+WC/GPQ==
+#  -----END CERTIFICATE-----
 
 [versions]
 Flask-Auth = 0.85
@@ -212,7 +212,7 @@ slapos.recipe.build = 0.11.5
 slapos.recipe.cmmi = 0.2
 slapos.recipe.download = 1.0.dev-r4053
 slapos.recipe.template = 2.4.2
-slapos.toolbox = 0.38.1
+slapos.toolbox = 0.39
 smmap = 0.8.2
 xml-marshaller = 0.9.7
 z3c.recipe.scripts = 1.0.1

software/slaprunner/template/slapos.cfg.in

@@ -17,6 +17,7 @@ supervisord_config = {{ supervisord_config }}
 runner_workdir = {{ runner_workdir }}
 runner_host = {{ ipv4 }}
 runner_port = {{ runner_port }}
+instance_monitoring_url = {{ instance_monitoring_url }}
 ipv4_address = {{ ipv4 }}
 ipv6_address = {{ ipv6 }}
 etc_dir = {{ etc_dir }}
@@ -26,6 +27,8 @@ console = {{ console }}
 verbose = {{ verbose }}
 debug = {{ debug }}
 auto_deploy = {{ auto_deploy }}
+auto_deploy_instance = {{ auto_deploy_instance }}
+autorun = {{ autorun }}
 
 [slapproxy]
 host = {{ proxy_host }}

stack/monitor/README.txt

+* This stack has for purpose to know if all promises, services, custom monitoring scripts went/are ok.
+* The second purpose of this stack is to implement a zero-knowledge feature : it means you can use its control interface to provide the user with sensible data. It can also let the user change some parameters
+* It also provides a web interface, to see which promises, services and custom scripts failed. It also provide a rss feed to easily know the actual state of your instance, and to know when it started to went bad. You can also add your own monitoring scripts, or cgi files (or just files) that you would want to check easily using a web interface.
+
+Implementation :
+----------------
+1/ In the software.cfg of your Software Release, extends the stack
+2/ In the template that will be copied for the buildout in the instance folder (instance.cfg ?), you have to add these parts:
+###Parts to add for monitoring
+  slap-parameters
+  certificate-authority
+  cron
+  cron-entry-monitor
+  cron-entry-rss
+  deploy-index
+  deploy-index-template
+  deploy-monitor-script
+  deploy-rss-script
+  deploy-settings-cgi
+  deploy-status-cgi
+  make-rss
+  monitor-promise
+  setup-static-files
+  certificate-authority
+  public
+  zero-parameters
+  cgi-httpd-wrappers
+  public-symlink
+
+* If you want to add a custom monitoring script, you can write it (in whatever language you wish) and save it in YOUR_INSTANCE_FOLDER/etc/monitor.
+The only thing to know, is that if your script successfully passed, do not return or print nothing. If there is a problem, you can print the explanation on stdout or stderr
+
+* Here are 2 promises that you can add to your instance buildout, to see if it is working (one is ok, not the other) :
+[google-promise]
+recipe = slapos.cookbook:check_url_available
+path = $${directory:promise}/google
+url = http://www.google.com
+dash_path = ${dash:location}/bin/dash
+curl_path = ${curl:location}/bin/curl
+
+[failing-promise]
+recipe = slapos.cookbook:check_url_available
+path = $${directory:promise}/fail
+url = http://127.0.0.2
+dash_path = ${dash:location}/bin/dash
+curl_path = ${curl:location}/bin/curl
+
+
+CGI Scripts:
+------------
+This stack also provides a web interface, in wich you can execute custom cgi scripts, or just print files. The web link is provided in the published parameters, as for the password that you have to change as soon as possible
+
+In that interface you will have access to the previous scripts and the RSS feed. You can also add your files/scripts.
+
+For that, there exists a folder /var/cgi-bin. You should see that directory as a tree having of deep 2. In /var/cgi-bin, you must create only folders, which are called categories. In each category, you can then add your own files.
+
+The backend system will automatically render the webpage according to the inside structure of the cgi-bin directory. Moreover, it will also let you access to your scripts only if you are logged in : you do not need do do your own authentication system ! 
+
+
+Notice :
+--------
+* /!\A default password is set up at the installation : "passwordtochange". It has to be rewritten in the control interface by the user itself
+* /!\ If you use the recipe zeroknown, never name a parameter "recipe" or "password". 
+* The control interface will let you change the values of the options declared in the [public] section of the config file (see zeroknown recipe). Other section's values will just be printed. These values won't be overwritten by buildout.
+* If you want to allow a user to change a parameter, use the recipe zeroknown, with the buildout section name : "[public]"
+* If you manually change a parameter, it could take some time for the modifications to be applied (at least 1 or 2 slapgrid-cp)
+* If you need to change the port of the web interface of the monitoring stack, just create in your software release file a part called [monitor-parameters] and give the new port value to the parameter "port".

stack/monitor/buildout.cfg

+[buildout]
+
+extends =
+  ../../component/apache/buildout.cfg
+  ../../component/curl/buildout.cfg
+  ../../component/dash/buildout.cfg
+  ../../component/dcron/buildout.cfg
+  ../../component/openssl/buildout.cfg
+  ../../component/nginx/buildout.cfg
+
+parts +=
+  backup-script-template
+  collective.recipe.template-egg
+  dcron
+  download-static-files
+  eggs
+  extra-eggs
+  make-rss
+  monitor-bin
+  monitor-template
+  nginx
+  rss-bin
+  slapos-cookbook
+  static-folder
+  template-nginx-conf
+
+[collective.recipe.template-egg]
+recipe = zc.recipe.egg
+eggs = collective.recipe.template
+  PyRSS2Gen
+  
+[extra-eggs]
+recipe = zc.recipe.egg
+interpreter = pythonwitheggs
+eggs =
+  PyRSS2Gen
+  Jinja2
+  
+[make-rss-script]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/make-rss.sh.in
+md5sum = 8d3a4b212784b591316b8b93d6bd163e
+output = ${buildout:directory}/make-rss.sh.in
+mode = 0644
+
+[monitor-template]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/monitor.cfg.in
+output = ${buildout:directory}/monitor.cfg
+filename = monitor.cfg
+md5sum = feeca02cd409457b7ee61697f2eb3eef
+mode = 0644
+
+[monitor-bin]
+recipe = hexagonit.recipe.download
+url = ${:_profile_base_location_}/${:filename}
+download-only = true
+md5sum = 05bbb70f6f69dc4f3fa83dc0f8c9960e
+filename = monitor.py.in
+mode = 0644
+
+[index]
+recipe = hexagonit.recipe.download
+url = ${:_profile_base_location_}/webfiles/${:filename}
+download-only = true
+md5sum = 91ac749f86aecc0c383d93e51e15a572
+filename = index.cgi.in
+mode = 0644
+
+[index-template]
+recipe = hexagonit.recipe.download
+url = ${:_profile_base_location_}/webfiles/${:filename}
+download-only = true
+md5sum = aa375a4225e2587b22f68c28cafd7871
+filename = index.html.jinja2
+mode = 0644
+
+[status-cgi]
+recipe = hexagonit.recipe.download
+url = ${:_profile_base_location_}/webfiles/${:filename}
+download-only = true
+md5sum = 4e5b7fc5c5c237836c7c81fe3e3bb903
+filename = status.cgi.in
+mode = 0644
+
+[settings-cgi]
+recipe = hexagonit.recipe.download
+url = ${:_profile_base_location_}/webfiles/${:filename}
+download-only = true
+md5sum = 18574b804da0c65d8670959f9e7c4774
+filename = settings.cgi.in
+mode = 0644
+
+[rss-bin]
+recipe = hexagonit.recipe.download
+url = ${:_profile_base_location_}/${:filename}
+download-only = true
+md5sum = 916f37f083b1ef391adea2f7a717bf8a
+filename = status2rss.py
+mode = 0644
+
+[dcron-service]
+recipe = slapos.recipe.template
+url = ${template-dcron-service:output}
+output = $${directory:services}/crond
+mode = 0700
+logfile = $${directory:log}/crond.log
+
+[download-static-files]
+recipe = hexagonit.recipe.download
+url = https://github.com/Sebatyne/staticForMonitoring/blob/3f0e93cec706c7ad311ddbf1ebf996965ce2f0a3/static-files.tar.gz?raw=true
+download-only = true
+md5sum = e98585b85634de48240b2c215e946769
+filename = static-files.tar.gz
+mode = 0644
+
+[eggs]
+recipe = z3c.recipe.scripts
+eggs =
+  slapos.cookbook
+  PyRSS2Gen
+
+[versions]
+PyRSS2Gen = 1.1
+Jinja2 = 2.6

stack/monitor/make-rss.sh.in

+#!${dash-output:dash}
+
+STATUS=$${monitor-parameters:result-dir}
+RSS_FILE=$${monitor-parameters:rss-path}
+PYTHON=${buildout:directory}/bin/${extra-eggs:interpreter}
+STATUS2RSS=${rss-bin:location}/${rss-bin:filename}
+
+NAME=`basename $STATUS`
+cat $STATUS/* | $PYTHON $STATUS2RSS "Monitoring RSS feed" "https://[$${slap-parameters:ipv6-random}]:$${monitor-parameters:port}/$${deploy-index:filename}" > $RSS_FILE

stack/monitor/monitor.cfg.in

+[slap-parameters]
+recipe = slapos.cookbook:slapconfiguration
+computer = $${slap-connection:computer-id}
+partition = $${slap-connection:partition-id}
+url = $${slap-connection:server-url}
+key = $${slap-connection:key-file}
+cert = $${slap-connection:cert-file}
+
+[monitor-parameters]
+monitor-dir = $${directory:var}/monitor
+result-dir = $${:monitor-dir}/bool
+json-filename = monitor.json
+json-path = $${:monitor-dir}/$${:json-filename}
+rss-path = $${:public-cgi}/$${:rss-filename}
+rss-filename = rssfeed.html
+executable = $${directory:bin}/monitor.py
+cgi-bin = $${directory:cgi-bin}
+monitoring-cgi = $${directory:monitoring-cgi}
+knowledge0-cgi = $${directory:knowledge0-cgi}
+public-cgi = $${directory:public-cgi}
+port = 9685
+
+[directory]
+home = $${buildout:directory}
+etc = $${:home}/etc
+bin = $${:home}/bin
+srv = $${:home}/srv
+var = $${:home}/var
+
+promises = $${:etc}/promise
+ca-dir = $${:srv}/ssl
+cgi-bin = $${:var}/cgi-bin
+monitoring-cgi = $${:cgi-bin}/monitoring
+knowledge0-cgi = $${:cgi-bin}/zero-knowledge
+cron-entries = $${:etc}/cron.d
+crontabs = $${:etc}/crontabs
+cronstamps = $${:etc}/cronstamps
+log = $${:var}/log
+monitor = $${:etc}/monitor
+monitor-result = $${monitor-parameters:monitor-dir}
+monitor-result-bool = $${monitor-parameters:result-dir}
+promise = $${:etc}/promise
+public-cgi = $${:cgi-bin}/public
+run = $${:var}/run
+service = $${:etc}/service/
+tmp = $${:home}/tmp
+www = $${:var}/www
+
+[public-symlink]
+recipe = cns.recipe.symlink
+symlink = $${monitor-parameters:public-cgi} = $${directory:www}/public
+autocreate = true
+
+[cron]
+recipe = slapos.cookbook:cron
+dcrond-binary = ${dcron:location}/sbin/crond
+cron-entries = $${directory:cron-entries}
+crontabs = $${directory:crontabs}
+cronstamps = $${directory:cronstamps}
+catcher = $${cron-simplelogger:wrapper}
+binary = $${directory:service}/crond
+
+# Add log to cron
+[cron-simplelogger]
+recipe = slapos.cookbook:simplelogger
+wrapper = $${directory:bin}/cron_simplelogger
+log = $${directory:log}/cron.log
+
+[cron-entry-monitor]
+<= cron
+recipe = slapos.cookbook:cron.d
+name = launch-monitor
+frequency = */5 * * * *
+command = $${monitor-parameters:executable} -a
+
+[cron-entry-rss]
+<= cron
+recipe = slapos.cookbook:cron.d
+name = build-rss
+frequency = */5 * * * *
+command = $${make-rss:output}
+
+[setup-static-files]
+recipe = hexagonit.recipe.download
+url = ${download-static-files:destination}/${download-static-files:filename}
+md5sum = e98585b85634de48240b2c215e946769
+filename = static
+destination = $${directory:www}
+ignore-existing = true
+mode = 0644
+
+[deploy-index]
+recipe = slapos.recipe.template:jinja2
+template = ${index:location}/${index:filename}
+rendered = $${directory:www}/$${:filename}
+filename = index.cgi
+mode = 0744
+context =
+  key cgi_directory monitor-parameters:cgi-bin
+  raw index_template $${deploy-index-template:location}/$${deploy-index-template:filename}
+  key password zero-parameters:monitor-password
+  raw extra_eggs_interpreter ${buildout:directory}/bin/${extra-eggs:interpreter}
+  raw default_page /index.cgi?script=$${monitor-parameters:knowledge0-cgi}%2F$${deploy-settings-cgi:filename}
+
+[deploy-index-template]
+recipe = hexagonit.recipe.download
+url = ${index-template:location}/$${:filename}
+destination = $${directory:www}
+filename = ${index-template:filename}
+download-only = true
+md5sum = aa375a4225e2587b22f68c28cafd7871
+mode = 0644
+
+[deploy-status-cgi]
+recipe = slapos.recipe.template:jinja2
+template = ${status-cgi:location}/${status-cgi:filename}
+rendered = $${monitor-parameters:monitoring-cgi}/$${:filename}
+filename = status.cgi
+mode = 0744
+context =
+  key json_file monitor-parameters:json-path
+  raw python_executable ${buildout:executable}
+
+[deploy-settings-cgi]
+recipe = slapos.recipe.template:jinja2
+template = ${settings-cgi:location}/${settings-cgi:filename}
+rendered = $${monitor-parameters:knowledge0-cgi}/$${:filename}
+filename = settings.cgi
+mode = 0744
+context =
+  raw config_cfg $${buildout:directory}/knowledge0.cfg
+  raw timestamp $${buildout:directory}/.timestamp
+  raw python_executable ${buildout:executable}
+  key pwd monitor-parameters:knowledge0-cgi
+  key this_file :filename
+
+[deploy-monitor-script]
+recipe = slapos.recipe.template:jinja2
+template = ${monitor-bin:location}/${monitor-bin:filename}
+rendered = $${monitor-parameters:executable}
+mode = 0744
+context =
+  section directory directory
+  key monitoring_file_json monitor-parameters:json-path
+  key monitoring_folder_bool monitor-parameters:result-dir
+  raw python_executable ${buildout:executable}
+
+[deploy-rss-script]
+recipe = hexagonit.recipe.download
+url = ${rss-bin:destination}/${rss-bin:filename}
+destination = $${directory:bin}
+filename = ${rss-bin:filename}
+md5sum = 916f37f083b1ef391adea2f7a717bf8a
+mode = 0744
+download-only = true
+
+[make-rss]
+recipe = slapos.recipe.template
+url = ${make-rss-script:output}
+output = $${directory:bin}/make-rss.sh
+md5sum = 8d3a4b212784b591316b8b93d6bd163e
+mode = 0744
+
+[cadirectory]
+recipe = slapos.cookbook:mkdirectory
+requests = $${directory:ca-dir}/requests/
+private = $${directory:ca-dir}/private/
+certs = $${directory:ca-dir}/certs/
+newcerts = $${directory:ca-dir}/newcerts/
+crl = $${directory:ca-dir}/crl/
+
+[certificate-authority]
+recipe = slapos.cookbook:certificate_authority
+openssl-binary = ${openssl:location}/bin/openssl
+ca-dir = $${directory:ca-dir}
+requests-directory = $${cadirectory:requests}
+wrapper = $${directory:service}/certificate_authority
+ca-private = $${cadirectory:private}
+ca-certs = $${cadirectory:certs}
+ca-newcerts = $${cadirectory:newcerts}
+ca-crl = $${cadirectory:crl}
+
+[ca-httpd]
+<= certificate-authority
+recipe = slapos.cookbook:certificate_authority.request
+key-file = $${cadirectory:certs}/httpd.key
+cert-file = $${cadirectory:certs}/httpd.crt
+executable = $${directory:bin}/cgi-httpd
+wrapper = $${directory:service}/cgi-httpd
+# Put domain name
+name = example.com
+
+###########
+# Deploy a webserver running cgi scripts for monitoring
+###########
+[public]
+recipe = slapos.cookbook:zeroknown.write
+filename = knowledge0.cfg
+monitor-password = passwordtochange
+
+[zero-parameters]
+recipe = slapos.cookbook:zeroknown.read
+filename = $${public:filename}
+
+# XXX could it be something lighter?
+[cgi-httpd-configuration-file]
+recipe = collective.recipe.template
+input = inline:
+  PidFile "$${:pid-file}"
+  ServerName example.com
+  ServerAdmin someone@email
+  <IfDefine !MonitorPort>
+  Listen [$${:listening-ip}]:$${monitor-parameters:port}
+  Define MonitorPort
+  </IfDefine>
+  DocumentRoot "$${:document-root}"
+  ErrorLog "$${:error-log}"
+  LoadModule unixd_module modules/mod_unixd.so
+  LoadModule access_compat_module modules/mod_access_compat.so
+  LoadModule authz_core_module modules/mod_authz_core.so
+  LoadModule authz_host_module modules/mod_authz_host.so
+  LoadModule mime_module modules/mod_mime.so
+  LoadModule cgid_module modules/mod_cgid.so
+  LoadModule dir_module modules/mod_dir.so
+  LoadModule ssl_module modules/mod_ssl.so
+  # SSL Configuration
+  <IfDefine !SSLConfigured>
+  Define SSLConfigured
+  SSLCertificateFile $${ca-httpd:cert-file}
+  SSLCertificateKeyFile $${ca-httpd:key-file}
+  SSLRandomSeed startup builtin
+  SSLRandomSeed connect builtin
+  SSLRandomSeed startup /dev/urandom 256
+  SSLRandomSeed connect builtin
+  SSLProtocol -ALL +SSLv3 +TLSv1
+  SSLHonorCipherOrder On
+  SSLCipherSuite RC4-SHA:HIGH:!ADH
+  </IfDefine>
+  SSLEngine   On
+  ScriptSock $${:cgid-pid-file}
+  <Directory $${:document-root}>
+    SSLVerifyDepth    1
+    SSLRequireSSL
+    SSLOptions        +StrictRequire
+    # XXX: security????
+    Options +ExecCGI
+    AddHandler cgi-script .cgi
+    DirectoryIndex $${deploy-index:filename}
+  </Directory>
+output = $${directory:etc}/cgi-httpd.conf
+# md5sum =
+listening-ip = $${slap-parameters:ipv6-random}
+# XXX: randomize-me
+htdocs = $${directory:www}
+pid-file = $${directory:run}/cgi-httpd.pid
+cgid-pid-file = $${directory:run}/cgi-httpd-cgid.pid
+document-root = $${directory:www}
+error-log = $${directory:log}/cgi-httpd-error-log
+
+[cgi-httpd-wrapper]
+recipe = slapos.cookbook:wrapper
+apache-executable = ${apache:location}/bin/httpd
+command-line = $${:apache-executable} -f $${cgi-httpd-configuration-file:output} -DFOREGROUND
+wrapper-path = $${ca-httpd:executable}
+
+[monitor-promise]
+recipe = slapos.cookbook:check_url_available
+path = $${directory:promises}/monitor
+url = https://[$${cgi-httpd-configuration-file:listening-ip}]:$${monitor-parameters:port}/$${deploy-index:filename}
+check-secure = 1
+dash_path = ${dash:location}/bin/dash
+curl_path = ${curl:location}/bin/curl
+
+[publish-connection-informations]
+recipe = slapos.cookbook:publish
+monitor_url = https://[$${cgi-httpd-configuration-file:listening-ip}]:$${monitor-parameters:port}
+IMPORTANT_monitor_info = Change the monitor_password as soon as possible ! Default is : $${public:monitor-password}

stack/monitor/monitor.py.in

+#!{{ python_executable }}
+
+import datetime
+import json
+import os
+import subprocess
+import sys
+import time
+from optparse import OptionParser, make_option
+
+instance_path = "{{ directory['home'] }}"
+monitor_dir = "{{ directory['monitor'] }}"
+pid_dir = "{{ directory['run'] }}"
+promise_dir = "{{ directory['promise'] }}"
+
+monitoring_file_json = "{{ monitoring_file_json }}"
+monitoring_folder_bool = "{{ monitoring_folder_bool }}"
+
+option_list = [
+  make_option("-a", "--all", action="store_true", dest="all",
+              help="test everything : promises, services, customs"),
+  make_option("-n", "--no-write", action="store_true", dest="only_stdout",
+              help="just show the json output on stdout"),
+  make_option("-m", "--monitors", action="store_true", dest="monitor",
+              help="add the custom monitoring file to the files to monitor"),
+  make_option("-p", "--promises", action="store_true", dest="promise",
+              help="add the promises\'file to the files to monitor"),
+  make_option("-s", "--services", action="store_true", dest="service",
+              help="add the file containing services\'pid to the files to monitor")
+]
+
+
+def getListOfScripts(directory):
+  scripts = []
+  if os.path.exists(directory) and os.path.isdir(directory):
+    for file in os.listdir(directory):
+      scripts.append(os.path.join(directory, file))
+  else:
+    exit("There is a problem in your directories" \
+          "of monitoring. Please check them")
+  return scripts
+
+def runServices(directory):
+  services = getListOfScripts(directory)
+  result = {}
+  for service in services:
+    service_path = os.path.join(pid_dir, service)
+    service_name = os.path.basename(service_path)
+    try:
+      pid = int(open(service_path).read())
+    ### because apache (or others) can write sockets
+    except IOError:
+      continue
+    try:
+      os.kill(pid, 0)
+      result[service_name] = ''
+    except OSError:
+      result[service_name] = "This service is not running anymore"
+  return result
+
+
+def runScripts(directory):
+  scripts = getListOfScripts(directory)
+  script_timeout = 3
+  result = {}
+  for script in scripts:
+    command = [os.path.join(promise_dir, script)]
+    script = os.path.basename(command[0])
+    result[script] = ''
+
+    process_handler = subprocess.Popen(command,
+                                       cwd=instance_path,
+                                       env=None if sys.platform == 'cygwin' else {},
+                                       stdout=subprocess.PIPE,
+                                       stderr=subprocess.PIPE,
+                                       stdin=subprocess.PIPE)
+    process_handler.stdin.flush()
+    process_handler.stdin.close()
+    process_handler.stdin = None
+
+    time.sleep(script_timeout)
+
+    if process_handler.poll() is None:
+      process_handler.terminate()
+      result[script] = "Time Out"
+    elif process_handler.poll() != 0:
+      stderr = process_handler.communicate()[1]
+      if stderr is not None:
+        result[script] = stderr.strip()
+  return result
+
+
+def writeFiles(monitors):
+  fail = False
+  for i in monitors.values():
+    if i != "" :
+      fail = True
+  if fail:
+    message = "FAILURE : something went wrong\n"
+  else:
+    message = "SUCCESS : everything is ok\n"
+  date = datetime.datetime.now().ctime()
+  monitors['datetime'] = date
+  file_bool = os.path.join(monitoring_folder_bool, str(time.time()))
+  open(file_bool, "w+").write(date + "," +  message)
+  open(monitoring_file_json, "w+").write(json.dumps(monitors))
+
+
+if __name__ == "__main__":
+  parser = OptionParser(option_list=option_list)
+  monitors = {}
+  (options, args) = parser.parse_args()
+
+  if not (options.monitor or options.promise
+         or options.service or options.all):
+    exit("Please provide at list one arg in : -a, -m, -p, -s")
+
+  if options.monitor or options.all:
+    monitors.update(runScripts(monitor_dir))
+  if options.promise or options.all:
+    monitors.update(runScripts(promise_dir))
+  if options.service or options.all:
+    monitors.update(runServices(pid_dir))
+
+  if options.only_stdout:
+    print json.dumps(monitors)
+  else:
+    writeFiles(monitors)
+  if len(monitors) == 0:
+    exit(0)
+  else:
+    exit(1)
+

stack/monitor/status2rss.py

+import datetime
+import uuid
+import PyRSS2Gen
+import sys
+from email.utils import parsedate_tz, mktime_tz
+import base64
+
+# Based on http://thehelpfulhacker.net/2011/03/27/a-rss-feed-for-your-crontabs/
+
+# ### Defaults
+TITLE = sys.argv[1]
+LINK = sys.argv[2]
+DESCRIPTION = TITLE
+
+items = []
+
+while 1:
+  try:
+    line = sys.stdin.readline()
+  except KeyboardInterrupt:
+    break
+
+  if not line:
+    break
+
+  time, desc = line.split(',', 1)
+
+  rss_item = PyRSS2Gen.RSSItem(
+    title = desc,
+    description = "%s, %s" % (time, desc),
+    link = LINK,
+    pubDate = datetime.datetime.fromtimestamp(mktime_tz(parsedate_tz(time))),
+    guid = PyRSS2Gen.Guid(base64.b64encode("%s, %s" % (time, desc)))
+    )
+  items.append(rss_item)
+
+### Build the rss feed
+rss_feed = PyRSS2Gen.RSS2 (
+  title = TITLE,
+  link = LINK,
+  description = DESCRIPTION,
+  lastBuildDate = datetime.datetime.utcnow(),
+  items = items
+  )
+
+print rss_feed.to_xml()
\ No newline at end of file

stack/monitor/webfiles/index.cgi.in

+#!{{ extra_eggs_interpreter }}
+
+import cgi
+import cgitb
+import Cookie
+import jinja2
+import json
+import os
+import subprocess
+import sys
+import urllib
+
+cgitb.enable(display=0, logdir="/tmp/cgi.log")
+
+form = cgi.FieldStorage()
+cookie = Cookie.SimpleCookie()
+
+cgi_path = "{{ cgi_directory }}"
+
+
+def forward_form():
+  command = os.path.join(cgi_path, form['posting-script'].value)
+  params_dict = {}
+  for f in form:
+    params_dict[f] = form[f].value
+  del params_dict['posting-script']
+  os.environ['QUERY_STRING'] = urllib.urlencode(params_dict)
+  try:
+    if os.access(command, os.X_OK):
+      print '\n', subprocess.check_output([command])
+  except subprocess.CalledProcessError:
+    print "There is a problem with sub-process"
+    pass
+
+
+def return_document():
+  command = os.path.join(cgi_path, form['script'].value)
+  #XXX this functions should be called only for display,
+  #so a priori it doesn't need form data
+  os.environ['QUERY_STRING'] = ''
+  try:
+    if os.access(command, os.X_OK):
+      print '\n', subprocess.check_output([command])
+    elif os.access(command, os.R_OK):
+      print open(command).read()
+    else:
+      raise OSError
+  except (subprocess.CalledProcessError, OSError):
+    print "<p>File cannot be found</p>"
+
+
+def make_menu():
+  # Transform deep-2 tree in json
+  folder_list = {}
+  for folder in os.listdir(cgi_path):
+    if os.path.isdir(os.path.join(cgi_path, folder)):
+      folder_list[folder] = []
+  for folder in folder_list:
+    for file in os.listdir(os.path.join(cgi_path, folder)):
+      if os.path.isfile(os.path.join(cgi_path, folder, file)):
+        folder_list[folder].append(file)
+  return folder_list
+
+
+# Beginning of response
+print "Content-Type: text/html"
+
+# Check if user is logged
+if "password" in form:
+  password = form['password'].value
+  if password == '{{ password }}' :
+    cookie['password'] = password
+    print cookie, "; Path=/; HttpOnly"
+else:
+  cookie_string = os.environ.get('HTTP_COOKIE')
+  if cookie_string:
+    cookie.load(cookie_string)
+    try:
+      password = cookie['password'].value
+    except KeyError:
+      password = None
+  else:
+    password = None
+
+print '\n'
+
+if not password or password != '{{ password }}':
+  print "<html><head>"
+  print """
+    <link rel="stylesheet" href="pure-min.css">
+    <link rel="stylesheet" href="/style.css">"""
+  print "</head><body>"
+  if password is None:
+    print "<h1>This is the monitoring interface</h1>"
+  else:
+    print "<h1>Error</h1><p>Wrong password</p>"
+  print """
+  <p>Please enter the monitor_password in the next field to access the data</p>
+  <form action="/index.cgi" method="post" class="pure-form-aligned">
+    Password : <input type="password" name="password">
+    <button type="submit" class="pure-button pure-button-primary">Access</button>
+  </form>
+  </body></html>"""
+
+# redirection to the required script/page
+else:
+  print
+  if "posting-script" in form:
+    forward_form()
+  elif "script" in form:
+    return_document()
+  else:
+    html_base = jinja2.Template(open('{{ index_template }}').read())
+    print
+    print html_base.render(tree=make_menu(), default_page="{{ default_page }}")

stack/monitor/webfiles/index.html.jinja2

+<html>
+<head>
+ <title>Monitoring Interface</title>
+ <link rel="stylesheet" href="pure-min.css">
+ <link rel="stylesheet" href="/style.css">
+ <script src="jquery-1.10.2.min.js"></script>
+ <script src="script.js"></script>
+</head>
+<body>
+ <div id="div-menu">
+  <h1>Monitoring</h1>
+ <div id="script-categories" class="pure-menu pure-menu-open">
+  <ul>
+   {% for category in tree %}
+   <li class="pure-menu-heading category">{{ category }}</li>
+    {% for script in tree[category] %}
+    <li><a href="{{ category }}/{{ script }}" class="script">{{ script }}</a></li>
+    {% endfor %}
+   {% endfor %}
+  </ul>
+ </div>
+ </div>
+ <div id="content">
+ <iframe src="{{ default_page }}">
+ </iframe>
+ </div>
+</body>
+</html>

stack/monitor/webfiles/settings.cgi.in

+#!{{ python_executable }}
+
+import cgi
+import cgitb
+import ConfigParser
+import os
+
+cgitb.enable()
+form = cgi.FieldStorage()
+
+print "<html><head>"
+print "<link rel=\"stylesheet\" href=\"pure-min.css\">"
+print "<link rel=\"stylesheet\" href=\"/style.css\">"
+print "</head><body>"
+
+config_file = "{{ config_cfg }}"
+
+if not os.path.exists(config_file):
+  print "Your software does <b>not</b> embed 0-knowledge. \
+  This interface is useless in this case"
+  exit(0)
+
+parser = ConfigParser.ConfigParser()
+parser.read(config_file)
+
+for name in form:
+  parser.set('public', name, form[name].value)
+with open(config_file, 'w') as file:
+  parser.write(file)
+
+if len(form) > 0:
+  try:
+    os.remove("{{ timestamp }}")
+  except OSError:
+    pass
+
+print "<h1>Values that can be defined :</h1>"
+print "<form action=\"/index.cgi\" method=\"post\" class=\"pure-form-aligned\">"
+print "<input type=\"hidden\" name=\"posting-script\" value=\"{{ pwd }}/{{ this_file }}\">"
+for option in parser.options("public"):
+  print "<div class=\"pure-control-group\">"
+  print "<label for=\"%s\">%s</label>"%(option, option)
+  print "<input type=\"text\" name=\"%s\" value=\"%s\">"%(option, parser.get('public', option))
+  print "</div>"
+print "<div class=\"pure-controls\"><button type=\"submit\" class=\"pure-button \
+ pure-button-primary\">Save</button></div></form>"
+
+print "<br><h1>Other values :</h1>"
+print "<form class=\"pure-form-aligned\">"
+for section in parser.sections():
+  if section != 'public':
+    for option in parser.options(section):
+      print "<div class=\"pure-control-group\">"
+      print "<label for=\"%s\">%s</label>"%(option, option)
+      print "<input type=\"text\" name=\"%s\" value=\"%s\" readonly>"%(option, parser.get(section, option))
+      print "</div>"
+print "</form>"
+
+print "</body></html>"

stack/monitor/webfiles/status.cgi.in

+#!{{ python_executable }}
+
+import cgitb
+import json
+
+cgitb.enable(display=0, logdir="/tmp/cgi.log")
+
+json_file = "{{ json_file }}"
+result = json.load(open(json_file))
+
+print "<html><head>"
+print "<link rel=\"stylesheet\" href=\"pure-min.css\">"
+print "<link rel=\"stylesheet\" href=\"/style.css\">"
+print "</head><body>"
+print "<h1>Monitoring :</h1>"
+print "<p><em>Last time of monitoring process : %s</em></p>" % (result['datetime'])
+del result['datetime']
+print "<br/>"
+
+print "<h2>These scripts and promises have failed :</h2>"
+for r in result:
+  if result[r] != '':
+    print "<h3>%s</h3><p style=\"padding-left:30px;\">%s</p>" % (r, result[r])
+print "<br/>"
+
+print "<h2>These scripts and promises were successful :</h2>"
+print "<ul>"
+for r in result:
+  if result[r] == '':
+    print "<li>%s</li>" % (r)
+print "</ul>"
+print "</body></html>"

stack/slapos.cfg

@@ -15,7 +15,8 @@ extensions +=
 # Use shacache and lxml
 extends =
   ../component/lxml-python/buildout.cfg
-  ../component/python-openssl/buildout.cfg
+  ../component/python-cffi/buildout.cfg
+  ../component/python-cryptography/buildout.cfg
 
 # Separate from site eggs
 allowed-eggs-from-site-packages =
@@ -63,7 +64,9 @@ networkcache-section = networkcache
 recipe = zc.recipe.egg
 eggs =
   ${lxml-python:egg}
-  ${python-openssl:egg}
+  ${python-cffi:egg}
+  ${python-cryptography:egg}
+  pyOpenSSL
   slapos.cookbook
   cliff
   hexagonit.recipe.download