Update Release Candidate

CHANGES.rst

 Changes
 =======
 
+1.0.41 (2016-10-26)
+-------------------
+
+  * dcron: new parameter to get a random time, with a frequency of once a day
+  * softwaretype: fix parse error on '+ =' when using buildout 2
+  * pbs: General Improvement and fixes.
+
 1.0.35 (2016-09-19)
 -------------------
 

component/swig/buildout.cfg

@@ -9,8 +9,8 @@ parts =
 
 [swig]
 recipe = slapos.recipe.cmmi
-url = http://downloads.sourceforge.net/project/swig/swig/swig-1.3.40/swig-1.3.40.tar.gz
-md5sum = 2df766c9e03e02811b1ab4bba1c7b9cc
+url = http://prdownloads.sourceforge.net/swig/swig-3.0.10.tar.gz
+md5sum = bb4ab8047159469add7d00910e203124
 configure-options =
   --disable-ccache
   --with-python=${buildout:executable}
@@ -32,5 +32,6 @@ configure-options =
   --without-r
   --without-ruby
   --without-tcl
+  --without-pcre
 environment =
   PATH = ${bison:location}/bin:%(PATH)s

setup.py

@@ -28,7 +28,7 @@ from setuptools import setup, find_packages
 import glob
 import os
 
-version = '1.0.36.dev0'
+version = '1.0.41'
 name = 'slapos.cookbook'
 long_description = open("README.rst").read() + "\n" + \
     open("CHANGES.rst").read() + "\n"

slapos/recipe/notifier.py

@@ -73,7 +73,7 @@ class Notify(GenericBaseRecipe):
     self.options = options
 
   def createNotifier(self, notifier_binary, wrapper, executable,
-                     log, title, notification_url, feed_url, pidfile=None,
+                     log, title, notification_url, feed_url, max_run='1', pidfile=None,
                      instance_root_name=None, log_url=None, status_item_directory=None):
 
     if not os.path.exists(log):
@@ -84,6 +84,7 @@ class Notify(GenericBaseRecipe):
             '-l', log,
             '--title', title,
             '--feed', feed_url,
+            '--max-run', str(max_run),
             '--notification-url',
             ]
     parameters.extend(notification_url.split(' '))
@@ -120,5 +121,6 @@ class Notify(GenericBaseRecipe):
                                  title=options['title'],
                                  pidfile=options['pidfile'],
                                  notification_url=options['notify'],
-                                 feed_url=feed_url)
+                                 feed_url=feed_url,
+                                 max_run=options.get('max-run', "1"))
     return [script]

software/agent/software.cfg

@@ -51,29 +51,29 @@ pycrypto = 2.6.1
 pycurl = 7.43.0
 slapos.recipe.download = 1.0
 slapos.recipe.template = 2.8
-slapos.toolbox = 0.60
+slapos.toolbox = 0.61
 smmap = 0.9.0
 
 # Required by:
-# slapos.toolbox = 0.60
+# slapos.toolbox = 0.61
 GitPython = 2.0.8
 
 # Required by:
-# slapos.toolbox = 0.60
+# slapos.toolbox = 0.61
 atomize = 0.2.0
 
 # Required by:
-# slapos.toolbox = 0.60
+# slapos.toolbox = 0.61
 feedparser = 5.2.1
 
 # Required by:
-# slapos.toolbox = 0.60
+# slapos.toolbox = 0.61
 lockfile = 0.12.2
 
 # Required by:
-# slapos.toolbox = 0.60
+# slapos.toolbox = 0.61
 paramiko = 2.0.1
 
 # Required by:
-# slapos.toolbox = 0.60
+# slapos.toolbox = 0.61
 rpdb = 0.1.5

software/apache-frontend/common.cfg

 [buildout]
 extends =
-# dev Stuff
-  ../../component/git/buildout.cfg
-
   ../../stack/slapos.cfg
+  ../../component/git/buildout.cfg
   ../../component/dash/buildout.cfg
   ../../component/lxml-python/buildout.cfg
   ../../component/apache/buildout.cfg
@@ -16,7 +14,10 @@ extends =
   ../../component/rdiff-backup/buildout.cfg
   ../../component/trafficserver/buildout.cfg
 
-# Monitoring stack
+  ../../component/pycurl/buildout.cfg
+  ../../component/nginx/buildout.cfg
+  ../../stack/nodejs.cfg
+# Monitoring stack (keep on bottom)
   ../../stack/monitor/buildout.cfg
 
 parts +=
@@ -34,6 +35,10 @@ parts +=
   dcron
   logrotate
   rdiff-backup
+  nginx-push-stream-output
+  npm-modules
+  proxy-by-url
+  http-proxy
 
 [slapos-toolbox]
 recipe = zc.recipe.egg
@@ -50,7 +55,17 @@ scripts =
 [eggs]
 recipe = zc.recipe.egg
 eggs =
+  ${python-cffi:egg}
+  ${python-cryptography:egg}
+  ${lxml-python:egg}
+  websockify
+  slapos.cookbook
+  slapos.toolbox
+  erp5.util
+  cns.recipe.symlink
+  collective.recipe.template
   plone.recipe.command
+  ${pycurl:egg}
 
 [check-recipe]
 recipe = plone.recipe.command
@@ -137,7 +152,7 @@ mode = 640
 [template-default-slave-virtualhost]
 recipe = slapos.recipe.build:download
 url = ${:_profile_base_location_}/templates/default-virtualhost.conf.in
-md5sum = c50959e7c38b307f1a1c274505a7e1c1 
+md5sum = 8975fd41fae2dcac92e18df3c6375f9a 
 mode = 640
 
 [template-cached-slave-virtualhost]
@@ -184,3 +199,34 @@ location = ${buildout:parts-directory}/${:_buildout_section_name_}
 filename = storage.config.jinja2
 download-only = true
 mode = 0644
+
+
+# Migrated from KVM recipe
+[http-proxy]
+# https://github.com/nodejitsu/node-http-proxy
+recipe = slapos.recipe.build:download-unpacked
+# use upstream when merged
+url = https://lab.nexedi.com/nexedi/node-http-proxy/repository/archive.zip?ref=a5d3aff428ee8d840068b439f6ce121077f1144f
+md5sum = 65602466066444c718215de41f546585
+
+[proxy-by-url]
+# https://github.com/dominictarr/proxy-by-url
+recipe = slapos.recipe.build:download-unpacked
+# use upstream when merged
+url = https://lab.nexedi.com/nexedi/proxy-by-url/repository/archive.zip?ref=59fcb11a3e00c45b4b0362e76f29653abb313072
+md5sum = c383e0c5ff31b56f7987466e8304c941
+
+[npm-modules]
+recipe = plone.recipe.command
+destination = ${buildout:parts-directory}/${:_buildout_section_name_}
+location = ${buildout:parts-directory}/${:_buildout_section_name_}
+command =
+  export HOME=${:location};
+  rm -fr ${:destination} &&
+  mkdir -p ${:destination} &&
+  cd ${:destination} &&
+  ${nodejs:location}/bin/node ${nodejs:location}/bin/npm install colors@0.6.0-1 &&
+  ${nodejs:location}/bin/node ${nodejs:location}/bin/npm install socket.io@0.8.7 &&
+  ${nodejs:location}/bin/node ${nodejs:location}/bin/npm install socket.io-client@0.8.7 &&
+  ${nodejs:location}/bin/node ${nodejs:location}/bin/npm install optimist@0.3.1 &&
+  ${nodejs:location}/bin/node ${nodejs:location}/bin/npm install pkginfo@0.2.3

software/apache-frontend/software.cfg

@@ -11,33 +11,33 @@ plone.recipe.command = 1.1
 pycrypto = 2.6.1
 rdiff-backup = 1.0.5
 slapos.recipe.template = 2.8
-slapos.toolbox = 0.60
+slapos.toolbox = 0.61
 smmap = 0.9.0
 
 # Required by:
-# slapos.toolbox = 0.60
+# slapos.toolbox = 0.61
 GitPython = 2.0.8
 
 # Required by:
-# slapos.toolbox = 0.60
+# slapos.toolbox = 0.61
 atomize = 0.2.0
 
 # Required by:
-# slapos.toolbox = 0.60
+# slapos.toolbox = 0.61
 feedparser = 5.2.1
 
 # Required by:
-# slapos.toolbox = 0.60
+# slapos.toolbox = 0.61
 lockfile = 0.12.2
 
 # Required by:
-# slapos.toolbox = 0.60
+# slapos.toolbox = 0.61
 paramiko = 2.0.1
 
 # Required by:
-# slapos.toolbox = 0.60
+# slapos.toolbox = 0.61
 pycurl = 7.43.0
 
 # Required by:
-# slapos.toolbox = 0.60
+# slapos.toolbox = 0.61
 rpdb = 0.1.5

software/apache-frontend/templates/default-virtualhost.conf.in

@@ -78,14 +78,14 @@
   # First, we check if we have a zope backend server
   # If so, let's use Virtual Host Monster rewrite
   # We suppose that Apache listens to 443 (even indirectly thanks to things like iptables)
-  RewriteRule ^/(.*)$ {{ slave_parameter.get('https-url', '') }}/VirtualHostBase/https//%{SERVER_NAME}:443/{{ slave_parameter.get('path', '') }}/VirtualHostRoot/$1 [L,P]
+  RewriteRule ^/(.*)$ {{ slave_parameter.get('https-url', slave_parameter.get('url', '')) }}/VirtualHostBase/https//%{SERVER_NAME}:443/{{ slave_parameter.get('path', '') }}/VirtualHostRoot/$1 [L,P]
 {% elif slave_parameter.get('type', '') ==  'redirect' -%}
-  RewriteRule     (.*)  {{ slave_parameter.get('https-url', '')}}$1 [R,L]
+  RewriteRule     (.*)  {{ slave_parameter.get('https-url', slave_parameter.get('url', ''))}}$1 [R,L]
 {% else -%}
   {% if 'default-path' in slave_parameter %}
   RewriteRule ^/?$ {{ slave_parameter.get('default-path') }} [R=301,L]
   {% endif -%}
-  RewriteRule ^/(.*)$ {{ slave_parameter.get('https-url', '') }}/$1 [L,P]
+  RewriteRule ^/(.*)$ {{ slave_parameter.get('https-url', slave_parameter.get('url', '')) }}/$1 [L,P]
 {% endif -%}
 </VirtualHost>
 

software/erp5/instance-erp5-input-schema.json

@@ -237,10 +237,10 @@
       "type": "array"
     },
     "jupyter": {
-      "description": "Jupyter slave instance parameters",
+      "description": "Jupyter subinstance parameters",
       "properties": {
         "enable": {
-          "description": "Whether to enable creation of associated slave Jupyter instance",
+          "description": "Whether to enable creation of associated Jupyter subinstance",
           "default": false,
           "type": "boolean"
         },

software/kvm/software.cfg

@@ -5,7 +5,7 @@ extends = common.cfg
 # XXX - use websockify = 0.5.1 for compatibility with kvm frontend
 websockify = 0.5.1
 
-slapos.toolbox = 0.60
+slapos.toolbox = 0.61
 erp5.util = 0.4.46
 apache-libcloud = 1.1.0
 collective.recipe.environment = 0.2.0
@@ -16,23 +16,23 @@ smmap = 0.9.0
 # websockify = 0.8.0
 
 # Required by:
-# slapos.toolbox==0.60
+# slapos.toolbox==0.61
 GitPython = 2.0.8
 
 # Required by:
-# slapos.toolbox==0.60
+# slapos.toolbox==0.61
 atomize = 0.2.0
 
 # Required by:
-# slapos.toolbox==0.60
+# slapos.toolbox==0.61
 dnspython = 1.14.0
 
 # Required by:
-# slapos.toolbox==0.60
+# slapos.toolbox==0.61
 feedparser = 5.2.1
 
 # Required by:
-# slapos.toolbox==0.60
+# slapos.toolbox==0.61
 lockfile = 0.12.2
 
 # Required by:
@@ -40,9 +40,9 @@ lockfile = 0.12.2
 numpy = 1.11.2rc1
 
 # Required by:
-# slapos.toolbox==0.60
+# slapos.toolbox==0.61
 paramiko = 2.0.2
 
 # Required by:
-# slapos.toolbox==0.60
+# slapos.toolbox==0.61
 passlib = 1.6.5
\ No newline at end of file

software/neoppod/software-common.cfg

@@ -103,8 +103,8 @@ md5sum = 81ab5e842ecf8385b12d735585497cc8
 [versions]
 slapos.recipe.template = 2.9
 ZODB3 = 3.10.7
-# Required by slapos.toolbox = 0.60
-slapos.toolbox = 0.60
+# Required by slapos.toolbox = 0.61
+slapos.toolbox = 0.61
 PyRSS2Gen = 1.1
 apache-libcloud = 1.2.1
 atomize = 0.2.0

software/powerdns/instance.cfg

@@ -43,7 +43,7 @@ extensions = jinja2.ext.do
 extra-context =
 # Must match the key id in [switch-softwaretype] which uses this section.
     raw software_type RootSoftwareInstance-default
-    raw template_monitor monitor-template:output
+    raw template_monitor ${monitor2-template:rendered}
 
 [slap-parameters]
 recipe = slapos.cookbook:slapconfiguration
@@ -52,4 +52,4 @@ partition = $${slap-connection:partition-id}
 url = $${slap-connection:server-url}
 key = $${slap-connection:key-file}
 cert = $${slap-connection:cert-file}
-slave-instance-list =
\ No newline at end of file
+slave-instance-list =

software/powerdns/software.cfg

@@ -18,7 +18,7 @@ eggs =
 [template]
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance.cfg
-md5sum = 19116ef08be2e9b5bda5e3e5139e45ac
+md5sum = da8be58db4255c07750f7a7583eab3ca
 output = ${buildout:directory}/template.cfg
 mode = 0644
 

software/re6stnet/software.cfg

@@ -115,15 +115,15 @@ plone.recipe.command = 1.1
 pycrypto = 2.6.1
 pycurl = 7.43.0
 slapos.recipe.template = 2.7
-slapos.toolbox = 0.60
+slapos.toolbox = 0.61
 smmap = 0.9.0
 
 # Required by:
-# slapos.toolbox = 0.60
+# slapos.toolbox = 0.61
 GitPython = 2.0.8
 
 # Required by:
-# slapos.toolbox = 0.60
+# slapos.toolbox = 0.61
 atomize = 0.2.0
 
 # Required by:
@@ -131,11 +131,11 @@ atomize = 0.2.0
 backports.ssl-match-hostname = 3.4.0.2
 
 # Required by:
-# slapos.toolbox = 0.60
+# slapos.toolbox = 0.61
 feedparser = 5.1.3
 
 # Required by:
-# slapos.toolbox = 0.60
+# slapos.toolbox = 0.61
 lockfile = 0.12.2
 
 # Required by:
@@ -143,10 +143,10 @@ lockfile = 0.12.2
 miniupnpc = 1.9
 
 # Required by:
-# slapos.toolbox = 0.60
+# slapos.toolbox = 0.61
 paramiko = 2.0.1
 
 # Required by:
-# slapos.toolbox = 0.60
+# slapos.toolbox = 0.61
 rpdb = 0.1.5
 

software/slapos-master/instance-erp5.cfg.in

@@ -6,7 +6,7 @@
 {% set publish_dict = {'site-id': site_id, 'inituser-login': inituser_login} -%}
 {% set has_posftix = slapparameter_dict.get('smtp', {}).get('postmaster') -%}
 {% set jupyter_dict = slapparameter_dict.get('jupyter', {}) -%}
-{% set has_jupyter = jupyter_dict.get('enable', jupyter_enable_default).lower() in ('true', 'yes') -%}
+{% set has_jupyter = jupyter_dict.get('enable', jupyter_enable_default.lower() in ('true', 'yes')) -%}
 {% set jupyter_zope_family = jupyter_dict.get('zope-family', '') -%}
 [request-common]
 <= request-common-base

software/slapos-master/software.cfg

@@ -54,7 +54,7 @@ mode = 644
 [template-erp5]
 < = download-base-part
 filename = instance-erp5.cfg.in
-md5sum = 6b7712c26dfdd105e8a1143d04839651 
+md5sum = 073b96888388a39721a8cef3e90c5c5b
 
 [template-balancer]
 < = download-base-part

software/slaprunner/software.cfg

@@ -17,41 +17,41 @@ gunicorn = 19.4.5
 prettytable = 0.7.2
 pycurl = 7.43.0
 slapos.recipe.template = 2.9
-slapos.toolbox = 0.60
+slapos.toolbox = 0.61
 smmap = 0.9.0
 
 # Required by:
-# slapos.toolbox==0.60
+# slapos.toolbox==0.61
 GitPython = 2.0.8
 
 # Required by:
-# slapos.toolbox==0.60
+# slapos.toolbox==0.61
 PyRSS2Gen = 1.1
 
 # Required by:
-# slapos.toolbox==0.60
+# slapos.toolbox==0.61
 atomize = 0.2.0
 
 # Required by:
-# slapos.toolbox==0.60
+# slapos.toolbox==0.61
 dnspython = 1.14.0
 
 # Required by:
-# slapos.toolbox==0.60
+# slapos.toolbox==0.61
 erp5.util = 0.4.46
 
 # Required by:
-# slapos.toolbox==0.60
+# slapos.toolbox==0.61
 feedparser = 5.2.1
 
 # Required by:
-# slapos.toolbox==0.60
+# slapos.toolbox==0.61
 lockfile = 0.12.2
 
 # Required by:
-# slapos.toolbox==0.60
+# slapos.toolbox==0.61
 paramiko = 2.0.2
 
 # Required by:
-# slapos.toolbox==0.60
+# slapos.toolbox==0.61
 passlib = 1.6.5

stack/erp5/buildout.cfg

@@ -331,7 +331,7 @@ rendered = ${monitor-template-dummy:target}
 [template-erp5]
 <= download-base
 filename = instance-erp5.cfg.in
-md5sum = cb0a8dfc15874939ec786a8b7ac5bff1
+md5sum = b5f49c90017684aa3389ef3f97ece509
 
 [template-zeo]
 <= download-base

stack/erp5/instance-erp5.cfg.in

@@ -6,7 +6,7 @@
 {% set publish_dict = {'site-id': site_id, 'inituser-login': inituser_login} -%}
 {% set has_posftix = slapparameter_dict.get('smtp', {}).get('postmaster') -%}
 {% set jupyter_dict = slapparameter_dict.get('jupyter', {}) -%}
-{% set has_jupyter = jupyter_dict.get('enable', jupyter_enable_default).lower() in ('true', 'yes') -%}
+{% set has_jupyter = jupyter_dict.get('enable', jupyter_enable_default.lower() in ('true', 'yes')) -%}
 {% set jupyter_zope_family = jupyter_dict.get('zope-family', '') -%}
 [request-common]
 <= request-common-base

stack/monitor/buildout.cfg

@@ -93,7 +93,7 @@ recipe = slapos.recipe.template:jinja2
 filename = template-monitor.cfg
 template = ${:_profile_base_location_}/instance-monitor.cfg.jinja2.in
 rendered = ${buildout:directory}/template-monitor.cfg
-md5sum = 0955530291105a322f2972244caf307d
+md5sum = 266f23bc836a938ac69ce0bc674f34fe
 context =
     key apache_location apache:location
     key gzip_location gzip:location
@@ -122,5 +122,5 @@ depends =
 PyRSS2Gen = 1.1
 cns.recipe.symlink = 0.2.3
 pycurl = 7.43.0
-slapos.toolbox = 0.60
+slapos.toolbox = 0.61
 

stack/monitor/instance-monitor.cfg.jinja2.in

@@ -373,16 +373,14 @@ check-secure = 1
 recipe = collective.recipe.template
 file = ${monitor-conf-parameters:promise-output-file}
 input = inline:#!{{ dash_executable_location }}
-  if [ -f "${monitor-conf-parameters:pid-file}" ]; then
-    PID=`cat ${monitor-conf-parameters:pid-file}`
+  pidfile=${monitor-conf-parameters:pid-file}
+  COMMAND=monitor.bootstrap
+  if [ -s $pidfile ]; then
     for i in {1..20}; do
-      # wait for monitor bootstrap script to finish (max 10 seconds)
-      kill -0 "$PID" > /dev/null 2>&1
-      RETURN_CODE=$?
-      if [ $RETURN_CODE -eq 0 ] ; then
-        break
-      else
+      if pid=`pgrep -F $pidfile -f "$COMMAND" 2>/dev/null`; then
         sleep 0.5
+      else
+        break
       fi
     done
   fi

stack/resilient/buildout.cfg

@@ -59,7 +59,7 @@ mode = 0644
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/pbsready-export.cfg.in
 output = ${buildout:directory}/pbsready-export.cfg
-md5sum = 031fb7785cd5e99c2d794fa7e00e5190
+md5sum = 793f1843a643b3c91b658eca2bad5abc
 mode = 0644
 
 [template-pull-backup]

stack/resilient/pbsready-export.cfg.in

@@ -57,6 +57,7 @@ executable = $${exporter:wrapper}
 wrapper = $${rootdirectory:bin}/exporter
 notify = $${slap-parameter:notify}
 pidfile = $${resilient-directory:pid}/$${:name}.pid
+max-run = 3
 
 [notifier-exporter-promise]
 recipe = slapos.recipe.template:jinja2

stack/slapos.cfg

@@ -133,8 +133,8 @@ pytz = 2016.6.1
 requests = 2.11.1
 setuptools = 19.6.2
 six = 1.10.0
-slapos.cookbook = 1.0.35
-slapos.core = 1.3.16
+slapos.cookbook = 1.0.41
+slapos.core = 1.3.17
 slapos.extension.strip = 0.1
 slapos.libnetworkcache = 0.14.5
 slapos.recipe.build = 0.25
@@ -144,7 +144,7 @@ unicodecsv = 0.14.1
 xml-marshaller = 0.9.7
 
 # Required by:
-# slapos.core==1.3.16
+# slapos.core==1.3.17
 Flask = 0.11.1
 
 # Required by:
@@ -164,11 +164,11 @@ functools32 = 3.2.3.post2
 ipaddress = 1.0.17
 
 # Required by:
-# slapos.cookbook==1.0.35
+# slapos.cookbook==1.0.41
 jsonschema = 2.5.1
 
 # Required by:
-# slapos.cookbook==1.0.35
+# slapos.cookbook==1.0.41
 lock-file = 2.0
 
 # Required by: