WIP rewrite of KVM software release.

Move all configuration to more generic recipes, so that recipes can be shared
between software releases.

Disable previous recipers.

Slave configuration is not finished and need to be migrated.
It is needed to publish the slave URL.

Hardcoded vifib rewrite map has been removed.
It is needed to use a frontend slave instance to achieve this functionnality.
This means, frontend slave instances need more parameters.

NBD software release has been merged into the KVM software release as a
software type. It allows to use the same binary for nbd and kvm, and so, save
space on servers and ease maintainance.

Remove software release's python dependency.

Add more promises to check status of nbd, kvm and frontend.

slapmonitor functionnality has to be restored.

component/gettext/buildout.cfg

@@ -20,6 +20,7 @@ configure-options =
   --without-emacs
   --disable-acl
   --disable-openmp
+  --without-git
 
 environment =
   CPPFLAGS=-I${libxml2:location}/include -I${zlib:location}/include -I${ncurses:location}/include

setup.py

@@ -58,7 +58,10 @@ setup(name=name,
           'erp5testnode = slapos.recipe.erp5testnode:Recipe',
           'generate.mac = slapos.recipe.generatemac:Recipe',
           'generic.kvm = slapos.recipe.generic_kvm:Recipe',
+          'generic.kvm.frontend = slapos.recipe.generic_kvm_frontend:Recipe',
+          'generic.nbdserver = slapos.recipe.generic_nbdserver:Recipe',
           'generic.novnc = slapos.recipe.generic_novnc:Recipe',
+          'generic.onetimeupload = slapos.recipe.generic_onetimeupload:Recipe',
           'helloworld = slapos.recipe.helloworld:Recipe',
           'generic.cloudooo = slapos.recipe.generic_cloudooo:Recipe',
           'fontconfig = slapos.recipe.fontconfig:Recipe',
@@ -66,8 +69,6 @@ setup(name=name,
           'kumofs = slapos.recipe.kumofs:Recipe',
           'generic.kumofs = slapos.recipe.generic_kumofs:Recipe',
           'haproxy = slapos.recipe.haproxy:Recipe',
-          'kvm = slapos.recipe.kvm:Recipe',
-          'kvm_frontend = slapos.recipe.kvm_frontend:Recipe',
           'libcloud = slapos.recipe.libcloud:Recipe',
           'libcloudrequest = slapos.recipe.libcloudrequest:Recipe',
           'lockfile = slapos.recipe.lockfile:Recipe',
@@ -77,7 +78,6 @@ setup(name=name,
           'mydumper = slapos.recipe.mydumper:Recipe',
           'generic.mysql = slapos.recipe.generic_mysql:Recipe',
           'mkdirectory = slapos.recipe.mkdirectory:Recipe',
-          'nbdserver = slapos.recipe.nbdserver:Recipe',
           'nosqltestbed = slapos.recipe.nosqltestbed:NoSQLTestBed',
           'notifier = slapos.recipe.notifier:Recipe',
           'notifier.callback = slapos.recipe.notifier:Callback',

slapos/recipe/generic_kvm_frontend/__init__.py

+##############################################################################
+#
+# Copyright (c) 2011 Vifib SARL and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+from slapos.recipe.librecipe import GenericBaseRecipe
+from json import loads as unjson
+
+class Recipe(GenericBaseRecipe):
+  """
+  kvm frontend instance configuration.
+  """
+
+  def _getRewriteRuleContent(self, slave_instance_list):
+    """Generate rewrite rules list from slaves list"""
+    rewrite_rule_list = []
+    for slave_instance in slave_instance_list:
+      current_slave_dict = dict()
+      # Get host, and if IPv6 address, remove "[" and "]"
+      current_slave_dict['host'] = current_slave_dict['host'].\
+          replace('[', '').replace(']', '')
+      current_slave_dict['port'] = slave_instance['port']
+      if current_slave_dict['host'] is None \
+          or current_slave_dict['port'] is None:
+        # XXX-Cedric: should raise warning because slave seems badly configured
+        continue
+      # Check if target is https or http
+      current_slave_dict['https'] = slave_instance.get('https', 'true')
+      if current_slave_dict['https'] in FALSE_VALUE_LIST:
+        current_slave_dict['https'] = 'false'
+      # Set reference and resource url
+      # Reference is raw reference from SlapOS Master, resource is
+      # URL-compatible name
+      reference = slave_instance.get('slave_reference')
+      current_slave_dict['reference'] = reference
+      current_slave_dict['resource'] = reference.replace('-', '')
+      rewrite_rule_list.append(current_slave_dict)
+    return rewrite_rule_list
+
+  def _getProxyTableContent(self, rewrite_rule_list):
+    """Generate proxy table file content from rewrite rules list"""
+    proxy_table_content = '{'
+    for rewrite_rule in rewrite_rule_list:
+      rewrite_part = self.substituteTemplate(
+         self.getTemplateFilename('proxytable-resource-snippet.json.in'),
+         rewrite_rule)
+      proxy_table_content += "%s," % rewrite_part
+#     proxy_table_content = '%s%s' % (proxy_table_content,
+#          open(self.getTemplateFilename('proxytable-vifib-snippet.json.in')).read())
+    proxy_table_content += '}\n'
+    return proxy_table_content
+
+  def install(self):
+    # Generate rewrite rules
+    rewrite_rule_list = self._getRewriteRuleContent(
+      unjson(self.options['slave-instance-list']))
+    # Create Map
+    map_content = self._getProxyTableContent(rewrite_rule_list)
+    map_file = self.createFile(self.options['map-path'], map_content)
+
+    # Create configuration
+    conf = open(self.getTemplateFilename('kvm-proxy.js'), 'r')
+    conf_file = self.createFile(self.options['conf-path'], conf.read())
+    conf.close()
+
+    config = dict(
+      ip=self.options['ip'],
+      port=self.options['port'],
+      key=self.options['ssl-key-path'],
+      certificate=self.options['ssl-cert-path'],
+      name=self.options['domain'],
+      shell_path=self.options['shell-path'],
+      node_path=self.options['node-binary'],
+      node_env=self.options['node-env'],
+      conf_path=conf_file,
+      map_path=map_file,
+      plain_http='',
+    )
+
+    runner_path = self.createExecutable(
+      self.options['wrapper-path'],
+      self.substituteTemplate(self.getTemplateFilename('nodejs_run.in'),
+                              config))
+
+    return [map_file, conf_file, runner_path]

slapos/recipe/kvm_frontend/template/kvm-proxy.js → slapos/recipe/generic_kvm_frontend/template/kvm-proxy.js

@@ -34,26 +34,10 @@ var middlewareNotFound = function(req, res, proxy) {
       'SlapOS administrator.');
 };
 
-/**
- * Rewrite URL to match Zope's virtual host monster if we use vifib
- */
-var middlewareVifib = function(req, res, next) {
-  // Completely hardcoded rewrite
-  var vifibPrefix = '/hosting';
-  if (req.url.indexOf(vifibPrefix) == 0) {
-    // Rewrite URL to match virtual host
-    req.url = vifibPrefix + '/VirtualHostBase/https/' + req.headers.host +
-              '/erp5/web_site_module/VirtualHostRoot' + req.url;
-    console.log('Vifib rewrite. New URL is : ' + req.url);
-  }
-  next();
-};
-
 /**
  * Create server
  */
 var proxyServer = httpProxy.createServer(
-  middlewareVifib,
   // We declare our proxyByUrl middleware
   proxyByUrl(proxyTable),
   // Then we add your dummy middleware, called when proxyByUrl doesn't find url.
@@ -62,12 +46,10 @@ var proxyServer = httpProxy.createServer(
   {
     https: {
       key: fs.readFileSync(
-        //'/Users/cedricdesaintmartin/Desktop/SlapOS/slapconsole-keys/cedric-owf-0/ssl.key',
         sslKeyFile,
         'utf8'
       ),
       cert: fs.readFileSync(
-        //'/Users/cedricdesaintmartin/Desktop/SlapOS/slapconsole-keys/cedric-owf-0/ssl.cert',
         sslCertFile,
         'utf8'
       )

slapos/recipe/generic_kvm_frontend/template/nodejs_run.in

+#!%(shell_path)s
+# BEWARE: This file is operated by slapgrid
+# BEWARE: It will be overwritten automatically
+export NODE_PATH=%(node_env)s
+exec %(node_path)s %(conf_path)s %(ip)s %(port)s %(key)s %(certificate)s %(map_path)s %(plain_http)s

slapos/recipe/generic_nbdserver/__init__.py

+##############################################################################
+#
+# Copyright (c) 2011 Vifib SARL and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+from slapos.recipe.librecipe import GenericBaseRecipe
+import binascii
+import os
+import sys
+
+class Recipe(GenericBaseRecipe):
+  """
+  nbd instance configuration.
+  """
+
+  def install(self):
+    config = dict(
+      ip=self.options['ip'],
+      port=self.options['port'],
+      image_path=self.options['image-path'],
+      qemu_path=self.options['qemu-path'],
+      shell_path=self.options['shell-path'],
+    )
+
+    # Runners
+    runner_path = self.createExecutable(
+      self.options['path'],
+      self.substituteTemplate(self.getTemplateFilename('nbdserver_run.in'),
+                              config))
+
+    return [runner_path]

slapos/recipe/nbdserver/template/nbdserver_run.in → slapos/recipe/generic_nbdserver/template/nbdserver_run.in

-#!/bin/sh
+#!%(shell_path)s
 # BEWARE: This file is operated by slapgrid
 # BEWARE: It will be overwritten automatically
 
 # 32767 is the maximum number of connections allowed by the nbd server
-exec %(qemu_path)s -b %(ip)s %(image)s -r -t -p %(port)s -e 32767
+exec %(qemu_path)s -b %(ip)s %(image_path)s -r -t -p %(port)s -e 32767

slapos/recipe/nbdserver/__init__.py → slapos/recipe/generic_onetimeupload/__init__.py

 ##############################################################################
 #
-# Copyright (c) 2010 Vifib SARL and Contributors. All Rights Reserved.
+# Copyright (c) 2011 Vifib SARL and Contributors. All Rights Reserved.
 #
 # WARNING: This program as such is intended to be used by professional
 # programmers who take the whole responsibility of assessing all potential
@@ -24,59 +24,36 @@
 # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
 #
 ##############################################################################
-import os
+from slapos.recipe.librecipe import GenericBaseRecipe
 import binascii
-from slapos.recipe.librecipe import BaseSlapRecipe
-
-import pkg_resources
-
-class Recipe(BaseSlapRecipe):
-
-  def _install(self):
-
-    # Image path
-    cdrom_iso = os.path.join(self.data_root_directory, 'cdrom.iso')
-
-    #Get the IP list
-    ip = self.getGlobalIPv6Address()
-    http_port = 9999
-    nbd_port = 1024
-
-    # Instanciate onetimeupload
-    onetimeupload_config = {}
-    onetimeupload_config.update(self.options)
-    onetimeupload_config['port'] = http_port
-    onetimeupload_config['ip'] = ip
-    onetimeupload_config['image'] = cdrom_iso
-    onetimeupload_config['key'] = binascii.hexlify(os.urandom(24))
-    onetimeupload_config['log_path'] = os.path.join(self.log_directory, 
-                                                    'onetimeupload.log')
-
-    wrapper_template_location = pkg_resources.resource_filename(
-                                       __name__, os.path.join(
-                                       'template', 'onetimeupload_run.in'))
-    onetimeupload_runner_path = self.createRunningWrapper("onetimeupload",
-        self.substituteTemplate(wrapper_template_location, 
-                                onetimeupload_config))
-
-    # Instanciate qemu
-    qemu_config = {}
-    qemu_config.update(self.options)
-    qemu_config['ip'] = ip
-    qemu_config['port'] = nbd_port
-    qemu_config['image'] = cdrom_iso
-
-    wrapper_template_location = pkg_resources.resource_filename(
-                                             __name__, os.path.join(
-                                             'template', 'nbdserver_run.in'))
-    nbdserver_runner_path = self.createRunningWrapper("nbdserver",
-        self.substituteTemplate(wrapper_template_location, qemu_config))
-
-    # Publish connection dict
-    self.computer_partition.setConnectionDict(dict(
-      upload_connection_string="https://[%s]:%s/" % (ip, http_port),
-      upload_key=onetimeupload_config['key'],
-      nbd_connection_string="nbd:[%s]:%s" % (ip, nbd_port),
-      ))
+import os
+import sys
+
+class Recipe(GenericBaseRecipe):
+  """
+  kvm instance configuration.
+  """
+
+  def __init__(self, buildout, name, options):
+    options['key'] = binascii.hexlify(os.urandom(24))
+    return GenericBaseRecipe.__init__(self, buildout, name, options)
+
+  def install(self):
+    config = dict(
+      ip=self.options['ip'],
+      port=self.options['port'],
+      onetimeupload_path=self.options['onetimeupload-path'],
+      shell_path=self.options['shell-path'],
+      log_path=self.options['log-path'],
+      image=self.options['image-path'],
+      key=self.options['key'],
+    )
+
+    # Runners
+    runner_path = self.createExecutable(
+      self.options['path'],
+      self.substituteTemplate(self.getTemplateFilename('onetimeupload_run.in'),
+                              config))
+
+    return [runner_path]
 
-    return [onetimeupload_runner_path, nbdserver_runner_path]

slapos/recipe/nbdserver/template/onetimeupload_run.in → slapos/recipe/generic_onetimeupload/template/onetimeupload_run.in

-#!/bin/sh
+#!%(shell_path)s
 # BEWARE: This file is operated by slapgrid
 # BEWARE: It will be overwritten automatically
 exec %(onetimeupload_path)s -l %(log_path)s %(ip)s %(port)s %(image)s %(key)s 

slapos/recipe/kvm/socket_connection_attempt.py

-import socket
-import sys
-
-def connection_attempt():
-
-  try:
-    hostname, port = sys.argv[1:3]
-  except ValueError:
-    print >> sys.stderr, """Bad command line.
-  Usage: %s hostname|ip port""" % sys.argv[0]
-    sys.exit(1)
-
-  connection_okay = False
-
-  try:
-    s = socket.create_connection((hostname, port))
-    connection_okay = True
-    s.close()
-  except (socket.error, socket.timeout):
-    connection_okay = False
-
-  if not connection_okay:
-    print >> sys.stderr, "%(port)s on %(ip)s isn't listening" % {
-      'port': port, 'ip': hostname
-    }
-    sys.exit(127)

slapos/recipe/kvm/template/kvm_controller_run.in

-#!%(python_path)s
-# BEWARE: This file is operated by slapgrid
-# BEWARE: It will be overwritten automatically
-
-# Echo client program
-import socket
-import time
-
-# Connect to KVM qmp socket
-so = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
-connected = False
-while not connected:
-  try:
-    so.connect('%(socket_path)s')
-  except socket.error:
-    time.sleep(1)
-  else:
-    connected = True
-data = so.recv(1024)
-
-# Enable qmp
-so.send('{ "execute": "qmp_capabilities" }')
-data = so.recv(1024)
-
-# Set VNC password
-so.send('{ "execute": "change", ' \
-        '"arguments": { "device": "vnc", "target": "password", ' \
-        '               "arg": "%(vnc_passwd)s" } }')
-data = so.recv(1024)
-
-# Finish
-so.close()

slapos/recipe/kvm/template/kvm_run.in

-#!/bin/sh
-# BEWARE: This file is operated by slapgrid
-# BEWARE: It will be overwritten automatically
-
-# TODO: -net nic,model=virtio, but OS installer has to provide the virtio_net
-# module
-exec %(qemu_path)s \
-  -net nic,macaddr=%(mac_address)s \
-  -net tap,ifname=%(tap_interface)s,script=no,downscript=no \
-  -smp %(smp_count)s \
-  -m %(ram_size)s \
-  -cdrom nbd:[%(nbd_ip)s]:%(nbd_port)s \
-  -drive file=%(disk_path)s,if=virtio,boot=on \
-  -vnc %(vnc_ip)s:1,ipv4,password \
-  -boot menu=on \
-  -qmp unix:%(socket_path)s,server \
-  -pidfile %(pid_file_path)s

slapos/recipe/kvm/template/port_listening_promise.in

-#!/usr/bin/env sh
-
-"%(check_port_listening_script)s" "%(hostname)s" "%(port)s"
-exit $?

software/kvm-frontend/instance.cfg

-[buildout]
-parts =
-  instance
-
-eggs-directory = ${buildout:eggs-directory}
-develop-eggs-directory = ${buildout:develop-eggs-directory}
-
-[instance]
-node_path = ${buildout:parts-directory}:${npm-modules:location}/node_modules
-recipe = ${instance-recipe:egg}:${instance-recipe:module}
-node_binary = ${nodejs:location}/bin/node
-#npm_binary = ${npm:location}/bin/npm
-openssl_binary = ${openssl:location}/bin/openssl
-dcrond_binary = ${dcron:location}/sbin/crond
-logrotate_binary = ${logrotate:location}/usr/sbin/logrotate
-rdiff_backup_binary = ${buildout:bin-directory}/rdiff-backup
-

software/kvm-frontend/software.cfg

-[buildout]
-extends =
-  ../../component/dcron/buildout.cfg
-  ../../component/logrotate/buildout.cfg
-  ../../component/rdiff-backup/buildout.cfg
-  ../../stack/nodejs.cfg
-
-parts =
-  template
-  nodejs
-  npm
-  http-proxy
-  proxy-by-url
-
-  dcron
-  logrotate
-  rdiff-backup
-
-# Buildoutish
-  eggs
-  instance-recipe-egg
-
-# XXX: Workaround of SlapOS limitation
-# Unzippig of eggs is required, as SlapOS do not yet provide nicely working
-# development / fast switching environment for whole software
-unzip = true
-
-[instance-recipe]
-# Note: In case if specific instantiation recipe is used this is the place to
-# put its name
-egg = slapos.cookbook
-module = kvm_frontend
-
-[instance-recipe-egg]
-recipe = zc.recipe.egg
-python = python2.7
-eggs = ${instance-recipe:egg}
-
-[template]
-# Default template for apache instance.
-recipe = slapos.recipe.template
-url = ${:_profile_base_location_}/instance.cfg
-#md5sum = 7686228221c684fe772d63a5fe581c74
-output = ${buildout:directory}/template.cfg
-mode = 0644
-
-[http-proxy]
-# https://github.com/nodejitsu/node-http-proxy
-recipe = slapos.recipe.build:download-unpacked
-#XXX-Cedric : use upstream when merged
-url = https://nodeload.github.com/desaintmartin/node-http-proxy/zipball/master
-md5sum = 20204d0b29c2cef26e1c91e99eedca6b
-
-[proxy-by-url]
-# https://github.com/dominictarr/proxy-by-url
-recipe = slapos.recipe.build:download-unpacked
-#XXX-Cedric : use upstream when merged
-url = https://nodeload.github.com/desaintmartin/proxy-by-url/zipball/master
-md5sum = f6a7c4f8f01f3049086a1dbde2cd141c
-
-[npm-modules]
-recipe = plone.recipe.command
-destination = ${buildout:parts-directory}/${:_buildout_section_name_}
-location = ${buildout:parts-directory}/${:_buildout_section_name_}
-command =
-  rm -fr ${:destination} &&
-  mkdir -p ${:destination} &&
-  cd ${:destination} &&
-  ${nodejs:location}/bin/node ${npm:location}/bin/npm install colors &&
-  ${nodejs:location}/bin/node ${npm:location}/bin/npm install socket.io &&
-  ${nodejs:location}/bin/node ${npm:location}/bin/npm install socket.io-client &&
-  ${nodejs:location}/bin/node ${npm:location}/bin/npm install optimist
-  ${nodejs:location}/bin/node ${npm:location}/bin/npm install pkginfo
-

software/kvm/instance-frontend.cfg

+#############################
+#
+# Instanciate kvm frontend
+#
+#############################
+[buildout]
+parts =
+  logrotate
+#   logrotate-entry-frontend
+  cron
+  cron-entry-logrotate
+  ca-frontend
+  certificate-authority
+  frontend-promise
+
+eggs-directory = ${buildout:eggs-directory}
+develop-eggs-directory = ${buildout:develop-eggs-directory}
+offline = true
+
+[rootdirectory]
+recipe = slapos.cookbook:mkdirectory
+etc = $${buildout:directory}/etc
+bin = $${buildout:directory}/bin
+srv = $${buildout:directory}/srv
+var = $${buildout:directory}/var
+
+[basedirectory]
+recipe = slapos.cookbook:mkdirectory
+services = $${rootdirectory:etc}/run
+promises = $${rootdirectory:etc}/promise
+nodejs-conf = $${rootdirectory:etc}/nodejs
+run = $${rootdirectory:var}/run
+log = $${rootdirectory:var}/log
+ca-dir = $${rootdirectory:srv}/ssl
+backup = $${rootdirectory:srv}/backup
+
+[directory]
+recipe = slapos.cookbook:mkdirectory
+cron-entries = $${rootdirectory:etc}/cron.d
+crontabs = $${rootdirectory:etc}/crontabs
+cronstamps = $${rootdirectory:etc}/cronstamps
+ca-dir = $${rootdirectory:srv}/ssl
+logrotate-backup = $${basedirectory:backup}/logrotate
+logrotate-entries = $${rootdirectory:etc}/logrotate.d
+
+[frontend-instance]
+recipe = slapos.cookbook:generic.kvm.frontend
+domain = $${ca-frontend:name}
+# port = $${slap-parameter:port}
+ip = $${slap-network-information:global-ipv6}
+port = 4443
+# http-port = 
+# https-port = 
+ssl-key-path = $${ca-frontend:key-file}
+ssl-cert-path = $${ca-frontend:cert-file}
+slave-instance-list = $${slap-parameter:slave_instance_list}
+map-path = $${basedirectory:nodejs-conf}/proxy_table.json
+conf-path = $${basedirectory:nodejs-conf}/kvm-proxy.js
+wrapper-path = $${rootdirectory:bin}/kvm_frontend
+node-binary = ${nodejs:location}/bin/node
+node-env = ${buildout:parts-directory}:${npm-modules:location}/node_modules
+shell-path = ${dash:location}/bin/dash
+
+[frontend-promise]
+recipe = slapos.cookbook:check_port_listening
+path = $${basedirectory:promises}/frontend_promise
+hostname = $${frontend-instance:ip}
+port = $${frontend-instance:port}
+
+[certificate-authority]
+recipe = slapos.cookbook:certificate_authority
+openssl-binary = ${openssl:location}/bin/openssl
+ca-dir = $${basedirectory:ca-dir}
+requests-directory = $${cadirectory:requests}
+wrapper = $${basedirectory:services}/certificate_authority
+ca-private = $${cadirectory:private}
+ca-certs = $${cadirectory:certs}
+ca-newcerts = $${cadirectory:newcerts}
+ca-crl = $${cadirectory:crl}
+
+[cadirectory]
+recipe = slapos.cookbook:mkdirectory
+requests = $${basedirectory:ca-dir}/requests/
+private = $${basedirectory:ca-dir}/private/
+certs = $${basedirectory:ca-dir}/certs/
+newcerts = $${basedirectory:ca-dir}/newcerts/
+crl = $${basedirectory:ca-dir}/crl/
+
+[ca-frontend]
+<= certificate-authority
+recipe = slapos.cookbook:certificate_authority.request
+key-file = $${basedirectory:nodejs-conf}/nodejs.key
+cert-file = $${basedirectory:nodejs-conf}/nodejs.crt
+executable = $${frontend-instance:wrapper-path}
+wrapper = $${basedirectory:services}/nodejs
+# Put domain name
+name = $${slap-parameter:domain}
+
+[cron]
+recipe = slapos.cookbook:cron
+dcrond-binary = ${dcron:location}/sbin/crond
+cron-entries = $${directory:cron-entries}
+crontabs = $${directory:crontabs}
+cronstamps = $${directory:cronstamps}
+catcher = $${cron-simplelogger:wrapper}
+binary = $${basedirectory:services}/crond
+
+[cron-simplelogger]
+recipe = slapos.cookbook:simplelogger
+wrapper = $${rootdirectory:bin}/cron_simplelogger
+log = $${basedirectory:log}/cron.log
+
+[cron-entry-logrotate]
+<= cron
+recipe = slapos.cookbook:cron.d
+name = logrotate
+frequency = 0 0 * * *
+command = $${logrotate:wrapper}
+
+[logrotate]
+recipe = slapos.cookbook:logrotate
+# Binaries
+logrotate-binary = ${logrotate:location}/usr/sbin/logrotate
+gzip-binary = ${gzip:location}/bin/gzip
+gunzip-binary = ${gzip:location}/bin/gunzip
+# Directories
+wrapper = $${rootdirectory:bin}/logrotate
+conf = $${rootdirectory:etc}/logrotate.conf
+logrotate-entries = $${directory:logrotate-entries}
+backup = $${directory:logrotate-backup}
+state-file = $${rootdirectory:srv}/logrotate.status
+
+# [logrotate-entry-frontend]
+# <= logrotate
+# recipe = slapos.cookbook:logrotate.d
+# name = frontend
+# log = $${mariadb-instance:error-log} $${mariadb-instance:slow-query-log}
+# post = $${mariadb-instance:mysql-binary} --no-defaults -B --socket=$${mariadb-instance:socket} -e "FLUSH LOGS"

software/kvm/instance-kvm.cfg

+#############################
+#
+# Instanciate kvm
+#
+#############################
+[buildout]
+parts =
+  request-slave-frontend
+  certificate-authority
+  kvm-promise
+  novnc-promise
+  publish-kvm-connection-information
+
+eggs-directory = ${buildout:eggs-directory}
+develop-eggs-directory = ${buildout:develop-eggs-directory}
+offline = true
+
+[rootdirectory]
+recipe = slapos.cookbook:mkdirectory
+etc = $${buildout:directory}/etc
+bin = $${buildout:directory}/bin
+srv = $${buildout:directory}/srv
+var = $${buildout:directory}/var
+
+[basedirectory]
+recipe = slapos.cookbook:mkdirectory
+services = $${rootdirectory:etc}/run
+promises = $${rootdirectory:etc}/promise
+novnc-conf = $${rootdirectory:etc}/novnc
+run = $${rootdirectory:var}/run
+ca-dir = $${rootdirectory:srv}/ssl
+
+[create-mac]
+recipe = slapos.cookbook:generate.mac
+
+[kvm-instance]
+recipe = slapos.cookbook:generic.kvm
+vnc-ip = $${slap-network-information:local-ipv4}
+vnc-port = 5901
+nbd-ip = $${slap-parameter:nbd_ip}
+nbd-port = $${slap-parameter:nbd_port}
+tap = $${slap-network-information:network-interface}
+disk-path = $${rootdirectory:srv}/virtual.qcow2
+disk-size = 10
+socket-path = $${rootdirectory:var}/qmp_socket
+pid-path = $${basedirectory:run}/pid_file
+smp-count = 1
+ram-size = 1024
+mac-address = $${create-mac:mac-address}
+runner-path = $${basedirectory:services}/kvm
+controller-path = $${basedirectory:services}/kvm_controller
+shell-path = ${dash:location}/bin/dash
+qemu-path = ${kvm:location}/bin/qemu-system-x86_64
+qemu-img-path = ${kvm:location}/bin/qemu-img
+
+[kvm-promise]
+recipe = slapos.cookbook:check_port_listening
+path = $${basedirectory:promises}/vnc_promise
+hostname = $${kvm-instance:vnc-ip}
+port = $${kvm-instance:vnc-port}
+
+[novnc-instance]
+recipe = slapos.cookbook:generic.novnc
+path = $${ca-novnc:executable}
+ip = $${slap-network-information:global-ipv6}
+port = 6080
+vnc-ip = $${kvm-instance:vnc-ip}
+vnc-port = $${kvm-instance:vnc-port}
+novnc-location = ${noVNC:location}
+websockify-path = ${buildout:directory}/bin/websockify
+ssl-key-path = $${ca-novnc:key-file}
+ssl-cert-path = $${ca-novnc:cert-file}
+
+[certificate-authority]
+recipe = slapos.cookbook:certificate_authority
+openssl-binary = ${openssl:location}/bin/openssl
+ca-dir = $${basedirectory:ca-dir}
+requests-directory = $${cadirectory:requests}
+wrapper = $${basedirectory:services}/certificate_authority
+ca-private = $${cadirectory:private}
+ca-certs = $${cadirectory:certs}
+ca-newcerts = $${cadirectory:newcerts}
+ca-crl = $${cadirectory:crl}
+
+[cadirectory]
+recipe = slapos.cookbook:mkdirectory
+requests = $${basedirectory:ca-dir}/requests/
+private = $${basedirectory:ca-dir}/private/
+certs = $${basedirectory:ca-dir}/certs/
+newcerts = $${basedirectory:ca-dir}/newcerts/
+crl = $${basedirectory:ca-dir}/crl/
+
+[ca-novnc]
+<= certificate-authority
+recipe = slapos.cookbook:certificate_authority.request
+key-file = $${basedirectory:novnc-conf}/novnc.key
+cert-file = $${basedirectory:novnc-conf}/novnc.crt
+executable = $${rootdirectory:bin}/novnc
+wrapper = $${basedirectory:services}/websockify
+
+[novnc-promise]
+recipe = slapos.cookbook:check_port_listening
+path = $${basedirectory:promises}/novnc_promise
+hostname = $${novnc-instance:ip}
+port = $${novnc-instance:port}
+
+[kvm-monitor]
+recipe = slapos.cookbook:generic.slapmonitor
+db-path = $${rootdirectory:srv}/slapmonitor_database
+
+[request-common]
+recipe = slapos.cookbook:request
+software-url = $${slap-connection:software-release-url}
+sla = computer_guid
+sla-computer_guid = $${slap-connection:computer-id}
+server-url = $${slap-connection:server-url}
+key-file = $${slap-connection:key-file}
+cert-file = $${slap-connection:cert-file}
+computer-id = $${slap-connection:computer-id}
+partition-id = $${slap-connection:partition-id}
+
+[request-nbd]
+<=request-common
+name = NBD
+software-type = nbd
+return = nbd_url
+
+[request-frontend]
+<=request-common
+name = Frontend
+software-type = frontend
+config = domain
+config-domain = example.org
+
+[request-slave-frontend]
+<=request-common
+name = SlaveFrontend
+software-type = frontend
+slave = true
+return = frontend_url
+
+[publish-kvm-connection-information]
+recipe = slapos.cookbook:publish
+backend_url = https://[$${novnc-instance:ip}]:$${novnc-instance:port}/vnc_auto.html?host=[$${novnc-instance:ip}]&port=$${novnc-instance:port}&encrypt=1
+frontend_url = $${request-slave-frontend:frontend_url}
+vnc_passwd = $${kvm-instance:passwd}

software/kvm/instance-nbd.cfg

+#############################
+#
+# Instanciate nbdserver
+#
+#############################
+[buildout]
+parts =
+  nbd-promise
+  onetimeupload-promise
+  publish-connection-information
+
+eggs-directory = ${buildout:eggs-directory}
+develop-eggs-directory = ${buildout:develop-eggs-directory}
+offline = true
+
+[rootdirectory]
+recipe = slapos.cookbook:mkdirectory
+etc = $${buildout:directory}/etc
+srv = $${buildout:directory}/srv
+log = $${buildout:directory}/log
+
+[basedirectory]
+recipe = slapos.cookbook:mkdirectory
+services = $${rootdirectory:etc}/run
+promises = $${rootdirectory:etc}/promise
+
+[nbd-instance]
+recipe = slapos.cookbook:generic.nbdserver
+ip = $${slap-network-information:global-ipv6}
+port = 1024
+image-path = $${onetimeupload-instance:image-path}
+qemu-path = ${kvm:location}/bin/qemu-nbd
+shell-path = ${dash:location}/bin/dash
+# XXX TODO: Wait for the iso to be uploaded (execute_wait)
+path = $${basedirectory:services}/nbdserver
+
+[nbd-promise]
+recipe = slapos.cookbook:check_port_listening
+path = $${basedirectory:promises}/nbd_promise
+hostname = $${nbd-instance:ip}
+port = $${nbd-instance:port}
+
+[onetimeupload-instance]
+recipe = slapos.cookbook:generic.onetimeupload
+ip = $${slap-network-information:global-ipv6}
+port = 9999
+image-path = $${rootdirectory:srv}/cdrom.iso
+log-path = $${rootdirectory:log}/onetimeupload.log
+shell-path = ${dash:location}/bin/dash
+onetimeupload-path = ${buildout:bin-directory}/onetimeupload
+path = $${basedirectory:services}/onetimeupload
+
+[onetimeupload-promise]
+recipe = slapos.cookbook:check_port_listening
+path = $${basedirectory:promises}/onetimeupload_promise
+hostname = $${onetimeupload-instance:ip}
+port = $${onetimeupload-instance:port}
+
+[publish-connection-information]
+recipe = slapos.cookbook:publish
+nbd_url = nbd://[$${nbd-instance:ip}]:$${nbd-instance:port}
+upload_url = http://[$${onetimeupload-instance:ip}]:$${onetimeupload-instance:port}/
+upload_key = $${onetimeupload-instance:key}

software/kvm/instance.cfg

 [buildout]
 parts =
-  kvminstance
+  switch-softwaretype
 
 eggs-directory = ${buildout:eggs-directory}
 develop-eggs-directory = ${buildout:develop-eggs-directory}
+offline = true
 
-[kvminstance]
-recipe = slapos.cookbook:kvm
-qemu_path = ${kvm:location}/bin/qemu-system-x86_64
-qemu_img_path = ${kvm:location}/bin/qemu-img
-#slapmonitor_path = ${buildout:bin-directory}/slapmonitor
-#slapreport_path = ${buildout:bin-directory}/slapreport
-websockify = ${buildout:directory}/bin/websockify
-noVNC_location = ${noVNC:location}
-openssl_binary = ${openssl:location}/bin/openssl
-rdiff_backup_binary = ${buildout:bin-directory}/rdiff-backup
-dcrond_binary = ${dcron:location}/sbin/crond
+[switch-softwaretype]
+recipe = slapos.cookbook:softwaretype
+default = ${template-kvm:output}
+kvm = ${template-kvm:output}
+nbd = ${template-nbd:output}
+frontend = ${template-frontend:output}
 
-smp_count = 1
-ram_size = 1024
-disk_size = 10
+[slap-connection]
+# part to migrate to new - separated words
+computer-id = $${slap_connection:computer_id}
+partition-id = $${slap_connection:partition_id}
+server-url = $${slap_connection:server_url}
+software-release-url = $${slap_connection:software_release_url}
+key-file = $${slap_connection:key_file}
+cert-file = $${slap_connection:cert_file}

software/kvm/software.cfg

@@ -3,25 +3,38 @@ extensions =
   buildout-versions
   
 extends =
+  ../../component/gzip/buildout.cfg
   ../../component/dcron/buildout.cfg
+  ../../component/logrotate/buildout.cfg
   ../../component/git/buildout.cfg
   ../../component/gnutls/buildout.cfg
   ../../component/libpng/buildout.cfg
   ../../component/libuuid/buildout.cfg
-  ../../component/lxml-python/buildout.cfg
   ../../component/noVNC/buildout.cfg
   ../../component/openssl/buildout.cfg
-  ../../component/rdiff-backup/buildout.cfg
+  ../../component/dash/buildout.cfg
+  ../../stack/nodejs.cfg
   ../../stack/shacache-client.cfg
 
 develop =
   ${:parts-directory}/websockify
+  /opt/slapdev
 
 parts =
   template
+  dash
   kvm
   eggs
   check-local-eggs
+  check-local-eggs2
+  nodejs
+  npm
+  http-proxy
+  proxy-by-url
+  npm-modules
+  dcron
+  logrotate
+  fail
 
 find-links +=
   http://www.nexedi.org/static/packages/source/slapos.buildout/
@@ -41,6 +54,12 @@ allow-hosts =
   psutil.googlecode.com
   www.dabeaz.com
   www.owlfish.com
+  launchpad.net
+
+# XXX: Workaround of SlapOS limitation
+# Unzippig of eggs is required, as SlapOS do not yet provide nicely working
+# development / fast switching environment for whole software
+unzip = true
 
 #XXX-Cedric : Currently, one can only access to KVM using noVNC.
 #             Ideally one should be able to access KVM by using either NoVNC or VNC.
@@ -99,24 +118,89 @@ update-command = ${:command}
 command = grep parts ${buildout:develop-eggs-directory}/websockify.egg-link
 depends = ${eggs:dummy}
 
+[check-local-eggs2]
+recipe = plone.recipe.command
+stop-on-error = true
+update-command = ${:command}
+command = grep slapdev ${buildout:develop-eggs-directory}/slapos.cookbook.egg-link
+depends = ${eggs:dummy}
+
 [eggs]
 recipe = z3c.recipe.scripts
 dummy =
   ${websockify:location}
 eggs =
-  ${lxml-python:egg}
-  slapos.cookbook
+#   ${lxml-python:egg}
   websockify
+  slapos.cookbook
+  slapos.toolbox
+
+[http-proxy]
+# https://github.com/nodejitsu/node-http-proxy
+recipe = slapos.recipe.build:download-unpacked
+#XXX-Cedric : use upstream when merged
+url = https://nodeload.github.com/desaintmartin/node-http-proxy/zipball/master
+md5sum = 20204d0b29c2cef26e1c91e99eedca6b
+
+[proxy-by-url]
+# https://github.com/dominictarr/proxy-by-url
+recipe = slapos.recipe.build:download-unpacked
+#XXX-Cedric : use upstream when merged
+url = https://nodeload.github.com/desaintmartin/proxy-by-url/zipball/master
+md5sum = f6a7c4f8f01f3049086a1dbde2cd141c
+
+[npm-modules]
+recipe = plone.recipe.command
+destination = ${buildout:parts-directory}/${:_buildout_section_name_}
+location = ${buildout:parts-directory}/${:_buildout_section_name_}
+command =
+  rm -fr ${:destination} &&
+  mkdir -p ${:destination} &&
+  cd ${:destination} &&
+  ${nodejs:location}/bin/node ${npm:location}/bin/npm install colors &&
+  ${nodejs:location}/bin/node ${npm:location}/bin/npm install socket.io &&
+  ${nodejs:location}/bin/node ${npm:location}/bin/npm install socket.io-client &&
+  ${nodejs:location}/bin/node ${npm:location}/bin/npm install optimist
+  ${nodejs:location}/bin/node ${npm:location}/bin/npm install pkginfo
+
+
+[template-kvm]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/instance-kvm.cfg
+md5sum = 285cc5ca336cbae23babecd1d5595d8e
+output = ${buildout:directory}/template-kvm.cfg
+mode = 0644
+
+[template-nbd]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/instance-nbd.cfg
+md5sum = 35bb719918248a90167b9c0c6a96ce75
+output = ${buildout:directory}/template-nbd.cfg
+mode = 0644
+
+[template-frontend]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/instance-frontend.cfg
+md5sum = 16a28991ce59f215f11d7cc61906d64e
+output = ${buildout:directory}/template-frontend.cfg
+mode = 0644
 
 [template]
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance.cfg
-md5sum = 298b146e4efce41bfd58b3f85d064ff1
+md5sum = 0a98e34aaec7097a84066c0665e3a49a
 output = ${buildout:directory}/template.cfg
 mode = 0644
 
 [versions]
-zc.buildout = 1.5.3-dev-SlapOS-010
+# Use SlapOS patched zc.buildout
+zc.buildout = 1.6.0-dev-SlapOS-003
+
+[fail]
+recipe = plone.recipe.command
+stop-on-error = true
+command = touch /couscous
+
 
 [networkcache]
 # signature certificates of the following uploaders.

software/nbd/instance.cfg

-[buildout]
-parts =
-  nbdserverinstance
-
-eggs-directory = ${buildout:eggs-directory}
-develop-eggs-directory = ${buildout:develop-eggs-directory} 
-
-[nbdserverinstance]
-recipe = ${instance-recipe:egg}:${instance-recipe:module}
-qemu_path = ${nbdserver:location}/bin/qemu-nbd
-onetimeupload_path = ${buildout:bin-directory}/onetimeupload

software/nbd/software.cfg

-[buildout]
-
-extends =
-  ../../stack/nbd.cfg
-  ../../stack/shacache-client.cfg
-
-parts +=
-  template
-
-[template]
-recipe = slapos.recipe.template
-url = ${:_profile_base_location_}/instance.cfg
-md5sum = 82e948e1c0cb0d5540ef185edeef3ec3
-output = ${buildout:directory}/template.cfg
-mode = 0644
-
-[versions]
-# XXX-CEDRIC Quick and dirty workaround to avoid m2crypto problems.
-# should not be used elsewhere unless for urgent cases. 
-slapos.libnetworkcache = 0.2

stack/nodejs.cfg

 [buildout]
 extends =
-  ../component/lxml-python/buildout.cfg
   ../component/nodejs/buildout.cfg
   ../stack/shacache-client.cfg
 
@@ -23,18 +22,11 @@ allow-hosts =
   www.dabeaz.com
 
 parts =
-#  template
   eggs
-#  instance-recipe-egg
   nodejs
   npm
 
 [eggs]
 recipe = zc.recipe.egg
 eggs =
-  ${lxml-python:egg}
   slapos.cookbook
-
-[versions]
-# Use SlapOS patched zc.buildout
-zc.buildout = 1.6.0-dev-SlapOS-003