Commit 7f077d29 authored by Cédric Le Ninivin's avatar Cédric Le Ninivin

Merge branch 'master' into slaprunner

parents 55b9c53e 7644b0f2
[buildout]
extends =
../readline/buildout.cfg
../gmp/buildout.cfg
../nettle/buildout.cfg
../ncurses/buildout.cfg
../readline/buildout.cfg
../zlib/buildout.cfg
parts = gnutls
......@@ -22,14 +24,13 @@ environment =
LDFLAGS=-lgpg-error -L${gpg-error:location}/lib -Wl,-rpath=${gpg-error:location}/lib
[gnutls]
# XXX-Cedric : update to latest gnutls
recipe = slapos.recipe.cmmi
url = ftp://ftp.gnutls.org/gcrypt/gnutls/v2.8/gnutls-2.8.6.tar.bz2
md5sum = eb0a6d7d3cb9ac684d971c14f9f6d3ba
url = ftp://ftp.gnutls.org/gcrypt/gnutls/v3.2/gnutls-3.2.0.tar.xz
md5sum = e0cba4ddd923420026ff9739b3bc069a
configure-options =
--with-libgcrypt-prefix=${gcrypt:location}
--disable-static
environment =
CPPFLAGS=-I${zlib:location}/include -I${readline:location}/include -I${ncurses:location}/include -I${ncurses:location}/include/ncursesw -I${gcrypt:location}/include -I${gpg-error:location}/include
LDFLAGS=-lgcrypt -L${readline:location}/lib -Wl,-rpath=${readline:location}/lib -L${ncurses:location}/lib -Wl,-rpath=${ncurses:location}/lib -L${gcrypt:location}/lib -Wl,-rpath=${gcrypt:location}/lib -L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib -L${gpg-error:location}/lib -Wl,-rpath=${gpg-error:location}/lib
PKG_CONFIG=${zlib:location}/lib/pkgconfig
CPPFLAGS=-I${zlib:location}/include -I${readline:location}/include -I${ncurses:location}/include -I${ncurses:location}/include/ncursesw -I${gmp:location}/include -I${gcrypt:location}/include -I${gpg-error:location}/include -I${nettle:location}/include
LDFLAGS=-lgcrypt -L${gmp:location}/lib -Wl,-rpath=${gmp:location}/lib -L${readline:location}/lib -Wl,-rpath=${readline:location}/lib -L${ncurses:location}/lib -Wl,-rpath=${ncurses:location}/lib -L${gcrypt:location}/lib -Wl,-rpath=${gcrypt:location}/lib -L${nettle:location}/lib -Wl,-rpath=${nettle:location}/lib -L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib -L${gpg-error:location}/lib -Wl,-rpath=${gpg-error:location}/lib
[buildout]
extends =
../gmp/buildout.cfg
../m4/buildout.cfg
[nettle-lib-location.patch]
recipe = hexagonit.recipe.download
download-only = true
filename = ${:_buildout_section_name_}
url = ${:_profile_base_location_}/${:filename}
md5sum = 41dd0ce2a73487929bdc637b75dd62c9
[nettle]
recipe = slapos.recipe.cmmi
url = http://www.lysator.liu.se/~nisse/archive/nettle-2.7.1.tar.gz
md5sum = 003d5147911317931dd453520eb234a5
patches =
${nettle-lib-location.patch:location}/${nettle-lib-location.patch:filename}
configure-option =
--disable-static
--disable-assembler
--disable-openssl
environment =
PATH=${m4:location}/bin:%(PATH)s
CPPFLAGS=-I${gmp:location}/include
LDFLAGS=-L${gmp:location}/lib -Wl,-rpath=${gmp:location}/lib
--- configure.orig 2013-07-05 15:37:28.000000000 +0200
+++ configure 2013-07-05 15:47:48.000000000 +0200
@@ -4680,52 +4680,6 @@
if test "x$ABI" != xstandard ; then
{ $as_echo "$as_me:${as_lineno-$LINENO}: Compiler uses $ABI-bit ABI. To change, set CC." >&5
$as_echo "$as_me: Compiler uses $ABI-bit ABI. To change, set CC." >&6;}
- if test "$libdir" = '${exec_prefix}/lib' ; then
- # Try setting a better default
- case "$host_cpu:$host_os:$ABI" in
- *:solaris*:32|*:sunos*:32)
- libdir='${exec_prefix}/lib'
- ;;
- *:solaris*:64|*:sunos*:64)
- libdir='${exec_prefix}/lib/64'
- ;;
- # Linux conventions are a mess... According to the Linux File
- # Hierarchy Standard, all architectures except IA64 puts 32-bit
- # libraries in lib, and 64-bit in lib64. Some distributions,
- # e.g., Fedora and Gentoo, adhere to this standard, while at
- # least Debian has decided to put 64-bit libraries in lib and
- # 32-bit libraries in lib32.
-
- # We try to figure out the convention, except if we're cross
- # compiling. We use lib${ABI} if /usr/lib${ABI} exists and
- # appears to not be a symlink to a different name.
- *:linux*:32|*:linux*:64)
- if test "$cross_compiling" = yes ; then
- { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: Cross compiling for linux. Can't guess if libraries go in lib${ABI} or lib." >&5
-$as_echo "$as_me: WARNING: Cross compiling for linux. Can't guess if libraries go in lib${ABI} or lib." >&2;}; else
- # The dash builtin pwd tries to be "helpful" and remember
- # symlink names. Use -P option, and hope it's portable enough.
- test -d /usr/lib${ABI} \
- && (cd /usr/lib${ABI} && pwd -P | grep >/dev/null "/lib${ABI}"'$') \
- && libdir='${exec_prefix}/'"lib${ABI}"
- fi
- ;;
- # On freebsd, it seems 32-bit libraries are in lib32,
- # and 64-bit in lib. Don't know about "kfreebsd", does
- # it follow the Linux fhs conventions?
- *:freebsd*:32)
- libdir='${exec_prefix}/lib32'
- ;;
- *:freebsd*:64)
- libdir='${exec_prefix}/lib'
- ;;
- *)
- { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: Don't know where to install $ABI-bit libraries on this system." >&5
-$as_echo "$as_me: WARNING: Don't know where to install $ABI-bit libraries on this system." >&2;};
- esac
- { $as_echo "$as_me:${as_lineno-$LINENO}: Libraries to be installed in $libdir." >&5
-$as_echo "$as_me: Libraries to be installed in $libdir." >&6;}
- fi
fi
# Select assembler code
[buildout]
extends =
extends =
../../component/gnutls/buildout.cfg
../../component/libpng/buildout.cfg
../../component/libuuid/buildout.cfg
../../component/pkgconfig/buildout.cfg
../../component/xorg/buildout.cfg
../../component/zlib/buildout.cfg
[kvm]
# Backward compatibility
<= qemu-kvm
# XXX Change all reference to kvm section to qemu section, then
# use qemu as main name section.
[qemu]
<= kvm
[qemu-kvm]
[kvm]
recipe = slapos.recipe.cmmi
# qemu-kvm and qemu are now the same since 1.3.
url = http://wiki.qemu-project.org/download/qemu-1.4.1.tar.bz2
md5sum = eb2d696956324722b5ecfa46e41f9a75
url = http://wiki.qemu-project.org/download/qemu-1.5.1.tar.bz2
md5sum = b56e73bdcfdb214d5c68e13111aca96f
depends =
${libpng:so_version}
configure-options =
--target-list=""
--target-list=x86_64-softmmu
--enable-system
--with-system-pixman
--disable-sdl
--disable-xen
--enable-vnc-tls
......@@ -36,4 +40,27 @@ configure-options =
environment =
PATH=${pkgconfig:location}/bin:%(PATH)s
PKG_CONFIG_PATH=${gnutls:location}/lib/pkgconfig:${glib:location}/lib/pkgconfig:${pixman:location}/lib/pkgconfig
LDFLAGS=-L${pixman:location}/lib -Wl,-rpath=${pixman:location}/lib
# The following is only available in buildout2, which we don't use yet.
[kvm-bits64]
configure-options =
--target-list=x86_64-softmmu
${kvm:configure-options}
[kvm-bits32]
configure-options =
--target-list=i386-softmmu
${kvm:configure-options}
[debian-amd64-netinst.iso]
# Download the installer of Debian 7 (Wheezy)
recipe = slapos.recipe.download
url = http://cdimage.debian.org/debian-cd/7.1.0/amd64/iso-cd/debian-7.1.0-amd64-netinst.iso
filename = ${:_buildout_section_name_}
md5sum = 80f498a1f9daa76bc911ae13692e4495
download-only = true
mode = 0644
location = ${buildout:parts-directory}/${:_buildout_section_name_}
......@@ -59,8 +59,6 @@ class Renamer(object):
cp_broken.rename(new_name=broken_new_ref)
cp_broken.stopped()
log.debug("Renaming {}: {}".format(cp_winner.getId(), cp_exporter_ref))
# update name (and later, software type) for the partition that will take over
......
......@@ -37,9 +37,11 @@ class Recipe(GenericSlapRecipe):
"""
def _install(self):
ip = self.options['ip']
backend_url = self.parameter_dict.get(
'backend-url',
self.parameter_dict.get('tidstorage-url') # BBB
backend_url = self.options.get('backend-url',
# BBB: Peeking in partition parameters directly. Eew.
self.parameter_dict.get('backend-url',
self.parameter_dict.get('tidstorage-url') # BBB
)
)
backend_server, backend_port = self._getBackendServer(backend_url)
path_list = []
......
......@@ -61,7 +61,8 @@ class Recipe(GenericBaseRecipe):
shell_path=self.options['shell-path'],
qemu_path=self.options['qemu-path'],
qemu_img_path=self.options['qemu-img-path'],
vnc_passwd=self.options['passwd']
vnc_passwd=self.options['passwd'],
default_disk_image=self.options['default-disk-image'],
)
# Runners
......
......@@ -7,6 +7,9 @@ import os
import socket
import subprocess
# XXX: give all of this through parameter, don't use this as template
default_disk_image = '%(default_disk_image)s'
def getSocketStatus(host, port):
s = None
for res in socket.getaddrinfo(host, port,
......@@ -26,7 +29,8 @@ def getSocketStatus(host, port):
break
return s
# create disk if doesn't exist
# Create disk if doesn't exist
# XXX: move to Buildout profile
disk_path = '%(disk_path)s'
if not os.path.exists(disk_path):
subprocess.Popen(['%(qemu_img_path)s', 'create' ,'-f', 'qcow2',
......@@ -58,4 +62,10 @@ for nbd_ip, nbd_port in (
'-drive',
'file=nbd:[%%s]:%%s,media=cdrom' %% (nbd_ip, nbd_port)])
# If no NBD is specified/available: use internal disk image
else:
kvm_argument_list.extend([
'-drive', 'file=%%s,media=cdrom' %% default_disk_image
])
os.execv('%(qemu_path)s', kvm_argument_list)
......@@ -87,8 +87,6 @@ class Recipe:
computer_partition_id)
self.parameter_dict = self.computer_partition.getInstanceParameterDict()
software_type = self.parameter_dict['slap_software_type']
self.logger.info('Deploying instance with software type %s' % \
software_type)
# Raise if request software_type does not exist ...
if software_type not in self.options:
......
......@@ -34,11 +34,15 @@ class Recipe(GenericSlapRecipe):
"""
def _install(self):
path_list = []
web_checker_mail_address = self.parameter_dict['web-checker-mail-address']
web_checker_smtp_host = self.parameter_dict['web-checker-smtp-host']
web_checker_frontend_url = self.parameter_dict.get(
'web-checker-frontend-url',
self.options['frontend-url'])
try:
web_checker_mail_address = self.options['mail-address']
web_checker_smtp_host = self.options['smtp-host']
web_checker_frontend_url = self.options['frontend-url']
except KeyError:
# BBB
web_checker_mail_address = self.parameter_dict['web-checker-mail-address']
web_checker_smtp_host = self.parameter_dict['web-checker-smtp-host']
web_checker_frontend_url = self.parameter_dict['web-checker-frontend-url']
web_checker_working_directory = \
self.options['web-checker-working-directory']
config = dict(
......
......@@ -17,7 +17,7 @@
"type": "string"
},
"type": "array"
}
},
"backend-count": {
"description": "Number of backend cloudooo instances",
"default": 1,
......
......@@ -9,13 +9,18 @@ extends =
../../component/logrotate/buildout.cfg
../../component/noVNC/buildout.cfg
../../component/openssl/buildout.cfg
../../component/dcron/buildout.cfg
../../stack/nodejs.cfg
../../stack/resilient/buildout.cfg
../../stack/slapos.cfg
parts =
template
eggs
# XXX: we have to manually add this for resilience
rdiff-backup
#XXX-Cedric : Currently, one can only access to KVM using noVNC.
# Ideally one should be able to access KVM by using either NoVNC or VNC.
# Problem is : no native crypto support in web browsers. So we have to disable ssl
......@@ -67,13 +72,58 @@ command =
${nodejs:location}/bin/node ${nodejs:location}/bin/npm install pkginfo@0.2.3
# Create all templates that will be used to deploy instances
[template]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance.cfg.in
md5sum = 0e84223169661462f439c164d62c2a6a
output = ${buildout:directory}/template.cfg
mode = 0644
[template-kvm]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance-kvm.cfg.in
md5sum = 87197471aa93863c310204e8865b5ac1
md5sum = f7c0e2172dac4ee70daae50f38d610ef
output = ${buildout:directory}/template-kvm.cfg
mode = 0644
[template-kvm-resilient]
recipe = hexagonit.recipe.download
url = ${:_profile_base_location_}/instance-kvm-resilient.cfg.jinja
mode = 644
md5sum = 3ee64c654aae503b93b39e9ccd6d3643
[template-kvm-import]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance-kvm-import.cfg.in
md5sum = 7b36d6c61154b7ec3113a1bfaa25a904
output = ${buildout:directory}/template-kvm-import.cfg
mode = 0644
[template-kvm-import-script]
recipe = hexagonit.recipe.download
url = ${:_profile_base_location_}/template/kvm-import.sh.in
filename = kvm-import.sh.in
md5sum = e03ed049cddd8d157228b09e1ebc071a
download-only = true
mode = 0755
[template-kvm-export]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance-kvm-export.cfg.in
md5sum = 64a1a505aff9fde52afac46240811047
output = ${buildout:directory}/template-kvm-export.cfg
mode = 0644
[template-kvm-export-script]
recipe = hexagonit.recipe.download
url = ${:_profile_base_location_}/template/kvm-export.sh.in
filename = kvm-export.sh.in
md5sum = 08cd8da2221f09095b14e35e6acd2a56
download-only = true
mode = 0755
[template-nbd]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance-nbd.cfg.in
......@@ -87,10 +137,3 @@ url = ${:_profile_base_location_}/instance-frontend.cfg.in
md5sum = cdb690495e9eb007d2b7d2f8e12f5c59
output = ${buildout:directory}/template-frontend.cfg
mode = 0644
[template]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance.cfg.in
md5sum = 0a98e34aaec7097a84066c0665e3a49a
output = ${buildout:directory}/template.cfg
mode = 0644
......@@ -31,3 +31,6 @@ update-command = ${:command}
command =
grep parts ${buildout:develop-eggs-directory}/slapos.cookbook.egg-link &&
grep parts ${buildout:develop-eggs-directory}/slapos.toolbox.egg-link
[versions]
websockify = 0.3.0
\ No newline at end of file
[buildout]
extends = ${template-kvm:output}
${pbsready-export:output}
parts +=
cron-entry-backup
certificate-authority
publish-connection-information
kvm-promise
websockify-sighandler
novnc-promise
cron
frontend-promise
# Create the exporter executable, which is a simple shell script
[exporter]
recipe = slapos.recipe.template
url = ${template-kvm-export-script:location}/${template-kvm-export-script:filename}
output = $${directory:bin}/$${slap-parameter:namebase}-exporter
mode = 0755
backup-disk-path = $${directory:backup}/virtual.qcow2
# Resilient stack wants a "wrapper" parameter
wrapper = $${:output}
# Extends publish section with resilient parameters
[publish-connection-information]
<= resilient-publish-connection-parameter
[buildout]
# Here, we don't need KVM to run to import data, so we don't
# even extend the kvm instance profile.
extends = ${pbsready-import:output}
eggs-directory = ${buildout:eggs-directory}
develop-eggs-directory = ${buildout:develop-eggs-directory}
offline = true
[directory]
recipe = slapos.cookbook:mkdirectory
etc = $${buildout:directory}/etc
bin = $${buildout:directory}/bin
srv = $${buildout:directory}/srv
var = $${buildout:directory}/var
log = $${:var}/log
scripts = $${:etc}/run
services = $${:etc}/service
promises = $${:etc}/promise
novnc-conf = $${:etc}/novnc
run = $${:var}/run
ca-dir = $${:srv}/ssl
cron-entries = $${:etc}/cron.d
crontabs = $${:etc}/crontabs
cronstamps = $${:etc}/cronstamps
[importer]
recipe = slapos.recipe.template
url = ${template-kvm-import-script:location}/${template-kvm-import-script:filename}
output = $${directory:bin}/$${slap-parameter:namebase}-importer
mode = 0755
backup-disk-path = $${directory:backup}/virtual.qcow2
disk-path = $${directory:srv}/virtual.qcow2
# Resilient stack wants a "wrapper" parameter
wrapper = $${:output}
backup-disk-path = $${directory:backup}/virtual.qcow2
# vim: set ft=cfg:
{% import 'parts' as parts %}
{% import 'replicated' as replicated with context %}
[buildout]
eggs-directory = {{ eggs_directory }}
develop-eggs-directory = {{ develop_eggs_directory }}
offline = true
# += because we need to take up parts (like instance-custom, slapmonitor etc) from the profile we extended
parts +=
{{ parts.replicate("kvm", "2") }}
publish-connection-informations
{{ replicated.replicate("kvm", "2", "kvm-export", "kvm-import") }}
# Bubble down the parameters of the requested instance to the user
[request-kvm]
# Note: += doesn't work.
return =
# Resilient related parameters
url ssh-public-key ssh-url notification-id ip
# KVM related parameters
backend-url url
[publish-connection-informations]
recipe = slapos.cookbook:publish
backend-url = ${request-kvm:connection-backend-url}
url = ${request-kvm:connection-url}
......@@ -6,10 +6,13 @@
[buildout]
parts =
certificate-authority
publish-kvm-connection-information
publish-connection-information
kvm-promise
websockify-sighandler
novnc-promise
# kvm-monitor
cron
# cron-entry-monitor
frontend-promise
eggs-directory = ${buildout:eggs-directory}
......@@ -22,12 +25,16 @@ etc = $${buildout:directory}/etc
bin = $${buildout:directory}/bin
srv = $${buildout:directory}/srv
var = $${buildout:directory}/var
log = $${:var}/log
scripts = $${:etc}/run
services = $${:etc}/service
promises = $${:etc}/promise
novnc-conf = $${:etc}/novnc
run = $${:var}/run
ca-dir = $${:srv}/ssl
cron-entries = $${:etc}/cron.d
crontabs = $${:etc}/crontabs
cronstamps = $${:etc}/cronstamps
[create-mac]
recipe = slapos.cookbook:generate.mac
......@@ -44,6 +51,7 @@ bytes = 8
recipe = slapos.cookbook:kvm
vnc-ip = $${slap-network-information:local-ipv4}
vnc-port = 5901
default-disk-image = ${debian-amd64-netinst.iso:location}/${debian-amd64-netinst.iso:filename}
nbd-host = $${slap-parameter:nbd-host}
nbd-port = $${slap-parameter:nbd-port}
nbd2-host = $${slap-parameter:nbd2-host}
......@@ -122,11 +130,43 @@ hostname = $${novnc-instance:ip}
port = $${novnc-instance:port}
[kvm-monitor]
recipe = slapos.cookbook:generic.slapmonitor
db-path = $${directory:srv}/slapmonitor_database
#[kvm-monitor]
#recipe = slapos.cookbook:wrapper
#wrapper-path = $${directory:services}/kvm_monitor
#command-line = ${buildout:bin-directory}/kvm.monitor.test
# $${buildout:directory}/buildout-switch-softwaretype.cfg
# $${buildout:directory}/report.xml
# -s slap-parameter
# -opts disk-size ram-size cpu-count
#----------------
#--
#-- Deploy cron.
[cron]
recipe = slapos.cookbook:cron
dcrond-binary = ${dcron:location}/sbin/crond
cron-entries = $${directory:cron-entries}
crontabs = $${directory:crontabs}
cronstamps = $${directory:cronstamps}
catcher = $${cron-simplelogger:wrapper}
binary = $${directory:services}/crond
[cron-simplelogger]
recipe = slapos.cookbook:simplelogger
wrapper = $${directory:bin}/cron_simplelogger
log = $${directory:log}/crond.log
#[cron-entry-monitor]
#<= cron
#recipe = slapos.cookbook:cron.d
#name = kvm_monitor
#frequency = 0 0 * * *
#command = $${kvm-monitor:wrapper-path}
[request-slave-frontend]
recipe = slapos.cookbook:requestoptional
software-url = $${slap-parameter:frontend-software-url}
......@@ -146,17 +186,16 @@ sla = instance_guid
sla-instance_guid = $${slap-parameter:frontend-instance-guid}
[publish-kvm-connection-information]
[publish-connection-information]
recipe = slapos.cookbook:publish
backend-url = https://[$${novnc-instance:ip}]:$${novnc-instance:port}/vnc_auto.html?host=[$${novnc-instance:ip}]&port=$${novnc-instance:port}&encrypt=1
password = $${kvm-instance:passwd}
url = $${request-slave-frontend:connection-url}/vnc_auto.html?host=$${request-slave-frontend:connection-domainname}&port=$${request-slave-frontend:connection-port}&encrypt=1&path=$${request-slave-frontend:connection-resource}
backend-url = https://[$${novnc-instance:ip}]:$${novnc-instance:port}/vnc_auto.html?host=[$${novnc-instance:ip}]&port=$${novnc-instance:port}&encrypt=1&password=$${kvm-instance:passwd}
url = $${request-slave-frontend:connection-url}/vnc_auto.html?host=$${request-slave-frontend:connection-domainname}&port=$${request-slave-frontend:connection-port}&encrypt=1&path=$${request-slave-frontend:connection-resource}&password=$${kvm-instance:passwd}
[frontend-promise]
recipe = slapos.cookbook:check_url_available
path = $${directory:promises}/frontend_promise
url = $${publish-kvm-connection-information:url}
url = $${publish-connection-information:url}
dash_path = ${dash:location}/bin/dash
curl_path = ${curl:location}/bin/curl
......@@ -164,9 +203,9 @@ curl_path = ${curl:location}/bin/curl
# Default values if not specified
frontend-software-type = frontend
frontend-software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/refs/tags/slapos-0.92:/software/kvm/software.cfg
frontend-instance-guid =
nbd-port = 1024
nbd-host = debian.nbd.vifib.net
nbd-host =
nbd2-port = 1024
nbd2-host =
......
......@@ -4,7 +4,6 @@ parts =
eggs-directory = ${buildout:eggs-directory}
develop-eggs-directory = ${buildout:develop-eggs-directory}
offline = true
[switch-softwaretype]
recipe = slapos.cookbook:softwaretype
......@@ -13,11 +12,33 @@ kvm = ${template-kvm:output}
nbd = ${template-nbd:output}
frontend = ${template-frontend:output}
[slap-connection]
# part to migrate to new - separated words
computer-id = $${slap_connection:computer_id}
partition-id = $${slap_connection:partition_id}
server-url = $${slap_connection:server_url}
software-release-url = $${slap_connection:software_release_url}
key-file = $${slap_connection:key_file}
cert-file = $${slap_connection:cert_file}
kvm-resilient = $${dynamic-template-kvm-resilient:rendered}
kvm-import = ${template-kvm-import:output}
kvm-export = ${template-kvm-export:output}
frozen = ${instance-frozen:output}
pull-backup = ${template-pull-backup:output}
[slap-configuration]
recipe = slapos.cookbook:slapconfiguration.serialised
computer = $${slap-connection:computer-id}
partition = $${slap-connection:partition-id}
url = $${slap-connection:server-url}
key = $${slap-connection:key-file}
cert = $${slap-connection:cert-file}
[dynamic-template-kvm-resilient]
recipe = slapos.recipe.template:jinja2
template = ${:_profile_base_location_}/instance-kvm-resilient.cfg.jinja2
md5sum = 1b7a2fcc884649b4d08f238e828899c1
rendered = $${buildout:directory}/template-kvm-resilient.cfg
context = key buildout buildout:bin-directory
key develop_eggs_directory buildout:develop-eggs-directory
key eggs_directory buildout:eggs-directory
key slapparameter_dict slap-configuration:configuration
template-parts-destination = ${template-parts:destination}
template-replicated-destination = ${template-replicated:destination}
import-list = file parts :template-parts-destination
file replicated :template-replicated-destination
mode = 0644
......@@ -47,98 +47,96 @@ signature-certificate-list =
x2IMeSwJ82BpdEI5niXxB+iT0HxhmR+XaMI=
-----END CERTIFICATE-----
[versions]
numpy = 1.6.2
Jinja2 = 2.6
Werkzeug = 0.8.3
apache-libcloud = 0.12.1
Werkzeug = 0.9.3
apache-libcloud = 0.13.0
async = 0.6.1
buildout-versions = 1.7
gitdb = 0.5.4
hexagonit.recipe.cmmi = 1.6
lxml = 3.1.0
itsdangerous = 0.22
lxml = 3.2.3
meld3 = 0.6.10
plone.recipe.command = 1.1
pycrypto = 2.6
slapos.cookbook = 0.73.1
rdiff-backup = 1.0.5
slapos.cookbook = 0.78.3
slapos.recipe.cmmi = 0.2
slapos.recipe.download = 1.0.dev-r4053
slapos.recipe.template = 2.4.2
slapos.toolbox = 0.33.1
slapos.toolbox = 0.35.0
smmap = 0.8.2
websockify = 0.3.0
websockify = 0.5.1
z3c.recipe.scripts = 1.0.1
# Required by:
# slapos.core==0.35.1
# slapos.toolbox==0.33.1
Flask = 0.9
# slapos.toolbox==0.35.0
Flask = 0.10.1
# Required by:
# slapos.toolbox==0.33.1
# slapos.toolbox==0.35.0
GitPython = 0.3.2.RC1
# Required by:
# slapos.toolbox==0.33.1
# slapos.toolbox==0.35.0
atomize = 0.1.1
# Required by:
# slapos.toolbox==0.33.1
# slapos.toolbox==0.35.0
feedparser = 5.1.3
# Required by:
# hexagonit.recipe.cmmi==1.6
hexagonit.recipe.download = 1.6nxd002
# slapos.cookbook==0.78.3
inotifyx = 0.2.0-1
# Required by:
# slapos.cookbook==0.73.1
inotifyx = 0.2.0
# slapos.cookbook==0.78.3
lock-file = 2.0
# Required by:
# slapos.cookbook==0.73.1
# slapos.cookbook==0.78.3
netaddr = 0.7.10
# Required by:
# slapos.core==0.35.1
netifaces = 0.8
netifaces = 0.8-1
# Required by:
# slapos.toolbox==0.33.1
paramiko = 1.10.0
# websockify==0.5.1
numpy = 1.7.1
# Required by:
# slapos.toolbox==0.33.1
psutil = 0.6.1
# slapos.toolbox==0.35.0
paramiko = 1.11.0
# Required by:
# slapos.core==0.35.1
pyflakes = 0.6.1
# slapos.toolbox==0.35.0
psutil = 1.0.1
# Required by:
# slapos.cookbook==0.73.1
pytz = 2012j
# slapos.core==0.35.1
pyflakes = 0.7.3
# Required by:
# slapos.cookbook==0.73.1
# slapos.core==0.35.1
# slapos.toolbox==0.33.1
setuptools = 0.6c12dev-r88846
# slapos.cookbook==0.78.3
pytz = 2013b
# Required by:
# slapos.cookbook==0.73.1
# slapos.toolbox==0.33.1
# slapos.cookbook==0.78.3
# slapos.toolbox==0.35.0
slapos.core = 0.35.1
# Required by:
# slapos.core==0.35.1
supervisor = 3.0b1
supervisor = 3.0b2
# Required by:
# slapos.core==0.35.1
unittest2 = 0.5.1
# Required by:
# slapos.cookbook==0.73.1
# slapos.toolbox==0.33.1
# slapos.cookbook==0.78.3
# slapos.toolbox==0.35.0
xml-marshaller = 0.9.7
# Required by:
......
#!/bin/bash
# Create a backup of the disk image of the virtual machine
QEMU_IMG=${kvm-instance:qemu-img-path}
SNAPSHOT_NAME=$(date +%s)
DISK_PATH=${kvm-instance:disk-path}
BACKUP_PATH=${:backup-disk-path}
if [ ! -f $DISK_PATH ]; then
echo "Nothing to backup, disk image doesn't exist yet."
exit 0;
fi
$QEMU_IMG snapshot -c $SNAPSHOT_NAME $DISK_PATH
if [ -f $BACKUP_PATH ]; then
rm $BACKUP_PATH
fi
$QEMU_IMG convert -f qcow2 -O qcow2 -s $SNAPSHOT_NAME $DISK_PATH $BACKUP_PATH
$QEMU_IMG snapshot -d $SNAPSHOT_NAME $DISK_PATH
#!/bin/bash
DISK_PATH=${:disk-path}
BACKUP_PATH=${:backup-disk-path}
# TODO: Use rdiff
rm $DISK_PATH
cp $BACKUP_PATH $DISK_PATH
{
"$schema": "http://json-schema.org/draft-04/schema#",
"extends": "./schema-definitions.json#",
"properties": {
"tcpv4-port": {
"allOf": [{
"$ref": "#/definitions/tcpv4port"
}, {
"description": "Start allocating ports at this value, going upward",
"default": 6001
}]
},
"backend-url": {
"description": "The backend url that varnish will cache",
"type": "string"
},
"web-checker": {
"description": "Controls automated cache checker, disabled if null or empty"
"properties": {
"frontend-url": {
"description": "Override entry-point-url web checker will check the HTTP headers of all links in the web site, '%(ip)s' and '%(port)s' being substituted with varnish's listening ip and port, respectively",
"default": "http://%(ip)s:%(port)s/",
"type": "string"
},
"mail-address": {
"description": "Email address to which web checker result is sent",
"type": "string"
},
"smtp-host": {
"description": "The smtp server to be used to send the web checker result",
"type": "string"
}
},
"type": "object"
}
}
}
{
"$schema": "http://json-schema.org/draft-04/schema#",
"description": "Values returned by Varnish instanciation",
"properties": {
"url": {
"description": "Varnish HTTP service access information",
"type": "string"
}
},
"type": "object"
}
{% set web_checker_parameter = slapparameter_dict['web-checker-mail-address'] -%}
{% set tcpv4_port = slapparameter_dict.get('tcpv4_port', 6001) -%}
{% set ip = (ipv4_set | list)[0] %}
[buildout]
parts =
publish-varnish-connection-information
varnish-instance
cron
cron-entry-logrotate
{# When web_checker related parameter is given, web_checker will be enabled.#}
{% if web_checker_parameter is defined %}
{# When web_checker related parameter is given, web_checker will be enabled.-#}
{% set web_checker_dict = slapparameter_dict.get('web-checker', {}) -%}
{% if web_checker_dict -%}
web-checker
cron-entry-web-checker
logrotate-entry-web-checker
{% endif %}
[cron-entry-web-checker]
<= cron
recipe = slapos.cookbook:cron.d
name = web-checker
frequency = 0 0 * * *
command = ${varnish-instance:web-checker} ${web-checker:web-checker-config}
[web-checker]
recipe = slapos.cookbook:webchecker
web-checker-config = ${rootdirectory:etc}/web_checker.cfg
web-checker-working-directory = ${directory:web-checker}
frontend-url = {{ web_checker_dict.get('frontend-url', 'http://%(ip)s:%(port)s/') % {
'ip': ip,
'port': tcpv4_port,
} }}
mail-address = {{ web_checker_dict['mail-address'] }}
smtp-host = {{ web_checker_dict['smtp-host'] }}
wget-binary-path = {{ parameter_dict['wget'] }}/bin/wget
varnishlog-binary-path = ${varnish-instance:varnishlog-wrapper}
web-checker-log = ${basedirectory:log}/web-checker.log
[logrotate-entry-web-checker]
<= logrotate
recipe = slapos.cookbook:logrotate.d
name = web-checker
log = ${web-checker:web-checker-log}
frequency = daily
rotate-num = 30
sharedscripts = true
notifempty = true
create = true
{%- endif %}
eggs-directory = {{ eggs_directory }}
develop-eggs-directory = {{ develop_eggs_directory }}
offline = true
[publish-varnish-connection-information]
recipe = slapos.cookbook:publishurl
recipe = slapos.cookbook:publish.serialised
url = http://${varnish-instance:ip}:${varnish-instance:server-port}/
[varnish-instance]
recipe = slapos.cookbook:generic.varnish
backend-url = {{ slapparameter_dict['backend-url'] }}
# Network options
ip = ${slap-network-information:local-ipv4}
server-port = 6001
manager-port = 6002
stunnel-port = 6003
ip = {{ ip }}
server-port = {{ tcpv4_port }}
manager-port = {{ tcpv4_port + 1 }}
stunnel-port = {{ tcpv4_port + 2}}
# Paths: Running wrappers
varnishd-wrapper = ${basedirectory:services}/varnishd
......@@ -50,22 +85,6 @@ varnish-data = ${directory:varnish-data}
varnish-instance-name = ${directory:varnish-instance}
web-checker = {{ parameter_dict['buildout-bin-directory'] }}/web_checker_utility
[cron-entry-web-checker]
<= cron
recipe = slapos.cookbook:cron.d
name = web-checker
frequency = 0 0 * * *
command = ${varnish-instance:web-checker} ${web-checker:web-checker-config}
[web-checker]
recipe = slapos.cookbook:webchecker
web-checker-config = ${rootdirectory:etc}/web_checker.cfg
web-checker-working-directory = ${directory:web-checker}
frontend-url = ${varnish-instance:ip}:${varnish-instance:server-port}
wget-binary-path = {{ parameter_dict['wget'] }}/bin/wget
varnishlog-binary-path = ${varnish-instance:varnishlog-wrapper}
web-checker-log = ${basedirectory:log}/web-checker.log
[cron]
recipe = slapos.cookbook:cron
dcrond-binary = {{ parameter_dict['dcron'] }}/sbin/crond
......@@ -100,17 +119,6 @@ logrotate-entries = ${directory:logrotate-entries}
backup = ${directory:logrotate-backup}
state-file = ${rootdirectory:srv}/logrotate.status
[logrotate-entry-web-checker]
<= logrotate
recipe = slapos.cookbook:logrotate.d
name = web-checker
log = ${web-checker:web-checker-log}
frequency = daily
rotate-num = 30
sharedscripts = true
notifempty = true
create = true
[basedirectory]
recipe = slapos.cookbook:mkdirectory
services = ${rootdirectory:etc}/run
......
......@@ -6,8 +6,8 @@ eggs-directory = {{ eggs_directory }}
develop-eggs-directory = {{ develop_eggs_directory }}
offline = true
[slap-parameters]
recipe = slapos.cookbook:slapconfiguration
[slap-configuration]
recipe = slapos.cookbook:slapconfiguration.serialised
computer = ${slap-connection:computer-id}
partition = ${slap-connection:partition-id}
url = ${slap-connection:server-url}
......@@ -21,7 +21,8 @@ extra-context =
context =
key eggs_directory buildout:eggs-directory
key develop_eggs_directory buildout:develop-eggs-directory
key slapparameter_dict slap-parameters:configuration
key slapparameter_dict slap-configuration:configuration
key ipv4_set slap-configuration:ipv4
${:extra-context}
[dynamic-template-varnish-parameters]
......
......@@ -44,7 +44,7 @@ parts =
[slapos.cookbook-repository]
recipe = slapos.recipe.build:gitclone
repository = http://git.erp5.org/repos/slapos.git
branch = erp5
branch = master
git-executable = ${git:location}/bin/git
[check-recipe]
......@@ -71,7 +71,7 @@ context =
# XXX: "template.cfg" is hardcoded in instanciation recipe
filename = template.cfg
template = ${:_profile_base_location_}/instance.cfg.in
md5sum = c380d2101bfb26b90f84a624809254a0
md5sum = 8e906d749e19ee13fe5b7f4d9bfcf896
extra-context =
key buildout_bin_directory buildout:bin-directory
key dash_location dash:location
......@@ -87,7 +87,7 @@ extra-context =
[template-varnish]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/instance-varnish.cfg.in
md5sum = 6b6d748bf089038234f864417a6182cc
md5sum = 4334d900f212d170fd0ca35865879bdf
mode = 640
[eggs]
......
[buildout]
extends =
../../component/dropbear/buildout.cfg
../../component/gzip/buildout.cfg
../../component/rdiff-backup/buildout.cfg
../../component/rsync/buildout.cfg
parts =
rdiff-backup
......@@ -11,7 +13,6 @@ parts =
template-replicated
template-parts
instance-frozen
template-resilient
# needed tools for resiliency
gzip
......@@ -38,7 +39,7 @@ mode = 0644
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/pbsready-import.cfg.in
output = ${buildout:directory}/pbsready-import.cfg
md5sum = 1b1308fd39476d48b5ca13db48ea6dc9
md5sum = 3c2e73f49abdc52282fc045e6d91f3e9
mode = 0644
[pbsready-export]
......@@ -47,20 +48,20 @@ mode = 0644
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/pbsready-export.cfg.in
output = ${buildout:directory}/pbsready-export.cfg
md5sum = 5d9e20c436fd307e8e4ab224a9a65792
md5sum = a0e22a5de727544c5767d6bee059a77a
mode = 0644
[template-pull-backup]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance-pull-backup.cfg.in
output = ${buildout:directory}/instance-pull-backup.cfg
md5sum = 453d96f5a6c1230c01c878cc7640bae6
md5sum = c67a9dad66490ae264f9e7003521bf59
mode = 0644
[template-replicated]
recipe = slapos.recipe.download
url = ${:_profile_base_location_}/template-replicated.cfg.in
md5sum = 9e20f283bf709c63c9c6692d5e1f8972
#md5sum = 9e20f283bf709c63c9c6692d5e1f8972
mode = 0644
destination = ${buildout:directory}/template-replicated.cfg.in
......@@ -77,12 +78,10 @@ destination = ${buildout:directory}/template-parts.cfg.in
# which will run without removing any content because it raises an error.
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance-frozen.cfg.in
md5sum = d21472f0e58f928fb827f2cbf22c4d4a
output = ${buildout:directory}/instance-frozen.cfg
[template-resilient]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/resilient.cfg.in
output = ${buildout:directory}/resilient.cfg
md5sum = 59e74d290d623de2c1e147e48f284fba
mode = 0644
[versions]
# Pin Jinja2 to 2.6, as 2.7 breaks current code
Jinja2 = 2.6
[buildout]
parts =
\ No newline at end of file
eggs-directory = ${buildout:eggs-directory}
develop-eggs-directory = ${buildout:develop-eggs-directory}
offline = true
parts =
......@@ -105,7 +105,8 @@ promises-directory = $${basedirectory:promises}
directory = $${directory:pbs-backup}
cron-entries = $${cron:cron-entries}
wrappers-directory = $${directory:pbs-wrappers}
notifier-url = http://[$${notifier:host}]:$${notifier:port}/
# XXX: this should be named "notifier-host"
notifier-url = http://[$${notifier:host}]:$${notifier:port}
slave-instance-list = $${slap-parameter:slave_instance_list}
......
......@@ -2,7 +2,21 @@
extends = ${pbsready:output}
parts +=
# Explicitely define extended parts from pbsready
# then add local parts
parts =
resiliency
logrotate
logrotate-entry-cron
logrotate-entry-equeue
cron
cron-entry-logrotate
sshkeys-authority
dropbear-server
sshkeys-dropbear
dropbear-server-pbs-authorized-key
notifier
cron-entry-backup
[resilient-publish-connection-parameter]
......
......@@ -2,7 +2,21 @@
extends = ${pbsready:output}
parts +=
# Explicitely define extended parts from pbsready
# then add local parts
parts =
resiliency
logrotate
logrotate-entry-cron
logrotate-entry-equeue
cron
cron-entry-logrotate
sshkeys-authority
dropbear-server
sshkeys-dropbear
dropbear-server-pbs-authorized-key
notifier
import-on-notification
resilient-publish-connection-parameter
......
## not used at the moment
[buildout]
parts =
request-pull-backup-server
[request-pull-backup-server]
<= slap-connection
recipe = slapos.cookbook:request
name = PBS (Pull Backup Server)
software-url = $${slap-connection:software-release-url}
software-type = pull-backup
return = ssh-key notification-url feeds-url
slave = false
\ No newline at end of file
......@@ -18,12 +18,22 @@ software-type = {{typeexport}}
name = {{namebase}}0
return = ssh-public-key ssh-url notification-id ip
config = number authorized-key notify ip-list namebase
config =
# Resilient related parameters
number authorized-key notify ip-list namebase
{% if slapparameter_dict is defined %}
# Software Instance related parameters
{% for parameter_name in slapparameter_dict.keys() %}{{parameter_name}} {% endfor %}
{% endif %}
config-number = 0
config-authorized-key = {% for id in range(1,nbbackup|int) %} ${request-pbs-{{namebase}}-{{id}}:connection-ssh-key}{% endfor %}
config-notify = {% for id in range(1,nbbackup|int) %} ${request-pbs-{{namebase}}-{{id}}:connection-notification-url}{% endfor %}
config-ip-list =
# Bubble up all the instance parameters to the requested export instance.
{% if slapparameter_dict is defined %}
{% for parameter_name, parameter_value in slapparameter_dict.items() %}config-{{parameter_name}} = {{parameter_value}}
{% endfor %}
{% endif %}
{% for id in range(1,nbbackup|int) %}
......@@ -135,3 +145,4 @@ pbs-{{namebase}}{{id}}-computer-guid =
{% endfor %}
{% endmacro %}
......@@ -78,6 +78,9 @@ zc.buildout = 1.6.0-dev-SlapOS-010
zc.recipe.egg = 1.3.2
# Use own version of h.r.download to be able to open xz-like archives
hexagonit.recipe.download = 1.6nxd002
# Use pinned version of setuptools. Other versions work, but changing
# version makes buildout recompile everything. Developers' nightmare.
setuptools = 0.9.8
[networkcache]
download-cache-url = http://www.shacache.org/shacache
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment