Commit f71776d4 authored by Alain Takoudjou's avatar Alain Takoudjou

Merge branch 'master' into openstack

Conflicts:
	component/qemu-kvm/buildout.cfg
parents bb2ed411 9eeac11b
Changes Changes
======= =======
0.78.4.dev (2013-07-18) 0.80 (2013-08-06)
----------------------- ----------------
* Add a simple readline recipe. [f4fce7e]
0.79 (2013-08-06)
-----------------
* KVM SR: Add support for NAT based networking (User Mode Network). [627895fe35]
* KVM SR: add virtual-hard-drive-url support. [aeb5df40cd, 8ce5a9aa1d0, a5034801aa9]
* Fix regression in GenericBaseRecipe.generatePassword. [3333b07d33c]
0.78.5 (2013-08-06)
-------------------
* check_url_available: add option to check secure links [6cbce4d8231]
0.78.4 (2013-08-06)
-------------------
* slapos.cookbook:slaprunner: Update to use https. [Cedric Le Ninivin]
0.78.3 (2013-07-18) 0.78.3 (2013-07-18)
......
...@@ -44,6 +44,13 @@ filename = cloud9-session-directory.patch ...@@ -44,6 +44,13 @@ filename = cloud9-session-directory.patch
download-only = true download-only = true
md5sum = 5dc8cc28447ed3747b8a53c768d872aa md5sum = 5dc8cc28447ed3747b8a53c768d872aa
[cloud9-socket.patch]
recipe = hexagonit.recipe.download
url = ${:_profile_base_location_}/${:filename}
filename = cloud9-socket.patch
download-only = true
#md5sum = 5dc8cc28447ed3747b8a53c768d872aa
[cloud9-git] [cloud9-git]
# Online IDE written in javascript/node.js # Online IDE written in javascript/node.js
# URL : c9.io # URL : c9.io
...@@ -55,7 +62,7 @@ commit = f7d102bc225c922f116d2cea52a746d64343ea59 ...@@ -55,7 +62,7 @@ commit = f7d102bc225c922f116d2cea52a746d64343ea59
repository = https://github.com/ajaxorg/cloud9.git repository = https://github.com/ajaxorg/cloud9.git
location = ${buildout:parts-directory}/${:_buildout_section_name_} location = ${buildout:parts-directory}/${:_buildout_section_name_}
environment = export GIT_SSL_NO_VERIFY=true; export PATH=${git:location}/bin:${nodejs:location}/bin:${node-sm:location}/node_modules/sm/bin:$PATH; export CPPFLAGS="-I${libxml2:location}/include -I${nodejs:location}/include"; export LDFLAGS="-L${libxml2:location}/lib -Wl,-rpath=${libxml2:location}/lib"; export HOME=${:location}; environment = export GIT_SSL_NO_VERIFY=true; export PATH=${git:location}/bin:${nodejs:location}/bin:${node-sm:location}/node_modules/sm/bin:$PATH; export CPPFLAGS="-I${libxml2:location}/include -I${nodejs:location}/include"; export LDFLAGS="-L${libxml2:location}/lib -Wl,-rpath=${libxml2:location}/lib"; export HOME=${:location};
command = ${:environment} (git clone --quiet ${:repository} ${:location} && cd ${:location} && git reset --hard ${:commit} && ${node-sm:location}/node_modules/.bin/sm install && patch -p1 < ${cloud9-session-directory.patch:location}/${cloud9-session-directory.patch:filename}) || (rm -fr ${:location}; exit 1) command = ${:environment} (git clone --quiet ${:repository} ${:location} && cd ${:location} && git reset --hard ${:commit} && ${node-sm:location}/node_modules/.bin/sm install && patch -p1 < ${cloud9-session-directory.patch:location}/${cloud9-session-directory.patch:filename} && ${node-sm:location}/node_modules/.bin/sm install && patch -p1 < ${cloud9-socket.patch:location}/${cloud9-socket.patch:filename}) || (rm -fr ${:location}; exit 1)
update-command = update-command =
executable = ${:location}/server.js executable = ${:location}/server.js
......
diff --git a/node_modules/smith.io/node_modules/engine.io/node_modules/engine.io-client/dist/engine.io-dev.js b/node_modules/smith.io/node_modules/engine.io/node_modules/engine.io-client/dist/engine.io-dev.js
index fa7e54a..14b8e67 100644
--- a/node_modules/smith.io/node_modules/engine.io/node_modules/engine.io-client/dist/engine.io-dev.js
+++ b/node_modules/smith.io/node_modules/engine.io/node_modules/engine.io-client/dist/engine.io-dev.js
@@ -2126,7 +2126,7 @@ Polling.prototype.uri = function () {
query = '?' + query;
}
- return schema + '://' + this.host + port + this.path + query;
+ return this.path + query;
};
});require.register("transports/websocket.js", function(module, exports, require, global){
[buildout] [buildout]
extends = extends =
../readline/buildout.cfg ../gmp/buildout.cfg
../nettle/buildout.cfg
../ncurses/buildout.cfg ../ncurses/buildout.cfg
../readline/buildout.cfg
../zlib/buildout.cfg ../zlib/buildout.cfg
parts = gnutls parts = gnutls
...@@ -22,14 +24,13 @@ environment = ...@@ -22,14 +24,13 @@ environment =
LDFLAGS=-lgpg-error -L${gpg-error:location}/lib -Wl,-rpath=${gpg-error:location}/lib LDFLAGS=-lgpg-error -L${gpg-error:location}/lib -Wl,-rpath=${gpg-error:location}/lib
[gnutls] [gnutls]
# XXX-Cedric : update to latest gnutls
recipe = slapos.recipe.cmmi recipe = slapos.recipe.cmmi
url = ftp://ftp.gnutls.org/gcrypt/gnutls/v2.8/gnutls-2.8.6.tar.bz2 url = ftp://ftp.gnutls.org/gcrypt/gnutls/v3.2/gnutls-3.2.0.tar.xz
md5sum = eb0a6d7d3cb9ac684d971c14f9f6d3ba md5sum = e0cba4ddd923420026ff9739b3bc069a
configure-options = configure-options =
--with-libgcrypt-prefix=${gcrypt:location} --with-libgcrypt-prefix=${gcrypt:location}
--disable-static --disable-static
environment = environment =
CPPFLAGS=-I${zlib:location}/include -I${readline:location}/include -I${ncurses:location}/include -I${ncurses:location}/include/ncursesw -I${gcrypt:location}/include -I${gpg-error:location}/include CPPFLAGS=-I${zlib:location}/include -I${readline:location}/include -I${ncurses:location}/include -I${ncurses:location}/include/ncursesw -I${gmp:location}/include -I${gcrypt:location}/include -I${gpg-error:location}/include -I${nettle:location}/include
LDFLAGS=-lgcrypt -L${readline:location}/lib -Wl,-rpath=${readline:location}/lib -L${ncurses:location}/lib -Wl,-rpath=${ncurses:location}/lib -L${gcrypt:location}/lib -Wl,-rpath=${gcrypt:location}/lib -L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib -L${gpg-error:location}/lib -Wl,-rpath=${gpg-error:location}/lib LDFLAGS=-lgcrypt -L${gmp:location}/lib -Wl,-rpath=${gmp:location}/lib -L${readline:location}/lib -Wl,-rpath=${readline:location}/lib -L${ncurses:location}/lib -Wl,-rpath=${ncurses:location}/lib -L${gcrypt:location}/lib -Wl,-rpath=${gcrypt:location}/lib -L${nettle:location}/lib -Wl,-rpath=${nettle:location}/lib -L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib -L${gpg-error:location}/lib -Wl,-rpath=${gpg-error:location}/lib
PKG_CONFIG=${zlib:location}/lib/pkgconfig
[buildout]
extends =
../gmp/buildout.cfg
../m4/buildout.cfg
[nettle-lib-location.patch]
recipe = hexagonit.recipe.download
download-only = true
filename = ${:_buildout_section_name_}
url = ${:_profile_base_location_}/${:filename}
md5sum = 41dd0ce2a73487929bdc637b75dd62c9
[nettle]
recipe = slapos.recipe.cmmi
url = http://www.lysator.liu.se/~nisse/archive/nettle-2.7.1.tar.gz
md5sum = 003d5147911317931dd453520eb234a5
patches =
${nettle-lib-location.patch:location}/${nettle-lib-location.patch:filename}
configure-option =
--disable-static
--disable-assembler
--disable-openssl
environment =
PATH=${m4:location}/bin:%(PATH)s
CPPFLAGS=-I${gmp:location}/include
LDFLAGS=-L${gmp:location}/lib -Wl,-rpath=${gmp:location}/lib
--- configure.orig 2013-07-05 15:37:28.000000000 +0200
+++ configure 2013-07-05 15:47:48.000000000 +0200
@@ -4680,52 +4680,6 @@
if test "x$ABI" != xstandard ; then
{ $as_echo "$as_me:${as_lineno-$LINENO}: Compiler uses $ABI-bit ABI. To change, set CC." >&5
$as_echo "$as_me: Compiler uses $ABI-bit ABI. To change, set CC." >&6;}
- if test "$libdir" = '${exec_prefix}/lib' ; then
- # Try setting a better default
- case "$host_cpu:$host_os:$ABI" in
- *:solaris*:32|*:sunos*:32)
- libdir='${exec_prefix}/lib'
- ;;
- *:solaris*:64|*:sunos*:64)
- libdir='${exec_prefix}/lib/64'
- ;;
- # Linux conventions are a mess... According to the Linux File
- # Hierarchy Standard, all architectures except IA64 puts 32-bit
- # libraries in lib, and 64-bit in lib64. Some distributions,
- # e.g., Fedora and Gentoo, adhere to this standard, while at
- # least Debian has decided to put 64-bit libraries in lib and
- # 32-bit libraries in lib32.
-
- # We try to figure out the convention, except if we're cross
- # compiling. We use lib${ABI} if /usr/lib${ABI} exists and
- # appears to not be a symlink to a different name.
- *:linux*:32|*:linux*:64)
- if test "$cross_compiling" = yes ; then
- { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: Cross compiling for linux. Can't guess if libraries go in lib${ABI} or lib." >&5
-$as_echo "$as_me: WARNING: Cross compiling for linux. Can't guess if libraries go in lib${ABI} or lib." >&2;}; else
- # The dash builtin pwd tries to be "helpful" and remember
- # symlink names. Use -P option, and hope it's portable enough.
- test -d /usr/lib${ABI} \
- && (cd /usr/lib${ABI} && pwd -P | grep >/dev/null "/lib${ABI}"'$') \
- && libdir='${exec_prefix}/'"lib${ABI}"
- fi
- ;;
- # On freebsd, it seems 32-bit libraries are in lib32,
- # and 64-bit in lib. Don't know about "kfreebsd", does
- # it follow the Linux fhs conventions?
- *:freebsd*:32)
- libdir='${exec_prefix}/lib32'
- ;;
- *:freebsd*:64)
- libdir='${exec_prefix}/lib'
- ;;
- *)
- { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: Don't know where to install $ABI-bit libraries on this system." >&5
-$as_echo "$as_me: WARNING: Don't know where to install $ABI-bit libraries on this system." >&2;};
- esac
- { $as_echo "$as_me:${as_lineno-$LINENO}: Libraries to be installed in $libdir." >&5
-$as_echo "$as_me: Libraries to be installed in $libdir." >&6;}
- fi
fi
# Select assembler code
...@@ -6,8 +6,9 @@ extends = ...@@ -6,8 +6,9 @@ extends =
[nginx] [nginx]
recipe = slapos.recipe.cmmi recipe = slapos.recipe.cmmi
url = http://nginx.org/download/nginx-1.2.7.tar.gz #url = http://nginx.org/download/nginx-1.2.7.tar.gz
md5sum = d252f5c689a14a668e241c744ccf5f06 url = http://nginx.org/download/nginx-1.4.2.tar.gz
#md5sum = d252f5c689a14a668e241c744ccf5f06
configure-options= configure-options=
--with-ipv6 --with-ipv6
--with-http_ssl_module --with-http_ssl_module
......
...@@ -8,6 +8,8 @@ parts = ...@@ -8,6 +8,8 @@ parts =
[python-openssl] [python-openssl]
recipe = zc.recipe.egg:custom recipe = zc.recipe.egg:custom
egg = pyOpenSSL egg = pyOpenSSL
include-dirs =
${openssl:location}/include/
library-dirs = library-dirs =
${openssl:location}/lib/ ${openssl:location}/lib/
rpath = rpath =
......
...@@ -28,7 +28,7 @@ from setuptools import setup, find_packages ...@@ -28,7 +28,7 @@ from setuptools import setup, find_packages
import glob import glob
import os import os
version = '0.78.4.dev' version = '0.81-dev'
name = 'slapos.cookbook' name = 'slapos.cookbook'
long_description = open("README.txt").read() + "\n" + \ long_description = open("README.txt").read() + "\n" + \
open("CHANGES.txt").read() + "\n" open("CHANGES.txt").read() + "\n"
...@@ -165,8 +165,7 @@ setup(name=name, ...@@ -165,8 +165,7 @@ setup(name=name,
'publish.serialised = slapos.recipe.publish:Serialised', 'publish.serialised = slapos.recipe.publish:Serialised',
'publishsection = slapos.recipe.publish:PublishSection', 'publishsection = slapos.recipe.publish:PublishSection',
'publishurl = slapos.recipe.publishurl:Recipe', 'publishurl = slapos.recipe.publishurl:Recipe',
'pwgen = slapos.recipe.pwgen:Recipe', 'readline = slapos.recipe.readline:Recipe',
'pwgen.stable = slapos.recipe.pwgen:StablePasswordGeneratorRecipe',
'redis.server = slapos.recipe.redis:Recipe', 'redis.server = slapos.recipe.redis:Recipe',
'request = slapos.recipe.request:Recipe', 'request = slapos.recipe.request:Recipe',
'request.serialised = slapos.recipe.request:Serialised', 'request.serialised = slapos.recipe.request:Serialised',
......
...@@ -59,8 +59,6 @@ class Renamer(object): ...@@ -59,8 +59,6 @@ class Renamer(object):
cp_broken.rename(new_name=broken_new_ref) cp_broken.rename(new_name=broken_new_ref)
cp_broken.stopped()
log.debug("Renaming {}: {}".format(cp_winner.getId(), cp_exporter_ref)) log.debug("Renaming {}: {}".format(cp_winner.getId(), cp_exporter_ref))
# update name (and later, software type) for the partition that will take over # update name (and later, software type) for the partition that will take over
......
...@@ -37,6 +37,7 @@ class Recipe(GenericBaseRecipe): ...@@ -37,6 +37,7 @@ class Recipe(GenericBaseRecipe):
'url': self.options['url'], 'url': self.options['url'],
'shell_path': self.options['dash_path'], 'shell_path': self.options['dash_path'],
'curl_path': self.options['curl_path'], 'curl_path': self.options['curl_path'],
'check_secure': self.options.get('check-secure', 0)
} }
# XXX-Cedric in this script, curl won't check certificate # XXX-Cedric in this script, curl won't check certificate
......
...@@ -31,6 +31,13 @@ if [ $CODE -eq 000 ]; then ...@@ -31,6 +31,13 @@ if [ $CODE -eq 000 ]; then
exit 1 exit 1
fi fi
if [ %(check_secure)s -eq 1 ]; then
if [ $CODE -eq 401 ]; then
echo "$URL is protected (returned $CODE)." >&2
exit 0
fi
fi
if ! [ $CODE -eq 200 ]; then if ! [ $CODE -eq 200 ]; then
echo "$URL is not available (returned $CODE)." >&2 echo "$URL is not available (returned $CODE)." >&2
exit 2 exit 2
......
...@@ -26,7 +26,11 @@ ...@@ -26,7 +26,11 @@
############################################################################## ##############################################################################
import os import os
from slapos.recipe.librecipe import GenericBaseRecipe if __name__ == '__main__': # Hack to easily run test below.
GenericBaseRecipe = object
else:
from slapos.recipe.librecipe import GenericBaseRecipe
from zc.buildout import UserError
class Recipe(GenericBaseRecipe): class Recipe(GenericBaseRecipe):
...@@ -51,18 +55,111 @@ class Recipe(GenericBaseRecipe): ...@@ -51,18 +55,111 @@ class Recipe(GenericBaseRecipe):
return [script] return [script]
class Part(GenericBaseRecipe): class Part(GenericBaseRecipe):
def install(self): def install(self):
try:
periodicity = self.options['frequency']
except KeyError:
periodicity = self.options['time']
try:
periodicity = systemd_to_cron(periodicity)
except Exception:
raise UserError("Invalid systemd calendar spec %r" % periodicity)
cron_d = self.options['cron-entries'] cron_d = self.options['cron-entries']
name = self.options['name'] name = self.options['name']
filename = os.path.join(cron_d, name) filename = os.path.join(cron_d, name)
with open(filename, 'w') as part: with open(filename, 'w') as part:
part.write('%(frequency)s %(command)s\n' % { part.write('%s %s\n' % (periodicity, self.options['command']))
'frequency': self.options['frequency'],
'command': self.options['command'],
})
return [filename] return [filename]
day_of_week_dict = dict((name, dow) for dow, name in enumerate(
"sunday monday tuesday wednesday thursday friday saturday".split())
for name in (name, name[:3]))
def systemd_to_cron(spec):
"""Convert from systemd.time(7) calendar spec to crontab spec"""
if spec in ("hourly", "daily", "monthly", "weekly"):
return '@' + spec
if not spec.strip():
raise ValueError
spec = spec.split(' ')
try:
dow = ','.join(sorted('-'.join(str(day_of_week_dict[x.lower()])
for x in x.split('-', 1))
for x in spec[0].split(',')
if x))
del spec[0]
except KeyError:
dow = '*'
day = spec.pop(0) if spec else '*-*'
if spec:
time, = spec
elif ':' in day:
time = day
day = '*-*'
else:
time = '0:0'
day = day.split('-')
time = time.split(':')
if (# years not supported
len(day) > 2 and day.pop(0) != '*' or
# some crons ignore day of month if day of week is given, and dcron
# treats day of month in a way that is not compatible with systemd
dow != '*' != day[1] or
# seconds not supported
len(time) > 2 and int(time.pop())):
raise ValueError
month, day = day
hour, minute = time
spec = minute, hour, day, month, dow
for x, (y, z) in zip(spec, ((0, 60), (0, 24), (1, 31), (1, 12))):
if x != '*':
for x in x.split(','):
x = map(int, x.split('/', 1))
x[0] -= y
if x[0] < 0 or len(x) > 1 and x[0] >= x[1] or z <= sum(x):
raise ValueError
return ' '.join(spec)
def test(self):
def _(systemd, cron):
self.assertEqual(systemd_to_cron(systemd), cron)
_("Sat,Mon-Thu,Sun", "0 0 * * 0,1-4,6")
_("mon,sun *-* 2,1:23", "23 2,1 * * 0,1")
_("Wed, 17:48", "48 17 * * 3")
_("Wed-Sat,Tue 10-* 1:2", "2 1 * 10 2,3-6")
_("*-*-7 0:0:0", "0 0 7 * *")
_("10-15", "0 0 15 10 *")
_("monday *-12-* 17:00", "00 17 * 12 1")
_("12,14,13,12:20,10,30", "20,10,30 12,14,13,12 * * *") # TODO: sort
_("*-1/2-1,3 *:30", "30 * 1,3 1/2 *")
_("03-05 08:05", "05 08 05 03 *")
_("08:05:00", "05 08 * * *")
_("05:40", "40 05 * * *")
_("Sat,Sun 12-* 08:05", "05 08 * 12 0,6")
_("Sat,Sun 08:05", "05 08 * * 0,6")
def _(systemd):
self.assertRaises(Exception, systemd_to_cron, systemd)
_("test")
_("")
_("7")
_("121212:1:2")
_("Wed *-1")
_("08:05:40")
_("2003-03-05")
_("0-1"); _("13-1"); _("6/4-1"); _("5/8-1")
_("1-0"); _("1-32"); _("1-4/3"); _("1-14/18")
_("24:0");_("9/9:0"); _("8/16:0")
_("0:60"); _("0:22/22"); _("0:15/45")
if __name__ == '__main__':
import unittest
unittest.TextTestRunner().run(type('', (unittest.TestCase,), {
'runTest': test})())
...@@ -26,27 +26,69 @@ ...@@ -26,27 +26,69 @@
# #
############################################################################## ##############################################################################
import binascii import errno
import os import os
import random
import string
from slapos.recipe.librecipe import GenericBaseRecipe def generatePassword(length):
return ''.join(random.SystemRandom().sample(string.ascii_lowercase, length))
class Recipe(GenericBaseRecipe): class Recipe(object):
"""Generate a password that is only composed of lowercase letters
This recipe only makes sure that ${:passwd} does not end up in `.installed`
file, which is world-readable by default. So be careful not to spread it
throughout the buildout configuration by referencing it directly: see
recipes like slapos.recipe.template:jinja2 to safely process the password.
Options:
- bytes: password length (default: 8 characters)
- storage-path: plain-text persistent storage for password,
that can only be accessed by the user
(default: ${buildout:parts-directory}/${:_buildout_section_name_})
"""
def __init__(self, buildout, name, options): def __init__(self, buildout, name, options):
if os.path.exists(options['storage-path']): options_get = options.get
open_file = open(options['storage-path'], 'r') try:
options['passwd'] = open_file.read() self.storage_path = options['storage-path']
open_file.close() except KeyError:
self.storage_path = options['storage-path'] = os.path.join(
buildout['buildout']['parts-directory'], name)
try:
with open(self.storage_path) as f:
passwd = f.read()
except IOError, e:
if e.errno != errno.ENOENT:
raise
passwd = None
if not passwd:
passwd = self.generatePassword(int(options_get('bytes', '8')))
self.update = self.install
self.passwd = passwd
# Password must not go into .installed file, for 2 reasons:
# security of course but also to prevent buildout to always reinstall.
options.get = lambda option, *args, **kw: passwd \
if option == 'passwd' else options_get(option, *args, **kw)
if options.get('passwd', '') == '': generatePassword = staticmethod(generatePassword)
options['passwd'] = binascii.hexlify(os.urandom(
int(options.get('bytes', '24'))))
return GenericBaseRecipe.__init__(self, buildout, name, options)
def install(self): def install(self):
with open(self.options['storage-path'], 'w') as fout: if self.storage_path:
fout.write(self.options['passwd']) try:
return [self.options['storage-path']] os.unlink(self.storage_path)
except OSError, e:
if e.errno != errno.ENOENT:
raise
fd = os.open(self.storage_path,
os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0600)
try:
os.write(fd, self.passwd)
finally:
os.close(fd)
return self.storage_path
def update(self):
return ()
...@@ -41,39 +41,40 @@ class Recipe(GenericBaseRecipe): ...@@ -41,39 +41,40 @@ class Recipe(GenericBaseRecipe):
'"virtio" value.' '"virtio" value.'
self.options['disk-type'] = 'virtio' self.options['disk-type'] = 'virtio'
config = dict( self.options['python-path'] = sys.executable
tap_interface=self.options['tap'],
vnc_ip=self.options['vnc-ip'], path_list = []
vnc_port=self.options['vnc-port'],
nbd_ip=self.options['nbd-host'], if not self.isTrueValue(self.options.get('use-tap')):
nbd_port=self.options['nbd-port'], # XXX This could be done using Jinja.
nbd2_ip=self.options.get('nbd2-host', ''), for port in self.options['nat-rules'].split():
nbd2_port=self.options.get('nbd2-port', 1024), ipv6_port = int(port) + 10000
disk_path=self.options['disk-path'], tunnel_path = self.createExecutable(
disk_size=self.options['disk-size'], '%s-%sto%s' % (self.options['6tunnel-wrapper-path'], port, ipv6_port),
disk_type=self.options['disk-type'], self.substituteTemplate(
mac_address=self.options['mac-address'], self.getTemplateFilename('6to4.in'),
smp_count=self.options['smp-count'], {
ram_size=self.options['ram-size'], 'ipv6': self.options['ipv6'],
socket_path=self.options['socket-path'], 'ipv6_port': ipv6_port,
pid_file_path=self.options['pid-path'], 'ipv4': self.options['ipv4'],
python_path=sys.executable, 'ipv4_port': port,
shell_path=self.options['shell-path'], 'shell_path': self.options['shell-path'],
qemu_path=self.options['qemu-path'], '6tunnel_path': self.options['6tunnel-path'],
qemu_img_path=self.options['qemu-img-path'], },
vnc_passwd=self.options['passwd'] ),
) )
path_list.append(tunnel_path)
# Runners
runner_path = self.createExecutable( runner_path = self.createExecutable(
self.options['runner-path'], self.options['runner-path'],
self.substituteTemplate(self.getTemplateFilename('kvm_run.in'), self.substituteTemplate(self.getTemplateFilename('kvm_run.in'),
config)) self.options))
path_list.append(runner_path)
controller_path = self.createExecutable( controller_path = self.createExecutable(
self.options['controller-path'], self.options['controller-path'],
self.substituteTemplate(self.getTemplateFilename('kvm_controller_run.in'), self.substituteTemplate(self.getTemplateFilename('kvm_controller_run.in'),
config)) self.options))
return [runner_path, controller_path] return path_list
#!%(shell_path)s
# BEWARE: This file is operated by slapgrid
# BEWARE: It will be overwritten automatically
exec %(6tunnel_path)s -6 -4 -d -l %(ipv6)s %(ipv6_port)s %(ipv4)s %(ipv4_port)s
#!%(python_path)s #!%(python-path)s
# BEWARE: This file is operated by slapgrid # BEWARE: This file is operated by slapgrid
# BEWARE: It will be overwritten automatically # BEWARE: It will be overwritten automatically
...@@ -6,12 +6,15 @@ ...@@ -6,12 +6,15 @@
import socket import socket
import time import time
socket_path = '%(socket-path)s'
vnc_password = '%(vnc-passwd)s'
# Connect to KVM qmp socket # Connect to KVM qmp socket
so = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) so = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
connected = False connected = False
while not connected: while not connected:
try: try:
so.connect('%(socket_path)s') so.connect(socket_path)
except socket.error: except socket.error:
time.sleep(1) time.sleep(1)
else: else:
...@@ -25,7 +28,7 @@ data = so.recv(1024) ...@@ -25,7 +28,7 @@ data = so.recv(1024)
# Set VNC password # Set VNC password
so.send('{ "execute": "change", ' \ so.send('{ "execute": "change", ' \
'"arguments": { "device": "vnc", "target": "password", ' \ '"arguments": { "device": "vnc", "target": "password", ' \
' "arg": "%(vnc_passwd)s" } }') ' "arg": "' + vnc_password + '" } }')
data = so.recv(1024) data = so.recv(1024)
# Finish # Finish
......
#!%(python_path)s #!%(python-path)s
# BEWARE: This file is operated by slapgrid # BEWARE: This file is operated by slapgrid
# BEWARE: It will be overwritten automatically # BEWARE: It will be overwritten automatically
# Echo client program import hashlib
import os import os
import socket import socket
import subprocess import subprocess
import urllib
# XXX: give all of this through parameter, don't use this as template, but as module
qemu_img_path = '%(qemu-img-path)s'
qemu_path = '%(qemu-path)s'
disk_size = '%(disk-size)s'
disk_type = '%(disk-type)s'
socket_path = '%(socket-path)s'
nbd_list = (('%(nbd-host)s', %(nbd-port)s), ('%(nbd2-host)s', %(nbd2-port)s))
default_disk_image = '%(default-disk-image)s'
disk_path = '%(disk-path)s'
virtual_hard_drive_url = '%(virtual-hard-drive-url)s'.strip()
virtual_hard_drive_md5_url = '%(virtual-hard-drive-md5-url)s'.strip()
nat_rules = '%(nat-rules)s'.strip()
use_tap = '%(use-tap)s'
tap_interface = '%(tap-interface)s'
listen_ip = '%(ipv4)s'
mac_address = '%(mac-address)s'
smp_count = '%(smp-count)s'
ram_size = '%(ram-size)s'
pid_file_path = '%(pid-file-path)s'
def md5Checksum(file_path):
with open(file_path, 'rb') as fh:
m = hashlib.md5()
while True:
data = fh.read(8192)
if not data:
break
m.update(data)
return m.hexdigest()
def getSocketStatus(host, port): def getSocketStatus(host, port):
s = None s = None
...@@ -26,27 +57,44 @@ def getSocketStatus(host, port): ...@@ -26,27 +57,44 @@ def getSocketStatus(host, port):
break break
return s return s
# create disk if doesn't exist # Download existing hard drive if needed at first boot
disk_path = '%(disk_path)s' if not os.path.exists(disk_path) and virtual_hard_drive_url != '':
urllib.urlretrieve(virtual_hard_drive_url, disk_path)
local_md5sum = md5Checksum(disk_path)
md5sum = urllib.urlopen(virtual_hard_drive_md5_url).read().strip()
if local_md5sum != md5sum:
os.remove(disk_path)
raise Exception('MD5 mismatch.')
# Create disk if doesn't exist
# XXX: move to Buildout profile
if not os.path.exists(disk_path): if not os.path.exists(disk_path):
subprocess.Popen(['%(qemu_img_path)s', 'create' ,'-f', 'qcow2', subprocess.Popen([qemu_img_path, 'create' ,'-f', 'qcow2',
disk_path, '%(disk_size)sG']) disk_path, '%%sG' %% disk_size])
kvm_argument_list = ['%(qemu_path)s', # Generate network parameters
'-enable-kvm', '-net', 'nic,macaddr=%(mac_address)s', # XXX: use_tap should be a boolean
'-net', 'tap,ifname=%(tap_interface)s,script=no,downscript=no', if use_tap == 'True':
'-smp', '%(smp_count)s', qemu_network_parameter = 'tap,ifname=%%s,script=no,downscript=no' %% tap_interface
'-m', '%(ram_size)s', else:
'-drive', 'file=%(disk_path)s,if=%(disk_type)s', qemu_network_parameter = 'user,' + ','.join('hostfwd=tcp:%%s:%%s-:%%s' %% (listen_ip, int(port) + 10000, port) for port in nat_rules.split())
'-vnc', '%(vnc_ip)s:1,ipv4,password',
kvm_argument_list = [qemu_path,
'-enable-kvm', '-net', 'nic,macaddr=%%s' %% mac_address,
'-net', qemu_network_parameter,
'-smp', smp_count,
'-m', ram_size,
'-drive', 'file=%%s,if=%%s' %% (disk_path, disk_type),
'-vnc', '%%s:1,ipv4,password' %% listen_ip,
'-boot', 'menu=on', '-boot', 'menu=on',
'-qmp', 'unix:%(socket_path)s,server', '-qmp', 'unix:%%s,server' %% socket_path,
'-pidfile', '%(pid_file_path)s', '-pidfile', pid_file_path,
] ]
# Try to connect to NBD server (and second nbd if defined) # Try to connect to NBD server (and second nbd if defined).
for nbd_ip, nbd_port in ( # If not available, don't even specify it in qemu command line parameters.
('%(nbd_ip)s', %(nbd_port)s), ('%(nbd2_ip)s', %(nbd2_port)s)): # Reason: if qemu starts with unavailable NBD drive, it will just crash.
for nbd_ip, nbd_port in nbd_list:
if nbd_ip and nbd_port: if nbd_ip and nbd_port:
s = getSocketStatus(nbd_ip, nbd_port) s = getSocketStatus(nbd_ip, nbd_port)
if s is None: if s is None:
...@@ -57,5 +105,10 @@ for nbd_ip, nbd_port in ( ...@@ -57,5 +105,10 @@ for nbd_ip, nbd_port in (
kvm_argument_list.extend([ kvm_argument_list.extend([
'-drive', '-drive',
'file=nbd:[%%s]:%%s,media=cdrom' %% (nbd_ip, nbd_port)]) 'file=nbd:[%%s]:%%s,media=cdrom' %% (nbd_ip, nbd_port)])
# If no NBD is specified/available: use internal disk image
else:
kvm_argument_list.extend([
'-drive', 'file=%%s,media=cdrom' %% default_disk_image
])
os.execv('%(qemu_path)s', kvm_argument_list) os.execv(qemu_path, kvm_argument_list)
...@@ -183,17 +183,13 @@ class GenericBaseRecipe(object): ...@@ -183,17 +183,13 @@ class GenericBaseRecipe(object):
'template/%s' % template_name) 'template/%s' % template_name)
def generatePassword(self, len_=32): def generatePassword(self, len_=32):
""" # TODO: Consider having generate.password recipe inherit this class,
The purpose of this method is to generate a password which doesn't change # so that it can be easily inheritable.
from one execution to the next, so the generated password doesn't change # In the long-term, it's probably better that passwords are provided
on each slapgrid-cp execution. # by software requesters, to avoid keeping unhashed secrets in
# partitions when possible.
Currently, it returns a hardcoded password because no decision has been self.logger.warning("GenericBaseRecipe.generatePassword is deprecated."
taken on where a generated password should be kept (so it is generated " Use generate.password recipe instead.")
once only).
"""
# TODO: implement a real password generator which remember the last
# call.
return "insecure" return "insecure"
def isTrueValue(self, value): def isTrueValue(self, value):
...@@ -247,7 +243,8 @@ class GenericBaseRecipe(object): ...@@ -247,7 +243,8 @@ class GenericBaseRecipe(object):
destination = self.location destination = self.location
if os.path.exists(destination): if os.path.exists(destination):
# leftovers from a previous failed attempt, removing it. # leftovers from a previous failed attempt, removing it.
log.warning('Removing already existing directory %s' % destination) self.logger.warning('Removing already existing directory %s',
destination)
shutil.rmtree(destination) shutil.rmtree(destination)
os.mkdir(destination) os.mkdir(destination)
......
# vim: set et sts=2:
############################################################################## ##############################################################################
# #
# Copyright (c) 2010 Vifib SARL and Contributors. All Rights Reserved. # Copyright (c) 2013 Vifib SARL and Contributors. All Rights Reserved.
# #
# WARNING: This program as such is intended to be used by professional # WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential # programmers who take the whole responsibility of assessing all potential
...@@ -24,37 +25,39 @@ ...@@ -24,37 +25,39 @@
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
# #
############################################################################## ##############################################################################
import subprocess
import os
from slapos.recipe.librecipe import GenericBaseRecipe import errno
class Recipe(GenericBaseRecipe): class Recipe(object):
"""Read the first line of a file.
def _options(self, options): As the result has to be provided as an options, it is mandatory that the
if not os.path.exists(self.options['file']): buildout profile fills the file content (if needed) before trying to read it.
password = subprocess.check_output([self.options['pwgen-binary'], '-1']).strip()
with open(self.options['file'], 'w') as password_file:
password_file.write(password)
else:
with open(self.options['file'], 'r') as password_file:
password = password_file.read()
options['password'] = password
def install(self): Options:
os.chmod(self.options['file'], 0600) - storage-path: file to read
return []
class StablePasswordGeneratorRecipe(GenericBaseRecipe): Result set in options:
- readline: first line of the file
""" """
The purpose of this class is to generate a password which doesn't change
from one execution to the next (hence "stable"), so the generated password
doesn't change on each slapgrid-cp execution.
See GenericBaseRecipe.generatePassword . def __init__(self, buildout, name, options):
""" storage_path = options['storage-path']
try:
with open(storage_path) as f:
readline = f.readline()
except IOError, e:
if e.errno != errno.ENOENT:
raise
readline = None
def _options(self, options): self.readline = readline
options['password'] = self.generatePassword() options['readline'] = readline
def install(self):
if self.readline is None:
raise ValueError('Unable to read the file content.')
return ()
update = install = lambda self: [] def update(self):
return ()
...@@ -40,8 +40,8 @@ class Recipe(GenericBaseRecipe): ...@@ -40,8 +40,8 @@ class Recipe(GenericBaseRecipe):
self.partition_amount = options['partition-amount'].strip() self.partition_amount = options['partition-amount'].strip()
self.cloud9_url = options.get('cloud9-url', '').strip() self.cloud9_url = options.get('cloud9-url', '').strip()
self.log_file = os.path.join(options['log_dir'].strip(), 'slaprunner.log') self.log_file = os.path.join(options['log_dir'].strip(), 'slaprunner.log')
# Set slaprunner access URL # Set slaprunner access URL, CLN Beware ipv6 access is made throught nginx
options['access-url'] = 'http://[%s]:%s' % (self.ipv6, self.runner_port) options['access-url'] = 'https://[%s]:%s' % (self.ipv6, self.runner_port)
def install(self): def install(self):
path_list = [] path_list = []
...@@ -62,7 +62,7 @@ class Recipe(GenericBaseRecipe): ...@@ -62,7 +62,7 @@ class Recipe(GenericBaseRecipe):
etc_dir=self.options['etc_dir'], etc_dir=self.options['etc_dir'],
run_dir=self.options['run_dir'], run_dir=self.options['run_dir'],
log_dir=self.options['log_dir'], log_dir=self.options['log_dir'],
runner_host=self.ipv6, runner_host=self.ipv4,
runner_port=self.runner_port, runner_port=self.runner_port,
ipv4_address=self.ipv4, ipv4_address=self.ipv4,
ipv6_address=self.ipv6, ipv6_address=self.ipv6,
...@@ -132,7 +132,7 @@ class Test(GenericBaseRecipe): ...@@ -132,7 +132,7 @@ class Test(GenericBaseRecipe):
etc_dir=self.options['etc_dir'], etc_dir=self.options['etc_dir'],
run_dir=self.options['etc_dir'], run_dir=self.options['etc_dir'],
log_dir=self.workdir, log_dir=self.workdir,
runner_host=self.ipv6, runner_host=self.ipv4,
runner_port=self.runner_port, runner_port=self.runner_port,
ipv4_address=self.ipv4, ipv4_address=self.ipv4,
ipv6_address=self.ipv6, ipv6_address=self.ipv6,
......
...@@ -71,7 +71,7 @@ class ExportRecipe(GenericBaseRecipe): ...@@ -71,7 +71,7 @@ class ExportRecipe(GenericBaseRecipe):
done done
} }
sync_element %(srv-directory)s/runner %(backup-directory)s/runner/ instance project proxy.db softwareLink sync_element %(srv-directory)s/runner %(backup-directory)s/runner/ instance project proxy.db softwareLink
sync_element %(etc-directory)s %(backup-directory)s/etc/ .rcode .project .users ssh sync_element %(etc-directory)s %(backup-directory)s/etc/ .rcode .project .users .htpasswd ssh
if [ -d %(backup-directory)s/runner/software ]; then if [ -d %(backup-directory)s/runner/software ]; then
rm %(backup-directory)s/runner/software/* rm %(backup-directory)s/runner/software/*
fi fi
...@@ -120,7 +120,7 @@ class ImportRecipe(GenericBaseRecipe): ...@@ -120,7 +120,7 @@ class ImportRecipe(GenericBaseRecipe):
done done
} }
restore_element %(backup-directory)s/runner/ %(srv-directory)s/runner instance project proxy.db softwareLink restore_element %(backup-directory)s/runner/ %(srv-directory)s/runner instance project proxy.db softwareLink
restore_element %(backup-directory)s/etc/ %(etc-directory)s .rcode .project .users ssh restore_element %(backup-directory)s/etc/ %(etc-directory)s .rcode .project .users .htpasswd ssh
ifs=$IFS IFS=';' ifs=$IFS IFS=';'
read user pass remaining < %(etc-directory)s/.users read user pass remaining < %(etc-directory)s/.users
IFS=$ifs IFS=$ifs
......
...@@ -87,8 +87,6 @@ class Recipe: ...@@ -87,8 +87,6 @@ class Recipe:
computer_partition_id) computer_partition_id)
self.parameter_dict = self.computer_partition.getInstanceParameterDict() self.parameter_dict = self.computer_partition.getInstanceParameterDict()
software_type = self.parameter_dict['slap_software_type'] software_type = self.parameter_dict['slap_software_type']
self.logger.info('Deploying instance with software type %s' % \
software_type)
# Raise if request software_type does not exist ... # Raise if request software_type does not exist ...
if software_type not in self.options: if software_type not in self.options:
......
...@@ -76,27 +76,21 @@ type = rsa ...@@ -76,27 +76,21 @@ type = rsa
[{{ slave_reference }}-backup-public_key] [{{ slave_reference }}-backup-public_key]
recipe = plone.recipe.command recipe = plone.recipe.command
stop-on-error = true stop-on-error = true
update-command = $${:command}
command = ${coreutils-output:rm} -f $${:key} && ${dropbear-output:keygen} -y -f {{ '$${' ~ slave_reference }}-backup-private_key:key} | ${grep-output:grep} {{ '$${' ~ slave_reference }}-backup-private_key:type} > $${:key} command = ${coreutils-output:rm} -f $${:key} && ${dropbear-output:keygen} -y -f {{ '$${' ~ slave_reference }}-backup-private_key:key} | ${grep-output:grep} {{ '$${' ~ slave_reference }}-backup-private_key:type} > $${:key}
key = {{ '$${' ~ slave_reference }}-backup-private_key:key}.pub key = {{ '$${' ~ slave_reference }}-backup-private_key:key}.pub
location = $${:key} location = $${:key}
# Insert as a beginning part, to ensure that all public keys are generated before trying to publish. This will reduce the number of slapgrid-cp run.
[{{ slave_reference }}-backup-check-public_key] {% do part_list.insert(0, "%s-backup-public_key" % slave_reference) -%}
recipe = plone.recipe.command
stop-on-error = true
update-command = $${:command}
command = grep ssh-{{ '$${' ~ slave_reference }}-backup-private_key:type} {{ '$${' ~ slave_reference }}-backup-public_key:key}
[{{ slave_reference }}-backup-read-public_key] [{{ slave_reference }}-backup-read-public_key]
recipe = slapos.cookbook:generate.password recipe = slapos.cookbook:readline
storage-path = {{ '$${' ~ slave_reference }}-backup-public_key:key} storage-path = {{ '$${' ~ slave_reference }}-backup-public_key:key}
bytes = 8
# Publish slave {{ slave_reference }} information # Publish slave {{ slave_reference }} information
[{{ slave_reference }}-backup-publish] [{{ slave_reference }}-backup-publish]
recipe = slapos.cookbook:publish recipe = slapos.cookbook:publish
-slave-reference = {{ slave_reference }} -slave-reference = {{ slave_reference }}
authorized_key = {{ '$${' ~ slave_reference }}-backup-read-public_key:passwd} authorized_key = {{ '$${' ~ slave_reference }}-backup-read-public_key:readline}
{% do part_list.append("%s-backup-publish" % slave_reference) -%} {% do part_list.append("%s-backup-publish" % slave_reference) -%}
[{{ slave_reference }}-backup-script] [{{ slave_reference }}-backup-script]
...@@ -133,7 +127,6 @@ frequency = {{ frequency }} ...@@ -133,7 +127,6 @@ frequency = {{ frequency }}
# XXX File is never removed # XXX File is never removed
recipe = plone.recipe.command recipe = plone.recipe.command
stop-on-error = true stop-on-error = true
update-command = $${:command}
command = ${coreutils-output:cat} ${template-crontab:output} {{ crontab_line_list_string }} | ${dcron-output:crontab} -c $${directory:crontabs} - command = ${coreutils-output:cat} ${template-crontab:output} {{ crontab_line_list_string }} | ${dcron-output:crontab} -c $${directory:crontabs} -
......
...@@ -197,7 +197,7 @@ mode = 0644 ...@@ -197,7 +197,7 @@ mode = 0644
[template-pullrdiffbackup] [template-pullrdiffbackup]
recipe = slapos.recipe.template recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance-pullrdiffbackup.cfg.in url = ${:_profile_base_location_}/instance-pullrdiffbackup.cfg.in
md5sum = 62c236773dadecac11eb9a47dbca9351 md5sum = 213256229360f57c778308825b161321
output = ${buildout:directory}/template-pullrdiffbackup.cfg output = ${buildout:directory}/template-pullrdiffbackup.cfg
mode = 0644 mode = 0644
...@@ -218,7 +218,7 @@ gunicorn = 17.5 ...@@ -218,7 +218,7 @@ gunicorn = 17.5
itsdangerous = 0.22 itsdangerous = 0.22
meld3 = 0.6.10 meld3 = 0.6.10
plone.recipe.command = 1.1 plone.recipe.command = 1.1
slapos.cookbook = 0.78.3 slapos.cookbook = 0.80
slapos.recipe.build = 0.11.6 slapos.recipe.build = 0.11.6
slapos.recipe.cmmi = 0.1.1 slapos.recipe.cmmi = 0.1.1
slapos.recipe.template = 2.4.2 slapos.recipe.template = 2.4.2
......
...@@ -7,7 +7,6 @@ offline = true ...@@ -7,7 +7,6 @@ offline = true
parts = parts =
connection-dict connection-dict
testnode testnode
pwgen
shell shell
shellinabox shellinabox
certificate-authority certificate-authority
...@@ -16,12 +15,11 @@ parts = ...@@ -16,12 +15,11 @@ parts =
[connection-dict] [connection-dict]
recipe = slapos.cookbook:publish recipe = slapos.cookbook:publish
url = http://[$${shellinabox:ipv6}]:$${shellinabox:port}/ url = http://[$${shellinabox:ipv6}]:$${shellinabox:port}/
password = $${pwgen:password} password = $${pwgen:passwd}
[pwgen] [pwgen]
recipe = slapos.cookbook:pwgen recipe = slapos.cookbook:generate.password
file = $${buildout:directory}/.password storage-path = $${buildout:directory}/.password
pwgen-binary = ${pwgen:location}/bin/pwgen
[testnode] [testnode]
recipe = slapos.cookbook:erp5testnode recipe = slapos.cookbook:erp5testnode
...@@ -82,7 +80,7 @@ port = 8080 ...@@ -82,7 +80,7 @@ port = 8080
shell = $${shell:wrapper} shell = $${shell:wrapper}
wrapper = $${rootdirectory:bin}/shellinaboxd wrapper = $${rootdirectory:bin}/shellinaboxd
shellinabox-binary = ${shellinabox:location}/bin/shellinaboxd shellinabox-binary = ${shellinabox:location}/bin/shellinaboxd
password = $${pwgen:password} password = $${pwgen:passwd}
directory = $${buildout:directory}/ directory = $${buildout:directory}/
login-shell = $${rootdirectory:bin}/login login-shell = $${rootdirectory:bin}/login
certificate-directory = $${directory:shellinabox} certificate-directory = $${directory:shellinabox}
......
...@@ -20,7 +20,6 @@ extends = ...@@ -20,7 +20,6 @@ extends =
../../component/zip/buildout.cfg ../../component/zip/buildout.cfg
../../component/busybox/buildout.cfg ../../component/busybox/buildout.cfg
../../component/shellinabox/buildout.cfg ../../component/shellinabox/buildout.cfg
../../component/pwgen/buildout.cfg
# Local development # Local development
develop = develop =
......
...@@ -13,14 +13,13 @@ parts = ...@@ -13,14 +13,13 @@ parts =
gitdaemon gitdaemon
git-http-backend-cgi git-http-backend-cgi
htpasswd htpasswd
pwgen
git-repos git-repos
[publish] [publish]
recipe = slapos.cookbook:publish recipe = slapos.cookbook:publish
url = http://[$${slap-network-information:global-ipv6}]:$${httpd-conf:port}/ url = http://[$${slap-network-information:global-ipv6}]:$${httpd-conf:port}/
user = $${pwgen:user} user = $${pwgen:user}
password = $${pwgen:password} password = $${pwgen:passwd}
[httpd] [httpd]
recipe = slapos.cookbook:wrapper recipe = slapos.cookbook:wrapper
...@@ -79,14 +78,12 @@ output = $${basedirectory:services}/git-daemon ...@@ -79,14 +78,12 @@ output = $${basedirectory:services}/git-daemon
recipe = collective.recipe.cmd recipe = collective.recipe.cmd
output = $${rootdirectory:etc}/httpd.htpasswd output = $${rootdirectory:etc}/httpd.htpasswd
on_install = true on_install = true
on_udptae = true on_update = true
cmds = cmds =
${apache:location}/bin/htpasswd -cb $${:output} $${pwgen:user} $${pwgen:password} ${apache:location}/bin/htpasswd -cb $${:output} $${pwgen:user} $${pwgen:passwd}
[pwgen] [pwgen]
recipe = slapos.cookbook:pwgen recipe = slapos.cookbook:generate.password
file = $${buildout:directory}/.password
pwgen-binary = ${pwgen:location}/bin/pwgen
user = slapos user = slapos
[rootdirectory] [rootdirectory]
......
...@@ -4,7 +4,6 @@ extends = ...@@ -4,7 +4,6 @@ extends =
../../component/apache/buildout.cfg ../../component/apache/buildout.cfg
../../component/perl/buildout.cfg ../../component/perl/buildout.cfg
../../component/git/buildout.cfg ../../component/git/buildout.cfg
../../component/pwgen/buildout.cfg
../../stack/slapos.cfg ../../stack/slapos.cfg
parts = parts =
......
[buildout] [buildout]
extends = extends =
../../component/6tunnel/buildout.cfg
../../component/curl/buildout.cfg ../../component/curl/buildout.cfg
../../component/dash/buildout.cfg ../../component/dash/buildout.cfg
../../component/dcron/buildout.cfg ../../component/dcron/buildout.cfg
...@@ -9,13 +10,18 @@ extends = ...@@ -9,13 +10,18 @@ extends =
../../component/logrotate/buildout.cfg ../../component/logrotate/buildout.cfg
../../component/noVNC/buildout.cfg ../../component/noVNC/buildout.cfg
../../component/openssl/buildout.cfg ../../component/openssl/buildout.cfg
../../component/dcron/buildout.cfg
../../stack/nodejs.cfg ../../stack/nodejs.cfg
../../stack/resilient/buildout.cfg
../../stack/slapos.cfg ../../stack/slapos.cfg
parts = parts =
template template
eggs eggs
# XXX: we have to manually add this for resilience
rdiff-backup
#XXX-Cedric : Currently, one can only access to KVM using noVNC. #XXX-Cedric : Currently, one can only access to KVM using noVNC.
# Ideally one should be able to access KVM by using either NoVNC or VNC. # Ideally one should be able to access KVM by using either NoVNC or VNC.
# Problem is : no native crypto support in web browsers. So we have to disable ssl # Problem is : no native crypto support in web browsers. So we have to disable ssl
...@@ -67,13 +73,58 @@ command = ...@@ -67,13 +73,58 @@ command =
${nodejs:location}/bin/node ${nodejs:location}/bin/npm install pkginfo@0.2.3 ${nodejs:location}/bin/node ${nodejs:location}/bin/npm install pkginfo@0.2.3
# Create all templates that will be used to deploy instances
[template]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance.cfg.in
md5sum = 0e84223169661462f439c164d62c2a6a
output = ${buildout:directory}/template.cfg
mode = 0644
[template-kvm] [template-kvm]
recipe = slapos.recipe.template recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance-kvm.cfg.in url = ${:_profile_base_location_}/instance-kvm.cfg.in
md5sum = 87197471aa93863c310204e8865b5ac1 #md5sum = c3c888c78bbff334135be9e8ad5885a9
output = ${buildout:directory}/template-kvm.cfg output = ${buildout:directory}/template-kvm.cfg
mode = 0644 mode = 0644
[template-kvm-resilient]
recipe = hexagonit.recipe.download
url = ${:_profile_base_location_}/instance-kvm-resilient.cfg.jinja
mode = 644
md5sum = 3ee64c654aae503b93b39e9ccd6d3643
[template-kvm-import]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance-kvm-import.cfg.in
md5sum = 7b36d6c61154b7ec3113a1bfaa25a904
output = ${buildout:directory}/template-kvm-import.cfg
mode = 0644
[template-kvm-import-script]
recipe = hexagonit.recipe.download
url = ${:_profile_base_location_}/template/kvm-import.sh.in
filename = kvm-import.sh.in
md5sum = e03ed049cddd8d157228b09e1ebc071a
download-only = true
mode = 0755
[template-kvm-export]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance-kvm-export.cfg.in
md5sum = 64a1a505aff9fde52afac46240811047
output = ${buildout:directory}/template-kvm-export.cfg
mode = 0644
[template-kvm-export-script]
recipe = hexagonit.recipe.download
url = ${:_profile_base_location_}/template/kvm-export.sh.in
filename = kvm-export.sh.in
md5sum = 08cd8da2221f09095b14e35e6acd2a56
download-only = true
mode = 0755
[template-nbd] [template-nbd]
recipe = slapos.recipe.template recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance-nbd.cfg.in url = ${:_profile_base_location_}/instance-nbd.cfg.in
...@@ -87,10 +138,3 @@ url = ${:_profile_base_location_}/instance-frontend.cfg.in ...@@ -87,10 +138,3 @@ url = ${:_profile_base_location_}/instance-frontend.cfg.in
md5sum = cdb690495e9eb007d2b7d2f8e12f5c59 md5sum = cdb690495e9eb007d2b7d2f8e12f5c59
output = ${buildout:directory}/template-frontend.cfg output = ${buildout:directory}/template-frontend.cfg
mode = 0644 mode = 0644
[template]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance.cfg.in
md5sum = 0a98e34aaec7097a84066c0665e3a49a
output = ${buildout:directory}/template.cfg
mode = 0644
...@@ -31,3 +31,6 @@ update-command = ${:command} ...@@ -31,3 +31,6 @@ update-command = ${:command}
command = command =
grep parts ${buildout:develop-eggs-directory}/slapos.cookbook.egg-link && grep parts ${buildout:develop-eggs-directory}/slapos.cookbook.egg-link &&
grep parts ${buildout:develop-eggs-directory}/slapos.toolbox.egg-link grep parts ${buildout:develop-eggs-directory}/slapos.toolbox.egg-link
[versions]
websockify = 0.3.0
\ No newline at end of file
[buildout]
extends = ${template-kvm:output}
${pbsready-export:output}
parts +=
cron-entry-backup
certificate-authority
publish-connection-information
kvm-promise
websockify-sighandler
novnc-promise
cron
frontend-promise
# Create the exporter executable, which is a simple shell script
[exporter]
recipe = slapos.recipe.template
url = ${template-kvm-export-script:location}/${template-kvm-export-script:filename}
output = $${directory:bin}/$${slap-parameter:namebase}-exporter
mode = 0755
backup-disk-path = $${directory:backup}/virtual.qcow2
# Resilient stack wants a "wrapper" parameter
wrapper = $${:output}
# Extends publish section with resilient parameters
[publish-connection-information]
<= resilient-publish-connection-parameter
[buildout]
# Here, we don't need KVM to run to import data, so we don't
# even extend the kvm instance profile.
extends = ${pbsready-import:output}
eggs-directory = ${buildout:eggs-directory}
develop-eggs-directory = ${buildout:develop-eggs-directory}
offline = true
[directory]
recipe = slapos.cookbook:mkdirectory
etc = $${buildout:directory}/etc
bin = $${buildout:directory}/bin
srv = $${buildout:directory}/srv
var = $${buildout:directory}/var
log = $${:var}/log
scripts = $${:etc}/run
services = $${:etc}/service
promises = $${:etc}/promise
novnc-conf = $${:etc}/novnc
run = $${:var}/run
ca-dir = $${:srv}/ssl
cron-entries = $${:etc}/cron.d
crontabs = $${:etc}/crontabs
cronstamps = $${:etc}/cronstamps
[importer]
recipe = slapos.recipe.template
url = ${template-kvm-import-script:location}/${template-kvm-import-script:filename}
output = $${directory:bin}/$${slap-parameter:namebase}-importer
mode = 0755
backup-disk-path = $${directory:backup}/virtual.qcow2
disk-path = $${directory:srv}/virtual.qcow2
# Resilient stack wants a "wrapper" parameter
wrapper = $${:output}
backup-disk-path = $${directory:backup}/virtual.qcow2
{ {
"name": "Input Parameters", "type": "object",
"$schema": "http://json-schema.org/draft-04/schema",
"title": "Input Parameters",
"properties": { "properties": {
"ram-size": { "ram-size": {
"title": "RAM size", "title": "RAM size",
...@@ -7,7 +10,7 @@ ...@@ -7,7 +10,7 @@
"type": "integer", "type": "integer",
"default": 1024, "default": 1024,
"minimum": 128, "minimum": 128,
"divisibleBy": 128, "multipleOf": 128,
"maximum": 16384 "maximum": 16384
}, },
"disk-size": { "disk-size": {
...@@ -34,7 +37,6 @@ ...@@ -34,7 +37,6 @@
"maximum": 8 "maximum": 8
}, },
"nbd-host": { "nbd-host": {
"title": "NBD hostname", "title": "NBD hostname",
"description": "hostname (or IP) of the NBD server containing the boot image.", "description": "hostname (or IP) of the NBD server containing the boot image.",
...@@ -65,6 +67,25 @@ ...@@ -65,6 +67,25 @@
"maximum": 65535 "maximum": 65535
}, },
"virtual-hard-drive-url": {
"title": "Existing disk image URL",
"description": "If specified, will download an existing disk image (qcow2, raw, ...), and will use it as main virtual hard drive. Can be used to download and use an already installed and customized virtual hard drive.",
"format": "uri",
"type": "string",
},
"use-tap": {
"title": "Use QEMU TAP network interface",
"description": "Use QEMU TAP network interface, requires a bridge on SlapOS Node. If false, use user-mode network stack (NAT).",
"type": "boolean",
"default": false
},
"nat-rules": {
"title": "List of rules for NAT of QEMU user mode network stack.",
"description": "List of rules for NAT of QEMU user mode network stack, as comma-separated list of ports. For each port specified, it will redirect port x of the VM (example: 80) to the port x + 10000 of the public IPv6 (example: 10080). Defaults to \"22 80 443\". Ignored if \"use-tap\" parameter is enabled.",
"type": "string",
},
"frontend-instance-guid": { "frontend-instance-guid": {
"title": "Frontend Instance ID", "title": "Frontend Instance ID",
......
...@@ -13,13 +13,6 @@ ...@@ -13,13 +13,6 @@
"description": "URL used to connect to the service.", "description": "URL used to connect to the service.",
"type": "uri", "type": "uri",
"required": false "required": false
},
"password": {
"title": "Password",
"description": "Password used to authenticate in the service webpage.",
"type": "uri",
"required": true
} }
} }
} }
# vim: set ft=cfg:
{% import 'parts' as parts %}
{% import 'replicated' as replicated with context %}
[buildout]
eggs-directory = {{ eggs_directory }}
develop-eggs-directory = {{ develop_eggs_directory }}
offline = true
# += because we need to take up parts (like instance-custom, slapmonitor etc) from the profile we extended
parts +=
{{ parts.replicate("kvm", "2") }}
publish-connection-informations
{{ replicated.replicate("kvm", "2", "kvm-export", "kvm-import") }}
# Bubble down the parameters of the requested instance to the user
[request-kvm]
# Note: += doesn't work.
return =
# Resilient related parameters
url ssh-public-key ssh-url notification-id ip
# KVM related parameters
backend-url url
[publish-connection-informations]
recipe = slapos.cookbook:publish
backend-url = ${request-kvm:connection-backend-url}
url = ${request-kvm:connection-url}
...@@ -6,10 +6,13 @@ ...@@ -6,10 +6,13 @@
[buildout] [buildout]
parts = parts =
certificate-authority certificate-authority
publish-kvm-connection-information publish-connection-information
kvm-promise kvm-promise
websockify-sighandler websockify-sighandler
novnc-promise novnc-promise
# kvm-monitor
cron
# cron-entry-monitor
frontend-promise frontend-promise
eggs-directory = ${buildout:eggs-directory} eggs-directory = ${buildout:eggs-directory}
...@@ -22,12 +25,16 @@ etc = $${buildout:directory}/etc ...@@ -22,12 +25,16 @@ etc = $${buildout:directory}/etc
bin = $${buildout:directory}/bin bin = $${buildout:directory}/bin
srv = $${buildout:directory}/srv srv = $${buildout:directory}/srv
var = $${buildout:directory}/var var = $${buildout:directory}/var
log = $${:var}/log
scripts = $${:etc}/run scripts = $${:etc}/run
services = $${:etc}/service services = $${:etc}/service
promises = $${:etc}/promise promises = $${:etc}/promise
novnc-conf = $${:etc}/novnc novnc-conf = $${:etc}/novnc
run = $${:var}/run run = $${:var}/run
ca-dir = $${:srv}/ssl ca-dir = $${:srv}/ssl
cron-entries = $${:etc}/cron.d
crontabs = $${:etc}/crontabs
cronstamps = $${:etc}/cronstamps
[create-mac] [create-mac]
recipe = slapos.cookbook:generate.mac recipe = slapos.cookbook:generate.mac
...@@ -38,31 +45,56 @@ recipe = slapos.cookbook:generate.password ...@@ -38,31 +45,56 @@ recipe = slapos.cookbook:generate.password
storage-path = $${directory:srv}/passwd storage-path = $${directory:srv}/passwd
bytes = 8 bytes = 8
[kvm-instance] [kvm-instance]
# XXX-Cedric: change "KVM" recipe to simple "create wrappers". No need for this # XXX-Cedric: change "KVM" recipe to simple "create wrappers". No need for this
# Specific code # Specific code. It needs Jinja.
recipe = slapos.cookbook:kvm recipe = slapos.cookbook:kvm
vnc-ip = $${slap-network-information:local-ipv4}
vnc-passwd = $${gen-passwd:passwd}
ipv4 = $${slap-network-information:local-ipv4}
ipv6 = $${slap-network-information:global-ipv6}
vnc-ip = $${:ipv4}
vnc-port = 5901 vnc-port = 5901
# XXX-Cedric: should be named "default-cdrom-iso"
default-disk-image = ${debian-amd64-netinst.iso:location}/${debian-amd64-netinst.iso:filename}
nbd-host = $${slap-parameter:nbd-host} nbd-host = $${slap-parameter:nbd-host}
nbd-port = $${slap-parameter:nbd-port} nbd-port = $${slap-parameter:nbd-port}
nbd2-host = $${slap-parameter:nbd2-host} nbd2-host = $${slap-parameter:nbd2-host}
nbd2-port = $${slap-parameter:nbd2-port} nbd2-port = $${slap-parameter:nbd2-port}
tap = $${slap-network-information:network-interface}
tap-interface = $${slap-network-information:network-interface}
disk-path = $${directory:srv}/virtual.qcow2 disk-path = $${directory:srv}/virtual.qcow2
disk-size = $${slap-parameter:disk-size} disk-size = $${slap-parameter:disk-size}
disk-type = $${slap-parameter:disk-type} disk-type = $${slap-parameter:disk-type}
socket-path = $${directory:var}/qmp_socket socket-path = $${directory:var}/qmp_socket
pid-path = $${directory:run}/pid_file pid-file-path = $${directory:run}/pid_file
smp-count = $${slap-parameter:cpu-count} smp-count = $${slap-parameter:cpu-count}
ram-size = $${slap-parameter:ram-size} ram-size = $${slap-parameter:ram-size}
mac-address = $${create-mac:mac-address} mac-address = $${create-mac:mac-address}
# XXX-Cedric: should be named runner-wrapper-path and controller-wrapper-path
runner-path = $${directory:services}/kvm runner-path = $${directory:services}/kvm
controller-path = $${directory:scripts}/kvm_controller controller-path = $${directory:scripts}/kvm_controller
use-tap = $${slap-parameter:use-tap}
nat-rules = $${slap-parameter:nat-rules}
6tunnel-wrapper-path = $${directory:services}/6tunnel
virtual-hard-drive-url = $${slap-parameter:virtual-hard-drive-url}
virtual-hard-drive-md5-url = $${slap-parameter:virtual-hard-drive-md5-url}
shell-path = ${dash:location}/bin/dash shell-path = ${dash:location}/bin/dash
qemu-path = ${kvm:location}/bin/qemu-system-x86_64 qemu-path = ${kvm:location}/bin/qemu-system-x86_64
qemu-img-path = ${kvm:location}/bin/qemu-img qemu-img-path = ${kvm:location}/bin/qemu-img
passwd = $${gen-passwd:passwd} 6tunnel-path = ${6tunnel:location}/bin/6tunnel
[kvm-promise] [kvm-promise]
recipe = slapos.cookbook:check_port_listening recipe = slapos.cookbook:check_port_listening
...@@ -122,10 +154,42 @@ hostname = $${novnc-instance:ip} ...@@ -122,10 +154,42 @@ hostname = $${novnc-instance:ip}
port = $${novnc-instance:port} port = $${novnc-instance:port}
[kvm-monitor] #[kvm-monitor]
recipe = slapos.cookbook:generic.slapmonitor #recipe = slapos.cookbook:wrapper
db-path = $${directory:srv}/slapmonitor_database #wrapper-path = $${directory:services}/kvm_monitor
#command-line = ${buildout:bin-directory}/kvm.monitor.test
# $${buildout:directory}/buildout-switch-softwaretype.cfg
# $${buildout:directory}/report.xml
# -s slap-parameter
# -opts disk-size ram-size cpu-count
#----------------
#--
#-- Deploy cron.
[cron]
recipe = slapos.cookbook:cron
dcrond-binary = ${dcron:location}/sbin/crond
cron-entries = $${directory:cron-entries}
crontabs = $${directory:crontabs}
cronstamps = $${directory:cronstamps}
catcher = $${cron-simplelogger:wrapper}
binary = $${directory:services}/crond
[cron-simplelogger]
recipe = slapos.cookbook:simplelogger
wrapper = $${directory:bin}/cron_simplelogger
log = $${directory:log}/crond.log
#[cron-entry-monitor]
#<= cron
#recipe = slapos.cookbook:cron.d
#name = kvm_monitor
#frequency = 0 0 * * *
#command = $${kvm-monitor:wrapper-path}
[request-slave-frontend] [request-slave-frontend]
recipe = slapos.cookbook:requestoptional recipe = slapos.cookbook:requestoptional
...@@ -146,17 +210,16 @@ sla = instance_guid ...@@ -146,17 +210,16 @@ sla = instance_guid
sla-instance_guid = $${slap-parameter:frontend-instance-guid} sla-instance_guid = $${slap-parameter:frontend-instance-guid}
[publish-kvm-connection-information] [publish-connection-information]
recipe = slapos.cookbook:publish recipe = slapos.cookbook:publish
backend-url = https://[$${novnc-instance:ip}]:$${novnc-instance:port}/vnc_auto.html?host=[$${novnc-instance:ip}]&port=$${novnc-instance:port}&encrypt=1 backend-url = https://[$${novnc-instance:ip}]:$${novnc-instance:port}/vnc_auto.html?host=[$${novnc-instance:ip}]&port=$${novnc-instance:port}&encrypt=1&password=$${kvm-instance:vnc-passwd}
password = $${kvm-instance:passwd} url = $${request-slave-frontend:connection-url}/vnc_auto.html?host=$${request-slave-frontend:connection-domainname}&port=$${request-slave-frontend:connection-port}&encrypt=1&path=$${request-slave-frontend:connection-resource}&password=$${kvm-instance:vnc-passwd}
url = $${request-slave-frontend:connection-url}/vnc_auto.html?host=$${request-slave-frontend:connection-domainname}&port=$${request-slave-frontend:connection-port}&encrypt=1&path=$${request-slave-frontend:connection-resource}
[frontend-promise] [frontend-promise]
recipe = slapos.cookbook:check_url_available recipe = slapos.cookbook:check_url_available
path = $${directory:promises}/frontend_promise path = $${directory:promises}/frontend_promise
url = $${publish-kvm-connection-information:url} url = $${publish-connection-information:url}
dash_path = ${dash:location}/bin/dash dash_path = ${dash:location}/bin/dash
curl_path = ${curl:location}/bin/curl curl_path = ${curl:location}/bin/curl
...@@ -164,9 +227,9 @@ curl_path = ${curl:location}/bin/curl ...@@ -164,9 +227,9 @@ curl_path = ${curl:location}/bin/curl
# Default values if not specified # Default values if not specified
frontend-software-type = frontend frontend-software-type = frontend
frontend-software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/refs/tags/slapos-0.92:/software/kvm/software.cfg frontend-software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/refs/tags/slapos-0.92:/software/kvm/software.cfg
frontend-instance-guid =
nbd-port = 1024 nbd-port = 1024
nbd-host = debian.nbd.vifib.net nbd-host =
nbd2-port = 1024 nbd2-port = 1024
nbd2-host = nbd2-host =
...@@ -175,3 +238,9 @@ disk-size = 10 ...@@ -175,3 +238,9 @@ disk-size = 10
disk-type = virtio disk-type = virtio
cpu-count = 1 cpu-count = 1
nat-rules = 22 80 443
use-tap = False
virtual-hard-drive-url =
virtual-hard-drive-md5-url =
...@@ -4,7 +4,6 @@ parts = ...@@ -4,7 +4,6 @@ parts =
eggs-directory = ${buildout:eggs-directory} eggs-directory = ${buildout:eggs-directory}
develop-eggs-directory = ${buildout:develop-eggs-directory} develop-eggs-directory = ${buildout:develop-eggs-directory}
offline = true
[switch-softwaretype] [switch-softwaretype]
recipe = slapos.cookbook:softwaretype recipe = slapos.cookbook:softwaretype
...@@ -13,11 +12,33 @@ kvm = ${template-kvm:output} ...@@ -13,11 +12,33 @@ kvm = ${template-kvm:output}
nbd = ${template-nbd:output} nbd = ${template-nbd:output}
frontend = ${template-frontend:output} frontend = ${template-frontend:output}
[slap-connection] kvm-resilient = $${dynamic-template-kvm-resilient:rendered}
# part to migrate to new - separated words kvm-import = ${template-kvm-import:output}
computer-id = $${slap_connection:computer_id} kvm-export = ${template-kvm-export:output}
partition-id = $${slap_connection:partition_id}
server-url = $${slap_connection:server_url} frozen = ${instance-frozen:output}
software-release-url = $${slap_connection:software_release_url} pull-backup = ${template-pull-backup:output}
key-file = $${slap_connection:key_file}
cert-file = $${slap_connection:cert_file} [slap-configuration]
recipe = slapos.cookbook:slapconfiguration.serialised
computer = $${slap-connection:computer-id}
partition = $${slap-connection:partition-id}
url = $${slap-connection:server-url}
key = $${slap-connection:key-file}
cert = $${slap-connection:cert-file}
[dynamic-template-kvm-resilient]
recipe = slapos.recipe.template:jinja2
template = ${:_profile_base_location_}/instance-kvm-resilient.cfg.jinja2
md5sum = 1b7a2fcc884649b4d08f238e828899c1
rendered = $${buildout:directory}/template-kvm-resilient.cfg
context = key buildout buildout:bin-directory
key develop_eggs_directory buildout:develop-eggs-directory
key eggs_directory buildout:eggs-directory
key slapparameter_dict slap-configuration:configuration
template-parts-destination = ${template-parts:destination}
template-replicated-destination = ${template-replicated:destination}
import-list = file parts :template-parts-destination
file replicated :template-replicated-destination
mode = 0644
...@@ -47,98 +47,96 @@ signature-certificate-list = ...@@ -47,98 +47,96 @@ signature-certificate-list =
x2IMeSwJ82BpdEI5niXxB+iT0HxhmR+XaMI= x2IMeSwJ82BpdEI5niXxB+iT0HxhmR+XaMI=
-----END CERTIFICATE----- -----END CERTIFICATE-----
[versions] [versions]
numpy = 1.6.2 Werkzeug = 0.9.3
Jinja2 = 2.6 apache-libcloud = 0.13.0
Werkzeug = 0.8.3
apache-libcloud = 0.12.1
async = 0.6.1 async = 0.6.1
buildout-versions = 1.7 buildout-versions = 1.7
gitdb = 0.5.4 gitdb = 0.5.4
hexagonit.recipe.cmmi = 1.6 itsdangerous = 0.22
lxml = 3.1.0 lxml = 3.2.3
meld3 = 0.6.10 meld3 = 0.6.10
plone.recipe.command = 1.1 plone.recipe.command = 1.1
pycrypto = 2.6 pycrypto = 2.6
slapos.cookbook = 0.73.1 rdiff-backup = 1.0.5
slapos.cookbook = 0.79
slapos.recipe.cmmi = 0.2
slapos.recipe.download = 1.0.dev-r4053
slapos.recipe.template = 2.4.2 slapos.recipe.template = 2.4.2
slapos.toolbox = 0.33.1 slapos.toolbox = 0.35.0
smmap = 0.8.2 smmap = 0.8.2
websockify = 0.3.0 websockify = 0.5.1
z3c.recipe.scripts = 1.0.1 z3c.recipe.scripts = 1.0.1
# Required by: # Required by:
# slapos.core==0.35.1 # slapos.core==0.35.1
# slapos.toolbox==0.33.1 # slapos.toolbox==0.35.0
Flask = 0.9 Flask = 0.10.1
# Required by: # Required by:
# slapos.toolbox==0.33.1 # slapos.toolbox==0.35.0
GitPython = 0.3.2.RC1 GitPython = 0.3.2.RC1
# Required by: # Required by:
# slapos.toolbox==0.33.1 # slapos.toolbox==0.35.0
atomize = 0.1.1 atomize = 0.1.1
# Required by: # Required by:
# slapos.toolbox==0.33.1 # slapos.toolbox==0.35.0
feedparser = 5.1.3 feedparser = 5.1.3
# Required by: # Required by:
# hexagonit.recipe.cmmi==1.6 # slapos.cookbook==0.79
hexagonit.recipe.download = 1.6nxd002 inotifyx = 0.2.0-1
# Required by: # Required by:
# slapos.cookbook==0.73.1 # slapos.cookbook==0.79
inotifyx = 0.2.0 lock-file = 2.0
# Required by: # Required by:
# slapos.cookbook==0.73.1 # slapos.cookbook==0.79
netaddr = 0.7.10 netaddr = 0.7.10
# Required by: # Required by:
# slapos.core==0.35.1 # slapos.core==0.35.1
netifaces = 0.8 netifaces = 0.8-1
# Required by: # Required by:
# slapos.toolbox==0.33.1 # websockify==0.5.1
paramiko = 1.10.0 numpy = 1.7.1
# Required by: # Required by:
# slapos.toolbox==0.33.1 # slapos.toolbox==0.35.0
psutil = 0.6.1 paramiko = 1.11.0
# Required by: # Required by:
# slapos.core==0.35.1 # slapos.toolbox==0.35.0
pyflakes = 0.6.1 psutil = 1.0.1
# Required by: # Required by:
# slapos.cookbook==0.73.1 # slapos.core==0.35.1
pytz = 2012j pyflakes = 0.7.3
# Required by: # Required by:
# slapos.cookbook==0.73.1 # slapos.cookbook==0.79
# slapos.core==0.35.1 pytz = 2013b
# slapos.toolbox==0.33.1
setuptools = 0.6c12dev-r88846
# Required by: # Required by:
# slapos.cookbook==0.73.1 # slapos.cookbook==0.79
# slapos.toolbox==0.33.1 # slapos.toolbox==0.35.0
slapos.core = 0.35.1 slapos.core = 0.35.1
# Required by: # Required by:
# slapos.core==0.35.1 # slapos.core==0.35.1
supervisor = 3.0b1 supervisor = 3.0b2
# Required by: # Required by:
# slapos.core==0.35.1 # slapos.core==0.35.1
unittest2 = 0.5.1 unittest2 = 0.5.1
# Required by: # Required by:
# slapos.cookbook==0.73.1 # slapos.cookbook==0.79
# slapos.toolbox==0.33.1 # slapos.toolbox==0.35.0
xml-marshaller = 0.9.7 xml-marshaller = 0.9.7
# Required by: # Required by:
......
#!/bin/bash
# Create a backup of the disk image of the virtual machine
QEMU_IMG=${kvm-instance:qemu-img-path}
SNAPSHOT_NAME=$(date +%s)
DISK_PATH=${kvm-instance:disk-path}
BACKUP_PATH=${:backup-disk-path}
if [ ! -f $DISK_PATH ]; then
echo "Nothing to backup, disk image doesn't exist yet."
exit 0;
fi
$QEMU_IMG snapshot -c $SNAPSHOT_NAME $DISK_PATH
if [ -f $BACKUP_PATH ]; then
rm $BACKUP_PATH
fi
$QEMU_IMG convert -f qcow2 -O qcow2 -s $SNAPSHOT_NAME $DISK_PATH $BACKUP_PATH
$QEMU_IMG snapshot -d $SNAPSHOT_NAME $DISK_PATH
#!/bin/bash
DISK_PATH=${:disk-path}
BACKUP_PATH=${:backup-disk-path}
# TODO: Use rdiff
rm $DISK_PATH
cp $BACKUP_PATH $DISK_PATH
...@@ -68,9 +68,8 @@ bridge = !!BRIDGE_NAME!! ...@@ -68,9 +68,8 @@ bridge = !!BRIDGE_NAME!!
interface = lxc$${slap-network-information:network-interface} interface = lxc$${slap-network-information:network-interface}
[passwd] [passwd]
recipe = slapos.cookbook:pwgen recipe = slapos.cookbook:generate.password
file = $${buildout:directory}/.password storage-path = $${buildout:directory}/.password
pwgen-binary = ${pwgen:location}/bin/pwgen
[shellinabox] [shellinabox]
recipe = slapos.cookbook:shellinabox recipe = slapos.cookbook:shellinabox
...@@ -79,7 +78,7 @@ port = 8080 ...@@ -79,7 +78,7 @@ port = 8080
shell = ${lxc:location}/bin/lxc-console -n $${uuid:uuid} shell = ${lxc:location}/bin/lxc-console -n $${uuid:uuid}
wrapper = $${rootdirectory:bin}/shellinaboxd_raw wrapper = $${rootdirectory:bin}/shellinaboxd_raw
shellinabox-binary = ${shellinabox:location}/bin/shellinaboxd shellinabox-binary = ${shellinabox:location}/bin/shellinaboxd
password = $${passwd:password} password = $${passwd:passwd}
directory = $${buildout:directory}/ directory = $${buildout:directory}/
login-shell = $${rootdirectory:bin}/login login-shell = $${rootdirectory:bin}/login
certificate-directory = $${directory:shellinabox} certificate-directory = $${directory:shellinabox}
......
...@@ -10,7 +10,6 @@ extends = ...@@ -10,7 +10,6 @@ extends =
../../component/xz-utils/buildout.cfg ../../component/xz-utils/buildout.cfg
../../component/tar/buildout.cfg ../../component/tar/buildout.cfg
../../component/shellinabox/buildout.cfg ../../component/shellinabox/buildout.cfg
../../component/pwgen/buildout.cfg
../../component/bash/buildout.cfg ../../component/bash/buildout.cfg
../../component/coreutils/buildout.cfg ../../component/coreutils/buildout.cfg
...@@ -23,7 +22,6 @@ parts = ...@@ -23,7 +22,6 @@ parts =
slapos-toolbox slapos-toolbox
lxc lxc
shellinabox shellinabox
pwgen
[template] [template]
recipe = slapos.recipe.template recipe = slapos.recipe.template
......
...@@ -7,6 +7,7 @@ extends = ...@@ -7,6 +7,7 @@ extends =
../../component/dropbear/buildout.cfg ../../component/dropbear/buildout.cfg
../../component/git/buildout.cfg ../../component/git/buildout.cfg
../../component/lxml-python/buildout.cfg ../../component/lxml-python/buildout.cfg
../../component/nginx/buildout.cfg
../../component/rsync/buildout.cfg ../../component/rsync/buildout.cfg
../../stack/flask.cfg ../../stack/flask.cfg
../../stack/shacache-client.cfg ../../stack/shacache-client.cfg
...@@ -14,57 +15,109 @@ extends = ...@@ -14,57 +15,109 @@ extends =
../../stack/slapos.cfg ../../stack/slapos.cfg
parts = parts =
slapos.cookbook-repository
rdiff-backup rdiff-backup
template template
eggs eggs
nginx
simple-proxy
node-frontend-template
http-proxy
npm-modules
instance-runner-import instance-runner-import
instance-runner-export instance-runner-export
slapos-cookbook
####################
## Node JS proxy
####################
[simple-proxy]
recipe = slapos.recipe.download
url = ${:_profile_base_location_}/simple-proxy.js
location = ${buildout:parts-directory}/${:_buildout_section_name_}
md5sum = 86e2231b3f65587b56d9be63e21a4e05
filename = simple-proxy.js
mode = 0644
[node-frontend-template]
recipe = slapos.recipe.download
url = ${:_profile_base_location_}/node-frontend.in
location = ${buildout:parts-directory}/${:_buildout_section_name_}
filename = node-frontend.in
md5sum = 72904152860dddb30ca936dac5bbf4cd
mode = 0644
[http-proxy]
# https://github.com/nodejitsu/node-http-proxy
recipe = slapos.recipe.build:download-unpacked
#XXX-Cedric : use upstream when merged
url = https://github.com/desaintmartin/node-http-proxy/archive/20120621.zip
md5sum = 621e5fca448cbea137c5d847d780d84d
[npm-modules]
recipe = plone.recipe.command
destination = ${buildout:parts-directory}/${:_buildout_section_name_}
location = ${buildout:parts-directory}/${:_buildout_section_name_}
command =
export HOME=${:location};
rm -fr ${:destination} &&
mkdir -p ${:destination} &&
cd ${:destination} &&
${nodejs:location}/bin/node ${nodejs:location}/bin/npm install colors@0.6.0-1 &&
${nodejs:location}/bin/node ${nodejs:location}/bin/npm install socket.io@0.8.7 &&
${nodejs:location}/bin/node ${nodejs:location}/bin/npm install socket.io-client@0.8.7 &&
${nodejs:location}/bin/node ${nodejs:location}/bin/npm install optimist@0.3.1 &&
${nodejs:location}/bin/node ${nodejs:location}/bin/npm install pkginfo@0.2.3
# slapos-cookbook
[template] [template]
recipe = slapos.recipe.template recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance.cfg url = ${:_profile_base_location_}/instance.cfg
output = ${buildout:directory}/template.cfg output = ${buildout:directory}/template.cfg
md5sum = 5de75f295f9382a587343718bb1be124 md5sum = b6ed8c30cc2ec51244796ce57315089f
mode = 0644 mode = 0644
[template-runner] [template-runner]
recipe = slapos.recipe.template recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance-runner.cfg url = ${:_profile_base_location_}/instance-runner.cfg
output = ${buildout:directory}/template-runner.cfg output = ${buildout:directory}/template-runner.cfg
md5sum = e1d9aeeb3b02dfb3578eddfddd44d053 md5sum = 4877c808972b6b04bfac1de94c14a301
mode = 0644
[instance-resilient]
recipe = slapos.recipe.template:jinja2
template = ${:_profile_base_location_}/instance-resilient.cfg.jinja2
rendered = ${buildout:directory}/instance-resilient.cfg
md5sum = f533d354da36e1bb10819fab8a90109a
context = key buildout buildout:bin-directory
key develop_eggs_directory buildout:develop-eggs-directory
key eggs_directory buildout:eggs-directory
import-list = file parts template-parts:destination
file replicated template-replicated:destination
mode = 0644 mode = 0644
[instance-runner-import] [instance-runner-import]
recipe = slapos.recipe.template recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance-runner-import.cfg.in url = ${:_profile_base_location_}/instance-runner-import.cfg.in
output = ${buildout:directory}/instance-runner-import.cfg output = ${buildout:directory}/instance-runner-import.cfg
md5sum = b37ec3af1898834041d8032ff755bac3 md5sum = f16cb60bb16632e652bea69cd5cdd9b7
mode = 0644 mode = 0644
[instance-runner-export] [instance-runner-export]
recipe = slapos.recipe.template recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance-runner-export.cfg.in url = ${:_profile_base_location_}/instance-runner-export.cfg.in
output = ${buildout:directory}/instance-runner-export.cfg output = ${buildout:directory}/instance-runner-export.cfg
md5sum = 4028924d0edb61bdcfbf03bb2bac43b8 md5sum = 7e71622c09271790b5cef21c8613b8ac
mode = 0644
[template-resilient]
recipe = slapos.recipe.download
url = ${:_profile_base_location_}/instance-resilient.cfg.jinja2
md5sum = fc253453da66583e7d515a6e2eb1475e
filename = instance-resilient.cfg.jinja2
mode = 0644
[template_nginx_conf]
recipe = slapos.recipe.download
url = ${:_profile_base_location_}/nginx_conf.in
md5sum = 09b7677dfc6b23c1f58e67fd06a7625e
filename = nginx_conf.in
mode = 0644
[template_launcher]
recipe = slapos.recipe.download
url = ${:_profile_base_location_}/launcher.in
md5sum = c7f8b6e9ae84aa94686a9cbaaa3dd693
filename = launcher.in
mode = 0644 mode = 0644
location = ${buildout:parts-directory}/${:_buildout_section_name_}
[eggs] [eggs]
recipe = z3c.recipe.scripts recipe = z3c.recipe.scripts
...@@ -73,6 +126,7 @@ eggs = ...@@ -73,6 +126,7 @@ eggs =
cns.recipe.symlink cns.recipe.symlink
hexagonit.recipe.download hexagonit.recipe.download
inotifyx inotifyx
lock-file
netaddr netaddr
slapos.libnetworkcache slapos.libnetworkcache
slapos.toolbox[flask_auth] slapos.toolbox[flask_auth]
......
...@@ -10,24 +10,22 @@ extends = common.cfg ...@@ -10,24 +10,22 @@ extends = common.cfg
parts += parts +=
slapos.cookbook-repository slapos.cookbook-repository
slapos.toolbox-repository
# slapos.toolbox-repository
# slapos.core-repository # slapos.core-repository
# check-recipe # check-recipe
develop = develop =
${:parts-directory}/slapos.cookbook-repository ${:parts-directory}/slapos.toolbox-repository
# ${:parts-directory}/slapos.toolbox-repository ${:parts-directory}/slapos.cookbook-repository
# ${:parts-directory}/slapos.core-repository # ${:parts-directory}/slapos.core-repository
#[slapos.toolbox-repository] [slapos.toolbox-repository]
#recipe = slapos.recipe.build:gitclone recipe = slapos.recipe.build:gitclone
#repository = http://git.erp5.org/repos/slapos.toolbox.git repository = http://git.erp5.org/repos/slapos.toolbox.git
#branch = slaprunner-resiliency branch = slaprunner-resiliency
#git-executable = ${git:location}/bin/git git-executable = ${git:location}/bin/git
[slapos.cookbook-repository] [slapos.cookbook-repository]
recipe = slapos.recipe.build:gitclone recipe = slapos.recipe.build:gitclone
......
...@@ -13,11 +13,11 @@ parts += ...@@ -13,11 +13,11 @@ parts +=
{{ parts.replicate("runner", "3") }} {{ parts.replicate("runner", "3") }}
publish-connection-informations publish-connection-informations
{{ replicated.replicate("runner", "3", "runner-export", "runner-import") }} {{ replicated.replicate("runner", "3", "runner-export", "runner-import", slapparameter_dict=slapparameter_dict) }}
# Bubble up the parameters # Bubble up the parameters
[request-runner] [request-runner]
return = url ssh-public-key ssh-url notification-id ip backend_url url cloud9_url ssh_command password_recovery_code return = url ssh-public-key ssh-url notification-id ip backend_url url cloud9_url ssh_command password_recovery_code cloud9_backend_url
config = instance-amount debug domain number authorized-key notify ip-list namebase runner1-computer-guid pbs-runner1-computer-guid runner2-computer-guid pbs-runner2-computer-guid runner3-computer-guid pbs-runner3-computer-guid config = instance-amount debug domain number authorized-key notify ip-list namebase runner1-computer-guid pbs-runner1-computer-guid runner2-computer-guid pbs-runner2-computer-guid runner3-computer-guid pbs-runner3-computer-guid
# XXX Cedric LN Ugly hack, resilient stack and slaprunner stack sharing too much ssh sections # XXX Cedric LN Ugly hack, resilient stack and slaprunner stack sharing too much ssh sections
config-authorized-key = ${request-pbs-runner-1:connection-ssh-key} ${request-pbs-runner-2:connection-ssh-key} ${slap-parameter:authorized-key} config-authorized-key = ${request-pbs-runner-1:connection-ssh-key} ${request-pbs-runner-2:connection-ssh-key} ${slap-parameter:authorized-key}
...@@ -34,9 +34,11 @@ config-domain = ${slap-parameter:domain} ...@@ -34,9 +34,11 @@ config-domain = ${slap-parameter:domain}
[publish-connection-informations] [publish-connection-informations]
recipe = slapos.cookbook:publish recipe = slapos.cookbook:publish
1_info = Set your passord in slaprunner in order to access cloud9
backend_url = ${request-runner:connection-backend_url} backend_url = ${request-runner:connection-backend_url}
url = ${request-runner:connection-url} url = ${request-runner:connection-url}
cloud9_url = ${request-runner:connection-cloud9_url} cloud9_url = ${request-runner:connection-cloud9_url}
cloud9_backend_url = ${request-runner:connection-cloud9_backend_url}
ssh_command = ${request-runner:connection-ssh_command} ssh_command = ${request-runner:connection-ssh_command}
password_recovery_code = ${request-runner:connection-password_recovery_code} password_recovery_code = ${request-runner:connection-password_recovery_code}
...@@ -54,4 +56,4 @@ pbs-runner3-computer-guid = ...@@ -54,4 +56,4 @@ pbs-runner3-computer-guid =
domain = domain =
authorized-key = authorized-key =
instance-amount = 10 instance-amount = 10
debug = false debug = false
\ No newline at end of file
...@@ -3,9 +3,27 @@ extends = ${template-runner:output} ...@@ -3,9 +3,27 @@ extends = ${template-runner:output}
${pbsready-export:output} ${pbsready-export:output}
parts += parts +=
urls nginx_conf
slaprunner nginx-launcher
cron-entry-backup cloud9
certificate-authority
ca-nginx
ca-node-frontend
slaprunner
test-runner
sshkeys-dropbear-runner
dropbear-server-add-authorized-key
sshkeys-authority
slaprunner-promise
slaprunner-frontend-promise
cloud9-promise
cloud9-frontend-promise
dropbear-promise
symlinks
node-frontend-promise
nginx-promise
urls
cron-entry-backup
[exporter] [exporter]
recipe = slapos.cookbook:slaprunner.export recipe = slapos.cookbook:slaprunner.export
...@@ -21,7 +39,8 @@ rsync-binary = ${rsync:location}/bin/rsync ...@@ -21,7 +39,8 @@ rsync-binary = ${rsync:location}/bin/rsync
[urls] [urls]
<= resilient-publish-connection-parameter <= resilient-publish-connection-parameter
backend_url = $${slaprunner:access-url} backend_url = $${slaprunner:access-url}
url = $${request-frontend:connection-site_url} url = https://$${request-frontend:connection-domain}
cloud9_url = $${cloud9:access-url} cloud9_backend_url = $${node-frontend:access-url}
cloud9_url = https://$${request-cloud9-frontend:connection-domain}
ssh_command = ssh $${dropbear-runner-server:host} -p $${dropbear-runner-server:port} ssh_command = ssh $${dropbear-runner-server:host} -p $${dropbear-runner-server:port}
password_recovery_code = $${recovery-code:passwd} password_recovery_code = $${recovery-code:passwd}
...@@ -2,11 +2,25 @@ ...@@ -2,11 +2,25 @@
extends = ${template-runner:output} extends = ${template-runner:output}
${pbsready-import:output} ${pbsready-import:output}
parts += parts +=
slaprunner nginx_conf
nginx-launcher
cloud9
certificate-authority
ca-nginx
ca-node-frontend
slaprunner
test-runner
sshkeys-dropbear-runner
dropbear-server-add-authorized-key
sshkeys-authority
slaprunner-promise
cloud9-promise
dropbear-promise
symlinks
nginx-promise
# have to repeat the next one, as it's not inherited from pbsready-import # have to repeat the next one, as it's not inherited from pbsready-import
import-on-notification import-on-notification
[importer] [importer]
recipe = slapos.cookbook:slaprunner.import recipe = slapos.cookbook:slaprunner.import
......
[buildout] [buildout]
parts = parts =
nginx_conf
nginx-launcher
cloud9 cloud9
certificate-authority
ca-nginx
ca-node-frontend
slaprunner slaprunner
test-runner test-runner
sshkeys-dropbear-runner sshkeys-dropbear-runner
...@@ -10,8 +15,12 @@ parts = ...@@ -10,8 +15,12 @@ parts =
slaprunner-promise slaprunner-promise
slaprunner-frontend-promise slaprunner-frontend-promise
cloud9-promise cloud9-promise
cloud9-frontend-promise
dropbear-promise dropbear-promise
symlinks symlinks
request-cloud9-frontend
node-frontend-promise
nginx-promise
eggs-directory = ${buildout:eggs-directory} eggs-directory = ${buildout:eggs-directory}
develop-eggs-directory = ${buildout:develop-eggs-directory} develop-eggs-directory = ${buildout:develop-eggs-directory}
...@@ -25,6 +34,7 @@ etc = $${buildout:directory}/etc/ ...@@ -25,6 +34,7 @@ etc = $${buildout:directory}/etc/
var = $${buildout:directory}/var/ var = $${buildout:directory}/var/
srv = $${buildout:directory}/srv/ srv = $${buildout:directory}/srv/
bin = $${buildout:directory}/bin/ bin = $${buildout:directory}/bin/
tmp = $${buildout:directory}/tmp/
sshkeys = $${:srv}/sshkeys sshkeys = $${:srv}/sshkeys
services = $${:etc}/service/ services = $${:etc}/service/
...@@ -35,6 +45,9 @@ run = $${:var}/run/ ...@@ -35,6 +45,9 @@ run = $${:var}/run/
backup = $${:srv}/backup/ backup = $${:srv}/backup/
promises = $${:etc}/promise/ promises = $${:etc}/promise/
test = $${:etc}/test/ test = $${:etc}/test/
nginx-data = $${directory:srv}/nginx
ca-dir = $${:srv}/ssl
[runnerdirectory] [runnerdirectory]
recipe = slapos.cookbook:mkdirectory recipe = slapos.cookbook:mkdirectory
...@@ -57,8 +70,8 @@ bytes = 4 ...@@ -57,8 +70,8 @@ bytes = 4
# Deploy cloud9 and slaprunner # Deploy cloud9 and slaprunner
[cloud9] [cloud9]
recipe = slapos.cookbook:cloud9 recipe = slapos.cookbook:cloud9
ip = $${slap-network-information:global-ipv6} ip = $${slap-network-information:local-ipv4}
port = 30000 port = 4443
wrapper = $${directory:services}/cloud9 wrapper = $${directory:services}/cloud9
working-directory = $${runnerdirectory:home} working-directory = $${runnerdirectory:home}
git-binary = ${git:location}/bin/git git-binary = ${git:location}/bin/git
...@@ -87,7 +100,7 @@ private_key = $${sshkeys-dropbear-runner:private-key} ...@@ -87,7 +100,7 @@ private_key = $${sshkeys-dropbear-runner:private-key}
ipv4 = $${slap-network-information:local-ipv4} ipv4 = $${slap-network-information:local-ipv4}
ipv6 = $${slap-network-information:global-ipv6} ipv6 = $${slap-network-information:global-ipv6}
proxy_port = 50000 proxy_port = 50000
runner_port = 50000 runner_port = 50005
partition-amount = $${slap-parameter:instance-amount} partition-amount = $${slap-parameter:instance-amount}
cloud9-url = $${cloud9:access-url} cloud9-url = $${cloud9:access-url}
wrapper = $${directory:services}/slaprunner wrapper = $${directory:services}/slaprunner
...@@ -145,31 +158,180 @@ wrapper = $${directory:services}/runner_sshd ...@@ -145,31 +158,180 @@ wrapper = $${directory:services}/runner_sshd
recipe = slapos.cookbook:dropbear.add_authorized_key recipe = slapos.cookbook:dropbear.add_authorized_key
key = $${slap-parameter:authorized-key} key = $${slap-parameter:authorized-key}
#---------------------
#--
#-- Set node frontend
[node-frontend]
launcher = $${directory:bin}/node-frontend
ip = $${slap-network-information:global-ipv6}
port = $${cloud9:port}
access-url = https://[$${:ip}]:$${:port}
[node-frontend-launcher]
recipe = slapos.recipe.template:jinja2
template = ${node-frontend-template:location}/${node-frontend-template:filename}
rendered = $${node-frontend:launcher}
mode = 700
context =
key ip node-frontend:ip
key port node-frontend:port
key key ca-node-frontend:key-file
key certificate ca-node-frontend:cert-file
key backend_ip nginx-frontend:local-ip
key backend_port nginx-frontend:port
raw shell_path ${bash:location}/bin/bash
raw node_env ${buildout:parts-directory}:${npm-modules:location}/node_modules
raw node_path ${nodejs:location}/bin/node
raw conf_path ${simple-proxy:location}/${simple-proxy:filename}
#---------------------------
#--
#-- Set nginx frontend
[tempdirectory]
recipe = slapos.cookbook:mkdirectory
client_body_temp_path = $${directory:tmp}/client_body_temp_path
proxy_temp_path = $${directory:tmp}/proxy_temp_path
fastcgi_temp_path = $${directory:tmp}/fastcgi_temp_path
uwsgi_temp_path = $${directory:tmp}/uwsgi_temp_path
scgi_temp_path = $${directory:tmp}/scgi_temp_path
[nginx-frontend]
# Options
nb_workers = 2
# Network
local-ip = $${slap-network-information:local-ipv4}
port = 30001
global-ip = $${slap-network-information:global-ipv6}
global-port = $${slaprunner:runner_port}
# Backend
cloud9-ip = $${cloud9:ip}
cloud9-port = $${cloud9:port}
runner-ip = $${slaprunner:ipv4}
runner-port = $${slaprunner:runner_port}
# SSL
ssl-certificate = $${ca-nginx:cert-file}
ssl-key = $${ca-nginx:key-file}
# Log
path_pid = $${directory:run}/nginx.pid
path_log = $${directory:log}/nginx.log
path_access_log = $${directory:log}/nginx.access.log
path_error_log = $${directory:log}/nginx.error.log
path_tmp = $${buildout:directory}/tmp
# Config files
path_nginx_conf = $${directory:etc}/nginx.conf
# Executables
bin_nginx = ${nginx:location}/sbin/nginx
bin_launcher = $${directory:bin}/launcher
# Utils
path_shell = ${dash:location}/bin/dash
# Misc.
etc_dir = $${directory:etc}
[nginx_conf]
recipe = slapos.recipe.template:jinja2
template = ${template_nginx_conf:location}/${template_nginx_conf:filename}
rendered = $${nginx-frontend:path_nginx_conf}
context =
section param_nginx_frontend nginx-frontend
section param_tempdir tempdirectory
[nginx-launcher]
recipe = slapos.recipe.template:jinja2
template = ${template_launcher:location}/${template_launcher:filename}
rendered = $${nginx-frontend:bin_launcher}
mode = 700
context =
section param_nginx_frontend nginx-frontend
#--------------------
#--
#-- ssl certificates
[certificate-authority]
recipe = slapos.cookbook:certificate_authority
openssl-binary = ${openssl:location}/bin/openssl
ca-dir = $${directory:ca-dir}
requests-directory = $${cadirectory:requests}
wrapper = $${directory:services}/certificate_authority
ca-private = $${cadirectory:private}
ca-certs = $${cadirectory:certs}
ca-newcerts = $${cadirectory:newcerts}
ca-crl = $${cadirectory:crl}
[cadirectory]
recipe = slapos.cookbook:mkdirectory
requests = $${directory:ca-dir}/requests/
private = $${directory:ca-dir}/private/
certs = $${directory:ca-dir}/certs/
newcerts = $${directory:ca-dir}/newcerts/
crl = $${directory:ca-dir}/crl/
[ca-nginx]
<= certificate-authority
recipe = slapos.cookbook:certificate_authority.request
key-file = $${cadirectory:certs}/nginx_frontend.key
cert-file = $${cadirectory:certs}/nginx_frontend.crt
executable = $${nginx-launcher:rendered}
wrapper = $${directory:services}/nginx-frontend
# Put domain name
name = example.com
[ca-node-frontend]
<= certificate-authority
recipe = slapos.cookbook:certificate_authority.request
key-file = $${cadirectory:certs}/nodejs.key
cert-file = $${cadirectory:certs}/nodejs.crt
executable = $${node-frontend-launcher:rendered}
wrapper = $${directory:services}/node-frontend
# Put domain name
name = example.com
#--------------------
#--
#-- Request frontend
# Request frontend
[request-frontend] [request-frontend]
<= slap-connection <= slap-connection
recipe = slapos.cookbook:requestoptional recipe = slapos.cookbook:requestoptional
name = Frontend name = SlapRunner Frontend
# XXX We have hardcoded SR URL here. # XXX We have hardcoded SR URL here.
software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg
slave = true slave = true
config = url config = url
config-url = $${slaprunner:access-url} config-url = $${slaprunner:access-url}
return = site_url return = site_url domain
[request-cloud9-frontend]
<= slap-connection
recipe = slapos.cookbook:requestoptional
name = Cloud9 Frontend
software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg
slave = true
config = url
config-url = $${node-frontend:access-url}
return = site_url domain
#--------------------------------------
#--
#-- Send informations to SlapOS Master
# Send informations to SlapOS Master
[publish-connection-informations] [publish-connection-informations]
recipe = slapos.cookbook:publish recipe = slapos.cookbook:publish
1_info = Set your passord in slaprunner in order to access cloud9
backend_url = $${slaprunner:access-url} backend_url = $${slaprunner:access-url}
url = $${request-frontend:connection-site_url} url = https://$${request-frontend:connection-domain}
cloud9_url = $${cloud9:access-url} cloud9_backend_url = $${node-frontend:access-url}
cloud9_url = https://$${request-cloud9-frontend:connection-domain}
ssh_command = ssh $${dropbear-runner-server:host} -p $${dropbear-runner-server:port} ssh_command = ssh $${dropbear-runner-server:host} -p $${dropbear-runner-server:port}
password_recovery_code = $${recovery-code:passwd} password_recovery_code = $${recovery-code:passwd}
#---------------------------
#--
#-- Deploy promises scripts
# Deploy promises scripts
[slaprunner-promise] [slaprunner-promise]
recipe = slapos.cookbook:check_port_listening recipe = slapos.cookbook:check_port_listening
path = $${directory:promises}/slaprunner path = $${directory:promises}/slaprunner
...@@ -179,7 +341,7 @@ port = $${slaprunner:runner_port} ...@@ -179,7 +341,7 @@ port = $${slaprunner:runner_port}
[slaprunner-frontend-promise] [slaprunner-frontend-promise]
recipe = slapos.cookbook:check_url_available recipe = slapos.cookbook:check_url_available
path = $${directory:promises}/slaprunner_frontend path = $${directory:promises}/slaprunner_frontend
url = $${request-frontend:connection-site_url} url = https://$${request-frontend:connection-domain}
dash_path = ${dash:location}/bin/dash dash_path = ${dash:location}/bin/dash
curl_path = ${curl:location}/bin/curl curl_path = ${curl:location}/bin/curl
...@@ -190,6 +352,26 @@ url = http://$${cloud9:ip}:$${cloud9:port} ...@@ -190,6 +352,26 @@ url = http://$${cloud9:ip}:$${cloud9:port}
dash_path = ${dash:location}/bin/dash dash_path = ${dash:location}/bin/dash
curl_path = ${curl:location}/bin/curl curl_path = ${curl:location}/bin/curl
[cloud9-frontend-promise]
recipe = slapos.cookbook:check_url_available
path = $${directory:promises}/cloud9-frontend-promise
url = $${publish-connection-informations:cloud9_url}
check-secure = 1
dash_path = ${dash:location}/bin/dash
curl_path = ${curl:location}/bin/curl
[node-frontend-promise]
recipe = slapos.cookbook:check_port_listening
path = $${directory:promises}/node-frontend
hostname = $${node-frontend:ip}
port = $${node-frontend:port}
[nginx-promise]
recipe = slapos.cookbook:check_port_listening
path = $${directory:promises}/nginx
hostname = $${nginx-frontend:local-ip}
port = $${nginx-frontend:port}
[dropbear-promise] [dropbear-promise]
recipe = slapos.cookbook:check_port_listening recipe = slapos.cookbook:check_port_listening
path = $${directory:promises}/dropbear path = $${directory:promises}/dropbear
...@@ -206,4 +388,4 @@ symlink_base = ${buildout:directory}/bin ...@@ -206,4 +388,4 @@ symlink_base = ${buildout:directory}/bin
authorized-key = authorized-key =
# Default value of instances number in slaprunner # Default value of instances number in slaprunner
instance-amount = 10 instance-amount = 10
debug = false debug = false
\ No newline at end of file
...@@ -4,15 +4,38 @@ parts = ...@@ -4,15 +4,38 @@ parts =
eggs-directory = ${buildout:eggs-directory} eggs-directory = ${buildout:eggs-directory}
develop-eggs-directory = ${buildout:develop-eggs-directory} develop-eggs-directory = ${buildout:develop-eggs-directory}
offline = true
[switch_softwaretype] [switch_softwaretype]
recipe = slapos.cookbook:softwaretype recipe = slapos.cookbook:softwaretype
default = ${template-runner:output} default = ${template-runner:output}
resilient = ${instance-resilient:rendered} resilient = $${instance-resilient:rendered}
runner = ${template-runner:output} runner = ${template-runner:output}
runner-import = ${instance-runner-import:output} runner-import = ${instance-runner-import:output}
runner-export = ${instance-runner-export:output} runner-export = ${instance-runner-export:output}
frozen = ${instance-frozen:output} frozen = ${instance-frozen:output}
pull-backup = ${template-pull-backup:output} pull-backup = ${template-pull-backup:output}
[instance-resilient]
recipe = slapos.recipe.template:jinja2
template = ${template-resilient:target}
rendered = $${buildout:directory}/instance-resilient.cfg
extensions = jinja2.ext.do
context = key buildout buildout:bin-directory
key develop_eggs_directory buildout:develop-eggs-directory
key eggs_directory buildout:eggs-directory
key slapparameter_dict slap-parameters:configuration
template-parts-destination = ${template-parts:destination}
template-replicated-destination = ${template-replicated:destination}
import-list = file parts :template-parts-destination
file replicated :template-replicated-destination
mode = 0644
[slap-parameters]
recipe = slapos.cookbook:slapconfiguration
computer = $${slap-connection:computer-id}
partition = $${slap-connection:partition-id}
url = $${slap-connection:server-url}
key = $${slap-connection:key-file}
cert = $${slap-connection:cert-file}
\ No newline at end of file
#! {{ param_nginx_frontend['path_shell'] }}
# BEWARE: This file is operated by slapgrid
# BEWARE: It will be overwritten automatically
# Run nginx
exec {{ param_nginx_frontend['bin_nginx'] }} -c {{ param_nginx_frontend['path_nginx_conf'] }}
worker_processes {{ param_nginx_frontend['nb_workers'] }};
pid {{ param_nginx_frontend['path_pid'] }};
error_log {{ param_nginx_frontend['path_error_log'] }};
daemon off;
events {
worker_connections 1024;
accept_mutex off;
}
http {
default_type application/octet-stream;
access_log {{ param_nginx_frontend['path_access_log'] }} combined;
map $http_upgrade $connection_upgrade {
default upgrade;
'' close;
}
server {
listen {{ param_nginx_frontend['local-ip'] }}:{{ param_nginx_frontend['port'] }};
server_name _;
keepalive_timeout 90s;
client_body_temp_path {{ param_tempdir['client_body_temp_path'] }};
proxy_temp_path {{ param_tempdir['proxy_temp_path'] }};
fastcgi_temp_path {{ param_tempdir['fastcgi_temp_path'] }};
uwsgi_temp_path {{ param_tempdir['uwsgi_temp_path'] }};
scgi_temp_path {{ param_tempdir['scgi_temp_path'] }};
location / {
auth_basic "Restricted";
auth_basic_user_file {{ param_nginx_frontend['etc_dir'] }}/.htpasswd;
proxy_pass http://{{ param_nginx_frontend['cloud9-ip'] }}:{{ param_nginx_frontend['cloud9-port'] }};
proxy_next_upstream error timeout invalid_header http_500 http_502 http_503 http_504;
proxy_redirect off;
proxy_buffering off;
proxy_http_version 1.1;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection $connection_upgrade;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
}
}
server {
listen [{{ param_nginx_frontend['global-ip'] }}]:{{ param_nginx_frontend['global-port'] }} ssl;
server_name _;
ssl_certificate {{ param_nginx_frontend['ssl-certificate'] }};
ssl_certificate_key {{ param_nginx_frontend['ssl-key'] }};
ssl_protocols SSLv3 TLSv1 TLSv1.1 TLSv1.2;
ssl_ciphers HIGH:!aNULL:!MD5;
keepalive_timeout 90s;
client_body_temp_path {{ param_tempdir['client_body_temp_path'] }};
proxy_temp_path {{ param_tempdir['proxy_temp_path'] }};
fastcgi_temp_path {{ param_tempdir['fastcgi_temp_path'] }};
uwsgi_temp_path {{ param_tempdir['uwsgi_temp_path'] }};
scgi_temp_path {{ param_tempdir['scgi_temp_path'] }};
location / {
proxy_pass http://{{ param_nginx_frontend['runner-ip'] }}:{{ param_nginx_frontend['runner-port'] }};
proxy_next_upstream error timeout invalid_header http_500 http_502 http_503 http_504;
proxy_redirect off;
proxy_buffering off;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Host $http_host;
}
}
}
#!{{ shell_path }}
# BEWARE: This file is operated by slapgrid
# BEWARE: It will be overwritten automatically
export NODE_PATH={{ node_env }}
exec {{ node_path }} {{ conf_path }} {{ ip }} {{ port }} {{ key }} {{ certificate }} {{ backend_ip }} {{ backend_port }}
\ No newline at end of file
/*****************************************************************************
*
* Copyright (c) 2013 Vifib SARL and Contributors. All Rights Reserved.
*
* WARNING: This program as such is intended to be used by professional
* programmers who take the whole responsibility of assessing all potential
* consequences resulting from its eventual inadequacies and bugs
* End users who are looking for a ready-to-use solution with commercial
* guarantees and support are strongly adviced to contract a Free Software
* Service Company
*
* This program is Free Software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 3
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
*
*****************************************************************************/
var fs = require('fs'),
util = require('util'),
colors = require('colors'),
http = require('http'),
httpProxy = require('http-proxy');
var listenInterface = process.argv[2],
port = process.argv[3],
sslKeyFile = process.argv[4],
sslCertFile = process.argv[5],
backendIp = process.argv[6],
backendPort = process.argv[7];
if (process.argv.length < 8) {
console.error("Too few arguments. Exiting.");
process.exit(1);
}
var middleware = function (req, res, proxy) {
return proxy.proxyRequest(req, res,{
host: backendIp,
port: backendPort
});
};
middleware.proxyWebSocketRequest = function (req, socket, head, proxy) {
return proxy.proxyWebSocketRequest(req, socket, head,{
host: backendIp,
port: backendPort
});
};
/**
* Create server
*/
var proxyServer = httpProxy.createServer(
middleware,
{
https: {
key: fs.readFileSync(
sslKeyFile,
'utf8'
),
cert: fs.readFileSync(
sslCertFile,
'utf8'
)
},
source: {
host: listenInterface,
port: port
}}
);
console.log('HTTPS server starting and trying to listen on ' +
listenInterface + ':' + port);
// Release the beast.
proxyServer.listen(port, listenInterface);
...@@ -5,7 +5,7 @@ ...@@ -5,7 +5,7 @@
# 2/ Define list of trusted certificates for the cache. # 2/ Define list of trusted certificates for the cache.
[buildout] [buildout]
extends = development.cfg extends = common.cfg
[networkcache] [networkcache]
# signature certificates of the following uploaders. # signature certificates of the following uploaders.
...@@ -65,7 +65,16 @@ slapos.recipe.template = 2.4.2 ...@@ -65,7 +65,16 @@ slapos.recipe.template = 2.4.2
smmap = 0.8.2 smmap = 0.8.2
xml-marshaller = 0.9.7 xml-marshaller = 0.9.7
z3c.recipe.scripts = 1.0.1 z3c.recipe.scripts = 1.0.1
lock-file = 2.0
rdiff-backup = 1.0.5
slapos.recipe.cmmi = 0.2
slapos.recipe.download = 1.0.dev-r4053
slapos.toolbox = 0.35.1
slapos.cookbook = 0.78.5
cliff = 1.4
cmd2 = 0.6.6
prettytable = 0.7.2
requests = 1.2.3
# Required by: # Required by:
# slapos.core==0.34 # slapos.core==0.34
# slapos.toolbox==0.34.0 # slapos.toolbox==0.34.0
......
...@@ -308,9 +308,8 @@ githttpbackend = ${git:location}/libexec/git-core/git-http-backend ...@@ -308,9 +308,8 @@ githttpbackend = ${git:location}/libexec/git-core/git-http-backend
base-directory = $${trac-config:project_dir}/git base-directory = $${trac-config:project_dir}/git
[trac-admin] [trac-admin]
recipe = slapos.cookbook:pwgen recipe = slapos.cookbook:generate.password
file = $${buildout:directory}/.password storage-path = $${buildout:directory}/.password
pwgen-binary = ${pwgen:location}/bin/pwgen
user = TracAdmin user = TracAdmin
#--------------------- #---------------------
...@@ -330,7 +329,7 @@ eggs-dirs = ...@@ -330,7 +329,7 @@ eggs-dirs =
python-lib = ${python2.7:location}/lib python-lib = ${python2.7:location}/lib
trac-admin = ${buildout:bin-directory}/trac-admin trac-admin = ${buildout:bin-directory}/trac-admin
admin-user = $${trac-admin:user} admin-user = $${trac-admin:user}
admin-password = $${trac-admin:password} admin-password = $${trac-admin:passwd}
#MySQL informations #MySQL informations
mysql-username = $${mariadb-urlparse:username} mysql-username = $${mariadb-urlparse:username}
mysql-password = $${mariadb-urlparse:password} mysql-password = $${mariadb-urlparse:password}
...@@ -401,7 +400,7 @@ port = 9000 ...@@ -401,7 +400,7 @@ port = 9000
shell = $${shell:wrapper} shell = $${shell:wrapper}
wrapper = $${rootdirectory:bin}/shellinaboxd_raw wrapper = $${rootdirectory:bin}/shellinaboxd_raw
shellinabox-binary = ${shellinabox:location}/bin/shellinaboxd shellinabox-binary = ${shellinabox:location}/bin/shellinaboxd
password = $${trac-admin:password} password = $${trac-admin:passwd}
directory = $${inittrac:site-dir} directory = $${inittrac:site-dir}
login-shell = $${rootdirectory:bin}/login login-shell = $${rootdirectory:bin}/login
certificate-directory = $${directory:shellinabox} certificate-directory = $${directory:shellinabox}
...@@ -454,7 +453,7 @@ frontend_url = $${request-frontend:connection-site_url} ...@@ -454,7 +453,7 @@ frontend_url = $${request-frontend:connection-site_url}
git = $${request-frontend:connection-site_url}git/ git = $${request-frontend:connection-site_url}git/
svn = $${request-frontend:connection-site_url}svn/ svn = $${request-frontend:connection-site_url}svn/
admin_user = $${trac-admin:user} admin_user = $${trac-admin:user}
admin_password = $${trac-admin:password} admin_password = $${trac-admin:passwd}
admin_shell = https://[$${shellinabox:ipv6}]:$${shellinabox:port}/ admin_shell = https://[$${shellinabox:ipv6}]:$${shellinabox:port}/
#---------------- #----------------
......
...@@ -41,7 +41,6 @@ extends = ...@@ -41,7 +41,6 @@ extends =
../../component/lxml-python/buildout.cfg ../../component/lxml-python/buildout.cfg
../../component/mysql-python/buildout.cfg ../../component/mysql-python/buildout.cfg
../../component/git/buildout.cfg ../../component/git/buildout.cfg
../../component/pwgen/buildout.cfg
../../component/shellinabox/buildout.cfg ../../component/shellinabox/buildout.cfg
../../component/perl/buildout.cfg ../../component/perl/buildout.cfg
......
...@@ -331,7 +331,7 @@ runzope-binary = {{ bin_directory }}/runzope ...@@ -331,7 +331,7 @@ runzope-binary = {{ bin_directory }}/runzope
bt5-repository-list = bt5-repository-list =
[deadlock-debugger-password] [deadlock-debugger-password]
recipe = slapos.cookbook:pwgen.stable recipe = slapos.cookbook:generate.password
[zope-conf-parameter-base] [zope-conf-parameter-base]
ip = {{ ipv4 }} ip = {{ ipv4 }}
...@@ -346,7 +346,7 @@ context = ...@@ -346,7 +346,7 @@ context =
key instance directory:instance key instance directory:instance
key instance_products directory:instance-products key instance_products directory:instance-products
raw deadlock_path /manage_debug_threads raw deadlock_path /manage_debug_threads
key deadlock_debugger_password deadlock-debugger-password:password key deadlock_debugger_password deadlock-debugger-password:passwd
key tidstorage_ip tidstorage:ip key tidstorage_ip tidstorage:ip
key tidstorage_port tidstorage:port key tidstorage_port tidstorage:port
key promise_path erp5-promise:promise-path key promise_path erp5-promise:promise-path
......
...@@ -214,12 +214,12 @@ parts += ...@@ -214,12 +214,12 @@ parts +=
{{ replicated.replicate("Name", "3", {{ replicated.replicate("Name", "3",
"mysoftware-export", "mysoftware-import", "mysoftware-export", "mysoftware-import",
"ArgLeader","ArgBackup") }} "ArgLeader","ArgBackup", slapparameter_dict=slapparameter_dict) }}
and it'll expend into the sections require to request Name0, Name1 and Name2, and it'll expend into the sections require to request Name0, Name1 and Name2,
backuped and resilient. The leader will expend the section [ArgLeader], backups backuped and resilient. The leader will expend the section [ArgLeader], backups
will expend [ArgBackup]. If you don't need to specify any options, you can will expend [ArgBackup]. slapparameter_dict is the dict containing the parameters given to the instance. If you don't need to specify any options, you can
omit the last two arguments in replicate(). omit the last three arguments in replicate().
Since you will compile your template with jinja2, there should be no $${}, Since you will compile your template with jinja2, there should be no $${},
because it is not yet possible to use jinja2 -> buildout template. because it is not yet possible to use jinja2 -> buildout template.
...@@ -227,3 +227,36 @@ because it is not yet possible to use jinja2 -> buildout template. ...@@ -227,3 +227,36 @@ because it is not yet possible to use jinja2 -> buildout template.
To compile with jinja2, see jinja2's recipe. To compile with jinja2, see jinja2's recipe.
Deploying your resilient software
---------------------------------
You can provide sla parameters to each request you make (a lot: for export, import and pbs).
example:
Here is a small example of parameters you can provide to control the deployment (case of a runner):
<?xml version='1.0' encoding='utf-8'?>
<instance>
<parameter id="-sla-1-computer_guid">COMP-GRP1</parameter>
<parameter id="-sla-pbs1-computer_guid">COMP-PBS1</parameter>
<parameter id="-sla-2-computer_guid">COMP-GROUP2</parameter>
<parameter id="-sla-runner2-computer_guid">COMP-RUN2</parameter>
<parameter id="-sla-2-network_guid">NET-2</parameter>
<parameter id="-sla-runner0-computer_guid">COMP-RUN0</parameter>
</instance>
Consequence on sla parameters by request:
* runner0: computer_guid = COMP-RUN0 (provided directly)
* runner1: computer_guid = COMP-GRP1 (provided by group 1)
* runner2: computer_guid = COMP-RUN2 (provided by group 2 but overided directly)
network_guid = NET-2 (provided by group 2)
* PBS 1: computer_guid = COMP-PBS1 (provided by group 1 but overided directly)
* PBS 2: computer_guid = COMP-GRP2 (provided by group 2)
network_guid = NET-2 (provided by group 2)
Parameters are analysed this way:
* If it starts with "-sla-" it is not transmitted to requested instance and is used to do the request as sla.
* -sla-foo-bar=example (foo being a magic key) will be use for each request considering "foo" as a key to use and the sla parameter is "bar". So for each group using the "foo" key, sla parameter "bar" is used with value "example"
About magic keys:
We can find 2 kinds of magic keys:
* id : example, in "-sla-2-foo" 2 is the magic key and the parameter will be used for each request with id 2 (in case of kvm: kvm2 and PBS 2)
* nameid : example, in "-sla-kvm2-foo", foo will be used for kvm2 request. Name for pbs is "pbs" -> "-sla-pbs2-foo".
IMPORTANT NOTE: in case the same foo parameter is asked for the group, the nameid key prevail
[buildout] [buildout]
extends = extends =
../../component/dropbear/buildout.cfg
../../component/gzip/buildout.cfg ../../component/gzip/buildout.cfg
../../component/rdiff-backup/buildout.cfg ../../component/rdiff-backup/buildout.cfg
../../component/rsync/buildout.cfg
parts = parts =
rdiff-backup rdiff-backup
...@@ -11,7 +13,6 @@ parts = ...@@ -11,7 +13,6 @@ parts =
template-replicated template-replicated
template-parts template-parts
instance-frozen instance-frozen
template-resilient
# needed tools for resiliency # needed tools for resiliency
gzip gzip
...@@ -38,7 +39,7 @@ mode = 0644 ...@@ -38,7 +39,7 @@ mode = 0644
recipe = slapos.recipe.template recipe = slapos.recipe.template
url = ${:_profile_base_location_}/pbsready-import.cfg.in url = ${:_profile_base_location_}/pbsready-import.cfg.in
output = ${buildout:directory}/pbsready-import.cfg output = ${buildout:directory}/pbsready-import.cfg
md5sum = 1b1308fd39476d48b5ca13db48ea6dc9 md5sum = 3c2e73f49abdc52282fc045e6d91f3e9
mode = 0644 mode = 0644
[pbsready-export] [pbsready-export]
...@@ -47,20 +48,20 @@ mode = 0644 ...@@ -47,20 +48,20 @@ mode = 0644
recipe = slapos.recipe.template recipe = slapos.recipe.template
url = ${:_profile_base_location_}/pbsready-export.cfg.in url = ${:_profile_base_location_}/pbsready-export.cfg.in
output = ${buildout:directory}/pbsready-export.cfg output = ${buildout:directory}/pbsready-export.cfg
md5sum = 5d9e20c436fd307e8e4ab224a9a65792 md5sum = a0e22a5de727544c5767d6bee059a77a
mode = 0644 mode = 0644
[template-pull-backup] [template-pull-backup]
recipe = slapos.recipe.template recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance-pull-backup.cfg.in url = ${:_profile_base_location_}/instance-pull-backup.cfg.in
output = ${buildout:directory}/instance-pull-backup.cfg output = ${buildout:directory}/instance-pull-backup.cfg
md5sum = 453d96f5a6c1230c01c878cc7640bae6 md5sum = c67a9dad66490ae264f9e7003521bf59
mode = 0644 mode = 0644
[template-replicated] [template-replicated]
recipe = slapos.recipe.download recipe = slapos.recipe.download
url = ${:_profile_base_location_}/template-replicated.cfg.in url = ${:_profile_base_location_}/template-replicated.cfg.in
md5sum = 9e20f283bf709c63c9c6692d5e1f8972 #md5sum = 9e20f283bf709c63c9c6692d5e1f8972
mode = 0644 mode = 0644
destination = ${buildout:directory}/template-replicated.cfg.in destination = ${buildout:directory}/template-replicated.cfg.in
...@@ -77,12 +78,10 @@ destination = ${buildout:directory}/template-parts.cfg.in ...@@ -77,12 +78,10 @@ destination = ${buildout:directory}/template-parts.cfg.in
# which will run without removing any content because it raises an error. # which will run without removing any content because it raises an error.
recipe = slapos.recipe.template recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance-frozen.cfg.in url = ${:_profile_base_location_}/instance-frozen.cfg.in
md5sum = d21472f0e58f928fb827f2cbf22c4d4a
output = ${buildout:directory}/instance-frozen.cfg output = ${buildout:directory}/instance-frozen.cfg
[template-resilient] [versions]
recipe = slapos.recipe.template # Pin Jinja2 to 2.6, as 2.7 breaks current code
url = ${:_profile_base_location_}/resilient.cfg.in Jinja2 = 2.6
output = ${buildout:directory}/resilient.cfg
md5sum = 59e74d290d623de2c1e147e48f284fba
mode = 0644
[buildout] [buildout]
parts = eggs-directory = ${buildout:eggs-directory}
\ No newline at end of file develop-eggs-directory = ${buildout:develop-eggs-directory}
offline = true
parts =
...@@ -105,7 +105,8 @@ promises-directory = $${basedirectory:promises} ...@@ -105,7 +105,8 @@ promises-directory = $${basedirectory:promises}
directory = $${directory:pbs-backup} directory = $${directory:pbs-backup}
cron-entries = $${cron:cron-entries} cron-entries = $${cron:cron-entries}
wrappers-directory = $${directory:pbs-wrappers} wrappers-directory = $${directory:pbs-wrappers}
notifier-url = http://[$${notifier:host}]:$${notifier:port}/ # XXX: this should be named "notifier-host"
notifier-url = http://[$${notifier:host}]:$${notifier:port}
slave-instance-list = $${slap-parameter:slave_instance_list} slave-instance-list = $${slap-parameter:slave_instance_list}
......
...@@ -2,7 +2,21 @@ ...@@ -2,7 +2,21 @@
extends = ${pbsready:output} extends = ${pbsready:output}
parts += # Explicitely define extended parts from pbsready
# then add local parts
parts =
resiliency
logrotate
logrotate-entry-cron
logrotate-entry-equeue
cron
cron-entry-logrotate
sshkeys-authority
dropbear-server
sshkeys-dropbear
dropbear-server-pbs-authorized-key
notifier
cron-entry-backup cron-entry-backup
[resilient-publish-connection-parameter] [resilient-publish-connection-parameter]
......
...@@ -2,7 +2,21 @@ ...@@ -2,7 +2,21 @@
extends = ${pbsready:output} extends = ${pbsready:output}
parts += # Explicitely define extended parts from pbsready
# then add local parts
parts =
resiliency
logrotate
logrotate-entry-cron
logrotate-entry-equeue
cron
cron-entry-logrotate
sshkeys-authority
dropbear-server
sshkeys-dropbear
dropbear-server-pbs-authorized-key
notifier
import-on-notification import-on-notification
resilient-publish-connection-parameter resilient-publish-connection-parameter
......
## not used at the moment
[buildout]
parts =
request-pull-backup-server
[request-pull-backup-server]
<= slap-connection
recipe = slapos.cookbook:request
name = PBS (Pull Backup Server)
software-url = $${slap-connection:software-release-url}
software-type = pull-backup
return = ssh-key notification-url feeds-url
slave = false
\ No newline at end of file
{% macro replicate(namebase, nbbackup, typeexport, typeimport, heriteLeader='', heriteBackup='') %} {% macro replicate(namebase, nbbackup, typeexport, typeimport, heriteLeader='', heriteBackup='', slapparameter_dict={}) %}
{% set sla_parameter_dict = {} -%}
# prepare sla-parameters
{% if slapparameter_dict is defined -%}
{% for key in slapparameter_dict.keys() -%}
{% if key.startswith('-sla-') -%}
{% do sla_parameter_dict.__setitem__(key, slapparameter_dict.pop(key)) -%}
{% endif -%}
{% endfor -%}
{% endif -%}
## Tells the Backupable recipe that we want a backup ## Tells the Backupable recipe that we want a backup
[resilient] [resilient]
...@@ -18,12 +29,42 @@ software-type = {{typeexport}} ...@@ -18,12 +29,42 @@ software-type = {{typeexport}}
name = {{namebase}}0 name = {{namebase}}0
return = ssh-public-key ssh-url notification-id ip return = ssh-public-key ssh-url notification-id ip
config = number authorized-key notify ip-list namebase config =
# Resilient related parameters
number authorized-key notify ip-list namebase
{% if slapparameter_dict is defined %}
# Software Instance related parameters
{% for parameter_name in slapparameter_dict.keys() %}{{parameter_name}} {% endfor %}
{% endif %}
config-number = 0 config-number = 0
config-authorized-key = {% for id in range(1,nbbackup|int) %} ${request-pbs-{{namebase}}-{{id}}:connection-ssh-key}{% endfor %} config-authorized-key = {% for id in range(1,nbbackup|int) %} ${request-pbs-{{namebase}}-{{id}}:connection-ssh-key}{% endfor %}
config-notify = {% for id in range(1,nbbackup|int) %} ${request-pbs-{{namebase}}-{{id}}:connection-notification-url}{% endfor %} config-notify = {% for id in range(1,nbbackup|int) %} ${request-pbs-{{namebase}}-{{id}}:connection-notification-url}{% endfor %}
config-ip-list = config-ip-list =
# Bubble up all the instance parameters to the requested export instance.
{% if slapparameter_dict is defined %}
{% for parameter_name, parameter_value in slapparameter_dict.items() %}config-{{parameter_name}} = {{parameter_value}}
{% endfor %}
{% endif %}
{% if sla_parameter_dict -%}
{% set sla_key_main = "-sla-%s%s-" % (namebase, 0) -%}
{% set sla_key_secondary = "-sla-%s-" % (0) -%}
{% set sla_key_main_length = sla_key_main | length -%}
{% set sla_key_secondary_length = sla_key_secondary | length -%}
{% set sla_dict = {} -%}
{% for key in sla_parameter_dict.keys() -%}
{% if key.startswith(sla_key_main) -%}
{% do sla_dict.__setitem__(key[sla_key_main_length:], sla_parameter_dict.get(key)) -%}
{% elif key.startswith(sla_key_secondary) and not sla_dict.has_key(key[sla_key_secondary_length:]) -%}
{% do sla_dict.__setitem__(key[sla_key_secondary_length:], sla_parameter_dict.get(key)) -%}
{% endif -%}
{% endfor -%}
{% if sla_dict %}
sla = {{ ' '.join(sla_dict.keys()) }}
{% for key, value in sla_dict.iteritems() -%}
sla-{{ key }} = {{ value }}
{% endfor -%}
{% endif -%}
{% endif -%}
{% for id in range(1,nbbackup|int) %} {% for id in range(1,nbbackup|int) %}
...@@ -45,12 +86,29 @@ config-number = {{id}} ...@@ -45,12 +86,29 @@ config-number = {{id}}
config-authorized-key = ${request-pbs-{{namebase}}-{{id}}:connection-ssh-key} config-authorized-key = ${request-pbs-{{namebase}}-{{id}}:connection-ssh-key}
config-on-notification = ${request-pbs-{{namebase}}-{{id}}:connection-feeds-url}${:pbs-notification-id} config-on-notification = ${request-pbs-{{namebase}}-{{id}}:connection-feeds-url}${:pbs-notification-id}
config-ip-list = config-ip-list =
{% if sla_parameter_dict -%}
sla = computer_guid {% set sla_key_main = "-sla-%s%s-" % (namebase, id) -%}
sla-computer_guid = ${slap-parameter:{{namebase}}{{id}}-computer-guid} {% set sla_key_secondary = "-sla-%s-" % (id) -%}
{% set sla_key_main_length = sla_key_main | length -%}
{% set sla_key_secondary_length = sla_key_secondary | length -%}
{% endfor %} {% set sla_dict = {} -%}
{% for key in sla_parameter_dict.keys() -%}
{% if key.startswith(sla_key_main) -%}
{% do sla_dict.__setitem__(key[sla_key_main_length:], sla_parameter_dict.get(key)) -%}
{% elif key.startswith(sla_key_secondary) and not sla_dict.has_key(key[sla_key_secondary_length:]) -%}
{% do sla_dict.__setitem__(key[sla_key_secondary_length:], sla_parameter_dict.get(key)) -%}
{% endif -%}
{% endfor -%}
{% if sla_dict %}
sla = {{ ' '.join(sla_dict.keys()) }}
{% for key, value in sla_dict.iteritems() -%}
sla-{{ key }} = {{ value }}
{% endfor -%}
{% endif %}
{% endif %}
{% endfor -%}
[iplist] [iplist]
config-ip-list = ${request-{{namebase}}:connection-ip}{% for j in range(1,nbbackup|int) %} ${request-{{namebase}}-pseudo-replicating-{{j}}:connection-ip}{% endfor %} config-ip-list = ${request-{{namebase}}:connection-ip}{% for j in range(1,nbbackup|int) %} ${request-{{namebase}}-pseudo-replicating-{{j}}:connection-ip}{% endfor %}
...@@ -90,8 +148,27 @@ software-type = pull-backup ...@@ -90,8 +148,27 @@ software-type = pull-backup
name = PBS ({{namebase}} / {{id}}) name = PBS ({{namebase}} / {{id}})
return = ssh-key notification-url feeds-url return = ssh-key notification-url feeds-url
slave = false slave = false
sla = computer_guid {% if sla_parameter_dict -%}
sla-computer_guid = ${slap-parameter:pbs-{{namebase}}{{id}}-computer-guid} {% set sla_key_main = "-sla-%s%s-" % ("pbs", id) -%}
{% set sla_key_secondary = "-sla-%s-" % (id) -%}
{% set sla_key_main_length = sla_key_main | length -%}
{% set sla_key_secondary_length = sla_key_secondary | length -%}
{% set sla_dict = {} -%}
{% for key in sla_parameter_dict.keys() -%}
{% if key.startswith(sla_key_main) -%}
{% do sla_dict.__setitem__(key[sla_key_main_length:], sla_parameter_dict.get(key)) -%}
{% elif key.startswith(sla_key_secondary) and not sla_dict.has_key(key[sla_key_secondary_length:]) -%}
{% do sla_dict.__setitem__(key[sla_key_secondary_length:], sla_parameter_dict.get(key)) -%}
{% endif -%}
{% endfor -%}
{% if sla_dict %}
sla = {{ ' '.join(sla_dict.keys()) }}
{% for key, value in sla_dict.iteritems() -%}
sla-{{ key }} = {{ value }}
{% endfor %}
{% endif %}
{% endif %}
[request-pull-backup-server-{{namebase}}-{{id}}] [request-pull-backup-server-{{namebase}}-{{id}}]
<= request-pbs-common <= request-pbs-common
...@@ -135,3 +212,4 @@ pbs-{{namebase}}{{id}}-computer-guid = ...@@ -135,3 +212,4 @@ pbs-{{namebase}}{{id}}-computer-guid =
{% endfor %} {% endfor %}
{% endmacro %} {% endmacro %}
...@@ -78,6 +78,9 @@ zc.buildout = 1.6.0-dev-SlapOS-010 ...@@ -78,6 +78,9 @@ zc.buildout = 1.6.0-dev-SlapOS-010
zc.recipe.egg = 1.3.2 zc.recipe.egg = 1.3.2
# Use own version of h.r.download to be able to open xz-like archives # Use own version of h.r.download to be able to open xz-like archives
hexagonit.recipe.download = 1.6nxd002 hexagonit.recipe.download = 1.6nxd002
# Use pinned version of setuptools. Other versions work, but changing
# version makes buildout recompile everything. Developers' nightmare.
setuptools = 0.9.8
[networkcache] [networkcache]
download-cache-url = http://www.shacache.org/shacache download-cache-url = http://www.shacache.org/shacache
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment