Commit dd97008d authored by Kazuhiko Shiozaki's avatar Kazuhiko Shiozaki

Merge branch 'erp5' into erp5-z2.13

parents 5cf88baf 5ff5acd9
...@@ -120,8 +120,9 @@ make-targets = ...@@ -120,8 +120,9 @@ make-targets =
[apache-2.2] [apache-2.2]
# inspired on http://old.aclark.net/team/aclark/blog/a-lamp-buildout-for-wordpress-and-other-php-apps/ # inspired on http://old.aclark.net/team/aclark/blog/a-lamp-buildout-for-wordpress-and-other-php-apps/
recipe = slapos.recipe.cmmi recipe = slapos.recipe.cmmi
url = http://mir2.ovh.net/ftp.apache.org/dist//httpd/httpd-2.2.24.tar.gz version = 2.2.25
md5sum = 64a3392018ad60583209a16d728180d3 url = http://mir2.ovh.net/ftp.apache.org/dist/httpd/httpd-${:version}.tar.bz2
md5sum = 9ebe3070c0bb4311f21a0cd0e34f0045
patch-options = -p1 patch-options = -p1
configure-options = --disable-static configure-options = --disable-static
--enable-authn-alias --enable-authn-alias
...@@ -182,6 +183,8 @@ environment = ...@@ -182,6 +183,8 @@ environment =
recipe = slapos.recipe.cmmi recipe = slapos.recipe.cmmi
url = http://sourceforge.net/projects/mod-antiloris/files/mod_antiloris-0.4.tar.bz2/download url = http://sourceforge.net/projects/mod-antiloris/files/mod_antiloris-0.4.tar.bz2/download
md5sum = 66862bf10e9be3a023e475604a28a0b4 md5sum = 66862bf10e9be3a023e475604a28a0b4
depends =
${apache-2.2:version}
configure-command = ${apache-2.2:location}/bin/apxs configure-command = ${apache-2.2:location}/bin/apxs
configure-options = -c mod_antiloris.c configure-options = -c mod_antiloris.c
make-binary = ${:configure-command} make-binary = ${:configure-command}
......
...@@ -14,8 +14,8 @@ parts = ...@@ -14,8 +14,8 @@ parts =
[nodejs-0.8] [nodejs-0.8]
# Server-side Javascript. # Server-side Javascript.
recipe = slapos.recipe.cmmi recipe = slapos.recipe.cmmi
url = http://nodejs.org/dist/v0.8.14/node-v0.8.14.tar.gz url = http://nodejs.org/dist/v0.8.23/node-v0.8.23.tar.gz
md5sum = 284fd2c7578064c339d9cf6a3a475ac7 md5sum = 22fe54ac365f52d3d80ecf748e7323d5
configure-options = configure-options =
--openssl-includes=${openssl:location}/include --openssl-includes=${openssl:location}/include
--openssl-libpath=${openssl:location}/lib --openssl-libpath=${openssl:location}/lib
......
...@@ -188,6 +188,8 @@ setup(name=name, ...@@ -188,6 +188,8 @@ setup(name=name,
'slapreport = slapos.recipe.slapreport:Recipe', 'slapreport = slapos.recipe.slapreport:Recipe',
'slaprunner = slapos.recipe.slaprunner:Recipe', 'slaprunner = slapos.recipe.slaprunner:Recipe',
'slaprunner.test = slapos.recipe.slaprunner:Test', 'slaprunner.test = slapos.recipe.slaprunner:Test',
'slaprunner.export = slapos.recipe.slaprunner.backup:ExportRecipe',
'slaprunner.import = slapos.recipe.slaprunner.backup:ImportRecipe',
'softwaretype = slapos.recipe.softwaretype:Recipe', 'softwaretype = slapos.recipe.softwaretype:Recipe',
'sphinx= slapos.recipe.sphinx:Recipe', 'sphinx= slapos.recipe.sphinx:Recipe',
'sshkeys_authority = slapos.recipe.sshkeys_authority:Recipe', 'sshkeys_authority = slapos.recipe.sshkeys_authority:Recipe',
......
...@@ -50,8 +50,6 @@ class NeoBaseRecipe(GenericBaseRecipe): ...@@ -50,8 +50,6 @@ class NeoBaseRecipe(GenericBaseRecipe):
#'-n', options['name'], #'-n', options['name'],
'-c', options['cluster'], '-c', options['cluster'],
] ]
if options['verbose']:
option_list.append('-v')
option_list.extend(self._getOptionList()) option_list.extend(self._getOptionList())
return [self.createPythonScript( return [self.createPythonScript(
options['wrapper'], options['wrapper'],
......
##############################################################################
#
# Copyright (c) 2013 Vifib SARL and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
import textwrap
from slapos.recipe.librecipe import GenericBaseRecipe
class ExportRecipe(GenericBaseRecipe):
"""\
This recipe creates an exporter script for using with the resilient stack.
Required options:
backup-directory
folder that will contain the dump file.
srv-directory
folder that contain the runner directory.
wrapper
full path of the exporter script to create.
"""
def install(self):
wrapper = self.options['wrapper']
self.createBackupScript(wrapper)
return [wrapper]
def createBackupScript(self, wrapper):
"""\
Create a script to backup the database in 'custom' format.
"""
content = textwrap.dedent("""\
#!%(shell-binary)s
umask 077
sync_element () {
path=$1
backup_path=$2
shift 2
element_list=$*
for element in $element_list
do
cd $path;
if [ -f $element ] || [ -d $element ]; then
%(rsync-binary)s -avz --safe-links --delete $element $backup_path;
fi
done
}
sync_element %(srv-directory)s/runner %(backup-directory)s/runner/ instance project proxy.db softwareLink
sync_element %(etc-directory)s %(backup-directory)s/etc/ .rcode .project .users ssh
if [ -d %(backup-directory)s/runner/software ]; then
rm %(backup-directory)s/runner/software/*
fi
""" % self.options)
self.createExecutable(wrapper, content=content)
class ImportRecipe(GenericBaseRecipe):
"""\
This recipe creates an importer script for using with the resilient stack.
Required options:
backup-directory
folder that will contain the dump file.
srv-directory
folder that contain the runner directory.
wrapper
full path of the exporter script to create.
"""
def install(self):
wrapper = self.options['wrapper']
self.createRestoreScript(wrapper)
return [wrapper]
def createRestoreScript(self, wrapper):
"""\
Create a script to restore the database from 'custom' format.
"""
content = textwrap.dedent("""\
#!%(shell-binary)s
umask 077
restore_element () {
backup_path=$1
restore_path=$2
shift 2
element_list=$*
for element in $element_list
do
cd $backup_path;
if [ -f $element ] || [ -d $element ]; then
%(rsync-binary)s -avz --delete $backup_path/$element $restore_path;
fi
done
}
restore_element %(backup-directory)s/runner/ %(srv-directory)s/runner instance project proxy.db softwareLink
restore_element %(backup-directory)s/etc/ %(etc-directory)s .rcode .project .users ssh
ifs=$IFS IFS=';'
read user pass remaining < %(etc-directory)s/.users
IFS=$ifs
%(curl-binary)s -vg6L -F clogin="$user" -F cpwd="$pass" --dump-header login_cookie %(backend-url)s/doLogin;
%(curl-binary)s -vg6LX POST --cookie login_cookie --max-time 5 %(backend-url)s/runSoftwareProfile;
rm -f login_cookie
""" % self.options)
self.createExecutable(wrapper, content=content)
...@@ -37,11 +37,6 @@ Defautls to 12. ...@@ -37,11 +37,6 @@ Defautls to 12.
Number of replicates. Number of replicates.
Defaults to 0 (no resilience). Defaults to 0 (no resilience).
'verbose' (int, optional)
~~~~~~~~~~~~~~~~~~~~~~~~~
Non-zero to increase verbosity.
Defaults to 0.
'mysql-storage-count' (int, optional) 'mysql-storage-count' (int, optional)
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Number of MySQL-based storage nodes to deploy. One master node is deployed Number of MySQL-based storage nodes to deploy. One master node is deployed
......
...@@ -19,10 +19,9 @@ partition-id = ${slap-connection:partition-id} ...@@ -19,10 +19,9 @@ partition-id = ${slap-connection:partition-id}
config-cluster = {{ slapparameter_dict['cluster'] }} config-cluster = {{ slapparameter_dict['cluster'] }}
config-partitions = {{ slapparameter_dict.get('partitions', 12) }} config-partitions = {{ slapparameter_dict.get('partitions', 12) }}
config-replicas = {{ slapparameter_dict.get('replicas', 0) }} config-replicas = {{ slapparameter_dict.get('replicas', 0) }}
config-verbose = {{ dumps(slapparameter_dict.get('verbose', 0)) }}
[request-config] [request-config]
base = cluster verbose base = cluster
master = partitions replicas master = partitions replicas
node = masters node = masters
......
...@@ -16,7 +16,6 @@ ip = ${publish:ip} ...@@ -16,7 +16,6 @@ ip = ${publish:ip}
port = ${publish:port} port = ${publish:port}
cluster = ${slap-parameter:cluster} cluster = ${slap-parameter:cluster}
masters = ${slap-parameter:masters} masters = ${slap-parameter:masters}
verbose = ${slap-parameter:verbose}
[publish] [publish]
recipe = slapos.cookbook:publish recipe = slapos.cookbook:publish
...@@ -33,6 +32,7 @@ link-binary = ${neo-admin:wrapper} ...@@ -33,6 +32,7 @@ link-binary = ${neo-admin:wrapper}
[directory] [directory]
recipe = slapos.cookbook:mkdirectory recipe = slapos.cookbook:mkdirectory
promises = ${buildout:directory}/etc/promises
run = ${buildout:directory}/etc/run run = ${buildout:directory}/etc/run
log = ${buildout:directory}/var/log log = ${buildout:directory}/var/log
bin = ${buildout:directory}/bin bin = ${buildout:directory}/bin
......
...@@ -17,7 +17,6 @@ cluster = ${slap-parameter:cluster} ...@@ -17,7 +17,6 @@ cluster = ${slap-parameter:cluster}
partitions = ${slap-parameter:partitions} partitions = ${slap-parameter:partitions}
replicas = ${slap-parameter:replicas} replicas = ${slap-parameter:replicas}
masters = ${slap-parameter:masters} masters = ${slap-parameter:masters}
verbose = ${slap-parameter:verbose}
[publish] [publish]
recipe = slapos.cookbook:publish recipe = slapos.cookbook:publish
...@@ -34,6 +33,7 @@ link-binary = ${neo-master:wrapper} ...@@ -34,6 +33,7 @@ link-binary = ${neo-master:wrapper}
[directory] [directory]
recipe = slapos.cookbook:mkdirectory recipe = slapos.cookbook:mkdirectory
promises = ${buildout:directory}/etc/promises
run = ${buildout:directory}/etc/run run = ${buildout:directory}/etc/run
log = ${buildout:directory}/var/log log = ${buildout:directory}/var/log
bin = ${buildout:directory}/bin bin = ${buildout:directory}/bin
......
...@@ -10,6 +10,7 @@ database = neo ...@@ -10,6 +10,7 @@ database = neo
conf-file = ${directory:etc}/mariadb.cnf conf-file = ${directory:etc}/mariadb.cnf
# No networking (ip & port options) # No networking (ip & port options)
data-directory = ${directory:srv_mariadb} data-directory = ${directory:srv_mariadb}
bin-directory = ${directory:bin}
pid-file = ${directory:var_run}/mariadb.pid pid-file = ${directory:var_run}/mariadb.pid
socket = ${directory:var_run}/mariadb.sock socket = ${directory:var_run}/mariadb.sock
error-log = ${directory:log}/mariadb_error.log error-log = ${directory:log}/mariadb_error.log
...@@ -33,7 +34,6 @@ logfile = ${directory:log}/neostorage.log ...@@ -33,7 +34,6 @@ logfile = ${directory:log}/neostorage.log
ip = [${slap-network-information:global-ipv6}] ip = [${slap-network-information:global-ipv6}]
cluster = ${slap-parameter:cluster} cluster = ${slap-parameter:cluster}
masters = ${slap-parameter:masters} masters = ${slap-parameter:masters}
verbose = ${slap-parameter:verbose}
database-adapter = MySQL database-adapter = MySQL
database-parameters = ${mariadb-instance:user}:${mariadb-instance:password}@${mariadb-instance:database}${mariadb-instance:socket} database-parameters = ${mariadb-instance:user}:${mariadb-instance:password}@${mariadb-instance:database}${mariadb-instance:socket}
wait-database = 60 wait-database = 60
......
# Note on COOKBOOK/END COOKBOOK: the areas delimited by those tags should be
# dropped when a cookbook version containing neoppod recipe is released as an
# egg on pypi.
# Note on LXML/END LXML: they delimit areas where lxml magic is needed. lxml is # Note on LXML/END LXML: they delimit areas where lxml magic is needed. lxml is
# a slapos.cookbook dependency, so it should be fetched automatically. But when # a slapos.cookbook dependency, so it should be fetched automatically. But when
# automatically fetched, it gets built against system headers/libs, which is # automatically fetched, it gets built against system headers/libs, which is
...@@ -10,9 +7,6 @@ ...@@ -10,9 +7,6 @@
[buildout] [buildout]
extends = extends =
../../stack/slapos.cfg ../../stack/slapos.cfg
#COOKBOOK
../../component/git/buildout.cfg
#END COOKBOOK
#LXML #LXML
../../component/lxml-python/buildout.cfg ../../component/lxml-python/buildout.cfg
#END LXML #END LXML
...@@ -20,19 +14,9 @@ extends = ...@@ -20,19 +14,9 @@ extends =
../../component/mariadb/buildout.cfg ../../component/mariadb/buildout.cfg
../../component/mysql-python/buildout.cfg ../../component/mysql-python/buildout.cfg
develop =
#COOKBOOK
# Note: as we develop a path we fetch, first buildout execution will fail as
# develop couldn't find the clone. Second (and later) runs will succeed.
${:parts-directory}/slapos.cookbook-repository
#END COOKBOOK
parts = parts =
#COOKBOOK
slapos.cookbook-repository
check-recipe
#END COOKBOOK
slapos-deps-eggs slapos-deps-eggs
slapos-cookbook
# NEO & dependencies # NEO & dependencies
python2.7 python2.7
mariadb mariadb
...@@ -47,29 +31,12 @@ eggs = ...@@ -47,29 +31,12 @@ eggs =
#LXML #LXML
${lxml-python:egg} ${lxml-python:egg}
#END LXML #END LXML
#COOKBOOK
slapos.cookbook
#END COOKBOOK
#LXML #LXML
[lxml-python] [lxml-python]
python = python2.7 python = python2.7
#END LXML #END LXML
#COOKBOOK
[slapos.cookbook-repository]
recipe = slapos.recipe.build:gitclone
repository = http://git.erp5.org/repos/slapos.git
branch = neoppod
git-executable = ${git:location}/bin/git
[check-recipe]
recipe = plone.recipe.command
stop-on-error = true
update-command = ${:command}
command = test -e ${buildout:develop-eggs-directory}/slapos.cookbook.egg-link
#END COOKBOOK
[base-template] [base-template]
recipe = slapos.recipe.template:jinja2 recipe = slapos.recipe.template:jinja2
template = ${:_profile_base_location_}/${:filename}.in template = ${:_profile_base_location_}/${:filename}.in
...@@ -85,12 +52,12 @@ context = ...@@ -85,12 +52,12 @@ context =
[template-neo-master] [template-neo-master]
< = base-template < = base-template
filename = instance-neo-master.cfg filename = instance-neo-master.cfg
md5sum = b48e9446fb8fdce24d954d7d4f4c7460 md5sum = 8947620b15535b95163c11a0efab546e
[template-neo-storage-mysql] [template-neo-storage-mysql]
< = base-template < = base-template
filename = instance-neo-storage-mysql.cfg filename = instance-neo-storage-mysql.cfg
md5sum = e767c66e7de58232de30bdf26188efe1 md5sum = 518c8c41ce73f0b608fbb1c3b889ca1b
extra-context = extra-context =
key mariadb_location mariadb:location key mariadb_location mariadb:location
key master_cfg template-neo-master:rendered key master_cfg template-neo-master:rendered
...@@ -98,12 +65,12 @@ extra-context = ...@@ -98,12 +65,12 @@ extra-context =
[template-neo-admin] [template-neo-admin]
< = base-template < = base-template
filename = instance-neo-admin.cfg filename = instance-neo-admin.cfg
md5sum = fcf1a5c937a609a934c8ccd7c1326726 md5sum = 987f69333c563898cab21455509f8024
[template-cluster] [template-cluster]
recipe = slapos.recipe.build:download recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/cluster.cfg.in url = ${:_profile_base_location_}/cluster.cfg.in
md5sum = fe37b2ff36fee65b44945393f63e51c3 md5sum = 7177d129e171b20044bf435ed549af43
[template-instance] [template-instance]
< = base-template < = base-template
......
- Use https to connect to the runner. !!! Critical
- resilient sr: Cloned instances should not launch slapgrid-sr if it was not launched on export instance
[buildout] [buildout]
extends = extends =
../../component/bash/buildout.cfg
../../component/cloud9/buildout.cfg ../../component/cloud9/buildout.cfg
../../component/curl/buildout.cfg
../../component/dash/buildout.cfg ../../component/dash/buildout.cfg
../../component/dropbear/buildout.cfg ../../component/dropbear/buildout.cfg
../../component/git/buildout.cfg ../../component/git/buildout.cfg
../../component/lxml-python/buildout.cfg ../../component/lxml-python/buildout.cfg
../../component/rsync/buildout.cfg
../../stack/flask.cfg ../../stack/flask.cfg
../../stack/shacache-client.cfg ../../stack/shacache-client.cfg
../../stack/resilient/buildout.cfg
../../stack/slapos.cfg ../../stack/slapos.cfg
parts = parts =
slapos.cookbook-repository
rdiff-backup
template template
slapos-cookbook
eggs eggs
instance-runner-import
instance-runner-export
# slapos-cookbook
[template] [template]
recipe = slapos.recipe.template recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance.cfg url = ${:_profile_base_location_}/instance.cfg
output = ${buildout:directory}/template.cfg output = ${buildout:directory}/template.cfg
mode = 0644 mode = 0644
md5sum = 5307e4200f044ae57b504ad68444491c #md5sum = 5307e4200f044ae57b504ad68444491c
[template-runner] [template-runner]
recipe = slapos.recipe.template recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance-runner.cfg url = ${:_profile_base_location_}/instance-runner.cfg
output = ${buildout:directory}/template-runner.cfg output = ${buildout:directory}/template-runner.cfg
md5sum = 91d6550c43b7a43a999724af4650ae40 #md5sum = 91d6550c43b7a43a999724af4650ae40
mode = 0644
[instance-resilient]
recipe = slapos.recipe.template:jinja2
template = ${:_profile_base_location_}/instance-resilient.cfg.jinja2
rendered = ${buildout:directory}/instance-resilient.cfg
context = key buildout buildout:bin-directory
key develop_eggs_directory buildout:develop-eggs-directory
key eggs_directory buildout:eggs-directory
import-list = file parts template-parts:destination
file replicated template-replicated:destination
mode = 0644
[instance-runner-import]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance-runner-import.cfg.in
output = ${buildout:directory}/instance-runner-import.cfg
mode = 0644
[instance-runner-export]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance-runner-export.cfg.in
output = ${buildout:directory}/instance-runner-export.cfg
mode = 0644 mode = 0644
[eggs] [eggs]
......
...@@ -10,39 +10,43 @@ extends = common.cfg ...@@ -10,39 +10,43 @@ extends = common.cfg
parts += parts +=
slapos.cookbook-repository slapos.cookbook-repository
slapos.core-repository
slapos.toolbox-repository
check-recipe # slapos.toolbox-repository
# slapos.core-repository
# check-recipe
develop = develop =
${:parts-directory}/slapos.cookbook-repository ${:parts-directory}/slapos.cookbook-repository
${:parts-directory}/slapos.core-repository # ${:parts-directory}/slapos.toolbox-repository
${:parts-directory}/slapos.toolbox-repository # ${:parts-directory}/slapos.core-repository
[slapos.toolbox-repository] #[slapos.toolbox-repository]
recipe = slapos.recipe.build:gitclone #recipe = slapos.recipe.build:gitclone
repository = http://git.erp5.org/repos/slapos.toolbox.git #repository = http://git.erp5.org/repos/slapos.toolbox.git
branch = slaprunner #branch = slaprunner-resiliency
git-executable = ${git:location}/bin/git #git-executable = ${git:location}/bin/git
[slapos.cookbook-repository] [slapos.cookbook-repository]
recipe = slapos.recipe.build:gitclone recipe = slapos.recipe.build:gitclone
repository = http://git.erp5.org/repos/slapos.git repository = http://git.erp5.org/repos/slapos.git
branch = slaprunner branch = slaprunner-resiliency
git-executable = ${git:location}/bin/git
[slapos.core-repository]
recipe = slapos.recipe.build:gitclone
repository = http://git.erp5.org/repos/slapos.core.git
branch = master
git-executable = ${git:location}/bin/git git-executable = ${git:location}/bin/git
#[slapos.core-repository]
#recipe = slapos.recipe.build:gitclone
#repository = http://git.erp5.org/repos/slapos.core.git
#branch = master
#git-executable = ${git:location}/bin/git
#
[check-recipe] [check-recipe]
recipe = plone.recipe.command recipe = plone.recipe.command
stop-on-error = true stop-on-error = true
update-command = ${:command} update-command = ${:command}
command = command =
grep parts ${buildout:develop-eggs-directory}/slapos.cookbook.egg-link && grep parts ${buildout:develop-eggs-directory}/slapos.cookbook.egg-link &&
grep parts ${buildout:develop-eggs-directory}/slapos.core.egg-link &&
grep parts ${buildout:develop-eggs-directory}/slapos.toolbox.egg-link grep parts ${buildout:develop-eggs-directory}/slapos.toolbox.egg-link
# grep parts ${buildout:develop-eggs-directory}/slapos.core.egg-link &&
# vim: set ft=cfg:
{% import 'parts' as parts %}
{% import 'replicated' as replicated %}
[buildout]
eggs-directory = {{ eggs_directory }}
develop-eggs-directory = {{ develop_eggs_directory }}
offline = true
# += because we need to take up parts (like instance-custom, slapmonitor etc) from the profile we extended
parts +=
{{ parts.replicate("runner", "3") }}
publish-connection-informations
{{ replicated.replicate("runner", "3", "runner-export", "runner-import") }}
# Bubble up the parameters
[request-runner]
return = url ssh-public-key ssh-url notification-id ip backend_url url cloud9_url ssh_command password_recovery_code
config = instance-amount debug domain number authorized-key notify ip-list namebase runner1-computer-guid pbs-runner1-computer-guid runner2-computer-guid pbs-runner2-computer-guid runner3-computer-guid pbs-runner3-computer-guid
# XXX Cedric LN Ugly hack, resilient stack and slaprunner stack sharing too much ssh sections
config-authorized-key = ${request-pbs-runner-1:connection-ssh-key} ${request-pbs-runner-2:connection-ssh-key} ${slap-parameter:authorized-key}
config-instance-amount = ${slap-parameter:instance-amount}
config-debug = ${slap-parameter:debug}
config-runner1-computer-guid = ${slap-parameter:runner1-computer-guid}
config-pbs-runner1-computer-guid = ${slap-parameter:pbs-runner1-computer-guid}
config-runner2-computer-guid = ${slap-parameter:runner2-computer-guid}
config-pbs-runner2-computer-guid = ${slap-parameter:pbs-runner2-computer-guid}
config-runner3-computer-guid = ${slap-parameter:runner3-computer-guid}
config-pbs-runner3-computer-guid = ${slap-parameter:pbs-runner3-computer-guid}
config-domain = ${slap-parameter:domain}
[publish-connection-informations]
recipe = slapos.cookbook:publish
backend_url = ${request-runner:connection-backend_url}
url = ${request-runner:connection-url}
cloud9_url = ${request-runner:connection-cloud9_url}
ssh_command = ${request-runner:connection-ssh_command}
password_recovery_code = ${request-runner:connection-password_recovery_code}
[slap-parameter]
# Default parameters for distributed deployment
# I.e state "backup1 of maria should go there, ..."
# XXX-Cedric: Hardcoded number of backups. Should be dynamically generated.
runner1-computer-guid =
pbs-runner1-computer-guid =
runner2-computer-guid =
pbs-runner2-computer-guid =
runner3-computer-guid =
pbs-runner3-computer-guid =
# XXX-Cedric: Hardcoded parameters. Should be dynamically generated.
domain =
authorized-key =
instance-amount = 10
debug = false
[buildout]
extends = ${template-runner:output}
${pbsready-export:output}
parts +=
urls
slaprunner
cron-entry-backup
[exporter]
recipe = slapos.cookbook:slaprunner.export
wrapper = $${directory:bin}/$${slap-parameter:namebase}-exporter
srv-directory = $${directory:srv}
etc-directory = $${directory:etc}
backup-directory = $${directory:backup}
shell-binary = ${dash:location}/bin/dash
rsync-binary = ${rsync:location}/bin/rsync
# Extends publish section with resilient parameters
[urls]
<= resilient-publish-connection-parameter
backend_url = $${slaprunner:access-url}
url = $${request-frontend:connection-site_url}
cloud9_url = $${cloud9:access-url}
ssh_command = ssh $${dropbear-runner-server:host} -p $${dropbear-runner-server:port}
password_recovery_code = $${recovery-code:passwd}
[buildout]
extends = ${template-runner:output}
${pbsready-import:output}
parts +=
slaprunner
# have to repeat the next one, as it's not inherited from pbsready-import
import-on-notification
[importer]
recipe = slapos.cookbook:slaprunner.import
wrapper = $${directory:bin}/$${slap-parameter:namebase}-importer
srv-directory = $${directory:srv}
etc-directory = $${directory:etc}
backup-directory = $${directory:backup}
shell-binary = ${dash:location}/bin/dash
rsync-binary = ${rsync:location}/bin/rsync
curl-binary = ${curl:location}/bin/curl
backend-url = $${slaprunner:access-url}
...@@ -3,7 +3,7 @@ parts = ...@@ -3,7 +3,7 @@ parts =
cloud9 cloud9
slaprunner slaprunner
test-runner test-runner
sshkeys-dropbear sshkeys-dropbear-runner
dropbear-server-add-authorized-key dropbear-server-add-authorized-key
sshkeys-authority sshkeys-authority
publish-connection-informations publish-connection-informations
...@@ -13,7 +13,6 @@ parts = ...@@ -13,7 +13,6 @@ parts =
dropbear-promise dropbear-promise
symlinks symlinks
eggs-directory = ${buildout:eggs-directory} eggs-directory = ${buildout:eggs-directory}
develop-eggs-directory = ${buildout:develop-eggs-directory} develop-eggs-directory = ${buildout:develop-eggs-directory}
offline = true offline = true
...@@ -82,9 +81,9 @@ instance-directory = $${runnerdirectory:instance-root} ...@@ -82,9 +81,9 @@ instance-directory = $${runnerdirectory:instance-root}
etc_dir = $${directory:etc} etc_dir = $${directory:etc}
log_dir = $${directory:log} log_dir = $${directory:log}
run_dir = $${directory:run} run_dir = $${directory:run}
ssh_client = $${sshkeys-dropbear:wrapper} ssh_client = $${sshkeys-dropbear-runner:wrapper}
public_key = $${sshkeys-dropbear:public-key} public_key = $${sshkeys-dropbear-runner:public-key}
private_key = $${sshkeys-dropbear:private-key} private_key = $${sshkeys-dropbear-runner:private-key}
ipv4 = $${slap-network-information:local-ipv4} ipv4 = $${slap-network-information:local-ipv4}
ipv6 = $${slap-network-information:global-ipv6} ipv6 = $${slap-network-information:global-ipv6}
proxy_port = 50000 proxy_port = 50000
...@@ -121,28 +120,28 @@ keys-directory = $${sshkeys-directory:keys} ...@@ -121,28 +120,28 @@ keys-directory = $${sshkeys-directory:keys}
wrapper = $${directory:services}/sshkeys_authority wrapper = $${directory:services}/sshkeys_authority
keygen-binary = ${dropbear:location}/bin/dropbearkey keygen-binary = ${dropbear:location}/bin/dropbearkey
[dropbear-server] [dropbear-runner-server]
recipe = slapos.cookbook:dropbear recipe = slapos.cookbook:dropbear
host = $${slap-network-information:global-ipv6} host = $${slap-network-information:global-ipv6}
port = 2222 port = 22222
home = $${directory:ssh} home = $${directory:ssh}
wrapper = $${directory:bin}/raw_sshd wrapper = $${directory:bin}/runner_sshd
shell = /bin/bash shell = ${bash:location}/bin/bash
rsa-keyfile = $${directory:ssh}/server_key.rsa rsa-keyfile = $${directory:ssh}/server_key.rsa
dropbear-binary = ${dropbear:location}/sbin/dropbear dropbear-binary = ${dropbear:location}/sbin/dropbear
[sshkeys-dropbear] [sshkeys-dropbear-runner]
<= sshkeys-authority <= sshkeys-authority
recipe = slapos.cookbook:sshkeys_authority.request recipe = slapos.cookbook:sshkeys_authority.request
name = dropbear name = dropbear
type = rsa type = rsa
executable = $${dropbear-server:wrapper} executable = $${dropbear-runner-server:wrapper}
public-key = $${dropbear-server:rsa-keyfile}.pub public-key = $${dropbear-runner-server:rsa-keyfile}.pub
private-key = $${dropbear-server:rsa-keyfile} private-key = $${dropbear-runner-server:rsa-keyfile}
wrapper = $${directory:services}/sshd wrapper = $${directory:services}/runner_sshd
[dropbear-server-add-authorized-key] [dropbear-server-add-authorized-key]
<= dropbear-server <= dropbear-runner-server
recipe = slapos.cookbook:dropbear.add_authorized_key recipe = slapos.cookbook:dropbear.add_authorized_key
key = $${slap-parameter:authorized-key} key = $${slap-parameter:authorized-key}
...@@ -166,7 +165,7 @@ recipe = slapos.cookbook:publish ...@@ -166,7 +165,7 @@ recipe = slapos.cookbook:publish
backend_url = $${slaprunner:access-url} backend_url = $${slaprunner:access-url}
url = $${request-frontend:connection-site_url} url = $${request-frontend:connection-site_url}
cloud9_url = $${cloud9:access-url} cloud9_url = $${cloud9:access-url}
ssh_command = ssh $${dropbear-server:host} -p $${dropbear-server:port} ssh_command = ssh $${dropbear-runner-server:host} -p $${dropbear-runner-server:port}
password_recovery_code = $${recovery-code:passwd} password_recovery_code = $${recovery-code:passwd}
...@@ -194,8 +193,8 @@ curl_path = ${curl:location}/bin/curl ...@@ -194,8 +193,8 @@ curl_path = ${curl:location}/bin/curl
[dropbear-promise] [dropbear-promise]
recipe = slapos.cookbook:check_port_listening recipe = slapos.cookbook:check_port_listening
path = $${directory:promises}/dropbear path = $${directory:promises}/dropbear
hostname = $${dropbear-server:host} hostname = $${dropbear-runner-server:host}
port = $${dropbear-server:port} port = $${dropbear-runner-server:port}
[symlinks] [symlinks]
recipe = cns.recipe.symlink recipe = cns.recipe.symlink
......
...@@ -8,4 +8,11 @@ offline = true ...@@ -8,4 +8,11 @@ offline = true
[switch_softwaretype] [switch_softwaretype]
recipe = slapos.cookbook:softwaretype recipe = slapos.cookbook:softwaretype
default = ${template-runner:output} default = ${template-runner:output}
\ No newline at end of file resilient = ${instance-resilient:rendered}
runner = ${template-runner:output}
runner-import = ${instance-runner-import:output}
runner-export = ${instance-runner-export:output}
frozen = ${instance-frozen:output}
pull-backup = ${template-pull-backup:output}
...@@ -5,7 +5,7 @@ ...@@ -5,7 +5,7 @@
# 2/ Define list of trusted certificates for the cache. # 2/ Define list of trusted certificates for the cache.
[buildout] [buildout]
extends = common.cfg extends = development.cfg
[networkcache] [networkcache]
# signature certificates of the following uploaders. # signature certificates of the following uploaders.
...@@ -56,12 +56,12 @@ netaddr = 0.7.10 ...@@ -56,12 +56,12 @@ netaddr = 0.7.10
plone.recipe.command = 1.1 plone.recipe.command = 1.1
pycrypto = 2.6 pycrypto = 2.6
pytz = 2012j pytz = 2012j
slapos.cookbook = 0.71.1 #slapos.cookbook = 0.71.1
slapos.core = 0.34 slapos.core = 0.35.1
slapos.libnetworkcache = 0.13.3 slapos.libnetworkcache = 0.13.4
slapos.recipe.build = 0.11.5 slapos.recipe.build = 0.11.5
slapos.recipe.template = 2.4.2 slapos.recipe.template = 2.4.2
slapos.toolbox = 0.34.0 #slapos.toolbox = 0.34.0
smmap = 0.8.2 smmap = 0.8.2
xml-marshaller = 0.9.7 xml-marshaller = 0.9.7
z3c.recipe.scripts = 1.0.1 z3c.recipe.scripts = 1.0.1
......
...@@ -127,6 +127,27 @@ wrapper. I suggest you only add options and specify your export/import recipe. ...@@ -127,6 +127,27 @@ wrapper. I suggest you only add options and specify your export/import recipe.
Checking that it works
----------------------
To check that your software instance is resilient you can proceed this way:
Once all instances are successfully deployed, go to your export instance, connect as the instance user and run:
$ ~/bin/exporter
It is the script responsible for triggering the resiliency stack on your instance. After doing a backup of your data, it will notify the pull-backup instances of a new backup, triggering the transfer of this data to the import instances.
Once this script is run successfully, go to your import instance, connect as its instance user and check ~/srv/backup/"your sofwtare"/, the location of the data you wanted to receive. The last part of the resiliency is up to your import script.
DEBUGGING:
Here is a partial list of things you can check to understand what is causing the problem:
- Check that your import script does not fail and successfully places your data in ~/srv/backup/"your software" (as the import instance user) by runnig:
$ ~/bin/"your software"-exporter
- Check the export instance script is run successfully as this instance user by running:
$ ~/bin/exporter
- Check the pull-instance system did its job by going to one of your pull-backup instance, connect as its user and check the log : ~/var/log/equeue.log
----------------------------------------------------------------------------------------- -----------------------------------------------------------------------------------------
Finally, instance-mysoftware-import.cfg.in and Finally, instance-mysoftware-import.cfg.in and
......
[buildout] [buildout]
extends =
../../component/gzip/buildout.cfg
../../component/rdiff-backup/buildout.cfg
parts = parts =
rdiff-backup
pbsready pbsready
pbsready-import pbsready-import
pbsready-export pbsready-export
...@@ -9,6 +13,9 @@ parts = ...@@ -9,6 +13,9 @@ parts =
instance-frozen instance-frozen
template-resilient template-resilient
# needed tools for resiliency
gzip
#---------------- #----------------
#-- #--
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment