Commit 1505c512 authored by Antoine Catton's avatar Antoine Catton

Switch lamp stack to pull backup server.

parent dc839285
......@@ -7,16 +7,32 @@ parts =
mariadb
mydumper
mysql-python
duplicity
rdiff-backup
dropbear
eggs
instance-recipe-egg
template
template-apache-php
template-mariadb
template-mariadb-remote-backuped
template-mariadb-pseudo-replicating
template-mariadb-pbsready
template-mariadb-pbsready-import
template-mariadb-pbsready-export
allow-hosts =
*.nexedi.org
*.python.org
*.sourceforge.net
alastairs-place.net
dist.repoze.org
effbot.org
github.com
peak.telecommunity.com
psutil.googlecode.com
www.dabeaz.com
launchpad.net
extends =
../shacache-client.cfg
../../component/mariadb/buildout.cfg
../../component/apache/buildout.cfg
../../component/apache-php/buildout.cfg
......@@ -29,14 +45,19 @@ extends =
../../component/sqlite3/buildout.cfg
../../component/lxml-python/buildout.cfg
../../component/zlib/buildout.cfg
../../component/duplicity/buildout.cfg
../../component/rdiff-backup/buildout.cfg
../../component/gzip/buildout.cfg
../../component/stunnel/buildout.cfg
../../component/mydumper/buildout.cfg
../../component/mysql-python/buildout.cfg
../../component/dropbear/buildout.cfg
versions = versions
[mariadb]
# Compile dir is for plugins, there's no plugin in LAMP
keep-compile-dir = false
[instance-recipe]
egg = slapos.cookbook
module = lamp.request
......@@ -55,42 +76,56 @@ strip-top-level-dir = true
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance.cfg
output = ${buildout:directory}/template.cfg
md5sum = 5cc5d92298e8b36067c9b7721561b81f
md5sum = f4e4cf75225685e1787c0bc6d1c16910
mode = 0644
[template-apache-php]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance-apache-php.cfg
output = ${buildout:directory}/template-apache-php.cfg
md5sum = b7a9c86166c49443ecc4db276ef09dec
md5sum = c6142b3d1606d2c98fedefb81f5a5638
mode = 0644
[template-backuped]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance-backuped.cfg
output = ${buildout:directory}/template-backuped.cfg
md5sum = e5cdaa121a6d66d119278e4cf90e7bb0
md5sum = 4190baa5e3f87a02e8721aeac3b70942
mode = 0644
[template-mariadb]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance-mariadb.cfg
output = ${buildout:directory}/template-mariadb.cfg
md5sum = 52237e8921b070d0de3bb2f6716d40c7
md5sum = d0670508b0c38879a2c7965ad5b2ab57
mode = 0644
[template-mariadb-pbsready]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance-mariadb-pbsready.cfg
output = ${buildout:directory}/template-mariadb-pbsready.cfg
md5sum = cfc87bb3bb1d4aa4ffd11e4b6dfcabe7
mode = 0644
[template-mariadb-pbsready-import]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance-mariadb-pbsready-import.cfg
output = ${buildout:directory}/template-mariadb-pbsready-import.cfg
md5sum = 483796d0e0ebdc86fb6fe6589bb8ed7e
mode = 0644
[template-mariadb-remote-backuped]
[template-mariadb-pbsready-export]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance-mariadb-remote-backuped.cfg
output = ${buildout:directory}/template-mariadb-remote-backuped.cfg
md5sum = 222bb187b5628536a20a809f3faf31f8
url = ${:_profile_base_location_}/instance-mariadb-pbsready-export.cfg
output = ${buildout:directory}/template-mariadb-pbsready-export.cfg
md5sum = 112f259b613eefa94f8cc83a0e2aa9f3
mode = 0644
[template-mariadb-pseudo-replicating]
[template-pull-backup]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance-mariadb-pseudo-replicating.cfg
output = ${buildout:directory}/template-mariadb-pseudo-replicating.cfg
md5sum = e9f1d5f6dd5f7995d0be06863d196de9
url = ${:_profile_base_location_}/instance-pull-backup.cfg
output = ${buildout:directory}/template-pull-backup.cfg
md5sum = 46bd46a0b46f39937ab3d604eeb20441
mode = 0644
[eggs]
......@@ -99,5 +134,15 @@ eggs =
${lxml-python:egg}
[versions]
slapos.cookbook =
setuptools = 0.6c12dev-r88795
zc.buildout = 1.6.0-dev-SlapOS-002
slapos.core =
slapos.cookbook =
collective.recipe.template = 1.8
hexagonit.recipe.cmmi = 1.5.0
hexagonit.recipe.download = 1.5.0
plone.recipe.command = 1.1
netaddr = 0.7.5
xml-marshaller = 0.9.7
......@@ -145,13 +145,13 @@ dcrond-binary = ${dcron:location}/sbin/crond
cron-entries = $${directory:cron-entries}
crontabs = $${directory:crontabs}
cronstamps = $${directory:cronstamps}
catcher = $${cron-simplelogger:binary}
catcher = $${cron-simplelogger:wrapper}
binary = $${basedirectory:services}/crond
[cron-simplelogger]
recipe = slapos.cookbook:simplelogger
binary = $${rootdirectory:bin}/cron_simplelogger
output = $${directory:cronoutput}
wrapper = $${rootdirectory:bin}/cron_simplelogger
log = $${basedirectory:log}/crond.log
[cron-entry-logrotate]
<= cron
......@@ -189,4 +189,3 @@ stunnel-conf = $${rootdirectory:etc}/stunnel/
cronstamps = $${rootdirectory:etc}/cronstamps/
cron-entries = $${rootdirectory:etc}/cron.d/
crontabs = $${rootdirectory:etc}/crontabs/
cronoutput = $${basedirectory:log}/cron-ouput/
......@@ -6,6 +6,9 @@ extends =
parts =
request-mariadb
request-mariadb-pseudo-replicating
request-pull-backup-server
request-pull-backup-server-mariadb
request-pull-backup-server-mariadb-backup
url
apache-php
stunnel
......@@ -17,16 +20,51 @@ parts =
cron
cron-entry-logrotate
[request-pull-backup-server]
<= slap-connection
recipe = slapos.cookbook:request
name = PBS (Pull Backup Server)
software-url = $${slap-connection:software-release-url}
software-type = pull-backup
return = ssh-key
slave = false
[request-mariadb]
software-type = mariadb-remote-backuped
config = backup-url
config-backup-url = $${slap-parameter:backup-url}
software-type = mariadb-pbsready-export
config = authorized-key
config-authorized-key = $${request-pull-backup-server:connection-ssh-key}
return = url ssh-public-key ssh-url
[request-mariadb-pseudo-replicating]
<= slap-connection
recipe = slapos.cookbook:request
name = MariaDB (pseudo-replicate)
name = MariaDB (backup)
software-url = $${slap-connection:software-release-url}
software-type = mariadb-pseudo-replicating
config = backup-url
config-backup-url = $${slap-parameter:backup-url}
software-type = mariadb-pbsready-import
return = url ssh-public-key ssh-url
config = authorized-key
config-authorized-key = $${request-pull-backup-server:connection-ssh-key}
[request-pull-backup-server-mariadb]
<= request-pull-backup-server
slap-connection
name = PBS pulling from MariaDB (backuped)
config = url name type server-key frequency
config-url = $${request-mariadb:connection-ssh-url}
config-name = $${slap-connection:computer-id}-$${slap-connection:partition-id}-mariadb
config-type = pull
config-server-key = $${request-mariadb:connection-ssh-public-key}
config-frequency = 15 0 * * *
slave = true
[request-pull-backup-server-mariadb-backup]
<= request-pull-backup-server
slap-connection
name = PBS pushing on MariaDB (ready to use backup)
config = url name type server-key frequency
config-url = $${request-mariadb-pseudo-replicating:connection-ssh-url}
config-name = $${slap-connection:computer-id}-$${slap-connection:partition-id}-mariadb
config-type = push
config-server-key = $${request-mariadb-pseudo-replicating:connection-ssh-public-key}
config-frequency = 45 0 * * *
slave = true
[buildout]
extends =
${template-mariadb:output}
extends = ${template-mariadb-pbsready:output}
parts =
url
mariadb
duplicity
stunnel
certificate-authority
ca-stunnel
logrotate
logrotate-entry-mariadb
logrotate-entry-stunnel
cron
cron-entry-logrotate
cron-entry-mariadb-backup
cron-entry-duplicity
parts += cron-entry-mariadb-backup
[mydumper]
recipe = slapos.cookbook:mydumper
......@@ -33,39 +19,9 @@ lock-file = $${basedirectory:run}/mysql-backup.lock
binary = $${mydumper:wrapper}
wrapper = $${rootdirectory:bin}/mydumper
[duplicity]
recipe = slapos.cookbook:duplicity
remote-backup = $${slap-parameter:backup-url}
local-directory = $${mydumper:backup-directory}
duplicity-binary = ${buildout:bin-directory}/duplicity
wrapper = $${rootdirectory:bin}/raw_duplicity
recover = false
cache = $${directory:duplicity-cache}
[duplicity-locked]
recipe = slapos.cookbook:lockfile
lock-file = $${mydumper-locked:lock-file}
binary = $${duplicity:wrapper}
wrapper = $${rootdirectory:bin}/duplicity
wait = true
[cron-entry-duplicity]
<= cron
recipe = slapos.cookbook:cron.d
name = remote_backup
frequency = 30 * * * *
command = $${duplicity-locked:wrapper}
[cron-entry-mariadb-backup]
<= cron
recipe = slapos.cookbook:cron.d
name = backup
frequency = 0 * * * *
command = $${mydumper-locked:wrapper}
[basedirectory]
cache = $${rootdirectory:var}/cache/
[directory]
mariadb-backup = $${basedirectory:backup}/mariadb/
duplicity-cache = $${basedirectory:cache}/duplicity/
[buildout]
extends = ${template-mariadb-pbsready:output}
parts =
url
urls
mariadb
duplicity-recover
mydumper-import
stunnel
certificate-authority
ca-stunnel
logrotate
logrotate-entry-mariadb
logrotate-entry-stunnel
logrotate-entry-cron
cron
cron-entry-logrotate
cron-entry-duplicity-recover
sshkeys-authority
dropbear-server
sshkeys-dropbear
dropbear-server-pbs-authorized-key
cron-entry-mariadb-import
extends = ${template-mariadb:output}
[duplicity-recover]
recipe = slapos.cookbook:duplicity
remote-backup = $${slap-parameter:backup-url}
local-directory = $${directory:duplicity-recover}
duplicity-binary = ${buildout:bin-directory}/duplicity
wrapper = $${rootdirectory:bin}/raw_duplicity
cache = $${directory:duplicity-cache}
recover = true
[duplicity-recover-locked]
recipe = slapos.cookbook:lockfile
lock-file = $${mydumper-import-locked:lock-file}
wrapper = $${rootdirectory:bin}/duplicity
binary = $${duplicity-recover:wrapper}
wait = true
[mydumper-import]
recipe = slapos.cookbook:mydumper
wrapper = $${rootdirectory:bin}/raw_myloader
backup-directory = $${directory:duplicity-recover}
backup-directory = $${directory:mariadb-backup}
socket = $${mariadb:socket}
user = root
myloader-binary = ${mydumper:location}/bin/myloader
......@@ -50,23 +37,9 @@ wrapper = $${rootdirectory:bin}/myloader
binary = $${mydumper-import:wrapper}
wait = true
[cron-entry-duplicity-recover]
<= cron
recipe = slapos.cookbook:cron.d
name = duplicity_recover
frequency = 30 * * * *
command = $${duplicity-recover-locked:wrapper}
[cron-entry-mariadb-import]
<= cron
recipe = slapos.cookbook:cron.d
name = mariadb_import
frequency = 0 * * * *
command = $${mydumper-import-locked:wrapper}
[basedirectory]
cache = $${rootdirectory:var}/cache/
[directory]
duplicity-recover = $${rootdirectory:srv}/recovering
duplicity-cache = $${basedirectory:cache}/duplicity
[buildout]
extends = ${template-mariadb:output}
parts =
urls
mariadb
stunnel
certificate-authority
ca-stunnel
logrotate
logrotate-entry-mariadb
logrotate-entry-stunnel
logrotate-entry-cron
cron
cron-entry-logrotate
sshkeys-authority
dropbear-server
sshkeys-dropbear
dropbear-server-pbs-authorized-key
[urls]
ssh-public-key = $${sshkeys-dropbear:public-key-value}
ssh-url = ssh://nobody@[$${dropbear-server:host}]:$${dropbear-server:port}/$${rdiff-backup-server:path}
[sshkeys-directory]
recipe = slapos.cookbook:mkdirectory
requests = $${directory:sshkeys}/requests/
keys = $${directory:sshkeys}/keys/
[sshkeys-authority]
recipe = slapos.cookbook:sshkeys_authority
request-directory = $${sshkeys-directory:requests}
keys-directory = $${sshkeys-directory:keys}
wrapper = $${basedirectory:services}/sshkeys_authority
keygen-binary = ${dropbear:location}/bin/dropbearkey
[sshkeys-dropbear]
<= sshkeys-authority
recipe = slapos.cookbook:sshkeys_authority.request
name = dropbear
type = rsa
executable = $${dropbear-server:wrapper}
public-key = $${dropbear-server:rsa-keyfile}.pub
private-key = $${dropbear-server:rsa-keyfile}
wrapper = $${basedirectory:services}/sshd
[dropbear-server]
recipe = slapos.cookbook:dropbear
host = $${slap-network-information:global-ipv6}
port = 2222
home = $${directory:ssh}
wrapper = $${rootdirectory:bin}/raw_sshd
shell = $${rdiff-backup-server:wrapper}
rsa-keyfile = $${directory:ssh}/server_key.rsa
dropbear-binary = ${dropbear:location}/sbin/dropbear
[dropbear-server-pbs-authorized-key]
<= dropbear-server
recipe = slapos.cookbook:dropbear.add_authorized_key
key = $${slap-parameter:authorized-key}
[rdiff-backup-server]
recipe = slapos.cookbook:rdiffbackup
client = false
path = $${directory:mariadb-backup}
wrapper = $${rootdirectory:bin}/rdiffbackup-server
rdiffbackup-binary = ${buildout:bin-directory}/rdiff-backup
[basedirectory]
cache = $${rootdirectory:var}/cache/
[directory]
mariadb-backup = $${basedirectory:backup}/mariadb/
ssh = $${rootdirectory:etc}/ssh/
sshkeys = $${rootdirectory:srv}/sshkeys
[buildout]
parts =
url
urls
mariadb
stunnel
certificate-authority
......@@ -8,6 +8,7 @@ parts =
logrotate
logrotate-entry-mariadb
logrotate-entry-stunnel
logrotate-entry-cron
cron
cron-entry-logrotate
......@@ -17,14 +18,9 @@ eggs-directory = ${buildout:eggs-directory}
develop-eggs-directory = ${buildout:develop-eggs-directory}
offline = true
[url]
recipe = slapos.cookbook:publishurl
scheme = mysqls
username = $${mariadb:user}
password = $${mariadb:password}
host = $${stunnel:remote-host}
port = $${stunnel:remote-port}
path = $${mariadb:database}
[urls]
recipe = slapos.cookbook:publish
url = mysqls://$${mariadb:user}:$${mariadb:password}@[$${stunnel:remote-host}]:$${stunnel:remote-port}/$${mariadb:database}
[mariadb]
recipe = slapos.cookbook:mysql
......@@ -131,19 +127,29 @@ rotate-num = 30
notifempty = true
create = true
[logrotate-entry-cron]
<= logrotate
recipe =slapos.cookbook:logrotate.d
name = crond
log = $${cron-simplelogger:log}
frequency = daily
rotate-num = 30
notifempty = true
create = true
[cron]
recipe = slapos.cookbook:cron
dcrond-binary = ${dcron:location}/sbin/crond
cron-entries = $${directory:cron-entries}
crontabs = $${directory:crontabs}
cronstamps = $${directory:cronstamps}
catcher = $${cron-simplelogger:binary}
catcher = $${cron-simplelogger:wrapper}
binary = $${basedirectory:services}/crond
[cron-simplelogger]
recipe = slapos.cookbook:simplelogger
binary = $${rootdirectory:bin}/cron_simplelogger
output = $${directory:cronoutput}
wrapper = $${rootdirectory:bin}/cron_simplelogger
log = $${basedirectory:log}/crond.log
[cron-entry-logrotate]
<= cron
......@@ -172,7 +178,6 @@ recipe = slapos.cookbook:mkdirectory
cron-entries = $${rootdirectory:etc}/cron.d/
crontabs = $${rootdirectory:etc}/crontabs/
cronstamps = $${rootdirectory:etc}/cronstamps/
cronoutput = $${basedirectory:log}/cron/
ca-dir = $${rootdirectory:srv}/ssl/
mariadb-data = $${rootdirectory:srv}/mariadb/
logrotate-backup = $${basedirectory:backup}/logrotate/
......
[buildout]
parts =
connection-dict
rdiff-backup
cron
cron-entry-logrotate
logrotate
sshkeys-authority
sshkeys-dropbear
eggs-directory = ${buildout:eggs-directory}
develop-eggs-directory = ${buildout:develop-eggs-directory}
offline = true
[connection-dict]
recipe = slapos.cookbook:publish
ssh-key = $${sshkeys-dropbear:public-key-value}
[dropbear-client]
recipe = slapos.cookbook:dropbear.client
dbclient-binary = ${dropbear:location}/bin/dbclient
wrapper = $${rootdirectory:bin}/ssh
home = $${basedirectory:ssh-home}
identity-file = $${basedirectory:ssh-home}/id_rsa
[sshkeys-directory]
recipe = slapos.cookbook:mkdirectory
requests = $${directory:sshkeys}/requests/
keys = $${directory:sshkeys}/keys/
[sshkeys-authority]
recipe = slapos.cookbook:sshkeys_authority
request-directory = $${sshkeys-directory:requests}
keys-directory = $${sshkeys-directory:keys}
keygen-binary = ${dropbear:location}/bin/dropbearkey
wrapper = $${basedirectory:services}/sshkeys_authority
[sshkeys-dropbear]
<= sshkeys-authority
recipe = slapos.cookbook:sshkeys_authority.request
name = rdiff-backup
type = rsa
executable = $${dropbear-client:wrapper}
public-key = $${dropbear-client:identity-file}.pub
private-key = $${dropbear-client:identity-file}
wrapper = $${rootdirectory:bin}/do_backup
[rdiff-backup]
recipe = slapos.cookbook:rdiffbackup
client = true
rdiffbackup-binary = ${buildout:bin-directory}/rdiff-backup
sshclient-binary = $${dropbear-client:wrapper}
known-hosts = $${directory:dot-ssh}/known_hosts
promises-directory = $${basedirectory:promises}
directory = $${directory:backups}
cron-entries = $${cron:cron-entries}
wrappers-directory = $${directory:pbs-wrappers}
[cron-simplelogger]
recipe = slapos.cookbook:simplelogger
wrapper = $${rootdirectory:bin}/cron_simplelogger
log = $${basedirectory:log}/crond.log
[cron]
recipe = slapos.cookbook:cron
dcrond-binary = ${dcron:location}/sbin/crond
cron-entries = $${directory:cron-entries}
crontabs = $${directory:crontabs}
cronstamps = $${directory:cronstamps}
catcher = $${cron-simplelogger:wrapper}
binary = $${basedirectory:services}/crond
[cron-entry-logrotate]
<= cron
recipe = slapos.cookbook:cron.d
name = logrotate
frequency = 0 0 * * *
command = $${logrotate:wrapper}
[logrotate]
recipe = slapos.cookbook:logrotate
# Binaries
logrotate-binary = ${logrotate:location}/sbin/logrotate
gzip-binary = ${gzip:location}/bin/gzip
gunzip-binary = ${gzip:location}/bin/gunzip
# Directories
wrapper = $${rootdirectory:bin}/logrotate
conf = $${rootdirectory:etc}/logrotate.conf
logrotate-entries = $${directory:logrotate-entries}
backup = $${directory:logrotate-backup}
state-file = $${rootdirectory:srv}/logrotate.status
[rootdirectory]
recipe = slapos.cookbook:mkdirectory
etc = $${buildout:directory}/etc/
var = $${buildout:directory}/var/
srv = $${buildout:directory}/srv/
bin = $${buildout:directory}/bin/
tmp = $${buildout:directory}/tmp/
[basedirectory]
recipe = slapos.cookbook:mkdirectory
log = $${rootdirectory:var}/log/
services = $${rootdirectory:etc}/run/
run = $${rootdirectory:var}/run/
backup = $${rootdirectory:srv}/backup/
promises = $${rootdirectory:etc}/promise/
[directory]
recipe = slapos.cookbook:mkdirectory
cronstamps = $${rootdirectory:etc}/cronstamps/
cron-entries = $${rootdirectory:etc}/cron.d/
crontabs = $${rootdirectory:etc}/crontabs/
cronoutput = $${basedirectory:log}/cron-ouput/
rdiff-backup-backup = $${basedirectory:backup}/rdiff-backup/
logrotate-entries = $${rootdirectory:etc}/logrotate.d/
logrotate-backup = $${basedirectory:backup}/logrotate/
rdiff-backup = $${rootdirectory:etc}/rdiff-backup/
sshkeys = $${rootdirectory:srv}/sshkeys
backups = $${basedirectory:backup}/backups/
pbs-wrappers = $${rootdirectory:bin}/rdiff-backup/
dot-ssh = $${basedirectory:ssh-home}/.ssh/
[basedirectory]
ssh-home = $${rootdirectory:home}/ssh
[rootdirectory]
home = $${buildout:directory}/home/
......@@ -11,5 +11,6 @@ recipe = slapos.cookbook:softwaretype
default = ${template-apache-php:output}
backuped = ${template-backuped:output}
mariadb = ${template-mariadb:output}
mariadb-remote-backuped = ${template-mariadb-remote-backuped:output}
mariadb-pseudo-replicating = ${template-mariadb-pseudo-replicating:output}
mariadb-pbsready-import = ${template-mariadb-pbsready-import:output}
mariadb-pbsready-export = ${template-mariadb-pbsready-export:output}
pull-backup = ${template-pull-backup:output}
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment