Skip to content

slapos
slapos
Commit d6a9aaa4 authored by Marco Mariani's avatar Marco Mariani
Browse files
Options

Merge branch 'lapp' into lapp-resilient2

parents 1de09c67 0f3b4c86
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 1561 additions and 917 deletions
slapos/recipe/addresiliency/__init__.py
View file @ d6a9aaa4
...@@ -26,56 +26,55 @@ ...@@ -26,56 +26,55 @@
############################################################################## ##############################################################################
from slapos.recipe.librecipe import GenericSlapRecipe from slapos.recipe.librecipe import GenericSlapRecipe
import sys
import os import os
class Recipe(GenericSlapRecipe): class Recipe(GenericSlapRecipe):
""" This class provides the installation of the resilience """ This class provides the installation of the resilience
script on the partition. script on the partition.
""" """
def _install(self): def _install(self):
path_list = [] path_list = []
self_id = int(self.parameter_dict['number'])
ip = self.parameter_dict['ip-list'].split(' ') confpath = os.path.join(self.options['etc'], 'bully.conf')
print 'Creating bully script with ips : %s\n' % ip
slap_connection = self.buildout['slap-connection'] ip_list = self.parameter_dict['ip-list']
print 'Creating bully configuration with ips : %s\n' % ip_list
conf = self.createFile(confpath,
self.substituteTemplate(
self.getTemplateFilename('bully.conf.in'),
{
'self_id': int(self.parameter_dict['number']),
'ip_list': ip_list
}
))
path_list.append(conf)
slap_connection = self.buildout['slap-connection']
if self.optionIsTrue('enable-bully-service', default=False):
wrapper_dir = self.options['services']
else:
wrapper_dir = self.options['bin']
wrapper = self.createPythonScript(
name=os.path.join(wrapper_dir, self.parameter_dict['wrapper']),
absolute_function='slapos.recipe.addresiliency.bully.run',
arguments={
'confpath': confpath,
'server_url': slap_connection['server-url'],
'key_file': slap_connection.get('key-file'),
'cert_file': slap_connection.get('cert-file'),
'computer_id': slap_connection['computer-id'],
'partition_id': slap_connection['partition-id'],
'software': slap_connection['software-release-url'],
'namebase': self.parameter_dict['namebase'],
})
path_list.append(wrapper)
return path_list
path_conf = os.path.join(self.options['script'], 'conf.in')
path_bully = os.path.join(self.options['script'], self.parameter_dict['script'])
path_bully_new = os.path.join(self.options['script'], 'new.py')
path_run = os.path.join(self.options['run'], self.parameter_dict['wrapper'])
print 'paths: %s\n%s\n' % (path_run, path_bully)
bully_conf = dict(self_id=self_id,
ip_list=ip,
executable=sys.executable,
syspath=sys.path,
server_url=slap_connection['server-url'],
key_file=slap_connection.get('key-file'),
cert_file=slap_connection.get('cert-file'),
computer_id=slap_connection['computer-id'],
partition_id=slap_connection['partition-id'],
software=slap_connection['software-release-url'],
namebase=self.parameter_dict['namebase'],
confpath=path_conf)
try:
conf = self.createFile(path_conf,
self.substituteTemplate(
self.getTemplateFilename('conf.in.in'),
bully_conf))
path_list.append(conf)
script = self.createExecutable(path_bully,
self.substituteTemplate(
self.getTemplateFilename('bully.py.in'),
bully_conf))
path_list.append(script)
wrapper = self.createPythonScript(
path_run,
'slapos.recipe.librecipe.execute.execute',
[path_bully])
path_list.append(wrapper)
except IOError:
pass
return path_list
slapos/recipe/addresiliency/bully.py 0 → 100644
View file @ d6a9aaa4
This diff is collapsed.
slapos/recipe/addresiliency/template/bully.py.in
View file @ d6a9aaa4
...@@ -8,6 +8,7 @@ import sys ...@@ -8,6 +8,7 @@ import sys
sys.path[:] = %(syspath)s sys.path[:] = %(syspath)s
import slapos
from slapos import slap as slapmodule from slapos import slap as slapmodule
port = 50000 port = 50000
...@@ -37,9 +38,11 @@ def rename_broken_and_stop(): ...@@ -37,9 +38,11 @@ def rename_broken_and_stop():
slap.initializeConnection('%(server_url)s', slap.initializeConnection('%(server_url)s',
'%(key_file)s', '%(key_file)s',
'%(cert_file)s') '%(cert_file)s')
computer_partition = slap.registerComputerPartition('%(computer_id)s', computer_partition = slap.registerComputerPartition(computer_guid='%(computer_id)s',
'%(partition_id)s') partition_id='%(partition_id)s')
broken = computer_partition.request('%(software)s', 'frozen', '%(namebase)s0') broken = computer_partition.request(software_release='%(software)s',
software_type='frozen',
partition_reference='%(namebase)s0')
broken.rename('broken-%%s' %% (time.strftime("%%d-%%b_%%H:%%M:%%S", time.gmtime()))) broken.rename('broken-%%s' %% (time.strftime("%%d-%%b_%%H:%%M:%%S", time.gmtime())))
broken.stopped() broken.stopped()
......
slapos/recipe/addresiliency/template/bully_new.py.in
View file @ d6a9aaa4
#!%(executable)s #!%(executable)s
import logging
import os
import socket import socket
import time
import sys import sys
import thread import thread
import time import time
import os
sys.path[:] = %(syspath)s sys.path[:] = %(syspath)s
from slapos import slap as slapmodule from slapos import slap as slapmodule
import slapos
BASE_PORT = 50000
SLEEPING_MINS = 2
log = logging.getLogger(__name__)
logging.basicConfig(level=logging.DEBUG)
class Renamer(object):
def __init__(self, server_url, key_file, cert_file, computer_guid,
partition_id, software_release, namebase):
self.server_url = server_url
self.key_file = key_file
self.cert_file = cert_file
self.computer_guid = computer_guid
self.partition_id = partition_id
self.software_release = software_release
self.namebase = namebase
def _failover(self):
slap = slapmodule.slap()
slap.initializeConnection(self.server_url,
self.key_file,
self.cert_file)
computer_partition = slap.registerComputerPartition(computer_guid=self.computer_guid,
partition_id=self.partition_id)
broken = computer_partition.request(software_release=self.software_release,
software_type='frozen',
partition_reference=self.namebase+'0')
broken.rename('broken-{}'.format(time.strftime("%%d-%%b_%%H:%%M:%%S", time.gmtime())))
broken.stopped()
computer_partition.rename(self.namebase+'0')
def failover(self):
try:
log.info('renaming done')
except slapos.slap.slap.ServerError:
log.info('Internal server error')
port = 50000
size = 1024
def rename_broken_and_stop():
try:
slap = slapmodule.slap()
slap.initializeConnection('%(server_url)s',
'%(key_file)s',
'%(cert_file)s')
computer_partition = slap.registerComputerPartition('%(computer_id)s',
'%(partition_id)s')
broken = computer_partition.request('%(software)s', 'frozen', '%(namebase)s0')
broken.rename('broken-%%s' %% (time.strftime("%%d-%%b_%%H:%%M:%%S", time.gmtime())))
broken.stopped()
computer_partition.rename('%(namebase)s0')
print 'renaming done\n'
except slapos.slap.slap.ServerError:
print 'Internal server error\n'
## Leader is always number 0 ## Leader is always number 0
class ResilientInstance(object): class ResilientInstance(object):
def __init__(self, comm): def __init__(self, comm, renamer, confpath):
self.comm = comm self.comm = comm
self.id = 0 self.id = 0
self.state = 'normal' self.state = 'normal'
self.halter = 0 self.halter = 0
self.nbComp = nbComp
self.inElection = False self.inElection = False
self.alive = True self.alive = True
self.lastPing = time.clock() self.lastPing = time.clock()
self.mainCanal = self.comm.canal(['ping', 'halt', self.mainCanal = self.comm.canal(['ping', 'halt', 'victory'])
'victory'])
self.renamer = renamer
self.okCanal = self.comm.canal(['ok']) self.okCanal = self.comm.canal(['ok'])
self.confpath = confpath
self.loadConnectionInfos() self.loadConnectionInfos()
def loadConnectionInfos(self): def loadConnectionInfos(self):
file = open('%(confpath)s', 'r') file = open(self.confpath, 'r')
params = file.read().split('\n') params = file.read().split('\n')
file.close() file.close()
self.nbComp = len([x.strip("' ") for x in params[0].strip('[],').split(',')]) self.nbComp = len([x.strip("' ") for x in params[0].strip('[],').split(',')])
...@@ -67,7 +87,8 @@ class ResilientInstance(object): ...@@ -67,7 +87,8 @@ class ResilientInstance(object):
## Needs to be changed to use the master ## Needs to be changed to use the master
def aliveManagement(self): def aliveManagement(self):
while self.alive: while self.alive:
time.sleep(30*60) log.info('XXX sleeping for %%d minutes' %% SLEEPING_MINS)
time.sleep(SLEEPING_MINS*60)
if self.id == 0: if self.id == 0:
continue continue
self.comm.send('ping', 0) self.comm.send('ping', 0)
...@@ -84,7 +105,7 @@ class ResilientInstance(object): ...@@ -84,7 +105,7 @@ class ResilientInstance(object):
while self.alive: while self.alive:
message, sender = self.mainCanal.get() message, sender = self.mainCanal.get()
if message == 'ping': if message == 'ping':
self.comm.send('ok', sender) self.comm.send('ok', sender)
elif message == 'halt': elif message == 'halt':
self.state = 'waitingConfirm' self.state = 'waitingConfirm'
...@@ -93,7 +114,7 @@ class ResilientInstance(object): ...@@ -93,7 +114,7 @@ class ResilientInstance(object):
elif message == 'victory': elif message == 'victory':
if int(sender) == int(self.halter) and self.state == 'waitingConfirm': if int(sender) == int(self.halter) and self.state == 'waitingConfirm':
print '%s thinks %s is the leader\n' % (self.id, sender) log.info('{} thinks {} is the leader'.format(self.id, sender))
self.comm.send('ok', sender) self.comm.send('ok', sender)
self.state = 'normal' self.state = 'normal'
...@@ -105,7 +126,7 @@ class ResilientInstance(object): ...@@ -105,7 +126,7 @@ class ResilientInstance(object):
self.comm.send('ping', higher) self.comm.send('ping', higher)
message, sender = self.okCanal.get() message, sender = self.okCanal.get()
if message: if message:
#print '%s is alive (%s)\n' % (higher, self.id) log.info('{} is alive ({})'.format(higher, self.id))
self.inElection = False self.inElection = False
return False return False
continue continue
...@@ -114,7 +135,7 @@ class ResilientInstance(object): ...@@ -114,7 +135,7 @@ class ResilientInstance(object):
return False return False
#I should be the new coordinator, halt those below me #I should be the new coordinator, halt those below me
print 'Should be ME : %s \n' % self.id log.info('Should be ME : {}'.format(self.id))
self.state = 'election' self.state = 'election'
self.halter = self.id self.halter = self.id
ups = [] ups = []
...@@ -131,13 +152,13 @@ class ResilientInstance(object): ...@@ -131,13 +152,13 @@ class ResilientInstance(object):
message, sender = self.okCanal.get() message, sender = self.okCanal.get()
if message: if message:
continue continue
print 'Something is wrong... let\'s start over\n' log.info('Something is wrong... let\'s start over')
return self.election() return self.election()
self.state = 'normal' self.state = 'normal'
self.active = True self.active = True
print '%s Is THE LEADER \n' % self.id log.info('{} Is THE LEADER'.format(self.id))
rename_broken_and_stop() self.renamer.failover()
self.inElection = False self.inElection = False
...@@ -164,27 +185,24 @@ class FilteredCanal(object): ...@@ -164,27 +185,24 @@ class FilteredCanal(object):
self.lock.acquire() self.lock.acquire()
if self.list: if self.list:
self.lock.release() self.lock.release()
val = self.list[0] return self.list.pop(0)
self.list = self.list[1:]
return val
self.lock.release() self.lock.release()
return [None, None] return [None, None]
class Wrapper(object): class Wrapper(object):
def __init__(self, timeout=20): def __init__(self, confpath, timeout=20):
self.read_pipes = [os.fdopen(x) for x in read_pipes]
self.write_pipes = write_pipes
self.canals = [] self.canals = []
self.ips = [] self.ips = []
self.id = 0 self.id = 0
self.timeout = timeout self.timeout = timeout
self.confpath = confpath
self.getConnectionInfos() self.getConnectionInfos()
self.socket = None self.socket = None
def getConnectionInfos(self): def getConnectionInfos(self):
file = open('%(confpath)s', 'r') file = open(self.confpath, 'r')
params = file.read().split('\n') params = file.read().split('\n')
file.close() file.close()
self.ips = [x.strip("' ") for x in params[0].strip('[],').split(',')] self.ips = [x.strip("' ") for x in params[0].strip('[],').split(',')]
...@@ -193,17 +211,17 @@ class Wrapper(object): ...@@ -193,17 +211,17 @@ class Wrapper(object):
def start(self): def start(self):
self.getConnectionInfos() self.getConnectionInfos()
self.socket = socket.socket(socket.AF_INET6, socket.SOCK_STREAM) self.socket = socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
self.socket.bind((self.ips[self.id], port + self.id)) self.socket.bind((self.ips[self.id], BASE_PORT + self.id))
s.listen(5) self.socket.listen(5)
def send(self, message, number): def send(self, message, number):
self.getConnectionInfos() self.getConnectionInfos()
try: try:
s = socket.socket(socket.AF_INET6, socket.SOCK_STREAM) s = socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
s.connect((self.ips[number], port + number)) s.connect((self.ips[number], BASE_PORT + number))
s.send(message + (' %s\n' % self.id)) s.send(message + (' {}\n'.format(self.id)))
except (socket.error, socket.herror, socket.gaierror, socket.timeout): except (socket.error, socket.herror, socket.gaierror, socket.timeout):
pass pass
finally: finally:
s.close() s.close()
...@@ -213,31 +231,48 @@ class Wrapper(object): ...@@ -213,31 +231,48 @@ class Wrapper(object):
return created return created
def recv(self): def recv(self):
client, _ = s.accept() client, _ = self.socket.accept()
client_message = client.recv(1024) client_message = client.recv(1024)
if client_message: if client_message:
message, sender = client_message.split() message, sender = client_message.split()
for canal in self.canals: for canal in self.canals:
canal.append(message, sender) canal.append(message, int(sender))
def main():
renamer = Renamer(server_url = '%(server_url)s',
key_file = '%(key_file)s',
cert_file = '%(cert_file)s',
computer_guid = '%(computer_id)s',
partition_id = '%(partition_id)s',
software_release = '%(software)s',
namebase = '%(namebase)s')
confpath = '%(confpath)s'
wrapper = Wrapper(confpath=confpath, timeout=20)
wrapper = createWrapper(20) computer = ResilientInstance(wrapper, renamer=renamer, confpath=confpath)
computer = ResilientInstance(wrapper) #idle waiting for connection infos
while computer.nbComp < 2 :
computer.loadConnectionInfos()
time.sleep(30)
#idle waiting for connection infos log.info('Starting')
while computer.nbComp < 2 :
computer.loadConnectionInfos()
time.sleep(30)
print 'Starting\n' computer.comm.start()
thread.start_new_thread(computer.listen, ())
thread.start_new_thread(computer.main, ())
thread.start_new_thread(computer.aliveManagement, ())
computer.comm.start() while True:
thread.start_new_thread(computer.listen, ()) # XXX tight loop
thread.start_new_thread(computer.main, ()) continue
thread.start_new_thread(computer.aliveManagement, ())
while True:
continue
if __name__ == '__main__':
main()
slapos/recipe/dcron.py
View file @ d6a9aaa4
...@@ -33,26 +33,22 @@ class Recipe(GenericBaseRecipe): ...@@ -33,26 +33,22 @@ class Recipe(GenericBaseRecipe):
def install(self): def install(self):
self.logger.info("Installing dcron...") self.logger.info("Installing dcron...")
path_list = [] options = self.options
script = self.createWrapper(name=options['binary'],
command=options['dcrond-binary'].strip(),
parameters=[
'-s', options['cron-entries'],
'-c', options['crontabs'],
'-t', options['cronstamps'],
'-f', '-l', '5',
'-M', options['catcher']
])
cronstamps = self.options['cronstamps']
cron_d = self.options['cron-entries']
crontabs = self.options['crontabs']
catcher = self.options['catcher']
binary = self.options['binary']
script = self.createPythonScript(binary,
'slapos.recipe.librecipe.execute.execute',
[self.options['dcrond-binary'].strip(), '-s', cron_d, '-c', crontabs,
'-t', cronstamps, '-f', '-l', '5', '-M', catcher]
)
path_list.append(script)
self.logger.debug('Main cron executable created at : %r', script) self.logger.debug('Main cron executable created at : %r', script)
self.logger.info("dcron successfully installed.") self.logger.info("dcron successfully installed.")
return path_list return [script]
......
slapos/recipe/dropbear.py
View file @ d6a9aaa4
...@@ -158,6 +158,24 @@ class Client(GenericBaseRecipe): ...@@ -158,6 +158,24 @@ class Client(GenericBaseRecipe):
return [wrapper] return [wrapper]
def keysplit(s):
"""
Split a string like "ssh-rsa AKLFKJSL..... ssh-rsa AAAASAF...."
and return the individual key_type + key strings.
"""
si = iter(s.split(' '))
while True:
key_type = next(si)
try:
key_value = next(si)
except StopIteration:
# odd number of elements, should not happen, yield the last one by itself
yield key_type
break
yield '%s %s' % (key_type, key_value)
class AddAuthorizedKey(GenericBaseRecipe): class AddAuthorizedKey(GenericBaseRecipe):
def install(self): def install(self):
...@@ -167,7 +185,9 @@ class AddAuthorizedKey(GenericBaseRecipe): ...@@ -167,7 +185,9 @@ class AddAuthorizedKey(GenericBaseRecipe):
path_list.append(ssh) path_list.append(ssh)
authorized_keys = AuthorizedKeysFile(os.path.join(ssh, 'authorized_keys')) authorized_keys = AuthorizedKeysFile(os.path.join(ssh, 'authorized_keys'))
for key in self.options['key'].split(' '): for key in keysplit(self.options['key']):
# XXX key might actually be the string 'None' or 'null'
authorized_keys.append(key) authorized_keys.append(key)
return path_list return path_list
slapos/recipe/equeue.py
View file @ d6a9aaa4
...@@ -30,17 +30,19 @@ class Recipe(GenericBaseRecipe): ...@@ -30,17 +30,19 @@ class Recipe(GenericBaseRecipe):
def install(self): def install(self):
commandline = [self.options['equeue-binary']] parameters = [
commandline.extend(['--database', self.options['database']]) '--database', self.options['database'],
commandline.extend(['-l', self.options['log']]) '-l', self.options['log'],
]
if 'loglevel' in self.options: if 'loglevel' in self.options:
commandline.extend(['--loglevel', self.options['loglevel']]) parameters.extend(['--loglevel', self.options['loglevel']])
commandline.append(self.options['socket']) parameters.append(self.options['socket'])
wrapper = self.createWrapper(name=self.options['wrapper'],
command=self.options['equeue-binary'],
parameters=parameters)
return [wrapper]
return [self.createPythonScript(
self.options['wrapper'],
'slapos.recipe.librecipe.execute.execute',
commandline,
)]
slapos/recipe/librecipe/generic.py
View file @ d6a9aaa4
# -*- coding: utf-8 -*-
# vim: set et sts=2:
############################################################################## ##############################################################################
# #
# Copyright (c) 2010 Vifib SARL and Contributors. All Rights Reserved. # Copyright (c) 2010 Vifib SARL and Contributors. All Rights Reserved.
...@@ -24,6 +26,7 @@ ...@@ -24,6 +26,7 @@
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
# #
############################################################################## ##############################################################################
import io
import logging import logging
import os import os
import sys import sys
...@@ -90,6 +93,21 @@ class GenericBaseRecipe(object): ...@@ -90,6 +93,21 @@ class GenericBaseRecipe(object):
def createExecutable(self, name, content, mode=0700): def createExecutable(self, name, content, mode=0700):
return self.createFile(name, content, mode) return self.createFile(name, content, mode)
def addLineToFile(self, filepath, line, encoding='utf8'):
"""Append a single line to a text file, if the line does not exist yet.
line must be unicode."""
try:
lines = io.open(filepath, 'r', encoding=encoding).readlines()
except IOError:
lines = []
if not line in lines:
lines.append(line)
with io.open(filepath, 'w+', encoding=encoding) as f:
f.write(u'\n'.join(lines))
def createPythonScript(self, name, absolute_function, arguments=''): def createPythonScript(self, name, absolute_function, arguments=''):
"""Create a python script using zc.buildout.easy_install.scripts """Create a python script using zc.buildout.easy_install.scripts
...@@ -115,7 +133,6 @@ class GenericBaseRecipe(object): ...@@ -115,7 +133,6 @@ class GenericBaseRecipe(object):
Takes care of quoting. Takes care of quoting.
""" """
q = shlex.quote
lines = [ lines = [
'#!/bin/sh', '#!/bin/sh',
'exec %s' % shlex.quote(command) 'exec %s' % shlex.quote(command)
......
slapos/recipe/mydumper.py
View file @ d6a9aaa4
...@@ -28,85 +28,76 @@ import subprocess ...@@ -28,85 +28,76 @@ import subprocess
from slapos.recipe.librecipe import GenericBaseRecipe from slapos.recipe.librecipe import GenericBaseRecipe
def dump(args):
mydumper_cmd = [args['mydumper']]
mydumper_cmd.extend(['-B', args['database']])
if args['socket'] is not None: def _mydumper_base_cmd(mydumper, database, user, password,
mydumper_cmd.extend(['-S', args['socket']]) socket=None, host=None, port=None, **kw):
else: cmd = [mydumper]
mydumper_cmd.extend(['-h', args['host']]) cmd.extend(['-B', database])
mydumper_cmd.etxned(['-P', args['port']])
mydumper_cmd.extend(['-u', args['user']])
if args['password'] is not None:
mydumper_cmd.extend(['-p', args['password']])
if args['compression']:
mydumper_cmd.append('--compress')
if args['rows'] is not None: if socket:
mydumper_cmd.extend(['-r', args['rows']]) cmd.extend(['-S', socket])
else:
mydumper_cmd.extend(['-o', args['directory']]) cmd.extend(['-h', host])
cmd.extend(['-P', port])
subprocess.check_call(mydumper_cmd) cmd.extend(['-u', user])
if password:
cmd.extend(['-p', password])
return cmd
def do_import(args): def do_export(args):
mydumper_cmd = [args['mydumper']] cmd = _mydumper_base_cmd(**args)
mydumper_cmd.extend(['-B', args['database']])
if args['socket'] is not None: if args['compression']:
mydumper_cmd.extend(['-S', args['socket']]) cmd.append('--compress')
else:
mydumper_cmd.extend(['-h', args['host']])
mydumper_cmd.etxned(['-P', args['port']])
mydumper_cmd.extend(['-u', args['user']]) if args['rows'] is not None:
if args['password'] is not None: cmd.extend(['-r', args['rows']])
mydumper_cmd.extend(['-p', args['password']])
mydumper_cmd.append('--overwrite-tables') cmd.extend(['-o', args['directory']])
mydumper_cmd.extend(['-d', args['directory']]) subprocess.check_call(cmd)
subprocess.check_call(mydumper_cmd)
def do_import(args):
cmd = _mydumper_base_cmd(**args)
cmd.append('--overwrite-tables')
cmd.extend(['-d', args['directory']])
subprocess.check_call(cmd)
class Recipe(GenericBaseRecipe): class Recipe(GenericBaseRecipe):
def install(self): def install(self):
# Host or socket should be defined config = {
try: 'database': self.options['database'],
self.options['host'] 'directory': self.options['backup-directory'],
except: 'user': self.options['user'],
self.options['socket'] 'password': self.options.get('password'),
}
config = dict(database=self.options['database'],
socket=self.options.get('socket'), if self.options.get('host'):
host=self.options.get('host'), config['host'] = self.options['host']
port=self.options.get('port', 3306), config['port'] = self.options.get('port', 3306)
directory=self.options['backup-directory'], elif self.options.get('socket'):
user=self.options['user'], config['socket'] = self.options['socket']
password=self.options.get('password'), else:
) raise ValueError("host or socket must be defined")
name = __name__
if self.optionIsTrue('import', False): if self.optionIsTrue('import', False):
config.update(mydumper=self.options['myloader-binary']) function = do_import
name += '.do_import' config['mydumper'] = self.options['myloader-binary']
else: else:
config.update(mydumper=self.options['mydumper-binary'], function = do_export
compression=self.optionIsTrue('compression', default=False), config['mydumper'] = self.options['mydumper-binary']
rows=self.options.get('rows'), config['compression'] = self.optionIsTrue('compression', default=False)
) config['rows'] = self.options.get('rows')
name += '.dump'
wrapper = self.createPythonScript(self.options['wrapper'], wrapper = self.createPythonScript(name=self.options['wrapper'],
name, absolute_function = '%s.%s' % (__name__, function.func_name),
config) arguments=config)
return [wrapper] return [wrapper]
slapos/recipe/notifier.py
View file @ d6a9aaa4
...@@ -31,62 +31,68 @@ from slapos.recipe.librecipe import GenericBaseRecipe ...@@ -31,62 +31,68 @@ from slapos.recipe.librecipe import GenericBaseRecipe
class Recipe(GenericBaseRecipe): class Recipe(GenericBaseRecipe):
def install(self): def install(self):
commandline = [self.options['server-binary']] options = self.options
commandline.extend(['--callbacks', self.options['callbacks']]) script = self.createWrapper(name=options['wrapper'],
commandline.extend(['--feeds', self.options['feeds']]) command=options['server-binary'],
commandline.extend(['--equeue-socket', self.options['equeue-socket']]) parameters=[
commandline.append(self.options['host']) '--callbacks', options['callbacks'],
commandline.append(self.options['port']) '--feeds', options['feeds'],
'--equeue-socket', options['equeue-socket'],
options['host'], options['port']
])
return [script]
return [self.createPythonScript(self.options['wrapper'],
'slapos.recipe.librecipe.execute.execute',
commandline)]
class Callback(GenericBaseRecipe): class Callback(GenericBaseRecipe):
def createCallback(self, notification_id, callback): def createCallback(self, notification_id, callback):
callback_id = sha512(notification_id).hexdigest() callback_id = sha512(notification_id).hexdigest()
callback = self.createFile(os.path.join(self.options['callbacks'],
callback_id), filepath = os.path.join(self.options['callbacks'], callback_id)
callback) self.addLineToFile(filepath, callback)
return callback return filepath
def install(self): def install(self):
# XXX this path is returned multiple times, one for each callback that has been added.
return [self.createCallback(self.options['on-notification-id'], return [self.createCallback(self.options['on-notification-id'],
self.options['callback'])] self.options['callback'])]
class Notify(GenericBaseRecipe): class Notify(GenericBaseRecipe):
def createNotifier(self, notifier_binary, executable, wrapper, **kwargs): def createNotifier(self, notifier_binary, wrapper, executable,
if not os.path.exists(kwargs['log']): log, title, notification_url, feed_url):
if not os.path.exists(log):
# Just a touch # Just a touch
open(kwargs['log'], 'w').close() open(log, 'w').close()
commandline = [notifier_binary, parameters = [
'-l', kwargs['log'], '-l', log,
'--title', kwargs['title'], '--title', title,
'--feed', kwargs['feed_url'], '--feed', feed_url,
'--notification-url'] '--notification-url',
]
parameters.extend(notification_url.split(' '))
parameters.extend(['--executable', executable])
commandline.extend(kwargs['notification_url'].split(' ')) return self.createWrapper(name=wrapper,
commandline.extend(['--executable', executable]) command=notifier_binary,
parameters=parameters)
return self.createPythonScript(wrapper,
'slapos.recipe.librecipe.execute.execute',
[str(i) for i in commandline])
def install(self): def install(self):
feedurl = self.unparseUrl(scheme='http', host=self.options['host'], feed_url = self.unparseUrl(scheme='http', host=self.options['host'],
port=self.options['port'], port=self.options['port'],
path='/get/%s' % self.options['name']) path='/get/%s' % self.options['name'])
script = self.createNotifier( log = os.path.join(self.options['feeds'], self.options['name'])
self.options['notifier-binary'],
wrapper=self.options['wrapper'], options = self.options
executable=self.options['executable'], script = self.createNotifier(notifier_binary=options['notifier-binary'],
log=os.path.join(self.options['feeds'], self.options['name']), wrapper=options['wrapper'],
title=self.options['title'], executable=options['executable'],
notification_url=self.options['notify'], log=log,
feed_url=feedurl, title=options['title'],
) notification_url=options['notify'],
feed_url=feed_url)
return [script] return [script]
slapos/recipe/pbs.py
View file @ d6a9aaa4
...@@ -24,14 +24,15 @@ ...@@ -24,14 +24,15 @@
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
# #
############################################################################## ##############################################################################
from json import loads as unjson
from hashlib import sha512 from hashlib import sha512
from urlparse import urlparse import inspect
import json
import os import os
import signal
import subprocess import subprocess
import sys import sys
import signal import urlparse
import inspect
from slapos.recipe.librecipe import GenericSlapRecipe from slapos.recipe.librecipe import GenericSlapRecipe
from slapos.recipe.dropbear import KnownHostsFile from slapos.recipe.dropbear import KnownHostsFile
...@@ -43,8 +44,7 @@ from slapos import slap as slapmodule ...@@ -43,8 +44,7 @@ from slapos import slap as slapmodule
def promise(args): def promise(args):
def failed_ssh(partition, ssh): def failed_ssh(partition, ssh):
# Bad python 2 syntax, looking forward python 3 to have print(file=) sys.stderr.write("SSH Connection failed\n")
print >> sys.stderr, "SSH Connection failed"
try: try:
ssh.terminate() ssh.terminate()
except: except:
...@@ -75,16 +75,20 @@ def promise(args): ...@@ -75,16 +75,20 @@ def promise(args):
slap = slapmodule.slap() slap = slapmodule.slap()
slap.initializeConnection(args['server_url'], slap.initializeConnection(args['server_url'],
key_file=args.get('key_file'), cert_file=args.get('cert_file')) key_file=args.get('key_file'),
cert_file=args.get('cert_file'))
partition = slap.registerComputerPartition(args['computer_id'], partition = slap.registerComputerPartition(args['computer_id'],
args['partition_id']) args['partition_id'])
ssh = subprocess.Popen([args['ssh_client'], '%(user)s@%(host)s/%(port)s' % args],
stdin=subprocess.PIPE,
stdout=open(os.devnull, 'w'),
stderr=open(os.devnull, 'w'))
# Rdiff Backup protocol quit command # Rdiff Backup protocol quit command
quitcommand = 'q' + chr(255) + chr(0) * 7 quitcommand = 'q' + chr(255) + chr(0) * 7
ssh_cmdline = [args['ssh_client'], '%(user)s@%(host)s/%(port)s' % args]
ssh = subprocess.Popen(ssh_cmdline, stdin=subprocess.PIPE,
stdout=open(os.devnull), stderr=open(os.devnull))
ssh.stdin.write(quitcommand) ssh.stdin.write(quitcommand)
ssh.stdin.flush() ssh.stdin.flush()
ssh.stdin.close() ssh.stdin.close()
...@@ -113,7 +117,7 @@ class Recipe(GenericSlapRecipe, Notify, Callback): ...@@ -113,7 +117,7 @@ class Recipe(GenericSlapRecipe, Notify, Callback):
promise_path = os.path.join(self.options['promises-directory'], promise_path = os.path.join(self.options['promises-directory'],
url_hash) url_hash)
parsed_url = urlparse(url) parsed_url = urlparse.urlparse(url)
promise_dict = self.promise_base_dict.copy() promise_dict = self.promise_base_dict.copy()
promise_dict.update(user=parsed_url.username, promise_dict.update(user=parsed_url.username,
host=parsed_url.hostname, host=parsed_url.hostname,
...@@ -134,8 +138,7 @@ class Recipe(GenericSlapRecipe, Notify, Callback): ...@@ -134,8 +138,7 @@ class Recipe(GenericSlapRecipe, Notify, Callback):
'host': parsed_url.hostname, 'host': parsed_url.hostname,
} }
command = [self.options['rdiffbackup-binary']] parameters = ['--remote-schema', remote_schema]
command.extend(['--remote-schema', remote_schema])
remote_directory = '%(port)s::%(path)s' % {'port': parsed_url.port, remote_directory = '%(port)s::%(path)s' % {'port': parsed_url.port,
'path': parsed_url.path} 'path': parsed_url.path}
...@@ -144,38 +147,36 @@ class Recipe(GenericSlapRecipe, Notify, Callback): ...@@ -144,38 +147,36 @@ class Recipe(GenericSlapRecipe, Notify, Callback):
name_hash) name_hash)
if entry['type'] == 'push': if entry['type'] == 'push':
command.extend(['--restore-as-of', 'now']) parameters.extend(['--restore-as-of', 'now'])
command.append('--force') parameters.append('--force')
command.extend([local_directory, remote_directory]) parameters.extend([local_directory, remote_directory])
else: else:
command.extend([remote_directory, local_directory]) parameters.extend([remote_directory, local_directory])
wrapper_basepath = os.path.join(self.options['wrappers-directory'], wrapper_basepath = os.path.join(self.options['wrappers-directory'],
url_hash) url_hash)
wrapper_path = wrapper_basepath
if 'notify' in entry: if 'notify' in entry:
wrapper_path = '%s_raw' % wrapper_basepath wrapper_path = wrapper_basepath + '_raw'
else:
wrapper_path = wrapper_basepath
wrapper = self.createPythonScript( wrapper = self.createWrapper(name=wrapper_path,
wrapper_path, command=self.options['rdiffbackup-binary'],
'slapos.recipe.librecipe.execute.execute', parameters=parameters)
[str(i) for i in command]
)
path_list.append(wrapper) path_list.append(wrapper)
if 'notify' in entry: if 'notify' in entry:
feed_url = '%s/get/%s' % (self.options['notifier-url'], feed_url = '%s/get/%s' % (self.options['notifier-url'],
entry['notification-id']) entry['notification-id'])
wrapper = self.createNotifier( wrapper = self.createNotifier(notifier_binary=self.options['notifier-binary'],
self.options['notifier-binary'], wrapper=wrapper_basepath,
wrapper=wrapper_basepath, executable=wrapper_path,
executable=wrapper_path, log=os.path.join(self.options['feeds'], entry['notification-id']),
log=os.path.join(self.options['feeds'], entry['notification-id']), title=entry.get('title', 'Untitled'),
title=entry.get('title', 'Untitled'), notification_url=entry['notify'],
notification_url=entry['notify'], feed_url=feed_url,
feed_url=feed_url, )
)
path_list.append(wrapper) path_list.append(wrapper)
#self.setConnectionDict(dict(feed_url=feed_url), entry['slave_reference']) #self.setConnectionDict(dict(feed_url=feed_url), entry['slave_reference'])
...@@ -190,40 +191,39 @@ class Recipe(GenericSlapRecipe, Notify, Callback): ...@@ -190,40 +191,39 @@ class Recipe(GenericSlapRecipe, Notify, Callback):
return path_list return path_list
def _install(self): def _install(self):
path_list = [] path_list = []
if self.optionIsTrue('client', True): if self.optionIsTrue('client', True):
self.logger.info("Client mode") self.logger.info("Client mode")
slap_connection = self.buildout['slap-connection'] slap_connection = self.buildout['slap-connection']
self.promise_base_dict = dict( self.promise_base_dict = {
server_url=slap_connection['server-url'], 'server_url': slap_connection['server-url'],
computer_id=slap_connection['computer-id'], 'computer_id': slap_connection['computer-id'],
cert_file=slap_connection.get('cert-file'), 'cert_file': slap_connection.get('cert-file'),
key_file=slap_connection.get('key-file'), 'key_file': slap_connection.get('key-file'),
partition_id=slap_connection['partition-id'], 'partition_id': slap_connection['partition-id'],
ssh_client=self.options['sshclient-binary'], 'ssh_client': self.options['sshclient-binary'],
) }
slaves = unjson(self.options['slave-instance-list']) slaves = json.loads(self.options['slave-instance-list'])
known_hosts = KnownHostsFile(self.options['known-hosts']) known_hosts = KnownHostsFile(self.options['known-hosts'])
with known_hosts: with known_hosts:
# XXX this API could be cleaner
for slave in slaves: for slave in slaves:
path_list.extend(self.add_slave(slave, known_hosts)) path_list.extend(self.add_slave(slave, known_hosts))
else: else:
command = [self.options['rdiffbackup-binary']]
self.logger.info("Server mode") self.logger.info("Server mode")
command.extend(['--restrict', self.options['path']])
command.append('--server')
wrapper = self.createPythonScript( wrapper = self.createWrapper(name=self.options['wrapper'],
self.options['wrapper'], command=self.options['rdiffbackup-binary'],
'slapos.recipe.librecipe.execute.execute', parameters=[
command) '--restrict', self.options['path'],
'--server'
])
path_list.append(wrapper) path_list.append(wrapper)
return path_list return path_list
slapos/recipe/request.py
View file @ d6a9aaa4
...@@ -123,20 +123,30 @@ class Recipe(object): ...@@ -123,20 +123,30 @@ class Recipe(object):
isSlave = options.get('slave', '').lower() in \ isSlave = options.get('slave', '').lower() in \
librecipe.GenericBaseRecipe.TRUE_VALUES librecipe.GenericBaseRecipe.TRUE_VALUES
self.instance = instance = request(software_url, software_type, self.instance = request(software_url, software_type,
name, partition_parameter_kw=partition_parameter_kw, name, partition_parameter_kw=partition_parameter_kw,
filter_kw=filter_kw, shared=isSlave) filter_kw=filter_kw, shared=isSlave)
self._raise_resource_not_ready = None
try:
# XXX what is the right way to get a global id?
options['instance_guid'] = self.instance.getId()
except slapmodule.ResourceNotReady as exc:
self._raise_resource_not_ready = exc
for param in return_parameters: for param in return_parameters:
try: try:
options['connection-%s' % param] = str( options['connection-%s' % param] = str(
instance.getConnectionParameter(param)) self.instance.getConnectionParameter(param))
except (slapmodule.NotFoundError, slapmodule.ServerError, slapmodule.ResourceNotReady): except (slapmodule.NotFoundError, slapmodule.ServerError, slapmodule.ResourceNotReady):
options['connection-%s' % param] = '' options['connection-%s' % param] = ''
if self.failed is None: if self.failed is None:
self.failed = param self.failed = param
def install(self): def install(self):
if self._raise_resource_not_ready:
raise slapmodule.ResourceNotReady(self._resource_not_ready)
if self.failed is not None: if self.failed is not None:
# Check instance status to know if instance has been deployed # Check instance status to know if instance has been deployed
try: try:
......
software/drupal/software.cfg
View file @ d6a9aaa4
...@@ -12,7 +12,6 @@ recipe = slapos.recipe.build:download-unpacked ...@@ -12,7 +12,6 @@ recipe = slapos.recipe.build:download-unpacked
url = http://ftp.drupal.org/files/projects/drupal-7.16.tar.gz url = http://ftp.drupal.org/files/projects/drupal-7.16.tar.gz
md5sum = 352497b2df94b5308e31cb8da020b631 md5sum = 352497b2df94b5308e31cb8da020b631
[download-patch-hide-dbsetup] [download-patch-hide-dbsetup]
recipe = hexagonit.recipe.download recipe = hexagonit.recipe.download
url = ${:_profile_base_location_}/${:filename} url = ${:_profile_base_location_}/${:filename}
......
stack/lamp/apache/instance-apache-backup.cfg → stack/lamp/apache/instance-apache-backup.cfg.in
View file @ d6a9aaa4
...@@ -16,32 +16,92 @@ eggs-directory = ${buildout:eggs-directory} ...@@ -16,32 +16,92 @@ eggs-directory = ${buildout:eggs-directory}
develop-eggs-directory = ${buildout:develop-eggs-directory} develop-eggs-directory = ${buildout:develop-eggs-directory}
offline = true offline = true
[urls]
recipe = slapos.cookbook:publish
url = http://[$${apache-proxy:ip}]:$${apache-proxy:port}/
ssh-public-key = $${sshkeys-dropbear:public-key-value}
ssh-url = ssh://nobody@[$${dropbear-server:host}]:$${dropbear-server:port}/$${rdiff-backup-server:path}
[apache-proxy] #----------------
recipe = slapos.cookbook:apacheproxy #--
url = $${slap-parameter:proxy-url} #-- Creation of all needed directories.
pid-file = $${basedirectory:run}/apache.pid
lock-file = $${basedirectory:run}/apache.lock
ip = $${slap-network-information:global-ipv6}
port = 8080
error-log = $${directory:httpd-log}/error.log
access-log = $${directory:httpd-log}/access.log
httpd-conf = $${rootdirectory:etc}/apache.conf
wrapper = $${basedirectory:services}/apache
promise = $${basedirectory:promises}/apache [rootdirectory]
recipe = slapos.cookbook:mkdirectory
etc = $${buildout:directory}/etc
var = $${buildout:directory}/var
srv = $${buildout:directory}/srv
bin = $${buildout:directory}/bin
tmp = $${buildout:directory}/tmp
httpd-binary = ${apache:location}/bin/httpd [basedirectory]
recipe = slapos.cookbook:mkdirectory
log = $${rootdirectory:var}/log
services = $${rootdirectory:etc}/run
run = $${rootdirectory:var}/run
backup = $${rootdirectory:srv}/backup
promises = $${rootdirectory:etc}/promise
[directory]
recipe = slapos.cookbook:mkdirectory
htdocs = $${rootdirectory:srv}/www
logrotate-entries = $${rootdirectory:etc}/logrotate.d
logrotate-backup = $${basedirectory:backup}/logrotate
cronstamps = $${rootdirectory:etc}/cronstamps
cron-entries = $${rootdirectory:etc}/cron.d
crontabs = $${rootdirectory:etc}/crontabs
ssh = $${rootdirectory:etc}/ssh
sshkeys = $${rootdirectory:srv}/sshkeys
httpd-log = $${basedirectory:log}/apache
#----------------
#--
#-- Deploy cron.
[cron]
recipe = slapos.cookbook:cron
dcrond-binary = ${dcron:location}/sbin/crond
cron-entries = $${directory:cron-entries}
crontabs = $${directory:crontabs}
cronstamps = $${directory:cronstamps}
catcher = $${cron-simplelogger:wrapper}
binary = $${basedirectory:services}/crond
[cron-simplelogger]
recipe = slapos.cookbook:simplelogger
wrapper = $${rootdirectory:bin}/cron_simplelogger
log = $${basedirectory:log}/crond.log
#----------------
#--
#-- Deploy logrotate.
[cron-entry-logrotate]
<= cron
recipe = slapos.cookbook:cron.d
name = logrotate
frequency = 0 0 * * *
command = $${logrotate:wrapper}
[logrotate]
recipe = slapos.cookbook:logrotate
# Binaries
logrotate-binary = ${logrotate:location}/usr/sbin/logrotate
gzip-binary = ${gzip:location}/bin/gzip
gunzip-binary = ${gzip:location}/bin/gunzip
# Directories
wrapper = $${rootdirectory:bin}/logrotate
conf = $${rootdirectory:etc}/logrotate.conf
logrotate-entries = $${directory:logrotate-entries}
backup = $${directory:logrotate-backup}
state-file = $${rootdirectory:srv}/logrotate.status
#----------------
#--
#-- sshkeys
[sshkeys-directory] [sshkeys-directory]
recipe = slapos.cookbook:mkdirectory recipe = slapos.cookbook:mkdirectory
requests = $${directory:sshkeys}/requests/ requests = $${directory:sshkeys}/requests
keys = $${directory:sshkeys}/keys/ keys = $${directory:sshkeys}/keys
[sshkeys-authority] [sshkeys-authority]
recipe = slapos.cookbook:sshkeys_authority recipe = slapos.cookbook:sshkeys_authority
...@@ -60,6 +120,11 @@ public-key = $${dropbear-server:rsa-keyfile}.pub ...@@ -60,6 +120,11 @@ public-key = $${dropbear-server:rsa-keyfile}.pub
private-key = $${dropbear-server:rsa-keyfile} private-key = $${dropbear-server:rsa-keyfile}
wrapper = $${basedirectory:services}/sshd wrapper = $${basedirectory:services}/sshd
#----------------
#--
#-- Dropbear.
[dropbear-server] [dropbear-server]
recipe = slapos.cookbook:dropbear recipe = slapos.cookbook:dropbear
host = $${slap-network-information:global-ipv6} host = $${slap-network-information:global-ipv6}
...@@ -75,6 +140,11 @@ dropbear-binary = ${dropbear:location}/sbin/dropbear ...@@ -75,6 +140,11 @@ dropbear-binary = ${dropbear:location}/sbin/dropbear
recipe = slapos.cookbook:dropbear.add_authorized_key recipe = slapos.cookbook:dropbear.add_authorized_key
key = $${slap-parameter:authorized-key} key = $${slap-parameter:authorized-key}
#----------------
#--
#-- rdiff
[rdiff-backup-server] [rdiff-backup-server]
recipe = slapos.cookbook:pbs recipe = slapos.cookbook:pbs
client = false client = false
...@@ -82,18 +152,26 @@ path = $${directory:htdocs} ...@@ -82,18 +152,26 @@ path = $${directory:htdocs}
wrapper = $${rootdirectory:bin}/rdiffbackup-server wrapper = $${rootdirectory:bin}/rdiffbackup-server
rdiffbackup-binary = ${buildout:bin-directory}/rdiff-backup rdiffbackup-binary = ${buildout:bin-directory}/rdiff-backup
[logrotate]
recipe = slapos.cookbook:logrotate #----------------
# Binaries #--
logrotate-binary = ${logrotate:location}/usr/sbin/logrotate #-- Apache Proxy.
gzip-binary = ${gzip:location}/bin/gzip
gunzip-binary = ${gzip:location}/bin/gunzip [apache-proxy]
# Directories recipe = slapos.cookbook:apacheproxy
wrapper = $${rootdirectory:bin}/logrotate url = $${slap-parameter:proxy-url}
conf = $${rootdirectory:etc}/logrotate.conf pid-file = $${basedirectory:run}/apache.pid
logrotate-entries = $${directory:logrotate-entries} lock-file = $${basedirectory:run}/apache.lock
backup = $${directory:logrotate-backup} ip = $${slap-network-information:global-ipv6}
state-file = $${rootdirectory:srv}/logrotate.status port = 8080
error-log = $${directory:httpd-log}/error.log
access-log = $${directory:httpd-log}/access.log
httpd-conf = $${rootdirectory:etc}/apache.conf
wrapper = $${basedirectory:services}/apache
promise = $${basedirectory:promises}/apache
httpd-binary = ${apache:location}/bin/httpd
[logrotate-entry-apache] [logrotate-entry-apache]
<= logrotate <= logrotate
...@@ -106,51 +184,14 @@ sharedscripts = true ...@@ -106,51 +184,14 @@ sharedscripts = true
notifempty = true notifempty = true
create = true create = true
[cron]
recipe = slapos.cookbook:cron
dcrond-binary = ${dcron:location}/sbin/crond
cron-entries = $${directory:cron-entries}
crontabs = $${directory:crontabs}
cronstamps = $${directory:cronstamps}
catcher = $${cron-simplelogger:wrapper}
binary = $${basedirectory:services}/crond
[cron-simplelogger]
recipe = slapos.cookbook:simplelogger
wrapper = $${rootdirectory:bin}/cron_simplelogger
log = $${basedirectory:log}/crond.log
[cron-entry-logrotate]
<= cron
recipe = slapos.cookbook:cron.d
name = logrotate
frequency = 0 0 * * *
command = $${logrotate:wrapper}
[rootdirectory] #----------------
recipe = slapos.cookbook:mkdirectory #--
etc = $${buildout:directory}/etc/ #-- Publish instance parameters.
var = $${buildout:directory}/var/
srv = $${buildout:directory}/srv/
bin = $${buildout:directory}/bin/
tmp = $${buildout:directory}/tmp/
[basedirectory] [urls]
recipe = slapos.cookbook:mkdirectory recipe = slapos.cookbook:publish
log = $${rootdirectory:var}/log/ url = http://[$${apache-proxy:ip}]:$${apache-proxy:port}/
services = $${rootdirectory:etc}/run/ ssh-public-key = $${sshkeys-dropbear:public-key-value}
run = $${rootdirectory:var}/run/ ssh-url = ssh://nobody@[$${dropbear-server:host}]:$${dropbear-server:port}/$${rdiff-backup-server:path}
backup = $${rootdirectory:srv}/backup/
promises = $${rootdirectory:etc}/promise/
[directory]
recipe = slapos.cookbook:mkdirectory
htdocs = $${rootdirectory:srv}/www/
logrotate-entries = $${rootdirectory:etc}/logrotate.d/
logrotate-backup = $${basedirectory:backup}/logrotate/
cronstamps = $${rootdirectory:etc}/cronstamps/
cron-entries = $${rootdirectory:etc}/cron.d/
crontabs = $${rootdirectory:etc}/crontabs/
ssh = $${rootdirectory:etc}/ssh/
sshkeys = $${rootdirectory:srv}/sshkeys
httpd-log = $${basedirectory:log}/apache/
stack/lamp/apache/instance-apache-import.cfg
View file @ d6a9aaa4
[buildout] [buildout]
extends = ${template-apache-php:output} extends = ${instance-apache-php:output}
${template-pbsready-export:output} ${pbsready-export:output}
parts = parts =
apache-proxy apache-proxy
...@@ -13,6 +13,7 @@ parts = ...@@ -13,6 +13,7 @@ parts =
dropbear-server dropbear-server
dropbear-server-pbs-authorized-key dropbear-server-pbs-authorized-key
[apache-proxy] [apache-proxy]
recipe = slapos.cookbook:apacheproxy recipe = slapos.cookbook:apacheproxy
url = $${slap-parameter:proxy-url} url = $${slap-parameter:proxy-url}
...@@ -29,10 +30,103 @@ promise = $${basedirectory:promises}/apache ...@@ -29,10 +30,103 @@ promise = $${basedirectory:promises}/apache
httpd-binary = ${apache:location}/bin/httpd httpd-binary = ${apache:location}/bin/httpd
#----------------
#--
#-- Creation of all needed directories.
[rootdirectory]
recipe = slapos.cookbook:mkdirectory
etc = $${buildout:directory}/etc
var = $${buildout:directory}/var
srv = $${buildout:directory}/srv
bin = $${buildout:directory}/bin
tmp = $${buildout:directory}/tmp
[basedirectory]
recipe = slapos.cookbook:mkdirectory
log = $${rootdirectory:var}/log
services = $${rootdirectory:etc}/run
run = $${rootdirectory:var}/run
backup = $${rootdirectory:srv}/backup
promises = $${rootdirectory:etc}/promise
[directory]
recipe = slapos.cookbook:mkdirectory
htdocs = $${rootdirectory:srv}/www
logrotate-entries = $${rootdirectory:etc}/logrotate.d
logrotate-backup = $${basedirectory:backup}/logrotate
cronstamps = $${rootdirectory:etc}/cronstamps
cron-entries = $${rootdirectory:etc}/cron.d
crontabs = $${rootdirectory:etc}/crontabs
ssh = $${rootdirectory:etc}/ssh
sshkeys = $${rootdirectory:srv}/sshkeys
httpd-log = $${basedirectory:log}/apache
#----------------
#--
#-- Deploy cron.
[cron]
recipe = slapos.cookbook:cron
dcrond-binary = ${dcron:location}/sbin/crond
cron-entries = $${directory:cron-entries}
crontabs = $${directory:crontabs}
cronstamps = $${directory:cronstamps}
catcher = $${cron-simplelogger:wrapper}
binary = $${basedirectory:services}/crond
[cron-simplelogger]
recipe = slapos.cookbook:simplelogger
wrapper = $${rootdirectory:bin}/cron_simplelogger
log = $${basedirectory:log}/crond.log
#----------------
#--
#-- Deploy logrotate.
[cron-entry-logrotate]
<= cron
recipe = slapos.cookbook:cron.d
name = logrotate
frequency = 0 0 * * *
command = $${logrotate:wrapper}
[logrotate]
recipe = slapos.cookbook:logrotate
# Binaries
logrotate-binary = ${logrotate:location}/usr/sbin/logrotate
gzip-binary = ${gzip:location}/bin/gzip
gunzip-binary = ${gzip:location}/bin/gunzip
# Directories
wrapper = $${rootdirectory:bin}/logrotate
conf = $${rootdirectory:etc}/logrotate.conf
logrotate-entries = $${directory:logrotate-entries}
backup = $${directory:logrotate-backup}
state-file = $${rootdirectory:srv}/logrotate.status
[logrotate-entry-apache]
<= logrotate
recipe = slapos.cookbook:logrotate.d
name = apache
log = $${apache-proxy:error-log} $${apache-proxy:access-log}
frequency = daily
rotate-num = 30
sharedscripts = true
notifempty = true
create = true
#----------------
#--
#-- sshkeys
[sshkeys-directory] [sshkeys-directory]
recipe = slapos.cookbook:mkdirectory recipe = slapos.cookbook:mkdirectory
requests = $${directory:sshkeys}/requests/ requests = $${directory:sshkeys}/requests
keys = $${directory:sshkeys}/keys/ keys = $${directory:sshkeys}/keys
[sshkeys-authority] [sshkeys-authority]
recipe = slapos.cookbook:sshkeys_authority recipe = slapos.cookbook:sshkeys_authority
...@@ -51,6 +145,11 @@ public-key = $${dropbear-server:rsa-keyfile}.pub ...@@ -51,6 +145,11 @@ public-key = $${dropbear-server:rsa-keyfile}.pub
private-key = $${dropbear-server:rsa-keyfile} private-key = $${dropbear-server:rsa-keyfile}
wrapper = $${basedirectory:services}/sshd wrapper = $${basedirectory:services}/sshd
#----------------
#--
#-- Dropbear.
[dropbear-server] [dropbear-server]
recipe = slapos.cookbook:dropbear recipe = slapos.cookbook:dropbear
host = $${slap-network-information:global-ipv6} host = $${slap-network-information:global-ipv6}
...@@ -66,6 +165,11 @@ dropbear-binary = ${dropbear:location}/sbin/dropbear ...@@ -66,6 +165,11 @@ dropbear-binary = ${dropbear:location}/sbin/dropbear
recipe = slapos.cookbook:dropbear.add_authorized_key recipe = slapos.cookbook:dropbear.add_authorized_key
key = $${slap-parameter:authorized-key} key = $${slap-parameter:authorized-key}
#----------------
#--
#-- rdiff
[rdiff-backup-server] [rdiff-backup-server]
recipe = slapos.cookbook:pbs recipe = slapos.cookbook:pbs
client = false client = false
...@@ -73,75 +177,3 @@ path = $${directory:htdocs} ...@@ -73,75 +177,3 @@ path = $${directory:htdocs}
wrapper = $${rootdirectory:bin}/rdiffbackup-server wrapper = $${rootdirectory:bin}/rdiffbackup-server
rdiffbackup-binary = ${buildout:bin-directory}/rdiff-backup rdiffbackup-binary = ${buildout:bin-directory}/rdiff-backup
[logrotate]
recipe = slapos.cookbook:logrotate
# Binaries
logrotate-binary = ${logrotate:location}/usr/sbin/logrotate
gzip-binary = ${gzip:location}/bin/gzip
gunzip-binary = ${gzip:location}/bin/gunzip
# Directories
wrapper = $${rootdirectory:bin}/logrotate
conf = $${rootdirectory:etc}/logrotate.conf
logrotate-entries = $${directory:logrotate-entries}
backup = $${directory:logrotate-backup}
state-file = $${rootdirectory:srv}/logrotate.status
[logrotate-entry-apache]
<= logrotate
recipe = slapos.cookbook:logrotate.d
name = apache
log = $${apache-proxy:error-log} $${apache-proxy:access-log}
frequency = daily
rotate-num = 30
sharedscripts = true
notifempty = true
create = true
[cron]
recipe = slapos.cookbook:cron
dcrond-binary = ${dcron:location}/sbin/crond
cron-entries = $${directory:cron-entries}
crontabs = $${directory:crontabs}
cronstamps = $${directory:cronstamps}
catcher = $${cron-simplelogger:wrapper}
binary = $${basedirectory:services}/crond
[cron-simplelogger]
recipe = slapos.cookbook:simplelogger
wrapper = $${rootdirectory:bin}/cron_simplelogger
log = $${basedirectory:log}/crond.log
[cron-entry-logrotate]
<= cron
recipe = slapos.cookbook:cron.d
name = logrotate
frequency = 0 0 * * *
command = $${logrotate:wrapper}
[rootdirectory]
recipe = slapos.cookbook:mkdirectory
etc = $${buildout:directory}/etc/
var = $${buildout:directory}/var/
srv = $${buildout:directory}/srv/
bin = $${buildout:directory}/bin/
tmp = $${buildout:directory}/tmp/
[basedirectory]
recipe = slapos.cookbook:mkdirectory
log = $${rootdirectory:var}/log/
services = $${rootdirectory:etc}/run/
run = $${rootdirectory:var}/run/
backup = $${rootdirectory:srv}/backup/
promises = $${rootdirectory:etc}/promise/
[directory]
recipe = slapos.cookbook:mkdirectory
htdocs = $${rootdirectory:srv}/www/
logrotate-entries = $${rootdirectory:etc}/logrotate.d/
logrotate-backup = $${basedirectory:backup}/logrotate/
cronstamps = $${rootdirectory:etc}/cronstamps/
cron-entries = $${rootdirectory:etc}/cron.d/
crontabs = $${rootdirectory:etc}/crontabs/
ssh = $${rootdirectory:etc}/ssh/
sshkeys = $${rootdirectory:srv}/sshkeys
httpd-log = $${basedirectory:log}/apache/
stack/lamp/apache/instance-apache-php.cfg → stack/lamp/apache/instance-apache-php.cfg.in
View file @ d6a9aaa4
...@@ -20,46 +20,96 @@ develop-eggs-directory = ${buildout:develop-eggs-directory} ...@@ -20,46 +20,96 @@ develop-eggs-directory = ${buildout:develop-eggs-directory}
offline = true offline = true
# Creation of all needed directories #----------------
#--
#-- Creation of all needed directories.
[rootdirectory] [rootdirectory]
recipe = slapos.cookbook:mkdirectory recipe = slapos.cookbook:mkdirectory
etc = $${buildout:directory}/etc/ etc = $${buildout:directory}/etc
var = $${buildout:directory}/var/ var = $${buildout:directory}/var
srv = $${buildout:directory}/srv/ srv = $${buildout:directory}/srv
bin = $${buildout:directory}/bin/ bin = $${buildout:directory}/bin
tmp = $${buildout:directory}/tmp/ tmp = $${buildout:directory}/tmp
[basedirectory] [basedirectory]
recipe = slapos.cookbook:mkdirectory recipe = slapos.cookbook:mkdirectory
log = $${rootdirectory:var}/log/ log = $${rootdirectory:var}/log
services = $${rootdirectory:etc}/run/ services = $${rootdirectory:etc}/run
run = $${rootdirectory:var}/run/ run = $${rootdirectory:var}/run
backup = $${rootdirectory:srv}/backup/ backup = $${rootdirectory:srv}/backup
promises = $${rootdirectory:etc}/promise/ promises = $${rootdirectory:etc}/promise
[directory] [directory]
recipe = slapos.cookbook:mkdirectory recipe = slapos.cookbook:mkdirectory
ca-dir = $${rootdirectory:srv}/ssl/ ca-dir = $${rootdirectory:srv}/ssl
httpd-log = $${basedirectory:log}/apache/ httpd-log = $${basedirectory:log}/apache
php-ini-dir = $${rootdirectory:etc}/php/ php-ini-dir = $${rootdirectory:etc}/php
tmp-php = $${rootdirectory:tmp}/php/ tmp-php = $${rootdirectory:tmp}/php
logrotate-entries = $${rootdirectory:etc}/logrotate.d/ logrotate-entries = $${rootdirectory:etc}/logrotate.d
logrotate-backup = $${basedirectory:backup}/logrotate/ logrotate-backup = $${basedirectory:backup}/logrotate
stunnel-conf = $${rootdirectory:etc}/stunnel/ stunnel-conf = $${rootdirectory:etc}/stunnel
cronstamps = $${rootdirectory:etc}/cronstamps/ cronstamps = $${rootdirectory:etc}/cronstamps
cron-entries = $${rootdirectory:etc}/cron.d/ cron-entries = $${rootdirectory:etc}/cron.d
crontabs = $${rootdirectory:etc}/crontabs/ crontabs = $${rootdirectory:etc}/crontabs
[cadirectory] [cadirectory]
recipe = slapos.cookbook:mkdirectory recipe = slapos.cookbook:mkdirectory
requests = $${directory:ca-dir}/requests/ requests = $${directory:ca-dir}/requests
private = $${directory:ca-dir}/private/ private = $${directory:ca-dir}/private
certs = $${directory:ca-dir}/certs/ certs = $${directory:ca-dir}/certs
newcerts = $${directory:ca-dir}/newcerts/ newcerts = $${directory:ca-dir}/newcerts
crl = $${directory:ca-dir}/crl/ crl = $${directory:ca-dir}/crl
#----------------
#--
#-- Deploy cron.
[cron]
recipe = slapos.cookbook:cron
dcrond-binary = ${dcron:location}/sbin/crond
cron-entries = $${directory:cron-entries}
crontabs = $${directory:crontabs}
cronstamps = $${directory:cronstamps}
catcher = $${cron-simplelogger:wrapper}
binary = $${basedirectory:services}/crond
[cron-simplelogger]
recipe = slapos.cookbook:simplelogger
wrapper = $${rootdirectory:bin}/cron_simplelogger
log = $${basedirectory:log}/crond.log
#----------------
#--
#-- Deploy logrotate.
[cron-entry-logrotate]
<= cron
recipe = slapos.cookbook:cron.d
name = logrotate
frequency = 0 0 * * *
command = $${logrotate:wrapper}
[logrotate]
recipe = slapos.cookbook:logrotate
# Binaries
logrotate-binary = ${logrotate:location}/usr/sbin/logrotate
gzip-binary = ${gzip:location}/bin/gzip
gunzip-binary = ${gzip:location}/bin/gunzip
# Directories
wrapper = $${rootdirectory:bin}/logrotate
conf = $${rootdirectory:etc}/logrotate.conf
logrotate-entries = $${directory:logrotate-entries}
backup = $${directory:logrotate-backup}
state-file = $${rootdirectory:srv}/logrotate.status
#----------------
#--
#-- Deploy stunnel.
# Deploy stunnel
[stunnel] [stunnel]
recipe = slapos.cookbook:stunnel recipe = slapos.cookbook:stunnel
client = true client = true
...@@ -76,8 +126,22 @@ pid-file = $${basedirectory:run}/stunnel.pid ...@@ -76,8 +126,22 @@ pid-file = $${basedirectory:run}/stunnel.pid
wrapper = $${rootdirectory:bin}/raw_stunnel wrapper = $${rootdirectory:bin}/raw_stunnel
post-rotate-script = $${rootdirectory:bin}/stunnel_post_rotate post-rotate-script = $${rootdirectory:bin}/stunnel_post_rotate
[logrotate-entry-stunnel]
<= logrotate
recipe = slapos.cookbook:logrotate.d
name = stunnel
log = $${stunnel:log-file}
frequency = daily
rotate-num = 30
notifempty = true
create = true
post = $${stunnel:post-rotate-script}
#----------------
#--
#-- Certificate stuff.
# Certificate stuffs
[certificate-authority] [certificate-authority]
recipe = slapos.cookbook:certificate_authority recipe = slapos.cookbook:certificate_authority
openssl-binary = ${openssl:location}/bin/openssl openssl-binary = ${openssl:location}/bin/openssl
...@@ -98,7 +162,10 @@ key-file = $${stunnel:key-file} ...@@ -98,7 +162,10 @@ key-file = $${stunnel:key-file}
cert-file = $${stunnel:cert-file} cert-file = $${stunnel:cert-file}
# Request MariaDB instance and parse its URL #----------------
#--
#-- Request MariaDB instance and parse its URL.
[request-mariadb] [request-mariadb]
<= slap-connection <= slap-connection
recipe = slapos.cookbook:request recipe = slapos.cookbook:request
...@@ -114,7 +181,10 @@ recipe = slapos.cookbook:urlparse ...@@ -114,7 +181,10 @@ recipe = slapos.cookbook:urlparse
url = $${request-mariadb:connection-url} url = $${request-mariadb:connection-url}
# Deploy Apache + PHP application #----------------
#--
#-- Deploy Apache + PHP application.
[apache-php] [apache-php]
recipe = slapos.cookbook:apachephp recipe = slapos.cookbook:apachephp
source = ${application:location} source = ${application:location}
...@@ -142,21 +212,6 @@ mysql-database = $${mariadb-urlparse:path} ...@@ -142,21 +212,6 @@ mysql-database = $${mariadb-urlparse:path}
mysql-host = $${stunnel:local-host} mysql-host = $${stunnel:local-host}
mysql-port = $${stunnel:local-port} mysql-port = $${stunnel:local-port}
# Deploy logrotate, cron, configure it
[logrotate]
recipe = slapos.cookbook:logrotate
# Binaries
logrotate-binary = ${logrotate:location}/usr/sbin/logrotate
gzip-binary = ${gzip:location}/bin/gzip
gunzip-binary = ${gzip:location}/bin/gunzip
# Directories
wrapper = $${rootdirectory:bin}/logrotate
conf = $${rootdirectory:etc}/logrotate.conf
logrotate-entries = $${directory:logrotate-entries}
backup = $${directory:logrotate-backup}
state-file = $${rootdirectory:srv}/logrotate.status
[logrotate-entry-apache] [logrotate-entry-apache]
<= logrotate <= logrotate
recipe = slapos.cookbook:logrotate.d recipe = slapos.cookbook:logrotate.d
...@@ -168,40 +223,11 @@ sharedscripts = true ...@@ -168,40 +223,11 @@ sharedscripts = true
notifempty = true notifempty = true
create = true create = true
[logrotate-entry-stunnel]
<= logrotate
recipe = slapos.cookbook:logrotate.d
name = stunnel
log = $${stunnel:log-file}
frequency = daily
rotate-num = 30
notifempty = true
create = true
post = $${stunnel:post-rotate-script}
[cron-simplelogger]
recipe = slapos.cookbook:simplelogger
wrapper = $${rootdirectory:bin}/cron_simplelogger
log = $${basedirectory:log}/crond.log
[cron]
recipe = slapos.cookbook:cron
dcrond-binary = ${dcron:location}/sbin/crond
cron-entries = $${directory:cron-entries}
crontabs = $${directory:crontabs}
cronstamps = $${directory:cronstamps}
catcher = $${cron-simplelogger:wrapper}
binary = $${basedirectory:services}/crond
[cron-entry-logrotate]
<= cron
recipe = slapos.cookbook:cron.d
name = logrotate
frequency = 0 0 * * *
command = $${logrotate:wrapper}
#----------------
#--
#-- Request frontend.
# Request frontend
[request-frontend] [request-frontend]
<= slap-connection <= slap-connection
recipe = slapos.cookbook:requestoptional recipe = slapos.cookbook:requestoptional
...@@ -215,7 +241,10 @@ return = site_url ...@@ -215,7 +241,10 @@ return = site_url
config-custom_domain = $${slap-parameter:domain} config-custom_domain = $${slap-parameter:domain}
# Deploy slapmonitor #----------------
#--
#-- Deploy slapmonitor.
[slapmonitor] [slapmonitor]
recipe = slapos.cookbook:slapmonitor recipe = slapos.cookbook:slapmonitor
pid-file = $${basedirectory:run}/apache.pid pid-file = $${basedirectory:run}/apache.pid
...@@ -238,14 +267,20 @@ slapreport-path = ${buildout:bin-directory}/slapreport ...@@ -238,14 +267,20 @@ slapreport-path = ${buildout:bin-directory}/slapreport
path = $${basedirectory:services}/slapreport path = $${basedirectory:services}/slapreport
# Publish all instance parameters (url of instance) #----------------
#--
#-- Publish all instance parameters (url of instance).
[publish-connection-informations] [publish-connection-informations]
recipe = slapos.cookbook:publish recipe = slapos.cookbook:publish
backend_url = $${apache-php:url} backend_url = $${apache-php:url}
url = $${request-frontend:connection-site_url} url = $${request-frontend:connection-site_url}
# Deploy promises scripts #----------------
#--
#-- Deploy promises scripts.
[promise] [promise]
recipe = slapos.cookbook:check_port_listening recipe = slapos.cookbook:check_port_listening
path = $${basedirectory:promises}/apache path = $${basedirectory:promises}/apache
...@@ -266,6 +301,9 @@ url = $${request-frontend:connection-site_url} ...@@ -266,6 +301,9 @@ url = $${request-frontend:connection-site_url}
dash_path = ${dash:location}/bin/dash dash_path = ${dash:location}/bin/dash
curl_path = ${curl:location}/bin/curl curl_path = ${curl:location}/bin/curl
[slap-parameter] [slap-parameter]
# Default value if no domain is specified # Default value if no domain is specified
domain = domain =
...@@ -274,3 +312,4 @@ logbox-ip = ...@@ -274,3 +312,4 @@ logbox-ip =
logbox-port = logbox-port =
logbox-user = logbox-user =
logbox-passwd = logbox-passwd =
stack/lamp/buildout.cfg
View file @ d6a9aaa4
...@@ -13,13 +13,16 @@ parts = ...@@ -13,13 +13,16 @@ parts =
instance-recipe-egg instance-recipe-egg
template template
template-apache-php instance-apache-php
template-mariadb template-mariadb
#Contains the importer and exporter recipes for mariadb #Contains the importer and exporter recipes for mariadb
template-mariadb-import template-mariadb-import
template-mariadb-export template-mariadb-export
instance-default-root
extends = extends =
../resilient/buildout.cfg ../resilient/buildout.cfg
../../component/mariadb/buildout.cfg ../../component/mariadb/buildout.cfg
...@@ -61,66 +64,80 @@ recipe = hexagonit.recipe.download ...@@ -61,66 +64,80 @@ recipe = hexagonit.recipe.download
#If provided tarball does not containt top directory this option shall be changed to false #If provided tarball does not containt top directory this option shall be changed to false
strip-top-level-dir = true strip-top-level-dir = true
#----------------
#-- Instance-level buildout profiles.
[template] [template]
recipe = slapos.recipe.template recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance.cfg url = ${:_profile_base_location_}/instance.cfg.in
output = ${buildout:directory}/template.cfg output = ${buildout:directory}/instance.cfg
md5sum = 8b4660ccaccda1fa8b0e73b8ac38be11 md5sum = 9e6a4adaa6b5ac923ad16e637bf5170c
mode = 0644 mode = 0644
[template-apache-php] [instance-apache-php]
recipe = slapos.recipe.template recipe = slapos.recipe.template
url = ${:_profile_base_location_}/apache/instance-apache-php.cfg url = ${:_profile_base_location_}/apache/instance-apache-php.cfg.in
output = ${buildout:directory}/template-apache-php.cfg output = ${buildout:directory}/instance-apache-php.cfg
md5sum = a5dd222b3faa4e1ef2df9b3b9bb47966 md5sum = dec36ec4fe54d87f4a5e5b8573f72fb7
mode = 0644 mode = 0644
[template-apache-backup] [instance-apache-backup]
recipe = slapos.recipe.template recipe = slapos.recipe.template
url = ${:_profile_base_location_}/apache/instance-apache-backup.cfg url = ${:_profile_base_location_}/apache/instance-apache-backup.cfg.in
output = ${buildout:directory}/template-apache-backup.cfg output = ${buildout:directory}/instance-apache-backup.cfg
md5sum = cfb77ac8785e0d125a785f69a5339014 md5sum = 48f969d82319a9d145570f5f0fd27672
mode = 0644 mode = 0644
[template-resilient-lamp] [template-resilient-lamp]
recipe = slapos.recipe.template:jinja2 recipe = slapos.recipe.template:jinja2
template = ${:_profile_base_location_}/instance-resilient.cfg template = ${:_profile_base_location_}/template-resilient.cfg.in
rendered = ${buildout:directory}/template-resilient.cfg rendered = ${buildout:directory}/instance-resilient.cfg
context = key templateapache template-apache-php:output context = key templateapache instance-apache-php:output
key dropbear dropbear:location key dropbear dropbear:location
key buildout buildout:bin-directory key buildout buildout:bin-directory
import-list = file parts template-parts:destination import-list = file parts template-parts:destination
file replicated template-replicated:destination file replicated template-replicated:destination
md5sum = 03aafcba5c626a4a1bd180d71007be1e md5sum = 5151295af3994499e2be6d68938787d3
mode = 0644 mode = 0644
[template-mariadb] [template-mariadb]
recipe = slapos.recipe.template recipe = slapos.recipe.template
url = ${:_profile_base_location_}/mariadb/instance-mariadb.cfg url = ${:_profile_base_location_}/mariadb/instance-mariadb.cfg.in
output = ${buildout:directory}/template-mariadb.cfg output = ${buildout:directory}/instance-mariadb.cfg
md5sum = fa9dc10efbcf61119f4cbab37c741322 md5sum = aa33c843f04753a34154467522f0c65e
mode = 0644 mode = 0644
[template-mariadb-import] [template-mariadb-import]
recipe = slapos.recipe.template recipe = slapos.recipe.template
url = ${:_profile_base_location_}/mariadb/instance-mariadb-import.cfg url = ${:_profile_base_location_}/mariadb/instance-mariadb-import.cfg.in
output = ${buildout:directory}/template-mariadb-import.cfg output = ${buildout:directory}/instance-mariadb-import.cfg
md5sum = fa696733db4bd5b2e3e9fb6e0b09c59b md5sum = 24a9286f1aed04f7393c92f251cbe0c9
mode = 0644 mode = 0644
[template-mariadb-export] [template-mariadb-export]
recipe = slapos.recipe.template recipe = slapos.recipe.template
url = ${:_profile_base_location_}/mariadb/instance-mariadb-export.cfg url = ${:_profile_base_location_}/mariadb/instance-mariadb-export.cfg.in
output = ${buildout:directory}/template-mariadb-export.cfg output = ${buildout:directory}/instance-mariadb-export.cfg
md5sum = 4b7dec765265b27c8235419b82ca7b02 md5sum = 96e915bca045e3ff62a3f8fc6e692685
mode = 0644
[instance-default-root]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance-default-root.cfg.in
output = ${buildout:directory}/instance-default-root.cfg
md5sum = 53c9020f7a0b5203f976e069e455787b
mode = 0644 mode = 0644
# Dummy parts in case no application configuration file is needed #----------------
#-- Dummy parts in case no application configuration file is needed
[application-template] [application-template]
filename = filename =
location = location =
...@@ -128,6 +145,8 @@ location = ...@@ -128,6 +145,8 @@ location =
[application-configuration] [application-configuration]
location = location =
#----------------
[eggs] [eggs]
recipe = zc.recipe.egg recipe = zc.recipe.egg
eggs = eggs =
......
stack/lamp/instance-default-root.cfg.in 0 → 100644
View file @ d6a9aaa4
[buildout]
parts = request-apache
[request-apache]
<= slap-connection
recipe = slapos.cookbook:request
software-url = $${slap-connection:software-release-url}
software-type = apache
name = Apache
stack/lamp/instance.cfg → stack/lamp/instance.cfg.in
View file @ d6a9aaa4
[buildout] [buildout]
extends =
${template-switchsoftware:output}
parts = parts =
switch_softwaretype switch_softwaretype
...@@ -12,12 +9,16 @@ offline = true ...@@ -12,12 +9,16 @@ offline = true
[switch_softwaretype] [switch_softwaretype]
recipe = slapos.cookbook:softwaretype recipe = slapos.cookbook:softwaretype
default = ${template-apache-php:output} default = ${instance-default-root:output}
apache = ${instance-apache-php:output}
resilient = ${template-resilient-lamp:rendered} resilient = ${template-resilient-lamp:rendered}
mariadb = ${template-mariadb:output} mariadb = ${template-mariadb:output}
mariadb-import = ${template-mariadb-import:output} mariadb-import = ${template-mariadb-import:output}
mariadb-export = ${template-mariadb-export:output} mariadb-export = ${template-mariadb-export:output}
apache-backup = ${instance-apache-backup:output}
#frozen creates a syntax error, meaning it can keep its data.
#It's dirty as hell, it needs to be replaced.
frozen = ${instance-frozen:output}
pull-backup = ${template-pull-backup:output} pull-backup = ${template-pull-backup:output}
apache-backup = ${template-apache-backup:output}
frozen = ${template-frozen:output}
stack/lamp/mariadb/instance-mariadb-export.cfg → stack/lamp/mariadb/instance-mariadb-export.cfg.in
View file @ d6a9aaa4
[buildout] [buildout]
extends = ${template-mariadb:output} extends = ${template-mariadb:output}
${template-pbsready-export:output} ${pbsready-export:output}
parts += mariadb parts += mariadb
...@@ -12,4 +12,4 @@ socket = $${mariadb:socket} ...@@ -12,4 +12,4 @@ socket = $${mariadb:socket}
user = root user = root
mydumper-binary = ${mydumper:location}/bin/mydumper mydumper-binary = ${mydumper:location}/bin/mydumper
database = $${mariadb:database} database = $${mariadb:database}
import = false import = false
\ No newline at end of file
stack/lamp/mariadb/instance-mariadb-import.cfg → stack/lamp/mariadb/instance-mariadb-import.cfg.in
View file @ d6a9aaa4
[buildout] [buildout]
extends = ${template-mariadb:output} extends = ${template-mariadb:output}
${template-pbsready-import:output} ${pbsready-import:output}
parts += mariadb parts += mariadb
......
stack/lamp/instance-resilient.cfg → stack/lamp/template-resilient.cfg.in
View file @ d6a9aaa4
...@@ -44,8 +44,8 @@ slave = false ...@@ -44,8 +44,8 @@ slave = false
[sshkeys-directory] [sshkeys-directory]
recipe = slapos.cookbook:mkdirectory recipe = slapos.cookbook:mkdirectory
requests = ${directory:sshkeys}/requests/ requests = ${directory:sshkeys}/requests
keys = ${directory:sshkeys}/keys/ keys = ${directory:sshkeys}/keys
[sshkeys-authority] [sshkeys-authority]
recipe = slapos.cookbook:sshkeys_authority recipe = slapos.cookbook:sshkeys_authority
...@@ -121,6 +121,8 @@ config-notify = ${request-pull-backup-server:connection-notification-url} ...@@ -121,6 +121,8 @@ config-notify = ${request-pull-backup-server:connection-notification-url}
config-notification-id = ${slap-connection:computer-id}-${slap-connection:partition-id}-apache-pull config-notification-id = ${slap-connection:computer-id}-${slap-connection:partition-id}-apache-pull
config-frequency = 30 * * * * config-frequency = 30 * * * *
slave = true slave = true
sla = instance_guid
sla-instance_guid = ${request-pull-backup-server:instance_guid}
[request-pull-backup-server-apache-2] [request-pull-backup-server-apache-2]
<= request-pbs-common <= request-pbs-common
...@@ -134,6 +136,8 @@ config-notify = ${request-pull-backup-server:connection-notification-url} ...@@ -134,6 +136,8 @@ config-notify = ${request-pull-backup-server:connection-notification-url}
config-notification-id = ${slap-connection:computer-id}-${slap-connection:partition-id}-apache-pull config-notification-id = ${slap-connection:computer-id}-${slap-connection:partition-id}-apache-pull
config-frequency = 30 * * * * config-frequency = 30 * * * *
slave = true slave = true
sla = instance_guid
sla-instance_guid = ${request-pull-backup-server:instance_guid}
[request-pull-backup-server-apache-backup-1] [request-pull-backup-server-apache-backup-1]
...@@ -146,6 +150,8 @@ config-type = push ...@@ -146,6 +150,8 @@ config-type = push
config-server-key = ${request-apache-backup-1:connection-ssh-public-key} config-server-key = ${request-apache-backup-1:connection-ssh-public-key}
config-on-notification = ${request-pull-backup-server:connection-feeds-url}${request-pull-backup-server-apache-1:config-notification-id} config-on-notification = ${request-pull-backup-server:connection-feeds-url}${request-pull-backup-server-apache-1:config-notification-id}
slave = true slave = true
sla = instance_guid
sla-instance_guid = ${request-pull-backup-server:instance_guid}
[request-pull-backup-server-apache-backup-2] [request-pull-backup-server-apache-backup-2]
<= request-pbs-common <= request-pbs-common
...@@ -157,8 +163,10 @@ config-type = push ...@@ -157,8 +163,10 @@ config-type = push
config-server-key = ${request-apache-backup-2:connection-ssh-public-key} config-server-key = ${request-apache-backup-2:connection-ssh-public-key}
config-on-notification = ${request-pull-backup-server:connection-feeds-url}${request-pull-backup-server-apache-2:config-notification-id} config-on-notification = ${request-pull-backup-server:connection-feeds-url}${request-pull-backup-server-apache-2:config-notification-id}
slave = true slave = true
sla = instance_guid
sla-instance_guid = ${request-pull-backup-server:instance_guid}
[directory] [directory]
ssh = ${rootdirectory:etc}/ssh/ ssh = ${rootdirectory:etc}/ssh
sshkeys = ${rootdirectory:srv}/sshkeys sshkeys = ${rootdirectory:srv}/sshkeys
stack/resilient/buildout.cfg
View file @ d6a9aaa4
[buildout] [buildout]
parts = parts =
#Templates needed to setup automatic backup pbsready
template-pbsready pbsready-import
template-pbsready-import pbsready-export
template-pbsready-export
template-replicated template-replicated
template-parts template-parts
instance-frozen
#Frozen is the state used to not destroy a broken instance's content
template-frozen
template-resilient template-resilient
template-switchsoftware
[template-pbsready] #----------------
#--
#-- Profiles needed to setup automated backup and recovery.
#--
[pbsready]
# Common parts for pbsready-import and pbsready-export.
# Provides rdiff-backup, notification queue, ssh authentication,
# dropbear server, and the bully script.
recipe = slapos.recipe.template recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance-pbsready.cfg url = ${:_profile_base_location_}/pbsready.cfg.in
output = ${buildout:directory}/template-pbsready.cfg output = ${buildout:directory}/pbsready.cfg
md5sum = 45e64cfb6afbcfda1f9f85e33c73bd99 md5sum = b6102416d000cae81dd2b06268946ea9
mode = 0644 mode = 0644
[template-pbsready-import] [pbsready-import]
# An import instance has an importer script, which is called
# by the parent PBS instance when the dump content is propagated.
recipe = slapos.recipe.template recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance-pbsready-import.cfg url = ${:_profile_base_location_}/pbsready-import.cfg.in
output = ${buildout:directory}/template-pbsready-import.cfg output = ${buildout:directory}/pbsready-import.cfg
md5sum = 5ba7477f9499a7dbde5f33ca96bd6ba4 md5sum = eda0c1574d8991f4f9e08e3707c2b04b
mode = 0644 mode = 0644
[template-pbsready-export] [pbsready-export]
# An export instance has an exporter script, and communicates
# to parent PBS instances to deliver the exported dump.
recipe = slapos.recipe.template recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance-pbsready-export.cfg url = ${:_profile_base_location_}/pbsready-export.cfg.in
output = ${buildout:directory}/template-pbsready-export.cfg output = ${buildout:directory}/pbsready-export.cfg
md5sum = 29d36aac2008b173cb9ce5da9e88c0fa md5sum = dd56f9c74e580475a17a9afb1d220390
mode = 0644 mode = 0644
[template-pull-backup] [template-pull-backup]
recipe = slapos.recipe.template recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance-pull-backup.cfg url = ${:_profile_base_location_}/instance-pull-backup.cfg.in
output = ${buildout:directory}/template-pull-backup.cfg output = ${buildout:directory}/instance-pull-backup.cfg
md5sum = f88cc9192a63c88f83a9e5191075534e md5sum = 18b88cd012e886fbaa457b03928c2d10
mode = 0644 mode = 0644
[template-replicated] [template-replicated]
recipe = slapos.recipe.download recipe = slapos.recipe.download
url = ${:_profile_base_location_}/template-replicated.cfg url = ${:_profile_base_location_}/template-replicated.cfg.in
md5sum = 61780842ccda1600a102456b0da69ac9 md5sum = 1017d919dbf41904f04f5c17dcb574fa
mode = 0644 mode = 0644
destination = ${buildout:directory}/template-replicated.cfg destination = ${buildout:directory}/template-replicated.cfg.in
[template-parts] [template-parts]
recipe = slapos.recipe.download recipe = slapos.recipe.download
url = ${:_profile_base_location_}/template-parts.cfg url = ${:_profile_base_location_}/template-parts.cfg.in
md5sum = f5fc27235725f05fdbde76a78ebc363e md5sum = c942f82552fcb42fc74a5f896e0cd5f3
mode = 0644 mode = 0644
destination = ${buildout:directory}/template-parts.cfg destination = ${buildout:directory}/template-parts.cfg.in
[template-frozen] [instance-frozen]
# When an instance is detected as broken, its software type is changed to "frozen".
# On the next run of slapgrid-cp, the buildout profile is replaced by instance-frozen.cfg,
# which will run without removing any content because it raises an error.
recipe = slapos.recipe.template recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance-frozen.cfg url = ${:_profile_base_location_}/instance-frozen.cfg.in
output = ${buildout:directory}/template-frozen.cfg output = ${buildout:directory}/instance-frozen.cfg
[template-resilient] [template-resilient]
recipe = slapos.recipe.template recipe = slapos.recipe.template
url = ${:_profile_base_location_}/resilient.cfg url = ${:_profile_base_location_}/resilient.cfg.in
output = ${buildout:directory}/resilient.cfg output = ${buildout:directory}/resilient.cfg
md5sum = 59e74d290d623de2c1e147e48f284fba md5sum = 59e74d290d623de2c1e147e48f284fba
mode = 0644 mode = 0644
[template-switchsoftware]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/switchsoftware.cfg
output = ${buildout:directory}/switchsoftware.cfg
md5sum = c94a0ed85fce2e72254ae956dce7e40d
mode = 0644
\ No newline at end of file
stack/resilient/instance-pull-backup.cfg → stack/resilient/instance-pull-backup.cfg.in
View file @ d6a9aaa4
[buildout] [buildout]
parts = parts =
connection-dict connection-dict
pbs pbs
cron logrotate
cron-entry-logrotate cron
logrotate cron-entry-logrotate
sshkeys-authority sshkeys-authority
sshkeys-dropbear sshkeys-dropbear
eggs-directory = ${buildout:eggs-directory} eggs-directory = ${buildout:eggs-directory}
develop-eggs-directory = ${buildout:develop-eggs-directory} develop-eggs-directory = ${buildout:develop-eggs-directory}
offline = true offline = true
[connection-dict]
recipe = slapos.cookbook:publish
ssh-key = $${sshkeys-dropbear:public-key-value}
notification-url = http://[$${notifier:host}]:$${notifier:port}/notify
feeds-url = http://[$${notifier:host}]:$${notifier:port}/get/
#----------------
#--
#-- Creation of all needed directories.
[rootdirectory]
recipe = slapos.cookbook:mkdirectory
etc = $${buildout:directory}/etc
home = $${buildout:directory}/home
srv = $${buildout:directory}/srv
bin = $${buildout:directory}/bin
tmp = $${buildout:directory}/tmp
var = $${buildout:directory}/var
[basedirectory]
recipe = slapos.cookbook:mkdirectory
log = $${rootdirectory:var}/log
services = $${rootdirectory:etc}/run
run = $${rootdirectory:var}/run
backup = $${rootdirectory:srv}/backup
promises = $${rootdirectory:etc}/promise
ssh-home = $${rootdirectory:home}/ssh
notifier = $${rootdirectory:etc}/notifier
[directory]
recipe = slapos.cookbook:mkdirectory
logrotate-entries = $${rootdirectory:etc}/logrotate.d
logrotate-backup = $${basedirectory:backup}/logrotate
cronstamps = $${rootdirectory:etc}/cronstamps
cron-entries = $${rootdirectory:etc}/cron.d
crontabs = $${rootdirectory:etc}/crontabs
cronoutput = $${basedirectory:log}/cron-ouput
pbs-backup = $${basedirectory:backup}/pbs
sshkeys = $${rootdirectory:srv}/sshkeys
pbs-wrappers = $${rootdirectory:bin}/pbs
dot-ssh = $${basedirectory:ssh-home}/.ssh
notifier-feeds = $${basedirectory:notifier}/feeds
notifier-callbacks = $${basedirectory:notifier}/callbacks
#----------------
#--
#-- Set up the equeue and notifier.
## sets up the equeue for the notifier
[equeue] [equeue]
recipe = slapos.cookbook:equeue recipe = slapos.cookbook:equeue
socket = $${basedirectory:run}/equeue.sock socket = $${basedirectory:run}/equeue.sock
...@@ -29,7 +66,8 @@ database = $${rootdirectory:srv}/equeue.db ...@@ -29,7 +66,8 @@ database = $${rootdirectory:srv}/equeue.db
wrapper = $${basedirectory:services}/equeue wrapper = $${basedirectory:services}/equeue
equeue-binary = ${buildout:bin-directory}/equeue equeue-binary = ${buildout:bin-directory}/equeue
# notifier.notify adds the [exporter, notifier] to the execution queue
# notifier.notify.callback sets up a callback
[notifier] [notifier]
recipe = slapos.cookbook:notifier recipe = slapos.cookbook:notifier
feeds = $${directory:notifier-feeds} feeds = $${directory:notifier-feeds}
...@@ -42,40 +80,20 @@ wrapper = $${basedirectory:services}/notifier ...@@ -42,40 +80,20 @@ wrapper = $${basedirectory:services}/notifier
server-binary = ${buildout:bin-directory}/pubsubserver server-binary = ${buildout:bin-directory}/pubsubserver
notifier-binary = ${buildout:bin-directory}/pubsubnotifier notifier-binary = ${buildout:bin-directory}/pubsubnotifier
[logrotate-entry-equeue]
<= logrotate
recipe = slapos.cookbook:logrotate.d
name = equeue
log = $${equeue:log}
frequency = daily
rotate-num = 30
## Dropbear Client to provide ssh
[dropbear-client]
recipe = slapos.cookbook:dropbear.client
dbclient-binary = ${dropbear:location}/bin/dbclient
wrapper = $${rootdirectory:bin}/ssh
home = $${basedirectory:ssh-home}
identity-file = $${basedirectory:ssh-home}/id_rsa
[sshkeys-directory]
recipe = slapos.cookbook:mkdirectory
requests = $${directory:sshkeys}/requests/
keys = $${directory:sshkeys}/keys/
[sshkeys-authority]
recipe = slapos.cookbook:sshkeys_authority
request-directory = $${sshkeys-directory:requests}
keys-directory = $${sshkeys-directory:keys}
keygen-binary = ${dropbear:location}/bin/dropbearkey
wrapper = $${basedirectory:services}/sshkeys_authority
[sshkeys-dropbear] #----------------
<= sshkeys-authority #--
recipe = slapos.cookbook:sshkeys_authority.request #-- The pull-backup-server contains every backup (incremental)
name = pbs #-- to prevent a corrupt dump from destroying everything.
type = rsa
executable = $${dropbear-client:wrapper}
public-key = $${dropbear-client:identity-file}.pub
private-key = $${dropbear-client:identity-file}
wrapper = $${rootdirectory:bin}/do_backup
## The pull-backup-server contains every backup (incremental).
## to prevent a corrupt dump from destroying everything.
[pbs] [pbs]
<= notifier <= notifier
recipe = slapos.cookbook:pbs recipe = slapos.cookbook:pbs
...@@ -91,10 +109,9 @@ notifier-url = http://[$${notifier:host}]:$${notifier:port}/ ...@@ -91,10 +109,9 @@ notifier-url = http://[$${notifier:host}]:$${notifier:port}/
slave-instance-list = $${slap-parameter:slave_instance_list} slave-instance-list = $${slap-parameter:slave_instance_list}
[cron-simplelogger] #----------------
recipe = slapos.cookbook:simplelogger #--
wrapper = $${rootdirectory:bin}/cron_simplelogger #-- Deploy cron.
log = $${basedirectory:log}/crond.log
[cron] [cron]
recipe = slapos.cookbook:cron recipe = slapos.cookbook:cron
...@@ -105,6 +122,16 @@ cronstamps = $${directory:cronstamps} ...@@ -105,6 +122,16 @@ cronstamps = $${directory:cronstamps}
catcher = $${cron-simplelogger:wrapper} catcher = $${cron-simplelogger:wrapper}
binary = $${basedirectory:services}/crond binary = $${basedirectory:services}/crond
[cron-simplelogger]
recipe = slapos.cookbook:simplelogger
wrapper = $${rootdirectory:bin}/cron_simplelogger
log = $${basedirectory:log}/crond.log
#----------------
#--
#-- Deploy logrotate.
[cron-entry-logrotate] [cron-entry-logrotate]
<= cron <= cron
recipe = slapos.cookbook:cron.d recipe = slapos.cookbook:cron.d
...@@ -115,7 +142,7 @@ command = $${logrotate:wrapper} ...@@ -115,7 +142,7 @@ command = $${logrotate:wrapper}
[logrotate] [logrotate]
recipe = slapos.cookbook:logrotate recipe = slapos.cookbook:logrotate
# Binaries # Binaries
logrotate-binary = ${logrotate:location}/sbin/logrotate logrotate-binary = ${logrotate:location}/usr/sbin/logrotate
gzip-binary = ${gzip:location}/bin/gzip gzip-binary = ${gzip:location}/bin/gzip
gunzip-binary = ${gzip:location}/bin/gunzip gunzip-binary = ${gzip:location}/bin/gunzip
# Directories # Directories
...@@ -125,15 +152,7 @@ logrotate-entries = $${directory:logrotate-entries} ...@@ -125,15 +152,7 @@ logrotate-entries = $${directory:logrotate-entries}
backup = $${directory:logrotate-backup} backup = $${directory:logrotate-backup}
state-file = $${rootdirectory:srv}/logrotate.status state-file = $${rootdirectory:srv}/logrotate.status
[logrotate-entry-equeue] [logrotate-entry-cron]
<= logrotate
recipe = slapos.cookbook:logrotate.d
name = equeue
log = $${equeue:log}
frequency = daily
rotate-num = 30
[logrotate-entry-equeue]
<= logrotate <= logrotate
recipe = slapos.cookbook:logrotate.d recipe = slapos.cookbook:logrotate.d
name = cron name = cron
...@@ -142,45 +161,61 @@ frequency = daily ...@@ -142,45 +161,61 @@ frequency = daily
rotate-num = 30 rotate-num = 30
[rootdirectory] #----------------
recipe = slapos.cookbook:mkdirectory #--
etc = $${buildout:directory}/etc/ #-- sshkeys
var = $${buildout:directory}/var/
srv = $${buildout:directory}/srv/
bin = $${buildout:directory}/bin/
tmp = $${buildout:directory}/tmp/
[basedirectory] [sshkeys-directory]
recipe = slapos.cookbook:mkdirectory recipe = slapos.cookbook:mkdirectory
log = $${rootdirectory:var}/log/ requests = $${directory:sshkeys}/requests
services = $${rootdirectory:etc}/run/ keys = $${directory:sshkeys}/keys
run = $${rootdirectory:var}/run/
backup = $${rootdirectory:srv}/backup/
promises = $${rootdirectory:etc}/promise/
[directory] [sshkeys-authority]
recipe = slapos.cookbook:mkdirectory recipe = slapos.cookbook:sshkeys_authority
cronstamps = $${rootdirectory:etc}/cronstamps/ request-directory = $${sshkeys-directory:requests}
cron-entries = $${rootdirectory:etc}/cron.d/ keys-directory = $${sshkeys-directory:keys}
crontabs = $${rootdirectory:etc}/crontabs/ wrapper = $${basedirectory:services}/sshkeys_authority
cronoutput = $${basedirectory:log}/cron-ouput/ keygen-binary = ${dropbear:location}/bin/dropbearkey
pbs-backup = $${basedirectory:backup}/pbs/
logrotate-entries = $${rootdirectory:etc}/logrotate.d/
logrotate-backup = $${basedirectory:backup}/logrotate/
sshkeys = $${rootdirectory:srv}/sshkeys
pbs-wrappers = $${rootdirectory:bin}/pbs/
dot-ssh = $${basedirectory:ssh-home}/.ssh/
notifier-feeds = $${basedirectory:notifier}/feeds/
notifier-callbacks = $${basedirectory:notifier}/callbacks/
[basedirectory] [sshkeys-dropbear]
ssh-home = $${rootdirectory:home}/ssh <= sshkeys-authority
notifier = $${rootdirectory:etc}/notifier/ recipe = slapos.cookbook:sshkeys_authority.request
name = pbs
type = rsa
executable = $${dropbear-client:wrapper}
public-key = $${dropbear-client:identity-file}.pub
private-key = $${dropbear-client:identity-file}
wrapper = $${rootdirectory:bin}/do_backup
[rootdirectory]
home = $${buildout:directory}/home/
#----------------
#--
#-- Dropbear.
[dropbear-client]
recipe = slapos.cookbook:dropbear.client
dbclient-binary = ${dropbear:location}/bin/dbclient
wrapper = $${rootdirectory:bin}/ssh
home = $${basedirectory:ssh-home}
identity-file = $${basedirectory:ssh-home}/id_rsa
#----------------
#--
#-- Slave instance list (empty default).
# Default values
[slap-parameter] [slap-parameter]
slave_instance_list = [] slave_instance_list = []
#----------------
#--
#-- Publish instance parameters.
[connection-dict]
recipe = slapos.cookbook:publish
ssh-key = $${sshkeys-dropbear:public-key-value}
notification-url = http://[$${notifier:host}]:$${notifier:port}/notify
feeds-url = http://[$${notifier:host}]:$${notifier:port}/get/
stack/resilient/instance-pbsready-export.cfg → stack/resilient/pbsready-export.cfg.in
View file @ d6a9aaa4
[buildout] [buildout]
extends = ${template-pbsready:output}
extends = ${pbsready:output}
parts += cron-entry-backup parts += cron-entry-backup
[urls] [urls]
notification-id = http://[$${notifier:host}]:$${notifier:port}/get/$${notifier-exporter:name} notification-id = http://[$${notifier:host}]:$${notifier:port}/get/$${notifier-exporter:name}
#notify launches executable, and once it's done, notifies the pull-backup-servers.
[notifier-exporter] [notifier-exporter]
# notifier.notify launches an (exporter) executable, and when finished,
# notifies the the pull-backup-servers.
<= notifier <= notifier
recipe = slapos.cookbook:notifier.notify recipe = slapos.cookbook:notifier.notify
name = exporter name = exporter
...@@ -16,8 +18,9 @@ executable = $${exporter:wrapper} ...@@ -16,8 +18,9 @@ executable = $${exporter:wrapper}
wrapper = $${rootdirectory:bin}/exporter wrapper = $${rootdirectory:bin}/exporter
notify = $${slap-parameter:notify} notify = $${slap-parameter:notify}
#adds the exporter to cron
[cron-entry-backup] [cron-entry-backup]
# Schedule the periodic database dump.
# Through notifications, this triggers (one or more) incremental backups on PBS instances.
<= cron <= cron
recipe = slapos.cookbook:cron.d recipe = slapos.cookbook:cron.d
name = backup name = backup
......
stack/resilient/instance-pbsready-import.cfg → stack/resilient/pbsready-import.cfg.in
View file @ d6a9aaa4
[buildout] [buildout]
extends = ${template-pbsready:output} extends = ${pbsready:output}
parts += import-on-notification parts += import-on-notification
[urls] [urls]
notification-url = http://[$${notifier:host}]:$${notifier:port}/notify notification-url = http://[$${notifier:host}]:$${notifier:port}/notify
#Launches callback, when a notification is received
[import-on-notification] [import-on-notification]
# notifier.callback runs a script when a notification (sent by a parent PBS)
# is received
<= notifier <= notifier
recipe = slapos.cookbook:notifier.callback recipe = slapos.cookbook:notifier.callback
on-notification-id = $${slap-parameter:on-notification} on-notification-id = $${slap-parameter:on-notification}
......
stack/resilient/instance-pbsready.cfg → stack/resilient/pbsready.cfg.in
View file @ d6a9aaa4
...@@ -18,21 +18,97 @@ parts = ...@@ -18,21 +18,97 @@ parts =
dropbear-server-pbs-authorized-key dropbear-server-pbs-authorized-key
notifier notifier
# adds the resiliency script for the bully algorithm
#----------------
#--
#-- Creation of all needed directories.
[basedirectory]
services = $${rootdirectory:etc}/run
cache = $${rootdirectory:var}/cache
notifier = $${rootdirectory:etc}/notifier
[directory]
backup = $${basedirectory:backup}/$${slap-parameter:namebase}
ssh = $${rootdirectory:etc}/ssh/
sshkeys = $${rootdirectory:srv}/sshkeys
notifier-feeds = $${basedirectory:notifier}/feeds
notifier-callbacks = $${basedirectory:notifier}/callbacks
#----------------
#--
#-- Resiliency script for the bully algorithm
[resiliency] [resiliency]
# If enable-bully-service is true, the scripts will be run automatically.
# If false, they can be run with bin/bullly for all the PBSReady instances.
enable-bully-service = False
recipe = slapos.cookbook:addresiliency recipe = slapos.cookbook:addresiliency
script = $${basedirectory:script} services = $${basedirectory:services}
run = $${basedirectory:services} bin = $${rootdirectory:bin}
etc = $${rootdirectory:etc}
#----------------
#--
#-- Sets up an rdiff-backup server (with a dropbear server for ssh)
# sets up an rdiff-backup server (with a dropbear server for ssh)
[urls] [urls]
ssh-public-key = $${sshkeys-dropbear:public-key-value} ssh-public-key = $${sshkeys-dropbear:public-key-value}
ssh-url = ssh://nobody@[$${dropbear-server:host}]:$${dropbear-server:port}/$${rdiff-backup-server:path} ssh-url = ssh://nobody@[$${dropbear-server:host}]:$${dropbear-server:port}/$${rdiff-backup-server:path}
[rdiff-backup-server]
recipe = slapos.cookbook:pbs
client = false
path = $${directory:backup}
wrapper = $${rootdirectory:bin}/rdiffbackup-server
rdiffbackup-binary = ${buildout:bin-directory}/rdiff-backup
#----------------
#--
#-- Set up the equeue and notifier.
[equeue]
recipe = slapos.cookbook:equeue
socket = $${basedirectory:run}/equeue.sock
log = $${basedirectory:log}/equeue.log
database = $${rootdirectory:srv}/equeue.db
wrapper = $${basedirectory:services}/equeue
equeue-binary = ${buildout:bin-directory}/equeue
# notifier.notify adds the [exporter, notifier] to the execution queue
# notifier.notify.callback sets up a callback
[notifier]
recipe = slapos.cookbook:notifier
feeds = $${directory:notifier-feeds}
callbacks = $${directory:notifier-callbacks}
id-file = $${rootdirectory:etc}/notifier.id
equeue-socket = $${equeue:socket}
host = $${slap-network-information:global-ipv6}
port = 8080
wrapper = $${basedirectory:services}/notifier
server-binary = ${buildout:bin-directory}/pubsubserver
notifier-binary = ${buildout:bin-directory}/pubsubnotifier
[logrotate-entry-equeue]
<= logrotate
recipe = slapos.cookbook:logrotate.d
name = equeue
log = $${equeue:log}
frequency = daily
rotate-num = 30
#----------------
#--
#-- sshkeys
[sshkeys-directory] [sshkeys-directory]
recipe = slapos.cookbook:mkdirectory recipe = slapos.cookbook:mkdirectory
requests = $${directory:sshkeys}/requests/ requests = $${directory:sshkeys}/requests
keys = $${directory:sshkeys}/keys/ keys = $${directory:sshkeys}/keys
[sshkeys-authority] [sshkeys-authority]
recipe = slapos.cookbook:sshkeys_authority recipe = slapos.cookbook:sshkeys_authority
...@@ -51,6 +127,11 @@ public-key = $${dropbear-server:rsa-keyfile}.pub ...@@ -51,6 +127,11 @@ public-key = $${dropbear-server:rsa-keyfile}.pub
private-key = $${dropbear-server:rsa-keyfile} private-key = $${dropbear-server:rsa-keyfile}
wrapper = $${basedirectory:services}/sshd wrapper = $${basedirectory:services}/sshd
#----------------
#--
#-- Dropbear.
[dropbear-server] [dropbear-server]
recipe = slapos.cookbook:dropbear recipe = slapos.cookbook:dropbear
host = $${slap-network-information:global-ipv6} host = $${slap-network-information:global-ipv6}
...@@ -66,57 +147,5 @@ dropbear-binary = ${dropbear:location}/sbin/dropbear ...@@ -66,57 +147,5 @@ dropbear-binary = ${dropbear:location}/sbin/dropbear
recipe = slapos.cookbook:dropbear.add_authorized_key recipe = slapos.cookbook:dropbear.add_authorized_key
key = $${slap-parameter:authorized-key} key = $${slap-parameter:authorized-key}
[rdiff-backup-server]
recipe = slapos.cookbook:pbs
client = false
path = $${directory:backup}
wrapper = $${rootdirectory:bin}/rdiffbackup-server
rdiffbackup-binary = ${buildout:bin-directory}/rdiff-backup
## Sets up the execution queue for the notifier
[logrotate-entry-equeue]
<= logrotate
recipe = slapos.cookbook:logrotate.d
name = equeue
log = $${equeue:log}
frequency = daily
rotate-num = 30
[equeue]
recipe = slapos.cookbook:equeue
socket = $${basedirectory:run}/equeue.sock
log = $${basedirectory:log}/equeue.log
database = $${rootdirectory:srv}/equeue.db
wrapper = $${basedirectory:services}/equeue
equeue-binary = ${buildout:bin-directory}/equeue
## notifier.notify adds the [exporter, notifier] to the execution queue
## notifier.notify.callback sets up a callback
[notifier]
recipe = slapos.cookbook:notifier
feeds = $${directory:notifier-feeds}
callbacks = $${directory:notifier-callbacks}
id-file = $${rootdirectory:etc}/notifier.id
equeue-socket = $${equeue:socket}
host = $${slap-network-information:global-ipv6}
port = 8080
wrapper = $${basedirectory:services}/notifier
server-binary = ${buildout:bin-directory}/pubsubserver
notifier-binary = ${buildout:bin-directory}/pubsubnotifier
[basedirectory]
script = $${rootdirectory:etc}/script/
services = $${rootdirectory:etc}/run/
cache = $${rootdirectory:var}/cache/
notifier = $${rootdirectory:etc}/notifier/
[directory]
backup = $${basedirectory:backup}/$${slap-parameter:namebase}
ssh = $${rootdirectory:etc}/ssh/
sshkeys = $${rootdirectory:srv}/sshkeys
notifier-feeds = $${basedirectory:notifier}/feeds/
notifier-callbacks = $${basedirectory:notifier}/callbacks/
script = $${basedirectory:script}
stack/resilient/switchsoftware.cfg deleted 100644 → 0
View file @ 1de09c67
[buildout]
parts =
switch_softwaretype
[switch_softwaretype]
recipe = slapos.cookbook:softwaretype
pull-backup = ${template-pull-backup:output}
#frozen creates a syntax error, meaning it can keep its data.
#It's dirty as hell, it needs to be replaced.
frozen = ${template-frozen:output}
stack/resilient/template-parts.cfg → stack/resilient/template-parts.cfg.in
View file @ d6a9aaa4
...@@ -18,4 +18,4 @@ ...@@ -18,4 +18,4 @@
{% endmacro %} {% endmacro %}
\ No newline at end of file
stack/resilient/template-replicated.cfg → stack/resilient/template-replicated.cfg.in
View file @ d6a9aaa4
...@@ -16,13 +16,10 @@ name = {{namebase}}0 ...@@ -16,13 +16,10 @@ name = {{namebase}}0
return = url ssh-public-key ssh-url notification-id ip return = url ssh-public-key ssh-url notification-id ip
config = number script wrapper authorized-key notify ip-list namebase config = number script wrapper authorized-key notify ip-list namebase
config-number = 0
config-authorized-key = {% for id in range(1,nbbackup|int) %} ${request-pbs-{{namebase}}-{{id}}:connection-ssh-key}{% endfor %} config-authorized-key = {% for id in range(1,nbbackup|int) %} ${request-pbs-{{namebase}}-{{id}}:connection-ssh-key}{% endfor %}
config-notify = {% for id in range(1,nbbackup|int) %} ${request-pbs-{{namebase}}-{{id}}:connection-notification-url}{% endfor %} config-notify = {% for id in range(1,nbbackup|int) %} ${request-pbs-{{namebase}}-{{id}}:connection-notification-url}{% endfor %}
config-ip-list = config-ip-list =
config-number = 0
{% for id in range(1,nbbackup|int) %} {% for id in range(1,nbbackup|int) %}
...@@ -37,14 +34,13 @@ software-url = ${slap-connection:software-release-url} ...@@ -37,14 +34,13 @@ software-url = ${slap-connection:software-release-url}
software-type = {{typeimport}} software-type = {{typeimport}}
return = url ssh-public-key ssh-url notification-url ip return = url ssh-public-key ssh-url notification-url ip
config = number script wrapper authorized-key on-notification ip-list namebase pbs-notification-id = ${slap-connection:computer-id}-${slap-connection:partition-id}-{{namebase}}-push
config = number script wrapper authorized-key on-notification ip-list namebase
config-number = {{id}}
config-authorized-key = ${request-pbs-{{namebase}}-{{id}}:connection-ssh-key} config-authorized-key = ${request-pbs-{{namebase}}-{{id}}:connection-ssh-key}
config-on-notification = ${request-pbs-{{namebase}}-{{id}}:connection-feeds-url}${:pbs-notification-id} config-on-notification = ${request-pbs-{{namebase}}-{{id}}:connection-feeds-url}${:pbs-notification-id}
pbs-notification-id = ${slap-connection:computer-id}-${slap-connection:partition-id}-{{namebase}}-push
config-ip-list = config-ip-list =
config-number = {{id}}
{% endfor %} {% endfor %}
...@@ -60,19 +56,15 @@ config-ip-list = ${request-{{namebase}}:connection-ip}{% for j in range(1,nbback ...@@ -60,19 +56,15 @@ config-ip-list = ${request-{{namebase}}:connection-ip}{% for j in range(1,nbback
recipe = slapos.cookbook:request recipe = slapos.cookbook:request
name = {{namebase}}0 name = {{namebase}}0
config = number script wrapper authorized-key notify ip-list namebase
software-url = ${slap-connection:software-release-url} software-url = ${slap-connection:software-release-url}
software-type = {{typeexport}} software-type = {{typeexport}}
return = url ssh-public-key ssh-url notification-id ip return = url ssh-public-key ssh-url notification-id ip
config = number script wrapper authorized-key notify ip-list namebase
config-number = 0
config-authorized-key = {% for id in range(1,nbbackup|int) %} ${request-pbs-{{namebase}}-{{id}}:connection-ssh-key}{% endfor %} config-authorized-key = {% for id in range(1,nbbackup|int) %} ${request-pbs-{{namebase}}-{{id}}:connection-ssh-key}{% endfor %}
config-notify = {% for id in range(1,nbbackup|int) %} ${request-pbs-{{namebase}}-{{id}}:connection-notification-url}{% endfor %} config-notify = {% for id in range(1,nbbackup|int) %} ${request-pbs-{{namebase}}-{{id}}:connection-notification-url}{% endfor %}
config-number=0
{% for id in range(1,nbbackup|int) %} {% for id in range(1,nbbackup|int) %}
[request-{{namebase}}-pseudo-replicating-{{id}}-2] [request-{{namebase}}-pseudo-replicating-{{id}}-2]
<= slap-connection <= slap-connection
...@@ -87,14 +79,13 @@ software-url = ${slap-connection:software-release-url} ...@@ -87,14 +79,13 @@ software-url = ${slap-connection:software-release-url}
software-type = {{typeimport}} software-type = {{typeimport}}
return = url ssh-public-key ssh-url notification-url return = url ssh-public-key ssh-url notification-url
config = number script wrapper authorized-key on-notification ip-list namebase pbs-notification-id = ${slap-connection:computer-id}-${slap-connection:partition-id}-{{namebase}}-push
config = number script wrapper authorized-key on-notification ip-list namebase
config-number = {{id}}
config-authorized-key = ${request-pbs-{{namebase}}-{{id}}:connection-ssh-key} config-authorized-key = ${request-pbs-{{namebase}}-{{id}}:connection-ssh-key}
config-on-notification = ${request-pbs-{{namebase}}-{{id}}:connection-feeds-url}${:pbs-notification-id} config-on-notification = ${request-pbs-{{namebase}}-{{id}}:connection-feeds-url}${:pbs-notification-id}
pbs-notification-id = ${slap-connection:computer-id}-${slap-connection:partition-id}-{{namebase}}-push
config-number = {{id}}
{% endfor %} {% endfor %}
...@@ -135,6 +126,8 @@ config-notify = ${request-pbs-{{namebase}}-{{id}}:connection-notification-url} ...@@ -135,6 +126,8 @@ config-notify = ${request-pbs-{{namebase}}-{{id}}:connection-notification-url}
config-notification-id = ${slap-connection:computer-id}-${slap-connection:partition-id}-{{namebase}}-{{id}}-pull config-notification-id = ${slap-connection:computer-id}-${slap-connection:partition-id}-{{namebase}}-{{id}}-pull
config-title = Pulling from {{namebase}} config-title = Pulling from {{namebase}}
slave = true slave = true
sla = instance_guid
sla-instance_guid = ${request-pbs-{{namebase}}-{{id}}:instance_guid}
[request-pull-backup-server-{{namebase}}-backup-{{id}}] [request-pull-backup-server-{{namebase}}-backup-{{id}}]
<= request-pbs-common <= request-pbs-common
...@@ -149,6 +142,8 @@ config-notify = ${request-{{namebase}}-pseudo-replicating-{{id}}:connection-noti ...@@ -149,6 +142,8 @@ config-notify = ${request-{{namebase}}-pseudo-replicating-{{id}}:connection-noti
config-notification-id = ${request-{{namebase}}-pseudo-replicating-{{id}}:pbs-notification-id} config-notification-id = ${request-{{namebase}}-pseudo-replicating-{{id}}:pbs-notification-id}
config-title = Pushing to {{namebase}} backup {{id}} config-title = Pushing to {{namebase}} backup {{id}}
slave = true slave = true
sla = instance_guid
sla-instance_guid = ${request-pbs-{{namebase}}-{{id}}:instance_guid}
{% endfor %} {% endfor %}
{% endmacro %} {% endmacro %}
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7