Commit 3d1dfe22 authored by Antoine Catton's avatar Antoine Catton

Adapt pbs recipe to new agent style.

parent ec216f2e
...@@ -26,20 +26,30 @@ ...@@ -26,20 +26,30 @@
############################################################################## ##############################################################################
from json import loads as unjson from json import loads as unjson
from hashlib import sha512 from hashlib import sha512
from urlparse import urlparse import urlparse
import os import os
import subprocess import subprocess
import sys import sys
import signal import signal
import inspect import inspect
import uuid
import base64
import urllib
from slapos.recipe.librecipe import GenericSlapRecipe from slapos.recipe.librecipe import GenericSlapRecipe
from slapos.recipe.dropbear import KnownHostsFile from slapos.recipe.dropbear import KnownHostsFile
from slapos.recipe.notifier import Notify
from slapos.recipe.notifier import Callback
from slapos import slap as slapmodule from slapos import slap as slapmodule
def process_backup(config):
from glob import glob
runafter = config['run_after']
pull_script = config['pull']
push_scripts = glob(config['push'])
os.execv(runafter, [runafter] + ['--first-one', pull_script] + push_scripts)
def promise(args): def promise(args):
def failed_ssh(partition, ssh): def failed_ssh(partition, ssh):
...@@ -97,23 +107,17 @@ def promise(args): ...@@ -97,23 +107,17 @@ def promise(args):
return ssh.returncode return ssh.returncode
class Recipe(GenericSlapRecipe):
class Recipe(GenericSlapRecipe, Notify, Callback): def _options(self, options):
if self.optionIsTrue('client', True):
def add_slave(self, entry, known_hosts_file): slaves = unjson(options['slave-instance-list'])
path_list = [] options['peers'] = '\n'.join([str(entry['request-agent-url'])
for entry in slaves])
url = entry.get('url')
if url is None:
url = ''
# We assume that thanks to sha512 there's no collisions
url_hash = sha512(url).hexdigest()
name_hash = sha512(entry['name']).hexdigest()
def add_slave_promise(self, parsed_url, instance_id):
promise_path = os.path.join(self.options['promises-directory'], promise_path = os.path.join(self.options['promises-directory'],
url_hash) instance_id)
parsed_url = urlparse(url)
promise_dict = self.promise_base_dict.copy() promise_dict = self.promise_base_dict.copy()
promise_dict.update(user=parsed_url.username, promise_dict.update(user=parsed_url.username,
host=parsed_url.hostname, host=parsed_url.hostname,
...@@ -121,27 +125,57 @@ class Recipe(GenericSlapRecipe, Notify, Callback): ...@@ -121,27 +125,57 @@ class Recipe(GenericSlapRecipe, Notify, Callback):
promise = self.createPythonScript(promise_path, promise = self.createPythonScript(promise_path,
__name__ + '.promise', __name__ + '.promise',
promise_dict) promise_dict)
path_list.append(promise) self.path_list.append(promise)
def add_slave(self, entry, known_hosts_file):
url = entry.get('url')
if url is None:
url = ''
# Base32 is filename safe
backup_id = base64.b32encode(uuid.UUID(entry['id']).bytes)
instance_id = '%(uuid)s%(type)s%(id)s' % {
'uuid': backup_id,
'type': entry['type'],
'id': entry['slave_reference']
}
parsed_url = urlparse.urlparse(url)
self.add_slave_promise(parsed_url, instance_id)
host = parsed_url.hostname host = parsed_url.hostname
known_hosts_file[host] = entry['server-key'] known_hosts_file[host] = entry['authorized-key']
remote_schema = '%(ssh)s -p %%s %(user)s@%(host)s' % \ remote_schema = '%(ssh)s -p %%s %(user)s@%(host)s' % {
{
'ssh': self.options['sshclient-binary'], 'ssh': self.options['sshclient-binary'],
'user': parsed_url.username, 'user': parsed_url.username,
'host': parsed_url.hostname, 'host': parsed_url.hostname,
} }
command = [self.options['rdiffbackup-binary']] agent_url = self.options['agent-url']
command.extend(['--remote-schema', remote_schema]) feed_url = urlparse.urljoin(self.options['agent-url'],
'log/%s' % urllib.quote(instance_id))
notify_url = urlparse.urljoin(self.options['agent-url'],
'notify')
command = [self.options['notifier-binary'],
'--write', feed_url,
'--title', entry.get('name', 'Untitled'),
]
for notify in entry.get('notify', '').split():
if notify: # Ignore empty values
import pdb; pdb.set_trace()
command.extend(['--notify', notify])
command.append('--')
command.extend([self.options['rdiffbackup-binary'], '--remote-schema', remote_schema])
remote_directory = '%(port)s::%(path)s' % {'port': parsed_url.port, remote_directory = '%(port)s::%(path)s' % {'port': parsed_url.port,
'path': parsed_url.path} 'path': parsed_url.path}
local_directory = self.createDirectory(self.options['directory'], local_directory = self.createDirectory(self.options['directory'],
name_hash) backup_id)
if entry['type'] == 'push': if entry['type'] == 'push':
command.extend(['--restore-as-of', 'now']) command.extend(['--restore-as-of', 'now'])
...@@ -150,48 +184,50 @@ class Recipe(GenericSlapRecipe, Notify, Callback): ...@@ -150,48 +184,50 @@ class Recipe(GenericSlapRecipe, Notify, Callback):
else: else:
command.extend([remote_directory, local_directory]) command.extend([remote_directory, local_directory])
wrapper_basepath = os.path.join(self.options['wrappers-directory'], wrapper_path = os.path.join(self.options['wrappers-directory'],
url_hash) instance_id)
wrapper_path = wrapper_basepath
if 'notify' in entry:
wrapper_path = '%s_raw' % wrapper_basepath
wrapper = self.createPythonScript( self.path_list.append(self.createPythonScript(
wrapper_path, wrapper_path,
'slapos.recipe.librecipe.execute.execute', 'slapos.recipe.librecipe.execute.execute',
[str(i) for i in command] command,
) ))
path_list.append(wrapper)
if entry['type'] == 'pull':
if 'notify' in entry: backup_wrapper = os.path.join(self.options['wrapper-directory'],
feed_url = '%s/get/%s' % (self.options['notifier-url'], backup_id)
entry['notification-id']) push_pattern = os.path.join(self.options['wrapper-directory'],
wrapper = self.createNotifier( '%spush*' % backup_id)
self.options['notifier-binary'], self.path_list.append(self.createPythonScript(
wrapper=wrapper_basepath, backup_wrapper,
executable=wrapper_path, __name__ + '.process_backup',
log=os.path.join(self.options['feeds'], entry['notification-id']), {
title=entry.get('title', 'Untitled'), 'run_after': self.options['runafter-binary'],
notification_url=entry['notify'], 'pull': wrapper_path,
feed_url=feed_url, 'push': push_pattern,
) }
path_list.append(wrapper) ))
#self.setConnectionDict(dict(feed_url=feed_url), entry['slave_reference'])
if 'frequency' in entry:
if 'on-notification' in entry: cron_entry_filename = os.path.join(self.options['cron-entries'],
path_list.append(self.createCallback(str(entry['on-notification']), '%s-%s' % (self.name, backup_id))
wrapper)) self.path_list.append(self.createFile(cron_entry_filename,
else: '%s %s' % (entry['frequency'], backup_wrapper)))
cron_entry = os.path.join(self.options['cron-entries'], url_hash) elif 'trigger-feed' in entry:
with open(cron_entry, 'w') as cron_entry_file: trigger_filename = os.path.join(self.options['agent-callbacks-directory'],
cron_entry_file.write('%s %s' % (entry['frequency'], wrapper)) sha512(entry['trigger-feed']).hexdigest())
path_list.append(cron_entry) with open(trigger_filename, 'w') as trigger_file:
trigger_file.write(backup_wrapper)
return path_list self.path_list.append(trigger_filename)
self.setConnectionDict({
'agent-url': agent_url,
'feed-url': feed_url,
'notify-url': notify_url,
}, entry['slave_reference'])
def _install(self): def _install(self):
path_list = [] self.path_list = []
if self.optionIsTrue('client', True): if self.optionIsTrue('client', True):
...@@ -211,7 +247,7 @@ class Recipe(GenericSlapRecipe, Notify, Callback): ...@@ -211,7 +247,7 @@ class Recipe(GenericSlapRecipe, Notify, Callback):
known_hosts = KnownHostsFile(self.options['known-hosts']) known_hosts = KnownHostsFile(self.options['known-hosts'])
with known_hosts: with known_hosts:
for slave in slaves: for slave in slaves:
path_list.extend(self.add_slave(slave, known_hosts)) self.add_slave(slave, known_hosts)
else: else:
command = [self.options['rdiffbackup-binary']] command = [self.options['rdiffbackup-binary']]
...@@ -223,6 +259,6 @@ class Recipe(GenericSlapRecipe, Notify, Callback): ...@@ -223,6 +259,6 @@ class Recipe(GenericSlapRecipe, Notify, Callback):
self.options['wrapper'], self.options['wrapper'],
'slapos.recipe.librecipe.execute.execute', 'slapos.recipe.librecipe.execute.execute',
command) command)
path_list.append(wrapper) self.path_list.append(wrapper)
return path_list return self.path_list
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment