Commit f1c3a7ff authored by Romain Courteaud's avatar Romain Courteaud

Provide RSS feeds about the backup status.

parent a07dffad
......@@ -12,6 +12,7 @@ etc = $${buildout:directory}/etc
bin = $${buildout:directory}/bin
srv = $${buildout:directory}/srv
var = $${buildout:directory}/var
run = $${:var}/run
log = $${:var}/log
# scripts = $${:etc}/run
services = $${:etc}/service
......@@ -20,8 +21,11 @@ cron-lines = $${:etc}/cron.lines
crontabs = $${:etc}/crontabs
cronstamps = $${:etc}/cronstamps
backup = $${:srv}/backup
status = $${:srv}/status
backupscript = $${:etc}/backup
www = $${:srv}/www
home = $${:etc}/home
ssl = $${:etc}/ssl
ssh = $${:home}/.ssh
#################################
......@@ -91,6 +95,7 @@ storage-path = {{ '$${' ~ slave_reference }}-backup-public_key:key}
recipe = slapos.cookbook:publish
-slave-reference = {{ slave_reference }}
authorized_key = {{ '$${' ~ slave_reference }}-backup-read-public_key:readline}
rss = https://[$${nginx-configuration:ip}]:$${nginx-configuration:port}/{{ '$${' ~ slave_reference }}-backup-script:status_name}.rss
{% do part_list.append("%s-backup-publish" % slave_reference) -%}
[{{ slave_reference }}-backup-script]
......@@ -105,6 +110,8 @@ hostname = {{ hostname }}
include = {{ include_string }}
exclude_string = {{ exclude_string }}
remote_schema = {{ remote_schema }}
status_name = {{ slave_reference }}_status.txt
status_log = $${directory:status}/$${:status_name}
[{{ slave_reference }}-backup-crontab-line]
recipe = slapos.recipe.template
......@@ -121,6 +128,25 @@ frequency = {{ frequency }}
#################################
# Generate crontab file
#################################
[update-rss-script]
recipe = slapos.recipe.template
url = ${template-update-rss-script:output}
output = $${directory:etc}/$${:_buildout_section_name_}
mode = 0700
global_rss = $${slap-connection:computer-id}-$${slap-connection:partition-id}.rss
[update-rss-crontab-line]
recipe = slapos.recipe.template
url = ${template-crontab-line:output}
output = $${directory:cron-lines}/$${:_buildout_section_name_}
mode = 0600
script = $${update-rss-script:output}
frequency = */5 * * * *
{% do crontab_line_list.append("$${update-rss-crontab-line:output}") -%}
[publish-global-rss]
recipe = slapos.cookbook:publish
rss = https://[$${nginx-configuration:ip}]:$${nginx-configuration:port}/$${update-rss-script:global_rss}
{% set crontab_line_list_string = " ".join(crontab_line_list) -%}
[activate-crontab-file]
......@@ -129,12 +155,37 @@ recipe = plone.recipe.command
stop-on-error = true
command = ${coreutils-output:cat} ${template-crontab:output} {{ crontab_line_list_string }} | ${dcron-output:crontab} -c $${directory:crontabs} -
#################################
# Nginx service
#################################
[nginx-service]
recipe = slapos.recipe.template
url = ${template-nginx-service:output}
output = $${directory:services}/nginx
mode = 0700
virtual-depends =
$${nginx-configuration:ip}
[nginx-configuration]
recipe = slapos.recipe.template
url = ${template-nginx-configuration:output}
output = $${directory:etc}/nginx.cfg
mode = 0600
access_log = $${directory:log}/nginx-access.log
error_log = $${directory:log}/nginx-error.log
ip = $${slap-network-information:global-ipv6}
port = 9443
ssl_key = $${directory:ssl}/nginx.key
ssl_csr = $${directory:ssl}/nginx.csr
ssl_crt = $${directory:ssl}/nginx.crt
# Add parts generated by template
[buildout]
parts =
dcron-service
nginx-service
activate-crontab-file
publish-global-rss
{% for part in part_list -%}
{{ ' %s' % part }}
{% endfor -%}
......
......@@ -3,7 +3,8 @@ extends =
../../component/dash/buildout.cfg
../../component/dcron/buildout.cfg
../../component/logrotate/buildout.cfg
# ../../component/nginx/buildout.cfg
../../component/openssl/buildout.cfg
../../component/nginx/buildout.cfg
../../component/rdiff-backup/buildout.cfg
# ../../component/duplicity/buildout.cfg
# ../../component/git/buildout.cfg
......@@ -15,11 +16,13 @@ extends =
../../stack/slapos.cfg
parts =
extra-eggs
rdiff-backup
# duplicity
dcron
logrotate
# nginx
nginx
openssl
# git
# subversion
rsync
......@@ -30,6 +33,12 @@ parts =
template-crontab-line
slapos-cookbook
[extra-eggs]
recipe = zc.recipe.egg
interpreter = pythonforrssgen
eggs =
PyRSS2Gen
[networkcache]
# signature certificates of the following uploaders.
# Romain Courteaud
......@@ -163,6 +172,20 @@ signature-certificate-list =
##########################################################
# Service startup scripts and configuration files
##########################################################
[template-nginx-service]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/template-nginx-service.sh.in
md5sum = 5c94d952305552dcbeaeaeb27dd28f3b
output = ${buildout:directory}/template-nginx-service.sh.in
mode = 0644
[template-nginx-configuration]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/template-nginx.cfg.in
md5sum = 1cf80a510a9ba9f6d9f12e2ce828615e
output = ${buildout:directory}/template-nginx.cfg.in
mode = 0644
[template-dcron-service]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/template-dcron-service.sh.in
......@@ -173,7 +196,7 @@ mode = 0644
[template-backup-script]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/template-backup-script.sh.in
md5sum = 19c3a72913cc285bf9745ab2f869018e
md5sum = ce63dd27aaea8b64f446a7aef9406873
output = ${buildout:directory}/template-backup-script.sh.in
mode = 0644
......@@ -191,13 +214,27 @@ md5sum = 072be0fd04896880c931d44d8eabde37
output = ${buildout:directory}/template-crontab.in
mode = 0644
[status2rss]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/status2rss.py
md5sum = 138c96e0836f2b06414b98ba2643f21c
output = ${buildout:directory}/status2rss.py
mode = 0644
[template-update-rss-script]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/template-update-rss.sh.in
md5sum = 529058c54e873ab26f7920c868b23c50
output = ${buildout:directory}/template-update-rss.sh.in
mode = 0644
##########################################################
# Buildout instance.cfg templates
##########################################################
[template-pullrdiffbackup]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance-pullrdiffbackup.cfg.in
md5sum = 213256229360f57c778308825b161321
md5sum = 935ee60f0454ab50df74d0bcc3364d37
output = ${buildout:directory}/template-pullrdiffbackup.cfg
mode = 0644
......@@ -223,6 +260,7 @@ slapos.recipe.build = 0.11.6
slapos.recipe.cmmi = 0.1.1
slapos.recipe.template = 2.4.2
zc.recipe.egg = 1.3.2
PyRSS2Gen = 1.1
# Required by:
# slapos.core==0.35.1
......
import datetime
import uuid
import PyRSS2Gen
import sys
from email.utils import parsedate_tz, mktime_tz
import base64
# Based on http://thehelpfulhacker.net/2011/03/27/a-rss-feed-for-your-crontabs/
# ### Defaults
TITLE = sys.argv[1]
LINK = sys.argv[2]
DESCRIPTION = TITLE
items = []
while 1:
try:
line = sys.stdin.readline()
except KeyboardInterrupt:
break
if not line:
break
time, desc = line.split(',', 1)
rss_item = PyRSS2Gen.RSSItem(
title = desc,
description = "%s, %s" % (time, desc),
link = LINK,
pubDate = datetime.datetime.fromtimestamp(mktime_tz(parsedate_tz(time))),
guid = PyRSS2Gen.Guid(base64.b64encode("%s, %s" % (time, desc)))
)
items.append(rss_item)
### Build the rss feed
rss_feed = PyRSS2Gen.RSS2 (
title = TITLE,
link = LINK,
description = DESCRIPTION,
lastBuildDate = datetime.datetime.utcnow(),
items = items
)
print rss_feed.to_xml()
......@@ -4,7 +4,13 @@
export HOME=$${directory:home}
set -e
# Clean status file (no history needed)
${coreutils-output:rm} -f $${:status_log}
# Inform about beginning of backup
${coreutils-output:echo} "`${coreutils-output:date} -u`, $${:hostname} backup running" >> $${:status_log}
# set -e
cd $${:datadirectory}
${rdiff-backup-output:rdiff-backup} \
$${:exclude_string} \
......@@ -13,5 +19,15 @@ ${rdiff-backup-output:rdiff-backup} \
--remote-schema '${dropbear-output:ssh} -T -y -i $${:sshkey} %s $${:remote_schema}' \
$${:connection}::/ ./
RESULT=$?
# Inform about backup status
if [ $RESULT == 0 ]
then
${coreutils-output:echo} "`${coreutils-output:date} -u`, $${:hostname} backup success" >> $${:status_log}
else
${coreutils-output:echo} "`${coreutils-output:date} -u`, $${:hostname} backup failed" >> $${:status_log}
fi
# python scripts/verify_with_sudo.py ./ $${:connection}:/
#!${dash-output:dash}
# BEWARE: This file is operated by slapgrid
# BEWARE: It will be overwritten automatically
if [ ! -e $${nginx-configuration:ssl_crt} ]
then
${openssl-output:openssl} genrsa -out $${nginx-configuration:ssl_key} 2048
${openssl-output:openssl} req -new \
-subj "/C=AA/ST=Denial/L=Nowhere/O=Dis/CN=$${nginx-configuration:ip}" \
-key $${nginx-configuration:ssl_key} -out $${nginx-configuration:ssl_csr}
${openssl-output:openssl} x509 -req -days 365 \
-in $${nginx-configuration:ssl_csr} \
-signkey $${nginx-configuration:ssl_key} \
-out $${nginx-configuration:ssl_crt}
fi
exec ${nginx-output:nginx} \
-c $${nginx-configuration:output}
daemon off; # run in the foreground so supervisord can look after it
worker_processes 4;
pid $${directory:run}/nginx.pid;
events {
worker_connections 768;
# multi_accept on;
}
http {
##
# Basic Settings
##
sendfile on;
tcp_nopush on;
tcp_nodelay on;
keepalive_timeout 65;
types_hash_max_size 2048;
server_tokens off;
# server_names_hash_bucket_size 64;
# server_name_in_redirect off;
default_type application/octet-stream;
include ${nginx-output:mime};
##
# Logging Settings
##
access_log $${nginx-configuration:access_log};
error_log $${nginx-configuration:error_log};
##
# Gzip Settings
##
gzip on;
gzip_disable "msie6";
# gzip_vary on;
# gzip_proxied any;
# gzip_comp_level 6;
# gzip_buffers 16 8k;
# gzip_http_version 1.1;
# gzip_types text/plain text/css application/json application/x-javascript text/xml application/xml application/xml+rss text/javascript;
server {
listen [$${nginx-configuration:ip}]:$${nginx-configuration:port};
ssl on;
ssl_certificate $${nginx-configuration:ssl_crt};
ssl_certificate_key $${nginx-configuration:ssl_key};
## Only allow GET and HEAD request methods
if ($request_method !~ ^(GET|HEAD)$ ) {
return 444;
}
## Serve an error 204 (No Content) for favicon.ico
location = /favicon.ico {
return 204;
}
location /
{
root $${directory:www};
# index index.html;
}
}
}
#!${dash-output:dash}
STATUS_DIR=$${directory:status}
RSS_DIR=$${directory:www}
CAT=${coreutils-output:cat}
PYTHON=${buildout:directory}/bin/${extra-eggs:interpreter}
STATUS2RSS=${status2rss:output}
BASENAME=${coreutils-output:basename}
for status in $STATUS_DIR/*
do
NAME=`$BASENAME $status`
$CAT $status | $PYTHON $STATUS2RSS "Backup status $NAME" "https://[$${nginx-configuration:ip}]:$${nginx-configuration:port}/$NAME.rss" > $RSS_DIR/$NAME.rss
done
$CAT $STATUS_DIR/* | $PYTHON $STATUS2RSS "Full backup status $${:global_rss}" "https://[$${nginx-configuration:ip}]:$${nginx-configuration:port}/$${:global_rss}" > $RSS_DIR/$${:global_rss}
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment