Commit 6734fca2 authored by Nicolas Wavrant's avatar Nicolas Wavrant

resilient: new tool directory for resiliency.

This first commit adds a script which generate a RSS feed from the statistics from rdiff-backup
parent d3c5fa41
...@@ -92,6 +92,7 @@ setup(name=name, ...@@ -92,6 +92,7 @@ setup(name=name,
'pubsubnotifier = slapos.pubsub.notifier:main', 'pubsubnotifier = slapos.pubsub.notifier:main',
'pubsubserver = slapos.pubsub:main', 'pubsubserver = slapos.pubsub:main',
'qemu-qmp-client = slapos.qemuqmpclient:main', 'qemu-qmp-client = slapos.qemuqmpclient:main',
'rdiffbackup.genstatrss = slapos.resilient.rdiffBackupStat2RSS:main',
'slapos-kill = slapos.systool:kill', 'slapos-kill = slapos.systool:kill',
'slaprunnertest = slapos.runner.runnertest:main', 'slaprunnertest = slapos.runner.runnertest:main',
'slaprunnerteststandalone = slapos.runner.runnertest:runStandaloneUnitTest', 'slaprunnerteststandalone = slapos.runner.runnertest:runStandaloneUnitTest',
......
import argparse
import datetime
import os
import re
import time
import PyRSS2Gen as RSS2
from collections import OrderedDict
def parseArguments():
"""
Parse arguments for rdiff-backup statistics Rss Generator.
"""
parser = argparse.ArgumentParser()
parser.add_argument('--rdiff_backup_data_folder',
help='Path where to find rdiff-backup statistical files')
parser.add_argument('--output',
help='Path where to save the feed')
parser.add_argument('--feed_url',
help='Url of this feed file.')
return parser.parse_args()
def makeDictFromStatFile(text_content):
m = re.search("([a-zA-Z]*) ([0-9 :.]*) \(([a-zA-Z0-9 :.]*)\)", text_content)
if m:
return {'metric': m.group(1),
'value': m.group(2),
'human_readable_value': m.group(3)}
m = re.search("([a-zA-Z]*) ([0-9]*)", text_content)
if m:
return {'metric': m.group(1),
'value': m.group(2),
'human_readable_value': m.group(2)}
def getRSSItemFromDict(item, option):
description = "Metric;Value;Human Readable Value\n"
for entry in item:
description += "%s;%s;%s\n" % (entry['metric'], entry['value'], entry['human_readable_value'])
if entry['metric'] == "EndTime":
pubDate = datetime.datetime.fromtimestamp(float(entry['value']))
return RSS2.RSSItem(
title="Rdiff-Backup Transfer Statistics",
link=option.feed_url,
pubDate=pubDate,
description=description)
def genRSS(option):
"""
Read statistics file from rdiff-backup and generate a RSS feed entry from it
"""
stat_file_list = sorted([file for file in os.listdir(option.rdiff_backup_data_folder)
if file.startswith('session_statistics')])
item_dict = OrderedDict()
for stat_file in stat_file_list:
with open(os.path.join(option.rdiff_backup_data_folder, stat_file), 'r') as file:
item_dict[stat_file] = [makeDictFromStatFile(line.strip()) for line in file.readlines()]
title="Rdiff-Backup Statistics"
rss_feed = RSS2.RSS2(
title=title,
link=option.feed_url,
description=title,
items = [getRSSItemFromDict(item_dict[item], option) for item in item_dict])
return rss_feed.to_xml()
def main():
option = parseArguments()
feed = genRSS(option)
with open(option.output, 'w') as rss_file:
rss_file.write(feed)
exit(0)
import csv
import feedparser
import os
import shutil
import tempfile
import unittest
from slapos.resilient.rdiffBackupStat2RSS import genRSS
class Option(dict):
def __init__(self, **kw):
self.__dict__.update(kw)
def __setitem__(i, y):
self.__dict__[i] = y
class TestRdiffBackupStat2RSS(unittest.TestCase):
def setUp(self):
self.data_directory = tempfile.mkdtemp(dir='.')
self.feed_path = os.path.join(self.data_directory)
def tearDown(self):
shutil.rmtree(self.data_directory)
def getOptionObject(self, **kw):
"""
Returns an object containing options as properties, to simulate a call
to the tested script
"""
option = {
'rdiff_backup_data_folder': self.data_directory,
'output': self.feed_path,
'feed_url': 'http://exemple.com',
}
option.update(kw)
return Option(**option)
def createSample(self):
"""
Writes 2 statistics file in rdiff-backup format
"""
with open(os.path.join(self.data_directory, 'session_statistics_1'), 'w') as stat_file:
stat_file.write("""\
StartTime 1473339659.00 (Thu Sep 8 15:00:59 2016)
EndTime 1473339667.81 (Thu Sep 8 15:01:07 2016)
ElapsedTime 8.81 (8.81 seconds)
SourceFiles 2381
SourceFileSize 142096473 (136 MB)
MirrorFiles 1
MirrorFileSize 0 (0 bytes)
NewFiles 2380
NewFileSize 142096473 (136 MB)
DeletedFiles 0
DeletedFileSize 0 (0 bytes)
ChangedFiles 1
ChangedSourceSize 0 (0 bytes)
ChangedMirrorSize 0 (0 bytes)
IncrementFiles 0
IncrementFileSize 0 (0 bytes)
TotalDestinationSizeChange 142096473 (136 MB)
Errors 0""")
with open(os.path.join(self.data_directory, 'session_statistics_2'), 'w') as stat_file:
stat_file.write("""\
StartTime 1473340154.00 (Thu Sep 8 15:09:14 2016)
EndTime 1473340154.95 (Thu Sep 8 15:09:14 2016)
ElapsedTime 0.95 (0.95 seconds)
SourceFiles 2381
SourceFileSize 142096473 (136 MB)
MirrorFiles 2381
MirrorFileSize 142096473 (136 MB)
NewFiles 0
NewFileSize 0 (0 bytes)
DeletedFiles 0
DeletedFileSize 0 (0 bytes)
ChangedFiles 15
ChangedSourceSize 230112 (225 KB)
ChangedMirrorSize 230112 (225 KB)
IncrementFiles 15
IncrementFileSize 2122 (2.07 KB)
TotalDestinationSizeChange 2122 (2.07 KB)
Errors 0""")
def test_generatedRSSIsCorrect(self):
self.createSample()
option = self.getOptionObject()
feed_content = genRSS(option)
feed = feedparser.parse(feed_content)
self.assertFalse(feed.bozo)
self.assertTrue(len(feed.entries), 2)
self.assertLess(feed.entries[0].published_parsed, feed.entries[1].published_parsed)
if __name__ == '__main__':
unittest.main()
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment