Commit 6c731311 authored by Łukasz Nowak's avatar Łukasz Nowak Committed by Łukasz Nowak

kedifa: Introduce asynchronous updater

Features:

 * by default runs with 60s sleep
 * allows to have master, updateable, certificate, which is used in case if
   specific certificate is not available
parent 24ac19af
...@@ -23,6 +23,7 @@ import requests ...@@ -23,6 +23,7 @@ import requests
import sys import sys
import app import app
from updater import Updater
def http(*args): def http(*args):
...@@ -123,3 +124,72 @@ def getter(*args): ...@@ -123,3 +124,72 @@ def getter(*args):
if len(response.text) > 0: if len(response.text) > 0:
with open(parsed.out, 'w') as out: with open(parsed.out, 'w') as out:
out.write(response.text.encode('utf-8')) out.write(response.text.encode('utf-8'))
def updater(*args):
"""Periodically downloads file with SSL client & server authentication,
updating it on change and success.
"""
if not args:
args = sys.argv[1:]
parser = argparse.ArgumentParser(description='KeDiFa updater')
parser.add_argument(
'mapping',
type=argparse.FileType('r'),
help='File mapping of URL to DESTINATION, where URL is the source of the '
'certificate, and DESTINATION is the output file.'
)
parser.add_argument(
'--identity',
type=argparse.FileType('r'),
help='Certificate to identify itself on the URL.',
required=True
)
parser.add_argument(
'--server-ca-certificate',
type=argparse.FileType('r'),
help='CA Certificate of the server.',
required=True
)
parser.add_argument(
'--master-certificate',
type=str,
help='Master certificate, to use in some cases. If it exsists in mapping '
'file, will be updated.',
)
parser.add_argument(
'--on-update',
type=str,
help='Executable to be run when update happens.'
)
parser.add_argument(
'--sleep',
type=int,
help='Sleep time in seconds.',
default=60
)
parser.add_argument(
'--once',
action='store_true',
help='Run only once.',
)
parsed = parser.parse_args(args)
u = Updater(
parsed.sleep, parsed.mapping.name, parsed.master_certificate,
parsed.on_update, parsed.identity.name, parsed.server_ca_certificate.name,
parsed.once
)
parsed.mapping.close()
parsed.identity.close()
parsed.server_ca_certificate.close()
u.loop()
...@@ -45,6 +45,8 @@ import caucase.http ...@@ -45,6 +45,8 @@ import caucase.http
from cli import getter from cli import getter
from cli import http from cli import http
from cli import updater
from updater import Updater
@contextlib.contextmanager @contextlib.contextmanager
...@@ -247,6 +249,22 @@ class KedifaIntegrationTest(unittest.TestCase): ...@@ -247,6 +249,22 @@ class KedifaIntegrationTest(unittest.TestCase):
self._getter_get(url, certificate, destination) self._getter_get(url, certificate, destination)
return destination return destination
def _updater_get(self, url, certificate, destination):
mapping = tempfile.NamedTemporaryFile(dir=self.testdir, delete=False)
mapping.write("%s %s" % (url, destination))
mapping.close()
updater(
'--once',
'--server-ca-certificate', self.ca_crt_pem,
'--identity', certificate,
mapping.name,
)
def updater_get(self, url, certificate):
destination = tempfile.NamedTemporaryFile(dir=self.testdir).name
self._updater_get(url, certificate, destination)
return destination
def reserveReference(self, *args, **kwargs): def reserveReference(self, *args, **kwargs):
result = requests.post( result = requests.post(
self.kedifa_url + 'reserve-id', self.kedifa_url + 'reserve-id',
...@@ -464,6 +482,37 @@ class KedifaIntegrationTest(unittest.TestCase): ...@@ -464,6 +482,37 @@ class KedifaIntegrationTest(unittest.TestCase):
self.assertFalse(os.path.isfile(result)) self.assertFalse(os.path.isfile(result))
self.assertEqual(1, code) self.assertEqual(1, code)
def test_GET_existing_updater(self):
self.put()
result = self.updater_get(
self.kedifa_url + self.reference, self.client_key_pem)
with open(result) as result_file:
self.assertEqual(
self.pem,
result_file.read()
)
def test_GET_updater_unsigned_identity(self):
self.put()
_, key_pem, _, certificate_pem = self.generateKeyCertificateData()
incorrect_key_pem = os.path.join(self.testdir, self.id())
with open(incorrect_key_pem, 'w') as out:
out.write(key_pem + certificate_pem)
result = self.updater_get(
self.kedifa_url + self.reference, incorrect_key_pem)
self.assertFalse(os.path.isfile(result))
def test_GET_existing_updater_name_does_not_match(self):
self.put()
result = self.updater_get(
self.kedifa_url + self.reference + 'MISSING', self.client_key_pem)
self.assertFalse(os.path.isfile(result))
def test_GET_existing_exact(self): def test_GET_existing_exact(self):
self.put() self.put()
result = self.requests_get( result = self.requests_get(
...@@ -1030,3 +1079,60 @@ class KedifaIntegrationTest(unittest.TestCase): ...@@ -1030,3 +1079,60 @@ class KedifaIntegrationTest(unittest.TestCase):
"Query string '&&==' was not correct.", "Query string '&&==' was not correct.",
result.text result.text
) )
class KedifaUpdaterTest(unittest.TestCase):
def setUp(self):
self.testdir = tempfile.mkdtemp()
def cleanTestDir():
shutil.rmtree(self.testdir)
self.addCleanup(cleanTestDir)
def setupMapping(self, mapping_content=''):
mapping = tempfile.NamedTemporaryFile(dir=self.testdir, delete=False)
mapping.write(mapping_content)
mapping.close()
self.mapping = mapping.name
def test_updateMapping_empty(self):
self.setupMapping()
u = Updater(1, self.mapping, None, None, None, None, True)
u.updateMapping()
self.assertEqual(u.mapping, {})
def test_updateMapping_normal(self):
self.setupMapping('url file')
u = Updater(1, self.mapping, None, None, None, None, True)
u.updateMapping()
self.assertEqual(u.mapping, {'file': 'url'})
def test_updateMapping_morewhite(self):
self.setupMapping('url \t file')
u = Updater(1, self.mapping, None, None, None, None, True)
u.updateMapping()
self.assertEqual(u.mapping, {'file': 'url'})
def test_updateMapping_one_empty(self):
self.setupMapping('url file\n \n')
u = Updater(1, self.mapping, None, None, None, None, True)
u.updateMapping()
self.assertEqual(u.mapping, {'file': 'url'})
def test_updateMapping_one_not_enough(self):
self.setupMapping('url file\nbuzz\n')
u = Updater(1, self.mapping, None, None, None, None, True)
u.updateMapping()
self.assertEqual(u.mapping, {'file': 'url'})
def test_updateMapping_one_too_much(self):
self.setupMapping('url file\nbuzz oink aff\n')
u = Updater(1, self.mapping, None, None, None, None, True)
u.updateMapping()
self.assertEqual(u.mapping, {'file': 'url'})
def test_updateMapping_one_comment(self):
self.setupMapping('url file\n#buzz uff\n')
u = Updater(1, self.mapping, None, None, None, None, True)
u.updateMapping()
self.assertEqual(u.mapping, {'file': 'url'})
import httplib
import os
import requests
import time
class Updater(object):
def __init__(self, sleep, mapping_file, master_certificate_file, on_update,
identity_file, server_ca_certificate_file, once):
self.sleep = sleep
self.mapping_file = mapping_file
self.master_certificate_file = master_certificate_file
self.on_update = on_update
self.identity_file = identity_file
self.server_ca_certificate_file = server_ca_certificate_file
self.once = once
def updateMapping(self):
self.mapping = {}
with open(self.mapping_file) as fh:
for line in fh.readlines():
line = line.strip()
if line.startswith('#'):
continue
if not line:
continue
line_content = line.split()
if len(line_content) != 2:
print 'Line %r is incorrect' % (line,)
continue
url, certificate = line_content
self.mapping[certificate] = url
def fetchCertificate(self, url, certificate_file):
certificate = ''
try:
response = requests.get(
url, verify=self.server_ca_certificate_file, cert=self.identity_file)
except Exception as e:
print 'Certificate %r: problem with %r not downloaded: %s' % (
certificate_file, url, e)
else:
if response.status_code != httplib.OK:
print 'Certificate %r: %r not downloaded, HTTP code %s' % (
certificate_file, url, response.status_code)
else:
certificate = response.text
if len(certificate) == 0:
print 'Certificate %r: %r is empty' % (certificate_file, url,)
return certificate
def updateCertificate(self, certificate_file, master_content=None):
url = self.mapping[certificate_file]
certificate = self.fetchCertificate(url, certificate_file)
current = ''
try:
with open(certificate_file, 'r') as fh:
current = fh.read()
except IOError:
current = ''
if not(certificate) and not current:
if master_content is not None:
url = self.master_certificate_file
certificate = master_content
else:
return False
if current != certificate:
with open(certificate_file, 'w') as fh:
fh.write(certificate)
print 'Certificate %r: updated from %r' % (certificate_file, url)
return True
else:
return False
def callOnUpdate(self):
if self.on_update is not None:
status = os.system(self.on_update)
print 'Called %r with status %i' % (self.on_update, status)
def loop(self):
while True:
self.updateMapping()
updated = False
if self.master_certificate_file in self.mapping:
updated = self.updateCertificate(self.master_certificate_file)
self.mapping.pop(self.master_certificate_file)
master_content = None
if self.master_certificate_file is not None:
try:
with open(self.master_certificate_file, 'r') as fh:
master_content = fh.read() or None
if master_content:
print 'Using master certificate from %r' % (
self.master_certificate_file,)
except IOError:
pass
for certificate_file in self.mapping.keys():
if self.updateCertificate(certificate_file, master_content):
updated = True
if updated:
self.callOnUpdate()
if self.once:
break
print 'Sleeping for %is' % (self.sleep,)
time.sleep(self.sleep)
...@@ -47,7 +47,7 @@ setup( ...@@ -47,7 +47,7 @@ setup(
packages=find_packages(), packages=find_packages(),
install_requires=[ install_requires=[
'cryptography', # for working with certificates 'cryptography', # for working with certificates
'requests', # for getter 'requests', # for getter and updater
'urllib3 >= 1.18', # https://github.com/urllib3/urllib3/issues/258 'urllib3 >= 1.18', # https://github.com/urllib3/urllib3/issues/258
'caucase', # provides utils for certificate management; 'caucase', # provides utils for certificate management;
# version requirement caucase >= 0.9.3 is dropped, as it # version requirement caucase >= 0.9.3 is dropped, as it
...@@ -61,6 +61,7 @@ setup( ...@@ -61,6 +61,7 @@ setup(
'console_scripts': [ 'console_scripts': [
'kedifa = kedifa.cli:http', 'kedifa = kedifa.cli:http',
'kedifa-getter = kedifa.cli:getter', 'kedifa-getter = kedifa.cli:getter',
'kedifa-updater = kedifa.cli:updater',
] ]
}, },
) )
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment