kedifa: Introduce asynchronous updater

Features:

 * by default runs with 60s sleep
 * allows to have master, updateable, certificate, which is used in case if
   specific certificate is not available

kedifa/cli.py

@@ -23,6 +23,7 @@ import requests
 import sys
 
 import app
+from updater import Updater
 
 
 def http(*args):
@@ -123,3 +124,72 @@ def getter(*args):
     if len(response.text) > 0:
       with open(parsed.out, 'w') as out:
         out.write(response.text.encode('utf-8'))
+
+
+def updater(*args):
+  """Periodically downloads file with SSL client & server authentication,
+    updating it on change and success.
+  """
+  if not args:
+    args = sys.argv[1:]
+
+  parser = argparse.ArgumentParser(description='KeDiFa updater')
+
+  parser.add_argument(
+    'mapping',
+    type=argparse.FileType('r'),
+    help='File mapping of URL to DESTINATION, where URL is the source of the '
+         'certificate, and DESTINATION is the output file.'
+  )
+
+  parser.add_argument(
+    '--identity',
+    type=argparse.FileType('r'),
+    help='Certificate to identify itself on the URL.',
+    required=True
+  )
+
+  parser.add_argument(
+    '--server-ca-certificate',
+    type=argparse.FileType('r'),
+    help='CA Certificate of the server.',
+    required=True
+  )
+
+  parser.add_argument(
+    '--master-certificate',
+    type=str,
+    help='Master certificate, to use in some cases. If it exsists in mapping '
+         'file, will be updated.',
+  )
+
+  parser.add_argument(
+    '--on-update',
+    type=str,
+    help='Executable to be run when update happens.'
+  )
+
+  parser.add_argument(
+    '--sleep',
+    type=int,
+    help='Sleep time in seconds.',
+    default=60
+  )
+
+  parser.add_argument(
+    '--once',
+    action='store_true',
+    help='Run only once.',
+  )
+
+  parsed = parser.parse_args(args)
+
+  u = Updater(
+    parsed.sleep, parsed.mapping.name, parsed.master_certificate,
+    parsed.on_update, parsed.identity.name, parsed.server_ca_certificate.name,
+    parsed.once
+  )
+  parsed.mapping.close()
+  parsed.identity.close()
+  parsed.server_ca_certificate.close()
+  u.loop()

kedifa/test.py

@@ -45,6 +45,8 @@ import caucase.http
 
 from cli import getter
 from cli import http
+from cli import updater
+from updater import Updater
 
 
 @contextlib.contextmanager
@@ -247,6 +249,22 @@ class KedifaIntegrationTest(unittest.TestCase):
     self._getter_get(url, certificate, destination)
     return destination
 
+  def _updater_get(self, url, certificate, destination):
+    mapping = tempfile.NamedTemporaryFile(dir=self.testdir, delete=False)
+    mapping.write("%s %s" % (url, destination))
+    mapping.close()
+    updater(
+      '--once',
+      '--server-ca-certificate', self.ca_crt_pem,
+      '--identity', certificate,
+      mapping.name,
+    )
+
+  def updater_get(self, url, certificate):
+    destination = tempfile.NamedTemporaryFile(dir=self.testdir).name
+    self._updater_get(url, certificate, destination)
+    return destination
+
   def reserveReference(self, *args, **kwargs):
     result = requests.post(
       self.kedifa_url + 'reserve-id',
@@ -464,6 +482,37 @@ class KedifaIntegrationTest(unittest.TestCase):
     self.assertFalse(os.path.isfile(result))
     self.assertEqual(1, code)
 
+  def test_GET_existing_updater(self):
+    self.put()
+    result = self.updater_get(
+      self.kedifa_url + self.reference, self.client_key_pem)
+    with open(result) as result_file:
+      self.assertEqual(
+        self.pem,
+        result_file.read()
+      )
+
+  def test_GET_updater_unsigned_identity(self):
+    self.put()
+
+    _, key_pem, _, certificate_pem = self.generateKeyCertificateData()
+    incorrect_key_pem = os.path.join(self.testdir, self.id())
+    with open(incorrect_key_pem, 'w') as out:
+      out.write(key_pem + certificate_pem)
+
+    result = self.updater_get(
+      self.kedifa_url + self.reference, incorrect_key_pem)
+
+    self.assertFalse(os.path.isfile(result))
+
+  def test_GET_existing_updater_name_does_not_match(self):
+    self.put()
+
+    result = self.updater_get(
+      self.kedifa_url + self.reference + 'MISSING', self.client_key_pem)
+
+    self.assertFalse(os.path.isfile(result))
+
   def test_GET_existing_exact(self):
     self.put()
     result = self.requests_get(
@@ -1030,3 +1079,60 @@ class KedifaIntegrationTest(unittest.TestCase):
       "Query string '&&==' was not correct.",
       result.text
     )
+
+
+class KedifaUpdaterTest(unittest.TestCase):
+  def setUp(self):
+    self.testdir = tempfile.mkdtemp()
+
+    def cleanTestDir():
+      shutil.rmtree(self.testdir)
+    self.addCleanup(cleanTestDir)
+
+  def setupMapping(self, mapping_content=''):
+    mapping = tempfile.NamedTemporaryFile(dir=self.testdir, delete=False)
+    mapping.write(mapping_content)
+    mapping.close()
+    self.mapping = mapping.name
+
+  def test_updateMapping_empty(self):
+    self.setupMapping()
+    u = Updater(1, self.mapping, None, None, None, None, True)
+    u.updateMapping()
+    self.assertEqual(u.mapping, {})
+
+  def test_updateMapping_normal(self):
+    self.setupMapping('url file')
+    u = Updater(1, self.mapping, None, None, None, None, True)
+    u.updateMapping()
+    self.assertEqual(u.mapping, {'file': 'url'})
+
+  def test_updateMapping_morewhite(self):
+    self.setupMapping('url \t file')
+    u = Updater(1, self.mapping, None, None, None, None, True)
+    u.updateMapping()
+    self.assertEqual(u.mapping, {'file': 'url'})
+
+  def test_updateMapping_one_empty(self):
+    self.setupMapping('url file\n     \n')
+    u = Updater(1, self.mapping, None, None, None, None, True)
+    u.updateMapping()
+    self.assertEqual(u.mapping, {'file': 'url'})
+
+  def test_updateMapping_one_not_enough(self):
+    self.setupMapping('url file\nbuzz\n')
+    u = Updater(1, self.mapping, None, None, None, None, True)
+    u.updateMapping()
+    self.assertEqual(u.mapping, {'file': 'url'})
+
+  def test_updateMapping_one_too_much(self):
+    self.setupMapping('url file\nbuzz oink aff\n')
+    u = Updater(1, self.mapping, None, None, None, None, True)
+    u.updateMapping()
+    self.assertEqual(u.mapping, {'file': 'url'})
+
+  def test_updateMapping_one_comment(self):
+    self.setupMapping('url file\n#buzz uff\n')
+    u = Updater(1, self.mapping, None, None, None, None, True)
+    u.updateMapping()
+    self.assertEqual(u.mapping, {'file': 'url'})

kedifa/updater.py

+import httplib
+import os
+import requests
+import time
+
+
+class Updater(object):
+  def __init__(self, sleep, mapping_file, master_certificate_file, on_update,
+               identity_file, server_ca_certificate_file, once):
+    self.sleep = sleep
+    self.mapping_file = mapping_file
+    self.master_certificate_file = master_certificate_file
+    self.on_update = on_update
+    self.identity_file = identity_file
+    self.server_ca_certificate_file = server_ca_certificate_file
+    self.once = once
+
+  def updateMapping(self):
+    self.mapping = {}
+    with open(self.mapping_file) as fh:
+      for line in fh.readlines():
+        line = line.strip()
+        if line.startswith('#'):
+          continue
+        if not line:
+          continue
+        line_content = line.split()
+        if len(line_content) != 2:
+          print 'Line %r is incorrect' % (line,)
+          continue
+        url, certificate = line_content
+        self.mapping[certificate] = url
+
+  def fetchCertificate(self, url, certificate_file):
+    certificate = ''
+    try:
+      response = requests.get(
+        url, verify=self.server_ca_certificate_file, cert=self.identity_file)
+    except Exception as e:
+      print 'Certificate %r: problem with %r not downloaded: %s' % (
+        certificate_file, url, e)
+    else:
+      if response.status_code != httplib.OK:
+        print 'Certificate %r: %r not downloaded, HTTP code %s' % (
+          certificate_file, url, response.status_code)
+      else:
+        certificate = response.text
+        if len(certificate) == 0:
+          print 'Certificate %r: %r is empty' % (certificate_file, url,)
+    return certificate
+
+  def updateCertificate(self, certificate_file, master_content=None):
+    url = self.mapping[certificate_file]
+    certificate = self.fetchCertificate(url, certificate_file)
+
+    current = ''
+    try:
+      with open(certificate_file, 'r') as fh:
+        current = fh.read()
+    except IOError:
+      current = ''
+
+    if not(certificate) and not current:
+      if master_content is not None:
+        url = self.master_certificate_file
+        certificate = master_content
+      else:
+        return False
+
+    if current != certificate:
+      with open(certificate_file, 'w') as fh:
+        fh.write(certificate)
+        print 'Certificate %r: updated from %r' % (certificate_file, url)
+        return True
+    else:
+      return False
+
+  def callOnUpdate(self):
+    if self.on_update is not None:
+      status = os.system(self.on_update)
+      print 'Called %r with status %i' % (self.on_update, status)
+
+  def loop(self):
+    while True:
+      self.updateMapping()
+      updated = False
+
+      if self.master_certificate_file in self.mapping:
+        updated = self.updateCertificate(self.master_certificate_file)
+        self.mapping.pop(self.master_certificate_file)
+
+      master_content = None
+      if self.master_certificate_file is not None:
+        try:
+          with open(self.master_certificate_file, 'r') as fh:
+            master_content = fh.read() or None
+            if master_content:
+              print 'Using master certificate from %r' % (
+                self.master_certificate_file,)
+        except IOError:
+          pass
+
+      for certificate_file in self.mapping.keys():
+        if self.updateCertificate(certificate_file, master_content):
+          updated = True
+
+      if updated:
+        self.callOnUpdate()
+
+      if self.once:
+        break
+      print 'Sleeping for %is' % (self.sleep,)
+      time.sleep(self.sleep)

setup.py

@@ -47,7 +47,7 @@ setup(
   packages=find_packages(),
   install_requires=[
     'cryptography',  # for working with certificates
-    'requests',  # for getter
+    'requests',  # for getter and updater
     'urllib3 >= 1.18',  # https://github.com/urllib3/urllib3/issues/258
     'caucase',  # provides utils for certificate management;
                 # version requirement caucase >= 0.9.3 is dropped, as it
@@ -61,6 +61,7 @@ setup(
     'console_scripts': [
       'kedifa = kedifa.cli:http',
       'kedifa-getter = kedifa.cli:getter',
+      'kedifa-updater = kedifa.cli:updater',
     ]
   },
 )