updater: Make stateful decision

If at least once certificate has been downloaded from KeDiFa it shall never
use again the fall-back, as otherwise it would result with a problem, that
next unsuccessful download from KeDiFa would result replacement with
fall-back.

In order to do so state file is introduced keeping list of overridden
certificates. As now there is critical path regarding fetching certificates,
the lock is created to avoid concurrent updates.

kedifa/cli.py

@@ -142,6 +142,13 @@ def updater(*args):
          'certificate, and DESTINATION is the output file.'
   )
 
+  parser.add_argument(
+    'state',
+    type=str,
+    help='Path to JSON state file for fallback recognition, on which locks '
+         'will happen.'
+  )
+
   parser.add_argument(
     '--identity',
     type=argparse.FileType('r'),
@@ -185,7 +192,7 @@ def updater(*args):
   parsed = parser.parse_args(args)
 
   u = Updater(
-    parsed.sleep, parsed.mapping.name, parsed.master_certificate,
+    parsed.sleep, parsed.mapping.name, parsed.state, parsed.master_certificate,
     parsed.on_update, parsed.identity.name, parsed.server_ca_certificate.name,
     parsed.once
   )

kedifa/updater.py

 import httplib
+import json
 import os
 import requests
+import sys
 import time
+import zc.lockfile
 
 
 class Updater(object):
-  def __init__(self, sleep, mapping_file, master_certificate_file, on_update,
-               identity_file, server_ca_certificate_file, once):
+  def __init__(self, sleep, mapping_file, state_file, master_certificate_file,
+               on_update, identity_file, server_ca_certificate_file, once):
     self.sleep = sleep
     self.mapping_file = mapping_file
+    self.state_file = state_file
+    self.state_lock_file = '%s.lock' % (state_file, )
     self.master_certificate_file = master_certificate_file
     self.on_update = on_update
     self.identity_file = identity_file
@@ -25,11 +30,15 @@ class Updater(object):
         if not line:
           continue
         line_content = line.split()
-        if len(line_content) != 2:
+        if len(line_content) == 2:
+          url, certificate = line_content
+          fallback = None
+        elif len(line_content) == 3:
+          url, certificate, fallback = line_content
+        else:
           print 'Line %r is incorrect' % (line,)
           continue
-        url, certificate = line_content
-        self.mapping[certificate] = url
+        self.mapping[certificate] = (url, fallback)
 
   def fetchCertificate(self, url, certificate_file):
     certificate = ''
@@ -50,9 +59,17 @@ class Updater(object):
     return certificate
 
   def updateCertificate(self, certificate_file, master_content=None):
-    url = self.mapping[certificate_file]
+    url, fallback_file = self.mapping[certificate_file]
     certificate = self.fetchCertificate(url, certificate_file)
 
+    fallback_overridden = self.state_dict.get(certificate_file, False)
+    fallback = ''
+    if fallback_file:
+      try:
+        with open(fallback_file, 'r') as fh:
+          fallback = fh.read() or None
+      except IOError:
+        pass
     current = ''
     try:
       with open(certificate_file, 'r') as fh:
@@ -61,11 +78,16 @@ class Updater(object):
       current = ''
 
     if not(certificate):
-      if not current and master_content is not None:
+      if fallback and not fallback_overridden:
+        certificate = fallback
+      elif not current and master_content is not None:
         url = self.master_certificate_file
         certificate = master_content
       else:
         return False
+    else:
+      self.state_dict[certificate_file] = True
+
     if current != certificate:
       with open(certificate_file, 'w') as fh:
         fh.write(certificate)
@@ -79,33 +101,63 @@ class Updater(object):
       status = os.system(self.on_update)
       print 'Called %r with status %i' % (self.on_update, status)
 
-  def loop(self):
-    while True:
-      self.updateMapping()
-      updated = False
-
-      if self.master_certificate_file in self.mapping:
-        updated = self.updateCertificate(self.master_certificate_file)
-        self.mapping.pop(self.master_certificate_file)
-
-      master_content = None
-      if self.master_certificate_file is not None:
+  def readState(self):
+    self.state_dict = {}
+    try:
+      with open(self.state_file, 'r') as fh:
         try:
-          with open(self.master_certificate_file, 'r') as fh:
-            master_content = fh.read() or None
-            if master_content:
-              print 'Using master certificate from %r' % (
-                self.master_certificate_file,)
-        except IOError:
+          self.state_dict = json.load(fh)
+        except ValueError:
           pass
+    except IOError:
+      pass
 
-      for certificate_file in self.mapping.keys():
-        if self.updateCertificate(certificate_file, master_content):
-          updated = True
+  def writeState(self):
+    with open(self.state_file, 'w') as fh:
+      json.dump(self.state_dict, fh, indent=2)
 
-      if updated:
-        self.callOnUpdate()
+  def action(self):
+    self.readState()
+    self.updateMapping()
+    updated = False
 
+    if self.master_certificate_file in self.mapping:
+      updated = self.updateCertificate(self.master_certificate_file)
+      self.mapping.pop(self.master_certificate_file)
+
+    master_content = None
+    if self.master_certificate_file is not None:
+      try:
+        with open(self.master_certificate_file, 'r') as fh:
+          master_content = fh.read() or None
+          if master_content:
+            print 'Using master certificate from %r' % (
+              self.master_certificate_file,)
+      except IOError:
+        pass
+
+    for certificate_file in self.mapping.keys():
+      if self.updateCertificate(certificate_file, master_content):
+        updated = True
+
+    if updated:
+      self.callOnUpdate()
+    self.writeState()
+
+  def loop(self):
+    while True:
+      try:
+        lock = zc.lockfile.LockFile(self.state_lock_file)
+      except zc.lockfile.LockError as e:
+        print e,
+        if self.once:
+          print '...exiting.'
+          sys.exit(1)
+        else:
+          print "...will try again later."
+      else:
+        self.action()
+        lock.close()
       if self.once:
         break
       print 'Sleeping for %is' % (self.sleep,)

setup.py

@@ -49,6 +49,7 @@ setup(
   install_requires=[
     'cryptography',  # for working with certificates
     'requests',  # for getter and updater
+    'zc.lockfile',  # for stateful updater
     'urllib3 >= 1.18',  # https://github.com/urllib3/urllib3/issues/258
     'caucase',  # provides utils for certificate management;
                 # version requirement caucase >= 0.9.3 is dropped, as it