slapgrid: binary cache only option

/reviewed-on !173

slapos.cfg.example

@@ -162,3 +162,7 @@ upload-to-binary-cache-url-blacklist =
   http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD
   http://git.erp5.org/gitweb/slapos.git/blob_plain/refs/heads
   /
+
+# List of URL(s) which are forced to be downloaded from binary cache
+#download-from-binary-cache-force-url-list =
+#  https://lab.node.vifib.com/nexedi/slapos/raw/1.0.

slapos/grid/SlapObject.py

@@ -112,6 +112,7 @@ class Software(object):
                download_binary_dir_url=None, upload_binary_dir_url=None,
                download_from_binary_cache_url_blacklist=None,
                upload_to_binary_cache_url_blacklist=None,
+               download_from_binary_cache_force_url_list=None,
                software_min_free_space=None,
                buildout_debug=False,
                shared_part_list=''):
@@ -124,6 +125,9 @@ class Software(object):
     if upload_to_binary_cache_url_blacklist is None:
       upload_to_binary_cache_url_blacklist = []
 
+    if download_from_binary_cache_force_url_list is None:
+      download_from_binary_cache_force_url_list = []
+
     self.url = url
     self.software_root = software_root
     self.software_url_hash = md5digest(self.url)
@@ -151,6 +155,8 @@ class Software(object):
         download_from_binary_cache_url_blacklist
     self.upload_to_binary_cache_url_blacklist = \
         upload_to_binary_cache_url_blacklist
+    self.download_from_binary_cache_force_url_list = \
+        download_from_binary_cache_force_url_list
     self.software_min_free_space = software_min_free_space
 
   def check_free_space(self):
@@ -175,6 +181,12 @@ class Software(object):
     try:
       tarpath = os.path.join(cache_dir, self.software_url_hash)
       # Check if we can download from cache
+      force_binary_cache = False
+      for url_match in self.download_from_binary_cache_force_url_list:
+        if self.url.startswith(url_match):
+          force_binary_cache = True
+          self.logger.debug('Binary cache forced for %r because of match %r', self.url, url_match)
+          break
       if (not os.path.exists(self.software_path)) \
           and download_network_cached(
               self.download_binary_cache_url,
@@ -190,6 +202,10 @@ class Software(object):
           tar.extractall(path=self.software_root)
         finally:
           tar.close()
+      elif force_binary_cache:
+        message = 'Binary cache forced for %r, but failed to download, will retry again' % (self.url,)
+        self.logger.error(message)
+        raise BuildoutFailedError(message)
       else:
         self._install_from_buildout()
         # Upload to binary cache if possible and allowed

slapos/grid/slapgrid.py

@@ -188,6 +188,9 @@ def merged_options(args, configp):
   options["upload-to-binary-cache-url-blacklist"] = [
       url.strip() for url in options.get(
           "upload-to-binary-cache-url-blacklist", "").split('\n') if url]
+  options["download-from-binary-cache-force-url-list"] = [
+      url.strip() for url in options.get(
+          "download-from-binary-cache-force-url-list", "").split('\n') if url]
 
   options['firewall'] = {}
   if configp.has_section('firewall'):
@@ -259,6 +262,8 @@ def create_slapgrid_object(options, logger):
                       op.get('download-from-binary-cache-url-blacklist', []),
                   upload_to_binary_cache_url_blacklist=
                       op.get('upload-to-binary-cache-url-blacklist', []),
+                  download_from_binary_cache_force_url_list=
+                      op.get('download-from-binary-cache-force-url-list', []),
                   upload_cache_url=op.get('upload-cache-url'),
                   download_binary_dir_url=op.get('download-binary-dir-url'),
                   upload_binary_dir_url=op.get('upload-binary-dir-url'),
@@ -317,6 +322,7 @@ class Slapgrid(object):
                upload_binary_cache_url=None,
                download_from_binary_cache_url_blacklist=None,
                upload_to_binary_cache_url_blacklist=None,
+               download_from_binary_cache_force_url_list=None,
                upload_cache_url=None,
                download_binary_dir_url=None,
                upload_binary_dir_url=None,
@@ -362,6 +368,8 @@ class Slapgrid(object):
         download_from_binary_cache_url_blacklist
     self.upload_to_binary_cache_url_blacklist = \
         upload_to_binary_cache_url_blacklist
+    self.download_from_binary_cache_force_url_list = \
+        download_from_binary_cache_force_url_list
     self.upload_cache_url = upload_cache_url
     self.download_binary_dir_url = download_binary_dir_url
     self.upload_binary_dir_url = upload_binary_dir_url
@@ -555,6 +563,8 @@ stderr_logfile_backups=1
                 self.download_from_binary_cache_url_blacklist,
             upload_to_binary_cache_url_blacklist=
                 self.upload_to_binary_cache_url_blacklist,
+            download_from_binary_cache_force_url_list=
+                self.download_from_binary_cache_force_url_list,
             upload_cache_url=self.upload_cache_url,
             download_binary_dir_url=self.download_binary_dir_url,
             upload_binary_dir_url=self.upload_binary_dir_url,