slapgrid: try 3 times to upload archive to binary cache

See merge request nexedi/slapos.core!185

slapos/grid/SlapObject.py

@@ -337,8 +337,9 @@ class Software(object):
       tar.add(self.software_path, arcname=self.software_url_hash)
     finally:
       tar.close()
-    self.logger.info("Trying to upload archive of software release...")
-    upload_network_cached(
+    for i in range(3):
+      self.logger.info("Trying to upload archive of software release (try %d)..." % (i))
+      if upload_network_cached(
           self.software_root,
           self.url, self.software_url_hash,
           self.upload_binary_cache_url,
@@ -350,7 +351,10 @@ class Software(object):
           self.shacache_key_file,
           self.shadir_ca_file,
           self.shadir_cert_file,
-        self.shadir_key_file)
+          self.shadir_key_file):
+        break
+      # let shacache process the upload before retrying
+      time.sleep(1800)
 
   def destroy(self):
     """Removes software release."""