Add download auto retrying function. This is necessary because downloading...

Add download auto retrying function. This is necessary because downloading binary cache often fails.

slapos/libnetworkcache.py

@@ -97,6 +97,67 @@ class CheckResponse(object):
             'Failed to download data to SHACACHE Server: invalid checksum.')
     return r
 
+class Retry(object):
+
+  def __init__(self, response, is_https, connection_kw, method, path, data, headers):
+    self.connection_kw = connection_kw
+    self.method = method
+    self.path = path
+    self.data = data
+    self.headers = headers
+    self.is_https = is_https
+    self.content_length = 0
+    self.position = 0
+    self.response = response
+    self.updateContentLength()
+
+  def updateContentLength(self):
+    if not self.content_length:
+      content_length_header = self.response.getheader('content-length')
+      if (content_length_header
+          and content_length_header.isdigit()
+          and self.response.getheader('accept-ranges') == 'bytes'):
+        self.content_length = int(content_length_header)
+
+  def getresponse(self):
+    if self.response is not None:
+      self.response.close()
+      self.response = None
+
+    if self.is_https:
+      connection = HTTPSConnection(**self.connection_kw)
+    else:
+      connection = HTTPConnection(**self.connection_kw)
+
+    new_headers = self.headers.copy()
+    if self.content_length > 0 and self.position > 0:
+      new_headers['Range'] = 'bytes=%d-' % self.position
+
+    try:
+      connection.request(self.method, self.path, self.data, new_headers)
+      self.response = connection.getresponse()
+      if 200 <= self.response.status < 300:
+        self.updateContentLength()
+        return self.response
+    finally:
+      connection.close()
+
+  def read(self, size=16*1024):
+    data = self.response.read(size)
+    if not data:
+      if self.position < self.content_length:
+        self.getresponse()
+        if self.response is not None:
+          data = self.response.read(size)
+    self.position += len(data)
+    return data
+
+  def close(self):
+    self.response.close()
+    self.response = None
+    self.content_length = 0
+    self.position = 0
+
 class NetworkcacheClient(object):
   '''
     NetworkcacheClient is a wrapper for httplib.
@@ -201,14 +262,19 @@ class NetworkcacheClient(object):
           cafile=self.config.get('sha%s-ca-file' % where)
         )
         context.set_ciphers('DEFAULT:@SECLEVEL=1') # XXX
+      is_https = True
       connection = HTTPSConnection(**connection_kw)
     else:
+      is_https = False
       connection = HTTPConnection(**connection_kw)
     try:
       connection.request(method, parsed_url.path, data, headers)
       r = connection.getresponse()
       if 200 <= r.status < 300:
-        return r
+        if method == 'GET':
+          return Retry(r, is_https, connection_kw, method, parsed_url.path, data, headers)
+        else:
+          return r
     finally:
       connection.close()
     raise HTTPError(url, r.status, r.reason, r.msg, r.fp)