erp5_bootstrap: add more connection controls

slapos/recipe/erp5_bootstrap/template/erp5_bootstrap.in

@@ -6,7 +6,7 @@ import base64
 import time
 
 MAX_INSTALLATION_TIME = 60*30
-MAX_TESTING_TIME = 60
+MAX_TESTING_TIME = 60*2
 MAX_GETTING_CONNECTION_TIME = 60*5
 
 user = "%(user)s"
@@ -33,41 +33,61 @@ def getConnection():
         return httplib.HTTPConnection(host)
       else:
         raise ValueError("Protocol not implemented")
-      except:
-        print "Getting new connection failed, retry"
+    except:
+      print "Getting new connection failed, retry"
+      time.sleep(10)
   raise ValueError("Cannot get new connection after %%d try (for %%s s)" %%(count, str(time.time()-start_time)))
 
 def testIfExist(page, unexcepted_content="Site Error"):
   print "Test if %%s exists" %%(page)
-  zope_connection = getConnection()
-  zope_connection.request('GET', '/%%s/%%s' %%(site_id, page))
-  result = zope_connection.getresponse()
-  return not unexcepted_content in result.read()
+  start_time = time.time()
+  count = 0
+  while MAX_TESTING_TIME > time.time()-start_time:
+    try:
+      count = count + 1
+      zope_connection = getConnection()
+      zope_connection.request('GET', '/%%s/%%s' %%(site_id, page))
+      result = zope_connection.getresponse()
+      return not unexcepted_content in result.read()
+    except:
+      print "Test if exists failed, retry..."
+      time.sleep(10)
+  raise ValueError("Cannot testIfExist after %%d try (for %%s s)" %%(count, str(time.time()-start_time)))
+  
   
 def waitFor0PendingActivities():
   # TODO: tolerate 1 pending activities ? (mail server...)
   print "waitFor0PendingActivities.."
   start_time = time.time()
+  count = 0
   while MAX_INSTALLATION_TIME > time.time()-start_time:
-    zope_connection = getConnection()
-    zope_connection.request(
-      'GET', '/%%s/portal_activities/getMessageList' %%(site_id),
-      headers=header_dict
-    )
-    result = zope_connection.getresponse()
-    message_list_text = result.read()
-    message_list = [s.strip() for s in message_list_text[1:-1].split(',')]
-    if len(message_list)==0:
-      print "There is no pending activities."
-      break
-    print "There is %%d pending activities" %%len(message_list)
-    time.sleep(5)
+    try:
+      count = count + 1
+      zope_connection = getConnection()
+      zope_connection.request(
+        'GET', '/%%s/portal_activities/getMessageList' %%(site_id),
+        headers=header_dict
+      )
+      result = zope_connection.getresponse()
+      message_list_text = result.read()
+      message_list = [s.strip() for s in message_list_text[1:-1].split(',')]
+      if len(message_list)==0:
+        print "There is no pending activities."
+        break
+      print "There is %%d pending activities" %%len(message_list)
+      time.sleep(5)
+
+      #Hack to do not take into account persistent Alarm_installMailServer acitivities
+      if len(message_list)==0 and testIfExist("/%%s/portal_activities" %%site_id, "Alarm_installMailServer"):
+        print "1 pending activity for 'Alarm_installMailServer'."
+        print "ok."
+        break
+        
+    except:
+      time.sleep(5)
+      print "Getting activities failed, retry."
+  raise ValueError("Cannot testIfExist after %%d try (for %%s s)" %%(count, str(time.time()-start_time)))
 
-    #Hack to do not take into account persistent Alarm_installMailServer acitivities
-    if len(message_list)==0 and testIfExist("/%%s/portal_activities" %%site_id, "Alarm_installMailServer"):
-      print "1 pending activity for 'Alarm_installMailServer'."
-      print "ok."
-      break
 
   
 # Check if an ERP5 site is already created, as ERP5 does support having