software/theia: Wait for proxy restart in import

Resiliency import script had a race condition where the proxy might not
have finished restarting before formatting the partitions, causing the
import script to randomly fail during format because the proxy is down
when format tries to send it updated computer information.

See merge request nexedi/slapos!1406

software/theia/buildout.hash.cfg

@@ -47,7 +47,7 @@ md5sum = e2f6c483cce09f87ab1e63ae8be0daf4
 
 [theia-import]
 _update_hash_filename_ = theia_import.py
-md5sum = 1a668d6203d42b4d46d56e24c7606cb2
+md5sum = 45e757f216374d22f0a92d5334dc00f0
 
 [slapos.css.in]
 _update_hash_filename_ = slapos.css.in

software/theia/software.cfg

@@ -125,6 +125,7 @@ eggs =
   ${slapos-toolbox:eggs}
   six
   zc.buildout
+  requests
 
 [python-for-standalone]
 <= python-with-eggs

software/theia/theia_import.py

@@ -4,8 +4,12 @@ import itertools
 import os
 import sys
 import subprocess as sp
+import time
 import traceback
 
+import requests
+from requests.exceptions import RequestException
+
 import six
 from six.moves import configparser
 
@@ -56,6 +60,7 @@ class TheiaImport(object):
     configp = configparser.SafeConfigParser()
     configp.read(cfg)
     self.proxy_db = configp.get('slapproxy', 'database_uri')
+    self.proxy_rest_url = configp.get('slapos', 'master_rest_url') # 200 OK
     self.instance_dir = configp.get('slapos', 'instance_root')
     mirror_dir = self.mirror_path(self.instance_dir)
     partitions = glob.glob(os.path.join(mirror_dir, 'slappart*'))
@@ -150,7 +155,27 @@ class TheiaImport(object):
       print(msg)
       raise Exception(msg)
 
+  def wait_for_proxy(self):
+    timeout = 10
+    sleep = 20
+    url = self.proxy_rest_url
+    for i in range(10):
+      try:
+        self.log(
+          '- GET proxy %s with timeout %d (attempt %d)' % (url, timeout, i))
+        response = requests.get(url, verify=True, timeout=timeout)
+      except RequestException:
+        if i == 9:
+          raise
+        else:
+          self.log('- Sleep %ds before retrying' % sleep)
+          time.sleep(sleep)
+      else:
+        response.raise_for_status()
+        break
+
   def log(self, msg):
+    msg = time.strftime("%Y-%B-%d %H:%M:%S - ") + msg
     print(msg)
     self.logs.append(msg)
 
@@ -214,6 +239,9 @@ class TheiaImport(object):
     self.log('Start slapproxy again')
     self.supervisorctl('start', 'slapos-proxy')
 
+    self.log('Wait until slapproxy is available')
+    self.wait_for_proxy()
+
     self.log('Reformat partitions')
     self.slapos('node', 'format', '--now')