Commit 63078069 authored by Xavier Thompson's avatar Xavier Thompson

software/theia: Wait for proxy restart in import

Resiliency import script had a race condition where the proxy might not
have finished restarting before formatting the partitions, causing the
import script to randomly fail during format because the proxy is down
when format tries to send it updated computer information.

See merge request !1406
parent ef076948
...@@ -47,7 +47,7 @@ md5sum = e2f6c483cce09f87ab1e63ae8be0daf4 ...@@ -47,7 +47,7 @@ md5sum = e2f6c483cce09f87ab1e63ae8be0daf4
[theia-import] [theia-import]
_update_hash_filename_ = theia_import.py _update_hash_filename_ = theia_import.py
md5sum = 1a668d6203d42b4d46d56e24c7606cb2 md5sum = 45e757f216374d22f0a92d5334dc00f0
[slapos.css.in] [slapos.css.in]
_update_hash_filename_ = slapos.css.in _update_hash_filename_ = slapos.css.in
......
...@@ -125,6 +125,7 @@ eggs = ...@@ -125,6 +125,7 @@ eggs =
${slapos-toolbox:eggs} ${slapos-toolbox:eggs}
six six
zc.buildout zc.buildout
requests
[python-for-standalone] [python-for-standalone]
<= python-with-eggs <= python-with-eggs
......
...@@ -4,8 +4,12 @@ import itertools ...@@ -4,8 +4,12 @@ import itertools
import os import os
import sys import sys
import subprocess as sp import subprocess as sp
import time
import traceback import traceback
import requests
from requests.exceptions import RequestException
import six import six
from six.moves import configparser from six.moves import configparser
...@@ -56,6 +60,7 @@ class TheiaImport(object): ...@@ -56,6 +60,7 @@ class TheiaImport(object):
configp = configparser.SafeConfigParser() configp = configparser.SafeConfigParser()
configp.read(cfg) configp.read(cfg)
self.proxy_db = configp.get('slapproxy', 'database_uri') self.proxy_db = configp.get('slapproxy', 'database_uri')
self.proxy_rest_url = configp.get('slapos', 'master_rest_url') # 200 OK
self.instance_dir = configp.get('slapos', 'instance_root') self.instance_dir = configp.get('slapos', 'instance_root')
mirror_dir = self.mirror_path(self.instance_dir) mirror_dir = self.mirror_path(self.instance_dir)
partitions = glob.glob(os.path.join(mirror_dir, 'slappart*')) partitions = glob.glob(os.path.join(mirror_dir, 'slappart*'))
...@@ -150,7 +155,27 @@ class TheiaImport(object): ...@@ -150,7 +155,27 @@ class TheiaImport(object):
print(msg) print(msg)
raise Exception(msg) raise Exception(msg)
def wait_for_proxy(self):
timeout = 10
sleep = 20
url = self.proxy_rest_url
for i in range(10):
try:
self.log(
'- GET proxy %s with timeout %d (attempt %d)' % (url, timeout, i))
response = requests.get(url, verify=True, timeout=timeout)
except RequestException:
if i == 9:
raise
else:
self.log('- Sleep %ds before retrying' % sleep)
time.sleep(sleep)
else:
response.raise_for_status()
break
def log(self, msg): def log(self, msg):
msg = time.strftime("%Y-%B-%d %H:%M:%S - ") + msg
print(msg) print(msg)
self.logs.append(msg) self.logs.append(msg)
...@@ -214,6 +239,9 @@ class TheiaImport(object): ...@@ -214,6 +239,9 @@ class TheiaImport(object):
self.log('Start slapproxy again') self.log('Start slapproxy again')
self.supervisorctl('start', 'slapos-proxy') self.supervisorctl('start', 'slapos-proxy')
self.log('Wait until slapproxy is available')
self.wait_for_proxy()
self.log('Reformat partitions') self.log('Reformat partitions')
self.slapos('node', 'format', '--now') self.slapos('node', 'format', '--now')
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment