Commit 000abb02 authored by Jérome Perrin's avatar Jérome Perrin

grid: cleanup the duplicate supervisord processes

After 1.6.1 was deployed we have two processes, stop the redundant process.
parent ad156d36
...@@ -66,10 +66,29 @@ def getSupervisorRPC(socket): ...@@ -66,10 +66,29 @@ def getSupervisorRPC(socket):
def _getSupervisordSocketPath(instance_root, logger): def _getSupervisordSocketPath(instance_root, logger):
legacy_socket_path = os.path.join(instance_root, 'supervisord.socket') legacy_socket_path = os.path.join(instance_root, 'supervisord.socket')
socket_path = os.path.join(instance_root, 'sv.sock')
if os.path.exists(legacy_socket_path): if os.path.exists(legacy_socket_path):
logger.info("Using legacy supervisor socket path %s", legacy_socket_path) logger.info("Using legacy supervisor socket path %s", legacy_socket_path)
# BBB slapos.core 1.6.1 had an issue that it started a new supervisord using
# `socket_path`, while leaving the supervisord using `legacy_socket_path`
# running with all processes attached.
# When we deployed slapos.core 1.6.1 all servers had two supervisord processes:
# - legacy_socket_path with all processes running
# - socket_path with some processes in EXIT state because ports are already taken.
# In this branch we clean up by stopping supervisord running on socket_path
if os.path.exists(socket_path) and os.access(socket_path, os.R_OK):
logger.critical(
"We have two supervisord running ! "
"Stopping the one using %s to keep using legacy %s instead",
socket_path, legacy_socket_path)
with getSupervisorRPC(socket_path) as supervisor:
supervisor.stopAllProcesses()
supervisor.shutdown()
return legacy_socket_path return legacy_socket_path
return os.path.join(instance_root, 'sv.sock')
return socket_path
def _getSupervisordConfigurationFilePath(instance_root): def _getSupervisordConfigurationFilePath(instance_root):
return os.path.join(instance_root, 'etc', 'supervisord.conf') return os.path.join(instance_root, 'etc', 'supervisord.conf')
......
...@@ -3823,3 +3823,91 @@ For help, use -c -h"""): ...@@ -3823,3 +3823,91 @@ For help, use -c -h"""):
slapos.grid.svcbackend.launchSupervisord(instance_root, logger) slapos.grid.svcbackend.launchSupervisord(instance_root, logger)
logger.debug.assert_called_with('Supervisord already running.') logger.debug.assert_called_with('Supervisord already running.')
def test_stop_second_supervisord_after_supervisor_socket_path_change(self):
"""Check that supervisor uses legacy socket path if available.
We changed the path of supervisor socket from supervisord.socket to sv.sock
but we don't want createSupervisordConfiguration or launchSupervisord to use
the new path if a supervisor is still running at the previous path.
"""
instance_root = tempfile.mkdtemp()
self.addCleanup(shutil.rmtree, instance_root)
logger = mock.create_autospec(logging.Logger)
slapos.grid.svcbackend.createSupervisordConfiguration(instance_root, logger)
supervisord_config_file_path = os.path.join(instance_root, 'etc', 'supervisord.conf')
with open(supervisord_config_file_path) as f:
supervisord_config_file_content = f.read()
self.assertIn('/sv.sock', supervisord_config_file_content)
# change this config to use old socket
with open(supervisord_config_file_path, 'w') as f:
f.write(supervisord_config_file_content.replace('/sv.sock', '/supervisord.socket'))
# start a first supervisor on the old socket
supervisord_process_old_socket = subprocess.Popen(
['supervisord', '--nodaemon', '-c', supervisord_config_file_path],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
)
self.addCleanup(self._stop_process, supervisord_process_old_socket)
# wait for process to be started
supervisord_legacy_socket_path = os.path.join(instance_root, 'supervisord.socket')
for i in range(20):
if os.path.exists(supervisord_legacy_socket_path):
break
time.sleep(.1 * i)
if supervisord_process_old_socket.poll() is not None:
self.fail(supervisord_process_old_socket.communicate())
# change this config to use new socket, as what happens when using slapos node 1.6.1
with open(supervisord_config_file_path, 'w') as f:
f.write(supervisord_config_file_content.replace('/supervisord.socket', '/sv.sock'))
# start a second supervisor on the old socket
supervisord_process_new_socket = subprocess.Popen(
['supervisord', '--nodaemon', '-c', supervisord_config_file_path],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
)
self.addCleanup(self._stop_process, supervisord_process_new_socket)
# wait for process to be started
supervisord_new_socket_path = os.path.join(instance_root, 'sv.sock')
for i in range(20):
if os.path.exists(supervisord_new_socket_path):
break
time.sleep(.1 * i)
if supervisord_process_new_socket.poll() is not None:
self.fail(supervisord_process_new_socket.communicate())
# Now we have two supervisord processes running, like in the bad state created
# by slapos.core 1.6.1
# calling createSupervisordConfiguration should stop the supervisord listening
# on the new socket
slapos.grid.svcbackend.createSupervisordConfiguration(instance_root, logger)
with open(supervisord_config_file_path) as f:
supervisord_config_file_content = f.read()
self.assertIn('/supervisord.socket', supervisord_config_file_content)
self.assertNotIn('/sv.sock', supervisord_config_file_content)
# process on new (sv.sock) will now exit
for i in range(20):
if supervisord_process_new_socket.poll() is not None:
break
time.sleep(.1 * i)
self.assertEqual(
supervisord_process_new_socket.poll(),
0,
supervisord_process_new_socket.communicate())
self.assertFalse(os.path.exists(supervisord_new_socket_path))
# process on old socket is still runnning
if supervisord_process_old_socket.poll() is not None:
self.fail(supervisord_process_old_socket.communicate())
logger.critical.assert_called_with(
'We have two supervisord running ! Stopping the one using %s to keep using legacy %s instead',
supervisord_new_socket_path,
supervisord_legacy_socket_path,
)
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment