svcbackend.py 7.93 KB
Newer Older
1
# -*- coding: utf-8 -*-
Marco Mariani's avatar
Marco Mariani committed
2
# vim: set et sts=2:
Łukasz Nowak's avatar
Łukasz Nowak committed
3 4
##############################################################################
#
5 6
# Copyright (c) 2010, 2011, 2012 Vifib SARL and Contributors.
# All Rights Reserved.
Łukasz Nowak's avatar
Łukasz Nowak committed
7 8 9 10 11
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
12
# guarantees and support are strongly advised to contract a Free Software
Łukasz Nowak's avatar
Łukasz Nowak committed
13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
#
##############################################################################

import os
32 33 34 35
import pkg_resources
import socket as socketlib
import subprocess
import stat
Łukasz Nowak's avatar
Łukasz Nowak committed
36
import sys
37
import time
Łukasz Nowak's avatar
Łukasz Nowak committed
38
import xmlrpclib
39 40

from slapos.grid.utils import (createPrivateDirectory, SlapPopen, updateFile)
Łukasz Nowak's avatar
Łukasz Nowak committed
41

42
from supervisor import xmlrpc, states
43

Łukasz Nowak's avatar
Łukasz Nowak committed
44 45 46 47 48 49 50 51 52

def getSupervisorRPC(socket):
  supervisor_transport = xmlrpc.SupervisorTransport('', '',
      'unix://' + socket)
  server_proxy = xmlrpclib.ServerProxy('http://127.0.0.1',
      supervisor_transport)
  return getattr(server_proxy, 'supervisor')


53 54 55 56 57 58 59 60 61
def _getSupervisordSocketPath(instance_root):
  return os.path.join(instance_root, 'supervisord.socket')

def _getSupervisordConfigurationFilePath(instance_root):
  return os.path.join(instance_root, 'etc', 'supervisord.conf')

def _getSupervisordConfigurationDirectory(instance_root):
  return os.path.join(instance_root, 'etc', 'supervisord.conf.d')

62
def createSupervisordConfiguration(instance_root, watchdog_command=''):
63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107
  """
  Create supervisord related files and directories.
  """
  if not os.path.isdir(instance_root):
    raise OSError('%s does not exist.' % instance_root)

  supervisord_configuration_file_path = _getSupervisordConfigurationFilePath(instance_root)
  supervisord_configuration_directory = _getSupervisordConfigurationDirectory(instance_root)
  supervisord_socket = _getSupervisordSocketPath(instance_root)

  # Create directory accessible for the instances.
  var_directory = os.path.join(instance_root, 'var')
  if not os.path.isdir(var_directory):
    os.mkdir(var_directory)
  os.chmod(var_directory, stat.S_IRWXU | stat.S_IROTH | stat.S_IXOTH | \
                          stat.S_IRGRP | stat.S_IXGRP )
  etc_directory = os.path.join(instance_root, 'etc')
  if not os.path.isdir(etc_directory):
    os.mkdir(etc_directory)

  # Creates instance_root structure
  createPrivateDirectory(os.path.join(instance_root, 'var', 'log'))
  createPrivateDirectory(os.path.join(instance_root, 'var', 'run'))

  createPrivateDirectory(os.path.join(instance_root, 'etc'))
  createPrivateDirectory(supervisord_configuration_directory)

  # Creates supervisord configuration
  updateFile(supervisord_configuration_file_path,
    pkg_resources.resource_stream(__name__,
      'templates/supervisord.conf.in').read() % {
          'supervisord_configuration_directory': supervisord_configuration_directory,
          'supervisord_socket': os.path.abspath(supervisord_socket),
          'supervisord_loglevel': 'info',
          'supervisord_logfile': os.path.abspath(
              os.path.join(instance_root, 'var', 'log', 'supervisord.log')),
          'supervisord_logfile_maxbytes': '50MB',
          'supervisord_nodaemon': 'false',
          'supervisord_pidfile': os.path.abspath(
              os.path.join(instance_root, 'var', 'run', 'supervisord.pid')),
          'supervisord_logfile_backups': '10',
          'watchdog_command': watchdog_command,
      }
  )

108 109 110 111 112 113 114 115 116 117 118 119 120 121 122
def _updateWatchdog(socket):
  """
  In special cases, supervisord can be started using configuration
  with empty watchdog parameter.
  Then, when running slapgrid, the real watchdog configuration is generated.
  We thus need to reload watchdog configuration if needed and start it.
  """
  supervisor = getSupervisorRPC(socket)
  if supervisor.getProcessInfo('watchdog')['state'] not in states.RUNNING_STATES:
    # XXX workaround for https://github.com/Supervisor/supervisor/issues/339
    # In theory, only reloadConfig is needed.
    supervisor.removeProcessGroup('watchdog')
    supervisor.reloadConfig()
    supervisor.addProcessGroup('watchdog')

123 124 125 126
def launchSupervisord(instance_root, logger,
                      supervisord_additional_argument_list=None):
  configuration_file = _getSupervisordConfigurationFilePath(instance_root)
  socket = _getSupervisordSocketPath(instance_root)
Łukasz Nowak's avatar
Łukasz Nowak committed
127
  if os.path.exists(socket):
128 129 130
    trynum = 1
    while trynum < 6:
      try:
131
        supervisor = getSupervisorRPC(socket)
132
        status = supervisor.getState()
Marco Mariani's avatar
Marco Mariani committed
133
      except xmlrpclib.Fault as e:
134 135 136 137
        if e.faultCode == 6 and e.faultString == 'SHUTDOWN_STATE':
          logger.info('Supervisor in shutdown procedure, will check again later.')
          trynum += 1
          time.sleep(2 * trynum)
138
      except Exception:
139 140 141
        # In case if there is problem with connection, assume that supervisord
        # is not running and try to run it
        break
Łukasz Nowak's avatar
Łukasz Nowak committed
142
      else:
143
        if status['statename'] == 'RUNNING' and status['statecode'] == 1:
144
          logger.debug('Supervisord already running.')
145
          _updateWatchdog(socket)
146 147 148 149 150 151 152 153 154
          return
        elif status['statename'] == 'SHUTDOWN_STATE' and status['statecode'] == 6:
          logger.info('Supervisor in shutdown procedure, will check again later.')
          trynum += 1
          time.sleep(2 * trynum)
        else:
          log_message = 'Unknown supervisord state %r. Will try to start.' % status
          logger.warning(log_message)
          break
Łukasz Nowak's avatar
Łukasz Nowak committed
155

156 157 158 159
  supervisord_argument_list = ['-c', configuration_file]
  if supervisord_additional_argument_list is not None:
    supervisord_argument_list.extend(supervisord_additional_argument_list)

Łukasz Nowak's avatar
Łukasz Nowak committed
160 161
  logger.info("Launching supervisord with clean environment.")
  # Extract python binary to prevent shebang size limit
162
  invocation_list = [sys.executable, '-c']
163 164 165 166 167
  invocation_list.append(
      "import sys ; sys.path=" + str(sys.path) + " ; " +
      "import supervisor.supervisord ; " +
      "sys.argv[1:1]=" + str(supervisord_argument_list) + " ; " +
      "supervisor.supervisord.main()")
Łukasz Nowak's avatar
Łukasz Nowak committed
168
  supervisord_popen = SlapPopen(invocation_list,
Marco Mariani's avatar
Marco Mariani committed
169 170
                                env={},
                                stdout=subprocess.PIPE,
171 172
                                stderr=subprocess.STDOUT,
                                logger=logger)
Marco Mariani's avatar
Marco Mariani committed
173

174 175 176
  result = supervisord_popen.communicate()[0]
  if supervisord_popen.returncode:
    logger.warning('Supervisord unknown problem: %s' % result)
177
    raise RuntimeError('Failed to launch supervisord : %s' % result)
Marco Mariani's avatar
Marco Mariani committed
178

179 180 181 182 183 184 185 186 187 188 189 190 191 192
  try:
    default_timeout = socketlib.getdefaulttimeout()
    current_timeout = 1
    trynum = 1
    while trynum < 6:
      try:
        socketlib.setdefaulttimeout(current_timeout)
        supervisor = getSupervisorRPC(socket)
        status = supervisor.getState()
        if status['statename'] == 'RUNNING' and status['statecode'] == 1:
          return
        logger.warning('Wrong status name %(statename)r and code '
          '%(statecode)r, trying again' % status)
        trynum += 1
193
      except Exception:
194 195 196 197 198 199 200 201
        current_timeout = 5 * trynum
        trynum += 1
      else:
        logger.info('Supervisord started correctly in try %s.' % trynum)
        return
    logger.warning('Issue while checking supervisord.')
  finally:
    socketlib.setdefaulttimeout(default_timeout)
202