Commit 128cc007 authored by Marco Mariani's avatar Marco Mariani

logger refactoring of slapgrid

parent 182878c8
...@@ -28,7 +28,6 @@ ...@@ -28,7 +28,6 @@
# #
############################################################################## ##############################################################################
import logging
import os import os
import pkg_resources import pkg_resources
import pwd import pwd
...@@ -58,7 +57,7 @@ REQUIRED_COMPUTER_PARTITION_PERMISSION = 0o750 ...@@ -58,7 +57,7 @@ REQUIRED_COMPUTER_PARTITION_PERMISSION = 0o750
class Software(object): class Software(object):
"""This class is responsible for installing a software release""" """This class is responsible for installing a software release"""
def __init__(self, url, software_root, buildout, def __init__(self, url, software_root, buildout, logger,
signature_private_key_file=None, signature_certificate_list=None, signature_private_key_file=None, signature_certificate_list=None,
upload_cache_url=None, upload_dir_url=None, shacache_cert_file=None, upload_cache_url=None, upload_dir_url=None, shacache_cert_file=None,
shacache_key_file=None, shadir_cert_file=None, shadir_key_file=None, shacache_key_file=None, shadir_cert_file=None, shadir_key_file=None,
...@@ -75,7 +74,7 @@ class Software(object): ...@@ -75,7 +74,7 @@ class Software(object):
self.software_path = os.path.join(self.software_root, self.software_path = os.path.join(self.software_root,
self.software_url_hash) self.software_url_hash)
self.buildout = buildout self.buildout = buildout
self.logger = logging.getLogger('BuildoutManager') self.logger = logger
self.signature_private_key_file = signature_private_key_file self.signature_private_key_file = signature_private_key_file
self.signature_certificate_list = signature_certificate_list self.signature_certificate_list = signature_certificate_list
self.upload_cache_url = upload_cache_url self.upload_cache_url = upload_cache_url
...@@ -139,7 +138,8 @@ class Software(object): ...@@ -139,7 +138,8 @@ class Software(object):
it. If it fails, we notify the server. it. If it fails, we notify the server.
""" """
root_stat_info = os.stat(self.software_root) root_stat_info = os.stat(self.software_root)
os.environ = getCleanEnvironment(pwd.getpwuid(root_stat_info.st_uid).pw_dir) os.environ = getCleanEnvironment(logger=self.logger,
home_path=pwd.getpwuid(root_stat_info.st_uid).pw_dir)
if not os.path.isdir(self.software_path): if not os.path.isdir(self.software_path):
os.mkdir(self.software_path) os.mkdir(self.software_path)
extends_cache = tempfile.mkdtemp() extends_cache = tempfile.mkdtemp()
...@@ -177,10 +177,13 @@ class Software(object): ...@@ -177,10 +177,13 @@ class Software(object):
self.createProfileIfMissing(buildout_cfg, self.url) self.createProfileIfMissing(buildout_cfg, self.url)
buildout_parameter_list.extend(['-c', buildout_cfg]) buildout_parameter_list.extend(['-c', buildout_cfg])
utils.bootstrapBuildout(self.software_path, self.buildout, utils.bootstrapBuildout(path=self.software_path,
buildout=self.buildout,
logger=self.logger,
additional_buildout_parametr_list=buildout_parameter_list) additional_buildout_parametr_list=buildout_parameter_list)
utils.launchBuildout(self.software_path, utils.launchBuildout(path=self.software_path,
os.path.join(self.software_path, 'bin', 'buildout'), buildout_binary=os.path.join(self.software_path, 'bin', 'buildout'),
logger=self.logger,
additional_buildout_parametr_list=buildout_parameter_list) additional_buildout_parametr_list=buildout_parameter_list)
finally: finally:
shutil.rmtree(extends_cache) shutil.rmtree(extends_cache)
...@@ -256,10 +259,12 @@ class Partition(object): ...@@ -256,10 +259,12 @@ class Partition(object):
server_url, server_url,
software_release_url, software_release_url,
buildout, buildout,
logger,
certificate_repository_path=None, certificate_repository_path=None,
): ):
"""Initialisation of class parameters""" """Initialisation of class parameters"""
self.buildout = buildout self.buildout = buildout
self.logger = logger
self.software_path = software_path self.software_path = software_path
self.instance_path = instance_path self.instance_path = instance_path
self.run_path = os.path.join(self.instance_path, 'etc', 'run') self.run_path = os.path.join(self.instance_path, 'etc', 'run')
...@@ -268,7 +273,6 @@ class Partition(object): ...@@ -268,7 +273,6 @@ class Partition(object):
supervisord_partition_configuration_path supervisord_partition_configuration_path
self.supervisord_socket = supervisord_socket self.supervisord_socket = supervisord_socket
self.computer_partition = computer_partition self.computer_partition = computer_partition
self.logger = logging.getLogger('Partition')
self.computer_id = computer_id self.computer_id = computer_id
self.partition_id = partition_id self.partition_id = partition_id
self.server_url = server_url self.server_url = server_url
...@@ -356,8 +360,8 @@ class Partition(object): ...@@ -356,8 +360,8 @@ class Partition(object):
'permissions are: 0%o, wanted are 0%o' % 'permissions are: 0%o, wanted are 0%o' %
(self.instance_path, permission, (self.instance_path, permission,
REQUIRED_COMPUTER_PARTITION_PERMISSION)) REQUIRED_COMPUTER_PARTITION_PERMISSION))
os.environ = getCleanEnvironment(pwd.getpwuid( os.environ = getCleanEnvironment(logger=self.logger,
instance_stat_info.st_uid).pw_dir) home_path=pwd.getpwuid(instance_stat_info.st_uid).pw_dir)
# Generates buildout part from template # Generates buildout part from template
template_location = os.path.join(self.software_path, 'instance.cfg') template_location = os.path.join(self.software_path, 'instance.cfg')
# Backward compatibility: "instance.cfg" file was named "template.cfg". # Backward compatibility: "instance.cfg" file was named "template.cfg".
...@@ -417,11 +421,13 @@ class Partition(object): ...@@ -417,11 +421,13 @@ class Partition(object):
self.logger.debug('Invoking %r in %r' % (' '.join(invocation_list), self.logger.debug('Invoking %r in %r' % (' '.join(invocation_list),
self.instance_path)) self.instance_path))
process_handler = SlapPopen(invocation_list, process_handler = SlapPopen(invocation_list,
preexec_fn=lambda: dropPrivileges(uid, gid), preexec_fn=lambda: dropPrivileges(uid, gid, logger=self.logger),
cwd=self.instance_path, cwd=self.instance_path,
env=getCleanEnvironment(pwd.getpwuid(uid).pw_dir), env=getCleanEnvironment(logger=self.logger,
home_path=pwd.getpwuid(uid).pw_dir),
stdout=subprocess.PIPE, stdout=subprocess.PIPE,
stderr=subprocess.STDOUT) stderr=subprocess.STDOUT,
logger=self.logger)
if process_handler.returncode is None or process_handler.returncode != 0: if process_handler.returncode is None or process_handler.returncode != 0:
message = 'Failed to bootstrap buildout in %r.' % (self.instance_path) message = 'Failed to bootstrap buildout in %r.' % (self.instance_path)
self.logger.error(message) self.logger.error(message)
...@@ -430,11 +436,17 @@ class Partition(object): ...@@ -430,11 +436,17 @@ class Partition(object):
if not os.path.exists(buildout_binary): if not os.path.exists(buildout_binary):
# use own buildout generation # use own buildout generation
utils.bootstrapBuildout(self.instance_path, self.buildout, utils.bootstrapBuildout(path=self.instance_path,
['buildout:bin-directory=%s' % os.path.join(self.instance_path, 'sbin')]) buildout=self.buildout,
logger=self.logger,
additional_buildout_parameter_list=
['buildout:bin-directory=%s' %
os.path.join(self.instance_path, 'sbin')])
buildout_binary = os.path.join(self.instance_path, 'sbin', 'buildout') buildout_binary = os.path.join(self.instance_path, 'sbin', 'buildout')
# Launches buildout # Launches buildout
utils.launchBuildout(self.instance_path, buildout_binary) utils.launchBuildout(path=self.instance_path,
buildout_binary=buildout_binary,
logger=self.logger)
# Generates supervisord configuration file from template # Generates supervisord configuration file from template
self.logger.info("Generating supervisord config file from template...") self.logger.info("Generating supervisord config file from template...")
# check if CP/etc/run exists and it is a directory # check if CP/etc/run exists and it is a directory
...@@ -510,11 +522,13 @@ class Partition(object): ...@@ -510,11 +522,13 @@ class Partition(object):
uid, gid = self.getUserGroupId() uid, gid = self.getUserGroupId()
self.logger.debug('Invoking %r' % destroy_executable_location) self.logger.debug('Invoking %r' % destroy_executable_location)
process_handler = SlapPopen([destroy_executable_location], process_handler = SlapPopen([destroy_executable_location],
preexec_fn=lambda: dropPrivileges(uid, gid), preexec_fn=lambda: dropPrivileges(uid, gid, logger=self.logger),
cwd=self.instance_path, cwd=self.instance_path,
env=getCleanEnvironment(pwd.getpwuid(uid).pw_dir), env=getCleanEnvironment(logger=self.logger,
home_path=pwd.getpwuid(uid).pw_dir),
stdout=subprocess.PIPE, stdout=subprocess.PIPE,
stderr=subprocess.STDOUT) stderr=subprocess.STDOUT,
logger=self.logger)
if process_handler.returncode is None or process_handler.returncode != 0: if process_handler.returncode is None or process_handler.returncode != 0:
message = 'Failed to destroy Computer Partition in %r.' % \ message = 'Failed to destroy Computer Partition in %r.' % \
self.instance_path self.instance_path
......
This diff is collapsed.
...@@ -46,13 +46,8 @@ def getSupervisorRPC(socket): ...@@ -46,13 +46,8 @@ def getSupervisorRPC(socket):
supervisor_transport) supervisor_transport)
return getattr(server_proxy, 'supervisor') return getattr(server_proxy, 'supervisor')
class dummylogger(object):
def info(self, *args):
print args
debug = info
def launchSupervisord(socket, configuration_file, logger): def launchSupervisord(socket, configuration_file, logger):
#logger = dummylogger()
if os.path.exists(socket): if os.path.exists(socket):
trynum = 1 trynum = 1
while trynum < 6: while trynum < 6:
...@@ -92,7 +87,8 @@ def launchSupervisord(socket, configuration_file, logger): ...@@ -92,7 +87,8 @@ def launchSupervisord(socket, configuration_file, logger):
env={}, env={},
executable=sys.executable, executable=sys.executable,
stdout=subprocess.PIPE, stdout=subprocess.PIPE,
stderr=subprocess.STDOUT) stderr=subprocess.STDOUT,
logger=logger)
result = supervisord_popen.communicate()[0] result = supervisord_popen.communicate()[0]
if supervisord_popen.returncode: if supervisord_popen.returncode:
......
...@@ -30,7 +30,6 @@ ...@@ -30,7 +30,6 @@
import grp import grp
import hashlib import hashlib
import logging
import os import os
import pkg_resources import pkg_resources
import pwd import pwd
...@@ -95,6 +94,7 @@ class SlapPopen(subprocess.Popen): ...@@ -95,6 +94,7 @@ class SlapPopen(subprocess.Popen):
log. log.
""" """
def __init__(self, *args, **kwargs): def __init__(self, *args, **kwargs):
logger = kwargs.pop('logger')
kwargs.update(stdin=subprocess.PIPE) kwargs.update(stdin=subprocess.PIPE)
if sys.platform == 'cygwin' and kwargs.get('env') == {}: if sys.platform == 'cygwin' and kwargs.get('env') == {}:
kwargs['env'] = None kwargs['env'] = None
...@@ -103,7 +103,6 @@ class SlapPopen(subprocess.Popen): ...@@ -103,7 +103,6 @@ class SlapPopen(subprocess.Popen):
self.stdin.close() self.stdin.close()
self.stdin = None self.stdin = None
logger = logging.getLogger('SlapProcessManager')
# XXX-Cedric: this algorithm looks overkill for simple logging. # XXX-Cedric: this algorithm looks overkill for simple logging.
output_lines = [] output_lines = []
while True: while True:
...@@ -119,8 +118,7 @@ def md5digest(url): ...@@ -119,8 +118,7 @@ def md5digest(url):
return hashlib.md5(url).hexdigest() return hashlib.md5(url).hexdigest()
def getCleanEnvironment(home_path='/tmp'): def getCleanEnvironment(logger, home_path='/tmp'):
logger = logging.getLogger('CleanEnvironment')
changed_env = {} changed_env = {}
removed_env = [] removed_env = []
env = os.environ.copy() env = os.environ.copy()
...@@ -137,12 +135,11 @@ def getCleanEnvironment(home_path='/tmp'): ...@@ -137,12 +135,11 @@ def getCleanEnvironment(home_path='/tmp'):
return env return env
def setRunning(pid_file): def setRunning(logger, pidfile):
"""Creates a pidfile. If a pidfile already exists, we exit""" """Creates a pidfile. If a pidfile already exists, we exit"""
logger = logging.getLogger('Slapgrid') if os.path.exists(pidfile):
if os.path.exists(pid_file):
try: try:
pid = int(open(pid_file, 'r').readline()) pid = int(open(pidfile, 'r').readline())
except ValueError: except ValueError:
pid = None pid = None
# XXX This could use psutil library. # XXX This could use psutil library.
...@@ -150,29 +147,28 @@ def setRunning(pid_file): ...@@ -150,29 +147,28 @@ def setRunning(pid_file):
logger.info('New slapos process started, but another slapos ' logger.info('New slapos process started, but another slapos '
'process is aleady running with pid %s, exiting.' % pid) 'process is aleady running with pid %s, exiting.' % pid)
sys.exit(10) sys.exit(10)
logger.info('Existing pid file %r was stale, overwritten' % pid_file) logger.info('Existing pid file %r was stale, overwritten' % pidfile)
# Start new process # Start new process
write_pid(pid_file) write_pid(logger, pidfile)
def setFinished(pid_file): def setFinished(pidfile):
try: try:
os.remove(pid_file) os.remove(pidfile)
except OSError: except OSError:
pass pass
def write_pid(pid_file): def write_pid(logger, pidfile):
logger = logging.getLogger('Slapgrid')
try: try:
with open(pid_file, 'w') as fout: with open(pidfile, 'w') as fout:
fout.write('%s' % os.getpid()) fout.write('%s' % os.getpid())
except (IOError, OSError): except (IOError, OSError):
logger.critical('slapgrid could not write pidfile %s' % pid_file) logger.critical('slapgrid could not write pidfile %s' % pidfile)
raise raise
def dropPrivileges(uid, gid): def dropPrivileges(uid, gid, logger):
"""Drop privileges to uid, gid if current uid is 0 """Drop privileges to uid, gid if current uid is 0
Do tests to check if dropping was successful and that no system call is able Do tests to check if dropping was successful and that no system call is able
...@@ -180,7 +176,6 @@ def dropPrivileges(uid, gid): ...@@ -180,7 +176,6 @@ def dropPrivileges(uid, gid):
Does nothing in case if uid and gid are not 0 Does nothing in case if uid and gid are not 0
""" """
logger = logging.getLogger('dropPrivileges')
# XXX-Cedric: remove format / just do a print, otherwise formatting is done # XXX-Cedric: remove format / just do a print, otherwise formatting is done
# twice # twice
current_uid, current_gid = os.getuid(), os.getgid() current_uid, current_gid = os.getuid(), os.getgid()
...@@ -232,11 +227,10 @@ def dropPrivileges(uid, gid): ...@@ -232,11 +227,10 @@ def dropPrivileges(uid, gid):
logger.debug('Succesfully dropped privileges to uid=%r gid=%r' % (uid, gid)) logger.debug('Succesfully dropped privileges to uid=%r gid=%r' % (uid, gid))
def bootstrapBuildout(path, buildout=None, def bootstrapBuildout(path, logger, buildout=None,
additional_buildout_parametr_list=None): additional_buildout_parametr_list=None):
if additional_buildout_parametr_list is None: if additional_buildout_parametr_list is None:
additional_buildout_parametr_list = [] additional_buildout_parametr_list = []
logger = logging.getLogger('BuildoutManager')
# Reads uid/gid of path, launches buildout with thoses privileges # Reads uid/gid of path, launches buildout with thoses privileges
stat_info = os.stat(path) stat_info = os.stat(path)
uid = stat_info.st_uid uid = stat_info.st_uid
...@@ -270,10 +264,11 @@ def bootstrapBuildout(path, buildout=None, ...@@ -270,10 +264,11 @@ def bootstrapBuildout(path, buildout=None,
logger.debug('Invoking: %r in directory %r' % (' '.join(invocation_list), logger.debug('Invoking: %r in directory %r' % (' '.join(invocation_list),
path)) path))
process_handler = SlapPopen(invocation_list, process_handler = SlapPopen(invocation_list,
preexec_fn=lambda: dropPrivileges(uid, gid), preexec_fn=lambda: dropPrivileges(uid, gid, logger=logger),
cwd=path, cwd=path,
stdout=subprocess.PIPE, stdout=subprocess.PIPE,
stderr=subprocess.STDOUT) stderr=subprocess.STDOUT,
logger=logger)
if process_handler.returncode is None or process_handler.returncode != 0: if process_handler.returncode is None or process_handler.returncode != 0:
message = 'Failed to run buildout profile in directory %r' % (path) message = 'Failed to run buildout profile in directory %r' % (path)
logger.error(message) logger.error(message)
...@@ -285,10 +280,9 @@ def bootstrapBuildout(path, buildout=None, ...@@ -285,10 +280,9 @@ def bootstrapBuildout(path, buildout=None,
logger.debug('Restore umask from %03o to %03o' % (old_umask, umask)) logger.debug('Restore umask from %03o to %03o' % (old_umask, umask))
def launchBuildout(path, buildout_binary, def launchBuildout(path, buildout_binary, logger,
additional_buildout_parametr_list=None): additional_buildout_parametr_list=None):
""" Launches buildout.""" """ Launches buildout."""
logger = logging.getLogger('BuildoutManager')
if additional_buildout_parametr_list is None: if additional_buildout_parametr_list is None:
additional_buildout_parametr_list = [] additional_buildout_parametr_list = []
# Reads uid/gid of path, launches buildout with thoses privileges # Reads uid/gid of path, launches buildout with thoses privileges
...@@ -311,11 +305,13 @@ def launchBuildout(path, buildout_binary, ...@@ -311,11 +305,13 @@ def launchBuildout(path, buildout_binary,
logger.debug('Invoking: %r in directory %r' % (' '.join(invocation_list), logger.debug('Invoking: %r in directory %r' % (' '.join(invocation_list),
path)) path))
process_handler = SlapPopen(invocation_list, process_handler = SlapPopen(invocation_list,
preexec_fn=lambda: dropPrivileges(uid, gid), preexec_fn=lambda: dropPrivileges(uid, gid, logger=logger),
cwd=path, cwd=path,
env=getCleanEnvironment(pwd.getpwuid(uid).pw_dir), env=getCleanEnvironment(logger=logger,
home_path=pwd.getpwuid(uid).pw_dir),
stdout=subprocess.PIPE, stdout=subprocess.PIPE,
stderr=subprocess.STDOUT) stderr=subprocess.STDOUT,
logger=logger)
if process_handler.returncode is None or process_handler.returncode != 0: if process_handler.returncode is None or process_handler.returncode != 0:
message = 'Failed to run buildout profile in directory %r' % (path) message = 'Failed to run buildout profile in directory %r' % (path)
logger.error(message) logger.error(message)
......
...@@ -117,10 +117,15 @@ class BasicMixin: ...@@ -117,10 +117,15 @@ class BasicMixin:
'supervisord') 'supervisord')
self.usage_report_periodicity = 1 self.usage_report_periodicity = 1
self.buildout = None self.buildout = None
self.grid = slapgrid.Slapgrid(self.software_root, self.instance_root, self.grid = slapgrid.Slapgrid(self.software_root,
self.master_url, self.computer_id, self.supervisord_socket, self.instance_root,
self.master_url,
self.computer_id,
self.supervisord_socket,
self.supervisord_configuration_path, self.supervisord_configuration_path,
self.buildout, develop=develop) self.buildout,
develop=develop,
logger=logging.getLogger())
# monkey patch buildout bootstrap # monkey patch buildout bootstrap
def dummy(*args, **kw): def dummy(*args, **kw):
pass pass
...@@ -1826,7 +1831,7 @@ exit 127""" % {'worked_file': worked_file}) ...@@ -1826,7 +1831,7 @@ exit 127""" % {'worked_file': worked_file})
slapos.grid.slapgrid.SLAPGRID_PROMISE_FAIL) slapos.grid.slapgrid.SLAPGRID_PROMISE_FAIL)
self.assertTrue(os.path.isfile(worked_file)) self.assertTrue(os.path.isfile(worked_file))
self.assertEqual(instance.error_log, 'Error') self.assertEqual(instance.error_log.split('\n')[-1], 'Error')
self.assertTrue(instance.error) self.assertTrue(instance.error)
self.assertIsNone(instance.state) self.assertIsNone(instance.state)
......
...@@ -25,12 +25,14 @@ ...@@ -25,12 +25,14 @@
# #
############################################################################## ##############################################################################
import logging
import os
import unittest
from slapos.grid import SlapObject from slapos.grid import SlapObject
from slapos.grid import utils from slapos.grid import utils
from slapos.grid import networkcache from slapos.grid import networkcache
from slapos.tests.slapgrid import BasicMixin from slapos.tests.slapgrid import BasicMixin
import os
import unittest
class FakeCallAndRead: class FakeCallAndRead:
...@@ -93,6 +95,7 @@ class TestSoftwareSlapObject(BasicMixin, unittest.TestCase): ...@@ -93,6 +95,7 @@ class TestSoftwareSlapObject(BasicMixin, unittest.TestCase):
url='http://example.com/software.cfg', url='http://example.com/software.cfg',
software_root=self.software_root, software_root=self.software_root,
buildout=self.buildout, buildout=self.buildout,
logger=logging.getLogger(),
signature_private_key_file='/signature/private/key_file', signature_private_key_file='/signature/private/key_file',
upload_cache_url='http://example.com/uploadcache', upload_cache_url='http://example.com/uploadcache',
upload_dir_url='http://example.com/uploaddir', upload_dir_url='http://example.com/uploaddir',
...@@ -129,7 +132,8 @@ class TestSoftwareSlapObject(BasicMixin, unittest.TestCase): ...@@ -129,7 +132,8 @@ class TestSoftwareSlapObject(BasicMixin, unittest.TestCase):
software = SlapObject.Software( software = SlapObject.Software(
url='http://example.com/software.cfg', url='http://example.com/software.cfg',
software_root=self.software_root, software_root=self.software_root,
buildout=self.buildout) buildout=self.buildout,
logger=logging.getLogger())
software.install() software.install()
...@@ -161,6 +165,7 @@ class TestSoftwareSlapObject(BasicMixin, unittest.TestCase): ...@@ -161,6 +165,7 @@ class TestSoftwareSlapObject(BasicMixin, unittest.TestCase):
url='http://example.com/software.cfg', url='http://example.com/software.cfg',
software_root=self.software_root, software_root=self.software_root,
buildout=self.buildout, buildout=self.buildout,
logger=logging.getLogger(),
signature_private_key_file='/signature/private/key_file', signature_private_key_file='/signature/private/key_file',
upload_cache_url='http://example.com/uploadcache', upload_cache_url='http://example.com/uploadcache',
upload_dir_url='http://example.com/uploaddir', upload_dir_url='http://example.com/uploaddir',
...@@ -192,6 +197,7 @@ class TestSoftwareSlapObject(BasicMixin, unittest.TestCase): ...@@ -192,6 +197,7 @@ class TestSoftwareSlapObject(BasicMixin, unittest.TestCase):
url='http://example.com/software.cfg', url='http://example.com/software.cfg',
software_root=self.software_root, software_root=self.software_root,
buildout=self.buildout, buildout=self.buildout,
logger=logging.getLogger(),
signature_private_key_file='/signature/private/key_file', signature_private_key_file='/signature/private/key_file',
upload_cache_url='http://example.com/uploadcache', upload_cache_url='http://example.com/uploadcache',
upload_dir_url='http://example.com/uploaddir', upload_dir_url='http://example.com/uploaddir',
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment