Always log into stdout. ALWAYS.

In order to do that, always use logger, and always in real time.
Logger is properly configured.
Then remove -c switch (kept in parameters but doesn't do anything).
Also factor code of whole slapos/grid directory about SlapPopen (now does the
log).
parent 84759eb9
...@@ -50,7 +50,7 @@ REQUIRED_COMPUTER_PARTITION_PERMISSION = '0750' ...@@ -50,7 +50,7 @@ REQUIRED_COMPUTER_PARTITION_PERMISSION = '0750'
class Software(object): class Software(object):
"""This class is responsible of installing a software release""" """This class is responsible of installing a software release"""
def __init__(self, url, software_root, console, buildout, def __init__(self, url, software_root, buildout,
signature_private_key_file=None, signature_certificate_list=None, signature_private_key_file=None, signature_certificate_list=None,
upload_cache_url=None, upload_dir_url=None, shacache_cert_file=None, upload_cache_url=None, upload_dir_url=None, shacache_cert_file=None,
shacache_key_file=None, shadir_cert_file=None, shadir_key_file=None, shacache_key_file=None, shadir_cert_file=None, shadir_key_file=None,
...@@ -67,7 +67,6 @@ class Software(object): ...@@ -67,7 +67,6 @@ class Software(object):
self.software_url_hash) self.software_url_hash)
self.buildout = buildout self.buildout = buildout
self.logger = logging.getLogger('BuildoutManager') self.logger = logging.getLogger('BuildoutManager')
self.console = console
self.signature_private_key_file = signature_private_key_file self.signature_private_key_file = signature_private_key_file
self.signature_certificate_list = signature_certificate_list self.signature_certificate_list = signature_certificate_list
self.upload_cache_url = upload_cache_url self.upload_cache_url = upload_cache_url
...@@ -185,12 +184,10 @@ class Software(object): ...@@ -185,12 +184,10 @@ class Software(object):
buildout_parameter_list.extend(['-c', self.url]) buildout_parameter_list.extend(['-c', self.url])
utils.bootstrapBuildout(self.software_path, self.buildout, utils.bootstrapBuildout(self.software_path, self.buildout,
additional_buildout_parametr_list=buildout_parameter_list, additional_buildout_parametr_list=buildout_parameter_list)
console=self.console)
utils.launchBuildout(self.software_path, utils.launchBuildout(self.software_path,
os.path.join(self.software_path, 'bin', 'buildout'), os.path.join(self.software_path, 'bin', 'buildout'),
additional_buildout_parametr_list=buildout_parameter_list, additional_buildout_parametr_list=buildout_parameter_list)
console=self.console)
finally: finally:
shutil.rmtree(extends_cache) shutil.rmtree(extends_cache)
...@@ -231,7 +228,6 @@ class Partition(object): ...@@ -231,7 +228,6 @@ class Partition(object):
software_release_url, software_release_url,
buildout, buildout,
certificate_repository_path=None, certificate_repository_path=None,
console=False
): ):
"""Initialisation of class parameters""" """Initialisation of class parameters"""
self.buildout = buildout self.buildout = buildout
...@@ -248,7 +244,6 @@ class Partition(object): ...@@ -248,7 +244,6 @@ class Partition(object):
self.partition_id = partition_id self.partition_id = partition_id
self.server_url = server_url self.server_url = server_url
self.software_release_url = software_release_url self.software_release_url = software_release_url
self.console = console
self.key_file = '' self.key_file = ''
self.cert_file = '' self.cert_file = ''
...@@ -378,18 +373,12 @@ class Partition(object): ...@@ -378,18 +373,12 @@ class Partition(object):
invocation_list.append(bootstrap_file) invocation_list.append(bootstrap_file)
self.logger.debug('Invoking %r in %r' % (' '.join(invocation_list), self.logger.debug('Invoking %r in %r' % (' '.join(invocation_list),
self.instance_path)) self.instance_path))
kw = dict() kw = dict(stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
if not self.console:
kw.update(stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
process_handler = utils.SlapPopen(invocation_list, process_handler = utils.SlapPopen(invocation_list,
preexec_fn=lambda: utils.dropPrivileges(uid, gid), cwd=self.instance_path, preexec_fn=lambda: utils.dropPrivileges(uid, gid), cwd=self.instance_path,
env=utils.getCleanEnvironment(pwd.getpwuid(uid).pw_dir), **kw) env=utils.getCleanEnvironment(pwd.getpwuid(uid).pw_dir), **kw)
result_std = process_handler.communicate()[0]
if self.console:
result_std = 'Please consult messages above.'
if process_handler.returncode is None or process_handler.returncode != 0: if process_handler.returncode is None or process_handler.returncode != 0:
message = 'Failed to bootstrap buildout in %r:\n%s\n' % ( message = 'Failed to bootstrap buildout in %r.' % (self.instance_path)
self.instance_path, result_std)
raise BuildoutFailedError(message) raise BuildoutFailedError(message)
buildout_binary = os.path.join(self.instance_path, 'sbin', 'buildout') buildout_binary = os.path.join(self.instance_path, 'sbin', 'buildout')
...@@ -397,11 +386,10 @@ class Partition(object): ...@@ -397,11 +386,10 @@ class Partition(object):
# use own buildout generation # use own buildout generation
utils.bootstrapBuildout(self.instance_path, self.buildout, utils.bootstrapBuildout(self.instance_path, self.buildout,
['buildout:bin-directory=%s'% os.path.join(self.instance_path, ['buildout:bin-directory=%s'% os.path.join(self.instance_path,
'sbin')], console=self.console) 'sbin')])
buildout_binary = os.path.join(self.instance_path, 'sbin', 'buildout') buildout_binary = os.path.join(self.instance_path, 'sbin', 'buildout')
# Launches buildout # Launches buildout
utils.launchBuildout(self.instance_path, utils.launchBuildout(self.instance_path, buildout_binary)
buildout_binary, console=self.console)
# Generates supervisord configuration file from template # Generates supervisord configuration file from template
self.logger.info("Generating supervisord config file from template...") self.logger.info("Generating supervisord config file from template...")
# check if CP/etc/run exists and it is a directory # check if CP/etc/run exists and it is a directory
...@@ -484,17 +472,13 @@ class Partition(object): ...@@ -484,17 +472,13 @@ class Partition(object):
gid = stat_info.st_gid gid = stat_info.st_gid
self.logger.debug('Invoking %r' % destroy_executable_location) self.logger.debug('Invoking %r' % destroy_executable_location)
kw = dict() kw = dict()
if not self.console: kw.update(stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
kw.update(stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
process_handler = utils.SlapPopen([destroy_executable_location], process_handler = utils.SlapPopen([destroy_executable_location],
preexec_fn=lambda: utils.dropPrivileges(uid, gid), cwd=self.instance_path, preexec_fn=lambda: utils.dropPrivileges(uid, gid), cwd=self.instance_path,
env=utils.getCleanEnvironment(pwd.getpwuid(uid).pw_dir), **kw) env=utils.getCleanEnvironment(pwd.getpwuid(uid).pw_dir), **kw)
result_std = process_handler.communicate()[0]
if self.console:
result_std = 'Please consult messages above'
if process_handler.returncode is None or process_handler.returncode != 0: if process_handler.returncode is None or process_handler.returncode != 0:
message = 'Failed to destroy Computer Partition in %r:\n%s\n' % ( message = 'Failed to destroy Computer Partition in %r.' % \
self.instance_path, result_std) self.instance_path
raise subprocess.CalledProcessError(message) raise subprocess.CalledProcessError(message)
# Manually cleans what remains # Manually cleans what remains
try: try:
......
...@@ -108,7 +108,7 @@ def parseArgumentTupleAndReturnSlapgridObject(*argument_tuple): ...@@ -108,7 +108,7 @@ def parseArgumentTupleAndReturnSlapgridObject(*argument_tuple):
parser.add_argument("--certificate_repository_path", parser.add_argument("--certificate_repository_path",
help="Path to directory where downloaded certificates would be stored.") help="Path to directory where downloaded certificates would be stored.")
parser.add_argument("-c", "--console", action="store_true", default=False, parser.add_argument("-c", "--console", action="store_true", default=False,
help="Enables console output and live output from subcommands.") help="Deprecated, doesn't do anything.")
parser.add_argument("-v", "--verbose", action="store_true", default=False, parser.add_argument("-v", "--verbose", action="store_true", default=False,
help="Be verbose.") help="Be verbose.")
parser.add_argument("--maximum-periodicity", type=int, default=None, parser.add_argument("--maximum-periodicity", type=int, default=None,
...@@ -158,16 +158,20 @@ def parseArgumentTupleAndReturnSlapgridObject(*argument_tuple): ...@@ -158,16 +158,20 @@ def parseArgumentTupleAndReturnSlapgridObject(*argument_tuple):
if argument_value is not None: if argument_value is not None:
option_dict.update({argument_key: argument_value}) option_dict.update({argument_key: argument_value})
# Configures logger. # Configures logger.
logger_format = '%(asctime)s %(name)-18s: %(levelname)-8s %(message)s'
if option_dict['verbose']: if option_dict['verbose']:
level = logging.DEBUG level = logging.DEBUG
else: else:
level = logging.INFO level = logging.INFO
logging.basicConfig(format='%(asctime)s %(levelname)-8s %(message)s',
level=level,
datefmt='%Y-%m-%dT%H:%M:%S')
if option_dict.get('logfile'): if option_dict.get('logfile'):
logging.basicConfig(filename=option_dict['logfile'], console = logging.FileHandler(option_dict['logfile'])
format=logger_format, level=level) console.setLevel(level)
if option_dict['console']: console.setFormatter(logging.Formatter(
logging.basicConfig(level=level) '%(asctime)s %(name)-18s: %(levelname)-8s %(message)s'))
logging.getLogger('').addHandler(console)
missing_mandatory_parameter_list = [] missing_mandatory_parameter_list = []
for mandatory_parameter in MANDATORY_PARAMETER_LIST: for mandatory_parameter in MANDATORY_PARAMETER_LIST:
if not mandatory_parameter in option_dict: if not mandatory_parameter in option_dict:
...@@ -294,7 +298,6 @@ def parseArgumentTupleAndReturnSlapgridObject(*argument_tuple): ...@@ -294,7 +298,6 @@ def parseArgumentTupleAndReturnSlapgridObject(*argument_tuple):
upload_binary_dir_url=\ upload_binary_dir_url=\
option_dict.get('upload-binary-dir-url', None), option_dict.get('upload-binary-dir-url', None),
upload_dir_url=option_dict.get('upload-dir-url', None), upload_dir_url=option_dict.get('upload-dir-url', None),
console=option_dict['console'],
buildout=option_dict.get('buildout'), buildout=option_dict.get('buildout'),
promise_timeout=option_dict['promise_timeout'], promise_timeout=option_dict['promise_timeout'],
shacache_cert_file=option_dict.get('shacache-cert-file', None), shacache_cert_file=option_dict.get('shacache-cert-file', None),
...@@ -390,7 +393,6 @@ class Slapgrid(object): ...@@ -390,7 +393,6 @@ class Slapgrid(object):
upload_dir_url=None, upload_dir_url=None,
master_ca_file=None, master_ca_file=None,
certificate_repository_path=None, certificate_repository_path=None,
console=False,
promise_timeout=3, promise_timeout=3,
shacache_cert_file=None, shacache_cert_file=None,
shacache_key_file=None, shacache_key_file=None,
...@@ -439,7 +441,6 @@ class Slapgrid(object): ...@@ -439,7 +441,6 @@ class Slapgrid(object):
self.instance_etc_directory = os.path.join(self.instance_root, 'etc') self.instance_etc_directory = os.path.join(self.instance_root, 'etc')
self.supervisord_configuration_directory = \ self.supervisord_configuration_directory = \
os.path.join(self.instance_etc_directory, 'supervisord.conf.d') os.path.join(self.instance_etc_directory, 'supervisord.conf.d')
self.console = console
self.buildout = buildout self.buildout = buildout
self.promise_timeout = promise_timeout self.promise_timeout = promise_timeout
self.develop = develop self.develop = develop
...@@ -526,7 +527,7 @@ class Slapgrid(object): ...@@ -526,7 +527,7 @@ class Slapgrid(object):
software_path = os.path.join(self.software_root, url_hash) software_path = os.path.join(self.software_root, url_hash)
software = Software(url=software_release_uri, software = Software(url=software_release_uri,
software_root=self.software_root, software_root=self.software_root,
console=self.console, buildout=self.buildout, buildout=self.buildout,
signature_private_key_file=self.signature_private_key_file, signature_private_key_file=self.signature_private_key_file,
signature_certificate_list=self.signature_certificate_list, signature_certificate_list=self.signature_certificate_list,
download_binary_cache_url=self.download_binary_cache_url, download_binary_cache_url=self.download_binary_cache_url,
...@@ -730,7 +731,7 @@ class Slapgrid(object): ...@@ -730,7 +731,7 @@ class Slapgrid(object):
server_url=self.master_url, server_url=self.master_url,
software_release_url=software_url, software_release_url=software_url,
certificate_repository_path=self.certificate_repository_path, certificate_repository_path=self.certificate_repository_path,
console=self.console, buildout=self.buildout) buildout=self.buildout)
computer_partition_state = computer_partition.getState() computer_partition_state = computer_partition.getState()
if computer_partition_state == "started": if computer_partition_state == "started":
...@@ -972,24 +973,17 @@ class Slapgrid(object): ...@@ -972,24 +973,17 @@ class Slapgrid(object):
#stat sys call to get statistics informations #stat sys call to get statistics informations
uid = stat_info.st_uid uid = stat_info.st_uid
gid = stat_info.st_gid gid = stat_info.st_gid
kw = dict() kw = dict(stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
if not self.console:
kw.update(stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
process_handler = SlapPopen(invocation_list, process_handler = SlapPopen(invocation_list,
preexec_fn=lambda: dropPrivileges(uid, gid), preexec_fn=lambda: dropPrivileges(uid, gid),
cwd=os.path.join(instance_path, 'etc', 'report'), cwd=os.path.join(instance_path, 'etc', 'report'),
env=None, **kw) env=None, **kw)
result = process_handler.communicate()[0]
if self.console:
result = 'Please consult messages above'
if process_handler.returncode is None: if process_handler.returncode is None:
process_handler.kill() process_handler.kill()
if process_handler.returncode != 0: if process_handler.returncode != 0:
clean_run = False clean_run = False
failed_script_list.append("Script %r failed with %s." % (script, failed_script_list.append("Script %r failed." % script)
result)) logger.warning("Failed to run %r" % invocation_list)
logger.warning("Failed to run %r, the result was. \n%s" %
(invocation_list, result))
if len(failed_script_list): if len(failed_script_list):
computer_partition.error('\n'.join(failed_script_list)) computer_partition.error('\n'.join(failed_script_list))
# Whatever happens, don't stop processing other instances # Whatever happens, don't stop processing other instances
...@@ -1102,7 +1096,7 @@ class Slapgrid(object): ...@@ -1102,7 +1096,7 @@ class Slapgrid(object):
server_url=self.master_url, server_url=self.master_url,
software_release_url=software_url, software_release_url=software_url,
certificate_repository_path=self.certificate_repository_path, certificate_repository_path=self.certificate_repository_path,
console=self.console, buildout=self.buildout buildout=self.buildout,
) )
local_partition.stop() local_partition.stop()
try: try:
......
...@@ -90,7 +90,9 @@ class AlreadyRunning(Exception): ...@@ -90,7 +90,9 @@ class AlreadyRunning(Exception):
class SlapPopen(subprocess.Popen): class SlapPopen(subprocess.Popen):
"""Almost normal subprocess with gridish features""" """
Almost normal subprocess with greedish features and logging.
"""
def __init__(self, *args, **kwargs): def __init__(self, *args, **kwargs):
kwargs.update(stdin=subprocess.PIPE) kwargs.update(stdin=subprocess.PIPE)
subprocess.Popen.__init__(self, *args, **kwargs) subprocess.Popen.__init__(self, *args, **kwargs)
...@@ -98,6 +100,15 @@ class SlapPopen(subprocess.Popen): ...@@ -98,6 +100,15 @@ class SlapPopen(subprocess.Popen):
self.stdin.close() self.stdin.close()
self.stdin = None self.stdin = None
logger = logging.getLogger('SlapProcessManager')
while True:
line = self.stdout.readline()
if line == '' and self.poll() != None:
break
if line[-1:] == '\n':
line = line[:-1]
logger.info(line)
def getSoftwareUrlHash(url): def getSoftwareUrlHash(url):
return md5(url).hexdigest() return md5(url).hexdigest()
...@@ -171,6 +182,8 @@ def dropPrivileges(uid, gid): ...@@ -171,6 +182,8 @@ def dropPrivileges(uid, gid):
Does nothing in case if uid and gid are not 0 Does nothing in case if uid and gid are not 0
""" """
logger = logging.getLogger('dropPrivileges') logger = logging.getLogger('dropPrivileges')
# XXX-Cedric: remove format / just do a print, otherwise formatting is done
# twice
current_uid, current_gid = os.getuid(), os.getgid() current_uid, current_gid = os.getuid(), os.getgid()
if uid == 0 or gid == 0: if uid == 0 or gid == 0:
raise OSError('Dropping privileges to uid = %r or ' \ raise OSError('Dropping privileges to uid = %r or ' \
...@@ -217,11 +230,11 @@ def dropPrivileges(uid, gid): ...@@ -217,11 +230,11 @@ def dropPrivileges(uid, gid):
else: else:
raise ValueError('%s it was possible to go back to uid = %r and gid = ' raise ValueError('%s it was possible to go back to uid = %r and gid = '
'%r which is fatal.' % (message_pre, current_uid, current_gid)) '%r which is fatal.' % (message_pre, current_uid, current_gid))
logger.info('Succesfully dropped privileges to uid=%r gid=%r' % (uid, gid)) logger.debug('Succesfully dropped privileges to uid=%r gid=%r' % (uid, gid))
def bootstrapBuildout(path, buildout=None, def bootstrapBuildout(path, buildout=None,
additional_buildout_parametr_list=None, console=False): additional_buildout_parametr_list=None):
if additional_buildout_parametr_list is None: if additional_buildout_parametr_list is None:
additional_buildout_parametr_list = [] additional_buildout_parametr_list = []
logger = logging.getLogger('BuildoutManager') logger = logging.getLogger('BuildoutManager')
...@@ -258,20 +271,14 @@ def bootstrapBuildout(path, buildout=None, ...@@ -258,20 +271,14 @@ def bootstrapBuildout(path, buildout=None,
logger.debug('Set umask from %03o to %03o' % (umask, SAFE_UMASK)) logger.debug('Set umask from %03o to %03o' % (umask, SAFE_UMASK))
logger.debug('Invoking: %r in directory %r' % (' '.join(invocation_list), logger.debug('Invoking: %r in directory %r' % (' '.join(invocation_list),
path)) path))
if not console: kw.update(stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
kw.update(stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
process_handler = SlapPopen(invocation_list, process_handler = SlapPopen(invocation_list,
preexec_fn=lambda: dropPrivileges(uid, gid), preexec_fn=lambda: dropPrivileges(uid, gid),
cwd=path, **kw) cwd=path, **kw)
result = process_handler.communicate()[0]
if console:
result = 'Please consult messages above'
if process_handler.returncode is None or process_handler.returncode != 0: if process_handler.returncode is None or process_handler.returncode != 0:
message = 'Failed to run buildout profile in directory %r:\n%s\n' % ( message = 'Failed to run buildout profile in directory %r.\n' % (path)
path, result)
raise BuildoutFailedError(message) raise BuildoutFailedError(message)
else:
logger.debug('Successful run:\n%s' % result)
except OSError as error: except OSError as error:
raise BuildoutFailedError(error) raise BuildoutFailedError(error)
finally: finally:
...@@ -280,7 +287,7 @@ def bootstrapBuildout(path, buildout=None, ...@@ -280,7 +287,7 @@ def bootstrapBuildout(path, buildout=None,
def launchBuildout(path, buildout_binary, def launchBuildout(path, buildout_binary,
additional_buildout_parametr_list=None, console=False): additional_buildout_parametr_list=None):
""" Launches buildout.""" """ Launches buildout."""
logger = logging.getLogger('BuildoutManager') logger = logging.getLogger('BuildoutManager')
if additional_buildout_parametr_list is None: if additional_buildout_parametr_list is None:
...@@ -306,21 +313,13 @@ def launchBuildout(path, buildout_binary, ...@@ -306,21 +313,13 @@ def launchBuildout(path, buildout_binary,
logger.debug('Set umask from %03o to %03o' % (umask, SAFE_UMASK)) logger.debug('Set umask from %03o to %03o' % (umask, SAFE_UMASK))
logger.debug('Invoking: %r in directory %r' % (' '.join(invocation_list), logger.debug('Invoking: %r in directory %r' % (' '.join(invocation_list),
path)) path))
kw = dict() kw = dict(stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
if not console:
kw.update(stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
process_handler = SlapPopen(invocation_list, process_handler = SlapPopen(invocation_list,
preexec_fn=lambda: dropPrivileges(uid, gid), cwd=path, preexec_fn=lambda: dropPrivileges(uid, gid), cwd=path,
env=getCleanEnvironment(pwd.getpwuid(uid).pw_dir), **kw) env=getCleanEnvironment(pwd.getpwuid(uid).pw_dir), **kw)
result = process_handler.communicate()[0]
if console:
result = 'Please consult messages above'
if process_handler.returncode is None or process_handler.returncode != 0: if process_handler.returncode is None or process_handler.returncode != 0:
message = 'Failed to run buildout profile in directory %r:\n%s\n' % ( message = 'Failed to run buildout profile in directory %r\n' % (path)
path, result)
raise BuildoutFailedError(message) raise BuildoutFailedError(message)
else:
logger.debug('Successful run:\n%s' % result)
except OSError as error: except OSError as error:
raise BuildoutFailedError(error) raise BuildoutFailedError(error)
finally: finally:
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment