Commit 65622b9a authored by Marco Mariani's avatar Marco Mariani

make pep8 happy

parent c1058afd
...@@ -43,7 +43,7 @@ from supervisor import xmlrpc ...@@ -43,7 +43,7 @@ from supervisor import xmlrpc
from slapos.grid.utils import (md5digest, getCleanEnvironment, from slapos.grid.utils import (md5digest, getCleanEnvironment,
SlapPopen, dropPrivileges, updateFile) SlapPopen, dropPrivileges, updateFile)
from slapos.grid import utils # for mocked methods from slapos.grid import utils # for mocked methods
from slapos.slap.slap import NotFoundError from slapos.slap.slap import NotFoundError
from slapos.grid.svcbackend import getSupervisorRPC from slapos.grid.svcbackend import getSupervisorRPC
from slapos.grid.exception import (BuildoutFailedError, WrongPermissionError, from slapos.grid.exception import (BuildoutFailedError, WrongPermissionError,
...@@ -166,8 +166,8 @@ class Software(object): ...@@ -166,8 +166,8 @@ class Software(object):
] ]
if (self.signature_private_key_file or if (self.signature_private_key_file or
self.upload_cache_url or self.upload_cache_url or
self.upload_dir_url): self.upload_dir_url):
buildout_parameter_list.append('buildout:networkcache-section=networkcache') buildout_parameter_list.append('buildout:networkcache-section=networkcache')
for buildout_option, value in [ for buildout_option, value in [
('%ssignature-private-key-file=%s', self.signature_private_key_file), ('%ssignature-private-key-file=%s', self.signature_private_key_file),
...@@ -306,10 +306,7 @@ class Partition(object): ...@@ -306,10 +306,7 @@ class Partition(object):
uid, gid = self.getUserGroupId() uid, gid = self.getUserGroupId()
for name, path in [ for name, path in [('certificate', self.cert_file), ('key', self.key_file)]:
('certificate', self.cert_file),
('key', self.key_file),
]:
new_content = partition_certificate[name] new_content = partition_certificate[name]
old_content = None old_content = None
if os.path.exists(path): if os.path.exists(path):
...@@ -325,7 +322,6 @@ class Partition(object): ...@@ -325,7 +322,6 @@ class Partition(object):
fout.write(new_content) fout.write(new_content)
os.chown(path, uid, gid) os.chown(path, uid, gid)
def getUserGroupId(self): def getUserGroupId(self):
"""Returns tuple of (uid, gid) of partition""" """Returns tuple of (uid, gid) of partition"""
stat_info = os.stat(self.instance_path) stat_info = os.stat(self.instance_path)
...@@ -341,17 +337,17 @@ class Partition(object): ...@@ -341,17 +337,17 @@ class Partition(object):
for runner in runner_list: for runner in runner_list:
self.partition_supervisor_configuration += '\n' + \ self.partition_supervisor_configuration += '\n' + \
program_partition_template % { program_partition_template % {
'program_id': '_'.join([partition_id, runner]), 'program_id': '_'.join([partition_id, runner]),
'program_directory': self.instance_path, 'program_directory': self.instance_path,
'program_command': os.path.join(path, runner), 'program_command': os.path.join(path, runner),
'program_name': runner + extension, 'program_name': runner + extension,
'instance_path': self.instance_path, 'instance_path': self.instance_path,
'user_id': uid, 'user_id': uid,
'group_id': gid, 'group_id': gid,
# As supervisord has no environment to inherit, setup a minimalistic one # As supervisord has no environment to inherit, setup a minimalistic one
'HOME': pwd.getpwuid(uid).pw_dir, 'HOME': pwd.getpwuid(uid).pw_dir,
'USER': pwd.getpwuid(uid).pw_name, 'USER': pwd.getpwuid(uid).pw_name,
} }
def updateSymlink(self, sr_symlink, software_path): def updateSymlink(self, sr_symlink, software_path):
if os.path.lexists(sr_symlink): if os.path.lexists(sr_symlink):
...@@ -391,7 +387,7 @@ class Partition(object): ...@@ -391,7 +387,7 @@ class Partition(object):
if not os.path.exists(self.software_path): if not os.path.exists(self.software_path):
# XXX What should it raise? # XXX What should it raise?
raise IOError('Software Release %s is not present on system.\n' raise IOError('Software Release %s is not present on system.\n'
'Cannot deploy instance.' % self.software_release_url) 'Cannot deploy instance.' % self.software_release_url)
# Generate buildout instance profile from template in Software Release # Generate buildout instance profile from template in Software Release
template_location = os.path.join(self.software_path, 'instance.cfg') template_location = os.path.join(self.software_path, 'instance.cfg')
...@@ -412,13 +408,13 @@ class Partition(object): ...@@ -412,13 +408,13 @@ class Partition(object):
buildout_text = open(config_location).read() buildout_text = open(config_location).read()
buildout_text += '\n\n' + pkg_resources.resource_string(__name__, buildout_text += '\n\n' + pkg_resources.resource_string(__name__,
'templates/buildout-tail.cfg.in') % { 'templates/buildout-tail.cfg.in') % {
'computer_id': self.computer_id, 'computer_id': self.computer_id,
'partition_id': self.partition_id, 'partition_id': self.partition_id,
'server_url': self.server_url, 'server_url': self.server_url,
'software_release_url': self.software_release_url, 'software_release_url': self.software_release_url,
'key_file': self.key_file, 'key_file': self.key_file,
'cert_file': self.cert_file, 'cert_file': self.cert_file,
} }
open(config_location, 'w').write(buildout_text) open(config_location, 'w').write(buildout_text)
os.chmod(config_location, 0o640) os.chmod(config_location, 0o640)
# Try to find the best possible buildout: # Try to find the best possible buildout:
...@@ -482,7 +478,6 @@ class Partition(object): ...@@ -482,7 +478,6 @@ class Partition(object):
logger=self.logger) logger=self.logger)
self.generateSupervisorConfigurationFile() self.generateSupervisorConfigurationFile()
def generateSupervisorConfigurationFile(self): def generateSupervisorConfigurationFile(self):
""" """
Generates supervisord configuration file from template. Generates supervisord configuration file from template.
...@@ -510,10 +505,10 @@ class Partition(object): ...@@ -510,10 +505,10 @@ class Partition(object):
group_partition_template = pkg_resources.resource_stream(__name__, group_partition_template = pkg_resources.resource_stream(__name__,
'templates/group_partition_supervisord.conf.in').read() 'templates/group_partition_supervisord.conf.in').read()
self.partition_supervisor_configuration = group_partition_template % { self.partition_supervisor_configuration = group_partition_template % {
'instance_id': partition_id, 'instance_id': partition_id,
'program_list': ','.join(['_'.join([partition_id, runner]) 'program_list': ','.join(['_'.join([partition_id, runner])
for runner in runner_list + service_list]) for runner in runner_list + service_list])
} }
# Same method to add to service and run # Same method to add to service and run
self.addServiceToGroup(partition_id, runner_list, self.run_path) self.addServiceToGroup(partition_id, runner_list, self.run_path)
self.addServiceToGroup(partition_id, service_list, self.service_path, self.addServiceToGroup(partition_id, service_list, self.service_path,
......
...@@ -62,4 +62,3 @@ def patched_linux_distribution(distname='', version='', id='', ...@@ -62,4 +62,3 @@ def patched_linux_distribution(distname='', version='', id='',
pass pass
return platform.linux_distribution(distname, version, id, supported_dists, full_distribution_name) return platform.linux_distribution(distname, version, id, supported_dists, full_distribution_name)
...@@ -100,23 +100,23 @@ def check_missing_parameters(options): ...@@ -100,23 +100,23 @@ def check_missing_parameters(options):
def check_missing_files(options): def check_missing_files(options):
req_files = [ req_files = [
options.get('key_file'), options.get('key_file'),
options.get('cert_file'), options.get('cert_file'),
options.get('master_ca_file'), options.get('master_ca_file'),
options.get('shacache-cert-file'), options.get('shacache-cert-file'),
options.get('shacache-key-file'), options.get('shacache-key-file'),
options.get('shadir-cert-file'), options.get('shadir-cert-file'),
options.get('shadir-key-file'), options.get('shadir-key-file'),
options.get('signature_private_key_file') options.get('signature_private_key_file')
] ]
req_dirs = [ req_dirs = [
options.get('certificate_repository_path') options.get('certificate_repository_path')
] ]
for f in req_files: for f in req_files:
if f and not os.path.exists(f): if f and not os.path.exists(f):
raise RuntimeError('File %r does not exist.' % f) raise RuntimeError('File %r does not exist.' % f)
for d in req_dirs: for d in req_dirs:
if d and not os.path.isdir(d): if d and not os.path.isdir(d):
...@@ -149,8 +149,8 @@ def merged_options(args, configp): ...@@ -149,8 +149,8 @@ def merged_options(args, configp):
# Parse cache / binary cache options # Parse cache / binary cache options
# Backward compatibility about "binary-cache-url-blacklist" deprecated option # Backward compatibility about "binary-cache-url-blacklist" deprecated option
if options.get("binary-cache-url-blacklist") and not \ if (options.get("binary-cache-url-blacklist") and not
options.get("download-from-binary-cache-url-blacklist"): options.get("download-from-binary-cache-url-blacklist")):
options["download-from-binary-cache-url-blacklist"] = \ options["download-from-binary-cache-url-blacklist"] = \
options["binary-cache-url-blacklist"] options["binary-cache-url-blacklist"]
options["download-from-binary-cache-url-blacklist"] = [ options["download-from-binary-cache-url-blacklist"] = [
...@@ -175,8 +175,8 @@ def random_delay(options, logger): ...@@ -175,8 +175,8 @@ def random_delay(options, logger):
maximal_delay = int(options.get('maximal_delay', '0')) maximal_delay = int(options.get('maximal_delay', '0'))
if maximal_delay: if maximal_delay:
duration = random.randint(1, maximal_delay) duration = random.randint(1, maximal_delay)
logger.info('Sleeping for %s seconds. To disable this feature, ' \ logger.info('Sleeping for %s seconds. To disable this feature, '
'check --now parameter in slapgrid help.' % duration) 'check --now parameter in slapgrid help.', duration)
time.sleep(duration) time.sleep(duration)
...@@ -185,10 +185,10 @@ def create_slapgrid_object(options, logger): ...@@ -185,10 +185,10 @@ def create_slapgrid_object(options, logger):
if 'signature-certificate-list' in options: if 'signature-certificate-list' in options:
cert_marker = '-----BEGIN CERTIFICATE-----' cert_marker = '-----BEGIN CERTIFICATE-----'
signature_certificate_list = [ signature_certificate_list = [
cert_marker + '\n' + q.strip() cert_marker + '\n' + q.strip()
for q in options['signature-certificate-list'].split(cert_marker) for q in options['signature-certificate-list'].split(cert_marker)
if q.strip() if q.strip()
] ]
op = options op = options
return Slapgrid(software_root=op['software_root'], return Slapgrid(software_root=op['software_root'],
...@@ -199,17 +199,17 @@ def create_slapgrid_object(options, logger): ...@@ -199,17 +199,17 @@ def create_slapgrid_object(options, logger):
supervisord_configuration_path=op['supervisord_configuration_path'], supervisord_configuration_path=op['supervisord_configuration_path'],
buildout=op.get('buildout'), buildout=op.get('buildout'),
logger=logger, logger=logger,
force_periodicity = op.get('force_periodicity', False), force_periodicity=op.get('force_periodicity', False),
maximum_periodicity = op.get('maximum_periodicity', 86400), maximum_periodicity=op.get('maximum_periodicity', 86400),
key_file=op.get('key_file'), key_file=op.get('key_file'),
cert_file=op.get('cert_file'), cert_file=op.get('cert_file'),
signature_private_key_file=op.get('signature_private_key_file'), signature_private_key_file=op.get('signature_private_key_file'),
signature_certificate_list=signature_certificate_list, signature_certificate_list=signature_certificate_list,
download_binary_cache_url=op.get('download-binary-cache-url'), download_binary_cache_url=op.get('download-binary-cache-url'),
upload_binary_cache_url=op.get('upload-binary-cache-url'), upload_binary_cache_url=op.get('upload-binary-cache-url'),
download_from_binary_cache_url_blacklist=\ download_from_binary_cache_url_blacklist=
op.get('download-from-binary-cache-url-blacklist', []), op.get('download-from-binary-cache-url-blacklist', []),
upload_to_binary_cache_url_blacklist=\ upload_to_binary_cache_url_blacklist=
op.get('upload-to-binary-cache-url-blacklist', []), op.get('upload-to-binary-cache-url-blacklist', []),
upload_cache_url=op.get('upload-cache-url'), upload_cache_url=op.get('upload-cache-url'),
download_binary_dir_url=op.get('download-binary-dir-url'), download_binary_dir_url=op.get('download-binary-dir-url'),
...@@ -232,8 +232,8 @@ def create_slapgrid_object(options, logger): ...@@ -232,8 +232,8 @@ def create_slapgrid_object(options, logger):
def check_required_only_partitions(existing, required): def check_required_only_partitions(existing, required):
missing = set(required) - set(existing) missing = set(required) - set(existing)
if missing: if missing:
plural = ['s', ''][len(missing)==1] plural = ['s', ''][len(missing) == 1]
raise ValueError('Unknown partition%s: %s' % (plural, ', '.join(sorted(missing)))) raise ValueError('Unknown partition%s: %s' % (plural, ', '.join(sorted(missing))))
class Slapgrid(object): class Slapgrid(object):
...@@ -334,8 +334,8 @@ class Slapgrid(object): ...@@ -334,8 +334,8 @@ class Slapgrid(object):
invocation_list = [WATCHDOG_PATH] invocation_list = [WATCHDOG_PATH]
invocation_list.append("--master-url '%s' " % self.master_url) invocation_list.append("--master-url '%s' " % self.master_url)
if self.certificate_repository_path: if self.certificate_repository_path:
invocation_list.append("--certificate-repository-path '%s'" \ invocation_list.append("--certificate-repository-path '%s'" %
% self.certificate_repository_path) self.certificate_repository_path)
invocation_list.append("--computer-id '%s'" % self.computer_id) invocation_list.append("--computer-id '%s'" % self.computer_id)
return ' '.join(invocation_list) return ' '.join(invocation_list)
...@@ -371,7 +371,8 @@ class Slapgrid(object): ...@@ -371,7 +371,8 @@ class Slapgrid(object):
'supervisord_pidfile': os.path.abspath(os.path.join(self.instance_root, 'var', 'run', 'supervisord.pid')), 'supervisord_pidfile': os.path.abspath(os.path.join(self.instance_root, 'var', 'run', 'supervisord.pid')),
'supervisord_logfile_backups': '10', 'supervisord_logfile_backups': '10',
'watchdog_command': self.getWatchdogLine(), 'watchdog_command': self.getWatchdogLine(),
}) }
)
def getComputerPartitionList(self): def getComputerPartitionList(self):
try: try:
...@@ -401,9 +402,9 @@ class Slapgrid(object): ...@@ -401,9 +402,9 @@ class Slapgrid(object):
signature_certificate_list=self.signature_certificate_list, signature_certificate_list=self.signature_certificate_list,
download_binary_cache_url=self.download_binary_cache_url, download_binary_cache_url=self.download_binary_cache_url,
upload_binary_cache_url=self.upload_binary_cache_url, upload_binary_cache_url=self.upload_binary_cache_url,
download_from_binary_cache_url_blacklist=\ download_from_binary_cache_url_blacklist=
self.download_from_binary_cache_url_blacklist, self.download_from_binary_cache_url_blacklist,
upload_to_binary_cache_url_blacklist=\ upload_to_binary_cache_url_blacklist=
self.upload_to_binary_cache_url_blacklist, self.upload_to_binary_cache_url_blacklist,
upload_cache_url=self.upload_cache_url, upload_cache_url=self.upload_cache_url,
download_binary_dir_url=self.download_binary_dir_url, download_binary_dir_url=self.download_binary_dir_url,
...@@ -415,10 +416,10 @@ class Slapgrid(object): ...@@ -415,10 +416,10 @@ class Slapgrid(object):
shadir_key_file=self.shadir_key_file) shadir_key_file=self.shadir_key_file)
if state == 'available': if state == 'available':
completed_tag = os.path.join(software_path, '.completed') completed_tag = os.path.join(software_path, '.completed')
if self.develop or (not os.path.exists(completed_tag) and \ if (self.develop or (not os.path.exists(completed_tag) and
len(self.software_release_filter_list) == 0) or \ len(self.software_release_filter_list) == 0) or
url_hash in self.software_release_filter_list or \ url_hash in self.software_release_filter_list or
url_hash in (md5digest(uri) for uri in self.software_release_filter_list): url_hash in (md5digest(uri) for uri in self.software_release_filter_list)):
try: try:
software_release.building() software_release.building()
except NotFoundError: except NotFoundError:
...@@ -471,7 +472,6 @@ class Slapgrid(object): ...@@ -471,7 +472,6 @@ class Slapgrid(object):
return SLAPGRID_FAIL return SLAPGRID_FAIL
return SLAPGRID_SUCCESS return SLAPGRID_SUCCESS
def _launchSupervisord(self): def _launchSupervisord(self):
launchSupervisord(self.supervisord_socket, launchSupervisord(self.supervisord_socket,
self.supervisord_configuration_path, self.supervisord_configuration_path,
...@@ -592,8 +592,8 @@ class Slapgrid(object): ...@@ -592,8 +592,8 @@ class Slapgrid(object):
# Check if timestamp from server is more recent than local one. # Check if timestamp from server is more recent than local one.
# If not: it's not worth processing this partition (nothing has # If not: it's not worth processing this partition (nothing has
# changed). # changed).
if computer_partition_id not in self.computer_partition_filter_list and \ if (computer_partition_id not in self.computer_partition_filter_list and
(not self.develop) and os.path.exists(timestamp_path): not self.develop and os.path.exists(timestamp_path)):
old_timestamp = open(timestamp_path).read() old_timestamp = open(timestamp_path).read()
last_runtime = int(os.path.getmtime(timestamp_path)) last_runtime = int(os.path.getmtime(timestamp_path))
if timestamp: if timestamp:
...@@ -808,7 +808,6 @@ class Slapgrid(object): ...@@ -808,7 +808,6 @@ class Slapgrid(object):
return SLAPGRID_PROMISE_FAIL return SLAPGRID_PROMISE_FAIL
return SLAPGRID_SUCCESS return SLAPGRID_SUCCESS
def validateXML(self, to_be_validated, xsd_model): def validateXML(self, to_be_validated, xsd_model):
"""Validates a given xml file""" """Validates a given xml file"""
#We retrieve the xsd model #We retrieve the xsd model
...@@ -819,8 +818,8 @@ class Slapgrid(object): ...@@ -819,8 +818,8 @@ class Slapgrid(object):
try: try:
document = etree.fromstring(to_be_validated) document = etree.fromstring(to_be_validated)
except (etree.XMLSyntaxError, etree.DocumentInvalid) as exc: except (etree.XMLSyntaxError, etree.DocumentInvalid) as exc:
self.logger.info('Failed to parse this XML report : %s\n%s' % \ self.logger.info('Failed to parse this XML report : %s\n%s' %
(to_be_validated, _formatXMLError(exc))) (to_be_validated, _formatXMLError(exc)))
self.logger.error(_formatXMLError(exc)) self.logger.error(_formatXMLError(exc))
return False return False
...@@ -967,9 +966,8 @@ class Slapgrid(object): ...@@ -967,9 +966,8 @@ class Slapgrid(object):
computer_partition.error('\n'.join(failed_script_list), logger=self.logger) computer_partition.error('\n'.join(failed_script_list), logger=self.logger)
# Whatever happens, don't stop processing other instances # Whatever happens, don't stop processing other instances
except Exception: except Exception:
self.logger.info('Cannot run usage script(s) for %r: %s' % ( self.logger.info('Cannot run usage script(s) for %r: %s' %
computer_partition.getId(), (computer_partition.getId(), traceback.format_exc()))
traceback.format_exc()))
#Now we loop through the different computer partitions to report #Now we loop through the different computer partitions to report
report_usage_issue_cp_list = [] report_usage_issue_cp_list = []
...@@ -1002,7 +1000,7 @@ class Slapgrid(object): ...@@ -1002,7 +1000,7 @@ class Slapgrid(object):
# report # report
else: else:
computer_partition_usage = self.slap.registerComputerPartition( computer_partition_usage = self.slap.registerComputerPartition(
self.computer_id, computer_partition_id) self.computer_id, computer_partition_id)
computer_partition_usage.setUsage(usage) computer_partition_usage.setUsage(usage)
computer_partition_usage_list.append(computer_partition_usage) computer_partition_usage_list.append(computer_partition_usage)
filename_delete_list.append(filename) filename_delete_list.append(filename)
...@@ -1015,14 +1013,12 @@ class Slapgrid(object): ...@@ -1015,14 +1013,12 @@ class Slapgrid(object):
# Whatever happens, don't stop processing other instances # Whatever happens, don't stop processing other instances
except Exception: except Exception:
self.logger.info('Cannot run usage script(s) for %r: %s' % ( self.logger.info('Cannot run usage script(s) for %r: %s' %
computer_partition.getId(), (computer_partition.getId(), traceback.format_exc()))
traceback.format_exc()))
for computer_partition_usage in computer_partition_usage_list: for computer_partition_usage in computer_partition_usage_list:
self.logger.info('computer_partition_usage_list: %s - %s' % ( self.logger.info('computer_partition_usage_list: %s - %s' %
computer_partition_usage.usage, (computer_partition_usage.usage, computer_partition_usage.getId()))
computer_partition_usage.getId()))
#If there is, at least, one report #If there is, at least, one report
if computer_partition_usage_list != []: if computer_partition_usage_list != []:
...@@ -1041,8 +1037,8 @@ class Slapgrid(object): ...@@ -1041,8 +1037,8 @@ class Slapgrid(object):
raise ValueError('XML file generated by asXML is not valid !') raise ValueError('XML file generated by asXML is not valid !')
except Exception: except Exception:
issue = "Cannot report usage for %r: %s" % ( issue = "Cannot report usage for %r: %s" % (
computer_partition.getId(), computer_partition.getId(),
traceback.format_exc()) traceback.format_exc())
self.logger.info(issue) self.logger.info(issue)
computer_partition.error(issue, logger=self.logger) computer_partition.error(issue, logger=self.logger)
report_usage_issue_cp_list.append(computer_partition_id) report_usage_issue_cp_list.append(computer_partition_id)
......
...@@ -124,7 +124,7 @@ def getCleanEnvironment(logger, home_path='/tmp'): ...@@ -124,7 +124,7 @@ def getCleanEnvironment(logger, home_path='/tmp'):
env = os.environ.copy() env = os.environ.copy()
# Clean python related environment variables # Clean python related environment variables
for k in PYTHON_ENVIRONMENT_REMOVE_LIST + SYSTEM_ENVIRONMENT_REMOVE_LIST \ for k in PYTHON_ENVIRONMENT_REMOVE_LIST + SYSTEM_ENVIRONMENT_REMOVE_LIST \
+ LOCALE_ENVIRONMENT_REMOVE_LIST: + LOCALE_ENVIRONMENT_REMOVE_LIST:
old = env.pop(k, None) old = env.pop(k, None)
if old is not None: if old is not None:
removed_env.append(k) removed_env.append(k)
...@@ -197,8 +197,7 @@ def dropPrivileges(uid, gid, logger): ...@@ -197,8 +197,7 @@ def dropPrivileges(uid, gid, logger):
# assert that privileges are dropped # assert that privileges are dropped
message_pre = 'After dropping to uid = %r and gid = %r ' \ message_pre = 'After dropping to uid = %r and gid = %r ' \
'and group_list = %s' % ( 'and group_list = %s' % (uid, gid, group_list)
uid, gid, group_list)
new_uid, new_gid, new_group_list = os.getuid(), os.getgid(), os.getgroups() new_uid, new_gid, new_group_list = os.getuid(), os.getgid(), os.getgroups()
if not (new_uid == uid and new_gid == gid and set(new_group_list) == group_list): if not (new_uid == uid and new_gid == gid and set(new_group_list) == group_list):
raise OSError('%s new_uid = %r and new_gid = %r and ' raise OSError('%s new_uid = %r and new_gid = %r and '
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment