Commit 16069398 authored by Bryton Lacquement's avatar Bryton Lacquement 🚪

slapos: add support for Python 3

parent 0d5ada46
......@@ -93,7 +93,7 @@ make-binary=
${:extra-env} PERL5LIB="${:inc}:${:install-inc}:${:perl-PERL5LIB}" make
# this post-make-hook is same for all users of the macro.
post-make-hook = ${:_profile_base_location_}/../../component/perl/perl-CPAN-package-create-wrapper.py#d012f7ba3300b14b2c6b173351d410be:post_make_hook
post-make-hook = ${:_profile_base_location_}/../../component/perl/perl-CPAN-package-create-wrapper.py#f28c45a0f473ae050ca3ebaed5c39a4e:post_make_hook
perl_location = ${perl:location}
......
......@@ -17,7 +17,7 @@ def post_make_hook(options, buildout, environmet):
export PERL5LIB="{site_perl}:$PERL5LIB"
exec {perl_location}/bin/perl "$@"
'''.format(**locals()))
os.chmod(perl_wrapper_path, 0755)
os.chmod(perl_wrapper_path, 0o755)
# create a wrapper for each scripts installed in perl-bin
for script_path in glob.glob(os.path.join(prefix, 'perl-bin', '*')):
......@@ -28,4 +28,4 @@ exec {perl_location}/bin/perl "$@"
export PERL5LIB="{site_perl}:$PERL5LIB"
exec {perl_location}/bin/perl {script_path} "$@"
'''.format(**locals()))
os.chmod(wrapper_path, 0755)
os.chmod(wrapper_path, 0o755)
......@@ -2,7 +2,6 @@
extends =
../fontconfig/buildout.cfg
../libexpat/buildout.cfg
../dash/buildout.cfg
parts =
phantomjs
......@@ -19,17 +18,19 @@ x86 = https://bitbucket.org/ariya/phantomjs/downloads/phantomjs-1.9.7-linux-i686
x86-64 = https://bitbucket.org/ariya/phantomjs/downloads/phantomjs-1.9.7-linux-x86_64.tar.bz2 f278996c3edd0e8d8ec4893807f27d71
script =
if not self.options.get('url'): self.options['url'], self.options['md5sum'] = self.options[guessPlatform()].split(' ')
extract_dir = self.extract(self.download(self.options['url'], self.options.get('md5sum')))
if not self.options.get('url'):
self.options['url'], self.options['md5sum'] = \
self.options[guessPlatform()].split(' ')
extract_dir = self.extract(self.download(self.options['url'],
self.options.get('md5sum')))
workdir = guessworkdir(extract_dir)
self.copyTree(workdir, "%(location)s")
wrapper_location = os.path.join("%(location)s", "phantomjs-slapos")
wrapper = open(wrapper_location, 'w')
wrapper.write("""#!${dash:location}/bin/dash
with open(wrapper_location, 'w') as wrapper:
wrapper.write("""#!/bin/sh
cd %(location)s
export LD_LIBRARY_PATH=%(location)s:${freetype:location}/lib/:${fontconfig:location}/lib/:${libexpat:location}/lib
export PATH=${fontconfig:location}/bin:$PATH
exec %(location)s/bin/phantomjs $*""")
wrapper.flush()
wrapper.close()
os.chmod(wrapper_location, 0755)
exec %(location)s/bin/phantomjs "$@"
""")
os.chmod(wrapper_location, 0o755)
......@@ -24,6 +24,8 @@
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
from __future__ import print_function
from slapos.recipe.librecipe import GenericBaseRecipe
import zc.buildout
import sys
......@@ -101,17 +103,17 @@ class Recipe(GenericBaseRecipe):
# TODO factor
if delete != []:
print "Creating lampconfigure with 'delete' arguments"
print("Creating lampconfigure with 'delete' arguments")
command = argument + delete
if rename != []:
for parameters in rename:
print "Creating lampconfigure with 'rename' arguments"
print("Creating lampconfigure with 'rename' arguments")
command = argument + rename
if chmod != []:
print "Creating lampconfigure with 'chmod' arguments"
print("Creating lampconfigure with 'chmod' arguments")
command = argument + chmod
if data != []:
print "Creating lampconfigure with 'run' arguments"
print("Creating lampconfigure with 'run' arguments")
command = argument + data
......
......@@ -94,7 +94,7 @@ class Recipe(GenericBaseRecipe):
dict(ip_address=self.options['ipv6'].strip(),
project=project,
middleware=type)))
os.chmod(config_info_file, 0744)
os.chmod(config_info_file, 0o744)
path_list.append(config_info)
update = install
......
......@@ -26,11 +26,11 @@
##############################################################################
import os
import hashlib
import ConfigParser
from six.moves import configparser
import tempfile
from slapos.recipe.librecipe import GenericBaseRecipe
from certificate_authority import popenCommunicate
from .certificate_authority import popenCommunicate
class Recipe(GenericBaseRecipe):
......@@ -119,7 +119,7 @@ class Request(Recipe):
open(certificate, 'w').write(cert_content)
request_needed = False
else:
parser = ConfigParser.RawConfigParser()
parser = configparser.RawConfigParser()
parser.add_section('certificate')
parser.set('certificate', 'name', name)
parser.set('certificate', 'key_file', key)
......
from __future__ import print_function
import os
import subprocess
import time
import ConfigParser
from six.moves import configparser
import uuid
......@@ -95,12 +97,12 @@ class CertificateAuthority:
def checkRequestDir(self):
for request_file in os.listdir(self.request_dir):
parser = ConfigParser.RawConfigParser()
parser = configparser.RawConfigParser()
parser.readfp(open(os.path.join(self.request_dir, request_file), 'r'))
if self._checkCertificate(parser.get('certificate', 'name'),
parser.get('certificate', 'key_file'), parser.get('certificate',
'certificate_file')):
print 'Created certificate %r' % parser.get('certificate', 'name')
print('Created certificate %r' % parser.get('certificate', 'name'))
def runCertificateAuthority(*args):
ca = CertificateAuthority(*args)
......
......@@ -29,7 +29,6 @@ import os
import subprocess
import zc.buildout
import filecmp
import urlparse
import shutil
import re
import json
......@@ -130,9 +129,9 @@ class Recipe(GenericBaseRecipe):
#create condor binary launcher for slapos
if not os.path.exists(self.wrapper_bin):
os.makedirs(self.wrapper_bin, int('0744', 8))
os.makedirs(self.wrapper_bin, int('0o744', 8))
if not os.path.exists(self.wrapper_sbin):
os.makedirs(self.wrapper_sbin, int('0744', 8))
os.makedirs(self.wrapper_sbin, int('0o744', 8))
#generate script for each file in prefix/bin
for binary in os.listdir(self.prefix+'/bin'):
wrapper_location = os.path.join(self.wrapper_bin, binary)
......@@ -153,7 +152,7 @@ class Recipe(GenericBaseRecipe):
wrapper.write(content)
wrapper.close()
path_list.append(wrapper_location)
os.chmod(wrapper_location, 0744)
os.chmod(wrapper_location, 0o744)
#generate script for each file in prefix/sbin
for binary in os.listdir(self.prefix+'/sbin'):
......@@ -175,7 +174,7 @@ class Recipe(GenericBaseRecipe):
wrapper.write(content)
wrapper.close()
path_list.append(wrapper_location)
os.chmod(wrapper_location, 0744)
os.chmod(wrapper_location, 0o744)
#generate script for start condor
wrapper = self.createPythonScript(
......@@ -228,7 +227,7 @@ class AppSubmit(GenericBaseRecipe):
for file in file_list:
if file and (file.startswith('http') or file.startswith('ftp')):
file_list[file] = self.download(file_list[file])
os.chmod(file_list[file], 0600)
os.chmod(file_list[file], 0o600)
else:
app_list[app]['files'] = {}
......@@ -236,11 +235,11 @@ class AppSubmit(GenericBaseRecipe):
if executable and (executable.startswith('http') or executable.startswith('ftp')):
app_list[app]['executable'] = self.download(executable,
app_list[app]['executable-name'])
os.chmod(app_list[app]['executable-name'], 0700)
os.chmod(app_list[app]['executable-name'], 0o700)
submit_file = app_list[app].get('description-file', '')
if submit_file and (submit_file.startswith('http') or submit_file.startswith('ftp')):
app_list[app]['description-file'] = self.download(submit_file, 'submit')
os.chmod(app_list[app]['description-file'], 0600)
os.chmod(app_list[app]['description-file'], 0o600)
return app_list
......
......@@ -25,7 +25,7 @@
#
##############################################################################
import subprocess
import httplib
from six.moves import http_client as httplib
import base64
import os
import shutil
......
......@@ -29,6 +29,8 @@ import os
from slapos.recipe.librecipe import GenericBaseRecipe
from zc.buildout import UserError
from six.moves import map
class Recipe(GenericBaseRecipe):
def install(self):
......@@ -124,7 +126,7 @@ def systemd_to_cron(spec):
x = spec[i]
if x != '*':
for x in x.split(','):
x = map(int, x.split('/', 1))
x = list(map(int, x.split('/', 1)))
a = x[0] - y
if 0 <= a < z:
if len(x) == 1:
......
......@@ -26,7 +26,7 @@
##############################################################################
from slapos.recipe.librecipe import GenericBaseRecipe
import ConfigParser
from six.moves import configparser
class Recipe(GenericBaseRecipe):
"""
......@@ -34,7 +34,7 @@ class Recipe(GenericBaseRecipe):
"""
def install(self):
promise_parser = ConfigParser.RawConfigParser()
promise_parser = configparser.RawConfigParser()
for section_name, option_id_list in (
('portal_templates', (
('repository', 'bt5-repository-url'),
......
......@@ -24,10 +24,10 @@
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
import ConfigParser
from six.moves import configparser
import io
import json
import os
import StringIO
from slapos.recipe.librecipe import GenericBaseRecipe
......@@ -40,13 +40,13 @@ class Recipe(GenericBaseRecipe):
CONFIG['PATH'] = os.environ['PATH']
if self.options['instance-dict']:
config_instance_dict = ConfigParser.ConfigParser()
config_instance_dict = configparser.ConfigParser()
config_instance_dict.add_section('instance_dict')
instance_dict = json.loads(self.options['instance-dict'])
for k ,v in instance_dict.iteritems():
config_instance_dict.set('instance_dict', k, v)
value = StringIO.StringIO()
value = io.StringIO()
config_instance_dict.write(value)
CONFIG['instance_dict'] = value.getvalue()
......
......@@ -25,7 +25,7 @@
#
##############################################################################
import ConfigParser
from six.moves import configparser
import os
import netaddr
import socket
......@@ -48,7 +48,7 @@ class Recipe(object):
# If this check isn't done, a new port would be picked for every upgrade
# of the software release
try:
parser = ConfigParser.RawConfigParser()
parser = configparser.RawConfigParser()
if os.path.exists(buildout['buildout']['installed']):
with open(buildout['buildout']['installed']) as config_file:
parser.readfp(config_file)
......@@ -59,7 +59,7 @@ class Recipe(object):
if port != '0':
self.options['port'] = port
return
except (IOError, ConfigParser.NoSectionError, ConfigParser.NoOptionError):
except (IOError, configparser.NoSectionError, configparser.NoOptionError):
pass
# Otherwise, let's find one
......
......@@ -24,9 +24,11 @@
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
from functools import cmp_to_key
import zc.buildout
from slapos.recipe.librecipe import GenericBaseRecipe
@cmp_to_key
def compareMimetypeEntryPair(a, b):
"""
Like comparing strings, but here the star `*` is stronger than any other
......@@ -115,7 +117,7 @@ class Recipe(GenericBaseRecipe):
if l and not l.isspace()
]
mimetype_entry_list.extend(default_mimetype_entry_list)
mimetype_entry_list.sort(compareMimetypeEntryPair)
mimetype_entry_list.sort(key=compareMimetypeEntryPair)
conversion_server_dict['MIMETYPE_ENTRY_LIST'] = \
"\n".join([" " + l for l in mimetype_entry_list])
config_file = self.createFile(self.options['configuration-file'],
......
......@@ -31,7 +31,7 @@ import pkg_resources
import zc.buildout
import sys
import zc.recipe.egg
import urlparse
from six.moves.urllib.parse import urlparse
# Warning : this recipe is deprecated and has been replaced by apachephp.
......@@ -264,7 +264,7 @@ class Request(BaseRecipe):
mysql = self.request(self.options['mariadb-software-url'],
software_type, 'MariaDB Server', partition_parameter_kw=parameters
).getConnectionParameter('url')
mysql_parsed = urlparse.urlparse(mysql)
mysql_parsed = urlparse(mysql)
mysql_host, mysql_port = mysql_parsed.hostname, mysql_parsed.port
if mysql_parsed.scheme == 'mysqls': # Listen over stunnel
......
......@@ -57,7 +57,7 @@ class Recipe(BaseSlapRecipe):
try:
self.slave_partition_configuration_dict_list.append(
self._installSlavePartition(slave_partition))
except SlavePartitionError, e:
except SlavePartitionError as e:
self.logger.warning('Slave Parttion %r not installed, issue: %r'%(
slave_partition.getId(), e))
# Installs wrappers
......
......@@ -34,13 +34,13 @@ import stat
import netaddr
import time
import re
import urlparse
from six.moves.urllib.parse import urlunparse
import json
# Use to do from slapos.recipe.librecipe import GenericBaseRecipe
from generic import GenericBaseRecipe
from genericslap import GenericSlapRecipe
from filehash import filehash
from .generic import GenericBaseRecipe
from .genericslap import GenericSlapRecipe
from .filehash import filehash
# Utility functions to (de)serialise live python objects in order to send them
# to master.
......@@ -324,7 +324,7 @@ class BaseSlapRecipe:
if port is not None:
netloc += ':%s' % port
url = urlparse.urlunparse((scheme, netloc, path, params, query, fragment))
url = urlunparse((scheme, netloc, path, params, query, fragment))
return url
from __future__ import print_function
import sys
import os
import signal
......@@ -5,6 +7,8 @@ import subprocess
from collections import defaultdict
from inotify_simple import INotify, flags
import six
def _wait_files_creation(file_list):
# Establish a list of directory and subfiles.
# and test existence before watching, so that we don't miss an event.
......@@ -14,7 +18,7 @@ def _wait_files_creation(file_list):
directories[dirname][filename] = os.path.lexists(f)
def all_files_exists():
return all(all(files.itervalues()) for files in directories.itervalues())
return all(all(six.itervalues(files)) for files in six.itervalues(directories))
with INotify() as inotify:
watchdescriptors = {inotify.add_watch(dirname,
......@@ -101,7 +105,7 @@ def generic_exec(args, extra_environ=None, wait_list=None,
child_pg = None
def sig_handler(sig, frame):
print 'Received signal %r, killing children and exiting' % sig
print('Received signal %r, killing children and exiting' % sig)
if child_pg is not None:
os.killpg(child_pg, signal.SIGHUP)
os.killpg(child_pg, signal.SIGTERM)
......@@ -116,7 +120,7 @@ def execute_with_signal_translation(args):
child = subprocess.Popen(args, close_fds=True, preexec_fn=os.setsid)
child_pg = child.pid
try:
print 'Process %r started' % (args, )
print('Process %r started' % (args, ))
signal.pause()
finally:
os.killpg(child_pg, signal.SIGHUP)
......
......@@ -24,6 +24,7 @@
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
from __future__ import print_function
import hashlib
import shutil
import os
......@@ -91,6 +92,6 @@ if __name__ == '__main__':
if len(sys.argv) == 1:
raise ValueError("Not enough command line arguments")
if len(sys.argv) == 2:
print sys.argv[1], '-', pathhash(sys.argv[1])
print(sys.argv[1], '-', pathhash(sys.argv[1]))
else:
print sys.argv[2], '-', pathhash(sys.argv[2], sys.argv[1])
print(sys.argv[2], '-', pathhash(sys.argv[2], sys.argv[1]))
......@@ -35,8 +35,12 @@ import inspect
import re
import shutil
import stat
import urllib
import urlparse
from six.moves.urllib.parse import quote
import itertools
import six
from six.moves import map
from six.moves.urllib.parse import urlunparse
import pkg_resources
import zc.buildout
......@@ -90,7 +94,7 @@ class GenericBaseRecipe(object):
"""Options Hook method. This method can be overriden in child classes"""
return
def createFile(self, name, content, mode=0600):
def createFile(self, name, content, mode=0o600):
"""Create a file with content
The parent directory should exists, else it would raise IOError"""
......@@ -117,7 +121,7 @@ class GenericBaseRecipe(object):
f.write(content)
return os.path.abspath(name)
def createExecutable(self, name, content, mode=0700):
def createExecutable(self, name, content, mode=0o700):
return self.createFile(name, content, mode)
def addLineToFile(self, filepath, line, encoding='utf8'):
......@@ -148,9 +152,9 @@ class GenericBaseRecipe(object):
module, function = function
path, filename = os.path.split(os.path.abspath(name))
assert not isinstance(args, (basestring, dict)), args
args = map(repr, args)
args += map('%s=%r'.__mod__, kw.iteritems())
assert not isinstance(args, (six.string_types, dict)), args
args = itertools.chain(map(repr, args),
map('%s=%r'.__mod__, six.iteritems(kw)))
return zc.buildout.easy_install.scripts(
[(filename, module, function)], self._ws, sys.executable,
......@@ -173,12 +177,12 @@ class GenericBaseRecipe(object):
lines = ['#!/bin/sh']
if env:
for k, v in sorted(env.iteritems()):
for k, v in sorted(six.iteritems(env)):
lines.append('export %s=%s' % (k, shlex.quote(v)))
lines.append('exec')
args = map(shlex.quote, args)
args = list(map(shlex.quote, args))
args.append('"$@"')
for arg in args:
if len(lines[-1]) < 40:
......@@ -188,9 +192,9 @@ class GenericBaseRecipe(object):
lines.append('\t' + arg)
lines.append('')
return self.createFile(path, '\n'.join(lines), 0700)
return self.createFile(path, '\n'.join(lines), 0o700)
def createDirectory(self, parent, name, mode=0700):
def createDirectory(self, parent, name, mode=0o700):
path = os.path.join(parent, name)
if not os.path.exists(path):
os.mkdir(path, mode)
......@@ -240,9 +244,9 @@ class GenericBaseRecipe(object):
netloc = ''
if auth is not None:
auth = tuple(auth)
netloc = urllib.quote(str(auth[0])) # Login
netloc = quote(str(auth[0])) # Login
if len(auth) > 1:
netloc += ':%s' % urllib.quote(auth[1]) # Password
netloc += ':%s' % quote(auth[1]) # Password
netloc += '@'
# host is an ipv6 address whithout brackets
......@@ -254,7 +258,7 @@ class GenericBaseRecipe(object):
if port is not None:
netloc += ':%s' % port
url = urlparse.urlunparse((scheme, netloc, path, params, query, fragment))
url = urlunparse((scheme, netloc, path, params, query, fragment))
return url
......
......@@ -27,7 +27,7 @@
from slapos import slap
import time
from generic import GenericBaseRecipe
from .generic import GenericBaseRecipe
CONNECTION_CACHE = {}
......
......@@ -27,6 +27,7 @@
import os
from slapos.recipe.librecipe import GenericBaseRecipe
import six
class Recipe(GenericBaseRecipe):
......@@ -36,7 +37,7 @@ class Recipe(GenericBaseRecipe):
self.mode = int(self.directory.pop('mode', '0777'), 8)
def install(self):
for path in sorted(self.directory.itervalues()):
for path in sorted(six.itervalues(self.directory)):
if path and not os.path.isdir(path):
os.makedirs(path, self.mode)
# WARNING: This recipe is currently used to create directories that will
......
......@@ -25,6 +25,8 @@
#
##############################################################################
from __future__ import print_function
import sys
import pkg_resources
from logging import Formatter
......@@ -41,7 +43,7 @@ class NoSQLTestBed(BaseSlapRecipe):
testbed = plugin_class()
except:
print Formatter().formatException(sys.exc_info())
print(Formatter().formatException(sys.exc_info()))
return None
software_type = self.parameter_dict.get('slap_software_type', 'default')
......
......@@ -25,12 +25,14 @@
#
##############################################################################
from __future__ import print_function
import json
import os
import subprocess
import sys
import textwrap
import urlparse
from six.moves.urllib.parse import urlparse
from slapos.recipe.librecipe import GenericSlapRecipe
from slapos.recipe.dropbear import KnownHostsFile
......@@ -208,7 +210,7 @@ class Recipe(GenericSlapRecipe, Notify, Callback):
# This behavior has been removed to accelerate deployment of the
# Software Release. The buildout, instead of failing, can process
# other sections, which will return parameters to the main instance faster
parsed_url = urlparse.urlparse(url)
parsed_url = urlparse(url)
slave_type = entry['type']
if not slave_type in ['pull', 'push']:
......@@ -216,7 +218,7 @@ class Recipe(GenericSlapRecipe, Notify, Callback):
slave_id = entry['notification-id']
print 'Processing PBS slave %s with type %s' % (slave_id, slave_type)
print('Processing PBS slave %s with type %s' % (slave_id, slave_type))
path_list.append(self.createPythonScript(
os.path.join(self.options['promises-directory'], "ssh-to-%s" % slave_id),
......
......@@ -25,7 +25,7 @@
#
##############################################################################
import md5
import hashlib
import os
import subprocess
import textwrap
......@@ -195,7 +195,7 @@ class Recipe(GenericBaseRecipe):
password = self.options['password']
# encrypt the password to avoid storing in the logs
enc_password = 'md5' + md5.md5(password+user).hexdigest()
enc_password = 'md5' + hashlib.md5(password+user).hexdigest()
self.runPostgresCommand(cmd="""ALTER USER "%s" ENCRYPTED PASSWORD '%s'""" % (user, enc_password))
......
......@@ -24,9 +24,11 @@
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
from __future__ import print_function
import zc.buildout
from slapos.recipe.librecipe import wrap
from slapos.recipe.librecipe import GenericSlapRecipe
import six
CONNECTION_PARAMETER_STRING = 'connection-'
......@@ -77,9 +79,9 @@ class PublishSection(GenericSlapRecipe):
for section in self.options['section-list'].strip().split():
section = section.strip()
options = self.buildout[section].copy()
for k, v in options.iteritems():
for k, v in six.iteritems(options):
if k.startswith(CONNECTION_PARAMETER_STRING):
print k, v
print(k, v)
publish_dict[k.lstrip(CONNECTION_PARAMETER_STRING)] = v
self.setConnectionDict(publish_dict)
return []
......
......@@ -27,6 +27,7 @@
from collections import defaultdict
from .librecipe import unwrap, wrap, GenericSlapRecipe
import six
def volatileOptions(options, volatile):
def copy():
......@@ -109,9 +110,9 @@ class Recipe(GenericSlapRecipe):
publish = False
publish_dict = {}
try:
for init_section, init in init.iteritems():
for init_section, init in six.iteritems(init):
override = {}
for k, v in init.iteritems():
for k, v in six.iteritems(init):
try:
override[v] = published_dict[k]
except KeyError:
......@@ -120,7 +121,7 @@ class Recipe(GenericSlapRecipe):
init_section = buildout[init_section]
assert buildout.Options is Options
new = {}
for k, v in init.iteritems():
for k, v in six.iteritems(init):
try:
publish_dict[k] = new[v] = init_section.pop(v)
except KeyError:
......
......@@ -172,7 +172,7 @@ class Password(object):
raise
fd = os.open(self.storage_path,
os.O_CREAT | os.O_EXCL | os.O_WRONLY | os.O_TRUNC, 0600)
os.O_CREAT | os.O_EXCL | os.O_WRONLY | os.O_TRUNC, 0o600)
try:
os.write(fd, self.passwd)
finally:
......
......@@ -123,7 +123,7 @@ class Recipe(GenericBaseRecipe):
token_dict[reference] = new_token
to_add_dict[reference] = new_token
for reference in token_dict.keys():
for reference in list(token_dict):
if not reference in reference_list:
# This slave instance is destroyed ?
to_remove_dict[reference] = token_dict.pop(reference)
......