Skip to content

slapos
slapos
Commit f5c6160c authored by Bryton Lacquement's avatar Bryton Lacquement 🚪
Browse files
Options

wip

parent de75a765
Show whitespace changes
Inline Side-by-side
Showing with 470 additions and 35 deletions
component/egg-patch/xml_marshaller/py3.patch 0 → 100644
View file @ f5c6160c
Index: src/xml_marshaller/xml_marshaller.py
===================================================================
--- src/xml_marshaller/xml_marshaller.py (revision 39721)
+++ src/xml_marshaller/xml_marshaller.py (working copy)
@@ -9,7 +9,8 @@
# same name, with four functions:
# dump(value, file), load(file)
# dumps(value), loads(string)
-from types import ClassType, TypeType
+from functools import cmp_to_key
+from six import reraise
import sys
from xml.sax import make_parser
from xml.sax.saxutils import escape, unescape
@@ -16,7 +17,7 @@
from lxml.sax import ElementTreeContentHandler, saxify
from lxml import etree
from lxml.builder import ElementMaker
-from cStringIO import StringIO
+from io import BytesIO
# Basic marshaller class, customizable by overriding it and
# changing various attributes and methods.
@@ -24,12 +25,19 @@
# also be a stupid hack.
MARSHAL_PREFIX = 'marshal'
+try:
+ cmp
+except NameError:
+ cmp = lambda x, y: (x > y) - (x < y)
+
+@cmp_to_key
def version_independent_cmp(a, b):
- ta = type(a)
- tb = type(b)
- if ta is not tb:
- return cmp(ta.__name__, tb.__name__)
- return cmp(a, b)
+ a = a[0]
+ b = b[0]
+ try:
+ return cmp(a, b)
+ except TypeError:
+ return cmp(repr(a), repr(b))
class Marshaller(object):
@@ -103,7 +111,7 @@
def _marshal(self, value, kw):
t = type(value)
i = str(id(value))
- if kw.has_key(i):
+ if i in kw:
return self.m_reference(value, kw)
else:
method_id = 'm_%s' % (type(value).__name__,)
@@ -119,7 +127,7 @@
# Utility function, used for types that aren't implemented
def m_unimplemented(self, value, kw):
- raise ValueError, ("Marshalling of object %r unimplemented or not supported in this DTD" % value)
+ raise ValueError("Marshalling of object %r unimplemented or not supported in this DTD" % value)
# The real entry point for marshalling, to handle properly
# and cleanly any root tag or tags necessary for the marshalled
@@ -137,29 +145,28 @@
return self.tag_reference(id='i%s' % i)
def m_string(self, value, kw):
- return self.tag_string(escape(str(value)))
+ return self.tag_string(escape(value))
- def m_unicode(self, value, kw):
- return self.tag_unicode(value)
# Since Python 2.2, the string type has a name of 'str'
# To avoid having to rewrite all classes that implement m_string
# we delegate m_str to m_string.
def m_str(self, value, kw):
return self.m_string(value, kw)
+
+ def m_bytes(self, value, kw):
+ return self.m_string(value.decode('utf-8'), kw)
+ m_unicode = m_str
+
def m_int(self, value, kw):
- return self.tag_int(str(value))
+ return (self.tag_int if -1e24 < value < 1e24 else
+ self.tag_long)(str(value))
+ m_long = m_int
+
def m_float(self, value, kw):
return self.tag_float(str(value))
- def m_long(self, value, kw):
- value = str(value)
- if value[-1] == 'L':
- # some Python versions append and 'L'
- value = value[:-1]
- return self.tag_long(value)
-
def m_tuple(self, value, kw):
xml_tree = self.tag_tuple()
for elem in value:
@@ -182,11 +189,10 @@
kw[str(id(value))] = i
kw[i] = value
xml_tree = self.tag_dictionary(id='i%s' % i)
- item_list = value.items()
# Sort the items to allow reproducable results across Python
# versions
- item_list.sort(version_independent_cmp)
- for key, v in item_list:
+ for key, v in sorted(value.items(),
+ key=version_independent_cmp):
xml_tree.append(self._marshal(key, kw))
xml_tree.append(self._marshal(v, kw))
return xml_tree
@@ -278,7 +284,6 @@
'float': ('um_start_float', 'um_end_float'),
'long': ('um_start_long', 'um_end_long'),
'string': ('um_start_string', 'um_end_string'),
- 'unicode': ('um_start_unicode', 'um_end_unicode'),
'tuple': ('um_start_tuple', 'um_end_tuple'),
'list': ('um_start_list', 'um_end_list'),
'dictionary': ('um_start_dictionary', 'um_end_dictionary'),
@@ -290,6 +295,7 @@
'set': ('um_start_set', 'um_end_set'),
'bool': ('um_start_bool', 'um_end_bool'),
}
+ unmarshal_meth['unicode'] = unmarshal_meth['string'] # BBB
def __init__(self):
# Find the named methods, and convert them to the actual
@@ -339,8 +345,7 @@
# Instantiate a new object; unmarshalling isn't thread-safe
# because it modifies attributes on the object.
m = self.__class__()
- file_object = StringIO(string)
- return m._load(file_object)
+ return m._load(BytesIO(string))
# Basic unmarshalling routine; it creates a SAX XML parser,
# registers self as the SAX handler, parses it, and returns
@@ -394,8 +399,7 @@
ns_uri, local_name = ns_name
sm, em = self.unmarshal_meth[local_name]
if sm is not None:
- attrib = {}
- [attrib.update({k[1]: v}) for k, v in attrs.items()]
+ attrib = {k[1]: v for k, v in attrs.items()}
return sm(local_name, attrib)
def characters(self, data):
@@ -420,20 +424,17 @@
# "empty" state.
def um_start_root(self, name, attrs):
if self.kw or self.data_stack:
- raise ValueError, "root element %s found elsewhere than root" \
- % repr(name)
+ raise ValueError("root element %r found elsewhere than root"
+ % name)
def um_start_reference(self, name, attrs):
- assert attrs.has_key('id')
- id = attrs['id']
- assert self.kw.has_key(id)
- self.data_stack.append(self.kw[id])
+ self.data_stack.append(self.kw[attrs['id']])
def um_start_generic(self, name, attrs):
self.data_stack.append([])
self.accumulating_chars = 1
- um_start_float = um_start_long = um_start_string = um_start_unicode = um_start_generic
+ um_start_float = um_start_long = um_start_string = um_start_generic
um_start_complex = um_start_code = um_start_none = um_start_generic
um_start_int = um_start_generic
um_start_bool = um_start_generic
@@ -440,20 +441,9 @@
def um_end_string(self, name):
ds = self.data_stack
- value = unescape(''.join(ds[-1]))
- if isinstance(value, unicode):
- value = value.encode('utf-8')
- ds[-1] = value
+ ds[-1] = unescape(''.join(ds[-1]))
self.accumulating_chars = 0
- def um_end_unicode(self, name):
- ds = self.data_stack
- value = unescape(''.join(ds[-1]))
- if not isinstance(value, unicode):
- value = value.decode('utf-8')
- ds[-1] = value
- self.accumulating_chars = 0
-
def um_end_int(self, name):
ds = self.data_stack
ds[-1] = ''.join(ds[-1])
@@ -463,7 +453,7 @@
def um_end_long(self, name):
ds = self.data_stack
ds[-1] = ''.join(ds[-1])
- ds[-1] = long(ds[-1])
+ ds[-1] = int(ds[-1])
self.accumulating_chars = 0
def um_end_float(self, name):
@@ -497,9 +487,8 @@
def um_start_list(self, name, attrs):
self.data_stack.append(LIST)
L = []
- if attrs.has_key('id'):
- id = attrs['id']
- self.kw[id] = L
+ if 'id' in attrs:
+ self.kw[attrs['id']] = L
self.data_stack.append(L)
def um_start_set(self, name, attrs):
@@ -553,9 +542,8 @@
def um_start_dictionary(self, name, attrs):
self.data_stack.append(DICT)
d = {}
- if attrs.has_key('id'):
- id = attrs['id']
- self.kw[id] = d
+ if 'id' in attrs:
+ self.kw[attrs['id']] = d
self.data_stack.append(d)
def um_end_dictionary(self, name):
@@ -575,9 +563,8 @@
module = attrs['module']
classname = attrs['class']
value = _EmptyClass()
- if attrs.has_key('id'):
- id = attrs['id']
- self.kw[id] = value
+ if 'id' in attrs:
+ self.kw[attrs['id']] = value
self.data_stack.append(value)
self.data_stack.append(module)
self.data_stack.append(classname)
@@ -585,18 +572,20 @@
def um_end_instance(self, name):
value, module, classname, initargs, kw = self.data_stack[-5:]
klass = self.find_class(module, classname)
- if (not initargs and (isinstance(klass, (ClassType, TypeType))) and
+ if (not initargs and isinstance(klass, type) and
not hasattr(klass, '__getinitargs__')):
value = klass()
else:
try:
- value = apply(klass, initargs)
- except TypeError, err:
- raise TypeError, 'in constructor for %s: %s' % (
- klass.__name__, str(err)), sys.exc_info()[2]
+ value = klass(*initargs)
+ except TypeError as err:
+ reraise(TypeError, TypeError('in constructor for %s: %s' % (
+ klass.__name__, err)), sys.exc_info()[2])
# Now set the object's attributes from the marshalled dictionary
for k, v in kw.items():
+ if not isinstance(k, str):
+ k = k.decode()
setattr(value, k, v)
self.data_stack[-5:] = [value]
Index: src/xml_marshaller/tests/test_xml_marshaller.py
===================================================================
--- src/xml_marshaller/tests/test_xml_marshaller.py (revision 39721)
+++ src/xml_marshaller/tests/test_xml_marshaller.py (working copy)
@@ -3,7 +3,7 @@
import xml_marshaller
from xml_marshaller.xml_marshaller import load, loads, dump, dumps, dump_ns,\
dumps_ns
-from StringIO import StringIO
+from io import BytesIO
from lxml import etree
import pkg_resources
@@ -42,13 +42,10 @@
# Disabled for xml with namespaces.
# Because URI is not predictable
return
- if not isinstance(xml_data, str):
- xml_string = xml_data.read()
- xml_data.seek(0)
- else:
- xml_string = xml_data
+ if not isinstance(xml_data, bytes):
+ xml_data = xml_data.getvalue().decode("utf-8")
- document_tree = etree.fromstring(xml_string)
+ document_tree = etree.fromstring(xml_data)
is_validated = xmlschema.validate(document_tree)
log = xmlschema.error_log
error = log.last_error
@@ -57,14 +54,14 @@
def test_string_serialisation(self):
"""
"""
- data_list = [None, 1, pow(2, 123L), 19.72, 1+5j,
+ data_list = [None, 1, 1<<123, 19.72, 1+5j,
"here is a string & a <fake tag>",
(1, 2, 3),
- ['alpha', 'beta', 'gamma', [None, 1, pow(2, 123L), 19.72,
+ ['alpha', 'beta', 'gamma', [None, 1, 1<<123, 19.72,
1+5j, "& a <fake tag>"]],
{'key': 'value', 1: 2},
- 'éàù^ç'.decode('utf-8'),
- set(('a', 1,)),
+ u'éàù^ç',
+ {'a', 1},
True,
False,
]
@@ -80,14 +77,14 @@
def test_file_serialisation(self):
"""
"""
- data_list = [None, 1, pow(2, 123L), 19.72, 1+5j,
+ data_list = [None, 1, 1<<123, 19.72, 1+5j,
"here is a string & a <fake tag>",
(1, 2, 3),
- ['alpha', 'beta', 'gamma', [None, 1, pow(2, 123L), 19.72,
- 1+5j, "& a <fake tag>"]],
+ ['alpha', 'beta', 'gamma', [None, 1, 1<<123, 19.72,
+ 1+5j, "& a <fake tag>"]],
{'key': 'value', 1: 2},
- 'éàù^ç'.decode('utf-8'),
- set(('a', 1,)),
+ u'éàù^ç',
+ {'a', 1},
True,
False,
]
@@ -96,7 +93,7 @@
else:
dumper = dump
for item in data_list:
- file_like_object = StringIO()
+ file_like_object = BytesIO()
dumper(item, file_like_object)
file_like_object.seek(0)
self._checkXML(file_like_object)
@@ -107,8 +104,8 @@
"""
instance = _A()
instance.subobject = _B()
- instance.subobject.list_attribute=[None, 1, pow(2, 123L), 19.72, 1+5j,
- "here is a string & a <fake tag>"]
+ instance.subobject.list_attribute=[None, 1, 1<<123, 19.72, 1+5j,
+ "here is a string & a <fake tag>"]
instance.self = instance
if self.use_namespace_uri:
Index: setup.py
===================================================================
--- setup.py (revision 39721)
+++ setup.py (working copy)
@@ -33,6 +33,6 @@
package_dir={'': 'src'},
include_package_data=True,
zip_safe=False,
- install_requires=['lxml',],
+ install_requires=['lxml','six'],
test_suite='xml_marshaller',
)
component/phantomjs/buildout.cfg
View file @ f5c6160c
......@@ -2,7 +2,6 @@
extends =
../fontconfig/buildout.cfg
../libexpat/buildout.cfg
../dash/buildout.cfg
parts =
phantomjs
......@@ -19,17 +18,19 @@ x86 = https://bitbucket.org/ariya/phantomjs/downloads/phantomjs-1.9.7-linux-i686
x86-64 = https://bitbucket.org/ariya/phantomjs/downloads/phantomjs-1.9.7-linux-x86_64.tar.bz2 f278996c3edd0e8d8ec4893807f27d71
script =
if not self.options.get('url'): self.options['url'], self.options['md5sum'] = self.options[guessPlatform()].split(' ')
extract_dir = self.extract(self.download(self.options['url'], self.options.get('md5sum')))
if not self.options.get('url'):
self.options['url'], self.options['md5sum'] = \
self.options[guessPlatform()].split(' ')
extract_dir = self.extract(self.download(self.options['url'],
self.options.get('md5sum')))
workdir = guessworkdir(extract_dir)
self.copyTree(workdir, "%(location)s")
wrapper_location = os.path.join("%(location)s", "phantomjs-slapos")
wrapper = open(wrapper_location, 'w')
wrapper.write("""#!${dash:location}/bin/dash
with open(wrapper_location, 'w') as wrapper:
wrapper.write("""#!/bin/sh
cd %(location)s
export LD_LIBRARY_PATH=%(location)s:${freetype:location}/lib/:${fontconfig:location}/lib/:${libexpat:location}/lib
export PATH=${fontconfig:location}/bin:$PATH
exec %(location)s/bin/phantomjs $*""")
wrapper.flush()
wrapper.close()
os.chmod(wrapper_location, 0755)
exec %(location)s/bin/phantomjs "$@"
""")
os.chmod(wrapper_location, 0o755)
slapos/recipe/librecipe/__init__.py
View file @ f5c6160c
......@@ -34,13 +34,13 @@ import stat
import netaddr
import time
import re
import urlparse
from six.moves.urllib import parse as urlparse
import json
# Use to do from slapos.recipe.librecipe import GenericBaseRecipe
from generic import GenericBaseRecipe
from genericslap import GenericSlapRecipe
from filehash import filehash
from .generic import GenericBaseRecipe
from .genericslap import GenericSlapRecipe
from .filehash import filehash
# Utility functions to (de)serialise live python objects in order to send them
# to master.
......
slapos/recipe/librecipe/filehash.py
View file @ f5c6160c
......@@ -24,6 +24,7 @@
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
from __future__ import print_function
import hashlib
import shutil
import os
......@@ -91,6 +92,6 @@ if __name__ == '__main__':
if len(sys.argv) == 1:
raise ValueError("Not enough command line arguments")
if len(sys.argv) == 2:
print sys.argv[1], '-', pathhash(sys.argv[1])
print(sys.argv[1], '-', pathhash(sys.argv[1]))
else:
print sys.argv[2], '-', pathhash(sys.argv[2], sys.argv[1])
print(sys.argv[2], '-', pathhash(sys.argv[2], sys.argv[1]))
slapos/recipe/librecipe/generic.py
View file @ f5c6160c
......@@ -34,7 +34,10 @@ import inspect
import re
import shutil
import urllib
import urlparse
from six import iteritems
from six.moves import map
from six.moves.urllib import parse
import pkg_resources
import zc.buildout
......@@ -88,7 +91,7 @@ class GenericBaseRecipe(object):
"""Options Hook method. This method can be overriden in child classes"""
return
def createFile(self, name, content, mode=0600):
def createFile(self, name, content, mode=0o600):
"""Create a file with content
The parent directory should exists, else it would raise IOError"""
......@@ -97,7 +100,7 @@ class GenericBaseRecipe(object):
os.chmod(fileobject.name, mode)
return os.path.abspath(name)
def createExecutable(self, name, content, mode=0700):
def createExecutable(self, name, content, mode=0o700):
return self.createFile(name, content, mode)
def addLineToFile(self, filepath, line, encoding='utf8'):
......@@ -153,12 +156,12 @@ class GenericBaseRecipe(object):
lines = ['#!/bin/sh']
if env:
for k, v in sorted(env.iteritems()):
for k, v in sorted(iteritems(env)):
lines.append('export %s=%s' % (k, shlex.quote(v)))
lines.append('exec')
args = map(shlex.quote, args)
args = list(map(shlex.quote, args))
args.append('"$@"')
for arg in args:
if len(lines[-1]) < 40:
......@@ -168,9 +171,9 @@ class GenericBaseRecipe(object):
lines.append('\t' + arg)
lines.append('')
return self.createFile(path, '\n'.join(lines), 0700)
return self.createFile(path, '\n'.join(lines), 0o700)
def createDirectory(self, parent, name, mode=0700):
def createDirectory(self, parent, name, mode=0o700):
path = os.path.join(parent, name)
if not os.path.exists(path):
os.mkdir(path, mode)
......@@ -234,7 +237,7 @@ class GenericBaseRecipe(object):
if port is not None:
netloc += ':%s' % port
url = urlparse.urlunparse((scheme, netloc, path, params, query, fragment))
url = parse.urlunparse((scheme, netloc, path, params, query, fragment))
return url
......
slapos/recipe/librecipe/genericslap.py
View file @ f5c6160c
......@@ -27,7 +27,7 @@
from slapos import slap
import time
from generic import GenericBaseRecipe
from .generic import GenericBaseRecipe
CONNECTION_CACHE = {}
......
slapos/recipe/mkdirectory.py
View file @ f5c6160c
......@@ -27,6 +27,7 @@
import os
from slapos.recipe.librecipe import GenericBaseRecipe
from six import itervalues
class Recipe(GenericBaseRecipe):
......@@ -36,7 +37,7 @@ class Recipe(GenericBaseRecipe):
self.mode = int(self.directory.pop('mode', '0777'), 8)
def install(self):
for path in sorted(self.directory.itervalues()):
for path in sorted(itervalues(self.directory)):
if path and not os.path.isdir(path):
os.makedirs(path, self.mode)
# WARNING: This recipe is currently used to create directories that will
......
slapos/recipe/slapconfiguration.py
View file @ f5c6160c
......@@ -31,7 +31,8 @@ import os
import slapos.slap
from slapos.recipe.librecipe import unwrap
from ConfigParser import RawConfigParser
import six
from six.moves.configparser import RawConfigParser
from netaddr import valid_ipv4, valid_ipv6
from slapos.util import mkdir_p
from slapos import format as slapformat
......@@ -115,7 +116,7 @@ class Recipe(object):
buildout['buildout']['directory'])
match = self.OPTCRE_match
for key, value in parameter_dict.iteritems():
for key, value in six.iteritems(parameter_dict):
if match(key) is not None:
continue
options['configuration.' + key] = value
......@@ -157,11 +158,10 @@ class Recipe(object):
options[his_key.replace('_', '-')] = value
# Get Instance and root instance title or return UNKNOWN if not set
options['instance-title'] = parameter_dict.pop('instance_title',
'UNKNOWN Instance').encode('UTF-8')
'UNKNOWN Instance')
options['root-instance-title'] = parameter_dict.pop('root_instance_title',
'UNKNOWN').encode('UTF-8')
options['instance-guid'] = computer_partition.getInstanceGuid() \
.encode('UTF-8')
'UNKNOWN')
options['instance-guid'] = computer_partition.getInstanceGuid()
ipv4_set = set()
v4_add = ipv4_set.add
......@@ -204,9 +204,23 @@ class Recipe(object):
# also export single ip values for those recipes that don't support sets.
if ipv4_set:
options['ipv4-random'] = list(ipv4_set)[0].encode('UTF-8')
options['ipv4-random'] = min(ipv4_set)
if ipv6_set:
options['ipv6-random'] = list(ipv6_set)[0].encode('UTF-8')
options['ipv6-random'] = min(ipv6_set)
if route_ipv4_set:
options['tap-ipv4'] = min(route_ipv4_set)
options['tap-network-information-dict'] = dict(ipv4=route_ipv4_set,
netmask=route_mask_set,
gateway=route_gw_set,
network=route_network_set)
else:
options['tap-network-information-dict'] = {}
if route_gw_set:
options['tap-gateway'] = min(route_gw_set)
if route_mask_set:
options['tap-netmask'] = min(route_mask_set)
if route_network_set:
options['tap-network'] = min(route_network_set)
storage_home = options.get('storage-home')
storage_dict = {}
......@@ -265,8 +279,11 @@ class JsonDump(Recipe):
def __init__(self, buildout, name, options):
parameter_dict = self.fetch_parameter_dict(options)
self._json_output = options['json-output']
with os.fdopen(os.open(self._json_output, os.O_WRONLY | os.O_CREAT, 0600), 'w') as fout:
fout.write(json.dumps(parameter_dict, indent=2, sort_keys=True))
# XXX: do not touch file if there's no change to avoid excessive IO
# (see https://lab.nexedi.com/nexedi/slapos.recipe.template/commit/14d26bc8c77a1940f389026bdbd3a9b229b241f4
# for an example to fix this)
with os.fdopen(os.open(self._json_output, os.O_WRONLY | os.O_CREAT, 0o600), 'w') as fout:
json.dump(parameter_dict, fout, indent=2, sort_keys=True)
def install(self):
return [self._json_output]
......
software/slapos-testing/software-py3.cfg 0 → 100644
View file @ f5c6160c
[buildout]
extends =
../../component/python3/buildout.cfg
software.cfg
python = python3.5
[nghttp2]
environment =
PATH=${autoconf:location}/bin:${automake:location}/bin:${libtool:location}/bin:${m4:location}/bin:${python3.5:location}/bin:%(PATH)s
[supervisor-repository]
<= git-clone-repository
repository = https://github.com/Supervisor/supervisor.git
[supervisor-develop]
recipe = zc.recipe.egg:develop
egg = supervisor
setup = ${supervisor-repository:location}
[slapos.core-repository]
branch = py3
[slapos.cookbook-repository]
branch = py3
[eggs]
recipe = zc.recipe.egg
eggs += ${supervisor-develop:egg}
patch-binary = ${patch:location}/bin/patch
xml-marshaller-patches = ${:_profile_base_location_}/../../component/egg-patch/xml_marshaller/py3.patch#db7aae991783d0de3e66490a3dae9911
xml-marshaller-options = -p0
[versions]
netifaces = 0.10.7
supervisor =
xml-marshaller = 0.9.7+SlapOSPatched001
slapos.recipe.build = 0.38
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7