wip

component/phantomjs/buildout.cfg

@@ -2,7 +2,6 @@
 extends =
   ../fontconfig/buildout.cfg
   ../libexpat/buildout.cfg
-  ../dash/buildout.cfg
 parts =
   phantomjs
 
@@ -19,17 +18,19 @@ x86 = https://bitbucket.org/ariya/phantomjs/downloads/phantomjs-1.9.7-linux-i686
 x86-64 = https://bitbucket.org/ariya/phantomjs/downloads/phantomjs-1.9.7-linux-x86_64.tar.bz2 f278996c3edd0e8d8ec4893807f27d71
 
 script =
-  if not self.options.get('url'): self.options['url'], self.options['md5sum'] = self.options[guessPlatform()].split(' ')
-  extract_dir = self.extract(self.download(self.options['url'], self.options.get('md5sum')))
+  if not self.options.get('url'):
+    self.options['url'], self.options['md5sum'] = \
+      self.options[guessPlatform()].split(' ')
+  extract_dir = self.extract(self.download(self.options['url'],
+                             self.options.get('md5sum')))
   workdir = guessworkdir(extract_dir)
   self.copyTree(workdir, "%(location)s")
   wrapper_location = os.path.join("%(location)s", "phantomjs-slapos")
-  wrapper = open(wrapper_location, 'w')
-  wrapper.write("""#!${dash:location}/bin/dash
+  with open(wrapper_location, 'w') as wrapper:
+    wrapper.write("""#!/bin/sh
   cd %(location)s
   export LD_LIBRARY_PATH=%(location)s:${freetype:location}/lib/:${fontconfig:location}/lib/:${libexpat:location}/lib
   export PATH=${fontconfig:location}/bin:$PATH
-  exec %(location)s/bin/phantomjs $*""")
-  wrapper.flush()
-  wrapper.close()
-  os.chmod(wrapper_location, 0755)
+  exec %(location)s/bin/phantomjs "$@"
+  """)
+  os.chmod(wrapper_location, 0o755)

slapos/recipe/librecipe/__init__.py

@@ -34,13 +34,13 @@ import stat
 import netaddr
 import time
 import re
-import urlparse
+from six.moves.urllib import parse as urlparse
 import json
 
 # Use to do from slapos.recipe.librecipe import GenericBaseRecipe
-from generic import GenericBaseRecipe
-from genericslap import GenericSlapRecipe
-from filehash import filehash
+from .generic import GenericBaseRecipe
+from .genericslap import GenericSlapRecipe
+from .filehash import filehash
 
 # Utility functions to (de)serialise live python objects in order to send them
 # to master.

slapos/recipe/librecipe/filehash.py

@@ -24,6 +24,7 @@
 # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
 #
 ##############################################################################
+from __future__ import print_function
 import hashlib
 import shutil
 import os
@@ -91,6 +92,6 @@ if __name__ == '__main__':
   if len(sys.argv) == 1:
     raise ValueError("Not enough command line arguments")
   if len(sys.argv) == 2:
-    print sys.argv[1], '-', pathhash(sys.argv[1])
+    print(sys.argv[1], '-', pathhash(sys.argv[1]))
   else:
-    print sys.argv[2], '-', pathhash(sys.argv[2], sys.argv[1])
+    print(sys.argv[2], '-', pathhash(sys.argv[2], sys.argv[1]))

slapos/recipe/librecipe/generic.py

@@ -34,7 +34,10 @@ import inspect
 import re
 import shutil
 import urllib
-import urlparse
+from six import iteritems
+from six.moves import map
+from six.moves.urllib import parse
+
 
 import pkg_resources
 import zc.buildout
@@ -88,7 +91,7 @@ class GenericBaseRecipe(object):
     """Options Hook method. This method can be overriden in child classes"""
     return
 
-  def createFile(self, name, content, mode=0600):
+  def createFile(self, name, content, mode=0o600):
     """Create a file with content
 
     The parent directory should exists, else it would raise IOError"""
@@ -97,7 +100,7 @@ class GenericBaseRecipe(object):
       os.chmod(fileobject.name, mode)
     return os.path.abspath(name)
 
-  def createExecutable(self, name, content, mode=0700):
+  def createExecutable(self, name, content, mode=0o700):
     return self.createFile(name, content, mode)
 
   def addLineToFile(self, filepath, line, encoding='utf8'):
@@ -153,12 +156,12 @@ class GenericBaseRecipe(object):
     lines = ['#!/bin/sh']
 
     if env:
-      for k, v in sorted(env.iteritems()):
+      for k, v in sorted(iteritems(env)):
         lines.append('export %s=%s' % (k, shlex.quote(v)))
 
     lines.append('exec')
 
-    args = map(shlex.quote, args)
+    args = list(map(shlex.quote, args))
     args.append('"$@"')
     for arg in args:
       if len(lines[-1]) < 40:
@@ -168,9 +171,9 @@ class GenericBaseRecipe(object):
         lines.append('\t' + arg)
 
     lines.append('')
-    return self.createFile(path, '\n'.join(lines), 0700)
+    return self.createFile(path, '\n'.join(lines), 0o700)
 
-  def createDirectory(self, parent, name, mode=0700):
+  def createDirectory(self, parent, name, mode=0o700):
     path = os.path.join(parent, name)
     if not os.path.exists(path):
       os.mkdir(path, mode)
@@ -234,7 +237,7 @@ class GenericBaseRecipe(object):
     if port is not None:
       netloc += ':%s' % port
 
-    url = urlparse.urlunparse((scheme, netloc, path, params, query, fragment))
+    url = parse.urlunparse((scheme, netloc, path, params, query, fragment))
 
     return url
 

slapos/recipe/librecipe/genericslap.py

@@ -27,7 +27,7 @@
 from slapos import slap
 import time
 
-from generic import GenericBaseRecipe
+from .generic import GenericBaseRecipe
 
 CONNECTION_CACHE = {}
 

slapos/recipe/mkdirectory.py

@@ -27,6 +27,7 @@
 import os
 
 from slapos.recipe.librecipe import GenericBaseRecipe
+from six import itervalues
 
 class Recipe(GenericBaseRecipe):
 
@@ -36,7 +37,7 @@ class Recipe(GenericBaseRecipe):
     self.mode = int(self.directory.pop('mode', '0777'), 8)
 
   def install(self):
-    for path in sorted(self.directory.itervalues()):
+    for path in sorted(itervalues(self.directory)):
       if path and not os.path.isdir(path):
         os.makedirs(path, self.mode)
     # WARNING: This recipe is currently used to create directories that will

slapos/recipe/slapconfiguration.py

@@ -31,7 +31,8 @@ import os
 
 import slapos.slap
 from slapos.recipe.librecipe import unwrap
-from ConfigParser import RawConfigParser
+import six
+from six.moves.configparser import RawConfigParser
 from netaddr import valid_ipv4, valid_ipv6
 from slapos.util import mkdir_p
 from slapos import format as slapformat
@@ -115,7 +116,7 @@ class Recipe(object):
                                       buildout['buildout']['directory'])
 
       match = self.OPTCRE_match
-      for key, value in parameter_dict.iteritems():
+      for key, value in six.iteritems(parameter_dict):
           if match(key) is not None:
               continue
           options['configuration.' + key] = value
@@ -157,11 +158,10 @@ class Recipe(object):
               options[his_key.replace('_', '-')] = value
       # Get Instance and root instance title or return UNKNOWN if not set
       options['instance-title'] = parameter_dict.pop('instance_title',
-                                            'UNKNOWN Instance').encode('UTF-8')
+                                            'UNKNOWN Instance')
       options['root-instance-title'] = parameter_dict.pop('root_instance_title',
-                                            'UNKNOWN').encode('UTF-8')
-      options['instance-guid'] = computer_partition.getInstanceGuid() \
-          .encode('UTF-8')
+                                            'UNKNOWN')
+      options['instance-guid'] = computer_partition.getInstanceGuid()
 
       ipv4_set = set()
       v4_add = ipv4_set.add
@@ -204,9 +204,23 @@ class Recipe(object):
 
       # also export single ip values for those recipes that don't support sets.
       if ipv4_set:
-          options['ipv4-random'] = list(ipv4_set)[0].encode('UTF-8')
+          options['ipv4-random'] = min(ipv4_set)
       if ipv6_set:
-          options['ipv6-random'] = list(ipv6_set)[0].encode('UTF-8')
+          options['ipv6-random'] = min(ipv6_set)
+      if route_ipv4_set:
+        options['tap-ipv4'] = min(route_ipv4_set)
+        options['tap-network-information-dict'] = dict(ipv4=route_ipv4_set,
+                                    netmask=route_mask_set,
+                                    gateway=route_gw_set,
+                                    network=route_network_set)
+      else:
+        options['tap-network-information-dict'] = {}
+      if route_gw_set:
+        options['tap-gateway'] = min(route_gw_set)
+      if route_mask_set:
+        options['tap-netmask'] = min(route_mask_set)
+      if route_network_set:
+        options['tap-network'] = min(route_network_set)
 
       storage_home = options.get('storage-home')
       storage_dict = {}
@@ -265,8 +279,11 @@ class JsonDump(Recipe):
   def __init__(self, buildout, name, options):
     parameter_dict = self.fetch_parameter_dict(options)
     self._json_output = options['json-output']
-    with os.fdopen(os.open(self._json_output, os.O_WRONLY | os.O_CREAT, 0600), 'w') as fout:
-      fout.write(json.dumps(parameter_dict, indent=2, sort_keys=True))
+    # XXX: do not touch file if there's no change to avoid excessive IO
+    #      (see https://lab.nexedi.com/nexedi/slapos.recipe.template/commit/14d26bc8c77a1940f389026bdbd3a9b229b241f4
+    #       for an example to fix this)
+    with os.fdopen(os.open(self._json_output, os.O_WRONLY | os.O_CREAT, 0o600), 'w') as fout:
+      json.dump(parameter_dict, fout, indent=2, sort_keys=True)
 
     def install(self):
         return [self._json_output]

software/slapos-testing/software-py3.cfg

+[buildout]
+extends =
+  ../../component/python3/buildout.cfg
+  software.cfg
+python = python3.5
+
+[nghttp2]
+environment =
+  PATH=${autoconf:location}/bin:${automake:location}/bin:${libtool:location}/bin:${m4:location}/bin:${python3.5:location}/bin:%(PATH)s
+
+[supervisor-repository]
+<= git-clone-repository
+repository = https://github.com/Supervisor/supervisor.git
+
+[supervisor-develop]
+recipe = zc.recipe.egg:develop
+egg = supervisor
+setup = ${supervisor-repository:location}
+
+[slapos.core-repository]
+branch = py3
+
+[slapos.cookbook-repository]
+branch = py3
+
+[eggs]
+recipe = zc.recipe.egg
+eggs += ${supervisor-develop:egg}
+patch-binary = ${patch:location}/bin/patch
+xml-marshaller-patches = ${:_profile_base_location_}/../../component/egg-patch/xml_marshaller/py3.patch#db7aae991783d0de3e66490a3dae9911
+xml-marshaller-options = -p0
+
+[versions]
+netifaces = 0.10.7
+supervisor =
+xml-marshaller = 0.9.7+SlapOSPatched001
+slapos.recipe.build = 0.38