Python 3.6 support

Fix a bug occurring when setting python3.6 as ${buildout:executable} in profiles.


    ************ REBOOTSTRAP: IMPORTANT NOTICE ************
    bin/buildout is being reinstalled right now, as new python:
      /srv/slapgrid/slappart14/srv/testnode/cup/soft/4281ee9e21ffccb8d017eba71a9cd9b1/parts/python3.6/bin/python3.6
    is available, and buildout is using another python:
      /opt/slapgrid/4cca7f1c7db00b5e7a4348f8df5c24ab/parts/python2.7/bin/python2.7
    Buildout will be restarted automatically to have this change applied.
    ************ REBOOTSTRAP: IMPORTANT NOTICE ************

    networkcache: Downloading pypi-index-96af89cdde216340df5aec1e9a1392e0-setuptools==40.4.3...
    networkcache: ignored unhandled exception at /srv/slapgrid/slappart14/srv/testnode/cup/soft/4281ee9e21ffccb8d017eba71a9cd9b1/eggs/slapos.libnetworkcache-0.20-py2.7.egg/slapos/networkcachehelper.py:104
    HTTPError: HTTP Error 404: Not Found
    networkcache: Downloading pypi-index-99e2b3234a294248d07c2bb8e6698302-setuptools==40.4.3...
    networkcache: ignored unhandled exception at /srv/slapgrid/slappart14/srv/testnode/cup/soft/4281ee9e21ffccb8d017eba71a9cd9b1/eggs/slapos.libnetworkcache-0.20-py2.7.egg/slapos/networkcachehelper.py:104
    HTTPError: HTTP Error 404: Not Found
    networkcache: Downloading pypi-index-a46be9bd3756ada5c303353adbb6407f-setuptools==40.4.3...
    networkcache: ignored unhandled exception at /srv/slapgrid/slappart14/srv/testnode/cup/soft/4281ee9e21ffccb8d017eba71a9cd9b1/eggs/slapos.libnetworkcache-0.20-py2.7.egg/slapos/networkcachehelper.py:104
    HTTPError: HTTP Error 404: Not Found
    networkcache: Downloading pypi-index-45c869bb967e16b5e30ed107541a36be-setuptools==40.4.3...
    networkcache: ignored unhandled exception at /srv/slapgrid/slappart14/srv/testnode/cup/soft/4281ee9e21ffccb8d017eba71a9cd9b1/eggs/slapos.libnetworkcache-0.20-py2.7.egg/slapos/networkcachehelper.py:104
    HTTPError: HTTP Error 404: Not Found
    networkcache: Downloading slapos-buildout-c4345cb200e7d70be4984e98a0864d72...
    Downloaded https://files.pythonhosted.org/packages/6e/9c/6a003320b00ef237f94aa74e4ad66c57a7618f6c79d67527136e2544b728/setuptools-40.4.3.zip#sha256=acbc5740dd63f243f46c2b4b8e2c7fd92259c2ddb55a4115b16418a2ed371b15 from network cache.
    networkcache: Downloading pypi-index-0b71f171beaaa2eb14aa9fd650059e50-zc.buildout==2.7.1+slapos001...
    Downloaded https://pypi.org/simple/zc.buildout/ from network cache.
    networkcache: Downloading slapos-buildout-fd4ac0343c559873f75e8e4be0b06366...
    Downloaded http://www.nexedi.org/static/packages/source/slapos.buildout/zc.buildout-2.7.1+slapos001.tar.gz from network cache.
    networkcache: Downloading pypi-index-f0a660f10b8bd0b2f65992c829d98404-slapos.libnetworkcache==0.20...
    Downloaded https://pypi.org/simple/slapos.libnetworkcache/ from network cache.
    networkcache: Downloading slapos-buildout-b7f445bbec09ae840c83ded70a305e7c...
    Downloaded https://files.pythonhosted.org/packages/b4/75/ae13749622d6ace9dae2442399d955d29241cfded7b48245320e2c0953b1/slapos.libnetworkcache-0.20.tar.gz#sha256=a6ceb542499fb572641abb1c7b675329bb235fda43d2f28b541906179e8db8db from network cache.
    Traceback (most recent call last):
      File "<string>", line 67, in <module>
      File "<string>", line 64, in main
      File "<string>", line 30, in setup_script
      File "/srv/slapgrid/slappart14/srv/testnode/cup/soft/4281ee9e21ffccb8d017eba71a9cd9b1/eggs/zc.buildout-2.7.1+slapos001-py3.6.egg/zc/buildout/easy_install.py", line 1313, in scripts
        sname = os.path.join(dest, sname)
      File "/srv/slapgrid/slappart14/srv/testnode/cup/soft/4281ee9e21ffccb8d017eba71a9cd9b1/parts/python3.6/lib/python3.6/posixpath.py", line 80, in join
        a = os.fspath(a)
    TypeError: expected str, bytes or os.PathLike object, not NoneType
    While:
      Installing.


This also fixes test suite to support python3. When running on python3.6 the problem occurred, but not with python3.5.

/reviewed-on nexedi/slapos.rebootstrap!2

setup.py

@@ -5,6 +5,13 @@ name = "slapos.rebootstrap"
 long_description = open("README.rst").read() + '\n\n'
 long_description += open("CHANGELOG.rst").read()
 
+extras_require = {
+    'test': [
+        'zope.testing',
+        'manuel',
+    ]
+}
+
 setup(
   name=name,
   version=version,
@@ -38,9 +45,7 @@ setup(
     'setuptools',
     'zc.buildout >=2.7.1+slapos001, <2.7.2',
     ],
-  tests_require=[
-      'zope.testing',
-      'manuel'
-    ],
+  extras_require=extras_require,
+  tests_require=extras_require['test'],
   test_suite='%s.tests.test_suite' % name,
 )

slapos/rebootstrap/bootstrap.py

@@ -21,13 +21,17 @@ def get_distributions():
 
 def setup_script(path, python=sys.executable):
   from zc.buildout import easy_install
+  executable_path = os.path.realpath(path)
+  assert os.path.isabs(executable_path)
   try:
     if sys.executable != python:
       easy_install.sys = FakeSysExecutable(python)
     easy_install.scripts(
-      ((os.path.realpath(path), 'zc.buildout.buildout', 'main'),),
+      ((os.path.basename(executable_path), 'zc.buildout.buildout', 'main'),),
       get_distributions(),
-      python)
+      python,
+      os.path.dirname(executable_path)
+    )
   finally:
     easy_install.sys = sys
 

slapos/rebootstrap/different_version.txt

@@ -38,8 +38,9 @@ TODO: The test is incomplete for 2 reasons:
 ...   def __getattr__(self, attr):
 ...     return getattr(sys, attr)
 ... """)
->>> print system(buildout),
+>>> print(system(buildout))
 Develop: '/sample-buildout/recipes'
+<BLANKLINE>
 >>> write(sample_buildout, 'buildout.cfg',
 ... """
 ... [buildout]
@@ -57,7 +58,7 @@ Develop: '/sample-buildout/recipes'
 ... recipe = recipes:pyshow
 ... """ % dict(syspython=sys.executable))
 
->>> print system(buildout), # doctest: +ELLIPSIS
+>>> print(system(buildout)) # doctest: +ELLIPSIS
 slapos.rebootstrap: Make sure that the section 'installpython' won't be reinstalled after rebootstrap.
 Develop: '/sample-buildout/recipes'
 Installing installpython.
@@ -73,3 +74,4 @@ Buildout will be restarted automatically to have this change applied.
 While:
   Installing.
 Error: Couldn't find a distribution for 'setuptools==...'
+<BLANKLINE>

slapos/rebootstrap/same_version.txt

@@ -2,8 +2,9 @@ Simple case: switch to a Python with same major/minor version
 -------------------------------------------------------------
 
 >>> import sys
->>> print system(buildout),
+>>> print(system(buildout))
 Develop: '/sample-buildout/recipes'
+<BLANKLINE>
 >>> write(sample_buildout, 'buildout.cfg',
 ... """
 ... [buildout]
@@ -21,7 +22,7 @@ Develop: '/sample-buildout/recipes'
 ... recipe = recipes:pyshow
 ... """ % dict(syspython=sys.executable))
 
->>> print system(buildout),
+>>> print(system(buildout))
 slapos.rebootstrap: Make sure that the section 'installpython' won't be reinstalled after rebootstrap.
 Develop: '/sample-buildout/recipes'
 Installing installpython.
@@ -39,15 +40,17 @@ Develop: '/sample-buildout/recipes'
 Updating installpython.
 Installing realrun.
 Running with: /sample_buildout/parts/installpython/bin/python
+<BLANKLINE>
 
->>> print system(buildout),
+>>> print(system(buildout))
 Develop: '/sample-buildout/recipes'
 Updating installpython.
 Updating realrun.
 Running with: /sample_buildout/parts/installpython/bin/python
+<BLANKLINE>
 
 >>> cp(buildout + '-orig', buildout)
->>> print system(buildout),
+>>> print(system(buildout))
 slapos.rebootstrap: Make sure that the section 'installpython' won't be reinstalled after rebootstrap.
 Develop: '/sample-buildout/recipes'
 Updating installpython.
@@ -64,3 +67,4 @@ Develop: '/sample-buildout/recipes'
 Updating installpython.
 Updating realrun.
 Running with: /sample_buildout/parts/installpython/bin/python
+<BLANKLINE>

slapos/rebootstrap/same_version_already_installed.txt

@@ -3,8 +3,9 @@
 An edge case is when the python is provided by a part that does not need to install anything.
 
 >>> import sys
->>> print system(buildout),
+>>> print(system(buildout))
 Develop: '/sample-buildout/recipes'
+<BLANKLINE>
 >>> write(sample_buildout, 'buildout.cfg',
 ... """
 ... [buildout]
@@ -22,9 +23,10 @@ Develop: '/sample-buildout/recipes'
 ... recipe = recipes:pyshow
 ... """ % dict(syspython=sys.executable))
 
->>> print system(buildout),
+>>> print(system(buildout))
 Using already installed /system_python
 Develop: '/sample-buildout/recipes'
 Installing already_installed_python.
 Installing realrun.
 Running with: /system_python
+<BLANKLINE>

slapos/rebootstrap/tests.py

@@ -59,12 +59,13 @@ class Pyinstall:
   update = install
 """)
   write(sample_buildout, 'recipes', 'pyalreadyinstalled.py', """
+from __future__ import print_function
 import os, zc.buildout, shutil, sys
 
 class PyAlreadyInstalled:
 
   def __init__(self, buildout, name, options):
-    print "Using already installed", sys.executable
+    print("Using already installed", sys.executable)
     options['executable'] = sys.executable
 
   def install(self):
@@ -73,6 +74,7 @@ class PyAlreadyInstalled:
   update = install
 """)
   write(sample_buildout, 'recipes', 'pyshow.py', """
+from __future__ import print_function
 import os, zc.buildout, shutil, sys
 
 class Pyshow:
@@ -81,7 +83,7 @@ class Pyshow:
     pass
 
   def install(self):
-    print 'Running with:', sys.executable
+    print('Running with:', sys.executable)
     return []
 
   update = install
@@ -95,7 +97,9 @@ parts =
 def setUp(test):
   zc.buildout.testing.buildoutSetUp(test)
   zc.buildout.testing.install_develop('slapos.rebootstrap', test)
-  test.globs['cp'] = shutil.copy
+  def cp(src, dst):
+    shutil.copy(src, dst)
+  test.globs['cp'] = cp
   _setUp(**test.globs)
 
 def test_suite():
@@ -105,9 +109,9 @@ def test_suite():
   kwargs = dict(setUp=setUp,
       tearDown=zc.buildout.testing.buildoutTearDown,
       checker=renormalizing.RENormalizing([
-                        (re.compile('--prefix=\S+sample-buildout'),
+                        (re.compile(r'--prefix=\S+sample-buildout'),
                          '--prefix=/sample_buildout'),
-                        (re.compile('\s/\S+sample-buildout'),
+                        (re.compile(r'\s/\S+sample-buildout'),
                          ' /sample_buildout'),
                         (re.compile(sys.executable),
                          '/system_python'),