Adapt rebootstrap to setuptools >= 52.0.0 without easy_install

See merge request !5

CHANGELOG.rst

 Changes
 =======
 
+4.7 (2024-05-22)
+----------------
+
+- Stop using deprecated setuptools.easy_install
+- Check zc.buildout version on import
+- Unify rebootstrap mechanisms
+
 4.6 (2024-03-26)
 ----------------
 

setup.py

 from setuptools import setup, find_packages
-version = '4.6'
+version = '4.7'
 name = "slapos.rebootstrap"
 
 long_description = open("README.rst").read() + '\n\n'

slapos/rebootstrap/__init__.py

@@ -12,8 +12,31 @@
 #
 ##############################################################################
 
-import errno, logging, os, shutil, subprocess, sys, tempfile
-from zc.buildout import easy_install, UserError
+import errno, logging, os, shutil, subprocess, sys
+import pkg_resources
+from zc.buildout import UserError
+
+
+buildout_dist = pkg_resources.get_distribution('zc.buildout')
+if 'slapos' not in str(buildout_dist.version):
+  raise UserError(
+    "Incompatible version %s\n"
+    "Consider installing e.g. zc.buildout==3.0.1+slapos001"
+    % buildout_dist)
+
+
+def get_paths():
+  # zc.buildout and dependencies
+  dists = pkg_resources.require('zc.buildout')
+  # propagate slapos.libnetworkcache availability
+  try:
+    import slapos.libnetworkcache
+    dists.append(pkg_resources.get_distribution('slapos.libnetworkcache'))
+  except ImportError:
+    pass
+  # keep same order as in sys.path
+  paths = {d.location for d in dists}
+  return [p for p in sys.path if p in paths]
 
 
 class extension(object):
@@ -69,55 +92,22 @@ Buildout will be restarted automatically to have this change applied.
       if e.errno != errno.ENOENT:
         raise
       x = None
-    if x != '#!' + self.wanted_python:
-      from .bootstrap import get_distributions, setup_script
-      if subprocess.call((self.wanted_python, '-c',
-          'import sys; sys.exit(sys.version_info[:2] == %r)'
-          % (sys.version_info[:2],))):
-        setup_script(new_bin, self.wanted_python)
-      else:
-        # With a different version of Python,
-        # we must reinstall required eggs from source.
-        from pkg_resources import resource_string
-        with Cache(buildout['buildout']) as cache:
-          subprocess.check_call([self.wanted_python, '-c',
-            resource_string(__name__, 'bootstrap.py'),
-            new_bin, cache._dest, cache.tmp,
-            ] + list(map(cache, get_distributions())))
-
-    shutil.copy(new_bin, installed)
+    if x == '#!' + self.wanted_python:
+      shutil.copy(new_bin, installed)
+    else:
+      old = installed + '-old'
+      shutil.move(installed, old)
+      try:
+        paths = get_paths()
+        args = [arg for arg in sys.argv if arg.startswith('buildout:')]
+        subprocess.check_call(
+          [self.wanted_python, '-c',
+          "import sys ; sys.path[0:0]=%r ; "
+          "import zc.buildout.buildout ; "
+          "sys.argv[1:1]=%r ; "
+          "zc.buildout.buildout.main()" % (paths, args + ['bootstrap'])])
+      except subprocess.CalledProcessError:
+        shutil.move(old, installed)
+        raise
+      shutil.copy(installed, new_bin)
     os.execve(self.wanted_python, [self.wanted_python] + sys.argv, self.environ)
-
-
-class Cache(easy_install.Installer):
-
-  def __init__(self, buildout):
-    easy_install.Installer.__init__(self,
-      buildout['eggs-directory'],
-      buildout.get('find-links', '').split())
-
-  def __enter__(self):
-    self.tmp = self._download_cache or tempfile.mkdtemp('get_dist')
-    return self
-
-  def __exit__(self, t, v, tb):
-    if self.tmp is not self._download_cache:
-      shutil.rmtree(self.tmp)
-    del self.tmp
-
-  def __call__(self, dist):
-    req = dist.as_requirement()
-    cache = self._download_cache
-    if cache:
-      from pkg_resources import SOURCE_DIST
-      for avail in self._index[dist.project_name]:
-        if (avail.version == dist.version and
-            avail.precedence == SOURCE_DIST and
-            cache == os.path.dirname(avail.location)):
-          return str(req)
-    avail = self._obtain(req, True)
-    if avail is None:
-      raise UserError("Couldn't find a distribution for %r" % str(req))
-    if self._fetch(avail, self.tmp, cache) is None:
-      raise UserError("Couldn't download distribution %s." % avail)
-    return str(req)

slapos/rebootstrap/bootstrap.py

-import errno, os, sys
-
-class FakeSysExecutable(object):
-
-  def __init__(self, python):
-    self.executable = python
-
-  def __getattr__(self, attr):
-    return getattr(sys, attr)
-
-def get_distributions():
-  from pkg_resources import get_distribution
-  distributions = ['setuptools', 'zc.buildout', 'wheel', 'pip']
-  try:
-    import slapos.libnetworkcache
-  except ImportError:
-    pass
-  else:
-    distributions.append('slapos.libnetworkcache')
-  return map(get_distribution, distributions)
-
-def setup_script(path, python=sys.executable):
-  from zc.buildout import easy_install
-  executable_path = os.path.realpath(path)
-  assert os.path.isabs(executable_path)
-  try:
-    if sys.executable != python:
-      easy_install.sys = FakeSysExecutable(python)
-    easy_install.scripts(
-      ((os.path.basename(executable_path), 'zc.buildout.buildout', 'main'),),
-      get_distributions(),
-      python,
-      os.path.dirname(executable_path)
-    )
-  finally:
-    easy_install.sys = sys
-
-def main():
-  import shutil, subprocess, tarfile, tempfile, zipfile
-  eggs_dir = sys.argv[2]
-  cache = sys.argv[3]
-  base = os.path.join(cache, sys.argv[4].replace('==', '-'))
-  # Install setuptools.
-  tmp = tempfile.mkdtemp()
-  try:
-    try:
-      with zipfile.ZipFile(base + '.zip') as zip_file:
-        zip_file.extractall(tmp)
-    except IOError as e:
-      if e.errno != errno.ENOENT:
-        raise
-      with tarfile.open(base + '.tar.gz') as tar_file:
-        tar_file.extractall(tmp)
-    src, = os.listdir(tmp)
-    subprocess.check_call((sys.executable, 'setup.py', '-q', 'bdist_egg',
-                          '--dist-dir', tmp), cwd=os.path.join(tmp, src))
-    egg = os.listdir(tmp)
-    egg.remove(src)
-    egg, = egg
-    dst = os.path.join(eggs_dir, egg)
-    os.path.exists(dst) or shutil.move(os.path.join(tmp, egg), dst)
-  finally:
-    shutil.rmtree(tmp)
-  sys.path.insert(0, dst)
-  # Install other requirements given on command line.
-  from pkg_resources import working_set, require
-  from setuptools.command import easy_install
-  reqs = sys.argv[5:]
-  easy_install.main(['-mZqNxd', eggs_dir, '-f', cache] + reqs)
-  working_set.add_entry(eggs_dir)
-  for req in reqs:
-    require(req)
-  # Generate bin/buildout-rebootstrap script.
-  setup_script(sys.argv[1])
-
-if __name__ == '__main__':
-  sys.exit(main())

slapos/rebootstrap/different_version.txt

@@ -62,6 +62,10 @@ Develop: '/sample-buildout/recipes'
 ... recipe = recipes:pyshow
 ... """ % dict(syspython=sys.executable))
 
+>>> cat(buildout) # doctest: +ELLIPSIS
+#!/system_python
+...
+
 >>> print(system(buildout, env={'PYTHONWARNINGS':'ignore'})) # doctest: +ELLIPSIS
 slapos.rebootstrap: Make sure that the section 'installpython' won't be reinstalled after rebootstrap.
 Develop: '/sample-buildout/recipes'
@@ -75,7 +79,13 @@ is available, and buildout is using another python:
 Buildout will be restarted automatically to have this change applied.
 ************ REBOOTSTRAP: IMPORTANT NOTICE ************
 <BLANKLINE>
-While:
-  Installing.
-Error: Couldn't find a distribution for 'setuptools==...'
+Generated script '/sample-buildout/bin/buildout'.
+Develop: '/sample-buildout/recipes'
+Updating installpython.
+Installing realrun.
+Running with: /sample_buildout/parts/installpython/bin/python
 <BLANKLINE>
+
+>>> cat(buildout) # doctest: +ELLIPSIS
+/sample-buildout/parts/installpython/bin/python
+...

slapos/rebootstrap/same_version.txt

@@ -22,6 +22,10 @@ Develop: '/sample-buildout/recipes'
 ... recipe = recipes:pyshow
 ... """ % dict(syspython=sys.executable))
 
+>>> cat(buildout) # doctest: +ELLIPSIS
+#!/system_python
+...
+
 >>> print(system(buildout, env={'PYTHONWARNINGS':'ignore'}))
 slapos.rebootstrap: Make sure that the section 'installpython' won't be reinstalled after rebootstrap.
 Develop: '/sample-buildout/recipes'
@@ -35,13 +39,17 @@ is available, and buildout is using another python:
 Buildout will be restarted automatically to have this change applied.
 ************ REBOOTSTRAP: IMPORTANT NOTICE ************
 <BLANKLINE>
-Generated script '/sample-buildout/bin/buildout-rebootstrap'.
+Generated script '/sample-buildout/bin/buildout'.
 Develop: '/sample-buildout/recipes'
 Updating installpython.
 Installing realrun.
 Running with: /sample_buildout/parts/installpython/bin/python
 <BLANKLINE>
 
+>>> cat(buildout) # doctest: +ELLIPSIS
+/sample-buildout/parts/installpython/bin/python
+...
+
 >>> print(system(buildout, env={'PYTHONWARNINGS':'ignore'}))
 Develop: '/sample-buildout/recipes'
 Updating installpython.
@@ -49,6 +57,10 @@ Updating realrun.
 Running with: /sample_buildout/parts/installpython/bin/python
 <BLANKLINE>
 
+>>> cat(buildout) # doctest: +ELLIPSIS
+/sample-buildout/parts/installpython/bin/python
+...
+
 >>> cp(buildout + '-orig', buildout)
 >>> print(system(buildout, env={'PYTHONWARNINGS':'ignore'}))
 slapos.rebootstrap: Make sure that the section 'installpython' won't be reinstalled after rebootstrap.
@@ -68,3 +80,7 @@ Updating installpython.
 Updating realrun.
 Running with: /sample_buildout/parts/installpython/bin/python
 <BLANKLINE>
+
+>>> cat(buildout) # doctest: +ELLIPSIS
+/sample-buildout/parts/installpython/bin/python
+...

slapos/rebootstrap/tests.py

@@ -50,15 +50,19 @@ class Pyinstall:
 
   def __init__(self, buildout, name, options):
     self.options = options
-    options['executable'] = os.path.join(buildout['buildout'][
-      'parts-directory'], name, 'bin', 'python')
+    self.part_dir = os.path.join(buildout['buildout']['parts-directory'], name)
+    options['executable'] = os.path.join(self.part_dir, 'bin', 'python')
 
   def install(self):
     python = self.options['executable']
     if not os.path.exists(python):
-      d = os.path.dirname(python)
-      os.path.exists(d) or os.makedirs(d)
-      shutil.copy(sys.executable, python)
+      try:
+        from venv import create
+        create(self.part_dir, clear=True)
+      except ImportError:
+        d = os.path.dirname(python)
+        os.path.exists(d) or os.makedirs(d)
+        shutil.copy(sys.executable, python)
     return []
 
   update = install
@@ -124,6 +128,7 @@ def test_suite():
                          ''),
                         zc.buildout.testing.normalize_path,
                         zc.buildout.testing.not_found,
+                        zc.buildout.testing.root_logger_messages,
                         ]),
     )
   test_list = []