Stop using deprecated setuptools.easy_install

Previously, slapos.rebootstrap switched from Python X.X to Python Y.Y
by first arranging to have source distributions for zc.buildout and its
dependencies, either already available in dowload cache, or downloading
them with Python X.X using internal APIs of zc.buildout, then launching
Python Y.Y to extract and bootstrap the installation of setuptools, and
then use setuptools.easy_install to install the other distributions,
meaning zc.buildout and any other dependency.

The versions of zc.buildout and its dependencies thus installed are
exactly the same as the one originally running with Python X.X. This
implies of course that these versions are compatible with Python X.X
and Python Y.Y - that is a hard constraint currently.

Thus instead of reinstalling them, we can actually just reuse the
already installed versions with Python Y.Y by passing the packages in
sys.path, giving us a working zc.buildout which we can use to properly
reinstall zc.buildout and its dependencies for Python Y.Y.

This allows for a much easier process, as we can merely achieve this
by calling buildout bootstrap with Python Y.Y without resorting to
using internal APIs of zc.buildout to download the source distributions
by hand, nor needing to manually bootstrap their installation by
extracting setuptools etc.

Note that the previous implementation also didn't respect :whl version
pins.

slapos/rebootstrap/__init__.py

@@ -13,9 +13,61 @@
 ##############################################################################
 
 import errno, logging, os, shutil, subprocess, sys, tempfile
+import pkg_resources
 from zc.buildout import easy_install, UserError
 
 
+class FakeSysExecutable(object):
+
+  def __init__(self, python):
+    self.executable = python
+
+  def __getattr__(self, attr):
+    return getattr(sys, attr)
+
+
+def get_distributions():
+  # zc.buildout and dependencies
+  dists = pkg_resources.require('zc.buildout')
+  # propagate slapos.libnetworkcache availability
+  try:
+    import slapos.libnetworkcache
+    dists.append(pkg_resources.get_distribution('slapos.libnetworkcache'))
+  except ImportError:
+    pass
+  # keep same order as in sys.path
+  dists.sort(key=lambda d: sys.path.index(d.location))
+  return dists
+
+
+def get_paths():
+  path = None
+  paths = []
+  # order preserving unique, knowing that duplicates can only be adjacent
+  for dist in get_distributions():
+    if path != dist.location:
+      path = dist.location
+      paths.append(path)
+  return paths
+
+
+def setup_script(path, python=sys.executable):
+  from zc.buildout import easy_install
+  executable_path = os.path.realpath(path)
+  assert os.path.isabs(executable_path)
+  try:
+    if sys.executable != python:
+      easy_install.sys = FakeSysExecutable(python)
+    easy_install.scripts(
+      ((os.path.basename(executable_path), 'zc.buildout.buildout', 'main'),),
+      get_distributions(),
+      python,
+      os.path.dirname(executable_path)
+    )
+  finally:
+    easy_install.sys = sys
+
+
 class extension(object):
 
   def __init__(self, buildout):
@@ -69,55 +121,28 @@ Buildout will be restarted automatically to have this change applied.
       if e.errno != errno.ENOENT:
         raise
       x = None
-    if x != '#!' + self.wanted_python:
-      from .bootstrap import get_distributions, setup_script
+    if x == '#!' + self.wanted_python:
+      shutil.copy(new_bin, installed)
+    else:
       if subprocess.call((self.wanted_python, '-c',
           'import sys; sys.exit(sys.version_info[:2] == %r)'
           % (sys.version_info[:2],))):
         setup_script(new_bin, self.wanted_python)
+        shutil.copy(new_bin, installed)
       else:
-        # With a different version of Python,
-        # we must reinstall required eggs from source.
-        from pkg_resources import resource_string
-        with Cache(buildout['buildout']) as cache:
-          subprocess.check_call([self.wanted_python, '-c',
-            resource_string(__name__, 'bootstrap.py'),
-            new_bin, cache._dest, cache.tmp,
-            ] + list(map(cache, get_distributions())))
-
-    shutil.copy(new_bin, installed)
+        old = installed + '-old'
+        shutil.move(installed, old)
+        try:
+          paths = get_paths()
+          args = [arg for arg in sys.argv if arg.startswith('buildout:')]
+          subprocess.check_call(
+            [self.wanted_python, '-c',
+            "import sys ; sys.path[0:0]=%r ; "
+            "import zc.buildout.buildout ; "
+            "sys.argv[1:1]=%r ; "
+            "zc.buildout.buildout.main()" % (paths, args + ['bootstrap'])])
+        except subprocess.CalledProcessError:
+          shutil.move(old, installed)
+          raise
+        shutil.copy(installed, new_bin)
     os.execve(self.wanted_python, [self.wanted_python] + sys.argv, self.environ)
-
-
-class Cache(easy_install.Installer):
-
-  def __init__(self, buildout):
-    easy_install.Installer.__init__(self,
-      buildout['eggs-directory'],
-      buildout.get('find-links', '').split())
-
-  def __enter__(self):
-    self.tmp = self._download_cache or tempfile.mkdtemp('get_dist')
-    return self
-
-  def __exit__(self, t, v, tb):
-    if self.tmp is not self._download_cache:
-      shutil.rmtree(self.tmp)
-    del self.tmp
-
-  def __call__(self, dist):
-    req = dist.as_requirement()
-    cache = self._download_cache
-    if cache:
-      from pkg_resources import SOURCE_DIST
-      for avail in self._index[dist.project_name]:
-        if (avail.version == dist.version and
-            avail.precedence == SOURCE_DIST and
-            cache == os.path.dirname(avail.location)):
-          return str(req)
-    avail = self._obtain(req, True)
-    if avail is None:
-      raise UserError("Couldn't find a distribution for %r" % str(req))
-    if self._fetch(avail, self.tmp, cache) is None:
-      raise UserError("Couldn't download distribution %s." % avail)
-    return str(req)

slapos/rebootstrap/bootstrap.py

-import errno, os, sys
-
-class FakeSysExecutable(object):
-
-  def __init__(self, python):
-    self.executable = python
-
-  def __getattr__(self, attr):
-    return getattr(sys, attr)
-
-def get_distributions():
-  from pkg_resources import get_distribution
-  distributions = ['setuptools', 'zc.buildout', 'wheel', 'pip']
-  try:
-    import slapos.libnetworkcache
-  except ImportError:
-    pass
-  else:
-    distributions.append('slapos.libnetworkcache')
-  return map(get_distribution, distributions)
-
-def setup_script(path, python=sys.executable):
-  from zc.buildout import easy_install
-  executable_path = os.path.realpath(path)
-  assert os.path.isabs(executable_path)
-  try:
-    if sys.executable != python:
-      easy_install.sys = FakeSysExecutable(python)
-    easy_install.scripts(
-      ((os.path.basename(executable_path), 'zc.buildout.buildout', 'main'),),
-      get_distributions(),
-      python,
-      os.path.dirname(executable_path)
-    )
-  finally:
-    easy_install.sys = sys
-
-def main():
-  import shutil, subprocess, tarfile, tempfile, zipfile
-  eggs_dir = sys.argv[2]
-  cache = sys.argv[3]
-  base = os.path.join(cache, sys.argv[4].replace('==', '-'))
-  # Install setuptools.
-  tmp = tempfile.mkdtemp()
-  try:
-    try:
-      with zipfile.ZipFile(base + '.zip') as zip_file:
-        zip_file.extractall(tmp)
-    except IOError as e:
-      if e.errno != errno.ENOENT:
-        raise
-      with tarfile.open(base + '.tar.gz') as tar_file:
-        tar_file.extractall(tmp)
-    src, = os.listdir(tmp)
-    subprocess.check_call((sys.executable, 'setup.py', '-q', 'bdist_egg',
-                          '--dist-dir', tmp), cwd=os.path.join(tmp, src))
-    egg = os.listdir(tmp)
-    egg.remove(src)
-    egg, = egg
-    dst = os.path.join(eggs_dir, egg)
-    os.path.exists(dst) or shutil.move(os.path.join(tmp, egg), dst)
-  finally:
-    shutil.rmtree(tmp)
-  sys.path.insert(0, dst)
-  # Install other requirements given on command line.
-  from pkg_resources import working_set, require
-  from setuptools.command import easy_install
-  reqs = sys.argv[5:]
-  easy_install.main(['-mZqNxd', eggs_dir, '-f', cache] + reqs)
-  working_set.add_entry(eggs_dir)
-  for req in reqs:
-    require(req)
-  # Generate bin/buildout-rebootstrap script.
-  setup_script(sys.argv[1])
-
-if __name__ == '__main__':
-  sys.exit(main())