[feat] Use pip install --editable in easy_install.develop

Instead of running python setup.py develop directly. This will allow
using zc.buildout.easy_install.develop on recent projects that have
only a pyproject.toml. It also fixes develop leaving build artifacts
in the source directory that caused later runs to do nothing, e.g.
preventing develop-eggs to be rebuilt when a build dependency passed
in setup-eggs option of zc.recipe.egg:develop changed.

A verbosity parameter to tune verbosity of pip is added, with adjusted
values for the case of buildout:develop and of zc.recipe.egg:develop,
so as to remain close to the previous behavior with regards to logs.

src/zc/buildout/buildout.py

@@ -1131,7 +1131,8 @@ class Buildout(DictMixin):
                     for setup in files:
                         self._logger.info("Develop: %r", setup)
                         __doing__ = 'Processing develop directory %r.', setup
-                        zc.buildout.easy_install.develop(setup, dest)
+                        zc.buildout.easy_install.develop(setup, dest,
+                                                         verbosity=-20)
             except:
                 # if we had an error, we need to roll back changes, by
                 # removing any files we created.

src/zc/buildout/easy_install.py

@@ -1113,22 +1113,6 @@ def _rm(*paths):
             os.remove(path)
 
 
-def _copyeggs(src, dest, suffix, undo):
-    result = []
-    undo.append(lambda : _rm(*result))
-    for name in os.listdir(src):
-        if name.endswith(suffix):
-            new = os.path.join(dest, name)
-            _rm(new)
-            os.rename(os.path.join(src, name), new)
-            result.append(new)
-
-    assert len(result) == 1, str(result)
-    undo.pop()
-
-    return result[0]
-
-
 _develop_distutils_scripts = {}
 
 
@@ -1175,13 +1159,11 @@ def _detect_distutils_scripts(directory):
         _develop_distutils_scripts[egg_name] = scripts_found
 
 
-def develop(setup, dest,
-            build_ext=None,
-            executable=sys.executable):
-    assert executable == sys.executable, (executable, sys.executable)
+def develop(setup, dest, build_ext=None, executable=None, verbosity=-10):
+    if executable is not None: # BBB
+        assert executable == sys.executable, (executable, sys.executable)
     if os.path.isdir(setup):
         directory = setup
-        setup = os.path.join(directory, 'setup.py')
     else:
         directory = os.path.dirname(setup)
 
@@ -1203,43 +1185,89 @@ def develop(setup, dest,
             setuptools.command.setopt.edit_config(
                 setup_cfg, dict(build_ext=build_ext))
 
-        fd, tsetup = tempfile.mkstemp()
-        undo.append(lambda: os.remove(tsetup))
-        undo.append(lambda: os.close(fd))
+        tmp3 = tempfile.mkdtemp('build', dir=dest)
+        undo.append(lambda : zc.buildout.rmtree.rmtree(tmp3))
 
-        extra_path = os.environ.get('PYTHONEXTRAPATH')
-        extra_path_list = []
-        if extra_path:
-          extra_path_list = extra_path.split(os.pathsep)
+        class options: pass
+        options._allow_picked_versions = allow_picked_versions()
 
-        os.write(fd, (runsetup_template % dict(
-            setupdir=directory,
-            setup=setup,
-            path_list=extra_path_list,
-            __file__ = setup,
-            )).encode())
+        call_pip_editable(directory, tmp3, options, verbosity=verbosity)
 
-        tmp3 = tempfile.mkdtemp('build', dir=dest)
-        undo.append(lambda : zc.buildout.rmtree.rmtree(tmp3))
+        _detect_distutils_scripts(tmp3)
 
-        args = [executable,  tsetup, '-q', 'develop', '-mN', '-d', tmp3]
+        egg_link, dist_info = [], []
+        for entry in os.listdir(tmp3):
+            if entry.endswith('.egg-link'):
+              egg_link.append(entry)
+            if entry.endswith('.dist-info') or entry.endswith('.egg-info'):
+              dist_info.append(entry)
 
-        log_level = logger.getEffectiveLevel()
-        if log_level <= 0:
-            if log_level == 0:
-                del args[2]
-            else:
-                args[2] == '-v'
-        if log_level < logging.DEBUG:
-            logger.debug("in: %r\n%s", directory, ' '.join(args))
+        assert len(egg_link) + len(dist_info) == 1, str(egg_link + dist_info)
+        entry, = (egg_link or dist_info)
 
-        if SETUPTOOLS_IGNORE_WARNINGS:
-            env = dict(os.environ, PYTHONWARNINGS='ignore')
+        entry_path = os.path.join(tmp3, entry)
+
+        entry_filename, ext = os.path.splitext(entry)
+        project_name = entry_filename.split('-', 1)[0]
+
+        def move(src, dst):
+            _rm(dst)
+            os.rename(src, dst)
+            return dst
+
+        if egg_link:
+            # case: legacy setup.py
+            return move(entry_path, os.path.join(dest, entry))
         else:
-            env=None
-        call_subprocess(args, env=env)
-        _detect_distutils_scripts(tmp3)
-        return _copyeggs(tmp3, dest, '.egg-link', undo)
+            # case: PEP 660 pyproject.toml
+            # This case can only happen in Python3
+            # (because pip versions that support PEP 660 are Python3-only).
+
+            # Resolve the package path(s) and import (parent) path(s).
+            # This may be a subpath of the project directory,
+            # e.g. zc.buildout/src.
+            with open(os.path.join(entry_path, 'top_level.txt')) as f:
+                top_level = f.read().splitlines()
+            try:
+                package_paths = subprocess.check_output([
+                    sys.executable, '-S', '-c',
+                    "import importlib.util, site; "
+                    "site.addsitedir(%(site)r); "
+                    "packages = %(packages)r; "
+                    "specs = [importlib.util.find_spec(p) for p in packages]; "
+                    "paths = sum((s.submodule_search_locations"
+                    " for s in specs), []); "
+                    "print('\\n'.join(paths))"  % {
+                        'site': tmp3,
+                        'packages': tuple(p for p in top_level if p),
+                        }
+                    ],
+                    env=dict(os.environ, PYTHONPATH=''),
+                    stderr=subprocess.STDOUT,
+                    universal_newlines=True,
+                ).splitlines()
+            except subprocess.CalledProcessError as e:
+                logger.error(e.output)
+                raise
+            # Resolve import path; warn if there are several and
+            # arbitrarily take the first one.
+            import_paths = [os.path.dirname(p) for p in package_paths]
+            import_path = import_paths[0]
+            if len(set(import_paths)) > 1:
+                logger.warning(
+                    "Found multiple package import paths in develop project %s"
+                    "\n(\n%s\n)\n"
+                    "Selected %s"
+                    % (directory, '\n'.join(set(import_paths)), import_path))
+            # Move the .dist-info folder to the import path like setuptools.
+            move(entry_path, os.path.join(import_path, project_name + ext))
+            # Create a temporary .egg-link.
+            tmp_egg_link = os.path.join(tmp3, project_name + '.egg-link')
+            with open(tmp_egg_link, 'w') as f:
+                f.write(import_path)
+            # Move the .egg-link to the destination directory.
+            egg_link_path = os.path.join(dest, project_name + '.egg-link')
+            return move(tmp_egg_link, egg_link_path)
 
     finally:
         undo.reverse()
@@ -1806,63 +1834,50 @@ class IncompatibleConstraintError(zc.buildout.UserError):
 IncompatibleVersionError = IncompatibleConstraintError # Backward compatibility
 
 
-def call_pip_wheel(spec, dest, options):
+try:
+    from pip._internal.cli.cmdoptions import no_python_version_warning
+    PIP_HAS_PYTHON_VERSION_WARNING_OPTION = True
+except ImportError:
+    PIP_HAS_PYTHON_VERSION_WARNING_OPTION = False
+
+
+def call_pip_command(command, operand, options, verbosity=-10):
     """
-    Call `pip wheel` from a subprocess to install a
-    distribution specified by `spec` into `dest`.
-    Returns all the paths inside `dest` created by the above.
+    Call `pip <command...> <operand...>` from a subprocess
+    with appropriate options and environment.
     """
     cleanup = []
     try:
-        args = [sys.executable, '-m', 'pip', 'wheel', '--no-deps', '-w', dest]
-        level = logger.getEffectiveLevel()
-        if level >= logging.INFO:
-            args.append('-q')
-        else:
-            args.append('-v')
+        env = os.environ.copy()
+        pythonpath = pip_path[:]
+        pythonpath.extend(
+            env.get(k) for k in ('PYTHONPATH', 'PYTHONEXTRAPATH'))
+        env['PYTHONPATH'] = os.pathsep.join(p for p in pythonpath if p)
+
+        args = [sys.executable, '-m', 'pip']
+        args.extend(command)
+        args.append('--no-deps')
+
+        log_level = logger.getEffectiveLevel()
+        pip_level = log_level - verbosity
+        if pip_level >= logging.WARNING:
+            args.append('-' + 'q' * (1 + (pip_level >= logging.ERROR)))
+        elif pip_level < logging.INFO:
+            args.append('-' + 'v' * (1 + (pip_level < logging.DEBUG)))
+        # Note: more recent pip accepts even -vvv and -qqq.
 
-        # Prevent pip from installing build dependencies on the fly
-        # without respecting pinned versions. This only works for
-        # PEP 517 specifications using pyproject.toml and not for
-        # dependencies in setup_requires option in legacy setup.py
         if not options._allow_picked_versions:
+            # Prevent pip from installing build dependencies on the fly
+            # without respecting pinned versions. This only works for
+            # PEP 517 specifications using pyproject.toml and not for
+            # dependencies in setup_requires option in legacy setup.py
             args.append('--no-index')
             args.append('--no-build-isolation')
 
-        args.append(spec)
-
-        try:
-            from pip._internal.cli.cmdoptions import no_python_version_warning
-            HAS_WARNING_OPTION = True
-        except ImportError:
-            HAS_WARNING_OPTION = False
-        if HAS_WARNING_OPTION:
-            if not hasattr(call_pip_wheel, 'displayed'):
-                call_pip_wheel.displayed = True
-            else:
-                args.append('--no-python-version-warning')
-
-        env = os.environ.copy()
-        python_path = pip_path[:]
-        env_paths = env.get('PYTHONPATH')
-        if env_paths:
-            python_path.append(env_paths)
-        extra_env_path = env.get('PYTHONEXTRAPATH')
-        if extra_env_path:
-            python_path.append(extra_env_path)
-        env['PYTHONPATH'] = os.pathsep.join(python_path)
-
-        if level <= logging.DEBUG:
-            logger.debug('Running pip install:\n"%s"\npath=%s\n',
-                            '" "'.join(args), pip_path)
-
-        sys.stdout.flush() # We want any pending output first
-
-        # Prevent setuptools from downloading and thus installing
-        # build dependencies specified in setup_requires option of
-        # legacy setup.py by providing a crafted .pydistutils.cfg.
-        # This is used in complement to --no-build-isolation.
-        if not options._allow_picked_versions:
+            # Prevent setuptools from downloading and thus installing
+            # build dependencies specified in setup_requires option of
+            # legacy setup.py by providing a crafted .pydistutils.cfg.
+            # This is used in complement to --no-build-isolation.
             pip_home = tempfile.mkdtemp('pip-pydistutils-home')
             cleanup.append(lambda: zc.buildout.rmtree.rmtree(pip_home))
             with open(os.path.join(pip_home, '.pydistutils.cfg'), 'w') as f:
@@ -1870,24 +1885,60 @@ def call_pip_wheel(spec, dest, options):
                         "index_url = file:///dev/null")
             env['HOME'] = pip_home
 
-        subprocess.check_call(args, env=env)
+        if PIP_HAS_PYTHON_VERSION_WARNING_OPTION:
+            # Let pip display Python warnings only on first run.
+            if not hasattr(call_pip_command, 'displayed'):
+                call_pip_command.displayed = True
+            else:
+                args.append('--no-python-version-warning')
 
-        entries = os.listdir(dest)
-        try:
-            assert len(entries) == 1, "Got multiple entries afer pip wheel"
-            wheel = entries[0]
-            assert os.path.splitext(wheel)[1] == '.whl', "Expected a .whl"
-        except AssertionError:
-            logger.error(
-                "No .whl after successful pip wheel of %s",
-                spec)
-            raise
+        args.extend(operand)
 
-        return make_egg_after_pip_wheel(dest, wheel)
+        if log_level < logging.DEBUG:
+            # Log this only when buildout log level is quite low
+            logger.debug('Running pip %s', ' '.join(command[0:1] + operand))
+        if log_level < 0:
+            # Log this only when buildout log level is even lower
+            logger.debug(
+                '%s\nPYTHONPATH=%s\n', ' '.join(args), env['PYTHONPATH'])
+
+        sys.stdout.flush() # We want any pending output first
+        subprocess.check_call(args, env=env)
     finally:
         for f in cleanup:
             f()
 
+
+def call_pip_editable(path, dest, options, verbosity=-10):
+    """
+    Call `pip install --editable` from a subprocess to install a
+    the project in `path` into `dest` in editable mode.
+    """
+    call_pip_command(
+        ['install', '-t', dest], ['--editable', path],
+        options, verbosity)
+
+
+def call_pip_wheel(spec, dest, options):
+    """
+    Call `pip wheel` from a subprocess to install a
+    distribution specified by `spec` into `dest`.
+    Returns all the paths inside `dest` created by the above.
+    """
+    call_pip_command(['wheel', '-w', dest], [spec], options)
+
+    entries = os.listdir(dest)
+    try:
+        assert len(entries) == 1, "Got multiple entries afer pip wheel"
+        wheel = entries[0]
+        assert os.path.splitext(wheel)[1] == '.whl', "Expected a .whl"
+    except AssertionError:
+        logger.error("No .whl after successful pip wheel of %s", spec)
+        raise
+
+    return make_egg_after_pip_wheel(dest, wheel)
+
+
 def make_egg_after_pip_wheel(dest, wheel):
     unpack_wheel(os.path.join(dest, wheel), dest)
     assert len(os.listdir(dest)) == 2

src/zc/buildout/tests/easy_install.txt

@@ -348,7 +348,7 @@ reporting that a version was picked automatically:
     zc.buildout.easy_install DEBUG
       Fetching demoneeded 1.1 from: http://.../demoneeded-1.1.zip
     zc.buildout.easy_install DEBUG
-      Running pip install:...
+      Running pip wheel...
     zc.buildout.easy_install INFO
       Got demoneeded 1.1.
     zc.buildout.easy_install DEBUG

src/zc/buildout/tests/test_all.py

@@ -204,8 +204,7 @@ We should be able to deal with setup scripts that aren't setuptools based.
     ... # doctest: +ELLIPSIS
     Installing...
     Develop: '/sample-buildout/foo'
-    ...
-    Installed /sample-buildout/foo
+    Running pip install --editable /sample-buildout/foo
     ...
 
     >>> ls('develop-eggs')
@@ -216,10 +215,9 @@ We should be able to deal with setup scripts that aren't setuptools based.
     ... # doctest: +ELLIPSIS
     Installing...
     Develop: '/sample-buildout/foo'
-    in: '/sample-buildout/foo'
-    ... -q develop -mN -d /sample-buildout/develop-eggs/...
-
-
+    Running pip install --editable /sample-buildout/foo
+    ... -m pip install -t ... --editable /sample-buildout/foo
+    ...
     """
 
 def buildout_error_handling():