build_dso: Fix support for develop install (pip install -e)

When a package is installed in development mode by `pip install -e`, it
calls `python setup.py develop` under the covers

    https://setuptools.readthedocs.io/en/latest/setuptools.html#development-mode

However it currently skips build_dso in that mode which leads to build
failure if anything - e.g. extension - is depending on a dso:

    + pip install -v -e .
    Created temporary directory: /tmp/pip-ephem-wheel-cache-OJgoBO
    Created temporary directory: /tmp/pip-req-tracker-CrRj5z
    Created requirements tracker '/tmp/pip-req-tracker-CrRj5z'
    Created temporary directory: /tmp/pip-install-hiu8rq
    Obtaining file:///home/kirr/src/tools/py/pypa/setuptools_dso/example
      Added file:///home/kirr/src/tools/py/pypa/setuptools_dso/example to build tracker '/tmp/pip-req-tracker-CrRj5z'
      Running setup.py (path:/home/kirr/src/tools/py/pypa/setuptools_dso/example/setup.py) egg_info for package from file:///home/kirr/src/tools/py/pypa/setuptools_dso/example
        Running command python setup.py egg_info
        running egg_info
        creating src/dsodemo.egg-info
        writing src/dsodemo.egg-info/PKG-INFO
        writing top-level names to src/dsodemo.egg-info/top_level.txt
        writing dependency_links to src/dsodemo.egg-info/dependency_links.txt
        writing manifest file 'src/dsodemo.egg-info/SOURCES.txt'
        reading manifest file 'src/dsodemo.egg-info/SOURCES.txt'
        writing manifest file 'src/dsodemo.egg-info/SOURCES.txt'
      Source in /home/kirr/src/tools/py/pypa/setuptools_dso/example has version 0.1, which satisfies requirement dsodemo==0.1 from file:///home/kirr/src/tools/py/pypa/setuptools_dso/example
      Removed dsodemo==0.1 from file:///home/kirr/src/tools/py/pypa/setuptools_dso/example from build tracker '/tmp/pip-req-tracker-CrRj5z'
    Installing collected packages: dsodemo
      Running setup.py develop for dsodemo
        Running command /home/kirr/src/tools/py/pypa/setuptools_dso/env/bin/python -c "import setuptools, tokenize;__file__='/home/kirr/src/tools/py/pypa/setuptools_dso/example/setup.py';f=getattr(tokenize, 'open', open)(__file__);code=f.read().replace('\r\n', '\n');f.close();exec(compile(code, __file__, 'exec'))" develop --no-deps
        running develop
        Checking .pth file support in /home/kirr/src/tools/py/pypa/setuptools_dso/env/lib/python2.7/site-packages/
        /home/kirr/src/tools/py/pypa/setuptools_dso/env/bin/python -E -c pass
        TEST PASSED: /home/kirr/src/tools/py/pypa/setuptools_dso/env/lib/python2.7/site-packages/ appears to support .pth files
        running egg_info
        writing src/dsodemo.egg-info/PKG-INFO
        writing top-level names to src/dsodemo.egg-info/top_level.txt
        writing dependency_links to src/dsodemo.egg-info/dependency_links.txt
        reading manifest file 'src/dsodemo.egg-info/SOURCES.txt'
        writing manifest file 'src/dsodemo.egg-info/SOURCES.txt'
        running build_ext
        Traceback (most recent call last):
          File "<string>", line 1, in <module>
          File "/home/kirr/src/tools/py/pypa/setuptools_dso/example/setup.py", line 26, in <module>
            x_dsos = [dso],
          File "/home/kirr/src/tools/py/pypa/setuptools_dso/env/local/lib/python2.7/site-packages/setuptools_dso/__init__.py", line 37, in setup
            _setup(**kws)
          File "/home/kirr/src/tools/py/pypa/setuptools_dso/env/local/lib/python2.7/site-packages/setuptools/__init__.py", line 145, in setup
            return distutils.core.setup(**attrs)
          File "/usr/lib/python2.7/distutils/core.py", line 151, in setup
            dist.run_commands()
          File "/usr/lib/python2.7/distutils/dist.py", line 953, in run_commands
            self.run_command(cmd)
          File "/usr/lib/python2.7/distutils/dist.py", line 972, in run_command
            cmd_obj.run()
          File "/home/kirr/src/tools/py/pypa/setuptools_dso/env/local/lib/python2.7/site-packages/setuptools/command/develop.py", line 38, in run
            self.install_for_development()
          File "/home/kirr/src/tools/py/pypa/setuptools_dso/env/local/lib/python2.7/site-packages/setuptools/command/develop.py", line 140, in install_for_development
            self.run_command('build_ext')
          File "/usr/lib/python2.7/distutils/cmd.py", line 326, in run_command
            self.distribution.run_command(command)
          File "/usr/lib/python2.7/distutils/dist.py", line 972, in run_command
            cmd_obj.run()
          File "/home/kirr/src/tools/py/pypa/setuptools_dso/env/local/lib/python2.7/site-packages/setuptools_dso/dsocmd.py", line 395, in run
            _build_ext.run(self)
          File "/home/kirr/src/tools/py/pypa/setuptools_dso/env/local/lib/python2.7/site-packages/setuptools/command/build_ext.py", line 78, in run
            _build_ext.run(self)
          File "/usr/lib/python2.7/distutils/command/build_ext.py", line 340, in run
            self.build_extensions()
          File "/usr/lib/python2.7/distutils/command/build_ext.py", line 449, in build_extensions
            self.build_extension(ext)
          File "/home/kirr/src/tools/py/pypa/setuptools_dso/env/local/lib/python2.7/site-packages/setuptools_dso/dsocmd.py", line 406, in build_extension
            self.dso2lib_pre(ext)
          File "/home/kirr/src/tools/py/pypa/setuptools_dso/env/local/lib/python2.7/site-packages/setuptools_dso/dsocmd.py", line 146, in dso2lib_pre
            raise RuntimeError("Unable to find DSO %s needed by extension %s"%(dso, ext.name))
        RuntimeError: Unable to find DSO dsodemo.lib.demo needed by extension dsodemo.ext.dtest

-> Fix it by making build_ext to depend on build_dso.

src/setuptools_dso/dsocmd.py

@@ -382,6 +382,10 @@ class build_dso(dso2libmixin, Command):
                 self.copy_file(outbaselib, baselib_dst)
 
 class build_ext(dso2libmixin, _build_ext):
+
+    # allow build_ext to depend on other commands
+    sub_commands = _build_ext.sub_commands[:]
+
     def finalize_options(self):
         _build_ext.finalize_options(self)
 
@@ -389,6 +393,10 @@ class build_ext(dso2libmixin, _build_ext):
         self.library_dirs = massage_dir_list([self.build_lib]  , self.library_dirs or [])
 
     def run(self):
+        # original setuptools/distutils don't call sub_commands for build_ext
+        for cmd_name in self.get_sub_commands():
+            self.run_command(cmd_name)
+
         # the Darwin linker errors if given non-existant directories :(
         [self.mkpath(D) for D in self.library_dirs]
         _build_ext.run(self)
@@ -466,3 +474,13 @@ def _needs_builddso(command, right_before=None):
 
 from distutils.command.build import build
 _needs_builddso(build, right_before='build_clib')
+
+
+# depend build_ext: build_dso, for DSOs to be automatically built on
+# `setup.py develop` (= `pip install -e`).
+#
+# `setup.py develop` does not call build and instead calls `build_ext -i`
+# directly without providing any kind of sub_commands support.
+#
+# -> so we hook into build_ext to make sure build_dso is also called.
+_needs_builddso(build_ext)

testme.sh

@@ -44,3 +44,17 @@ pip install -v .
 cd ..
 
 python -m dsodemo.cli
+
+
+# install in development mode
+pip uninstall -y dsodemo
+python -m dsodemo.cli 2>/dev/null && die "error: dsodemo not uninstalled"
+
+cd example
+python setup.py clean -a
+git clean -fdx
+(cd src && python -m dsodemo.cli 2>/dev/null) && die "error: worktree must be clean"
+pip install -v -e .
+
+cd ..
+python -m dsodemo.cli