Merge pull request #930 from GandaG/build_clib

Add timestamp-based dependency system to build_clib

setuptools/command/__init__.py

@@ -2,7 +2,7 @@ __all__ = [
     'alias', 'bdist_egg', 'bdist_rpm', 'build_ext', 'build_py', 'develop',
     'easy_install', 'egg_info', 'install', 'install_lib', 'rotate', 'saveopts',
     'sdist', 'setopt', 'test', 'install_egg_info', 'install_scripts',
-    'register', 'bdist_wininst', 'upload_docs', 'upload',
+    'register', 'bdist_wininst', 'upload_docs', 'upload', 'build_clib',
 ]
 
 from distutils.command.bdist import bdist

setuptools/command/build_clib.py

+import distutils.command.build_clib as orig
+from distutils.errors import DistutilsSetupError
+from distutils import log
+from setuptools.dep_util import newer_pairwise_group
+
+
+class build_clib(orig.build_clib):
+    """
+    Override the default build_clib behaviour to do the following:
+
+    1. Implement a rudimentary timestamp-based dependency system
+       so 'compile()' doesn't run every time.
+    2. Add more keys to the 'build_info' dictionary:
+        * obj_deps - specify dependencies for each object compiled.
+                     this should be a dictionary mapping a key
+                     with the source filename to a list of
+                     dependencies. Use an empty string for global
+                     dependencies.
+        * cflags   - specify a list of additional flags to pass to
+                     the compiler.
+    """
+
+    def build_libraries(self, libraries):
+        for (lib_name, build_info) in libraries:
+            sources = build_info.get('sources')
+            if sources is None or not isinstance(sources, (list, tuple)):
+                raise DistutilsSetupError(
+                       "in 'libraries' option (library '%s'), "
+                       "'sources' must be present and must be "
+                       "a list of source filenames" % lib_name)
+            sources = list(sources)
+
+            log.info("building '%s' library", lib_name)
+
+            # Make sure everything is the correct type.
+            # obj_deps should be a dictionary of keys as sources
+            # and a list/tuple of files that are its dependencies.
+            obj_deps = build_info.get('obj_deps', dict())
+            if not isinstance(obj_deps, dict):
+                raise DistutilsSetupError(
+                       "in 'libraries' option (library '%s'), "
+                       "'obj_deps' must be a dictionary of "
+                       "type 'source: list'" % lib_name)
+            dependencies = []
+
+            # Get the global dependencies that are specified by the '' key.
+            # These will go into every source's dependency list.
+            global_deps = obj_deps.get('', list())
+            if not isinstance(global_deps, (list, tuple)):
+                raise DistutilsSetupError(
+                       "in 'libraries' option (library '%s'), "
+                       "'obj_deps' must be a dictionary of "
+                       "type 'source: list'" % lib_name)
+
+            # Build the list to be used by newer_pairwise_group
+            # each source will be auto-added to its dependencies.
+            for source in sources:
+                src_deps = [source]
+                src_deps.extend(global_deps)
+                extra_deps = obj_deps.get(source, list())
+                if not isinstance(extra_deps, (list, tuple)):
+                    raise DistutilsSetupError(
+                           "in 'libraries' option (library '%s'), "
+                           "'obj_deps' must be a dictionary of "
+                           "type 'source: list'" % lib_name)
+                src_deps.extend(extra_deps)
+                dependencies.append(src_deps)
+
+            expected_objects = self.compiler.object_filenames(
+                    sources,
+                    output_dir=self.build_temp
+                    )
+
+            if newer_pairwise_group(dependencies, expected_objects) != ([], []):
+                # First, compile the source code to object files in the library
+                # directory.  (This should probably change to putting object
+                # files in a temporary build directory.)
+                macros = build_info.get('macros')
+                include_dirs = build_info.get('include_dirs')
+                cflags = build_info.get('cflags')
+                objects = self.compiler.compile(
+                        sources,
+                        output_dir=self.build_temp,
+                        macros=macros,
+                        include_dirs=include_dirs,
+                        extra_postargs=cflags,
+                        debug=self.debug
+                        )
+
+            # Now "link" the object files together into a static library.
+            # (On Unix at least, this isn't really linking -- it just
+            # builds an archive.  Whatever.)
+            self.compiler.create_static_lib(
+                    expected_objects,
+                    lib_name,
+                    output_dir=self.build_clib,
+                    debug=self.debug
+                    )

setuptools/dep_util.py

+from distutils.dep_util import newer_group
+
+# yes, this is was almost entirely copy-pasted from
+# 'newer_pairwise()', this is just another convenience
+# function.
+def newer_pairwise_group(sources_groups, targets):
+    """Walk both arguments in parallel, testing if each source group is newer
+    than its corresponding target. Returns a pair of lists (sources_groups,
+    targets) where sources is newer than target, according to the semantics
+    of 'newer_group()'.
+    """
+    if len(sources_groups) != len(targets):
+        raise ValueError("'sources_group' and 'targets' must be the same length")
+
+    # build a pair of lists (sources_groups, targets) where source is newer
+    n_sources = []
+    n_targets = []
+    for i in range(len(sources_groups)):
+        if newer_group(sources_groups[i], targets[i]):
+            n_sources.append(sources_groups[i])
+            n_targets.append(targets[i])
+
+    return n_sources, n_targets

setuptools/tests/test_build_clib.py

+import pytest
+import os
+import shutil
+
+from unittest import mock
+from distutils.errors import DistutilsSetupError
+from setuptools.command.build_clib import build_clib
+from setuptools.dist import Distribution
+
+
+class TestBuildCLib:
+    @mock.patch(
+            'setuptools.command.build_clib.newer_pairwise_group'
+            )
+    def test_build_libraries(self, mock_newer):
+        dist = Distribution()
+        cmd = build_clib(dist)
+
+        # this will be a long section, just making sure all
+        # exceptions are properly raised
+        libs = [('example', {'sources': 'broken.c'})]
+        with pytest.raises(DistutilsSetupError):
+            cmd.build_libraries(libs)
+
+        obj_deps = 'some_string'
+        libs = [('example', {'sources': ['source.c'], 'obj_deps': obj_deps})]
+        with pytest.raises(DistutilsSetupError):
+            cmd.build_libraries(libs)
+
+        obj_deps = {'': ''}
+        libs = [('example', {'sources': ['source.c'], 'obj_deps': obj_deps})]
+        with pytest.raises(DistutilsSetupError):
+            cmd.build_libraries(libs)
+
+        obj_deps = {'source.c': ''}
+        libs = [('example', {'sources': ['source.c'], 'obj_deps': obj_deps})]
+        with pytest.raises(DistutilsSetupError):
+            cmd.build_libraries(libs)
+
+        # with that out of the way, let's see if the crude dependency
+        # system works
+        cmd.compiler = mock.MagicMock(spec=cmd.compiler)
+        mock_newer.return_value = ([],[])
+
+        obj_deps = {'': ('global.h',), 'example.c': ('example.h',)}
+        libs = [('example', {'sources': ['example.c'] ,'obj_deps': obj_deps})]
+
+        cmd.build_libraries(libs)
+        assert [['example.c', 'global.h', 'example.h']] in mock_newer.call_args[0]
+        assert not cmd.compiler.compile.called
+        assert cmd.compiler.create_static_lib.call_count == 1
+
+        # reset the call numbers so we can test again
+        cmd.compiler.reset_mock()
+
+        mock_newer.return_value = ''  # anything as long as it's not ([],[])
+        cmd.build_libraries(libs)
+        assert cmd.compiler.compile.call_count == 1
+        assert cmd.compiler.create_static_lib.call_count == 1

setuptools/tests/test_dep_util.py

+from setuptools.dep_util import newer_pairwise_group
+import os
+import pytest
+
+
+@pytest.fixture
+def groups_target(tmpdir):
+    """Sets up some older sources, a target and newer sources.
+    Returns a 3-tuple in this order.
+    """
+    creation_order = ['older.c', 'older.h', 'target.o', 'newer.c', 'newer.h']
+    mtime = 0
+
+    for i in range(len(creation_order)):
+        creation_order[i] = os.path.join(str(tmpdir), creation_order[i])
+        with open(creation_order[i], 'w'):
+            pass
+
+        # make sure modification times are sequential
+        os.utime(creation_order[i], (mtime, mtime))
+        mtime += 1
+
+    return creation_order[:2], creation_order[2], creation_order[3:]
+
+
+def test_newer_pairwise_group(groups_target):
+    older = newer_pairwise_group([groups_target[0]], [groups_target[1]])
+    newer = newer_pairwise_group([groups_target[2]], [groups_target[1]])
+    assert older == ([], [])
+    assert newer == ([groups_target[2]], [groups_target[1]])