Implement "entry points" for dynamic discovery of drivers and plugins.

Change setuptools to discover setup commands using an entry point group
called "distutils.commands".  Thanks to Ian Bicking for the suggestion that
led to designing this super-cool feature.

--HG--
branch : setuptools
extra : convert_revision : svn%3A6015fed2-1504-0410-9fe1-9d1591cc4771/sandbox/trunk/setuptools%4041152

setup.py

@@ -18,6 +18,9 @@ def get_description():
 VERSION = "0.6a0"
 
 from setuptools import setup, find_packages
+import sys
+
+from setuptools.command import __all__ as SETUP_COMMANDS
 
 setup(
     name="setuptools",
@@ -35,9 +38,15 @@ setup(
     packages = find_packages(),
     py_modules = ['pkg_resources', 'easy_install'],
     scripts = ['easy_install.py'],
+    
     zip_safe = False,   # We want 'python -m easy_install' to work  :(
 
-
+    entry_points = {
+        "distutils.commands" : [
+            "%(cmd)s = setuptools.command.%(cmd)s:%(cmd)s" % locals()
+            for cmd in SETUP_COMMANDS if cmd!="build_py" or sys.version>="2.4"
+        ],
+    },
 
     classifiers = [f.strip() for f in """
     Development Status :: 3 - Alpha
@@ -63,15 +72,6 @@ setup(
 
 
 
-
-
-
-
-
-
-
-
-
 
 
 

setuptools.egg-info/entry_points.txt

+[distutils.commands]
+rotate = setuptools.command.rotate:rotate
+develop = setuptools.command.develop:develop
+setopt = setuptools.command.setopt:setopt
+saveopts = setuptools.command.saveopts:saveopts
+egg_info = setuptools.command.egg_info:egg_info
+depends = setuptools.command.depends:depends
+upload = setuptools.command.upload:upload
+alias = setuptools.command.alias:alias
+easy_install = setuptools.command.easy_install:easy_install
+bdist_egg = setuptools.command.bdist_egg:bdist_egg
+install = setuptools.command.install:install
+test = setuptools.command.test:test
+install_lib = setuptools.command.install_lib:install_lib
+build_ext = setuptools.command.build_ext:build_ext
+sdist = setuptools.command.sdist:sdist
+

setuptools.txt

@@ -153,21 +153,20 @@ unless you need the associated ``setuptools`` feature.
     A string or list of strings specifying what other distributions need to
     be installed when this one is.  See the section below on `Declaring
     Dependencies`_ for details and examples of the format of this argument.
-    
+
+``entry_points``
+    A dictionary mapping entry point group names to strings or lists of strings
+    defining the entry points.  Entry points are used to support dynamic
+    discovery of services or plugins provided by a project.  See `Dynamic
+    Discovery of Services and Plugins`_ for details and examples of the format
+    of this argument.
+
 ``extras_require``
     A dictionary mapping names of "extras" (optional features of your project)
     to strings or lists of strings specifying what other distributions must be
     installed to support those features.  See the section below on `Declaring
     Dependencies`_ for details and examples of the format of this argument.
 
-``test_suite``
-    A string naming a ``unittest.TestCase`` subclass (or a module containing
-    one or more of them, or a method of such a subclass), or naming a function
-    that can be called with no arguments and returns a ``unittest.TestSuite``.
-    Specifying this argument enables use of the `test`_ command to run the
-    specified test suite, e.g. via ``setup.py test``.  See the section on the
-    `test`_ command below for more details.
-
 ``namespace_packages``
     A list of strings naming the project's "namespace packages".  A namespace
     package is a package that may be split across multiple project
@@ -180,6 +179,14 @@ unless you need the associated ``setuptools`` feature.
     does not contain any code.  See the section below on `Namespace Packages`_
     for more information.
 
+``test_suite``
+    A string naming a ``unittest.TestCase`` subclass (or a module containing
+    one or more of them, or a method of such a subclass), or naming a function
+    that can be called with no arguments and returns a ``unittest.TestSuite``.
+    Specifying this argument enables use of the `test`_ command to run the
+    specified test suite, e.g. via ``setup.py test``.  See the section on the
+    `test`_ command below for more details.
+
 ``eager_resources``
     A list of strings naming resources that should be extracted together, if
     any of them is needed, or if any C extensions included in the project are
@@ -516,7 +523,72 @@ either C or an external program that needs "real" files in your project before
 there's any possibility of ``eager_resources`` being relevant to your project.
 
 
+Extensible Applications and Frameworks
+======================================
+
+
+Dynamic Discovery of Services and Plugins
+-----------------------------------------
+
+``setuptools`` supports creating libraries that "plug in" to extensible
+applications and frameworks, by letting you register "entry points" in your
+project that can be imported by the application or framework.
+
+For example, suppose that a blogging tool wants to support plugins
+that provide translation for various file types to the blog's output format.
+The framework might define an "entry point group" called ``blogtool.parsers``,
+and then allow plugins to register entry points for the file extensions they
+support.
+
+This would allow people to create distributions that contain one or more
+parsers for different file types, and then the blogging tool would be able to
+find the parsers at runtime by looking up an entry point for the file
+extension (or mime type, or however it wants to).
+
+Note that if the blogging tool includes parsers for certain file formats, it
+can register these as entry points in its own setup script, which means it
+doesn't have to special-case its built-in formats.  They can just be treated
+the same as any other plugin's entry points would be.
 
+If you're creating a project that plugs in to an existing application or
+framework, you'll need to know what entry points or entry point groups are
+defined by that application or framework.  Then, you can register entry points
+in your setup script.  Here are a few examples of ways you might register an
+``.rst`` file parser entry point in the ``blogtool.parsers`` entry point group,
+for our hypothetical blogging tool::
+
+    setup(
+        # ...
+        entry_points = {'blogtool.parsers': '.rst = some_module:SomeClass'}
+    )
+
+    setup(
+        # ...
+        entry_points = {'blogtool.parsers': ['.rst = some_module:a_func']}
+    )
+
+    setup(
+        # ...
+        entry_points = """
+            [blogtool.parsers]
+            .rst = some.nested.module:SomeClass.some_classmethod [reST]
+        """,
+        extras_require = dict(reST = "Docutils>=0.3.5")
+    )
+
+The ``entry_points`` argument to ``setup()`` accepts either a string with
+``.ini``-style sections, or a dictionary mapping entry point group names to
+either strings or lists of strings containing entry point specifiers.  An
+entry point specifier consists of a name and value, separated by an ``=``
+sign.  The value consists of a dotted module name, optionally followed by a
+``:`` and a dotted identifier naming an object within the module.  It can
+also include a bracketed list of "extras" that are required for the entry
+point to be used.  When the invoking application or framework requests loading
+of an entry point, any requirements implied by the associated extras will be
+passed to ``pkg_resources.require()``, so that an appropriate error message
+can be displayed if the needed package(s) are missing.  (Of course, the
+invoking app or framework can ignore such errors if it wants to make an entry
+point optional if a requirement isn't installed.)
 
 
 "Development Mode"
@@ -1072,12 +1144,13 @@ Last, but not least, the ``develop`` command invokes the ``build_ext -i``
 command to ensure any C extensions in the project have been built and are
 up-to-date, and the ``egg_info`` command to ensure the project's metadata is
 updated (so that the runtime and wrappers know what the project's dependencies
-are).  If you make changes to the project's metadata or C extensions, you
-should rerun the ``develop`` command (or ``egg_info``, or ``build_ext -i``) in
-order to keep the project up-to-date.  If you add or rename any of the
-project's scripts, you should re-run ``develop`` against all relevant staging
-areas to update the wrapper scripts.  Most other kinds of changes to your
-project should not require any build operations or rerunning ``develop``.
+are).  If you make any changes to the project's setup script or C extensions,
+you should rerun the ``develop`` command against all relevant staging areas to
+keep the project's scripts, metadata and extensions up-to-date.  Most other
+kinds of changes to your project should not require any build operations or
+rerunning ``develop``, but keep in mind that even minor changes to the setup
+script (e.g. changing an entry point definition) require you to re-run the
+``develop`` or ``test`` commands to keep the distribution updated.
 
 Here are the options that the ``develop`` command accepts.  Note that they
 affect the project's dependencies as well as the project itself, so if you have
@@ -1442,8 +1515,39 @@ in this page <setuptools?action=subscribe>`_ to see when new documentation is
 added or updated.
 
 
+Adding Commands
+===============
+
+You can create add-on packages that extend setuptools with additional commands
+by defining entry points in the ``distutils.commands`` group.  For example, if
+you wanted to add a ``foo`` command, you might add something like this to your
+setup script::
+
+    setup(
+        # ...
+        entry_points = {
+            "distutils.commands": [
+                "foo = mypackage.some_module:foo",
+            ],
+        },
+    )
+
+Assuming, of course, that the ``foo`` class in ``mypackage.some_module`` is
+a ``setuptools.Command`` subclass.
+
+Once a project containing such entry points has been activated on ``sys.path``,
+(e.g. by running "install" or "develop" with a site-packages installation
+directory) the command(s) will be available to any ``setuptools``-based setup
+scripts.  It is not necessary to use the ``--command-packages`` option or
+to monkeypatch the ``distutils.command`` package to install your commands;
+``setuptools`` automatically adds a wrapper to the distutils to search for
+entry points in the active distributions on ``sys.path``.  In fact, this is
+how setuptools' own commands are installed: the setuptools setup script defines
+entry points for them.
+
+
 Subclassing ``Command``
-=======================
+-----------------------
 
 XXX
 
@@ -1492,9 +1596,15 @@ Release Notes/Change History
  * Fixed ``pkg_resources.resource_exists()`` not working correctly, along with
    some other resource API bugs.
 
+ * Added ``entry_points`` argument to ``setup()``
 
  * Many ``pkg_resources`` API changes and enhancements:
 
+   * Added ``EntryPoint``, ``get_entry_map``, ``load_entry_point``, and
+     ``get_entry_info`` APIs for dynamic plugin discovery.
+
+   * ``list_resources`` is now ``resource_listdir`` (and it actually works)
+
    * Resource API functions like ``resource_string()`` that accepted a package
      name and resource name, will now also accept a ``Requirement`` object in
      place of the package name (to allow access to non-package data files in
@@ -1532,7 +1642,8 @@ Release Notes/Change History
 
    * Distribution objects no longer have an ``installed_on()`` method, and the
      ``install_on()`` method is now ``activate()`` (but may go away altogether
-     soon).  The ``depends()`` method has also been renamed to ``requires()``.
+     soon).  The ``depends()`` method has also been renamed to ``requires()``,
+     and ``InvalidOption`` is now ``UnknownExtra``.
 
    * ``find_distributions()`` now takes an additional argument called ``only``,
      that tells it to only yield distributions whose location is the passed-in

setuptools/command/__init__.py

-import distutils.command
-
 __all__ = [
-    'test', 'develop', 'bdist_egg', 'saveopts', 'setopt', 'rotate', 'alias'
+    'alias', 'bdist_egg', 'build_ext', 'build_py', 'depends', 'develop',
+    'easy_install', 'egg_info', 'install', 'install_lib', 'rotate', 'saveopts',
+    'sdist', 'setopt', 'test', 'upload',
 ]
 
 
-# Make our commands available as though they were part of the distutils
-
-distutils.command.__path__.extend(__path__)
-distutils.command.__all__.extend(
-    [cmd for cmd in __all__ if cmd not in distutils.command.__all__]
-    )
-
 from distutils.command.bdist import bdist
 
 if 'egg' not in bdist.format_commands:

setuptools/command/egg_info.py

@@ -8,7 +8,7 @@ from setuptools import Command
 from distutils.errors import *
 from distutils import log
 from pkg_resources import parse_requirements, safe_name, \
-    safe_version, yield_lines
+    safe_version, yield_lines, EntryPoint
 from setuptools.dist import iter_distribution_names
 
 class egg_info(Command):
@@ -95,7 +95,7 @@ class egg_info(Command):
                 metadata.write_pkg_info(self.egg_info)
             finally:
                 metadata.name, metadata.version = oldname, oldver
-
+        self.write_entry_points()
         self.write_requirements()
         self.write_toplevel_names()
         self.write_or_delete_dist_arg('namespace_packages')
@@ -183,23 +183,23 @@ class egg_info(Command):
             if not self.dry_run:
                 os.unlink(filename)
 
+    def write_entry_points(self):
+        ep = getattr(self.distribution,'entry_points',None)
+        if ep is None:
+            return
 
+        epname = os.path.join(self.egg_info,"entry_points.txt")
+        log.info("writing %s", epname)
 
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+        if not self.dry_run:
+            f = open(epname, 'wt')
+            if isinstance(ep,basestring):
+                f.write(ep)
+            else:
+                for section, contents in ep.items():
+                    if not isinstance(contents,basestring):
+                        contents = EntryPoint.parse_list(section, contents)
+                        contents = '\n'.join(map(str,contents.values()))
+                    f.write('[%s]\n%s\n\n' % (section,contents))
+            f.close()
 

setuptools/dist.py

@@ -11,6 +11,32 @@ from distutils.errors import DistutilsOptionError, DistutilsPlatformError
 from distutils.errors import DistutilsSetupError
 import setuptools, pkg_resources
 
+def get_command_class(self, command):
+    """Pluggable version of get_command_class()"""
+    if command in self.cmdclass:
+        return self.cmdclass[command]
+
+    for dist in pkg_resources.working_set:
+        if dist.get_entry_info('distutils.commands',command):
+            cmdclass = dist.load_entry_point('distutils.commands',command)
+            self.cmdclass[command] = cmdclass
+            return cmdclass
+    else:
+        return _old_get_command_class(self, command)
+
+def print_commands(self):
+    for dist in pkg_resources.working_set:
+        for cmd,ep in dist.get_entry_map('distutils.commands').items():
+            if cmd not in self.cmdclass:
+                cmdclass = ep.load() # don't require extras, we're not running
+                self.cmdclass[cmd] = cmdclass
+    return _old_print_commands(self)
+
+for meth in 'print_commands', 'get_command_class':
+    if getattr(_Distribution,meth).im_func.func_globals is not globals():
+        globals()['_old_'+meth] = getattr(_Distribution,meth)
+        setattr(_Distribution, meth, globals()[meth])
+
 sequence = tuple, list
 
 class Distribution(_Distribution):
@@ -80,6 +106,21 @@ class Distribution(_Distribution):
     distribution for the included and excluded features.
     """
 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
     def __init__ (self, attrs=None):
         have_package_data = hasattr(self, "package_data")
         if not have_package_data:
@@ -93,15 +134,9 @@ class Distribution(_Distribution):
         self.zip_safe = None
         self.namespace_packages = None
         self.eager_resources = None
+        self.entry_points = None
         _Distribution.__init__(self,attrs)
-        if not have_package_data:
-            from setuptools.command.build_py import build_py
-            self.cmdclass.setdefault('build_py',build_py)
 
-        self.cmdclass.setdefault('build_ext',build_ext)
-        self.cmdclass.setdefault('install',install)
-        self.cmdclass.setdefault('install_lib',install_lib)
-        self.cmdclass.setdefault('sdist',sdist)
 
     def parse_command_line(self):
         """Process features after parsing command line options"""
@@ -121,6 +156,12 @@ class Distribution(_Distribution):
 
 
 
+
+
+
+
+
+
     def finalize_options(self):
         _Distribution.finalize_options(self)
 
@@ -171,6 +212,12 @@ class Distribution(_Distribution):
                     "namespace package %r" % nsp
                 )
 
+        if self.entry_points is not None:
+            try:
+                pkg_resources.EntryPoint.parse_map(self.entry_points)
+            except ValueError, e:
+                raise DistutilsSetupError(e)
+
     def _set_global_opts_from_features(self):
         """Add --with-X/--without-X options based on optional features"""
 
@@ -197,12 +244,6 @@ class Distribution(_Distribution):
 
 
 
-
-
-
-
-
-
     def _finalize_features(self):
         """Add/remove features and resolve dependencies between them"""
 
@@ -420,7 +461,7 @@ class Distribution(_Distribution):
             src,alias = aliases[command]
             del aliases[command]    # ensure each alias can expand only once!
             import shlex
-            args[:1] = shlex.split(alias,True)            
+            args[:1] = shlex.split(alias,True)
             command = args[0]
 
         nargs = _Distribution._parse_command_opts(self, parser, args)

setuptools/tests/test_resources.py

@@ -185,7 +185,7 @@ class DistroTests(TestCase):
             d,"Twisted>=1.5 fcgiapp>=0.1 ZConfig>=2.0 docutils>=0.3".split(),
             ["fastcgi", "docgen"]
         )
-        self.assertRaises(InvalidOption, d.requires, ["foo"])
+        self.assertRaises(UnknownExtra, d.requires, ["foo"])
 
 
 
@@ -203,6 +203,88 @@ class DistroTests(TestCase):
 
 
 
+class EntryPointTests(TestCase):
+
+    def assertfields(self, ep):
+        self.assertEqual(ep.name,"foo")
+        self.assertEqual(ep.module_name,"setuptools.tests.test_resources")
+        self.assertEqual(ep.attrs, ("EntryPointTests",))
+        self.assertEqual(ep.extras, ("x",))
+        self.failUnless(ep.load() is EntryPointTests)
+        self.assertEqual(
+            str(ep),
+            "foo = setuptools.tests.test_resources:EntryPointTests [x]"
+        )
+
+    def testBasics(self):
+        ep = EntryPoint(
+            "foo", "setuptools.tests.test_resources", ["EntryPointTests"],
+            ["x"]
+        )
+        self.assertfields(ep)
+
+    def testParse(self):
+        s = "foo = setuptools.tests.test_resources:EntryPointTests [x]"
+        ep = EntryPoint.parse(s)
+        self.assertfields(ep)
+
+        ep = EntryPoint.parse("bar baz=  spammity[PING]")
+        self.assertEqual(ep.name,"bar baz")
+        self.assertEqual(ep.module_name,"spammity")
+        self.assertEqual(ep.attrs, ())
+        self.assertEqual(ep.extras, ("ping",))
+
+        ep = EntryPoint.parse(" fizzly =  wocka:foo")
+        self.assertEqual(ep.name,"fizzly")
+        self.assertEqual(ep.module_name,"wocka")
+        self.assertEqual(ep.attrs, ("foo",))
+        self.assertEqual(ep.extras, ())
+
+
+
+
+
+    def testRejects(self):
+        for ep in [
+            "foo", "x=1=2", "x=a:b:c", "q=x/na", "fez=pish:tush-z", "x=f[a]>2",
+        ]:
+            try: EntryPoint.parse(ep)
+            except ValueError: pass
+            else: raise AssertionError("Should've been bad", ep)
+
+    def checkSubMap(self, m):
+        self.assertEqual(str(m),
+            "{"
+            "'feature2': EntryPoint.parse("
+                "'feature2 = another.module:SomeClass [extra1,extra2]'), "
+            "'feature1': EntryPoint.parse("
+                "'feature1 = somemodule:somefunction')"
+            "}"
+        )
+
+    submap_str = """
+            # define features for blah blah
+            feature1 = somemodule:somefunction
+            feature2 = another.module:SomeClass [extra1,extra2]
+    """
+
+    def testParseList(self):
+        self.checkSubMap(EntryPoint.parse_list("xyz", self.submap_str))
+        self.assertRaises(ValueError, EntryPoint.parse_list, "x a", "foo=bar")
+        self.assertRaises(ValueError, EntryPoint.parse_list, "x",
+            ["foo=baz", "foo=bar"])
+
+    def testParseMap(self):
+        m = EntryPoint.parse_map({'xyz':self.submap_str})
+        self.checkSubMap(m['xyz'])
+        self.assertEqual(m.keys(),['xyz'])
+        m = EntryPoint.parse_map("[xyz]\n"+self.submap_str)
+        self.checkSubMap(m['xyz'])
+        self.assertEqual(m.keys(),['xyz'])
+        self.assertRaises(ValueError, EntryPoint.parse_map, ["[xyz]", "[xyz]"])
+        self.assertRaises(ValueError, EntryPoint.parse_map, self.submap_str)
+
+
 class RequirementsTests(TestCase):
 
     def testBasics(self):