Distribution metadata parsing: distribution objects can now extract their

version from PKG-INFO and their dependencies from depends.txt, including
optional dependencies.

--HG--
branch : setuptools
extra : convert_revision : svn%3A6015fed2-1504-0410-9fe1-9d1591cc4771/sandbox/trunk/setuptools%4041007

pkg_resources.py

@@ -14,23 +14,19 @@ The package resource API is designed to work with normal filesystem packages,
 method.
 """
 __all__ = [
-    'register_loader_type', 'get_provider', 'IResourceProvider',
+    'register_loader_type', 'get_provider', 'IResourceProvider', 
     'ResourceManager', 'AvailableDistributions', 'require', 'resource_string',
     'resource_stream', 'resource_filename', 'set_extraction_path',
     'cleanup_resources', 'parse_requirements', 'parse_version',
-    'compatible_platforms', 'get_platform',
+    'compatible_platforms', 'get_platform', 'IMetadataProvider',
     'ResolutionError', 'VersionConflict', 'DistributionNotFound',
-    'Distribution', 'Requirement', # 'glob_resources'
+    'InvalidOption', 'Distribution', 'Requirement', 'yield_lines',
+    'split_sections', # 'glob_resources'
 ]
 
 import sys, os, zipimport, time, re
 
-def _sort_dists(dists):
-    tmp = [(dist.version,dist) for dist in dists]
-    tmp.sort()
-    dists[::-1] = [d for v,d in tmp]
-
-class ResolutionError(ImportError):
+class ResolutionError(Exception):
     """Abstract base for dependency resolution errors"""
 
 class VersionConflict(ResolutionError):
@@ -39,6 +35,10 @@ class VersionConflict(ResolutionError):
 class DistributionNotFound(ResolutionError):
     """A requested distribution was not found"""
 
+class InvalidOption(ResolutionError):
+    """Invalid or unrecognized option name for a distribution"""
+
+
 _provider_factories = {}
 
 def register_loader_type(loader_type, provider_factory):
@@ -80,7 +80,22 @@ def compatible_platforms(provided,required):
     return False
 
 
-class IResourceProvider:
+class IMetadataProvider:
+
+    def has_metadata(name):
+        """Does the package's distribution contain the named metadata?"""
+
+    def get_metadata(name):
+        """The named metadata resource as a string"""
+
+    def get_metadata_lines(name):
+        """Yield named metadata resource as list of non-blank non-comment lines
+
+       Leading and trailing whitespace is stripped from each line, and lines
+       with ``#`` as the first non-blank character are omitted.
+       """
+
+class IResourceProvider(IMetadataProvider):
 
     """An object that provides access to package resources"""
 
@@ -102,25 +117,10 @@ class IResourceProvider:
     def has_resource(resource_name):
         """Does the package contain the named resource?"""
 
-    def has_metadata(name):
-        """Does the package's distribution contain the named metadata?"""
-
-    def get_metadata(name):
-        """The named metadata resource as a string"""
-
-    def get_metadata_lines(name):
-        """Yield named metadata resource as list of non-blank non-comment lines
-
-       Leading and trailing whitespace is stripped from each line, and lines
-       with ``#`` as the first non-blank character are omitted.
-       """
-
     # XXX list_resources?  glob_resources?
 
 
 
-
-
 class AvailableDistributions(object):
     """Searchable snapshot of distributions on a search path"""
 
@@ -417,7 +417,6 @@ def require(*requirements):
     XXX This doesn't work yet, because:
 
         * get_distro_source() isn't implemented
-        * Distribution.depends() isn't implemented
         * Distribution.install_on() isn't implemented
         * Requirement.options isn't implemented
         * AvailableDistributions.resolve() is untested
@@ -449,6 +448,7 @@ def require(*requirements):
 
 
 
+
 class DefaultProvider:
     """Provides access to package resources in the filesystem"""
 
@@ -746,11 +746,12 @@ class Distribution(object):
         if name:
             self.name = name.replace('_','-')
         if version:
-            self.version = version.replace('_','-')
+            self._version = version.replace('_','-')
 
         self.py_version = py_version
         self.platform = platform
         self.path = path_str
+        self.metadata = metadata
 
     def installed_on(self,path=None):
         """Is this distro installed on `path`? (defaults to ``sys.path``)"""
@@ -776,7 +777,6 @@ class Distribution(object):
 
 
 
-
     # These properties have to be lazy so that we don't have to load any
     # metadata until/unless it's actually needed.  (i.e., some distributions
     # may not know their name or version without loading PKG-INFO)
@@ -800,17 +800,58 @@ class Distribution(object):
 
     parsed_version = property(parsed_version)
 
+    #@property
+    def version(self):
+        try:
+            return self._version
+        except AttributeError:
+            for line in self.metadata.get_metadata_lines('PKG-INFO'):
+                if line.lower().startswith('version:'):
+                    self._version = line.split(':',1)[1].strip()
+                    return self._version
+            else:
+                raise AttributeError(
+                    "Missing Version: header in PKG-INFO", self
+                )
+    version = property(version)
 
 
 
 
+    #@property
+    def _dep_map(self):
+        try:
+            return self.__dep_map
+        except AttributeError:
+            dm = self.__dep_map = {None: []}
+            if self.metadata.has_metadata('depends.txt'):
+                for section,contents in split_sections(
+                    self.metadata.get_metadata_lines('depends.txt')
+                ):
+                    dm[section] = list(parse_requirements(contents))
+            return dm
 
+    _dep_map = property(_dep_map)
 
+    def depends(self,options=()):
+        """List of Requirements needed for this distro if `options` are used"""
+        dm = self._dep_map
+        deps = []
+        deps.extend(dm.get(None,()))
 
+        for opt in options:
+            try:
+                deps.extend(dm[opt.lower()])
+            except KeyError:
+                raise InvalidOption("No such option", self, opt)
+        return deps
 
 
 
-
+def _sort_dists(dists):
+    tmp = [(dist.version,dist) for dist in dists]
+    tmp.sort()
+    dists[::-1] = [d for v,d in tmp]
 
 
 
@@ -941,6 +982,33 @@ def _ensure_directory(path):
         os.makedirs(dirname)
 
 
+def split_sections(s):
+    """Split a string or iterable thereof into (section,content) pairs
+
+    Each ``section`` is a lowercase version of the section header ("[section]")
+    and each ``content`` is a list of stripped lines excluding blank lines and
+    comment-only lines.  If there are any such lines before the first section
+    header, they're returned in a first ``section`` of ``None``.
+    """
+    section = None
+    content = []
+    for line in yield_lines(s):
+        if line.startswith("["):
+            if line.endswith("]"):
+                if content:
+                    yield section, content
+                section = line[1:-1].strip().lower()
+                content = []
+            else:
+                raise ValueError("Invalid section heading", line)
+        else:
+            content.append(line)
+
+    # wrap up last segment
+    if content:
+        yield section, content
+
+
 # Set up global resource manager
 
 _manager = ResourceManager()
@@ -952,3 +1020,6 @@ def _initialize(g):
 _initialize(globals())
 
 
+
+
+

setuptools/tests/test_resources.py

@@ -2,6 +2,23 @@ from unittest import TestCase, makeSuite
 from pkg_resources import *
 import pkg_resources, sys
 
+class Metadata:
+    """Mock object to return metadata as if from an on-disk distribution"""
+
+    def __init__(self,*pairs):
+        self.metadata = dict(pairs)
+
+    def has_metadata(self,name):
+        return name in self.metadata
+
+    def get_metadata(self,name):
+        return self.metadata[name]
+
+    def get_metadata_lines(self,name):
+        return yield_lines(self.get_metadata(name))
+
+
+
 class DistroTests(TestCase):
 
     def testCollection(self):
@@ -26,13 +43,11 @@ class DistroTests(TestCase):
         self.assertEqual(
             [dist.version for dist in ad['FooPkg']], ['1.4','1.3-1','1.2']
         )
-
         # Removing a distribution leaves sequence alone
         ad.remove(ad['FooPkg'][1])
         self.assertEqual(
             [dist.version for dist in ad.get('FooPkg')], ['1.4','1.2']
         )
-
         # And inserting adds them in order
         ad.add(Distribution.from_filename("FooPkg-1.9.egg"))
         self.assertEqual(
@@ -51,34 +66,11 @@ class DistroTests(TestCase):
 
         # If the first matching distro is unsuitable, it's a version conflict
         path.insert(0,"FooPkg-1.2-py2.4.egg")
-        self.assertRaises(VersionConflict, ad.best_match, req, path)        
+        self.assertRaises(VersionConflict, ad.best_match, req, path)
 
         # If more than one match on the path, the first one takes precedence
         path.insert(0,"FooPkg-1.4-py2.4-win32.egg")
         self.assertEqual(ad.best_match(req,path).version, '1.4')
-        
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
 
     def checkFooPkg(self,d):
         self.assertEqual(d.name, "FooPkg")
@@ -105,6 +97,55 @@ class DistroTests(TestCase):
         d = Distribution.from_filename("FooPkg-1.3_1-py2.4-win32.egg")
         self.checkFooPkg(d)
 
+    def testDistroMetadata(self):
+        d = Distribution(
+            "/some/path", name="FooPkg", py_version="2.4", platform="win32",
+            metadata = Metadata(
+                ('PKG-INFO',"Metadata-Version: 1.0\nVersion: 1.3-1\n")
+            )
+        )
+        self.checkFooPkg(d)
+
+
+    def distDepends(self, txt):
+        return Distribution("/foo", metadata=Metadata(('depends.txt', txt)))
+
+    def checkDepends(self, dist, txt, opts=()):
+        self.assertEqual(
+            list(dist.depends(opts)),
+            list(parse_requirements(txt))
+        )
+
+    def testDistroDependsSimple(self):
+        for v in "Twisted>=1.5", "Twisted>=1.5\nZConfig>=2.0":
+            self.checkDepends(self.distDepends(v), v)
+
+
+    def testDistroDependsOptions(self):
+        d = self.distDepends("""
+            Twisted>=1.5
+            [docgen]
+            ZConfig>=2.0
+            docutils>=0.3
+            [fastcgi]
+            fcgiapp>=0.1""")
+        self.checkDepends(d,"Twisted>=1.5")
+        self.checkDepends(
+            d,"Twisted>=1.5 ZConfig>=2.0 docutils>=0.3".split(), ["docgen"]
+        )
+        self.checkDepends(
+            d,"Twisted>=1.5 fcgiapp>=0.1".split(), ["fastcgi"]
+        )
+        self.checkDepends(
+            d,"Twisted>=1.5 ZConfig>=2.0 docutils>=0.3 fcgiapp>=0.1".split(),
+            ["docgen","fastcgi"]
+        )
+        self.checkDepends(
+            d,"Twisted>=1.5 fcgiapp>=0.1 ZConfig>=2.0 docutils>=0.3".split(),
+            ["fastcgi", "docgen"]
+        )
+        self.assertRaises(InvalidOption, d.depends, ["foo"])
+
 
 
 
@@ -174,6 +215,35 @@ class ParseTests(TestCase):
         ]:
             self.assertEqual(list(pkg_resources.yield_lines(inp)),out)
 
+    def testSplitting(self):
+        self.assertEqual(
+            list(
+                pkg_resources.split_sections("""
+                    x
+                    [Y]
+                    z
+
+                    a
+                    [b ]
+                    # foo
+                    c
+                    [ d]
+                    [q]
+                    v
+                    """
+                )
+            ),
+            [(None,["x"]), ("y",["z","a"]), ("b",["c"]), ("q",["v"])]
+        )
+        self.assertRaises(ValueError,list,pkg_resources.split_sections("[foo"))
+
+
+
+
+
+
+
+
     def testSimpleRequirements(self):
         self.assertEqual(
             list(parse_requirements('Twis-Ted>=1.2-1')),
@@ -194,6 +264,18 @@ class ParseTests(TestCase):
         self.assertRaises(ValueError,Requirement.parse,"#")
 
 
+    def testVersionEquality(self):
+        def c(s1,s2):
+            p1, p2 = parse_version(s1),parse_version(s2)
+            self.assertEqual(p1,p2, (s1,s2,p1,p2))
+
+        c('0.4', '0.4.0')
+        c('0.4.0.0', '0.4.0')
+        c('0.4.0-0', '0.4-0')
+        c('0pl1', '0.0pl1')
+        c('0pre1', '0.0c1')
+        c('0.0.0preview1', '0c1')
+        c('0.0c1', '0rc1')
 
 
 
@@ -235,47 +317,6 @@ class ParseTests(TestCase):
             for v2 in torture[p+1:]:
                 c(v2,v1)
 
-
-
-
-
-
-
-
-
-
-    def testVersionEquality(self):
-        def c(s1,s2):
-            p1, p2 = parse_version(s1),parse_version(s2)
-            self.assertEqual(p1,p2, (s1,s2,p1,p2))
-
-        c('0.4', '0.4.0')
-        c('0.4.0.0', '0.4.0')
-        c('0.4.0-0', '0.4-0')
-        c('0pl1', '0.0pl1')
-        c('0pre1', '0.0c1')
-        c('0.0.0preview1', '0c1')
-        c('0.0c1', '0rc1')
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-