cyclonedx: WIP export dependencies

nxdbom/__init__.py

@@ -26,32 +26,92 @@ Usage: nxd-bom software   <path-to-installed-software>
 An example of generated bill of material is provided in example/ors-bom.txt .
 """
 
-from collections import namedtuple
+from collections import defaultdict
+from dataclasses import dataclass
 import datetime
 from glob import glob
 import importlib.metadata
 from os.path import basename
 from urllib.parse import unquote, urlparse
+from typing import Dict, List, NamedTuple, Set, Tuple
 import argparse
 import json
 import sys, configparser, re, codecs
 import uuid
+from functools import cached_property
 
 
-# PkgInfo represents information about a package
-PkgInfo = namedtuple('PkgInfo', [
-    'name',
-    'version',
-    'kind',
-    'url'])
-    # TODO patches
+# The key of a package in the bom
+class PkgKey(NamedTuple):
+    name: str
+    kind: str
+    version: str
+
+
+# PkgInfo represents information about a component
+@dataclass
+class PkgInfo:
+    name: str
+    version: str
+    kind: str
+    url: str
+    dependencies: Set['PkgInfo']
+
+    @cached_property
+    def purl(self):
+        if self.kind == 'egg':
+            purl_type = 'pypi'
+        elif self.kind == 'gem':
+            purl_type = 'gem'
+        else:
+            purl_type = 'generic'
+        return f'pkg:{purl_type}/{self.name}@{self.version}'
+
+    def __hash__(self):
+        return hash(self.purl)
+
+    @cached_property
+    def cpe(self):
+        cpe_vendor = '*'
+        parsed_url = urlparse(self.url)
+        if parsed_url.hostname == 'github.com':
+            cpe_vendor = parsed_url.path.split('/')[1]
+        return f'cpe:2.3:*:{cpe_vendor}:{self.name}:{self.version}:*:*:*:*:*:*:*'
+
+
+class Bom(Dict[PkgKey, PkgInfo]):
+    dependencies: Set['PkgInfo']
+    """Direct dependencies of the software
+    """
+    def __init__(self, *args, **kw):
+        super().__init__(*args, **kw)
+        self.dependencies = set()
 
 
 
 # bom_software retrieves BOM from .installed.cfg generated by buildout along the build.
-def bom_software(installed_software_path): # -> {} (name,kind) -> PkgInfo
-    bom = {}
-    def addbom(urlpath, kind, version=None):
+def bom_software(installed_software_path: str) -> Bom:
+    bom = Bom()
+
+    # bom component key per buildout section name
+    bom_key_per_buildout_sections:Dict[str, List[PkgKey]] = defaultdict(list)
+    # list of buildout sections referenced by each bom component
+    dependencies_by_pkg_key:Dict[PkgKey, List[str]] = defaultdict(list)
+    # buildout sections installing direct dependencies
+    bom_dependencies_sections:List[str] = []
+    pkg_key_by_egg_name:Dict[str, PkgKey] = {}
+    egg_dependencies:Dict[str, List[str]] = defaultdict(list)
+
+    # direct dependencies of the software
+    bom_dependencies : List[str] = []
+
+    def adddeps(part: configparser.SectionProxy, bkey:PkgKey) -> None:
+        for dep in part.get('__buildout_signature__', raw=True, fallback='').split():
+            if ':' in dep:
+                dependencies_by_pkg_key[bkey].append(dep.split(':')[0])
+        bom_key_per_buildout_sections[s].append(bkey)
+
+    def addbom(urlpath, kind, version=None) -> PkgKey:
         name, ver = namever(urlpath)
         if version is not None:
             assert ver is None
@@ -67,6 +127,24 @@ def bom_software(installed_software_path): # -> {} (name,kind) -> PkgInfo
             url = urlpath
         else:
             if kind == 'egg':
+                # Compute egg dependencies
+                if name not in egg_dependencies:
+                    for prefix in '', installed_software_path + '/eggs/', installed_software_path + '/develop-eggs/':
+                        try:
+                            with open(prefix + urlpath + '/EGG-INFO/requires.txt') as f:
+                                requirements = f.readlines()
+                        except (FileNotFoundError, NotADirectoryError):
+                            pass
+                        else:
+                            req_name_re = re.compile(r'^[^><=!\[]+')
+                            for req_line in requirements:
+                                req_line = req_line.strip()
+                                if req_line and not req_line.startswith(('#', '[')):
+                                    req_name_match = req_name_re.match(req_line)
+                                    if req_name_match:
+                                        req_name = req_name_match.group().strip()
+                                        egg_dependencies[name].append(req_name)
+                # Compute URL
                 # XXX not strictly correct -> better retrieve the actual URL, but buildout does not save it in installed.cfg
                 # remove +slapospatcheXXX suffix from egg url
                 v = ver
@@ -86,14 +164,15 @@ def bom_software(installed_software_path): # -> {} (name,kind) -> PkgInfo
             else:
                 raise NotImplementedError('TODO url for kind %r  (urlpath: %r)' % (kind, urlpath))
 
-
-        info = PkgInfo(name, ver, kind, url)
-        bkey = (name, kind, ver)
+        info = PkgInfo(name, ver, kind, url, set())
+        bkey = PkgKey(name, kind, ver)
         if bkey in bom:
             assert bom[bkey] == info,  (bom[bkey], info)
         else:
             bom[bkey] = info
-
+        if kind == 'egg':
+            pkg_key_by_egg_name[name] = bkey
+        return bkey
 
     _ = {
         '__buildout_space__':   ' ',
@@ -121,8 +200,8 @@ def bom_software(installed_software_path): # -> {} (name,kind) -> PkgInfo
             else:
                 url = geturl(part)
             if url:
-                addbom(url, '')    # XXX detect kind?
-
+                bkey = addbom(url, '')    # XXX detect kind?
+                adddeps(part, bkey)
         elif recipe == 'slapos.recipe.build':
             # slapos.recipe.build is often used in creative ways to actually
             # run python code during the build. Let's detect this via lack of
@@ -137,7 +216,8 @@ def bom_software(installed_software_path): # -> {} (name,kind) -> PkgInfo
                     raise NotImplementedError('%s might be using url with %s in an unsupported way' % (s, recipe))
 
             else:
-                addbom(url, '')     # XXX detect kind?
+                bkey = addbom(url, '')     # XXX detect kind?
+                adddeps(part, bkey)
 
         elif recipe in ('slapos.recipe.build:download', 'slapos.recipe.build:download-unpacked', 'hexagonit.recipe.download', ):
             url = geturl(part)
@@ -171,6 +251,11 @@ def bom_software(installed_software_path): # -> {} (name,kind) -> PkgInfo
 
         elif recipe.startswith('slapos.recipe.template') or \
              recipe == 'collective.recipe.template':
+            # dependences of the section installing the instance templates are considered direct dependencies
+            if part.get('__buildout_installed__', raw=True, fallback='').endswith(('/template.cfg', '/instance.cfg')):
+                for dep in part.get('__buildout_signature__', raw=True, fallback='').split():
+                    if ':' in dep:
+                        bom_dependencies.append(dep.split(':')[0])
             url = geturl(part, None)
             if url is None:
                 if recipe.startswith('slapos'):
@@ -200,8 +285,9 @@ def bom_software(installed_software_path): # -> {} (name,kind) -> PkgInfo
             isegg  = eggpath.endswith('.egg')
             islink = eggpath.endswith('.egg-link'),
             assert isegg or islink, eggpath
-            if isegg:                   # ignore .egg-link - we declare it through the place
-                addbom(eggpath, 'egg')  # from where destination is downloaded from
+            if isegg:                          # ignore .egg-link - we declare it through the place
+                bkey = addbom(eggpath, 'egg')  # from where destination is downloaded from
+                adddeps(part, bkey)
 
         elif recipe in ('zc.recipe.egg', 'zc.recipe.egg:eggs', 'zc.recipe.egg:script', 'zc.recipe.egg:scripts'):
             # zc.recipe.egg:* installs
@@ -213,7 +299,7 @@ def bom_software(installed_software_path): # -> {} (name,kind) -> PkgInfo
             # when no scripts are installed at all, but is better than nothing.
             # An alternative is to fix zc.recipe.egg itself to emit information
             # about all eggs it installs:
-            xeggs = {} # xeggname -> xeggname-ver-....egg on the filesystem
+            xeggs:Dict[str, str] = {} # xeggname -> xeggname-ver-....egg on the filesystem
             installedv = part['__buildout_installed__'].split()
             for f in installedv:
                 for xeggpath in eggscript_imports(f):
@@ -257,7 +343,9 @@ def bom_software(installed_software_path): # -> {} (name,kind) -> PkgInfo
                 if len(eggv) > 1:
                     eggv.sort()
                     raise ValueError('egg %s is present multiple times: %s' % (eggname, eggv))
-                addbom(eggv[0], 'egg')
+                bkey = addbom(eggv[0], 'egg')
+                # eggs listed in section are direct dependencies
+                adddeps(part, bkey)
 
         elif recipe == 'slapos.recipe.build:gitclone':
             repo = part['repository']
@@ -280,6 +368,24 @@ def bom_software(installed_software_path): # -> {} (name,kind) -> PkgInfo
         else:
             raise NotImplementedError('TODO: add support for recipe %s' % recipe)
 
+    for k, pkginfo in bom.items():
+        # resolve buildout part dependencies
+        for dependency_buildout_section in dependencies_by_pkg_key[k]:
+            for dep_key in bom_key_per_buildout_sections.get(dependency_buildout_section, []):
+                dep = bom[dep_key]
+                if dep.kind != 'egg' and dep.name not in ('patch', ):
+                    pkginfo.dependencies.add(dep)
+        # resolve eggs dependencies
+        if pkginfo.kind == 'egg':
+            for dep_egg in egg_dependencies[pkginfo.name]:
+                egg_dep_key = pkg_key_by_egg_name.get(dep_egg)
+                if egg_dep_key:
+                    pkginfo.dependencies.add(bom[egg_dep_key])
+
+    # resolve direct dependencies
+    for bom_dependency_buildout_part in bom_dependencies:
+        for dep_key in bom_key_per_buildout_sections.get(bom_dependency_buildout_part, []):
+            bom.dependencies.add(bom[dep_key])
 
     return bom
 
@@ -546,8 +652,8 @@ def fmt_bom(bom): # -> str
 
     return ''.join(outv)
 
-def fmt_bom_cyclonedx_json(bom, software_path):
 
+def fmt_bom_cyclonedx_json(bom:Bom, software_path:str):
     # possible future extensions:
     # - describe patches applied to components (using components[*].pedigree.patches )
     # - for egg components, include metadata (licence, author, description) by reading
@@ -557,8 +663,9 @@ def fmt_bom_cyclonedx_json(bom, software_path):
     software_url = cfgparser.get('buildout', 'extends')
     name = software_url.split('/')[-2]  # slapos convention
 
+    bom_ref = f'urn:uuid:{uuid.uuid4()}'
     bom_json = {
-        "serialNumber": f'urn:uuid:{uuid.uuid4()}',
+        "serialNumber": bom_ref,
         "version": 1,
         "$schema": "http://cyclonedx.org/schema/bom-1.5.schema.json",
         "bomFormat": "CycloneDX",
@@ -567,6 +674,7 @@ def fmt_bom_cyclonedx_json(bom, software_path):
             "timestamp": datetime.datetime.now(datetime.timezone.utc).isoformat(),
             "component": {
                 "name": name,
+                "bom-ref": bom_ref,
                 "type": "application",
                 "externalReferences": [
                     {
@@ -593,6 +701,12 @@ def fmt_bom_cyclonedx_json(bom, software_path):
         }
     }
     components = bom_json["components"] = []
+    dependencies = bom_json["dependencies"] = [
+        {
+            "ref": bom_ref,
+            "dependsOn": [pkgdep.purl for pkgdep in bom.dependencies]
+        }
+    ]
     for _, pkginfo in sorted(bom.items()):
         cpe = None
         externalReferences = []
@@ -607,29 +721,24 @@ def fmt_bom_cyclonedx_json(bom, software_path):
                     ),
                 }
             )
-        purl_type = 'generic'
-        if pkginfo.kind == 'egg':
-            purl_type = 'pypi'
-        elif pkginfo.kind == 'gem':
-            purl_type = 'gem'
-        else:
-            cpe_vendor = '*'
-            parsed_url = urlparse(pkginfo.url)
-            if parsed_url.hostname == 'github.com':
-              cpe_vendor = parsed_url.path.split('/')[1]
-            cpe = f'cpe:2.3:*:{cpe_vendor}:{pkginfo.name}:{pkginfo.version}:*:*:*:*:*:*:*'
-        purl = f'pkg:{purl_type}/{pkginfo.name}@{pkginfo.version}'
         component = {
             'name': pkginfo.name,
-            'purl': purl,
+            'purl': pkginfo.purl,
+            'bom-ref': pkginfo.purl,
             'type': 'library',
             'version': pkginfo.version,
         }
-        if cpe:
-            component['cpe'] = cpe
+        if pkginfo.kind not in ('egg', 'gem'):
+            component['cpe'] = pkginfo.cpe
         if externalReferences:
             component['externalReferences'] = externalReferences
         components.append(component)
+        dependencies.append(
+            {
+                "ref": pkginfo.purl,
+                "dependsOn": [pkgdep.purl for pkgdep in pkginfo.dependencies]
+            }
+        )
 
     return bom_json
 

nxdbom/nxdbom_test.py

@@ -745,7 +745,7 @@ extends = https://slapos.example.invalid/software/example/software.cfg
 @pytest.mark.parametrize('build,bomok', testv)
 def test_bom_software(tmpdir, build, bomok):
     populate_software_directory_from_build(tmpdir, build)
-    bom = {}
+    bom = nxdbom.Bom()
     if isinstance(bomok, Exception):
         with pytest.raises(type(bomok)) as e:
             nxdbom.bom_software(tmpdir)