check_software: check egg vulnerabilities for every used python versions

pkg_resources.Environment only scans packages for the current python
version, but the checked software might be using a different python
version than the python version used to check software (the slapos
python). The checked software might also include packages for
multiples python versions.

Compute the list used python packages from the eggs directory and use
an Environment for each python version, this way we check all python
versions and also when slapos python and software python are different.

slapos/testing/check_software.py

@@ -253,22 +253,37 @@ def checkSoftware(slap, software_url):
     # type: (List[str], Dict) -> Iterable[str]
     """Check eggs against known vulnerabilities database from https://github.com/pyupio/safety-db
     """
-    env = pkg_resources.Environment(egg_directories)
-    for egg in env:
-      known_vulnerabilities = safety_db.get(egg)
-      if known_vulnerabilities:
-        for distribution in env[egg]:
-          for known_vulnerability in known_vulnerabilities:
-            for vulnerable_spec in known_vulnerability['specs']:
-              for req in pkg_resources.parse_requirements(egg +
-                                                          vulnerable_spec):
-                vulnerability_description = "\n".join(
-                    u"{}: {}".format(*item)
-                    for item in known_vulnerability.items())
-                if distribution in req:
-                  yield (
-                      u"{egg} use vulnerable version {distribution.version} because {vulnerable_spec}.\n"
-                      "{vulnerability_description}\n".format(**locals()))
+    def get_python_versions():
+      # () -> Iterable[str]
+      """Returns all python versions used in egg_directories 
+      """
+      python_versions = set()
+      for egg_dir in egg_directories:
+        basename, _ = os.path.splitext(os.path.basename(egg_dir))
+        match = pkg_resources.EGG_NAME(basename)
+        if match:
+          pyver = match.group('pyver')
+          if pyver:
+            python_versions.add(pyver)
+      return python_versions
+
+    for python in get_python_versions():
+      env = pkg_resources.Environment(egg_directories, python=python)
+      for egg in env:
+        known_vulnerabilities = safety_db.get(egg)
+        if known_vulnerabilities:
+          for distribution in env[egg]:
+            for known_vulnerability in known_vulnerabilities:
+              for vulnerable_spec in known_vulnerability['specs']:
+                for req in pkg_resources.parse_requirements(egg +
+                                                            vulnerable_spec):
+                  vulnerability_description = "\n".join(
+                      u"{}: {}".format(*item)
+                      for item in known_vulnerability.items())
+                  if distribution in req:
+                    yield (
+                        u"{egg} use vulnerable version {distribution.version} because {vulnerable_spec}.\n"
+                        "{vulnerability_description}\n".format(**locals()))
 
   warning_list.extend(
       checkEggsVersionsKnownVulnerabilities(

slapos/tests/test_check_software.py

@@ -32,13 +32,14 @@ import os
 import glob
 import tempfile
 import textwrap
+import warnings
 
 from slapos.testing.check_software import checkSoftware
-
+from slapos.grid.utils import md5digest
 from .test_standalone import SlapOSStandaloneTestCase
 
 
-class TestCheckSoftware(SlapOSStandaloneTestCase):
+class TestCheckSoftwareLDD(SlapOSStandaloneTestCase):
   # BBB python2
   assertRaisesRegex = getattr(
       unittest.TestCase,
@@ -138,3 +139,77 @@ class TestCheckSoftware(SlapOSStandaloneTestCase):
     # parts
     self.test_shared_part_using_system_libraries()
     self.test_ok()
+
+
+class TestCheckSoftwareEggVulnerability(SlapOSStandaloneTestCase):
+
+  def _install_software(self):
+    """install a fake software with vulnerable packages, installed as
+    eggs and develop-eggs, and for different python versions.
+    """
+    software_url = '/fake/path/software.cfg'
+    software_hash = md5digest(software_url)
+    self.standalone.supply(software_url)
+
+    software_dir = os.path.join(
+        self.standalone.software_directory,
+        software_hash,
+    )
+    eggs_dir = os.path.join(software_dir, 'eggs')
+
+    os.makedirs(os.path.join(eggs_dir, 'Werkzeug-1.0.1-py3.7.egg', 'EGG-INFO'))
+    with open(
+        os.path.join(
+            eggs_dir,
+            'Werkzeug-1.0.1-py3.7.egg',
+            'EGG-INFO',
+            'PKG-INFO',
+        ), 'w') as f:
+      f.write(
+          textwrap.dedent('''\
+              Metadata-Version: 2.1
+              Name: Werkzeug
+              Version: 1.0.1
+              '''))
+
+    develop_eggs_dir = os.path.join(software_dir, 'develop-eggs')
+    os.makedirs(
+        os.path.join(
+            develop_eggs_dir,
+            'lxml-4.6.3-py2.7-linux-x86_64.egg',
+            'EGG-INFO',
+        ))
+    with open(
+        os.path.join(
+            develop_eggs_dir,
+            'lxml-4.6.3-py2.7-linux-x86_64.egg',
+            'EGG-INFO',
+            'PKG-INFO',
+        ), 'w') as f:
+      f.write(
+          textwrap.dedent('''\
+              Metadata-Version: 2.1
+              Name: lxml
+              Version: 4.6.3
+              '''))
+    with open(os.path.join(software_dir, '.completed'), 'w') as f:
+      f.write('Thu Dec  2 01:35:02 2021')
+    with open(os.path.join(software_dir, '.installed.cfg'), 'w') as f:
+      f.write('[buildout]')
+    return software_url
+
+  def test_egg_known_vulnerability(self):
+    software_url = self._install_software()
+    with warnings.catch_warnings(record=True) as warning_context:
+      warnings.simplefilter("always")
+      checkSoftware(self.standalone, software_url)
+
+    warning, = [w for w in warning_context if 'vulnerable' in str(w.message)]
+    self.assertIn(
+        'Werkzeug version 2.0.2 improves the security of the debugger cookies',
+        str(warning.message),
+    )
+    self.assertIn(
+        'Lxml 4.6.5 includes a fix for CVE-2021-43818',
+        str(warning.message),
+    )