Extend tests to detect shared libraries using system libraries

On some test machines, we saw test failures because openssl version mismatch:

```
Traceback (most recent call last):
  File "/srv/slapgrid/slappart14/srv/testnode/cqg/soft/18d28ecd49e5e0f732e2ce0c00c57519/parts/slapos.core-repository/slapos/testing/testcase.py", line 227, in installSoftwareUrlList
    checkSoftware(cls.slap, software_url)
  File "/srv/slapgrid/slappart14/srv/testnode/cqg/soft/18d28ecd49e5e0f732e2ce0c00c57519/parts/slapos.core-repository/slapos/testing/testcase.py", line 211, in checkSoftware
    raise RuntimeError('\n'.join(error_list))
RuntimeError: ./parts/proftpd/libexec/mod_auth_web.so:
./parts/proftpd/libexec/mod_auth_web.so: /usr/lib/x86_64-linux-gnu/libssl.so.1.1: version `OPENSSL_1_1_1' not found (required by /srv/slapgrid/slappart14/srv/testnode/cqg/inst/test0-0/tmp/shared/curl/724c785f86fff6993ff24ef745d4dbb9/lib/libcurl.so.4)
```

What we see in this traceback is that the version of openssl used was the one from system ( in `/usr/lib/` ), which should not happen because we are using setting rpath to slapos libraries. We had a test checking for missing rpath that could not resolved, but our test did not catch missing rpath that can be resolved on a system library, which is also wrong because this system library might be a different version like here or even be missing.

/reviewed-on nexedi/slapos.core!172

slapos/testing/testcase.py

@@ -29,6 +29,7 @@
 import unittest
 import os
 import fnmatch
+import re
 import glob
 import logging
 import shutil
@@ -37,7 +38,7 @@ from six.moves.urllib.parse import urlparse
 try:
   import subprocess32 as subprocess
 except ImportError:
-  import subprocess
+  import subprocess  # type: ignore
   subprocess  # pyflakes
 
 from .utils import getPortFromPath
@@ -49,7 +50,7 @@ from ..grid.utils import md5digest
 from ..util import mkdir_p
 
 try:
-  from typing import Iterable, Tuple, Callable, Type
+  from typing import Iterable, Tuple, Callable, Type, Dict, List, Optional
 except ImportError:
   pass
 
@@ -63,9 +64,9 @@ def makeModuleSetUpAndTestCaseClass(
     verbose=bool(int(os.environ.get('SLAPOS_TEST_VERBOSE', 0))),
     shared_part_list=os.environ.get('SLAPOS_TEST_SHARED_PART_LIST',
                                     '').split(os.pathsep),
-    snapshot_directory=os.environ.get('SLAPOS_TEST_LOG_DIRECTORY')
+    snapshot_directory=os.environ.get('SLAPOS_TEST_LOG_DIRECTORY'),
 ):
-  # type: (str, str, str, str, bool, bool, List[str]) -> Tuple[Callable[[], None], Type[SlapOSInstanceTestCase]]
+  # type: (str, str, str, str, bool, bool, Iterable[str], Optional[str]) -> Tuple[Callable[[], None], Type[SlapOSInstanceTestCase]]
   """Create a setup module function and a testcase for testing `software_url`.
 
   This function returns a tuple of two arguments:
@@ -124,7 +125,6 @@ def makeModuleSetUpAndTestCaseClass(
   if not snapshot_directory:
     snapshot_directory = os.path.join(base_directory, "snapshots")
 
-
   cls = type(
       'SlapOSInstanceTestCase for {}'.format(software_url),
       (SlapOSInstanceTestCase,), {
@@ -139,7 +139,9 @@ def makeModuleSetUpAndTestCaseClass(
           '_test_file_snapshot_directory': snapshot_directory
       })
 
-  class SlapOSInstanceTestCase_(cls, SlapOSInstanceTestCase):
+  class SlapOSInstanceTestCase_(
+      cls,  # type: ignore # https://github.com/python/mypy/issues/2813
+      SlapOSInstanceTestCase):
     # useless intermediate class so that editors provide completion anyway.
     pass
 
@@ -161,53 +163,148 @@ def checkSoftware(slap, software_url):
   This perform a few basic static checks for common problems
   with software installations.
   """
-  software_hash = md5digest(software_url)
 
-  error_list = []
   # Check that all components set rpath correctly and we don't have miss linking any libraries.
-  for path in (os.path.join(slap.software_directory,
-                            software_hash), slap.shared_directory):
-    if not glob.glob(os.path.join(path, '*')):
-      # shared might be empty (when using a slapos command that does not support shared yet).
-      continue
-    out = ''
+  # Also check that they are not linked against system libraries, except a white list of core
+  # system libraries.
+  system_lib_white_list = set((
+      'libc',
+      'libcrypt',
+      'libdl',
+      'libgcc_s',
+      'libgomp',
+      'libm',
+      'libnsl',
+      'libpthread',
+      'libresolv',
+      'librt',
+      'libstdc++',
+      'libutil',
+  ))
+
+  # we also ignore a few patterns for part that are known to be binary distributions,
+  # for which we generate LD_LIBRARY_PATH wrappers or we don't use directly.
+  ignored_file_patterns = set((
+      '*/parts/java-re*/*',
+      '*/parts/firefox*/*',
+      '*/parts/chromium-*/*',
+      '*/parts/chromedriver*/*',
+      # nss is not a binary distribution, but for some reason it has invalid rpath, but it does
+      # not seem to be a problem in our use cases.
+      '*/parts/nss/*',
+      '*/node_modules/phantomjs*/*',
+      '*/grafana/tools/phantomjs/*',
+  ))
+
+  software_hash = md5digest(software_url)
+  error_list = []
+
+  ldd_so_resolved_re = re.compile(
+      r'\t(?P<library_name>.*) => (?P<library_path>.*) \(0x')
+  ldd_already_loaded_re = re.compile(r'\t(?P<library_name>.*) \(0x')
+  ldd_not_found_re = re.compile(r'.*not found.*')
+
+  class DynamicLibraryNotFound(Exception):
+    """Exception raised when ldd cannot resolve a library.
+    """
+  def getLddOutput(path):
+    # type: (str) -> Dict[str, str]
+    """Parse ldd output on executable as `path` and returns a mapping
+    of library paths or None when library is not found, keyed by library so name.
+
+    Raises a `DynamicLibraryNotFound` if any dynamic library is not found.
+
+    Special entries, like VDSO ( linux-vdso.so.1 ) or ELF interpreter
+    ( /lib64/ld-linux-x86-64.so.2 ) are ignored.
+    """
+    libraries = {}  # type: Dict[str, str]
     try:
-      out = subprocess.check_output(
-          "find . -type f -executable "
-
-          # We ignore parts that are binary distributions.
-          "| egrep -v /parts/java-re.*/ "
-          "| egrep -v /parts/firefox-.*/ "
-          "| egrep -v /parts/chromium-.*/ "
-          "| egrep -v /parts/chromedriver-.*/ "
-          "| egrep -v /parts/renderjs-repository.git/node_modules/.* "
-
-          # nss has no valid rpath. It does not seem to be a problem in our case.
-          "| egrep -v /parts/nss/ "
-          "| xargs ldd "
-          r"| egrep '(^\S|not found)' "
-          "| grep -B1 'not found'",
-          shell=True,
+      ldd_output = subprocess.check_output(
+          ('ldd', path),
           stderr=subprocess.STDOUT,
-          cwd=path,
+          universal_newlines=True,
       )
     except subprocess.CalledProcessError as e:
-      # The "good case" is when grep does not match anything, but in
-      # that case, it exists with exit code 1, so we accept this case.
-      if e.returncode != 1 or e.output:
-        error_list.append(e.output)
-    if out:
-      error_list.append(out)
+      if e.output not in ('\tnot a dynamic executable\n',):
+        raise
+      return libraries
+    if ldd_output == '\tstatically linked\n':
+      return libraries
+
+    not_found = []
+    for line in ldd_output.splitlines():
+      resolved_so_match = ldd_so_resolved_re.match(line)
+      ldd_already_loaded_match = ldd_already_loaded_re.match(line)
+      not_found_match = ldd_not_found_re.match(line)
+      if resolved_so_match:
+        libraries[resolved_so_match.group(
+            'library_name')] = resolved_so_match.group('library_path')
+      elif ldd_already_loaded_match:
+        # VDSO or ELF, ignore . See https://stackoverflow.com/a/35805410/7294664 for more about this
+        pass
+      elif not_found_match:
+        not_found.append(line)
+      else:
+        raise RuntimeError('Unknown ldd line %s for %s.' % (line, path))
+    if not_found:
+      not_found_text = '\n'.join(not_found)
+      raise DynamicLibraryNotFound(
+          '{path} has some not found libraries:\n{not_found_text}'.format(
+              **locals()))
+    return libraries
+
+  def checkExecutableLink(paths_to_check, valid_paths_for_libs):
+    # type: (Iterable[str], Iterable[str]) -> List[str]
+    """Check shared libraries linked with executables in `paths_to_check`.
+    Only libraries from `valid_paths_for_libs` are accepted.
+    Returns a list of error messages.
+    """
+    executable_link_error_list = []
+    for path in paths_to_check:
+      for root, dirs, files in os.walk(path):
+        for f in files:
+          f = os.path.join(root, f)
+          if any(fnmatch.fnmatch(f, ignored_pattern)
+                 for ignored_pattern in ignored_file_patterns):
+            continue
+          if os.access(f, os.X_OK):
+            try:
+              libs = getLddOutput(f)
+            except DynamicLibraryNotFound as e:
+              executable_link_error_list.append(str(e))
+            else:
+              for lib, lib_path in libs.items():
+                if lib.split('.')[0] in system_lib_white_list:
+                  continue
+                # dynamically linked programs can only be linked with libraries
+                # present in software or in shared parts repository.
+                if any(lib_path.startswith(valid_path)
+                       for valid_path in valid_paths_for_libs):
+                  continue
+                executable_link_error_list.append(
+                    '{f} uses system library {lib_path} for {lib}'.format(
+                        **locals()))
+    return executable_link_error_list
+
+  paths_to_check = (
+      os.path.join(slap.software_directory, software_hash),
+      slap.shared_directory,
+  )
+  error_list.extend(
+      checkExecutableLink(
+          paths_to_check,
+          paths_to_check + tuple(slap._shared_part_list),
+      ))
 
   # check this software is not referenced in any shared parts.
   for signature_file in glob.glob(os.path.join(slap.shared_directory, '*', '*',
                                                '.*slapos.*.signature')):
     with open(signature_file) as f:
       signature_content = f.read()
-      if software_hash in signature_content:
-        error_list.append(
-            "Software hash present in signature {}\n{}\n".format(
-                signature_file, signature_content))
+    if software_hash in signature_content:
+      error_list.append(
+          "Software hash present in signature {}\n{}\n".format(
+              signature_file, signature_content))
 
   if error_list:
     raise RuntimeError('\n'.join(error_list))
@@ -403,7 +500,8 @@ class SlapOSInstanceTestCase(unittest.TestCase):
     """
     cls._cleanup("{}.{}.tearDownClass".format(cls.__module__, cls.__name__))
     if not cls._debug:
-      cls.logger.debug("cleaning up slapos log files in %s", cls.slap._log_directory)
+      cls.logger.debug(
+          "cleaning up slapos log files in %s", cls.slap._log_directory)
       for log_file in glob.glob(os.path.join(cls.slap._log_directory, '*')):
         os.unlink(log_file)
 
@@ -524,11 +622,12 @@ class SlapOSInstanceTestCase(unittest.TestCase):
           "{}._cleanup leaked_partitions".format(snapshot_name))
       for cp in leaked_partitions:
         try:
+          # XXX is this really the reference ?
+          partition_reference = cp.getInstanceParameterDict()['instance_title']
           cls.slap.request(
               software_release=cp.getSoftwareRelease().getURI(),
               # software_type=cp.getType(), # TODO
-              # XXX is this really the reference ?
-              partition_reference=cp.getInstanceParameterDict()['instance_title'],
+              partition_reference=partition_reference,
               state="destroyed")
         except:
           cls.logger.exception(
@@ -549,10 +648,14 @@ class SlapOSInstanceTestCase(unittest.TestCase):
       cls.logger.exception("Error during stop")
       cls._storeSystemSnapshot("{}._cleanup stop".format(snapshot_name))
     leaked_supervisor_configs = glob.glob(
-        os.path.join(cls.slap.instance_directory, 'etc', 'supervisord.conf.d', '*.conf'))
+        os.path.join(
+            cls.slap.instance_directory, 'etc', 'supervisord.conf.d', '*.conf'))
     if leaked_supervisor_configs:
-      [os.unlink(config) for config in leaked_supervisor_configs]
-      raise AssertionError("Test leaked supervisor configurations: %s" % leaked_supervisor_configs)
+      for config in leaked_supervisor_configs:
+        os.unlink(config)
+      raise AssertionError(
+          "Test leaked supervisor configurations: %s" %
+          leaked_supervisor_configs)
 
   @classmethod
   def requestDefaultInstance(cls, state='started'):