From 2731aa6e25e4f0b9fd19518523379a0b2fc4ba37 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9rome=20Perrin?= Date: Wed, 6 Jun 2018 05:14:39 +0200 Subject: [PATCH 1/4] Introduce an InstallMonitor To monitor which files are created by buildout when installing parts This version is based on git because it was easy to implement, but with git tools, it's easy to inspect what exactly installing a part had changed and also to revert to a previous state by using git commands with the same `--git-dir` and `--work-dir`. --- src/zc/buildout/buildout.py | 13 +++ src/zc/buildout/install_monitor.py | 156 +++++++++++++++++++++++++++++ 2 files changed, 169 insertions(+) create mode 100644 src/zc/buildout/install_monitor.py diff --git a/src/zc/buildout/buildout.py b/src/zc/buildout/buildout.py index 0f8d4407..fb53f46a 100644 --- a/src/zc/buildout/buildout.py +++ b/src/zc/buildout/buildout.py @@ -51,6 +51,8 @@ import pprint import zc.buildout import zc.buildout.download +from zc.buildout.install_monitor import InstallMonitor + PY3 = sys.version_info[0] == 3 if PY3: text_type = str @@ -257,6 +259,8 @@ class Buildout(DictMixin): data = dict(buildout=_buildout_default_options.copy()) self._buildout_dir = os.getcwd() + self._install_monitor = InstallMonitor(self) + if config_file and not _isurl(config_file): config_file = os.path.abspath(config_file) base = os.path.dirname(config_file) @@ -634,6 +638,7 @@ class Buildout(DictMixin): self.install(()) def install(self, install_args): + self._install_monitor.begin_install(install_args) try: self._install_parts(install_args) finally: @@ -644,6 +649,7 @@ class Buildout(DictMixin): del self.installed_part_options if self.show_picked_versions or self.update_versions_file: self._print_picked_versions() + self._install_monitor.end_install() self._unload_extensions() def _install_parts(self, install_args): @@ -754,7 +760,10 @@ class Buildout(DictMixin): elif not uninstall_missing: continue + self._install_monitor.begin_uninstall_part(part, installed_part_options) self._uninstall_part(part, installed_part_options) + self._install_monitor.end_uninstall_part(part, installed_part_options) + installed_parts = [p for p in installed_parts if p != part] installed_part_options['buildout']['parts'] = ( ' '.join(installed_parts)) @@ -785,7 +794,9 @@ class Buildout(DictMixin): part) try: + self._install_monitor.begin_install_part(part, update=True) updated_files = self[part]._call(update) + self._install_monitor.end_install_part(part, updated_files, update=True) except Exception: installed_parts.remove(part) self._uninstall(installed_files) @@ -804,7 +815,9 @@ class Buildout(DictMixin): self._logger.info(*__doing__) if self.dry_run: continue + self._install_monitor.begin_install_part(part) installed_files = self[part]._call(recipe.install) + self._install_monitor.end_install_part(part, installed_files) if installed_files is None: self._logger.warning( "The %s install returned None. A path or " diff --git a/src/zc/buildout/install_monitor.py b/src/zc/buildout/install_monitor.py new file mode 100644 index 00000000..66927d8f --- /dev/null +++ b/src/zc/buildout/install_monitor.py @@ -0,0 +1,156 @@ +import os +import logging +import subprocess +import time +import textwrap +import pprint + + +""" +Monitor what files are installed by buildout + +# Notes + +## Packaging + +Instead of direct patches to buildout, this could be a buildout extension + + * parts installation: ? monkey patchs ? + * `end_install` -> `zc.buildout.unloadextension` + +## random notes about monitor + +I had in mind we could have a ptrace monitor that knows were part is +supposed to install and abort the process before it writes in a place +where it is not supposed to write. + +Hard part is to defined what's an "allowed directory". + +For "custom" recipes, a section can install in: + + * `${buildout:parts-directory}/${:_buildout_section_name}` + * `${buildout:bin-directory}` ( but is this really needed ? if it is, it must + be with restrictions, to prevent a part from overriding scripts from another part) + * the corresponding cache subdirectory for recipes using cache + +A `LD_PRELOAD` or [PRoot](https://github.com/proot-me/PRoot) based solution seems also +possible, but the dynamic aspects of the cache seems to make this hard. + + +For eggs, installing egg `X` should only be able to write to `eggs/X-{version_spec}`. + +Setuptools have some [sandboxing](https://github.com/pypa/setuptools/blob/c2262d9fe4eaac507ff128ae60b6682e8d132e4d/setuptools/sandbox.py) support (that I have not studied at this point), but it only monitor the +python part of the installation process. + +Develop eggs seems to be similar - only write to `develop-eggs/X-{version_spec}` but this also +probably need to include some hash in version spec. + +( because `eggs` and `develop-eggs` can be shared too ) + +Related bugs that could be solved by such a "don't write here" approach: + https://nexedi.erp5.net/bug_module/20150413-15EC498 + https://nexedi.erp5.net/bug_module/20110718-8E43A9 + +""" + +class GitInstallMonitor(object): + """Stupid monitor making a (big) git commit to hold everything installed by each part. + + Assumes git command is in path + """ + def __init__(self, buildout): + self._buildout = buildout + self._logger = logging.getLogger('zc.buildout.installmonitor') + self._start_install_time = {} + + def _record_changes(self, message): + self._logger.debug("Adding %s to git, this may take some time", self._work_tree) + if self._git("status", "--porcelain"): + self._git("add", self._work_tree) + self._git("commit", "-m", message) + self._logger.debug("Added %s to git", self._work_tree) + else: + self._logger.info("No changes to record") + + def _git(self, *args): + return subprocess.check_output( + ( + 'git', + '--git-dir', self._git_dir, + '--work-tree', self._work_tree, + ) + args, + env={ + 'GIT_COMMITTER_NAME': 'SlapOS Install Monitor', + 'GIT_COMMITTER_EMAIL': 'noemail@example.com', + 'GIT_AUTHOR_NAME': 'SlapOS Install Monitor', + 'GIT_AUTHOR_EMAIL': 'noemail@example.com', + }) + + def begin_install(self, install_args): + """Called at the beginning of buildout installation. + """ + self._start_install_time[self] = time.time() + self._work_tree = os.path.dirname( + os.path.commonprefix(( + self._buildout['buildout']['parts-directory'], + self._buildout['buildout']['directory'], + self._buildout['buildout']['slapos-recipe-cmmi-shared-path'], + ))) + self._git_dir = os.path.join( + self._buildout['buildout']['directory'], + 'slapos.gitinstallmonitor.git' + ) + + self._git("init") + self._logger.info( + "Initialised git dir in %s with work tree %s", + self._git_dir, + self._work_tree + ) + + # ignore git dir ( as a path relative to work tree ) + with open(os.path.join(self._git_dir, 'info', 'exclude'), 'w') as gitignore: + gitignore.write(self._git_dir[len(self._work_tree):]) + + self._record_changes("Begin install with args {install_args}".format(**locals())) + + def end_install(self): + took = time.time() - self._start_install_time[self] + self._record_changes("Finished installation in {took:.2f} seconds".format(**locals())) + + def begin_install_part(self, name, update=False): + self._start_install_time[name] = time.time() + + def end_install_part(self, name, installed_files, update=False): + took = time.time() - self._start_install_time[name] + recipe = self._buildout[name]['recipe'] + installed_files = '\n '.join(installed_files or []) + options = '\n '.join(pprint.pformat(dict(self._buildout[name])).splitlines()) + did = "Updated" if update else "Installed" + + self._record_changes( + textwrap.dedent("""\ + {did} part {name} in {took:.2f} seconds + + Recipe: {recipe} + Options: + {options} + Installed files: + {installed_files} + """).format(**locals())) + + def begin_uninstall_part(self, name, installed_part_options): + self._start_install_time[name] = time.time() + + def end_uninstall_part(self, name, installed_part_options): + took = time.time() - self._start_install_time[name] + self._record_changes( + textwrap.dedent("""\ + Uninstalled part {name} in {took:.2f} seconds + + Installed part options: + {installed_part_options} + """).format(**locals())) + + +InstallMonitor = GitInstallMonitor -- 2.30.9 From 6f60a5ebf51800bac993deabb0500c507b8ffd33 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9rome=20Perrin?= Date: Fri, 8 Jun 2018 08:07:58 +0200 Subject: [PATCH 2/4] fixup! Introduce an InstallMonitor Monitor together all buildout using the cache Store the git database in a place that will be ignored by other buildouts --- src/zc/buildout/install_monitor.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/src/zc/buildout/install_monitor.py b/src/zc/buildout/install_monitor.py index 66927d8f..0fe9ffff 100644 --- a/src/zc/buildout/install_monitor.py +++ b/src/zc/buildout/install_monitor.py @@ -60,15 +60,16 @@ class GitInstallMonitor(object): """ def __init__(self, buildout): self._buildout = buildout - self._logger = logging.getLogger('zc.buildout.installmonitor') + self._logger = logging.getLogger('slapos.buildout.installmonitor') self._start_install_time = {} def _record_changes(self, message): + start = time.time() self._logger.debug("Adding %s to git, this may take some time", self._work_tree) if self._git("status", "--porcelain"): self._git("add", self._work_tree) self._git("commit", "-m", message) - self._logger.debug("Added %s to git", self._work_tree) + self._logger.debug("Added %s to git in %.2f seconds", self._work_tree, time.time() - start) else: self._logger.info("No changes to record") @@ -97,8 +98,8 @@ class GitInstallMonitor(object): self._buildout['buildout']['slapos-recipe-cmmi-shared-path'], ))) self._git_dir = os.path.join( - self._buildout['buildout']['directory'], - 'slapos.gitinstallmonitor.git' + self._work_tree, + 'slapos.buildout.installmonitor.git' ) self._git("init") @@ -110,9 +111,13 @@ class GitInstallMonitor(object): # ignore git dir ( as a path relative to work tree ) with open(os.path.join(self._git_dir, 'info', 'exclude'), 'w') as gitignore: - gitignore.write(self._git_dir[len(self._work_tree):]) + gitignore.write('slapos.buildout.installmonitor.git') - self._record_changes("Begin install with args {install_args}".format(**locals())) + software_release = os.path.basename( + self._buildout['buildout']['directory']) + + self._record_changes( + "Begin install {software_release} with args {install_args}".format(**locals())) def end_install(self): took = time.time() - self._start_install_time[self] -- 2.30.9 From 92a1ac671193e8c026f2fdebb74db980258c5932 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9rome=20Perrin?= Date: Fri, 8 Jun 2018 05:45:26 +0200 Subject: [PATCH 3/4] install_monitor: handle submodules describes the options, but in fact ignore them for now --- src/zc/buildout/install_monitor.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/zc/buildout/install_monitor.py b/src/zc/buildout/install_monitor.py index 0fe9ffff..23300a10 100644 --- a/src/zc/buildout/install_monitor.py +++ b/src/zc/buildout/install_monitor.py @@ -66,7 +66,18 @@ class GitInstallMonitor(object): def _record_changes(self, message): start = time.time() self._logger.debug("Adding %s to git, this may take some time", self._work_tree) - if self._git("status", "--porcelain"): + # One problem is that buildout directory contain git repositories, wich `git add` below + # will treat as submodules. + # I see several options: + # * add the content of these git repositories as normal files using plumbing commands (see + # https://www.git-scm.com/book/en/v2/Git-Internals-Git-Objects) + # * manage them as submodules (see https://stackoverflow.com/a/4162672 https://stackoverflow.com/a/4185579 + # because git add does not seem to make a managed submodule ) + # * just ignore them. + # + # for now, they are ignored. + + if self._git("status", "--ignore-submodules", "--porcelain"): self._git("add", self._work_tree) self._git("commit", "-m", message) self._logger.debug("Added %s to git in %.2f seconds", self._work_tree, time.time() - start) -- 2.30.9 From 6979c3103a185b52be2caaf6641c2982b5fda841 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9rome=20Perrin?= Date: Fri, 8 Jun 2018 08:40:51 +0200 Subject: [PATCH 4/4] fixup! Introduce an InstallMonitor installed_part_options contains options for all parts, so it can be very long and cause OSError 7 . Just include options for the part being uninstalled. --- src/zc/buildout/install_monitor.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/zc/buildout/install_monitor.py b/src/zc/buildout/install_monitor.py index 23300a10..de7e4294 100644 --- a/src/zc/buildout/install_monitor.py +++ b/src/zc/buildout/install_monitor.py @@ -160,12 +160,13 @@ class GitInstallMonitor(object): def end_uninstall_part(self, name, installed_part_options): took = time.time() - self._start_install_time[name] + this_installed_part_options = installed_part_options.get(name) self._record_changes( textwrap.dedent("""\ Uninstalled part {name} in {took:.2f} seconds Installed part options: - {installed_part_options} + {this_installed_part_options} """).format(**locals())) -- 2.30.9