From b3901440f4e9c1b414c7368c95b1c90f773c647a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Nowak?= Date: Tue, 26 Nov 2019 14:56:06 +0100 Subject: [PATCH] slapos/cache: Implement report command Report allows to check which software releases are available on which distribution in a binary cache. --- setup.py | 1 + slapos/cli/cache.py | 67 ++++++++++++++++++++++++++++++++++++++++ slapos/tests/test_cli.py | 35 +++++++++++++++++++++ 3 files changed, 103 insertions(+) diff --git a/setup.py b/setup.py index 0e05f59a2..5850678c6 100644 --- a/setup.py +++ b/setup.py @@ -87,6 +87,7 @@ setup(name=name, 'slapos.cli': [ # Utilities 'cache lookup = slapos.cli.cache:CacheLookupCommand', + 'cache report = slapos.cli.cache:CacheReportCommand', 'cache source = slapos.cli.cache_source:CacheLookupCommand', # SlapOS Node commands 'node bang = slapos.cli.bang:BangCommand', diff --git a/slapos/cli/cache.py b/slapos/cli/cache.py index 877536324..dd10821d0 100644 --- a/slapos/cli/cache.py +++ b/slapos/cli/cache.py @@ -66,6 +66,26 @@ class CacheLookupCommand(ConfigCommand): sys.exit( do_lookup(self.app.log, cache_dir, args.software_url)) +class CacheReportCommand(ConfigCommand): + """ + perform a report for given list of software relases against + given distribution + """ + + def get_parser(self, prog_name): + ap = super(CacheReportCommand, self).get_parser(prog_name) + ap.add_argument('--distribution', + help='Distribution in a form , like debian 9') + ap.add_argument('software_url', nargs='+', + help='Your software url or MD5 hash') + return ap + + def take_action(self, args): + configp = self.fetch_config(args) + cache_dir = configp.get('networkcache', 'download-binary-dir-url') + sys.exit( + do_report(self.app.log, cache_dir, args.software_url, args.distribution)) + def looks_like_md5(s): """ Return True if the parameter looks like an hashed value. @@ -118,3 +138,50 @@ def do_lookup(logger, cache_dir, software_url): logger.info(line) return 0 + +def do_report(logger, cache_dir, software_url_list, distribution): + if distribution: + linux_distribution = distribution.split() + linux_distribution.append('') + else: + linux_distribution = distribution_tuple() + for software_url in software_url_list: + status = '' + if looks_like_md5(software_url): + md5 = software_url + else: + md5 = hashlib.md5(str2bytes(software_url)).hexdigest() + try: + url = '%s/%s' % (cache_dir, md5) + logger.debug('Connecting to %s', url) + req = requests.get(url, timeout=5) + except (requests.Timeout, requests.ConnectionError): + msg = 'Cannot connect to cache server %s' % (url,) + else: + if not req.ok: + if req.status_code == 404: + status = "not in cache" + else: + status = "error while looking up" + else: + entries = req.json() + + if not entries: + status = "no binary in cache" + + is_compatible = False + if status == '': + ostable = sorted(ast.literal_eval(json.loads(entry[0])['os']) for entry in entries) + + meta = json.loads(entries[0][0]) + for os in ostable: + if networkcache.os_matches(os, linux_distribution): + is_compatible = True + break + if is_compatible: + msg = "%r available for %r is compatible with %r." % (software_url, ' '.join(os[:3]).strip(), ' '.join(linux_distribution).strip(),) + else: + msg = "%r compatible with %r NOT found." % (software_url, ' '.join(linux_distribution).strip(),) + else: + msg = "%r is NOT available because of %s" % (software_url, status) + logger.info(msg) diff --git a/slapos/tests/test_cli.py b/slapos/tests/test_cli.py index 40f5b89f0..5091b65c2 100644 --- a/slapos/tests/test_cli.py +++ b/slapos/tests/test_cli.py @@ -48,6 +48,7 @@ import slapos.cli.computer_token import slapos.cli.supervisorctl from slapos.cli.proxy_show import do_show, StringIO from slapos.cli.cache import do_lookup as cache_do_lookup +from slapos.cli.cache import do_report from slapos.cli.cache_source import do_lookup as cache_source_do_lookup from slapos.client import ClientConfig import slapos.grid.svcbackend @@ -105,6 +106,40 @@ class TestCliCache(CliMixin): 'http://xxx.shacache.org/cccdc51a07e8c575c880f2d70dd4d458') +class TestCliCacheReport(CliMixin): + + test_url = "https://lab.nexedi.com/nexedi/slapos/raw/1.0.102/software/slaprunner/software.cfg" + def test_cached_binary(self): + do_report( + self.logger, + cache_dir="http://dir.shacache.org", + software_url_list=[self.test_url], + distribution='debian 8') + + self.logger.info.assert_any_call( + "'https://lab.nexedi.com/nexedi/slapos/raw/1.0.102/software/slaprunner/software.cfg' available for 'debian 8.10' is compatible with 'debian 8'.") + + def test_uncached_binary(self): + do_report( + self.logger, + cache_dir="http://dir.shacache.org", + software_url_list=["this_is_uncached_url"], + distribution='debian 8') + + self.logger.info.assert_any_call( + "'this_is_uncached_url' is NOT available because of not in cache") + + def test_bad_cache_dir(self): + do_report( + self.logger, + cache_dir="http://xxx.shacache.org", + software_url_list=[self.test_url], + distribution='debian 8') + + self.logger.info.assert_any_call( + 'Cannot connect to cache server http://xxx.shacache.org/cccdc51a07e8c575c880f2d70dd4d458') + + class TestCliCacheSource(CliMixin): test_url = "https://mirrors.edge.kernel.org/pub/software/scm/git/git-2.17.1.tar.xz" -- 2.30.9