Merge branch 'cache-lookup'

CHANGES.txt

@@ -4,7 +4,8 @@ Changes
 0.33.2 (unreleased)
 -------------------
 
- * networkcache: only match major release number in debian [Marco Mariani]
+ * networkcache: only match major release number in Debian,
+                 fixed platform detection for Ubuntu [Marco Mariani]
  * slapproxy: Filter by instance_guid, allow computer partition renames
               and change of software_type and requested_state [Marco Mariani]
  * slapproxy: Stop instance even if buildout/reporting is wrong [Cedric de Saint Martin]

slapos/cache.py

+# -*- coding: utf-8 -*-
+
+import ast
+import argparse
+import ConfigParser
+import hashlib
+import json
+import re
+import sys
+import urllib2
+
+from slapos.grid import networkcache
+from slapos.grid.distribution import patched_linux_distribution
+
+
+def maybe_md5(s):
+    return re.match('[0-9a-f]{32}', s)
+
+def cache_lookup():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("configuration_file", help="SlapOS configuration file")
+    parser.add_argument("software_url", help="Your software url or MD5 hash")
+    args = parser.parse_args()
+
+    configuration_parser = ConfigParser.SafeConfigParser()
+    configuration_parser.read(args.configuration_file)
+
+    configuration_parser.items('networkcache')
+
+    cache_dir = configuration_parser.get('networkcache', 'download-binary-dir-url')
+
+    if maybe_md5(args.software_url):
+        md5 = args.software_url
+    else:
+        md5 = hashlib.md5(args.software_url).hexdigest()
+
+    try:
+        response = urllib2.urlopen('%s/%s' % (cache_dir, md5))
+    except urllib2.HTTPError as e:
+        if e.code == 404:
+            print 'Object not in cache: %s' % args.software_url
+        else:
+            print 'Error during cache lookup: %s (%s)' % (e.code, e.reason)
+        sys.exit(10)
+
+    entries = json.loads(response.read())
+
+    linux_distribution = patched_linux_distribution()
+
+    header_printed = False
+
+    ostable = []
+    for entry in entries:
+        meta = json.loads(entry[0])
+        os = ast.literal_eval(meta['os'])
+        if not header_printed:
+            print 'Software URL: %s' % meta['software_url']
+            print 'MD5:          %s' % md5
+            print '-------------'
+            print 'Available for: '
+            print 'distribution     |   version    |       id       | compatible?'
+            print '-----------------+--------------+----------------+-------------'
+            header_printed = True
+        ostable.append(os)
+    ostable.sort()
+
+    for os in ostable:
+        compatible = 'yes' if networkcache.os_matches(os, linux_distribution) else 'no'
+        print '%-16s | %12s | %s | %s' % (os[0], os[1], os[2].center(14), compatible)
+

slapos/entry.py

@@ -36,6 +36,7 @@ from slapos.client import request as request
 from slapos.client import remove as remove
 from slapos.client import supply as supply
 from slapos.format import main as format
+from slapos.cache import cache_lookup
 from slapos.grid.slapgrid import runComputerPartition as instance
 from slapos.grid.slapgrid import runSoftwareRelease as software
 from slapos.grid.slapgrid import runUsageReport as report
@@ -152,6 +153,8 @@ def dispatch(command, is_node_command):
     raise EntryPointNotImplementedError(command)
   elif command == 'console':
     call(console, config=USER_SLAPOS_CONFIGURATION)
+  elif command == 'cache-lookup':
+    call(cache_lookup, config=GLOBAL_SLAPOS_CONFIGURATION)
   else:
     return False
 
@@ -177,6 +180,7 @@ Client subcommands usage:
   slapos request <instance-name> <software-url> [--configuration arg1=value1 arg2=value2 ... argN=valueN]
   slapos supply <software-url> <node-id>
   slapos console
+  slapos cache-lookup <software-url-or-md5>
 Node subcommands usage:
   slapos node
   slapos node register <node-id>

slapos/grid/distribution.py

+# -*- coding: utf-8 -*-
+
+"""
+Provides helper functions to check if two binary caches are compatible.
+
+os_matches(...):
+    returns True if the arguments reference compatible platforms.
+
+patched_linux_distribution(...):
+    a patched version of platform.linux_distribution()
+
+    this is the same function provided with the python package in Debian and Ubuntu:
+        see http://bugs.python.org/issue9514
+
+    otherwise, Ubuntu will always be reported as an unstable Debian, regardless of the version.
+"""
+
+import platform
+import re
+
+
+def _debianize(os):
+    """
+    keep only the major release number in case of debian, otherwise
+    minor releases would be seen as not compatible to each other.
+    """
+    distname, version, id_ = os
+    if distname == 'debian' and '.' in version:
+        version = version.split('.')[0]
+    return distname, version, id_
+
+
+def os_matches(os1, os2):
+    return _debianize(os1) == _debianize(os2)
+
+
+_distributor_id_file_re = re.compile("(?:DISTRIB_ID\s*=)\s*(.*)", re.I)
+_release_file_re = re.compile("(?:DISTRIB_RELEASE\s*=)\s*(.*)", re.I)
+_codename_file_re = re.compile("(?:DISTRIB_CODENAME\s*=)\s*(.*)", re.I)
+
+
+def patched_linux_distribution(distname='', version='', id='',
+                               supported_dists=platform._supported_dists,
+                               full_distribution_name=1):
+    # check for the Debian/Ubuntu /etc/lsb-release file first, needed so
+    # that the distribution doesn't get identified as Debian.
+    try:
+        etclsbrel = open("/etc/lsb-release", "rU")
+        for line in etclsbrel:
+            m = _distributor_id_file_re.search(line)
+            if m:
+                _u_distname = m.group(1).strip()
+            m = _release_file_re.search(line)
+            if m:
+                _u_version = m.group(1).strip()
+            m = _codename_file_re.search(line)
+            if m:
+                _u_id = m.group(1).strip()
+        if _u_distname and _u_version:
+            return (_u_distname, _u_version, _u_id)
+    except (EnvironmentError, UnboundLocalError):
+            pass
+
+    return platform.linux_distribution(distname, version, id, supported_dists, full_distribution_name)
+

slapos/grid/networkcache.py

@@ -18,6 +18,8 @@ import platform
 import shutil
 import traceback
 
+from slapos.grid.distribution import os_matches, patched_linux_distribution
+
 try:
     try:
         from slapos.libnetworkcache import NetworkcacheClient, UploadError, \
@@ -32,6 +34,8 @@ except:
     LIBNETWORKCACHE_ENABLED = False
     print 'Networkcache forced to be disabled.'
 
+
+
 def fallback_call(function):
     """Decorator which disallow to have any problem while calling method"""
     def wrapper(self, *args, **kwd):
@@ -48,18 +52,6 @@ def fallback_call(function):
     return wrapper
 
 
-def debianize(os):
-    # keep only the major release number in case of debian
-    distname, version, id_ = os
-    if distname == 'debian' and '.' in version:
-        version = version.split('.')[0]
-    return distname, version, id_
-
-
-def os_matches(os1, os2):
-    return debianize(os1) == debianize(os2)
-
-
 @fallback_call
 def download_network_cached(cache_url, dir_url, software_url, software_root,
                             key, path, logger, signature_certificate_list,
@@ -99,7 +91,7 @@ def download_network_cached(cache_url, dir_url, software_url, software_root,
                 if tags.get('machine') != platform.machine():
                     continue
                 if not os_matches(ast.literal_eval(tags.get('os')),
-                                  platform.linux_distribution()):
+                                  patched_linux_distribution()):
                     continue
                 if tags.get('software_url') != software_url:
                     continue
@@ -145,7 +137,7 @@ def upload_network_cached(software_root, software_url, cached_key,
       software_url=software_url,
       software_root=software_root,
       machine=platform.machine(),
-      os=str(platform.linux_distribution())
+      os=str(patched_linux_distribution())
     )
 
     f = open(path, 'r')