networkcache.py 5.74 KB
Newer Older
1 2
##############################################################################
#
3
# Copyright (c) 2010, 2011, 2012 ViFiB SARL and Contributors.
4 5 6 7 8 9 10 11 12 13 14 15 16
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################

import shutil
import traceback
Yingjie Xu's avatar
Yingjie Xu committed
17
import json
Yingjie Xu's avatar
Yingjie Xu committed
18
import platform
19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50

try:
    try:
        from slapos.libnetworkcache import NetworkcacheClient, UploadError, \
          DirectoryNotFound
    except ImportError:
        LIBNETWORKCACHE_ENABLED = False
    else:
        LIBNETWORKCACHE_ENABLED = True
except:
    print 'There was problem while trying to import slapos.libnetworkcache:'\
        '\n%s' % traceback.format_exc()
    LIBNETWORKCACHE_ENABLED = False
    print 'Networkcache forced to be disabled.'

def fallback_call(function):
    """Decorator which disallow to have any problem while calling method"""
    def wrapper(self, *args, **kwd):
        """
        Log the call, and the result of the call
        """
        try:
            return function(self, *args, **kwd)
        except: # indeed, *any* exception is swallowed
            print 'There was problem while calling method %r:\n%s' % (
                function.__name__, traceback.format_exc())
            return False
    wrapper.__doc__ = function.__doc__
    return wrapper


@fallback_call
Yingjie Xu's avatar
Yingjie Xu committed
51
def download_network_cached(cache_url, dir_url, software_url, software_root,
52 53
                            key, path, logger, signature_certificate_list,
                            binary_cache_url_blacklist=None):
54 55 56 57 58 59 60
    """Downloads from a network cache provider

    return True if download succeeded.
    """
    if not LIBNETWORKCACHE_ENABLED:
        return False

61
    if not(cache_url and dir_url and software_url and software_root):
62 63
        return False

64 65 66 67
    for url in binary_cache_url_blacklist:
      if software_url.startswith(url):
        return False

Yingjie Xu's avatar
Yingjie Xu committed
68
    # In order to call nc nicely.
69 70 71 72 73 74 75 76 77
    if len(signature_certificate_list) == 0:
        signature_certificate_list = None
    try:
        nc = NetworkcacheClient(cache_url, dir_url,
            signature_certificate_list=signature_certificate_list)
    except TypeError:
      logger.warning('Incompatible version of networkcache, not using it.')
      return False

Yingjie Xu's avatar
Yingjie Xu committed
78
    logger.info('Downloading %s binary from network cache.' % software_url)
79
    try:
Yingjie Xu's avatar
Yingjie Xu committed
80
        file_descriptor = None
Yingjie Xu's avatar
Yingjie Xu committed
81 82 83 84 85
        json_entry_list = nc.select_generic(key)
        for entry in json_entry_list:
            json_information, _ = entry
            try:
                tags = json.loads(json_information)
Yingjie Xu's avatar
Yingjie Xu committed
86 87 88 89 90 91 92 93
                if tags.get('machine') != platform.machine():
                    continue
                if tags.get('os') != str(platform.linux_distribution()):
                    continue
                if tags.get('software_url') != software_url:
                    continue
                if tags.get('software_root') != software_root:
                    continue
Yingjie Xu's avatar
Yingjie Xu committed
94 95 96 97 98
                sha512 = tags.get('sha512')
                file_descriptor = nc.download(sha512)
                break
            except Exception:
                continue
Yingjie Xu's avatar
Yingjie Xu committed
99 100 101 102 103 104 105 106
        if file_descriptor is not None:
            f = open(path, 'w+b')
            try:
                shutil.copyfileobj(file_descriptor, f)
            finally:
                f.close()
                file_descriptor.close()
            return True
107 108
    except (IOError, DirectoryNotFound), e:
        logger.info('Failed to download from network cache %s: %s' % \
Yingjie Xu's avatar
Yingjie Xu committed
109
                                                       (software_url, str(e)))
Yingjie Xu's avatar
Yingjie Xu committed
110
    return False
111 112 113


@fallback_call
Yingjie Xu's avatar
Yingjie Xu committed
114 115
def upload_network_cached(software_root, software_url, cached_key,
    cache_url, dir_url, path, logger, signature_private_key_file,
Yingjie Xu's avatar
Yingjie Xu committed
116
    shacache_cert_file, shacache_key_file, shadir_cert_file, shadir_key_file):
117 118 119 120
    """Upload file to a network cache server"""
    if not LIBNETWORKCACHE_ENABLED:
        return False

Yingjie Xu's avatar
Yingjie Xu committed
121 122
    if not (software_root and software_url and cached_key \
                          and cache_url and dir_url):
123 124
        return False

Yingjie Xu's avatar
Yingjie Xu committed
125
    logger.info('Uploading %s binary into network cache.' % software_url)
126

Yingjie Xu's avatar
Yingjie Xu committed
127
    # YXU: "file" and "urlmd5" should be removed when server side is ready
Yingjie Xu's avatar
Yingjie Xu committed
128
    kw = dict(
Yingjie Xu's avatar
Yingjie Xu committed
129 130
      file="file",
      urlmd5="urlmd5",
Yingjie Xu's avatar
Yingjie Xu committed
131 132
      software_url=software_url,
      software_root=software_root,
Yingjie Xu's avatar
Yingjie Xu committed
133
      machine=platform.machine(),
Yingjie Xu's avatar
Yingjie Xu committed
134
      os=str(platform.linux_distribution())
Yingjie Xu's avatar
Yingjie Xu committed
135
    )
136 137 138 139 140 141 142 143 144 145 146 147 148 149

    f = open(path, 'r')
    # convert '' into None in order to call nc nicely
    if not signature_private_key_file:
        signature_private_key_file = None
    if not shacache_cert_file:
        shacache_cert_file = None
    if not shacache_key_file:
        shacache_key_file = None
    if not shadir_cert_file:
        shadir_cert_file = None
    if not shadir_key_file:
        shadir_key_file = None
    try:
Yingjie Xu's avatar
Yingjie Xu committed
150
        nc = NetworkcacheClient(cache_url, dir_url,
151 152 153 154 155 156 157 158 159 160
            signature_private_key_file=signature_private_key_file,
            shacache_cert_file=shacache_cert_file,
            shacache_key_file=shacache_key_file,
            shadir_cert_file=shadir_cert_file,
            shadir_key_file=shadir_key_file)
    except TypeError:
        logger.warning('Incompatible version of networkcache, not using it.')
        return False

    try:
Yingjie Xu's avatar
Yingjie Xu committed
161
        return nc.upload_generic(f, cached_key, **kw)
162
    except (IOError, UploadError), e:
Yingjie Xu's avatar
Yingjie Xu committed
163
        logger.info('Fail to upload file. %s' % (str(e)))
164 165 166 167 168
        return False
    finally:
      f.close()

    return True