Commit 9e88abe9 authored by Yingjie Xu's avatar Yingjie Xu

Merge branch 'binary_cache'

parents 77717fd8 6b18ffec
......@@ -38,6 +38,8 @@ import utils
from svcbackend import getSupervisorRPC
from exception import BuildoutFailedError, WrongPermissionError, \
PathDoesNotExistError
from networkcache import download_network_cached, upload_network_cached
import tarfile
REQUIRED_COMPUTER_PARTITION_PERMISSION = '0750'
......@@ -45,27 +47,75 @@ REQUIRED_COMPUTER_PARTITION_PERMISSION = '0750'
class Software(object):
"""This class is responsible of installing a software release"""
def __init__(self, url, software_root, console, buildout,
signature_private_key_file=None, upload_cache_url=None,
upload_dir_url=None, shacache_cert_file=None, shacache_key_file=None,
shadir_cert_file=None, shadir_key_file=None):
signature_private_key_file=None, signature_certificate_list=None,
upload_cache_url=None, upload_dir_url=None, shacache_cert_file=None,
shacache_key_file=None, shadir_cert_file=None, shadir_key_file=None,
download_binary_cache_url=None, upload_binary_cache_url=None,
download_binary_dir_url=None, upload_binary_dir_url=None):
"""Initialisation of class parameters
"""
self.url = url
self.software_root = software_root
self.software_url_hash = utils.getSoftwareUrlHash(self.url)
self.software_path = os.path.join(self.software_root,
utils.getSoftwareUrlHash(self.url))
self.software_url_hash)
self.buildout = buildout
self.logger = logging.getLogger('BuildoutManager')
self.console = console
self.signature_private_key_file = signature_private_key_file
self.signature_certificate_list = signature_certificate_list
self.upload_cache_url = upload_cache_url
self.upload_dir_url = upload_dir_url
self.shacache_cert_file = shacache_cert_file
self.shacache_key_file = shacache_key_file
self.shadir_cert_file = shadir_cert_file
self.shadir_key_file = shadir_key_file
self.download_binary_cache_url = download_binary_cache_url
self.upload_binary_cache_url = upload_binary_cache_url
self.download_binary_dir_url = download_binary_dir_url
self.upload_binary_dir_url = upload_binary_dir_url
def install(self):
""" Fetches binary cache if possible.
Installs from buildout otherwise.
"""
tarname = self.software_url_hash
cache_dir = tempfile.mkdtemp()
tarpath = os.path.join(cache_dir, tarname)
if (not os.path.exists(self.software_path)) \
and download_network_cached(
self.download_binary_cache_url,
self.download_binary_dir_url,
self.url, self.software_root,
self.software_url_hash,
tarpath, self.logger,
self.signature_certificate_list):
tar = tarfile.open(tarpath)
try:
tar.extractall(path=self.software_root)
finally:
tar.close()
else:
self._install_from_buildout()
tar = tarfile.open(tarpath, "w:gz")
try:
tar.add(self.software_path, arcname=self.software_url_hash)
finally:
tar.close()
upload_network_cached(
self.software_root,
self.url, self.software_url_hash,
self.upload_binary_cache_url,
self.upload_binary_dir_url,
tarpath, self.logger,
self.signature_private_key_file,
self.shacache_cert_file,
self.shacache_key_file,
self.shadir_cert_file,
self.shadir_key_file)
shutil.rmtree(cache_dir)
def _install_from_buildout(self):
""" Fetches buildout configuration from the server, run buildout with
it. If it fails, we notify the server.
"""
......
##############################################################################
#
# Copyright (c) 2010 ViFiB SARL and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
import hashlib
import os
import posixpath
import re
import shutil
import urlparse
import traceback
import utils
import json
import platform
try:
try:
from slapos.libnetworkcache import NetworkcacheClient, UploadError, \
DirectoryNotFound
except ImportError:
LIBNETWORKCACHE_ENABLED = False
else:
LIBNETWORKCACHE_ENABLED = True
except:
print 'There was problem while trying to import slapos.libnetworkcache:'\
'\n%s' % traceback.format_exc()
LIBNETWORKCACHE_ENABLED = False
print 'Networkcache forced to be disabled.'
def fallback_call(function):
"""Decorator which disallow to have any problem while calling method"""
def wrapper(self, *args, **kwd):
"""
Log the call, and the result of the call
"""
try:
return function(self, *args, **kwd)
except: # indeed, *any* exception is swallowed
print 'There was problem while calling method %r:\n%s' % (
function.__name__, traceback.format_exc())
return False
wrapper.__doc__ = function.__doc__
return wrapper
@fallback_call
def download_network_cached(cache_url, dir_url, software_url, software_root,
key, path, logger, signature_certificate_list):
"""Downloads from a network cache provider
return True if download succeeded.
"""
if not LIBNETWORKCACHE_ENABLED:
return False
if not(cache_url and dir_url and software_url and software_root):
return False
# In order to call nc nicely.
if len(signature_certificate_list) == 0:
signature_certificate_list = None
try:
nc = NetworkcacheClient(cache_url, dir_url,
signature_certificate_list=signature_certificate_list)
except TypeError:
logger.warning('Incompatible version of networkcache, not using it.')
return False
logger.info('Downloading %s binary from network cache.' % software_url)
try:
file_descriptor = None
json_entry_list = nc.select_generic(key)
for entry in json_entry_list:
json_information, _ = entry
try:
tags = json.loads(json_information)
if tags.get('machine') != platform.machine():
continue
if tags.get('os') != str(platform.linux_distribution()):
continue
if tags.get('software_url') != software_url:
continue
if tags.get('software_root') != software_root:
continue
sha512 = tags.get('sha512')
file_descriptor = nc.download(sha512)
break
except Exception:
continue
if file_descriptor is not None:
f = open(path, 'w+b')
try:
shutil.copyfileobj(file_descriptor, f)
finally:
f.close()
file_descriptor.close()
return True
except (IOError, DirectoryNotFound), e:
logger.info('Failed to download from network cache %s: %s' % \
(software_url, str(e)))
return False
@fallback_call
def upload_network_cached(software_root, software_url, cached_key,
cache_url, dir_url, path, logger, signature_private_key_file,
shacache_cert_file, shacache_key_file, shadir_cert_file, shadir_key_file):
"""Upload file to a network cache server"""
if not LIBNETWORKCACHE_ENABLED:
return False
if not (software_root and software_url and cached_key \
and cache_url and dir_url):
return False
logger.info('Uploading %s binary into network cache.' % software_url)
# YXU: "file" and "urlmd5" should be removed when server side is ready
kw = dict(
file="file",
urlmd5="urlmd5",
software_url=software_url,
software_root=software_root,
machine=platform.machine(),
os=str(platform.linux_distribution())
)
f = open(path, 'r')
# convert '' into None in order to call nc nicely
if not signature_private_key_file:
signature_private_key_file = None
if not shacache_cert_file:
shacache_cert_file = None
if not shacache_key_file:
shacache_key_file = None
if not shadir_cert_file:
shadir_cert_file = None
if not shadir_key_file:
shadir_key_file = None
try:
nc = NetworkcacheClient(cache_url, dir_url,
signature_private_key_file=signature_private_key_file,
shacache_cert_file=shacache_cert_file,
shacache_key_file=shacache_key_file,
shadir_cert_file=shadir_cert_file,
shadir_key_file=shadir_key_file)
except TypeError:
logger.warning('Incompatible version of networkcache, not using it.')
return False
try:
return nc.upload_generic(f, cached_key, **kw)
except (IOError, UploadError), e:
logger.info('Fail to upload file. %s' % (str(e)))
return False
finally:
f.close()
return True
......@@ -204,6 +204,17 @@ def parseArgumentTupleAndReturnSlapgridObject(*argument_tuple):
if not option_dict.get('supervisord_socket'):
option_dict['supervisord_socket'] = \
os.path.join(option_dict['instance_root'], 'supervisord.socket')
signature_certificate_list_string = \
option_dict.get('signature-certificate-list', None)
if signature_certificate_list_string is not None:
cert_marker = "-----BEGIN CERTIFICATE-----"
signature_certificate_list = [cert_marker + '\n' + q.strip() \
for q in signature_certificate_list_string.split(cert_marker) \
if q.strip()]
else:
signature_certificate_list = None
# Returning new Slapgrid instance and options
return ([Slapgrid(software_root=option_dict['software_root'],
instance_root=option_dict['instance_root'],
......@@ -218,7 +229,16 @@ def parseArgumentTupleAndReturnSlapgridObject(*argument_tuple):
master_ca_file=master_ca_file,
certificate_repository_path=certificate_repository_path,
signature_private_key_file=signature_private_key_file,
signature_certificate_list=signature_certificate_list,
download_binary_cache_url=\
option_dict.get('download-binary-cache-url', None),
upload_binary_cache_url=\
option_dict.get('upload-binary-cache-url', None),
upload_cache_url=option_dict.get('upload-cache-url', None),
download_binary_dir_url=\
option_dict.get('download-binary-dir-url', None),
upload_binary_dir_url=\
option_dict.get('upload-binary-dir-url', None),
upload_dir_url=option_dict.get('upload-dir-url', None),
console=option_dict['console'],
buildout=option_dict.get('buildout'),
......@@ -299,7 +319,12 @@ class Slapgrid(object):
key_file=None,
cert_file=None,
signature_private_key_file=None,
signature_certificate_list=None,
download_binary_cache_url=None,
upload_binary_cache_url=None,
upload_cache_url=None,
download_binary_dir_url=None,
upload_binary_dir_url=None,
upload_dir_url=None,
master_ca_file=None,
certificate_repository_path=None,
......@@ -323,7 +348,12 @@ class Slapgrid(object):
self.master_ca_file = master_ca_file
self.certificate_repository_path = certificate_repository_path
self.signature_private_key_file = signature_private_key_file
self.signature_certificate_list = signature_certificate_list
self.download_binary_cache_url = download_binary_cache_url
self.upload_binary_cache_url = upload_binary_cache_url
self.upload_cache_url = upload_cache_url
self.download_binary_dir_url = download_binary_dir_url
self.upload_binary_dir_url = upload_binary_dir_url
self.upload_dir_url = upload_dir_url
self.shacache_cert_file = shacache_cert_file
self.shacache_key_file = shacache_key_file
......@@ -403,7 +433,12 @@ class Slapgrid(object):
Software(url=software_release_uri, software_root=self.software_root,
console=self.console, buildout=self.buildout,
signature_private_key_file=self.signature_private_key_file,
signature_certificate_list=self.signature_certificate_list,
download_binary_cache_url=self.download_binary_cache_url,
upload_binary_cache_url=self.upload_binary_cache_url,
upload_cache_url=self.upload_cache_url,
download_binary_dir_url=self.download_binary_dir_url,
upload_binary_dir_url=self.upload_binary_dir_url,
upload_dir_url=self.upload_dir_url,
shacache_cert_file=self.shacache_cert_file,
shacache_key_file=self.shacache_key_file,
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment