Commit 47d7a87e by Rafael Monnerat

Initial Commit

0 parents
Changes
========
0.1.1 (2015-06-11)
---------------------
* Added sha256 verification. [Rafael Monnerat]
0.1.0 (2015-06-10)
---------------------
* Initial Release of slapcache. [Rafael Monnerat]
include README.txt
include CHANGES.txt
include slapcache/template/*
File mode changed
from setuptools import setup, find_packages
version = '0.1.1'
name = 'slapcache'
long_description = open("README.txt").read() + "\n" + \
open("CHANGES.txt").read() + "\n"
setup(name=name,
version=version,
description="SlapOS Cache Utils",
long_description=long_description,
classifiers=[
"Programming Language :: Python",
],
keywords='slapos shacache',
license='GPLv3',
url='http://www.slapos.org',
author='VIFIB',
namespace_packages=['slapcache'],
packages=find_packages(),
include_package_data=True,
install_requires=[
'slapos.libnetworkcache>=0.14.1',
'iniparse',
],
zip_safe=False,
entry_points={
'console_scripts': [
# Those entry points are development version and
# self updatable API
'slapcache-upload = slapcache.upload:main',
'slapcache-conf = slapcache.conf:do_conf',
'slapcache-download = slapcache.download:main',
],
},
test_suite="slapcache.test",
)
File mode changed
No preview for this file type
#!/usr/bin/python
# -*- coding: utf-8 -*-
##############################################################################
#
# Copyright (c) 2012-2014 Vifib SARL and Contributors.
# All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly advised to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
import datetime
import shutil
import logging
from optparse import OptionParser, Option
from time import time
import sys
from signature import Config
import os
class Parser(OptionParser):
"""
Parse all arguments.
"""
def __init__(self, usage=None, version=None):
"""
Initialize all options possibles.
"""
OptionParser.__init__(self, usage=usage, version=version,
option_list=[
Option("--slapos-configuration",
default='/etc/opt/slapcache.cfg',
help="Path to slapos configuration file"),
Option("--key",
default="slapos-global-key",
help="Upgrade Key for configuration file"),
Option("--force",
default=False,
action="store_true",
help="Overwrite configuration file"),
])
def check_args(self):
"""
Check arguments
"""
(options, args) = self.parse_args()
return options
def get_template(name):
fd = open('/'.join(__file__.split('/')[:-1]) + '/template/%s' % name, "r")
try:
content = fd.read()
finally:
fd.close()
return content
import os, errno
def mkdir_p(path):
if not os.path.exists(path):
os.makedirs(path)
def create(path, text=None, mode=0666):
fd = os.open(path, os.O_CREAT | os.O_WRONLY | os.O_TRUNC, mode)
try:
os.write(fd, text)
finally:
os.close(fd)
def do_conf():
"""Generate Default Configuration file """
usage = "usage: %s [options] " % sys.argv[0]
run_config(Config(Parser(usage=usage).check_args()))
sys.exit()
def run_config(config):
if not config.force and os.path.exists(config.slapos_configuration):
raise ValueError("%s already exists!" % config.slapos_configuration)
if os.path.exists(config.slapos_configuration):
shutil.move(config.slapos_configuration,
config.slapos_configuration + ".old.%s" % time())
mkdir_p('/'.join(config.slapos_configuration.split('/')[:-1]))
configuration_content = get_template("update.cfg.in")
configuration_dict = {"key": config.key}
create(path=config.slapos_configuration,
text=configuration_content % configuration_dict)
No preview for this file type
#!/usr/bin/python
# -*- coding: utf-8 -*-
##############################################################################
#
# Copyright (c) 2012-2014 Vifib SARL and Contributors.
# All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly advised to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
import ConfigParser
import datetime
from optparse import OptionParser, Option
import sys
from signature import Signature, Config
class Parser(OptionParser):
"""
Parse all arguments.
"""
def __init__(self, usage=None, version=None):
"""
Initialize all options possibles.
"""
OptionParser.__init__(self, usage=usage, version=version,
option_list=[
Option("--slapos-configuration",
default='/etc/opt/slapcache.cfg',
help="Configuration File used to upload the key."),
Option("--destination",
default='/etc/opt/slapcache',
help="File used as reference to upgrade."),
])
def check_args(self):
"""
Check arguments
"""
(options, args) = self.parse_args()
return options
def main():
"""Upload file to update computer and slapos"""
usage = "usage: [options] "
# Parse arguments
config = Config(Parser(usage=usage).check_args())
Signature(config).download()
sys.exit()
No preview for this file type
# -*- coding: utf-8 -*-
##############################################################################
#
# Copyright (c) 2012-2014 Vifib SARL and Contributors.
# All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly advised to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
import ConfigParser
import os
import time
import traceback
import tempfile
import datetime
import shutil
import hashlib
import base64
from random import choice
from string import ascii_lowercase
from slapos.networkcachehelper import NetworkcacheClient, \
helper_download_network_cached
class NetworkCache:
def __init__(self, configuration_path):
if not os.path.exists(configuration_path):
raise ValueError("You need configuration file")
self.configuration = configuration_path
self._load()
def _load(self):
network_cache_info = ConfigParser.RawConfigParser()
network_cache_info.read(self.configuration)
network_cache_info_dict = dict(network_cache_info.items('networkcache'))
def get_(name):
return network_cache_info_dict.get(name)
self.download_binary_cache_url = get_('download-binary-cache-url')
self.download_cache_url = get_('download-cache-url')
self.download_binary_dir_url = get_('download-binary-dir-url')
self.signature_certificate_list = get_('signature-certificate-list')
# Not mandatory
self.dir_url = get_('upload-dir-url')
self.cache_url = get_('upload-cache-url')
self.signature_private_key_file = get_('signature_private_key_file')
self.shacache_cert_file = get_('shacache-cert-file')
self.shacache_key_file = get_('shacache-key-file')
self.shadir_cert_file = get_('shadir-cert-file')
self.shadir_key_file = get_('shadir-key-file')
if network_cache_info.has_section('shacache'):
self.directory_key = network_cache_info.get('shacache', 'key')
else:
self.directory_key = "slapos-upgrade-testing-key"
def upload(self, path, metadata_dict, is_sha256file=False):
"""
Upload an existing file, using a file_descriptor.
"""
if is_sha256file:
key = self.directory_key + "-sha256-content"
else:
key = self.directory_key
file_descriptor = open(path, 'r')
if not (self.dir_url and self.cache_url):
raise ValueError("upload-dir-url and/or upload-cache-url is not defined")
# backward compatibility
metadata_dict.setdefault('file', 'notused')
metadata_dict.setdefault('urlmd5', 'notused')
# convert '' into None in order to call nc nicely
with NetworkcacheClient(self.cache_url, self.dir_url,
signature_private_key_file=self.signature_private_key_file or None,
shacache_cert_file=self.shacache_cert_file or None,
shacache_key_file=self.shacache_key_file or None,
shadir_cert_file=self.shadir_cert_file or None,
shadir_key_file=self.shadir_key_file or None,
) as nc:
return nc.upload(file_descriptor, key, **metadata_dict)
def download(self, path, wanted_metadata_dict={},
required_key_list=[], strategy=None, is_sha256file=False):
if is_sha256file:
key = self.directory_key + "-sha256-content"
else:
key = self.directory_key
result = helper_download_network_cached(
self.download_binary_dir_url,
self.download_binary_cache_url,
self.signature_certificate_list,
key, wanted_metadata_dict,
required_key_list, strategy)
if result:
# XXX check if nc filters signature_certificate_list!
# Creates a file with content to desired path.
file_descriptor, metadata_dict = result
f = open(path, 'w+b')
try:
shutil.copyfileobj(file_descriptor, f)
# XXX method should check MD5.
return metadata_dict
finally:
f.close()
file_descriptor.close()
return False
def strategy(entry_list):
"""Get the latest entry. """
timestamp = 0
best_entry = None
for entry in entry_list:
if entry['timestamp'] > timestamp:
best_entry = entry
timestamp = entry['timestamp']
return best_entry
class Signature:
def __init__(self, config, logger=None):
self.config = config
self.logger = logger
def log(self, message):
if self.logger is not None:
self.logger.debug(message)
else:
print message
def get_file_hash(self, path):
with open(path) as f:
h = hashlib.sha256()
h.update(f.read())
base = base64.b64encode(h.digest())
return base
def save_file_hash(self, path, destination):
base = self.get_file_hash(path)
with open(destination, "w") as f:
f.write(base)
def _download(self, path):
"""
Download a tar of the repository from cache, and untar it.
"""
info, sha256path = tempfile.mkstemp()
shacache = NetworkCache(self.config.slapos_configuration)
if shacache.signature_certificate_list is None:
raise ValueError("You need at least one valid signature for download")
download_metadata_dict = shacache.download(path=path,
required_key_list=['timestamp'], strategy=strategy)
if download_metadata_dict:
self.log('File downloaded on temp %s' % path)
current_sha256 = self.get_file_hash(path)
if shacache.download(path=sha256path, required_key_list=['timestamp'],
strategy=strategy, is_sha256file=True):
self.log('File downloaded on temp. %s' % sha256path)
with open(sha256path) as f:
expected_sha256 = f.read()
if current_sha256 == expected_sha256:
return True
else:
raise ValueError("%s != %s" % (current_sha256, expected_sha256))
def download(self):
"""
Get status information and return its path
"""
info, path = tempfile.mkstemp()
if not self._download(path) == False:
shutil.move(path, self.config.destination)
else:
raise ValueError("No result from shacache")
def _upload(self, path):
"""
Creates uploads repository to cache.
"""
shacache = NetworkCache(self.config.slapos_configuration)
sha256path = path + ".sha256"
self.save_file_hash(path, sha256path)
metadata_dict = {
# XXX: we set date from client side. It can be potentially dangerous
# as it can be badly configured.
'timestamp': time.time(),
'token': ''.join([choice(ascii_lowercase) for _ in range(128)])
}
try:
if shacache.upload(path=path,
metadata_dict=metadata_dict):
self.log('Uploaded upload file to cache.')
if shacache.upload(path=sha256path,
metadata_dict=metadata_dict, is_sha256file=True):
self.log('Uploaded sha256file file to cache.')
else:
self.log('Fail to upload sha256file file to cache.')
else:
self.log('Fail to upload file to cache.')
except Exception:
self.log('Unable to upload to cache:\n%s.' % traceback.format_exc())
return
def upload(self):
self._upload(self.config.file)
# Class containing all parameters needed for configuration
class Config:
def __init__(self, option_dict=None):
if option_dict is not None:
# Set options parameters
for option, value in option_dict.__dict__.items():
setattr(self, option, value)
No preview for this file type
[shacache]
key = %(key)s
[networkcache]
download-binary-cache-url = http://www.shacache.org/shacache
download-cache-url = https://www.shacache.org/shacache
download-binary-dir-url = http://www.shacache.org/shadir
# Standard Signature handled by SlapOS Administrators
signature-certificate-list =
-----BEGIN CERTIFICATE-----
MIIB8DCCAVmgAwIBAgIJAPFf61p8y809MA0GCSqGSIb3DQEBBQUAMBAxDjAMBgNV
BAMMBUNPTVAtMCAXDTE0MDIxNzE2NDgxN1oYDzIxMTQwMTI0MTY0ODE3WjAQMQ4w
DAYDVQQDDAVDT01QLTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAsiqCyuv1
HO9FmtwnMbEa1/u8Dn7T0k7hVKYXVQYof+59Ltbb3cA3nLjFSJDr/wQT6N89MccS
PneRzkWqZKL06Kmj+N+XJfRyVaTz1qQtNzjdbYkO6RgQq+fvq2CO0+PSnL6NttLU
/a9nQMcVm7wZ8kmY+AG5LbVo8lmxDD16Wq0CAwEAAaNQME4wHQYDVR0OBBYEFEVi
YyWHF3W7/O4NaTjn4lElLpp7MB8GA1UdIwQYMBaAFEViYyWHF3W7/O4NaTjn4lEl
Lpp7MAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADgYEAgIPGoxhUa16AgjZx
Jr1kUrs8Fg3ig8eRFQlBSLYfANIUxcQ2ScFAkmsvwXY3Md7uaSvMJsEl2jcjdmdi
eSreNkx85j9GtMLY/2cv0kF4yAQNRtibtDkbg6fRNkmUopDosJNVf79l1GKX8JFL
zZBOFdOaLYY/6dLRwiTUKHU6su8=
-----END CERTIFICATE-----
No preview for this file type
# -*- coding: utf-8 -*-
##############################################################################
#
# Copyright (c) 2012-2014 Vifib SARL and Contributors.
# All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly advised to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
def _fake_call(self, *args, **kw):
self.last_call = (args, kw)
CONFIGURATION_FILE = """
[networkcache]
download-binary-cache-url = http://www.shacache.org/shacache
download-cache-url = https://www.shacache.org/shacache
download-binary-dir-url = http://www.shacache.org/shadir
signature-certificate-list =
-----BEGIN CERTIFICATE-----
MIIB8DCCAVmgAwIBAgIJAPFf61p8y809MA0GCSqGSIb3DQEBBQUAMBAxDjAMBgNV
BAMMBUNPTVAtMCAXDTE0MDIxNzE2NDgxN1oYDzIxMTQwMTI0MTY0ODE3WjAQMQ4w
DAYDVQQDDAVDT01QLTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAsiqCyuv1
HO9FmtwnMbEa1/u8Dn7T0k7hVKYXVQYof+59Ltbb3cA3nLjFSJDr/wQT6N89MccS
PneRzkWqZKL06Kmj+N+XJfRyVaTz1qQtNzjdbYkO6RgQq+fvq2CO0+PSnL6NttLU
/a9nQMcVm7wZ8kmY+AG5LbVo8lmxDD16Wq0CAwEAAaNQME4wHQYDVR0OBBYEFEVi
YyWHF3W7/O4NaTjn4lElLpp7MB8GA1UdIwQYMBaAFEViYyWHF3W7/O4NaTjn4lEl
Lpp7MAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADgYEAgIPGoxhUa16AgjZx
Jr1kUrs8Fg3ig8eRFQlBSLYfANIUxcQ2ScFAkmsvwXY3Md7uaSvMJsEl2jcjdmdi
eSreNkx85j9GtMLY/2cv0kF4yAQNRtibtDkbg6fRNkmUopDosJNVf79l1GKX8JFL
zZBOFdOaLYY/6dLRwiTUKHU6su8=
-----END CERTIFICATE-----
[slapupdate]
upgrade_key = slapos-upgrade-testing-key-with-config-file
"""
UPGRADE_KEY = """[debian-default]
repository-list =
main = http://ftp.fr.debian.org/debian/ wheezy main
main-src = http://ftp.fr.debian.org/debian/ wheezy main
update = http://ftp.fr.debian.org/debian/ wheezy-updates main
update-src = http://ftp.fr.debian.org/debian/ wheezy-updates main
slapos = http://download.opensuse.org/repositories/home:/VIFIBnexedi:/branches:/home:/VIFIBnexedi/Debian_7.0/ ./
re6stnet = http://git.erp5.org/dist/deb ./
key-list =
slapos = http://git.erp5.org/gitweb/slapos.package.git/blob_plain/HEAD:/debian-preseed/slapos.openbuildservice.key
re6st = http://git.erp5.org/gitweb/slapos.package.git/blob_plain/HEAD:/debian-preseed/git.erp5.org.key
filter-package-list =
ntp
openvpn
slapos.node
re6stnet
filter-promise-list =
core
signature-list =
debian+++jessie/sid+++
debian+++7.4+++
debian+++7.5+++
debian+++7.3+++
debian+++7+++
[opensuse-legacy]
repository-list =
suse = http://download.opensuse.org/distribution/12.1/repo/oss/
slapos = http://download.opensuse.org/repositories/home:/VIFIBnexedi:/branches:/home:/VIFIBnexedi/openSUSE_12.1/
re6st = http://git.erp5.org/dist/rpm
key-list =
filter-promise-list =
core
filter-package-list =
ntp
openvpn
slapos.node
re6stnet
signature-list =
opensuse+++12.1+++x86_64
[system]
reboot = 2011-10-10
upgrade = 2014-06-04
"""
UPGRADE_KEY_WITHOUT_KEY_LIST = """[debian-default]
repository-list =
main = http://ftp.fr.debian.org/debian/ wheezy main
main-src = http://ftp.fr.debian.org/debian/ wheezy main
update = http://ftp.fr.debian.org/debian/ wheezy-updates main
update-src = http://ftp.fr.debian.org/debian/ wheezy-updates main
slapos = http://download.opensuse.org/repositories/home:/VIFIBnexedi:/branches:/home:/VIFIBnexedi/Debian_7.0/ ./
re6stnet = http://git.erp5.org/dist/deb ./
filter-package-list =
ntp
openvpn
slapos.node
re6stnet
filter-promise-list =
core
signature-list =
debian+++jessie/sid+++
debian+++7.4+++
debian+++7.5+++
debian+++7.3+++
debian+++7+++
[opensuse-legacy]
repository-list =
suse = http://download.opensuse.org/distribution/12.1/repo/oss/
slapos = http://download.opensuse.org/repositories/home:/VIFIBnexedi:/branches:/home:/VIFIBnexedi/openSUSE_12.1/
re6st = http://git.erp5.org/dist/rpm
filter-promise-list =
core
filter-package-list =
ntp
openvpn
slapos.node
re6stnet
signature-list =
opensuse+++12.1+++x86_64
[system]
reboot = 2011-10-10
upgrade = 2014-06-04
"""
No preview for this file type
No preview for this file type
#!/usr/bin/python
# -*- coding: utf-8 -*-
##############################################################################
#
# Copyright (c) 2012-2014 Vifib SARL and Contributors.
# All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly advised to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
import ConfigParser
import datetime
from optparse import OptionParser, Option
import sys
from signature import Signature, Config
class Parser(OptionParser):
"""
Parse all arguments.
"""
def __init__(self, usage=None, version=None):
"""
Initialize all options possibles.
"""
OptionParser.__init__(self, usage=usage, version=version,
option_list=[
Option("--slapos-configuration",
default='/etc/opt/slapcache.cfg',
help="Configuration File used to upload the key."),
Option("--file",
default='/etc/opt/slapos/slapos-upgrade',
help="File used as reference to upgrade."),
Option("-n", "--dry-run",
help="Simulate the execution steps",
default=False,
action="store_true"),
])
def check_args(self):
"""
Check arguments
"""
(options, args) = self.parse_args()
return options
# Utility fonction to get yes/no answers
def get_yes_no(prompt):
while True:
answer = raw_input(prompt + " [y,n]: ")
if answer.upper() in ['Y', 'YES']:
return True
if answer.upper() in ['N', 'NO']:
return False
def main():
"""Upload file to update computer and slapos"""
usage = "usage: [options] "
# Parse arguments
config = Config(Parser(usage=usage).check_args())
Signature(config).upload()
sys.exit()
No preview for this file type
Styling with Markdown is supported
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!