Commit fed06c39 authored by Łukasz Nowak's avatar Łukasz Nowak

Imported shacache.

parent 17662fc5
Introduction
===================
The network cache server is a NoSQL storage with a REST API.
How does it works
===================
_______________
/ \
| |
------>| NETWORKCACHED |
| ----| |<-----
| | \_______________/ |
GET /key | | |
| | File | PUT / <- data
__|__v______ ______|_____
| | | |
| Client | | Client |
|____________| |____________|
Basically, the networkcached archives the files through HTTP PUT method.
When a client want to download the file it just need to provide the key
value and the server will send a response with the file data.
API:
PUT / :
parameter: file uploaded
Used to upload/modify an entry
GET /<key>
Return raw content
Raise HTTP error (404) if key does not exist
Installation
==============
$ python2.6 setup.py install
Now it is time to create the 'networkcached.conf' file in /etc/networkcached.conf
directory, using your preferred text editor (gedit, kate, vim.).
Follow text shall be put in this file:
[networkcached]
host = 127.0.0.1
port = 5001
cache_base_folder = /var/cache/networkcached/
Run the server:
# networkcached networkcached.conf
Setup Develoment Environment
===============================
$ mkdir -p ~/networkcached/downloads
$ cd ~/networkcached
Now it is time to create 'buildout.cfg' file in ~/networkcached directory,
using your preferred text editor (gedit, kate, vim.).
Follow text shall be put in this file:
[buildout]
extensions = mr.developer
auto-checkout = slapos.tool.networkcached
download-cache = /nexedi/buildout-networkcached/downloads
eggs-directory = /nexedi/buildout-networkcached/eggs
parts =
networkcached
[sources]
slapos.tool.networkcached = svn https://svn.erp5.org/repos/vifib/trunk/utils/slapos.tool.networkcached
[networkcached]
recipe = zc.recipe.egg
eggs =
slapos.tool.networkcached
Now you bootstrap the buildout:
$ python -S -c 'import urllib;print urllib.urlopen(\
"http://svn.zope.org/*checkout*/zc.buildout/trunk/bootstrap/bootstrap.py"\
).read()' | python -S -
Run the buildout:
$ bin/buildout -v
Now it is time to create the 'networkcached.conf' file in ~/networkcached
directory, using your preferred text editor (gedit, kate, vim.).
Follow text shall be put in this file:
[networkcached]
host = 127.0.0.1
port = 5001
cache_base_folder = ~/networkcached/networkcached-database
Now you can start your networkcached server:
$ bin/networkcached networkcached.conf
or
$ bin/networkcached -d networkcached-database -a 127.0.0.1 -p 5002
[egg_info]
tag_build = .dev
tag_svn_revision = 1
from setuptools import setup, find_packages
import os
name = "slapos.tool.networkcached"
version = '1.1'
def read(*rnames):
return open(os.path.join(os.path.dirname(__file__), *rnames)).read()
long_description=(
read('README.txt')
+ '\n' +
read('CHANGES.txt')
)
setup(
name = name,
version = version,
description = "networkcached - The networkcached is cache provider.",
long_description=long_description,
license = "GPLv3",
keywords = "vifib slapos cache",
classifiers=[
],
packages = find_packages('src'),
include_package_data = True,
package_dir = {'':'src'},
namespace_packages = ['slapos', 'slapos.tool'],
install_requires = [
'Flask', # used to create this
'setuptools', # namespaces
],
zip_safe=False,
entry_points = """
[console_scripts]
networkcached = %(name)s:main
runNetorkcacheTest = %(name)s.test:run
""" % dict(name=name),
)
# See http://peak.telecommunity.com/DevCenter/setuptools#namespace-packages
try:
__import__('pkg_resources').declare_namespace(__name__)
except ImportError:
from pkgutil import extend_path
__path__ = extend_path(__path__, __name__)
# See http://peak.telecommunity.com/DevCenter/setuptools#namespace-packages
try:
__import__('pkg_resources').declare_namespace(__name__)
except ImportError:
from pkgutil import extend_path
__path__ = extend_path(__path__, __name__)
##############################################################################
#
# Copyright (c) 2010 ViFiB SARL and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
import hashlib
import os
import sys
import traceback
from flask import Flask, request, helpers, abort
from config import NetworkcacheConfiguration, NetworkcacheParser
app = Flask(__name__)
@app.route('/', methods=['PUT'])
def put():
""" Save the file on the cache server. """
if getattr(request, 'data', None) is None:
abort(400, 'PUT requires data.')
cache_base_folder = app.config.get('CACHE_BASE_FOLDER')
file_name = hashlib.sha512(request.data).hexdigest()
try:
f = open(os.path.join(cache_base_folder, file_name), "w+")
try:
f.write(request.data)
finally:
f.close()
except:
app.logger.info(traceback.format_exc())
abort(500, "Faile to upload the file.")
return 'Success'
@app.route('/<key>', methods=['GET'])
def get(key):
""" Return the file if found, otherwise 404 is raised. """
cache_base_folder = app.config.get('CACHE_BASE_FOLDER')
file_path = os.path.join(cache_base_folder, key)
if not os.path.exists(file_path):
abort(404, 'File not found.')
return helpers.send_file(file_path)
def main():
"Run default configuration."
usage = "usage: %s [options] CONFIGURATION_FILE" % sys.argv[0]
try:
# Parse arguments
configuration = NetworkcacheConfiguration()
configuration.setConfig(*NetworkcacheParser(usage=usage).check_args())
app.config['CACHE_BASE_FOLDER'] = configuration.cache_base_folder
app.run(host=configuration.host, port=int(configuration.port), debug=True)
return_code = 0
except SystemExit, err:
# Catch exception raise by optparse
return_code = err
sys.exit(return_code)
##############################################################################
#
# Copyright (c) 2010 ViFiB SARL and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
import os
import logging
import logging.handlers
import ConfigParser
from optparse import OptionParser, Option
class NetworkcacheParser(OptionParser):
"""
Parse all arguments.
"""
def __init__(self, usage=None, version=None):
"""
Initialize all options possibles.
"""
OptionParser.__init__(self, usage=usage, version=version,
option_list=[
Option("-l", "--log_file",
help="The path to the log file used by the script.",
type=str),
Option("-v", "--verbose",
default=False,
action="store_true",
help="Verbose output."),
Option("-c", "--console",
default=False,
action="store_true",
help="Console output."),
Option("-d", "--cache_base_folder",
type=str,
help="Folder to store the files."),
Option("-a", "--host",
type=str,
help="Host address."),
Option("-p", "--port",
type=str,
help="Port number."),
])
def check_args(self):
"""
Check arguments
"""
(options, args) = self.parse_args()
configuration_file_path = []
if len(args) > 1:
self.error("Incorrect number of arguments")
if len(args) == 1:
configuration_file_path = args[0]
return options, configuration_file_path
class NetworkcacheConfiguration:
def setConfig(self, option_dict, configuration_file_path):
"""
Set options given by parameters.
"""
# Set options parameters
for option, value in option_dict.__dict__.items():
setattr(self, option, value)
# Load configuration file
if configuration_file_path:
configuration_parser = ConfigParser.SafeConfigParser()
configuration_parser.read(configuration_file_path)
# Merges the arguments and configuration
for section in ("networkcached",):
configuration_dict = dict(configuration_parser.items(section))
for key in configuration_dict:
if not getattr(self, key, None):
setattr(self, key, configuration_dict[key])
# set up logging
self.logger = logging.getLogger("networkcached")
self.logger.setLevel(logging.INFO)
if self.console:
self.logger.addHandler(logging.StreamHandler())
# cache base folder is required
if not self.cache_base_folder:
raise ValueError('cache_base_folder is required.')
if self.log_file:
if not os.path.isdir(os.path.dirname(self.log_file)):
# fallback to console only if directory for logs does not exists and
# continue to run
raise ValueError('Please create directory %r to store %r log file' % (
os.path.dirname(self.log_file), self.log_file))
else:
file_handler = logging.FileHandler(self.log_file)
format = "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
file_handler.setFormatter(logging.Formatter(format))
self.logger.addHandler(file_handler)
self.logger.info('Configured logging to file %r' % self.log_file)
self.logger.info("Started.")
if self.verbose:
self.logger.setLevel(logging.DEBUG)
self.logger.debug("Verbose mode enabled.")
[networkcached]
host = 127.0.0.1
port = 5001
cache_base_folder = /opt/slapos/networkcached
##############################################################################
#
# Copyright (c) 2010 ViFiB SARL and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
import hashlib
import os
import shutil
import slapos.tool.networkcached as networkcached
import tempfile
import unittest
class NetworkcacheTestCase(unittest.TestCase):
def setUp(self):
networkcached.app.config['CACHE_BASE_FOLDER'] = tempfile.mkdtemp()
self.cache_base_folder = networkcached.app.config['CACHE_BASE_FOLDER']
self.app = networkcached.app.test_client()
self.file_content = 'test file content.'
def tearDown(self):
shutil.rmtree(self.cache_base_folder)
def create_file_into_cache_base_folder(self):
"""
Create a binary file into the directory.
"""
file_name = hashlib.sha512(self.file_content).hexdigest()
file_path = os.path.join(self.cache_base_folder, file_name)
f = open(file_path, 'w')
try:
f.write(self.file_content)
finally:
f.close()
return file_path
def test_get_non_existing_file(self):
"""
When the client tries to download a file which does not exists, it must
return a 404 error.
"""
result = self.app.get('/does_not_exists')
self.assertEquals('404 NOT FOUND', result.status)
self.assertTrue('File not found.' in result.data)
def test_get_existing_file(self):
"""
Check if the file is returned if it exists.
"""
file_path = self.create_file_into_cache_base_folder()
result = self.app.get('/%s' % file_path.split('/')[-1])
self.assertEquals('200 OK', result.status)
self.assertEquals('application/octet-stream',
result.headers['Content-Type'])
self.assertEquals(self.file_content, result.data)
def test_puti_file(self):
"""
Check if the file will be upload just fine.
"""
file_content = 'file does not exist yet'
file_name = hashlib.sha512(file_content).hexdigest()
result = self.app.put('/', data=file_content)
self.assertEquals('200 OK', result.status)
self.assertEquals('Success', result.data)
f = open(os.path.join(self.cache_base_folder, file_name), 'r')
try:
data = f.read()
finally:
f.close()
self.assertEquals(file_content, data)
def run():
unittest.main(module=networkcached.test)
if __name__ == '__main__':
run()
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment