Skip to content

S
slapos.buildout
Commit b1accb72 authored by Łukasz Nowak's avatar Łukasz Nowak
Browse files
Options

Merge branch 'bug_778748'

parents b19763cf 41062088
Show whitespace changes
Inline Side-by-side
Showing with 237 additions and 35 deletions
networkcached.cfg
View file @ b1accb72
[buildout]
extends = buildout.cfg
#develop += path_to_slapos.libnetworkcache
parts += nc pync testnc
parts += cached
[cached]
[nc]
recipe = z3c.recipe.scripts
eggs =
slapos.libnetworkcache
zc.buildout
scripts =
buildout=buildout_cached
buildout=buildout_nc
[pync]
recipe = z3c.recipe.scripts
eggs = slapos.libnetworkcache
interpreter =
pync
executable = ${buildout:bin-directory}/pync
[test]
eggs +=
[testnc]
recipe = ${test:recipe}
eggs = ${test:eggs}
slapos.libnetworkcache
python = pync
src/zc/buildout/buildout.py
View file @ b1accb72
......@@ -337,7 +337,7 @@ class Buildout(UserDict.DictMixin):
os.chdir(options['directory'])
if not LIBNETWORKCACHE_ENABLED:
self._logger.warning('Networkache disabled, no '
self._logger.debug('Networkache disabled, no '
'slapos.libnetworkache available.')
self.download_cache_url = None
self.download_dir_url = None
......@@ -367,7 +367,7 @@ class Buildout(UserDict.DictMixin):
options['__networkcache__upload-cache-url'] = self.upload_cache_url
options['__networkcache__upload-dir-url'] = self.upload_dir_url
else:
self._logger.warning('Networkcache functionality not enabled.'
self._logger.debug('Networkcache functionality not enabled. '
'In order to activate use networkcache-section.')
self.download_cache_url = None
self.download_dir_url = None
......
src/zc/buildout/networkcache.py
View file @ b1accb72
......@@ -17,6 +17,7 @@ import hashlib
import os
import posixpath
import re
import shutil
try:
from slapos.libnetworkcache import NetworkcacheClient, UploadError, \
......@@ -67,27 +68,28 @@ def download_network_cached(dir_url, cache_url, path, url, logger, md5sum=None):
directory_key = get_directory_key(url)
file_name = os.path.basename(url)
url = os.path.basename(url)
logger.info('Downloading %s from network cache.' % file_name)
logger.info('Downloading %s from network cache.' % url)
try:
nc = NetworkcacheClient(shacache=cache_url, shadir=dir_url)
file_content = nc.select(directory_key)
file_descriptor = nc.select(directory_key)
buffer_size = min(1024, os.path.getsize(file_descriptor.name))
f = open(path, 'w+b')
try:
f.seek(0)
f.write(file_content)
shutil.copyfileobj(file_descriptor, f, buffer_size)
finally:
f.close()
file_descriptor.close()
if not check_md5sum(path, md5sum):
logger.info('MD5 checksum mismatch downloading %s' % file_name)
logger.info('MD5 checksum mismatch downloading %s' % url)
return False
except (IOError, DirectoryNotFound), e:
logger.info('Failed to download from network cache %s: %s' % \
(file_name, str(e)))
(url, str(e)))
return False
return True
......
src/zc/buildout/networkcache.txt
View file @ b1accb72
Using buildout with cache over network
======================================
Note: We are in process of preparing docs. This is only showcase.
>>> nc_data = tmpdir('nc_data')
>>> nc_url = start_nc(nc_data)
Networkcache in buildout is optional, and during normal run no information
is shown:
>>> write(sample_buildout, 'buildout.cfg',
... '''
... [buildout]
... parts =
... ''')
>>> print system(buildout)
But whenever networkcache configuration is enabled, it is possible to use it:
>>> write(sample_buildout, 'buildout.cfg',
... '''
... [buildout]
...
... networkcache-section = networkcache
... find-links = %(link_server)s
... parts = eggs
...
... [eggs]
... recipe = zc.recipe.egg:eggs
... eggs = demoneeded ==1.2c1
...
... [networkcache]
... download-cache-url = %(nc_url)sshacache
... download-dir-url = %(nc_url)sshadir
... upload-cache-url = %(nc_url)sshacache
... upload-dir-url = %(nc_url)sshadir
... ''' % globals())
>>> print system(buildout)
Networkcache enabled.
Networkcache download cache: 'http://localhost/shacache', directory 'http://localhost/shadir'
Networkcache upload cache: 'http://localhost/shacache', directory 'http://localhost/shadir'
Installing eggs.
Getting distribution for 'demoneeded==1.2c1'.
Downloading demoneeded-1.2c1.zip from network cache.
Failed to download from network cache demoneeded-1.2c1.zip:
Uploading http://localhost/demoneeded-1.2c1.zip into network cache.
Got demoneeded 1.2c1.
<BLANKLINE>
Now lets clean up buildout directory:
>>> import os, glob
>>> [remove(q) for q in glob.glob(os.path.join(sample_buildout, 'eggs', 'demoneeded*'))]
[None]
>>> remove('.installed.cfg')
And run it again, data will be downloaded from networkcache:
>>> print system(buildout)
Networkcache enabled.
Networkcache download cache: 'http://localhost/shacache', directory 'http://localhost/shadir'
Networkcache upload cache: 'http://localhost/shacache', directory 'http://localhost/shadir'
Installing eggs.
Getting distribution for 'demoneeded==1.2c1'.
Downloading demoneeded-1.2c1.zip from network cache.
Got demoneeded 1.2c1.
<BLANKLINE>
Now lets clean up buildout directory:
>>> import os, glob
>>> [remove(q) for q in glob.glob(os.path.join(sample_buildout, 'eggs', 'demoneeded*'))]
[None]
>>> remove('.installed.cfg')
As one can see, in case if upload is not working, there is no problem with running buildout:
>>> write(sample_buildout, 'buildout.cfg',
... '''
... [buildout]
...
... networkcache-section = networkcache
... find-links = %(link_server)s
... parts = eggs
...
... [eggs]
... recipe = zc.recipe.egg:eggs
... eggs = demo
...
... [networkcache]
... download-cache-url = %(nc_url)sshacache
... download-dir-url = %(nc_url)sshadir
... upload-cache-url = http://127.0.0.1:1/sshacache
... upload-dir-url = http://127.0.0.1:1/sshadir
... ''' % globals())
>>> print system(buildout)
Networkcache enabled.
Networkcache download cache: 'http://localhost/shacache', directory 'http://localhost/shadir'
Networkcache upload cache: 'http://127.0.0.1:1/sshacache', directory 'http://127.0.0.1:1/sshadir'
Installing eggs.
Getting distribution for 'demo'.
Downloading demo-0.4c1-py2.7.egg from network cache.
Failed to download from network cache demo-0.4c1-py2.7.egg:
Uploading http://localhost/demo-0.4c1-py2.7.egg into network cache.
Fail to upload file. [Errno 111] Connection refused
Got demo 0.4c1.
Getting distribution for 'demoneeded'.
Downloading demoneeded-1.2c1.zip from network cache.
Got demoneeded 1.2c1.
<BLANKLINE>
Now lets clean up buildout directory:
>>> import os, glob
>>> [remove(q) for q in glob.glob(os.path.join(sample_buildout, 'eggs', 'demo*'))]
[None, None]
>>> remove('.installed.cfg')
Upload is optional, and then no upload will be tried:
>>> write(sample_buildout, 'buildout.cfg',
... '''
......@@ -13,31 +127,64 @@ Note: We are in process of preparing docs. This is only showcase.
...
... [eggs]
... recipe = zc.recipe.egg:eggs
... eggs = demo ==0.2
... eggs = demo
...
... [networkcache]
... download-cache-url = http://127.0.0.1:1/
... download-dir-url = http://127.0.0.1:1/
... upload-cache-url = http://127.0.0.1:1/
... upload-dir-url = http://127.0.0.1:1/
... download-cache-url = %(nc_url)sshacache
... download-dir-url = %(nc_url)sshadir
... ''' % globals())
>>> print system(buildout)
Networkcache enabled.
Networkcache download cache: 'http://localhost/shacache', directory 'http://localhost/shadir'
Networkcache upload cache: '', directory ''
Installing eggs.
Getting distribution for 'demo'.
Downloading demo-0.4c1-py2.7.egg from network cache.
Failed to download from network cache demo-0.4c1-py2.7.egg:
Got demo 0.4c1.
Getting distribution for 'demoneeded'.
Downloading demoneeded-1.2c1.zip from network cache.
Got demoneeded 1.2c1.
<BLANKLINE>
Now lets clean up buildout directory:
>>> import os, glob
>>> [remove(q) for q in glob.glob(os.path.join(sample_buildout, 'eggs', 'demo*'))]
[None, None]
>>> remove('.installed.cfg')
>>> from glob import glob
>>> from os.path import join
>>> for egg in glob(join(sample_buildout, 'eggs', 'demoneeded*.egg')):
... remove(sample_buildout, 'eggs', egg)
In case if download cache is failing, original resources are used:
>>> write(sample_buildout, 'buildout.cfg',
... '''
... [buildout]
...
... networkcache-section = networkcache
... find-links = %(link_server)s
... parts = eggs
...
... [eggs]
... recipe = zc.recipe.egg:eggs
... eggs = demo
...
... [networkcache]
... download-cache-url = http://127.0.0.1:1/shacache
... download-dir-url = http://127.0.0.1:1/shadir
... upload-cache-url = http://127.0.0.1:1/sshacache
... upload-dir-url = http://127.0.0.1:1/sshadir
... ''' % globals())
>>> print system(buildout)
Networkcache enabled.
Networkcache download cache: 'http://127.0.0.1:1/', directory 'http://127.0.0.1:1/'
Networkcache upload cache: 'http://127.0.0.1:1/', directory 'http://127.0.0.1:1/'
Networkcache download cache: 'http://127.0.0.1:1/shacache', directory 'http://127.0.0.1:1/shadir'
Networkcache upload cache: 'http://127.0.0.1:1/sshacache', directory 'http://127.0.0.1:1/sshadir'
Installing eggs.
Getting distribution for 'demo==0.2'.
Downloading demo-0.2-py2.7.egg from network cache.
Failed to download from network cache demo-0.2-py2.7.egg: [Errno 111] Connection refused
Uploading http://localhost/demo-0.2-py2.7.egg into network cache.
Getting distribution for 'demo'.
Downloading demo-0.4c1-py2.7.egg from network cache.
Failed to download from network cache demo-0.4c1-py2.7.egg: [Errno 111] Connection refused
Uploading http://localhost/demo-0.4c1-py2.7.egg into network cache.
Fail to upload file. [Errno 111] Connection refused
Got demo 0.2.
Got demo 0.4c1.
Getting distribution for 'demoneeded'.
Downloading demoneeded-1.2c1.zip from network cache.
Failed to download from network cache demoneeded-1.2c1.zip: [Errno 111] Connection refused
......
src/zc/buildout/testing.py
View file @ b1accb72
......@@ -17,6 +17,7 @@ $Id$
"""
import BaseHTTPServer
import SimpleHTTPServer
import errno
import logging
import os
......@@ -32,6 +33,7 @@ import textwrap
import threading
import time
import urllib2
import json
import zc.buildout.buildout
import zc.buildout.easy_install
......@@ -267,7 +269,7 @@ def make_buildout(executable=None):
"""Make a buildout that uses this version of zc.buildout."""
# Create a basic buildout.cfg to avoid a warning from buildout.
open('buildout.cfg', 'w').write(
"[buildout]\nparts =\n"
"[buildout]\nparts =\nfind-links = http://pypi.python.org/pypi/slapos.libnetworkcache"
)
# Get state of installer defaults so we can reinstate them (instantiating
# a Buildout will force the Buildout's defaults on the installer).
......@@ -347,6 +349,12 @@ def buildoutSetUp(test):
os.chdir(sample)
make_buildout()
def start_nc(path):
port, thread = _start_nc(path, name=path)
url = 'http://localhost:%s/' % port
register_teardown(lambda: stop_server(url, thread))
return url
def start_server(path):
port, thread = _start_server(path, name=path)
url = 'http://localhost:%s/' % port
......@@ -416,6 +424,7 @@ def buildoutSetUp(test):
sdist = sdist,
bdist_egg = bdist_egg,
start_server = start_server,
start_nc = start_nc,
buildout = os.path.join(sample, 'bin', 'buildout'),
wait_until = wait_until,
make_py = make_py
......@@ -511,11 +520,37 @@ class Handler(BaseHTTPServer.BaseHTTPRequestHandler):
if self.__server.__log:
print '%s %s %s' % (self.command, code, self.path)
class NCHandler(SimpleHTTPServer.SimpleHTTPRequestHandler):
def do_GET(self):
if '__stop__' in self.path:
raise SystemExit
SimpleHTTPServer.SimpleHTTPRequestHandler.do_GET(self)
def do_PUT(self):
path = self.translate_path(self.path)
d = os.path.dirname(path)
if not os.path.exists(d):
os.makedirs(d)
data = self.rfile.read(int(self.headers.getheader('content-length')))
try:
d = json.loads(data)
except ValueError:
pass
else:
data = json.dumps([d])
open(path, 'wb').write(data)
self.send_response(201)
def _run(tree, port):
server_address = ('localhost', port)
httpd = Server(tree, server_address, Handler)
httpd.serve_forever()
def _run_nc(tree, port):
server_address = ('localhost', port)
httpd = Server(tree, server_address, NCHandler)
httpd.serve_forever()
def get_port():
for i in range(10):
port = random.randrange(20000, 30000)
......@@ -529,6 +564,17 @@ def get_port():
s.close()
raise RuntimeError, "Can't find port"
def _start_nc(tree, name=''):
port = get_port()
thread = threading.Thread(target=_run_nc, args=(tree, port), name=name)
thread.setDaemon(True)
thread.start()
wait(port, up=True)
return port, thread
def start_nc(tree):
return _start_nc(tree)[0]
def _start_server(tree, name=''):
port = get_port()
thread = threading.Thread(target=_run, args=(tree, port), name=name)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7