Merge branch 'bug_778748'

networkcached.cfg

 [buildout]
 extends = buildout.cfg
 
-#develop += path_to_slapos.libnetworkcache
+parts += nc pync testnc
 
-parts += cached
-
-[cached]
+[nc]
 recipe = z3c.recipe.scripts
 eggs =
   slapos.libnetworkcache
   zc.buildout
 scripts =
-  buildout=buildout_cached
+  buildout=buildout_nc
+
+[pync]
+recipe = z3c.recipe.scripts
+eggs = slapos.libnetworkcache
+interpreter =
+  pync
+executable = ${buildout:bin-directory}/pync
 
-[test]
-eggs +=
+[testnc]
+recipe = ${test:recipe}
+eggs = ${test:eggs}
   slapos.libnetworkcache
+python = pync

src/zc/buildout/buildout.py

@@ -337,7 +337,7 @@ class Buildout(UserDict.DictMixin):
         os.chdir(options['directory'])
 
         if not LIBNETWORKCACHE_ENABLED:
-            self._logger.warning('Networkache disabled, no '
+            self._logger.debug('Networkache disabled, no '
                 'slapos.libnetworkache available.')
             self.download_cache_url = None
             self.download_dir_url = None
@@ -367,7 +367,7 @@ class Buildout(UserDict.DictMixin):
                 options['__networkcache__upload-cache-url'] = self.upload_cache_url
                 options['__networkcache__upload-dir-url'] = self.upload_dir_url
             else:
-                self._logger.warning('Networkcache functionality not enabled.'
+                self._logger.debug('Networkcache functionality not enabled. '
                     'In order to activate use networkcache-section.')
                 self.download_cache_url = None
                 self.download_dir_url = None

src/zc/buildout/networkcache.py

@@ -17,6 +17,7 @@ import hashlib
 import os
 import posixpath
 import re
+import shutil
 
 try:
     from slapos.libnetworkcache import NetworkcacheClient, UploadError, \
@@ -67,27 +68,28 @@ def download_network_cached(dir_url, cache_url, path, url, logger, md5sum=None):
 
 
     directory_key = get_directory_key(url)
-    file_name = os.path.basename(url)
+    url = os.path.basename(url)
 
-    logger.info('Downloading %s from network cache.' % file_name)
+    logger.info('Downloading %s from network cache.' % url)
     try:
         nc = NetworkcacheClient(shacache=cache_url, shadir=dir_url)
-        file_content = nc.select(directory_key)
+        file_descriptor = nc.select(directory_key)
 
+        buffer_size = min(1024, os.path.getsize(file_descriptor.name))
         f = open(path, 'w+b')
         try:
-          f.seek(0)
-          f.write(file_content)
+            shutil.copyfileobj(file_descriptor, f, buffer_size)
         finally:
             f.close()
+            file_descriptor.close()
 
         if not check_md5sum(path, md5sum):
-            logger.info('MD5 checksum mismatch downloading %s' % file_name)
+            logger.info('MD5 checksum mismatch downloading %s' % url)
             return False
 
     except (IOError, DirectoryNotFound), e:
         logger.info('Failed to download from network cache %s: %s' % \
-                                                       (file_name, str(e)))
+                                                       (url, str(e)))
         return False
     return True
 

src/zc/buildout/networkcache.txt

 Using buildout with cache over network
 ======================================
 
-Note: We are in process of preparing docs. This is only showcase.
+   >>> nc_data = tmpdir('nc_data')
+   >>> nc_url = start_nc(nc_data)
+
+Networkcache in buildout is optional, and during normal run no information
+is shown:
+
+   >>> write(sample_buildout, 'buildout.cfg',
+   ... '''
+   ... [buildout]
+   ... parts =
+   ... ''')
+   >>> print system(buildout)
+
+But whenever networkcache configuration is enabled, it is possible to use it:
+
+   >>> write(sample_buildout, 'buildout.cfg',
+   ... '''
+   ... [buildout]
+   ...
+   ... networkcache-section = networkcache
+   ... find-links = %(link_server)s
+   ... parts = eggs
+   ...
+   ... [eggs]
+   ... recipe = zc.recipe.egg:eggs
+   ... eggs = demoneeded ==1.2c1
+   ...
+   ... [networkcache]
+   ... download-cache-url = %(nc_url)sshacache
+   ... download-dir-url = %(nc_url)sshadir
+   ... upload-cache-url = %(nc_url)sshacache
+   ... upload-dir-url = %(nc_url)sshadir
+   ... ''' % globals())
+   >>> print system(buildout)
+   Networkcache enabled.
+   Networkcache download cache: 'http://localhost/shacache', directory 'http://localhost/shadir'
+   Networkcache upload cache: 'http://localhost/shacache', directory 'http://localhost/shadir'
+   Installing eggs.
+   Getting distribution for 'demoneeded==1.2c1'.
+   Downloading demoneeded-1.2c1.zip from network cache.
+   Failed to download from network cache demoneeded-1.2c1.zip: 
+   Uploading http://localhost/demoneeded-1.2c1.zip into network cache.
+   Got demoneeded 1.2c1.
+   <BLANKLINE>
+
+Now lets clean up buildout directory:
+
+   >>> import os, glob
+   >>> [remove(q) for q in glob.glob(os.path.join(sample_buildout, 'eggs', 'demoneeded*'))]
+   [None]
+   >>> remove('.installed.cfg')
+
+And run it again, data will be downloaded from networkcache:
+
+   >>> print system(buildout)
+   Networkcache enabled.
+   Networkcache download cache: 'http://localhost/shacache', directory 'http://localhost/shadir'
+   Networkcache upload cache: 'http://localhost/shacache', directory 'http://localhost/shadir'
+   Installing eggs.
+   Getting distribution for 'demoneeded==1.2c1'.
+   Downloading demoneeded-1.2c1.zip from network cache.
+   Got demoneeded 1.2c1.
+   <BLANKLINE>
+
+Now lets clean up buildout directory:
+
+   >>> import os, glob
+   >>> [remove(q) for q in glob.glob(os.path.join(sample_buildout, 'eggs', 'demoneeded*'))]
+   [None]
+   >>> remove('.installed.cfg')
+
+As one can see, in case if upload is not working, there is no problem with running buildout:
+
+   >>> write(sample_buildout, 'buildout.cfg',
+   ... '''
+   ... [buildout]
+   ...
+   ... networkcache-section = networkcache
+   ... find-links = %(link_server)s
+   ... parts = eggs
+   ...
+   ... [eggs]
+   ... recipe = zc.recipe.egg:eggs
+   ... eggs = demo
+   ...
+   ... [networkcache]
+   ... download-cache-url = %(nc_url)sshacache
+   ... download-dir-url = %(nc_url)sshadir
+   ... upload-cache-url = http://127.0.0.1:1/sshacache
+   ... upload-dir-url = http://127.0.0.1:1/sshadir
+   ... ''' % globals())
+   >>> print system(buildout)
+   Networkcache enabled.
+   Networkcache download cache: 'http://localhost/shacache', directory 'http://localhost/shadir'
+   Networkcache upload cache: 'http://127.0.0.1:1/sshacache', directory 'http://127.0.0.1:1/sshadir'
+   Installing eggs.
+   Getting distribution for 'demo'.
+   Downloading demo-0.4c1-py2.7.egg from network cache.
+   Failed to download from network cache demo-0.4c1-py2.7.egg: 
+   Uploading http://localhost/demo-0.4c1-py2.7.egg into network cache.
+   Fail to upload file. [Errno 111] Connection refused
+   Got demo 0.4c1.
+   Getting distribution for 'demoneeded'.
+   Downloading demoneeded-1.2c1.zip from network cache.
+   Got demoneeded 1.2c1.
+   <BLANKLINE>
+
+Now lets clean up buildout directory:
+
+   >>> import os, glob
+   >>> [remove(q) for q in glob.glob(os.path.join(sample_buildout, 'eggs', 'demo*'))]
+   [None, None]
+   >>> remove('.installed.cfg')
+
+Upload is optional, and then no upload will be tried:
 
    >>> write(sample_buildout, 'buildout.cfg',
    ... '''
@@ -13,31 +127,64 @@ Note: We are in process of preparing docs. This is only showcase.
    ...
    ... [eggs]
    ... recipe = zc.recipe.egg:eggs
-   ... eggs = demo ==0.2
+   ... eggs = demo
    ...
    ... [networkcache]
-   ... download-cache-url = http://127.0.0.1:1/
-   ... download-dir-url = http://127.0.0.1:1/
-   ... upload-cache-url = http://127.0.0.1:1/
-   ... upload-dir-url = http://127.0.0.1:1/
+   ... download-cache-url = %(nc_url)sshacache
+   ... download-dir-url = %(nc_url)sshadir
    ... ''' % globals())
+   >>> print system(buildout)
+   Networkcache enabled.
+   Networkcache download cache: 'http://localhost/shacache', directory 'http://localhost/shadir'
+   Networkcache upload cache: '', directory ''
+   Installing eggs.
+   Getting distribution for 'demo'.
+   Downloading demo-0.4c1-py2.7.egg from network cache.
+   Failed to download from network cache demo-0.4c1-py2.7.egg: 
+   Got demo 0.4c1.
+   Getting distribution for 'demoneeded'.
+   Downloading demoneeded-1.2c1.zip from network cache.
+   Got demoneeded 1.2c1.
+   <BLANKLINE>
+
+Now lets clean up buildout directory:
+
+   >>> import os, glob
+   >>> [remove(q) for q in glob.glob(os.path.join(sample_buildout, 'eggs', 'demo*'))]
+   [None, None]
+   >>> remove('.installed.cfg')
 
-   >>> from glob import glob
-   >>> from os.path import join
-   >>> for egg in glob(join(sample_buildout, 'eggs', 'demoneeded*.egg')):
-   ...     remove(sample_buildout, 'eggs', egg)
+In case if download cache is failing, original resources are used:
 
+   >>> write(sample_buildout, 'buildout.cfg',
+   ... '''
+   ... [buildout]
+   ...
+   ... networkcache-section = networkcache
+   ... find-links = %(link_server)s
+   ... parts = eggs
+   ...
+   ... [eggs]
+   ... recipe = zc.recipe.egg:eggs
+   ... eggs = demo
+   ...
+   ... [networkcache]
+   ... download-cache-url = http://127.0.0.1:1/shacache
+   ... download-dir-url = http://127.0.0.1:1/shadir
+   ... upload-cache-url = http://127.0.0.1:1/sshacache
+   ... upload-dir-url = http://127.0.0.1:1/sshadir
+   ... ''' % globals())
    >>> print system(buildout)
    Networkcache enabled.
-   Networkcache download cache: 'http://127.0.0.1:1/', directory 'http://127.0.0.1:1/'
-   Networkcache upload cache: 'http://127.0.0.1:1/', directory 'http://127.0.0.1:1/'
+   Networkcache download cache: 'http://127.0.0.1:1/shacache', directory 'http://127.0.0.1:1/shadir'
+   Networkcache upload cache: 'http://127.0.0.1:1/sshacache', directory 'http://127.0.0.1:1/sshadir'
    Installing eggs.
-   Getting distribution for 'demo==0.2'.
-   Downloading demo-0.2-py2.7.egg from network cache.
-   Failed to download from network cache demo-0.2-py2.7.egg: [Errno 111] Connection refused
-   Uploading http://localhost/demo-0.2-py2.7.egg into network cache.
+   Getting distribution for 'demo'.
+   Downloading demo-0.4c1-py2.7.egg from network cache.
+   Failed to download from network cache demo-0.4c1-py2.7.egg: [Errno 111] Connection refused
+   Uploading http://localhost/demo-0.4c1-py2.7.egg into network cache.
    Fail to upload file. [Errno 111] Connection refused
-   Got demo 0.2.
+   Got demo 0.4c1.
    Getting distribution for 'demoneeded'.
    Downloading demoneeded-1.2c1.zip from network cache.
    Failed to download from network cache demoneeded-1.2c1.zip: [Errno 111] Connection refused

src/zc/buildout/testing.py

@@ -17,6 +17,7 @@ $Id$
 """
 
 import BaseHTTPServer
+import SimpleHTTPServer
 import errno
 import logging
 import os
@@ -32,6 +33,7 @@ import textwrap
 import threading
 import time
 import urllib2
+import json
 
 import zc.buildout.buildout
 import zc.buildout.easy_install
@@ -267,7 +269,7 @@ def make_buildout(executable=None):
     """Make a buildout that uses this version of zc.buildout."""
     # Create a basic buildout.cfg to avoid a warning from buildout.
     open('buildout.cfg', 'w').write(
-        "[buildout]\nparts =\n"
+        "[buildout]\nparts =\nfind-links = http://pypi.python.org/pypi/slapos.libnetworkcache"
         )
     # Get state of installer defaults so we can reinstate them (instantiating
     # a Buildout will force the Buildout's defaults on the installer).
@@ -347,6 +349,12 @@ def buildoutSetUp(test):
     os.chdir(sample)
     make_buildout()
 
+    def start_nc(path):
+        port, thread = _start_nc(path, name=path)
+        url = 'http://localhost:%s/' % port
+        register_teardown(lambda: stop_server(url, thread))
+        return url
+
     def start_server(path):
         port, thread = _start_server(path, name=path)
         url = 'http://localhost:%s/' % port
@@ -416,6 +424,7 @@ def buildoutSetUp(test):
         sdist = sdist,
         bdist_egg = bdist_egg,
         start_server = start_server,
+        start_nc = start_nc,
         buildout = os.path.join(sample, 'bin', 'buildout'),
         wait_until = wait_until,
         make_py = make_py
@@ -511,11 +520,37 @@ class Handler(BaseHTTPServer.BaseHTTPRequestHandler):
         if self.__server.__log:
             print '%s %s %s' % (self.command, code, self.path)
 
+class NCHandler(SimpleHTTPServer.SimpleHTTPRequestHandler):
+    def do_GET(self):
+        if '__stop__' in self.path:
+            raise SystemExit
+        SimpleHTTPServer.SimpleHTTPRequestHandler.do_GET(self)
+
+    def do_PUT(self):
+        path = self.translate_path(self.path)
+        d = os.path.dirname(path)
+        if not os.path.exists(d):
+            os.makedirs(d)
+        data = self.rfile.read(int(self.headers.getheader('content-length')))
+        try:
+            d = json.loads(data)
+        except ValueError:
+            pass
+        else:
+            data = json.dumps([d])
+        open(path, 'wb').write(data)
+        self.send_response(201)
+
 def _run(tree, port):
     server_address = ('localhost', port)
     httpd = Server(tree, server_address, Handler)
     httpd.serve_forever()
 
+def _run_nc(tree, port):
+    server_address = ('localhost', port)
+    httpd = Server(tree, server_address, NCHandler)
+    httpd.serve_forever()
+
 def get_port():
     for i in range(10):
         port = random.randrange(20000, 30000)
@@ -529,6 +564,17 @@ def get_port():
             s.close()
     raise RuntimeError, "Can't find port"
 
+def _start_nc(tree, name=''):
+    port = get_port()
+    thread = threading.Thread(target=_run_nc, args=(tree, port), name=name)
+    thread.setDaemon(True)
+    thread.start()
+    wait(port, up=True)
+    return port, thread
+
+def start_nc(tree):
+    return _start_nc(tree)[0]
+
 def _start_server(tree, name=''):
     port = get_port()
     thread = threading.Thread(target=_run, args=(tree, port), name=name)