From f16efa06bc3a2c54e71de8f5d39ffeef6f480f33 Mon Sep 17 00:00:00 2001 From: Jim Fulton <jim@zope.com> Date: Mon, 19 Mar 2007 15:03:18 +0000 Subject: [PATCH] Added support for a download cache. --- CHANGES.txt | 9 ++ src/zc/buildout/buildout.py | 23 +++++ src/zc/buildout/downloadcache.txt | 136 ++++++++++++++++++++++++++++++ 3 files changed, 168 insertions(+) create mode 100644 src/zc/buildout/downloadcache.txt diff --git a/CHANGES.txt b/CHANGES.txt index ae41390..66e9f5e 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -14,6 +14,15 @@ Change History 1.0.0b23 (2007-03-??) ===================== +Feature Changes +--------------- + +- Added support for download caches. A buildout can specify a cache + for distribution downloads. The cache can be shared among buildouts + to reduce network access and to support creating source + distributions for applications allowing install without network + access. + Bugs Fixed ---------- diff --git a/src/zc/buildout/buildout.py b/src/zc/buildout/buildout.py index 2687065..a2ee56e 100644 --- a/src/zc/buildout/buildout.py +++ b/src/zc/buildout/buildout.py @@ -149,6 +149,29 @@ class Buildout(UserDict.DictMixin): if versions: zc.buildout.easy_install.default_versions(dict(self[versions])) + download_cache = options.get('download-cache') + if download_cache: + download_cache = os.path.join(options['directory'], download_cache) + if not os.path.isdir(download_cache): + raise zc.buildout.UserError( + 'The specified download cache:\n' + '%r\n' + "Doesn't exist.\n" + % download_cache) + download_cache = os.path.join(download_cache, 'dist') + if not os.path.isdir(download_cache): + os.mkdir(download_cache) + + zc.buildout.easy_install.download_cache(download_cache) + + install_from_cache = options.get('install-from-cache') + if install_from_cache: + if install_from_cache not in ('true', 'false'): + self._error('Invalid value for install-from-cache option: %s', + install_from_cache) + if install_from_cache == 'true': + zc.buildout.easy_install.install_from_cache(True) + # "Use" each of the defaults so they aren't reported as unused options. for name in _buildout_default_options: options[name] diff --git a/src/zc/buildout/downloadcache.txt b/src/zc/buildout/downloadcache.txt new file mode 100644 index 0000000..f6148e5 --- /dev/null +++ b/src/zc/buildout/downloadcache.txt @@ -0,0 +1,136 @@ +Using a download cache +====================== + +Normally, when distributions are installed, if any processing is +needed, they are downloaded from the internet to a temporary directory +and then installed from there. A download cache can be used to avoid +the download step. This can be useful to reduce network access and to +create source distributions of an entire buildout. + +The buildout download-cache option can be used to specify a directory +to be used as a download cache. + +In this example, we'll create a directory to hold the cache: + + >>> cache = tmpdir('cache') + +And set up a buildout that downloads some eggs: + + >>> write('buildout.cfg', + ... ''' + ... [buildout] + ... parts = eggs + ... download-cache = %(cache)s + ... find-links = %(link_server)s + ... + ... [eggs] + ... recipe = zc.recipe.egg + ... eggs = demo ==0.2 + ... ''' % globals()) + +We specified a link server that has some distributions available for +download: + + >>> print get(link_server), + <html><body> + <a href="demo-0.1-py2.4.egg">demo-0.1-py2.4.egg</a><br> + <a href="demo-0.2-py2.4.egg">demo-0.2-py2.4.egg</a><br> + <a href="demo-0.3-py2.4.egg">demo-0.3-py2.4.egg</a><br> + <a href="demoneeded-1.0.zip">demoneeded-1.0.zip</a><br> + <a href="demoneeded-1.1.zip">demoneeded-1.1.zip</a><br> + <a href="extdemo-1.4.zip">extdemo-1.4.zip</a><br> + <a href="index/">index/</a><br> + <a href="other-1.0-py2.4.egg">other-1.0-py2.4.egg</a><br> + </body></html> + + +We'll enable logging on the link server so we can see what's going on: + + >>> get(link_server+'enable_server_logging') + GET 200 /enable_server_logging + '' + +We also specified a download cache. + +If we run the buildout, we'll see the eggs installed from the link +server as usual: + + >>> print system(buildout), + GET 200 / + GET 200 /demo-0.2-py2.4.egg + GET 200 /demoneeded-1.1.zip + buildout: Installing eggs + zc.buildout.easy_install: Getting new distribution for demo==0.2 + zc.buildout.easy_install: Got demo 0.2 + zc.buildout.easy_install: Getting new distribution for demoneeded + zc.buildout.easy_install: Got demoneeded 1.1 + +We'll also get the download cache populated. The buildout doesn't put +files in the cache directly. It creates an intermediate directory, +dist: + + + >>> ls(cache) + d dist + + >>> ls(cache, 'dist') + - demo-0.2-py2.4.egg + - demoneeded-1.1.zip + +If we remove the installed eggs from eggs directory and re-run the buildout: + + >>> import os + >>> for f in os.listdir('eggs'): + ... if f.startswith('demo'): + ... remove('eggs', f) + + >>> print system(buildout), + GET 200 / + buildout: Updating eggs + zc.buildout.easy_install: Getting new distribution for demo==0.2 + zc.buildout.easy_install: Got demo 0.2 + zc.buildout.easy_install: Getting new distribution for demoneeded + zc.buildout.easy_install: Got demoneeded 1.1 + +We see that the distributions aren't downloaded, because they're +downloaded from the cache. + +Installing solely from a download cache +--------------------------------------- + +A download cache can be used as the basis of application source +releases. In an application source release, we want to distribute an +application that can be built without making any network accesses. In +this case, we distribute a buildout with download cache and tell the +buildout to install from the download cache only, without making +network accesses. The buildout install-from-cache option can be used +to signal that packages should be installed only from the download +cache. + +Let's remove our installed eggs and run the buildout with the +install-from-cache option set to true: + + >>> for f in os.listdir('eggs'): + ... if f.startswith('demo'): + ... remove('eggs', f) + + >>> write('buildout.cfg', + ... ''' + ... [buildout] + ... parts = eggs + ... download-cache = %(cache)s + ... install-from-cache = true + ... find-links = %(link_server)s + ... + ... [eggs] + ... recipe = zc.recipe.egg + ... eggs = demo + ... ''' % globals()) + + >>> print system(buildout), + buildout: Uninstalling eggs + buildout: Installing eggs + zc.buildout.easy_install: Getting new distribution for demo + zc.buildout.easy_install: Got demo 0.2 + zc.buildout.easy_install: Getting new distribution for demoneeded + zc.buildout.easy_install: Got demoneeded 1.1 -- 2.30.9