Merged master

.travis.yml

@@ -6,11 +6,19 @@ env:
     - PYTHON_VER=3.3
     - PYTHON_VER=3.4
 
+sudo: false
+cache:
+  directories:
+  - pythons
+# Note on this cache directories: "make build" downloads pythons elsewhere,
+# but installs the compiled bin/lib/share in pythons/pythonx.y/ .
+
 notifications:
   email:
     - buildout-development@googlegroups.com
 
 install:
+    - ls -al pythons
     - deactivate
     - make build
 

CHANGES.rst

 Change History
 **************
 
-Unreleased
-==========
+2.4.0 (unreleased)
+==================
 
 - Undo breakage on Windows machines where ``sys.prefix`` can also be a
   ``site-packages`` directory:  don't remove it from ``sys.path``.  See
@@ -22,6 +22,13 @@ Unreleased
   https://github.com/buildout/buildout/pull/222 .
   [lrowe]
 
+- Note: zc.recipe.egg has also been updated to 2.0.2 together with this
+  zc.buildout release. Fixed: In ``zc.recipe.egg#custom`` recipe's ``rpath``
+  support, don't assume path elements are buildout-relative if they start with
+  one of the "special" tokens (e.g., ``$ORIGIN``).  See:
+  https://github.com/buildout/buildout/issues/225.
+  [tseaver]
+
 - Bootstrap script now accepts ``--to-dir``. Setuptools is installed there. If
   already available there, it is reused. This can be used to bootstrap
   buildout without internet access. Similarly, a local ``ez_setup.py`` is used
@@ -33,6 +40,17 @@ Unreleased
   filename. Fixes #89 and #148.
   [reinout]
 
+- Updated buildout's `travis-ci <https://travis-ci.org/buildout/buildout>`_
+  configuration so that tests run much quicker so that buildout is easier and
+  quicker to develop.
+
+- ``download-cache``, ``eggs-directory`` and ``extends-cache`` are now
+  automatically created if their parent directory exists. Also they can be
+  relative directories (relative to the location of the buildout config file
+  that defines them). Also they can now be in the form ``~/subdir``, with the
+  usual convention that the ``~`` char means the home directory of the user
+  running buildout.
+  [lelit]
 
 2.3.1 (2014-12-16)
 ==================

DEVELOPERS.txt

@@ -7,6 +7,10 @@ When you're developing buildout itself, you need to know two things:
   will find your already-installed setuptools, leading to test differences
   when setuptools' presence is explicitly tested.
 
+- Also the presence of ``~/.buildout/default.cfg`` may interfere with the
+  tests so you may want to temporarily rename it so that it does not get in
+  the way.
+
 - Don't bootstrap with ``python bootstrap/bootstrap.py`` but with ``python
   dev.py``.
 

Makefile

 HERE = $(shell pwd)
 PYTHON_VER ?= 2.7
-PYTHON_PATH = $(HERE)/python$(PYTHON_VER)
+PYTHON_PATH = $(HERE)/pythons/$(PYTHON_VER)
+PYTHON_BUILD_DIR = $(HERE)/python_builds
 
 ifeq ($(PYTHON_VER),2.6)
 	PYTHON_MINOR ?= 2.6.8
@@ -32,28 +33,29 @@ BUILD_DIRS = $(PYTHON_PATH) bin build develop-eggs eggs parts
 
 all: build
 
-$(PYTHON_PATH):
+$(PYTHON_PATH)/bin/$(PYTHON_EXE):
 	@echo "Installing Python"
 	mkdir -p $(PYTHON_PATH)
-	cd $(PYTHON_PATH) && \
+	mkdir -p $(PYTHON_BUILD_DIR)
+	cd $(PYTHON_BUILD_DIR) && \
 	curl --progress-bar --location $(PYTHON_DOWNLOAD) | tar -zx
 ifeq ($(PYTHON_VER),2.6)
-	cd $(PYTHON_PATH) && \
+	cd $(PYTHON_BUILD_DIR) && \
 	curl --progress-bar -L https://raw.github.com/collective/buildout.python/ad45adb78bfa37542d62a394392d5146fce5af34/src/issue12012-sslv2-py26.patch > ssl.patch
-	cd $(PYTHON_PATH)/$(PYTHON_ARCHIVE) && \
+	cd $(PYTHON_BUILD_DIR)/$(PYTHON_ARCHIVE) && \
 	patch -p0 < ../ssl.patch
 endif
-	cd $(PYTHON_PATH)/$(PYTHON_ARCHIVE) && \
+	cd $(PYTHON_BUILD_DIR)/$(PYTHON_ARCHIVE) && \
 	./configure --prefix $(PYTHON_PATH) $(PYTHON_CONFIGURE_ARGS) >/dev/null 2>&1 && \
 	make >/dev/null 2>&1 && \
 	make install >/dev/null 2>&1
 	@echo "Finished installing Python"
 
-build: $(PYTHON_PATH)
+build: $(PYTHON_PATH)/bin/$(PYTHON_EXE)
 	$(PYTHON_PATH)/bin/$(PYTHON_EXE) dev.py
 
 clean:
-	rm -rf $(BUILD_DIRS)
+	rm -rf $(BUILD_DIRS) $(PYTHON_BUILD_DIR)
 
 test:
 	$(HERE)/bin/test -1 -v

setup.py

@@ -12,7 +12,7 @@
 #
 ##############################################################################
 name = "zc.buildout"
-version = "2.3.1"
+version = "2.4.0.dev0"
 
 import os
 from setuptools import setup

src/zc/buildout/bootstrap.txt

@@ -24,9 +24,9 @@ Make sure the bootstrap script actually works::
     ...     zc.buildout.easy_install._safe_arg(sys.executable)+' '+
     ...     'bootstrap.py')); print_('X') # doctest: +ELLIPSIS
     X...
+    Creating directory '/sample/eggs'.
     Creating directory '/sample/bin'.
     Creating directory '/sample/parts'.
-    Creating directory '/sample/eggs'.
     Creating directory '/sample/develop-eggs'.
     Generated script '/sample/bin/buildout'.
     ...

src/zc/buildout/bootstrap_cl_settings.test

@@ -27,9 +27,9 @@ Some people pass buildout settings to bootstrap.
     ...     ' -c'+join('buildout', 'buildout.cfg')
     ...    )); print_('X') # doctest: +ELLIPSIS
     X...
+    Creating directory '/top/eggs'.
     Creating directory '/top/bin'.
     Creating directory '/top/parts'.
-    Creating directory '/top/eggs'.
     Creating directory '/top/develop-eggs'.
     Generated script '/top/bin/buildout'.
     ...
@@ -52,9 +52,9 @@ They might do it with init, but no worries:
     ...    )); print_('X') # doctest: +ELLIPSIS
     X...
     Creating '/top/buildout/buildout.cfg'.
+    Creating directory '/top/eggs'.
     Creating directory '/top/bin'.
     Creating directory '/top/parts'.
-    Creating directory '/top/eggs'.
     Creating directory '/top/develop-eggs'.
     Generated script '/top/bin/buildout'.
     ...

src/zc/buildout/buildout.py

@@ -266,6 +266,37 @@ class Buildout(DictMixin):
                  if k not in versions
                  ))
 
+        # Absolutize some particular directory, handling also the ~/foo form,
+        # and considering the location of the configuration file that generated
+        # the setting as the base path, falling back to the main configuration
+        # file location
+        for name in ('download-cache', 'eggs-directory', 'extends-cache'):
+            if name in data['buildout']:
+                origdir, src = data['buildout'][name]
+                if '${' in origdir:
+                    continue
+                if not os.path.isabs(origdir):
+                    if src in ('DEFAULT_VALUE',
+                               'COMPUTED_VALUE',
+                               'COMMAND_LINE_VALUE'):
+                        if 'directory' in data['buildout']:
+                            basedir = data['buildout']['directory'][0]
+                        else:
+                            basedir = self._buildout_dir
+                    else:
+                        if _isurl(src):
+                            raise zc.buildout.UserError(
+                                'Setting "%s" to a non absolute location ("%s") '
+                                'within a\n'
+                                'remote configuration file ("%s") is ambiguous.' % (
+                                    name, origdir, src))
+                        basedir = os.path.dirname(src)
+                    absdir = os.path.expanduser(origdir)
+                    if not os.path.isabs(absdir):
+                        absdir = os.path.join(basedir, absdir)
+                    absdir = os.path.abspath(absdir)
+                    data['buildout'][name] = (absdir, src)
+
         self._annotated = copy.deepcopy(data)
         self._raw = _unannotate(data)
         self._data = {}
@@ -362,18 +393,22 @@ class Buildout(DictMixin):
                                                    self.update_versions_file)
 
         download_cache = options.get('download-cache')
+        extends_cache = options.get('extends-cache')
+        eggs_cache = options.get('eggs-directory')
+        for cache in [download_cache, extends_cache, eggs_cache]:
+            if cache:
+                cache = os.path.join(options['directory'], cache)
+                if not os.path.exists(cache):
+                    # Note: os.mkdir only creates the dir if the parent
+                    # exists. This is the way we want it.
+                    self._logger.info('Creating directory %r.', cache)
+                    os.mkdir(cache)
+
         if download_cache:
-            download_cache = os.path.join(options['directory'], download_cache)
-            if not os.path.isdir(download_cache):
-                raise zc.buildout.UserError(
-                    'The specified download cache:\n'
-                    '%r\n'
-                    "Doesn't exist.\n"
-                    % download_cache)
+            # Actually, we want to use a subdirectory in there called 'dist'.
             download_cache = os.path.join(download_cache, 'dist')
-            if not os.path.isdir(download_cache):
+            if not os.path.exists(download_cache):
                 os.mkdir(download_cache)
-
             zc.buildout.easy_install.download_cache(download_cache)
 
         if bool_option(options, 'install-from-cache'):
@@ -389,11 +424,6 @@ class Buildout(DictMixin):
         for name in _buildout_default_options:
             options[name]
 
-        # Do the same for extends-cache which is not among the defaults but
-        # wasn't recognized as having been used since it was used before
-        # tracking was turned on.
-        options.get('extends-cache')
-
         os.chdir(options['directory'])
 
     def _buildout_path(self, name):
@@ -996,9 +1026,6 @@ class Buildout(DictMixin):
                 path.append(self['buildout']['eggs-directory'])
             else:
                 dest = self['buildout']['eggs-directory']
-                if not os.path.exists(dest):
-                    self._logger.info('Creating directory %r.', dest)
-                    os.mkdir(dest)
 
             zc.buildout.easy_install.install(
                 specs, dest, path=path,

src/zc/buildout/buildout.txt

@@ -810,7 +810,7 @@ COMMAND_LINE_VALUE).
         DEFAULT_VALUE
     directory= /sample-buildout
         COMPUTED_VALUE
-    eggs-directory= eggs
+    eggs-directory= /sample-buildout/eggs
         DEFAULT_VALUE
     executable= ...
         DEFAULT_VALUE
@@ -2446,9 +2446,9 @@ provide alternate locations, and even names for these directories.
     ... ))
 
     >>> print_(system(buildout), end='')
+    Creating directory '/sample-alt/basket'.
     Creating directory '/sample-alt/scripts'.
     Creating directory '/sample-alt/work'.
-    Creating directory '/sample-alt/basket'.
     Creating directory '/sample-alt/developbasket'.
     Develop: '/sample-buildout/recipes'
     Uninstalling d4.
@@ -2482,9 +2482,9 @@ You can also specify an alternate buildout directory:
     ...    ))
 
     >>> print_(system(buildout), end='')
+    Creating directory '/sample-alt/eggs'.
     Creating directory '/sample-alt/bin'.
     Creating directory '/sample-alt/parts'.
-    Creating directory '/sample-alt/eggs'.
     Creating directory '/sample-alt/develop-eggs'.
     Develop: '/sample-buildout/recipes'
 
@@ -2772,9 +2772,9 @@ local buildout scripts.
     ...              +' -c'+os.path.join(sample_bootstrapped, 'setup.cfg')
     ...              +' init'), end='')
     Creating '/sample-bootstrapped/setup.cfg'.
+    Creating directory '/sample-bootstrapped/eggs'.
     Creating directory '/sample-bootstrapped/bin'.
     Creating directory '/sample-bootstrapped/parts'.
-    Creating directory '/sample-bootstrapped/eggs'.
     Creating directory '/sample-bootstrapped/develop-eggs'.
     Generated script '/sample-bootstrapped/bin/buildout'.
 
@@ -2833,9 +2833,9 @@ if there isn't a configuration file:
     >>> print_(system(buildout
     ...              +' -c'+os.path.join(sample_bootstrapped2, 'setup.cfg')
     ...              +' bootstrap'), end='')
+    Creating directory '/sample-bootstrapped2/eggs'.
     Creating directory '/sample-bootstrapped2/bin'.
     Creating directory '/sample-bootstrapped2/parts'.
-    Creating directory '/sample-bootstrapped2/eggs'.
     Creating directory '/sample-bootstrapped2/develop-eggs'.
     Generated script '/sample-bootstrapped2/bin/buildout'.
 

src/zc/buildout/downloadcache.txt

@@ -46,7 +46,7 @@ download:
     <a href="index/">index/</a><br>
     <a href="other-1.0-py2.4.egg">other-1.0-py2.4.egg</a><br>
     </body></html>
-    
+
 
 We'll enable logging on the link server so we can see what's going on:
 
@@ -87,7 +87,7 @@ If we remove the installed eggs from eggs directory and re-run the buildout:
     >>> for  f in os.listdir('eggs'):
     ...     if f.startswith('demo'):
     ...         remove('eggs', f)
-   
+
     >>> print_(system(buildout), end='')
     GET 200 /
     Updating eggs.
@@ -139,3 +139,79 @@ install-from-cache option set to true:
     Getting distribution for 'demoneeded'.
     Got demoneeded 1.1.
     Generated script '/sample-buildout/bin/demo'.
+
+Auto-creation of download cache directory
+-----------------------------------------
+
+With zc.buildout version 2.2.2 or higher the cache directory is automatically
+created, provided it is within an already existing directory::
+
+    >>> write('buildout.cfg',
+    ... '''
+    ... [buildout]
+    ... parts =
+    ... download-cache = %(cache)s/newdir
+    ... ''' % globals())
+
+    >>> print_(system(buildout), end='')
+    Creating directory '/cache/newdir'.
+    Uninstalling eggs.
+
+    >>> ls(cache)
+    d  dist
+    d  newdir
+
+Using relative paths
+--------------------
+
+You can use a relative path for ``download-cache`` (the same logic is applied to
+``eggs-directory`` and to ``extends-cache`` too) and in such case it is considered
+relative to the location of the configuration file that sets its value.
+
+As an example, we create a ``base.cfg`` configuration in a different directory::
+
+    >>> basedir = tmpdir('basecfg')
+    >>> write(basedir, 'base.cfg',
+    ... '''
+    ... [buildout]
+    ... download-cache = cache
+    ... ''')
+
+and a ``buildout.cfg`` that extends from there::
+
+    >>> write('buildout.cfg',
+    ... '''
+    ... [buildout]
+    ... extends = %(basedir)s/base.cfg
+    ... parts =
+    ... ''' % globals())
+
+    >>> dummy = system(buildout)
+    >>> ls(basedir)
+    -  base.cfg
+    d  cache
+
+Of course this cannot be used when the base configuration is not on the local
+filesystem because it wouldn't make any sense having a remote cache::
+
+    >>> server_data = tmpdir('server_data')
+    >>> server_url = start_server(server_data)
+    >>> cd(sample_buildout)
+
+    >>> write(server_data, 'base.cfg', """\
+    ... [buildout]
+    ... download-cache = cache
+    ... """)
+
+    >>> write('buildout.cfg',
+    ... '''
+    ... [buildout]
+    ... extends = %(server_url)s/base.cfg
+    ... parts =
+    ... ''' % globals())
+
+    >>> print_(system(buildout), end='') # doctest: +ELLIPSIS
+    While:
+      Initializing.
+    Error: Setting "download-cache" to a non absolute location ("cache") within a
+    remote configuration file...

src/zc/buildout/setup.txt

@@ -30,6 +30,7 @@ We'll use the buildout script from our sample buildout:
 
     >>> print_(system(buildout+' setup'), end='')
     ... # doctest: +NORMALIZE_WHITESPACE
+    Creating directory '/sample-buildout/test/eggs'.
     Error: The setup command requires the path to a setup script or
     directory containing a setup script, and its arguments.
 

src/zc/buildout/tests.py

@@ -647,9 +647,9 @@ bootstrapping.
     >>> os.chdir(d)
     >>> print_(system(os.path.join(sample_buildout, 'bin', 'buildout')
     ...              + ' bootstrap'), end='')
+    Creating directory '/sample-bootstrap/eggs'.
     Creating directory '/sample-bootstrap/bin'.
     Creating directory '/sample-bootstrap/parts'.
-    Creating directory '/sample-bootstrap/eggs'.
     Creating directory '/sample-bootstrap/develop-eggs'.
     Generated script '/sample-bootstrap/bin/buildout'.
     """
@@ -674,9 +674,9 @@ def bug_92891_bootstrap_crashes_with_egg_recipe_in_buildout_section():
     >>> os.chdir(d)
     >>> print_(system(os.path.join(sample_buildout, 'bin', 'buildout')
     ...              + ' bootstrap'), end='')
+    Creating directory '/sample-bootstrap/eggs'.
     Creating directory '/sample-bootstrap/bin'.
     Creating directory '/sample-bootstrap/parts'.
-    Creating directory '/sample-bootstrap/eggs'.
     Creating directory '/sample-bootstrap/develop-eggs'.
     Generated script '/sample-bootstrap/bin/buildout'.
 

src/zc/buildout/update.txt

@@ -175,9 +175,9 @@ directory:
 
     >>> cd(sample_buildout2)
     >>> print_(system(buildout), end='')
+    Creating directory '/sample_buildout2/eggs'.
     Creating directory '/sample_buildout2/bin'.
     Creating directory '/sample_buildout2/parts'.
-    Creating directory '/sample_buildout2/eggs'.
     Creating directory '/sample_buildout2/develop-eggs'.
     Getting distribution for 'zc.buildout>=1.99'.
     Got zc.buildout 99.99.

zc.recipe.egg_/CHANGES.txt

 Change History
 **************
 
+2.0.2 (unreleased)
+==================
+
+- Fixed: In ``zc.recipe.egg#custom`` recipe's ``rpath`` support, don't
+  assume path elements are buildout-relative if they start with one of the
+  "special" tokens (e.g., ``$ORIGIN``).  See:
+  https://github.com/buildout/buildout/issues/225.
+  [tseaver]
+
 2.0.1 (2013-09-05)
 ==================
 
@@ -129,7 +138,7 @@ Feature Changes
 
   - Cause develop eggs to be created after other parts.
 
-- The develop and build recipes now return the paths created, so that 
+- The develop and build recipes now return the paths created, so that
   created eggs or egg links are removed when a part is removed (or
   changed).
 

zc.recipe.egg_/setup.py

@@ -14,7 +14,7 @@
 """Setup for zc.recipe.egg package
 """
 
-version = '2.0.1'
+version = '2.0.2.dev0'
 
 import os
 from setuptools import setup, find_packages

zc.recipe.egg_/src/zc/recipe/egg/custom.py

@@ -130,7 +130,7 @@ class Develop(Base):
 
 def build_ext(buildout, options):
     result = {}
-    for be_option in ('include-dirs', 'library-dirs', 'rpath'):
+    for be_option in ('include-dirs', 'library-dirs'):
         value = options.get(be_option)
         if value is None:
             continue
@@ -145,6 +145,25 @@ def build_ext(buildout, options):
         result[be_option] = os.pathsep.join(value)
         options[be_option] = os.pathsep.join(value)
 
+    # rpath has special symbolic dirnames which must not be prefixed
+    # with the buildout dir.  See:
+    # http://man7.org/linux/man-pages/man8/ld.so.8.html
+    RPATH_SPECIAL = [
+        '$ORIGIN', '$LIB', '$PLATFORM', '${ORIGIN}', '${LIB}', '${PLATFORM}']
+    def _prefix_non_special(x):
+        x = x.strip()
+        for special in RPATH_SPECIAL:
+            if x.startswith(special):
+                return x
+        return os.path.join( buildout['buildout']['directory'], x)
+
+    value = options.get('rpath')
+    if value is not None:
+        values = [_prefix_non_special(v)
+                    for v in value.strip().split('\n') if v.strip()]
+        result['rpath'] = os.pathsep.join(values)
+        options['rpath'] = os.pathsep.join(values)
+
     swig = options.get('swig')
     if swig:
         options['swig'] = result['swig'] = os.path.join(