Merged master and re-ordered changelog

CHANGES.rst

 Change History
 **************
 
-Unreleased
-==========
+2.4.0 (unreleased)
+==================
+
+- Buildout no longer breaks on packages that contain a file with a non-ascii
+  filename. Fixes #89 and #148.
+  [reinout]
 
 - Undo breakage on Windows machines where ``sys.prefix`` can also be a
   ``site-packages`` directory:  don't remove it from ``sys.path``.  See
@@ -27,6 +31,21 @@ Unreleased
   quicker to develop.
   [reinout]
 
+- Note: zc.recipe.egg has also been updated to 2.0.2 together with this
+  zc.buildout release. Fixed: In ``zc.recipe.egg#custom`` recipe's ``rpath``
+  support, don't assume path elements are buildout-relative if they start with
+  one of the "special" tokens (e.g., ``$ORIGIN``).  See:
+  https://github.com/buildout/buildout/issues/225.
+  [tseaver]
+
+- ``download-cache``, ``eggs-directory`` and ``extends-cache`` are now
+  automatically created if their parent directory exists. Also they can be
+  relative directories (relative to the location of the buildout config file
+  that defines them). Also they can now be in the form ``~/subdir``, with the
+  usual convention that the ``~`` char means the home directory of the user
+  running buildout.
+  [lelit]
+
 - A new boostrap.py file is released (version 2015-07-01).
 
 - When bootstrapping, the ``develop-eggs/`` directory is first removed. This
@@ -34,11 +53,11 @@ Unreleased
   package collection mechanism.
   [reinout]
 
-- Bootstrap script now accepts ``--to-dir``. Setuptools is installed there. If
-  already available there, it is reused. This can be used to bootstrap
-  buildout without internet access. Similarly, a local ``ez_setup.py`` is used
-  when available instead of it being downloaded. You need setuptools 14.0 or
-  higher for this functionality.
+- The bootstrap script now accepts ``--to-dir``. Setuptools is installed
+  there. If already available there, it is reused. This can be used to
+  bootstrap buildout without internet access. Similarly, a local
+  ``ez_setup.py`` is used when available instead of it being downloaded. You
+  need setuptools 14.0 or higher for this functionality.
   [lrowe]
 
 - The bootstrap script now uses ``--buildout-version`` instead of

DEVELOPERS.txt

@@ -7,6 +7,10 @@ When you're developing buildout itself, you need to know two things:
   will find your already-installed setuptools, leading to test differences
   when setuptools' presence is explicitly tested.
 
+- Also the presence of ``~/.buildout/default.cfg`` may interfere with the
+  tests so you may want to temporarily rename it so that it does not get in
+  the way.
+
 - Don't bootstrap with ``python bootstrap/bootstrap.py`` but with ``python
   dev.py``.
 

setup.py

@@ -12,7 +12,7 @@
 #
 ##############################################################################
 name = "zc.buildout"
-version = "2.3.1"
+version = "2.4.0.dev0"
 
 import os
 from setuptools import setup

src/zc/buildout/bootstrap.txt

@@ -24,9 +24,9 @@ Make sure the bootstrap script actually works::
     ...     zc.buildout.easy_install._safe_arg(sys.executable)+' '+
     ...     'bootstrap.py')); print_('X') # doctest: +ELLIPSIS
     X...
+    Creating directory '/sample/eggs'.
     Creating directory '/sample/bin'.
     Creating directory '/sample/parts'.
-    Creating directory '/sample/eggs'.
     Creating directory '/sample/develop-eggs'.
     Generated script '/sample/bin/buildout'.
     ...

src/zc/buildout/bootstrap_cl_settings.test

@@ -27,9 +27,9 @@ Some people pass buildout settings to bootstrap.
     ...     ' -c'+join('buildout', 'buildout.cfg')
     ...    )); print_('X') # doctest: +ELLIPSIS
     X...
+    Creating directory '/top/eggs'.
     Creating directory '/top/bin'.
     Creating directory '/top/parts'.
-    Creating directory '/top/eggs'.
     Creating directory '/top/develop-eggs'.
     Generated script '/top/bin/buildout'.
     ...
@@ -52,9 +52,9 @@ They might do it with init, but no worries:
     ...    )); print_('X') # doctest: +ELLIPSIS
     X...
     Creating '/top/buildout/buildout.cfg'.
+    Creating directory '/top/eggs'.
     Creating directory '/top/bin'.
     Creating directory '/top/parts'.
-    Creating directory '/top/eggs'.
     Creating directory '/top/develop-eggs'.
     Generated script '/top/bin/buildout'.
     ...

src/zc/buildout/buildout.py

@@ -44,6 +44,19 @@ import tempfile
 import zc.buildout
 import zc.buildout.download
 
+PY3 = sys.version_info[0] == 3
+if PY3:
+    text_type = str
+else:
+    text_type = unicode
+
+def fs_to_text(fs_name):
+    """Return filesystem name always as unicode(2)/str(3)."""
+    if not isinstance(fs_name, text_type):
+        fs_name = fs_name.decode(sys.getfilesystemencoding(),
+                                 'surrogateescape')
+    return fs_name
+
 def _print_options(sep=' ', end='\n', file=None):
     return sep, end, file
 
@@ -253,6 +266,37 @@ class Buildout(DictMixin):
                  if k not in versions
                  ))
 
+        # Absolutize some particular directory, handling also the ~/foo form,
+        # and considering the location of the configuration file that generated
+        # the setting as the base path, falling back to the main configuration
+        # file location
+        for name in ('download-cache', 'eggs-directory', 'extends-cache'):
+            if name in data['buildout']:
+                origdir, src = data['buildout'][name]
+                if '${' in origdir:
+                    continue
+                if not os.path.isabs(origdir):
+                    if src in ('DEFAULT_VALUE',
+                               'COMPUTED_VALUE',
+                               'COMMAND_LINE_VALUE'):
+                        if 'directory' in data['buildout']:
+                            basedir = data['buildout']['directory'][0]
+                        else:
+                            basedir = self._buildout_dir
+                    else:
+                        if _isurl(src):
+                            raise zc.buildout.UserError(
+                                'Setting "%s" to a non absolute location ("%s") '
+                                'within a\n'
+                                'remote configuration file ("%s") is ambiguous.' % (
+                                    name, origdir, src))
+                        basedir = os.path.dirname(src)
+                    absdir = os.path.expanduser(origdir)
+                    if not os.path.isabs(absdir):
+                        absdir = os.path.join(basedir, absdir)
+                    absdir = os.path.abspath(absdir)
+                    data['buildout'][name] = (absdir, src)
+
         self._annotated = copy.deepcopy(data)
         self._raw = _unannotate(data)
         self._data = {}
@@ -349,18 +393,22 @@ class Buildout(DictMixin):
                                                    self.update_versions_file)
 
         download_cache = options.get('download-cache')
+        extends_cache = options.get('extends-cache')
+        eggs_cache = options.get('eggs-directory')
+        for cache in [download_cache, extends_cache, eggs_cache]:
+            if cache:
+                cache = os.path.join(options['directory'], cache)
+                if not os.path.exists(cache):
+                    # Note: os.mkdir only creates the dir if the parent
+                    # exists. This is the way we want it.
+                    self._logger.info('Creating directory %r.', cache)
+                    os.mkdir(cache)
+
         if download_cache:
-            download_cache = os.path.join(options['directory'], download_cache)
-            if not os.path.isdir(download_cache):
-                raise zc.buildout.UserError(
-                    'The specified download cache:\n'
-                    '%r\n'
-                    "Doesn't exist.\n"
-                    % download_cache)
+            # Actually, we want to use a subdirectory in there called 'dist'.
             download_cache = os.path.join(download_cache, 'dist')
-            if not os.path.isdir(download_cache):
+            if not os.path.exists(download_cache):
                 os.mkdir(download_cache)
-
             zc.buildout.easy_install.download_cache(download_cache)
 
         if bool_option(options, 'install-from-cache'):
@@ -376,11 +424,6 @@ class Buildout(DictMixin):
         for name in _buildout_default_options:
             options[name]
 
-        # Do the same for extends-cache which is not among the defaults but
-        # wasn't recognized as having been used since it was used before
-        # tracking was turned on.
-        options.get('extends-cache')
-
         os.chdir(options['directory'])
 
     def _buildout_path(self, name):
@@ -989,9 +1032,6 @@ class Buildout(DictMixin):
                 path.append(self['buildout']['eggs-directory'])
             else:
                 dest = self['buildout']['eggs-directory']
-                if not os.path.exists(dest):
-                    self._logger.info('Creating directory %r.', dest)
-                    os.mkdir(dest)
 
             zc.buildout.easy_install.install(
                 specs, dest, path=path,
@@ -1598,18 +1638,23 @@ def _open(base, filename, seen, dl_options, override, downloaded):
 ignore_directories = '.svn', 'CVS', '__pycache__'
 _dir_hashes = {}
 def _dir_hash(dir):
+    dir = fs_to_text(dir)
+    # ^^^ fs_to_text ensures unicode, needed for os.walk() on python2 to work
+    # well with non-ascii filenames.
     dir_hash = _dir_hashes.get(dir, None)
     if dir_hash is not None:
         return dir_hash
     hash = md5()
     for (dirpath, dirnames, filenames) in os.walk(dir):
+        dirnames[:] = [fs_to_text(dirname) for dirname in dirnames]
+        filenames[:] = [fs_to_text(filename) for filename in filenames]
         dirnames[:] = sorted(n for n in dirnames if n not in ignore_directories)
         filenames[:] = sorted(f for f in filenames
                               if (not (f.endswith('pyc') or f.endswith('pyo'))
                                   and os.path.exists(os.path.join(dirpath, f)))
-                              )
-        hash.update(' '.join(dirnames).encode())
-        hash.update(' '.join(filenames).encode())
+                          )
+        hash.update(' '.join(dirnames).encode('utf-8'))
+        hash.update(' '.join(filenames).encode('utf-8'))
         for name in filenames:
             path = os.path.join(dirpath, name)
             if name == 'entry_points.txt':

src/zc/buildout/buildout.txt

@@ -810,7 +810,7 @@ COMMAND_LINE_VALUE).
         DEFAULT_VALUE
     directory= /sample-buildout
         COMPUTED_VALUE
-    eggs-directory= eggs
+    eggs-directory= /sample-buildout/eggs
         DEFAULT_VALUE
     executable= ...
         DEFAULT_VALUE
@@ -2446,9 +2446,9 @@ provide alternate locations, and even names for these directories.
     ... ))
 
     >>> print_(system(buildout), end='')
+    Creating directory '/sample-alt/basket'.
     Creating directory '/sample-alt/scripts'.
     Creating directory '/sample-alt/work'.
-    Creating directory '/sample-alt/basket'.
     Creating directory '/sample-alt/developbasket'.
     Develop: '/sample-buildout/recipes'
     Uninstalling d4.
@@ -2482,9 +2482,9 @@ You can also specify an alternate buildout directory:
     ...    ))
 
     >>> print_(system(buildout), end='')
+    Creating directory '/sample-alt/eggs'.
     Creating directory '/sample-alt/bin'.
     Creating directory '/sample-alt/parts'.
-    Creating directory '/sample-alt/eggs'.
     Creating directory '/sample-alt/develop-eggs'.
     Develop: '/sample-buildout/recipes'
 
@@ -2772,9 +2772,9 @@ local buildout scripts.
     ...              +' -c'+os.path.join(sample_bootstrapped, 'setup.cfg')
     ...              +' init'), end='')
     Creating '/sample-bootstrapped/setup.cfg'.
+    Creating directory '/sample-bootstrapped/eggs'.
     Creating directory '/sample-bootstrapped/bin'.
     Creating directory '/sample-bootstrapped/parts'.
-    Creating directory '/sample-bootstrapped/eggs'.
     Creating directory '/sample-bootstrapped/develop-eggs'.
     Generated script '/sample-bootstrapped/bin/buildout'.
 
@@ -2833,9 +2833,9 @@ if there isn't a configuration file:
     >>> print_(system(buildout
     ...              +' -c'+os.path.join(sample_bootstrapped2, 'setup.cfg')
     ...              +' bootstrap'), end='')
+    Creating directory '/sample-bootstrapped2/eggs'.
     Creating directory '/sample-bootstrapped2/bin'.
     Creating directory '/sample-bootstrapped2/parts'.
-    Creating directory '/sample-bootstrapped2/eggs'.
     Creating directory '/sample-bootstrapped2/develop-eggs'.
     Generated script '/sample-bootstrapped2/bin/buildout'.
 

src/zc/buildout/downloadcache.txt

@@ -46,7 +46,7 @@ download:
     <a href="index/">index/</a><br>
     <a href="other-1.0-py2.4.egg">other-1.0-py2.4.egg</a><br>
     </body></html>
-    
+
 
 We'll enable logging on the link server so we can see what's going on:
 
@@ -87,7 +87,7 @@ If we remove the installed eggs from eggs directory and re-run the buildout:
     >>> for  f in os.listdir('eggs'):
     ...     if f.startswith('demo'):
     ...         remove('eggs', f)
-   
+
     >>> print_(system(buildout), end='')
     GET 200 /
     Updating eggs.
@@ -139,3 +139,79 @@ install-from-cache option set to true:
     Getting distribution for 'demoneeded'.
     Got demoneeded 1.1.
     Generated script '/sample-buildout/bin/demo'.
+
+Auto-creation of download cache directory
+-----------------------------------------
+
+With zc.buildout version 2.2.2 or higher the cache directory is automatically
+created, provided it is within an already existing directory::
+
+    >>> write('buildout.cfg',
+    ... '''
+    ... [buildout]
+    ... parts =
+    ... download-cache = %(cache)s/newdir
+    ... ''' % globals())
+
+    >>> print_(system(buildout), end='')
+    Creating directory '/cache/newdir'.
+    Uninstalling eggs.
+
+    >>> ls(cache)
+    d  dist
+    d  newdir
+
+Using relative paths
+--------------------
+
+You can use a relative path for ``download-cache`` (the same logic is applied to
+``eggs-directory`` and to ``extends-cache`` too) and in such case it is considered
+relative to the location of the configuration file that sets its value.
+
+As an example, we create a ``base.cfg`` configuration in a different directory::
+
+    >>> basedir = tmpdir('basecfg')
+    >>> write(basedir, 'base.cfg',
+    ... '''
+    ... [buildout]
+    ... download-cache = cache
+    ... ''')
+
+and a ``buildout.cfg`` that extends from there::
+
+    >>> write('buildout.cfg',
+    ... '''
+    ... [buildout]
+    ... extends = %(basedir)s/base.cfg
+    ... parts =
+    ... ''' % globals())
+
+    >>> dummy = system(buildout)
+    >>> ls(basedir)
+    -  base.cfg
+    d  cache
+
+Of course this cannot be used when the base configuration is not on the local
+filesystem because it wouldn't make any sense having a remote cache::
+
+    >>> server_data = tmpdir('server_data')
+    >>> server_url = start_server(server_data)
+    >>> cd(sample_buildout)
+
+    >>> write(server_data, 'base.cfg', """\
+    ... [buildout]
+    ... download-cache = cache
+    ... """)
+
+    >>> write('buildout.cfg',
+    ... '''
+    ... [buildout]
+    ... extends = %(server_url)s/base.cfg
+    ... parts =
+    ... ''' % globals())
+
+    >>> print_(system(buildout), end='') # doctest: +ELLIPSIS
+    While:
+      Initializing.
+    Error: Setting "download-cache" to a non absolute location ("cache") within a
+    remote configuration file...

src/zc/buildout/setup.txt

@@ -30,6 +30,7 @@ We'll use the buildout script from our sample buildout:
 
     >>> print_(system(buildout+' setup'), end='')
     ... # doctest: +NORMALIZE_WHITESPACE
+    Creating directory '/sample-buildout/test/eggs'.
     Error: The setup command requires the path to a setup script or
     directory containing a setup script, and its arguments.
 

src/zc/buildout/tests.py

+# -*- coding: utf-8 -*-
 ##############################################################################
 #
 # Copyright (c) 2004-2009 Zope Foundation and Contributors.
@@ -646,9 +647,9 @@ bootstrapping.
     >>> os.chdir(d)
     >>> print_(system(os.path.join(sample_buildout, 'bin', 'buildout')
     ...              + ' bootstrap'), end='')
+    Creating directory '/sample-bootstrap/eggs'.
     Creating directory '/sample-bootstrap/bin'.
     Creating directory '/sample-bootstrap/parts'.
-    Creating directory '/sample-bootstrap/eggs'.
     Creating directory '/sample-bootstrap/develop-eggs'.
     Generated script '/sample-bootstrap/bin/buildout'.
     """
@@ -673,9 +674,9 @@ def bug_92891_bootstrap_crashes_with_egg_recipe_in_buildout_section():
     >>> os.chdir(d)
     >>> print_(system(os.path.join(sample_buildout, 'bin', 'buildout')
     ...              + ' bootstrap'), end='')
+    Creating directory '/sample-bootstrap/eggs'.
     Creating directory '/sample-bootstrap/bin'.
     Creating directory '/sample-bootstrap/parts'.
-    Creating directory '/sample-bootstrap/eggs'.
     Creating directory '/sample-bootstrap/develop-eggs'.
     Generated script '/sample-bootstrap/bin/buildout'.
 
@@ -1156,6 +1157,20 @@ because of the missing target file.
 
     """
 
+def unicode_filename_doesnt_break_hash():
+    """
+Buildout's _dir_hash() used to break on non-ascii filenames on python 2.
+
+    >>> mkdir('héhé')
+    >>> write('héhé', 'héhé.py',
+    ... '''
+    ... print('Example filename from pyramid tests')
+    ... ''')
+    >>> from zc.buildout.buildout import _dir_hash
+    >>> dont_care = _dir_hash('héhé')
+
+    """
+
 def o_option_sets_offline():
     """
     >>> print_(system(join(sample_buildout, 'bin', 'buildout')+' -vvo'), end='')

src/zc/buildout/update.txt

@@ -175,9 +175,9 @@ directory:
 
     >>> cd(sample_buildout2)
     >>> print_(system(buildout), end='')
+    Creating directory '/sample_buildout2/eggs'.
     Creating directory '/sample_buildout2/bin'.
     Creating directory '/sample_buildout2/parts'.
-    Creating directory '/sample_buildout2/eggs'.
     Creating directory '/sample_buildout2/develop-eggs'.
     Getting distribution for 'zc.buildout>=1.99'.
     Got zc.buildout 99.99.

zc.recipe.egg_/CHANGES.txt

 Change History
 **************
 
+2.0.2 (unreleased)
+==================
+
+- Fixed: In ``zc.recipe.egg#custom`` recipe's ``rpath`` support, don't
+  assume path elements are buildout-relative if they start with one of the
+  "special" tokens (e.g., ``$ORIGIN``).  See:
+  https://github.com/buildout/buildout/issues/225.
+  [tseaver]
+
 2.0.1 (2013-09-05)
 ==================
 
@@ -129,7 +138,7 @@ Feature Changes
 
   - Cause develop eggs to be created after other parts.
 
-- The develop and build recipes now return the paths created, so that 
+- The develop and build recipes now return the paths created, so that
   created eggs or egg links are removed when a part is removed (or
   changed).
 

zc.recipe.egg_/setup.py

@@ -14,7 +14,7 @@
 """Setup for zc.recipe.egg package
 """
 
-version = '2.0.1'
+version = '2.0.2.dev0'
 
 import os
 from setuptools import setup, find_packages

zc.recipe.egg_/src/zc/recipe/egg/custom.py

@@ -130,7 +130,7 @@ class Develop(Base):
 
 def build_ext(buildout, options):
     result = {}
-    for be_option in ('include-dirs', 'library-dirs', 'rpath'):
+    for be_option in ('include-dirs', 'library-dirs'):
         value = options.get(be_option)
         if value is None:
             continue
@@ -145,6 +145,25 @@ def build_ext(buildout, options):
         result[be_option] = os.pathsep.join(value)
         options[be_option] = os.pathsep.join(value)
 
+    # rpath has special symbolic dirnames which must not be prefixed
+    # with the buildout dir.  See:
+    # http://man7.org/linux/man-pages/man8/ld.so.8.html
+    RPATH_SPECIAL = [
+        '$ORIGIN', '$LIB', '$PLATFORM', '${ORIGIN}', '${LIB}', '${PLATFORM}']
+    def _prefix_non_special(x):
+        x = x.strip()
+        for special in RPATH_SPECIAL:
+            if x.startswith(special):
+                return x
+        return os.path.join( buildout['buildout']['directory'], x)
+
+    value = options.get('rpath')
+    if value is not None:
+        values = [_prefix_non_special(v)
+                    for v in value.strip().split('\n') if v.strip()]
+        result['rpath'] = os.pathsep.join(values)
+        options['rpath'] = os.pathsep.join(values)
+
     swig = options.get('swig')
     if swig:
         options['swig'] = result['swig'] = os.path.join(