Merge branch 'upstream'

CHANGES.txt

 Change History
 **************
 
-1.5.3 (unreleased)
+1.6.0 (unreleased)
 ==================
 
+- The buildout init command now accepts distribution requirements and
+  paths to set up a custom interpreter part that has the distributions
+  or parts in the path. For example::
+
+     python bootstrap.py init BeautifulSoup
+
 - Introduce a cache for the expensive `buildout._dir_hash` function.
 
 - Remove duplicate path from script's sys.path setup.
@@ -27,6 +33,9 @@ Bugs fixed:
 - Removed any traces of the implementation of ``extended-by``. Raise a
   UserError if the option is encountered instead of ignoring it, though.
 
+- https://bugs.launchpad.net/bugs/697913 : Buildout doesn't honor exit code
+  from scripts. Fixed.
+
 1.5.2 (2010-10-11)
 ==================
 

bootstrap/bootstrap.py

@@ -18,7 +18,7 @@ The script accepts buildout command-line options, so you can
 use the -c option to specify an alternate configuration file.
 """
 
-import os, shutil, sys, tempfile, textwrap, urllib, urllib2, subprocess
+import os, shutil, sys, tempfile, urllib, urllib2, subprocess
 from optparse import OptionParser
 
 if sys.platform == 'win32':
@@ -57,13 +57,13 @@ if not has_broken_dash_S and 'site' in sys.modules:
 # out any namespace packages from site-packages that might have been
 # loaded by .pth files.
 clean_path = sys.path[:]
-import site
+import site  # imported because of its side effects
 sys.path[:] = clean_path
 for k, v in sys.modules.items():
     if k in ('setuptools', 'pkg_resources') or (
         hasattr(v, '__path__') and
-        len(v.__path__)==1 and
-        not os.path.exists(os.path.join(v.__path__[0],'__init__.py'))):
+        len(v.__path__) == 1 and
+        not os.path.exists(os.path.join(v.__path__[0], '__init__.py'))):
         # This is a namespace package.  Remove it.
         sys.modules.pop(k)
 
@@ -72,6 +72,7 @@ is_jython = sys.platform.startswith('java')
 setuptools_source = 'http://peak.telecommunity.com/dist/ez_setup.py'
 distribute_source = 'http://python-distribute.org/distribute_setup.py'
 
+
 # parsing arguments
 def normalize_to_url(option, opt_str, value, parser):
     if value:
@@ -111,7 +112,7 @@ parser.add_option("--setup-source", action="callback", dest="setup_source",
                   help=("Specify a URL or file location for the setup file. "
                         "If you use Setuptools, this will default to " +
                         setuptools_source + "; if you use Distribute, this "
-                        "will default to " + distribute_source +"."))
+                        "will default to " + distribute_source + "."))
 parser.add_option("--download-base", action="callback", dest="download_base",
                   callback=normalize_to_url, nargs=1, type="string",
                   help=("Specify a URL or directory for downloading "
@@ -218,6 +219,7 @@ if version is None and not options.accept_buildout_test_releases:
     # Figure out the most recent final version of zc.buildout.
     import setuptools.package_index
     _final_parts = '*final-', '*final'
+
     def _final_version(parsed_version):
         for part in parsed_version:
             if (part[:1] == '*') and (part not in _final_parts):

src/zc/buildout/bootstrap.txt

@@ -73,7 +73,7 @@ customized site.py.
     import zc.buildout.buildout
     <BLANKLINE>
     if __name__ == '__main__':
-        zc.buildout.buildout.main()
+        sys.exit(zc.buildout.buildout.main())
     <BLANKLINE>
 
 The bootstrap process prefers final versions of zc.buildout, so it has

src/zc/buildout/buildout.py

@@ -167,7 +167,8 @@ _buildout_default_options = _annotate_section({
 class Buildout(UserDict.DictMixin):
 
     def __init__(self, config_file, cloptions,
-                 user_defaults=True, windows_restart=False, command=None):
+                 user_defaults=True, windows_restart=False,
+                 command=None, args=()):
 
         __doing__ = 'Initializing.'
 
@@ -182,8 +183,7 @@ class Buildout(UserDict.DictMixin):
             base = os.path.dirname(config_file)
             if not os.path.exists(config_file):
                 if command == 'init':
-                    print 'Creating %r.' % config_file
-                    open(config_file, 'w').write('[buildout]\nparts = \n')
+                    self._init_config(config_file, args)
                 elif command == 'setup':
                     # Sigh. This model of a buildout instance
                     # with methods is breaking down. :(
@@ -192,6 +192,9 @@ class Buildout(UserDict.DictMixin):
                 else:
                     raise zc.buildout.UserError(
                         "Couldn't open %s" % config_file)
+            elif command == 'init':
+                raise zc.buildout.UserError(
+                    "%r already exists." % config_file)
 
             if config_file:
                 data['buildout']['directory'] = (os.path.dirname(config_file),
@@ -214,7 +217,8 @@ class Buildout(UserDict.DictMixin):
                                        '.buildout', 'default.cfg')
             if os.path.exists(user_config):
                 _update(data, _open(os.path.dirname(user_config), user_config,
-                                    [], data['buildout'].copy(), override, set()))
+                                    [], data['buildout'].copy(), override,
+                                    set()))
 
         # load configuration files
         if config_file:
@@ -515,9 +519,10 @@ class Buildout(UserDict.DictMixin):
         relative_paths = options.get('relative-paths', 'false')
         if relative_paths == 'true':
             relative_paths = options['directory']
-        else:
-            assert relative_paths == 'false'
+        elif relative_paths == 'false':
             relative_paths = ''
+        else:
+            raise zc.buildout.UserError("relative_paths must be true or false")
         if (self.accept_buildout_test_releases and
             self._annotated['buildout']['accept-buildout-test-releases'][1] ==
             'COMMAND_LINE_VALUE'):
@@ -534,7 +539,38 @@ class Buildout(UserDict.DictMixin):
             exec_sitecustomize=self.exec_sitecustomize,
             )
 
-    init = bootstrap
+    def _init_config(self, config_file, args):
+        print 'Creating %r.' % config_file
+        f = open(config_file, 'w')
+        sep = re.compile(r'[\\/]')
+        if args:
+            eggs = '\n  '.join(a for a in args if not sep.search(a))
+            paths = '\n  '.join(
+                sep.sub(os.path.sep, a) for a in args if sep.search(a))
+            f.write('[buildout]\n'
+                    'parts = py\n'
+                    '\n'
+                    '[py]\n'
+                    'recipe = zc.recipe.egg\n'
+                    'interpreter = py\n'
+                    'eggs =\n'
+                    )
+            if eggs:
+                f.write('  %s\n' % eggs)
+            if paths:
+                f.write('extra-paths =\n  %s\n' % paths)
+                for p in [a for a in args if sep.search(a)]:
+                    if not os.path.exists(p):
+                        os.mkdir(p)
+
+        else:
+            f.write('[buildout]\nparts =\n')
+        f.close()
+
+    def init(self, args):
+        self.bootstrap(())
+        if args:
+            self.install(())
 
     def install(self, install_args):
         __doing__ = 'Installing.'
@@ -725,7 +761,7 @@ class Buildout(UserDict.DictMixin):
                 self._save_installed_options(installed_part_options)
                 installed_exists = True
             else:
-                assert installed_exists
+                assert installed_exists  # nothing to tell the user here
                 self._update_installed(parts=' '.join(installed_parts))
 
         if installed_develop_eggs:
@@ -1056,9 +1092,10 @@ class Buildout(UserDict.DictMixin):
         relative_paths = options.get('relative-paths', 'false')
         if relative_paths == 'true':
             relative_paths = options['directory']
-        else:
-            assert relative_paths == 'false'
+        elif relative_paths == 'false':
             relative_paths = ''
+        else:
+            raise zc.buildout.UserError("relative_paths must be true or false")
         zc.buildout.easy_install.sitepackage_safe_scripts(
             options['bin-directory'], ws, options['executable'], partsdir,
             reqs=['zc.buildout'], relative_paths=relative_paths,
@@ -1971,7 +2008,8 @@ def main(args=None):
     try:
         try:
             buildout = Buildout(config_file, options,
-                                user_defaults, windows_restart, command)
+                                user_defaults, windows_restart,
+                                command, args)
             getattr(buildout, command)(args)
         except Exception, v:
             _doing()

src/zc/buildout/buildout.txt

@@ -2527,7 +2527,15 @@ local buildout scripts.
     Creating directory '/sample-bootstrapped/develop-eggs'.
     Generated script '/sample-bootstrapped/bin/buildout'.
 
-Note that a basic setup.cfg was created for us.
+Note that a basic setup.cfg was created for us.  This is because we
+provided an 'init' argument.  By default, the generated
+``setup.cfg`` is as minimal as it could be:
+
+    >>> cat(sample_bootstrapped, 'setup.cfg')
+    [buildout]
+    parts =
+
+We also get other buildout artifacts:
 
     >>> ls(sample_bootstrapped)
     d  bin
@@ -2593,7 +2601,7 @@ If relative-paths is ``true``, the buildout script uses relative paths.
     import zc.buildout.buildout
     <BLANKLINE>
     if __name__ == '__main__':
-        zc.buildout.buildout.main()
+        sys.exit(zc.buildout.buildout.main())
     <BLANKLINE>
 
 
@@ -2629,6 +2637,93 @@ if there isn't a configuration file:
     Creating directory '/sample-bootstrapped2/develop-eggs'.
     Generated script '/sample-bootstrapped2/bin/buildout'.
 
+Similarly, if there is a configuration file and we use the init
+command, we'll get an error that the configuration file already
+exists:
+
+    >>> print system(buildout
+    ...              +' -c'+os.path.join(sample_bootstrapped, 'setup.cfg')
+    ...              +' init'),
+    While:
+      Initializing.
+    Error: '/sample-bootstrapped/setup.cfg' already exists.
+
+Initial eggs
+------------
+
+When using the ``init`` command, you can specify distribution requirements
+or paths to use:
+
+    >>> cd(sample_bootstrapped)
+    >>> remove('setup.cfg')
+    >>> print system(buildout + ' -csetup.cfg init demo other ./src'),
+    Creating '/sample-bootstrapped/setup.cfg'.
+    Generated script '/sample-bootstrapped/bin/buildout'.
+    Getting distribution for 'zc.recipe.egg'.
+    Got zc.recipe.egg 1.3.3dev.
+    Installing py.
+    Getting distribution for 'demo'.
+    Got demo 0.4c1.
+    Getting distribution for 'other'.
+    Got other 1.0.
+    Getting distribution for 'demoneeded'.
+    Got demoneeded 1.2c1.
+    Generated script '/sample-bootstrapped/bin/demo'.
+    Generated interpreter '/sample-bootstrapped/bin/py'.
+
+This causes a ``py`` part to be included that sets up a custom python
+interpreter with the given requirements or paths:
+
+    >>> cat('setup.cfg')
+    [buildout]
+    parts = py
+    <BLANKLINE>
+    [py]
+    recipe = zc.recipe.egg
+    interpreter = py
+    eggs =
+      demo
+      other
+    extra-paths =
+      ./src
+
+Passing requirements or paths causes the the builout to be run as part
+of initialization.  In the example above, we got a number of
+distributions installed and 2 scripts generated.  The first, ``demo``,
+was defined by the ``demo`` project. The second, ``py`` was defined by
+the generated configuration.  It's a "custom interpreter" that behaves
+like a standard Python interpeter, except that includes the specified
+eggs and extra paths in it's Python path.
+
+We specified a source directory that didn't exist. Buildout created it
+for us:
+
+    >>> ls('.')
+    -  .installed.cfg
+    d  bin
+    d  develop-eggs
+    d  eggs
+    d  parts
+    -  setup.cfg
+    d  src
+
+    >>> uncd()
+
+.. Make sure it works if the dir is already there:
+
+    >>> cd(sample_bootstrapped)
+    >>> _ = system(buildout + ' -csetup.cfg buildout:parts=')
+    >>> remove('setup.cfg')
+    >>> print system(buildout + ' -csetup.cfg init demo other ./src'),
+    Creating '/sample-bootstrapped/setup.cfg'.
+    Installing py.
+    Generated script '/sample-bootstrapped/bin/demo'.
+    Generated interpreter '/sample-bootstrapped/bin/py'.
+
+.. cleanup
+
+    >>> _ = system(buildout + ' -csetup.cfg buildout:parts=')
+    >>> uncd()
 
 Newest and Offline Modes
 ------------------------

src/zc/buildout/easy_install.py

@@ -1568,7 +1568,7 @@ sys.path[0:0] = [
 import %(module_name)s
 
 if __name__ == '__main__':
-    %(module_name)s.%(attrs)s(%(arguments)s)
+    sys.exit(%(module_name)s.%(attrs)s(%(arguments)s))
 '''
 
 # These are used only by the older ``scripts`` function.

src/zc/buildout/easy_install.txt

@@ -677,7 +677,7 @@ The demo script run the entry point defined in the demo egg:
     import eggrecipedemo
     <BLANKLINE>
     if __name__ == '__main__':
-        eggrecipedemo.main()
+        sys.exit(eggrecipedemo.main())
 
 Some things to note:
 
@@ -714,7 +714,7 @@ rather than passing a requirement:
     import eggrecipedemo
     <BLANKLINE>
     if __name__ == '__main__':
-        eggrecipedemo.main()
+        sys.exit(eggrecipedemo.main())
 
 Passing entry-point information directly is handy when using eggs (or
 distributions) that don't declare their entry points, such as
@@ -845,7 +845,7 @@ to be included in the a generated script:
     import eggrecipedemo
     <BLANKLINE>
     if __name__ == '__main__':
-        eggrecipedemo.main()
+        sys.exit(eggrecipedemo.main())
 
 The ``scripts`` function: Providing script arguments
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -869,7 +869,7 @@ parentheses in the call:
     import eggrecipedemo
     <BLANKLINE>
     if __name__ == '__main__':
-        eggrecipedemo.main(1, 2)
+        sys.exit(eggrecipedemo.main(1, 2))
 
 The ``scripts`` function: Passing initialization code
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -895,7 +895,7 @@ You can also pass script initialization code:
     import eggrecipedemo
     <BLANKLINE>
     if __name__ == '__main__':
-        eggrecipedemo.main(1, 2)
+        sys.exit(eggrecipedemo.main(1, 2))
 
 The ``scripts`` function: Relative paths
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -941,7 +941,7 @@ to pass a common base directory of the scripts and eggs:
     import eggrecipedemo
     <BLANKLINE>
     if __name__ == '__main__':
-        eggrecipedemo.main()
+        sys.exit(eggrecipedemo.main())
 
 Note that the extra path we specified that was outside the directory
 passed as relative_paths wasn't converted to a relative path.
@@ -1518,7 +1518,7 @@ The demo script runs the entry point defined in the demo egg:
     import eggrecipedemo
     <BLANKLINE>
     if __name__ == '__main__':
-        eggrecipedemo.main()
+        sys.exit(eggrecipedemo.main())
 
     >>> demo_call = join(interpreter_bin_dir, 'demo')
     >>> if sys.platform == 'win32':
@@ -1565,7 +1565,7 @@ Let's see ``script_arguments`` and ``script_initialization`` in action.
     import eggrecipedemo
     <BLANKLINE>
     if __name__ == '__main__':
-        eggrecipedemo.main(1, 2)
+        sys.exit(eggrecipedemo.main(1, 2))
 
 Handling custom build options for extensions provided in source distributions
 -----------------------------------------------------------------------------

src/zc/buildout/testing.py

@@ -151,14 +151,18 @@ def _runsetup(setup, executable, *args):
     if executable == sys.executable:
         env['PYTHONPATH'] = setuptools_location
     # else pass an executable that has setuptools! See testselectingpython.py.
-    args.append(env)
 
     here = os.getcwd()
     try:
         os.chdir(d)
-        os.spawnle(os.P_WAIT, executable,
-                   zc.buildout.easy_install._safe_arg(executable),
-                   setup, *args)
+        p = subprocess.Popen(
+            [zc.buildout.easy_install._safe_arg(executable), setup] + args,
+            stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
+            close_fds=True, env=env)
+        out = p.stdout.read()
+        if p.wait():
+            print out
+
         if os.path.exists('build'):
             rmtree('build')
     finally:
@@ -437,6 +441,15 @@ def buildoutSetUp(test):
         return (
             os.path.join(buildout, 'bin', 'py'), site_packages_dir)
 
+    cdpaths = []
+    def cd(*path):
+        path = os.path.join(*path)
+        cdpaths.append(os.path.abspath(os.getcwd()))
+        os.chdir(path)
+
+    def uncd():
+        os.chdir(cdpaths.pop())
+
     test.globs.update(dict(
         sample_buildout = sample,
         ls = ls,
@@ -449,7 +462,7 @@ def buildoutSetUp(test):
         system = system,
         call_py = call_py,
         get = get,
-        cd = (lambda *path: os.chdir(os.path.join(*path))),
+        cd = cd, uncd = uncd,
         join = os.path.join,
         sdist = sdist,
         bdist_egg = bdist_egg,

src/zc/buildout/testing.txt

@@ -79,6 +79,11 @@ number of names to the test namespace:
 
     The directory will be reset at the end of the test.
 
+``uncd()``
+    Change to the directory that was current prior to the previous
+    call to ``cd``. You can call ``cd`` multiple times and then
+    ``uncd`` the same number of times to return to the same location.
+
 ``join(*path)``
     A convenient reference to os.path.join.
 

src/zc/buildout/tests.py

@@ -3837,6 +3837,29 @@ def easy_install_SetUp(test):
     zc.buildout.testing.install_develop('zc.recipe.egg', test)
     zc.buildout.testing.install_develop('z3c.recipe.scripts', test)
 
+def buildout_txt_setup(test):
+    zc.buildout.testing.buildoutSetUp(test)
+    mkdir = test.globs['mkdir']
+    eggs = os.environ['buildout-testing-index-url'][7:]
+    test.globs['sample_eggs'] = eggs
+    create_sample_eggs(test)
+
+    for name in os.listdir(eggs):
+        if '-' in name:
+            pname = name.split('-')[0]
+            if not os.path.exists(os.path.join(eggs, pname)):
+                mkdir(eggs, pname)
+            shutil.move(os.path.join(eggs, name),
+                        os.path.join(eggs, pname, name))
+
+    dist = pkg_resources.working_set.find(
+        pkg_resources.Requirement.parse('zc.recipe.egg'))
+    mkdir(eggs, 'zc.recipe.egg')
+    zc.buildout.testing.sdist(
+        os.path.dirname(dist.location),
+        os.path.join(eggs, 'zc.recipe.egg'),
+        )
+
 egg_parse = re.compile('([0-9a-zA-Z_.]+)-([0-9a-zA-Z_.]+)-py(\d[.]\d).egg$'
                        ).match
 def makeNewRelease(project, ws, dest, version='99.99'):
@@ -3943,7 +3966,42 @@ hide_first_index_page_message = (
 def test_suite():
     test_suite = [
         doctest.DocFileSuite(
-            'buildout.txt', 'runsetup.txt', 'repeatable.txt', 'setup.txt',
+            'buildout.txt',
+            setUp=buildout_txt_setup,
+            tearDown=zc.buildout.testing.buildoutTearDown,
+            checker=renormalizing.RENormalizing([
+                zc.buildout.testing.normalize_path,
+                zc.buildout.testing.normalize_endings,
+                zc.buildout.testing.normalize_script,
+                zc.buildout.testing.normalize_egg_py,
+                zc.buildout.tests.hide_distribute_additions,
+                hide_zip_safe_message,
+                (re.compile('__buildout_signature__ = recipes-\S+'),
+                 '__buildout_signature__ = recipes-SSSSSSSSSSS'),
+                (re.compile('executable = [\S ]+python\S*', re.I),
+                 'executable = python'),
+                (re.compile('[-d]  (setuptools|distribute)-\S+[.]egg'),
+                 'setuptools.egg'),
+                (re.compile('zc.buildout(-\S+)?[.]egg(-link)?'),
+                 'zc.buildout.egg'),
+                (re.compile('creating \S*setup.cfg'), 'creating setup.cfg'),
+                (re.compile('hello\%ssetup' % os.path.sep), 'hello/setup'),
+                (re.compile('Picked: (\S+) = \S+'),
+                 'Picked: \\1 = V.V'),
+                (re.compile(r'We have a develop egg: zc.buildout (\S+)'),
+                 'We have a develop egg: zc.buildout X.X.'),
+                (re.compile(r'\\[\\]?'), '/'),
+                (re.compile('WindowsError'), 'OSError'),
+                (re.compile(r'\[Error \d+\] Cannot create a file '
+                            r'when that file already exists: '),
+                 '[Errno 17] File exists: '
+                 ),
+                (re.compile('distribute'), 'setuptools'),
+                (re.compile('Got zc.recipe.egg \S+'), 'Got zc.recipe.egg'),
+                ])
+            ),
+        doctest.DocFileSuite(
+            'runsetup.txt', 'repeatable.txt', 'setup.txt',
             setUp=zc.buildout.testing.buildoutSetUp,
             tearDown=zc.buildout.testing.buildoutTearDown,
             checker=renormalizing.RENormalizing([

zc.recipe.egg_/src/zc/recipe/egg/README.txt

@@ -388,7 +388,7 @@ Let's look at the script that was generated:
     import eggrecipedemo
     <BLANKLINE>
     if __name__ == '__main__':
-        eggrecipedemo.main()
+        sys.exit(eggrecipedemo.main())
 
 Relative egg paths
 ------------------
@@ -441,7 +441,7 @@ Let's look at the script that was generated:
     import eggrecipedemo
     <BLANKLINE>
     if __name__ == '__main__':
-        eggrecipedemo.main()
+        sys.exit(eggrecipedemo.main())
 
 You can specify relative paths in the buildout section, rather than in
 each individual script section:
@@ -488,7 +488,7 @@ each individual script section:
     import eggrecipedemo
     <BLANKLINE>
     if __name__ == '__main__':
-        eggrecipedemo.main()
+        sys.exit(eggrecipedemo.main())
 
 Specifying initialialization code and arguments
 -----------------------------------------------
@@ -538,7 +538,7 @@ to be included in generated scripts:
     import eggrecipedemo
     <BLANKLINE>
     if __name__ == '__main__':
-        eggrecipedemo.main(a, 2)
+        sys.exit(eggrecipedemo.main(a, 2))
 
 Here we see that the initialization code we specified was added after
 setting the path.  Note, as mentioned above, that leading whitespace
@@ -593,7 +593,7 @@ declare entry points using the entry-points option:
     import foo.bar
     <BLANKLINE>
     if __name__ == '__main__':
-        foo.bar.a.b.c()
+        sys.exit(foo.bar.a.b.c())
 
 Generating all scripts
 ----------------------

zc.recipe.egg_/src/zc/recipe/egg/selecting-python.txt

@@ -91,7 +91,7 @@ And the generated scripts invoke Python 2.5:
     import eggrecipedemo
     <BLANKLINE>
     if __name__ == '__main__':
-        eggrecipedemo.main()
+        sys.exit(eggrecipedemo.main())
 
     >>> if sys.platform == 'win32':
     ...     f = open(os.path.join(sample_buildout, 'bin', 'py-demo-script.py'))