Improved error reporting/handling:

- Added "logical tracebacks" that show functionally what the buildout
  was doing when an error occurs.  Don't show a Python traceback
  unless the -D option is used.

- Added a -D option that causes the buildout to print a traceback and
  start the pdb post-mortem debugger when an error occurs.

src/zc/buildout/buildout.txt

@@ -120,9 +120,7 @@ and then we'll create a source file for our mkdir recipe:
     ... class Mkdir:
     ...
     ...     def __init__(self, buildout, name, options):
-    ...         self.buildout = buildout
-    ...         self.name = name
-    ...         self.options = options
+    ...         self.name, self.options = name, options
     ...         options['path'] = os.path.join(
     ...                               buildout['buildout']['directory'],
     ...                               options['path'],
@@ -367,10 +365,10 @@ Error reporting
 If a user makes an error, an error needs to be printed and work needs
 to stop.  This is accomplished by logging a detailed error message and
 then raising a (or an instance of a subclass of a)
-zc.buildout.UserError exception.  Raising UserError causes the
-buildout to print the error and exit without printing a traceback.  In
-the sample above, of someone gives a non-existant directory to create
-the directory in:
+zc.buildout.UserError exception.  Raising an error other than a
+UserError still displays the error, but labels it as a bug in the
+buildout software or recipe. In the sample above, of someone gives a
+non-existant directory to create the directory in:
 
 
     >>> write(sample_buildout, 'buildout.cfg',
@@ -389,6 +387,10 @@ We'll get a user error, not a traceback.
     >>> print system(buildout),
     buildout: Develop: /sample-buildout/recipes
     data-dir: Cannot create /xxx/mydata. /xxx is not a directory.
+    While:
+      Installing
+      Getting section data-dir
+      Initializing part data-dir
     Error: Invalid Path
 
 
@@ -813,6 +815,8 @@ buildout.
     ... """)
 
     >>> print system(buildout + ' -c ' + server_url + '/remote.cfg'),
+    While:
+      Initializing
     Error: Missing option: buildout:directory
 
 Normally, the buildout directory defaults to directory

src/zc/buildout/debugging.txt

+Debugging buildouts
+===================
+
+Buildouts can be pretty complex.  When things go wrong, it isn't
+always obvious why.  Errors can occur due to problems in user input or
+due to bugs in zc.buildout or recipes.  When an error occurs, Python's
+post-mortem debugger can be used to inspect the state of the buildout
+or recipe code were there error occured.  To enable this, use the -D
+option to the buildout.  Let's create a recipe that has a bug:
+
+    >>> mkdir(sample_buildout, 'recipes')
+
+    >>> write(sample_buildout, 'recipes', 'mkdir.py', 
+    ... """
+    ... import os, zc.buildout
+    ...
+    ... class Mkdir:
+    ...
+    ...     def __init__(self, buildout, name, options):
+    ...         self.name, self.options = name, options
+    ...         options['path'] = os.path.join(
+    ...                               buildout['buildout']['directory'],
+    ...                               options['path'],
+    ...                               )
+    ...
+    ...     def install(self):
+    ...         directory = self.options['directory']
+    ...         os.mkdir(directory)
+    ...         return directory
+    ...
+    ...     def update(self):
+    ...         pass
+    ... """)
+
+    >>> write(sample_buildout, 'recipes', 'setup.py',
+    ... """
+    ... from setuptools import setup
+    ... 
+    ... setup(name = "recipes",
+    ...       entry_points = {'zc.buildout': ['mkdir = mkdir:Mkdir']},
+    ...       )
+    ... """)
+
+And create a buildout that uses it:
+
+    >>> write(sample_buildout, 'buildout.cfg',
+    ... """
+    ... [buildout]
+    ... develop = recipes
+    ... parts = data-dir
+    ...
+    ... [data-dir]
+    ... recipe = recipes:mkdir
+    ... path = mystuff
+    ... """)
+
+If we run the buildout, we'll get an error:
+
+    >>> print system(buildout),
+    buildout: Develop: /sample-buildout/recipes
+    buildout: Installing data-dir
+    While:
+      Installing data-dir
+    Error: Missing option: data-dir:directory
+
+
+If we want to debug the error, we can add the -D option. Here's we'll
+supply some input:
+
+    >>> print system(buildout+" -D", """\
+    ... up
+    ... p self.options.keys()
+    ... q
+    ... """),
+    buildout: Develop: /tmp/tmpozk_tH/_TEST_/sample-buildout/recipes
+    buildout: Installing data-dir
+    While:
+      Installing data-dir
+    Traceback (most recent call last):
+      File "/zc/buildout/buildout.py", line 1173, in main
+        getattr(buildout, command)(args)
+      File "/zc/buildout/buildout.py", line 324, in install
+        installed_files = recipe.install()
+      File "/sample-buildout/recipes/mkdir.py", line 14, in install
+        directory = self.options['directory']
+      File "/zc/buildout/buildout.py", line 815, in __getitem__
+        raise MissingOption("Missing option: %s:%s" % (self.name, key))
+    MissingOption: Missing option: data-dir:directory
+    <BLANKLINE>
+    Starting pdb:
+    > /zc/buildout/buildout.py(815)__getitem__()
+    -> raise MissingOption("Missing option: %s:%s" % (self.name, key))
+    (Pdb) > /sample-buildout/recipes/mkdir.py(14)install()
+    -> directory = self.options['directory']
+    (Pdb) ['path', 'recipe']
+    (Pdb) 

src/zc/buildout/tests.py

@@ -116,9 +116,15 @@ It is an error to create a variable-reference cycle:
 
     >>> print system(os.path.join(sample_buildout, 'bin', 'buildout')),
     ... # doctest: +NORMALIZE_WHITESPACE +ELLIPSIS
+    While:
+      Initializing
+      Getting section buildout
+      Initializing section buildout
+      Getting option buildout:y
+      Getting option buildout:z
+      Getting option buildout:x
+      Getting option buildout:y
     Error: Circular reference in substitutions.
-    We're evaluating buildout:y, buildout:z, buildout:x
-    and are referencing: buildout:y.
 
 It is an error to use funny characters in variable refereces:
 
@@ -131,6 +137,11 @@ It is an error to use funny characters in variable refereces:
     ... ''')
 
     >>> print system(os.path.join(sample_buildout, 'bin', 'buildout')),
+    While:
+      Initializing
+      Getting section buildout
+      Initializing section buildout
+      Getting option buildout:x
     Error: The section name in substitution, ${bui$ldout:y},
     has invalid characters.
 
@@ -143,6 +154,11 @@ It is an error to use funny characters in variable refereces:
     ... ''')
 
     >>> print system(os.path.join(sample_buildout, 'bin', 'buildout')),
+    While:
+      Initializing
+      Getting section buildout
+      Initializing section buildout
+      Getting option buildout:x
     Error: The option name in substitution, ${buildout:y{z},
     has invalid characters.
 
@@ -157,6 +173,11 @@ and too have too many or too few colons:
     ... ''')
 
     >>> print system(os.path.join(sample_buildout, 'bin', 'buildout')),
+    While:
+      Initializing
+      Getting section buildout
+      Initializing section buildout
+      Getting option buildout:x
     Error: The substitution, ${parts},
     doesn't contain a colon.
 
@@ -169,6 +190,11 @@ and too have too many or too few colons:
     ... ''')
 
     >>> print system(os.path.join(sample_buildout, 'bin', 'buildout')),
+    While:
+      Initializing
+      Getting section buildout
+      Initializing section buildout
+      Getting option buildout:x
     Error: The substitution, ${buildout:y:z},
     has too many colons.
 
@@ -181,6 +207,9 @@ Al parts have to have a section:
     ... ''')
 
     >>> print system(os.path.join(sample_buildout, 'bin', 'buildout')),
+    While:
+      Installing
+      Getting section x
     Error: The referenced section, 'x', was not defined.
 
 and all parts have to have a specified recipe:
@@ -196,6 +225,8 @@ and all parts have to have a specified recipe:
     ... ''')
 
     >>> print system(os.path.join(sample_buildout, 'bin', 'buildout')),
+    While:
+      Installing
     Error: Missing option: x:recipe
 
 """
@@ -477,6 +508,12 @@ Options:
     buildout:newest=false.  With this setting, buildout will not seek
     new distributions if installed distributions satisfy it's
     requirements. 
+<BLANKLINE>
+  -D
+<BLANKLINE>
+    Debug errors.  If an error occurs, then the post-mortem debugger
+    will be started. This is especially useful for debuging recipe
+    problems.
 <BLANKLINE>
 Assignments are of the form: section:option=value and are used to
 provide configuration options that override those given in the
@@ -554,6 +591,12 @@ Options:
     buildout:newest=false.  With this setting, buildout will not seek
     new distributions if installed distributions satisfy it's
     requirements. 
+<BLANKLINE>
+  -D
+<BLANKLINE>
+    Debug errors.  If an error occurs, then the post-mortem debugger
+    will be started. This is especially useful for debuging recipe
+    problems.
 <BLANKLINE>
 Assignments are of the form: section:option=value and are used to
 provide configuration options that override those given in the
@@ -1233,6 +1276,55 @@ def log_when_there_are_not_local_distros():
     
     """
 
+def internal_errors():
+    """Internal errors are clearly marked and don't generate tracebacks:
+
+    >>> mkdir(sample_buildout, 'recipes')
+
+    >>> write(sample_buildout, 'recipes', 'mkdir.py', 
+    ... '''
+    ... class Mkdir:
+    ...     def __init__(self, buildout, name, options):
+    ...         self.name, self.options = name, options
+    ...         options['path'] = os.path.join(
+    ...                               buildout['buildout']['directory'],
+    ...                               options['path'],
+    ...                               )
+    ... ''')
+
+    >>> write(sample_buildout, 'recipes', 'setup.py',
+    ... '''
+    ... from setuptools import setup
+    ... setup(name = "recipes",
+    ...       entry_points = {'zc.buildout': ['mkdir = mkdir:Mkdir']},
+    ...       )
+    ... ''')
+
+    >>> write(sample_buildout, 'buildout.cfg',
+    ... '''
+    ... [buildout]
+    ... develop = recipes
+    ... parts = data-dir
+    ...
+    ... [data-dir]
+    ... recipe = recipes:mkdir
+    ... ''')
+
+    >>> print system(buildout),
+    buildout: Develop: /sample-buildout/recipes
+    While:
+      Installing
+      Getting section data-dir
+      Initializing part data-dir
+    <BLANKLINE>
+    An internal error occured due to a bug in either zc.buildout or in a
+    recipe being used:
+    <BLANKLINE>
+    NameError:
+    global name 'os' is not defined
+    """
+    
+
 ######################################################################
     
 def create_sample_eggs(test, executable=sys.executable):
@@ -1440,6 +1532,17 @@ def test_suite():
                 'picked \\1 = V.V'),
                ])
             ),
+        doctest.DocFileSuite(
+            'debugging.txt',
+            setUp=zc.buildout.testing.buildoutSetUp,
+            tearDown=zc.buildout.testing.buildoutTearDown,
+            checker=renormalizing.RENormalizing([
+               zc.buildout.testing.normalize_path,
+               (re.compile(r'\S+buildout.py'), 'buildout.py'),
+               (re.compile(r'line \d+'), 'line NNN'),
+               (re.compile(r'py\(\d+\)'), 'py(NNN)'),
+               ])
+            ),
 
         doctest.DocFileSuite(
             'update.txt',