Update Demo/parser directory; backport unparse fixes from py3k.

Demo/parser/README

@@ -6,26 +6,27 @@ It also contains examples for the AST parser.
 Files:
 ------
 
-	FILES	     -- list of files associated with the parser module.
+    FILES        -- list of files associated with the parser module.
 
-	README	     -- this file.
+    README       -- this file.
 
-	example.py   --	module that uses the `parser' module to extract
-			information from the parse tree of Python source
-			code.
+    docstring.py -- sample source file containing only a module docstring.
 
-	docstring.py -- sample source file containing only a module docstring.
+    example.py   -- module that uses the `parser' module to extract
+                    information from the parse tree of Python source
+                    code.
 
-	simple.py    -- sample source containing a "short form" definition.
+    simple.py    -- sample source containing a "short form" definition.
 
-	source.py    --	sample source code used to demonstrate ability to
-			handle nested constructs easily using the functions
-			and classes in example.py.
+    source.py    -- sample source code used to demonstrate ability to
+                    handle nested constructs easily using the functions
+                    and classes in example.py.
 
-	test_parser.py  program to put the parser module through its paces.
+    test_parser.py  program to put the parser module through its paces.
 
-	unparse.py	AST (2.5) based example to recreate source code
-			from an AST. This is incomplete; contributions
-			are welcome.
+    test_unparse.py tests for the unparse module
+
+    unparse.py      AST (2.7) based example to recreate source code
+                    from an AST.
 
 Enjoy!

Demo/parser/test_parser.py

@@ -11,19 +11,19 @@ def testChunk(t, fileName):
     global _numFailed
     print '----', fileName,
     try:
-        ast = parser.suite(t)
-        tup = parser.ast2tuple(ast)
-        # this discards the first AST; a huge memory savings when running
+        st = parser.suite(t)
+        tup = parser.st2tuple(st)
+        # this discards the first ST; a huge memory savings when running
         # against a large source file like Tkinter.py.
-        ast = None
-        new = parser.tuple2ast(tup)
+        st = None
+        new = parser.tuple2st(tup)
     except parser.ParserError, err:
         print
         print 'parser module raised exception on input file', fileName + ':'
         traceback.print_exc()
         _numFailed = _numFailed + 1
     else:
-        if tup != parser.ast2tuple(new):
+        if tup != parser.st2tuple(new):
             print
             print 'parser module failed on input file', fileName
             _numFailed = _numFailed + 1

Demo/parser/test_unparse.py

 import unittest
 from test import test_support
-
 import cStringIO
+import sys
+import os
+import tokenize
 import ast
-import _ast
 import unparse
 
-forelse = """\
+def read_pyfile(filename):
+    """Read and return the contents of a Python source file (as a
+    string), taking into account the file encoding."""
+    with open(filename, "r") as pyfile:
+        source = pyfile.read()
+    return source
+
+for_else = """\
 def f():
     for x in range(10):
         break
@@ -15,7 +23,7 @@ def f():
     z = 3
 """
 
-whileelse = """\
+while_else = """\
 def g():
     while True:
         break
@@ -24,16 +32,63 @@ def g():
     z = 3
 """
 
-class UnparseTestCase(unittest.TestCase):
-    # Tests for specific bugs found in earlier versions of unparse
+relative_import = """\
+from . import fred
+from .. import barney
+from .australia import shrimp as prawns
+"""
+
+class_decorator = """\
+@f1(arg)
+@f2
+class Foo: pass
+"""
+
+elif1 = """\
+if cond1:
+    suite1
+elif cond2:
+    suite2
+else:
+    suite3
+"""
+
+elif2 = """\
+if cond1:
+    suite1
+elif cond2:
+    suite2
+"""
+
+try_except_finally = """\
+try:
+    suite1
+except ex1:
+    suite2
+except ex2:
+    suite3
+else:
+    suite4
+finally:
+    suite5
+"""
+
+class ASTTestCase(unittest.TestCase):
+    def assertASTEqual(self, ast1, ast2):
+        dump1 = ast.dump(ast1)
+        dump2 = ast.dump(ast2)
+        self.assertEqual(ast.dump(ast1), ast.dump(ast2))
 
     def check_roundtrip(self, code1, filename="internal"):
-        ast1 = compile(code1, filename, "exec", _ast.PyCF_ONLY_AST)
+        ast1 = compile(code1, filename, "exec", ast.PyCF_ONLY_AST)
         unparse_buffer = cStringIO.StringIO()
         unparse.Unparser(ast1, unparse_buffer)
         code2 = unparse_buffer.getvalue()
-        ast2 = compile(code2, filename, "exec", _ast.PyCF_ONLY_AST)
-        self.assertEqual(ast.dump(ast1), ast.dump(ast2))
+        ast2 = compile(code2, filename, "exec", ast.PyCF_ONLY_AST)
+        self.assertASTEqual(ast1, ast2)
+
+class UnparseTestCase(ASTTestCase):
+    # Tests for specific bugs found in earlier versions of unparse
 
     def test_del_statement(self):
         self.check_roundtrip("del x, y, z")
@@ -43,23 +98,116 @@ class UnparseTestCase(unittest.TestCase):
         self.check_roundtrip("13 >> 7")
 
     def test_for_else(self):
-        self.check_roundtrip(forelse)
+        self.check_roundtrip(for_else)
 
     def test_while_else(self):
-        self.check_roundtrip(whileelse)
+        self.check_roundtrip(while_else)
 
     def test_unary_parens(self):
         self.check_roundtrip("(-1)**7")
+        self.check_roundtrip("(-1.)**8")
+        self.check_roundtrip("(-1j)**6")
         self.check_roundtrip("not True or False")
         self.check_roundtrip("True or not False")
 
+    def test_integer_parens(self):
+        self.check_roundtrip("3 .__abs__()")
+
+    def test_huge_float(self):
+        self.check_roundtrip("1e1000")
+        self.check_roundtrip("-1e1000")
+        self.check_roundtrip("1e1000j")
+        self.check_roundtrip("-1e1000j")
+
+    def test_min_int(self):
+        self.check_roundtrip(str(-sys.maxint-1))
+        self.check_roundtrip("-(%s)" % (sys.maxint + 1))
+
+    def test_imaginary_literals(self):
+        self.check_roundtrip("7j")
+        self.check_roundtrip("-7j")
+        self.check_roundtrip("-(7j)")
+        self.check_roundtrip("0j")
+        self.check_roundtrip("-0j")
+        self.check_roundtrip("-(0j)")
+
+    def test_negative_zero(self):
+        self.check_roundtrip("-0")
+        self.check_roundtrip("-(0)")
+        self.check_roundtrip("-0b0")
+        self.check_roundtrip("-(0b0)")
+        self.check_roundtrip("-0o0")
+        self.check_roundtrip("-(0o0)")
+        self.check_roundtrip("-0x0")
+        self.check_roundtrip("-(0x0)")
+
+    def test_lambda_parentheses(self):
+        self.check_roundtrip("(lambda: int)()")
+
     def test_chained_comparisons(self):
         self.check_roundtrip("1 < 4 <= 5")
         self.check_roundtrip("a is b is c is not d")
 
+    def test_function_arguments(self):
+        self.check_roundtrip("def f(): pass")
+        self.check_roundtrip("def f(a): pass")
+        self.check_roundtrip("def f(b = 2): pass")
+        self.check_roundtrip("def f(a, b): pass")
+        self.check_roundtrip("def f(a, b = 2): pass")
+        self.check_roundtrip("def f(a = 5, b = 2): pass")
+        self.check_roundtrip("def f(*args, **kwargs): pass")
+
+    def test_relative_import(self):
+        self.check_roundtrip(relative_import)
+
+    def test_bytes(self):
+        self.check_roundtrip("b'123'")
+
+    def test_set_literal(self):
+        self.check_roundtrip("{'a', 'b', 'c'}")
+
+    def test_set_comprehension(self):
+        self.check_roundtrip("{x for x in range(5)}")
+
+    def test_dict_comprehension(self):
+        self.check_roundtrip("{x: x*x for x in range(10)}")
+
+    def test_class_decorators(self):
+        self.check_roundtrip(class_decorator)
+
+    def test_elifs(self):
+        self.check_roundtrip(elif1)
+        self.check_roundtrip(elif2)
+
+    def test_try_except_finally(self):
+        self.check_roundtrip(try_except_finally)
+
+class DirectoryTestCase(ASTTestCase):
+    """Test roundtrip behaviour on all files in Lib and Lib/test."""
+
+    # test directories, relative to the root of the distribution
+    test_directories = 'Lib', os.path.join('Lib', 'test')
+
+    def test_files(self):
+        # get names of files to test
+        dist_dir = os.path.join(os.path.dirname(__file__), os.pardir, os.pardir)
+
+        names = []
+        for d in self.test_directories:
+            test_dir = os.path.join(dist_dir, d)
+            for n in os.listdir(test_dir):
+                if n.endswith('.py') and not n.startswith('bad'):
+                    names.append(os.path.join(test_dir, n))
+
+        for filename in names:
+            if test_support.verbose:
+                print('Testing %s' % filename)
+            source = read_pyfile(filename)
+            self.check_roundtrip(source)
+
 
 def test_main():
-    test_support.run_unittest(UnparseTestCase)
+    test_support.run_unittest(UnparseTestCase, DirectoryTestCase)
 
 if __name__ == '__main__':
     test_main()

Demo/parser/texipre.dat

-\input texinfo   @c -*-texinfo-*-
-@c %**start of header
-@setfilename parser.info
-@settitle Python Parser Module Reference
-@setchapternewpage odd
-@footnotestyle end
-@c %**end of header
-
-@ifinfo
-This file describes the interfaces
-published by the optional @code{parser} module and gives examples of
-how they may be used.  It contains the same text as the chapter on the
-@code{parser} module in the @cite{Python Library Reference}, but is
-presented as a separate document.
-
-Copyright 1995-1996 by Fred L. Drake, Jr., Reston, Virginia, USA, and
-Virginia Polytechnic Institute and State University, Blacksburg,
-Virginia, USA.  Portions of the software copyright 1991-1995 by
-Stichting Mathematisch Centrum, Amsterdam, The Netherlands.  Copying is
-permitted under the terms associated with the main Python distribution,
-with the additional restriction that this additional notice be included
-and maintained on all distributed copies.
-
-                        All Rights Reserved
-
-Permission to use, copy, modify, and distribute this software and its
-documentation for any purpose and without fee is hereby granted,
-provided that the above copyright notice appear in all copies and that
-both that copyright notice and this permission notice appear in
-supporting documentation, and that the names of Fred L. Drake, Jr. and
-Virginia Polytechnic Institute and State University not be used in
-advertising or publicity pertaining to distribution of the software
-without specific, written prior permission.
-
-FRED L. DRAKE, JR. AND VIRGINIA POLYTECHNIC INSTITUTE AND STATE
-UNIVERSITY DISCLAIM ALL WARRANTIES WITH REGARD TO THIS SOFTWARE,
-INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN NO
-EVENT SHALL FRED L. DRAKE, JR. OR VIRGINIA POLYTECHNIC INSTITUTE AND
-STATE UNIVERSITY BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL
-DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR
-PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER
-TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
-PERFORMANCE OF THIS SOFTWARE.
-@end ifinfo
-
-@titlepage
-@title Python Parser Module Reference
-@author Fred L. Drake, Jr.
-
-@c  The following two commands start the copyright page.
-@page
-@vskip 0pt plus 1filll
-Copyright 1995-1996 by Fred L. Drake, Jr., Reston, Virginia, USA, and
-Virginia Polytechnic Institute and State University, Blacksburg,
-Virginia, USA.  Portions of the software copyright 1991-1995 by
-Stichting Mathematisch Centrum, Amsterdam, The Netherlands.  Copying is
-permitted under the terms associated with the main Python distribution,
-with the additional restriction that this additional notice be included
-and maintained on all distributed copies.
-
-@center All Rights Reserved
-
-Permission to use, copy, modify, and distribute this software and its
-documentation for any purpose and without fee is hereby granted,
-provided that the above copyright notice appear in all copies and that
-both that copyright notice and this permission notice appear in
-supporting documentation, and that the names of Fred L. Drake, Jr. and
-Virginia Polytechnic Institute and State University not be used in
-advertising or publicity pertaining to distribution of the software
-without specific, written prior permission.
-
-FRED L. DRAKE, JR. AND VIRGINIA POLYTECHNIC INSTITUTE AND STATE
-UNIVERSITY DISCLAIM ALL WARRANTIES WITH REGARD TO THIS SOFTWARE,
-INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN NO
-EVENT SHALL FRED L. DRAKE, JR. OR VIRGINIA POLYTECHNIC INSTITUTE AND
-STATE UNIVERSITY BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL
-DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR
-PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER
-TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
-PERFORMANCE OF THIS SOFTWARE.
-@end titlepage
-
-
-@node Top, Overview, (dir), (dir)
-@top The Python Parser Module
-
-@ifinfo
-This file describes the interfaces
-published by the optional @code{parser} module and gives examples of
-how they may be used.  It contains the same text as the chapter on the
-@code{parser} module in the @cite{Python Library Reference}, but is
-presented as a separate document.
-
-This version corresponds to Python version 1.4 (1 Sept. 1996).
-
-@end ifinfo
-
-@c placeholder for the master menu -- patched by texinfo-all-menus-update
-@menu
-@end menu