compiler.transformer: correct lineno attribute when possible

SF patch #1015989

The basic idea of this patch is to compute lineno attributes for all AST nodes.  The actual
implementation lead to a lot of restructing and code cleanup.

The generated AST nodes now have an optional lineno argument to constructor.  Remove the
top-level asList(), since it didn't seem to serve any purpose.  Add an __iter__ to ast nodes.
Use isinstance() instead of explicit type tests.

Change transformer to use the new lineno attribute, which replaces three lines of code with one.
Use universal newlines so that we can get rid of special-case code for line endings.  Use
lookup_node() in a few more frequently called, but simple com_xxx methods().  Change string
exception to class exception.

Lib/test/test_compiler.py

@@ -33,6 +33,65 @@ class CompilerTest(unittest.TestCase):
                 else:
                     compiler.compile(buf, basename, "exec")
 
+    def testLineNo(self):
+        # Test that all nodes except Module have a correct lineno attribute.
+        filename = __file__
+        if filename.endswith(".pyc") or filename.endswith(".pyo"):
+            filename = filename[:-1]
+        tree = compiler.parseFile(filename)
+        self.check_lineno(tree)
+
+    def check_lineno(self, node):
+        try:
+            self._check_lineno(node)
+        except AssertionError:
+            print node.__class__, node.lineno
+            raise
+
+    def _check_lineno(self, node):
+        if not node.__class__ in NOLINENO:
+            self.assert_(isinstance(node.lineno, int), 
+                "lineno=%s on %s" % (node.lineno, node.__class__))
+            self.assert_(node.lineno > 0, 
+                "lineno=%s on %s" % (node.lineno, node.__class__))
+        for child in node.getChildNodes():
+            self.check_lineno(child)
+
+NOLINENO = (compiler.ast.Module, compiler.ast.Stmt, compiler.ast.Discard)
+
+###############################################################################
+# code below is just used to trigger some possible errors, for the benefit of
+# testLineNo
+###############################################################################
+
+class Toto:
+    """docstring"""
+    pass
+
+a, b = 2, 3
+[c, d] = 5, 6
+l = [(x, y) for x, y in zip(range(5), range(5,10))]
+l[0]
+l[3:4]
+if l:
+    pass
+else:
+    a, b = b, a
+
+try:
+    print yo
+except:
+    yo = 3
+else:
+    yo += 3
+    
+try:
+    a += b
+finally:
+    b = 0
+    
+###############################################################################
+
 def test_main():
     global TEST_ALL
     TEST_ALL = test.test_support.is_resource_enabled("compiler")

Lib/test/test_transformer.py

@@ -29,9 +29,7 @@ class Tests(unittest.TestCase):
             assert vals['b'] == 2
 
 def test_main():
-    test_support.run_unittest(
-        Tests
-    )
+    test_support.run_unittest(Tests)
 
 if __name__ == "__main__":
     test_main()

Misc/ACKS

@@ -563,6 +563,7 @@ Michael Stone
 Ken Stox
 Daniel Stutzbach
 Paul Swartz
+Thenault Sylvain
 Geoff Talvola
 William Tanksley
 Christian Tanzer

Tools/compiler/astgen.py

@@ -94,7 +94,6 @@ class NodeInfo:
     def gen_source(self):
         buf = StringIO()
         print >> buf, "class %s(Node):" % self.name
-        print >> buf, '    nodes["%s"] = "%s"' % (self.name.lower(), self.name)
         self._gen_init(buf)
         print >> buf
         self._gen_getChildren(buf)
@@ -106,12 +105,14 @@ class NodeInfo:
         return buf.read()
 
     def _gen_init(self, buf):
-        print >> buf, "    def __init__(self, %s):" % self.args
+        if self.args:
+            print >> buf, "    def __init__(self, %s, lineno=None):" % self.args
+        else:
+            print >> buf, "    def __init__(self, lineno=None):"
         if self.argnames:
             for name in self.argnames:
                 print >> buf, "        self.%s = %s" % (name, name)
-        else:
-            print >> buf, "        pass"
+        print >> buf, "        self.lineno = lineno"
         if self.init:
             print >> buf, "".join(["    " + line for line in self.init])
 
@@ -128,15 +129,18 @@ class NodeInfo:
                 else:
                     print >> buf, "        return %s" % clist
             else:
-                print >> buf, "        children = []"
-                template = "        children.%s(%sself.%s%s)"
-                for name in self.argnames:
-                    if self.argprops[name] == P_NESTED:
-                        print >> buf, template % ("extend", "flatten(",
-                                                  name, ")")
-                    else:
-                        print >> buf, template % ("append", "", name, "")
-                print >> buf, "        return tuple(children)"
+                if len(self.argnames) == 1:
+                    print >> buf, "        return tuple(flatten(self.%s))" % self.argnames[0]
+                else:
+                    print >> buf, "        children = []"
+                    template = "        children.%s(%sself.%s%s)"
+                    for name in self.argnames:
+                        if self.argprops[name] == P_NESTED:
+                            print >> buf, template % ("extend", "flatten(",
+                                                      name, ")")
+                        else:
+                            print >> buf, template % ("append", "", name, "")
+                    print >> buf, "        return tuple(children)"
 
     def _gen_getChildNodes(self, buf):
         print >> buf, "    def getChildNodes(self):"
@@ -158,7 +162,7 @@ class NodeInfo:
                 template = "        nodelist.%s(%sself.%s%s)"
                 for name in self.argnames:
                     if self.argprops[name] == P_NONE:
-                        tmp = ("        if self.%s is not None:"
+                        tmp = ("        if self.%s is not None:\n"
                                "            nodelist.append(self.%s)")
                         print >> buf, tmp % (name, name)
                     elif self.argprops[name] == P_NESTED:
@@ -226,16 +230,15 @@ if __name__ == "__main__":
 ### PROLOGUE
 """Python abstract syntax node definitions
 
-This file is automatically generated.
+This file is automatically generated by Tools/compiler/astgen.py
 """
-from types import TupleType, ListType
 from consts import CO_VARARGS, CO_VARKEYWORDS
 
 def flatten(list):
     l = []
     for elt in list:
         t = type(elt)
-        if t is TupleType or t is ListType:
+        if t is tuple or t is list:
             for elt2 in flatten(elt):
                 l.append(elt2)
         else:
@@ -245,29 +248,17 @@ def flatten(list):
 def flatten_nodes(list):
     return [n for n in flatten(list) if isinstance(n, Node)]
 
-def asList(nodearg):
-    l = []
-    for item in nodearg:
-        if hasattr(item, "asList"):
-            l.append(item.asList())
-        else:
-            t = type(item)
-            if t is TupleType or t is ListType:
-                l.append(tuple(asList(item)))
-            else:
-                l.append(item)
-    return l
-
 nodes = {}
 
-class Node: # an abstract base class
-    lineno = None # provide a lineno for nodes that don't have one
-    def getType(self):
-        pass # implemented by subclass
+class Node:
+    """Abstract base class for ast nodes."""
     def getChildren(self):
         pass # implemented by subclasses
-    def asList(self):
-        return tuple(asList(self.getChildren()))
+    def __iter__(self):
+        for n in self.getChildren():
+            yield n
+    def asList(self): # for backwards compatibility
+        return self.getChildren()
     def getChildNodes(self):
         pass # implemented by subclasses
 
@@ -290,6 +281,6 @@ class Expression(Node):
         return "Expression(%s)" % (repr(self.node))
 
 ### EPILOGUE
-klasses = globals()
-for k in nodes.keys():
-    nodes[k] = klasses[nodes[k]]
+for name, obj in globals().items():
+    if isinstance(obj, type) and issubclass(obj, Node):
+       nodes[name.lower()] = obj