Patch #1440601: Add col_offset attribute to AST nodes.

Doc/lib/libast.tex

@@ -34,7 +34,13 @@ names of all child nodes.
 
 Each instance of a concrete class has one attribute for each child node,
 of the type as defined in the grammar. For example, \code{_ast.BinOp}
-instances have an attribute \code{left} of type \code{_ast.expr}.
+instances have an attribute \code{left} of type \code{_ast.expr}.  
+Instances of \code{_ast.expr} and \code{_ast.stmt} subclasses also
+have lineno and col_offset attributes.  The lineno is the line number
+of source text (1 indexed so the first line is line 1) and the
+col_offset is the utf8 byte offset of the first token that generated
+the node.  The utf8 offset is recorded because the parser uses utf8 
+internally.
 
 If these attributes are marked as optional in the grammar (using a
 question mark), the value might be \code{None}. If the attributes

Include/Python-ast.h

@@ -178,6 +178,7 @@ struct _stmt {
                 
         } v;
         int lineno;
+        int col_offset;
 };
 
 struct _expr {
@@ -288,6 +289,7 @@ struct _expr {
                 
         } v;
         int lineno;
+        int col_offset;
 };
 
 struct _slice {
@@ -346,68 +348,79 @@ mod_ty Interactive(asdl_seq * body, PyArena *arena);
 mod_ty Expression(expr_ty body, PyArena *arena);
 mod_ty Suite(asdl_seq * body, PyArena *arena);
 stmt_ty FunctionDef(identifier name, arguments_ty args, asdl_seq * body,
-                    asdl_seq * decorators, int lineno, PyArena *arena);
+                    asdl_seq * decorators, int lineno, int col_offset, PyArena
+                    *arena);
 stmt_ty ClassDef(identifier name, asdl_seq * bases, asdl_seq * body, int
-                 lineno, PyArena *arena);
-stmt_ty Return(expr_ty value, int lineno, PyArena *arena);
-stmt_ty Delete(asdl_seq * targets, int lineno, PyArena *arena);
-stmt_ty Assign(asdl_seq * targets, expr_ty value, int lineno, PyArena *arena);
+                 lineno, int col_offset, PyArena *arena);
+stmt_ty Return(expr_ty value, int lineno, int col_offset, PyArena *arena);
+stmt_ty Delete(asdl_seq * targets, int lineno, int col_offset, PyArena *arena);
+stmt_ty Assign(asdl_seq * targets, expr_ty value, int lineno, int col_offset,
+               PyArena *arena);
 stmt_ty AugAssign(expr_ty target, operator_ty op, expr_ty value, int lineno,
-                  PyArena *arena);
-stmt_ty Print(expr_ty dest, asdl_seq * values, bool nl, int lineno, PyArena
-              *arena);
+                  int col_offset, PyArena *arena);
+stmt_ty Print(expr_ty dest, asdl_seq * values, bool nl, int lineno, int
+              col_offset, PyArena *arena);
 stmt_ty For(expr_ty target, expr_ty iter, asdl_seq * body, asdl_seq * orelse,
-            int lineno, PyArena *arena);
-stmt_ty While(expr_ty test, asdl_seq * body, asdl_seq * orelse, int lineno,
-              PyArena *arena);
-stmt_ty If(expr_ty test, asdl_seq * body, asdl_seq * orelse, int lineno,
-           PyArena *arena);
+            int lineno, int col_offset, PyArena *arena);
+stmt_ty While(expr_ty test, asdl_seq * body, asdl_seq * orelse, int lineno, int
+              col_offset, PyArena *arena);
+stmt_ty If(expr_ty test, asdl_seq * body, asdl_seq * orelse, int lineno, int
+           col_offset, PyArena *arena);
 stmt_ty With(expr_ty context_expr, expr_ty optional_vars, asdl_seq * body, int
-             lineno, PyArena *arena);
-stmt_ty Raise(expr_ty type, expr_ty inst, expr_ty tback, int lineno, PyArena
-              *arena);
+             lineno, int col_offset, PyArena *arena);
+stmt_ty Raise(expr_ty type, expr_ty inst, expr_ty tback, int lineno, int
+              col_offset, PyArena *arena);
 stmt_ty TryExcept(asdl_seq * body, asdl_seq * handlers, asdl_seq * orelse, int
-                  lineno, PyArena *arena);
-stmt_ty TryFinally(asdl_seq * body, asdl_seq * finalbody, int lineno, PyArena
-                   *arena);
-stmt_ty Assert(expr_ty test, expr_ty msg, int lineno, PyArena *arena);
-stmt_ty Import(asdl_seq * names, int lineno, PyArena *arena);
+                  lineno, int col_offset, PyArena *arena);
+stmt_ty TryFinally(asdl_seq * body, asdl_seq * finalbody, int lineno, int
+                   col_offset, PyArena *arena);
+stmt_ty Assert(expr_ty test, expr_ty msg, int lineno, int col_offset, PyArena
+               *arena);
+stmt_ty Import(asdl_seq * names, int lineno, int col_offset, PyArena *arena);
 stmt_ty ImportFrom(identifier module, asdl_seq * names, int level, int lineno,
-                   PyArena *arena);
-stmt_ty Exec(expr_ty body, expr_ty globals, expr_ty locals, int lineno, PyArena
-             *arena);
-stmt_ty Global(asdl_seq * names, int lineno, PyArena *arena);
-stmt_ty Expr(expr_ty value, int lineno, PyArena *arena);
-stmt_ty Pass(int lineno, PyArena *arena);
-stmt_ty Break(int lineno, PyArena *arena);
-stmt_ty Continue(int lineno, PyArena *arena);
-expr_ty BoolOp(boolop_ty op, asdl_seq * values, int lineno, PyArena *arena);
-expr_ty BinOp(expr_ty left, operator_ty op, expr_ty right, int lineno, PyArena
-              *arena);
-expr_ty UnaryOp(unaryop_ty op, expr_ty operand, int lineno, PyArena *arena);
-expr_ty Lambda(arguments_ty args, expr_ty body, int lineno, PyArena *arena);
-expr_ty IfExp(expr_ty test, expr_ty body, expr_ty orelse, int lineno, PyArena
-              *arena);
-expr_ty Dict(asdl_seq * keys, asdl_seq * values, int lineno, PyArena *arena);
-expr_ty ListComp(expr_ty elt, asdl_seq * generators, int lineno, PyArena
-                 *arena);
-expr_ty GeneratorExp(expr_ty elt, asdl_seq * generators, int lineno, PyArena
-                     *arena);
-expr_ty Yield(expr_ty value, int lineno, PyArena *arena);
+                   int col_offset, PyArena *arena);
+stmt_ty Exec(expr_ty body, expr_ty globals, expr_ty locals, int lineno, int
+             col_offset, PyArena *arena);
+stmt_ty Global(asdl_seq * names, int lineno, int col_offset, PyArena *arena);
+stmt_ty Expr(expr_ty value, int lineno, int col_offset, PyArena *arena);
+stmt_ty Pass(int lineno, int col_offset, PyArena *arena);
+stmt_ty Break(int lineno, int col_offset, PyArena *arena);
+stmt_ty Continue(int lineno, int col_offset, PyArena *arena);
+expr_ty BoolOp(boolop_ty op, asdl_seq * values, int lineno, int col_offset,
+               PyArena *arena);
+expr_ty BinOp(expr_ty left, operator_ty op, expr_ty right, int lineno, int
+              col_offset, PyArena *arena);
+expr_ty UnaryOp(unaryop_ty op, expr_ty operand, int lineno, int col_offset,
+                PyArena *arena);
+expr_ty Lambda(arguments_ty args, expr_ty body, int lineno, int col_offset,
+               PyArena *arena);
+expr_ty IfExp(expr_ty test, expr_ty body, expr_ty orelse, int lineno, int
+              col_offset, PyArena *arena);
+expr_ty Dict(asdl_seq * keys, asdl_seq * values, int lineno, int col_offset,
+             PyArena *arena);
+expr_ty ListComp(expr_ty elt, asdl_seq * generators, int lineno, int
+                 col_offset, PyArena *arena);
+expr_ty GeneratorExp(expr_ty elt, asdl_seq * generators, int lineno, int
+                     col_offset, PyArena *arena);
+expr_ty Yield(expr_ty value, int lineno, int col_offset, PyArena *arena);
 expr_ty Compare(expr_ty left, asdl_seq * ops, asdl_seq * comparators, int
-                lineno, PyArena *arena);
+                lineno, int col_offset, PyArena *arena);
 expr_ty Call(expr_ty func, asdl_seq * args, asdl_seq * keywords, expr_ty
-             starargs, expr_ty kwargs, int lineno, PyArena *arena);
-expr_ty Repr(expr_ty value, int lineno, PyArena *arena);
-expr_ty Num(object n, int lineno, PyArena *arena);
-expr_ty Str(string s, int lineno, PyArena *arena);
+             starargs, expr_ty kwargs, int lineno, int col_offset, PyArena
+             *arena);
+expr_ty Repr(expr_ty value, int lineno, int col_offset, PyArena *arena);
+expr_ty Num(object n, int lineno, int col_offset, PyArena *arena);
+expr_ty Str(string s, int lineno, int col_offset, PyArena *arena);
 expr_ty Attribute(expr_ty value, identifier attr, expr_context_ty ctx, int
-                  lineno, PyArena *arena);
+                  lineno, int col_offset, PyArena *arena);
 expr_ty Subscript(expr_ty value, slice_ty slice, expr_context_ty ctx, int
-                  lineno, PyArena *arena);
-expr_ty Name(identifier id, expr_context_ty ctx, int lineno, PyArena *arena);
-expr_ty List(asdl_seq * elts, expr_context_ty ctx, int lineno, PyArena *arena);
-expr_ty Tuple(asdl_seq * elts, expr_context_ty ctx, int lineno, PyArena *arena);
+                  lineno, int col_offset, PyArena *arena);
+expr_ty Name(identifier id, expr_context_ty ctx, int lineno, int col_offset,
+             PyArena *arena);
+expr_ty List(asdl_seq * elts, expr_context_ty ctx, int lineno, int col_offset,
+             PyArena *arena);
+expr_ty Tuple(asdl_seq * elts, expr_context_ty ctx, int lineno, int col_offset,
+              PyArena *arena);
 slice_ty Ellipsis(PyArena *arena);
 slice_ty Slice(expr_ty lower, expr_ty upper, expr_ty step, PyArena *arena);
 slice_ty ExtSlice(asdl_seq * dims, PyArena *arena);

Include/node.h

@@ -11,13 +11,14 @@ typedef struct _node {
     short		n_type;
     char		*n_str;
     int			n_lineno;
+    int			n_col_offset;
     int			n_nchildren;
     struct _node	*n_child;
 } node;
 
 PyAPI_FUNC(node *) PyNode_New(int type);
 PyAPI_FUNC(int) PyNode_AddChild(node *n, int type,
-                                      char *str, int lineno);
+                                      char *str, int lineno, int col_offset);
 PyAPI_FUNC(void) PyNode_Free(node *n);
 
 /* Node access functions */

Lib/test/test_ast.py

 import sys, itertools
+import _ast
 
 def to_tuple(t):
     if t is None or isinstance(t, (basestring, int, long, complex)):
@@ -6,6 +7,8 @@ def to_tuple(t):
     elif isinstance(t, list):
         return [to_tuple(e) for e in t]
     result = [t.__class__.__name__]
+    if hasattr(t, 'lineno') and hasattr(t, 'col_offset'):
+        result.append((t.lineno, t.col_offset))
     if t._fields is None:
         return tuple(result)
     for f in t._fields:
@@ -106,7 +109,10 @@ eval_tests = [
   # List
   "[1,2,3]",
   # Tuple
-  "1,2,3"
+  "1,2,3",
+  # Combination
+  "a.b.c.d(a.b[1:2])",
+
 ]
 
 # TODO: expr_context, slice, boolop, operator, unaryop, cmpop, comprehension
@@ -121,58 +127,77 @@ if __name__=='__main__' and sys.argv[1:] == ['-g']:
     print "run_tests()"
     raise SystemExit
 
+def test_order(ast_node, parent_pos):
+    
+    if not isinstance(ast_node, _ast.AST) or ast_node._fields == None:
+        return
+    if isinstance(ast_node, (_ast.expr, _ast.stmt)):
+        node_pos = (ast_node.lineno, ast_node.col_offset)
+        assert node_pos >= parent_pos, (node_pos, parent_pos)
+        parent_pos = (ast_node.lineno, ast_node.col_offset)
+    for name in ast_node._fields:
+        value = getattr(ast_node, name)
+        if isinstance(value, list):
+            for child in value:
+                test_order(child, parent_pos)
+        elif value != None:
+            test_order(value, parent_pos)
+
 def run_tests():
     for input, output, kind in ((exec_tests, exec_results, "exec"),
                                         (single_tests, single_results, "single"),
                                         (eval_tests, eval_results, "eval")):
         for i, o in itertools.izip(input, output):
-            assert to_tuple(compile(i, "?", kind, 0x400)) == o
+            ast_tree = compile(i, "?", kind, 0x400)
+            assert to_tuple(ast_tree) == o
+            test_order(ast_tree, (0, 0))
 
 #### EVERYTHING BELOW IS GENERATED #####
 exec_results = [
-('Module', [('FunctionDef', 'f', ('arguments', [], None, None, []), [('Pass',)], [])]),
-('Module', [('ClassDef', 'C', [], [('Pass',)])]),
-('Module', [('FunctionDef', 'f', ('arguments', [], None, None, []), [('Return', ('Num', 1))], [])]),
-('Module', [('Delete', [('Name', 'v', ('Del',))])]),
-('Module', [('Assign', [('Name', 'v', ('Store',))], ('Num', 1))]),
-('Module', [('AugAssign', ('Name', 'v', ('Load',)), ('Add',), ('Num', 1))]),
-('Module', [('Print', ('Name', 'f', ('Load',)), [('Num', 1)], False)]),
-('Module', [('For', ('Name', 'v', ('Store',)), ('Name', 'v', ('Load',)), [('Pass',)], [])]),
-('Module', [('While', ('Name', 'v', ('Load',)), [('Pass',)], [])]),
-('Module', [('If', ('Name', 'v', ('Load',)), [('Pass',)], [])]),
-('Module', [('Raise', ('Name', 'Exception', ('Load',)), ('Str', 'string'), None)]),
-('Module', [('TryExcept', [('Pass',)], [('excepthandler', ('Name', 'Exception', ('Load',)), None, [('Pass',)])], [])]),
-('Module', [('TryFinally', [('Pass',)], [('Pass',)])]),
-('Module', [('Assert', ('Name', 'v', ('Load',)), None)]),
-('Module', [('Import', [('alias', 'sys', None)])]),
-('Module', [('ImportFrom', 'sys', [('alias', 'v', None)], 0)]),
-('Module', [('Exec', ('Str', 'v'), None, None)]),
-('Module', [('Global', ['v'])]),
-('Module', [('Expr', ('Num', 1))]),
-('Module', [('Pass',)]),
-('Module', [('Break',)]),
-('Module', [('Continue',)]),
+('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [], None, None, []), [('Pass', (1, 9))], [])]),
+('Module', [('ClassDef', (1, 0), 'C', [], [('Pass', (1, 8))])]),
+('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [], None, None, []), [('Return', (1, 8), ('Num', (1, 15), 1))], [])]),
+('Module', [('Delete', (1, 0), [('Name', (1, 4), 'v', ('Del',))])]),
+('Module', [('Assign', (1, 0), [('Name', (1, 0), 'v', ('Store',))], ('Num', (1, 4), 1))]),
+('Module', [('AugAssign', (1, 0), ('Name', (1, 0), 'v', ('Load',)), ('Add',), ('Num', (1, 5), 1))]),
+('Module', [('Print', (1, 0), ('Name', (1, 8), 'f', ('Load',)), [('Num', (1, 11), 1)], False)]),
+('Module', [('For', (1, 0), ('Name', (1, 4), 'v', ('Store',)), ('Name', (1, 9), 'v', ('Load',)), [('Pass', (1, 11))], [])]),
+('Module', [('While', (1, 0), ('Name', (1, 6), 'v', ('Load',)), [('Pass', (1, 8))], [])]),
+('Module', [('If', (1, 0), ('Name', (1, 3), 'v', ('Load',)), [('Pass', (1, 5))], [])]),
+('Module', [('Raise', (1, 0), ('Name', (1, 6), 'Exception', ('Load',)), ('Str', (1, 17), 'string'), None)]),
+('Module', [('TryExcept', (1, 0), [('Pass', (2, 2))], [('excepthandler', ('Name', (3, 7), 'Exception', ('Load',)), None, [('Pass', (4, 2))])], [])]),
+('Module', [('TryFinally', (1, 0), [('Pass', (2, 2))], [('Pass', (4, 2))])]),
+('Module', [('Assert', (1, 0), ('Name', (1, 7), 'v', ('Load',)), None)]),
+('Module', [('Import', (1, 0), [('alias', 'sys', None)])]),
+('Module', [('ImportFrom', (1, 0), 'sys', [('alias', 'v', None)], 0)]),
+('Module', [('Exec', (1, 0), ('Str', (1, 5), 'v'), None, None)]),
+('Module', [('Global', (1, 0), ['v'])]),
+('Module', [('Expr', (1, 0), ('Num', (1, 0), 1))]),
+('Module', [('Pass', (1, 0))]),
+('Module', [('Break', (1, 0))]),
+('Module', [('Continue', (1, 0))]),
 ]
 single_results = [
-('Interactive', [('Expr', ('BinOp', ('Num', 1), ('Add',), ('Num', 2)))]),
+('Interactive', [('Expr', (1, 0), ('BinOp', (1, 0), ('Num', (1, 0), 1), ('Add',), ('Num', (1, 2), 2)))]),
 ]
 eval_results = [
-('Expression', ('BoolOp', ('And',), [('Name', 'a', ('Load',)), ('Name', 'b', ('Load',))])),
-('Expression', ('BinOp', ('Name', 'a', ('Load',)), ('Add',), ('Name', 'b', ('Load',)))),
-('Expression', ('UnaryOp', ('Not',), ('Name', 'v', ('Load',)))),
-('Expression', ('Lambda', ('arguments', [], None, None, []), ('Name', 'None', ('Load',)))),
-('Expression', ('Dict', [('Num', 1)], [('Num', 2)])),
-('Expression', ('ListComp', ('Name', 'a', ('Load',)), [('comprehension', ('Name', 'b', ('Store',)), ('Name', 'c', ('Load',)), [('Name', 'd', ('Load',))])])),
-('Expression', ('GeneratorExp', ('Name', 'a', ('Load',)), [('comprehension', ('Name', 'b', ('Store',)), ('Name', 'c', ('Load',)), [('Name', 'd', ('Load',))])])),
-('Expression', ('Compare', ('Num', 1), [('Lt',), ('Lt',)], [('Num', 2), ('Num', 3)])),
-('Expression', ('Call', ('Name', 'f', ('Load',)), [('Num', 1), ('Num', 2)], [('keyword', 'c', ('Num', 3))], ('Name', 'd', ('Load',)), ('Name', 'e', ('Load',)))),
-('Expression', ('Repr', ('Name', 'v', ('Load',)))),
-('Expression', ('Num', 10L)),
-('Expression', ('Str', 'string')),
-('Expression', ('Attribute', ('Name', 'a', ('Load',)), 'b', ('Load',))),
-('Expression', ('Subscript', ('Name', 'a', ('Load',)), ('Slice', ('Name', 'b', ('Load',)), ('Name', 'c', ('Load',)), None), ('Load',))),
-('Expression', ('Name', 'v', ('Load',))),
-('Expression', ('List', [('Num', 1), ('Num', 2), ('Num', 3)], ('Load',))),
-('Expression', ('Tuple', [('Num', 1), ('Num', 2), ('Num', 3)], ('Load',))),
+('Expression', ('BoolOp', (1, 0), ('And',), [('Name', (1, 0), 'a', ('Load',)), ('Name', (1, 6), 'b', ('Load',))])),
+('Expression', ('BinOp', (1, 0), ('Name', (1, 0), 'a', ('Load',)), ('Add',), ('Name', (1, 4), 'b', ('Load',)))),
+('Expression', ('UnaryOp', (1, 0), ('Not',), ('Name', (1, 4), 'v', ('Load',)))),
+('Expression', ('Lambda', (1, 0), ('arguments', [], None, None, []), ('Name', (1, 7), 'None', ('Load',)))),
+('Expression', ('Dict', (1, 0), [('Num', (1, 2), 1)], [('Num', (1, 4), 2)])),
+('Expression', ('ListComp', (1, 1), ('Name', (1, 1), 'a', ('Load',)), [('comprehension', ('Name', (1, 7), 'b', ('Store',)), ('Name', (1, 12), 'c', ('Load',)), [('Name', (1, 17), 'd', ('Load',))])])),
+('Expression', ('GeneratorExp', (1, 1), ('Name', (1, 1), 'a', ('Load',)), [('comprehension', ('Name', (1, 7), 'b', ('Store',)), ('Name', (1, 12), 'c', ('Load',)), [('Name', (1, 17), 'd', ('Load',))])])),
+('Expression', ('Compare', (1, 0), ('Num', (1, 0), 1), [('Lt',), ('Lt',)], [('Num', (1, 4), 2), ('Num', (1, 8), 3)])),
+('Expression', ('Call', (1, 0), ('Name', (1, 0), 'f', ('Load',)), [('Num', (1, 2), 1), ('Num', (1, 4), 2)], [('keyword', 'c', ('Num', (1, 8), 3))], ('Name', (1, 11), 'd', ('Load',)), ('Name', (1, 15), 'e', ('Load',)))),
+('Expression', ('Repr', (1, 0), ('Name', (1, 1), 'v', ('Load',)))),
+('Expression', ('Num', (1, 0), 10L)),
+('Expression', ('Str', (1, 0), 'string')),
+('Expression', ('Attribute', (1, 0), ('Name', (1, 0), 'a', ('Load',)), 'b', ('Load',))),
+('Expression', ('Subscript', (1, 0), ('Name', (1, 0), 'a', ('Load',)), ('Slice', ('Name', (1, 2), 'b', ('Load',)), ('Name', (1, 4), 'c', ('Load',)), None), ('Load',))),
+('Expression', ('Name', (1, 0), 'v', ('Load',))),
+('Expression', ('List', (1, 0), [('Num', (1, 1), 1), ('Num', (1, 3), 2), ('Num', (1, 5), 3)], ('Load',))),
+('Expression', ('Tuple', (1, 0), [('Num', (1, 0), 1), ('Num', (1, 2), 2), ('Num', (1, 4), 3)], ('Load',))),
+('Expression', ('Call', (1, 0), ('Attribute', (1, 0), ('Attribute', (1, 0), ('Attribute', (1, 0), ('Name', (1, 0), 'a', ('Load',)), 'b', ('Load',)), 'c', ('Load',)), 'd', ('Load',)), [('Subscript', (1, 8), ('Attribute', (1, 8), ('Name', (1, 8), 'a', ('Load',)), 'b', ('Load',)), ('Slice', ('Num', (1, 12), 1), ('Num', (1, 14), 2), None), ('Load',))], [], None, None)),
 ]
 run_tests()

Modules/parsermodule.c

@@ -715,7 +715,7 @@ build_node_children(PyObject *tuple, node *root, int *line_num)
             Py_XDECREF(elem);
             return (0);
         }
-        err = PyNode_AddChild(root, type, strn, *line_num);
+        err = PyNode_AddChild(root, type, strn, *line_num, 0);
         if (err == E_NOMEM) {
             PyMem_DEL(strn);
             return (node *) PyErr_NoMemory();

Parser/Python.asdl

@@ -46,7 +46,8 @@ module Python version "$Revision$"
 	      | Pass | Break | Continue
 
 	      -- XXX Jython will be different
-	      attributes (int lineno)
+	      -- col_offset is the byte offset in the utf8 string the parser uses
+	      attributes (int lineno, int col_offset)
 
 	      -- BoolOp() can use left & right?
 	expr = BoolOp(boolop op, expr* values)
@@ -76,7 +77,8 @@ module Python version "$Revision$"
 	     | List(expr* elts, expr_context ctx) 
 	     | Tuple(expr *elts, expr_context ctx)
 
-	      attributes (int lineno)
+	      -- col_offset is the byte offset in the utf8 string the parser uses
+	      attributes (int lineno, int col_offset)
 
 	expr_context = Load | Store | Del | AugLoad | AugStore | Param
 

Parser/asdl.py

@@ -156,6 +156,8 @@ class ASDLParser(spark.GenericParser, object):
         if id.value != "attributes":
             raise ASDLSyntaxError(id.lineno,
                                   msg="expected attributes, found %s" % id)
+        if attributes:
+            attributes.reverse()
         return Sum(sum, attributes)
 
     def p_product(self, (_0, fields, _1)):

Parser/node.c

@@ -76,7 +76,7 @@ fancy_roundup(int n)
 
 
 int
-PyNode_AddChild(register node *n1, int type, char *str, int lineno)
+PyNode_AddChild(register node *n1, int type, char *str, int lineno, int col_offset)
 {
 	const int nch = n1->n_nchildren;
 	int current_capacity;
@@ -103,6 +103,7 @@ PyNode_AddChild(register node *n1, int type, char *str, int lineno)
 	n->n_type = type;
 	n->n_str = str;
 	n->n_lineno = lineno;
+	n->n_col_offset = col_offset;
 	n->n_nchildren = 0;
 	n->n_child = NULL;
 	return 0;

Parser/parser.c

@@ -105,11 +105,11 @@ PyParser_Delete(parser_state *ps)
 /* PARSER STACK OPERATIONS */
 
 static int
-shift(register stack *s, int type, char *str, int newstate, int lineno)
+shift(register stack *s, int type, char *str, int newstate, int lineno, int col_offset)
 {
 	int err;
 	assert(!s_empty(s));
-	err = PyNode_AddChild(s->s_top->s_parent, type, str, lineno);
+	err = PyNode_AddChild(s->s_top->s_parent, type, str, lineno, col_offset);
 	if (err)
 		return err;
 	s->s_top->s_state = newstate;
@@ -117,13 +117,13 @@ shift(register stack *s, int type, char *str, int newstate, int lineno)
 }
 
 static int
-push(register stack *s, int type, dfa *d, int newstate, int lineno)
+push(register stack *s, int type, dfa *d, int newstate, int lineno, int col_offset)
 {
 	int err;
 	register node *n;
 	n = s->s_top->s_parent;
 	assert(!s_empty(s));
-	err = PyNode_AddChild(n, type, (char *)NULL, lineno);
+	err = PyNode_AddChild(n, type, (char *)NULL, lineno, col_offset);
 	if (err)
 		return err;
 	s->s_top->s_state = newstate;
@@ -213,7 +213,7 @@ future_hack(parser_state *ps)
 
 int
 PyParser_AddToken(register parser_state *ps, register int type, char *str,
-	          int lineno, int *expected_ret)
+	          int lineno, int col_offset, int *expected_ret)
 {
 	register int ilabel;
 	int err;
@@ -245,7 +245,7 @@ PyParser_AddToken(register parser_state *ps, register int type, char *str,
 					dfa *d1 = PyGrammar_FindDFA(
 						ps->p_grammar, nt);
 					if ((err = push(&ps->p_stack, nt, d1,
-						arrow, lineno)) > 0) {
+						arrow, lineno, col_offset)) > 0) {
 						D(printf(" MemError: push\n"));
 						return err;
 					}
@@ -255,7 +255,7 @@ PyParser_AddToken(register parser_state *ps, register int type, char *str,
 				
 				/* Shift the token */
 				if ((err = shift(&ps->p_stack, type, str,
-						x, lineno)) > 0) {
+						x, lineno, col_offset)) > 0) {
 					D(printf(" MemError: shift.\n"));
 					return err;
 				}

Parser/parser.h

@@ -32,7 +32,7 @@ typedef struct {
 
 parser_state *PyParser_New(grammar *g, int start);
 void PyParser_Delete(parser_state *ps);
-int PyParser_AddToken(parser_state *ps, int type, char *str, int lineno,
+int PyParser_AddToken(parser_state *ps, int type, char *str, int lineno, int col_offset,
                       int *expected_ret);
 void PyGrammar_AddAccelerators(grammar *g);
 

Parser/parsetok.c

@@ -130,6 +130,7 @@ parsetok(struct tok_state *tok, grammar *g, int start, perrdetail *err_ret,
 		int type;
 		size_t len;
 		char *str;
+		int col_offset;
 
 		type = PyTokenizer_Get(tok, &a, &b);
 		if (type == ERRORTOKEN) {
@@ -185,9 +186,13 @@ parsetok(struct tok_state *tok, grammar *g, int start, perrdetail *err_ret,
 			 len == 4 && str[0] == 'w' && strcmp(str, "with") == 0)
 			handling_with = 1;
 #endif
-
+		if (a >= tok->line_start)
+			col_offset = a - tok->line_start;
+		else
+			col_offset = -1;
+			
 		if ((err_ret->error =
-		     PyParser_AddToken(ps, (int)type, str, tok->lineno,
+		     PyParser_AddToken(ps, (int)type, str, tok->lineno, col_offset,
 				       &(err_ret->expected))) != E_OK) {
 			if (err_ret->error != E_DONE)
 				PyObject_FREE(str);

Parser/tokenizer.c

@@ -764,6 +764,7 @@ tok_nextc(register struct tok_state *tok)
 			}
 			if (tok->start == NULL)
 				tok->buf = tok->cur;
+			tok->line_start = tok->cur;
 			tok->lineno++;
 			tok->inp = end;
 			return Py_CHARMASK(*tok->cur++);
@@ -798,6 +799,7 @@ tok_nextc(register struct tok_state *tok)
 				}
 				tok->buf = buf;
 				tok->cur = tok->buf + oldlen;
+				tok->line_start = tok->cur;
 				strcpy(tok->buf + oldlen, new);
 				PyMem_FREE(new);
 				tok->inp = tok->buf + newlen;
@@ -809,7 +811,9 @@ tok_nextc(register struct tok_state *tok)
 				if (tok->buf != NULL)
 					PyMem_DEL(tok->buf);
 				tok->buf = new;
+				tok->line_start = tok->buf;
 				tok->cur = tok->buf;
+				tok->line_start = tok->buf;
 				tok->inp = strchr(tok->buf, '\0');
 				tok->end = tok->inp + 1;
 			}
@@ -877,6 +881,7 @@ tok_nextc(register struct tok_state *tok)
 				done = tok->inp[-1] == '\n';
 			}
 			tok->cur = tok->buf + cur;
+			tok->line_start = tok->cur;
 			/* replace "\r\n" with "\n" */
 			/* For Mac we leave the \r, giving a syntax error */
 			pt = tok->inp - 2;

Parser/tokenizer.h

@@ -45,6 +45,7 @@ struct tok_state {
 	int read_coding_spec;	/* whether 'coding:...' has been read  */
 	char *encoding;
 	int cont_line;          /* whether we are in a continuation line. */
+	const char* line_start;	/* pointer to start of current line */
 #ifndef PGEN
 	PyObject *decoding_readline; /* codecs.open(...).readline */
 	PyObject *decoding_buffer;

Python/compile.c

@@ -3635,7 +3635,7 @@ compiler_augassign(struct compiler *c, stmt_ty s)
 	switch (e->kind) {
 		case Attribute_kind:
 		auge = Attribute(e->v.Attribute.value, e->v.Attribute.attr,
-				 AugLoad, e->lineno, c->c_arena);
+				 AugLoad, e->lineno, e->col_offset, c->c_arena);
 		if (auge == NULL)
 		    return 0;
 		VISIT(c, expr, auge);
@@ -3646,7 +3646,7 @@ compiler_augassign(struct compiler *c, stmt_ty s)
 		break;
 	case Subscript_kind:
 		auge = Subscript(e->v.Subscript.value, e->v.Subscript.slice,
-				 AugLoad, e->lineno, c->c_arena);
+				 AugLoad, e->lineno, e->col_offset, c->c_arena);
 		if (auge == NULL)
 		    return 0;
 		VISIT(c, expr, auge);