Patch #1440601: Add col_offset attribute to AST nodes.

Doc/lib/libast.tex

@@ -35,6 +35,12 @@ names of all child nodes.
 Each instance of a concrete class has one attribute for each child node,
 of the type as defined in the grammar. For example, \code{_ast.BinOp}
 instances have an attribute \code{left} of type \code{_ast.expr}.  
+Instances of \code{_ast.expr} and \code{_ast.stmt} subclasses also
+have lineno and col_offset attributes.  The lineno is the line number
+of source text (1 indexed so the first line is line 1) and the
+col_offset is the utf8 byte offset of the first token that generated
+the node.  The utf8 offset is recorded because the parser uses utf8 
+internally.
 
 If these attributes are marked as optional in the grammar (using a
 question mark), the value might be \code{None}. If the attributes

Include/Python-ast.h

@@ -178,6 +178,7 @@ struct _stmt {
                 
         } v;
         int lineno;
+        int col_offset;
 };
 
 struct _expr {
@@ -288,6 +289,7 @@ struct _expr {
                 
         } v;
         int lineno;
+        int col_offset;
 };
 
 struct _slice {
@@ -346,68 +348,79 @@ mod_ty Interactive(asdl_seq * body, PyArena *arena);
 mod_ty Expression(expr_ty body, PyArena *arena);
 mod_ty Suite(asdl_seq * body, PyArena *arena);
 stmt_ty FunctionDef(identifier name, arguments_ty args, asdl_seq * body,
-                    asdl_seq * decorators, int lineno, PyArena *arena);
+                    asdl_seq * decorators, int lineno, int col_offset, PyArena
+                    *arena);
 stmt_ty ClassDef(identifier name, asdl_seq * bases, asdl_seq * body, int
-                 lineno, PyArena *arena);
-stmt_ty Return(expr_ty value, int lineno, PyArena *arena);
-stmt_ty Delete(asdl_seq * targets, int lineno, PyArena *arena);
-stmt_ty Assign(asdl_seq * targets, expr_ty value, int lineno, PyArena *arena);
-stmt_ty AugAssign(expr_ty target, operator_ty op, expr_ty value, int lineno,
+                 lineno, int col_offset, PyArena *arena);
+stmt_ty Return(expr_ty value, int lineno, int col_offset, PyArena *arena);
+stmt_ty Delete(asdl_seq * targets, int lineno, int col_offset, PyArena *arena);
+stmt_ty Assign(asdl_seq * targets, expr_ty value, int lineno, int col_offset,
                PyArena *arena);
-stmt_ty Print(expr_ty dest, asdl_seq * values, bool nl, int lineno, PyArena
-              *arena);
+stmt_ty AugAssign(expr_ty target, operator_ty op, expr_ty value, int lineno,
+                  int col_offset, PyArena *arena);
+stmt_ty Print(expr_ty dest, asdl_seq * values, bool nl, int lineno, int
+              col_offset, PyArena *arena);
 stmt_ty For(expr_ty target, expr_ty iter, asdl_seq * body, asdl_seq * orelse,
-            int lineno, PyArena *arena);
-stmt_ty While(expr_ty test, asdl_seq * body, asdl_seq * orelse, int lineno,
-              PyArena *arena);
-stmt_ty If(expr_ty test, asdl_seq * body, asdl_seq * orelse, int lineno,
-           PyArena *arena);
+            int lineno, int col_offset, PyArena *arena);
+stmt_ty While(expr_ty test, asdl_seq * body, asdl_seq * orelse, int lineno, int
+              col_offset, PyArena *arena);
+stmt_ty If(expr_ty test, asdl_seq * body, asdl_seq * orelse, int lineno, int
+           col_offset, PyArena *arena);
 stmt_ty With(expr_ty context_expr, expr_ty optional_vars, asdl_seq * body, int
-             lineno, PyArena *arena);
-stmt_ty Raise(expr_ty type, expr_ty inst, expr_ty tback, int lineno, PyArena
-              *arena);
+             lineno, int col_offset, PyArena *arena);
+stmt_ty Raise(expr_ty type, expr_ty inst, expr_ty tback, int lineno, int
+              col_offset, PyArena *arena);
 stmt_ty TryExcept(asdl_seq * body, asdl_seq * handlers, asdl_seq * orelse, int
-                  lineno, PyArena *arena);
-stmt_ty TryFinally(asdl_seq * body, asdl_seq * finalbody, int lineno, PyArena
+                  lineno, int col_offset, PyArena *arena);
+stmt_ty TryFinally(asdl_seq * body, asdl_seq * finalbody, int lineno, int
+                   col_offset, PyArena *arena);
+stmt_ty Assert(expr_ty test, expr_ty msg, int lineno, int col_offset, PyArena
                *arena);
-stmt_ty Assert(expr_ty test, expr_ty msg, int lineno, PyArena *arena);
-stmt_ty Import(asdl_seq * names, int lineno, PyArena *arena);
+stmt_ty Import(asdl_seq * names, int lineno, int col_offset, PyArena *arena);
 stmt_ty ImportFrom(identifier module, asdl_seq * names, int level, int lineno,
+                   int col_offset, PyArena *arena);
+stmt_ty Exec(expr_ty body, expr_ty globals, expr_ty locals, int lineno, int
+             col_offset, PyArena *arena);
+stmt_ty Global(asdl_seq * names, int lineno, int col_offset, PyArena *arena);
+stmt_ty Expr(expr_ty value, int lineno, int col_offset, PyArena *arena);
+stmt_ty Pass(int lineno, int col_offset, PyArena *arena);
+stmt_ty Break(int lineno, int col_offset, PyArena *arena);
+stmt_ty Continue(int lineno, int col_offset, PyArena *arena);
+expr_ty BoolOp(boolop_ty op, asdl_seq * values, int lineno, int col_offset,
                PyArena *arena);
-stmt_ty Exec(expr_ty body, expr_ty globals, expr_ty locals, int lineno, PyArena
-             *arena);
-stmt_ty Global(asdl_seq * names, int lineno, PyArena *arena);
-stmt_ty Expr(expr_ty value, int lineno, PyArena *arena);
-stmt_ty Pass(int lineno, PyArena *arena);
-stmt_ty Break(int lineno, PyArena *arena);
-stmt_ty Continue(int lineno, PyArena *arena);
-expr_ty BoolOp(boolop_ty op, asdl_seq * values, int lineno, PyArena *arena);
-expr_ty BinOp(expr_ty left, operator_ty op, expr_ty right, int lineno, PyArena
-              *arena);
-expr_ty UnaryOp(unaryop_ty op, expr_ty operand, int lineno, PyArena *arena);
-expr_ty Lambda(arguments_ty args, expr_ty body, int lineno, PyArena *arena);
-expr_ty IfExp(expr_ty test, expr_ty body, expr_ty orelse, int lineno, PyArena
-              *arena);
-expr_ty Dict(asdl_seq * keys, asdl_seq * values, int lineno, PyArena *arena);
-expr_ty ListComp(expr_ty elt, asdl_seq * generators, int lineno, PyArena
-                 *arena);
-expr_ty GeneratorExp(expr_ty elt, asdl_seq * generators, int lineno, PyArena
-                     *arena);
-expr_ty Yield(expr_ty value, int lineno, PyArena *arena);
+expr_ty BinOp(expr_ty left, operator_ty op, expr_ty right, int lineno, int
+              col_offset, PyArena *arena);
+expr_ty UnaryOp(unaryop_ty op, expr_ty operand, int lineno, int col_offset,
+                PyArena *arena);
+expr_ty Lambda(arguments_ty args, expr_ty body, int lineno, int col_offset,
+               PyArena *arena);
+expr_ty IfExp(expr_ty test, expr_ty body, expr_ty orelse, int lineno, int
+              col_offset, PyArena *arena);
+expr_ty Dict(asdl_seq * keys, asdl_seq * values, int lineno, int col_offset,
+             PyArena *arena);
+expr_ty ListComp(expr_ty elt, asdl_seq * generators, int lineno, int
+                 col_offset, PyArena *arena);
+expr_ty GeneratorExp(expr_ty elt, asdl_seq * generators, int lineno, int
+                     col_offset, PyArena *arena);
+expr_ty Yield(expr_ty value, int lineno, int col_offset, PyArena *arena);
 expr_ty Compare(expr_ty left, asdl_seq * ops, asdl_seq * comparators, int
-                lineno, PyArena *arena);
+                lineno, int col_offset, PyArena *arena);
 expr_ty Call(expr_ty func, asdl_seq * args, asdl_seq * keywords, expr_ty
-             starargs, expr_ty kwargs, int lineno, PyArena *arena);
-expr_ty Repr(expr_ty value, int lineno, PyArena *arena);
-expr_ty Num(object n, int lineno, PyArena *arena);
-expr_ty Str(string s, int lineno, PyArena *arena);
+             starargs, expr_ty kwargs, int lineno, int col_offset, PyArena
+             *arena);
+expr_ty Repr(expr_ty value, int lineno, int col_offset, PyArena *arena);
+expr_ty Num(object n, int lineno, int col_offset, PyArena *arena);
+expr_ty Str(string s, int lineno, int col_offset, PyArena *arena);
 expr_ty Attribute(expr_ty value, identifier attr, expr_context_ty ctx, int
-                  lineno, PyArena *arena);
+                  lineno, int col_offset, PyArena *arena);
 expr_ty Subscript(expr_ty value, slice_ty slice, expr_context_ty ctx, int
-                  lineno, PyArena *arena);
-expr_ty Name(identifier id, expr_context_ty ctx, int lineno, PyArena *arena);
-expr_ty List(asdl_seq * elts, expr_context_ty ctx, int lineno, PyArena *arena);
-expr_ty Tuple(asdl_seq * elts, expr_context_ty ctx, int lineno, PyArena *arena);
+                  lineno, int col_offset, PyArena *arena);
+expr_ty Name(identifier id, expr_context_ty ctx, int lineno, int col_offset,
+             PyArena *arena);
+expr_ty List(asdl_seq * elts, expr_context_ty ctx, int lineno, int col_offset,
+             PyArena *arena);
+expr_ty Tuple(asdl_seq * elts, expr_context_ty ctx, int lineno, int col_offset,
+              PyArena *arena);
 slice_ty Ellipsis(PyArena *arena);
 slice_ty Slice(expr_ty lower, expr_ty upper, expr_ty step, PyArena *arena);
 slice_ty ExtSlice(asdl_seq * dims, PyArena *arena);

Include/node.h

@@ -11,13 +11,14 @@ typedef struct _node {
     short		n_type;
     char		*n_str;
     int			n_lineno;
+    int			n_col_offset;
     int			n_nchildren;
     struct _node	*n_child;
 } node;
 
 PyAPI_FUNC(node *) PyNode_New(int type);
 PyAPI_FUNC(int) PyNode_AddChild(node *n, int type,
-                                      char *str, int lineno);
+                                      char *str, int lineno, int col_offset);
 PyAPI_FUNC(void) PyNode_Free(node *n);
 
 /* Node access functions */

Lib/test/test_ast.py

 import sys, itertools
+import _ast
 
 def to_tuple(t):
     if t is None or isinstance(t, (basestring, int, long, complex)):
@@ -6,6 +7,8 @@ def to_tuple(t):
     elif isinstance(t, list):
         return [to_tuple(e) for e in t]
     result = [t.__class__.__name__]
+    if hasattr(t, 'lineno') and hasattr(t, 'col_offset'):
+        result.append((t.lineno, t.col_offset))
     if t._fields is None:
         return tuple(result)
     for f in t._fields:
@@ -106,7 +109,10 @@ eval_tests = [
   # List
   "[1,2,3]",
   # Tuple
-  "1,2,3"
+  "1,2,3",
+  # Combination
+  "a.b.c.d(a.b[1:2])",
+
 ]
 
 # TODO: expr_context, slice, boolop, operator, unaryop, cmpop, comprehension
@@ -121,58 +127,77 @@ if __name__=='__main__' and sys.argv[1:] == ['-g']:
     print "run_tests()"
     raise SystemExit
 
+def test_order(ast_node, parent_pos):
+    
+    if not isinstance(ast_node, _ast.AST) or ast_node._fields == None:
+        return
+    if isinstance(ast_node, (_ast.expr, _ast.stmt)):
+        node_pos = (ast_node.lineno, ast_node.col_offset)
+        assert node_pos >= parent_pos, (node_pos, parent_pos)
+        parent_pos = (ast_node.lineno, ast_node.col_offset)
+    for name in ast_node._fields:
+        value = getattr(ast_node, name)
+        if isinstance(value, list):
+            for child in value:
+                test_order(child, parent_pos)
+        elif value != None:
+            test_order(value, parent_pos)
+
 def run_tests():
     for input, output, kind in ((exec_tests, exec_results, "exec"),
                                         (single_tests, single_results, "single"),
                                         (eval_tests, eval_results, "eval")):
         for i, o in itertools.izip(input, output):
-            assert to_tuple(compile(i, "?", kind, 0x400)) == o
+            ast_tree = compile(i, "?", kind, 0x400)
+            assert to_tuple(ast_tree) == o
+            test_order(ast_tree, (0, 0))
 
 #### EVERYTHING BELOW IS GENERATED #####
 exec_results = [
-('Module', [('FunctionDef', 'f', ('arguments', [], None, None, []), [('Pass',)], [])]),
-('Module', [('ClassDef', 'C', [], [('Pass',)])]),
-('Module', [('FunctionDef', 'f', ('arguments', [], None, None, []), [('Return', ('Num', 1))], [])]),
-('Module', [('Delete', [('Name', 'v', ('Del',))])]),
-('Module', [('Assign', [('Name', 'v', ('Store',))], ('Num', 1))]),
-('Module', [('AugAssign', ('Name', 'v', ('Load',)), ('Add',), ('Num', 1))]),
-('Module', [('Print', ('Name', 'f', ('Load',)), [('Num', 1)], False)]),
-('Module', [('For', ('Name', 'v', ('Store',)), ('Name', 'v', ('Load',)), [('Pass',)], [])]),
-('Module', [('While', ('Name', 'v', ('Load',)), [('Pass',)], [])]),
-('Module', [('If', ('Name', 'v', ('Load',)), [('Pass',)], [])]),
-('Module', [('Raise', ('Name', 'Exception', ('Load',)), ('Str', 'string'), None)]),
-('Module', [('TryExcept', [('Pass',)], [('excepthandler', ('Name', 'Exception', ('Load',)), None, [('Pass',)])], [])]),
-('Module', [('TryFinally', [('Pass',)], [('Pass',)])]),
-('Module', [('Assert', ('Name', 'v', ('Load',)), None)]),
-('Module', [('Import', [('alias', 'sys', None)])]),
-('Module', [('ImportFrom', 'sys', [('alias', 'v', None)], 0)]),
-('Module', [('Exec', ('Str', 'v'), None, None)]),
-('Module', [('Global', ['v'])]),
-('Module', [('Expr', ('Num', 1))]),
-('Module', [('Pass',)]),
-('Module', [('Break',)]),
-('Module', [('Continue',)]),
+('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [], None, None, []), [('Pass', (1, 9))], [])]),
+('Module', [('ClassDef', (1, 0), 'C', [], [('Pass', (1, 8))])]),
+('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [], None, None, []), [('Return', (1, 8), ('Num', (1, 15), 1))], [])]),
+('Module', [('Delete', (1, 0), [('Name', (1, 4), 'v', ('Del',))])]),
+('Module', [('Assign', (1, 0), [('Name', (1, 0), 'v', ('Store',))], ('Num', (1, 4), 1))]),
+('Module', [('AugAssign', (1, 0), ('Name', (1, 0), 'v', ('Load',)), ('Add',), ('Num', (1, 5), 1))]),
+('Module', [('Print', (1, 0), ('Name', (1, 8), 'f', ('Load',)), [('Num', (1, 11), 1)], False)]),
+('Module', [('For', (1, 0), ('Name', (1, 4), 'v', ('Store',)), ('Name', (1, 9), 'v', ('Load',)), [('Pass', (1, 11))], [])]),
+('Module', [('While', (1, 0), ('Name', (1, 6), 'v', ('Load',)), [('Pass', (1, 8))], [])]),
+('Module', [('If', (1, 0), ('Name', (1, 3), 'v', ('Load',)), [('Pass', (1, 5))], [])]),
+('Module', [('Raise', (1, 0), ('Name', (1, 6), 'Exception', ('Load',)), ('Str', (1, 17), 'string'), None)]),
+('Module', [('TryExcept', (1, 0), [('Pass', (2, 2))], [('excepthandler', ('Name', (3, 7), 'Exception', ('Load',)), None, [('Pass', (4, 2))])], [])]),
+('Module', [('TryFinally', (1, 0), [('Pass', (2, 2))], [('Pass', (4, 2))])]),
+('Module', [('Assert', (1, 0), ('Name', (1, 7), 'v', ('Load',)), None)]),
+('Module', [('Import', (1, 0), [('alias', 'sys', None)])]),
+('Module', [('ImportFrom', (1, 0), 'sys', [('alias', 'v', None)], 0)]),
+('Module', [('Exec', (1, 0), ('Str', (1, 5), 'v'), None, None)]),
+('Module', [('Global', (1, 0), ['v'])]),
+('Module', [('Expr', (1, 0), ('Num', (1, 0), 1))]),
+('Module', [('Pass', (1, 0))]),
+('Module', [('Break', (1, 0))]),
+('Module', [('Continue', (1, 0))]),
 ]
 single_results = [
-('Interactive', [('Expr', ('BinOp', ('Num', 1), ('Add',), ('Num', 2)))]),
+('Interactive', [('Expr', (1, 0), ('BinOp', (1, 0), ('Num', (1, 0), 1), ('Add',), ('Num', (1, 2), 2)))]),
 ]
 eval_results = [
-('Expression', ('BoolOp', ('And',), [('Name', 'a', ('Load',)), ('Name', 'b', ('Load',))])),
-('Expression', ('BinOp', ('Name', 'a', ('Load',)), ('Add',), ('Name', 'b', ('Load',)))),
-('Expression', ('UnaryOp', ('Not',), ('Name', 'v', ('Load',)))),
-('Expression', ('Lambda', ('arguments', [], None, None, []), ('Name', 'None', ('Load',)))),
-('Expression', ('Dict', [('Num', 1)], [('Num', 2)])),
-('Expression', ('ListComp', ('Name', 'a', ('Load',)), [('comprehension', ('Name', 'b', ('Store',)), ('Name', 'c', ('Load',)), [('Name', 'd', ('Load',))])])),
-('Expression', ('GeneratorExp', ('Name', 'a', ('Load',)), [('comprehension', ('Name', 'b', ('Store',)), ('Name', 'c', ('Load',)), [('Name', 'd', ('Load',))])])),
-('Expression', ('Compare', ('Num', 1), [('Lt',), ('Lt',)], [('Num', 2), ('Num', 3)])),
-('Expression', ('Call', ('Name', 'f', ('Load',)), [('Num', 1), ('Num', 2)], [('keyword', 'c', ('Num', 3))], ('Name', 'd', ('Load',)), ('Name', 'e', ('Load',)))),
-('Expression', ('Repr', ('Name', 'v', ('Load',)))),
-('Expression', ('Num', 10L)),
-('Expression', ('Str', 'string')),
-('Expression', ('Attribute', ('Name', 'a', ('Load',)), 'b', ('Load',))),
-('Expression', ('Subscript', ('Name', 'a', ('Load',)), ('Slice', ('Name', 'b', ('Load',)), ('Name', 'c', ('Load',)), None), ('Load',))),
-('Expression', ('Name', 'v', ('Load',))),
-('Expression', ('List', [('Num', 1), ('Num', 2), ('Num', 3)], ('Load',))),
-('Expression', ('Tuple', [('Num', 1), ('Num', 2), ('Num', 3)], ('Load',))),
+('Expression', ('BoolOp', (1, 0), ('And',), [('Name', (1, 0), 'a', ('Load',)), ('Name', (1, 6), 'b', ('Load',))])),
+('Expression', ('BinOp', (1, 0), ('Name', (1, 0), 'a', ('Load',)), ('Add',), ('Name', (1, 4), 'b', ('Load',)))),
+('Expression', ('UnaryOp', (1, 0), ('Not',), ('Name', (1, 4), 'v', ('Load',)))),
+('Expression', ('Lambda', (1, 0), ('arguments', [], None, None, []), ('Name', (1, 7), 'None', ('Load',)))),
+('Expression', ('Dict', (1, 0), [('Num', (1, 2), 1)], [('Num', (1, 4), 2)])),
+('Expression', ('ListComp', (1, 1), ('Name', (1, 1), 'a', ('Load',)), [('comprehension', ('Name', (1, 7), 'b', ('Store',)), ('Name', (1, 12), 'c', ('Load',)), [('Name', (1, 17), 'd', ('Load',))])])),
+('Expression', ('GeneratorExp', (1, 1), ('Name', (1, 1), 'a', ('Load',)), [('comprehension', ('Name', (1, 7), 'b', ('Store',)), ('Name', (1, 12), 'c', ('Load',)), [('Name', (1, 17), 'd', ('Load',))])])),
+('Expression', ('Compare', (1, 0), ('Num', (1, 0), 1), [('Lt',), ('Lt',)], [('Num', (1, 4), 2), ('Num', (1, 8), 3)])),
+('Expression', ('Call', (1, 0), ('Name', (1, 0), 'f', ('Load',)), [('Num', (1, 2), 1), ('Num', (1, 4), 2)], [('keyword', 'c', ('Num', (1, 8), 3))], ('Name', (1, 11), 'd', ('Load',)), ('Name', (1, 15), 'e', ('Load',)))),
+('Expression', ('Repr', (1, 0), ('Name', (1, 1), 'v', ('Load',)))),
+('Expression', ('Num', (1, 0), 10L)),
+('Expression', ('Str', (1, 0), 'string')),
+('Expression', ('Attribute', (1, 0), ('Name', (1, 0), 'a', ('Load',)), 'b', ('Load',))),
+('Expression', ('Subscript', (1, 0), ('Name', (1, 0), 'a', ('Load',)), ('Slice', ('Name', (1, 2), 'b', ('Load',)), ('Name', (1, 4), 'c', ('Load',)), None), ('Load',))),
+('Expression', ('Name', (1, 0), 'v', ('Load',))),
+('Expression', ('List', (1, 0), [('Num', (1, 1), 1), ('Num', (1, 3), 2), ('Num', (1, 5), 3)], ('Load',))),
+('Expression', ('Tuple', (1, 0), [('Num', (1, 0), 1), ('Num', (1, 2), 2), ('Num', (1, 4), 3)], ('Load',))),
+('Expression', ('Call', (1, 0), ('Attribute', (1, 0), ('Attribute', (1, 0), ('Attribute', (1, 0), ('Name', (1, 0), 'a', ('Load',)), 'b', ('Load',)), 'c', ('Load',)), 'd', ('Load',)), [('Subscript', (1, 8), ('Attribute', (1, 8), ('Name', (1, 8), 'a', ('Load',)), 'b', ('Load',)), ('Slice', ('Num', (1, 12), 1), ('Num', (1, 14), 2), None), ('Load',))], [], None, None)),
 ]
 run_tests()

Modules/parsermodule.c

@@ -715,7 +715,7 @@ build_node_children(PyObject *tuple, node *root, int *line_num)
             Py_XDECREF(elem);
             return (0);
         }
-        err = PyNode_AddChild(root, type, strn, *line_num);
+        err = PyNode_AddChild(root, type, strn, *line_num, 0);
         if (err == E_NOMEM) {
             PyMem_DEL(strn);
             return (node *) PyErr_NoMemory();

Parser/Python.asdl

@@ -46,7 +46,8 @@ module Python version "$Revision$"
 	      | Pass | Break | Continue
 
 	      -- XXX Jython will be different
-	      attributes (int lineno)
+	      -- col_offset is the byte offset in the utf8 string the parser uses
+	      attributes (int lineno, int col_offset)
 
 	      -- BoolOp() can use left & right?
 	expr = BoolOp(boolop op, expr* values)
@@ -76,7 +77,8 @@ module Python version "$Revision$"
 	     | List(expr* elts, expr_context ctx) 
 	     | Tuple(expr *elts, expr_context ctx)
 
-	      attributes (int lineno)
+	      -- col_offset is the byte offset in the utf8 string the parser uses
+	      attributes (int lineno, int col_offset)
 
 	expr_context = Load | Store | Del | AugLoad | AugStore | Param
 

Parser/asdl.py

@@ -156,6 +156,8 @@ class ASDLParser(spark.GenericParser, object):
         if id.value != "attributes":
             raise ASDLSyntaxError(id.lineno,
                                   msg="expected attributes, found %s" % id)
+        if attributes:
+            attributes.reverse()
         return Sum(sum, attributes)
 
     def p_product(self, (_0, fields, _1)):

Parser/node.c

@@ -76,7 +76,7 @@ fancy_roundup(int n)
 
 
 int
-PyNode_AddChild(register node *n1, int type, char *str, int lineno)
+PyNode_AddChild(register node *n1, int type, char *str, int lineno, int col_offset)
 {
 	const int nch = n1->n_nchildren;
 	int current_capacity;
@@ -103,6 +103,7 @@ PyNode_AddChild(register node *n1, int type, char *str, int lineno)
 	n->n_type = type;
 	n->n_str = str;
 	n->n_lineno = lineno;
+	n->n_col_offset = col_offset;
 	n->n_nchildren = 0;
 	n->n_child = NULL;
 	return 0;

Parser/parser.c

@@ -105,11 +105,11 @@ PyParser_Delete(parser_state *ps)
 /* PARSER STACK OPERATIONS */
 
 static int
-shift(register stack *s, int type, char *str, int newstate, int lineno)
+shift(register stack *s, int type, char *str, int newstate, int lineno, int col_offset)
 {
 	int err;
 	assert(!s_empty(s));
-	err = PyNode_AddChild(s->s_top->s_parent, type, str, lineno);
+	err = PyNode_AddChild(s->s_top->s_parent, type, str, lineno, col_offset);
 	if (err)
 		return err;
 	s->s_top->s_state = newstate;
@@ -117,13 +117,13 @@ shift(register stack *s, int type, char *str, int newstate, int lineno)
 }
 
 static int
-push(register stack *s, int type, dfa *d, int newstate, int lineno)
+push(register stack *s, int type, dfa *d, int newstate, int lineno, int col_offset)
 {
 	int err;
 	register node *n;
 	n = s->s_top->s_parent;
 	assert(!s_empty(s));
-	err = PyNode_AddChild(n, type, (char *)NULL, lineno);
+	err = PyNode_AddChild(n, type, (char *)NULL, lineno, col_offset);
 	if (err)
 		return err;
 	s->s_top->s_state = newstate;
@@ -213,7 +213,7 @@ future_hack(parser_state *ps)
 
 int
 PyParser_AddToken(register parser_state *ps, register int type, char *str,
-	          int lineno, int *expected_ret)
+	          int lineno, int col_offset, int *expected_ret)
 {
 	register int ilabel;
 	int err;
@@ -245,7 +245,7 @@ PyParser_AddToken(register parser_state *ps, register int type, char *str,
 					dfa *d1 = PyGrammar_FindDFA(
 						ps->p_grammar, nt);
 					if ((err = push(&ps->p_stack, nt, d1,
-						arrow, lineno)) > 0) {
+						arrow, lineno, col_offset)) > 0) {
 						D(printf(" MemError: push\n"));
 						return err;
 					}
@@ -255,7 +255,7 @@ PyParser_AddToken(register parser_state *ps, register int type, char *str,
 				
 				/* Shift the token */
 				if ((err = shift(&ps->p_stack, type, str,
-						x, lineno)) > 0) {
+						x, lineno, col_offset)) > 0) {
 					D(printf(" MemError: shift.\n"));
 					return err;
 				}

Parser/parser.h

@@ -32,7 +32,7 @@ typedef struct {
 
 parser_state *PyParser_New(grammar *g, int start);
 void PyParser_Delete(parser_state *ps);
-int PyParser_AddToken(parser_state *ps, int type, char *str, int lineno,
+int PyParser_AddToken(parser_state *ps, int type, char *str, int lineno, int col_offset,
                       int *expected_ret);
 void PyGrammar_AddAccelerators(grammar *g);
 

Parser/parsetok.c

@@ -130,6 +130,7 @@ parsetok(struct tok_state *tok, grammar *g, int start, perrdetail *err_ret,
 		int type;
 		size_t len;
 		char *str;
+		int col_offset;
 
 		type = PyTokenizer_Get(tok, &a, &b);
 		if (type == ERRORTOKEN) {
@@ -185,9 +186,13 @@ parsetok(struct tok_state *tok, grammar *g, int start, perrdetail *err_ret,
 			 len == 4 && str[0] == 'w' && strcmp(str, "with") == 0)
 			handling_with = 1;
 #endif
+		if (a >= tok->line_start)
+			col_offset = a - tok->line_start;
+		else
+			col_offset = -1;
 			
 		if ((err_ret->error =
-		     PyParser_AddToken(ps, (int)type, str, tok->lineno,
+		     PyParser_AddToken(ps, (int)type, str, tok->lineno, col_offset,
 				       &(err_ret->expected))) != E_OK) {
 			if (err_ret->error != E_DONE)
 				PyObject_FREE(str);

Parser/tokenizer.c

@@ -764,6 +764,7 @@ tok_nextc(register struct tok_state *tok)
 			}
 			if (tok->start == NULL)
 				tok->buf = tok->cur;
+			tok->line_start = tok->cur;
 			tok->lineno++;
 			tok->inp = end;
 			return Py_CHARMASK(*tok->cur++);
@@ -798,6 +799,7 @@ tok_nextc(register struct tok_state *tok)
 				}
 				tok->buf = buf;
 				tok->cur = tok->buf + oldlen;
+				tok->line_start = tok->cur;
 				strcpy(tok->buf + oldlen, new);
 				PyMem_FREE(new);
 				tok->inp = tok->buf + newlen;
@@ -809,7 +811,9 @@ tok_nextc(register struct tok_state *tok)
 				if (tok->buf != NULL)
 					PyMem_DEL(tok->buf);
 				tok->buf = new;
+				tok->line_start = tok->buf;
 				tok->cur = tok->buf;
+				tok->line_start = tok->buf;
 				tok->inp = strchr(tok->buf, '\0');
 				tok->end = tok->inp + 1;
 			}
@@ -877,6 +881,7 @@ tok_nextc(register struct tok_state *tok)
 				done = tok->inp[-1] == '\n';
 			}
 			tok->cur = tok->buf + cur;
+			tok->line_start = tok->cur;
 			/* replace "\r\n" with "\n" */
 			/* For Mac we leave the \r, giving a syntax error */
 			pt = tok->inp - 2;

Parser/tokenizer.h

@@ -45,6 +45,7 @@ struct tok_state {
 	int read_coding_spec;	/* whether 'coding:...' has been read  */
 	char *encoding;
 	int cont_line;          /* whether we are in a continuation line. */
+	const char* line_start;	/* pointer to start of current line */
 #ifndef PGEN
 	PyObject *decoding_readline; /* codecs.open(...).readline */
 	PyObject *decoding_buffer;

Python/Python-ast.c

@@ -25,6 +25,7 @@ static char *Suite_fields[]={
 static PyTypeObject *stmt_type;
 static char *stmt_attributes[] = {
         "lineno",
+        "col_offset",
 };
 static PyObject* ast2obj_stmt(void*);
 static PyTypeObject *FunctionDef_type;
@@ -142,6 +143,7 @@ static PyTypeObject *Continue_type;
 static PyTypeObject *expr_type;
 static char *expr_attributes[] = {
         "lineno",
+        "col_offset",
 };
 static PyObject* ast2obj_expr(void*);
 static PyTypeObject *BoolOp_type;
@@ -450,7 +452,7 @@ static int init_types(void)
         if (!Suite_type) return 0;
         stmt_type = make_type("stmt", AST_type, NULL, 0);
         if (!stmt_type) return 0;
-        if (!add_attributes(stmt_type, stmt_attributes, 1)) return 0;
+        if (!add_attributes(stmt_type, stmt_attributes, 2)) return 0;
         FunctionDef_type = make_type("FunctionDef", stmt_type,
                                      FunctionDef_fields, 4);
         if (!FunctionDef_type) return 0;
@@ -502,7 +504,7 @@ static int init_types(void)
         if (!Continue_type) return 0;
         expr_type = make_type("expr", AST_type, NULL, 0);
         if (!expr_type) return 0;
-        if (!add_attributes(expr_type, expr_attributes, 1)) return 0;
+        if (!add_attributes(expr_type, expr_attributes, 2)) return 0;
         BoolOp_type = make_type("BoolOp", expr_type, BoolOp_fields, 2);
         if (!BoolOp_type) return 0;
         BinOp_type = make_type("BinOp", expr_type, BinOp_fields, 3);
@@ -783,7 +785,7 @@ Suite(asdl_seq * body, PyArena *arena)
 
 stmt_ty
 FunctionDef(identifier name, arguments_ty args, asdl_seq * body, asdl_seq *
-            decorators, int lineno, PyArena *arena)
+            decorators, int lineno, int col_offset, PyArena *arena)
 {
         stmt_ty p;
         if (!name) {
@@ -807,12 +809,13 @@ FunctionDef(identifier name, arguments_ty args, asdl_seq * body, asdl_seq *
         p->v.FunctionDef.body = body;
         p->v.FunctionDef.decorators = decorators;
         p->lineno = lineno;
+        p->col_offset = col_offset;
         return p;
 }
 
 stmt_ty
-ClassDef(identifier name, asdl_seq * bases, asdl_seq * body, int lineno,
-         PyArena *arena)
+ClassDef(identifier name, asdl_seq * bases, asdl_seq * body, int lineno, int
+         col_offset, PyArena *arena)
 {
         stmt_ty p;
         if (!name) {
@@ -830,11 +833,12 @@ ClassDef(identifier name, asdl_seq * bases, asdl_seq * body, int lineno,
         p->v.ClassDef.bases = bases;
         p->v.ClassDef.body = body;
         p->lineno = lineno;
+        p->col_offset = col_offset;
         return p;
 }
 
 stmt_ty
-Return(expr_ty value, int lineno, PyArena *arena)
+Return(expr_ty value, int lineno, int col_offset, PyArena *arena)
 {
         stmt_ty p;
         p = (stmt_ty)PyArena_Malloc(arena, sizeof(*p));
@@ -845,11 +849,12 @@ Return(expr_ty value, int lineno, PyArena *arena)
         p->kind = Return_kind;
         p->v.Return.value = value;
         p->lineno = lineno;
+        p->col_offset = col_offset;
         return p;
 }
 
 stmt_ty
-Delete(asdl_seq * targets, int lineno, PyArena *arena)
+Delete(asdl_seq * targets, int lineno, int col_offset, PyArena *arena)
 {
         stmt_ty p;
         p = (stmt_ty)PyArena_Malloc(arena, sizeof(*p));
@@ -860,11 +865,13 @@ Delete(asdl_seq * targets, int lineno, PyArena *arena)
         p->kind = Delete_kind;
         p->v.Delete.targets = targets;
         p->lineno = lineno;
+        p->col_offset = col_offset;
         return p;
 }
 
 stmt_ty
-Assign(asdl_seq * targets, expr_ty value, int lineno, PyArena *arena)
+Assign(asdl_seq * targets, expr_ty value, int lineno, int col_offset, PyArena
+       *arena)
 {
         stmt_ty p;
         if (!value) {
@@ -881,12 +888,13 @@ Assign(asdl_seq * targets, expr_ty value, int lineno, PyArena *arena)
         p->v.Assign.targets = targets;
         p->v.Assign.value = value;
         p->lineno = lineno;
+        p->col_offset = col_offset;
         return p;
 }
 
 stmt_ty
-AugAssign(expr_ty target, operator_ty op, expr_ty value, int lineno, PyArena
-          *arena)
+AugAssign(expr_ty target, operator_ty op, expr_ty value, int lineno, int
+          col_offset, PyArena *arena)
 {
         stmt_ty p;
         if (!target) {
@@ -914,11 +922,13 @@ AugAssign(expr_ty target, operator_ty op, expr_ty value, int lineno, PyArena
         p->v.AugAssign.op = op;
         p->v.AugAssign.value = value;
         p->lineno = lineno;
+        p->col_offset = col_offset;
         return p;
 }
 
 stmt_ty
-Print(expr_ty dest, asdl_seq * values, bool nl, int lineno, PyArena *arena)
+Print(expr_ty dest, asdl_seq * values, bool nl, int lineno, int col_offset,
+      PyArena *arena)
 {
         stmt_ty p;
         p = (stmt_ty)PyArena_Malloc(arena, sizeof(*p));
@@ -931,12 +941,13 @@ Print(expr_ty dest, asdl_seq * values, bool nl, int lineno, PyArena *arena)
         p->v.Print.values = values;
         p->v.Print.nl = nl;
         p->lineno = lineno;
+        p->col_offset = col_offset;
         return p;
 }
 
 stmt_ty
 For(expr_ty target, expr_ty iter, asdl_seq * body, asdl_seq * orelse, int
-    lineno, PyArena *arena)
+    lineno, int col_offset, PyArena *arena)
 {
         stmt_ty p;
         if (!target) {
@@ -960,12 +971,13 @@ For(expr_ty target, expr_ty iter, asdl_seq * body, asdl_seq * orelse, int
         p->v.For.body = body;
         p->v.For.orelse = orelse;
         p->lineno = lineno;
+        p->col_offset = col_offset;
         return p;
 }
 
 stmt_ty
-While(expr_ty test, asdl_seq * body, asdl_seq * orelse, int lineno, PyArena
-      *arena)
+While(expr_ty test, asdl_seq * body, asdl_seq * orelse, int lineno, int
+      col_offset, PyArena *arena)
 {
         stmt_ty p;
         if (!test) {
@@ -983,11 +995,13 @@ While(expr_ty test, asdl_seq * body, asdl_seq * orelse, int lineno, PyArena
         p->v.While.body = body;
         p->v.While.orelse = orelse;
         p->lineno = lineno;
+        p->col_offset = col_offset;
         return p;
 }
 
 stmt_ty
-If(expr_ty test, asdl_seq * body, asdl_seq * orelse, int lineno, PyArena *arena)
+If(expr_ty test, asdl_seq * body, asdl_seq * orelse, int lineno, int
+   col_offset, PyArena *arena)
 {
         stmt_ty p;
         if (!test) {
@@ -1005,12 +1019,13 @@ If(expr_ty test, asdl_seq * body, asdl_seq * orelse, int lineno, PyArena *arena)
         p->v.If.body = body;
         p->v.If.orelse = orelse;
         p->lineno = lineno;
+        p->col_offset = col_offset;
         return p;
 }
 
 stmt_ty
 With(expr_ty context_expr, expr_ty optional_vars, asdl_seq * body, int lineno,
-     PyArena *arena)
+     int col_offset, PyArena *arena)
 {
         stmt_ty p;
         if (!context_expr) {
@@ -1028,11 +1043,13 @@ With(expr_ty context_expr, expr_ty optional_vars, asdl_seq * body, int lineno,
         p->v.With.optional_vars = optional_vars;
         p->v.With.body = body;
         p->lineno = lineno;
+        p->col_offset = col_offset;
         return p;
 }
 
 stmt_ty
-Raise(expr_ty type, expr_ty inst, expr_ty tback, int lineno, PyArena *arena)
+Raise(expr_ty type, expr_ty inst, expr_ty tback, int lineno, int col_offset,
+      PyArena *arena)
 {
         stmt_ty p;
         p = (stmt_ty)PyArena_Malloc(arena, sizeof(*p));
@@ -1045,12 +1062,13 @@ Raise(expr_ty type, expr_ty inst, expr_ty tback, int lineno, PyArena *arena)
         p->v.Raise.inst = inst;
         p->v.Raise.tback = tback;
         p->lineno = lineno;
+        p->col_offset = col_offset;
         return p;
 }
 
 stmt_ty
 TryExcept(asdl_seq * body, asdl_seq * handlers, asdl_seq * orelse, int lineno,
-          PyArena *arena)
+          int col_offset, PyArena *arena)
 {
         stmt_ty p;
         p = (stmt_ty)PyArena_Malloc(arena, sizeof(*p));
@@ -1063,11 +1081,13 @@ TryExcept(asdl_seq * body, asdl_seq * handlers, asdl_seq * orelse, int lineno,
         p->v.TryExcept.handlers = handlers;
         p->v.TryExcept.orelse = orelse;
         p->lineno = lineno;
+        p->col_offset = col_offset;
         return p;
 }
 
 stmt_ty
-TryFinally(asdl_seq * body, asdl_seq * finalbody, int lineno, PyArena *arena)
+TryFinally(asdl_seq * body, asdl_seq * finalbody, int lineno, int col_offset,
+           PyArena *arena)
 {
         stmt_ty p;
         p = (stmt_ty)PyArena_Malloc(arena, sizeof(*p));
@@ -1079,11 +1099,12 @@ TryFinally(asdl_seq * body, asdl_seq * finalbody, int lineno, PyArena *arena)
         p->v.TryFinally.body = body;
         p->v.TryFinally.finalbody = finalbody;
         p->lineno = lineno;
+        p->col_offset = col_offset;
         return p;
 }
 
 stmt_ty
-Assert(expr_ty test, expr_ty msg, int lineno, PyArena *arena)
+Assert(expr_ty test, expr_ty msg, int lineno, int col_offset, PyArena *arena)
 {
         stmt_ty p;
         if (!test) {
@@ -1100,11 +1121,12 @@ Assert(expr_ty test, expr_ty msg, int lineno, PyArena *arena)
         p->v.Assert.test = test;
         p->v.Assert.msg = msg;
         p->lineno = lineno;
+        p->col_offset = col_offset;
         return p;
 }
 
 stmt_ty
-Import(asdl_seq * names, int lineno, PyArena *arena)
+Import(asdl_seq * names, int lineno, int col_offset, PyArena *arena)
 {
         stmt_ty p;
         p = (stmt_ty)PyArena_Malloc(arena, sizeof(*p));
@@ -1115,12 +1137,13 @@ Import(asdl_seq * names, int lineno, PyArena *arena)
         p->kind = Import_kind;
         p->v.Import.names = names;
         p->lineno = lineno;
+        p->col_offset = col_offset;
         return p;
 }
 
 stmt_ty
-ImportFrom(identifier module, asdl_seq * names, int level, int lineno, PyArena
-           *arena)
+ImportFrom(identifier module, asdl_seq * names, int level, int lineno, int
+           col_offset, PyArena *arena)
 {
         stmt_ty p;
         if (!module) {
@@ -1138,11 +1161,13 @@ ImportFrom(identifier module, asdl_seq * names, int level, int lineno, PyArena
         p->v.ImportFrom.names = names;
         p->v.ImportFrom.level = level;
         p->lineno = lineno;
+        p->col_offset = col_offset;
         return p;
 }
 
 stmt_ty
-Exec(expr_ty body, expr_ty globals, expr_ty locals, int lineno, PyArena *arena)
+Exec(expr_ty body, expr_ty globals, expr_ty locals, int lineno, int col_offset,
+     PyArena *arena)
 {
         stmt_ty p;
         if (!body) {
@@ -1160,11 +1185,12 @@ Exec(expr_ty body, expr_ty globals, expr_ty locals, int lineno, PyArena *arena)
         p->v.Exec.globals = globals;
         p->v.Exec.locals = locals;
         p->lineno = lineno;
+        p->col_offset = col_offset;
         return p;
 }
 
 stmt_ty
-Global(asdl_seq * names, int lineno, PyArena *arena)
+Global(asdl_seq * names, int lineno, int col_offset, PyArena *arena)
 {
         stmt_ty p;
         p = (stmt_ty)PyArena_Malloc(arena, sizeof(*p));
@@ -1175,11 +1201,12 @@ Global(asdl_seq * names, int lineno, PyArena *arena)
         p->kind = Global_kind;
         p->v.Global.names = names;
         p->lineno = lineno;
+        p->col_offset = col_offset;
         return p;
 }
 
 stmt_ty
-Expr(expr_ty value, int lineno, PyArena *arena)
+Expr(expr_ty value, int lineno, int col_offset, PyArena *arena)
 {
         stmt_ty p;
         if (!value) {
@@ -1195,11 +1222,12 @@ Expr(expr_ty value, int lineno, PyArena *arena)
         p->kind = Expr_kind;
         p->v.Expr.value = value;
         p->lineno = lineno;
+        p->col_offset = col_offset;
         return p;
 }
 
 stmt_ty
-Pass(int lineno, PyArena *arena)
+Pass(int lineno, int col_offset, PyArena *arena)
 {
         stmt_ty p;
         p = (stmt_ty)PyArena_Malloc(arena, sizeof(*p));
@@ -1209,11 +1237,12 @@ Pass(int lineno, PyArena *arena)
         }
         p->kind = Pass_kind;
         p->lineno = lineno;
+        p->col_offset = col_offset;
         return p;
 }
 
 stmt_ty
-Break(int lineno, PyArena *arena)
+Break(int lineno, int col_offset, PyArena *arena)
 {
         stmt_ty p;
         p = (stmt_ty)PyArena_Malloc(arena, sizeof(*p));
@@ -1223,11 +1252,12 @@ Break(int lineno, PyArena *arena)
         }
         p->kind = Break_kind;
         p->lineno = lineno;
+        p->col_offset = col_offset;
         return p;
 }
 
 stmt_ty
-Continue(int lineno, PyArena *arena)
+Continue(int lineno, int col_offset, PyArena *arena)
 {
         stmt_ty p;
         p = (stmt_ty)PyArena_Malloc(arena, sizeof(*p));
@@ -1237,11 +1267,13 @@ Continue(int lineno, PyArena *arena)
         }
         p->kind = Continue_kind;
         p->lineno = lineno;
+        p->col_offset = col_offset;
         return p;
 }
 
 expr_ty
-BoolOp(boolop_ty op, asdl_seq * values, int lineno, PyArena *arena)
+BoolOp(boolop_ty op, asdl_seq * values, int lineno, int col_offset, PyArena
+       *arena)
 {
         expr_ty p;
         if (!op) {
@@ -1258,11 +1290,13 @@ BoolOp(boolop_ty op, asdl_seq * values, int lineno, PyArena *arena)
         p->v.BoolOp.op = op;
         p->v.BoolOp.values = values;
         p->lineno = lineno;
+        p->col_offset = col_offset;
         return p;
 }
 
 expr_ty
-BinOp(expr_ty left, operator_ty op, expr_ty right, int lineno, PyArena *arena)
+BinOp(expr_ty left, operator_ty op, expr_ty right, int lineno, int col_offset,
+      PyArena *arena)
 {
         expr_ty p;
         if (!left) {
@@ -1290,11 +1324,13 @@ BinOp(expr_ty left, operator_ty op, expr_ty right, int lineno, PyArena *arena)
         p->v.BinOp.op = op;
         p->v.BinOp.right = right;
         p->lineno = lineno;
+        p->col_offset = col_offset;
         return p;
 }
 
 expr_ty
-UnaryOp(unaryop_ty op, expr_ty operand, int lineno, PyArena *arena)
+UnaryOp(unaryop_ty op, expr_ty operand, int lineno, int col_offset, PyArena
+        *arena)
 {
         expr_ty p;
         if (!op) {
@@ -1316,11 +1352,13 @@ UnaryOp(unaryop_ty op, expr_ty operand, int lineno, PyArena *arena)
         p->v.UnaryOp.op = op;
         p->v.UnaryOp.operand = operand;
         p->lineno = lineno;
+        p->col_offset = col_offset;
         return p;
 }
 
 expr_ty
-Lambda(arguments_ty args, expr_ty body, int lineno, PyArena *arena)
+Lambda(arguments_ty args, expr_ty body, int lineno, int col_offset, PyArena
+       *arena)
 {
         expr_ty p;
         if (!args) {
@@ -1342,11 +1380,13 @@ Lambda(arguments_ty args, expr_ty body, int lineno, PyArena *arena)
         p->v.Lambda.args = args;
         p->v.Lambda.body = body;
         p->lineno = lineno;
+        p->col_offset = col_offset;
         return p;
 }
 
 expr_ty
-IfExp(expr_ty test, expr_ty body, expr_ty orelse, int lineno, PyArena *arena)
+IfExp(expr_ty test, expr_ty body, expr_ty orelse, int lineno, int col_offset,
+      PyArena *arena)
 {
         expr_ty p;
         if (!test) {
@@ -1374,11 +1414,13 @@ IfExp(expr_ty test, expr_ty body, expr_ty orelse, int lineno, PyArena *arena)
         p->v.IfExp.body = body;
         p->v.IfExp.orelse = orelse;
         p->lineno = lineno;
+        p->col_offset = col_offset;
         return p;
 }
 
 expr_ty
-Dict(asdl_seq * keys, asdl_seq * values, int lineno, PyArena *arena)
+Dict(asdl_seq * keys, asdl_seq * values, int lineno, int col_offset, PyArena
+     *arena)
 {
         expr_ty p;
         p = (expr_ty)PyArena_Malloc(arena, sizeof(*p));
@@ -1390,11 +1432,13 @@ Dict(asdl_seq * keys, asdl_seq * values, int lineno, PyArena *arena)
         p->v.Dict.keys = keys;
         p->v.Dict.values = values;
         p->lineno = lineno;
+        p->col_offset = col_offset;
         return p;
 }
 
 expr_ty
-ListComp(expr_ty elt, asdl_seq * generators, int lineno, PyArena *arena)
+ListComp(expr_ty elt, asdl_seq * generators, int lineno, int col_offset,
+         PyArena *arena)
 {
         expr_ty p;
         if (!elt) {
@@ -1411,11 +1455,13 @@ ListComp(expr_ty elt, asdl_seq * generators, int lineno, PyArena *arena)
         p->v.ListComp.elt = elt;
         p->v.ListComp.generators = generators;
         p->lineno = lineno;
+        p->col_offset = col_offset;
         return p;
 }
 
 expr_ty
-GeneratorExp(expr_ty elt, asdl_seq * generators, int lineno, PyArena *arena)
+GeneratorExp(expr_ty elt, asdl_seq * generators, int lineno, int col_offset,
+             PyArena *arena)
 {
         expr_ty p;
         if (!elt) {
@@ -1432,11 +1478,12 @@ GeneratorExp(expr_ty elt, asdl_seq * generators, int lineno, PyArena *arena)
         p->v.GeneratorExp.elt = elt;
         p->v.GeneratorExp.generators = generators;
         p->lineno = lineno;
+        p->col_offset = col_offset;
         return p;
 }
 
 expr_ty
-Yield(expr_ty value, int lineno, PyArena *arena)
+Yield(expr_ty value, int lineno, int col_offset, PyArena *arena)
 {
         expr_ty p;
         p = (expr_ty)PyArena_Malloc(arena, sizeof(*p));
@@ -1447,12 +1494,13 @@ Yield(expr_ty value, int lineno, PyArena *arena)
         p->kind = Yield_kind;
         p->v.Yield.value = value;
         p->lineno = lineno;
+        p->col_offset = col_offset;
         return p;
 }
 
 expr_ty
-Compare(expr_ty left, asdl_seq * ops, asdl_seq * comparators, int lineno,
-        PyArena *arena)
+Compare(expr_ty left, asdl_seq * ops, asdl_seq * comparators, int lineno, int
+        col_offset, PyArena *arena)
 {
         expr_ty p;
         if (!left) {
@@ -1470,12 +1518,13 @@ Compare(expr_ty left, asdl_seq * ops, asdl_seq * comparators, int lineno,
         p->v.Compare.ops = ops;
         p->v.Compare.comparators = comparators;
         p->lineno = lineno;
+        p->col_offset = col_offset;
         return p;
 }
 
 expr_ty
 Call(expr_ty func, asdl_seq * args, asdl_seq * keywords, expr_ty starargs,
-     expr_ty kwargs, int lineno, PyArena *arena)
+     expr_ty kwargs, int lineno, int col_offset, PyArena *arena)
 {
         expr_ty p;
         if (!func) {
@@ -1495,11 +1544,12 @@ Call(expr_ty func, asdl_seq * args, asdl_seq * keywords, expr_ty starargs,
         p->v.Call.starargs = starargs;
         p->v.Call.kwargs = kwargs;
         p->lineno = lineno;
+        p->col_offset = col_offset;
         return p;
 }
 
 expr_ty
-Repr(expr_ty value, int lineno, PyArena *arena)
+Repr(expr_ty value, int lineno, int col_offset, PyArena *arena)
 {
         expr_ty p;
         if (!value) {
@@ -1515,11 +1565,12 @@ Repr(expr_ty value, int lineno, PyArena *arena)
         p->kind = Repr_kind;
         p->v.Repr.value = value;
         p->lineno = lineno;
+        p->col_offset = col_offset;
         return p;
 }
 
 expr_ty
-Num(object n, int lineno, PyArena *arena)
+Num(object n, int lineno, int col_offset, PyArena *arena)
 {
         expr_ty p;
         if (!n) {
@@ -1535,11 +1586,12 @@ Num(object n, int lineno, PyArena *arena)
         p->kind = Num_kind;
         p->v.Num.n = n;
         p->lineno = lineno;
+        p->col_offset = col_offset;
         return p;
 }
 
 expr_ty
-Str(string s, int lineno, PyArena *arena)
+Str(string s, int lineno, int col_offset, PyArena *arena)
 {
         expr_ty p;
         if (!s) {
@@ -1555,12 +1607,13 @@ Str(string s, int lineno, PyArena *arena)
         p->kind = Str_kind;
         p->v.Str.s = s;
         p->lineno = lineno;
+        p->col_offset = col_offset;
         return p;
 }
 
 expr_ty
-Attribute(expr_ty value, identifier attr, expr_context_ty ctx, int lineno,
-          PyArena *arena)
+Attribute(expr_ty value, identifier attr, expr_context_ty ctx, int lineno, int
+          col_offset, PyArena *arena)
 {
         expr_ty p;
         if (!value) {
@@ -1588,12 +1641,13 @@ Attribute(expr_ty value, identifier attr, expr_context_ty ctx, int lineno,
         p->v.Attribute.attr = attr;
         p->v.Attribute.ctx = ctx;
         p->lineno = lineno;
+        p->col_offset = col_offset;
         return p;
 }
 
 expr_ty
-Subscript(expr_ty value, slice_ty slice, expr_context_ty ctx, int lineno,
-          PyArena *arena)
+Subscript(expr_ty value, slice_ty slice, expr_context_ty ctx, int lineno, int
+          col_offset, PyArena *arena)
 {
         expr_ty p;
         if (!value) {
@@ -1621,11 +1675,13 @@ Subscript(expr_ty value, slice_ty slice, expr_context_ty ctx, int lineno,
         p->v.Subscript.slice = slice;
         p->v.Subscript.ctx = ctx;
         p->lineno = lineno;
+        p->col_offset = col_offset;
         return p;
 }
 
 expr_ty
-Name(identifier id, expr_context_ty ctx, int lineno, PyArena *arena)
+Name(identifier id, expr_context_ty ctx, int lineno, int col_offset, PyArena
+     *arena)
 {
         expr_ty p;
         if (!id) {
@@ -1647,11 +1703,13 @@ Name(identifier id, expr_context_ty ctx, int lineno, PyArena *arena)
         p->v.Name.id = id;
         p->v.Name.ctx = ctx;
         p->lineno = lineno;
+        p->col_offset = col_offset;
         return p;
 }
 
 expr_ty
-List(asdl_seq * elts, expr_context_ty ctx, int lineno, PyArena *arena)
+List(asdl_seq * elts, expr_context_ty ctx, int lineno, int col_offset, PyArena
+     *arena)
 {
         expr_ty p;
         if (!ctx) {
@@ -1668,11 +1726,13 @@ List(asdl_seq * elts, expr_context_ty ctx, int lineno, PyArena *arena)
         p->v.List.elts = elts;
         p->v.List.ctx = ctx;
         p->lineno = lineno;
+        p->col_offset = col_offset;
         return p;
 }
 
 expr_ty
-Tuple(asdl_seq * elts, expr_context_ty ctx, int lineno, PyArena *arena)
+Tuple(asdl_seq * elts, expr_context_ty ctx, int lineno, int col_offset, PyArena
+      *arena)
 {
         expr_ty p;
         if (!ctx) {
@@ -1689,6 +1749,7 @@ Tuple(asdl_seq * elts, expr_context_ty ctx, int lineno, PyArena *arena)
         p->v.Tuple.elts = elts;
         p->v.Tuple.ctx = ctx;
         p->lineno = lineno;
+        p->col_offset = col_offset;
         return p;
 }
 
@@ -2264,6 +2325,9 @@ ast2obj_stmt(void* _o)
         value = ast2obj_int(o->lineno);
         if (!value) goto failed;
         PyObject_SetAttrString(result, "lineno", value);
+        value = ast2obj_int(o->col_offset);
+        if (!value) goto failed;
+        PyObject_SetAttrString(result, "col_offset", value);
         return result;
 failed:
         Py_XDECREF(value);
@@ -2580,6 +2644,9 @@ ast2obj_expr(void* _o)
         value = ast2obj_int(o->lineno);
         if (!value) goto failed;
         PyObject_SetAttrString(result, "lineno", value);
+        value = ast2obj_int(o->col_offset);
+        if (!value) goto failed;
+        PyObject_SetAttrString(result, "col_offset", value);
         return result;
 failed:
         Py_XDECREF(value);

Python/ast.c

@@ -243,7 +243,7 @@ PyAST_FromNode(const node *n, PyCompilerFlags *flags, const char *filename,
                 stmts = asdl_seq_new(1, arena);
                 if (!stmts)
 		    goto error;
-                asdl_seq_SET(stmts, 0, Pass(n->n_lineno, arena));
+                asdl_seq_SET(stmts, 0, Pass(n->n_lineno, n->n_col_offset, arena));
                 return Interactive(stmts, arena);
             }
             else {
@@ -564,7 +564,7 @@ compiler_complex_args(struct compiling *c, const node *n)
 	    		ast_error(child, "assignment to None");
 		    	return NULL;
 		    }   
-            arg = Name(NEW_IDENTIFIER(child), Store, LINENO(child),
+            arg = Name(NEW_IDENTIFIER(child), Store, LINENO(child), child->n_col_offset,
                        c->c_arena);
 	    }
         else {
@@ -573,7 +573,7 @@ compiler_complex_args(struct compiling *c, const node *n)
         asdl_seq_SET(args, i, arg);
     }
 
-    result = Tuple(args, Store, LINENO(n), c->c_arena);
+    result = Tuple(args, Store, LINENO(n), n->n_col_offset, c->c_arena);
     if (!set_context(result, Store, n))
         return NULL;
     return result;
@@ -651,7 +651,7 @@ ast_for_arguments(struct compiling *c, const node *n)
 			    goto error;
 		    }
                     name = Name(NEW_IDENTIFIER(CHILD(ch, 0)),
-                                Param, LINENO(ch), c->c_arena);
+                                Param, LINENO(ch), ch->n_col_offset, c->c_arena);
                     if (!name)
                         goto error;
                     asdl_seq_SET(args, k++, name);
@@ -696,14 +696,18 @@ ast_for_dotted_name(struct compiling *c, const node *n)
 {
     expr_ty e;
     identifier id;
+    int lineno, col_offset;
     int i;
 
     REQ(n, dotted_name);
 
+    lineno = LINENO(n);
+    col_offset = n->n_col_offset;
+
     id = NEW_IDENTIFIER(CHILD(n, 0));
     if (!id)
         return NULL;
-    e = Name(id, Load, LINENO(n), c->c_arena);
+    e = Name(id, Load, lineno, col_offset, c->c_arena);
     if (!e)
 	return NULL;
 
@@ -711,7 +715,7 @@ ast_for_dotted_name(struct compiling *c, const node *n)
         id = NEW_IDENTIFIER(CHILD(n, i));
 	if (!id)
 	    return NULL;
-	e = Attribute(e, id, Load, LINENO(CHILD(n, i)), c->c_arena);
+	e = Attribute(e, id, Load, lineno, col_offset, c->c_arena);
 	if (!e)
 	    return NULL;
     }
@@ -739,7 +743,7 @@ ast_for_decorator(struct compiling *c, const node *n)
 	name_expr = NULL;
     }
     else if (NCH(n) == 5) { /* Call with no arguments */
-	d = Call(name_expr, NULL, NULL, NULL, NULL, LINENO(n), c->c_arena);
+	d = Call(name_expr, NULL, NULL, NULL, NULL, LINENO(n), n->n_col_offset, c->c_arena);
 	if (!d)
 	    return NULL;
 	name_expr = NULL;
@@ -811,7 +815,7 @@ ast_for_funcdef(struct compiling *c, const node *n)
     if (!body)
 	return NULL;
 
-    return FunctionDef(name, args, body, decorator_seq, LINENO(n), c->c_arena);
+    return FunctionDef(name, args, body, decorator_seq, LINENO(n), n->n_col_offset, c->c_arena);
 }
 
 static expr_ty
@@ -838,7 +842,7 @@ ast_for_lambdef(struct compiling *c, const node *n)
             return NULL;
     }
 
-    return Lambda(args, expression, LINENO(n), c->c_arena);
+    return Lambda(args, expression, LINENO(n), n->n_col_offset, c->c_arena);
 }
 
 static expr_ty
@@ -857,7 +861,7 @@ ast_for_ifexpr(struct compiling *c, const node *n)
     orelse = ast_for_expr(c, CHILD(n, 4));
     if (!orelse)
 	return NULL;
-    return IfExp(expression, body, orelse, LINENO(n), c->c_arena);
+    return IfExp(expression, body, orelse, LINENO(n), n->n_col_offset, c->c_arena);
 }
 
 /* Count the number of 'for' loop in a list comprehension.
@@ -968,7 +972,7 @@ ast_for_listcomp(struct compiling *c, const node *n)
 	    lc = comprehension(asdl_seq_GET(t, 0), expression, NULL,
                                c->c_arena);
 	else
-	    lc = comprehension(Tuple(t, Store, LINENO(ch), c->c_arena),
+	    lc = comprehension(Tuple(t, Store, LINENO(ch), ch->n_col_offset, c->c_arena),
                                expression, NULL, c->c_arena);
         if (!lc)
             return NULL;
@@ -1003,7 +1007,7 @@ ast_for_listcomp(struct compiling *c, const node *n)
 	    asdl_seq_SET(listcomps, i, lc);
     }
 
-    return ListComp(elt, listcomps, LINENO(n), c->c_arena);
+    return ListComp(elt, listcomps, LINENO(n), n->n_col_offset, c->c_arena);
 }
 
 /*
@@ -1113,7 +1117,7 @@ ast_for_genexp(struct compiling *c, const node *n)
             ge = comprehension(asdl_seq_GET(t, 0), expression,
                                NULL, c->c_arena);
         else
-            ge = comprehension(Tuple(t, Store, LINENO(ch), c->c_arena),
+            ge = comprehension(Tuple(t, Store, LINENO(ch), ch->n_col_offset, c->c_arena),
                                expression, NULL, c->c_arena);
 
         if (!ge)
@@ -1152,7 +1156,7 @@ ast_for_genexp(struct compiling *c, const node *n)
         asdl_seq_SET(genexps, i, ge);
     }
     
-    return GeneratorExp(elt, genexps, LINENO(n), c->c_arena);
+    return GeneratorExp(elt, genexps, LINENO(n), n->n_col_offset, c->c_arena);
 }
 
 static expr_ty
@@ -1167,14 +1171,14 @@ ast_for_atom(struct compiling *c, const node *n)
     case NAME:
 	/* All names start in Load context, but may later be
 	   changed. */
-	return Name(NEW_IDENTIFIER(ch), Load, LINENO(n), c->c_arena);
+	return Name(NEW_IDENTIFIER(ch), Load, LINENO(n), n->n_col_offset, c->c_arena);
     case STRING: {
 	PyObject *str = parsestrplus(c, n);
 	if (!str)
 	    return NULL;
 
 	PyArena_AddPyObject(c->c_arena, str);
-	return Str(str, LINENO(n), c->c_arena);
+	return Str(str, LINENO(n), n->n_col_offset, c->c_arena);
     }
     case NUMBER: {
 	PyObject *pynum = parsenumber(STR(ch));
@@ -1182,13 +1186,13 @@ ast_for_atom(struct compiling *c, const node *n)
 	    return NULL;
 
 	PyArena_AddPyObject(c->c_arena, pynum);
-	return Num(pynum, LINENO(n), c->c_arena);
+	return Num(pynum, LINENO(n), n->n_col_offset, c->c_arena);
     }
     case LPAR: /* some parenthesized expressions */
 	ch = CHILD(n, 1);
 	
 	if (TYPE(ch) == RPAR)
-	    return Tuple(NULL, Load, LINENO(n), c->c_arena);
+	    return Tuple(NULL, Load, LINENO(n), n->n_col_offset, c->c_arena);
 	
 	if (TYPE(ch) == yield_expr)
 	    return ast_for_expr(c, ch);
@@ -1201,7 +1205,7 @@ ast_for_atom(struct compiling *c, const node *n)
 	ch = CHILD(n, 1);
 	
 	if (TYPE(ch) == RSQB)
-	    return List(NULL, Load, LINENO(n), c->c_arena);
+	    return List(NULL, Load, LINENO(n), n->n_col_offset, c->c_arena);
 	
 	REQ(ch, listmaker);
 	if (NCH(ch) == 1 || TYPE(CHILD(ch, 1)) == COMMA) {
@@ -1209,7 +1213,7 @@ ast_for_atom(struct compiling *c, const node *n)
 	    if (!elts)
 		return NULL;
 
-	    return List(elts, Load, LINENO(n), c->c_arena);
+	    return List(elts, Load, LINENO(n), n->n_col_offset, c->c_arena);
 	}
 	else
 	    return ast_for_listcomp(c, ch);
@@ -1243,14 +1247,14 @@ ast_for_atom(struct compiling *c, const node *n)
 
 	    asdl_seq_SET(values, i / 4, expression);
 	}
-	return Dict(keys, values, LINENO(n), c->c_arena);
+	return Dict(keys, values, LINENO(n), n->n_col_offset, c->c_arena);
     }
     case BACKQUOTE: { /* repr */
 	expr_ty expression = ast_for_testlist(c, CHILD(n, 1));
 	if (!expression)
 	    return NULL;
 
-	return Repr(expression, LINENO(n), c->c_arena);
+	return Repr(expression, LINENO(n), n->n_col_offset, c->c_arena);
     }
     default:
 	PyErr_Format(PyExc_SystemError, "unhandled atom %d", TYPE(ch));
@@ -1353,7 +1357,7 @@ ast_for_binop(struct compiling *c, const node *n)
         if (!operator)
             return NULL;
 
-	result = BinOp(expr1, operator, expr2, LINENO(n), c->c_arena);
+	result = BinOp(expr1, operator, expr2, LINENO(n), n->n_col_offset, c->c_arena);
 	if (!result)
             return NULL;
 
@@ -1371,7 +1375,7 @@ ast_for_binop(struct compiling *c, const node *n)
                     return NULL;
 
                 tmp_result = BinOp(result, operator, tmp, 
-				   LINENO(next_oper), c->c_arena);
+				   LINENO(next_oper), next_oper->n_col_offset, c->c_arena);
 		if (!tmp) 
 			return NULL;
 		result = tmp_result;
@@ -1389,13 +1393,13 @@ ast_for_trailer(struct compiling *c, const node *n, expr_ty left_expr)
     REQ(n, trailer);
     if (TYPE(CHILD(n, 0)) == LPAR) {
         if (NCH(n) == 2)
-            return Call(left_expr, NULL, NULL, NULL, NULL, LINENO(n), c->c_arena);
+            return Call(left_expr, NULL, NULL, NULL, NULL, LINENO(n), n->n_col_offset, c->c_arena);
         else
             return ast_for_call(c, CHILD(n, 1), left_expr);
     }
     else if (TYPE(CHILD(n, 0)) == DOT ) {
         return Attribute(left_expr, NEW_IDENTIFIER(CHILD(n, 1)), Load,
-                         LINENO(n), c->c_arena);
+                         LINENO(n), n->n_col_offset, c->c_arena);
     }
     else {
         REQ(CHILD(n, 0), LSQB);
@@ -1405,7 +1409,7 @@ ast_for_trailer(struct compiling *c, const node *n, expr_ty left_expr)
             slice_ty slc = ast_for_slice(c, CHILD(n, 0));
             if (!slc)
                 return NULL;
-            return Subscript(left_expr, slc, Load, LINENO(n), c->c_arena);
+            return Subscript(left_expr, slc, Load, LINENO(n), n->n_col_offset, c->c_arena);
         }
         else {
             /* The grammar is ambiguous here. The ambiguity is resolved 
@@ -1430,7 +1434,7 @@ ast_for_trailer(struct compiling *c, const node *n, expr_ty left_expr)
             }
             if (!simple) {
                 return Subscript(left_expr, ExtSlice(slices, c->c_arena),
-                                 Load, LINENO(n), c->c_arena);
+                                 Load, LINENO(n), n->n_col_offset, c->c_arena);
             }
             /* extract Index values and put them in a Tuple */
             elts = asdl_seq_new(asdl_seq_LEN(slices), c->c_arena);
@@ -1439,11 +1443,11 @@ ast_for_trailer(struct compiling *c, const node *n, expr_ty left_expr)
                 assert(slc->kind == Index_kind  && slc->v.Index.value);
                 asdl_seq_SET(elts, j, slc->v.Index.value);
             }
-            e = Tuple(elts, Load, LINENO(n), c->c_arena);
+            e = Tuple(elts, Load, LINENO(n), n->n_col_offset, c->c_arena);
             if (!e)
                 return NULL;
             return Subscript(left_expr, Index(e, c->c_arena),
-                             Load, LINENO(n), c->c_arena);
+                             Load, LINENO(n), n->n_col_offset, c->c_arena);
         }
     }
 }
@@ -1468,13 +1472,15 @@ ast_for_power(struct compiling *c, const node *n)
         tmp = ast_for_trailer(c, ch, e);
         if (!tmp)
             return NULL;
+	tmp->lineno = e->lineno;
+	tmp->col_offset = e->col_offset;
         e = tmp;
     }
     if (TYPE(CHILD(n, NCH(n) - 1)) == factor) {
         expr_ty f = ast_for_expr(c, CHILD(n, NCH(n) - 1));
         if (!f)
             return NULL;
-        tmp = BinOp(e, Pow, f, LINENO(n), c->c_arena);
+        tmp = BinOp(e, Pow, f, LINENO(n), n->n_col_offset, c->c_arena);
         if (!tmp)
             return NULL;
         e = tmp;
@@ -1542,9 +1548,9 @@ ast_for_expr(struct compiling *c, const node *n)
                 asdl_seq_SET(seq, i / 2, e);
             }
             if (!strcmp(STR(CHILD(n, 1)), "and"))
-                return BoolOp(And, seq, LINENO(n), c->c_arena);
+                return BoolOp(And, seq, LINENO(n), n->n_col_offset, c->c_arena);
             assert(!strcmp(STR(CHILD(n, 1)), "or"));
-            return BoolOp(Or, seq, LINENO(n), c->c_arena);
+            return BoolOp(Or, seq, LINENO(n), n->n_col_offset, c->c_arena);
         case not_test:
             if (NCH(n) == 1) {
                 n = CHILD(n, 0);
@@ -1555,7 +1561,7 @@ ast_for_expr(struct compiling *c, const node *n)
                 if (!expression)
                     return NULL;
 
-                return UnaryOp(Not, expression, LINENO(n), c->c_arena);
+                return UnaryOp(Not, expression, LINENO(n), n->n_col_offset, c->c_arena);
             }
         case comparison:
             if (NCH(n) == 1) {
@@ -1594,7 +1600,7 @@ ast_for_expr(struct compiling *c, const node *n)
                     return NULL;
 		}
                     
-                return Compare(expression, ops, cmps, LINENO(n), c->c_arena);
+                return Compare(expression, ops, cmps, LINENO(n), n->n_col_offset, c->c_arena);
             }
             break;
 
@@ -1620,7 +1626,7 @@ ast_for_expr(struct compiling *c, const node *n)
 		if (!exp)
 		    return NULL;
 	    }
-	    return Yield(exp, LINENO(n), c->c_arena);
+	    return Yield(exp, LINENO(n), n->n_col_offset, c->c_arena);
 	}
         case factor: {
             expr_ty expression;
@@ -1636,11 +1642,11 @@ ast_for_expr(struct compiling *c, const node *n)
 
             switch (TYPE(CHILD(n, 0))) {
                 case PLUS:
-                    return UnaryOp(UAdd, expression, LINENO(n), c->c_arena);
+                    return UnaryOp(UAdd, expression, LINENO(n), n->n_col_offset, c->c_arena);
                 case MINUS:
-                    return UnaryOp(USub, expression, LINENO(n), c->c_arena);
+                    return UnaryOp(USub, expression, LINENO(n), n->n_col_offset, c->c_arena);
                 case TILDE:
-                    return UnaryOp(Invert, expression, LINENO(n), c->c_arena);
+                    return UnaryOp(Invert, expression, LINENO(n), n->n_col_offset, c->c_arena);
             }
             PyErr_Format(PyExc_SystemError, "unhandled factor: %d",
 	    		 TYPE(CHILD(n, 0)));
@@ -1761,7 +1767,7 @@ ast_for_call(struct compiling *c, const node *n, expr_ty func)
 	}
     }
 
-    return Call(func, args, keywords, vararg, kwarg, LINENO(n), c->c_arena);
+    return Call(func, args, keywords, vararg, kwarg, func->lineno, func->col_offset, c->c_arena);
 }
 
 static expr_ty
@@ -1787,7 +1793,7 @@ ast_for_testlist(struct compiling *c, const node* n)
         asdl_seq *tmp = seq_for_testlist(c, n);
         if (!tmp)
             return NULL;
-	return Tuple(tmp, Load, LINENO(n), c->c_arena);
+	return Tuple(tmp, Load, LINENO(n), n->n_col_offset, c->c_arena);
     }
 }
 
@@ -1841,7 +1847,7 @@ ast_for_expr_stmt(struct compiling *c, const node *n)
         if (!e)
             return NULL;
 
-	return Expr(e, LINENO(n), c->c_arena);
+	return Expr(e, LINENO(n), n->n_col_offset, c->c_arena);
     }
     else if (TYPE(CHILD(n, 1)) == augassign) {
         expr_ty expr1, expr2;
@@ -1851,7 +1857,7 @@ ast_for_expr_stmt(struct compiling *c, const node *n)
 	if (TYPE(ch) == testlist)
 	    expr1 = ast_for_testlist(c, ch);
 	else
-	    expr1 = Yield(ast_for_expr(c, CHILD(ch, 0)), LINENO(ch),
+	    expr1 = Yield(ast_for_expr(c, CHILD(ch, 0)), LINENO(ch), n->n_col_offset,
                           c->c_arena);
 
         if (!expr1)
@@ -1883,7 +1889,7 @@ ast_for_expr_stmt(struct compiling *c, const node *n)
 	if (TYPE(ch) == testlist)
 	    expr2 = ast_for_testlist(c, ch);
 	else
-	    expr2 = Yield(ast_for_expr(c, ch), LINENO(ch), c->c_arena);
+	    expr2 = Yield(ast_for_expr(c, ch), LINENO(ch), ch->n_col_offset, c->c_arena);
         if (!expr2)
             return NULL;
 
@@ -1891,7 +1897,7 @@ ast_for_expr_stmt(struct compiling *c, const node *n)
         if (!operator)
             return NULL;
 
-	return AugAssign(expr1, operator, expr2, LINENO(n), c->c_arena);
+	return AugAssign(expr1, operator, expr2, LINENO(n), n->n_col_offset, c->c_arena);
     }
     else {
 	int i;
@@ -1929,7 +1935,7 @@ ast_for_expr_stmt(struct compiling *c, const node *n)
 	    expression = ast_for_expr(c, value);
 	if (!expression)
 	    return NULL;
-	return Assign(targets, expression, LINENO(n), c->c_arena);
+	return Assign(targets, expression, LINENO(n), n->n_col_offset, c->c_arena);
     }
 }
 
@@ -1961,7 +1967,7 @@ ast_for_print_stmt(struct compiling *c, const node *n)
     	asdl_seq_SET(seq, j, expression);
     }
     nl = (TYPE(CHILD(n, NCH(n) - 1)) == COMMA) ? false : true;
-    return Print(dest, seq, nl, LINENO(n), c->c_arena);
+    return Print(dest, seq, nl, LINENO(n), n->n_col_offset, c->c_arena);
 }
 
 static asdl_seq *
@@ -1998,7 +2004,7 @@ ast_for_del_stmt(struct compiling *c, const node *n)
     expr_list = ast_for_exprlist(c, CHILD(n, 1), Del);
     if (!expr_list)
         return NULL;
-    return Delete(expr_list, LINENO(n), c->c_arena);
+    return Delete(expr_list, LINENO(n), n->n_col_offset, c->c_arena);
 }
 
 static stmt_ty
@@ -2020,32 +2026,32 @@ ast_for_flow_stmt(struct compiling *c, const node *n)
     ch = CHILD(n, 0);
     switch (TYPE(ch)) {
         case break_stmt:
-            return Break(LINENO(n), c->c_arena);
+            return Break(LINENO(n), n->n_col_offset, c->c_arena);
         case continue_stmt:
-            return Continue(LINENO(n), c->c_arena);
+            return Continue(LINENO(n), n->n_col_offset, c->c_arena);
         case yield_stmt: { /* will reduce to yield_expr */
 	    expr_ty exp = ast_for_expr(c, CHILD(ch, 0));
 	    if (!exp)
 		return NULL;
-            return Expr(exp, LINENO(n), c->c_arena);
+            return Expr(exp, LINENO(n), n->n_col_offset, c->c_arena);
         }
         case return_stmt:
             if (NCH(ch) == 1)
-                return Return(NULL, LINENO(n), c->c_arena);
+                return Return(NULL, LINENO(n), n->n_col_offset, c->c_arena);
             else {
                 expr_ty expression = ast_for_testlist(c, CHILD(ch, 1));
                 if (!expression)
                     return NULL;
-                return Return(expression, LINENO(n), c->c_arena);
+                return Return(expression, LINENO(n), n->n_col_offset, c->c_arena);
             }
         case raise_stmt:
             if (NCH(ch) == 1)
-                return Raise(NULL, NULL, NULL, LINENO(n), c->c_arena);
+                return Raise(NULL, NULL, NULL, LINENO(n), n->n_col_offset, c->c_arena);
             else if (NCH(ch) == 2) {
                 expr_ty expression = ast_for_expr(c, CHILD(ch, 1));
                 if (!expression)
                     return NULL;
-                return Raise(expression, NULL, NULL, LINENO(n), c->c_arena);
+                return Raise(expression, NULL, NULL, LINENO(n), n->n_col_offset, c->c_arena);
             }
             else if (NCH(ch) == 4) {
                 expr_ty expr1, expr2;
@@ -2057,7 +2063,7 @@ ast_for_flow_stmt(struct compiling *c, const node *n)
                 if (!expr2)
                     return NULL;
 
-                return Raise(expr1, expr2, NULL, LINENO(n), c->c_arena);
+                return Raise(expr1, expr2, NULL, LINENO(n), n->n_col_offset, c->c_arena);
             }
             else if (NCH(ch) == 6) {
                 expr_ty expr1, expr2, expr3;
@@ -2072,7 +2078,7 @@ ast_for_flow_stmt(struct compiling *c, const node *n)
                 if (!expr3)
                     return NULL;
                     
-                return Raise(expr1, expr2, expr3, LINENO(n), c->c_arena);
+                return Raise(expr1, expr2, expr3, LINENO(n), n->n_col_offset, c->c_arena);
             }
         default:
             PyErr_Format(PyExc_SystemError,
@@ -2167,10 +2173,14 @@ ast_for_import_stmt(struct compiling *c, const node *n)
       import_from: 'from' ('.'* dotted_name | '.') 'import'
                           ('*' | '(' import_as_names ')' | import_as_names)
     */
+    int lineno;
+    int col_offset;
     int i;
     asdl_seq *aliases;
 
     REQ(n, import_stmt);
+    lineno = LINENO(n);
+    col_offset = n->n_col_offset;
     n = CHILD(n, 0);
     if (TYPE(n) == import_name) {
         n = CHILD(n, 1);
@@ -2184,11 +2194,10 @@ ast_for_import_stmt(struct compiling *c, const node *n)
                 return NULL;
 	    asdl_seq_SET(aliases, i / 2, import_alias);
         }
-	return Import(aliases, LINENO(n), c->c_arena);
+	return Import(aliases, lineno, col_offset, c->c_arena);
     }
     else if (TYPE(n) == import_from) {
         int n_children;
-	int lineno = LINENO(n);
 	int idx, ndots = 0;
 	alias_ty mod = NULL;
 	identifier modname;
@@ -2259,7 +2268,7 @@ ast_for_import_stmt(struct compiling *c, const node *n)
             modname = mod->name;
         else
             modname = new_identifier("", c->c_arena);
-        return ImportFrom(modname, aliases, ndots, lineno,
+        return ImportFrom(modname, aliases, ndots, lineno, col_offset,
                           c->c_arena);
     }
     PyErr_Format(PyExc_SystemError,
@@ -2286,7 +2295,7 @@ ast_for_global_stmt(struct compiling *c, const node *n)
 	    return NULL;
 	asdl_seq_SET(s, i / 2, name);
     }
-    return Global(s, LINENO(n), c->c_arena);
+    return Global(s, LINENO(n), n->n_col_offset, c->c_arena);
 }
 
 static stmt_ty
@@ -2317,7 +2326,7 @@ ast_for_exec_stmt(struct compiling *c, const node *n)
             return NULL;
     }
 
-    return Exec(expr1, globals, locals, LINENO(n), c->c_arena);
+    return Exec(expr1, globals, locals, LINENO(n), n->n_col_offset, c->c_arena);
 }
 
 static stmt_ty
@@ -2329,7 +2338,7 @@ ast_for_assert_stmt(struct compiling *c, const node *n)
         expr_ty expression = ast_for_expr(c, CHILD(n, 1));
         if (!expression)
             return NULL;
-	return Assert(expression, NULL, LINENO(n), c->c_arena);
+	return Assert(expression, NULL, LINENO(n), n->n_col_offset, c->c_arena);
     }
     else if (NCH(n) == 4) {
         expr_ty expr1, expr2;
@@ -2341,7 +2350,7 @@ ast_for_assert_stmt(struct compiling *c, const node *n)
         if (!expr2)
             return NULL;
             
-	return Assert(expr1, expr2, LINENO(n), c->c_arena);
+	return Assert(expr1, expr2, LINENO(n), n->n_col_offset, c->c_arena);
     }
     PyErr_Format(PyExc_SystemError,
                  "improper number of parts to 'assert' statement: %d",
@@ -2436,7 +2445,7 @@ ast_for_if_stmt(struct compiling *c, const node *n)
         if (!suite_seq)
             return NULL;
             
-	return If(expression, suite_seq, NULL, LINENO(n), c->c_arena);
+	return If(expression, suite_seq, NULL, LINENO(n), n->n_col_offset, c->c_arena);
     }
 
     s = STR(CHILD(n, 4));
@@ -2458,7 +2467,7 @@ ast_for_if_stmt(struct compiling *c, const node *n)
         if (!seq2)
             return NULL;
 
-	return If(expression, seq1, seq2, LINENO(n), c->c_arena);
+	return If(expression, seq1, seq2, LINENO(n), n->n_col_offset, c->c_arena);
     }
     else if (s[2] == 'i') {
 	int i, n_elif, has_else = 0;
@@ -2491,7 +2500,7 @@ ast_for_if_stmt(struct compiling *c, const node *n)
                 return NULL;
 
 	    asdl_seq_SET(orelse, 0, If(expression, seq1, seq2, 
-				       LINENO(CHILD(n, NCH(n) - 6)),
+				       LINENO(CHILD(n, NCH(n) - 6)), CHILD(n, NCH(n) - 6)->n_col_offset,
                                        c->c_arena));
 	    /* the just-created orelse handled the last elif */
 	    n_elif--;
@@ -2513,12 +2522,12 @@ ast_for_if_stmt(struct compiling *c, const node *n)
 
 	    asdl_seq_SET(new, 0,
 			 If(expression, suite_seq, orelse, 
-			    LINENO(CHILD(n, off)), c->c_arena));
+			    LINENO(CHILD(n, off)), CHILD(n, off)->n_col_offset, c->c_arena));
 	    orelse = new;
 	}
 	return If(ast_for_expr(c, CHILD(n, 1)),
 		  ast_for_suite(c, CHILD(n, 3)),
-		  orelse, LINENO(n), c->c_arena);
+		  orelse, LINENO(n), n->n_col_offset, c->c_arena);
     }
 
     PyErr_Format(PyExc_SystemError,
@@ -2542,7 +2551,7 @@ ast_for_while_stmt(struct compiling *c, const node *n)
         suite_seq = ast_for_suite(c, CHILD(n, 3));
         if (!suite_seq)
             return NULL;
-	return While(expression, suite_seq, NULL, LINENO(n), c->c_arena);
+	return While(expression, suite_seq, NULL, LINENO(n), n->n_col_offset, c->c_arena);
     }
     else if (NCH(n) == 7) {
         expr_ty expression;
@@ -2558,7 +2567,7 @@ ast_for_while_stmt(struct compiling *c, const node *n)
         if (!seq2)
             return NULL;
 
-	return While(expression, seq1, seq2, LINENO(n), c->c_arena);
+	return While(expression, seq1, seq2, LINENO(n), n->n_col_offset, c->c_arena);
     }
 
     PyErr_Format(PyExc_SystemError,
@@ -2588,7 +2597,7 @@ ast_for_for_stmt(struct compiling *c, const node *n)
     if (asdl_seq_LEN(_target) == 1)
 	target = asdl_seq_GET(_target, 0);
     else
-	target = Tuple(_target, Store, LINENO(n), c->c_arena);
+	target = Tuple(_target, Store, LINENO(n), n->n_col_offset, c->c_arena);
 
     expression = ast_for_testlist(c, CHILD(n, 3));
     if (!expression)
@@ -2597,7 +2606,7 @@ ast_for_for_stmt(struct compiling *c, const node *n)
     if (!suite_seq)
         return NULL;
 
-    return For(target, expression, suite_seq, seq, LINENO(n), c->c_arena);
+    return For(target, expression, suite_seq, seq, LINENO(n), n->n_col_offset, c->c_arena);
 }
 
 static excepthandler_ty
@@ -2711,7 +2720,7 @@ ast_for_try_stmt(struct compiling *c, const node *n)
             asdl_seq_SET(handlers, i, e);
         }
 
-	except_st = TryExcept(body, handlers, orelse, LINENO(n), c->c_arena);
+	except_st = TryExcept(body, handlers, orelse, LINENO(n), n->n_col_offset, c->c_arena);
         if (!finally)
 	    return except_st;
 
@@ -2725,7 +2734,7 @@ ast_for_try_stmt(struct compiling *c, const node *n)
 
     /* must be a try ... finally (except clauses are in body, if any exist) */
     assert(finally != NULL);
-    return TryFinally(body, finally, LINENO(n), c->c_arena);
+    return TryFinally(body, finally, LINENO(n), n->n_col_offset, c->c_arena);
 }
 
 static expr_ty
@@ -2765,7 +2774,8 @@ ast_for_with_stmt(struct compiling *c, const node *n)
     if (!suite_seq) {
         return NULL;
     }
-    return With(context_expr, optional_vars, suite_seq, LINENO(n), c->c_arena);
+    return With(context_expr, optional_vars, suite_seq, LINENO(n), 
+		n->n_col_offset, c->c_arena);
 }
 
 static stmt_ty
@@ -2785,7 +2795,7 @@ ast_for_classdef(struct compiling *c, const node *n)
         s = ast_for_suite(c, CHILD(n, 3));
         if (!s)
             return NULL;
-	return ClassDef(NEW_IDENTIFIER(CHILD(n, 1)), NULL, s, LINENO(n),
+	return ClassDef(NEW_IDENTIFIER(CHILD(n, 1)), NULL, s, LINENO(n), n->n_col_offset,
                         c->c_arena);
     }
     /* check for empty base list */
@@ -2793,7 +2803,7 @@ ast_for_classdef(struct compiling *c, const node *n)
 	s = ast_for_suite(c, CHILD(n,5));
 	if (!s)
 		return NULL;
-	return ClassDef(NEW_IDENTIFIER(CHILD(n, 1)), NULL, s, LINENO(n),
+	return ClassDef(NEW_IDENTIFIER(CHILD(n, 1)), NULL, s, LINENO(n), n->n_col_offset,
                         c->c_arena);
     }
 
@@ -2805,7 +2815,7 @@ ast_for_classdef(struct compiling *c, const node *n)
     s = ast_for_suite(c, CHILD(n, 6));
     if (!s)
         return NULL;
-    return ClassDef(NEW_IDENTIFIER(CHILD(n, 1)), bases, s, LINENO(n),
+    return ClassDef(NEW_IDENTIFIER(CHILD(n, 1)), bases, s, LINENO(n), n->n_col_offset,
                     c->c_arena);
 }
 
@@ -2835,7 +2845,7 @@ ast_for_stmt(struct compiling *c, const node *n)
             case del_stmt:
                 return ast_for_del_stmt(c, n);
             case pass_stmt:
-                return Pass(LINENO(n), c->c_arena);
+                return Pass(LINENO(n), n->n_col_offset, c->c_arena);
             case flow_stmt:
                 return ast_for_flow_stmt(c, n);
             case import_stmt:

Python/compile.c

@@ -3635,7 +3635,7 @@ compiler_augassign(struct compiler *c, stmt_ty s)
 	switch (e->kind) {
 		case Attribute_kind:
 		auge = Attribute(e->v.Attribute.value, e->v.Attribute.attr,
-				 AugLoad, e->lineno, c->c_arena);
+				 AugLoad, e->lineno, e->col_offset, c->c_arena);
 		if (auge == NULL)
 		    return 0;
 		VISIT(c, expr, auge);
@@ -3646,7 +3646,7 @@ compiler_augassign(struct compiler *c, stmt_ty s)
 		break;
 	case Subscript_kind:
 		auge = Subscript(e->v.Subscript.value, e->v.Subscript.slice,
-				 AugLoad, e->lineno, c->c_arena);
+				 AugLoad, e->lineno, e->col_offset, c->c_arena);
 		if (auge == NULL)
 		    return 0;
 		VISIT(c, expr, auge);