Skip to content

C
cpython
Commit 49c5da1d authored by Martin v. Löwis's avatar Martin v. Löwis
Browse files
Options

Patch #1440601: Add col_offset attribute to AST nodes.

parent 3ffa59b1
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 382 additions and 244 deletions
Doc/lib/libast.tex
View file @ 49c5da1d
......@@ -34,7 +34,13 @@ names of all child nodes.
Each instance of a concrete class has one attribute for each child node,
of the type as defined in the grammar. For example, \code{_ast.BinOp}
instances have an attribute \code{left} of type \code{_ast.expr}.
instances have an attribute \code{left} of type \code{_ast.expr}.
Instances of \code{_ast.expr} and \code{_ast.stmt} subclasses also
have lineno and col_offset attributes. The lineno is the line number
of source text (1 indexed so the first line is line 1) and the
col_offset is the utf8 byte offset of the first token that generated
the node. The utf8 offset is recorded because the parser uses utf8
internally.
If these attributes are marked as optional in the grammar (using a
question mark), the value might be \code{None}. If the attributes
......
Include/Python-ast.h
View file @ 49c5da1d
......@@ -178,6 +178,7 @@ struct _stmt {
} v;
int lineno;
int col_offset;
};
struct _expr {
......@@ -288,6 +289,7 @@ struct _expr {
} v;
int lineno;
int col_offset;
};
struct _slice {
......@@ -346,68 +348,79 @@ mod_ty Interactive(asdl_seq * body, PyArena *arena);
mod_ty Expression(expr_ty body, PyArena *arena);
mod_ty Suite(asdl_seq * body, PyArena *arena);
stmt_ty FunctionDef(identifier name, arguments_ty args, asdl_seq * body,
asdl_seq * decorators, int lineno, PyArena *arena);
asdl_seq * decorators, int lineno, int col_offset, PyArena
*arena);
stmt_ty ClassDef(identifier name, asdl_seq * bases, asdl_seq * body, int
lineno, PyArena *arena);
stmt_ty Return(expr_ty value, int lineno, PyArena *arena);
stmt_ty Delete(asdl_seq * targets, int lineno, PyArena *arena);
stmt_ty Assign(asdl_seq * targets, expr_ty value, int lineno, PyArena *arena);
lineno, int col_offset, PyArena *arena);
stmt_ty Return(expr_ty value, int lineno, int col_offset, PyArena *arena);
stmt_ty Delete(asdl_seq * targets, int lineno, int col_offset, PyArena *arena);
stmt_ty Assign(asdl_seq * targets, expr_ty value, int lineno, int col_offset,
PyArena *arena);
stmt_ty AugAssign(expr_ty target, operator_ty op, expr_ty value, int lineno,
PyArena *arena);
stmt_ty Print(expr_ty dest, asdl_seq * values, bool nl, int lineno, PyArena
*arena);
int col_offset, PyArena *arena);
stmt_ty Print(expr_ty dest, asdl_seq * values, bool nl, int lineno, int
col_offset, PyArena *arena);
stmt_ty For(expr_ty target, expr_ty iter, asdl_seq * body, asdl_seq * orelse,
int lineno, PyArena *arena);
stmt_ty While(expr_ty test, asdl_seq * body, asdl_seq * orelse, int lineno,
PyArena *arena);
stmt_ty If(expr_ty test, asdl_seq * body, asdl_seq * orelse, int lineno,
PyArena *arena);
int lineno, int col_offset, PyArena *arena);
stmt_ty While(expr_ty test, asdl_seq * body, asdl_seq * orelse, int lineno, int
col_offset, PyArena *arena);
stmt_ty If(expr_ty test, asdl_seq * body, asdl_seq * orelse, int lineno, int
col_offset, PyArena *arena);
stmt_ty With(expr_ty context_expr, expr_ty optional_vars, asdl_seq * body, int
lineno, PyArena *arena);
stmt_ty Raise(expr_ty type, expr_ty inst, expr_ty tback, int lineno, PyArena
*arena);
lineno, int col_offset, PyArena *arena);
stmt_ty Raise(expr_ty type, expr_ty inst, expr_ty tback, int lineno, int
col_offset, PyArena *arena);
stmt_ty TryExcept(asdl_seq * body, asdl_seq * handlers, asdl_seq * orelse, int
lineno, PyArena *arena);
stmt_ty TryFinally(asdl_seq * body, asdl_seq * finalbody, int lineno, PyArena
*arena);
stmt_ty Assert(expr_ty test, expr_ty msg, int lineno, PyArena *arena);
stmt_ty Import(asdl_seq * names, int lineno, PyArena *arena);
lineno, int col_offset, PyArena *arena);
stmt_ty TryFinally(asdl_seq * body, asdl_seq * finalbody, int lineno, int
col_offset, PyArena *arena);
stmt_ty Assert(expr_ty test, expr_ty msg, int lineno, int col_offset, PyArena
*arena);
stmt_ty Import(asdl_seq * names, int lineno, int col_offset, PyArena *arena);
stmt_ty ImportFrom(identifier module, asdl_seq * names, int level, int lineno,
PyArena *arena);
stmt_ty Exec(expr_ty body, expr_ty globals, expr_ty locals, int lineno, PyArena
*arena);
stmt_ty Global(asdl_seq * names, int lineno, PyArena *arena);
stmt_ty Expr(expr_ty value, int lineno, PyArena *arena);
stmt_ty Pass(int lineno, PyArena *arena);
stmt_ty Break(int lineno, PyArena *arena);
stmt_ty Continue(int lineno, PyArena *arena);
expr_ty BoolOp(boolop_ty op, asdl_seq * values, int lineno, PyArena *arena);
expr_ty BinOp(expr_ty left, operator_ty op, expr_ty right, int lineno, PyArena
*arena);
expr_ty UnaryOp(unaryop_ty op, expr_ty operand, int lineno, PyArena *arena);
expr_ty Lambda(arguments_ty args, expr_ty body, int lineno, PyArena *arena);
expr_ty IfExp(expr_ty test, expr_ty body, expr_ty orelse, int lineno, PyArena
*arena);
expr_ty Dict(asdl_seq * keys, asdl_seq * values, int lineno, PyArena *arena);
expr_ty ListComp(expr_ty elt, asdl_seq * generators, int lineno, PyArena
*arena);
expr_ty GeneratorExp(expr_ty elt, asdl_seq * generators, int lineno, PyArena
*arena);
expr_ty Yield(expr_ty value, int lineno, PyArena *arena);
int col_offset, PyArena *arena);
stmt_ty Exec(expr_ty body, expr_ty globals, expr_ty locals, int lineno, int
col_offset, PyArena *arena);
stmt_ty Global(asdl_seq * names, int lineno, int col_offset, PyArena *arena);
stmt_ty Expr(expr_ty value, int lineno, int col_offset, PyArena *arena);
stmt_ty Pass(int lineno, int col_offset, PyArena *arena);
stmt_ty Break(int lineno, int col_offset, PyArena *arena);
stmt_ty Continue(int lineno, int col_offset, PyArena *arena);
expr_ty BoolOp(boolop_ty op, asdl_seq * values, int lineno, int col_offset,
PyArena *arena);
expr_ty BinOp(expr_ty left, operator_ty op, expr_ty right, int lineno, int
col_offset, PyArena *arena);
expr_ty UnaryOp(unaryop_ty op, expr_ty operand, int lineno, int col_offset,
PyArena *arena);
expr_ty Lambda(arguments_ty args, expr_ty body, int lineno, int col_offset,
PyArena *arena);
expr_ty IfExp(expr_ty test, expr_ty body, expr_ty orelse, int lineno, int
col_offset, PyArena *arena);
expr_ty Dict(asdl_seq * keys, asdl_seq * values, int lineno, int col_offset,
PyArena *arena);
expr_ty ListComp(expr_ty elt, asdl_seq * generators, int lineno, int
col_offset, PyArena *arena);
expr_ty GeneratorExp(expr_ty elt, asdl_seq * generators, int lineno, int
col_offset, PyArena *arena);
expr_ty Yield(expr_ty value, int lineno, int col_offset, PyArena *arena);
expr_ty Compare(expr_ty left, asdl_seq * ops, asdl_seq * comparators, int
lineno, PyArena *arena);
lineno, int col_offset, PyArena *arena);
expr_ty Call(expr_ty func, asdl_seq * args, asdl_seq * keywords, expr_ty
starargs, expr_ty kwargs, int lineno, PyArena *arena);
expr_ty Repr(expr_ty value, int lineno, PyArena *arena);
expr_ty Num(object n, int lineno, PyArena *arena);
expr_ty Str(string s, int lineno, PyArena *arena);
starargs, expr_ty kwargs, int lineno, int col_offset, PyArena
*arena);
expr_ty Repr(expr_ty value, int lineno, int col_offset, PyArena *arena);
expr_ty Num(object n, int lineno, int col_offset, PyArena *arena);
expr_ty Str(string s, int lineno, int col_offset, PyArena *arena);
expr_ty Attribute(expr_ty value, identifier attr, expr_context_ty ctx, int
lineno, PyArena *arena);
lineno, int col_offset, PyArena *arena);
expr_ty Subscript(expr_ty value, slice_ty slice, expr_context_ty ctx, int
lineno, PyArena *arena);
expr_ty Name(identifier id, expr_context_ty ctx, int lineno, PyArena *arena);
expr_ty List(asdl_seq * elts, expr_context_ty ctx, int lineno, PyArena *arena);
expr_ty Tuple(asdl_seq * elts, expr_context_ty ctx, int lineno, PyArena *arena);
lineno, int col_offset, PyArena *arena);
expr_ty Name(identifier id, expr_context_ty ctx, int lineno, int col_offset,
PyArena *arena);
expr_ty List(asdl_seq * elts, expr_context_ty ctx, int lineno, int col_offset,
PyArena *arena);
expr_ty Tuple(asdl_seq * elts, expr_context_ty ctx, int lineno, int col_offset,
PyArena *arena);
slice_ty Ellipsis(PyArena *arena);
slice_ty Slice(expr_ty lower, expr_ty upper, expr_ty step, PyArena *arena);
slice_ty ExtSlice(asdl_seq * dims, PyArena *arena);
......
Include/node.h
View file @ 49c5da1d
......@@ -11,13 +11,14 @@ typedef struct _node {
short n_type;
char *n_str;
int n_lineno;
int n_col_offset;
int n_nchildren;
struct _node *n_child;
} node;
PyAPI_FUNC(node *) PyNode_New(int type);
PyAPI_FUNC(int) PyNode_AddChild(node *n, int type,
char *str, int lineno);
char *str, int lineno, int col_offset);
PyAPI_FUNC(void) PyNode_Free(node *n);
/* Node access functions */
......
Lib/test/test_ast.py
View file @ 49c5da1d
import sys, itertools
import _ast
def to_tuple(t):
if t is None or isinstance(t, (basestring, int, long, complex)):
......@@ -6,6 +7,8 @@ def to_tuple(t):
elif isinstance(t, list):
return [to_tuple(e) for e in t]
result = [t.__class__.__name__]
if hasattr(t, 'lineno') and hasattr(t, 'col_offset'):
result.append((t.lineno, t.col_offset))
if t._fields is None:
return tuple(result)
for f in t._fields:
......@@ -106,7 +109,10 @@ eval_tests = [
# List
"[1,2,3]",
# Tuple
"1,2,3"
"1,2,3",
# Combination
"a.b.c.d(a.b[1:2])",
]
# TODO: expr_context, slice, boolop, operator, unaryop, cmpop, comprehension
......@@ -121,58 +127,77 @@ if __name__=='__main__' and sys.argv[1:] == ['-g']:
print "run_tests()"
raise SystemExit
def test_order(ast_node, parent_pos):
if not isinstance(ast_node, _ast.AST) or ast_node._fields == None:
return
if isinstance(ast_node, (_ast.expr, _ast.stmt)):
node_pos = (ast_node.lineno, ast_node.col_offset)
assert node_pos >= parent_pos, (node_pos, parent_pos)
parent_pos = (ast_node.lineno, ast_node.col_offset)
for name in ast_node._fields:
value = getattr(ast_node, name)
if isinstance(value, list):
for child in value:
test_order(child, parent_pos)
elif value != None:
test_order(value, parent_pos)
def run_tests():
for input, output, kind in ((exec_tests, exec_results, "exec"),
(single_tests, single_results, "single"),
(eval_tests, eval_results, "eval")):
for i, o in itertools.izip(input, output):
assert to_tuple(compile(i, "?", kind, 0x400)) == o
ast_tree = compile(i, "?", kind, 0x400)
assert to_tuple(ast_tree) == o
test_order(ast_tree, (0, 0))
#### EVERYTHING BELOW IS GENERATED #####
exec_results = [
('Module', [('FunctionDef', 'f', ('arguments', [], None, None, []), [('Pass',)], [])]),
('Module', [('ClassDef', 'C', [], [('Pass',)])]),
('Module', [('FunctionDef', 'f', ('arguments', [], None, None, []), [('Return', ('Num', 1))], [])]),
('Module', [('Delete', [('Name', 'v', ('Del',))])]),
('Module', [('Assign', [('Name', 'v', ('Store',))], ('Num', 1))]),
('Module', [('AugAssign', ('Name', 'v', ('Load',)), ('Add',), ('Num', 1))]),
('Module', [('Print', ('Name', 'f', ('Load',)), [('Num', 1)], False)]),
('Module', [('For', ('Name', 'v', ('Store',)), ('Name', 'v', ('Load',)), [('Pass',)], [])]),
('Module', [('While', ('Name', 'v', ('Load',)), [('Pass',)], [])]),
('Module', [('If', ('Name', 'v', ('Load',)), [('Pass',)], [])]),
('Module', [('Raise', ('Name', 'Exception', ('Load',)), ('Str', 'string'), None)]),
('Module', [('TryExcept', [('Pass',)], [('excepthandler', ('Name', 'Exception', ('Load',)), None, [('Pass',)])], [])]),
('Module', [('TryFinally', [('Pass',)], [('Pass',)])]),
('Module', [('Assert', ('Name', 'v', ('Load',)), None)]),
('Module', [('Import', [('alias', 'sys', None)])]),
('Module', [('ImportFrom', 'sys', [('alias', 'v', None)], 0)]),
('Module', [('Exec', ('Str', 'v'), None, None)]),
('Module', [('Global', ['v'])]),
('Module', [('Expr', ('Num', 1))]),
('Module', [('Pass',)]),
('Module', [('Break',)]),
('Module', [('Continue',)]),
('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [], None, None, []), [('Pass', (1, 9))], [])]),
('Module', [('ClassDef', (1, 0), 'C', [], [('Pass', (1, 8))])]),
('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [], None, None, []), [('Return', (1, 8), ('Num', (1, 15), 1))], [])]),
('Module', [('Delete', (1, 0), [('Name', (1, 4), 'v', ('Del',))])]),
('Module', [('Assign', (1, 0), [('Name', (1, 0), 'v', ('Store',))], ('Num', (1, 4), 1))]),
('Module', [('AugAssign', (1, 0), ('Name', (1, 0), 'v', ('Load',)), ('Add',), ('Num', (1, 5), 1))]),
('Module', [('Print', (1, 0), ('Name', (1, 8), 'f', ('Load',)), [('Num', (1, 11), 1)], False)]),
('Module', [('For', (1, 0), ('Name', (1, 4), 'v', ('Store',)), ('Name', (1, 9), 'v', ('Load',)), [('Pass', (1, 11))], [])]),
('Module', [('While', (1, 0), ('Name', (1, 6), 'v', ('Load',)), [('Pass', (1, 8))], [])]),
('Module', [('If', (1, 0), ('Name', (1, 3), 'v', ('Load',)), [('Pass', (1, 5))], [])]),
('Module', [('Raise', (1, 0), ('Name', (1, 6), 'Exception', ('Load',)), ('Str', (1, 17), 'string'), None)]),
('Module', [('TryExcept', (1, 0), [('Pass', (2, 2))], [('excepthandler', ('Name', (3, 7), 'Exception', ('Load',)), None, [('Pass', (4, 2))])], [])]),
('Module', [('TryFinally', (1, 0), [('Pass', (2, 2))], [('Pass', (4, 2))])]),
('Module', [('Assert', (1, 0), ('Name', (1, 7), 'v', ('Load',)), None)]),
('Module', [('Import', (1, 0), [('alias', 'sys', None)])]),
('Module', [('ImportFrom', (1, 0), 'sys', [('alias', 'v', None)], 0)]),
('Module', [('Exec', (1, 0), ('Str', (1, 5), 'v'), None, None)]),
('Module', [('Global', (1, 0), ['v'])]),
('Module', [('Expr', (1, 0), ('Num', (1, 0), 1))]),
('Module', [('Pass', (1, 0))]),
('Module', [('Break', (1, 0))]),
('Module', [('Continue', (1, 0))]),
]
single_results = [
('Interactive', [('Expr', ('BinOp', ('Num', 1), ('Add',), ('Num', 2)))]),
('Interactive', [('Expr', (1, 0), ('BinOp', (1, 0), ('Num', (1, 0), 1), ('Add',), ('Num', (1, 2), 2)))]),
]
eval_results = [
('Expression', ('BoolOp', ('And',), [('Name', 'a', ('Load',)), ('Name', 'b', ('Load',))])),
('Expression', ('BinOp', ('Name', 'a', ('Load',)), ('Add',), ('Name', 'b', ('Load',)))),
('Expression', ('UnaryOp', ('Not',), ('Name', 'v', ('Load',)))),
('Expression', ('Lambda', ('arguments', [], None, None, []), ('Name', 'None', ('Load',)))),
('Expression', ('Dict', [('Num', 1)], [('Num', 2)])),
('Expression', ('ListComp', ('Name', 'a', ('Load',)), [('comprehension', ('Name', 'b', ('Store',)), ('Name', 'c', ('Load',)), [('Name', 'd', ('Load',))])])),
('Expression', ('GeneratorExp', ('Name', 'a', ('Load',)), [('comprehension', ('Name', 'b', ('Store',)), ('Name', 'c', ('Load',)), [('Name', 'd', ('Load',))])])),
('Expression', ('Compare', ('Num', 1), [('Lt',), ('Lt',)], [('Num', 2), ('Num', 3)])),
('Expression', ('Call', ('Name', 'f', ('Load',)), [('Num', 1), ('Num', 2)], [('keyword', 'c', ('Num', 3))], ('Name', 'd', ('Load',)), ('Name', 'e', ('Load',)))),
('Expression', ('Repr', ('Name', 'v', ('Load',)))),
('Expression', ('Num', 10L)),
('Expression', ('Str', 'string')),
('Expression', ('Attribute', ('Name', 'a', ('Load',)), 'b', ('Load',))),
('Expression', ('Subscript', ('Name', 'a', ('Load',)), ('Slice', ('Name', 'b', ('Load',)), ('Name', 'c', ('Load',)), None), ('Load',))),
('Expression', ('Name', 'v', ('Load',))),
('Expression', ('List', [('Num', 1), ('Num', 2), ('Num', 3)], ('Load',))),
('Expression', ('Tuple', [('Num', 1), ('Num', 2), ('Num', 3)], ('Load',))),
('Expression', ('BoolOp', (1, 0), ('And',), [('Name', (1, 0), 'a', ('Load',)), ('Name', (1, 6), 'b', ('Load',))])),
('Expression', ('BinOp', (1, 0), ('Name', (1, 0), 'a', ('Load',)), ('Add',), ('Name', (1, 4), 'b', ('Load',)))),
('Expression', ('UnaryOp', (1, 0), ('Not',), ('Name', (1, 4), 'v', ('Load',)))),
('Expression', ('Lambda', (1, 0), ('arguments', [], None, None, []), ('Name', (1, 7), 'None', ('Load',)))),
('Expression', ('Dict', (1, 0), [('Num', (1, 2), 1)], [('Num', (1, 4), 2)])),
('Expression', ('ListComp', (1, 1), ('Name', (1, 1), 'a', ('Load',)), [('comprehension', ('Name', (1, 7), 'b', ('Store',)), ('Name', (1, 12), 'c', ('Load',)), [('Name', (1, 17), 'd', ('Load',))])])),
('Expression', ('GeneratorExp', (1, 1), ('Name', (1, 1), 'a', ('Load',)), [('comprehension', ('Name', (1, 7), 'b', ('Store',)), ('Name', (1, 12), 'c', ('Load',)), [('Name', (1, 17), 'd', ('Load',))])])),
('Expression', ('Compare', (1, 0), ('Num', (1, 0), 1), [('Lt',), ('Lt',)], [('Num', (1, 4), 2), ('Num', (1, 8), 3)])),
('Expression', ('Call', (1, 0), ('Name', (1, 0), 'f', ('Load',)), [('Num', (1, 2), 1), ('Num', (1, 4), 2)], [('keyword', 'c', ('Num', (1, 8), 3))], ('Name', (1, 11), 'd', ('Load',)), ('Name', (1, 15), 'e', ('Load',)))),
('Expression', ('Repr', (1, 0), ('Name', (1, 1), 'v', ('Load',)))),
('Expression', ('Num', (1, 0), 10L)),
('Expression', ('Str', (1, 0), 'string')),
('Expression', ('Attribute', (1, 0), ('Name', (1, 0), 'a', ('Load',)), 'b', ('Load',))),
('Expression', ('Subscript', (1, 0), ('Name', (1, 0), 'a', ('Load',)), ('Slice', ('Name', (1, 2), 'b', ('Load',)), ('Name', (1, 4), 'c', ('Load',)), None), ('Load',))),
('Expression', ('Name', (1, 0), 'v', ('Load',))),
('Expression', ('List', (1, 0), [('Num', (1, 1), 1), ('Num', (1, 3), 2), ('Num', (1, 5), 3)], ('Load',))),
('Expression', ('Tuple', (1, 0), [('Num', (1, 0), 1), ('Num', (1, 2), 2), ('Num', (1, 4), 3)], ('Load',))),
('Expression', ('Call', (1, 0), ('Attribute', (1, 0), ('Attribute', (1, 0), ('Attribute', (1, 0), ('Name', (1, 0), 'a', ('Load',)), 'b', ('Load',)), 'c', ('Load',)), 'd', ('Load',)), [('Subscript', (1, 8), ('Attribute', (1, 8), ('Name', (1, 8), 'a', ('Load',)), 'b', ('Load',)), ('Slice', ('Num', (1, 12), 1), ('Num', (1, 14), 2), None), ('Load',))], [], None, None)),
]
run_tests()
Modules/parsermodule.c
View file @ 49c5da1d
......@@ -715,7 +715,7 @@ build_node_children(PyObject *tuple, node *root, int *line_num)
Py_XDECREF(elem);
return (0);
}
err = PyNode_AddChild(root, type, strn, *line_num);
err = PyNode_AddChild(root, type, strn, *line_num, 0);
if (err == E_NOMEM) {
PyMem_DEL(strn);
return (node *) PyErr_NoMemory();
......
Parser/Python.asdl
View file @ 49c5da1d
......@@ -46,7 +46,8 @@ module Python version "$Revision$"
| Pass | Break | Continue
-- XXX Jython will be different
attributes (int lineno)
-- col_offset is the byte offset in the utf8 string the parser uses
attributes (int lineno, int col_offset)
-- BoolOp() can use left & right?
expr = BoolOp(boolop op, expr* values)
......@@ -76,7 +77,8 @@ module Python version "$Revision$"
| List(expr* elts, expr_context ctx)
| Tuple(expr *elts, expr_context ctx)
attributes (int lineno)
-- col_offset is the byte offset in the utf8 string the parser uses
attributes (int lineno, int col_offset)
expr_context = Load | Store | Del | AugLoad | AugStore | Param
......
Parser/asdl.py
View file @ 49c5da1d
......@@ -156,6 +156,8 @@ class ASDLParser(spark.GenericParser, object):
if id.value != "attributes":
raise ASDLSyntaxError(id.lineno,
msg="expected attributes, found %s" % id)
if attributes:
attributes.reverse()
return Sum(sum, attributes)
def p_product(self, (_0, fields, _1)):
......
Parser/node.c
View file @ 49c5da1d
......@@ -76,7 +76,7 @@ fancy_roundup(int n)
int
PyNode_AddChild(register node *n1, int type, char *str, int lineno)
PyNode_AddChild(register node *n1, int type, char *str, int lineno, int col_offset)
{
const int nch = n1->n_nchildren;
int current_capacity;
......@@ -103,6 +103,7 @@ PyNode_AddChild(register node *n1, int type, char *str, int lineno)
n->n_type = type;
n->n_str = str;
n->n_lineno = lineno;
n->n_col_offset = col_offset;
n->n_nchildren = 0;
n->n_child = NULL;
return 0;
......
Parser/parser.c
View file @ 49c5da1d
......@@ -105,11 +105,11 @@ PyParser_Delete(parser_state *ps)
/* PARSER STACK OPERATIONS */
static int
shift(register stack *s, int type, char *str, int newstate, int lineno)
shift(register stack *s, int type, char *str, int newstate, int lineno, int col_offset)
{
int err;
assert(!s_empty(s));
err = PyNode_AddChild(s->s_top->s_parent, type, str, lineno);
err = PyNode_AddChild(s->s_top->s_parent, type, str, lineno, col_offset);
if (err)
return err;
s->s_top->s_state = newstate;
......@@ -117,13 +117,13 @@ shift(register stack *s, int type, char *str, int newstate, int lineno)
}
static int
push(register stack *s, int type, dfa *d, int newstate, int lineno)
push(register stack *s, int type, dfa *d, int newstate, int lineno, int col_offset)
{
int err;
register node *n;
n = s->s_top->s_parent;
assert(!s_empty(s));
err = PyNode_AddChild(n, type, (char *)NULL, lineno);
err = PyNode_AddChild(n, type, (char *)NULL, lineno, col_offset);
if (err)
return err;
s->s_top->s_state = newstate;
......@@ -213,7 +213,7 @@ future_hack(parser_state *ps)
int
PyParser_AddToken(register parser_state *ps, register int type, char *str,
int lineno, int *expected_ret)
int lineno, int col_offset, int *expected_ret)
{
register int ilabel;
int err;
......@@ -245,7 +245,7 @@ PyParser_AddToken(register parser_state *ps, register int type, char *str,
dfa *d1 = PyGrammar_FindDFA(
ps->p_grammar, nt);
if ((err = push(&ps->p_stack, nt, d1,
arrow, lineno)) > 0) {
arrow, lineno, col_offset)) > 0) {
D(printf(" MemError: push\n"));
return err;
}
......@@ -255,7 +255,7 @@ PyParser_AddToken(register parser_state *ps, register int type, char *str,
/* Shift the token */
if ((err = shift(&ps->p_stack, type, str,
x, lineno)) > 0) {
x, lineno, col_offset)) > 0) {
D(printf(" MemError: shift.\n"));
return err;
}
......
Parser/parser.h
View file @ 49c5da1d
......@@ -32,7 +32,7 @@ typedef struct {
parser_state *PyParser_New(grammar *g, int start);
void PyParser_Delete(parser_state *ps);
int PyParser_AddToken(parser_state *ps, int type, char *str, int lineno,
int PyParser_AddToken(parser_state *ps, int type, char *str, int lineno, int col_offset,
int *expected_ret);
void PyGrammar_AddAccelerators(grammar *g);
......
Parser/parsetok.c
View file @ 49c5da1d
......@@ -130,6 +130,7 @@ parsetok(struct tok_state *tok, grammar *g, int start, perrdetail *err_ret,
int type;
size_t len;
char *str;
int col_offset;
type = PyTokenizer_Get(tok, &a, &b);
if (type == ERRORTOKEN) {
......@@ -185,9 +186,13 @@ parsetok(struct tok_state *tok, grammar *g, int start, perrdetail *err_ret,
len == 4 && str[0] == 'w' && strcmp(str, "with") == 0)
handling_with = 1;
#endif
if (a >= tok->line_start)
col_offset = a - tok->line_start;
else
col_offset = -1;
if ((err_ret->error =
PyParser_AddToken(ps, (int)type, str, tok->lineno,
PyParser_AddToken(ps, (int)type, str, tok->lineno, col_offset,
&(err_ret->expected))) != E_OK) {
if (err_ret->error != E_DONE)
PyObject_FREE(str);
......
Parser/tokenizer.c
View file @ 49c5da1d
......@@ -764,6 +764,7 @@ tok_nextc(register struct tok_state *tok)
}
if (tok->start == NULL)
tok->buf = tok->cur;
tok->line_start = tok->cur;
tok->lineno++;
tok->inp = end;
return Py_CHARMASK(*tok->cur++);
......@@ -798,6 +799,7 @@ tok_nextc(register struct tok_state *tok)
}
tok->buf = buf;
tok->cur = tok->buf + oldlen;
tok->line_start = tok->cur;
strcpy(tok->buf + oldlen, new);
PyMem_FREE(new);
tok->inp = tok->buf + newlen;
......@@ -809,7 +811,9 @@ tok_nextc(register struct tok_state *tok)
if (tok->buf != NULL)
PyMem_DEL(tok->buf);
tok->buf = new;
tok->line_start = tok->buf;
tok->cur = tok->buf;
tok->line_start = tok->buf;
tok->inp = strchr(tok->buf, '\0');
tok->end = tok->inp + 1;
}
......@@ -877,6 +881,7 @@ tok_nextc(register struct tok_state *tok)
done = tok->inp[-1] == '\n';
}
tok->cur = tok->buf + cur;
tok->line_start = tok->cur;
/* replace "\r\n" with "\n" */
/* For Mac we leave the \r, giving a syntax error */
pt = tok->inp - 2;
......
Parser/tokenizer.h
View file @ 49c5da1d
......@@ -45,6 +45,7 @@ struct tok_state {
int read_coding_spec; /* whether 'coding:...' has been read */
char *encoding;
int cont_line; /* whether we are in a continuation line. */
const char* line_start; /* pointer to start of current line */
#ifndef PGEN
PyObject *decoding_readline; /* codecs.open(...).readline */
PyObject *decoding_buffer;
......
Python/Python-ast.c
View file @ 49c5da1d
This diff is collapsed.
Python/ast.c
View file @ 49c5da1d
This diff is collapsed.
Python/compile.c
View file @ 49c5da1d
......@@ -3635,7 +3635,7 @@ compiler_augassign(struct compiler *c, stmt_ty s)
switch (e->kind) {
case Attribute_kind:
auge = Attribute(e->v.Attribute.value, e->v.Attribute.attr,
AugLoad, e->lineno, c->c_arena);
AugLoad, e->lineno, e->col_offset, c->c_arena);
if (auge == NULL)
return 0;
VISIT(c, expr, auge);
......@@ -3646,7 +3646,7 @@ compiler_augassign(struct compiler *c, stmt_ty s)
break;
case Subscript_kind:
auge = Subscript(e->v.Subscript.value, e->v.Subscript.slice,
AugLoad, e->lineno, c->c_arena);
AugLoad, e->lineno, e->col_offset, c->c_arena);
if (auge == NULL)
return 0;
VISIT(c, expr, auge);
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7