Commit 37c9351c authored by Jeremy Hylton's avatar Jeremy Hylton

Handle more syntax errors.

Invoke compiler.syntax.check() after building AST.  If a SyntaxError
occurs, print the error and exit without generating a .pyc file.

Refactor code to use compiler.misc.set_filename() rather than passing
filename argument around to each CodeGenerator instance.
parent 09392b77
...@@ -8,7 +8,7 @@ import sys ...@@ -8,7 +8,7 @@ import sys
import types import types
from cStringIO import StringIO from cStringIO import StringIO
from compiler import ast, parse, walk from compiler import ast, parse, walk, syntax
from compiler import pyassem, misc, future, symbols from compiler import pyassem, misc, future, symbols
from compiler.consts import SC_LOCAL, SC_GLOBAL, SC_FREE, SC_CELL from compiler.consts import SC_LOCAL, SC_GLOBAL, SC_FREE, SC_CELL
from compiler.consts import CO_VARARGS, CO_VARKEYWORDS, CO_NEWLOCALS,\ from compiler.consts import CO_VARARGS, CO_VARKEYWORDS, CO_NEWLOCALS,\
...@@ -41,17 +41,19 @@ class BlockStack(misc.Stack): ...@@ -41,17 +41,19 @@ class BlockStack(misc.Stack):
self.__super_init(self) self.__super_init(self)
self.loop = None self.loop = None
def compile(filename, display=0): def compile(filename, display=0):
f = open(filename) f = open(filename)
buf = f.read() buf = f.read()
f.close() f.close()
mod = Module(buf, filename) mod = Module(buf, filename)
mod.compile(display) try:
f = open(filename + "c", "wb") mod.compile(display)
mod.dump(f) except SyntaxError, err:
f.close() print "SyntaxError:", err
else:
f = open(filename + "c", "wb")
mod.dump(f)
f.close()
class Module: class Module:
def __init__(self, source, filename): def __init__(self, source, filename):
...@@ -61,7 +63,9 @@ class Module: ...@@ -61,7 +63,9 @@ class Module:
def compile(self, display=0): def compile(self, display=0):
tree = parse(self.source) tree = parse(self.source)
gen = ModuleCodeGenerator(self.filename, tree) misc.set_filename(self.filename, tree)
syntax.check(tree)
gen = ModuleCodeGenerator(tree)
if display: if display:
import pprint import pprint
print pprint.pprint(tree) print pprint.pprint(tree)
...@@ -149,12 +153,11 @@ class CodeGenerator: ...@@ -149,12 +153,11 @@ class CodeGenerator:
__initialized = None __initialized = None
class_name = None # provide default for instance variable class_name = None # provide default for instance variable
def __init__(self, filename): def __init__(self):
if self.__initialized is None: if self.__initialized is None:
self.initClass() self.initClass()
self.__class__.__initialized = 1 self.__class__.__initialized = 1
self.checkClass() self.checkClass()
self.filename = filename
self.locals = misc.Stack() self.locals = misc.Stack()
self.setups = misc.Stack() self.setups = misc.Stack()
self.curStack = 0 self.curStack = 0
...@@ -306,7 +309,7 @@ class CodeGenerator: ...@@ -306,7 +309,7 @@ class CodeGenerator:
self._visitFuncOrLambda(node, isLambda=1) self._visitFuncOrLambda(node, isLambda=1)
def _visitFuncOrLambda(self, node, isLambda=0): def _visitFuncOrLambda(self, node, isLambda=0):
gen = self.FunctionGen(node, self.filename, self.scopes, isLambda, gen = self.FunctionGen(node, self.scopes, isLambda,
self.class_name, self.get_module()) self.class_name, self.get_module())
walk(node.code, gen) walk(node.code, gen)
gen.finish() gen.finish()
...@@ -324,7 +327,7 @@ class CodeGenerator: ...@@ -324,7 +327,7 @@ class CodeGenerator:
self.emit('MAKE_FUNCTION', len(node.defaults)) self.emit('MAKE_FUNCTION', len(node.defaults))
def visitClass(self, node): def visitClass(self, node):
gen = self.ClassGen(node, self.filename, self.scopes, gen = self.ClassGen(node, self.scopes,
self.get_module()) self.get_module())
if node.doc: if node.doc:
self.emit('LOAD_CONST', node.doc) self.emit('LOAD_CONST', node.doc)
...@@ -430,14 +433,14 @@ class CodeGenerator: ...@@ -430,14 +433,14 @@ class CodeGenerator:
def visitBreak(self, node): def visitBreak(self, node):
if not self.setups: if not self.setups:
raise SyntaxError, "'break' outside loop (%s, %d)" % \ raise SyntaxError, "'break' outside loop (%s, %d)" % \
(self.filename, node.lineno) (node.filename, node.lineno)
self.set_lineno(node) self.set_lineno(node)
self.emit('BREAK_LOOP') self.emit('BREAK_LOOP')
def visitContinue(self, node): def visitContinue(self, node):
if not self.setups: if not self.setups:
raise SyntaxError, "'continue' outside loop (%s, %d)" % \ raise SyntaxError, "'continue' outside loop (%s, %d)" % \
(self.filename, node.lineno) (node.filename, node.lineno)
kind, block = self.setups.top() kind, block = self.setups.top()
if kind == LOOP: if kind == LOOP:
self.set_lineno(node) self.set_lineno(node)
...@@ -454,12 +457,12 @@ class CodeGenerator: ...@@ -454,12 +457,12 @@ class CodeGenerator:
break break
if kind != LOOP: if kind != LOOP:
raise SyntaxError, "'continue' outside loop (%s, %d)" % \ raise SyntaxError, "'continue' outside loop (%s, %d)" % \
(self.filename, node.lineno) (node.filename, node.lineno)
self.emit('CONTINUE_LOOP', loop_block) self.emit('CONTINUE_LOOP', loop_block)
self.nextBlock() self.nextBlock()
elif kind == END_FINALLY: elif kind == END_FINALLY:
msg = "'continue' not allowed inside 'finally' clause (%s, %d)" msg = "'continue' not allowed inside 'finally' clause (%s, %d)"
raise SyntaxError, msg % (self.filename, node.lineno) raise SyntaxError, msg % (node.filename, node.lineno)
def visitTest(self, node, jump): def visitTest(self, node, jump):
end = self.newBlock() end = self.newBlock()
...@@ -1085,10 +1088,10 @@ class ModuleCodeGenerator(NestedScopeMixin, CodeGenerator): ...@@ -1085,10 +1088,10 @@ class ModuleCodeGenerator(NestedScopeMixin, CodeGenerator):
scopes = None scopes = None
def __init__(self, filename, tree): def __init__(self, tree):
self.graph = pyassem.PyFlowGraph("<module>", filename) self.graph = pyassem.PyFlowGraph("<module>", tree.filename)
self.futures = future.find_futures(tree) self.futures = future.find_futures(tree)
self.__super_init(filename) self.__super_init()
walk(tree, self) walk(tree, self)
def get_module(self): def get_module(self):
...@@ -1098,7 +1101,7 @@ class AbstractFunctionCode: ...@@ -1098,7 +1101,7 @@ class AbstractFunctionCode:
optimized = 1 optimized = 1
lambdaCount = 0 lambdaCount = 0
def __init__(self, func, filename, scopes, isLambda, class_name, mod): def __init__(self, func, scopes, isLambda, class_name, mod):
self.class_name = class_name self.class_name = class_name
self.module = mod self.module = mod
if isLambda: if isLambda:
...@@ -1108,10 +1111,10 @@ class AbstractFunctionCode: ...@@ -1108,10 +1111,10 @@ class AbstractFunctionCode:
else: else:
name = func.name name = func.name
args, hasTupleArg = generateArgList(func.argnames) args, hasTupleArg = generateArgList(func.argnames)
self.graph = pyassem.PyFlowGraph(name, filename, args, self.graph = pyassem.PyFlowGraph(name, func.filename, args,
optimized=1) optimized=1)
self.isLambda = isLambda self.isLambda = isLambda
self.super_init(filename) self.super_init()
if not isLambda and func.doc: if not isLambda and func.doc:
self.setDocstring(func.doc) self.setDocstring(func.doc)
...@@ -1162,10 +1165,10 @@ class FunctionCodeGenerator(NestedScopeMixin, AbstractFunctionCode, ...@@ -1162,10 +1165,10 @@ class FunctionCodeGenerator(NestedScopeMixin, AbstractFunctionCode,
__super_init = AbstractFunctionCode.__init__ __super_init = AbstractFunctionCode.__init__
def __init__(self, func, filename, scopes, isLambda, class_name, mod): def __init__(self, func, scopes, isLambda, class_name, mod):
self.scopes = scopes self.scopes = scopes
self.scope = scopes[func] self.scope = scopes[func]
self.__super_init(func, filename, scopes, isLambda, class_name, mod) self.__super_init(func, scopes, isLambda, class_name, mod)
self.graph.setFreeVars(self.scope.get_free_vars()) self.graph.setFreeVars(self.scope.get_free_vars())
self.graph.setCellVars(self.scope.get_cell_vars()) self.graph.setCellVars(self.scope.get_cell_vars())
if self.graph.checkFlag(CO_GENERATOR_ALLOWED): if self.graph.checkFlag(CO_GENERATOR_ALLOWED):
...@@ -1174,12 +1177,12 @@ class FunctionCodeGenerator(NestedScopeMixin, AbstractFunctionCode, ...@@ -1174,12 +1177,12 @@ class FunctionCodeGenerator(NestedScopeMixin, AbstractFunctionCode,
class AbstractClassCode: class AbstractClassCode:
def __init__(self, klass, filename, scopes, module): def __init__(self, klass, scopes, module):
self.class_name = klass.name self.class_name = klass.name
self.module = module self.module = module
self.graph = pyassem.PyFlowGraph(klass.name, filename, self.graph = pyassem.PyFlowGraph(klass.name, klass.filename,
optimized=0, klass=1) optimized=0, klass=1)
self.super_init(filename) self.super_init()
lnf = walk(klass.code, self.NameFinder(), verbose=0) lnf = walk(klass.code, self.NameFinder(), verbose=0)
self.locals.push(lnf.getLocals()) self.locals.push(lnf.getLocals())
self.graph.setFlag(CO_NEWLOCALS) self.graph.setFlag(CO_NEWLOCALS)
...@@ -1200,10 +1203,10 @@ class ClassCodeGenerator(NestedScopeMixin, AbstractClassCode, CodeGenerator): ...@@ -1200,10 +1203,10 @@ class ClassCodeGenerator(NestedScopeMixin, AbstractClassCode, CodeGenerator):
__super_init = AbstractClassCode.__init__ __super_init = AbstractClassCode.__init__
def __init__(self, klass, filename, scopes, module): def __init__(self, klass, scopes, module):
self.scopes = scopes self.scopes = scopes
self.scope = scopes[klass] self.scope = scopes[klass]
self.__super_init(klass, filename, scopes, module) self.__super_init(klass, scopes, module)
self.graph.setFreeVars(self.scope.get_free_vars()) self.graph.setFreeVars(self.scope.get_free_vars())
self.graph.setCellVars(self.scope.get_cell_vars()) self.graph.setCellVars(self.scope.get_cell_vars())
## self.graph.setFlag(CO_NESTED) ## self.graph.setFlag(CO_NESTED)
......
"""Check for errs in the AST.
The Python parser does not catch all syntax errors. Others, like
assignments with invalid targets, are caught in the code generation
phase.
The compiler package catches some errors in the transformer module.
But it seems clearer to write checkers that use the AST to detect
errors.
"""
from compiler import ast, walk
def check(tree, multi=None):
v = SyntaxErrorChecker(multi)
walk(tree, v)
return v.errors
class SyntaxErrorChecker:
"""A visitor to find syntax errors in the AST."""
def __init__(self, multi=None):
"""Create new visitor object.
If optional argument multi is not None, then print messages
for each error rather than raising a SyntaxError for the
first.
"""
self.multi = multi
self.errors = 0
def error(self, node, msg):
self.errors = self.errors + 1
if self.multi is not None:
print "%s:%s: %s" % (node.filename, node.lineno, msg)
else:
raise SyntaxError, "%s (%s:%s)" % (msg, node.filename, node.lineno)
def visitAssign(self, node):
# the transformer module handles many of these
for target in node.nodes:
if isinstance(target, ast.AssList):
if target.lineno is None:
target.lineno = node.lineno
self.error(target, "can't assign to list comprehension")
...@@ -8,7 +8,7 @@ import sys ...@@ -8,7 +8,7 @@ import sys
import types import types
from cStringIO import StringIO from cStringIO import StringIO
from compiler import ast, parse, walk from compiler import ast, parse, walk, syntax
from compiler import pyassem, misc, future, symbols from compiler import pyassem, misc, future, symbols
from compiler.consts import SC_LOCAL, SC_GLOBAL, SC_FREE, SC_CELL from compiler.consts import SC_LOCAL, SC_GLOBAL, SC_FREE, SC_CELL
from compiler.consts import CO_VARARGS, CO_VARKEYWORDS, CO_NEWLOCALS,\ from compiler.consts import CO_VARARGS, CO_VARKEYWORDS, CO_NEWLOCALS,\
...@@ -41,17 +41,19 @@ class BlockStack(misc.Stack): ...@@ -41,17 +41,19 @@ class BlockStack(misc.Stack):
self.__super_init(self) self.__super_init(self)
self.loop = None self.loop = None
def compile(filename, display=0): def compile(filename, display=0):
f = open(filename) f = open(filename)
buf = f.read() buf = f.read()
f.close() f.close()
mod = Module(buf, filename) mod = Module(buf, filename)
mod.compile(display) try:
f = open(filename + "c", "wb") mod.compile(display)
mod.dump(f) except SyntaxError, err:
f.close() print "SyntaxError:", err
else:
f = open(filename + "c", "wb")
mod.dump(f)
f.close()
class Module: class Module:
def __init__(self, source, filename): def __init__(self, source, filename):
...@@ -61,7 +63,9 @@ class Module: ...@@ -61,7 +63,9 @@ class Module:
def compile(self, display=0): def compile(self, display=0):
tree = parse(self.source) tree = parse(self.source)
gen = ModuleCodeGenerator(self.filename, tree) misc.set_filename(self.filename, tree)
syntax.check(tree)
gen = ModuleCodeGenerator(tree)
if display: if display:
import pprint import pprint
print pprint.pprint(tree) print pprint.pprint(tree)
...@@ -149,12 +153,11 @@ class CodeGenerator: ...@@ -149,12 +153,11 @@ class CodeGenerator:
__initialized = None __initialized = None
class_name = None # provide default for instance variable class_name = None # provide default for instance variable
def __init__(self, filename): def __init__(self):
if self.__initialized is None: if self.__initialized is None:
self.initClass() self.initClass()
self.__class__.__initialized = 1 self.__class__.__initialized = 1
self.checkClass() self.checkClass()
self.filename = filename
self.locals = misc.Stack() self.locals = misc.Stack()
self.setups = misc.Stack() self.setups = misc.Stack()
self.curStack = 0 self.curStack = 0
...@@ -306,7 +309,7 @@ class CodeGenerator: ...@@ -306,7 +309,7 @@ class CodeGenerator:
self._visitFuncOrLambda(node, isLambda=1) self._visitFuncOrLambda(node, isLambda=1)
def _visitFuncOrLambda(self, node, isLambda=0): def _visitFuncOrLambda(self, node, isLambda=0):
gen = self.FunctionGen(node, self.filename, self.scopes, isLambda, gen = self.FunctionGen(node, self.scopes, isLambda,
self.class_name, self.get_module()) self.class_name, self.get_module())
walk(node.code, gen) walk(node.code, gen)
gen.finish() gen.finish()
...@@ -324,7 +327,7 @@ class CodeGenerator: ...@@ -324,7 +327,7 @@ class CodeGenerator:
self.emit('MAKE_FUNCTION', len(node.defaults)) self.emit('MAKE_FUNCTION', len(node.defaults))
def visitClass(self, node): def visitClass(self, node):
gen = self.ClassGen(node, self.filename, self.scopes, gen = self.ClassGen(node, self.scopes,
self.get_module()) self.get_module())
if node.doc: if node.doc:
self.emit('LOAD_CONST', node.doc) self.emit('LOAD_CONST', node.doc)
...@@ -430,14 +433,14 @@ class CodeGenerator: ...@@ -430,14 +433,14 @@ class CodeGenerator:
def visitBreak(self, node): def visitBreak(self, node):
if not self.setups: if not self.setups:
raise SyntaxError, "'break' outside loop (%s, %d)" % \ raise SyntaxError, "'break' outside loop (%s, %d)" % \
(self.filename, node.lineno) (node.filename, node.lineno)
self.set_lineno(node) self.set_lineno(node)
self.emit('BREAK_LOOP') self.emit('BREAK_LOOP')
def visitContinue(self, node): def visitContinue(self, node):
if not self.setups: if not self.setups:
raise SyntaxError, "'continue' outside loop (%s, %d)" % \ raise SyntaxError, "'continue' outside loop (%s, %d)" % \
(self.filename, node.lineno) (node.filename, node.lineno)
kind, block = self.setups.top() kind, block = self.setups.top()
if kind == LOOP: if kind == LOOP:
self.set_lineno(node) self.set_lineno(node)
...@@ -454,12 +457,12 @@ class CodeGenerator: ...@@ -454,12 +457,12 @@ class CodeGenerator:
break break
if kind != LOOP: if kind != LOOP:
raise SyntaxError, "'continue' outside loop (%s, %d)" % \ raise SyntaxError, "'continue' outside loop (%s, %d)" % \
(self.filename, node.lineno) (node.filename, node.lineno)
self.emit('CONTINUE_LOOP', loop_block) self.emit('CONTINUE_LOOP', loop_block)
self.nextBlock() self.nextBlock()
elif kind == END_FINALLY: elif kind == END_FINALLY:
msg = "'continue' not allowed inside 'finally' clause (%s, %d)" msg = "'continue' not allowed inside 'finally' clause (%s, %d)"
raise SyntaxError, msg % (self.filename, node.lineno) raise SyntaxError, msg % (node.filename, node.lineno)
def visitTest(self, node, jump): def visitTest(self, node, jump):
end = self.newBlock() end = self.newBlock()
...@@ -1085,10 +1088,10 @@ class ModuleCodeGenerator(NestedScopeMixin, CodeGenerator): ...@@ -1085,10 +1088,10 @@ class ModuleCodeGenerator(NestedScopeMixin, CodeGenerator):
scopes = None scopes = None
def __init__(self, filename, tree): def __init__(self, tree):
self.graph = pyassem.PyFlowGraph("<module>", filename) self.graph = pyassem.PyFlowGraph("<module>", tree.filename)
self.futures = future.find_futures(tree) self.futures = future.find_futures(tree)
self.__super_init(filename) self.__super_init()
walk(tree, self) walk(tree, self)
def get_module(self): def get_module(self):
...@@ -1098,7 +1101,7 @@ class AbstractFunctionCode: ...@@ -1098,7 +1101,7 @@ class AbstractFunctionCode:
optimized = 1 optimized = 1
lambdaCount = 0 lambdaCount = 0
def __init__(self, func, filename, scopes, isLambda, class_name, mod): def __init__(self, func, scopes, isLambda, class_name, mod):
self.class_name = class_name self.class_name = class_name
self.module = mod self.module = mod
if isLambda: if isLambda:
...@@ -1108,10 +1111,10 @@ class AbstractFunctionCode: ...@@ -1108,10 +1111,10 @@ class AbstractFunctionCode:
else: else:
name = func.name name = func.name
args, hasTupleArg = generateArgList(func.argnames) args, hasTupleArg = generateArgList(func.argnames)
self.graph = pyassem.PyFlowGraph(name, filename, args, self.graph = pyassem.PyFlowGraph(name, func.filename, args,
optimized=1) optimized=1)
self.isLambda = isLambda self.isLambda = isLambda
self.super_init(filename) self.super_init()
if not isLambda and func.doc: if not isLambda and func.doc:
self.setDocstring(func.doc) self.setDocstring(func.doc)
...@@ -1162,10 +1165,10 @@ class FunctionCodeGenerator(NestedScopeMixin, AbstractFunctionCode, ...@@ -1162,10 +1165,10 @@ class FunctionCodeGenerator(NestedScopeMixin, AbstractFunctionCode,
__super_init = AbstractFunctionCode.__init__ __super_init = AbstractFunctionCode.__init__
def __init__(self, func, filename, scopes, isLambda, class_name, mod): def __init__(self, func, scopes, isLambda, class_name, mod):
self.scopes = scopes self.scopes = scopes
self.scope = scopes[func] self.scope = scopes[func]
self.__super_init(func, filename, scopes, isLambda, class_name, mod) self.__super_init(func, scopes, isLambda, class_name, mod)
self.graph.setFreeVars(self.scope.get_free_vars()) self.graph.setFreeVars(self.scope.get_free_vars())
self.graph.setCellVars(self.scope.get_cell_vars()) self.graph.setCellVars(self.scope.get_cell_vars())
if self.graph.checkFlag(CO_GENERATOR_ALLOWED): if self.graph.checkFlag(CO_GENERATOR_ALLOWED):
...@@ -1174,12 +1177,12 @@ class FunctionCodeGenerator(NestedScopeMixin, AbstractFunctionCode, ...@@ -1174,12 +1177,12 @@ class FunctionCodeGenerator(NestedScopeMixin, AbstractFunctionCode,
class AbstractClassCode: class AbstractClassCode:
def __init__(self, klass, filename, scopes, module): def __init__(self, klass, scopes, module):
self.class_name = klass.name self.class_name = klass.name
self.module = module self.module = module
self.graph = pyassem.PyFlowGraph(klass.name, filename, self.graph = pyassem.PyFlowGraph(klass.name, klass.filename,
optimized=0, klass=1) optimized=0, klass=1)
self.super_init(filename) self.super_init()
lnf = walk(klass.code, self.NameFinder(), verbose=0) lnf = walk(klass.code, self.NameFinder(), verbose=0)
self.locals.push(lnf.getLocals()) self.locals.push(lnf.getLocals())
self.graph.setFlag(CO_NEWLOCALS) self.graph.setFlag(CO_NEWLOCALS)
...@@ -1200,10 +1203,10 @@ class ClassCodeGenerator(NestedScopeMixin, AbstractClassCode, CodeGenerator): ...@@ -1200,10 +1203,10 @@ class ClassCodeGenerator(NestedScopeMixin, AbstractClassCode, CodeGenerator):
__super_init = AbstractClassCode.__init__ __super_init = AbstractClassCode.__init__
def __init__(self, klass, filename, scopes, module): def __init__(self, klass, scopes, module):
self.scopes = scopes self.scopes = scopes
self.scope = scopes[klass] self.scope = scopes[klass]
self.__super_init(klass, filename, scopes, module) self.__super_init(klass, scopes, module)
self.graph.setFreeVars(self.scope.get_free_vars()) self.graph.setFreeVars(self.scope.get_free_vars())
self.graph.setCellVars(self.scope.get_cell_vars()) self.graph.setCellVars(self.scope.get_cell_vars())
## self.graph.setFlag(CO_NESTED) ## self.graph.setFlag(CO_NESTED)
......
"""Check for errs in the AST.
The Python parser does not catch all syntax errors. Others, like
assignments with invalid targets, are caught in the code generation
phase.
The compiler package catches some errors in the transformer module.
But it seems clearer to write checkers that use the AST to detect
errors.
"""
from compiler import ast, walk
def check(tree, multi=None):
v = SyntaxErrorChecker(multi)
walk(tree, v)
return v.errors
class SyntaxErrorChecker:
"""A visitor to find syntax errors in the AST."""
def __init__(self, multi=None):
"""Create new visitor object.
If optional argument multi is not None, then print messages
for each error rather than raising a SyntaxError for the
first.
"""
self.multi = multi
self.errors = 0
def error(self, node, msg):
self.errors = self.errors + 1
if self.multi is not None:
print "%s:%s: %s" % (node.filename, node.lineno, msg)
else:
raise SyntaxError, "%s (%s:%s)" % (msg, node.filename, node.lineno)
def visitAssign(self, node):
# the transformer module handles many of these
for target in node.nodes:
if isinstance(target, ast.AssList):
if target.lineno is None:
target.lineno = node.lineno
self.error(target, "can't assign to list comprehension")
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment