Skip to content

C
cpython
Commit 9c048f9f authored by Jeremy Hylton's avatar Jeremy Hylton
Browse files
Options

Now supports entire Python 2.0 language and still supports Python 

1.5.2.  The compiler generates code for the version of the interpreter
it is run under.

ast.py:
    Print and Printnl add dest attr for extended print
    new node AugAssign for augmented assignments
    new nodes ListComp, ListCompFor, and ListCompIf for list
        comprehensions

pyassem.py:
    add work around for string-Unicode comparison raising UnicodeError
        on comparison of two objects in code object's const table

pycodegen.py:
    define VERSION, the Python major version number
    get magic number using imp.get_magic() instead of hard coding
    implement list comprehensions, extended print, and augmented
        assignment; augmented assignment uses Delegator classes (see
        doc string)
    fix import and tuple unpacking for 1.5.2

transformer.py:
    various changes to support new 2.0 grammar and old 1.5 grammar
    add debug_tree helper than converts and symbol and token numbers
    to their names
parent 5bad5a4b
Show whitespace changes
Inline Side-by-side
Showing with 926 additions and 186 deletions
Lib/compiler/ast.py
View file @ 9c048f9f
...@@ -279,22 +279,24 @@ class Const(Node): ...@@ -279,22 +279,24 @@ class Const(Node):
class Print(Node): class Print(Node):
nodes['print'] = 'Print' nodes['print'] = 'Print'
def __init__(self, nodes): def __init__(self, nodes, dest):
self.nodes = nodes self.nodes = nodes
self._children = ('print', nodes) self.dest = dest
self._children = ('print', nodes, dest)
def __repr__(self): def __repr__(self):
return "Print(%s)" % self._children[1:] return "Print(%s, %s)" % (self._children[1:-1], self._children[-1])
class Printnl(Node): class Printnl(Node):
nodes['printnl'] = 'Printnl' nodes['printnl'] = 'Printnl'
def __init__(self, nodes): def __init__(self, nodes, dest):
self.nodes = nodes self.nodes = nodes
self._children = ('printnl', nodes) self.dest = dest
self._children = ('printnl', nodes, dest)
def __repr__(self): def __repr__(self):
return "Printnl(%s)" % self._children[1:] return "Printnl(%s, %s)" % (self._children[1:-1], self._children[-1])
class Discard(Node): class Discard(Node):
nodes['discard'] = 'Discard' nodes['discard'] = 'Discard'
...@@ -306,6 +308,18 @@ class Discard(Node): ...@@ -306,6 +308,18 @@ class Discard(Node):
def __repr__(self): def __repr__(self):
return "Discard(%s)" % self._children[1:] return "Discard(%s)" % self._children[1:]
class AugAssign(Node):
nodes['augassign'] = 'AugAssign'
def __init__(self, node, op, expr):
self.node = node
self.op = op
self.expr = expr
self._children = ('augassign', node, op, expr)
def __repr__(self):
return "AugAssign(%s)" % str(self._children[1:])
class Assign(Node): class Assign(Node):
nodes['assign'] = 'Assign' nodes['assign'] = 'Assign'
...@@ -360,6 +374,41 @@ class AssAttr(Node): ...@@ -360,6 +374,41 @@ class AssAttr(Node):
def __repr__(self): def __repr__(self):
return "AssAttr(%s,%s,%s)" % self._children[1:] return "AssAttr(%s,%s,%s)" % self._children[1:]
class ListComp(Node):
nodes['listcomp'] = 'ListComp'
def __init__(self, expr, quals):
self.expr = expr
self.quals = quals
self._children = ('listcomp', expr, quals)
def __repr__(self):
return "ListComp(%s, %s)" % self._children[1:]
class ListCompFor(Node):
nodes['listcomp_for'] = 'ListCompFor'
# transformer fills in ifs after node is created
def __init__(self, assign, list, ifs):
self.assign = assign
self.list = list
self.ifs = ifs
self._children = ('listcomp_for', assign, list, ifs)
def __repr__(self):
return "ListCompFor(%s, %s, %s)" % self._children[1:]
class ListCompIf(Node):
nodes['listcomp_if'] = 'ListCompIf'
def __init__(self, test):
self.test = test
self._children = ('listcomp_if', test)
def __repr__(self):
return "ListCompIf(%s)" % self._children[1:]
class List(Node): class List(Node):
nodes['list'] = 'List' nodes['list'] = 'List'
......
Lib/compiler/pyassem.py
View file @ 9c048f9f
...@@ -253,8 +253,14 @@ class PyFlowGraph(FlowGraph): ...@@ -253,8 +253,14 @@ class PyFlowGraph(FlowGraph):
def _lookupName(self, name, list): def _lookupName(self, name, list):
"""Return index of name in list, appending if necessary""" """Return index of name in list, appending if necessary"""
if name in list: found = None
i = list.index(name) t = type(name)
for i in range(len(list)):
# must do a comparison on type first to prevent UnicodeErrors
if t == type(list[i]) and list[i] == name:
found = 1
break
if found:
# this is cheap, but incorrect in some cases, e.g 2 vs. 2L # this is cheap, but incorrect in some cases, e.g 2 vs. 2L
if type(name) == type(list[i]): if type(name) == type(list[i]):
return i return i
......
Lib/compiler/pycodegen.py
View file @ 9c048f9f
import imp
import os import os
import marshal import marshal
import stat import stat
import string import string
import struct import struct
import sys
import types import types
from cStringIO import StringIO from cStringIO import StringIO
...@@ -10,6 +12,12 @@ from compiler import ast, parse, walk ...@@ -10,6 +12,12 @@ from compiler import ast, parse, walk
from compiler import pyassem, misc from compiler import pyassem, misc
from compiler.pyassem import CO_VARARGS, CO_VARKEYWORDS, CO_NEWLOCALS, TupleArg from compiler.pyassem import CO_VARARGS, CO_VARKEYWORDS, CO_NEWLOCALS, TupleArg
# Do we have Python 1.x or Python 2.x?
try:
VERSION = sys.version_info[0]
except AttributeError:
VERSION = 1
callfunc_opcode_info = { callfunc_opcode_info = {
# (Have *args, Have **args) : opcode # (Have *args, Have **args) : opcode
(0,0) : "CALL_FUNCTION", (0,0) : "CALL_FUNCTION",
...@@ -18,12 +26,12 @@ callfunc_opcode_info = { ...@@ -18,12 +26,12 @@ callfunc_opcode_info = {
(1,1) : "CALL_FUNCTION_VAR_KW", (1,1) : "CALL_FUNCTION_VAR_KW",
} }
def compile(filename): def compile(filename, display=0):
f = open(filename) f = open(filename)
buf = f.read() buf = f.read()
f.close() f.close()
mod = Module(buf, filename) mod = Module(buf, filename)
mod.compile() mod.compile(display)
f = open(filename + "c", "wb") f = open(filename + "c", "wb")
mod.dump(f) mod.dump(f)
f.close() f.close()
...@@ -34,28 +42,30 @@ class Module: ...@@ -34,28 +42,30 @@ class Module:
self.source = source self.source = source
self.code = None self.code = None
def compile(self): def compile(self, display=0):
ast = parse(self.source) ast = parse(self.source)
root, filename = os.path.split(self.filename) root, filename = os.path.split(self.filename)
gen = ModuleCodeGenerator(filename) gen = ModuleCodeGenerator(filename)
walk(ast, gen, 1) walk(ast, gen, 1)
if display:
import pprint
print pprint.pprint(ast)
self.code = gen.getCode() self.code = gen.getCode()
def dump(self, f): def dump(self, f):
f.write(self.getPycHeader()) f.write(self.getPycHeader())
marshal.dump(self.code, f) marshal.dump(self.code, f)
MAGIC = (50823 | (ord('\r')<<16) | (ord('\n')<<24)) MAGIC = imp.get_magic()
def getPycHeader(self): def getPycHeader(self):
# compile.c uses marshal to write a long directly, with # compile.c uses marshal to write a long directly, with
# calling the interface that would also generate a 1-byte code # calling the interface that would also generate a 1-byte code
# to indicate the type of the value. simplest way to get the # to indicate the type of the value. simplest way to get the
# same effect is to call marshal and then skip the code. # same effect is to call marshal and then skip the code.
magic = marshal.dumps(self.MAGIC)[1:]
mtime = os.stat(self.filename)[stat.ST_MTIME] mtime = os.stat(self.filename)[stat.ST_MTIME]
mtime = struct.pack('i', mtime) mtime = struct.pack('i', mtime)
return magic + mtime return self.MAGIC + mtime
class CodeGenerator: class CodeGenerator:
...@@ -63,7 +73,7 @@ class CodeGenerator: ...@@ -63,7 +73,7 @@ class CodeGenerator:
def __init__(self, filename): def __init__(self, filename):
## Subclasses must define a constructor that intializes self.graph ## Subclasses must define a constructor that intializes self.graph
## before calling this init function ## before calling this init function, e.g.
## self.graph = pyassem.PyFlowGraph() ## self.graph = pyassem.PyFlowGraph()
self.filename = filename self.filename = filename
self.locals = misc.Stack() self.locals = misc.Stack()
...@@ -142,7 +152,6 @@ class CodeGenerator: ...@@ -142,7 +152,6 @@ class CodeGenerator:
def visitLambda(self, node): def visitLambda(self, node):
self._visitFuncOrLambda(node, isLambda=1) self._visitFuncOrLambda(node, isLambda=1)
## self.storeName("<lambda>")
def _visitFuncOrLambda(self, node, isLambda): def _visitFuncOrLambda(self, node, isLambda):
gen = FunctionCodeGenerator(node, self.filename, isLambda) gen = FunctionCodeGenerator(node, self.filename, isLambda)
...@@ -180,10 +189,6 @@ class CodeGenerator: ...@@ -180,10 +189,6 @@ class CodeGenerator:
test, suite = node.tests[i] test, suite = node.tests[i]
self.set_lineno(test) self.set_lineno(test)
self.visit(test) self.visit(test)
## if i == numtests - 1 and not node.else_:
## nextTest = end
## else:
## nextTest = self.newBlock()
nextTest = self.newBlock() nextTest = self.newBlock()
self.emit('JUMP_IF_FALSE', nextTest) self.emit('JUMP_IF_FALSE', nextTest)
self.nextBlock() self.nextBlock()
...@@ -304,6 +309,70 @@ class CodeGenerator: ...@@ -304,6 +309,70 @@ class CodeGenerator:
self.emit('POP_TOP') self.emit('POP_TOP')
self.nextBlock(end) self.nextBlock(end)
# list comprehensions
__list_count = 0
def visitListComp(self, node):
# XXX would it be easier to transform the AST into the form it
# would have if the list comp were expressed as a series of
# for and if stmts and an explicit append?
self.set_lineno(node)
# setup list
append = "$append%d" % self.__list_count
self.__list_count = self.__list_count + 1
self.emit('BUILD_LIST', 0)
self.emit('DUP_TOP')
self.emit('LOAD_ATTR', 'append')
self.storeName(append)
l = len(node.quals)
stack = []
for i, for_ in zip(range(l), node.quals):
start, anchor = self.visit(for_)
cont = None
for if_ in for_.ifs:
if cont is None:
cont = self.newBlock()
self.visit(if_, cont)
stack.insert(0, (start, cont, anchor))
self.loadName(append)
self.visit(node.expr)
self.emit('CALL_FUNCTION', 1)
self.emit('POP_TOP')
for start, cont, anchor in stack:
if cont:
skip_one = self.newBlock()
self.emit('JUMP_FORWARD', skip_one)
self.nextBlock(cont)
self.emit('POP_TOP')
self.nextBlock(skip_one)
self.emit('JUMP_ABSOLUTE', start)
self.nextBlock(anchor)
self.delName(append)
self.__list_count = self.__list_count - 1
def visitListCompFor(self, node):
self.set_lineno(node)
start = self.newBlock()
anchor = self.newBlock()
self.visit(node.list)
self.visit(ast.Const(0))
self.emit('SET_LINENO', node.lineno)
self.nextBlock(start)
self.emit('FOR_LOOP', anchor)
self.visit(node.assign)
return start, anchor
def visitListCompIf(self, node, branch):
self.set_lineno(node)
self.visit(node.test)
self.emit('JUMP_IF_FALSE', branch)
self.newBlock()
self.emit('POP_TOP')
# exception related # exception related
def visitAssert(self, node): def visitAssert(self, node):
...@@ -397,10 +466,6 @@ class CodeGenerator: ...@@ -397,10 +466,6 @@ class CodeGenerator:
# misc # misc
## def visitStmt(self, node):
## # nothing to do except walk the children
## pass
def visitDiscard(self, node): def visitDiscard(self, node):
self.visit(node.expr) self.visit(node.expr)
self.emit('POP_TOP') self.emit('POP_TOP')
...@@ -426,17 +491,20 @@ class CodeGenerator: ...@@ -426,17 +491,20 @@ class CodeGenerator:
def visitImport(self, node): def visitImport(self, node):
self.set_lineno(node) self.set_lineno(node)
for name, alias in node.names: for name, alias in node.names:
if VERSION > 1:
self.emit('LOAD_CONST', None) self.emit('LOAD_CONST', None)
self.emit('IMPORT_NAME', name) self.emit('IMPORT_NAME', name)
self._resolveDots(name) mod = string.split(name, ".")[0]
self.storeName(alias or name) self.storeName(alias or mod)
def visitFrom(self, node): def visitFrom(self, node):
self.set_lineno(node) self.set_lineno(node)
fromlist = map(lambda (name, alias): name, node.names) fromlist = map(lambda (name, alias): name, node.names)
if VERSION > 1:
self.emit('LOAD_CONST', tuple(fromlist)) self.emit('LOAD_CONST', tuple(fromlist))
self.emit('IMPORT_NAME', node.modname) self.emit('IMPORT_NAME', node.modname)
for name, alias in node.names: for name, alias in node.names:
if VERSION > 1:
if name == '*': if name == '*':
self.namespace = 0 self.namespace = 0
self.emit('IMPORT_STAR') self.emit('IMPORT_STAR')
...@@ -447,6 +515,8 @@ class CodeGenerator: ...@@ -447,6 +515,8 @@ class CodeGenerator:
self.emit('IMPORT_FROM', name) self.emit('IMPORT_FROM', name)
self._resolveDots(name) self._resolveDots(name)
self.storeName(alias or name) self.storeName(alias or name)
else:
self.emit('IMPORT_FROM', name)
self.emit('POP_TOP') self.emit('POP_TOP')
def _resolveDots(self, name): def _resolveDots(self, name):
...@@ -491,13 +561,85 @@ class CodeGenerator: ...@@ -491,13 +561,85 @@ class CodeGenerator:
print "warning: unexpected flags:", node.flags print "warning: unexpected flags:", node.flags
print node print node
def visitAssTuple(self, node): def _visitAssSequence(self, node, op='UNPACK_SEQUENCE'):
if findOp(node) != 'OP_DELETE': if findOp(node) != 'OP_DELETE':
self.emit('UNPACK_SEQUENCE', len(node.nodes)) self.emit(op, len(node.nodes))
for child in node.nodes: for child in node.nodes:
self.visit(child) self.visit(child)
visitAssList = visitAssTuple if VERSION > 1:
visitAssTuple = _visitAssSequence
visitAssList = _visitAssSequence
else:
def visitAssTuple(self, node):
self._visitAssSequence(node, 'UNPACK_TUPLE')
def visitAssList(self, node):
self._visitAssSequence(node, 'UNPACK_LIST')
# augmented assignment
def visitAugAssign(self, node):
aug_node = wrap_aug(node.node)
self.visit(aug_node, "load")
self.visit(node.expr)
self.emit(self._augmented_opcode[node.op])
self.visit(aug_node, "store")
_augmented_opcode = {
'+=' : 'INPLACE_ADD',
'-=' : 'INPLACE_SUBTRACT',
'*=' : 'INPLACE_MULTIPLY',
'/=' : 'INPLACE_DIVIDE',
'%=' : 'INPLACE_MODULO',
'**=': 'INPLACE_POWER',
'>>=': 'INPLACE_RSHIFT',
'<<=': 'INPLACE_LSHIFT',
'&=' : 'INPLACE_AND',
'^=' : 'INPLACE_XOR',
'|=' : 'INPLACE_OR',
}
def visitAugName(self, node, mode):
if mode == "load":
self.loadName(node.name)
elif mode == "store":
self.storeName(node.name)
def visitAugGetattr(self, node, mode):
if mode == "load":
self.visit(node.expr)
self.emit('DUP_TOP')
self.emit('LOAD_ATTR', node.attrname)
elif mode == "store":
self.emit('ROT_TWO')
self.emit('STORE_ATTR', node.attrname)
def visitAugSlice(self, node, mode):
if mode == "load":
self.visitSlice(node, 1)
elif mode == "store":
slice = 0
if node.lower:
slice = slice | 1
if node.upper:
slice = slice | 2
if slice == 0:
self.emit('ROT_TWO')
elif slice == 3:
self.emit('ROT_FOUR')
else:
self.emit('ROT_THREE')
self.emit('STORE_SLICE+%d' % slice)
def visitAugSubscript(self, node, mode):
if len(node.subs) > 1:
raise SyntaxError, "augmented assignment to tuple is not possible"
if mode == "load":
self.visitSubscript(node, 1)
elif mode == "store":
self.emit('ROT_THREE')
self.emit('STORE_SUBSCR')
def visitExec(self, node): def visitExec(self, node):
self.visit(node.expr) self.visit(node.expr)
...@@ -533,12 +675,23 @@ class CodeGenerator: ...@@ -533,12 +675,23 @@ class CodeGenerator:
def visitPrint(self, node): def visitPrint(self, node):
self.set_lineno(node) self.set_lineno(node)
if node.dest:
self.visit(node.dest)
for child in node.nodes: for child in node.nodes:
if node.dest:
self.emit('DUP_TOP')
self.visit(child) self.visit(child)
if node.dest:
self.emit('ROT_TWO')
self.emit('PRINT_ITEM_TO')
else:
self.emit('PRINT_ITEM') self.emit('PRINT_ITEM')
def visitPrintnl(self, node): def visitPrintnl(self, node):
self.visitPrint(node) self.visitPrint(node)
if node.dest:
self.emit('PRINT_NEWLINE_TO')
else:
self.emit('PRINT_NEWLINE') self.emit('PRINT_NEWLINE')
def visitReturn(self, node): def visitReturn(self, node):
...@@ -548,7 +701,8 @@ class CodeGenerator: ...@@ -548,7 +701,8 @@ class CodeGenerator:
# slice and subscript stuff # slice and subscript stuff
def visitSlice(self, node): def visitSlice(self, node, aug_flag=None):
# aug_flag is used by visitAugSlice
self.visit(node.expr) self.visit(node.expr)
slice = 0 slice = 0
if node.lower: if node.lower:
...@@ -557,6 +711,13 @@ class CodeGenerator: ...@@ -557,6 +711,13 @@ class CodeGenerator:
if node.upper: if node.upper:
self.visit(node.upper) self.visit(node.upper)
slice = slice | 2 slice = slice | 2
if aug_flag:
if slice == 0:
self.emit('DUP_TOP')
elif slice == 3:
self.emit('DUP_TOPX', 3)
else:
self.emit('DUP_TOPX', 2)
if node.flags == 'OP_APPLY': if node.flags == 'OP_APPLY':
self.emit('SLICE+%d' % slice) self.emit('SLICE+%d' % slice)
elif node.flags == 'OP_ASSIGN': elif node.flags == 'OP_ASSIGN':
...@@ -567,10 +728,12 @@ class CodeGenerator: ...@@ -567,10 +728,12 @@ class CodeGenerator:
print "weird slice", node.flags print "weird slice", node.flags
raise raise
def visitSubscript(self, node): def visitSubscript(self, node, aug_flag=None):
self.visit(node.expr) self.visit(node.expr)
for sub in node.subs: for sub in node.subs:
self.visit(sub) self.visit(sub)
if aug_flag:
self.emit('DUP_TOPX', 2)
if len(node.subs) > 1: if len(node.subs) > 1:
self.emit('BUILD_TUPLE', len(node.subs)) self.emit('BUILD_TUPLE', len(node.subs))
if node.flags == 'OP_APPLY': if node.flags == 'OP_APPLY':
...@@ -740,7 +903,10 @@ class FunctionCodeGenerator(CodeGenerator): ...@@ -740,7 +903,10 @@ class FunctionCodeGenerator(CodeGenerator):
self.unpackSequence(arg) self.unpackSequence(arg)
def unpackSequence(self, tup): def unpackSequence(self, tup):
if VERSION > 1:
self.emit('UNPACK_SEQUENCE', len(tup)) self.emit('UNPACK_SEQUENCE', len(tup))
else:
self.emit('UNPACK_TUPLE', len(tup))
for elt in tup: for elt in tup:
if type(elt) == types.TupleType: if type(elt) == types.TupleType:
self.unpackSequence(elt) self.unpackSequence(elt)
...@@ -765,7 +931,6 @@ class ClassCodeGenerator(CodeGenerator): ...@@ -765,7 +931,6 @@ class ClassCodeGenerator(CodeGenerator):
self.emit('LOAD_LOCALS') self.emit('LOAD_LOCALS')
self.emit('RETURN_VALUE') self.emit('RETURN_VALUE')
def generateArgList(arglist): def generateArgList(arglist):
"""Generate an arg list marking TupleArgs""" """Generate an arg list marking TupleArgs"""
args = [] args = []
...@@ -838,6 +1003,45 @@ class OpFinder: ...@@ -838,6 +1003,45 @@ class OpFinder:
elif self.op != node.flags: elif self.op != node.flags:
raise ValueError, "mixed ops in stmt" raise ValueError, "mixed ops in stmt"
class Delegator:
"""Base class to support delegation for augmented assignment nodes
To generator code for augmented assignments, we use the following
wrapper classes. In visitAugAssign, the left-hand expression node
is visited twice. The first time the visit uses the normal method
for that node . The second time the visit uses a different method
that generates the appropriate code to perform the assignment.
These delegator classes wrap the original AST nodes in order to
support the variant visit methods.
"""
def __init__(self, obj):
self.obj = obj
def __getattr__(self, attr):
return getattr(self.obj, attr)
class AugGetattr(Delegator):
pass
class AugName(Delegator):
pass
class AugSlice(Delegator):
pass
class AugSubscript(Delegator):
pass
wrapper = {
ast.Getattr: AugGetattr,
ast.Name: AugName,
ast.Slice: AugSlice,
ast.Subscript: AugSubscript,
}
def wrap_aug(node):
return wrapper[node.__class__](node)
if __name__ == "__main__": if __name__ == "__main__":
import sys import sys
......
Lib/compiler/transformer.py
View file @ 9c048f9f
#
# Copyright (C) 1997-1998 Greg Stein. All Rights Reserved.
#
# This module is provided under a BSD-ish license. See
# http://www.opensource.org/licenses/bsd-license.html
# and replace OWNER, ORGANIZATION, and YEAR as appropriate.
#
#
# Written by Greg Stein (gstein@lyra.org)
# and Bill Tutt (rassilon@lima.mudlib.org)
# February 1997.
#
# Support for ast.Node subclasses written and other revisions by
# Jeremy Hylton (jeremy@beopen.com)
#
"""Parse tree transformation module. """Parse tree transformation module.
Transforms Python source code into an abstract syntax tree (AST) Transforms Python source code into an abstract syntax tree (AST)
...@@ -24,7 +8,21 @@ parse(buf) -> AST ...@@ -24,7 +8,21 @@ parse(buf) -> AST
parseFile(path) -> AST parseFile(path) -> AST
""" """
# Original version written by Greg Stein (gstein@lyra.org)
# and Bill Tutt (rassilon@lima.mudlib.org)
# February 1997.
#
# Modifications and improvements for Python 2.0 by Jeremy Hylton and
# Mark Hammond
# Portions of this file are:
# Copyright (C) 1997-1998 Greg Stein. All Rights Reserved.
# #
# This module is provided under a BSD-ish license. See
# http://www.opensource.org/licenses/bsd-license.html
# and replace OWNER, ORGANIZATION, and YEAR as appropriate.
# The output tree has the following nodes: # The output tree has the following nodes:
# #
# Source Python line #'s appear at the end of each of all of these nodes # Source Python line #'s appear at the end of each of all of these nodes
...@@ -49,9 +47,10 @@ parseFile(path) -> AST ...@@ -49,9 +47,10 @@ parseFile(path) -> AST
# tryexcept: trySuiteNode, [ (exprNode, assgnNode, suiteNode), ... ], elseNode # tryexcept: trySuiteNode, [ (exprNode, assgnNode, suiteNode), ... ], elseNode
# return: valueNode # return: valueNode
# const: value # const: value
# print: [ node1, ..., nodeN ] # print: [ node1, ..., nodeN ] [, dest]
# printnl: [ node1, ..., nodeN ] # printnl: [ node1, ..., nodeN ] [, dest]
# discard: exprNode # discard: exprNode
# augassign: node, op, expr
# assign: [ node1, ..., nodeN ], exprNode # assign: [ node1, ..., nodeN ], exprNode
# ass_tuple: [ node1, ..., nodeN ] # ass_tuple: [ node1, ..., nodeN ]
# ass_list: [ node1, ..., nodeN ] # ass_list: [ node1, ..., nodeN ]
...@@ -97,12 +96,12 @@ parseFile(path) -> AST ...@@ -97,12 +96,12 @@ parseFile(path) -> AST
import ast import ast
import parser import parser
# Care must be taken to use only symbols and tokens defined in Python
# 1.5.2 for code branches executed in 1.5.2
import symbol import symbol
import token import token
import string import string
import pprint
error = 'walker.error' error = 'walker.error'
from consts import CO_VARARGS, CO_VARKEYWORDS from consts import CO_VARARGS, CO_VARKEYWORDS
...@@ -328,27 +327,44 @@ class Transformer: ...@@ -328,27 +327,44 @@ class Transformer:
# #
def expr_stmt(self, nodelist): def expr_stmt(self, nodelist):
# testlist ('=' testlist)* # augassign testlist | testlist ('=' testlist)*
exprNode = self.com_node(nodelist[-1]) exprNode = self.com_node(nodelist[-1])
if len(nodelist) == 1: if len(nodelist) == 1:
return Node('discard', exprNode) return Node('discard', exprNode)
if nodelist[1][0] == token.EQUAL:
nodes = [ ] nodes = [ ]
for i in range(0, len(nodelist) - 2, 2): for i in range(0, len(nodelist) - 2, 2):
nodes.append(self.com_assign(nodelist[i], OP_ASSIGN)) nodes.append(self.com_assign(nodelist[i], OP_ASSIGN))
n = Node('assign', nodes, exprNode) n = Node('assign', nodes, exprNode)
n.lineno = nodelist[1][2] n.lineno = nodelist[1][2]
else:
lval = self.com_augassign(nodelist[0])
op = self.com_augassign_op(nodelist[1])
n = Node('augassign', lval, op[1], exprNode)
n.lineno = op[2]
return n return n
def print_stmt(self, nodelist): def print_stmt(self, nodelist):
# print: (test ',')* [test] # print ([ test (',' test)* [','] ] | '>>' test [ (',' test)+ [','] ])
items = [ ] items = [ ]
for i in range(1, len(nodelist), 2): if len(nodelist) == 1:
start = 1
dest = None
elif nodelist[1][0] == token.RIGHTSHIFT:
assert len(nodelist) == 3 \
or nodelist[3][0] == token.COMMA
dest = self.com_node(nodelist[2])
start = 4
else:
dest = None
start = 1
for i in range(start, len(nodelist), 2):
items.append(self.com_node(nodelist[i])) items.append(self.com_node(nodelist[i]))
if nodelist[-1][0] == token.COMMA: if nodelist[-1][0] == token.COMMA:
n = Node('print', items) n = Node('print', items, dest)
n.lineno = nodelist[0][2] n.lineno = nodelist[0][2]
return n return n
n = Node('printnl', items) n = Node('printnl', items, dest)
n.lineno = nodelist[0][2] n.lineno = nodelist[0][2]
return n return n
...@@ -405,15 +421,22 @@ class Transformer: ...@@ -405,15 +421,22 @@ class Transformer:
# import_stmt: 'import' dotted_as_name (',' dotted_as_name)* | # import_stmt: 'import' dotted_as_name (',' dotted_as_name)* |
# from: 'from' dotted_name 'import' # from: 'from' dotted_name 'import'
# ('*' | import_as_name (',' import_as_name)*) # ('*' | import_as_name (',' import_as_name)*)
names = []
is_as = 0
if nodelist[0][1] == 'from': if nodelist[0][1] == 'from':
names = []
if nodelist[3][0] == token.NAME:
for i in range(3, len(nodelist), 2):
names.append((nodelist[i][1], None))
else:
for i in range(3, len(nodelist), 2): for i in range(3, len(nodelist), 2):
names.append(self.com_import_as_name(nodelist[i][1])) names.append(self.com_import_as_name(nodelist[i][1]))
n = Node('from', self.com_dotted_name(nodelist[1]), names) n = Node('from', self.com_dotted_name(nodelist[1]), names)
n.lineno = nodelist[0][2] n.lineno = nodelist[0][2]
return n return n
if nodelist[1][0] == symbol.dotted_name:
names = [(self.com_dotted_name(nodelist[1][1:]), None)]
else:
names = []
for i in range(1, len(nodelist), 2): for i in range(1, len(nodelist), 2):
names.append(self.com_dotted_as_name(nodelist[i])) names.append(self.com_dotted_as_name(nodelist[i]))
n = Node('import', names) n = Node('import', names)
...@@ -737,7 +760,7 @@ class Transformer: ...@@ -737,7 +760,7 @@ class Transformer:
return Node('discard', Node('const', None)) return Node('discard', Node('const', None))
if node[0] not in _legal_node_types: if node[0] not in _legal_node_types:
raise error, 'illegal node passed to com_node: %s' % node[0] raise error, 'illegal node passed to com_node: %s' % `node`
# print "dispatch", self._dispatch[node[0]].__name__, node # print "dispatch", self._dispatch[node[0]].__name__, node
return self._dispatch[node[0]](node[1:]) return self._dispatch[node[0]](node[1:])
...@@ -818,8 +841,11 @@ class Transformer: ...@@ -818,8 +841,11 @@ class Transformer:
def com_dotted_as_name(self, node): def com_dotted_as_name(self, node):
dot = self.com_dotted_name(node[1]) dot = self.com_dotted_name(node[1])
if len(node) == 2: if len(node) <= 2:
return dot, None return dot, None
if node[0] == symbol.dotted_name:
pass
else:
assert node[2][1] == 'as' assert node[2][1] == 'as'
assert node[3][0] == token.NAME assert node[3][0] == token.NAME
return dot, node[3][1] return dot, node[3][1]
...@@ -872,6 +898,20 @@ class Transformer: ...@@ -872,6 +898,20 @@ class Transformer:
n.lineno = nodelist[0][2] n.lineno = nodelist[0][2]
return n return n
def com_augassign_op(self, node):
assert node[0] == symbol.augassign
return node[1]
def com_augassign(self, node):
"""Return node suitable for lvalue of augmented assignment
Names, slices, and attributes are the only allowable nodes.
"""
l = self.com_node(node)
if l[0] in ('name', 'slice', 'subscript', 'getattr'):
return l
raise SyntaxError, "can't assign to %s" % l[0]
def com_assign(self, node, assigning): def com_assign(self, node, assigning):
# return a node suitable for use as an "lvalue" # return a node suitable for use as an "lvalue"
# loop to avoid trivial recursion # loop to avoid trivial recursion
...@@ -955,7 +995,6 @@ class Transformer: ...@@ -955,7 +995,6 @@ class Transformer:
return Node(type, items) return Node(type, items)
def com_stmt(self, node): def com_stmt(self, node):
#pprint.pprint(node)
result = self.com_node(node) result = self.com_node(node)
try: try:
result[0] result[0]
...@@ -976,6 +1015,59 @@ class Transformer: ...@@ -976,6 +1015,59 @@ class Transformer:
else: else:
stmts.append(result) stmts.append(result)
if hasattr(symbol, 'list_for'):
def com_list_constructor(self, nodelist):
# listmaker: test ( list_for | (',' test)* [','] )
values = [ ]
for i in range(1, len(nodelist)):
if nodelist[i][0] == symbol.list_for:
assert len(nodelist[i:]) == 1
return self.com_list_comprehension(values[0],
nodelist[i])
elif nodelist[i][0] == token.COMMA:
continue
values.append(self.com_node(nodelist[i]))
return Node('list', values)
def com_list_comprehension(self, expr, node):
# list_iter: list_for | list_if
# list_for: 'for' exprlist 'in' testlist [list_iter]
# list_if: 'if' test [list_iter]
lineno = node[1][2]
fors = []
while node:
if node[1][1] == 'for':
assignNode = self.com_assign(node[2], OP_ASSIGN)
listNode = self.com_node(node[4])
newfor = Node('listcomp_for', assignNode,
listNode, [])
newfor.lineno = node[1][2]
fors.append(newfor)
if len(node) == 5:
node = None
else:
node = self.com_list_iter(node[5])
elif node[1][1] == 'if':
test = self.com_node(node[2])
newif = Node('listcomp_if', test)
newif.lineno = node[1][2]
newfor.ifs.append(newif)
if len(node) == 3:
node = None
else:
node = self.com_list_iter(node[3])
else:
raise SyntaxError, \
("unexpected list comprehension element: %s %d"
% (node, lineno))
n = Node('listcomp', expr, fors)
n.lineno = lineno
return n
def com_list_iter(self, node):
assert node[0] == symbol.list_iter
return node[1]
else:
def com_list_constructor(self, nodelist): def com_list_constructor(self, nodelist):
values = [ ] values = [ ]
for i in range(1, len(nodelist), 2): for i in range(1, len(nodelist), 2):
...@@ -986,7 +1078,8 @@ class Transformer: ...@@ -986,7 +1078,8 @@ class Transformer:
# dictmaker: test ':' test (',' test ':' value)* [','] # dictmaker: test ':' test (',' test ':' value)* [',']
items = [ ] items = [ ]
for i in range(1, len(nodelist), 4): for i in range(1, len(nodelist), 4):
items.append((self.com_node(nodelist[i]), self.com_node(nodelist[i+2]))) items.append((self.com_node(nodelist[i]),
self.com_node(nodelist[i+2])))
return Node('dict', items) return Node('dict', items)
def com_apply_trailer(self, primaryNode, nodelist): def com_apply_trailer(self, primaryNode, nodelist):
...@@ -1250,3 +1343,21 @@ _assign_types = [ ...@@ -1250,3 +1343,21 @@ _assign_types = [
symbol.term, symbol.term,
symbol.factor, symbol.factor,
] ]
import types
_names = {}
for k, v in symbol.sym_name.items():
_names[k] = v
for k, v in token.tok_name.items():
_names[k] = v
def debug_tree(tree):
l = []
for elt in tree:
if type(elt) == types.IntType:
l.append(_names.get(elt, elt))
elif type(elt) == types.StringType:
l.append(elt)
else:
l.append(debug_tree(elt))
return l
Tools/compiler/compiler/ast.py
View file @ 9c048f9f
...@@ -279,22 +279,24 @@ class Const(Node): ...@@ -279,22 +279,24 @@ class Const(Node):
class Print(Node): class Print(Node):
nodes['print'] = 'Print' nodes['print'] = 'Print'
def __init__(self, nodes): def __init__(self, nodes, dest):
self.nodes = nodes self.nodes = nodes
self._children = ('print', nodes) self.dest = dest
self._children = ('print', nodes, dest)
def __repr__(self): def __repr__(self):
return "Print(%s)" % self._children[1:] return "Print(%s, %s)" % (self._children[1:-1], self._children[-1])
class Printnl(Node): class Printnl(Node):
nodes['printnl'] = 'Printnl' nodes['printnl'] = 'Printnl'
def __init__(self, nodes): def __init__(self, nodes, dest):
self.nodes = nodes self.nodes = nodes
self._children = ('printnl', nodes) self.dest = dest
self._children = ('printnl', nodes, dest)
def __repr__(self): def __repr__(self):
return "Printnl(%s)" % self._children[1:] return "Printnl(%s, %s)" % (self._children[1:-1], self._children[-1])
class Discard(Node): class Discard(Node):
nodes['discard'] = 'Discard' nodes['discard'] = 'Discard'
...@@ -306,6 +308,18 @@ class Discard(Node): ...@@ -306,6 +308,18 @@ class Discard(Node):
def __repr__(self): def __repr__(self):
return "Discard(%s)" % self._children[1:] return "Discard(%s)" % self._children[1:]
class AugAssign(Node):
nodes['augassign'] = 'AugAssign'
def __init__(self, node, op, expr):
self.node = node
self.op = op
self.expr = expr
self._children = ('augassign', node, op, expr)
def __repr__(self):
return "AugAssign(%s)" % str(self._children[1:])
class Assign(Node): class Assign(Node):
nodes['assign'] = 'Assign' nodes['assign'] = 'Assign'
...@@ -360,6 +374,41 @@ class AssAttr(Node): ...@@ -360,6 +374,41 @@ class AssAttr(Node):
def __repr__(self): def __repr__(self):
return "AssAttr(%s,%s,%s)" % self._children[1:] return "AssAttr(%s,%s,%s)" % self._children[1:]
class ListComp(Node):
nodes['listcomp'] = 'ListComp'
def __init__(self, expr, quals):
self.expr = expr
self.quals = quals
self._children = ('listcomp', expr, quals)
def __repr__(self):
return "ListComp(%s, %s)" % self._children[1:]
class ListCompFor(Node):
nodes['listcomp_for'] = 'ListCompFor'
# transformer fills in ifs after node is created
def __init__(self, assign, list, ifs):
self.assign = assign
self.list = list
self.ifs = ifs
self._children = ('listcomp_for', assign, list, ifs)
def __repr__(self):
return "ListCompFor(%s, %s, %s)" % self._children[1:]
class ListCompIf(Node):
nodes['listcomp_if'] = 'ListCompIf'
def __init__(self, test):
self.test = test
self._children = ('listcomp_if', test)
def __repr__(self):
return "ListCompIf(%s)" % self._children[1:]
class List(Node): class List(Node):
nodes['list'] = 'List' nodes['list'] = 'List'
......
Tools/compiler/compiler/pyassem.py
View file @ 9c048f9f
...@@ -253,8 +253,14 @@ class PyFlowGraph(FlowGraph): ...@@ -253,8 +253,14 @@ class PyFlowGraph(FlowGraph):
def _lookupName(self, name, list): def _lookupName(self, name, list):
"""Return index of name in list, appending if necessary""" """Return index of name in list, appending if necessary"""
if name in list: found = None
i = list.index(name) t = type(name)
for i in range(len(list)):
# must do a comparison on type first to prevent UnicodeErrors
if t == type(list[i]) and list[i] == name:
found = 1
break
if found:
# this is cheap, but incorrect in some cases, e.g 2 vs. 2L # this is cheap, but incorrect in some cases, e.g 2 vs. 2L
if type(name) == type(list[i]): if type(name) == type(list[i]):
return i return i
......
Tools/compiler/compiler/pycodegen.py
View file @ 9c048f9f
import imp
import os import os
import marshal import marshal
import stat import stat
import string import string
import struct import struct
import sys
import types import types
from cStringIO import StringIO from cStringIO import StringIO
...@@ -10,6 +12,12 @@ from compiler import ast, parse, walk ...@@ -10,6 +12,12 @@ from compiler import ast, parse, walk
from compiler import pyassem, misc from compiler import pyassem, misc
from compiler.pyassem import CO_VARARGS, CO_VARKEYWORDS, CO_NEWLOCALS, TupleArg from compiler.pyassem import CO_VARARGS, CO_VARKEYWORDS, CO_NEWLOCALS, TupleArg
# Do we have Python 1.x or Python 2.x?
try:
VERSION = sys.version_info[0]
except AttributeError:
VERSION = 1
callfunc_opcode_info = { callfunc_opcode_info = {
# (Have *args, Have **args) : opcode # (Have *args, Have **args) : opcode
(0,0) : "CALL_FUNCTION", (0,0) : "CALL_FUNCTION",
...@@ -18,12 +26,12 @@ callfunc_opcode_info = { ...@@ -18,12 +26,12 @@ callfunc_opcode_info = {
(1,1) : "CALL_FUNCTION_VAR_KW", (1,1) : "CALL_FUNCTION_VAR_KW",
} }
def compile(filename): def compile(filename, display=0):
f = open(filename) f = open(filename)
buf = f.read() buf = f.read()
f.close() f.close()
mod = Module(buf, filename) mod = Module(buf, filename)
mod.compile() mod.compile(display)
f = open(filename + "c", "wb") f = open(filename + "c", "wb")
mod.dump(f) mod.dump(f)
f.close() f.close()
...@@ -34,28 +42,30 @@ class Module: ...@@ -34,28 +42,30 @@ class Module:
self.source = source self.source = source
self.code = None self.code = None
def compile(self): def compile(self, display=0):
ast = parse(self.source) ast = parse(self.source)
root, filename = os.path.split(self.filename) root, filename = os.path.split(self.filename)
gen = ModuleCodeGenerator(filename) gen = ModuleCodeGenerator(filename)
walk(ast, gen, 1) walk(ast, gen, 1)
if display:
import pprint
print pprint.pprint(ast)
self.code = gen.getCode() self.code = gen.getCode()
def dump(self, f): def dump(self, f):
f.write(self.getPycHeader()) f.write(self.getPycHeader())
marshal.dump(self.code, f) marshal.dump(self.code, f)
MAGIC = (50823 | (ord('\r')<<16) | (ord('\n')<<24)) MAGIC = imp.get_magic()
def getPycHeader(self): def getPycHeader(self):
# compile.c uses marshal to write a long directly, with # compile.c uses marshal to write a long directly, with
# calling the interface that would also generate a 1-byte code # calling the interface that would also generate a 1-byte code
# to indicate the type of the value. simplest way to get the # to indicate the type of the value. simplest way to get the
# same effect is to call marshal and then skip the code. # same effect is to call marshal and then skip the code.
magic = marshal.dumps(self.MAGIC)[1:]
mtime = os.stat(self.filename)[stat.ST_MTIME] mtime = os.stat(self.filename)[stat.ST_MTIME]
mtime = struct.pack('i', mtime) mtime = struct.pack('i', mtime)
return magic + mtime return self.MAGIC + mtime
class CodeGenerator: class CodeGenerator:
...@@ -63,7 +73,7 @@ class CodeGenerator: ...@@ -63,7 +73,7 @@ class CodeGenerator:
def __init__(self, filename): def __init__(self, filename):
## Subclasses must define a constructor that intializes self.graph ## Subclasses must define a constructor that intializes self.graph
## before calling this init function ## before calling this init function, e.g.
## self.graph = pyassem.PyFlowGraph() ## self.graph = pyassem.PyFlowGraph()
self.filename = filename self.filename = filename
self.locals = misc.Stack() self.locals = misc.Stack()
...@@ -142,7 +152,6 @@ class CodeGenerator: ...@@ -142,7 +152,6 @@ class CodeGenerator:
def visitLambda(self, node): def visitLambda(self, node):
self._visitFuncOrLambda(node, isLambda=1) self._visitFuncOrLambda(node, isLambda=1)
## self.storeName("<lambda>")
def _visitFuncOrLambda(self, node, isLambda): def _visitFuncOrLambda(self, node, isLambda):
gen = FunctionCodeGenerator(node, self.filename, isLambda) gen = FunctionCodeGenerator(node, self.filename, isLambda)
...@@ -180,10 +189,6 @@ class CodeGenerator: ...@@ -180,10 +189,6 @@ class CodeGenerator:
test, suite = node.tests[i] test, suite = node.tests[i]
self.set_lineno(test) self.set_lineno(test)
self.visit(test) self.visit(test)
## if i == numtests - 1 and not node.else_:
## nextTest = end
## else:
## nextTest = self.newBlock()
nextTest = self.newBlock() nextTest = self.newBlock()
self.emit('JUMP_IF_FALSE', nextTest) self.emit('JUMP_IF_FALSE', nextTest)
self.nextBlock() self.nextBlock()
...@@ -304,6 +309,70 @@ class CodeGenerator: ...@@ -304,6 +309,70 @@ class CodeGenerator:
self.emit('POP_TOP') self.emit('POP_TOP')
self.nextBlock(end) self.nextBlock(end)
# list comprehensions
__list_count = 0
def visitListComp(self, node):
# XXX would it be easier to transform the AST into the form it
# would have if the list comp were expressed as a series of
# for and if stmts and an explicit append?
self.set_lineno(node)
# setup list
append = "$append%d" % self.__list_count
self.__list_count = self.__list_count + 1
self.emit('BUILD_LIST', 0)
self.emit('DUP_TOP')
self.emit('LOAD_ATTR', 'append')
self.storeName(append)
l = len(node.quals)
stack = []
for i, for_ in zip(range(l), node.quals):
start, anchor = self.visit(for_)
cont = None
for if_ in for_.ifs:
if cont is None:
cont = self.newBlock()
self.visit(if_, cont)
stack.insert(0, (start, cont, anchor))
self.loadName(append)
self.visit(node.expr)
self.emit('CALL_FUNCTION', 1)
self.emit('POP_TOP')
for start, cont, anchor in stack:
if cont:
skip_one = self.newBlock()
self.emit('JUMP_FORWARD', skip_one)
self.nextBlock(cont)
self.emit('POP_TOP')
self.nextBlock(skip_one)
self.emit('JUMP_ABSOLUTE', start)
self.nextBlock(anchor)
self.delName(append)
self.__list_count = self.__list_count - 1
def visitListCompFor(self, node):
self.set_lineno(node)
start = self.newBlock()
anchor = self.newBlock()
self.visit(node.list)
self.visit(ast.Const(0))
self.emit('SET_LINENO', node.lineno)
self.nextBlock(start)
self.emit('FOR_LOOP', anchor)
self.visit(node.assign)
return start, anchor
def visitListCompIf(self, node, branch):
self.set_lineno(node)
self.visit(node.test)
self.emit('JUMP_IF_FALSE', branch)
self.newBlock()
self.emit('POP_TOP')
# exception related # exception related
def visitAssert(self, node): def visitAssert(self, node):
...@@ -397,10 +466,6 @@ class CodeGenerator: ...@@ -397,10 +466,6 @@ class CodeGenerator:
# misc # misc
## def visitStmt(self, node):
## # nothing to do except walk the children
## pass
def visitDiscard(self, node): def visitDiscard(self, node):
self.visit(node.expr) self.visit(node.expr)
self.emit('POP_TOP') self.emit('POP_TOP')
...@@ -426,17 +491,20 @@ class CodeGenerator: ...@@ -426,17 +491,20 @@ class CodeGenerator:
def visitImport(self, node): def visitImport(self, node):
self.set_lineno(node) self.set_lineno(node)
for name, alias in node.names: for name, alias in node.names:
if VERSION > 1:
self.emit('LOAD_CONST', None) self.emit('LOAD_CONST', None)
self.emit('IMPORT_NAME', name) self.emit('IMPORT_NAME', name)
self._resolveDots(name) mod = string.split(name, ".")[0]
self.storeName(alias or name) self.storeName(alias or mod)
def visitFrom(self, node): def visitFrom(self, node):
self.set_lineno(node) self.set_lineno(node)
fromlist = map(lambda (name, alias): name, node.names) fromlist = map(lambda (name, alias): name, node.names)
if VERSION > 1:
self.emit('LOAD_CONST', tuple(fromlist)) self.emit('LOAD_CONST', tuple(fromlist))
self.emit('IMPORT_NAME', node.modname) self.emit('IMPORT_NAME', node.modname)
for name, alias in node.names: for name, alias in node.names:
if VERSION > 1:
if name == '*': if name == '*':
self.namespace = 0 self.namespace = 0
self.emit('IMPORT_STAR') self.emit('IMPORT_STAR')
...@@ -447,6 +515,8 @@ class CodeGenerator: ...@@ -447,6 +515,8 @@ class CodeGenerator:
self.emit('IMPORT_FROM', name) self.emit('IMPORT_FROM', name)
self._resolveDots(name) self._resolveDots(name)
self.storeName(alias or name) self.storeName(alias or name)
else:
self.emit('IMPORT_FROM', name)
self.emit('POP_TOP') self.emit('POP_TOP')
def _resolveDots(self, name): def _resolveDots(self, name):
...@@ -491,13 +561,85 @@ class CodeGenerator: ...@@ -491,13 +561,85 @@ class CodeGenerator:
print "warning: unexpected flags:", node.flags print "warning: unexpected flags:", node.flags
print node print node
def visitAssTuple(self, node): def _visitAssSequence(self, node, op='UNPACK_SEQUENCE'):
if findOp(node) != 'OP_DELETE': if findOp(node) != 'OP_DELETE':
self.emit('UNPACK_SEQUENCE', len(node.nodes)) self.emit(op, len(node.nodes))
for child in node.nodes: for child in node.nodes:
self.visit(child) self.visit(child)
visitAssList = visitAssTuple if VERSION > 1:
visitAssTuple = _visitAssSequence
visitAssList = _visitAssSequence
else:
def visitAssTuple(self, node):
self._visitAssSequence(node, 'UNPACK_TUPLE')
def visitAssList(self, node):
self._visitAssSequence(node, 'UNPACK_LIST')
# augmented assignment
def visitAugAssign(self, node):
aug_node = wrap_aug(node.node)
self.visit(aug_node, "load")
self.visit(node.expr)
self.emit(self._augmented_opcode[node.op])
self.visit(aug_node, "store")
_augmented_opcode = {
'+=' : 'INPLACE_ADD',
'-=' : 'INPLACE_SUBTRACT',
'*=' : 'INPLACE_MULTIPLY',
'/=' : 'INPLACE_DIVIDE',
'%=' : 'INPLACE_MODULO',
'**=': 'INPLACE_POWER',
'>>=': 'INPLACE_RSHIFT',
'<<=': 'INPLACE_LSHIFT',
'&=' : 'INPLACE_AND',
'^=' : 'INPLACE_XOR',
'|=' : 'INPLACE_OR',
}
def visitAugName(self, node, mode):
if mode == "load":
self.loadName(node.name)
elif mode == "store":
self.storeName(node.name)
def visitAugGetattr(self, node, mode):
if mode == "load":
self.visit(node.expr)
self.emit('DUP_TOP')
self.emit('LOAD_ATTR', node.attrname)
elif mode == "store":
self.emit('ROT_TWO')
self.emit('STORE_ATTR', node.attrname)
def visitAugSlice(self, node, mode):
if mode == "load":
self.visitSlice(node, 1)
elif mode == "store":
slice = 0
if node.lower:
slice = slice | 1
if node.upper:
slice = slice | 2
if slice == 0:
self.emit('ROT_TWO')
elif slice == 3:
self.emit('ROT_FOUR')
else:
self.emit('ROT_THREE')
self.emit('STORE_SLICE+%d' % slice)
def visitAugSubscript(self, node, mode):
if len(node.subs) > 1:
raise SyntaxError, "augmented assignment to tuple is not possible"
if mode == "load":
self.visitSubscript(node, 1)
elif mode == "store":
self.emit('ROT_THREE')
self.emit('STORE_SUBSCR')
def visitExec(self, node): def visitExec(self, node):
self.visit(node.expr) self.visit(node.expr)
...@@ -533,12 +675,23 @@ class CodeGenerator: ...@@ -533,12 +675,23 @@ class CodeGenerator:
def visitPrint(self, node): def visitPrint(self, node):
self.set_lineno(node) self.set_lineno(node)
if node.dest:
self.visit(node.dest)
for child in node.nodes: for child in node.nodes:
if node.dest:
self.emit('DUP_TOP')
self.visit(child) self.visit(child)
if node.dest:
self.emit('ROT_TWO')
self.emit('PRINT_ITEM_TO')
else:
self.emit('PRINT_ITEM') self.emit('PRINT_ITEM')
def visitPrintnl(self, node): def visitPrintnl(self, node):
self.visitPrint(node) self.visitPrint(node)
if node.dest:
self.emit('PRINT_NEWLINE_TO')
else:
self.emit('PRINT_NEWLINE') self.emit('PRINT_NEWLINE')
def visitReturn(self, node): def visitReturn(self, node):
...@@ -548,7 +701,8 @@ class CodeGenerator: ...@@ -548,7 +701,8 @@ class CodeGenerator:
# slice and subscript stuff # slice and subscript stuff
def visitSlice(self, node): def visitSlice(self, node, aug_flag=None):
# aug_flag is used by visitAugSlice
self.visit(node.expr) self.visit(node.expr)
slice = 0 slice = 0
if node.lower: if node.lower:
...@@ -557,6 +711,13 @@ class CodeGenerator: ...@@ -557,6 +711,13 @@ class CodeGenerator:
if node.upper: if node.upper:
self.visit(node.upper) self.visit(node.upper)
slice = slice | 2 slice = slice | 2
if aug_flag:
if slice == 0:
self.emit('DUP_TOP')
elif slice == 3:
self.emit('DUP_TOPX', 3)
else:
self.emit('DUP_TOPX', 2)
if node.flags == 'OP_APPLY': if node.flags == 'OP_APPLY':
self.emit('SLICE+%d' % slice) self.emit('SLICE+%d' % slice)
elif node.flags == 'OP_ASSIGN': elif node.flags == 'OP_ASSIGN':
...@@ -567,10 +728,12 @@ class CodeGenerator: ...@@ -567,10 +728,12 @@ class CodeGenerator:
print "weird slice", node.flags print "weird slice", node.flags
raise raise
def visitSubscript(self, node): def visitSubscript(self, node, aug_flag=None):
self.visit(node.expr) self.visit(node.expr)
for sub in node.subs: for sub in node.subs:
self.visit(sub) self.visit(sub)
if aug_flag:
self.emit('DUP_TOPX', 2)
if len(node.subs) > 1: if len(node.subs) > 1:
self.emit('BUILD_TUPLE', len(node.subs)) self.emit('BUILD_TUPLE', len(node.subs))
if node.flags == 'OP_APPLY': if node.flags == 'OP_APPLY':
...@@ -740,7 +903,10 @@ class FunctionCodeGenerator(CodeGenerator): ...@@ -740,7 +903,10 @@ class FunctionCodeGenerator(CodeGenerator):
self.unpackSequence(arg) self.unpackSequence(arg)
def unpackSequence(self, tup): def unpackSequence(self, tup):
if VERSION > 1:
self.emit('UNPACK_SEQUENCE', len(tup)) self.emit('UNPACK_SEQUENCE', len(tup))
else:
self.emit('UNPACK_TUPLE', len(tup))
for elt in tup: for elt in tup:
if type(elt) == types.TupleType: if type(elt) == types.TupleType:
self.unpackSequence(elt) self.unpackSequence(elt)
...@@ -765,7 +931,6 @@ class ClassCodeGenerator(CodeGenerator): ...@@ -765,7 +931,6 @@ class ClassCodeGenerator(CodeGenerator):
self.emit('LOAD_LOCALS') self.emit('LOAD_LOCALS')
self.emit('RETURN_VALUE') self.emit('RETURN_VALUE')
def generateArgList(arglist): def generateArgList(arglist):
"""Generate an arg list marking TupleArgs""" """Generate an arg list marking TupleArgs"""
args = [] args = []
...@@ -838,6 +1003,45 @@ class OpFinder: ...@@ -838,6 +1003,45 @@ class OpFinder:
elif self.op != node.flags: elif self.op != node.flags:
raise ValueError, "mixed ops in stmt" raise ValueError, "mixed ops in stmt"
class Delegator:
"""Base class to support delegation for augmented assignment nodes
To generator code for augmented assignments, we use the following
wrapper classes. In visitAugAssign, the left-hand expression node
is visited twice. The first time the visit uses the normal method
for that node . The second time the visit uses a different method
that generates the appropriate code to perform the assignment.
These delegator classes wrap the original AST nodes in order to
support the variant visit methods.
"""
def __init__(self, obj):
self.obj = obj
def __getattr__(self, attr):
return getattr(self.obj, attr)
class AugGetattr(Delegator):
pass
class AugName(Delegator):
pass
class AugSlice(Delegator):
pass
class AugSubscript(Delegator):
pass
wrapper = {
ast.Getattr: AugGetattr,
ast.Name: AugName,
ast.Slice: AugSlice,
ast.Subscript: AugSubscript,
}
def wrap_aug(node):
return wrapper[node.__class__](node)
if __name__ == "__main__": if __name__ == "__main__":
import sys import sys
......
Tools/compiler/compiler/transformer.py
View file @ 9c048f9f
#
# Copyright (C) 1997-1998 Greg Stein. All Rights Reserved.
#
# This module is provided under a BSD-ish license. See
# http://www.opensource.org/licenses/bsd-license.html
# and replace OWNER, ORGANIZATION, and YEAR as appropriate.
#
#
# Written by Greg Stein (gstein@lyra.org)
# and Bill Tutt (rassilon@lima.mudlib.org)
# February 1997.
#
# Support for ast.Node subclasses written and other revisions by
# Jeremy Hylton (jeremy@beopen.com)
#
"""Parse tree transformation module. """Parse tree transformation module.
Transforms Python source code into an abstract syntax tree (AST) Transforms Python source code into an abstract syntax tree (AST)
...@@ -24,7 +8,21 @@ parse(buf) -> AST ...@@ -24,7 +8,21 @@ parse(buf) -> AST
parseFile(path) -> AST parseFile(path) -> AST
""" """
# Original version written by Greg Stein (gstein@lyra.org)
# and Bill Tutt (rassilon@lima.mudlib.org)
# February 1997.
#
# Modifications and improvements for Python 2.0 by Jeremy Hylton and
# Mark Hammond
# Portions of this file are:
# Copyright (C) 1997-1998 Greg Stein. All Rights Reserved.
# #
# This module is provided under a BSD-ish license. See
# http://www.opensource.org/licenses/bsd-license.html
# and replace OWNER, ORGANIZATION, and YEAR as appropriate.
# The output tree has the following nodes: # The output tree has the following nodes:
# #
# Source Python line #'s appear at the end of each of all of these nodes # Source Python line #'s appear at the end of each of all of these nodes
...@@ -49,9 +47,10 @@ parseFile(path) -> AST ...@@ -49,9 +47,10 @@ parseFile(path) -> AST
# tryexcept: trySuiteNode, [ (exprNode, assgnNode, suiteNode), ... ], elseNode # tryexcept: trySuiteNode, [ (exprNode, assgnNode, suiteNode), ... ], elseNode
# return: valueNode # return: valueNode
# const: value # const: value
# print: [ node1, ..., nodeN ] # print: [ node1, ..., nodeN ] [, dest]
# printnl: [ node1, ..., nodeN ] # printnl: [ node1, ..., nodeN ] [, dest]
# discard: exprNode # discard: exprNode
# augassign: node, op, expr
# assign: [ node1, ..., nodeN ], exprNode # assign: [ node1, ..., nodeN ], exprNode
# ass_tuple: [ node1, ..., nodeN ] # ass_tuple: [ node1, ..., nodeN ]
# ass_list: [ node1, ..., nodeN ] # ass_list: [ node1, ..., nodeN ]
...@@ -97,12 +96,12 @@ parseFile(path) -> AST ...@@ -97,12 +96,12 @@ parseFile(path) -> AST
import ast import ast
import parser import parser
# Care must be taken to use only symbols and tokens defined in Python
# 1.5.2 for code branches executed in 1.5.2
import symbol import symbol
import token import token
import string import string
import pprint
error = 'walker.error' error = 'walker.error'
from consts import CO_VARARGS, CO_VARKEYWORDS from consts import CO_VARARGS, CO_VARKEYWORDS
...@@ -328,27 +327,44 @@ class Transformer: ...@@ -328,27 +327,44 @@ class Transformer:
# #
def expr_stmt(self, nodelist): def expr_stmt(self, nodelist):
# testlist ('=' testlist)* # augassign testlist | testlist ('=' testlist)*
exprNode = self.com_node(nodelist[-1]) exprNode = self.com_node(nodelist[-1])
if len(nodelist) == 1: if len(nodelist) == 1:
return Node('discard', exprNode) return Node('discard', exprNode)
if nodelist[1][0] == token.EQUAL:
nodes = [ ] nodes = [ ]
for i in range(0, len(nodelist) - 2, 2): for i in range(0, len(nodelist) - 2, 2):
nodes.append(self.com_assign(nodelist[i], OP_ASSIGN)) nodes.append(self.com_assign(nodelist[i], OP_ASSIGN))
n = Node('assign', nodes, exprNode) n = Node('assign', nodes, exprNode)
n.lineno = nodelist[1][2] n.lineno = nodelist[1][2]
else:
lval = self.com_augassign(nodelist[0])
op = self.com_augassign_op(nodelist[1])
n = Node('augassign', lval, op[1], exprNode)
n.lineno = op[2]
return n return n
def print_stmt(self, nodelist): def print_stmt(self, nodelist):
# print: (test ',')* [test] # print ([ test (',' test)* [','] ] | '>>' test [ (',' test)+ [','] ])
items = [ ] items = [ ]
for i in range(1, len(nodelist), 2): if len(nodelist) == 1:
start = 1
dest = None
elif nodelist[1][0] == token.RIGHTSHIFT:
assert len(nodelist) == 3 \
or nodelist[3][0] == token.COMMA
dest = self.com_node(nodelist[2])
start = 4
else:
dest = None
start = 1
for i in range(start, len(nodelist), 2):
items.append(self.com_node(nodelist[i])) items.append(self.com_node(nodelist[i]))
if nodelist[-1][0] == token.COMMA: if nodelist[-1][0] == token.COMMA:
n = Node('print', items) n = Node('print', items, dest)
n.lineno = nodelist[0][2] n.lineno = nodelist[0][2]
return n return n
n = Node('printnl', items) n = Node('printnl', items, dest)
n.lineno = nodelist[0][2] n.lineno = nodelist[0][2]
return n return n
...@@ -405,15 +421,22 @@ class Transformer: ...@@ -405,15 +421,22 @@ class Transformer:
# import_stmt: 'import' dotted_as_name (',' dotted_as_name)* | # import_stmt: 'import' dotted_as_name (',' dotted_as_name)* |
# from: 'from' dotted_name 'import' # from: 'from' dotted_name 'import'
# ('*' | import_as_name (',' import_as_name)*) # ('*' | import_as_name (',' import_as_name)*)
names = []
is_as = 0
if nodelist[0][1] == 'from': if nodelist[0][1] == 'from':
names = []
if nodelist[3][0] == token.NAME:
for i in range(3, len(nodelist), 2):
names.append((nodelist[i][1], None))
else:
for i in range(3, len(nodelist), 2): for i in range(3, len(nodelist), 2):
names.append(self.com_import_as_name(nodelist[i][1])) names.append(self.com_import_as_name(nodelist[i][1]))
n = Node('from', self.com_dotted_name(nodelist[1]), names) n = Node('from', self.com_dotted_name(nodelist[1]), names)
n.lineno = nodelist[0][2] n.lineno = nodelist[0][2]
return n return n
if nodelist[1][0] == symbol.dotted_name:
names = [(self.com_dotted_name(nodelist[1][1:]), None)]
else:
names = []
for i in range(1, len(nodelist), 2): for i in range(1, len(nodelist), 2):
names.append(self.com_dotted_as_name(nodelist[i])) names.append(self.com_dotted_as_name(nodelist[i]))
n = Node('import', names) n = Node('import', names)
...@@ -737,7 +760,7 @@ class Transformer: ...@@ -737,7 +760,7 @@ class Transformer:
return Node('discard', Node('const', None)) return Node('discard', Node('const', None))
if node[0] not in _legal_node_types: if node[0] not in _legal_node_types:
raise error, 'illegal node passed to com_node: %s' % node[0] raise error, 'illegal node passed to com_node: %s' % `node`
# print "dispatch", self._dispatch[node[0]].__name__, node # print "dispatch", self._dispatch[node[0]].__name__, node
return self._dispatch[node[0]](node[1:]) return self._dispatch[node[0]](node[1:])
...@@ -818,8 +841,11 @@ class Transformer: ...@@ -818,8 +841,11 @@ class Transformer:
def com_dotted_as_name(self, node): def com_dotted_as_name(self, node):
dot = self.com_dotted_name(node[1]) dot = self.com_dotted_name(node[1])
if len(node) == 2: if len(node) <= 2:
return dot, None return dot, None
if node[0] == symbol.dotted_name:
pass
else:
assert node[2][1] == 'as' assert node[2][1] == 'as'
assert node[3][0] == token.NAME assert node[3][0] == token.NAME
return dot, node[3][1] return dot, node[3][1]
...@@ -872,6 +898,20 @@ class Transformer: ...@@ -872,6 +898,20 @@ class Transformer:
n.lineno = nodelist[0][2] n.lineno = nodelist[0][2]
return n return n
def com_augassign_op(self, node):
assert node[0] == symbol.augassign
return node[1]
def com_augassign(self, node):
"""Return node suitable for lvalue of augmented assignment
Names, slices, and attributes are the only allowable nodes.
"""
l = self.com_node(node)
if l[0] in ('name', 'slice', 'subscript', 'getattr'):
return l
raise SyntaxError, "can't assign to %s" % l[0]
def com_assign(self, node, assigning): def com_assign(self, node, assigning):
# return a node suitable for use as an "lvalue" # return a node suitable for use as an "lvalue"
# loop to avoid trivial recursion # loop to avoid trivial recursion
...@@ -955,7 +995,6 @@ class Transformer: ...@@ -955,7 +995,6 @@ class Transformer:
return Node(type, items) return Node(type, items)
def com_stmt(self, node): def com_stmt(self, node):
#pprint.pprint(node)
result = self.com_node(node) result = self.com_node(node)
try: try:
result[0] result[0]
...@@ -976,6 +1015,59 @@ class Transformer: ...@@ -976,6 +1015,59 @@ class Transformer:
else: else:
stmts.append(result) stmts.append(result)
if hasattr(symbol, 'list_for'):
def com_list_constructor(self, nodelist):
# listmaker: test ( list_for | (',' test)* [','] )
values = [ ]
for i in range(1, len(nodelist)):
if nodelist[i][0] == symbol.list_for:
assert len(nodelist[i:]) == 1
return self.com_list_comprehension(values[0],
nodelist[i])
elif nodelist[i][0] == token.COMMA:
continue
values.append(self.com_node(nodelist[i]))
return Node('list', values)
def com_list_comprehension(self, expr, node):
# list_iter: list_for | list_if
# list_for: 'for' exprlist 'in' testlist [list_iter]
# list_if: 'if' test [list_iter]
lineno = node[1][2]
fors = []
while node:
if node[1][1] == 'for':
assignNode = self.com_assign(node[2], OP_ASSIGN)
listNode = self.com_node(node[4])
newfor = Node('listcomp_for', assignNode,
listNode, [])
newfor.lineno = node[1][2]
fors.append(newfor)
if len(node) == 5:
node = None
else:
node = self.com_list_iter(node[5])
elif node[1][1] == 'if':
test = self.com_node(node[2])
newif = Node('listcomp_if', test)
newif.lineno = node[1][2]
newfor.ifs.append(newif)
if len(node) == 3:
node = None
else:
node = self.com_list_iter(node[3])
else:
raise SyntaxError, \
("unexpected list comprehension element: %s %d"
% (node, lineno))
n = Node('listcomp', expr, fors)
n.lineno = lineno
return n
def com_list_iter(self, node):
assert node[0] == symbol.list_iter
return node[1]
else:
def com_list_constructor(self, nodelist): def com_list_constructor(self, nodelist):
values = [ ] values = [ ]
for i in range(1, len(nodelist), 2): for i in range(1, len(nodelist), 2):
...@@ -986,7 +1078,8 @@ class Transformer: ...@@ -986,7 +1078,8 @@ class Transformer:
# dictmaker: test ':' test (',' test ':' value)* [','] # dictmaker: test ':' test (',' test ':' value)* [',']
items = [ ] items = [ ]
for i in range(1, len(nodelist), 4): for i in range(1, len(nodelist), 4):
items.append((self.com_node(nodelist[i]), self.com_node(nodelist[i+2]))) items.append((self.com_node(nodelist[i]),
self.com_node(nodelist[i+2])))
return Node('dict', items) return Node('dict', items)
def com_apply_trailer(self, primaryNode, nodelist): def com_apply_trailer(self, primaryNode, nodelist):
...@@ -1250,3 +1343,21 @@ _assign_types = [ ...@@ -1250,3 +1343,21 @@ _assign_types = [
symbol.term, symbol.term,
symbol.factor, symbol.factor,
] ]
import types
_names = {}
for k, v in symbol.sym_name.items():
_names[k] = v
for k, v in token.tok_name.items():
_names[k] = v
def debug_tree(tree):
l = []
for elt in tree:
if type(elt) == types.IntType:
l.append(_names.get(elt, elt))
elif type(elt) == types.StringType:
l.append(elt)
else:
l.append(debug_tree(elt))
return l
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7