Skip to content

C
cpython
Commit 9c048f9f authored by Jeremy Hylton's avatar Jeremy Hylton
Browse files
Options

Now supports entire Python 2.0 language and still supports Python 

1.5.2.  The compiler generates code for the version of the interpreter
it is run under.

ast.py:
    Print and Printnl add dest attr for extended print
    new node AugAssign for augmented assignments
    new nodes ListComp, ListCompFor, and ListCompIf for list
        comprehensions

pyassem.py:
    add work around for string-Unicode comparison raising UnicodeError
        on comparison of two objects in code object's const table

pycodegen.py:
    define VERSION, the Python major version number
    get magic number using imp.get_magic() instead of hard coding
    implement list comprehensions, extended print, and augmented
        assignment; augmented assignment uses Delegator classes (see
        doc string)
    fix import and tuple unpacking for 1.5.2

transformer.py:
    various changes to support new 2.0 grammar and old 1.5 grammar
    add debug_tree helper than converts and symbol and token numbers
    to their names
parent 5bad5a4b
Hide whitespace changes
Inline Side-by-side
Showing with 926 additions and 186 deletions
Lib/compiler/ast.py
View file @ 9c048f9f
......@@ -279,22 +279,24 @@ class Const(Node):
class Print(Node):
nodes['print'] = 'Print'
def __init__(self, nodes):
def __init__(self, nodes, dest):
self.nodes = nodes
self._children = ('print', nodes)
self.dest = dest
self._children = ('print', nodes, dest)
def __repr__(self):
return "Print(%s)" % self._children[1:]
return "Print(%s, %s)" % (self._children[1:-1], self._children[-1])
class Printnl(Node):
nodes['printnl'] = 'Printnl'
def __init__(self, nodes):
def __init__(self, nodes, dest):
self.nodes = nodes
self._children = ('printnl', nodes)
self.dest = dest
self._children = ('printnl', nodes, dest)
def __repr__(self):
return "Printnl(%s)" % self._children[1:]
return "Printnl(%s, %s)" % (self._children[1:-1], self._children[-1])
class Discard(Node):
nodes['discard'] = 'Discard'
......@@ -306,6 +308,18 @@ class Discard(Node):
def __repr__(self):
return "Discard(%s)" % self._children[1:]
class AugAssign(Node):
nodes['augassign'] = 'AugAssign'
def __init__(self, node, op, expr):
self.node = node
self.op = op
self.expr = expr
self._children = ('augassign', node, op, expr)
def __repr__(self):
return "AugAssign(%s)" % str(self._children[1:])
class Assign(Node):
nodes['assign'] = 'Assign'
......@@ -360,6 +374,41 @@ class AssAttr(Node):
def __repr__(self):
return "AssAttr(%s,%s,%s)" % self._children[1:]
class ListComp(Node):
nodes['listcomp'] = 'ListComp'
def __init__(self, expr, quals):
self.expr = expr
self.quals = quals
self._children = ('listcomp', expr, quals)
def __repr__(self):
return "ListComp(%s, %s)" % self._children[1:]
class ListCompFor(Node):
nodes['listcomp_for'] = 'ListCompFor'
# transformer fills in ifs after node is created
def __init__(self, assign, list, ifs):
self.assign = assign
self.list = list
self.ifs = ifs
self._children = ('listcomp_for', assign, list, ifs)
def __repr__(self):
return "ListCompFor(%s, %s, %s)" % self._children[1:]
class ListCompIf(Node):
nodes['listcomp_if'] = 'ListCompIf'
def __init__(self, test):
self.test = test
self._children = ('listcomp_if', test)
def __repr__(self):
return "ListCompIf(%s)" % self._children[1:]
class List(Node):
nodes['list'] = 'List'
......
Lib/compiler/pyassem.py
View file @ 9c048f9f
......@@ -253,8 +253,14 @@ class PyFlowGraph(FlowGraph):
def _lookupName(self, name, list):
"""Return index of name in list, appending if necessary"""
if name in list:
i = list.index(name)
found = None
t = type(name)
for i in range(len(list)):
# must do a comparison on type first to prevent UnicodeErrors
if t == type(list[i]) and list[i] == name:
found = 1
break
if found:
# this is cheap, but incorrect in some cases, e.g 2 vs. 2L
if type(name) == type(list[i]):
return i
......
Lib/compiler/pycodegen.py
View file @ 9c048f9f
import imp
import os
import marshal
import stat
import string
import struct
import sys
import types
from cStringIO import StringIO
......@@ -10,6 +12,12 @@ from compiler import ast, parse, walk
from compiler import pyassem, misc
from compiler.pyassem import CO_VARARGS, CO_VARKEYWORDS, CO_NEWLOCALS, TupleArg
# Do we have Python 1.x or Python 2.x?
try:
VERSION = sys.version_info[0]
except AttributeError:
VERSION = 1
callfunc_opcode_info = {
# (Have *args, Have **args) : opcode
(0,0) : "CALL_FUNCTION",
......@@ -18,12 +26,12 @@ callfunc_opcode_info = {
(1,1) : "CALL_FUNCTION_VAR_KW",
}
def compile(filename):
def compile(filename, display=0):
f = open(filename)
buf = f.read()
f.close()
mod = Module(buf, filename)
mod.compile()
mod.compile(display)
f = open(filename + "c", "wb")
mod.dump(f)
f.close()
......@@ -34,28 +42,30 @@ class Module:
self.source = source
self.code = None
def compile(self):
def compile(self, display=0):
ast = parse(self.source)
root, filename = os.path.split(self.filename)
gen = ModuleCodeGenerator(filename)
walk(ast, gen, 1)
if display:
import pprint
print pprint.pprint(ast)
self.code = gen.getCode()
def dump(self, f):
f.write(self.getPycHeader())
marshal.dump(self.code, f)
MAGIC = (50823 | (ord('\r')<<16) | (ord('\n')<<24))
MAGIC = imp.get_magic()
def getPycHeader(self):
# compile.c uses marshal to write a long directly, with
# calling the interface that would also generate a 1-byte code
# to indicate the type of the value. simplest way to get the
# same effect is to call marshal and then skip the code.
magic = marshal.dumps(self.MAGIC)[1:]
mtime = os.stat(self.filename)[stat.ST_MTIME]
mtime = struct.pack('i', mtime)
return magic + mtime
return self.MAGIC + mtime
class CodeGenerator:
......@@ -63,7 +73,7 @@ class CodeGenerator:
def __init__(self, filename):
## Subclasses must define a constructor that intializes self.graph
## before calling this init function
## before calling this init function, e.g.
## self.graph = pyassem.PyFlowGraph()
self.filename = filename
self.locals = misc.Stack()
......@@ -142,7 +152,6 @@ class CodeGenerator:
def visitLambda(self, node):
self._visitFuncOrLambda(node, isLambda=1)
## self.storeName("<lambda>")
def _visitFuncOrLambda(self, node, isLambda):
gen = FunctionCodeGenerator(node, self.filename, isLambda)
......@@ -180,10 +189,6 @@ class CodeGenerator:
test, suite = node.tests[i]
self.set_lineno(test)
self.visit(test)
## if i == numtests - 1 and not node.else_:
## nextTest = end
## else:
## nextTest = self.newBlock()
nextTest = self.newBlock()
self.emit('JUMP_IF_FALSE', nextTest)
self.nextBlock()
......@@ -304,6 +309,70 @@ class CodeGenerator:
self.emit('POP_TOP')
self.nextBlock(end)
# list comprehensions
__list_count = 0
def visitListComp(self, node):
# XXX would it be easier to transform the AST into the form it
# would have if the list comp were expressed as a series of
# for and if stmts and an explicit append?
self.set_lineno(node)
# setup list
append = "$append%d" % self.__list_count
self.__list_count = self.__list_count + 1
self.emit('BUILD_LIST', 0)
self.emit('DUP_TOP')
self.emit('LOAD_ATTR', 'append')
self.storeName(append)
l = len(node.quals)
stack = []
for i, for_ in zip(range(l), node.quals):
start, anchor = self.visit(for_)
cont = None
for if_ in for_.ifs:
if cont is None:
cont = self.newBlock()
self.visit(if_, cont)
stack.insert(0, (start, cont, anchor))
self.loadName(append)
self.visit(node.expr)
self.emit('CALL_FUNCTION', 1)
self.emit('POP_TOP')
for start, cont, anchor in stack:
if cont:
skip_one = self.newBlock()
self.emit('JUMP_FORWARD', skip_one)
self.nextBlock(cont)
self.emit('POP_TOP')
self.nextBlock(skip_one)
self.emit('JUMP_ABSOLUTE', start)
self.nextBlock(anchor)
self.delName(append)
self.__list_count = self.__list_count - 1
def visitListCompFor(self, node):
self.set_lineno(node)
start = self.newBlock()
anchor = self.newBlock()
self.visit(node.list)
self.visit(ast.Const(0))
self.emit('SET_LINENO', node.lineno)
self.nextBlock(start)
self.emit('FOR_LOOP', anchor)
self.visit(node.assign)
return start, anchor
def visitListCompIf(self, node, branch):
self.set_lineno(node)
self.visit(node.test)
self.emit('JUMP_IF_FALSE', branch)
self.newBlock()
self.emit('POP_TOP')
# exception related
def visitAssert(self, node):
......@@ -397,10 +466,6 @@ class CodeGenerator:
# misc
## def visitStmt(self, node):
## # nothing to do except walk the children
## pass
def visitDiscard(self, node):
self.visit(node.expr)
self.emit('POP_TOP')
......@@ -426,27 +491,32 @@ class CodeGenerator:
def visitImport(self, node):
self.set_lineno(node)
for name, alias in node.names:
self.emit('LOAD_CONST', None)
if VERSION > 1:
self.emit('LOAD_CONST', None)
self.emit('IMPORT_NAME', name)
self._resolveDots(name)
self.storeName(alias or name)
mod = string.split(name, ".")[0]
self.storeName(alias or mod)
def visitFrom(self, node):
self.set_lineno(node)
fromlist = map(lambda (name, alias): name, node.names)
self.emit('LOAD_CONST', tuple(fromlist))
if VERSION > 1:
self.emit('LOAD_CONST', tuple(fromlist))
self.emit('IMPORT_NAME', node.modname)
for name, alias in node.names:
if name == '*':
self.namespace = 0
self.emit('IMPORT_STAR')
# There can only be one name w/ from ... import *
assert len(node.names) == 1
return
if VERSION > 1:
if name == '*':
self.namespace = 0
self.emit('IMPORT_STAR')
# There can only be one name w/ from ... import *
assert len(node.names) == 1
return
else:
self.emit('IMPORT_FROM', name)
self._resolveDots(name)
self.storeName(alias or name)
else:
self.emit('IMPORT_FROM', name)
self._resolveDots(name)
self.storeName(alias or name)
self.emit('POP_TOP')
def _resolveDots(self, name):
......@@ -491,13 +561,85 @@ class CodeGenerator:
print "warning: unexpected flags:", node.flags
print node
def visitAssTuple(self, node):
def _visitAssSequence(self, node, op='UNPACK_SEQUENCE'):
if findOp(node) != 'OP_DELETE':
self.emit('UNPACK_SEQUENCE', len(node.nodes))
self.emit(op, len(node.nodes))
for child in node.nodes:
self.visit(child)
visitAssList = visitAssTuple
if VERSION > 1:
visitAssTuple = _visitAssSequence
visitAssList = _visitAssSequence
else:
def visitAssTuple(self, node):
self._visitAssSequence(node, 'UNPACK_TUPLE')
def visitAssList(self, node):
self._visitAssSequence(node, 'UNPACK_LIST')
# augmented assignment
def visitAugAssign(self, node):
aug_node = wrap_aug(node.node)
self.visit(aug_node, "load")
self.visit(node.expr)
self.emit(self._augmented_opcode[node.op])
self.visit(aug_node, "store")
_augmented_opcode = {
'+=' : 'INPLACE_ADD',
'-=' : 'INPLACE_SUBTRACT',
'*=' : 'INPLACE_MULTIPLY',
'/=' : 'INPLACE_DIVIDE',
'%=' : 'INPLACE_MODULO',
'**=': 'INPLACE_POWER',
'>>=': 'INPLACE_RSHIFT',
'<<=': 'INPLACE_LSHIFT',
'&=' : 'INPLACE_AND',
'^=' : 'INPLACE_XOR',
'|=' : 'INPLACE_OR',
}
def visitAugName(self, node, mode):
if mode == "load":
self.loadName(node.name)
elif mode == "store":
self.storeName(node.name)
def visitAugGetattr(self, node, mode):
if mode == "load":
self.visit(node.expr)
self.emit('DUP_TOP')
self.emit('LOAD_ATTR', node.attrname)
elif mode == "store":
self.emit('ROT_TWO')
self.emit('STORE_ATTR', node.attrname)
def visitAugSlice(self, node, mode):
if mode == "load":
self.visitSlice(node, 1)
elif mode == "store":
slice = 0
if node.lower:
slice = slice | 1
if node.upper:
slice = slice | 2
if slice == 0:
self.emit('ROT_TWO')
elif slice == 3:
self.emit('ROT_FOUR')
else:
self.emit('ROT_THREE')
self.emit('STORE_SLICE+%d' % slice)
def visitAugSubscript(self, node, mode):
if len(node.subs) > 1:
raise SyntaxError, "augmented assignment to tuple is not possible"
if mode == "load":
self.visitSubscript(node, 1)
elif mode == "store":
self.emit('ROT_THREE')
self.emit('STORE_SUBSCR')
def visitExec(self, node):
self.visit(node.expr)
......@@ -533,13 +675,24 @@ class CodeGenerator:
def visitPrint(self, node):
self.set_lineno(node)
if node.dest:
self.visit(node.dest)
for child in node.nodes:
if node.dest:
self.emit('DUP_TOP')
self.visit(child)
self.emit('PRINT_ITEM')
if node.dest:
self.emit('ROT_TWO')
self.emit('PRINT_ITEM_TO')
else:
self.emit('PRINT_ITEM')
def visitPrintnl(self, node):
self.visitPrint(node)
self.emit('PRINT_NEWLINE')
if node.dest:
self.emit('PRINT_NEWLINE_TO')
else:
self.emit('PRINT_NEWLINE')
def visitReturn(self, node):
self.set_lineno(node)
......@@ -548,7 +701,8 @@ class CodeGenerator:
# slice and subscript stuff
def visitSlice(self, node):
def visitSlice(self, node, aug_flag=None):
# aug_flag is used by visitAugSlice
self.visit(node.expr)
slice = 0
if node.lower:
......@@ -557,6 +711,13 @@ class CodeGenerator:
if node.upper:
self.visit(node.upper)
slice = slice | 2
if aug_flag:
if slice == 0:
self.emit('DUP_TOP')
elif slice == 3:
self.emit('DUP_TOPX', 3)
else:
self.emit('DUP_TOPX', 2)
if node.flags == 'OP_APPLY':
self.emit('SLICE+%d' % slice)
elif node.flags == 'OP_ASSIGN':
......@@ -567,10 +728,12 @@ class CodeGenerator:
print "weird slice", node.flags
raise
def visitSubscript(self, node):
def visitSubscript(self, node, aug_flag=None):
self.visit(node.expr)
for sub in node.subs:
self.visit(sub)
if aug_flag:
self.emit('DUP_TOPX', 2)
if len(node.subs) > 1:
self.emit('BUILD_TUPLE', len(node.subs))
if node.flags == 'OP_APPLY':
......@@ -740,7 +903,10 @@ class FunctionCodeGenerator(CodeGenerator):
self.unpackSequence(arg)
def unpackSequence(self, tup):
self.emit('UNPACK_SEQUENCE', len(tup))
if VERSION > 1:
self.emit('UNPACK_SEQUENCE', len(tup))
else:
self.emit('UNPACK_TUPLE', len(tup))
for elt in tup:
if type(elt) == types.TupleType:
self.unpackSequence(elt)
......@@ -765,7 +931,6 @@ class ClassCodeGenerator(CodeGenerator):
self.emit('LOAD_LOCALS')
self.emit('RETURN_VALUE')
def generateArgList(arglist):
"""Generate an arg list marking TupleArgs"""
args = []
......@@ -838,6 +1003,45 @@ class OpFinder:
elif self.op != node.flags:
raise ValueError, "mixed ops in stmt"
class Delegator:
"""Base class to support delegation for augmented assignment nodes
To generator code for augmented assignments, we use the following
wrapper classes. In visitAugAssign, the left-hand expression node
is visited twice. The first time the visit uses the normal method
for that node . The second time the visit uses a different method
that generates the appropriate code to perform the assignment.
These delegator classes wrap the original AST nodes in order to
support the variant visit methods.
"""
def __init__(self, obj):
self.obj = obj
def __getattr__(self, attr):
return getattr(self.obj, attr)
class AugGetattr(Delegator):
pass
class AugName(Delegator):
pass
class AugSlice(Delegator):
pass
class AugSubscript(Delegator):
pass
wrapper = {
ast.Getattr: AugGetattr,
ast.Name: AugName,
ast.Slice: AugSlice,
ast.Subscript: AugSubscript,
}
def wrap_aug(node):
return wrapper[node.__class__](node)
if __name__ == "__main__":
import sys
......
Lib/compiler/transformer.py
View file @ 9c048f9f
#
# Copyright (C) 1997-1998 Greg Stein. All Rights Reserved.
#
# This module is provided under a BSD-ish license. See
# http://www.opensource.org/licenses/bsd-license.html
# and replace OWNER, ORGANIZATION, and YEAR as appropriate.
#
#
# Written by Greg Stein (gstein@lyra.org)
# and Bill Tutt (rassilon@lima.mudlib.org)
# February 1997.
#
# Support for ast.Node subclasses written and other revisions by
# Jeremy Hylton (jeremy@beopen.com)
#
"""Parse tree transformation module.
Transforms Python source code into an abstract syntax tree (AST)
......@@ -24,7 +8,21 @@ parse(buf) -> AST
parseFile(path) -> AST
"""
# Original version written by Greg Stein (gstein@lyra.org)
# and Bill Tutt (rassilon@lima.mudlib.org)
# February 1997.
#
# Modifications and improvements for Python 2.0 by Jeremy Hylton and
# Mark Hammond
# Portions of this file are:
# Copyright (C) 1997-1998 Greg Stein. All Rights Reserved.
#
# This module is provided under a BSD-ish license. See
# http://www.opensource.org/licenses/bsd-license.html
# and replace OWNER, ORGANIZATION, and YEAR as appropriate.
# The output tree has the following nodes:
#
# Source Python line #'s appear at the end of each of all of these nodes
......@@ -49,9 +47,10 @@ parseFile(path) -> AST
# tryexcept: trySuiteNode, [ (exprNode, assgnNode, suiteNode), ... ], elseNode
# return: valueNode
# const: value
# print: [ node1, ..., nodeN ]
# printnl: [ node1, ..., nodeN ]
# print: [ node1, ..., nodeN ] [, dest]
# printnl: [ node1, ..., nodeN ] [, dest]
# discard: exprNode
# augassign: node, op, expr
# assign: [ node1, ..., nodeN ], exprNode
# ass_tuple: [ node1, ..., nodeN ]
# ass_list: [ node1, ..., nodeN ]
......@@ -97,12 +96,12 @@ parseFile(path) -> AST
import ast
import parser
# Care must be taken to use only symbols and tokens defined in Python
# 1.5.2 for code branches executed in 1.5.2
import symbol
import token
import string
import pprint
error = 'walker.error'
from consts import CO_VARARGS, CO_VARKEYWORDS
......@@ -328,27 +327,44 @@ class Transformer:
#
def expr_stmt(self, nodelist):
# testlist ('=' testlist)*
# augassign testlist | testlist ('=' testlist)*
exprNode = self.com_node(nodelist[-1])
if len(nodelist) == 1:
return Node('discard', exprNode)
nodes = [ ]
for i in range(0, len(nodelist) - 2, 2):
nodes.append(self.com_assign(nodelist[i], OP_ASSIGN))
n = Node('assign', nodes, exprNode)
n.lineno = nodelist[1][2]
if nodelist[1][0] == token.EQUAL:
nodes = [ ]
for i in range(0, len(nodelist) - 2, 2):
nodes.append(self.com_assign(nodelist[i], OP_ASSIGN))
n = Node('assign', nodes, exprNode)
n.lineno = nodelist[1][2]
else:
lval = self.com_augassign(nodelist[0])
op = self.com_augassign_op(nodelist[1])
n = Node('augassign', lval, op[1], exprNode)
n.lineno = op[2]
return n
def print_stmt(self, nodelist):
# print: (test ',')* [test]
# print ([ test (',' test)* [','] ] | '>>' test [ (',' test)+ [','] ])
items = [ ]
for i in range(1, len(nodelist), 2):
if len(nodelist) == 1:
start = 1
dest = None
elif nodelist[1][0] == token.RIGHTSHIFT:
assert len(nodelist) == 3 \
or nodelist[3][0] == token.COMMA
dest = self.com_node(nodelist[2])
start = 4
else:
dest = None
start = 1
for i in range(start, len(nodelist), 2):
items.append(self.com_node(nodelist[i]))
if nodelist[-1][0] == token.COMMA:
n = Node('print', items)
n = Node('print', items, dest)
n.lineno = nodelist[0][2]
return n
n = Node('printnl', items)
n = Node('printnl', items, dest)
n.lineno = nodelist[0][2]
return n
......@@ -405,17 +421,24 @@ class Transformer:
# import_stmt: 'import' dotted_as_name (',' dotted_as_name)* |
# from: 'from' dotted_name 'import'
# ('*' | import_as_name (',' import_as_name)*)
names = []
is_as = 0
if nodelist[0][1] == 'from':
for i in range(3, len(nodelist), 2):
names.append(self.com_import_as_name(nodelist[i][1]))
names = []
if nodelist[3][0] == token.NAME:
for i in range(3, len(nodelist), 2):
names.append((nodelist[i][1], None))
else:
for i in range(3, len(nodelist), 2):
names.append(self.com_import_as_name(nodelist[i][1]))
n = Node('from', self.com_dotted_name(nodelist[1]), names)
n.lineno = nodelist[0][2]
return n
for i in range(1, len(nodelist), 2):
names.append(self.com_dotted_as_name(nodelist[i]))
if nodelist[1][0] == symbol.dotted_name:
names = [(self.com_dotted_name(nodelist[1][1:]), None)]
else:
names = []
for i in range(1, len(nodelist), 2):
names.append(self.com_dotted_as_name(nodelist[i]))
n = Node('import', names)
n.lineno = nodelist[0][2]
return n
......@@ -737,7 +760,7 @@ class Transformer:
return Node('discard', Node('const', None))
if node[0] not in _legal_node_types:
raise error, 'illegal node passed to com_node: %s' % node[0]
raise error, 'illegal node passed to com_node: %s' % `node`
# print "dispatch", self._dispatch[node[0]].__name__, node
return self._dispatch[node[0]](node[1:])
......@@ -818,11 +841,14 @@ class Transformer:
def com_dotted_as_name(self, node):
dot = self.com_dotted_name(node[1])
if len(node) == 2:
if len(node) <= 2:
return dot, None
assert node[2][1] == 'as'
assert node[3][0] == token.NAME
return dot, node[3][1]
if node[0] == symbol.dotted_name:
pass
else:
assert node[2][1] == 'as'
assert node[3][0] == token.NAME
return dot, node[3][1]
def com_import_as_name(self, node):
if node == '*':
......@@ -872,6 +898,20 @@ class Transformer:
n.lineno = nodelist[0][2]
return n
def com_augassign_op(self, node):
assert node[0] == symbol.augassign
return node[1]
def com_augassign(self, node):
"""Return node suitable for lvalue of augmented assignment
Names, slices, and attributes are the only allowable nodes.
"""
l = self.com_node(node)
if l[0] in ('name', 'slice', 'subscript', 'getattr'):
return l
raise SyntaxError, "can't assign to %s" % l[0]
def com_assign(self, node, assigning):
# return a node suitable for use as an "lvalue"
# loop to avoid trivial recursion
......@@ -955,7 +995,6 @@ class Transformer:
return Node(type, items)
def com_stmt(self, node):
#pprint.pprint(node)
result = self.com_node(node)
try:
result[0]
......@@ -976,17 +1015,71 @@ class Transformer:
else:
stmts.append(result)
def com_list_constructor(self, nodelist):
values = [ ]
for i in range(1, len(nodelist), 2):
values.append(self.com_node(nodelist[i]))
return Node('list', values)
if hasattr(symbol, 'list_for'):
def com_list_constructor(self, nodelist):
# listmaker: test ( list_for | (',' test)* [','] )
values = [ ]
for i in range(1, len(nodelist)):
if nodelist[i][0] == symbol.list_for:
assert len(nodelist[i:]) == 1
return self.com_list_comprehension(values[0],
nodelist[i])
elif nodelist[i][0] == token.COMMA:
continue
values.append(self.com_node(nodelist[i]))
return Node('list', values)
def com_list_comprehension(self, expr, node):
# list_iter: list_for | list_if
# list_for: 'for' exprlist 'in' testlist [list_iter]
# list_if: 'if' test [list_iter]
lineno = node[1][2]
fors = []
while node:
if node[1][1] == 'for':
assignNode = self.com_assign(node[2], OP_ASSIGN)
listNode = self.com_node(node[4])
newfor = Node('listcomp_for', assignNode,
listNode, [])
newfor.lineno = node[1][2]
fors.append(newfor)
if len(node) == 5:
node = None
else:
node = self.com_list_iter(node[5])
elif node[1][1] == 'if':
test = self.com_node(node[2])
newif = Node('listcomp_if', test)
newif.lineno = node[1][2]
newfor.ifs.append(newif)
if len(node) == 3:
node = None
else:
node = self.com_list_iter(node[3])
else:
raise SyntaxError, \
("unexpected list comprehension element: %s %d"
% (node, lineno))
n = Node('listcomp', expr, fors)
n.lineno = lineno
return n
def com_list_iter(self, node):
assert node[0] == symbol.list_iter
return node[1]
else:
def com_list_constructor(self, nodelist):
values = [ ]
for i in range(1, len(nodelist), 2):
values.append(self.com_node(nodelist[i]))
return Node('list', values)
def com_dictmaker(self, nodelist):
# dictmaker: test ':' test (',' test ':' value)* [',']
items = [ ]
for i in range(1, len(nodelist), 4):
items.append((self.com_node(nodelist[i]), self.com_node(nodelist[i+2])))
items.append((self.com_node(nodelist[i]),
self.com_node(nodelist[i+2])))
return Node('dict', items)
def com_apply_trailer(self, primaryNode, nodelist):
......@@ -1250,3 +1343,21 @@ _assign_types = [
symbol.term,
symbol.factor,
]
import types
_names = {}
for k, v in symbol.sym_name.items():
_names[k] = v
for k, v in token.tok_name.items():
_names[k] = v
def debug_tree(tree):
l = []
for elt in tree:
if type(elt) == types.IntType:
l.append(_names.get(elt, elt))
elif type(elt) == types.StringType:
l.append(elt)
else:
l.append(debug_tree(elt))
return l
Tools/compiler/compiler/ast.py
View file @ 9c048f9f
......@@ -279,22 +279,24 @@ class Const(Node):
class Print(Node):
nodes['print'] = 'Print'
def __init__(self, nodes):
def __init__(self, nodes, dest):
self.nodes = nodes
self._children = ('print', nodes)
self.dest = dest
self._children = ('print', nodes, dest)
def __repr__(self):
return "Print(%s)" % self._children[1:]
return "Print(%s, %s)" % (self._children[1:-1], self._children[-1])
class Printnl(Node):
nodes['printnl'] = 'Printnl'
def __init__(self, nodes):
def __init__(self, nodes, dest):
self.nodes = nodes
self._children = ('printnl', nodes)
self.dest = dest
self._children = ('printnl', nodes, dest)
def __repr__(self):
return "Printnl(%s)" % self._children[1:]
return "Printnl(%s, %s)" % (self._children[1:-1], self._children[-1])
class Discard(Node):
nodes['discard'] = 'Discard'
......@@ -306,6 +308,18 @@ class Discard(Node):
def __repr__(self):
return "Discard(%s)" % self._children[1:]
class AugAssign(Node):
nodes['augassign'] = 'AugAssign'
def __init__(self, node, op, expr):
self.node = node
self.op = op
self.expr = expr
self._children = ('augassign', node, op, expr)
def __repr__(self):
return "AugAssign(%s)" % str(self._children[1:])
class Assign(Node):
nodes['assign'] = 'Assign'
......@@ -360,6 +374,41 @@ class AssAttr(Node):
def __repr__(self):
return "AssAttr(%s,%s,%s)" % self._children[1:]
class ListComp(Node):
nodes['listcomp'] = 'ListComp'
def __init__(self, expr, quals):
self.expr = expr
self.quals = quals
self._children = ('listcomp', expr, quals)
def __repr__(self):
return "ListComp(%s, %s)" % self._children[1:]
class ListCompFor(Node):
nodes['listcomp_for'] = 'ListCompFor'
# transformer fills in ifs after node is created
def __init__(self, assign, list, ifs):
self.assign = assign
self.list = list
self.ifs = ifs
self._children = ('listcomp_for', assign, list, ifs)
def __repr__(self):
return "ListCompFor(%s, %s, %s)" % self._children[1:]
class ListCompIf(Node):
nodes['listcomp_if'] = 'ListCompIf'
def __init__(self, test):
self.test = test
self._children = ('listcomp_if', test)
def __repr__(self):
return "ListCompIf(%s)" % self._children[1:]
class List(Node):
nodes['list'] = 'List'
......
Tools/compiler/compiler/pyassem.py
View file @ 9c048f9f
......@@ -253,8 +253,14 @@ class PyFlowGraph(FlowGraph):
def _lookupName(self, name, list):
"""Return index of name in list, appending if necessary"""
if name in list:
i = list.index(name)
found = None
t = type(name)
for i in range(len(list)):
# must do a comparison on type first to prevent UnicodeErrors
if t == type(list[i]) and list[i] == name:
found = 1
break
if found:
# this is cheap, but incorrect in some cases, e.g 2 vs. 2L
if type(name) == type(list[i]):
return i
......
Tools/compiler/compiler/pycodegen.py
View file @ 9c048f9f
import imp
import os
import marshal
import stat
import string
import struct
import sys
import types
from cStringIO import StringIO
......@@ -10,6 +12,12 @@ from compiler import ast, parse, walk
from compiler import pyassem, misc
from compiler.pyassem import CO_VARARGS, CO_VARKEYWORDS, CO_NEWLOCALS, TupleArg
# Do we have Python 1.x or Python 2.x?
try:
VERSION = sys.version_info[0]
except AttributeError:
VERSION = 1
callfunc_opcode_info = {
# (Have *args, Have **args) : opcode
(0,0) : "CALL_FUNCTION",
......@@ -18,12 +26,12 @@ callfunc_opcode_info = {
(1,1) : "CALL_FUNCTION_VAR_KW",
}
def compile(filename):
def compile(filename, display=0):
f = open(filename)
buf = f.read()
f.close()
mod = Module(buf, filename)
mod.compile()
mod.compile(display)
f = open(filename + "c", "wb")
mod.dump(f)
f.close()
......@@ -34,28 +42,30 @@ class Module:
self.source = source
self.code = None
def compile(self):
def compile(self, display=0):
ast = parse(self.source)
root, filename = os.path.split(self.filename)
gen = ModuleCodeGenerator(filename)
walk(ast, gen, 1)
if display:
import pprint
print pprint.pprint(ast)
self.code = gen.getCode()
def dump(self, f):
f.write(self.getPycHeader())
marshal.dump(self.code, f)
MAGIC = (50823 | (ord('\r')<<16) | (ord('\n')<<24))
MAGIC = imp.get_magic()
def getPycHeader(self):
# compile.c uses marshal to write a long directly, with
# calling the interface that would also generate a 1-byte code
# to indicate the type of the value. simplest way to get the
# same effect is to call marshal and then skip the code.
magic = marshal.dumps(self.MAGIC)[1:]
mtime = os.stat(self.filename)[stat.ST_MTIME]
mtime = struct.pack('i', mtime)
return magic + mtime
return self.MAGIC + mtime
class CodeGenerator:
......@@ -63,7 +73,7 @@ class CodeGenerator:
def __init__(self, filename):
## Subclasses must define a constructor that intializes self.graph
## before calling this init function
## before calling this init function, e.g.
## self.graph = pyassem.PyFlowGraph()
self.filename = filename
self.locals = misc.Stack()
......@@ -142,7 +152,6 @@ class CodeGenerator:
def visitLambda(self, node):
self._visitFuncOrLambda(node, isLambda=1)
## self.storeName("<lambda>")
def _visitFuncOrLambda(self, node, isLambda):
gen = FunctionCodeGenerator(node, self.filename, isLambda)
......@@ -180,10 +189,6 @@ class CodeGenerator:
test, suite = node.tests[i]
self.set_lineno(test)
self.visit(test)
## if i == numtests - 1 and not node.else_:
## nextTest = end
## else:
## nextTest = self.newBlock()
nextTest = self.newBlock()
self.emit('JUMP_IF_FALSE', nextTest)
self.nextBlock()
......@@ -304,6 +309,70 @@ class CodeGenerator:
self.emit('POP_TOP')
self.nextBlock(end)
# list comprehensions
__list_count = 0
def visitListComp(self, node):
# XXX would it be easier to transform the AST into the form it
# would have if the list comp were expressed as a series of
# for and if stmts and an explicit append?
self.set_lineno(node)
# setup list
append = "$append%d" % self.__list_count
self.__list_count = self.__list_count + 1
self.emit('BUILD_LIST', 0)
self.emit('DUP_TOP')
self.emit('LOAD_ATTR', 'append')
self.storeName(append)
l = len(node.quals)
stack = []
for i, for_ in zip(range(l), node.quals):
start, anchor = self.visit(for_)
cont = None
for if_ in for_.ifs:
if cont is None:
cont = self.newBlock()
self.visit(if_, cont)
stack.insert(0, (start, cont, anchor))
self.loadName(append)
self.visit(node.expr)
self.emit('CALL_FUNCTION', 1)
self.emit('POP_TOP')
for start, cont, anchor in stack:
if cont:
skip_one = self.newBlock()
self.emit('JUMP_FORWARD', skip_one)
self.nextBlock(cont)
self.emit('POP_TOP')
self.nextBlock(skip_one)
self.emit('JUMP_ABSOLUTE', start)
self.nextBlock(anchor)
self.delName(append)
self.__list_count = self.__list_count - 1
def visitListCompFor(self, node):
self.set_lineno(node)
start = self.newBlock()
anchor = self.newBlock()
self.visit(node.list)
self.visit(ast.Const(0))
self.emit('SET_LINENO', node.lineno)
self.nextBlock(start)
self.emit('FOR_LOOP', anchor)
self.visit(node.assign)
return start, anchor
def visitListCompIf(self, node, branch):
self.set_lineno(node)
self.visit(node.test)
self.emit('JUMP_IF_FALSE', branch)
self.newBlock()
self.emit('POP_TOP')
# exception related
def visitAssert(self, node):
......@@ -397,10 +466,6 @@ class CodeGenerator:
# misc
## def visitStmt(self, node):
## # nothing to do except walk the children
## pass
def visitDiscard(self, node):
self.visit(node.expr)
self.emit('POP_TOP')
......@@ -426,27 +491,32 @@ class CodeGenerator:
def visitImport(self, node):
self.set_lineno(node)
for name, alias in node.names:
self.emit('LOAD_CONST', None)
if VERSION > 1:
self.emit('LOAD_CONST', None)
self.emit('IMPORT_NAME', name)
self._resolveDots(name)
self.storeName(alias or name)
mod = string.split(name, ".")[0]
self.storeName(alias or mod)
def visitFrom(self, node):
self.set_lineno(node)
fromlist = map(lambda (name, alias): name, node.names)
self.emit('LOAD_CONST', tuple(fromlist))
if VERSION > 1:
self.emit('LOAD_CONST', tuple(fromlist))
self.emit('IMPORT_NAME', node.modname)
for name, alias in node.names:
if name == '*':
self.namespace = 0
self.emit('IMPORT_STAR')
# There can only be one name w/ from ... import *
assert len(node.names) == 1
return
if VERSION > 1:
if name == '*':
self.namespace = 0
self.emit('IMPORT_STAR')
# There can only be one name w/ from ... import *
assert len(node.names) == 1
return
else:
self.emit('IMPORT_FROM', name)
self._resolveDots(name)
self.storeName(alias or name)
else:
self.emit('IMPORT_FROM', name)
self._resolveDots(name)
self.storeName(alias or name)
self.emit('POP_TOP')
def _resolveDots(self, name):
......@@ -491,13 +561,85 @@ class CodeGenerator:
print "warning: unexpected flags:", node.flags
print node
def visitAssTuple(self, node):
def _visitAssSequence(self, node, op='UNPACK_SEQUENCE'):
if findOp(node) != 'OP_DELETE':
self.emit('UNPACK_SEQUENCE', len(node.nodes))
self.emit(op, len(node.nodes))
for child in node.nodes:
self.visit(child)
visitAssList = visitAssTuple
if VERSION > 1:
visitAssTuple = _visitAssSequence
visitAssList = _visitAssSequence
else:
def visitAssTuple(self, node):
self._visitAssSequence(node, 'UNPACK_TUPLE')
def visitAssList(self, node):
self._visitAssSequence(node, 'UNPACK_LIST')
# augmented assignment
def visitAugAssign(self, node):
aug_node = wrap_aug(node.node)
self.visit(aug_node, "load")
self.visit(node.expr)
self.emit(self._augmented_opcode[node.op])
self.visit(aug_node, "store")
_augmented_opcode = {
'+=' : 'INPLACE_ADD',
'-=' : 'INPLACE_SUBTRACT',
'*=' : 'INPLACE_MULTIPLY',
'/=' : 'INPLACE_DIVIDE',
'%=' : 'INPLACE_MODULO',
'**=': 'INPLACE_POWER',
'>>=': 'INPLACE_RSHIFT',
'<<=': 'INPLACE_LSHIFT',
'&=' : 'INPLACE_AND',
'^=' : 'INPLACE_XOR',
'|=' : 'INPLACE_OR',
}
def visitAugName(self, node, mode):
if mode == "load":
self.loadName(node.name)
elif mode == "store":
self.storeName(node.name)
def visitAugGetattr(self, node, mode):
if mode == "load":
self.visit(node.expr)
self.emit('DUP_TOP')
self.emit('LOAD_ATTR', node.attrname)
elif mode == "store":
self.emit('ROT_TWO')
self.emit('STORE_ATTR', node.attrname)
def visitAugSlice(self, node, mode):
if mode == "load":
self.visitSlice(node, 1)
elif mode == "store":
slice = 0
if node.lower:
slice = slice | 1
if node.upper:
slice = slice | 2
if slice == 0:
self.emit('ROT_TWO')
elif slice == 3:
self.emit('ROT_FOUR')
else:
self.emit('ROT_THREE')
self.emit('STORE_SLICE+%d' % slice)
def visitAugSubscript(self, node, mode):
if len(node.subs) > 1:
raise SyntaxError, "augmented assignment to tuple is not possible"
if mode == "load":
self.visitSubscript(node, 1)
elif mode == "store":
self.emit('ROT_THREE')
self.emit('STORE_SUBSCR')
def visitExec(self, node):
self.visit(node.expr)
......@@ -533,13 +675,24 @@ class CodeGenerator:
def visitPrint(self, node):
self.set_lineno(node)
if node.dest:
self.visit(node.dest)
for child in node.nodes:
if node.dest:
self.emit('DUP_TOP')
self.visit(child)
self.emit('PRINT_ITEM')
if node.dest:
self.emit('ROT_TWO')
self.emit('PRINT_ITEM_TO')
else:
self.emit('PRINT_ITEM')
def visitPrintnl(self, node):
self.visitPrint(node)
self.emit('PRINT_NEWLINE')
if node.dest:
self.emit('PRINT_NEWLINE_TO')
else:
self.emit('PRINT_NEWLINE')
def visitReturn(self, node):
self.set_lineno(node)
......@@ -548,7 +701,8 @@ class CodeGenerator:
# slice and subscript stuff
def visitSlice(self, node):
def visitSlice(self, node, aug_flag=None):
# aug_flag is used by visitAugSlice
self.visit(node.expr)
slice = 0
if node.lower:
......@@ -557,6 +711,13 @@ class CodeGenerator:
if node.upper:
self.visit(node.upper)
slice = slice | 2
if aug_flag:
if slice == 0:
self.emit('DUP_TOP')
elif slice == 3:
self.emit('DUP_TOPX', 3)
else:
self.emit('DUP_TOPX', 2)
if node.flags == 'OP_APPLY':
self.emit('SLICE+%d' % slice)
elif node.flags == 'OP_ASSIGN':
......@@ -567,10 +728,12 @@ class CodeGenerator:
print "weird slice", node.flags
raise
def visitSubscript(self, node):
def visitSubscript(self, node, aug_flag=None):
self.visit(node.expr)
for sub in node.subs:
self.visit(sub)
if aug_flag:
self.emit('DUP_TOPX', 2)
if len(node.subs) > 1:
self.emit('BUILD_TUPLE', len(node.subs))
if node.flags == 'OP_APPLY':
......@@ -740,7 +903,10 @@ class FunctionCodeGenerator(CodeGenerator):
self.unpackSequence(arg)
def unpackSequence(self, tup):
self.emit('UNPACK_SEQUENCE', len(tup))
if VERSION > 1:
self.emit('UNPACK_SEQUENCE', len(tup))
else:
self.emit('UNPACK_TUPLE', len(tup))
for elt in tup:
if type(elt) == types.TupleType:
self.unpackSequence(elt)
......@@ -765,7 +931,6 @@ class ClassCodeGenerator(CodeGenerator):
self.emit('LOAD_LOCALS')
self.emit('RETURN_VALUE')
def generateArgList(arglist):
"""Generate an arg list marking TupleArgs"""
args = []
......@@ -838,6 +1003,45 @@ class OpFinder:
elif self.op != node.flags:
raise ValueError, "mixed ops in stmt"
class Delegator:
"""Base class to support delegation for augmented assignment nodes
To generator code for augmented assignments, we use the following
wrapper classes. In visitAugAssign, the left-hand expression node
is visited twice. The first time the visit uses the normal method
for that node . The second time the visit uses a different method
that generates the appropriate code to perform the assignment.
These delegator classes wrap the original AST nodes in order to
support the variant visit methods.
"""
def __init__(self, obj):
self.obj = obj
def __getattr__(self, attr):
return getattr(self.obj, attr)
class AugGetattr(Delegator):
pass
class AugName(Delegator):
pass
class AugSlice(Delegator):
pass
class AugSubscript(Delegator):
pass
wrapper = {
ast.Getattr: AugGetattr,
ast.Name: AugName,
ast.Slice: AugSlice,
ast.Subscript: AugSubscript,
}
def wrap_aug(node):
return wrapper[node.__class__](node)
if __name__ == "__main__":
import sys
......
Tools/compiler/compiler/transformer.py
View file @ 9c048f9f
#
# Copyright (C) 1997-1998 Greg Stein. All Rights Reserved.
#
# This module is provided under a BSD-ish license. See
# http://www.opensource.org/licenses/bsd-license.html
# and replace OWNER, ORGANIZATION, and YEAR as appropriate.
#
#
# Written by Greg Stein (gstein@lyra.org)
# and Bill Tutt (rassilon@lima.mudlib.org)
# February 1997.
#
# Support for ast.Node subclasses written and other revisions by
# Jeremy Hylton (jeremy@beopen.com)
#
"""Parse tree transformation module.
Transforms Python source code into an abstract syntax tree (AST)
......@@ -24,7 +8,21 @@ parse(buf) -> AST
parseFile(path) -> AST
"""
# Original version written by Greg Stein (gstein@lyra.org)
# and Bill Tutt (rassilon@lima.mudlib.org)
# February 1997.
#
# Modifications and improvements for Python 2.0 by Jeremy Hylton and
# Mark Hammond
# Portions of this file are:
# Copyright (C) 1997-1998 Greg Stein. All Rights Reserved.
#
# This module is provided under a BSD-ish license. See
# http://www.opensource.org/licenses/bsd-license.html
# and replace OWNER, ORGANIZATION, and YEAR as appropriate.
# The output tree has the following nodes:
#
# Source Python line #'s appear at the end of each of all of these nodes
......@@ -49,9 +47,10 @@ parseFile(path) -> AST
# tryexcept: trySuiteNode, [ (exprNode, assgnNode, suiteNode), ... ], elseNode
# return: valueNode
# const: value
# print: [ node1, ..., nodeN ]
# printnl: [ node1, ..., nodeN ]
# print: [ node1, ..., nodeN ] [, dest]
# printnl: [ node1, ..., nodeN ] [, dest]
# discard: exprNode
# augassign: node, op, expr
# assign: [ node1, ..., nodeN ], exprNode
# ass_tuple: [ node1, ..., nodeN ]
# ass_list: [ node1, ..., nodeN ]
......@@ -97,12 +96,12 @@ parseFile(path) -> AST
import ast
import parser
# Care must be taken to use only symbols and tokens defined in Python
# 1.5.2 for code branches executed in 1.5.2
import symbol
import token
import string
import pprint
error = 'walker.error'
from consts import CO_VARARGS, CO_VARKEYWORDS
......@@ -328,27 +327,44 @@ class Transformer:
#
def expr_stmt(self, nodelist):
# testlist ('=' testlist)*
# augassign testlist | testlist ('=' testlist)*
exprNode = self.com_node(nodelist[-1])
if len(nodelist) == 1:
return Node('discard', exprNode)
nodes = [ ]
for i in range(0, len(nodelist) - 2, 2):
nodes.append(self.com_assign(nodelist[i], OP_ASSIGN))
n = Node('assign', nodes, exprNode)
n.lineno = nodelist[1][2]
if nodelist[1][0] == token.EQUAL:
nodes = [ ]
for i in range(0, len(nodelist) - 2, 2):
nodes.append(self.com_assign(nodelist[i], OP_ASSIGN))
n = Node('assign', nodes, exprNode)
n.lineno = nodelist[1][2]
else:
lval = self.com_augassign(nodelist[0])
op = self.com_augassign_op(nodelist[1])
n = Node('augassign', lval, op[1], exprNode)
n.lineno = op[2]
return n
def print_stmt(self, nodelist):
# print: (test ',')* [test]
# print ([ test (',' test)* [','] ] | '>>' test [ (',' test)+ [','] ])
items = [ ]
for i in range(1, len(nodelist), 2):
if len(nodelist) == 1:
start = 1
dest = None
elif nodelist[1][0] == token.RIGHTSHIFT:
assert len(nodelist) == 3 \
or nodelist[3][0] == token.COMMA
dest = self.com_node(nodelist[2])
start = 4
else:
dest = None
start = 1
for i in range(start, len(nodelist), 2):
items.append(self.com_node(nodelist[i]))
if nodelist[-1][0] == token.COMMA:
n = Node('print', items)
n = Node('print', items, dest)
n.lineno = nodelist[0][2]
return n
n = Node('printnl', items)
n = Node('printnl', items, dest)
n.lineno = nodelist[0][2]
return n
......@@ -405,17 +421,24 @@ class Transformer:
# import_stmt: 'import' dotted_as_name (',' dotted_as_name)* |
# from: 'from' dotted_name 'import'
# ('*' | import_as_name (',' import_as_name)*)
names = []
is_as = 0
if nodelist[0][1] == 'from':
for i in range(3, len(nodelist), 2):
names.append(self.com_import_as_name(nodelist[i][1]))
names = []
if nodelist[3][0] == token.NAME:
for i in range(3, len(nodelist), 2):
names.append((nodelist[i][1], None))
else:
for i in range(3, len(nodelist), 2):
names.append(self.com_import_as_name(nodelist[i][1]))
n = Node('from', self.com_dotted_name(nodelist[1]), names)
n.lineno = nodelist[0][2]
return n
for i in range(1, len(nodelist), 2):
names.append(self.com_dotted_as_name(nodelist[i]))
if nodelist[1][0] == symbol.dotted_name:
names = [(self.com_dotted_name(nodelist[1][1:]), None)]
else:
names = []
for i in range(1, len(nodelist), 2):
names.append(self.com_dotted_as_name(nodelist[i]))
n = Node('import', names)
n.lineno = nodelist[0][2]
return n
......@@ -737,7 +760,7 @@ class Transformer:
return Node('discard', Node('const', None))
if node[0] not in _legal_node_types:
raise error, 'illegal node passed to com_node: %s' % node[0]
raise error, 'illegal node passed to com_node: %s' % `node`
# print "dispatch", self._dispatch[node[0]].__name__, node
return self._dispatch[node[0]](node[1:])
......@@ -818,11 +841,14 @@ class Transformer:
def com_dotted_as_name(self, node):
dot = self.com_dotted_name(node[1])
if len(node) == 2:
if len(node) <= 2:
return dot, None
assert node[2][1] == 'as'
assert node[3][0] == token.NAME
return dot, node[3][1]
if node[0] == symbol.dotted_name:
pass
else:
assert node[2][1] == 'as'
assert node[3][0] == token.NAME
return dot, node[3][1]
def com_import_as_name(self, node):
if node == '*':
......@@ -872,6 +898,20 @@ class Transformer:
n.lineno = nodelist[0][2]
return n
def com_augassign_op(self, node):
assert node[0] == symbol.augassign
return node[1]
def com_augassign(self, node):
"""Return node suitable for lvalue of augmented assignment
Names, slices, and attributes are the only allowable nodes.
"""
l = self.com_node(node)
if l[0] in ('name', 'slice', 'subscript', 'getattr'):
return l
raise SyntaxError, "can't assign to %s" % l[0]
def com_assign(self, node, assigning):
# return a node suitable for use as an "lvalue"
# loop to avoid trivial recursion
......@@ -955,7 +995,6 @@ class Transformer:
return Node(type, items)
def com_stmt(self, node):
#pprint.pprint(node)
result = self.com_node(node)
try:
result[0]
......@@ -976,17 +1015,71 @@ class Transformer:
else:
stmts.append(result)
def com_list_constructor(self, nodelist):
values = [ ]
for i in range(1, len(nodelist), 2):
values.append(self.com_node(nodelist[i]))
return Node('list', values)
if hasattr(symbol, 'list_for'):
def com_list_constructor(self, nodelist):
# listmaker: test ( list_for | (',' test)* [','] )
values = [ ]
for i in range(1, len(nodelist)):
if nodelist[i][0] == symbol.list_for:
assert len(nodelist[i:]) == 1
return self.com_list_comprehension(values[0],
nodelist[i])
elif nodelist[i][0] == token.COMMA:
continue
values.append(self.com_node(nodelist[i]))
return Node('list', values)
def com_list_comprehension(self, expr, node):
# list_iter: list_for | list_if
# list_for: 'for' exprlist 'in' testlist [list_iter]
# list_if: 'if' test [list_iter]
lineno = node[1][2]
fors = []
while node:
if node[1][1] == 'for':
assignNode = self.com_assign(node[2], OP_ASSIGN)
listNode = self.com_node(node[4])
newfor = Node('listcomp_for', assignNode,
listNode, [])
newfor.lineno = node[1][2]
fors.append(newfor)
if len(node) == 5:
node = None
else:
node = self.com_list_iter(node[5])
elif node[1][1] == 'if':
test = self.com_node(node[2])
newif = Node('listcomp_if', test)
newif.lineno = node[1][2]
newfor.ifs.append(newif)
if len(node) == 3:
node = None
else:
node = self.com_list_iter(node[3])
else:
raise SyntaxError, \
("unexpected list comprehension element: %s %d"
% (node, lineno))
n = Node('listcomp', expr, fors)
n.lineno = lineno
return n
def com_list_iter(self, node):
assert node[0] == symbol.list_iter
return node[1]
else:
def com_list_constructor(self, nodelist):
values = [ ]
for i in range(1, len(nodelist), 2):
values.append(self.com_node(nodelist[i]))
return Node('list', values)
def com_dictmaker(self, nodelist):
# dictmaker: test ':' test (',' test ':' value)* [',']
items = [ ]
for i in range(1, len(nodelist), 4):
items.append((self.com_node(nodelist[i]), self.com_node(nodelist[i+2])))
items.append((self.com_node(nodelist[i]),
self.com_node(nodelist[i+2])))
return Node('dict', items)
def com_apply_trailer(self, primaryNode, nodelist):
......@@ -1250,3 +1343,21 @@ _assign_types = [
symbol.term,
symbol.factor,
]
import types
_names = {}
for k, v in symbol.sym_name.items():
_names[k] = v
for k, v in token.tok_name.items():
_names[k] = v
def debug_tree(tree):
l = []
for elt in tree:
if type(elt) == types.IntType:
l.append(_names.get(elt, elt))
elif type(elt) == types.StringType:
l.append(elt)
else:
l.append(debug_tree(elt))
return l
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7