Synced up with Python's release21-maint branch, with a bugfix for FOR_LOOP

stack size computation and relative rather than absolute imports.

lib/python/RestrictedPython/compiler/ast.py

@@ -40,8 +40,6 @@ class Node:
         return flatten(self._getChildren())
     def asList(self):
         return tuple(asList(self.getChildren()))
-    def getChildNodes(self):
-        return [n for n in self.getChildren() if isinstance(n, Node)]
 
 class EmptyNode(Node):
     def __init__(self):
@@ -190,7 +188,7 @@ class Sub(Node):
     def _getChildren(self):
         return self.left, self.right
     def __repr__(self):
-        return "Sub((%s, %s))" % (repr(self.left), repr(self.right))
+        return "Sub(%s, %s)" % (repr(self.left), repr(self.right))
 
 class ListCompIf(Node):
     nodes["listcompif"] = "ListCompIf"
@@ -209,7 +207,7 @@ class Div(Node):
     def _getChildren(self):
         return self.left, self.right
     def __repr__(self):
-        return "Div((%s, %s))" % (repr(self.left), repr(self.right))
+        return "Div(%s, %s)" % (repr(self.left), repr(self.right))
 
 class Discard(Node):
     nodes["discard"] = "Discard"
@@ -237,7 +235,7 @@ class RightShift(Node):
     def _getChildren(self):
         return self.left, self.right
     def __repr__(self):
-        return "RightShift((%s, %s))" % (repr(self.left), repr(self.right))
+        return "RightShift(%s, %s)" % (repr(self.left), repr(self.right))
 
 class Continue(Node):
     nodes["continue"] = "Continue"
@@ -277,7 +275,7 @@ class LeftShift(Node):
     def _getChildren(self):
         return self.left, self.right
     def __repr__(self):
-        return "LeftShift((%s, %s))" % (repr(self.left), repr(self.right))
+        return "LeftShift(%s, %s)" % (repr(self.left), repr(self.right))
 
 class Mul(Node):
     nodes["mul"] = "Mul"
@@ -287,16 +285,7 @@ class Mul(Node):
     def _getChildren(self):
         return self.left, self.right
     def __repr__(self):
-        return "Mul((%s, %s))" % (repr(self.left), repr(self.right))
-
-class Yield(Node):
-    nodes["yield"] = "Yield"
-    def __init__(self, value):
-        self.value = value
-    def _getChildren(self):
-        return self.value,
-    def __repr__(self):
-        return "Yield(%s)" % (repr(self.value),)
+        return "Mul(%s, %s)" % (repr(self.left), repr(self.right))
 
 class List(Node):
     nodes["list"] = "List"
@@ -365,7 +354,7 @@ class Mod(Node):
     def _getChildren(self):
         return self.left, self.right
     def __repr__(self):
-        return "Mod((%s, %s))" % (repr(self.left), repr(self.right))
+        return "Mod(%s, %s)" % (repr(self.left), repr(self.right))
 
 class Class(Node):
     nodes["class"] = "Class"
@@ -463,16 +452,16 @@ class Power(Node):
     def _getChildren(self):
         return self.left, self.right
     def __repr__(self):
-        return "Power((%s, %s))" % (repr(self.left), repr(self.right))
+        return "Power(%s, %s)" % (repr(self.left), repr(self.right))
 
-class Ellipsis(Node):
-    nodes["ellipsis"] = "Ellipsis"
-    def __init__(self, ):
-        pass
+class Import(Node):
+    nodes["import"] = "Import"
+    def __init__(self, names):
+        self.names = names
     def _getChildren(self):
-        return ()
+        return self.names,
     def __repr__(self):
-        return "Ellipsis()"
+        return "Import(%s)" % (repr(self.names),)
 
 class Return(Node):
     nodes["return"] = "Return"
@@ -491,7 +480,7 @@ class Add(Node):
     def _getChildren(self):
         return self.left, self.right
     def __repr__(self):
-        return "Add((%s, %s))" % (repr(self.left), repr(self.right))
+        return "Add(%s, %s)" % (repr(self.left), repr(self.right))
 
 class Function(Node):
     nodes["function"] = "Function"
@@ -536,14 +525,14 @@ class Subscript(Node):
     def __repr__(self):
         return "Subscript(%s, %s, %s)" % (repr(self.expr), repr(self.flags), repr(self.subs))
 
-class Import(Node):
-    nodes["import"] = "Import"
-    def __init__(self, names):
-        self.names = names
+class Ellipsis(Node):
+    nodes["ellipsis"] = "Ellipsis"
+    def __init__(self, ):
+        pass
     def _getChildren(self):
-        return self.names,
+        return ()
     def __repr__(self):
-        return "Import(%s)" % (repr(self.names),)
+        return "Ellipsis()"
 
 class Print(Node):
     nodes["print"] = "Print"

lib/python/RestrictedPython/compiler/ast.txt

+Module: doc, node
+Stmt: nodes
+Function: name, argnames, defaults, flags, doc, code
+Lambda: argnames, defaults, flags, code
+Class: name, bases, doc, code
+Pass: 
+Break: 
+Continue: 
+For: assign, list, body, else_
+While: test, body, else_
+If: tests, else_
+Exec: expr, locals, globals
+From: modname, names
+Import: names
+Raise: expr1, expr2, expr3
+TryFinally: body, final
+TryExcept: body, handlers, else_
+Return: value
+Const: value
+Print: nodes, dest
+Printnl: nodes, dest
+Discard: expr
+AugAssign: node, op, expr
+Assign: nodes, expr
+AssTuple: nodes
+AssList: nodes
+AssName: name, flags
+AssAttr: expr, attrname, flags
+ListComp: expr, quals
+ListCompFor: assign, list, ifs
+ListCompIf: test
+List: nodes
+Dict: items
+Not: expr
+Compare: expr, ops
+Name: name
+Global: names
+Backquote: expr
+Getattr: expr, attrname
+CallFunc: node, args, star_args = None, dstar_args = None
+Keyword: name, expr
+Subscript: expr, flags, subs
+Ellipsis: 
+Sliceobj: nodes
+Slice: expr, flags, lower, upper
+Assert: test, fail
+Tuple: nodes
+Or: nodes
+And: nodes
+Bitor: nodes
+Bitxor: nodes
+Bitand: nodes
+LeftShift: (left, right)
+RightShift: (left, right)
+Add: (left, right)
+Sub: (left, right)
+Mul: (left, right)
+Div: (left, right)
+Mod: (left, right)
+Power: (left, right)
+UnaryAdd: expr
+UnarySub: expr
+Invert: expr
+
+init(Function):
+    self.varargs = self.kwargs = None
+    if flags & CO_VARARGS:
+        self.varargs = 1
+    if flags & CO_VARKEYWORDS:
+        self.kwargs = 1
+
+init(Lambda):
+    self.varargs = self.kwargs = None
+    if flags & CO_VARARGS:
+        self.varargs = 1
+    if flags & CO_VARKEYWORDS:
+        self.kwargs = 1

lib/python/RestrictedPython/compiler/astgen.py

@@ -81,8 +81,6 @@ class NodeInfo:
         print >> buf, "    def __repr__(self):"
         if self.argnames:
             fmt = COMMA.join(["%s"] * self.nargs)
-            if '(' in self.args:
-                fmt = '(%s)' % fmt
             vals = ["repr(self.%s)" % name for name in self.argnames]
             vals = COMMA.join(vals)
             if self.nargs == 1:
@@ -173,8 +171,6 @@ class Node:
         return flatten(self._getChildren())
     def asList(self):
         return tuple(asList(self.getChildren()))
-    def getChildNodes(self):
-        return [n for n in self.getChildren() if isinstance(n, Node)]
 
 class EmptyNode(Node):
     def __init__(self):

lib/python/RestrictedPython/compiler/consts.py

-# code flags
-CO_VARARGS = 1
-CO_VARKEYWORDS = 2
-
 # operation flags
 OP_ASSIGN = 'OP_ASSIGN'
 OP_DELETE = 'OP_DELETE'
@@ -12,3 +8,9 @@ SC_GLOBAL = 2
 SC_FREE = 3
 SC_CELL = 4
 SC_UNKNOWN = 5
+
+CO_OPTIMIZED = 0x0001
+CO_NEWLOCALS = 0x0002
+CO_VARARGS = 0x0004
+CO_VARKEYWORDS = 0x0008
+CO_NESTED = 0x0010

lib/python/RestrictedPython/compiler/future.py

@@ -16,14 +16,19 @@ def is_future(stmt):
 
 class FutureParser:
 
-    features = ("nested_scopes", "generators", "division")
+    features = ("nested_scopes",)
     
     def __init__(self):
         self.found = {} # set
 
     def visitModule(self, node):
+        if node.doc is None:
+            off = 0
+        else:
+            off = 1
+
         stmt = node.node
-        for s in stmt.nodes:
+        for s in stmt.nodes[off:]:
             if not self.check_stmt(s):
                 break
 
@@ -62,7 +67,7 @@ def find_futures(node):
 
 if __name__ == "__main__":
     import sys
-    from compiler import parseFile, walk
+    from transformer import parseFile
 
     for file in sys.argv[1:]:
         print file

lib/python/RestrictedPython/compiler/misc.py

@@ -39,3 +39,29 @@ class Stack:
         self.stack.append(elt)
     def top(self):
         return self.stack[-1]
+    def __getitem__(self, index): # needed by visitContinue()
+        return self.stack[index]
+
+MANGLE_LEN = 256 # magic constant from compile.c
+
+def mangle(name, klass):
+    if not name.startswith('__'):
+        return name
+    if len(name) + 2 >= MANGLE_LEN:
+        return name
+    if name.endswith('__'):
+        return name
+    try:
+        i = 0
+        while klass[i] == '_':
+            i = i + 1
+    except IndexError:
+        return name
+    klass = klass[i:]
+
+    tlen = len(klass) + len(name)
+    if tlen > MANGLE_LEN:
+        klass = klass[:MANGLE_LEN-tlen]
+
+    return "_%s%s" % (klass, name)
+

lib/python/RestrictedPython/compiler/pyassem.py

 """A flow graph representation for Python bytecode"""
 
 from __future__ import nested_scopes
+
 import dis
 import new
 import string
@@ -8,6 +9,7 @@ import sys
 import types
 
 import misc
+from consts import CO_OPTIMIZED, CO_NEWLOCALS, CO_VARARGS, CO_VARKEYWORDS
 
 def xxx_sort(l):
     l = l[:]
@@ -54,7 +56,7 @@ class FlowGraph:
         # these edges to get the blocks emitted in the right order,
         # however. :-(  If a client needs to remove these edges, call
         # pruneEdges().
-        
+
         self.current.addNext(block)
         self.startBlock(block)
 
@@ -109,13 +111,13 @@ class FlowGraph:
 
         # XXX This is a total mess.  There must be a better way to get
         # the code blocks in the right order.
-        
+
         self.fixupOrderHonorNext(blocks, default_next)
         self.fixupOrderForward(blocks, default_next)
 
     def fixupOrderHonorNext(self, blocks, default_next):
         """Fix one problem with DFS.
-        
+
         The DFS uses child block, but doesn't know about the special
         "next" block.  As a result, the DFS can order blocks so that a
         block isn't next to the right block for implicit control
@@ -195,19 +197,18 @@ class FlowGraph:
             chains.remove(c)
             chains.insert(goes_before, c)
 
-
         del blocks[:]
         for c in chains:
             for b in c:
                 blocks.append(b)
-            
+
     def getBlocks(self):
         return self.blocks.elements()
 
     def getRoot(self):
         """Return nodes appropriate for use with dominator"""
         return self.entry
-    
+
     def getContainedGraphs(self):
         l = []
         for b in self.getBlocks():
@@ -246,7 +247,7 @@ class Block:
     def __str__(self):
         insts = map(str, self.insts)
         return "<block %s %d:\n%s>" % (self.label, self.bid,
-                                       string.join(insts, '\n')) 
+                                       string.join(insts, '\n'))
 
     def emit(self, inst):
         op = inst[0]
@@ -268,7 +269,7 @@ class Block:
         assert len(self.next) == 1, map(str, self.next)
 
     _uncond_transfer = ('RETURN_VALUE', 'RAISE_VARARGS',
-                        'JUMP_ABSOLUTE', 'JUMP_FORWARD')
+                        'JUMP_ABSOLUTE', 'JUMP_FORWARD', 'CONTINUE_LOOP')
 
     def pruneNext(self):
         """Remove bogus edge for unconditional transfers
@@ -312,11 +313,6 @@ class Block:
         return contained
 
 # flags for code objects
-CO_OPTIMIZED = 0x0001
-CO_NEWLOCALS = 0x0002
-CO_VARARGS = 0x0004
-CO_VARKEYWORDS = 0x0008
-CO_NESTED = 0x0010
 
 # the FlowGraph is transformed in place; it exists in one of these states
 RAW = "RAW"
@@ -327,15 +323,17 @@ DONE = "DONE"
 class PyFlowGraph(FlowGraph):
     super_init = FlowGraph.__init__
 
-    def __init__(self, name, filename, args=(), optimized=0):
+    def __init__(self, name, filename, args=(), optimized=0, klass=None):
         self.super_init()
         self.name = name
+        assert isinstance(filename, types.StringType)
         self.filename = filename
         self.docstring = None
         self.args = args # XXX
         self.argcount = getArgCount(args)
+        self.klass = klass
         if optimized:
-            self.flags = CO_OPTIMIZED | CO_NEWLOCALS 
+            self.flags = CO_OPTIMIZED | CO_NEWLOCALS
         else:
             self.flags = 0
         self.consts = []
@@ -364,6 +362,10 @@ class PyFlowGraph(FlowGraph):
         if flag == CO_VARARGS:
             self.argcount = self.argcount - 1
 
+    def checkFlag(self, flag):
+        if self.flags & flag:
+            return 1
+
     def setFreeVars(self, names):
         self.freevars = list(names)
 
@@ -462,7 +464,6 @@ class PyFlowGraph(FlowGraph):
                 insts[i] = opname, offset
             elif self.hasjabs.has_elt(opname):
                 insts[i] = opname, begin[inst[1]]
-        self.stacksize = findDepth(self.insts)
         self.stage = FLAT
 
     hasjrel = misc.Set()
@@ -480,8 +481,7 @@ class PyFlowGraph(FlowGraph):
         for i in range(len(self.insts)):
             t = self.insts[i]
             if len(t) == 2:
-                opname = t[0]
-                oparg = t[1]
+                opname, oparg = t
                 conv = self._converters.get(opname, None)
                 if conv:
                     self.insts[i] = opname, conv(self, oparg)
@@ -501,10 +501,16 @@ class PyFlowGraph(FlowGraph):
         self.closure = self.cellvars + self.freevars
 
     def _lookupName(self, name, list):
-        """Return index of name in list, appending if necessary"""
+        """Return index of name in list, appending if necessary
+
+        This routine uses a list instead of a dictionary, because a
+        dictionary can't store two different keys if the keys have the
+        same value but different types, e.g. 2 and 2L.  The compiler
+        must treat these two separately, so it does an explicit type
+        comparison before comparing the values.
+        """
         t = type(name)
         for i in range(len(list)):
-            # must do a comparison on type first to prevent UnicodeErrors 
             if t == type(list[i]) and list[i] == name:
                 return i
         end = len(list)
@@ -523,9 +529,15 @@ class PyFlowGraph(FlowGraph):
     _convert_STORE_FAST = _convert_LOAD_FAST
     _convert_DELETE_FAST = _convert_LOAD_FAST
 
+    def _convert_LOAD_NAME(self, arg):
+        if self.klass is None:
+            self._lookupName(arg, self.varnames)
+        return self._lookupName(arg, self.names)
+
     def _convert_NAME(self, arg):
+        if self.klass is None:
+            self._lookupName(arg, self.varnames)
         return self._lookupName(arg, self.names)
-    _convert_LOAD_NAME = _convert_NAME
     _convert_STORE_NAME = _convert_NAME
     _convert_DELETE_NAME = _convert_NAME
     _convert_IMPORT_NAME = _convert_NAME
@@ -557,7 +569,7 @@ class PyFlowGraph(FlowGraph):
     for name, obj in locals().items():
         if name[:9] == "_convert_":
             opname = name[9:]
-            _converters[opname] = obj            
+            _converters[opname] = obj
     del name, obj, opname
 
     def makeByteCode(self):
@@ -587,13 +599,14 @@ class PyFlowGraph(FlowGraph):
 
     def newCodeObject(self):
         assert self.stage == DONE
-        if self.flags == 0:
+        if (self.flags & CO_NEWLOCALS) == 0:
             nlocals = 0
         else:
             nlocals = len(self.varnames)
         argcount = self.argcount
         if self.flags & CO_VARKEYWORDS:
             argcount = argcount - 1
+
         return new.code(argcount, nlocals, self.stacksize, self.flags,
                         self.lnotab.getCode(), self.getConsts(),
                         tuple(self.names), tuple(self.varnames),
@@ -613,7 +626,7 @@ class PyFlowGraph(FlowGraph):
                 elt = elt.getCode()
             l.append(elt)
         return tuple(l)
-            
+
 def isJump(opname):
     if opname[:4] == 'JUMP':
         return 1
@@ -707,17 +720,19 @@ class LineAddrTable:
 
     def getTable(self):
         return string.join(map(chr, self.lnotab), '')
-    
+
 class StackDepthTracker:
     # XXX 1. need to keep track of stack depth on jumps
     # XXX 2. at least partly as a result, this code is broken
 
-    def findDepth(self, insts):
+    def findDepth(self, insts, debug=0):
         depth = 0
         maxDepth = 0
         for i in insts:
             opname = i[0]
-            delta = self.effect.get(opname)
+            if debug:
+                print i,
+            delta = self.effect.get(opname, None)
             if delta is not None:
                 depth = depth + delta
             else:
@@ -732,10 +747,10 @@ class StackDepthTracker:
                     meth = getattr(self, opname, None)
                     if meth is not None:
                         depth = depth + meth(i[1])
-            if depth < 0:
-                depth = 0
             if depth > maxDepth:
                 maxDepth = depth
+            if debug:
+                print depth, maxDepth
         return maxDepth
 
     effect = {
@@ -756,9 +771,8 @@ class StackDepthTracker:
         'DELETE_SUBSCR': -2,
         # PRINT_EXPR?
         'PRINT_ITEM': -1,
-        'LOAD_LOCALS': 1,
         'RETURN_VALUE': -1,
-        'EXEC_STMT': -2,
+        'EXEC_STMT': -3,
         'BUILD_CLASS': -2,
         'STORE_NAME': -1,
         'STORE_ATTR': -2,
@@ -774,23 +788,20 @@ class StackDepthTracker:
         # close enough...
         'SETUP_EXCEPT': 3,
         'SETUP_FINALLY': 3,
-        'FOR_ITER': 1,
+        'FOR_LOOP': 1,
         }
     # use pattern match
     patterns = [
         ('BINARY_', -1),
         ('LOAD_', 1),
         ]
-    
-    # special cases:
-    # UNPACK_SEQUENCE, BUILD_TUPLE,
-    # BUILD_LIST, CALL_FUNCTION, MAKE_FUNCTION, BUILD_SLICE
+
     def UNPACK_SEQUENCE(self, count):
         return count-1
     def BUILD_TUPLE(self, count):
-        return 1-count
+        return -count+1
     def BUILD_LIST(self, count):
-        return 1-count
+        return -count+1
     def CALL_FUNCTION(self, argc):
         hi, lo = divmod(argc, 256)
         return -(lo + hi * 2)
@@ -802,6 +813,9 @@ class StackDepthTracker:
         return self.CALL_FUNCTION(argc)-2
     def MAKE_FUNCTION(self, argc):
         return -argc
+    def MAKE_CLOSURE(self, argc):
+        # XXX need to account for free variables too!
+        return -argc
     def BUILD_SLICE(self, argc):
         if argc == 2:
             return -1
@@ -809,5 +823,5 @@ class StackDepthTracker:
             return -2
     def DUP_TOPX(self, argc):
         return argc
-    
+
 findDepth = StackDepthTracker().findDepth

lib/python/RestrictedPython/compiler/pycodegen.py

@@ -13,8 +13,8 @@ from transformer import parse
 from visitor import walk
 import pyassem, misc, future, symbols
 from consts import SC_LOCAL, SC_GLOBAL, SC_FREE, SC_CELL
-from pyassem import CO_VARARGS, CO_VARKEYWORDS, CO_NEWLOCALS,\
-     CO_NESTED, TupleArg
+from consts import CO_VARARGS, CO_VARKEYWORDS, CO_NEWLOCALS, CO_NESTED
+from pyassem import TupleArg
 
 # Do we have Python 1.x or Python 2.x?
 try:
@@ -30,15 +30,24 @@ callfunc_opcode_info = {
     (1,1) : "CALL_FUNCTION_VAR_KW",
 }
 
+LOOP = 1
+EXCEPT = 2
+TRY_FINALLY = 3
+END_FINALLY = 4
+
 def compile(filename, display=0):
     f = open(filename)
     buf = f.read()
     f.close()
     mod = Module(buf, filename)
-    mod.compile(display)
-    f = open(filename + "c", "wb")
-    mod.dump(f)
-    f.close()
+    try:
+        mod.compile(display)
+    except SyntaxError:
+        raise
+    else:
+        f = open(filename + "c", "wb")
+        mod.dump(f)
+        f.close()
 
 class Module:
     def __init__(self, source, filename):
@@ -117,6 +126,12 @@ class LocalNameFinder:
     def visitAssName(self, node):
         self.names.add(node.name)
 
+def is_constant_false(node):
+    if isinstance(node, ast.Const):
+        if not node.value:
+            return 1
+    return 0
+
 class CodeGenerator:
     """Defines basic code generator for Python bytecode
 
@@ -133,6 +148,7 @@ class CodeGenerator:
 
     optimized = 0 # is namespace access optimized?
     __initialized = None
+    class_name = None # provide default for instance variable
 
     def __init__(self, filename):
         if self.__initialized is None:
@@ -141,7 +157,7 @@ class CodeGenerator:
         self.checkClass()
         self.filename = filename
         self.locals = misc.Stack()
-        self.loops = misc.Stack()
+        self.setups = misc.Stack()
         self.curStack = 0
         self.maxStack = 0
         self.last_lineno = None
@@ -172,6 +188,17 @@ class CodeGenerator:
         """Return a code object"""
         return self.graph.getCode()
 
+    def mangle(self, name):
+        if self.class_name is not None:
+            return misc.mangle(name, self.class_name)
+        else:
+            return name
+
+    def parseSymbols(self, tree):
+        s = symbols.SymbolVisitor()
+        walk(tree, s)
+        return s.scopes
+
     # Next five methods handle name access
 
     def isLocalName(self, name):
@@ -187,13 +214,23 @@ class CodeGenerator:
         self._nameOp('DELETE', name)
 
     def _nameOp(self, prefix, name):
-        if not self.optimized:
-            self.emit(prefix + '_NAME', name)
-            return
-        if self.isLocalName(name):
-            self.emit(prefix + '_FAST', name)
+        name = self.mangle(name)
+        scope = self.scope.check_name(name)
+        if scope == SC_LOCAL:
+            if not self.optimized:
+                self.emit(prefix + '_NAME', name)
+            else:
+                self.emit(prefix + '_FAST', name)
+        elif scope == SC_GLOBAL:
+            if not self.optimized:
+                self.emit(prefix + '_NAME', name)
+            else:
+                self.emit(prefix + '_GLOBAL', name)
+        elif scope == SC_FREE or scope == SC_CELL:
+            self.emit(prefix + '_DEREF', name)
         else:
-            self.emit(prefix + '_GLOBAL', name)
+            raise RuntimeError, "unsupported scope for var %s: %d" % \
+                  (name, scope)
 
     def _implicitNameOp(self, prefix, name):
         """Emit name ops for names generated implicitly by for loops
@@ -235,11 +272,14 @@ class CodeGenerator:
     ClassGen = None
 
     def visitModule(self, node):
+        self.scopes = self.parseSymbols(node)
+        self.scope = self.scopes[node]
         self.emit('SET_LINENO', 0)
-        lnf = walk(node.node, self.NameFinder(), 0)
-        self.locals.push(lnf.getLocals())
         if node.doc:
-            self.fixDocstring(node.node)
+            self.emit('LOAD_CONST', node.doc)
+            self.storeName('__doc__')
+        lnf = walk(node.node, self.NameFinder(), verbose=0)
+        self.locals.push(lnf.getLocals())
         self.visit(node.node)
         self.emit('LOAD_CONST', None)
         self.emit('RETURN_VALUE')
@@ -254,7 +294,8 @@ class CodeGenerator:
         self._visitFuncOrLambda(node, isLambda=1)
 
     def _visitFuncOrLambda(self, node, isLambda=0):
-        gen = self.FunctionGen(node, self.filename, self.scopes, isLambda)
+        gen = self.FunctionGen(node, self.filename, self.scopes, isLambda,
+                               self.class_name)
         walk(node.code, gen)
         gen.finish()
         self.set_lineno(node)
@@ -264,9 +305,7 @@ class CodeGenerator:
         self.emit('MAKE_FUNCTION', len(node.defaults))
 
     def visitClass(self, node):
-        gen = self.ClassGen(node, self.filename, self.scopes)
-        if node.doc:
-            self.fixDocstring(node.code)
+        gen = self.ClassGen(node, self.scopes, self.filename)
         walk(node.code, gen)
         gen.finish()
         self.set_lineno(node)
@@ -280,19 +319,6 @@ class CodeGenerator:
         self.emit('BUILD_CLASS')
         self.storeName(node.name)
 
-    def fixDocstring(self, node):
-        """Rewrite the ast for a class with a docstring.
-
-        The AST includes a Discard(Const(docstring)) node.  Replace
-        this with an Assign([AssName('__doc__', ...])
-        """
-        assert isinstance(node, ast.Stmt)
-        stmts = node.nodes
-        discard = stmts[0]
-        assert isinstance(discard, ast.Discard)
-        stmts[0] = ast.Assign([ast.AssName('__doc__', 'OP_ASSIGN')],
-                              discard.expr)
-        stmts[0].lineno = discard.lineno
     # The rest are standard visitor methods
 
     # The next few implement control-flow statements
@@ -302,6 +328,8 @@ class CodeGenerator:
         numtests = len(node.tests)
         for i in range(numtests):
             test, suite = node.tests[i]
+            if is_constant_false(test):
+                continue
             self.set_lineno(test)
             self.visit(test)
             nextTest = self.newBlock()
@@ -326,7 +354,7 @@ class CodeGenerator:
         self.emit('SETUP_LOOP', after)
 
         self.nextBlock(loop)
-        self.loops.push(loop)
+        self.setups.push((LOOP, loop))
 
         self.set_lineno(node, force=1)
         self.visit(node.test)
@@ -340,7 +368,7 @@ class CodeGenerator:
         self.startBlock(else_) # or just the POPs if not else clause
         self.emit('POP_TOP')
         self.emit('POP_BLOCK')
-        self.loops.pop()
+        self.setups.pop()
         if node.else_:
             self.visit(node.else_)
         self.nextBlock(after)
@@ -349,7 +377,7 @@ class CodeGenerator:
         start = self.newBlock()
         anchor = self.newBlock()
         after = self.newBlock()
-        self.loops.push(start)
+        self.setups.push((LOOP, start))
 
         self.set_lineno(node)
         self.emit('SETUP_LOOP', after)
@@ -358,32 +386,52 @@ class CodeGenerator:
         self.nextBlock(start)
         self.set_lineno(node, force=1)
         self.emit('FOR_LOOP', anchor)
-        self.nextBlock()
+##        self.nextBlock()
         self.visit(node.assign)
         self.visit(node.body)
         self.emit('JUMP_ABSOLUTE', start)
-        self.startBlock(anchor)
+##        self.startBlock(anchor)
+        self.nextBlock(anchor)
         self.emit('POP_BLOCK')
-        self.loops.pop()
+        self.setups.pop()
         if node.else_:
             self.visit(node.else_)
         self.nextBlock(after)
 
     def visitBreak(self, node):
-        if not self.loops:
+        if not self.setups:
             raise SyntaxError, "'break' outside loop (%s, %d)" % \
                   (self.filename, node.lineno)
         self.set_lineno(node)
         self.emit('BREAK_LOOP')
 
     def visitContinue(self, node):
-        if not self.loops:
+        # XXX test_grammar.py, line 351
+        if not self.setups:
             raise SyntaxError, "'continue' outside loop (%s, %d)" % \
                   (self.filename, node.lineno)
-        l = self.loops.top()
-        self.set_lineno(node)
-        self.emit('JUMP_ABSOLUTE', l)
-        self.nextBlock()
+        kind, block = self.setups.top()
+        if kind == LOOP:
+            self.set_lineno(node)
+            self.emit('JUMP_ABSOLUTE', block)
+            self.nextBlock()
+        elif kind == EXCEPT or kind == TRY_FINALLY:
+            self.set_lineno(node)
+            # find the block that starts the loop
+            top = len(self.setups)
+            while top > 0:
+                top = top - 1
+                kind, loop_block = self.setups[top]
+                if kind == LOOP:
+                    break
+            if kind != LOOP:
+                raise SyntaxError, "'continue' outside loop (%s, %d)" % \
+                      (self.filename, node.lineno)
+            self.emit('CONTINUE_LOOP', loop_block)
+            self.nextBlock()
+        elif kind == END_FINALLY:
+            msg = "'continue' not allowed inside 'finally' clause (%s, %d)"
+            raise SyntaxError, msg % (self.filename, node.lineno)
 
     def visitTest(self, node, jump):
         end = self.newBlock()
@@ -528,6 +576,7 @@ class CodeGenerator:
         self.emit('RAISE_VARARGS', n)
 
     def visitTryExcept(self, node):
+        body = self.newBlock()
         handlers = self.newBlock()
         end = self.newBlock()
         if node.else_:
@@ -536,9 +585,11 @@ class CodeGenerator:
             lElse = end
         self.set_lineno(node)
         self.emit('SETUP_EXCEPT', handlers)
-        self.nextBlock()
+        self.nextBlock(body)
+        self.setups.push((EXCEPT, body))
         self.visit(node.body)
         self.emit('POP_BLOCK')
+        self.setups.pop()
         self.emit('JUMP_FORWARD', lElse)
         self.startBlock(handlers)
         
@@ -574,16 +625,21 @@ class CodeGenerator:
         self.nextBlock(end)
     
     def visitTryFinally(self, node):
+        body = self.newBlock()
         final = self.newBlock()
         self.set_lineno(node)
         self.emit('SETUP_FINALLY', final)
-        self.nextBlock()
+        self.nextBlock(body)
+        self.setups.push((TRY_FINALLY, body))
         self.visit(node.body)
         self.emit('POP_BLOCK')
+        self.setups.pop()
         self.emit('LOAD_CONST', None)
         self.nextBlock(final)
+        self.setups.push((END_FINALLY, final))
         self.visit(node.final)
         self.emit('END_FINALLY')
+        self.setups.pop()
 
     # misc
 
@@ -650,7 +706,7 @@ class CodeGenerator:
 
     def visitGetattr(self, node):
         self.visit(node.expr)
-        self.emit('LOAD_ATTR', node.attrname)
+        self.emit('LOAD_ATTR', self.mangle(node.attrname))
 
     # next five implement assignments
 
@@ -669,6 +725,7 @@ class CodeGenerator:
         if node.flags == 'OP_ASSIGN':
             self.storeName(node.name)
         elif node.flags == 'OP_DELETE':
+            self.set_lineno(node)
             self.delName(node.name)
         else:
             print "oops", node.flags
@@ -676,9 +733,9 @@ class CodeGenerator:
     def visitAssAttr(self, node):
         self.visit(node.expr)
         if node.flags == 'OP_ASSIGN':
-            self.emit('STORE_ATTR', node.attrname)
+            self.emit('STORE_ATTR', self.mangle(node.attrname))
         elif node.flags == 'OP_DELETE':
-            self.emit('DELETE_ATTR', node.attrname)
+            self.emit('DELETE_ATTR', self.mangle(node.attrname))
         else:
             print "warning: unexpected flags:", node.flags
             print node
@@ -702,6 +759,7 @@ class CodeGenerator:
     # augmented assignment
 
     def visitAugAssign(self, node):
+        self.set_lineno(node)
         aug_node = wrap_aug(node.node)
         self.visit(aug_node, "load")
         self.visit(node.expr)
@@ -732,10 +790,10 @@ class CodeGenerator:
         if mode == "load":
             self.visit(node.expr)
             self.emit('DUP_TOP')
-            self.emit('LOAD_ATTR', node.attrname)
+            self.emit('LOAD_ATTR', self.mangle(node.attrname))
         elif mode == "store":
             self.emit('ROT_TWO')
-            self.emit('STORE_ATTR', node.attrname)
+            self.emit('STORE_ATTR', self.mangle(node.attrname))
 
     def visitAugSlice(self, node, mode):
         if mode == "load":
@@ -812,7 +870,7 @@ class CodeGenerator:
             self.emit('POP_TOP')
 
     def visitPrintnl(self, node):
-        self.visitPrint(node, 1)
+        self.visitPrint(node, newline=1)
         if node.dest:
             self.emit('PRINT_NEWLINE_TO')
         else:
@@ -945,11 +1003,13 @@ class CodeGenerator:
         self.emit('LOAD_CONST', Ellipsis)
 
     def visitTuple(self, node):
+        self.set_lineno(node)
         for elt in node.nodes:
             self.visit(elt)
         self.emit('BUILD_TUPLE', len(node.nodes))
 
     def visitList(self, node):
+        self.set_lineno(node)
         for elt in node.nodes:
             self.visit(elt)
         self.emit('BUILD_LIST', len(node.nodes))
@@ -962,7 +1022,7 @@ class CodeGenerator:
     def visitDict(self, node):
         lineno = getattr(node, 'lineno', None)
         if lineno:
-            set.emit('SET_LINENO', lineno)
+            self.emit('SET_LINENO', lineno)
         self.emit('BUILD_MAP', 0)
         for k, v in node.items:
             lineno2 = getattr(node, 'lineno', None)
@@ -991,6 +1051,7 @@ class NestedScopeCodeGenerator(CodeGenerator):
         self.__super_visitModule(node)
 
     def _nameOp(self, prefix, name):
+        name = self.mangle(name)
         scope = self.scope.check_name(name)
         if scope == SC_LOCAL:
             if not self.optimized:
@@ -1006,7 +1067,8 @@ class NestedScopeCodeGenerator(CodeGenerator):
                   (name, scope)
 
     def _visitFuncOrLambda(self, node, isLambda=0):
-        gen = self.FunctionGen(node, self.filename, self.scopes, isLambda)
+        gen = self.FunctionGen(node, self.filename, self.scopes, isLambda,
+                               self.class_name)
         walk(node.code, gen)
         gen.finish()
         self.set_lineno(node)
@@ -1023,9 +1085,7 @@ class NestedScopeCodeGenerator(CodeGenerator):
             self.emit('MAKE_FUNCTION', len(node.defaults))
 
     def visitClass(self, node):
-        gen = self.ClassGen(node, self.filename, self.scopes)
-        if node.doc:
-            self.fixDocstring(node.code)
+        gen = self.ClassGen(node, self.scopes, self.filename)
         walk(node.code, gen)
         gen.finish()
         self.set_lineno(node)
@@ -1082,7 +1142,10 @@ class AbstractFunctionCode:
     optimized = 1
     lambdaCount = 0
 
-    def __init__(self, func, filename, scopes, isLambda):
+    def __init__(self, func, filename, scopes, isLambda, class_name):
+        self.scopes = scopes
+        self.scope = scopes[func]
+        self.class_name = class_name
         if isLambda:
             klass = FunctionCodeGenerator
             name = "<lambda.%d>" % klass.lambdaCount
@@ -1098,7 +1161,7 @@ class AbstractFunctionCode:
         if not isLambda and func.doc:
             self.setDocstring(func.doc)
 
-        lnf = walk(func.code, self.NameFinder(args), 0)
+        lnf = walk(func.code, self.NameFinder(args), verbose=0)
         self.locals.push(lnf.getLocals())
         if func.varargs:
             self.graph.setFlag(CO_VARARGS)
@@ -1145,17 +1208,19 @@ class NestedFunctionCodeGenerator(AbstractFunctionCode,
     super_init = NestedScopeCodeGenerator.__init__ # call be other init
     __super_init = AbstractFunctionCode.__init__
 
-    def __init__(self, func, filename, scopes, isLambda):
+    def __init__(self, func, filename, scopes, isLambda, class_name):
         self.scopes = scopes
         self.scope = scopes[func]
-        self.__super_init(func, filename, scopes, isLambda)
+        self.__super_init(func, filename, scopes, isLambda, class_name)
         self.graph.setFreeVars(self.scope.get_free_vars())
         self.graph.setCellVars(self.scope.get_cell_vars())
         self.graph.setFlag(CO_NESTED)
 
 class AbstractClassCode:
 
-    def __init__(self, klass, filename, scopes):
+    def __init__(self, klass, scopes, filename):
+        assert isinstance(filename, types.StringType)
+        assert isinstance(scopes, types.DictType)
         self.graph = pyassem.PyFlowGraph(klass.name, filename,
                                            optimized=0)
         self.super_init(filename)
@@ -1178,16 +1243,26 @@ class ClassCodeGenerator(LGBScopeMixin, AbstractClassCode, CodeGenerator):
     super_init = CodeGenerator.__init__
     scopes = None
 
+    __super_init = AbstractClassCode.__init__
+
+    def __init__(self, klass, scopes, filename):
+        self.scopes = scopes
+        self.scope = scopes[klass]
+        self.__super_init(klass, scopes, filename)
+        self.graph.setFreeVars(self.scope.get_free_vars())
+        self.graph.setCellVars(self.scope.get_cell_vars())
+
 class NestedClassCodeGenerator(AbstractClassCode,
                                NestedScopeMixin,
                                NestedScopeCodeGenerator):
     super_init = NestedScopeCodeGenerator.__init__ # call be other init
     __super_init = AbstractClassCode.__init__
 
-    def __init__(self, klass, filename, scopes):
+    def __init__(self, klass, scopes, filename):
+        assert isinstance(filename, types.StringType)
         self.scopes = scopes
         self.scope = scopes[klass]
-        self.__super_init(klass, filename, scopes)
+        self.__super_init(klass, scopes, filename)
         self.graph.setFreeVars(self.scope.get_free_vars())
         self.graph.setCellVars(self.scope.get_cell_vars())
         self.graph.setFlag(CO_NESTED)
@@ -1212,7 +1287,7 @@ def generateArgList(arglist):
 def findOp(node):
     """Find the op (DELETE, LOAD, STORE) in an AssTuple tree"""
     v = OpFinder()
-    walk(node, v, 0)
+    walk(node, v, verbose=0)
     return v.op
 
 class OpFinder:
@@ -1224,6 +1299,7 @@ class OpFinder:
         elif self.op != node.flags:
             raise ValueError, "mixed ops in stmt"
     visitAssAttr = visitAssName
+    visitSubscript = visitAssName
 
 class Delegator:
     """Base class to support delegation for augmented assignment nodes
@@ -1254,6 +1330,7 @@ class AugSlice(Delegator):
 class AugSubscript(Delegator):
     pass
 
+
 wrapper = {
     ast.Getattr: AugGetattr,
     ast.Name: AugName,

lib/python/RestrictedPython/compiler/symbols.py

@@ -2,6 +2,7 @@
 
 import ast
 from consts import SC_LOCAL, SC_GLOBAL, SC_FREE, SC_CELL, SC_UNKNOWN
+from misc import mangle
 import types
 
 import sys
@@ -36,13 +37,7 @@ class Scope:
     def mangle(self, name):
         if self.klass is None:
             return name
-        if not name.startswith('__'):
-            return name
-        if len(name) + 2 >= MANGLE_LEN:
-            return name
-        if name.endswith('__'):
-            return name
-        return "_%s%s" % (self.klass, name)
+        return mangle(name, self.klass)
 
     def add_def(self, name):
         self.defs[self.mangle(name)] = 1
@@ -295,6 +290,27 @@ class SymbolVisitor:
                 name = name[:i]
             scope.add_def(asname or name)
 
+    def visitGlobal(self, node, scope):
+        for name in node.names:
+            scope.add_global(name)
+
+    def visitAssign(self, node, scope):
+        """Propagate assignment flag down to child nodes.
+
+        The Assign node doesn't itself contains the variables being
+        assigned to.  Instead, the children in node.nodes are visited
+        with the assign flag set to true.  When the names occur in
+        those nodes, they are marked as defs.
+
+        Some names that occur in an assignment target are not bound by
+        the assignment, e.g. a name occurring inside a slice.  The
+        visitor handles these nodes specially; they do not propagate
+        the assign flag to their children.
+        """
+        for n in node.nodes:
+            self.visit(n, scope, 1)
+        self.visit(node.expr, scope)
+
     def visitAssName(self, node, scope, assign=1):
         scope.add_def(node.name)
 
@@ -306,6 +322,13 @@ class SymbolVisitor:
         for n in node.subs:
             self.visit(n, scope, 0)
 
+    def visitSlice(self, node, scope, assign=0):
+        self.visit(node.expr, scope, 0)
+        if node.lower:
+            self.visit(node.lower, scope, 0)
+        if node.upper:
+            self.visit(node.upper, scope, 0)
+
     def visitAugAssign(self, node, scope):
         # If the LHS is a name, then this counts as assignment.
         # Otherwise, it's just use.
@@ -314,15 +337,6 @@ class SymbolVisitor:
             self.visit(node.node, scope, 1) # XXX worry about this
         self.visit(node.expr, scope)
 
-    def visitAssign(self, node, scope):
-        for n in node.nodes:
-            self.visit(n, scope, 1)
-        self.visit(node.expr, scope)
-
-    def visitGlobal(self, node, scope):
-        for name in node.names:
-            scope.add_global(name)
-
     # prune if statements if tests are false
 
     _const_types = types.StringType, types.IntType, types.FloatType
@@ -348,7 +362,8 @@ def list_eq(l1, l2):
 
 if __name__ == "__main__":
     import sys
-    from compiler import parseFile, walk
+    from transformer import parseFile
+    from visitor import walk
     import symtable
 
     def get_names(syms):

lib/python/RestrictedPython/compiler/transformer.py

@@ -153,12 +153,8 @@ class Transformer:
 
     def file_input(self, nodelist):
         doc = self.get_docstring(nodelist, symbol.file_input)
-        if doc is not None:
-            i = 1
-        else:
-            i = 0
         stmts = []
-        for node in nodelist[i:]:
+        for node in nodelist:
             if node[0] != token.ENDMARKER and node[0] != token.NEWLINE:
                 self.com_append_stmt(stmts, node)
         return Module(doc, Stmt(stmts))
@@ -344,11 +340,6 @@ class Transformer:
         n.lineno = nodelist[0][2]
         return n
 
-    def yield_stmt(self, nodelist):
-        n = Yield(self.com_node(nodelist[1]))
-        n.lineno = nodelist[0][2]
-        return n
-
     def raise_stmt(self, nodelist):
         # raise: [test [',' test [',' test]]]
         if len(nodelist) > 5:
@@ -1254,7 +1245,6 @@ _legal_node_types = [
     symbol.continue_stmt,
     symbol.return_stmt,
     symbol.raise_stmt,
-    #symbol.yield_stmt,
     symbol.import_stmt,
     symbol.global_stmt,
     symbol.exec_stmt,
@@ -1281,9 +1271,6 @@ _legal_node_types = [
     symbol.atom,
     ]
 
-if hasattr(symbol, 'yield_stmt'):
-    _legal_node_types.append(symbol.yield_stmt)
-
 _assign_types = [
     symbol.test,
     symbol.and_test,