Merge

Cython/CodeWriter.py

-from Cython.Compiler.Visitor import TreeVisitor, get_temp_name_handle_desc
+from Cython.Compiler.Visitor import TreeVisitor
 from Cython.Compiler.Nodes import *
 from Cython.Compiler.ExprNodes import *
 
@@ -37,6 +37,7 @@ class CodeWriter(TreeVisitor):
         self.result = result
         self.numindents = 0
         self.tempnames = {}
+        self.tempblockindex = 0
     
     def write(self, tree):
         self.visit(tree)
@@ -60,12 +61,6 @@ class CodeWriter(TreeVisitor):
         self.startline(s)
         self.endline()
 
-    def putname(self, name):
-        tmpdesc = get_temp_name_handle_desc(name)
-        if tmpdesc is not None:
-            name = self.tempnames.setdefault(tmpdesc, u"$" +tmpdesc)
-        self.put(name)
-    
     def comma_seperated_list(self, items, output_rhs=False):
         if len(items) > 0:
             for item in items[:-1]:
@@ -132,7 +127,7 @@ class CodeWriter(TreeVisitor):
         self.endline()
     
     def visit_NameNode(self, node):
-        self.putname(node.name)
+        self.put(node.name)
     
     def visit_IntNode(self, node):
         self.put(node.value)
@@ -312,3 +307,18 @@ class CodeWriter(TreeVisitor):
         self.visit(node.operand)
         self.put(u")")
 
+    def visit_TempsBlockNode(self, node):
+        """
+        Temporaries are output like $1_1', where the first number is
+        an index of the TempsBlockNode and the second number is an index
+        of the temporary which that block allocates.
+        """
+        idx = 0
+        for handle in node.temps:
+            self.tempnames[handle] = "$%d_%d" % (self.tempblockindex, idx)
+            idx += 1
+        self.tempblockindex += 1
+        self.visit(node.body)
+
+    def visit_TempRefNode(self, node):
+        self.put(self.tempnames[node.handle])

Cython/Compiler/Buffer.py

-from Cython.Compiler.Visitor import VisitorTransform, temp_name_handle, CythonTransform
+from Cython.Compiler.Visitor import VisitorTransform, CythonTransform
 from Cython.Compiler.ModuleNode import ModuleNode
 from Cython.Compiler.Nodes import *
 from Cython.Compiler.ExprNodes import *
-from Cython.Compiler.TreeFragment import TreeFragment
 from Cython.Compiler.StringEncoding import EncodedString
 from Cython.Compiler.Errors import CompileError
 import Interpreter

Cython/Compiler/CodeGeneration.py

-from Cython.Compiler.Visitor import VisitorTransform, temp_name_handle, CythonTransform
+from Cython.Compiler.Visitor import VisitorTransform, CythonTransform
 from Cython.Compiler.ModuleNode import ModuleNode
 from Cython.Compiler.Nodes import *
 from Cython.Compiler.ExprNodes import *

Cython/Compiler/ExprNodes.py

@@ -168,6 +168,7 @@ class ExprNode(Node):
     
     saved_subexpr_nodes = None
     is_temp = 0
+    is_target = 0
 
     def get_child_attrs(self):
         return self.subexprs
@@ -206,10 +207,10 @@ class ExprNode(Node):
         return self.saved_subexpr_nodes
         
     def result(self):
-		if self.is_temp:
-			return self.result_code
-		else:
+        if not self.is_temp or self.is_target:
             return self.calculate_result_code()
+        else: # i.e. self.is_temp:
+            return self.result_code
     
     def result_as(self, type = None):
         #  Return the result code cast to the specified C type.
@@ -335,7 +336,7 @@ class ExprNode(Node):
         if debug_temp_alloc:
             print("%s Allocating target temps" % self)
         self.allocate_subexpr_temps(env)
-        self.result_code = self.target_code()
+        self.is_target = True
         if rhs:
             rhs.release_temp(env)
         self.release_subexpr_temps(env)
@@ -560,6 +561,66 @@ class ExprNode(Node):
         return self.result_in_temp()
 
 
+class NewTempExprNode(ExprNode):
+    backwards_compatible_result = None
+    
+    def result(self):
+        if self.is_temp:
+            return self.temp_code
+        else:
+            return self.calculate_result_code()
+
+    def allocate_target_temps(self, env, rhs):
+        self.allocate_subexpr_temps(env)
+        rhs.release_temp(rhs)
+        self.releasesubexpr_temps(env)
+
+    def allocate_temps(self, env, result = None):
+        self.allocate_subexpr_temps(env)
+        self.backwards_compatible_result = result
+        if self.is_temp:
+            self.release_subexpr_temps(env)
+
+    def allocate_temp(self, env, result = None):
+        assert result is None
+
+    def release_temp(self, env):
+        pass
+
+    def generate_result_code(self, code):
+        if self.is_temp:
+            type = self.type
+            if not type.is_void:
+                if type.is_pyobject:
+                    type = PyrexTypes.py_object_type
+                if self.backwards_compatible_result:
+                    self.temp_code = self.backwards_compatible_result
+                else:
+                    self.temp_code = code.funcstate.allocate_temp(type)
+            else:
+                self.temp_code = None
+
+    def generate_disposal_code(self, code):
+        if self.is_temp:
+            if self.type.is_pyobject:
+                code.put_decref_clear(self.result(), self.ctype())
+            if not self.backwards_compatible_result:
+                code.funcstate.release_temp(self.temp_code)
+        else:
+            self.generate_subexpr_disposal_code(code)
+
+    def generate_post_assignment_code(self, code):
+        if self.is_temp:
+            if self.type.is_pyobject:
+                code.putln("%s = 0;" % self.temp_code)
+            if not self.backwards_compatible_result:
+                code.funcstate.release_temp(self.temp_code)
+        else:
+            self.generate_subexpr_disposal_code(code)
+
+    
+        
+
 class AtomicExprNode(ExprNode):
     #  Abstract base class for expression nodes which have
     #  no sub-expressions.
@@ -1499,7 +1560,8 @@ class IndexNode(ExprNode):
 
     def generate_result_code(self, code):
         if self.is_buffer_access:
-            # buffer_pointer_code is returned by result()
+            if code.globalstate.directives['nonecheck']:
+                self.put_nonecheck(code)
             ptrcode = self.buffer_lookup_code(code)
             code.putln("%s = *%s;" % (
                 self.result(),
@@ -1543,6 +1605,8 @@ class IndexNode(ExprNode):
 
     def generate_buffer_setitem_code(self, rhs, code, op=""):
         # Used from generate_assignment_code and InPlaceAssignmentNode
+        if code.globalstate.directives['nonecheck']:
+            self.put_nonecheck(code)
         ptrexpr = self.buffer_lookup_code(code)
         if self.buffer_type.dtype.is_pyobject:
             # Must manage refcounts. Decref what is already there
@@ -1609,6 +1673,13 @@ class IndexNode(ExprNode):
                                              options=code.globalstate.directives,
                                              pos=self.pos, code=code)
 
+    def put_nonecheck(self, code):
+        code.globalstate.use_utility_code(raise_noneindex_error_utility_code)
+        code.putln("if (%s) {" % code.unlikely("%s == Py_None") % self.base.result_as(PyrexTypes.py_object_type))
+        code.putln("__Pyx_RaiseNoneIndexingError();")
+        code.putln(code.error_goto(self.pos))
+        code.putln("}")
+
 class SliceIndexNode(ExprNode):
     #  2-element slice indexing
     #
@@ -3211,7 +3282,7 @@ def get_compile_time_binop(node):
                 % node.operator)
     return func
 
-class BinopNode(ExprNode):
+class BinopNode(NewTempExprNode):
     #  operator     string
     #  operand1     ExprNode
     #  operand2     ExprNode
@@ -3261,6 +3332,7 @@ class BinopNode(ExprNode):
         self.operand2.check_const()
     
     def generate_result_code(self, code):
+        NewTempExprNode.generate_result_code(self, code)
         #print "BinopNode.generate_result_code:", self.operand1, self.operand2 ###
         if self.operand1.type.is_pyobject:
             function = self.py_operation_function()
@@ -4589,3 +4661,12 @@ static INLINE void __Pyx_RaiseNoneAttributeError(char* attrname) {
     PyErr_Format(PyExc_AttributeError, "'NoneType' object has no attribute '%s'", attrname);
 }
 """]
+
+raise_noneindex_error_utility_code = [
+"""
+static INLINE void __Pyx_RaiseNoneIndexingError();
+""", """
+static INLINE void __Pyx_RaiseNoneIndexingError() {
+    PyErr_SetString(PyExc_TypeError, "'NoneType' object is unsubscriptable");
+}
+"""]

Cython/Compiler/Nodes.py

@@ -4188,6 +4188,7 @@ class FromImportStatNode(StatNode):
         self.module.generate_disposal_code(code)
 
 
+
 #------------------------------------------------------------------------------------
 #
 #  Runtime support code

Cython/Compiler/ParseTreeTransforms.py

-from Cython.Compiler.Visitor import VisitorTransform, temp_name_handle, CythonTransform
+from Cython.Compiler.Visitor import VisitorTransform, CythonTransform
 from Cython.Compiler.ModuleNode import ModuleNode
 from Cython.Compiler.Nodes import *
 from Cython.Compiler.ExprNodes import *
+from Cython.Compiler.UtilNodes import *
 from Cython.Compiler.TreeFragment import TreeFragment
 from Cython.Compiler.StringEncoding import EncodedString
 from Cython.Compiler.Errors import CompileError
@@ -409,7 +410,7 @@ class WithTransform(CythonTransform):
         finally:
             if EXC:
                 EXIT(None, None, None)
-    """, temps=[u'MGR', u'EXC', u"EXIT", u"SYS)"],
+    """, temps=[u'MGR', u'EXC', u"EXIT"],
     pipeline=[NormalizeTree(None)])
 
     template_with_target = TreeFragment(u"""
@@ -428,32 +429,32 @@ class WithTransform(CythonTransform):
         finally:
             if EXC:
                 EXIT(None, None, None)
-    """, temps=[u'MGR', u'EXC', u"EXIT", u"VALUE", u"SYS"],
+    """, temps=[u'MGR', u'EXC', u"EXIT", u"VALUE"],
     pipeline=[NormalizeTree(None)])
 
     def visit_WithStatNode(self, node):
-        excinfo_name = temp_name_handle('EXCINFO')
-        excinfo_namenode = NameNode(pos=node.pos, name=excinfo_name)
-        excinfo_target = NameNode(pos=node.pos, name=excinfo_name)
+        excinfo_temp = TempHandle(PyrexTypes.py_object_type)
         if node.target is not None:
             result = self.template_with_target.substitute({
                 u'EXPR' : node.manager,
                 u'BODY' : node.body,
                 u'TARGET' : node.target,
-                u'EXCINFO' : excinfo_namenode
+                u'EXCINFO' : excinfo_temp.ref(node.pos)
                 }, pos=node.pos)
             # Set except excinfo target to EXCINFO
-            result.stats[4].body.stats[0].except_clauses[0].excinfo_target = excinfo_target
+            result.body.stats[4].body.stats[0].except_clauses[0].excinfo_target = (
+                excinfo_temp.ref(node.pos))
         else:
             result = self.template_without_target.substitute({
                 u'EXPR' : node.manager,
                 u'BODY' : node.body,
-                u'EXCINFO' : excinfo_namenode
+                u'EXCINFO' : excinfo_temp.ref(node.pos)
                 }, pos=node.pos)
             # Set except excinfo target to EXCINFO
-            result.stats[4].body.stats[0].except_clauses[0].excinfo_target = excinfo_target
+            result.body.stats[4].body.stats[0].except_clauses[0].excinfo_target = (
+                excinfo_temp.ref(node.pos))
 
-        return result.stats
+        return TempsBlockNode(node.pos, temps=[excinfo_temp], body=result)
 
 class DecoratorTransform(CythonTransform):
 

Cython/Compiler/Tests/TestParseTreeTransforms.py

@@ -95,20 +95,20 @@ class TestWithTransform(TransformTest):
 
         self.assertCode(u"""
 
-        $MGR = x
-        $EXIT = $MGR.__exit__
-        $MGR.__enter__()
-        $EXC = True
+        $1_0 = x
+        $1_2 = $1_0.__exit__
+        $1_0.__enter__()
+        $1_1 = True
         try:
             try:
                 y = z ** 3
             except:
-                $EXC = False
-                if (not $EXIT($EXCINFO)):
+                $1_1 = False
+                if (not $1_2($0_0)):
                     raise
         finally:
-            if $EXC:
-                $EXIT(None, None, None)
+            if $1_1:
+                $1_2(None, None, None)
 
         """, t)
 
@@ -119,21 +119,21 @@ class TestWithTransform(TransformTest):
         """)
         self.assertCode(u"""
 
-        $MGR = x
-        $EXIT = $MGR.__exit__
-        $VALUE = $MGR.__enter__()
-        $EXC = True
+        $1_0 = x
+        $1_2 = $1_0.__exit__
+        $1_3 = $1_0.__enter__()
+        $1_1 = True
         try:
             try:
-                y = $VALUE
+                y = $1_3
                 y = z ** 3
             except:
-                $EXC = False
-                if (not $EXIT($EXCINFO)):
+                $1_1 = False
+                if (not $1_2($0_0)):
                     raise
         finally:
-            if $EXC:
-                $EXIT(None, None, None)
+            if $1_1:
+                $1_2(None, None, None)
 
         """, t)
                           

Cython/Compiler/Tests/TestTreeFragment.py

 from Cython.TestUtils import CythonTest
 from Cython.Compiler.TreeFragment import *
 from Cython.Compiler.Nodes import *
+from Cython.Compiler.UtilNodes import *
 import Cython.Compiler.Naming as Naming
 
 class TestTreeFragments(CythonTest):
@@ -54,10 +55,10 @@ class TestTreeFragments(CythonTest):
             x = TMP
         """)
         T = F.substitute(temps=[u"TMP"])
-        s = T.stats
-        self.assert_(s[0].expr.name == Naming.temp_prefix +  u"1_TMP", s[0].expr.name)
-        self.assert_(s[1].rhs.name == Naming.temp_prefix + u"1_TMP")
-        self.assert_(s[0].expr.name !=  u"TMP")
+        s = T.body.stats
+        self.assert_(isinstance(s[0].expr, TempRefNode))
+        self.assert_(isinstance(s[1].rhs, TempRefNode))
+        self.assert_(s[0].expr.handle is s[1].rhs.handle)
 
 if __name__ == "__main__":
     import unittest

Cython/Compiler/TreeFragment.py

@@ -8,11 +8,12 @@ from Scanning import PyrexScanner, StringSourceDescriptor
 from Symtab import BuiltinScope, ModuleScope
 import Symtab
 import PyrexTypes
-from Visitor import VisitorTransform, temp_name_handle
+from Visitor import VisitorTransform
 from Nodes import Node, StatListNode
 from ExprNodes import NameNode
 import Parsing
 import Main
+import UtilNodes
 
 """
 Support for parsing strings into code trees.
@@ -111,12 +112,20 @@ class TemplateTransform(VisitorTransform):
 
     def __call__(self, node, substitutions, temps, pos):
         self.substitutions = substitutions
-        tempdict = {}
-        for key in temps:
-            tempdict[key] = temp_name_handle(key) # pending result_code refactor: Symtab.new_temp(PyrexTypes.py_object_type, key)
-        self.temp_key_to_entries = tempdict
         self.pos = pos
-        return super(TemplateTransform, self).__call__(node)
+        tempmap = {}
+        temphandles = []
+        for temp in temps:
+            handle = UtilNodes.TempHandle(PyrexTypes.py_object_type)
+            tempmap[temp] = handle
+            temphandles.append(handle)
+        self.tempmap = tempmap
+        result = super(TemplateTransform, self).__call__(node)
+        if temps:
+            result = UtilNodes.TempsBlockNode(self.get_pos(node),
+                                              temps=temphandles,
+                                              body=result)
+        return result
 
     def get_pos(self, node):
         if self.pos:
@@ -143,13 +152,11 @@ class TemplateTransform(VisitorTransform):
         else:
             return self.visit_Node(node) # make copy as usual
             
-    
     def visit_NameNode(self, node):
-        tempentry = self.temp_key_to_entries.get(node.name)
-        if tempentry is not None:
+        temphandle = self.tempmap.get(node.name)
+        if temphandle:
             # Replace name with temporary
-            return NameNode(self.get_pos(node), name=tempentry)
-            # Pending result_code refactor: return NameNode(self.get_pos(node), entry=tempentry)
+            return temphandle.ref(self.get_pos(node))
         else:
             return self.try_substitution(node, node.name)
 

Cython/Compiler/UtilNodes.py

+#
+# Nodes used as utilities and support for transforms etc.
+# These often make up sets including both Nodes and ExprNodes
+# so it is convenient to have them in a seperate module.
+#
+
+import Nodes
+import ExprNodes
+from Nodes import Node
+from ExprNodes import ExprNode
+
+class TempHandle(object):
+    temp = None
+    def __init__(self, type):
+        self.type = type
+
+    def ref(self, pos):
+        return TempRefNode(pos, handle=self, type=self.type)
+
+class TempRefNode(ExprNode):
+    # handle   TempHandle
+    subexprs = []
+
+    def analyse_types(self, env):
+        assert self.type == self.handle.type
+    
+    def analyse_target_types(self, env):
+        assert self.type == self.handle.type
+
+    def analyse_target_declaration(self, env):
+        pass
+
+    def calculate_result_code(self):
+        result = self.handle.temp
+        if result is None: result = "<error>" # might be called and overwritten
+        return result
+
+    def generate_result_code(self, code):
+        pass
+
+    def generate_assignment_code(self, rhs, code):
+        if self.type.is_pyobject:
+            rhs.make_owned_reference(code)
+            code.put_xdecref(self.result(), self.ctype())
+        code.putln('%s = %s;' % (self.result(), rhs.result_as(self.ctype())))
+        rhs.generate_post_assignment_code(code)
+
+class TempsBlockNode(Node):
+    """
+    Creates a block which allocates temporary variables.
+    This is used by transforms to output constructs that need
+    to make use of a temporary variable. Simply pass the types
+    of the needed temporaries to the constructor.
+    
+    The variables can be referred to using a TempRefNode
+    (which can be constructed by calling get_ref_node).
+    """
+
+    # temps   [TempHandle]
+    # body    StatNode
+    
+    child_attrs = ["body"]
+
+    def generate_execution_code(self, code):
+        for handle in self.temps:
+            handle.temp = code.funcstate.allocate_temp(handle.type)
+        self.body.generate_execution_code(code)
+        for handle in self.temps:
+            code.funcstate.release_temp(handle.temp)
+
+    def analyse_control_flow(self, env):
+        self.body.analyse_control_flow(env)
+
+    def analyse_declarations(self, env):
+        self.body.analyse_declarations(env)
+    
+    def analyse_expressions(self, env):
+        self.body.analyse_expressions(env)
+    
+    def generate_function_definitions(self, env, code):
+        self.body.generate_function_definitions(env, code)
+            
+    def annotate(self, code):
+        self.body.annotate(code)
+

Cython/Compiler/Visitor.py

@@ -199,23 +199,6 @@ def replace_node(ptr, value):
     else:
         getattr(parent, attrname)[listidx] = value
 
-tmpnamectr = 0
-def temp_name_handle(description=None):
-    global tmpnamectr
-    tmpnamectr += 1
-    if description is not None:
-        name = u"%d_%s" % (tmpnamectr, description)
-    else:
-        name = u"%d" % tmpnamectr
-    return EncodedString(Naming.temp_prefix + name)
-
-def get_temp_name_handle_desc(handle):
-    if not handle.startswith(u"__cyt_"):
-        return None
-    else:
-        idx = handle.find(u"_", 6)
-        return handle[idx+1:]
-    
 class PrintTree(TreeVisitor):
     """Prints a representation of the tree to standard output.
     Subclass and override repr_of to provide more information

Cython/TestUtils.py

@@ -47,10 +47,16 @@ class CythonTest(unittest.TestCase):
         self.assertEqual(len(expected), len(result),
             "Unmatched lines. Got:\n%s\nExpected:\n%s" % ("\n".join(expected), u"\n".join(result)))
 
-    def assertCode(self, expected, result_tree):
+    def codeToLines(self, tree):
         writer = CodeWriter()
-        writer.write(result_tree)
-        result_lines = writer.result.lines
+        writer.write(tree)
+        return writer.result.lines
+
+    def codeToString(self, tree):
+        return "\n".join(self.codeToLines(tree))
+
+    def assertCode(self, expected, result_tree):
+        result_lines = self.codeToLines(result_tree)
                 
         expected_lines = strip_common_indent(expected.split("\n"))
         

tests/run/nonecheck.pyx

@@ -32,6 +32,16 @@ Traceback (most recent call last):
    ...
 AttributeError: 'NoneType' object has no attribute 'a'
 
+>>> check_buffer_get(None)
+Traceback (most recent call last):
+   ...
+TypeError: 'NoneType' object is unsubscriptable
+
+>>> check_buffer_set(None)
+Traceback (most recent call last):
+   ...
+TypeError: 'NoneType' object is unsubscriptable
+
 """
 
 cimport cython
@@ -70,3 +80,11 @@ def check_and_assign(MyClass var):
         var = None
         print var.a
 
+@cython.nonecheck(True)
+def check_buffer_get(object[int] buf):
+    return buf[0]
+
+@cython.nonecheck(True)
+def check_buffer_set(object[int] buf):
+    buf[0] = 1
+