large merge of cython-dagss as of revision 764

Cython/CodeWriter.py

-from Cython.Compiler.Visitor import TreeVisitor
+from Cython.Compiler.Visitor import TreeVisitor, get_temp_name_handle_desc
 from Cython.Compiler.Nodes import *
+from Cython.Compiler.ExprNodes import *
 
 """
 Serializes a Cython code tree to Cython code. This is primarily useful for
@@ -35,6 +36,7 @@ class CodeWriter(TreeVisitor):
             result = LinesResult()
         self.result = result
         self.numindents = 0
+        self.tempnames = {}
     
     def write(self, tree):
         self.visit(tree)
@@ -57,6 +59,12 @@ class CodeWriter(TreeVisitor):
     def line(self, s):
         self.startline(s)
         self.endline()
+
+    def putname(self, name):
+        tmpdesc = get_temp_name_handle_desc(name)
+        if tmpdesc is not None:
+            name = self.tempnames.setdefault(tmpdesc, u"$" +tmpdesc)
+        self.put(name)
     
     def comma_seperated_list(self, items, output_rhs=False):
         if len(items) > 0:
@@ -116,7 +124,7 @@ class CodeWriter(TreeVisitor):
         self.endline()
     
     def visit_NameNode(self, node):
-        self.put(node.name)
+        self.putname(node.name)
     
     def visit_IntNode(self, node):
         self.put(node.value)
@@ -185,10 +193,23 @@ class CodeWriter(TreeVisitor):
         self.comma_seperated_list(node.args) # Might need to discover whether we need () around tuples...hmm...
     
     def visit_SimpleCallNode(self, node):
-        self.put(node.function.name + u"(")
+        self.visit(node.function)
+        self.put(u"(")
         self.comma_seperated_list(node.args)
         self.put(")")
 
+    def visit_GeneralCallNode(self, node):
+        self.visit(node.function)
+        self.put(u"(")
+        posarg = node.positional_args
+        if isinstance(posarg, AsTupleNode):
+            self.visit(posarg.arg)
+        else:
+            self.comma_seperated_list(posarg)
+        if node.keyword_args is not None or node.starstar_arg is not None:
+            raise Exception("Not implemented yet")
+        self.put(u")")
+
     def visit_ExprStatNode(self, node):
         self.startline()
         self.visit(node.expr)
@@ -197,9 +218,73 @@ class CodeWriter(TreeVisitor):
     def visit_InPlaceAssignmentNode(self, node):
         self.startline()
         self.visit(node.lhs)
-        self.put(" %s= " % node.operator)
+        self.put(u" %s= " % node.operator)
         self.visit(node.rhs)
         self.endline()
-    
-    
+        
+    def visit_WithStatNode(self, node):
+        self.startline()
+        self.put(u"with ")
+        self.visit(node.manager)
+        if node.target is not None:
+            self.put(u" as ")
+            self.visit(node.target)
+        self.endline(u":")
+        self.indent()
+        self.visit(node.body)
+        self.dedent()
+        
+    def visit_AttributeNode(self, node):
+        self.visit(node.obj)
+        self.put(u".%s" % node.attribute)
+
+    def visit_BoolNode(self, node):
+        self.put(str(node.value))
+
+    def visit_TryFinallyStatNode(self, node):
+        self.line(u"try:")
+        self.indent()
+        self.visit(node.body)
+        self.dedent()
+        self.line(u"finally:")
+        self.indent()
+        self.visit(node.finally_clause)
+        self.dedent()
+
+    def visit_TryExceptStatNode(self, node):
+        self.line(u"try:")
+        self.indent()
+        self.visit(node.body)
+        self.dedent()
+        for x in node.except_clauses:
+            self.visit(x)
+        if node.else_clause is not None:
+            self.visit(node.else_clause)
+
+    def visit_ExceptClauseNode(self, node):
+        self.startline(u"except")
+        if node.pattern is not None:
+            self.put(u" ")
+            self.visit(node.pattern)
+        if node.target is not None:
+            self.put(u", ")
+            self.visit(node.target)
+        self.endline(":")
+        self.indent()
+        self.visit(node.body)
+        self.dedent()
+
+    def visit_ReraiseStatNode(self, node):
+        self.line("raise")
+
+    def visit_NoneNode(self, node):
+        self.put(u"None")
+
+    def visit_ImportNode(self, node):
+        self.put(u"(import %s)" % node.module_name.value)
+
+    def visit_NotNode(self, node):
+        self.put(u"(not ")
+        self.visit(node.operand)
+        self.put(u")")
 

Cython/Compiler/Code.py

@@ -200,6 +200,8 @@ class CCodeWriter:
     def put_var_declaration(self, entry, static = 0, dll_linkage = None,
             definition = True):
         #print "Code.put_var_declaration:", entry.name, "definition =", definition ###
+        if entry.in_closure:
+            return
         visibility = entry.visibility
         if visibility == 'private' and not definition:
             #print "...private and not definition, skipping" ###

Cython/Compiler/Errors.py

@@ -32,6 +32,7 @@ class CompileError(PyrexError):
     
     def __init__(self, position = None, message = ""):
         self.position = position
+        self.message_only = message
     # Deprecated and withdrawn in 2.6:
     #   self.message = message
         if position:
@@ -91,6 +92,7 @@ def error(position, message):
     #print "Errors.error:", repr(position), repr(message) ###
     global num_errors
     err = CompileError(position, message)
+#    if position is not None: raise Exception(err) # debug
     line = "%s\n" % err
     if listing_file:
         listing_file.write(line)

Cython/Compiler/ExprNodes.py

@@ -972,7 +972,7 @@ class NameNode(AtomicExprNode):
             if entry.is_builtin:
                 namespace = Naming.builtins_cname
             else: # entry.is_pyglobal
-                namespace = entry.namespace_cname
+                namespace = entry.scope.namespace_cname
             code.putln(
                 '%s = __Pyx_GetName(%s, %s); %s' % (
                 self.result_code,
@@ -997,7 +997,7 @@ class NameNode(AtomicExprNode):
         # is_pyglobal seems to be True for module level-globals only.
         # We use this to access class->tp_dict if necessary.
         if entry.is_pyglobal:
-            namespace = self.entry.namespace_cname
+            namespace = self.entry.scope.namespace_cname
             if entry.is_member:
                 # if the entry is a member we have to cheat: SetAttr does not work
                 # on types, so we create a descriptor which is then added to tp_dict
@@ -1060,7 +1060,6 @@ class NameNode(AtomicExprNode):
             else:
                 code.annotate(pos, AnnotationItem('c_call', 'c function', size=len(self.name)))
             
-
 class BackquoteNode(ExprNode):
     #  `expr`
     #
@@ -1212,6 +1211,9 @@ class ExcValueNode(AtomicExprNode):
     def generate_result_code(self, code):
         pass
 
+    def analyse_types(self, env):
+        pass
+
 
 class TempNode(AtomicExprNode):
     #  Node created during analyse_types phase
@@ -1273,36 +1275,59 @@ class IndexNode(ExprNode):
         self.analyse_base_and_index_types(env, setting = 1)
     
     def analyse_base_and_index_types(self, env, getting = 0, setting = 0):
+        self.is_buffer_access = False
+
         self.base.analyse_types(env)
-        self.index.analyse_types(env)
-        if self.base.type.is_pyobject:
-            if self.index.type.is_int:
-                self.original_index_type = self.index.type
-                self.index = self.index.coerce_to(PyrexTypes.c_py_ssize_t_type, env).coerce_to_simple(env)
-                if getting:
-                    env.use_utility_code(getitem_int_utility_code)
-                if setting:
-                    env.use_utility_code(setitem_int_utility_code)
+        
+        if self.base.type.buffer_options is not None:
+            if isinstance(self.index, TupleNode):
+                indices = self.index.args
+#                is_int_indices = 0 == sum([1 for i in self.index.args if not i.type.is_int])
             else:
-                self.index = self.index.coerce_to_pyobject(env)
-            self.type = py_object_type
-            self.gil_check(env)
-            self.is_temp = 1
-        else:
-            if self.base.type.is_ptr or self.base.type.is_array:
-                self.type = self.base.type.base_type
+#                is_int_indices = self.index.type.is_int
+                indices = [self.index]
+            all_ints = True
+            for index in indices:
+                index.analyse_types(env)
+                if not index.type.is_int:
+                    all_ints = False
+            if all_ints:
+                self.indices = indices
+                self.index = None
+                self.type = self.base.type.buffer_options.dtype
+                self.is_temp = 1
+                self.is_buffer_access = True
+
+        if not self.is_buffer_access:
+            self.index.analyse_types(env) # ok to analyse as tuple
+            if self.base.type.is_pyobject:
+                if self.index.type.is_int:
+                    self.original_index_type = self.index.type
+                    self.index = self.index.coerce_to(PyrexTypes.c_py_ssize_t_type, env).coerce_to_simple(env)
+                    if getting:
+                        env.use_utility_code(getitem_int_utility_code)
+                    if setting:
+                        env.use_utility_code(setitem_int_utility_code)
+                else:
+                    self.index = self.index.coerce_to_pyobject(env)
+                self.type = py_object_type
+                self.gil_check(env)
+                self.is_temp = 1
             else:
-                error(self.pos,
-                    "Attempting to index non-array type '%s'" %
-                        self.base.type)
-                self.type = PyrexTypes.error_type
-            if self.index.type.is_pyobject:
-                self.index = self.index.coerce_to(
-                    PyrexTypes.c_py_ssize_t_type, env)
-            if not self.index.type.is_int:
-                error(self.pos,
-                    "Invalid index type '%s'" %
-                        self.index.type)
+                if self.base.type.is_ptr or self.base.type.is_array:
+                    self.type = self.base.type.base_type
+                else:
+                    error(self.pos,
+                        "Attempting to index non-array type '%s'" %
+                            self.base.type)
+                    self.type = PyrexTypes.error_type
+                if self.index.type.is_pyobject:
+                    self.index = self.index.coerce_to(
+                        PyrexTypes.c_py_ssize_t_type, env)
+                if not self.index.type.is_int:
+                    error(self.pos,
+                        "Invalid index type '%s'" %
+                            self.index.type)
 
     gil_message = "Indexing Python object"
 
@@ -1328,11 +1353,17 @@ class IndexNode(ExprNode):
 
     def generate_subexpr_evaluation_code(self, code):
         self.base.generate_evaluation_code(code)
-        self.index.generate_evaluation_code(code)
+        if self.index is not None:
+            self.index.generate_evaluation_code(code)
+        else:
+            for i in self.indices: i.generate_evaluation_code(code)
         
     def generate_subexpr_disposal_code(self, code):
         self.base.generate_disposal_code(code)
-        self.index.generate_disposal_code(code)
+        if self.index is not None:
+            self.index.generate_disposal_code(code)
+        else:
+            for i in self.indices: i.generate_disposal_code(code)
 
     def generate_result_code(self, code):
         if self.type.is_pyobject:

Cython/Compiler/Future.py

@@ -7,5 +7,6 @@ def _get_feature(name):
         return object()
 
 unicode_literals = _get_feature("unicode_literals")
+with_statement = _get_feature("with_statement")
 
 del _get_feature

Cython/Compiler/ModuleNode.py

@@ -25,6 +25,10 @@ from PyrexTypes import py_object_type
 from Cython.Utils import open_new_file, replace_suffix
 
 
+def check_c_classes(module_node):
+    module_node.scope.check_c_classes()
+    return module_node
+
 class ModuleNode(Nodes.Node, Nodes.BlockNode):
     #  doc       string or None
     #  body      StatListNode
@@ -32,6 +36,9 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
     #  referenced_modules   [ModuleScope]
     #  module_temp_cname    string
     #  full_module_name     string
+    #
+    #  scope                The module scope.
+    #  compilation_source   A CompilationSource (see Main)
 
     child_attrs = ["body"]
     
@@ -44,10 +51,8 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
             env.doc = self.doc
         self.body.analyse_declarations(env)
     
-    def process_implementation(self, env, options, result):
-        self.analyse_declarations(env)
-        env.check_c_classes()
-        self.body.analyse_expressions(env)
+    def process_implementation(self, options, result):
+        env = self.scope
         env.return_type = PyrexTypes.c_void_type
         self.referenced_modules = []
         self.find_referenced_modules(env, self.referenced_modules, {})
@@ -254,6 +259,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
         self.generate_module_cleanup_func(env, code)
         self.generate_filename_table(code)
         self.generate_utility_functions(env, code)
+        self.generate_buffer_compatability_functions(env, code)
 
         self.generate_declarations_for_modules(env, modules, code.h)
 
@@ -433,6 +439,9 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
         code.putln("  #define PyBUF_F_CONTIGUOUS (0x0040 | PyBUF_STRIDES)")
         code.putln("  #define PyBUF_ANY_CONTIGUOUS (0x0080 | PyBUF_STRIDES)")
         code.putln("  #define PyBUF_INDIRECT (0x0100 | PyBUF_STRIDES)")
+        code.putln("")
+        code.putln("  static int PyObject_GetBuffer(PyObject *obj, Py_buffer *view, int flags);")
+        code.putln("  static void PyObject_ReleaseBuffer(PyObject *obj, Py_buffer *view);")
         code.putln("#endif")
 
         code.put(builtin_module_name_utility_code[0])
@@ -1940,6 +1949,60 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
             code.h.put(utility_code[0])
             code.put(utility_code[1])
         code.put(PyrexTypes.type_conversion_functions)
+        code.putln("")
+
+    def generate_buffer_compatability_functions(self, env, code):
+        # will be refactored
+        code.put("""
+static int numpy_getbuffer(PyObject *obj, Py_buffer *view, int flags) {
+  /* This function is always called after a type-check */
+  PyArrayObject *arr = (PyArrayObject*)obj;
+  PyArray_Descr *type = (PyArray_Descr*)arr->descr;
+  
+  view->buf = arr->data;
+  view->readonly = 0; /*fixme*/
+  view->format = "B"; /*fixme*/
+  view->ndim = arr->nd;
+  view->strides = arr->strides;
+  view->shape = arr->dimensions;
+  view->suboffsets = 0;
+  
+  view->itemsize = type->elsize;
+  view->internal = 0;
+  return 0;
+}
+
+static void numpy_releasebuffer(PyObject *obj, Py_buffer *view) {
+}
+
+""")
+        
+        # For now, hard-code numpy imported as "numpy"
+        ndarrtype = env.entries[u'numpy'].as_module.entries['ndarray'].type
+        types = [
+            (ndarrtype.typeptr_cname, "numpy_getbuffer", "numpy_releasebuffer")
+        ]
+        
+#        typeptr_cname = ndarrtype.typeptr_cname
+        code.putln("static int PyObject_GetBuffer(PyObject *obj, Py_buffer *view, int flags) {")
+        clause = "if"
+        for t, get, release in types:
+            code.putln("%s (__Pyx_TypeTest(obj, %s)) return %s(obj, view, flags);" % (clause, t, get))
+            clause = "else if"
+        code.putln("else {")
+        code.putln("PyErr_Format(PyExc_TypeError, \"'%100s' does not have the buffer interface\", Py_TYPE(obj)->tp_name);")
+        code.putln("return -1;")
+        code.putln("}")
+        code.putln("}")
+        code.putln("")
+        code.putln("static void PyObject_ReleaseBuffer(PyObject *obj, Py_buffer *view) {")
+        clause = "if"
+        for t, get, release in types:
+            code.putln("%s (__Pyx_TypeTest(obj, %s)) %s(obj, view);" % (clause, t, release))
+            clause = "else if"
+        code.putln("}")
+        code.putln("")
+
 
 #------------------------------------------------------------------------------------
 #

Cython/Compiler/Naming.py

@@ -8,6 +8,8 @@
 
 pyrex_prefix    = "__pyx_"
 
+temp_prefix       = u"__cyt_"
+
 builtin_prefix    = pyrex_prefix + "builtin_"
 arg_prefix        = pyrex_prefix + "arg_"
 funcdoc_prefix    = pyrex_prefix + "doc_"
@@ -70,6 +72,8 @@ optional_args_cname = pyrex_prefix + "optional_args"
 no_opt_args      = pyrex_prefix + "no_opt_args"
 import_star      = pyrex_prefix + "import_star"
 import_star_set  = pyrex_prefix + "import_star_set"
+cur_scope_cname  = pyrex_prefix + "cur_scope"
+enc_scope_cname  = pyrex_prefix + "enc_scope"
 
 line_c_macro = "__LINE__"
 

Cython/Compiler/Parsing.py

@@ -312,6 +312,7 @@ def p_call(s, function):
         if s.sy != ',':
             break
         s.next()
+
     if s.sy == '*':
         s.next()
         star_arg = p_simple_expr(s)
@@ -1159,13 +1160,13 @@ def p_for_from_step(s):
 
 inequality_relations = ('<', '<=', '>', '>=')
 
-def p_for_target(s):
+def p_target(s, terminator):
     pos = s.position()
     expr = p_bit_expr(s)
     if s.sy == ',':
         s.next()
         exprs = [expr]
-        while s.sy != 'in':
+        while s.sy != terminator:
             exprs.append(p_bit_expr(s))
             if s.sy != ',':
                 break
@@ -1174,6 +1175,9 @@ def p_for_target(s):
     else:
         return expr
 
+def p_for_target(s):
+    return p_target(s, 'in')
+
 def p_for_iterator(s):
     pos = s.position()
     expr = p_testlist(s)
@@ -1253,8 +1257,17 @@ def p_with_statement(s):
         body = p_suite(s)
         return Nodes.GILStatNode(pos, state = state, body = body)
     else:
-        s.error("Only 'with gil' and 'with nogil' implemented",
-                pos = pos)
+        manager = p_expr(s)
+        target = None
+        if s.sy == 'IDENT' and s.systring == 'as':
+            s.next()
+            allow_multi = (s.sy == '(')
+            target = p_target(s, ':')
+            if not allow_multi and isinstance(target, ExprNodes.TupleNode):
+                s.error("Multiple with statement target values not allowed without paranthesis")
+        body = p_suite(s)
+	return Nodes.WithStatNode(pos, manager = manager, 
+	       			       target = target, body = body)
     
 def p_simple_statement(s, first_statement = 0):
     #print "p_simple_statement:", s.sy, s.systring ###
@@ -1447,6 +1460,71 @@ def p_suite(s, ctx = Ctx(), with_doc = 0, with_pseudo_doc = 0):
     else:
         return body
 
+def p_positional_and_keyword_args(s, end_sy_set, type_positions=(), type_keywords=()):
+    """
+    Parses positional and keyword arguments. end_sy_set
+    should contain any s.sy that terminate the argument list.
+    Argument expansion (* and **) are not allowed.
+
+    type_positions and type_keywords specifies which argument
+    positions and/or names which should be interpreted as
+    types. Other arguments will be treated as expressions.
+
+    Returns: (positional_args, keyword_args)
+    """
+    positional_args = []
+    keyword_args = []
+    pos_idx = 0
+
+    while s.sy not in end_sy_set:
+        if s.sy == '*' or s.sy == '**':
+            s.error('Argument expansion not allowed here.')
+
+        was_keyword = False
+        parsed_type = False
+        if s.sy == 'IDENT':
+            # Since we can have either types or expressions as positional args,
+            # we use a strategy of looking an extra step forward for a '=' and
+            # if it is a positional arg we backtrack.
+            ident = s.systring
+            s.next()
+            if s.sy == '=':
+                s.next()
+                # Is keyword arg
+                if ident in type_keywords:
+                    arg = p_c_base_type(s)
+                    parsed_type = True
+                else:
+                    arg = p_simple_expr(s)
+                keyword_node = ExprNodes.IdentifierStringNode(arg.pos,
+                                value = Utils.EncodedString(ident))
+                keyword_args.append((keyword_node, arg))
+                was_keyword = True
+            else:
+                s.put_back('IDENT', ident)
+                
+        if not was_keyword:
+            if pos_idx in type_positions:
+                arg = p_c_base_type(s)
+                parsed_type = True
+            else:
+                arg = p_simple_expr(s)
+            positional_args.append(arg)
+            pos_idx += 1
+            if len(keyword_args) > 0:
+                s.error("Non-keyword arg following keyword arg",
+                        pos = arg.pos)
+
+        if s.sy != ',':
+            if s.sy not in end_sy_set:
+                if parsed_type:
+                    s.error("Expected: type")
+                else:
+                    s.error("Expected: expression")
+            break
+        s.next()
+    return positional_args, keyword_args
+
 def p_c_base_type(s, self_flag = 0, nonempty = 0):
     # If self_flag is true, this is the base type for the
     # self argument of a C method of an extension type.
@@ -1519,11 +1597,43 @@ def p_c_simple_base_type(s, self_flag, nonempty):
     else:
         #print "p_c_simple_base_type: not looking at type at", s.position()
         name = None
-    return Nodes.CSimpleBaseTypeNode(pos, 
+
+    type_node = Nodes.CSimpleBaseTypeNode(pos, 
         name = name, module_path = module_path,
         is_basic_c_type = is_basic, signed = signed,
         longness = longness, is_self_arg = self_flag)
 
+
+    # Treat trailing [] on type as buffer access
+    if s.sy == '[':
+        if is_basic:
+            p.error("Basic C types do not support buffer access")
+        return p_buffer_access(s, type_node)
+    else:
+        return type_node
+
+def p_buffer_access(s, type_node):
+    # s.sy == '['
+    pos = s.position()
+    s.next()
+    positional_args, keyword_args = (
+        p_positional_and_keyword_args(s, (']',), (0,), ('dtype',))
+    )
+    s.expect(']')
+
+    keyword_dict = ExprNodes.DictNode(pos,
+        key_value_pairs = [
+            ExprNodes.DictItemNode(pos=key.pos, key=key, value=value)
+            for key, value in keyword_args
+        ])
+
+    result = Nodes.CBufferAccessTypeNode(pos,
+        positional_args = positional_args,
+        keyword_args = keyword_dict,
+        base_type_node = type_node)
+    return result
+    
+
 def looking_at_type(s):
     return looking_at_base_type(s) or s.looking_at_type_name()
 

Cython/Compiler/PyrexTypes.py

@@ -4,6 +4,23 @@
 
 from Cython import Utils
 import Naming
+import copy
+
+class BufferOptions:
+    # dtype         PyrexType
+    # ndim          int
+    def __init__(self, dtype, ndim):
+        self.dtype = dtype
+        self.ndim = ndim
+
+
+def create_buffer_type(base_type, buffer_options):
+    # Make a shallow copy of base_type and then annotate it
+    # with the buffer information
+    result = copy.copy(base_type)
+    result.buffer_options = buffer_options
+    return result
+
 
 class BaseType:
     #
@@ -92,6 +109,7 @@ class PyrexType(BaseType):
     default_value = ""
     parsetuple_format = ""
     pymemberdef_typecode = None
+    buffer_options = None # can contain a BufferOptions instance
     
     def resolve(self):
         # If a typedef, returns the base type.
@@ -183,7 +201,6 @@ class CTypedefType(BaseType):
     def __getattr__(self, name):
         return getattr(self.typedef_base_type, name)
 
-
 class PyObjectType(PyrexType):
     #
     #  Base class for all Python object types (reference-counted).

Cython/Compiler/Symtab.py

 #
-#   Pyrex - Symbol Table
+#   Symbol Table
 #
 
 import re
@@ -19,6 +19,14 @@ import __builtin__
 possible_identifier = re.compile(ur"(?![0-9])\w+$", re.U).match
 nice_identifier = re.compile('^[a-zA-Z0-0_]+$').match
 
+class BufferAux:
+    def __init__(self, buffer_info_var, stridevars, shapevars):
+        self.buffer_info_var = buffer_info_var
+        self.stridevars = stridevars
+        self.shapevars = shapevars
+    def __repr__(self):
+        return "<BufferAux %r>" % self.__dict__
+
 class Entry:
     # A symbol table entry in a Scope or ModuleNamespace.
     #
@@ -47,6 +55,7 @@ class Entry:
     # is_self_arg      boolean    Is the "self" arg of an exttype method
     # is_arg           boolean    Is the arg of a method
     # is_local         boolean    Is a local variable
+    # in_closure       boolean    Is referenced in an inner scope
     # is_readonly      boolean    Can't be assigned to
     # func_cname       string     C func implementing Python func
     # pos              position   Source position where declared
@@ -75,6 +84,8 @@ class Entry:
     # defined_in_pxd   boolean    Is defined in a .pxd file (not just declared)
     # api              boolean    Generate C API for C class or function
     # utility_code     string     Utility code needed when this entry is used
+    #
+    # buffer_aux      BufferAux or None  Extra information needed for buffer variables
 
     borrowed = 0
     init = ""
@@ -96,6 +107,7 @@ class Entry:
     is_self_arg = 0
     is_arg = 0
     is_local = 0
+    in_closure = 0
     is_declared_generic = 0
     is_readonly = 0
     func_cname = None
@@ -115,6 +127,7 @@ class Entry:
     api = 0
     utility_code = None
     is_overridable = 0
+    buffer_aux = None
 
     def __init__(self, name, cname, type, pos = None, init = None):
         self.name = name
@@ -163,6 +176,8 @@ class Scope:
     in_cinclude = 0
     nogil = 0
     
+    temp_prefix = Naming.pyrex_prefix
+    
     def __init__(self, name, outer_scope, parent_scope):
         # The outer_scope is the next scope in the lookup chain.
         # The parent_scope is used to derive the qualified name of this scope.
@@ -447,7 +462,14 @@ class Scope:
         # Look up name in this scope or an enclosing one.
         # Return None if not found.
         return (self.lookup_here(name)
-            or (self.outer_scope and self.outer_scope.lookup(name))
+            or (self.outer_scope and self.outer_scope.lookup_from_inner(name))
+            or None)
+
+    def lookup_from_inner(self, name):
+        # Look up name in this scope or an enclosing one.
+        # This is only called from enclosing scopes.
+        return (self.lookup_here(name)
+            or (self.outer_scope and self.outer_scope.lookup_from_inner(name))
             or None)
 
     def lookup_here(self, name):
@@ -562,7 +584,7 @@ class Scope:
                 return entry.cname
         n = self.temp_counter
         self.temp_counter = n + 1
-        cname = "%s%d" % (Naming.pyrex_prefix, n)
+        cname = "%s%d" % (self.temp_prefix, n)
         entry = Entry("", cname, type)
         entry.used = 1
         if type.is_pyobject or type == PyrexTypes.c_py_ssize_t_type:
@@ -608,6 +630,9 @@ class Scope:
         return 0
 
 class PreImportScope(Scope):
+
+    namespace_cname = Naming.preimport_cname
+
     def __init__(self):
         Scope.__init__(self, Options.pre_import, None, None)
         
@@ -615,7 +640,6 @@ class PreImportScope(Scope):
         entry = self.declare(name, name, py_object_type, pos)
         entry.is_variable = True
         entry.is_pyglobal = True
-        entry.namespace_cname = Naming.preimport_cname
         return entry
 
 
@@ -761,6 +785,7 @@ class ModuleScope(Scope):
         self.has_extern_class = 0
         self.cached_builtins = []
         self.undeclared_cached_builtins = []
+        self.namespace_cname = self.module_cname
     
     def qualifying_scope(self):
         return self.parent_module
@@ -876,7 +901,6 @@ class ModuleScope(Scope):
                 raise InternalError(
                     "Non-cdef global variable is not a generic Python object")
             entry.is_pyglobal = 1
-            entry.namespace_cname = self.module_cname
         else:
             entry.is_cglobal = 1
             self.var_entries.append(entry)
@@ -1075,8 +1099,7 @@ class ModuleScope(Scope):
         var_entry.is_readonly = 1
         entry.as_variable = var_entry
         
-
-class LocalScope(Scope):
+class LocalScope(Scope):    
 
     def __init__(self, name, outer_scope):
         Scope.__init__(self, name, outer_scope, outer_scope)
@@ -1119,6 +1142,33 @@ class LocalScope(Scope):
             entry = self.global_scope().lookup_target(name)
             self.entries[name] = entry
         
+    def lookup_from_inner(self, name):
+        entry = self.lookup_here(name)
+        if entry:
+            entry.in_closure = 1
+            return entry
+        else:
+            return (self.outer_scope and self.outer_scope.lookup_from_inner(name)) or None
+            
+    def mangle_closure_cnames(self, scope_var):
+        for entry in self.entries.values():
+            if entry.in_closure:
+                if not hasattr(entry, 'orig_cname'):
+                    entry.orig_cname = entry.cname
+                entry.cname = scope_var + "->" + entry.cname
+                
+
+class GeneratorLocalScope(LocalScope):
+
+    temp_prefix = Naming.cur_scope_cname + "->" + LocalScope.temp_prefix
+    
+    def mangle_closure_cnames(self, scope_var):
+        for entry in self.entries.values() + self.temp_entries:
+            entry.in_closure = 1
+        LocalScope.mangle_closure_cnames(self, scope_var)
+    
+#    def mangle(self, prefix, name):
+#        return "%s->%s" % (Naming.scope_obj_cname, name)
 
 class StructOrUnionScope(Scope):
     #  Namespace of a C struct or union.
@@ -1198,7 +1248,6 @@ class PyClassScope(ClassScope):
         entry = Scope.declare_var(self, name, type, pos, 
             cname, visibility, is_cdef)
         entry.is_pyglobal = 1
-        entry.namespace_cname = self.class_obj_cname
         return entry
 
     def allocate_temp(self, type):
@@ -1295,7 +1344,7 @@ class CClassScope(ClassScope):
             entry.is_pyglobal = 1 # xxx: is_pyglobal changes behaviour in so many places that
                                   # I keep it in for now. is_member should be enough
                                   # later on
-            entry.namespace_cname = "(PyObject *)%s" % self.parent_type.typeptr_cname
+            self.namespace_cname = "(PyObject *)%s" % self.parent_type.typeptr_cname
             entry.interned_cname = self.intern_identifier(name)
             return entry
 

Cython/Compiler/Tests/TestBuffer.py

+from Cython.TestUtils import CythonTest
+import Cython.Compiler.Errors as Errors
+from Cython.Compiler.Nodes import *
+from Cython.Compiler.ParseTreeTransforms import *
+
+
+class TestBufferParsing(CythonTest):
+    # First, we only test the raw parser, i.e.
+    # the number and contents of arguments are NOT checked.
+    # However "dtype"/the first positional argument is special-cased
+    #  to parse a type argument rather than an expression
+
+    def parse(self, s):
+        return self.should_not_fail(lambda: self.fragment(s)).root
+
+    def not_parseable(self, expected_error, s):
+        e = self.should_fail(lambda: self.fragment(s),  Errors.CompileError)
+        self.assertEqual(expected_error, e.message_only)
+    
+    def test_basic(self):
+        t = self.parse(u"cdef object[float, 4, ndim=2, foo=foo] x")
+        bufnode = t.stats[0].base_type
+        self.assert_(isinstance(bufnode, CBufferAccessTypeNode))
+        self.assertEqual(2, len(bufnode.positional_args))
+#        print bufnode.dump()
+        # should put more here...
+        
+    def test_type_fail(self):
+        self.not_parseable("Expected: type",
+                           u"cdef object[2] x")
+    
+    def test_type_pos(self):
+        self.parse(u"cdef object[short unsigned int, 3] x")
+
+    def test_type_keyword(self):
+        self.parse(u"cdef object[foo=foo, dtype=short unsigned int] x")
+
+    def test_notype_as_expr1(self):
+        self.not_parseable("Expected: expression",
+                           u"cdef object[foo2=short unsigned int] x")
+
+    def test_notype_as_expr2(self):
+        self.not_parseable("Expected: expression",
+                           u"cdef object[int, short unsigned int] x")
+
+    def test_pos_after_key(self):
+        self.not_parseable("Non-keyword arg following keyword arg",
+                           u"cdef object[foo=1, 2] x")
+
+class TestBufferOptions(CythonTest):
+    # Tests the full parsing of the options within the brackets
+
+    def parse_opts(self, opts):
+        s = u"cdef object[%s] x" % opts
+        root = self.fragment(s, pipeline=[PostParse(self)]).root
+        buftype = root.stats[0].base_type
+        self.assert_(isinstance(buftype, CBufferAccessTypeNode))
+        self.assert_(isinstance(buftype.base_type_node, CSimpleBaseTypeNode))
+        self.assertEqual(u"object", buftype.base_type_node.name)
+        return buftype
+
+    def non_parse(self, expected_err, opts):
+        e = self.should_fail(lambda: self.parse_opts(opts))
+        self.assertEqual(expected_err, e.message_only)
+        
+    def test_basic(self):
+        buf = self.parse_opts(u"unsigned short int, 3")
+        self.assert_(isinstance(buf.dtype_node, CSimpleBaseTypeNode))
+        self.assert_(buf.dtype_node.signed == 0 and buf.dtype_node.longness == -1)
+        self.assertEqual(3, buf.ndim)
+
+    def test_dict(self):
+        buf = self.parse_opts(u"ndim=3, dtype=unsigned short int")
+        self.assert_(isinstance(buf.dtype_node, CSimpleBaseTypeNode))
+        self.assert_(buf.dtype_node.signed == 0 and buf.dtype_node.longness == -1)
+        self.assertEqual(3, buf.ndim)
+        
+    def test_dtype(self):
+        self.non_parse(ERR_BUF_MISSING % 'dtype', u"")
+    
+    def test_ndim(self):
+        self.parse_opts(u"int, 2")
+        self.non_parse(ERR_BUF_INT % 'ndim', u"int, 'a'")
+        self.non_parse(ERR_BUF_NONNEG % 'ndim', u"int, -34")
+
+    def test_use_DEF(self):
+        t = self.fragment(u"""
+        DEF ndim = 3
+        cdef object[int, ndim] x
+        cdef object[ndim=ndim, dtype=int] y
+        """, pipeline=[PostParse(self)]).root
+        self.assert_(t.stats[1].base_type.ndim == 3)
+        self.assert_(t.stats[2].base_type.ndim == 3)
+
+    # add exotic and impossible combinations as they come along

Cython/Compiler/Tests/TestParseTreeTransforms.py

+from Cython.TestUtils import TransformTest
+from Cython.Compiler.ParseTreeTransforms import *
+from Cython.Compiler.Nodes import *
+
+class TestNormalizeTree(TransformTest):
+    def test_parserbehaviour_is_what_we_coded_for(self):
+        t = self.fragment(u"if x: y").root
+        self.assertLines(u"""
+(root): StatListNode
+  stats[0]: IfStatNode
+    if_clauses[0]: IfClauseNode
+      condition: NameNode
+      body: ExprStatNode
+        expr: NameNode
+""", self.treetypes(t))
+        
+    def test_wrap_singlestat(self):
+    	t = self.run_pipeline([NormalizeTree(None)], u"if x: y")
+        self.assertLines(u"""
+(root): StatListNode
+  stats[0]: IfStatNode
+    if_clauses[0]: IfClauseNode
+      condition: NameNode
+      body: StatListNode
+        stats[0]: ExprStatNode
+          expr: NameNode
+""", self.treetypes(t))
+
+    def test_wrap_multistat(self):
+        t = self.run_pipeline([NormalizeTree(None)], u"""
+            if z:
+                x
+                y
+        """)
+        self.assertLines(u"""
+(root): StatListNode
+  stats[0]: IfStatNode
+    if_clauses[0]: IfClauseNode
+      condition: NameNode
+      body: StatListNode
+        stats[0]: ExprStatNode
+          expr: NameNode
+        stats[1]: ExprStatNode
+          expr: NameNode
+""", self.treetypes(t))
+
+    def test_statinexpr(self):
+        t = self.run_pipeline([NormalizeTree(None)], u"""
+            a, b = x, y
+        """)
+        self.assertLines(u"""
+(root): StatListNode
+  stats[0]: ParallelAssignmentNode
+    stats[0]: SingleAssignmentNode
+      lhs: NameNode
+      rhs: NameNode
+    stats[1]: SingleAssignmentNode
+      lhs: NameNode
+      rhs: NameNode
+""", self.treetypes(t))
+
+    def test_wrap_offagain(self):
+        t = self.run_pipeline([NormalizeTree(None)], u"""
+            x
+            y
+            if z:
+                x
+        """)
+        self.assertLines(u"""
+(root): StatListNode
+  stats[0]: ExprStatNode
+    expr: NameNode
+  stats[1]: ExprStatNode
+    expr: NameNode
+  stats[2]: IfStatNode
+    if_clauses[0]: IfClauseNode
+      condition: NameNode
+      body: StatListNode
+        stats[0]: ExprStatNode
+          expr: NameNode
+""", self.treetypes(t))
+        
+
+    def test_pass_eliminated(self):
+        t = self.run_pipeline([NormalizeTree(None)], u"pass")
+        self.assert_(len(t.stats) == 0)
+
+class TestWithTransform(TransformTest):
+
+    def test_simplified(self):
+        t = self.run_pipeline([WithTransform(None)], u"""
+        with x:
+            y = z ** 3
+        """)
+        
+        self.assertCode(u"""
+
+        $MGR = x
+        $EXIT = $MGR.__exit__
+        $MGR.__enter__()
+        $EXC = True
+        try:
+            try:
+                y = z ** 3
+            except:
+                $EXC = False
+                if (not $EXIT($EXCINFO)):
+                    raise
+        finally:
+            if $EXC:
+                $EXIT(None, None, None)
+
+        """, t)
+
+    def test_basic(self):
+        t = self.run_pipeline([WithTransform(None)], u"""
+        with x as y:
+            y = z ** 3
+        """)
+        self.assertCode(u"""
+
+        $MGR = x
+        $EXIT = $MGR.__exit__
+        $VALUE = $MGR.__enter__()
+        $EXC = True
+        try:
+            try:
+                y = $VALUE
+                y = z ** 3
+            except:
+                $EXC = False
+                if (not $EXIT($EXCINFO)):
+                    raise
+        finally:
+            if $EXC:
+                $EXIT(None, None, None)
+
+        """, t)
+                          
+
+if __name__ == "__main__":
+    import unittest
+    unittest.main()

Cython/Compiler/Tests/TestTreeFragment.py

 from Cython.TestUtils import CythonTest
 from Cython.Compiler.TreeFragment import *
 from Cython.Compiler.Nodes import *
+import Cython.Compiler.Naming as Naming
 
 class TestTreeFragments(CythonTest):
+    
     def test_basic(self):
         F = self.fragment(u"x = 4")
         T = F.copy()
@@ -12,15 +14,15 @@ class TestTreeFragments(CythonTest):
         F = self.fragment(u"if True: x = 4")
         T1 = F.root
         T2 = F.copy()
-        self.assertEqual("x", T2.body.if_clauses[0].body.lhs.name)
-        T2.body.if_clauses[0].body.lhs.name = "other"
-        self.assertEqual("x", T1.body.if_clauses[0].body.lhs.name)
+        self.assertEqual("x", T2.stats[0].if_clauses[0].body.lhs.name)
+        T2.stats[0].if_clauses[0].body.lhs.name = "other"
+        self.assertEqual("x", T1.stats[0].if_clauses[0].body.lhs.name)
 
     def test_substitutions_are_copied(self):
         T = self.fragment(u"y + y").substitute({"y": NameNode(pos=None, name="x")})
-        self.assertEqual("x", T.body.expr.operand1.name)
-        self.assertEqual("x", T.body.expr.operand2.name)
-        self.assert_(T.body.expr.operand1 is not T.body.expr.operand2)
+        self.assertEqual("x", T.stats[0].expr.operand1.name)
+        self.assertEqual("x", T.stats[0].expr.operand2.name)
+        self.assert_(T.stats[0].expr.operand1 is not T.stats[0].expr.operand2)
 
     def test_substitution(self):
         F = self.fragment(u"x = 4")
@@ -32,7 +34,7 @@ class TestTreeFragments(CythonTest):
         F = self.fragment(u"PASS")
         pass_stat = PassStatNode(pos=None)
         T = F.substitute({"PASS" : pass_stat})
-        self.assert_(isinstance(T.body, PassStatNode), T.body)
+        self.assert_(isinstance(T.stats[0], PassStatNode), T)
 
     def test_pos_is_transferred(self):
         F = self.fragment(u"""
@@ -40,11 +42,22 @@ class TestTreeFragments(CythonTest):
         x = u * v ** w
         """)
         T = F.substitute({"v" : NameNode(pos=None, name="a")})
-        v = F.root.body.stats[1].rhs.operand2.operand1
-        a = T.body.stats[1].rhs.operand2.operand1
+        v = F.root.stats[1].rhs.operand2.operand1
+        a = T.stats[1].rhs.operand2.operand1
         self.assertEquals(v.pos, a.pos)
         
-        
+    def test_temps(self):
+        import Cython.Compiler.Visitor as v
+        v.tmpnamectr = 0
+        F = self.fragment(u"""
+            TMP
+            x = TMP
+        """)
+        T = F.substitute(temps=[u"TMP"])
+        s = T.stats
+        self.assert_(s[0].expr.name == Naming.temp_prefix +  u"1_TMP", s[0].expr.name)
+        self.assert_(s[1].rhs.name == Naming.temp_prefix + u"1_TMP")
+        self.assert_(s[0].expr.name !=  u"TMP")
 
 if __name__ == "__main__":
     import unittest

Cython/Compiler/TreeFragment.py

@@ -6,8 +6,8 @@ import re
 from cStringIO import StringIO
 from Scanning import PyrexScanner, StringSourceDescriptor
 from Symtab import BuiltinScope, ModuleScope
-from Visitor import VisitorTransform
-from Nodes import Node
+from Visitor import VisitorTransform, temp_name_handle
+from Nodes import Node, StatListNode
 from ExprNodes import NameNode
 import Parsing
 import Main
@@ -53,7 +53,7 @@ def parse_from_strings(name, code, pxds={}):
     buf = StringIO(code.encode(encoding))
 
     scanner = PyrexScanner(buf, code_source, source_encoding = encoding,
-                     type_names = scope.type_names, context = context)
+                     scope = scope, context = context)
     tree = Parsing.p_module(scanner, 0, module_name)
     return tree
 
@@ -92,27 +92,56 @@ class TemplateTransform(VisitorTransform):
        if its name is listed in the substitutions dictionary in the
        same way. It is the responsibility of the caller to make sure
        that the replacement nodes is a valid expression.
+
+    Also a list "temps" should be passed. Any names listed will
+    be transformed into anonymous, temporary names.
+   
+    Currently supported for tempnames is:
+    NameNode
+    (various function and class definition nodes etc. should be added to this)
     
     Each replacement node gets the position of the substituted node
     recursively applied to every member node.
     """
+
+    def __call__(self, node, substitutions, temps, pos):
+        self.substitutions = substitutions
+        tempdict = {}
+        for key in temps:
+            tempdict[key] = temp_name_handle(key)
+        self.temps = tempdict
+        self.pos = pos
+        return super(TemplateTransform, self).__call__(node)
+
+
     def visit_Node(self, node):
         if node is None:
-            return node
+            return None
         else:
             c = node.clone_node()
+            if self.pos is not None:
+                c.pos = self.pos
             self.visitchildren(c)
             return c
     
     def try_substitution(self, node, key):
         sub = self.substitutions.get(key)
-        if sub is None:
-            return self.visit_Node(node) # make copy as usual
+        if sub is not None:
+            pos = self.pos
+            if pos is None: pos = node.pos
+            return ApplyPositionAndCopy(pos)(sub)
         else:
-            return ApplyPositionAndCopy(node.pos)(sub)
+            return self.visit_Node(node) # make copy as usual
+            
     
     def visit_NameNode(self, node):
-        return self.try_substitution(node, node.name)
+        tempname = self.temps.get(node.name)
+        if tempname is not None:
+            # Replace name with temporary
+            node.name = tempname
+            return self.visit_Node(node)
+        else:
+            return self.try_substitution(node, node.name)
 
     def visit_ExprStatNode(self, node):
         # If an expression-as-statement consists of only a replaceable
@@ -122,10 +151,6 @@ class TemplateTransform(VisitorTransform):
         else:
             return self.visit_Node(node)
     
-    def __call__(self, node, substitutions):
-        self.substitutions = substitutions
-        return super(TemplateTransform, self).__call__(node)
-
 def copy_code_tree(node):
     return TreeCopier()(node)
 
@@ -133,12 +158,12 @@ INDENT_RE = re.compile(ur"^ *")
 def strip_common_indent(lines):
     "Strips empty lines and common indentation from the list of strings given in lines"
     lines = [x for x in lines if x.strip() != u""]
-    minindent = min(len(INDENT_RE.match(x).group(0)) for x in lines)
+    minindent = min([len(INDENT_RE.match(x).group(0)) for x in lines])
     lines = [x[minindent:] for x in lines]
     return lines
     
 class TreeFragment(object):
-    def __init__(self, code, name, pxds={}):
+    def __init__(self, code, name="(tree fragment)", pxds={}, temps=[], pipeline=[]):
         if isinstance(code, unicode):
             def fmt(x): return u"\n".join(strip_common_indent(x.split(u"\n"))) 
             
@@ -147,18 +172,28 @@ class TreeFragment(object):
             for key, value in pxds.iteritems():
                 fmt_pxds[key] = fmt(value)
                 
-            self.root = parse_from_strings(name, fmt_code, fmt_pxds)
+            t = parse_from_strings(name, fmt_code, fmt_pxds)
+            mod = t
+            t = t.body # Make sure a StatListNode is at the top
+            if not isinstance(t, StatListNode):
+                t = StatListNode(pos=mod.pos, stats=[t])
+            for transform in pipeline:
+                t = transform(t)
+            self.root = t
         elif isinstance(code, Node):
             if pxds != {}: raise NotImplementedError()
             self.root = code
         else:
             raise ValueError("Unrecognized code format (accepts unicode and Node)")
+        self.temps = temps
 
     def copy(self):
         return copy_code_tree(self.root)
 
-    def substitute(self, nodes={}):
-        return TemplateTransform()(self.root, substitutions = nodes)
+    def substitute(self, nodes={}, temps=[], pos = None):
+        return TemplateTransform()(self.root,
+                                   substitutions = nodes,
+                                   temps = self.temps + temps, pos = pos)
 
 
 

Cython/Compiler/Visitor.py

 #
 #   Tree visitor and transform framework
 #
+import inspect
 import Nodes
 import ExprNodes
-import inspect
+import Naming
+from Cython.Utils import EncodedString
 
 class BasicVisitor(object):
     """A generic visitor base class which can be used for visiting any kind of object."""
@@ -129,7 +131,6 @@ class VisitorTransform(TreeVisitor):
     was not, an exception will be raised. (Typically you want to ensure that you
     are within a StatListNode or similar before doing this.)
     """
-    
     def visitchildren(self, parent, attrs=None):
         result = super(VisitorTransform, self).visitchildren(parent, attrs)
         for attr, newnode in result.iteritems():
@@ -150,6 +151,19 @@ class VisitorTransform(TreeVisitor):
     def __call__(self, root):
         return self.visit(root)
 
+class CythonTransform(VisitorTransform):
+    """
+    Certain common conventions and utilitues for Cython transforms.
+    """
+    def __init__(self, context):
+        super(CythonTransform, self).__init__()
+        self.context = context
+
+    def visit_Node(self, node):
+        self.visitchildren(node)
+        return node
+
+
 # Utils
 def ensure_statlist(node):
     if not isinstance(node, Nodes.StatListNode):
@@ -166,6 +180,19 @@ def replace_node(ptr, value):
     else:
         getattr(parent, attrname)[listidx] = value
 
+tmpnamectr = 0
+def temp_name_handle(description):
+    global tmpnamectr
+    tmpnamectr += 1
+    return EncodedString(Naming.temp_prefix + u"%d_%s" % (tmpnamectr, description))
+
+def get_temp_name_handle_desc(handle):
+    if not handle.startswith(u"__cyt_"):
+        return None
+    else:
+        idx = handle.find(u"_", 6)
+        return handle[idx+1:]
+    
 class PrintTree(TreeVisitor):
     """Prints a representation of the tree to standard output.
     Subclass and override repr_of to provide more information

Cython/TestUtils.py

@@ -4,8 +4,49 @@ import unittest
 from Cython.Compiler.ModuleNode import ModuleNode
 import Cython.Compiler.Main as Main
 from Cython.Compiler.TreeFragment import TreeFragment, strip_common_indent
+from Cython.Compiler.Visitor import TreeVisitor
+
+class NodeTypeWriter(TreeVisitor):
+    def __init__(self):
+        super(NodeTypeWriter, self).__init__()
+        self._indents = 0
+        self.result = []
+    def visit_Node(self, node):
+        if len(self.access_path) == 0:
+            name = u"(root)"
+        else:
+            tip = self.access_path[-1]
+            if tip[2] is not None:
+                name = u"%s[%d]" % tip[1:3]
+            else:
+                name = tip[1]
+            
+        self.result.append(u"  " * self._indents +
+                           u"%s: %s" % (name, node.__class__.__name__))
+        self._indents += 1
+        self.visitchildren(node)
+        self._indents -= 1
+
+def treetypes(root):
+    """Returns a string representing the tree by class names.
+    There's a leading and trailing whitespace so that it can be
+    compared by simple string comparison while still making test
+    cases look ok."""
+    w = NodeTypeWriter()
+    w.visit(root)
+    return u"\n".join([u""] + w.result + [u""])
 
 class CythonTest(unittest.TestCase):
+
+    def assertLines(self, expected, result):
+        "Checks that the given strings or lists of strings are equal line by line"
+        if not isinstance(expected, list): expected = expected.split(u"\n")
+        if not isinstance(result, list): result = result.split(u"\n")
+        for idx, (expected_line, result_line) in enumerate(zip(expected, result)):
+            self.assertEqual(expected_line, result_line, "Line %d:\nExp: %s\nGot: %s" % (idx, expected_line, result_line))
+        self.assertEqual(len(expected), len(result),
+            "Unmatched lines. Got:\n%s\nExpected:\n%s" % ("\n".join(expected), u"\n".join(result)))
+
     def assertCode(self, expected, result_tree):
         writer = CodeWriter()
         writer.write(result_tree)
@@ -18,13 +59,35 @@ class CythonTest(unittest.TestCase):
         self.assertEqual(len(result_lines), len(expected_lines),
             "Unmatched lines. Got:\n%s\nExpected:\n%s" % ("\n".join(result_lines), expected))
 
-    def fragment(self, code, pxds={}):
+    def fragment(self, code, pxds={}, pipeline=[]):
         "Simply create a tree fragment using the name of the test-case in parse errors."
         name = self.id()
         if name.startswith("__main__."): name = name[len("__main__."):]
         name = name.replace(".", "_")
-        return TreeFragment(code, name, pxds)
-        
+        return TreeFragment(code, name, pxds, pipeline=pipeline)
+
+    def treetypes(self, root):
+        return treetypes(root)
+
+    def should_fail(self, func, exc_type=Exception):
+        """Calls "func" and fails if it doesn't raise the right exception
+        (any exception by default). Also returns the exception in question.
+        """
+        try:
+            func()
+            self.fail("Expected an exception of type %r" % exc_type)
+        except exc_type, e:
+            self.assert_(isinstance(e, exc_type))
+            return e
+
+    def should_not_fail(self, func):
+        """Calls func and succeeds if and only if no exception is raised
+        (i.e. converts exception raising into a failed testcase). Returns
+        the return value of func."""
+        try:
+            return func()
+        except:
+            self.fail()
 
 class TransformTest(CythonTest):
     """
@@ -37,8 +100,8 @@ class TransformTest(CythonTest):
     To create a test case:
      - Call run_pipeline. The pipeline should at least contain the transform you
        are testing; pyx should be either a string (passed to the parser to
-       create a post-parse tree) or a ModuleNode representing input to pipeline.
-       The result will be a transformed result (usually a ModuleNode).
+       create a post-parse tree) or a node representing input to pipeline.
+       The result will be a transformed result.
        
      - Check that the tree is correct. If wanted, assertCode can be used, which
        takes a code string as expected, and a ModuleNode in result_tree
@@ -53,7 +116,6 @@ class TransformTest(CythonTest):
     
     def run_pipeline(self, pipeline, pyx, pxds={}):
         tree = self.fragment(pyx, pxds).root
-        assert isinstance(tree, ModuleNode)
         # Run pipeline
         for T in pipeline:
             tree = T(tree)

Cython/Tests/TestCodeWriter.py

@@ -72,6 +72,9 @@ class TestCodeWriter(CythonTest):
 
     def test_inplace_assignment(self):
         self.t(u"x += 43")
+
+    def test_attribute(self):
+        self.t(u"a.x")
     
 if __name__ == "__main__":
     import unittest

Includes/__cython__.pxd

+cdef extern from "Python.h":
+    ctypedef struct PyObject
+
+
+
+    ctypedef struct Py_buffer:
+        void *buf
+        Py_ssize_t len
+        int readonly
+        char *format
+        int ndim
+        Py_ssize_t *shape
+        Py_ssize_t *strides
+        Py_ssize_t *suboffsets
+        Py_ssize_t itemsize
+        void *internal
+
+    
+    int PyObject_GetBuffer(PyObject* obj, Py_buffer* view, int flags) except -1
+    void PyObject_ReleaseBuffer(PyObject* obj, Py_buffer* view)
+
+#                  int PyObject_GetBuffer(PyObject *obj, Py_buffer *view,
+#                       int flags)

runtests.py

@@ -238,6 +238,29 @@ class CythonRunTestCase(CythonCompileTestCase):
         except Exception:
             pass
 
+def collect_unittests(path, suite, selectors):
+    def file_matches(filename):
+        return filename.startswith("Test") and filename.endswith(".py")
+
+    def package_matches(dirname):
+        return dirname == "Tests"
+
+    loader = unittest.TestLoader()
+
+    for dirpath, dirnames, filenames in os.walk(path):
+        parentname = os.path.split(dirpath)[-1]
+        if package_matches(parentname):
+            for f in filenames:
+                if file_matches(f):
+                    filepath = os.path.join(dirpath, f)[:-len(".py")]
+                    modulename = filepath[len(path)+1:].replace(os.path.sep, '.')
+                    if not [ 1 for match in selectors if match(modulename) ]:
+                        continue
+                    module = __import__(modulename)
+                    for x in modulename.split('.')[1:]:
+                        module = getattr(module, x)
+                    suite.addTests(loader.loadTestsFromModule(module))
+
 if __name__ == '__main__':
     from optparse import OptionParser
     parser = OptionParser()
@@ -247,6 +270,12 @@ if __name__ == '__main__':
     parser.add_option("--no-cython", dest="with_cython",
                       action="store_false", default=True,
                       help="do not run the Cython compiler, only the C compiler")
+    parser.add_option("--no-unit", dest="unittests",
+                      action="store_false", default=True,
+                      help="do not run the unit tests")
+    parser.add_option("--no-file", dest="filetests",
+                      action="store_false", default=True,
+                      help="do not run the file based tests")
     parser.add_option("-C", "--coverage", dest="coverage",
                       action="store_true", default=False,
                       help="collect source coverage data for the Compiler")
@@ -296,9 +325,15 @@ if __name__ == '__main__':
     if not selectors:
         selectors = [ lambda x:True ]
 
-    tests = TestBuilder(ROOTDIR, WORKDIR, selectors,
-                        options.annotate_source, options.cleanup_workdir)
-    test_suite = tests.build_suite()
+    test_suite = unittest.TestSuite()
+
+    if options.unittests:
+        collect_unittests(os.getcwd(), test_suite, selectors)
+
+    if options.filetests:
+        filetests = TestBuilder(ROOTDIR, WORKDIR, selectors,
+                                options.annotate_source, options.cleanup_workdir)
+        test_suite.addTests(filetests.build_suite())
 
     unittest.TextTestRunner(verbosity=options.verbosity).run(test_suite)
 

tests/run/withstat.pyx

+from __future__ import with_statement
+
+__doc__ = u"""
+>>> no_as()
+enter
+hello
+exit <type 'NoneType'> <type 'NoneType'> <type 'NoneType'>
+>>> basic()
+enter
+value
+exit <type 'NoneType'> <type 'NoneType'> <type 'NoneType'>
+>>> with_exception(None)
+enter
+value
+exit <type 'type'> <class 'withstat.MyException'> <type 'traceback'>
+outer except
+>>> with_exception(True)
+enter
+value
+exit <type 'type'> <class 'withstat.MyException'> <type 'traceback'>
+>>> multitarget()
+enter
+1 2 3 4 5
+exit <type 'NoneType'> <type 'NoneType'> <type 'NoneType'>
+>>> tupletarget()
+enter
+(1, 2, (3, (4, 5)))
+exit <type 'NoneType'> <type 'NoneType'> <type 'NoneType'>
+>>> typed()
+enter
+10
+exit <type 'NoneType'> <type 'NoneType'> <type 'NoneType'>
+"""
+
+class MyException(Exception):
+    pass
+
+class ContextManager:
+    def __init__(self, value, exit_ret = None):
+        self.value = value
+        self.exit_ret = exit_ret
+
+    def __exit__(self, a, b, tb):
+        print "exit", type(a), type(b), type(tb)
+        return self.exit_ret
+        
+    def __enter__(self):
+        print "enter"
+        return self.value
+
+def no_as():
+    with ContextManager("value"):
+        print "hello"
+        
+def basic():
+    with ContextManager("value") as x:
+        print x
+
+def with_exception(exit_ret):
+    try:
+        with ContextManager("value", exit_ret=exit_ret) as value:
+            print value
+            raise MyException()
+    except:
+        print "outer except"
+
+def multitarget():
+    with ContextManager((1, 2, (3, (4, 5)))) as (a, b, (c, (d, e))):
+        print a, b, c, d, e
+
+def tupletarget():
+    with ContextManager((1, 2, (3, (4, 5)))) as t:
+        print t
+
+def typed():
+    cdef unsigned char i
+    c = ContextManager(255)
+    with c as i:
+        i += 11
+        print i