large merge of cython-dagss as of revision 764

Cython/CodeWriter.py

-from Cython.Compiler.Visitor import TreeVisitor
+from Cython.Compiler.Visitor import TreeVisitor, get_temp_name_handle_desc
 from Cython.Compiler.Nodes import *
+from Cython.Compiler.ExprNodes import *
 
 """
 Serializes a Cython code tree to Cython code. This is primarily useful for
@@ -35,6 +36,7 @@ class CodeWriter(TreeVisitor):
             result = LinesResult()
         self.result = result
         self.numindents = 0
+        self.tempnames = {}
     
     def write(self, tree):
         self.visit(tree)
@@ -57,6 +59,12 @@ class CodeWriter(TreeVisitor):
     def line(self, s):
         self.startline(s)
         self.endline()
+
+    def putname(self, name):
+        tmpdesc = get_temp_name_handle_desc(name)
+        if tmpdesc is not None:
+            name = self.tempnames.setdefault(tmpdesc, u"$" +tmpdesc)
+        self.put(name)
     
     def comma_seperated_list(self, items, output_rhs=False):
         if len(items) > 0:
@@ -116,7 +124,7 @@ class CodeWriter(TreeVisitor):
         self.endline()
     
     def visit_NameNode(self, node):
-        self.put(node.name)
+        self.putname(node.name)
     
     def visit_IntNode(self, node):
         self.put(node.value)
@@ -185,10 +193,23 @@ class CodeWriter(TreeVisitor):
         self.comma_seperated_list(node.args) # Might need to discover whether we need () around tuples...hmm...
     
     def visit_SimpleCallNode(self, node):
-        self.put(node.function.name + u"(")
+        self.visit(node.function)
+        self.put(u"(")
         self.comma_seperated_list(node.args)
         self.put(")")
 
+    def visit_GeneralCallNode(self, node):
+        self.visit(node.function)
+        self.put(u"(")
+        posarg = node.positional_args
+        if isinstance(posarg, AsTupleNode):
+            self.visit(posarg.arg)
+        else:
+            self.comma_seperated_list(posarg)
+        if node.keyword_args is not None or node.starstar_arg is not None:
+            raise Exception("Not implemented yet")
+        self.put(u")")
+
     def visit_ExprStatNode(self, node):
         self.startline()
         self.visit(node.expr)
@@ -197,9 +218,73 @@ class CodeWriter(TreeVisitor):
     def visit_InPlaceAssignmentNode(self, node):
         self.startline()
         self.visit(node.lhs)
-        self.put(" %s= " % node.operator)
+        self.put(u" %s= " % node.operator)
         self.visit(node.rhs)
         self.endline()
-    
-    
+        
+    def visit_WithStatNode(self, node):
+        self.startline()
+        self.put(u"with ")
+        self.visit(node.manager)
+        if node.target is not None:
+            self.put(u" as ")
+            self.visit(node.target)
+        self.endline(u":")
+        self.indent()
+        self.visit(node.body)
+        self.dedent()
+        
+    def visit_AttributeNode(self, node):
+        self.visit(node.obj)
+        self.put(u".%s" % node.attribute)
+
+    def visit_BoolNode(self, node):
+        self.put(str(node.value))
+
+    def visit_TryFinallyStatNode(self, node):
+        self.line(u"try:")
+        self.indent()
+        self.visit(node.body)
+        self.dedent()
+        self.line(u"finally:")
+        self.indent()
+        self.visit(node.finally_clause)
+        self.dedent()
+
+    def visit_TryExceptStatNode(self, node):
+        self.line(u"try:")
+        self.indent()
+        self.visit(node.body)
+        self.dedent()
+        for x in node.except_clauses:
+            self.visit(x)
+        if node.else_clause is not None:
+            self.visit(node.else_clause)
+
+    def visit_ExceptClauseNode(self, node):
+        self.startline(u"except")
+        if node.pattern is not None:
+            self.put(u" ")
+            self.visit(node.pattern)
+        if node.target is not None:
+            self.put(u", ")
+            self.visit(node.target)
+        self.endline(":")
+        self.indent()
+        self.visit(node.body)
+        self.dedent()
+
+    def visit_ReraiseStatNode(self, node):
+        self.line("raise")
+
+    def visit_NoneNode(self, node):
+        self.put(u"None")
+
+    def visit_ImportNode(self, node):
+        self.put(u"(import %s)" % node.module_name.value)
+
+    def visit_NotNode(self, node):
+        self.put(u"(not ")
+        self.visit(node.operand)
+        self.put(u")")
 

Cython/Compiler/Code.py

@@ -200,6 +200,8 @@ class CCodeWriter:
     def put_var_declaration(self, entry, static = 0, dll_linkage = None,
             definition = True):
         #print "Code.put_var_declaration:", entry.name, "definition =", definition ###
+        if entry.in_closure:
+            return
         visibility = entry.visibility
         if visibility == 'private' and not definition:
             #print "...private and not definition, skipping" ###

Cython/Compiler/Errors.py

@@ -32,6 +32,7 @@ class CompileError(PyrexError):
     
     def __init__(self, position = None, message = ""):
         self.position = position
+        self.message_only = message
     # Deprecated and withdrawn in 2.6:
     #   self.message = message
         if position:
@@ -91,6 +92,7 @@ def error(position, message):
     #print "Errors.error:", repr(position), repr(message) ###
     global num_errors
     err = CompileError(position, message)
+#    if position is not None: raise Exception(err) # debug
     line = "%s\n" % err
     if listing_file:
         listing_file.write(line)

Cython/Compiler/ExprNodes.py

@@ -972,7 +972,7 @@ class NameNode(AtomicExprNode):
             if entry.is_builtin:
                 namespace = Naming.builtins_cname
             else: # entry.is_pyglobal
-                namespace = entry.namespace_cname
+                namespace = entry.scope.namespace_cname
             code.putln(
                 '%s = __Pyx_GetName(%s, %s); %s' % (
                 self.result_code,
@@ -997,7 +997,7 @@ class NameNode(AtomicExprNode):
         # is_pyglobal seems to be True for module level-globals only.
         # We use this to access class->tp_dict if necessary.
         if entry.is_pyglobal:
-            namespace = self.entry.namespace_cname
+            namespace = self.entry.scope.namespace_cname
             if entry.is_member:
                 # if the entry is a member we have to cheat: SetAttr does not work
                 # on types, so we create a descriptor which is then added to tp_dict
@@ -1060,7 +1060,6 @@ class NameNode(AtomicExprNode):
             else:
                 code.annotate(pos, AnnotationItem('c_call', 'c function', size=len(self.name)))
             
-
 class BackquoteNode(ExprNode):
     #  `expr`
     #
@@ -1212,6 +1211,9 @@ class ExcValueNode(AtomicExprNode):
     def generate_result_code(self, code):
         pass
 
+    def analyse_types(self, env):
+        pass
+
 
 class TempNode(AtomicExprNode):
     #  Node created during analyse_types phase
@@ -1273,36 +1275,59 @@ class IndexNode(ExprNode):
         self.analyse_base_and_index_types(env, setting = 1)
     
     def analyse_base_and_index_types(self, env, getting = 0, setting = 0):
+        self.is_buffer_access = False
+
         self.base.analyse_types(env)
-        self.index.analyse_types(env)
-        if self.base.type.is_pyobject:
-            if self.index.type.is_int:
-                self.original_index_type = self.index.type
-                self.index = self.index.coerce_to(PyrexTypes.c_py_ssize_t_type, env).coerce_to_simple(env)
-                if getting:
-                    env.use_utility_code(getitem_int_utility_code)
-                if setting:
-                    env.use_utility_code(setitem_int_utility_code)
+        
+        if self.base.type.buffer_options is not None:
+            if isinstance(self.index, TupleNode):
+                indices = self.index.args
+#                is_int_indices = 0 == sum([1 for i in self.index.args if not i.type.is_int])
             else:
-                self.index = self.index.coerce_to_pyobject(env)
-            self.type = py_object_type
-            self.gil_check(env)
-            self.is_temp = 1
-        else:
-            if self.base.type.is_ptr or self.base.type.is_array:
-                self.type = self.base.type.base_type
+#                is_int_indices = self.index.type.is_int
+                indices = [self.index]
+            all_ints = True
+            for index in indices:
+                index.analyse_types(env)
+                if not index.type.is_int:
+                    all_ints = False
+            if all_ints:
+                self.indices = indices
+                self.index = None
+                self.type = self.base.type.buffer_options.dtype
+                self.is_temp = 1
+                self.is_buffer_access = True
+
+        if not self.is_buffer_access:
+            self.index.analyse_types(env) # ok to analyse as tuple
+            if self.base.type.is_pyobject:
+                if self.index.type.is_int:
+                    self.original_index_type = self.index.type
+                    self.index = self.index.coerce_to(PyrexTypes.c_py_ssize_t_type, env).coerce_to_simple(env)
+                    if getting:
+                        env.use_utility_code(getitem_int_utility_code)
+                    if setting:
+                        env.use_utility_code(setitem_int_utility_code)
+                else:
+                    self.index = self.index.coerce_to_pyobject(env)
+                self.type = py_object_type
+                self.gil_check(env)
+                self.is_temp = 1
             else:
-                error(self.pos,
-                    "Attempting to index non-array type '%s'" %
-                        self.base.type)
-                self.type = PyrexTypes.error_type
-            if self.index.type.is_pyobject:
-                self.index = self.index.coerce_to(
-                    PyrexTypes.c_py_ssize_t_type, env)
-            if not self.index.type.is_int:
-                error(self.pos,
-                    "Invalid index type '%s'" %
-                        self.index.type)
+                if self.base.type.is_ptr or self.base.type.is_array:
+                    self.type = self.base.type.base_type
+                else:
+                    error(self.pos,
+                        "Attempting to index non-array type '%s'" %
+                            self.base.type)
+                    self.type = PyrexTypes.error_type
+                if self.index.type.is_pyobject:
+                    self.index = self.index.coerce_to(
+                        PyrexTypes.c_py_ssize_t_type, env)
+                if not self.index.type.is_int:
+                    error(self.pos,
+                        "Invalid index type '%s'" %
+                            self.index.type)
 
     gil_message = "Indexing Python object"
 
@@ -1328,11 +1353,17 @@ class IndexNode(ExprNode):
 
     def generate_subexpr_evaluation_code(self, code):
         self.base.generate_evaluation_code(code)
-        self.index.generate_evaluation_code(code)
+        if self.index is not None:
+            self.index.generate_evaluation_code(code)
+        else:
+            for i in self.indices: i.generate_evaluation_code(code)
         
     def generate_subexpr_disposal_code(self, code):
         self.base.generate_disposal_code(code)
-        self.index.generate_disposal_code(code)
+        if self.index is not None:
+            self.index.generate_disposal_code(code)
+        else:
+            for i in self.indices: i.generate_disposal_code(code)
 
     def generate_result_code(self, code):
         if self.type.is_pyobject:

Cython/Compiler/Future.py

@@ -7,5 +7,6 @@ def _get_feature(name):
         return object()
 
 unicode_literals = _get_feature("unicode_literals")
+with_statement = _get_feature("with_statement")
 
 del _get_feature

Cython/Compiler/Main.py

@@ -25,22 +25,6 @@ from Cython import Utils
 
 module_name_pattern = re.compile(r"[A-Za-z_][A-Za-z0-9_]*(\.[A-Za-z_][A-Za-z0-9_]*)*$")
 
-# Note: PHASES and TransformSet should be removed soon; but that's for
-# another day and another commit.
-PHASES = [
-    'before_analyse_function', # run in FuncDefNode.generate_function_definitions
-    'after_analyse_function'   # run in FuncDefNode.generate_function_definitions
-]
-
-class TransformSet(dict):
-    def __init__(self):
-        for name in PHASES:
-            self[name] = []
-    def run(self, name, node, **options):
-        assert name in self, "Transform phase %s not defined" % name
-        for transform in self[name]:
-            transform(node, phase=name, **options)
-
 verbose = 0
 
 class Context:
@@ -58,6 +42,8 @@ class Context:
         #self.modules = {"__builtin__" : BuiltinScope()}
         import Builtin
         self.modules = {"__builtin__" : Builtin.builtin_scope}
+        self.pxds = {}
+        self.pyxs = {}
         self.include_directories = include_directories
         self.future_directives = set()
         
@@ -305,55 +291,25 @@ class Context:
         names.reverse()
         return ".".join(names)
 
-    def compile(self, source, options = None, full_module_name = None):
-        # Compile a Pyrex implementation file in this context
-        # and return a CompilationResult.
-        if not options:
-            options = default_options
-        result = CompilationResult()
-        cwd = os.getcwd()
-
-        source = os.path.join(cwd, source)
-        result.main_source_file = source
-
+    def setup_errors(self, options):
         if options.use_listing_file:
             result.listing_file = Utils.replace_suffix(source, ".lis")
             Errors.open_listing_file(result.listing_file,
                 echo_to_stderr = options.errors_to_stderr)
         else:
             Errors.open_listing_file(None)
-        if options.output_file:
-            result.c_file = os.path.join(cwd, options.output_file)
-        else:
-            if options.cplus:
-                c_suffix = ".cpp"
-            else:
-                c_suffix = ".c"
-            result.c_file = Utils.replace_suffix(source, c_suffix)
-        c_stat = None
-        if result.c_file:
-            try:
-                c_stat = os.stat(result.c_file)
-            except EnvironmentError:
-                pass
-        full_module_name = full_module_name or self.extract_module_name(source, options)
-        source = FileSourceDescriptor(source)
-        initial_pos = (source, 1, 0)
-        scope = self.find_module(full_module_name, pos = initial_pos, need_pxd = 0)
-        errors_occurred = False
-        try:
-            tree = self.parse(source, scope, pxd = 0,
-                              full_module_name = full_module_name)
-            tree.process_implementation(scope, options, result)
-        except CompileError:
-            errors_occurred = True
+
+    def teardown_errors(self, errors_occurred, options, result):
+        source_desc = result.compilation_source.source_desc
+        if not isinstance(source_desc, FileSourceDescriptor):
+            raise RuntimeError("Only file sources for code supported")
         Errors.close_listing_file()
         result.num_errors = Errors.num_errors
         if result.num_errors > 0:
             errors_occurred = True
         if errors_occurred and result.c_file:
             try:
-                Utils.castrate_file(result.c_file, os.stat(source.filename))
+                Utils.castrate_file(result.c_file, os.stat(source_desc.filename))
             except EnvironmentError:
                 pass
             result.c_file = None
@@ -366,7 +322,98 @@ class Context:
                     extra_objects = options.objects,
                     verbose_flag = options.show_version,
                     cplus = options.cplus)
+
+    def run_pipeline(self, pipeline, source):
+        errors_occurred = False
+        data = source
+        try:
+            for phase in pipeline:
+                data = phase(data)
+        except CompileError:
+            errors_occurred = True
+        return (errors_occurred, data)
+
+def create_parse(context):
+    def parse(compsrc):
+        source_desc = compsrc.source_desc
+        full_module_name = compsrc.full_module_name
+        initial_pos = (source_desc, 1, 0)
+        scope = context.find_module(full_module_name, pos = initial_pos, need_pxd = 0)
+        tree = context.parse(source_desc, scope, pxd = 0, full_module_name = full_module_name)
+        tree.compilation_source = compsrc
+        tree.scope = scope
+        return tree
+    return parse
+
+def create_generate_code(context, options, result):
+    def generate_code(module_node):
+        scope = module_node.scope
+        module_node.process_implementation(options, result)
+        result.compilation_source = module_node.compilation_source
         return result
+    return generate_code
+
+def create_default_pipeline(context, options, result):
+    from ParseTreeTransforms import WithTransform, NormalizeTree, PostParse, BufferTransform
+    from ParseTreeTransforms import AnalyseDeclarationsTransform, AnalyseExpressionsTransform
+    from ParseTreeTransforms import CreateClosureClasses, MarkClosureVisitor
+    from ModuleNode import check_c_classes
+    
+    return [
+        create_parse(context),
+        NormalizeTree(context),
+        PostParse(context),
+        WithTransform(context),
+        AnalyseDeclarationsTransform(context),
+        check_c_classes,
+        AnalyseExpressionsTransform(context),
+        BufferTransform(context),
+#        CreateClosureClasses(context),
+        create_generate_code(context, options, result)
+    ]
+
+def create_default_resultobj(compilation_source, options):
+    result = CompilationResult()
+    result.main_source_file = compilation_source.source_desc.filename
+    result.compilation_source = compilation_source
+    source_desc = compilation_source.source_desc
+    if options.output_file:
+        result.c_file = os.path.join(compilation_source.cwd, options.output_file)
+    else:
+        if options.cplus:
+            c_suffix = ".cpp"
+        else:
+            c_suffix = ".c"
+        result.c_file = Utils.replace_suffix(source_desc.filename, c_suffix)
+    # The below doesn't make any sense? Why is it there?
+    c_stat = None
+    if result.c_file:
+        try:
+            c_stat = os.stat(result.c_file)
+        except EnvironmentError:
+            pass
+    return result
+
+def run_pipeline(source, options, full_module_name = None):
+    # Set up context
+    context = Context(options.include_path)
+
+    # Set up source object
+    cwd = os.getcwd()
+    source_desc = FileSourceDescriptor(os.path.join(cwd, source))
+    full_module_name = full_module_name or context.extract_module_name(source, options)
+    source = CompilationSource(source_desc, full_module_name, cwd)
+
+    # Set up result object
+    result = create_default_resultobj(source, options)
+    
+    # Get pipeline
+    pipeline = create_default_pipeline(context, options, result)
+
+    context.setup_errors(options)
+    errors_occurred, enddata = context.run_pipeline(pipeline, source)
+    context.teardown_errors(errors_occurred, options, result)
+    return result
 
 #------------------------------------------------------------------------
 #
@@ -374,6 +421,16 @@ class Context:
 #
 #------------------------------------------------------------------------
 
+class CompilationSource(object):
+    """
+    Contains the data necesarry to start up a compilation pipeline for
+    a single compilation unit.
+    """
+    def __init__(self, source_desc, full_module_name, cwd):
+        self.source_desc = source_desc
+        self.full_module_name = full_module_name
+        self.cwd = cwd
+
 class CompilationOptions:
     """
     Options to the Cython compiler:
@@ -389,7 +446,6 @@ class CompilationOptions:
                                 defaults to true when recursive is true.
     verbose           boolean   Always print source names being compiled
     quiet             boolean   Don't print source names in recursive mode
-    transforms        Transform.TransformSet Transforms to use on the parse tree
     
     Following options are experimental and only used on MacOSX:
     
@@ -427,6 +483,7 @@ class CompilationResult:
     object_file      string or None   Result of compiling the C file
     extension_file   string or None   Result of linking the object file
     num_errors       integer          Number of compilation errors
+    compilation_source CompilationSource
     """
     
     def __init__(self):
@@ -464,8 +521,10 @@ def compile_single(source, options, full_module_name = None):
     Always compiles a single file; does not perform timestamp checking or
     recursion.
     """
-    context = Context(options.include_path)
-    return context.compile(source, options, full_module_name)
+    return run_pipeline(source, options, full_module_name)
+#    context = Context(options.include_path)
+#    return context.compile(source, options, full_module_name)
+
 
 def compile_multiple(sources, options):
     """
@@ -478,21 +537,21 @@ def compile_multiple(sources, options):
     sources = [os.path.abspath(source) for source in sources]
     processed = set()
     results = CompilationResultSet()
-    context = Context(options.include_path)
     recursive = options.recursive
     timestamps = options.timestamps
     if timestamps is None:
         timestamps = recursive
     verbose = options.verbose or ((recursive or timestamps) and not options.quiet)
     for source in sources:
+        context = Context(options.include_path) # to be removed later
         if source not in processed:
             if not timestamps or context.c_file_out_of_date(source):
                 if verbose:
                     sys.stderr.write("Compiling %s\n" % source)
+
                 result = context.compile(source, options)
                 # Compiling multiple sources in one context doesn't quite
                 # work properly yet.
-                context = Context(options.include_path) # to be removed later
                 results.add(source, result)
             processed.add(source)
             if recursive:
@@ -522,7 +581,10 @@ def compile(source, options = None, c_compile = 0, c_link = 0,
             and not options.recursive:
         return compile_single(source, options, full_module_name)
     else:
-        return compile_multiple(source, options)
+        # Hack it for wednesday dev1
+        assert len(source) == 1
+        return compile_single(source[0], options)
+#        return compile_multiple(source, options)
 
 #------------------------------------------------------------------------
 #
@@ -571,9 +633,9 @@ default_options = dict(
     output_file = None,
     annotate = False,
     generate_pxi = 0,
-    transforms = TransformSet(),
     working_path = "",
     recursive = 0,
+    transforms = None, # deprecated
     timestamps = None,
     verbose = 0,
     quiet = 0)

Cython/Compiler/ModuleNode.py

@@ -25,6 +25,10 @@ from PyrexTypes import py_object_type
 from Cython.Utils import open_new_file, replace_suffix
 
 
+def check_c_classes(module_node):
+    module_node.scope.check_c_classes()
+    return module_node
+
 class ModuleNode(Nodes.Node, Nodes.BlockNode):
     #  doc       string or None
     #  body      StatListNode
@@ -32,6 +36,9 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
     #  referenced_modules   [ModuleScope]
     #  module_temp_cname    string
     #  full_module_name     string
+    #
+    #  scope                The module scope.
+    #  compilation_source   A CompilationSource (see Main)
 
     child_attrs = ["body"]
     
@@ -44,10 +51,8 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
             env.doc = self.doc
         self.body.analyse_declarations(env)
     
-    def process_implementation(self, env, options, result):
-        self.analyse_declarations(env)
-        env.check_c_classes()
-        self.body.analyse_expressions(env)
+    def process_implementation(self, options, result):
+        env = self.scope
         env.return_type = PyrexTypes.c_void_type
         self.referenced_modules = []
         self.find_referenced_modules(env, self.referenced_modules, {})
@@ -254,6 +259,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
         self.generate_module_cleanup_func(env, code)
         self.generate_filename_table(code)
         self.generate_utility_functions(env, code)
+        self.generate_buffer_compatability_functions(env, code)
 
         self.generate_declarations_for_modules(env, modules, code.h)
 
@@ -433,6 +439,9 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
         code.putln("  #define PyBUF_F_CONTIGUOUS (0x0040 | PyBUF_STRIDES)")
         code.putln("  #define PyBUF_ANY_CONTIGUOUS (0x0080 | PyBUF_STRIDES)")
         code.putln("  #define PyBUF_INDIRECT (0x0100 | PyBUF_STRIDES)")
+        code.putln("")
+        code.putln("  static int PyObject_GetBuffer(PyObject *obj, Py_buffer *view, int flags);")
+        code.putln("  static void PyObject_ReleaseBuffer(PyObject *obj, Py_buffer *view);")
         code.putln("#endif")
 
         code.put(builtin_module_name_utility_code[0])
@@ -1940,6 +1949,60 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
             code.h.put(utility_code[0])
             code.put(utility_code[1])
         code.put(PyrexTypes.type_conversion_functions)
+        code.putln("")
+
+    def generate_buffer_compatability_functions(self, env, code):
+        # will be refactored
+        code.put("""
+static int numpy_getbuffer(PyObject *obj, Py_buffer *view, int flags) {
+  /* This function is always called after a type-check */
+  PyArrayObject *arr = (PyArrayObject*)obj;
+  PyArray_Descr *type = (PyArray_Descr*)arr->descr;
+  
+  view->buf = arr->data;
+  view->readonly = 0; /*fixme*/
+  view->format = "B"; /*fixme*/
+  view->ndim = arr->nd;
+  view->strides = arr->strides;
+  view->shape = arr->dimensions;
+  view->suboffsets = 0;
+  
+  view->itemsize = type->elsize;
+  view->internal = 0;
+  return 0;
+}
+
+static void numpy_releasebuffer(PyObject *obj, Py_buffer *view) {
+}
+
+""")
+        
+        # For now, hard-code numpy imported as "numpy"
+        ndarrtype = env.entries[u'numpy'].as_module.entries['ndarray'].type
+        types = [
+            (ndarrtype.typeptr_cname, "numpy_getbuffer", "numpy_releasebuffer")
+        ]
+        
+#        typeptr_cname = ndarrtype.typeptr_cname
+        code.putln("static int PyObject_GetBuffer(PyObject *obj, Py_buffer *view, int flags) {")
+        clause = "if"
+        for t, get, release in types:
+            code.putln("%s (__Pyx_TypeTest(obj, %s)) return %s(obj, view, flags);" % (clause, t, get))
+            clause = "else if"
+        code.putln("else {")
+        code.putln("PyErr_Format(PyExc_TypeError, \"'%100s' does not have the buffer interface\", Py_TYPE(obj)->tp_name);")
+        code.putln("return -1;")
+        code.putln("}")
+        code.putln("}")
+        code.putln("")
+        code.putln("static void PyObject_ReleaseBuffer(PyObject *obj, Py_buffer *view) {")
+        clause = "if"
+        for t, get, release in types:
+            code.putln("%s (__Pyx_TypeTest(obj, %s)) %s(obj, view);" % (clause, t, release))
+            clause = "else if"
+        code.putln("}")
+        code.putln("")
+
 
 #------------------------------------------------------------------------------------
 #

Cython/Compiler/Naming.py

@@ -8,6 +8,8 @@
 
 pyrex_prefix    = "__pyx_"
 
+temp_prefix       = u"__cyt_"
+
 builtin_prefix    = pyrex_prefix + "builtin_"
 arg_prefix        = pyrex_prefix + "arg_"
 funcdoc_prefix    = pyrex_prefix + "doc_"
@@ -70,6 +72,8 @@ optional_args_cname = pyrex_prefix + "optional_args"
 no_opt_args      = pyrex_prefix + "no_opt_args"
 import_star      = pyrex_prefix + "import_star"
 import_star_set  = pyrex_prefix + "import_star_set"
+cur_scope_cname  = pyrex_prefix + "cur_scope"
+enc_scope_cname  = pyrex_prefix + "enc_scope"
 
 line_c_macro = "__LINE__"
 

Cython/Compiler/Nodes.py

@@ -10,7 +10,7 @@ import Naming
 import PyrexTypes
 import TypeSlots
 from PyrexTypes import py_object_type, error_type, CTypedefType, CFuncType
-from Symtab import ModuleScope, LocalScope, \
+from Symtab import ModuleScope, LocalScope, GeneratorLocalScope, \
     StructOrUnionScope, PyClassScope, CClassScope
 from Cython.Utils import open_new_file, replace_suffix, EncodedString
 import Options
@@ -172,7 +172,27 @@ class Node(object):
             self._end_pos = pos
             return pos
 
-
+    def dump(self, level=0, filter_out=("pos",)):
+        def dump_child(x, level):
+            if isinstance(x, Node):
+                return x.dump(level)
+            elif isinstance(x, list):
+                return "[%s]" % ", ".join(dump_child(item, level) for item in x)
+            else:
+                return repr(x)
+            
+        
+        attrs = [(key, value) for key, value in self.__dict__.iteritems() if key not in filter_out]
+        if len(attrs) == 0:
+            return "<%s>" % self.__class__.__name__
+        else:
+            indent = "  " * level
+            res = "<%s\n" % (self.__class__.__name__)
+            for key, value in attrs:
+                res += "%s  %s: %s\n" % (indent, key, dump_child(value, level + 1))
+            res += "%s>" % indent
+            return res
+        
 class BlockNode:
     #  Mixin class for nodes representing a declaration block.
 
@@ -545,7 +565,6 @@ class CBaseTypeNode(Node):
     
     pass
 
-
 class CSimpleBaseTypeNode(CBaseTypeNode):
     # name             string
     # module_path      [string]     Qualifying name components
@@ -587,6 +606,30 @@ class CSimpleBaseTypeNode(CBaseTypeNode):
         else:
             return PyrexTypes.error_type
 
+class CBufferAccessTypeNode(Node):
+    #  After parsing:
+    #  positional_args  [ExprNode]        List of positional arguments
+    #  keyword_args     DictNode          Keyword arguments
+    #  base_type_node   CBaseTypeNode
+
+    #  After PostParse:
+    #  dtype_node       CBaseTypeNode
+    #  ndim             int
+
+    #  After analysis:
+    #  type             PyrexType.PyrexType
+
+    child_attrs = ["base_type_node", "positional_args", "keyword_args",
+                   "dtype_node"]
+
+    dtype_node = None
+    
+    def analyse(self, env):
+        base_type = self.base_type_node.analyse(env)
+        dtype = self.dtype_node.analyse(env)
+        options = PyrexTypes.BufferOptions(dtype=dtype, ndim=self.ndim)
+        self.type = PyrexTypes.create_buffer_type(base_type, options)
+        return self.type
 
 class CComplexBaseTypeNode(CBaseTypeNode):
     # base_type   CBaseTypeNode
@@ -773,9 +816,11 @@ class FuncDefNode(StatNode, BlockNode):
     #  return_type     PyrexType
     #  #filename        string        C name of filename string const
     #  entry           Symtab.Entry
+    #  needs_closure   boolean        Whether or not this function has inner functions/classes/yield
     
     py_func = None
     assmt = None
+    needs_closure = False
     
     def analyse_default_values(self, env):
         genv = env.global_scope()
@@ -802,26 +847,28 @@ class FuncDefNode(StatNode, BlockNode):
     
     def need_gil_acquisition(self, lenv):
         return 0
-                
-    def generate_function_definitions(self, env, code, transforms):
-        code.mark_pos(self.pos)
-        # Generate C code for header and body of function
-        genv = env.global_scope()
-        lenv = LocalScope(name = self.entry.name, outer_scope = genv)
+        
+    def create_local_scope(self, env):
+        genv = env
+        while env.is_py_class_scope or env.is_c_class_scope:
+            env = env.outer_scope
+        if self.needs_closure:
+            lenv = GeneratorLocalScope(name = self.entry.name, outer_scope = genv)
+        else:
+            lenv = LocalScope(name = self.entry.name, outer_scope = genv)
         lenv.return_type = self.return_type
         type = self.entry.type
         if type.is_cfunction:
             lenv.nogil = type.nogil and not type.with_gil
+        self.local_scope = lenv
+        return lenv
+                
+    def generate_function_definitions(self, env, code, transforms):
+        # Generate C code for header and body of function
         code.init_labels()
-        self.declare_arguments(lenv)
-        transforms.run('before_analyse_function', self, env=env, lenv=lenv, genv=genv)
-        self.body.analyse_control_flow(lenv)
-        self.body.analyse_declarations(lenv)
-        self.body.analyse_expressions(lenv)
-        transforms.run('after_analyse_function', self, env=env, lenv=lenv, genv=genv)
-        # Code for nested function definitions would go here
-        # if we supported them, which we probably won't.
+        lenv = self.local_scope
         # ----- Top-level constants used by this function
+        code.mark_pos(self.pos)
         self.generate_interned_num_decls(lenv, code)
         self.generate_interned_string_decls(lenv, code)
         self.generate_py_string_decls(lenv, code)
@@ -838,7 +885,10 @@ class FuncDefNode(StatNode, BlockNode):
         self.generate_function_header(code,
             with_pymethdef = env.is_py_class_scope)
         # ----- Local variable declarations
+        lenv.mangle_closure_cnames(Naming.cur_scope_cname)
         self.generate_argument_declarations(lenv, code)
+        if self.needs_closure:
+            code.putln("/* TODO: declare and create scope object */")
         code.put_var_declarations(lenv.var_entries)
         init = ""
         if not self.return_type.is_void:
@@ -919,6 +969,7 @@ class FuncDefNode(StatNode, BlockNode):
         self.put_stararg_decrefs(code)
         if acquire_gil:
             code.putln("PyGILState_Release(_save);")
+        code.putln("/* TODO: decref scope object */")
         # ----- Return
         if not self.return_type.is_void:
             code.putln("return %s;" % Naming.retval_cname)
@@ -975,8 +1026,9 @@ class CFuncDefNode(FuncDefNode):
     #
     #  with_gil      boolean    Acquire GIL around body
     #  type          CFuncType
+    #  py_func       wrapper for calling from Python
     
-    child_attrs = ["base_type", "declarator", "body"]
+    child_attrs = ["base_type", "declarator", "body", "py_func"]
 
     def unqualified_name(self):
         return self.entry.name
@@ -1838,6 +1890,8 @@ class OverrideCheckNode(StatNode):
     #  body
     
     child_attrs = ['body']
+    
+    body = None
 
     def analyse_expressions(self, env):
         self.args = env.arg_entries
@@ -1881,9 +1935,10 @@ class OverrideCheckNode(StatNode):
 #        code.put_decref(self.func_temp, PyrexTypes.py_object_type)
         code.putln("}")
 
+class ClassDefNode(StatNode, BlockNode):
+    pass
 
-
-class PyClassDefNode(StatNode, BlockNode):
+class PyClassDefNode(ClassDefNode):
     #  A Python class definition.
     #
     #  name     EncodedString   Name of the class
@@ -1915,19 +1970,27 @@ class PyClassDefNode(StatNode, BlockNode):
         self.classobj = ExprNodes.ClassNode(pos, name = name,
             bases = bases, dict = self.dict, doc = doc_node)
         self.target = ExprNodes.NameNode(pos, name = name)
+        
+    def create_scope(self, env):
+        genv = env
+        while env.is_py_class_scope or env.is_c_class_scope:
+            env = env.outer_scope
+        cenv = self.scope = PyClassScope(name = self.name, outer_scope = genv)
+        return cenv
     
     def analyse_declarations(self, env):
         self.target.analyse_target_declaration(env)
+        cenv = self.create_scope(env)
+        cenv.class_obj_cname = self.target.entry.cname
+        self.body.analyse_declarations(cenv)
     
     def analyse_expressions(self, env):
         self.dict.analyse_expressions(env)
         self.classobj.analyse_expressions(env)
         genv = env.global_scope()
-        cenv = PyClassScope(name = self.name, outer_scope = genv)
+        cenv = self.scope
         cenv.class_dict_cname = self.dict.result_code
-        cenv.class_obj_cname = self.classobj.result_code
-        self.scope = cenv
-        self.body.analyse_declarations(cenv)
+        cenv.namespace_cname = cenv.class_obj_cname = self.classobj.result_code
         self.body.analyse_expressions(cenv)
         self.target.analyse_target_expression(env, self.classobj)
         self.dict.release_temp(env)
@@ -1947,7 +2010,7 @@ class PyClassDefNode(StatNode, BlockNode):
         self.dict.generate_disposal_code(code)
 
 
-class CClassDefNode(StatNode, BlockNode):
+class CClassDefNode(ClassDefNode):
     #  An extension type definition.
     #
     #  visibility         'private' or 'public' or 'extern'
@@ -2356,7 +2419,7 @@ class InPlaceAssignmentNode(AssignmentNode):
     #  Fortunately, the type of the lhs node is fairly constrained 
     #  (it must be a NameNode, AttributeNode, or IndexNode).     
     
-    child_attrs = ["lhs", "rhs", "dup"]
+    child_attrs = ["lhs", "rhs"]
     dup = None
 
     def analyse_declarations(self, env):
@@ -2991,7 +3054,7 @@ class ForInStatNode(LoopNode, StatNode):
     #  else_clause   StatNode
     #  item          NextNode       used internally
     
-    child_attrs = ["target", "iterator", "body", "else_clause", "item"]
+    child_attrs = ["target", "iterator", "body", "else_clause"]
     item = None
     
     def analyse_declarations(self, env):
@@ -3225,6 +3288,18 @@ class ForFromStatNode(LoopNode, StatNode):
             self.else_clause.annotate(code)
 
 
+class WithStatNode(StatNode):
+    """
+    Represents a Python with statement.
+    
+    This is only used at parse tree level; and is not present in
+    analysis or generation phases.
+    """
+    #  manager          The with statement manager object
+    #  target            Node (lhs expression)
+    #  body             StatNode
+    child_attrs = ["manager", "target", "body"]
+
 class TryExceptStatNode(StatNode):
     #  try .. except statement
     #
@@ -3317,17 +3392,26 @@ class ExceptClauseNode(Node):
     #  pattern        ExprNode
     #  target         ExprNode or None
     #  body           StatNode
+    #  excinfo_target NameNode or None   optional target for exception info
     #  match_flag     string             result of exception match
     #  exc_value      ExcValueNode       used internally
     #  function_name  string             qualified name of enclosing function
     #  exc_vars       (string * 3)       local exception variables
+
+    # excinfo_target is never set by the parser, but can be set by a transform
+    # in order to extract more extensive information about the exception as a
+    # sys.exc_info()-style tuple into a target variable
     
-    child_attrs = ["pattern", "target", "body", "exc_value"]
+    child_attrs = ["pattern", "target", "body", "exc_value", "excinfo_target"]
+
     exc_value = None
+    excinfo_target = None
 
     def analyse_declarations(self, env):
         if self.target:
             self.target.analyse_target_declaration(env)
+        if self.excinfo_target is not None:
+            self.excinfo_target.analyse_target_declaration(env)
         self.body.analyse_declarations(env)
     
     def analyse_expressions(self, env):
@@ -3345,6 +3429,17 @@ class ExceptClauseNode(Node):
             self.exc_value = ExprNodes.ExcValueNode(self.pos, env, self.exc_vars[1])
             self.exc_value.allocate_temps(env)
             self.target.analyse_target_expression(env, self.exc_value)
+        if self.excinfo_target is not None:
+            import ExprNodes
+            self.excinfo_tuple = ExprNodes.TupleNode(pos=self.pos, args=[
+                ExprNodes.ExcValueNode(pos=self.pos, env=env, var=self.exc_vars[0]),
+                ExprNodes.ExcValueNode(pos=self.pos, env=env, var=self.exc_vars[1]),
+                ExprNodes.ExcValueNode(pos=self.pos, env=env, var=self.exc_vars[2])
+            ])
+            self.excinfo_tuple.analyse_expressions(env)
+            self.excinfo_tuple.allocate_temps(env)
+            self.excinfo_target.analyse_target_expression(env, self.excinfo_tuple)
+
         self.body.analyse_expressions(env)
         for var in self.exc_vars:
             env.release_temp(var)
@@ -3374,6 +3469,10 @@ class ExceptClauseNode(Node):
         if self.target:
             self.exc_value.generate_evaluation_code(code)
             self.target.generate_assignment_code(self.exc_value, code)
+        if self.excinfo_target is not None:
+            self.excinfo_tuple.generate_evaluation_code(code)
+            self.excinfo_target.generate_assignment_code(self.excinfo_tuple, code)
+
         old_exc_vars = code.exc_vars
         code.exc_vars = self.exc_vars
         self.body.generate_execution_code(code)
@@ -4484,6 +4583,7 @@ bad:
     Py_XDECREF(*tb);
     return -1;
 }
+
 """]
 
 #------------------------------------------------------------------------------------

Cython/Compiler/ParseTreeTransforms.py

+
+from Cython.Compiler.Visitor import VisitorTransform, temp_name_handle, CythonTransform
+from Cython.Compiler.ModuleNode import ModuleNode
+from Cython.Compiler.Nodes import *
+from Cython.Compiler.ExprNodes import *
+from Cython.Compiler.TreeFragment import TreeFragment
+from Cython.Utils import EncodedString
+from Cython.Compiler.Errors import CompileError
+from sets import Set as set
+
+class NormalizeTree(CythonTransform):
+    """
+    This transform fixes up a few things after parsing
+    in order to make the parse tree more suitable for
+    transforms.
+
+    a) After parsing, blocks with only one statement will
+    be represented by that statement, not by a StatListNode.
+    When doing transforms this is annoying and inconsistent,
+    as one cannot in general remove a statement in a consistent
+    way and so on. This transform wraps any single statements
+    in a StatListNode containing a single statement.
+
+    b) The PassStatNode is a noop and serves no purpose beyond
+    plugging such one-statement blocks; i.e., once parsed a
+`    "pass" can just as well be represented using an empty
+    StatListNode. This means less special cases to worry about
+    in subsequent transforms (one always checks to see if a
+    StatListNode has no children to see if the block is empty).
+    """
+
+    def __init__(self, context):
+        super(NormalizeTree, self).__init__(context)
+        self.is_in_statlist = False
+        self.is_in_expr = False
+
+    def visit_ExprNode(self, node):
+        stacktmp = self.is_in_expr
+        self.is_in_expr = True
+        self.visitchildren(node)
+        self.is_in_expr = stacktmp
+        return node
+
+    def visit_StatNode(self, node, is_listcontainer=False):
+        stacktmp = self.is_in_statlist
+        self.is_in_statlist = is_listcontainer
+        self.visitchildren(node)
+        self.is_in_statlist = stacktmp
+        if not self.is_in_statlist and not self.is_in_expr:
+            return StatListNode(pos=node.pos, stats=[node])
+        else:
+            return node
+
+    def visit_PassStatNode(self, node):
+        if not self.is_in_statlist:
+            return StatListNode(pos=node.pos, stats=[])
+        else:
+            return []
+
+    def visit_StatListNode(self, node):
+        self.is_in_statlist = True
+        self.visitchildren(node)
+        self.is_in_statlist = False
+        return node
+
+    def visit_ParallelAssignmentNode(self, node):
+        return self.visit_StatNode(node, True)
+    
+    def visit_CEnumDefNode(self, node):
+        return self.visit_StatNode(node, True)
+
+    def visit_CStructOrUnionDefNode(self, node):
+        return self.visit_StatNode(node, True)
+
+
+class PostParseError(CompileError): pass
+
+# error strings checked by unit tests, so define them
+ERR_BUF_OPTION_UNKNOWN = '"%s" is not a buffer option'
+ERR_BUF_TOO_MANY = 'Too many buffer options'
+ERR_BUF_DUP = '"%s" buffer option already supplied'
+ERR_BUF_MISSING = '"%s" missing'
+ERR_BUF_INT = '"%s" must be an integer'
+ERR_BUF_NONNEG = '"%s" must be non-negative'
+
+class PostParse(CythonTransform):
+    """
+    Basic interpretation of the parse tree, as well as validity
+    checking that can be done on a very basic level on the parse
+    tree (while still not being a problem with the basic syntax,
+    as such).
+
+    Specifically:
+    - CBufferAccessTypeNode has its options interpreted:
+    Any first positional argument goes into the "dtype" attribute,
+    any "ndim" keyword argument goes into the "ndim" attribute and
+    so on. Also it is checked that the option combination is valid.
+
+    Note: Currently Parsing.py does a lot of interpretation and
+    reorganization that can be refactored into this transform
+    if a more pure Abstract Syntax Tree is wanted.
+    """
+
+    buffer_options = ("dtype", "ndim") # ordered!
+    def visit_CBufferAccessTypeNode(self, node):
+        options = {}
+        # Fetch positional arguments
+        if len(node.positional_args) > len(self.buffer_options):
+            self.context.error(ERR_BUF_TOO_MANY)
+        for arg, unicode_name in zip(node.positional_args, self.buffer_options):
+            name = str(unicode_name)
+            options[name] = arg
+        # Fetch named arguments
+        for item in node.keyword_args.key_value_pairs:
+            name = str(item.key.value)
+            if not name in self.buffer_options:
+                raise PostParseError(item.key.pos,
+                                     ERR_BUF_UNKNOWN % name)
+            if name in options.keys():
+                raise PostParseError(item.key.pos,
+                                     ERR_BUF_DUP % key)
+            options[name] = item.value
+
+        provided = options.keys()
+        # get dtype
+        dtype = options.get("dtype")
+        if dtype is None: raise PostParseError(node.pos, ERR_BUF_MISSING % 'dtype')
+        node.dtype_node = dtype
+
+        # get ndim
+        if "ndim" in provided:
+            ndimnode = options["ndim"]
+            if not isinstance(ndimnode, IntNode):
+                # Compile-time values (DEF) are currently resolved by the parser,
+                # so nothing more to do here
+                raise PostParseError(ndimnode.pos, ERR_BUF_INT % 'ndim')
+            ndim_value = int(ndimnode.value)
+            if ndim_value < 0:
+                raise PostParseError(ndimnode.pos, ERR_BUF_NONNEG % 'ndim')
+            node.ndim = int(ndimnode.value)
+        else:
+            node.ndim = 1
+        
+        # We're done with the parse tree args
+        node.positional_args = None
+        node.keyword_args = None
+        return node
+
+class BufferTransform(CythonTransform):
+    """
+    Run after type analysis. Takes care of the buffer functionality.
+    """
+    scope = None
+
+    def __call__(self, node):
+        cymod = self.context.modules[u'__cython__']
+        self.buffer_type = cymod.entries[u'Py_buffer'].type
+        return super(BufferTransform, self).__call__(node)
+
+    def handle_scope(self, node, scope):
+        # For all buffers, insert extra variables in the scope.
+        # The variables are also accessible from the buffer_info
+        # on the buffer entry
+        bufvars = [(name, entry) for name, entry
+                   in scope.entries.iteritems()
+                   if entry.type.buffer_options is not None]
+                   
+        for name, entry in bufvars:
+            # Variable has buffer opts, declare auxiliary vars
+            bufopts = entry.type.buffer_options
+
+            bufinfo = scope.declare_var(temp_name_handle(u"%s_bufinfo" % name),
+                                        self.buffer_type, node.pos)
+
+            temp_var =  scope.declare_var(temp_name_handle(u"%s_tmp" % name),
+                                        entry.type, node.pos)
+            
+            
+            stridevars = []
+            shapevars = []
+            for idx in range(bufopts.ndim):
+                # stride
+                varname = temp_name_handle(u"%s_%s%d" % (name, "stride", idx))
+                var = scope.declare_var(varname, PyrexTypes.c_int_type, node.pos, is_cdef=True)
+                stridevars.append(var)
+                # shape
+                varname = temp_name_handle(u"%s_%s%d" % (name, "shape", idx))
+                var = scope.declare_var(varname, PyrexTypes.c_uint_type, node.pos, is_cdef=True)
+                shapevars.append(var)
+            entry.buffer_aux = Symtab.BufferAux(bufinfo, stridevars, 
+                                                shapevars)
+            entry.buffer_aux.temp_var = temp_var
+        self.scope = scope
+
+            
+    def visit_ModuleNode(self, node):
+        self.handle_scope(node, node.scope)
+        self.visitchildren(node)
+        return node
+
+    def visit_FuncDefNode(self, node):
+        self.handle_scope(node, node.local_scope)
+        self.visitchildren(node)
+        return node
+
+    acquire_buffer_fragment = TreeFragment(u"""
+        TMP = LHS
+        if TMP is not None:
+            __cython__.PyObject_ReleaseBuffer(<__cython__.PyObject*>TMP, &BUFINFO)
+        TMP = RHS
+        __cython__.PyObject_GetBuffer(<__cython__.PyObject*>TMP, &BUFINFO, 0)
+        ASSIGN_AUX
+        LHS = TMP
+    """)
+
+    fetch_strides = TreeFragment(u"""
+        TARGET = BUFINFO.strides[IDX]
+    """)
+
+    fetch_shape = TreeFragment(u"""
+        TARGET = BUFINFO.shape[IDX]
+    """)
+
+#                ass = SingleAssignmentNode(pos=node.pos,
+#                    lhs=NameNode(node.pos, name=entry.name),
+#                    rhs=IndexNode(node.pos,
+#                        base=AttributeNode(node.pos,
+#                            obj=NameNode(node.pos, name=bufaux.buffer_info_var.name),
+#                            attribute=EncodedString("strides")),
+#                        index=IntNode(node.pos, value=EncodedString(idx))))
+#                print ass.dump()
+    def visit_SingleAssignmentNode(self, node):
+        self.visitchildren(node)
+        bufaux = node.lhs.entry.buffer_aux
+        if bufaux is not None:
+            auxass = []
+            for idx, entry in enumerate(bufaux.stridevars):
+                entry.used = True
+                ass = self.fetch_strides.substitute({
+                    u"TARGET": NameNode(node.pos, name=entry.name),
+                    u"BUFINFO": NameNode(node.pos, name=bufaux.buffer_info_var.name),
+                    u"IDX": IntNode(node.pos, value=EncodedString(idx))
+                })
+                auxass.append(ass)
+
+            for idx, entry in enumerate(bufaux.shapevars):
+                entry.used = True
+                ass = self.fetch_shape.substitute({
+                    u"TARGET": NameNode(node.pos, name=entry.name),
+                    u"BUFINFO": NameNode(node.pos, name=bufaux.buffer_info_var.name),
+                    u"IDX": IntNode(node.pos, value=EncodedString(idx))
+                })
+                auxass.append(ass)
+                
+            bufaux.buffer_info_var.used = True
+            acq = self.acquire_buffer_fragment.substitute({
+                u"TMP" : NameNode(pos=node.pos, name=bufaux.temp_var.name),
+                u"LHS" : node.lhs,
+                u"RHS": node.rhs,
+                u"ASSIGN_AUX": StatListNode(node.pos, stats=auxass),
+                u"BUFINFO": NameNode(pos=node.pos, name=bufaux.buffer_info_var.name)
+            }, pos=node.pos)
+            # Note: The below should probably be refactored into something
+            # like fragment.substitute(..., context=self.context), with
+            # TreeFragment getting context.pipeline_until_now() and
+            # applying it on the fragment.
+            acq.analyse_declarations(self.scope)
+            acq.analyse_expressions(self.scope)
+            stats = acq.stats
+#            stats += [node] # Do assignment after successful buffer acquisition
+         #   print acq.dump()
+            return stats
+        else:
+            return node
+
+    buffer_access = TreeFragment(u"""
+        (<unsigned char*>(BUF.buf + OFFSET))[0]
+    """)
+    def visit_IndexNode(self, node):
+        if node.is_buffer_access:
+            assert node.index is None
+            assert node.indices is not None
+            bufaux = node.base.entry.buffer_aux
+            assert bufaux is not None
+            to_sum = [ IntBinopNode(node.pos, operator='*', operand1=index,
+                                    operand2=NameNode(node.pos, name=stride.name))
+                for index, stride in zip(node.indices, bufaux.stridevars)]
+            print to_sum
+
+            indices = node.indices
+            # reduce * on indices
+            expr = to_sum[0]
+            for next in to_sum[1:]:
+                expr = IntBinopNode(node.pos, operator='+', operand1=expr, operand2=next)
+            tmp= self.buffer_access.substitute({
+                'BUF': NameNode(node.pos, name=bufaux.buffer_info_var.name),
+                'OFFSET': expr
+                })
+            tmp.analyse_expressions(self.scope)
+            return tmp.stats[0].expr
+        else:
+            return node
+
+    def visit_CallNode(self, node):
+###        print node.dump()
+        return node
+    
+#    def visit_FuncDefNode(self, node):
+#        print node.dump()
+    
+
+class WithTransform(CythonTransform):
+
+    # EXCINFO is manually set to a variable that contains
+    # the exc_info() tuple that can be generated by the enclosing except
+    # statement.
+    template_without_target = TreeFragment(u"""
+        MGR = EXPR
+        EXIT = MGR.__exit__
+        MGR.__enter__()
+        EXC = True
+        try:
+            try:
+                BODY
+            except:
+                EXC = False
+                if not EXIT(*EXCINFO):
+                    raise
+        finally:
+            if EXC:
+                EXIT(None, None, None)
+    """, temps=[u'MGR', u'EXC', u"EXIT", u"SYS)"],
+    pipeline=[NormalizeTree(None)])
+
+    template_with_target = TreeFragment(u"""
+        MGR = EXPR
+        EXIT = MGR.__exit__
+        VALUE = MGR.__enter__()
+        EXC = True
+        try:
+            try:
+                TARGET = VALUE
+                BODY
+            except:
+                EXC = False
+                if not EXIT(*EXCINFO):
+                    raise
+        finally:
+            if EXC:
+                EXIT(None, None, None)
+    """, temps=[u'MGR', u'EXC', u"EXIT", u"VALUE", u"SYS"],
+    pipeline=[NormalizeTree(None)])
+
+    def visit_WithStatNode(self, node):
+        excinfo_name = temp_name_handle('EXCINFO')
+        excinfo_namenode = NameNode(pos=node.pos, name=excinfo_name)
+        excinfo_target = NameNode(pos=node.pos, name=excinfo_name)
+        if node.target is not None:
+            result = self.template_with_target.substitute({
+                u'EXPR' : node.manager,
+                u'BODY' : node.body,
+                u'TARGET' : node.target,
+                u'EXCINFO' : excinfo_namenode
+                }, pos = node.pos)
+            # Set except excinfo target to EXCINFO
+            result.stats[4].body.stats[0].except_clauses[0].excinfo_target = excinfo_target
+        else:
+            result = self.template_without_target.substitute({
+                u'EXPR' : node.manager,
+                u'BODY' : node.body,
+                u'EXCINFO' : excinfo_namenode
+                }, pos = node.pos)
+            # Set except excinfo target to EXCINFO
+            result.stats[4].body.stats[0].except_clauses[0].excinfo_target = excinfo_target
+        
+        return result.stats
+
+class AnalyseDeclarationsTransform(CythonTransform):
+
+    def __call__(self, root):
+        self.env_stack = [root.scope]
+        return super(AnalyseDeclarationsTransform, self).__call__(root)        
+    
+    def visit_ModuleNode(self, node):
+        node.analyse_declarations(self.env_stack[-1])
+        self.visitchildren(node)
+        return node
+
+    def visit_FuncDefNode(self, node):
+        lenv = node.create_local_scope(self.env_stack[-1])
+        node.body.analyse_control_flow(lenv) # this will be totally refactored
+        node.declare_arguments(lenv)
+        node.body.analyse_declarations(lenv)
+        self.env_stack.append(lenv)
+        self.visitchildren(node)
+        self.env_stack.pop()
+        return node
+        
+class AnalyseExpressionsTransform(CythonTransform):
+    def visit_ModuleNode(self, node):
+        node.body.analyse_expressions(node.scope)
+        self.visitchildren(node)
+        return node
+        
+    def visit_FuncDefNode(self, node):
+        node.body.analyse_expressions(node.local_scope)
+        self.visitchildren(node)
+        return node
+        
+class MarkClosureVisitor(CythonTransform):
+    
+    needs_closure = False
+    
+    def visit_FuncDefNode(self, node):
+        self.needs_closure = False
+        self.visitchildren(node)
+        node.needs_closure = self.needs_closure
+        self.needs_closure = True
+        return node
+        
+    def visit_ClassDefNode(self, node):
+        self.visitchildren(node)
+        self.needs_closure = True
+        return node
+        
+    def visit_YieldNode(self, node):
+        self.needs_closure = True
+        
+class CreateClosureClasses(CythonTransform):
+    # Output closure classes in module scope for all functions
+    # that need it. 
+    
+    def visit_ModuleNode(self, node):
+        self.module_scope = node.scope
+        self.visitchildren(node)
+        return node
+
+    def create_class_from_scope(self, node, target_module_scope):
+        as_name = temp_name_handle("closure")
+        func_scope = node.local_scope
+
+        entry = target_module_scope.declare_c_class(name = as_name,
+            pos = node.pos, defining = True, implementing = True)
+        class_scope = entry.type.scope
+        for entry in func_scope.entries.values():
+            class_scope.declare_var(pos=node.pos,
+                                    name=entry.name,
+                                    cname=entry.cname,
+                                    type=entry.type,
+                                    is_cdef=True)
+            
+    def visit_FuncDefNode(self, node):
+        self.create_class_from_scope(node, self.module_scope)
+        return node
+        
+

Cython/Compiler/Parsing.py

@@ -312,6 +312,7 @@ def p_call(s, function):
         if s.sy != ',':
             break
         s.next()
+
     if s.sy == '*':
         s.next()
         star_arg = p_simple_expr(s)
@@ -1159,13 +1160,13 @@ def p_for_from_step(s):
 
 inequality_relations = ('<', '<=', '>', '>=')
 
-def p_for_target(s):
+def p_target(s, terminator):
     pos = s.position()
     expr = p_bit_expr(s)
     if s.sy == ',':
         s.next()
         exprs = [expr]
-        while s.sy != 'in':
+        while s.sy != terminator:
             exprs.append(p_bit_expr(s))
             if s.sy != ',':
                 break
@@ -1174,6 +1175,9 @@ def p_for_target(s):
     else:
         return expr
 
+def p_for_target(s):
+    return p_target(s, 'in')
+
 def p_for_iterator(s):
     pos = s.position()
     expr = p_testlist(s)
@@ -1253,8 +1257,17 @@ def p_with_statement(s):
         body = p_suite(s)
         return Nodes.GILStatNode(pos, state = state, body = body)
     else:
-        s.error("Only 'with gil' and 'with nogil' implemented",
-                pos = pos)
+        manager = p_expr(s)
+        target = None
+        if s.sy == 'IDENT' and s.systring == 'as':
+            s.next()
+            allow_multi = (s.sy == '(')
+            target = p_target(s, ':')
+            if not allow_multi and isinstance(target, ExprNodes.TupleNode):
+                s.error("Multiple with statement target values not allowed without paranthesis")
+        body = p_suite(s)
+	return Nodes.WithStatNode(pos, manager = manager, 
+	       			       target = target, body = body)
     
 def p_simple_statement(s, first_statement = 0):
     #print "p_simple_statement:", s.sy, s.systring ###
@@ -1447,6 +1460,71 @@ def p_suite(s, ctx = Ctx(), with_doc = 0, with_pseudo_doc = 0):
     else:
         return body
 
+def p_positional_and_keyword_args(s, end_sy_set, type_positions=(), type_keywords=()):
+    """
+    Parses positional and keyword arguments. end_sy_set
+    should contain any s.sy that terminate the argument list.
+    Argument expansion (* and **) are not allowed.
+
+    type_positions and type_keywords specifies which argument
+    positions and/or names which should be interpreted as
+    types. Other arguments will be treated as expressions.
+
+    Returns: (positional_args, keyword_args)
+    """
+    positional_args = []
+    keyword_args = []
+    pos_idx = 0
+
+    while s.sy not in end_sy_set:
+        if s.sy == '*' or s.sy == '**':
+            s.error('Argument expansion not allowed here.')
+
+        was_keyword = False
+        parsed_type = False
+        if s.sy == 'IDENT':
+            # Since we can have either types or expressions as positional args,
+            # we use a strategy of looking an extra step forward for a '=' and
+            # if it is a positional arg we backtrack.
+            ident = s.systring
+            s.next()
+            if s.sy == '=':
+                s.next()
+                # Is keyword arg
+                if ident in type_keywords:
+                    arg = p_c_base_type(s)
+                    parsed_type = True
+                else:
+                    arg = p_simple_expr(s)
+                keyword_node = ExprNodes.IdentifierStringNode(arg.pos,
+                                value = Utils.EncodedString(ident))
+                keyword_args.append((keyword_node, arg))
+                was_keyword = True
+            else:
+                s.put_back('IDENT', ident)
+                
+        if not was_keyword:
+            if pos_idx in type_positions:
+                arg = p_c_base_type(s)
+                parsed_type = True
+            else:
+                arg = p_simple_expr(s)
+            positional_args.append(arg)
+            pos_idx += 1
+            if len(keyword_args) > 0:
+                s.error("Non-keyword arg following keyword arg",
+                        pos = arg.pos)
+
+        if s.sy != ',':
+            if s.sy not in end_sy_set:
+                if parsed_type:
+                    s.error("Expected: type")
+                else:
+                    s.error("Expected: expression")
+            break
+        s.next()
+    return positional_args, keyword_args
+
 def p_c_base_type(s, self_flag = 0, nonempty = 0):
     # If self_flag is true, this is the base type for the
     # self argument of a C method of an extension type.
@@ -1519,11 +1597,43 @@ def p_c_simple_base_type(s, self_flag, nonempty):
     else:
         #print "p_c_simple_base_type: not looking at type at", s.position()
         name = None
-    return Nodes.CSimpleBaseTypeNode(pos, 
+
+    type_node = Nodes.CSimpleBaseTypeNode(pos, 
         name = name, module_path = module_path,
         is_basic_c_type = is_basic, signed = signed,
         longness = longness, is_self_arg = self_flag)
 
+
+    # Treat trailing [] on type as buffer access
+    if s.sy == '[':
+        if is_basic:
+            p.error("Basic C types do not support buffer access")
+        return p_buffer_access(s, type_node)
+    else:
+        return type_node
+
+def p_buffer_access(s, type_node):
+    # s.sy == '['
+    pos = s.position()
+    s.next()
+    positional_args, keyword_args = (
+        p_positional_and_keyword_args(s, (']',), (0,), ('dtype',))
+    )
+    s.expect(']')
+
+    keyword_dict = ExprNodes.DictNode(pos,
+        key_value_pairs = [
+            ExprNodes.DictItemNode(pos=key.pos, key=key, value=value)
+            for key, value in keyword_args
+        ])
+
+    result = Nodes.CBufferAccessTypeNode(pos,
+        positional_args = positional_args,
+        keyword_args = keyword_dict,
+        base_type_node = type_node)
+    return result
+    
+
 def looking_at_type(s):
     return looking_at_base_type(s) or s.looking_at_type_name()
 

Cython/Compiler/PyrexTypes.py

@@ -4,6 +4,23 @@
 
 from Cython import Utils
 import Naming
+import copy
+
+class BufferOptions:
+    # dtype         PyrexType
+    # ndim          int
+    def __init__(self, dtype, ndim):
+        self.dtype = dtype
+        self.ndim = ndim
+
+
+def create_buffer_type(base_type, buffer_options):
+    # Make a shallow copy of base_type and then annotate it
+    # with the buffer information
+    result = copy.copy(base_type)
+    result.buffer_options = buffer_options
+    return result
+
 
 class BaseType:
     #
@@ -92,6 +109,7 @@ class PyrexType(BaseType):
     default_value = ""
     parsetuple_format = ""
     pymemberdef_typecode = None
+    buffer_options = None # can contain a BufferOptions instance
     
     def resolve(self):
         # If a typedef, returns the base type.
@@ -183,7 +201,6 @@ class CTypedefType(BaseType):
     def __getattr__(self, name):
         return getattr(self.typedef_base_type, name)
 
-
 class PyObjectType(PyrexType):
     #
     #  Base class for all Python object types (reference-counted).

Cython/Compiler/Symtab.py

 #
-#   Pyrex - Symbol Table
+#   Symbol Table
 #
 
 import re
@@ -19,6 +19,14 @@ import __builtin__
 possible_identifier = re.compile(ur"(?![0-9])\w+$", re.U).match
 nice_identifier = re.compile('^[a-zA-Z0-0_]+$').match
 
+class BufferAux:
+    def __init__(self, buffer_info_var, stridevars, shapevars):
+        self.buffer_info_var = buffer_info_var
+        self.stridevars = stridevars
+        self.shapevars = shapevars
+    def __repr__(self):
+        return "<BufferAux %r>" % self.__dict__
+
 class Entry:
     # A symbol table entry in a Scope or ModuleNamespace.
     #
@@ -47,6 +55,7 @@ class Entry:
     # is_self_arg      boolean    Is the "self" arg of an exttype method
     # is_arg           boolean    Is the arg of a method
     # is_local         boolean    Is a local variable
+    # in_closure       boolean    Is referenced in an inner scope
     # is_readonly      boolean    Can't be assigned to
     # func_cname       string     C func implementing Python func
     # pos              position   Source position where declared
@@ -75,6 +84,8 @@ class Entry:
     # defined_in_pxd   boolean    Is defined in a .pxd file (not just declared)
     # api              boolean    Generate C API for C class or function
     # utility_code     string     Utility code needed when this entry is used
+    #
+    # buffer_aux      BufferAux or None  Extra information needed for buffer variables
 
     borrowed = 0
     init = ""
@@ -96,6 +107,7 @@ class Entry:
     is_self_arg = 0
     is_arg = 0
     is_local = 0
+    in_closure = 0
     is_declared_generic = 0
     is_readonly = 0
     func_cname = None
@@ -115,6 +127,7 @@ class Entry:
     api = 0
     utility_code = None
     is_overridable = 0
+    buffer_aux = None
 
     def __init__(self, name, cname, type, pos = None, init = None):
         self.name = name
@@ -163,6 +176,8 @@ class Scope:
     in_cinclude = 0
     nogil = 0
     
+    temp_prefix = Naming.pyrex_prefix
+    
     def __init__(self, name, outer_scope, parent_scope):
         # The outer_scope is the next scope in the lookup chain.
         # The parent_scope is used to derive the qualified name of this scope.
@@ -447,7 +462,14 @@ class Scope:
         # Look up name in this scope or an enclosing one.
         # Return None if not found.
         return (self.lookup_here(name)
-            or (self.outer_scope and self.outer_scope.lookup(name))
+            or (self.outer_scope and self.outer_scope.lookup_from_inner(name))
+            or None)
+
+    def lookup_from_inner(self, name):
+        # Look up name in this scope or an enclosing one.
+        # This is only called from enclosing scopes.
+        return (self.lookup_here(name)
+            or (self.outer_scope and self.outer_scope.lookup_from_inner(name))
             or None)
 
     def lookup_here(self, name):
@@ -562,7 +584,7 @@ class Scope:
                 return entry.cname
         n = self.temp_counter
         self.temp_counter = n + 1
-        cname = "%s%d" % (Naming.pyrex_prefix, n)
+        cname = "%s%d" % (self.temp_prefix, n)
         entry = Entry("", cname, type)
         entry.used = 1
         if type.is_pyobject or type == PyrexTypes.c_py_ssize_t_type:
@@ -608,6 +630,9 @@ class Scope:
         return 0
 
 class PreImportScope(Scope):
+
+    namespace_cname = Naming.preimport_cname
+
     def __init__(self):
         Scope.__init__(self, Options.pre_import, None, None)
         
@@ -615,7 +640,6 @@ class PreImportScope(Scope):
         entry = self.declare(name, name, py_object_type, pos)
         entry.is_variable = True
         entry.is_pyglobal = True
-        entry.namespace_cname = Naming.preimport_cname
         return entry
 
 
@@ -761,6 +785,7 @@ class ModuleScope(Scope):
         self.has_extern_class = 0
         self.cached_builtins = []
         self.undeclared_cached_builtins = []
+        self.namespace_cname = self.module_cname
     
     def qualifying_scope(self):
         return self.parent_module
@@ -876,7 +901,6 @@ class ModuleScope(Scope):
                 raise InternalError(
                     "Non-cdef global variable is not a generic Python object")
             entry.is_pyglobal = 1
-            entry.namespace_cname = self.module_cname
         else:
             entry.is_cglobal = 1
             self.var_entries.append(entry)
@@ -1075,8 +1099,7 @@ class ModuleScope(Scope):
         var_entry.is_readonly = 1
         entry.as_variable = var_entry
         
-
-class LocalScope(Scope):
+class LocalScope(Scope):    
 
     def __init__(self, name, outer_scope):
         Scope.__init__(self, name, outer_scope, outer_scope)
@@ -1119,6 +1142,33 @@ class LocalScope(Scope):
             entry = self.global_scope().lookup_target(name)
             self.entries[name] = entry
         
+    def lookup_from_inner(self, name):
+        entry = self.lookup_here(name)
+        if entry:
+            entry.in_closure = 1
+            return entry
+        else:
+            return (self.outer_scope and self.outer_scope.lookup_from_inner(name)) or None
+            
+    def mangle_closure_cnames(self, scope_var):
+        for entry in self.entries.values():
+            if entry.in_closure:
+                if not hasattr(entry, 'orig_cname'):
+                    entry.orig_cname = entry.cname
+                entry.cname = scope_var + "->" + entry.cname
+                
+
+class GeneratorLocalScope(LocalScope):
+
+    temp_prefix = Naming.cur_scope_cname + "->" + LocalScope.temp_prefix
+    
+    def mangle_closure_cnames(self, scope_var):
+        for entry in self.entries.values() + self.temp_entries:
+            entry.in_closure = 1
+        LocalScope.mangle_closure_cnames(self, scope_var)
+    
+#    def mangle(self, prefix, name):
+#        return "%s->%s" % (Naming.scope_obj_cname, name)
 
 class StructOrUnionScope(Scope):
     #  Namespace of a C struct or union.
@@ -1198,7 +1248,6 @@ class PyClassScope(ClassScope):
         entry = Scope.declare_var(self, name, type, pos, 
             cname, visibility, is_cdef)
         entry.is_pyglobal = 1
-        entry.namespace_cname = self.class_obj_cname
         return entry
 
     def allocate_temp(self, type):
@@ -1295,7 +1344,7 @@ class CClassScope(ClassScope):
             entry.is_pyglobal = 1 # xxx: is_pyglobal changes behaviour in so many places that
                                   # I keep it in for now. is_member should be enough
                                   # later on
-            entry.namespace_cname = "(PyObject *)%s" % self.parent_type.typeptr_cname
+            self.namespace_cname = "(PyObject *)%s" % self.parent_type.typeptr_cname
             entry.interned_cname = self.intern_identifier(name)
             return entry
 

Cython/Compiler/Tests/TestBuffer.py

+from Cython.TestUtils import CythonTest
+import Cython.Compiler.Errors as Errors
+from Cython.Compiler.Nodes import *
+from Cython.Compiler.ParseTreeTransforms import *
+
+
+class TestBufferParsing(CythonTest):
+    # First, we only test the raw parser, i.e.
+    # the number and contents of arguments are NOT checked.
+    # However "dtype"/the first positional argument is special-cased
+    #  to parse a type argument rather than an expression
+
+    def parse(self, s):
+        return self.should_not_fail(lambda: self.fragment(s)).root
+
+    def not_parseable(self, expected_error, s):
+        e = self.should_fail(lambda: self.fragment(s),  Errors.CompileError)
+        self.assertEqual(expected_error, e.message_only)
+    
+    def test_basic(self):
+        t = self.parse(u"cdef object[float, 4, ndim=2, foo=foo] x")
+        bufnode = t.stats[0].base_type
+        self.assert_(isinstance(bufnode, CBufferAccessTypeNode))
+        self.assertEqual(2, len(bufnode.positional_args))
+#        print bufnode.dump()
+        # should put more here...
+        
+    def test_type_fail(self):
+        self.not_parseable("Expected: type",
+                           u"cdef object[2] x")
+    
+    def test_type_pos(self):
+        self.parse(u"cdef object[short unsigned int, 3] x")
+
+    def test_type_keyword(self):
+        self.parse(u"cdef object[foo=foo, dtype=short unsigned int] x")
+
+    def test_notype_as_expr1(self):
+        self.not_parseable("Expected: expression",
+                           u"cdef object[foo2=short unsigned int] x")
+
+    def test_notype_as_expr2(self):
+        self.not_parseable("Expected: expression",
+                           u"cdef object[int, short unsigned int] x")
+
+    def test_pos_after_key(self):
+        self.not_parseable("Non-keyword arg following keyword arg",
+                           u"cdef object[foo=1, 2] x")
+
+class TestBufferOptions(CythonTest):
+    # Tests the full parsing of the options within the brackets
+
+    def parse_opts(self, opts):
+        s = u"cdef object[%s] x" % opts
+        root = self.fragment(s, pipeline=[PostParse(self)]).root
+        buftype = root.stats[0].base_type
+        self.assert_(isinstance(buftype, CBufferAccessTypeNode))
+        self.assert_(isinstance(buftype.base_type_node, CSimpleBaseTypeNode))
+        self.assertEqual(u"object", buftype.base_type_node.name)
+        return buftype
+
+    def non_parse(self, expected_err, opts):
+        e = self.should_fail(lambda: self.parse_opts(opts))
+        self.assertEqual(expected_err, e.message_only)
+        
+    def test_basic(self):
+        buf = self.parse_opts(u"unsigned short int, 3")
+        self.assert_(isinstance(buf.dtype_node, CSimpleBaseTypeNode))
+        self.assert_(buf.dtype_node.signed == 0 and buf.dtype_node.longness == -1)
+        self.assertEqual(3, buf.ndim)
+
+    def test_dict(self):
+        buf = self.parse_opts(u"ndim=3, dtype=unsigned short int")
+        self.assert_(isinstance(buf.dtype_node, CSimpleBaseTypeNode))
+        self.assert_(buf.dtype_node.signed == 0 and buf.dtype_node.longness == -1)
+        self.assertEqual(3, buf.ndim)
+        
+    def test_dtype(self):
+        self.non_parse(ERR_BUF_MISSING % 'dtype', u"")
+    
+    def test_ndim(self):
+        self.parse_opts(u"int, 2")
+        self.non_parse(ERR_BUF_INT % 'ndim', u"int, 'a'")
+        self.non_parse(ERR_BUF_NONNEG % 'ndim', u"int, -34")
+
+    def test_use_DEF(self):
+        t = self.fragment(u"""
+        DEF ndim = 3
+        cdef object[int, ndim] x
+        cdef object[ndim=ndim, dtype=int] y
+        """, pipeline=[PostParse(self)]).root
+        self.assert_(t.stats[1].base_type.ndim == 3)
+        self.assert_(t.stats[2].base_type.ndim == 3)
+
+    # add exotic and impossible combinations as they come along

Cython/Compiler/Tests/TestParseTreeTransforms.py

+from Cython.TestUtils import TransformTest
+from Cython.Compiler.ParseTreeTransforms import *
+from Cython.Compiler.Nodes import *
+
+class TestNormalizeTree(TransformTest):
+    def test_parserbehaviour_is_what_we_coded_for(self):
+        t = self.fragment(u"if x: y").root
+        self.assertLines(u"""
+(root): StatListNode
+  stats[0]: IfStatNode
+    if_clauses[0]: IfClauseNode
+      condition: NameNode
+      body: ExprStatNode
+        expr: NameNode
+""", self.treetypes(t))
+        
+    def test_wrap_singlestat(self):
+    	t = self.run_pipeline([NormalizeTree(None)], u"if x: y")
+        self.assertLines(u"""
+(root): StatListNode
+  stats[0]: IfStatNode
+    if_clauses[0]: IfClauseNode
+      condition: NameNode
+      body: StatListNode
+        stats[0]: ExprStatNode
+          expr: NameNode
+""", self.treetypes(t))
+
+    def test_wrap_multistat(self):
+        t = self.run_pipeline([NormalizeTree(None)], u"""
+            if z:
+                x
+                y
+        """)
+        self.assertLines(u"""
+(root): StatListNode
+  stats[0]: IfStatNode
+    if_clauses[0]: IfClauseNode
+      condition: NameNode
+      body: StatListNode
+        stats[0]: ExprStatNode
+          expr: NameNode
+        stats[1]: ExprStatNode
+          expr: NameNode
+""", self.treetypes(t))
+
+    def test_statinexpr(self):
+        t = self.run_pipeline([NormalizeTree(None)], u"""
+            a, b = x, y
+        """)
+        self.assertLines(u"""
+(root): StatListNode
+  stats[0]: ParallelAssignmentNode
+    stats[0]: SingleAssignmentNode
+      lhs: NameNode
+      rhs: NameNode
+    stats[1]: SingleAssignmentNode
+      lhs: NameNode
+      rhs: NameNode
+""", self.treetypes(t))
+
+    def test_wrap_offagain(self):
+        t = self.run_pipeline([NormalizeTree(None)], u"""
+            x
+            y
+            if z:
+                x
+        """)
+        self.assertLines(u"""
+(root): StatListNode
+  stats[0]: ExprStatNode
+    expr: NameNode
+  stats[1]: ExprStatNode
+    expr: NameNode
+  stats[2]: IfStatNode
+    if_clauses[0]: IfClauseNode
+      condition: NameNode
+      body: StatListNode
+        stats[0]: ExprStatNode
+          expr: NameNode
+""", self.treetypes(t))
+        
+
+    def test_pass_eliminated(self):
+        t = self.run_pipeline([NormalizeTree(None)], u"pass")
+        self.assert_(len(t.stats) == 0)
+
+class TestWithTransform(TransformTest):
+
+    def test_simplified(self):
+        t = self.run_pipeline([WithTransform(None)], u"""
+        with x:
+            y = z ** 3
+        """)
+        
+        self.assertCode(u"""
+
+        $MGR = x
+        $EXIT = $MGR.__exit__
+        $MGR.__enter__()
+        $EXC = True
+        try:
+            try:
+                y = z ** 3
+            except:
+                $EXC = False
+                if (not $EXIT($EXCINFO)):
+                    raise
+        finally:
+            if $EXC:
+                $EXIT(None, None, None)
+
+        """, t)
+
+    def test_basic(self):
+        t = self.run_pipeline([WithTransform(None)], u"""
+        with x as y:
+            y = z ** 3
+        """)
+        self.assertCode(u"""
+
+        $MGR = x
+        $EXIT = $MGR.__exit__
+        $VALUE = $MGR.__enter__()
+        $EXC = True
+        try:
+            try:
+                y = $VALUE
+                y = z ** 3
+            except:
+                $EXC = False
+                if (not $EXIT($EXCINFO)):
+                    raise
+        finally:
+            if $EXC:
+                $EXIT(None, None, None)
+
+        """, t)
+                          
+
+if __name__ == "__main__":
+    import unittest
+    unittest.main()

Cython/Compiler/Tests/TestTreeFragment.py

 from Cython.TestUtils import CythonTest
 from Cython.Compiler.TreeFragment import *
 from Cython.Compiler.Nodes import *
+import Cython.Compiler.Naming as Naming
 
 class TestTreeFragments(CythonTest):
+    
     def test_basic(self):
         F = self.fragment(u"x = 4")
         T = F.copy()
@@ -12,15 +14,15 @@ class TestTreeFragments(CythonTest):
         F = self.fragment(u"if True: x = 4")
         T1 = F.root
         T2 = F.copy()
-        self.assertEqual("x", T2.body.if_clauses[0].body.lhs.name)
-        T2.body.if_clauses[0].body.lhs.name = "other"
-        self.assertEqual("x", T1.body.if_clauses[0].body.lhs.name)
+        self.assertEqual("x", T2.stats[0].if_clauses[0].body.lhs.name)
+        T2.stats[0].if_clauses[0].body.lhs.name = "other"
+        self.assertEqual("x", T1.stats[0].if_clauses[0].body.lhs.name)
 
     def test_substitutions_are_copied(self):
         T = self.fragment(u"y + y").substitute({"y": NameNode(pos=None, name="x")})
-        self.assertEqual("x", T.body.expr.operand1.name)
-        self.assertEqual("x", T.body.expr.operand2.name)
-        self.assert_(T.body.expr.operand1 is not T.body.expr.operand2)
+        self.assertEqual("x", T.stats[0].expr.operand1.name)
+        self.assertEqual("x", T.stats[0].expr.operand2.name)
+        self.assert_(T.stats[0].expr.operand1 is not T.stats[0].expr.operand2)
 
     def test_substitution(self):
         F = self.fragment(u"x = 4")
@@ -32,7 +34,7 @@ class TestTreeFragments(CythonTest):
         F = self.fragment(u"PASS")
         pass_stat = PassStatNode(pos=None)
         T = F.substitute({"PASS" : pass_stat})
-        self.assert_(isinstance(T.body, PassStatNode), T.body)
+        self.assert_(isinstance(T.stats[0], PassStatNode), T)
 
     def test_pos_is_transferred(self):
         F = self.fragment(u"""
@@ -40,11 +42,22 @@ class TestTreeFragments(CythonTest):
         x = u * v ** w
         """)
         T = F.substitute({"v" : NameNode(pos=None, name="a")})
-        v = F.root.body.stats[1].rhs.operand2.operand1
-        a = T.body.stats[1].rhs.operand2.operand1
+        v = F.root.stats[1].rhs.operand2.operand1
+        a = T.stats[1].rhs.operand2.operand1
         self.assertEquals(v.pos, a.pos)
         
-        
+    def test_temps(self):
+        import Cython.Compiler.Visitor as v
+        v.tmpnamectr = 0
+        F = self.fragment(u"""
+            TMP
+            x = TMP
+        """)
+        T = F.substitute(temps=[u"TMP"])
+        s = T.stats
+        self.assert_(s[0].expr.name == Naming.temp_prefix +  u"1_TMP", s[0].expr.name)
+        self.assert_(s[1].rhs.name == Naming.temp_prefix + u"1_TMP")
+        self.assert_(s[0].expr.name !=  u"TMP")
 
 if __name__ == "__main__":
     import unittest

Cython/Compiler/TreeFragment.py

@@ -6,8 +6,8 @@ import re
 from cStringIO import StringIO
 from Scanning import PyrexScanner, StringSourceDescriptor
 from Symtab import BuiltinScope, ModuleScope
-from Visitor import VisitorTransform
-from Nodes import Node
+from Visitor import VisitorTransform, temp_name_handle
+from Nodes import Node, StatListNode
 from ExprNodes import NameNode
 import Parsing
 import Main
@@ -53,7 +53,7 @@ def parse_from_strings(name, code, pxds={}):
     buf = StringIO(code.encode(encoding))
 
     scanner = PyrexScanner(buf, code_source, source_encoding = encoding,
-                     type_names = scope.type_names, context = context)
+                     scope = scope, context = context)
     tree = Parsing.p_module(scanner, 0, module_name)
     return tree
 
@@ -92,27 +92,56 @@ class TemplateTransform(VisitorTransform):
        if its name is listed in the substitutions dictionary in the
        same way. It is the responsibility of the caller to make sure
        that the replacement nodes is a valid expression.
+
+    Also a list "temps" should be passed. Any names listed will
+    be transformed into anonymous, temporary names.
+   
+    Currently supported for tempnames is:
+    NameNode
+    (various function and class definition nodes etc. should be added to this)
     
     Each replacement node gets the position of the substituted node
     recursively applied to every member node.
     """
+
+    def __call__(self, node, substitutions, temps, pos):
+        self.substitutions = substitutions
+        tempdict = {}
+        for key in temps:
+            tempdict[key] = temp_name_handle(key)
+        self.temps = tempdict
+        self.pos = pos
+        return super(TemplateTransform, self).__call__(node)
+
+
     def visit_Node(self, node):
         if node is None:
-            return node
+            return None
         else:
             c = node.clone_node()
+            if self.pos is not None:
+                c.pos = self.pos
             self.visitchildren(c)
             return c
     
     def try_substitution(self, node, key):
         sub = self.substitutions.get(key)
-        if sub is None:
-            return self.visit_Node(node) # make copy as usual
+        if sub is not None:
+            pos = self.pos
+            if pos is None: pos = node.pos
+            return ApplyPositionAndCopy(pos)(sub)
         else:
-            return ApplyPositionAndCopy(node.pos)(sub)
+            return self.visit_Node(node) # make copy as usual
+            
     
     def visit_NameNode(self, node):
-        return self.try_substitution(node, node.name)
+        tempname = self.temps.get(node.name)
+        if tempname is not None:
+            # Replace name with temporary
+            node.name = tempname
+            return self.visit_Node(node)
+        else:
+            return self.try_substitution(node, node.name)
 
     def visit_ExprStatNode(self, node):
         # If an expression-as-statement consists of only a replaceable
@@ -122,10 +151,6 @@ class TemplateTransform(VisitorTransform):
         else:
             return self.visit_Node(node)
     
-    def __call__(self, node, substitutions):
-        self.substitutions = substitutions
-        return super(TemplateTransform, self).__call__(node)
-
 def copy_code_tree(node):
     return TreeCopier()(node)
 
@@ -133,12 +158,12 @@ INDENT_RE = re.compile(ur"^ *")
 def strip_common_indent(lines):
     "Strips empty lines and common indentation from the list of strings given in lines"
     lines = [x for x in lines if x.strip() != u""]
-    minindent = min(len(INDENT_RE.match(x).group(0)) for x in lines)
+    minindent = min([len(INDENT_RE.match(x).group(0)) for x in lines])
     lines = [x[minindent:] for x in lines]
     return lines
     
 class TreeFragment(object):
-    def __init__(self, code, name, pxds={}):
+    def __init__(self, code, name="(tree fragment)", pxds={}, temps=[], pipeline=[]):
         if isinstance(code, unicode):
             def fmt(x): return u"\n".join(strip_common_indent(x.split(u"\n"))) 
             
@@ -147,18 +172,28 @@ class TreeFragment(object):
             for key, value in pxds.iteritems():
                 fmt_pxds[key] = fmt(value)
                 
-            self.root = parse_from_strings(name, fmt_code, fmt_pxds)
+            t = parse_from_strings(name, fmt_code, fmt_pxds)
+            mod = t
+            t = t.body # Make sure a StatListNode is at the top
+            if not isinstance(t, StatListNode):
+                t = StatListNode(pos=mod.pos, stats=[t])
+            for transform in pipeline:
+                t = transform(t)
+            self.root = t
         elif isinstance(code, Node):
             if pxds != {}: raise NotImplementedError()
             self.root = code
         else:
             raise ValueError("Unrecognized code format (accepts unicode and Node)")
+        self.temps = temps
 
     def copy(self):
         return copy_code_tree(self.root)
 
-    def substitute(self, nodes={}):
-        return TemplateTransform()(self.root, substitutions = nodes)
+    def substitute(self, nodes={}, temps=[], pos = None):
+        return TemplateTransform()(self.root,
+                                   substitutions = nodes,
+                                   temps = self.temps + temps, pos = pos)
 
 
 

Cython/Compiler/Visitor.py

 #
 #   Tree visitor and transform framework
 #
+import inspect
 import Nodes
 import ExprNodes
-import inspect
+import Naming
+from Cython.Utils import EncodedString
 
 class BasicVisitor(object):
     """A generic visitor base class which can be used for visiting any kind of object."""
@@ -129,7 +131,6 @@ class VisitorTransform(TreeVisitor):
     was not, an exception will be raised. (Typically you want to ensure that you
     are within a StatListNode or similar before doing this.)
     """
-    
     def visitchildren(self, parent, attrs=None):
         result = super(VisitorTransform, self).visitchildren(parent, attrs)
         for attr, newnode in result.iteritems():
@@ -150,6 +151,19 @@ class VisitorTransform(TreeVisitor):
     def __call__(self, root):
         return self.visit(root)
 
+class CythonTransform(VisitorTransform):
+    """
+    Certain common conventions and utilitues for Cython transforms.
+    """
+    def __init__(self, context):
+        super(CythonTransform, self).__init__()
+        self.context = context
+
+    def visit_Node(self, node):
+        self.visitchildren(node)
+        return node
+
+
 # Utils
 def ensure_statlist(node):
     if not isinstance(node, Nodes.StatListNode):
@@ -166,6 +180,19 @@ def replace_node(ptr, value):
     else:
         getattr(parent, attrname)[listidx] = value
 
+tmpnamectr = 0
+def temp_name_handle(description):
+    global tmpnamectr
+    tmpnamectr += 1
+    return EncodedString(Naming.temp_prefix + u"%d_%s" % (tmpnamectr, description))
+
+def get_temp_name_handle_desc(handle):
+    if not handle.startswith(u"__cyt_"):
+        return None
+    else:
+        idx = handle.find(u"_", 6)
+        return handle[idx+1:]
+    
 class PrintTree(TreeVisitor):
     """Prints a representation of the tree to standard output.
     Subclass and override repr_of to provide more information

Cython/TestUtils.py

@@ -4,8 +4,49 @@ import unittest
 from Cython.Compiler.ModuleNode import ModuleNode
 import Cython.Compiler.Main as Main
 from Cython.Compiler.TreeFragment import TreeFragment, strip_common_indent
+from Cython.Compiler.Visitor import TreeVisitor
+
+class NodeTypeWriter(TreeVisitor):
+    def __init__(self):
+        super(NodeTypeWriter, self).__init__()
+        self._indents = 0
+        self.result = []
+    def visit_Node(self, node):
+        if len(self.access_path) == 0:
+            name = u"(root)"
+        else:
+            tip = self.access_path[-1]
+            if tip[2] is not None:
+                name = u"%s[%d]" % tip[1:3]
+            else:
+                name = tip[1]
+            
+        self.result.append(u"  " * self._indents +
+                           u"%s: %s" % (name, node.__class__.__name__))
+        self._indents += 1
+        self.visitchildren(node)
+        self._indents -= 1
+
+def treetypes(root):
+    """Returns a string representing the tree by class names.
+    There's a leading and trailing whitespace so that it can be
+    compared by simple string comparison while still making test
+    cases look ok."""
+    w = NodeTypeWriter()
+    w.visit(root)
+    return u"\n".join([u""] + w.result + [u""])
 
 class CythonTest(unittest.TestCase):
+
+    def assertLines(self, expected, result):
+        "Checks that the given strings or lists of strings are equal line by line"
+        if not isinstance(expected, list): expected = expected.split(u"\n")
+        if not isinstance(result, list): result = result.split(u"\n")
+        for idx, (expected_line, result_line) in enumerate(zip(expected, result)):
+            self.assertEqual(expected_line, result_line, "Line %d:\nExp: %s\nGot: %s" % (idx, expected_line, result_line))
+        self.assertEqual(len(expected), len(result),
+            "Unmatched lines. Got:\n%s\nExpected:\n%s" % ("\n".join(expected), u"\n".join(result)))
+
     def assertCode(self, expected, result_tree):
         writer = CodeWriter()
         writer.write(result_tree)
@@ -18,13 +59,35 @@ class CythonTest(unittest.TestCase):
         self.assertEqual(len(result_lines), len(expected_lines),
             "Unmatched lines. Got:\n%s\nExpected:\n%s" % ("\n".join(result_lines), expected))
 
-    def fragment(self, code, pxds={}):
+    def fragment(self, code, pxds={}, pipeline=[]):
         "Simply create a tree fragment using the name of the test-case in parse errors."
         name = self.id()
         if name.startswith("__main__."): name = name[len("__main__."):]
         name = name.replace(".", "_")
-        return TreeFragment(code, name, pxds)
-        
+        return TreeFragment(code, name, pxds, pipeline=pipeline)
+
+    def treetypes(self, root):
+        return treetypes(root)
+
+    def should_fail(self, func, exc_type=Exception):
+        """Calls "func" and fails if it doesn't raise the right exception
+        (any exception by default). Also returns the exception in question.
+        """
+        try:
+            func()
+            self.fail("Expected an exception of type %r" % exc_type)
+        except exc_type, e:
+            self.assert_(isinstance(e, exc_type))
+            return e
+
+    def should_not_fail(self, func):
+        """Calls func and succeeds if and only if no exception is raised
+        (i.e. converts exception raising into a failed testcase). Returns
+        the return value of func."""
+        try:
+            return func()
+        except:
+            self.fail()
 
 class TransformTest(CythonTest):
     """
@@ -37,8 +100,8 @@ class TransformTest(CythonTest):
     To create a test case:
      - Call run_pipeline. The pipeline should at least contain the transform you
        are testing; pyx should be either a string (passed to the parser to
-       create a post-parse tree) or a ModuleNode representing input to pipeline.
-       The result will be a transformed result (usually a ModuleNode).
+       create a post-parse tree) or a node representing input to pipeline.
+       The result will be a transformed result.
        
      - Check that the tree is correct. If wanted, assertCode can be used, which
        takes a code string as expected, and a ModuleNode in result_tree
@@ -53,7 +116,6 @@ class TransformTest(CythonTest):
     
     def run_pipeline(self, pipeline, pyx, pxds={}):
         tree = self.fragment(pyx, pxds).root
-        assert isinstance(tree, ModuleNode)
         # Run pipeline
         for T in pipeline:
             tree = T(tree)

Cython/Tests/TestCodeWriter.py

@@ -72,6 +72,9 @@ class TestCodeWriter(CythonTest):
 
     def test_inplace_assignment(self):
         self.t(u"x += 43")
+
+    def test_attribute(self):
+        self.t(u"a.x")
     
 if __name__ == "__main__":
     import unittest

Includes/__cython__.pxd

+cdef extern from "Python.h":
+    ctypedef struct PyObject
+
+
+
+    ctypedef struct Py_buffer:
+        void *buf
+        Py_ssize_t len
+        int readonly
+        char *format
+        int ndim
+        Py_ssize_t *shape
+        Py_ssize_t *strides
+        Py_ssize_t *suboffsets
+        Py_ssize_t itemsize
+        void *internal
+
+    
+    int PyObject_GetBuffer(PyObject* obj, Py_buffer* view, int flags) except -1
+    void PyObject_ReleaseBuffer(PyObject* obj, Py_buffer* view)
+
+#                  int PyObject_GetBuffer(PyObject *obj, Py_buffer *view,
+#                       int flags)

runtests.py

@@ -238,6 +238,29 @@ class CythonRunTestCase(CythonCompileTestCase):
         except Exception:
             pass
 
+def collect_unittests(path, suite, selectors):
+    def file_matches(filename):
+        return filename.startswith("Test") and filename.endswith(".py")
+
+    def package_matches(dirname):
+        return dirname == "Tests"
+
+    loader = unittest.TestLoader()
+
+    for dirpath, dirnames, filenames in os.walk(path):
+        parentname = os.path.split(dirpath)[-1]
+        if package_matches(parentname):
+            for f in filenames:
+                if file_matches(f):
+                    filepath = os.path.join(dirpath, f)[:-len(".py")]
+                    modulename = filepath[len(path)+1:].replace(os.path.sep, '.')
+                    if not [ 1 for match in selectors if match(modulename) ]:
+                        continue
+                    module = __import__(modulename)
+                    for x in modulename.split('.')[1:]:
+                        module = getattr(module, x)
+                    suite.addTests(loader.loadTestsFromModule(module))
+
 if __name__ == '__main__':
     from optparse import OptionParser
     parser = OptionParser()
@@ -247,6 +270,12 @@ if __name__ == '__main__':
     parser.add_option("--no-cython", dest="with_cython",
                       action="store_false", default=True,
                       help="do not run the Cython compiler, only the C compiler")
+    parser.add_option("--no-unit", dest="unittests",
+                      action="store_false", default=True,
+                      help="do not run the unit tests")
+    parser.add_option("--no-file", dest="filetests",
+                      action="store_false", default=True,
+                      help="do not run the file based tests")
     parser.add_option("-C", "--coverage", dest="coverage",
                       action="store_true", default=False,
                       help="collect source coverage data for the Compiler")
@@ -296,9 +325,15 @@ if __name__ == '__main__':
     if not selectors:
         selectors = [ lambda x:True ]
 
-    tests = TestBuilder(ROOTDIR, WORKDIR, selectors,
-                        options.annotate_source, options.cleanup_workdir)
-    test_suite = tests.build_suite()
+    test_suite = unittest.TestSuite()
+
+    if options.unittests:
+        collect_unittests(os.getcwd(), test_suite, selectors)
+
+    if options.filetests:
+        filetests = TestBuilder(ROOTDIR, WORKDIR, selectors,
+                                options.annotate_source, options.cleanup_workdir)
+        test_suite.addTests(filetests.build_suite())
 
     unittest.TextTestRunner(verbosity=options.verbosity).run(test_suite)
 

tests/run/withstat.pyx

+from __future__ import with_statement
+
+__doc__ = u"""
+>>> no_as()
+enter
+hello
+exit <type 'NoneType'> <type 'NoneType'> <type 'NoneType'>
+>>> basic()
+enter
+value
+exit <type 'NoneType'> <type 'NoneType'> <type 'NoneType'>
+>>> with_exception(None)
+enter
+value
+exit <type 'type'> <class 'withstat.MyException'> <type 'traceback'>
+outer except
+>>> with_exception(True)
+enter
+value
+exit <type 'type'> <class 'withstat.MyException'> <type 'traceback'>
+>>> multitarget()
+enter
+1 2 3 4 5
+exit <type 'NoneType'> <type 'NoneType'> <type 'NoneType'>
+>>> tupletarget()
+enter
+(1, 2, (3, (4, 5)))
+exit <type 'NoneType'> <type 'NoneType'> <type 'NoneType'>
+>>> typed()
+enter
+10
+exit <type 'NoneType'> <type 'NoneType'> <type 'NoneType'>
+"""
+
+class MyException(Exception):
+    pass
+
+class ContextManager:
+    def __init__(self, value, exit_ret = None):
+        self.value = value
+        self.exit_ret = exit_ret
+
+    def __exit__(self, a, b, tb):
+        print "exit", type(a), type(b), type(tb)
+        return self.exit_ret
+        
+    def __enter__(self):
+        print "enter"
+        return self.value
+
+def no_as():
+    with ContextManager("value"):
+        print "hello"
+        
+def basic():
+    with ContextManager("value") as x:
+        print x
+
+def with_exception(exit_ret):
+    try:
+        with ContextManager("value", exit_ret=exit_ret) as value:
+            print value
+            raise MyException()
+    except:
+        print "outer except"
+
+def multitarget():
+    with ContextManager((1, 2, (3, (4, 5)))) as (a, b, (c, (d, e))):
+        print a, b, c, d, e
+
+def tupletarget():
+    with ContextManager((1, 2, (3, (4, 5)))) as t:
+        print t
+
+def typed():
+    cdef unsigned char i
+    c = ContextManager(255)
+    with c as i:
+        i += 11
+        print i