merge

Cython/CodeWriter.py

-from Cython.Compiler.Visitor import TreeVisitor
+from Cython.Compiler.Visitor import TreeVisitor, get_temp_name_handle_desc
 from Cython.Compiler.Nodes import *
+from Cython.Compiler.ExprNodes import *
 
 """
 Serializes a Cython code tree to Cython code. This is primarily useful for
@@ -35,6 +36,7 @@ class CodeWriter(TreeVisitor):
             result = LinesResult()
         self.result = result
         self.numindents = 0
+        self.tempnames = {}
     
     def write(self, tree):
         self.visit(tree)
@@ -57,6 +59,12 @@ class CodeWriter(TreeVisitor):
     def line(self, s):
         self.startline(s)
         self.endline()
+
+    def putname(self, name):
+        tmpdesc = get_temp_name_handle_desc(name)
+        if tmpdesc is not None:
+            name = self.tempnames.setdefault(tmpdesc, u"$" +tmpdesc)
+        self.put(name)
     
     def comma_seperated_list(self, items, output_rhs=False):
         if len(items) > 0:
@@ -116,7 +124,7 @@ class CodeWriter(TreeVisitor):
         self.endline()
     
     def visit_NameNode(self, node):
-        self.put(node.name)
+        self.putname(node.name)
     
     def visit_IntNode(self, node):
         self.put(node.value)
@@ -185,10 +193,23 @@ class CodeWriter(TreeVisitor):
         self.comma_seperated_list(node.args) # Might need to discover whether we need () around tuples...hmm...
     
     def visit_SimpleCallNode(self, node):
-        self.put(node.function.name + u"(")
+        self.visit(node.function)
+        self.put(u"(")
         self.comma_seperated_list(node.args)
         self.put(")")
 
+    def visit_GeneralCallNode(self, node):
+        self.visit(node.function)
+        self.put(u"(")
+        posarg = node.positional_args
+        if isinstance(posarg, AsTupleNode):
+            self.visit(posarg.arg)
+        else:
+            self.comma_seperated_list(posarg)
+        if node.keyword_args is not None or node.starstar_arg is not None:
+            raise Exception("Not implemented yet")
+        self.put(u")")
+
     def visit_ExprStatNode(self, node):
         self.startline()
         self.visit(node.expr)
@@ -197,9 +218,83 @@ class CodeWriter(TreeVisitor):
     def visit_InPlaceAssignmentNode(self, node):
         self.startline()
         self.visit(node.lhs)
-        self.put(" %s= " % node.operator)
+        self.put(u" %s= " % node.operator)
         self.visit(node.rhs)
         self.endline()
-    
-    
+        
+    def visit_WithStatNode(self, node):
+        self.startline()
+        self.put(u"with ")
+        self.visit(node.manager)
+        if node.target is not None:
+            self.put(u" as ")
+            self.visit(node.target)
+        self.endline(u":")
+        self.indent()
+        self.visit(node.body)
+        self.dedent()
+        
+    def visit_AttributeNode(self, node):
+        self.visit(node.obj)
+        self.put(u".%s" % node.attribute)
+
+    def visit_BoolNode(self, node):
+        self.put(str(node.value))
+
+    def visit_TryFinallyStatNode(self, node):
+        self.line(u"try:")
+        self.indent()
+        self.visit(node.body)
+        self.dedent()
+        self.line(u"finally:")
+        self.indent()
+        self.visit(node.finally_clause)
+        self.dedent()
+
+    def visit_TryExceptStatNode(self, node):
+        self.line(u"try:")
+        self.indent()
+        self.visit(node.body)
+        self.dedent()
+        for x in node.except_clauses:
+            self.visit(x)
+        if node.else_clause is not None:
+            self.visit(node.else_clause)
+
+    def visit_ExceptClauseNode(self, node):
+        self.startline(u"except")
+        if node.pattern is not None:
+            self.put(u" ")
+            self.visit(node.pattern)
+        if node.target is not None:
+            self.put(u", ")
+            self.visit(node.target)
+        self.endline(":")
+        self.indent()
+        self.visit(node.body)
+        self.dedent()
+
+    def visit_ReturnStatNode(self, node):
+        self.startline("return ")
+        self.visit(node.value)
+        self.endline()
+
+    def visit_DecoratorNode(self, node):
+        self.startline("@")
+        self.visit(node.decorator)
+        self.endline()
+
+    def visit_ReraiseStatNode(self, node):
+        self.line("raise")
+
+    def visit_NoneNode(self, node):
+        self.put(u"None")
+
+    def visit_ImportNode(self, node):
+        self.put(u"(import %s)" % node.module_name.value)
+
+    def visit_NotNode(self, node):
+        self.put(u"(not ")
+        self.visit(node.operand)
+        self.put(u")")
 

Cython/Compiler/Code.py

@@ -64,13 +64,17 @@ class CCodeWriter:
         dl = code.count("{") - code.count("}")
         if dl < 0:
             self.level += dl
+        elif dl == 0 and code.startswith('}'):
+            self.level -= 1
         if self.bol:
             self.indent()
         self._write(code)
         self.bol = 0
         if dl > 0:
             self.level += dl
-    
+        elif dl == 0 and code.startswith('}'):
+            self.level += 1
+
     def increase_indent(self):
         self.level = self.level + 1
     
@@ -200,6 +204,8 @@ class CCodeWriter:
     def put_var_declaration(self, entry, static = 0, dll_linkage = None,
             definition = True):
         #print "Code.put_var_declaration:", entry.name, "definition =", definition ###
+        if entry.in_closure:
+            return
         visibility = entry.visibility
         if visibility == 'private' and not definition:
             #print "...private and not definition, skipping" ###

Cython/Compiler/Errors.py

@@ -32,6 +32,7 @@ class CompileError(PyrexError):
     
     def __init__(self, position = None, message = ""):
         self.position = position
+        self.message_only = message
     # Deprecated and withdrawn in 2.6:
     #   self.message = message
         if position:
@@ -91,6 +92,7 @@ def error(position, message):
     #print "Errors.error:", repr(position), repr(message) ###
     global num_errors
     err = CompileError(position, message)
+#    if position is not None: raise Exception(err) # debug
     line = "%s\n" % err
     if listing_file:
         listing_file.write(line)

Cython/Compiler/Future.py

@@ -7,5 +7,7 @@ def _get_feature(name):
         return object()
 
 unicode_literals = _get_feature("unicode_literals")
+with_statement = _get_feature("with_statement")
+division = _get_feature("division")
 
 del _get_feature

Cython/Compiler/Lexicon.py

@@ -65,6 +65,7 @@ def make_lexicon():
     escapeseq = Str("\\") + (two_oct | three_oct | two_hex |
                              Str('u') + four_hex | Str('x') + two_hex | AnyChar)
     
+    deco = Str("@")
     bra = Any("([{")
     ket = Any(")]}")
     punct = Any(":,;+-*/|&<>=.%`~^?")
@@ -82,6 +83,7 @@ def make_lexicon():
         (longconst, 'LONG'),
         (fltconst, 'FLOAT'),
         (imagconst, 'IMAG'),
+        (deco, 'DECORATOR'),
         (punct | diphthong, TEXT),
         
         (bra, Method('open_bracket_action')),

Cython/Compiler/ModuleNode.py

@@ -5,6 +5,7 @@
 import os, time
 from cStringIO import StringIO
 from PyrexTypes import CPtrType
+import Future
 
 try:
     set
@@ -25,6 +26,10 @@ from PyrexTypes import py_object_type
 from Cython.Utils import open_new_file, replace_suffix
 
 
+def check_c_classes(module_node):
+    module_node.scope.check_c_classes()
+    return module_node
+
 class ModuleNode(Nodes.Node, Nodes.BlockNode):
     #  doc       string or None
     #  body      StatListNode
@@ -32,6 +37,9 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
     #  referenced_modules   [ModuleScope]
     #  module_temp_cname    string
     #  full_module_name     string
+    #
+    #  scope                The module scope.
+    #  compilation_source   A CompilationSource (see Main)
 
     child_attrs = ["body"]
     
@@ -44,10 +52,8 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
             env.doc = self.doc
         self.body.analyse_declarations(env)
     
-    def process_implementation(self, env, options, result):
-        self.analyse_declarations(env)
-        env.check_c_classes()
-        self.body.analyse_expressions(env)
+    def process_implementation(self, options, result):
+        env = self.scope
         env.return_type = PyrexTypes.c_void_type
         self.referenced_modules = []
         self.find_referenced_modules(env, self.referenced_modules, {})
@@ -254,6 +260,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
         self.generate_module_cleanup_func(env, code)
         self.generate_filename_table(code)
         self.generate_utility_functions(env, code)
+        self.generate_buffer_compatability_functions(env, code)
 
         self.generate_declarations_for_modules(env, modules, code.h)
 
@@ -433,6 +440,9 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
         code.putln("  #define PyBUF_F_CONTIGUOUS (0x0040 | PyBUF_STRIDES)")
         code.putln("  #define PyBUF_ANY_CONTIGUOUS (0x0080 | PyBUF_STRIDES)")
         code.putln("  #define PyBUF_INDIRECT (0x0100 | PyBUF_STRIDES)")
+        code.putln("")
+        code.putln("  static int PyObject_GetBuffer(PyObject *obj, Py_buffer *view, int flags);")
+        code.putln("  static void PyObject_ReleaseBuffer(PyObject *obj, Py_buffer *view);")
         code.putln("#endif")
 
         code.put(builtin_module_name_utility_code[0])
@@ -458,8 +468,12 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
         code.putln("  #define PyInt_AsSsize_t              PyLong_AsSsize_t")
         code.putln("  #define PyInt_AsUnsignedLongMask     PyLong_AsUnsignedLongMask")
         code.putln("  #define PyInt_AsUnsignedLongLongMask PyLong_AsUnsignedLongLongMask")
-        code.putln("  #define PyNumber_Divide(x,y)         PyNumber_TrueDivide(x,y)")
+        code.putln("  #define __Pyx_PyNumber_Divide(x,y)         PyNumber_TrueDivide(x,y)")
         code.putln("#else")
+        if Future.division in env.context.future_directives:
+            code.putln("  #define __Pyx_PyNumber_Divide(x,y)         PyNumber_TrueDivide(x,y)")
+        else:
+            code.putln("  #define __Pyx_PyNumber_Divide(x,y)         PyNumber_Divide(x,y)")
         code.putln("  #define PyBytes_Type                 PyString_Type")
         code.putln("#endif")
 
@@ -473,6 +487,9 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
         code.putln("#ifndef __cdecl")
         code.putln("  #define __cdecl")
         code.putln("#endif")
+        code.putln('');
+        code.putln('#define %s 0xB0000000B000B0BBLL' % Naming.default_error);
+        code.putln('');
         self.generate_extern_c_macro_definition(code)
         code.putln("#include <math.h>")
         code.putln("#define %s" % Naming.api_guard_prefix + self.api_name(env))
@@ -1940,6 +1957,90 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
             code.h.put(utility_code[0])
             code.put(utility_code[1])
         code.put(PyrexTypes.type_conversion_functions)
+        code.putln("")
+
+    def generate_buffer_compatability_functions(self, env, code):
+        # will be refactored
+        try:
+            env.entries[u'numpy']
+            code.put("""
+static int numpy_getbuffer(PyObject *obj, Py_buffer *view, int flags) {
+  /* This function is always called after a type-check; safe to cast */
+  PyArrayObject *arr = (PyArrayObject*)obj;
+  PyArray_Descr *type = (PyArray_Descr*)arr->descr;
+
+  
+  int typenum = PyArray_TYPE(obj);
+  if (!PyTypeNum_ISNUMBER(typenum)) {
+    PyErr_Format(PyExc_TypeError, "Only numeric NumPy types currently supported.");
+    return -1;
+  }
+
+  /*
+  NumPy format codes doesn't completely match buffer codes;
+  seems safest to retranslate.
+                            01234567890123456789012345*/
+  const char* base_codes = "?bBhHiIlLqQfdgfdgO";
+
+  char* format = (char*)malloc(4);
+  char* fp = format;
+  *fp++ = type->byteorder;
+  if (PyTypeNum_ISCOMPLEX(typenum)) *fp++ = 'Z';
+  *fp++ = base_codes[typenum];
+  *fp = 0;
+
+  view->buf = arr->data;
+  view->readonly = !PyArray_ISWRITEABLE(obj);
+  view->ndim = PyArray_NDIM(arr);
+  view->strides = PyArray_STRIDES(arr);
+  view->shape = PyArray_DIMS(arr);
+  view->suboffsets = NULL;
+  view->format = format;
+  view->itemsize = type->elsize;
+
+  view->internal = 0;
+  return 0;
+}
+
+static void numpy_releasebuffer(PyObject *obj, Py_buffer *view) {
+  free((char*)view->format);
+  view->format = NULL;
+}
+
+""")
+        except KeyError:
+            pass
+        
+        # For now, hard-code numpy imported as "numpy"
+        types = []
+        try:
+            ndarrtype = env.entries[u'numpy'].as_module.entries['ndarray'].type
+            types.append((ndarrtype.typeptr_cname, "numpy_getbuffer", "numpy_releasebuffer"))
+        except KeyError:
+            pass
+        code.putln("#if PY_VERSION_HEX < 0x02060000")
+        code.putln("static int PyObject_GetBuffer(PyObject *obj, Py_buffer *view, int flags) {")
+        if len(types) > 0:
+            clause = "if"
+            for t, get, release in types:
+                code.putln("%s (__Pyx_TypeTest(obj, %s)) return %s(obj, view, flags);" % (clause, t, get))
+                clause = "else if"
+            code.putln("else {")
+        code.putln("PyErr_Format(PyExc_TypeError, \"'%100s' does not have the buffer interface\", Py_TYPE(obj)->tp_name);")
+        code.putln("return -1;")
+        if len(types) > 0: code.putln("}")
+        code.putln("}")
+        code.putln("")
+        code.putln("static void PyObject_ReleaseBuffer(PyObject *obj, Py_buffer *view) {")
+        if len(types) > 0:
+            clause = "if"
+            for t, get, release in types:
+                code.putln("%s (__Pyx_TypeTest(obj, %s)) %s(obj, view);" % (clause, t, release))
+                clause = "else if"
+        code.putln("}")
+        code.putln("")
+        code.putln("#endif")
+
 
 #------------------------------------------------------------------------------------
 #

Cython/Compiler/Naming.py

@@ -8,6 +8,8 @@
 
 pyrex_prefix    = "__pyx_"
 
+temp_prefix       = u"__cyt_"
+
 builtin_prefix    = pyrex_prefix + "builtin_"
 arg_prefix        = pyrex_prefix + "arg_"
 funcdoc_prefix    = pyrex_prefix + "doc_"
@@ -70,6 +72,9 @@ optional_args_cname = pyrex_prefix + "optional_args"
 no_opt_args      = pyrex_prefix + "no_opt_args"
 import_star      = pyrex_prefix + "import_star"
 import_star_set  = pyrex_prefix + "import_star_set"
+cur_scope_cname  = pyrex_prefix + "cur_scope"
+enc_scope_cname  = pyrex_prefix + "enc_scope"
+default_error    = pyrex_prefix + "ERROR"
 
 line_c_macro = "__LINE__"
 

Cython/Compiler/Optimize.py

 import Nodes
 import ExprNodes
+import PyrexTypes
 import Visitor
 
+def unwrap_node(node):
+    while isinstance(node, ExprNodes.PersistentNode):
+        node = node.arg
+    return node
 
 def is_common_value(a, b):
+    a = unwrap_node(a)
+    b = unwrap_node(b)
     if isinstance(a, ExprNodes.NameNode) and isinstance(b, ExprNodes.NameNode):
         return a.name == b.name
     if isinstance(a, ExprNodes.AttributeNode) and isinstance(b, ExprNodes.AttributeNode):
@@ -11,13 +18,20 @@ def is_common_value(a, b):
     return False
 
 
-class SwitchTransformVisitor(Visitor.VisitorTransform):
-
+class SwitchTransform(Visitor.VisitorTransform):
+    """
+    This transformation tries to turn long if statements into C switch statements. 
+    The requirement is that every clause be an (or of) var == value, where the var
+    is common among all clauses and both var and value are not Python objects. 
+    """
     def extract_conditions(self, cond):
     
         if isinstance(cond, ExprNodes.CoerceToTempNode):
             cond = cond.arg
-        
+
+        if isinstance(cond, ExprNodes.TypecastNode):
+            cond = cond.operand
+    
         if (isinstance(cond, ExprNodes.PrimaryCmpNode) 
                 and cond.cascade is None 
                 and cond.operator == '=='
@@ -40,14 +54,8 @@ class SwitchTransformVisitor(Visitor.VisitorTransform):
                 return t1, c1+c2
         return None, None
         
-    def is_common_value(self, a, b):
-        if isinstance(a, ExprNodes.NameNode) and isinstance(b, ExprNodes.NameNode):
-            return a.name == b.name
-        if isinstance(a, ExprNodes.AttributeNode) and isinstance(b, ExprNodes.AttributeNode):
-            return not a.is_py_attr and is_common_value(a.obj, b.obj)
-        return False
-    
     def visit_IfStatNode(self, node):
+        self.visitchildren(node)
         if len(node.if_clauses) < 3:
             return node
         common_var = None
@@ -56,19 +64,73 @@ class SwitchTransformVisitor(Visitor.VisitorTransform):
             var, conditions = self.extract_conditions(if_clause.condition)
             if var is None:
                 return node
-            elif common_var is not None and not self.is_common_value(var, common_var):
+            elif common_var is not None and not is_common_value(var, common_var):
                 return node
             else:
                 common_var = var
                 cases.append(Nodes.SwitchCaseNode(pos = if_clause.pos,
                                                   conditions = conditions,
                                                   body = if_clause.body))
+        
+        common_var = unwrap_node(common_var)
         return Nodes.SwitchStatNode(pos = node.pos,
                                     test = common_var,
                                     cases = cases,
                                     else_clause = node.else_clause)
-                                    
+
+
     def visit_Node(self, node):
         self.visitchildren(node)
         return node
                               
+
+class FlattenInListTransform(Visitor.VisitorTransform):
+    """
+    This transformation flattens "x in [val1, ..., valn]" into a sequential list
+    of comparisons. 
+    """
+    
+    def visit_PrimaryCmpNode(self, node):
+        self.visitchildren(node)
+        if node.cascade is not None:
+            return node
+        elif node.operator == 'in':
+            conjunction = 'or'
+            eq_or_neq = '=='
+        elif node.operator == 'not_in':
+            conjunction = 'and'
+            eq_or_neq = '!='
+        else:
+            return node
+
+        if isinstance(node.operand2, ExprNodes.TupleNode) or isinstance(node.operand2, ExprNodes.ListNode):
+            args = node.operand2.args
+            if len(args) == 0:
+                return ExprNodes.BoolNode(pos = node.pos, value = node.operator == 'not_in')
+            else:
+                lhs = ExprNodes.PersistentNode(node.operand1, len(args))
+                conds = []
+                for arg in args:
+                    cond = ExprNodes.PrimaryCmpNode(
+                                        pos = node.pos,
+                                        operand1 = lhs,
+                                        operator = eq_or_neq,
+                                        operand2 = arg,
+                                        cascade = None)
+                    conds.append(ExprNodes.TypecastNode(
+                                        pos = node.pos, 
+                                        operand = cond,
+                                        type = PyrexTypes.c_bint_type))
+                def concat(left, right):
+                    return ExprNodes.BoolBinopNode(
+                                        pos = node.pos, 
+                                        operator = conjunction,
+                                        operand1 = left,
+                                        operand2 = right)
+                return reduce(concat, conds)
+        else:
+            return node
+        
+    def visit_Node(self, node):
+        self.visitchildren(node)
+        return node

Cython/Compiler/Parsing.py

@@ -312,6 +312,7 @@ def p_call(s, function):
         if s.sy != ',':
             break
         s.next()
+
     if s.sy == '*':
         s.next()
         star_arg = p_simple_expr(s)
@@ -1159,13 +1160,13 @@ def p_for_from_step(s):
 
 inequality_relations = ('<', '<=', '>', '>=')
 
-def p_for_target(s):
+def p_target(s, terminator):
     pos = s.position()
     expr = p_bit_expr(s)
     if s.sy == ',':
         s.next()
         exprs = [expr]
-        while s.sy != 'in':
+        while s.sy != terminator:
             exprs.append(p_bit_expr(s))
             if s.sy != ',':
                 break
@@ -1174,6 +1175,9 @@ def p_for_target(s):
     else:
         return expr
 
+def p_for_target(s):
+    return p_target(s, 'in')
+
 def p_for_iterator(s):
     pos = s.position()
     expr = p_testlist(s)
@@ -1253,8 +1257,17 @@ def p_with_statement(s):
         body = p_suite(s)
         return Nodes.GILStatNode(pos, state = state, body = body)
     else:
-        s.error("Only 'with gil' and 'with nogil' implemented",
-                pos = pos)
+        manager = p_expr(s)
+        target = None
+        if s.sy == 'IDENT' and s.systring == 'as':
+            s.next()
+            allow_multi = (s.sy == '(')
+            target = p_target(s, ':')
+            if not allow_multi and isinstance(target, ExprNodes.TupleNode):
+                s.error("Multiple with statement target values not allowed without paranthesis")
+        body = p_suite(s)
+	return Nodes.WithStatNode(pos, manager = manager, 
+	       			       target = target, body = body)
     
 def p_simple_statement(s, first_statement = 0):
     #print "p_simple_statement:", s.sy, s.systring ###
@@ -1359,6 +1372,14 @@ def p_statement(s, ctx, first_statement = 0):
         return p_DEF_statement(s)
     elif s.sy == 'IF':
         return p_IF_statement(s, ctx)
+    elif s.sy == 'DECORATOR':
+        if ctx.level not in ('module', 'class', 'c_class', 'property'):
+            s.error('decorator not allowed here')
+        s.level = ctx.level
+        decorators = p_decorators(s)
+        if s.sy != 'def':
+            s.error("Decorators can only be followed by functions ")
+        return p_def_statement(s, decorators)
     else:
         overridable = 0
         if s.sy == 'cdef':
@@ -1447,6 +1468,71 @@ def p_suite(s, ctx = Ctx(), with_doc = 0, with_pseudo_doc = 0):
     else:
         return body
 
+def p_positional_and_keyword_args(s, end_sy_set, type_positions=(), type_keywords=()):
+    """
+    Parses positional and keyword arguments. end_sy_set
+    should contain any s.sy that terminate the argument list.
+    Argument expansion (* and **) are not allowed.
+
+    type_positions and type_keywords specifies which argument
+    positions and/or names which should be interpreted as
+    types. Other arguments will be treated as expressions.
+
+    Returns: (positional_args, keyword_args)
+    """
+    positional_args = []
+    keyword_args = []
+    pos_idx = 0
+
+    while s.sy not in end_sy_set:
+        if s.sy == '*' or s.sy == '**':
+            s.error('Argument expansion not allowed here.')
+
+        was_keyword = False
+        parsed_type = False
+        if s.sy == 'IDENT':
+            # Since we can have either types or expressions as positional args,
+            # we use a strategy of looking an extra step forward for a '=' and
+            # if it is a positional arg we backtrack.
+            ident = s.systring
+            s.next()
+            if s.sy == '=':
+                s.next()
+                # Is keyword arg
+                if ident in type_keywords:
+                    arg = p_c_base_type(s)
+                    parsed_type = True
+                else:
+                    arg = p_simple_expr(s)
+                keyword_node = ExprNodes.IdentifierStringNode(arg.pos,
+                                value = Utils.EncodedString(ident))
+                keyword_args.append((keyword_node, arg))
+                was_keyword = True
+            else:
+                s.put_back('IDENT', ident)
+                
+        if not was_keyword:
+            if pos_idx in type_positions:
+                arg = p_c_base_type(s)
+                parsed_type = True
+            else:
+                arg = p_simple_expr(s)
+            positional_args.append(arg)
+            pos_idx += 1
+            if len(keyword_args) > 0:
+                s.error("Non-keyword arg following keyword arg",
+                        pos = arg.pos)
+
+        if s.sy != ',':
+            if s.sy not in end_sy_set:
+                if parsed_type:
+                    s.error("Expected: type")
+                else:
+                    s.error("Expected: expression")
+            break
+        s.next()
+    return positional_args, keyword_args
+
 def p_c_base_type(s, self_flag = 0, nonempty = 0):
     # If self_flag is true, this is the base type for the
     # self argument of a C method of an extension type.
@@ -1519,11 +1605,43 @@ def p_c_simple_base_type(s, self_flag, nonempty):
     else:
         #print "p_c_simple_base_type: not looking at type at", s.position()
         name = None
-    return Nodes.CSimpleBaseTypeNode(pos, 
+
+    type_node = Nodes.CSimpleBaseTypeNode(pos, 
         name = name, module_path = module_path,
         is_basic_c_type = is_basic, signed = signed,
         longness = longness, is_self_arg = self_flag)
 
+
+    # Treat trailing [] on type as buffer access
+    if 0: # s.sy == '[':
+        if is_basic:
+            s.error("Basic C types do not support buffer access")
+        return p_buffer_access(s, type_node)
+    else:
+        return type_node
+
+def p_buffer_access(s, type_node):
+    # s.sy == '['
+    pos = s.position()
+    s.next()
+    positional_args, keyword_args = (
+        p_positional_and_keyword_args(s, (']',), (0,), ('dtype',))
+    )
+    s.expect(']')
+
+    keyword_dict = ExprNodes.DictNode(pos,
+        key_value_pairs = [
+            ExprNodes.DictItemNode(pos=key.pos, key=key, value=value)
+            for key, value in keyword_args
+        ])
+
+    result = Nodes.CBufferAccessTypeNode(pos,
+        positional_args = positional_args,
+        keyword_args = keyword_dict,
+        base_type_node = type_node)
+    return result
+    
+
 def looking_at_type(s):
     return looking_at_base_type(s) or s.looking_at_type_name()
 
@@ -1668,7 +1786,7 @@ def p_c_simple_declarator(s, ctx, empty, is_type, cmethod_flag,
             name = ""
             cname = None
         result = Nodes.CNameDeclaratorNode(pos,
-            name = name, cname = cname, rhs = rhs)
+            name = name, cname = cname, default = rhs)
     result.calling_convention = calling_convention
     return result
 
@@ -1993,7 +2111,21 @@ def p_ctypedef_statement(s, ctx):
             declarator = declarator, visibility = visibility,
             in_pxd = ctx.level == 'module_pxd')
 
-def p_def_statement(s):
+def p_decorators(s):
+    decorators = []
+    while s.sy == 'DECORATOR':
+        pos = s.position()
+        s.next()
+        decorator = ExprNodes.NameNode(
+            pos, name = Utils.EncodedString(
+                p_dotted_name(s, as_allowed=0)[2] ))
+        if s.sy == '(':
+            decorator = p_call(s, decorator)
+        decorators.append(Nodes.DecoratorNode(pos, decorator=decorator))
+        s.expect_newline("Expected a newline after decorator")
+    return decorators
+
+def p_def_statement(s, decorators=None):
     # s.sy == 'def'
     pos = s.position()
     s.next()
@@ -2022,7 +2154,7 @@ def p_def_statement(s):
     doc, body = p_suite(s, Ctx(level = 'function'), with_doc = 1)
     return Nodes.DefNode(pos, name = name, args = args, 
         star_arg = star_arg, starstar_arg = starstar_arg,
-        doc = doc, body = body)
+        doc = doc, body = body, decorators = decorators)
 
 def p_py_arg_decl(s):
     pos = s.position()

Cython/Compiler/PyrexTypes.py

@@ -4,6 +4,23 @@
 
 from Cython import Utils
 import Naming
+import copy
+
+class BufferOptions:
+    # dtype         PyrexType
+    # ndim          int
+    def __init__(self, dtype, ndim):
+        self.dtype = dtype
+        self.ndim = ndim
+
+
+def create_buffer_type(base_type, buffer_options):
+    # Make a shallow copy of base_type and then annotate it
+    # with the buffer information
+    result = copy.copy(base_type)
+    result.buffer_options = buffer_options
+    return result
+
 
 class BaseType:
     #
@@ -44,6 +61,7 @@ class PyrexType(BaseType):
     #  default_value         string      Initial value
     #  parsetuple_format     string      Format char for PyArg_ParseTuple
     #  pymemberdef_typecode  string      Type code for PyMemberDef struct
+    #  typestring            string      String char defining the type (see Python struct module)
     #
     #  declaration_code(entity_code, 
     #      for_display = 0, dll_linkage = None, pyrex = 0)
@@ -92,6 +110,7 @@ class PyrexType(BaseType):
     default_value = ""
     parsetuple_format = ""
     pymemberdef_typecode = None
+    buffer_options = None # can contain a BufferOptions instance
     
     def resolve(self):
         # If a typedef, returns the base type.
@@ -183,7 +202,6 @@ class CTypedefType(BaseType):
     def __getattr__(self, name):
         return getattr(self.typedef_base_type, name)
 
-
 class PyObjectType(PyrexType):
     #
     #  Base class for all Python object types (reference-counted).
@@ -399,9 +417,10 @@ class CNumericType(CType):
     
     sign_words = ("unsigned ", "", "signed ")
     
-    def __init__(self, rank, signed = 1, pymemberdef_typecode = None):
+    def __init__(self, rank, signed = 1, pymemberdef_typecode = None, typestring = None):
         self.rank = rank
         self.signed = signed
+        self.typestring = typestring
         ptf = self.parsetuple_formats[signed][rank]
         if ptf == '?':
             ptf = None
@@ -434,8 +453,9 @@ class CIntType(CNumericType):
     from_py_function = "__pyx_PyInt_AsLong"
     exception_value = -1
 
-    def __init__(self, rank, signed, pymemberdef_typecode = None, is_returncode = 0):
-        CNumericType.__init__(self, rank, signed, pymemberdef_typecode)
+    def __init__(self, rank, signed, pymemberdef_typecode = None, is_returncode = 0,
+                 typestring=None):
+        CNumericType.__init__(self, rank, signed, pymemberdef_typecode, typestring=typestring)
         self.is_returncode = is_returncode
         if self.from_py_function == '__pyx_PyInt_AsLong':
             self.from_py_function = self.get_type_conversion()
@@ -526,8 +546,8 @@ class CFloatType(CNumericType):
     to_py_function = "PyFloat_FromDouble"
     from_py_function = "__pyx_PyFloat_AsDouble"
     
-    def __init__(self, rank, pymemberdef_typecode = None):
-        CNumericType.__init__(self, rank, 1, pymemberdef_typecode)
+    def __init__(self, rank, pymemberdef_typecode = None, typestring=None):
+        CNumericType.__init__(self, rank, 1, pymemberdef_typecode, typestring = typestring)
     
     def assignable_from_resolved_type(self, src_type):
         return src_type.is_numeric or src_type is error_type
@@ -835,9 +855,12 @@ class CFuncTypeArg:
     #  type       PyrexType
     #  pos        source file position
     
-    def __init__(self, name, type, pos):
+    def __init__(self, name, type, pos, cname=None):
         self.name = name
-        self.cname = Naming.var_prefix + name
+        if cname is not None:
+            self.cname = cname
+        else:
+            self.cname = Naming.var_prefix + name
         self.type = type
         self.pos = pos
         self.not_none = False
@@ -1033,29 +1056,29 @@ c_void_type =         CVoidType()
 c_void_ptr_type =     CPtrType(c_void_type)
 c_void_ptr_ptr_type = CPtrType(c_void_ptr_type)
 
-c_uchar_type =       CIntType(0, 0, "T_UBYTE")
-c_ushort_type =      CIntType(1, 0, "T_USHORT")
-c_uint_type =        CUIntType(2, 0, "T_UINT")
-c_ulong_type =       CULongType(3, 0, "T_ULONG")
-c_ulonglong_type =   CULongLongType(4, 0, "T_ULONGLONG")
-
-c_char_type =        CIntType(0, 1, "T_CHAR")
-c_short_type =       CIntType(1, 1, "T_SHORT")
-c_int_type =         CIntType(2, 1, "T_INT")
-c_long_type =        CIntType(3, 1, "T_LONG")
-c_longlong_type =    CLongLongType(4, 1, "T_LONGLONG")
+c_uchar_type =       CIntType(0, 0, "T_UBYTE", typestring="B")
+c_ushort_type =      CIntType(1, 0, "T_USHORT", typestring="H")
+c_uint_type =        CUIntType(2, 0, "T_UINT", typestring="I")
+c_ulong_type =       CULongType(3, 0, "T_ULONG", typestring="L")
+c_ulonglong_type =   CULongLongType(4, 0, "T_ULONGLONG", typestring="Q")
+
+c_char_type =        CIntType(0, 1, "T_CHAR", typestring="b")
+c_short_type =       CIntType(1, 1, "T_SHORT", typestring="h")
+c_int_type =         CIntType(2, 1, "T_INT", typestring="i")
+c_long_type =        CIntType(3, 1, "T_LONG", typestring="l")
+c_longlong_type =    CLongLongType(4, 1, "T_LONGLONG", typestring="q")
 c_py_ssize_t_type =  CPySSizeTType(5, 1)
-c_bint_type =        CBIntType(2, 1, "T_INT")
+c_bint_type =        CBIntType(2, 1, "T_INT", typestring="i")
 
-c_schar_type =       CIntType(0, 2, "T_CHAR")
-c_sshort_type =      CIntType(1, 2, "T_SHORT")
-c_sint_type =        CIntType(2, 2, "T_INT")
-c_slong_type =       CIntType(3, 2, "T_LONG")
-c_slonglong_type =   CLongLongType(4, 2, "T_LONGLONG")
+c_schar_type =       CIntType(0, 2, "T_CHAR", typestring="b")
+c_sshort_type =      CIntType(1, 2, "T_SHORT", typestring="h")
+c_sint_type =        CIntType(2, 2, "T_INT", typestring="i")
+c_slong_type =       CIntType(3, 2, "T_LONG", typestring="l")
+c_slonglong_type =   CLongLongType(4, 2, "T_LONGLONG", typestring="q")
 
-c_float_type =       CFloatType(6, "T_FLOAT")
-c_double_type =      CFloatType(7, "T_DOUBLE")
-c_longdouble_type =  CFloatType(8)
+c_float_type =       CFloatType(6, "T_FLOAT", typestring="f")
+c_double_type =      CFloatType(7, "T_DOUBLE", typestring="d")
+c_longdouble_type =  CFloatType(8, typestring="g")
 
 c_null_ptr_type =     CNullPtrType(c_void_type)
 c_char_array_type =   CCharArrayType(None)
@@ -1070,7 +1093,8 @@ c_returncode_type =   CIntType(2, 1, "T_INT", is_returncode = 1)
 c_anon_enum_type =    CAnonEnumType(-1, 1)
 
 # the Py_buffer type is defined in Builtin.py
-c_py_buffer_ptr_type = CPtrType(CStructOrUnionType("Py_buffer", "struct", None, 1, "Py_buffer"))
+c_py_buffer_type = CStructOrUnionType("Py_buffer", "struct", None, 1, "Py_buffer")
+c_py_buffer_ptr_type = CPtrType(c_py_buffer_type)
 
 error_type =    ErrorType()
 

Cython/Compiler/Symtab.py

 #
-#   Pyrex - Symbol Table
+#   Symbol Table
 #
 
 import re
@@ -19,6 +19,16 @@ import __builtin__
 possible_identifier = re.compile(ur"(?![0-9])\w+$", re.U).match
 nice_identifier = re.compile('^[a-zA-Z0-0_]+$').match
 
+class BufferAux:
+    def __init__(self, buffer_info_var, stridevars, shapevars, tschecker):
+        self.buffer_info_var = buffer_info_var
+        self.stridevars = stridevars
+        self.shapevars = shapevars
+        self.tschecker = tschecker
+        
+    def __repr__(self):
+        return "<BufferAux %r>" % self.__dict__
+
 class Entry:
     # A symbol table entry in a Scope or ModuleNamespace.
     #
@@ -47,6 +57,7 @@ class Entry:
     # is_self_arg      boolean    Is the "self" arg of an exttype method
     # is_arg           boolean    Is the arg of a method
     # is_local         boolean    Is a local variable
+    # in_closure       boolean    Is referenced in an inner scope
     # is_readonly      boolean    Can't be assigned to
     # func_cname       string     C func implementing Python func
     # pos              position   Source position where declared
@@ -75,6 +86,8 @@ class Entry:
     # defined_in_pxd   boolean    Is defined in a .pxd file (not just declared)
     # api              boolean    Generate C API for C class or function
     # utility_code     string     Utility code needed when this entry is used
+    #
+    # buffer_aux      BufferAux or None  Extra information needed for buffer variables
 
     borrowed = 0
     init = ""
@@ -96,6 +109,7 @@ class Entry:
     is_self_arg = 0
     is_arg = 0
     is_local = 0
+    in_closure = 0
     is_declared_generic = 0
     is_readonly = 0
     func_cname = None
@@ -115,6 +129,7 @@ class Entry:
     api = 0
     utility_code = None
     is_overridable = 0
+    buffer_aux = None
 
     def __init__(self, name, cname, type, pos = None, init = None):
         self.name = name
@@ -163,6 +178,8 @@ class Scope:
     in_cinclude = 0
     nogil = 0
     
+    temp_prefix = Naming.pyrex_prefix
+    
     def __init__(self, name, outer_scope, parent_scope):
         # The outer_scope is the next scope in the lookup chain.
         # The parent_scope is used to derive the qualified name of this scope.
@@ -447,7 +464,14 @@ class Scope:
         # Look up name in this scope or an enclosing one.
         # Return None if not found.
         return (self.lookup_here(name)
-            or (self.outer_scope and self.outer_scope.lookup(name))
+            or (self.outer_scope and self.outer_scope.lookup_from_inner(name))
+            or None)
+
+    def lookup_from_inner(self, name):
+        # Look up name in this scope or an enclosing one.
+        # This is only called from enclosing scopes.
+        return (self.lookup_here(name)
+            or (self.outer_scope and self.outer_scope.lookup_from_inner(name))
             or None)
 
     def lookup_here(self, name):
@@ -562,7 +586,7 @@ class Scope:
                 return entry.cname
         n = self.temp_counter
         self.temp_counter = n + 1
-        cname = "%s%d" % (Naming.pyrex_prefix, n)
+        cname = "%s%d" % (self.temp_prefix, n)
         entry = Entry("", cname, type)
         entry.used = 1
         if type.is_pyobject or type == PyrexTypes.c_py_ssize_t_type:
@@ -608,6 +632,9 @@ class Scope:
         return 0
 
 class PreImportScope(Scope):
+
+    namespace_cname = Naming.preimport_cname
+
     def __init__(self):
         Scope.__init__(self, Options.pre_import, None, None)
         
@@ -615,7 +642,6 @@ class PreImportScope(Scope):
         entry = self.declare(name, name, py_object_type, pos)
         entry.is_variable = True
         entry.is_pyglobal = True
-        entry.namespace_cname = Naming.preimport_cname
         return entry
 
 
@@ -761,6 +787,7 @@ class ModuleScope(Scope):
         self.has_extern_class = 0
         self.cached_builtins = []
         self.undeclared_cached_builtins = []
+        self.namespace_cname = self.module_cname
     
     def qualifying_scope(self):
         return self.parent_module
@@ -876,7 +903,6 @@ class ModuleScope(Scope):
                 raise InternalError(
                     "Non-cdef global variable is not a generic Python object")
             entry.is_pyglobal = 1
-            entry.namespace_cname = self.module_cname
         else:
             entry.is_cglobal = 1
             self.var_entries.append(entry)
@@ -1075,8 +1101,7 @@ class ModuleScope(Scope):
         var_entry.is_readonly = 1
         entry.as_variable = var_entry
         
-
-class LocalScope(Scope):
+class LocalScope(Scope):    
 
     def __init__(self, name, outer_scope):
         Scope.__init__(self, name, outer_scope, outer_scope)
@@ -1119,6 +1144,33 @@ class LocalScope(Scope):
             entry = self.global_scope().lookup_target(name)
             self.entries[name] = entry
         
+    def lookup_from_inner(self, name):
+        entry = self.lookup_here(name)
+        if entry:
+            entry.in_closure = 1
+            return entry
+        else:
+            return (self.outer_scope and self.outer_scope.lookup_from_inner(name)) or None
+            
+    def mangle_closure_cnames(self, scope_var):
+        for entry in self.entries.values():
+            if entry.in_closure:
+                if not hasattr(entry, 'orig_cname'):
+                    entry.orig_cname = entry.cname
+                entry.cname = scope_var + "->" + entry.cname
+                
+
+class GeneratorLocalScope(LocalScope):
+
+    temp_prefix = Naming.cur_scope_cname + "->" + LocalScope.temp_prefix
+    
+    def mangle_closure_cnames(self, scope_var):
+        for entry in self.entries.values() + self.temp_entries:
+            entry.in_closure = 1
+        LocalScope.mangle_closure_cnames(self, scope_var)
+    
+#    def mangle(self, prefix, name):
+#        return "%s->%s" % (Naming.scope_obj_cname, name)
 
 class StructOrUnionScope(Scope):
     #  Namespace of a C struct or union.
@@ -1198,7 +1250,6 @@ class PyClassScope(ClassScope):
         entry = Scope.declare_var(self, name, type, pos, 
             cname, visibility, is_cdef)
         entry.is_pyglobal = 1
-        entry.namespace_cname = self.class_obj_cname
         return entry
 
     def allocate_temp(self, type):
@@ -1295,7 +1346,7 @@ class CClassScope(ClassScope):
             entry.is_pyglobal = 1 # xxx: is_pyglobal changes behaviour in so many places that
                                   # I keep it in for now. is_member should be enough
                                   # later on
-            entry.namespace_cname = "(PyObject *)%s" % self.parent_type.typeptr_cname
+            self.namespace_cname = "(PyObject *)%s" % self.parent_type.typeptr_cname
             entry.interned_cname = self.intern_identifier(name)
             return entry
 

Cython/Compiler/Tests/TestBuffer.py

+from Cython.TestUtils import CythonTest
+import Cython.Compiler.Errors as Errors
+from Cython.Compiler.Nodes import *
+from Cython.Compiler.ParseTreeTransforms import *
+
+
+class TestBufferParsing(CythonTest):
+    # First, we only test the raw parser, i.e.
+    # the number and contents of arguments are NOT checked.
+    # However "dtype"/the first positional argument is special-cased
+    #  to parse a type argument rather than an expression
+
+    def parse(self, s):
+        return self.should_not_fail(lambda: self.fragment(s)).root
+
+    def not_parseable(self, expected_error, s):
+        e = self.should_fail(lambda: self.fragment(s),  Errors.CompileError)
+        self.assertEqual(expected_error, e.message_only)
+    
+    def test_basic(self):
+        t = self.parse(u"cdef object[float, 4, ndim=2, foo=foo] x")
+        bufnode = t.stats[0].base_type
+        self.assert_(isinstance(bufnode, CBufferAccessTypeNode))
+        self.assertEqual(2, len(bufnode.positional_args))
+#        print bufnode.dump()
+        # should put more here...
+        
+    def test_type_fail(self):
+        self.not_parseable("Expected: type",
+                           u"cdef object[2] x")
+    
+    def test_type_pos(self):
+        self.parse(u"cdef object[short unsigned int, 3] x")
+
+    def test_type_keyword(self):
+        self.parse(u"cdef object[foo=foo, dtype=short unsigned int] x")
+
+    def test_notype_as_expr1(self):
+        self.not_parseable("Expected: expression",
+                           u"cdef object[foo2=short unsigned int] x")
+
+    def test_notype_as_expr2(self):
+        self.not_parseable("Expected: expression",
+                           u"cdef object[int, short unsigned int] x")
+
+    def test_pos_after_key(self):
+        self.not_parseable("Non-keyword arg following keyword arg",
+                           u"cdef object[foo=1, 2] x")
+
+class TestBufferOptions(CythonTest):
+    # Tests the full parsing of the options within the brackets
+
+    def parse_opts(self, opts):
+        s = u"cdef object[%s] x" % opts
+        root = self.fragment(s, pipeline=[PostParse(self)]).root
+        buftype = root.stats[0].base_type
+        self.assert_(isinstance(buftype, CBufferAccessTypeNode))
+        self.assert_(isinstance(buftype.base_type_node, CSimpleBaseTypeNode))
+        self.assertEqual(u"object", buftype.base_type_node.name)
+        return buftype
+
+    def non_parse(self, expected_err, opts):
+        e = self.should_fail(lambda: self.parse_opts(opts))
+        self.assertEqual(expected_err, e.message_only)
+        
+    def test_basic(self):
+        buf = self.parse_opts(u"unsigned short int, 3")
+        self.assert_(isinstance(buf.dtype_node, CSimpleBaseTypeNode))
+        self.assert_(buf.dtype_node.signed == 0 and buf.dtype_node.longness == -1)
+        self.assertEqual(3, buf.ndim)
+
+    def test_dict(self):
+        buf = self.parse_opts(u"ndim=3, dtype=unsigned short int")
+        self.assert_(isinstance(buf.dtype_node, CSimpleBaseTypeNode))
+        self.assert_(buf.dtype_node.signed == 0 and buf.dtype_node.longness == -1)
+        self.assertEqual(3, buf.ndim)
+        
+    def test_dtype(self):
+        self.non_parse(ERR_BUF_MISSING % 'dtype', u"")
+    
+    def test_ndim(self):
+        self.parse_opts(u"int, 2")
+        self.non_parse(ERR_BUF_INT % 'ndim', u"int, 'a'")
+        self.non_parse(ERR_BUF_NONNEG % 'ndim', u"int, -34")
+
+    def test_use_DEF(self):
+        t = self.fragment(u"""
+        DEF ndim = 3
+        cdef object[int, ndim] x
+        cdef object[ndim=ndim, dtype=int] y
+        """, pipeline=[PostParse(self)]).root
+        self.assert_(t.stats[1].base_type.ndim == 3)
+        self.assert_(t.stats[2].base_type.ndim == 3)
+
+    # add exotic and impossible combinations as they come along

Cython/Compiler/Tests/TestDecorators.py

+import unittest
+from Cython.TestUtils import TransformTest
+from Cython.Compiler.ParseTreeTransforms import DecoratorTransform
+
+class TestDecorator(TransformTest):
+
+    def test_decorator(self):
+        t = self.run_pipeline([DecoratorTransform(None)], u"""
+        def decorator(fun):
+            return fun
+        @decorator
+        def decorated():
+            pass
+        """)
+        
+        self.assertCode(u"""
+        def decorator(fun):
+            return fun
+        def decorated():
+            pass
+        decorated = decorator(decorated)
+        """, t)
+
+if __name__ == '__main__':
+    unittest.main()

Cython/Compiler/Tests/TestParseTreeTransforms.py

+from Cython.TestUtils import TransformTest
+from Cython.Compiler.ParseTreeTransforms import *
+from Cython.Compiler.Nodes import *
+
+class TestNormalizeTree(TransformTest):
+    def test_parserbehaviour_is_what_we_coded_for(self):
+        t = self.fragment(u"if x: y").root
+        self.assertLines(u"""
+(root): StatListNode
+  stats[0]: IfStatNode
+    if_clauses[0]: IfClauseNode
+      condition: NameNode
+      body: ExprStatNode
+        expr: NameNode
+""", self.treetypes(t))
+        
+    def test_wrap_singlestat(self):
+    	t = self.run_pipeline([NormalizeTree(None)], u"if x: y")
+        self.assertLines(u"""
+(root): StatListNode
+  stats[0]: IfStatNode
+    if_clauses[0]: IfClauseNode
+      condition: NameNode
+      body: StatListNode
+        stats[0]: ExprStatNode
+          expr: NameNode
+""", self.treetypes(t))
+
+    def test_wrap_multistat(self):
+        t = self.run_pipeline([NormalizeTree(None)], u"""
+            if z:
+                x
+                y
+        """)
+        self.assertLines(u"""
+(root): StatListNode
+  stats[0]: IfStatNode
+    if_clauses[0]: IfClauseNode
+      condition: NameNode
+      body: StatListNode
+        stats[0]: ExprStatNode
+          expr: NameNode
+        stats[1]: ExprStatNode
+          expr: NameNode
+""", self.treetypes(t))
+
+    def test_statinexpr(self):
+        t = self.run_pipeline([NormalizeTree(None)], u"""
+            a, b = x, y
+        """)
+        self.assertLines(u"""
+(root): StatListNode
+  stats[0]: ParallelAssignmentNode
+    stats[0]: SingleAssignmentNode
+      lhs: NameNode
+      rhs: NameNode
+    stats[1]: SingleAssignmentNode
+      lhs: NameNode
+      rhs: NameNode
+""", self.treetypes(t))
+
+    def test_wrap_offagain(self):
+        t = self.run_pipeline([NormalizeTree(None)], u"""
+            x
+            y
+            if z:
+                x
+        """)
+        self.assertLines(u"""
+(root): StatListNode
+  stats[0]: ExprStatNode
+    expr: NameNode
+  stats[1]: ExprStatNode
+    expr: NameNode
+  stats[2]: IfStatNode
+    if_clauses[0]: IfClauseNode
+      condition: NameNode
+      body: StatListNode
+        stats[0]: ExprStatNode
+          expr: NameNode
+""", self.treetypes(t))
+        
+
+    def test_pass_eliminated(self):
+        t = self.run_pipeline([NormalizeTree(None)], u"pass")
+        self.assert_(len(t.stats) == 0)
+
+class TestWithTransform(TransformTest):
+
+    def test_simplified(self):
+        t = self.run_pipeline([WithTransform(None)], u"""
+        with x:
+            y = z ** 3
+        """)
+        
+        self.assertCode(u"""
+
+        $MGR = x
+        $EXIT = $MGR.__exit__
+        $MGR.__enter__()
+        $EXC = True
+        try:
+            try:
+                y = z ** 3
+            except:
+                $EXC = False
+                if (not $EXIT($EXCINFO)):
+                    raise
+        finally:
+            if $EXC:
+                $EXIT(None, None, None)
+
+        """, t)
+
+    def test_basic(self):
+        t = self.run_pipeline([WithTransform(None)], u"""
+        with x as y:
+            y = z ** 3
+        """)
+        self.assertCode(u"""
+
+        $MGR = x
+        $EXIT = $MGR.__exit__
+        $VALUE = $MGR.__enter__()
+        $EXC = True
+        try:
+            try:
+                y = $VALUE
+                y = z ** 3
+            except:
+                $EXC = False
+                if (not $EXIT($EXCINFO)):
+                    raise
+        finally:
+            if $EXC:
+                $EXIT(None, None, None)
+
+        """, t)
+                          
+
+if __name__ == "__main__":
+    import unittest
+    unittest.main()

Cython/Compiler/Tests/TestTreeFragment.py

 from Cython.TestUtils import CythonTest
 from Cython.Compiler.TreeFragment import *
 from Cython.Compiler.Nodes import *
+import Cython.Compiler.Naming as Naming
 
 class TestTreeFragments(CythonTest):
+    
     def test_basic(self):
         F = self.fragment(u"x = 4")
         T = F.copy()
@@ -12,15 +14,15 @@ class TestTreeFragments(CythonTest):
         F = self.fragment(u"if True: x = 4")
         T1 = F.root
         T2 = F.copy()
-        self.assertEqual("x", T2.body.if_clauses[0].body.lhs.name)
-        T2.body.if_clauses[0].body.lhs.name = "other"
-        self.assertEqual("x", T1.body.if_clauses[0].body.lhs.name)
+        self.assertEqual("x", T2.stats[0].if_clauses[0].body.lhs.name)
+        T2.stats[0].if_clauses[0].body.lhs.name = "other"
+        self.assertEqual("x", T1.stats[0].if_clauses[0].body.lhs.name)
 
     def test_substitutions_are_copied(self):
         T = self.fragment(u"y + y").substitute({"y": NameNode(pos=None, name="x")})
-        self.assertEqual("x", T.body.expr.operand1.name)
-        self.assertEqual("x", T.body.expr.operand2.name)
-        self.assert_(T.body.expr.operand1 is not T.body.expr.operand2)
+        self.assertEqual("x", T.stats[0].expr.operand1.name)
+        self.assertEqual("x", T.stats[0].expr.operand2.name)
+        self.assert_(T.stats[0].expr.operand1 is not T.stats[0].expr.operand2)
 
     def test_substitution(self):
         F = self.fragment(u"x = 4")
@@ -32,7 +34,7 @@ class TestTreeFragments(CythonTest):
         F = self.fragment(u"PASS")
         pass_stat = PassStatNode(pos=None)
         T = F.substitute({"PASS" : pass_stat})
-        self.assert_(isinstance(T.body, PassStatNode), T.body)
+        self.assert_(isinstance(T.stats[0], PassStatNode), T)
 
     def test_pos_is_transferred(self):
         F = self.fragment(u"""
@@ -40,11 +42,22 @@ class TestTreeFragments(CythonTest):
         x = u * v ** w
         """)
         T = F.substitute({"v" : NameNode(pos=None, name="a")})
-        v = F.root.body.stats[1].rhs.operand2.operand1
-        a = T.body.stats[1].rhs.operand2.operand1
+        v = F.root.stats[1].rhs.operand2.operand1
+        a = T.stats[1].rhs.operand2.operand1
         self.assertEquals(v.pos, a.pos)
         
-        
+    def test_temps(self):
+        import Cython.Compiler.Visitor as v
+        v.tmpnamectr = 0
+        F = self.fragment(u"""
+            TMP
+            x = TMP
+        """)
+        T = F.substitute(temps=[u"TMP"])
+        s = T.stats
+        self.assert_(s[0].expr.name == Naming.temp_prefix +  u"1_TMP", s[0].expr.name)
+        self.assert_(s[1].rhs.name == Naming.temp_prefix + u"1_TMP")
+        self.assert_(s[0].expr.name !=  u"TMP")
 
 if __name__ == "__main__":
     import unittest

Cython/Compiler/TreeFragment.py

@@ -6,8 +6,8 @@ import re
 from cStringIO import StringIO
 from Scanning import PyrexScanner, StringSourceDescriptor
 from Symtab import BuiltinScope, ModuleScope
-from Visitor import VisitorTransform
-from Nodes import Node
+from Visitor import VisitorTransform, temp_name_handle
+from Nodes import Node, StatListNode
 from ExprNodes import NameNode
 import Parsing
 import Main
@@ -53,7 +53,7 @@ def parse_from_strings(name, code, pxds={}):
     buf = StringIO(code.encode(encoding))
 
     scanner = PyrexScanner(buf, code_source, source_encoding = encoding,
-                     type_names = scope.type_names, context = context)
+                     scope = scope, context = context)
     tree = Parsing.p_module(scanner, 0, module_name)
     return tree
 
@@ -92,27 +92,56 @@ class TemplateTransform(VisitorTransform):
        if its name is listed in the substitutions dictionary in the
        same way. It is the responsibility of the caller to make sure
        that the replacement nodes is a valid expression.
+
+    Also a list "temps" should be passed. Any names listed will
+    be transformed into anonymous, temporary names.
+   
+    Currently supported for tempnames is:
+    NameNode
+    (various function and class definition nodes etc. should be added to this)
     
     Each replacement node gets the position of the substituted node
     recursively applied to every member node.
     """
+
+    def __call__(self, node, substitutions, temps, pos):
+        self.substitutions = substitutions
+        tempdict = {}
+        for key in temps:
+            tempdict[key] = temp_name_handle(key)
+        self.temps = tempdict
+        self.pos = pos
+        return super(TemplateTransform, self).__call__(node)
+
+
     def visit_Node(self, node):
         if node is None:
-            return node
+            return None
         else:
             c = node.clone_node()
+            if self.pos is not None:
+                c.pos = self.pos
             self.visitchildren(c)
             return c
     
     def try_substitution(self, node, key):
         sub = self.substitutions.get(key)
-        if sub is None:
-            return self.visit_Node(node) # make copy as usual
+        if sub is not None:
+            pos = self.pos
+            if pos is None: pos = node.pos
+            return ApplyPositionAndCopy(pos)(sub)
         else:
-            return ApplyPositionAndCopy(node.pos)(sub)
+            return self.visit_Node(node) # make copy as usual
+            
     
     def visit_NameNode(self, node):
-        return self.try_substitution(node, node.name)
+        tempname = self.temps.get(node.name)
+        if tempname is not None:
+            # Replace name with temporary
+            node.name = tempname
+            return self.visit_Node(node)
+        else:
+            return self.try_substitution(node, node.name)
 
     def visit_ExprStatNode(self, node):
         # If an expression-as-statement consists of only a replaceable
@@ -122,10 +151,6 @@ class TemplateTransform(VisitorTransform):
         else:
             return self.visit_Node(node)
     
-    def __call__(self, node, substitutions):
-        self.substitutions = substitutions
-        return super(TemplateTransform, self).__call__(node)
-
 def copy_code_tree(node):
     return TreeCopier()(node)
 
@@ -133,12 +158,12 @@ INDENT_RE = re.compile(ur"^ *")
 def strip_common_indent(lines):
     "Strips empty lines and common indentation from the list of strings given in lines"
     lines = [x for x in lines if x.strip() != u""]
-    minindent = min(len(INDENT_RE.match(x).group(0)) for x in lines)
+    minindent = min([len(INDENT_RE.match(x).group(0)) for x in lines])
     lines = [x[minindent:] for x in lines]
     return lines
     
 class TreeFragment(object):
-    def __init__(self, code, name, pxds={}):
+    def __init__(self, code, name="(tree fragment)", pxds={}, temps=[], pipeline=[]):
         if isinstance(code, unicode):
             def fmt(x): return u"\n".join(strip_common_indent(x.split(u"\n"))) 
             
@@ -147,18 +172,28 @@ class TreeFragment(object):
             for key, value in pxds.iteritems():
                 fmt_pxds[key] = fmt(value)
                 
-            self.root = parse_from_strings(name, fmt_code, fmt_pxds)
+            t = parse_from_strings(name, fmt_code, fmt_pxds)
+            mod = t
+            t = t.body # Make sure a StatListNode is at the top
+            if not isinstance(t, StatListNode):
+                t = StatListNode(pos=mod.pos, stats=[t])
+            for transform in pipeline:
+                t = transform(t)
+            self.root = t
         elif isinstance(code, Node):
             if pxds != {}: raise NotImplementedError()
             self.root = code
         else:
             raise ValueError("Unrecognized code format (accepts unicode and Node)")
+        self.temps = temps
 
     def copy(self):
         return copy_code_tree(self.root)
 
-    def substitute(self, nodes={}):
-        return TemplateTransform()(self.root, substitutions = nodes)
+    def substitute(self, nodes={}, temps=[], pos = None):
+        return TemplateTransform()(self.root,
+                                   substitutions = nodes,
+                                   temps = self.temps + temps, pos = pos)
 
 
 

Cython/Compiler/Visitor.py

 #
 #   Tree visitor and transform framework
 #
+import inspect
 import Nodes
 import ExprNodes
-import inspect
+import Naming
+from Cython.Utils import EncodedString
 
 class BasicVisitor(object):
     """A generic visitor base class which can be used for visiting any kind of object."""
@@ -129,7 +131,6 @@ class VisitorTransform(TreeVisitor):
     was not, an exception will be raised. (Typically you want to ensure that you
     are within a StatListNode or similar before doing this.)
     """
-    
     def visitchildren(self, parent, attrs=None):
         result = super(VisitorTransform, self).visitchildren(parent, attrs)
         for attr, newnode in result.iteritems():
@@ -150,6 +151,19 @@ class VisitorTransform(TreeVisitor):
     def __call__(self, root):
         return self.visit(root)
 
+class CythonTransform(VisitorTransform):
+    """
+    Certain common conventions and utilitues for Cython transforms.
+    """
+    def __init__(self, context):
+        super(CythonTransform, self).__init__()
+        self.context = context
+
+    def visit_Node(self, node):
+        self.visitchildren(node)
+        return node
+
+
 # Utils
 def ensure_statlist(node):
     if not isinstance(node, Nodes.StatListNode):
@@ -166,12 +180,29 @@ def replace_node(ptr, value):
     else:
         getattr(parent, attrname)[listidx] = value
 
+tmpnamectr = 0
+def temp_name_handle(description=None):
+    global tmpnamectr
+    tmpnamectr += 1
+    if description is not None:
+        name = u"%d_%s" % (tmpnamectr, description)
+    else:
+        name = u"%d" % tmpnamectr
+    return EncodedString(Naming.temp_prefix + name)
+
+def get_temp_name_handle_desc(handle):
+    if not handle.startswith(u"__cyt_"):
+        return None
+    else:
+        idx = handle.find(u"_", 6)
+        return handle[idx+1:]
+    
 class PrintTree(TreeVisitor):
     """Prints a representation of the tree to standard output.
     Subclass and override repr_of to provide more information
     about nodes. """
     def __init__(self):
-        Transform.__init__(self)
+        TreeVisitor.__init__(self)
         self._indent = ""
 
     def indent(self):
@@ -181,6 +212,8 @@ class PrintTree(TreeVisitor):
 
     def __call__(self, tree, phase=None):
         print("Parse tree dump at phase '%s'" % phase)
+        self.visit(tree)
+        return tree
 
     # Don't do anything about process_list, the defaults gives
     # nice-looking name[idx] nodes which will visually appear

Cython/TestUtils.py

@@ -4,8 +4,49 @@ import unittest
 from Cython.Compiler.ModuleNode import ModuleNode
 import Cython.Compiler.Main as Main
 from Cython.Compiler.TreeFragment import TreeFragment, strip_common_indent
+from Cython.Compiler.Visitor import TreeVisitor
+
+class NodeTypeWriter(TreeVisitor):
+    def __init__(self):
+        super(NodeTypeWriter, self).__init__()
+        self._indents = 0
+        self.result = []
+    def visit_Node(self, node):
+        if len(self.access_path) == 0:
+            name = u"(root)"
+        else:
+            tip = self.access_path[-1]
+            if tip[2] is not None:
+                name = u"%s[%d]" % tip[1:3]
+            else:
+                name = tip[1]
+            
+        self.result.append(u"  " * self._indents +
+                           u"%s: %s" % (name, node.__class__.__name__))
+        self._indents += 1
+        self.visitchildren(node)
+        self._indents -= 1
+
+def treetypes(root):
+    """Returns a string representing the tree by class names.
+    There's a leading and trailing whitespace so that it can be
+    compared by simple string comparison while still making test
+    cases look ok."""
+    w = NodeTypeWriter()
+    w.visit(root)
+    return u"\n".join([u""] + w.result + [u""])
 
 class CythonTest(unittest.TestCase):
+
+    def assertLines(self, expected, result):
+        "Checks that the given strings or lists of strings are equal line by line"
+        if not isinstance(expected, list): expected = expected.split(u"\n")
+        if not isinstance(result, list): result = result.split(u"\n")
+        for idx, (expected_line, result_line) in enumerate(zip(expected, result)):
+            self.assertEqual(expected_line, result_line, "Line %d:\nExp: %s\nGot: %s" % (idx, expected_line, result_line))
+        self.assertEqual(len(expected), len(result),
+            "Unmatched lines. Got:\n%s\nExpected:\n%s" % ("\n".join(expected), u"\n".join(result)))
+
     def assertCode(self, expected, result_tree):
         writer = CodeWriter()
         writer.write(result_tree)
@@ -18,13 +59,35 @@ class CythonTest(unittest.TestCase):
         self.assertEqual(len(result_lines), len(expected_lines),
             "Unmatched lines. Got:\n%s\nExpected:\n%s" % ("\n".join(result_lines), expected))
 
-    def fragment(self, code, pxds={}):
+    def fragment(self, code, pxds={}, pipeline=[]):
         "Simply create a tree fragment using the name of the test-case in parse errors."
         name = self.id()
         if name.startswith("__main__."): name = name[len("__main__."):]
         name = name.replace(".", "_")
-        return TreeFragment(code, name, pxds)
-        
+        return TreeFragment(code, name, pxds, pipeline=pipeline)
+
+    def treetypes(self, root):
+        return treetypes(root)
+
+    def should_fail(self, func, exc_type=Exception):
+        """Calls "func" and fails if it doesn't raise the right exception
+        (any exception by default). Also returns the exception in question.
+        """
+        try:
+            func()
+            self.fail("Expected an exception of type %r" % exc_type)
+        except exc_type, e:
+            self.assert_(isinstance(e, exc_type))
+            return e
+
+    def should_not_fail(self, func):
+        """Calls func and succeeds if and only if no exception is raised
+        (i.e. converts exception raising into a failed testcase). Returns
+        the return value of func."""
+        try:
+            return func()
+        except:
+            self.fail()
 
 class TransformTest(CythonTest):
     """
@@ -37,8 +100,8 @@ class TransformTest(CythonTest):
     To create a test case:
      - Call run_pipeline. The pipeline should at least contain the transform you
        are testing; pyx should be either a string (passed to the parser to
-       create a post-parse tree) or a ModuleNode representing input to pipeline.
-       The result will be a transformed result (usually a ModuleNode).
+       create a post-parse tree) or a node representing input to pipeline.
+       The result will be a transformed result.
        
      - Check that the tree is correct. If wanted, assertCode can be used, which
        takes a code string as expected, and a ModuleNode in result_tree
@@ -53,7 +116,6 @@ class TransformTest(CythonTest):
     
     def run_pipeline(self, pipeline, pyx, pxds={}):
         tree = self.fragment(pyx, pxds).root
-        assert isinstance(tree, ModuleNode)
         # Run pipeline
         for T in pipeline:
             tree = T(tree)

Cython/Tests/TestCodeWriter.py

@@ -72,6 +72,9 @@ class TestCodeWriter(CythonTest):
 
     def test_inplace_assignment(self):
         self.t(u"x += 43")
+
+    def test_attribute(self):
+        self.t(u"a.x")
     
 if __name__ == "__main__":
     import unittest

Demos/Setup.py

@@ -7,6 +7,7 @@ from Cython.Distutils import build_ext
 ext_modules=[ 
     Extension("primes",       ["primes.pyx"]),
     Extension("spam",         ["spam.pyx"]),
+#    Extension("optargs",      ["optargs.pyx"], language = "c++"),
 ]
 
 for file in glob.glob("*.pyx"):

Includes/__cython__.pxd

+cdef extern from "Python.h":
+    ctypedef struct PyObject
+
+
+
+    ctypedef struct Py_buffer:
+        void *buf
+        Py_ssize_t len
+        int readonly
+        char *format
+        int ndim
+        Py_ssize_t *shape
+        Py_ssize_t *strides
+        Py_ssize_t *suboffsets
+        Py_ssize_t itemsize
+        void *internal
+
+    
+    int PyObject_GetBuffer(PyObject* obj, Py_buffer* view, int flags) except -1
+    void PyObject_ReleaseBuffer(PyObject* obj, Py_buffer* view)
+
+    void PyErr_Format(int, char*, ...)
+
+    enum:
+        PyExc_TypeError
+
+#                  int PyObject_GetBuffer(PyObject *obj, Py_buffer *view,
+#                       int flags)

Includes/numpy.pxd

+cdef extern from "Python.h":
+    ctypedef int Py_intptr_t
+    
+cdef extern from "numpy/arrayobject.h":
+    ctypedef class numpy.ndarray [object PyArrayObject]:
+        cdef char *data
+        cdef int nd
+        cdef Py_intptr_t *dimensions
+        cdef Py_intptr_t *strides
+        cdef object base
+        # descr not implemented yet here...
+        cdef int flags
+        cdef int itemsize
+        cdef object weakreflist
+
+    ctypedef unsigned int npy_uint8
+    ctypedef unsigned int npy_uint16
+    ctypedef unsigned int npy_uint32
+    ctypedef unsigned int npy_uint64
+    ctypedef unsigned int npy_uint96
+    ctypedef unsigned int npy_uint128
+    ctypedef signed int   npy_int64
+
+    ctypedef float        npy_float32
+    ctypedef float        npy_float64
+    ctypedef float        npy_float80
+    ctypedef float        npy_float96
+    ctypedef float        npy_float128
+
+ctypedef npy_int64 Tint64

runtests.py

@@ -9,8 +9,8 @@ from distutils.core import Extension
 from distutils.command.build_ext import build_ext
 distutils_distro = Distribution()
 
-TEST_DIRS = ['compile', 'errors', 'run']
-TEST_RUN_DIRS = ['run']
+TEST_DIRS = ['compile', 'errors', 'run', 'pyregr']
+TEST_RUN_DIRS = ['run', 'pyregr']
 
 INCLUDE_DIRS = [ d for d in os.getenv('INCLUDE', '').split(os.pathsep) if d ]
 CFLAGS = os.getenv('CFLAGS', '').split()
@@ -78,15 +78,21 @@ class TestBuilder(object):
         filenames = os.listdir(path)
         filenames.sort()
         for filename in filenames:
-            if not filename.endswith(".pyx"):
+            if not (filename.endswith(".pyx") or filename.endswith(".py")):
                 continue
-            module = filename[:-4]
+            if context == 'pyregr' and not filename.startswith('test_'):
+                continue
+            module = os.path.splitext(filename)[0]
             fqmodule = "%s.%s" % (context, module)
             if not [ 1 for match in self.selectors
                      if match(fqmodule) ]:
                 continue
             if context in TEST_RUN_DIRS:
-                test = CythonRunTestCase(
+                if module.startswith("test_"):
+                    build_test = CythonUnitTestCase
+                else:
+                    build_test = CythonRunTestCase
+                test = build_test(
                     path, workdir, module,
                     annotate=self.annotate,
                     cleanup_workdir=self.cleanup_workdir)
@@ -133,11 +139,21 @@ class CythonCompileTestCase(unittest.TestCase):
             os.makedirs(self.workdir)
 
     def runTest(self):
+        self.runCompileTest()
+
+    def runCompileTest(self):
         self.compile(self.directory, self.module, self.workdir,
                      self.directory, self.expect_errors, self.annotate)
 
+    def find_module_source_file(self, source_file):
+        if not os.path.exists(source_file):
+            source_file = source_file[:-1]
+        return source_file
+
     def split_source_and_output(self, directory, module, workdir):
-        source_and_output = open(os.path.join(directory, module + '.pyx'), 'rU')
+        source_file = os.path.join(directory, module) + '.pyx'
+        source_and_output = open(
+            self.find_module_source_file(source_file), 'rU')
         out = open(os.path.join(workdir, module + '.pyx'), 'w')
         for line in source_and_output:
             last_line = line
@@ -157,7 +173,8 @@ class CythonCompileTestCase(unittest.TestCase):
         include_dirs = INCLUDE_DIRS[:]
         if incdir:
             include_dirs.append(incdir)
-        source = os.path.join(directory, module + '.pyx')
+        source = self.find_module_source_file(
+            os.path.join(directory, module + '.pyx'))
         target = os.path.join(targetdir, module + '.c')
         options = CompilationOptions(
             pyrex_default_options,
@@ -228,7 +245,7 @@ class CythonRunTestCase(CythonCompileTestCase):
             result = self.defaultTestResult()
         result.startTest(self)
         try:
-            self.runTest()
+            self.runCompileTest()
             doctest.DocTestSuite(self.module).run(result)
         except Exception:
             result.addError(self, sys.exc_info())
@@ -238,6 +255,48 @@ class CythonRunTestCase(CythonCompileTestCase):
         except Exception:
             pass
 
+class CythonUnitTestCase(CythonCompileTestCase):
+    def shortDescription(self):
+        return "compiling and running unit tests in " + self.module
+
+    def run(self, result=None):
+        if result is None:
+            result = self.defaultTestResult()
+        result.startTest(self)
+        try:
+            self.runCompileTest()
+            unittest.defaultTestLoader.loadTestsFromName(self.module).run(result)
+        except Exception:
+            result.addError(self, sys.exc_info())
+            result.stopTest(self)
+        try:
+            self.tearDown()
+        except Exception:
+            pass
+
+def collect_unittests(path, suite, selectors):
+    def file_matches(filename):
+        return filename.startswith("Test") and filename.endswith(".py")
+
+    def package_matches(dirname):
+        return dirname == "Tests"
+
+    loader = unittest.TestLoader()
+
+    for dirpath, dirnames, filenames in os.walk(path):
+        parentname = os.path.split(dirpath)[-1]
+        if package_matches(parentname):
+            for f in filenames:
+                if file_matches(f):
+                    filepath = os.path.join(dirpath, f)[:-len(".py")]
+                    modulename = filepath[len(path)+1:].replace(os.path.sep, '.')
+                    if not [ 1 for match in selectors if match(modulename) ]:
+                        continue
+                    module = __import__(modulename)
+                    for x in modulename.split('.')[1:]:
+                        module = getattr(module, x)
+                    suite.addTests([loader.loadTestsFromModule(module)])
+
 if __name__ == '__main__':
     from optparse import OptionParser
     parser = OptionParser()
@@ -247,6 +306,12 @@ if __name__ == '__main__':
     parser.add_option("--no-cython", dest="with_cython",
                       action="store_false", default=True,
                       help="do not run the Cython compiler, only the C compiler")
+    parser.add_option("--no-unit", dest="unittests",
+                      action="store_false", default=True,
+                      help="do not run the unit tests")
+    parser.add_option("--no-file", dest="filetests",
+                      action="store_false", default=True,
+                      help="do not run the file based tests")
     parser.add_option("-C", "--coverage", dest="coverage",
                       action="store_true", default=False,
                       help="collect source coverage data for the Compiler")
@@ -296,9 +361,15 @@ if __name__ == '__main__':
     if not selectors:
         selectors = [ lambda x:True ]
 
-    tests = TestBuilder(ROOTDIR, WORKDIR, selectors,
-                        options.annotate_source, options.cleanup_workdir)
-    test_suite = tests.build_suite()
+    test_suite = unittest.TestSuite()
+
+    if options.unittests:
+        collect_unittests(os.getcwd(), test_suite, selectors)
+
+    if options.filetests:
+        filetests = TestBuilder(ROOTDIR, WORKDIR, selectors,
+                                options.annotate_source, options.cleanup_workdir)
+        test_suite.addTests([filetests.build_suite()])
 
     unittest.TextTestRunner(verbosity=options.verbosity).run(test_suite)
 

tests/broken/assert2.pyx

+def f(a, b):
+    assert a, a+b

tests/broken/b_extimpinherit.pyx

+cdef class Parrot:
+
+    cdef describe(self):
+        print "This is a parrot."
+    
+    cdef action(self):
+        print "Polly wants a cracker!"

tests/broken/big_t.pyx

+cdef extern from "foo.h":
+    ctypedef long long big_t
+    cdef void spam(big_t b)
+
+spam(grail)

tests/broken/builtinconst.pyx

+cdef int f() except -1:
+    cdef type t
+    cdef object x
+    t = buffer
+    t = enumerate
+    t = file
+    t = float
+    t = int
+    t = long
+    t = open
+    t = property
+    t = str
+    t = tuple
+    t = xrange
+    x = True
+    x = False
+    x = Ellipsis
+    x = Exception
+    x = StopIteration
+    x = StandardError
+    x = ArithmeticError
+    x = LookupError
+    x = AssertionError
+    x = AssertionError
+    x = EOFError
+    x = FloatingPointError
+    x = EnvironmentError
+    x = IOError
+    x = OSError
+    x = ImportError
+    x = IndexError
+    x = KeyError
+    x = KeyboardInterrupt
+    x = MemoryError
+    x = NameError
+    x = OverflowError
+    x = RuntimeError
+    x = NotImplementedError
+    x = SyntaxError
+    x = IndentationError
+    x = TabError
+    x = ReferenceError
+    x = SystemError
+    x = SystemExit
+    x = TypeError
+    x = UnboundLocalError
+    x = UnicodeError
+    x = UnicodeEncodeError
+    x = UnicodeDecodeError
+    x = UnicodeTranslateError
+    x = ValueError
+    x = ZeroDivisionError
+    x = MemoryErrorInst
+    x = Warning
+    x = UserWarning
+    x = DeprecationWarning
+    x = PendingDeprecationWarning
+    x = SyntaxWarning
+    #x = OverflowWarning # Does not seem to exist in 2.5
+    x = RuntimeWarning
+    x = FutureWarning
+    typecheck(x, Exception)
+    try:
+        pass
+    except ValueError:
+        pass

tests/broken/builtindict.pyx

+cdef int f() except -1:
+    cdef dict d
+    cdef object x, z
+    cdef int i
+    z = dict
+    d = dict(x)
+    d = dict(*x)
+    d.clear()
+    z = d.copy()
+    z = d.items()
+    z = d.keys()
+    z = d.values()
+    d.merge(x, i)
+    d.update(x)
+    d.merge_pairs(x, i)

tests/broken/builtinlist.pyx

+cdef int f() except -1:
+    cdef list l
+    cdef object x, y, z
+    z = list
+    l = list(x)
+    l = list(*y)
+    z = l.insert
+    l.insert(17, 42)
+    l.append(88)
+    l.sort()
+    l.reverse()
+    z = l.as_tuple()

tests/broken/builtinslice.pyx

+cdef int f() except -1:
+    cdef slice s
+    cdef object z
+    cdef int i
+    z = slice
+    s = slice(1, 2, 3)
+    z = slice.indices()
+    i = s.start
+    i = s.stop
+    i = s.step

tests/broken/builtintype.pyx

+cdef int f() except -1:
+    cdef type t1, t2
+    cdef object x
+    cdef int b
+    b = typecheck(x, t1)
+    b = issubtype(t1, t2)

tests/broken/cascadedass.pyx

+cdef void foo():
+    cdef int i, j, k
+    i = j = k
+    a = b = c
+    i = j = c
+    a = b = k
+    (a, b), c = (d, e), f = (x, y), z
+#	a, b = p, q = x, y
\ No newline at end of file

tests/broken/cdefemptysue.pyx

+cdef extern from "cdefemptysue.h":
+
+    cdef struct spam:
+        pass
+    
+    ctypedef union eggs:
+        pass
+    
+    cdef enum ham:
+        pass
+    
+cdef extern spam s
+cdef extern eggs e
+cdef extern ham h

tests/broken/cdefexternblock.pyx

+cdef extern from "cheese.h":
+
+    ctypedef int camembert
+    
+    struct roquefort:
+        int x
+        
+    char *swiss
+    
+    void cheddar()
+
+    class external.runny [object runny_obj]:
+        cdef int a
+        def __init__(self):
+            pass
+
+cdef runny r
+r = x
+r.a = 42

tests/broken/cdefexternempty.pyx

+cdef extern from "cheese.h":
+    pass
+

tests/broken/cexportfunc.pyx

+cdef int f():
+    pass
+
+cdef char *g(int k, float z):
+    pass

tests/broken/cimport.pyx

+cimport spam
+cimport pkg.eggs
+
+cdef spam.Spam yummy
+cdef pkg.eggs.Eggs fried
+
+spam.eat(yummy)
+spam.tons = 3.14
+ova = pkg.eggs
+fried = pkg.eggs.Eggs()

tests/broken/cimportfrom.pyx

+from spam cimport Spam
+from pkg.eggs cimport Eggs as ova
+
+cdef extern Spam yummy
+cdef ova fried
+
+fried = None

tests/broken/cimportfrompkgdir.pyx

+from package.inpackage cimport Spam
+
+cdef Spam s2

tests/broken/cimportfunc.pyx

+from cexportfunc cimport f, g

tests/broken/ctypedefextern.pyx

+cdef extern from "ctypedefextern.h":
+
+    ctypedef int some_int
+    ctypedef some_int *some_ptr
+
+cdef void spam():
+    cdef some_int i
+    cdef some_ptr p
+    p[0] = i

tests/broken/e_exestmtinexttype.pyx

+cdef class Spam:
+    answer = 42

tests/broken/e_nogil.pyx

+cdef object f(object x) nogil:
+    pass
+
+cdef void g(int x) nogil:
+    cdef object z
+    z = None
+
+cdef void h(int x) nogil:
+    p()
+
+cdef object p() nogil:
+    pass
+
+cdef void r() nogil:
+    q()
+    
+cdef void (*fp)()
+cdef void (*fq)() nogil
+cdef extern void u()
+
+cdef object m():
+    global fp, fq
+    cdef object x, y, obj
+    cdef int i, j, k
+    global fred
+    q()
+    with nogil:
+        r()
+        q()
+        i = 42
+        obj = None
+        17L
+        7j
+        asdf
+        `"Hello"`
+        import fred
+        from fred import obj
+        for x in obj:
+            pass
+        obj[i]
+        obj[i:j]
+        obj[i:j:k]
+        obj.fred
+        (x, y)
+        [x, y]
+        {x: y}
+        obj and x
+        t(obj)
+        f(42)
+        x + obj
+        -obj
+        x = y = obj
+        x, y = y, x
+        obj[i] = x
+        obj.fred = x
+        print obj
+        del fred
+        return obj
+        raise obj
+        if obj:
+            pass
+        while obj:
+            pass
+        for x <= obj <= y:
+            pass
+        try:
+            pass
+        except:
+            pass
+        try:
+            pass
+        finally:
+            pass
+        fq = u
+        fq = fp
+
+cdef void q():
+    pass
+
+cdef class C:
+    pass
+
+cdef void t(C c) nogil:
+    pass

tests/broken/externfunc.pyx

+cdef extern from "foo.h":
+
+    int fred()

tests/broken/externsue.pyx

+cdef extern from "externsue.h":
+
+    enum Eggs:
+        runny, firm, hard
+    
+    struct Spam:
+        int i
+    
+    union Soviet:
+        char c
+
+cdef extern Eggs e
+cdef extern Spam s
+cdef extern Soviet u
+
+cdef void tomato():
+    global e
+    e = runny
+    e = firm
+    e = hard
+

tests/broken/fwddeclcclass.pyx

+cdef class Widget:
+    pass
+
+cdef class Container:
+    pass
+
+cdef Widget w
+cdef Container c
+w.parent = c

tests/broken/getattr.pyx

+cdef class Spam:
+    cdef public object eggs
+    
+    def __getattr__(self, name):
+        print "Spam getattr:", name

tests/broken/getattr3ref.pyx

+cdef int f() except -1:
+    g = getattr3

tests/broken/i_public.pyx

+cdef public int grail
+
+cdef public spam(int servings):
+    pass
+
+cdef public class sandwich [object sandwich, type sandwich_Type]:
+    cdef int tomato
+    cdef float lettuce

tests/broken/includepublic.pyx

+include "i_public.pxi"
+

tests/broken/inplace_lhs.pyx

+cdef struct S:
+    int q
+
+cdef int f() except -1:
+    cdef int i, j, k
+    cdef float x, y, z
+    cdef object a, b, c, d, e
+    cdef int m[3]
+    cdef S s
+    global g
+    i += j + k
+    x += y + z
+    x += i
+    a += b + c
+    g += a
+    m[i] += j
+    a[i] += b + c
+    a[b + c] += d
+    (a + b)[c] += d
+    a[i : j] += b
+    (a + b)[i : j] += c
+    a.b += c + d
+    (a + b).c += d
+    s.q += i

tests/broken/inplace_ops.pyx

+cdef int f() except -1:
+    cdef object a, b
+    cdef char *p
+    a += b
+    a -= b
+    a *= b
+    a /= b
+    a %= b
+    a **= b
+    a <<= b
+    a >>= b
+    a &= b
+    a ^= b
+    a |= b
+    p += 42
+    p -= 42
+    p += a

tests/broken/intindex.pyx

+cdef int f() except -1:
+    cdef object x, y, z
+    cdef int i
+    cdef unsigned int ui
+    z = x[y]
+    z = x[i]
+    x[y] = z
+    x[i] = z
+    z = x[ui]
+    x[ui] = z

tests/broken/l_capi.pyx

+cdef api float f(float x):
+    return 0.5 * x * x

tests/broken/l_cfuncexport.pyx

+cdef int f(int x):
+    return x * x
+
+cdef int g(int x):
+    return 5 * x

tests/broken/naanou_1.pyx

+def f(a, *p, **n):
+    pass

tests/broken/newstyleintforloop.pyx

+cdef int f() except -1:
+    cdef int i, x, y
+    for x < i < y:
+        pass

tests/broken/oldstyleintforloop.pyx

+cdef int f() except -1:
+    cdef int i, x, y
+    for i from x < i < y:
+        pass

tests/broken/pkg.cimport.pyx

+cimport spam, eggs
+
+cdef extern spam.Spam yummy
+cdef eggs.Eggs fried
+
+fried = None

tests/broken/pkg.cimportfrom.pyx

+from spam cimport Spam
+from eggs cimport Eggs
+
+cdef extern Spam yummy
+cdef Eggs fried
+
+fried = None

tests/broken/r_builtinlist.pyx

+def f():
+    cdef list l
+    l = list()
+    l.append("second")
+    l.insert(0, "first")
+    return l

tests/broken/r_capi.pyx

+cdef extern from "l_capi_api.h":
+    float f(float)
+    int import_l_capi() except -1
+
+def test():
+    print f(3.1415)
+
+import_l_capi()

tests/broken/r_cfuncimport.pyx

+cimport l_cfuncexport
+from l_cfuncexport cimport g
+
+print l_cfuncexport.f(42)
+print g(42)

tests/broken/r_classdoc.pyx

+class Spam:
+    """Spam, glorious spam!"""
+

tests/broken/r_classmodname.pyx

+class Spam:
+    pass

tests/broken/r_excval.pyx

+cdef int tomato() except -1:
+    print "Entering tomato"
+    raise Exception("Eject! Eject! Eject!")
+    print "Leaving tomato"
+    
+cdef void sandwich():
+    print "Entering sandwich"
+    tomato()
+    print "Leaving sandwich"
+
+def snack():
+    print "Entering snack"
+    tomato()
+    print "Leaving snack"
+
+def lunch():
+    print "Entering lunch"
+    sandwich()
+    print "Leaving lunch"

tests/broken/r_extcmethod.pyx

+cdef class SpamDish:
+    cdef int spam
+
+    cdef void describe(self):
+        print "This dish contains", self.spam, "tons of spam."
+
+
+cdef class FancySpamDish(SpamDish):
+    cdef int lettuce
+    
+    cdef void describe(self):
+        print "This dish contains", self.spam, "tons of spam",
+        print "and", self.lettuce, "milligrams of lettuce."
+
+
+cdef void describe_dish(SpamDish d):
+    d.describe()
+
+def test():
+    cdef SpamDish s
+    cdef FancySpamDish ss
+    s = SpamDish()
+    s.spam = 42
+    ss = FancySpamDish()
+    ss.spam = 88
+    ss.lettuce = 5
+    describe_dish(s)
+    describe_dish(ss)

tests/broken/r_extimpinherit.pyx

+from b_extimpinherit cimport Parrot
+
+
+cdef class Norwegian(Parrot):
+
+    cdef action(self):
+        print "This parrot is resting."
+    
+    cdef plumage(self):
+        print "Lovely plumage!"
+
+
+def main():
+    cdef Parrot p
+    cdef Norwegian n
+    p = Parrot()
+    n = Norwegian()
+    print "Parrot:"
+    p.describe()
+    p.action()
+    print "Norwegian:"
+    n.describe()
+    n.action()
+    n.plumage()

tests/broken/r_extinherit.pyx

+cdef class Parrot:
+
+    cdef object plumage
+    
+    def __init__(self):
+        self.plumage = "yellow"
+    
+    def describe(self):
+        print "This bird has lovely", self.plumage, "plumage."
+
+
+cdef class Norwegian(Parrot):
+
+    def __init__(self):
+        self.plumage = "blue"
+

tests/broken/r_extmember.pyx

+cdef class Spam:
+    cdef public int tons
+    cdef readonly float tastiness
+    cdef int temperature
+    
+    def __init__(self, tons, tastiness, temperature):
+        self.tons = tons
+        self.tastiness = tastiness
+        self.temperature = temperature
+    
+    def get_temperature(self):
+        return self.temperature

tests/broken/r_extnumeric2.pyx

+cdef extern from "numeric.h":
+    
+    struct PyArray_Descr:
+        int type_num, elsize
+        char type
+    
+    ctypedef class Numeric.ArrayType [object PyArrayObject]:
+        cdef char *data
+        cdef int nd
+        cdef int *dimensions, *strides
+        cdef object base
+        cdef PyArray_Descr *descr
+        cdef int flags
+    
+def ogle(ArrayType a):
+    print "No. of dimensions:", a.nd
+    print "  Dim Value"
+    for i in range(a.nd):
+        print "%5d %5d" % (i, a.dimensions[i])
+    print "flags:", a.flags
+    print "Type no.", a.descr.type_num
+    print "Element size:", a.descr.elsize

tests/broken/r_extproperty.pyx

+cdef class CheeseShop:
+    cdef object cheeses
+    
+    def __cinit__(self):
+        self.cheeses = []
+
+    property cheese:
+    
+        "A senseless waste of a property."
+        
+        def __get__(self):
+            return "We don't have: %s" % self.cheeses
+
+        def __set__(self, value):
+            self.cheeses.append(value)
+        
+        def __del__(self):
+            del self.cheeses[:]

tests/broken/r_extweakref.pyx

+cdef class Animal:
+    cdef object __weakref__
+    cdef public object name

tests/broken/r_getattr3.pyx

+def test(obj, attr, dflt):
+    return getattr3(obj, attr, dflt)

tests/broken/r_import.pyx

+import spam
+print "Imported spam"
+print dir(spam)
+
+import sys
+print "Imported sys"
+print sys
+

tests/broken/r_inhcmethcall.pyx

+cdef class Parrot:
+
+  cdef void describe(self):
+    print "This parrot is resting."
+
+
+cdef class Norwegian(Parrot):
+
+  cdef void describe(self):
+    Parrot.describe(self)
+    print "Lovely plumage!"
+
+
+cdef Parrot p1, p2
+p1 = Parrot()
+p2 = Norwegian()
+p1.describe()
+p2.describe()

tests/broken/r_kwonlyargs.pyx

+def pd(d):
+    l = []
+    i = d.items()
+    i.sort()
+    for kv in i:
+        l.append("%r: %r" % kv)
+    return "{%s}" % ", ".join(l)
+
+def c(a, b, c):
+    print "a =", a, "b =", b, "c =", c
+
+def d(a, b, *, c = 88):
+    print "a =", a, "b =", b, "c =", c
+
+def e(a, b, c = 88, **kwds):
+    print "a =", a, "b =", b, "c =", c, "kwds =", pd(kwds)
+
+def f(a, b, *, c, d = 42):
+    print "a =", a, "b =", b, "c =", c, "d =", d
+
+def g(a, b, *, c, d = 42, e = 17, f, **kwds):
+    print "a =", a, "b =", b, "c =", c, "d =", d, "e =", e, "f =", f, "kwds =", pd(kwds)
+
+def h(a, b, *args, c, d = 42, e = 17, f, **kwds):
+    print "a =", a, "b =", b, "args =", args, "c =", c, "d =", d, "e =", e, "f =", f, "kwds =", pd(kwds)

tests/broken/r_newstyleclass.pyx

+class Inquisition(object):
+    """Something that nobody expects."""
+
+    def __repr__(self):
+        return "Surprise!"

tests/broken/r_simpcall.pyx

+def f():
+    print "Spam!"
+
+f()

tests/broken/r_tbfilename.pyx

+def foo():
+    raise Exception

tests/broken/r_traceback.pyx

+cdef int spam() except -1:
+    raise Exception("Spam error")
+
+cdef int grail() except -1:
+    spam()
+
+def tomato():
+    grail()
+

tests/broken/r_unpack.pyx

+seq = [1, [2, 3]]
+
+def f():
+    a, (b, c) = [1, [2, 3]]
+    print a
+    print b
+    print c
+
+def g():
+    a, b, c = seq
+
+def h():
+    a, = seq

tests/broken/retconvert.pyx

+def f():
+    return 42
+
+cdef int g():
+    cdef object x
+    return x

tests/broken/specialfloatvals.pyx

+DEF nan = float('nan')
+DEF inf = float('inf')
+DEF minf = -float('inf')
+
+cdef int f() except -1:
+    cdef float x, y, z
+    x = nan
+    y = inf
+    z = minf

tests/broken/test_include_options.pyx

+import sys
+from Pyrex.Compiler.Main import main
+
+sys.argv[1:] = "-I spam -Ieggs --include-dir ham".split()
+main(command_line = 1)

tests/broken/tslots.pyx

+cdef class Spam:
+    pass
+
+def probe():
+    pass

tests/broken/typeofexttype.pyx

+cdef class Spam:
+    pass
+
+cdef int f() except -1:
+    cdef type t
+    t = Spam

tests/broken/voidstarcast.pyx

+cdef class C:
+    cdef int i
+
+cdef int f() except -1:
+    cdef object x
+    cdef void *p
+    cdef int i
+    x = <object>p
+    p = <void *>x
+    x = (<object>p).foo
+    i = (<C>p).i
+    (<C>p).i = i

tests/errors/e_addop.pyx

@@ -2,6 +2,7 @@ def f():
 	cdef int int1, int3
 	cdef int *ptr1, *ptr2, *ptr3
 	ptr1 = ptr2 + ptr3 # error
+
 _ERRORS = u"""
-/Local/Projects/D/Pyrex/Source/Tests/Errors2/e_addop.pyx:4:13: Invalid operand types for '+' (int *; int *)
+4:13: Invalid operand types for '+' (int *; int *)
 """

tests/run/carrays.pyx

+__doc__ = """
+>>> test()
+2
+"""
+
+def test():
+    cdef int x[2][2]
+    x[0][0] = 1
+    x[0][1] = 2
+    x[1][0] = 3
+    x[1][1] = 4
+    return f(x)[1]
+
+cdef int* f(int x[2][2]):
+    return x[0]

tests/run/decorators.pyx

+__doc__ = u"""
+  >>> f(1,2)
+  4
+  >>> f.HERE
+  1
+
+  >>> g(1,2)
+  5
+  >>> g.HERE
+  5
+
+  >>> h(1,2)
+  6
+  >>> h.HERE
+  1
+"""
+
+class wrap:
+    def __init__(self, func):
+        self.func = func
+        self.HERE = 1
+    def __call__(self, *args, **kwargs):
+        return self.func(*args, **kwargs)
+
+def decorate(func):
+    try:
+        func.HERE += 1
+    except AttributeError:
+        func = wrap(func)
+    return func
+
+def decorate2(a,b):
+    return decorate
+
+@decorate
+def f(a,b):
+    return a+b+1
+
+@decorate
+@decorate
+@decorate
+@decorate
+@decorate
+def g(a,b):
+    return a+b+2
+
+@decorate2(1,2)
+def h(a,b):
+    return a+b+3

tests/run/flatin.pyx

+__doc__ = u"""
+>>> test_in('ABC')
+1
+>>> test_in('abc')
+2
+>>> test_in('X')
+3
+>>> test_in('XYZ')
+4
+>>> test_in('ABCXYZ')
+5
+>>> test_in('')
+5
+
+>>> test_not_in('abc')
+1
+>>> test_not_in('CDE')
+2
+>>> test_not_in('CDEF')
+3
+>>> test_not_in('BCD')
+4
+"""
+
+def test_in(s):
+    if s in ('ABC', 'BCD'):
+        return 1
+    elif s.upper() in ('ABC', 'BCD'):
+        return 2
+    elif len(s) in (1,2):
+        return 3
+    elif len(s) in (3,4):
+        return 4
+    else:
+        return 5
+
+def test_not_in(s):
+    if s not in ('ABC', 'BCD', 'CDE', 'CDEF'):
+        return 1
+    elif s.upper() not in ('ABC', 'BCD', 'CDEF'):
+        return 2
+    elif len(s) not in [3]:
+        return 3
+    elif len(s) not in [1,2]:
+        return 4
+    else:
+        return 5

tests/run/future_division.pyx

+from __future__ import division
+
+__doc__ = """
+>>> from future_division import doit
+>>> doit(1,2)
+(0.5, 0)
+>>> doit(4,3)
+(1.3333333333333333, 1)
+>>> doit(4,3.0)
+(1.3333333333333333, 1.0)
+>>> doit(4,2)
+(2.0, 2)
+"""
+
+def doit(x,y):
+    return x/y, x//y

tests/run/r_forloop.pyx

 __doc__ = u"""
-  >>> go()
+  >>> go_py()
+  Spam!
+  Spam!
+  Spam!
+  Spam!
+  Spam!
+
+  >>> go_c()
   Spam!
   Spam!
   Spam!
@@ -7,7 +14,11 @@ __doc__ = u"""
   Spam!
 """
 
-def go():
+def go_py():
     for i in range(5):
         print u"Spam!"
 
+def go_c():
+    cdef int i
+    for i in range(5):
+        print u"Spam!"