merge

CHANGES.rst

@@ -13,7 +13,9 @@ Features added
   the module's Python namespace.  Cpdef enums in pxd files export
   their values to their own module, iff it exists.
 
-* Calls to ``slice()`` are translated to a straight C-API call.
+* Allow @staticmethod decorator to declare static cdef methods.
+  This is especially useful for declaring "constructors" for
+  cdef classes that can take non-Python arguments.
 
 * Taking a ``char*`` from a temporary Python string object is safer
   in more cases and can be done inside of non-trivial expressions,
@@ -21,16 +23,6 @@ Features added
   is raised only when such a pointer is assigned to a variable and
   would thus exceed the lifetime of the string itself.
 
-* The "and"/"or" operators try to avoid unnecessary coercions of their
-  arguments.  They now evaluate the truth value of each argument
-  independently and only coerce the final result of the whole expression
-  to the target type (e.g. the type on the left side of an assignment).
-  This also avoids reference counting overhead for Python values during
-  evaluation and generally improves the code flow in the generated C code.
-
-* Cascaded assignments (a = b = ...) try to minimise the number of
-  type coercions.
-
 * Generators have new properties ``__name__`` and ``__qualname__``
   that provide the plain/qualified name of the generator function
   (following CPython 3.5).  See http://bugs.python.org/issue21205
@@ -46,9 +38,6 @@ Features added
 * HTML output of annotated code uses Pygments for code highlighting
   and generally received a major overhaul by Matthias Bussonier.
 
-* The Python expression "2 ** N" is optimised into bit shifting.
-  See http://bugs.python.org/issue21420
-
 * Simple support for declaring Python object types in Python signature
   annotations.  Currently requires setting the compiler directive
   ``annotation_typing=True``.
@@ -56,6 +45,29 @@ Features added
 * New directive ``use_switch`` (defaults to True) to optionally disable
   the optimization of chained if statement to C switch statements.
 
+* Defines dynamic_cast et al. in `libcpp.cast`.
+
+Optimizations
+-------------
+
+* The "and"/"or" operators try to avoid unnecessary coercions of their
+  arguments.  They now evaluate the truth value of each argument
+  independently and only coerce the final result of the whole expression
+  to the target type (e.g. the type on the left side of an assignment).
+  This also avoids reference counting overhead for Python values during
+  evaluation and generally improves the code flow in the generated C code.
+
+* Cascaded assignments (a = b = ...) try to minimise the number of
+  type coercions.
+
+* The Python expression "2 ** N" is optimised into bit shifting.
+  See http://bugs.python.org/issue21420
+
+* Cascaded assignments (a = b = ...) try to minimise the number of
+  type coercions.
+
+* Calls to ``slice()`` are translated to a straight C-API call.
+
 Bugs fixed
 ----------
 
@@ -91,6 +103,8 @@ Bugs fixed
 
 * Correctly handle ``from cython.submodule cimport name``.
 
+* Fix infinite recursion when using super with cpdef methods.
+
 Other changes
 -------------
 

Cython/Compiler/ExprNodes.py

@@ -4456,16 +4456,24 @@ class SimpleCallNode(CallNode):
         return func_type
 
     def analyse_c_function_call(self, env):
-        if self.function.type is error_type:
+        func_type = self.function.type
+        if func_type is error_type:
             self.type = error_type
             return
 
-        if self.self:
+        if func_type.is_cfunction and func_type.is_static_method:
+            if self.self and self.self.type.is_extension_type:
+                # To support this we'd need to pass self to determine whether
+                # it was overloaded in Python space (possibly via a Cython
+                # superclass turning a cdef method into a cpdef one).
+                error(self.pos, "Cannot call a static method on an instance variable.")
+            args = self.args
+        elif self.self:
             args = [self.self] + self.args
         else:
             args = self.args
 
-        if self.function.type.is_cpp_class:
+        if func_type.is_cpp_class:
             overloaded_entry = self.function.type.scope.lookup("operator()")
             if overloaded_entry is None:
                 self.type = PyrexTypes.error_type
@@ -4515,7 +4523,7 @@ class SimpleCallNode(CallNode):
             self.is_temp = 1
 
         # check 'self' argument
-        if entry and entry.is_cmethod and func_type.args:
+        if entry and entry.is_cmethod and func_type.args and not func_type.is_static_method:
             formal_arg = func_type.args[0]
             arg = args[0]
             if formal_arg.not_none:
@@ -5302,10 +5310,12 @@ class AttributeNode(ExprNode):
         # C method of an extension type or builtin type.  If successful,
         # creates a corresponding NameNode and returns it, otherwise
         # returns None.
-        type = self.obj.analyse_as_extension_type(env)
-        if type:
+        if self.obj.is_string_literal:
+            return
+        type = self.obj.analyse_as_type(env)
+        if type and (type.is_extension_type or type.is_builtin_type or type.is_cpp_class):
             entry = type.scope.lookup_here(self.attribute)
-            if entry and entry.is_cmethod:
+            if entry and (entry.is_cmethod or type.is_cpp_class and entry.type.is_cfunction):
                 if type.is_builtin_type:
                     if not self.is_called:
                         # must handle this as Python object
@@ -5314,9 +5324,22 @@ class AttributeNode(ExprNode):
                 else:
                     # Create a temporary entry describing the C method
                     # as an ordinary function.
-                    ubcm_entry = Symtab.Entry(entry.name,
-                        "%s->%s" % (type.vtabptr_cname, entry.cname),
-                        entry.type)
+                    if entry.func_cname and not hasattr(entry.type, 'op_arg_struct'):
+                        cname = entry.func_cname
+                        if entry.type.is_static_method:
+                            ctype = entry.type
+                        elif type.is_cpp_class:
+                            error(self.pos, "%s not a static member of %s" % (entry.name, type))
+                            ctype = PyrexTypes.error_type
+                        else:
+                            # Fix self type.
+                            ctype = copy.copy(entry.type)
+                            ctype.args = ctype.args[:]
+                            ctype.args[0] = PyrexTypes.CFuncTypeArg('self', type, 'self', None)
+                    else:
+                        cname = "%s->%s" % (type.vtabptr_cname, entry.cname)
+                        ctype = entry.type
+                    ubcm_entry = Symtab.Entry(entry.name, cname, ctype)
                     ubcm_entry.is_cfunction = 1
                     ubcm_entry.func_cname = entry.func_cname
                     ubcm_entry.is_unbound_cmethod = 1

Cython/Compiler/ModuleNode.py

@@ -826,7 +826,9 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
             has_virtual_methods = False
             has_destructor = False
             for attr in scope.var_entries:
-                if attr.type.is_cfunction and attr.name != "<init>":
+                if attr.type.is_cfunction and attr.type.is_static_method:
+                    code.put("static ")
+                elif attr.type.is_cfunction and attr.name != "<init>":
                     code.put("virtual ")
                     has_virtual_methods = True
                 if attr.cname[0] == '~':

Cython/Compiler/Options.py

@@ -209,6 +209,7 @@ directive_types = {
     'cfunc' : None, # decorators do not take directive value
     'ccall' : None,
     'inline' : None,
+    'staticmethod' : None,
     'cclass' : None,
     'returns' : type,
     'set_initial_path': str,
@@ -225,6 +226,7 @@ directive_scopes = { # defaults to available everywhere
     # 'module', 'function', 'class', 'with statement'
     'final' : ('cclass', 'function'),
     'inline' : ('function',),
+    'staticmethod' : ('function',),  # FIXME: analysis currently lacks more specific function scope
     'no_gc_clear' : ('cclass',),
     'internal' : ('cclass',),
     'autotestdict' : ('module',),

Cython/Compiler/ParseTreeTransforms.py

@@ -674,7 +674,7 @@ class InterpretCompilerDirectives(CythonTransform, SkipDeclarations):
         for key, value in compilation_directive_defaults.items():
             self.compilation_directive_defaults[unicode(key)] = copy.deepcopy(value)
         self.cython_module_names = set()
-        self.directive_names = {}
+        self.directive_names = {'staticmethod': 'staticmethod'}
         self.parallel_directives = {}
 
     def check_directive_scope(self, pos, directive, scope):
@@ -951,11 +951,11 @@ class InterpretCompilerDirectives(CythonTransform, SkipDeclarations):
         for name, value in directives.iteritems():
             if name == 'locals':
                 node.directive_locals = value
-            elif name != 'final':
+            elif name not in ('final', 'staticmethod'):
                 self.context.nonfatal_error(PostParseError(
                     node.pos,
-                    "Cdef functions can only take cython.locals() "
-                    "or final decorators, got %s." % name))
+                    "Cdef functions can only take cython.locals(), "
+                    "staticmethod, or final decorators, got %s." % name))
         body = Nodes.StatListNode(node.pos, stats=[node])
         return self.visit_with_directives(body, directives)
 
@@ -966,6 +966,13 @@ class InterpretCompilerDirectives(CythonTransform, SkipDeclarations):
         body = Nodes.StatListNode(node.pos, stats=[node])
         return self.visit_with_directives(body, directives)
 
+    def visit_CppClassNode(self, node):
+        directives = self._extract_directives(node, 'cppclass')
+        if not directives:
+            return self.visit_Node(node)
+        body = Nodes.StatListNode(node.pos, stats=[node])
+        return self.visit_with_directives(body, directives)
+
     def visit_PyClassDefNode(self, node):
         directives = self._extract_directives(node, 'class')
         if not directives:
@@ -979,18 +986,23 @@ class InterpretCompilerDirectives(CythonTransform, SkipDeclarations):
         # Split the decorators into two lists -- real decorators and directives
         directives = []
         realdecs = []
+        both = []
         for dec in node.decorators:
             new_directives = self.try_to_parse_directives(dec.decorator)
             if new_directives is not None:
                 for directive in new_directives:
                     if self.check_directive_scope(node.pos, directive[0], scope_name):
-                        directives.append(directive)
+                        name, value = directive
+                        if self.directives.get(name, object()) != value:
+                            directives.append(directive)
+                        if directive[0] == 'staticmethod':
+                            both.append(dec)
             else:
                 realdecs.append(dec)
         if realdecs and isinstance(node, (Nodes.CFuncDefNode, Nodes.CClassDefNode, Nodes.CVarDefNode)):
             raise PostParseError(realdecs[0].pos, "Cdef functions/classes cannot take arbitrary decorators.")
         else:
-            node.decorators = realdecs
+            node.decorators = realdecs + both
         # merge or override repeated directives
         optdict = {}
         directives.reverse() # Decorators coming first take precedence
@@ -2261,8 +2273,8 @@ class MarkClosureVisitor(CythonTransform):
 
     def visit_CFuncDefNode(self, node):
         self.visit_FuncDefNode(node)
-        if node.needs_closure:
-            error(node.pos, "closures inside cdef functions not yet supported")
+        if node.needs_closure and node.overridable:
+            error(node.pos, "closures inside cpdef functions not yet supported")
         return node
 
     def visit_LambdaNode(self, node):
@@ -2401,8 +2413,11 @@ class CreateClosureClasses(CythonTransform):
         return node
 
     def visit_CFuncDefNode(self, node):
-        self.visitchildren(node)
-        return node
+        if not node.overridable:
+            return self.visit_FuncDefNode(node)
+        else:
+            self.visitchildren(node)
+            return node
 
 
 class GilCheck(VisitorTransform):
@@ -2705,6 +2720,8 @@ class TransformBuiltinMethods(EnvTransform):
                     node.cdivision = True
             elif function == u'set':
                 node.function = ExprNodes.NameNode(node.pos, name=EncodedString('set'))
+            elif function == u'staticmethod':
+                node.function = ExprNodes.NameNode(node.pos, name=EncodedString('staticmethod'))
             elif self.context.cython_scope.lookup_qualified_name(function):
                 pass
             else:

Cython/Compiler/Parsing.pxd

@@ -187,3 +187,4 @@ cdef p_doc_string(PyrexScanner s)
 cdef p_ignorable_statement(PyrexScanner s)
 cdef p_compiler_directive_comments(PyrexScanner s)
 cdef p_cpp_class_definition(PyrexScanner s, pos, ctx)
+cdef p_cpp_class_attribute(PyrexScanner s, ctx)

Cython/Compiler/Parsing.py

@@ -3231,12 +3231,8 @@ def p_cpp_class_definition(s, pos,  ctx):
         body_ctx = Ctx(visibility = ctx.visibility, level='cpp_class', nogil=nogil or ctx.nogil)
         body_ctx.templates = templates
         while s.sy != 'DEDENT':
-            if s.systring == 'cppclass':
-                attributes.append(
-                    p_cpp_class_definition(s, s.position(), body_ctx))
-            elif s.sy != 'pass':
-                attributes.append(
-                    p_c_func_or_var_declaration(s, s.position(), body_ctx))
+            if s.sy != 'pass':
+                attributes.append(p_cpp_class_attribute(s, body_ctx))
             else:
                 s.next()
                 s.expect_newline("Expected a newline")
@@ -3253,6 +3249,23 @@ def p_cpp_class_definition(s, pos,  ctx):
         attributes = attributes,
         templates = templates)
 
+def p_cpp_class_attribute(s, ctx):
+    decorators = None
+    if s.sy == '@':
+        decorators = p_decorators(s)
+    if s.systring == 'cppclass':
+        return p_cpp_class_definition(s, s.position(), ctx)
+    else:
+        node = p_c_func_or_var_declaration(s, s.position(), ctx)
+        if decorators is not None:
+            tup = Nodes.CFuncDefNode, Nodes.CVarDefNode, Nodes.CClassDefNode
+            if ctx.allow_struct_enum_decorator:
+                tup += Nodes.CStructOrUnionDefNode, Nodes.CEnumDefNode
+            if not isinstance(node, tup):
+                s.error("Decorators can only be followed by functions or classes")
+            node.decorators = decorators
+        return node
+
 
 #----------------------------------------------
 #

Cython/Compiler/PyrexTypes.py

@@ -2315,6 +2315,7 @@ class CFuncType(CType):
     #  is_strict_signature boolean  function refuses to accept coerced arguments
     #                               (used for optimisation overrides)
     #  is_const_method  boolean
+    #  is_static_method boolean
 
     is_cfunction = 1
     original_sig = None
@@ -2327,7 +2328,8 @@ class CFuncType(CType):
     def __init__(self, return_type, args, has_varargs = 0,
             exception_value = None, exception_check = 0, calling_convention = "",
             nogil = 0, with_gil = 0, is_overridable = 0, optional_arg_count = 0,
-            is_const_method = False, templates = None, is_strict_signature = False):
+            is_const_method = False, is_static_method=False,
+            templates = None, is_strict_signature = False):
         self.return_type = return_type
         self.args = args
         self.has_varargs = has_varargs
@@ -2339,6 +2341,7 @@ class CFuncType(CType):
         self.with_gil = with_gil
         self.is_overridable = is_overridable
         self.is_const_method = is_const_method
+        self.is_static_method = is_static_method
         self.templates = templates
         self.is_strict_signature = is_strict_signature
 
@@ -2563,6 +2566,7 @@ class CFuncType(CType):
                            is_overridable = self.is_overridable,
                            optional_arg_count = self.optional_arg_count,
                            is_const_method = self.is_const_method,
+                           is_static_method = self.is_static_method,
                            templates = self.templates)
 
         result.from_fused = self.is_fused
@@ -3106,7 +3110,7 @@ class CppClassType(CType):
         # Need to do these *after* self.specializations[key] is set
         # to avoid infinite recursion on circular references.
         specialized.base_classes = [b.specialize(values) for b in self.base_classes]
-        specialized.scope = self.scope.specialize(values)
+        specialized.scope = self.scope.specialize(values, specialized)
         if self.namespace is not None:
             specialized.namespace = self.namespace.specialize(values)
         return specialized

Cython/Compiler/Symtab.py

@@ -1947,11 +1947,12 @@ class CClassScope(ClassScope):
         if get_special_method_signature(name) and not self.parent_type.is_builtin_type:
             error(pos, "Special methods must be declared with 'def', not 'cdef'")
         args = type.args
-        if not args:
-            error(pos, "C method has no self argument")
-        elif not self.parent_type.assignable_from(args[0].type):
-            error(pos, "Self argument (%s) of C method '%s' does not match parent type (%s)" %
-                  (args[0].type, name, self.parent_type))
+        if not type.is_static_method:
+            if not args:
+                error(pos, "C method has no self argument")
+            elif not self.parent_type.assignable_from(args[0].type):
+                error(pos, "Self argument (%s) of C method '%s' does not match parent type (%s)" %
+                      (args[0].type, name, self.parent_type))
         entry = self.lookup_here(name)
         if cname is None:
             cname = c_safe_identifier(name)
@@ -2106,7 +2107,8 @@ class CppClassScope(Scope):
             entry = self.declare(name, cname, type, pos, visibility)
         entry.is_variable = 1
         if type.is_cfunction and self.type:
-            entry.func_cname = "%s::%s" % (self.type.declaration_code(""), cname)
+            if not self.type.templates or not any(T.is_fused for T in self.type.templates):
+                entry.func_cname = "%s::%s" % (self.type.declaration_code(""), cname)
         if name != "this" and (defining or name != "<init>"):
             self.var_entries.append(entry)
         if type.is_pyobject and not allow_pyobject:
@@ -2189,8 +2191,9 @@ class CppClassScope(Scope):
                                            utility_code = base_entry.utility_code)
             entry.is_inherited = 1
 
-    def specialize(self, values):
+    def specialize(self, values, type_entry):
         scope = CppClassScope(self.name, self.outer_scope)
+        scope.type = type_entry
         for entry in self.entries.values():
             if entry.is_type:
                 scope.declare_type(entry.name,

Cython/Includes/libcpp/__init__.pxd

-# empty file
-
 cdef extern from *:
     ctypedef bint bool

Cython/Includes/libcpp/cast.pxd

+# Defines the standard C++ cast operators.
+#
+# Due to type restrictions, these are only defined for pointer parameters,
+# however that is the only case where they are significantly more interesting
+# than the standard C cast operator which can be written "<T>(expression)" in
+# Cython.
+
+cdef extern from *:
+    cdef T dynamic_cast[T](void *) except +   # nullptr may also indicate failure
+    cdef T static_cast[T](void *)
+    cdef T reinterpret_cast[T](void *)
+    cdef T const_cast[T](void *)

Cython/Plex/Scanners.pxd

@@ -24,7 +24,7 @@ cdef class Scanner:
     cdef public list queue
     cdef public bint trace
     cdef public cur_char
-    cdef public int input_state
+    cdef public long input_state
 
     cdef public level
 

Cython/Utils.py

@@ -399,3 +399,16 @@ def print_bytes(s, end=b'\n', file=sys.stdout, flush=True):
         out.write(end)
     if flush:
         out.flush()
+
+class LazyStr:
+    def __init__(self, callback):
+        self.callback = callback
+    def __str__(self):
+        return self.callback()
+    def __repr__(self):
+        return self.callback()
+    def __add__(self, right):
+        return self.callback() + right
+    def __radd__(self, left):
+        return left + self.callback()
+

docs/src/userguide/extension_types.rst

@@ -345,7 +345,7 @@ functions, C methods are declared using :keyword:`cdef` or :keyword:`cpdef` inst
 :keyword:`def`. C methods are "virtual", and may be overridden in derived
 extension types. In addition, :keyword:`cpdef` methods can even be overridden by python
 methods when called as C method. This adds a little to their calling overhead
-compared to a :keyword:`cdef` methd::
+compared to a :keyword:`cdef` method::
 
     # pets.pyx
     cdef class Parrot:
@@ -382,21 +382,29 @@ method using the usual Python technique, i.e.::
 
     Parrot.describe(self)
 
+`cdef` methods can be declared static by using the @staticmethod decorator.
+This can be especially useful for constructing classes that take non-Python
+compatible types.::
 
-Forward-declaring extension types
-===================================
+    cdef class OwnedPointer:
+        cdef void* ptr
 
-Extension types can be forward-declared, like :keyword:`struct` and
-:keyword:`union` types. This will be necessary if you have two extension types
-that need to refer to each other, e.g.::
+        cdef __dealloc__(self):
+            if ptr != NULL:
+                free(ptr)
 
-    cdef class Shrubbery # forward declaration
+        @staticmethod
+        cdef create(void* ptr):
+            p = OwnedPointer()
+            p.ptr = ptr
+            return ptr
 
-    cdef class Shrubber:
-        cdef Shrubbery work_in_progress
 
-    cdef class Shrubbery:
-        cdef Shrubber creator
+Forward-declaring extension types
+===================================
+
+Extension types can be forward-declared, like :keyword:`struct` and
+:keyword:`union` types. This is usually necessary.
 
 If you are forward-declaring an extension type that has a base class, you must
 specify the base class in both the forward declaration and its subsequent

docs/src/userguide/wrapping_CPlusPlus.rst

@@ -529,9 +529,10 @@ If the Rectangle class has a static member:
         };
     }
 
-you can declare it as a function living in the class namespace, i.e.::
+you can declare it using the Python @staticmethod decorator, i.e.::
 
-    cdef extern from "Rectangle.h" namespace "shapes::Rectangle":
+    cdef extern from "Rectangle.h" namespace "shapes":
+        @staticmethod
         void do_something()
 
 

runtests.py

@@ -115,6 +115,77 @@ def unpatch_inspect_isfunction():
     else:
         inspect.isfunction = orig_isfunction
 
+def def_to_cdef(source):
+    '''
+    Converts the module-level def methods into cdef methods, i.e.
+
+        @decorator
+        def foo([args]):
+            """
+            [tests]
+            """
+            [body]
+
+    becomes
+
+        def foo([args]):
+            """
+            [tests]
+            """
+            return foo_c([args])
+
+        cdef foo_c([args]):
+            [body]
+    '''
+    output = []
+    skip = False
+    def_node = re.compile(r'def (\w+)\(([^()*]*)\):').match
+    lines = iter(source.split('\n'))
+    for line in lines:
+        if not line.strip():
+            output.append(line)
+            continue
+
+        if skip:
+            if line[0] != ' ':
+                skip = False
+            else:
+                continue
+
+        if line[0] == '@':
+            skip = True
+            continue
+
+        m = def_node(line)
+        if m:
+            name = m.group(1)
+            args = m.group(2)
+            if args:
+                args_no_types = ", ".join(arg.split()[-1] for arg in args.split(','))
+            else:
+                args_no_types = ""
+            output.append("def %s(%s):" % (name, args_no_types))
+            line = next(lines)
+            if '"""' in line:
+                has_docstring = True
+                output.append(line)
+                for line in lines:
+                    output.append(line)
+                    if '"""' in line:
+                        break
+            else:
+                has_docstring = False
+            output.append("    return %s_c(%s)" % (name, args_no_types))
+            output.append('')
+            output.append("cdef %s_c(%s):" % (name, args))
+            if not has_docstring:
+                output.append(line)
+
+        else:
+            output.append(line)
+
+    return '\n'.join(output)
+
 def update_linetrace_extension(ext):
     ext.define_macros.append(('CYTHON_TRACE', 1))
     return ext
@@ -331,7 +402,7 @@ def parse_tags(filepath):
                 if tag == 'tags':
                     tag = 'tag'
                     print("WARNING: test tags use the 'tag' directive, not 'tags' (%s)" % filepath)
-                if tag not in ('mode', 'tag', 'ticket', 'cython', 'distutils'):
+                if tag not in ('mode', 'tag', 'ticket', 'cython', 'distutils', 'preparse'):
                     print("WARNING: unknown test directive '%s' found (%s)" % (tag, filepath))
                 values = values.split(',')
                 tags[tag].extend(filter(None, [value.strip() for value in values]))
@@ -532,19 +603,25 @@ class TestBuilder(object):
         elif 'no-cpp' in tags['tag'] and 'cpp' in self.languages:
             languages = list(languages)
             languages.remove('cpp')
+
+        preparse_list = tags.get('preparse', ['id'])
         tests = [ self.build_test(test_class, path, workdir, module, tags,
-                                  language, expect_errors, warning_errors)
-                  for language in languages ]
+                                  language, expect_errors, warning_errors, preparse)
+                  for language in languages
+                  for preparse in preparse_list ]
         return tests
 
     def build_test(self, test_class, path, workdir, module, tags,
-                   language, expect_errors, warning_errors):
+                   language, expect_errors, warning_errors, preparse):
         language_workdir = os.path.join(workdir, language)
         if not os.path.exists(language_workdir):
             os.makedirs(language_workdir)
         workdir = os.path.join(language_workdir, module)
+        if preparse != 'id':
+            workdir += '_%s' % str(preparse)
         return test_class(path, workdir, module, tags,
                           language=language,
+                          preparse=preparse,
                           expect_errors=expect_errors,
                           annotate=self.annotate,
                           cleanup_workdir=self.cleanup_workdir,
@@ -556,7 +633,7 @@ class TestBuilder(object):
                           warning_errors=warning_errors)
 
 class CythonCompileTestCase(unittest.TestCase):
-    def __init__(self, test_directory, workdir, module, tags, language='c',
+    def __init__(self, test_directory, workdir, module, tags, language='c', preparse='id',
                  expect_errors=False, annotate=False, cleanup_workdir=True,
                  cleanup_sharedlibs=True, cleanup_failures=True, cython_only=False,
                  fork=True, language_level=2, warning_errors=False):
@@ -565,6 +642,8 @@ class CythonCompileTestCase(unittest.TestCase):
         self.workdir = workdir
         self.module = module
         self.language = language
+        self.preparse = preparse
+        self.name = module if self.preparse == "id" else "%s_%s" % (module, preparse)
         self.expect_errors = expect_errors
         self.annotate = annotate
         self.cleanup_workdir = cleanup_workdir
@@ -577,7 +656,7 @@ class CythonCompileTestCase(unittest.TestCase):
         unittest.TestCase.__init__(self)
 
     def shortDescription(self):
-        return "compiling (%s) %s" % (self.language, self.module)
+        return "compiling (%s) %s" % (self.language, self.name)
 
     def setUp(self):
         from Cython.Compiler import Options
@@ -660,11 +739,16 @@ class CythonCompileTestCase(unittest.TestCase):
                 if is_related(filename)]
 
     def copy_files(self, test_directory, target_directory, file_list):
-        # use symlink on Unix, copy on Windows
-        try:
-            copy = os.symlink
-        except AttributeError:
-            copy = shutil.copy
+        if self.preparse and self.preparse != 'id':
+            preparse_func = globals()[self.preparse]
+            def copy(src, dest):
+                open(dest, 'w').write(preparse_func(open(src).read()))
+        else:
+            # use symlink on Unix, copy on Windows
+            try:
+                copy = os.symlink
+            except AttributeError:
+                copy = shutil.copy
 
         join = os.path.join
         for filename in file_list:
@@ -707,6 +791,12 @@ class CythonCompileTestCase(unittest.TestCase):
             include_dirs.append(incdir)
         source = self.find_module_source_file(
             os.path.join(test_directory, module + '.pyx'))
+        if self.preparse == 'id':
+            source = self.find_module_source_file(
+                os.path.join(test_directory, module + '.pyx'))
+        else:
+            self.copy_files(test_directory, targetdir, [module + '.pyx'])
+            source = os.path.join(targetdir, module + '.pyx')
         target = os.path.join(targetdir, self.build_target_filename(module))
 
         if extra_compile_options is None:
@@ -903,7 +993,7 @@ class CythonRunTestCase(CythonCompileTestCase):
         if self.cython_only:
             return CythonCompileTestCase.shortDescription(self)
         else:
-            return "compiling (%s) and running %s" % (self.language, self.module)
+            return "compiling (%s) and running %s" % (self.language, self.name)
 
     def run(self, result=None):
         if result is None:
@@ -1105,7 +1195,7 @@ class PartialTestResult(_TextTestResult):
 
 class CythonUnitTestCase(CythonRunTestCase):
     def shortDescription(self):
-        return "compiling (%s) tests in %s" % (self.language, self.module)
+        return "compiling (%s) tests in %s" % (self.language, self.name)
 
     def run_tests(self, result, ext_so_path):
         module = import_ext(self.module, ext_so_path)

tests/bugs.txt

@@ -6,7 +6,6 @@ unsignedbehaviour_T184
 missing_baseclass_in_predecl_T262
 cfunc_call_tuple_args_T408
 cpp_structs
-closure_inside_cdef_T554
 genexpr_iterable_lookup_T600
 generator_expressions_in_class
 for_from_pyvar_loop_T601

tests/compile/builtinfuncs.pyx

@@ -5,18 +5,20 @@ cdef int f() except -1:
     cdef str sstring
     cdef basestring sustring
     cdef int i
+    cdef long lng
+    cdef Py_ssize_t s
     x = abs(y)
     delattr(x, 'spam')
     x = dir(y)
     x = divmod(y, z)
     x = getattr(y, 'spam')
     i = hasattr(y, 'spam')
-    i = hash(y)
+    lng = hash(y)
     x = intern(y)
     i = isinstance(y, z)
     i = issubclass(y, z)
     x = iter(y)
-    i = len(x)
+    s = len(x)
     x = open(y, z)
     x = pow(y, z, w)
     x = pow(y, z)

tests/errors/e_cdef_closure.pyx

 # mode: error
 
-cdef cdef_yield():
-    def inner():
-        pass
-
 cpdef cpdef_yield():
     def inner():
         pass
 
 _ERRORS = u"""
-3:5: closures inside cdef functions not yet supported
-7:6: closures inside cdef functions not yet supported
+3:6: closures inside cpdef functions not yet supported
 """

tests/run/closure_self.pyx

@@ -38,3 +38,19 @@ cdef class SelfInClosure(object):
         def nested():
             return self.x, t.x
         return nested
+
+    def call_closure_method_cdef_attr_c(self, Test t):
+        """
+        >>> o = SelfInClosure()
+        >>> o.call_closure_method_cdef_attr_c(Test())()
+        (1, 2)
+        """
+        return self.closure_method_cdef_attr_c(t)
+
+    cdef closure_method_cdef_attr_c(self, Test t):
+        t.x = 2
+        self._t = t
+        self.x = 1
+        def nested():
+            return self.x, t.x
+        return nested

tests/run/closure_tests_1.pyx

 # mode: run
 # tag: closures
+# preparse: id
+# preparse: def_to_cdef
 #
 # closure_tests_1.pyx
 #

tests/run/closure_tests_2.pyx

 # mode: run
 # tag: closures
+# preparse: id
+# preparse: def_to_cdef
 #
 # closure_tests_2.pyx
 #

tests/run/closure_tests_3.pyx

 # mode: run
 # tag: closures
+# preparse: id
+# preparse: def_to_cdef
 #
 # closure_tests_3.pyx
 #

tests/run/closure_tests_4.pyx

 # mode: run
 # tag: closures
+# preparse: id
+# preparse: def_to_cdef
 #
 # closure_tests_4.pyx
 #

tests/run/closures_T82.pyx

 # mode: run
 # tag: closures
 # ticket: 82
+# preparse: id
+# preparse: def_to_cdef
 
 cimport cython
 

tests/run/cpp_classes_def.pyx

@@ -42,6 +42,21 @@ def test_Poly(int n, float radius=1):
         del poly
 
 
+cdef cppclass WithStatic:
+    @staticmethod
+    double square(double x):
+        return x * x
+
+def test_Static(x):
+    """
+    >>> test_Static(2)
+    4.0
+    >>> test_Static(0.5)
+    0.25
+    """
+    return WithStatic.square(x)
+
+
 cdef cppclass InitDealloc:
     __init__():
         print "Init"

tests/run/cpp_templates.pyx

@@ -22,6 +22,10 @@ cdef extern from "cpp_templates_helper.h":
     cdef cppclass SubClass[T2, T3](SuperClass[T2, T3]):
         pass
 
+    cdef cppclass Div[T]:
+        @staticmethod
+        T half(T value)
+
 def test_int(int x, int y):
     """
     >>> test_int(3, 4)
@@ -104,3 +108,12 @@ def test_cast_template_pointer():
 
     sup = sub
     sup = <SubClass[int, float] *> sub
+
+def test_static(x):
+    """
+    >>> test_static(2)
+    (1, 1.0)
+    >>> test_static(3)
+    (1, 1.5)
+    """
+    return Div[int].half(x), Div[double].half(x)

tests/run/cpp_templates_helper.h

@@ -30,3 +30,9 @@ public:
 template <class T2, class T3>
 class SubClass : public SuperClass<T2, T3> {
 };
+
+template <class T>
+class Div {
+public:
+    static T half(T value) { return value / 2; }
+};

tests/run/py2_super.pyx

+# mode: run
+# tag: py3k_super
+
+class A(object):
+    def method(self):
+        return 1
+
+    @classmethod
+    def class_method(cls):
+        return 2
+
+    @staticmethod
+    def static_method():
+        return 3
+
+    def generator_test(self):
+        return [1, 2, 3]
+
+
+class B(A):
+    """
+    >>> obj = B()
+    >>> obj.method()
+    1
+    >>> B.class_method()
+    2
+    >>> B.static_method(obj)
+    3
+    >>> list(obj.generator_test())
+    [1, 2, 3]
+    """
+    def method(self):
+        return super(B, self).method()
+
+    @classmethod
+    def class_method(cls):
+        return super(B, cls).class_method()
+
+    @staticmethod
+    def static_method(instance):
+        return super(B, instance).static_method()
+
+    def generator_test(self):
+        for i in super(B, self).generator_test():
+            yield i
+
+
+cdef class CClassBase(object):
+    def method(self):
+        return 'def'
+    cpdef method_cp(self):
+        return 'cpdef'
+
+#     cdef method_c(self):
+#         return 'cdef'
+#     def call_method_c(self):
+#         return self.method_c()
+
+cdef class CClassSub(CClassBase):
+    """
+    >>> CClassSub().method()
+    'def'
+    >>> CClassSub().method_cp()
+    'cpdef'
+    """
+#     >>> CClassSub().call_method_c()
+#     'cdef'
+
+    def method(self):
+        return super(CClassSub, self).method()
+    cpdef method_cp(self):
+        return super(CClassSub, self).method_cp()
+
+#     cdef method_c(self):
+#         return super(CClassSub, self).method_c()
+
+cdef class Base(object):
+    """
+    >>> Base().method()
+    'Base'
+    >>> Base.method(Base())
+    'Base'
+    """
+    cpdef method(self):
+        return "Base"
+
+cdef class Sub(Base):
+    """
+    >>> Sub().method()
+    'Sub'
+    >>> Sub.method(Sub())
+    'Sub'
+    >>> Base.method(Sub())
+    'Base'
+    """
+    cpdef method(self):
+        return "Sub"

tests/run/py3k_super.pyx

@@ -63,13 +63,29 @@ def test_class_cell_empty():
 
 cdef class CClassBase(object):
     def method(self):
-        return 1
+        return 'def'
+
+#     cpdef method_cp(self):
+#         return 'cpdef'
+#     cdef method_c(self):
+#         return 'cdef'
+#     def call_method_c(self):
+#         return self.method_c()
 
-cdef class CClassSuper(CClassBase):
+cdef class CClassSub(CClassBase):
     """
-    >>> CClassSuper().method()
-    1
+    >>> CClassSub().method()
+    'def'
     """
+#     >>> CClassSub().method_cp()
+#     'cpdef'
+#     >>> CClassSub().call_method_c()
+#     'cdef'
 
     def method(self):
         return super().method()
+
+#     cpdef method_cp(self):
+#         return super().method_cp()
+#     cdef method_c(self):
+#         return super().method_c()

tests/run/static_methods.pyx

+cdef class A:
+    @staticmethod
+    def static_def(int x):
+        """
+        >>> A.static_def(2)
+        ('def', 2)
+        >>> A().static_def(2)
+        ('def', 2)
+        """
+        return 'def', x
+
+    @staticmethod
+    cdef static_cdef(int* x):
+        return 'cdef', x[0]
+
+#     @staticmethod
+#     cpdef static_cpdef(int x):
+#         """
+#         >>> A.static_def
+#         >>> A.static_cpdef
+#
+#         >>> A().static_def
+#         >>> A().static_cpdef
+#
+#         >>> A.static_cpdef(2)
+#         ('cpdef', 2)
+#         >>> A().static_cpdef(2)
+#         ('cpdef', 2)
+#         """
+#         return 'cpdef', x
+
+def call_static_def(int x):
+    """
+    >>> call_static_def(2)
+    ('def', 2)
+    """
+    return A.static_def(x)
+
+def call_static_cdef(int x):
+    """
+    >>> call_static_cdef(2)
+    ('cdef', 2)
+    """
+    cdef int *x_ptr = &x
+    return A.static_cdef(x_ptr)
+
+# def call_static_cpdef(int x):
+#     """
+#     >>> call_static_cpdef(2)
+#     ('cpdef', 2)
+#     """
+#     return A.static_cpdef(x)