Merge branch 'master' of github.com:cython/cython into cdef_closure

CHANGES.rst

@@ -21,6 +21,13 @@ Features added
   is raised only when such a pointer is assigned to a variable and
   would thus exceed the lifetime of the string itself.
 
+* The "and"/"or" operators try to avoid unnecessary coercions of their
+  arguments.  They now evaluate the truth value of each argument
+  independently and only coerce the final result of the whole expression
+  to the target type (e.g. the type on the left side of an assignment).
+  This also avoids reference counting overhead for Python values during
+  evaluation and generally improves the code flow in the generated C code.
+
 * Cascaded assignments (a = b = ...) try to minimise the number of
   type coercions.
 
@@ -84,6 +91,8 @@ Bugs fixed
 
 * Correctly handle ``from cython.submodule cimport name``.
 
+* Fix infinite recursion when using super with cpdef methods.
+
 Other changes
 -------------
 

Cython/Compiler/Nodes.py

@@ -27,6 +27,7 @@ from .Code import UtilityCode
 from .StringEncoding import EncodedString, escape_byte_string, split_string_literal
 from . import Options
 from . import DebugFlags
+from Cython.Utils import LazyStr
 
 absolute_path_length = 0
 
@@ -2294,10 +2295,16 @@ class CFuncDefNode(FuncDefNode):
         if is_module_scope:
             cfunc = ExprNodes.NameNode(self.pos, name=self.entry.name)
         else:
-            self_arg = ExprNodes.NameNode(self.pos, name=arg_names[0])
-            cfunc = ExprNodes.AttributeNode(self.pos, obj=self_arg, attribute=self.entry.name)
+            type_entry = self.type.args[0].type.entry
+            type_arg = ExprNodes.NameNode(self.pos, name=type_entry.name)
+            type_arg.entry = type_entry
+            cfunc = ExprNodes.AttributeNode(self.pos, obj=type_arg, attribute=self.entry.name)
         skip_dispatch = not is_module_scope or Options.lookup_module_cpdef
-        c_call = ExprNodes.SimpleCallNode(self.pos, function=cfunc, args=[ExprNodes.NameNode(self.pos, name=n) for n in arg_names[1-is_module_scope:]], wrapper_call=skip_dispatch)
+        c_call = ExprNodes.SimpleCallNode(
+            self.pos,
+            function=cfunc,
+            args=[ExprNodes.NameNode(self.pos, name=n) for n in arg_names],
+            wrapper_call=skip_dispatch)
         return ReturnStatNode(pos=self.pos, return_type=PyrexTypes.py_object_type, value=c_call)
 
     def declare_arguments(self, env):

Cython/Compiler/Optimize.py

@@ -816,6 +816,8 @@ class SwitchTransform(Visitor.CythonTransform):
             if isinstance(cond, (ExprNodes.CoerceToTempNode,
                                  ExprNodes.CoerceToBooleanNode)):
                 cond = cond.arg
+            elif isinstance(cond, ExprNodes.BoolBinopResultNode):
+                cond = cond.arg.arg
             elif isinstance(cond, UtilNodes.EvalWithTempExprNode):
                 # this is what we get from the FlattenInListTransform
                 cond = cond.subexpression
@@ -860,7 +862,7 @@ class SwitchTransform(Visitor.CythonTransform):
                     elif getattr(cond.operand1, 'entry', None) \
                              and cond.operand1.entry.is_const:
                         return not_in, cond.operand2, [cond.operand1]
-        elif isinstance(cond, ExprNodes.BoolBinopNode):
+        elif isinstance(cond, (ExprNodes.BoolBinopNode, ExprNodes.GenericBoolBinopNode)):
             if cond.operator == 'or' or (allow_not_in and cond.operator == 'and'):
                 allow_not_in = (cond.operator == 'and')
                 not_in_1, t1, c1 = self.extract_conditions(cond.operand1, allow_not_in)

Cython/Compiler/PyrexTypes.py

@@ -3711,6 +3711,9 @@ def independent_spanning_type(type1, type2):
         return py_object_type
     span_type = _spanning_type(type1, type2)
     if span_type is None:
+        if type1.is_ptr and type2.is_ptr:
+            # incompatible pointers, void* will do as a result
+            return c_void_ptr_type
         return error_type
     return span_type
 

Cython/Plex/Scanners.pxd

@@ -24,7 +24,7 @@ cdef class Scanner:
     cdef public list queue
     cdef public bint trace
     cdef public cur_char
-    cdef public int input_state
+    cdef public long input_state
 
     cdef public level
 

Cython/Utils.py

@@ -399,3 +399,16 @@ def print_bytes(s, end=b'\n', file=sys.stdout, flush=True):
         out.write(end)
     if flush:
         out.flush()
+
+class LazyStr:
+    def __init__(self, callback):
+        self.callback = callback
+    def __str__(self):
+        return self.callback()
+    def __repr__(self):
+        return self.callback()
+    def __add__(self, right):
+        return self.callback() + right
+    def __radd__(self, left):
+        return left + self.callback()
+

cythonize.py

+#!/usr/bin/env python
+
+#
+#   Cython -- enhanced main program
+#
+
+if __name__ == '__main__':
+    from Cython.Build.Cythonize import main
+    main()

setup.py

@@ -74,14 +74,15 @@ if 'setuptools' in sys.modules:
     setuptools_extra_args['entry_points'] = {
         'console_scripts': [
             'cython = Cython.Compiler.Main:setuptools_main',
+            'cythonize = Cython.Build.Cythonize:main'
         ]
     }
     scripts = []
 else:
     if os.name == "posix":
-        scripts = ["bin/cython"]
+        scripts = ["bin/cython", 'bin/cythonize']
     else:
-        scripts = ["cython.py"]
+        scripts = ["cython.py", "cythonize.py"]
 
 if include_debugger:
     if 'setuptools' in sys.modules:

tests/compile/builtinfuncs.pyx

@@ -5,18 +5,20 @@ cdef int f() except -1:
     cdef str sstring
     cdef basestring sustring
     cdef int i
+    cdef long lng
+    cdef Py_ssize_t s
     x = abs(y)
     delattr(x, 'spam')
     x = dir(y)
     x = divmod(y, z)
     x = getattr(y, 'spam')
     i = hasattr(y, 'spam')
-    i = hash(y)
+    lng = hash(y)
     x = intern(y)
     i = isinstance(y, z)
     i = issubclass(y, z)
     x = iter(y)
-    i = len(x)
+    s = len(x)
     x = open(y, z)
     x = pow(y, z, w)
     x = pow(y, z)

tests/run/constant_folding.py

@@ -390,8 +390,8 @@ def combined():
     '//IntNode[@value = "4"]',
     '//IntNode[@value = "5"]',
     '//IntNode[@value = "7"]',
-    '//BoolBinopNode//PrimaryCmpNode',
-    '//BoolBinopNode[.//PrimaryCmpNode//IntNode[@value = "4"] and .//PrimaryCmpNode//IntNode[@value = "5"]]',
+    '//GenericBoolBinopNode//PrimaryCmpNode',
+    '//GenericBoolBinopNode[.//PrimaryCmpNode//IntNode[@value = "4"] and .//PrimaryCmpNode//IntNode[@value = "5"]]',
     '//PrimaryCmpNode[.//IntNode[@value = "2"] and .//IntNode[@value = "4"]]',
     '//PrimaryCmpNode[.//IntNode[@value = "5"] and .//IntNode[@value = "7"]]',
 )
@@ -423,11 +423,11 @@ def cascaded_cmp_with_partial_constants(a, b):
     '//IntNode[@value = "4"]',
     '//IntNode[@value = "5"]',
     '//IntNode[@value = "7"]',
-    '//BoolBinopNode',
-    '//SingleAssignmentNode//BoolBinopNode',
-    '//SingleAssignmentNode//BoolBinopNode//NameNode[@name = "a"]',
-    '//SingleAssignmentNode//BoolBinopNode//NameNode[@name = "b"]',
-    '//BoolBinopNode[.//PrimaryCmpNode//IntNode[@value = "4"] and .//PrimaryCmpNode//IntNode[@value = "5"]]',
+    '//GenericBoolBinopNode',
+    '//SingleAssignmentNode//GenericBoolBinopNode',
+    '//SingleAssignmentNode//GenericBoolBinopNode//NameNode[@name = "a"]',
+    '//SingleAssignmentNode//GenericBoolBinopNode//NameNode[@name = "b"]',
+    '//GenericBoolBinopNode[.//PrimaryCmpNode//IntNode[@value = "4"] and .//PrimaryCmpNode//IntNode[@value = "5"]]',
     '//BoolNode[@value = False]',
 )
 @cython.test_fail_if_path_exists(

tests/run/cpdef_enums_import.srctree

@@ -34,12 +34,12 @@ from enums cimport *
 # We can import enums with a star import.
 from enums import *
 
-print dir()
+print(dir())
 assert 'BAR' in dir() and 'FOO' in dir()
 
 # enums not generated in the wrong module
 import no_enums
-print dir(no_enums)
+print(dir(no_enums))
 assert 'FOO' not in dir(no_enums)
 assert 'foo' not in dir(no_enums)
 

tests/run/notinop.pyx

@@ -83,7 +83,7 @@ def m_tuple(int a):
     return result
 
 @cython.test_assert_path_exists("//SwitchStatNode")
-@cython.test_fail_if_path_exists("//BoolBinopNode", "//PrimaryCmpNode")
+@cython.test_fail_if_path_exists("//BoolBinopNode", "//GenericBoolBinopNode", "//PrimaryCmpNode")
 def m_set(int a):
     """
     >>> m_set(2)
@@ -97,7 +97,7 @@ def m_set(int a):
 cdef bytes bytes_string = b'abcdefg'
 
 @cython.test_assert_path_exists("//PrimaryCmpNode")
-@cython.test_fail_if_path_exists("//SwitchStatNode", "//BoolBinopNode")
+@cython.test_fail_if_path_exists("//SwitchStatNode", "//BoolBinopNode", "//GenericBoolBinopNode")
 def m_bytes(char a):
     """
     >>> m_bytes(ord('f'))
@@ -109,7 +109,7 @@ def m_bytes(char a):
     return result
 
 @cython.test_assert_path_exists("//SwitchStatNode")
-@cython.test_fail_if_path_exists("//BoolBinopNode", "//PrimaryCmpNode")
+@cython.test_fail_if_path_exists("//BoolBinopNode", "//GenericBoolBinopNode", "//PrimaryCmpNode")
 def m_bytes_literal(char a):
     """
     >>> m_bytes_literal(ord('f'))
@@ -127,7 +127,7 @@ cdef unicode klingon_character = u'\uF8D2'
 py_klingon_character = klingon_character
 
 @cython.test_assert_path_exists("//PrimaryCmpNode")
-@cython.test_fail_if_path_exists("//SwitchStatNode", "//BoolBinopNode")
+@cython.test_fail_if_path_exists("//SwitchStatNode", "//GenericBoolBinopNode", "//BoolBinopNode")
 def m_unicode(Py_UNICODE a, unicode unicode_string):
     """
     >>> m_unicode(ord('f'), py_unicode_string)
@@ -147,7 +147,7 @@ def m_unicode(Py_UNICODE a, unicode unicode_string):
     return result
 
 @cython.test_assert_path_exists("//SwitchStatNode")
-@cython.test_fail_if_path_exists("//BoolBinopNode", "//PrimaryCmpNode")
+@cython.test_fail_if_path_exists("//GenericBoolBinopNode", "//BoolBinopNode", "//PrimaryCmpNode")
 def m_unicode_literal(Py_UNICODE a):
     """
     >>> m_unicode_literal(ord('f'))
@@ -160,7 +160,7 @@ def m_unicode_literal(Py_UNICODE a):
     cdef int result = a not in u'abcdefg\u1234\uF8D2'
     return result
 
-@cython.test_assert_path_exists("//SwitchStatNode", "//BoolBinopNode")
+@cython.test_assert_path_exists("//SwitchStatNode", "//GenericBoolBinopNode")
 @cython.test_fail_if_path_exists("//PrimaryCmpNode")
 def m_tuple_in_or_notin(int a):
     """
@@ -174,7 +174,7 @@ def m_tuple_in_or_notin(int a):
     cdef int result = a not in (1,2,3,4) or a in (3,4)
     return result
 
-@cython.test_assert_path_exists("//SwitchStatNode", "//BoolBinopNode")
+@cython.test_assert_path_exists("//SwitchStatNode", "//GenericBoolBinopNode")
 @cython.test_fail_if_path_exists("//PrimaryCmpNode")
 def m_tuple_notin_or_notin(int a):
     """
@@ -189,7 +189,7 @@ def m_tuple_notin_or_notin(int a):
     return result
 
 @cython.test_assert_path_exists("//SwitchStatNode")
-@cython.test_fail_if_path_exists("//BoolBinopNode", "//PrimaryCmpNode")
+@cython.test_fail_if_path_exists("//GenericBoolBinopNode", "//BoolBinopNode", "//PrimaryCmpNode")
 def m_tuple_notin_and_notin(int a):
     """
     >>> m_tuple_notin_and_notin(2)
@@ -202,7 +202,7 @@ def m_tuple_notin_and_notin(int a):
     cdef int result = a not in (1,2,3,4) and a not in (6,7)
     return result
 
-@cython.test_assert_path_exists("//SwitchStatNode", "//BoolBinopNode")
+@cython.test_assert_path_exists("//SwitchStatNode", "//GenericBoolBinopNode")
 @cython.test_fail_if_path_exists("//PrimaryCmpNode")
 def m_tuple_notin_and_notin_overlap(int a):
     """
@@ -217,7 +217,7 @@ def m_tuple_notin_and_notin_overlap(int a):
     return result
 
 @cython.test_assert_path_exists("//SwitchStatNode")
-@cython.test_fail_if_path_exists("//BoolBinopNode", "//PrimaryCmpNode")
+@cython.test_fail_if_path_exists("//GenericBoolBinopNode", "//BoolBinopNode", "//PrimaryCmpNode")
 def conditional_int(int a):
     """
     >>> conditional_int(1)
@@ -230,7 +230,7 @@ def conditional_int(int a):
     return 1 if a not in (1,2,3,4) else 2
 
 @cython.test_assert_path_exists("//SwitchStatNode")
-@cython.test_fail_if_path_exists("//BoolBinopNode", "//PrimaryCmpNode")
+@cython.test_fail_if_path_exists("//GenericBoolBinopNode", "//BoolBinopNode", "//PrimaryCmpNode")
 def conditional_object(int a):
     """
     >>> conditional_object(1)
@@ -243,7 +243,7 @@ def conditional_object(int a):
     return 1 if a not in (1,2,3,4) else '2'
 
 @cython.test_assert_path_exists("//SwitchStatNode")
-@cython.test_fail_if_path_exists("//BoolBinopNode", "//PrimaryCmpNode")
+@cython.test_fail_if_path_exists("//GenericBoolBinopNode", "//BoolBinopNode", "//PrimaryCmpNode")
 def conditional_bytes(char a):
     """
     >>> conditional_bytes(ord('a'))
@@ -256,7 +256,7 @@ def conditional_bytes(char a):
     return 1 if a not in b'abc' else '2'
 
 @cython.test_assert_path_exists("//SwitchStatNode")
-@cython.test_fail_if_path_exists("//BoolBinopNode", "//PrimaryCmpNode")
+@cython.test_fail_if_path_exists("//GenericBoolBinopNode", "//BoolBinopNode", "//PrimaryCmpNode")
 def conditional_unicode(Py_UNICODE a):
     """
     >>> conditional_unicode(ord('a'))
@@ -269,7 +269,7 @@ def conditional_unicode(Py_UNICODE a):
     return 1 if a not in u'abc' else '2'
 
 @cython.test_assert_path_exists("//SwitchStatNode")
-@cython.test_fail_if_path_exists("//BoolBinopNode", "//PrimaryCmpNode")
+@cython.test_fail_if_path_exists("//GenericBoolBinopNode", "//BoolBinopNode", "//PrimaryCmpNode")
 def conditional_none(int a):
     """
     >>> conditional_none(1)

tests/run/pointers.pyx

+cimport cython
 
 cdef char* c_string = b'abcdefg'
 cdef void* void_ptr = c_string
@@ -60,3 +61,29 @@ def bool_binop_truth(int x):
         print True
     if c_string and x or not (void_ptr or int_ptr and float_ptr) or x:
         print True
+
+
+def binop_voidptr(int x, long y, char* z):
+    """
+    >>> binop_voidptr(1, 3, b'abc')
+    'void *'
+    """
+    result = &x and &y and z
+    return cython.typeof(result)
+
+
+def cond_expr_voidptr(int x, long y, char* z):
+    """
+    >>> cond_expr_voidptr(0, -1, b'abc')
+    ('void *', 0)
+    >>> cond_expr_voidptr(-1, 0, b'abc')
+    ('void *', -1)
+    >>> cond_expr_voidptr(-1, 0, b'')
+    ('void *', 0)
+    >>> cond_expr_voidptr(0, -1, b'')
+    ('void *', -1)
+    """
+    result = &x if len(z) else &y
+    assert sizeof(long) >= sizeof(int)
+    assert -1 == <int>(-1L)
+    return cython.typeof(result), (<int*>result)[0]

tests/run/py2_super.pyx

+# mode: run
+# tag: py3k_super
+
+class A(object):
+    def method(self):
+        return 1
+
+    @classmethod
+    def class_method(cls):
+        return 2
+
+    @staticmethod
+    def static_method():
+        return 3
+
+    def generator_test(self):
+        return [1, 2, 3]
+
+
+class B(A):
+    """
+    >>> obj = B()
+    >>> obj.method()
+    1
+    >>> B.class_method()
+    2
+    >>> B.static_method(obj)
+    3
+    >>> list(obj.generator_test())
+    [1, 2, 3]
+    """
+    def method(self):
+        return super(B, self).method()
+
+    @classmethod
+    def class_method(cls):
+        return super(B, cls).class_method()
+
+    @staticmethod
+    def static_method(instance):
+        return super(B, instance).static_method()
+
+    def generator_test(self):
+        for i in super(B, self).generator_test():
+            yield i
+
+
+cdef class CClassBase(object):
+    def method(self):
+        return 'def'
+    cpdef method_cp(self):
+        return 'cpdef'
+
+#     cdef method_c(self):
+#         return 'cdef'
+#     def call_method_c(self):
+#         return self.method_c()
+
+cdef class CClassSub(CClassBase):
+    """
+    >>> CClassSub().method()
+    'def'
+    >>> CClassSub().method_cp()
+    'cpdef'
+    """
+#     >>> CClassSub().call_method_c()
+#     'cdef'
+
+    def method(self):
+        return super(CClassSub, self).method()
+    cpdef method_cp(self):
+        return super(CClassSub, self).method_cp()
+
+#     cdef method_c(self):
+#         return super(CClassSub, self).method_c()
+
+cdef class Base(object):
+    """
+    >>> Base().method()
+    'Base'
+    >>> Base.method(Base())
+    'Base'
+    """
+    cpdef method(self):
+        return "Base"
+
+cdef class Sub(Base):
+    """
+    >>> Sub().method()
+    'Sub'
+    >>> Sub.method(Sub())
+    'Sub'
+    >>> Base.method(Sub())
+    'Base'
+    """
+    cpdef method(self):
+        return "Sub"

tests/run/py3k_super.pyx

@@ -63,13 +63,29 @@ def test_class_cell_empty():
 
 cdef class CClassBase(object):
     def method(self):
-        return 1
+        return 'def'
+
+#     cpdef method_cp(self):
+#         return 'cpdef'
+#     cdef method_c(self):
+#         return 'cdef'
+#     def call_method_c(self):
+#         return self.method_c()
 
-cdef class CClassSuper(CClassBase):
+cdef class CClassSub(CClassBase):
     """
-    >>> CClassSuper().method()
-    1
+    >>> CClassSub().method()
+    'def'
     """
+#     >>> CClassSub().method_cp()
+#     'cpdef'
+#     >>> CClassSub().call_method_c()
+#     'cdef'
 
     def method(self):
         return super().method()
+
+#     cpdef method_cp(self):
+#         return super().method_cp()
+#     cdef method_c(self):
+#         return super().method_c()