Merge branch 'master' of git+ssh://github.com/cython/cython

CHANGES.rst

@@ -28,6 +28,9 @@ Features added
 * ``len(memoryview)`` can be used in nogil sections to get the size of the
   first dimension of a memory view (``shape[0]``). (Github issue #1733)
 
+* C++ classes can now contain (properly refcounted) Python objects.
+
+
 Bugs fixed
 ----------
 
@@ -39,6 +42,9 @@ Bugs fixed
 * Compile time ``DEF`` assignments were evaluated even when they occur inside of
   falsy ``IF`` blocks. (Github issue #1796)
 
+* abs(signed int) now returns a signed rather than unsigned int.
+  (Github issue #1837)
+
 
 
 0.26.1 (2017-??-??)

Cython/Compiler/Builtin.py

@@ -95,16 +95,25 @@ builtin_function_table = [
                     is_strict_signature = True),
     BuiltinFunction('abs',        "f",    "f",     "fabsf",
                     is_strict_signature = True),
+    BuiltinFunction('abs',        "i",    "i",     "abs",
+                    is_strict_signature = True),
+    BuiltinFunction('abs',        "l",    "l",     "labs",
+                    is_strict_signature = True),
+    BuiltinFunction('abs',        None,    None,   "__Pyx_abs_longlong",
+                utility_code = UtilityCode.load("abs_longlong", "Builtins.c"),
+                func_type = PyrexTypes.CFuncType(
+                    PyrexTypes.c_longlong_type, [
+                        PyrexTypes.CFuncTypeArg("arg", PyrexTypes.c_longlong_type, None)
+                        ],
+                    is_strict_signature = True, nogil=True)),
     ] + list(
         # uses getattr to get PyrexTypes.c_uint_type etc to allow easy iteration over a list
-        BuiltinFunction('abs',        None,    None,   "__Pyx_abs_{0}".format(t),
-                    utility_code = UtilityCode.load("abs_{0}".format(t), "Builtins.c"),
+        BuiltinFunction('abs',        None,    None,   "/*abs_{0}*/".format(t.specialization_name()),
                     func_type = PyrexTypes.CFuncType(
-                        getattr(PyrexTypes,"c_u{0}_type".format(t)), [
-                            PyrexTypes.CFuncTypeArg("arg", getattr(PyrexTypes,"c_{0}_type".format(t)), None)
-                            ],
+                        t,
+                        [PyrexTypes.CFuncTypeArg("arg", t, None)],
                         is_strict_signature = True, nogil=True))
-                            for t in ("int", "long", "longlong")
+                            for t in (PyrexTypes.c_uint_type, PyrexTypes.c_ulong_type, PyrexTypes.c_ulonglong_type)
              ) + list(
         BuiltinFunction('abs',        None,    None,   "__Pyx_c_abs{0}".format(t.funcsuffix),
                     func_type = PyrexTypes.CFuncType(

Cython/Compiler/Code.py

@@ -274,7 +274,7 @@ class UtilityCodeBase(object):
                 elif not values:
                     values = None
                 elif len(values) == 1:
-                    values = values[0]
+                    values = list(values)[0]
                 kwargs[name] = values
 
         if proto is not None:
@@ -1967,9 +1967,12 @@ class CCodeWriter(object):
         if entry.type.is_pyobject:
             self.putln("__Pyx_XDECREF(%s);" % self.entry_as_pyobject(entry))
 
-    def put_var_xdecref(self, entry):
+    def put_var_xdecref(self, entry, nanny=True):
         if entry.type.is_pyobject:
+            if nanny:
                 self.putln("__Pyx_XDECREF(%s);" % self.entry_as_pyobject(entry))
+            else:
+                self.putln("Py_XDECREF(%s);" % self.entry_as_pyobject(entry))
 
     def put_var_decref_clear(self, entry):
         self._put_var_decref_clear(entry, null_check=False)

Cython/Compiler/ExprNodes.py

@@ -5201,6 +5201,7 @@ class SimpleCallNode(CallNode):
     has_optional_args = False
     nogil = False
     analysed = False
+    overflowcheck = False
 
     def compile_time_value(self, denv):
         function = self.function.compile_time_value(denv)
@@ -5510,6 +5511,8 @@ class SimpleCallNode(CallNode):
             if func_type.exception_value is None:
                 env.use_utility_code(UtilityCode.load_cached("CppExceptionConversion", "CppSupport.cpp"))
 
+        self.overflowcheck = env.directives['overflowcheck']
+
     def calculate_result_code(self):
         return self.c_call_code()
 
@@ -5620,7 +5623,12 @@ class SimpleCallNode(CallNode):
                     translate_cpp_exception(code, self.pos, '%s%s;' % (lhs, rhs),
                                             func_type.exception_value, self.nogil)
                 else:
-                    if exc_checks:
+                    if (self.overflowcheck
+                        and self.type.is_int
+                        and self.type.signed
+                        and self.function.result() in ('abs', 'labs', '__Pyx_abs_longlong')):
+                        goto_error = 'if (%s < 0) { PyErr_SetString(PyExc_OverflowError, "value too large"); %s; }' % (self.result(), code.error_goto(self.pos))
+                    elif exc_checks:
                         goto_error = code.error_goto_if(" && ".join(exc_checks), self.pos)
                     else:
                         goto_error = ""

Cython/Compiler/ModuleNode.py

@@ -897,19 +897,60 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
                     [base_class.empty_declaration_code() for base_class in type.base_classes])
                 code.put(" : public %s" % base_class_decl)
             code.putln(" {")
+            py_attrs = [e for e in scope.entries.values()
+                        if e.type.is_pyobject and not e.is_inherited]
             has_virtual_methods = False
-            has_destructor = False
+            constructor = None
+            destructor = None
             for attr in scope.var_entries:
                 if attr.type.is_cfunction and attr.type.is_static_method:
                     code.put("static ")
-                elif attr.type.is_cfunction and attr.name != "<init>":
+                elif attr.name == "<init>":
+                    constructor = attr
+                elif attr.name == "<del>":
+                    destructor = attr
+                elif attr.type.is_cfunction:
                     code.put("virtual ")
                     has_virtual_methods = True
-                if attr.cname[0] == '~':
-                    has_destructor = True
                 code.putln("%s;" % attr.type.declaration_code(attr.cname))
-            if has_virtual_methods and not has_destructor:
-                code.putln("virtual ~%s() { }" % type.cname)
+            if constructor or py_attrs:
+                if constructor:
+                    arg_decls = []
+                    arg_names = []
+                    for arg in constructor.type.args[:len(constructor.type.args)-constructor.type.optional_arg_count]:
+                        arg_decls.append(arg.declaration_code())
+                        arg_names.append(arg.cname)
+                    if constructor.type.optional_arg_count:
+                        arg_decls.append(constructor.type.op_arg_struct.declaration_code(Naming.optional_args_cname))
+                        arg_names.append(Naming.optional_args_cname)
+                    if not arg_decls:
+                        arg_decls = ["void"]
+                else:
+                    arg_decls = ["void"]
+                    arg_names = []
+                code.putln("%s(%s) {" % (type.cname, ", ".join(arg_decls)))
+                if py_attrs:
+                    code.put_ensure_gil()
+                    for attr in py_attrs:
+                        code.put_init_var_to_py_none(attr, nanny=False);
+                if constructor:
+                    code.putln("%s(%s);" % (constructor.cname, ", ".join(arg_names)))
+                if py_attrs:
+                    code.put_release_ensured_gil()
+                code.putln("}")
+            if destructor or py_attrs or has_virtual_methods:
+                if has_virtual_methods:
+                    code.put("virtual ")
+                code.putln("~%s() {" % type.cname)
+                if py_attrs:
+                    code.put_ensure_gil()
+                if destructor:
+                    code.putln("%s();" % destructor.cname)
+                if py_attrs:
+                    for attr in py_attrs:
+                        code.put_var_xdecref(attr, nanny=False);
+                    code.put_release_ensured_gil()
+                code.putln("}")
             code.putln("};")
 
     def generate_enum_definition(self, entry, code):

Cython/Compiler/Symtab.py

@@ -2255,8 +2255,7 @@ class CppClassScope(Scope):
 
     def declare_var(self, name, type, pos,
                     cname = None, visibility = 'extern',
-                    api = 0, in_pxd = 0, is_cdef = 0,
-                    allow_pyobject = 0, defining = 0):
+                    api = 0, in_pxd = 0, is_cdef = 0, defining = 0):
         # Add an entry for an attribute.
         if not cname:
             cname = name
@@ -2275,22 +2274,20 @@ class CppClassScope(Scope):
                 entry.func_cname = "%s::%s" % (self.type.empty_declaration_code(), cname)
         if name != "this" and (defining or name != "<init>"):
             self.var_entries.append(entry)
-        if type.is_pyobject and not allow_pyobject:
-            error(pos,
-                "C++ class member cannot be a Python object")
         return entry
 
     def declare_cfunction(self, name, type, pos,
                           cname=None, visibility='extern', api=0, in_pxd=0,
                           defining=0, modifiers=(), utility_code=None, overridable=False):
-        if name in (self.name.split('::')[-1], '__init__') and cname is None:
-            cname = self.type.cname
+        class_name = self.name.split('::')[-1]
+        if name in (class_name, '__init__') and cname is None:
+            cname = "%s__init__%s" % (Naming.func_prefix, class_name)
             name = '<init>'
-            type.return_type = PyrexTypes.InvisibleVoidType()
+            type.return_type = PyrexTypes.CVoidType()
         elif name == '__dealloc__' and cname is None:
-            cname = "~%s" % self.type.cname
+            cname = "%s__dealloc__%s" % (Naming.func_prefix, class_name)
             name = '<del>'
-            type.return_type = PyrexTypes.InvisibleVoidType()
+            type.return_type = PyrexTypes.CVoidType()
         prev_entry = self.lookup_here(name)
         entry = self.declare_var(name, type, pos,
                                  defining=defining,
@@ -2315,8 +2312,8 @@ class CppClassScope(Scope):
         # to work with this type.
         for base_entry in \
             base_scope.inherited_var_entries + base_scope.var_entries:
-                #contructor is not inherited
-                if base_entry.name == "<init>":
+                #contructor/destructor is not inherited
+                if base_entry.name in ("<init>", "<del>"):
                     continue
                 #print base_entry.name, self.entries
                 if base_entry.name in self.entries:
@@ -2324,6 +2321,7 @@ class CppClassScope(Scope):
                 entry = self.declare(base_entry.name, base_entry.cname,
                     base_entry.type, None, 'extern')
                 entry.is_variable = 1
+                entry.is_inherited = 1
                 self.inherited_var_entries.append(entry)
         for base_entry in base_scope.cfunc_entries:
             entry = self.declare_cfunction(base_entry.name, base_entry.type,

Cython/Compiler/TypeInference.py

@@ -306,6 +306,14 @@ class MarkOverflowingArithmetic(CythonTransform):
         else:
             return self.visit_dangerous_node(node)
 
+    def visit_SimpleCallNode(self, node):
+        if (isinstance(node.function, ExprNodes.NameNode)
+            and node.function.name == 'abs'):
+          # Overflows for minimum value of fixed size ints.
+          return self.visit_dangerous_node(node)
+        else:
+          return self.visit_neutral_node(node)
+
     visit_UnopNode = visit_neutral_node
 
     visit_UnaryMinusNode = visit_dangerous_node

Cython/Utility/Builtins.c

@@ -245,9 +245,7 @@ static CYTHON_INLINE unsigned long __Pyx_abs_long(long x) {
 
 //////////////////// abs_longlong.proto ////////////////////
 
-static CYTHON_INLINE unsigned PY_LONG_LONG __Pyx_abs_longlong(PY_LONG_LONG x) {
-    if (unlikely(x == -PY_LLONG_MAX-1))
-        return ((unsigned PY_LONG_LONG)PY_LLONG_MAX) + 1U;
+static CYTHON_INLINE PY_LONG_LONG __Pyx_abs_longlong(PY_LONG_LONG x) {
 #if defined (__cplusplus) && __cplusplus >= 201103L
     return (unsigned PY_LONG_LONG) std::abs(x);
 #elif defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L

Cython/Utility/ModuleSetupCode.c

@@ -690,6 +690,7 @@ typedef struct {PyObject **p; const char *s; const Py_ssize_t n; const char* enc
                 const char is_unicode; const char is_str; const char intern; } __Pyx_StringTabEntry; /*proto*/
 
 /////////////// ForceInitThreads.proto ///////////////
+//@proto_block: utility_code_proto_before_types
 
 #ifndef __PYX_FORCE_INIT_THREADS
   #define __PYX_FORCE_INIT_THREADS 0
@@ -1081,6 +1082,7 @@ __Pyx_FastGilFuncInit();
 #endif
 
 /////////////// NoFastGil.proto ///////////////
+//@proto_block: utility_code_proto_before_types
 
 #define __Pyx_PyGILState_Ensure PyGILState_Ensure
 #define __Pyx_PyGILState_Release PyGILState_Release
@@ -1089,6 +1091,7 @@ __Pyx_FastGilFuncInit();
 #define __Pyx_FastGilFuncInit()
 
 /////////////// FastGil.proto ///////////////
+//@proto_block: utility_code_proto_before_types
 
 struct __Pyx_FastGilVtab {
   PyGILState_STATE (*Fast_PyGILState_Ensure)(void);

tests/run/builtin_abs.pyx

@@ -31,25 +31,37 @@ def py_abs(a):
     return abs(a)
 
 @cython.test_assert_path_exists("//ReturnStatNode//NameNode[@entry.name = 'abs']",
-                                "//ReturnStatNode//NameNode[@entry.cname = '__Pyx_abs_int']")
+                                "//ReturnStatNode//NameNode[@entry.cname = 'abs']")
+def sub_abs(int a):
+    """
+    >>> sub_abs(5)
+    (-5, 95)
+    >>> sub_abs(105)
+    (-105, -5)
+    """
+    return -abs(a), 100 - abs(a)
+
+@cython.overflowcheck(True)
+@cython.test_assert_path_exists("//ReturnStatNode//NameNode[@entry.name = 'abs']",
+                                "//ReturnStatNode//NameNode[@entry.cname = 'abs']")
 def int_abs(int a):
     """
     >>> int_abs(-5) == 5
     True
     >>> int_abs(-5.1) == 5
     True
-    >>> int_abs(-max_int-1) > 0
-    True
-    >>> int_abs(-max_int-1) == abs(-max_int-1)   or (max_int, int_abs(-max_int-1), abs(-max_int-1))
-    True
+    >>> int_abs(-max_int-1)     #doctest: +ELLIPSIS
+    Traceback (most recent call last):
+        ...
+    OverflowError: ...
     >>> int_abs(max_int) == abs(max_int)         or (max_int, int_abs(max_int), abs(max_int))
     True
     """
     return abs(a)
 
 @cython.test_assert_path_exists("//ReturnStatNode//NameNode[@entry.name = 'abs']")
-@cython.test_fail_if_path_exists("//ReturnStatNode//NameNode[@entry.cname = '__Pyx_abs_int']",
-                                 "//ReturnStatNode//NameNode[@entry.cname = '__Pyx_abs_long']")
+@cython.test_fail_if_path_exists("//ReturnStatNode//NameNode[@entry.cname = 'abs']",
+                                 "//ReturnStatNode//NameNode[@entry.cname = 'labs']")
 def uint_abs(unsigned int a):
     """
     >>> uint_abs(max_int) == abs(max_int)         or (max_int, uint_abs(max_int), abs(max_int))
@@ -57,43 +69,47 @@ def uint_abs(unsigned int a):
     """
     return abs(a)
 
+@cython.overflowcheck(True)
 @cython.test_assert_path_exists("//ReturnStatNode//NameNode[@entry.name = 'abs']",
-                                "//ReturnStatNode//NameNode[@entry.cname = '__Pyx_abs_long']")
+                                "//ReturnStatNode//NameNode[@entry.cname = 'labs']")
 def long_abs(long a):
     """
     >>> long_abs(-5) == 5
     True
     >>> long_abs(-5.1) == 5
     True
-    >>> long_abs(-max_long-1) > 0
-    True
-    >>> long_abs(-max_long-1) == abs(-max_long-1)   or (max_long, long_abs(-max_long-1), abs(-max_long-1))
-    True
+    >>> long_abs(-max_long-1)     #doctest: +ELLIPSIS
+    Traceback (most recent call last):
+        ...
+    OverflowError: ...
     >>> long_abs(max_long) == abs(max_long)         or (max_long, long_abs(max_long), abs(max_long))
     True
     """
     return abs(a)
 
 @cython.test_assert_path_exists("//ReturnStatNode//NameNode[@entry.name = 'abs']")
-@cython.test_fail_if_path_exists("//ReturnStatNode//NameNode[@entry.cname = '__Pyx_abs_int']",
-                                 "//ReturnStatNode//NameNode[@entry.cname = '__Pyx_abs_long']")
+@cython.test_fail_if_path_exists("//ReturnStatNode//NameNode[@entry.cname = 'abs']",
+                                 "//ReturnStatNode//NameNode[@entry.cname = 'labs']")
 def ulong_abs(unsigned long a):
     """
     >>> ulong_abs(max_long) == abs(max_long)         or (max_int, ulong_abs(max_long), abs(max_long))
     True
+    >>> ulong_abs(max_long + 5) == abs(max_long + 5)         or (max_long + 5, ulong_abs(max_long + 5), abs(max_long + 5))
+    True
     """
     return abs(a)
 
+@cython.overflowcheck(True)
 @cython.test_assert_path_exists("//ReturnStatNode//NameNode[@entry.name = 'abs']",
                                 "//ReturnStatNode//NameNode[@entry.cname = '__Pyx_abs_longlong']")
 def long_long_abs(long long a):
     """
     >>> long_long_abs(-(2**33)) == 2**33
     True
-    >>> long_long_abs(-max_long_long-1) > 0
-    True
-    >>> long_long_abs(-max_long_long-1) == abs(-max_long_long-1)  or (max_long_long, long_long_abs(-max_long_long-1), abs(-max_long_long-1))
-    True
+    >>> long_long_abs(-max_long_long-1)     #doctest: +ELLIPSIS
+    Traceback (most recent call last):
+        ...
+    OverflowError: ...
     >>> long_long_abs(max_long_long) == abs(max_long_long)        or (max_long_long, long_long_abs(max_long_long), abs(max_long_long))
     True
     """

tests/run/cpp_classes_def.pyx

@@ -151,3 +151,40 @@ def test_default_init_no_gil():
   with nogil:
     s = new Simple()
     del s
+
+
+cdef class NoisyAlloc(object):
+    cdef public name
+    def __init__(self, name):
+        print "NoisyAlloc.__init__", name
+        self.name = name
+    def __dealloc__(self):
+        try:
+            print "NoisyAlloc.__dealloc__", self.name
+        except:
+            pass  # Suppress unraisable exception warning.
+
+cdef cppclass CppClassWithObjectMember:
+    NoisyAlloc o
+    __init__(name):
+        try:
+            print "CppClassWithObjectMember.__init__", name
+            this.o = NoisyAlloc(name)
+        except:
+            pass  # Suppress unraisable exception warning.
+    __dealloc__():
+        try:
+            print "CppClassWithObjectMember.__dealloc__", this.o.name
+        except:
+            pass  # Suppress unraisable exception warning.
+
+def test_CppClassWithObjectMember(name):
+    """
+    >>> test_CppClassWithObjectMember("gertrude")
+    CppClassWithObjectMember.__init__ gertrude
+    NoisyAlloc.__init__ gertrude
+    CppClassWithObjectMember.__dealloc__ gertrude
+    NoisyAlloc.__dealloc__ gertrude
+    """
+    x = new CppClassWithObjectMember(name)
+    del x

tests/run/type_inference.pyx

@@ -486,6 +486,11 @@ def safe_only():
     for j in range(10):
         res = -j
     assert typeof(j) == "Python object", typeof(j)
+    h = 1
+    res = abs(h)
+    assert typeof(h) == "Python object", typeof(h)
+    cdef int c_int = 1
+    assert typeof(abs(c_int)) == "int", typeof(abs(c_int))
 
 @infer_types(None)
 def safe_c_functions():