Acquisition count arguments, attributes + coerce memslice to objects

Cython/Compiler/ExprNodes.py

@@ -1362,6 +1362,7 @@ class NameNode(AtomicExprNode):
                     node.entry = var_entry
                     node.analyse_rvalue_entry(env)
                     return node
+
         return super(NameNode, self).coerce_to(dst_type, env)
 
     def analyse_as_module(self, env):
@@ -1662,17 +1663,7 @@ class NameNode(AtomicExprNode):
                 rhs.free_temps(code)
         else:
             if self.type.is_memoryviewslice:
-                import MemoryView
-                MemoryView.put_acquire_memoryviewslice(rhs, self.type,
-                        self.entry.is_cglobal, self.result(), self.pos, code)
-
-                if isinstance(rhs, CoerceToMemViewSliceNode):
-                    # We had a new reference, the lhs now has another,
-                    # dispose of ours.
-                    # code.put_xdecref_memoryviewslice(rhs.result())
-                    code.put_decref("%s.memview" % rhs.result(),
-                                    cython_memoryview_ptr_type,
-                                    nanny=False)
+                self.generate_acquire_memoryviewslice(rhs, code)
 
             elif self.type.is_buffer:
                 # Generate code for doing the buffer release/acquisition.
@@ -1716,6 +1707,21 @@ class NameNode(AtomicExprNode):
                 rhs.generate_post_assignment_code(code)
             rhs.free_temps(code)
 
+    def generate_acquire_memoryviewslice(self, rhs, code):
+        """
+        If the value was coerced to a memoryviewslice, its a new reference.
+        Otherwise we're simply using a borrowed reference from another slice
+        """
+        import MemoryView
+
+        MemoryView.put_acquire_memoryviewslice(
+            lhs_cname=self.result(),
+            lhs_type=self.type,
+            lhs_pos=self.pos,
+            rhs=rhs,
+            code=code,
+            incref_rhs=not isinstance(rhs, CoerceToMemViewSliceNode))
+
     def generate_acquire_buffer(self, rhs, code):
         # rhstmp is only used in case the rhs is a complicated expression leading to
         # the object, to avoid repeating the same C expression for every reference
@@ -4009,10 +4015,9 @@ class AttributeNode(ExprNode):
                 code.put_decref(select_code, self.ctype())
             elif self.type.is_memoryviewslice:
                 import MemoryView
-                MemoryView.put_assign_to_memviewslice(select_code, rhs.result(), self.type,
-                        pos=self.pos, code=code)
-                #if rhs.is_temp:
-                #    code.put_xdecref_clear("%s.memview" % rhs.result(), cython_memoryview_ptr_type)
+                MemoryView.put_assign_to_memviewslice(
+                        select_code, rhs.result(), self.type, code)
+
             if not self.type.is_memoryviewslice:
                 code.putln(
                     "%s = %s;" % (
@@ -7863,12 +7868,17 @@ class CoerceToPyTypeNode(CoercionNode):
         pass
 
     def generate_result_code(self, code):
-        function = self.arg.type.to_py_function
-        code.putln('%s = %s(%s); %s' % (
+        if self.arg.type.is_memoryviewslice:
+            funccall = self.arg.type.get_to_py_function(self.arg)
+        else:
+            funccall = "%s(%s)" % (self.arg.type.to_py_function,
+                                   self.arg.result())
+
+        code.putln('%s = %s; %s' % (
             self.result(),
-            function,
-            self.arg.result(),
+            funccall,
             code.error_goto_if_null(self.result(), self.pos)))
+
         code.put_gotref(self.py_result())
 
 

Cython/Compiler/MemoryView.py

 from Errors import CompileError
+import ExprNodes
 from ExprNodes import IntNode, NoneNode, IntBinopNode, NameNode, AttributeNode
 from Visitor import CythonTransform
 import Options
@@ -74,8 +75,8 @@ def mangle_dtype_name(dtype):
 def axes_to_str(axes):
     return "".join([access[0].upper()+packing[0] for (access, packing) in axes])
 
-def put_acquire_memoryviewslice(rhs, lhs_type, lhs_is_cglobal, lhs_result, lhs_pos, code):
-    # import MemoryView
+def put_acquire_memoryviewslice(lhs_cname, lhs_type, lhs_pos, rhs, code,
+                                incref_rhs=True, have_gil=False):
     assert rhs.type.is_memoryviewslice
 
     pretty_rhs = isinstance(rhs, NameNode) or rhs.result_in_temp()
@@ -86,14 +87,15 @@ def put_acquire_memoryviewslice(rhs, lhs_type, lhs_is_cglobal, lhs_result, lhs_p
         code.putln("%s = %s;" % (rhstmp, rhs.result_as(lhs_type)))
 
     code.putln(code.error_goto_if_null("%s.memview" % rhstmp, lhs_pos))
-    put_assign_to_memviewslice(lhs_result, rhstmp, lhs_type,
-                                     lhs_pos, code=code)
+    put_assign_to_memviewslice(lhs_cname, rhstmp, lhs_type, code, incref_rhs)
 
     if not pretty_rhs:
         code.funcstate.release_temp(rhstmp)
 
-def put_assign_to_memviewslice(lhs_cname, rhs_cname, memviewslicetype, pos, code):
+def put_assign_to_memviewslice(lhs_cname, rhs_cname, memviewslicetype, code,
+                               incref_rhs=True):
     code.put_xdecref_memoryviewslice(lhs_cname)
+    if incref_rhs:
         code.put_incref_memoryviewslice(rhs_cname)
 
     code.putln("%s.memview = %s.memview;" % (lhs_cname, rhs_cname))
@@ -196,7 +198,7 @@ class MemoryViewSliceBufferEntry(Buffer.BufferEntry):
 
             if access == 'full' and packing in ('strided', 'follow'):
                 code.globalstate.use_utility_code(memviewslice_index_helpers)
-                bufp = ('__pyx_memviewslice_index_full(%s, %s, %s)' %
+                bufp = ('__pyx_memviewslice_index_full(%s, %s, %s, %s)' %
                                             (bufp, index, stride, suboffset))
 
             elif access == 'full' and packing == 'contig':
@@ -722,47 +724,6 @@ def _resolve_AttributeNode(env, node):
         scope = scope.lookup(modname).as_module
     return scope.lookup(path[-1])
 
-class MemoryViewSliceTransform(CythonTransform):
-
-    memviews_exist = False
-
-    def __call__(self, node):
-        return super(MemoryViewSliceTransform, self).__call__(node)
-
-    def inspect_scope(self, node, scope):
-
-        memviewvars = [entry for name, entry
-                in scope.entries.iteritems()
-                if entry.type.is_memoryviewslice]
-        if memviewvars:
-            self.memviews_exist = True
-
-    def visit_FuncDefNode(self, node):
-        # check for the existence of memview entries here.
-        self.inspect_scope(node, node.local_scope)
-        self.visitchildren(node)
-        return node
-
-    def visit_ModuleNode(self, node):
-        # check for memviews here.
-        self.inspect_scope(node, node.scope)
-        self.visitchildren(node)
-        return node
-
-    def visit_ClassDefNode(self, node):
-        # check for memviews in the class scope
-        if hasattr(node, 'scope'):
-            scope = node.scope
-        else:
-            scope = node.entry.type.scope
-        self.inspect_scope(node, scope)
-        self.visitchildren(node)
-        return node
-
-    def visit_SingleAssignmentNode(self, node):
-        return node
-
-
 def load_memview_cy_utility(util_code_name, context=None, **kwargs):
     return CythonUtilityCode.load(util_code_name, "MemoryView.pyx",
                                   context=context, **kwargs)

Cython/Compiler/ModuleNode.py

@@ -1251,8 +1251,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
                 code.put_init_var_to_py_none(entry, "p->%s", nanny=False)
         for entry in memviewslice_attrs:
             code.putln("p->%s.data = NULL;" % entry.cname)
-            code.put_init_to_py_none("p->%s.memview" % entry.cname,
-                    PyrexTypes.cython_memoryview_ptr_type, nanny=False)
+            code.putln("p->%s.memview = NULL;" % entry.cname)
         entry = scope.lookup_here("__new__")
         if entry and entry.is_special:
             if entry.trivial_signature:

Cython/Compiler/Nodes.py

@@ -1432,9 +1432,11 @@ class FuncDefNode(StatNode, BlockNode):
             if entry.type.is_pyobject:
                 if (acquire_gil or entry.assignments) and not entry.in_closure:
                     code.put_var_incref(entry)
-            if entry.type.is_memoryviewslice:
-                code.put_incref_memoryviewslice(entry.cname,
-                                                have_gil=not lenv.nogil)
+            # Note: all memoryviewslices are already newly acquired references
+            #       or increffed defaults!
+            #if entry.type.is_memoryviewslice:
+            #    code.put_incref_memoryviewslice(entry.cname,
+            #                                    have_gil=not lenv.nogil)
                 #code.put_incref("%s.memview" % entry.cname, cython_memoryview_ptr_type)
         # ----- Initialise local buffer auxiliary variables
         for entry in lenv.var_entries + lenv.arg_entries:
@@ -1482,6 +1484,8 @@ class FuncDefNode(StatNode, BlockNode):
             # Clean up buffers -- this calls a Python function
             # so need to save and restore error state
             buffers_present = len(lenv.buffer_entries) > 0
+            memslice_entries = [e for e in lenv.entries.itervalues()
+                                      if e.type.is_memoryviewslice]
             if buffers_present:
                 code.globalstate.use_utility_code(restore_exception_utility_code)
                 code.putln("{ PyObject *__pyx_type, *__pyx_value, *__pyx_tb;")
@@ -1489,6 +1493,8 @@ class FuncDefNode(StatNode, BlockNode):
                 for entry in lenv.buffer_entries:
                     Buffer.put_release_buffer_code(code, entry)
                     #code.putln("%s = 0;" % entry.cname)
+                #for entry in memslice_entries:
+                #    code.put_xdecref_memoryviewslice(entry.cname)
                 code.putln("__Pyx_ErrRestore(__pyx_type, __pyx_value, __pyx_tb);}")
 
             err_val = self.error_value()
@@ -1548,10 +1554,12 @@ class FuncDefNode(StatNode, BlockNode):
         for entry in lenv.var_entries:
             if not entry.used or entry.in_closure:
                 continue
+
             if entry.type.is_memoryviewslice:
                 #code.put_xdecref("%s.memview" % entry.cname, cython_memoryview_ptr_type)
                 code.put_xdecref_memoryviewslice(entry.cname)
-            if entry.type.is_pyobject:
+
+            elif entry.type.is_pyobject:
                 code.put_var_decref(entry)
 
         # Decref any increfed args
@@ -1559,7 +1567,7 @@ class FuncDefNode(StatNode, BlockNode):
             if entry.type.is_pyobject:
                 if (acquire_gil or entry.assignments) and not entry.in_closure:
                     code.put_var_decref(entry)
-            if entry.type.is_memoryviewslice:
+            if entry.type.is_memoryviewslice and isinstance(self, DefNode):
                 code.put_xdecref_memoryviewslice(entry.cname)
                 #code.put_decref("%s.memview" % entry.cname, cython_memoryview_ptr_type)
         if self.needs_closure:
@@ -1574,9 +1582,6 @@ class FuncDefNode(StatNode, BlockNode):
                 err_val = default_retval
             if self.return_type.is_pyobject:
                 code.put_xgiveref(self.return_type.as_pyobject(Naming.retval_cname))
-            #elif self.return_type.is_memoryviewslice:
-            #    code.put_xgiveref(code.as_pyobject("%s.memview" % Naming.retval_cname,cython_memoryview_ptr_type))
-            #    code.put_xgiveref_memoryviewslice(Naming.retval_cname)
 
         if self.entry.is_special and self.entry.name == "__hash__":
             # Returning -1 for __hash__ is supposed to signal an error
@@ -2805,6 +2810,9 @@ class DefNode(FuncDefNode):
                     "%s = %s;" % (
                         arg.entry.cname,
                         arg.calculate_default_value_code(code)))
+                if arg.type.is_memoryviewslice:
+                    code.put_incref_memoryviewslice(arg.entry.cname,
+                                                    have_gil=True)
 
     def generate_stararg_init_code(self, max_positional_args, code):
         if self.starstar_arg:
@@ -2983,6 +2991,22 @@ class DefNode(FuncDefNode):
                 code.error_goto(self.pos)))
         code.putln('}')
 
+        # convert arg values to their final type and assign them
+        for i, arg in enumerate(all_args):
+            if arg.default and not arg.type.is_pyobject:
+                code.putln("if (values[%d]) {" % i)
+            self.generate_arg_assignment(arg, "values[%d]" % i, code)
+            if arg.default and not arg.type.is_pyobject:
+                code.putln('} else {')
+                code.putln(
+                    "%s = %s;" % (
+                        arg.entry.cname,
+                        arg.calculate_default_value_code(code)))
+                if arg.type.is_memoryviewslice:
+                    code.put_incref_memoryviewslice(arg.entry.cname,
+                                                    have_gil=True)
+                code.putln('}')
+
     def generate_argument_conversion_code(self, code):
         # Generate code to convert arguments from signature type to
         # declared type, if needed.  Also copies signature arguments
@@ -4291,8 +4315,12 @@ class ReturnStatNode(StatNode):
             self.value.generate_evaluation_code(code)
             if self.return_type.is_memoryviewslice:
                 import MemoryView
-                MemoryView.put_acquire_memoryviewslice(self.value, self.return_type,
-                        False, Naming.retval_cname, None, code)
+                MemoryView.put_acquire_memoryviewslice(
+                        lhs_cname=Naming.retval_cname,
+                        lhs_type=self.return_type,
+                        lhs_pos=self.value.pos,
+                        rhs=self.value,
+                        code=code)
             else:
                 self.value.make_owned_reference(code)
                 code.putln(

Cython/Compiler/PyrexTypes.py

@@ -376,12 +376,13 @@ class MemoryViewSliceType(PyrexType):
         the *first* axis' packing spec and 'follow' for all other packing
         specs.
         '''
+        import MemoryView
 
         self.dtype = base_dtype
         self.axes = axes
         self.ndim = len(axes)
+        self.flags = MemoryView.get_buf_flag(self.axes)
 
-        import MemoryView
         self.is_c_contig, self.is_f_contig = MemoryView.is_cf_contig(self.axes)
         assert not (self.is_c_contig and self.is_f_contig)
 
@@ -540,7 +541,7 @@ class MemoryViewSliceType(PyrexType):
 
         context = dict(
             MemoryView.context,
-            buf_flag = MemoryView.get_buf_flag(self.axes),
+            buf_flag = self.flags,
             ndim = self.ndim,
             axes_specs = ', '.join(self.axes_specs_to_code()),
             dtype_typedecl = self.dtype.declaration_code(""),
@@ -554,6 +555,13 @@ class MemoryViewSliceType(PyrexType):
 
         return True
 
+    def create_to_py_utility_code(self, env):
+        return True
+
+    def get_to_py_function(self, obj):
+        return "__pyx_memoryview_fromslice(&%s, %s.memview->obj, %s, %s);" % (
+                                obj.result(), obj.result(), self.flags, self.ndim)
+
     def axes_specs_to_code(self):
         "Return a list of code constants for each axis"
         import MemoryView

Cython/Utility/MemoryView.pyx

@@ -166,7 +166,7 @@ cdef extern from "pythread.h":
     void PyThread_release_lock(PyThread_type_lock) nogil
 
 cdef extern from *:
-    int __Pyx_GetBuffer(object, Py_buffer *, int)
+    int __Pyx_GetBuffer(object, Py_buffer *, int) except -1
     void __Pyx_ReleaseBuffer(Py_buffer *)
 
     ctypedef struct {{memviewslice_name}}:
@@ -175,6 +175,9 @@ cdef extern from *:
         Py_ssize_t strides[{{max_dims}}]
         Py_ssize_t suboffsets[{{max_dims}}]
 
+    void puts(char *)
+    void printf(char *, ...)
+
 
 @cname('__pyx_MemviewEnum')
 cdef class Enum(object):
@@ -197,10 +200,9 @@ cdef indirect_contiguous = Enum("<contiguous and indirect>")
 cdef class memoryview(object):
 
     cdef object obj
-    cdef Py_buffer view
     cdef PyThread_type_lock lock
     cdef int acquisition_count
-
+    cdef Py_buffer view
 
     def __cinit__(memoryview self, object obj, int flags):
         self.obj = obj
@@ -212,6 +214,7 @@ cdef class memoryview(object):
 
     def __dealloc__(memoryview self):
         __Pyx_ReleaseBuffer(&self.view)
+        if self.lock != NULL:
             PyThread_free_lock(self.lock)
 
     @cname('__pyx_memoryview_getitem')
@@ -255,6 +258,9 @@ cdef class memoryview(object):
         bytesitem = itemp[:self.view.itemsize]
         return struct.unpack(fmt, bytesitem)
 
+    def __str__(self):
+        return "<MemoryView of %r at 0x%x>" % (self.obj, id(self))
+
 
 @cname('__pyx_memoryviewslice')
 cdef class _memoryviewslice(memoryview):
@@ -277,9 +283,9 @@ cdef memoryview_cwrapper(object o, int flags):
     return memoryview(o, flags)
 
 @cname('__pyx_memoryview_fromslice')
-cdef memoryview memoryview_from_memview_cwrapper(memoryview m,  int flags,
-                        int new_ndim, {{memviewslice_name}} *memviewslice):
-    cdef _memoryviewslice result = _memoryviewslice(m.obj, flags)
+cdef memoryview memoryview_from_memview_cwrapper(
+        {{memviewslice_name}} *memviewslice, object orig_obj, int flags, int new_ndim):
+    cdef _memoryviewslice result = _memoryviewslice(orig_obj, flags)
 
     result.from_slice = memviewslice[0]
 

Cython/Utility/MemoryView_C.c

@@ -76,7 +76,6 @@ static CYTHON_INLINE {{memviewslice_name}} {{funcname}}(PyObject *obj) {
     if (unlikely(retcode == -1))
         goto __pyx_fail;
 
-    memview->acquisition_count = 1;
     return result;
 __pyx_fail:
     Py_XDECREF(memview);
@@ -165,7 +164,7 @@ static int __Pyx_ValidateAndInit_memviewslice(
             }
         }
 
-        if (spec & (__Pyx_MEMVIEW_PTR | __Pyx_MEMVIEW_FULL)) {
+        if (spec & __Pyx_MEMVIEW_PTR) {
             if (!buf->suboffsets) {
                 PyErr_SetString(PyExc_ValueError,
                     "Buffer not able to be indirectly accessed.");
@@ -252,6 +251,7 @@ static int __Pyx_init_memviewslice(
 
     memviewslice->memview = memview;
     memviewslice->data = (char *)buf->buf;
+    memview->acquisition_count++;
     retval = 0;
     goto no_fail;
 
@@ -265,23 +265,40 @@ no_fail:
     return retval;
 }
 
+
+static CYTHON_INLINE void __pyx_fatalerror(char *fmt, ...) {
+    va_list vargs;
+    char msg[200];
+
+    va_start(vargs, fmt);
+
+#ifdef HAVE_STDARG_PROTOTYPES
+    va_start(vargs, fmt);
+#else
+    va_start(vargs);
+#endif
+
+    vsnprintf(msg, 200, fmt, vargs);
+    Py_FatalError(msg);
+
+    va_end(vargs);
+}
+
 static CYTHON_INLINE void __Pyx_INC_MEMVIEW({{memviewslice_name}} *memslice,
                                             int have_gil, int lineno) {
     int first_time;
     struct {{memview_struct_name}} *memview = memslice->memview;
-    if (!memview) {
-        char msg[50];
-        snprintf(msg, 50, "memoryslice is not initialized (line %d)", lineno);
-        Py_FatalError(msg);
-    }
+    if (!memview)
+        __pyx_fatalerror("memoryslice is not initialized (line %d)", lineno);
+
+    if (memview->acquisition_count <= 0)
+        __pyx_fatalerror("Acquisition count is %d (line %d)",
+                         memview->acquisition_count, lineno);
 
     PyThread_acquire_lock(memview->lock, 1);
     first_time = (memview->acquisition_count++ == 0);
     PyThread_release_lock(memview->lock);
 
-    /* printf("INCREF %d: acquisition_count=%d, refcount=%d\n", lineno,
-           memview->acquisition_count, memview->ob_refcnt); */
-
     if (first_time) {
         if (have_gil) {
             Py_INCREF((PyObject *) memview);
@@ -298,26 +315,26 @@ static CYTHON_INLINE void __Pyx_XDEC_MEMVIEW({{memviewslice_name}} *memslice,
     int last_time;
     struct {{memview_struct_name}} *memview = memslice->memview;
 
-    if (!memview) {
+    if (!memview)
         return;
-    }
+
+    if (memview->acquisition_count <= 0)
+        __pyx_fatalerror("Acquisition count is %d (line %d)",
+                         memview->acquisition_count, lineno);
 
     PyThread_acquire_lock(memview->lock, 1);
     last_time = (memview->acquisition_count-- == 1);
     PyThread_release_lock(memview->lock);
 
-    /* printf("DECREF %d: acquisition_count=%d, refcount=%d\n", lineno,
-           memview->acquisition_count, memview->ob_refcnt); */
-
     if (last_time) {
         if (have_gil) {
-            Py_CLEAR(memview);
+            Py_CLEAR(memslice->memview);
         } else {
             PyGILState_STATE _gilstate = PyGILState_Ensure();
-            Py_CLEAR(memview);
+            Py_CLEAR(memslice->memview);
             PyGILState_Release(_gilstate);
         }
-        memslice->data = NULL;
+        memslice->memview = NULL;
     }
 }
 

tests/run/memslice.pyx

@@ -48,13 +48,13 @@ def acquire_release(o1, o2):
     >>> B = IntMockBuffer("B", range(6))
     >>> acquire_release(A, B)
     acquired A
-    released A
     acquired B
+    released A
     released B
-    >>> acquire_release(None, None)
     >>> acquire_release(None, B)
-    acquired B
-    released B
+    Traceback (most recent call last):
+       ...
+    TypeError: 'NoneType' does not have the buffer interface
     """
     cdef int[:] buf
     buf = o1
@@ -122,8 +122,6 @@ def acquire_failure3():
     >>> acquire_failure3()
     acquired working
     0 3
-    released working
-    acquired working
     0 3
     released working
     """
@@ -149,6 +147,10 @@ def acquire_nonbuffer1(first, second=None):
       ...
     TypeError: 'type' does not have the buffer interface
     >>> acquire_nonbuffer1(None, 2)
+    Traceback (most recent call last):
+      ...
+    TypeError: 'NoneType' does not have the buffer interface
+    >>> acquire_nonbuffer1(4, object())
     Traceback (most recent call last):
       ...
     TypeError: 'int' does not have the buffer interface
@@ -163,8 +165,6 @@ def acquire_nonbuffer2():
     >>> acquire_nonbuffer2()
     acquired working
     0 3
-    released working
-    acquired working
     0 3
     released working
     """
@@ -195,9 +195,7 @@ def as_argument(int[:] bufarg, int n):
 def as_argument_defval(int[:] bufarg=IntMockBuffer('default', range(6)), int n=6):
     """
     >>> as_argument_defval()
-    acquired default
     0 1 2 3 4 5 END
-    released default
     >>> A = IntMockBuffer("A", range(6))
     >>> as_argument_defval(A, 6)
     acquired A
@@ -233,14 +231,14 @@ def forin_assignment(objs, int pick):
     >>> forin_assignment([A, B, A, A], 2)
     acquired A
     2
-    released A
     acquired B
+    released A
     2
-    released B
     acquired A
+    released B
     2
-    released A
     acquired A
+    released A
     2
     released A
     """
@@ -532,7 +530,7 @@ def c_contig_2d(int[:, ::1] buf):
 @testcase
 def f_contig(int[::1, :] buf):
     """
-    >>> A = IntMockBuffer(None, range(4), shape=(2, 2))
+    >>> A = IntMockBuffer(None, range(4), shape=(2, 2), strides=(1, 2))
     >>> f_contig(A)
     2
     >>> [str(x) for x in A.recieved_flags]
@@ -688,7 +686,7 @@ def printbuf_int_2d(o, shape):
     released A
     """
     # should make shape builtin
-    cdef int[:, :] buf
+    cdef int[::view.generic, ::view.generic] buf
     buf = o
     cdef int i, j
     for i in range(shape[0]):