implement coercion between C array and list/tuple

--HG--
rename : Cython/Utility/CFuncConvert.pyx => Cython/Utility/CConvert.pyx

Cython/Compiler/ExprNodes.py

@@ -3895,10 +3895,17 @@ class SliceIndexNode(ExprNode):
         elif base_type.is_ptr:
             self.type = base_type
         elif base_type.is_array:
-            # we need a ptr type here instead of an array type, as
-            # array types can result in invalid type casts in the C
-            # code
-            self.type = PyrexTypes.CPtrType(base_type.base_type)
+            if getting:
+                # we need a ptr type here instead of an array type, as
+                # array types can result in invalid type casts in the C
+                # code
+                self.type = PyrexTypes.CPtrType(base_type.base_type)
+            else:
+                # try to assign 'by value' (i.e. make a copy)
+                if not self.start and not self.stop:
+                    self.type = base_type
+                else:
+                    self.type = PyrexTypes.CPtrType(base_type.base_type)
         else:
             self.base = self.base.coerce_to_pyobject(env)
             self.type = py_object_type
@@ -6528,7 +6535,7 @@ class ListNode(SequenceNode):
                 error(self.pos, "Cannot coerce list to type '%s'" % dst_type)
         elif self.mult_factor:
             error(self.pos, "Cannot coerce multiplied list to '%s'" % dst_type)
-        elif dst_type.is_ptr and dst_type.base_type is not PyrexTypes.c_void_type:
+        elif (dst_type.is_array or dst_type.is_ptr) and dst_type.base_type is not PyrexTypes.c_void_type:
             base_type = dst_type.base_type
             self.type = PyrexTypes.CArrayType(base_type, len(self.args))
             for i in range(len(self.original_args)):
@@ -11110,6 +11117,7 @@ class CoerceToPyTypeNode(CoercionNode):
     #  to a Python object.
 
     type = py_object_type
+    target_type = py_object_type
     is_temp = 1
 
     def __init__(self, arg, env, type=py_object_type):
@@ -11129,22 +11137,17 @@ class CoerceToPyTypeNode(CoercionNode):
                 self.type = unicode_type
             elif arg.type.is_complex:
                 self.type = Builtin.complex_type
+            self.target_type = self.type
         elif arg.type.is_string or arg.type.is_cpp_string:
             if (type not in (bytes_type, bytearray_type)
                     and not env.directives['c_string_encoding']):
                 error(arg.pos,
                     "default encoding required for conversion from '%s' to '%s'" %
                     (arg.type, type))
-            self.type = type
+            self.type = self.target_type = type
         else:
             # FIXME: check that the target type and the resulting type are compatible
-            pass
-
-        if arg.type.is_memoryviewslice:
-            # Register utility codes at this point
-            arg.type.get_to_py_function(env, arg)
-
-        self.env = env
+            self.target_type = type
 
     gil_message = "Converting to Python object"
 
@@ -11172,21 +11175,11 @@ class CoerceToPyTypeNode(CoercionNode):
         return self
 
     def generate_result_code(self, code):
-        arg_type = self.arg.type
-        if arg_type.is_memoryviewslice:
-            funccall = arg_type.get_to_py_function(self.env, self.arg)
-        else:
-            func = arg_type.to_py_function
-            if arg_type.is_string or arg_type.is_cpp_string:
-                if self.type in (bytes_type, str_type, unicode_type):
-                    func = func.replace("Object", self.type.name.title(), 1)
-                elif self.type is bytearray_type:
-                    func = func.replace("Object", "ByteArray", 1)
-            funccall = "%s(%s)" % (func, self.arg.result() or 'NULL')
-
-        code.putln('%s = %s; %s' % (
-            self.result(),
-            funccall,
+        code.putln('%s; %s' % (
+            self.arg.type.to_py_call_code(
+                self.arg.result(),
+                self.result(),
+                self.target_type),
             code.error_goto_if_null(self.result(), self.pos)))
 
         code.put_gotref(self.py_result())
@@ -11257,15 +11250,8 @@ class CoerceFromPyTypeNode(CoercionNode):
         return self.type.is_ptr and self.arg.is_ephemeral()
 
     def generate_result_code(self, code):
-        function = self.type.from_py_function
-        operand = self.arg.py_result()
-        rhs = "%s(%s)" % (function, operand)
-        if self.type.is_enum:
-            rhs = typecast(self.type, c_long_type, rhs)
-        code.putln('%s = %s; %s' % (
-            self.result(),
-            rhs,
-            code.error_goto_if(self.type.error_condition(self.result()), self.pos)))
+        code.putln(self.type.from_py_call_code(
+            self.arg.py_result(), self.result(), self.pos, code))
         if self.type.is_pyobject:
             code.put_gotref(self.py_result())
 

Cython/Utility/CConvert.pyx

+#################### FromPyStructUtility ####################
+
+cdef extern from *:
+    ctypedef struct PyTypeObject:
+        char* tp_name
+    PyTypeObject *Py_TYPE(obj)
+    bint PyMapping_Check(obj)
+    object PyErr_Format(exc, const char *format, ...)
+
+@cname("{{funcname}}")
+cdef {{struct_name}} {{funcname}}(obj) except *:
+    cdef {{struct_name}} result
+    if not PyMapping_Check(obj):
+        PyErr_Format(TypeError, b"Expected %.16s, got %.200s", b"a mapping", Py_TYPE(obj).tp_name)
+
+    {{for member in var_entries:}}
+    try:
+        value = obj['{{member.name}}']
+    except KeyError:
+        raise ValueError("No value specified for struct attribute '{{member.name}}'")
+    result.{{member.cname}}{{'[:]' if member.type.is_array else ''}} = value
+    {{endfor}}
+    return result
+
+
+#################### cfunc.to_py ####################
+
+@cname("{{cname}}")
+cdef object {{cname}}({{return_type.ctype}} (*f)({{ ', '.join(arg.type_cname for arg in args) }}) {{except_clause}}):
+    def wrap({{ ', '.join('{arg.ctype} {arg.name}'.format(arg=arg) for arg in args) }}):
+        """wrap({{', '.join(('{arg.name}: {arg.type_displayname}'.format(arg=arg) if arg.type_displayname else arg.name) for arg in args)}}){{if return_type.type_displayname}} -> {{return_type.type_displayname}}{{endif}}"""
+        {{'' if return_type.type.is_void else 'return '}}f({{ ', '.join(arg.name for arg in args) }})
+    return wrap
+
+
+#################### carray.from_py ####################
+
+cdef extern from *:
+    object PyErr_Format(exc, const char *format, ...)
+
+@cname("{{cname}}")
+cdef int {{cname}}(object o, {{base_type}} *v, Py_ssize_t length) except -1:
+    cdef Py_ssize_t i = length
+    try:
+        i = len(o)
+    except (TypeError, OverflowError):
+        pass
+    if i == length:
+        for i, item in enumerate(o):
+            if i >= length:
+                break
+            v[i] = item
+        else:
+            i += 1  # convert index to length
+            if i == length:
+                return 0
+
+    PyErr_Format(
+        IndexError,
+        ("too many values found during array assignment, expected %zd"
+         if i >= length else
+         "not enough values found during array assignment, expected %zd, got %zd"),
+        length, i)
+
+
+#################### carray.to_py ####################
+
+cdef extern from *:
+    ctypedef struct PyObject
+    PyObject* {{to_py_func}}({{base_type}}) except NULL
+
+    tuple PyTuple_New(Py_ssize_t size)
+    void PyTuple_SET_ITEM(object  p, Py_ssize_t pos, PyObject*  o)
+
+    list PyList_New(Py_ssize_t size)
+    void PyList_SET_ITEM(object  p, Py_ssize_t pos, PyObject*  o)
+
+
+@cname("{{cname}}")
+cdef inline list {{cname}}({{base_type}} *v, Py_ssize_t length):
+    cdef size_t i
+    l = PyList_New(length)
+    for i in range(<size_t>length):
+        PyList_SET_ITEM(l, i, {{to_py_func}}(v[i]))
+    return l
+
+
+@cname("{{to_tuple_cname}}")
+cdef inline tuple {{to_tuple_cname}}({{base_type}} *v, Py_ssize_t length):
+    cdef size_t i
+    t = PyTuple_New(length)
+    for i in range(<size_t>length):
+        PyTuple_SET_ITEM(t, i, {{to_py_func}}(v[i]))
+    return t

Cython/Utility/CFuncConvert.pyx

-#################### cfunc.to_py ####################
-
-@cname("{{cname}}")
-cdef object {{cname}}({{return_type.ctype}} (*f)({{ ', '.join(arg.type_cname for arg in args) }}) {{except_clause}}):
-    def wrap({{ ', '.join('{arg.ctype} {arg.name}'.format(arg=arg) for arg in args) }}):
-        """wrap({{', '.join(('{arg.name}: {arg.type_displayname}'.format(arg=arg) if arg.type_displayname else arg.name) for arg in args)}}){{if return_type.type_displayname}} -> {{return_type.type_displayname}}{{endif}}"""
-        {{'' if return_type.type.is_void else 'return '}}f({{ ', '.join(arg.name for arg in args) }})
-    return wrap

Cython/Utility/TypeConversion.c

@@ -311,42 +311,6 @@ static CYTHON_INLINE PyObject * __Pyx_PyInt_FromSize_t(size_t ival) {
 }
 
 
-/////////////// FromPyStructUtility.proto ///////////////
-{{struct_type_decl}};
-static {{struct_type_decl}} {{funcname}}(PyObject *);
-
-/////////////// FromPyStructUtility ///////////////
-static {{struct_type_decl}} {{funcname}}(PyObject * o) {
-    {{struct_type_decl}} result;
-    PyObject *value = NULL;
-
-    if (!PyMapping_Check(o)) {
-        PyErr_Format(PyExc_TypeError, "Expected %.16s, got %.200s", "a mapping", Py_TYPE(o)->tp_name);
-        goto bad;
-    }
-
-    {{for member in var_entries:}}
-        {{py:attr = "result." + member.cname}}
-
-        value = PyObject_GetItem(o, PYIDENT("{{member.name}}"));
-        if (!value) {
-            PyErr_Format(PyExc_ValueError, \
-                "No value specified for struct attribute '%.{{max(200, len(member.name))}}s'", "{{member.name}}");
-            goto bad;
-        }
-        {{attr}} = {{member.type.from_py_function}}(value);
-        if ({{member.type.error_condition(attr)}})
-            goto bad;
-
-        Py_DECREF(value);
-    {{endfor}}
-
-    return result;
-bad:
-    Py_XDECREF(value);
-    return result;
-}
-
 /////////////// ObjectAsUCS4.proto ///////////////
 
 static CYTHON_INLINE Py_UCS4 __Pyx_PyObject_AsPy_UCS4(PyObject*);

tests/memoryview/memslice.pyx

@@ -1935,9 +1935,9 @@ def test_borrowed_slice():
     5
     5
     """
-    cdef int i, carray[10]
-    for i in range(10):
-        carray[i] = i
+    cdef int i
+    cdef int[10] carray
+    carray[:] = range(10)
     _borrowed(carray)
     _not_borrowed(carray)
     _not_borrowed2(carray)

tests/run/carray_coercion.pyx

+
+def from_int_array():
+    """
+    >>> from_int_array()
+    [1, 2, 3]
+    """
+    cdef int[3] v
+    v[0] = 1
+    v[1] = 2
+    v[2] = 3
+    return v
+
+
+cpdef tuple tuple_from_int_array():
+    """
+    >>> tuple_from_int_array()
+    (1, 2, 3)
+    """
+    cdef int[3] v
+    v[0] = 1
+    v[1] = 2
+    v[2] = 3
+    assert isinstance(<tuple>v, tuple)
+    return v
+
+
+cdef extern from "stdint.h":
+    ctypedef unsigned long uint32_t
+
+
+def from_typedef_int_array():
+    """
+    >>> from_typedef_int_array()
+    [1, 2, 3]
+    """
+    cdef uint32_t[3] v
+    v[0] = 1
+    v[1] = 2
+    v[2] = 3
+    return v
+
+
+cpdef tuple tuple_from_typedef_int_array():
+    """
+    >>> tuple_from_typedef_int_array()
+    (1, 2, 3)
+    """
+    cdef uint32_t[3] v
+    v[0] = 1
+    v[1] = 2
+    v[2] = 3
+    return v
+
+
+ctypedef struct MyStructType:
+    int x
+    double y
+
+
+cdef struct MyStruct:
+    int x
+    double y
+
+
+def from_struct_array():
+    """
+    >>> a, b = from_struct_array()
+    >>> a['x'], a['y']
+    (1, 2.0)
+    >>> b['x'], b['y']
+    (3, 4.0)
+    """
+    cdef MyStructType[2] v
+    cdef MyStruct[2] w
+    v[0] = MyStructType(1, 2)
+    v[1] = MyStructType(3, 4)
+    assert isinstance(<tuple>v, tuple)
+    assert isinstance(v, list)
+
+    w[0] = MyStruct(1, 2)
+    w[1] = MyStruct(3, 4)
+    assert (<object>w) == v
+    assert w == (<object>v)
+
+    return v
+
+
+def to_int_array(x):
+    """
+    >>> to_int_array([1, 2, 3])
+    (1, 2, 3)
+    >>> to_int_array([1, 2])
+    Traceback (most recent call last):
+    IndexError: not enough values found during array assignment, expected 3, got 2
+    >>> to_int_array([1, 2, 3, 4])
+    Traceback (most recent call last):
+    IndexError: too many values found during array assignment, expected 3
+    """
+    cdef int[3] v
+    v[:] = x[:3]
+    assert v[0] == x[0]
+    assert v[1] == x[1]
+    assert v[2] == x[2]
+    v[:3] = [0, 0, 0]
+    assert v[0] == 0
+    assert v[1] == 0
+    assert v[2] == 0
+    v[:] = x
+    return v[0], v[1], v[2]
+
+
+def iterable_to_int_array(x):
+    """
+    >>> iterable_to_int_array(iter([1, 2, 3]))
+    (1, 2, 3)
+    >>> iterable_to_int_array(iter([1, 2]))
+    Traceback (most recent call last):
+    IndexError: not enough values found during array assignment, expected 3, got 2
+    >>> iterable_to_int_array(iter([1, 2, 3, 4]))
+    Traceback (most recent call last):
+    IndexError: too many values found during array assignment, expected 3
+    """
+    cdef int[3] v
+    v[:] = x
+    return v[0], v[1], v[2]
+
+
+def to_struct_array(x):
+    """
+    >>> a, b = to_struct_array(({'x': 1, 'y': 2}, {'x': 3, 'y': 4}))
+    >>> a['x'], a['y']
+    (1, 2.0)
+    >>> b['x'], b['y']
+    (3, 4.0)
+    """
+    cdef MyStructType[2] v
+    v[:] = x
+
+    cdef MyStruct[2] w
+    w[:] = x
+
+    assert w[0].x == v[0].x
+    assert w[0].y == v[0].y
+    assert w[1].x == v[1].x
+    assert w[1].y == v[1].y
+
+    return v[0], w[1]