parsing memory view array declarations.

Cython/Compiler/MemoryView.py

+from Errors import CompileError
+from ExprNodes import IntNode, NoneNode, IntBinopNode, NameNode, AttributeNode
+
+START_ERR = "there must be nothing or the value 0 (zero) in the start slot."
+STOP_ERR = "Axis specification only allowed in the 'stop' slot."
+STEP_ERR = "Only the value 1 (one) or valid axis specification allowed in the step slot."
+ONE_ERR = "The value 1 (one) may appear in the first or last axis specification only."
+BOTH_CF_ERR = "Cannot specify an array that is both C and Fortran contiguous."
+NOT_AMP_ERR = "Invalid operator, only an ampersand '&' is allowed."
+INVALID_ERR = "Invalid axis specification."
+EXPR_ERR = "no expressions allowed in axis spec, only names (e.g. cython.view.contig)."
+CF_ERR = "Invalid axis specification for a C/Fortran contiguous array."
+
+def get_axes_specs(env, axes):
+    '''
+    get_axes_specs(env, axes) -> list of (access, packing) specs for each axis.
+
+    access is one of 'full', 'ptr' or 'direct'
+    packing is one of 'contig', 'strided' or 'follow'
+    '''
+
+    cythonscope = env.context.cython_scope
+    viewscope = cythonscope.viewscope
+
+    access_specs = tuple([viewscope.lookup(name)
+                    for name in ('full', 'direct', 'ptr')])
+    packing_specs = tuple([viewscope.lookup(name)
+                    for name in ('contig', 'strided', 'follow')])
+
+    is_f_contig, is_c_contig = False, False
+    default_access, default_packing = 'direct', 'strided'
+    cf_access, cf_packing = default_access, 'follow'
+
+    # set the is_{c,f}_contig flag.
+    for idx, axis in ((0,axes[0]), (-1,axes[-1])):
+        if isinstance(axis.step, IntNode):
+            if axis.step.compile_time_value(env) != 1:
+                raise CompileError(axis.step.pos, STEP_ERR)
+            if len(axes) > 1 and (is_c_contig or is_f_contig):
+                raise CompileError(axis.step.pos, BOTH_CF_ERR)
+            if not idx:
+                is_f_contig = True
+            else:
+                is_c_contig = True
+            if len(axes) == 1:
+                break
+
+    assert not (is_c_contig and is_f_contig)
+
+    axes_specs = []
+    # analyse all axes.
+    for idx, axis in enumerate(axes):
+
+        # start slot can be either a literal '0' or None.
+        if isinstance(axis.start, IntNode):
+            if axis.start.compile_time_value(env):
+                raise CompileError(axis.start.pos,  START_ERR)
+        elif not isinstance(axis.start, NoneNode):
+            raise CompileError(axis.start.pos,  START_ERR)
+
+        # stop slot must be None.
+        if not isinstance(axis.stop, NoneNode):
+            raise CompileError(axis.stop.pos, STOP_ERR)
+
+        # step slot can be None, the value 1, 
+        # a single axis spec, or an IntBinopNode.
+        if isinstance(axis.step, NoneNode):
+            if is_c_contig or is_f_contig:
+                axes_specs.append((cf_access, cf_packing))
+            else:
+                axes_specs.append((default_access, default_packing))
+
+        elif isinstance(axis.step, IntNode):
+            if idx not in (0, len(axes)-1):
+                raise CompileError(axis.step.pos, ONE_ERR)
+            # the packing for the ::1 axis is contiguous, 
+            # all others are cf_packing.
+            axes_specs.append((cf_access, 'contig'))
+
+        elif isinstance(axis.step, IntBinopNode):
+            if is_c_contig or is_f_contig:
+                raise CompileError(axis.step.pos, CF_ERR)
+            if axis.step.operator != u'&':
+                raise CompileError(axis.step.pos, NOT_AMP_ERR)
+            operand1, operand2 = axis.step.operand1, axis.step.operand2
+            spec1, spec2 = [_get_resolved_spec(env, op)
+                    for op in (operand1, operand2)]
+            if spec1 in access_specs and spec2 in packing_specs:
+                axes_specs.append((spec1.name, spec2.name))
+            elif spec2 in access_specs and spec1 in packing_specs:
+                axes_specs.append((spec2.name, spec1.name))
+            else:
+                raise CompileError(axis.step.pos, INVALID_ERR)
+        
+        elif isinstance(axis.step, (NameNode, AttributeNode)):
+            if is_c_contig or is_f_contig:
+                raise CompileError(axis.step.pos, CF_ERR)
+            resolved_spec = _get_resolved_spec(env, axis.step)
+            if resolved_spec in access_specs:
+                axes_specs.append((resolved_spec.name, default_packing))
+            elif resolved_spec in packing_specs:
+                axes_specs.append((default_access, resolved_spec.name))
+            else:
+                raise CompileError(axis.step.pos, INVALID_ERR)
+
+        else:
+            raise CompileError(axis.step.pos, INVALID_ERR)
+
+    return axes_specs
+
+
+def _get_resolved_spec(env, spec):
+    # spec must be a NameNode or an AttributeNode
+    if isinstance(spec, NameNode):
+        return _resolve_NameNode(env, spec)
+    elif isinstance(spec, AttributeNode):
+        return _resolve_AttributeNode(env, spec)
+    else:
+        raise CompileError(spec.pos, INVALID_ERR)
+
+def _resolve_NameNode(env, node):
+    try:
+        resolved_name = env.lookup(node.name).name
+    except AttributeError:
+        raise CompileError(node.pos, INVALID_ERR)
+    viewscope = env.context.cython_scope.viewscope
+    return viewscope.lookup(resolved_name)
+
+def _resolve_AttributeNode(env, node):
+    path = []
+    while isinstance(node, AttributeNode):
+        path.insert(0, node.attribute)
+        node = node.obj
+    if isinstance(node, NameNode):
+        path.insert(0, node.name)
+    else:
+        raise CompileError(node.pos, EXPR_ERR)
+    modnames = path[:-1]
+    # must be at least 1 module name, o/w not an AttributeNode.
+    assert modnames
+    scope = env.lookup(modnames[0]).as_module
+    for modname in modnames[1:]:
+        scope = scope.lookup(modname).as_module
+    return scope.lookup(path[-1])
+

Cython/Compiler/Nodes.py

@@ -13,7 +13,7 @@ cython.declare(sys=object, os=object, time=object, copy=object,
 import sys, os, time, copy
 
 import Builtin
-from Errors import error, warning, InternalError
+from Errors import error, warning, InternalError, CompileError
 import Naming
 import PyrexTypes
 import TypeSlots
@@ -804,6 +804,27 @@ class CSimpleBaseTypeNode(CBaseTypeNode):
         else:
             return PyrexTypes.error_type
 
+class MemoryViewTypeNode(CBaseTypeNode):
+
+    child_attrs = ['base_type_node', 'axes']
+
+    def analyse(self, env, could_be_name = False):
+
+        base_type = self.base_type_node.analyse(env)
+        if base_type.is_error: return base_type
+
+        import MemoryView
+
+        try:
+            axes_specs = MemoryView.get_axes_specs(env, self.axes)
+        except CompileError, e:
+            error(e.position, e.message_only)
+            self.type = PyrexTypes.ErrorType()
+            return self.type
+
+        self.type = PyrexTypes.MemoryViewType(base_type, axes_specs)
+        return base_type # XXX: just for testing!!!
+
 class CNestedBaseTypeNode(CBaseTypeNode):
     # For C++ classes that live inside other C++ classes.
 
@@ -825,6 +846,7 @@ class CNestedBaseTypeNode(CBaseTypeNode):
             return PyrexTypes.error_type
         return type_entry.type
 
+
 class TemplatedTypeNode(CBaseTypeNode):
     #  After parsing:
     #  positional_args  [ExprNode]        List of positional arguments

Cython/Compiler/Parsing.py

@@ -1938,7 +1938,7 @@ def p_c_complex_base_type(s):
     # s.sy == '('
     pos = s.position()
     s.next()
-    base_type = p_c_base_type(s)
+    base_type = p_c_base_type(s, empty = 1)
     declarator = p_c_declarator(s, empty = 1)
     s.expect(')')
     return Nodes.CComplexBaseTypeNode(pos,
@@ -2003,6 +2003,9 @@ def p_c_simple_base_type(s, self_flag, nonempty, templates = None):
         is_self_arg = self_flag, templates = templates)
 
     if s.sy == '[':
+        if is_memoryviewslice_access(s):
+            type_node = p_memoryview_access(s, type_node)
+        else:
             type_node = p_buffer_or_template(s, type_node, templates)
 
     if s.sy == '.':
@@ -2034,6 +2037,61 @@ def p_buffer_or_template(s, base_type_node, templates):
         base_type_node = base_type_node)
     return result
 
+def p_bracketed_base_type(s, base_type_node, nonempty, empty):
+    # s.sy == '['
+    if empty and not nonempty:
+        # sizeof-like thing.  Only anonymous C arrays allowed (int[SIZE]).
+        return base_type_node
+    elif not empty and nonempty:
+        # declaration of either memoryview or buffer.
+        if is_memoryview_access(s):
+            return p_memoryview_access(s, base_type_node)
+        else:
+            return p_buffer_access(s, base_type_node)
+    elif not empty and not nonempty:
+        # only anonymous C arrays and memoryview arrays here.  We disallow buffer
+        # declarations for now, due to ambiguity with anonymous C arrays.
+        if is_memoryview_access(s):
+            return p_memoryview_access(s, base_type_node)
+        else:
+            return base_type_node
+
+def is_memoryview_access(s):
+    # s.sy == '['
+    # a memoryview declaration is distinguishable from a buffer access
+    # declaration by the first entry in the bracketed list.  The buffer will
+    # not have an unnested colon in the first entry; the memoryview will.
+    saved = [(s.sy, s.systring)]
+    s.next()
+    retval = False
+    if s.systring == ':':
+        retval = True
+    elif s.sy == 'INT':
+        saved.append((s.sy, s.systring))
+        s.next()
+        if s.sy == ':':
+            retval = True
+
+    for sv in reversed(saved):
+        s.put_back(*sv)
+
+    return retval
+
+def p_memoryview_access(s, base_type_node):
+    # s.sy == '['
+    pos = s.position()
+    s.next()
+    subscripts = p_subscript_list(s)
+    # make sure each entry in subscripts is a slice
+    for subscript in subscripts:
+        if len(subscript) < 2:
+            s.error("An axis specification in memoryview declaration does not have a ':'.")
+    s.expect(']')
+    indexes = make_slice_nodes(pos, subscripts)
+    result = Nodes.MemoryViewTypeNode(pos,
+            base_type_node = base_type_node,
+            axes = indexes)
+    return result
 
 def looking_at_name(s):
     return s.sy == 'IDENT' and not s.systring in calling_convention_words
@@ -2977,4 +3035,3 @@ def print_parse_tree(f, node, level, key = None):
             f.write("%s]\n" % ind)
             return
     f.write("%s%s\n" % (ind, node))
-

Cython/Compiler/PyrexTypes.py

@@ -308,6 +308,11 @@ class CTypedefType(BaseType):
     def __getattr__(self, name):
         return getattr(self.typedef_base_type, name)
 
+class MemoryViewType(BaseType):
+
+    def __init__(self, base, axes):
+        self.base = base
+        self.axes = axes
         
 class BufferType(BaseType):
     #

Cython/Compiler/Tests/TestBuffer.py

@@ -98,3 +98,8 @@ class TestBufferOptions(CythonTest):
         self.assert_(stats[1].base_type.ndim == 3)
 
     # add exotic and impossible combinations as they come along...
+
+if __name__ == '__main__':
+    import unittest
+    unittest.main()
+

Cython/Compiler/Tests/TestMemView.py

+from Cython.TestUtils import CythonTest
+import Cython.Compiler.Errors as Errors
+from Cython.Compiler.Nodes import *
+from Cython.Compiler.ParseTreeTransforms import *
+from Cython.Compiler.Buffer import *
+
+
+class TestMemviewParsing(CythonTest):
+
+    def parse(self, s):
+        return self.should_not_fail(lambda: self.fragment(s)).root
+
+    def not_parseable(self, expected_error, s):
+        e = self.should_fail(lambda: self.fragment(s),  Errors.CompileError)
+        self.assertEqual(expected_error, e.message_only)
+    
+    def test_default_1dim(self):
+        self.parse(u"cdef int[:] x")
+        self.parse(u"cdef short int[:] x")
+
+    def test_default_ndim(self):
+        self.parse(u"cdef int[:,:,:,:,:] x")
+        self.parse(u"cdef unsigned long int[:,:,:,:,:] x")
+        self.parse(u"cdef unsigned int[:,:,:,:,:] x")
+
+    def test_zero_offset(self):
+        self.parse(u"cdef long double[0:] x")
+        self.parse(u"cdef int[0:] x")
+
+    def test_zero_offset_ndim(self):
+        self.parse(u"cdef int[0:,0:,0:,0:] x")
+
+    def test_general_slice(self):
+        self.parse(u'cdef float[::ptr, ::direct & contig, 0::full & strided] x')
+
+    def test_non_slice_memview(self):
+        self.not_parseable(u"An axis specification in memoryview declaration does not have a ':'.",
+                u"cdef double[:foo, bar] x")
+        self.not_parseable(u"An axis specification in memoryview declaration does not have a ':'.",
+                u"cdef double[0:foo, bar] x")
+
+    def test_basic(self):
+        t = self.parse(u"cdef int[:] x")
+        memv_node = t.stats[0].base_type
+        self.assert_(isinstance(memv_node, MemoryViewTypeNode))
+
+    # we also test other similar declarations (buffers, anonymous C arrays)
+    # since the parsing has to distinguish between them.
+
+    def disable_test_no_buf_arg(self): # TODO
+        self.not_parseable(u"Expected ']'",
+                u"cdef extern foo(object[int, ndim=2])")
+
+    def disable_test_parse_sizeof(self): # TODO
+        self.parse(u"sizeof(int[NN])")
+        self.parse(u"sizeof(int[])")
+        self.parse(u"sizeof(int[][NN])")
+        self.not_parseable(u"Expected an identifier or literal",
+                u"sizeof(int[:NN])")
+        self.not_parseable(u"Expected ']'",
+                u"sizeof(foo[dtype=bar]")
+
+if __name__ == '__main__':
+    import unittest
+    unittest.main()

tests/compile/memview_declaration.pyx

+cimport cython
+from cython.view cimport contig as foo, full as bar, follow
+from cython cimport view
+
+cdef char[:] one_dim
+cdef char[:,:,:] three_dim
+cdef unsigned int[::1, :] view1
+cdef unsigned int[:, ::1] view2
+cdef long long[::1, :, :, :] fort_contig
+cdef unsigned long[:, :, :, ::1] c_contig
+cdef unsigned short int[::1] c_and_fort
+cdef long long[0x0::0x1, 00:, -0   :,0 :] fort_contig0
+cdef unsigned long[0:, 0:, 0:, 0::0x0001] c_contig0
+
+cdef float[::foo & bar, ::cython.view.direct & cython.view.follow] view4
+cdef int[::view.full & foo] view3
+cdef int[::view.ptr & follow] view1000

tests/errors/memview_declarations.pyx

+cimport cython
+from cython.view cimport contig as foo, full as bar, follow
+from cython cimport view
+
+biz = cython.view.contig
+foz = cython.view.full
+
+adict = {'view': cython.view}
+alist = [adict]
+
+cdef signed short[::1, ::1] both
+cdef signed short[::1, :, :, ::1] both2
+cdef signed char[::2] err0
+cdef signed char[::-100] err1
+cdef signed char[::-1] err2
+cdef long long[01::1, 0x01:, '0'   :, False:] fort_contig0
+cdef signed char[1::] bad_start
+cdef unsigned long[:,:1] bad_stop
+cdef unsigned long[:,::1,:] neither_c_or_f
+cdef signed char[::1, ::view.follow & view.direct] bad_f_contig
+cdef signed char[::1, ::view.follow] bad_f_contig2
+cdef signed char[::view.contig | view.direct] not_ampersand
+cdef signed char[::view.ptr & view.direct] no_access_spec
+cdef signed char[::1-1+1] expr_spec
+cdef signed char[::blargh] bad_name
+cdef double[::alist[0]['view'].full] expr_attribute
+
+_ERRORS = u'''
+11:25: Cannot specify an array that is both C and Fortran contiguous.
+12:31: Cannot specify an array that is both C and Fortran contiguous.
+13:19: Only the value 1 (one) or valid axis specification allowed in the step slot.
+14:20: Only the value 1 (one) or valid axis specification allowed in the step slot.
+15:20: Only the value 1 (one) or valid axis specification allowed in the step slot.
+16:17: there must be nothing or the value 0 (zero) in the start slot.
+17:18: there must be nothing or the value 0 (zero) in the start slot.
+18:22: Axis specification only allowed in the 'stop' slot.
+19:23: The value 1 (one) may appear in the first or last axis specification only.
+20:36: Invalid axis specification for a C/Fortran contiguous array.
+21:28: Invalid axis specification for a C/Fortran contiguous array.
+22:31: Invalid operator, only an ampersand '&' is allowed.
+23:28: Invalid axis specification.
+24:22: Invalid axis specification.
+25:25: Invalid axis specification.
+26:22: no expressions allowed in axis spec, only names (e.g. cython.view.contig).
+'''