Commit 174beabd authored by Kurt Smith's avatar Kurt Smith Committed by Mark Florisson

parsing memory view array declarations.

parent eabb40f2
from Errors import CompileError
from ExprNodes import IntNode, NoneNode, IntBinopNode, NameNode, AttributeNode
START_ERR = "there must be nothing or the value 0 (zero) in the start slot."
STOP_ERR = "Axis specification only allowed in the 'stop' slot."
STEP_ERR = "Only the value 1 (one) or valid axis specification allowed in the step slot."
ONE_ERR = "The value 1 (one) may appear in the first or last axis specification only."
BOTH_CF_ERR = "Cannot specify an array that is both C and Fortran contiguous."
NOT_AMP_ERR = "Invalid operator, only an ampersand '&' is allowed."
INVALID_ERR = "Invalid axis specification."
EXPR_ERR = "no expressions allowed in axis spec, only names (e.g. cython.view.contig)."
CF_ERR = "Invalid axis specification for a C/Fortran contiguous array."
def get_axes_specs(env, axes):
'''
get_axes_specs(env, axes) -> list of (access, packing) specs for each axis.
access is one of 'full', 'ptr' or 'direct'
packing is one of 'contig', 'strided' or 'follow'
'''
cythonscope = env.context.cython_scope
viewscope = cythonscope.viewscope
access_specs = tuple([viewscope.lookup(name)
for name in ('full', 'direct', 'ptr')])
packing_specs = tuple([viewscope.lookup(name)
for name in ('contig', 'strided', 'follow')])
is_f_contig, is_c_contig = False, False
default_access, default_packing = 'direct', 'strided'
cf_access, cf_packing = default_access, 'follow'
# set the is_{c,f}_contig flag.
for idx, axis in ((0,axes[0]), (-1,axes[-1])):
if isinstance(axis.step, IntNode):
if axis.step.compile_time_value(env) != 1:
raise CompileError(axis.step.pos, STEP_ERR)
if len(axes) > 1 and (is_c_contig or is_f_contig):
raise CompileError(axis.step.pos, BOTH_CF_ERR)
if not idx:
is_f_contig = True
else:
is_c_contig = True
if len(axes) == 1:
break
assert not (is_c_contig and is_f_contig)
axes_specs = []
# analyse all axes.
for idx, axis in enumerate(axes):
# start slot can be either a literal '0' or None.
if isinstance(axis.start, IntNode):
if axis.start.compile_time_value(env):
raise CompileError(axis.start.pos, START_ERR)
elif not isinstance(axis.start, NoneNode):
raise CompileError(axis.start.pos, START_ERR)
# stop slot must be None.
if not isinstance(axis.stop, NoneNode):
raise CompileError(axis.stop.pos, STOP_ERR)
# step slot can be None, the value 1,
# a single axis spec, or an IntBinopNode.
if isinstance(axis.step, NoneNode):
if is_c_contig or is_f_contig:
axes_specs.append((cf_access, cf_packing))
else:
axes_specs.append((default_access, default_packing))
elif isinstance(axis.step, IntNode):
if idx not in (0, len(axes)-1):
raise CompileError(axis.step.pos, ONE_ERR)
# the packing for the ::1 axis is contiguous,
# all others are cf_packing.
axes_specs.append((cf_access, 'contig'))
elif isinstance(axis.step, IntBinopNode):
if is_c_contig or is_f_contig:
raise CompileError(axis.step.pos, CF_ERR)
if axis.step.operator != u'&':
raise CompileError(axis.step.pos, NOT_AMP_ERR)
operand1, operand2 = axis.step.operand1, axis.step.operand2
spec1, spec2 = [_get_resolved_spec(env, op)
for op in (operand1, operand2)]
if spec1 in access_specs and spec2 in packing_specs:
axes_specs.append((spec1.name, spec2.name))
elif spec2 in access_specs and spec1 in packing_specs:
axes_specs.append((spec2.name, spec1.name))
else:
raise CompileError(axis.step.pos, INVALID_ERR)
elif isinstance(axis.step, (NameNode, AttributeNode)):
if is_c_contig or is_f_contig:
raise CompileError(axis.step.pos, CF_ERR)
resolved_spec = _get_resolved_spec(env, axis.step)
if resolved_spec in access_specs:
axes_specs.append((resolved_spec.name, default_packing))
elif resolved_spec in packing_specs:
axes_specs.append((default_access, resolved_spec.name))
else:
raise CompileError(axis.step.pos, INVALID_ERR)
else:
raise CompileError(axis.step.pos, INVALID_ERR)
return axes_specs
def _get_resolved_spec(env, spec):
# spec must be a NameNode or an AttributeNode
if isinstance(spec, NameNode):
return _resolve_NameNode(env, spec)
elif isinstance(spec, AttributeNode):
return _resolve_AttributeNode(env, spec)
else:
raise CompileError(spec.pos, INVALID_ERR)
def _resolve_NameNode(env, node):
try:
resolved_name = env.lookup(node.name).name
except AttributeError:
raise CompileError(node.pos, INVALID_ERR)
viewscope = env.context.cython_scope.viewscope
return viewscope.lookup(resolved_name)
def _resolve_AttributeNode(env, node):
path = []
while isinstance(node, AttributeNode):
path.insert(0, node.attribute)
node = node.obj
if isinstance(node, NameNode):
path.insert(0, node.name)
else:
raise CompileError(node.pos, EXPR_ERR)
modnames = path[:-1]
# must be at least 1 module name, o/w not an AttributeNode.
assert modnames
scope = env.lookup(modnames[0]).as_module
for modname in modnames[1:]:
scope = scope.lookup(modname).as_module
return scope.lookup(path[-1])
......@@ -13,7 +13,7 @@ cython.declare(sys=object, os=object, time=object, copy=object,
import sys, os, time, copy
import Builtin
from Errors import error, warning, InternalError
from Errors import error, warning, InternalError, CompileError
import Naming
import PyrexTypes
import TypeSlots
......@@ -804,6 +804,27 @@ class CSimpleBaseTypeNode(CBaseTypeNode):
else:
return PyrexTypes.error_type
class MemoryViewTypeNode(CBaseTypeNode):
child_attrs = ['base_type_node', 'axes']
def analyse(self, env, could_be_name = False):
base_type = self.base_type_node.analyse(env)
if base_type.is_error: return base_type
import MemoryView
try:
axes_specs = MemoryView.get_axes_specs(env, self.axes)
except CompileError, e:
error(e.position, e.message_only)
self.type = PyrexTypes.ErrorType()
return self.type
self.type = PyrexTypes.MemoryViewType(base_type, axes_specs)
return base_type # XXX: just for testing!!!
class CNestedBaseTypeNode(CBaseTypeNode):
# For C++ classes that live inside other C++ classes.
......@@ -825,6 +846,7 @@ class CNestedBaseTypeNode(CBaseTypeNode):
return PyrexTypes.error_type
return type_entry.type
class TemplatedTypeNode(CBaseTypeNode):
# After parsing:
# positional_args [ExprNode] List of positional arguments
......
......@@ -1938,7 +1938,7 @@ def p_c_complex_base_type(s):
# s.sy == '('
pos = s.position()
s.next()
base_type = p_c_base_type(s)
base_type = p_c_base_type(s, empty = 1)
declarator = p_c_declarator(s, empty = 1)
s.expect(')')
return Nodes.CComplexBaseTypeNode(pos,
......@@ -2003,6 +2003,9 @@ def p_c_simple_base_type(s, self_flag, nonempty, templates = None):
is_self_arg = self_flag, templates = templates)
if s.sy == '[':
if is_memoryviewslice_access(s):
type_node = p_memoryview_access(s, type_node)
else:
type_node = p_buffer_or_template(s, type_node, templates)
if s.sy == '.':
......@@ -2034,6 +2037,61 @@ def p_buffer_or_template(s, base_type_node, templates):
base_type_node = base_type_node)
return result
def p_bracketed_base_type(s, base_type_node, nonempty, empty):
# s.sy == '['
if empty and not nonempty:
# sizeof-like thing. Only anonymous C arrays allowed (int[SIZE]).
return base_type_node
elif not empty and nonempty:
# declaration of either memoryview or buffer.
if is_memoryview_access(s):
return p_memoryview_access(s, base_type_node)
else:
return p_buffer_access(s, base_type_node)
elif not empty and not nonempty:
# only anonymous C arrays and memoryview arrays here. We disallow buffer
# declarations for now, due to ambiguity with anonymous C arrays.
if is_memoryview_access(s):
return p_memoryview_access(s, base_type_node)
else:
return base_type_node
def is_memoryview_access(s):
# s.sy == '['
# a memoryview declaration is distinguishable from a buffer access
# declaration by the first entry in the bracketed list. The buffer will
# not have an unnested colon in the first entry; the memoryview will.
saved = [(s.sy, s.systring)]
s.next()
retval = False
if s.systring == ':':
retval = True
elif s.sy == 'INT':
saved.append((s.sy, s.systring))
s.next()
if s.sy == ':':
retval = True
for sv in reversed(saved):
s.put_back(*sv)
return retval
def p_memoryview_access(s, base_type_node):
# s.sy == '['
pos = s.position()
s.next()
subscripts = p_subscript_list(s)
# make sure each entry in subscripts is a slice
for subscript in subscripts:
if len(subscript) < 2:
s.error("An axis specification in memoryview declaration does not have a ':'.")
s.expect(']')
indexes = make_slice_nodes(pos, subscripts)
result = Nodes.MemoryViewTypeNode(pos,
base_type_node = base_type_node,
axes = indexes)
return result
def looking_at_name(s):
return s.sy == 'IDENT' and not s.systring in calling_convention_words
......@@ -2977,4 +3035,3 @@ def print_parse_tree(f, node, level, key = None):
f.write("%s]\n" % ind)
return
f.write("%s%s\n" % (ind, node))
......@@ -308,6 +308,11 @@ class CTypedefType(BaseType):
def __getattr__(self, name):
return getattr(self.typedef_base_type, name)
class MemoryViewType(BaseType):
def __init__(self, base, axes):
self.base = base
self.axes = axes
class BufferType(BaseType):
#
......
......@@ -98,3 +98,8 @@ class TestBufferOptions(CythonTest):
self.assert_(stats[1].base_type.ndim == 3)
# add exotic and impossible combinations as they come along...
if __name__ == '__main__':
import unittest
unittest.main()
from Cython.TestUtils import CythonTest
import Cython.Compiler.Errors as Errors
from Cython.Compiler.Nodes import *
from Cython.Compiler.ParseTreeTransforms import *
from Cython.Compiler.Buffer import *
class TestMemviewParsing(CythonTest):
def parse(self, s):
return self.should_not_fail(lambda: self.fragment(s)).root
def not_parseable(self, expected_error, s):
e = self.should_fail(lambda: self.fragment(s), Errors.CompileError)
self.assertEqual(expected_error, e.message_only)
def test_default_1dim(self):
self.parse(u"cdef int[:] x")
self.parse(u"cdef short int[:] x")
def test_default_ndim(self):
self.parse(u"cdef int[:,:,:,:,:] x")
self.parse(u"cdef unsigned long int[:,:,:,:,:] x")
self.parse(u"cdef unsigned int[:,:,:,:,:] x")
def test_zero_offset(self):
self.parse(u"cdef long double[0:] x")
self.parse(u"cdef int[0:] x")
def test_zero_offset_ndim(self):
self.parse(u"cdef int[0:,0:,0:,0:] x")
def test_general_slice(self):
self.parse(u'cdef float[::ptr, ::direct & contig, 0::full & strided] x')
def test_non_slice_memview(self):
self.not_parseable(u"An axis specification in memoryview declaration does not have a ':'.",
u"cdef double[:foo, bar] x")
self.not_parseable(u"An axis specification in memoryview declaration does not have a ':'.",
u"cdef double[0:foo, bar] x")
def test_basic(self):
t = self.parse(u"cdef int[:] x")
memv_node = t.stats[0].base_type
self.assert_(isinstance(memv_node, MemoryViewTypeNode))
# we also test other similar declarations (buffers, anonymous C arrays)
# since the parsing has to distinguish between them.
def disable_test_no_buf_arg(self): # TODO
self.not_parseable(u"Expected ']'",
u"cdef extern foo(object[int, ndim=2])")
def disable_test_parse_sizeof(self): # TODO
self.parse(u"sizeof(int[NN])")
self.parse(u"sizeof(int[])")
self.parse(u"sizeof(int[][NN])")
self.not_parseable(u"Expected an identifier or literal",
u"sizeof(int[:NN])")
self.not_parseable(u"Expected ']'",
u"sizeof(foo[dtype=bar]")
if __name__ == '__main__':
import unittest
unittest.main()
cimport cython
from cython.view cimport contig as foo, full as bar, follow
from cython cimport view
cdef char[:] one_dim
cdef char[:,:,:] three_dim
cdef unsigned int[::1, :] view1
cdef unsigned int[:, ::1] view2
cdef long long[::1, :, :, :] fort_contig
cdef unsigned long[:, :, :, ::1] c_contig
cdef unsigned short int[::1] c_and_fort
cdef long long[0x0::0x1, 00:, -0 :,0 :] fort_contig0
cdef unsigned long[0:, 0:, 0:, 0::0x0001] c_contig0
cdef float[::foo & bar, ::cython.view.direct & cython.view.follow] view4
cdef int[::view.full & foo] view3
cdef int[::view.ptr & follow] view1000
cimport cython
from cython.view cimport contig as foo, full as bar, follow
from cython cimport view
biz = cython.view.contig
foz = cython.view.full
adict = {'view': cython.view}
alist = [adict]
cdef signed short[::1, ::1] both
cdef signed short[::1, :, :, ::1] both2
cdef signed char[::2] err0
cdef signed char[::-100] err1
cdef signed char[::-1] err2
cdef long long[01::1, 0x01:, '0' :, False:] fort_contig0
cdef signed char[1::] bad_start
cdef unsigned long[:,:1] bad_stop
cdef unsigned long[:,::1,:] neither_c_or_f
cdef signed char[::1, ::view.follow & view.direct] bad_f_contig
cdef signed char[::1, ::view.follow] bad_f_contig2
cdef signed char[::view.contig | view.direct] not_ampersand
cdef signed char[::view.ptr & view.direct] no_access_spec
cdef signed char[::1-1+1] expr_spec
cdef signed char[::blargh] bad_name
cdef double[::alist[0]['view'].full] expr_attribute
_ERRORS = u'''
11:25: Cannot specify an array that is both C and Fortran contiguous.
12:31: Cannot specify an array that is both C and Fortran contiguous.
13:19: Only the value 1 (one) or valid axis specification allowed in the step slot.
14:20: Only the value 1 (one) or valid axis specification allowed in the step slot.
15:20: Only the value 1 (one) or valid axis specification allowed in the step slot.
16:17: there must be nothing or the value 0 (zero) in the start slot.
17:18: there must be nothing or the value 0 (zero) in the start slot.
18:22: Axis specification only allowed in the 'stop' slot.
19:23: The value 1 (one) may appear in the first or last axis specification only.
20:36: Invalid axis specification for a C/Fortran contiguous array.
21:28: Invalid axis specification for a C/Fortran contiguous array.
22:31: Invalid operator, only an ampersand '&' is allowed.
23:28: Invalid axis specification.
24:22: Invalid axis specification.
25:25: Invalid axis specification.
26:22: no expressions allowed in axis spec, only names (e.g. cython.view.contig).
'''
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment