merged in quick fix by Dag

Cython/Compiler/Buffer.py

+from Cython.Compiler.Visitor import VisitorTransform, temp_name_handle, CythonTransform
+from Cython.Compiler.ModuleNode import ModuleNode
+from Cython.Compiler.Nodes import *
+from Cython.Compiler.ExprNodes import *
+from Cython.Compiler.TreeFragment import TreeFragment
+from Cython.Utils import EncodedString
+from Cython.Compiler.Errors import CompileError
+import PyrexTypes
+from sets import Set as set
+
+class PureCFuncNode(Node):
+    def __init__(self, pos, cname, type, c_code, visibility='private'):
+        self.pos = pos
+        self.cname = cname
+        self.type = type
+        self.c_code = c_code
+        self.visibility = visibility
+
+    def analyse_types(self, env):
+        self.entry = env.declare_cfunction(
+            "<pure c function:%s>" % self.cname,
+            self.type, self.pos, cname=self.cname,
+            defining=True, visibility=self.visibility)
+
+    def generate_function_definitions(self, env, code, transforms):
+        assert self.type.optional_arg_count == 0
+        visibility = self.entry.visibility
+        if visibility != 'private':
+            storage_class = "%s " % Naming.extern_c_macro
+        else:
+            storage_class = "static "
+        arg_decls = [arg.declaration_code() for arg in self.type.args]
+        sig = self.type.return_type.declaration_code(
+            self.type.function_header_code(self.cname, ", ".join(arg_decls)))
+        code.putln("")
+        code.putln("%s%s {" % (storage_class, sig))
+        code.put(self.c_code)
+        code.putln("}")
+
+    def generate_execution_code(self, code):
+        pass
+
+
+tschecker_functype = PyrexTypes.CFuncType(
+    PyrexTypes.c_char_ptr_type,
+    [PyrexTypes.CFuncTypeArg(EncodedString("ts"), PyrexTypes.c_char_ptr_type,
+                             (0, 0, None), cname="ts")],
+    exception_value = "NULL"
+)  
+
+tsprefix = "__Pyx_tsc"
+
+class BufferTransform(CythonTransform):
+    """
+    Run after type analysis. Takes care of the buffer functionality.
+
+    Expects to be run on the full module. If you need to process a fragment
+    one should look into refactoring this transform.
+    """
+    # Abbreviations:
+    # "ts" means typestring and/or typestring checking stuff
+    
+    scope = None
+
+    #
+    # Entry point
+    #
+
+    def __call__(self, node):
+        assert isinstance(node, ModuleNode)
+        
+        try:
+            cymod = self.context.modules[u'__cython__']
+        except KeyError:
+            # No buffer fun for this module
+            return node
+        self.bufstruct_type = cymod.entries[u'Py_buffer'].type
+        self.tscheckers = {}
+        self.ts_funcs = []
+        self.ts_item_checkers = {}
+        self.module_scope = node.scope
+        self.module_pos = node.pos
+        result = super(BufferTransform, self).__call__(node)
+        # Register ts stuff
+        if "endian.h" not in node.scope.include_files:
+            node.scope.include_files.append("endian.h")
+        result.body.stats += self.ts_funcs
+        return result
+
+
+    #
+    # Basic operations for transforms
+    #
+    def handle_scope(self, node, scope):
+        # For all buffers, insert extra variables in the scope.
+        # The variables are also accessible from the buffer_info
+        # on the buffer entry
+        bufvars = [(name, entry) for name, entry
+                   in scope.entries.iteritems()
+                   if entry.type.buffer_options is not None]
+                   
+        for name, entry in bufvars:
+            
+            bufopts = entry.type.buffer_options
+
+            # Get or make a type string checker
+            tschecker = self.tschecker(bufopts.dtype)
+
+            # Declare auxiliary vars
+            bufinfo = scope.declare_var(temp_name_handle(u"%s_bufinfo" % name),
+                                        self.bufstruct_type, node.pos)
+
+            temp_var =  scope.declare_var(temp_name_handle(u"%s_tmp" % name),
+                                        entry.type, node.pos)
+            
+            
+            stridevars = []
+            shapevars = []
+            for idx in range(bufopts.ndim):
+                # stride
+                varname = temp_name_handle(u"%s_%s%d" % (name, "stride", idx))
+                var = scope.declare_var(varname, PyrexTypes.c_int_type, node.pos, is_cdef=True)
+                stridevars.append(var)
+                # shape
+                varname = temp_name_handle(u"%s_%s%d" % (name, "shape", idx))
+                var = scope.declare_var(varname, PyrexTypes.c_uint_type, node.pos, is_cdef=True)
+                shapevars.append(var)
+            entry.buffer_aux = Symtab.BufferAux(bufinfo, stridevars, 
+                                                shapevars, tschecker)
+            entry.buffer_aux.temp_var = temp_var
+        self.scope = scope
+
+    # Notes: The cast to <char*> gets around Cython not supporting const types
+    acquire_buffer_fragment = TreeFragment(u"""
+        TMP = LHS
+        if TMP is not None:
+            __cython__.PyObject_ReleaseBuffer(<__cython__.PyObject*>TMP, &BUFINFO)
+        TMP = RHS
+        if TMP is not None:
+            __cython__.PyObject_GetBuffer(<__cython__.PyObject*>TMP, &BUFINFO, 0)
+            TSCHECKER(<char*>BUFINFO.format)
+            ASSIGN_AUX
+        LHS = TMP
+    """)
+
+    fetch_strides = TreeFragment(u"""
+        TARGET = BUFINFO.strides[IDX]
+    """)
+
+    fetch_shape = TreeFragment(u"""
+        TARGET = BUFINFO.shape[IDX]
+    """)
+
+    def reacquire_buffer(self, node):
+        bufaux = node.lhs.entry.buffer_aux
+        auxass = []
+        for idx, entry in enumerate(bufaux.stridevars):
+            entry.used = True
+            ass = self.fetch_strides.substitute({
+                u"TARGET": NameNode(node.pos, name=entry.name),
+                u"BUFINFO": NameNode(node.pos, name=bufaux.buffer_info_var.name),
+                u"IDX": IntNode(node.pos, value=EncodedString(idx)),
+            })
+            auxass.append(ass)
+
+        for idx, entry in enumerate(bufaux.shapevars):
+            entry.used = True
+            ass = self.fetch_shape.substitute({
+                u"TARGET": NameNode(node.pos, name=entry.name),
+                u"BUFINFO": NameNode(node.pos, name=bufaux.buffer_info_var.name),
+                u"IDX": IntNode(node.pos, value=EncodedString(idx))
+            })
+            auxass.append(ass)
+
+        bufaux.buffer_info_var.used = True
+        acq = self.acquire_buffer_fragment.substitute({
+            u"TMP" : NameNode(pos=node.pos, name=bufaux.temp_var.name),
+            u"LHS" : node.lhs,
+            u"RHS": node.rhs,
+            u"ASSIGN_AUX": StatListNode(node.pos, stats=auxass),
+            u"BUFINFO": NameNode(pos=node.pos, name=bufaux.buffer_info_var.name),
+            u"TSCHECKER": NameNode(node.pos, name=bufaux.tschecker.name)
+        }, pos=node.pos)
+        # Note: The below should probably be refactored into something
+        # like fragment.substitute(..., context=self.context), with
+        # TreeFragment getting context.pipeline_until_now() and
+        # applying it on the fragment.
+        acq.analyse_declarations(self.scope)
+        acq.analyse_expressions(self.scope)
+        stats = acq.stats
+        return stats
+
+    def assign_into_buffer(self, node):
+        result = SingleAssignmentNode(node.pos,
+                                      rhs=self.visit(node.rhs),
+                                      lhs=self.buffer_index(node.lhs))
+        result.analyse_expressions(self.scope)
+        return result
+        
+
+    def buffer_index(self, node):
+        pos = node.pos
+        bufaux = node.base.entry.buffer_aux
+        assert bufaux is not None
+        # indices * strides...
+        to_sum = [ IntBinopNode(pos, operator='*',
+                                operand1=index, #PhaseEnvelopeNode(PhaseEnvelopeNode.ANALYSED, index),
+                                operand2=NameNode(node.pos, name=stride.name))
+            for index, stride in zip(node.indices, bufaux.stridevars)]
+
+        # then sum them with the buffer pointer
+        expr = AttributeNode(pos,
+            obj=NameNode(pos, name=bufaux.buffer_info_var.name),
+            attribute=EncodedString("buf"))
+        for next in to_sum:
+            expr = AddNode(pos, operator='+', operand1=expr, operand2=next)
+
+        casted = TypecastNode(pos, operand=expr,
+                              type=PyrexTypes.c_ptr_type(node.base.entry.type.buffer_options.dtype))
+        result = IndexNode(pos, base=casted, index=IntNode(pos, value='0'))
+
+        return result
+
+
+    #
+    # Transforms
+    #
+    def visit_ModuleNode(self, node):
+        self.handle_scope(node, node.scope)
+        self.visitchildren(node)
+        return node
+
+    def visit_FuncDefNode(self, node):
+        self.handle_scope(node, node.local_scope)
+        self.visitchildren(node)
+        return node
+
+    def visit_SingleAssignmentNode(self, node):
+        # On assignments, two buffer-related things can happen:
+        # a) A buffer variable is assigned to (reacquisition)
+        # b) Buffer access assignment: arr[...] = ...
+        # Since we don't allow nested buffers, these don't overlap.
+        self.visitchildren(node)
+        # Only acquire buffers on vars (not attributes) for now.
+        if isinstance(node.lhs, NameNode) and node.lhs.entry.buffer_aux:
+            # Is buffer variable
+            return self.reacquire_buffer(node)
+        elif (isinstance(node.lhs, IndexNode) and
+              isinstance(node.lhs.base, NameNode) and
+              node.lhs.base.entry.buffer_aux is not None):
+            return self.assign_into_buffer(node)
+        else:
+            return node
+        
+    def visit_IndexNode(self, node):
+        # Only occurs when the IndexNode is an rvalue
+        if node.is_buffer_access:
+            assert node.index is None
+            assert node.indices is not None
+            result = self.buffer_index(node)
+            result.analyse_expressions(self.scope)
+            return result
+        else:
+            return node
+
+    #
+    # Utils for creating type string checkers
+    #
+    
+    def new_ts_func(self, name, code):
+        cname = "%s_%s" % (tsprefix, name)
+        funcnode = PureCFuncNode(self.module_pos, cname, tschecker_functype, code)
+        funcnode.analyse_types(self.module_scope)
+        self.ts_funcs.append(funcnode)
+        return funcnode        
+    
+    def mangle_dtype_name(self, dtype):
+        # Use prefixes to seperate user defined types from builtins
+        # (consider "typedef float unsigned_int")
+        return dtype.declaration_code("").replace(" ", "_")
+        
+    def get_ts_check_item(self, dtype):
+        # See if we can consume one (unnamed) dtype as next item
+        funcnode = self.ts_item_checkers.get(dtype)
+        if funcnode is None:
+            char = dtype.typestring
+            if char is not None and len(char) > 1:
+                # Can use direct comparison
+                funcnode = self.new_ts_func("natitem_%s" % self.mangle_dtype_name(dtype), """\
+  if (*ts != '%s') {
+    PyErr_Format(PyExc_TypeError, "Buffer datatype mismatch (rejecting on '%%s')", ts);
+  return NULL;
+  } else return ts + 1;
+""" % char)
+            else:
+                # Must deduce sign and length; rely on int vs. float to be correctly declared
+                ctype = dtype.declaration_code("")
+                
+                code = """\
+  int ok;
+  switch (*ts) {"""
+                if dtype.is_int:
+                    types = [
+                        ('b', 'char'), ('h', 'short'), ('i', 'int'),
+                        ('l', 'long'), ('q', 'long long')
+                    ]
+                    code += "".join(["""\
+    case '%s': ok = (sizeof(%s) == sizeof(%s) && (%s)-1 < 0); break;
+    case '%s': ok = (sizeof(%s) == sizeof(%s) && (%s)-1 > 0); break;
+""" % (char, ctype, against, ctype, char.upper(), ctype, "unsigned " + against, ctype) for
+                                     char, against in types])
+                    code += """\
+    default: ok = 0;
+  }
+  if (!ok) {
+    PyErr_Format(PyExc_TypeError, "Buffer datatype mismatch (rejecting on '%s')", ts);
+    return NULL;
+  } else return ts + 1;
+"""
+                
+                funcnode = self.new_ts_func("tdefitem_%s" % self.mangle_dtype_name(dtype), code)
+                
+            self.ts_item_checkers[dtype] = funcnode
+        return funcnode.entry.cname
+
+    ts_consume_whitespace_cname = None
+    ts_check_endian_cname = None
+
+    def ensure_ts_utils(self):
+        # Makes sure that the typechecker utils are in scope
+        # (and constructs them if not)
+        if self.ts_consume_whitespace_cname is None:
+            self.ts_consume_whitespace_cname = self.new_ts_func("consume_whitespace", """\
+  while (1) {
+    switch (*ts) {
+      case 10:
+      case 13:
+      case ' ':
+        ++ts;
+      default:
+        return ts;
+    }
+  }
+""").entry.cname
+        if self.ts_check_endian_cname is None:
+            self.ts_check_endian_cname = self.new_ts_func("check_endian", """\
+  int ok = 1;
+  switch (*ts) {
+    case '@':
+    case '=':
+      ++ts; break;
+    case '<':
+      if (__BYTE_ORDER == __LITTLE_ENDIAN) ++ts;
+      else ok = 0;
+      break;
+    case '>':
+    case '!':
+      if (__BYTE_ORDER == __BIG_ENDIAN) ++ts;
+      else ok = 0;
+      break;
+  }
+  if (!ok) {
+    PyErr_Format(PyExc_TypeError, "Data has wrong endianness (rejecting on '%s')", ts);
+    return NULL;
+  }
+  return ts;
+""").entry.cname
+            
+    def create_ts_check_simple(self, dtype):
+        # Check whole string for single unnamed item
+        consume_whitespace = self.ts_consume_whitespace_cname
+        check_endian = self.ts_check_endian_cname
+        check_item = self.get_ts_check_item(dtype)
+        return self.new_ts_func("simple_%s" % self.mangle_dtype_name(dtype), """\
+  ts = %(consume_whitespace)s(ts);
+  ts = %(check_endian)s(ts);
+  if (!ts) return NULL;
+  ts = %(consume_whitespace)s(ts);
+  ts = %(check_item)s(ts);
+  if (!ts) return NULL;
+  ts = %(consume_whitespace)s(ts);
+  if (*ts != 0) {
+    PyErr_Format(PyExc_TypeError, "Data too long (rejecting on '%%s')", ts);
+    return NULL;
+  }
+  return ts;
+""" % locals())
+
+    def tschecker(self, dtype):
+        # Creates a type string checker function for the given type.
+        # Each checker is created as a function entry in the module scope
+        # and a PureCNode and put in the self.ts_checkers dict.
+        # Also the entry is returned.
+        #
+        # TODO: __eq__ and __hash__ for types
+
+        self.ensure_ts_utils()
+        funcnode = self.tscheckers.get(dtype)
+        if funcnode is None:
+            if dtype.is_struct_or_union:
+                assert False
+            elif dtype.is_int or dtype.is_float:
+                # This includes simple typedef-ed types
+                funcnode = self.create_ts_check_simple(dtype)
+            else:
+                assert False
+            self.tscheckers[dtype] = funcnode
+        return funcnode.entry
+
+
+
+# TODO:
+# - buf must be NULL before getting new buffer
+

Cython/Compiler/ExprNodes.py

@@ -1251,8 +1251,19 @@ class IndexNode(ExprNode):
     #
     #  base     ExprNode
     #  index    ExprNode
+    #  indices  [ExprNode]
+    #  is_buffer_access boolean Whether this is a buffer access.
+    #
+    #  indices is used on buffer access, index on non-buffer access.
+    #  The former contains a clean list of index parameters, the
+    #  latter whatever Python object is needed for index access.
     
-    subexprs = ['base', 'index']
+    subexprs = ['base', 'index', 'indices']
+    indices = None
+
+    def __init__(self, pos, index, *args, **kw):
+        ExprNode.__init__(self, pos, index=index, *args, **kw)
+        self._index = index
     
     def compile_time_value(self, denv):
         base = self.base.compile_time_value(denv)
@@ -1273,33 +1284,45 @@ class IndexNode(ExprNode):
     
     def analyse_target_types(self, env):
         self.analyse_base_and_index_types(env, setting = 1)
-    
+
     def analyse_base_and_index_types(self, env, getting = 0, setting = 0):
         self.is_buffer_access = False
 
         self.base.analyse_types(env)
-        
+
+        skip_child_analysis = False
+        buffer_access = False
         if self.base.type.buffer_options is not None:
             if isinstance(self.index, TupleNode):
                 indices = self.index.args
-#                is_int_indices = 0 == sum([1 for i in self.index.args if not i.type.is_int])
             else:
-#                is_int_indices = self.index.type.is_int
                 indices = [self.index]
-            all_ints = True
-            for index in indices:
-                index.analyse_types(env)
-                if not index.type.is_int:
-                    all_ints = False
-            if all_ints:
-                self.indices = indices
-                self.index = None
-                self.type = self.base.type.buffer_options.dtype
-                self.is_temp = 1
-                self.is_buffer_access = True
+            if len(indices) == self.base.type.buffer_options.ndim:
+                buffer_access = True
+                skip_child_analysis = True
+                for x in indices:
+                    x.analyse_types(env)
+                    if not x.type.is_int:
+                        buffer_access = False
+                if buffer_access:
+                    # self.indices = [
+                    #   x.coerce_to(PyrexTypes.c_py_ssize_t_type, env).coerce_to_simple(env)
+                    # for x in  indices]
+                    self.indices = indices
+                    self.index = None
+                    self.type = self.base.type.buffer_options.dtype 
+                    self.is_temp = 1
+                    self.is_buffer_access = True
+            
+        # Note: This might be cleaned up by having IndexNode
+        # parsed in a saner way and only construct the tuple if
+        # needed.
 
-        if not self.is_buffer_access:
-            self.index.analyse_types(env) # ok to analyse as tuple
+        if not buffer_access:
+            if isinstance(self.index, TupleNode):
+                self.index.analyse_types(env, skip_children=skip_child_analysis)
+            elif not skip_child_analysis:
+                self.index.analyse_types(env)
             if self.base.type.is_pyobject:
                 if self.index.type.is_int:
                     self.original_index_type = self.index.type
@@ -1339,8 +1362,11 @@ class IndexNode(ExprNode):
         return 1
     
     def calculate_result_code(self):
-        return "(%s[%s])" % (
-            self.base.result_code, self.index.result_code)
+        if self.is_buffer_access:
+            return "<not needed>"
+        else:
+            return "(%s[%s])" % (
+                self.base.result_code, self.index.result_code)
             
     def index_unsigned_parameter(self):
         if self.index.type.is_int:
@@ -2197,10 +2223,10 @@ class SequenceNode(ExprNode):
         for arg in self.args:
             arg.analyse_target_declaration(env)
 
-    def analyse_types(self, env):
+    def analyse_types(self, env, skip_children=False):
         for i in range(len(self.args)):
             arg = self.args[i]
-            arg.analyse_types(env)
+            if not skip_children: arg.analyse_types(env)
             self.args[i] = arg.coerce_to_pyobject(env)
         self.type = py_object_type
         self.gil_check(env)
@@ -2305,12 +2331,12 @@ class TupleNode(SequenceNode):
 
     gil_message = "Constructing Python tuple"
 
-    def analyse_types(self, env):
+    def analyse_types(self, env, skip_children=False):
         if len(self.args) == 0:
             self.is_temp = 0
             self.is_literal = 1
         else:
-            SequenceNode.analyse_types(self, env)
+            SequenceNode.analyse_types(self, env, skip_children)
         self.type = tuple_type
             
     def calculate_result_code(self):
@@ -2813,15 +2839,20 @@ def unop_node(pos, operator, operand):
 class TypecastNode(ExprNode):
     #  C type cast
     #
+    #  operand      ExprNode
     #  base_type    CBaseTypeNode
     #  declarator   CDeclaratorNode
-    #  operand      ExprNode
+    #
+    #  If used from a transform, one can if wanted specify the attribute
+    #  "type" directly and leave base_type and declarator to None
     
     subexprs = ['operand']
+    base_type = declarator = type = None
     
     def analyse_types(self, env):
-        base_type = self.base_type.analyse(env)
-        _, self.type = self.declarator.analyse(base_type, env)
+        if self.type is None:
+            base_type = self.base_type.analyse(env)
+            _, self.type = self.declarator.analyse(base_type, env)
         if self.type.is_cfunction:
             error(self.pos,
                 "Cannot cast to a function type")
@@ -3842,6 +3873,10 @@ class CoerceToPyTypeNode(CoercionNode):
 
     gil_message = "Converting to Python object"
 
+    def analyse_types(self, env):
+        # The arg is always already analysed
+        pass
+
     def generate_result_code(self, code):
         function = self.arg.type.to_py_function
         code.putln('%s = %s(%s); %s' % (
@@ -3866,6 +3901,10 @@ class CoerceFromPyTypeNode(CoercionNode):
             error(arg.pos,
                 "Obtaining char * from temporary Python value")
     
+    def analyse_types(self, env):
+        # The arg is always already analysed
+        pass
+
     def generate_result_code(self, code):
         function = self.type.from_py_function
         operand = self.arg.py_result()
@@ -3924,6 +3963,10 @@ class CoerceToTempNode(CoercionNode):
 
     gil_message = "Creating temporary Python reference"
 
+    def analyse_types(self, env):
+        # The arg is always already analysed
+        pass
+
     def generate_result_code(self, code):
         #self.arg.generate_evaluation_code(code) # Already done
         # by generic generate_subexpr_evaluation_code!

Cython/Compiler/Main.py

@@ -354,9 +354,10 @@ def create_generate_code(context, options, result):
     return generate_code
 
 def create_default_pipeline(context, options, result):
-    from ParseTreeTransforms import WithTransform, NormalizeTree, PostParse, BufferTransform
+    from ParseTreeTransforms import WithTransform, NormalizeTree, PostParse
     from ParseTreeTransforms import AnalyseDeclarationsTransform, AnalyseExpressionsTransform
     from ParseTreeTransforms import CreateClosureClasses, MarkClosureVisitor
+    from Buffer import BufferTransform
     from ModuleNode import check_c_classes
     
     return [

Cython/Compiler/ModuleNode.py

@@ -1953,53 +1953,81 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
 
     def generate_buffer_compatability_functions(self, env, code):
         # will be refactored
-        code.put("""
+        try:
+            env.entries[u'numpy']
+            code.put("""
 static int numpy_getbuffer(PyObject *obj, Py_buffer *view, int flags) {
-  /* This function is always called after a type-check */
+  /* This function is always called after a type-check; safe to cast */
   PyArrayObject *arr = (PyArrayObject*)obj;
   PyArray_Descr *type = (PyArray_Descr*)arr->descr;
+
   
+  int typenum = PyArray_TYPE(obj);
+  if (!PyTypeNum_ISNUMBER(typenum)) {
+    PyErr_Format(PyExc_TypeError, "Only numeric NumPy types currently supported.");
+    return -1;
+  }
+
+  /*
+  NumPy format codes doesn't completely match buffer codes;
+  seems safest to retranslate.
+                            01234567890123456789012345*/
+  const char* base_codes = "?bBhHiIlLqQfdgfdgO";
+
+  char* format = (char*)malloc(4);
+  char* fp = format;
+  *fp++ = type->byteorder;
+  if (PyTypeNum_ISCOMPLEX(typenum)) *fp++ = 'Z';
+  *fp++ = base_codes[typenum];
+  *fp = 0;
+
   view->buf = arr->data;
-  view->readonly = 0; /*fixme*/
-  view->format = "B"; /*fixme*/
-  view->ndim = arr->nd;
-  view->strides = arr->strides;
-  view->shape = arr->dimensions;
-  view->suboffsets = 0;
-  
+  view->readonly = !PyArray_ISWRITEABLE(obj);
+  view->ndim = PyArray_NDIM(arr);
+  view->strides = PyArray_STRIDES(arr);
+  view->shape = PyArray_DIMS(arr);
+  view->suboffsets = NULL;
+  view->format = format;
   view->itemsize = type->elsize;
+
   view->internal = 0;
   return 0;
 }
 
 static void numpy_releasebuffer(PyObject *obj, Py_buffer *view) {
+  free((char*)view->format);
+  view->format = NULL;
 }
 
 """)
+        except KeyError:
+            pass
         
         # For now, hard-code numpy imported as "numpy"
-        ndarrtype = env.entries[u'numpy'].as_module.entries['ndarray'].type
-        types = [
-            (ndarrtype.typeptr_cname, "numpy_getbuffer", "numpy_releasebuffer")
-        ]
-        
-#        typeptr_cname = ndarrtype.typeptr_cname
+        types = []
+        try:
+            ndarrtype = env.entries[u'numpy'].as_module.entries['ndarray'].type
+            types.append((ndarrtype.typeptr_cname, "numpy_getbuffer", "numpy_releasebuffer"))
+        except KeyError:
+            pass
         code.putln("static int PyObject_GetBuffer(PyObject *obj, Py_buffer *view, int flags) {")
-        clause = "if"
-        for t, get, release in types:
-            code.putln("%s (__Pyx_TypeTest(obj, %s)) return %s(obj, view, flags);" % (clause, t, get))
-            clause = "else if"
-        code.putln("else {")
+        if len(types) > 0:
+            clause = "if"
+            for t, get, release in types:
+                code.putln("%s (__Pyx_TypeTest(obj, %s)) return %s(obj, view, flags);" % (clause, t, get))
+                clause = "else if"
+            code.putln("else {")
         code.putln("PyErr_Format(PyExc_TypeError, \"'%100s' does not have the buffer interface\", Py_TYPE(obj)->tp_name);")
         code.putln("return -1;")
-        code.putln("}")
+        if len(types) > 0: code.putln("}")
         code.putln("}")
         code.putln("")
         code.putln("static void PyObject_ReleaseBuffer(PyObject *obj, Py_buffer *view) {")
-        clause = "if"
-        for t, get, release in types:
-            code.putln("%s (__Pyx_TypeTest(obj, %s)) %s(obj, view);" % (clause, t, release))
-            clause = "else if"
+        if len(types) > 0:
+            clause = "if"
+            for t, get, release in types:
+                code.putln("%s (__Pyx_TypeTest(obj, %s)) %s(obj, view);" % (clause, t, release))
+                clause = "else if"
         code.putln("}")
         code.putln("")
 

Cython/Compiler/Nodes.py

@@ -184,10 +184,10 @@ class Node(object):
         
         attrs = [(key, value) for key, value in self.__dict__.iteritems() if key not in filter_out]
         if len(attrs) == 0:
-            return "<%s>" % self.__class__.__name__
+            return "<%s (%d)>" % (self.__class__.__name__, id(self))
         else:
             indent = "  " * level
-            res = "<%s\n" % (self.__class__.__name__)
+            res = "<%s (%d)\n" % (self.__class__.__name__, id(self))
             for key, value in attrs:
                 res += "%s  %s: %s\n" % (indent, key, dump_child(value, level + 1))
             res += "%s>" % indent

Cython/Compiler/ParseTreeTransforms.py

-
 from Cython.Compiler.Visitor import VisitorTransform, temp_name_handle, CythonTransform
 from Cython.Compiler.ModuleNode import ModuleNode
 from Cython.Compiler.Nodes import *
@@ -51,12 +50,6 @@ class NormalizeTree(CythonTransform):
         else:
             return node
 
-    def visit_PassStatNode(self, node):
-        if not self.is_in_statlist:
-            return StatListNode(pos=node.pos, stats=[])
-        else:
-            return []
-
     def visit_StatListNode(self, node):
         self.is_in_statlist = True
         self.visitchildren(node)
@@ -72,6 +65,18 @@ class NormalizeTree(CythonTransform):
     def visit_CStructOrUnionDefNode(self, node):
         return self.visit_StatNode(node, True)
 
+    # Eliminate PassStatNode
+    def visit_PassStatNode(self, node):
+        if not self.is_in_statlist:
+            return StatListNode(pos=node.pos, stats=[])
+        else:
+            return []
+
+    # Eliminate CascadedAssignmentNode
+    def visit_CascadedAssignmentNode(self, node):
+        tmpname = temp_name_handle()
+        
+
 
 class PostParseError(CompileError): pass
 
@@ -146,169 +151,6 @@ class PostParse(CythonTransform):
         node.keyword_args = None
         return node
 
-class BufferTransform(CythonTransform):
-    """
-    Run after type analysis. Takes care of the buffer functionality.
-    """
-    scope = None
-
-    def __call__(self, node):
-        cymod = self.context.modules[u'__cython__']
-        self.buffer_type = cymod.entries[u'Py_buffer'].type
-        return super(BufferTransform, self).__call__(node)
-
-    def handle_scope(self, node, scope):
-        # For all buffers, insert extra variables in the scope.
-        # The variables are also accessible from the buffer_info
-        # on the buffer entry
-        bufvars = [(name, entry) for name, entry
-                   in scope.entries.iteritems()
-                   if entry.type.buffer_options is not None]
-                   
-        for name, entry in bufvars:
-            # Variable has buffer opts, declare auxiliary vars
-            bufopts = entry.type.buffer_options
-
-            bufinfo = scope.declare_var(temp_name_handle(u"%s_bufinfo" % name),
-                                        self.buffer_type, node.pos)
-
-            temp_var =  scope.declare_var(temp_name_handle(u"%s_tmp" % name),
-                                        entry.type, node.pos)
-            
-            
-            stridevars = []
-            shapevars = []
-            for idx in range(bufopts.ndim):
-                # stride
-                varname = temp_name_handle(u"%s_%s%d" % (name, "stride", idx))
-                var = scope.declare_var(varname, PyrexTypes.c_int_type, node.pos, is_cdef=True)
-                stridevars.append(var)
-                # shape
-                varname = temp_name_handle(u"%s_%s%d" % (name, "shape", idx))
-                var = scope.declare_var(varname, PyrexTypes.c_uint_type, node.pos, is_cdef=True)
-                shapevars.append(var)
-            entry.buffer_aux = Symtab.BufferAux(bufinfo, stridevars, 
-                                                shapevars)
-            entry.buffer_aux.temp_var = temp_var
-        self.scope = scope
-
-            
-    def visit_ModuleNode(self, node):
-        self.handle_scope(node, node.scope)
-        self.visitchildren(node)
-        return node
-
-    def visit_FuncDefNode(self, node):
-        self.handle_scope(node, node.local_scope)
-        self.visitchildren(node)
-        return node
-
-    acquire_buffer_fragment = TreeFragment(u"""
-        TMP = LHS
-        if TMP is not None:
-            __cython__.PyObject_ReleaseBuffer(<__cython__.PyObject*>TMP, &BUFINFO)
-        TMP = RHS
-        __cython__.PyObject_GetBuffer(<__cython__.PyObject*>TMP, &BUFINFO, 0)
-        ASSIGN_AUX
-        LHS = TMP
-    """)
-
-    fetch_strides = TreeFragment(u"""
-        TARGET = BUFINFO.strides[IDX]
-    """)
-
-    fetch_shape = TreeFragment(u"""
-        TARGET = BUFINFO.shape[IDX]
-    """)
-
-#                ass = SingleAssignmentNode(pos=node.pos,
-#                    lhs=NameNode(node.pos, name=entry.name),
-#                    rhs=IndexNode(node.pos,
-#                        base=AttributeNode(node.pos,
-#                            obj=NameNode(node.pos, name=bufaux.buffer_info_var.name),
-#                            attribute=EncodedString("strides")),
-#                        index=IntNode(node.pos, value=EncodedString(idx))))
-#                print ass.dump()
-    def visit_SingleAssignmentNode(self, node):
-        self.visitchildren(node)
-        bufaux = node.lhs.entry.buffer_aux
-        if bufaux is not None:
-            auxass = []
-            for idx, entry in enumerate(bufaux.stridevars):
-                entry.used = True
-                ass = self.fetch_strides.substitute({
-                    u"TARGET": NameNode(node.pos, name=entry.name),
-                    u"BUFINFO": NameNode(node.pos, name=bufaux.buffer_info_var.name),
-                    u"IDX": IntNode(node.pos, value=EncodedString(idx))
-                })
-                auxass.append(ass)
-
-            for idx, entry in enumerate(bufaux.shapevars):
-                entry.used = True
-                ass = self.fetch_shape.substitute({
-                    u"TARGET": NameNode(node.pos, name=entry.name),
-                    u"BUFINFO": NameNode(node.pos, name=bufaux.buffer_info_var.name),
-                    u"IDX": IntNode(node.pos, value=EncodedString(idx))
-                })
-                auxass.append(ass)
-                
-            bufaux.buffer_info_var.used = True
-            acq = self.acquire_buffer_fragment.substitute({
-                u"TMP" : NameNode(pos=node.pos, name=bufaux.temp_var.name),
-                u"LHS" : node.lhs,
-                u"RHS": node.rhs,
-                u"ASSIGN_AUX": StatListNode(node.pos, stats=auxass),
-                u"BUFINFO": NameNode(pos=node.pos, name=bufaux.buffer_info_var.name)
-            }, pos=node.pos)
-            # Note: The below should probably be refactored into something
-            # like fragment.substitute(..., context=self.context), with
-            # TreeFragment getting context.pipeline_until_now() and
-            # applying it on the fragment.
-            acq.analyse_declarations(self.scope)
-            acq.analyse_expressions(self.scope)
-            stats = acq.stats
-#            stats += [node] # Do assignment after successful buffer acquisition
-         #   print acq.dump()
-            return stats
-        else:
-            return node
-
-    buffer_access = TreeFragment(u"""
-        (<unsigned char*>(BUF.buf + OFFSET))[0]
-    """)
-    def visit_IndexNode(self, node):
-        if node.is_buffer_access:
-            assert node.index is None
-            assert node.indices is not None
-            bufaux = node.base.entry.buffer_aux
-            assert bufaux is not None
-            to_sum = [ IntBinopNode(node.pos, operator='*', operand1=index,
-                                    operand2=NameNode(node.pos, name=stride.name))
-                for index, stride in zip(node.indices, bufaux.stridevars)]
-            print to_sum
-
-            indices = node.indices
-            # reduce * on indices
-            expr = to_sum[0]
-            for next in to_sum[1:]:
-                expr = IntBinopNode(node.pos, operator='+', operand1=expr, operand2=next)
-            tmp= self.buffer_access.substitute({
-                'BUF': NameNode(node.pos, name=bufaux.buffer_info_var.name),
-                'OFFSET': expr
-                })
-            tmp.analyse_expressions(self.scope)
-            return tmp.stats[0].expr
-        else:
-            return node
-
-    def visit_CallNode(self, node):
-###        print node.dump()
-        return node
-    
-#    def visit_FuncDefNode(self, node):
-#        print node.dump()
-    
-
 class WithTransform(CythonTransform):
 
     # EXCINFO is manually set to a variable that contains

Cython/Compiler/PyrexTypes.py

@@ -61,6 +61,7 @@ class PyrexType(BaseType):
     #  default_value         string      Initial value
     #  parsetuple_format     string      Format char for PyArg_ParseTuple
     #  pymemberdef_typecode  string      Type code for PyMemberDef struct
+    #  typestring            string      String char defining the type (see Python struct module)
     #
     #  declaration_code(entity_code, 
     #      for_display = 0, dll_linkage = None, pyrex = 0)
@@ -416,9 +417,10 @@ class CNumericType(CType):
     
     sign_words = ("unsigned ", "", "signed ")
     
-    def __init__(self, rank, signed = 1, pymemberdef_typecode = None):
+    def __init__(self, rank, signed = 1, pymemberdef_typecode = None, typestring = None):
         self.rank = rank
         self.signed = signed
+        self.typestring = typestring
         ptf = self.parsetuple_formats[signed][rank]
         if ptf == '?':
             ptf = None
@@ -451,8 +453,9 @@ class CIntType(CNumericType):
     from_py_function = "__pyx_PyInt_AsLong"
     exception_value = -1
 
-    def __init__(self, rank, signed, pymemberdef_typecode = None, is_returncode = 0):
-        CNumericType.__init__(self, rank, signed, pymemberdef_typecode)
+    def __init__(self, rank, signed, pymemberdef_typecode = None, is_returncode = 0,
+                 typestring=None):
+        CNumericType.__init__(self, rank, signed, pymemberdef_typecode, typestring=typestring)
         self.is_returncode = is_returncode
         if self.from_py_function == '__pyx_PyInt_AsLong':
             self.from_py_function = self.get_type_conversion()
@@ -543,8 +546,8 @@ class CFloatType(CNumericType):
     to_py_function = "PyFloat_FromDouble"
     from_py_function = "__pyx_PyFloat_AsDouble"
     
-    def __init__(self, rank, pymemberdef_typecode = None):
-        CNumericType.__init__(self, rank, 1, pymemberdef_typecode)
+    def __init__(self, rank, pymemberdef_typecode = None, typestring=None):
+        CNumericType.__init__(self, rank, 1, pymemberdef_typecode, typestring = typestring)
     
     def assignable_from_resolved_type(self, src_type):
         return src_type.is_numeric or src_type is error_type
@@ -852,9 +855,12 @@ class CFuncTypeArg:
     #  type       PyrexType
     #  pos        source file position
     
-    def __init__(self, name, type, pos):
+    def __init__(self, name, type, pos, cname=None):
         self.name = name
-        self.cname = Naming.var_prefix + name
+        if cname is not None:
+            self.cname = cname
+        else:
+            self.cname = Naming.var_prefix + name
         self.type = type
         self.pos = pos
         self.not_none = False
@@ -1050,29 +1056,29 @@ c_void_type =         CVoidType()
 c_void_ptr_type =     CPtrType(c_void_type)
 c_void_ptr_ptr_type = CPtrType(c_void_ptr_type)
 
-c_uchar_type =       CIntType(0, 0, "T_UBYTE")
-c_ushort_type =      CIntType(1, 0, "T_USHORT")
-c_uint_type =        CUIntType(2, 0, "T_UINT")
-c_ulong_type =       CULongType(3, 0, "T_ULONG")
-c_ulonglong_type =   CULongLongType(4, 0, "T_ULONGLONG")
-
-c_char_type =        CIntType(0, 1, "T_CHAR")
-c_short_type =       CIntType(1, 1, "T_SHORT")
-c_int_type =         CIntType(2, 1, "T_INT")
-c_long_type =        CIntType(3, 1, "T_LONG")
-c_longlong_type =    CLongLongType(4, 1, "T_LONGLONG")
+c_uchar_type =       CIntType(0, 0, "T_UBYTE", typestring="B")
+c_ushort_type =      CIntType(1, 0, "T_USHORT", typestring="H")
+c_uint_type =        CUIntType(2, 0, "T_UINT", typestring="I")
+c_ulong_type =       CULongType(3, 0, "T_ULONG", typestring="L")
+c_ulonglong_type =   CULongLongType(4, 0, "T_ULONGLONG", typestring="Q")
+
+c_char_type =        CIntType(0, 1, "T_CHAR", typestring="b")
+c_short_type =       CIntType(1, 1, "T_SHORT", typestring="h")
+c_int_type =         CIntType(2, 1, "T_INT", typestring="i")
+c_long_type =        CIntType(3, 1, "T_LONG", typestring="l")
+c_longlong_type =    CLongLongType(4, 1, "T_LONGLONG", typestring="q")
 c_py_ssize_t_type =  CPySSizeTType(5, 1)
-c_bint_type =        CBIntType(2, 1, "T_INT")
+c_bint_type =        CBIntType(2, 1, "T_INT", typestring="i")
 
-c_schar_type =       CIntType(0, 2, "T_CHAR")
-c_sshort_type =      CIntType(1, 2, "T_SHORT")
-c_sint_type =        CIntType(2, 2, "T_INT")
-c_slong_type =       CIntType(3, 2, "T_LONG")
-c_slonglong_type =   CLongLongType(4, 2, "T_LONGLONG")
+c_schar_type =       CIntType(0, 2, "T_CHAR", typestring="b")
+c_sshort_type =      CIntType(1, 2, "T_SHORT", typestring="h")
+c_sint_type =        CIntType(2, 2, "T_INT", typestring="i")
+c_slong_type =       CIntType(3, 2, "T_LONG", typestring="l")
+c_slonglong_type =   CLongLongType(4, 2, "T_LONGLONG", typestring="q")
 
-c_float_type =       CFloatType(6, "T_FLOAT")
-c_double_type =      CFloatType(7, "T_DOUBLE")
-c_longdouble_type =  CFloatType(8)
+c_float_type =       CFloatType(6, "T_FLOAT", typestring="f")
+c_double_type =      CFloatType(7, "T_DOUBLE", typestring="d")
+c_longdouble_type =  CFloatType(8, typestring="g")
 
 c_null_ptr_type =     CNullPtrType(c_void_type)
 c_char_array_type =   CCharArrayType(None)
@@ -1087,7 +1093,8 @@ c_returncode_type =   CIntType(2, 1, "T_INT", is_returncode = 1)
 c_anon_enum_type =    CAnonEnumType(-1, 1)
 
 # the Py_buffer type is defined in Builtin.py
-c_py_buffer_ptr_type = CPtrType(CStructOrUnionType("Py_buffer", "struct", None, 1, "Py_buffer"))
+c_py_buffer_type = CStructOrUnionType("Py_buffer", "struct", None, 1, "Py_buffer")
+c_py_buffer_ptr_type = CPtrType(c_py_buffer_type)
 
 error_type =    ErrorType()
 

Cython/Compiler/Symtab.py

@@ -20,10 +20,12 @@ possible_identifier = re.compile(ur"(?![0-9])\w+$", re.U).match
 nice_identifier = re.compile('^[a-zA-Z0-0_]+$').match
 
 class BufferAux:
-    def __init__(self, buffer_info_var, stridevars, shapevars):
+    def __init__(self, buffer_info_var, stridevars, shapevars, tschecker):
         self.buffer_info_var = buffer_info_var
         self.stridevars = stridevars
         self.shapevars = shapevars
+        self.tschecker = tschecker
+        
     def __repr__(self):
         return "<BufferAux %r>" % self.__dict__
 

Cython/Compiler/Visitor.py

@@ -181,10 +181,14 @@ def replace_node(ptr, value):
         getattr(parent, attrname)[listidx] = value
 
 tmpnamectr = 0
-def temp_name_handle(description):
+def temp_name_handle(description=None):
     global tmpnamectr
     tmpnamectr += 1
-    return EncodedString(Naming.temp_prefix + u"%d_%s" % (tmpnamectr, description))
+    if description is not None:
+        name = u"%d_%s" % (tmpnamectr, description)
+    else:
+        name = u"%d" % tmpnamectr
+    return EncodedString(Naming.temp_prefix + name)
 
 def get_temp_name_handle_desc(handle):
     if not handle.startswith(u"__cyt_"):
@@ -198,7 +202,7 @@ class PrintTree(TreeVisitor):
     Subclass and override repr_of to provide more information
     about nodes. """
     def __init__(self):
-        Transform.__init__(self)
+        TreeVisitor.__init__(self)
         self._indent = ""
 
     def indent(self):
@@ -208,6 +212,7 @@ class PrintTree(TreeVisitor):
 
     def __call__(self, tree, phase=None):
         print("Parse tree dump at phase '%s'" % phase)
+        self.visit(tree)
 
     # Don't do anything about process_list, the defaults gives
     # nice-looking name[idx] nodes which will visually appear

Includes/__cython__.pxd

@@ -19,5 +19,10 @@ cdef extern from "Python.h":
     int PyObject_GetBuffer(PyObject* obj, Py_buffer* view, int flags) except -1
     void PyObject_ReleaseBuffer(PyObject* obj, Py_buffer* view)
 
+    void PyErr_Format(int, char*, ...)
+
+    enum:
+        PyExc_TypeError
+
 #                  int PyObject_GetBuffer(PyObject *obj, Py_buffer *view,
 #                       int flags)

Includes/numpy.pxd

+cdef extern from "Python.h":
+    ctypedef int Py_intptr_t
+    
+cdef extern from "numpy/arrayobject.h":
+    ctypedef class numpy.ndarray [object PyArrayObject]:
+        cdef char *data
+        cdef int nd
+        cdef Py_intptr_t *dimensions
+        cdef Py_intptr_t *strides
+        cdef object base
+        # descr not implemented yet here...
+        cdef int flags
+        cdef int itemsize
+        cdef object weakreflist
+
+    ctypedef unsigned int npy_uint8
+    ctypedef unsigned int npy_uint16
+    ctypedef unsigned int npy_uint32
+    ctypedef unsigned int npy_uint64
+    ctypedef unsigned int npy_uint96
+    ctypedef unsigned int npy_uint128
+    ctypedef signed int   npy_int64
+
+    ctypedef float        npy_float32
+    ctypedef float        npy_float64
+    ctypedef float        npy_float80
+    ctypedef float        npy_float96
+    ctypedef float        npy_float128
+
+ctypedef npy_int64 Tint64