merge

Cython/Compiler/CythonScope.py

@@ -15,6 +15,12 @@ class CythonScope(ModuleScope):
                                                   pos=None,
                                                   defining = 1,
                                                   cname='<error>')
+                                                  
+    def lookup_type(self, name):
+        # This function should go away when types are all first-level objects. 
+        type = parse_basic_type(name)
+        if type:
+            return type
 
 def create_cython_scope(context):
     return CythonScope(context)

Cython/Compiler/ExprNodes.py

@@ -568,6 +568,9 @@ class ExprNode(Node):
         #  a constant, local var, C global var, struct member
         #  reference, or temporary.
         return self.result_in_temp()
+        
+    def as_cython_attribute(self):
+        return None
 
 
 class NewTempExprNode(ExprNode):
@@ -780,6 +783,9 @@ class StringNode(ConstNode):
         self.entry = env.add_string_const(self.value)
         
     def analyse_as_type(self, env):
+        type = PyrexTypes.parse_basic_type(self.value)
+        if type is not None:    
+            return type
         from TreeFragment import TreeFragment
         pos = (self.pos[0], self.pos[1], self.pos[2]-7)
         declaration = TreeFragment(u"sizeof(%s)" % self.value, name=pos[0].filename, initial_pos=pos)
@@ -914,7 +920,8 @@ class NameNode(AtomicExprNode):
     #  entry           Entry     Symbol table entry
     #  interned_cname  string
     
-    is_name = 1
+    is_name = True
+    is_cython_module = False
     skip_assignment_decref = False
     entry = None
 
@@ -960,8 +967,9 @@ class NameNode(AtomicExprNode):
         return None
         
     def analyse_as_type(self, env):
-        if self.name in PyrexTypes.rank_to_type_name:
-            return PyrexTypes.simple_c_type(1, 0, self.name)
+        type = PyrexTypes.parse_basic_type(self.name)
+        if type:
+            return type
         entry = self.entry
         if not entry:
             entry = env.lookup(self.name)
@@ -1869,6 +1877,21 @@ class SimpleCallNode(CallNode):
             return function(*args)
         except Exception, e:
             self.compile_time_value_error(e)
+            
+    def analyse_as_type(self, env):
+        attr = self.function.as_cython_attribute()
+        if attr == 'pointer':
+            if len(self.args) != 1:
+                error(self.args.pos, "only one type allowed.")
+            else:
+                type = self.args[0].analyse_as_type(env)
+                if not type:
+                    error(self.args[0].pos, "Unknown type")
+                else:
+                    return PyrexTypes.CPtrType(type)
+
+    def explicit_args_kwds(self):
+        return self.args, None
 
     def analyse_types(self, env):
         function = self.function
@@ -2098,6 +2121,12 @@ class GeneralCallNode(CallNode):
             return function(*positional_args, **keyword_args)
         except Exception, e:
             self.compile_time_value_error(e)
+            
+    def explicit_args_kwds(self):
+        if self.starstar_arg or not isinstance(self.positional_args, TupleNode):
+            raise PostParseError(self.pos,
+                'Compile-time keyword arguments must be explicit.')
+        return self.positional_args.args, self.keyword_args
 
     def analyse_types(self, env):
         self.function.analyse_types(env)
@@ -2201,6 +2230,10 @@ class AttributeNode(ExprNode):
     is_called = 0
     needs_none_check = True
 
+    def as_cython_attribute(self):
+        if isinstance(self.obj, NameNode) and self.obj.is_cython_module:
+            return self.attribute
+
     def coerce_to(self, dst_type, env):
         #  If coercing to a generic pyobject and this is a cpdef function
         #  we can create the corresponding attribute
@@ -2278,9 +2311,7 @@ class AttributeNode(ExprNode):
     def analyse_as_type(self, env):
         module_scope = self.obj.analyse_as_module(env)
         if module_scope:
-            entry = module_scope.lookup_here(self.attribute)
-            if entry and entry.is_type:
-                return entry.type
+            return module_scope.lookup_type(self.attribute)
         return None
     
     def analyse_as_extension_type(self, env):

Cython/Compiler/Main.py

@@ -74,12 +74,13 @@ class Context:
             os.path.join(os.path.dirname(__file__), '..', 'Includes'))
         self.include_directories = include_directories + [standard_include_path]
 
-    def create_pipeline(self, pxd):
+    def create_pipeline(self, pxd, py=False):
         from Visitor import PrintTree
         from ParseTreeTransforms import WithTransform, NormalizeTree, PostParse, PxdPostParse
         from ParseTreeTransforms import AnalyseDeclarationsTransform, AnalyseExpressionsTransform
         from ParseTreeTransforms import CreateClosureClasses, MarkClosureVisitor, DecoratorTransform
         from ParseTreeTransforms import InterpretCompilerDirectives, TransformBuiltinMethods
+        from ParseTreeTransforms import AlignFunctionDefinitions
         from AutoDocTransforms import EmbedSignature
         from Optimize import FlattenInListTransform, SwitchTransform, FinalOptimizePhase
         from Buffer import IntroduceBufferAuxiliaryVars
@@ -91,11 +92,17 @@ class Context:
         else:
             _check_c_classes = check_c_classes
             _specific_post_parse = None
+            
+        if py and not pxd:
+            _align_function_definitions = AlignFunctionDefinitions(self)
+        else:
+            _align_function_definitions = None
  
         return [
             NormalizeTree(self),
             PostParse(self),
             _specific_post_parse,
+            _align_function_definitions,
             InterpretCompilerDirectives(self, self.pragma_overrides),
             FlattenInListTransform(),
             WithTransform(self),
@@ -112,7 +119,7 @@ class Context:
             #        CreateClosureClasses(context),
             ]
 
-    def create_pyx_pipeline(self, options, result):
+    def create_pyx_pipeline(self, options, result, py=False):
         def generate_pyx_code(module_node):
             module_node.process_implementation(options, result)
             result.compilation_source = module_node.compilation_source
@@ -134,7 +141,7 @@ class Context:
 
         return ([
                 create_parse(self),
-            ] + self.create_pipeline(pxd=False) + [
+            ] + self.create_pipeline(pxd=False, py=py) + [
                 inject_pxd_code,
                 generate_pyx_code,
             ])
@@ -154,6 +161,10 @@ class Context:
         return [parse_pxd] + self.create_pipeline(pxd=True) + [
             ExtractPxdCode(self),
             ]
+            
+    def create_py_pipeline(self, options, result):
+        return self.create_pyx_pipeline(options, result, py=True)
+
 
     def process_pxd(self, source_desc, scope, module_name):
         pipeline = self.create_pxd_pipeline(scope, module_name)
@@ -504,7 +515,10 @@ def run_pipeline(source, options, full_module_name = None):
     result = create_default_resultobj(source, options)
     
     # Get pipeline
-    pipeline = context.create_pyx_pipeline(options, result)
+    if source_desc.filename.endswith(".py"):
+        pipeline = context.create_py_pipeline(options, result)
+    else:
+        pipeline = context.create_pyx_pipeline(options, result)
 
     context.setup_errors(options)
     err, enddata = context.run_pipeline(pipeline, source)

Cython/Compiler/ParseTreeTransforms.py

@@ -313,10 +313,18 @@ class InterpretCompilerDirectives(CythonTransform):
     def visit_SingleAssignmentNode(self, node):
         if (isinstance(node.rhs, ImportNode) and
                 node.rhs.module_name.value == u'cython'):
-            self.cython_module_names.add(node.lhs.name)
+            node = CImportStatNode(node.pos, 
+                                   module_name = u'cython',
+                                   as_name = node.lhs.name)
+            self.visit_CImportStatNode(node)
         else:
             self.visitchildren(node)
-            return node
+        return node
+            
+    def visit_NameNode(self, node):
+        if node.name in self.cython_module_names:
+            node.is_cython_module = True
+        return node
 
     def visit_Node(self, node):
         self.visitchildren(node)
@@ -339,15 +347,7 @@ class InterpretCompilerDirectives(CythonTransform):
         if optname:
             optiontype = Options.option_types.get(optname)
             if optiontype:
-                if isinstance(node, SimpleCallNode):
-                    args = node.args
-                    kwds = None
-                else:
-                    if node.starstar_arg or not isinstance(node.positional_args, TupleNode):
-                        raise PostParseError(dec.function.pos,
-                            'Compile-time keyword arguments must be explicit.' % optname)
-                    args = node.positional_args.args
-                    kwds = node.keyword_args
+                args, kwds = node.explicit_args_kwds()
                 if optiontype is bool:
                     if kwds is not None or len(args) != 1 or not isinstance(args[0], BoolNode):
                         raise PostParseError(dec.function.pos,
@@ -516,7 +516,7 @@ property NAME:
         node.analyse_declarations(self.env_stack[-1])
         self.visitchildren(node)
         return node
-
+        
     def visit_FuncDefNode(self, node):
         lenv = node.create_local_scope(self.env_stack[-1])
         node.body.analyse_control_flow(lenv) # this will be totally refactored
@@ -573,6 +573,54 @@ class AnalyseExpressionsTransform(CythonTransform):
         node.body.analyse_expressions(node.local_scope)
         self.visitchildren(node)
         return node
+        
+class AlignFunctionDefinitions(CythonTransform):
+    """
+    This class takes the signatures from a .pxd file and applies them to 
+    the def methods in a .py file. 
+    """
+    
+    def visit_ModuleNode(self, node):
+        self.scope = node.scope
+        self.visitchildren(node)
+        return node
+    
+    def visit_PyClassDefNode(self, node):
+        pxd_def = self.scope.lookup(node.name)
+        if pxd_def:
+            if pxd_def.is_cclass:
+                return self.visit_CClassDefNode(node.as_cclass(), pxd_def)
+            else:
+                error(node.pos, "'%s' redeclared" % node.name)
+                error(pxd_def.pos, "previous declaration here")
+                return None
+        self.visitchildren(node)
+        return node
+        
+    def visit_CClassDefNode(self, node, pxd_def=None):
+        if pxd_def is None:
+            pxd_def = self.scope.lookup(node.class_name)
+        if pxd_def:
+            outer_scope = self.scope
+            self.scope = pxd_def.type.scope
+        self.visitchildren(node)
+        if pxd_def:
+            self.scope = outer_scope
+        return node
+        
+    def visit_DefNode(self, node):
+        pxd_def = self.scope.lookup(node.name)
+        if pxd_def:
+            if pxd_def.is_cfunction:
+                node = node.as_cfunction(pxd_def)
+            else:
+                error(node.pos, "'%s' redeclared" % node.name)
+                error(pxd_def.pos, "previous declaration here")
+                return None
+        # Enable this when internal def functions are allowed. 
+        # self.visitchildren(node)
+        return node
+        
 
 class MarkClosureVisitor(CythonTransform):
     
@@ -638,24 +686,20 @@ class EnvTransform(CythonTransform):
 
 class TransformBuiltinMethods(EnvTransform):
 
-    def cython_attribute(self, node):
-        if (isinstance(node, AttributeNode) and 
-            isinstance(node.obj, NameNode) and
-            node.obj.name in self.cython_module_names):
-            return node.attribute
+    def visit_SingleAssignmentNode(self, node):
+        if node.declaration_only:
+            return None
+        else:
+            self.visitchildren(node)
+            return node
     
-    def visit_ModuleNode(self, node):
-        self.cython_module_names = node.cython_module_names
-        self.visitchildren(node)
-        return node
-        
     def visit_AttributeNode(self, node):
-        attribute = self.cython_attribute(node)
+        attribute = node.as_cython_attribute()
         if attribute:
             if attribute == u'compiled':
                 node = BoolNode(node.pos, value=True)
             else:
-                error(node.function.pos, u"'%s' not a valid cython attribute" % function)
+                error(node.pos, u"'%s' not a valid cython attribute" % attribute)
         return node
 
     def visit_SimpleCallNode(self, node):
@@ -671,7 +715,7 @@ class TransformBuiltinMethods(EnvTransform):
                 return ExprNodes.DictNode(pos, key_value_pairs=items)
 
         # cython.foo
-        function = self.cython_attribute(node.function)
+        function = node.function.as_cython_attribute()
         if function:
             if function == u'cast':
                 if len(node.args) != 2:
@@ -691,6 +735,11 @@ class TransformBuiltinMethods(EnvTransform):
                         node = SizeofTypeNode(node.function.pos, arg_type=type)
                     else:
                         node = SizeofVarNode(node.function.pos, operand=node.args[0])
+            elif function == 'address':
+                if len(node.args) != 1:
+                    error(node.function.pos, u"sizeof takes exactly one argument" % function)
+                else:
+                    node = AmpersandNode(node.function.pos, operand=node.args[0])
             else:
                 error(node.function.pos, u"'%s' not a valid cython language construct" % function)
         

Cython/Compiler/PyrexTypes.py

@@ -1187,6 +1187,7 @@ modifiers_and_name_to_type = {
     (2, 0, "Py_ssize_t"): c_py_ssize_t_type,
     
     (1, 0, "long"): c_long_type,
+    (1, 0, "longlong"): c_longlong_type,
     (1, 0, "bint"): c_bint_type,
 }
 
@@ -1210,6 +1211,21 @@ def simple_c_type(signed, longness, name):
     # Find type descriptor for simple type given name and modifiers.
     # Returns None if arguments don't make sense.
     return modifiers_and_name_to_type.get((signed, longness, name))
+    
+def parse_basic_type(name):
+    base = None
+    if name.startswith('p_'):
+        base = parse_basic_type(name[2:])
+    elif name.startswith('p'):
+        base = parse_basic_type(name[1:])
+    elif name.endswith('*'):
+        base = parse_basic_type(name[:-1])
+    if base:
+        return CPtrType(base)
+    elif name.startswith('u'):
+        return simple_c_type(0, 0, name[1:])
+    else:
+        return simple_c_type(1, 0, name)
 
 def c_array_type(base_type, size):
     # Construct a C array type.

Cython/Compiler/Symtab.py

@@ -56,6 +56,7 @@ class Entry:
     # is_cmethod       boolean    Is a C method of an extension type
     # is_unbound_cmethod boolean  Is an unbound C method of an extension type
     # is_type          boolean    Is a type definition
+    # is_cclass        boolean    Is an extension class
     # is_const         boolean    Is a constant
     # is_property      boolean    Is a property of an extension type:
     # doc_cname        string or None  C const holding the docstring
@@ -108,6 +109,7 @@ class Entry:
     is_cmethod = 0
     is_unbound_cmethod = 0
     is_type = 0
+    is_cclass = 0
     is_const = 0
     is_property = 0
     doc_cname = None
@@ -500,6 +502,11 @@ class Scope:
         if not entry:
             entry = self.declare_var(name, py_object_type, None)
         return entry
+        
+    def lookup_type(self, name):
+        entry = self.lookup(name)
+        if entry and entry.is_type:
+            return entry.type
 
     def add_string_const(self, value, identifier = False):
         # Add an entry for a string constant.
@@ -989,6 +996,7 @@ class ModuleScope(Scope):
             type.typeptr_cname = self.mangle(Naming.typeptr_prefix, name)
             entry = self.declare_type(name, type, pos, visibility = visibility,
                 defining = 0)
+            entry.is_cclass = True
             if objstruct_cname:
                 type.objstruct_cname = objstruct_cname
             elif not entry.in_cinclude:

Cython/Shadow.py

@@ -6,17 +6,157 @@ def empty_decorator(x):
 def locals(**arg_types):
     return empty_decorator
 
+
+# Emulated language constructs
+
 def cast(type, arg):
-    # can/should we emulate anything here?
-    return arg
+    if callable(type):
+        return type(arg)
+    else:
+        return arg
 
 def sizeof(arg):
-    # can/should we emulate anything here?
     return 1
+    
+def address(arg):
+    return pointer(type(arg))([arg])
+    
+def declare(type):
+    if callable(type):
+        return type()
+    else:
+        return None
+
+# Emulated types
+
+class CythonType(object):
+
+    def _pointer(self, n=1):
+        for i in range(n):
+            self = pointer(self)
+        return self
+
+    def __getitem__(self, ix):
+        return array(self, ix)
+
+
+class PointerType(CythonType):
+
+    def __init__(self, value=None):
+        if isinstance(value, ArrayType):
+            self._items = [cast(self._basetype, a) for a in value._items]
+        elif isinstance(value, list):
+            self._items = [cast(self._basetype, a) for a in value]
+        elif value is None:
+            self._items = []
+        else:
+            raise ValueError
+            
+    def __getitem__(self, ix):
+        if ix < 0:
+            raise IndexError, "negative indexing not allowed in C"
+        return self._items[ix]
+        
+    def __setitem__(self, ix, value):
+        if ix < 0:
+            raise IndexError, "negative indexing not allowed in C"
+        self._items[ix] = cast(self._basetype, value)
+        
+class ArrayType(PointerType):
+    
+    def __init__(self):
+        self._items = [None] * self._n
+
+
+class StructType(CythonType):
+    
+    def __init__(self, **data):
+        for key, value in data.items():
+            setattr(self, key, value)
+            
+    def __setattr__(self, key, value):
+        if key in self._members:
+            self.__dict__[key] = cast(self._members[key], value)
+        else:
+            raise AttributeError, "Struct has no member '%s'" % key
+    
+
+class UnionType(CythonType):
+
+    def __init__(self, **data):
+        if len(data) > 0:
+            raise AttributeError, "Union can only store one field at a time."
+        for key, value in data.items():
+            setattr(self, key, value)
+            
+    def __setattr__(self, key, value):
+        if key in '__dict__':
+            CythonType.__setattr__(self, key, value)
+        elif key in self._members:
+            self.__dict__ = {key: cast(self._members[key], value)}
+        else:
+            raise AttributeError, "Union has no member '%s'" % key
+
+def pointer(basetype):
+    class PointerInstance(PointerType):
+        _basetype = basetype
+    return PointerInstance
+
+def array(basetype, n):
+    class ArrayInstance(ArrayType):
+        _basetype = basetype
+        _n = n
+    return ArrayInstance
+
+def struct(**members):
+    class StructInstance(StructType):
+        _members = members
+    for key in members.keys():
+        setattr(StructInstance, key, None)
+    return StructInstance
+
+def union(**members):
+    class UnionInstance(UnionType):
+        _members = members
+    for key in members.keys():
+        setattr(UnionInstance, key, None)
+    return UnionInstance
+
+class typedef(CythonType):
+
+    def __init__(self, type):
+        self._basetype = type
+    
+    def __call__(self, value=None):
+        if value is not None:
+            value = cast(self._basetype, value)
+        return value
+        
+
 
 py_int = int
 py_long = long
 py_float = float
 
-# They just have to exist...
-int = long = char = bint = uint = ulong = longlong = ulonglong = Py_ssize_t = float = double = None
+
+# Predefined types
+
+int_types = ['char', 'short', 'int', 'long', 'longlong', 'Py_ssize_t'] 
+float_types = ['double', 'float']
+other_types = ['bint', 'Py_ssize_t', 'void']
+gs = globals()
+
+for name in int_types:
+    gs[name] = typedef(py_int)
+    gs['u'+name] = typedef(py_int)
+    
+double = float = typedef(py_float)
+bint = typedef(bool)
+void = typedef(int)
+
+for t in int_types + float_types + other_types:
+    for i in range(1, 4):
+        gs["%s_%s" % ('p'*i, t)] = globals()[t]._pointer(i)
+
+void = typedef(None)
+NULL = None