merge

Cython/Compiler/AnalysedTreeTransforms.py

 from Cython.Compiler.Visitor import VisitorTransform, ScopeTrackingTransform, TreeVisitor
 from Nodes import StatListNode, SingleAssignmentNode, CFuncDefNode
-from ExprNodes import (DictNode, DictItemNode, NameNode, UnicodeNode, NoneNode,
-                      ExprNode, AttributeNode, ModuleRefNode, DocstringRefNode)
+from ExprNodes import DictNode, DictItemNode, NameNode, UnicodeNode, NoneNode, \
+                      ExprNode, AttributeNode, ModuleRefNode, DocstringRefNode
 from PyrexTypes import py_object_type
 from Builtin import dict_type
 from StringEncoding import EncodedString

Cython/Compiler/Main.py

@@ -87,6 +87,7 @@ class Context(object):
         from ParseTreeTransforms import AnalyseDeclarationsTransform, AnalyseExpressionsTransform
         from ParseTreeTransforms import CreateClosureClasses, MarkClosureVisitor, DecoratorTransform
         from ParseTreeTransforms import InterpretCompilerDirectives, TransformBuiltinMethods
+        from TypeInference import MarkAssignments
         from ParseTreeTransforms import AlignFunctionDefinitions, GilCheck
         from AnalysedTreeTransforms import AutoTestDictTransform
         from AutoDocTransforms import EmbedSignature
@@ -129,6 +130,7 @@ class Context(object):
             AnalyseDeclarationsTransform(self),
             AutoTestDictTransform(self),
             EmbedSignature(self),
+            MarkAssignments(self),
             TransformBuiltinMethods(self),
             IntroduceBufferAuxiliaryVars(self),
             _check_c_declarations,

Cython/Compiler/Nodes.py

@@ -4,6 +4,12 @@
 
 import sys, os, time, copy
 
+try:
+    set
+except NameError:
+    # Python 2.3
+    from sets import Set as set
+
 import Code
 import Builtin
 from Errors import error, warning, InternalError
@@ -3197,6 +3203,10 @@ class InPlaceAssignmentNode(AssignmentNode):
         self.lhs.annotate(code)
         self.rhs.annotate(code)
         self.dup.annotate(code)
+    
+    def create_binop_node(self):
+        import ExprNodes
+        return ExprNodes.binop_node(self.pos, self.operator, self.lhs, self.rhs)
 
 
 class PrintStatNode(StatNode):

Cython/Compiler/Options.py

@@ -68,6 +68,7 @@ option_defaults = {
     'c99_complex' : False, # Don't use macro wrappers for complex arith, not sure what to name this...
     'callspec' : "",
     'profile': False,
+    'infer_types': False,
     'autotestdict': True,
 
 # test support

Cython/Compiler/ParseTreeTransforms.py

@@ -328,7 +328,7 @@ class InterpretCompilerDirectives(CythonTransform, SkipDeclarations):
     duplication of functionality has to occur: We manually track cimports
     and which names the "cython" module may have been imported to.
     """
-    special_methods = set(['declare', 'union', 'struct', 'typedef', 'sizeof', 'cast', 'address', 'pointer', 'compiled', 'NULL'])
+    special_methods = set(['declare', 'union', 'struct', 'typedef', 'sizeof', 'typeof', 'cast', 'address', 'pointer', 'compiled', 'NULL'])
 
     def __init__(self, context, compilation_option_overrides):
         super(InterpretCompilerDirectives, self).__init__(context)
@@ -785,11 +785,13 @@ property NAME:
 class AnalyseExpressionsTransform(CythonTransform):
 
     def visit_ModuleNode(self, node):
+        node.scope.infer_types()
         node.body.analyse_expressions(node.scope)
         self.visitchildren(node)
         return node
         
     def visit_FuncDefNode(self, node):
+        node.local_scope.infer_types()
         node.body.analyse_expressions(node.local_scope)
         self.visitchildren(node)
         return node
@@ -1007,6 +1009,11 @@ class TransformBuiltinMethods(EnvTransform):
                         node = SizeofTypeNode(node.function.pos, arg_type=type)
                     else:
                         node = SizeofVarNode(node.function.pos, operand=node.args[0])
+            elif function == 'typeof':
+                if len(node.args) != 1:
+                    error(node.function.pos, u"sizeof takes exactly one argument" % function)
+                else:
+                    node = TypeofNode(node.function.pos, operand=node.args[0])
             elif function == 'address':
                 if len(node.args) != 1:
                     error(node.function.pos, u"sizeof takes exactly one argument" % function)

Cython/Compiler/PyrexTypes.py

@@ -77,6 +77,7 @@ class PyrexType(BaseType):
     #
         
     is_pyobject = 0
+    is_unspecified = 0
     is_extension_type = 0
     is_builtin_type = 0
     is_numeric = 0
@@ -849,9 +850,17 @@ class CComplexType(CNumericType):
     def __hash__(self):
         return ~hash(self.real_type)
     
+    def declaration_code(self, entity_code, 
+            for_display = 0, dll_linkage = None, pyrex = 0):
+        if for_display:
+            base = public_decl(self.real_type.sign_and_name() + " complex", dll_linkage)
+        else:
+            base = public_decl(self.sign_and_name(), dll_linkage)
+        return self.base_declaration_code(base,  entity_code)
+
     def sign_and_name(self):
         return Naming.type_prefix + self.real_type.specalization_name() + "_complex"
-
+    
     def assignable_from_resolved_type(self, src_type):
         return (src_type.is_complex and self.real_type.assignable_from_resolved_type(src_type.real_type)
                     or src_type.is_numeric and self.real_type.assignable_from_resolved_type(src_type) 
@@ -1591,6 +1600,8 @@ class CUCharPtrType(CStringType, CPtrType):
 
 class UnspecifiedType(PyrexType):
     # Used as a placeholder until the type can be determined.
+    
+    is_unspecified = 1
         
     def declaration_code(self, entity_code, 
             for_display = 0, dll_linkage = None, pyrex = 0):
@@ -1788,6 +1799,23 @@ def widest_numeric_type(type1, type2):
         return sign_and_rank_to_type[min(type1.signed, type2.signed), max(type1.rank, type2.rank)]
     return widest_type
 
+def spanning_type(type1, type2):
+    # Return a type assignable from both type1 and type2.
+    if type1 is py_object_type or type2 is py_object_type:
+        return py_object_type
+    elif type1 == type2:
+        return type1
+    elif type1.is_numeric and type2.is_numeric:
+        return widest_numeric_type(type1, type2)
+    elif type1.is_pyobject ^ type2.is_pyobject:
+        return py_object_type
+    elif type1.assignable_from(type2):
+        return type1
+    elif type2.assignable_from(type1):
+        return type2
+    else:
+        return py_object_type
+    
 def simple_c_type(signed, longness, name):
     # Find type descriptor for simple type given name and modifiers.
     # Returns None if arguments don't make sense.

Cython/Compiler/Symtab.py

@@ -8,7 +8,7 @@ from Errors import warning, error, InternalError
 from StringEncoding import EncodedString
 import Options, Naming
 import PyrexTypes
-from PyrexTypes import py_object_type
+from PyrexTypes import py_object_type, unspecified_type
 import TypeSlots
 from TypeSlots import \
     pyfunction_signature, pymethod_signature, \
@@ -114,9 +114,10 @@ class Entry(object):
     # api              boolean    Generate C API for C class or function
     # utility_code     string     Utility code needed when this entry is used
     #
-    # buffer_aux      BufferAux or None  Extra information needed for buffer variables
+    # buffer_aux       BufferAux or None  Extra information needed for buffer variables
     # inline_func_in_pxd boolean  Hacky special case for inline function in pxd file.
     #                             Ideally this should not be necesarry.
+    # assignments      [ExprNode] List of expressions that get assigned to this entry.
 
     inline_func_in_pxd = False
     borrowed = 0
@@ -171,6 +172,10 @@ class Entry(object):
         self.type = type
         self.pos = pos
         self.init = init
+        self.assignments = []
+    
+    def __repr__(self):
+        return "Entry(name=%s, type=%s)" % (self.name, self.type)
         
     def redeclared(self, pos):
         error(pos, "'%s' does not match previous declaration" % self.name)
@@ -542,6 +547,10 @@ class Scope(object):
             if name in self.entries:    
                 return 1
         return 0
+    
+    def infer_types(self):
+        from TypeInference import get_type_inferer
+        get_type_inferer().infer_types(self)
 
 class PreImportScope(Scope):
 
@@ -814,6 +823,8 @@ class ModuleScope(Scope):
         if not visibility in ('private', 'public', 'extern'):
             error(pos, "Module-level variable cannot be declared %s" % visibility)
         if not is_cdef:
+            if type is unspecified_type:
+                type = py_object_type
             if not (type.is_pyobject and not type.is_extension_type):
                 raise InternalError(
                     "Non-cdef global variable is not a generic Python object")
@@ -1043,6 +1054,10 @@ class ModuleScope(Scope):
         var_entry.is_cglobal = 1
         var_entry.is_readonly = 1
         entry.as_variable = var_entry
+    
+    def infer_types(self):
+        from TypeInference import PyObjectTypeInferer
+        PyObjectTypeInferer().infer_types(self)
         
 class LocalScope(Scope):    
 
@@ -1074,7 +1089,7 @@ class LocalScope(Scope):
             cname, visibility, is_cdef)
         if type.is_pyobject and not Options.init_local_none:
             entry.init = "0"
-        entry.init_to_none = type.is_pyobject and Options.init_local_none
+        entry.init_to_none = (type.is_pyobject or type.is_unspecified) and Options.init_local_none
         entry.is_local = 1
         self.var_entries.append(entry)
         return entry
@@ -1189,6 +1204,8 @@ class PyClassScope(ClassScope):
     
     def declare_var(self, name, type, pos, 
             cname = None, visibility = 'private', is_cdef = 0):
+        if type is unspecified_type:
+            type = py_object_type
         # Add an entry for a class attribute.
         entry = Scope.declare_var(self, name, type, pos, 
             cname, visibility, is_cdef)
@@ -1275,6 +1292,8 @@ class CClassScope(ClassScope):
                     "Non-generic Python attribute cannot be exposed for writing from Python")
             return entry
         else:
+            if type is unspecified_type:
+                type = py_object_type
             # Add an entry for a class attribute.
             entry = Scope.declare_var(self, name, type, pos, 
                 cname, visibility, is_cdef)

Cython/Compiler/TypeInference.py

+import ExprNodes
+from PyrexTypes import py_object_type, unspecified_type, spanning_type
+from Visitor import CythonTransform
+
+try:
+    set
+except NameError:
+    # Python 2.3
+    from sets import Set as set
+
+
+class TypedExprNode(ExprNodes.ExprNode):
+    # Used for declaring assignments of a specified type whithout a known entry.
+    def __init__(self, type):
+        self.type = type
+
+object_expr = TypedExprNode(py_object_type)
+
+class MarkAssignments(CythonTransform):
+    
+    def mark_assignment(self, lhs, rhs):
+        if isinstance(lhs, ExprNodes.NameNode):
+            if lhs.entry is None:
+                # TODO: This shouldn't happen...
+                # It looks like comprehension loop targets are not declared soon enough.
+                return
+            lhs.entry.assignments.append(rhs)
+        elif isinstance(lhs, ExprNodes.SequenceNode):
+            for arg in lhs.args:
+                self.mark_assignment(arg, object_expr)
+        else:
+            # Could use this info to infer cdef class attributes...
+            pass
+    
+    def visit_SingleAssignmentNode(self, node):
+        self.mark_assignment(node.lhs, node.rhs)
+        self.visitchildren(node)
+        return node
+
+    def visit_CascadedAssignmentNode(self, node):
+        for lhs in node.lhs_list:
+            self.mark_assignment(lhs, node.rhs)
+        self.visitchildren(node)
+        return node
+    
+    def visit_InPlaceAssignmentNode(self, node):
+        self.mark_assignment(node.lhs, node.create_binop_node())
+        self.visitchildren(node)
+        return node
+
+    def visit_ForInStatNode(self, node):
+        # TODO: Remove redundancy with range optimization...
+        is_range = False
+        sequence = node.iterator.sequence
+        if isinstance(sequence, ExprNodes.SimpleCallNode):
+            function = sequence.function
+            if sequence.self is None and \
+                    isinstance(function, ExprNodes.NameNode) and \
+                    function.name in ('range', 'xrange'):
+                is_range = True
+                self.mark_assignment(node.target, sequence.args[0])
+                if len(sequence.args) > 1:
+                    self.mark_assignment(node.target, sequence.args[1])
+                    if len(sequence.args) > 2:
+                        self.mark_assignment(node.target, 
+                                 ExprNodes.binop_node(node.pos,
+                                                      '+',
+                                                      sequence.args[0],
+                                                      sequence.args[2]))
+        if not is_range:
+            self.mark_assignment(node.target, object_expr)
+        self.visitchildren(node)
+        return node
+
+    def visit_ForFromStatNode(self, node):
+        self.mark_assignment(node.target, node.bound1)
+        if node.step is not None:
+            self.mark_assignment(node.target,
+                    ExprNodes.binop_node(node.pos, 
+                                         '+', 
+                                         node.bound1, 
+                                         node.step))
+        self.visitchildren(node)
+        return node
+
+    def visit_ExceptClauseNode(self, node):
+        if node.target is not None:
+            self.mark_assignment(node.target, object_expr)
+        self.visitchildren(node)
+        return node
+    
+    def visit_FromCImportStatNode(self, node):
+        pass # Can't be assigned to...
+
+    def visit_FromImportStatNode(self, node):
+        for name, target in node.items:
+            if name != "*":
+                self.mark_assignment(target, object_expr)
+        self.visitchildren(node)
+        return node
+
+
+class PyObjectTypeInferer:
+    """
+    If it's not declared, it's a PyObject.
+    """
+    def infer_types(self, scope):
+        """
+        Given a dict of entries, map all unspecified types to a specified type.
+        """
+        for name, entry in scope.entries.items():
+            if entry.type is unspecified_type:
+                entry.type = py_object_type
+
+class SimpleAssignmentTypeInferer:
+    """
+    Very basic type inference.
+    """
+    # TODO: Implement a real type inference algorithm.
+    # (Something more powerful than just extending this one...)
+    def infer_types(self, scope):
+        dependancies_by_entry = {} # entry -> dependancies
+        entries_by_dependancy = {} # dependancy -> entries
+        ready_to_infer = []
+        for name, entry in scope.entries.items():
+            if entry.type is unspecified_type:
+                all = set()
+                for expr in entry.assignments:
+                    all.update(expr.type_dependencies(scope))
+                if all:
+                    dependancies_by_entry[entry] = all
+                    for dep in all:
+                        if dep not in entries_by_dependancy:
+                            entries_by_dependancy[dep] = set([entry])
+                        else:
+                            entries_by_dependancy[dep].add(entry)
+                else:
+                    ready_to_infer.append(entry)
+        def resolve_dependancy(dep):
+            if dep in entries_by_dependancy:
+                for entry in entries_by_dependancy[dep]:
+                    entry_deps = dependancies_by_entry[entry]
+                    entry_deps.remove(dep)
+                    if not entry_deps and entry != dep:
+                        del dependancies_by_entry[entry]
+                        ready_to_infer.append(entry)
+        # Try to infer things in order...
+        while True:
+            while ready_to_infer:
+                entry = ready_to_infer.pop()
+                types = [expr.infer_type(scope) for expr in entry.assignments]
+                if types:
+                    entry.type = reduce(spanning_type, types)
+                else:
+                    # List comprehension?
+                    # print "No assignments", entry.pos, entry
+                    entry.type = py_object_type
+                resolve_dependancy(entry)
+            # Deal with simple circular dependancies...
+            for entry, deps in dependancies_by_entry.items():
+                if len(deps) == 1 and deps == set([entry]):
+                    types = [expr.infer_type(scope) for expr in entry.assignments if expr.type_dependencies(scope) == ()]
+                    if types:
+                        entry.type = reduce(spanning_type, types)
+                        types = [expr.infer_type(scope) for expr in entry.assignments]
+                        entry.type = reduce(spanning_type, types) # might be wider...
+                        resolve_dependancy(entry)
+                        del dependancies_by_entry[entry]
+                        if ready_to_infer:
+                            break
+            if not ready_to_infer:
+                break
+                    
+        # We can't figure out the rest with this algorithm, let them be objects.
+        for entry in dependancies_by_entry:
+            entry.type = py_object_type
+
+def get_type_inferer():
+    return SimpleAssignmentTypeInferer()

Cython/Shadow.py

@@ -30,6 +30,9 @@ def cast(type, arg):
 
 def sizeof(arg):
     return 1
+
+def typeof(arg):
+    return type(arg)
     
 def address(arg):
     return pointer(type(arg))([arg])

Demos/embed/main.c

-#include "Python.h"
-#include "embedded.h"
-
-int main(int argc, char *argv) {
-  Py_Initialize();
-  initembedded();
-  spam();
-  Py_Finalize();
-}

Demos/primes.pyx

 print "starting"
 
 def primes(int kmax):
-    cdef int n, k, i
+    # cdef int n, k, i
     cdef int p[1000]
     result = []
     if kmax > 1000:

runtests.py

@@ -8,7 +8,10 @@ import shutil
 import unittest
 import doctest
 import operator
-from StringIO import StringIO
+try:
+    from StringIO import StringIO
+except ImportError:
+    from io import StringIO
 
 try:
     import cPickle as pickle
@@ -394,7 +397,7 @@ class CythonRunTestCase(CythonCompileTestCase):
                 try:
                     partial_result = PartialTestResult(result)
                     doctest.DocTestSuite(module_name).run(partial_result)
-                except Exception, e:
+                except Exception:
                     partial_result.addError(module_name, sys.exc_info())
                     result_code = 1
                 pickle.dump(partial_result.data(), output)

tests/compile/cassign.pyx

@@ -2,6 +2,7 @@ cdef void foo():
     cdef int i1, i2=0
     cdef char c1=0, c2
     cdef char *p1, *p2=NULL
+    cdef object obj1
     i1 = i2
     i1 = c1
     p1 = p2

tests/run/ctypedef_int_types_T333.pyx

@@ -576,15 +576,15 @@ def test_DefSInt(defs.SInt x):
    """
    return x
 
-def test_DefUInt(defs.UInt x):
+def test_DefUChar(defs.UChar x):
    u"""
-   >>> test_DefUInt(-1) #doctest: +ELLIPSIS
+   >>> test_DefUChar(-1) #doctest: +ELLIPSIS
    Traceback (most recent call last):
        ...
    OverflowError: ...
-   >>> test_DefUInt(0)
+   >>> test_DefUChar(0)
    0
-   >>> test_DefUInt(1)
+   >>> test_DefUChar(1)
    1
    """
    return x

tests/run/isinstance.pyx

@@ -35,7 +35,7 @@ def test_all():
     assert not isinstance(u"foo", int)
     
     # Non-optimized
-    foo = A
+    cdef object foo = A
     assert isinstance(A(), foo)
     assert isinstance(0, (int, long))
     assert not isinstance(u"xyz", (int, long))

tests/run/pycmp.pyx

@@ -5,6 +5,7 @@ __doc__ = u"""
 
 def f():
     cdef int bool, int1, int2
+    cdef object obj1, obj2
     int1 = 0
     int2 = 0
     obj1 = 1

tests/run/type_inference.pyx

+# cython: infer_types = True
+
+__doc__ = u"""
+    >>> simple()
+    >>> multiple_assignments()
+    >>> arithmatic()
+    >>> cascade()
+    >>> increment()
+    >>> loop()
+"""
+
+from cython cimport typeof
+
+def simple():
+    i = 3
+    assert typeof(i) == "long"
+    x = 1.41
+    assert typeof(x) == "double"
+    xptr = &x
+    assert typeof(xptr) == "double *"
+    xptrptr = &xptr
+    assert typeof(xptrptr) == "double **"
+    s = "abc"
+    assert typeof(s) == "char *"
+    u = u"xyz"
+    assert typeof(u) == "unicode object"
+    L = [1,2,3]
+    assert typeof(L) == "list object"
+    t = (4,5,6)
+    assert typeof(t) == "tuple object"
+
+def multiple_assignments():
+    a = 3
+    a = 4
+    a = 5
+    assert typeof(a) == "long"
+    b = a
+    b = 3.1
+    b = 3.14159
+    assert typeof(b) == "double"
+    c = a
+    c = b
+    c = [1,2,3]
+    assert typeof(c) == "Python object"
+
+def arithmatic():
+    a = 1 + 2
+    assert typeof(a) == "long"
+    b = 1 + 1.5
+    assert typeof(b) == "double"
+    c = 1 + <object>2
+    assert typeof(c) == "Python object"
+    d = "abc %s" % "x"
+    assert typeof(d) == "Python object"
+    
+def cascade():
+    a = 1.0
+    b = a + 2
+    c = b + 3
+    d = c + 4
+    assert typeof(d) == "double"
+    e = a + b + c + d
+    assert typeof(e) == "double"
+
+def increment():
+    a = 5
+    a += 1
+    assert typeof(a) == "long"
+
+def loop():
+    for a in range(10):
+        pass
+    assert typeof(a) == "long"
+    b = 1.0
+    for b in range(5):
+        pass
+    assert typeof(b) == "double"
+    for c from 0 <= c < 10 by .5:
+        pass
+    assert typeof(c) == "double"

tests/run/typeof.pyx

+__doc__ = u"""
+    >>> simple()
+    int
+    long
+    long long
+    int *
+    int **
+    A
+    B
+    X
+    Python object
+    
+    >>> expression()
+    double
+    double complex
+    int
+    unsigned int
+"""
+
+from cython cimport typeof
+
+cdef class A:
+    pass
+
+cdef class B(A):
+    pass
+
+cdef struct X:
+    double a
+    double complex b
+
+def simple():
+    cdef int i = 0
+    cdef long l = 0
+    cdef long long ll = 0
+    cdef int* iptr = &i
+    cdef int** iptrptr = &iptr
+    cdef A a = None
+    cdef B b = None
+    cdef X x = X(a=1, b=2)
+    print typeof(i)
+    print typeof(l)
+    print typeof(ll)
+    print typeof(iptr)
+    print typeof(iptrptr)
+    print typeof(a)
+    print typeof(b)
+    print typeof(x)
+    print typeof(None)
+    used = i, l, ll, <long>iptr, <long>iptrptr, a, b, x
+    
+def expression():
+    cdef X x = X(a=1, b=2)
+    cdef X *xptr = &x
+    cdef short s = 0
+    cdef int i = 0
+    cdef unsigned int ui = 0
+    print typeof(x.a)
+    print typeof(xptr.b)
+    print typeof(s + i)
+    print typeof(i + ui)
+    used = x, <long>xptr, s, i, ui