merged changes from 0.9.6.10

Cython/Compiler/CmdLine.py

@@ -26,6 +26,7 @@ Options:
                                  performing any binary operations.
   -D, --no-docstrings            Remove docstrings.
   -a, --annotate                 Produce an colorized version of the source.
+  --convert-range                Convert for loops using range() function to for...from loops. 
 """  
 #The following experimental options are supported only on MacOSX:
 #  -C, --compile    Compile generated .c file to .o file
@@ -88,6 +89,8 @@ def parse_command_line(args):
                 Options.docstrings = False
             elif option in ("-a", "--annotate"):
                 Options.annotate = True
+            elif option == "--convert-range":
+                Options.convert_range = True
             else:
                 bad_usage()
         else:

Cython/Compiler/Code.py

@@ -34,6 +34,7 @@ class CCodeWriter:
         self.level = 0
         self.bol = 1
         self.marker = None
+        self.last_marker = 1
         self.label_counter = 1
         self.error_label = None
         self.filename_table = {}
@@ -53,6 +54,7 @@ class CCodeWriter:
         self.f.write("\n");
         self.indent()
         self.f.write("/* %s */\n" % self.marker)
+        self.last_marker = self.marker
         self.marker = None
 
     def put(self, code):
@@ -96,8 +98,6 @@ class CCodeWriter:
         return "0x%02X%02X%02X%02X" % (tuple(pyversion) + (0,0,0,0))[:4]
 
     def mark_pos(self, pos):
-#        if self.marker is not None:
-#            print "new marker"
         file, line, col = pos
         contents = self.file_contents(file)
 
@@ -108,7 +108,9 @@ class CCodeWriter:
                 s = s.rstrip() + '             # <<<<<<<<<<<<<< ' + '\n'
             context += " * " + s
         
-        self.marker = '"%s":%s\n%s' % (file, line, context)
+        marker = '"%s":%s\n%s' % (file, line, context)
+        if self.last_marker != marker:
+            self.marker = marker
 
     def init_labels(self):
         self.label_counter = 0

Cython/Compiler/ModuleNode.py

@@ -279,6 +279,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
         self.generate_includes(env, cimported_modules, code)
         code.putln('')
         code.put(Nodes.utility_function_predeclarations)
+        code.put(PyrexTypes.type_conversion_predeclarations)
         code.put(Nodes.branch_prediction_macros)
         code.putln('')
         code.putln('static PyObject *%s;' % env.module_cname)
@@ -1598,6 +1599,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
         for utility_code in env.utility_code_used:
             code.h.put(utility_code[0])
             code.put(utility_code[1])
+        code.put(PyrexTypes.type_conversion_functions)
 
 #------------------------------------------------------------------------------------
 #
@@ -1745,11 +1747,11 @@ bad:
 
 register_cleanup_utility_code = [
 """
-static int __Pyx_RegisterCleanup(); /*proto*/
+static int __Pyx_RegisterCleanup(void); /*proto*/
 static PyObject* cleanup(PyObject *self, PyObject *unused); /*proto*/
 static PyMethodDef cleanup_def = {"__cleanup", (PyCFunction)&cleanup, METH_NOARGS, 0};
 ""","""
-static int __Pyx_RegisterCleanup() {
+static int __Pyx_RegisterCleanup(void) {
     /* Don't use Py_AtExit because that has a 32-call limit 
      * and is called after python finalization. 
      */

Cython/Compiler/Nodes.py

@@ -2412,10 +2412,57 @@ class ForInStatNode(StatNode):
         if self.else_clause:
             self.else_clause.analyse_declarations(env)
             
+    def analyse_range_step(self, args):
+        import ExprNodes
+        # The direction must be determined at compile time to set relations. 
+        # Otherwise, return False. 
+        if len(args) < 3:
+            self.step = ExprNodes.IntNode(pos = args[0].pos, value='1')
+            self.relation1 = '<='
+            self.relation2 = '<'
+            return True
+        else:
+            step = args[2]
+            if isinstance(step, ExprNodes.UnaryMinusNode) and isinstance(step.operand, ExprNodes.IntNode):
+                step = ExprNodes.IntNode(pos = step.pos, value=-int(step.operand.value))
+            if isinstance(step, ExprNodes.IntNode):
+                if step.value > 0:
+                    self.step = step
+                    self.relation1 = '<='
+                    self.relation2 = '<'
+                    return True
+                elif step.value < 0:
+                    self.step = ExprNodes.IntNode(pos = step.pos, value=-int(step.value))
+                    self.relation1 = '>='
+                    self.relation2 = '>'
+                    return True
+        return False
+                
+    
     def analyse_expressions(self, env):
         import ExprNodes
-        self.iterator.analyse_expressions(env)
         self.target.analyse_target_types(env)
+        if Options.convert_range and self.target.type.is_int:
+            sequence = self.iterator.sequence
+            if isinstance(sequence, ExprNodes.SimpleCallNode) \
+                  and sequence.self is None \
+                  and isinstance(sequence.function, ExprNodes.NameNode) \
+                  and sequence.function.name == 'range':
+                args = sequence.args
+                # Make sure we can determine direction from step
+                if self.analyse_range_step(args):
+                    # Mutate to ForFrom loop type
+                    self.__class__ = ForFromStatNode
+                    if len(args) == 1:
+                        self.bound1 = ExprNodes.IntNode(pos = sequence.pos, value='0')
+                        self.bound2 = args[0]
+                    else:
+                        self.bound1 = args[0]
+                        self.bound2 = args[1]
+                    ForFromStatNode.analyse_expressions(self, env)
+                    return
+                    
+        self.iterator.analyse_expressions(env)
         self.item = ExprNodes.NextNode(self.iterator, env)
         self.item = self.item.coerce_to(self.target.type, env)
         self.item.allocate_temps(env)
@@ -2485,8 +2532,8 @@ class ForFromStatNode(StatNode):
         self.target.analyse_target_types(env)
         self.bound1.analyse_types(env)
         self.bound2.analyse_types(env)
-        self.bound1 = self.bound1.coerce_to_integer(env)
-        self.bound2 = self.bound2.coerce_to_integer(env)
+        self.bound1 = self.bound1.coerce_to(self.target.type, env)
+        self.bound2 = self.bound2.coerce_to(self.target.type, env)
         if self.step is not None:
             if isinstance(self.step, ExprNodes.UnaryMinusNode):
                 warning(self.step.pos, "Probable infinite loop in for-from-by statment. Consider switching the directions of the relations.", 2)
@@ -3068,22 +3115,6 @@ utility_function_predeclarations = \
 typedef struct {PyObject **p; char *s;} __Pyx_InternTabEntry; /*proto*/
 typedef struct {PyObject **p; char *s; long n; int is_unicode;} __Pyx_StringTabEntry; /*proto*/
 
-static INLINE Py_ssize_t __pyx_PyIndex_AsSsize_t(PyObject* b) {
-  Py_ssize_t ival;
-  PyObject* x = PyNumber_Index(b);
-  if (!x) return -1;
-  ival = PyInt_AsSsize_t(x);
-  Py_DECREF(x);
-  return ival;
-}
-
-#define __Pyx_PyBool_FromLong(b) ((b) ? (Py_INCREF(Py_True), Py_True) : (Py_INCREF(Py_False), Py_False))
-static INLINE int __Pyx_PyObject_IsTrue(PyObject* x) {
-   if (x == Py_True) return 1;
-   else if (x == Py_False) return 0;
-   else return PyObject_IsTrue(x);
-}
-
 """ + """
 
 static int %(skip_dispatch_cname)s = 0;

Cython/Compiler/Options.py

@@ -19,6 +19,17 @@ incref_local_binop = 0
 
 # Decref global variables in this module on exit for garbage collection. 
 # 0: None, 1+: interned objects, 2+: cdef globals, 3+: types objects
+# Mostly for reducing noise for Valgrind, only executes at process exit
+# (when all memory will be reclaimed anyways). 
 generate_cleanup_code = 3
 
 annotate = 0
+
+# This will convert statements of the form "for i in range(...)" 
+# to "for i from ..." when i is a cdef'd integer type, and the direction
+# (i.e. sign of step) can be determined. 
+# WARNING: This may change the symantics if the range causes assignment to 
+# i to overflow. Specifically, if this option is set, an error will be
+# raised before the loop is entered, wheras without this option the loop
+# will execute util a overflowing value is encountered. 
+convert_range = 0

Cython/Compiler/Parsing.py

@@ -1344,7 +1344,7 @@ def p_statement(s, level, cdef_flag = 0, visibility = 'private', api = 0):
             elif s.sy == 'pass' and level <> 'property':
                 return p_pass_statement(s, with_newline = 1)
             else:
-                if level in ('c_class', 'c_class_pxd', 'property'):
+                if level in ('c_class_pxd', 'property'):
                     s.error("Executable statement not allowed here")
                 if s.sy == 'if':
                     return p_if_statement(s)

Cython/Compiler/PyrexTypes.py

@@ -361,17 +361,56 @@ class CNumericType(CType):
         return self.base_declaration_code(base,  entity_code)
 
 
+int_conversion_list = {}
+type_conversion_functions = ""
+type_conversion_predeclarations = ""
+
 class CIntType(CNumericType):
     
     is_int = 1
     typedef_flag = 0
     to_py_function = "PyInt_FromLong"
-    from_py_function = "PyInt_AsLong"
+    from_py_function = "__pyx_PyInt_AsLong"
     exception_value = -1
 
     def __init__(self, rank, signed, pymemberdef_typecode = None, is_returncode = 0):
         CNumericType.__init__(self, rank, signed, pymemberdef_typecode)
         self.is_returncode = is_returncode
+        if self.from_py_function == '__pyx_PyInt_AsLong':
+            self.from_py_function = self.get_type_conversion()
+
+    def get_type_conversion(self):
+        # error on overflow
+        c_type = self.sign_and_name()
+        c_name = c_type.replace(' ', '_');
+        func_name = "__pyx_PyInt_%s" % c_name;
+        if not int_conversion_list.has_key(func_name):
+            # no env to add utility code to
+            global type_conversion_predeclarations, type_conversion_functions
+            if self.signed:
+                neg_test = ""
+            else:
+                neg_test = " || (long_val < 0)"
+            type_conversion_predeclarations += """
+static INLINE %(c_type)s %(func_name)s(PyObject* x);""" % {'c_type': c_type, 'c_name': c_name, 'func_name': func_name }
+            type_conversion_functions +=  """
+static INLINE %(c_type)s %(func_name)s(PyObject* x) {
+    if (sizeof(%(c_type)s) < sizeof(long)) {
+        long long_val = __pyx_PyInt_AsLong(x);
+        %(c_type)s val = (%(c_type)s)long_val;
+        if (unlikely((val != long_val) %(neg_test)s)) {
+            PyErr_SetString(PyExc_OverflowError, "value too large to convert to %(c_type)s");
+            return (%(c_type)s)-1;
+        }
+        return val;
+    }
+    else {
+        return __pyx_PyInt_AsLong(x);
+    }
+}
+""" % {'c_type': c_type, 'c_name': c_name, 'func_name': func_name, 'neg_test': neg_test }
+            int_conversion_list[func_name] = True
+        return func_name
     
     def assignable_from_resolved_type(self, src_type):
         return src_type.is_int or src_type.is_enum or src_type is error_type
@@ -405,13 +444,13 @@ class CULongType(CUIntType):
 class CLongLongType(CUIntType):
 
     to_py_function = "PyLong_FromLongLong"
-    from_py_function = "PyInt_AsUnsignedLongLongMask"
+    from_py_function = "__pyx_PyInt_AsLongLong"
 
 
 class CULongLongType(CUIntType):
 
     to_py_function = "PyLong_FromUnsignedLongLong"
-    from_py_function = "PyInt_AsUnsignedLongLongMask"
+    from_py_function = "__pyx_PyInt_AsUnsignedLongLong"
 
 
 class CPySSizeTType(CIntType):
@@ -424,7 +463,7 @@ class CFloatType(CNumericType):
 
     is_float = 1
     to_py_function = "PyFloat_FromDouble"
-    from_py_function = "PyFloat_AsDouble"
+    from_py_function = "__pyx_PyFloat_AsDouble"
     
     def __init__(self, rank, pymemberdef_typecode = None):
         CNumericType.__init__(self, rank, 1, pymemberdef_typecode)
@@ -828,6 +867,18 @@ class ErrorType(PyrexType):
         return 1
 
 
+rank_to_type_name = (
+    "char",         # 0
+    "short",        # 1
+    "int",          # 2
+    "long",         # 3
+    "PY_LONG_LONG", # 4
+    "Py_ssize_t",   # 5
+    "float",        # 6
+    "double",       # 7
+    "long double",  # 8
+)
+
 py_object_type = PyObjectType()
 
 c_void_type =         CVoidType()
@@ -872,18 +923,6 @@ error_type =    ErrorType()
 
 lowest_float_rank = 6
 
-rank_to_type_name = (
-    "char",         # 0
-    "short",        # 1
-    "int",          # 2
-    "long",         # 3
-    "PY_LONG_LONG", # 4
-    "Py_ssize_t",   # 5
-    "float",        # 6
-    "double",       # 7
-    "long double",  # 8
-)
-
 sign_and_rank_to_type = {
     #(signed, rank)
     (0, 0, ): c_uchar_type, 
@@ -988,3 +1027,72 @@ def typecast(to_type, from_type, expr_code):
     else:
         #print "typecast: to", to_type, "from", from_type ###
         return to_type.cast_code(expr_code)
+
+
+type_conversion_predeclarations = """
+/* Type Conversion Predeclarations */
+
+#define __Pyx_PyBool_FromLong(b) ((b) ? (Py_INCREF(Py_True), Py_True) : (Py_INCREF(Py_False), Py_False))
+static INLINE int __Pyx_PyObject_IsTrue(PyObject* x);
+static INLINE PY_LONG_LONG __pyx_PyInt_AsLongLong(PyObject* x);
+static INLINE unsigned PY_LONG_LONG __pyx_PyInt_AsUnsignedLongLong(PyObject* x);
+static INLINE Py_ssize_t __pyx_PyIndex_AsSsize_t(PyObject* b);
+
+#define __pyx_PyInt_AsLong(x) (PyInt_CheckExact(x) ? PyInt_AS_LONG(x) : PyInt_AsLong(x))
+#define __pyx_PyFloat_AsDouble(x) (PyFloat_CheckExact(x) ? PyFloat_AS_DOUBLE(x) : PyFloat_AsDouble(x))
+""" + type_conversion_predeclarations
+
+type_conversion_functions = """
+/* Type Conversion Functions */
+
+static INLINE Py_ssize_t __pyx_PyIndex_AsSsize_t(PyObject* b) {
+  Py_ssize_t ival;
+  PyObject* x = PyNumber_Index(b);
+  if (!x) return -1;
+  ival = PyInt_AsSsize_t(x);
+  Py_DECREF(x);
+  return ival;
+}
+
+static INLINE int __Pyx_PyObject_IsTrue(PyObject* x) {
+   if (x == Py_True) return 1;
+   else if (x == Py_False) return 0;
+   else return PyObject_IsTrue(x);
+}
+
+static INLINE PY_LONG_LONG __pyx_PyInt_AsLongLong(PyObject* x) {
+    if (PyInt_CheckExact(x)) {
+        return PyInt_AS_LONG(x);
+    }
+    else if (PyLong_CheckExact(x)) {
+        return PyLong_AsLongLong(x);
+    }
+    else {
+        PyObject* tmp = PyNumber_Int(x); if (!tmp) return (PY_LONG_LONG)-1;
+        PY_LONG_LONG val = __pyx_PyInt_AsLongLong(tmp);
+        Py_DECREF(tmp);
+        return val;
+    }
+}
+
+static INLINE unsigned PY_LONG_LONG __pyx_PyInt_AsUnsignedLongLong(PyObject* x) {
+    if (PyInt_CheckExact(x)) {
+        long val = PyInt_AS_LONG(x);
+        if (unlikely(val < 0)) {
+            PyErr_SetString(PyExc_TypeError, "Negative assignment to unsigned type.");
+            return (unsigned PY_LONG_LONG)-1;
+        }
+        return val;
+    }
+    else if (PyLong_CheckExact(x)) {
+        return PyLong_AsUnsignedLongLong(x);
+    }
+    else {
+        PyObject* tmp = PyNumber_Int(x); if (!tmp) return (PY_LONG_LONG)-1;
+        PY_LONG_LONG val = __pyx_PyInt_AsUnsignedLongLong(tmp);
+        Py_DECREF(tmp);
+        return val;
+    }
+}
+
+""" + type_conversion_functions

Cython/Compiler/Symtab.py

@@ -1071,6 +1071,21 @@ class ClassScope(Scope):
     def add_string_const(self, value):
         return self.outer_scope.add_string_const(value)
 
+    def lookup(self, name):
+        if name == "classmethod":
+            # We don't want to use the builtin classmethod here 'cause it won't do the 
+            # right thing in this scope (as the class memebers aren't still functions). 
+            # Don't want to add a cfunction to this scope 'cause that would mess with 
+            # the type definition, so we just return the right entry. 
+            self.use_utility_code(classmethod_utility_code)
+            entry = Entry("classmethod", 
+                          "__Pyx_Method_ClassMethod", 
+                          CFuncType(py_object_type, [CFuncTypeArg("", py_object_type, None)], 0, 0))
+            entry.is_cfunction = 1
+            return entry
+        else:
+            return Scope.lookup(self, name)
+    
 
 class PyClassScope(ClassScope):
     #  Namespace of a Python class.
@@ -1290,21 +1305,6 @@ class CClassScope(ClassScope):
     def release_temp(self, cname):
         return Scope.release_temp(self.global_scope(), cname)
         
-    def lookup(self, name):
-        if name == "classmethod":
-            # We don't want to use the builtin classmethod here 'cause it won't do the 
-            # right thing in this scope (as the class memebers aren't still functions). 
-            # Don't want to add a cfunction to this scope 'cause that would mess with 
-            # the type definition, so we just return the right entry. 
-            self.use_utility_code(classmethod_utility_code)
-            entry = Entry("classmethod", 
-                          "__Pyx_Method_ClassMethod", 
-                          CFuncType(py_object_type, [CFuncTypeArg("", py_object_type, None)], 0, 0))
-            entry.is_cfunction = 1
-            return entry
-        else:
-            return Scope.lookup(self, name)
-    
         
 class PropertyScope(Scope):
     #  Scope holding the __get__, __set__ and __del__ methods for
@@ -1337,12 +1337,15 @@ static PyObject* __Pyx_Method_ClassMethod(PyObject *method); /*proto*/
 static PyObject* __Pyx_Method_ClassMethod(PyObject *method) {
     /* It appears that PyMethodDescr_Type is not anywhere exposed in the Python/C API */
     /* if (!PyObject_TypeCheck(method, &PyMethodDescr_Type)) { */ 
-    if (strcmp(method->ob_type->tp_name, "method_descriptor") != 0) {
-        PyErr_Format(PyExc_TypeError, "Extension type classmethod() can only be called on a method_descriptor.");
-        return NULL;
-    }
+    if (strcmp(method->ob_type->tp_name, "method_descriptor") == 0) { /* cdef classes */
         PyMethodDescrObject *descr = (PyMethodDescrObject *)method;
         return PyDescr_NewClassMethod(descr->d_type, descr->d_method);
+    }
+    else if (PyMethod_Check(method)) {                                /* python classes */
+        return PyClassMethod_New(PyMethod_GET_FUNCTION(method));
+    }
+    PyErr_Format(PyExc_TypeError, "Class-level classmethod() can only be called on a method_descriptor or instance method.");
+    return NULL;
 }
 """
 ]

Cython/Compiler/Version.py

-version = '0.9.6.9'
+version = '0.9.6.10'